[ { "path": ".ci/gitlab/forward_dotenv_variables.py", "content": "import sys\nfrom typing import Dict\n\nimport yaml\n\n\ndef read_dotenv(file_name: str) -> Dict[str, str]:\n result = []\n with open(file_name, \"r\", encoding=\"utf-8\") as fd:\n for field in fd:\n if field.strip()[0] == \"#\":\n continue\n\n data = field.strip(\"\\n\").split(\"=\", 1)\n try:\n result.append((data[0], data[1]))\n except IndexError:\n print(f\"Skipping bad value: {field}\")\n\n return dict(result)\n\n\nif __name__ == \"__main__\":\n dotenv = read_dotenv(sys.argv[1])\n if not dotenv:\n exit(0)\n\n with open(sys.argv[2], \"r\", encoding=\"utf-8\") as fd:\n conf = yaml.load(fd, Loader=yaml.Loader)\n\n if \"variables\" not in conf:\n conf[\"variables\"] = {}\n conf[\"variables\"].update(dotenv)\n\n with open(sys.argv[2], \"w\", encoding=\"utf-8\") as fd:\n yaml.dump(conf, fd, Dumper=yaml.Dumper)\n" }, { "path": ".ci/gitlab-ci.yml", "content": "# Gitlab configuraiton for spack/spack\n\nstages:\n - packages\n\nvariables:\n SPACK_PACKAGES_CHECKOUT_VERSION: develop\n\n.clone_packages: &clone_packages\n - mkdir -p ${REPO_DESTINATION}\n - cd ${REPO_DESTINATION}\n - git init\n - git remote add origin https://github.com/spack/spack-packages.git\n - git fetch --depth 1 origin ${SPACK_PACKAGES_CHECKOUT_VERSION}\n - git checkout FETCH_HEAD\n - cd -\n\ndotenv:\n stage: .pre\n image: ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18\n tags: [ spack, service ]\n script:\n - export REPO_DESTINATION=etc/spack-packages\n - *clone_packages\n - repo_commit=$(git -C ${REPO_DESTINATION} rev-parse FETCH_HEAD)\n - echo \"SPACK_CHECKOUT_VERSION=${repo_commit}\" >> ${CI_PROJECT_DIR}/env\n - echo \"SPACK_CHECKOUT_REPO=spack/spack-packages\" >> ${CI_PROJECT_DIR}/env\n - cat ${CI_PROJECT_DIR}/env\n - python3 ${CI_PROJECT_DIR}/.ci/gitlab/forward_dotenv_variables.py\n ${CI_PROJECT_DIR}/env\n ${REPO_DESTINATION}/.ci/gitlab/.gitlab-ci.yml\n\n artifacts:\n paths:\n - etc/spack-packages/.ci/gitlab/.gitlab-ci.yml\n\nspack-packages:\n stage: packages\n trigger:\n strategy: depend\n include:\n - artifact: etc/spack-packages/.ci/gitlab/.gitlab-ci.yml\n job: dotenv\n\n" }, { "path": ".codecov.yml", "content": "coverage:\n precision: 2\n round: nearest\n range: 60...90\n status:\n project:\n default:\n threshold: 2.0%\n\nignore:\n - lib/spack/spack/test/.*\n - lib/spack/docs/.*\n - lib/spack/spack/vendor/.*\n - share/spack/qa/.*\n\ncomment: off\n\n# Inline codecov annotations make the code hard to read, and they add\n# annotations in files that seemingly have nothing to do with the PR.\ngithub_checks:\n annotations: false\n\n# Attempt to fix \"Missing base commit\" messages in the codecov UI.\n# Because we do not run full tests on package PRs, package PRs' merge\n# commits on `develop` don't have coverage info. It appears that\n# codecov will give you an error if the pseudo-base's coverage data\n# doesn't all apply properly to the real PR base.\n#\n# See here for docs:\n# https://docs.codecov.com/docs/comparing-commits#pseudo-comparison\n# See here for another potential solution:\n# https://community.codecov.com/t/2480/15\ncodecov:\n allow_coverage_offsets: true\n" }, { "path": ".devcontainer/postCreateCommand.sh", "content": "#!/bin/bash\n\n# Load spack environment at terminal startup\ncat <> /root/.bashrc\n. /workspaces/spack/share/spack/setup-env.sh\nEOF\n\n# Load spack environment in this script\n. /workspaces/spack/share/spack/setup-env.sh\n\n# Ensure generic targets for maximum matching with buildcaches\nspack config --scope site add \"packages:all:require:[target=x86_64_v3]\"\nspack config --scope site add \"concretizer:targets:granularity:generic\"\n\n# Find compiler and install gcc-runtime\nspack compiler find --scope site\n\n# Setup buildcaches\nspack mirror add --scope site develop https://binaries.spack.io/develop\nspack buildcache keys --install --trust\n" }, { "path": ".devcontainer/ubuntu20.04/devcontainer.json", "content": "{\n \"name\": \"Ubuntu 20.04\",\n \"image\": \"ghcr.io/spack/ubuntu20.04-runner-amd64-gcc-11.4:2023.08.01\",\n \"postCreateCommand\": \"./.devcontainer/postCreateCommand.sh\"\n}\n" }, { "path": ".devcontainer/ubuntu22.04/devcontainer.json", "content": "{\n \"name\": \"Ubuntu 22.04\",\n \"image\": \"ghcr.io/spack/ubuntu-22.04:v2024-05-07\",\n \"postCreateCommand\": \"./.devcontainer/postCreateCommand.sh\"\n}\n" }, { "path": ".dockerignore", "content": ".git/*\nopt/spack/*\n\n/etc/spack/*\n!/etc/spack/defaults\n\nshare/spack/dotkit/*\nshare/spack/lmod/*\nshare/spack/modules/*\nlib/spack/spack/test/*\nvar/spack/cache/*\n" }, { "path": ".flake8", "content": "# -*- conf -*-\n# flake8 settings for Spack.\n#\n# These exceptions are for Spack core files. We're slightly more lenient\n# with packages. See .flake8_packages for that.\n#\n# This is the only flake8 rule Spack violates somewhat flagrantly\n# - E731: do not assign a lambda expression, use a def\n#\n# This is the only flake8 exception needed when using Black.\n# - E203: white space around slice operators can be required, ignore : warn\n#\n# We still allow these in packages (Would like to get rid of them or rely on mypy\n# in the future)\n# - F403: from/import * used; unable to detect undefined names\n# - F405: undefined name or from *\n# - F821: undefined name (needed with from/import *)\n#\n[flake8]\n#ignore = E129,,W503,W504,F999,N801,N813,N814,F403,F405,E203\nextend-ignore = E731,E203\nmax-line-length = 99\n\n# F4: Import\n# - F405: `name` may be undefined, or undefined from star imports: `module`\n#\n# F8: Name\n# - F821: undefined name `name`\n#\nper-file-ignores =\n var/spack/*/package.py:F403,F405,F821\n *-ci-package.py:F403,F405,F821\n\n# exclude things we usually do not want linting for.\n# These still get linted when passed explicitly, as when spack flake8 passes\n# them on the command line.\nexclude =\n .git\n etc/\n opt/\n share/\n var/spack/cache/\n var/spack/gpg*/\n var/spack/junit-report/\n var/spack/mock-configs/\n lib/spack/spack/vendor/\n __pycache__\n var\n\nformat = spack\n\n[flake8:local-plugins]\nreport =\n spack = flake8_formatter:SpackFormatter\npaths =\n ./share/spack/qa/\n" }, { "path": ".git-blame-ignore-revs", "content": "# .git-blame-ignore-revs\n# Formatted entire codebase with black 23\n603569e321013a1a63a637813c94c2834d0a0023\n# Formatted entire codebase with black 22\nf52f6e99dbf1131886a80112b8c79dfc414afb7c\n# Formatted all rst files\n1377d42c16c6912faa77259c0a1f665210ccfd85\n" }, { "path": ".gitattributes", "content": "*.bat text eol=crlf\n*.py diff=python\n*.py text eol=lf\nlib/spack/spack/vendor/* linguist-vendored\n" }, { "path": ".github/CODE_OF_CONDUCT.md", "content": "# Spack Community Code of Conduct\n\n## Our Pledge\n\nIn the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.\n\n## Our Standards\n\nExamples of behavior that contributes to creating a positive environment include:\n\n* Using welcoming and inclusive language\n* Being respectful of differing viewpoints and experiences\n* Gracefully accepting constructive criticism\n* Focusing on what is best for the community\n* Showing empathy towards other community members\n\nExamples of unacceptable behavior by participants include:\n\n* The use of sexualized language or imagery and unwelcome sexual attention or advances\n* Trolling, insulting/derogatory comments, and personal or political attacks\n* Public or private harassment\n* Publishing others' private information, such as a physical or electronic address, without explicit permission\n* Other conduct which could reasonably be considered inappropriate in a professional setting\n\n## Our Responsibilities\n\nProject maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.\n\nProject maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.\n\n## Scope\n\nThis Code of Conduct applies both within project spaces and in public spaces when an individual is representing the Spack project or its community. Examples of representing the project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of the project may be further defined and clarified by Spack maintainers.\n\n## Enforcement\n\nInstances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at maintainers@spack.io. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.\n\nProject maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.\n\n## Attribution\n\nThis Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version]\n\n[homepage]: http://contributor-covenant.org\n[version]: http://contributor-covenant.org/version/1/4/\n" }, { "path": ".github/CONTRIBUTING.md", "content": "# Contributing to Spack\n\nAll contributions to Spack must be made under both the Apache License,\nVersion 2.0 (Apache-2.0) and the MIT license (MIT).\n\nBefore contributing to Spack, you should read the\n[Contribution Guide](https://spack.readthedocs.io/en/latest/contribution_guide.html),\nwhich is maintained as part of Spack's documentation.\n" }, { "path": ".github/ISSUE_TEMPLATE/bug_report.yml", "content": "name: \"\\U0001F41E Bug report\"\ndescription: Report a bug in the core of Spack (command not working as expected, etc.)\nlabels: [bug, triage]\nbody:\n - type: textarea\n id: reproduce\n attributes:\n label: Steps to reproduce\n description: |\n Explain, in a clear and concise way, the command you ran and the result you were trying to achieve.\n Example: \"I ran `spack find` to list all the installed packages and ...\"\n placeholder: |\n ```console\n $ spack \n $ spack \n ...\n ```\n validations:\n required: true\n - type: textarea\n id: error\n attributes:\n label: Error message\n description: |\n If Spack reported an error, provide the error message. If it did not report an error but the output appears incorrect, provide the incorrect output. If there was no error message and no output but the result is incorrect, describe how it does not match what you expect.\n placeholder: |\n ```console\n $ spack --debug --stacktrace \n ```\n - type: textarea\n id: information\n attributes:\n label: Information on your system\n description: Please include the output of `spack debug report`\n validations:\n required: true\n - type: markdown\n attributes:\n value: |\n If you have any relevant configuration detail (custom `packages.yaml` or `modules.yaml`, etc.) you can add that here as well.\n - type: checkboxes\n id: checks\n attributes:\n label: General information\n options:\n - label: I have run `spack debug report` and reported the version of Spack/Python/Platform\n required: true\n - label: I have searched the issues of this repo and believe this is not a duplicate\n required: true\n - label: I have run the failing commands in debug mode and reported the output\n required: true\n - type: markdown\n attributes:\n value: |\n We encourage you to try, as much as possible, to reduce your problem to the minimal example that still reproduces the issue. That would help us a lot in fixing it quickly and effectively!\n If you want to ask a question about the tool (how to use it, what it can currently do, etc.), try the `#general` channel on [our Slack](https://slack.spack.io/) first. We have a welcoming community and chances are you'll get your reply faster and without opening an issue.\n \n Other than that, thanks for taking the time to contribute to Spack!\n" }, { "path": ".github/ISSUE_TEMPLATE/config.yml", "content": "blank_issues_enabled: true\ncontact_links:\n - name: \"\\U0001F4A5 Package build error\"\n url: https://github.com/spack/spack-packages/issues/new?template=build_error.yml\n about: Report installation issues in the spack/spack-packages repository\n - name: \"\\U0001F4A5 Package test error\"\n url: https://github.com/spack/spack-packages/issues/new?template=test_error.yml\n about: Report standalone package test issues in the spack/spack-packages repository\n" }, { "path": ".github/ISSUE_TEMPLATE/feature_request.yml", "content": "name: \"\\U0001F38A Feature request\"\ndescription: Suggest adding a feature that is not yet in Spack\nlabels: [feature]\nbody:\n - type: textarea\n id: summary\n attributes:\n label: Summary\n description: Please add a concise summary of your suggestion here.\n validations:\n required: true\n - type: textarea\n id: rationale\n attributes:\n label: Rationale\n description: Is your feature request related to a problem? Please describe it!\n - type: textarea\n id: description\n attributes:\n label: Description\n description: Describe the solution you'd like and the alternatives you have considered.\n - type: textarea\n id: additional_information\n attributes:\n label: Additional information\n description: Add any other context about the feature request here.\n - type: checkboxes\n id: checks\n attributes:\n label: General information\n options:\n - label: I have searched the issues of this repo and believe this is not a duplicate\n required: true\n - type: markdown\n attributes:\n value: |\n If you want to ask a question about the tool (how to use it, what it can currently do, etc.), try the `#general` channel on [our Slack](https://slack.spack.io/) first. We have a welcoming community and chances are you'll get your reply faster and without opening an issue.\n\n Other than that, thanks for taking the time to contribute to Spack!\n" }, { "path": ".github/dependabot.yml", "content": "version: 2\nupdates:\n - package-ecosystem: \"github-actions\"\n directory: \"/\"\n schedule:\n interval: \"daily\"\n\n - package-ecosystem: \"pip\"\n directories:\n - \"/.github/workflows/requirements/coverage\"\n - \"/.github/workflows/requirements/style\"\n - \"/.github/workflows/requirements/unit_tests\"\n - \"/lib/spack/docs\"\n schedule:\n interval: \"daily\"\n" }, { "path": ".github/labeler.yml", "content": "bootstrap:\n- changed-files:\n - any-glob-to-any-file: lib/spack/spack/bootstrap/**\n\nbinary-caches:\n- changed-files:\n - any-glob-to-any-file: lib/spack/spack/binary_distribution.py\n - any-glob-to-any-file: lib/spack/spack/cmd/buildcache.py\n\nci:\n- changed-files:\n - any-glob-to-any-file: .ci/**\n - any-glob-to-any-file: .github/**\n - any-glob-to-any-file: lib/spack/spack/ci/**\n\ncommands:\n- changed-files:\n - any-glob-to-any-file: lib/spack/spack/cmd/**\n\nconfig:\n- changed-files:\n - any-glob-to-any-file: etc/spack/**\n - any-glob-to-any-file: lib/spack/spack/cmd/config.py\n - any-glob-to-any-file: lib/spack/spack/config.py\n - any-glob-to-any-file: lib/spack/spack/schema/**\n\ndocs:\n- changed-files:\n - any-glob-to-any-file: .readthedocs.yml\n - any-glob-to-any-file: lib/spack/docs/**\n\nenvironments:\n- changed-files:\n - any-glob-to-any-file: lib/spack/spack/cmd/env.py\n - any-glob-to-any-file: lib/spack/spack/environment/**\n\nmirrors:\n- changed-files:\n - any-glob-to-any-file: lib/spack/spack/cmd/mirror.py\n - any-glob-to-any-file: lib/spack/spack/mirrors/**\n\nmodules:\n- changed-files:\n - any-glob-to-any-file: lib/spack/spack/cmd/module.py\n - any-glob-to-any-file: lib/spack/spack/modules/**\n\nsolver:\n- changed-files:\n - any-glob-to-any-file: lib/spack/spack/solver/**\n\nstyle:\n- changed-files:\n - any-glob-to-any-file: .flake8\n - any-glob-to-any-file: .github/workflows/prechecks.yml\n - any-glob-to-any-file: .github/workflows/requirements/style/**\n - any-glob-to-any-file: lib/spack/spack/cmd/style.py\n - any-glob-to-any-file: pyproject.toml\n\nunit-tests:\n- changed-files:\n - any-glob-to-any-file: .codecov.yml\n - any-glob-to-any-file: lib/spack/spack/cmd/unit_test.py\n - any-glob-to-any-file: lib/spack/spack/test/**\n - any-glob-to-any-file: pyproject.toml\n - any-glob-to-any-file: pytest.ini\n - any-glob-to-any-file: var/spack/test_repos/**\n\nvendor:\n- changed-files:\n - any-glob-to-any-file: lib/spack/spack/vendor/**\n\nversions:\n- changed-files:\n - any-glob-to-any-file: lib/spack/spack/version/**\n" }, { "path": ".github/pull_request_template.md", "content": "\n" }, { "path": ".github/workflows/bin/canonicalize.py", "content": "#!/usr/bin/env python3\n# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport argparse\nimport ast\nimport os\nimport subprocess\nimport sys\nfrom itertools import product\nfrom typing import List\n\n\ndef run_git_command(*args: str, dir: str) -> None:\n \"\"\"Run a git command in the output directory.\"\"\"\n subprocess.run(\n [\n \"git\",\n \"-c\",\n \"user.email=example@example.com\",\n \"-c\",\n \"user.name=Example\",\n \"-c\",\n \"init.defaultBranch=main\",\n \"-c\",\n \"color.ui=always\",\n \"-C\",\n dir,\n *args,\n ],\n check=True,\n stdout=sys.stdout,\n stderr=sys.stderr,\n )\n\n\ndef run(root: str, output_dir: str) -> None:\n \"\"\"Recurse over a directory and canonicalize all Python files.\"\"\"\n from spack.util.package_hash import RemoveDocstrings, unparse\n\n count = 0\n stack = [root]\n\n while stack:\n current = stack.pop()\n for entry in os.scandir(current):\n if entry.is_dir(follow_symlinks=False):\n stack.append(entry.path)\n elif entry.is_file(follow_symlinks=False) and entry.name.endswith(\".py\"):\n try:\n with open(entry.path, \"r\") as f:\n src = f.read()\n except OSError:\n continue\n\n canonical_dir = os.path.join(output_dir, os.path.relpath(current, root))\n os.makedirs(canonical_dir, exist_ok=True)\n with open(os.path.join(canonical_dir, entry.name), \"w\") as f:\n f.write(\n unparse(RemoveDocstrings().visit(ast.parse(src)), py_ver_consistent=True)\n )\n count += 1\n\n assert count > 0, \"No Python files found in the specified directory.\"\n\n\ndef compare(\n input_dir: str, output_dir: str, python_versions: List[str], spack_versions: List[str]\n) -> None:\n \"\"\"Compare canonicalized files across different Python versions and error if they differ.\"\"\"\n # Create a git repo in output_dir to track changes\n os.makedirs(output_dir, exist_ok=True)\n run_git_command(\"init\", dir=output_dir)\n\n pairs = list(product(spack_versions, python_versions))\n\n if len(pairs) < 2:\n raise ValueError(\"At least two Python or two Spack versions must be given for comparison.\")\n\n changes_with_previous: List[int] = []\n\n for i, (spack_dir, python_exe) in enumerate(pairs):\n print(f\"\\033[1;97mCanonicalizing with {python_exe} and {spack_dir}...\\033[0m\", flush=True)\n\n # Point PYTHONPATH to the given Spack library for the subprocess\n if not os.path.isdir(spack_dir):\n raise ValueError(f\"Invalid Spack dir: {spack_dir}\")\n env = os.environ.copy()\n spack_pythonpath = os.path.join(spack_dir, \"lib\", \"spack\")\n if \"PYTHONPATH\" in env and env[\"PYTHONPATH\"]:\n env[\"PYTHONPATH\"] = f\"{spack_pythonpath}{os.pathsep}{env['PYTHONPATH']}\"\n else:\n env[\"PYTHONPATH\"] = spack_pythonpath\n\n subprocess.run(\n [python_exe, __file__, \"--run\", \"--input-dir\", input_dir, \"--output-dir\", output_dir],\n check=True,\n stdout=sys.stdout,\n stderr=sys.stderr,\n env=env,\n )\n if i > 0:\n try:\n run_git_command(\"diff\", \"--exit-code\", \"HEAD\", dir=output_dir)\n except subprocess.CalledProcessError:\n changes_with_previous.append(i)\n\n # The first run creates a commit for reference\n run_git_command(\"add\", \".\", dir=output_dir)\n run_git_command(\n \"commit\",\n \"--quiet\",\n \"--allow-empty\", # makes this idempotent when running locally\n \"-m\",\n f\"Canonicalized with {python_exe} and {spack_dir}\",\n dir=output_dir,\n )\n\n for i in changes_with_previous:\n previous_spack, previous_python = pairs[i - 1]\n current_spack, current_python = pairs[i]\n print(\n f\"\\033[1;31mChanges detected between {previous_python} ({previous_spack}) and \"\n f\"{current_python} ({current_spack})\\033[0m\"\n )\n\n if changes_with_previous:\n exit(1)\n\n\nif __name__ == \"__main__\":\n parser = argparse.ArgumentParser(description=\"Canonicalize Spack package files.\")\n parser.add_argument(\"--run\", action=\"store_true\", help=\"Generate canonicalized sources.\")\n parser.add_argument(\"--spack\", nargs=\"+\", help=\"Specify one or more Spack versions.\")\n parser.add_argument(\"--python\", nargs=\"+\", help=\"Specify one or more Python versions.\")\n parser.add_argument(\"--input-dir\", type=str, required=True, help=\"A repo's packages dir.\")\n parser.add_argument(\n \"--output-dir\",\n type=str,\n required=True,\n help=\"The output directory for canonicalized package files.\",\n )\n args = parser.parse_args()\n\n if args.run:\n run(args.input_dir, args.output_dir)\n else:\n compare(args.input_dir, args.output_dir, args.python, args.spack)\n" }, { "path": ".github/workflows/bin/execute_installer.ps1", "content": "$ proc = Start-Process ${{ env.spack_installer }}\\spack.exe \"/install /quiet\" -Passthru\n$handle = $proc.Handle # cache proc.Handle\n$proc.WaitForExit();\n\nif ($proc.ExitCode -ne 0) {\n Write-Warning \"$_ exited with status code $($proc.ExitCode)\"\n}\n" }, { "path": ".github/workflows/bin/format-rst.py", "content": "#!/usr/bin/env python3\n# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"This script formats reStructuredText files to ensure one sentence per line and no trailing\nwhitespace. It exits with a non-zero status if any files were modified.\"\"\"\n\nimport difflib\nimport importlib\nimport io\nimport json\nimport os\nimport re\nimport subprocess\nimport sys\nfrom typing import List\n\nimport black\nfrom docutils import nodes\nfrom docutils.core import publish_doctree\nfrom docutils.parsers.rst import Directive, directives\nfrom ruamel.yaml import YAML\n\nfrom spack.vendor import jsonschema\n\nimport spack.schema\n\n#: Map Spack config sections to their corresponding JSON schema\nSECTION_AND_SCHEMA = [\n # The first property's key is the config section name\n (next(iter(m.schema[\"properties\"])), m.schema)\n # Dynamically load all modules in spack.schema to be future-proof\n for m in (\n importlib.import_module(f\"spack.schema.{f[:-3]}\")\n for f in os.listdir(os.path.dirname(spack.schema.__file__))\n if f.endswith(\".py\") and f != \"__init__.py\"\n )\n if hasattr(m, \"schema\") and len(m.schema.get(\"properties\", {})) == 1\n]\n\nassert SECTION_AND_SCHEMA, \"no schemas found\"\n\nEND_OF_SENTENCE = re.compile(\n r\"\"\"\n(\n (?:\n (? str:\n return f\"\\033[1;33mwarning:\\033[0m {msg}\"\n\n\nclass Warning:\n def __init__(self, path: str, line: int, message: str) -> None:\n self.path = path\n self.line = line\n self.message = message\n\n def __str__(self) -> str:\n return _warning(f\"{self.path}:{self.line}: {self.message}\")\n\n\nclass CodeBlockWarning(Warning):\n def __init__(self, path: str, line: int, message: str, diff: str):\n super().__init__(path, line, f\"{message}\\n{diff}\")\n\n def __str__(self) -> str:\n return _warning(f\"{self.path}:{self.line}: {self.message}\")\n\n\nclass ValidationWarning(Warning):\n pass\n\n\nclass SphinxCodeBlock(Directive):\n \"\"\"Defines a code-block directive with the options Sphinx supports.\"\"\"\n\n has_content = True\n optional_arguments = 1 # language\n required_arguments = 0\n option_spec = {\n \"force\": directives.unchanged,\n \"linenos\": directives.unchanged,\n \"dedent\": directives.unchanged,\n \"lineno-start\": directives.unchanged,\n \"emphasize-lines\": directives.unchanged,\n \"caption\": directives.unchanged,\n \"class\": directives.unchanged,\n \"name\": directives.unchanged,\n }\n\n def run(self) -> List[nodes.Node]:\n # Produce a literal block with block.attributes[\"language\"] set.\n language = self.arguments[0] if self.arguments else \"python\"\n literal = nodes.literal_block(\"\\n\".join(self.content), \"\\n\".join(self.content))\n literal[\"language\"] = language\n return [literal]\n\n\ndirectives.register_directive(\"code-block\", SphinxCodeBlock)\n\n\nclass ParagraphInfo:\n lineno: int\n end_lineno: int\n src: str\n lines: List[str]\n\n def __init__(self, line: int, src: str) -> None:\n self.lineno = line\n self.src = src\n self.lines = src.splitlines()\n self.end_lineno = line + len(self.lines) - 1\n\n\ndef _is_node_in_table(node: nodes.Node) -> bool:\n \"\"\"Check if a node is inside a table by walking up the parent chain.\"\"\"\n while node.parent:\n node = node.parent\n if isinstance(node, nodes.table):\n return True\n return False\n\n\ndef _validate_schema(data: object) -> None:\n if not isinstance(data, dict):\n return\n for section, schema in SECTION_AND_SCHEMA:\n if section in data:\n jsonschema.validate(data, schema)\n\n\ndef _format_code_blocks(document: nodes.document, path: str) -> List[Warning]:\n \"\"\"Try to parse and format Python, YAML, and JSON code blocks. This does *not* update the\n sources, but collects issues for later reporting. Returns a list of warnings.\"\"\"\n issues: List[Warning] = []\n for code_block in document.findall(nodes.literal_block):\n language = code_block.attributes.get(\"language\", \"\")\n if language not in (\"python\", \"yaml\", \"json\"):\n continue\n original = code_block.astext()\n line = code_block.line if code_block.line else 0\n possible_config_data = None\n\n try:\n if language == \"python\":\n formatted = black.format_str(original, mode=black.FileMode(line_length=99))\n elif language == \"yaml\":\n yaml = YAML(pure=True)\n yaml.width = 10000 # do not wrap lines\n yaml.preserve_quotes = True # do not force particular quotes\n buf = io.BytesIO()\n possible_config_data = yaml.load(original)\n yaml.dump(possible_config_data, buf)\n formatted = buf.getvalue().decode(\"utf-8\")\n elif language == \"json\":\n formatted = json.dumps(json.loads(original), indent=2)\n else:\n assert False\n except Exception as e:\n issues.append(Warning(path, line, f\"formatting failed: {e}: {original!r}\"))\n continue\n\n try:\n _validate_schema(possible_config_data)\n except jsonschema.ValidationError as e:\n issues.append(ValidationWarning(path, line, f\"schema validation failed: {e.message}\"))\n\n if formatted == original:\n continue\n diff = \"\\n\".join(\n difflib.unified_diff(\n original.splitlines(),\n formatted.splitlines(),\n lineterm=\"\",\n fromfile=f\"{path}:{line} (original)\",\n tofile=f\"{path}:{line} (suggested, NOT required)\",\n )\n )\n\n # ignore suggestions to quote double colons like this:\n #\n # - build_stage::\n # + 'build_stage:':\n #\n if diff and not DOUBLE_COLON_WARNING.search(diff):\n issues.append(CodeBlockWarning(path, line, \"formatting suggested:\", diff))\n return issues\n\n\ndef _format_paragraphs(document: nodes.document, path: str, src_lines: List[str]) -> bool:\n \"\"\"Format paragraphs in the document. Returns True if ``src_lines`` was modified.\"\"\"\n\n paragraphs = [\n ParagraphInfo(line=p.line, src=p.rawsource)\n for p in document.findall(nodes.paragraph)\n if p.line is not None and p.rawsource and not _is_node_in_table(p)\n ]\n\n # Work from bottom to top to avoid messing up line numbers\n paragraphs.sort(key=lambda p: p.lineno, reverse=True)\n modified = False\n\n for p in paragraphs:\n # docutils does not give us the column offset, so we'll find it ourselves.\n col_offset = src_lines[p.lineno - 1].rfind(p.lines[0])\n assert col_offset >= 0, f\"{path}:{p.lineno}: rst parsing error.\"\n prefix = lambda i: \" \" * col_offset if i > 0 else src_lines[p.lineno - 1][:col_offset]\n\n # Defensive check to ensure the source paragraph matches the docutils paragraph\n for i, line in enumerate(p.lines):\n line_lhs = f\"{prefix(i)}{line}\"\n line_rhs = src_lines[p.lineno - 1 + i].rstrip() # docutils trims trailing whitespace\n assert line_lhs == line_rhs, f\"{path}:{p.lineno + i}: rst parsing error.\"\n\n # Replace current newlines with whitespace, and then split sentences.\n new_paragraph_src = END_OF_SENTENCE.sub(r\"\\1\\n\", p.src.replace(\"\\n\", \" \"))\n new_paragraph_lines = [\n f\"{prefix(i)}{line.lstrip()}\" for i, line in enumerate(new_paragraph_src.splitlines())\n ]\n\n if new_paragraph_lines != src_lines[p.lineno - 1 : p.end_lineno]:\n modified = True\n src_lines[p.lineno - 1 : p.end_lineno] = new_paragraph_lines\n\n return modified\n\n\ndef reformat_rst_file(path: str, warnings: List[Warning]) -> bool:\n \"\"\"Reformat a reStructuredText file \"in-place\". Returns True if modified, False otherwise.\"\"\"\n with open(path, \"r\", encoding=\"utf-8\") as f:\n src = f.read()\n\n src_lines = src.splitlines()\n document: nodes.document = publish_doctree(src, settings_overrides=DOCUTILS_SETTING)\n\n warnings.extend(_format_code_blocks(document, path))\n\n if not _format_paragraphs(document, path, src_lines):\n return False\n\n with open(f\"{path}.tmp\", \"w\", encoding=\"utf-8\") as f:\n f.write(\"\\n\".join(src_lines))\n f.write(\"\\n\")\n os.rename(f\"{path}.tmp\", path)\n print(f\"Fixed reStructuredText formatting: {path}\", flush=True)\n return True\n\n\ndef main(*files: str) -> None:\n modified = False\n warnings: List[Warning] = []\n for f in files:\n modified |= reformat_rst_file(f, warnings)\n\n if modified:\n subprocess.run([\"git\", \"--no-pager\", \"diff\", \"--color=always\", \"--\", *files])\n\n for warning in sorted(warnings, key=lambda w: isinstance(w, ValidationWarning)):\n print(warning, flush=True, file=sys.stderr)\n\n if warnings:\n print(\n _warning(f\"completed with {len(warnings)} potential issues\"),\n flush=True,\n file=sys.stderr,\n )\n sys.exit(1 if modified else 0)\n\n\nif __name__ == \"__main__\":\n main(*sys.argv[1:])\n" }, { "path": ".github/workflows/bin/generate_spack_yaml_containerize.sh", "content": "#!/bin/bash\n (echo \"spack:\" \\\n&& echo \" specs: []\" \\\n&& echo \" container:\" \\\n&& echo \" format: docker\" \\\n&& echo \" images:\" \\\n&& echo \" os: \\\"${SPACK_YAML_OS}\\\"\" \\\n&& echo \" spack:\" \\\n&& echo \" ref: ${GITHUB_REF}\") > spack.yaml\n" }, { "path": ".github/workflows/bin/setup_git.ps1", "content": "git config --global user.email \"spack@example.com\"\ngit config --global user.name \"Test User\"\ngit config --global core.longpaths true\n\nif ($(git branch --show-current) -ne \"develop\")\n{\n git branch develop origin/develop\n}\n" }, { "path": ".github/workflows/bin/setup_git.sh", "content": "#!/bin/bash -e\ngit config --global user.email \"spack@example.com\"\ngit config --global user.name \"Test User\"\n\n# create a local pr base branch\nif [[ -n $GITHUB_BASE_REF ]]; then\n git fetch origin \"${GITHUB_BASE_REF}:${GITHUB_BASE_REF}\"\nfi\n" }, { "path": ".github/workflows/bin/system_shortcut_check.ps1", "content": "param ($systemFolder, $shortcut)\n\n$start = [System.Environment]::GetFolderPath(\"$systemFolder\")\nInvoke-Item \"$start\\Programs\\Spack\\$shortcut\"\n" }, { "path": ".github/workflows/bootstrap.yml", "content": "name: Bootstrapping\n\non:\n # This Workflow can be triggered manually\n workflow_dispatch:\n workflow_call:\n schedule:\n # nightly at 2:16 AM\n - cron: \"16 2 * * *\"\n\nconcurrency:\n group: bootstrap-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}\n cancel-in-progress: true\n\njobs:\n distros-clingo-sources:\n if: github.repository == 'spack/spack'\n runs-on: ubuntu-latest\n container: ${{ matrix.image }}\n strategy:\n matrix:\n image: [\"fedora:latest\", \"opensuse/leap:latest\"]\n steps:\n - name: Setup Fedora\n if: ${{ matrix.image == 'fedora:latest' }}\n run: |\n dnf install -y \\\n bzip2 curl file gcc-c++ gcc gcc-gfortran git gzip \\\n make patch unzip which xz python3 python3-devel tree \\\n cmake bison bison-devel libstdc++-static gawk\n - name: Setup OpenSUSE\n if: ${{ matrix.image == 'opensuse/leap:latest' }}\n run: |\n # Harden CI by applying the workaround described here: https://www.suse.com/support/kb/doc/?id=000019505\n zypper update -y || zypper update -y\n zypper install -y \\\n bzip2 curl file gcc-c++ gcc gcc-fortran tar git gpg2 gzip \\\n make patch unzip which xz python3 python3-devel tree \\\n cmake bison\n - name: Checkout\n uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n with:\n fetch-depth: 0\n - name: Bootstrap clingo\n run: |\n . share/spack/setup-env.sh\n spack config add config:installer:new\n spack bootstrap disable github-actions-v2\n spack bootstrap disable github-actions-v0.6\n spack solve zlib\n tree ~/.spack/bootstrap/store/\n\n clingo-sources:\n if: github.repository == 'spack/spack'\n runs-on: ${{ matrix.runner }}\n strategy:\n matrix:\n runner: [\"macos-15-intel\", \"macos-latest\", \"ubuntu-latest\"]\n steps:\n - name: Setup macOS\n if: ${{ matrix.runner != 'ubuntu-latest' }}\n run: brew install bison tree\n - name: Checkout\n uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n with:\n fetch-depth: 0\n - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0\n with:\n python-version: \"3.12\"\n - name: Bootstrap clingo\n run: |\n . share/spack/setup-env.sh\n spack config add config:installer:new\n spack bootstrap disable github-actions-v2\n spack bootstrap disable github-actions-v0.6\n export PATH=\"$(brew --prefix bison)/bin:$(brew --prefix cmake)/bin:$PATH\"\n spack solve zlib\n tree ~/.spack/bootstrap/store/\n\n gnupg-sources:\n if: github.repository == 'spack/spack'\n runs-on: ${{ matrix.runner }}\n strategy:\n matrix:\n runner: [\"macos-15-intel\", \"macos-latest\", \"ubuntu-latest\"]\n steps:\n - name: Setup macOS\n if: ${{ matrix.runner != 'ubuntu-latest' }}\n run: brew install tree gawk\n - name: Remove system executables\n run: |\n while [ -n \"$(command -v gpg gpg2 patchelf)\" ]; do\n sudo rm $(command -v gpg gpg2 patchelf)\n done\n - name: Checkout\n uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n with:\n fetch-depth: 0\n - name: Bootstrap GnuPG\n run: |\n . share/spack/setup-env.sh\n spack config add config:installer:new\n spack solve zlib\n spack bootstrap disable github-actions-v2\n spack bootstrap disable github-actions-v0.6\n spack gpg list\n tree ~/.spack/bootstrap/store/\n\n from-binaries:\n if: github.repository == 'spack/spack'\n runs-on: ${{ matrix.runner }}\n strategy:\n matrix:\n runner: [\"macos-15-intel\", \"macos-latest\", \"ubuntu-latest\"]\n steps:\n - name: Setup macOS\n if: ${{ matrix.runner != 'ubuntu-latest' }}\n run: brew install tree\n - name: Remove system executables\n run: |\n while [ -n \"$(command -v gpg gpg2 patchelf)\" ]; do\n sudo rm $(command -v gpg gpg2 patchelf)\n done\n - name: Checkout\n uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n with:\n fetch-depth: 0\n - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0\n with:\n python-version: |\n 3.8\n 3.9\n 3.10\n 3.11\n 3.12\n 3.13\n 3.14\n - name: Set bootstrap sources\n run: |\n . share/spack/setup-env.sh\n spack bootstrap disable github-actions-v0.6\n spack bootstrap disable spack-install\n - name: Bootstrap clingo\n run: |\n . share/spack/setup-env.sh\n for ver in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do\n ver_dir=\"$(find \"$RUNNER_TOOL_CACHE/Python\" -wholename \"*/${ver}.*/*/bin\" | grep . || true)\"\n export SPACK_PYTHON=\"$ver_dir/python3\"\n if [ ! -d \"$ver_dir\" ] || ! \"$SPACK_PYTHON\" --version; then\n echo \"Python $ver not found\"\n exit 1\n fi\n spack solve zlib\n done\n tree ~/.spack/bootstrap/store\n - name: Bootstrap GnuPG\n run: |\n . share/spack/setup-env.sh\n spack config add config:installer:new\n spack gpg list\n tree ~/.spack/bootstrap/store/\n\n windows:\n if: github.repository == 'spack/spack'\n runs-on: \"windows-latest\"\n steps:\n - name: Checkout\n uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n with:\n fetch-depth: 0\n - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0\n with:\n python-version: \"3.12\"\n - name: Setup Windows\n run: |\n Remove-Item -Path (Get-Command gpg).Path\n Remove-Item -Path (Get-Command file).Path\n - name: Bootstrap clingo\n run: |\n ./share/spack/setup-env.ps1\n spack bootstrap disable github-actions-v2\n spack bootstrap disable github-actions-v0.6\n spack -d solve zlib\n ./share/spack/qa/validate_last_exit.ps1\n tree $env:userprofile/.spack/bootstrap/store/\n - name: Bootstrap GnuPG\n run: |\n ./share/spack/setup-env.ps1\n spack -d gpg list\n ./share/spack/qa/validate_last_exit.ps1\n tree $env:userprofile/.spack/bootstrap/store/\n\n dev-bootstrap:\n runs-on: ubuntu-latest\n container: registry.access.redhat.com/ubi8/ubi\n steps:\n - name: Install dependencies\n run: |\n dnf install -y \\\n bzip2 curl gcc-c++ gcc gcc-gfortran git gnupg2 gzip \\\n make patch python3.11 tcl unzip which xz\n - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n - name: Setup repo and non-root user\n run: |\n git --version\n git config --global --add safe.directory '*'\n git fetch --unshallow\n . .github/workflows/bin/setup_git.sh\n - name: Setup a virtual environment with platform-python\n run: |\n python3.11 -m venv ~/platform-spack-311\n source ~/platform-spack-311/bin/activate\n pip install --upgrade pip clingo\n - name: Bootstrap Spack development environment\n run: |\n source ~/platform-spack-311/bin/activate\n source share/spack/setup-env.sh\n spack debug report\n spack -d bootstrap now --dev\n" }, { "path": ".github/workflows/build-containers.yml", "content": "name: Containers\n\non:\n # This Workflow can be triggered manually\n workflow_dispatch:\n # Build new Spack develop containers nightly.\n schedule:\n - cron: '34 0 * * *'\n # Run on pull requests that modify this file\n pull_request:\n branches:\n - develop\n paths:\n - '.github/workflows/build-containers.yml'\n - 'share/spack/docker/*'\n - 'share/spack/templates/container/*'\n - 'lib/spack/spack/container/*'\n # Let's also build & tag Spack containers on releases.\n release:\n types: [published]\n\nconcurrency:\n group: build_containers-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}\n cancel-in-progress: true\n\njobs:\n deploy-images:\n runs-on: ubuntu-latest\n permissions:\n packages: write\n strategy:\n # Even if one container fails to build we still want the others\n # to continue their builds.\n fail-fast: false\n # A matrix of Dockerfile paths, associated tags, and which architectures\n # they support.\n matrix:\n # Meaning of the various items in the matrix list\n # 0: Container name (e.g. ubuntu-bionic)\n # 1: Platforms to build for\n # 2: Base image (e.g. ubuntu:22.04)\n dockerfile: [[amazon-linux, 'linux/amd64,linux/arm64', 'amazonlinux:2'],\n [centos-stream9, 'linux/amd64,linux/arm64', 'centos:stream9'],\n [leap15, 'linux/amd64,linux/arm64', 'opensuse/leap:15'],\n [ubuntu-focal, 'linux/amd64,linux/arm64', 'ubuntu:20.04'],\n [ubuntu-jammy, 'linux/amd64,linux/arm64', 'ubuntu:22.04'],\n [ubuntu-noble, 'linux/amd64,linux/arm64', 'ubuntu:24.04'],\n [almalinux8, 'linux/amd64,linux/arm64', 'almalinux:8'],\n [almalinux9, 'linux/amd64,linux/arm64', 'almalinux:9'],\n [rockylinux8, 'linux/amd64,linux/arm64', 'rockylinux:8'],\n [rockylinux9, 'linux/amd64,linux/arm64', 'rockylinux:9'],\n [fedora39, 'linux/amd64,linux/arm64', 'fedora:39'],\n [fedora40, 'linux/amd64,linux/arm64', 'fedora:40']]\n name: Build ${{ matrix.dockerfile[0] }}\n if: github.repository == 'spack/spack'\n steps:\n - name: Checkout\n uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n\n - name: Determine latest release tag\n id: latest\n run: |\n git fetch --quiet --tags\n echo \"tag=$(git tag --list --sort=-v:refname | grep -E '^v[0-9]+\\.[0-9]+\\.[0-9]+$' | head -n 1)\" | tee -a $GITHUB_OUTPUT\n\n - uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0\n id: docker_meta\n with:\n images: |\n ghcr.io/${{ github.repository_owner }}/${{ matrix.dockerfile[0] }}\n ${{ github.repository_owner }}/${{ matrix.dockerfile[0] }}\n tags: |\n type=schedule,pattern=nightly\n type=schedule,pattern=develop\n type=semver,pattern={{version}}\n type=semver,pattern={{major}}.{{minor}}\n type=semver,pattern={{major}}\n type=ref,event=branch\n type=ref,event=pr\n type=raw,value=latest,enable=${{ github.ref == format('refs/tags/{0}', steps.latest.outputs.tag) }}\n\n - name: Generate the Dockerfile\n env:\n SPACK_YAML_OS: \"${{ matrix.dockerfile[2] }}\"\n run: |\n .github/workflows/bin/generate_spack_yaml_containerize.sh\n . share/spack/setup-env.sh\n mkdir -p dockerfiles/${{ matrix.dockerfile[0] }}\n spack containerize --last-stage=bootstrap | tee dockerfiles/${{ matrix.dockerfile[0] }}/Dockerfile\n printf \"Preparing to build ${{ env.container }} from dockerfiles/${{ matrix.dockerfile[0] }}/Dockerfile\"\n if [ ! -f \"dockerfiles/${{ matrix.dockerfile[0] }}/Dockerfile\" ]; then\n printf \"dockerfiles/${{ matrix.dockerfile[0] }}/Dockerfile does not exist\"\n exit 1;\n fi\n\n - name: Upload Dockerfile\n uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1\n with:\n name: dockerfiles_${{ matrix.dockerfile[0] }}\n path: dockerfiles\n\n - name: Set up QEMU\n uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0\n\n - name: Set up Docker Buildx\n uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0\n\n - name: Log in to GitHub Container Registry\n uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0\n with:\n registry: ghcr.io\n username: ${{ github.actor }}\n password: ${{ secrets.GITHUB_TOKEN }}\n\n - name: Log in to DockerHub\n if: github.event_name != 'pull_request'\n uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0\n with:\n username: ${{ secrets.DOCKERHUB_USERNAME }}\n password: ${{ secrets.DOCKERHUB_TOKEN }}\n\n - name: Build & Deploy ${{ matrix.dockerfile[0] }}\n uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0\n with:\n context: dockerfiles/${{ matrix.dockerfile[0] }}\n platforms: ${{ matrix.dockerfile[1] }}\n push: ${{ github.event_name != 'pull_request' }}\n tags: ${{ steps.docker_meta.outputs.tags }}\n labels: ${{ steps.docker_meta.outputs.labels }}\n\n merge-dockerfiles:\n runs-on: ubuntu-latest\n needs: deploy-images\n steps:\n - name: Merge Artifacts\n uses: actions/upload-artifact/merge@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1\n with:\n name: dockerfiles\n pattern: dockerfiles_*\n delete-merged: true\n" }, { "path": ".github/workflows/ci.yaml", "content": "name: ci\n\non:\n push:\n branches:\n - develop\n - releases/**\n pull_request:\n branches:\n - develop\n - releases/**\n merge_group:\n\nconcurrency:\n group: ci-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}\n cancel-in-progress: true\n\njobs:\n # Check which files have been updated by the PR\n changes:\n runs-on: ubuntu-latest\n # Set job outputs to values from filter step\n outputs:\n bootstrap: ${{ steps.filter.outputs.bootstrap }}\n core: ${{ steps.filter.outputs.core }}\n packages: ${{ steps.filter.outputs.packages }}\n steps:\n - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n if: ${{ github.event_name == 'push' || github.event_name == 'merge_group' }}\n with:\n fetch-depth: 0\n # For pull requests it's not necessary to checkout the code\n - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1\n id: filter\n with:\n # For merge group events, compare against the target branch (main)\n base: ${{ github.event_name == 'merge_group' && github.event.merge_group.base_ref || '' }}\n # For merge group events, use the merge group head ref\n ref: ${{ github.event_name == 'merge_group' && github.event.merge_group.head_sha || github.ref }}\n # See https://github.com/dorny/paths-filter/issues/56 for the syntax used below\n # Don't run if we only modified packages in the\n # built-in repository or documentation\n filters: |\n bootstrap:\n - 'lib/spack/**'\n - 'share/spack/**'\n - '.github/workflows/bootstrap.yml'\n - '.github/workflows/ci.yaml'\n core:\n - './!(var/**)/**'\n - 'var/spack/test_repos/**'\n packages:\n - 'var/**'\n # Some links for easier reference:\n #\n # \"github\" context: https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#github-context\n # job outputs: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idoutputs\n # setting environment variables from earlier steps: https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable\n #\n bootstrap:\n if: ${{ github.repository == 'spack/spack' && needs.changes.outputs.bootstrap == 'true' }}\n needs: [ prechecks, changes ]\n uses: ./.github/workflows/bootstrap.yml\n secrets: inherit\n\n unit-tests:\n if: ${{ github.repository == 'spack/spack' && needs.changes.outputs.core == 'true' }}\n needs: [ prechecks, changes ]\n uses: ./.github/workflows/unit_tests.yaml\n secrets: inherit\n\n prechecks:\n needs: [ changes ]\n uses: ./.github/workflows/prechecks.yml\n secrets: inherit\n with:\n with_coverage: ${{ needs.changes.outputs.core }}\n with_packages: ${{ needs.changes.outputs.packages }}\n\n import-check:\n needs: [ changes ]\n uses: ./.github/workflows/import-check.yaml\n\n all-prechecks:\n needs: [ prechecks ]\n if: ${{ always() }}\n runs-on: ubuntu-latest\n steps:\n - name: Success\n run: |\n [ \"${{ needs.prechecks.result }}\" = \"success\" ] && exit 0\n [ \"${{ needs.prechecks.result }}\" = \"skipped\" ] && exit 0\n echo \"Unit tests failed.\"\n exit 1\n\n coverage:\n needs: [ unit-tests, prechecks ]\n if: ${{ needs.changes.outputs.core }}\n uses: ./.github/workflows/coverage.yml\n secrets: inherit\n\n all:\n needs: [ unit-tests, coverage, bootstrap ]\n if: ${{ always() }}\n runs-on: ubuntu-latest\n # See https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/accessing-contextual-information-about-workflow-runs#needs-context\n steps:\n - name: Status summary\n run: |\n if [ \"${{ needs.unit-tests.result }}\" = \"success\" ] || [ \"${{ needs.unit-tests.result }}\" = \"skipped\" ]; then\n if [ \"${{ needs.bootstrap.result }}\" = \"success\" ] || [ \"${{ needs.bootstrap.result }}\" = \"skipped\" ]; then\n exit 0\n else\n echo \"Bootstrap tests failed.\"\n exit 1\n fi\n else\n echo \"Unit tests failed.\"\n exit 1\n fi\n" }, { "path": ".github/workflows/coverage.yml", "content": "name: coverage\n\non:\n workflow_call:\n\njobs:\n # Upload coverage reports to codecov once as a single bundle\n upload:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0\n with:\n python-version: '3.14'\n\n - name: Install python dependencies\n run: pip install -r .github/workflows/requirements/coverage/requirements.txt\n\n - name: Download coverage artifact files\n uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1\n with:\n pattern: coverage-*\n path: coverage\n merge-multiple: true\n\n - run: ls -la coverage\n - run: coverage combine -a coverage/.coverage*\n - run: coverage xml\n\n - name: \"Upload coverage report to CodeCov\"\n uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2\n with:\n verbose: true\n fail_ci_if_error: false\n token: ${{ secrets.CODECOV_TOKEN }}\n" }, { "path": ".github/workflows/import-check.yaml", "content": "name: import-check\n\non:\n workflow_call:\n\njobs:\n # Check we don't make the situation with circular imports worse\n import-check:\n continue-on-error: true\n runs-on: ubuntu-latest\n steps:\n - uses: julia-actions/setup-julia@4a12c5f801ca5ef0458bba44687563ef276522dd # v3.0.0\n with:\n version: '1.10'\n - uses: julia-actions/cache@9a93c5fb3e9c1c20b60fc80a478cae53e38618a4 # v3.0.2\n\n # PR: use the base of the PR as the old commit\n - name: Checkout PR base commit\n if: github.event_name == 'pull_request'\n uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n with:\n ref: ${{ github.event.pull_request.base.sha }}\n path: old\n # not a PR: use the previous commit as the old commit\n - name: Checkout previous commit\n if: github.event_name != 'pull_request'\n uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n with:\n fetch-depth: 2\n path: old\n - name: Checkout previous commit\n if: github.event_name != 'pull_request'\n run: git -C old reset --hard HEAD^\n\n - name: Checkout new commit\n uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n with:\n path: new\n - name: Install circular import checker\n uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n with:\n repository: haampie/circular-import-fighter\n ref: f1c56367833f3c82f6a85dc58595b2cd7995ad48\n path: circular-import-fighter\n - name: Install dependencies\n working-directory: circular-import-fighter\n run: make -j dependencies\n - name: Circular import check (without inline imports)\n working-directory: circular-import-fighter\n run: make -j compare \"SPACK_ROOT=../old ../new\"\n - name: Circular import check (with inline imports)\n working-directory: circular-import-fighter\n run: make clean-graph && make -j compare \"SPACK_ROOT=../old ../new\" IMPORTS_FLAGS=--inline\n" }, { "path": ".github/workflows/prechecks.yml", "content": "name: prechecks\n\non:\n workflow_call:\n inputs:\n with_coverage:\n required: true\n type: string\n with_packages:\n required: true\n type: string\n\nconcurrency:\n group: style-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}\n cancel-in-progress: true\n\n\njobs:\n validate:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0\n with:\n python-version: '3.13'\n - name: Install Python Packages\n run: |\n pip install -r .github/workflows/requirements/style/requirements.txt\n # Validate that the code can be run on all the Python versions supported by Spack\n - name: vermin\n run: |\n vermin --backport importlib \\\n --backport argparse \\\n --violations \\\n --backport typing \\\n -t=3.6- \\\n -vvv \\\n --exclude-regex lib/spack/spack/vendor \\\n lib/spack/spack/ lib/spack/llnl/ bin/ var/spack/test_repos\n # Check that __slots__ are used properly\n - name: slotscheck\n run: |\n ./bin/spack python -m slotscheck --exclude-modules=\"spack.test|spack.vendor\" lib/spack/spack/\n\n # Run style checks on the files that have been changed\n style:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n with:\n fetch-depth: 2\n - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0\n with:\n python-version: '3.13'\n - name: Install Python packages\n run: |\n pip install -r .github/workflows/requirements/style/requirements.txt\n echo \"PYTHONPATH=$PWD/lib/spack\" >> $GITHUB_ENV\n - name: Run style tests (code)\n run: |\n bin/spack style --base HEAD^1\n bin/spack license verify\n pylint -j $(nproc) --disable=all --enable=unspecified-encoding --ignore-paths=lib/spack/spack/vendor lib\n - name: Run style tests (docs)\n run: .github/workflows/bin/format-rst.py $(git ls-files 'lib/spack/docs/*.rst')\n" }, { "path": ".github/workflows/requirements/coverage/requirements.txt", "content": "coverage==7.13.5\n" }, { "path": ".github/workflows/requirements/style/requirements.txt", "content": "black==25.12.0\nclingo==5.8.0\nflake8==7.3.0\nisort==7.0.0\nmypy==1.20.1\nvermin==1.8.0\npylint==4.0.5\ndocutils==0.22.4\nruamel.yaml==0.19.1\nslotscheck==0.19.1\nruff==0.15.11\n" }, { "path": ".github/workflows/requirements/unit_tests/requirements.txt", "content": "pytest==9.0.3\npytest-cov==7.1.0\npytest-xdist==3.8.0\ncoverage[toml]<=7.11.0\nclingo==5.8.0\n" }, { "path": ".github/workflows/stale.yaml", "content": "name: 'Close stale issues and PRs'\non:\n schedule:\n # Run every day at 1:14 UTC\n - cron: '14 1 * * *'\n workflow_dispatch:\n\njobs:\n stale:\n runs-on: ubuntu-latest\n permissions:\n actions: write\n issues: write\n pull-requests: write\n steps:\n - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0\n with:\n # Issues configuration\n stale-issue-message: >\n This issue has been automatically marked as stale because it has not had any activity in the last 6 months.\n It will be closed in 30 days if there is no further activity.\n \n \n If the issue is waiting for a reply from maintainers, feel free to ping them as a reminder.\n If it is waiting and has no comments yet, feel free to ping `@spack/spack-releasers` or simply leave a comment saying this should not be marked stale.\n This will also reset the issue's stale state.\n \n \n Thank you for your contributions!\n close-issue-message: >\n This issue was closed because it had no activity for 30 days after being marked stale.\n If you feel this is in error, please feel free to reopen this issue.\n stale-issue-label: 'stale'\n any-of-issue-labels: 'build-error,unreproducible,question,documentation,environments'\n exempt-issue-labels: 'pinned,triage,impact-low,impact-medium,impact-high'\n\n # Pull requests configuration\n stale-pr-message: >\n This pull request has been automatically marked as stale because it has not had any activity in the last 6 months. \n It will be closed in 30 days if there is no further activity.\n \n \n If the pull request is waiting for a reply from reviewers, feel free to ping them as a reminder.\n If it is waiting and has no assigned reviewer, feel free to ping `@spack/spack-releasers` or simply leave a comment saying this should not be marked stale.\n This will reset the pull request's stale state.\n \n \n To get more eyes on your pull request, you can post a link in the #pull-requests channel of the Spack Slack.\n \n Thank you for your contributions!\n close-pr-message: >\n This pull request was closed because it had no activity for 30 days after being marked stale.\n If you feel this is in error, please feel free to reopen this pull request.\n stale-pr-label: 'stale'\n any-of-pr-labels: 'new-package,update-package'\n exempt-pr-labels: 'pinned'\n\n # General configuration\n ascending: true\n operations-per-run: 1000\n remove-stale-when-updated: true\n enable-statistics: true\n days-before-stale: 180\n days-before-close: 30\n" }, { "path": ".github/workflows/triage.yml", "content": "#-----------------------------------------------------------------------\n# DO NOT modify unless you really know what you are doing.\n#\n# See https://stackoverflow.com/a/74959635 for more info.\n# Talk to @alecbcs if you have questions/are not sure of a change's\n# possible impact to security.\n#-----------------------------------------------------------------------\nname: triage\non:\n pull_request_target:\n branches:\n - develop\n\njobs:\n pr:\n runs-on: ubuntu-latest\n permissions:\n contents: read\n pull-requests: write\n issues: write\n steps:\n - uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b # v6.0.1\n" }, { "path": ".github/workflows/unit_tests.yaml", "content": "name: unit tests\n\non:\n workflow_dispatch:\n workflow_call:\n\nconcurrency:\n group: unit_tests-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}\n cancel-in-progress: true\n\njobs:\n # Run unit tests with different configurations on linux\n ubuntu:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n os: [ubuntu-latest]\n python-version: ['3.8', '3.9', '3.10', '3.11', '3.14']\n on_develop:\n - ${{ github.ref == 'refs/heads/develop' }}\n include:\n - python-version: '3.7'\n os: ubuntu-22.04\n on_develop: ${{ github.ref == 'refs/heads/develop' }}\n exclude:\n - python-version: '3.8'\n os: ubuntu-latest\n on_develop: false\n - python-version: '3.9'\n os: ubuntu-latest\n on_develop: false\n - python-version: '3.10'\n os: ubuntu-latest\n on_develop: false\n - python-version: '3.11'\n os: ubuntu-latest\n on_develop: false\n\n steps:\n - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n with:\n fetch-depth: 0\n - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0\n with:\n python-version: ${{ matrix.python-version }}\n - name: Install System packages\n run: |\n sudo apt-get -y update\n # Needed for unit tests\n sudo apt-get -y install \\\n coreutils cvs gfortran graphviz gnupg2 mercurial ninja-build \\\n cmake bison libbison-dev subversion\n # On ubuntu 24.04, kcov was removed. It may come back in some future Ubuntu\n - name: Set up Homebrew\n id: set-up-homebrew\n uses: Homebrew/actions/setup-homebrew@40e9946c182a64b3db1bf51be0dcb915f7802aa9\n - name: Install kcov with brew\n run: \"brew install kcov\"\n - name: Install Python packages\n run: |\n # See https://github.com/coveragepy/coveragepy/issues/2082\n pip install --upgrade pip pytest pytest-xdist pytest-cov \"coverage<=7.11.0\"\n pip install --upgrade \"mypy>=0.900\" \"click\" \"ruff\"\n - name: Setup git configuration\n run: |\n # Need this for the git tests to succeed.\n git --version\n . .github/workflows/bin/setup_git.sh\n - name: Bootstrap clingo\n if: ${{ matrix.concretizer == 'clingo' }}\n env:\n SPACK_PYTHON: python\n run: |\n . share/spack/setup-env.sh\n spack bootstrap disable spack-install\n spack bootstrap now\n spack -v solve zlib\n - name: Run unit tests\n env:\n SPACK_PYTHON: python\n SPACK_TEST_PARALLEL: 4\n COVERAGE: true\n COVERAGE_FILE: coverage/.coverage-${{ matrix.os }}-python${{ matrix.python-version }}\n UNIT_TEST_COVERAGE: ${{ matrix.python-version == '3.14' }}\n run: |\n share/spack/qa/run-unit-tests\n - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1\n with:\n name: coverage-${{ matrix.os }}-python${{ matrix.python-version }}\n path: coverage\n include-hidden-files: true\n # Test shell integration\n shell:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n with:\n fetch-depth: 0\n - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0\n with:\n python-version: '3.11'\n - name: Install System packages\n run: |\n sudo apt-get -y update\n # Needed for shell tests\n sudo apt-get install -y coreutils csh zsh tcsh fish dash bash subversion\n # On ubuntu 24.04, kcov was removed. It may come back in some future Ubuntu\n - name: Set up Homebrew\n id: set-up-homebrew\n uses: Homebrew/actions/setup-homebrew@40e9946c182a64b3db1bf51be0dcb915f7802aa9\n - name: Install kcov with brew\n run: \"brew install kcov\"\n - name: Install Python packages\n run: |\n pip install --upgrade pip -r .github/workflows/requirements/unit_tests/requirements.txt\n - name: Setup git configuration\n run: |\n # Need this for the git tests to succeed.\n git --version\n . .github/workflows/bin/setup_git.sh\n - name: Run shell tests\n env:\n COVERAGE: true\n run: |\n share/spack/qa/run-shell-tests\n - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1\n with:\n name: coverage-shell\n path: coverage\n include-hidden-files: true\n\n # Test RHEL8 UBI with platform Python. This job is run\n # only on PRs modifying core Spack\n rhel8-platform-python:\n runs-on: ubuntu-latest\n container: registry.access.redhat.com/ubi8/ubi\n steps:\n - name: Install dependencies\n run: |\n dnf install -y \\\n bzip2 curl gcc-c++ gcc gcc-gfortran git gnupg2 gzip \\\n make patch tcl unzip which xz\n - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n - name: Setup repo and non-root user\n run: |\n git --version\n git config --global --add safe.directory '*'\n git fetch --unshallow\n . .github/workflows/bin/setup_git.sh\n - name: Setup a virtual environment with platform-python\n run: |\n /usr/libexec/platform-python -m venv ~/platform-spack\n source ~/platform-spack/bin/activate\n pip install --upgrade pip pytest coverage[toml] pytest-xdist\n - name: Bootstrap Spack development environment and run unit tests\n run: |\n source ~/platform-spack/bin/activate\n source share/spack/setup-env.sh\n spack debug report\n spack -d bootstrap now\n pytest --verbose -x -n3 --dist loadfile -k 'not cvs and not svn and not hg'\n # Test for the clingo based solver (using clingo-cffi)\n clingo-cffi:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n with:\n fetch-depth: 0\n - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0\n with:\n python-version: '3.13'\n - name: Install System packages\n run: |\n sudo apt-get -y update\n sudo apt-get -y install coreutils gfortran graphviz gnupg2\n - name: Install Python packages\n run: |\n pip install --upgrade pip -r .github/workflows/requirements/unit_tests/requirements.txt\n pip install --upgrade -r .github/workflows/requirements/style/requirements.txt\n - name: Run unit tests (full suite with coverage)\n env:\n COVERAGE: true\n COVERAGE_FILE: coverage/.coverage-clingo-cffi\n run: |\n . share/spack/setup-env.sh\n spack bootstrap disable spack-install\n spack bootstrap disable github-actions-v0.6\n spack bootstrap disable github-actions-v2\n spack bootstrap status\n spack solve zlib\n pytest --verbose --cov --cov-config=pyproject.toml --cov-report=xml:coverage.xml -x -n3 lib/spack/spack/test/concretization/core.py\n - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1\n with:\n name: coverage-clingo-cffi\n path: coverage\n include-hidden-files: true\n # Run unit tests on MacOS\n macos:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n os: [macos-15-intel, macos-latest]\n python-version: [\"3.14\"]\n steps:\n - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n with:\n fetch-depth: 0\n - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0\n with:\n python-version: ${{ matrix.python-version }}\n - name: Install Python packages\n run: |\n pip install --upgrade pip\n # See https://github.com/coveragepy/coveragepy/issues/2082\n pip install --upgrade -r .github/workflows/requirements/unit_tests/requirements.txt\n - name: Setup Homebrew packages\n run: |\n brew install dash fish gcc gnupg kcov\n - name: Run unit tests\n env:\n COVERAGE_FILE: coverage/.coverage-${{ matrix.os }}-python${{ matrix.python-version }}\n run: |\n git --version\n . .github/workflows/bin/setup_git.sh\n . share/spack/setup-env.sh\n spack bootstrap disable spack-install\n spack solve zlib\n python3 -m pytest --verbose --cov --cov-config=pyproject.toml --cov-report=xml:coverage.xml --dist loadfile -x -n4\n - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1\n with:\n name: coverage-${{ matrix.os }}-python${{ matrix.python-version }}\n path: coverage\n include-hidden-files: true\n # Run unit tests on Windows\n windows:\n defaults:\n run:\n shell:\n powershell Invoke-Expression -Command \"./share/spack/qa/windows_test_setup.ps1\"; {0}\n runs-on: windows-latest\n steps:\n - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n with:\n fetch-depth: 0\n - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0\n with:\n python-version: '3.14'\n - name: Install Python packages\n run: |\n python -m pip install --upgrade pip pywin32 -r .github/workflows/requirements/unit_tests/requirements.txt\n python -m pip install --upgrade pip -r .github/workflows/requirements/style/requirements.txt\n - name: Create local develop\n run: |\n ./.github/workflows/bin/setup_git.ps1\n - name: Unit Test\n env:\n COVERAGE_FILE: coverage/.coverage-windows\n run: |\n python -m pytest -x --verbose --cov --cov-config=pyproject.toml\n ./share/spack/qa/validate_last_exit.ps1\n - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1\n with:\n name: coverage-windows\n path: coverage\n include-hidden-files: true\n\n canonicalization:\n name: package.py canonicalization\n runs-on: ubuntu-latest\n container:\n image: ghcr.io/spack/all-pythons:2025-10-10\n\n steps:\n - name: Checkout Spack (current)\n uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n with:\n path: spack-current\n - name: Checkout Spack (previous)\n uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n with:\n path: spack-previous\n ref: ${{ github.event.pull_request.base.sha || github.event.before }}\n - name: Checkout Spack Packages\n uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n with:\n repository: spack/spack-packages\n path: spack-packages\n - name: Test package.py canonicalization\n run: spack-current/.github/workflows/bin/canonicalize.py\n --spack $PWD/spack-previous $PWD/spack-current\n --python python3.6 python3.7 python3.8 python3.9 python3.10 python3.11 python3.12 python3.13 python3.14\n --input-dir spack-packages/repos/spack_repo/builtin/packages/\n --output-dir canonicalized\n" }, { "path": ".gitignore", "content": "##########################\n# Spack-specific ignores #\n##########################\n\n/var/spack/stage\n/var/spack/cache\n/var/spack/environments\n/opt\n/share/spack/modules\n/share/spack/lmod\n# Debug logs\nspack-db.*\n*.in.log\n*.out.log\nCLAUDE.md\n\n# Configuration: Ignore everything in /etc/spack,\n# except defaults and site scopes that ship with spack\n/etc/spack/*\n!/etc/spack/defaults\n!/etc/spack/site/README.md\n\n###########################\n# Coding agent state\n###########################\n.claude/\n.gemini/\n.codex/\n\n###########################\n# Python-specific ignores #\n###########################\n\n# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\n#lib/\n#lib64/\nparts/\nsdist/\n#var/\nwheels/\nshare/python-wheels/\n*.egg-info/\n.installed.cfg\n*.egg\nMANIFEST\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.nox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n*.py,cover\n.hypothesis/\n.pytest_cache/\ncover/\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\nlocal_settings.py\ndb.sqlite3\ndb.sqlite3-journal\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\n.pybuilder/\ntarget/\n\n# Jupyter Notebook\n.ipynb_checkpoints\n\n# IPython\nprofile_default/\nipython_config.py\n\n# pyenv\n# For a library or package, you might want to ignore these files since the code is\n# intended to run in multiple environments; otherwise, check them in:\n# .python-version\n\n# pipenv\n# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.\n# However, in case of collaboration, if having platform-specific dependencies or dependencies\n# having no cross-platform support, pipenv may install dependencies that don't work, or not\n# install all needed dependencies.\n#Pipfile.lock\n\n# PEP 582; used by e.g. github.com/David-OConnor/pyflow\n__pypackages__/\n\n# Celery stuff\ncelerybeat-schedule\ncelerybeat.pid\n\n# SageMath parsed files\n*.sage.py\n\n# Environments\n.env\n.venv\nenv/\nvenv/\nENV/\nenv.bak/\nvenv.bak/\n!/lib/spack/env\n\n# Spyder project settings\n.spyderproject\n.spyproject\n\n# Rope project settings\n.ropeproject\n\n# mkdocs documentation\n/site\n\n# mypy\n.mypy_cache/\n.dmypy.json\ndmypy.json\n\n# Pyre type checker\n.pyre/\n\n# pytype static type analyzer\n.pytype/\n\n# Cython debug symbols\ncython_debug/\n\n########################\n# Vim-specific ignores #\n########################\n\n# Swap\n[._]*.s[a-v][a-z]\n!*.svg # comment out if you don't need vector files\n[._]*.sw[a-p]\n[._]s[a-rt-v][a-z]\n[._]ss[a-gi-z]\n[._]sw[a-p]\n\n# Session\nSession.vim\nSessionx.vim\n\n# Temporary\n.netrwhist\n*~\n# Auto-generated tag files\ntags\n# Persistent undo\n[._]*.un~\n\n##########################\n# Emacs-specific ignores #\n##########################\n\n*~\n\\#*\\#\n/.emacs.desktop\n/.emacs.desktop.lock\n*.elc\nauto-save-list\ntramp\n.\\#*\n\n# Org-mode\n.org-id-locations\n\n# flymake-mode\n*_flymake.*\n\n# eshell files\n/eshell/history\n/eshell/lastdir\n\n# zsh byte-compiled files\n*.zwc\n\n# elpa packages\n/elpa/\n\n# reftex files\n*.rel\n\n# AUCTeX auto folder\n/auto/\n\n# cask packages\n.cask/\ndist/\n\n# Flycheck\nflycheck_*.el\n\n# server auth directory\n/server/\n\n# projectiles files\n.projectile\n\n# directory configuration\n.dir-locals.el\n\n# network security\n/network-security.data\n\n############################\n# Eclipse-specific ignores #\n############################\n\n.metadata\n#bin/\ntmp/\n*.tmp\n*.bak\n*.swp\n*~.nib\nlocal.properties\n.settings/\n.loadpath\n.recommenders\n\n# External tool builders\n.externalToolBuilders/\n\n# Locally stored \"Eclipse launch configurations\"\n*.launch\n\n# PyDev specific (Python IDE for Eclipse)\n*.pydevproject\n\n# CDT-specific (C/C++ Development Tooling)\n.cproject\n\n# CDT- autotools\n.autotools\n\n# Java annotation processor (APT)\n.factorypath\n\n# PDT-specific (PHP Development Tools)\n.buildpath\n\n# sbteclipse plugin\n.target\n\n# Tern plugin\n.tern-project\n\n# TeXlipse plugin\n.texlipse\n\n# STS (Spring Tool Suite)\n.springBeans\n\n# Code Recommenders\n.recommenders/\n\n# Annotation Processing\n.apt_generated/\n.apt_generated_test/\n\n# Scala IDE specific (Scala & Java development for Eclipse)\n.cache-main\n.scala_dependencies\n.worksheet\n\n# Uncomment this line if you wish to ignore the project description file.\n# Typically, this file would be tracked if it contains build/dependency configurations:\n#.project\n\n##################################\n# Visual Studio-specific ignores #\n##################################\n\n.vscode/*\n!.vscode/settings.json\n!.vscode/tasks.json\n!.vscode/launch.json\n!.vscode/extensions.json\n*.code-workspace\n\n# Local History for Visual Studio Code\n.history/\n\n#################################\n# Sublime Text-specific ignores #\n#################################\n\n# Cache files for Sublime Text\n*.tmlanguage.cache\n*.tmPreferences.cache\n*.stTheme.cache\n\n# Workspace files are user-specific\n*.sublime-workspace\n\n# Project files should be checked into the repository, unless a significant\n# proportion of contributors will probably not be using Sublime Text\n# *.sublime-project\n\n# SFTP configuration file\nsftp-config.json\nsftp-config-alt*.json\n\n# Package control specific files\nPackage Control.last-run\nPackage Control.ca-list\nPackage Control.ca-bundle\nPackage Control.system-ca-bundle\nPackage Control.cache/\nPackage Control.ca-certs/\nPackage Control.merged-ca-bundle\nPackage Control.user-ca-bundle\noscrypto-ca-bundle.crt\nbh_unicode_properties.cache\n\n# Sublime-github package stores a github token in this file\n# https://packagecontrol.io/packages/sublime-github\nGitHub.sublime-settings\n\n##############################\n# JetBrains-specific ignores #\n##############################\n\n# Ignore the entire folder since it may conatin more files than\n# just the ones listed below\n.idea/\n\n# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider\n# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839\n\n# User-specific stuff\n.idea/**/workspace.xml\n.idea/**/tasks.xml\n.idea/**/usage.statistics.xml\n.idea/**/dictionaries\n.idea/**/shelf\n\n# Generated files\n.idea/**/contentModel.xml\n\n# Sensitive or high-churn files\n.idea/**/dataSources/\n.idea/**/dataSources.ids\n.idea/**/dataSources.local.xml\n.idea/**/sqlDataSources.xml\n.idea/**/dynamic.xml\n.idea/**/uiDesigner.xml\n.idea/**/dbnavigator.xml\n\n# Gradle\n.idea/**/gradle.xml\n.idea/**/libraries\n\n# Gradle and Maven with auto-import\n# When using Gradle or Maven with auto-import, you should exclude module files,\n# since they will be recreated, and may cause churn. Uncomment if using\n# auto-import.\n# .idea/artifacts\n# .idea/compiler.xml\n# .idea/jarRepositories.xml\n# .idea/modules.xml\n# .idea/*.iml\n# .idea/modules\n# *.iml\n# *.ipr\n\n# CMake\ncmake-build-*/\n\n# Mongo Explorer plugin\n.idea/**/mongoSettings.xml\n\n# File-based project format\n*.iws\n\n# IntelliJ\nout/\n\n# mpeltonen/sbt-idea plugin\n.idea_modules/\n\n# JIRA plugin\natlassian-ide-plugin.xml\n\n# Cursive Clojure plugin\n.idea/replstate.xml\n\n# Crashlytics plugin (for Android Studio and IntelliJ)\ncom_crashlytics_export_strings.xml\ncrashlytics.properties\ncrashlytics-build.properties\nfabric.properties\n\n# Editor-based Rest Client\n.idea/httpRequests\n\n# Android studio 3.1+ serialized cache file\n.idea/caches/build_file_checksums.ser\n\n##########################\n# macOS-specific ignores #\n##########################\n\n# General\n.DS_Store\n.AppleDouble\n.LSOverride\n\n# Icon must end with two \\r\nIcon\r\n\n# Thumbnails\n._*\n\n# Files that might appear in the root of a volume\n.DocumentRevisions-V100\n.fseventsd\n.Spotlight-V100\n.TemporaryItems\n.Trashes\n.VolumeIcon.icns\n.com.apple.timemachine.donotpresent\n\n# Directories potentially created on remote AFP share\n.AppleDB\n.AppleDesktop\nNetwork Trash Folder\nTemporary Items\n.apdisk\n\n##########################\n# Linux-specific ignores #\n##########################\n\n*~\n\n# temporary files which can be created if a process still has a handle open of a deleted file\n.fuse_hidden*\n\n# KDE directory preferences\n.directory\n\n# Linux trash folder which might appear on any partition or disk\n.Trash-*\n\n# .nfs files are created when an open file is removed but is still being accessed\n.nfs*\n\n############################\n# Windows-specific ignores #\n############################\n\n# Windows thumbnail cache files\nThumbs.db\nThumbs.db:encryptable\nehthumbs.db\nehthumbs_vista.db\n\n# Dump file\n*.stackdump\n\n# Folder config file\n[Dd]esktop.ini\n\n# Recycle Bin used on file shares\n$RECYCLE.BIN/\n\n# Windows Installer files\n*.cab\n*.msi\n*.msix\n*.msm\n*.msp\n\n# Windows shortcuts\n*.lnk\r\n" }, { "path": ".mailmap", "content": "Abhinav Bhatele Abhinav Bhatele \nAdam Moody Adam T. Moody \nAlfredo Gimenez Alfredo Gimenez \nAlfredo Gimenez Alfredo Adolfo Gimenez \nAndrew Williams Andrew Williams \nAxel Huebl Axel Huebl \nAxel Huebl Axel Huebl \nBen Boeckel Ben Boeckel \nBen Boeckel Ben Boeckel \nBenedikt Hegner Benedikt Hegner \nBrett Viren Brett Viren \nDavid Boehme David Boehme \nDavid Boehme David Boehme \nDavid Poliakoff David Poliakoff \nDhanannjay Deo Dhanannjay 'Djay' Deo \nElizabeth Fischer Elizabeth F \nElizabeth Fischer Elizabeth F \nElizabeth Fischer Elizabeth Fischer \nElizabeth Fischer citibeth \nGeoffrey Oxberry Geoffrey Oxberry \nGlenn Johnson Glenn Johnson \nGlenn Johnson Glenn Johnson \nGregory Becker Gregory Becker \nGregory Becker Gregory Becker \nGregory Becker Gregory Becker \nGregory L. Lee Greg Lee \nGregory L. Lee Gregory L. Lee \nGregory L. Lee Gregory L. Lee \nGregory L. Lee Gregory L. Lee \nGregory L. Lee Gregory L. Lee \nGregory L. Lee Gregory Lee \nHarmen Stoppels Harmen Stoppels \nIan Lee Ian Lee \nJames Wynne III James Riley Wynne III \nJames Wynne III James Wynne III \nJoachim Protze jprotze \nKathleen Shea kshea21 \nKelly (KT) Thompson \nKelly (KT) Thompson Kelly Thompson \nKevin Brandstatter Kevin Brandstatter \nLuc Jaulmes Luc Jaulmes \nMario Melara Mario Melara \nMark Miller miller86 \nMassimiliano Culpo Massimiliano Culpo \nMassimiliano Culpo alalazo \nMayeul d'Avezac Mayeul d'Avezac \nMitchell Devlin Mitchell Devlin \nNicolas Richart Nicolas \nNicolas Richart Nicolas Richart \nPeter Scheibel scheibelp \nRobert D. French Robert D. French \nRobert D. French Robert.French \nRobert D. French robertdfrench \nSaravan Pantham Saravan Pantham \nSergey Kosukhin Sergey Kosukhin \nStephen Herbein Stephen Herbein \nTodd Gamblin George Todd Gamblin \nTodd Gamblin Todd Gamblin \nTom Scogland Tom Scogland \nTom Scogland Tom Scogland \nTzanio Kolev Tzanio \n" }, { "path": ".readthedocs.yml", "content": "version: 2\n\nbuild:\n os: \"ubuntu-24.04\"\n apt_packages:\n - graphviz\n - inkscape\n - xindy\n tools:\n python: \"3.14\"\n jobs:\n post_checkout:\n - git fetch --unshallow || true # get accurate \"Last updated on\" info\n\nsphinx:\n configuration: lib/spack/docs/conf.py\n fail_on_warning: true\n\nformats:\n - pdf\n\npython:\n install:\n - requirements: lib/spack/docs/requirements.txt\n\nsearch:\n ranking:\n _modules/*: -10\n spack_repo.*.html: -10\n spack_repo.html: -10\n spack.*.html: -10\n spack.html: -10\n command_index.html: 4\n advanced_topics.html: 5\n binary_caches.html: 5\n bootstrapping.html: 5\n build_settings.html: 5\n build_systems.html: 5\n build_systems/*.html: 5\n chain.html: 5\n config_yaml.html: 5\n configuring_compilers.html: 5\n containers.html: 5\n roles_and_responsibilities.html: 5\n contribution_guide.html: 5\n developer_guide.html: 5\n package_review_guide.html: 5\n env_vars_yaml.html: 5\n environments.html: 5\n extensions.html: 5\n features.html: 5\n getting_help.html: 5\n getting_started.html: 5\n gpu_configuration.html: 5\n include_yaml.html: 5\n installing_prerequisites.html: 5\n mirrors.html: 5\n module_file_support.html: 5\n package_api.html: 5\n package_fundamentals.html: 5\n packages_yaml.html: 5\n packaging_guide_advanced.html: 5\n packaging_guide_build.html: 5\n packaging_guide_creation.html: 5\n packaging_guide_testing.html: 5\n pipelines.html: 5\n replace_conda_homebrew.html: 5\n repositories.html: 5\n signing.html: 5\n spec_syntax.html: 5\n windows.html: 5\n frequently_asked_questions.html: 6\n" }, { "path": "CHANGELOG.md", "content": "# v1.1.1 (2026-01-14)\n\n## Usability and performance enhancements\n\n* solver: do a precheck for non-existing and deprecated versions #51555\n* improvements to solver performance (PRs 51591, 51605, 51612, 51625)\n* python 3.14 support (PRs 51686, 51687, 51688, 51689, 51663)\n* display when conditions with dependencies in spack info #51588\n* spack repo remove: allow removing from unspecified scope #51563\n* spack compiler info: show non-external compilers too #51718\n\n## Improvements to the experimental new installer\n\n* support forkserver #51788 (for python 3.14 support)\n* support --dirty, --keep-stage, and `skip patch` arguments #51558\n* implement --use-buildcache, --cache-only, --use-cache and --only arguments #51593\n* implement overwrite, keep_prefix #51622\n* implement --dont-restage #51623\n* fix logging #51787\n\n## Bugfixes\n\n* repo.py: support rhel 7 #51617\n* solver: match glibc constraints by hash #51559\n* buildache list: list the component prefix not the root #51635\n* solver: fix issue with conditional language dependencies #51692\n* repo.py: fix checking out commits #51695\n* spec parser: ensure toolchains are expanded to different objects #51731\n* RHEL7 git 1.8.3.1 fix #51779\n* RewireTask.complete: return value from \\_process\\_binary\\_cache\\_tarball #51825\n\n## Documentation\n\n* docs: fix default projections setting discrepancy #51640\n\n\n# v1.1.0 (2025-11-14)\n\n`v1.1.0` features major improvements to **compiler handling** and **configuration management**, a significant refactoring of **externals**, and exciting new **experimental features** like a console UI for parallel installations and concretization caching.\n\n## Major new features\n\n1. **Enhanced Compiler Control and Unmixing**\n\n * Compiler unmixing (#51135)\n * Propagated compiler preferences (#51383)\n\n In Spack v1.0, support for compilers as nodes made it much easier to mix compilers for the same language on different packages in a Spec. This increased flexibility, but did not offer options to constrain compiler selection when needed.\n\n * #51135 introduces the `concretizer:compiler_mixing` config option. When disabled, all specs in the \"root unification set\" (root specs and their transitive link/run deps) will be assigned a single compiler for each language. You can also specify a list of packages to be excepted from the restriction.\n\n * #51383 introduces the `%%` sigil in the spec syntax. While `%` specifies a direct dependency for a single node, `%%` specifies a dependency for that node and a preference for its transitive link/run dependencies (at the same priority as the `prefer` key in `packages.yaml` config).\n\n2. **Customizable configuration** (#51162)\n\n All configuration now stems from `$spack/etc/spack` and `$spack/etc/spack/defaults`, so the owner of a Spack instance can have full control over what configuration scopes exist.\n\n * Scopes included in configuration can be named, and the builtin `site`, `user`, `system`, etc. scopes are now defined in configuration rather than hard-coded.\n * `$spack/etc/spack/defaults` is the lowest priority.\n * `$spack/etc/spack` *includes* the other scopes at lower precedence than itself.\n * You can override with any scopes *except* the defaults with `include::`. e.g., `include::[]` in an environment allows you to ignore everything but defaults entirely.\n\n Here is `$spack/etc/spack/include.yaml`:\n\n ```yaml\n include:\n # user configuration scope\n - name: \"user\"\n path: \"~/.spack\"\n optional: true\n when: '\"SPACK_DISABLE_LOCAL_CONFIG\" not in env'\n\n # site configuration scope\n - name: \"site\"\n path: \"$spack/etc/spack/site\"\n optional: true\n\n # system configuration scope\n - name: \"system\"\n path: \"/etc/spack\"\n optional: true\n when: '\"SPACK_DISABLE_LOCAL_CONFIG\" not in env'\n ```\n\n NOTE: This change inverts the priority order of configuration in `$spack/etc/spack` and `~/.spack`.\n\n See the [configuration docs](https://spack.readthedocs.io/en/latest/configuration.html) and\n [include docs](https://spack.readthedocs.io/en/latest/include_yaml.html) for\n more information.\n\n3. **Git includes** (#51191)\n\n Configuration files can now be included directly from a **remote Git repository**. This allows for easier sharing and versioning of complex configurations across teams or projects. These entries accept the same syntax as remote repository configuration, and can likewise be conditional with `when:`.\n\n ```yaml\n include:\n - git: https://github.com/spack/spack-configs\n branch: main\n when: os == \"centos7\"\n paths:\n - USC/config/config.yaml\n - USC/config/packages.yaml\n ```\n\n See [the docs](https://spack.readthedocs.io/en/latest/include_yaml.html#git-repository-files) for details.\n\n4. **Externals Can Now Have Dependencies** (#51118)\n\n Externals are treated as concrete specs, so there is a 1:1 mapping between an entry in `packages.yaml` and any installed external spec (for a fixed repository).\n\n Their YAML specification has been extended to allow modeling dependencies of external specs. This might be quite useful to better capture e.g. ROCm installations that are already installed on a given system, or in similar cases.\n\n To be backward compatible with external specs specifying a compiler, for instance `mpich %gcc@9`, Spack will match the compiler specification to an existing external. It will fail when the specification is ambiguous, or if it does not match any other externals.\n\n\n## Experimental Features\n\n5. **New installer UI** (experimental, see #51434)\n\n New, experimental console UI for the Spack installer that allows:\n\n * Spack to show progress on multiple parallel processes concurrently;\n * Users to view logs for different installations independently; and\n * Spack to share a jobserver among multiple parallel builds.\n\n Demo: https://asciinema.org/a/755827\n\n Usage:\n\n * Run this to enable by default (and persist across runs):\n ```\n spack config add config:installer:new\n ```\n or use:\n ```\n spack -c config:installer:new install ...\n ```\n to try one run with the new UI.\n * The `-j` flag in spack install `-j ...` is all you need, it will build packages in parallel. There is no need to set `-p`; the installer spawns as many builds as it can and shares work by default.\n * Use `n` for next logs and `p/N` for previous logs\n * Use `v` to toggle between logs and overview\n * Use `q` or `Esc` to go from logs back to overview.\n * Use `/` to enter search mode: filters the overview as you type; press `Enter` to follow logs or `Esc` to exit search mode.\n\n > [!WARNING]\n > This feature is experimental because it is not feature-complete to match the existing installer. See the issue #51515 for a list of features that are not completed. Particularly note that the new installer locks the entire database, and other spack instances will not install concurrently while it is running.\n\n6. **Concretization Caching** (experimental, see #50905, #51448)\n\n Spack can cache concretization outputs for performance. With caching, Spack will still set up the concretization problem, but it can look up the solve result and avoid long solve times. This feature is currently off by default, but you can enable it with:\n\n ```\n spack config add concretizer:concretization_cache:enable:true\n ```\n\n > [!WARNING]\n > Currently there is a bug that the cache will return results that do not properly reflect changes in the `package_hash` (that is, changes in the `package.py` source code). We will enable caching by default in a future release, when this bug is fixed.\n\n## Potentially breaking changes\n* Configurable configuration changes the precedence of the `site` scope.\n * The `spack` scope (in `/etc/spack` within the Spack installation) is now the highest precedence scope\n * The `site` scope is now *lower* precedence than `spack` and `user`.\n * If you previously had configuration files in in `$spack/etc/spack`, they will take precedence over configuration in `~/.spack`. If you do not want that, move them to `$spack/etc/spack/site`.\n * See #51162 for details.\n* Fixed a bug with command-line and environment scope ordering. The environment scope could previously override custom command-line scopes. Now, the active environment is *always* lower precedence than any configuration scopes provided on the command line. (#51461)\n\n## Other notable improvements\n\n### Improved error messages\n* solver: catch invalid dependencies during concretization (#51176)\n* improved errors for requirements (#45800)\n\n### Performance Improvements\n* `spack mirror create --all` now runs in parallel (#50901)\n* `spack develop`: fast automatic reconcretization (#51140)\n* Don't spawn a process for `--fake` installs (#51491)\n* Use `gethostname` instead of `getfqdn` (#51481)\n* Check for `commit` variant only if not developing (#51507)\n* Concretization performance improvements (#51160, #51152, #51416)\n* spack diff: fix performance bug (#51270)\n\n### Concretizer improvements\n* concretizer: fix direct dep w/ virtuals issue (#51037)\n* solver: reduce items in edge optimizations (#51503)\n\n### UI and Commands\n* Managed environments can now be organized into folders (#50994)\n* `spack info` shows full info about conditional dependencies and can filter by spec. (#51137)\n* `spack help` is now reorganized and has color sections (#51484)\n* `spack clean --all` means all (no exception for bootstrap cache) (#50984)\n* `--variants-by-name` no longer used (#51450)\n* `spack env create`: allow creation from env or env dir (#51433)\n\n## Notable Bugfixes\n* mirror: clean up stage when retrying (#43519)\n* Many smaller concretization fixes (#51361, #51355, #51341, #51347, #51282, #51190, #51226, #51065, #51064, #51074)\n* Bugfix for failed multi-node parallel installations (#50933)\n\n## Spack community stats\n\n* 1,681 commits\n* 8,611 packages in the 2025.11.0 release, 112 new since 2025.07.0\n* 276 people contributed to this release\n* 265 committers to packages\n* 31 committers to core\n\nSee the [2025.11.0 release](https://github.com/spack/spack-packages/releases/tag/v2025.11.0) of [spack-packages](https://github.com/spack/spack-packages/) for more details.\n\n\n# v1.0.4 (2026-02-23)\n\n## Bug fixes\n\n* Concretizer bugfixes:\n * solver: remove a special case for provider weighting #51347\n * solver: improve timeout handling and add Ctrl-C interrupt safety #51341\n * solver: simplify interrupt/timeout logic #51349\n* Repo management bugfixes:\n * repo.py: support rhel 7 #51617\n * repo.py: fix checking out commits #51695\n * git: pull_checkout_branch RHEL7 git 1.8.3.1 fix #51779\n * git: fix locking issue in pull_checkout_branch #51854\n * spack repo remove: allow removing from unspecified scope #51563\n* build_environment.py: Prevent deadlock on install process join #51429\n* Fix typo in untrack_env #51554\n* audit.py: fix re.sub(..., N) positional count arg #51735\n\n## Enhancements\n\n* Support Macos Tahoe (#51373, #51394, #51479)\n* Support for Python 3.14, except for t-strings (#51686, #51687, #51688, #51697, #51663)\n* spack info: show conditional dependencies and licenses; allow filtering #51137\n* Spack fetch less likely to fail due to AI download protections #51496\n* config: relax concurrent_packages to minimum 0 #51840\n * This avoids forward-incompatibility with Spack v1.2\n* Documentation improvements (#51315, #51640)\n\n\n# v1.0.3 (2026-02-20)\n\nSkipped due to a failure in the release process.\n\n\n# v1.0.2 (2025-09-11)\n\n## Bug Fixes\n\n* `spack config edit` can now open malformed YAML files. (#51088)\n* `spack edit -b` supports specifying the repository path or its namespace. (#51084)\n* `spack repo list` escapes the color code for paths that contain `@g`. (#51178)\n* Fixed various issues on the solver:\n * Improved the error message when an invalid dependency is specified in the input. (#51176)\n * Build the preferred compiler with itself by default. (#51201)\n * Fixed a performance regression when using `unify:when_possible`. (#51226)\n * Fixed an issue with strong preferences, when provider details are given. (#51263)\n * Fixed an issue when specifying flags on a package that appears multiple times in the DAG. (#51218)\n* Fixed a regression for `zsh` in `spack env activate --prompt`. (#51258)\n* Fix a few cases where the `when` context manager was not dealing with direct dependencies correctly. (#51259)\n* Various fixes to string representations of specs. (#51207)\n\n## Enhancements\n\n* Various improvements to the documentation (#51145, #51151, #51147, #51181, #51172, #51188, #51195)\n* Greatly improve the performance of `spack diff`. (#51270)\n* `spack solve` highlights optimization weights in a more intuitive way. (#51198)\n\n# v1.0.1 (2025-08-11)\n\n## Bug Fixes\n\n* Ensure forward compatibility of package hashing with the upcoming Python 3.14 release. (#51042)\n* The `spack diff` command now shows differences in runtime dependencies (e.g., `gcc-runtime`, `glibc`), which were previously hidden. (#51076)\n* Fix a regression where the solver would mishandle a compiler that was required as both a build and a link dependency. (#51074)\n* Resolved issues with selecting external packages that have a specific compiler specified. (#51064)\n* Fix a bug where the concretizer would compute solution scores incorrectly when the package does not depend on a compiler. (#51037)\n* The solver now correctly evaluates and respects package requirements that specify a hash. (#51065)\n* Fix an issue where sparse checkouts for different packages could overwrite each other in a source cache or mirror. (#51080)\n* Prevent `spack repo add` from overwriting the default branch when initially cloning a repository. (#51105)\n* Add exception handling for bad URLs when fetching git provenance information. (#51022)\n* Spack no longer conflates git warning messages with command output. (#51045)\n* Fix an issue with non-path-based package repositories in environments. (#51055)\n* Spack now validates the terminal size and will fall back to `LINES` and `COLUMNS` environment variables if detection fails. (#51090)\n* Fix an issue where the package's fetcher was not being set correctly. (#51108)\n* Ensure `spack tutorial` clones Spack v1.0 instead of v0.23. (#51091)\n\n## Enhancements\n\n* Various improvements to the documentation (#51014, #51033, #51039, #51049, #51066, #51073, #51079, #51082, #51083, #51086, #51126, #51131, #51132, #51025)\n\n\n# v1.0.0 (2025-07-20)\n\n`v1.0.0` is a major feature release and a significant milestone. It introduces compiler\ndependencies, a foundational change that has been in development for almost seven years,\nand the project's first stable package API.\n\nIf you are interested in more information, you can find more details on the road to\nv1.0, as well as its features, in talks from the 2025 Spack User Meeting. For example:\n* [State of the Spack Community](https://www.youtube.com/watch?v=4rInmUfuiZQ&list=PLRKq_yxxHw29-JcpG2CZ-xKK2U8Hw8O1t&index=2)\n* [Spack v1.0 overview](https://www.youtube.com/watch?v=nFksqSDNwQA&list=PLRKq_yxxHw29-JcpG2CZ-xKK2U8Hw8O1t&index=4)\n\nIntroducing some of these features required us to make breaking changes. In most cases,\nwe've also provided tools (in the form of Spack commands) that you can use to\nautomatically migrate your packages and configuration.\n\n## Overview\n\n- [Overview](#overview)\n- [Stable Package API](#stable-package-api)\n - [Separate Package Repository](#separate-package-repository)\n - [Updating and Pinning Packages](#updating-and-pinning-packages)\n - [Breaking changes related to package repositories](#breaking-changes-related-to-package-repositories)\n - [Migrating to the new package API](#migrating-to-the-new-package-api)\n- [Compiler dependencies](#compiler-dependencies)\n - [Compiler configuration](#compiler-configuration)\n - [Languages are virtual dependencies](#languages-are-virtual-dependencies)\n - [The meaning of % has changed](#the-meaning-of-%25-has-changed)\n - [Virtual assignment syntax](#virtual-assignment-syntax)\n - [Toolchains](#toolchains)\n - [Ordering of variants and compilers now matters](#ordering-of-variants-and-compilers-now-matters)\n- [Additional Major Features](#additional-major-features)\n - [Concurrent Package Builds](#concurrent-package-builds)\n - [Content-addressed build caches](#content-addressed-build-caches)\n - [Better provenance and mirroring for git](#better-provenance-and-mirroring-for-git)\n - [Environment variables in environments](#environment-variables-in-environments)\n - [Better include functionality](#better-include-functionality)\n- [New commands and options](#new-commands-and-options)\n- [Notable refactors](#notable-refactors)\n- [Documentation](#documentation)\n- [Notable Bugfixes](#notable-bugfixes)\n- [Additional deprecations, removals, and breaking changes](#additional-deprecations-removals-and-breaking-changes)\n- [Spack community stats](#spack-community-stats)\n\n## Stable Package API\n\nIn Spack `v1.0`, the package repository is separate from the Spack tool, giving you more\ncontrol over the versioning of package recipes. There is also a stable\n[Package API](https://spack.readthedocs.io/en/latest/package_api.html) that is versioned\nseparately from Spack.\n\nThis release of Spack supports package API from `v1.0` up to `v2.2`. The older `v1.0`\npackage API is deprecated and may be removed in a future release, but we are\nguaranteeing that any Spack `v1.x` release will be backward compatible with Package API\n`v.2.x` -- i.e., it can execute code from the packages in *this* Spack release.\n\nSee the\n[Package API Documentation](https://spack.readthedocs.io/en/latest/package_api.html) for\nfull details on package versioning and compatibility. The high level details are:\n\n 1. The `spack.package` Python module defines the Package API;\n 2. The Package API *minor version* is incremented when new functions or classes are exported from `spack.package`; and\n 3. The major version is incremented when functions or classes are removed or have breaking changes to their signatures (a rare occurrence).\n\nThis independent versioning allows package authors to utilize new Spack features without\nwaiting for a new Spack release. Older Spack packages (API `v1.0`) may import code from\noutside of `spack.package`, e.g., from `spack.*` or `llnl.util.*`. This is deprecated\nand *not* included in the API guarantee. We will remove support for these packages in a\nfuture Spack release.\n\n### Separate Package Repository\n\nThe Spack `builtin` package repository no longer lives in the Spack git repository. You\ncan find it here:\n\n* https://github.com/spack/spack-packages\n\nSpack clones the package repository automatically when you first run, so you do not have\nto manage this manually. By default, Spack version `v1.0` uses the `v2025.07` release of\n`spack-packages`. You can find out more about it by looking at the\n[package releases](https://github.com/spack/spack-packages/releases).\n\nDownloaded package repos are stored by default within `~/.spack`, but the fetch\ndestination can be configured. (#50650). If you want your package repository to live\nsomewhere else, run, e.g.:\n\n```\nspack repo set --destination ~/spack-packages builtin\n```\n\nYou can also configure your *own* package repositories to be fetched automatically from\ngit urls, just as you can with `builtin`. See the\n[repository configuration docs](https://spack.readthedocs.io/en/latest/repositories.html)\nfor details.\n\n### Updating and Pinning Packages\n\nYou can tell Spack to update the core package repository from a branch. For example, on\n`develop` or on a release, you can run commands like:\n\n ```shell\n # pull the latest packages\n spack repo update\n ```\nor\n\n ```shell\n # check out a specific commit of the spack-packages repo\n spack repo update --commit 2bf4ab9585c8d483cc8581d65912703d3f020393 builtin\n ```\n\nwhich will set up your configuration like this:\n\n ```yaml\n repos:\n builtin:\n git: \"https://github.com/spack/spack-packages.git\"\n commit: 2bf4ab9585c8d483cc8581d65912703d3f020393\n ```\n\nYou can use this within an environment to pin a specific version of its package files.\nSee the\n[repository configuration docs](https://spack.readthedocs.io/en/latest/repositories.html)\nfor more details (#50868, #50997, #51021).\n\n### Breaking changes related to package repositories\n\n1. The builtin repo now lives in `var/spack/repos/spack_repo/builtin` instead of\n `var/spack/repos/builtin`, and it has a new layout, which you can learn about in the\n [repo docs](https://spack.readthedocs.io/en/latest/repositories.html).\n\n2. The module `spack.package` no longer exports the following symbols, mostly related to\n build systems: `AspellDictPackage`, `AutotoolsPackage`, `BundlePackage`,\n `CachedCMakePackage`, `cmake_cache_filepath`, `cmake_cache_option`,\n `cmake_cache_path`, `cmake_cache_string`, `CargoPackage`, `CMakePackage`,\n `generator`, `CompilerPackage`, `CudaPackage`, `Package`, `GNUMirrorPackage`,\n `GoPackage`, `IntelPackage`, `IntelOneApiLibraryPackageWithSdk`,\n `IntelOneApiLibraryPackage`, `IntelOneApiStaticLibraryList`, `IntelOneApiPackage`,\n `INTEL_MATH_LIBRARIES`, `LuaPackage`, `MakefilePackage`, `MavenPackage`,\n `MesonPackage`, `MSBuildPackage`, `NMakePackage`, `OctavePackage`, `PerlPackage`,\n `PythonExtension`, `PythonPackage`, `QMakePackage`, `RacketPackage`, `RPackage`,\n `ROCmPackage`, `RubyPackage`, `SConsPackage`, `SIPPackage`, `SourceforgePackage`,\n `SourcewarePackage`, `WafPackage`, `XorgPackage`\n\n These are now part of the `builtin` package repository, not part of core spack or its\n package API. When using repositories with package API `v2.0` and higher, *you must\n explicitly import these package classes* from the appropriate module in\n `spack_repo.builtin.build_systems` (see #50452 for more).\n\n e.g., for `CMakePackage`, you would write:\n\n ```python\n from spack_repo.builtin.build_systems.cmake import CMakePackage\n ```\n\n Note that `GenericBuilder` and `Package` *are* part of the core package API. They are\n currently re-exported from `spack_repo.builtin.build_systems.generic` for backward\n compatibility but may be removed from the package repo. You should prefer to import\n them from `spack.package`.\n\n The original names will still work for old-style (`v1.0`) package repositories but\n *not* in `v2.0` package repositories. Note that this means that the API stability\n promise does *not* include old-style package repositories. They are deprecated and\n will be removed in a future version. So, you should update as soon as you can.\n\n3. Package directory names within `v2.0` repositories are now valid Python modules\n\n | Old | New | Description |\n |-----------------------|-----------------------|-------------------------------------|\n | `py-numpy/package.py` | `py_numpy/package.py` | hyphen is replaced by underscore. |\n | `7zip/package.py` | `_7zip/package.py` | leading digits now preceded by _ |\n | `pass/package.py` | `_pass/package.py` | Python reserved words preceded by _ |\n\n\n4. Spack has historically injected `import` statements into package recipes, so there\n was no need to use `from spack.package import *` (though we have included it in\n `builtin` packages for years. `from spack.package import *` (or more specific\n imports) will be necessary in packages. The magic we added in the early days of Spack\n was causing IDEs, code editors, and other tools not to be able to understand Spack\n packages. Now they use standard Python import semantics and should be compatible with\n modern Python tooling. This change was also necessary to support Python 3.13. (see\n #47947 for more details).\n\n### Migrating to the new package API\n\nSupport will remain in place for the old repository layout for *at least a year*, so\nthat you can continue to use old-style repos in conjunction with earlier versions. If\nyou have custom repositories that need to migrate to the new layout, you can upgrade\nthem to package API `v2.x` by running:\n\n```\nspack repo migrate\n```\n\nThis will make the following changes to your repository:\n\n 1. If you used to import from `spack.pkg.builtin` in Python, you now need to import\n from `spack_repo.builtin` instead:\n\n ```python\n # OLD: no longer supported\n from spack.pkg.builtin.my_pkg import MyPackage\n\n # NEW: spack_repo is a Python namespace package\n from spack_repo.builtin.packages.my_pkg.package import MyPackage\n ```\n 2. Normalized directory names for packages\n 3. New-style `spack.package` imports\n\nSee #50507, #50579, and #50594 for more.\n\n## Compiler dependencies\n\nPrior to `v1.0`, compilers in Spack were attributes on nodes in the spec graph, with a\nname and a version (e.g., `gcc@12.0.0`). In `v1.0` compilers are packages like any other\npackage in Spack (see #45189). This means that they can have variants, targets, and\nother attributes that regular nodes have.\n\nHere, we list the major changes that users should be aware of for this new model.\n\n### Compiler configuration\n\nIn Spack `v1.0`, `compilers.yaml` is deprecated. `compilers.yaml` is still read by\nSpack, if present. We will continue to support this for at least a year, but we may\nremove it after that. Users are encouraged to migrate their configuration to use\n`packages.yaml` instead.\n\nOld style `compilers.yaml` specification:\n\n```yaml\ncompilers:\n - compiler:\n spec: gcc@12.3.1\n paths:\n c: /usr/bin/gcc\n cxx: /usr/bin/g++\n fc: /usr/bin/gfortran\n modules: [...]\n```\n\nNew style `packages.yaml` compiler specification:\n\n```yaml\npackages:\n gcc:\n externals:\n - spec: gcc@12.3.1+binutils\n prefix: /usr\n extra_attributes:\n compilers:\n c: /usr/bin/gcc\n cxx: /usr/bin/g++\n fc: /usr/bin/gfortran\n modules: [...]\n```\n\nSee\n[Configuring Compilers](https://spack.readthedocs.io/en/latest/configuring_compilers.html)\nfor more details.\n\n\n### Languages are virtual dependencies\n\nPackages that need a C, C++, or Fortran compiler now **must** depend on `c`, `cxx`, or\n`fortran` as a build dependency, e.g.:\n\n```python\nclass MyPackage(Package):\n depends_on(\"c\", type=\"build\")\n depends_on(\"cxx\", type=\"build\")\n depends_on(\"fortran\", type=\"build\")\n```\n\nHistorically, Spack assumed that *every* package was compiled with C, C++, and Fortran.\nIn Spack `v1.0`, we allow packages to simply not have a compiler if they do not need\none. For example, pure Python packages would not depend on any of these, and you should\nnot add these dependencies to packages that do not need them.\n\n[Spack `v0.23`](https://github.com/spack/spack/releases/tag/v0.23.0) introduced language\nvirtual dependencies, and we have back-ported them to `0.21.3` and `v0.22.2`. In pre-1.0\nSpack releases, these are a no-op. They are present so that language dependencies do not\ncause an error. This allows you to more easily use older Spack versions together with\n`v1.0`.\n\nSee #45217 for more details.\n\n### The meaning of `%` has changed\n\nIn Spack `v0.x`, `%` specified a compiler with a name and an optional version. In Spack\n`v1.0`, it simply means \"direct dependency\". It is similar to the caret `^`, which means\n\"direct *or* transitive dependency\".\n\nUnlike `^`, which specifies a dependency that needs to be unified for the whole graph,\n`%` can specify direct dependencies of particular nodes. This means you can use it to\nmix and match compilers, or `cmake` versions, or any other package for which *multiple*\nversions of the same build dependency are needed in the same graph. For example, in this\nspec:\n\n```\nfoo ^hdf5 %cmake@3.1.2 ^zlib-ng %cmake@3.2.4\n```\n\n`hdf5` and `zlib-ng` are both transitive dependencies of `foo`, but `hdf5` will be built\nwith `cmake@3.1.2` and `zlib-ng` will be built with `%cmake@3.2.4`. This is similar to\nmixing compilers, but you can now use `%` with other types of build dependencies, as\nwell. You can have multiple versions of packages in the same graph, as long as they are\npurely build dependencies.\n\n### Virtual assignment syntax\n\nYou can still specify compilers with `foo %gcc`, in which case `gcc` will be used to\nsatisfy any `c`, `cxx`, and `fortran` dependencies of `foo`, but you can also be\nspecific about the compiler that should be used for each language. To mix, e.g., `clang`\nand `gfortran`, you can now use *virtual assignment* like so:\n\n```console\nspack install foo %c,cxx=gcc %fortran=gfortran\n```\n\nThis says to use `gcc` for `c` and `cxx`, and `gfortran` for `fortran`.\nIt is functionally equivalent to the already supported edge attribute syntax:\n\n```\nspack install foo %[virtuals=c,cxx] gcc %[virtuals=fortran] gfortran\n```\n\nBut, virtual assignment is more legible. We use it as the default formatting for virtual\nedge attributes, and we print it in the output of `spack spec`, `spack find`, etc. For\nexample:\n\n```console\n> spack spec zlib\n - zlib@1.3.1+optimize+pic+shared build_system=makefile arch=darwin-sequoia-m1 %c,cxx=apple-clang@16.0.0\n[e] ^apple-clang@16.0.0 build_system=bundle arch=darwin-sequoia-m1\n - ^compiler-wrapper@1.0 build_system=generic arch=darwin-sequoia-m1\n[+] ^gmake@4.4.1~guile build_system=generic arch=darwin-sequoia-m1 %c=apple-clang@16.0.0\n[+] ^compiler-wrapper@1.0 build_system=generic arch=darwin-sequoia-m1\n```\n\nYou can see above that only `zlib` and `gmake` are compiled, and `gmake` uses only `c`.\nThe other nodes are either external, and we cannot detect the compiler (`apple-clang`)\nor they are not compiled (`compiler-wrapper` is a shell script).\n\n### Toolchains\n\nSpack now has a concept of a \"toolchain\", which can be configured in `toolchains.yaml`.\nA toolchain is an alias for common dependencies, flags, and other spec properties that\nyou can attach to a node in a graph with `%`.\n\nToolchains are versatile and composable as they are simply aliases for regular specs.\nYou can use them to represent mixed compiler combinations, compiler/MPI/numerical\nlibrary groups, particular runtime libraries, and flags -- all to be applied together.\nThis allows you to do with compiler dependencies what we used to do with\n`compilers.yaml`, and more.\n\nExample mixed clang/gfortran toolchain:\n\n```yaml\ntoolchains:\n clang_gfortran:\n - spec: %c=clang\n when: %c\n - spec: %cxx=clang\n when: %cxx\n - spec: %fortran=gcc\n when: %fortran\n - spec: cflags=\"-O3 -g\"\n - spec: cxxflags=\"-O3 -g\"\n - spec: fflags=\"-O3 -g\"\n```\n\nThis enables you to write `spack install foo %clang_gfortran`, and Spack will resolve\nthe `%clang_gfortran` toolchain to include the dependencies and flags listed in\n`toolchains.yaml`.\n\nYou could also couple the intel compilers with `mvapich2` like so:\n\n```yaml\ntoolchains:\n intel_mvapich2:\n - spec: %c=intel-oneapi-compilers @2025.1.1\n when: %c\n - spec: %cxx=intel-oneapi-compilers @2025.1.1\n when: %cxx\n - spec: %fortran=intel-oneapi-compilers @2025.1.1\n when: %fortran\n - spec: %mpi=mvapich2 @2.3.7-1 +cuda\n when: %mpi\n```\n\nThe `when:` conditions here ensure that toolchain constraints are only applied when\nneeded. See the\n[toolchains documentation](https://spack.readthedocs.io/en/latest/advanced_topics.html#defining-and-using-toolchains)\nor #50481 for details.\n\n### Ordering of variants and compilers now matters\n\nIn Spack `v0.x`, these two specs parse the same:\n\n```\npkg %gcc +foo\npkg +foo %gcc\n```\n\nThe `+foo` variant applies to `pkg` in either case. In Spack `v1.0`, there is a breaking\nchange, and `+foo` in `pkg %gcc +foo` now applies to `gcc`, since `gcc` is a normal\npackage. This ensures we have the following symmetry:\n\n```\npkg +foo %dep +bar # `pkg +foo` depends on `dep +bar` directly\npkg +foo ^dep +bar # `pkg +foo` depends on `dep +bar` directly or transitively\n```\n\nIn Spack `v1.0` you may get errors at concretization time if `+foo` is not a variant of\n`gcc` in specs like`%pkg %gcc +foo`.\n\nYou can use the `spack style --spec-strings` command to update `package.py` files,\n`spack.yaml` files:\n\n ```shell\n # dry run\n spack style --spec-strings $(git ls-files) # if you have a git repo\n spack style --spec-strings spack.yaml # environments\n ```\n\n ```shell\n # use --fix to perform the changes listed by the dry run\n spack style --fix --spec-strings $(git ls-files)\n spack style --fix --spec-strings spack.yaml\n ```\n\n See #49808, #49438, #49439 for details.\n\n## Additional Major Features\n\n### Concurrent Package Builds\n\nThis release has completely reworked Spack's build scheduler, and it adds a `-p`/\n`--concurrent-packages` argument to `spack install`, which can greatly accelerate builds\nwith many packages. You can use it in combination with `spack install -j`. For example,\nthis command:\n\n```\nspack install -p 4 -j 16\n```\n\nruns up to 4 package builds at once, each with up to 16 make jobs. The default for\n`--concurrent-packages` is currently 1, so you must enable this feature yourself, either\non the command line or by setting `config:concurrent_packages` (#50856):\n\n```yaml\nconfig:\n concurrent_packages: 1\n```\n\nAs before, you can run `spack install` on multiple nodes in a cluster, if the filesystem\nwhere Spack's `install_tree` is located supports locking.\n\nWe will make concurrent package builds the default in `1.1`, when we plan to include\nsupport for `gmake`'s jobserver protocol and for line-synced output. Currently, setting\n`-p` higher than 1 can make Spack's output difficult to read.\n\n### Content-addressed build caches\n\nSpack `v1.0` changes the format of build caches to address a number of scaling and\nconsistency issues with our old (aging) buildcache layout. The new buildcache format is\ncontent-addressed and enables us to make many operations atomic (and therefore safer).\nIt is also more extensible than the old buildcache format and can enable features like\nsplit debug info and different signing methods in the future. See #48713 for more\ndetails.\n\nSpack `v1.0` can still read, but not write to, the old build caches. The new build cache\nformat is *not* backward compatible with the old format, *but* you can have a new build\ncache and an old build cache coexist beside each other. If you push to an old build\ncache, new binaries will start to show up in the new format.\n\nYou can migrate an old buildcache to the new format using the `spack buildcache migrate`\ncommand. It is nondestructive and can migrate an old build cache to a new one in-place.\nThat is, it creates the new buildcache within the same directory, alongside the old\nbuildcache.\n\nAs with other major changes, the old buildcache format is deprecated in `v1.0`, but will\nnot be removed for at least a year.\n\n### Better provenance and mirroring for git\n\nSpack now resolves and preserves the commit of any git-based version at concretization\ntime, storing the precise commit built on the Spec in a reserved `commit` variant. This\nallows us to better reproduce git builds. See #48702 for details.\n\nHistorically, Spack has only stored the ref name, e.g. the branch or tag, for git\nversions that did not already contain full commits. Now we can know exactly what was\nbuilt regardless of how it was fetched.\n\nAs a consequence of this change, mirroring git repositories is also more robust. See\n#50604, #50906 for details.\n\n### Environment variables in environments\n\nYou can now specify environment variables in your environment that should be set on\nactivation (and unset on deactivation):\n\n```yaml\nspack:\n specs:\n - cmake%gcc\n env_vars:\n set:\n MY_FAVORITE_VARIABLE: \"TRUE\"\n```\n\nThe syntax allows the same modifications that are allowed for modules: `set:`, `unset:`,\n`prepend_path:`, `append_path:`, etc.\n\nSee [the docs](https://spack.readthedocs.io/en/latest/env_vars_yaml.html or #47587 for more.\n\n### Better include functionality\n\nSpack allows you to include local or remote configuration files through `include.yaml`,\nand includes can be optional (i.e. include them only if they exist) or conditional (only\ninclude them under certain conditions:\n\n```yaml\nspack:\n include:\n - /path/to/a/required/config.yaml\n - path: /path/to/$os/$target/config\n optional: true\n - path: /path/to/os-specific/config-dir\n when: os == \"ventura\"\n```\n\nYou can use this in an environment, or in an `include.yaml` in an existing configuration\nscope. Included configuration files are required *unless* they are explicitly optional\nor the entry's condition evaluates to `false`. Optional includes are specified with the\n`optional` clause and conditional ones with the ``when`` clause.\n\nConditionals use the same syntax as\n[spec list references](https://spack.readthedocs.io/en/latest/environments.html#spec-list-references)\nThe [docs on `include.yaml`](https://spack.readthedocs.io/en/latest/include_yaml.html)\nhave more information. You can also look at #48784.\n\n\n## New commands and options\n* `spack repo update` will pull the latest packages (#50868, #50997)\n* `spack style --spec-strings` fixes old configuration file and packages (#49485)\n* `spack repo migrate`: migrates old repositories to the new layout (#50507)\n* `spack ci` no longer has a `--keep-stage` flag (#49467)\n* The new `spack config scopes` subcommand will list active configuration scopes (#41455, #50703)\n* `spack cd --repo ` (#50845)\n* `spack location --repo ` (#50845)\n* `--force` is now a common argument for all commands that do concretization (#48838)\n\n## Notable refactors\n\n* All of Spack is now in one top-level `spack` Python package\n * The `spack_installable` package is gone as it's no longer needed (#50996)\n * The top-level `llnl` package has been moved to `spack.llnl` and will likely be\n refactored more later (#50989)\n * Vendored dependencies that were previously in `_vendoring` are now in `spack.vendor` (#51005)\n\n* Increased determinism when generating inputs for the ASP solver, leading to more\n consistent concretization results (#49471)\n\n* Added fast, stable spec comparison, which also increases determinism of concretizer\n inputs, and more consistent results (#50625)\n\n* Test deps are now part of the DAG hash, so builds with tests enabled will (correctly)\n have different hashes from builds without tests enabled (#48936)\n\n* `spack spec` in an environment or on the command line will show unified output with\n the specs provided as roots (#47574)\n\n* users can now set a timeout in `concretizer.yaml` in case they frequently hit long\n solves (#47661)\n\n* GoPackage: respect `-j`` concurrency (#48421)\n\n* We are using static analysis to speed up concretization (#48729)\n\n## Documentation\n\nWe have overhauled a number of sections of the documentation.\n\n* The basics section of the documentation has been reorganized and updated (#50932)\n\n* The [packaging guide](https://spack.readthedocs.io/en/latest/packaging_guide_creation.html)\n has been rewritten and broken into four separate, logically ordered sections (#50884).\n\n* As mentioned above the entire\n [`spack.package` API](https://spack.readthedocs.io/en/latest/package_api.html) has\n been documented and consolidated to one package (#51010)\n\n## Notable Bugfixes\n\n* A race that would cause timeouts in certain parallel builds has been fixed. Every\n build now stages its own patches and cannot fight over them (causing a timeout) with\n other builds (#50697)\n* The `command_line` scope is now *always* the top level. Previously environments could\n override command line settings (#48255)\n* `setup-env.csh` is now hardened to avoid conflicts with user aliases (#49670)\n\n## Additional deprecations, removals, and breaking changes\n\n1. `spec[\"pkg\"]` searches only direct dependencies and transitive link/run dependencies,\n ordered by depth. This avoids situations where we pick up unwanted deps of build/test\n deps. To reach those, you need to do `spec[\"build_dep\"][\"pkg\"]` explicitly (#49016).\n\n2. `spec[\"mpi\"]` no longer works to refer to `spec` itself on specs like `openmpi` and\n `mpich` that could provide `mpi`. We only find `\"mpi\"` if it is provided by some\n dependency (see #48984).\n\n3. We have removed some long-standing internal API methods on `spack.spec.Spec` so that\n we can decouple internal modules in the Spack code. `spack.spec` was including too\n many different parts of Spack.\n * `Spec.concretize()` and `Spec.concretized()` have been removed. Use\n `spack.concretize.concretize_one(spec)` instead (#47971, #47978)\n * `Spec.is_virtual`` is now spack.repo.PATH.is_virtual (#48986)\n * `Spec.virtual_dependencies` has been removed (#49079)\n\n4. #50603: Platform config scopes are now opt-in. If you want to use subdirectories like\n `darwin` or `linux` in your scopes, you'll need to include them explicitly in an\n `include.yaml` or in your `spack.yaml` file, like so:\n\n ```yaml\n include:\n - include: \"${platform}\"\n optional: true\n ```\n\n5. #48488, #48502: buildcache entries created with Spack 0.19 and older using `spack\n buildcache create --rel` will no longer be relocated upon install. These old binaries\n should continue to work, except when they are installed with different\n `config:install_tree:projections` compared to what they were built with. Similarly,\n buildcache entries created with Spack 0.15 and older that contain long shebang lines\n wrapped with sbang will no longer be relocated.\n\n6. #50462: the `package.py` globals `std_cmake_args`, `std_pip_args`, `std_meson_args`\n were removed. They were deprecated in Spack 0.23. Use `CMakeBuilder.std_args(pkg)`,\n `PythonPipBuilder.std_args(pkg)` and `MesonBuilder.std_args(pkg)` instead.\n\n7. #50605, #50616: If you were using `update_external_dependencies()` in your private\n packages, note that it is going away in 1.0 to get it out of the package API. It is\n instead being moved into the concretizer, where it can change in the future, when we\n have a better way to deal with dependencies of externals, without breaking the\n package API. We suspect that nobody was doing this, but it's technically a breaking\n change.\n\n8. #48838: Two breaking command changes:\n * `spack install` no longer has a `-f` / `--file` option --\n write `spack install ./path/to/spec.json` instead.\n * `spack mirror create` no longer has a short `-f` option --\n use `spack mirror create --file` instead.\n\n9. We no longer support the PGI compilers. They have been replaced by `nvhpc` (#47195)\n\n10. Python 3.8 is deprecated in the Python package, as it is EOL (#46913)\n\n11. The `target=fe` / `target=frontend` and `target=be` / `target=backend` targets from\n Spack's orignal compilation model for cross-compiled Cray and BlueGene systems are\n now deprecated (#47756)\n\n## Spack community stats\n\n* 2,276 commits updated package recipes\n* 8,499 total packages, 214 new since v0.23.0\n* 372 people contributed to this release\n* 363 committers to packages\n* 63 committers to core\n\n\n# v0.23.1 (2025-02-19)\n\n## Bugfixes\n- Fix a correctness issue of `ArchSpec.intersects` (#48741)\n- Make `extra_attributes` order independent in Spec hashing (#48615, #48854)\n- Fix issue where system proxy settings were not respected in OCI build caches (#48783)\n- Fix an issue where the `--test` concretizer flag was not forwarded correctly (#48417)\n- Fix an issue where `codesign` and `install_name_tool` would not preserve hardlinks on\n Darwin (#47808)\n- Fix an issue on Darwin where codesign would run on unmodified binaries (#48568)\n- Patch configure scripts generated with libtool < 2.5.4, to avoid redundant flags when\n creating shared libraries on Darwin (#48671)\n- Fix issue related to mirror URL paths on Windows (#47898)\n- Esnure proper UTF-8 encoding/decoding in logging (#48005)\n- Fix issues related to `filter_file` (#48038, #48108)\n- Fix issue related to creating bootstrap source mirrors (#48235)\n- Fix issue where command line config arguments were not always top level (#48255)\n- Fix an incorrect typehint of `concretized()` (#48504)\n- Improve mention of next Spack version in warning (#47887)\n- Tests: fix forward compatibility with Python 3.13 (#48209)\n- Docs: encourage use of `--oci-username-variable` and `--oci-password-variable` (#48189)\n- Docs: ensure Getting Started has bootstrap list output in correct place (#48281)\n- CI: allow GitHub actions to run on forks of Spack with different project name (#48041)\n- CI: make unit tests work on Ubuntu 24.04 (#48151)\n- CI: re-enable cray pipelines (#47697)\n\n## Package updates\n- `qt-base`: fix rpath for dependents (#47424)\n- `gdk-pixbuf`: fix outdated URL (#47825)\n\n\n# v0.23.0 (2024-11-13)\n\n`v0.23.0` is a major feature release.\n\nWe are planning to make this the last major release before Spack `v1.0`\nin June 2025. Alongside `v0.23`, we will be making pre-releases (alpha,\nbeta, etc.) of `v1.0`, and we encourage users to try them and send us\nfeedback, either on GitHub or on Slack. You can track the road to\n`v1.0` here:\n\n * https://github.com/spack/spack/releases\n * https://github.com/spack/spack/discussions/30634\n\n## Features in this Release\n\n1. **Language virtuals**\n\n Your packages can now explicitly depend on the languages they require.\n Historically, Spack has considered C, C++, and Fortran compiler\n dependencies to be implicit. In `v0.23`, you should ensure that\n new packages add relevant C, C++, and Fortran dependencies like this:\n\n ```python\n depends_on(\"c\", type=\"build\")\n depends_on(\"cxx\", type=\"build\")\n depends_on(\"fortran\", type=\"build\")\n ```\n\n We encourage you to add these annotations to your packages now, to prepare\n for Spack `v1.0.0`. In `v1.0.0`, these annotations will be necessary for\n your package to use C, C++, and Fortran compilers. Note that you should\n *not* add language dependencies to packages that don't need them, e.g.,\n pure python packages.\n\n We have already auto-generated these dependencies for packages in the\n `builtin` repository (see #45217), based on the types of source files\n present in each package's source code. We *may* have added too many or too\n few language dependencies, so please submit pull requests to correct\n packages if you find that the language dependencies are incorrect.\n\n Note that we have also backported support for these dependencies to\n `v0.21.3` and `v0.22.2`, to make all of them forward-compatible with\n `v0.23`. This should allow you to move easily between older and newer Spack\n releases without breaking your packages.\n\n2. **Spec splicing**\n\n We are working to make binary installation more seamless in Spack. `v0.23`\n introduces \"splicing\", which allows users to deploy binaries using local,\n optimized versions of a binary interface, even if they were not built with\n that interface. For example, this would allow you to build binaries in the\n cloud using `mpich` and install them on a system using a local, optimized\n version of `mvapich2` *without rebuilding*. Spack preserves full provenance\n for the installed packages and knows that they were built one way but\n deployed another.\n\n Our intent is to leverage this across many key HPC binary packages,\n e.g. MPI, CUDA, ROCm, and libfabric.\n\n Fundamentally, splicing allows Spack to redeploy an existing spec with\n different dependencies than how it was built. There are two interfaces to\n splicing.\n\n a. Explicit Splicing\n\n #39136 introduced the explicit splicing interface. In the\n concretizer config, you can specify a target spec and a replacement\n by hash.\n\n ```yaml\n concretizer:\n splice:\n explicit:\n - target: mpi\n replacement: mpich/abcdef\n ```\n\n Here, every installation that would normally use the target spec will\n instead use its replacement. Above, any spec using *any* `mpi` will be\n spliced to depend on the specific `mpich` installation requested. This\n *can* go wrong if you try to replace something built with, e.g.,\n `openmpi` with `mpich`, and it is on the user to ensure ABI\n compatibility between target and replacement specs. This currently\n requires some expertise to use, but it will allow users to reuse the\n binaries they create across more machines and environments.\n\n b. Automatic Splicing (experimental)\n\n #46729 introduced automatic splicing. In the concretizer config, enable\n automatic splicing:\n\n ```yaml\n concretizer:\n splice:\n automatic: true\n ```\n\n or run:\n\n ```console\n spack config add concretizer:splice:automatic:true\n ```\n\n The concretizer will select splices for ABI compatibility to maximize\n package reuse. Packages can denote ABI compatibility using the\n `can_splice` directive. No packages in Spack yet use this directive, so\n if you want to use this feature you will need to add `can_splice`\n annotations to your packages. We are working on ways to add more ABI\n compatibility information to the Spack package repository, and this\n directive may change in the future.\n\n See the documentation for more details:\n * https://spack.readthedocs.io/en/latest/build_settings.html#splicing\n * https://spack.readthedocs.io/en/latest/packaging_guide.html#specifying-abi-compatibility\n\n3. Broader variant propagation\n\n Since #42931, you can specify propagated variants like `hdf5\n build_type==RelWithDebInfo` or `trilinos ++openmp` to propagate a variant\n to all dependencies for which it is relevant. This is valid *even* if the\n variant does not exist on the package or its dependencies.\n\n See https://spack.readthedocs.io/en/latest/basic_usage.html#variants.\n\n4. Query specs by namespace\n\n #45416 allows a package's namespace (indicating the repository it came from)\n to be treated like a variant. You can request packages from particular repos\n like this:\n\n ```console\n spack find zlib namespace=builtin\n spack find zlib namespace=myrepo\n ```\n\n Previously, the spec syntax only allowed namespaces to be prefixes of spec\n names, e.g. `builtin.zlib`. The previous syntax still works.\n\n5. `spack spec` respects environment settings and `unify:true`\n\n `spack spec` did not previously respect environment lockfiles or\n unification settings, which made it difficult to see exactly how a spec\n would concretize within an environment. Now it does, so the output you get\n with `spack spec` will be *the same* as what your environment will\n concretize to when you run `spack concretize`. Similarly, if you provide\n multiple specs on the command line with `spack spec`, it will concretize\n them together if `unify:true` is set.\n\n See #47556 and #44843.\n\n6. Less noisy `spack spec` output\n\n `spack spec` previously showed output like this:\n\n ```console\n > spack spec /v5fn6xo\n Input spec\n --------------------------------\n - /v5fn6xo\n\n Concretized\n --------------------------------\n [+] openssl@3.3.1%apple-clang@16.0.0~docs+shared arch=darwin-sequoia-m1\n ...\n ```\n\n But the input spec is redundant, and we know we run `spack spec` to concretize\n the input spec. `spack spec` now *only* shows the concretized spec. See #47574.\n\n7. Better output for `spack find -c`\n\n In an environmnet, `spack find -c` lets you search the concretized, but not\n yet installed, specs, just as you would the installed ones. As with `spack\n spec`, this should make it easier for you to see what *will* be built\n before building and installing it. See #44713.\n\n8. `spack -C `: use an environment's configuration without activation\n\n Spack environments allow you to associate:\n 1. a set of (possibly concretized) specs, and\n 2. configuration\n\n When you activate an environment, you're using both of these. Previously, we\n supported:\n * `spack -e ` to run spack in the context of a specific environment, and\n * `spack -C ` to run spack using a directory with configuration files.\n\n You can now also pass an environment to `spack -C` to use *only* the environment's\n configuration, but not the specs or lockfile. See #45046.\n\n## New commands, options, and directives\n\n* The new `spack env track` command (#41897) takes a non-managed Spack\n environment and adds a symlink to Spack's `$environments_root` directory, so\n that it will be included for reference counting for commands like `spack\n uninstall` and `spack gc`. If you use free-standing directory environments,\n this is useful for preventing Spack from removing things required by your\n environments. You can undo this tracking with the `spack env untrack`\n command.\n\n* Add `-t` short option for `spack --backtrace` (#47227)\n\n `spack -d / --debug` enables backtraces on error, but it can be very\n verbose, and sometimes you just want the backtrace. `spack -t / --backtrace`\n provides that option.\n\n* `gc`: restrict to specific specs (#46790)\n\n If you only want to garbage-collect specific packages, you can now provide\n them on the command line. This gives users finer-grained control over what\n is uninstalled.\n\n* oci buildcaches now support `--only=package`. You can now push *just* a\n package and not its dependencies to an OCI registry. This allows dependents\n of non-redistributable specs to be stored in OCI registries without an\n error. See #45775.\n\n## Notable refactors\n* Variants are now fully conditional\n\n The `variants` dictionary on packages was previously keyed by variant name,\n and allowed only one definition of any given variant. Spack is now smart\n enough to understand that variants may have different values and defaults\n for different versions. For example, `warpx` prior to `23.06` only supported\n builds for one dimensionality, and newer `warpx` versions could be built\n with support for many different dimensions:\n\n ```python\n variant(\n \"dims\",\n default=\"3\",\n values=(\"1\", \"2\", \"3\", \"rz\"),\n multi=False,\n description=\"Number of spatial dimensions\",\n when=\"@:23.05\",\n )\n variant(\n \"dims\",\n default=\"1,2,rz,3\",\n values=(\"1\", \"2\", \"3\", \"rz\"),\n multi=True,\n description=\"Number of spatial dimensions\",\n when=\"@23.06:\",\n )\n ```\n\n Previously, the default for the old version of `warpx` was not respected and\n had to be specified manually. Now, Spack will select the right variant\n definition for each version at concretization time. This allows variants to\n evolve more smoothly over time. See #44425 for details.\n\n## Highlighted bugfixes\n\n1. Externals no longer override the preferred provider (#45025).\n\n External definitions could interfere with package preferences. Now, if\n `openmpi` is the preferred `mpi`, and an external `mpich` is defined, a new\n `openmpi` *will* be built if building it is possible. Previously we would\n prefer `mpich` despite the preference.\n\n2. Composable `cflags` (#41049).\n\n This release fixes a longstanding bug that concretization would fail if\n there were different `cflags` specified in `packages.yaml`,\n `compilers.yaml`, or on `the` CLI. Flags and their ordering are now tracked\n in the concretizer and flags from multiple sources will be merged.\n\n3. Fix concretizer Unification for included environments (#45139).\n\n## Deprecations, removals, and syntax changes\n\n1. The old concretizer has been removed from Spack, along with the\n `config:concretizer` config option. Spack will emit a warning if the option\n is present in user configuration, since it now has no effect. Spack now\n uses a simpler bootstrapping mechanism, where a JSON prototype is tweaked\n slightly to get an initial concrete spec to download. See #45215.\n\n2. Best-effort expansion of spec matrices has been removed. This feature did\n not work with the \"new\" ASP-based concretizer, and did not work with\n `unify: True` or `unify: when_possible`. Use the\n [exclude key](https://spack.readthedocs.io/en/latest/environments.html#spec-matrices)\n for the environment to exclude invalid components, or use multiple spec\n matrices to combine the list of specs for which the constraint is valid and\n the list of specs for which it is not. See #40792.\n\n3. The old Cray `platform` (based on Cray PE modules) has been removed, and\n `platform=cray` is no longer supported. Since `v0.19`, Spack has handled\n Cray machines like Linux clusters with extra packages, and we have\n encouraged using this option to support Cray. The new approach allows us to\n correctly handle Cray machines with non-SLES operating systems, and it is\n much more reliable than making assumptions about Cray modules. See the\n `v0.19` release notes and #43796 for more details.\n\n4. The `config:install_missing_compilers` config option has been deprecated,\n and it is a no-op when set in `v0.23`. Our new compiler dependency model\n will replace it with a much more reliable and robust mechanism in `v1.0`.\n See #46237.\n\n5. Config options that deprecated in `v0.21` have been removed in `v0.23`. You\n can now only specify preferences for `compilers`, `targets`, and\n `providers` globally via the `packages:all:` section. Similarly, you can\n only specify `versions:` locally for a specific package. See #44061 and\n #31261 for details.\n\n6. Spack's old test interface has been removed (#45752), having been\n deprecated in `v0.22.0` (#34236). All `builtin` packages have been updated\n to use the new interface. See the [stand-alone test documentation](\n https://spack.readthedocs.io/en/latest/packaging_guide.html#stand-alone-tests)\n\n7. The `spack versions --safe-only` option, deprecated since `v0.21.0`, has\n been removed. See #45765.\n\n* The `--dependencies` and `--optimize` arguments to `spack ci` have been\n deprecated. See #45005.\n\n## Binary caches\n1. Public binary caches now include an ML stack for Linux/aarch64 (#39666)We\n now build an ML stack for Linux/aarch64 for all pull requests and on\n develop. The ML stack includes both CPU-only and CUDA builds for Horovod,\n Hugging Face, JAX, Keras, PyTorch,scikit-learn, TensorBoard, and\n TensorFlow, and related packages. The CPU-only stack also includes XGBoost.\n See https://cache.spack.io/tag/develop/?stack=ml-linux-aarch64-cuda.\n\n2. There is also now an stack of developer tools for macOS (#46910), which is\n analogous to the Linux devtools stack. You can use this to avoid building\n many common build dependencies. See\n https://cache.spack.io/tag/develop/?stack=developer-tools-darwin.\n\n## Architecture support\n* archspec has been updated to `v0.2.5`, with support for `zen5`\n* Spack's CUDA package now supports the Grace Hopper `9.0a` compute capability (#45540)\n\n## Windows\n* Windows bootstrapping: `file` and `gpg` (#41810)\n* `scripts` directory added to PATH on Windows for python extensions (#45427)\n* Fix `spack load --list` and `spack unload` on Windows (#35720)\n\n## Other notable changes\n* Bugfix: `spack find -x` in environments (#46798)\n* Spec splices are now robust to duplicate nodes with the same name in a spec (#46382)\n* Cache per-compiler libc calculations for performance (#47213)\n* Fixed a bug in external detection for openmpi (#47541)\n* Mirror configuration allows username/password as environment variables (#46549)\n* Default library search caps maximum depth (#41945)\n* Unify interface for `spack spec` and `spack solve` commands (#47182)\n* Spack no longer RPATHs directories in the default library search path (#44686)\n* Improved performance of Spack database (#46554)\n* Enable package reuse for packages with versions from git refs (#43859)\n* Improved handling for `uuid` virtual on macos (#43002)\n* Improved tracking of task queueing/requeueing in the installer (#46293)\n\n## Spack community stats\n\n* Over 2,000 pull requests updated package recipes\n* 8,307 total packages, 329 new since `v0.22.0`\n * 140 new Python packages\n * 14 new R packages\n* 373 people contributed to this release\n * 357 committers to packages\n * 60 committers to core\n\n\n# v0.22.2 (2024-09-21)\n\n## Bugfixes\n- Forward compatibility with Spack 0.23 packages with language dependencies (#45205, #45191)\n- Forward compatibility with `urllib` from Python 3.12.6+ (#46453, #46483)\n- Bump vendored `archspec` for better aarch64 support (#45721, #46445)\n- Support macOS Sequoia (#45018, #45127)\n- Fix regression in `{variants.X}` and `{variants.X.value}` format strings (#46206)\n- Ensure shell escaping of environment variable values in load and activate commands (#42780)\n- Fix an issue where `spec[pkg]` considers specs outside the current DAG (#45090)\n- Do not halt concretization on unknown variants in externals (#45326)\n- Improve validation of `develop` config section (#46485)\n- Explicitly disable `ccache` if turned off in config, to avoid cache pollution (#45275)\n- Improve backwards compatibility in `include_concrete` (#45766)\n- Fix issue where package tags were sometimes repeated (#45160)\n- Make `setup-env.sh` \"sourced only\" by dropping execution bits (#45641)\n- Make certain source/binary fetch errors recoverable instead of a hard error (#45683)\n- Remove debug statements in package hash computation (#45235)\n- Remove redundant clingo warnings (#45269)\n- Remove hard-coded layout version (#45645)\n- Do not initialize previous store state in `use_store` (#45268)\n- Docs improvements (#46475)\n\n## Package updates\n- `chapel` major update (#42197, #44931, #45304)\n\n# v0.22.1 (2024-07-04)\n\n## Bugfixes\n- Fix reuse of externals on Linux (#44316)\n- Ensure parent gcc-runtime version >= child (#44834, #44870)\n- Ensure the latest gcc-runtime is rpath'ed when multiple exist among link deps (#44219)\n- Improve version detection of glibc (#44154)\n- Improve heuristics for solver (#44893, #44976, #45023)\n- Make strong preferences override reuse (#44373)\n- Reduce verbosity when C compiler is missing (#44182)\n- Make missing ccache executable an error when required (#44740)\n- Make every environment view containing `python` a `venv` (#44382)\n- Fix external detection for compilers with os but no target (#44156)\n- Fix version optimization for roots (#44272)\n- Handle common implementations of pagination of tags in OCI build caches (#43136)\n- Apply fetched patches to develop specs (#44950)\n- Avoid Windows wrappers for filesystem utilities on non-Windows (#44126)\n- Fix issue with long filenames in build caches on Windows (#43851)\n- Fix formatting issue in `spack audit` (#45045)\n- CI fixes (#44582, #43965, #43967, #44279, #44213)\n\n## Package updates\n- protobuf: fix 3.4:3.21 patch checksum (#44443)\n- protobuf: update hash for patch needed when=\"@3.4:3.21\" (#44210)\n- git: bump v2.39 to 2.45; deprecate unsafe versions (#44248)\n- gcc: use -rpath {rpath_dir} not -rpath={rpath dir} (#44315)\n- Remove mesa18 and libosmesa (#44264)\n- Enforce consistency of `gl` providers (#44307)\n- Require libiconv for iconv (#44335, #45026).\n Notice that glibc/musl also provide iconv, but are not guaranteed to be\n complete. Set `packages:iconv:require:[glibc]` to restore the old behavior.\n- py-matplotlib: qualify when to do a post install (#44191)\n- rust: fix v1.78.0 instructions (#44127)\n- suite-sparse: improve setting of the `libs` property (#44214)\n- netlib-lapack: provide blas and lapack together (#44981)\n\n# v0.22.0 (2024-05-12)\n\n`v0.22.0` is a major feature release.\n\n## Features in this release\n\n1. **Compiler dependencies**\n\n We are in the process of making compilers proper dependencies in Spack, and a number\n of changes in `v0.22` support that effort. You may notice nodes in your dependency\n graphs for compiler runtime libraries like `gcc-runtime` or `libgfortran`, and you\n may notice that Spack graphs now include `libc`. We've also begun moving compiler\n configuration from `compilers.yaml` to `packages.yaml` to make it consistent with\n other externals. We are trying to do this with the least disruption possible, so\n your existing `compilers.yaml` files should still work. We expect to be done with\n this transition by the `v0.23` release in November.\n\n * #41104: Packages compiled with `%gcc` on Linux, macOS and FreeBSD now depend on a\n new package `gcc-runtime`, which contains a copy of the shared compiler runtime\n libraries. This enables gcc runtime libraries to be installed and relocated when\n using a build cache. When building minimal Spack-generated container images it is\n no longer necessary to install libgfortran, libgomp etc. using the system package\n manager.\n\n * #42062: Packages compiled with `%oneapi` now depend on a new package\n `intel-oneapi-runtime`. This is similar to `gcc-runtime`, and the runtimes can\n provide virtuals and compilers can inject dependencies on virtuals into compiled\n packages. This allows us to model library soname compatibility and allows\n compilers like `%oneapi` to provide virtuals like `sycl` (which can also be\n provided by standalone libraries). Note that until we have an agreement in place\n with intel, Intel packages are marked `redistribute(source=False, binary=False)`\n and must be downloaded outside of Spack.\n\n * #43272: changes to the optimization criteria of the solver improve the hit-rate of\n buildcaches by a fair amount. The solver more relaxed compatibility rules and will\n not try to strictly match compilers or targets of reused specs. Users can still\n enforce the previous strict behavior with `require:` sections in `packages.yaml`.\n Note that to enforce correct linking, Spack will *not* reuse old `%gcc` and\n `%oneapi` specs that do not have the runtime libraries as a dependency.\n\n * #43539: Spack will reuse specs built with compilers that are *not* explicitly\n configured in `compilers.yaml`. Because we can now keep runtime libraries in build\n cache, we do not require you to also have a local configured compiler to *use* the\n runtime libraries. This improves reuse in buildcaches and avoids conflicts with OS\n updates that happen underneath Spack.\n\n * #43190: binary compatibility on `linux` is now based on the `libc` version,\n instead of on the `os` tag. Spack builds now detect the host `libc` (`glibc` or\n `musl`) and add it as an implicit external node in the dependency graph. Binaries\n with a `libc` with the same name and a version less than or equal to that of the\n detected `libc` can be reused. This is only on `linux`, not `macos` or `Windows`.\n\n * #43464: each package that can provide a compiler is now detectable using `spack\n external find`. External packages defining compiler paths are effectively used as\n compilers, and `spack external find -t compiler` can be used as a substitute for\n `spack compiler find`. More details on this transition are in\n [the docs](https://spack.readthedocs.io/en/latest/getting_started.html#manual-compiler-configuration)\n\n2. **Improved `spack find` UI for Environments**\n\n If you're working in an environment, you likely care about:\n\n * What are the roots\n * Which ones are installed / not installed\n * What's been added that still needs to be concretized\n\n We've tweaked `spack find` in environments to show this information much more\n clearly. Installation status is shown next to each root, so you can see what is\n installed. Roots are also shown in bold in the list of installed packages. There is\n also a new option for `spack find -r` / `--only-roots` that will only show env\n roots, if you don't want to look at all the installed specs.\n\n More details in #42334.\n\n3. **Improved command-line string quoting**\n\n We are making some breaking changes to how Spack parses specs on the CLI in order to\n respect shell quoting instead of trying to fight it. If you (sadly) had to write\n something like this on the command line:\n\n ```\n spack install zlib cflags=\\\"-O2 -g\\\"\n ```\n\n That will now result in an error, but you can now write what you probably expected\n to work in the first place:\n\n ```\n spack install zlib cflags=\"-O2 -g\"\n ```\n\n Quoted can also now include special characters, so you can supply flags like:\n\n ```\n spack install zlib ldflags='-Wl,-rpath=$ORIGIN/_libs'\n ```\n\n To reduce ambiguity in parsing, we now require that you *not* put spaces around `=`\n and `==` when for flags or variants. This would not have broken before but will now\n result in an error:\n\n ```\n spack install zlib cflags = \"-O2 -g\"\n ```\n\n More details and discussion in #30634.\n\n4. **Revert default `spack install` behavior to `--reuse`**\n\n We changed the default concretizer behavior from `--reuse` to `--reuse-deps` in\n #30990 (in `v0.20`), which meant that *every* `spack install` invocation would\n attempt to build a new version of the requested package / any environment roots.\n While this is a common ask for *upgrading* and for *developer* workflows, we don't\n think it should be the default for a package manager.\n\n We are going to try to stick to this policy:\n 1. Prioritize reuse and build as little as possible by default.\n 2. Only upgrade or install duplicates if they are explicitly asked for, or if there\n is a known security issue that necessitates an upgrade.\n\n With the install command you now have three options:\n\n * `--reuse` (default): reuse as many existing installations as possible.\n * `--reuse-deps` / `--fresh-roots`: upgrade (freshen) roots but reuse dependencies if possible.\n * `--fresh`: install fresh versions of requested packages (roots) and their dependencies.\n\n We've also introduced `--fresh-roots` as an alias for `--reuse-deps` to make it more clear\n that it may give you fresh versions. More details in #41302 and #43988.\n\n5. **More control over reused specs**\n\n You can now control which packages to reuse and how. There is a new\n `concretizer:reuse` config option, which accepts the following properties:\n\n - `roots`: `true` to reuse roots, `false` to reuse just dependencies\n - `exclude`: list of constraints used to select which specs *not* to reuse\n - `include`: list of constraints used to select which specs *to* reuse\n - `from`: list of sources for reused specs (some combination of `local`,\n `buildcache`, or `external`)\n\n For example, to reuse only specs compiled with GCC, you could write:\n\n ```yaml\n concretizer:\n reuse:\n roots: true\n include:\n - \"%gcc\"\n ```\n\n Or, if `openmpi` must be used from externals, and it must be the only external used:\n\n ```yaml\n concretizer:\n reuse:\n roots: true\n from:\n - type: local\n exclude: [\"openmpi\"]\n - type: buildcache\n exclude: [\"openmpi\"]\n - type: external\n include: [\"openmpi\"]\n ```\n\n6. **New `redistribute()` directive**\n\n Some packages can't be redistributed in source or binary form. We need an explicit\n way to say that in a package.\n\n Now there is a `redistribute()` directive so that package authors can write:\n\n ```python\n class MyPackage(Package):\n redistribute(source=False, binary=False)\n ```\n\n Like other directives, this works with `when=`:\n\n ```python\n class MyPackage(Package):\n # 12.0 and higher are proprietary\n redistribute(source=False, binary=False, when=\"@12.0:\")\n\n # can't redistribute when we depend on some proprietary dependency\n redistribute(source=False, binary=False, when=\"^proprietary-dependency\")\n ```\n\n More in #20185.\n\n7. **New `conflict:` and `prefer:` syntax for package preferences**\n\n Previously, you could express conflicts and preferences in `packages.yaml` through\n some contortions with `require:`:\n\n ```yaml\n packages:\n zlib-ng:\n require:\n - one_of: [\"%clang\", \"@:\"] # conflict on %clang\n - any_of: [\"+shared\", \"@:\"] # strong preference for +shared\n ```\n\n You can now use `require:` and `prefer:` for a much more readable configuration:\n\n ```yaml\n packages:\n zlib-ng:\n conflict:\n - \"%clang\"\n prefer:\n - \"+shared\"\n ```\n\n See [the documentation](https://spack.readthedocs.io/en/latest/packages_yaml.html#conflicts-and-strong-preferences)\n and #41832 for more details.\n\n8. **`include_concrete` in environments**\n\n You may want to build on the *concrete* contents of another environment without\n changing that environment. You can now include the concrete specs from another\n environment's `spack.lock` with `include_concrete`:\n\n ```yaml\n spack:\n specs: []\n concretizer:\n unify: true\n include_concrete:\n - /path/to/environment1\n - /path/to/environment2\n ```\n\n Now, when *this* environment is concretized, it will bring in the already concrete\n specs from `environment1` and `environment2`, and build on top of them without\n changing them. This is useful if you have phased deployments, where old deployments\n should not be modified but you want to use as many of them as possible. More details\n in #33768.\n\n9. **`python-venv` isolation**\n\n Spack has unique requirements for Python because it:\n 1. installs every package in its own independent directory, and\n 2. allows users to register *external* python installations.\n\n External installations may contain their own installed packages that can interfere\n with Spack installations, and some distributions (Debian and Ubuntu) even change the\n `sysconfig` in ways that alter the installation layout of installed Python packages\n (e.g., with the addition of a `/local` prefix on Debian or Ubuntu). To isolate Spack\n from these and other issues, we now insert a small `python-venv` package in between\n `python` and packages that need to install Python code. This isolates Spack's build\n environment, isolates Spack from any issues with an external python, and resolves a\n large number of issues we've had with Python installations.\n\n See #40773 for further details.\n\n## New commands, options, and directives\n\n* Allow packages to be pushed to build cache after install from source (#42423)\n* `spack develop`: stage build artifacts in same root as non-dev builds #41373\n * Don't delete `spack develop` build artifacts after install (#43424)\n* `spack find`: add options for local/upstream only (#42999)\n* `spack logs`: print log files for packages (either partially built or installed) (#42202)\n* `patch`: support reversing patches (#43040)\n* `develop`: Add -b/--build-directory option to set build_directory package attribute (#39606)\n* `spack list`: add `--namespace` / `--repo` option (#41948)\n* directives: add `checked_by` field to `license()`, add some license checks\n* `spack gc`: add options for environments and build dependencies (#41731)\n* Add `--create` to `spack env activate` (#40896)\n\n## Performance improvements\n\n* environment.py: fix excessive re-reads (#43746)\n* ruamel yaml: fix quadratic complexity bug (#43745)\n* Refactor to improve `spec format` speed (#43712)\n* Do not acquire a write lock on the env post install if no views (#43505)\n* asp.py: fewer calls to `spec.copy()` (#43715)\n* spec.py: early return in `__str__`\n* avoid `jinja2` import at startup unless needed (#43237)\n\n## Other new features of note\n\n* `archspec`: update to `v0.2.4`: support for Windows, bugfixes for `neoverse-v1` and\n `neoverse-v2` detection.\n* `spack config get`/`blame`: with no args, show entire config\n* `spack env create `: dir if dir-like (#44024)\n* ASP-based solver: update os compatibility for macOS (#43862)\n* Add handling of custom ssl certs in urllib ops (#42953)\n* Add ability to rename environments (#43296)\n* Add config option and compiler support to reuse across OS's (#42693)\n* Support for prereleases (#43140)\n* Only reuse externals when configured (#41707)\n* Environments: Add support for including views (#42250)\n\n## Binary caches\n* Build cache: make signed/unsigned a mirror property (#41507)\n* tools stack\n\n## Removals, deprecations, and syntax changes\n* remove `dpcpp` compiler and package (#43418)\n* spack load: remove --only argument (#42120)\n\n## Notable Bugfixes\n* repo.py: drop deleted packages from provider cache (#43779)\n* Allow `+` in module file names (#41999)\n* `cmd/python`: use runpy to allow multiprocessing in scripts (#41789)\n* Show extension commands with spack -h (#41726)\n* Support environment variable expansion inside module projections (#42917)\n* Alert user to failed concretizations (#42655)\n* shell: fix zsh color formatting for PS1 in environments (#39497)\n* spack mirror create --all: include patches (#41579)\n\n## Spack community stats\n\n* 7,994 total packages; 525 since `v0.21.0`\n * 178 new Python packages, 5 new R packages\n* 358 people contributed to this release\n * 344 committers to packages\n * 45 committers to core\n\n# v0.21.3 (2024-10-02)\n\n## Bugfixes\n- Forward compatibility with Spack 0.23 packages with language dependencies (#45205, #45191)\n- Forward compatibility with `urllib` from Python 3.12.6+ (#46453, #46483)\n- Bump `archspec` to 0.2.5-dev for better aarch64 and Windows support (#42854, #44005,\n #45721, #46445)\n- Support macOS Sequoia (#45018, #45127, #43862)\n- CI and test maintenance (#42909, #42728, #46711, #41943, #43363)\n\n# v0.21.2 (2024-03-01)\n\n## Bugfixes\n\n- Containerize: accommodate nested or pre-existing spack-env paths (#41558)\n- Fix setup-env script, when going back and forth between instances (#40924)\n- Fix using fully-qualified namespaces from root specs (#41957)\n- Fix a bug when a required provider is requested for multiple virtuals (#42088)\n- OCI buildcaches:\n - only push in parallel when forking (#42143)\n - use pickleable errors (#42160)\n- Fix using sticky variants in externals (#42253)\n- Fix a rare issue with conditional requirements and multi-valued variants (#42566)\n\n## Package updates\n- rust: add v1.75, rework a few variants (#41161,#41903)\n- py-transformers: add v4.35.2 (#41266)\n- mgard: fix OpenMP on AppleClang (#42933)\n\n# v0.21.1 (2024-01-11)\n\n## New features\n- Add support for reading buildcaches created by Spack v0.22 (#41773)\n\n## Bugfixes\n\n- spack graph: fix coloring with environments (#41240)\n- spack info: sort variants in --variants-by-name (#41389)\n- Spec.format: error on old style format strings (#41934)\n- ASP-based solver:\n - fix infinite recursion when computing concretization errors (#41061)\n - don't error for type mismatch on preferences (#41138)\n - don't emit spurious debug output (#41218)\n- Improve the error message for deprecated preferences (#41075)\n- Fix MSVC preview version breaking clingo build on Windows (#41185)\n- Fix multi-word aliases (#41126)\n- Add a warning for unconfigured compiler (#41213)\n- environment: fix an issue with deconcretization/reconcretization of specs (#41294)\n- buildcache: don't error if a patch is missing, when installing from binaries (#41986)\n- Multiple improvements to unit-tests (#41215,#41369,#41495,#41359,#41361,#41345,#41342,#41308,#41226)\n\n## Package updates\n- root: add a webgui patch to address security issue (#41404)\n- BerkeleyGW: update source urls (#38218)\n\n# v0.21.0 (2023-11-11)\n\n`v0.21.0` is a major feature release.\n\n## Features in this release\n\n1. **Better error messages with condition chaining**\n\n In v0.18, we added better error messages that could tell you what problem happened,\n but they couldn't tell you *why* it happened. `0.21` adds *condition chaining* to the\n solver, and Spack can now trace back through the conditions that led to an error and\n build a tree of causes potential causes and where they came from. For example:\n\n ```console\n $ spack solve hdf5 ^cmake@3.0.1\n ==> Error: concretization failed for the following reasons:\n\n 1. Cannot satisfy 'cmake@3.0.1'\n 2. Cannot satisfy 'cmake@3.0.1'\n required because hdf5 ^cmake@3.0.1 requested from CLI\n 3. Cannot satisfy 'cmake@3.18:' and 'cmake@3.0.1\n required because hdf5 ^cmake@3.0.1 requested from CLI\n required because hdf5 depends on cmake@3.18: when @1.13:\n required because hdf5 ^cmake@3.0.1 requested from CLI\n 4. Cannot satisfy 'cmake@3.12:' and 'cmake@3.0.1\n required because hdf5 depends on cmake@3.12:\n required because hdf5 ^cmake@3.0.1 requested from CLI\n required because hdf5 ^cmake@3.0.1 requested from CLI\n ```\n\n More details in #40173.\n\n2. **OCI build caches**\n\n You can now use an arbitrary [OCI](https://opencontainers.org) registry as a build\n cache:\n\n ```console\n $ spack mirror add my_registry oci://user/image # Dockerhub\n $ spack mirror add my_registry oci://ghcr.io/haampie/spack-test # GHCR\n $ spack mirror set --push --oci-username ... --oci-password ... my_registry # set login creds\n $ spack buildcache push my_registry [specs...]\n ```\n\n And you can optionally add a base image to get *runnable* images:\n\n ```console\n $ spack buildcache push --base-image ubuntu:23.04 my_registry python\n Pushed ... as [image]:python-3.11.2-65txfcpqbmpawclvtasuog4yzmxwaoia.spack\n\n $ docker run --rm -it [image]:python-3.11.2-65txfcpqbmpawclvtasuog4yzmxwaoia.spack\n ```\n\n This creates a container image from the Spack installations on the host system,\n without the need to run `spack install` from a `Dockerfile` or `sif` file. It also\n addresses the inconvenience of losing binaries of dependencies when `RUN spack\n install` fails inside `docker build`.\n\n Further, the container image layers and build cache tarballs are the same files. This\n means that `spack install` and `docker pull` use the exact same underlying binaries.\n If you previously used `spack install` inside of `docker build`, this feature helps\n you save storage by a factor two.\n\n More details in #38358.\n\n3. **Multiple versions of build dependencies**\n\n Increasingly, complex package builds require multiple versions of some build\n dependencies. For example, Python packages frequently require very specific versions\n of `setuptools`, `cython`, and sometimes different physics packages require different\n versions of Python to build. The concretizer enforced that every solve was *unified*,\n i.e., that there only be one version of every package. The concretizer now supports\n \"duplicate\" nodes for *build dependencies*, but enforces unification through\n transitive link and run dependencies. This will allow it to better resolve complex\n dependency graphs in ecosystems like Python, and it also gets us very close to\n modeling compilers as proper dependencies.\n\n This change required a major overhaul of the concretizer, as well as a number of\n performance optimizations. See #38447, #39621.\n\n4. **Cherry-picking virtual dependencies**\n\n You can now select only a subset of virtual dependencies from a spec that may provide\n more. For example, if you want `mpich` to be your `mpi` provider, you can be explicit\n by writing:\n\n ```\n hdf5 ^[virtuals=mpi] mpich\n ```\n\n Or, if you want to use, e.g., `intel-parallel-studio` for `blas` along with an external\n `lapack` like `openblas`, you could write:\n\n ```\n strumpack ^[virtuals=mpi] intel-parallel-studio+mkl ^[virtuals=lapack] openblas\n ```\n\n The `virtuals=mpi` is an edge attribute, and dependency edges in Spack graphs now\n track which virtuals they satisfied. More details in #17229 and #35322.\n\n Note for packaging: in Spack 0.21 `spec.satisfies(\"^virtual\")` is true if and only if\n the package specifies `depends_on(\"virtual\")`. This is different from Spack 0.20,\n where depending on a provider implied depending on the virtual provided. See #41002\n for an example where `^mkl` was being used to test for several `mkl` providers in a\n package that did not depend on `mkl`.\n\n5. **License directive**\n\n Spack packages can now have license metadata, with the new `license()` directive:\n\n ```python\n license(\"Apache-2.0\")\n ```\n\n Licenses use [SPDX identifiers](https://spdx.org/licenses), and you can use SPDX\n expressions to combine them:\n\n ```python\n license(\"Apache-2.0 OR MIT\")\n ```\n\n Like other directives in Spack, it's conditional, so you can handle complex cases like\n Spack itself:\n\n ```python\n license(\"LGPL-2.1\", when=\"@:0.11\")\n license(\"Apache-2.0 OR MIT\", when=\"@0.12:\")\n ```\n\n More details in #39346, #40598.\n\n6. **`spack deconcretize` command**\n\n We are getting close to having a `spack update` command for environments, but we're\n not quite there yet. This is the next best thing. `spack deconcretize` gives you\n control over what you want to update in an already concrete environment. If you have\n an environment built with, say, `meson`, and you want to update your `meson` version,\n you can run:\n\n ```console\n spack deconcretize meson\n ```\n\n and have everything that depends on `meson` rebuilt the next time you run `spack\n concretize`. In a future Spack version, we'll handle all of this in a single command,\n but for now you can use this to drop bits of your lockfile and resolve your\n dependencies again. More in #38803.\n\n7. **UI Improvements**\n\n The venerable `spack info` command was looking shabby compared to the rest of Spack's\n UI, so we reworked it to have a bit more flair. `spack info` now makes much better\n use of terminal space and shows variants, their values, and their descriptions much\n more clearly. Conditional variants are grouped separately so you can more easily\n understand how packages are structured. More in #40998.\n\n `spack checksum` now allows you to filter versions from your editor, or by version\n range. It also notifies you about potential download URL changes. See #40403.\n\n8. **Environments can include definitions**\n\n Spack did not previously support using `include:` with The\n [definitions](https://spack.readthedocs.io/en/latest/environments.html#spec-list-references)\n section of an environment, but now it does. You can use this to curate lists of specs\n and more easily reuse them across environments. See #33960.\n\n9. **Aliases**\n\n You can now add aliases to Spack commands in `config.yaml`, e.g. this might enshrine\n your favorite args to `spack find` as `spack f`:\n\n ```yaml\n config:\n aliases:\n f: find -lv\n ```\n\n See #17229.\n\n10. **Improved autoloading of modules**\n\n Spack 0.20 was the first release to enable autoloading of direct dependencies in\n module files.\n\n The downside of this was that `module avail` and `module load` tab completion would\n show users too many modules to choose from, and many users disabled generating\n modules for dependencies through `exclude_implicits: true`. Further, it was\n necessary to keep hashes in module names to avoid file name clashes.\n\n In this release, you can start using `hide_implicits: true` instead, which exposes\n only explicitly installed packages to the user, while still autoloading\n dependencies. On top of that, you can safely use `hash_length: 0`, as this config\n now only applies to the modules exposed to the user -- you don't have to worry about\n file name clashes for hidden dependencies.\n\n Note: for `tcl` this feature requires Modules 4.7 or higher\n\n11. **Updated container labeling**\n\n Nightly Docker images from the `develop` branch will now be tagged as `:develop` and\n `:nightly`. The `:latest` tag is no longer associated with `:develop`, but with the\n latest stable release. Releases will be tagged with `:{major}`, `:{major}.{minor}`\n and `:{major}.{minor}.{patch}`. `ubuntu:18.04` has also been removed from the list of\n generated Docker images, as it is no longer supported. See #40593.\n\n## Other new commands and directives\n\n* `spack env activate` without arguments now loads a `default` environment that you do\n not have to create (#40756).\n* `spack find -H` / `--hashes`: a new shortcut for piping `spack find` output to\n other commands (#38663)\n* Add `spack checksum --verify`, fix `--add` (#38458)\n* New `default_args` context manager factors out common args for directives (#39964)\n* `spack compiler find --[no]-mixed-toolchain` lets you easily mix `clang` and\n `gfortran` on Linux (#40902)\n\n## Performance improvements\n\n* `spack external find` execution is now much faster (#39843)\n* `spack location -i` now much faster on success (#40898)\n* Drop redundant rpaths post install (#38976)\n* ASP-based solver: avoid cycles in clingo using hidden directive (#40720)\n* Fix multiple quadratic complexity issues in environments (#38771)\n\n## Other new features of note\n\n* archspec: update to v0.2.2, support for Sapphire Rapids, Power10, Neoverse V2 (#40917)\n* Propagate variants across nodes that don't have that variant (#38512)\n* Implement fish completion (#29549)\n* Can now distinguish between source/binary mirror; don't ping mirror.spack.io as much (#34523)\n* Improve status reporting on install (add [n/total] display) (#37903)\n\n## Windows\n\nThis release has the best Windows support of any Spack release yet, with numerous\nimprovements and much larger swaths of tests passing:\n\n* MSVC and SDK improvements (#37711, #37930, #38500, #39823, #39180)\n* Windows external finding: update default paths; treat .bat as executable on Windows (#39850)\n* Windows decompression: fix removal of intermediate file (#38958)\n* Windows: executable/path handling (#37762)\n* Windows build systems: use ninja and enable tests (#33589)\n* Windows testing (#36970, #36972, #36973, #36840, #36977, #36792, #36834, #34696, #36971)\n* Windows PowerShell support (#39118, #37951)\n* Windows symlinking and libraries (#39933, #38599, #34701, #38578, #34701)\n\n## Notable refactors\n* User-specified flags take precedence over others in Spack compiler wrappers (#37376)\n* Improve setup of build, run, and test environments (#35737, #40916)\n* `make` is no longer a required system dependency of Spack (#40380)\n* Support Python 3.12 (#40404, #40155, #40153)\n* docs: Replace package list with packages.spack.io (#40251)\n* Drop Python 2 constructs in Spack (#38720, #38718, #38703)\n\n## Binary cache and stack updates\n* e4s arm stack: duplicate and target neoverse v1 (#40369)\n* Add macOS ML CI stacks (#36586)\n* E4S Cray CI Stack (#37837)\n* e4s cray: expand spec list (#38947)\n* e4s cray sles ci: expand spec list (#39081)\n\n## Removals, deprecations, and syntax changes\n* ASP: targets, compilers and providers soft-preferences are only global (#31261)\n* Parser: fix ambiguity with whitespace in version ranges (#40344)\n* Module file generation is disabled by default; you'll need to enable it to use it (#37258)\n* Remove deprecated \"extra_instructions\" option for containers (#40365)\n* Stand-alone test feature deprecation postponed to v0.22 (#40600)\n* buildcache push: make `--allow-root` the default and deprecate the option (#38878)\n\n## Notable Bugfixes\n* Bugfix: propagation of multivalued variants (#39833)\n* Allow `/` in git versions (#39398)\n* Fetch & patch: actually acquire stage lock, and many more issues (#38903)\n* Environment/depfile: better escaping of targets with Git versions (#37560)\n* Prevent \"spack external find\" to error out on wrong permissions (#38755)\n* lmod: allow core compiler to be specified with a version range (#37789)\n\n## Spack community stats\n\n* 7,469 total packages, 303 new since `v0.20.0`\n * 150 new Python packages\n * 34 new R packages\n* 353 people contributed to this release\n * 336 committers to packages\n * 65 committers to core\n\n\n# v0.20.3 (2023-10-31)\n\n## Bugfixes\n\n- Fix a bug where `spack mirror set-url` would drop configured connection info (reverts #34210)\n- Fix a minor issue with package hash computation for Python 3.12 (#40328)\n\n\n# v0.20.2 (2023-10-03)\n\n## Features in this release\n\nSpack now supports Python 3.12 (#40155)\n\n## Bugfixes\n\n- Improve escaping in Tcl module files (#38375)\n- Make repo cache work on repositories with zero mtime (#39214)\n- Ignore errors for newer, incompatible buildcache version (#40279)\n- Print an error when git is required, but missing (#40254)\n- Ensure missing build dependencies get installed when using `spack install --overwrite` (#40252)\n- Fix an issue where Spack freezes when the build process unexpectedly exits (#39015)\n- Fix a bug where installation failures cause an unrelated `NameError` to be thrown (#39017)\n- Fix an issue where Spack package versions would be incorrectly derived from git tags (#39414)\n- Fix a bug triggered when file locking fails internally (#39188)\n- Prevent \"spack external find\" to error out when a directory cannot be accessed (#38755)\n- Fix multiple performance regressions in environments (#38771)\n- Add more ignored modules to `pyproject.toml` for `mypy` (#38769)\n\n\n# v0.20.1 (2023-07-10)\n\n## Spack Bugfixes\n\n- Spec removed from an environment where not actually removed if `--force` was not given (#37877)\n- Speed-up module file generation (#37739)\n- Hotfix for a few recipes that treat CMake as a link dependency (#35816)\n- Fix re-running stand-alone test a second time, which was getting a trailing spurious failure (#37840)\n- Fixed reading JSON manifest on Cray, reporting non-concrete specs (#37909)\n- Fixed a few bugs when generating Dockerfiles from Spack (#37766,#37769)\n- Fixed a few long-standing bugs when generating module files (#36678,#38347,#38465,#38455)\n- Fixed issues with building Python extensions using an external Python (#38186)\n- Fixed compiler removal from command line (#38057)\n- Show external status as [e] (#33792)\n- Backported `archspec` fixes (#37793)\n- Improved a few error messages (#37791)\n\n\n# v0.20.0 (2023-05-21)\n\n`v0.20.0` is a major feature release.\n\n## Features in this release\n\n1. **`requires()` directive and enhanced package requirements**\n\n We've added some more enhancements to requirements in Spack (#36286).\n\n There is a new `requires()` directive for packages. `requires()` is the opposite of\n `conflicts()`. You can use it to impose constraints on this package when certain\n conditions are met:\n\n ```python\n requires(\n \"%apple-clang\",\n when=\"platform=darwin\",\n msg=\"This package builds only with clang on macOS\"\n )\n ```\n\n More on this in [the docs](\n https://spack.rtfd.io/en/latest/packaging_guide.html#conflicts-and-requirements).\n\n You can also now add a `when:` clause to `requires:` in your `packages.yaml`\n configuration or in an environment:\n\n ```yaml\n packages:\n openmpi:\n require:\n - any_of: [\"%gcc\"]\n when: \"@:4.1.4\"\n message: \"Only OpenMPI 4.1.5 and up can build with fancy compilers\"\n ```\n\n More details can be found [here](\n https://spack.readthedocs.io/en/latest/build_settings.html#package-requirements)\n\n2. **Exact versions**\n\n Spack did not previously have a way to distinguish a version if it was a prefix of\n some other version. For example, `@3.2` would match `3.2`, `3.2.1`, `3.2.2`, etc. You\n can now match *exactly* `3.2` with `@=3.2`. This is useful, for example, if you need\n to patch *only* the `3.2` version of a package. The new syntax is described in [the docs](\n https://spack.readthedocs.io/en/latest/basic_usage.html#version-specifier).\n\n Generally, when writing packages, you should prefer to use ranges like `@3.2` over\n the specific versions, as this allows the concretizer more leeway when selecting\n versions of dependencies. More details and recommendations are in the [packaging guide](\n https://spack.readthedocs.io/en/latest/packaging_guide.html#ranges-versus-specific-versions).\n\n See #36273 for full details on the version refactor.\n\n3. **New testing interface**\n\n Writing package tests is now much simpler with a new [test interface](\n https://spack.readthedocs.io/en/latest/packaging_guide.html#stand-alone-tests).\n\n Writing a test is now as easy as adding a method that starts with `test_`:\n\n ```python\n class MyPackage(Package):\n ...\n\n def test_always_fails(self):\n \"\"\"use assert to always fail\"\"\"\n assert False\n\n def test_example(self):\n \"\"\"run installed example\"\"\"\n example = which(self.prefix.bin.example)\n example()\n ```\n\n You can use Python's native `assert` statement to implement your checks -- no more\n need to fiddle with `run_test` or other test framework methods. Spack will\n introspect the class and run `test_*` methods when you run `spack test`,\n\n4. **More stable concretization**\n\n * Now, `spack concretize` will *only* concretize the new portions of the environment\n and will not change existing parts of an environment unless you specify `--force`.\n This has always been true for `unify:false`, but not for `unify:true` and\n `unify:when_possible` environments. Now it is true for all of them (#37438, #37681).\n\n * The concretizer has a new `--reuse-deps` argument that *only* reuses dependencies.\n That is, it will always treat the *roots* of your environment as it would with\n `--fresh`. This allows you to upgrade just the roots of your environment while\n keeping everything else stable (#30990).\n\n5. **Weekly develop snapshot releases**\n\n Since last year, we have maintained a buildcache of `develop` at\n https://binaries.spack.io/develop, but the cache can grow to contain so many builds\n as to be unwieldy. When we get a stable `develop` build, we snapshot the release and\n add a corresponding tag the Spack repository. So, you can use a stack from a specific\n day. There are now tags in the spack repository like:\n\n * `develop-2023-05-14`\n * `develop-2023-05-18`\n\n that correspond to build caches like:\n\n * https://binaries.spack.io/develop-2023-05-14/e4s\n * https://binaries.spack.io/develop-2023-05-18/e4s\n\n We plan to store these snapshot releases weekly.\n\n6. **Specs in buildcaches can be referenced by hash.**\n\n * Previously, you could run `spack buildcache list` and see the hashes in\n buildcaches, but referring to them by hash would fail.\n * You can now run commands like `spack spec` and `spack install` and refer to\n buildcache hashes directly, e.g. `spack install /abc123` (#35042)\n\n7. **New package and buildcache index websites**\n\n Our public websites for searching packages have been completely revamped and updated.\n You can check them out here:\n\n * *Package Index*: https://packages.spack.io\n * *Buildcache Index*: https://cache.spack.io\n\n Both are searchable and more interactive than before. Currently major releases are\n shown; UI for browsing `develop` snapshots is coming soon.\n\n8. **Default CMake and Meson build types are now Release**\n\n Spack has historically defaulted to building with optimization and debugging, but\n packages like `llvm` can be enormous with debug turned on. Our default build type for\n all Spack packages is now `Release` (#36679, #37436). This has a number of benefits:\n\n * much smaller binaries;\n * higher default optimization level; and\n * defining `NDEBUG` disables assertions, which may lead to further speedups.\n\n You can still get the old behavior back through requirements and package preferences.\n\n## Other new commands and directives\n\n* `spack checksum` can automatically add new versions to package (#24532)\n* new command: `spack pkg grep` to easily search package files (#34388)\n* New `maintainers` directive (#35083)\n* Add `spack buildcache push` (alias to `buildcache create`) (#34861)\n* Allow using `-j` to control the parallelism of concretization (#37608)\n* Add `--exclude` option to 'spack external find' (#35013)\n\n## Other new features of note\n\n* editing: add higher-precedence `SPACK_EDITOR` environment variable\n* Many YAML formatting improvements from updating `ruamel.yaml` to the latest version\n supporting Python 3.6. (#31091, #24885, #37008).\n* Requirements and preferences should not define (non-git) versions (#37687, #37747)\n* Environments now store spack version/commit in `spack.lock` (#32801)\n* User can specify the name of the `packages` subdirectory in repositories (#36643)\n* Add container images supporting RHEL alternatives (#36713)\n* make version(...) kwargs explicit (#36998)\n\n## Notable refactors\n\n* buildcache create: reproducible tarballs (#35623)\n* Bootstrap most of Spack dependencies using environments (#34029)\n* Split `satisfies(..., strict=True/False)` into two functions (#35681)\n* spack install: simplify behavior when inside environments (#35206)\n\n## Binary cache and stack updates\n\n* Major simplification of CI boilerplate in stacks (#34272, #36045)\n* Many improvements to our CI pipeline's reliability\n\n## Removals, Deprecations, and disablements\n* Module file generation is disabled by default; you'll need to enable it to use it (#37258)\n* Support for Python 2 was deprecated in `v0.19.0` and has been removed. `v0.20.0` only\n supports Python 3.6 and higher.\n* Deprecated target names are no longer recognized by Spack. Use generic names instead:\n * `graviton` is now `cortex_a72`\n * `graviton2` is now `neoverse_n1`\n * `graviton3` is now `neoverse_v1`\n* `blacklist` and `whitelist` in module configuration were deprecated in `v0.19.0` and are\n removed in this release. Use `exclude` and `include` instead.\n* The `ignore=` parameter of the `extends()` directive has been removed. It was not used by\n any builtin packages and is no longer needed to avoid conflicts in environment views (#35588).\n* Support for the old YAML buildcache format has been removed. It was deprecated in `v0.19.0` (#34347).\n* `spack find --bootstrap` has been removed. It was deprecated in `v0.19.0`. Use `spack\n --bootstrap find` instead (#33964).\n* `spack bootstrap trust` and `spack bootstrap untrust` are now removed, having been\n deprecated in `v0.19.0`. Use `spack bootstrap enable` and `spack bootstrap disable`.\n* The `--mirror-name`, `--mirror-url`, and `--directory` options to buildcache and\n mirror commands were deprecated in `v0.19.0` and have now been removed. They have been\n replaced by positional arguments (#37457).\n* Deprecate `env:` as top level environment key (#37424)\n* deprecate buildcache create --rel, buildcache install --allow-root (#37285)\n* Support for very old perl-like spec format strings (e.g., `$_$@$%@+$+$=`) has been\n removed (#37425). This was deprecated in in `v0.15` (#10556).\n\n## Notable Bugfixes\n\n* bugfix: don't fetch package metadata for unknown concrete specs (#36990)\n* Improve package source code context display on error (#37655)\n* Relax environment manifest filename requirements and lockfile identification criteria (#37413)\n* `installer.py`: drop build edges of installed packages by default (#36707)\n* Bugfix: package requirements with git commits (#35057, #36347)\n* Package requirements: allow single specs in requirement lists (#36258)\n* conditional variant values: allow boolean (#33939)\n* spack uninstall: follow run/link edges on --dependents (#34058)\n\n## Spack community stats\n\n* 7,179 total packages, 499 new since `v0.19.0`\n * 329 new Python packages\n * 31 new R packages\n* 336 people contributed to this release\n * 317 committers to packages\n * 62 committers to core\n\n\n# v0.19.1 (2023-02-07)\n\n### Spack Bugfixes\n\n* `buildcache create`: make \"file exists\" less verbose (#35019)\n* `spack mirror create`: don't change paths to urls (#34992)\n* Improve error message for requirements (#33988)\n* uninstall: fix accidental cubic complexity (#34005)\n* scons: fix signature for `install_args` (#34481)\n* Fix `combine_phase_logs` text encoding issues (#34657)\n* Use a module-like object to propagate changes in the MRO, when setting build env (#34059)\n* PackageBase should not define builder legacy attributes (#33942)\n* Forward lookup of the \"run_tests\" attribute (#34531)\n* Bugfix for timers (#33917, #33900)\n* Fix path handling in prefix inspections (#35318)\n* Fix libtool filter for Fujitsu compilers (#34916)\n* Bug fix for duplicate rpath errors on macOS when creating build caches (#34375)\n* FileCache: delete the new cache file on exception (#34623)\n* Propagate exceptions from Spack python console (#34547)\n* Tests: Fix a bug/typo in a `config_values.py` fixture (#33886)\n* Various CI fixes (#33953, #34560, #34560, #34828)\n* Docs: remove monitors and analyzers, typos (#34358, #33926)\n* bump release version for tutorial command (#33859)\n\n\n# v0.19.0 (2022-11-11)\n\n`v0.19.0` is a major feature release.\n\n## Major features in this release\n\n1. **Package requirements**\n\n Spack's traditional [package preferences](\n https://spack.readthedocs.io/en/latest/build_settings.html#package-preferences)\n are soft, but we've added hard requirements to `packages.yaml` and `spack.yaml`\n (#32528, #32369). Package requirements use the same syntax as specs:\n\n ```yaml\n packages:\n libfabric:\n require: \"@1.13.2\"\n mpich:\n require:\n - one_of: [\"+cuda\", \"+rocm\"]\n ```\n\n More details in [the docs](\n https://spack.readthedocs.io/en/latest/build_settings.html#package-requirements).\n\n2. **Environment UI Improvements**\n\n * Fewer surprising modifications to `spack.yaml` (#33711):\n\n * `spack install` in an environment will no longer add to the `specs:` list; you'll\n need to either use `spack add ` or `spack install --add `.\n\n * Similarly, `spack uninstall` will not remove from your environment's `specs:`\n list; you'll need to use `spack remove` or `spack uninstall --remove`.\n\n This will make it easier to manage an environment, as there is clear separation\n between the stack to be installed (`spack.yaml`/`spack.lock`) and which parts of\n it should be installed (`spack install` / `spack uninstall`).\n\n * `concretizer:unify:true` is now the default mode for new environments (#31787)\n\n We see more users creating `unify:true` environments now. Users who need\n `unify:false` can add it to their environment to get the old behavior. This will\n concretize every spec in the environment independently.\n\n * Include environment configuration from URLs (#29026, [docs](\n https://spack.readthedocs.io/en/latest/environments.html#included-configurations))\n\n You can now include configuration in your environment directly from a URL:\n\n ```yaml\n spack:\n include:\n - https://github.com/path/to/raw/config/compilers.yaml\n ```\n\n4. **Multiple Build Systems**\n\n An increasing number of packages in the ecosystem need the ability to support\n multiple build systems (#30738, [docs](\n https://spack.readthedocs.io/en/latest/packaging_guide.html#multiple-build-systems)),\n either across versions, across platforms, or within the same version of the software.\n This has been hard to support through multiple inheritance, as methods from different\n build system superclasses would conflict. `package.py` files can now define separate\n builder classes with installation logic for different build systems, e.g.:\n\n ```python\n class ArpackNg(CMakePackage, AutotoolsPackage):\n\n build_system(\n conditional(\"cmake\", when=\"@0.64:\"),\n conditional(\"autotools\", when=\"@:0.63\"),\n default=\"cmake\",\n )\n\n class CMakeBuilder(spack.build_systems.cmake.CMakeBuilder):\n def cmake_args(self):\n pass\n\n class Autotoolsbuilder(spack.build_systems.autotools.AutotoolsBuilder):\n def configure_args(self):\n pass\n ```\n\n5. **Compiler and variant propagation**\n\n Currently, compiler flags and variants are inconsistent: compiler flags set for a\n package are inherited by its dependencies, while variants are not. We should have\n these be consistent by allowing for inheritance to be enabled or disabled for both\n variants and compiler flags.\n\n Example syntax:\n - `package ++variant`:\n enabled variant that will be propagated to dependencies\n - `package +variant`:\n enabled variant that will NOT be propagated to dependencies\n - `package ~~variant`:\n disabled variant that will be propagated to dependencies\n - `package ~variant`:\n disabled variant that will NOT be propagated to dependencies\n - `package cflags==-g`:\n `cflags` will be propagated to dependencies\n - `package cflags=-g`:\n `cflags` will NOT be propagated to dependencies\n\n Syntax for non-boolean variants is similar to compiler flags. More in the docs for\n [variants](\n https://spack.readthedocs.io/en/latest/basic_usage.html#variants) and [compiler flags](\n https://spack.readthedocs.io/en/latest/basic_usage.html#compiler-flags).\n\n6. **Enhancements to git version specifiers**\n\n * `v0.18.0` added the ability to use git commits as versions. You can now use the\n `git.` prefix to specify git tags or branches as versions. All of these are valid git\n versions in `v0.19` (#31200):\n\n ```console\n foo@abcdef1234abcdef1234abcdef1234abcdef1234 # raw commit\n foo@git.abcdef1234abcdef1234abcdef1234abcdef1234 # commit with git prefix\n foo@git.develop # the develop branch\n foo@git.0.19 # use the 0.19 tag\n ```\n\n * `v0.19` also gives you more control over how Spack interprets git versions, in case\n Spack cannot detect the version from the git repository. You can suffix a git\n version with `=` to force Spack to concretize it as a particular version\n (#30998, #31914, #32257):\n\n ```console\n # use mybranch, but treat it as version 3.2 for version comparison\n foo@git.mybranch=3.2\n\n # use the given commit, but treat it as develop for version comparison\n foo@git.abcdef1234abcdef1234abcdef1234abcdef1234=develop\n ```\n\n More in [the docs](\n https://spack.readthedocs.io/en/latest/basic_usage.html#version-specifier)\n\n7. **Changes to Cray EX Support**\n\n Cray machines have historically had their own \"platform\" within Spack, because we\n needed to go through the module system to leverage compilers and MPI installations on\n these machines. The Cray EX programming environment now provides standalone `craycc`\n executables and proper `mpicc` wrappers, so Spack can treat EX machines like Linux\n with extra packages (#29392).\n\n We expect this to greatly reduce bugs, as external packages and compilers can now be\n used by prefix instead of through modules. We will also no longer be subject to\n reproducibility issues when modules change from Cray PE release to release and from\n site to site. This also simplifies dealing with the underlying Linux OS on cray\n systems, as Spack will properly model the machine's OS as either SuSE or RHEL.\n\n8. **Improvements to tests and testing in CI**\n\n * `spack ci generate --tests` will generate a `.gitlab-ci.yml` file that not only does\n builds but also runs tests for built packages (#27877). Public GitHub pipelines now\n also run tests in CI.\n\n * `spack test run --explicit` will only run tests for packages that are explicitly\n installed, instead of all packages.\n\n9. **Experimental binding link model**\n\n You can add a new option to `config.yaml` to make Spack embed absolute paths to\n needed shared libraries in ELF executables and shared libraries on Linux (#31948, [docs](\n https://spack.readthedocs.io/en/latest/config_yaml.html#shared-linking-bind)):\n\n ```yaml\n config:\n shared_linking:\n type: rpath\n bind: true\n ```\n\n This can improve launch time at scale for parallel applications, and it can make\n installations less susceptible to environment variables like `LD_LIBRARY_PATH`, even\n especially when dealing with external libraries that use `RUNPATH`. You can think of\n this as a faster, even higher-precedence version of `RPATH`.\n\n## Other new features of note\n\n* `spack spec` prints dependencies more legibly. Dependencies in the output now appear\n at the *earliest* level of indentation possible (#33406)\n* You can override `package.py` attributes like `url`, directly in `packages.yaml`\n (#33275, [docs](\n https://spack.readthedocs.io/en/latest/build_settings.html#assigning-package-attributes))\n* There are a number of new architecture-related format strings you can use in Spack\n configuration files to specify paths (#29810, [docs](\n https://spack.readthedocs.io/en/latest/configuration.html#config-file-variables))\n* Spack now supports bootstrapping Clingo on Windows (#33400)\n* There is now support for an `RPATH`-like library model on Windows (#31930)\n\n## Performance Improvements\n\n* Major performance improvements for installation from binary caches (#27610, #33628,\n #33636, #33608, #33590, #33496)\n* Test suite can now be parallelized using `xdist` (used in GitHub Actions) (#32361)\n* Reduce lock contention for parallel builds in environments (#31643)\n\n## New binary caches and stacks\n\n* We now build nearly all of E4S with `oneapi` in our buildcache (#31781, #31804,\n #31804, #31803, #31840, #31991, #32117, #32107, #32239)\n* Added 3 new machine learning-centric stacks to binary cache: `x86_64_v3`, CUDA, ROCm\n (#31592, #33463)\n\n## Removals and Deprecations\n\n* Support for Python 3.5 is dropped (#31908). Only Python 2.7 and 3.6+ are officially\n supported.\n\n* This is the last Spack release that will support Python 2 (#32615). Spack `v0.19`\n will emit a deprecation warning if you run it with Python 2, and Python 2 support will\n soon be removed from the `develop` branch.\n\n* `LD_LIBRARY_PATH` is no longer set by default by `spack load` or module loads.\n\n Setting `LD_LIBRARY_PATH` in Spack environments/modules can cause binaries from\n outside of Spack to crash, and Spack's own builds use `RPATH` and do not need\n `LD_LIBRARY_PATH` set in order to run. If you still want the old behavior, you\n can run these commands to configure Spack to set `LD_LIBRARY_PATH`:\n\n ```console\n spack config add modules:prefix_inspections:lib64:[LD_LIBRARY_PATH]\n spack config add modules:prefix_inspections:lib:[LD_LIBRARY_PATH]\n ```\n\n* The `spack:concretization:[together|separately]` has been removed after being\n deprecated in `v0.18`. Use `concretizer:unify:[true|false]`.\n* `config:module_roots` is no longer supported after being deprecated in `v0.18`. Use\n configuration in module sets instead (#28659, [docs](\n https://spack.readthedocs.io/en/latest/module_file_support.html)).\n* `spack activate` and `spack deactivate` are no longer supported, having been\n deprecated in `v0.18`. Use an environment with a view instead of\n activating/deactivating ([docs](\n https://spack.readthedocs.io/en/latest/environments.html#configuration-in-spack-yaml)).\n* The old YAML format for buildcaches is now deprecated (#33707). If you are using an\n old buildcache with YAML metadata you will need to regenerate it with JSON metadata.\n* `spack bootstrap trust` and `spack bootstrap untrust` are deprecated in favor of\n `spack bootstrap enable` and `spack bootstrap disable` and will be removed in `v0.20`.\n (#33600)\n* The `graviton2` architecture has been renamed to `neoverse_n1`, and `graviton3`\n is now `neoverse_v1`. Buildcaches using the old architecture names will need to be rebuilt.\n* The terms `blacklist` and `whitelist` have been replaced with `include` and `exclude`\n in all configuration files (#31569). You can use `spack config update` to\n automatically fix your configuration files.\n\n## Notable Bugfixes\n\n* Permission setting on installation now handles effective uid properly (#19980)\n* `buildable:true` for an MPI implementation now overrides `buildable:false` for `mpi` (#18269)\n* Improved error messages when attempting to use an unconfigured compiler (#32084)\n* Do not punish explicitly requested compiler mismatches in the solver (#30074)\n* `spack stage`: add missing --fresh and --reuse (#31626)\n* Fixes for adding build system executables like `cmake` to package scope (#31739)\n* Bugfix for binary relocation with aliased strings produced by newer `binutils` (#32253)\n\n## Spack community stats\n\n* 6,751 total packages, 335 new since `v0.18.0`\n * 141 new Python packages\n * 89 new R packages\n* 303 people contributed to this release\n * 287 committers to packages\n * 57 committers to core\n\n\n# v0.18.1 (2022-07-19)\n\n### Spack Bugfixes\n* Fix several bugs related to bootstrapping (#30834,#31042,#31180)\n* Fix a regression that was causing spec hashes to differ between\n Python 2 and Python 3 (#31092)\n* Fixed compiler flags for oneAPI and DPC++ (#30856)\n* Fixed several issues related to concretization (#31142,#31153,#31170,#31226)\n* Improved support for Cray manifest file and `spack external find` (#31144,#31201,#31173,#31186)\n* Assign a version to openSUSE Tumbleweed according to the GLIBC version\n in the system (#19895)\n* Improved Dockerfile generation for `spack containerize` (#29741,#31321)\n* Fixed a few bugs related to concurrent execution of commands (#31509,#31493,#31477)\n\n### Package updates\n* WarpX: add v22.06, fixed libs property (#30866,#31102)\n* openPMD: add v0.14.5, update recipe for @develop (#29484,#31023)\n\n# v0.18.0 (2022-05-28)\n\n`v0.18.0` is a major feature release.\n\n## Major features in this release\n\n1. **Concretizer now reuses by default**\n\n `spack install --reuse` was introduced in `v0.17.0`, and `--reuse`\n is now the default concretization mode. Spack will try hard to\n resolve dependencies using installed packages or binaries (#30396).\n\n To avoid reuse and to use the latest package configurations, (the\n old default), you can use `spack install --fresh`, or add\n configuration like this to your environment or `concretizer.yaml`:\n\n ```yaml\n concretizer:\n reuse: false\n ```\n\n2. **Finer-grained hashes**\n\n Spack hashes now include `link`, `run`, *and* `build` dependencies,\n as well as a canonical hash of package recipes. Previously, hashes\n only included `link` and `run` dependencies (though `build`\n dependencies were stored by environments). We coarsened the hash to\n reduce churn in user installations, but the new default concretizer\n behavior mitigates this concern and gets us reuse *and* provenance.\n You will be able to see the build dependencies of new installations\n with `spack find`. Old installations will not change and their\n hashes will not be affected. (#28156, #28504, #30717, #30861)\n\n3. **Improved error messages**\n\n Error handling with the new concretizer is now done with\n optimization criteria rather than with unsatisfiable cores, and\n Spack reports many more details about conflicting constraints.\n (#30669)\n\n4. **Unify environments when possible**\n\n Environments have thus far supported `concretization: together` or\n `concretization: separately`. These have been replaced by a new\n preference in `concretizer.yaml`:\n\n ```yaml\n concretizer:\n unify: [true|false|when_possible]\n ```\n\n `concretizer:unify:when_possible` will *try* to resolve a fully\n unified environment, but if it cannot, it will create multiple\n configurations of some packages where it has to. For large\n environments that previously had to be concretized separately, this\n can result in a huge speedup (40-50x). (#28941)\n\n5. **Automatically find externals on Cray machines**\n\n Spack can now automatically discover installed packages in the Cray\n Programming Environment by running `spack external find` (or `spack\n external read-cray-manifest` to *only* query the PE). Packages from\n the PE (e.g., `cray-mpich` are added to the database with full\n dependency information, and compilers from the PE are added to\n `compilers.yaml`. Available with the June 2022 release of the Cray\n Programming Environment. (#24894, #30428)\n\n6. **New binary format and hardened signing**\n\n Spack now has an updated binary format, with improvements for\n security. The new format has a detached signature file, and Spack\n verifies the signature before untarring or decompressing the binary\n package. The previous format embedded the signature in a `tar`\n file, which required the client to run `tar` *before* verifying\n (#30750). Spack can still install from build caches using the old\n format, but we encourage users to switch to the new format going\n forward.\n\n Production GitLab pipelines have been hardened to securely sign\n binaries. There is now a separate signing stage so that signing\n keys are never exposed to build system code, and signing keys are\n ephemeral and only live as long as the signing pipeline stage.\n (#30753)\n\n7. **Bootstrap mirror generation**\n\n The `spack bootstrap mirror` command can automatically create a\n mirror for bootstrapping the concretizer and other needed\n dependencies in an air-gapped environment. (#28556)\n\n8. **Nascent Windows support**\n\n Spack now has initial support for Windows. Spack core has been\n refactored to run in the Windows environment, and a small number of\n packages can now build for Windows. More details are\n [in the documentation](https://spack.rtfd.io/en/latest/getting_started.html#spack-on-windows)\n (#27021, #28385, many more)\n\n9. **Makefile generation**\n\n `spack env depfile` can be used to generate a `Makefile` from an\n environment, which can be used to build packages the environment\n in parallel on a single node. e.g.:\n\n ```console\n spack -e myenv env depfile > Makefile\n make\n ```\n\n Spack propagates `gmake` jobserver information to builds so that\n their jobs can share cores. (#30039, #30254, #30302, #30526)\n\n10. **New variant features**\n\n In addition to being conditional themselves, variants can now have\n [conditional *values*](https://spack.readthedocs.io/en/latest/packaging_guide.html#conditional-possible-values)\n that are only possible for certain configurations of a package. (#29530)\n\n Variants can be\n [declared \"sticky\"](https://spack.readthedocs.io/en/latest/packaging_guide.html#sticky-variants),\n which prevents them from being enabled or disabled by the\n concretizer. Sticky variants must be set explicitly by users\n on the command line or in `packages.yaml`. (#28630)\n\n* Allow conditional possible values in variants\n* Add a \"sticky\" property to variants\n\n\n## Other new features of note\n\n* Environment views can optionally link only `run` dependencies\n with `link:run` (#29336)\n* `spack external find --all` finds library-only packages in\n addition to build dependencies (#28005)\n* Customizable `config:license_dir` option (#30135)\n* `spack external find --path PATH` takes a custom search path (#30479)\n* `spack spec` has a new `--format` argument like `spack find` (#27908)\n* `spack concretize --quiet` skips printing concretized specs (#30272)\n* `spack info` now has cleaner output and displays test info (#22097)\n* Package-level submodule option for git commit versions (#30085, #30037)\n* Using `/hash` syntax to refer to concrete specs in an environment\n now works even if `/hash` is not installed. (#30276)\n\n## Major internal refactors\n\n* full hash (see above)\n* new develop versioning scheme `0.19.0-dev0`\n* Allow for multiple dependencies/dependents from the same package (#28673)\n* Splice differing virtual packages (#27919)\n\n## Performance Improvements\n\n* Concretization of large environments with `unify: when_possible` is\n much faster than concretizing separately (#28941, see above)\n* Single-pass view generation algorithm is 2.6x faster (#29443)\n\n## Archspec improvements\n\n* `oneapi` and `dpcpp` flag support (#30783)\n* better support for `M1` and `a64fx` (#30683)\n\n## Removals and Deprecations\n\n* Spack no longer supports Python `2.6` (#27256)\n* Removed deprecated `--run-tests` option of `spack install`;\n use `spack test` (#30461)\n* Removed deprecated `spack flake8`; use `spack style` (#27290)\n\n* Deprecate `spack:concretization` config option; use\n `concretizer:unify` (#30038)\n* Deprecate top-level module configuration; use module sets (#28659)\n* `spack activate` and `spack deactivate` are deprecated in favor of\n environments; will be removed in `0.19.0` (#29430; see also `link:run`\n in #29336 above)\n\n## Notable Bugfixes\n\n* Fix bug that broke locks with many parallel builds (#27846)\n* Many bugfixes and consistency improvements for the new concretizer\n and `--reuse` (#30357, #30092, #29835, #29933, #28605, #29694, #28848)\n\n## Packages\n\n* `CMakePackage` uses `CMAKE_INSTALL_RPATH_USE_LINK_PATH` (#29703)\n* Refactored `lua` support: `lua-lang` virtual supports both\n `lua` and `luajit` via new `LuaPackage` build system(#28854)\n* PythonPackage: now installs packages with `pip` (#27798)\n* Python: improve site_packages_dir handling (#28346)\n* Extends: support spec, not just package name (#27754)\n* `find_libraries`: search for both .so and .dylib on macOS (#28924)\n* Use stable URLs and `?full_index=1` for all github patches (#29239)\n\n## Spack community stats\n\n* 6,416 total packages, 458 new since `v0.17.0`\n * 219 new Python packages\n * 60 new R packages\n* 377 people contributed to this release\n * 337 committers to packages\n * 85 committers to core\n\n# v0.17.3 (2022-07-14)\n\n### Spack bugfixes\n\n* Fix missing chgrp on symlinks in package installations (#30743)\n* Allow having non-existing upstreams (#30744, #30746)\n* Fix `spack stage` with custom paths (#30448)\n* Fix failing call for `spack buildcache save-specfile` (#30637)\n* Fix globbing in compiler wrapper (#30699)\n\n# v0.17.2 (2022-04-13)\n\n### Spack bugfixes\n* Fix --reuse with upstreams set in an environment (#29680)\n* config add: fix parsing of validator error to infer type from oneOf (#29475)\n* Fix spack -C command_line_scope used in conjunction with other flags (#28418)\n* Use Spec.constrain to construct spec lists for stacks (#28783)\n* Fix bug occurring when searching for inherited patches in packages (#29574)\n* Fixed a few bugs when manipulating symlinks (#28318, #29515, #29636)\n* Fixed a few minor bugs affecting command prompt, terminal title and argument completion (#28279, #28278, #28939, #29405, #29070, #29402)\n* Fixed a few bugs affecting the spack ci command (#29518, #29419)\n* Fix handling of Intel compiler environment (#29439)\n* Fix a few edge cases when reindexing the DB (#28764)\n* Remove \"Known issues\" from documentation (#29664)\n* Other miscellaneous bugfixes (0b72e070583fc5bcd016f5adc8a84c99f2b7805f, #28403, #29261)\n\n# v0.17.1 (2021-12-23)\n\n### Spack Bugfixes\n* Allow locks to work under high contention (#27846)\n* Improve errors messages from clingo (#27707 #27970)\n* Respect package permissions for sbang (#25764)\n* Fix --enable-locks behavior (#24675)\n* Fix log-format reporter ignoring install errors (#25961)\n* Fix overloaded argparse keys (#27379)\n* Allow style commands to run with targets other than \"develop\" (#27472)\n* Log lock messages to debug level, instead of verbose level (#27408)\n* Handle invalid unicode while logging (#21447)\n* spack audit: fix API calls to variants (#27713)\n* Provide meaningful message for empty environment installs (#28031)\n* Added opensuse leap containers to spack containerize (#27837)\n* Revert \"patches: make re-applied patches idempotent\" (#27625)\n* MANPATH can use system defaults (#21682)\n* Add \"setdefault\" subcommand to `spack module tcl` (#14686)\n* Regenerate views when specs already installed (#28113)\n\n### Package bugfixes\n* Fix external package detection for OpenMPI (#27255)\n* Update the UPC++ package to 2021.9.0 (#26996)\n* Added py-vermin v1.3.2 (#28072)\n\n# v0.17.0 (2021-11-05)\n\n`v0.17.0` is a major feature release.\n\n## Major features in this release\n\n1. **New concretizer is now default**\n The new concretizer introduced as an experimental feature in `v0.16.0`\n is now the default (#25502). The new concretizer is based on the\n [clingo](https://github.com/potassco/clingo) logic programming system,\n and it enables us to do much higher quality and faster dependency solving\n The old concretizer is still available via the `concretizer: original`\n setting, but it is deprecated and will be removed in `v0.18.0`.\n\n2. **Binary Bootstrapping**\n To make it easier to use the new concretizer and binary packages,\n Spack now bootstraps `clingo` and `GnuPG` from public binaries. If it\n is not able to bootstrap them from binaries, it installs them from\n source code. With these changes, you should still be able to clone Spack\n and start using it almost immediately. (#21446, #22354, #22489, #22606,\n #22720, #22720, #23677, #23946, #24003, #25138, #25607, #25964, #26029,\n #26399, #26599).\n\n3. **Reuse existing packages (experimental)**\n The most wanted feature from our\n [2020 user survey](https://spack.io/spack-user-survey-2020/) and\n the most wanted Spack feature of all time (#25310). `spack install`,\n `spack spec`, and `spack concretize` now have a `--reuse` option, which\n causes Spack to minimize the number of rebuilds it does. The `--reuse`\n option will try to find existing installations and binary packages locally\n and in registered mirrors, and will prefer to use them over building new\n versions. This will allow users to build from source *far* less than in\n prior versions of Spack. This feature will continue to be improved, with\n configuration options and better CLI expected in `v0.17.1`. It will become\n the *default* concretization mode in `v0.18.0`.\n\n4. **Better error messages**\n We have improved the error messages generated by the new concretizer by\n using *unsatisfiable cores*. Spack will now print a summary of the types\n of constraints that were violated to make a spec unsatisfiable (#26719).\n\n5. **Conditional variants**\n Variants can now have a `when=\"\"` clause, allowing them to be\n conditional based on the version or other attributes of a package (#24858).\n\n6. **Git commit versions**\n In an environment and on the command-line, you can now provide a full,\n 40-character git commit as a version for any package with a top-level\n `git` URL. e.g., `spack install hdf5@45bb27f58240a8da7ebb4efc821a1a964d7712a8`.\n Spack will compare the commit to tags in the git repository to understand\n what versions it is ahead of or behind.\n\n7. **Override local config and cache directories**\n You can now set `SPACK_DISABLE_LOCAL_CONFIG` to disable the `~/.spack` and\n `/etc/spack` configuration scopes. `SPACK_USER_CACHE_PATH` allows you to\n move caches out of `~/.spack`, as well (#27022, #26735). This addresses\n common problems where users could not isolate CI environments from local\n configuration.\n\n8. **Improvements to Spack Containerize**\n For added reproducibility, you can now pin the Spack version used by\n `spack containerize` (#21910). The container build will only build\n with the Spack version pinned at build recipe creation instead of the\n latest Spack version.\n\n9. **New commands for dealing with tags**\n The `spack tags` command allows you to list tags on packages (#26136), and you\n can list tests and filter tags with `spack test list` (#26842).\n\n## Other new features of note\n\n* Copy and relocate environment views as stand-alone installations (#24832)\n* `spack diff` command can diff two installed specs (#22283, #25169)\n* `spack -c ` can set one-off config parameters on CLI (#22251)\n* `spack load --list` is an alias for `spack find --loaded` (#27184)\n* `spack gpg` can export private key with `--secret` (#22557)\n* `spack style` automatically bootstraps dependencies (#24819)\n* `spack style --fix` automatically invokes `isort` (#24071)\n* build dependencies can be installed from build caches with `--include-build-deps` (#19955)\n* `spack audit` command for checking package constraints (#23053)\n* Spack can now fetch from `CVS` repositories (yep, really) (#23212)\n* `spack monitor` lets you upload analysis about installations to a\n [spack monitor server](https://github.com/spack/spack-monitor) (#23804, #24321,\n #23777, #25928))\n* `spack python --path` shows which `python` Spack is using (#22006)\n* `spack env activate --temp` can create temporary environments (#25388)\n* `--preferred` and `--latest` options for `spack checksum` (#25830)\n* `cc` is now pure posix and runs on Alpine (#26259)\n* `SPACK_PYTHON` environment variable sets which `python` spack uses (#21222)\n* `SPACK_SKIP_MODULES` lets you source `setup-env.sh` faster if you don't need modules (#24545)\n\n## Major internal refactors\n\n* `spec.yaml` files are now `spec.json`, yielding a large speed improvement (#22845)\n* Splicing allows Spack specs to store mixed build provenance (#20262)\n* More extensive hooks API for installations (#21930)\n* New internal API for getting the active environment (#25439)\n\n## Performance Improvements\n\n* Parallelize separate concretization in environments; Previously 55 min E4S solve\n now takes 2.5 min (#26264)\n* Drastically improve YamlFilesystemView file removal performance via batching (#24355)\n* Speed up spec comparison (#21618)\n* Speed up environment activation (#25633)\n\n## Archspec improvements\n* support for new generic `x86_64_v2`, `x86_64_v3`, `x86_64_v4` targets\n (see [archspec#31](https://github.com/archspec/archspec-json/pull/31))\n* `spack arch --generic` lets you get the best generic architecture for\n your node (#27061)\n* added support for aocc (#20124), `arm` compiler on `graviton2` (#24904)\n and on `a64fx` (#24524),\n\n## Infrastructure, buildcaches, and services\n\n* Add support for GCS Bucket Mirrors (#26382)\n* Add `spackbot` to help package maintainers with notifications. See\n [spack.github.io/spackbot](https://spack.github.io/spackbot/)\n* Reproducible pipeline builds with `spack ci rebuild` (#22887)\n* Removed redundant concretizations from GitLab pipeline generation (#26622)\n* Spack CI no longer generates jobs for unbuilt specs (#20435)\n* Every pull request pipeline has its own buildcache (#25529)\n* `--no-add` installs only specified specs and only if already present in… (#22657)\n* Add environment-aware `spack buildcache sync` command (#25470)\n* Binary cache installation speedups and improvements (#19690, #20768)\n\n## Deprecations and Removals\n\n* `spack setup` was deprecated in v0.16.0, and has now been removed.\n Use `spack develop` and `spack dev-build`.\n* Remove unused `--dependencies` flag from `spack load` (#25731)\n* Remove stubs for `spack module [refresh|find|rm|loads]`, all of which\n were deprecated in 2018.\n\n## Notable Bugfixes\n\n* Deactivate previous env before activating new one (#25409)\n* Many fixes to error codes from `spack install` (#21319, #27012, #25314)\n* config add: infer type based on JSON schema validation errors (#27035)\n* `spack config edit` now works even if `spack.yaml` is broken (#24689)\n\n## Packages\n\n* Allow non-empty version ranges like `1.1.0:1.1` (#26402)\n* Remove `.99`'s from many version ranges (#26422)\n* Python: use platform-specific site packages dir (#25998)\n* `CachedCMakePackage` for using *.cmake initial config files (#19316)\n* `lua-lang` allows swapping `lua` and `luajit` (#22492)\n* Better support for `ld.gold` and `ld.lld` (#25626)\n* build times are now stored as metadata in `$prefix/.spack` (#21179)\n* post-install tests can be reused in smoke tests (#20298)\n* Packages can use `pypi` attribute to infer `homepage`/`url`/`list_url` (#17587)\n* Use gnuconfig package for `config.guess` file replacement (#26035)\n* patches: make re-applied patches idempotent (#26784)\n\n## Spack community stats\n\n* 5969 total packages, 920 new since `v0.16.0`\n * 358 new Python packages, 175 new R packages\n* 513 people contributed to this release\n * 490 committers to packages\n * 105 committers to core\n* Lots of GPU updates:\n * ~77 CUDA-related commits\n * ~66 AMD-related updates\n * ~27 OneAPI-related commits\n * 30 commits from AMD toolchain support\n* `spack test` usage in packages is increasing\n * 1669 packages with tests (mostly generic python tests)\n * 93 packages with their own tests\n\n\n# v0.16.3 (2021-09-21)\n\n* clang/llvm: fix version detection (#19978)\n* Fix use of quotes in Python build system (#22279)\n* Cray: fix extracting paths from module files (#23472)\n* Use AWS CloudFront for source mirror (#23978)\n* Ensure all roots of an installed environment are marked explicit in db (#24277)\n* Fix fetching for Python 3.8 and 3.9 (#24686)\n* locks: only open lockfiles once instead of for every lock held (#24794)\n* Remove the EOL centos:6 docker image\n\n# v0.16.2 (2021-05-22)\n\n* Major performance improvement for `spack load` and other commands. (#23661)\n* `spack fetch` is now environment-aware. (#19166)\n* Numerous fixes for the new, `clingo`-based concretizer. (#23016, #23307,\n #23090, #22896, #22534, #20644, #20537, #21148)\n* Support for automatically bootstrapping `clingo` from source. (#20652, #20657\n #21364, #21446, #21913, #22354, #22444, #22460, #22489, #22610, #22631)\n* Python 3.10 support: `collections.abc` (#20441)\n* Fix import issues by using `__import__` instead of Spack package importe.\n (#23288, #23290)\n* Bugfixes and `--source-dir` argument for `spack location`. (#22755, #22348,\n #22321)\n* Better support for externals in shared prefixes. (#22653)\n* `spack build-env` now prefers specs defined in the active environment.\n (#21642)\n* Remove erroneous warnings about quotes in `from_sourcing_files`. (#22767)\n* Fix clearing cache of `InternalConfigScope`. (#22609)\n* Bugfix for active when pkg is already active error. (#22587)\n* Make `SingleFileScope` able to repopulate the cache after clearing it.\n (#22559)\n* Channelflow: Fix the package. (#22483)\n* More descriptive error message for bugs in `package.py` (#21811)\n* Use package-supplied `autogen.sh`. (#20319)\n* Respect `-k/verify-ssl-false` in `_existing_url` method. (#21864)\n\n\n# v0.16.1 (2021-02-22)\n\nThis minor release includes a new feature and associated fixes:\n* intel-oneapi support through new packages (#20411, #20686, #20693, #20717,\n #20732, #20808, #21377, #21448)\n\nThis release also contains bug fixes/enhancements for:\n* HIP/ROCm support (#19715, #20095)\n* concretization (#19988, #20020, #20082, #20086, #20099, #20102, #20128,\n #20182, #20193, #20194, #20196, #20203, #20247, #20259, #20307, #20362,\n #20383, #20423, #20473, #20506, #20507, #20604, #20638, #20649, #20677,\n #20680, #20790)\n* environment install reporting fix (#20004)\n* avoid import in ABI compatibility info (#20236)\n* restore ability of dev-build to skip patches (#20351)\n* spack find -d spec grouping (#20028)\n* spack smoke test support (#19987, #20298)\n* macOS fixes (#20038, #21662)\n* abstract spec comparisons (#20341)\n* continuous integration (#17563)\n* performance improvements for binary relocation (#19690, #20768)\n* additional sanity checks for variants in builtin packages (#20373)\n* do not pollute auto-generated configuration files with empty lists or\n dicts (#20526)\n\nplus assorted documentation (#20021, #20174) and package bug fixes/enhancements\n(#19617, #19933, #19986, #20006, #20097, #20198, #20794, #20906, #21411).\n\n\n# v0.16.0 (2020-11-18)\n\n`v0.16.0` is a major feature release.\n\n## Major features in this release\n\n1. **New concretizer (experimental)** Our new backtracking concretizer is\n now in Spack as an experimental feature. You will need to install\n `clingo@master+python` and set `concretizer: clingo` in `config.yaml`\n to use it. The original concretizer is not exhaustive and is not\n guaranteed to find a solution if one exists. We encourage you to use\n the new concretizer and to report any bugs you find with it. We\n anticipate making the new concretizer the default and including all\n required dependencies for it in Spack `v0.17`. For more details, see\n #19501.\n\n2. **spack test (experimental)** Users can add `test()` methods to their\n packages to run smoke tests on installations with the new `spack test`\n command (the old `spack test` is now `spack unit-test`). `spack test`\n is environment-aware, so you can `spack install` an environment and\n `spack test run` smoke tests on all of its packages. Historical test\n logs can be perused with `spack test results`. Generic smoke tests for\n MPI implementations, C, C++, and Fortran compilers as well as specific\n smoke tests for 18 packages. This is marked experimental because the\n test API (`self.run_test()`) is likely to be change, but we encourage\n users to upstream tests, and we will maintain and refactor any that\n are added to mainline packages (#15702).\n\n3. **spack develop** New `spack develop` command allows you to develop\n several packages at once within a Spack environment. Running\n `spack develop foo@v1` and `spack develop bar@v2` will check\n out specific versions of `foo` and `bar` into subdirectories, which you\n can then build incrementally with `spack install ` (#15256).\n\n4. **More parallelism** Spack previously installed the dependencies of a\n _single_ spec in parallel. Entire environments can now be installed in\n parallel, greatly accelerating builds of large environments. get\n parallelism from individual specs. Spack now parallelizes entire\n environment builds (#18131).\n\n5. **Customizable base images for spack containerize**\n `spack containerize` previously only output a `Dockerfile` based\n on `ubuntu`. You may now specify any base image of your choosing (#15028).\n\n6. **more external finding** `spack external find` was added in `v0.15`,\n but only `cmake` had support. `spack external find` can now find\n `bison`, `cuda`, `findutils`, `flex`, `git`, `lustre` `m4`, `mpich`,\n `mvapich2`, `ncurses`, `openmpi`, `perl`, `spectrum-mpi`, `tar`, and\n `texinfo` on your system and add them automatically to\n `packages.yaml`.\n\n7. **Support aocc, nvhpc, and oneapi compilers** We are aggressively\n pursuing support for the newest vendor compilers, especially those for\n the U.S. exascale and pre-exascale systems. Compiler classes and\n auto-detection for `aocc`, `nvhpc`, `oneapi` are now in Spack (#19345,\n #19294, #19330).\n\n## Additional new features of note\n\n* New `spack mark` command can be used to designate packages as explicitly\n installed, so that `spack gc` will not garbage-collect them (#16662).\n* `install_tree` can be customized with Spack's projection format (#18341)\n* `sbang` now lives in the `install_tree` so that all users can access it (#11598)\n* `csh` and `tcsh` users no longer need to set `SPACK_ROOT` before\n sourcing `setup-env.csh` (#18225)\n* Spec syntax now supports `variant=*` syntax for finding any package\n that has a particular variant (#19381).\n* Spack respects `SPACK_GNUPGHOME` variable for custom GPG directories (#17139)\n* Spack now recognizes Graviton chips\n\n## Major refactors\n\n* Use spawn instead of fork on Python >= 3.8 on macOS (#18205)\n* Use indexes for public build caches (#19101, #19117, #19132, #19141, #19209)\n* `sbang` is an external package now (https://github.com/spack/sbang, #19582)\n* `archspec` is an external package now (https://github.com/archspec/archspec, #19600)\n\n## Deprecations and Removals\n\n* `spack bootstrap` was deprecated in v0.14.0, and has now been removed.\n* `spack setup` is deprecated as of v0.16.0.\n* What was `spack test` is now called `spack unit-test`. `spack test` is\n now the smoke testing feature in (2) above.\n\n## Bugfixes\n\nSome of the most notable bugfixes in this release include:\n\n* Better warning messages for deprecated syntax in `packages.yaml` (#18013)\n* `buildcache list --allarch` now works properly (#17827)\n* Many fixes and tests for buildcaches and binary relcoation (#15687,\n *#17455, #17418, #17455, #15687, #18110)\n\n## Package Improvements\n\nSpack now has 5050 total packages, 720 of which were added since `v0.15`.\n\n* ROCm packages (`hip`, `aomp`, more) added by AMD (#19957, #19832, others)\n* Many improvements for ARM support\n* `llvm-flang`, `flang`, and `f18` removed, as `llvm` has real `flang`\n support since Flang was merged to LLVM mainline\n* Emerging support for `spack external find` and `spack test` in packages.\n\n## Infrastructure\n\n* Major infrastructure improvements to pipelines on `gitlab.spack.io`\n* Support for testing PRs from forks (#19248) is being enabled for all\n forks to enable rolling, up-to-date binary builds on `develop`\n\n\n# v0.15.4 (2020-08-12)\n\nThis release contains one feature addition:\n\n* Users can set `SPACK_GNUPGHOME` to override Spack's GPG path (#17139)\n\nSeveral bugfixes for CUDA, binary packaging, and `spack -V`:\n\n* CUDA package's `.libs` method searches for `libcudart` instead of `libcuda` (#18000)\n* Don't set `CUDAHOSTCXX` in environments that contain CUDA (#17826)\n* `buildcache create`: `NoOverwriteException` is a warning, not an error (#17832)\n* Fix `spack buildcache list --allarch` (#17884)\n* `spack -V` works with `releases/latest` tag and shallow clones (#17884)\n\nAnd fixes for GitHub Actions and tests to ensure that CI passes on the\nrelease branch (#15687, #17279, #17328, #17377, #17732).\n\n# v0.15.3 (2020-07-28)\n\nThis release contains the following bugfixes:\n\n* Fix handling of relative view paths (#17721)\n* Fixes for binary relocation (#17418, #17455)\n* Fix redundant printing of error messages in build environment (#17709)\n\nIt also adds a support script for Spack tutorials:\n\n* Add a tutorial setup script to share/spack (#17705, #17722)\n\n# v0.15.2 (2020-07-23)\n\nThis minor release includes two new features:\n\n* Spack install verbosity is decreased, and more debug levels are added (#17546)\n* The $spack/share/spack/keys directory contains public keys that may be optionally trusted for public binary mirrors (#17684)\n\nThis release also includes several important fixes:\n\n* MPICC and related variables are now cleand in the build environment (#17450)\n* LLVM flang only builds CUDA offload components when +cuda (#17466)\n* CI pipelines no longer upload user environments that can contain secrets to the internet (#17545)\n* CI pipelines add bootstrapped compilers to the compiler config (#17536)\n* `spack buildcache list` does not exit on first failure and lists later mirrors (#17565)\n* Apple's \"gcc\" executable that is an apple-clang compiler does not generate a gcc compiler config (#17589)\n* Mixed compiler toolchains are merged more naturally across different compiler suffixes (#17590)\n* Cray Shasta platforms detect the OS properly (#17467)\n* Additional more minor fixes.\n\n# v0.15.1 (2020-07-10)\n\nThis minor release includes several important fixes:\n\n* Fix shell support on Cray (#17386)\n* Fix use of externals installed with other Spack instances (#16954)\n* Fix gcc+binutils build (#9024)\n* Fixes for usage of intel-mpi (#17378 and #17382)\n* Fixes to Autotools config.guess detection (#17333 and #17356)\n* Update `spack install` message to prompt user when an environment is not\n explicitly activated (#17454)\n\nThis release also adds a mirror for all sources that are\nfetched in Spack (#17077). It is expected to be useful when the\nofficial website for a Spack package is unavailable.\n\n# v0.15.0 (2020-06-28)\n\n`v0.15.0` is a major feature release.\n\n## Major Features in this release\n\n1. **Cray support** Spack will now work properly on Cray \"Cluster\"\nsystems (non XC systems) and after a `module purge` command on Cray\nsystems. See #12989\n\n2. **Virtual package configuration** Virtual packages are allowed in\npackages.yaml configuration. This allows users to specify a virtual\npackage as non-buildable without needing to specify for each\nimplementation. See #14934\n\n3. **New config subcommands** This release adds `spack config add` and\n`spack config remove` commands to add to and remove from yaml\nconfiguration files from the CLI. See #13920\n\n4. **Environment activation** Anonymous environments are **no longer**\nautomatically activated in the current working directory. To activate\nan environment from a `spack.yaml` file in the current directory, use\nthe `spack env activate .` command. This removes a concern that users\nwere too easily polluting their anonymous environments with accidental\ninstallations. See #17258\n\n5. **Apple clang compiler** The clang compiler and the apple-clang\ncompiler are now separate compilers in Spack. This allows Spack to\nimprove support for the apple-clang compiler. See #17110\n\n6. **Finding external packages** Spack packages can now support an API\nfor finding external installations. This allows the `spack external\nfind` command to automatically add installations of those packages to\nthe user's configuration. See #15158\n\n\n## Additional new features of note\n\n* support for using Spack with the fish shell (#9279)\n* `spack load --first` option to load first match (instead of prompting user) (#15622)\n* support the Cray cce compiler both new and classic versions (#17256, #12989)\n* `spack dev-build` command:\n * supports stopping before a specified phase (#14699)\n * supports automatically launching a shell in the build environment (#14887)\n* `spack install --fail-fast` allows builds to fail at the first error (rather than best-effort) (#15295)\n* environments: SpecList references can be dereferenced as compiler or dependency constraints (#15245)\n* `spack view` command: new support for a copy/relocate view type (#16480)\n* ci pipelines: see documentation for several improvements\n* `spack mirror -a` command now supports excluding packages (#14154)\n* `spack buildcache create` is now environment-aware (#16580)\n* module generation: more flexible format for specifying naming schemes (#16629)\n* lmod module generation: packages can be configured as core specs for lmod hierarchy (#16517)\n\n## Deprecations and Removals\n\nThe following commands were deprecated in v0.13.0, and have now been removed:\n\n* `spack configure`\n* `spack build`\n* `spack diy`\n\nThe following commands were deprecated in v0.14.0, and will be removed in the next major release:\n\n* `spack bootstrap`\n\n## Bugfixes\n\nSome of the most notable bugfixes in this release include:\n\n* Spack environments can now contain the string `-h` (#15429)\n* The `spack install` gracefully handles being backgrounded (#15723, #14682)\n* Spack uses `-isystem` instead of `-I` in cases that the underlying build system does as well (#16077)\n* Spack no longer prints any specs that cannot be safely copied into a Spack command (#16462)\n* Incomplete Spack environments containing python no longer cause problems (#16473)\n* Several improvements to binary package relocation\n\n## Package Improvements\n\nThe Spack project is constantly engaged in routine maintenance,\nbugfixes, and improvements for the package ecosystem. Of particular\nnote in this release are the following:\n\n* Spack now contains 4339 packages. There are 430 newly supported packages in v0.15.0\n* GCC now builds properly on ARM architectures (#17280)\n* Python: patched to support compiling mixed C/C++ python modules through distutils (#16856)\n* improvements to pytorch and py-tensorflow packages\n* improvements to major MPI implementations: mvapich2, mpich, openmpi, and others\n\n## Spack Project Management:\n\n* Much of the Spack CI infrastructure has moved from Travis to GitHub Actions (#16610, #14220, #16345)\n* All merges to the `develop` branch run E4S CI pipeline (#16338)\n* New `spack debug report` command makes reporting bugs easier (#15834)\n\n# v0.14.2 (2020-04-15)\n\nThis is a minor release on the `0.14` series. It includes performance\nimprovements and bug fixes:\n\n* Improvements to how `spack install` handles foreground/background (#15723)\n* Major performance improvements for reading the package DB (#14693, #15777)\n* No longer check for the old `index.yaml` database file (#15298)\n* Properly activate environments with '-h' in the name (#15429)\n* External packages have correct `.prefix` in environments/views (#15475)\n* Improvements to computing env modifications from sourcing files (#15791)\n* Bugfix on Cray machines when getting `TERM` env variable (#15630)\n* Avoid adding spurious `LMOD` env vars to Intel modules (#15778)\n* Don't output [+] for mock installs run during tests (#15609)\n\n# v0.14.1 (2020-03-20)\n\nThis is a bugfix release on top of `v0.14.0`. Specific fixes include:\n\n* several bugfixes for parallel installation (#15339, #15341, #15220, #15197)\n* `spack load` now works with packages that have been renamed (#14348)\n* bugfix for `suite-sparse` installation (#15326)\n* deduplicate identical suffixes added to module names (#14920)\n* fix issues with `configure_args` during module refresh (#11084)\n* increased test coverage and test fixes (#15237, #15354, #15346)\n* remove some unused code (#15431)\n\n# v0.14.0 (2020-02-23)\n\n`v0.14.0` is a major feature release, with 3 highlighted features:\n\n1. **Distributed builds.** Multiple Spack instances will now coordinate\n properly with each other through locks. This works on a single node\n (where you've called `spack` several times) or across multiple nodes\n with a shared filesystem. For example, with SLURM, you could build\n `trilinos` and its dependencies on 2 24-core nodes, with 3 Spack\n instances per node and 8 build jobs per instance, with `srun -N 2 -n 6\n spack install -j 8 trilinos`. This requires a filesystem with locking\n enabled, but not MPI or any other library for parallelism.\n\n2. **Build pipelines.** You can also build in parallel through Gitlab\n CI. Simply create a Spack environment and push it to Gitlab to build\n on Gitlab runners. Pipeline support is now integrated into a single\n `spack ci` command, so setting it up is easier than ever. See the\n [Pipelines section](https://spack.readthedocs.io/en/v0.14.0/pipelines.html)\n in the docs.\n\n3. **Container builds.** The new `spack containerize` command allows you\n to create a Docker or Singularity recipe from any Spack environment.\n There are options to customize the build if you need them. See the\n [Container Images section](https://spack.readthedocs.io/en/latest/containers.html)\n in the docs.\n\nIn addition, there are several other new commands, many bugfixes and\nimprovements, and `spack load` no longer requires modules, so you can use\nit the same way on your laptop or on your supercomputer.\n\nSpack grew by over 300 packages since our last release in November 2019,\nand the project grew to over 500 contributors. Thanks to all of you for\nmaking yet another great release possible. Detailed notes below.\n\n## Major new core features\n* Distributed builds: spack instances coordinate and build in parallel (#13100)\n* New `spack ci` command to manage CI pipelines (#12854)\n* Generate container recipes from environments: `spack containerize` (#14202)\n* `spack load` now works without using modules (#14062, #14628)\n* Garbage collect old/unused installations with `spack gc` (#13534)\n* Configuration files all set environment modifications the same way (#14372,\n [docs](https://spack.readthedocs.io/en/v0.14.0/configuration.html#environment-modifications))\n* `spack commands --format=bash` auto-generates completion (#14393, #14607)\n* Packages can specify alternate fetch URLs in case one fails (#13881)\n\n## Improvements\n* Improved locking for concurrency with environments (#14676, #14621, #14692)\n* `spack test` sends args to `pytest`, supports better listing (#14319)\n* Better support for aarch64 and cascadelake microarch (#13825, #13780, #13820)\n* Archspec is now a separate library (see https://github.com/archspec/archspec)\n* Many improvements to the `spack buildcache` command (#14237, #14346,\n #14466, #14467, #14639, #14642, #14659, #14696, #14698, #14714, #14732,\n #14929, #15003, #15086, #15134)\n\n## Selected Bugfixes\n* Compilers now require an exact match on version (#8735, #14730, #14752)\n* Bugfix for patches that specified specific versions (#13989)\n* `spack find -p` now works in environments (#10019, #13972)\n* Dependency queries work correctly in `spack find` (#14757)\n* Bugfixes for locking upstream Spack instances chains (#13364)\n* Fixes for PowerPC clang optimization flags (#14196)\n* Fix for issue with compilers and specific microarchitectures (#13733, #14798)\n\n## New commands and options\n* `spack ci` (#12854)\n* `spack containerize` (#14202)\n* `spack gc` (#13534)\n* `spack load` accepts `--only package`, `--only dependencies` (#14062, #14628)\n* `spack commands --format=bash` (#14393)\n* `spack commands --update-completion` (#14607)\n* `spack install --with-cache` has new option: `--no-check-signature` (#11107)\n* `spack test` now has `--list`, `--list-long`, and `--list-names` (#14319)\n* `spack install --help-cdash` moves CDash help out of the main help (#13704)\n\n## Deprecations\n* `spack release-jobs` has been rolled into `spack ci`\n* `spack bootstrap` will be removed in a future version, as it is no longer\n needed to set up modules (see `spack load` improvements above)\n\n## Documentation\n* New section on building container images with Spack (see\n [docs](https://spack.readthedocs.io/en/latest/containers.html))\n* New section on using `spack ci` command to build pipelines (see\n [docs](https://spack.readthedocs.io/en/latest/pipelines.html))\n* Document how to add conditional dependencies (#14694)\n* Document how to use Spack to replace Homebrew/Conda (#13083, see\n [docs](https://spack.readthedocs.io/en/latest/workflows.html#using-spack-to-replace-homebrew-conda))\n\n## Important package changes\n* 3,908 total packages (345 added since 0.13.0)\n* Added first cut at a TensorFlow package (#13112)\n* We now build R without \"recommended\" packages, manage them w/Spack (#12015)\n* Elpa and OpenBLAS now leverage microarchitecture support (#13655, #14380)\n* Fix `octave` compiler wrapper usage (#14726)\n* Enforce that packages in `builtin` aren't missing dependencies (#13949)\n\n\n# v0.13.4 (2020-02-07)\n\nThis release contains several bugfixes:\n\n* bugfixes for invoking python in various environments (#14349, #14496, #14569)\n* brought tab completion up to date (#14392)\n* bugfix for removing extensions from views in order (#12961)\n* bugfix for nondeterministic hashing for specs with externals (#14390)\n\n# v0.13.3 (2019-12-23)\n\nThis release contains more major performance improvements for Spack\nenvironments, as well as bugfixes for mirrors and a `python` issue with\nRHEL8.\n\n* mirror bugfixes: symlinks, duplicate patches, and exception handling (#13789)\n* don't try to fetch `BundlePackages` (#13908)\n* avoid re-fetching patches already added to a mirror (#13908)\n* avoid re-fetching already added patches (#13908)\n* avoid re-fetching already added patches (#13908)\n* allow repeated invocations of `spack mirror create` on the same dir (#13908)\n* bugfix for RHEL8 when `python` is unavailable (#14252)\n* improve concretization performance in environments (#14190)\n* improve installation performance in environments (#14263)\n\n# v0.13.2 (2019-12-04)\n\nThis release contains major performance improvements for Spack environments, as\nwell as some bugfixes and minor changes.\n\n* allow missing modules if they are blacklisted (#13540)\n* speed up environment activation (#13557)\n* mirror path works for unknown versions (#13626)\n* environments: don't try to modify run-env if a spec is not installed (#13589)\n* use semicolons instead of newlines in module/python command (#13904)\n* verify.py: os.path.exists exception handling (#13656)\n* Document use of the maintainers field (#13479)\n* bugfix with config caching (#13755)\n* hwloc: added 'master' version pointing at the HEAD of the master branch (#13734)\n* config option to allow gpg warning suppression (#13744)\n* fix for relative symlinks when relocating binary packages (#13727)\n* allow binary relocation of strings in relative binaries (#13724)\n\n# v0.13.1 (2019-11-05)\n\nThis is a bugfix release on top of `v0.13.0`. Specific fixes include:\n\n* `spack find` now displays variants and other spec constraints\n* bugfix: uninstall should find concrete specs by DAG hash (#13598)\n* environments: make shell modifications partially unconditional (#13523)\n* binary distribution: relocate text files properly in relative binaries (#13578)\n* bugfix: fetch prefers to fetch local mirrors over remote resources (#13545)\n* environments: only write when necessary (#13546)\n* bugfix: spack.util.url.join() now handles absolute paths correctly (#13488)\n* sbang: use utf-8 for encoding when patching (#13490)\n* Specs with quoted flags containing spaces are parsed correctly (#13521)\n* targets: print a warning message before downgrading (#13513)\n* Travis CI: Test Python 3.8 (#13347)\n* Documentation: Database.query methods share docstrings (#13515)\n* cuda: fix conflict statements for x86-64 targets (#13472)\n* cpu: fix clang flags for generic x86_64 (#13491)\n* syaml_int type should use int.__repr__ rather than str.__repr__ (#13487)\n* elpa: prefer 2016.05.004 until sse/avx/avx2 issues are resolved (#13530)\n* trilinos: temporarily constrain netcdf@:4.7.1 (#13526)\n\n# v0.13.0 (2019-10-25)\n\n`v0.13.0` is our biggest Spack release yet, with *many* new major features.\nFrom facility deployment to improved environments, microarchitecture\nsupport, and auto-generated build farms, this release has features for all of\nour users.\n\nSpack grew by over 700 packages in the past year, and the project now has\nover 450 contributors. Thanks to all of you for making this release possible.\n\n## Major new core features\n- Chaining: use dependencies from external \"upstream\" Spack instances\n- Environments now behave more like virtualenv/conda\n - Each env has a *view*: a directory with all packages symlinked in\n - Activating an environment sets `PATH`, `LD_LIBRARY_PATH`, `CPATH`,\n `CMAKE_PREFIX_PATH`, `PKG_CONFIG_PATH`, etc. to point to this view.\n- Spack detects and builds specifically for your microarchitecture\n - named, understandable targets like `skylake`, `broadwell`, `power9`, `zen2`\n - Spack knows which compilers can build for which architectures\n - Packages can easily query support for features like `avx512` and `sse3`\n - You can pick a target with, e.g. `spack install foo target=icelake`\n- Spack stacks: combinatorial environments for facility deployment\n - Environments can now build cartesian products of specs (with `matrix:`)\n - Conditional syntax support to exclude certain builds from the stack\n- Projections: ability to build easily navigable symlink trees environments\n- Support no-source packages (BundlePackage) to aggregate related packages\n- Extensions: users can write custom commands that live outside of Spack repo\n- Support ARM and Fujitsu compilers\n\n## CI/build farm support\n- `spack release-jobs` can detect `package.py` changes and generate\n `.gitlab-ci.yml` to create binaries for an environment or stack\n\tin parallel (initial support -- will change in future release).\n- Results of build pipelines can be uploaded to a CDash server.\n- Spack can now upload/fetch from package mirrors in Amazon S3\n\n## New commands/options\n- `spack mirror create --all` downloads *all* package sources/resources/patches\n- `spack dev-build` runs phases of the install pipeline on the working directory\n- `spack deprecate` permanently symlinks an old, unwanted package to a new one\n- `spack verify` chcecks that packages' files match what was originally installed\n- `spack find --json` prints `JSON` that is easy to parse with, e.g. `jq`\n- `spack find --format FORMAT` allows you to flexibly print package metadata\n- `spack spec --json` prints JSON version of `spec.yaml`\n\n## Selected improvements\n- Auto-build requested compilers if they do not exist\n- Spack automatically adds `RPATHs` needed to make executables find compiler\n runtime libraries (e.g., path to newer `libstdc++` in `icpc` or `g++`)\n- setup-env.sh is now compatible with Bash, Dash, and Zsh\n- Spack now caps build jobs at min(16, ncores) by default\n- `spack compiler find` now also throttles number of spawned processes\n- Spack now writes stage directories directly to `$TMPDIR` instead of\n symlinking stages within `$spack/var/spack/cache`.\n- Improved and more powerful `spec` format strings\n- You can pass a `spec.yaml` file anywhere in the CLI you can type a spec.\n- Many improvements to binary caching\n- Gradually supporting new features from Environment Modules v4\n- `spack edit` respects `VISUAL` environment variable\n- Simplified package syntax for specifying build/run environment modifications\n- Numerous improvements to support for environments across Spack commands\n- Concretization improvements\n\n## Documentation\n- Multi-lingual documentation (Started a Japanese translation)\n- Tutorial now has its own site at spack-tutorial.readthedocs.io\n - This enables us to keep multiple versions of the tutorial around\n\n## Deprecations\n- Spack no longer supports dotkit (LLNL's homegrown, now deprecated module tool)\n- `spack build`, `spack configure`, `spack diy` deprecated in favor of\n `spack dev-build` and `spack install`\n\n## Important package changes\n- 3,563 total packages (718 added since 0.12.1)\n- Spack now defaults to Python 3 (previously preferred 2.7 by default)\n- Much improved ARM support thanks to Fugaku (RIKEN) and SNL teams\n- Support new special versions: master, trunk, and head (in addition to develop)\n- Better finding logic for libraries and headers\n\n\n# v0.12.1 (2018-11-13)\n\nThis is a minor bugfix release, with a minor fix in the tutorial and a `flake8` fix.\n\nBugfixes\n* Add `r` back to regex strings in binary distribution\n* Fix gcc install version in the tutorial\n\n\n# v0.12.0 (2018-11-13)\n\n## Major new features\n- Spack environments\n- `spack.yaml` and `spack.lock` files for tracking dependencies\n- Custom configurations via command line\n- Better support for linking Python packages into view directories\n- Packages have more control over compiler flags via flag handlers\n- Better support for module file generation\n- Better support for Intel compilers, Intel MPI, etc.\n- Many performance improvements, improved startup time\n\n## License\n- As of this release, all of Spack is permissively licensed under Apache-2.0 or MIT, at the user's option.\n- Consents from over 300 contributors were obtained to make this relicense possible.\n- Previous versions were distributed under the LGPL license, version 2.1.\n\n## New packages\nOver 2,900 packages (800 added since last year)\n\nSpack would not be possible without our community. Thanks to all of our\n[contributors](https://github.com/spack/spack/graphs/contributors) for the\nnew features and packages in this release!\n\n\n# v0.11.2 (2018-02-07)\n\nThis release contains the following fixes:\n\n* Fixes for `gfortran` 7 compiler detection (#7017)\n* Fixes for exceptions thrown during module generation (#7173)\n\n\n# v0.11.1 (2018-01-19)\n\nThis release contains bugfixes for compiler flag handling. There were issues in `v0.11.0` that caused some packages to be built without proper optimization.\n\nFixes:\n* Issue #6999: FFTW installed with Spack 0.11.0 gets built without optimisations\n\nIncludes:\n* PR #6415: Fixes for flag handling behavior\n* PR #6960: Fix type issues with setting flag handlers\n* 880e319: Upstream fixes to `list_url` in various R packages\n\n\n# v0.11.0 (2018-01-17)\n\nSpack v0.11.0 contains many improvements since v0.10.0.\nBelow is a summary of the major features, broken down by category.\n\n## New packages\n- Spack now has 2,178 packages (from 1,114 in v0.10.0)\n- Many more Python packages (356) and R packages (471)\n- 48 Exascale Proxy Apps (try `spack list -t proxy-app`)\n\n\n## Core features for users\n- Relocatable binary packages (`spack buildcache`, #4854)\n- Spack now fully supports Python 3 (#3395)\n- Packages can be tagged and searched by tags (#4786)\n- Custom module file templates using Jinja (#3183)\n- `spack bootstrap` command now sets up a basic module environment (#3057)\n- Simplified and better organized help output (#3033)\n- Improved, less redundant `spack install` output (#5714, #5950)\n- Reworked `spack dependents` and `spack dependencies` commands (#4478)\n\n\n## Major new features for packagers\n- Multi-valued variants (#2386)\n- New `conflicts()` directive (#3125)\n- New dependency type: `test` dependencies (#5132)\n- Packages can require their own patches on dependencies (#5476)\n - `depends_on(..., patches=)`\n- Build interface for passing linker information through Specs (#1875)\n - Major packages that use blas/lapack now use this interface\n- Flag handlers allow packages more control over compiler flags (#6415)\n- Package subclasses support many more build systems:\n - autotools, perl, qmake, scons, cmake, makefile, python, R, WAF\n - package-level support for installing Intel HPC products (#4300)\n- `spack blame` command shows contributors to packages (#5522)\n- `spack create` now guesses many more build systems (#2707)\n- Better URL parsing to guess package version URLs (#2972)\n- Much improved `PythonPackage` support (#3367)\n\n\n## Core\n- Much faster concretization (#5716, #5783)\n- Improved output redirection (redirecting build output works properly #5084)\n- Numerous improvements to internal structure and APIs\n\n\n## Tutorials & Documentation\n- Many updates to documentation\n- [New tutorial material from SC17](https://spack.readthedocs.io/en/latest/tutorial.html)\n - configuration\n - build systems\n - build interface\n - working with module generation\n- Documentation on docker workflows and best practices\n\n\n## Selected improvements and bug fixes\n- No longer build Python eggs -- installations are plain directories (#3587)\n- Improved filtering of system paths from build PATHs and RPATHs (#2083, #3910)\n- Git submodules are properly handled on fetch (#3956)\n- Can now set default number of parallel build jobs in `config.yaml`\n- Improvements to `setup-env.csh` (#4044)\n- Better default compiler discovery on Mac OS X (#3427)\n - clang will automatically mix with gfortran\n- Improved compiler detection on Cray machines (#3075)\n- Better support for IBM XL compilers\n- Better tab completion\n- Resume gracefully after prematurely terminated partial installs (#4331)\n- Better mesa support (#5170)\n\n\nSpack would not be possible without our community. Thanks to all of our\n[contributors](https://github.com/spack/spack/graphs/contributors) for the\nnew features and packages in this release!\n\n\n# v0.10.0 (2017-01-17)\n\nThis is Spack `v0.10.0`. With this release, we will start to push Spack\nreleases more regularly. This is the last Spack release without\nautomated package testing. With the next release, we will begin to run\npackage tests in addition to unit tests.\n\nSpack has grown rapidly from 422 to\n[1,114 packages](https://spack.readthedocs.io/en/v0.10.0/package_list.html),\nthanks to the hard work of over 100 contributors. Below is a condensed\nversion of all the changes since `v0.9.1`.\n\n### Packages\n- Grew from 422 to 1,114 packages\n - Includes major updates like X11, Qt\n - Expanded HPC, R, and Python ecosystems\n\n### Core\n- Major speed improvements for spack find and concretization\n- Completely reworked architecture support\n - Platforms can have front-end and back-end OS/target combinations\n - Much better support for Cray and BG/Q cross-compiled environments\n- Downloads are now cached locally\n- Support installations in deeply nested directories: patch long shebangs using `sbang`\n\n### Basic usage\n- Easier global configuration via config.yaml\n - customize install, stage, and cache locations\n- Hierarchical configuration scopes: default, site, user\n - Platform-specific scopes allow better per-platform defaults\n- Ability to set `cflags`, `cxxflags`, `fflags` on the command line\n- YAML-configurable support for both Lmod and tcl modules in mainline\n- `spack install` supports --dirty option for emergencies\n\n### For developers\n- Support multiple dependency types: `build`, `link`, and `run`\n- Added `Package` base classes for custom build systems\n - `AutotoolsPackage`, `CMakePackage`, `PythonPackage`, etc.\n - `spack create` now guesses many more build systems\n- Development environment integration with `spack setup`\n- New interface to pass linking information via `spec` objects\n - Currently used for `BLAS`/`LAPACK`/`SCALAPACK` libraries\n - Polymorphic virtual dependency attributes: `spec['blas'].blas_libs`\n\n### Testing & Documentation\n- Unit tests run continuously on Travis CI for Mac and Linux\n- Switched from `nose` to `pytest` for unit tests.\n - Unit tests take 1 minute now instead of 8\n- Massively expanded documentation\n- Docs are now hosted on [spack.readthedocs.io](https://spack.readthedocs.io)\n" }, { "path": "CITATION.cff", "content": "# If you are referencing Spack in a publication, please cite the SC'15 paper\n# described here.\n#\n# Here's the raw citation:\n#\n# Todd Gamblin, Matthew P. LeGendre, Michael R. Collette, Gregory L. Lee,\n# Adam Moody, Bronis R. de Supinski, and W. Scott Futral.\n# The Spack Package Manager: Bringing Order to HPC Software Chaos.\n# In Supercomputing 2015 (SC’15), Austin, Texas, November 15-20 2015. LLNL-CONF-669890.\n#\n# Or, in BibTeX:\n#\n# @inproceedings{Gamblin_The_Spack_Package_2015,\n# address = {Austin, Texas, USA},\n# author = {Gamblin, Todd and LeGendre, Matthew and\n# Collette, Michael R. and Lee, Gregory L. and\n# Moody, Adam and de Supinski, Bronis R. and Futral, Scott},\n# doi = {10.1145/2807591.2807623},\n# month = {November 15-20},\n# note = {LLNL-CONF-669890},\n# series = {Supercomputing 2015 (SC’15)},\n# title = {{The Spack Package Manager: Bringing Order to HPC Software Chaos}},\n# url = {https://github.com/spack/spack},\n# year = {2015}\n# }\n#\n# And here's the CITATION.cff format:\n#\ncff-version: 1.2.0\ntype: software\nmessage: \"If you are referencing Spack in a publication, please cite the paper below.\"\ntitle: \"The Spack Package Manager: Bringing Order to HPC Software Chaos\"\nabstract: >-\n Large HPC centers spend considerable time supporting software for thousands of users, but the\n complexity of HPC software is quickly outpacing the capabilities of existing software management\n tools. Scientific applications require specific versions of compilers, MPI, and other dependency\n libraries, so using a single, standard software stack is infeasible. However, managing many\n configurations is difficult because the configuration space is combinatorial in size. We\n introduce Spack, a tool used at Lawrence Livermore National Laboratory to manage this complexity.\n Spack provides a novel, re- cursive specification syntax to invoke parametric builds of packages\n and dependencies. It allows any number of builds to coexist on the same system, and it ensures\n that installed packages can find their dependencies, regardless of the environment. We show\n through real-world use cases that Spack supports diverse and demanding applications, bringing\n order to HPC software chaos.\npreferred-citation:\n title: \"The Spack Package Manager: Bringing Order to HPC Software Chaos\"\n type: conference-paper\n url: \"https://tgamblin.github.io/pubs/spack-sc15.pdf\"\n authors:\n - family-names: \"Gamblin\"\n given-names: \"Todd\"\n - family-names: \"LeGendre\"\n given-names: \"Matthew\"\n - family-names: \"Collette\"\n given-names: \"Michael R.\"\n - family-names: \"Lee\"\n given-names: \"Gregory L.\"\n - family-names: \"Moody\"\n given-names: \"Adam\"\n - family-names: \"de Supinski\"\n given-names: \"Bronis R.\"\n - family-names: \"Futral\"\n given-names: \"Scott\"\n conference:\n name: \"Supercomputing 2015 (SC’15)\"\n city: \"Austin\"\n region: \"Texas\"\n country: \"US\"\n date-start: 2015-11-15\n date-end: 2015-11-20\n month: 11\n year: 2015\n identifiers:\n - description: \"The concept DOI of the work.\"\n type: doi\n value: 10.1145/2807591.2807623\n - description: \"The DOE Document Release Number of the work\"\n type: other\n value: \"LLNL-CONF-669890\"\nauthors:\n - family-names: \"Gamblin\"\n given-names: \"Todd\"\n - family-names: \"LeGendre\"\n given-names: \"Matthew\"\n - family-names: \"Collette\"\n given-names: \"Michael R.\"\n - family-names: \"Lee\"\n given-names: \"Gregory L.\"\n - family-names: \"Moody\"\n given-names: \"Adam\"\n - family-names: \"de Supinski\"\n given-names: \"Bronis R.\"\n - family-names: \"Futral\"\n given-names: \"Scott\"\n" }, { "path": "COPYRIGHT", "content": "Intellectual Property Notice\n------------------------------\n\nSpack is licensed under the Apache License, Version 2.0 (LICENSE-APACHE\nor http://www.apache.org/licenses/LICENSE-2.0) or the MIT license,\n(LICENSE-MIT or http://opensource.org/licenses/MIT), at your option.\n\nCopyrights and patents in the Spack project are retained by contributors.\nNo copyright assignment is required to contribute to Spack.\n\nSpack was originally developed in 2013 by Lawrence Livermore National\nSecurity, LLC. It was originally distributed under the LGPL-2.1 license.\nConsent from contributors to relicense to Apache-2.0/MIT is documented at\nhttps://github.com/spack/spack/issues/9137.\n\n\nSPDX usage\n------------\n\nIndividual files contain SPDX tags instead of the full license text.\nThis enables machine processing of license information based on the SPDX\nLicense Identifiers that are available here: https://spdx.org/licenses/\n\nFiles that are dual-licensed as Apache-2.0 OR MIT contain the following\ntext in the license header:\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\nExternal Packages\n-------------------\n\nSpack bundles most external dependencies in `lib/spack/spack/vendor/`.\nThis directory is automatically maintained using the `vendoring` tool.\nSpack also includes other vendored components like `ast.unparse` (in\n`lib/spack/spack/util/unparse/`), `sbang` (in `lib/spack/spack/hooks/`\nand `bin/`), and `spack.util.ctest_log_parser`. These packages are\ncovered by various permissive licenses. A summary listing follows.\nSee the license included with each for full details.\n\nPackageName: altgraph\nPackageHomePage: https://altgraph.readthedocs.io/en/latest/index.html\nPackageLicenseDeclared: MIT\n\nPackageName: archspec\nPackageHomePage: https://github.com/archspec/archspec\nPackageLicenseDeclared: Apache-2.0 OR MIT\n\nPackageName: ast.unparse\nPackageHomePage: https://www.python.org/\nPackageLicenseDeclared: PSF-2.0\n\nPackageName: attr\nPackageHomePage: https://www.attrs.org/\nPackageLicenseDeclared: MIT\n\nPackageName: attrs\nPackageHomePage: https://www.attrs.org/\nPackageLicenseDeclared: MIT\n\nPackageName: ctest_log_parser\nPackageHomePage: https://github.com/Kitware/CMake\nPackageLicenseDeclared: BSD-3-Clause\n\nPackageName: distro\nPackageHomePage: https://pypi.python.org/pypi/distro\nPackageLicenseDeclared: Apache-2.0\n\nPackageName: jinja2\nPackageHomePage: https://pypi.python.org/pypi/Jinja2\nPackageLicenseDeclared: BSD-3-Clause\n\nPackageName: jsonschema\nPackageHomePage: https://pypi.python.org/pypi/jsonschema\nPackageLicenseDeclared: MIT\n\nPackageName: macholib\nPackageHomePage: https://macholib.readthedocs.io/en/latest/index.html\nPackageLicenseDeclared: MIT\n\nPackageName: markupsafe\nPackageHomePage: https://pypi.python.org/pypi/MarkupSafe\nPackageLicenseDeclared: BSD-3-Clause\n\nPackageName: pyrsistent\nPackageHomePage: http://github.com/tobgu/pyrsistent\nPackageLicenseDeclared: MIT\n\nPackageName: ruamel.yaml\nPackageHomePage: https://yaml.readthedocs.io/\nPackageLicenseDeclared: MIT\n\nPackageName: sbang\nPackageHomePage: https://github.com/spack/sbang\nPackageLicenseDeclared: Apache-2.0 OR MIT\n\nPackageName: six\nPackageHomePage: https://pypi.org/project/six/\nPackageLicenseDeclared: MIT\n\nPackageName: typing_extensions\nPackageHomePage: https://pypi.org/project/typing-extensions/\nPackageLicenseDeclared: Python-2.0\n" }, { "path": "LICENSE-APACHE", "content": "\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "LICENSE-MIT", "content": "MIT License\n\nCopyright (c) Spack Project Developers.\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "NEWS.md", "content": "## Package API v2.4\n- Added the `%%` sigil to spec syntax, to propagate compiler preferences.\n\n## Spack v1.0.0\nDeprecated the implicit attributes:\n- `PackageBase.legacy_buildsystem`\n- `Builder.legacy_methods`\n- `Builder.legacy_attributes`\n- `Builder.legacy_long_methods`\n\n## Package API v2.3\n- `spack.package.version` directive: added `git_sparse_paths` parameter.\n\n## Package API v2.2\nAdded to `spack.package`:\n- `BuilderWithDefaults`\n- `ClassProperty`\n- `CompilerPropertyDetector`\n- `GenericBuilder`\n- `HKEY`\n- `LC_ID_DYLIB`\n- `LinkTree`\n- `MachO`\n- `ModuleChangePropagator`\n- `Package`\n- `WindowsRegistryView`\n- `apply_macos_rpath_fixups`\n- `classproperty`\n- `compare_output_file`\n- `compare_output`\n- `compile_c_and_execute`\n- `compiler_spec`\n- `create_builder`\n- `dedupe`\n- `delete_needed_from_elf`\n- `delete_rpath`\n- `environment_modifications_for_specs`\n- `execute_install_time_tests`\n- `filter_shebang`\n- `filter_system_paths`\n- `find_all_libraries`\n- `find_compilers`\n- `get_cmake_prefix_path`\n- `get_effective_jobs`\n- `get_elf_compat`\n- `get_path_args_from_module_line`\n- `get_user`\n- `has_shebang`\n- `host_platform`\n- `is_system_path`\n- `join_url`\n- `kernel_version`\n- `libc_from_dynamic_linker`\n- `macos_version`\n- `make_package_test_rpath`\n- `memoized`\n- `microarchitecture_flags_from_target`\n- `microarchitecture_flags`\n- `module_command`\n- `parse_dynamic_linker`\n- `parse_elf`\n- `path_contains_subdirectory`\n- `readlink`\n- `safe_remove`\n- `sbang_install_path`\n- `sbang_shebang_line`\n- `set_env`\n- `shared_library_suffix`\n- `spack_script`\n- `static_library_suffix`\n- `substitute_version_in_url`\n- `windows_sfn`\n\n## Package API v2.1\nAdded to `spack.package`:\n- `CompilerError`\n- `SpackError`\n" }, { "path": "NOTICE", "content": "This work was produced under the auspices of the U.S. Department of\nEnergy by Lawrence Livermore National Laboratory under Contract\nDE-AC52-07NA27344.\n\nThis work was prepared as an account of work sponsored by an agency of\nthe United States Government. Neither the United States Government nor\nLawrence Livermore National Security, LLC, nor any of their employees\nmakes any warranty, expressed or implied, or assumes any legal liability\nor responsibility for the accuracy, completeness, or usefulness of any\ninformation, apparatus, product, or process disclosed, or represents that\nits use would not infringe privately owned rights.\n\nReference herein to any specific commercial product, process, or service\nby trade name, trademark, manufacturer, or otherwise does not necessarily\nconstitute or imply its endorsement, recommendation, or favoring by the\nUnited States Government or Lawrence Livermore National Security, LLC.\n\nThe views and opinions of authors expressed herein do not necessarily\nstate or reflect those of the United States Government or Lawrence\nLivermore National Security, LLC, and shall not be used for advertising\nor product endorsement purposes.\n" }, { "path": "README.md", "content": "
\n\n

\n\n \n \n \"Spack\"\n\n\n
\n
\n\n\"CI\n\"Bootstrap\n\"Containers\n\"Documentation\n\"Code\n\"Slack\"/\n\"Matrix\"/\n\n

\n\n**[Getting Started]   •   [Config]   •   [Community]   •   [Contributing]   •   [Packaging Guide]   •   [Packages]**\n\n[Getting Started]: https://spack.readthedocs.io/en/latest/getting_started.html\n[Config]: https://spack.readthedocs.io/en/latest/configuration.html\n[Community]: #community\n[Contributing]: https://spack.readthedocs.io/en/latest/contribution_guide.html\n[Packaging Guide]: https://spack.readthedocs.io/en/latest/packaging_guide_creation.html\n[Packages]: https://github.com/spack/spack-packages\n\n
\n\nSpack is a multi-platform package manager that builds and installs\nmultiple versions and configurations of software. It works on Linux,\nmacOS, Windows, and many supercomputers. Spack is non-destructive: installing a\nnew version of a package does not break existing installations, so many\nconfigurations of the same package can coexist.\n\nSpack offers a simple \"spec\" syntax that allows users to specify versions\nand configuration options. Package files are written in pure Python, and\nspecs allow package authors to write a single script for many different\nbuilds of the same package. With Spack, you can build your software\n*all* the ways you want to.\n\nSee the\n[Feature Overview](https://spack.readthedocs.io/en/latest/features.html)\nfor examples and highlights.\n\nInstallation\n----------------\n\nTo install spack, first make sure you have Python & Git.\nThen:\n\n```bash\ngit clone --depth=2 https://github.com/spack/spack.git\n```\n\n```bash\n# For bash/zsh/sh\n. spack/share/spack/setup-env.sh\n\n# For tcsh/csh\nsource spack/share/spack/setup-env.csh\n\n# For fish\n. spack/share/spack/setup-env.fish\n```\n\n```bash\n# Now you're ready to install a package!\nspack install zlib-ng\n```\n\nDocumentation\n----------------\n\n[**Full documentation**](https://spack.readthedocs.io/) is available, or\nrun `spack help` or `spack help --all`.\n\nFor a cheat sheet on Spack syntax, run `spack help --spec`.\n\nTutorial\n----------------\n\nWe maintain a\n[**hands-on tutorial**](https://spack-tutorial.readthedocs.io/).\nIt covers basic to advanced usage, packaging, developer features, and large HPC\ndeployments. You can do all of the exercises on your own laptop using a\nDocker container.\n\nFeel free to use these materials to teach users at your organization\nabout Spack.\n\nCommunity\n------------------------\n\nSpack is an open source project. Questions, discussion, and\ncontributions are welcome. Contributions can be anything from new\npackages to bugfixes, documentation, or even new core features.\n\nResources:\n\n* **Slack workspace**: [spackpm.slack.com](https://spackpm.slack.com).\n To get an invitation, visit [slack.spack.io](https://slack.spack.io).\n* **Matrix space**: [#spack-space:matrix.org](https://matrix.to/#/#spack-space:matrix.org):\n [bridged](https://github.com/matrix-org/matrix-appservice-slack#matrix-appservice-slack) to Slack.\n* [**Github Discussions**](https://github.com/spack/spack/discussions):\n for Q&A and discussions. Note the pinned discussions for announcements.\n* **X**: [@spackpm](https://twitter.com/spackpm). Be sure to\n `@mention` us!\n* **Mailing list**: [groups.google.com/d/forum/spack](https://groups.google.com/d/forum/spack):\n only for announcements. Please use other venues for discussions.\n\nContributing\n------------------------\nContributing to Spack is relatively easy. Just send us a\n[pull request](https://help.github.com/articles/using-pull-requests/).\n\nMost contributors will want to contribute to Spack's community package\nrecipes. To do that, you should visit the\n**[spack-packages repository][Packages]**.\n\nIf you want to contribute to Spack itself, you can submit a pull request\nto the [spack repository](https://github.com/spack/spack) (this repository).\n\nYour PR must:\n\n 1. Make ``develop`` the destination branch;\n 2. Pass Spack's unit tests, documentation tests, and package build tests;\n 3. Be [PEP 8](https://www.python.org/dev/peps/pep-0008/) compliant;\n 4. Sign off all commits with `git commit --signoff`. Signoff says that you\n agree to the [Developer Certificate of Origin](https://developercertificate.org).\n Note that this is different from [signing commits](https://docs.github.com/en/authentication/managing-commit-signature-verification/signing-commits),\n which you may also do, but it's not required.\n\nWe enforce these guidelines with our continuous integration (CI) process.\nTo run tests locally, and for helpful tips on git, see our\n[Contribution Guide](https://spack.readthedocs.io/en/latest/contribution_guide.html).\n\nReleases\n--------\n\nFor multi-user site deployments or other use cases that need very stable\nsoftware installations, we recommend using Spack's\n[stable releases](https://github.com/spack/spack/releases).\n\nEach Spack release series also has a corresponding branch, e.g.\n`releases/v0.14` has `0.14.x` versions of Spack, and `releases/v0.13` has\n`0.13.x` versions. We backport important bug fixes to these branches but\nwe do not advance the package versions or make other changes that would\nchange the way Spack concretizes dependencies within a release branch.\nSo, you can base your Spack deployment on a release branch and `git pull`\nto get fixes, without the package churn that comes with `develop`.\n\nThe latest release is always available with the `releases/latest` tag.\n\nSee the [docs on releases](https://spack.readthedocs.io/en/latest/developer_guide.html#releases)\nfor more details.\n\nCode of Conduct\n------------------------\n\nPlease note that Spack has a\n[**Code of Conduct**](.github/CODE_OF_CONDUCT.md). By participating in\nthe Spack community, you agree to abide by its rules.\n\nAuthors\n----------------\nMany thanks go to Spack's [contributors](https://github.com/spack/spack/graphs/contributors).\n\nSpack was created by Todd Gamblin, tgamblin@llnl.gov.\n\n### Citing Spack\n\nIf you are referencing Spack in a publication, please cite the following paper:\n\n * Todd Gamblin, Matthew P. LeGendre, Michael R. Collette, Gregory L. Lee,\n Adam Moody, Bronis R. de Supinski, and W. Scott Futral.\n [**The Spack Package Manager: Bringing Order to HPC Software Chaos**](https://www.computer.org/csdl/proceedings/sc/2015/3723/00/2807623.pdf).\n In *Supercomputing 2015 (SC’15)*, Austin, Texas, November 15-20 2015. LLNL-CONF-669890.\n\nOn GitHub, you can copy this citation in APA or BibTeX format via the \"Cite this repository\"\nbutton. Or, see the comments in `CITATION.cff` for the raw BibTeX.\n\nLicense\n----------------\n\nSpack is distributed under the terms of both the MIT license and the\nApache License (Version 2.0). Users may choose either license, at their\noption.\n\nAll new contributions must be made under both the MIT and Apache-2.0\nlicenses.\n\nSee [LICENSE-MIT](https://github.com/spack/spack/blob/develop/LICENSE-MIT),\n[LICENSE-APACHE](https://github.com/spack/spack/blob/develop/LICENSE-APACHE),\n[COPYRIGHT](https://github.com/spack/spack/blob/develop/COPYRIGHT), and\n[NOTICE](https://github.com/spack/spack/blob/develop/NOTICE) for details.\n\nSPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nLLNL-CODE-811652\n" }, { "path": "SECURITY.md", "content": "# Security Policy\n\n## Supported Versions\n\nWe provide security updates for `develop` and for the last two\nstable (`0.x`) release series of Spack. Security updates will be\nmade available as patch (`0.x.1`, `0.x.2`, etc.) releases.\n\nFor more on Spack's release structure, see\n[`README.md`](https://github.com/spack/spack#releases).\n\n## Reporting a Vulnerability\n\nYou can report a vulnerability using GitHub's private reporting\nfeature:\n\n1. Go to [github.com/spack/spack/security](https://github.com/spack/spack/security).\n2. Click \"Report a vulnerability\" in the upper right corner of that page.\n3. Fill out the form and submit your draft security advisory.\n\nMore details are available in\n[GitHub's docs](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing/privately-reporting-a-security-vulnerability).\n\nYou can expect to hear back about security issues within two days.\nIf your security issue is accepted, we will do our best to release\na fix within a week. If fixing the issue will take longer than\nthis, we will discuss timeline options with you.\n" }, { "path": "bin/haspywin.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport subprocess\nimport sys\n\n\ndef getpywin():\n try:\n import win32con # noqa: F401\n except ImportError:\n print(\"pyWin32 not installed but is required...\\nInstalling via pip:\")\n subprocess.check_call([sys.executable, \"-m\", \"pip\", \"-q\", \"install\", \"--upgrade\", \"pip\"])\n subprocess.check_call([sys.executable, \"-m\", \"pip\", \"-q\", \"install\", \"pywin32\"])\n\n\nif __name__ == \"__main__\":\n getpywin()\n" }, { "path": "bin/sbang", "content": "#!/bin/sh\n#\n# Copyright sbang project developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n#\n# `sbang`: Run scripts with long shebang lines.\n#\n# Many operating systems limit the length and number of possible\n# arguments in shebang lines, making it hard to use interpreters that are\n# deep in the directory hierarchy or require special arguments.\n#\n# To use, put the long shebang on the second line of your script, and\n# make sbang the interpreter, like this:\n#\n# #!/bin/sh /path/to/sbang\n# #!/long/path/to/real/interpreter with arguments\n#\n# `sbang` will run the real interpreter with the script as its argument.\n#\n# See https://github.com/spack/sbang for more details.\n#\n\n# Generic error handling\ndie() {\n echo \"$@\" 1>&2;\n exit 1\n}\n\n# set SBANG_DEBUG to make the script print what would normally be executed.\nexec=\"exec\"\nif [ -n \"${SBANG_DEBUG}\" ]; then\n exec=\"echo \"\nfi\n\n# First argument is the script we want to actually run.\nscript=\"$1\"\n\n# ensure that the script actually exists\nif [ -z \"$script\" ]; then\n die \"error: sbang requires exactly one argument\"\nelif [ ! -f \"$script\" ]; then\n die \"$script: no such file or directory\"\nfi\n\n# Search the first two lines of script for interpreters.\nlines=0\nwhile read -r line && [ $lines -ne 2 ]; do\n if [ \"${line#\\#!}\" != \"$line\" ]; then\n shebang_line=\"${line#\\#!}\"\n elif [ \"${line#//!}\" != \"$line\" ]; then # // comments\n shebang_line=\"${line#//!}\"\n elif [ \"${line#--!}\" != \"$line\" ]; then # -- lua comments\n shebang_line=\"${line#--!}\"\n elif [ \"${line#}\"\n fi\n lines=$((lines+1))\ndone < \"$script\"\n\n# error if we did not find any interpreter\nif [ -z \"$shebang_line\" ]; then\n die \"error: sbang found no interpreter in $script\"\nfi\n\n# parse out the interpreter and first argument\nIFS=' ' read -r interpreter arg1 rest < /dev/null \"$cmd\"; then\n export SPACK_PYTHON=\"$(command -v \"$cmd\")\"\n exec \"${SPACK_PYTHON}\" \"$0\" \"$@\"\n fi\ndone\n\necho \"==> Error: spack could not find a python interpreter!\" >&2\nexit 1\n\":\"\"\"\n# Line above is a shell no-op, and ends a python multi-line comment.\n# The code above runs this file with our preferred python interpreter.\n\nimport os\nimport sys\n\nmin_python3 = (3, 6)\n\nif sys.version_info[:2] < min_python3:\n v_info = sys.version_info[:3]\n msg = \"Spack requires Python %d.%d or higher \" % min_python3\n msg += \"You are running spack with Python %d.%d.%d.\" % v_info\n sys.exit(msg)\n\n# Find spack's location and its prefix.\nspack_file = os.path.realpath(os.path.expanduser(__file__))\nspack_prefix = os.path.dirname(os.path.dirname(spack_file))\n\n# Allow spack libs to be imported in our scripts\nsys.path.insert(0, os.path.join(spack_prefix, \"lib\", \"spack\"))\n\nfrom spack.main import main # noqa: E402\n\n# Once we've set up the system path, run the spack main method\nif __name__ == \"__main__\":\n sys.exit(main())\n" }, { "path": "bin/spack-python", "content": "#!/bin/sh\n#\n# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n#\n# spack-python\n#\n# If you want to write your own executable Python script that uses Spack\n# modules, on Mac OS or maybe some others, you may be able to do it like\n# this:\n#\n# #!/usr/bin/env spack python\n#\n# Mac OS supports the above syntax, but it's not standard and most Linuxes\n# don't support more than one argument after the shebang command. This\n# script is a workaround. Do this in your Python script instead:\n#\n# #!/usr/bin/env spack-python\n#\n# This is compatible across platforms.\n#\nexec spack python \"$@\"\n" }, { "path": "bin/spack-tmpconfig", "content": "#!/bin/bash\nset -euo pipefail\n[[ -n \"${TMPCONFIG_DEBUG:=}\" ]] && set -x\nDIR=\"$(cd -P \"$(dirname \"${BASH_SOURCE[0]}\")\" && pwd)\"\nmkdir -p \"${XDG_RUNTIME_DIR:=/tmp}/spack-tests\"\nexport TMPDIR=\"${XDG_RUNTIME_DIR}\"\nexport TMP_DIR=\"$(mktemp -d -t spack-test-XXXXX)\"\nclean_up() {\n [[ -n \"$TMPCONFIG_DEBUG\" ]] && printf \"cleaning up: $TMP_DIR\\n\"\n rm -rf \"$TMP_DIR\"\n}\ntrap clean_up EXIT\ntrap clean_up ERR\n\n[[ -n \"$TMPCONFIG_DEBUG\" ]] && printf \"Redirecting TMP_DIR and spack directories to $TMP_DIR\\n\"\n\nexport BOOTSTRAP=\"${SPACK_USER_CACHE_PATH:=$HOME/.spack}/bootstrap\"\nexport SPACK_USER_CACHE_PATH=\"$TMP_DIR/user_cache\"\nmkdir -p \"$SPACK_USER_CACHE_PATH\"\n\nprivate_bootstrap=\"$SPACK_USER_CACHE_PATH/bootstrap\"\nuse_spack=''\nuse_bwrap=''\n# argument handling\nwhile (($# >= 1)) ; do\n case \"$1\" in\n -b) # privatize bootstrap too, useful for CI but not always cheap\n shift\n export BOOTSTRAP=\"$private_bootstrap\"\n ;;\n -B) # use specified bootstrap dir\n export BOOTSTRAP=\"$2\"\n shift 2\n ;;\n -s) # run spack directly with remaining args\n shift\n use_spack=1\n ;;\n --contain=bwrap)\n if bwrap --help 2>&1 > /dev/null ; then\n use_bwrap=1\n else\n echo Bubblewrap containment requested, but no bwrap command found\n exit 1\n fi\n shift\n ;;\n --)\n shift\n break\n ;;\n *)\n break\n ;;\n esac\ndone\ntypeset -a CMD\nif [[ -n \"$use_spack\" ]] ; then\n CMD=(\"$DIR/spack\" \"$@\")\nelse\n CMD=(\"$@\")\nfi\n\nmkdir -p \"$BOOTSTRAP\"\n\nexport SPACK_SYSTEM_CONFIG_PATH=\"$TMP_DIR/sys_conf\"\nexport SPACK_USER_CONFIG_PATH=\"$TMP_DIR/user_conf\"\nmkdir -p \"$SPACK_USER_CONFIG_PATH\"\ncat >\"$SPACK_USER_CONFIG_PATH/config.yaml\" <\"$SPACK_USER_CONFIG_PATH/bootstrap.yaml\" < 1)) && echo \"Running: ${CMD[@]}\"\n\"${CMD[@]}\"\n" }, { "path": "bin/spack.bat", "content": ":: Copyright Spack Project Developers. See COPYRIGHT file for details.\r\n::\r\n:: SPDX-License-Identifier: (Apache-2.0 OR MIT)\r\n::#######################################################################\r\n::\r\n:: This file is part of Spack and sets up the spack environment for batch,\r\n:: This includes environment modules and lmod support,\r\n:: and it also puts spack in your path. The script also checks that at least\r\n:: module support exists, and provides suggestions if it doesn't. Source\r\n:: it like this:\r\n::\r\n:: . /path/to/spack/install/spack_cmd.bat\r\n::\r\n@echo off\r\n\r\nset spack=\"%SPACK_ROOT%\\bin\\spack\"\r\n\r\n::#######################################################################\r\n:: This is a wrapper around the spack command that forwards calls to\r\n:: 'spack load' and 'spack unload' to shell functions. This in turn\r\n:: allows them to be used to invoke environment modules functions.\r\n::\r\n:: 'spack load' is smarter than just 'load' because it converts its\r\n:: arguments into a unique Spack spec that is then passed to module\r\n:: commands. This allows the user to use packages without knowing all\r\n:: their installation details.\r\n::\r\n:: e.g., rather than requiring a full spec for libelf, the user can type:\r\n::\r\n:: spack load libelf\r\n::\r\n:: This will first find the available libelf module file and use a\r\n:: matching one. If there are two versions of libelf, the user would\r\n:: need to be more specific, e.g.:\r\n::\r\n:: spack load libelf@0.8.13\r\n::\r\n:: This is very similar to how regular spack commands work and it\r\n:: avoids the need to come up with a user-friendly naming scheme for\r\n:: spack module files.\r\n::#######################################################################\r\n\r\n:_sp_shell_wrapper\r\nset \"_sp_flags=\"\r\nset \"_sp_args=\"\r\nset \"_sp_subcommand=\"\r\nsetlocal enabledelayedexpansion\r\n:: commands have the form '[flags] [subcommand] [args]'\r\n:: flags will always start with '-', e.g. --help or -V\r\n:: subcommands will never start with '-'\r\n:: everything after the subcommand is an arg\r\n\r\n\r\n:process_cl_args\r\nrem Set first cl argument (denoted by %1) to be processed\r\nset t=%1\r\nrem shift moves all cl positional arguments left by one\r\nrem meaning %2 is now %1, this allows us to iterate over each\r\nrem argument\r\nshift\r\nrem assign next \"first\" cl argument to cl_args, will be null when\r\nrem there are now further arguments to process\r\nset cl_args=%1\r\nif \"!t:~0,1!\" == \"-\" (\r\n if defined _sp_subcommand (\r\n rem We already have a subcommand, processing args now\r\n if not defined _sp_args (\r\n set \"_sp_args=!t!\"\r\n ) else (\r\n set \"_sp_args=!_sp_args! !t!\"\r\n )\r\n ) else (\r\n if not defined _sp_flags (\r\n set \"_sp_flags=!t!\"\r\n ) else (\r\n set \"_sp_flags=!_sp_flags! !t!\"\r\n )\r\n )\r\n) else if not defined _sp_subcommand (\r\n set \"_sp_subcommand=!t!\"\r\n) else (\r\n if not defined _sp_args (\r\n set \"_sp_args=!t!\"\r\n ) else (\r\n set \"_sp_args=!_sp_args! !t!\"\r\n )\r\n)\r\n\r\nrem if this is not nu;ll, we have more tokens to process\r\nrem start above process again with remaining unprocessed cl args\r\nif defined cl_args goto :process_cl_args\r\n\r\n\r\n:: --help, -h and -V flags don't require further output parsing.\r\n:: If we encounter, execute and exit\r\nif defined _sp_flags (\r\n if NOT \"%_sp_flags%\"==\"%_sp_flags:-h=%\" (\r\n python %spack% %_sp_flags%\r\n exit /B 0\r\n ) else if NOT \"%_sp_flags%\"==\"%_sp_flags:--help=%\" (\r\n python %spack% %_sp_flags%\r\n exit /B 0\r\n ) else if NOT \"%_sp_flags%\"==\"%_sp_flags:-V=%\" (\r\n python %spack% %_sp_flags%\r\n exit /B 0\r\n )\r\n)\r\nif not defined _sp_subcommand (\r\n if not defined _sp_args (\r\n if not defined _sp_flags (\r\n python %spack% --help\r\n exit /B 0\r\n )\r\n )\r\n)\r\n\r\n\r\n:: pass parsed variables outside of local scope. Need to do\r\n:: this because delayedexpansion can only be set by setlocal\r\nendlocal & (\r\n set \"_sp_flags=%_sp_flags%\"\r\n set \"_sp_args=%_sp_args%\"\r\n set \"_sp_subcommand=%_sp_subcommand%\"\r\n)\r\n\r\n\r\n:: Filter out some commands. For any others, just run the command.\r\nif \"%_sp_subcommand%\" == \"cd\" (\r\n goto :case_cd\r\n) else if \"%_sp_subcommand%\" == \"env\" (\r\n goto :case_env\r\n) else if \"%_sp_subcommand%\" == \"load\" (\r\n goto :case_load\r\n) else if \"%_sp_subcommand%\" == \"unload\" (\r\n goto :case_load\r\n) else (\r\n goto :default_case\r\n)\r\n\r\n::#######################################################################\r\n\r\n:case_cd\r\n:: Check for --help or -h\r\n:: TODO: This is not exactly the same as setup-env.\r\n:: In setup-env, '--help' or '-h' must follow the cd\r\n:: Here, they may be anywhere in the args\r\nif defined _sp_args (\r\n if NOT \"%_sp_args%\"==\"%_sp_args:--help=%\" (\r\n python %spack% cd -h\r\n goto :end_switch\r\n ) else if NOT \"%_sp_args%\"==\"%_sp_args:-h=%\" (\r\n python %spack% cd -h\r\n goto :end_switch\r\n )\r\n)\r\n\r\nfor /F \"tokens=* USEBACKQ\" %%F in (\r\n `python %spack% location %_sp_args%`) do (\r\n set \"LOC=%%F\"\r\n)\r\nfor %%Z in (\"%LOC%\") do if EXIST %%~sZ\\NUL (cd /d \"%LOC%\")\r\ngoto :end_switch\r\n\r\n:case_env\r\n:: If no args or args contain --bat or -h/--help: just execute.\r\nif NOT defined _sp_args (\r\n goto :default_case\r\n)\r\n\r\nif NOT \"%_sp_args%\"==\"%_sp_args:--help=%\" (\r\n goto :default_case\r\n) else if NOT \"%_sp_args%\"==\"%_sp_args: -h=%\" (\r\n goto :default_case\r\n) else if NOT \"%_sp_args%\"==\"%_sp_args:--bat=%\" (\r\n goto :default_case\r\n) else if NOT \"%_sp_args%\"==\"%_sp_args:deactivate=%\" (\r\n for /f \"tokens=* USEBACKQ\" %%I in (\r\n `call python %spack% %_sp_flags% env deactivate --bat %_sp_args:deactivate=%`\r\n ) do %%I\r\n) else if NOT \"%_sp_args%\"==\"%_sp_args:activate=%\" (\r\n for /f \"tokens=* USEBACKQ\" %%I in (\r\n `python %spack% %_sp_flags% env activate --bat %_sp_args:activate=%`\r\n ) do %%I\r\n) else (\r\n goto :default_case\r\n)\r\ngoto :end_switch\r\n\r\n:case_load\r\nif NOT defined _sp_args (\r\n exit /B 0\r\n)\r\n\r\n:: If args contain --bat, or -h/--help: just execute.\r\nif NOT \"%_sp_args%\"==\"%_sp_args:--help=%\" (\r\n goto :default_case\r\n) else if NOT \"%_sp_args%\"==\"%_sp_args:-h=%\" (\r\n goto :default_case\r\n) else if NOT \"%_sp_args%\"==\"%_sp_args:--bat=%\" (\r\n goto :default_case\r\n) else if NOT \"%_sp_args%\"==\"%_sp_args:--list=%\" (\r\n goto :default_case\r\n)\r\n\r\nfor /f \"tokens=* USEBACKQ\" %%I in (\r\n `python %spack% %_sp_flags% %_sp_subcommand% --bat %_sp_args%`\r\n ) do %%I\r\n\r\ngoto :end_switch\r\n\r\n:default_case\r\npython %spack% %_sp_flags% %_sp_subcommand% %_sp_args%\r\ngoto :end_switch\r\n\r\n:end_switch\r\nexit /B %ERRORLEVEL%\r\n\r\n\r\n::########################################################################\r\n:: Prepends directories to path, if they exist.\r\n:: pathadd /path/to/dir # add to PATH\r\n:: or pathadd OTHERPATH /path/to/dir # add to OTHERPATH\r\n::########################################################################\r\n\r\n:_spack_pathadd\r\nset \"_pa_varname=PATH\"\r\nset \"_pa_new_path=%~1\"\r\nif NOT \"%~2\" == \"\" (\r\n set \"_pa_varname=%~1\"\r\n set \"_pa_new_path=%~2\"\r\n )\r\nset \"_pa_oldvalue=%_pa_varname%\"\r\nfor %%Z in (\"%_pa_new_path%\") do if EXIST %%~sZ\\NUL (\r\n if defined %_pa_oldvalue% (\r\n set \"_pa_varname=%_pa_new_path%:%_pa_oldvalue%\"\r\n ) else (\r\n set \"_pa_varname=%_pa_new_path%\"\r\n )\r\n)\r\nexit /b 0\r\n\r\n:: set module system roots\r\n:_sp_multi_pathadd\r\nfor %%I in (%~2) do (\r\n for %%Z in (%_sp_compatible_sys_types%) do (\r\n :pathadd \"%~1\" \"%%I\\%%Z\"\r\n )\r\n)\r\nexit /B %ERRORLEVEL%\r\n" }, { "path": "bin/spack.ps1", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n# #######################################################################\n\nfunction Compare-CommonArgs {\n $CMDArgs = $args[0]\n # These arguments take precedence and call for no further parsing of arguments\n # invoke actual Spack entrypoint with that context and exit after\n \"--help\", \"-h\", \"--version\", \"-V\" | ForEach-Object {\n $arg_opt = $_\n if(($CMDArgs) -and ([bool]($CMDArgs.Where({$_ -eq $arg_opt})))) {\n return $true\n }\n }\n return $false\n}\n\nfunction Read-SpackArgs {\n $SpackCMD_params = @()\n $SpackSubCommand = $NULL\n $SpackSubCommandArgs = @()\n $args_ = $args[0]\n $args_ | ForEach-Object {\n if (!$SpackSubCommand) {\n if($_.SubString(0,1) -eq \"-\")\n {\n $SpackCMD_params += $_\n }\n else{\n $SpackSubCommand = $_\n }\n }\n else{\n $SpackSubCommandArgs += $_\n }\n }\n return $SpackCMD_params, $SpackSubCommand, $SpackSubCommandArgs\n}\n\nfunction Set-SpackEnv {\n # This method is responsible\n # for processing the return from $(spack )\n # which are returned as System.Object[]'s containing\n # a list of env commands\n # Invoke-Expression can only handle one command at a time\n # so we iterate over the list to invoke the env modification\n # expressions one at a time\n foreach($envop in $args[0]){\n Invoke-Expression $envop\n }\n}\n\n\nfunction Invoke-SpackCD {\n if (Compare-CommonArgs $SpackSubCommandArgs) {\n python \"$Env:SPACK_ROOT/bin/spack\" cd -h\n }\n else {\n $LOC = $(python \"$Env:SPACK_ROOT/bin/spack\" location $SpackSubCommandArgs)\n if (($NULL -ne $LOC)){\n if ( Test-Path -Path $LOC){\n Set-Location $LOC\n }\n else{\n exit 1\n }\n }\n else {\n exit 1\n }\n }\n}\n\nfunction Invoke-SpackEnv {\n if (Compare-CommonArgs $SpackSubCommandArgs[0]) {\n python \"$Env:SPACK_ROOT/bin/spack\" env -h\n }\n else {\n $SubCommandSubCommand = $SpackSubCommandArgs[0]\n $SubCommandSubCommandArgs = $SpackSubCommandArgs[1..$SpackSubCommandArgs.Count]\n switch ($SubCommandSubCommand) {\n \"activate\" {\n if (Compare-CommonArgs $SubCommandSubCommandArgs) {\n python \"$Env:SPACK_ROOT/bin/spack\" env activate $SubCommandSubCommandArgs\n }\n elseif ([bool]($SubCommandSubCommandArgs.Where({$_ -eq \"--pwsh\"}))) {\n python \"$Env:SPACK_ROOT/bin/spack\" env activate $SubCommandSubCommandArgs\n }\n elseif (!$SubCommandSubCommandArgs) {\n python \"$Env:SPACK_ROOT/bin/spack\" env activate $SubCommandSubCommandArgs\n }\n else {\n $SpackEnv = $(python \"$Env:SPACK_ROOT/bin/spack\" $SpackCMD_params env activate \"--pwsh\" $SubCommandSubCommandArgs)\n Set-SpackEnv $SpackEnv\n }\n }\n \"deactivate\" {\n if ([bool]($SubCommandSubCommandArgs.Where({$_ -eq \"--pwsh\"}))) {\n python\"$Env:SPACK_ROOT/bin/spack\" env deactivate $SubCommandSubCommandArgs\n }\n elseif($SubCommandSubCommandArgs) {\n python \"$Env:SPACK_ROOT/bin/spack\" env deactivate -h\n }\n else {\n $SpackEnv = $(python \"$Env:SPACK_ROOT/bin/spack\" $SpackCMD_params env deactivate \"--pwsh\")\n Set-SpackEnv $SpackEnv\n }\n }\n default {python \"$Env:SPACK_ROOT/bin/spack\" $SpackCMD_params $SpackSubCommand $SpackSubCommandArgs}\n }\n }\n}\n\nfunction Invoke-SpackLoad {\n if (Compare-CommonArgs $SpackSubCommandArgs) {\n python \"$Env:SPACK_ROOT/bin/spack\" $SpackCMD_params $SpackSubCommand $SpackSubCommandArgs\n }\n elseif ([bool]($SpackSubCommandArgs.Where({($_ -eq \"--pwsh\") -or ($_ -eq \"--list\")}))) {\n python \"$Env:SPACK_ROOT/bin/spack\" $SpackCMD_params $SpackSubCommand $SpackSubCommandArgs\n }\n else {\n $SpackEnv = $(python \"$Env:SPACK_ROOT/bin/spack\" $SpackCMD_params $SpackSubCommand \"--pwsh\" $SpackSubCommandArgs)\n Set-SpackEnv $SpackEnv\n }\n}\n\n\n$SpackCMD_params, $SpackSubCommand, $SpackSubCommandArgs = Read-SpackArgs $args\n\nif (Compare-CommonArgs $SpackCMD_params) {\n python \"$Env:SPACK_ROOT/bin/spack\" $SpackCMD_params $SpackSubCommand $SpackSubCommandArgs\n exit $LASTEXITCODE\n}\n\n# Process Spack commands with special conditions\n# all other commands are piped directly to Spack\nswitch($SpackSubCommand)\n{\n \"cd\" {Invoke-SpackCD}\n \"env\" {Invoke-SpackEnv}\n \"load\" {Invoke-SpackLoad}\n \"unload\" {Invoke-SpackLoad}\n default {python \"$Env:SPACK_ROOT/bin/spack\" $SpackCMD_params $SpackSubCommand $SpackSubCommandArgs}\n}\n\nexit $LASTEXITCODE\n" }, { "path": "bin/spack_cmd.bat", "content": "@ECHO OFF\r\n:: (c) 2021 Lawrence Livermore National Laboratory\r\n:: To use this file independently of Spack's installer, execute this script in its directory, or add the\r\n:: associated bin directory to your PATH. Invoke to launch Spack Shell.\r\n::\r\n:: source_dir/spack/bin/spack_cmd.bat\r\n::\r\n\r\ncall \"%~dp0..\\share\\spack\\setup-env.bat\"\r\npushd %SPACK_ROOT%\r\n%comspec% /K\r\n" }, { "path": "bin/spack_pwsh.ps1", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n$Env:SPACK_PS1_PATH=\"$PSScriptRoot\\..\\share\\spack\\setup-env.ps1\"\n& (Get-Process -Id $pid).Path -NoExit {\n . $Env:SPACK_PS1_PATH ; \n Push-Location $ENV:SPACK_ROOT\n }\n" }, { "path": "etc/spack/defaults/base/concretizer.yaml", "content": "# -------------------------------------------------------------------------\n# This is the default spack configuration file.\n#\n# Settings here are versioned with Spack and are intended to provide\n# sensible defaults out of the box. Spack maintainers should edit this\n# file to keep it current.\n#\n# Users can override these settings by editing\n# `$SPACK_ROOT/etc/spack/concretizer.yaml`, `~/.spack/concretizer.yaml`,\n# or by adding a `concretizer:` section to an environment.\n# -------------------------------------------------------------------------\nconcretizer:\n # Whether to consider installed packages or packages from buildcaches when\n # concretizing specs. If `true`, we'll try to use as many installs/binaries\n # as possible, rather than building. If `false`, we'll always give you a fresh\n # concretization. If `dependencies`, we'll only reuse dependencies but\n # give you a fresh concretization for your root specs.\n reuse: true\n # Options that tune which targets are considered for concretization. The\n # concretization process is very sensitive to the number targets, and the time\n # needed to reach a solution increases noticeably with the number of targets\n # considered.\n targets:\n # Determine whether we want to target specific or generic\n # microarchitectures. Valid values are: \"microarchitectures\" or \"generic\".\n # An example of \"microarchitectures\" would be \"skylake\" or \"bulldozer\",\n # while an example of \"generic\" would be \"aarch64\" or \"x86_64_v4\".\n granularity: microarchitectures\n # If \"false\" allow targets that are incompatible with the current host (for\n # instance concretize with target \"icelake\" while running on \"haswell\").\n # If \"true\" only allow targets that are compatible with the host.\n host_compatible: true\n # When \"true\" concretize root specs of environments together, so that each unique\n # package in an environment corresponds to one concrete spec. This ensures\n # environments can always be activated. When \"false\" perform concretization separately\n # on each root spec, allowing different versions and variants of the same package in\n # an environment.\n unify: true\n # Option to deal with possible duplicate nodes (i.e. different nodes from the same package) in the DAG.\n duplicates:\n # \"none\": allows a single node for any package in the DAG.\n # \"minimal\": allows the duplication of 'build-tools' nodes only\n # (e.g. py-setuptools, cmake etc.)\n # \"full\" (experimental): allows separation of the entire build-tool stack (e.g. the entire \"cmake\" subDAG)\n strategy: minimal\n # Maximum number of duplicates in a DAG, when using a strategy that allows duplicates. \"default\" is the\n # number used if there isn't a more specific alternative\n max_dupes:\n default: 1\n # Virtuals\n c: 2\n cxx: 2\n fortran: 1\n # Regular packages\n cmake: 2\n gmake: 2\n python: 2\n python-venv: 2\n py-cython: 2\n py-flit-core: 2\n py-pip: 2\n py-setuptools: 2\n py-versioneer: 2\n py-wheel: 2\n xcb-proto: 2\n # Compilers\n gcc: 2\n llvm: 2\n # Option to specify compatibility between operating systems for reuse of compilers and packages\n # Specified as a key: [list] where the key is the os that is being targeted, and the list contains the OS's\n # it can reuse. Note this is a directional compatibility so mutual compatibility between two OS's\n # requires two entries i.e. os_compatible: {sonoma: [monterey], monterey: [sonoma]}\n os_compatible: {}\n # If false, force all link/run dependencies of root to match c/c++/Fortran compiler. If this is\n # a list, then the listed packages are allowed to use a different compiler, but all others must\n # match.\n compiler_mixing: true\n # Option to specify whether to support splicing. Splicing allows for\n # the relinking of concrete package dependencies in order to better\n # reuse already built packages with ABI compatible dependencies\n splice:\n explicit: []\n automatic: false\n # Maximum time, in seconds, allowed for the 'solve' phase. If set to 0, there is no time limit.\n timeout: 0\n # If set to true, exceeding the timeout will always result in a concretization error. If false,\n # the best (suboptimal) model computed before the timeout is used.\n #\n # Setting this to false yields unreproducible results, so we advise to use that value only\n # for debugging purposes (e.g. check which constraints can help Spack concretize faster).\n error_on_timeout: true\n\n # Static analysis may reduce the concretization time by generating smaller ASP problems, in\n # cases where there are requirements that prevent part of the search space to be explored.\n static_analysis: false\n\n # If enabled, concretizations are cached in the misc_cache. The cache is keyed by the hash\n # of solver inputs, so we only need to run setup (not solve) if there is a cache hit.\n # Feature is experimental: enabling may result in potentially invalid concretizations\n # if package recipes are changed, see: https://github.com/spack/spack/issues/51553\n concretization_cache:\n enable: false\n\n # Options to control the behavior of the concretizer with external specs\n externals:\n # Either 'architecture_only', to complete external specs with just the architecture of the\n # current host, or 'default_variants' to complete external specs also with missing variants,\n # using their default value.\n completion: default_variants\n" }, { "path": "etc/spack/defaults/base/config.yaml", "content": "# -------------------------------------------------------------------------\n# This is the default spack configuration file.\n#\n# Settings here are versioned with Spack and are intended to provide\n# sensible defaults out of the box. Spack maintainers should edit this\n# file to keep it current.\n#\n# Users can override these settings by editing files in:\n#\n# * $SPACK_ROOT/etc/spack/ - Spack instance settings\n# * ~/.spack/ - user settings\n# * $SPACK_ROOT/etc/spack/site - Spack \"site\" settings. Like instance settings\n# but lower priority then user settings.\n#\n# -------------------------------------------------------------------------\nconfig:\n # This is the path to the root of the Spack install tree.\n # You can use $spack here to refer to the root of the spack instance.\n install_tree:\n root: $spack/opt/spack\n projections:\n all: \"{architecture.platform}-{architecture.target}/{name}-{version}-{hash}\"\n # install_tree can include an optional padded length (int or boolean)\n # default is False (do not pad)\n # if padded_length is True, Spack will pad as close to the system max path\n # length as possible\n # if padded_length is an integer, Spack will pad to that many characters,\n # assuming it is higher than the length of the install_tree root.\n # padded_length: 128\n\n\n # Locations where templates should be found\n template_dirs:\n - $spack/share/spack/templates\n\n # Directory where licenses should be located\n license_dir: $spack/etc/spack/licenses\n\n # Temporary locations Spack can try to use for builds.\n #\n # Recommended options are given below.\n #\n # Builds can be faster in temporary directories on some (e.g., HPC) systems.\n # Specifying `$tempdir` will ensure use of the default temporary directory\n # (i.e., ``$TMP` or ``$TMPDIR``).\n #\n # Another option that prevents conflicts and potential permission issues is\n # to specify `$user_cache_path/stage`, which ensures each user builds in their\n # home directory.\n #\n # A more traditional path uses the value of `$spack/var/spack/stage`, which\n # builds directly inside Spack's instance without staging them in a\n # temporary space. Problems with specifying a path inside a Spack instance\n # are that it precludes its use as a system package and its ability to be\n # pip installable.\n #\n # In Spack environment files, chaining onto existing system Spack\n # installations, the $env variable can be used to download, cache and build\n # into user-writable paths that are relative to the currently active\n # environment.\n #\n # In any case, if the username is not already in the path, Spack will append\n # the value of `$user` in an attempt to avoid potential conflicts between\n # users in shared temporary spaces.\n #\n # The build stage can be purged with `spack clean --stage` and\n # `spack clean -a`, so it is important that the specified directory uniquely\n # identifies Spack staging to avoid accidentally wiping out non-Spack work.\n build_stage:\n - $tempdir/$user/spack-stage\n - $user_cache_path/stage\n # - $spack/var/spack/stage\n\n # Naming format for individual stage directories\n stage_name: \"spack-stage-{name}-{version}-{hash}\"\n\n # Directory in which to run tests and store test results.\n # Tests will be stored in directories named by date/time and package\n # name/hash.\n test_stage: $user_cache_path/test\n\n # Cache directory for already downloaded source tarballs and archived\n # repositories. This can be purged with `spack clean --downloads`.\n source_cache: $spack/var/spack/cache\n\n\n ## Directory where spack managed environments are created and stored\n # environments_root: $spack/var/spack/environments\n\n\n # Cache directory for miscellaneous files, like the package index.\n # This can be purged with `spack clean --misc-cache`\n misc_cache: $user_cache_path/cache\n\n # Abort downloads after this many seconds if not data is received.\n # Setting this to 0 will disable the timeout.\n connect_timeout: 30\n\n\n # If this is false, tools like curl that use SSL will not verify\n # certificates. (e.g., curl will use use the -k option)\n verify_ssl: true\n\n\n # This is where custom certs for proxy/firewall are stored.\n # It can be a path or environment variable. To match ssl env configuration\n # the default is the environment variable SSL_CERT_FILE\n ssl_certs: $SSL_CERT_FILE\n\n\n # Suppress gpg warnings from binary package verification\n # Only suppresses warnings, gpg failure will still fail the install\n # Potential rationale to set True: users have already explicitly trusted the\n # gpg key they are using, and may not want to see repeated warnings that it\n # is self-signed or something of the sort.\n suppress_gpg_warnings: false\n\n\n # If set to true, Spack will always check checksums after downloading\n # archives. If false, Spack skips the checksum step.\n checksum: true\n\n\n # If set to true, Spack will fetch deprecated versions without warning.\n # If false, Spack will raise an error when trying to install a deprecated version.\n deprecated: false\n\n\n # If set to true, `spack install` and friends will NOT clean\n # potentially harmful variables from the build environment. Use wisely.\n dirty: false\n\n\n # The language the build environment will use. This will produce English\n # compiler messages by default, so the log parser can highlight errors.\n # If set to C, it will use English (see man locale).\n # If set to the empty string (''), it will use the language from the\n # user's environment.\n build_language: C\n\n\n # When set to true, concurrent instances of Spack will use locks to\n # avoid modifying the install tree, database file, etc. If false, Spack\n # will disable all locking, but you must NOT run concurrent instances\n # of Spack. For filesystems that don't support locking, you should set\n # this to false and run one Spack at a time, but otherwise we recommend\n # enabling locks.\n locks: true\n\n # The default url fetch method to use.\n # If set to 'curl', Spack will require curl on the user's system\n # If set to 'urllib', Spack will use python built-in libs to fetch\n url_fetch_method: urllib\n\n # The maximum number of jobs to use for the build. When using the old\n # installer, this is the number of jobs per package. In the new installer,\n # this is the global maximum number of jobs across all packages. When fewer\n # cores are available, Spack will use fewer jobs. The `-j` command line\n # argument overrides this option.\n build_jobs: 16\n\n # The maximum number of concurrent package builds a single Spack process\n # will perform. The default value of 0 means no package parallelism when using\n # the old installer, and unlimited package parallelism (other than the limit\n # set by build_jobs) when using the new installer. Setting this to 1 will\n # disable package parallelism in both installers. This option is ignored on\n # Windows.\n concurrent_packages: 0\n\n # Which installer to use: \"old\" or \"new\".\n installer: new\n\n # If set to true, Spack will use ccache to cache C compiles.\n ccache: false\n\n # How long to wait to lock the Spack installation database. This lock is used\n # when Spack needs to manage its own package metadata and all operations are\n # expected to complete within the default time limit. The timeout should\n # therefore generally be left untouched.\n db_lock_timeout: 60\n\n\n # How long to wait when attempting to modify a package (e.g. to install it).\n # This value should typically be 'null' (never time out) unless the Spack\n # instance only ever has a single user at a time, and only if the user\n # anticipates that a significant delay indicates that the lock attempt will\n # never succeed.\n package_lock_timeout: null\n\n\n # Control how shared libraries are located at runtime on Linux. See the\n # the Spack documentation for details.\n shared_linking:\n # Spack automatically embeds runtime search paths in ELF binaries for their\n # dependencies. Their type can either be \"rpath\" or \"runpath\". For glibc, rpath is\n # inherited and has precedence over LD_LIBRARY_PATH; runpath is not inherited\n # and of lower precedence. DO NOT MIX these within the same install tree.\n type: rpath\n\n\n # (Experimental) Embed absolute paths of dependent libraries directly in ELF\n # binaries to avoid runtime search. This can improve startup time of\n # executables with many dependencies, in particular on slow filesystems.\n bind: false\n\n # Controls the handling of missing dynamic libraries after installation.\n # Options are ignore (default), warn, or error. If set to error, the\n # installation fails if installed binaries reference dynamic libraries that\n # are not found in their specified rpaths.\n missing_library_policy: ignore\n\n\n # Set to 'false' to allow installation on filesystems that doesn't allow setgid bit\n # manipulation by unprivileged user (e.g. AFS)\n allow_sgid: true\n\n # Whether to show status information during building and installing packages.\n # This gives information about Spack's current progress as well as the current\n # and total number of packages. Information is shown both in the terminal\n # title and inline.\n install_status: true\n\n # Number of seconds a buildcache's index.json is cached locally before probing\n # for updates, within a single Spack invocation. Defaults to 10 minutes.\n binary_index_ttl: 600\n\n flags:\n # Whether to keep -Werror flags active in package builds.\n keep_werror: 'none'\n\n # A mapping of aliases that can be used to define new commands. For instance,\n # `sp: spec -I` will define a new command `sp` that will execute `spec` with\n # the `-I` argument. Aliases cannot override existing commands.\n aliases:\n concretise: concretize\n containerise: containerize\n rm: remove\n" }, { "path": "etc/spack/defaults/base/mirrors.yaml", "content": "mirrors:\n spack-public:\n binary: false\n url: https://mirror.spack.io\n" }, { "path": "etc/spack/defaults/base/modules.yaml", "content": "# -------------------------------------------------------------------------\n# This is the default configuration for Spack's module file generation.\n#\n# Settings here are versioned with Spack and are intended to provide\n# sensible defaults out of the box. Spack maintainers should edit this\n# file to keep it current.\n#\n# Users can override these settings by editing the following files.\n#\n# Per-spack-instance settings (overrides defaults):\n# $SPACK_ROOT/etc/spack/modules.yaml\n#\n# Per-user settings (overrides default and site settings):\n# ~/.spack/modules.yaml\n# -------------------------------------------------------------------------\nmodules:\n # This maps paths in the package install prefix to environment variables\n # they should be added to. For example, /bin should be in PATH.\n prefix_inspections:\n ./bin:\n - PATH\n ./man:\n - MANPATH\n ./share/man:\n - MANPATH\n ./share/aclocal:\n - ACLOCAL_PATH\n ./lib/pkgconfig:\n - PKG_CONFIG_PATH\n ./lib64/pkgconfig:\n - PKG_CONFIG_PATH\n ./share/pkgconfig:\n - PKG_CONFIG_PATH\n ./:\n - CMAKE_PREFIX_PATH\n\n # These are configurations for the module set named \"default\"\n default:\n # Where to install modules\n roots:\n tcl: $spack/share/spack/modules\n lmod: $spack/share/spack/lmod\n # What type of modules to use (\"tcl\" and/or \"lmod\")\n enable: []\n\n tcl:\n all:\n autoload: direct\n\n # Default configurations if lmod is enabled\n lmod:\n all:\n autoload: direct\n hierarchy:\n - mpi\n" }, { "path": "etc/spack/defaults/base/packages.yaml", "content": "# -------------------------------------------------------------------------\n# This file controls default concretization preferences for Spack.\n#\n# Settings here are versioned with Spack and are intended to provide\n# sensible defaults out of the box. Spack maintainers should edit this\n# file to keep it current.\n#\n# Users can override these settings by editing the following files.\n#\n# Per-spack-instance settings (overrides defaults):\n# $SPACK_ROOT/etc/spack/packages.yaml\n#\n# Per-user settings (overrides default and site settings):\n# ~/.spack/packages.yaml\n# -------------------------------------------------------------------------\npackages:\n all:\n providers:\n awk: [gawk]\n armci: [armcimpi]\n blas: [openblas]\n c: [gcc, llvm, intel-oneapi-compilers]\n cuda-lang: [cuda]\n cxx: [gcc, llvm, intel-oneapi-compilers]\n daal: [intel-oneapi-daal]\n elf: [elfutils]\n fftw-api: [fftw, amdfftw]\n flame: [libflame, amdlibflame]\n fortran: [gcc, llvm, intel-oneapi-compilers]\n fortran-rt: [gcc-runtime, intel-oneapi-runtime]\n fuse: [libfuse]\n gl: [glx, osmesa]\n glu: [mesa-glu, openglu]\n golang: [go, gcc]\n go-or-gccgo-bootstrap: [go-bootstrap, gcc]\n hip-lang: [llvm-amdgpu]\n iconv: [libiconv]\n ipp: [intel-oneapi-ipp]\n java: [openjdk, jdk]\n jpeg: [libjpeg-turbo, libjpeg]\n lapack: [openblas]\n libc: [glibc, musl]\n libgfortran: [gcc-runtime]\n libglx: [mesa+glx]\n libifcore: [intel-oneapi-runtime]\n libllvm: [llvm]\n lua-lang: [lua, lua-luajit-openresty, lua-luajit]\n luajit: [lua-luajit-openresty, lua-luajit]\n mariadb-client: [mariadb-c-client, mariadb]\n mkl: [intel-oneapi-mkl]\n mpe: [mpe2]\n mpi: [openmpi, mpich]\n mysql-client: [mysql, mariadb-c-client]\n opencl: [pocl]\n onedal: [intel-oneapi-dal]\n pbs: [openpbs, torque]\n pil: [py-pillow]\n pkgconfig: [pkgconf, pkg-config]\n qmake: [qt-base, qt]\n rpc: [libtirpc]\n scalapack: [netlib-scalapack, amdscalapack]\n sycl: [hipsycl]\n szip: [libaec, libszip]\n tbb: [intel-tbb]\n unwind: [libunwind]\n uuid: [util-linux-uuid, libuuid]\n wasi-sdk: [wasi-sdk-prebuilt]\n xkbdata-api: [xkeyboard-config, xkbdata]\n xxd: [xxd-standalone, vim]\n yacc: [bison, byacc]\n ziglang: [zig]\n zlib-api: [zlib-ng+compat, zlib]\n permissions:\n read: world\n write: user\n cce:\n buildable: false\n cray-fftw:\n buildable: false\n cray-libsci:\n buildable: false\n cray-mpich:\n buildable: false\n cray-mvapich2:\n buildable: false\n cray-pmi:\n buildable: false\n egl:\n buildable: false\n essl:\n buildable: false\n fj:\n buildable: false\n fujitsu-mpi:\n buildable: false\n fujitsu-ssl2:\n buildable: false\n glibc:\n buildable: false\n hpcx-mpi:\n buildable: false\n iconv:\n prefer: [libiconv]\n mpt:\n buildable: false\n musl:\n buildable: false\n opengl:\n buildable: false\n spectrum-mpi:\n buildable: false\n xl:\n buildable: false\n" }, { "path": "etc/spack/defaults/base/repos.yaml", "content": "# -------------------------------------------------------------------------\n# This is the default spack repository configuration. It includes the\n# builtin spack package repository.\n#\n# Users can override these settings by editing the following files.\n#\n# Per-spack-instance settings (overrides defaults):\n# $SPACK_ROOT/etc/spack/repos.yaml\n#\n# Per-user settings (overrides default and site settings):\n# ~/.spack/repos.yaml\n# -------------------------------------------------------------------------\nrepos:\n builtin:\n git: https://github.com/spack/spack-packages.git\n" }, { "path": "etc/spack/defaults/bootstrap.yaml", "content": "bootstrap:\n # If set to false Spack will not bootstrap missing software,\n # but will instead raise an error.\n enable: true\n # Root directory for bootstrapping work. The software bootstrapped\n # by Spack is installed in a \"store\" subfolder of this root directory\n root: $user_cache_path/bootstrap\n # Methods that can be used to bootstrap software. Each method may or\n # may not be able to bootstrap all the software that Spack needs,\n # depending on its type.\n sources:\n - name: github-actions-v2\n metadata: $spack/share/spack/bootstrap/github-actions-v2\n - name: github-actions-v0.6\n metadata: $spack/share/spack/bootstrap/github-actions-v0.6\n - name: spack-install\n metadata: $spack/share/spack/bootstrap/spack-install\n trusted:\n # By default we trust bootstrapping from sources and from binaries\n # produced on Github via the workflow\n github-actions-v2: true\n github-actions-v0.6: true\n spack-install: true\n" }, { "path": "etc/spack/defaults/darwin/modules.yaml", "content": "# -------------------------------------------------------------------------\n# This is the default configuration for Spack's module file generation.\n#\n# Settings here are versioned with Spack and are intended to provide\n# sensible defaults out of the box. Spack maintainers should edit this\n# file to keep it current.\n#\n# Users can override these settings by editing the following files.\n#\n# Per-spack-instance settings (overrides defaults):\n# $SPACK_ROOT/etc/spack/modules.yaml\n#\n# Per-user settings (overrides default and site settings):\n# ~/.spack/modules.yaml\n# -------------------------------------------------------------------------\nmodules:\n prefix_inspections:\n ./lib:\n - DYLD_FALLBACK_LIBRARY_PATH\n ./lib64:\n - DYLD_FALLBACK_LIBRARY_PATH\n" }, { "path": "etc/spack/defaults/darwin/packages.yaml", "content": "# -------------------------------------------------------------------------\n# This file controls default concretization preferences for Spack.\n#\n# Settings here are versioned with Spack and are intended to provide\n# sensible defaults out of the box. Spack maintainers should edit this\n# file to keep it current.\n#\n# Users can override these settings by editing the following files.\n#\n# Per-spack-instance settings (overrides defaults):\n# $SPACK_ROOT/etc/spack/packages.yaml\n#\n# Per-user settings (overrides default and site settings):\n# ~/.spack/packages.yaml\n# -------------------------------------------------------------------------\npackages:\n all:\n providers:\n c: [apple-clang, llvm, gcc]\n cxx: [apple-clang, llvm, gcc]\n elf: [libelf]\n fortran: [gcc]\n fuse: [macfuse]\n gl: [apple-gl]\n glu: [apple-glu]\n unwind: [apple-libunwind]\n uuid: [apple-libuuid]\n apple-clang:\n buildable: false\n apple-gl:\n buildable: false\n externals:\n - spec: apple-gl@4.1.0\n prefix: /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk\n apple-glu:\n buildable: false\n externals:\n - spec: apple-glu@1.3.0\n prefix: /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk\n apple-libunwind:\n buildable: false\n externals:\n # Apple bundles libunwind version 35.3 with macOS 10.9 and later,\n # although the version number used here isn't critical\n - spec: apple-libunwind@35.3\n prefix: /usr\n apple-libuuid:\n buildable: false\n externals:\n # Apple bundles libuuid in libsystem_c version 1353.100.2,\n # although the version number used here isn't critical\n - spec: apple-libuuid@1353.100.2\n prefix: /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk\n c:\n prefer:\n - apple-clang\n cxx:\n prefer:\n - apple-clang\n fortran:\n prefer:\n - gcc\n" }, { "path": "etc/spack/defaults/include.yaml", "content": "include:\n # default platform-specific configuration\n - path: \"${platform}\"\n optional: true\n\n # base packages.yaml overridable by platform-specific settings\n - path: base\n" }, { "path": "etc/spack/defaults/linux/modules.yaml", "content": "# -------------------------------------------------------------------------\n# This is the default configuration for Spack's module file generation.\n#\n# Settings here are versioned with Spack and are intended to provide\n# sensible defaults out of the box. Spack maintainers should edit this\n# file to keep it current.\n#\n# Users can override these settings by editing the following files.\n#\n# Per-spack-instance settings (overrides defaults):\n# $SPACK_ROOT/etc/spack/modules.yaml\n#\n# Per-user settings (overrides default and site settings):\n# ~/.spack/modules.yaml\n# -------------------------------------------------------------------------\nmodules: {}\n" }, { "path": "etc/spack/defaults/windows/config.yaml", "content": "config:\n locks: false\n build_stage::\n - '$user_cache_path/stage'\n stage_name: '{name}-{version}-{hash:7}'\n installer: old\n" }, { "path": "etc/spack/defaults/windows/packages.yaml", "content": "# -------------------------------------------------------------------------\n# This file controls default concretization preferences for Spack.\n#\n# Settings here are versioned with Spack and are intended to provide\n# sensible defaults out of the box. Spack maintainers should edit this\n# file to keep it current.\n#\n# Users can override these settings by editing the following files.\n#\n# Per-spack-instance settings (overrides defaults):\n# $SPACK_ROOT/etc/spack/packages.yaml\n#\n# Per-user settings (overrides default and site settings):\n# ~/.spack/packages.yaml\n# -------------------------------------------------------------------------\npackages:\n all:\n providers:\n c : [msvc]\n cxx: [msvc]\n mpi: [msmpi]\n gl: [wgl]\n mpi:\n require:\n - one_of: [msmpi]\n msvc:\n buildable: false\n" }, { "path": "lib/spack/_vendoring/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\nimport warnings\n\nimport spack.vendor\n\nimport spack.error\n\nwarnings.warn(\n \"The `_vendoring` module will be removed in Spack v1.1\",\n category=spack.error.SpackAPIWarning,\n stacklevel=2,\n)\n\n__path__ = spack.vendor.__path__\n" }, { "path": "lib/spack/docs/.gitignore", "content": ".spack/spack-packages\n.spack/packages.yaml\n.spack-env\n_build\n_spack_root\ncommand_index.rst\nspack*.rst\nspack.lock\n" }, { "path": "lib/spack/docs/.spack/repos.yaml", "content": "repos:\n builtin:\n destination: ./spack-packages\n" }, { "path": "lib/spack/docs/Makefile", "content": "# Makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line.\nSPHINXOPTS = -W --keep-going\nSPHINXBUILD = sphinx-build\nPAPER =\nBUILDDIR = _build\n\nexport PYTHONPATH := ../../spack:$(PYTHONPATH)\nAPIDOC_FILES = spack*.rst\n\n# Internal variables.\nPAPEROPT_a4 = -D latex_paper_size=a4\nPAPEROPT_letter = -D latex_paper_size=letter\nALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .\n# the i18n builder cannot share the environment and doctrees with the others\nI18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .\n\n.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext apidoc dashdoc\n\nall: html\n\n#\n# This creates a git repository and commits generated html docs.\n# It them pushes the new branch into THIS repository as gh-pages.\n#\n# github for some reason runs jekyll automatically on gh-pages\n# files, but we don't want that. 'touch .nojekyll' takes care\n# of it.\n#\ngh-pages: _build/html\n\troot=\"$$(git rev-parse --show-toplevel)\" && \\\n\tcd _build/html && \\\n\trm -rf .git && \\\n\ttouch .nojekyll && \\\n\tgit init && \\\n\tgit add . && \\\n\tgit commit -m \"Spack Documentation\" && \\\n\tgit push -f $$root master:gh-pages && \\\n\trm -rf .git\n\n# This version makes gh-pages into a single page that redirects\n# to spack.readthedocs.io\ngh-pages-redirect:\n\troot=\"$$(git rev-parse --show-toplevel)\" && \\\n\tcd _gh_pages_redirect && \\\n\trm -rf .git && \\\n\tgit init && \\\n\tgit add . && \\\n\tgit commit -m \"Spack Documentation\" && \\\n\tgit push -f $$root master:gh-pages && \\\n\trm -rf .git\n\nupload:\n\trsync -avz --rsh=ssh --delete _build/html/ cab:/usr/global/web-pages/lc/www/adept/docs/spack\n\tgit push -f github gh-pages\n\napidoc:\n\tsphinx-apidoc -f -T -o . ../spack\n\tsphinx-apidoc -f -T -o . ../llnl\n\nhelp:\n\t@echo \"Please use \\`make ' where is one of\"\n\t@echo \" html to make standalone HTML files\"\n\t@echo \" dirhtml to make HTML files named index.html in directories\"\n\t@echo \" singlehtml to make a single large HTML file\"\n\t@echo \" pickle to make pickle files\"\n\t@echo \" json to make JSON files\"\n\t@echo \" htmlhelp to make HTML files and a HTML help project\"\n\t@echo \" qthelp to make HTML files and a qthelp project\"\n\t@echo \" devhelp to make HTML files and a Devhelp project\"\n\t@echo \" epub to make an epub\"\n\t@echo \" latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter\"\n\t@echo \" latexpdf to make LaTeX files and run them through pdflatex\"\n\t@echo \" text to make text files\"\n\t@echo \" man to make manual pages\"\n\t@echo \" texinfo to make Texinfo files\"\n\t@echo \" info to make Texinfo files and run them through makeinfo\"\n\t@echo \" gettext to make PO message catalogs\"\n\t@echo \" changes to make an overview of all changed/added/deprecated items\"\n\t@echo \" linkcheck to check all external links for integrity\"\n\t@echo \" doctest to run all doctests embedded in the documentation (if enabled)\"\n\nclean:\n\t-rm -f command_index.rst\n\t-rm -rf $(BUILDDIR)/* $(APIDOC_FILES)\n\t-rm -rf .spack/spack-packages\n\nhtml:\n\t$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html\n\t@echo\n\t@echo \"Build finished. The HTML pages are in $(BUILDDIR)/html.\"\n\ndirhtml:\n\t$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml\n\t@echo\n\t@echo \"Build finished. The HTML pages are in $(BUILDDIR)/dirhtml.\"\n\nsinglehtml:\n\t$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml\n\t@echo\n\t@echo \"Build finished. The HTML page is in $(BUILDDIR)/singlehtml.\"\n\npickle:\n\t$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle\n\t@echo\n\t@echo \"Build finished; now you can process the pickle files.\"\n\njson:\n\t$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json\n\t@echo\n\t@echo \"Build finished; now you can process the JSON files.\"\n\nhtmlhelp:\n\t$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp\n\t@echo\n\t@echo \"Build finished; now you can run HTML Help Workshop with the\" \\\n\t \".hhp project file in $(BUILDDIR)/htmlhelp.\"\n\nqthelp:\n\t$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp\n\t@echo\n\t@echo \"Build finished; now you can run \"qcollectiongenerator\" with the\" \\\n\t \".qhcp project file in $(BUILDDIR)/qthelp, like this:\"\n\t@echo \"# qcollectiongenerator $(BUILDDIR)/qthelp/Spack.qhcp\"\n\t@echo \"To view the help file:\"\n\t@echo \"# assistant -collectionFile $(BUILDDIR)/qthelp/Spack.qhc\"\n\ndevhelp:\n\t$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp\n\t@echo\n\t@echo \"Build finished.\"\n\t@echo \"To view the help file:\"\n\t@echo \"# mkdir -p $$HOME/.local/share/devhelp/Spack\"\n\t@echo \"# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Spack\"\n\t@echo \"# devhelp\"\n\nepub:\n\t$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub\n\t@echo\n\t@echo \"Build finished. The epub file is in $(BUILDDIR)/epub.\"\n\nlatex:\n\t$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex\n\t@echo\n\t@echo \"Build finished; the LaTeX files are in $(BUILDDIR)/latex.\"\n\t@echo \"Run \\`make' in that directory to run these through (pdf)latex\" \\\n\t \"(use \\`make latexpdf' here to do that automatically).\"\n\nlatexpdf:\n\t$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex\n\t@echo \"Running LaTeX files through pdflatex...\"\n\t$(MAKE) -C $(BUILDDIR)/latex all-pdf\n\t@echo \"pdflatex finished; the PDF files are in $(BUILDDIR)/latex.\"\n\ntext:\n\t$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text\n\t@echo\n\t@echo \"Build finished. The text files are in $(BUILDDIR)/text.\"\n\nman:\n\t$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man\n\t@echo\n\t@echo \"Build finished. The manual pages are in $(BUILDDIR)/man.\"\n\ntexinfo:\n\t$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo\n\t@echo\n\t@echo \"Build finished. The Texinfo files are in $(BUILDDIR)/texinfo.\"\n\t@echo \"Run \\`make' in that directory to run these through makeinfo\" \\\n\t \"(use \\`make info' here to do that automatically).\"\n\ninfo:\n\t$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo\n\t@echo \"Running Texinfo files through makeinfo...\"\n\tmake -C $(BUILDDIR)/texinfo info\n\t@echo \"makeinfo finished; the Info files are in $(BUILDDIR)/texinfo.\"\n\ngettext:\n\t$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale\n\t@echo\n\t@echo \"Build finished. The message catalogs are in $(BUILDDIR)/locale.\"\n\nchanges:\n\t$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes\n\t@echo\n\t@echo \"The overview file is in $(BUILDDIR)/changes.\"\n\nlinkcheck:\n\t$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck\n\t@echo\n\t@echo \"Link check complete; look for any errors in the above output \" \\\n\t \"or in $(BUILDDIR)/linkcheck/output.txt.\"\n\ndoctest:\n\t$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest\n\t@echo \"Testing of doctests in the sources finished, look at the \" \\\n\t \"results in $(BUILDDIR)/doctest/output.txt.\"\n\ndashdoc:\n\t$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/dashdoc\n\tdoc2dash -A -v -n spack -d $(BUILDDIR)/ -f -I index.html -j $(BUILDDIR)/dashdoc\n\t@echo\n\t@echo \"Build finished. The Docset is in $(BUILDDIR)/dashdoc.\"\n" }, { "path": "lib/spack/docs/_gh_pages_redirect/.nojekyll", "content": "" }, { "path": "lib/spack/docs/_gh_pages_redirect/index.html", "content": "\n \n \n \n \n

\n This page has moved to https://spack.readthedocs.io/\n

\n \n\n" }, { "path": "lib/spack/docs/_static/css/custom.css", "content": "div.versionadded {\n border-left: 3px solid #0c731f;\n color: #0c731f;\n padding-left: 1rem;\n}\n\n.py.property {\n display: block !important;\n}\n\ndiv.version-switch {\n text-align: center;\n min-height: 2em;\n}\n\ndiv.version-switch>select {\n display: inline-block;\n text-align-last: center;\n background: none;\n border: none;\n border-radius: 0.5em;\n box-shadow: none;\n color: var(--color-foreground-primary);\n cursor: pointer;\n appearance: none;\n padding: 0.2em;\n -webkit-appearance: none;\n -moz-appearance: none;\n}\n\ndiv.version-switch select:active,\ndiv.version-switch select:focus,\ndiv.version-switch select:hover {\n color: var(--color-foreground-secondary);\n background: var(--color-background-hover);\n}\n\n.toc-tree li.scroll-current>.reference {\n font-weight: normal;\n}\n\n.search-results span {\n background-color: #fff3cd;\n padding: 0.1rem 0.2rem;\n border-radius: 2px;\n}\n\n@media (prefers-color-scheme: dark) {\n body:not([data-theme=\"light\"]) .search-results span {\n background-color: #664d03;\n color: #fff3cd;\n }\n}\n\n.highlight .go {\n color: #333;\n}" }, { "path": "lib/spack/docs/_static/js/versions.js", "content": "// based on https://github.com/readthedocs/sphinx_rtd_theme/blob/3.0.2/sphinx_rtd_theme/static/js/versions.js_t\n\nfunction onSelectorSwitch(event) {\n const option = event.target.selectedIndex;\n const item = event.target.options[option];\n window.location.href = item.dataset.url;\n}\n\nfunction initVersionSelector(config) {\n const versionSwitch = document.querySelector(\".version-switch\");\n if (!versionSwitch) { return; }\n let versions = config.versions.active;\n if (config.versions.current.hidden || config.versions.current.type === \"external\") {\n versions.unshift(config.versions.current);\n }\n const versionSelect = `\n \n`;\n\n versionSwitch.innerHTML = versionSelect;\n versionSwitch.firstElementChild.addEventListener(\"change\", onSelectorSwitch);\n}\n\nfunction initSearch(currentVersion) {\n // Numeric versions are PRs which have no search results; use latest instead.\n const searchVersion = /^\\d+$/.test(currentVersion) ? 'latest' : currentVersion;\n let searchTimeout;\n let originalContent;\n const searchInput = document.querySelector(\".sidebar-search\");\n const mainContent = document.getElementById(\"furo-main-content\");\n const searchForm = document.querySelector(\".sidebar-search-container\");\n\n if (!searchInput || !mainContent || !searchForm) { return; }\n\n // Store original content\n originalContent = mainContent.innerHTML;\n\n searchInput.addEventListener(\"input\", handleSearchInput);\n searchInput.addEventListener(\"keydown\", handleTabNavigation);\n searchForm.addEventListener(\"submit\", handleFormSubmit);\n\n function handleSearchInput(e) {\n const query = e.target.value.trim();\n clearTimeout(searchTimeout);\n if (query.length === 0) {\n mainContent.innerHTML = originalContent;\n return;\n }\n searchTimeout = setTimeout(function () {\n performSearch(query);\n }, 300);\n }\n\n function handleFormSubmit(e) {\n e.preventDefault();\n const query = searchInput.value.trim();\n if (query) {\n performSearch(query);\n }\n }\n\n function handleTabNavigation(e) {\n // Check if we're tabbing throught search results\n if (e.key !== 'Tab' || e.shiftKey) { return; }\n const searchResults = document.querySelector(\".search-results\");\n if (!searchResults) { return; }\n\n // Focus on the first link in search results instead of default behavior\n e.preventDefault();\n const firstLink = searchResults.querySelector(\"a\");\n if (firstLink) {\n firstLink.focus();\n }\n }\n\n function performSearch(query) {\n const fullQuery = `project:spack/${searchVersion} ${query}`;\n const searchUrl = `/_/api/v3/search/?q=${encodeURIComponent(fullQuery)}`;\n\n fetch(searchUrl)\n .then(function (response) {\n if (!response.ok) { throw new Error(\"HTTP error! status: \" + response.status); }\n return response.json();\n })\n .then(function (data) {\n displaySearchResults(data, query);\n })\n .catch(function (error) {\n mainContent.innerHTML = \"

Error performing search.

\";\n });\n }\n\n function displaySearchResults(data, query) {\n if (!data.results?.length) {\n mainContent.innerHTML = `

No Results Found

No results found for \"${query}\".

`;\n return;\n }\n\n let html = '

Search Results

';\n\n data.results.forEach((result, index) => {\n const title = result.highlights?.title?.[0] ?? result.title;\n html += `

${title}

`;\n\n result.blocks?.forEach(block => {\n const blockTitle = block.highlights?.title?.[0] ?? block.title;\n html += `

${blockTitle}

`;\n html += block.highlights?.content?.map(content => `

${content}

`).join('') ?? '';\n });\n\n if (index < data.results.length - 1) {\n html += `
`;\n }\n });\n\n html += \"
\";\n mainContent.innerHTML = html;\n }\n}\n\ndocument.addEventListener(\"readthedocs-addons-data-ready\", function (event) {\n const config = event.detail.data();\n initVersionSelector(config);\n initSearch(config.versions.current.slug);\n});\n" }, { "path": "lib/spack/docs/_templates/base.html", "content": "{% extends \"!base.html\" %}\n\n{%- block extrahead %} \n \n \n \n {%- if READTHEDOCS %}\n \n \n {%- endif %}\n{% endblock %}\n" }, { "path": "lib/spack/docs/_templates/sidebar/brand.html", "content": "\n {%- block brand_content %}\n {#- Remember to update the prefetch logic in `block logo_prefetch_links` in base.html #}\n {%- if logo_url %}\n
\n \"Logo\"/\n
\n {%- endif %}\n {%- if theme_light_logo and theme_dark_logo %}\n
\n \n \n
\n {%- endif %}\n {% if not theme_sidebar_hide_name %}\n {{ docstitle if docstitle else project }}\n {%- endif %}\n {% endblock brand_content %}\n\n\n{%- if READTHEDOCS %}\n
\n{%- endif %}\n" }, { "path": "lib/spack/docs/advanced_topics.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Explore advanced topics in Spack, including auditing packages and configuration, and verifying installations.\n\n.. _cmd-spack-audit:\n\nAuditing Packages and Configuration\n===================================\n\nThe ``spack audit`` command detects potential issues with configuration and packages:\n\n.. command-output:: spack audit -h\n\nFor instance, it can detect duplicate external specs in ``packages.yaml``, or the use of non-existing variants in directives.\nA detailed list of the checks currently implemented for each subcommand can be printed with:\n\n.. command-output:: spack -v audit list\n\nDepending on the use case, users can run the appropriate subcommands to obtain diagnostics.\nIf issues are found, they are reported to stdout:\n\n.. code-block:: console\n\n % spack audit packages lammps\n PKG-DIRECTIVES: 1 issue found\n 1. lammps: wrong variant in \"conflicts\" directive\n the variant 'adios' does not exist\n in spack_repo/builtin/packages/lammps/package.py\n\n.. _cmd-spack-verify:\n\nVerifying Installations\n=======================\n\nThe ``spack verify`` command can be used to verify the validity of Spack-installed packages any time after installation.\n\n\n``spack verify manifest``\n-------------------------\n\nAt installation time, Spack creates a manifest of every file in the installation prefix.\nFor links, Spack tracks the mode, ownership, and destination.\nFor directories, Spack tracks the mode and ownership.\nFor files, Spack tracks the mode, ownership, modification time, hash, and size.\nThe ``spack verify manifest`` command will check, for every file in each package, whether any of those attributes have changed.\nIt will also check for newly added files or deleted files from the installation prefix.\nSpack can either check all installed packages using the ``-a,--all`` option or accept specs listed on the command line to verify.\n\nThe ``spack verify manifest`` command can also verify that individual files haven't been altered since installation time.\nIf the given file is not in a Spack installation prefix, Spack will report that it is not owned by any package.\nTo check individual files instead of specs, use the ``-f,--files`` option.\n\nSpack installation manifests are included in the tarball signed by Spack for binary package distribution.\nWhen installed from a binary package, Spack uses the packaged installation manifest instead of creating one at install time.\n\nThe ``spack verify`` command also accepts the ``-l,--local`` option to check only local packages (as opposed to those used transparently from ``upstream`` Spack instances) and the ``-j,--json`` option to output machine-readable JSON data for any errors.\n\n``spack verify libraries``\n--------------------------\n\nThe ``spack verify libraries`` command can be used to verify that packages do not have accidental system dependencies.\nThis command scans the install prefixes of packages for executables and shared libraries, and resolves their needed libraries in their RPATHs.\nWhen needed libraries cannot be located, an error is reported.\nThis typically indicates that a package was linked against a system library instead of a library provided by a Spack package.\n\nThis verification can also be enabled as a post-install hook by setting ``config:shared_linking:missing_library_policy`` to ``error`` or ``warn`` in :ref:`config.yaml `.\n\n.. _filesystem-requirements:\n\nFilesystem Requirements\n=======================\n\nBy default, Spack needs to be run from a filesystem that supports ``flock`` locking semantics.\nNearly all local filesystems and recent versions of NFS support this, but parallel filesystems or NFS volumes may be configured without ``flock`` support enabled.\nYou can determine how your filesystems are mounted with ``mount``.\nThe output for a Lustre filesystem might look like this:\n\n.. code-block:: console\n\n $ mount | grep lscratch\n mds1-lnet0@o2ib100:/lsd on /p/lscratchd type lustre (rw,nosuid,lazystatfs,flock)\n mds2-lnet0@o2ib100:/lse on /p/lscratche type lustre (rw,nosuid,lazystatfs,flock)\n\nNote the ``flock`` option on both Lustre mounts.\n\nIf you do not see this or a similar option for your filesystem, you have a few options.\nFirst, you can move your Spack installation to a filesystem that supports locking.\nSecond, you could ask your system administrator to enable ``flock`` for your filesystem.\n\nIf none of those work, you can disable locking in one of two ways:\n\n1. Run Spack with the ``-L`` or ``--disable-locks`` option to disable locks on a call-by-call basis.\n2. Edit :ref:`config.yaml ` and set the ``locks`` option to ``false`` to always disable locking.\n\n.. warning::\n\n If you disable locking, concurrent instances of Spack will have no way to avoid stepping on each other.\n You must ensure that there is only **one** instance of Spack running at a time.\n Otherwise, Spack may end up with a corrupted database, or you may not be able to see all installed packages when running commands like ``spack find``.\n\n If you are unfortunate enough to run into this situation, you may be able to fix it by running ``spack reindex``.\n\nThis issue typically manifests with the error below:\n\n.. code-block:: console\n\n $ ./spack find\n Traceback (most recent call last):\n File \"./spack\", line 176, in \n main()\n File \"./spack\", line 154, in main\n return_val = command(parser, args)\n File \"./spack/lib/spack/spack/cmd/find.py\", line 170, in find\n specs = set(spack.installed_db.query(\\**q_args))\n File \"./spack/lib/spack/spack/database.py\", line 551, in query\n with self.read_transaction():\n File \"./spack/lib/spack/spack/database.py\", line 598, in __enter__\n if self._enter() and self._acquire_fn:\n File \"./spack/lib/spack/spack/database.py\", line 608, in _enter\n return self._db.lock.acquire_read(self._timeout)\n File \"./spack/lib/spack/llnl/util/lock.py\", line 103, in acquire_read\n self._lock(fcntl.LOCK_SH, timeout) # can raise LockError.\n File \"./spack/lib/spack/llnl/util/lock.py\", line 64, in _lock\n fcntl.lockf(self._fd, op | fcntl.LOCK_NB)\n IOError: [Errno 38] Function not implemented\n" }, { "path": "lib/spack/docs/binary_caches.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Discover how to create, use, and manage build caches in Spack to share pre-built binary packages and speed up installations.\n\n.. _binary_caches:\n\nBuild Caches\n============\n\nTo avoid recompilation of Spack packages, installed packages can be pushed to a build cache, and then downloaded and installed by others.\n\nWhenever a mirror provides prebuilt packages, Spack will take these packages into account during concretization and installation, making ``spack install`` significantly faster.\n\n\n.. note::\n\n We use the terms \"build cache\" and \"mirror\" often interchangeably.\n Mirrors are used during installation for both sources and prebuilt packages.\n Build caches refer to mirrors that provide prebuilt packages.\n\n\nCreating a Build Cache\n----------------------\n\nBuild caches are created via:\n\n.. code-block:: console\n\n $ spack buildcache push \n\nThis command takes the locally installed spec and its dependencies, and creates tarballs of their install prefixes.\nIt also generates metadata files, signed with GPG.\nThese tarballs and metadata files are then pushed to the provided build cache, which can be a local directory or a remote URL.\n\nHere is an example where a build cache is created in a local directory named \"spack-cache\", to which we push the \"ninja\" spec:\n\n.. code-block:: console\n\n $ spack buildcache push ./spack-cache ninja\n ==> Selected 30 specs to push to file:///home/spackuser/spack/spack-cache\n ...\n ==> [30/30] Pushed ninja@1.12.1/ngldn2k\n\nNote that ``ninja`` must be installed locally for this to work.\n\nOnce you have a build cache, you can add it as a mirror, as discussed next.\n\nFinding or Installing Build Cache Files\n---------------------------------------\n\nTo find or install build cache files, a Spack mirror must be configured with:\n\n.. code-block:: console\n\n $ spack mirror add \n\n\nBoth URLs and local paths on the filesystem can be specified.\nIn the previous example, you might add the directory \"spack-cache\" and call it ``mymirror``:\n\n\n.. code-block:: console\n\n $ spack mirror add mymirror ./spack-cache\n\n\nYou can see that the mirror is added with ``spack mirror list`` as follows:\n\n.. code-block:: console\n\n\n $ spack mirror list\n mymirror file:///home/spackuser/spack/spack-cache\n spack-public https://spack-llnl-mirror.s3-us-west-2.amazonaws.com/\n\n\nAt this point, you've created a build cache, but Spack hasn't indexed it, so if you run ``spack buildcache list``, you won't see any results.\nYou need to index this new build cache as follows:\n\n.. code-block:: console\n\n $ spack buildcache update-index ./spack-cache\n\nNow you can use ``list``:\n\n.. code-block:: console\n\n $ spack buildcache list\n ==> 24 cached builds.\n -- linux-ubuntu22.04-sapphirerapids / gcc@12.3.0 ----------------\n [ ... ]\n ninja@1.12.1\n\nWith ``mymirror`` configured and an index available, Spack will automatically use it during concretization and installation.\nThat means that you can expect ``spack install ninja`` to fetch prebuilt packages from the mirror.\nLet's verify by reinstalling ninja:\n\n.. code-block:: spec\n\n $ spack uninstall ninja\n $ spack install ninja\n [ ... ]\n ==> Installing ninja-1.12.1-ngldn2kpvb6lqc44oqhhow7fzg7xu7lh [24/24]\n gpg: Signature made Thu 06 Mar 2025 10:03:38 AM MST\n gpg: using RSA key 75BC0528114909C076E2607418010FFAD73C9B07\n gpg: Good signature from \"example (GPG created for Spack) \" [ultimate]\n ==> Fetching file:///home/spackuser/spack/spack-cache/blobs/sha256/f0/f08eb62661ad159d2d258890127fc6053f5302a2f490c1c7f7bd677721010ee0\n ==> Fetching file:///home/spackuser/spack/spack-cache/blobs/sha256/c7/c79ac6e40dfdd01ac499b020e52e57aa91151febaea3ad183f90c0f78b64a31a\n ==> Extracting ninja-1.12.1-ngldn2kpvb6lqc44oqhhow7fzg7xu7lh from binary cache\n ==> ninja: Successfully installed ninja-1.12.1-ngldn2kpvb6lqc44oqhhow7fzg7xu7lh\n Search: 0.00s. Fetch: 0.11s. Install: 0.11s. Extract: 0.10s. Relocate: 0.00s. Total: 0.22s\n [+] /home/spackuser/spack/opt/spack/linux-ubuntu22.04-sapphirerapids/gcc-12.3.0/ninja-1.12.1-ngldn2kpvb6lqc44oqhhow7fzg7xu7lh\n\nIt worked!\nYou've just completed a full example of creating a build cache with a spec of interest, adding it as a mirror, updating its index, listing the contents, and finally, installing from it.\n\nBy default, Spack falls back to building from sources when the mirror is not available or when the package is simply not already available.\nTo force Spack to install only prebuilt packages, you can use:\n\n.. code-block:: console\n\n $ spack install --use-buildcache only \n\nFor example, to combine all of the commands above to add the E4S build cache and then install from it exclusively, you would do:\n\n.. code-block:: console\n\n $ spack mirror add E4S https://cache.e4s.io\n $ spack buildcache keys --install --trust\n $ spack install --use-buildcache only \n\nThe ``--install`` and ``--trust`` flags install keys to the keyring and trust all downloaded keys.\n\n\nBuild Cache Index Views\n^^^^^^^^^^^^^^^^^^^^^^^\n\n.. note::\n Introduced in Spack v1.2.\n The addition of this feature does not increment the build cache version (v3).\n\n.. note::\n Build cache index views are not supported in OCI build caches.\n\nBuild caches can quickly become large and inefficient to search as binaries are added over time.\nA common work around to this problem is to break the build cache into stacks that target specific applications or workflows.\nThis allows for curation of binaries as smaller collections of packages that push to their own mirrors that each maintain a smaller search area.\nHowever, this approach comes with the trade off of requiring much larger storage and computational footprints due to duplication of common dependencies between stacks.\nSplitting build caches can also reduce direct fetch hits by reducing the breadth of binaries available in a single mirror.\n\nTo better address the issues with large search areas, build cache index views (or just \"views\" in this section) were introduced.\nA view is a named index which provides a curated view into a larger build cache.\nThis allows build cache maintainers to provide the same granularity of build caches split by stacks without having to pay for the extra storage and compute required for the duplicated dependencies.\n\nViews can be created or updated using an active environment, or a list of environment names or paths.\nThe ``spack buildcache`` commands for views are alias of the command ``spack buildcache update-index``.\n\nView indices are stored similarly to the top level build cache index, but use an additional prefix of the view name ``/v3/manifests/index/my-stack/index.manifest.json``.\n\n.. _cmd-spack-buildcache-create-view:\n\nCreating a Build Cache Index View\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nHere is an example of creating a view using an active environment.\n\n.. code-block:: console\n\n $ spack env activate my-stack\n $ spack install\n $ spack buildcache push my-mirror\n $ spack buildcache update-index --name my-view my-mirror\n\nIt is also possible to create a view from a list of one or more environments by passing the environment names or paths.\nIf a list of environments is passed while inside of an active environment, the active environment is ignored and only the passed environments are considered.\n\n.. code-block:: console\n\n $ spack buildcache update-index --name my-view my-mirror my-stack /path/to/environment/my-other-stack\n\n.. _cmd-spack-buildcache-update-view:\n\nUpdating a Build Cache Index View\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nTo prevent accidentally overwriting an existing view, it is required to specify how a view should be updated.\nIt is possible to use one of two options for updating a view index: ``--force`` or ``--append``.\nUsing the ``--force`` option will replace the index as if the previous one did not exist.\nThe ``--append`` option will first read the existing index, and then add the new specs to it.\n\n.. code-block:: console\n\n $ spack buildcache push my-mirror\n $ spack buildcache update-index --append --name my-view my-mirror my-stack\n\n\n.. warning::\n\n Using the ``--append`` option with build cache index views is a non-atomic operation.\n In the case where multiple writers are appending to the same view, the result will only include the state of the last to write.\n When using ``--append`` for build cache workflows it is up to the user to correctly serialize the update operations.\n\n\n\nList of Popular Build Caches\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n* `Spack Public Build Cache `_: `spack build cache `_\n* `Extreme-scale Scientific Software Stack (E4S) `_: `e4s build cache `_\n\n\nCreating and Trusting GPG keys\n------------------------------\n\n.. _cmd-spack-gpg:\n\n``spack gpg``\n^^^^^^^^^^^^^\n\nSpack has support for signing and verifying packages using GPG keys.\nA separate keyring is used for Spack, so any keys available in the user's home directory are not used.\n\n``spack gpg init``\n^^^^^^^^^^^^^^^^^^\n\nWhen Spack is first installed, its keyring is empty.\nKeys stored in :file:`var/spack/gpg` are the default keys for a Spack installation.\nThese keys may be imported by running ``spack gpg init``.\nThis will import the default keys into the keyring as trusted keys.\n\nTrusting keys\n^^^^^^^^^^^^^\n\nAdditional keys may be added to the keyring using:\n\n.. code-block:: console\n\n $ spack gpg trust \n\nOnce a key is trusted, packages signed by the owner of the key may be installed.\n\nTo remove keys from your keyring, use:\n\n.. code-block:: console\n\n $ spack gpg untrust \n\nKey IDs can be email addresses, names, or (preferably) fingerprints.\n\nCreating keys\n^^^^^^^^^^^^^\n\nYou may also create your own key so that you may sign your own packages using\n\n.. code-block:: console\n\n $ spack gpg create \n\nBy default, the key has no expiration, but it may be set with the ``--expires `` flag.\nIt is also recommended to add a comment as to the use of the key using the ``--comment `` flag.\nThe public half of the key can also be exported for sharing with others so that they may use packages you have signed using the ``--export `` flag.\nSecret keys may also be later exported using the ``spack gpg export [...]`` command.\n\n.. admonition:: Key creation speed\n :class: tip\n\n The creation of a new GPG key requires generating a lot of random numbers.\n Depending on the entropy produced on your system, the entire process may take a long time (and may even appear to hang).\n Virtual machines and cloud instances are particularly likely to display this behavior.\n\n To speed it up, you may install tools like ``rngd``, which is usually available as a package in the host OS.\n Another alternative is ``haveged``, which can be installed on RHEL/CentOS machines.\n\n `This Digital Ocean tutorial `_ provides a good overview of sources of randomness.\n\nBuild Cache Signing\n-------------------\n\nBy default, Spack will add a cryptographic signature to each package pushed to a build cache and verify the signature when installing from a build cache.\n\nKeys for signing can be managed with the :ref:`spack gpg ` command, as well as ``spack buildcache keys``, as mentioned above.\n\nYou can disable signing when pushing with ``spack buildcache push --unsigned`` and disable verification when installing from any build cache with ``spack install --no-check-signature``.\n\nAlternatively, signing and verification can be enabled or disabled on a per-build-cache basis:\n\n.. code-block:: console\n\n $ spack mirror add --signed # enable signing and verification\n $ spack mirror add --unsigned # disable signing and verification\n\n $ spack mirror set --signed # enable signing and verification for an existing mirror\n $ spack mirror set --unsigned # disable signing and verification for an existing mirror\n\nAlternatively, you can edit the ``mirrors.yaml`` configuration file directly:\n\n.. code-block:: yaml\n\n mirrors:\n :\n url: \n signed: false # disable signing and verification\n\nSee also :ref:`mirrors`.\n\nRelocation\n----------\n\nWhen using build caches across different machines, it is likely that the install root is different from the one used to build the binaries.\n\nTo address this issue, Spack automatically relocates all paths encoded in binaries and scripts to their new location upon installation.\n\nNote that there are some cases where this is not possible: if binaries are built in a relatively short path and then installed to a longer path, there may not be enough space in the binary to encode the new path.\nIn this case, Spack will fail to install the package from the build cache, and a source build is required.\n\nTo reduce the likelihood of this happening, it is highly recommended to add padding to the install root during the build, as specified in the :ref:`config ` section of the configuration:\n\n.. code-block:: yaml\n\n config:\n install_tree:\n root: /opt/spack\n padded_length: 128\n\n\nAutomatic Push to a Build Cache\n---------------------------------\n\nSometimes it is convenient to push packages to a build cache immediately after they are installed.\nSpack can do this by setting the ``--autopush`` flag when adding a mirror:\n\n.. code-block:: console\n\n $ spack mirror add --autopush \n\nOr the ``--autopush`` flag can be set for an existing mirror:\n\n.. code-block:: console\n\n $ spack mirror set --autopush # enable automatic push for an existing mirror\n $ spack mirror set --no-autopush # disable automatic push for an existing mirror\n\nThen, after installing a package, it is automatically pushed to all mirrors with ``autopush: true``.\nThe command\n\n.. code-block:: console\n\n $ spack install \n\nwill have the same effect as\n\n.. code-block:: console\n\n $ spack install \n $ spack buildcache push # for all caches with autopush: true\n\n.. note::\n\n Packages are automatically pushed to a build cache only if they are built from source.\n\n.. _binary_caches_oci:\n\nOCI / Docker V2 Registries as Build Cache\n-----------------------------------------\n\nSpack can also use OCI or Docker V2 registries such as Docker Hub, Quay.io, Amazon ECR, GitHub Packages, GitLab Container Registry, JFrog Artifactory, and others as build caches.\nThis is a convenient way to share binaries using public infrastructure or to cache Spack-built binaries in GitHub Actions and GitLab CI.\nThese registries can be used not only to share Spack binaries but also to create and distribute runnable container images.\n\nTo get started, configure an OCI mirror using ``oci://`` as the scheme and optionally specify variables that hold the username and password (or personal access token) for the registry:\n\n.. code-block:: console\n\n $ spack mirror add --oci-username-variable REGISTRY_USER \\\n --oci-password-variable REGISTRY_TOKEN \\\n my_registry oci://example.com/my_image\n\nThis registers a mirror in your ``mirrors.yaml`` configuration file that looks as follows:\n\n.. code-block:: yaml\n\n mirrors:\n my_registry:\n url: oci://example.com/my_image\n access_pair:\n id_variable: REGISTRY_USER\n secret_variable: REGISTRY_TOKEN\n\nSpack follows the naming conventions of Docker, with Docker Hub as the default registry.\nTo use Docker Hub, you can omit the registry domain:\n\n.. code-block:: console\n\n $ spack mirror add ... my_registry oci://username/my_image\n\nFrom here, you can use the mirror as any other build cache:\n\n.. code-block:: console\n\n $ export REGISTRY_USER=...\n $ export REGISTRY_TOKEN=...\n $ spack buildcache push my_registry # push to the registry\n $ spack install # or install from the registry\n\n.. note::\n\n Spack defaults to ``https`` for OCI registries, and does not fall back to ``http`` in case of failure.\n For local registries which use ``http`` instead of ``https``, you can specify ``oci+http://localhost:5000/my_image``.\n\n.. _oci-authentication:\n\nAuthentication with popular Container Registries\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nBelow are instructions for authenticating with some of the most popular container registries.\nIn all cases, you need to generate a (temporary) token to use as the password -- this is not the same as your account password.\n\nGHCR\n\"\"\"\"\"\"\n\nTo authenticate with GitHub Container Registry (GHCR), you can use your GitHub username as the username.\nFor the password, you can use either:\n\n#. A personal access token (PAT) with ``write:packages`` scope.\n#. A GitHub Actions token (``GITHUB_TOKEN``) with ``packages:write`` permission.\n\nSee also `GitHub's documentation `_ and :ref:`github-actions-build-cache` below.\n\nDocker Hub\n\"\"\"\"\"\"\"\"\"\"\n\nTo authenticate with Docker Hub, you can use your Docker Hub username as the username.\nFor the password, you need to generate a personal access token (PAT) on the Docker Hub website.\nSee `Docker's documentation `_ for more information.\n\nAmazon ECR\n\"\"\"\"\"\"\"\"\"\"\n\nTo authenticate with Amazon ECR, you can use the AWS CLI to generate a temporary password.\nThe username is always ``AWS``.\n\n.. code-block:: console\n\n $ export AWS_ECR_PASSWORD=$(aws ecr get-login-password --region )\n $ spack mirror add \\\n --oci-username AWS \\\n --oci-password-variable AWS_ECR_PASSWORD \\\n my_registry \\\n oci://XXX.dkr.ecr..amazonaws.com/my/image\n\nSee also `AWS's documentation `_.\n\nAzure Container Registry\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nTo authenticate with an Azure Container Registry that has RBAC enabled, you can use the Azure CLI to generate a temporary password for your managed identity.\nThe username is always ``00000000-0000-0000-0000-000000000000``.\n\n.. code-block:: console\n\n $ export AZURE_ACR_PASSWORD=$(az acr login --name --expose-token --output tsv --query accessToken)\n $ spack mirror add \\\n --oci-username 00000000-0000-0000-0000-000000000000 \\\n --oci-password-variable AZURE_ACR_PASSWORD \\\n my_registry \\\n oci://.azurecr.io/my/image\n\nSee also `Azure's documentation `_.\n\n\nBuild Cache and Container Images\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nA unique feature of build caches on top of OCI registries is that it's incredibly easy to generate a runnable container image with the binaries installed.\nThis is a great way to make applications available to users without requiring them to install Spack -- all you need is Docker, Podman, or any other OCI-compatible container runtime.\n\nTo produce container images, all you need to do is add the ``--base-image`` flag when pushing to the build cache:\n\n.. code-block:: console\n\n $ spack buildcache push --base-image ubuntu:20.04 my_registry ninja\n Pushed to example.com/my_image:ninja-1.11.1-yxferyhmrjkosgta5ei6b4lqf6bxbscz.spack\n\n $ docker run -it example.com/my_image:ninja-1.11.1-yxferyhmrjkosgta5ei6b4lqf6bxbscz.spack\n root@e4c2b6f6b3f4:/# ninja --version\n 1.11.1\n\nIf ``--base-image`` is not specified, Spack produces distroless images.\nIn practice, you won't be able to run these as containers because they don't come with libc and other system dependencies.\nHowever, they are still compatible with tools like ``skopeo``, ``podman``, and ``docker`` for pulling and pushing.\n\nSee the section :ref:`exporting-images` for more details on how to create container images with Spack.\n\n.. _github-actions-build-cache:\n\nSpack Build Cache for GitHub Actions\n------------------------------------\n\nTo significantly speed up Spack in GitHub Actions, binaries can be cached in GitHub Packages.\nThis service is an OCI registry that can be linked to a GitHub repository.\n\nSpack offers a public build cache for GitHub Actions with a set of common packages, which lets you get started quickly.\nSee the following resources for more information:\n\n* `spack/setup-spack `_ for setting up Spack in GitHub Actions\n* `spack/github-actions-buildcache `_ for more details on the public build cache\n\n.. _cmd-spack-buildcache:\n\n``spack buildcache``\n--------------------\n\n``spack buildcache push``\n^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nCreate a tarball of an installed Spack package and all its dependencies.\nTarballs and specfiles are compressed and checksummed; manifests are signed if GPG2 is available.\nCommands like ``spack buildcache install`` will search Spack mirrors to get the list of build caches.\n\n============== ========================================================================================================================\nArguments Description\n============== ========================================================================================================================\n```` list of partial specs or hashes with a leading ``/`` to match from installed packages and used for creating build caches\n``-d `` directory in which ``v3`` and ``blobs`` directories are created, defaults to ``.``\n``-f`` overwrite compressed tarball and spec metadata files if they already exist\n``-k `` the key to sign package with. In the case where multiple keys exist, the package will be unsigned unless ``-k`` is used.\n``-r`` make paths in binaries relative before creating tarball\n``-y`` answer yes to all questions about creating unsigned build caches\n============== ========================================================================================================================\n\n``spack buildcache list``\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nRetrieves all specs for build caches available on a Spack mirror.\n\n============== =====================================================================================\nArguments Description\n============== =====================================================================================\n```` list of partial package specs to be matched against specs downloaded for build caches\n============== =====================================================================================\n\nE.g., ``spack buildcache list gcc`` will print only commands to install ``gcc`` package(s).\n\n``spack buildcache install``\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nRetrieves all specs for build caches available on a Spack mirror and installs build caches with specs matching the input specs.\n\n============== ==============================================================================================\nArguments Description\n============== ==============================================================================================\n```` list of partial package specs or hashes with a leading ``/`` to be installed from build caches\n``-f`` remove install directory if it exists before unpacking tarball\n``-y`` answer yes to all to don't verify package with gpg questions\n============== ==============================================================================================\n\n``spack buildcache keys``\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nList public keys available on a Spack mirror.\n\n========= ==============================================\nArguments Description\n========= ==============================================\n``-it`` trust the keys downloaded with prompt for each\n``-y`` answer yes to all trust all keys downloaded\n========= ==============================================\n\n.. _build_cache_layout:\n\nBuild Cache Layout\n------------------\n\nThis section describes the structure and content of URL-style build caches, as distinguished from OCI-style build caches.\n\nThe entry point for a binary package is a manifest JSON file that references at least two other files stored as content-addressed blobs.\nThese files include a spec metadata file, as well as the installation directory of the package stored as a compressed archive file.\nBinary package manifest files are named to indicate the package name and version, as well as the hash of the concrete spec.\nFor example:\n\n.. code-block:: text\n\n gcc-runtime-12.3.0-qyu2lvgt3nxh7izxycugdbgf5gsdpkjt.spec.manifest.json\n\nwould contain the manifest for a binary package of ``gcc-runtime@12.3.0``.\nThe ID of the built package is defined to be the DAG hash of the concrete spec and exists in the name of the file as well.\nThe ID distinguishes a particular binary package from all other binary packages with the same package name and version.\nBelow is an example binary package manifest file.\nSuch a file would live in the versioned spec manifests directory of a binary mirror, for example, ``v3/manifests/spec/``:\n\n.. code-block:: json\n\n {\n \"version\": 3,\n \"data\": [\n {\n \"contentLength\": 10731083,\n \"mediaType\": \"application/vnd.spack.install.v2.tar+gzip\",\n \"compression\": \"gzip\",\n \"checksumAlgorithm\": \"sha256\",\n \"checksum\": \"0f24aa6b5dd7150067349865217acd3f6a383083f9eca111d2d2fed726c88210\"\n },\n {\n \"contentLength\": 1000,\n \"mediaType\": \"application/vnd.spack.spec.v5+json\",\n \"compression\": \"gzip\",\n \"checksumAlgorithm\": \"sha256\",\n \"checksum\": \"fba751c4796536737c9acbb718dad7429be1fa485f5585d450ab8b25d12ae041\"\n }\n ]\n }\n\nThe manifest references both the compressed tar file as well as the compressed spec metadata file, and contains the checksum of each.\nThis checksum is also used as the address of the associated file and, hence, must be known in order to locate the tarball or spec file within the mirror.\nOnce the tarball or spec metadata file is downloaded, the checksum should be computed locally and compared to the checksum in the manifest to ensure the contents have not changed since the binary package was pushed.\nSpack stores all data files (including compressed tar files, spec metadata, indices, public keys, etc.) within a ``blobs//`` directory, using the first two characters of the checksum as a subdirectory to reduce the number of files in a single folder.\nHere is a depiction of the organization of binary mirror contents:\n\n.. code-block:: text\n\n mirror_directory/\n v3/\n layout.json\n manifests/\n spec/\n gcc-runtime/\n gcc-runtime-12.3.0-s2nqujezsce4x6uhtvxscu7jhewqzztx.spec.manifest.json\n gmake/\n gmake-4.4.1-lpr4j77rcgkg5536tmiuzwzlcjsiomph.spec.manifest.json\n compiler-wrapper/\n compiler-wrapper-1.0-s7ieuyievp57vwhthczhaq2ogowf3ohe.spec.manifest.json\n index/\n index.manifest.json\n key/\n 75BC0528114909C076E2607418010FFAD73C9B07.key.manifest.json\n keys.manifest.json\n blobs/\n sha256/\n 0f/\n 0f24aa6b5dd7150067349865217acd3f6a383083f9eca111d2d2fed726c88210\n fb/\n fba751c4796536737c9acbb718dad7429be1fa485f5585d450ab8b25d12ae041\n 2a/\n 2a21836d206ccf0df780ab0be63fdf76d24501375306a35daa6683c409b7922f\n ...\n\nFiles within the ``manifests`` directory are organized into subdirectories by the type of entity they represent.\nBinary package manifests live in the ``spec/`` directory, build cache index manifests live in the ``index/`` directory, and manifests for public keys and their indices live in the ``key/`` subdirectory.\nRegardless of the type of entity they represent, all manifest files are named with an extension ``.manifest.json``.\n\nEvery manifest contains a ``data`` array, each element of which refers to an associated file stored as a content-addressed blob.\nConsidering the example spec manifest shown above, the compressed installation archive can be found by picking out the data blob with the appropriate ``mediaType``, which in this case would be ``application/vnd.spack.install.v2.tar+gzip``.\nThe associated file is found by looking in the blobs directory under ``blobs/sha256/fb/`` for the file named with the complete checksum value.\n\nAs mentioned above, every entity in a build cache is stored as a content-addressed blob pointed to by a manifest.\nWhile an example spec manifest (i.e., a manifest for a binary package) is shown above, here is what the manifest of a build cache index looks like:\n\n.. code-block:: json\n\n {\n \"version\": 3,\n \"data\": [\n {\n \"contentLength\": 6411,\n \"mediaType\": \"application/vnd.spack.db.v8+json\",\n \"compression\": \"none\",\n \"checksumAlgorithm\": \"sha256\",\n \"checksum\": \"225a3e9da24d201fdf9d8247d66217f5b3f4d0fc160db1498afd998bfd115234\"\n }\n ]\n }\n\nSome things to note about this manifest are that it points to a blob that is not compressed (``compression: \"none\"``) and that the ``mediaType`` is one we have not seen yet, ``application/vnd.spack.db.v8+json``.\nThe decision not to compress build cache indices stems from the fact that Spack does not yet sign build cache index manifests.\nOnce that changes, you may start to see these indices stored as compressed blobs.\n\nFor completeness, here are examples of manifests for the other two types of entities you might find in a Spack build cache.\nFirst, a public key manifest:\n\n.. code-block:: json\n\n {\n \"version\": 3,\n \"data\": [\n {\n \"contentLength\": 2472,\n \"mediaType\": \"application/pgp-keys\",\n \"compression\": \"none\",\n \"checksumAlgorithm\": \"sha256\",\n \"checksum\": \"9fc18374aebc84deb2f27898da77d4d4410e5fb44c60c6238cb57fb36147e5c7\"\n }\n ]\n }\n\nNote the ``mediaType`` of ``application/pgp-keys``.\nFinally, a public key index manifest:\n\n.. code-block:: json\n\n {\n \"version\": 3,\n \"data\": [\n {\n \"contentLength\": 56,\n \"mediaType\": \"application/vnd.spack.keyindex.v1+json\",\n \"compression\": \"none\",\n \"checksumAlgorithm\": \"sha256\",\n \"checksum\": \"29b3a0eb6064fd588543bc43ac7d42d708a69058dafe4be0859e3200091a9a1c\"\n }\n ]\n }\n\nAgain, note the ``mediaType`` of ``application/vnd.spack.keyindex.v1+json``.\nAlso, note that both the above manifest examples refer to uncompressed blobs; this is for the same reason Spack does not yet compress build cache index blobs.\n" }, { "path": "lib/spack/docs/bootstrapping.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Learn how Spack's bootstrapping feature automatically fetches and installs essential build tools when they are not available on the host system.\n\n.. _bootstrapping:\n.. _cmd-spack-bootstrap:\n.. _cmd-spack-bootstrap-status:\n.. _cmd-spack-bootstrap-now:\n\nBootstrapping\n=============\n\nIn the :ref:`Getting started ` section, we already mentioned that Spack can bootstrap some of its dependencies, including ``clingo``.\nIn fact, there is an entire command dedicated to the management of every aspect of bootstrapping:\n\n.. command-output:: spack bootstrap --help\n\nSpack bootstraps its dependencies automatically the first time they are needed.\nYou can readily check if any prerequisite for using Spack is missing by running:\n\n.. code-block:: console\n\n % spack bootstrap status\n Spack v0.19.0 - python@3.8\n\n [FAIL] Core Functionalities\n [B] MISSING \"clingo\": required to concretize specs\n\n [FAIL] Binary packages\n [B] MISSING \"gpg2\": required to sign/verify buildcaches\n\n\n Spack will take care of bootstrapping any missing dependency marked as [B]. Dependencies marked as [-] are instead required to be found on the system.\n\n % echo $?\n 1\n\nIn the case of the output shown above, Spack detected that both ``clingo`` and ``gnupg`` are missing, and it's giving detailed information on why they are needed and whether they can be bootstrapped.\nThe return code of this command summarizes the results; if any dependencies are missing, the return code is ``1``, otherwise ``0``.\nRunning a command that concretizes a spec, like:\n\n.. code-block:: console\n\n % spack solve zlib\n ==> Installing \"clingo-bootstrap@spack%apple-clang@12.0.0~docs~ipo+python build_type=Release arch=darwin-catalina-x86_64\" from a buildcache\n [ ... ]\n\nautomatically triggers the bootstrapping of clingo from pre-built binaries as expected.\n\nUsers can also bootstrap all Spack's dependencies in a single command, which is useful to set up containers or other similar environments:\n\n.. code-block:: console\n\n $ spack bootstrap now\n ==> Installing \"clingo-bootstrap@spack%gcc@10.2.1~docs~ipo+python+static_libstdcpp build_type=Release arch=linux-centos7-x86_64\" from a buildcache\n ==> Installing \"patchelf@0.15.0%gcc@10.2.1 ldflags=\"-static-libstdc++ -static-libgcc\" arch=linux-centos7-x86_64\" from a buildcache\n\n.. _cmd-spack-bootstrap-root:\n\nThe Bootstrapping Store\n-----------------------\n\nThe software installed for bootstrapping purposes is deployed in a separate store.\nYou can check its location with the following command:\n\n.. code-block:: console\n\n % spack bootstrap root\n\nYou can also change it by specifying the desired path:\n\n.. code-block:: console\n\n % spack bootstrap root /opt/spack/bootstrap\n\nYou can check what is installed in the bootstrapping store at any time using:\n\n.. code-block:: console\n\n % spack -b find\n ==> Showing internal bootstrap store at \"/Users/spack/.spack/bootstrap/store\"\n ==> 11 installed packages\n -- darwin-catalina-x86_64 / apple-clang@12.0.0 ------------------\n clingo-bootstrap@spack libassuan@2.5.5 libgpg-error@1.42 libksba@1.5.1 pinentry@1.1.1 zlib@1.2.11\n gnupg@2.3.1 libgcrypt@1.9.3 libiconv@1.16 npth@1.6 python@3.8\n\nIf needed, you can remove all the software in the current bootstrapping store with:\n\n.. code-block:: console\n\n % spack clean -b\n ==> Removing bootstrapped software and configuration in \"/Users/spack/.spack/bootstrap\"\n\n % spack -b find\n ==> Showing internal bootstrap store at \"/Users/spack/.spack/bootstrap/store\"\n ==> 0 installed packages\n\n.. _cmd-spack-bootstrap-list:\n.. _cmd-spack-bootstrap-disable:\n.. _cmd-spack-bootstrap-enable:\n.. _cmd-spack-bootstrap-reset:\n\nEnabling and Disabling Bootstrapping Methods\n--------------------------------------------\n\nBootstrapping is performed by trying the methods listed by:\n\n.. command-output:: spack bootstrap list\n\nin the order they appear, from top to bottom.\nBy default, Spack is configured to try bootstrapping from pre-built binaries first and to fall back to bootstrapping from sources if that fails.\n\nIf needed, you can disable bootstrapping altogether by running:\n\n.. code-block:: console\n\n % spack bootstrap disable\n\nin which case, it's your responsibility to ensure Spack runs in an environment where all its prerequisites are installed.\nYou can also configure Spack to skip certain bootstrapping methods by disabling them specifically:\n\n.. code-block:: console\n\n % spack bootstrap disable github-actions\n ==> \"github-actions\" is now disabled and will not be used for bootstrapping\n\ntells Spack to skip trying to bootstrap from binaries.\nTo add the \"github-actions\" method back, you can:\n\n.. code-block:: console\n\n % spack bootstrap enable github-actions\n\nYou can also reset the bootstrapping configuration to Spack's defaults:\n\n.. code-block:: console\n\n % spack bootstrap reset\n ==> Bootstrapping configuration is being reset to Spack's defaults. Current configuration will be lost.\n Do you want to continue? [Y/n]\n %\n\n.. _cmd-spack-bootstrap-mirror:\n.. _cmd-spack-bootstrap-add:\n\nCreating a Mirror for Air-Gapped Systems\n----------------------------------------\n\nSpack's default bootstrapping configuration requires internet connection to fetch precompiled binaries or source tarballs.\nSometimes, though, Spack is deployed on air-gapped systems where such access is denied.\n\nTo help in these situations, Spack provides a command to create a local mirror containing the source tarballs and/or binary packages needed for bootstrapping.\n\n.. code-block:: console\n\n % spack bootstrap mirror --binary-packages /opt/bootstrap\n ==> Adding \"clingo-bootstrap@spack+python %apple-clang target=x86_64\" and dependencies to the mirror at /opt/bootstrap/local-mirror\n ==> Adding \"gnupg@2.3: %apple-clang target=x86_64\" and dependencies to the mirror at /opt/bootstrap/local-mirror\n ==> Adding \"patchelf@0.13.1:0.13.99 %apple-clang target=x86_64\" and dependencies to the mirror at /opt/bootstrap/local-mirror\n ==> Adding binary packages from \"https://github.com/alalazo/spack-bootstrap-mirrors/releases/download/v0.1-rc.2/bootstrap-buildcache.tar.gz\" to the mirror at /opt/bootstrap/local-mirror\n\n To register the mirror on the platform where it's supposed to be used run the following command(s):\n % spack bootstrap add --trust local-sources /opt/bootstrap/metadata/sources\n % spack bootstrap add --trust local-binaries /opt/bootstrap/metadata/binaries\n % spack buildcache update-index /opt/bootstrap/bootstrap_cache\n\nRun this command on a machine with internet access, then move the resulting folder to the air-gapped system.\nOnce the local sources are added using the commands suggested at the prompt, they can be used to bootstrap Spack.\n" }, { "path": "lib/spack/docs/build_settings.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Understand how to control the build process in Spack by customizing package-specific build settings and environment variables.\n\n.. _concretizer-options:\n\nConcretization Settings (concretizer.yaml)\n==========================================\n\nThe ``concretizer.yaml`` configuration file allows users to customize aspects of the algorithm used to select the dependencies they install.\nThe default configuration is the following:\n\n.. literalinclude:: _spack_root/etc/spack/defaults/base/concretizer.yaml\n :language: yaml\n\n\nCompletion of external nodes\n----------------------------\n\n:ref:`The external packages ` available from the ``packages.yaml`` configuration file are usually reporting only a few of the variants defined in the corresponding recipe.\nUsers can configure how Spack deals with missing information for externals via the ``concretizer:externals:completion`` attribute:\n\n.. code-block:: yaml\n\n concretizer:\n externals:\n completion: default_variants\n\nThis attribute currently allows two possible values:\n\n- ``architecture_only``: only the mandatory architectural information is completed on externals\n- ``default_variants``: external specs are also completed with missing variants, using their default values\n\n\nReuse Already Installed Packages\n--------------------------------\n\nThe ``reuse`` attribute controls how aggressively Spack reuses binary packages during concretization.\nThe attribute can be either a single value or an object for more complex configurations.\n\nIn the former case (\"single value\"), it allows Spack to:\n\n1. Reuse installed packages and build caches for all the specs to be concretized, when ``true``.\n2. Reuse installed packages and build caches only for the dependencies of the root specs, when ``dependencies``.\n3. Disregard reusing installed packages and build caches, when ``false``.\n\nIn case finer control over which specs are reused is needed, the value of this attribute can be an object with the following keys:\n\n1. ``roots``: if ``true`` root specs are reused, if ``false`` only dependencies of root specs are reused\n2. ``from``: list of sources from which reused specs are taken\n\nEach source in ``from`` is itself an object with the following attributes:\n\n.. list-table:: Attributes for a source or reusable specs\n :header-rows: 1\n\n * - Attribute name\n - Description\n * - type (mandatory, string)\n - Can be ``local``, ``buildcache``, or ``external``.\n * - include (optional, list of specs)\n - If present, reusable specs must match at least one of the constraints in the list.\n * - exclude (optional, list of specs)\n - If present, reusable specs must not match any of the constraints in the list.\n\nFor instance, the following configuration:\n\n.. code-block:: yaml\n\n concretizer:\n reuse:\n roots: true\n from:\n - type: local\n include:\n - \"%gcc\"\n - \"%clang\"\n\ntells the concretizer to reuse all specs compiled with either ``gcc`` or ``clang`` that are installed in the local store.\nAny spec from remote build caches is disregarded.\n\nTo reduce the boilerplate in configuration files, default values for the ``include`` and ``exclude`` options can be pushed up one level:\n\n.. code-block:: yaml\n\n concretizer:\n reuse:\n roots: true\n include:\n - \"%gcc\"\n from:\n - type: local\n - type: buildcache\n - type: local\n include:\n - \"foo %oneapi\"\n\nIn the example above, we reuse all specs compiled with ``gcc`` from the local store and remote build caches, and we also reuse ``foo %oneapi``.\nNote that the last source of specs overrides the default ``include`` attribute.\n\nFor one-off concretizations, there are command-line arguments for each of the simple \"single value\" configurations.\nThis means a user can:\n\n.. code-block:: console\n\n % spack install --reuse \n\nto enable reuse for a single installation, or:\n\n.. code-block:: console\n\n $ spack install --fresh \n\nto do a fresh install if ``reuse`` is enabled by default.\n\n.. seealso::\n\n FAQ: :ref:`Why does Spack pick particular versions and variants? `\n\nSelection of Target Microarchitectures\n------------------------------------------\n\nThe options under the ``targets`` attribute control which targets are considered during a solve.\nCurrently, the options in this section are only configurable from the ``concretizer.yaml`` file, and there are no corresponding command-line arguments to enable them for a single solve.\n\nThe ``granularity`` option can take two possible values: ``microarchitectures`` and ``generic``.\nIf set to:\n\n.. code-block:: yaml\n\n concretizer:\n targets:\n granularity: microarchitectures\n\nSpack will consider all the microarchitectures known to ``archspec`` to label nodes for compatibility.\nIf instead the option is set to:\n\n.. code-block:: yaml\n\n concretizer:\n targets:\n granularity: generic\n\nSpack will consider only generic microarchitectures.\nFor instance, when running on a Haswell machine, Spack will consider ``haswell`` as the best target in the former case and ``x86_64_v3`` as the best target in the latter case.\n\nThe ``host_compatible`` option is a Boolean option that determines whether or not the microarchitectures considered during the solve are constrained to be compatible with the host Spack is currently running on.\nFor instance, if this option is set to ``true``, a user cannot concretize for ``target=icelake`` while running on a Haswell machine.\n\nDuplicate Nodes\n---------------\n\nThe ``duplicates`` attribute controls whether the DAG can contain multiple configurations of the same package.\nThis is mainly relevant for build dependencies, which may have their version pinned by some nodes and thus be required at different versions by different nodes in the same DAG.\n\nThe ``strategy`` option controls how the solver deals with duplicates.\nIf the value is ``none``, then a single configuration per package is allowed in the DAG.\nThis means, for instance, that only a single ``cmake`` or a single ``py-setuptools`` version is allowed.\nThe result would be a slightly faster concretization at the expense of making a few specs unsolvable.\n\nIf the value is ``minimal``, Spack will allow packages tagged as ``build-tools`` to have duplicates.\nThis allows, for instance, to concretize specs whose nodes require different and incompatible ranges of some build tool.\nFor instance, in the figure below, the latest `py-shapely` requires a newer `py-setuptools`, while `py-numpy` still needs an older version:\n\n.. figure:: images/shapely_duplicates.svg\n :width: 5580\n :height: 1842\n\nUp to Spack v0.20, ``duplicates:strategy:none`` was the default (and only) behavior.\nFrom Spack v0.21, the default behavior is ``duplicates:strategy:minimal``.\n\nSplicing\n--------\n\nThe ``splice`` key covers configuration attributes for splicing specs in the solver.\n\n\"Splicing\" is a method for replacing a dependency with another spec that provides the same package or virtual.\nThere are two types of splices, referring to different behaviors for shared dependencies between the root spec and the new spec replacing a dependency: \"transitive\" and \"intransitive\".\nA \"transitive\" splice is one that resolves all conflicts by taking the dependency from the new node.\nAn \"intransitive\" splice is one that resolves all conflicts by taking the dependency from the original root.\nFrom a theory perspective, hybrid splices are possible but are not modeled by Spack.\n\nAll spliced specs retain a ``build_spec`` attribute that points to the original spec before any splice occurred.\nThe ``build_spec`` for a non-spliced spec is itself.\n\nThe figure below shows examples of transitive and intransitive splices:\n\n.. figure:: images/splices.png\n :width: 2308\n :height: 1248\n\nThe concretizer can be configured to explicitly splice particular replacements for a target spec.\nSplicing will allow the user to make use of generically built public binary caches while swapping in highly optimized local builds for performance-critical components and/or components that interact closely with the specific hardware details of the system.\nThe most prominent candidate for splicing is MPI providers.\nMPI packages have relatively well-understood ABI characteristics, and most High Performance Computing facilities deploy highly optimized MPI packages tailored to their particular hardware.\nThe following configuration block configures Spack to replace whatever MPI provider each spec was concretized to use with the particular package of ``mpich`` with the hash that begins ``abcdef``.\n\n.. code-block:: yaml\n\n concretizer:\n splice:\n explicit:\n - target: mpi\n replacement: mpich/abcdef\n transitive: false\n\n.. warning::\n\n When configuring an explicit splice, you as the user take on the responsibility for ensuring ABI compatibility between the specs matched by the target and the replacement you provide.\n If they are not compatible, Spack will not warn you, and your application will fail to run.\n\nThe ``target`` field of an explicit splice can be any abstract spec.\nThe ``replacement`` field must be a spec that includes the hash of a concrete spec, and the replacement must either be the same package as the target, provide the virtual that is the target, or provide a virtual that the target provides.\nThe ``transitive`` field is optional -- by default, splices will be transitive.\n\n.. note::\n\n With explicit splices configured, it is possible for Spack to concretize to a spec that does not satisfy the input.\n For example, with the configuration above, ``hdf5 ^mvapich2`` will concretize to use ``mpich/abcdef`` instead of ``mvapich2`` as the MPI provider.\n Spack will warn the user in this case, but will not fail the concretization.\n\n.. _automatic_splicing:\n\nAutomatic Splicing\n^^^^^^^^^^^^^^^^^^\n\nThe Spack solver can be configured to do automatic splicing for ABI-compatible packages.\nAutomatic splices are enabled in the concretizer configuration section:\n\n.. code-block:: yaml\n\n concretizer:\n splice:\n automatic: true\n\nPackages can include ABI-compatibility information using the ``can_splice`` directive.\nSee :ref:`the packaging guide ` for instructions on specifying ABI compatibility using the ``can_splice`` directive.\n\n.. note::\n\n The ``can_splice`` directive is experimental and may be changed in future versions.\n\nWhen automatic splicing is enabled, the concretizer will combine any number of ABI-compatible specs if possible to reuse installed packages and packages available from binary caches.\nThe end result of these specs is equivalent to a series of transitive/intransitive splices, but the series may be non-obvious.\n" }, { "path": "lib/spack/docs/build_systems/autotoolspackage.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n An overview of the Autotools build system in Spack for packages that use GNU Autotools.\n\n.. _autotoolspackage:\n\nAutotools\n---------\n\nAutotools is a GNU build system that provides a build-script generator.\nBy running the platform-independent ``./configure`` script that comes with the package, you can generate a platform-dependent Makefile.\n\nPhases\n^^^^^^\n\nThe ``AutotoolsBuilder`` and ``AutotoolsPackage`` base classes come with the following phases:\n\n#. ``autoreconf`` - generate the configure script\n#. ``configure`` - generate the Makefiles\n#. ``build`` - build the package\n#. ``install`` - install the package\n\nMost of the time, the ``autoreconf`` phase will do nothing, but if the package is missing a ``configure`` script, ``autoreconf`` will generate one for you.\n\nThe other phases run:\n\n.. code-block:: console\n\n $ ./configure --prefix=/path/to/installation/prefix\n $ make\n $ make check # optional\n $ make install\n $ make installcheck # optional\n\n\nOf course, you may need to add a few arguments to the ``./configure`` line.\n\nImportant files\n^^^^^^^^^^^^^^^\n\nThe most important file for an Autotools-based package is the ``configure`` script.\nThis script is automatically generated by Autotools and generates the appropriate Makefile when run.\n\n.. warning::\n\n Watch out for fake Autotools packages!\n\n Autotools is a very popular build system, and many people are used to the classic steps to install a package:\n\n .. code-block:: console\n\n $ ./configure\n $ make\n $ make install\n\n\n For this reason, some developers will write their own ``configure`` scripts that have nothing to do with Autotools.\n These packages may not accept the same flags as other Autotools packages, so it is better to use the ``Package`` base class and create a :ref:`custom build system `.\n You can tell if a package uses Autotools by running ``./configure --help`` and comparing the output to other known Autotools packages.\n You should also look for files like:\n\n * ``configure.ac``\n * ``configure.in``\n * ``Makefile.am``\n\n Packages that don't use Autotools aren't likely to have these files.\n\nBuild system dependencies\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nWhether or not your package requires Autotools to install depends on how the source code is distributed.\nMost of the time, when developers distribute tarballs, they will already contain the ``configure`` script necessary for installation.\nIf this is the case, your package does not require any Autotools dependencies.\n\nHowever, a basic rule of version control systems is to never commit code that can be generated.\nThe source code repository itself likely does not have a ``configure`` script.\nDevelopers typically write (or auto-generate) a ``configure.ac`` script that contains configuration preferences and a ``Makefile.am`` script that contains build instructions.\nThen, ``autoconf`` is used to convert ``configure.ac`` into ``configure``, while ``automake`` is used to convert ``Makefile.am`` into ``Makefile.in``.\n``Makefile.in`` is used by ``configure`` to generate a platform-dependent ``Makefile`` for you.\nThe following diagram provides a high-level overview of the process:\n\n.. figure:: Autoconf-automake-process.*\n :target: https://commons.wikimedia.org/w/index.php?curid=15581407\n\n `GNU autoconf and automake process for generating makefiles `_\n by `Jdthood` under `CC BY-SA 3.0 `_\n\nIf a ``configure`` script is not present in your tarball, you will need to generate one yourself.\nLuckily, Spack already has an ``autoreconf`` phase to do most of the work for you.\nBy default, the ``autoreconf`` phase runs:\n\n.. code-block:: console\n\n $ autoreconf --install --verbose --force -I /share/aclocal\n\nIn case you need to add more arguments, override ``autoreconf_extra_args`` in your ``package.py`` on class scope like this:\n\n.. code-block:: python\n\n autoreconf_extra_args = [\"-Im4\"]\n\nAll you need to do is add a few Autotools dependencies to the package.\nMost stable releases will come with a ``configure`` script, but if you check out a commit from the ``master`` branch, you would want to add:\n\n.. code-block:: python\n\n depends_on(\"autoconf\", type=\"build\", when=\"@master\")\n depends_on(\"automake\", type=\"build\", when=\"@master\")\n depends_on(\"libtool\", type=\"build\", when=\"@master\")\n\nIt is typically redundant to list the ``m4`` macro processor package as a dependency, since ``autoconf`` already depends on it.\n\nUsing a custom autoreconf phase\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nIn some cases, it might be needed to replace the default implementation of the autoreconf phase with one running a script interpreter.\nIn this example, the ``bash`` shell is used to run the ``autogen.sh`` script.\n\n.. code-block:: python\n\n def autoreconf(self, spec, prefix):\n which(\"bash\")(\"autogen.sh\")\n\nIf the ``package.py`` has build instructions in a separate :ref:`builder class `, the signature for a phase changes slightly:\n\n.. code-block:: python\n\n class AutotoolsBuilder(AutotoolsBuilder):\n def autoreconf(self, pkg, spec, prefix):\n which(\"bash\")(\"autogen.sh\")\n\npatching configure or Makefile.in files\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nIn some cases, developers might need to distribute a patch that modifies one of the files used to generate ``configure`` or ``Makefile.in``.\nIn this case, these scripts will need to be regenerated.\nIt is preferable to regenerate these manually using the patch, and then create a new patch that directly modifies ``configure``.\nThat way, Spack can use the secondary patch and additional build system dependencies aren't necessary.\n\nOld Autotools helper scripts\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nAutotools based tarballs come with helper scripts such as ``config.sub`` and ``config.guess``.\nIt is the responsibility of the developers to keep these files up to date so that they run on every platform, but for very old software releases this is impossible.\nIn these cases Spack can help to replace these files with newer ones, without having to add the heavy dependency on ``automake``.\n\nAutomatic helper script replacement is currently enabled by default on ``ppc64le`` and ``aarch64``, as these are the known cases where old scripts fail.\nOn these targets, ``AutotoolsPackage`` adds a build dependency on ``gnuconfig``, which is a very lightweight package with newer versions of the helper files.\nSpack then tries to run all the helper scripts it can find in the release, and replaces them on failure with the helper scripts from ``gnuconfig``.\n\nTo opt out of this feature, use the following setting:\n\n.. code-block:: python\n\n patch_config_files = False\n\nTo enable it conditionally on different architectures, define a property and make the package depend on ``gnuconfig`` as a build dependency:\n\n.. code-block:: python\n\n depends_on(\"gnuconfig\", when=\"@1.0:\")\n\n\n @property\n def patch_config_files(self):\n return self.spec.satisfies(\"@1.0:\")\n\n.. note::\n\n On some exotic architectures it is necessary to use system provided ``config.sub`` and ``config.guess`` files.\n In this case, the most transparent solution is to mark the ``gnuconfig`` package as external and non-buildable, with a prefix set to the directory containing the files:\n\n .. code-block:: yaml\n\n gnuconfig:\n buildable: false\n externals:\n - spec: gnuconfig@master\n prefix: /usr/share/configure_files/\n\n\nforce_autoreconf\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nIf for whatever reason you really want to add the original patch and tell Spack to regenerate ``configure``, you can do so using the following setting:\n\n.. code-block:: python\n\n force_autoreconf = True\n\nThis line tells Spack to wipe away the existing ``configure`` script and generate a new one.\nIf you only need to do this for a single version, this can be done like so:\n\n.. code-block:: python\n\n @property\n def force_autoreconf(self):\n return self.version == Version(\"1.2.3\")\n\nFinding configure flags\n^^^^^^^^^^^^^^^^^^^^^^^\n\nOnce you have a ``configure`` script present, the next step is to determine what option flags are available.\nThese flags can be found by running:\n\n.. code-block:: console\n\n $ ./configure --help\n\n``configure`` will display a list of valid flags separated into some or all of the following sections:\n\n* Configuration\n* Installation directories\n* Fine tuning of the installation directories\n* Program names\n* X features\n* System types\n* **Optional Features**\n* **Optional Packages**\n* **Some influential environment variables**\n\nFor the most part, you can ignore all but the last 3 sections.\nThe \"Optional Features\" section lists flags that enable/disable features you may be interested in.\nThe \"Optional Packages\" section often lists dependencies and the flags needed to locate them.\nThe \"environment variables\" section lists environment variables that the build system uses to pass flags to the compiler and linker.\n\nAdding flags to configure\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nFor most of the flags you encounter, you will want a variant to optionally enable/disable them.\nYou can then optionally pass these flags to the ``configure`` call by overriding the ``configure_args`` function like so:\n\n.. code-block:: python\n\n def configure_args(self):\n args = []\n ...\n if self.spec.satisfies(\"+mpi\"):\n args.append(\"--enable-mpi\")\n else:\n args.append(\"--disable-mpi\")\n\n return args\n\n\nAlternatively, you can use the :ref:`enable_or_disable ` helper:\n\n.. code-block:: python\n\n def configure_args(self):\n args = []\n ...\n args.extend(self.enable_or_disable(\"mpi\"))\n return args\n\n\nNote that we are explicitly disabling MPI support if it is not requested.\nThis is important, as many Autotools packages will enable options by default if the dependencies are found, and disable them otherwise.\nWe want Spack installations to be as deterministic as possible.\nIf two users install a package with the same variants, the goal is that both installations work the same way.\nSee `here `__ and `here `__ for a rationale as to why these so-called \"automagic\" dependencies are a problem.\n\n.. note::\n\n By default, Autotools installs packages to ``/usr``.\n We don't want this, so Spack automatically adds ``--prefix=/path/to/installation/prefix`` to your list of ``configure_args``.\n You don't need to add this yourself.\n\n.. _autotools_helper_functions:\n\nHelper functions\n^^^^^^^^^^^^^^^^\n\nYou may have noticed that most of the Autotools flags are of the form ``--enable-foo``, ``--disable-bar``, ``--with-baz=``, or ``--without-baz``.\nSince these flags are so common, Spack provides a couple of helper functions to make your life easier.\n\n.. _autotools_enable_or_disable:\n\n``enable_or_disable``\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nAutotools flags for simple boolean variants can be automatically generated by calling the ``enable_or_disable`` method.\nThis is typically used to enable or disable some feature within the package.\n\n.. code-block:: python\n\n variant(\n \"memchecker\",\n default=False,\n description=\"Memchecker support for debugging [degrades performance]\",\n )\n ...\n\n\n def configure_args(self):\n args = []\n ...\n args.extend(self.enable_or_disable(\"memchecker\"))\n\n return args\n\nIn this example, specifying the variant ``+memchecker`` will generate the following configuration options:\n\n.. code-block:: console\n\n --enable-memchecker\n\n``with_or_without``\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nAutotools flags for more complex variants, including boolean variants and multi-valued variants, can be automatically generated by calling the ``with_or_without`` method.\n\n.. code-block:: python\n\n variant(\n \"schedulers\",\n values=disjoint_sets(\n (\"auto\",), (\"alps\", \"lsf\", \"tm\", \"slurm\", \"sge\", \"loadleveler\")\n ).with_non_feature_values(\"auto\", \"none\"),\n description=\"List of schedulers for which support is enabled; \"\n \"'auto' lets openmpi determine\",\n )\n if not spec.satisfies(\"schedulers=auto\"):\n config_args.extend(self.with_or_without(\"schedulers\"))\n\nIn this example, specifying the variant ``schedulers=slurm,sge`` will generate the following configuration options:\n\n.. code-block:: console\n\n --with-slurm --with-sge\n\n``enable_or_disable`` is actually functionally equivalent to ``with_or_without``, and accepts the same arguments and variant types; but idiomatic Autotools packages often follow these naming conventions.\n\n``activation_value``\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nAutotools parameters that require an option can still be automatically generated, using the ``activation_value`` argument to ``with_or_without`` (or, rarely, ``enable_or_disable``).\n\n.. code-block:: python\n\n variant(\n \"fabrics\",\n values=disjoint_sets(\n (\"auto\",), (\"psm\", \"psm2\", \"verbs\", \"mxm\", \"ucx\", \"libfabric\")\n ).with_non_feature_values(\"auto\", \"none\"),\n description=\"List of fabrics that are enabled; 'auto' lets openmpi determine\",\n )\n if not spec.satisfies(\"fabrics=auto\"):\n config_args.extend(self.with_or_without(\"fabrics\", activation_value=\"prefix\"))\n\n``activation_value`` accepts a callable that generates the configure parameter value given the variant value; but the special value ``prefix`` tells Spack to automatically use the dependency's installation prefix, which is the most common use for such parameters.\nIn this example, specifying the variant ``fabrics=libfabric`` will generate the following configuration options:\n\n.. code-block:: console\n\n --with-libfabric=\n\nThe ``variant`` keyword\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nWhen Spack variants and configure flags do not correspond one-to-one, the ``variant`` keyword can be passed to ``with_or_without`` and ``enable_or_disable``.\nFor example:\n\n.. code-block:: python\n\n variant(\"debug_tools\", default=False)\n config_args += self.enable_or_disable(\"debug-tools\", variant=\"debug_tools\")\n\nOr when one variant controls multiple flags:\n\n.. code-block:: python\n\n variant(\"debug_tools\", default=False)\n config_args += self.with_or_without(\"memchecker\", variant=\"debug_tools\")\n config_args += self.with_or_without(\"profiler\", variant=\"debug_tools\")\n\n\nConditional variants\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nWhen a variant is conditional and its condition is not met on the concrete spec, the ``with_or_without`` and ``enable_or_disable`` methods will simply return an empty list.\n\nFor example:\n\n.. code-block:: python\n\n variant(\"profiler\", when=\"@2.0:\")\n config_args += self.with_or_without(\"profiler\")\n\nwill neither add ``--with-profiler`` nor ``--without-profiler`` when the version is below ``2.0``.\n\nActivation overrides\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nFinally, the behavior of either ``with_or_without`` or ``enable_or_disable`` can be overridden for specific variant values.\nThis is most useful for multi-value variants where some of the variant values require atypical behavior.\n\n.. code-block:: python\n\n def with_or_without_verbs(self, activated):\n # Up through version 1.6, this option was named --with-openib.\n # In version 1.7, it was renamed to be --with-verbs.\n opt = \"verbs\" if self.spec.satisfies(\"@1.7:\") else \"openib\"\n if not activated:\n return f\"--without-{opt}\"\n return f\"--with-{opt}={self.spec['rdma-core'].prefix}\"\n\nDefining ``with_or_without_verbs`` overrides the behavior of a ``fabrics=verbs`` variant, changing the configure-time option to ``--with-openib`` for older versions of the package and specifying an alternative dependency name:\n\n.. code-block:: text\n\n --with-openib=\n\nConfigure script in a sub-directory\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nOccasionally, developers will hide their source code and ``configure`` script in a subdirectory like ``src``.\nIf this happens, Spack won't be able to automatically detect the build system properly when running ``spack create``.\nYou will have to manually change the package base class and tell Spack where the ``configure`` script resides.\nYou can do this like so:\n\n.. code-block:: python\n\n configure_directory = \"src\"\n\nBuilding out of source\n^^^^^^^^^^^^^^^^^^^^^^\n\nSome packages like ``gcc`` recommend building their software in a different directory than the source code to prevent build pollution.\nThis can be done using the ``build_directory`` variable:\n\n.. code-block:: python\n\n build_directory = \"spack-build\"\n\nBy default, Spack will build the package in the same directory that contains the ``configure`` script.\n\nBuild and install targets\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nFor most Autotools packages, the usual:\n\n.. code-block:: console\n\n $ configure\n $ make\n $ make install\n\nis sufficient to install the package.\nHowever, if you need to run make with any other targets, for example, to build an optional library or build the documentation, you can add these like so:\n\n.. code-block:: python\n\n build_targets = [\"all\", \"docs\"]\n install_targets = [\"install\", \"docs\"]\n\nTesting\n^^^^^^^\n\nAutotools-based packages typically provide unit testing via the ``check`` and ``installcheck`` targets.\nIf you build your software with ``spack install --test=root``, Spack will check for the presence of a ``check`` or ``test`` target in the Makefile and run ``make check`` for you.\nAfter installation, it will check for an ``installcheck`` target and run ``make installcheck`` if it finds one.\n\nExternal documentation\n^^^^^^^^^^^^^^^^^^^^^^\n\nFor more information on the Autotools build system, see: https://www.gnu.org/software/automake/manual/html_node/Autotools-Introduction.html\n" }, { "path": "lib/spack/docs/build_systems/bundlepackage.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Discover how to create meta-packages known as \"bundles\" in Spack to group multiple packages together for a single installation.\n\n.. _bundlepackage:\n\nBundle\n------\n\n``BundlePackage`` represents a set of packages that are expected to work well together, such as a collection of commonly used software libraries.\nThe associated software is specified as dependencies.\n\nIf it makes sense, variants, conflicts, and requirements can be added to the package.\n:ref:`Variants ` ensure that common build options are consistent across the packages supporting them.\n:ref:`Conflicts ` prevent attempts to build with known bugs and limitations.\n:ref:`Requirements ` prevent attempts to build without critical options.\n\nFor example, if ``MyBundlePackage`` is known to only build on ``linux``, it could use the ``require`` directive as follows:\n\n.. code-block:: python\n\n require(\"platform=linux\", msg=\"MyBundlePackage only builds on linux\")\n\nSpack has a number of built-in bundle packages, such as:\n\n* `AmdAocl `_\n* `EcpProxyApps `_\n* `Libc `_\n* `Xsdk `_\n\nwhere ``Xsdk`` also inherits from ``CudaPackage`` and ``RocmPackage`` and ``Libc`` is a virtual bundle package for the C standard library.\n\n\nCreation\n^^^^^^^^\n\nBe sure to specify the ``bundle`` template if you are using ``spack create`` to generate a package from the template.\nFor example, use the following command to create a bundle package whose class name will be ``Mybundle``:\n\n.. code-block:: console\n\n $ spack create --template bundle --name mybundle\n\n\n\nPhases\n^^^^^^\n\nThe ``BundlePackage`` base class does not provide any phases by default since the bundle does not represent a build system.\n\n\nURL\n^^^^^^\n\nThe ``url`` property does not have meaning since there is no package-specific code to fetch.\n\n\nVersion\n^^^^^^^\n\nAt least one ``version`` must be specified in order for the package to build.\n" }, { "path": "lib/spack/docs/build_systems/cachedcmakepackage.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Learn about the CachedCMakePackage build system in Spack for CMake-based projects.\n\n.. _cachedcmakepackage:\n\nCachedCMake\n-----------\n\nThe CachedCMakePackage base class is used for CMake-based workflows that create a CMake cache file prior to running ``cmake``.\nThis is useful for packages with arguments longer than the system limit, and for reproducibility.\n\nThe documentation for this class assumes that the user is familiar with the ``CMakePackage`` class from which it inherits.\nSee the documentation for :ref:`CMakePackage `.\n\nPhases\n^^^^^^\n\nThe ``CachedCMakePackage`` base class comes with the following phases:\n\n#. ``initconfig`` - generate the CMake cache file\n#. ``cmake`` - generate the Makefile\n#. ``build`` - build the package\n#. ``install`` - install the package\n\nBy default, these phases run:\n\n.. code-block:: console\n\n $ mkdir spack-build\n $ cd spack-build\n $ cat << EOF > name-arch-compiler@version.cmake\n # Write information on compilers and dependencies\n # includes information on mpi and cuda if applicable\n $ cmake .. -DCMAKE_INSTALL_PREFIX=/path/to/installation/prefix -C name-arch-compiler@version.cmake\n $ make\n $ make test # optional\n $ make install\n\nThe ``CachedCMakePackage`` class inherits from the ``CMakePackage`` class, and accepts all of the same options and adds all of the same flags to the ``cmake`` command.\nSimilar to the ``CMakePackage`` class, you may need to add a few arguments yourself, and the ``CachedCMakePackage`` provides the same interface to add those flags.\n\nAdding entries to the CMake cache\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nIn addition to adding flags to the ``cmake`` command, you may need to add entries to the CMake cache in the ``initconfig`` phase.\nThis can be done by overriding one of four methods:\n\n#. ``CachedCMakePackage.initconfig_compiler_entries``\n#. ``CachedCMakePackage.initconfig_mpi_entries``\n#. ``CachedCMakePackage.initconfig_hardware_entries``\n#. ``CachedCMakePackage.initconfig_package_entries``\n\nEach of these methods returns a list of CMake cache strings.\nThe distinction between these methods is merely to provide a well-structured and legible CMake cache file -- otherwise, entries from each of these methods are handled identically.\n\nSpack also provides convenience methods for generating CMake cache entries.\nThese methods are available at module scope in every Spack package.\nBecause CMake parses boolean options, strings, and paths differently, there are three such methods:\n\n#. ``cmake_cache_option``\n#. ``cmake_cache_string``\n#. ``cmake_cache_path``\n\nThese methods each accept three parameters -- the name of the CMake variable associated with the entry, the value of the entry, and an optional comment -- and return strings in the appropriate format to be returned from any of the ``initconfig*`` methods.\nAdditionally, these methods may return comments beginning with the ``#`` character.\n\nA typical usage of these methods may look something like this:\n\n.. code-block:: python\n\n def initconfig_mpi_entries(self):\n # Get existing MPI configurations\n entries = super(self, Foo).initconfig_mpi_entries()\n\n # The existing MPI configurations key on whether ``mpi`` is in the spec\n # This spec has an MPI variant, and we need to enable MPI when it is on.\n # This hypothetical package controls MPI with the ``FOO_MPI`` option to\n # cmake.\n if self.spec.satisfies(\"+mpi\"):\n entries.append(cmake_cache_option(\"FOO_MPI\", True, \"enable mpi\"))\n else:\n entries.append(cmake_cache_option(\"FOO_MPI\", False, \"disable mpi\"))\n\n\n def initconfig_package_entries(self):\n # Package specific options\n entries = []\n\n entries.append(\"#Entries for build options\")\n\n bar_on = self.spec.satisfies(\"+bar\")\n entries.append(cmake_cache_option(\"FOO_BAR\", bar_on, \"toggle bar\"))\n\n entries.append(\"#Entries for dependencies\")\n\n if self.spec[\"blas\"].name == \"baz\": # baz is our blas provider\n entries.append(cmake_cache_string(\"FOO_BLAS\", \"baz\", \"Use baz\"))\n entries.append(cmake_cache_path(\"BAZ_PREFIX\", self.spec[\"baz\"].prefix))\n\nExternal documentation\n^^^^^^^^^^^^^^^^^^^^^^\n\nFor more information on CMake cache files, see: https://cmake.org/cmake/help/latest/manual/cmake.1.html\n" }, { "path": "lib/spack/docs/build_systems/cmakepackage.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Learn how to package software that uses the CMake build system with Spack, covering common practices and customization options for CMake-based packages.\n\n.. _cmakepackage:\n\nCMake\n------\n\nLike Autotools, CMake is a widely-used build-script generator.\nDesigned by Kitware, CMake is the most popular build system for new C, C++, and Fortran projects, and many older projects are switching to it as well.\n\nUnlike Autotools, CMake can generate build scripts for builders other than Make: Ninja, Visual Studio, etc.\nIt is therefore cross-platform, whereas Autotools is Unix-only.\n\nPhases\n^^^^^^\n\nThe ``CMakeBuilder`` and ``CMakePackage`` base classes come with the following phases:\n\n#. ``cmake`` - generate the Makefile\n#. ``build`` - build the package\n#. ``install`` - install the package\n\nBy default, these phases run:\n\n.. code-block:: console\n\n $ mkdir spack-build\n $ cd spack-build\n $ cmake .. -DCMAKE_INSTALL_PREFIX=/path/to/installation/prefix\n $ make\n $ make test # optional\n $ make install\n\n\nA few more flags are passed to ``cmake`` by default, including flags for setting the build type and flags for locating dependencies.\nOf course, you may need to add a few arguments yourself.\n\nImportant files\n^^^^^^^^^^^^^^^\n\nA CMake-based package can be identified by the presence of a ``CMakeLists.txt`` file.\nThis file defines the build flags that can be passed to the CMake invocation, as well as linking instructions.\nIf you are familiar with CMake, it can prove very useful for determining dependencies and dependency version requirements.\n\nOne thing to look for is the ``cmake_minimum_required`` function:\n\n.. code-block:: cmake\n\n cmake_minimum_required(VERSION 2.8.12)\n\n\nThis means that CMake 2.8.12 is the earliest release that will work.\nYou should specify this in a ``depends_on`` statement.\n\nCMake-based packages may also contain ``CMakeLists.txt`` in subdirectories.\nThis modularization helps to manage complex builds in a hierarchical fashion.\nSometimes these nested ``CMakeLists.txt`` require additional dependencies not mentioned in the top-level file.\n\nThere's also usually a ``cmake`` or ``CMake`` directory containing additional macros, find scripts, etc.\nThese may prove useful in determining dependency version requirements.\n\nBuild system dependencies\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nEvery package that uses the CMake build system requires a ``cmake`` dependency.\nSince this is always the case, the ``CMakePackage`` base class already contains:\n\n.. code-block:: python\n\n depends_on(\"cmake\", type=\"build\")\n\n\nIf you need to specify a particular version requirement, you can override this in your package:\n\n.. code-block:: python\n\n depends_on(\"cmake@2.8.12:\", type=\"build\")\n\n\nFinding cmake flags\n^^^^^^^^^^^^^^^^^^^\n\nTo get a list of valid flags that can be passed to ``cmake``, run the following command in the directory that contains ``CMakeLists.txt``:\n\n.. code-block:: console\n\n $ cmake . -LAH\n\n\nCMake will start by checking for compilers and dependencies.\nEventually it will begin to list build options.\nYou'll notice that most of the build options at the top are prefixed with ``CMAKE_``.\nYou can safely ignore most of these options as Spack already sets them for you.\nThis includes flags needed to locate dependencies, RPATH libraries, set the installation directory, and set the build type.\n\nThe rest of the flags are the ones you should consider adding to your package.\nThey often include flags to enable/disable support for certain features and locate specific dependencies.\nOne thing you'll notice that makes CMake different from Autotools is that CMake has an understanding of build flag hierarchy.\nThat is, certain flags will not display unless their parent flag has been selected.\nFor example, flags to specify the ``lib`` and ``include`` directories for a package might not appear unless CMake found the dependency it was looking for.\nYou may need to manually specify certain flags to explore the full depth of supported build flags, or check the ``CMakeLists.txt`` yourself.\n\n.. _cmake_args:\n\nAdding flags to cmake\n^^^^^^^^^^^^^^^^^^^^^\n\nTo add additional flags to the ``cmake`` call, simply override the ``cmake_args`` function.\nThe following example defines values for the flags ``WHATEVER``, ``ENABLE_BROKEN_FEATURE``, ``DETECT_HDF5``, and ``THREADS`` with and without the :meth:`~spack_repo.builtin.build_systems.cmake.CMakeBuilder.define` and :meth:`~spack_repo.builtin.build_systems.cmake.CMakeBuilder.define_from_variant` helper functions:\n\n.. code-block:: python\n\n def cmake_args(self):\n args = [\n \"-DWHATEVER:STRING=somevalue\",\n self.define(\"ENABLE_BROKEN_FEATURE\", False),\n self.define_from_variant(\"DETECT_HDF5\", \"hdf5\"),\n self.define_from_variant(\"THREADS\"), # True if +threads\n ]\n\n return args\n\nSpack supports CMake defines from conditional variants too.\nWhenever the condition on the variant is not met, ``define_from_variant()`` will simply return an empty string, and CMake simply ignores the empty command line argument.\nFor example, the following\n\n.. code-block:: python\n\n variant(\"example\", default=True, when=\"@2.0:\")\n\n\n def cmake_args(self):\n return [self.define_from_variant(\"EXAMPLE\", \"example\")]\n\nwill generate ``'cmake' '-DEXAMPLE=ON' ...`` when `@2.0: +example` is met, but will result in ``'cmake' '' ...`` when the spec version is below ``2.0``.\n\nCMake arguments provided by Spack\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nThe following default arguments are controlled by Spack:\n\n\n``CMAKE_INSTALL_PREFIX``\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nIs set to the package's install directory.\n\n\n``CMAKE_PREFIX_PATH``\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nCMake finds dependencies through calls to ``find_package()``, ``find_program()``, ``find_library()``, ``find_file()``, and ``find_path()``, which use a list of search paths from ``CMAKE_PREFIX_PATH``.\nSpack sets this variable to a list of prefixes of the spec's transitive dependencies.\n\nFor troubleshooting cases where CMake fails to find a dependency, add the ``--debug-find`` flag to ``cmake_args``.\n\n``CMAKE_BUILD_TYPE``\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nEvery CMake-based package accepts a ``-DCMAKE_BUILD_TYPE`` flag to dictate which level of optimization to use.\nIn order to ensure uniformity across packages, the ``CMakePackage`` base class adds a variant to control this:\n\n.. code-block:: python\n\n variant(\n \"build_type\",\n default=\"RelWithDebInfo\",\n description=\"CMake build type\",\n values=(\"Debug\", \"Release\", \"RelWithDebInfo\", \"MinSizeRel\"),\n )\n\nHowever, not every CMake package accepts all four of these options.\nGrep the ``CMakeLists.txt`` file to see if the default values are missing or replaced.\nFor example, the `dealii `_ package overrides the default variant with:\n\n.. code-block:: python\n\n variant(\n \"build_type\",\n default=\"DebugRelease\",\n description=\"The build type to build\",\n values=(\"Debug\", \"Release\", \"DebugRelease\"),\n )\n\nFor more information on ``CMAKE_BUILD_TYPE``, see: https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html\n\n\n``CMAKE_INSTALL_RPATH`` and ``CMAKE_INSTALL_RPATH_USE_LINK_PATH=ON``\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nCMake uses different RPATHs during the build and after installation, so that executables can locate the libraries they're linked to during the build, and installed executables do not have RPATHs to build directories.\nIn Spack, we have to make sure that RPATHs are set properly after installation.\n\nSpack sets ``CMAKE_INSTALL_RPATH`` to a list of ``/lib`` or ``/lib64`` directories of the spec's link-type dependencies.\nApart from that, it sets ``-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON``, which should add RPATHs for directories of linked libraries not in the directories covered by ``CMAKE_INSTALL_RPATH``.\n\nUsually it's enough to set only ``-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON``, but the reason to provide both options is that packages may dynamically open shared libraries, which CMake cannot detect.\nIn those cases, the RPATHs from ``CMAKE_INSTALL_RPATH`` are used as search paths.\n\n.. note::\n\n Some packages provide stub libraries, which contain an interface for linking without an implementation.\n When using such libraries, it's best to override the option ``-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=OFF`` in ``cmake_args``, so that stub libraries are not used at runtime.\n\n\nGenerators\n^^^^^^^^^^\n\nCMake and Autotools are build-script generation tools; they \"generate\" the Makefiles that are used to build a software package.\nCMake actually supports multiple generators, not just Makefiles.\nAnother common generator is Ninja.\nTo switch to the Ninja generator, simply add:\n\n.. code-block:: python\n\n generator(\"ninja\")\n\n\n``CMakePackage`` defaults to \"Unix Makefiles\".\nIf you switch to the Ninja generator, make sure to add:\n\n.. code-block:: python\n\n depends_on(\"ninja\", type=\"build\")\n\nto the package as well.\nAside from that, you shouldn't need to do anything else.\nSpack will automatically detect that you are using Ninja and run:\n\n.. code-block:: console\n\n $ cmake .. -G Ninja\n $ ninja\n $ ninja install\n\nSpack currently only supports \"Unix Makefiles\" and \"Ninja\" as valid generators, but it should be simple to add support for alternative generators.\nFor more information on CMake generators, see: https://cmake.org/cmake/help/latest/manual/cmake-generators.7.html\n\nCMakeLists.txt in a sub-directory\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nOccasionally, developers will hide their source code and ``CMakeLists.txt`` in a subdirectory like ``src``.\nIf this happens, Spack won't be able to automatically detect the build system properly when running ``spack create``.\nYou will have to manually change the package base class and tell Spack where ``CMakeLists.txt`` resides.\nYou can do this like so:\n\n.. code-block:: python\n\n root_cmakelists_dir = \"src\"\n\n\nNote that this path is relative to the root of the extracted tarball, not to the ``build_directory``.\nIt defaults to the current directory.\n\nBuilding out of source\n^^^^^^^^^^^^^^^^^^^^^^\n\nBy default, Spack builds every ``CMakePackage`` in a ``spack-build`` sub-directory.\nIf, for whatever reason, you would like to build in a different sub-directory, simply override ``build_directory`` like so:\n\n.. code-block:: python\n\n build_directory = \"my-build\"\n\nBuild and install targets\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nFor most CMake packages, the usual:\n\n.. code-block:: console\n\n $ cmake\n $ make\n $ make install\n\nis sufficient to install the package.\nHowever, if you need to run make with any other targets, for example, to build an optional library or build the documentation, you can add these like so:\n\n.. code-block:: python\n\n build_targets = [\"all\", \"docs\"]\n install_targets = [\"install\", \"docs\"]\n\nTesting\n^^^^^^^\n\nCMake-based packages typically provide unit testing via the ``test`` target.\nIf you build your software with ``--test=root``, Spack will check for the presence of a ``test`` target in the Makefile and run ``make test`` for you.\nIf you want to run a different test instead, simply override the ``check`` method.\n\nExternal documentation\n^^^^^^^^^^^^^^^^^^^^^^\n\nFor more information on the CMake build system, see: https://cmake.org/cmake/help/latest/\n" }, { "path": "lib/spack/docs/build_systems/cudapackage.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n A guide to packaging CUDA applications with Spack, including helpers for managing CUDA dependencies and architecture-specific builds.\n\n.. _cudapackage:\n\nCuda\n------\n\nDifferent from other packages, ``CudaPackage`` does not represent a build system.\nInstead its goal is to simplify and unify usage of ``CUDA`` in other packages by providing a `mixin-class `_.\n\nYou can find source for the package at ``__.\n\nVariants\n^^^^^^^^\n\nThis package provides the following variants:\n\n* **cuda**\n\n This variant is used to enable/disable building with ``CUDA``.\n The default is disabled (or ``False``).\n\n* **cuda_arch**\n\n This variant supports the optional specification of one or more architectures.\n Valid values are maintained in the ``cuda_arch_values`` property and are the numeric character equivalent of the compute capability version (e.g., '10' for version 1.0).\n Each provided value affects associated ``CUDA`` dependencies and compiler conflicts.\n\n The variant builds both PTX code for the *virtual* architecture (e.g. ``compute_10``) and binary code for the *real* architecture (e.g. ``sm_10``).\n\n GPUs and their compute capability versions are listed at https://developer.nvidia.com/cuda-gpus.\n\nConflicts\n^^^^^^^^^\n\nConflicts are used to prevent builds with known bugs or issues.\nWhile base ``CUDA`` conflicts have been included with this package, you may want to add more for your software.\n\nFor example, if your package requires ``cuda_arch`` to be specified when ``cuda`` is enabled, you can add the following conflict to your package to terminate such build attempts with a suitable message:\n\n.. code-block:: python\n\n conflicts(\"cuda_arch=none\", when=\"+cuda\", msg=\"CUDA architecture is required\")\n\nSimilarly, if your software does not support all versions of the property, you could add ``conflicts`` to your package for those versions.\nFor example, suppose your software does not work with CUDA compute capability versions prior to SM 5.0 (``50``).\nYou can add the following code to display a custom message should a user attempt such a build:\n\n.. code-block:: python\n\n unsupported_cuda_archs = [\"10\", \"11\", \"12\", \"13\", \"20\", \"21\", \"30\", \"32\", \"35\", \"37\"]\n for value in unsupported_cuda_archs:\n conflicts(\n f\"cuda_arch={value}\", when=\"+cuda\", msg=f\"CUDA architecture {value} is not supported\"\n )\n\nMethods\n^^^^^^^\n\nThis package provides one custom helper method, which is used to build standard CUDA compiler flags.\n\n**cuda_flags**\n This built-in static method returns a list of command line flags for the chosen ``cuda_arch`` value(s).\n The flags are intended to be passed to the CUDA compiler driver (i.e., ``nvcc``).\n\n This method must be explicitly called when you are creating the arguments for your build in order to use the values.\n\nUsage\n^^^^^^\n\nThis helper package can be added to your package by adding it as a base class of your package.\nFor example, you can add it to your :ref:`CMakePackage `-based package as follows:\n\n.. code-block:: python\n :emphasize-lines: 1,8-17\n\n class MyCudaPackage(CMakePackage, CudaPackage):\n ...\n\n def cmake_args(self):\n spec = self.spec\n args = []\n ...\n if spec.satisfies(\"+cuda\"):\n # Set up the CUDA macros needed by the build\n args.append(\"-DWITH_CUDA=ON\")\n cuda_arch_list = spec.variants[\"cuda_arch\"].value\n cuda_arch = cuda_arch_list[0]\n if cuda_arch != \"none\":\n args.append(f\"-DCUDA_FLAGS=-arch=sm_{cuda_arch}\")\n else:\n # Ensure build with CUDA is disabled\n args.append(\"-DWITH_CUDA=OFF\")\n ...\n return args\n\nassuming only the ``WITH_CUDA`` and ``CUDA_FLAGS`` flags are required.\nYou will need to customize options as needed for your build.\n\nThis example also illustrates how to check for the ``cuda`` variant using ``self.spec`` and how to retrieve the ``cuda_arch`` variant's value, which is a list, using ``self.spec.variants[\"cuda_arch\"].value``.\n\nWith over 70 packages using ``CudaPackage`` as of January 2021 there are lots of examples to choose from to get more ideas for using this package.\n" }, { "path": "lib/spack/docs/build_systems/custompackage.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n A guide to creating custom build systems in Spack for packaging software with its own build scripts or adding support for new build systems.\n\n.. _custompackage:\n\nCustom Build Systems\n--------------------\n\nWhile the built-in build systems should meet your needs for the vast majority of packages, some packages provide custom build scripts.\nThis guide is intended for the following use cases:\n\n* Packaging software with its own custom build system\n* Adding support for new build systems\n\nIf you want to add support for a new build system, a good place to start is to look at the definitions of other build systems.\nThis guide focuses mostly on how Spack's build systems work.\n\nIn this guide, we will be using the `perl `_ and `cmake `_ packages as examples.\n``perl``'s build system is a hand-written ``Configure`` shell script, while ``cmake`` bootstraps itself during installation.\nBoth of these packages require custom build systems.\n\nBase class\n^^^^^^^^^^\n\nIf your package does not belong to any of the built-in build systems that Spack already supports, you should inherit from the ``Package`` base class.\n``Package`` is a simple base class with a single phase: ``install``.\nIf your package is simple, you may be able to simply write an ``install`` method that gets the job done.\nHowever, if your package is more complex and installation involves multiple steps, you should add separate phases as mentioned in the next section.\n\nIf you are creating a new build system base class, you should inherit from ``PackageBase``.\nThis is the superclass for all build systems in Spack.\n\nPhases\n^^^^^^\n\nThe most important concept in Spack's build system support is the idea of phases.\nEach build system defines a set of phases that are necessary to install the package.\nThey usually follow some sort of \"configure\", \"build\", \"install\" guideline, but any of those phases may be missing or combined with another phase.\n\nIf you look at the ``perl`` package, you'll see:\n\n.. code-block:: python\n\n phases = (\"configure\", \"build\", \"install\")\n\nSimilarly, ``cmake`` defines:\n\n.. code-block:: python\n\n phases = (\"bootstrap\", \"build\", \"install\")\n\nIf we look at the ``cmake`` example, this tells Spack's ``PackageBase`` class to run the ``bootstrap``, ``build``, and ``install`` functions in that order.\nIt is now up to you to define these methods.\n\nPhase and phase_args functions\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nIf we look at ``perl``, we see that it defines a ``configure`` method:\n\n.. code-block:: python\n\n def configure(self, spec, prefix):\n configure = Executable(\"./Configure\")\n configure(*self.configure_args())\n\nThere is also a corresponding ``configure_args`` function that handles all of the arguments to pass to ``Configure``, just like in ``AutotoolsPackage``.\nComparatively, the ``build`` and ``install`` phases are pretty simple:\n\n.. code-block:: python\n\n def build(self, spec, prefix):\n make()\n\n\n def install(self, spec, prefix):\n make(\"install\")\n\nThe ``cmake`` package looks very similar, but with a ``bootstrap`` function instead of ``configure``:\n\n.. code-block:: python\n\n def bootstrap(self, spec, prefix):\n bootstrap = Executable(\"./bootstrap\")\n bootstrap(*self.bootstrap_args())\n\n\n def build(self, spec, prefix):\n make()\n\n\n def install(self, spec, prefix):\n make(\"install\")\n\nAgain, there is a ``bootstrap_args`` function that determines the correct bootstrap flags to use.\n\n``run_before`` / ``run_after``\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nOccasionally, you may want to run extra steps either before or after a given phase.\nThis applies not just to custom build systems, but to existing build systems as well.\nYou may need to patch a file that is generated by configure, or install extra files in addition to what ``make install`` copies to the installation prefix.\nThis is where ``@run_before`` and ``@run_after`` come in.\n\nThese Python decorators allow you to write functions that are called before or after a particular phase.\nFor example, in ``perl``, we see:\n\n.. code-block:: python\n\n @run_after(\"install\")\n def install_cpanm(self):\n spec = self.spec\n maker = make\n cpan_dir = join_path(\"cpanm\", \"cpanm\")\n if sys.platform == \"win32\":\n maker = nmake\n cpan_dir = join_path(self.stage.source_path, cpan_dir)\n cpan_dir = windows_sfn(cpan_dir)\n if \"+cpanm\" in spec:\n with working_dir(cpan_dir):\n perl = spec[\"perl\"].command\n perl(\"Makefile.PL\")\n maker()\n maker(\"install\")\n\nThis extra step automatically installs ``cpanm`` in addition to the base Perl installation.\n\n``on_package_attributes``\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nThe ``run_before`` / ``run_after`` logic discussed above becomes particularly powerful when combined with the ``@on_package_attributes`` decorator.\nThis decorator allows you to conditionally run certain functions depending on the attributes of that package.\nThe most common example is conditional testing.\nMany unit tests are prone to failure, even when there is nothing wrong with the installation.\nUnfortunately, non-portable unit tests and tests that are \"supposed to fail\" are more common than we would like.\nInstead of always running unit tests on installation, Spack lets users conditionally run tests with the ``--test=root`` flag.\n\nIf we wanted to define a function that would conditionally run if and only if this flag is set, we would use the following:\n\n.. code-block:: python\n\n @on_package_attributes(run_tests=True)\n def my_test_function(self): ...\n\nTesting\n^^^^^^^\n\nLet's put everything together and add unit tests to be optionally run during the installation of our package.\nIn the ``perl`` package, we can see:\n\n.. code-block:: python\n\n @run_after(\"build\")\n @on_package_attributes(run_tests=True)\n def build_test(self):\n if sys.platform == \"win32\":\n win32_dir = os.path.join(self.stage.source_path, \"win32\")\n win32_dir = windows_sfn(win32_dir)\n with working_dir(win32_dir):\n nmake(\"test\", ignore_quotes=True)\n else:\n make(\"test\")\n\nAs you can guess, this runs ``make test`` *after* building the package, if and only if testing is requested.\nAgain, this is not specific to custom build systems, it can be added to existing build systems as well.\n\n.. warning::\n\n The order of decorators matters.\n The following ordering:\n\n .. code-block:: python\n\n @run_after(\"install\")\n @on_package_attributes(run_tests=True)\n def my_test_function(self): ...\n\n works as expected.\n However, if you reverse the ordering:\n\n .. code-block:: python\n\n @on_package_attributes(run_tests=True)\n @run_after(\"install\")\n def my_test_function(self): ...\n\n the tests will always be run regardless of whether or not ``--test=root`` is requested.\n See https://github.com/spack/spack/issues/3833 for more information\n\nIdeally, every package in Spack will have some sort of test to ensure that it was built correctly.\nIt is up to the package authors to make sure this happens.\nIf you are adding a package for some software and the developers list commands to test the installation, please add these tests to your ``package.py``.\n\nFor more information on other forms of package testing, refer to :ref:`Checking an installation `.\n" }, { "path": "lib/spack/docs/build_systems/inteloneapipackage.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n A guide to using the Intel oneAPI packages in Spack, including how to build with icx, use the oneAPI Spack environment, and configure externally installed oneAPI tools.\n\n.. _inteloneapipackage:\n\n\nIntelOneapi\n===========\n\n\nSpack can install and use the Intel oneAPI products.\nYou may either use Spack to install the oneAPI tools or use the `Intel installers`_.\nAfter installation, you may use the tools directly, or use Spack to build packages with the tools.\n\nThe Spack Python class ``IntelOneapiPackage`` is a base class that is used by ``IntelOneapiCompilers``, ``IntelOneapiMkl``, ``IntelOneapiTbb`` and other classes to implement the oneAPI packages.\nSearch for ``oneAPI`` at `packages.spack.io `_ for the full list of available oneAPI packages, or use:\n\n.. code-block:: console\n\n $ spack list -d oneAPI\n\nFor more information on a specific package, do:\n\n.. code-block:: console\n\n $ spack info --all \n\n\nBuilding a Package With icx\n---------------------------\n\nIn this example, we build patchelf with ``icc`` and ``icx``.\nThe compilers are installed with Spack.\n\nInstall the oneAPI compilers:\n\n.. code-block:: spec\n\n $ spack install intel-oneapi-compilers\n\n\nTo build the ``patchelf`` Spack package with ``icx``, do:\n\n.. code-block:: spec\n\n $ spack install patchelf%oneapi\n\n\nUsing oneAPI Spack environment\n-------------------------------\n\nIn this example, we build LAMMPS with ``icx`` using Spack environment for oneAPI packages created by Intel.\nThe compilers are installed with Spack like in example above.\n\nInstall the oneAPI compilers:\n\n.. code-block:: spec\n\n $ spack install intel-oneapi-compilers\n\nClone `spack-configs `_ repo and activate Intel oneAPI CPU environment:\n\n.. code-block:: console\n\n $ git clone https://github.com/spack/spack-configs\n $ spack env activate spack-configs/INTEL/CPU\n $ spack concretize -f\n\n`Intel oneAPI CPU environment `_ contains applications tested and validated by Intel.\nThis list is constantly extended.\nCurrently, it supports:\n\n- `Devito `_\n- `GROMACS `_\n- `HPCG `_\n- `HPL `_\n- `LAMMPS `_\n- `OpenFOAM `_\n- `Quantum Espresso `_\n- `STREAM `_\n- `WRF `_\n\nTo build LAMMPS with oneAPI compiler from this environment just run:\n\n.. code-block:: spec\n\n $ spack install lammps\n\nCompiled binaries can be found using:\n\n.. code-block:: console\n\n $ spack cd -i lammps\n\nYou can do the same for all other applications from this environment.\n\n\nUsing oneAPI MPI to Satisfy a Virtual Dependence\n------------------------------------------------\n\nThe ``hdf5`` package works with any compatible MPI implementation.\nTo build ``hdf5`` with Intel oneAPI MPI do:\n\n.. code-block:: spec\n\n $ spack install hdf5 +mpi ^intel-oneapi-mpi\n\nUsing Externally Installed oneAPI Tools\n---------------------------------------\n\nSpack can also use oneAPI tools that are manually installed with `Intel Installers`_.\nThe procedures for configuring Spack to use external compilers and libraries are different.\n\nCompilers\n^^^^^^^^^\n\nTo use the compilers, add some information about the installation to ``packages.yaml``.\nFor most users, it is sufficient to do:\n\n.. code-block:: console\n\n $ spack compiler add /opt/intel/oneapi/compiler/latest/bin\n\nAdapt the paths above if you did not install the tools in the default location.\nAfter adding the compilers, using them is the same as if you had installed the ``intel-oneapi-compilers`` package.\nAnother option is to manually add the configuration to ``packages.yaml`` as described in :ref:`Compiler configuration `.\n\nBefore 2024, the directory structure was different:\n\n.. code-block:: console\n \n $ spack compiler add /opt/intel/oneapi/compiler/latest/linux/bin/intel64\n $ spack compiler add /opt/intel/oneapi/compiler/latest/linux/bin\n\n\nLibraries\n^^^^^^^^^\n\nIf you want Spack to use oneMKL that you have installed without Spack in the default location, then add the following to ``~/.spack/packages.yaml``, adjusting the version as appropriate:\n\n.. code-block:: yaml\n\n intel-oneapi-mkl:\n externals:\n - spec: intel-oneapi-mkl@2021.1.1\n prefix: /opt/intel/oneapi/\n\n\nUsing oneAPI Tools Installed by Spack\n-------------------------------------\n\nSpack can be a convenient way to install and configure compilers and libraries, even if you do not intend to build a Spack package.\nIf you want to build a Makefile project using Spack-installed oneAPI compilers, then use Spack to configure your environment:\n\n.. code-block:: spec\n\n $ spack load intel-oneapi-compilers\n\nAnd then you can build with:\n\n.. code-block:: console\n\n $ CXX=icpx make\n\nYou can also use Spack-installed libraries.\nFor example:\n\n.. code-block:: spec\n\n $ spack load intel-oneapi-mkl\n\nThis updates your environment CPATH, LIBRARY_PATH, and other environment variables for building an application with oneMKL.\n\n\n.. _`Intel installers`: https://software.intel.com/content/www/us/en/develop/documentation/installation-guide-for-intel-oneapi-toolkits-linux/top.html\n" }, { "path": "lib/spack/docs/build_systems/luapackage.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\n.. meta::\n :description lang=en:\n Learn about the Lua build system in Spack for packages using rockspec files.\n\n.. _luapackage:\n\nLua\n------\n\nThe ``Lua`` build system is a helper for the common case of Lua packages that provide a rockspec file.\nThis is not meant to take a rock archive, but to build a source archive or repository that provides a rockspec, which should cover most Lua packages.\nIn the case a Lua package builds by Make rather than LuaRocks, prefer MakefilePackage.\n\nPhases\n^^^^^^\n\nThe ``LuaBuilder`` and ``LuaPackage`` base classes come with the following phases:\n\n#. ``unpack`` - if using a rock, unpacks the rock and moves into the source directory\n#. ``preprocess`` - adjust sources or rockspec to fix build\n#. ``install`` - install the project\n\nBy default, these phases run:\n\n.. code-block:: console\n\n # If the archive is a source rock\n $ luarocks unpack .src.rock\n $ # preprocess is a no-op by default\n $ luarocks make .rockspec\n\n\nAny of these phases can be overridden in your package as necessary.\n\nImportant files\n^^^^^^^^^^^^^^^\n\nPackages that use the Lua/LuaRocks build system can be identified by the presence of a ``*.rockspec`` file in their source tree, or can be fetched as a source rock archive (``.src.rock``).\nThis file declares things like build instructions and dependencies.\nThe ``.src.rock`` also contains all code.\n\nIt is common for the rockspec file to list the Lua version required in a dependency.\nThe LuaPackage class adds appropriate dependencies on a Lua implementation, but it is a good idea to specify the version required with a ``depends_on`` statement.\nThe block normally will be a table definition like this:\n\n.. code-block:: lua\n\n dependencies = {\n \"lua >= 5.1\",\n }\n\nThe LuaPackage class supports source repositories and archives containing a rockspec and directly downloading source rock files.\nIt *does not* support downloading dependencies listed inside a rockspec, and thus does not support directly downloading a rockspec as an archive.\n\nBuild system dependencies\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nAll base dependencies are added by the build system, but LuaRocks is run to avoid downloading extra Lua dependencies during build.\nIf the package needs Lua libraries outside the standard set, they should be added as dependencies.\n\nTo specify a Lua version constraint but allow all Lua implementations, prefer to use ``depends_on(\"lua-lang@5.1:5.1.99\")`` to express any 5.1 compatible version.\nIf the package requires LuaJit rather than Lua, a ``depends_on(\"luajit\")`` should be used to ensure a LuaJit distribution is used instead of the Lua interpreter.\nAlternately, if only interpreted Lua will work, ``depends_on(\"lua\")`` will express that.\n\nPassing arguments to luarocks make\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nIf you need to pass any arguments to the ``luarocks make`` call, you can override the ``luarocks_args`` method like so:\n\n.. code-block:: python\n\n def luarocks_args(self):\n return [\"flag1\", \"flag2\"]\n\nOne common use of this is to override warnings or flags for newer compilers, as in:\n\n.. code-block:: python\n\n def luarocks_args(self):\n return [\"CFLAGS='-Wno-error=implicit-function-declaration'\"]\n\nExternal documentation\n^^^^^^^^^^^^^^^^^^^^^^\n\nFor more information on the LuaRocks build system, see: https://luarocks.org/\n" }, { "path": "lib/spack/docs/build_systems/makefilepackage.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n A guide to using the Makefile build system in Spack for packages that use plain Makefiles.\n\n.. _makefilepackage:\n\nMakefile\n--------\n\nThe most primitive build system a package can use is a plain Makefile.\nMakefiles are simple to write for small projects, but they usually require you to edit the Makefile to set platform and compiler-specific variables.\n\nPhases\n^^^^^^\n\nThe ``MakefileBuilder`` and ``MakefilePackage`` base classes come with 3 phases:\n\n#. ``edit`` - edit the Makefile\n#. ``build`` - build the project\n#. ``install`` - install the project\n\nBy default, ``edit`` does nothing, but you can override it to replace hardcoded Makefile variables.\nThe ``build`` and ``install`` phases run:\n\n.. code-block:: console\n\n $ make\n $ make install\n\n\nImportant files\n^^^^^^^^^^^^^^^\n\nThe main file that matters for a ``MakefilePackage`` is the Makefile.\nThis file will be named one of the following ways:\n\n* GNUmakefile (only works with GNU Make)\n* Makefile (most common)\n* makefile\n\nSome Makefiles also *include* other configuration files.\nCheck for an ``include`` directive in the Makefile.\n\nBuild system dependencies\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nSpack assumes that the operating system will have a valid ``make`` utility installed already, so you don't need to add a dependency on ``make``.\nHowever, if the package uses a ``GNUmakefile`` or the developers recommend using GNU Make, you should add a dependency on ``gmake``:\n\n.. code-block:: python\n\n depends_on(\"gmake\", type=\"build\")\n\n\nTypes of Makefile packages\n^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nMost of the work involved in packaging software that uses Makefiles involves overriding or replacing hard-coded variables.\nMany packages make the mistake of hard-coding compilers, usually for GCC or Intel.\nThis is fine if you happen to be using that particular compiler, but Spack is designed to work with *any* compiler, and you need to ensure that this is the case.\n\nDepending on how the Makefile is designed, there are 4 common strategies that can be used to set or override the appropriate variables:\n\nEnvironment variables\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nMake has multiple types of `assignment operators `_.\nSome Makefiles use ``=`` to assign variables.\nThe only way to override these variables is to edit the Makefile or override them on the command-line.\nHowever, Makefiles that use ``?=`` for assignment honor environment variables.\nSince Spack already sets ``CC``, ``CXX``, ``F77``, and ``FC``, you won't need to worry about setting these variables.\nIf there are any other variables you need to set, you can do this in the ``setup_build_environment`` method:\n\n.. code-block:: python\n\n def setup_build_environment(self, env: EnvironmentModifications) -> None:\n env.set(\"PREFIX\", prefix)\n env.set(\"BLASLIB\", spec[\"blas\"].libs.ld_flags)\n\n\n`cbench `_ is a good example of a simple package that does this, while `esmf `_ is a good example of a more complex package.\n\nCommand-line arguments\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nIf the Makefile ignores environment variables, the next thing to try is command-line arguments.\nYou can do this by overriding the ``build_targets`` attribute.\nIf you don't need access to the spec, you can do this like so:\n\n.. code-block:: python\n\n build_targets = [\"CC=cc\"]\n\n\nIf you do need access to the spec, you can create a property like so:\n\n.. code-block:: python\n\n @property\n def build_targets(self):\n spec = self.spec\n\n return [\n \"CC=cc\",\n f\"BLASLIB={spec['blas'].libs.ld_flags}\",\n ]\n\n\n`cloverleaf `_ is a good example of a package that uses this strategy.\n\nEdit Makefile\n\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nSome Makefiles are just plain stubborn and will ignore command-line variables.\nThe only way to ensure that these packages build correctly is to directly edit the Makefile.\nSpack provides a ``FileFilter`` class and a ``filter`` method to help with this.\nFor example:\n\n.. code-block:: python\n\n def edit(self, spec, prefix):\n makefile = FileFilter(\"Makefile\")\n\n makefile.filter(r\"^\\s*CC\\s*=.*\", f\"CC = {spack_cc}\")\n makefile.filter(r\"^\\s*CXX\\s*=.*\", f\"CXX = {spack_cxx}\")\n makefile.filter(r\"^\\s*F77\\s*=.*\", f\"F77 = {spack_f77}\")\n makefile.filter(r\"^\\s*FC\\s*=.*\", f\"FC = {spack_fc}\")\n\n\n`stream `_ is a good example of a package that involves editing a Makefile to set the appropriate variables.\n\nConfig file\n\"\"\"\"\"\"\"\"\"\"\"\n\nMore complex packages often involve Makefiles that *include* a configuration file.\nThese configuration files are primarily composed of variables relating to the compiler, platform, and the location of dependencies or names of libraries.\nSince these config files are dependent on the compiler and platform, you will often see entire directories of examples for common compilers and architectures.\nUse these examples to help determine what possible values to use.\n\nIf the config file is long and only contains one or two variables that need to be modified, you can use the technique above to edit the config file.\nHowever, if you end up needing to modify most of the variables, it may be easier to write a new file from scratch.\n\nIf each variable is independent of each other, a dictionary works well for storing variables:\n\n.. code-block:: python\n\n def edit(self, spec, prefix):\n config = {\n \"CC\": \"cc\",\n \"MAKE\": \"make\",\n }\n\n if spec.satisfies(\"+blas\"):\n config[\"BLAS_LIBS\"] = spec[\"blas\"].libs.joined()\n\n with open(\"make.inc\", \"w\") as inc:\n for key in config:\n inc.write(f\"{key} = {config[key]}\\n\")\n\n\n`elk `_ is a good example of a package that uses a dictionary to store configuration variables.\n\nIf the order of variables is important, it may be easier to store them in a list:\n\n.. code-block:: python\n\n def edit(self, spec, prefix):\n config = [\n f\"INSTALL_DIR = {prefix}\",\n \"INCLUDE_DIR = $(INSTALL_DIR)/include\",\n \"LIBRARY_DIR = $(INSTALL_DIR)/lib\",\n ]\n\n with open(\"make.inc\", \"w\") as inc:\n for var in config:\n inc.write(f\"{var}\\n\")\n\n\n`hpl `_ is a good example of a package that uses a list to store configuration variables.\n\nVariables to watch out for\n^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nThe following is a list of common variables to watch out for.\nThe first two sections are `implicit variables `_ defined by Make and will always use the same name, while the rest are user-defined variables and may vary from package to package.\n\n* **Compilers**\n\n This includes variables such as ``CC``, ``CXX``, ``F77``, ``F90``, and ``FC``, as well as variables related to MPI compiler wrappers, like ``MPICC`` and friends.\n\n* **Compiler flags**\n\n This includes variables for compiler flags, such as ``CFLAGS``, ``CXXFLAGS``, ``F77FLAGS``, ``F90FLAGS``, ``FCFLAGS``, and ``CPPFLAGS``.\n These variables are often hard-coded to contain flags specific to a certain compiler.\n If these flags don't work for every compiler, you may want to consider filtering them.\n\n* **Variables that enable or disable features**\n\n This includes variables like ``MPI``, ``OPENMP``, ``PIC``, and ``DEBUG``.\n These flags often require you to create a variant so that you can either build with or without MPI support, for example.\n These flags are often compiler-dependent.\n You should replace them with the appropriate compiler flags, such as ``self.compiler.openmp_flag`` or ``self.compiler.pic_flag``.\n\n* **Platform flags**\n\n These flags control the type of architecture that the executable is compiled for.\n Watch out for variables like ``PLAT`` or ``ARCH``.\n\n* **Dependencies**\n\n Look out for variables that sound like they could be used to locate dependencies, such as ``JAVA_HOME``, ``JPEG_ROOT``, or ``ZLIBDIR``.\n Also watch out for variables that control linking, such as ``LIBS``, ``LDFLAGS``, and ``INCLUDES``.\n These variables need to be set to the installation prefix of a dependency, or to the correct linker flags to link to that dependency.\n\n* **Installation prefix**\n\n If your Makefile has an ``install`` target, it needs some way of knowing where to install.\n By default, many packages install to ``/usr`` or ``/usr/local``.\n Since many Spack users won't have sudo privileges, it is imperative that each package is installed to the proper prefix.\n Look for variables like ``PREFIX`` or ``INSTALL``.\n\nMakefiles in a sub-directory\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nNot every package places their Makefile in the root of the package tarball.\nIf the Makefile is in a sub-directory like ``src``, you can tell Spack where to locate it like so:\n\n.. code-block:: python\n\n build_directory = \"src\"\n\n\nManual installation\n^^^^^^^^^^^^^^^^^^^\n\nNot every Makefile includes an ``install`` target.\nIf this is the case, you can override the default ``install`` method to manually install the package:\n\n.. code-block:: python\n\n def install(self, spec, prefix):\n mkdir(prefix.bin)\n install(\"foo\", prefix.bin)\n install_tree(\"lib\", prefix.lib)\n\n\nExternal documentation\n^^^^^^^^^^^^^^^^^^^^^^\n\nFor more information on reading and writing Makefiles, see: https://www.gnu.org/software/make/manual/make.html\n" }, { "path": "lib/spack/docs/build_systems/mavenpackage.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Learn about the Maven build system in Spack for building and managing Java-based projects.\n\n.. _mavenpackage:\n\nMaven\n------\n\nApache Maven is a general-purpose build system that does not rely on Makefiles to build software.\nIt is designed for building and managing Java-based projects.\n\nPhases\n^^^^^^\n\nThe ``MavenBuilder`` and ``MavenPackage`` base classes come with the following phases:\n\n#. ``build`` - compile code and package into a JAR file\n#. ``install`` - copy to installation prefix\n\nBy default, these phases run:\n\n.. code-block:: console\n\n $ mvn package\n $ install . \n\n\nImportant files\n^^^^^^^^^^^^^^^\n\nMaven packages can be identified by the presence of a ``pom.xml`` file.\nThis file lists dependencies and other metadata about the project.\nThere may also be configuration files in the ``.mvn`` directory.\n\nBuild system dependencies\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nMaven requires the ``mvn`` executable to build the project.\nIt also requires Java at both build- and run-time.\nBecause of this, the base class automatically adds the following dependencies:\n\n.. code-block:: python\n\n depends_on(\"java\", type=(\"build\", \"run\"))\n depends_on(\"maven\", type=\"build\")\n\n\nIn the ``pom.xml`` file, you may see sections like:\n\n.. code-block:: xml\n\n \n [1.7,)\n \n \n [3.5.4,)\n \n\n\nThis specifies the versions of Java and Maven that are required to build the package.\nSee https://docs.oracle.com/middleware/1212/core/MAVEN/maven_version.htm#MAVEN402 for a description of this version range syntax.\nIn this case, you should add:\n\n.. code-block:: python\n\n depends_on(\"java@7:\", type=\"build\")\n depends_on(\"maven@3.5.4:\", type=\"build\")\n\n\nPassing arguments to the build phase\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nThe default build and install phases should be sufficient to install most packages.\nHowever, you may want to pass additional flags to the build phase.\nFor example:\n\n.. code-block:: python\n\n def build_args(self):\n return [\"-Pdist,native\", \"-Dtar\", \"-Dmaven.javadoc.skip=true\"]\n\n\nExternal documentation\n^^^^^^^^^^^^^^^^^^^^^^\n\nFor more information on the Maven build system, see: https://maven.apache.org/index.html\n" }, { "path": "lib/spack/docs/build_systems/mesonpackage.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Learn how to use the Meson build system in Spack for projects that use Meson for their build process.\n\n.. _mesonpackage:\n\nMeson\n------\n\nMuch like Autotools and CMake, Meson is a build system.\nBut it is meant to be both fast and as user friendly as possible.\nGNOME's goal is to port modules to use the Meson build system.\n\nPhases\n^^^^^^\n\nThe ``MesonBuilder`` and ``MesonPackage`` base classes come with the following phases:\n\n#. ``meson`` - generate ninja files\n#. ``build`` - build the project\n#. ``install`` - install the project\n\nBy default, these phases run:\n\n.. code-block:: console\n\n $ mkdir spack-build\n $ cd spack-build\n $ meson .. --prefix=/path/to/installation/prefix\n $ ninja\n $ ninja test # optional\n $ ninja install\n\n\nAny of these phases can be overridden in your package as necessary.\nThere is also a ``check`` method that looks for a ``test`` target in the build file.\nIf a ``test`` target exists and the user runs:\n\n.. code-block:: console\n\n $ spack install --test=root \n\n\nSpack will run ``ninja test`` after the build phase.\n\nImportant files\n^^^^^^^^^^^^^^^\n\nPackages that use the Meson build system can be identified by the presence of a ``meson.build`` file.\nThis file declares things like build instructions and dependencies.\n\nOne thing to look for is the ``meson_version`` key that gets passed to the ``project`` function:\n\n.. code-block:: none\n :emphasize-lines: 10\n\n project('gtk+', 'c',\n version: '3.94.0',\n default_options: [\n 'buildtype=debugoptimized',\n 'warning_level=1',\n # We only need c99, but glib needs GNU-specific features\n # https://github.com/mesonbuild/meson/issues/2289\n 'c_std=gnu99',\n ],\n meson_version: '>= 0.43.0',\n license: 'LGPLv2.1+')\n\n\nThis means that Meson 0.43.0 is the earliest release that will work.\nYou should specify this in a ``depends_on`` statement.\n\nBuild system dependencies\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nAt the bare minimum, packages that use the Meson build system need ``meson`` and ``ninja`` dependencies.\nSince this is always the case, the ``MesonPackage`` base class already contains:\n\n.. code-block:: python\n\n depends_on(\"meson\", type=\"build\")\n depends_on(\"ninja\", type=\"build\")\n\n\nIf you need to specify a particular version requirement, you can override this in your package:\n\n.. code-block:: python\n\n depends_on(\"meson@0.43.0:\", type=\"build\")\n depends_on(\"ninja\", type=\"build\")\n\n\nFinding meson flags\n^^^^^^^^^^^^^^^^^^^\n\nTo get a list of valid flags that can be passed to ``meson``, run the following command in the directory that contains ``meson.build``:\n\n.. code-block:: console\n\n $ meson setup --help\n\n\nPassing arguments to meson\n^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nIf you need to pass any arguments to the ``meson`` call, you can override the ``meson_args`` method like so:\n\n.. code-block:: python\n\n def meson_args(self):\n return [\"--warnlevel=3\"]\n\n\nThis method can be used to pass flags as well as variables.\n\nNote that the ``MesonPackage`` base class already defines variants for ``buildtype``, ``default_library`` and ``strip``, which are mapped to default Meson arguments, meaning that you don't have to specify these.\n\nExternal documentation\n^^^^^^^^^^^^^^^^^^^^^^\n\nFor more information on the Meson build system, see: https://mesonbuild.com/index.html\n" }, { "path": "lib/spack/docs/build_systems/octavepackage.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Learn about the Octave build system in Spack for installing Octave packages.\n\n.. _octavepackage:\n\nOctave\n------\n\nOctave has its own build system for installing packages.\n\nPhases\n^^^^^^\n\nThe ``OctaveBuilder`` and ``OctavePackage`` base classes have a single phase:\n\n#. ``install`` - install the package\n\nBy default, this phase runs the following command:\n\n.. code-block:: console\n\n $ octave '--eval' 'pkg prefix ; pkg install '\n\n\nBeware that uninstallation is not implemented at the moment.\nAfter uninstalling a package via Spack, you also need to manually uninstall it from Octave via ``pkg uninstall ``.\n\nFinding Octave packages\n^^^^^^^^^^^^^^^^^^^^^^^\n\nMost Octave packages are listed at https://octave.sourceforge.io/packages.php.\n\nDependencies\n^^^^^^^^^^^^\n\nUsually, the homepage of a package will list dependencies, i.e., ``Dependencies: Octave >= 3.6.0 struct >= 1.0.12``.\nThe same information should be available in the ``DESCRIPTION`` file in the root of each archive.\n\nExternal Documentation\n^^^^^^^^^^^^^^^^^^^^^^\n\nFor more information on the Octave build system, see: https://octave.org/doc/v4.4.0/Installing-and-Removing-Packages.html\n" }, { "path": "lib/spack/docs/build_systems/perlpackage.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n A guide to packaging Perl modules with Spack, covering when to add a package and build system integration.\n\n.. _perlpackage:\n\nPerl\n------\n\nMuch like Octave, Perl has its own language-specific build system.\nThis documentation includes information on when **not** to add a Spack package for a Perl module.\n\n\n.. _suitable_perl_modules:\n\nSuitable Modules\n^^^^^^^^^^^^^^^^\n\nIn general, modules that are part of the standard Perl installation should not be added to Spack.\nA possible exception is if the module was not part of the standard installation for earlier versions of ``perl`` that are still listed in the package, which you can check by running ``spack info perl``.\n\nHow do you know if the module is in the standard Perl installation?\nYou check if it is included in the ``CORE`` by entering the following on the command line:\n\n.. code-block:: console\n\n $ corelist \n\nwhere is case sensitive.\n\nExamples of outputs for modules that are and are not in the ``CORE`` using perl v5.42.0 are:\n\n.. code-block:: console\n\n $ corelist Carp\n\n Data for 2025-07-02\n Carp was first released with perl 5\n\n $ corelist XML::Writer\n\n Data for 2025-07-02\n XML::Writer was not in CORE (or so I think)\n\n\nPhases\n^^^^^^\n\nThe ``PerlBuilder`` and ``PerlPackage`` base classes come with three phases that can be overridden:\n\n#. ``configure`` - configure the package\n#. ``build`` - build the package\n#. ``install`` - install the package\n\nPerl packages have two common modules used for module installation:\n\n``ExtUtils::MakeMaker``\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nThe ``ExtUtils::MakeMaker`` module is just what it sounds like, a module designed to generate Makefiles.\nIt can be identified by the presence of a ``Makefile.PL`` file, and has the following installation steps:\n\n.. code-block:: console\n\n $ perl Makefile.PL INSTALL_BASE=/path/to/installation/prefix\n $ make\n $ make test # optional\n $ make install\n\n\n``Module::Build``\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nThe ``Module::Build`` module is a pure-Perl build system, and can be identified by the presence of a ``Build.PL`` file.\nIt has the following installation steps:\n\n.. code-block:: console\n\n $ perl Build.PL --install_base /path/to/installation/prefix\n $ ./Build\n $ ./Build test # optional\n $ ./Build install\n\n\nIf both ``Makefile.PL`` and ``Build.PL`` files exist in the package, Spack will use ``Makefile.PL`` by default.\nIf your package uses a different module, ``PerlPackage`` will need to be extended to support it.\n\n``PerlPackage`` automatically detects which build steps to use, so there shouldn't be much work on the package developer's side to get things working.\n\nFinding Perl packages\n^^^^^^^^^^^^^^^^^^^^^\n\nMost Perl modules are hosted on CPAN, the Comprehensive Perl Archive Network.\nIf you need to find a package for ``XML::Parser``, for example, you should search for \"CPAN XML::Parser\".\nJust make sure that the module is not included in the ``CORE`` (see :ref:`suitable_perl_modules`).\n\nSome CPAN pages are versioned.\nCheck for a link to the \"Latest Release\" to make sure you have the latest version.\n\n\nPackage name\n^^^^^^^^^^^^\n\nWhen you use ``spack create`` to create a new Perl package, Spack will automatically prepend ``perl-`` to the front of the package name.\nThis helps to keep Perl modules separate from other packages.\nThe same naming scheme is used for other language extensions, like Python and R.\nSee :ref:`creating-and-editing-packages` for more information on the command.\n\nDescription\n^^^^^^^^^^^\n\nMost CPAN pages have a short description under \"NAME\" and a longer description under \"DESCRIPTION\".\nUse whichever you think is more useful while still being succinct.\n\nHomepage\n^^^^^^^^\n\nIn the top-right corner of the CPAN page, you'll find a \"permalink\" for the package.\nThis should be used instead of the current URL, as it doesn't contain the version number and will always link to the latest release.\n\nURL\n^^^^^^\n\nIf you haven't found it already, the download URL is on the right side of the page below the permalink.\nSearch for \"Download\".\n\nBuild system dependencies\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nEvery ``PerlPackage`` obviously depends on Perl at build and run-time, so ``PerlPackage`` contains:\n\n.. code-block:: python\n\n extends(\"perl\")\n\n\nIf your package requires a specific version of Perl, you should specify this.\n\nAlthough newer versions of Perl include ``ExtUtils::MakeMaker`` and ``Module::Build`` as \"core\" modules, you may want to add dependencies on ``perl-extutils-makemaker`` and ``perl-module-build`` anyway.\nMany people add Perl as an external package, and we want the build to work properly.\nIf your package uses ``Makefile.PL`` to build, add:\n\n.. code-block:: python\n\n depends_on(\"perl-extutils-makemaker\", type=\"build\")\n\n\nIf your package uses ``Build.PL`` to build, add:\n\n.. code-block:: python\n\n depends_on(\"perl-module-build\", type=\"build\")\n\n\nPerl dependencies\n^^^^^^^^^^^^^^^^^\n\nBelow the download URL, you will find a \"Dependencies\" link, which takes you to a page listing all of the dependencies of the package.\nPackages listed as \"Core module\" don't need to be added as dependencies, but all direct dependencies should be added.\nDon't add dependencies of dependencies.\nThese should be added as dependencies to the dependency, not to your package.\n\nPassing arguments to configure\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nPackages that have non-Perl dependencies often use command-line variables to specify their installation directory.\nYou can pass arguments to ``Makefile.PL`` or ``Build.PL`` by overriding ``configure_args`` like so:\n\n.. code-block:: python\n\n def configure_args(self):\n expat = self.spec[\"expat\"].prefix\n\n return [\n \"EXPATLIBPATH={0}\".format(expat.lib),\n \"EXPATINCPATH={0}\".format(expat.include),\n ]\n\n\nTesting\n^^^^^^^\n\n``PerlPackage`` provides a simple stand-alone test of the successfully installed package to confirm that installed Perl module(s) can be used.\nThese tests can be performed any time after the installation using ``spack -v test run``.\n(For more information on the command, see :ref:`cmd-spack-test-run`.)\n\nThe base class automatically detects Perl modules based on the presence of ``*.pm`` files under the package's library directory.\nFor example, the files under ``perl-bignum``'s Perl library are:\n\n.. code-block:: console\n\n $ find . -name \"*.pm\"\n ./bigfloat.pm\n ./bigrat.pm\n ./Math/BigFloat/Trace.pm\n ./Math/BigInt/Trace.pm\n ./Math/BigRat/Trace.pm\n ./bigint.pm\n ./bignum.pm\n\n\nwhich results in the package having the ``use_modules`` property containing:\n\n.. code-block:: python\n\n use_modules = [\n \"bigfloat\",\n \"bigrat\",\n \"Math::BigFloat::Trace\",\n \"Math::BigInt::Trace\",\n \"Math::BigRat::Trace\",\n \"bigint\",\n \"bignum\",\n ]\n\n.. note::\n\n This list can often be used to catch missing dependencies.\n\nIf the list is somehow wrong, you can provide the names of the modules yourself by overriding ``use_modules`` like so:\n\n.. code-block:: python\n\n use_modules = [\"bigfloat\", \"bigrat\", \"bigint\", \"bignum\"]\n\nIf you only want a subset of the automatically detected modules to be tested, you could instead define the ``skip_modules`` property on the package.\nSo, instead of overriding ``use_modules`` as shown above, you could define the following:\n\n.. code-block:: python\n\n skip_modules = [\n \"Math::BigFloat::Trace\",\n \"Math::BigInt::Trace\",\n \"Math::BigRat::Trace\",\n ]\n\nfor the same use tests.\n\nAlternatives to Spack\n^^^^^^^^^^^^^^^^^^^^^\n\nIf you need to maintain a stack of Perl modules for a user and don't want to add all of them to Spack, a good alternative is ``cpanm``.\nIf Perl is already installed on your system, it should come with a ``cpan`` executable.\nTo install ``cpanm``, run the following command:\n\n.. code-block:: console\n\n $ cpan App::cpanminus\n\n\nNow, you can install any Perl module you want by running:\n\n.. code-block:: console\n\n $ cpanm Module::Name\n\n\nObviously, these commands can only be run if you have root privileges.\nFurthermore, ``cpanm`` is not capable of installing non-Perl dependencies.\nIf you need to install to your home directory or need to install a module with non-Perl dependencies, Spack is a better option.\n\nExternal documentation\n^^^^^^^^^^^^^^^^^^^^^^\n\nYou can find more information on installing Perl modules from source at: http://www.perlmonks.org/?node_id=128077\n\nMore generic Perl module installation instructions can be found at: http://www.cpan.org/modules/INSTALL.html\n" }, { "path": "lib/spack/docs/build_systems/pythonpackage.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n A guide to packaging Python libraries with Spack, covering PyPI downloads, dependency management, and build system integration.\n\n.. _pythonpackage:\n\nPython\n------\n\nPython packages and modules have their own special build system.\nThis documentation covers everything you'll need to know in order to write a Spack build recipe for a Python library.\n\nTerminology\n^^^^^^^^^^^\n\nIn the Python ecosystem, there are a number of terms that are important to understand.\n\n**PyPI**\n The `Python Package Index `_, where most Python libraries are hosted.\n\n**sdist**\n Source distributions, distributed as tarballs (.tar.gz) and zip files (.zip).\n Contain the source code of the package.\n\n**bdist**\n Built distributions, distributed as wheels (.whl).\n Contain the pre-built library.\n\n**wheel**\n A binary distribution format common in the Python ecosystem.\n This file is actually just a zip file containing specific metadata and code.\n See the `documentation `_ for more details.\n\n**build frontend**\n Command-line tools used to build and install wheels.\n Examples include `pip `_, `build `_, and `installer `_.\n\n**build backend**\n Libraries used to define how to build a wheel.\n Examples include `setuptools `__, `flit `_, `poetry `_, `hatchling `_, `meson `_, and `pdm `_.\n\nDownloading\n^^^^^^^^^^^\n\nThe first step in packaging a Python library is to figure out where to download it from.\nThe vast majority of Python packages are hosted on `PyPI `_, which is :ref:`preferred over GitHub ` for downloading packages.\nSearch for the package name on PyPI to find the project page.\nThe project page is usually located at:\n\n.. code-block:: text\n\n https://pypi.org/project/\n\nOn the project page, there is a \"Download files\" tab containing download URLs.\nWhenever possible, we prefer to build Spack packages from source.\nIf PyPI only has wheels, check to see if the project is hosted on GitHub and see if GitHub has source distributions.\nThe project page usually has a \"Homepage\" and/or \"Source code\" link for this.\nIf the project is closed-source, it may only have wheels available.\nFor example, ``py-azureml-sdk`` is closed-source and can be downloaded from:\n\n.. code-block:: text\n\n https://pypi.io/packages/py3/a/azureml_sdk/azureml_sdk-1.11.0-py3-none-any.whl\n\nOnce you've found a URL to download the package from, run:\n\n.. code-block:: console\n\n $ spack create \n\n\nto create a new package template.\n\n.. _pypi-vs-github:\n\nPyPI vs. GitHub\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nMany packages are hosted on PyPI, but are developed on GitHub or another version control system hosting service.\nThe source code can be downloaded from either location, but PyPI is preferred for the following reasons:\n\n#. PyPI contains the bare minimum number of files needed to install the package.\n\n You may notice that the tarball you download from PyPI does not have the same checksum as the tarball you download from GitHub.\n When a developer uploads a new release to PyPI, it doesn't contain every file in the repository, only the files necessary to install the package.\n PyPI tarballs are therefore smaller.\n\n#. PyPI is the official source for package managers like ``pip``.\n\n Let's be honest, ``pip`` is much more popular than Spack.\n If the GitHub tarball contains a file not present in the PyPI tarball that causes a bug, the developers may not realize this for quite some time.\n If the bug was in a file contained in the PyPI tarball, users would notice the bug much more quickly.\n\n#. GitHub release may be a beta version.\n\n When a developer releases a new version of a package on GitHub, it may not be intended for most users.\n Until that release also makes its way to PyPI, it should be assumed that the release is not yet ready for general use.\n\n#. The checksum for a GitHub release may change.\n\n Unfortunately, some developers have a habit of patching releases without incrementing the version number.\n This results in a change in tarball checksum.\n Package managers like Spack that use checksums to verify the integrity of a download tarball grind to a halt when the checksum for a known version changes.\n Most of the time, the change is intentional, and contains a needed bug fix.\n However, sometimes the change indicates a download source that has been compromised, and a tarball that contains a virus.\n If this happens, you must contact the developers to determine which is the case.\n PyPI is nice because it makes it physically impossible to re-release the same version of a package with a different checksum.\n\nThe only reason to use GitHub instead of PyPI is if PyPI only has wheels or if the PyPI sdist is missing a file needed to build the package.\nIf this is the case, please add a comment above the ``url`` explaining this.\n\nPyPI\n^^^^^^\n\nSince PyPI is so commonly used to host Python libraries, the ``PythonPackage`` base class has a ``pypi`` attribute that can be set.\nOnce set, ``pypi`` will be used to define the ``homepage``, ``url``, and ``list_url``.\nFor example, the following:\n\n.. code-block:: python\n\n homepage = \"https://pypi.org/project/setuptools/\"\n url = \"https://pypi.org/packages/source/s/setuptools/setuptools-49.2.0.zip\"\n list_url = \"https://pypi.org/simple/setuptools/\"\n\n\nis equivalent to:\n\n.. code-block:: python\n\n pypi = \"setuptools/setuptools-49.2.0.zip\"\n\n\nIf a package has a different homepage listed on PyPI, you can override it by setting your own ``homepage``.\n\nDescription\n^^^^^^^^^^^\n\nThe top of the PyPI project page contains a short description of the package.\nThe \"Project description\" tab may also contain a longer description of the package.\nEither of these can be used to populate the package docstring.\n\nDependencies\n^^^^^^^^^^^^\n\nOnce you've determined the basic metadata for a package, the next step is to determine the build backend.\n``PythonPackage`` uses `pip `_ to install the package, but pip requires a backend to actually build the package.\n\nTo determine the build backend, look for a ``pyproject.toml`` file.\nIf there is no ``pyproject.toml`` file and only a ``setup.py`` or ``setup.cfg`` file, you can assume that the project uses :ref:`setuptools`.\nIf there is a ``pyproject.toml`` file, see if it contains a ``[build-system]`` section.\nFor example:\n\n.. code-block:: toml\n\n [build-system]\n requires = [\n \"setuptools>=42\",\n \"wheel\",\n ]\n build-backend = \"setuptools.build_meta\"\n\n\nThis section does two things: the ``requires`` key lists build dependencies of the project, and the ``build-backend`` key defines the build backend.\nAll of these build dependencies should be added as dependencies to your package:\n\n.. code-block:: python\n\n depends_on(\"py-setuptools@42:\", type=\"build\")\n\n\nNote that ``py-wheel`` is already listed as a build dependency in the ``PythonPackage`` base class, so you don't need to add it unless you need to specify a specific version requirement or change the dependency type.\n\nSee `PEP 517 `__ and `PEP 518 `_ for more information on the design of ``pyproject.toml``.\n\nDepending on which build backend a project uses, there are various places that run-time dependencies can be listed.\nMost modern build backends support listing dependencies directly in ``pyproject.toml``.\nLook for dependencies under the following keys:\n\n* ``requires-python`` under ``[project]``\n\n This specifies the version of Python that is required.\n\n* ``dependencies`` under ``[project]``\n\n These packages are required for building and installation.\n You can add them with ``type=(\"build\", \"run\")``.\n\n* ``[project.optional-dependencies]``\n\n This section includes keys with lists of optional dependencies needed to enable those features.\n You should add a variant that optionally adds these dependencies.\n This variant should be ``False`` by default.\n\nSome build backends may have additional locations where dependencies can be found.\n\ndistutils\n\"\"\"\"\"\"\"\"\"\n\nBefore the introduction of setuptools and other build backends, Python packages had to rely on the built-in distutils library.\nDistutils is missing many of the features that setuptools and other build backends offer, and users are encouraged to use setuptools instead.\nIn fact, distutils was deprecated in Python 3.10 and will be removed in Python 3.12.\nBecause of this, pip actually replaces all imports of distutils with setuptools.\nIf a package uses distutils, you should instead add a build dependency on setuptools.\nCheck for a ``requirements.txt`` file that may list dependencies of the project.\n\n.. _setuptools:\n\nsetuptools\n\"\"\"\"\"\"\"\"\"\"\n\nIf the ``pyproject.toml`` lists ``setuptools.build_meta`` as a ``build-backend``, or if the package has a ``setup.py`` that imports ``setuptools``, or if the package has a ``setup.cfg`` file, then it uses setuptools to build.\nSetuptools is a replacement for the distutils library, and has almost the exact same API.\nIn addition to ``pyproject.toml``, dependencies can be listed in the ``setup.py`` or ``setup.cfg`` file.\nLook for the following arguments:\n\n* ``python_requires``\n\n This specifies the version of Python that is required.\n\n* ``setup_requires``\n\n These packages are usually only needed at build-time, so you can add them with ``type=\"build\"``.\n\n* ``install_requires``\n\n These packages are required for building and installation.\n You can add them with ``type=(\"build\", \"run\")``.\n\n* ``extras_require``\n\n These packages are optional dependencies that enable additional functionality.\n You should add a variant that optionally adds these dependencies.\n This variant should be ``False`` by default.\n\n* ``tests_require``\n\n These are packages that are required to run the unit tests for the package.\n These dependencies can be specified using the ``type=\"test\"`` dependency type.\n However, the PyPI tarballs rarely contain unit tests, so there is usually no reason to add these.\n\nSee https://setuptools.pypa.io/en/latest/userguide/dependency_management.html for more information on how setuptools handles dependency management.\nSee `PEP 440 `_ for documentation on version specifiers in setuptools.\n\nflit\n\"\"\"\"\"\"\n\nThere are actually two possible build backends for flit, ``flit`` and ``flit_core``.\nIf you see these in the ``pyproject.toml``, add a build dependency to your package.\nWith flit, all dependencies are listed directly in the ``pyproject.toml`` file.\nOlder versions of flit used to store this info in a ``flit.ini`` file, so check for this too.\n\nIn addition to the default ``pyproject.toml`` keys listed above, older versions of flit may use the following keys:\n\n* ``requires`` under ``[tool.flit.metadata]``\n\n These packages are required for building and installation.\n You can add them with ``type=(\"build\", \"run\")``.\n\n* ``[tool.flit.metadata.requires-extra]``\n\n This section includes keys with lists of optional dependencies needed to enable those features.\n You should add a variant that optionally adds these dependencies.\n This variant should be ``False`` by default.\n\nSee https://flit.pypa.io/en/latest/pyproject_toml.html for more information.\n\npoetry\n\"\"\"\"\"\"\n\nLike flit, poetry also has two possible build backends, ``poetry`` and ``poetry_core``.\nIf you see these in the ``pyproject.toml``, add a build dependency to your package.\nWith poetry, all dependencies are listed directly in the ``pyproject.toml`` file.\nDependencies are listed in a ``[tool.poetry.dependencies]`` section, and use a `custom syntax `_ for specifying the version requirements.\nNote that ``~=`` works differently in poetry than in setuptools and flit for versions that start with a zero.\n\nhatchling\n\"\"\"\"\"\"\"\"\"\n\nIf the ``pyproject.toml`` lists ``hatchling.build`` as the ``build-backend``, it uses the hatchling build system.\nHatchling uses the default ``pyproject.toml`` keys to list dependencies.\n\nSee https://hatch.pypa.io/latest/config/dependency/ for more information.\n\nmeson\n\"\"\"\"\"\"\n\nIf the ``pyproject.toml`` lists ``mesonpy`` as the ``build-backend``, it uses the meson build system.\nMeson uses the default ``pyproject.toml`` keys to list dependencies.\n\nSee https://meson-python.readthedocs.io/en/latest/tutorials/introduction.html for more information.\n\npdm\n\"\"\"\"\"\"\n\nIf the ``pyproject.toml`` lists ``pdm.pep517.api`` as the ``build-backend``, it uses the PDM build system.\nPDM uses the default ``pyproject.toml`` keys to list dependencies.\n\nSee https://pdm.fming.dev/latest/ for more information.\n\nwheels\n\"\"\"\"\"\"\n\nSome Python packages are closed-source and are distributed as Python wheels.\nFor example, ``py-azureml-sdk`` downloads a ``.whl`` file.\nThis file is simply a zip file, and can be extracted using:\n\n.. code-block:: console\n\n $ unzip *.whl\n\n\nThe zip file will not contain a ``setup.py``, but it will contain a ``METADATA`` file which contains all the information you need to write a ``package.py`` build recipe.\nCheck for lines like:\n\n.. code-block:: text\n\n Requires-Python: >=3.5,<4\n Requires-Dist: azureml-core (~=1.11.0)\n Requires-Dist: azureml-dataset-runtime[fuse] (~=1.11.0)\n Requires-Dist: azureml-train (~=1.11.0)\n Requires-Dist: azureml-train-automl-client (~=1.11.0)\n Requires-Dist: azureml-pipeline (~=1.11.0)\n Provides-Extra: accel-models\n Requires-Dist: azureml-accel-models (~=1.11.0); extra == 'accel-models'\n Provides-Extra: automl\n Requires-Dist: azureml-train-automl (~=1.11.0); extra == 'automl'\n\n\n``Requires-Python`` is equivalent to ``python_requires`` and ``Requires-Dist`` is equivalent to ``install_requires``.\n``Provides-Extra`` is used to name optional features (variants) and a ``Requires-Dist`` with ``extra == 'foo'`` will list any dependencies needed for that feature.\n\nPassing arguments to setup.py\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nThe default install phase should be sufficient to install most packages.\nHowever, the installation instructions for a package may suggest passing certain flags to the ``setup.py`` call.\nThe ``PythonPackage`` class has two techniques for doing this.\n\nConfig settings\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nThese settings are passed to `PEP 517 `__ build backends.\nFor example, ``py-scipy`` package allows you to specify the name of the BLAS/LAPACK library you want pkg-config to search for:\n\n.. code-block:: python\n\n depends_on(\"py-pip@22.1:\", type=\"build\")\n\n\n def config_settings(self, spec, prefix):\n return {\n \"blas\": spec[\"blas\"].libs.names[0],\n \"lapack\": spec[\"lapack\"].libs.names[0],\n }\n\n\n.. note::\n\n This flag only works for packages that define a ``build-backend`` in ``pyproject.toml``.\n Also, it is only supported by pip 22.1+, which requires Python 3.7+.\n For packages that still support Python 3.6 and older, ``install_options`` should be used instead.\n\n\nGlobal options\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nThese flags are added directly after ``setup.py`` when pip runs ``python setup.py install``.\nFor example, the ``py-pyyaml`` package has an optional dependency on ``libyaml`` that can be enabled like so:\n\n.. code-block:: python\n\n def global_options(self, spec, prefix):\n options = []\n if spec.satisfies(\"+libyaml\"):\n options.append(\"--with-libyaml\")\n else:\n options.append(\"--without-libyaml\")\n return options\n\n\n.. note::\n\n Direct invocation of ``setup.py`` is `deprecated `_.\n This flag forces pip to use a deprecated installation procedure.\n It should only be used in packages that don't define a ``build-backend`` in ``pyproject.toml`` or packages that still support Python 3.6 and older.\n\n\nInstall options\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nThese flags are added directly after ``install`` when pip runs ``python setup.py install``.\nFor example, the ``py-pyyaml`` package allows you to specify the directories to search for ``libyaml``:\n\n.. code-block:: python\n\n def install_options(self, spec, prefix):\n options = []\n if spec.satisfies(\"+libyaml\"):\n options.extend(\n [\n spec[\"libyaml\"].libs.search_flags,\n spec[\"libyaml\"].headers.include_flags,\n ]\n )\n return options\n\n\n.. note::\n\n Direct invocation of ``setup.py`` is `deprecated `_.\n This flag forces pip to use a deprecated installation procedure.\n It should only be used in packages that don't define a ``build-backend`` in ``pyproject.toml`` or packages that still support Python 3.6 and older.\n\n\nTesting\n^^^^^^^\n\n``PythonPackage`` provides a couple of options for testing packages both during and after the installation process.\n\nImport tests\n\"\"\"\"\"\"\"\"\"\"\"\"\n\nJust because a package successfully built does not mean that it built correctly.\nThe most reliable test of whether or not the package was correctly installed is to attempt to import all of the modules that get installed.\nTo get a list of modules, run the following command in the source directory:\n\n.. code-block:: pycon\n\n >>> import setuptools\n >>> setuptools.find_packages()\n ['numpy', 'numpy._build_utils', 'numpy.compat', 'numpy.core', 'numpy.distutils', 'numpy.doc', 'numpy.f2py', 'numpy.fft', 'numpy.lib', 'numpy.linalg', 'numpy.ma', 'numpy.matrixlib', 'numpy.polynomial', 'numpy.random', 'numpy.testing', 'numpy.core.code_generators', 'numpy.distutils.command', 'numpy.distutils.fcompiler']\n\n\nLarge, complex packages like ``numpy`` will return a long list of packages, while other packages like ``six`` will return an empty list.\n``py-six`` installs a single ``six.py`` file.\nIn Python packaging lingo, a \"package\" is a directory containing files like:\n\n.. code-block:: none\n\n foo/__init__.py\n foo/bar.py\n foo/baz.py\n\n\nwhereas a \"module\" is a single Python file.\n\nThe ``PythonPackage`` base class automatically detects these package and module names for you.\nIf, for whatever reason, the module names detected are wrong, you can provide the names yourself by overriding ``import_modules`` like so:\n\n.. code-block:: python\n\n import_modules = [\"six\"]\n\n\nSometimes the list of module names to import depends on how the package was built.\nFor example, the ``py-pyyaml`` package has a ``+libyaml`` variant that enables the build of a faster optimized version of the library.\nIf the user chooses ``~libyaml``, only the ``yaml`` library will be importable.\nIf the user chooses ``+libyaml``, both the ``yaml`` and ``yaml.cyaml`` libraries will be available.\nThis can be expressed like so:\n\n.. code-block:: python\n\n @property\n def import_modules(self):\n modules = [\"yaml\"]\n if self.spec.satisfies(\"+libyaml\"):\n modules.append(\"yaml.cyaml\")\n return modules\n\n\nThese tests often catch missing dependencies and non-RPATHed libraries.\nMake sure not to add modules/packages containing the word \"test\", as these likely won't end up in the installation directory, or may require test dependencies like pytest to be installed.\n\nInstead of defining the ``import_modules`` explicitly, only the subset of module names to be skipped can be defined by using ``skip_modules``.\nIf a defined module has submodules, they are skipped as well, e.g., in case the ``plotting`` modules should be excluded from the automatically detected ``import_modules`` ``[\"nilearn\", \"nilearn.surface\", \"nilearn.plotting\", \"nilearn.plotting.data\"]`` set:\n\n.. code-block:: python\n\n skip_modules = [\"nilearn.plotting\"]\n\nThis will set ``import_modules`` to ``[\"nilearn\", \"nilearn.surface\"]``.\n\nImport tests can be run during the installation using ``spack install --test=root`` or at any time after the installation using ``spack test run``.\n\nUnit tests\n\"\"\"\"\"\"\"\"\"\"\n\nThe package may have its own unit or regression tests.\nSpack can run these tests during the installation by adding test methods after installation.\n\nFor example, ``py-numpy`` adds the following as a check to run after the ``install`` phase:\n\n.. code-block:: python\n\n @run_after(\"install\")\n @on_package_attributes(run_tests=True)\n def install_test(self):\n with working_dir(\"spack-test\", create=True):\n python(\"-c\", \"import numpy; numpy.test('full', verbose=2)\")\n\n\nwhen testing is enabled during the installation (i.e., ``spack install --test=root``).\n\n.. note::\n\n Additional information is available on :ref:`install phase tests `.\n\nSetup file in a sub-directory\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nMany C/C++ libraries provide optional Python bindings in a subdirectory.\nTo tell pip which directory to build from, you can override the ``build_directory`` attribute.\nFor example, if a package provides Python bindings in a ``python`` directory, you can use:\n\n.. code-block:: python\n\n build_directory = \"python\"\n\n\nPythonPackage vs. packages that use Python\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nThere are many packages that make use of Python, but packages that depend on Python are not necessarily ``PythonPackage``'s.\n\nChoosing a build system\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nFirst of all, you need to select a build system.\n``spack create`` usually does this for you, but if for whatever reason you need to do this manually, choose ``PythonPackage`` if and only if the package contains one of the following files:\n\n* ``pyproject.toml``\n* ``setup.py``\n* ``setup.cfg``\n\nChoosing a package name\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nSelecting the appropriate package name is a little more complicated than choosing the build system.\nBy default, ``spack create`` will prepend ``py-`` to the beginning of the package name if it detects that the package uses the ``PythonPackage`` build system.\nHowever, there are occasionally packages that use ``PythonPackage`` that shouldn't start with ``py-``.\nFor example:\n\n* awscli\n* aws-parallelcluster\n* busco\n* easybuild\n* httpie\n* mercurial\n* scons\n* snakemake\n\nThe thing these packages have in common is that they are command-line tools that just so happen to be written in Python.\nSomeone who wants to install ``mercurial`` with Spack isn't going to realize that it is written in Python, and they certainly aren't going to assume the package is called ``py-mercurial``.\nFor this reason, we manually renamed the package to ``mercurial``.\n\nLikewise, there are occasionally packages that don't use the ``PythonPackage`` build system but should still be prepended with ``py-``.\nFor example:\n\n* py-genders\n* py-py2cairo\n* py-pygobject\n* py-pygtk\n* py-pyqt\n* py-pyserial\n* py-sip\n* py-xpyb\n\nThese packages are primarily used as Python libraries, not as command-line tools.\nYou may see C/C++ packages that have optional Python language bindings, such as:\n\n* antlr\n* cantera\n* conduit\n* pagmo\n* vtk\n\nDon't prepend these kinds of packages with ``py-``.\nWhen in doubt, think about how this package will be used.\nIs it primarily a Python library that will be imported in other Python scripts?\nOr is it a command-line tool, or C/C++/Fortran program with optional Python modules?\nThe former should be prepended with ``py-``, while the latter should not.\n\n``extends`` vs. ``depends_on``\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nAs mentioned in the :ref:`Packaging Guide `, ``extends`` and ``depends_on`` are very similar, but ``extends`` ensures that the extension and extendee share the same prefix in views.\nThis allows the user to import a Python module without having to add that module to ``PYTHONPATH``.\n\nAdditionally, ``extends(\"python\")`` adds a dependency on the package ``python-venv``.\nThis improves isolation from the system, whether it's during the build or at runtime: user and system site packages cannot accidentally be used by any package that ``extends(\"python\")``.\n\nAs a rule of thumb: if a package does not install any Python modules of its own, and merely puts a Python script in the ``bin`` directory, then there is no need for ``extends``.\nIf the package installs modules in the ``site-packages`` directory, it requires ``extends``.\n\nExecuting ``python`` during the build\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nWhenever you need to execute a Python command or pass the path of the Python interpreter to the build system, it is best to use the global variable ``python`` directly.\nFor example:\n\n.. code-block:: python\n\n @run_before(\"install\")\n def recythonize(self):\n python(\"setup.py\", \"clean\") # use the `python` global\n\nAs mentioned in the previous section, ``extends(\"python\")`` adds an automatic dependency on ``python-venv``, which is a virtual environment that guarantees build isolation.\nThe ``python`` global always refers to the correct Python interpreter, whether the package uses ``extends(\"python\")`` or ``depends_on(\"python\")``.\n\nAlternatives to Spack\n^^^^^^^^^^^^^^^^^^^^^\n\nPyPI has hundreds of thousands of packages that are not yet in Spack, and ``pip`` may be a perfectly valid alternative to using Spack.\nThe main advantage of Spack over ``pip`` is its ability to compile non-Python dependencies.\nIt can also build cythonized versions of a package or link to an optimized BLAS/LAPACK library like MKL, resulting in calculations that run orders of magnitude faster.\nSpack does not offer a significant advantage over other Python-management systems for installing and using tools like flake8 and sphinx.\nBut if you need packages with non-Python dependencies like numpy and scipy, Spack will be very valuable to you.\n\nAnaconda is another great alternative to Spack, and comes with its own ``conda`` package manager.\nLike Spack, Anaconda is capable of compiling non-Python dependencies.\nAnaconda contains many Python packages that are not yet in Spack, and Spack contains many Python packages that are not yet in Anaconda.\nThe main advantage of Spack over Anaconda is its ability to choose a specific compiler and BLAS/LAPACK or MPI library.\nSpack also has better platform support for supercomputers, and can build optimized binaries for your specific microarchitecture.\n\nExternal documentation\n^^^^^^^^^^^^^^^^^^^^^^\n\nFor more information on Python packaging, see:\n\n* https://packaging.python.org/\n\nFor more information on build and installation frontend tools, see:\n\n* pip: https://pip.pypa.io/\n* build: https://pypa-build.readthedocs.io/\n* installer: https://installer.readthedocs.io/\n\nFor more information on build backend tools, see:\n\n* setuptools: https://setuptools.pypa.io/\n* flit: https://flit.pypa.io/\n* poetry: https://python-poetry.org/\n* hatchling: https://hatch.pypa.io/latest/\n* meson: https://meson-python.readthedocs.io/\n* pdm: https://pdm.fming.dev/latest/\n" }, { "path": "lib/spack/docs/build_systems/qmakepackage.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Learn about the QMake build system in Spack, a script generator for Qt-based projects.\n\n.. _qmakepackage:\n\nQMake\n------\n\nMuch like Autotools and CMake, QMake is a build-script generator designed by the developers of Qt.\nIn its simplest form, Spack's ``QMakePackage`` runs the following steps:\n\n.. code-block:: console\n\n $ qmake\n $ make\n $ make check # optional\n $ make install\n\n\nQMake does not appear to have a standardized way of specifying the installation directory, so you may have to set environment variables or edit ``*.pro`` files to get things working properly.\n\nQMake packages will depend on the virtual ``qmake`` package which is provided by multiple versions of Qt: ``qt`` provides Qt up to Qt5, and ``qt-base`` provides Qt from version Qt6 onwards.\nThis split was motivated by the desire to split the single Qt package into its components to allow for more fine-grained installation.\nTo depend on a specific version, refer to the documentation on :ref:`virtual-dependencies`.\n\nPhases\n^^^^^^\n\nThe ``QMakeBuilder`` and ``QMakePackage`` base classes come with the following phases:\n\n#. ``qmake`` - generate Makefiles\n#. ``build`` - build the project\n#. ``install`` - install the project\n\nBy default, these phases run:\n\n.. code-block:: console\n\n $ qmake\n $ make\n $ make install\n\n\nAny of these phases can be overridden in your package as necessary.\nThere is also a ``check`` method that looks for a ``check`` target in the Makefile.\nIf a ``check`` target exists and the user runs:\n\n.. code-block:: console\n\n $ spack install --test=root \n\n\nSpack will run ``make check`` after the build phase.\n\nImportant files\n^^^^^^^^^^^^^^^\n\nPackages that use the QMake build system can be identified by the presence of a ``.pro`` file.\nThis file declares things like build instructions and dependencies.\n\nOne thing to look for is the ``minQtVersion`` function:\n\n.. code-block:: none\n\n minQtVersion(5, 6, 0)\n\n\nThis means that Qt 5.6.0 is the earliest release that will work.\nYou should specify this in a ``depends_on`` statement.\n\nBuild system dependencies\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nAt the bare minimum, packages that use the QMake build system need a ``qt`` dependency.\nSince this is always the case, the ``QMakePackage`` base class already contains:\n\n.. code-block:: python\n\n depends_on(\"qt\", type=\"build\")\n\n\nIf you want to specify a particular version requirement, or need to link to the ``qt`` libraries, you can override this in your package:\n\n.. code-block:: python\n\n depends_on(\"qt@5.6.0:\")\n\nPassing arguments to qmake\n^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nIf you need to pass any arguments to the ``qmake`` call, you can override the ``qmake_args`` method like so:\n\n.. code-block:: python\n\n def qmake_args(self):\n return [\"-recursive\"]\n\n\nThis method can be used to pass flags as well as variables.\n\n``*.pro`` file in a sub-directory\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nIf the ``*.pro`` file used to tell QMake how to build the package is found in a sub-directory, you can tell Spack to run all phases in this sub-directory by adding the following to the package:\n\n.. code-block:: python\n\n build_directory = \"src\"\n\n\nExternal documentation\n^^^^^^^^^^^^^^^^^^^^^^\n\nFor more information on the QMake build system, see: http://doc.qt.io/qt-5/qmake-manual.html\n" }, { "path": "lib/spack/docs/build_systems/racketpackage.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Learn about the Racket build system in Spack for installing Racket packages and modules.\n\n.. _racketpackage:\n\nRacket\n------\n\nMuch like Python, Racket packages and modules have their own special build system.\nTo learn more about the specifics of the Racket package system, please refer to the `Racket Docs `_.\n\nPhases\n^^^^^^\n\nThe ``RacketBuilder`` and ``RacketPackage`` base classes provide an ``install`` phase that can be overridden, corresponding to the use of:\n\n.. code-block:: console\n\n $ raco pkg install\n\nCaveats\n^^^^^^^\n\nIn principle, ``raco`` supports a second, ``setup`` phase; however, we have not implemented this separately, as in normal circumstances, ``install`` also handles running ``setup`` automatically.\n\nUnlike Python, Racket currently only supports two installation scopes for packages, user or system, and keeps a registry of installed packages at each scope in its configuration files.\nThis means we can't simply compose a \"``RACKET_PATH``\" environment variable listing all of the places packages are installed, and update this at will.\n\nUnfortunately, this means that all currently installed packages which extend Racket via ``raco pkg install`` are accessible whenever Racket is accessible.\n\nAdditionally, because Spack does not implement uninstall hooks, uninstalling a Spack ``rkt-`` package will have no effect on the ``raco`` installed packages visible to your Racket installation.\nInstead, you must manually run ``raco pkg remove`` to keep the two package managers in a mutually consistent state.\n" }, { "path": "lib/spack/docs/build_systems/rocmpackage.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Learn about the ROCmPackage helper in Spack, which provides standard variants, dependencies, and conflicts for building packages that target AMD GPUs.\n\n.. _rocmpackage:\n\nROCm\n------\n\nThe ``ROCmPackage`` is not a build system but a helper package.\nLike ``CudaPackage``, it provides standard variants, dependencies, and conflicts to facilitate building packages targeting AMD GPUs.\n\nYou can find the source for this package (and suggestions for setting up your ``packages.yaml`` file) at ``__.\n\nVariants\n^^^^^^^^\n\nThis package provides the following variants:\n\n* **rocm**\n\n This variant is used to enable/disable building with ``rocm``.\n The default is disabled (or ``False``).\n\n* **amdgpu_target**\n\n This variant supports the optional specification of the AMD GPU architecture.\n Valid values are the names of the GPUs (e.g., ``gfx701``), which are maintained in the ``amdgpu_targets`` property.\n\nDependencies\n^^^^^^^^^^^^\n\nThis package defines basic ROCm dependencies, including ``llvm`` and ``hip``.\n\nConflicts\n^^^^^^^^^\n\nConflicts are used to prevent builds with known bugs or issues.\nThis package already requires that the ``amdgpu_target`` always be specified for ROCm builds.\nIt also defines a conflict that prevents builds with an ``amdgpu_target`` when ``rocm`` is disabled.\n\nRefer to :ref:`packaging_conflicts` for more information on package conflicts.\n\nMethods\n^^^^^^^\n\nThis package provides one custom helper method, which is used to build standard AMD HIP compiler flags.\n\n**hip_flags**\n This built-in static method returns the appropriately formatted ``--amdgpu-target`` build option for ``hipcc``.\n\n This method must be explicitly called when you are creating the arguments for your build in order to use the values.\n\nUsage\n^^^^^^\n\nThis helper package can be added to your package by adding it as a base class of your package.\nFor example, you can add it to your :ref:`CMakePackage `-based package as follows:\n\n.. code-block:: python\n :emphasize-lines: 1,3-6,13-21\n\n class MyRocmPackage(CMakePackage, ROCmPackage):\n ...\n # Ensure +rocm and amdgpu_targets are passed to dependencies\n depends_on(\"mydeppackage\", when=\"+rocm\")\n for val in ROCmPackage.amdgpu_targets:\n depends_on(f\"mydeppackage amdgpu_target={val}\", when=f\"amdgpu_target={val}\")\n ...\n\n def cmake_args(self):\n spec = self.spec\n args = []\n ...\n if spec.satisfies(\"+rocm\"):\n # Set up the HIP macros needed by the build\n args.extend([\"-DENABLE_HIP=ON\", f\"-DHIP_ROOT_DIR={spec['hip'].prefix}\"])\n rocm_archs = spec.variants[\"amdgpu_target\"].value\n if \"none\" not in rocm_archs:\n args.append(f\"-DHIP_HIPCC_FLAGS=--amdgpu-target={','.join(rocm_archs)}\")\n else:\n # Ensure build with HIP is disabled\n args.append(\"-DENABLE_HIP=OFF\")\n ...\n return args\n\n ...\n\nassuming only the ``ENABLE_HIP``, ``HIP_ROOT_DIR``, and ``HIP_HIPCC_FLAGS`` macros are required to be set and the only dependency needing ROCm options is ``mydeppackage``.\nYou will need to customize the flags as needed for your build.\n\nThis example also illustrates how to check for the ``rocm`` variant using ``self.spec`` and how to retrieve the ``amdgpu_target`` variant's value using ``self.spec.variants[\"amdgpu_target\"].value``.\n\nAll five packages using ``ROCmPackage`` as of January 2021 also use the :ref:`CudaPackage `.\nSo, it is worth looking at those packages to get ideas for creating a package that can support both ``cuda`` and ``rocm``.\n" }, { "path": "lib/spack/docs/build_systems/rpackage.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n A guide to packaging R libraries and applications with Spack, including support for CRAN, Bioconductor, and GitHub sources.\n\n.. _rpackage:\n\nR\n------\n\nLike Python, R has its own built-in build system.\n\nThe R build system is remarkably uniform and well-tested.\nThis makes it one of the easiest build systems to create new Spack packages for.\n\nPhases\n^^^^^^\n\nThe ``RBuilder`` and ``RPackage`` base classes have a single phase:\n\n#. ``install`` - install the package\n\nBy default, this phase runs the following command:\n\n.. code-block:: console\n\n $ R CMD INSTALL --library=/path/to/installation/prefix/rlib/R/library .\n\n\nFinding R packages\n^^^^^^^^^^^^^^^^^^\n\nThe vast majority of R packages are hosted on CRAN - The Comprehensive R Archive Network.\nIf you are looking for a particular R package, search for \"CRAN \" and you should quickly find what you want.\nIf it isn't on CRAN, try Bioconductor, another common R repository.\n\nFor the purposes of this tutorial, we will be walking through `r-caret `_ as an example.\nIf you search for \"CRAN caret\", you will quickly find what you are looking for at https://cran.r-project.org/package=caret. https://cran.r-project.org is the main CRAN website.\nHowever, CRAN also has a https://cloud.r-project.org site that automatically redirects to `mirrors around the world `_.\nFor stability and performance reasons, we will use https://cloud.r-project.org/package=caret.\nIf you search for \"Package source\", you will find the download URL for the latest release.\nUse this URL with ``spack create`` to create a new package.\n\nPackage name\n^^^^^^^^^^^^\n\nThe first thing you'll notice is that Spack prepends ``r-`` to the front of the package name.\nThis is how Spack separates R extensions from the rest of the packages in Spack.\nWithout this, we would end up with package name collisions more frequently than we would like.\nFor instance, there are already packages for both:\n\n* ``ape`` and ``r-ape``\n* ``curl`` and ``r-curl``\n* ``gmp`` and ``r-gmp``\n* ``jpeg`` and ``r-jpeg``\n* ``openssl`` and ``r-openssl``\n* ``uuid`` and ``r-uuid``\n* ``xts`` and ``r-xts``\n\nMany popular programs written in C/C++ are later ported to R as a separate project.\n\nDescription\n^^^^^^^^^^^\n\nThe first thing you'll need to add to your new package is a description.\nThe top of the homepage for ``caret`` lists the following description:\n\n Classification and Regression Training\n\n Misc functions for training and plotting classification and regression models.\n\nThe first line is a short description (title) and the second line is a long description.\nIn this case the description is only one line but often the description is several lines.\nSpack makes use of both short and long descriptions and convention is to use both when creating an R package.\n\nHomepage\n^^^^^^^^\n\nIf you look at the bottom of the page, you'll see:\n\n Linking:\n\n Please use the canonical form https://CRAN.R-project.org/package=caret to link to this page.\n\nPlease uphold the wishes of the CRAN admins and use https://cloud.r-project.org/package=caret as the homepage instead of https://cloud.r-project.org/web/packages/caret/index.html.\nThe latter may change without notice.\n\nURL\n^^^^^^\n\nAs previously mentioned, the download URL for the latest release can be found by searching \"Package source\" on the homepage.\n\nList URL\n^^^^^^^^\n\nCRAN maintains a single webpage containing the latest release of every single package: https://cloud.r-project.org/src/contrib/\n\nOf course, as soon as a new release comes out, the version you were using in your package is no longer available at that URL.\nIt is moved to an archive directory.\nIf you search for \"Old sources\", you will find: https://cloud.r-project.org/src/contrib/Archive/caret\n\nIf you only specify the URL for the latest release, your package will no longer be able to fetch that version as soon as a new release comes out.\nTo get around this, add the archive directory as a ``list_url``.\n\nBioconductor packages\n^^^^^^^^^^^^^^^^^^^^^\n\nBioconductor packages are set up in a similar way to CRAN packages, but there are some very important distinctions.\nBioconductor packages can be found at: https://bioconductor.org/.\nBioconductor packages are R packages and so follow the same packaging scheme as CRAN packages.\nWhat is different is that Bioconductor itself is versioned and released.\nThis scheme, using the Bioconductor package installer, allows further specification of the minimum version of R as well as further restrictions on the dependencies between packages than what is possible with the native R packaging system.\nSpack cannot replicate these extra features and thus Bioconductor packages in Spack need to be managed as a group during updates in order to maintain package consistency with Bioconductor itself.\n\nAnother key difference is that, while previous versions of packages are available, they are not available from a site that can be programmatically set, thus a ``list_url`` attribute cannot be used.\nHowever, each package is also available in a git repository, with branches corresponding to each Bioconductor release.\nThus, it is always possible to retrieve the version of any package corresponding to a Bioconductor release simply by fetching the branch that corresponds to the Bioconductor release of the package repository.\nFor this reason, Spack Bioconductor R packages use the git repository, with the commit of the respective branch used in the ``version()`` attribute of the package.\n\ncran and bioc attributes\n^^^^^^^^^^^^^^^^^^^^^^^^\n\nMuch like the ``pypi`` attribute for Python packages, due to the fact that R packages are obtained from specific repositories, it is possible to set up shortcut attributes that can be used to set ``homepage``, ``url``, ``list_url``, and ``git``.\nFor example, the following ``cran`` attribute:\n\n.. code-block:: python\n\n cran = \"caret\"\n\nis equivalent to:\n\n.. code-block:: python\n\n homepage = \"https://cloud.r-project.org/package=caret\"\n url = \"https://cloud.r-project.org/src/contrib/caret_6.0-86.tar.gz\"\n list_url = \"https://cloud.r-project.org/src/contrib/Archive/caret\"\n\nLikewise, the following ``bioc`` attribute:\n\n.. code-block:: python\n\n bioc = \"BiocVersion\"\n\nis equivalent to:\n\n.. code-block:: python\n\n homepage = \"https://bioconductor.org/packages/BiocVersion/\"\n git = \"https://git.bioconductor.org/packages/BiocVersion\"\n\n\nBuild system dependencies\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nAs an extension of the R ecosystem, your package will obviously depend on R to build and run.\nNormally, we would use ``depends_on`` to express this, but for R packages, we use ``extends``.\nThis implies a special dependency on R, which is used to set environment variables such as ``R_LIBS`` uniformly.\nSince every R package needs this, the ``RPackage`` base class contains:\n\n.. code-block:: python\n\n extends(\"r\")\n\n\nTake a close look at the homepage for ``caret``.\nIf you look at the \"Depends\" section, you'll notice that ``caret`` depends on \"R (≥ 3.2.0)\".\nYou should add this to your package like so:\n\n.. code-block:: python\n\n depends_on(\"r@3.2.0:\", type=(\"build\", \"run\"))\n\n\nR dependencies\n^^^^^^^^^^^^^^\n\nR packages are often small and follow the classic Unix philosophy of doing one thing well.\nThey are modular and usually depend on several other packages.\nYou may find a single package with over a hundred dependencies.\nLuckily, R packages are well-documented and list all of their dependencies in the following sections:\n\n* Depends\n* Imports\n* LinkingTo\n\nAs far as Spack is concerned, all three of these dependency types correspond to ``type=(\"build\", \"run\")``, so you don't have to worry about the details.\nIf you are curious what they mean, https://github.com/spack/spack/issues/2951 has a pretty good summary:\n\n ``Depends`` is required and will cause those R packages to be *attached*, that is, their APIs are exposed to the user.\n ``Imports`` *loads* packages so that *the package* importing these packages can access their APIs, while *not* being exposed to the user.\n When a user calls ``library(foo)`` s/he *attaches* package ``foo`` and all of the packages under ``Depends``.\n Any function in one of these packages can be called directly as ``bar()``.\n If there are conflicts, a user can also specify ``pkgA::bar()`` and ``pkgB::bar()`` to distinguish between them.\n Historically, there was only ``Depends`` and ``Suggests``, hence the confusing names.\n Today, maybe ``Depends`` would have been named ``Attaches``.\n\n The ``LinkingTo`` is not perfect and there was recently an extensive discussion about API/ABI among other things on the R-devel mailing list among very skilled R developers:\n\n * https://stat.ethz.ch/pipermail/r-devel/2016-December/073505.html\n * https://stat.ethz.ch/pipermail/r-devel/2017-January/073647.html\n\nSome packages also have a fourth section:\n\n* Suggests\n\nThese are optional, rarely-used dependencies that a user might find useful.\nYou should **NOT** add these dependencies to your package.\nR packages already have enough dependencies as it is, and adding optional dependencies can really slow down the concretization process.\nThey can also introduce circular dependencies.\n\nA fifth rarely used section is:\n\n* Enhances\n\nThis means that the package can be used as an optional dependency for another package.\nAgain, these packages should **NOT** be listed as dependencies.\n\nCore, recommended, and non-core packages\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nIf you look at \"Depends\", \"Imports\", and \"LinkingTo\", you will notice 3 different types of packages:\n\nCore packages\n\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nIf you look at the ``caret`` homepage, you'll notice a few dependencies that don't have a link to the package, like ``methods``, ``stats``, and ``utils``.\nThese packages are part of the core R distribution and are tied to the R version installed.\nYou can basically consider these to be \"R itself\".\nThese are so essential to R that it would not make sense for them to be updated via CRAN.\nIf you did, you would basically get a different version of R.\nThus, they're updated when R is updated.\n\nYou can find a list of these core libraries at: https://github.com/wch/r-source/tree/trunk/src/library\n\nRecommended packages\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nWhen you install R, there is an option called ``--with-recommended-packages``.\nThis flag causes the R installation to include a few \"Recommended\" packages (legacy term).\nThey are for historical reasons quite tied to the core R distribution, developed by the R core team or people closely related to it.\nThe R core distribution \"knows\" about these packages, but they are indeed distributed via CRAN.\nBecause they're distributed via CRAN, they can also be updated between R version releases.\n\nSpack explicitly adds the ``--without-recommended-packages`` flag to prevent the installation of these packages.\nDue to the way Spack handles package activation (symlinking packages to the R installation directory), pre-existing recommended packages will cause conflicts for already-existing files.\nWe could either not include these recommended packages in Spack and require them to be installed through ``--with-recommended-packages``, or we could not install them with R and let users choose the version of the package they want to install.\nWe chose the latter.\n\nSince these packages are so commonly distributed with the R system, many developers may assume these packages exist and fail to list them as dependencies.\nWatch out for this.\n\nYou can find a list of these recommended packages at: https://github.com/wch/r-source/blob/trunk/share/make/vars.mk\n\nNon-core packages\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nThese are packages that are neither \"core\" nor \"recommended\".\nThere are more than 10,000 of these packages hosted on CRAN alone.\n\nFor each of these package types, if you see that a specific version is required, for example, \"lattice (≥ 0.20)\", please add this information to the dependency:\n\n.. code-block:: python\n\n depends_on(\"r-lattice@0.20:\", type=(\"build\", \"run\"))\n\n\nNon-R dependencies\n^^^^^^^^^^^^^^^^^^\n\nSome packages depend on non-R libraries for linking.\nCheck out the `r-stringi `_ package for an example: https://cloud.r-project.org/package=stringi.\nIf you search for the text \"SystemRequirements\", you will see:\n\n ICU4C (>= 52, optional)\n\nThis is how non-R dependencies are listed.\nMake sure to add these dependencies.\nThe default dependency type should suffice.\n\nPassing arguments to the installation\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nSome R packages provide additional flags that can be passed to ``R CMD INSTALL``, often to locate non-R dependencies.\n`r-rmpi `_ is an example of this, as it uses flags for linking to an MPI library.\nTo pass these to the installation command, you can override ``configure_args`` like so:\n\n.. code-block:: python\n\n def configure_args(self):\n mpi_name = self.spec[\"mpi\"].name\n\n # The type of MPI. Supported values are:\n # OPENMPI, LAM, MPICH, MPICH2, or CRAY\n if mpi_name == \"openmpi\":\n Rmpi_type = \"OPENMPI\"\n elif mpi_name == \"mpich\":\n Rmpi_type = \"MPICH2\"\n else:\n raise InstallError(\"Unsupported MPI type\")\n\n return [\n \"--with-Rmpi-type={0}\".format(Rmpi_type),\n \"--with-mpi={0}\".format(spec[\"mpi\"].prefix),\n ]\n\n\nThere is a similar ``configure_vars`` function that can be overridden to pass variables to the build.\n\nAlternatives to Spack\n^^^^^^^^^^^^^^^^^^^^^\n\nCRAN hosts over 10,000 R packages, most of which are not in Spack.\nMany users may not need the advanced features of Spack, and may prefer to install R packages the normal way:\n\n.. code-block:: console\n\n $ R\n > install.packages(\"ggplot2\")\n\n\nR will search CRAN for the ``ggplot2`` package and install all necessary dependencies for you.\nIf you want to update all installed R packages to the latest release, you can use:\n\n.. code-block:: console\n\n > update.packages(ask = FALSE)\n\n\nThis works great for users who have internet access, but those on an air-gapped cluster will find it easier to let Spack build a download mirror and install these packages for you.\n\nWhere Spack really shines is its ability to install non-R dependencies and link to them properly, something the R installation mechanism cannot handle.\n\nExternal documentation\n^^^^^^^^^^^^^^^^^^^^^^\n\nFor more information on installing R packages, see: https://stat.ethz.ch/R-manual/R-devel/library/utils/html/INSTALL.html\n\nFor more information on writing R packages, see: https://cloud.r-project.org/doc/manuals/r-release/R-exts.html\n\nIn particular, https://cloud.r-project.org/doc/manuals/r-release/R-exts.html#Package-Dependencies has a great explanation of the difference between Depends, Imports, and LinkingTo.\n" }, { "path": "lib/spack/docs/build_systems/rubypackage.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Discover the Ruby build system in Spack for installing Ruby gems, with support for gemspec, Rakefile, and pre-packaged .gem files.\n\n.. _rubypackage:\n\nRuby\n------\n\nLike Perl, Python, and R, Ruby has its own build system for installing Ruby gems.\n\nPhases\n^^^^^^\n\nThe ``RubyBuilder`` and ``RubyPackage`` base classes provide the following phases that can be overridden:\n\n#. ``build`` - build everything needed to install\n#. ``install`` - install everything from build directory\n\nFor packages that come with a ``*.gemspec`` file, these phases run:\n\n.. code-block:: console\n\n $ gem build *.gemspec\n $ gem install *.gem\n\n\nFor packages that come with a ``Rakefile`` file, these phases run:\n\n.. code-block:: console\n\n $ rake package\n $ gem install *.gem\n\n\nFor packages that come pre-packaged as a ``*.gem`` file, the build phase is skipped and the install phase runs:\n\n.. code-block:: console\n\n $ gem install *.gem\n\n\nThese are all standard ``gem`` commands and can be found by running:\n\n.. code-block:: console\n\n $ gem help commands\n\n\nFor packages that only distribute ``*.gem`` files, these files can be downloaded with the ``expand=False`` option in the ``version`` directive.\nThe build phase will be automatically skipped.\n\nImportant files\n^^^^^^^^^^^^^^^\n\nWhen building from source, Ruby packages can be identified by the presence of any of the following files:\n\n* ``*.gemspec``\n* ``Rakefile``\n* ``setup.rb`` (not yet supported)\n\nHowever, not all Ruby packages are released as source code.\nSome are only released as ``*.gem`` files.\nThese files can be extracted using:\n\n.. code-block:: console\n\n $ gem unpack *.gem\n\n\nDescription\n^^^^^^^^^^^\n\nThe ``*.gemspec`` file may contain something like:\n\n.. code-block:: ruby\n\n summary = \"An implementation of the AsciiDoc text processor and publishing toolchain\"\n description = \"A fast, open source text processor and publishing toolchain for converting AsciiDoc content to HTML 5, DocBook 5, and other formats.\"\n\n\nEither of these can be used for the description of the Spack package.\n\nHomepage\n^^^^^^^^\n\nThe ``*.gemspec`` file may contain something like:\n\n.. code-block:: ruby\n\n homepage = \"https://asciidoctor.org\"\n\n\nThis should be used as the official homepage of the Spack package.\n\nBuild system dependencies\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nAll Ruby packages require Ruby at build and run-time.\nFor this reason, the base class contains:\n\n.. code-block:: python\n\n extends(\"ruby\")\n\n\nThe ``*.gemspec`` file may contain something like:\n\n.. code-block:: ruby\n\n required_ruby_version = \">= 2.3.0\"\n\n\nThis can be added to the Spack package using:\n\n.. code-block:: python\n\n depends_on(\"ruby@2.3.0:\", type=(\"build\", \"run\"))\n\n\nRuby dependencies\n^^^^^^^^^^^^^^^^^\n\nWhen you install a package with ``gem``, it reads the ``*.gemspec`` file in order to determine the dependencies of the package.\nIf the dependencies are not yet installed, ``gem`` downloads them and installs them for you.\nThis may sound convenient, but Spack cannot rely on this behavior for two reasons:\n\n#. Spack needs to be able to install packages on air-gapped networks.\n\n If there is no internet connection, ``gem`` can't download the package dependencies.\n By explicitly listing every dependency in the ``package.py``, Spack knows what to download ahead of time.\n\n#. Duplicate installations of the same dependency may occur.\n\n Spack supports *activation* of Ruby extensions, which involves symlinking the package installation prefix to the Ruby installation prefix.\n If your package is missing a dependency, that dependency will be installed to the installation directory of the same package.\n If you try to activate the package + dependency, it may cause a problem if that package has already been activated.\n\nFor these reasons, you must always explicitly list all dependencies.\nAlthough the documentation may list the package's dependencies, often the developers assume people will use ``gem`` and won't have to worry about it.\nAlways check the ``*.gemspec`` file to find the true dependencies.\n\nCheck for the following clues in the ``*.gemspec`` file:\n\n* ``add_runtime_dependency``\n\n These packages are required for installation.\n\n* ``add_dependency``\n\n This is an alias for ``add_runtime_dependency``\n\n* ``add_development_dependency``\n\n These packages are optional dependencies used for development.\n They should not be added as dependencies of the package.\n\nExternal documentation\n^^^^^^^^^^^^^^^^^^^^^^\n\nFor more information on Ruby packaging, see: https://guides.rubygems.org/\n" }, { "path": "lib/spack/docs/build_systems/sconspackage.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Learn about the SCons build system in Spack, a Python-based tool that handles building and linking without relying on Makefiles.\n\n.. _sconspackage:\n\nSCons\n------\n\nSCons is a general-purpose build system that does not rely on Makefiles to build software.\nSCons is written in Python, and handles all building and linking itself.\n\nAs far as build systems go, SCons is very non-uniform.\nIt provides a common framework for developers to write build scripts, but the build scripts themselves can vary drastically.\nSome developers add subcommands like:\n\n.. code-block:: console\n\n $ scons clean\n $ scons build\n $ scons test\n $ scons install\n\n\nOthers don't add any subcommands.\nSome have configuration options that can be specified through variables on the command line.\nOthers don't.\n\nPhases\n^^^^^^\n\nAs previously mentioned, SCons allows developers to add subcommands like ``build`` and ``install``, but by default, installation usually looks like:\n\n.. code-block:: console\n\n $ scons\n $ scons install\n\n\nTo facilitate this, the ``SConsBuilder`` and ``SConsPackage`` base classes provide the following phases:\n\n#. ``build`` - build the package\n#. ``install`` - install the package\n\nPackage developers often add unit tests that can be invoked with ``scons test`` or ``scons check``.\nSpack provides a ``build_test`` method to handle this.\nSince we don't know which one the package developer chose, the ``build_test`` method does nothing by default, but can be easily overridden like so:\n\n.. code-block:: python\n\n def build_test(self):\n scons(\"check\")\n\n\nImportant files\n^^^^^^^^^^^^^^^\n\nSCons packages can be identified by their ``SConstruct`` files.\nThese files handle everything from setting up subcommands and command-line options to linking and compiling.\n\nOne thing to look for is the ``EnsureSConsVersion`` function:\n\n.. code-block:: none\n\n EnsureSConsVersion(2, 3, 0)\n\n\nThis means that SCons 2.3.0 is the earliest release that will work.\nYou should specify this in a ``depends_on`` statement.\n\nBuild system dependencies\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nAt the bare minimum, packages that use the SCons build system need a ``scons`` dependency.\nSince this is always the case, the ``SConsPackage`` base class already contains:\n\n.. code-block:: python\n\n depends_on(\"scons\", type=\"build\")\n\n\nIf you want to specify a particular version requirement, you can override this in your package:\n\n.. code-block:: python\n\n depends_on(\"scons@2.3.0:\", type=\"build\")\n\n\nFinding available options\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nThe first place to start when looking for a list of valid options to build a package is ``scons --help``.\nSome packages like `kahip `_ don't bother overwriting the default SCons help message, so this isn't very useful, but other packages like `serf `_ print a list of valid command-line variables:\n\n.. code-block:: console\n\n $ scons --help\n scons: Reading SConscript files ...\n Checking for GNU-compatible C compiler...yes\n scons: done reading SConscript files.\n\n PREFIX: Directory to install under ( /path/to/PREFIX )\n default: /usr/local\n actual: /usr/local\n\n LIBDIR: Directory to install architecture dependent libraries under ( /path/to/LIBDIR )\n default: $PREFIX/lib\n actual: /usr/local/lib\n\n APR: Path to apr-1-config, or to APR's install area ( /path/to/APR )\n default: /usr\n actual: /usr\n\n APU: Path to apu-1-config, or to APR's install area ( /path/to/APU )\n default: /usr\n actual: /usr\n\n OPENSSL: Path to OpenSSL's install area ( /path/to/OPENSSL )\n default: /usr\n actual: /usr\n\n ZLIB: Path to zlib's install area ( /path/to/ZLIB )\n default: /usr\n actual: /usr\n\n GSSAPI: Path to GSSAPI's install area ( /path/to/GSSAPI )\n default: None\n actual: None\n\n DEBUG: Enable debugging info and strict compile warnings (yes|no)\n default: False\n actual: False\n\n APR_STATIC: Enable using a static compiled APR (yes|no)\n default: False\n actual: False\n\n CC: Command name or path of the C compiler\n default: None\n actual: gcc\n\n CFLAGS: Extra flags for the C compiler (space-separated)\n default: None\n actual:\n\n LIBS: Extra libraries passed to the linker, e.g. \"-l -l\" (space separated)\n default: None\n actual: None\n\n LINKFLAGS: Extra flags for the linker (space-separated)\n default: None\n actual:\n\n CPPFLAGS: Extra flags for the C preprocessor (space separated)\n default: None\n actual: None\n\n Use scons -H for help about command-line options.\n\n\nMore advanced packages like `cantera `_ use ``scons --help`` to print a list of subcommands:\n\n.. code-block:: console\n\n $ scons --help\n scons: Reading SConscript files ...\n\n SCons build script for Cantera\n\n Basic usage:\n 'scons help' - print a description of user-specifiable options.\n\n 'scons build' - Compile Cantera and the language interfaces using\n default options.\n\n 'scons clean' - Delete files created while building Cantera.\n\n '[sudo] scons install' - Install Cantera.\n\n '[sudo] scons uninstall' - Uninstall Cantera.\n\n 'scons test' - Run all tests which did not previously pass or for which the\n results may have changed.\n\n 'scons test-reset' - Reset the passing status of all tests.\n\n 'scons test-clean' - Delete files created while running the tests.\n\n 'scons test-help' - List available tests.\n\n 'scons test-NAME' - Run the test named \"NAME\".\n\n 'scons dump' - Dump the state of the SCons environment to the\n screen instead of doing , e.g.\n 'scons build dump'. For debugging purposes.\n\n 'scons samples' - Compile the C++ and Fortran samples.\n\n 'scons msi' - Build a Windows installer (.msi) for Cantera.\n\n 'scons sphinx' - Build the Sphinx documentation\n\n 'scons doxygen' - Build the Doxygen documentation\n\n\nYou'll notice that cantera provides a ``scons help`` subcommand.\nRunning ``scons help`` prints a list of valid command-line variables.\n\nPassing arguments to SCons\n^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nNow that you know what arguments the project accepts, you can add them to the package build phase.\nThis is done by overriding ``build_args`` like so:\n\n.. code-block:: python\n\n def build_args(self, spec, prefix):\n args = [\n f\"PREFIX={prefix}\",\n f\"ZLIB={spec['zlib'].prefix}\",\n ]\n\n if spec.satisfies(\"+debug\"):\n args.append(\"DEBUG=yes\")\n else:\n args.append(\"DEBUG=no\")\n\n return args\n\n\n``SConsPackage`` also provides an ``install_args`` function that you can override to pass additional arguments to ``scons install``.\n\nCompiler wrappers\n^^^^^^^^^^^^^^^^^\n\nBy default, SCons builds all packages in a separate execution environment, and doesn't pass any environment variables from the user environment.\nEven changes to ``PATH`` are not propagated unless the package developer does so.\n\nThis is particularly troublesome for Spack's compiler wrappers, which depend on environment variables to manage dependencies and linking flags.\nIn many cases, SCons packages are not compatible with Spack's compiler wrappers, and linking must be done manually.\n\nFirst of all, check the list of valid options for anything relating to environment variables.\nFor example, cantera has the following option:\n\n.. code-block:: none\n\n * env_vars: [ string ]\n Environment variables to propagate through to SCons. Either the\n string \"all\" or a comma separated list of variable names, e.g.\n \"LD_LIBRARY_PATH,HOME\".\n - default: \"LD_LIBRARY_PATH,PYTHONPATH\"\n\n\nIn the case of cantera, using ``env_vars=all`` allows us to use Spack's compiler wrappers.\nIf you don't see an option related to environment variables, try using Spack's compiler wrappers by passing ``spack_cc``, ``spack_cxx``, and ``spack_fc`` via the ``CC``, ``CXX``, and ``FC`` arguments, respectively.\nIf you pass them to the build and you see an error message like:\n\n.. code-block:: none\n\n Spack compiler must be run from Spack! Input 'SPACK_PREFIX' is missing.\n\n\nyou'll know that the package isn't compatible with Spack's compiler wrappers.\nIn this case, you'll have to use the path to the actual compilers, which are stored in ``self.compiler.cc`` and friends.\nNote that this may involve passing additional flags to the build to locate dependencies, a task normally done by the compiler wrappers. serf is an example of a package with this limitation.\n\nExternal documentation\n^^^^^^^^^^^^^^^^^^^^^^\n\nFor more information on the SCons build system, see: http://scons.org/documentation.html\n" }, { "path": "lib/spack/docs/build_systems/sippackage.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n A guide to using the SIP build system in Spack for creating Python bindings for C and C++ libraries.\n\n.. _sippackage:\n\nSIP\n------\n\nSIP is a tool that makes it very easy to create Python bindings for C and C++ libraries.\nIt was originally developed to create PyQt, the Python bindings for the Qt toolkit, but can be used to create bindings for any C or C++ library.\n\nSIP comprises a code generator and a Python module.\nThe code generator processes a set of specification files and generates C or C++ code which is then compiled to create the bindings extension module.\nThe SIP Python module provides support functions to the automatically generated code.\n\nPhases\n^^^^^^\n\nThe ``SIPBuilder`` and ``SIPPackage`` base classes come with the following phases:\n\n#. ``configure`` - configure the package\n#. ``build`` - build the package\n#. ``install`` - install the package\n\nBy default, these phases run:\n\n.. code-block:: console\n\n $ sip-build --verbose --target-dir ...\n $ make\n $ make install\n\n\nImportant files\n^^^^^^^^^^^^^^^\n\nEach SIP package comes with a custom configuration file written in Python.\nFor newer packages, this is called ``project.py``, while in older packages, it may be called ``configure.py``.\nThis script contains instructions to build the project.\n\nBuild system dependencies\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\n``SIPPackage`` requires several dependencies.\nPython and SIP are needed at build-time to run the aforementioned configure script.\nPython is also needed at run-time to actually use the installed Python library.\nAnd as we are building Python bindings for C/C++ libraries, Python is also needed as a link dependency.\nAll of these dependencies are automatically added via the base class.\n\n.. code-block:: python\n\n extends(\"python\", type=(\"build\", \"link\", \"run\"))\n depends_on(\"py-sip\", type=\"build\")\n\n\nPassing arguments to ``sip-build``\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nEach phase comes with a ```` function that can be used to pass arguments to that particular phase.\nFor example, if you need to pass arguments to the configure phase, you can use:\n\n.. code-block:: python\n\n def configure_args(self):\n return [\"--no-python-dbus\"]\n\n\nA list of valid options can be found by running ``sip-build --help``.\n\nTesting\n^^^^^^^\n\nJust because a package successfully built does not mean that it built correctly.\nThe most reliable test of whether or not the package was correctly installed is to attempt to import all of the modules that get installed.\nTo get a list of modules, run the following command in the site-packages directory:\n\n.. code-block:: pycon\n\n >>> import setuptools\n >>> setuptools.find_packages()\n [\n 'PyQt5', 'PyQt5.QtCore', 'PyQt5.QtGui', 'PyQt5.QtHelp',\n 'PyQt5.QtMultimedia', 'PyQt5.QtMultimediaWidgets', 'PyQt5.QtNetwork',\n 'PyQt5.QtOpenGL', 'PyQt5.QtPrintSupport', 'PyQt5.QtQml',\n 'PyQt5.QtQuick', 'PyQt5.QtSvg', 'PyQt5.QtTest', 'PyQt5.QtWebChannel',\n 'PyQt5.QtWebSockets', 'PyQt5.QtWidgets', 'PyQt5.QtXml',\n 'PyQt5.QtXmlPatterns'\n ]\n\n\nLarge, complex packages like ``py-pyqt5`` will return a long list of packages, while other packages may return an empty list.\nThese packages only install a single ``foo.py`` file.\nIn Python packaging lingo, a \"package\" is a directory containing files like:\n\n.. code-block:: none\n\n foo/__init__.py\n foo/bar.py\n foo/baz.py\n\n\nwhereas a \"module\" is a single Python file.\n\nThe ``SIPPackage`` base class automatically detects these module names for you.\nIf, for whatever reason, the module names detected are wrong, you can provide the names yourself by overriding ``import_modules`` like so:\n\n.. code-block:: python\n\n import_modules = [\"PyQt5\"]\n\n\nThese tests often catch missing dependencies and non-RPATHed libraries.\nMake sure not to add modules/packages containing the word \"test\", as these likely won't end up in the installation directory, or may require test dependencies like pytest to be installed.\n\nThese tests can be triggered by running ``spack install --test=root`` or by running ``spack test run`` after the installation has finished.\n\nExternal documentation\n^^^^^^^^^^^^^^^^^^^^^^\n\nFor more information on the SIP build system, see:\n\n* https://www.riverbankcomputing.com/software/sip/intro\n* https://www.riverbankcomputing.com/static/Docs/sip/\n* https://wiki.python.org/moin/SIP\n" }, { "path": "lib/spack/docs/build_systems/sourceforgepackage.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Discover how to use the SourceforgePackage mixin in Spack to automatically generate download URLs for packages hosted on SourceForge.\n\n.. _sourceforgepackage:\n\nSourceForge\n-----------\n\n``SourceforgePackage`` is a `mixin-class `_.\nIt automatically sets the URL based on a list of SourceForge mirrors listed in ``sourceforge_mirror_path``, which defaults to a half dozen known mirrors.\nRefer to the `package source `__ for the current list of mirrors used by Spack.\n\n\nMethods\n^^^^^^^\n\nThis package provides a method for populating mirror URLs.\n\n**urls**\n This method returns a list of possible URLs for package source.\n It is decorated with `property` so its results are treated as a package attribute.\n\n Refer to :ref:`mirrors-of-the-main-url` for information on how Spack uses the ``urls`` attribute during fetching.\n\nUsage\n^^^^^^\n\nThis helper package can be added to your package by adding it as a base class of your package and defining the relative location of an archive file for one version of your software.\n\n.. code-block:: python\n :emphasize-lines: 1,3\n\n class MyPackage(AutotoolsPackage, SourceforgePackage):\n ...\n sourceforge_mirror_path = \"my-package/mypackage.1.0.0.tar.gz\"\n ...\n\nOver 40 packages are using ``SourceforgePackage`` this mix-in as of July 2022 so there are multiple packages to choose from if you want to see a real example.\n" }, { "path": "lib/spack/docs/build_systems/wafpackage.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Explore the Waf build system in Spack, a Python-based tool for configuring and building software projects without Makefiles.\n\n.. _wafpackage:\n\nWaf\n------\n\nLike SCons, Waf is a general-purpose build system that does not rely on Makefiles to build software.\n\nPhases\n^^^^^^\n\nThe ``WafBuilder`` and ``WafPackage`` base classes come with the following phases:\n\n#. ``configure`` - configure the project\n#. ``build`` - build the project\n#. ``install`` - install the project\n\nBy default, these phases run:\n\n.. code-block:: console\n\n $ python waf configure --prefix=/path/to/installation/prefix\n $ python waf build\n $ python waf install\n\n\nEach of these are standard Waf commands and can be found by running:\n\n.. code-block:: console\n\n $ python waf --help\n\n\nEach phase provides a ```` function that runs:\n\n.. code-block:: console\n\n $ python waf -j \n\n\nwhere ```` is the number of parallel jobs to build with.\nEach phase also has a ```` function that can pass arguments to this call.\nAll of these functions are empty.\nThe ``configure`` phase automatically adds ``--prefix=/path/to/installation/prefix``, so you don't need to add that in the ``configure_args``.\n\nTesting\n^^^^^^^\n\n``WafPackage`` also provides ``test`` and ``installtest`` methods, which are run after the ``build`` and ``install`` phases, respectively.\nBy default, these phases do nothing, but you can override them to run package-specific unit tests.\n\n.. code-block:: python\n\n def installtest(self):\n with working_dir(\"test\"):\n pytest = which(\"py.test\")\n pytest()\n\n\nImportant files\n^^^^^^^^^^^^^^^\n\nEach Waf package comes with a custom ``waf`` build script, written in Python.\nThis script contains instructions to build the project.\n\nThe package also comes with a ``wscript`` file.\nThis file is used to override the default ``configure``, ``build``, and ``install`` phases to customize the Waf project.\nIt also allows developers to override the default ``./waf --help`` message.\nCheck this file to find useful information about dependencies and the minimum versions that are supported.\n\nBuild system dependencies\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\n``WafPackage`` does not require ``waf`` to build.\n``waf`` is only needed to create the ``./waf`` script.\nSince ``./waf`` is a Python script, Python is needed to build the project.\n``WafPackage`` adds the following dependency automatically:\n\n.. code-block:: python\n\n depends_on(\"python@2.5:\", type=\"build\")\n\n\nWaf only supports Python 2.5 and up.\n\nPassing arguments to Waf\n^^^^^^^^^^^^^^^^^^^^^^^^\n\nAs previously mentioned, each phase comes with a ```` function that can be used to pass arguments to that particular phase.\nFor example, if you need to pass arguments to the build phase, you can use:\n\n.. code-block:: python\n\n def build_args(self, spec, prefix):\n args = []\n\n if self.run_tests:\n args.append(\"--test\")\n\n return args\n\n\nA list of valid options can be found by running ``./waf --help``.\n\nExternal documentation\n^^^^^^^^^^^^^^^^^^^^^^\n\nFor more information on the Waf build system, see: https://waf.io/book/\n" }, { "path": "lib/spack/docs/build_systems.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n An overview of the build systems supported by Spack, with links to detailed documentation for each system.\n\n.. _build-systems:\n\nBuild Systems\n=============\n\nSpack defines a number of classes that understand how to use common `build systems `_ (Makefiles, CMake, etc.).\nSpack package definitions can inherit these classes in order to streamline their builds.\n\nThis guide provides information specific to each particular build system.\nIt assumes that you've read the Packaging Guide :doc:`part 1 ` and :doc:`part 2 ` and expands on these ideas for each distinct build system that Spack supports:\n\n.. toctree::\n :maxdepth: 1\n :caption: Make-based\n\n build_systems/makefilepackage\n\n.. toctree::\n :maxdepth: 1\n :caption: Make-incompatible\n\n build_systems/mavenpackage\n build_systems/sconspackage\n build_systems/wafpackage\n\n.. toctree::\n :maxdepth: 1\n :caption: Build-script generation\n\n build_systems/autotoolspackage\n build_systems/cmakepackage\n build_systems/cachedcmakepackage\n build_systems/mesonpackage\n build_systems/qmakepackage\n build_systems/sippackage\n\n.. toctree::\n :maxdepth: 1\n :caption: Language-specific\n\n build_systems/luapackage\n build_systems/octavepackage\n build_systems/perlpackage\n build_systems/pythonpackage\n build_systems/rpackage\n build_systems/racketpackage\n build_systems/rubypackage\n\n.. toctree::\n :maxdepth: 1\n :caption: Other\n\n build_systems/bundlepackage\n build_systems/cudapackage\n build_systems/custompackage\n build_systems/inteloneapipackage\n build_systems/rocmpackage\n build_systems/sourceforgepackage\n\nFor reference, the :py:mod:`Build System API docs ` provide a list of build systems and methods/attributes that can be overridden.\nIf you are curious about the implementation of a particular build system, you can view the source code by running:\n\n.. code-block:: console\n\n $ spack edit --build-system autotools\n\n\nThis will open up the ``AutotoolsPackage`` definition in your favorite editor.\nIn addition, if you are working with a less common build system like QMake, SCons, or Waf, it may be useful to see examples of other packages.\nYou can quickly find examples by running:\n\n.. code-block:: console\n\n $ spack cd --packages\n $ grep -l QMakePackage */package.py\n\n\nYou can then view these packages with ``spack edit``.\n\nThis guide is intended to supplement the :py:mod:`Build System API docs ` with examples of how to override commonly used methods.\nIt also provides rules of thumb and suggestions for package developers who are unfamiliar with a particular build system.\n" }, { "path": "lib/spack/docs/chain.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Learn how to chain Spack installations by pointing one Spack instance to another to use its installed packages.\n\nChaining Spack Installations (upstreams.yaml)\n=============================================\n\nYou can point your Spack installation to another Spack installation to use any packages that are installed there.\nTo register the other Spack instance, you can add it as an entry to ``upstreams.yaml`` at any of the :ref:`configuration-scopes`:\n\n.. code-block:: yaml\n\n upstreams:\n spack-instance-1:\n install_tree: /path/to/other/spack/opt/spack\n spack-instance-2:\n install_tree: /path/to/another/spack/opt/spack\n\nThe ``install_tree`` must point to the ``opt/spack`` directory inside of the Spack base directory, or the location of the ``install_tree`` defined in :ref:`config.yaml `.\n\nOnce the upstream Spack instance has been added, ``spack find`` will automatically check the upstream instance when querying installed packages, and new package installations for the local Spack installation will use any dependencies that are installed in the upstream instance.\n\nThe upstream Spack instance has no knowledge of the local Spack instance and may not have the same permissions or ownership as the local Spack instance.\nThis has the following consequences:\n\n#. Upstream Spack instances are not locked.\n Therefore, it is up to users to make sure that the local instance is not using an upstream instance when it is being modified.\n\n#. Users should not uninstall packages from the upstream instance.\n Since the upstream instance does not know about the local instance, it cannot prevent the uninstallation of packages that the local instance depends on.\n\nOther details about upstream Spack installations:\n\n#. If a package is installed both locally and upstream, the local installation will always be used as a dependency.\n This can occur if the local Spack installs a package which is not present in the upstream, but later on the upstream Spack instance also installs that package.\n\n#. If an upstream Spack instance registers and installs an external package, the local Spack instance will treat this the same as a Spack-installed package.\n This feature will only work if the upstream Spack instance includes the upstream functionality (i.e., if its commit is after March 27, 2019).\n\nUsing Multiple Upstream Spack Instances\n---------------------------------------\n\nA single Spack instance can use multiple upstream Spack installations.\nSpack will search upstream instances in the order that you list them in your configuration.\nIf your Spack installation refers to instances X and Y, in that order, then instance X must list Y as an upstream in its own ``upstreams.yaml``.\n\nUsing Modules for Upstream Packages\n-----------------------------------\n\nThe local Spack instance does not generate modules for packages that are installed upstream.\nThe local Spack instance can be configured to use the modules generated by the upstream Spack instance.\n\nThere are two requirements to use the modules created by an upstream Spack instance: firstly, the upstream instance must do a ``spack module tcl refresh``, which generates an index file that maps installed packages to their modules; secondly, the local Spack instance must add a ``modules`` entry to the configuration:\n\n.. code-block:: yaml\n\n upstreams:\n spack-instance-1:\n install_tree: /path/to/other/spack/opt/spack\n modules:\n tcl: /path/to/other/spack/share/spack/modules\n\nEach time new packages are installed in the upstream Spack instance, the upstream Spack maintainer should run ``spack module tcl refresh`` (or the corresponding command for the type of module that they intend to use).\n\n.. note::\n\n Spack can generate modules that :ref:`automatically load ` the modules of dependency packages.\n Spack cannot currently do this for modules in upstream packages.\n" }, { "path": "lib/spack/docs/command_index.in", "content": "=================\nCommand Reference\n=================\n\nThis is a reference for all commands in the Spack command line interface.\nThe same information is available through :ref:`spack-help`.\n\nCommands that also have sections in the main documentation have a link to\n\"More documentation\".\n" }, { "path": "lib/spack/docs/conf.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n# flake8: noqa\n# -*- coding: utf-8 -*-\n#\n# Spack documentation build configuration file, created by\n# sphinx-quickstart on Mon Dec 9 15:32:41 2013.\n#\n# This file is execfile()d with the current directory set to its containing dir.\n#\n# Note that not all possible configuration values are present in this\n# autogenerated file.\n#\n# All configuration values have a default; values that are commented out\n# serve to show the default.\n\nimport os\nimport subprocess\nimport sys\nfrom glob import glob\nfrom typing import List\n\nfrom docutils.statemachine import StringList\nfrom pygments.formatters.html import HtmlFormatter\nfrom pygments.lexer import RegexLexer, default\nfrom pygments.token import *\nfrom sphinx.domains.python import PythonDomain\nfrom sphinx.ext.apidoc import main as sphinx_apidoc\nfrom sphinx.highlighting import PygmentsBridge\nfrom sphinx.parsers import RSTParser\n\n# -- Spack customizations -----------------------------------------------------\n# If extensions (or modules to document with autodoc) are in another directory,\n# add these directories to sys.path here. If the directory is relative to the\n# documentation root, use os.path.abspath to make it absolute, like shown here.\nlink_name = os.path.abspath(\"_spack_root\")\nif not os.path.exists(link_name):\n os.symlink(os.path.abspath(\"../../..\"), link_name, target_is_directory=True)\n\n# Add the Spack bin directory to the path so that we can use its output in docs.\nos.environ[\"SPACK_ROOT\"] = os.path.abspath(\"_spack_root\")\nos.environ[\"SPACK_USER_CONFIG_PATH\"] = os.path.abspath(\".spack\")\nos.environ[\"PATH\"] += os.pathsep + os.path.abspath(\"_spack_root/bin\")\n\n# Set an environment variable so that colify will print output like it would to\n# a terminal.\nos.environ[\"COLIFY_SIZE\"] = \"25x120\"\nos.environ[\"COLUMNS\"] = \"120\"\n\nsys.path[0:0] = [\n os.path.abspath(\"_spack_root/lib/spack/\"),\n os.path.abspath(\".spack/spack-packages/repos\"),\n]\n\n# Init the package repo with all git history, so \"Last updated on\" is accurate.\nsubprocess.call([\"spack\", \"repo\", \"update\"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\nif os.path.exists(\".spack/spack-packages/.git/shallow\"):\n subprocess.call(\n [\"git\", \"fetch\", \"--unshallow\"],\n cwd=\".spack/spack-packages\",\n stdout=subprocess.DEVNULL,\n stderr=subprocess.DEVNULL,\n )\n\n# Generate a command index if an update is needed\nsubprocess.call(\n [\n \"spack\",\n \"commands\",\n \"--format=rst\",\n \"--header=command_index.in\",\n \"--update=command_index.rst\",\n *glob(\"*rst\"),\n ]\n)\n\n#\n# Run sphinx-apidoc\n#\n# Remove any previous API docs\n# Read the Docs doesn't clean up after previous builds\n# Without this, the API Docs will never actually update\n#\napidoc_args = [\n \"--force\", # Overwrite existing files\n \"--no-toc\", # Don't create a table of contents file\n \"--output-dir=.\", # Directory to place all output\n \"--module-first\", # emit module docs before submodule docs\n]\nsphinx_apidoc(\n apidoc_args\n + [\n \"_spack_root/lib/spack/spack\",\n \"_spack_root/lib/spack/spack/vendor\",\n \"_spack_root/lib/spack/spack/test\",\n \"_spack_root/lib/spack/spack/package.py\",\n ]\n)\nsphinx_apidoc(\n apidoc_args\n + [\n \"--implicit-namespaces\",\n \".spack/spack-packages/repos/spack_repo\",\n \".spack/spack-packages/repos/spack_repo/builtin/packages\",\n \".spack/spack-packages/repos/spack_repo/builtin/build_systems/generic.py\",\n ]\n)\n\n\nclass NoWhitespaceHtmlFormatter(HtmlFormatter):\n \"\"\"HTML formatter that suppresses redundant span elements for Text.Whitespace tokens.\"\"\"\n\n def _get_css_classes(self, ttype):\n # For Text.Whitespace return an empty string, which avoids \n # elements from being generated.\n return \"\" if ttype is Text.Whitespace else super()._get_css_classes(ttype)\n\n\nclass CustomPygmentsBridge(PygmentsBridge):\n def get_formatter(self, **options):\n return NoWhitespaceHtmlFormatter(**options)\n\n\n# Use custom HTML formatter to avoid redundant elements.\n# See https://github.com/pygments/pygments/issues/1905#issuecomment-3170486995.\nPygmentsBridge.html_formatter = NoWhitespaceHtmlFormatter\n\n\nfrom spack.llnl.util.lang import classproperty\nfrom spack.spec_parser import SpecTokens\n\n# replace classproperty.__get__ to return `self` so Sphinx can document it correctly. Otherwise\n# it evaluates the callback, and it documents the result, which is not what we want.\nclassproperty.__get__ = lambda self, instance, owner: self\n\n\nclass SpecLexer(RegexLexer):\n \"\"\"A custom lexer for Spack spec strings and spack commands.\"\"\"\n\n name = \"Spack spec\"\n aliases = [\"spec\"]\n filenames = []\n tokens = {\n \"root\": [\n # Looks for `$ command`, which may need spec highlighting.\n (r\"^\\$\\s+\", Generic.Prompt, \"command\"),\n (r\"#.*?\\n\", Comment.Single),\n # Alternatively, we just get a literal spec string, so we move to spec mode. We just\n # look ahead here, without consuming the spec string.\n (r\"(?=\\S+)\", Generic.Prompt, \"spec\"),\n ],\n \"command\": [\n # A spack install command is followed by a spec string, which we highlight.\n (\n r\"spack(?:\\s+(?:-[eC]\\s+\\S+|--?\\S+))*\\s+(?:install|uninstall|spec|load|unload|find|info|list|versions|providers|mark|diff|add)(?: +(?:--?\\S+)?)*\",\n Text,\n \"spec\",\n ),\n # Comment\n (r\"\\s+#.*?\\n\", Comment.Single, \"command_output\"),\n # Escaped newline should leave us in this mode\n (r\".*?\\\\\\n\", Text),\n # Otherwise, it's the end of the command\n (r\".*?\\n\", Text, \"command_output\"),\n ],\n \"command_output\": [\n (r\"^\\$\\s+\", Generic.Prompt, \"#pop\"), # new command\n (r\"#.*?\\n\", Comment.Single), # comments\n (r\".*?\\n\", Generic.Output), # command output\n ],\n \"spec\": [\n # New line terminates the spec string\n (r\"\\s*?$\", Text, \"#pop\"),\n # Dependency, with optional virtual assignment specifier\n (SpecTokens.START_EDGE_PROPERTIES.regex, Name.Variable, \"edge_properties\"),\n (SpecTokens.DEPENDENCY.regex, Name.Variable),\n # versions\n (SpecTokens.VERSION_HASH_PAIR.regex, Keyword.Pseudo),\n (SpecTokens.GIT_VERSION.regex, Keyword.Pseudo),\n (SpecTokens.VERSION.regex, Keyword.Pseudo),\n # variants\n (SpecTokens.PROPAGATED_BOOL_VARIANT.regex, Name.Function),\n (SpecTokens.BOOL_VARIANT.regex, Name.Function),\n (SpecTokens.PROPAGATED_KEY_VALUE_PAIR.regex, Name.Function),\n (SpecTokens.KEY_VALUE_PAIR.regex, Name.Function),\n # filename\n (SpecTokens.FILENAME.regex, Text),\n # Package name\n (SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME.regex, Name.Class),\n (SpecTokens.UNQUALIFIED_PACKAGE_NAME.regex, Name.Class),\n # DAG hash\n (SpecTokens.DAG_HASH.regex, Text),\n (SpecTokens.WS.regex, Text),\n # Also stop at unrecognized tokens (without consuming them)\n default(\"#pop\"),\n ],\n \"edge_properties\": [\n (SpecTokens.KEY_VALUE_PAIR.regex, Name.Function),\n (SpecTokens.END_EDGE_PROPERTIES.regex, Name.Variable, \"#pop\"),\n ],\n }\n\n\n# Enable todo items\ntodo_include_todos = True\n\n\n#\n# Disable duplicate cross-reference warnings.\n#\nclass PatchedPythonDomain(PythonDomain):\n def resolve_xref(self, env, fromdocname, builder, type, target, node, contnode):\n if \"refspecific\" in node:\n del node[\"refspecific\"]\n return super().resolve_xref(env, fromdocname, builder, type, target, node, contnode)\n\n\n#\n# Disable tabs to space expansion in code blocks\n# since Makefiles require tabs.\n#\nclass NoTabExpansionRSTParser(RSTParser):\n def parse(self, inputstring, document):\n if isinstance(inputstring, str):\n lines = inputstring.splitlines()\n inputstring = StringList(lines, document.current_source)\n super().parse(inputstring, document)\n\n\ndef add_package_api_version_line(app, what, name: str, obj, options, lines: List[str]):\n \"\"\"Add versionadded directive to package API docstrings\"\"\"\n # We're adding versionadded directive here instead of in spack/package.py because most symbols\n # are re-exported, and we don't want to modify __doc__ of symbols we don't own.\n if name.startswith(\"spack.package.\"):\n symbol = name[len(\"spack.package.\") :]\n for version, symbols in spack.package.api.items():\n if symbol in symbols:\n lines.extend([\"\", f\".. versionadded:: {version}\"])\n break\n\n\ndef skip_member(app, what, name, obj, skip, options):\n # Do not skip (Make)Executable.__call__\n if name == \"__call__\" and \"Executable\" in obj.__qualname__:\n return False\n return skip\n\n\ndef setup(sphinx):\n # autodoc-process-docstring\n sphinx.connect(\"autodoc-process-docstring\", add_package_api_version_line)\n sphinx.connect(\"autodoc-skip-member\", skip_member)\n sphinx.add_domain(PatchedPythonDomain, override=True)\n sphinx.add_source_parser(NoTabExpansionRSTParser, override=True)\n sphinx.add_lexer(\"spec\", SpecLexer)\n\n\n# -- General configuration -----------------------------------------------------\n\n# If your documentation needs a minimal Sphinx version, state it here.\nneeds_sphinx = \"3.4\"\n\n# Add any Sphinx extension module names here, as strings. They can be extensions\n# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.\nextensions = [\n \"sphinx.ext.autodoc\",\n \"sphinx.ext.graphviz\",\n \"sphinx.ext.intersphinx\",\n \"sphinx.ext.napoleon\",\n \"sphinx.ext.todo\",\n \"sphinx.ext.viewcode\",\n \"sphinx_copybutton\",\n \"sphinx_last_updated_by_git\",\n \"sphinx_sitemap\",\n \"sphinxcontrib.inkscapeconverter\",\n \"sphinxcontrib.programoutput\",\n]\n\ncopybutton_exclude = \".linenos, .gp, .go\"\n\n# Set default graphviz options\ngraphviz_dot_args = [\n \"-Grankdir=LR\",\n \"-Gbgcolor=transparent\",\n \"-Nshape=box\",\n \"-Nfontname=monaco\",\n \"-Nfontsize=10\",\n]\n\n# Get nice vector graphics\ngraphviz_output_format = \"svg\"\n\n# Add any paths that contain templates here, relative to this directory.\ntemplates_path = [\"_templates\"]\n\n# The suffix of source filenames.\nsource_suffix = \".rst\"\n\n# The encoding of source files.\nsource_encoding = \"utf-8-sig\"\n\n# The master toctree document.\nmaster_doc = \"index\"\n\n# General information about the project.\nproject = \"Spack\"\ncopyright = \"Spack Project Developers\"\n\n# The version info for the project you're documenting, acts as replacement for\n# |version| and |release|, also used in various other places throughout the\n# built documents.\n#\n# The short X.Y version.\nimport spack\nimport spack.package\n\nversion = \".\".join(str(s) for s in spack.spack_version_info[:2])\n# The full version, including alpha/beta/rc tags.\nrelease = spack.spack_version\n\n# The language for content autogenerated by Sphinx. Refer to documentation\n# for a list of supported languages.\n# language = None\n\n# Places to look for .po/.mo files for doc translations\n# locale_dirs = []\n\n# Sphinx gettext settings\ngettext_compact = True\ngettext_uuid = False\n\n# There are two options for replacing |today|: either, you set today to some\n# non-false value, then it is used:\n# today = ''\n# Else, today_fmt is used as the format for a strftime call.\n# today_fmt = '%B %d, %Y'\n\n# List of patterns, relative to source directory, that match files and\n# directories to ignore when looking for source files.\nexclude_patterns = [\"_build\", \"_spack_root\", \".spack-env\", \".spack\", \".venv\"]\n\nautodoc_mock_imports = [\"llnl\"]\nautodoc_default_options = {\"no-value\": True}\nautodoc_preserve_defaults = True\n\nnitpicky = True\nnitpick_ignore = [\n # Python classes that intersphinx is unable to resolve\n (\"py:class\", \"argparse.HelpFormatter\"),\n (\"py:class\", \"concurrent.futures._base.Executor\"),\n (\"py:class\", \"hashlib._Hash\"),\n (\"py:class\", \"multiprocessing.context.BaseContext\"),\n (\"py:class\", \"posix.DirEntry\"),\n # Spack classes that are private and we don't want to expose\n (\"py:class\", \"spack_repo.builtin.build_systems._checks.BuilderWithDefaults\"),\n (\"py:class\", \"spack.repo._PrependFileLoader\"),\n # Spack classes that intersphinx is unable to resolve\n (\"py:class\", \"GitOrStandardVersion\"),\n (\"py:class\", \"spack.bootstrap._common.QueryInfo\"),\n (\"py:class\", \"spack.filesystem_view.SimpleFilesystemView\"),\n (\"py:class\", \"spack.spec.ArchSpec\"),\n (\"py:class\", \"spack.spec.DependencySpec\"),\n (\"py:class\", \"spack.spec.InstallStatus\"),\n (\"py:class\", \"spack.spec.SpecfileReaderBase\"),\n (\"py:class\", \"spack.traverse.EdgeAndDepth\"),\n (\"py:class\", \"spack.vendor.archspec.cpu.microarchitecture.Microarchitecture\"),\n (\"py:class\", \"spack.vendor.jinja2.Environment\"),\n (\"py:class\", \"SpecFiltersFactory\"),\n # TypeVar that is not handled correctly\n (\"py:class\", \"spack.llnl.util.lang.ClassPropertyType\"),\n (\"py:class\", \"spack.llnl.util.lang.K\"),\n (\"py:class\", \"spack.llnl.util.lang.KT\"),\n (\"py:class\", \"spack.llnl.util.lang.T\"),\n (\"py:class\", \"spack.llnl.util.lang.V\"),\n (\"py:class\", \"spack.llnl.util.lang.VT\"),\n (\"py:obj\", \"spack.llnl.util.lang.ClassPropertyType\"),\n (\"py:obj\", \"spack.llnl.util.lang.K\"),\n (\"py:obj\", \"spack.llnl.util.lang.KT\"),\n (\"py:obj\", \"spack.llnl.util.lang.V\"),\n (\"py:obj\", \"spack.llnl.util.lang.VT\"),\n (\"py:class\", \"_P\"),\n (\"py:class\", \"spack.util.web._R\"),\n]\n\n# The reST default role (used for this markup: `text`) to use for all documents.\n# default_role = None\n\n# If true, '()' will be appended to :func: etc. cross-reference text.\n# add_function_parentheses = True\n\n# If true, the current module name will be prepended to all description\n# unit titles (such as .. function::).\n# add_module_names = True\n\n# If true, sectionauthor and moduleauthor directives will be shown in the\n# output. They are ignored by default.\n# show_authors = False\n\n# A list of ignored prefixes for module index sorting.\n# modindex_common_prefix = []\n\n\n# -- Options for HTML output ---------------------------------------------------\n\n# The theme to use for HTML and HTML Help pages. See the documentation for\n# a list of builtin themes.\nhtml_theme = \"furo\"\n\n# Add any paths that contain custom themes here, relative to this directory.\n# html_theme_path = [\"_themes\"]\n\n# Google Search Console verification file\nhtml_extra_path = [\"google5fda5f94b4ffb8de.html\"]\n\n# The name for this set of Sphinx documents. If None, it defaults to\n# \" v documentation\".\n# html_title = None\n\n# A shorter title for the navigation bar. Default is the same as html_title.\n# html_short_title = None\n\n# The name of an image file (relative to this directory) to place at the top\n# of the sidebar.\nhtml_theme_options = {\n \"sidebar_hide_name\": True,\n \"light_logo\": \"spack-logo-text.svg\",\n \"dark_logo\": \"spack-logo-white-text.svg\",\n}\n\n# The name of an image file (within the static path) to use as favicon of the\n# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32\n# pixels large.\nhtml_favicon = \"_spack_root/share/spack/logo/favicon.ico\"\n\n# Add any paths that contain custom static files (such as style sheets) here,\n# relative to this directory. They are copied after the builtin static files,\n# so a file named \"default.css\" will overwrite the builtin \"default.css\".\nhtml_static_path = [\"_static\"]\n\n# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,\n# using the given strftime format.\nhtml_last_updated_fmt = \"%b %d, %Y\"\npygments_style = \"default\"\npygments_dark_style = \"monokai\"\n\n# If true, SmartyPants will be used to convert quotes and dashes to\n# typographically correct entities.\n# html_use_smartypants = True\n\n# Custom sidebar templates, maps document names to template names.\n# html_sidebars = {}\n\n# Additional templates that should be rendered to pages, maps page names to\n# template names.\n# html_additional_pages = {}\n\n# If false, no module index is generated.\n# html_domain_indices = True\n\n# If false, no index is generated.\n# html_use_index = True\n\n# If true, the index is split into individual pages for each letter.\n# html_split_index = False\n\n# If true, links to the reST sources are added to the pages.\n# html_show_sourcelink = True\n\n# If true, \"Created using Sphinx\" is shown in the HTML footer. Default is True.\nhtml_show_sphinx = False\n\n# If true, \"(C) Copyright ...\" is shown in the HTML footer. Default is True.\n# html_show_copyright = True\n\n# If true, an OpenSearch description file will be output, and all pages will\n# contain a tag referring to it. The value of this option must be the\n# base URL from which the finished HTML is served.\n# html_use_opensearch = ''\n\n# This is the file name suffix for HTML files (e.g. \".xhtml\").\n# html_file_suffix = None\n\n# Base URL for the documentation, used to generate for better indexing\nhtml_baseurl = \"https://spack.readthedocs.io/en/latest/\"\n\n# Output file base name for HTML help builder.\nhtmlhelp_basename = \"Spackdoc\"\n\n# Sitemap settings\nsitemap_show_lastmod = True\nsitemap_url_scheme = \"{link}\"\nsitemap_excludes = [\"search.html\", \"_modules/*\"]\n\n# -- Options for LaTeX output --------------------------------------------------\n\nlatex_engine = \"lualatex\"\n\nlatex_elements = {\n # The paper size ('letterpaper' or 'a4paper').\n # 'papersize': 'letterpaper',\n # The font size ('10pt', '11pt' or '12pt').\n # 'pointsize': '10pt',\n # Additional stuff for the LaTeX preamble.\n # 'preamble': '',\n}\n\n# Grouping the document tree into LaTeX files. List of tuples\n# (source start file, target name, title, author, documentclass [howto/manual]).\nlatex_documents = [(\"index\", \"Spack.tex\", \"Spack Documentation\", \"\", \"manual\")]\n\n# The name of an image file (relative to this directory) to place at the top of\n# the title page.\n# latex_logo = None\n\n# For \"manual\" documents, if this is true, then toplevel headings are parts,\n# not chapters.\n# latex_use_parts = False\n\n# If true, show page references after internal links.\n# latex_show_pagerefs = False\n\n# If true, show URL addresses after external links.\n# latex_show_urls = False\n\n# Documents to append as an appendix to all manuals.\n# latex_appendices = []\n\n# If false, no module index is generated.\n# latex_domain_indices = True\n\n\n# -- Options for manual page output --------------------------------------------\n\n# One entry per manual page. List of tuples\n# (source start file, name, description, authors, manual section).\nman_pages = [(\"index\", \"spack\", \"Spack Documentation\", [\"Todd Gamblin\"], 1)]\n\n# If true, show URL addresses after external links.\n# man_show_urls = False\n\n\n# -- Options for Texinfo output ------------------------------------------------\n\n# Grouping the document tree into Texinfo files. List of tuples\n# (source start file, target name, title, author,\n# dir menu entry, description, category)\ntexinfo_documents = [\n (\n \"index\",\n \"Spack\",\n \"Spack Documentation\",\n \"Todd Gamblin\",\n \"Spack\",\n \"One line description of project.\",\n \"Miscellaneous\",\n )\n]\n\n# Documents to append as an appendix to all manuals.\n# texinfo_appendices = []\n\n# If false, no module index is generated.\n# texinfo_domain_indices = True\n\n# How to display URL addresses: 'footnote', 'no', or 'inline'.\n# texinfo_show_urls = 'footnote'\n\n\n# -- Extension configuration -------------------------------------------------\n\n# sphinx.ext.intersphinx\nintersphinx_mapping = {\"python\": (\"https://docs.python.org/3\", None)}\n\nrst_epilog = f\"\"\"\n.. |package_api_version| replace:: v{spack.package_api_version[0]}.{spack.package_api_version[1]}\n.. |min_package_api_version| replace:: v{spack.min_package_api_version[0]}.{spack.min_package_api_version[1]}\n.. |spack_version| replace:: {spack.spack_version}\n\"\"\"\n\nhtml_static_path = [\"_static\"]\nhtml_css_files = [\"css/custom.css\"]\nhtml_context = {}\n\nif os.environ.get(\"READTHEDOCS\", \"\") == \"True\":\n html_context[\"READTHEDOCS\"] = True\n" }, { "path": "lib/spack/docs/config_yaml.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. _config-yaml:\n\n.. meta::\n :description lang=en:\n A detailed guide to the config.yaml file in Spack, which allows you to set core configuration options like installation paths, build parallelism, and trusted sources.\n\nSpack Settings (config.yaml)\n============================\n\nSpack's basic configuration options are set in ``config.yaml``.\nYou can see the default settings by looking at ``etc/spack/defaults/config.yaml``:\n\n.. literalinclude:: _spack_root/etc/spack/defaults/base/config.yaml\n :language: yaml\n\nThese settings can be overridden in ``etc/spack/config.yaml`` or ``~/.spack/config.yaml``.\nSee :ref:`configuration-scopes` for details.\n\n``install_tree:root``\n---------------------\n\nThe location where Spack will install packages and their dependencies.\nThe default is ``$spack/opt/spack``.\n\n``projections``\n---------------\n\n.. warning::\n\n Modifying projections of the install tree is strongly discouraged.\n\nBy default, Spack installs all packages into a unique directory relative to the install tree root with the following layout:\n\n.. code-block:: text\n\n {architecture.platform}-{architecture.target}/{name}-{version}-{hash}\n\nIn very rare cases, it may be necessary to reduce the length of this path.\nFor example, very old versions of the Intel compiler are known to segfault when input paths are too long:\n\n.. code-block:: console\n\n : internal error: ** The compiler has encountered an unexpected problem.\n ** Segmentation violation signal raised. **\n Access violation or stack overflow. Please contact Intel Support for assistance.\n\nAnother case is Python and R packages with many runtime dependencies, which can result in very large ``PYTHONPATH`` and ``R_LIBS`` environment variables.\nThis can cause the ``execve`` system call to fail with ``E2BIG``, preventing processes from starting.\n\nFor this reason, Spack allows users to modify the installation layout through custom projections.\nFor example:\n\n.. code-block:: yaml\n\n config:\n install_tree:\n root: $spack/opt/spack\n projections:\n all: \"{name}/{version}/{hash:16}\"\n\nwould install packages into subdirectories using only the package name, version, and a hash length of 16 characters.\n\nNotice that reducing the hash length increases the likelihood of hash collisions.\n\n``build_stage``\n--------------------\n\nSpack is designed to run from a user home directory, and on many systems, the home directory is a (slow) network file system.\nOn most systems, building in a temporary file system is faster.\nUsually, there is also more space available in the temporary location than in the home directory.\nIf the username is not already in the path, Spack will append the value of ``$user`` to the selected ``build_stage`` path.\n\n.. warning::\n We highly recommend specifying ``build_stage`` paths that distinguish between staging and other activities to ensure ``spack clean`` does not inadvertently remove unrelated files.\n Spack prepends ``spack-stage-`` to temporary staging directory names to reduce this risk.\n Using a combination of ``spack`` and or ``stage`` in each specified path, as shown in the default settings and documented examples, will add another layer of protection.\n\nBy default, Spack's ``build_stage`` is configured like this:\n\n.. code-block:: yaml\n\n build_stage:\n - $tempdir/$user/spack-stage\n - ~/.spack/stage\n\nThis can be an ordered list of paths that Spack should search when trying to find a temporary directory for the build stage.\nThe list is searched in order, and Spack will use the first directory to which it has write access.\n\nSpecifying `~/.spack/stage` first will ensure each user builds in their home directory.\nThe historic Spack stage path `$spack/var/spack/stage` will build directly inside the Spack instance.\nSee :ref:`config-file-variables` for more on ``$tempdir`` and ``$spack``.\n\nWhen Spack builds a package, it creates a temporary directory within the ``build_stage``.\nAfter the package is successfully installed, Spack deletes the temporary directory it used to build.\nUnsuccessful builds are not deleted, but you can manually purge them with ``spack clean --stage``.\n\n.. note::\n\n The build will fail if there is no writable directory in the ``build_stage`` list, where any user- and site-specific setting will be searched first.\n\n``source_cache``\n--------------------\n\nLocation to cache downloaded tarballs and repositories.\nBy default, these are stored in ``$spack/var/spack/cache``.\nThese are stored indefinitely by default and can be purged with ``spack clean --downloads``.\n\n.. _Misc Cache:\n\n``misc_cache``\n--------------------\n\nTemporary directory to store long-lived cache files, such as indices of packages available in repositories.\nDefaults to ``~/.spack/cache``.\nCan be purged with ``spack clean --misc-cache``.\n\nIn some cases, e.g., if you work with many Spack instances or many different versions of Spack, it makes sense to have a cache per instance or per version.\nYou can do that by changing the value to either:\n\n* ``~/.spack/$spack_instance_id/cache`` for per-instance caches, or\n* ``~/.spack/$spack_short_version/cache`` for per-spack-version caches.\n\n``verify_ssl``\n--------------------\n\nWhen set to ``true`` (default), Spack will verify certificates of remote hosts when making ``ssl`` connections.\nSet to ``false`` to disable, and tools like ``curl`` will use their ``--insecure`` options.\nDisabling this can expose you to attacks.\nUse at your own risk.\n\n``ssl_certs``\n--------------------\n\nPath to custom certificates for SSL verification.\nThe value can be a filesystem path, or an environment variable that expands to an absolute file path.\nThe default value is set to the environment variable ``SSL_CERT_FILE`` to use the same syntax used by many other applications that automatically detect custom certificates.\nWhen ``url_fetch_method:curl``, the ``config:ssl_certs`` should resolve to a single file.\nSpack will then set the environment variable ``CURL_CA_BUNDLE`` in the subprocess calling ``curl``.\nIf additional ``curl`` arguments are required, they can be set in the config, e.g., ``url_fetch_method:'curl -k -q'``.\nIf ``url_fetch_method:urllib``, then files and directories are supported, i.e., ``config:ssl_certs:$SSL_CERT_FILE`` or ``config:ssl_certs:$SSL_CERT_DIR`` will work.\nIn all cases, the expanded path must be absolute for Spack to use the certificates.\nCertificates relative to an environment can be created by prepending the path variable with the Spack configuration variable ``$env``.\n\n``checksum``\n--------------------\n\nWhen set to ``true``, Spack verifies downloaded source code using a checksum and will refuse to build packages that it cannot verify.\nSet to ``false`` to disable these checks.\nDisabling this can expose you to attacks.\nUse at your own risk.\n\n``locks``\n--------------------\n\nWhen set to ``true``, concurrent instances of Spack will use locks to avoid modifying the install tree, database file, etc.\nIf ``false``, Spack will disable all locking, but you must **not** run concurrent instances of Spack.\nFor file systems that do not support locking, you should set this to ``false`` and run one Spack instance at a time; otherwise, we recommend enabling locks.\n\n``dirty``\n--------------------\n\nBy default, Spack unsets variables in your environment that can change the way packages build.\nThis includes ``LD_LIBRARY_PATH``, ``CPATH``, ``LIBRARY_PATH``, ``DYLD_LIBRARY_PATH``, and others.\n\nBy default, builds are ``clean``, but on some machines, compilers and other tools may need custom ``LD_LIBRARY_PATH`` settings to run.\nYou can set ``dirty`` to ``true`` to skip the cleaning step and make all builds \"dirty\" by default.\nBe aware that this will reduce the reproducibility of builds.\n\n.. _build-jobs:\n\n``build_jobs``\n--------------\n\nUnless overridden in a package or on the command line, Spack builds all packages in parallel.\nThe default parallelism is equal to the number of cores available to the process, up to 16 (the default of ``build_jobs``).\nFor a build system that uses Makefiles, ``spack install`` runs:\n\n- ``make -j``, when ``build_jobs`` is less than the number of cores available\n- ``make -j``, when ``build_jobs`` is greater or equal to the number of cores available\n\nIf you work on a shared login node or have a strict ulimit, it may be necessary to set the default to a lower value.\nBy setting ``build_jobs`` to 4, for example, commands like ``spack install`` will run ``make -j4`` instead of using every core.\nTo build all software in serial, set ``build_jobs`` to 1.\n\nNote that specifying the number of jobs on the command line always takes priority, so that ``spack install -j`` always runs ``make -j``, even when that exceeds the number of cores available.\n\n``ccache``\n--------------------\n\nWhen set to ``true``, Spack will use ccache to cache compiles.\nThis is useful specifically in two cases: (1) when using ``spack dev-build`` and (2) when building the same package with many different variants.\nThe default is ``false``.\n\nWhen enabled, Spack will look inside your ``PATH`` for a ``ccache`` executable and stop if it is not found.\nSome systems come with ``ccache``, but it can also be installed using ``spack install ccache``.\n``ccache`` comes with reasonable defaults for cache size and location.\n(See the *Configuration settings* section of ``man ccache`` to learn more about the default settings and how to change them.)\nPlease note that we currently disable ccache's ``hash_dir`` feature to avoid an issue with the stage directory (see https://github.com/spack/spack/pull/3761#issuecomment-294352232).\n\n``shared_linking:type``\n-----------------------\n\nControls whether Spack embeds ``RPATH`` or ``RUNPATH`` attributes in ELF binaries so that they can find their dependencies.\nThis has no effect on macOS.\nTwo options are allowed:\n\n1. ``rpath`` uses ``RPATH`` and forces the ``--disable-new-tags`` flag to be passed to the linker.\n2. ``runpath`` uses ``RUNPATH`` and forces the ``--enable-new-tags`` flag to be passed to the linker.\n\n``RPATH`` search paths have higher precedence than ``LD_LIBRARY_PATH``, and ``ld.so`` will search for libraries in transitive RPATHs of parent objects.\n\n``RUNPATH`` search paths have lower precedence than ``LD_LIBRARY_PATH``, and ``ld.so`` will ONLY search for dependencies in the ``RUNPATH`` of the loading object.\n\nDO NOT MIX the two options within the same install tree.\n\n``shared_linking:bind``\n-----------------------\n\nThis is an *experimental option* that controls whether Spack embeds absolute paths to needed shared libraries in ELF executables and shared libraries on Linux.\nSetting this option to ``true`` has two advantages:\n\n1. **Improved startup time**: when running an executable, the dynamic loader does not have to search for needed libraries.\n They are loaded directly.\n2. **Reliability**: libraries loaded at runtime are those that were linked during the build.\n This minimizes the risk of accidentally picking up system libraries.\n\nIn the current implementation, Spack sets the soname (shared object name) of libraries to their install path upon installation.\nThis has two implications:\n\n1. Binding does not apply to libraries installed *before* the option was enabled.\n2. Disabling the option does *not* prevent binding of libraries installed when the option was still enabled.\n\nIt is also worth noting that:\n\n1. Applications relying on ``dlopen(3)`` will continue to work, even when they open a library by name.\n This is because RPATHs are retained in binaries also when ``bind`` is enabled.\n2. ``LD_PRELOAD`` continues to work for the typical use case of overriding symbols, such as preloading a library with a more efficient ``malloc``.\n However, the preloaded library will be loaded *in addition to*, rather than *in place of*, another library with the same name -- which can be problematic in rare cases where libraries rely on a particular ``init`` or ``fini`` order.\n\n.. note::\n\n In some cases, packages provide *stub libraries* that only contain an interface for linking but lack an implementation for runtime.\n An example of this is ``libcuda.so``, provided by the CUDA toolkit; it can be used to link against, but the library needed at runtime is the one installed with the CUDA driver.\n To avoid binding those libraries, they can be marked as non-bindable using a property in the package:\n\n .. code-block:: python\n\n class Example(Package):\n non_bindable_shared_objects = [\"libinterface.so\"]\n\n``install_status``\n----------------------\n\nWhen set to ``true``, Spack will show information about its current progress as well as the current and total package numbers.\nProgress is shown both in the terminal title and inline.\nSetting it to ``false`` will not show any progress information.\n\nTo work properly, this requires your terminal to reset its title after Spack has finished its work; otherwise, Spack's status information will remain in the terminal's title indefinitely.\nMost terminals should already be set up this way and clear Spack's status information.\n\n``aliases``\n-----------\n\nAliases can be used to define new Spack commands.\nThey can be either shortcuts for longer commands or include specific arguments for convenience.\nFor instance, if users want to use ``spack install``'s ``-v`` argument all the time, they can create a new alias called ``inst`` that will always call ``install -v``:\n\n.. code-block:: yaml\n\n aliases:\n inst: install -v\n\n``concretization_cache:enable``\n-------------------------------\n\nWhen set to ``true``, Spack will utilize a cache of solver outputs from successful concretization runs.\nWhen enabled, Spack will check the concretization cache prior to running the solver.\nIf a previous request to solve a given problem is present in the cache, Spack will load the concrete specs and other solver data from the cache rather than running the solver.\nSpecs not previously concretized will be added to the cache on a successful solve.\nThe cache additionally holds solver statistics, so commands like ``spack solve`` will still return information about the run that produced a given solver result.\n\nThis cache is a subcache of the :ref:`Misc Cache` and as such will be cleaned when the Misc Cache is cleaned.\n\nWhen ``false`` or omitted, all concretization requests will be performed from scratch\n\n``concretization_cache:url``\n----------------------------\n\nPath to the location where Spack will root the concretization cache.\nCurrently this only supports paths on the local filesystem.\n\nDefault location is under the :ref:`Misc Cache` at: ``$misc_cache/concretization``\n\n``concretization_cache:entry_limit``\n------------------------------------\n\nSets a limit on the number of concretization results that Spack will cache.\nThe limit is evaluated after each concretization run; if Spack has stored more results than the limit allows, the oldest concretization results are pruned until 10% of the limit has been removed.\n\nSetting this value to 0 disables automatic pruning.\nIt is expected that users will be responsible for maintaining this cache.\n\n``concretization_cache:size_limit``\n-----------------------------------\n\nSets a limit on the size of the concretization cache in bytes.\nThe limit is evaluated after each concretization run; if Spack has stored more results than the limit allows, the oldest concretization results are pruned until 10% of the limit has been removed.\n\nSetting this value to 0 disables automatic pruning.\nIt is expected that users will be responsible for maintaining this cache.\n" }, { "path": "lib/spack/docs/configuration.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Learn how to configure Spack using its flexible YAML-based system. This guide covers the different configuration scopes and provides links to detailed documentation for each configuration file, helping you customize Spack to your specific needs.\n\n.. _configuration:\n\nConfiguration Files\n===================\n\nSpack has many configuration files.\nHere is a quick list of them, in case you want to skip directly to specific docs:\n\n* :ref:`concretizer.yaml `\n* :ref:`config.yaml `\n* :ref:`include.yaml `\n* :ref:`mirrors.yaml `\n* :ref:`modules.yaml `\n* :ref:`packages.yaml ` (including :ref:`compiler configuration `)\n* :ref:`repos.yaml `\n* :ref:`toolchains.yaml `\n\nYou can also add any of these as inline configuration in the YAML manifest file (``spack.yaml``) describing an :ref:`environment `.\n\nYAML Format\n-----------\n\nSpack configuration files are written in YAML.\nWe chose YAML because it's human-readable but also versatile in that it supports dictionaries, lists, and nested sections.\nFor more details on the format, see `yaml.org `_.\nHere is an example ``config.yaml`` file:\n\n.. code-block:: yaml\n\n config:\n install_tree:\n root: $spack/opt/spack\n build_stage:\n - $tempdir/$user/spack-stage\n - ~/.spack/stage\n\nEach Spack configuration file is nested under a top-level section corresponding to its name.\nSo, ``config.yaml`` starts with ``config:``, ``mirrors.yaml`` starts with ``mirrors:``, etc.\n\n.. tip::\n\n Validation and autocompletion of Spack config files can be enabled in your editor using `JSON Schema Store `_.\n\n.. _configuration-scopes:\n\nConfiguration Scopes\n--------------------\n\nSpack pulls configuration data from files in several directories.\nThere are multiple configuration scopes.\nFrom lowest to highest precedence:\n\n#. **defaults**: Stored in ``$(prefix)/etc/spack/defaults/``.\n These are the \"factory\" settings.\n Users should generally not modify the settings here, but should override them in other configuration scopes.\n The defaults here will change from version to version of Spack.\n\n#. **system**: Stored in ``/etc/spack/``.\n These are settings for this machine or for all machines on which this file system is mounted.\n The system scope overrides the defaults scope.\n It can be used for settings idiosyncratic to a particular machine, such as the locations of compilers or external packages.\n Be careful when modifying this scope, as changes here affect all Spack users on a machine.\n Before putting configuration here, instead consider using the ``site`` scope, which only affects the spack instance it's part of.\n\n#. **site**: Stored in ``$(prefix)/etc/spack/site/``.\n Settings here affect only *this instance* of Spack, and they override the defaults and system scopes.\n The site scope is intended for site-wide settings on multi-user machines (e.g., for a common Spack instance).\n\n#. **plugin**: Read from a Python package's entry points.\n Settings here affect all instances of Spack running with the same Python installation.\n This scope takes higher precedence than site, system, and default scopes.\n\n#. **user**: Stored in the home directory: ``~/.spack/``.\n These settings affect all instances of Spack and take higher precedence than site, system, plugin, or defaults scopes.\n\n#. **spack**: Stored in ``$(prefix)/etc/spack/``.\n Settings here affect only *this instance* of Spack, and they override ``user`` and lower configuration scopes.\n This is intended for project-specific or single-user spack installations.\n This is the topmost built-in spack scope, and modifying it gives you full control over configuration scopes.\n For example, it defines the ``user``, ``site``, and ``system`` scopes, so you can use it to remove them completely if you want.\n\n#. **environment**: When using Spack :ref:`environments`, Spack reads additional configuration from the environment file.\n See :ref:`environment-configuration` for further details on these scopes.\n Environment scopes can be referenced from the command line as ``env:name`` (e.g., to reference environment ``foo``, use ``env:foo``).\n\n#. **custom**: Stored in a custom directory specified by ``--config-scope``.\n If multiple scopes are listed on the command line, they are ordered from lowest to highest precedence.\n\n#. **command line**: Build settings specified on the command line take precedence over all other scopes.\n\nEach configuration directory may contain several configuration files, such as ``config.yaml``, ``packages.yaml``, or ``mirrors.yaml``.\nWhen configurations conflict, settings from higher-precedence scopes override lower-precedence settings.\n\nAll of these except ``spack`` and ``defaults`` are initially empty, so you don't have to think about the others unless you need them.\nThe most commonly used scopes are ``environment``, ``user``, and ``spack``.\nIf you forget, you can always see the available configuration scopes in order of precedence with the ``spack config scopes`` command::\n\n > spack config scopes -p\n Scope Path\n command_line\n spack /home/username/spack/etc/spack\n user /home/username/.spack/\n site /home/username/spack/etc/spack/site/\n defaults /home/username/spack/etc/spack/defaults/\n defaults:darwin /home/username/spack/etc/spack/defaults/darwin/\n defaults:base /home/username/spack/etc/spack/defaults/base/\n _builtin\n\nCommands that modify scopes (e.g., ``spack compilers``, ``spack repo``, ``spack external find``, etc.) take a ``--scope=`` parameter that you can use to control which scope is modified.\nBy default, they modify the highest-precedence available scope that is not read-only (like `defaults`).\n\n.. _custom-scopes:\n\nCustom scopes\n^^^^^^^^^^^^^\n\nYou may add configuration scopes directly on the command line with the ``--config-scope`` argument, or ``-C`` for short.\nCustom command-line scopes override any active environments, as well as the ``defaults``, ``system``, ``site``, ``user``, and ``spack`` scopes,\n\nFor example, the following adds two configuration scopes, named ``scope-a`` and ``scope-b``, to a ``spack spec`` command:\n\n.. code-block:: spec\n\n $ spack -C ~/myscopes/scope-a -C ~/myscopes/scope-b spec ncurses\n\nCustom scopes come *after* the ``spack`` command and *before* the subcommand, and they specify a single path to a directory containing configuration files.\nYou can add the same configuration files to that directory that you can add to any other scope (e.g., ``config.yaml``, ``packages.yaml``, etc.).\n\nIf multiple scopes are provided:\n\n#. Each must be preceded with the ``--config-scope`` or ``-C`` flag.\n#. They must be ordered from lowest to highest precedence.\n\nExample: scopes for release and development\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nSuppose that you need to support simultaneous building of release and development versions of ``mypackage``, where ``mypackage`` depends on ``pkg-a``, which in turn depends on ``pkg-b``.\nYou could create the following files:\n\n.. code-block:: yaml\n :caption: ``~/myscopes/release/packages.yaml``\n :name: code-example-release-packages-yaml\n\n packages:\n mypackage:\n prefer: [\"@1.7\"]\n pkg-a:\n prefer: [\"@2.3\"]\n pkg-b:\n prefer: [\"@0.8\"]\n\n.. code-block:: yaml\n :caption: ``~/myscopes/develop/packages.yaml``\n :name: code-example-develop-packages-yaml\n\n packages:\n mypackage:\n prefer: [\"@develop\"]\n pkg-a:\n prefer: [\"@develop\"]\n pkg-b:\n prefer: [\"@develop\"]\n\nYou can switch between ``release`` and ``develop`` configurations using configuration arguments.\nYou would type ``spack -C ~/myscopes/release`` when you want to build the designated release versions of ``mypackage``, ``pkg-a``, and ``pkg-b``, and you would type ``spack -C ~/myscopes/develop`` when you want to build all of these packages at the ``develop`` version.\n\nExample: swapping MPI providers\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nSuppose that you need to build two software packages, ``pkg-a`` and ``pkg-b``.\nFor ``pkg-b`` you want a newer Python version and a different MPI implementation than for ``pkg-a``.\nYou can create different configuration scopes for use with ``pkg-a`` and ``pkg-b``:\n\n.. code-block:: yaml\n :caption: ``~/myscopes/pkg-a/packages.yaml``\n :name: code-example-pkg-a-packages-yaml\n\n packages:\n python:\n require: [\"@3.11\"]\n mpi:\n require: [openmpi]\n\n.. code-block:: yaml\n :caption: ``~/myscopes/pkg-b/packages.yaml``\n :name: code-example-pkg-b-packages-yaml\n\n packages:\n python:\n require: [\"@3.13\"]\n mpi:\n require: [mpich]\n\n\n.. _plugin-scopes:\n\nPlugin scopes\n^^^^^^^^^^^^^\n\n.. note::\n Python version >= 3.8 is required to enable plugin configuration.\n\nSpack can be made aware of configuration scopes that are installed as part of a Python package.\nTo do so, register a function that returns the scope's path to the ``\"spack.config\"`` entry point.\nConsider the Python package ``my_package`` that includes Spack configurations:\n\n.. code-block:: console\n\n my-package/\n ├── src\n │ ├── my_package\n │ │ ├── __init__.py\n │ │ └── spack/\n │ │ │ └── config.yaml\n └── pyproject.toml\n\nAdding the following to ``my_package``'s ``pyproject.toml`` will make ``my_package``'s ``spack/`` configurations visible to Spack when ``my_package`` is installed:\n\n.. code-block:: toml\n\n [project.entry_points.\"spack.config\"]\n my_package = \"my_package:get_config_path\"\n\nThe function ``my_package.get_config_path`` (matching the entry point definition) in ``my_package/__init__.py`` might look like:\n\n.. code-block:: python\n\n import importlib.resources\n\n\n def get_config_path():\n dirname = importlib.resources.files(\"my_package\").joinpath(\"spack\")\n if dirname.exists():\n return str(dirname)\n\n.. _platform-scopes:\n\nPlatform-specific Configuration\n-------------------------------\n\n.. warning::\n\n Prior to v1.0, each scope above -- except environment scopes -- had a corresponding platform-specific scope (e.g., ``defaults/linux``, ``system/windows``).\n This can now be accomplished through a suitably placed :ref:`include.yaml ` file.\n\nThere is often a need for platform-specific configuration settings.\nFor example, on most platforms, GCC is the preferred compiler.\nHowever, on macOS (darwin), Clang often works for more packages, and is set as the default compiler.\nThis configuration is set in ``$(prefix)/etc/spack/defaults/darwin/packages.yaml``, which is included by ``$(prefix)/etc/spack/defaults/include.yaml``.\nSince it is an included configuration of the ``defaults`` scope, settings in the ``defaults`` scope will take precedence.\n\nFor example, if ``$(prefix)/etc/spack/defaults/include.yaml`` contains:\n\n.. code-block:: yaml\n\n include:\n - path: \"${platform}\"\n optional: true\n - path: base\n\nthen, on macOS (``darwin``), configuration settings for files under the ``$(prefix)/etc/spack/defaults/darwin`` directory would be picked up if they are present.\nBecause ``${platform}`` is above the ``base`` include in the list, ``${platform}`` settings will override anything in ``base`` if there are conflicts.\n\n.. note::\n\n You can get the name to use for ```` by running ``spack arch --platform``.\n\nPlatform-specific configuration files can similarly be set up for any other scope by creating an ``include.yaml`` similar to the one above for ``defaults`` -- under the appropriate configuration paths (see :ref:`config-overrides`) and creating a subdirectory with the platform name that contains the configurations.\n\n.. _config-scope-precedence:\n\nScope Precedence\n----------------\n\nWhen Spack queries for configuration parameters, it searches in higher-precedence scopes first.\nSo, settings in a higher-precedence file can override those with the same key in a lower-precedence one.\nFor list-valued settings, Spack merges lists by *prepending* items from higher-precedence configurations to items from lower-precedence configurations by default.\nCompletely ignoring lower-precedence configuration options is supported with the ``::`` notation for keys (see :ref:`config-overrides` below).\n\n.. note::\n\n Settings in a scope take precedence over those provided in any included configuration files (i.e., files listed in :ref:`include.yaml ` or an ``include:`` section in ``spack.yaml``).\n\nThere are also special notations for string concatenation and precedence override:\n\n* ``+:`` will force *prepending* strings or lists.\n For lists, this is the default behavior.\n* ``-:`` works similarly, but for *appending* values.\n\nSee :ref:`config-prepend-append` for more details.\n\nSimple keys\n^^^^^^^^^^^\n\nLet's look at an example of overriding a single key in a Spack configuration file.\nIf your configurations look like this:\n\n.. code-block:: yaml\n :caption: ``$(prefix)/etc/spack/defaults/config.yaml``\n :name: code-example-defaults-config-yaml\n\n config:\n install_tree:\n root: $spack/opt/spack\n build_stage:\n - $tempdir/$user/spack-stage\n - ~/.spack/stage\n\n\n.. code-block:: yaml\n :caption: ``~/.spack/config.yaml``\n :name: code-example-user-config-yaml\n\n config:\n install_tree:\n root: /some/other/directory\n\n\nSpack will only override ``install_tree`` in the ``config`` section, and will take the site preferences for other settings.\nYou can see the final, combined configuration with the ``spack config get `` command:\n\n.. code-block:: console\n :emphasize-lines: 3\n\n $ spack config get config\n config:\n install_tree:\n root: /some/other/directory\n build_stage:\n - $tempdir/$user/spack-stage\n - ~/.spack/stage\n\n\n.. _config-prepend-append:\n\nString Concatenation\n^^^^^^^^^^^^^^^^^^^^\n\nAbove, the user ``config.yaml`` *completely* overrides specific settings in the default ``config.yaml``.\nSometimes, it is useful to add a suffix/prefix to a path or name.\nTo do this, you can use the ``-:`` notation for *append* string concatenation at the end of a key in a configuration file.\nFor example:\n\n.. code-block:: yaml\n :emphasize-lines: 1\n :caption: ``~/.spack/config.yaml``\n :name: code-example-append-install-tree\n\n config:\n install_tree:\n root-: /my/custom/suffix/\n\nSpack will then append to the lower-precedence configuration under the ``root`` key:\n\n.. code-block:: console\n\n $ spack config get config\n config:\n install_tree:\n root: /some/other/directory/my/custom/suffix\n build_stage:\n - $tempdir/$user/spack-stage\n - ~/.spack/stage\n\n\nSimilarly, ``+:`` can be used to *prepend* to a path or name:\n\n.. code-block:: yaml\n :emphasize-lines: 1\n :caption: ``~/.spack/config.yaml``\n :name: code-example-prepend-install-tree\n\n config:\n install_tree:\n root+: /my/custom/suffix/\n\n\n.. _config-overrides:\n\nOverriding entire sections\n^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nAbove, the user ``config.yaml`` only overrides specific settings in the default ``config.yaml``.\nSometimes, it is useful to *completely* override lower-precedence settings.\nTo do this, you can use *two* colons at the end of a key in a configuration file.\nFor example:\n\n.. code-block:: yaml\n :emphasize-lines: 1\n :caption: ``~/.spack/config.yaml``\n :name: code-example-override-config-section\n\n config::\n install_tree:\n root: /some/other/directory\n\nSpack will ignore all lower-precedence configuration under the ``config::`` section:\n\n.. code-block:: console\n\n $ spack config get config\n config:\n install_tree:\n root: /some/other/directory\n\n\nList-valued settings\n^^^^^^^^^^^^^^^^^^^^\n\nLet's revisit the ``config.yaml`` example one more time.\nThe ``build_stage`` setting's value is an ordered list of directories:\n\n.. code-block:: yaml\n :caption: ``$(prefix)/etc/spack/defaults/config.yaml``\n :name: code-example-defaults-build-stage\n\n config:\n build_stage:\n - $tempdir/$user/spack-stage\n - ~/.spack/stage\n\n\nSuppose the user configuration adds its *own* list of ``build_stage`` paths:\n\n.. code-block:: yaml\n :caption: ``~/.spack/config.yaml``\n :name: code-example-user-build-stage\n\n config:\n build_stage:\n - /lustre-scratch/$user/spack\n - ~/mystage\n\n\nSpack will first look at the paths in the defaults ``config.yaml``, then the paths in the user's ``~/.spack/config.yaml``.\nThe list in the higher-precedence scope is *prepended* to the defaults.\n``spack config get config`` shows the result:\n\n.. code-block:: console\n :emphasize-lines: 5-8\n\n $ spack config get config\n config:\n install_tree:\n root: /some/other/directory\n build_stage:\n - /lustre-scratch/$user/spack\n - ~/mystage\n - $tempdir/$user/spack-stage\n - ~/.spack/stage\n\n\nAs in :ref:`config-overrides`, the higher-precedence scope can *completely* override the lower-precedence scope using ``::``.\nSo if the user config looked like this:\n\n.. code-block:: yaml\n :emphasize-lines: 1\n :caption: ``~/.spack/config.yaml``\n :name: code-example-override-build-stage\n\n config:\n build_stage::\n - /lustre-scratch/$user/spack\n - ~/mystage\n\n\nThe merged configuration would look like this:\n\n.. code-block:: console\n :emphasize-lines: 5-6\n\n $ spack config get config\n config:\n install_tree:\n root: /some/other/directory\n build_stage:\n - /lustre-scratch/$user/spack\n - ~/mystage\n\n\n.. _config-file-variables:\n\nConfig File Variables\n---------------------\n\nSpack understands several variables which can be used in config file paths wherever they appear.\nThere are three sets of these variables: Spack-specific variables, environment variables, and user path variables.\nSpack-specific variables and environment variables are both indicated by prefixing the variable name with ``$``.\nUser path variables are indicated at the start of the path with ``~`` or ``~user``.\n\nSpack-specific variables\n^^^^^^^^^^^^^^^^^^^^^^^^\n\nSpack understands over a dozen special variables.\nThese are:\n\n* ``$env``: name of the currently active :ref:`environment `\n* ``$spack``: path to the prefix of this Spack installation\n* ``$tempdir``: default system temporary directory (as specified in Python's `tempfile.tempdir `_ variable.\n* ``$user``: name of the current user\n* ``$user_cache_path``: user cache directory (``~/.spack`` unless :ref:`overridden `)\n* ``$architecture``: the architecture triple of the current host, as detected by Spack.\n* ``$arch``: alias for ``$architecture``.\n* ``$platform``: the platform of the current host, as detected by Spack.\n* ``$operating_system``: the operating system of the current host, as detected by the ``distro`` Python module.\n* ``$os``: alias for ``$operating_system``.\n* ``$target``: the ISA target for the current host, as detected by ArchSpec.\n E.g.\n ``skylake`` or ``neoverse-n1``.\n* ``$target_family``.\n The target family for the current host, as detected by ArchSpec.\n E.g.\n ``x86_64`` or ``aarch64``.\n* ``$date``: the current date in the format YYYY-MM-DD\n* ``$spack_short_version``: the Spack version truncated to the first components.\n\n\nNote that, as with shell variables, you can write these as ``$varname`` or with braces to distinguish the variable from surrounding characters: ``${varname}``.\nTheir names are also case insensitive, meaning that ``$SPACK`` works just as well as ``$spack``.\nThese special variables are substituted first, so any environment variables with the same name will not be used.\n\nEnvironment variables\n^^^^^^^^^^^^^^^^^^^^^\n\nAfter Spack-specific variables are evaluated, environment variables are expanded.\nThese are formatted like Spack-specific variables, e.g., ``${varname}``.\nYou can use this to insert environment variables in your Spack configuration.\n\nUser home directories\n^^^^^^^^^^^^^^^^^^^^^\n\nSpack performs Unix-style tilde expansion on paths in configuration files.\nThis means that tilde (``~``) will expand to the current user's home directory, and ``~user`` will expand to a specified user's home directory.\nThe ``~`` must appear at the beginning of the path, or Spack will not expand it.\n\n.. _configuration_environment_variables:\n\nEnvironment Modifications\n-------------------------\n\nSpack allows users to prescribe custom environment modifications in a few places within its configuration files.\nEvery time these modifications are allowed, they are specified as a dictionary, like in the following example:\n\n.. code-block:: yaml\n\n environment:\n set:\n LICENSE_FILE: \"/path/to/license\"\n unset:\n - CPATH\n - LIBRARY_PATH\n append_path:\n PATH: \"/new/bin/dir\"\n\nThe possible actions that are permitted are ``set``, ``unset``, ``append_path``, ``prepend_path``, and finally ``remove_path``.\nThey all require a dictionary of variable names mapped to the values used for the modification, with the exception of ``unset``, which requires just a list of variable names.\nNo particular order is ensured for the execution of each of these modifications.\n\nSeeing Spack's Configuration\n----------------------------\n\nWith so many scopes overriding each other, it can sometimes be difficult to understand what Spack's final configuration looks like.\n\nSpack provides two useful ways to view the final \"merged\" version of any configuration file: ``spack config get`` and ``spack config blame``.\n\n.. _cmd-spack-config-get:\n\n``spack config get``\n^^^^^^^^^^^^^^^^^^^^\n\n``spack config get`` shows a fully merged configuration file, taking into account all scopes.\nFor example, to see the fully merged ``config.yaml``, you can type:\n\n.. code-block:: console\n\n $ spack config get config\n config:\n debug: false\n checksum: true\n verify_ssl: true\n dirty: false\n build_jobs: 8\n install_tree:\n root: $spack/opt/spack\n template_dirs:\n - $spack/templates\n directory_layout: {architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}\n build_stage:\n - $tempdir/$user/spack-stage\n - ~/.spack/stage\n - $spack/var/spack/stage\n source_cache: $spack/var/spack/cache\n misc_cache: ~/.spack/cache\n locks: true\n\nLikewise, this will show the fully merged ``packages.yaml``:\n\n.. code-block:: console\n\n $ spack config get packages\n\nYou can use this in conjunction with the ``-C`` / ``--config-scope`` argument to see how your scope will affect Spack's configuration:\n\n.. code-block:: console\n\n $ spack -C /path/to/my/scope config get packages\n\n\n.. _cmd-spack-config-blame:\n\n``spack config blame``\n^^^^^^^^^^^^^^^^^^^^^^\n\n``spack config blame`` functions much like ``spack config get``, but it shows exactly which configuration file each setting came from.\nIf you do not know why Spack is behaving a certain way, this command can help you track down the source of the configuration:\n\n.. code-block:: console\n\n $ spack --insecure -C ./my-scope -C ./my-scope-2 config blame config\n ==> Warning: You asked for --insecure. Will NOT check SSL certificates.\n --- config:\n _builtin debug: False\n /home/myuser/spack/etc/spack/defaults/config.yaml:72 checksum: True\n command_line verify_ssl: False\n ./my-scope-2/config.yaml:2 dirty: False\n _builtin build_jobs: 8\n ./my-scope/config.yaml:2 install_tree: /path/to/some/tree\n /home/myuser/spack/etc/spack/defaults/config.yaml:23 template_dirs:\n /home/myuser/spack/etc/spack/defaults/config.yaml:24 - $spack/templates\n /home/myuser/spack/etc/spack/defaults/config.yaml:28 directory_layout: {architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}\n /home/myuser/spack/etc/spack/defaults/config.yaml:49 build_stage:\n /home/myuser/spack/etc/spack/defaults/config.yaml:50 - $tempdir/$user/spack-stage\n /home/myuser/spack/etc/spack/defaults/config.yaml:51 - ~/.spack/stage\n /home/myuser/spack/etc/spack/defaults/config.yaml:52 - $spack/var/spack/stage\n /home/myuser/spack/etc/spack/defaults/config.yaml:57 source_cache: $spack/var/spack/cache\n /home/myuser/spack/etc/spack/defaults/config.yaml:62 misc_cache: ~/.spack/cache\n /home/myuser/spack/etc/spack/defaults/config.yaml:86 locks: True\n\nYou can see above that the ``build_jobs`` and ``debug`` settings are built-in and are not overridden by a configuration file.\nThe ``verify_ssl`` setting comes from the ``--insecure`` option on the command line.\nThe ``dirty`` and ``install_tree`` settings come from the custom scopes ``./my-scope`` and ``./my-scope-2``, and all other configuration options come from the default configuration files that ship with Spack.\n\n.. _local-config-overrides:\n\nOverriding Local Configuration\n------------------------------\n\nSpack's ``system`` and ``user`` scopes provide ways for administrators and users to set global defaults for all Spack instances, but for use cases where one wants a clean Spack installation, these scopes can be undesirable.\nFor example, users may want to opt out of global system configuration, or they may want to ignore their own home directory settings when running in a continuous integration environment.\n\nSpack also, by default, keeps various caches and user data in ``~/.spack``, but users may want to override these locations.\n\nSpack provides three environment variables that allow you to override or opt out of configuration locations:\n\n* ``SPACK_USER_CONFIG_PATH``: Override the path to use for the ``user`` scope (``~/.spack`` by default).\n* ``SPACK_SYSTEM_CONFIG_PATH``: Override the path to use for the ``system`` scope (``/etc/spack`` by default).\n* ``SPACK_DISABLE_LOCAL_CONFIG``: Set this environment variable to completely disable **both** the system and user configuration directories.\n Spack will then only consider its own defaults and ``site`` configuration locations.\n\nAnd one that allows you to move the default cache location:\n\n* ``SPACK_USER_CACHE_PATH``: Override the default path to use for user data (misc_cache, tests, reports, etc.)\n\nWith these settings, if you want to isolate Spack in a CI environment, you can do this:\n\n.. code-block:: console\n\n $ export SPACK_DISABLE_LOCAL_CONFIG=true\n $ export SPACK_USER_CACHE_PATH=/tmp/spack\n" }, { "path": "lib/spack/docs/configuring_compilers.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Discover how to configure compilers in Spack, whether by specifying them as externals, or by installing them with Spack.\n\n.. _compiler-config:\n\nConfiguring Compilers\n=====================\n\nSpack has the ability to build packages with multiple compilers and compiler versions.\nCompilers can be made available to Spack by:\n\n1. Specifying them as externals in ``packages.yaml``, or\n2. Having them installed in the current Spack store, or\n3. Having them available as binaries in a build cache\n\nFor convenience, Spack will automatically detect compilers as externals the first time it needs them, if no compiler is available.\n\n.. _cmd-spack-compilers:\n\n``spack compiler list``\n-----------------------\n\nYou can see which compilers are available to Spack by running ``spack compiler list``:\n\n.. code-block:: spec\n\n $ spack compiler list\n ==> Available compilers\n -- gcc ubuntu20.04-x86_64 ---------------------------------------\n [e] gcc@10.5.0 [+] gcc@15.1.0 [+] gcc@14.3.0\n\nCompilers marked with an ``[e]`` are system compilers (externals), and those marked with a ``[+]`` have been installed by Spack.\nCompilers from remote build caches are marked as ``-``, but are not shown by default.\nTo see them you need a specific option:\n\n.. code-block:: console\n\n $ spack compiler list --remote\n ==> Available compilers\n -- gcc ubuntu20.04-x86_64 ---------------------------------------\n [e] gcc@10.5.0 [+] gcc@15.1.0 [+] gcc@14.3.0\n\n -- gcc ubuntu20.04-x86_64 ---------------------------------------\n - gcc@12.4.0\n\nAny of these compilers can be used to build Spack packages.\nMore details on how this is done can be found in :ref:`sec-specs`.\n\n.. _cmd-spack-compiler-find:\n\n``spack compiler find``\n-----------------------\n\nIf you do not see a compiler in the list shown by:\n\n.. code-block:: console\n\n $ spack compiler list\n\nbut you want to use it with Spack, you can simply run ``spack compiler find`` with the path to where the compiler is installed.\nFor example:\n\n.. code-block:: console\n\n $ spack compiler find /opt/intel/oneapi/compiler/2025.1/bin/\n ==> Added 1 new compiler to /home/user/.spack/packages.yaml\n intel-oneapi-compilers@2025.1.0\n ==> Compilers are defined in the following files:\n /home/user/.spack/packages.yaml\n\nOr you can run ``spack compiler find`` with no arguments to force auto-detection.\nThis is useful if you do not know where compilers are installed, but you know that new compilers have been added to your ``PATH``.\nFor example, you might load a module, like this:\n\n.. code-block:: console\n\n $ module load gcc/4.9.0\n $ spack compiler find\n ==> Added 1 new compiler to /home/user/.spack/packages.yaml\n gcc@4.9.0\n\nThis loads the environment module for gcc-4.9.0 to add it to ``PATH``, and then it adds the compiler to Spack.\n\n.. note::\n\n By default, Spack does not fill in the ``modules:`` field in the ``packages.yaml`` file.\n If you are using a compiler from a module, then you should add this field manually.\n See the section on :ref:`compilers-requiring-modules`.\n\n.. _cmd-spack-compiler-info:\n\n``spack compiler info``\n-----------------------\n\nIf you want to see additional information about specific compilers, you can run ``spack compiler info``:\n\n.. code-block:: console\n\n $ spack compiler info gcc\n gcc@=8.4.0 languages='c,c++,fortran' arch=linux-ubuntu20.04-x86_64:\n prefix: /usr\n compilers:\n c: /usr/bin/gcc-8\n cxx: /usr/bin/g++-8\n fortran: /usr/bin/gfortran-8\n\n gcc@=9.4.0 languages='c,c++,fortran' arch=linux-ubuntu20.04-x86_64:\n prefix: /usr\n compilers:\n c: /usr/bin/gcc\n cxx: /usr/bin/g++\n fortran: /usr/bin/gfortran\n\n gcc@=10.5.0 languages='c,c++,fortran' arch=linux-ubuntu20.04-x86_64:\n prefix: /usr\n compilers:\n c: /usr/bin/gcc-10\n cxx: /usr/bin/g++-10\n fortran: /usr/bin/gfortran-10\n\nThis shows the details of the compilers that were detected by Spack.\nNotice also that we didn't have to be too specific about the version.\nWe just said ``gcc``, and we got information about all the matching compilers.\n\nManual configuration of external compilers\n------------------------------------------\n\nIf auto-detection fails, you can manually configure a compiler by editing your ``packages`` configuration.\nYou can do this by running:\n\n.. code-block:: console\n\n $ spack config edit packages\n\nwhich will open the file in :ref:`your favorite editor `.\n\nEach compiler has an \"external\" entry in the file with ``extra_attributes``:\n\n.. code-block:: yaml\n\n packages:\n gcc:\n externals:\n - spec: gcc@10.5.0 languages='c,c++,fortran'\n prefix: /usr\n extra_attributes:\n compilers:\n c: /usr/bin/gcc-10\n cxx: /usr/bin/g++-10\n fortran: /usr/bin/gfortran-10\n\nThe compiler executables are listed under ``extra_attributes:compilers``, and are keyed by language.\nOnce you save the file, the configured compilers will show up in the list displayed by ``spack compilers``.\n\nYou can also add compiler flags to manually configured compilers.\nThese flags should be specified in the ``flags`` section of the compiler specification.\nThe valid flags are ``cflags``, ``cxxflags``, ``fflags``, ``cppflags``, ``ldflags``, and ``ldlibs``.\nFor example:\n\n.. code-block:: yaml\n\n packages:\n gcc:\n externals:\n - spec: gcc@10.5.0 languages='c,c++,fortran'\n prefix: /usr\n extra_attributes:\n compilers:\n c: /usr/bin/gcc-10\n cxx: /usr/bin/g++-10\n fortran: /usr/bin/gfortran-10\n flags:\n cflags: -O3 -fPIC\n cxxflags: -O3 -fPIC\n cppflags: -O3 -fPIC\n\nThese flags will be treated by Spack as if they were entered from the command line each time this compiler is used.\nThe compiler wrappers then inject those flags into the compiler command.\nCompiler flags entered from the command line will be discussed in more detail in the following section.\n\nSome compilers also require additional environment configuration.\nExamples include Intel's oneAPI and AMD's AOCC compiler suites, which have custom scripts for loading environment variables and setting paths.\nThese variables should be specified in the ``environment`` section of the compiler specification.\nThe operations available to modify the environment are ``set``, ``unset``, ``prepend_path``, ``append_path``, and ``remove_path``.\nFor example:\n\n.. code-block:: yaml\n\n packages:\n intel-oneapi-compilers:\n externals:\n - spec: intel-oneapi-compilers@2025.1.0\n prefix: /opt/intel/oneapi\n extra_attributes:\n compilers:\n c: /opt/intel/oneapi/compiler/2025.1/bin/icx\n cxx: /opt/intel/oneapi/compiler/2025.1/bin/icpx\n fortran: /opt/intel/oneapi/compiler/2025.1/bin/ifx\n environment:\n set:\n MKL_ROOT: \"/path/to/mkl/root\"\n unset: # A list of environment variables to unset\n - CC\n prepend_path: # Similar for append|remove_path\n LD_LIBRARY_PATH: /ld/paths/added/by/setvars/sh\n\nIt is also possible to specify additional ``RPATHs`` that the compiler will add to all executables generated by that compiler.\nThis is useful for forcing certain compilers to RPATH their own runtime libraries so that executables will run without the need to set ``LD_LIBRARY_PATH``:\n\n.. code-block:: yaml\n\n packages:\n gcc:\n externals:\n - spec: gcc@4.9.3\n prefix: /opt/gcc\n extra_attributes:\n compilers:\n c: /opt/gcc/bin/gcc\n cxx: /opt/gcc/bin/g++\n fortran: /opt/gcc/bin/gfortran\n extra_rpaths:\n - /path/to/some/compiler/runtime/directory\n - /path/to/some/other/compiler/runtime/directory\n\n.. _compilers-requiring-modules:\n\nCompilers Requiring Modules\n---------------------------\n\nMany installed compilers will work regardless of the environment from which they are called.\nHowever, some installed compilers require environment variables to be set in order to run.\n\nOn typical HPC clusters, these environment modifications are usually delegated to some \"module\" system.\nIn such a case, you should tell Spack which module(s) to load in order to run the chosen compiler:\n\n.. code-block:: yaml\n\n packages:\n gcc:\n externals:\n - spec: gcc@10.5.0 languages='c,c++,fortran'\n prefix: /opt/compilers\n extra_attributes:\n compilers:\n c: /opt/compilers/bin/gcc-10\n cxx: /opt/compilers/bin/g++-10\n fortran: /opt/compilers/bin/gfortran-10\n modules: [gcc/10.5.0]\n\nSome compilers require special environment settings to be loaded not just to run, but also to execute the code they build, breaking packages that need to execute code they just compiled.\nIf it's not possible or practical to use a better compiler, you'll need to ensure that environment settings are preserved for compilers like this (i.e., you'll need to load the module or source the compiler's shell script).\n\nBy default, Spack tries to ensure that builds are reproducible by cleaning the environment before building.\nIf this interferes with your compiler settings, you CAN use ``spack install --dirty`` as a workaround.\nNote that this MAY interfere with package builds.\n\nBuild Your Own Compiler\n-----------------------\n\nIf you require a specific compiler and version, you can have Spack build it for you.\nFor example:\n\n.. code-block:: spec\n\n $ spack install gcc@14\n\nOnce the compiler is installed, you can start using it without additional configuration:\n\n.. code-block:: spec\n\n $ spack install hdf5~mpi %gcc@14\n\nMixing Compilers\n----------------\n\nFor more options on configuring Spack to mix different compilers for different languages, see :ref:`the toolchains configuration docs `.\n\nTo disable mixing (e.g. if you have multiple compilers defined, but want each concretized DAG to use one of them consistently), you can set:\n\n.. code-block:: yaml\n\n concretizer:\n compiler_mixing: false\n\nThis affects root specs and any (transitive) link or run dependencies.\nBuild-only dependencies are allowed to use different compilers (even when this is set).\n\nSome packages are difficult to build with high performance compilers, and it may be necessary to enable compiler mixing just for those packages.\nTo enable mixing for specific packages, specify an allow-list in the ``compiler_mixing`` config:\n\n.. code-block:: yaml\n\n concretizer:\n compiler_mixing: [\"openssl\"]\n\nAdding ``openssl`` to the compiler mixing allow-list does not allow mixing for dependencies of ``openssl``.\n" }, { "path": "lib/spack/docs/containers.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Learn how to turn Spack packages and Spack environments into OCI-compatible container images, either by exporting existing installations or by generating recipes for Docker and Singularity.\n\n.. _containers:\n\nContainer Images\n================\n\nWhether you want to share applications with others who do not use Spack, deploy on cloud services that run container images, or move workloads to HPC clusters, containers are an effective way to package and distribute software.\n\nSpack offers two fundamentally different paradigms for creating container images, each with distinct advantages.\nYou can either export software packages already built on your host system as a container image, or you can generate a traditional recipe file (``Dockerfile`` or Singularity Definition File) to build the software from scratch inside the container.\n\n.. list-table:: Comparison of Spack container image creation methods\n :widths: 15 42 43\n :header-rows: 1\n\n * - \n - :ref:`Container Image Export `\n - :ref:`Recipe Generation `\n * - **Purpose**\n - Exports existing installations from the host system as a container image\n - Runs ``spack install`` to build software from source *inside* the container build process\n * - **Spack Command**\n - ``spack buildcache push``\n - ``spack containerize``\n * - **Reproducibility**\n - Limited: depends on the host system\n - High: controlled build environment\n * - **Input**\n - Installed Spack packages or environments\n - A ``spack.yaml`` file\n * - **Speed**\n - Faster: copies existing binaries\n - Slower: typically builds from source\n * - **Troubleshooting**\n - Build issues are resolved on the host, where debugging is simpler\n - Build issues must be resolved inside the container build process\n * - **Build Tools**\n - None\n - Docker, Podman, Singularity, or similar\n * - **Privileges**\n - None (rootless)\n - May require elevated privileges, depending on the container build tool (root)\n * - **Output destination**\n - OCI-compatible registry\n - Local Docker or Singularity image\n\n\n.. _exporting-images:\n\nExporting Spack installations as Container Images\n-------------------------------------------------\n\nThe command\n\n.. code-block:: text\n\n spack buildcache push [--base-image BASE_IMAGE] [--tag TAG] mirror [specs...]\n\ncreates and pushes a container image to an OCI-compatible container registry, with the ``mirror`` argument specifying a registry (see below).\n\nThink of this command less as \"building a container\" and more as archiving a working software stack into a portable image.\n\nContainer images created this way are **minimal**: they contain only runtime dependencies of the specified specs, the base image, and nothing else.\nSpack itself is *not* included in the resulting image.\n\nThe arguments are as follows:\n\n``--base-image BASE_IMAGE``\n Specifies the base image to use for the container.\n This should be a minimal Linux distribution with a libc that is compatible with the host system.\n For example, if your host system is Ubuntu 22.04, you can use ``ubuntu:22.04``, ``ubuntu:24.04``, or newer: the libc in the container image must be at least the version of the host system, assuming ABI compatibility.\n It is also perfectly fine to use a completely different Linux distribution as long as the libc is compatible.\n\n``--tag TAG``\n Specifies a container image tag to use.\n This tag is used for the image consisting of all specs specified in the command line together.\n\n``mirror`` argument\n Either the name of a configured OCI registry image (in ``mirrors.yaml``), or a URL specifying the registry and image name.\n\n * When pushing to remote registries, you will typically :ref:`specify the name of a registry ` from your Spack configuration.\n * When pushing to a local registry, you can simply specify a URL like ``oci+http://localhost:5000/[image]``, where ``[image]`` is the name of the image to create, and ``oci+http://`` indicates that the registry does not support HTTPS.\n\n``specs...`` arguments\n is a list of Spack specs to include in the image.\n These are packages that have already been installed by Spack.\n When a Spack environment is activated, only the packages in the environment are included in the image.\n If no specs are given, and a Spack environment is active, all packages in the environment are included.\n\nSpack publishes every individual dependency as a separate image layer, which allows for efficient storage and transfer of images with overlapping dependencies.\n\n.. note::\n The Docker ``overlayfs2`` storage driver is limited to 128 layers, above which a ``max depth exceeded`` error may be produced when pulling the image.\n You can hit this limit when exporting container images from larger environments or packages with many dependencies.\n There are `alternative drivers `_ to work around this limitation.\n\nThe ``spack buildcache push --base-image ...`` command serves a **dual purpose**:\n\n1. It makes container images available for container runtimes like Docker and Podman.\n2. It makes the *same* binaries available :ref:`as a build cache ` for ``spack install``.\n\n.. _configuring-container-registries:\n\nContainer registries\n^^^^^^^^^^^^^^^^^^^^\n\nThe ``spack buildcache push`` command exports container images directly to an OCI-compatible container registry, such as Docker Hub, GitHub Container Registry (GHCR), Amazon ECR, Google GCR, Azure ACR, or a private registry.\n\nThese services require authentication, which is configured with the ``spack mirror add`` command:\n\n.. code-block:: spec\n\n $ spack mirror add \\\n --oci-username-variable REGISTRY_USER \\\n --oci-password-variable REGISTRY_TOKEN \\\n example-registry \\\n oci://example.com/name/image\n\nThis registers a mirror named ``example-registry`` in your ``mirrors.yaml`` configuration file that is associated with a container registry and image ``example.com/name/image``.\nThe registry can then be referred to by its name, e.g. ``spack buildcache push example-registry ...``.\n\nThe ``oci://`` scheme in the URL indicates that this is an OCI-compatible registry with HTTPS support.\nIf you only specify ``oci://name/image``, Spack will assume the registry is hosted on Docker Hub.\n\nThe ``--oci-username-variable`` and ``--oci-password-variable`` options specify the names of *environment variables* that will be used to authenticate with the registry.\nSpack does not store your credentials in configuration files; it expects you to set the corresponding environment variables in your shell before running the ``spack buildcache push`` command:\n\n.. code-block:: console\n\n $ REGISTRY_USER=user REGISTRY_TOKEN=token spack buildcache push ...\n\n.. seealso::\n\n The registry password is typically a *personal access token* (PAT) generated on the registry website or a command line tool.\n In the section :ref:`oci-authentication` we list specific examples for popular registries.\n\nIf you don't have access to a remote registry, or wish to experiment with container images locally, you can run a *local registry* on your machine and let Spack push to it.\nThis is as simple as running the `official registry image `_ in the background:\n\n.. code-block:: console\n\n $ docker run -d -p 5000:5000 --name registry registry\n\nIn this case, it is not necessary to configure a named mirror, you can simply refer to it by URL using ``oci+http://localhost:5000/[image]``, where ``[image]`` is the name of the image to create, and ``oci+http://`` indicates that the registry does not support HTTPS.\n\n.. _local-registry-example:\n\nExample 1: pushing selected specs as container images\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nAssume we have ``python@3.13`` and ``cmake@3`` already installed by Spack, and we want to push them as a combined container image ``software_stack:latest`` to a local registry.\n\nFirst we verify that the specs are indeed installed:\n\n.. code-block:: spec\n\n $ spack find --long python@3.13 cmake@3\n\n -- linux-ubuntu24.04-zen2 / %c,cxx=gcc@13.3.0 -------------------\n scpgv2h cmake@3.31.8 n54tvjw python@3.13.5\n\nSince these are the only installations on our system, we can simply refer to them by their spec strings.\nIn case there are multiple installations, we could use ``python/n54tvjw`` and ``cmake/scpgv2h`` to uniquely refer to them by hashes.\n\nWe now use ``spack buildcache push`` to publish these packages as a container image with ``ubuntu:24.04`` as a base image:\n\n.. code-block:: console\n\n $ spack buildcache push \\\n --base-image ubuntu:24.04 \\\n --tag latest \\\n oci+http://localhost:5000/software_stack \\\n python@3.13 cmake@3\n\nThey can now be pulled and run with Docker or any other OCI-compatible container runtime:\n\n.. code-block:: console\n\n $ docker run -it localhost:5000/software_stack:latest\n root@container-id:/# python3 --version\n Python 3.13.5\n root@container-id:/# cmake --version\n cmake version 3.31.8\n\n.. _installed-environments-as-containers:\n\nExample 2: pushing entire Spack environments as container images\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nIn this example we show how to export an installed :ref:`Spack environment ` as a container image and push it to a remote registry.\n\n.. code-block:: spec\n\n # Create and install an environment\n $ spack env create .\n $ spack -e . add python@3.13 cmake@3\n $ spack -e . install\n\n # Configure a remote registry\n $ spack -e . mirror add \\\n --oci-username-variable REGISTRY_USER \\\n --oci-password-variable REGISTRY_TOKEN \\\n container-registry \\\n oci://example.com/name/image\n\n # Push the image\n $ REGISTRY_USER=user REGISTRY_TOKEN=token \\\n spack -e . buildcache push \\\n --update-index \\\n --base-image ubuntu:24.04 \\\n --tag my_env \\\n container-registry\n\nThe resulting container image can then be run as follows:\n\n.. code-block:: console\n\n $ docker run -it example.com/name/image:my_env\n root@container-id:/# python3 --version\n Python 3.13.5\n root@container-id:/# cmake --version\n cmake version 3.31.8\n\nThe advantage of using a Spack environment is that we do not have to specify the individual specs on the command line when pushing the image.\nWith environments, all root specs and their runtime dependencies are included in the container image.\n\nIf you do specify specs in ``spack buildcache push`` with an environment active, only those matching specs from the environment are included in the image.\n\n\n.. _generating-recipes:\n\nGenerating recipes for Docker and Singularity\n---------------------------------------------\n\nApart from exporting existing installations into container images, Spack can also generate recipes for container images.\nThis is useful if you want to run Spack itself in a sandboxed environment instead of on the host system.\n\nThis approach requires you to have a container runtime like Docker or Singularity installed on your system, and can only be used using Spack environments.\n\nSince recipes need a little more boilerplate than:\n\n.. code-block:: docker\n\n COPY spack.yaml /environment\n RUN spack -e /environment install\n\nSpack provides a command to generate customizable recipes for container images.\nCustomizations include minimizing the size of the image, installing packages in the base image using the system package manager, and setting up a proper entry point to run the image.\n\n.. _cmd-spack-containerize:\n\nA Quick Introduction\n^^^^^^^^^^^^^^^^^^^^\n\nConsider having a Spack environment like the following:\n\n.. code-block:: yaml\n\n spack:\n specs:\n - gromacs+mpi\n - mpich\n\nProducing a ``Dockerfile`` from it is as simple as changing directories to where the ``spack.yaml`` file is stored and running the following command:\n\n.. code-block:: console\n\n $ spack containerize > Dockerfile\n\nThe ``Dockerfile`` that gets created uses multi-stage builds and other techniques to minimize the size of the final image:\n\n.. code-block:: docker\n\n # Build stage with Spack pre-installed and ready to be used\n FROM spack/ubuntu-jammy:develop AS builder\n\n\n # What we want to install and how we want to install it\n # is specified in a manifest file (spack.yaml)\n RUN mkdir -p /opt/spack-environment && \\\n set -o noclobber \\\n && (echo spack: \\\n && echo ' specs:' \\\n && echo ' - gromacs+mpi' \\\n && echo ' - mpich' \\\n && echo ' concretizer:' \\\n && echo ' unify: true' \\\n && echo ' config:' \\\n && echo ' install_tree:' \\\n && echo ' root: /opt/software' \\\n && echo ' view: /opt/views/view') > /opt/spack-environment/spack.yaml\n\n # Install the software, remove unnecessary deps\n RUN cd /opt/spack-environment && spack env activate . && spack install --fail-fast && spack gc -y\n\n # Strip all the binaries\n RUN find -L /opt/views/view/* -type f -exec readlink -f '{}' \\; | \\\n xargs file -i | \\\n grep 'charset=binary' | \\\n grep 'x-executable\\|x-archive\\|x-sharedlib' | \\\n awk -F: '{print $1}' | xargs strip\n\n # Modifications to the environment that are necessary to run\n RUN cd /opt/spack-environment && \\\n spack env activate --sh -d . > activate.sh\n\n\n # Bare OS image to run the installed executables\n FROM ubuntu:22.04\n\n COPY --from=builder /opt/spack-environment /opt/spack-environment\n COPY --from=builder /opt/software /opt/software\n COPY --from=builder /opt/views /opt/views\n\n RUN { \\\n echo '#!/bin/sh' \\\n && echo '.' /opt/spack-environment/activate.sh \\\n && echo 'exec \"$@\"'; \\\n } > /entrypoint.sh \\\n && chmod a+x /entrypoint.sh \\\n && ln -s /opt/views/view /opt/view\n\n\n ENTRYPOINT [ \"/entrypoint.sh\" ]\n CMD [ \"/bin/bash\" ]\n\n\nThe image itself can then be built and run in the usual way with any of the tools suitable for the task.\nFor instance, if we decided to use Docker:\n\n.. code-block:: bash\n\n $ spack containerize > Dockerfile\n $ docker build -t myimage .\n [ ... ]\n $ docker run -it myimage\n\nThe various components involved in the generation of the recipe and their configuration are discussed in detail in the sections below.\n\n.. _container_spack_images:\n\nOfficial Container Images for Spack\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nContainer images with Spack preinstalled are available on `Docker Hub `_ and `GitHub Container Registry `_.\nThese images are based on popular distributions and are named accordingly (e.g. ``spack/ubuntu-noble`` for Spack on top of ``ubuntu:24.04``).\n\nThe table below summarizes the available base images and their corresponding Spack images:\n\n.. _containers-supported-os:\n\n.. list-table:: Supported base container images\n :header-rows: 1\n\n * - Base Distribution\n - Base Image\n - Spack Image\n * - Ubuntu 20.04\n - ``ubuntu:20.04``\n - ``spack/ubuntu-focal``\n * - Ubuntu 22.04\n - ``ubuntu:22.04``\n - ``spack/ubuntu-jammy``\n * - Ubuntu 24.04\n - ``ubuntu:24.04``\n - ``spack/ubuntu-noble``\n * - CentOS Stream 9\n - ``quay.io/centos/centos:stream9``\n - ``spack/centos-stream9``\n * - openSUSE Leap\n - ``opensuse/leap``\n - ``spack/leap15``\n * - Amazon Linux 2\n - ``amazonlinux:2``\n - ``spack/amazon-linux``\n * - AlmaLinux 8\n - ``almalinux:8``\n - ``spack/almalinux8``\n * - AlmaLinux 9\n - ``almalinux:9``\n - ``spack/almalinux9``\n * - Rocky Linux 8\n - ``rockylinux:8``\n - ``spack/rockylinux8``\n * - Rocky Linux 9\n - ``rockylinux:9``\n - ``spack/rockylinux9``\n * - Fedora Linux 39\n - ``fedora:39``\n - ``spack/fedora39``\n * - Fedora Linux 40\n - ``fedora:40``\n - ``spack/fedora40``\n\nAll container images are tagged with the version of Spack they contain.\n\n.. list-table:: Spack container image tags\n :header-rows: 1\n\n * - Tag\n - Meaning\n * - ``:latest``\n - Latest *stable* release of Spack\n * - ``:1``\n - Latest ``1.x.y`` release of Spack\n * - ``:1.0``\n - Latest ``1.0.y`` release of Spack\n * - ``:1.0.2``\n - Specific ``1.0.2`` release of Spack\n * - ``:develop``\n - Latest *development* version of Spack\n\nThese images are available for anyone to use and take care of all the repetitive tasks that are necessary to set up Spack within a container.\nThe container recipes generated by Spack use them as default base images for their ``build`` stage, even though options to use custom base images provided by users are available to accommodate complex use cases.\n\nConfiguring the Container Recipe\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nAny Spack environment can be used for the automatic generation of container recipes.\nSensible defaults are provided for things like the base image or the version of Spack used in the image.\nIf finer tuning is needed, it can be obtained by adding the relevant metadata under the ``container`` attribute of environments:\n\n.. code-block:: yaml\n\n spack:\n specs:\n - gromacs+mpi\n - mpich\n\n container:\n # Select the format of the recipe e.g. docker,\n # singularity or anything else that is currently supported\n format: docker\n\n # Sets the base images for the stages where Spack builds the\n # software or where the software gets installed after being built.\n images:\n os: \"almalinux:9\"\n spack: develop\n\n # Whether or not to strip binaries\n strip: true\n\n # Additional system packages that are needed at runtime\n os_packages:\n final:\n - libgomp\n\n # Labels for the image\n labels:\n app: \"gromacs\"\n mpi: \"mpich\"\n\nA detailed description of the options available can be found in the :ref:`container_config_options` section.\n\nSetting Base Images\n^^^^^^^^^^^^^^^^^^^\n\nThe ``images`` subsection is used to select both the image where Spack builds the software and the image where the built software is installed.\nThis attribute can be set in different ways and which one to use depends on the use case at hand.\n\nUse Official Spack Images From Dockerhub\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nTo generate a recipe that uses an official Docker image from the Spack organization to build the software and the corresponding official OS image to install the built software, all the user has to do is specify:\n\n1. An operating system under ``images:os``\n2. A Spack version under ``images:spack``\n\nAny combination of these two values that can be mapped to one of the images discussed in :ref:`container_spack_images` is allowed.\nFor instance, the following ``spack.yaml``:\n\n.. code-block:: yaml\n\n spack:\n specs:\n - gromacs+mpi\n - mpich\n\n container:\n images:\n os: almalinux:9\n spack: \"1.0\"\n\nuses ``spack/almalinux9:1.0`` and ``almalinux:9`` for the stages where the software is respectively built and installed:\n\n.. code-block:: docker\n\n # Build stage with Spack pre-installed and ready to be used\n FROM spack/almalinux9:1.0 AS builder\n\n\n # What we want to install and how we want to install it\n # is specified in a manifest file (spack.yaml)\n RUN mkdir -p /opt/spack-environment && \\\n set -o noclobber \\\n && (echo spack: \\\n && echo ' specs:' \\\n && echo ' - gromacs+mpi' \\\n && echo ' - mpich' \\\n && echo ' concretizer:' \\\n && echo ' unify: true' \\\n && echo ' config:' \\\n && echo ' install_tree:' \\\n && echo ' root: /opt/software' \\\n && echo ' view: /opt/views/view') > /opt/spack-environment/spack.yaml\n\n # ...\n\n # Bare OS image to run the installed executables\n FROM quay.io/almalinuxorg/almalinux:9\n\n COPY --from=builder /opt/spack-environment /opt/spack-environment\n COPY --from=builder /opt/software /opt/software\n COPY --from=builder /opt/views /opt/views\n\n RUN { \\\n echo '#!/bin/sh' \\\n && echo '.' /opt/spack-environment/activate.sh \\\n && echo 'exec \"$@\"'; \\\n } > /entrypoint.sh \\\n && chmod a+x /entrypoint.sh \\\n && ln -s /opt/views/view /opt/view\n\n\n ENTRYPOINT [ \"/entrypoint.sh\" ]\n CMD [ \"/bin/bash\" ]\n\n\n\nThis is the simplest available method of selecting base images, and we advise its use whenever possible.\nThere are cases, though, where using Spack official images is not enough to fit production needs.\nIn these situations, users can extend the recipe to start with the bootstrapping of Spack at a certain pinned version or manually select which base image to start from in the recipe, as we'll see next.\n\n\nUse a Bootstrap Stage for Spack\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nIn some cases, users may want to pin the commit SHA that is used for Spack to ensure later reproducibility or start from a fork of the official Spack repository to try a bugfix or a feature in an early stage of development.\nThis is possible by being just a little more verbose when specifying information about Spack in the ``spack.yaml`` file:\n\n.. code-block:: yaml\n\n images:\n os: amazonlinux:2\n spack:\n # URL of the Spack repository to be used in the container image\n url: \n # Either a commit SHA, a branch name, or a tag\n ref: \n # If true, turn a branch name or a tag into the corresponding commit\n # SHA at the time of recipe generation\n resolve_sha: \n\n``url`` specifies the URL from which to clone Spack and defaults to https://github.com/spack/spack.\nThe ``ref`` attribute can be either a commit SHA, a branch name, or a tag.\nThe default value in this case is to use the ``develop`` branch, but it may change in the future to point to the latest stable release.\nFinally, ``resolve_sha`` transforms branch names or tags into the corresponding commit SHAs at the time of recipe generation to allow for greater reproducibility of the results at a later time.\n\nThe list of operating systems that can be used to bootstrap Spack can be obtained with:\n\n.. command-output:: spack containerize --list-os\n\n.. note::\n\n The ``resolve_sha`` option uses ``git rev-parse`` under the hood and thus requires checking out the corresponding Spack repository in a temporary folder before generating the recipe.\n Recipe generation may take longer when this option is set to true because of this additional step.\n\n\nUse Custom Images Provided by Users\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nConsider, as an example, building a production-grade image for a CUDA application.\nThe best strategy would probably be to build on top of images provided by the vendor and regard CUDA as an external package.\n\nSpack does not currently provide an official image with CUDA configured this way, but users can build it on their own and then configure the environment to explicitly pull it.\nThis requires users to:\n\n1. Specify the image used to build the software under ``images:build``\n2. Specify the image used to install the built software under ``images:final``\n\nA ``spack.yaml`` like the following:\n\n.. code-block:: yaml\n\n spack:\n specs:\n - gromacs@2019.4+cuda build_type=Release\n - mpich\n - fftw precision=float\n packages:\n cuda:\n buildable: false\n externals:\n - spec: cuda%gcc\n prefix: /usr/local/cuda\n\n container:\n images:\n build: custom/cuda-13.0.1-ubuntu22.04:latest\n final: nvidia/cuda:13.0.1-base-ubuntu22.04\n\nproduces, for instance, the following ``Dockerfile``:\n\n.. code-block:: docker\n\n # Build stage with Spack pre-installed and ready to be used\n FROM custom/cuda-13.0.1-ubuntu22.04:latest AS builder\n\n\n # What we want to install and how we want to install it\n # is specified in a manifest file (spack.yaml)\n RUN mkdir -p /opt/spack-environment && \\\n set -o noclobber \\\n && (echo spack: \\\n && echo ' specs:' \\\n && echo ' - gromacs@2019.4+cuda build_type=Release' \\\n && echo ' - mpich' \\\n && echo ' - fftw precision=float' \\\n && echo ' packages:' \\\n && echo ' cuda:' \\\n && echo ' buildable: false' \\\n && echo ' externals:' \\\n && echo ' - spec: cuda%gcc' \\\n && echo ' prefix: /usr/local/cuda' \\\n && echo '' \\\n && echo ' concretizer:' \\\n && echo ' unify: true' \\\n && echo ' config:' \\\n && echo ' install_tree:' \\\n && echo ' root: /opt/software' \\\n && echo ' view: /opt/views/view') > /opt/spack-environment/spack.yaml\n\n # Install the software, remove unnecessary deps\n RUN cd /opt/spack-environment && spack env activate . && spack install --fail-fast && spack gc -y\n\n # Strip all the binaries\n RUN find -L /opt/views/view/* -type f -exec readlink -f '{}' \\; | \\\n xargs file -i | \\\n grep 'charset=binary' | \\\n grep 'x-executable\\|x-archive\\|x-sharedlib' | \\\n awk -F: '{print $1}' | xargs strip\n\n # Modifications to the environment that are necessary to run\n RUN cd /opt/spack-environment && \\\n spack env activate --sh -d . > activate.sh\n\n\n # Bare OS image to run the installed executables\n FROM nvidia/cuda:13.0.1-base-ubuntu22.04\n\n COPY --from=builder /opt/spack-environment /opt/spack-environment\n COPY --from=builder /opt/software /opt/software\n COPY --from=builder /opt/views /opt/views\n\n RUN { \\\n echo '#!/bin/sh' \\\n && echo '.' /opt/spack-environment/activate.sh \\\n && echo 'exec \"$@\"'; \\\n } > /entrypoint.sh \\\n && chmod a+x /entrypoint.sh \\\n && ln -s /opt/views/view /opt/view\n\n\n ENTRYPOINT [ \"/entrypoint.sh\" ]\n CMD [ \"/bin/bash\" ]\n\n\nwhere the base images for both stages are completely custom.\n\nThis second mode of selection for base images is more flexible than just choosing an operating system and a Spack version but is also more demanding.\nUsers may need to generate their base images themselves, and it's also their responsibility to ensure that:\n\n1. Spack is available in the ``build`` stage and set up correctly to install the required software\n2. The artifacts produced in the ``build`` stage can be executed in the ``final`` stage\n\nTherefore, we do not recommend its use in cases that can be otherwise covered by the simplified mode shown first.\n\nSingularity Definition Files\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nIn addition to producing recipes in ``Dockerfile`` format, Spack can produce Singularity Definition Files by just changing the value of the ``format`` attribute:\n\n.. code-block:: console\n\n $ cat spack.yaml\n spack:\n specs:\n - hdf5~mpi\n container:\n format: singularity\n\n $ spack containerize > hdf5.def\n $ sudo singularity build hdf5.sif hdf5.def\n\nThe minimum version of Singularity required to build a SIF (Singularity Image Format) image from the recipes generated by Spack is ``3.5.3``.\n\nExtending the Jinja2 Templates\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nThe ``Dockerfile`` and the Singularity definition file that Spack can generate are based on a few Jinja2 templates that are rendered according to the Spack environment being containerized.\nEven though Spack allows a great deal of customization by just setting appropriate values for the configuration options, sometimes that is not enough.\n\nIn those cases, a user can directly extend the template that Spack uses to render the image to, e.g., set additional environment variables or perform specific operations either before or after a given stage of the build.\nLet's consider as an example the following structure:\n\n.. code-block:: console\n\n $ tree /opt/environment\n /opt/environment\n ├── data\n │ └── data.csv\n ├── spack.yaml\n ├── data\n └── templates\n └── container\n └── CustomDockerfile\n\ncontaining both the custom template extension and the Spack environment manifest file.\nTo use a custom template, the Spack environment must register the directory containing it and declare its use under the ``container`` configuration:\n\n.. code-block:: yaml\n :emphasize-lines: 7-8,12\n\n spack:\n specs:\n - hdf5~mpi\n concretizer:\n unify: true\n config:\n template_dirs:\n - /opt/environment/templates\n container:\n format: docker\n depfile: true\n template: container/CustomDockerfile\n\nThe template extension can override two blocks, named ``build_stage`` and ``final_stage``, similarly to the example below:\n\n.. code-block:: text\n :caption: /opt/environment/templates/container/CustomDockerfile\n :emphasize-lines: 3,8\n\n {% extends \"container/Dockerfile\" %}\n {% block build_stage %}\n RUN echo \"Start building\"\n {{ super() }}\n {% endblock %}\n {% block final_stage %}\n {{ super() }}\n COPY data /share/myapp/data\n {% endblock %}\n\nThe Dockerfile is generated by running:\n\n.. code-block:: console\n\n $ spack -e /opt/environment containerize\n\nNote that the Spack environment must be active for Spack to read the template.\nThe recipe that gets generated contains the two extra instructions that we added in our template extension:\n\n.. code-block:: Dockerfile\n :emphasize-lines: 4,55\n\n # Build stage with Spack pre-installed and ready to be used\n FROM spack/ubuntu-jammy:develop AS builder\n\n RUN echo \"Start building\"\n\n # What we want to install and how we want to install it\n # is specified in a manifest file (spack.yaml)\n RUN mkdir -p /opt/spack-environment && \\\n set -o noclobber \\\n && (echo spack: \\\n && echo ' specs:' \\\n && echo ' - hdf5~mpi' \\\n && echo ' concretizer:' \\\n && echo ' unify: true' \\\n && echo ' config:' \\\n && echo ' template_dirs:' \\\n && echo ' - /tmp/tmp.xvyLqAZpZg' \\\n && echo ' install_tree:' \\\n && echo ' root: /opt/software' \\\n && echo ' view: /opt/views/view') > /opt/spack-environment/spack.yaml\n\n # Install the software, remove unnecessary deps\n RUN cd /opt/spack-environment && spack env activate . && spack concretize && spack env depfile -o Makefile && make -j $(nproc) && spack gc -y\n\n # Strip all the binaries\n RUN find -L /opt/views/view/* -type f -exec readlink -f '{}' \\; | \\\n xargs file -i | \\\n grep 'charset=binary' | \\\n grep 'x-executable\\|x-archive\\|x-sharedlib' | \\\n awk -F: '{print $1}' | xargs strip\n\n # Modifications to the environment that are necessary to run\n RUN cd /opt/spack-environment && \\\n spack env activate --sh -d . > activate.sh\n\n\n\n # Bare OS image to run the installed executables\n FROM ubuntu:22.04\n\n COPY --from=builder /opt/spack-environment /opt/spack-environment\n COPY --from=builder /opt/software /opt/software\n COPY --from=builder /opt/views /opt/views\n\n RUN { \\\n echo '#!/bin/sh' \\\n && echo '.' /opt/spack-environment/activate.sh \\\n && echo 'exec \"$@\"'; \\\n } > /entrypoint.sh \\\n && chmod a+x /entrypoint.sh \\\n && ln -s /opt/views/view /opt/view\n\n\n\n COPY data /share/myapp/data\n ENTRYPOINT [ \"/entrypoint.sh\" ]\n CMD [ \"/bin/bash\" ]\n\n\n.. _container_config_options:\n\nConfiguration Reference\n^^^^^^^^^^^^^^^^^^^^^^^\n\nThe tables below describe all the configuration options that are currently supported to customize the generation of container recipes:\n\n.. list-table:: General configuration options for the ``container`` section of ``spack.yaml``\n :header-rows: 1\n\n * - Option Name\n - Description\n - Allowed Values\n - Required\n * - ``format``\n - The format of the recipe\n - ``docker`` or ``singularity``\n - Yes\n * - ``depfile``\n - Whether to use a depfile for installation, or not\n - True or False (default)\n - No\n * - ``images:os``\n - Operating system used as a base for the image\n - See :ref:`containers-supported-os`\n - Yes, if using constrained selection of base images\n * - ``images:spack``\n - Version of Spack used in the ``build`` stage\n - Valid tags for ``base:image``\n - Yes, if using constrained selection of base images\n * - ``images:spack:url``\n - Repository from which Spack is cloned\n - Any fork of Spack\n - No\n * - ``images:spack:ref``\n - Reference for the checkout of Spack\n - Either a commit SHA, a branch name, or a tag\n - No\n * - ``images:spack:resolve_sha``\n - Resolve branches and tags in ``spack.yaml`` to commits in the generated recipe\n - True or False (default: False)\n - No\n * - ``images:build``\n - Image to be used in the ``build`` stage\n - Any valid container image\n - Yes, if using custom selection of base images\n * - ``images:final``\n - Image to be used in the ``final`` stage (runtime)\n - Any valid container image\n - Yes, if using custom selection of base images\n * - ``strip``\n - Whether to strip binaries\n - ``true`` (default) or ``false``\n - No\n * - ``os_packages:command``\n - Tool used to manage system packages\n - ``apt``, ``yum``, ``dnf``, ``dnf_epel``, ``zypper``, ``apk``, ``yum_amazon``\n - Only with custom base images\n * - ``os_packages:update``\n - Whether or not to update the list of available packages\n - True or False (default: True)\n - No\n * - ``os_packages:build``\n - System packages needed at build-time\n - Valid packages for the current OS\n - No\n * - ``os_packages:final``\n - System packages needed at run-time\n - Valid packages for the current OS\n - No\n * - ``labels``\n - Labels to tag the image\n - Pairs of key-value strings\n - No\n\n.. list-table:: Configuration options specific to Singularity\n :header-rows: 1\n\n * - Option Name\n - Description\n - Allowed Values\n - Required\n * - ``singularity:runscript``\n - Content of ``%runscript``\n - Any valid script\n - No\n * - ``singularity:startscript``\n - Content of ``%startscript``\n - Any valid script\n - No\n * - ``singularity:test``\n - Content of ``%test``\n - Any valid script\n - No\n * - ``singularity:help``\n - Description of the image\n - Description string\n - No\n\nBest Practices\n^^^^^^^^^^^^^^\n\nMPI\n\"\"\"\"\"\"\n\nDue to the dependency on Fortran for OpenMPI, which is the Spack default implementation, consider adding ``gfortran`` to the ``apt-get install`` list.\n\nRecent versions of OpenMPI will require you to pass ``--allow-run-as-root`` to your ``mpirun`` calls if started as root user inside Docker.\n\nFor execution on HPC clusters, it can be helpful to import the Docker image into Singularity in order to start a program with an *external* MPI.\nOtherwise, also add ``openssh-server`` to the ``apt-get install`` list.\n\nCUDA\n\"\"\"\"\"\"\n\nStarting from CUDA 9.0, NVIDIA provides minimal CUDA images based on Ubuntu.\nPlease see `their instructions `_.\nAvoid double-installing CUDA by adding, e.g.:\n\n.. code-block:: yaml\n\n packages:\n cuda:\n externals:\n - spec: \"cuda@9.0.176 arch=linux-ubuntu16-x86_64 %gcc@5.4.0\"\n prefix: /usr/local/cuda\n buildable: false\n\nto your ``spack.yaml``.\n\nUsers will either need ``nvidia-docker`` or, e.g., Singularity to *execute* device kernels.\n\nDocker on Windows and macOS\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nOn macOS and Windows, Docker runs on a hypervisor that is not allocated much memory by default, and some Spack packages may fail to build due to lack of memory.\nTo work around this issue, consider configuring your Docker installation to use more of your host memory.\nIn some cases, you can also ease the memory pressure on parallel builds by limiting the parallelism in your ``config.yaml``.\n\n.. code-block:: yaml\n\n config:\n build_jobs: 2\n" }, { "path": "lib/spack/docs/contribution_guide.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n A guide for developers and administrators on contributing new packages, features, or bug fixes to Spack, covering Git workflows, pull requests, and continuous integration testing.\n\n.. _contribution-guide:\n\nContribution Guide\n==================\n\nThis guide is intended for developers or administrators who want to contribute a new package, feature, or bug fix to Spack.\nIt assumes that you have at least some familiarity with Git and GitHub.\nThe guide will show a few examples of contributing workflows and discuss the granularity of pull requests (PRs).\nIt will also discuss the tests your PR must pass in order to be accepted into Spack.\n\nFirst, what is a PR?\nQuoting `Bitbucket's tutorials `_:\n\n Pull requests are a mechanism for a developer to notify team members that they have **completed a feature**.\n The pull request is more than just a notification -- it's a dedicated forum for discussing the proposed feature.\n\nImportant is **completed feature**.\nThe changes one proposes in a PR should correspond to one feature, bug fix, extension, etc.\nOne can create PRs with changes relevant to different ideas; however, reviewing such PRs becomes tedious and error-prone.\nIf possible, try to follow the **one-PR-one-package/feature** rule.\n\nBranches\n--------\n\nSpack's ``develop`` branch has the latest contributions.\nNearly all pull requests should start from ``develop`` and target ``develop``.\n\nThere is a branch for each major release series.\nRelease branches originate from ``develop`` and have tags for each point release in the series.\nFor example, ``releases/v0.14`` has tags for ``v0.14.0``, ``v0.14.1``, ``v0.14.2``, etc., versions of Spack.\nWe backport important bug fixes to these branches, but we do not advance the package versions or make other changes that would change the way Spack concretizes dependencies.\nCurrently, the maintainers manage these branches by cherry-picking from ``develop``.\nSee :ref:`releases` for more information.\n\nContinuous Integration\n----------------------\n\nSpack uses `GitHub Actions `_ for Continuous Integration (CI) testing.\nThis means that every time you submit a pull request, a series of tests will be run to make sure you did not accidentally introduce any bugs into Spack.\n**Your PR will not be accepted until it passes all of these tests.**\nWhile you can certainly wait for the results of these tests after submitting a PR, we recommend that you run them locally to speed up the review process.\n\n.. note::\n\n Oftentimes, CI will fail for reasons other than a problem with your PR.\n For example, ``apt-get``, ``pip``, or ``brew`` (Homebrew) might fail to download one of the dependencies for the test suite, or a transient bug might cause the unit tests to timeout.\n If any job fails, click the \"Details\" link and click on the test(s) that is failing.\n If it does not look like it is failing for reasons related to your PR, you have two options.\n If you have write permissions for the Spack repository, you should see a \"Restart workflow\" button on the right-hand side.\n If not, you can close and reopen your PR to rerun all of the tests.\n If the same test keeps failing, there may be a problem with your PR.\n If you notice that every recent PR is failing with the same error message, it may be that an issue occurred with the CI infrastructure, or one of Spack's dependencies put out a new release that is causing problems.\n If this is the case, please file an issue.\n\n\nWe currently test against Python 3.6 and up on both macOS and Linux and perform three types of tests:\n\n.. _cmd-spack-unit-test:\n\nUnit Tests\n^^^^^^^^^^\n\nUnit tests ensure that core Spack features like fetching or spec resolution are working as expected.\nIf your PR only adds new packages or modifies existing ones, there's very little chance that your changes could cause the unit tests to fail.\nHowever, if you make changes to Spack's core libraries, you should run the unit tests to make sure you didn't break anything.\n\nSince they test things like fetching from VCS repos, the unit tests require `git `_, `mercurial `_, and `subversion `_ to run.\nMake sure these are installed on your system and can be found in your ``PATH``.\nAll of these can be installed with Spack or with your system package manager.\n\nTo run *all* of the unit tests, use:\n\n.. code-block:: console\n\n $ spack unit-test\n\nThese tests may take several minutes to complete.\nIf you know you are only modifying a single Spack feature, you can run subsets of tests at a time.\nFor example, this would run all the tests in ``lib/spack/spack/test/architecture.py``:\n\n.. code-block:: console\n\n $ spack unit-test lib/spack/spack/test/architecture.py\n\nAnd this would run the ``test_platform`` test from that file:\n\n.. code-block:: console\n\n $ spack unit-test lib/spack/spack/test/architecture.py::test_platform\n\nThis allows you to develop iteratively: make a change, test that change, make another change, test that change, etc.\nWe use `pytest `_ as our tests framework, and these types of arguments are just passed to the ``pytest`` command underneath.\nSee `the pytest docs `_ for more details on test selection syntax.\n\n``spack unit-test`` has a few special options that can help you understand what tests are available.\nTo get a list of all available unit test files, run:\n\n.. command-output:: spack unit-test --list\n :ellipsis: 5\n\nTo see a more detailed list of available unit tests, use ``spack unit-test --list-long``:\n\n.. command-output:: spack unit-test --list-long\n :ellipsis: 10\n\nAnd to see the fully qualified names of all tests, use ``--list-names``:\n\n.. command-output:: spack unit-test --list-names\n :ellipsis: 5\n\nYou can combine these with ``pytest`` arguments to restrict which tests you want to know about.\nFor example, to see just the tests in ``architecture.py``:\n\n.. command-output:: spack unit-test --list-long lib/spack/spack/test/architecture.py\n\nYou can also combine any of these options with a ``pytest`` keyword search.\nSee the `pytest usage documentation `_ for more details on test selection syntax.\nFor example, to see the names of all tests that have \"spec\" or \"concretize\" somewhere in their names:\n\n.. command-output:: spack unit-test --list-names -k \"spec and concretize\"\n\nBy default, ``pytest`` captures the output of all unit tests, and it will print any captured output for failed tests.\nSometimes it is helpful to see your output interactively while the tests run (e.g., if you add print statements to unit tests).\nTo see the output *live*, use the ``-s`` argument to ``pytest``:\n\n.. code-block:: console\n\n $ spack unit-test -s --list-long lib/spack/spack/test/architecture.py::test_platform\n\nUnit tests are crucial to making sure bugs are not introduced into Spack.\nIf you are modifying core Spack libraries or adding new functionality, please add new unit tests for your feature and consider strengthening existing tests.\nYou will likely be asked to do this if you submit a pull request to the Spack project on GitHub.\nCheck out the `pytest documentation `_ and feel free to ask for guidance on how to write tests!\n\n.. note::\n\n You may notice the ``share/spack/qa/run-unit-tests`` script in the repository.\n This script is designed for CI.\n It runs the unit tests and reports coverage statistics back to Codecov.\n If you want to run the unit tests yourself, we suggest you use ``spack unit-test``.\n\nStyle Tests\n^^^^^^^^^^^^\n\nSpack uses `Flake8 `_ to test for `PEP 8 `_ conformance and `mypy `_ for type checking.\nPEP 8 is a series of style guides for Python that provide suggestions for everything from variable naming to indentation.\nIn order to limit the number of PRs that were mostly style changes, we decided to enforce PEP 8 conformance.\nYour PR needs to comply with PEP 8 in order to be accepted, and if it modifies the Spack library, it needs to successfully type-check with mypy as well.\n\nTesting for compliance with Spack's style is easy.\nSimply run the ``spack style`` command:\n\n.. code-block:: console\n\n $ spack style\n\n``spack style`` has a couple advantages over running the tools by hand:\n\n#. It only tests files that you have modified since branching off of ``develop``.\n\n#. It works regardless of what directory you are in.\n\n#. It automatically adds approved exemptions from the ``flake8`` checks.\n For example, URLs are often longer than 80 characters, so we exempt them from line length checks.\n We also exempt lines that start with ``homepage =``, ``url =``, ``version()``, ``variant()``, ``depends_on()``, and ``extends()`` in ``package.py`` files.\n This is now also possible when directly running Flake8 if you can use the ``spack`` formatter plugin included with Spack.\n\nMore approved Flake8 exemptions can be found `here `_.\n\nIf all is well, you'll see something like this:\n\n.. code-block:: console\n\n $ run-flake8-tests\n Dependencies found.\n =======================================================\n flake8: running flake8 code checks on spack.\n\n Modified files:\n\n var/spack/repos/spack_repo/builtin/packages/hdf5/package.py\n var/spack/repos/spack_repo/builtin/packages/hdf/package.py\n var/spack/repos/spack_repo/builtin/packages/netcdf/package.py\n =======================================================\n Flake8 checks were clean.\n\nHowever, if you are not compliant with PEP 8, Flake8 will complain:\n\n.. code-block:: console\n\n var/spack/repos/spack_repo/builtin/packages/netcdf/package.py:26: [F401] 'os' imported but unused\n var/spack/repos/spack_repo/builtin/packages/netcdf/package.py:61: [E303] too many blank lines (2)\n var/spack/repos/spack_repo/builtin/packages/netcdf/package.py:106: [E501] line too long (92 > 79 characters)\n Flake8 found errors.\n\nMost of the error messages are straightforward, but if you do not understand what they mean, just ask questions about them when you submit your PR.\nThe line numbers will change if you add or delete lines, so simply run ``spack style`` again to update them.\n\n.. tip::\n\n Try fixing Flake8 errors in reverse order.\n This eliminates the need for multiple runs of ``spack style`` just to re-compute line numbers and makes it much easier to fix errors directly off of the CI output.\n\n\nDocumentation Tests\n^^^^^^^^^^^^^^^^^^^\n\nSpack uses `Sphinx `_ to build its documentation.\nIn order to prevent things like broken links and missing imports, we added documentation tests that build the documentation and fail if there are any warning or error messages.\n\nBuilding the documentation requires several dependencies:\n\n* sphinx\n* sphinxcontrib-programoutput\n* sphinx-rtd-theme\n* graphviz\n* git\n* mercurial\n* subversion\n\nAll of these can be installed with Spack, e.g.:\n\n.. code-block:: console\n\n $ spack install py-sphinx py-sphinxcontrib-programoutput py-sphinx-rtd-theme graphviz git mercurial subversion\n\n.. warning::\n\n Sphinx has `several required dependencies `_.\n If you are using a Python from Spack and you installed ``py-sphinx`` and friends, you need to make them available to your Python interpreter.\n The easiest way to do this is to run:\n\n .. code-block:: console\n\n $ spack load py-sphinx py-sphinx-rtd-theme py-sphinxcontrib-programoutput\n\n so that all of the dependencies are added to ``PYTHONPATH``.\n If you see an error message like:\n\n .. code-block:: console\n\n Extension error:\n Could not import extension sphinxcontrib.programoutput (exception: No module named sphinxcontrib.programoutput)\n make: *** [html] Error 1\n\n that means Sphinx could not find ``py-sphinxcontrib-programoutput`` in your ``PYTHONPATH``.\n\nOnce all of the dependencies are installed, you can try building the documentation:\n\n.. code-block:: console\n\n $ cd path/to/spack/lib/spack/docs/\n $ make clean\n $ make\n\nIf you see any warning or error messages, you will have to correct those before your PR is accepted.\nIf you are editing the documentation, you should be running the documentation tests to make sure there are no errors.\nDocumentation changes can result in some obfuscated warning messages.\nIf you do not understand what they mean, feel free to ask when you submit your PR.\n\n.. _spack-builders-and-pipelines:\n\nGitLab CI\n^^^^^^^^^\n\nBuild Cache Stacks\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nSpack welcomes the contribution of software stacks of interest to the community.\nThese stacks are used to test package recipes and generate publicly available build caches.\nSpack uses GitLab CI for managing the orchestration of build jobs.\n\nGitLab Entry Point\n~~~~~~~~~~~~~~~~~~\n\nAdd a stack entry point to ``share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml``.\nThere are two stages required for each new stack: the generation stage and the build stage.\n\nThe generate stage is defined using the job template ``.generate`` configured with environment variables defining the name of the stack in ``SPACK_CI_STACK_NAME``, the platform (``SPACK_TARGET_PLATFORM``) and architecture (``SPACK_TARGET_ARCH``) configuration, and the tags associated with the class of runners to build on.\n\n.. note::\n\n The ``SPACK_CI_STACK_NAME`` must match the name of the directory containing the stack's ``spack.yaml`` file.\n\n\n.. note::\n\n The platform and architecture variables are specified in order to select the correct configurations from the generic configurations used in Spack CI.\n The configurations currently available are:\n\n * ``.cray_rhel_zen4``\n * ``.cray_sles_zen4``\n * ``.darwin_aarch64``\n * ``.darwin_x86_64``\n * ``.linux_aarch64``\n * ``.linux_icelake``\n * ``.linux_neoverse_n1``\n * ``.linux_neoverse_v1``\n * ``.linux_neoverse_v2``\n * ``.linux_skylake``\n * ``.linux_x86_64``\n * ``.linux_x86_64_v4``\n\n New configurations can be added to accommodate new platforms and architectures.\n\n\nThe build stage is defined as a trigger job that consumes the GitLab CI pipeline generated in the generate stage for this stack.\nBuild stage jobs use the ``.build`` job template, which handles the basic configuration.\n\nAn example entry point for a new stack called ``my-super-cool-stack``\n\n.. code-block:: yaml\n\n .my-super-cool-stack:\n extends: [\".linux_x86_64_v3\"]\n variables:\n SPACK_CI_STACK_NAME: my-super-cool-stack\n tags: [\"all\", \"tags\", \"your\", \"job\", \"needs\"]\n\n my-super-cool-stack-generate:\n extends: [\".generate\", \".my-super-cool-stack\"]\n image: my-super-cool-stack-image:0.0.1\n\n my-super-cool-stack-build:\n extends: [\".build\", \".my-super-cool-stack\"]\n trigger:\n include:\n - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml\n job: my-super-cool-stack-generate\n strategy: depend\n needs:\n - artifacts: true\n job: my-super-cool-stack-generate\n\n\nStack Configuration\n~~~~~~~~~~~~~~~~~~~\n\nThe stack configuration is a Spack environment file with two additional sections added.\nStack configurations should be located in ``share/spack/gitlab/cloud_pipelines/stacks//spack.yaml``.\n\nThe ``ci`` section is generally used to define stack-specific mappings such as image or tags.\nFor more information on what can go into the ``ci`` section, refer to the docs on pipelines.\n\nThe ``cdash`` section is used for defining where to upload the results of builds.\nSpack configures most of the details for posting pipeline results to `cdash.spack.io `_.\nThe only requirement in the stack configuration is to define a ``build-group`` that is unique; this is usually the long name of the stack.\n\nAn example stack that builds ``zlib``.\n\n.. code-block:: yaml\n\n spack:\n view: false\n packages:\n all:\n require: [\"%gcc\", \"target=x86_64_v3\"]\n specs:\n - zlib\n\n ci:\n pipeline-gen:\n - build-job:\n image: my-super-cool-stack-image:0.0.1\n\n cdash:\n build-group: My Super Cool Stack\n\n.. note::\n\n The ``image`` used in the ``*-generate`` job must match exactly the ``image`` used in the ``build-job``.\n When the images do not match, the build job may fail.\n\n\nRegistering Runners\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nContributing computational resources to Spack's CI build farm is one way to help expand the capabilities and offerings of the public Spack build caches.\nCurrently, Spack utilizes Linux runners from AWS, Google, and the University of Oregon (UO).\n\nRunners require four key pieces:\n\n* Runner Registration Token\n* Accurate tags\n* OIDC Authentication script\n* GPG keys\n\nMinimum GitLab Runner Version: ``16.1.0`` `Installation instructions `_\n\nRegistration Token\n~~~~~~~~~~~~~~~~~~\n\nThe first step to contribute new runners is to open an issue in the `Spack infrastructure `_ project.\nThis will be reported to the Spack infrastructure team, who will guide users through the process of registering new runners for Spack CI.\n\nThe information needed to register a runner is the motivation for the new resources, a semi-detailed description of the runner, and finally the point of contact for maintaining the software on the runner.\n\nThe point of contact will then work with the infrastructure team to obtain runner registration token(s) for interacting with Spack's GitLab instance.\nOnce the runner is active, this point of contact will also be responsible for updating the GitLab runner software to keep pace with Spack's GitLab.\n\nTagging\n~~~~~~~\n\nIn the initial stages of runner registration, it is important to **exclude** the special tag ``spack``.\nThis will prevent the new runner(s) from being picked up for production CI jobs while it is configured and evaluated.\nOnce it is determined that the runner is ready for production use, the ``spack`` tag will be added.\n\nBecause GitLab has no concept of tag exclusion, runners that provide specialized resources also require specialized tags.\nFor example, a basic CPU-only x86_64 runner may have a tag ``x86_64`` associated with it.\nHowever, a runner containing a CUDA-capable GPU may have the tag ``x86_64-cuda`` to denote that it should only be used for packages that will benefit from a CUDA-capable resource.\n\nOIDC\n~~~~\n\nSpack runners use OIDC authentication for connecting to the appropriate AWS bucket, which is used for coordinating the communication of binaries between build jobs.\nIn order to configure OIDC authentication, Spack CI runners use a Python script with minimal dependencies.\nThis script can be configured for runners as seen here using the ``pre_build_script``.\n\n.. code-block:: toml\n\n [[runners]]\n pre_build_script = \"\"\"\n echo 'Executing Spack pre-build setup script'\n\n for cmd in \"${PY3:-}\" python3 python; do\n if command -v > /dev/null \"$cmd\"; then\n export PY3=\"$(command -v \"$cmd\")\"\n break\n fi\n done\n\n if [ -z \"${PY3:-}\" ]; then\n echo \"Unable to find python3 executable\"\n exit 1\n fi\n\n $PY3 -c \"import urllib.request; urllib.request.urlretrieve('https://raw.githubusercontent.com/spack/spack-infrastructure/main/scripts/gitlab_runner_pre_build/pre_build.py', 'pre_build.py')\"\n $PY3 pre_build.py > envvars\n\n . ./envvars\n rm -f envvars\n unset GITLAB_OIDC_TOKEN\n \"\"\"\n\nGPG Keys\n~~~~~~~~\n\nRunners that may be utilized for ``protected`` CI require the registration of an intermediate signing key that can be used to sign packages.\nFor more information on package signing, read :ref:`key_architecture`.\n\nCoverage\n--------\n\nSpack uses `Codecov `_ to generate and report unit test coverage.\nThis helps us tell what percentage of lines of code in Spack are covered by unit tests.\nAlthough code covered by unit tests can still contain bugs, it is much less error-prone than code that is not covered by unit tests.\n\nCodecov provides `browser extensions `_ for Google Chrome and Firefox.\nThese extensions integrate with GitHub and allow you to see coverage line-by-line when viewing the Spack repository.\nIf you are new to Spack, a great way to get started is to write unit tests to increase coverage!\n\nUnlike with CI on GitHub Actions, Codecov tests are not required to pass in order for your PR to be merged.\nIf you modify core Spack libraries, we would greatly appreciate unit tests that cover these changed lines.\nOtherwise, we have no way of knowing whether or not your changes introduce a bug.\nIf you make substantial changes to the core, we may request unit tests to increase coverage.\n\n.. note::\n\n If the only files you modified are package files, we do not care about coverage on your PR.\n You may notice that the Codecov tests fail even though you did not modify any core files.\n This means that Spack's overall coverage has increased since you branched off of ``develop``.\n This is a good thing!\n If you really want to get the Codecov tests to pass, you can rebase off of the latest ``develop``, but again, this is not required.\n\n\nGit Workflows\n-------------\n\nSpack is still in the beta stages of development.\nMost of our users run off of the ``develop`` branch, and fixes and new features are constantly being merged.\nSo, how do you keep up-to-date with upstream while maintaining your own local differences and contributing PRs to Spack?\n\nBranching\n^^^^^^^^^\n\nThe easiest way to contribute a pull request is to make all of your changes on new branches.\nMake sure your ``develop`` branch is up-to-date and create a new branch off of it:\n\n.. code-block:: console\n\n $ git checkout develop\n $ git pull upstream develop\n $ git branch \n $ git checkout \n\nHere we assume that the local ``develop`` branch tracks the upstream ``develop`` branch of Spack.\nThis is not a requirement, and you could also do the same with remote branches.\nBut for some, it is more convenient to have a local branch that tracks upstream.\n\nNormally, we prefer that commits pertaining to a package ```` have a message in the format ``: descriptive message``.\nIt is important to add a descriptive message so that others who might be looking at your changes later (in a year or maybe two) can understand the rationale behind them.\n\nNow, you can make your changes while keeping the ``develop`` branch clean.\nEdit a few files and commit them by running:\n\n.. code-block:: console\n\n $ git add \n $ git commit --message \n\nNext, push it to your remote fork and create a PR:\n\n.. code-block:: console\n\n $ git push origin --set-upstream\n\nGitHub provides a `tutorial `_ on how to file a pull request.\nWhen you send the request, make ``develop`` the destination branch.\n\nIf you need this change immediately and do not have time to wait for your PR to be merged, you can always work on this branch.\nBut if you have multiple PRs, another option is to maintain a \"Frankenstein\" branch that combines all of your other branches:\n\n.. code-block:: console\n\n $ git co develop\n $ git branch \n $ git checkout \n $ git merge \n\nThis can be done with each new PR you submit.\nJust make sure to keep this local branch up-to-date with the upstream ``develop`` branch too.\n\nCherry-Picking\n^^^^^^^^^^^^^^\n\nWhat if you made some changes to your local modified ``develop`` branch and already committed them, but later decided to contribute them to Spack?\nYou can use cherry-picking to create a new branch with only these commits.\n\nFirst, check out your local modified ``develop`` branch:\n\n.. code-block:: console\n\n $ git checkout \n\nNow, get the hashes of the commits you want from the output of ``git log``:\n\n.. code-block:: console\n\n $ git log\n\nNext, create a new branch off of the upstream ``develop`` branch and copy the commits that you want in your PR:\n\n.. code-block:: console\n\n $ git checkout develop\n $ git pull upstream develop\n $ git branch \n $ git checkout \n $ git cherry-pick \n $ git push origin --set-upstream\n\nNow you can create a PR from the web interface of GitHub.\nThe net result is as follows:\n\n#. You patched your local version of Spack and can use it further.\n#. You \"cherry-picked\" these changes into a standalone branch and submitted it as a PR upstream.\n\nShould you have several commits to contribute, you could follow the same procedure by getting hashes of all of them and cherry-picking them to the PR branch.\n\n.. note::\n\n It is important that whenever you change something that might be of importance upstream, create a pull request as soon as possible.\n Do not wait for weeks or months to do this, because:\n\n #. you might forget why you modified certain files.\n #. it could get difficult to isolate this change into a standalone, clean PR.\n\nRebasing\n^^^^^^^^\n\nOther developers are constantly making contributions to Spack, possibly on the same files that your PR changed.\nIf their PR is merged before yours, it can create a merge conflict.\nThis means that your PR can no longer be automatically merged without a chance of breaking your changes.\nIn this case, you will be asked to rebase on top of the latest upstream ``develop`` branch.\n\nFirst, make sure your ``develop`` branch is up-to-date:\n\n.. code-block:: console\n\n $ git checkout develop\n $ git pull upstream develop\n\nNow, we need to switch to the branch you submitted for your PR and rebase it on top of ``develop``:\n\n.. code-block:: console\n\n $ git checkout \n $ git rebase develop\n\nGit will likely ask you to resolve conflicts.\nEdit the file that it says cannot be merged automatically and resolve the conflict.\nThen, run:\n\n.. code-block:: console\n\n $ git add \n $ git rebase --continue\n\nYou may have to repeat this process multiple times until all conflicts are resolved.\nOnce this is done, simply force push your rebased branch to your remote fork:\n\n.. code-block:: console\n\n $ git push --force origin \n\nRebasing with cherry-pick\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nYou can also perform a rebase using ``cherry-pick``.\nFirst, create a temporary backup branch:\n\n.. code-block:: console\n\n $ git checkout \n $ git branch tmp\n\nIf anything goes wrong, you can always go back to your ``tmp`` branch.\nNow, look at the logs and save the hashes of any commits you would like to keep:\n\n.. code-block:: console\n\n $ git log\n\nNext, go back to the original branch and reset it to ``develop``.\nBefore doing so, make sure that your local ``develop`` branch is up-to-date with upstream:\n\n.. code-block:: console\n\n $ git checkout develop\n $ git pull upstream develop\n $ git checkout \n $ git reset --hard develop\n\nNow you can cherry-pick relevant commits:\n\n.. code-block:: console\n\n $ git cherry-pick \n $ git cherry-pick \n\nPush the modified branch to your fork:\n\n.. code-block:: console\n\n $ git push --force origin \n\nIf everything looks good, delete the backup branch:\n\n.. code-block:: console\n\n $ git branch --delete --force tmp\n\nRe-writing History\n^^^^^^^^^^^^^^^^^^\n\nSometimes you may end up on a branch that has diverged so much from ``develop`` that it cannot easily be rebased.\nIf the current commit history is more of an experimental nature and only the net result is important, you may rewrite the history.\n\nFirst, merge upstream ``develop`` and reset your branch to it.\nOn the branch in question, run:\n\n.. code-block:: console\n\n $ git merge develop\n $ git reset develop\n\nAt this point, your branch will point to the same commit as ``develop``, and thereby the two are indistinguishable.\nHowever, all the files that were previously modified will stay as such.\nIn other words, you do not lose the changes you made.\nChanges can be reviewed by looking at diffs:\n\n.. code-block:: console\n\n $ git status\n $ git diff\n\nThe next step is to rewrite the history by adding files and creating commits:\n\n.. code-block:: console\n\n $ git add \n $ git commit --message \n\nAfter all changed files are committed, you can push the branch to your fork and create a PR:\n\n.. code-block:: console\n\n $ git push origin --set-upstream\n" }, { "path": "lib/spack/docs/developer_guide.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n A comprehensive guide for developers working on Spack itself, covering the directory structure, code organization, and key concepts like specs and packages.\n\n.. _developer_guide:\n\nDeveloper Guide\n===============\n\nThis guide is intended for people who want to work on Spack itself.\nIf you just want to develop packages, see the :doc:`Packaging Guide `.\n\nIt is assumed that you have read the :ref:`basic-usage` and :doc:`packaging guide ` sections and that you are familiar with the concepts discussed there.\n\nOverview\n--------\n\nSpack is designed with three separate roles in mind:\n\n#. **Users**, who need to install software *without* knowing all the details about how it is built.\n#. **Packagers**, who know how a particular software package is built and encode this information in package files.\n#. **Developers**, who work on Spack, add new features, and try to make the jobs of packagers and users easier.\n\nUsers could be end-users installing software in their home directory or administrators installing software to a shared directory on a shared machine.\nPackagers could be administrators who want to automate software builds or application developers who want to make their software more accessible to users.\n\nAs you might expect, there are many types of users with different levels of sophistication, and Spack is designed to accommodate both simple and complex use cases for packages.\nA user who only knows that they need a certain package should be able to type something simple, like ``spack install ``, and get the package that they want.\nIf a user wants to ask for a specific version, use particular compilers, or build several versions with different configurations, then that should be possible with a minimal amount of additional specification.\n\nThis gets us to the two key concepts in Spack's software design:\n\n#. **Specs**: expressions for describing builds of software, and\n#. **Packages**: Python modules that build software according to a spec.\n\nA package is a template for building particular software, and a spec is a descriptor for one or more instances of that template.\nUsers express the configuration they want using a spec, and a package turns the spec into a complete build.\n\nThe obvious difficulty with this design is that users underspecify what they want.\nTo build a software package, the package object needs a *complete* specification.\nIn Spack, if a spec describes only one instance of a package, then we say it is **concrete**.\nIf a spec could describe many instances (i.e., it is underspecified in one way or another), then we say it is **abstract**.\n\nSpack's job is to take an *abstract* spec from the user, find a *concrete* spec that satisfies the constraints, and hand the task of building the software off to the package object.\n\nPackages are managed through Spack's **package repositories**, which allow packages to be stored in multiple repositories with different namespaces.\nThe built-in packages are hosted in a separate Git repository and automatically managed by Spack, while custom repositories can be added for organization-specific or experimental packages.\n\nThe rest of this document describes all the pieces that come together to make that happen.\n\nDirectory Structure\n-------------------\n\nSo that you can familiarize yourself with the project, we will start with a high-level view of Spack's directory structure:\n\n.. code-block:: none\n\n spack/ <- installation root\n bin/\n spack <- main spack executable\n\n etc/\n spack/ <- Spack config files.\n Can be overridden by files in ~/.spack.\n\n var/\n spack/\n test_repos/ <- contains package repositories for tests\n cache/ <- saves resources downloaded during installs\n\n opt/\n spack/ <- packages are installed here\n\n lib/\n spack/\n docs/ <- source for this documentation\n\n external/ <- external libs included in Spack distribution\n\n spack/ <- spack module; contains Python code\n build_systems/ <- modules for different build systems\n cmd/ <- each file in here is a Spack subcommand\n compilers/ <- compiler description files\n container/ <- module for spack containerize\n hooks/ <- hook modules to run at different points\n modules/ <- modules for Lmod, Tcl, etc.\n operating_systems/ <- operating system modules\n platforms/ <- different Spack platforms\n reporters/ <- reporters like CDash, JUnit\n schema/ <- schemas to validate data structures\n solver/ <- the Spack solver\n test/ <- unit test modules\n util/ <- common code\n\nSpack is designed so that it could live within a `standard UNIX directory hierarchy `_, so ``lib``, ``var``, and ``opt`` all contain a ``spack`` subdirectory in case Spack is installed alongside other software.\nMost of the interesting parts of Spack live in ``lib/spack``.\n\n.. note::\n\n **Package Repositories**: Built-in packages are hosted in a separate Git repository at `spack/spack-packages `_ and are automatically cloned to ``~/.spack/package_repos/`` when needed.\n The ``var/spack/test_repos/`` directory is used for unit tests only.\n See :ref:`repositories` for details on package repositories.\n\nSpack has *one* directory layout, and there is no installation process.\nMost Python programs do not look like this (they use ``distutils``, ``setup.py``, etc.), but we wanted to make Spack *very* easy to use.\nThe simple layout spares users from the need to install Spack into a Python environment.\nMany users do not have write access to a Python installation, and installing an entire new instance of Python to bootstrap Spack would be very complicated.\nUsers should not have to install a big, complicated package to use the thing that is supposed to spare them from the details of big, complicated packages.\nThe end result is that Spack works out of the box: clone it and add ``bin`` to your ``PATH``, and you are ready to go.\n\nCode Structure\n--------------\n\nThis section gives an overview of the various Python modules in Spack, grouped by functionality.\n\nPackage-related modules\n^^^^^^^^^^^^^^^^^^^^^^^\n\n:mod:`spack.package_base`\n Contains the :class:`~spack.package_base.PackageBase` class, which is the superclass for all packages in Spack.\n\n:mod:`spack.util.naming`\n Contains functions for mapping between Spack package names, Python module names, and Python class names.\n\n:mod:`spack.directives`\n *Directives* are functions that can be called inside a package definition to modify the package, like :func:`~spack.directives.depends_on` and :func:`~spack.directives.provides`.\n See :ref:`dependencies` and :ref:`virtual-dependencies`.\n\n:mod:`spack.multimethod`\n Implementation of the :func:`@when ` decorator, which allows :ref:`multimethods ` in packages.\n\nSpec-related modules\n^^^^^^^^^^^^^^^^^^^^\n\n:mod:`spack.spec`\n Contains :class:`~spack.spec.Spec`.\n Also implements most of the logic for concretization of specs.\n\n:mod:`spack.spec_parser`\n Contains :class:`~spack.spec_parser.SpecParser` and functions related to parsing specs.\n\n:mod:`spack.version`\n Implements a simple :class:`~spack.version.Version` class with simple comparison semantics.\n It also implements :class:`~spack.version.VersionRange` and :class:`~spack.version.VersionList`.\n All three are comparable with each other and offer union and intersection operations.\n Spack uses these classes to compare versions and to manage version constraints on specs.\n Comparison semantics are similar to the ``LooseVersion`` class in ``distutils`` and to the way RPM compares version strings.\n\n:mod:`spack.compilers`\n Submodules contains descriptors for all valid compilers in Spack.\n This is used by the build system to set up the build environment.\n\n .. warning::\n\n Not yet implemented.\n Currently has two compiler descriptions, but compilers aren't fully integrated with the build process yet.\n\nBuild environment\n^^^^^^^^^^^^^^^^^\n\n:mod:`spack.stage`\n Handles creating temporary directories for builds.\n\n:mod:`spack.build_environment`\n This contains utility functions used by the compiler wrapper script, ``cc``.\n\n:mod:`spack.directory_layout`\n Classes that control the way an installation directory is laid out.\n Create more implementations of this to change the hierarchy and naming scheme in ``$spack_prefix/opt``\n\nSpack Subcommands\n^^^^^^^^^^^^^^^^^\n\n:mod:`spack.cmd`\n Each module in this package implements a Spack subcommand.\n See :ref:`writing commands ` for details.\n\nUnit tests\n^^^^^^^^^^\n\n``spack.test``\n Implements Spack's test suite.\n Add a module and put its name in the test suite in ``__init__.py`` to add more unit tests.\n\n\nOther Modules\n^^^^^^^^^^^^^\n\n:mod:`spack.url`\n URL parsing, for deducing names and versions of packages from tarball URLs.\n\n:mod:`spack.error`\n :class:`~spack.error.SpackError`, the base class for Spack's exception hierarchy.\n\n:mod:`spack.llnl.util.tty`\n Basic output functions for all of the messages Spack writes to the terminal.\n\n:mod:`spack.llnl.util.tty.color`\n Implements a color formatting syntax used by ``spack.tty``.\n\n:mod:`spack.llnl.util`\n In this package are a number of utility modules for the rest of Spack.\n\n.. _package-repositories:\n\nPackage Repositories\n^^^^^^^^^^^^^^^^^^^^\n\nSpack's package repositories allow developers to manage packages from multiple sources.\nUnderstanding this system is important for developing Spack itself.\n\n:mod:`spack.repo`\n The core module for managing package repositories.\n Contains the ``Repo`` and ``RepoPath`` classes that handle loading and searching packages from multiple repositories.\n\nBuilt-in packages are stored in a separate Git repository (`spack/spack-packages `_) rather than being included directly in the Spack source tree.\nThis repository is automatically cloned to ``~/.spack/package_repos/`` when needed.\n\nKey concepts:\n\n* **Repository namespaces**: Each repository has a unique namespace (e.g., ``builtin``)\n* **Repository search order**: Packages are found by searching repositories in order\n* **Git-based repositories**: Remote repositories can be automatically cloned and managed\n* **Repository configuration**: Managed through ``repos.yaml`` configuration files\n\nSee :ref:`repositories` for complete details on configuring and managing package repositories.\n\n.. _package_class_structure:\n\nPackage class architecture\n--------------------------\n\n.. note::\n\n This section aims to provide a high-level knowledge of how the package class architecture evolved in Spack, and provides some insights on the current design.\n\nPackages in Spack were originally designed to support only a single build system.\nThe overall class structure for a package looked like:\n\n.. image:: images/original_package_architecture.png\n :scale: 60 %\n :align: center\n\nIn this architecture the base class ``AutotoolsPackage`` was responsible for both the metadata related to the ``autotools`` build system (e.g. dependencies or variants common to all packages using it), and for encoding the default installation procedure.\n\nIn reality, a non-negligible number of packages are either changing their build system during the evolution of the project, or using different build systems for different platforms.\nAn architecture based on a single class requires hacks or other workarounds to deal with these cases.\n\nTo support a model more adherent to reality, Spack v0.19 changed its internal design by extracting the attributes and methods related to building a software into a separate hierarchy:\n\n.. image:: images/builder_package_architecture.png\n :scale: 60 %\n :align: center\n\nIn this new format each ``package.py`` contains one ``*Package`` class that gathers all the metadata, and one or more ``*Builder`` classes that encode the installation procedure.\nA specific builder object is created just before the software is built, so at a time where Spack knows which build system needs to be used for the current installation, and receives a ``package`` object during initialization.\n\nCompatibility with single-class format\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nInternally, Spack always uses builders to perform operations related to the installation of a specific software.\nThe builders are created in the ``spack.builder.create`` function.\n\n.. literalinclude:: _spack_root/lib/spack/spack/builder.py\n :pyobject: create\n\nTo achieve backward compatibility with the single-class format Spack creates in this function a special \"adapter builder\", if no custom builder is detected in the recipe:\n\n.. image:: images/adapter.png\n :scale: 60 %\n :align: center\n\nOverall the role of the adapter is to route access to attributes of methods first through the ``*Package`` hierarchy, and then back to the base class builder.\nThis is schematically shown in the diagram above, where the adapter role is to \"emulate\" a method resolution order like the one represented by the red arrows.\n\n\n.. _writing-commands:\n\nWriting commands\n----------------\n\nAdding a new command to Spack is easy.\nSimply add a ``.py`` file to ``lib/spack/spack/cmd/``, where ```` is the name of the subcommand.\nAt a bare minimum, two functions are required in this file:\n\n``setup_parser()``\n^^^^^^^^^^^^^^^^^^\n\nUnless your command does not accept any arguments, a ``setup_parser()`` function is required to define what arguments and flags your command takes.\nSee the `Argparse documentation `_ for more details on how to add arguments.\n\nSome commands have a set of subcommands, like ``spack compiler find`` or ``spack module lmod refresh``.\nYou can add subparsers to your parser to handle this.\nCheck out ``spack edit --command compiler`` for an example of this.\n\nMany commands take the same arguments and flags.\nThese arguments should be defined in ``lib/spack/spack/cmd/common/arguments.py`` so that they do not need to be redefined in multiple commands.\n\n``()``\n^^^^^^^^^^^^\n\nIn order to run your command, Spack searches for a function with the same name as your command in ``.py``.\nThis is the main method for your command and can call other helper methods to handle common tasks.\n\nRemember, before adding a new command, think to yourself whether or not this new command is actually necessary.\nSometimes, the functionality you desire can be added to an existing command.\nAlso, remember to add unit tests for your command.\nIf it is not used very frequently, changes to the rest of Spack can cause your command to break without sufficient unit tests to prevent this from happening.\n\nWhenever you add/remove/rename a command or flags for an existing command, make sure to update Spack's `Bash tab completion script `_.\n\n\nWriting Hooks\n-------------\n\nA hook is a callback that makes it easy to design functions that run for different events.\nWe do this by defining hook types and then inserting them at different places in the Spack codebase.\nWhenever a hook type triggers by way of a function call, we find all the hooks of that type and run them.\n\nSpack defines hooks by way of a module in the ``lib/spack/spack/hooks`` directory.\nThis module has to be registered in ``lib/spack/spack/hooks/__init__.py`` so that Spack is aware of it.\nThis section will cover the basic kind of hooks and how to write them.\n\nTypes of Hooks\n^^^^^^^^^^^^^^\n\nThe following hooks are currently implemented to make it easy for you, the developer, to add hooks at different stages of a Spack install or similar.\nIf there is a hook that you would like and it is missing, you can propose to add a new one.\n\n``pre_install(spec)``\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nA ``pre_install`` hook is run within the install subprocess, directly before the installation starts.\nIt expects a single argument of a spec.\n\n\n``post_install(spec, explicit=None)``\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nA ``post_install`` hook is run within the install subprocess, directly after the installation finishes, but before the build stage is removed and the spec is registered in the database.\nIt expects two arguments: the spec and an optional boolean indicating whether this spec is being installed explicitly.\n\n``pre_uninstall(spec)`` and ``post_uninstall(spec)``\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nThese hooks are currently used for cleaning up module files after uninstall.\n\n\nAdding a New Hook Type\n^^^^^^^^^^^^^^^^^^^^^^\n\nAdding a new hook type is very simple!\nIn ``lib/spack/spack/hooks/__init__.py``, you can simply create a new ``HookRunner`` that is named to match your new hook.\nFor example, let's say you want to add a new hook called ``post_log_write`` to trigger after anything is written to a logger.\nYou would add it as follows:\n\n.. code-block:: python\n\n # pre/post install and run by the install subprocess\n pre_install = HookRunner(\"pre_install\")\n post_install = HookRunner(\"post_install\")\n\n # hooks related to logging\n post_log_write = HookRunner(\"post_log_write\") # <- here is my new hook!\n\n\nYou then need to decide what arguments your hook would expect.\nSince this is related to logging, let's say that you want a message and level.\nThat means that when you add a Python file to the ``lib/spack/spack/hooks`` folder with one or more callbacks intended to be triggered by this hook, you might use your new hook as follows:\n\n.. code-block:: python\n\n def post_log_write(message, level):\n \"\"\"Do something custom with the message and level every time we write\n to the log\n \"\"\"\n print(\"running post_log_write!\")\n\n\nTo use the hook, we would call it as follows somewhere in the logic to do logging.\nIn this example, we use it outside of a logger that is already defined:\n\n.. code-block:: python\n\n import spack.hooks\n\n # We do something here to generate a logger and message\n spack.hooks.post_log_write(message, logger.level)\n\n\nThis is not to say that this would be the best way to implement an integration with the logger (you would probably want to write a custom logger, or you could have the hook defined within the logger), but it serves as an example of writing a hook.\n\nUnit tests\n----------\n\nUnit testing\n------------\n\nDebugging Unit Tests in CI\n--------------------------\n\nSpack runs its CI for unit tests via Github Actions from the Spack repo.\nThe unit tests are run for each platform Spack supports, Windows, Linux, and MacOS.\nIt may be the case that a unit test fails or passes on just one of these platforms.\nWhen the platform is one the PR author does not have access to, it can be difficult to reproduce, diagnose, and fix a CI failure.\nThankfully, PR authors can take advantage of a Github Actions Action to gain temporary access to the failing platform from the context of their PRs.\nSimply copy the following Github actions yaml stanza into `the GHA workflow file `__ in the `steps` section of whatever unit test needs debugging.\n\n.. code-block:: yaml\n\n - name: Setup tmate session\n uses: mxschmitt/action-tmate@c0afd6f790e3a5564914980036ebf83216678101\n\nIdeally this would be inserted somewhere after GHA checks out Spack and does any setup, but before the unit tests themselves are run.\nYou can of course put this stanza after the unit-tests, but then you'll be stuck waiting for the unit tests to complete (potentially up to ~30m) and will need to add additional logic to the yaml in the case where the unit tests fail.\n\nFor example, if you were to add this step to the Linux unit test CI, it would look something like:\n\n.. code-block:: yaml\n\n - name: Bootstrap clingo\n if: ${{ matrix.concretizer == 'clingo' }}\n env:\n SPACK_PYTHON: python\n run: |\n . share/spack/setup-env.sh\n spack bootstrap disable spack-install\n spack bootstrap now\n spack -v solve zlib\n - name: Setup tmate session\n uses: mxschmitt/action-tmate@c0afd6f790e3a5564914980036ebf83216678101\n - name: Run unit tests\n env:\n SPACK_PYTHON: python\n SPACK_TEST_PARALLEL: 4\n COVERAGE: true\n COVERAGE_FILE: coverage/.coverage-${{ matrix.os }}-python${{ matrix.python-version }}\n UNIT_TEST_COVERAGE: ${{ matrix.python-version == '3.14' }}\n run: |-\n share/spack/qa/run-unit-tests\n\n\nNote that the ssh session comes after Spack does its setup but before it runs the unit tests.\n\nOnce this step is present in the job definition, it will be triggered for each CI run.\nThis action provides access to an SSH server running on the GHA runner that is hosting a given CI run.\nAs the action runs, you should observe output similar to:\n\n.. code-block:: console\n\n ssh 5RjFs7LPdtwGG8cwSPkGrdMNg@sfo2.tmate.io\n https://tmate.io/t/5RjFs7LPdtwGG8cwSPkGrdMNg\n\nThe first line is the ssh command necessary to connect to the server, the second line is a tmate web UI that also provides access to the ssh server on the runner.\n\n.. note:: The web UI has occasionally been unresponsive, if it does not respond within ~10s, you'll need to use your local ssh utility.\n\nOnce connected via SSH, you have the same level of access to the machine that the CI job's user does.\nSpack's source should be available already (depending on where the step was inserted).\nSo you can just setup the shell to run Spack via the setup scripts and then debug as needed.\n\n.. note:: If you have configured your Github profile with SSH keys, the action will be aware of this and require those keys to access the SSH session.\n\n.. note:: If you are on Windows you'll be dropped into an MSYS shell, Spack is not supported inside MSYS, so it is strongly recommended to drop into a CMD or powershell prompt.\n\nYou will have access to this ssh session for as long as Github allows a job to be alive.\n\nOnce you have finished debugging, remove this action from the Github actions workflow.\n\nIf you want to continue a workflow and you are inside a session, just create a empty file with the name continue either in the root directory or in the project directory.\n\nThis action has a few option to configure behavior like ssh key handling, tmate server, detached mode, etc.\nFor more on how to use those options, see the actions docs at https://github.com/mxschmitt/action-tmate\n\nDeveloper environment\n---------------------\n\n.. warning::\n\n This is an experimental feature.\n It is expected to change and you should not use it in a production environment.\n\n\nWhen installing a package, we currently have support to export environment variables to specify adding debug flags to the build.\nBy default, a package installation will build without any debug flags.\nHowever, if you want to add them, you can export:\n\n.. code-block:: console\n\n export SPACK_ADD_DEBUG_FLAGS=true\n spack install zlib\n\n\nIf you want to add custom flags, you should export an additional variable:\n\n.. code-block:: console\n\n export SPACK_ADD_DEBUG_FLAGS=true\n export SPACK_DEBUG_FLAGS=\"-g\"\n spack install zlib\n\nThese environment variables will eventually be integrated into Spack so they are set from the command line.\n\nDeveloper commands\n------------------\n\n.. _cmd-spack-doc:\n\n``spack doc``\n^^^^^^^^^^^^^\n\n.. _cmd-spack-style:\n\n``spack style``\n^^^^^^^^^^^^^^^\n\n``spack style`` exists to help the developer check imports and style with mypy, Flake8, isort, and (soon) Black.\nTo run all style checks, simply do:\n\n.. code-block:: console\n\n $ spack style\n\nTo run automatic fixes for isort, you can do:\n\n.. code-block:: console\n\n $ spack style --fix\n\nYou do not need any of these Python packages installed on your system for the checks to work!\nSpack will bootstrap install them from packages for your use.\n\n``spack unit-test``\n^^^^^^^^^^^^^^^^^^^\n\nSee the :ref:`contributor guide section ` on ``spack unit-test``.\n\n.. _cmd-spack-python:\n\n``spack python``\n^^^^^^^^^^^^^^^^\n\n``spack python`` is a command that lets you import and debug things as if you were in a Spack interactive shell.\nWithout any arguments, it is similar to a normal interactive Python shell, except you can import ``spack`` and any other Spack modules:\n\n.. code-block:: console\n\n $ spack python\n >>> from spack.version import Version\n >>> a = Version(\"1.2.3\")\n >>> b = Version(\"1_2_3\")\n >>> a == b\n True\n >>> c = Version(\"1.2.3b\")\n >>> c > a\n True\n >>>\n\nIf you prefer using an IPython interpreter, given that IPython is installed, you can specify the interpreter with ``-i``:\n\n.. code-block:: console\n\n $ spack python -i ipython\n In [1]:\n\n\nWith either interpreter you can run a single command:\n\n.. code-block:: console\n\n $ spack python -c 'from spack.concretize import concretize_one; concretize_one(\"python\")'\n ...\n\n $ spack python -i ipython -c 'from spack.concretize import concretize_one; concretize_one(\"python\")'\n Out[1]: ...\n\nor a file:\n\n.. code-block:: console\n\n $ spack python ~/test_fetching.py\n $ spack python -i ipython ~/test_fetching.py\n\njust like you would with the normal Python command.\n\n\n.. _cmd-spack-blame:\n\n``spack blame``\n^^^^^^^^^^^^^^^\n\n``spack blame`` is a way to quickly see contributors to packages or files in Spack's source tree.\nFor built-in packages, this shows contributors to the package files in the separate ``spack/spack-packages`` repository.\nYou should provide a target package name or file name to the command.\nHere is an example asking to see contributions for the package \"python\":\n\n.. code-block:: console\n\n $ spack blame python\n LAST_COMMIT LINES % AUTHOR EMAIL\n 2 weeks ago 3 0.3 Mickey Mouse \n a month ago 927 99.7 Minnie Mouse \n\n 2 weeks ago 930 100.0\n\n\nBy default, you will get a table view (shown above) sorted by date of contribution, with the most recent contribution at the top.\nIf you want to sort instead by percentage of code contribution, then add ``-p``:\n\n.. code-block:: console\n\n $ spack blame -p python\n\n\nAnd to see the Git blame view, add ``-g`` instead:\n\n\n.. code-block:: console\n\n $ spack blame -g python\n\n\nFinally, to get a JSON export of the data, add ``--json``:\n\n.. code-block:: console\n\n $ spack blame --json python\n\n\n.. _cmd-spack-url:\n\n``spack url``\n^^^^^^^^^^^^^\n\nA package containing a single URL can be used to download several different versions of the package.\nIf you have ever wondered how this works, all of the magic is in :mod:`spack.url`.\nThis module contains methods for extracting the name and version of a package from its URL.\nThe name is used by ``spack create`` to guess the name of the package.\nBy determining the version from the URL, Spack can replace it with other versions to determine where to download them from.\n\nThe regular expressions in ``parse_name_offset`` and ``parse_version_offset`` are used to extract the name and version, but they are not perfect.\nIn order to debug Spack's URL parsing support, the ``spack url`` command can be used.\n\n\n.. _cmd-spack-url-parse:\n\n``spack url parse``\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nIf you need to debug a single URL, you can use the following command:\n\n.. command-output:: spack url parse http://cache.ruby-lang.org/pub/ruby/2.2/ruby-2.2.0.tar.gz\n\nYou will notice that the name and version of this URL are correctly detected, and you can even see which regular expressions it was matched to.\nHowever, you will notice that when it substitutes the version number in, it does not replace the ``2.2`` with ``9.9`` where we would expect ``9.9.9b`` to live.\nThis particular package may require a ``list_url`` or ``url_for_version`` function.\n\nThis command also accepts a ``--spider`` flag.\nIf provided, Spack searches for other versions of the package and prints the matching URLs.\n\n\n.. _cmd-spack-url-list:\n\n``spack url list``\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nThis command lists every URL in every package in Spack.\nIf given the ``--color`` and ``--extrapolation`` flags, it also colors the part of the string that it detected to be the name and version.\nThe ``--incorrect-name`` and ``--incorrect-version`` flags can be used to print URLs that were not being parsed correctly.\n\n\n.. _cmd-spack-url-summary:\n\n``spack url summary``\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nThis command attempts to parse every URL for every package in Spack and prints a summary of how many of them are being correctly parsed.\nIt also prints a histogram showing which regular expressions are being matched and how frequently:\n\n.. command-output:: spack url summary\n\nThis command is essential for anyone adding or changing the regular expressions that parse names and versions.\nBy running this command before and after the change, you can make sure that your regular expression fixes more packages than it breaks.\n\nProfiling\n---------\n\nTo profile Spack, use Python's built-in `cProfile `_ module directly:\n\n.. code-block:: console\n\n $ python3 -m cProfile -s cumtime bin/spack find\n $ python3 -m cProfile -o profile.out bin/spack find\n\n.. _releases:\n\nReleases\n--------\n\nThis section documents Spack's release process.\nIt is intended for project maintainers, as the tasks described here require maintainer privileges on the Spack repository.\nFor others, we hope this section at least provides some insight into how the Spack project works.\n\n.. _release-branches:\n\nRelease branches\n^^^^^^^^^^^^^^^^\n\nThere are currently two types of Spack releases: :ref:`minor releases ` (``1.1.0``, ``1.2.0``, etc.) and :ref:`patch releases ` (``1.1.1``, ``1.1.2``, ``1.1.3``, etc.).\nHere is a diagram of how Spack release branches work:\n\n.. code-block:: text\n\n o branch: develop (latest version, v1.2.0.dev0)\n |\n o\n | o branch: releases/v1.1, tag: v1.1.1\n o |\n | o tag: v1.1.0\n o |\n | o\n |/\n o\n |\n o\n | o branch: releases/v1.0, tag: v1.0.2\n o |\n | o tag: v1.0.1\n o |\n | o tag: v1.0.0\n o |\n | o\n |/\n o\n\nThe ``develop`` branch has the latest contributions, and nearly all pull requests target ``develop``.\nThe ``develop`` branch will report that its version is that of the next **minor** release with a ``.dev0`` suffix.\n\nEach Spack release series also has a corresponding branch, e.g., ``releases/v1.1`` has ``v1.1.x`` versions of Spack, and ``releases/v1.0`` has ``v1.0.x`` versions.\nA minor release is the first tagged version on a release branch.\nPatch releases are back-ported from develop onto release branches.\nThis is typically done by cherry-picking bugfix commits off of ``develop``.\n\nTo avoid version churn for users of a release series, patch releases **should not** make changes that would change the concretization of packages.\nThey should generally only contain fixes to the Spack core.\nHowever, sometimes priorities are such that new functionality needs to be added to a patch release.\n\nBoth minor and patch releases are tagged.\nAs a convenience, we also tag the latest release as ``releases/latest``, so that users can easily check it out to get the latest stable version.\nSee :ref:`updating-latest-release` for more details.\n\n.. admonition:: PEP 440 compliance\n :class: note\n\n Spack releases up to ``v0.17`` were merged back into the ``develop`` branch to ensure that release tags would appear among its ancestors.\n Since ``v0.18`` we opted to have a linear history of the ``develop`` branch, for reasons explained `here `_.\n At the same time, we converted to using `PEP 440 `_ compliant versions.\n\nScheduling work for releases\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nWe schedule work for **minor releases** through `milestones `_ and `GitHub Projects `_, while **patch releases** use `labels `_.\n\nWhile there can be multiple milestones open at a given time, only one is usually active.\nIts name corresponds to the next major/minor version, for example ``v1.1.0``.\nImportant issues and pull requests should be assigned to this milestone by core developers, so that they are not forgotten at the time of release.\nThe milestone is closed when the release is made, and a new milestone is created for the next major/minor release, if not already there.\n\nBug reports in GitHub issues are automatically labelled ``bug`` and ``triage``.\nSpack developers assign one of the labels ``impact-low``, ``impact-medium`` or ``impact-high``.\nThis will make the issue appear in the `Triaged bugs `_ project board.\nImportant issues should be assigned to the next milestone as well, so they appear at the top of the project board.\n\nSpack's milestones are not firm commitments so we move work between releases frequently.\nIf we need to make a release and some tasks are not yet done, we will simply move them to the next minor release milestone, rather than delaying the release to complete them.\n\nBackporting bug fixes\n^^^^^^^^^^^^^^^^^^^^^\n\nWhen a bug is fixed in the ``develop`` branch, it is often necessary to backport the fix to one (or more) of the ``releases/vX.Y`` branches.\nOnly the release manager is responsible for doing backports, but Spack maintainers are responsible for labelling pull requests (and issues if no bug fix is available yet) with ``vX.Y.Z`` labels.\nThe labels should correspond to the future patch versions that the bug fix should be backported to.\n\nBackports are done publicly by the release manager using a pull request named ``Backports vX.Y.Z``.\nThis pull request is opened from the ``backports/vX.Y.Z`` branch, targets the ``releases/vX.Y`` branch and contains a (growing) list of cherry-picked commits from the ``develop`` branch.\nTypically there are one or two backport pull requests open at any given time.\n\n.. _minor-releases:\n\nMaking minor releases\n^^^^^^^^^^^^^^^^^^^^^\n\nAssuming all required work from the milestone is completed, the steps to make the minor release are:\n\n#. `Create a new milestone `_ for the next major/minor release.\n\n#. `Create a new label `_ for the next patch release.\n\n#. Move any optional tasks that are not done to the next milestone.\n\n#. Create a branch for the release, based on ``develop``:\n\n .. code-block:: console\n\n $ git checkout -b releases/v1.1 develop\n\n For a version ``vX.Y.Z``, the branch's name should be ``releases/vX.Y``.\n That is, you should create a ``releases/vX.Y`` branch if you are preparing the ``X.Y.0`` release.\n\n#. Remove the ``dev0`` development release segment from the version tuple in ``lib/spack/spack/__init__.py``.\n\n The version number itself should already be correct and should not be modified.\n\n#. Update ``CHANGELOG.md`` with major highlights in bullet form.\n\n Use proper Markdown formatting, like `this example from v1.0.0 `_.\n\n#. Push the release branch to GitHub.\n\n#. Make sure CI passes on the release branch, including:\n\n * Regular unit tests\n * Build tests\n * The E4S pipeline at `gitlab.spack.io `_\n\n If CI is not passing, submit pull requests to ``develop`` as normal and keep rebasing the release branch on ``develop`` until CI passes.\n\n#. Make sure the entire documentation is up to date.\n If documentation is outdated, submit pull requests to ``develop`` as normal and keep rebasing the release branch on ``develop``.\n\n#. Bump the minor version in the ``develop`` branch.\n\n Create a pull request targeting the ``develop`` branch, bumping the minor version in ``lib/spack/spack/__init__.py`` with a ``dev0`` release segment.\n For instance, when you have just released ``v1.1.0``, set the version to ``(1, 2, 0, 'dev0')`` on ``develop``.\n\n#. Follow the steps in :ref:`publishing-releases`.\n\n#. Follow the steps in :ref:`updating-latest-release`.\n\n#. Follow the steps in :ref:`announcing-releases`.\n\n\n.. _patch-releases:\n\nMaking patch releases\n^^^^^^^^^^^^^^^^^^^^^\n\nTo make the patch release process both efficient and transparent, we use a *backports pull request* which contains cherry-picked commits from the ``develop`` branch.\nThe majority of the work is to cherry-pick the bug fixes, which ideally should be done as soon as they land on ``develop``; this ensures cherry-picking happens in order and makes conflicts easier to resolve since the changes are fresh in the mind of the developer.\n\nThe backports pull request is always titled ``Backports vX.Y.Z`` and is labelled ``backports``.\nIt is opened from a branch named ``backports/vX.Y.Z`` and targets the ``releases/vX.Y`` branch.\n\nThe first commit on the ``backports/vX.Y.Z`` branch should update the Spack version to ``X.Y.Z.dev0``, and should have the commit message ``set version to X.Y.Z.dev0``.\nThis ensures that if users check out an intermediate commit between two patch releases, Spack reports the version correctly.\n\nWhenever a pull request labelled ``vX.Y.Z`` is merged, cherry-pick the associated squashed commit on ``develop`` to the ``backports/vX.Y.Z`` branch.\nFor pull requests that were rebased (or not squashed), cherry-pick each associated commit individually.\nNever force-push to the ``backports/vX.Y.Z`` branch.\n\n.. warning::\n\n Sometimes you may **still** get merge conflicts even if you have cherry-picked all the commits in order.\n This generally means there is some other intervening pull request that the one you are trying to pick depends on.\n In these cases, you will need to make a judgment call regarding those pull requests.\n Consider the number of affected files and/or the resulting differences.\n\n 1. If the changes are small, you might just cherry-pick it.\n\n 2. If the changes are large, then you may decide that this fix is not worth including in a patch release, in which case you should remove the label from the pull request.\n Remember that large, manual backports are seldom the right choice for a patch release.\n\nWhen all commits are cherry-picked in the ``backports/vX.Y.Z`` branch, make the patch release as follows:\n\n#. `Create a new label `_ ``vX.Y.{Z+1}`` for the next patch release.\n\n#. Replace the label ``vX.Y.Z`` with ``vX.Y.{Z+1}`` for all PRs and issues that are not yet done.\n\n#. Manually push a single commit with commit message ``Set version to vX.Y.Z`` to the ``backports/vX.Y.Z`` branch, that both bumps the Spack version number and updates the changelog:\n\n 1. Bump the version in ``lib/spack/spack/__init__.py``.\n 2. Update ``CHANGELOG.md`` with a list of the changes.\n\n This is typically a summary of the commits you cherry-picked onto the release branch.\n See `the changelog from v1.0.2 `_.\n\n#. Make sure CI passes on the **backports pull request**, including:\n\n * Regular unit tests\n * Build tests\n * The E4S pipeline at `gitlab.spack.io `_\n\n#. Merge the ``Backports vX.Y.Z`` PR with the **Rebase and merge** strategy.\n This is needed to keep track in the release branch of all the commits that were cherry-picked.\n\n#. Make sure CI passes on the last commit of the **release branch**.\n\n#. In the rare case you need to include additional commits in the patch release after the backports PR is merged, it is best to delete the last commit ``Set version to vX.Y.Z`` from the release branch with a single force-push, open a new backports PR named ``Backports vX.Y.Z (2)``, and repeat the process.\n Avoid repeated force-pushes to the release branch.\n\n#. Follow the steps in :ref:`publishing-releases`.\n\n#. Follow the steps in :ref:`updating-latest-release`.\n\n#. Follow the steps in :ref:`announcing-releases`.\n\n#. Submit a PR to update the ``CHANGELOG.md`` in the ``develop`` branch with the addition of this patch release.\n\n.. _publishing-releases:\n\nPublishing a release on GitHub\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n#. Create the release in GitHub.\n\n * Go to `github.com/spack/spack/releases `_ and click ``Draft a new release``.\n\n * Set ``Tag version`` to the name of the tag that will be created.\n\n The name should start with ``v`` and contain *all three* parts of the version (e.g., ``v1.1.0`` or ``v1.1.1``).\n\n * Set ``Target`` to the ``releases/vX.Y`` branch (e.g., ``releases/v1.0``).\n\n * Set ``Release title`` to ``vX.Y.Z`` to match the tag (e.g., ``v1.0.1``).\n\n * Paste the latest release Markdown from your ``CHANGELOG.md`` file as the text.\n\n * Save the draft so you can keep coming back to it as you prepare the release.\n\n#. When you are ready to finalize the release, click ``Publish release``.\n\n#. Immediately after publishing, go back to `github.com/spack/spack/releases `_ and download the auto-generated ``.tar.gz`` file for the release.\n It is the ``Source code (tar.gz)`` link.\n\n#. Click ``Edit`` on the release you just made and attach the downloaded release tarball as a binary.\n This does two things:\n\n #. Makes sure that the hash of our releases does not change over time.\n\n GitHub sometimes annoyingly changes the way they generate tarballs that can result in the hashes changing if you rely on the auto-generated tarball links.\n\n #. Gets download counts on releases visible through the GitHub API.\n\n GitHub tracks downloads of artifacts, but *not* the source links.\n See the `releases page `_ and search for ``download_count`` to see this.\n\n#. Go to `readthedocs.org `_ and activate the release tag.\n\n This builds the documentation and makes the released version selectable in the versions menu.\n\n\n.. _updating-latest-release:\n\nUpdating `releases/latest`\n^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nIf the new release is the **highest** Spack release yet, you should also tag it as ``releases/latest``.\nFor example, suppose the highest release is currently ``v1.1.3``:\n\n* If you are releasing ``v1.1.4`` or ``v1.2.0``, then you should tag it with ``releases/latest``, as these are higher than ``v1.1.3``.\n\n* If you are making a new release of an **older** minor version of Spack, e.g., ``v1.0.5``, then you should not tag it as ``releases/latest`` (as there are newer major/minor versions).\n\nTo do so, first fetch the latest tag created on GitHub, since you may not have it locally:\n\n.. code-block:: console\n\n $ git fetch --force git@github.com:spack/spack tag vX.Y.Z\n\nThen tag ``vX.Y.Z`` as ``releases/latest`` and push the individual tag to GitHub.\n\n.. code-block:: console\n\n $ git tag --force releases/latest vX.Y.Z\n $ git push --force git@github.com:spack/spack releases/latest\n\nThe ``--force`` argument to ``git tag`` makes Git overwrite the existing ``releases/latest`` tag with the new one.\nDo **not** use the ``--tags`` flag when pushing, as this will push *all* local tags.\n\n\n.. _announcing-releases:\n\nAnnouncing a release\n^^^^^^^^^^^^^^^^^^^^\n\nWe announce releases in all of the major Spack communication channels.\nPublishing the release takes care of GitHub.\nThe remaining channels are X, Slack, and the mailing list.\nHere are the steps:\n\n#. Announce the release on X.\n\n * Compose the tweet on the ``@spackpm`` account per the ``spack-twitter`` slack channel.\n\n * Be sure to include a link to the release's page on GitHub.\n\n You can base the tweet on `this example `_.\n\n#. Announce the release on Slack.\n\n * Compose a message in the ``#announcements`` Slack channel (`spackpm.slack.com `_).\n\n * Preface the message with ``@channel`` to notify even those people not currently logged in.\n\n * Be sure to include a link to the tweet above.\n\n The tweet will be shown inline so that you do not have to retype your release announcement.\n\n#. Announce the release on the Spack mailing list.\n\n * Compose an email to the Spack mailing list.\n\n * Be sure to include a link to the release's page on GitHub.\n\n * It is also helpful to include some information directly in the email.\n\n You can base your announcement on this `example email `_.\n\nOnce you have completed the above steps, congratulations, you are done!\nYou have finished making the release!\n" }, { "path": "lib/spack/docs/env_vars_yaml.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Learn how to modify shell environment variables within a Spack environment using the env_vars.yaml file.\n\n.. _env-vars-yaml:\n\nEnvironment Variable Settings (env_vars.yaml)\n=============================================\n\nSpack allows you to include shell environment variable modifications for a Spack environment by including an ``env_vars.yaml`` file.\nEnvironment variables can be modified by setting, unsetting, appending, and prepending variables in the shell environment.\nThe changes to the shell environment will take effect when the Spack environment is activated.\n\nFor example:\n\n.. code-block:: yaml\n\n env_vars:\n set:\n ENVAR_TO_SET_IN_ENV_LOAD: \"FOO\"\n unset:\n - ENVAR_TO_UNSET_IN_ENV_LOAD\n prepend_path:\n PATH_LIST: \"path/to/prepend\"\n append_path:\n PATH_LIST: \"path/to/append\"\n remove_path:\n PATH_LIST: \"path/to/remove\"\n\n\n" }, { "path": "lib/spack/docs/environments.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Learn how to use Spack environments to manage reproducible software stacks, making it easy to share and recreate specific sets of packages and their dependencies.\n\n.. _environments:\n\nEnvironments (spack.yaml, spack.lock)\n=====================================\n\nAn environment is used to group a set of specs intended for some purpose to be built, rebuilt, and deployed in a coherent fashion.\nEnvironments define aspects of the installation of the software, such as:\n\n#. *which* specs to install;\n#. *how* those specs are configured; and\n#. *where* the concretized software will be installed.\n\nAggregating this information into an environment for processing has advantages over the *à la carte* approach of building and loading individual Spack modules.\n\nWith environments, you concretize, install, or load (activate) all of the specs with a single command.\nConcretization fully configures the specs and dependencies of the environment in preparation for installing the software.\nThis is a more robust solution than ad-hoc installation scripts.\nAnd you can share an environment or even re-use it on a different computer.\n\nEnvironment definitions, especially *how* specs are configured, allow the software to remain stable and repeatable even when Spack packages are upgraded.\nChanges are only picked up when the environment is explicitly re-concretized.\n\nDefining *where* specs are installed supports a filesystem view of the environment.\nYet Spack maintains a single installation of the software that can be re-used across multiple environments.\n\nActivating an environment determines *when* all of the associated (and installed) specs are loaded so limits the software loaded to those specs actually needed by the environment.\nSpack can even generate a script to load all modules related to an environment.\n\nOther packaging systems also provide environments that are similar in some ways to Spack environments; for example, `Conda environments `_ or `Python Virtual Environments `_.\nSpack environments provide some distinctive features though:\n\n#. A spec installed \"in\" an environment is no different from the same spec installed anywhere else in Spack.\n#. Spack environments may contain more than one spec of the same package.\n\nSpack uses a \"manifest and lock\" model similar to `Bundler gemfiles `_ and other package managers.\nThe environment's user input file (or manifest), is named ``spack.yaml``.\nThe lock file, which contains the fully configured and concretized specs, is named ``spack.lock``.\n\n.. _environments-using:\n\nUsing Environments\n------------------\n\nHere we follow a typical use case of creating, concretizing, installing and loading an environment.\n\n.. _cmd-spack-env-create:\n\nCreating a managed Environment\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nAn environment is created by:\n\n.. code-block:: console\n\n $ spack env create myenv\n\nThe directory ``$SPACK_ROOT/var/spack/environments/myenv`` is created to manage the environment.\n\n.. note::\n\n By default, all managed environments are stored in the ``$SPACK_ROOT/var/spack/environments`` folder.\n This location can be changed by setting the ``environments_root`` variable in ``config.yaml``.\n\nSpack creates the file ``spack.yaml``, hidden directory ``.spack-env``, and ``spack.lock`` file under ``$SPACK_ROOT/var/spack/environments/myenv``.\nUser interaction occurs through the ``spack.yaml`` file and the Spack commands that affect it.\nMetadata and, by default, the view are stored in the ``.spack-env`` directory.\nWhen the environment is concretized, Spack creates the ``spack.lock`` file with the fully configured specs and dependencies for the environment.\n\nThe ``.spack-env`` subdirectory also contains:\n\n* ``repo/``: A subdirectory acting as the repo consisting of the Spack packages used in the environment.\n It allows the environment to build the same, in theory, even on different versions of Spack with different packages!\n* ``logs/``: A subdirectory containing the build logs for the packages in this environment.\n\nSpack Environments can also be created from another environment.\nEnvironments can be created from the manifest file (the user input), the lockfile, or the entire environment at once.\nCreate an environment from a manifest using:\n\n.. code-block:: console\n\n $ spack env create myenv spack.yaml\n\nThe resulting environment is guaranteed to have the same root specs as the original but may concretize differently in the presence of different explicit or default configuration settings (e.g., a different version of Spack or for a different user account).\n\nEnvironments created from a manifest will copy any included configs from relative paths inside the environment.\nRelative paths from outside the environment will cause errors, and absolute paths will be kept absolute.\nFor example, if ``spack.yaml`` includes:\n\n.. code-block:: yaml\n\n spack:\n include: [./config.yaml]\n\nthen the created environment will have its own copy of the file ``config.yaml`` copied from the location in the original environment.\n\nCreate an environment from a ``spack.lock`` file using:\n\n.. code-block:: console\n\n $ spack env create myenv spack.lock\n\nThe resulting environment, when on the same or a compatible machine, is guaranteed to initially have the same concrete specs as the original.\n\nCreate an environment from an entire environment using either the environment name or path:\n\n.. code-block:: console\n\n $ spack env create myenv /path/to/env\n $ spack env create myenv2 myenv\n\nThe resulting environment will include the concrete specs from the original if the original is concretized (as when created from a lockfile) and all of the config options and abstract specs specified in the original (as when created from a manifest file).\nIt will also include any other files included in the environment directory, such as repos or source code, as they could be referenced in the environment by relative path.\n\n.. note::\n\n Environment creation also accepts a full path to the file.\n\n If the path is not under the ``$SPACK_ROOT/var/spack/environments`` directory then the source is referred to as an :ref:`independent environment `.\n\nThe name of an environment can be a nested path to help organize environments via subdirectories.\n\n.. code-block:: console\n\n $ spack env create projectA/configA/myenv\n\nThis will create a managed environment under ``$environments_root/projectA/configA/myenv``.\nChanging ``environment_root`` can therefore also be used to make a whole group of nested environments available.\n\n.. _cmd-spack-env-activate:\n.. _cmd-spack-env-deactivate:\n\nActivating an Environment\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nTo activate an environment, use the following command:\n\n.. code-block:: console\n\n $ spack env activate myenv\n\nBy default, the ``spack env activate`` will load the view associated with the environment into the user environment.\nThe ``-v, --with-view`` argument ensures this behavior, and the ``-V, --without-view`` argument activates the environment without changing the user environment variables.\n\nThe ``-p`` option to the ``spack env activate`` command modifies the user's prompt to begin with the environment name in brackets.\n\n.. code-block:: console\n\n $ spack env activate -p myenv\n [myenv] $ ...\n\nThe ``activate`` command can also be used to create a new environment, if it is not already defined, by adding the ``--create`` flag.\nManaged and independent environments can both be created using the same flags that `spack env create` accepts.\nIf an environment already exists then Spack will simply activate it and ignore the create-specific flags.\n\n.. code-block:: console\n\n $ spack env activate --create -p myenv\n # ...\n # [creates if myenv does not exist yet]\n # ...\n [myenv] $ ...\n\nTo deactivate an environment, use the command:\n\n.. code-block:: console\n\n $ spack env deactivate\n\nor the shortcut alias\n\n.. code-block:: console\n\n $ despacktivate\n\nIf the environment was activated with its view, deactivating the environment will remove the view from the user environment.\n\n.. _independent_environments:\n\nIndependent Environments\n^^^^^^^^^^^^^^^^^^^^^^^^\n\nIndependent environments can be located in any directory outside of Spack.\n\n.. note::\n\n When uninstalling packages, Spack asks the user to confirm the removal of packages that are still used in a managed environment.\n This is not the case for independent environments.\n\nTo create an independent environment, use one of the following commands:\n\n.. code-block:: console\n\n $ spack env create --dir my_env\n $ spack env create ./my_env\n\nAs a shorthand, you can also create an independent environment upon activation if it does not already exist:\n\n.. code-block:: console\n\n $ spack env activate --create ./my_env\n\nFor convenience, Spack can also place an independent environment in a temporary directory for you:\n\n.. code-block:: console\n\n $ spack env activate --temp\n\n\nEnvironment-Aware Commands\n^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nSpack commands are environment-aware.\nFor example, the ``find`` command shows only the specs in the active environment if an environment has been activated.\nOtherwise it shows all specs in the Spack instance.\nThe same rule applies to the ``install`` and ``uninstall`` commands.\n\n.. code-block:: spec\n\n $ spack find\n ==> 0 installed packages\n\n $ spack install zlib@1.2.11\n [+] q6cqrdt zlib@1.2.11 ~/spack/opt/spack/linux-rhel7-broadwell/gcc-8.1.0/zlib-1.2.11-q6cqrdto4iktfg6qyqcc5u4vmfmwb7iv (12s)\n\n $ spack env activate myenv\n\n $ spack find\n ==> In environment myenv\n ==> No root specs\n ==> 0 installed packages\n\n $ spack install zlib@1.2.8\n [+] yfc7epf zlib@1.2.8 ~/spack/opt/spack/linux-rhel7-broadwell/gcc-8.1.0/zlib-1.2.8-yfc7epf57nsfn2gn4notccaiyxha6z7x (12s)\n ==> Updating view at ~/spack/var/spack/environments/myenv/.spack-env/view\n\n $ spack find\n ==> In environment myenv\n ==> Root specs\n zlib@1.2.8\n\n ==> 1 installed package\n -- linux-rhel7-broadwell / gcc@8.1.0 ----------------------------\n zlib@1.2.8\n\n $ despacktivate\n\n $ spack find\n ==> 2 installed packages\n -- linux-rhel7-broadwell / gcc@8.1.0 ----------------------------\n zlib@1.2.8 zlib@1.2.11\n\n\nNote that when we installed the abstract spec ``zlib@1.2.8``, it was presented as a root of the environment.\nAll explicitly installed packages will be listed as roots of the environment.\n\nAll of the Spack commands that act on the list of installed specs are environment-aware in this way, including ``install``, ``uninstall``, ``find``, ``extensions``, etc.\nIn the :ref:`environment-configuration` section we will discuss environment-aware commands further.\n\n.. _cmd-spack-add:\n\nAdding Abstract Specs\n^^^^^^^^^^^^^^^^^^^^^\n\nAn abstract spec is the user-specified spec before Spack applies defaults or dependency information.\n\nYou can add abstract specs to an environment using the ``spack add`` command.\nThis adds the abstract spec as a root of the environment in the ``spack.yaml`` file.\nThe most important component of an environment is a list of abstract specs.\n\nAdding abstract specs does not immediately install anything, nor does it affect the ``spack.lock`` file.\nTo update the lockfile, the environment must be :ref:`re-concretized `, and to update any installations, the environment must be :ref:`(re)installed `.\n\nThe ``spack add`` command is environment-aware.\nIt adds the spec to the currently active environment.\nAn error is generated if there isn't an active environment.\n\n.. code-block:: spec\n\n $ spack env activate myenv\n $ spack add mpileaks\n\nor\n\n.. code-block:: spec\n\n $ spack -e myenv add python\n\n.. note::\n\n All environment-aware commands can also be called using the ``spack -e`` flag to specify the environment.\n\n.. _cmd-spack-concretize:\n\nConcretizing\n^^^^^^^^^^^^\n\nOnce user specs have been added to an environment, they can be concretized.\nThere are three different modes of operation to concretize an environment, explained in detail in :ref:`environments_concretization_config`.\nRegardless of which mode of operation is chosen, the following command will ensure all of the root specs are concretized according to the constraints that are prescribed in the configuration:\n\n.. code-block:: console\n\n [myenv]$ spack concretize\n\nIn the case of specs that are not concretized together, the command above will concretize only the specs that were added and not yet concretized.\nForcing a re-concretization of all of the specs can be done by adding the ``-f`` option:\n\n.. code-block:: console\n\n [myenv]$ spack concretize -f\n\nWithout the option, Spack guarantees that already concretized specs are unchanged in the environment.\n\nThe ``concretize`` command does not install any packages.\nFor packages that have already been installed outside of the environment, the process of adding the spec and concretizing is identical to installing the spec assuming it concretizes to the exact spec that was installed outside of the environment.\n\nThe ``spack find`` command can show concretized specs separately from installed specs using the ``-c`` (``--concretized``) flag.\n\n.. code-block:: console\n\n [myenv]$ spack add zlib\n [myenv]$ spack concretize\n [myenv]$ spack find -c\n ==> In environment myenv\n ==> Root specs\n zlib\n\n ==> Concretized roots\n -- linux-rhel7-x86_64 / gcc@4.9.3 -------------------------------\n zlib@1.2.11\n\n ==> 0 installed packages\n\n\n.. _installing-environment:\n\nInstalling an Environment\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nIn addition to adding individual specs to an environment, one can install the entire environment at once using the command\n\n.. code-block:: console\n\n [myenv]$ spack install\n\nIf the environment has been concretized, Spack will install the concretized specs.\nOtherwise, ``spack install`` will concretize the environment before installing the concretized specs.\n\n.. note::\n\n Every ``spack install`` process builds one package at a time with multiple build jobs, controlled by the ``-j`` flag and the ``config:build_jobs`` option (see :ref:`build-jobs`).\n To speed up environment builds further, independent packages can be installed in parallel by launching more Spack instances.\n For example, the following will build at most four packages in parallel using three background jobs:\n\n .. code-block:: console\n\n [myenv]$ spack install & spack install & spack install & spack install\n\n Another option is to generate a ``Makefile`` and run ``make -j`` to control the number of parallel install processes.\n See :ref:`cmd-spack-env-depfile` for details.\n\n\nAs it installs, ``spack install`` creates symbolic links in the ``logs/`` directory in the environment, allowing for easy inspection of build logs related to that environment.\nThe ``spack install`` command also stores a Spack repo containing the ``package.py`` file used at install time for each package in the ``repos/`` directory in the environment.\n\nThe ``--no-add`` option can be used in a concrete environment to tell Spack to install specs already present in the environment but not to add any new root specs to the environment.\nFor root specs provided to ``spack install`` on the command line, ``--no-add`` is the default, while for dependency specs, it is optional.\nIn other words, if there is an unambiguous match in the active concrete environment for a root spec provided to ``spack install`` on the command line, Spack does not require you to specify the ``--no-add`` option to prevent the spec from being added again.\nAt the same time, a spec that already exists in the environment, but only as a dependency, will be added to the environment as a root spec without the ``--no-add`` option.\n\n.. _cmd-spack-develop:\n\nDeveloping Packages in a Spack Environment\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nThe ``spack develop`` command allows one to develop Spack packages in an environment.\nIt will configure Spack to install the package from local source.\nBy default, ``spack develop`` will also clone the package to a subdirectory in the environment for the local source.\nThese choices can be overridden with the ``--path`` argument, and the ``--no-clone`` argument.\nRelative paths provided to the ``--path`` argument will be resolved relative to the environment directory.\nAll of these options are recorded in the environment manifest, although default values may be left implied.\n\n.. code-block:: console\n\n $ spack develop --path src/foo foo@develop\n $ cat `spack location -e`/spack.yaml\n spack:\n ...\n develop\n foo:\n spec: foo@develop\n path: src/foo\n\nWhen ``spack develop`` is run in a concretized environment, Spack will modify the concrete specs in the environment to reflect the modified provenance.\nAny package built from local source will have a ``dev_path`` variant, and the hash of any dependent of those packages will be modified to reflect the change.\nThe value of the ``dev_path`` variant will be the absolute path to the package source directory.\nIf the develop spec conflicts with the concrete specs in the environment, Spack will raise an exception and require the ``spack develop --no-modify-concrete-specs`` option, followed by a ``spack concretize --force`` to apply the ``dev_path`` variant and constraints from the develop spec.\n\nWhen concretizing an environment with develop specs, the version, variants, and other attributes of the spec provided to the ``spack develop`` command will be treated as constraints by the concretizer (in addition to any constraints from the packages ``specs`` list).\nIf the ``develop`` configuration for the package does not include a spec version, Spack will choose the **highest** version of the package.\nThis means that any \"infinity\" versions (``develop``, ``main``, etc.) will be preferred for specs marked with the ``spack develop`` command, which is different from the standard Spack behavior to prefer the highest **numeric** version.\nThese packages will have an automatic ``dev_path`` variant added by the concretizer, with a value of the absolute path to the local source Spack is building from.\n\nSpack will ensure the package and its dependents are rebuilt any time the environment is installed if the package's local source code has been modified.\nSpack's native implementation is to check if ``mtime`` is newer than the installation.\nA custom check can be created by overriding the ``detect_dev_src_change`` method in your package class.\nThis is particularly useful for projects using custom Spack repos to drive development and want to optimize performance.\n\nWhen ``spack develop`` is run without any arguments, Spack will clone any develop specs in the environment for which the specified path does not exist.\n\nWhen working deep in the graph it is often desirable to have multiple specs marked as ``develop`` so you don't have to restage and/or do full rebuilds each time you call ``spack install``.\nThe ``--recursive`` flag can be used in these scenarios to ensure that all the dependents of the initial spec you provide are also marked as develop specs.\nThe ``--recursive`` flag requires a pre-concretized environment so the graph can be traversed from the supplied spec all the way to the root specs.\n\nFor packages with ``git`` attributes, git branches, tags, and commits can also be used as valid concrete versions (see :ref:`version-specifier`).\nThis means that for a package ``foo``, ``spack develop foo@git.main`` will clone the ``main`` branch of the package, and ``spack install`` will install from that git clone if ``foo`` is in the environment.\nFurther development on ``foo`` can be tested by re-installing the environment, and eventually committed and pushed to the upstream git repo.\n\nIf the package being developed supports out-of-source builds then users can use the ``--build_directory`` flag to control the location and name of the build directory.\nThis is a shortcut to set the ``package_attributes:build_directory`` in the ``packages`` configuration (see :ref:`assigning-package-attributes`).\nThe supplied location will become the build-directory for that package in all future builds.\n\n.. admonition:: Potential pitfalls of setting the build directory\n :class: warning\n\n Spack does not check for out-of-source build compatibility with the packages and so the onus of making sure the package supports out-of-source builds is on the user.\n For example, most ``autotool`` and ``makefile`` packages do not support out-of-source builds while all ``CMake`` packages do.\n Understanding these nuances is up to the software developers and we strongly encourage developers to only redirect the build directory if they understand their package's build-system.\n\nModifying Specs in an Environment\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nThe ``spack change`` command allows the user to change individual specs in a Spack environment.\n\nBy default, ``spack change`` operates on the abstract specs of an environment.\nThe command a list of spec arguments.\nFor each argument, the root spec with the same name as the provided spec is modified to satisfy the provided spec.\nFor example, in an environment with the root spec ``hdf5+mpi+fortran``, then\n\n.. code-block:: console\n\n spack change hdf5~mpi+cxx\n\nwill change the root spec to ``hdf5~mpi+cxx+fortran``.\n\nWhen more complex matching semantics are necessary, the ``--match-spec`` argument replaces the spec name as the selection criterion.\nWhen using the ``--match-spec`` argument, the spec name is not required.\nIn the same environment,\n\n.. code-block:: console\n\n spack change --match-spec \"+fortran\" +hl\n\nwill constrain the ``hdf5`` spec to ``+hl``.\n\nBy default, the ``spack change`` command will result in an error and no change to the environment if it will modify more than one abstract spec.\nUse the ``--all`` option to allow ``spack change`` to modify multiple abstract specs.\n\nThe ``--concrete`` option allows ``spack change`` to modify the concrete specs of an environment as well as the abstract specs.\nMultiple concrete specs may be modified, even for a change that modifies only a single abstract spec.\nThe ``--all`` option does not affect how many concrete specs may be modified.\n\n.. warning::\n\n Concrete specs are modified without any constraints from the packages.\n The ``spack change --concrete`` command may create invalid specs that will not build properly if applied without caution.\n\nThe ``--concrete-only`` option allows for modifying concrete specs without modifying abstract specs.\nIt allows changes to be applied to non-root nodes in the environment, and other changes that do not modify any root specs.\n\nLoading\n^^^^^^^\n\nOnce an environment has been installed, the following creates a load script for it:\n\n.. code-block:: console\n\n $ spack env loads -r\n\nThis creates a file called ``loads`` in the environment directory.\nSourcing that file in Bash will make the environment available to the user, and can be included in ``.bashrc`` files, etc.\nThe ``loads`` file may also be copied out of the environment, renamed, etc.\n\n\n.. _environment_include_concrete:\n\nIncluding Concrete Environments\n-------------------------------\n\nSpack can create an environment that includes information from already concretized environments.\nYou can think of the new environment as a combination of existing environments.\nIt uses information from the existing environments' ``spack.lock`` files in the creation of the new environment.\nWhen such an environment is concretized it will generate its own ``spack.lock`` file that contains relevant information from the included environments.\n\n\nCreating combined concrete environments\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nTo create a combined concrete environment, you must have at least one existing concrete environment.\nYou will use the command ``spack env create`` with the argument ``--include-concrete`` followed by the name or path of the environment you'd like to include.\nHere is an example of how to create a combined environment from the command line::\n\n $ spack env create myenv\n $ spack -e myenv add python\n $ spack -e myenv concretize\n $ spack env create --include-concrete myenv combined_env\n\nYou can also include concrete environments directly in the ``spack.yaml`` file.\nIt involves adding the absolute paths to the concrete environments ``spack.lock`` under the new environment's ``include`` heading.\nSpack-specific configuration variables, such as ``$spack``, and environment variables can be used in the include paths as long as the expression expands to an absolute path.\n(See :ref:`config-file-variables` for more information.)\n\nFor example,\n\n.. code-block:: yaml\n\n spack:\n include:\n - /absolute/path/to/environment1/spack.lock\n - $spack/../path/to/environment2/spack.lock\n specs: []\n concretizer:\n unify: true\n\nwill include the specs from ``environment1`` and ``environment2`` where the second environment's path is the absolute path of the directory that is relative to the spack root.\n\n.. note::\n\n Once the ``spack.yaml`` file is updated you must concretize the new environment to get the concrete specs from the included environments.\n This will produce the combined ``spack.lock`` file.\n\nUpdating a combined environment\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nIf you want changes made to one of the included environments reflected in the combined environment, then you will need to re-concretize the included environment **then** the combined environment for the change to be incorporated.\nFor example::\n\n $ spack env create myenv\n $ spack -e myenv add python\n $ spack -e myenv concretize\n $ spack env create --include-concrete myenv combined_env\n\n $ spack -e myenv find\n ==> In environment myenv\n ==> Root specs\n python\n\n ==> 0 installed packages\n\n $ spack -e combined_env find\n ==> In environment combined_env\n ==> No root specs\n ==> Included specs\n python\n\n ==> 0 installed packages\n\nHere we see that ``combined_env`` contains the python package from ``myenv`` environment.\nBut if we were to add another spec to ``myenv``, ``combined_env`` will not know about the other spec.\n\n.. code-block:: spec\n\n $ spack -e myenv add perl\n $ spack -e myenv concretize\n $ spack -e myenv find\n ==> In environment myenv\n ==> Root specs\n perl python\n\n ==> 0 installed packages\n\n $ spack -e combined_env find\n ==> In environment combined_env\n ==> No root specs\n ==> Included specs\n python\n\n ==> 0 installed packages\n\nIt isn't until you run the ``spack concretize`` command that the combined environment will get the updated information from the re-concretized ``myenv``.\n\n.. code-block:: console\n\n $ spack -e combined_env concretize\n $ spack -e combined_env find\n ==> In environment combined_env\n ==> No root specs\n ==> Included specs\n perl python\n\n ==> 0 installed packages\n\n.. _environment-configuration:\n\nConfiguring Environments\n------------------------\n\nA variety of Spack behaviors are changed through Spack configuration files, covered in more detail in the :ref:`configuration` section.\n\nSpack Environments provide an additional level of configuration scope between the custom scope and the user scope discussed in the configuration documentation.\n\nThere are two ways to include configuration information in a Spack Environment:\n\n#. Inline in the ``spack.yaml`` file\n\n#. Included in the ``spack.yaml`` file from another file.\n\nMany Spack commands also affect configuration information in files automatically.\nThose commands take a ``--scope`` argument, and the environment can be specified by ``env:NAME`` (to affect environment ``foo``, set ``--scope env:foo``).\nThese commands will automatically manipulate configuration inline in the ``spack.yaml`` file.\n\nInline configurations\n^^^^^^^^^^^^^^^^^^^^^\n\nInline environment-scope configuration is done using the same yaml format as standard Spack configuration scopes, covered in the :ref:`configuration` section.\nEach section is contained under a top-level yaml object with its name.\nFor example, a ``spack.yaml`` manifest file containing some package preference configuration (as in a ``packages.yaml`` file) could contain:\n\n.. code-block:: yaml\n\n spack:\n # ...\n packages:\n all:\n providers:\n mpi: [openmpi]\n # ...\n\nThis configuration sets the default ``mpi`` provider to be ``openmpi``.\n\nIncluded configurations\n^^^^^^^^^^^^^^^^^^^^^^^\n\nSpack environments allow an ``include`` heading in their yaml schema.\nThis heading pulls in external configuration files and applies them to the environment.\n\n.. code-block:: yaml\n\n spack:\n include:\n - environment/relative/path/to/config.yaml\n - path: https://github.com/path/to/raw/config/compilers.yaml\n sha256: 26e871804a92cd07bb3d611b31b4156ae93d35b6a6d6e0ef3a67871fcb1d258b\n - /absolute/path/to/packages.yaml\n - path: /path/to/$os/$target/environment\n optional: true\n - path: /path/to/os-specific/config-dir\n when: os == \"ventura\"\n\nIncluded configuration files are required *unless* they are explicitly optional or the entry's condition evaluates to ``false``.\nOptional includes are specified with the ``optional`` clause and conditional with the ``when`` clause.\n(See :ref:`include-yaml` for more information on optional and conditional entries.)\n\nFiles are listed using paths to individual files or directories containing them.\nPath entries may be absolute or relative to the environment or specified as URLs.\nURLs to individual files must link to the **raw** form of the file's contents (e.g., `GitHub `_ or `GitLab `_) **and** include a valid sha256 for the file.\nOnly the ``file``, ``ftp``, ``http`` and ``https`` protocols (or schemes) are supported.\nSpack-specific, environment and user path variables can be used.\n(See :ref:`config-file-variables` for more information.)\n\n.. warning::\n\n Recursive includes are not currently processed in a breadth-first manner so the value of a configuration option that is altered by multiple included files may not be what you expect.\n This will be addressed in a future update.\n\n\nConfiguration precedence\n^^^^^^^^^^^^^^^^^^^^^^^^\n\nInline configurations take precedence over included configurations, so you don't have to change shared configuration files to make small changes to an individual environment.\nIncluded configurations listed earlier will have higher precedence, as the included configs are applied in reverse order.\n\nManually Editing the Specs List\n-------------------------------\n\nThe list of abstract/root specs in the environment is maintained in the ``spack.yaml`` manifest under the heading ``specs``.\n\n.. code-block:: yaml\n\n spack:\n specs:\n - ncview\n - netcdf\n - nco\n - py-sphinx\n\nAppending to this list in the yaml is identical to using the ``spack add`` command from the command line.\nHowever, there is more power available from the yaml file.\n\n.. _environments_concretization_config:\n\nSpec concretization\n^^^^^^^^^^^^^^^^^^^\nAn environment can be concretized in three different modes and the behavior active under any environment is determined by the ``concretizer:unify`` configuration option.\n\nThe *default* mode is to unify all specs:\n\n.. code-block:: yaml\n\n spack:\n specs:\n - hdf5+mpi\n - zlib@1.2.8\n concretizer:\n unify: true\n\nThis means that any package in the environment corresponds to a single concrete spec.\nIn the above example, when ``hdf5`` depends down the line of ``zlib``, it is required to take ``zlib@1.2.8`` instead of a newer version.\nThis mode of concretization is particularly useful when environment views are used: if every package occurs in only one flavor, it is usually possible to merge all install directories into a view.\n\nA downside of unified concretization is that it can be overly strict.\nFor example, a concretization error would happen when both ``hdf5+mpi`` and ``hdf5~mpi`` are specified in an environment.\n\nThe second mode is to *unify when possible*: this makes concretization of root specs more independent.\nInstead of requiring reuse of dependencies across different root specs, it is only maximized:\n\n.. code-block:: yaml\n\n spack:\n specs:\n - hdf5~mpi\n - hdf5+mpi\n - zlib@1.2.8\n concretizer:\n unify: when_possible\n\nThis means that both ``hdf5`` installations will use ``zlib@1.2.8`` as a dependency even if newer versions of that library are available.\n\nThe third mode of operation is to concretize root specs entirely independently by disabling unified concretization:\n\n.. code-block:: yaml\n\n spack:\n specs:\n - hdf5~mpi\n - hdf5+mpi\n - zlib@1.2.8\n concretizer:\n unify: false\n\nIn this example ``hdf5`` is concretized separately, and does not consider ``zlib@1.2.8`` as a constraint or preference.\nInstead, it will take the latest possible version.\n\nThe last two concretization options are typically useful for system administrators and user support groups providing a large software stack for their HPC center.\n\n.. note::\n\n The ``concretizer:unify`` config option was introduced in Spack 0.18 to replace the ``concretization`` property.\n For reference, ``concretization: together`` is replaced by ``concretizer:unify:true``, and ``concretization: separately`` is replaced by ``concretizer:unify:false``.\n\n.. admonition:: Re-concretization of user specs\n\n The ``spack concretize`` command without additional arguments will *not* change any previously concretized specs.\n This may prevent it from finding a solution when using ``unify: true``, and it may prevent it from finding a minimal solution when using ``unify: when_possible``.\n You can force Spack to ignore the existing concrete environment with ``spack concretize -f``.\n\n.. _environment-spec-matrices:\n\nSpec Matrices\n^^^^^^^^^^^^^\n\nEntries in the ``specs`` list can be individual abstract specs or a spec matrix.\n\nA spec matrix is a yaml object containing multiple lists of specs, and evaluates to the cross-product of those specs.\nSpec matrices also contain an ``excludes`` directive, which eliminates certain combinations from the evaluated result.\n\nThe following two environment manifests are identical:\n\n.. code-block:: yaml\n\n spack:\n specs:\n - zlib %gcc@7.1.0\n - zlib %gcc@4.9.3\n - libelf %gcc@7.1.0\n - libelf %gcc@4.9.3\n - libdwarf %gcc@7.1.0\n - cmake\n\n.. code-block:: yaml\n\n spack:\n specs:\n - matrix:\n - [zlib, libelf, libdwarf]\n - [\"%gcc@7.1.0\", \"%gcc@4.9.3\"]\n exclude:\n - libdwarf%gcc@4.9.3\n - cmake\n\nSpec matrices can be used to install swaths of software across various toolchains.\n\n.. _spec-list-references:\n\nSpec List References\n^^^^^^^^^^^^^^^^^^^^\n\nThe last type of possible entry in the specs list is a reference.\n\nThe Spack Environment manifest yaml schema contains an additional heading ``definitions``.\nUnder definitions is an array of yaml objects.\nEach object has one or two fields.\nThe one required field is a name, and the optional field is a ``when`` clause.\n\nThe named field is a spec list.\nThe spec list uses the same syntax as the ``specs`` entry.\nEach entry in the spec list can be a spec, a spec matrix, or a reference to an earlier named list.\nReferences are specified using the ``$`` sigil, and are \"splatted\" into place (i.e. the elements of the referent are at the same level as the elements listed separately).\nAs an example, the following two manifest files are identical.\n\n.. code-block:: yaml\n\n spack:\n definitions:\n - first: [libelf, libdwarf]\n - compilers: [\"%gcc\", \"%intel\"]\n - second:\n - $first\n - matrix:\n - [zlib]\n - [$compilers]\n specs:\n - $second\n - cmake\n\n.. code-block:: yaml\n\n spack:\n specs:\n - libelf\n - libdwarf\n - zlib%gcc\n - zlib%intel\n - cmake\n\n.. note::\n\n Named spec lists in the definitions section may only refer to a named list defined above itself.\n Order matters.\n\nIn short files like the example, it may be easier to simply list the included specs.\nHowever for more complicated examples involving many packages across many toolchains, separately factored lists make environments substantially more manageable.\n\nAdditionally, the ``-l`` option to the ``spack add`` command allows one to add to named lists in the definitions section of the manifest file directly from the command line.\n\nThe ``when`` directive can be used to conditionally add specs to a named list.\nThe ``when`` directive takes a string of Python code referring to a restricted set of variables, and evaluates to a boolean.\nThe specs listed are appended to the named list if the ``when`` string evaluates to ``True``.\nIn the following snippet, the named list ``compilers`` is ``[\"%gcc\", \"%clang\", \"%intel\"]`` on ``x86_64`` systems and ``[\"%gcc\", \"%clang\"]`` on all other systems.\n\n.. code-block:: yaml\n\n spack:\n definitions:\n - compilers: [\"%gcc\", \"%clang\"]\n - when: arch.satisfies(\"target=x86_64:\")\n compilers: [\"%intel\"]\n\n.. note::\n\n Any definitions with the same named list with true ``when`` clauses (or absent ``when`` clauses) will be appended together\n\nThe valid variables for a ``when`` clause are:\n\n#. ``platform``.\n The platform string of the default Spack architecture on the system.\n\n#. ``os``.\n The OS string of the default Spack architecture on the system.\n\n#. ``target``.\n The target string of the default Spack architecture on the system.\n\n#. ``architecture`` or ``arch``.\n A Spack spec satisfying the default Spack architecture on the system.\n This supports querying via the ``satisfies`` method, as shown above.\n\n#. ``arch_str``.\n The architecture string of the default Spack architecture on the system.\n\n#. ``re``.\n The standard regex module in Python.\n\n#. ``env``.\n The user environment (usually ``os.environ`` in Python).\n\n#. ``hostname``.\n The hostname of the system (if ``hostname`` is an executable in the user's PATH).\n\nSpecLists as Constraints\n^^^^^^^^^^^^^^^^^^^^^^^^\n\nDependencies and compilers in Spack can be both packages in an environment and constraints on other packages.\nReferences to SpecLists allow a shorthand to treat packages in a list as either a compiler or a dependency using the ``$%`` or ``$^`` syntax respectively.\n\nFor example, the following environment has three root packages: ``gcc@8.1.0``, ``mvapich2@2.3.1 %gcc@8.1.0``, and ``hdf5+mpi %gcc@8.1.0 ^mvapich2@2.3.1``.\n\n.. code-block:: yaml\n\n spack:\n definitions:\n - compilers: [gcc@8.1.0]\n - mpis: [mvapich2@2.3.1]\n - packages: [hdf5+mpi]\n\n specs:\n - $compilers\n - matrix:\n - [$mpis]\n - [$%compilers]\n - matrix:\n - [$packages]\n - [$^mpis]\n - [$%compilers]\n\nThis allows for a much-needed reduction in redundancy between packages and constraints.\n\n.. _environment-spec-groups:\n\nSpec Groups\n^^^^^^^^^^^\n\n.. versionadded:: 1.2\n\nEnvironments can be organized with named spec groups, enabling you to apply localized configuration overrides and establish concretization dependencies.\nThis is extremely useful in a couple of common scenarios, as detailed below.\n\n.. _environment-spec-groups-bootstrapping-compiler:\n\nBuilding and using a compiler in a single environment\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nA common use case is to build a recent compiler on top of an existing system and then compile a stack of software with it.\nFor instance, assume we are interested in building ``hdf5`` and ``libtree`` with ``gcc@15.2``.\nThe following manifest file would do exactly that:\n\n.. code-block:: yaml\n\n spack:\n specs:\n - group: compiler\n specs:\n - gcc@15.2\n\n - group: apps\n needs: [compiler]\n specs:\n - hdf5 %gcc@15.2\n - libtree %gcc@15.2\n\nThe ``group:`` attribute allows to name a group of specs, which are then listed under the ``specs:`` attribute in the same object.\nThe simplest example is the ``compiler`` group composed of just the ``gcc@15.2`` spec.\n\nTo express dependencies among groups of specs the ``needs:`` attribute is used, which is a list of names corresponding to the groups we depend on.\nThe way this works is that group dependencies are always concretized *before* the current group, and their specs are *always* available for reuse when the current group is concretized.\n\n.. _environment-spec-groups-configuring-groups:\n\nConfiguring a group of specs\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nAnother common scenario is the deployment of different configurations (e.g. CUDA enabled vs.\nROCm enabled) of the same set of software.\nAs an example, assume we want to install ``gromacs`` and ``quantum-espresso`` for both ``target=x86_64_v3`` and ``target=x86_64_v4``.\nThat can be done with the following manifest file:\n\n.. code-block:: yaml\n\n spack:\n - group: apps-x86_64_v3\n specs:\n - gromacs\n - quantum-espresso\n override:\n packages:\n all:\n prefer:\n - target=x86_64_v3\n\n - group: apps-x86_64_v4\n specs:\n - gromacs\n - quantum-espresso\n override:\n packages:\n all:\n prefer:\n - target=x86_64_v4\n\nThe ``override:`` attribute allows us to override the configuration for a single group of specs.\nThe overridden part is always added as the *topmost* scope when the current group is concretized.\nThis ensures the override always takes precedence over other sources of configuration.\n\n.. _environment-spec-groups-explicit:\n\nControlling garbage collection with ``explicit: false``\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nBy default every spec group is treated as a set of *explicit* roots.\nThis means its specs are preserved by ``spack gc`` even when nothing else depends on them.\nSetting ``explicit: false`` on a group marks its specs as *implicit*, making them eligible for garbage collection once no other installed spec depends on them:\n\n.. code-block:: yaml\n\n spack:\n specs:\n - group: compiler\n explicit: false\n specs:\n - gcc@15.2\n\n - group: apps\n needs: [compiler]\n specs:\n - hdf5 %gcc@15.2\n - libtree %gcc@15.2\n\nAfter the apps are installed, ``spack gc`` will remove the compiler once no installed spec has a link or run dependency on it.\n\n.. note::\n\n Flipping ``explicit: false`` on a group that has already been installed does **not** retroactively update the database record for the already-installed specs.\n The flag takes effect only for specs installed, or re-installed, after the change.\n To immediately mark an existing spec as implicit, use ``spack mark -i ``.\n\n\nModifying Environment Variables\n-------------------------------\n\nSpack Environments can modify the active shell's environment variables when activated.\nThe environment can be configured to set, unset, prepend, or append using ``env_vars`` configuration in ``spack.yaml``:\n\n.. code-block:: yaml\n\n spack:\n env_vars:\n set:\n ENVAR_TO_SET_IN_ENV_LOAD: \"FOO\"\n unset:\n - ENVAR_TO_UNSET_IN_ENV_LOAD\n prepend_path:\n PATH_LIST: \"path/to/prepend\"\n append_path:\n PATH_LIST: \"path/to/append\"\n remove_path:\n PATH_LIST: \"path/to/remove\"\n\nEnvironment Views\n-----------------\n\nSpack Environments can have an associated filesystem view, which is a directory with a more traditional structure ``/bin``, ``/lib``, ``/include`` in which all files of the installed packages are linked.\n\nBy default a view is created for each environment, thanks to the ``view: true`` option in the ``spack.yaml`` manifest file:\n\n.. code-block:: yaml\n\n spack:\n specs: [perl, python]\n view: true\n\nThe view is created in a hidden directory ``.spack-env/view`` relative to the environment.\nIf you've used ``spack env activate``, you may have already interacted with this view.\nSpack prepends its ``/bin`` dir to ``PATH`` when the environment is activated, so that you can directly run executables from all installed packages in the environment.\n\nViews are highly customizable: you can control where they are put, modify their structure, include and exclude specs, change how files are linked, and you can even generate multiple views for a single environment.\n\n.. _configuring_environment_views:\n\nMinimal view configuration\n^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nThe minimal configuration\n\n.. code-block:: yaml\n\n spack:\n # ...\n view: true\n\nlets Spack generate a single view with default settings under the ``.spack-env/view`` directory of the environment.\n\nAnother short way to configure a view is to specify just where to put it:\n\n.. code-block:: yaml\n\n spack:\n # ...\n view: /path/to/view\n\nViews can also be disabled by setting ``view: false``.\n\n.. _cmd-spack-env-view:\n\nAdvanced view configuration\n^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nOne or more **view descriptors** can be defined under ``view``, keyed by a name.\nThe example from the previous section with ``view: /path/to/view`` is equivalent to defining a view descriptor named ``default`` with a ``root`` attribute:\n\n.. code-block:: yaml\n\n spack:\n # ...\n view:\n default: # name of the view\n root: /path/to/view # view descriptor attribute\n\nThe ``default`` view descriptor name is special: when you ``spack env activate`` your environment, this view will be used to update (among other things) your ``PATH`` variable.\n\nView descriptors must contain the root of the view, and optionally projections, ``select`` and ``exclude`` lists and link information via ``link`` and ``link_type``.\n\nAs a more advanced example, in the following manifest file snippet we define a view named ``mpis``, rooted at ``/path/to/view`` in which all projections use the package name, version, and compiler name to determine the path for a given package.\nThis view selects all packages that depend on MPI, and excludes those built with the GCC compiler at version 8.5.\nThe root specs with their (transitive) link and run type dependencies will be put in the view due to the ``link: all`` option, and the files in the view will be symlinks to the Spack install directories.\n\n.. code-block:: yaml\n\n spack:\n # ...\n view:\n mpis:\n root: /path/to/view\n select: [^mpi]\n exclude: [\"%gcc@8.5\"]\n projections:\n all: \"{name}/{version}-{compiler.name}\"\n link: all\n link_type: symlink\n\nThe default for the ``select`` and ``exclude`` values is to select everything and exclude nothing.\nThe default projection is the default view projection (``{}``).\nThe ``link`` attribute allows the following values:\n\n#. ``link: all`` include root specs with their transitive run and link type dependencies (default);\n#. ``link: run`` include root specs with their transitive run type dependencies;\n#. ``link: roots`` include root specs without their dependencies.\n\nThe ``link_type`` defaults to ``symlink`` but can also take the value of ``hardlink`` or ``copy``.\n\n.. tip::\n\n The option ``link: run`` can be used to create small environment views for Python packages.\n Python will be able to import packages *inside* of the view even when the environment is not activated, and linked libraries will be located *outside* of the view thanks to rpaths.\n\nFrom the command line, the ``spack env create`` command takes an argument ``--with-view [PATH]`` that sets the path for a single, default view.\nIf no path is specified, the default path is used (``view: true``).\nThe argument ``--without-view`` can be used to create an environment without any view configured.\n\nThe ``spack env view`` command can be used to manage views of an environment.\nThe subcommand ``spack env view enable`` will add a view named ``default`` to an environment.\nIt takes an optional argument to specify the path for the new default view.\nThe subcommand ``spack env view disable`` will remove the view named ``default`` from an environment if one exists.\nThe subcommand ``spack env view regenerate`` will regenerate the views for the environment.\nThis will apply any updates in the environment configuration that have not yet been applied.\n\n.. _view_projections:\n\nView Projections\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\nThe default projection into a view is to link every package into the root of the view.\nThe projections attribute is a mapping of partial specs to spec format strings, defined by the :meth:`~spack.spec.Spec.format` function, as shown in the example below:\n\n.. code-block:: yaml\n\n projections:\n zlib: \"{name}-{version}\"\n ^mpi: \"{name}-{version}/{^mpi.name}-{^mpi.version}-{compiler.name}-{compiler.version}\"\n all: \"{name}-{version}/{compiler.name}-{compiler.version}\"\n\nProjections also permit environment and Spack configuration variable expansions as shown below:\n\n.. code-block:: yaml\n\n projections:\n all: \"{name}-{version}/{compiler.name}-{compiler.version}/$date/$SYSTEM_ENV_VARIABLE\"\n\nwhere ``$date`` is the Spack configuration variable that will expand with the ``YYYY-MM-DD`` format and ``$SYSTEM_ENV_VARIABLE`` is an environment variable defined in the shell.\n\nThe entries in the projections configuration file must all be either specs or the keyword ``all``.\nFor each spec, the projection used will be the first non-``all`` entry that the spec satisfies, or ``all`` if there is an entry for ``all`` and no other entry is satisfied by the spec.\nWhere the keyword ``all`` appears in the file does not matter.\n\nGiven the example above, the spec ``zlib@1.2.8`` will be linked into ``/my/view/zlib-1.2.8/``, the spec ``hdf5@1.8.10+mpi %gcc@4.9.3 ^mvapich2@2.2`` will be linked into ``/my/view/hdf5-1.8.10/mvapich2-2.2-gcc-4.9.3``, and the spec ``hdf5@1.8.10~mpi %gcc@4.9.3`` will be linked into ``/my/view/hdf5-1.8.10/gcc-4.9.3``.\n\nIf the keyword ``all`` does not appear in the projections configuration file, any spec that does not satisfy any entry in the file will be linked into the root of the view as in a single-prefix view.\nAny entries that appear below the keyword ``all`` in the projections configuration file will not be used, as all specs will use the projection under ``all`` before reaching those entries.\n\nGroup of Specs\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nViews can also be applied to a selected list of :ref:`spec groups `.\nThis can be done by specifying the ``group:`` attribute in the view configuration.\nFor instance, with the following manifest:\n\n.. code-block:: yaml\n\n spack:\n concretizer:\n unify: true\n\n packages:\n all:\n require:\n - target=x86_64_v4\n\n specs:\n - group: compiler\n specs:\n - gcc@15.2\n\n - group: apps\n needs: [compiler]\n specs:\n - hdf5~mpi %gcc@15.2\n - libtree %gcc@15.2\n\n view:\n apps:\n root: ./views/apps\n group: apps\n\nThe view will only contain entries from the ``apps`` group, and will not include specs from the ``compiler`` group.\n\nActivating environment views\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nThe ``spack env activate `` command has two effects:\n\n1. It activates the environment so that further Spack commands such as ``spack install`` will run in the context of the environment.\n2. It activates the view so that environment variables such as ``PATH`` are updated to include the view.\n\nWithout further arguments, the ``default`` view of the environment is activated.\nIf a view with a different name has to be activated, ``spack env activate --with-view `` can be used instead.\nYou can also activate the environment without modifying further environment variables using ``--without-view``.\n\nThe environment variables affected by the ``spack env activate`` command and the paths that are used to update them are determined by the :ref:`prefix inspections ` defined in your modules configuration; the defaults are summarized in the following table.\n\n=================== =========\nVariable Paths\n=================== =========\nPATH bin\nMANPATH man, share/man\nACLOCAL_PATH share/aclocal\nPKG_CONFIG_PATH lib/pkgconfig, lib64/pkgconfig, share/pkgconfig\nCMAKE_PREFIX_PATH .\n=================== =========\n\nEach of these paths are appended to the view root, and added to the relevant variable if the path exists.\nFor this reason, it is not recommended to use non-default projections with the default view of an environment.\n\nThe ``spack env deactivate`` command will remove the active view of the Spack environment from the user's environment variables.\n\n\n.. _cmd-spack-env-depfile:\n\n\nGenerating Depfiles from Environments\n------------------------------------------\n\nSpack can generate ``Makefile``\\s to make it easier to build multiple packages in an environment in parallel.\n\n.. note::\n\n Since Spack v1.1, there is a new experimental installer that supports package-level parallelism out of the box with POSIX jobserver support.\n You can enable it with ``spack config add config:installer:new``.\n This new installer may provide a simpler alternative to the ``spack env depfile`` workflow described in this section for users primarily interested in speeding up environment installations.\n\nGenerated ``Makefile``\\s expose targets that can be included in existing ``Makefile``\\s, to allow other targets to depend on the environment installation.\n\nA typical workflow is as follows:\n\n.. code-block:: spec\n\n $ spack env create -d .\n $ spack -e . add perl\n $ spack -e . concretize\n $ spack -e . env depfile -o Makefile\n $ make -j64\n\nThis generates a ``Makefile`` from a concretized environment in the current working directory, and ``make -j64`` installs the environment, exploiting parallelism across packages as much as possible.\nSpack respects the Make jobserver and forwards it to the build environment of packages, meaning that a single ``-j`` flag is enough to control the load, even when packages are built in parallel.\n\nBy default the following phony convenience targets are available:\n\n- ``make all``: installs the environment (default target);\n- ``make clean``: cleans files used by make, but does not uninstall packages.\n\n.. tip::\n\n GNU Make version 4.3 and above have great support for output synchronization through the ``-O`` and ``--output-sync`` flags, which ensure that output is printed orderly per package install.\n To get synchronized output with colors, use ``make -j SPACK_COLOR=always --output-sync=recurse``.\n\nSpecifying dependencies on generated ``make`` targets\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nAn interesting question is how to include generated ``Makefile``\\s in your own ``Makefile``\\s.\nThis comes up when you want to install an environment that provides executables required in a command for a make target of your own.\n\nThe example below shows how to accomplish this: the ``env`` target specifies the generated ``spack/env`` target as a prerequisite, meaning that the environment gets installed and is available for use in the ``env`` target.\n\n.. code-block:: Makefile\n\n SPACK ?= spack\n\n .PHONY: all clean env\n\n all: env\n\n spack.lock: spack.yaml\n \t$(SPACK) -e . concretize -f\n\n env.mk: spack.lock\n \t$(SPACK) -e . env depfile -o $@ --make-prefix spack\n\n env: spack/env\n \t$(info environment installed!)\n\n clean:\n \trm -rf spack.lock env.mk spack/\n\n ifeq (,$(filter clean,$(MAKECMDGOALS)))\n include env.mk\n endif\n\nThis works as follows: when ``make`` is invoked, it first \"remakes\" the missing include ``env.mk`` as there is a target for it.\nThis triggers concretization of the environment and makes Spack output ``env.mk``.\nAt that point the generated target ``spack/env`` becomes available through ``include env.mk``.\n\nAs it is typically undesirable to remake ``env.mk`` as part of ``make clean``, the include is conditional.\n\n.. note::\n\n When including generated ``Makefile``\\s, it is important to use the ``--make-prefix`` flag and use the non-phony target ``/env`` as prerequisite, instead of the phony target ``/all``.\n\nBuilding a subset of the environment\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nThe generated ``Makefile``\\s contain install targets for each spec, identified by ``--``.\nThis allows you to install only a subset of the packages in the environment.\nWhen packages are unique in the environment, it's enough to know the name and let tab-completion fill out the version and hash.\n\nThe following phony targets are available: ``install/`` to install the spec with its dependencies, and ``install-deps/`` to *only* install its dependencies.\nThis can be useful when certain flags should only apply to dependencies.\nBelow we show a use case where a spec is installed with verbose output (``spack install --verbose``) while its dependencies are installed silently:\n\n.. code-block:: console\n\n $ spack env depfile -o Makefile\n\n # Install dependencies in parallel, only show a log on error.\n $ make -j16 install-deps/python-3.11.0- SPACK_INSTALL_FLAGS=--show-log-on-error\n\n # Install the root spec with verbose output.\n $ make -j16 install/python-3.11.0- SPACK_INSTALL_FLAGS=--verbose\n\nAdding post-install hooks\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nAnother advanced use-case of generated ``Makefile``\\s is running a post-install command for each package.\nThese \"hooks\" could be anything from printing a post-install message, running tests, or pushing just-built binaries to a build cache.\n\nThis can be accomplished through the generated ``[/]SPACK_PACKAGE_IDS`` variable.\nAssuming we have an active and concrete environment, we generate the associated ``Makefile`` with a prefix ``example``:\n\n.. code-block:: console\n\n $ spack env depfile -o env.mk --make-prefix example\n\nAnd we now include it in a different ``Makefile``, in which we create a target ``example/push/%`` with ``%`` referring to a package identifier.\nThis target depends on the particular package installation.\nIn this target we automatically have the target-specific ``HASH`` and ``SPEC`` variables at our disposal.\nThey are respectively the spec hash (excluding leading ``/``), and a human-readable spec.\nFinally, we have an entry point target ``push`` that will update the build cache index once every package is pushed.\nNote how this target uses the generated ``example/SPACK_PACKAGE_IDS`` variable to define its prerequisites.\n\n.. code-block:: Makefile\n\n SPACK ?= spack\n BUILDCACHE_DIR = $(CURDIR)/tarballs\n\n .PHONY: all\n\n all: push\n\n include env.mk\n\n example/push/%: example/install/%\n \t@mkdir -p $(dir $@)\n \t$(info About to push $(SPEC) to a buildcache)\n \t$(SPACK) -e . buildcache push --only=package $(BUILDCACHE_DIR) /$(HASH)\n \t@touch $@\n\n push: $(addprefix example/push/,$(example/SPACK_PACKAGE_IDS))\n \t$(info Updating the buildcache index)\n \t$(SPACK) -e . buildcache update-index $(BUILDCACHE_DIR)\n \t$(info Done!)\n \t@touch $@\n" }, { "path": "lib/spack/docs/environments_basics.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Learn how to use Spack environments to manage reproducible software stacks on a local machine.\n\nSpack Environments\n==================\n\nSpack is a powerful package manager designed for the complex software needs of supercomputers.\nThese same robust features for managing versions and dependencies also make it an excellent tool for local development on a laptop or workstation.\n\nIf you are used to tools like Conda, Homebrew or pip for managing local command-line tools and development projects, you will find Spack environments to be a powerful and flexible alternative.\nSpack environments allow you to create self-contained, reproducible software collections, a concept similar to Conda environments and Python's virtual environments.\n\nUnlike other package managers, Spack environments do not contain copies of the software themselves.\nInstead, they reference installations in the Spack store, which is a central location where Spack keeps all installed packages.\nThis means that multiple environments can share the same package installations, saving disk space and reducing duplication.\n\nIn this section, we will walk through creating a simple environment to manage a personal software stack.\n\nCreating and Activating an Environment\n--------------------------------------\n\nFirst, let's create and activate a new environment.\nThis places you \"inside\" the environment, so all subsequent Spack commands apply to it by default.\n\n.. code-block:: console\n\n $ spack env create myenv\n ==> Created environment myenv in /path/to/spack/var/spack/environments/myenv\n $ spack env activate myenv\n\nHere, *myenv* is the name of our new environment.\n\nYou can verify you are in the environment using:\n\n.. code-block:: console\n\n $ spack env status\n ==> In environment myenv\n\nAdding Specs to the Environment\n-------------------------------\n\nNow that our environment is active, we can add the packages we want to install.\nLet's say we want a newer version of curl and a few Python libraries.\n\n.. code-block:: spec\n\n $ spack add curl@8 python py-numpy py-scipy py-matplotlib\n\nYou can add packages one at a time or all at once.\nNotice that we didn't need to specify the environment name, as Spack knows we are working inside ``myenv``.\nThese packages are now added to the environment's manifest file, ``spack.yaml``.\n\nYou can view the manifest at any time by running:\n\n.. code-block:: console\n\n $ spack config edit\n\nThis will open your ``spack.yaml``, which should look like this:\n\n.. code-block:: yaml\n :caption: Example ``spack.yaml`` for our environment\n\n # This is a Spack Environment file.\n #\n # It describes a set of packages to be installed, along with\n # configuration settings.\n spack:\n # add package specs to the `specs` list\n specs:\n - curl@8\n - python\n - py-numpy\n - py-scipy\n - py-matplotlib\n view: true\n concretizer:\n unify: true\n\nThe ``view: true`` setting tells Spack to create a single directory where all executables, libraries, etc., are symlinked together, similar to a traditional Unix prefix.\nBy default, this view is located inside the environment directory.\n\nInstalling the Software\n-----------------------\n\nWith our specs defined, the next step is to have Spack solve the dependency graph.\nThis is called \"concretization.\"\n\n.. code-block:: console\n\n $ spack concretize\n ==> Concretized ...\n ...\n\nSpack will find a consistent set of versions and dependencies for the packages you requested.\nOnce this is done, you can install everything with a single command:\n\n.. code-block:: console\n\n $ spack install\n\nSpack will now download, build, and install all the necessary packages.\nAfter the installation is complete, the environment's view is automatically updated.\nBecause the environment is active, your ``PATH`` and other variables are already configured.\n\nYou can verify the installation:\n\n.. code-block:: console\n\n $ which python3\n /path/to/spack/var/spack/environments/myenv/.spack-env/view/bin/python3\n\nWhen you are finished working in the environment, you can deactivate it:\n\n.. code-block:: console\n\n $ spack env deactivate\n\nKeeping Up With Updates\n-----------------------\n\nOver time, you may want to update the packages in your environment to their latest versions.\nSpack makes this easy.\n\nFirst, update Spack's package repository to make the latest package versions available:\n\n.. code-block:: console\n\n $ spack repo update\n\nThen, activate the environment, re-concretize and reinstall.\n\n.. code-block:: console\n\n $ spack env activate myenv\n $ spack concretize --fresh-roots --force\n $ spack install\n\nThe ``--fresh-roots`` flag tells the concretizer to prefer the latest available package versions you've added explicitly to the environment, while allowing existing dependencies to remain unchanged if possible.\nAlternatively, you can use the ``--fresh`` flag to prefer the latest versions of all packages including dependencies, but that might lead to longer install times and more changes.\nThe ``--force`` flag allows it to overwrite the previously solved dependencies.\nThe ``install`` command is smart and will only build packages that are not already installed for the new configuration.\n\nCleaning Up Old Packages\n------------------------\n\nAfter an update, you may have old, unused packages taking up space.\nYou can safely remove any package that is no longer part of an environment's dependency tree.\n\n.. code-block:: console\n\n $ spack gc --except-any-environment\n\nThis runs Spack's garbage collector, which will find and uninstall any package versions that are no longer referenced by *any* of your environments.\n\nRemoving the Environment\n------------------------\n\nIf you no longer need an environment, you can completely remove it.\n\nFirst, ensure the environment is not active:\n\n.. code-block:: console\n\n $ spack env deactivate\n\nThen, remove the environment.\n\n.. code-block:: console\n\n $ spack env rm myenv\n\nThis removes the environment's directory and its view, but the packages that were installed for it remain in the Spack store.\nTo actually remove the installations from the Spack store and free up disk space, you can run the garbage collector again.\n\n.. code-block:: console\n\n $ spack gc --except-any-environment\n\nThis command will safely uninstall any packages that are no longer referenced by any of your remaining environments.\n\nNext steps\n----------\n\nSpack has many other features for managing software environments.\nSee :doc:`environments` for more advanced usage.\n" }, { "path": "lib/spack/docs/extensions.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Discover how to extend Spack's core functionality by creating custom commands and plugins.\n\nCustom Extensions\n=================\n\n*Spack extensions* allow you to add custom subcommands to the ``spack`` command.\nThis is extremely useful when developing and maintaining a command whose purpose is too specific to be included in the Spack codebase.\nIt's also useful for evolving a command through its early stages before starting a discussion to merge it upstream.\n\nFrom Spack's point of view, an extension is any path in your filesystem that respects the following naming and layout for files:\n\n.. code-block:: console\n\n spack-scripting/ # The top level directory must match the format 'spack-{extension_name}'\n ├── pytest.ini # Optional file if the extension ships its own tests\n ├── scripting # Folder that may contain modules that are needed for the extension commands\n │ ├── cmd # Folder containing extension commands\n │ │ └── filter.py # A new command that will be available\n │ └── functions.py # Module with internal details\n ├── tests # Tests for this extension\n │ ├── conftest.py\n │ └── test_filter.py\n └── templates # Templates that may be needed by the extension\n\nIn the example above, the extension is named *scripting*.\nIt adds an additional command (``spack filter``) and unit tests to verify its behavior.\n\nThe extension can import any core Spack module in its implementation.\nWhen loaded by the ``spack`` command, the extension itself is imported as a Python package in the ``spack.extensions`` namespace.\nIn the example above, since the extension is named \"scripting\", the corresponding Python module is ``spack.extensions.scripting``.\n\nThe code for this example extension can be obtained by cloning the corresponding git repository:\n\n.. code-block:: console\n\n $ git -C /tmp clone https://github.com/spack/spack-scripting.git\n\nConfigure Spack to Use Extensions\n---------------------------------\n\nTo make your current Spack instance aware of extensions you should add their root paths to ``config.yaml``.\nIn the case of our example, this means ensuring that:\n\n.. code-block:: yaml\n\n config:\n extensions:\n - /tmp/spack-scripting\n\nis part of your configuration file.\nOnce this is set up, any command that the extension provides will be available from the command line:\n\n.. code-block:: console\n\n $ spack filter --help\n usage: spack filter [-h] [--installed | --not-installed]\n [--explicit | --implicit] [--output OUTPUT]\n ...\n\n filter specs based on their properties\n\n positional arguments:\n specs specs to be filtered\n\n optional arguments:\n -h, --help show this help message and exit\n --installed select installed specs\n --not-installed select specs that are not yet installed\n --explicit select specs that were installed explicitly\n --implicit select specs that are not installed or were installed implicitly\n --output OUTPUT where to dump the result\n\nThe corresponding unit tests can be run giving the appropriate options to ``spack unit-test``:\n\n.. code-block:: console\n\n $ spack unit-test --extension=scripting\n ========================================== test session starts ===========================================\n platform linux -- Python 3.11.5, pytest-7.4.3, pluggy-1.3.0\n rootdir: /home/culpo/github/spack-scripting\n configfile: pytest.ini\n testpaths: tests\n plugins: xdist-3.5.0\n collected 5 items\n\n tests/test_filter.py ..... [100%]\n\n ========================================== slowest 30 durations ==========================================\n 2.31s setup tests/test_filter.py::test_filtering_specs[kwargs0-specs0-expected0]\n 0.57s call tests/test_filter.py::test_filtering_specs[kwargs2-specs2-expected2]\n 0.56s call tests/test_filter.py::test_filtering_specs[kwargs4-specs4-expected4]\n 0.54s call tests/test_filter.py::test_filtering_specs[kwargs3-specs3-expected3]\n 0.54s call tests/test_filter.py::test_filtering_specs[kwargs1-specs1-expected1]\n 0.48s call tests/test_filter.py::test_filtering_specs[kwargs0-specs0-expected0]\n 0.01s setup tests/test_filter.py::test_filtering_specs[kwargs4-specs4-expected4]\n 0.01s setup tests/test_filter.py::test_filtering_specs[kwargs2-specs2-expected2]\n 0.01s setup tests/test_filter.py::test_filtering_specs[kwargs1-specs1-expected1]\n 0.01s setup tests/test_filter.py::test_filtering_specs[kwargs3-specs3-expected3]\n\n (5 durations < 0.005s hidden. Use -vv to show these durations.)\n =========================================== 5 passed in 5.06s ============================================\n\nRegistering Extensions via Entry Points\n---------------------------------------\n\n.. note::\n Python version >= 3.8 is required to register extensions via entry points.\n\nSpack can be made aware of extensions that are installed as part of a Python package.\nTo do so, register a function that returns the extension path, or paths, to the ``\"spack.extensions\"`` entry point.\nConsider the Python package ``my_package`` that includes a Spack extension:\n\n.. code-block:: console\n\n my-package/\n ├── src\n │ ├── my_package\n │ │ └── __init__.py\n │ └── spack-scripting/ # the spack extensions\n └── pyproject.toml\n\nadding the following to ``my_package``'s ``pyproject.toml`` will make the ``spack-scripting`` extension visible to Spack when ``my_package`` is installed:\n\n.. code-block:: toml\n\n [project.entry_points.\"spack.extensions\"]\n my_package = \"my_package:get_extension_path\"\n\nThe function ``my_package.get_extension_path`` in ``my_package/__init__.py`` might look like\n\n.. code-block:: python\n\n import importlib.resources\n\n\n def get_extension_path():\n dirname = importlib.resources.files(\"my_package\").joinpath(\"spack-scripting\")\n if dirname.exists():\n return str(dirname)\n" }, { "path": "lib/spack/docs/features.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n An overview of the key features that distinguish Spack from other package managers, including simple installation, custom configurations, and non-destructive installs.\n\nFeature Overview\n================\n\nThis is a high-level overview of features that make Spack different from other `package managers `_ and `port systems `_.\n\nSimple package installation\n---------------------------\n\nInstalling the default version of a package is simple.\nThis will install the latest version of the ``mpileaks`` package and all of its dependencies:\n\n.. code-block:: spec\n\n $ spack install mpileaks\n\nCustom versions & configurations\n--------------------------------\n\nSpack allows installation to be customized.\nUsers can specify the version, compile-time options, and target architecture, all on the command line.\n\n.. code-block:: spec\n\n # Install a particular version by appending @\n $ spack install hdf5@1.14\n\n # Add special compile-time options by name\n $ spack install hdf5@1.14 api=v110\n\n # Add special boolean compile-time options with +\n $ spack install hdf5@1.14 +hl\n\n # Add compiler flags using the conventional names\n $ spack install hdf5@1.14 cflags=\"-O3 -floop-block\"\n\n # Target a specific micro-architecture\n $ spack install hdf5@1.14 target=icelake\n\nUsers can specify as many or as few options as they care about.\nSpack will fill in the unspecified values with sensible defaults.\n\nCustomize dependencies\n----------------------\n\nSpack allows *dependencies* of a particular installation to be customized extensively.\nUsers can specify both *direct* dependencies of a package, using the ``%`` sigil, or *transitive* dependencies, using the ``^`` sigil:\n\n.. code-block:: spec\n\n # Install hdf5 using gcc@15 as a compiler (direct dependency of hdf5)\n $ spack install hdf5@1.14 %gcc@15\n\n # Install hdf5 using hwloc with CUDA enabled (transitive dependency)\n $ spack install hdf5@1.14 ^hwloc+cuda\n\nThe expression on the command line can be as simple or as complicated as the user needs:\n\n.. code-block:: spec\n\n # Install hdf5 compiled with gcc@15, linked to mpich compiled with gcc@14\n $ spack install hdf5@1.14 %gcc@15 ^mpich %gcc@14\n\nNon-destructive installs\n------------------------\n\nSpack installs every unique package/dependency configuration into its own prefix, so new installs will not break existing ones.\n\nPackages can peacefully coexist\n-------------------------------\n\nSpack avoids library misconfiguration by using ``RPATH`` to link dependencies.\nWhen a user links a library or runs a program, it is tied to the dependencies it was built with, so there is no need to manipulate ``LD_LIBRARY_PATH`` at runtime.\n\nUnprivileged user installs\n--------------------------\n\nSpack does not require administrator privileges to install packages.\nYou can install software in any directory you choose, making it easy to manage packages in your home directory or shared project locations without needing sudo access.\n\nFrom source and binary\n----------------------\n\nSpack's core strength is creating highly customized, optimized software builds from source code.\nWhile it's primarily a from-source package manager, it also supports fast binary installations through build caches.\n\nContributing is easy\n--------------------\n\nTo contribute a new package, all Spack needs is a URL for the source archive.\nThe ``spack create`` command will create a boilerplate package file, and the package authors can fill in specific build steps in pure Python.\n\nFor example, this command:\n\n.. code-block:: console\n\n $ spack create https://ftp.osuosl.org/pub/blfs/conglomeration/libelf/libelf-0.8.13.tar.gz\n\ncreates a simple Python file:\n\n.. code-block:: python\n\n from spack.package import *\n\n\n class Libelf(AutotoolsPackage):\n \"\"\"FIXME: Put a proper description of your package here.\"\"\"\n\n # FIXME: Add a proper url for your package's homepage here.\n homepage = \"https://www.example.com\"\n url = \"https://ftp.osuosl.org/pub/blfs/conglomeration/libelf/libelf-0.8.13.tar.gz\"\n\n # FIXME: Add a list of GitHub accounts to\n # notify when the package is updated.\n # maintainers(\"github_user1\", \"github_user2\")\n\n version(\"0.8.13\", sha256=\"591a9b4ec81c1f2042a97aa60564e0cb79d041c52faa7416acb38bc95bd2c76d\")\n\n # FIXME: Add dependencies if required.\n # depends_on(\"foo\")\n\n def configure_args(self):\n # FIXME: Add arguments other than --prefix\n # FIXME: If not needed delete this function\n args = []\n return args\n\nIt doesn't take much Python coding to get from there to a working package:\n\n.. literalinclude:: .spack/spack-packages/repos/spack_repo/builtin/packages/libelf/package.py\n :lines: 5-\n\n\nUnderstanding Spack's scope\n---------------------------\n\nSpack is a package manager designed for performance and customization of software.\nTo clarify its role and prevent common misconceptions, it's helpful to understand what falls outside of its current scope:\n\n1. Spack is a user-space tool, not an operating system.\n It runs on top of your existing OS (like Linux, macOS, or Windows) and complements the system's native package manager (like ``yum`` or ``apt``), but does not replace it.\n Spack relies on the host system for essentials like the C runtime libraries.\n Building a software stack with a custom `libc` is a planned future capability but is not yet implemented.\n\n2. Spack performs native builds, not cross-compilation.\n It builds software for the same processor architecture it is running on.\n Support for cross-compilation (e.g., building for an ARM processor on an x86 machine) is a planned future capability but is not yet implemented.\n" }, { "path": "lib/spack/docs/frequently_asked_questions.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Answers to common Spack questions, including version and variant selection, package preferences, compiler configuration, and concretizer behavior, with practical YAML and command-line examples.\n\nFrequently Asked Questions\n==========================\n\nThis page contains answers to frequently asked questions about Spack.\nIf you have questions that are not answered here, feel free to ask on `Slack `_ or `GitHub Discussions `_.\nIf you've learned the answer to a question that you think should be here, please consider contributing to this page.\n\n.. _faq-concretizer-precedence:\n\nWhy does Spack pick particular versions and variants?\n-----------------------------------------------------\n\nThis question comes up in a variety of forms:\n\n1. Why does Spack seem to ignore my package preferences from ``packages.yaml`` configuration?\n2. Why does Spack toggle a variant instead of using the default from the ``package.py`` file?\n\nThe short answer is that Spack always picks an optimal configuration based on a complex set of criteria\\ [#f1]_.\nThese criteria are more nuanced than always choosing the latest versions or default variants.\n\n.. note::\n\n As a rule of thumb: requirements + constraints > strong preferences > reuse > preferences > defaults.\n\nThe following set of criteria (from lowest to highest precedence) explains common cases where concretization output may seem surprising at first.\n\n1. :ref:`Package preferences ` configured in ``packages.yaml`` override variant defaults from ``package.py`` files, and influence the optimal ordering of versions.\n Preferences are specified as follows:\n\n .. code-block:: yaml\n\n packages:\n foo:\n version: [1.0, 1.1]\n variants: ~mpi\n\n2. :ref:`Reuse concretization ` configured in ``concretizer.yaml`` overrides preferences, since it's typically faster to reuse an existing spec than to build a preferred one from sources.\n When build caches are enabled, specs may be reused from a remote location too.\n Reuse concretization is configured as follows:\n\n .. code-block:: yaml\n\n concretizer:\n reuse: dependencies # other options are 'true' and 'false'\n\n3. :ref:`Strong preferences ` configured in ``packages.yaml`` are higher priority than reuse, and can be used to strongly prefer a specific version or variant, without erroring out if it's not possible.\n Strong preferences are specified as follows:\n\n .. code-block:: yaml\n\n packages:\n foo:\n prefer:\n - \"@1.1: ~mpi\"\n\n4. :ref:`Package requirements ` configured in ``packages.yaml``, and constraints from the command line as well as ``package.py`` files override all of the above.\n Requirements are specified as follows:\n\n .. code-block:: yaml\n\n packages:\n foo:\n require:\n - \"@1.2: +mpi\"\n conflict:\n - \"@1.4\"\n\nRequirements and constraints restrict the set of possible solutions, while reuse behavior and preferences influence what an optimal solution looks like.\n\nHow do I use a specific compiler?\n---------------------------------\n\nWhen you have multiple compilers available in :ref:`spack-compiler-list`, and want to build your packages with a specific one, you have the following options:\n\n1. Specify your compiler preferences globally for all packages in configuration files.\n2. Specify them on the level of individual specs, like ``pkg %gcc@15`` or ``pkg %c,cxx=gcc@15``.\n\nWe'll explore both options in more detail.\n\nSpecific compiler for all packages\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nIf you want to use a specific compiler for all packages, it's best to use :ref:`strong preferences in packages.yaml config `.\nThe following example prefers GCC 15 for all languages ``c``, ``cxx``, and ``fortran``:\n\n.. code-block:: yaml\n :caption: Recommended: *prefer* a specific compiler\n :name: code-example-prefer-compiler\n\n packages:\n c:\n prefer:\n - gcc@15\n cxx:\n prefer:\n - gcc@15\n fortran:\n prefer:\n - gcc@15\n\nYou can also replace ``prefer:`` with ``require:`` if you want Spack to produce an error if the preferred compiler cannot be used.\nSee also :ref:`the previous FAQ entry `.\n\nIn Spack, the languages ``c``, ``cxx`` and ``fortran`` are :ref:`virtual packages `, on which packages depend if they need a compiler for that language.\nCompiler packages provide these language virtuals.\nWhen you specify these strong preferences, Spack determines whether the package depends on any of the language virtuals, and if so, it applies the associated compiler spec when possible.\n\nWhat is **not recommended** is to define ``%gcc`` as a required dependency of all packages:\n\n.. code-block:: yaml\n :caption: Incorrect: requiring a dependency on a compiler for all packages\n :name: code-example-typical-mistake-require-compiler\n\n packages:\n all:\n require:\n - \"%gcc@15\"\n\nThis is *incorrect*, because some packages do not need a compiler at all (e.g. pure Python packages).\n\nSpecific compiler for individual specs\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nIf different parts of your software stack need to be built with different compilers, it's best to specify compilers as dependencies of the relevant specs (whether on the command line or in Spack environments).\n\n.. code-block:: spec\n :caption: Example of specifying different compilers for different specs\n :name: console-example-different-compilers\n\n $ spack install foo %gcc@15 ^bar %intel-oneapi-compilers\n\nWhat this means is that ``foo`` will depend on GCC 15, while ``bar`` will depend on ``intel-oneapi-compilers``.\n\nYou can also be more specific about what compiler to use for a particular language:\n\n.. code-block:: spec\n :caption: Example of specifying different compilers for different languages\n :name: console-example-different-languages\n\n $ spack install foo %c,cxx=gcc@15 %fortran=intel-oneapi-compilers\n\nThese input specs can be simplified using :doc:`toolchains_yaml`.\nSee also :ref:`pitfalls-without-toolchains` for common mistakes to avoid.\n\n.. rubric:: Footnotes\n\n.. [#f1] The exact list of criteria can be retrieved with the :ref:`spack-solve` command.\n" }, { "path": "lib/spack/docs/getting_help.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Find out how to get help with Spack, including using the spack help command.\n\nGetting Help\n============\n\n.. _cmd-spack-help:\n\n``spack help``\n--------------\n\nIf you don't find what you need here, the ``help`` subcommand will print out a list of *all* of Spack's options and subcommands:\n\n.. command-output:: spack help\n\nAdding an argument, e.g., ``spack help ``, will print out usage information for a particular subcommand:\n\n.. command-output:: spack help install\n\nAlternatively, you can use ``spack --help`` in place of ``spack help``, or ``spack --help`` to get help on a particular subcommand.\n" }, { "path": "lib/spack/docs/getting_started.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n A beginner's guide to Spack, walking you through the initial setup, basic commands, and core concepts to get you started with managing software.\n\n.. _getting_started:\n\nGetting Started\n===============\n\nGetting Spack is easy.\nYou can clone it from the `GitHub repository `_ using this command:\n\n.. code-block:: console\n\n $ git clone --depth=2 https://github.com/spack/spack.git\n\nThis will create a directory called ``spack``.\nOnce you have cloned Spack, we recommend sourcing the appropriate script for your shell.\n\nFor *bash*, *zsh* and *sh* users:\n\n.. code-block:: console\n\n $ . spack/share/spack/setup-env.sh\n\nFor *csh* and *tcsh* users:\n\n.. code-block:: console\n\n $ source spack/share/spack/setup-env.csh\n\nFor *fish* users:\n\n.. code-block:: console\n\n $ . spack/share/spack/setup-env.fish\n\nNow you're ready to use Spack!\n\nList packages you can install\n-----------------------------\n\nOnce Spack is ready, you can list all the packages it knows about with the following command:\n\n.. code-block:: spec\n\n $ spack list\n\nIf you want to get more information on a specific package, for instance ``hdf5``, you can use:\n\n.. code-block:: spec\n\n $ spack info hdf5\n\nThis command shows information about ``hdf5``, including a brief description, the versions of the package Spack knows about, and all the options you can activate when installing.\n\nAs you can see, it's quite simple to gather basic information on packages before you install them!\n\n.. admonition:: Slowdown on the very first command\n :class: warning\n\n The first command you run with Spack may take a while, as Spack builds caches to speed up future commands.\n\nInstalling your first package\n-----------------------------\n\nTo install most packages, Spack needs a compiler suite to be available.\nTo search your machine for available compilers, you can run:\n\n.. code-block:: console\n\n $ spack compiler find\n\nThe command shows users whether any compilers were found and where their configuration is stored.\nIf the search was successful, you can now list known compilers, and get an output similar to the following:\n\n.. code-block:: console\n\n $ spack compiler list\n ==> Available compilers\n -- gcc ubuntu20.04-x86_64 ---------------------------------------\n [e] gcc@9.4.0 [e] gcc@8.4.0 [e] gcc@10.5.0\n\nIf no compilers were found, you need to either:\n\n* Install further prerequisites, see :ref:`verify-spack-prerequisites`, and repeat the search above.\n* Register a build cache that provides a compiler already available as a binary\n\nOnce a compiler is available, you can proceed installing your first package:\n\n.. code-block:: spec\n\n $ spack install tcl\n\nThe output of this command should look similar to the following:\n\n.. code-block:: text\n\n [e] zmjbkxx gcc@10.5.0 /usr (0s)\n [e] rawvy4p glibc@2.31 /usr (0s)\n [+] 5qfbgng compiler-wrapper@1.0 /home/spack/.local/spack/opt/linux-icelake/compiler-wrapper-1.0-5qfbgngzoqcjfbwrjn2vh75fr3g25c35 (0s)\n [+] vchaib2 gcc-runtime@10.5.0 /home/spack/.local/spack/opt/linux-icelake/gcc-runtime-10.5.0-vchaib2njqlk2cud4a2n33tabq526qjj (0s)\n [+] vzazvty gmake@4.4.1 /home/spack/.local/spack/opt/linux-icelake/gmake-4.4.1-vzazvtyn5cjdmg3vkkuau35x7hzu7pyl (12s)\n [+] soedrhb zlib-ng@2.3.3 /home/spack/.local/spack/opt/linux-icelake/zlib-ng-2.3.3-soedrhbnpeordiixaib6utcple6tpgya (3s)\n [+] u6nztpk tcl@8.6.17 /home/spack/.local/spack/opt/linux-icelake/tcl-8.6.17-u6nztpkhzbga4ul665qqhxucxqk3cins (49s)\n\n\nCongratulations!\nYou just installed your first package with Spack!\n\nUse the software you just installed\n-----------------------------------\n\nOnce you have installed ``tcl``, you can immediately use it by starting the ``tclsh`` with its absolute path:\n\n.. code-block:: console\n\n $ /home/spack/.local/spack/opt/linux-icelake/tcl-8.6.17-u6nztpkhzbga4ul665qqhxucxqk3cins/bin/tclsh\n >% echo \"Hello world!\"\n Hello world!\n\nThis works, but using such a long absolute path is not the most convenient way to run an executable.\n\nThe simplest way to have ``tclsh`` available on the command line is:\n\n.. code-block:: spec\n\n $ spack load tcl\n\nThe environment of the current shell has now been modified, and you can run\n\n.. code-block:: console\n\n $ tclsh\n\ndirectly.\nTo undo these modifications, you can:\n\n.. code-block:: spec\n\n $ spack unload tcl\n\n.. admonition:: Environments and views\n :class: tip\n\n :doc:`Spack Environments ` are a better way to install and load a set of packages that are frequently used together.\n The discussion of this topic goes beyond this ``Getting Started`` guide, and we refer to :ref:`environments` for more information.\n\nNext steps\n----------\n\nThis section helped you get Spack installed and running quickly.\nThere are further resources in the documentation that cover both basic and advanced topics in more detail:\n\nBasic Usage\n 1. :ref:`basic-usage`\n 2. :ref:`compiler-config`\n 3. :doc:`environments_basics`\n\nAdvanced Topics\n 1. :ref:`toolchains`\n 2. :ref:`cmd-spack-audit`\n 3. :ref:`cmd-spack-verify`\n" }, { "path": "lib/spack/docs/google5fda5f94b4ffb8de.html", "content": "google-site-verification: google5fda5f94b4ffb8de.html" }, { "path": "lib/spack/docs/gpu_configuration.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n A guide to configuring Spack to use external GPU support, including ROCm and CUDA installations, as well as the OpenGL API.\n\nUsing External GPU Support\n==========================\n\nMany packages come with a ``+cuda`` or ``+rocm`` variant.\nWith no added configuration, Spack will download and install the needed components.\nIt may be preferable to use existing system support: the following sections help with using a system installation of GPU libraries.\n\nUsing an External ROCm Installation\n-----------------------------------\n\nSpack breaks down ROCm into many separate component packages.\nThe following is an example ``packages.yaml`` that organizes a consistent set of ROCm components for use by dependent packages:\n\n.. code-block:: yaml\n\n packages:\n all:\n variants: amdgpu_target=gfx90a\n hip:\n buildable: false\n externals:\n - spec: hip@5.3.0\n prefix: /opt/rocm-5.3.0/hip\n hsa-rocr-dev:\n buildable: false\n externals:\n - spec: hsa-rocr-dev@5.3.0\n prefix: /opt/rocm-5.3.0/\n comgr:\n buildable: false\n externals:\n - spec: comgr@5.3.0\n prefix: /opt/rocm-5.3.0/\n hipsparse:\n buildable: false\n externals:\n - spec: hipsparse@5.3.0\n prefix: /opt/rocm-5.3.0/\n hipblas:\n buildable: false\n externals:\n - spec: hipblas@5.3.0\n prefix: /opt/rocm-5.3.0/\n rocblas:\n buildable: false\n externals:\n - spec: rocblas@5.3.0\n prefix: /opt/rocm-5.3.0/\n rocprim:\n buildable: false\n externals:\n - spec: rocprim@5.3.0\n prefix: /opt/rocm-5.3.0/rocprim/\n\nThis is in combination with the following compiler definition:\n\n.. code-block:: yaml\n\n packages:\n llvm-amdgpu:\n externals:\n - spec: llvm-amdgpu@=5.3.0\n prefix: /opt/rocm-5.3.0\n extra_attributes:\n compilers:\n c: /opt/rocm-5.3.0/bin/amdclang\n cxx: /opt/rocm-5.3.0/bin/amdclang++\n\nThis includes the following considerations:\n\n- Each of the listed externals specifies ``buildable: false`` to force Spack to use only the externals we defined.\n- ``spack external find`` can automatically locate some of the ``hip``/``rocm`` packages, but not all of them, and furthermore not in a manner that guarantees a complementary set if multiple ROCm installations are available.\n- The ``prefix`` is the same for several components, but note that others require listing one of the subdirectories as a prefix.\n\nUsing an External CUDA Installation\n-----------------------------------\n\nCUDA is split into fewer components and is simpler to specify:\n\n.. code-block:: yaml\n\n packages:\n all:\n variants:\n - cuda_arch=70\n cuda:\n buildable: false\n externals:\n - spec: cuda@11.0.2\n prefix: /opt/cuda/cuda-11.0.2/\n\nwhere ``/opt/cuda/cuda-11.0.2/lib/`` contains ``libcudart.so``.\n\n\n\nUsing an External OpenGL API\n----------------------------\nDepending on whether we have a graphics card or not, we may choose to use OSMesa or GLX to implement the OpenGL API.\n\nIf a graphics card is unavailable, OSMesa is recommended and can typically be built with Spack.\nHowever, if we prefer to utilize the system GLX tailored to our graphics card, we need to declare it as an external.\nHere's how to do it:\n\n\n.. code-block:: yaml\n\n packages:\n libglx:\n require: [opengl]\n opengl:\n buildable: false\n externals:\n - prefix: /usr/\n spec: opengl@4.6\n\nNote that the prefix has to be the root of both the libraries and the headers (e.g., ``/usr``), not the path to the ``lib`` directory.\nTo know which spec for OpenGL is available, use ``cd /usr/include/GL && grep -Ri gl_version``.\n" }, { "path": "lib/spack/docs/images/packaging.excalidrawlib", "content": "{\n \"type\": \"excalidrawlib\",\n \"version\": 2,\n \"source\": \"https://excalidraw.com\",\n \"libraryItems\": [\n {\n \"status\": \"unpublished\",\n \"elements\": [\n {\n \"type\": \"rectangle\",\n \"version\": 601,\n \"versionNonce\": 158569138,\n \"isDeleted\": false,\n \"id\": \"8MYJkzMoNEhDhGH1FB83g\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 2,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 445.75,\n \"y\": 129,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#ced4da\",\n \"width\": 736,\n \"height\": 651,\n \"seed\": 448140078,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664627195460,\n \"link\": null,\n \"locked\": false\n },\n {\n \"type\": \"rectangle\",\n \"version\": 195,\n \"versionNonce\": 1239338030,\n \"isDeleted\": false,\n \"id\": \"2CKbNSYnk0z80hSe6axnR\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 2,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 470.25,\n \"y\": 164,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#228be6\",\n \"width\": 495,\n \"height\": 455,\n \"seed\": 566918834,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [\n {\n \"id\": \"IU_VoaKHNHswI8HaxNWt5\",\n \"type\": \"arrow\"\n }\n ],\n \"updated\": 1664627105795,\n \"link\": null,\n \"locked\": false\n },\n {\n \"type\": \"rectangle\",\n \"version\": 403,\n \"versionNonce\": 56919410,\n \"isDeleted\": false,\n \"id\": \"XUzv2kfpdxMahaSVVS42X\",\n \"fillStyle\": \"solid\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 509.25,\n \"y\": 407.5,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#ced4da\",\n \"width\": 396.99999999999994,\n \"height\": 112,\n \"seed\": 354909550,\n \"groupIds\": [\n \"LYqioPcAzrIgJBDV3IaDA\",\n \"SsaCg2uTI9sJjhD323wkh\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [\n {\n \"id\": \"71z_J7hoepiXas8Fk5x0B\",\n \"type\": \"arrow\"\n },\n {\n \"id\": \"IU_VoaKHNHswI8HaxNWt5\",\n \"type\": \"arrow\"\n }\n ],\n \"updated\": 1664627099901,\n \"link\": null,\n \"locked\": false\n },\n {\n \"type\": \"text\",\n \"version\": 300,\n \"versionNonce\": 925254318,\n \"isDeleted\": false,\n \"id\": \"lkCxvsSEn-AuBHtfj1N0d\",\n \"fillStyle\": \"solid\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 547.25,\n \"y\": 441,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#ced4da\",\n \"width\": 321,\n \"height\": 45,\n \"seed\": 1361827954,\n \"groupIds\": [\n \"LYqioPcAzrIgJBDV3IaDA\",\n \"SsaCg2uTI9sJjhD323wkh\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664627099902,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 36,\n \"fontFamily\": 1,\n \"text\": \"AutotoolsPackage\",\n \"baseline\": 32,\n \"textAlign\": \"center\",\n \"verticalAlign\": \"top\",\n \"containerId\": null,\n \"originalText\": \"AutotoolsPackage\"\n },\n {\n \"type\": \"rectangle\",\n \"version\": 377,\n \"versionNonce\": 1733756722,\n \"isDeleted\": false,\n \"id\": \"aCDb2PgRdoFKA8e-GqQzR\",\n \"fillStyle\": \"solid\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 509.25,\n \"y\": 200,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#ced4da\",\n \"width\": 396.99999999999994,\n \"height\": 112,\n \"seed\": 175218606,\n \"groupIds\": [\n \"WEeFev8dTdo9KgzR3hPki\",\n \"SsaCg2uTI9sJjhD323wkh\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [\n {\n \"id\": \"71z_J7hoepiXas8Fk5x0B\",\n \"type\": \"arrow\"\n }\n ],\n \"updated\": 1664627099902,\n \"link\": null,\n \"locked\": false\n },\n {\n \"type\": \"text\",\n \"version\": 161,\n \"versionNonce\": 585481454,\n \"isDeleted\": false,\n \"id\": \"fXYOlmw0CV0WFTNUDity0\",\n \"fillStyle\": \"solid\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 627.75,\n \"y\": 233.5,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#ced4da\",\n \"width\": 160,\n \"height\": 45,\n \"seed\": 1186724402,\n \"groupIds\": [\n \"WEeFev8dTdo9KgzR3hPki\",\n \"SsaCg2uTI9sJjhD323wkh\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664627099902,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 36,\n \"fontFamily\": 1,\n \"text\": \"ArpackNg\",\n \"baseline\": 32,\n \"textAlign\": \"center\",\n \"verticalAlign\": \"top\",\n \"containerId\": null,\n \"originalText\": \"ArpackNg\"\n },\n {\n \"type\": \"arrow\",\n \"version\": 290,\n \"versionNonce\": 890458354,\n \"isDeleted\": false,\n \"id\": \"71z_J7hoepiXas8Fk5x0B\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 4,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 707.8516807799414,\n \"y\": 403,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 0,\n \"height\": 85,\n \"seed\": 247298542,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664627099902,\n \"link\": null,\n \"locked\": false,\n \"startBinding\": {\n \"focus\": 0.02318227093169459,\n \"gap\": 3,\n \"elementId\": \"XUzv2kfpdxMahaSVVS42X\"\n },\n \"endBinding\": {\n \"focus\": -0.02318227093169459,\n \"gap\": 6,\n \"elementId\": \"aCDb2PgRdoFKA8e-GqQzR\"\n },\n \"lastCommittedPoint\": null,\n \"startArrowhead\": null,\n \"endArrowhead\": \"triangle\",\n \"points\": [\n [\n 0,\n 0\n ],\n [\n 0,\n -85\n ]\n ]\n },\n {\n \"type\": \"text\",\n \"version\": 673,\n \"versionNonce\": 1429991214,\n \"isDeleted\": false,\n \"id\": \"bsoYa0EVTdXYsTx5nsFJk\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 4,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 783.25,\n \"y\": 518.3821170339361,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 164,\n \"height\": 90,\n \"seed\": 1633805298,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [\n {\n \"id\": \"IU_VoaKHNHswI8HaxNWt5\",\n \"type\": \"arrow\"\n }\n ],\n \"updated\": 1664627099902,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 36,\n \"fontFamily\": 1,\n \"text\": \"Package \\nHierarchy\",\n \"baseline\": 77,\n \"textAlign\": \"center\",\n \"verticalAlign\": \"top\",\n \"containerId\": null,\n \"originalText\": \"Package \\nHierarchy\"\n },\n {\n \"type\": \"rectangle\",\n \"version\": 903,\n \"versionNonce\": 1712814318,\n \"isDeleted\": false,\n \"id\": \"qRi5xNnAOqg-SFwtYBpoN\",\n \"fillStyle\": \"solid\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 510.25,\n \"y\": 657.5,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#ced4da\",\n \"width\": 396.99999999999994,\n \"height\": 112,\n \"seed\": 1226050606,\n \"groupIds\": [\n \"-wCL8N0qNvseDw29hpA8g\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [\n {\n \"id\": \"IU_VoaKHNHswI8HaxNWt5\",\n \"type\": \"arrow\"\n }\n ],\n \"updated\": 1664627118807,\n \"link\": null,\n \"locked\": false\n },\n {\n \"type\": \"text\",\n \"version\": 623,\n \"versionNonce\": 492299954,\n \"isDeleted\": false,\n \"id\": \"9h25d9NB-Q9Wc79boMEnC\",\n \"fillStyle\": \"solid\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 552.25,\n \"y\": 691,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#ced4da\",\n \"width\": 313,\n \"height\": 45,\n \"seed\": 186946994,\n \"groupIds\": [\n \"-wCL8N0qNvseDw29hpA8g\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664627118807,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 36,\n \"fontFamily\": 1,\n \"text\": \"Builder Forwarder\",\n \"baseline\": 32,\n \"textAlign\": \"center\",\n \"verticalAlign\": \"top\",\n \"containerId\": null,\n \"originalText\": \"Builder Forwarder\"\n },\n {\n \"type\": \"text\",\n \"version\": 1188,\n \"versionNonce\": 351671150,\n \"isDeleted\": false,\n \"id\": \"IlomIIocRvEmmYro4MZ68\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 4,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 1002.75,\n \"y\": 168.5,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 157,\n \"height\": 90,\n \"seed\": 1428885362,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664627188273,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 36,\n \"fontFamily\": 1,\n \"text\": \"Package\\n Wrapper\",\n \"baseline\": 77,\n \"textAlign\": \"center\",\n \"verticalAlign\": \"top\",\n \"containerId\": null,\n \"originalText\": \"Package\\n Wrapper\"\n },\n {\n \"type\": \"arrow\",\n \"version\": 832,\n \"versionNonce\": 1121332014,\n \"isDeleted\": false,\n \"id\": \"IU_VoaKHNHswI8HaxNWt5\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 4,\n \"strokeStyle\": \"dotted\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 707.7778281289579,\n \"y\": 653.5,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 7.847537838213611,\n \"height\": 130.23576593212783,\n \"seed\": 1301783086,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664627118807,\n \"link\": null,\n \"locked\": false,\n \"startBinding\": {\n \"elementId\": \"qRi5xNnAOqg-SFwtYBpoN\",\n \"focus\": 0.013062197564634722,\n \"gap\": 4\n },\n \"endBinding\": {\n \"elementId\": \"XUzv2kfpdxMahaSVVS42X\",\n \"focus\": 0.056574233332975385,\n \"gap\": 3.7642340678721666\n },\n \"lastCommittedPoint\": null,\n \"startArrowhead\": null,\n \"endArrowhead\": \"triangle\",\n \"points\": [\n [\n 0,\n 0\n ],\n [\n -7.847537838213611,\n -130.23576593212783\n ]\n ]\n }\n ],\n \"id\": \"mulubEO9Lw-HgC00sx7G-\",\n \"created\": 1664627205632\n },\n {\n \"status\": \"unpublished\",\n \"elements\": [\n {\n \"type\": \"rectangle\",\n \"version\": 360,\n \"versionNonce\": 699609906,\n \"isDeleted\": false,\n \"id\": \"ai3MIBTq8Rkokk4d2NJ_k\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 2,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 441.5,\n \"y\": 56,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#228be6\",\n \"width\": 479,\n \"height\": 642,\n \"seed\": 725687342,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664622926148,\n \"link\": null,\n \"locked\": false\n },\n {\n \"type\": \"rectangle\",\n \"version\": 327,\n \"versionNonce\": 1239118706,\n \"isDeleted\": false,\n \"id\": \"7tuXfM91g28UGae9gJkis\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 2,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 993.25,\n \"y\": 53,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#228be6\",\n \"width\": 479,\n \"height\": 642,\n \"seed\": 860539570,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [\n {\n \"id\": \"F6E1EQxM-PyPeNjQXH6NZ\",\n \"type\": \"arrow\"\n }\n ],\n \"updated\": 1664623054904,\n \"link\": null,\n \"locked\": false\n },\n {\n \"type\": \"rectangle\",\n \"version\": 482,\n \"versionNonce\": 616506034,\n \"isDeleted\": false,\n \"id\": \"TmgDkNmbU86sH2Ssf1mL2\",\n \"fillStyle\": \"solid\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 1030.75,\n \"y\": 503.5,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#ced4da\",\n \"width\": 396.99999999999994,\n \"height\": 112,\n \"seed\": 329380206,\n \"groupIds\": [\n \"rqi4zfKDNJjqgRyIIknBO\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [\n {\n \"id\": \"RQl1RtMzUcPE_zXHt8Ldm\",\n \"type\": \"arrow\"\n },\n {\n \"id\": \"F6E1EQxM-PyPeNjQXH6NZ\",\n \"type\": \"arrow\"\n },\n {\n \"id\": \"Iey2r9ev3NqXShFhDRa3t\",\n \"type\": \"arrow\"\n }\n ],\n \"updated\": 1664623131360,\n \"link\": null,\n \"locked\": false\n },\n {\n \"type\": \"text\",\n \"version\": 377,\n \"versionNonce\": 1649618094,\n \"isDeleted\": false,\n \"id\": \"M6LF3AKrGIzDW8p00PLeg\",\n \"fillStyle\": \"solid\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 1068.75,\n \"y\": 537,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#ced4da\",\n \"width\": 321,\n \"height\": 45,\n \"seed\": 1690477682,\n \"groupIds\": [\n \"rqi4zfKDNJjqgRyIIknBO\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664622926151,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 36,\n \"fontFamily\": 1,\n \"text\": \"AutotoolsPackage\",\n \"baseline\": 32,\n \"textAlign\": \"center\",\n \"verticalAlign\": \"top\",\n \"containerId\": null,\n \"originalText\": \"AutotoolsPackage\"\n },\n {\n \"type\": \"rectangle\",\n \"version\": 466,\n \"versionNonce\": 378147058,\n \"isDeleted\": false,\n \"id\": \"-34MaUc1fQDbeqLTRUx91\",\n \"fillStyle\": \"solid\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 1030.625,\n \"y\": 296,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#ced4da\",\n \"width\": 396.99999999999994,\n \"height\": 112,\n \"seed\": 964531118,\n \"groupIds\": [\n \"TtAdfrQjw8FIlPZMGmWhX\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [\n {\n \"id\": \"RQl1RtMzUcPE_zXHt8Ldm\",\n \"type\": \"arrow\"\n },\n {\n \"id\": \"7czUS_PAuM5hdRJoQRDRT\",\n \"type\": \"arrow\"\n },\n {\n \"id\": \"Iey2r9ev3NqXShFhDRa3t\",\n \"type\": \"arrow\"\n }\n ],\n \"updated\": 1664623131360,\n \"link\": null,\n \"locked\": false\n },\n {\n \"type\": \"text\",\n \"version\": 250,\n \"versionNonce\": 1826973422,\n \"isDeleted\": false,\n \"id\": \"85YHNomCStJoIV17Sp0A6\",\n \"fillStyle\": \"solid\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 1093.625,\n \"y\": 329.5,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#ced4da\",\n \"width\": 271,\n \"height\": 45,\n \"seed\": 1436108338,\n \"groupIds\": [\n \"TtAdfrQjw8FIlPZMGmWhX\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664622926151,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 36,\n \"fontFamily\": 1,\n \"text\": \"builtin.ArpackNg\",\n \"baseline\": 32,\n \"textAlign\": \"center\",\n \"verticalAlign\": \"top\",\n \"containerId\": null,\n \"originalText\": \"builtin.ArpackNg\"\n },\n {\n \"type\": \"arrow\",\n \"version\": 476,\n \"versionNonce\": 1270564594,\n \"isDeleted\": false,\n \"id\": \"RQl1RtMzUcPE_zXHt8Ldm\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 4,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 1233.8516807799415,\n \"y\": 499,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 0,\n \"height\": 85,\n \"seed\": 1613426158,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664622926151,\n \"link\": null,\n \"locked\": false,\n \"startBinding\": {\n \"elementId\": \"TmgDkNmbU86sH2Ssf1mL2\",\n \"focus\": 0.023182270931695163,\n \"gap\": 4.5\n },\n \"endBinding\": {\n \"elementId\": \"-34MaUc1fQDbeqLTRUx91\",\n \"focus\": -0.02381199385360952,\n \"gap\": 6\n },\n \"lastCommittedPoint\": null,\n \"startArrowhead\": null,\n \"endArrowhead\": \"triangle\",\n \"points\": [\n [\n 0,\n 0\n ],\n [\n 0,\n -85\n ]\n ]\n },\n {\n \"type\": \"text\",\n \"version\": 693,\n \"versionNonce\": 1438013742,\n \"isDeleted\": false,\n \"id\": \"wSIdF9zegc69r2D38BVMs\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 4,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 1061.75,\n \"y\": 632.3821170339361,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 335,\n \"height\": 45,\n \"seed\": 1052094450,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664622926151,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 36,\n \"fontFamily\": 1,\n \"text\": \"Old-style packages\",\n \"baseline\": 32,\n \"textAlign\": \"center\",\n \"verticalAlign\": \"top\",\n \"containerId\": null,\n \"originalText\": \"Old-style packages\"\n },\n {\n \"type\": \"rectangle\",\n \"version\": 556,\n \"versionNonce\": 1760787058,\n \"isDeleted\": false,\n \"id\": \"lYxakYKLpAmo_DvzDJ27b\",\n \"fillStyle\": \"solid\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 1030.625,\n \"y\": 95,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#ced4da\",\n \"width\": 396.99999999999994,\n \"height\": 112,\n \"seed\": 1302932978,\n \"groupIds\": [\n \"-WCCzMWoqGFfWxksMC6LG\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [\n {\n \"id\": \"RQl1RtMzUcPE_zXHt8Ldm\",\n \"type\": \"arrow\"\n },\n {\n \"id\": \"8Z8HX6DlXqC-qL-63w1ol\",\n \"type\": \"arrow\"\n },\n {\n \"id\": \"ia8wHuSmOVJLvGe5blR5g\",\n \"type\": \"arrow\"\n },\n {\n \"id\": \"7czUS_PAuM5hdRJoQRDRT\",\n \"type\": \"arrow\"\n }\n ],\n \"updated\": 1664623123836,\n \"link\": null,\n \"locked\": false\n },\n {\n \"type\": \"text\",\n \"version\": 341,\n \"versionNonce\": 1412367214,\n \"isDeleted\": false,\n \"id\": \"hF1874wuKYmbBjYAQwrVJ\",\n \"fillStyle\": \"solid\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 1088.125,\n \"y\": 128.5,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#ced4da\",\n \"width\": 282,\n \"height\": 45,\n \"seed\": 524182062,\n \"groupIds\": [\n \"-WCCzMWoqGFfWxksMC6LG\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664622926152,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 36,\n \"fontFamily\": 1,\n \"text\": \"myrepo.ArpackNg\",\n \"baseline\": 32,\n \"textAlign\": \"center\",\n \"verticalAlign\": \"top\",\n \"containerId\": null,\n \"originalText\": \"myrepo.ArpackNg\"\n },\n {\n \"type\": \"arrow\",\n \"version\": 593,\n \"versionNonce\": 214413938,\n \"isDeleted\": false,\n \"id\": \"8Z8HX6DlXqC-qL-63w1ol\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 4,\n \"strokeStyle\": \"dotted\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 1226.4453379157953,\n \"y\": 297.5,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 2.434529927712447,\n \"height\": 84,\n \"seed\": 1326581486,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664622926152,\n \"link\": null,\n \"locked\": false,\n \"startBinding\": null,\n \"endBinding\": {\n \"elementId\": \"lYxakYKLpAmo_DvzDJ27b\",\n \"focus\": -0.00782655608584947,\n \"gap\": 6.5\n },\n \"lastCommittedPoint\": null,\n \"startArrowhead\": null,\n \"endArrowhead\": \"triangle\",\n \"points\": [\n [\n 0,\n 0\n ],\n [\n 2.434529927712447,\n -84\n ]\n ]\n },\n {\n \"type\": \"rectangle\",\n \"version\": 733,\n \"versionNonce\": 390297266,\n \"isDeleted\": false,\n \"id\": \"G4--cV2YGQSrSijvYiNDB\",\n \"fillStyle\": \"solid\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 482.5,\n \"y\": 507,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#ced4da\",\n \"width\": 396.99999999999994,\n \"height\": 112,\n \"seed\": 85080878,\n \"groupIds\": [\n \"qZhg7KFANDHKWmTH71Lm0\",\n \"FSKOW2oS76ubMa6DTOrDh\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [\n {\n \"id\": \"RQl1RtMzUcPE_zXHt8Ldm\",\n \"type\": \"arrow\"\n },\n {\n \"id\": \"BkpnKUCjV1uqDGHPNuNZK\",\n \"type\": \"arrow\"\n },\n {\n \"id\": \"aQdIO4VQx_J6SzCz-xt64\",\n \"type\": \"arrow\"\n },\n {\n \"id\": \"F6E1EQxM-PyPeNjQXH6NZ\",\n \"type\": \"arrow\"\n }\n ],\n \"updated\": 1664623061069,\n \"link\": null,\n \"locked\": false\n },\n {\n \"type\": \"text\",\n \"version\": 577,\n \"versionNonce\": 2001681906,\n \"isDeleted\": false,\n \"id\": \"MbNSUrN26Lx1aERuxunnt\",\n \"fillStyle\": \"solid\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 541.5,\n \"y\": 540.5,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#ced4da\",\n \"width\": 279,\n \"height\": 45,\n \"seed\": 950326962,\n \"groupIds\": [\n \"qZhg7KFANDHKWmTH71Lm0\",\n \"FSKOW2oS76ubMa6DTOrDh\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664622926152,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 36,\n \"fontFamily\": 1,\n \"text\": \"Default Builder\",\n \"baseline\": 32,\n \"textAlign\": \"center\",\n \"verticalAlign\": \"top\",\n \"containerId\": null,\n \"originalText\": \"Default Builder\"\n },\n {\n \"type\": \"rectangle\",\n \"version\": 722,\n \"versionNonce\": 1372930162,\n \"isDeleted\": false,\n \"id\": \"WIS84sS48dCmi8q81Hh9F\",\n \"fillStyle\": \"solid\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 482.5,\n \"y\": 99,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#ced4da\",\n \"width\": 396.99999999999994,\n \"height\": 112,\n \"seed\": 1977410350,\n \"groupIds\": [\n \"_CQwHz-xftDZzy8u9u4YO\",\n \"FSKOW2oS76ubMa6DTOrDh\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [\n {\n \"id\": \"RQl1RtMzUcPE_zXHt8Ldm\",\n \"type\": \"arrow\"\n },\n {\n \"id\": \"BkpnKUCjV1uqDGHPNuNZK\",\n \"type\": \"arrow\"\n },\n {\n \"id\": \"aQdIO4VQx_J6SzCz-xt64\",\n \"type\": \"arrow\"\n },\n {\n \"id\": \"ia8wHuSmOVJLvGe5blR5g\",\n \"type\": \"arrow\"\n }\n ],\n \"updated\": 1664623105535,\n \"link\": null,\n \"locked\": false\n },\n {\n \"type\": \"text\",\n \"version\": 531,\n \"versionNonce\": 1851174834,\n \"isDeleted\": false,\n \"id\": \"qIbTXN1LbDYGZzSceYynz\",\n \"fillStyle\": \"solid\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 540.5,\n \"y\": 132.5,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#ced4da\",\n \"width\": 281,\n \"height\": 45,\n \"seed\": 221818546,\n \"groupIds\": [\n \"_CQwHz-xftDZzy8u9u4YO\",\n \"FSKOW2oS76ubMa6DTOrDh\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664622926152,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 36,\n \"fontFamily\": 1,\n \"text\": \"Adapter Builder\",\n \"baseline\": 32,\n \"textAlign\": \"center\",\n \"verticalAlign\": \"top\",\n \"containerId\": null,\n \"originalText\": \"Adapter Builder\"\n },\n {\n \"type\": \"arrow\",\n \"version\": 85,\n \"versionNonce\": 50141422,\n \"isDeleted\": false,\n \"id\": \"aQdIO4VQx_J6SzCz-xt64\",\n \"fillStyle\": \"solid\",\n \"strokeWidth\": 4,\n \"strokeStyle\": \"solid\",\n \"roughness\": 0,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 670,\n \"y\": 505,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#ced4da\",\n \"width\": 2,\n \"height\": 291,\n \"seed\": 417372974,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664622926152,\n \"link\": null,\n \"locked\": false,\n \"startBinding\": {\n \"elementId\": \"G4--cV2YGQSrSijvYiNDB\",\n \"focus\": -0.05731267980406219,\n \"gap\": 2\n },\n \"endBinding\": {\n \"elementId\": \"WIS84sS48dCmi8q81Hh9F\",\n \"focus\": 0.04321344955983103,\n \"gap\": 3\n },\n \"lastCommittedPoint\": null,\n \"startArrowhead\": null,\n \"endArrowhead\": \"triangle\",\n \"points\": [\n [\n 0,\n 0\n ],\n [\n 2,\n -291\n ]\n ]\n },\n {\n \"type\": \"arrow\",\n \"version\": 720,\n \"versionNonce\": 1494556718,\n \"isDeleted\": false,\n \"id\": \"ia8wHuSmOVJLvGe5blR5g\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 4,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 4.706831282597808,\n \"x\": 932.4285606227004,\n \"y\": 52.69401049592016,\n \"strokeColor\": \"#c92a2a\",\n \"backgroundColor\": \"transparent\",\n \"width\": 47.10077935537049,\n \"height\": 145.9883132350331,\n \"seed\": 314146734,\n \"groupIds\": [],\n \"strokeSharpness\": \"round\",\n \"boundElements\": [],\n \"updated\": 1664623039605,\n \"link\": null,\n \"locked\": false,\n \"startBinding\": {\n \"elementId\": \"WIS84sS48dCmi8q81Hh9F\",\n \"focus\": 0.6597923311816741,\n \"gap\": 2.0985583595166872\n },\n \"endBinding\": {\n \"elementId\": \"lYxakYKLpAmo_DvzDJ27b\",\n \"focus\": -0.6857137990945498,\n \"gap\": 3.0336827015810286\n },\n \"lastCommittedPoint\": null,\n \"startArrowhead\": null,\n \"endArrowhead\": \"triangle\",\n \"points\": [\n [\n 0,\n 0\n ],\n [\n 45.89517648378751,\n 72.7218231059162\n ],\n [\n -1.2056028715829825,\n 145.9883132350331\n ]\n ]\n },\n {\n \"type\": \"text\",\n \"version\": 727,\n \"versionNonce\": 549636846,\n \"isDeleted\": false,\n \"id\": \"JRrvIVZ9KAv56BYbRbCLA\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 4,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 527.5,\n \"y\": 633.5,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 295,\n \"height\": 45,\n \"seed\": 2130028978,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664622926153,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 36,\n \"fontFamily\": 1,\n \"text\": \"Builder Hierarchy\",\n \"baseline\": 32,\n \"textAlign\": \"center\",\n \"verticalAlign\": \"top\",\n \"containerId\": null,\n \"originalText\": \"Builder Hierarchy\"\n },\n {\n \"type\": \"text\",\n \"version\": 281,\n \"versionNonce\": 777063918,\n \"isDeleted\": false,\n \"id\": \"BBj29IYUUwcEAk0aGGgEe\",\n \"fillStyle\": \"solid\",\n \"strokeWidth\": 4,\n \"strokeStyle\": \"solid\",\n \"roughness\": 0,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 746,\n \"y\": 2,\n \"strokeColor\": \"#c92a2a\",\n \"backgroundColor\": \"#ced4da\",\n \"width\": 438,\n \"height\": 35,\n \"seed\": 344107566,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664623034966,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 28,\n \"fontFamily\": 1,\n \"text\": \"Defer to the old-style package\",\n \"baseline\": 25,\n \"textAlign\": \"center\",\n \"verticalAlign\": \"top\",\n \"containerId\": null,\n \"originalText\": \"Defer to the old-style package\"\n },\n {\n \"type\": \"arrow\",\n \"version\": 864,\n \"versionNonce\": 353999662,\n \"isDeleted\": false,\n \"id\": \"F6E1EQxM-PyPeNjQXH6NZ\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 4,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 1.5656833824867196,\n \"x\": 932.5276780900645,\n \"y\": 511.2079252998286,\n \"strokeColor\": \"#c92a2a\",\n \"backgroundColor\": \"transparent\",\n \"width\": 47.10077935537049,\n \"height\": 145.9883132350331,\n \"seed\": 2119154546,\n \"groupIds\": [],\n \"strokeSharpness\": \"round\",\n \"boundElements\": [],\n \"updated\": 1664623061069,\n \"link\": null,\n \"locked\": false,\n \"startBinding\": {\n \"elementId\": \"TmgDkNmbU86sH2Ssf1mL2\",\n \"focus\": 0.700636908798286,\n \"gap\": 3.7338363313426726\n },\n \"endBinding\": {\n \"elementId\": \"G4--cV2YGQSrSijvYiNDB\",\n \"focus\": -0.7137516210459195,\n \"gap\": 1.5235945037890133\n },\n \"lastCommittedPoint\": null,\n \"startArrowhead\": null,\n \"endArrowhead\": \"triangle\",\n \"points\": [\n [\n 0,\n 0\n ],\n [\n 45.89517648378751,\n 72.7218231059162\n ],\n [\n -1.2056028715829825,\n 145.9883132350331\n ]\n ]\n },\n {\n \"type\": \"text\",\n \"version\": 318,\n \"versionNonce\": 1988243186,\n \"isDeleted\": false,\n \"id\": \"VIOq-st9nvReenpiJkr7q\",\n \"fillStyle\": \"solid\",\n \"strokeWidth\": 4,\n \"strokeStyle\": \"solid\",\n \"roughness\": 0,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 828,\n \"y\": 724.5,\n \"strokeColor\": \"#c92a2a\",\n \"backgroundColor\": \"#ced4da\",\n \"width\": 274,\n \"height\": 70,\n \"seed\": 2086072882,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664623095297,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 28,\n \"fontFamily\": 1,\n \"text\": \"Fall-back to the \\nAdapter base class\",\n \"baseline\": 60,\n \"textAlign\": \"center\",\n \"verticalAlign\": \"top\",\n \"containerId\": null,\n \"originalText\": \"Fall-back to the \\nAdapter base class\"\n },\n {\n \"type\": \"arrow\",\n \"version\": 971,\n \"versionNonce\": 1844256174,\n \"isDeleted\": false,\n \"id\": \"7czUS_PAuM5hdRJoQRDRT\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 4,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 6.272294617229998,\n \"x\": 1433.5276780900645,\n \"y\": 163.20792529982862,\n \"strokeColor\": \"#c92a2a\",\n \"backgroundColor\": \"transparent\",\n \"width\": 47.10077935537049,\n \"height\": 145.9883132350331,\n \"seed\": 142056302,\n \"groupIds\": [],\n \"strokeSharpness\": \"round\",\n \"boundElements\": [],\n \"updated\": 1664623123836,\n \"link\": null,\n \"locked\": false,\n \"startBinding\": {\n \"elementId\": \"lYxakYKLpAmo_DvzDJ27b\",\n \"focus\": -0.8331982906950285,\n \"gap\": 5.098981289624589\n },\n \"endBinding\": {\n \"elementId\": \"-34MaUc1fQDbeqLTRUx91\",\n \"focus\": 0.7587321286266477,\n \"gap\": 5.483331940596372\n },\n \"lastCommittedPoint\": null,\n \"startArrowhead\": null,\n \"endArrowhead\": \"triangle\",\n \"points\": [\n [\n 0,\n 0\n ],\n [\n 45.89517648378751,\n 72.7218231059162\n ],\n [\n -1.2056028715829825,\n 145.9883132350331\n ]\n ]\n },\n {\n \"type\": \"arrow\",\n \"version\": 1075,\n \"versionNonce\": 2073112366,\n \"isDeleted\": false,\n \"id\": \"Iey2r9ev3NqXShFhDRa3t\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 4,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 6.272294617229998,\n \"x\": 1434.451400933309,\n \"y\": 387.7559332541056,\n \"strokeColor\": \"#c92a2a\",\n \"backgroundColor\": \"transparent\",\n \"width\": 47.10077935537049,\n \"height\": 145.9883132350331,\n \"seed\": 840513518,\n \"groupIds\": [],\n \"strokeSharpness\": \"round\",\n \"boundElements\": [],\n \"updated\": 1664623131360,\n \"link\": null,\n \"locked\": false,\n \"startBinding\": {\n \"elementId\": \"-34MaUc1fQDbeqLTRUx91\",\n \"focus\": -0.7723329153292293,\n \"gap\": 6.037577244264867\n },\n \"endBinding\": {\n \"elementId\": \"TmgDkNmbU86sH2Ssf1mL2\",\n \"focus\": 0.808011962769455,\n \"gap\": 6.296927895236422\n },\n \"lastCommittedPoint\": null,\n \"startArrowhead\": null,\n \"endArrowhead\": \"triangle\",\n \"points\": [\n [\n 0,\n 0\n ],\n [\n 45.89517648378751,\n 72.7218231059162\n ],\n [\n -1.2056028715829825,\n 145.9883132350331\n ]\n ]\n }\n ],\n \"id\": \"sJP5ES4-kuhrqaBed7Feh\",\n \"created\": 1664623142493\n },\n {\n \"status\": \"unpublished\",\n \"elements\": [\n {\n \"type\": \"rectangle\",\n \"version\": 351,\n \"versionNonce\": 94847218,\n \"isDeleted\": false,\n \"id\": \"QfhQQY4Kvx8RLvCd6qXsx\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 2,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 1011.5,\n \"y\": 249,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#228be6\",\n \"width\": 479,\n \"height\": 438,\n \"seed\": 1024685106,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664612347442,\n \"link\": null,\n \"locked\": false\n },\n {\n \"type\": \"rectangle\",\n \"version\": 156,\n \"versionNonce\": 2082406190,\n \"isDeleted\": false,\n \"id\": \"rMQqqzkSZsBVWvOk137wO\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 2,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 511,\n \"y\": 247,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#228be6\",\n \"width\": 479,\n \"height\": 438,\n \"seed\": 250617778,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664612347443,\n \"link\": null,\n \"locked\": false\n },\n {\n \"type\": \"rectangle\",\n \"version\": 392,\n \"versionNonce\": 414601906,\n \"isDeleted\": false,\n \"id\": \"h2lcAgJBn6WsPKAj3vWS8\",\n \"fillStyle\": \"solid\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 545.5,\n \"y\": 490.5,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#ced4da\",\n \"width\": 396.99999999999994,\n \"height\": 112,\n \"seed\": 721668433,\n \"groupIds\": [\n \"ETPwHpdW1CXh0DtqZ_2na\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [\n {\n \"id\": \"r2Lq0kGXd6aTn5T-ki1aL\",\n \"type\": \"arrow\"\n }\n ],\n \"updated\": 1664612347443,\n \"link\": null,\n \"locked\": false\n },\n {\n \"type\": \"text\",\n \"version\": 293,\n \"versionNonce\": 848488814,\n \"isDeleted\": false,\n \"id\": \"eaxk_MzyrjAjXKf0vmFuU\",\n \"fillStyle\": \"solid\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 583.5,\n \"y\": 524,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#ced4da\",\n \"width\": 321,\n \"height\": 45,\n \"seed\": 1324675135,\n \"groupIds\": [\n \"ETPwHpdW1CXh0DtqZ_2na\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664612347443,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 36,\n \"fontFamily\": 1,\n \"text\": \"AutotoolsPackage\",\n \"baseline\": 32,\n \"textAlign\": \"center\",\n \"verticalAlign\": \"top\",\n \"containerId\": null,\n \"originalText\": \"AutotoolsPackage\"\n },\n {\n \"type\": \"rectangle\",\n \"version\": 370,\n \"versionNonce\": 595405938,\n \"isDeleted\": false,\n \"id\": \"6TAhmS7GKN_ppUHjSVGLb\",\n \"fillStyle\": \"solid\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 545.5,\n \"y\": 283,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#ced4da\",\n \"width\": 396.99999999999994,\n \"height\": 112,\n \"seed\": 2083634783,\n \"groupIds\": [\n \"biKtN87UToAb_UBhyub5I\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [\n {\n \"id\": \"r2Lq0kGXd6aTn5T-ki1aL\",\n \"type\": \"arrow\"\n }\n ],\n \"updated\": 1664612347443,\n \"link\": null,\n \"locked\": false\n },\n {\n \"type\": \"text\",\n \"version\": 155,\n \"versionNonce\": 1066372014,\n \"isDeleted\": false,\n \"id\": \"xyXchzGRLKRPuMVGo17mr\",\n \"fillStyle\": \"solid\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 664,\n \"y\": 316.5,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#ced4da\",\n \"width\": 160,\n \"height\": 45,\n \"seed\": 2066951921,\n \"groupIds\": [\n \"biKtN87UToAb_UBhyub5I\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664612347443,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 36,\n \"fontFamily\": 1,\n \"text\": \"ArpackNg\",\n \"baseline\": 32,\n \"textAlign\": \"center\",\n \"verticalAlign\": \"top\",\n \"containerId\": null,\n \"originalText\": \"ArpackNg\"\n },\n {\n \"type\": \"arrow\",\n \"version\": 285,\n \"versionNonce\": 1807928882,\n \"isDeleted\": false,\n \"id\": \"r2Lq0kGXd6aTn5T-ki1aL\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 4,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 748.6016807799414,\n \"y\": 486,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 0,\n \"height\": 85,\n \"seed\": 1479060383,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664612347443,\n \"link\": null,\n \"locked\": false,\n \"startBinding\": {\n \"elementId\": \"h2lcAgJBn6WsPKAj3vWS8\",\n \"focus\": 0.02318227093169459,\n \"gap\": 3\n },\n \"endBinding\": {\n \"elementId\": \"6TAhmS7GKN_ppUHjSVGLb\",\n \"focus\": -0.02318227093169459,\n \"gap\": 6\n },\n \"lastCommittedPoint\": null,\n \"startArrowhead\": null,\n \"endArrowhead\": \"triangle\",\n \"points\": [\n [\n 0,\n 0\n ],\n [\n 0,\n -85\n ]\n ]\n },\n {\n \"type\": \"text\",\n \"version\": 572,\n \"versionNonce\": 1094575598,\n \"isDeleted\": false,\n \"id\": \"pUx1_v_UyKhu5zXISU4-f\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 4,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 653,\n \"y\": 619.3821170339361,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 182,\n \"height\": 45,\n \"seed\": 1608256017,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664612347443,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 36,\n \"fontFamily\": 1,\n \"text\": \"Metadata\",\n \"baseline\": 32,\n \"textAlign\": \"center\",\n \"verticalAlign\": \"top\",\n \"containerId\": null,\n \"originalText\": \"Metadata\"\n },\n {\n \"type\": \"rectangle\",\n \"version\": 734,\n \"versionNonce\": 1401317810,\n \"isDeleted\": false,\n \"id\": \"4YBPHTc5sQiOKGM9NOZwg\",\n \"fillStyle\": \"solid\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 1045.5,\n \"y\": 490.5,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#ced4da\",\n \"width\": 396.99999999999994,\n \"height\": 112,\n \"seed\": 1687989426,\n \"groupIds\": [\n \"lxE4hLtUAF2X7993lwk8q\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [\n {\n \"id\": \"M8cWqpsa0-iwN_cVJeXEQ\",\n \"type\": \"arrow\"\n }\n ],\n \"updated\": 1664612347443,\n \"link\": null,\n \"locked\": false\n },\n {\n \"type\": \"text\",\n \"version\": 436,\n \"versionNonce\": 1572061806,\n \"isDeleted\": false,\n \"id\": \"P2U0ucf_QPvJcOWlMLp2K\",\n \"fillStyle\": \"solid\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 1183,\n \"y\": 524,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#ced4da\",\n \"width\": 122,\n \"height\": 45,\n \"seed\": 276038958,\n \"groupIds\": [\n \"lxE4hLtUAF2X7993lwk8q\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664612347443,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 36,\n \"fontFamily\": 1,\n \"text\": \"Builder\",\n \"baseline\": 32,\n \"textAlign\": \"center\",\n \"verticalAlign\": \"top\",\n \"containerId\": null,\n \"originalText\": \"Builder\"\n },\n {\n \"type\": \"arrow\",\n \"version\": 489,\n \"versionNonce\": 1663911086,\n \"isDeleted\": false,\n \"id\": \"M8cWqpsa0-iwN_cVJeXEQ\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 4,\n \"strokeStyle\": \"dashed\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 942,\n \"y\": 337,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 303,\n \"height\": 143,\n \"seed\": 1698960686,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664612347443,\n \"link\": null,\n \"locked\": false,\n \"startBinding\": null,\n \"endBinding\": {\n \"elementId\": \"4YBPHTc5sQiOKGM9NOZwg\",\n \"focus\": 0.04820781382766574,\n \"gap\": 10.5\n },\n \"lastCommittedPoint\": null,\n \"startArrowhead\": null,\n \"endArrowhead\": \"dot\",\n \"points\": [\n [\n 0,\n 0\n ],\n [\n 295,\n 0\n ],\n [\n 303,\n 143\n ]\n ]\n },\n {\n \"type\": \"text\",\n \"version\": 841,\n \"versionNonce\": 2059173614,\n \"isDeleted\": false,\n \"id\": \"QGyg9pXnTgByg9Lw9oZKC\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 4,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 1043.5,\n \"y\": 621.5,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 401,\n \"height\": 45,\n \"seed\": 1012078510,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664612347443,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 36,\n \"fontFamily\": 1,\n \"text\": \"Installation Procedure\",\n \"baseline\": 32,\n \"textAlign\": \"center\",\n \"verticalAlign\": \"top\",\n \"containerId\": null,\n \"originalText\": \"Installation Procedure\"\n }\n ],\n \"id\": \"tezI4Q4gBH7mr-Q_us1KO\",\n \"created\": 1664612353293\n },\n {\n \"status\": \"unpublished\",\n \"elements\": [\n {\n \"type\": \"rectangle\",\n \"version\": 273,\n \"versionNonce\": 1078330865,\n \"isDeleted\": false,\n \"id\": \"h2lcAgJBn6WsPKAj3vWS8\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 545.5,\n \"y\": 489,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 396.99999999999994,\n \"height\": 112,\n \"seed\": 721668433,\n \"groupIds\": [\n \"ETPwHpdW1CXh0DtqZ_2na\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [\n {\n \"id\": \"r2Lq0kGXd6aTn5T-ki1aL\",\n \"type\": \"arrow\"\n }\n ],\n \"updated\": 1664534889868,\n \"link\": null,\n \"locked\": false\n },\n {\n \"type\": \"text\",\n \"version\": 174,\n \"versionNonce\": 1400524191,\n \"isDeleted\": false,\n \"id\": \"eaxk_MzyrjAjXKf0vmFuU\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 583.5,\n \"y\": 522.5,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 321,\n \"height\": 45,\n \"seed\": 1324675135,\n \"groupIds\": [\n \"ETPwHpdW1CXh0DtqZ_2na\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664534889868,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 36,\n \"fontFamily\": 1,\n \"text\": \"AutotoolsPackage\",\n \"baseline\": 32,\n \"textAlign\": \"center\",\n \"verticalAlign\": \"top\",\n \"containerId\": null,\n \"originalText\": \"AutotoolsPackage\"\n },\n {\n \"type\": \"text\",\n \"version\": 108,\n \"versionNonce\": 438728849,\n \"isDeleted\": false,\n \"id\": \"xyXchzGRLKRPuMVGo17mr\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 664,\n \"y\": 316.5,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 160,\n \"height\": 45,\n \"seed\": 2066951921,\n \"groupIds\": [\n \"1wm7ikIN28k9zdVSKTLKQ\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664540120970,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 36,\n \"fontFamily\": 1,\n \"text\": \"ArpackNg\",\n \"baseline\": 32,\n \"textAlign\": \"center\",\n \"verticalAlign\": \"top\",\n \"containerId\": null,\n \"originalText\": \"ArpackNg\"\n },\n {\n \"type\": \"rectangle\",\n \"version\": 322,\n \"versionNonce\": 1389146591,\n \"isDeleted\": false,\n \"id\": \"6TAhmS7GKN_ppUHjSVGLb\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 545.5,\n \"y\": 283,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 396.99999999999994,\n \"height\": 112,\n \"seed\": 2083634783,\n \"groupIds\": [\n \"1wm7ikIN28k9zdVSKTLKQ\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [\n {\n \"id\": \"r2Lq0kGXd6aTn5T-ki1aL\",\n \"type\": \"arrow\"\n }\n ],\n \"updated\": 1664534889868,\n \"link\": null,\n \"locked\": false\n },\n {\n \"type\": \"arrow\",\n \"version\": 94,\n \"versionNonce\": 787416433,\n \"isDeleted\": false,\n \"id\": \"r2Lq0kGXd6aTn5T-ki1aL\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 4,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 748.6016807799414,\n \"y\": 486,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 0,\n \"height\": 85,\n \"seed\": 1479060383,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664534889868,\n \"link\": null,\n \"locked\": false,\n \"startBinding\": {\n \"elementId\": \"h2lcAgJBn6WsPKAj3vWS8\",\n \"focus\": 0.02318227093169459,\n \"gap\": 3\n },\n \"endBinding\": {\n \"elementId\": \"6TAhmS7GKN_ppUHjSVGLb\",\n \"focus\": -0.02318227093169459,\n \"gap\": 6\n },\n \"lastCommittedPoint\": null,\n \"startArrowhead\": null,\n \"endArrowhead\": \"triangle\",\n \"points\": [\n [\n 0,\n 0\n ],\n [\n 0,\n -85\n ]\n ]\n },\n {\n \"type\": \"text\",\n \"version\": 227,\n \"versionNonce\": 117980031,\n \"isDeleted\": false,\n \"id\": \"pUx1_v_UyKhu5zXISU4-f\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 4,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 969,\n \"y\": 386.5,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 442,\n \"height\": 90,\n \"seed\": 1608256017,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1664534908931,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 36,\n \"fontFamily\": 1,\n \"text\": \"Metadata \\n+ Installation Procedure\",\n \"baseline\": 77,\n \"textAlign\": \"center\",\n \"verticalAlign\": \"top\",\n \"containerId\": null,\n \"originalText\": \"Metadata \\n+ Installation Procedure\"\n }\n ],\n \"id\": \"_c7AOn60omrTlppZHlLQh\",\n \"created\": 1664540190548\n },\n {\n \"status\": \"unpublished\",\n \"elements\": [\n {\n \"type\": \"rectangle\",\n \"version\": 367,\n \"versionNonce\": 963584621,\n \"isDeleted\": false,\n \"id\": \"oAei2n-Ha1gpjnYdK7AwC\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 240.5,\n \"y\": 642.75,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#228be6\",\n \"width\": 392,\n \"height\": 80,\n \"seed\": 701868237,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [\n {\n \"id\": \"slfbd0bbRqA8648kZ5fns\",\n \"type\": \"text\"\n },\n {\n \"id\": \"slfbd0bbRqA8648kZ5fns\",\n \"type\": \"text\"\n },\n {\n \"type\": \"text\",\n \"id\": \"slfbd0bbRqA8648kZ5fns\"\n }\n ],\n \"updated\": 1663329462351,\n \"link\": null,\n \"locked\": false\n },\n {\n \"type\": \"text\",\n \"version\": 373,\n \"versionNonce\": 1698441027,\n \"isDeleted\": false,\n \"id\": \"slfbd0bbRqA8648kZ5fns\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 245.5,\n \"y\": 670.25,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 382,\n \"height\": 25,\n \"seed\": 1179637379,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1663329462351,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 20,\n \"fontFamily\": 1,\n \"text\": \"Execute the installation process\",\n \"baseline\": 18,\n \"textAlign\": \"center\",\n \"verticalAlign\": \"middle\",\n \"containerId\": \"oAei2n-Ha1gpjnYdK7AwC\",\n \"originalText\": \"Execute the installation process\"\n },\n {\n \"type\": \"rectangle\",\n \"version\": 208,\n \"versionNonce\": 844908259,\n \"isDeleted\": false,\n \"id\": \"cLwg2WXUit_OTQmXLIdIW\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 815.5,\n \"y\": 517.5,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 392,\n \"height\": 80,\n \"seed\": 557411811,\n \"groupIds\": [\n \"D1SCf714tngJFHk8TFX8T\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [\n {\n \"id\": \"SpG_8HxzMHjM2HYK6Fgwx\",\n \"type\": \"arrow\"\n }\n ],\n \"updated\": 1663329462351,\n \"link\": null,\n \"locked\": false\n },\n {\n \"type\": \"text\",\n \"version\": 274,\n \"versionNonce\": 1704611021,\n \"isDeleted\": false,\n \"id\": \"1r8FMl26VYSKpPKlHA_Oc\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 916.5,\n \"y\": 545,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 207,\n \"height\": 25,\n \"seed\": 961881101,\n \"groupIds\": [\n \"D1SCf714tngJFHk8TFX8T\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1663329462351,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 20,\n \"fontFamily\": 1,\n \"text\": \"CMakeBuilder.cmake()\",\n \"baseline\": 18,\n \"textAlign\": \"left\",\n \"verticalAlign\": \"top\",\n \"containerId\": null,\n \"originalText\": \"CMakeBuilder.cmake()\"\n },\n {\n \"type\": \"rectangle\",\n \"version\": 264,\n \"versionNonce\": 295137923,\n \"isDeleted\": false,\n \"id\": \"CSwjuAw6Nl67sqQ6p21ty\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 815.5,\n \"y\": 642.75,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 392,\n \"height\": 80,\n \"seed\": 1011629069,\n \"groupIds\": [\n \"D1SCf714tngJFHk8TFX8T\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [\n {\n \"id\": \"SpG_8HxzMHjM2HYK6Fgwx\",\n \"type\": \"arrow\"\n },\n {\n \"id\": \"zvmLoAH5oICRD5og-pBvu\",\n \"type\": \"arrow\"\n }\n ],\n \"updated\": 1663329462351,\n \"link\": null,\n \"locked\": false\n },\n {\n \"type\": \"text\",\n \"version\": 466,\n \"versionNonce\": 196160301,\n \"isDeleted\": false,\n \"id\": \"WX4axTU0IR7PJb0GkR-jq\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 922.5,\n \"y\": 670.25,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 193,\n \"height\": 25,\n \"seed\": 716117827,\n \"groupIds\": [\n \"D1SCf714tngJFHk8TFX8T\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1663329462351,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 20,\n \"fontFamily\": 1,\n \"text\": \"CMakeBuilder.build()\",\n \"baseline\": 18,\n \"textAlign\": \"left\",\n \"verticalAlign\": \"top\",\n \"containerId\": null,\n \"originalText\": \"CMakeBuilder.build()\"\n },\n {\n \"type\": \"rectangle\",\n \"version\": 301,\n \"versionNonce\": 1545420173,\n \"isDeleted\": false,\n \"id\": \"coUXke3Fv_DpjqG9zgEjQ\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 815.5,\n \"y\": 768,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 392,\n \"height\": 80,\n \"seed\": 1934529891,\n \"groupIds\": [\n \"D1SCf714tngJFHk8TFX8T\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [\n {\n \"type\": \"text\",\n \"id\": \"yVIbU03yFYvpXnh9xIgET\"\n },\n {\n \"id\": \"zvmLoAH5oICRD5og-pBvu\",\n \"type\": \"arrow\"\n }\n ],\n \"updated\": 1663329462351,\n \"link\": null,\n \"locked\": false\n },\n {\n \"type\": \"text\",\n \"version\": 273,\n \"versionNonce\": 1837690307,\n \"isDeleted\": false,\n \"id\": \"yVIbU03yFYvpXnh9xIgET\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 820.5,\n \"y\": 795.5,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 382,\n \"height\": 25,\n \"seed\": 1611291683,\n \"groupIds\": [\n \"D1SCf714tngJFHk8TFX8T\"\n ],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1663329462351,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 20,\n \"fontFamily\": 1,\n \"text\": \"CMakeBuilder.install()\",\n \"baseline\": 18,\n \"textAlign\": \"center\",\n \"verticalAlign\": \"middle\",\n \"containerId\": \"coUXke3Fv_DpjqG9zgEjQ\",\n \"originalText\": \"CMakeBuilder.install()\"\n },\n {\n \"type\": \"arrow\",\n \"version\": 564,\n \"versionNonce\": 1041761261,\n \"isDeleted\": false,\n \"id\": \"SpG_8HxzMHjM2HYK6Fgwx\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 1209,\n \"y\": 558.5,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 96,\n \"height\": 109,\n \"seed\": 732445197,\n \"groupIds\": [\n \"D1SCf714tngJFHk8TFX8T\"\n ],\n \"strokeSharpness\": \"round\",\n \"boundElements\": [],\n \"updated\": 1663329462351,\n \"link\": null,\n \"locked\": false,\n \"startBinding\": {\n \"elementId\": \"cLwg2WXUit_OTQmXLIdIW\",\n \"focus\": -0.7327371048252911,\n \"gap\": 1.5\n },\n \"endBinding\": {\n \"elementId\": \"CSwjuAw6Nl67sqQ6p21ty\",\n \"focus\": 0.6494341563786008,\n \"gap\": 2.5\n },\n \"lastCommittedPoint\": null,\n \"startArrowhead\": null,\n \"endArrowhead\": \"triangle\",\n \"points\": [\n [\n 0,\n 0\n ],\n [\n 96,\n 54\n ],\n [\n 1,\n 109\n ]\n ]\n },\n {\n \"type\": \"arrow\",\n \"version\": 642,\n \"versionNonce\": 1380728163,\n \"isDeleted\": false,\n \"id\": \"zvmLoAH5oICRD5og-pBvu\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 1216,\n \"y\": 680,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 98,\n \"height\": 124.33745608356844,\n \"seed\": 708861581,\n \"groupIds\": [\n \"D1SCf714tngJFHk8TFX8T\"\n ],\n \"strokeSharpness\": \"round\",\n \"boundElements\": [],\n \"updated\": 1663329462351,\n \"link\": null,\n \"locked\": false,\n \"startBinding\": {\n \"elementId\": \"CSwjuAw6Nl67sqQ6p21ty\",\n \"focus\": -0.7839018302828619,\n \"gap\": 8.5\n },\n \"endBinding\": {\n \"elementId\": \"coUXke3Fv_DpjqG9zgEjQ\",\n \"focus\": 0.7841576120638036,\n \"gap\": 6.5\n },\n \"lastCommittedPoint\": null,\n \"startArrowhead\": null,\n \"endArrowhead\": \"triangle\",\n \"points\": [\n [\n 0,\n 0\n ],\n [\n 96,\n 54\n ],\n [\n -2,\n 124.33745608356844\n ]\n ]\n },\n {\n \"type\": \"text\",\n \"version\": 613,\n \"versionNonce\": 909390253,\n \"isDeleted\": false,\n \"id\": \"fAHH1YdSlMq8ioLIj36Of\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 644,\n \"y\": 353.7484662576685,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 166,\n \"height\": 567.2515337423315,\n \"seed\": 1455993539,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1663329499644,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 493.16790307261914,\n \"fontFamily\": 2,\n \"text\": \"{\",\n \"baseline\": 454.2515337423315,\n \"textAlign\": \"center\",\n \"verticalAlign\": \"top\",\n \"containerId\": null,\n \"originalText\": \"{\"\n }\n ],\n \"id\": \"KBV_I9pxrJD2zPuaP6vBc\",\n \"created\": 1663329511286\n },\n {\n \"status\": \"unpublished\",\n \"elements\": [\n {\n \"type\": \"rectangle\",\n \"version\": 93,\n \"versionNonce\": 42296109,\n \"isDeleted\": false,\n \"id\": \"cLwg2WXUit_OTQmXLIdIW\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 625.5,\n \"y\": 298,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 392,\n \"height\": 80,\n \"seed\": 557411811,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [\n {\n \"id\": \"SpG_8HxzMHjM2HYK6Fgwx\",\n \"type\": \"arrow\"\n }\n ],\n \"updated\": 1663324636434,\n \"link\": null,\n \"locked\": false\n },\n {\n \"type\": \"text\",\n \"version\": 99,\n \"versionNonce\": 1537897869,\n \"isDeleted\": false,\n \"id\": \"1r8FMl26VYSKpPKlHA_Oc\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 726.5,\n \"y\": 325.5,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 179,\n \"height\": 25,\n \"seed\": 961881101,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1663324636434,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 20,\n \"fontFamily\": 1,\n \"text\": \"Fetch source files\",\n \"baseline\": 18,\n \"textAlign\": \"left\",\n \"verticalAlign\": \"top\",\n \"containerId\": null,\n \"originalText\": \"Fetch source files\"\n },\n {\n \"type\": \"rectangle\",\n \"version\": 149,\n \"versionNonce\": 1653290435,\n \"isDeleted\": false,\n \"id\": \"CSwjuAw6Nl67sqQ6p21ty\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 625.5,\n \"y\": 423.25,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 392,\n \"height\": 80,\n \"seed\": 1011629069,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [\n {\n \"id\": \"SpG_8HxzMHjM2HYK6Fgwx\",\n \"type\": \"arrow\"\n },\n {\n \"id\": \"zvmLoAH5oICRD5og-pBvu\",\n \"type\": \"arrow\"\n }\n ],\n \"updated\": 1663324636434,\n \"link\": null,\n \"locked\": false\n },\n {\n \"type\": \"text\",\n \"version\": 272,\n \"versionNonce\": 1195260909,\n \"isDeleted\": false,\n \"id\": \"WX4axTU0IR7PJb0GkR-jq\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 645.5,\n \"y\": 450.75,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 352,\n \"height\": 25,\n \"seed\": 716117827,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1663324636434,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 20,\n \"fontFamily\": 1,\n \"text\": \"Expand them in the stage directory\",\n \"baseline\": 18,\n \"textAlign\": \"left\",\n \"verticalAlign\": \"top\",\n \"containerId\": null,\n \"originalText\": \"Expand them in the stage directory\"\n },\n {\n \"type\": \"rectangle\",\n \"version\": 185,\n \"versionNonce\": 2143651171,\n \"isDeleted\": false,\n \"id\": \"coUXke3Fv_DpjqG9zgEjQ\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 625.5,\n \"y\": 548.5,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 392,\n \"height\": 80,\n \"seed\": 1934529891,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [\n {\n \"type\": \"text\",\n \"id\": \"yVIbU03yFYvpXnh9xIgET\"\n },\n {\n \"id\": \"zvmLoAH5oICRD5og-pBvu\",\n \"type\": \"arrow\"\n },\n {\n \"id\": \"5yqrFWV-hhJ4RoVewqAC0\",\n \"type\": \"arrow\"\n }\n ],\n \"updated\": 1663324636434,\n \"link\": null,\n \"locked\": false\n },\n {\n \"type\": \"text\",\n \"version\": 135,\n \"versionNonce\": 1833580109,\n \"isDeleted\": false,\n \"id\": \"yVIbU03yFYvpXnh9xIgET\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 630.5,\n \"y\": 563.5,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 382,\n \"height\": 50,\n \"seed\": 1611291683,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1663324636434,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 20,\n \"fontFamily\": 1,\n \"text\": \"Set the stage directory as the \\ncurrent working directory\",\n \"baseline\": 43,\n \"textAlign\": \"center\",\n \"verticalAlign\": \"middle\",\n \"containerId\": \"coUXke3Fv_DpjqG9zgEjQ\",\n \"originalText\": \"Set the stage directory as the current working directory\"\n },\n {\n \"type\": \"rectangle\",\n \"version\": 253,\n \"versionNonce\": 1704770627,\n \"isDeleted\": false,\n \"id\": \"tBTBRiEA6AJABK4wnKF_-\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 625.5,\n \"y\": 673.75,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 392,\n \"height\": 80,\n \"seed\": 1257829773,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [\n {\n \"id\": \"GoE9udjDQxUqdsYCUnbVI\",\n \"type\": \"text\"\n },\n {\n \"type\": \"text\",\n \"id\": \"GoE9udjDQxUqdsYCUnbVI\"\n },\n {\n \"id\": \"5yqrFWV-hhJ4RoVewqAC0\",\n \"type\": \"arrow\"\n },\n {\n \"id\": \"v-9Voh5erXQ8iqoQ_9BVO\",\n \"type\": \"arrow\"\n }\n ],\n \"updated\": 1663324636434,\n \"link\": null,\n \"locked\": false\n },\n {\n \"type\": \"text\",\n \"version\": 194,\n \"versionNonce\": 1557028205,\n \"isDeleted\": false,\n \"id\": \"GoE9udjDQxUqdsYCUnbVI\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 630.5,\n \"y\": 701.25,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 382,\n \"height\": 25,\n \"seed\": 895792579,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1663324636434,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 20,\n \"fontFamily\": 1,\n \"text\": \"Fork a new build environment\",\n \"baseline\": 18,\n \"textAlign\": \"center\",\n \"verticalAlign\": \"middle\",\n \"containerId\": \"tBTBRiEA6AJABK4wnKF_-\",\n \"originalText\": \"Fork a new build environment\"\n },\n {\n \"type\": \"rectangle\",\n \"version\": 321,\n \"versionNonce\": 1675770851,\n \"isDeleted\": false,\n \"id\": \"oAei2n-Ha1gpjnYdK7AwC\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 625.5,\n \"y\": 799,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"#228be6\",\n \"width\": 392,\n \"height\": 80,\n \"seed\": 701868237,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [\n {\n \"id\": \"slfbd0bbRqA8648kZ5fns\",\n \"type\": \"text\"\n },\n {\n \"id\": \"slfbd0bbRqA8648kZ5fns\",\n \"type\": \"text\"\n },\n {\n \"type\": \"text\",\n \"id\": \"slfbd0bbRqA8648kZ5fns\"\n },\n {\n \"id\": \"v-9Voh5erXQ8iqoQ_9BVO\",\n \"type\": \"arrow\"\n }\n ],\n \"updated\": 1663324636434,\n \"link\": null,\n \"locked\": false\n },\n {\n \"type\": \"text\",\n \"version\": 328,\n \"versionNonce\": 1868179405,\n \"isDeleted\": false,\n \"id\": \"slfbd0bbRqA8648kZ5fns\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 630.5,\n \"y\": 826.5,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 382,\n \"height\": 25,\n \"seed\": 1179637379,\n \"groupIds\": [],\n \"strokeSharpness\": \"sharp\",\n \"boundElements\": [],\n \"updated\": 1663324636434,\n \"link\": null,\n \"locked\": false,\n \"fontSize\": 20,\n \"fontFamily\": 1,\n \"text\": \"Execute the installation process\",\n \"baseline\": 18,\n \"textAlign\": \"center\",\n \"verticalAlign\": \"middle\",\n \"containerId\": \"oAei2n-Ha1gpjnYdK7AwC\",\n \"originalText\": \"Execute the installation process\"\n },\n {\n \"type\": \"arrow\",\n \"version\": 221,\n \"versionNonce\": 1777917731,\n \"isDeleted\": false,\n \"id\": \"SpG_8HxzMHjM2HYK6Fgwx\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 1019,\n \"y\": 339,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 96,\n \"height\": 109,\n \"seed\": 732445197,\n \"groupIds\": [],\n \"strokeSharpness\": \"round\",\n \"boundElements\": [],\n \"updated\": 1663324636434,\n \"link\": null,\n \"locked\": false,\n \"startBinding\": {\n \"elementId\": \"cLwg2WXUit_OTQmXLIdIW\",\n \"focus\": -0.7533277870216306,\n \"gap\": 7\n },\n \"endBinding\": {\n \"elementId\": \"CSwjuAw6Nl67sqQ6p21ty\",\n \"focus\": 0.7554869684499315,\n \"gap\": 6\n },\n \"lastCommittedPoint\": null,\n \"startArrowhead\": null,\n \"endArrowhead\": \"triangle\",\n \"points\": [\n [\n 0,\n 0\n ],\n [\n 96,\n 54\n ],\n [\n 1,\n 109\n ]\n ]\n },\n {\n \"type\": \"arrow\",\n \"version\": 299,\n \"versionNonce\": 309789379,\n \"isDeleted\": false,\n \"id\": \"zvmLoAH5oICRD5og-pBvu\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 1026,\n \"y\": 460.5,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 98,\n \"height\": 124.33745608356844,\n \"seed\": 708861581,\n \"groupIds\": [],\n \"strokeSharpness\": \"round\",\n \"boundElements\": [],\n \"updated\": 1663324636435,\n \"link\": null,\n \"locked\": false,\n \"startBinding\": {\n \"elementId\": \"CSwjuAw6Nl67sqQ6p21ty\",\n \"focus\": -0.7021630615640598,\n \"gap\": 12\n },\n \"endBinding\": {\n \"elementId\": \"coUXke3Fv_DpjqG9zgEjQ\",\n \"focus\": 0.8530521262002744,\n \"gap\": 12\n },\n \"lastCommittedPoint\": null,\n \"startArrowhead\": null,\n \"endArrowhead\": \"triangle\",\n \"points\": [\n [\n 0,\n 0\n ],\n [\n 96,\n 54\n ],\n [\n -2,\n 124.33745608356844\n ]\n ]\n },\n {\n \"type\": \"arrow\",\n \"version\": 301,\n \"versionNonce\": 914472685,\n \"isDeleted\": false,\n \"id\": \"5yqrFWV-hhJ4RoVewqAC0\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 1019,\n \"y\": 586.6789496258876,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 99,\n \"height\": 123.78306157234579,\n \"seed\": 642378381,\n \"groupIds\": [],\n \"strokeSharpness\": \"round\",\n \"boundElements\": [],\n \"updated\": 1663324636435,\n \"link\": null,\n \"locked\": false,\n \"startBinding\": {\n \"elementId\": \"coUXke3Fv_DpjqG9zgEjQ\",\n \"focus\": -0.6501663893510815,\n \"gap\": 7\n },\n \"endBinding\": {\n \"elementId\": \"tBTBRiEA6AJABK4wnKF_-\",\n \"focus\": 0.8705418381344308,\n \"gap\": 8\n },\n \"lastCommittedPoint\": null,\n \"startArrowhead\": null,\n \"endArrowhead\": \"triangle\",\n \"points\": [\n [\n 0,\n 0\n ],\n [\n 99,\n 42.82105037411236\n ],\n [\n 10.000000000000227,\n 123.78306157234579\n ]\n ]\n },\n {\n \"type\": \"arrow\",\n \"version\": 351,\n \"versionNonce\": 984592995,\n \"isDeleted\": false,\n \"id\": \"v-9Voh5erXQ8iqoQ_9BVO\",\n \"fillStyle\": \"hachure\",\n \"strokeWidth\": 1,\n \"strokeStyle\": \"solid\",\n \"roughness\": 2,\n \"opacity\": 100,\n \"angle\": 0,\n \"x\": 1031,\n \"y\": 714.8662394200408,\n \"strokeColor\": \"#000000\",\n \"backgroundColor\": \"transparent\",\n \"width\": 90,\n \"height\": 137.7637151210173,\n \"seed\": 698547757,\n \"groupIds\": [],\n \"strokeSharpness\": \"round\",\n \"boundElements\": [],\n \"updated\": 1663324636435,\n \"link\": null,\n \"locked\": false,\n \"startBinding\": {\n \"elementId\": \"tBTBRiEA6AJABK4wnKF_-\",\n \"focus\": -0.6014975041597337,\n \"gap\": 10\n },\n \"endBinding\": {\n \"elementId\": \"oAei2n-Ha1gpjnYdK7AwC\",\n \"focus\": 0.9573045267489712,\n \"gap\": 12\n },\n \"lastCommittedPoint\": null,\n \"startArrowhead\": null,\n \"endArrowhead\": \"triangle\",\n \"points\": [\n [\n 0,\n 0\n ],\n [\n 90,\n 33.633760579959244\n ],\n [\n 4,\n 137.7637151210173\n ]\n ]\n }\n ],\n \"id\": \"RzNgncGu1938Ma5Teh6qZ\",\n \"created\": 1663324659550\n }\n ]\n}" }, { "path": "lib/spack/docs/include_yaml.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Learn how to use include directives to modularize your Spack YAML configuration files for better organization and reusability.\n\n.. _include-yaml:\n\nInclude Settings (include.yaml)\n===============================\n\nSpack allows you to include configuration files through ``include.yaml``, or in the ``include:`` section in an environment.\nYou can specify includes using local paths, remote paths, and ``git`` URLs.\nIncluded paths become configuration scopes in Spack and can even be used to override built-in scopes.\n\nLocal files\n~~~~~~~~~~~\n\nYou can include a single configuration file or an entire configuration *scope* like this:\n\n.. code-block:: yaml\n\n include:\n - /path/to/a/required/config.yaml\n - $MY_SPECIAL_CONFIG_FILE\n - path: $HOME/path/to/my/project/packages.yaml\n - path: /path/to/$os/$target/config\n optional: true\n - path: /path/to/os-specific/config-dir\n when: os == \"ventura\"\n\nIncluded paths may be absolute, relative (to the configuration file), specified as URLs, or provided in environment variables (e.g., ``$MY_SPECIAL_CONFIG_FILE``).\n\n* ``optional``: Spack will raise an error when an included configuration file does not exist, *unless* it is explicitly made ``optional: true``, like the second path above.\n* ``when``: Configuration scopes can also be included *conditionally* with ``when``.\n ``when:`` conditions are evaluated as described for :ref:`Spec List References `.\n\n\nThe same conditions and variables in :ref:`Spec List References ` can be used for conditional activation in the ``when`` clauses.\n\nRemote file URLs\n~~~~~~~~~~~~~~~~\n\nOnly the ``ftp``, ``http``, and ``https`` protocols (or schemes) are supported for remote file URLs.\nSpack-specific, environment, and user path variables can be used.\n(See :ref:`config-file-variables` for more information.)\n\nA ``sha256`` is required.\nFor example, suppose you have a ``/etc/spack/include.yaml`` file that specifies a remote ``config.yaml`` file as follows::\n\n include:\n - path: https://github.com/path/to/raw/config/config.yaml\n sha256: 26e871804a92cd07bb3d611b31b4156ae93d35b6a6d6e0ef3a67871fcb1d258b\n\nThe ``config.yaml`` file is downloaded to a subdirectory of ``/etc/spack``.\nThe contents of the downloaded file are read and included in Spack's configuration when Spack configuration files are processed.\n\n.. note::\n\n You can check the destination of the downloaded file by running: ``spack config scopes -p``.\n\n.. warning::\n\n Remote file URLs must link to the **raw** form of the file's contents (e.g., `GitHub `_ or `GitLab `_).\n\n If the directory containing the ``include.yaml`` file is not writable when the remote file is downloaded, then the destination will be a temporary directory.\n\n\n``git`` repository files\n~~~~~~~~~~~~~~~~~~~~~~~~\n\nYou can also include configuration files from a ``git`` repository.\nThe ``branch``, ``commit``, or ``tag`` to be checked out is required.\nA list of relative paths in which to find the configuration files is also required.\nInclusion of the repository (and its paths) can be optional or conditional.\nIf you want to control the :ref:`name of the configuration scope `, you can provide a ``name``.\n\nFor example, suppose we only want to include the ``config.yaml`` and ``packages.yaml`` files from the `spack/spack-configs `_ repository's ``USC/config`` directory when using the ``centos7`` operating system.\nAnd we want the configuration scope name to start ``common``.\nWe could then configure the include in, for example, the user scope include file (i.e., ``$HOME/.spack/include.yaml`` by default), as follows::\n\n include:\n - name: common\n git: https://github.com/spack/spack-configs.git\n branch: main\n when: os == \"centos7\"\n paths:\n - USC/config/config.yaml\n - USC/config/packages.yaml\n\n.. note::\n\n The git URL could be specified through an environment variable (e.g., ``$MY_USC_CONFIG_URL``).\n\nIf the condition is satisfied, then the ``main`` branch of the repository will be cloned -- under ``$HOME/.spack/includes`` -- when configuration scopes are initially created.\nOnce cloned, the settings for the two files under the ``USC/config`` directory will be integrated into Spack's configuration.\nIn this example, the new scopes and their paths can be seen by running::\n\n $ spack config scopes -p\n Scope Path\n command_line\n spack /Users/username/spack/etc/spack/\n user /Users/username/.spack/\n common:USC/config/config.yaml /Users/username/.spack/includes/common/USC/config/config.yaml\n common:USC/config/packages.yaml /Users/username/.spack/includes/common/USC/config/packages.yaml\n site /Users/username/spack/etc/spack/site/\n system /etc/spack/\n defaults /Users/username/spack/etc/spack/defaults/\n defaults:darwin /Users/username/spack/etc/spack/defaults/darwin/\n defaults:base /Users/username/spack/etc/spack/defaults/base/\n _builtin\n\nSince there are two unique paths, each results in a separate configuration scope.\nIf only the ``USC/config`` directory was listed under ``paths``, then there would be only one configuration scope, named ``USC``, and the configuration settings from all of the configuration files within that directory would be integrated.\n\n.. versionadded:: 1.1\n ``git:``, ``branch:``, ``commit:``, and ``tag:`` attributes.\n\n.. versionadded:: 1.2\n ``name:`` attribute and git environment variable support.\n\nPrecedence\n~~~~~~~~~~\n\nUsing ``include:`` adds the included files as a configuration scope *below* the including file.\nThis is so that you can override settings from files you include.\nIf you want one file to take precedence over another, you can put the include with higher precedence earlier in the list:\n\n.. code-block:: yaml\n\n include:\n - /path/to/higher/precedence/scope/\n - /path/to/middle/precedence/scope/\n - git: https://github.com/org/git-repo-scope\n commit: 95c59784bd02ea248bf905d79d063df38e087b19\n\n``prefer_modify``\n^^^^^^^^^^^^^^^^^\n\nWhen you use commands like ``spack compiler find``, ``spack external find``, ``spack config edit`` or ``spack config add``, they modify the topmost writable scope in the current configuration.\nScopes can tell Spack to prefer to edit their included scopes instead, using ``prefer_modify``:\n\n.. code-block:: yaml\n\n include:\n - name: \"preferred\"\n path: /path/to/scope/we/want-to-write\n prefer_modify: true\n\nNow, if the including scope is the highest precedence scope and would otherwise be selected automatically by one of these commands, they will instead prefer to edit ``preferred``.\nThe including scope can still be modified by using the ``--scope`` argument (e.g., ``spack compiler find --scope NAME``).\n\n.. warning::\n\n Recursive includes are not currently processed in a breadth-first manner, so the value of a configuration option that is altered by multiple included files may not be what you expect.\n This will be addressed in a future update.\n\n.. versionadded:: 1.1 The ``prefer_modify:`` attribute.\n\nOverriding local paths\n~~~~~~~~~~~~~~~~~~~~~~\n\nOptionally, you can enable a local path to be overridden by an environment variable using ``path_override_env_var:``:\n\n.. code-block:: yaml\n\n include:\n - path_override_env_var: SPECIAL_CONFIG_PATH\n path: /path/to/special/config.yaml\n\nHere, If ``SPECIAL_CONFIG_PATH`` is set, its value will be used as the path.\nIf not, Spack will instead use the ``path:`` specified in configuration.\n\n.. note::\n\n ``path_override_env_var:`` is currently only supported for ``path:`` includes, not ``git:`` includes.\n\n.. versionadded:: 1.1\n The ``path_override_env_var:`` attribute.\n\n.. _named-config-scopes:\n\nNamed configuration scopes\n~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nBy default, the included scope names are constructed by appending ``:`` and the included scope's basename to the parent scope name.\nFor example, Spack's own ``defaults`` scope includes a ``base`` scope and a platform-specific scope::\n\n $ spack config scopes -p\n Scope Path\n command_line\n spack /home/username/spack/etc/spack/\n user /home/username/.spack/\n site /home/username/spack/etc/spack/site/\n defaults /home/username/spack/etc/spack/defaults/\n defaults:darwin /home/username/spack/etc/spack/defaults/darwin/\n defaults:base /home/username/spack/etc/spack/defaults/base/\n _builtin\n\nYou can see ``defaults`` and the included ``defaults:base`` and ``defaults:darwin`` scopes here.\n\nIf you want to define your own name for an included scope, you can supply an optional ``name:`` argument when you include it:\n\n.. code-block:: yaml\n\n spack:\n include:\n - path: foo\n name: myscope\n\nYou can see the ``myscope`` name when we activate this environment::\n\n > spack -e ./env config scopes -p\n Scope Path\n command_line\n env:/home/username/env /home/username/env/spack.yaml/\n myscope /home/username/env/foo/\n spack /home/username/spack/etc/spack/\n user /home/username/.spack/\n site /home/username/spack/etc/spack/site/\n defaults /home/username/spack/etc/spack/defaults/\n defaults:darwin /home/username/spack/etc/spack/defaults/darwin/\n defaults:base /home/username/spack/etc/spack/defaults/base/\n _builtin\n\nYou can now use the argument ``myscope`` to refer to this, for example with ``spack config --scope myscope add ...``.\n\nBuilt-in configuration scopes\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nThe default ``user``, ``system``, and ``site`` scopes are defined using ``include:`` in ``$spack/etc/spack/include.yaml``:\n\n.. literalinclude:: _spack_root/etc/spack/include.yaml\n :language: yaml\n\nYou can see that all three of these scopes are given meaningful names, and all three are ``optional``, i.e., they'll be ignored if their directories do not exist.\nThe ``user`` and ``system`` scopes can also be disabled by setting ``SPACK_DISABLE_LOCAL_CONFIG``.\nFinally, the ``user`` scope can be overridden with a path in ``SPACK_USER_CONFIG_PATH`` if it is set.\n\nOverriding scopes by name\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nConfiguration scopes have unique names.\nThis means that you can use the ``name:`` attribute to *replace* a builtin scope.\nIf you supply an environment like this:\n\n.. code-block:: yaml\n\n spack:\n include:\n - path: foo\n name: user\n\nThe newly included ``user`` scope will *completely* override the builtin ``user`` scope::\n\n > spack -e ~/env config scopes -p\n Scope Path\n command_line\n env:/home/username/env /home/username/env/spack.yaml/\n user /home/username/env/foo/\n spack /home/username/spack/etc/spack/\n site /home/username/spack/etc/spack/site/\n defaults /home/username/spack/etc/spack/defaults/\n defaults:darwin /home/username/spack/etc/spack/defaults/darwin/\n defaults:base /home/username/spack/etc/spack/defaults/base/\n _builtin\n\n.. warning::\n\n Overriding the ``defaults`` scope can have **very** unexpected consequences and is not advised.\n\n.. versionadded:: 1.1\n The ``name:`` attribute.\n\nOverriding built-in scopes with ``include::``\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nIn some cases, you may want to override *all* of the built-in configuration scopes.\nThe ``user`` and ``system`` scopes depend on the user and the machine on which Spack is running, and they can end up bringing in unexpected configuration settings in surprising ways.\n\nIf you want to eliminate them completely from an environment, you can write:\n\n.. code-block:: yaml\n\n spack:\n include:: []\n\nThis overrides all scopes except the ``defaults`` that Spack needs in order to function.\nYou can see that ``spack``, ``user``, and ``site`` are overridden::\n\n > spack -e ~/env config scopes -vp\n Scope Type Status Path\n command_line internal active\n env:/home/username/env env,path active /home/username/env/spack.yaml/\n spack path override /home/username/spack/etc/spack/\n user include,path override /home/username/.spack/\n site include,path override /home/username/spack/etc/spack/site/\n defaults path active /home/username/spack/etc/spack/defaults/\n defaults:darwin include,path active /home/username/spack/etc/spack/defaults/darwin/\n defaults:base include,path active /home/username/spack/etc/spack/defaults/base/\n _builtin internal active\n\nAnd if you run ``spack config blame``, the settings from these scopes will no longer show up.\n``defaults`` are not overridden as they are needed by Spack to function.\nThis allows you to create completely isolated environments that do not bring in external settings.\n\n.. versionadded:: 1.1\n ``include::`` with two colons for overriding.\n" }, { "path": "lib/spack/docs/index.rst", "content": ".. \n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n Spack documentation master file, created by\n sphinx-quickstart on Mon Dec 9 15:32:41 2013.\n You can adapt this file completely to your liking, but it should at least\n contain the root `toctree` directive.\n\n.. meta::\n :description lang=en:\n Documentation of Spack, a flexible package manager for high-performance computing, designed to support multiple versions and configurations of software on a wide variety of platforms.\n\nSpack\n===================\n\nSpack is a package management tool designed to support multiple versions and configurations of software on a wide variety of platforms and environments.\nIt was designed for large supercomputing centers, where many users and application teams share common installations of software on clusters with exotic architectures, using libraries that do not have a standard ABI.\nSpack is non-destructive: installing a new version does not break existing installations, so many configurations can coexist on the same system.\n\nMost importantly, Spack is *simple*.\nIt offers a simple *spec* syntax so that users can specify versions and configuration options concisely.\nSpack is also simple for package authors: package files are written in pure Python, and specs allow package authors to maintain a single file for many different builds of the same package.\n\nSee the :doc:`features` for examples and highlights.\n\nGet Spack from the `GitHub repository `_ and install your first package:\n\n.. code-block:: console\n\n $ git clone --depth=2 https://github.com/spack/spack.git\n $ cd spack/bin\n $ ./spack install libelf\n\n.. note::\n ``--depth=2`` prunes the git history to reduce the size of the Spack installation.\n\nIf you're new to Spack and want to start using it, see :doc:`getting_started`, or refer to the full manual below.\n\n\n.. toctree::\n :maxdepth: 2\n :caption: Introduction\n\n features\n getting_started\n spec_syntax\n installing_prerequisites\n windows\n\n.. toctree::\n :maxdepth: 2\n :caption: Basic Usage\n\n package_fundamentals\n installing\n configuring_compilers\n environments_basics\n frequently_asked_questions\n getting_help\n\n.. toctree::\n :maxdepth: 2\n :caption: Links\n\n Tutorial (spack-tutorial.rtfd.io) \n Packages (packages.spack.io) \n Binaries (binaries.spack.io) \n\n.. toctree::\n :maxdepth: 2\n :caption: Configuration\n\n configuration\n config_yaml\n packages_yaml\n toolchains_yaml\n build_settings\n repositories\n mirrors\n chain\n module_file_support\n include_yaml\n env_vars_yaml\n\n.. toctree::\n :maxdepth: 2\n :caption: Reference\n\n environments\n containers\n binary_caches\n bootstrapping\n command_index\n extensions\n pipelines\n signing\n gpu_configuration\n\n.. toctree::\n :maxdepth: 2\n :caption: Contributing\n\n packaging_guide_creation\n packaging_guide_build\n packaging_guide_testing\n packaging_guide_advanced\n build_systems\n roles_and_responsibilities\n contribution_guide\n developer_guide\n package_review_guide\n\n.. toctree::\n :maxdepth: 2\n :caption: Advanced Topics\n\n advanced_topics\n\n.. toctree::\n :maxdepth: 2\n :caption: API Docs\n\n Spack Package API \n Spack Builtin Repo \n Spack API Docs \n\nIndices and tables\n------------------\n\n* :ref:`genindex`\n* :ref:`modindex`\n* :ref:`search`\n" }, { "path": "lib/spack/docs/installing.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Learn how Spack installs packages: the interactive terminal UI, parallelism\n via a POSIX jobserver, multi-process installs, background execution, and\n handling build failures.\n\n.. _installing:\n\nInstalling Packages\n===================\n\nThis page covers the ``spack install`` experience in detail, including the interactive terminal UI (TUI), parallelism, background execution, and handling build failures.\n\nBefore diving in, ensure you are familiar with :doc:`package_fundamentals` for basic usage and spec syntax.\n\n.. versionadded:: 1.2\n The TUI and POSIX jobserver are new in Spack 1.2 and require a Unix-like platform.\n\n\nInteractive terminal UI\n-----------------------\n\nBy default, ``spack install`` shows live progress inline in the terminal.\nCompleted packages scroll into terminal history, while active builds update dynamically below the progress header.\n\nEvery package in the install plan is shown with its current status:\n\n.. code-block:: text\n\n $ spack install -j16 python\n [+] abc1234 zlib@1.3.1 /home/user/spack/opt/spack/... (4s)\n [+] def5678 pkgconf@2.2.0 /home/user/spack/opt/spack/... (6s)\n [+] 9ab0123 ncurses@6.5 /home/user/spack/opt/spack/... (23s)\n Progress: 3/7 +/-: 4 jobs /: filter v: logs n/p: next/prev\n [/] cde4567 readline@8.2 configure (11s)\n [/] fgh8901 openssl@3.4.1 build (18s)\n\nStatus indicators:\n\n* ``[+]`` finished successfully\n* ``[x]`` failed\n* ``[/]``, ``[-]``, ``[\\]``, ``[|]`` building (rotating spinner)\n* ``[e]`` external\n\n**Log-following mode**: press ``v`` to switch from the overview to a live view of build output.\nPress ``v``, ``q``, or ``Esc`` to return to the overview.\n\nWhile in log-following mode, press ``n`` / ``p`` to cycle to the next or previous build.\nPress ``/``, type a pattern, and press ``Enter`` to jump to a matching build (``Esc`` cancels the filter).\n\nWhen a build fails, press ``v`` to see a parsed error summary and the path to the full log.\n\n\nParallelism\n-----------\n\nSpack controls parallelism at two levels: the number of build jobs shared across all packages (``-j``), and the number of packages building concurrently (``-p``).\n\nBuild-level parallelism (``-j``)\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nThe ``-j`` flag controls the **total** number of concurrent build jobs via a POSIX jobserver.\nAll build processes (``make``, ``cmake``, ``ninja``, etc.) share the same jobserver, so ``-j16`` means at most 16 build jobs across *all* packages combined.\nThis is the primary concurrency knob.\n\n.. code-block:: console\n\n $ spack install -j16 python\n\nSpack creates a POSIX jobserver compatible with GNU Make's jobserver protocol.\nChild build systems automatically respect it through ``MAKEFLAGS``, so total CPU usage stays bounded regardless of how many packages are building concurrently.\n\n.. note::\n\n If an external jobserver is already present in ``MAKEFLAGS``, for example when Spack itself is invoked from inside a larger ``make`` build, Spack attaches to the existing jobserver instead of creating its own.\n\nPackage-level parallelism (``-p``)\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nThe ``-p`` / ``--concurrent-packages`` flag limits how many packages can be in the build queue simultaneously.\nBy default there is no limit, and packages are started as jobserver tokens become available.\n\n.. code-block:: console\n\n $ spack install -j16 -p4 python\n\nThis builds with 16 total make-jobs but never more than 4 packages at once.\n\nDynamic adjustment\n^^^^^^^^^^^^^^^^^^\n\nYou can adjust parallelism while a build is running:\n\n* Press ``+`` to add a job (increases ``-j`` by 1)\n* Press ``-`` to remove a job (decreases ``-j`` by 1)\n\nWhen reducing parallelism, Spack waits for currently running jobs to finish before the new limit takes effect; it does not kill active processes.\nThe progress header shows the adjustment in progress, e.g. ``+/-: 4=>2 jobs``, until the actual count reaches the target.\n\n\nMulti-process and multi-node installs\n--------------------------------------\n\nMultiple ``spack install`` processes can safely run concurrently, whether on the same machine or across multiple nodes in a cluster with a shared filesystem.\nSpack coordinates through :ref:`per-prefix filesystem locks `: before building a package, the process acquires an exclusive lock on its install prefix.\nIf another process already holds the lock, Spack waits rather than building a second copy.\nWhen a process encounters a prefix that was already installed, it simply skips it and moves on to the next install.\n\nFor best results on a cluster, it's recommended to limit per-process package-level parallelism (e.g., ``spack install -p2``) for better load balancing.\n\n\nNon-interactive mode\n--------------------\n\nWhen the controlling process is not a tty, such as in CI pipelines, when redirecting output to a file, or when running in the background, Spack skips the TUI and prints simple line-based status updates instead.\nUse ``spack install -v`` to also print build output.\n\nYou can also background builds:\n\n* **Suspend and resume**: press ``Ctrl-Z`` to suspend the install, then ``bg`` to let it continue in the background or ``fg`` to bring it back.\n Child builds are paused while suspended, and resumed when continued in the background or foreground.\n The TUI is suppressed while backgrounded and restored on ``fg``.\n* **Start in the background**: run ``spack install ... &`` to skip the TUI entirely and build in the background from the start.\n\n.. tip::\n\n You don't need a new terminal or SSH session to keep a build running — just suspend it with ``Ctrl-Z`` and ``bg``, then continue working.\n\n\nHandling failures\n-----------------\n\nBy default, Spack continues building other packages when one fails (best-effort).\nUse ``--fail-fast`` to stop immediately on the first failure.\n\n.. code-block:: console\n\n $ spack install --fail-fast python\n\nFailed builds show ``[x]`` in the overview.\nNavigate to a failed build and press ``v`` to see a parsed error summary and the path to the full log.\n\nSee :ref:`spack install ` for the full set of flags related to debugging and controlling build behavior.\n" }, { "path": "lib/spack/docs/installing_prerequisites.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Find instructions on how to install the necessary prerequisites for Spack on various operating systems, including Linux and macOS.\n\n.. _verify-spack-prerequisites:\n\nSpack Prerequisites\n===================\n\nSpack relies on a few basic utilities to be present on the system where it runs, depending on the operating system.\nTo install them, follow the instructions below.\n\nLinux\n-----\n\nFor **Debian** and **Ubuntu** users:\n\n.. code-block:: console\n\n $ apt update\n $ apt install file bzip2 ca-certificates g++ gcc gfortran git gzip lsb-release patch python3 tar unzip xz-utils zstd\n\nFor **RHEL**, **AlmaLinux**, and **Rocky Linux** users:\n\n.. code-block:: console\n\n $ dnf install epel-release\n $ dnf install file bzip2 ca-certificates git gzip patch python3 tar unzip xz zstd gcc gcc-c++ gcc-gfortran\n\nmacOS\n-----\n\nOn macOS, the Command Line Tools package is required, and the full Xcode suite may be necessary for some packages, such as Qt and apple-gl.\nTo install Xcode, you can use the following command:\n\n.. code-block:: console\n\n $ xcode-select --install\n\nFor most packages, the Xcode command-line tools are sufficient.\nHowever, some packages like ``qt`` require the full Xcode suite.\nYou can check to see which you have installed by running:\n\n.. code-block:: console\n\n $ xcode-select -p\n\nIf the output is:\n\n.. code-block:: none\n\n /Applications/Xcode.app/Contents/Developer\n\nyou already have the full Xcode suite installed.\nIf the output is:\n\n.. code-block:: none\n\n /Library/Developer/CommandLineTools\n\nyou only have the command-line tools installed.\nThe full Xcode suite can be installed through the App Store.\nMake sure to launch the Xcode application and accept the license agreement before using Spack.\nIt may ask you to install additional components.\nAlternatively, the Xcode license can be accepted through the command line:\n\n.. code-block:: console\n\n $ sudo xcodebuild -license accept\n" }, { "path": "lib/spack/docs/mirrors.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Discover how to set up and manage mirrors in Spack to provide a local repository of tarballs for offline package fetching.\n\n.. _mirrors:\n\nMirrors (mirrors.yaml)\n======================\n\nSome sites may not have access to the internet for fetching packages.\nThese sites will need a local repository of tarballs from which they can get their files.\nSpack has support for this with *mirrors*.\nA mirror is a URL that points to a directory, either on the local filesystem or on some server, containing tarballs for all of Spack's packages.\n\nHere's an example of a mirror's directory structure:\n\n.. code-block:: none\n\n mirror/\n cmake/\n cmake-2.8.10.2.tar.gz\n dyninst/\n dyninst-8.1.1.tgz\n dyninst-8.1.2.tgz\n libdwarf/\n libdwarf-20130126.tar.gz\n libdwarf-20130207.tar.gz\n libdwarf-20130729.tar.gz\n libelf/\n libelf-0.8.12.tar.gz\n libelf-0.8.13.tar.gz\n libunwind/\n libunwind-1.1.tar.gz\n mpich/\n mpich-3.0.4.tar.gz\n mvapich2/\n mvapich2-1.9.tgz\n\nThe structure is very simple.\nThere is a top-level directory.\nThe second level directories are named after packages, and the third level contains tarballs for each package, named after each package.\n\n.. note::\n\n Archives are **not** named exactly the way they were in the package's fetch URL.\n They have the form ``-.``, where ```` is Spack's name for the package, ```` is the version of the tarball, and ```` is whatever format the package's fetch URL contains.\n\n In order to make mirror creation reasonably fast, we copy the tarball in its original format to the mirror directory, but we do not standardize on a particular compression algorithm, because this would potentially require expanding and recompressing each archive.\n\n.. _cmd-spack-mirror:\n\n``spack mirror``\n----------------\n\nMirrors are managed with the ``spack mirror`` command.\nThe help for ``spack mirror`` looks like this:\n\n.. command-output:: spack help mirror\n\nThe ``create`` command actually builds a mirror by fetching all of its packages from the internet and checksumming them.\n\nThe other three commands are for managing mirror configuration.\nThey control the URL(s) from which Spack downloads its packages.\n\n.. _cmd-spack-mirror-create:\n\n``spack mirror create``\n-----------------------\n\nYou can create a mirror using the ``spack mirror create`` command, assuming you're on a machine where you can access the internet.\n\nThe command will iterate through all of Spack's packages and download the safe ones into a directory structure like the one above.\nHere is what it looks like:\n\n.. code-block:: console\n\n $ spack mirror create libelf libdwarf\n ==> Created new mirror in spack-mirror-2014-06-24\n ==> Trying to fetch from http://www.mr511.de/software/libelf-0.8.13.tar.gz\n ########################################################## 81.6%\n ==> Checksum passed for libelf@0.8.13\n ==> Added libelf@0.8.13\n ==> Trying to fetch from http://www.mr511.de/software/libelf-0.8.12.tar.gz\n ###################################################################### 98.6%\n ==> Checksum passed for libelf@0.8.12\n ==> Added libelf@0.8.12\n ==> Trying to fetch from http://www.prevanders.net/libdwarf-20130207.tar.gz\n ###################################################################### 97.3%\n ==> Checksum passed for libdwarf@20130207\n ==> Added libdwarf@20130207\n ==> Trying to fetch from http://www.prevanders.net/libdwarf-20130126.tar.gz\n ######################################################## 78.9%\n ==> Checksum passed for libdwarf@20130126\n ==> Added libdwarf@20130126\n ==> Trying to fetch from http://www.prevanders.net/libdwarf-20130729.tar.gz\n ############################################################# 84.7%\n ==> Added libdwarf@20130729\n ==> Added spack-mirror-2014-06-24/libdwarf/libdwarf-20130729.tar.gz to mirror\n ==> Added python@2.7.8.\n ==> Successfully updated mirror in spack-mirror-2015-02-24.\n Archive stats:\n 0 already present\n 5 added\n 0 failed to fetch.\n\nOnce this is done, you can tar up the ``spack-mirror-2014-06-24`` directory and copy it over to the machine you want it hosted on.\n\nCustom package sets\n^^^^^^^^^^^^^^^^^^^\n\nNormally, ``spack mirror create`` downloads all the archives it has checksums for.\nIf you want to only create a mirror for a subset of packages, you can do that by supplying a list of package specs on the command line after ``spack mirror create``.\nFor example, this command:\n\n.. code-block:: console\n\n $ spack mirror create libelf@0.8.12: boost@1.44:\n\nWill create a mirror for libelf versions greater than or equal to 0.8.12 and boost versions greater than or equal to 1.44.\n\nMirror files\n^^^^^^^^^^^^\n\nIf you have a *very* large number of packages you want to mirror, you can supply a file with specs in it, one per line:\n\n.. code-block:: console\n\n $ cat specs.txt\n libdwarf\n libelf@0.8.12:\n boost@1.44:\n boost@1.39.0\n ...\n $ spack mirror create --file specs.txt\n ...\n\nThis is useful if there is a specific suite of software managed by your site.\n\nMirror environment\n^^^^^^^^^^^^^^^^^^\n\nTo create a mirror of all packages required by a concrete environment, activate the environment and run ``spack mirror create -a``.\nThis is especially useful to create a mirror of an environment that was concretized on another machine.\n\nOptionally specify ``-j `` to control the number of workers used to create a full mirror.\nIf not specified, the optimal number of workers is determined dynamically.\nFor a full mirror, the number of workers used is the minimum of 16 workers, available CPU cores, and number of packages to mirror.\nFor individual packages, 1 worker is used.\n\n.. code-block:: console\n\n [remote] $ spack env create myenv\n [remote] $ spack env activate myenv\n [remote] $ spack add ...\n [remote] $ spack concretize\n \n $ sftp remote:/spack/var/environment/myenv/spack.lock\n $ spack env create myenv spack.lock\n $ spack env activate myenv\n $ spack mirror create -a\n \n\n\n.. _cmd-spack-mirror-add:\n\n``spack mirror add``\n--------------------\n\nOnce you have a mirror, you need to let Spack know about it.\nThis is relatively simple.\nFirst, figure out the URL for the mirror.\nIf it's a directory, you can use a file URL like this one:\n\n.. code-block:: none\n\n file://$HOME/spack-mirror-2014-06-24\n\nThat points to the directory on the local filesystem.\nIf it were on a web server, you could use a URL like this one:\n\nhttps://example.com/some/web-hosted/directory/spack-mirror-2014-06-24\n\nSpack will use the URL as the root for all of the packages it fetches.\nYou can tell your Spack installation to use that mirror like this:\n\n.. code-block:: console\n\n $ spack mirror add local_filesystem file://$HOME/spack-mirror-2014-06-24\n\nEach mirror has a name so that you can refer to it again later.\n\n.. _cmd-spack-mirror-list:\n\n``spack mirror list``\n---------------------\n\nTo see all the mirrors Spack knows about, run ``spack mirror list``:\n\n.. code-block:: console\n\n $ spack mirror list\n local_filesystem file:///home/username/spack-mirror-2014-06-24\n\n.. _cmd-spack-mirror-remove:\n\n``spack mirror remove``\n-----------------------\n\nTo remove a mirror by name, run:\n\n.. code-block:: console\n\n $ spack mirror remove local_filesystem\n $ spack mirror list\n ==> No mirrors configured.\n\nMirror precedence\n-----------------\n\nAdding a mirror really adds a line in ``~/.spack/mirrors.yaml``:\n\n.. code-block:: yaml\n\n mirrors:\n local_filesystem: file:///home/username/spack-mirror-2014-06-24\n remote_server: https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24\n\nIf you want to change the order in which mirrors are searched for packages, you can edit this file and reorder the sections.\nSpack will search the topmost mirror first and the bottom-most mirror last.\n\n.. _caching:\n\nLocal Default Cache\n-------------------\n\nSpack caches resources that are downloaded as part of installations.\nThe cache is a valid Spack mirror: it uses the same directory structure and naming scheme as other Spack mirrors (so it can be copied anywhere and referenced with a URL like other mirrors).\nThe mirror is maintained locally (within the Spack installation directory) at :file:`var/spack/cache/`.\nIt is always enabled (and is always searched first when attempting to retrieve files for an installation) but can be cleared with ``spack clean --misc-cache``; the cache directory can also be deleted manually without issue.\n\nCaching includes retrieved tarball archives and source control repositories, but only resources with an associated digest or commit ID (e.g. a revision number for SVN) will be cached.\n" }, { "path": "lib/spack/docs/module_file_support.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Learn how to configure and customize module file generation in Spack for Environment Modules and Lmod.\n\n.. _modules:\n\nModules (modules.yaml)\n======================\n\nThe use of module systems to manage user environments in a controlled way is a common practice at HPC centers that is sometimes embraced also by individual programmers on their development machines.\nTo support this common practice Spack integrates with `Environment Modules `_ and `Lmod `_ by providing post-install hooks that generate module files and commands to manipulate them.\n\nModules are one of several ways you can use Spack packages.\nFor other options that may fit your use case better, you should also look at :ref:`spack load ` and :ref:`environments `.\n\nQuick start\n-----------\n\nIn the current version of Spack, module files are not generated by default.\nTo get started, you can generate module files for all currently installed packages by running either\n\n.. code-block:: console\n\n $ spack module tcl refresh\n\nor\n\n.. code-block:: console\n\n $ spack module lmod refresh\n\nSpack can also generate module files for all future installations automatically through the following configuration:\n\n.. code-block:: console\n\n $ spack config add modules:default:enable:[tcl]\n\nor\n\n.. code-block:: console\n\n $ spack config add modules:default:enable:[lmod]\n\nAssuming you have a module system installed, you should now be able to use the ``module`` command to interact with them:\n\n.. code-block:: console\n\n $ module avail\n\n --------------------------------------------------------------- ~/spack/share/spack/modules/linux-ubuntu14-x86_64 ---------------------------------------------------------------\n autoconf/2.69-gcc-4.8-qextxkq hwloc/1.11.6-gcc-6.3.0-akcisez m4/1.4.18-gcc-4.8-ev2znoc openblas/0.2.19-gcc-6.3.0-dhkmed6 py-setuptools/34.2.0-gcc-6.3.0-fadur4s\n automake/1.15-gcc-4.8-maqvukj isl/0.18-gcc-4.8-afi6taq m4/1.4.18-gcc-6.3.0-uppywnz openmpi/2.1.0-gcc-6.3.0-go2s4z5 py-six/1.10.0-gcc-6.3.0-p4dhkaw\n binutils/2.28-gcc-4.8-5s7c6rs libiconv/1.15-gcc-4.8-at46wg3 mawk/1.3.4-gcc-4.8-acjez57 openssl/1.0.2k-gcc-4.8-dkls5tk python/2.7.13-gcc-6.3.0-tyehea7\n bison/3.0.4-gcc-4.8-ek4luo5 libpciaccess/0.13.4-gcc-6.3.0-gmufnvh mawk/1.3.4-gcc-6.3.0-ostdoms openssl/1.0.2k-gcc-6.3.0-gxgr5or readline/7.0-gcc-4.8-xhufqhn\n bzip2/1.0.6-gcc-4.8-iffrxzn libsigsegv/2.11-gcc-4.8-pp2cvte mpc/1.0.3-gcc-4.8-g5mztc5 pcre/8.40-gcc-4.8-r5pbrxb readline/7.0-gcc-6.3.0-zzcyicg\n bzip2/1.0.6-gcc-6.3.0-bequudr libsigsegv/2.11-gcc-6.3.0-7enifnh mpfr/3.1.5-gcc-4.8-o7xm7az perl/5.24.1-gcc-4.8-dg5j65u sqlite/3.8.5-gcc-6.3.0-6zoruzj\n cmake/3.7.2-gcc-6.3.0-fowuuby libtool/2.4.6-gcc-4.8-7a523za mpich/3.2-gcc-6.3.0-dmvd3aw perl/5.24.1-gcc-6.3.0-6uzkpt6 tar/1.29-gcc-4.8-wse2ass\n curl/7.53.1-gcc-4.8-3fz46n6 libtool/2.4.6-gcc-6.3.0-n7zmbzt ncurses/6.0-gcc-4.8-dcpe7ia pkg-config/0.29.2-gcc-4.8-ib33t75 tcl/8.6.6-gcc-4.8-tfxzqbr\n expat/2.2.0-gcc-4.8-mrv6bd4 libxml2/2.9.4-gcc-4.8-ryzxnsu ncurses/6.0-gcc-6.3.0-ucbhcdy pkg-config/0.29.2-gcc-6.3.0-jpgubk3 util-macros/1.19.1-gcc-6.3.0-xorz2x2\n flex/2.6.3-gcc-4.8-yf345oo libxml2/2.9.4-gcc-6.3.0-rltzsdh netlib-lapack/3.6.1-gcc-6.3.0-js33dog py-appdirs/1.4.0-gcc-6.3.0-jxawmw7 xz/5.2.3-gcc-4.8-mew4log\n gcc/6.3.0-gcc-4.8-24puqve lmod/7.4.1-gcc-4.8-je4srhr netlib-scalapack/2.0.2-gcc-6.3.0-5aidk4l py-numpy/1.12.0-gcc-6.3.0-oemmoeu xz/5.2.3-gcc-6.3.0-3vqeuvb\n gettext/0.19.8.1-gcc-4.8-yymghlh lua/5.3.4-gcc-4.8-im75yaz netlib-scalapack/2.0.2-gcc-6.3.0-hjsemcn py-packaging/16.8-gcc-6.3.0-i2n3dtl zip/3.0-gcc-4.8-rwar22d\n gmp/6.1.2-gcc-4.8-5ub2wu5 lua-luafilesystem/1_6_3-gcc-4.8-wkey3nl netlib-scalapack/2.0.2-gcc-6.3.0-jva724b py-pyparsing/2.1.10-gcc-6.3.0-tbo6gmw zlib/1.2.11-gcc-4.8-pgxsxv7\n help2man/1.47.4-gcc-4.8-kcnqmau lua-luaposix/33.4.0-gcc-4.8-mdod2ry netlib-scalapack/2.0.2-gcc-6.3.0-rgqfr6d py-scipy/0.19.0-gcc-6.3.0-kr7nat4 zlib/1.2.11-gcc-6.3.0-7cqp6cj\n\nThe names should look familiar, as they resemble the output from ``spack find``.\nFor example, you could type the following command to load the ``cmake`` module:\n\n.. code-block:: console\n\n $ module load cmake/3.7.2-gcc-6.3.0-fowuuby\n\nNeither of these is particularly pretty, easy to remember, or easy to type.\nLuckily, Spack offers many facilities for customizing the module scheme used at your site.\n\nModule file customization\n-------------------------\n\nThe table below summarizes the essential information associated with the different file formats that can be generated by Spack:\n\n\n+-----------+--------------+------------------------------+----------------------------------------------+----------------------+\n| | Hierarchical | **Default root directory** | **Default template file** | **Compatible tools** |\n+===========+==============+==============================+==============================================+======================+\n| ``tcl`` | No | share/spack/modules | share/spack/templates/modules/modulefile.tcl | Env. Modules/Lmod |\n+-----------+--------------+------------------------------+----------------------------------------------+----------------------+\n| ``lmod`` | Yes | share/spack/lmod | share/spack/templates/modules/modulefile.lua | Lmod |\n+-----------+--------------+------------------------------+----------------------------------------------+----------------------+\n\n\nSpack ships with sensible defaults for the generation of module files, but you can customize many aspects of it to accommodate package or site specific needs.\nIn general you can override or extend the default behavior by:\n\n1. overriding certain callback APIs in the Python packages\n2. writing specific rules in the ``modules.yaml`` configuration file\n3. writing your own templates to override or extend the defaults\n\nThe former method lets you express changes in the run-time environment that are needed to use the installed software properly, e.g. injecting variables from language interpreters into their extensions.\nThe latter two instead permit to fine tune the filesystem layout, content and creation of module files to meet site specific conventions.\n\n.. _overide-api-calls-in-package-py:\n\nSetting environment variables dynamically in ``package.py``\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nThere are two methods that you can implement in any ``package.py`` to dynamically affect the content of the module files generated by Spack.\nThe most important one is ``setup_run_environment``, which can be used to set environment variables in the module file that depend on the spec:\n\n.. code-block:: python\n\n def setup_run_environment(self, env: EnvironmentModifications) -> None:\n if self.spec.satisfies(\"+foo\"):\n env.set(\"FOO\", \"bar\")\n\nThe second, less commonly used, is ``setup_dependent_run_environment(self, env, dependent_spec)``, which allows a dependency to set variables in the module file of its dependents.\nThis is typically used in packages like ``python``, ``r``, or ``perl`` to prepend the dependent's prefix to the search path of the interpreter (``PYTHONPATH``, ``R_LIBS``, ``PERL5LIB`` resp.), so it can locate the packages at runtime.\n\nFor example, a simplified version of the ``python`` package could look like this:\n\n.. code-block:: python\n\n def setup_dependent_run_environment(\n self, env: EnvironmentModifications, dependent_spec: Spec\n ) -> None:\n if dependent_spec.package.extends(self.spec):\n env.prepend_path(\"PYTHONPATH\", dependent_spec.prefix.lib.python)\n\nand would make any package that ``extends(\"python\")`` have its library directory added to the ``PYTHONPATH`` environment variable in the module file.\nIt's much more convenient to set this variable here, than to repeat it in every Python extension's ``setup_run_environment`` method.\n\n.. _modules-yaml:\n\nThe ``modules.yaml`` config file and module sets\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nThe configuration files that control module generation behavior are named ``modules.yaml``.\nThe default configuration looks like this:\n\n.. literalinclude:: _spack_root/etc/spack/defaults/base/modules.yaml\n :language: yaml\n\nYou can define one or more **module sets**, each of which can be configured separately with regard to install location, naming scheme, inclusion and exclusion, autoloading, et cetera.\n\nThe default module set is aptly named ``default``.\nAll :ref:`Spack commands that operate on modules ` apply to the ``default`` module set, unless another module set is specified explicitly (with the ``--name`` flag).\n\n\nChanging the modules root\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nAs shown in the table above, the default module root for ``lmod`` is ``$spack/share/spack/lmod`` and the default root for ``tcl`` is ``$spack/share/spack/modules``.\nThis can be overridden for any module set by changing the ``roots`` key of the configuration.\n\n.. code-block:: yaml\n\n modules:\n default:\n roots:\n tcl: /path/to/install/tcl/modules\n my_custom_lmod_modules:\n roots:\n lmod: /path/to/install/custom/lmod/modules\n # ...\n\nThis configuration will create two module sets.\nThe default module set will install its ``tcl`` modules to ``/path/to/install/tcl/modules`` (and still install its lmod modules, if any, to the default location).\nThe set ``my_custom_lmod_modules`` will install its lmod modules to ``/path/to/install/custom/lmod/modules`` (and still install its tcl modules, if any, to the default location).\n\nBy default, an architecture-specific directory is added to the root directory.\nA module set may override that behavior by setting the ``arch_folder`` config value to ``False``.\n\n.. code-block:: yaml\n\n modules:\n default:\n roots:\n tcl: /path/to/install/tcl/modules\n arch_folder: false\n\nObviously, having multiple module sets install modules to the default location could be confusing to users of your modules.\nIn the next section, we will discuss enabling and disabling module types (module file generators) for each module set.\n\nAutomatically generating module files\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nSpack can be configured to automatically generate module files as part of package installation.\nThis is done by adding the desired module systems to the ``enable`` list.\n\n.. code-block:: yaml\n\n modules:\n default:\n enable:\n - tcl\n - lmod\n\nConfiguring ``tcl`` and ``lmod`` modules\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nYou can configure the behavior of either module system separately, under a key corresponding to the generator being customized:\n\n.. code-block:: yaml\n\n modules:\n default:\n tcl:\n # contains environment modules specific customizations\n lmod:\n # contains lmod specific customizations\n\nIn general, the configuration options that you can use in ``modules.yaml`` will either change the layout of the module files on the filesystem, or they will affect their content.\nFor the latter point it is possible to use anonymous specs to fine tune the set of packages on which the modifications should be applied.\n\n.. _autoloading-dependencies:\n\nAutoloading and hiding dependencies\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nA module file should set the variables that are needed for an application to work.\nBut since an application often has many dependencies, where should all the environment variables for those be set?\nIn Spack the rule is that each package sets the runtime variables that are needed by the package itself, and no more.\nThis way, dependencies can be loaded standalone too, and duplication of environment variables is avoided.\n\nThat means however that if you want to use an application, you need to load the modules for all its dependencies.\nOf course this is not something you would want users to do manually.\n\nSince Spack knows the dependency graph of every package, it can easily generate module files that automatically load the modules for its dependencies recursively.\nIt is enabled by default for both Lmod and Environment Modules under the ``autoload: direct`` config option.\nThe former system has builtin support through the ``depends_on`` function, the latter simply uses a ``module load`` statement.\nBoth module systems (at least in newer versions) do reference counting, so that if a module is loaded by two different modules, it will only be unloaded after the others are.\n\nThe ``autoload`` key accepts the values:\n\n* ``none``: no autoloading\n* ``run``: autoload direct *run* type dependencies\n* ``direct``: autoload direct *link and run* type dependencies\n* ``all``: autoload all dependencies\n\nIn case of ``run`` and ``direct``, a ``module load`` triggers a recursive load.\n\nThe ``direct`` option is most correct: there are cases where pure link dependencies need to set variables for themselves, or need to have variables of their own dependencies set.\n\nIn practice however, ``run`` is often sufficient, and may make ``module load`` snappier.\n\nThe ``all`` option is discouraged and seldomly used.\n\nA common complaint about autoloading is the large number of modules that are visible to the user.\nSpack has a solution for this as well: ``hide_implicits: true``.\nThis ensures that only those packages you've explicitly installed are exposed by ``module avail``, but still allows for autoloading of hidden dependencies.\nLmod should support hiding implicits in general, while Environment Modules requires version 4.7 or higher.\n\n.. note::\n If supported by your module system, we highly encourage the following configuration that enables autoloading and hiding of implicits.\n It ensures all runtime variables are set correctly, including those for dependencies, without overwhelming the user with a large number of available modules.\n Further, it makes it easier to get readable module names without collisions, see the section below on :ref:`modules-projections`.\n\n .. code-block:: yaml\n\n modules:\n default:\n tcl:\n hide_implicits: true\n all:\n autoload: direct # or `run`\n lmod:\n hide_implicits: true\n all:\n autoload: direct # or `run`\n\n.. _anonymous_specs:\n\nSetting environment variables for selected packages in config\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nIn the configuration file you can filter particular specs, and make further changes to the environment variables that go into their module files.\nThis is very powerful when you want to avoid :ref:`modifying the package itself `, or when you want to set certain variables on multiple selected packages at once.\n\nFor instance, in the snippet below:\n\n.. code-block:: yaml\n\n modules:\n default:\n tcl:\n # The keyword `all` selects every package\n all:\n environment:\n set:\n BAR: \"bar\"\n # This anonymous spec selects any package that\n # depends on mpi. The double colon at the\n # end clears the set of rules that matched so far.\n ^mpi::\n environment:\n prepend_path:\n PATH: \"{^mpi.prefix}/bin\"\n set:\n BAR: \"baz\"\n # Selects any zlib package\n zlib:\n environment:\n prepend_path:\n LD_LIBRARY_PATH: \"foo\"\n # Selects zlib compiled with gcc@4.8\n zlib%gcc@4.8:\n environment:\n unset:\n - FOOBAR\n\nyou are instructing Spack to set the environment variable ``BAR=bar`` for every module, unless the associated spec satisfies the abstract dependency ``^mpi`` in which case ``BAR=baz``, and the directory containing the respective MPI executables is prepended to the ``PATH`` variable.\nIn addition in any spec that satisfies ``zlib`` the value ``foo`` will be prepended to ``LD_LIBRARY_PATH`` and in any spec that satisfies ``zlib%gcc@4.8`` the variable ``FOOBAR`` will be unset.\n\n.. admonition:: Note: order does matter\n :class: note\n\n The modifications associated with the ``all`` keyword are always evaluated first, no matter where they appear in the configuration file.\n All the other changes to environment variables for matching specs are evaluated from top to bottom.\n\n.. warning::\n\n As general advice, it's often better to set as few unnecessary variables as possible.\n For example, the following seemingly innocent and potentially useful configuration\n\n .. code-block:: yaml\n\n all:\n environment:\n set:\n \"{name}_ROOT\": \"{prefix}\"\n\n sets ``BINUTILS_ROOT`` to its prefix in modules for ``binutils``, which happens to break the ``gcc`` compiler: it uses this variable as its default search path for certain object files and libraries, and by merely setting it, everything fails to link.\n\nExclude or include specific module files\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nYou can use anonymous specs also to prevent module files from being written or to force them to be written.\nConsider the case where you want to hide from users all the boilerplate software that you had to build in order to bootstrap a new compiler.\nSuppose for instance that ``gcc@4.4.7`` is the compiler provided by your system.\nIf you write a configuration file like:\n\n.. code-block:: yaml\n\n modules:\n default:\n tcl:\n include: [\"gcc\", \"llvm\"] # include will have precedence over exclude\n exclude: [\"%gcc@4.4.7\"] # Assuming gcc@4.4.7 is the system compiler\n\nyou will prevent the generation of module files for any package that is compiled with ``gcc@4.4.7``, with the only exception of any ``gcc`` or any ``llvm`` installation.\n\nIt is safe to combine ``exclude`` and ``autoload`` :ref:`mentioned above `.\nWhen ``exclude`` prevents a module file to be generated for a dependency, the ``autoload`` feature will simply not generate a statement to load it.\n\n\n.. _modules-projections:\n\nCustomize the naming of modules\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nThe names of environment modules generated by Spack are not always easy to fully comprehend due to the long hash in the name.\nThere are three module configuration options to help with that.\nThe first is a global setting to adjust the hash length.\nIt can be set anywhere from 0 to 32 and has a default length of 7.\nThis is the representation of the hash in the module file name and does not affect the size of the package hash.\nBe aware that the smaller the hash length the more likely naming conflicts will occur.\nThe following snippet shows how to set hash length in the module file names:\n\n.. code-block:: yaml\n\n modules:\n default:\n tcl:\n hash_length: 7\n\n.. tip::\n\n Using ``hide_implicits: true`` (see :ref:`autoloading-dependencies`) vastly reduces the number modules exposed to the user.\n The hidden modules always contain the hash in their name, and are not influenced by the ``hash_length`` setting.\n Hidden implicits thus make it easier to use a short hash length or no hash at all, without risking name conflicts.\n\nTo help make module names more readable, and to help alleviate name conflicts with a short hash, one can use the ``suffixes`` option in the modules configuration file.\nThis option will add strings to modules that match a spec.\nFor instance, the following config options,\n\n.. code-block:: yaml\n\n modules:\n default:\n tcl:\n all:\n suffixes:\n ^python@3: \"python{^python.version.up_to_2}\"\n ^openblas: \"openblas\"\n\nwill add a ``python3.12`` to module names of packages compiled with Python 3.12, and similarly for all specs depending on ``python@3``.\nThis is useful to know which version of Python a set of Python extensions is associated with.\nLikewise, the ``openblas`` string is attached to any program that has ``openblas`` in the spec, most likely via the ``+blas`` variant specification.\n\nThe most heavyweight solution to module naming is to change the entire naming convention for module files.\nThis uses the projections format covered in :ref:`view_projections`.\n\n.. code-block:: yaml\n\n modules:\n default:\n tcl:\n projections:\n all: \"{name}/{version}-{compiler.name}-{compiler.version}-module\"\n ^mpi: \"{name}/{version}-{^mpi.name}-{^mpi.version}-{compiler.name}-{compiler.version}-module\"\n\nwill create module files that are nested in directories by package name, contain the version and compiler name and version, and have the word ``module`` before the hash for all specs that do not depend on ``mpi``, and will have the same information plus the MPI implementation name and version for all packages that depend on ``mpi``.\n\nWhen specifying module names by projection for Lmod modules, we recommend NOT including names of dependencies (e.g., MPI, compilers) that are already in the Lmod hierarchy.\n\n\n\n.. note::\n\n Tcl and Lua modules also allow for explicit conflicts between module files.\n\n .. code-block:: yaml\n\n modules:\n default:\n enable:\n - tcl\n tcl:\n projections:\n all: \"{name}/{version}-{compiler.name}-{compiler.version}\"\n all:\n conflict:\n - \"{name}\"\n - \"intel/14.0.1\"\n\n will create module files that will conflict with ``intel/14.0.1`` and with the base directory of the same module, effectively preventing the possibility to load two or more versions of the same software at the same time.\n The tokens that are available for use in this directive are those understood by the :meth:`~spack.spec.Spec.format` method.\n\n For Lmod and Environment Modules versions prior to 4.2, it is important to express the conflict on both module files conflicting with each other.\n\n\n.. admonition:: Note: Lmod hierarchical module files\n :class: note\n\n When ``lmod`` is activated Spack will generate a set of hierarchical lua module files that are understood by Lmod.\n The hierarchy always contains the ``Core`` and ``Compiler`` layers, but can be extended to include any package or virtual package in Spack.\n A case that could be useful in practice is for instance:\n\n .. code-block:: yaml\n\n modules:\n default:\n enable:\n - lmod\n lmod:\n core_compilers:\n - \"gcc@4.8\"\n core_specs:\n - \"r\"\n hierarchy:\n - \"mpi\"\n - \"lapack\"\n - \"python\"\n\n that will generate a hierarchy in which the ``python``, ``lapack`` and ``mpi`` layer can be switched independently.\n This allows a site to build the same libraries or applications against different implementations of ``mpi`` and ``lapack``, and with different versions of those implementations and of ``python``, and let Lmod switch safely from among the resulting installs.\n\n All packages built with a compiler in ``core_compilers`` and all packages that satisfy a spec in ``core_specs`` will be put in the ``Core`` hierarchy of the lua modules.\n\n.. admonition:: Warning: consistency of core packages\n :class: warning\n\n The user is responsible for maintaining consistency among core packages, as ``core_specs`` bypasses the hierarchy that allows Lmod to safely switch between coherent software stacks.\n\n.. admonition:: Warning: deep hierarchies\n :class: warning\n\n For hierarchies that are deeper than three layers ``lmod spider`` may have some issues.\n See `this discussion on the Lmod project `_.\n\n.. _customize-env-modifications:\n\nCustomize environment modifications\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nYou can control which prefixes in a Spack package are added to environment variables with the ``prefix_inspections`` section; this section maps relative prefixes to the list of environment variables which should be updated with those prefixes.\n\nThe ``prefix_inspections`` configuration is different from other settings in that a ``prefix_inspections`` configuration at the ``modules`` level of the configuration file applies to all module sets.\nThis allows users to make general overrides to the default inspections and customize them per-module-set.\n\n.. code-block:: yaml\n\n modules:\n prefix_inspections:\n ./bin:\n - PATH\n ./man:\n - MANPATH\n ./:\n - CMAKE_PREFIX_PATH\n\nPrefix inspections are only applied if the relative path inside the installation prefix exists.\nIn this case, for a Spack package ``foo`` installed to ``/spack/prefix/foo``, if ``foo`` installs executables to ``bin`` but no manpages in ``man``, the generated module file for ``foo`` would update ``PATH`` to contain ``/spack/prefix/foo/bin`` and ``CMAKE_PREFIX_PATH`` to contain ``/spack/prefix/foo``, but would not update ``MANPATH``.\n\nThe default list of environment variables in this config section includes ``PATH``, ``MANPATH``, ``ACLOCAL_PATH``, ``PKG_CONFIG_PATH`` and ``CMAKE_PREFIX_PATH``, as well as ``DYLD_FALLBACK_LIBRARY_PATH`` on macOS.\nOn Linux however, the corresponding ``LD_LIBRARY_PATH`` variable is *not* set, because it affects the behavior of system executables too.\n\n.. note::\n\n In general, the ``LD_LIBRARY_PATH`` variable is not required when using packages built with Spack, thanks to the use of RPATH.\n Some packages may still need the variable, which is best handled on a per-package basis instead of globally, as explained in :ref:`overide-api-calls-in-package-py`.\n\nThere is a special case for prefix inspections relative to environment views.\nIf all of the following conditions hold for a module set configuration:\n\n#. The configuration is for an :ref:`environment ` and will never be applied outside the environment,\n#. The environment in question is configured to use a view,\n#. The :ref:`environment view is configured ` with a projection that ensures every package is linked to a unique directory,\n\nthen the module set may be configured to create modules relative to the environment view.\nThis is specified by the ``use_view`` configuration option in the module set.\nIf ``True``, the module set is constructed relative to the default view of the environment.\nOtherwise, the value must be the name of the environment view relative to which to construct modules, or ``False-ish`` to disable the feature explicitly (the default is ``False``).\n\nIf the ``use_view`` value is set in the config, then the prefix inspections for the package are done relative to the package's path in the view.\n\n.. code-block:: yaml\n\n spack:\n modules:\n view_relative_modules:\n use_view: my_view\n prefix_inspections:\n ./bin:\n - PATH\n view:\n my_view:\n root: /path/to/my/view\n projections:\n all: \"{name}-{hash}\"\n\nThe ``spack`` key is relevant to :ref:`environment ` configuration, and the view key is discussed in detail in the section on :ref:`Configuring environment views `.\nWith this configuration the generated module for package ``foo`` would set ``PATH`` to include ``/path/to/my/view/foo-/bin`` instead of ``/spack/prefix/foo/bin``.\n\nThe ``use_view`` option is useful when deploying a large software stack to users who are likely to inspect the modules to find full paths to software, when it is desirable to present the users with a simpler set of paths than those generated by the Spack install tree.\n\nFilter out environment modifications\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nModifications to certain environment variables in module files are there by default, for instance because they are generated by prefix inspections.\nIf you want to prevent modifications to some environment variables, you can do so by using the ``exclude_env_vars``:\n\n.. code-block:: yaml\n\n modules:\n default:\n tcl:\n all:\n filter:\n # Exclude changes to any of these variables\n exclude_env_vars: [\"CPATH\", \"LIBRARY_PATH\"]\n\nThe configuration above will generate module files that will not contain modifications to either ``CPATH`` or ``LIBRARY_PATH``.\n\nSelect default modules\n^^^^^^^^^^^^^^^^^^^^^^\n\nBy default, when multiple modules of the same name share a directory, the highest version number will be the default module.\nThis behavior of the ``module`` command can be overridden with a symlink named ``default`` to the desired default module.\nIf you wish to configure default modules with Spack, add a ``defaults`` key to your modules configuration:\n\n.. code-block:: yaml\n\n modules:\n my-module-set:\n tcl:\n defaults:\n - gcc@10.2.1\n - hdf5@1.2.10+mpi+hl%gcc\n\nThese defaults may be arbitrarily specific.\nFor any package that satisfies a default, Spack will generate the module file in the appropriate path, and will generate a default symlink to the module file as well.\n\n.. warning::\n\n If Spack is configured to generate multiple default packages in the same directory, the last modulefile to be generated will be the default module.\n\n.. _maintaining-module-files:\n\nMaintaining Module Files\n------------------------\n\nEach type of module file has a command with the same name associated with it.\nThe actions these commands permit are usually associated with the maintenance of a production environment.\nHere's, for instance, a sample of the features of the ``spack module tcl`` command:\n\n.. command-output:: spack module tcl --help\n\n.. _cmd-spack-module-refresh:\n\nRefresh the set of modules\n^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nThe subcommand that regenerates module files to update their content or their layout is ``refresh``:\n\n.. command-output:: spack module tcl refresh --help\n\nA set of packages can be selected using anonymous specs for the optional ``constraint`` positional argument.\nOptionally the entire tree can be deleted before regeneration if the change in layout is radical.\n\n.. _cmd-spack-module-rm:\n\nDelete module files\n^^^^^^^^^^^^^^^^^^^\n\nIf instead what you need is just to delete a few module files, then the right subcommand is ``rm``:\n\n.. command-output:: spack module tcl rm --help\n\n.. note::\n\n We care about your module files!\n Every modification done on modules that are already existing will ask for a confirmation by default.\n If the command is used in a script it is possible though to pass the ``-y`` argument, that will skip this safety measure.\n\n\n.. _modules-in-shell-scripts:\n\nUsing Spack modules in shell scripts\n------------------------------------\n\nTo enable additional Spack commands for loading and unloading module files, and to add the correct path to ``MODULEPATH``, you need to source the appropriate setup file.\nAssuming Spack is installed in ``$SPACK_ROOT``, run the appropriate command for your shell:\n\n.. code-block:: console\n\n # For bash/zsh/sh\n $ . $SPACK_ROOT/share/spack/setup-env.sh\n\n # For tcsh/csh\n $ source $SPACK_ROOT/share/spack/setup-env.csh\n\n # For fish\n $ . $SPACK_ROOT/share/spack/setup-env.fish\n\nIf you want to have Spack's shell support available on the command line at any login you can put this source line in one of the files that are sourced at startup (like ``.profile``, ``.bashrc`` or ``.cshrc``).\nBe aware that the shell startup time may increase slightly as a result.\n\n.. _cmd-spack-module-loads:\n\n``spack module tcl loads``\n^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nIn some cases, it is desirable to use a Spack-generated module, rather than relying on Spack's built-in user-environment modification capabilities.\nTo translate a spec into a module name, use ``spack module tcl loads`` or ``spack module lmod loads`` depending on the module system desired.\n\n\nTo load not just a module, but also all the modules it depends on, use the ``--dependencies`` option.\nThis is not required for most modules because Spack builds binaries with RPATH support.\nHowever, not all packages use RPATH to find their dependencies: this can be true in particular for Python extensions, which are currently *not* built with RPATH.\n\nScripts to load modules recursively may be made with the command:\n\n.. code-block:: console\n\n $ spack module tcl loads --dependencies \n\nAn equivalent alternative using `process substitution `_ is:\n\n.. code-block:: console\n\n $ source <( spack module tcl loads --dependencies )\n\n\nModule Commands for Shell Scripts\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nAlthough Spack is flexible, the ``module`` command is much faster.\nThis can become an issue when emitting a series of ``spack load`` commands inside a shell script.\nBy adding the ``--dependencies`` flag, ``spack module tcl loads`` may also be used to generate code that can be cut-and-pasted into a shell script.\nFor example:\n\n.. code-block:: console\n\n $ spack module tcl loads --dependencies py-numpy git\n # bzip2@1.0.6%gcc@4.9.3=linux-x86_64\n module load bzip2/1.0.6-gcc-4.9.3-ktnrhkrmbbtlvnagfatrarzjojmkvzsx\n # ncurses@6.0%gcc@4.9.3=linux-x86_64\n module load ncurses/6.0-gcc-4.9.3-kaazyneh3bjkfnalunchyqtygoe2mncv\n # zlib@1.2.8%gcc@4.9.3=linux-x86_64\n module load zlib/1.2.8-gcc-4.9.3-v3ufwaahjnviyvgjcelo36nywx2ufj7z\n # sqlite@3.8.5%gcc@4.9.3=linux-x86_64\n module load sqlite/3.8.5-gcc-4.9.3-a3eediswgd5f3rmto7g3szoew5nhehbr\n # readline@6.3%gcc@4.9.3=linux-x86_64\n module load readline/6.3-gcc-4.9.3-se6r3lsycrwxyhreg4lqirp6xixxejh3\n # python@3.5.1%gcc@4.9.3=linux-x86_64\n module load python/3.5.1-gcc-4.9.3-5q5rsrtjld4u6jiicuvtnx52m7tfhegi\n # py-setuptools@20.5%gcc@4.9.3=linux-x86_64\n module load py-setuptools/20.5-gcc-4.9.3-4qr2suj6p6glepnedmwhl4f62x64wxw2\n # py-nose@1.3.7%gcc@4.9.3=linux-x86_64\n module load py-nose/1.3.7-gcc-4.9.3-pwhtjw2dvdvfzjwuuztkzr7b4l6zepli\n # openblas@0.2.17%gcc@4.9.3+shared=linux-x86_64\n module load openblas/0.2.17-gcc-4.9.3-pw6rmlom7apfsnjtzfttyayzc7nx5e7y\n # py-numpy@1.11.0%gcc@4.9.3+blas+lapack=linux-x86_64\n module load py-numpy/1.11.0-gcc-4.9.3-mulodttw5pcyjufva4htsktwty4qd52r\n # curl@7.47.1%gcc@4.9.3=linux-x86_64\n module load curl/7.47.1-gcc-4.9.3-ohz3fwsepm3b462p5lnaquv7op7naqbi\n # autoconf@2.69%gcc@4.9.3=linux-x86_64\n module load autoconf/2.69-gcc-4.9.3-bkibjqhgqm5e3o423ogfv2y3o6h2uoq4\n # cmake@3.5.0%gcc@4.9.3~doc+ncurses+openssl~qt=linux-x86_64\n module load cmake/3.5.0-gcc-4.9.3-x7xnsklmgwla3ubfgzppamtbqk5rwn7t\n # expat@2.1.0%gcc@4.9.3=linux-x86_64\n module load expat/2.1.0-gcc-4.9.3-6pkz2ucnk2e62imwakejjvbv6egncppd\n # git@2.8.0-rc2%gcc@4.9.3+curl+expat=linux-x86_64\n module load git/2.8.0-rc2-gcc-4.9.3-3bib4hqtnv5xjjoq5ugt3inblt4xrgkd\n\nThe script may be further edited by removing unnecessary modules.\n\n\nModule Prefixes\n^^^^^^^^^^^^^^^\n\nOn some systems, modules are automatically prefixed with a certain string; ``spack module tcl loads`` needs to know about that prefix when it issues ``module load`` commands.\nAdd the ``--prefix`` option to your ``spack module tcl loads`` commands if this is necessary.\n\nFor example, consider the following on one system:\n\n.. code-block:: console\n\n $ module avail\n linux-SuSE11-x86_64/antlr/2.7.7-gcc-5.3.0-bdpl46y\n\n $ spack module tcl loads antlr # WRONG!\n # antlr@2.7.7%gcc@5.3.0~csharp+cxx~java~python arch=linux-SuSE11-x86_64\n module load antlr/2.7.7-gcc-5.3.0-bdpl46y\n\n $ spack module tcl loads --prefix linux-SuSE11-x86_64/ antlr\n # antlr@2.7.7%gcc@5.3.0~csharp+cxx~java~python arch=linux-SuSE11-x86_64\n module load linux-SuSE11-x86_64/antlr/2.7.7-gcc-5.3.0-bdpl46y\n" }, { "path": "lib/spack/docs/package_api.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n An overview of the Spack Package API, a stable interface for package authors to interact with the Spack framework.\n\nSpack Package API\n=================\n\nThis document describes the Spack Package API (:mod:`spack.package`), the stable interface for Spack package authors.\nIt is assumed you have already read the :doc:`Spack Packaging Guide `.\n\nThe Spack Package API is the *only* module in the Spack codebase considered public API.\nIt re-exports essential functions and classes from various Spack modules, allowing package authors to import them directly from :mod:`spack.package` without needing to know Spack's internal structure.\n\nSpack Package API Versioning\n----------------------------\n\nThe current Package API version is |package_api_version|, defined in :attr:`spack.package_api_version`.\nNotice that the Package API is versioned independently from Spack itself:\n\n* The **minor version** is incremented when new functions or classes are exported from :mod:`spack.package`.\n* The **major version** is incremented when functions or classes are removed or have breaking changes to their signatures (a rare occurrence).\n\nThis independent versioning allows package authors to utilize new Spack features without waiting for a new Spack release.\n\nCompatibility between Spack and :doc:`package repositories ` is managed as follows:\n\n* Package repositories declare their minimum required Package API version in their ``repo.yaml`` file using the ``api: vX.Y`` format.\n* Spack checks if the declared API version falls within its supported range, specifically between :attr:`spack.min_package_api_version` and :attr:`spack.package_api_version`.\n\nSpack version |spack_version| supports package repositories with a Package API version between |min_package_api_version| and |package_api_version|, inclusive.\n\nChangelog\n---------\n\n**v2.4** *(Spack v1.0.3)*\n\n* The ``%%`` operator can be used on input specs to set propagated preferences, which is particularly useful for ``unify: false`` environments.\n\n**v2.3** *(Spack v1.0.3)*\n\n* The :func:`~spack.package.version` directive now supports the ``git_sparse_paths`` parameter, allowing sparse checkouts when fetching from git repositories.\n\n**v2.2** *(Spack v1.0.0)*\n\n* Renamed implicit builder attributes with backward compatibility:\n\n * ``legacy_buildsystem`` to ``default_buildsystem``,\n * ``legacy_methods`` to ``package_methods``,\n * ``legacy_attributes`` to ``package_attributes``,\n * ``legacy_long_methods`` to ``package_long_methods``.\n\n* Exported :class:`~spack.package.GenericBuilder`, :class:`~spack.package.Package`, and :class:`~spack.package.BuilderWithDefaults` from :mod:`spack.package`.\n* Exported numerous utility functions and classes for file operations, library/header search, macOS/Windows support, compiler detection, and build system helpers.\n\n**v2.1** *(Spack v1.0.0)*\n\n* Exported :class:`~spack.package.CompilerError` and :class:`~spack.package.SpackError` from :mod:`spack.package`.\n\nSpack Package API Reference\n---------------------------\n\n.. automodule:: spack.package\n :members:\n :show-inheritance:\n :undoc-members:\n :no-value:\n" }, { "path": "lib/spack/docs/package_fundamentals.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Learn the fundamental Spack commands for managing software packages, including how to find, inspect, install, and remove them.\n\n.. _basic-usage:\n\nPackage Fundamentals\n====================\n\nSpack provides a comprehensive ecosystem of software packages that you can install.\nIn this section you'll learn:\n\n1. How to discover which packages are available,\n2. How to get detailed information about specific packages,\n3. How to install/uninstall packages, and\n4. How to discover, and use, software that has been installed\n\n\n\n.. _basic-list-and-info-packages:\n\nListing Available Packages\n--------------------------\n\nTo install software with Spack, you need to know what software is available.\nYou can search for available packages on the `packages.spack.io `_ website or by using the ``spack list`` command.\n\n.. _cmd-spack-list:\n\n``spack list``\n^^^^^^^^^^^^^^\n\nThe ``spack list`` command prints out a list of all of the packages Spack can install:\n\n.. code-block:: spec\n\n $ spack list\n\nPackages are listed by name in alphabetical order.\nA pattern can be used to narrow the list, and the following rules apply:\n\n* A pattern with no wildcards (``*`` or ``?``) is treated as if it starts and ends with ``*``\n* All patterns are case-insensitive\n\nTo search for all packages whose names contain the word ``sql`` you can run the following command:\n\n.. code-block:: spec\n\n $ spack list sql\n\nA few options are also provided for more specific searches.\nFor instance, it is possible to search the description of packages for a match.\nA way to list all the packages whose names or descriptions contain the word ``quantum`` is the following:\n\n.. code-block:: spec\n\n $ spack list -d quantum\n\n\n.. _cmd-spack-info:\n\n``spack info``\n^^^^^^^^^^^^^^\n\nTo get more information about a particular package from `spack list`, use `spack info`.\nJust supply the name of a package:\n\n.. command-output:: spack info mpich\n :language: spec\n\nMost of the information is self-explanatory.\nThe *safe versions* are versions for which Spack knows the checksum.\nSpack uses this checksum to verify that the versions are downloaded without errors or malicious changes.\n\n:ref:`Dependencies ` and :ref:`virtual dependencies ` are described in more detail later.\n\n.. _cmd-spack-versions:\n\n``spack versions``\n^^^^^^^^^^^^^^^^^^\n\nTo see *more* available versions of a package, run ``spack versions``.\nFor example:\n\n.. command-output:: spack versions libelf\n :language: spec\n\nThere are two sections in the output.\n*Safe versions* are versions for which Spack has a checksum on file.\nIt can verify that these versions are downloaded correctly.\n\nIn many cases, Spack can also show you what versions are available out on the web -- these are *remote versions*.\nSpack gets this information by scraping it directly from package web pages.\nDepending on the package and how its releases are organized, Spack may or may not be able to find remote versions.\n\n.. _cmd-spack-providers:\n\n``spack providers``\n^^^^^^^^^^^^^^^^^^^\n\nYou can see what packages provide a particular virtual package using ``spack providers``.\nIf you wanted to see what packages provide ``mpi``, you would just run:\n\n.. command-output:: spack providers mpi\n :language: spec\n\nAnd if you *only* wanted to see packages that provide MPI-2, you would add a version specifier to the spec:\n\n.. command-output:: spack providers mpi@2\n :language: spec\n\nNotice that the package versions that provide insufficient MPI versions are now filtered out.\n\nInstalling and Uninstalling\n---------------------------\n\n.. _cmd-spack-install:\n\n``spack install``\n^^^^^^^^^^^^^^^^^\n\n``spack install`` will install any package shown by ``spack list``.\nFor example, to install the latest version of the ``mpileaks`` package, you might type this:\n\n.. code-block:: spec\n\n $ spack install mpileaks\n\nIf ``mpileaks`` depends on other packages, Spack will install the dependencies first.\nIt then fetches the ``mpileaks`` tarball, expands it, verifies that it was downloaded without errors, builds it, and installs it in its own directory under ``$SPACK_ROOT/opt``.\n\n.. code-block:: spec\n\n $ spack install mpileaks\n ... dependency build output ...\n [+] ph7pbnh mpileaks@1.0 ~/spack/opt/linux-rhel7-broadwell/gcc-8.1.0/mpileaks-1.0-ph7pbnhl334wuhogmugriohcwempqry2 (5s)\n\nThe last line, with the ``[+]``, indicates where the package is installed.\n\nBuilding a specific version\n^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nSpack can also build *specific versions* of a package.\nTo do this, just add ``@`` after the package name, followed by a version:\n\n.. code-block:: spec\n\n $ spack install mpich@3.0.4\n\nAny number of versions of the same package can be installed at once without interfering with each other.\nThis is useful for multi-user sites, as installing a version that one user needs will not disrupt existing installations for other users.\n\nIn addition to different versions, Spack can customize the compiler, compile-time options (variants), compiler flags, and target architecture of an installation.\nSpack is unique in that it can also configure the *dependencies* a package is built with.\nFor example, two configurations of the same version of a package, one built with boost 1.39.0, and the other version built with version 1.43.0, can coexist.\n\nThis can all be done on the command line using the *spec* syntax.\nSpack calls the descriptor used to refer to a particular package configuration a **spec**.\nIn the commands above, ``mpileaks`` and ``mpileaks@3.0.4`` are both valid *specs*.\nWe'll talk more about how you can use them to customize an installation in :ref:`sec-specs`.\n\nReusing installed dependencies\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nBy default, when you run ``spack install``, Spack tries hard to reuse existing installations as dependencies, either from a local store or from remote build caches, if configured.\nThis minimizes unwanted rebuilds of common dependencies, in particular if you update Spack frequently.\n\nIn case you want the latest versions and configurations to be installed instead, you can add the ``--fresh`` option:\n\n.. code-block:: spec\n\n $ spack install --fresh mpich\n\nReusing installations in this mode is \"accidental\" and happens only if there's a match between existing installations and what Spack would have installed anyway.\n\nYou can use the ``spack spec -I mpich`` command to see what will be reused and what will be built before you install.\n\nYou can configure Spack to use the ``--fresh`` behavior by default in ``concretizer.yaml``:\n\n.. code-block:: yaml\n\n concretizer:\n reuse: false\n\n.. _cmd-spack-uninstall:\n\n``spack uninstall``\n^^^^^^^^^^^^^^^^^^^\n\nTo uninstall a package, run ``spack uninstall ``.\nThis will ask the user for confirmation before completely removing the directory in which the package was installed.\n\n.. code-block:: spec\n\n $ spack uninstall mpich\n\nIf there are still installed packages that depend on the package to be uninstalled, Spack will refuse to uninstall it.\n\nTo uninstall a package and every package that depends on it, you may give the ``--dependents`` option.\n\n.. code-block:: spec\n\n $ spack uninstall --dependents mpich\n\nwill display a list of all the packages that depend on ``mpich`` and, upon confirmation, will uninstall them in the correct order.\n\nA command like\n\n.. code-block:: spec\n\n $ spack uninstall mpich\n\nmay be ambiguous if multiple ``mpich`` configurations are installed.\nFor example, if both ``mpich@3.0.2`` and ``mpich@3.1`` are installed, ``mpich`` could refer to either one.\nBecause it cannot determine which one to uninstall, Spack will ask you either to provide a version number to remove the ambiguity or use the ``--all`` option to uninstall all matching packages.\n\nYou may force uninstall a package with the ``--force`` option\n\n.. code-block:: spec\n\n $ spack uninstall --force mpich\n\nbut you risk breaking other installed packages.\nIn general, it is safer to remove dependent packages *before* removing their dependencies or to use the ``--dependents`` option.\n\n\n.. _cmd-spack-gc:\n\nGarbage collection\n^^^^^^^^^^^^^^^^^^\n\nWhen Spack builds software from sources, it often installs tools that are needed only to build or test other software.\nThese are not necessary at runtime.\nTo support cases where removing these tools can be a benefit, Spack provides the ``spack gc`` (\"garbage collector\") command, which will uninstall all unneeded packages:\n\n.. code-block:: console\n\n $ spack find\n ==> 24 installed packages\n -- linux-ubuntu18.04-broadwell / gcc@9.0.1 ----------------------\n autoconf@2.69 findutils@4.6.0 libiconv@1.16 libszip@2.1.1 m4@1.4.18 openjpeg@2.3.1 pkgconf@1.6.3 util-macros@1.19.1\n automake@1.16.1 gdbm@1.18.1 libpciaccess@0.13.5 libtool@2.4.6 mpich@3.3.2 openssl@1.1.1d readline@8.0 xz@5.2.4\n cmake@3.16.1 hdf5@1.10.5 libsigsegv@2.12 libxml2@2.9.9 ncurses@6.1 perl@5.30.0 texinfo@6.5 zlib@1.2.11\n\n $ spack gc\n ==> The following packages will be uninstalled:\n\n -- linux-ubuntu18.04-broadwell / gcc@9.0.1 ----------------------\n vn47edz autoconf@2.69 6m3f2qn findutils@4.6.0 ubl6bgk libtool@2.4.6 pksawhz openssl@1.1.1d urdw22a readline@8.0\n ki6nfw5 automake@1.16.1 fklde6b gdbm@1.18.1 b6pswuo m4@1.4.18 k3s2csy perl@5.30.0 lp5ya3t texinfo@6.5\n ylvgsov cmake@3.16.1 5omotir libsigsegv@2.12 leuzbbh ncurses@6.1 5vmfbrq pkgconf@1.6.3 5bmv4tg util-macros@1.19.1\n\n ==> Do you want to proceed? [y/N] y\n\n [ ... ]\n\n $ spack find\n ==> 9 installed packages\n -- linux-ubuntu18.04-broadwell / gcc@9.0.1 ----------------------\n hdf5@1.10.5 libiconv@1.16 libpciaccess@0.13.5 libszip@2.1.1 libxml2@2.9.9 mpich@3.3.2 openjpeg@2.3.1 xz@5.2.4 zlib@1.2.11\n\nIn the example above, ``spack gc`` scans the package database.\nIt keeps only the packages that were explicitly installed by a user, along with their required ``link`` and ``run`` dependencies (including transitive dependencies).\nAll other packages, such as build-only dependencies or orphaned packages, are identified as \"garbage\" and removed.\n\nYou can check :ref:`cmd-spack-find-metadata` to see how to query for explicitly installed packages or :ref:`dependency-types` for a more thorough treatment of dependency types.\n\n.. _cmd-spack-mark:\n\nMarking packages explicit or implicit\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nBy default, Spack will mark packages a user installs as explicitly installed, while all of its dependencies will be marked as implicitly installed.\nPackages can be marked manually as explicitly or implicitly installed by using ``spack mark``.\nThis can be used in combination with ``spack gc`` to clean up packages that are no longer required.\n\n.. code-block:: spec\n\n $ spack install m4\n ==> 29005: Installing libsigsegv\n [...]\n ==> 29005: Installing m4\n [...]\n\n $ spack install m4 ^libsigsegv@2.11\n ==> 39798: Installing libsigsegv\n [...]\n ==> 39798: Installing m4\n [...]\n\n $ spack find -d\n ==> 4 installed packages\n -- linux-fedora32-haswell / gcc@10.1.1 --------------------------\n libsigsegv@2.11\n\n libsigsegv@2.12\n\n m4@1.4.18\n libsigsegv@2.12\n\n m4@1.4.18\n libsigsegv@2.11\n\n $ spack gc\n ==> There are no unused specs. Spack's store is clean.\n\n $ spack mark -i m4 ^libsigsegv@2.11\n ==> m4@1.4.18 : marking the package implicit\n\n $ spack gc\n ==> The following packages will be uninstalled:\n\n -- linux-fedora32-haswell / gcc@10.1.1 --------------------------\n 5fj7p2o libsigsegv@2.11 c6ensc6 m4@1.4.18\n\n ==> Do you want to proceed? [y/N]\n\nIn the example above, we ended up with two versions of ``m4`` because they depend on different versions of ``libsigsegv``.\n``spack gc`` will not remove any of the packages because both versions of ``m4`` have been installed explicitly and both versions of ``libsigsegv`` are required by the ``m4`` packages.\n\n``spack mark`` can also be used to implement upgrade workflows.\nThe following example demonstrates how ``spack mark`` and ``spack gc`` can be used to only keep the current version of a package installed.\n\nWhen updating Spack via ``git pull``, new versions for either ``libsigsegv`` or ``m4`` might be introduced.\nThis will cause Spack to install duplicates.\nBecause we only want to keep one version, we mark everything as implicitly installed before updating Spack.\nIf there is no new version for either of the packages, ``spack install`` will simply mark them as explicitly installed, and ``spack gc`` will not remove them.\n\n.. code-block:: spec\n\n $ spack install m4\n ==> 62843: Installing libsigsegv\n [...]\n ==> 62843: Installing m4\n [...]\n\n $ spack mark -i -a\n ==> m4@1.4.18 : marking the package implicit\n\n $ git pull\n [...]\n\n $ spack install m4\n [...]\n ==> m4@1.4.18 : marking the package explicit\n [...]\n\n $ spack gc\n ==> There are no unused specs. Spack's store is clean.\n\nWhen using this workflow for installations that contain more packages, care must be taken to either only mark selected packages or issue ``spack install`` for all packages that should be kept.\n\nYou can check :ref:`cmd-spack-find-metadata` to see how to query for explicitly or implicitly installed packages.\n\n.. _nondownloadable:\n\nNon-Downloadable Tarballs\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nThe tarballs for some packages cannot be automatically downloaded by Spack.\nThis could be for a number of reasons:\n\n#. The author requires users to manually accept a license agreement before downloading (e.g., ``jdk`` and ``galahad``).\n\n#. The software is proprietary and cannot be downloaded on the open Internet.\n\nTo install these packages, one must create a mirror and manually add the tarballs in question to it (see :ref:`mirrors`):\n\n#. Create a directory for the mirror.\n You can create this directory anywhere you like, it does not have to be inside ``~/.spack``:\n\n .. code-block:: console\n\n $ mkdir ~/.spack/manual_mirror\n\n#. Register the mirror with Spack by creating ``~/.spack/mirrors.yaml``:\n\n .. code-block:: yaml\n\n mirrors:\n manual: file://~/.spack/manual_mirror\n\n#. Put your tarballs in it.\n Tarballs should be named ``/-.tar.gz``.\n For example:\n\n .. code-block:: console\n\n $ ls -l manual_mirror/galahad\n\n -rw-------. 1 me me 11657206 Jun 21 19:25 galahad-2.60003.tar.gz\n\n#. Install as usual:\n\n .. code-block:: console\n\n $ spack install galahad\n\n\nSeeing Installed Packages\n-------------------------\n\nWe know that ``spack list`` shows you the names of available packages, but how do you figure out which are already installed?\n\n.. _cmd-spack-find:\n\n``spack find``\n^^^^^^^^^^^^^^\n\n``spack find`` shows the *specs* of installed packages.\nA spec is like a name, but it has a version, compiler, architecture, and build options associated with it.\nIn Spack, you can have many installations of the same package with different specs.\n\nRunning ``spack find`` with no arguments lists installed packages:\n\n.. code-block:: spec\n\n $ spack find\n ==> 74 installed packages.\n -- linux-debian7-x86_64 / gcc@4.4.7 --------------------------------\n ImageMagick@6.8.9-10 libdwarf@20130729 py-dateutil@2.4.0\n adept-utils@1.0 libdwarf@20130729 py-ipython@2.3.1\n atk@2.14.0 libelf@0.8.12 py-matplotlib@1.4.2\n boost@1.55.0 libelf@0.8.13 py-nose@1.3.4\n bzip2@1.0.6 libffi@3.1 py-numpy@1.9.1\n cairo@1.14.0 libmng@2.0.2 py-pygments@2.0.1\n callpath@1.0.2 libpng@1.6.16 py-pyparsing@2.0.3\n cmake@3.0.2 libtiff@4.0.3 py-pyside@1.2.2\n dbus@1.8.6 libtool@2.4.2 py-pytz@2014.10\n dbus@1.9.0 libxcb@1.11 py-setuptools@11.3.1\n dyninst@8.1.2 libxml2@2.9.2 py-six@1.9.0\n fontconfig@2.11.1 libxml2@2.9.2 python@2.7.8\n freetype@2.5.3 llvm@3.0 qhull@1.0\n gdk-pixbuf@2.31.2 memaxes@0.5 qt@4.8.6\n glib@2.42.1 mesa@8.0.5 qt@5.4.0\n graphlib@2.0.0 mpich@3.0.4 readline@6.3\n gtkplus@2.24.25 mpileaks@1.0 sqlite@3.8.5\n harfbuzz@0.9.37 mrnet@4.1.0 stat@2.1.0\n hdf5@1.8.13 ncurses@5.9 tcl@8.6.3\n icu@54.1 netcdf@4.3.3 tk@src\n jpeg@9a openssl@1.0.1h vtk@6.1.0\n launchmon@1.0.1 pango@1.36.8 xcb-proto@1.11\n lcms@2.6 pixman@0.32.6 xz@5.2.0\n libdrm@2.4.33 py-dateutil@2.4.0 zlib@1.2.8\n\n -- linux-debian7-x86_64 / gcc@4.9.2 --------------------------------\n libelf@0.8.10 mpich@3.0.4\n\nPackages are divided into groups according to their architecture and compiler.\nWithin each group, Spack tries to keep the view simple and only shows the version of installed packages.\n\n.. _cmd-spack-find-metadata:\n\nViewing more metadata\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\n``spack find`` can filter the package list based on the package name, spec, or a number of properties of their installation status.\nFor example, missing dependencies of a spec can be shown with ``--missing``, deprecated packages can be included with ``--deprecated``, packages that were explicitly installed with ``spack install `` can be singled out with ``--explicit``, and those that have been pulled in only as dependencies with ``--implicit``.\n\nIn some cases, there may be different configurations of the *same* version of a package installed.\nFor example, there are two installations of ``libdwarf@20130729`` above.\nWe can look at them in more detail using ``spack find --deps`` and by asking only to show ``libdwarf`` packages:\n\n.. code-block:: spec\n\n $ spack find --deps libdwarf\n ==> 2 installed packages.\n -- linux-debian7-x86_64 / gcc@4.4.7 --------------------------------\n libdwarf@20130729-d9b90962\n ^libelf@0.8.12\n libdwarf@20130729-b52fac98\n ^libelf@0.8.13\n\nNow we see that the two instances of ``libdwarf`` depend on *different* versions of ``libelf``: 0.8.12 and 0.8.13.\nThis view can become complicated for packages with many dependencies.\nIf you just want to know whether two packages' dependencies differ, you can use ``spack find --long``:\n\n.. code-block:: spec\n\n $ spack find --long libdwarf\n ==> 2 installed packages.\n -- linux-debian7-x86_64 / gcc@4.4.7 --------------------------------\n libdwarf@20130729-d9b90962 libdwarf@20130729-b52fac98\n\nNow the ``libdwarf`` installs have hashes after their names.\nThese are hashes over all of the dependencies of each package.\nIf the hashes are the same, then the packages have the same dependency configuration.\n\nIf you want to know the path where each package is installed, you can use ``spack find --paths``:\n\n.. code-block:: spec\n\n $ spack find --paths\n ==> 74 installed packages.\n -- linux-debian7-x86_64 / gcc@4.4.7 --------------------------------\n ImageMagick@6.8.9-10 ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/ImageMagick@6.8.9-10-4df950dd\n adept-utils@1.0 ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/adept-utils@1.0-5adef8da\n atk@2.14.0 ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/atk@2.14.0-3d09ac09\n boost@1.55.0 ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/boost@1.55.0\n bzip2@1.0.6 ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/bzip2@1.0.6\n cairo@1.14.0 ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/cairo@1.14.0-fcc2ab44\n callpath@1.0.2 ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/callpath@1.0.2-5dce4318\n ...\n\nYou can restrict your search to a particular package by supplying its name:\n\n.. code-block:: spec\n\n $ spack find --paths libelf\n -- linux-debian7-x86_64 / gcc@4.4.7 --------------------------------\n libelf@0.8.11 ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/libelf@0.8.11\n libelf@0.8.12 ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/libelf@0.8.12\n libelf@0.8.13 ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/libelf@0.8.13\n\nSpec queries\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\n``spack find`` actually does a lot more than this.\nYou can use *specs* to query for specific configurations and builds of each package.\nIf you want to find only libelf versions greater than version 0.8.12, you could say:\n\n.. code-block:: spec\n\n $ spack find libelf@0.8.12:\n -- linux-debian7-x86_64 / gcc@4.4.7 --------------------------------\n libelf@0.8.12 libelf@0.8.13\n\nFinding just the versions of ``libdwarf`` built with a particular version of libelf would look like this:\n\n.. code-block:: spec\n\n $ spack find --long libdwarf ^libelf@0.8.12\n ==> 1 installed packages.\n -- linux-debian7-x86_64 / gcc@4.4.7 --------------------------------\n libdwarf@20130729-d9b90962\n\nWe can also search for packages that have a certain attribute.\nFor example, ``spack find libdwarf +debug`` will show only installations of ``libdwarf`` with the 'debug' compile-time option enabled.\n\nThe full spec syntax is discussed in detail in :ref:`sec-specs`.\n\n\nMachine-readable output\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nIf you only want to see very specific things about installed packages, Spack has some options for you.\n``spack find --format`` can be used to output only specific fields:\n\n.. code-block:: console\n\n $ spack find --format \"{name}-{version}-{hash}\"\n autoconf-2.69-icynozk7ti6h4ezzgonqe6jgw5f3ulx4\n automake-1.16.1-o5v3tc77kesgonxjbmeqlwfmb5qzj7zy\n bzip2-1.0.6-syohzw57v2jfag5du2x4bowziw3m5p67\n bzip2-1.0.8-zjny4jwfyvzbx6vii3uuekoxmtu6eyuj\n cmake-3.15.1-7cf6onn52gywnddbmgp7qkil4hdoxpcb\n ...\n\nor:\n\n.. code-block:: console\n\n $ spack find --format \"{hash:7}\"\n icynozk\n o5v3tc7\n syohzw5\n zjny4jw\n 7cf6onn\n ...\n\nThis uses the same syntax as described in the documentation for :meth:`~spack.spec.Spec.format` -- you can use any of the options there.\nThis is useful for passing metadata about packages to other command-line tools.\n\nAlternatively, if you want something even more machine readable, you can output each spec as JSON records using ``spack find --json``.\nThis will output metadata on specs and all dependencies as JSON:\n\n.. code-block:: spec\n\n $ spack find --json sqlite@3.28.0\n [\n {\n \"name\": \"sqlite\",\n \"hash\": \"3ws7bsihwbn44ghf6ep4s6h4y2o6eznv\",\n \"version\": \"3.28.0\",\n \"arch\": {\n \"platform\": \"darwin\",\n \"platform_os\": \"mojave\",\n \"target\": \"x86_64\"\n },\n \"compiler\": {\n \"name\": \"apple-clang\",\n \"version\": \"10.0.0\"\n },\n \"namespace\": \"builtin\",\n \"parameters\": {\n \"fts\": true,\n \"functions\": false,\n \"cflags\": [],\n \"cppflags\": [],\n \"cxxflags\": [],\n \"fflags\": [],\n \"ldflags\": [],\n \"ldlibs\": []\n },\n \"dependencies\": {\n \"readline\": {\n \"hash\": \"722dzmgymxyxd6ovjvh4742kcetkqtfs\",\n \"type\": [\n \"build\",\n \"link\"\n ]\n }\n }\n },\n ...\n ]\n\nYou can use this with tools like `jq `_ to quickly create JSON records structured the way you want:\n\n.. code-block:: console\n\n $ spack find --json sqlite@3.28.0 | jq -C '.[] | { name, version, hash }'\n {\n \"name\": \"sqlite\",\n \"version\": \"3.28.0\",\n \"hash\": \"3ws7bsihwbn44ghf6ep4s6h4y2o6eznv\"\n }\n {\n \"name\": \"readline\",\n \"version\": \"7.0\",\n \"hash\": \"722dzmgymxyxd6ovjvh4742kcetkqtfs\"\n }\n {\n \"name\": \"ncurses\",\n \"version\": \"6.1\",\n \"hash\": \"zvaa4lhlhilypw5quj3akyd3apbq5gap\"\n }\n\n.. _cmd-spack-diff:\n\n``spack diff``\n^^^^^^^^^^^^^^\n\nIt's often the case that you have two versions of a spec that you need to disambiguate.\nLet's say that we've installed two variants of ``zlib``, one with and one without the optimize variant:\n\n.. code-block:: spec\n\n $ spack install zlib\n $ spack install zlib -optimize\n\nWhen we do ``spack find``, we see the two versions.\n\n.. code-block:: spec\n\n $ spack find zlib\n ==> 2 installed packages\n -- linux-ubuntu20.04-skylake / gcc@9.3.0 ------------------------\n zlib@1.2.11 zlib@1.2.11\n\n\nLet's say we want to uninstall ``zlib``.\nWe run the command and quickly encounter a problem because two versions are installed.\n\n.. code-block:: spec\n\n $ spack uninstall zlib\n ==> Error: zlib matches multiple packages:\n\n -- linux-ubuntu20.04-skylake / gcc@9.3.0 ------------------------\n efzjziy zlib@1.2.11 sl7m27m zlib@1.2.11\n\n ==> Error: You can either:\n a) use a more specific spec, or\n b) specify the spec by its hash (e.g. `spack uninstall /hash`), or\n c) use `spack uninstall --all` to uninstall ALL matching specs.\n\nOh no!\nWe can see from the above that we have two different versions of ``zlib`` installed, and the only difference between the two is the hash.\nThis is a good use case for ``spack diff``, which can easily show us the \"diff\" or set difference between properties for two packages.\nLet's try it out.\nBecause the only difference we see in the ``spack find`` view is the hash, let's use ``spack diff`` to look for more detail.\nWe will provide the two hashes:\n\n.. code-block:: diff\n\n $ spack diff /efzjziy /sl7m27m\n\n --- zlib@1.2.11efzjziyc3dmb5h5u5azsthgbgog5mj7g\n +++ zlib@1.2.11sl7m27mzkbejtkrajigj3a3m37ygv4u2\n @@ variant_value @@\n - zlib optimize False\n + zlib optimize True\n\n\nThe output is colored and written in the style of a git diff.\nThis means that you can copy and paste it into a GitHub markdown as a code block with language \"diff\" and it will render nicely!\nHere is an example:\n\n.. code-block:: diff\n\n --- zlib@1.2.11/efzjziyc3dmb5h5u5azsthgbgog5mj7g\n +++ zlib@1.2.11/sl7m27mzkbejtkrajigj3a3m37ygv4u2\n @@ variant_value @@\n - zlib optimize False\n + zlib optimize True\n\nAwesome!\nNow let's read the diff.\nIt tells us that our first ``zlib`` was built with ``~optimize`` (``False``) and the second was built with ``+optimize`` (``True``).\nYou can't see it in the docs here, but the output above is also colored based on the content being an addition (+) or subtraction (-).\n\nThis is a small example, but you will be able to see differences for any attributes on the installation spec.\nRunning ``spack diff A B`` means we'll see which spec attributes are on ``B`` but not on ``A`` (green) and which are on ``A`` but not on ``B`` (red).\nHere is another example with an additional difference type, ``version``:\n\n.. code-block:: diff\n\n $ spack diff python@2.7.8 python@3.8.11\n\n --- python@2.7.8/tsxdi6gl4lihp25qrm4d6nys3nypufbf\n +++ python@3.8.11/yjtseru4nbpllbaxb46q7wfkyxbuvzxx\n @@ variant_value @@\n - python patches a8c52415a8b03c0e5f28b5d52ae498f7a7e602007db2b9554df28cd5685839b8\n + python patches 0d98e93189bc278fbc37a50ed7f183bd8aaf249a8e1670a465f0db6bb4f8cf87\n @@ version @@\n - openssl 1.0.2u\n + openssl 1.1.1k\n - python 2.7.8\n + python 3.8.11\n\nLet's say that we were only interested in one kind of attribute above, ``version``.\nWe can ask the command to only output this attribute.\nTo do this, you'd add the ``--attribute`` for attribute parameter, which defaults to all.\nHere is how you would filter to show just versions:\n\n.. code-block:: diff\n\n $ spack diff --attribute version python@2.7.8 python@3.8.11\n\n --- python@2.7.8/tsxdi6gl4lihp25qrm4d6nys3nypufbf\n +++ python@3.8.11/yjtseru4nbpllbaxb46q7wfkyxbuvzxx\n @@ version @@\n - openssl 1.0.2u\n + openssl 1.1.1k\n - python 2.7.8\n + python 3.8.11\n\nAnd you can add as many attributes as you'd like with multiple ``--attribute`` arguments (for lots of attributes, you can use ``-a`` for short).\nFinally, if you want to view the data as JSON (and possibly pipe into an output file), just add ``--json``:\n\n\n.. code-block:: spec\n\n $ spack diff --json python@2.7.8 python@3.8.11\n\n\nThis data will be much longer because along with the differences for ``A`` vs.\n``B`` and ``B`` vs.\n``A``, the JSON output also shows the intersection.\n\n\nUsing Installed Packages\n------------------------\n\nAs you've seen, Spack packages are installed into long paths with hashes, and you need a way to get them into your path.\nSpack has three different ways to solve this problem, which fit different use cases:\n\n1. Spack provides :ref:`environments `, and views, with which you can \"activate\" a number of related packages all at once.\n This is likely the best method for most use cases.\n2. Spack can generate :ref:`environment modules `, which are commonly used on supercomputing clusters.\n Module files can be generated for every installation automatically, and you can customize how this is done.\n3. For one-off use, Spack provides the :ref:`spack load ` command\n\n\n.. _cmd-spack-load:\n.. _cmd-spack-unload:\n\n``spack load / unload``\n^^^^^^^^^^^^^^^^^^^^^^^\n\nIf you sourced the appropriate shell script, as shown in :ref:`getting_started`, you can use the ``spack load`` command to quickly add a package to your ``PATH``.\n\nFor example, this will add the ``mpich`` package built with ``gcc`` to your path:\n\n.. code-block:: spec\n\n $ spack install mpich %gcc@4.4.7\n\n # ... wait for install ...\n\n $ spack load mpich %gcc@4.4.7\n $ which mpicc\n ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/mpich@3.0.4/bin/mpicc\n\nThese commands will add appropriate directories to your ``PATH`` and ``MANPATH`` according to the :ref:`prefix inspections ` defined in your modules configuration.\nWhen you no longer want to use a package, you can type ``spack unload``:\n\n.. code-block:: spec\n\n $ spack unload mpich %gcc@4.4.7\n\n\nAmbiguous specs\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nIf a spec used with load/unload is ambiguous (i.e., more than one installed package matches it), then Spack will warn you:\n\n.. code-block:: spec\n\n $ spack load libelf\n ==> Error: libelf matches multiple packages.\n Matching packages:\n qmm4kso libelf@0.8.13%gcc@4.4.7 arch=linux-debian7-x86_64\n cd2u6jt libelf@0.8.13%intel@15.0.0 arch=linux-debian7-x86_64\n Use a more specific spec\n\nYou can either type the ``spack load`` command again with a fully qualified argument, or you can add just enough extra constraints to identify one package.\nFor example, above, the key differentiator is that one ``libelf`` is built with the Intel compiler, while the other used ``gcc``.\nYou could therefore just type:\n\n.. code-block:: spec\n\n $ spack load libelf %intel\n\nTo identify just the one built with the Intel compiler.\nIf you want to be *very* specific, you can load it by its hash.\nFor example, to load the first ``libelf`` above, you would run:\n\n.. code-block:: spec\n\n $ spack load /qmm4kso\n\nTo see which packages that you have loaded into your environment, you would use ``spack find --loaded``.\n\n.. code-block:: console\n\n $ spack find --loaded\n ==> 2 installed packages\n -- linux-debian7 / gcc@4.4.7 ------------------------------------\n libelf@0.8.13\n\n -- linux-debian7 / intel@15.0.0 ---------------------------------\n libelf@0.8.13\n\nYou can also use ``spack load --list`` to get the same output, but it does not have the full set of query options that ``spack find`` offers.\n\nWe'll learn more about Spack's spec syntax in :ref:`a later section `.\n\n.. _extensions:\n\nSpack environments\n^^^^^^^^^^^^^^^^^^\n\nSpack can install a large number of Python packages.\nTheir names are typically prefixed with ``py-``.\nInstalling and using them is no different from any other package:\n\n.. code-block:: spec\n\n $ spack install py-numpy\n $ spack load py-numpy\n $ python3\n >>> import numpy\n\nThe ``spack load`` command sets the ``PATH`` variable so that the correct Python executable is used and makes sure that ``numpy`` and its dependencies can be located in the ``PYTHONPATH``.\n\nSpack is different from other Python package managers in that it installs every package into its *own* prefix.\nThis is in contrast to ``pip``, which installs all packages into the same prefix, whether in a virtual environment or not.\n\nFor many users, **virtual environments** are more convenient than repeated ``spack load`` commands, particularly when working with multiple Python packages.\nFortunately, Spack supports environments itself, which together with a view are no different from Python virtual environments.\n\nThe recommended way of working with Python extensions such as ``py-numpy`` is through :ref:`Environments `.\nThe following example creates a Spack environment with ``numpy`` in the current working directory.\nIt also puts a filesystem view in ``./view``, which is a more traditional combined prefix for all packages in the environment.\n\n.. code-block:: spec\n\n $ spack env create --with-view view --dir .\n $ spack -e . add py-numpy\n $ spack -e . concretize\n $ spack -e . install\n\nNow you can activate the environment and start using the packages:\n\n.. code-block:: console\n\n $ spack env activate .\n $ python3\n >>> import numpy\n\nThe environment view is also a virtual environment, which is useful if you are sharing the environment with others who are unfamiliar with Spack.\nThey can either use the Python executable directly:\n\n.. code-block:: console\n\n $ ./view/bin/python3\n >>> import numpy\n\nor use the activation script:\n\n.. code-block:: console\n\n $ source ./view/bin/activate\n $ python3\n >>> import numpy\n\nIn general, there should not be much difference between ``spack env activate`` and using the virtual environment.\nThe main advantage of ``spack env activate`` is that it knows about more packages than just Python packages, and it may set additional runtime variables that are not covered by the virtual environment activation script.\n\nSee :ref:`environments` for a more in-depth description of Spack environments and customizations to views.\n" }, { "path": "lib/spack/docs/package_review_guide.rst", "content": ".. Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n This is a guide for people who review package pull requests and includes criteria for them to be merged into the develop branch.\n\n.. _package-review-guide:\n\nPackage Review Guide\n====================\n\nPackage reviews are performed with the goals of minimizing build errors and making packages as **uniform and stable** as possible.\n\nThis section establishes guidelines to help assess Spack community `package repository `_ pull requests (PRs).\nIt describes the considerations and actions to be taken when reviewing new and updated `Spack packages `_.\nIn some cases, there are possible solutions to common issues.\n\nHow to use this guide\n---------------------\n\nWhether you are a :ref:`Package Reviewer `, :ref:`Maintainer `, or :ref:`Committer `, this guide highlights relevant aspects to consider when reviewing package pull requests.\nIf you are a :ref:`Package Contributor ` (or simply ``Contributor``), you may also find the information and solutions useful in your work.\nWhile we provide information on what to look for, the changes themselves should drive the actual review process.\n\n.. note::\n\n :ref:`Confirmation of successful package builds ` of **all** affected versions can reduce the amount of effort needed to review a PR.\n However, packaging conventions and the combinatorial nature of versions and directives mean each change should still be checked.\n\nReviewing a new package\n~~~~~~~~~~~~~~~~~~~~~~~\n\nIf the pull request includes a new package, then focus on answering the following questions:\n\n* Should the :ref:`package ` be added to the repository?\n* Does the package :ref:`structure ` conform to conventions?\n* Are the directives and their options correct?\n* Do all :ref:`automated checks ` pass?\n If not, are there easy-to-resolve CI and/or test issues that can be addressed or does the submitter need to investigate the failures?\n* Is there :ref:`confirmation ` that every version builds successfully on at least one platform?\n\nRefer to the relevant sections below for more guidance.\n\nReviewing changes to an existing package\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nIf the pull request includes changes to an existing package, then focus on answering the following questions:\n\n* Are there any changes to the package :ref:`structure ` and, if so, do they conform to conventions?\n* If there are new or updated directives, then are they and their options correct?\n* If there are changes to the :ref:`url or its equivalent `, are the older versions still correct?\n* If there are changes to the :ref:`git or equivalent URL `, do older branches exist in the new location?\n* Do all :ref:`automated checks ` pass?\n If not, are there easy-to-resolve CI and/or test issues that can be addressed or does the submitter need to investigate the failures?\n* Is there :ref:`confirmation ` that every new version builds successfully on at least one platform?\n\nRefer to the relevant sections below for more guidance.\n\n.. _suitable_package:\n\nPackage suitability\n-------------------\n\nIt is rare that a package would be considered inappropriate for inclusion in the public `Spack package `_ repository.\nOne exception is making packages for standard Perl modules.\n\n**Action.**\nShould you find the software is not appropriate, ask that the package be removed from the PR if it is one of multiple affected files or suggest the PR be closed.\nIn either case, explain the reason for the request.\n\nCORE Perl modules\n~~~~~~~~~~~~~~~~~\n\nIn general, modules that are part of the standard installation for all listed Perl versions (i.e., ``CORE``) should **not be implemented or contributed**.\nDetails on the exceptions and process for checking Perl modules can be found in the :ref:`Perl build system ` documentation.\n\n.. _structure_reviews:\n\nPackage structure\n-----------------\n\nThe `convention `_ for structuring Spack packages has metadata (key properties) listed first followed by directives then methods:\n\n* :ref:`url_equivalent_reviews`;\n* :ref:`vcs_url_reviews`;\n* :ref:`maintainers_reviews`;\n* :ref:`license_reviews`;\n* :ref:`version_reviews`;\n* :ref:`variant_reviews`;\n* :ref:`depends_on_reviews`;\n* :ref:`packaging_conflicts` and :ref:`packaging_requires` directives; then\n* methods.\n\n`Groupings `_ using ``with`` context managers can affect the order of dependency, conflict, and requires directives to some degree.\nHowever, they do cut down on visual clutter and make packages more readable.\n\n**Action.**\nIf you see clear deviations from the convention, request that they be addressed.\nWhen in doubt, ask others with merge privileges for advice.\n\n.. _url_equivalent_reviews:\n\n``url``, ``url_for_version``, or URL equivalent\n-----------------------------------------------\n\nChanges to URLs may invalidate existing versions, which should be checked when there is a URL-related modification.\nAll packages have a URL, though for some :ref:`build-systems` it is derived automatically and not visible in the package.\n\nReasons :ref:`versions ` may become invalid include:\n\n* the new URL does not support Spack version extrapolation;\n* the addition of or changes to ``url_for_version`` involve checks of the ``spec``'s version instead of the ``version`` argument or the (usually older) versions are not covered;\n* extrapolation of the derived URL no longer matches that of older versions; and\n* the older versions are no longer available.\n\n**Action.**\nChecking existing version directives with checksums can usually be done manually with the modified package using `spack checksum `_.\n\n**Solutions.**\nOptions for resolving the problem that can be suggested for investigation depend on the source.\n\nIn simpler cases involving ``url`` or ``url_for_version``, invalid versions can sometimes be corrected by ensuring all versions are covered by ``url_for_version``.\nAlternatively, especially for older versions, the version-specific URL can be added as an argument to the ``version`` directive.\n\nSometimes the derived URLs of versions on the hosting system can vary.\nThis commonly happens with Python packages.\nFor example, the case of one or more letters in the package name may change at some point (e.g., `py-sphinx `_).\nAlso, dashes may be replaced with underscores (e.g., `py-scikit-build `_).\nIn some cases, both changes can occur for the same package.\nAs these examples illustrate, it is sometimes possible to add a ``url_for_version`` method to override the default derived URL to ensure the correct one is returned.\n\nIf older versions are no longer available and there is a chance someone has the package in a build cache, the usual approach is to first suggest :ref:`deprecating ` them in the package.\n\n.. _vcs_url_reviews:\n\n``git``, ``hg``, ``svn``, or ``cvs``\n------------------------------------\n\nIf the :ref:`repository-specific URL ` for fetching branches or the version control system (VCS) equivalent changes, there is a risk that the listed versions are no longer accessible.\n\n**Action.**\nYou may need to check the new source repository to confirm the presence of all of the listed versions.\n\n.. _maintainers_reviews:\n\n``maintainers`` directive\n-------------------------\n\n**Action.**\nIf the new package does not have a :ref:`maintainers ` directive, ask the Contributor to add one.\n\n.. note::\n\n This request is optional for existing packages.\n\n Be prepared for them to refuse.\n\n.. _license_reviews:\n\n``license`` directive\n---------------------\n\n**Action.**\nIf the new package does not have a :ref:`license ` directive, ask the Contributor to investigate and add it.\n\n.. note::\n\n This request is optional for existing packages.\n\n Be prepared for them to refuse.\n\n.. _version_reviews:\n\n``version`` directives\n----------------------\n\nIn general, Spack packages are expected to be built from source code.\nThere are a few exceptions (e.g., :ref:`BundlePackage `).\nTypically every package will have at least one :ref:`version ` directive.\n\nThe goals of reviewing version directives are to confirm that versions are listed in the proper order **and** that the arguments for new and updated versions are correct.\n\n.. note::\n\n Additions and removals of version directives should generally trigger a review of :ref:`dependencies `.\n\n``version`` directive order\n~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nBy :ref:`convention ` version directives should be listed in descending order, from newest to oldest.\nIf branch versions are included, then they should be listed first.\n\n**Action.**\nWhen versions are being added, check the ordering of the directives.\nRequest that the directives be re-ordered if any of the directives do not conform to the convention.\n\n.. note::\n\n Edge cases, such as manually downloaded software, may be difficult to confirm.\n\nChecksums, commits, tags, and branches\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n**Checksums, commits, and tags**\n Normally these version arguments are automatically validated by GitHub Actions using `spack ci verify-versions `_.\n\n **Action.**\n Review the PR's ``verify-checksums`` precheck to confirm.\n If necessary, checksums can usually be manually confirmed using `spack checksum `_.\n\n .. warning::\n\n From a security and reproducibility standpoint, it is important that Spack be able to verify downloaded source.\n This is accomplished using a hash (e.g., checksum or commit).\n See :ref:`checksum verification ` for more information.\n\n Exceptions are allowed in rare cases, such as software supplied from reputable vendors.\n When in doubt, ask others with merge privileges for advice.\n\n**Tags**\n If a ``tag`` is provided without a ``commit``, the downloaded software will not be trusted.\n\n **Action.**\n Suggest that the ``commit`` argument be included in the ``version`` directive.\n\n**Branches**\n Confirming new branch versions involves checking that the branches exist in the repository *and* that the version and branch names are consistent.\n Let's take each in turn.\n\n **Action.**\n Confirming branch existence often involves checking the source repository though is not necessary if there is confirmation that the branch was built successfully from the package.\n\n In general, the version and branch names should match.\n When they do not, it is sometimes the result of people not being aware of how Spack handles :ref:`version-comparison`.\n\n **Action.**\n If there is a name mismatch, especially for the most common branch names (e.g., ``develop``, ``main``, and ``master``), ask why and suggest the arguments be changed such that they match the actual branch name.\n\n**Manual downloads**\n\n **Action.**\n Since these can be difficult to confirm, it is acceptable to rely on the package's Maintainers, if any.\n\nDeprecating versions\n~~~~~~~~~~~~~~~~~~~~\n\nIf someone is deprecating versions, it is good to find out why.\nSometimes there are concerns, such as security or lack of availability.\n\n**Action.**\nSuggest the Contributor review the :ref:`deprecation guidelines ` before finalizing the changes if they haven't already explained why they made the choice in the PR description or comments.\n\n.. _variant_reviews:\n\n``variant`` directives\n----------------------\n\n:ref:`Variants ` represent build options so any changes involving these directives should be reflected elsewhere in the package.\n\nAdding or modifying variants\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n**Action.**\nConfirm that new or modified variants are actually used in the package.\nThe most common uses are additions and changes to:\n\n* :ref:`dependencies `;\n* configure options; and/or\n* build arguments.\n\nRemoving or disabling variants\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nIf the variant is still relevant to listed version directives, it may be preferable to adjust or add `conditions `_.\n\n**Action.**\nConsider asking why the variant (or build option) is being removed and suggest making it conditional when it is still relevant.\n\n.. warning::\n\n If the default value of a variant is changed in the PR, then there is a risk that other packages relying on that value will no longer build as others expect.\n This may be worth noting in the review.\n\n.. _depends_on_reviews:\n\n``depends_on`` directives\n-------------------------\n\n:ref:`Dependencies ` represent software that must be installed before the package builds or is able to work correctly.\n\nUpdating dependencies\n~~~~~~~~~~~~~~~~~~~~~\n\nIt is important that dependencies reflect the requirements of listed versions.\nThey only need to be checked in a review when versions are being added or removed or the dependencies are being changed.\n\n**Action.**\nDependencies affected by such changes should be confirmed, when possible, and *at least* when the Contributor is not a Maintainer of the package.\n\n**Solutions.**\nIn some cases, the needed change may be as simple as ensuring the version ranges (see :ref:`version_compatibility`) and/or variant options in the dependency are accurate.\nIn others, one or more of the dependencies needed by new versions are missing and need to be added.\nOr there may be dependencies that are no longer relevant when versions requiring them are removed, meaning the dependencies should be removed as well.\n\nFor example, it is not uncommon for Python package dependencies to be out of date when new versions are added.\nIn this case, check Python package dependencies by following the build system `guidelines `_.\n\n.. tip::\n\n In general, refer to the relevant dependencies section, if any, for the package’s :ref:`build-systems` for guidance.\n\nUpdating language and compiler dependencies\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nWhen :ref:`language and compiler dependencies ` were introduced, their ``depends_on`` directives were derived from the source for existing packages.\nThese dependencies are flagged with ``# generated`` comments when they have not been confirmed.\nUnfortunately, the generated dependencies are not always complete or necessarily required.\n\n**Action.**\nIf these dependencies are being updated, ask that the ``# generated`` comments be removed if the Contributor can confirm they are relevant.\nDefinitely make sure Contributors do **not** include ``# generated`` on the dependencies they are adding to the package.\n\n.. _automated_checks_reviews:\n\nFailed automated checks\n-----------------------\n\nAll PRs are expected to pass **at least the required** automated checks.\n\nStyle failures\n~~~~~~~~~~~~~~\n\nThe PR may fail one or more style checks.\n\n**Action.**\nIf the failure is due to issues raised by the ``black`` style checker *and* the PR is otherwise ready to be merged, you can add ``@spackbot fix style`` in a comment to see if Spack will fix the errors.\nOtherwise, inform the Contributor that the style failures need to be addressed.\n\nCI stack failures\n~~~~~~~~~~~~~~~~~\n\nExisting packages **may** be included in GitLab CI pipelines through inclusion in one or more `stacks `_.\n\n**Action.**\nIt is worth checking **at least a sampling** of the failed job logs, if present, to determine the possible cause and take or suggest an action accordingly.\n\n**CI Runner Failures**\n Sometimes CI runners time out or the pods become unavailable.\n\n **Action.**\n If that is the case, the resolution may be as simple as restarting the pipeline by adding a ``@spackbot run pipeline`` comment.\n Otherwise, the Contributor will need to investigate and resolve the problem.\n\n**Stand-alone Test Failures**\n Sometimes :ref:`stand-alone tests ` could be causing the build job to time out.\n If the tests take too long, the issue could be that the package is running too many and/or long running tests.\n Or the tests may be trying to use resources (e.g., a batch scheduler) that are not available on runners.\n\n **Action.**\n If the tests for a package are hanging, at a minimum create a `new issue `_ if there is not one already, to flag the package.\n\n **(Temporary) Solution.**\n Look at the package implementation to see if the tests are using a batch scheduler or there appear to be too many or long running tests.\n If that is the case, then a pull request should be created in the ``spack/spack-packages`` repository that adds the package to the ``broken-tests-packages`` list in the `ci configuration `_.\n Once the fix PR is merged, then the affected PR can be rebased to pick up the change.\n\n.. _build_success_reviews:\n\nSuccessful builds\n-----------------\n\nIs there evidence that the package builds successfully on at least one platform?\nFor a new package, we would ideally have confirmation for every version; whereas, we would want confirmation of only the affected versions for changes to an existing package.\n\nAcceptable forms of confirmation are **one or more of**:\n\n* the Contributor or another reviewer explicitly confirms that a successful build of **each new version on at least one platform**;\n* the software is built successfully by Spack CI by **at least one of the CI stacks**; and\n* **at least one Maintainer** explicitly confirms they are able to successfully build the software.\n\nIndividuals are expected to update the PR description or add a comment to explicitly confirm the builds.\nYou may need to check the CI stacks and/or outputs to confirm that there is a stack that builds the new version.\n\n.. note::\n\n When builds are confirmed by individuals, we would prefer the output of ``spack debug report`` be included in either the PR description or a comment.\n" }, { "path": "lib/spack/docs/packages_yaml.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n A guide to customizing package settings in Spack using the packages.yaml file, including configuring compilers, specifying external packages, package requirements, and permissions.\n\n.. _packages-config:\n\nPackage Settings (packages.yaml)\n================================\n\nSpack allows you to customize how your software is built through the ``packages.yaml`` file.\nUsing it, you can make Spack prefer particular implementations of virtual dependencies (e.g., MPI or BLAS/LAPACK), or you can make it prefer to build with particular compilers.\nYou can also tell Spack to use *external* software installations already present on your system.\n\nAt a high level, the ``packages.yaml`` file is structured like this:\n\n.. code-block:: yaml\n\n packages:\n package1:\n # settings for package1\n package2:\n # settings for package2\n # ...\n all:\n # settings that apply to all packages.\n\nYou can either set build preferences specifically for *one* package, or you can specify that certain settings should apply to *all* packages.\nThe types of settings you can customize are described in detail below.\n\nSpack's build defaults are in the default ``etc/spack/defaults/packages.yaml`` file.\nYou can override them in ``~/.spack/packages.yaml`` or ``etc/spack/packages.yaml``.\nFor more details on how this works, see :ref:`configuration-scopes`.\n\n.. _sec-external-packages:\n\nExternal packages\n-----------------\n\nSpack can be configured to use externally-installed packages rather than building its own packages.\nThis may be desirable if machines ship with system packages, such as a customized MPI, which should be used instead of Spack building its own MPI.\n\nExternal packages are configured through the ``packages.yaml`` file.\nHere's an example of an external configuration:\n\n.. code-block:: yaml\n\n packages:\n openmpi:\n externals:\n - spec: \"openmpi@1.4.3~debug\"\n prefix: /opt/openmpi-1.4.3\n - spec: \"openmpi@1.4.3+debug\"\n prefix: /opt/openmpi-1.4.3-debug\n\nThis example lists two installations of OpenMPI, one with debug information, and one without.\nIf Spack is asked to build a package that uses one of these MPIs as a dependency, it will use the pre-installed OpenMPI in the given directory.\nNote that the specified path is the top-level install prefix, not the ``bin`` subdirectory.\n\n``packages.yaml`` can also be used to specify modules to load instead of the installation prefixes.\nThe following example says that module ``CMake/3.7.2`` provides CMake version 3.7.2.\n\n.. code-block:: yaml\n\n cmake:\n externals:\n - spec: cmake@3.7.2\n modules:\n - CMake/3.7.2\n\nEach ``packages.yaml`` begins with a ``packages:`` attribute, followed by a list of package names.\nTo specify externals, add an ``externals:`` attribute under the package name, which lists externals.\nEach external should specify a ``spec:`` string that should be as well-defined as reasonably possible.\nIf a package lacks a spec component, such as missing a compiler or package version, then Spack will guess the missing component based on its most-favored packages, and it may guess incorrectly.\n\n\n.. _cmd-spack-external-find:\n\nAutomatically find external packages\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nYou can run the :ref:`spack external find ` command to search for system-provided packages and add them to ``packages.yaml``.\nAfter running this command your ``packages.yaml`` may include new entries:\n\n.. code-block:: yaml\n\n packages:\n cmake:\n externals:\n - spec: cmake@3.17.2\n prefix: /usr\n\nGenerally this is useful for detecting a small set of commonly-used packages; for now this is generally limited to finding build-only dependencies.\nSpecific limitations include:\n\n* Packages are not discoverable by default: For a package to be discoverable with ``spack external find``, it needs to add special logic.\n See :ref:`here ` for more details.\n* The logic does not search through module files, it can only detect packages with executables defined in ``PATH``; you can help Spack locate externals which use module files by loading any associated modules for packages that you want Spack to know about before running ``spack external find``.\n* Spack does not overwrite existing entries in the package configuration: If there is an external defined for a spec at any configuration scope, then Spack will not add a new external entry (``spack config blame packages`` can help locate all external entries).\n\nPrevent packages from being built from sources\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nAdding an external spec in ``packages.yaml`` allows Spack to use an external location, but it does not prevent Spack from building packages from sources.\nIn the above example, Spack might choose for many valid reasons to start building and linking with the latest version of OpenMPI rather than continue using the pre-installed OpenMPI versions.\n\nTo prevent this, the ``packages.yaml`` configuration also allows packages to be flagged as non-buildable.\nThe previous example could be modified to be:\n\n.. code-block:: yaml\n\n packages:\n openmpi:\n externals:\n - spec: \"openmpi@1.4.3~debug\"\n prefix: /opt/openmpi-1.4.3\n - spec: \"openmpi@1.4.3+debug\"\n prefix: /opt/openmpi-1.4.3-debug\n buildable: false\n\nThe addition of the ``buildable`` flag tells Spack that it should never build its own version of OpenMPI from sources, and it will instead always rely on a pre-built OpenMPI.\n\n.. note::\n\n If ``concretizer:reuse`` is on (see :ref:`concretizer-options` for more information on that flag) pre-built specs are taken from: the local store, an upstream store, a registered buildcache and externals in ``packages.yaml``.\n If ``concretizer:reuse`` is off, only external specs in ``packages.yaml`` are included in the list of pre-built specs.\n\nIf an external module is specified as not buildable, then Spack will load the external module into the build environment which can be used for linking.\n\nThe ``buildable`` attribute does not need to be paired with external packages.\nIt could also be used alone to forbid packages that may be buggy or otherwise undesirable.\n\nNon-buildable virtual packages\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nVirtual packages in Spack can also be specified as not buildable, and external implementations can be provided.\nIn the example above, OpenMPI is configured as not buildable, but Spack will often prefer other MPI implementations over the externally available OpenMPI.\nSpack can be configured with every MPI provider not buildable individually, but more conveniently:\n\n.. code-block:: yaml\n\n packages:\n mpi:\n buildable: false\n openmpi:\n externals:\n - spec: \"openmpi@1.4.3~debug\"\n prefix: /opt/openmpi-1.4.3\n - spec: \"openmpi@1.4.3+debug\"\n prefix: /opt/openmpi-1.4.3-debug\n\nSpack can then use any of the listed external implementations of MPI to satisfy a dependency, and will choose among them depending on the compiler and architecture.\n\nIn cases where the concretizer is configured to reuse specs, and other ``mpi`` providers (available via stores or build caches) are not desirable, Spack can be configured to require specs matching only the available externals:\n\n.. code-block:: yaml\n\n packages:\n mpi:\n buildable: false\n require:\n - one_of:\n - \"openmpi@1.4.3~debug\"\n - \"openmpi@1.4.3+debug\"\n openmpi:\n externals:\n - spec: \"openmpi@1.4.3~debug\"\n prefix: /opt/openmpi-1.4.3\n - spec: \"openmpi@1.4.3+debug\"\n prefix: /opt/openmpi-1.4.3-debug\n\nThis configuration prevents any spec using MPI and originating from stores or build caches to be reused, unless it matches the requirements under ``packages:mpi:require``.\nFor more information on requirements see :ref:`package-requirements`.\n\nSpecifying dependencies among external packages\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nExternal packages frequently have dependencies on other software components.\nExplicitly modeling these relationships provides Spack with a more complete representation of the software stack.\nThis ensures that:\n\n- Runtime environments include all necessary components.\n- Build-time dependencies are accurately represented.\n\nThis comprehensive view, in turn, enables Spack to more reliably build software that depends on these externals.\n\nSpack provides two methods for configuring dependency relationships among externals, each offering different trade-offs between conciseness and explicit control:\n\n- An \"inline\" spec syntax.\n- A structured YAML configuration that is more verbose but also more explicit.\n\nThe following sections will detail both approaches.\n\nDependencies using inline spec syntax\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nSpack allows you to define external package dependencies using the standard spec syntax directly within your package configuration.\nThis approach is concise and leverages the familiar spec syntax that you already use elsewhere in Spack.\n\nWhen configuring an external package with dependencies using the spec syntax, you can include dependency specifications directly in the main ``spec:`` field:\n\n.. code-block:: yaml\n\n # Specification for the following DAG:\n #\n # o mpileaks@2.3\n # |\\\n # | o callpath@1.0\n # |/\n # o mpich@3.0.4\n packages:\n mpileaks:\n externals:\n - spec: \"mpileaks@2.3~debug+opt %mpich@3 %callpath\"\n prefix: /user/path\n callpath:\n externals:\n - spec: \"callpath@1.0 %mpi=mpich\"\n prefix: /user/path\n mpich:\n externals:\n - spec: \"mpich@3.0.4\"\n prefix: /user/path\n\nIn this example ``mpileaks`` depends on both ``mpich`` and ``callpath``.\nSpack will parse the ``mpileaks`` spec string, and create the appropriate dependency relationships automatically.\n\nUsers *need* to ensure that each dependency maps exactly to a single other external package.\nIn case multiple externals can satisfy the same dependency, or in case no external can satisfy a dependency, Spack will error and point to the configuration line causing the issue.\n\nWhenever no information is given about the dependency type, Spack will infer it from the current package recipe.\nFor instance, the dependencies in the configuration above are inferred to be of ``build,link`` type from the recipe of ``mpileaks`` and ``callpath``:\n\n.. code-block:: console\n\n $ spack -m spec --types -l --cover edges mpileaks\n [e] oelprl6 [ ] mpileaks@2.3~debug+opt+shared+static build_system=generic platform=linux os=ubuntu20.04 target=icelake\n [e] jdhzy2t [bl ] ^callpath@1.0 build_system=generic platform=linux os=ubuntu20.04 target=icelake\n [e] pgem3yp [bl ] ^mpich@3.0.4~debug build_system=generic platform=linux os=ubuntu20.04 target=icelake\n [e] pgem3yp [bl ] ^mpich@3.0.4~debug build_system=generic platform=linux os=ubuntu20.04 target=icelake\n\nWhen inferring the dependency types, Spack will also infer virtuals if they are not already specified.\n\nThis method's conciseness comes with a strict requirement: each dependency must resolve to a single, unambiguous external package.\nThis makes the approach suitable for simple or temporary configurations.\nIn larger, more dynamic environments, however, it can become a maintenance challenge, as adding new external packages over time may require frequent updates to existing specs to preserve their uniqueness.\n\nDependencies using YAML configuration\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nWhile the spec syntax offers a concise way to specify dependencies, Spack's YAML-based explicit dependency configuration provides more control and clarity, especially for complex dependency relationships.\nThis approach uses the ``dependencies:`` field to precisely define each dependency relationship.\nThe example in the previous section, written using the YAML configuration, becomes:\n\n.. code-block:: yaml\n\n # Specification for the following DAG:\n #\n # o mpileaks@2.3\n # |\\\n # | o callpath@1.0\n # |/\n # o mpich@3.0.4\n packages:\n mpileaks:\n externals:\n - spec: \"mpileaks@2.3~debug+opt\"\n prefix: /user/path\n dependencies:\n - id: callpath_id\n deptypes: link\n - spec: mpich\n deptypes:\n - \"build\"\n - \"link\"\n virtuals: \"mpi\"\n callpath:\n externals:\n - spec: \"callpath@1.0\"\n prefix: /user/path\n id: callpath_id\n dependencies:\n - spec: mpich\n deptypes:\n - \"build\"\n - \"link\"\n virtuals: \"mpi\"\n mpich:\n externals:\n - spec: \"mpich@3.0.4\"\n prefix: /user/path\n\nEach dependency can be specified either by:\n\n- A ``spec:`` that matches an available external package, like in the previous case, or by\n- An ``id`` that explicitly references another external package.\n\nUsing the ``id`` provides an unambiguous reference to a specific external package, which is essential for differentiating between externals that have similar specs but differ, for example, only by their installation prefix.\n\nThe dependency types can be specified in the optional ``deptypes`` field, while virtuals can be specified in the optional ``virtuals`` field.\nAs before, when the dependency types are not specified, Spack will infer them from the package recipe.\n\n.. _extra-attributes-for-externals:\n\nExtra attributes for external packages\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nSometimes external packages require additional attributes to be used effectively.\nThis information can be defined on a per-package basis and stored in the ``extra_attributes`` section of the external package configuration.\nIn addition to per-package information, this section can be used to define environment modifications to be performed whenever the package is used.\nFor example, if an external package is built without ``rpath`` support, it may require ``LD_LIBRARY_PATH`` settings to find its dependencies.\nThis could be configured as follows:\n\n.. code-block:: yaml\n\n packages:\n mpich:\n externals:\n - spec: mpich@3.3 +hwloc\n prefix: /path/to/mpich\n extra_attributes:\n environment:\n prepend_path:\n LD_LIBRARY_PATH: /path/to/hwloc/lib64\n\nSee :ref:`configuration_environment_variables` for more information on how to configure environment modifications in Spack config files.\n\n.. _configuring-system-compilers-as-external-packages:\n\nConfiguring system compilers as external packages\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nIn Spack, compilers are treated as packages like any other.\nThis means that you can also configure system compilers as external packages and use them in Spack.\n\nSpack automatically detects system compilers and configures them in ``packages.yaml`` for you.\nYou can also run :ref:`spack-compiler-find` to find and configure new system compilers.\n\nWhen configuring compilers as external packages, you need to set a few :ref:`extra attributes ` for them to work properly.\nThe ``compilers`` extra attribute field is required to clarify which paths within the compiler prefix are used for which languages:\n\n.. code-block:: yaml\n\n packages:\n gcc:\n externals:\n - spec: gcc@10.5.0 languages='c,c++,fortran'\n prefix: /usr\n extra_attributes:\n compilers:\n c: /usr/bin/gcc-10\n cxx: /usr/bin/g++-10\n fortran: /usr/bin/gfortran-10\n\nOther fields accepted by compilers under ``extra_attributes`` are ``flags``, ``environment``, ``extra_rpaths``, and ``implicit_rpaths``.\n\n.. code-block:: yaml\n\n packages:\n gcc:\n externals:\n - spec: gcc@10.5.0 languages='c,c++,fortran'\n prefix: /usr\n extra_attributes:\n compilers:\n c: /usr/bin/gcc-10\n cxx: /usr/bin/g++-10\n fortran: /usr/bin/gfortran-10\n flags:\n cflags: -O3\n fflags: -g -O2\n environment:\n set:\n GCC_ROOT: /usr\n prepend_path:\n PATH: /usr/unusual_path_for_ld/bin\n implicit_rpaths:\n - /usr/lib/gcc\n extra_rpaths:\n - /usr/lib/unusual_gcc_path\n\nThe ``flags`` attribute specifies compiler flags to apply to every spec that depends on this compiler.\nThe accepted flag types are ``cflags``, ``cxxflags``, ``fflags``, ``cppflags``, ``ldflags``, and ``ldlibs``.\nIn the example above, every spec compiled with this compiler will pass the flags ``-g -O2`` to ``/usr/bin/gfortran-10`` and will pass the flag ``-O3`` to ``/usr/bin/gcc-10``.\n\nThe ``environment`` attribute specifies user environment modifications to apply before every time the compiler is invoked.\nThe available operations are ``set``, ``unset``, ``prepend_path``, ``append_path``, and ``remove_path``.\nIn the example above, Spack will set ``GCC_ROOT=/usr`` and set ``PATH=/usr/unusual_path_for_ld/bin:$PATH`` before handing control to the build system that will use this compiler.\n\nThe ``extra_rpaths`` and ``implicit_rpaths`` fields specify additional paths to pass as rpaths to the linker when using this compiler.\nThe ``implicit_rpaths`` field is filled in automatically by Spack when detecting compilers, and the ``extra_rpaths`` field is available for users to configure necessary rpaths that have not been detected by Spack.\nIn addition, paths from ``extra_rpaths`` are added as library search paths for the linker.\nIn the example above, both ``/usr/lib/gcc`` and ``/usr/lib/unusual_gcc_path`` would be added as rpaths to the linker, and ``-L/usr/lib/unusual_gcc_path`` would be added as well.\n\n.. _package-requirements:\n\nPackage Requirements\n--------------------\n\nSpack can be configured to always use certain compilers, package versions, and variants during concretization through package requirements.\n\nPackage requirements are useful when you find yourself repeatedly specifying the same constraints on the command line, and wish that Spack respects these constraints whether you mention them explicitly or not.\nAnother use case is specifying constraints that should apply to all root specs in an environment, without having to repeat the constraint everywhere.\n\nApart from that, requirements config is more flexible than constraints on the command line, because it can specify constraints on packages *when they occur* as a dependency.\nIn contrast, on the command line it is not possible to specify constraints on dependencies while also keeping those dependencies optional.\n\n.. seealso::\n\n FAQ: :ref:`Why does Spack pick particular versions and variants? `\n\n\nRequirements syntax\n^^^^^^^^^^^^^^^^^^^\n\nThe package requirements configuration is specified in ``packages.yaml``, keyed by package name and expressed using the Spec syntax.\nIn the simplest case you can specify attributes that you always want the package to have by providing a single spec string to ``require``:\n\n.. code-block:: yaml\n\n packages:\n libfabric:\n require: \"@1.13.2\"\n\nIn the above example, ``libfabric`` will always build with version 1.13.2.\nIf you need to compose multiple configuration scopes ``require`` accepts a list of strings:\n\n.. code-block:: yaml\n\n packages:\n libfabric:\n require:\n - \"@1.13.2\"\n - \"%gcc\"\n\nIn this case ``libfabric`` will always build with version 1.13.2 **and** using GCC as a compiler.\n\nFor more complex use cases, require accepts also a list of objects.\nThese objects must have either a ``any_of`` or a ``one_of`` field, containing a list of spec strings, and they can optionally have a ``when`` and a ``message`` attribute:\n\n.. code-block:: yaml\n\n packages:\n openmpi:\n require:\n - any_of: [\"@4.1.5\", \"%c,cxx,fortran=gcc\"]\n message: \"in this example only 4.1.5 can build with other compilers\"\n\n``any_of`` is a list of specs.\nOne of those specs must be satisfied and it is also allowed for the concretized spec to match more than one.\nIn the above example, that means you could build ``openmpi@4.1.5%gcc``, ``openmpi@4.1.5%clang`` or ``openmpi@3.9%gcc``, but not ``openmpi@3.9%clang``.\n\nIf a custom message is provided, and the requirement is not satisfiable, Spack will print the custom error message:\n\n.. code-block:: spec\n\n $ spack spec openmpi@3.9%clang\n ==> Error: in this example only 4.1.5 can build with other compilers\n\nWe could express a similar requirement using the ``when`` attribute:\n\n.. code-block:: yaml\n\n packages:\n openmpi:\n require:\n - any_of: [\"%c,cxx,fortran=gcc\"]\n when: \"@:4.1.4\"\n message: \"in this example only 4.1.5 can build with other compilers\"\n\nIn the example above, if the version turns out to be 4.1.4 or less, we require the compiler to be GCC.\nFor readability, Spack also allows a ``spec`` key accepting a string when there is only a single constraint:\n\n.. code-block:: yaml\n\n packages:\n openmpi:\n require:\n - spec: \"%c,cxx,fortran=gcc\"\n when: \"@:4.1.4\"\n message: \"in this example only 4.1.5 can build with other compilers\"\n\nThis code snippet and the one before it are semantically equivalent.\n\nFinally, instead of ``any_of`` you can use ``one_of`` which also takes a list of specs.\nThe final concretized spec must match one and only one of them:\n\n.. code-block:: yaml\n\n packages:\n mpich:\n require:\n - one_of: [\"+cuda\", \"+rocm\"]\n\nIn the example above, that means you could build ``mpich+cuda`` or ``mpich+rocm`` but not ``mpich+cuda+rocm``.\n\n.. note::\n\n For ``any_of`` and ``one_of``, the order of specs indicates a preference: items that appear earlier in the list are preferred (note that these preferences can be ignored in favor of others).\n\n.. note::\n\n When using a conditional requirement, Spack is allowed to actively avoid the triggering condition (the ``when=...`` spec) if that leads to a concrete spec with better scores in the optimization criteria.\n To check the current optimization criteria and their priorities you can run ``spack solve zlib``.\n\nSetting default requirements\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nYou can also set default requirements for all packages under ``all`` like this:\n\n.. code-block:: yaml\n\n packages:\n all:\n require: \"%[when=%c]c=clang %[when=%cxx]cxx=clang\"\n\nwhich means every spec will be required to use ``clang`` as the compiler for C and C++ code.\n\n.. warning::\n\n The simpler config ``require: %clang`` will fail to build any package that does not include compiled code, because those packages cannot depend on ``clang`` (alias for ``llvm+clang``).\n In most contexts, default requirements must use either conditional dependencies or a :ref:`toolchain ` that combines conditional dependencies.\n\nRequirements on variants for all packages are possible too, but note that they are only enforced for those packages that define these variants, otherwise they are disregarded.\nFor example:\n\n.. code-block:: yaml\n\n packages:\n all:\n require:\n - \"+shared\"\n - \"+cuda\"\n\nwill just enforce ``+shared`` on ``zlib``, which has a boolean ``shared`` variant but no ``cuda`` variant.\n\nConstraints in a single spec literal are always considered as a whole, so in a case like:\n\n.. code-block:: yaml\n\n packages:\n all:\n require: \"+shared +cuda\"\n\nthe default requirement will be enforced only if a package has both a ``cuda`` and a ``shared`` variant, and will never be partially enforced.\n\nFinally, ``all`` represents a *default set of requirements* - if there are specific package requirements, then the default requirements under ``all`` are disregarded.\nFor example, with a configuration like this:\n\n.. code-block:: yaml\n\n packages:\n all:\n require:\n - \"build_type=Debug\"\n - \"%[when=%c]c=clang %[when=%cxx]cxx=clang\"\n cmake:\n require:\n - \"build_type=Debug\"\n - \"%c,cxx=gcc\"\n\nSpack requires ``cmake`` to use ``gcc`` and all other nodes (including ``cmake`` dependencies) to use ``clang``.\nIf enforcing ``build_type=Debug`` is needed also on ``cmake``, it must be repeated in the specific ``cmake`` requirements.\n\n.. _setting-requirements-on-virtual-specs:\n\nSetting requirements on virtual specs\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nA requirement on a virtual spec applies whenever that virtual is present in the DAG.\nThis can be useful for fixing which virtual provider you want to use:\n\n.. code-block:: yaml\n\n packages:\n mpi:\n require: \"mvapich2 %c,cxx,fortran=gcc\"\n\nWith the configuration above the only allowed ``mpi`` provider is ``mvapich2`` built with ``gcc``/``g++``/``gfortran``.\n\nRequirements on the virtual spec and on the specific provider are both applied, if present.\nFor instance with a configuration like:\n\n.. code-block:: yaml\n\n packages:\n mpi:\n require: \"mvapich2 %c,cxx,fortran=gcc\"\n mvapich2:\n require: \"~cuda\"\n\nyou will use ``mvapich2~cuda %c,cxx,fortran=gcc`` as an ``mpi`` provider.\n\n.. _package-strong-preferences:\n\nConflicts and strong preferences\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nIf the semantic of requirements is too strong, you can also express \"strong preferences\" and \"conflicts\" from configuration files:\n\n.. code-block:: yaml\n\n packages:\n all:\n prefer:\n - \"%c,cxx=clang\"\n conflict:\n - \"+shared\"\n\nThe ``prefer`` and ``conflict`` sections can be used whenever a ``require`` section is allowed.\nThe argument is always a list of constraints, and each constraint can be either a simple string, or a more complex object:\n\n.. code-block:: yaml\n\n packages:\n all:\n conflict:\n - spec: \"%c,cxx=clang\"\n when: \"target=x86_64_v3\"\n message: \"reason why clang cannot be used\"\n\nThe ``spec`` attribute is mandatory, while both ``when`` and ``message`` are optional.\n\n.. note::\n\n Requirements allow for expressing both \"strong preferences\" and \"conflicts\".\n The syntax for doing so, though, may not be immediately clear.\n For instance, if we want to prevent any package from using ``%clang``, we can set:\n\n .. code-block:: yaml\n\n packages:\n all:\n require:\n - one_of: [\"%clang\", \"@:\"]\n\n Since only one of the requirements must hold, and ``@:`` is always true, the rule above is equivalent to a conflict.\n For \"strong preferences\" the same construction works, with the ``any_of`` policy instead of the ``one_of`` policy.\n\n.. _package-preferences:\n\nPackage Preferences\n-------------------\n\nIn some cases package requirements can be too strong, and package preferences are the better option.\nPackage preferences do not impose constraints on packages for particular versions or variants values, they rather only set defaults.\nThe concretizer is free to change them if it must, due to other constraints, and also prefers reusing installed packages over building new ones that are a better match for preferences.\n\n.. seealso::\n\n FAQ: :ref:`Why does Spack pick particular versions and variants? `\n\n\nThe ``target`` and ``providers`` preferences can only be set globally under the ``all`` section of ``packages.yaml``:\n\n.. code-block:: yaml\n\n packages:\n all:\n target: [x86_64_v3]\n providers:\n mpi: [mvapich2, mpich, openmpi]\n\nThese preferences override Spack's default and effectively reorder priorities when looking for the best compiler, target or virtual package provider.\nEach preference takes an ordered list of spec constraints, with earlier entries in the list being preferred over later entries.\n\nIn the example above all packages prefer to target the ``x86_64_v3`` microarchitecture and to use ``mvapich2`` if they depend on ``mpi``.\n\nThe ``variants`` and ``version`` preferences can be set under package specific sections of the ``packages.yaml`` file:\n\n.. code-block:: yaml\n\n packages:\n opencv:\n variants: +debug\n gperftools:\n version: [2.2, 2.4, 2.3]\n\nIn this case, the preference for ``opencv`` is to build with debug options, while ``gperftools`` prefers version 2.2 over 2.4.\n\nAny preference can be overwritten on the command line if explicitly requested.\n\nPreferences cannot overcome explicit constraints, as they only set a preferred ordering among homogeneous attribute values.\nGoing back to the example, if ``gperftools@2.3:`` was requested, then Spack will install version 2.4 since the most preferred version 2.2 is prohibited by the version constraint.\n\n.. _package_permissions:\n\nPackage Permissions\n-------------------\n\nSpack can be configured to assign permissions to the files installed by a package.\n\nIn the ``packages.yaml`` file under ``permissions``, the attributes ``read``, ``write``, and ``group`` control the package permissions.\nThese attributes can be set per-package, or for all packages under ``all``.\nIf permissions are set under ``all`` and for a specific package, the package-specific settings take precedence.\n\nThe ``read`` and ``write`` attributes take one of ``user``, ``group``, and ``world``.\n\n.. code-block:: yaml\n\n packages:\n all:\n permissions:\n write: group\n group: spack\n my_app:\n permissions:\n read: group\n group: my_team\n\nThe permissions settings describe the broadest level of access to installations of the specified packages.\nThe execute permissions of the file are set to the same level as read permissions for those files that are executable.\nThe default setting for ``read`` is ``world``, and for ``write`` is ``user``.\nIn the example above, installations of ``my_app`` will be installed with user and group permissions but no world permissions, and owned by the group ``my_team``.\nAll other packages will be installed with user and group write privileges, and world read privileges.\nThose packages will be owned by the group ``spack``.\n\nThe ``group`` attribute assigns a Unix-style group to a package.\nAll files installed by the package will be owned by the assigned group, and the sticky group bit will be set on the install prefix and all directories inside the install prefix.\nThis will ensure that even manually placed files within the install prefix are owned by the assigned group.\nIf no group is assigned, Spack will allow the OS default behavior to go as expected.\n\n.. _assigning-package-attributes:\n\nAssigning Package Attributes\n----------------------------\n\nYou can assign class-level attributes in the configuration:\n\n.. code-block:: yaml\n\n packages:\n mpileaks:\n package_attributes:\n # Override existing attributes\n url: http://www.somewhereelse.com/mpileaks-1.0.tar.gz\n # ... or add new ones\n x: 1\n\nAttributes set this way will be accessible to any method executed in the package.py file (e.g. the ``install()`` method).\nValues for these attributes may be any value parseable by yaml.\n\nThese can only be applied to specific packages, not \"all\" or virtual packages.\n" }, { "path": "lib/spack/docs/packaging_guide_advanced.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Advanced topics in Spack packaging, covering packages with multiple build systems, making packages discoverable with spack external find, and specifying ABI compatibility.\n\n.. list-table::\n :widths: 25 25 25 25\n :header-rows: 0\n :width: 100%\n\n * - :doc:`1. Creation `\n - :doc:`2. Build `\n - :doc:`3. Testing `\n - **4. Advanced**\n\nPackaging Guide: advanced topics\n================================\n\nThis section of the packaging guide covers a few advanced topics.\n\n.. _multiple_build_systems:\n\nMultiple build systems\n----------------------\n\nPackages may use different build systems over time or across platforms.\nSpack is designed to handle this seamlessly within a single ``package.py`` file.\n\nLet's assume we work with ``curl`` and that the package is built using Autotools so far:\n\n.. code-block:: python\n\n from spack_repo.builtin.build_systems.autotools import AutotoolsPackage\n\n\n class Curl(AutotoolsPackage):\n\n depends_on(\"zlib-api\")\n\n def configure_args(self):\n return [f\"--with-zlib={self.spec['zlib-api'].prefix}\"]\n\nTo add CMake as a further build system we need to:\n\n1. Add another base to the ``Curl`` package class (in our case ``cmake.CMakePackage``),\n2. Explicitly declare which build systems are supported using the ``build_system`` directive,\n3. Move the :doc:`build instructions ` in *separate builder classes*.\n\n.. code-block:: python\n\n from spack_repo.builtin.build_systems import autotools, cmake\n\n\n class Curl(cmake.CMakePackage, autotools.AutotoolsPackage):\n\n build_system(\"autotools\", \"cmake\", default=\"cmake\")\n\n depends_on(\"zlib-api\")\n\n\n class AutotoolsBuilder(autotools.AutotoolsBuilder):\n def configure_args(self):\n return [f\"--with-zlib={self.spec['zlib-api'].prefix}\"]\n\n\n class CMakeBuilder(cmake.CMakeBuilder):\n def cmake_args(self):\n return [self.define_from_variant(\"USE_NGHTTP2\", \"nghttp2\")]\n\nIn general, with multiple build systems there is a clear split between the :doc:`package metadata ` and the :doc:`build instructions `:\n\n1. The directives such as ``depends_on``, ``variant``, ``patch`` go into the package class\n2. The build phase functions like ``configure``, ``build`` and ``install``, and helper functions such as ``cmake_args`` or ``configure_args`` go into the builder classes\n\nWhen ``curl`` is concretized, we can select its build system using the ``build_system`` variant, which is available for every package:\n\n.. code-block:: spec\n\n $ spack install curl build_system=cmake\n\nOverride \"phases\" of a build system\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nSometimes package recipes need to override entire :ref:`phases ` of a build system.\nLet's assume this happens for ``cp2k``:\n\n.. code-block:: python\n\n from spack.package import *\n from spack_repo.builtin.build_systems import autotools\n\n\n class Cp2k(autotools.AutotoolsPackage):\n def install(self, spec: Spec, prefix: str) -> None:\n # ...existing code...\n pass\n\nIf we want to add CMake as another build system we need to remember that the signature of phases changes when moving from the ``Package`` to the ``Builder`` class:\n\n.. code-block:: python\n\n from spack.package import *\n from spack_repo.builtin.build_systems import autotools, cmake\n\n\n class Cp2k(autotools.AutotoolsPackage, cmake.CMakePackage):\n build_system(\"autotools\", \"cmake\", default=\"cmake\")\n\n\n class AutotoolsBuilder(autotools.AutotoolsBuilder):\n def install(self, pkg: Cp2k, spec: Spec, prefix: str) -> None:\n # ...existing code...\n pass\n\nThe ``install`` method now takes the ``Package`` instance as the first argument, since ``self`` refers to the builder class.\n\nAdd dependencies conditional on a build system\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nMany build dependencies are conditional on which build system is chosen.\nAn effective way to handle this is to use a ``with when(\"build_system=...\")`` block to specify dependencies that are only relevant for a specific build system:\n\n.. code-block:: python\n\n from spack.package import *\n from spack_repo.builtin.build_systems import cmake, autotools\n\n\n class Cp2k(cmake.CMakePackage, autotools.AutotoolsPackage):\n\n build_system(\"cmake\", \"autotools\", default=\"cmake\")\n\n # Runtime dependencies\n depends_on(\"ncurses\")\n depends_on(\"libxml2\")\n\n # Lowerbounds for cmake only apply when using cmake as the build system\n with when(\"build_system=cmake\"):\n depends_on(\"cmake@3.18:\", when=\"@2.0:\", type=\"build\")\n depends_on(\"cmake@3:\", type=\"build\")\n\n # Specify extra build dependencies used only in the configure script\n with when(\"build_system=autotools\"):\n depends_on(\"perl\", type=\"build\")\n depends_on(\"pkgconfig\", type=\"build\")\n\nTransition from one build system to another\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nPackages that transition from one build system to another can be modeled using :ref:`conditional variant values `:\n\n.. code-block:: python\n\n from spack.package import *\n from spack_repo.builtin.build_systems import cmake, autotools\n\n\n class Cp2k(cmake.CMakePackage, autotools.AutotoolsPackage):\n\n build_system(\n conditional(\"cmake\", when=\"@0.64:\"),\n conditional(\"autotools\", when=\"@:0.63\"),\n default=\"cmake\",\n )\n\nIn the example, the directive imposes a change from ``Autotools`` to ``CMake`` going from ``v0.63`` to ``v0.64``.\n\nInherit from a package with multiple build systems\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nCustomizing a package supporting multiple build systems is straightforward.\nIf we need to only customize the metadata, we can just define the derived package class.\n\nFor instance, let's assume we want to add a new version to the ``silo`` package:\n\n.. code-block:: python\n\n from spack_repo.builtin.packages.silo.package import Silo as BuiltinSilo\n\n class Silo(BuiltinSilo):\n # Version not in builtin.silo\n version(\"special_version\")\n\nIf we don't define any builder, Spack will reuse the custom builder from ``builtin.silo`` by default.\nIf we need to customize the builder too, we just have to inherit from it, like any other Python class:\n\n.. code-block:: python\n\n from spack_repo.builtin.packages.silo.package import CMakeBuilder as SiloCMakeBuilder\n\n class CMakeBuilder(SiloCMakeBuilder):\n def cmake_args(self):\n return [self.define_from_variant(\"USE_NGHTTP2\", \"nghttp2\")]\n\n.. _make-package-findable:\n\nMaking a package discoverable with ``spack external find``\n----------------------------------------------------------\n\nThe simplest way to make a package discoverable with :ref:`spack external find ` is to:\n\n1. Define the executables associated with the package.\n2. Implement a method to determine the versions of these executables.\n\nMinimal detection\n^^^^^^^^^^^^^^^^^\n\nThe first step is fairly simple, as it requires only to specify a package-level ``executables`` attribute:\n\n.. code-block:: python\n\n class Foo(Package):\n # Each string provided here is treated as a regular expression, and\n # would match for example \"foo\", \"foobar\", and \"bazfoo\".\n executables = [\"foo\"]\n\nThis attribute must be a list of strings.\nEach string is a regular expression (e.g. \"gcc\" would match \"gcc\", \"gcc-8.3\", \"my-weird-gcc\", etc.) to determine a set of system executables that might be part of this package.\nNote that to match only executables named \"gcc\" the regular expression ``\"^gcc$\"`` must be used.\n\nFinally, to determine the version of each executable the ``determine_version`` method must be implemented:\n\n.. code-block:: python\n\n @classmethod\n def determine_version(cls, exe):\n \"\"\"Return either the version of the executable passed as argument\n or ``None`` if the version cannot be determined.\n\n Args:\n exe (str): absolute path to the executable being examined\n \"\"\"\n\nThis method receives as input the path to a single executable and must return as output its version as a string.\nIf the version cannot be determined, or if the executable turns out to be a false positive, the value ``None`` must be returned, which ensures that the executable is discarded as a candidate.\nImplementing the two steps above is mandatory, and gives the package the basic ability to detect if a spec is present on the system at a given version.\n\n.. note::\n Any executable for which the ``determine_version`` method returns ``None`` will be discarded and won't appear in later stages of the workflow described below.\n\nAdditional functionality\n^^^^^^^^^^^^^^^^^^^^^^^^\n\nBesides the two mandatory steps described above, there are also optional methods that can be implemented to either increase the amount of details being detected or improve the robustness of the detection logic in a package.\n\nVariants and custom attributes\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nThe ``determine_variants`` method can be optionally implemented in a package to detect additional details of the spec:\n\n.. code-block:: python\n\n @classmethod\n def determine_variants(cls, exes, version_str):\n \"\"\"Return either a variant string, a tuple of a variant string\n and a dictionary of extra attributes that will be recorded in\n packages.yaml or a list of those items.\n\n Args:\n exes (list of str): list of executables (absolute paths) that\n live in the same prefix and share the same version\n version_str (str): version associated with the list of\n executables, as detected by ``determine_version``\n \"\"\"\n\nThis method takes as input a list of executables that live in the same prefix and share the same version string, and returns either:\n\n1. A variant string\n2. A tuple of a variant string and a dictionary of extra attributes\n3. A list of items matching either 1 or 2 (if multiple specs are detected from the set of executables)\n\nIf extra attributes are returned, they will be recorded in ``packages.yaml`` and be available for later reuse.\nAs an example, the ``gcc`` package will record by default the different compilers found and an entry in ``packages.yaml`` would look like:\n\n.. code-block:: yaml\n\n packages:\n gcc:\n externals:\n - spec: \"gcc@9.0.1 languages=c,c++,fortran\"\n prefix: /usr\n extra_attributes:\n compilers:\n c: /usr/bin/x86_64-linux-gnu-gcc-9\n c++: /usr/bin/x86_64-linux-gnu-g++-9\n fortran: /usr/bin/x86_64-linux-gnu-gfortran-9\n\nThis allows us, for instance, to keep track of executables that would be named differently if built by Spack (e.g. ``x86_64-linux-gnu-gcc-9`` instead of just ``gcc``).\n\n.. TODO: we need to gather some more experience on overriding \"prefix\"\n and other special keywords in extra attributes, but as soon as we are\n confident that this is the way to go we should document the process.\n See https://github.com/spack/spack/pull/16526#issuecomment-653783204\n\nFilter matching executables\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nSometimes defining the appropriate regex for the ``executables`` attribute might prove to be difficult, especially if one has to deal with corner cases or exclude \"red herrings\".\nTo help keep the regular expressions as simple as possible, each package can optionally implement a ``filter_detected_exes`` method:\n\n.. code-block:: python\n\n @classmethod\n def filter_detected_exes(cls, prefix, exes_in_prefix):\n \"\"\"Return a filtered list of the executables in prefix\"\"\"\n\nwhich takes as input a prefix and a list of matching executables and returns a filtered list of said executables.\n\nUsing this method has the advantage of allowing custom logic for filtering, and does not restrict the user to regular expressions only.\nConsider the case of detecting the GNU C++ compiler.\nIf we try to search for executables that match ``g++``, that would have the unwanted side effect of selecting also ``clang++`` - which is a C++ compiler provided by another package - if present on the system.\nTrying to select executables that contain ``g++`` but not ``clang`` would be quite complicated to do using only regular expressions.\nEmploying the ``filter_detected_exes`` method it becomes:\n\n.. code-block:: python\n\n class Gcc(Package):\n executables = [\"g++\"]\n\n @classmethod\n def filter_detected_exes(cls, prefix, exes_in_prefix):\n return [x for x in exes_in_prefix if \"clang\" not in x]\n\nAnother possibility that this method opens is to apply certain filtering logic when specific conditions are met (e.g. take some decisions on an OS and not on another).\n\nValidate detection\n^^^^^^^^^^^^^^^^^^\n\nTo increase detection robustness, packagers may also implement a method to validate the detected Spec objects:\n\n.. code-block:: python\n\n @classmethod\n def validate_detected_spec(cls, spec, extra_attributes):\n \"\"\"Validate a detected spec. Raise an exception if validation fails.\"\"\"\n\nThis method receives a detected spec along with its extra attributes and can be used to check that certain conditions are met by the spec.\nPackagers can either use assertions or raise an ``InvalidSpecDetected`` exception when the check fails.\nIf the conditions are not honored the spec will be discarded and any message associated with the assertion or the exception will be logged as the reason for discarding it.\n\nAs an example, a package that wants to check that the ``compilers`` attribute is in the extra attributes can implement this method like this:\n\n.. code-block:: python\n\n @classmethod\n def validate_detected_spec(cls, spec, extra_attributes):\n \"\"\"Check that \"compilers\" is in the extra attributes.\"\"\"\n msg = \"the extra attribute 'compilers' must be set for the detected spec '{0}'\".format(spec)\n assert \"compilers\" in extra_attributes, msg\n\nor like this:\n\n.. code-block:: python\n\n @classmethod\n def validate_detected_spec(cls, spec, extra_attributes):\n \"\"\"Check that \"compilers\" is in the extra attributes.\"\"\"\n if \"compilers\" not in extra_attributes:\n msg = \"the extra attribute 'compilers' must be set for the detected spec '{0}'\".format(\n spec\n )\n raise InvalidSpecDetected(msg)\n\n.. _determine_spec_details:\n\nCustom detection workflow\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nIn the rare case when the mechanisms described so far don't fit the detection of a package, the implementation of all the methods above can be disregarded and instead a custom ``determine_spec_details`` method can be implemented directly in the package class (note that the definition of the ``executables`` attribute is still required):\n\n.. code-block:: python\n\n @classmethod\n def determine_spec_details(cls, prefix, exes_in_prefix):\n # exes_in_prefix = a set of paths, each path is an executable\n # prefix = a prefix that is common to each path in exes_in_prefix\n\n # return None or [] if none of the exes represent an instance of\n # the package. Return one or more Specs for each instance of the\n # package which is thought to be installed in the provided prefix\n ...\n\nThis method takes as input a set of discovered executables (which match those specified by the user) as well as a common prefix shared by all of those executables.\nThe function must return one or more :py:class:`spack.package.Spec` associated with the executables (it can also return ``None`` to indicate that no provided executables are associated with the package).\n\nAs an example, consider a made-up package called ``foo-package`` which builds an executable called ``foo``.\n``FooPackage`` would appear as follows:\n\n.. code-block:: python\n\n class FooPackage(Package):\n homepage = \"...\"\n url = \"...\"\n\n version(...)\n\n # Each string provided here is treated as a regular expression, and\n # would match for example \"foo\", \"foobar\", and \"bazfoo\".\n executables = [\"foo\"]\n\n @classmethod\n def determine_spec_details(cls, prefix, exes_in_prefix):\n candidates = [x for x in exes_in_prefix if os.path.basename(x) == \"foo\"]\n if not candidates:\n return\n # This implementation is lazy and only checks the first candidate\n exe_path = candidates[0]\n exe = Executable(exe_path)\n output = exe(\"--version\", output=str, error=str)\n version_str = ... # parse output for version string\n return Spec.from_detection(\"foo-package@{0}\".format(version_str))\n\nAdd detection tests to packages\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nTo ensure that software is detected correctly for multiple configurations and on different systems users can write a ``detection_test.yaml`` file and put it in the package directory alongside the ``package.py`` file.\nThis YAML file contains enough information for Spack to mock an environment and try to check if the detection logic yields the results that are expected.\n\nAs a general rule, attributes at the top-level of ``detection_test.yaml`` represent search mechanisms and they each map to a list of tests that should confirm the validity of the package's detection logic.\n\nThe detection tests can be run with the following command:\n\n.. code-block:: console\n\n $ spack audit externals\n\nErrors that have been detected are reported to screen.\n\nTests for PATH inspections\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nDetection tests insisting on ``PATH`` inspections are listed under the ``paths`` attribute:\n\n.. code-block:: yaml\n\n paths:\n - layout:\n - executables:\n - \"bin/clang-3.9\"\n - \"bin/clang++-3.9\"\n script: |\n echo \"clang version 3.9.1-19ubuntu1 (tags/RELEASE_391/rc2)\"\n echo \"Target: x86_64-pc-linux-gnu\"\n echo \"Thread model: posix\"\n echo \"InstalledDir: /usr/bin\"\n platforms: [\"linux\", \"darwin\"]\n results:\n - spec: \"llvm@3.9.1 +clang~lld~lldb\"\n\nIf the ``platforms`` attribute is present, tests are run only if the current host matches one of the listed platforms.\nEach test is performed by first creating a temporary directory structure as specified in the corresponding ``layout`` and by then running package detection and checking that the outcome matches the expected ``results``.\nThe exact details on how to specify both the ``layout`` and the ``results`` are reported in the table below:\n\n.. list-table:: Test based on PATH inspections\n :header-rows: 1\n\n * - Option Name\n - Description\n - Allowed Values\n - Required Field\n * - ``layout``\n - Specifies the filesystem tree used for the test\n - List of objects\n - Yes\n * - ``layout:[0]:executables``\n - Relative paths for the mock executables to be created\n - List of strings\n - Yes\n * - ``layout:[0]:script``\n - Mock logic for the executable\n - Any valid shell script\n - Yes\n * - ``results``\n - List of expected results\n - List of objects (empty if no result is expected)\n - Yes\n * - ``results:[0]:spec``\n - A spec that is expected from detection\n - Any valid spec\n - Yes\n * - ``results:[0]:extra_attributes``\n - Extra attributes expected on the associated Spec\n - Nested dictionary with string as keys, and regular expressions as leaf values\n - No\n\nReuse tests from other packages\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nWhen using a custom repository, it is possible to customize a package that already exists in ``builtin`` and reuse its external tests.\nTo do so, just write a ``detection_test.yaml`` alongside the customized ``package.py`` with an ``includes`` attribute.\nFor instance the ``detection_test.yaml`` for ``myrepo.llvm`` might look like:\n\n.. code-block:: yaml\n\n includes:\n - \"builtin.llvm\"\n\nThis YAML file instructs Spack to run the detection tests defined in ``builtin.llvm`` in addition to those locally defined in the file.\n\n.. _abi_compatibility:\n\nSpecifying ABI Compatibility\n----------------------------\n\n.. warning::\n\n The ``can_splice`` directive is experimental, and may be replaced by a higher-level interface in future versions of Spack.\n\nPackages can include ABI-compatibility information using the ``can_splice`` directive.\nFor example, if ``Foo`` version 1.1 can always replace version 1.0, then the package could have:\n\n.. code-block:: python\n\n can_splice(\"foo@1.0\", when=\"@1.1\")\n\nFor virtual packages, packages can also specify ABI compatibility with other packages providing the same virtual.\nFor example, ``zlib-ng`` could specify:\n\n.. code-block:: python\n\n can_splice(\"zlib@1.3.1\", when=\"@2.2+compat\")\n\nSome packages have ABI-compatibility that is dependent on matching variant values, either for all variants or for some set of ABI-relevant variants.\nIn those cases, it is not necessary to specify the full combinatorial explosion.\nThe ``match_variants`` keyword can cover all single-value variants.\n\n.. code-block:: python\n\n # any value for bar as long as they're the same\n can_splice(\"foo@1.1\", when=\"@1.2\", match_variants=[\"bar\"])\n\n # any variant values if all single-value variants match\n can_splice(\"foo@1.2\", when=\"@1.3\", match_variants=\"*\")\n\nThe concretizer will use ABI compatibility to determine automatic splices when :ref:`automatic splicing` is enabled.\n\nCustomizing Views\n-----------------\n\n.. warning::\n\n This is advanced functionality documented for completeness, and rarely needs customization.\n\nSpack environments manage a view of their packages, which is a single directory that merges all installed packages through symlinks, so users can easily access them.\nThe methods of ``PackageViewMixin`` can be overridden to customize how packages are added to views.\nSometimes it's impossible to get an application to work just through symlinking its executables, and patching is necessary.\nFor example, Python scripts in a ``bin`` directory may have a shebang that points to the Python interpreter in Python's install prefix and not to the Python interpreter in the view.\nHowever, it's more convenient to have the shebang point to the Python interpreter in the view, since that interpreter can locate other Python packages in the view without ``PYTHONPATH`` being set.\nTherefore, Python extension packages (those inheriting from ``PythonPackage``) override ``add_files_to_view`` in order to rewrite shebang lines.\n" }, { "path": "lib/spack/docs/packaging_guide_build.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n A guide to customizing the build process in Spack, covering installation procedures, build systems, and how to control the build with spec objects and environment variables.\n\n.. list-table::\n :widths: 25 25 25 25\n :header-rows: 0\n :width: 100%\n\n * - :doc:`1. Creation `\n - **2. Build**\n - :doc:`3. Testing `\n - :doc:`4. Advanced `\n\nPackaging Guide: customizing the build\n======================================\n\nIn the first part of the packaging guide, we covered the basic structure of a package, how to specify dependencies, and how to define variants.\nIn the second part, we will cover the installation procedure, build systems, and how to customize the build process.\n\n.. _installation_procedure:\n\nOverview of the installation procedure\n--------------------------------------\n\nWhenever Spack installs software, it goes through a series of predefined steps:\n\n.. image:: images/installation_pipeline.png\n :scale: 60 %\n :align: center\n\nAll these steps are influenced by the metadata in each ``package.py`` and by the current Spack configuration.\nSince build systems are different from one another, the execution of the last block in the figure is further expanded in a build system specific way.\nAn example for ``CMake`` is, for instance:\n\n.. image:: images/builder_phases.png\n :align: center\n :scale: 60 %\n\nThe predefined steps for each build system are called \"phases\".\nIn general, the name and order in which the phases will be executed can be obtained by either reading the API docs at :py:mod:`~.spack_repo.builtin.build_systems`, or using the ``spack info`` command:\n\n.. code-block:: console\n :emphasize-lines: 13,14\n\n $ spack info --phases m4\n AutotoolsPackage: m4\n Homepage: https://www.gnu.org/software/m4/m4.html\n\n Safe versions:\n 1.4.17 ftp://ftp.gnu.org/gnu/m4/m4-1.4.17.tar.gz\n\n Variants:\n Name Default Description\n\n sigsegv on Build the libsigsegv dependency\n\n Installation Phases:\n autoreconf configure build install\n\n Build Dependencies:\n libsigsegv\n\n ...\n\nAn extensive list of available build systems and phases is provided in :ref:`installation_process`.\n\nControlling the build process\n-----------------------------\n\nAs we have seen in the first part of the packaging guide, the usual workflow for creating a package is to start with ``spack create ``, which generates a ``package.py`` file for you with a boilerplate package class.\nThis typically includes a package base class (e.g. ``AutotoolsPackage`` or ``CMakePackage``), a URL, and one or more versions.\nAfter you have added required dependencies and variants, you can start customizing the build process.\nThere are various ways to do this, depending on the build system and the package itself.\n\nFrom simplest to most complex, the following are the most common ways to customize the build process:\n\n1. **Implementing build system helper methods and properties**.\n Most build systems provide a set of helper methods that can be overridden to customize the build process without overriding entire phases.\n For example, for ``AutotoolsPackage`` you can specify the command line arguments for ``./configure`` by implementing ``configure_args``:\n\n .. code-block:: python\n\n class MyPkg(AutotoolsPackage):\n def configure_args(self):\n # FIXME: Add arguments other than --prefix\n # FIXME: If not needed delete this function\n args = []\n return args\n\n Similarly for ``CMakePackage`` you can influence how ``cmake`` is invoked by implementing ``cmake_args``:\n\n .. code-block:: python\n\n class MyPkg(CMakePackage):\n def cmake_args(self):\n # FIXME: Add arguments other than\n # FIXME: CMAKE_INSTALL_PREFIX and CMAKE_BUILD_TYPE\n # FIXME: If not needed delete this function\n args = []\n return args\n\n The exact methods and properties available depend on the build system you are using.\n See :doc:`build_systems` for a complete list of available build systems and their specific helper functions and properties.\n\n2. **Setting environment variables**.\n Some build systems require specific environment variables to be set before the build starts.\n You can set these variables by overriding the ``setup_build_environment`` method in your package class:\n\n .. code-block:: python\n\n def setup_build_environment(self, env):\n env.set(\"MY_ENV_VAR\", \"value\")\n\n This is useful for setting paths or other variables that the build system needs to find dependencies or configure itself correctly.\n\n See :ref:`setup-environment`.\n\n3. **Complementing the build system with pre- or post-build steps**.\n In some cases, you may need to run additional commands before or after the build system phases.\n This is useful for installing additional files missed by the build system, or for running custom scripts.\n\n .. code-block:: python\n\n @run_after(\"install\")\n def install_missing_files(self):\n install_tree(\"extra_files\", self.prefix.bin)\n\n See :ref:`before_after_build_phases`.\n\n4. **Overriding entire build phases**.\n If the default implementation of a build phase does not fit your needs, you can override the entire phase.\n See :ref:`overriding-phases` for examples.\n\nIn any of the functions above, you can\n\n1. **Make instructions dynamic**.\n Build instructions typically depend on the package's variants, version and its dependencies.\n For example, you can use\n\n .. code-block:: python\n\n if self.spec.satisfies(\"+variant_name\"):\n ...\n\n to check if a variant is enabled, or\n\n .. code-block:: python\n\n self.spec[\"dependency_name\"].prefix\n\n to get the prefix of a dependency.\n See :ref:`spec-objects` for more details on how to use specs in your package.\n2. **Use Spack's Python Package API**.\n The ``from spack.package import *`` statement at the top of a ``package.py`` file allows you to access Spack's utilities and helper functions, such as ``which``, ``install_tree``, ``filter_file`` and others.\n See :ref:`python-package-api` for more details.\n\n\n.. _installation_process:\n\nWhat are build systems?\n-----------------------\n\nEvery package in Spack has an associated build system.\nFor most packages, this will be a well-known system for which Spack provides a base class, like ``CMakePackage`` or ``AutotoolsPackage``.\nEven for packages that have no formal build process (e.g., just copying files), Spack still associates them with a generic build system class.\nBuild systems have the following responsibilities:\n\n1. **Define and implement the build phases**.\n Each build system defines a set of phases that are executed in a specific order.\n For example, ``AutotoolsPackage`` has the following phases: ``autoreconf``, ``configure``, ``build``, and ``install``.\n These phases are Python methods with a sensible default implementation that can be overridden by the package author.\n2. **Add dependencies and variants**.\n Build systems can define dependencies and variants that are specific to the build system.\n For example, ``CMakePackage`` adds a ``cmake`` as a build dependency, and defines ``build_type`` as a variant (which maps to the ``CMAKE_BUILD_TYPE`` CMake variable).\n All build systems also define a special variant ``build_system``, which is useful in case of :ref:`multiple build systems `.\n3. **Provide helper methods**.\n Build systems often provide helper functions and properties that the package author can use to customize the build configuration, without having to override entire phases.\n For example:\n\n * The ``CMakePackage`` lets users implement the ``cmake_args`` method to specify additional arguments for the ``cmake`` command\n * The ``MakefilePackage`` lets users set ``build_targets`` and ``install_targets`` properties to specify the targets to build and install.\n\n There are typically also helper functions to map variants to CMake or Autotools options:\n\n * The ``CMakePackage`` provides the ``self.define_from_variant(\"VAR_NAME\", \"variant_name\")`` method to generate the appropriate ``-DVAR_NAME:BOOL=ON/OFF`` arguments for the ``cmake`` command.\n * The ``AutotoolsPackage`` provides helper functions like ``self.with_or_without(\"foo\")`` to generate the appropriate ``--with-foo`` or ``--without-foo`` arguments for the ``./configure`` script.\n\nHere is a table of the most common build systems available in Spack:\n\n.. list-table::\n :widths: 40 60\n :header-rows: 1\n\n * - Package Class\n - Description\n * - :doc:`AutotoolsPackage `\n - For packages that use GNU Autotools (autoconf, automake, libtool).\n * - :doc:`CMakePackage `\n - For packages that use CMake.\n * - :doc:`MakefilePackage `\n - For packages that use plain Makefiles.\n * - :doc:`MesonPackage `\n - For packages that use the Meson build system.\n * - :doc:`PythonPackage `\n - For Python packages (setuptools, pip, etc.).\n * - :doc:`BundlePackage `\n - For installing a collection of other packages.\n * - :doc:`Package `\n - Generic package for custom builds, provides only an ``install`` phase.\n\nAll build systems are defined in the ``spack_repo.builtin.build_systems`` module, which is part of the Spack builtin package repository.\nTo use a particular build system, you need to import it in your ``package.py`` file, and then derive your package class from the appropriate base class:\n\n.. code-block:: python\n\n from spack_repo.builtin.build_systems.cmake import CMakePackage\n\n\n class MyPkg(CMakePackage):\n pass\n\nFor a complete list of build systems and their specific helper functions and properties, see the :doc:`build_systems` documentation.\n\n\n.. _spec-objects:\n\nConfiguring the build with spec objects\n---------------------------------------\n\nConfiguring a build is typically the first step in the build process.\nIn many build systems it involves passing the right command line arguments to the configure script, and in some build systems it is a matter of setting the right environment variables.\nIn this section we will use an Autotools package as an example, where we just need to implement the ``configure_args`` helper function.\n\nIn general, whenever you implement helper functions of a build system or complement or override its build phases, you often need to make decisions based on the package's configuration.\nSpack is unique in that it allows you to write a *single* ``package.py`` for all configurations of a package.\n\nThe central object in Spack that encodes the package's configuration is the **concrete spec**, which is available as ``self.spec`` in the package class.\nThis is the object you need to query to make decisions about how to configure the build.\n\nQuerying ``self.spec``\n^^^^^^^^^^^^^^^^^^^^^^\n\n**Variants**.\nIn the previous section of the packaging guide, we've seen :ref:`how to define variants `.\nAs a packager, you are responsible for implementing the logic that translates the selected variant values into build instructions the build system can understand.\nIf you want to pass a flag to the configure script only if the package is built with a specific variant, you can do so like this:\n\n.. code-block:: python\n\n variant(\"foo\", default=False, description=\"Enable foo feature\")\n\n\n def configure_args(self):\n args = []\n if self.spec.satisfies(\"+foo\"):\n args.append(\"--enable-foo\")\n else:\n args.append(\"--disable-foo\")\n return args\n\nFor multi-valued variants, you can use the ``key=value`` syntax to test whether a specific value is selected:\n\n.. code-block:: python\n\n variant(\"threads\", default=\"none\", values=(\"pthreads\", \"openmp\", \"none\"), multi=False, ...)\n\n\n def configure_args(self):\n args = []\n if self.spec.satisfies(\"threads=pthreads\"):\n args.append(\"--enable-threads=pthreads\")\n elif self.spec.satisfies(\"threads=openmp\"):\n args.append(\"--enable-threads=openmp\")\n elif self.spec.satisfies(\"threads=none\"):\n args.append(\"--disable-threads\")\n return args\n\nEven if *multiple* values are selected, you can still use ``key=value`` to test for specific values:\n\n.. code-block:: python\n\n variant(\"languages\", default=\"c,c++\", values=(\"c\", \"c++\", \"fortran\"), multi=True, ...)\n\n\n def configure_args(self):\n args = []\n if self.spec.satisfies(\"languages=c\"):\n args.append(\"--enable-c\")\n if self.spec.satisfies(\"languages=c++\"):\n args.append(\"--enable-c++\")\n if self.spec.satisfies(\"languages=fortran\"):\n args.append(\"--enable-fortran\")\n return args\n\nNotice that many build systems provide helper functions to make the above code more concise.\nSee :ref:`the Autotools docs ` and :ref:`the CMake docs `.\n\nOther than testing for certain variant values, you can also obtain the variant value directly with ``self.spec.variants[\"variant_name\"].value``.\nThis is useful when you want to pass the variant value as a command line argument to the build system.\nThe type of this value depends on the variant type:\n\n* For boolean variants this is :data:`True` or :data:`False`.\n* For single-valued variants this is a :class:`str` value.\n* For multi-valued variants it is a tuple of :class:`str` values.\n\nAn example of using this is shown below:\n\n.. code-block:: python\n\n variant(\n \"cxxstd\",\n default=\"11\",\n values=(\"11\", \"14\", \"17\", \"20\", \"23\"),\n multi=False,\n description=\"C++ standard\",\n )\n\n\n def configure_args(self):\n return [f\"--with-cxxstd={self.spec.variants['cxxstd'].value}\"]\n\n**Versions**.\nSimilarly, versions are often used to dynamically change the build configuration:\n\n.. code-block:: python\n\n def configure_args(self):\n args = []\n if self.spec.satisfies(\"@1.2:\"):\n args.append(\"--enable-new-feature\")\n return args\n\nThis adds a flag only if the package is on version 1.2 or higher.\n\n**Dependencies**.\nYou can also use the ``self.spec.satisfies`` method to test whether a dependency is present or not, and whether it is built with a specific variant or version.\n\nThe ``^`` character is used to refer to packages that are required at runtime as well as build dependencies.\nMore precisely, it includes all direct dependencies of ``build`` type and transitive dependencies of ``link`` or ``run`` type.\n\n.. code-block:: python\n\n if self.spec.satisfies(\"^python@3.8:\"):\n args.append(\"--min-python-version=3.8\")\n\nHere we test whether the package has a (possibly transitive) dependency on Python version 3.8 or higher.\n\nThe ``%`` character is used to refer to direct dependencies only.\nA typical use case is when you want to test the compiler used to build the package.\n\n.. code-block:: python\n\n if self.spec.satisfies(\"%c=gcc@8:\"):\n args.append(\"--enable-profile-guided-optimization\")\n\nThis example adds a flag when the C compiler is from GCC version 8 or higher.\nThe ``%c=gcc`` syntax technically means that ``gcc`` is the provider for the ``c`` language virtual.\n\n.. tip::\n\n Historically, many packages have been written using ``^dep`` to refer to a dependency.\n Modern Spack packages should consider using ``%dep`` instead, which is more precise: it can only match direct dependencies, which are listed in the ``depends_on`` statements.\n\n\n**Target specific configuration**.\nSpack always makes the special ``platform``, ``os`` and ``target`` variants available in the spec.\nThese variants can be used to test the target platform, operating system and CPU microarchitecture the package.\n\nThe following example shows how we can add a configure option only if the package is built for Apple Silicon:\n\n.. code-block:: python\n\n if self.spec.satisfies(\"platform=darwin target=aarch64:\"):\n args.append(\"--enable-apple-silicon\")\n\nNotice that ``target=aarch64:`` is a range which matches the whole family of ``aarch64`` microarchitectures, including ``m1``, ``m2``, and so on.\n\nYou can use ranges starting at a specific microarchitecture as well, for example:\n\n.. code-block:: python\n\n if self.spec.satisfies(\"target=haswell:\"):\n args.append(\"--enable-haswell\")\n\n.. note::\n\n The ``spec`` object encodes the *target* platform, os and architecture the package is being built for.\n This is different from the *host* platform (typically accessed via ``sys.platform``) which is the platform where Spack is running.\n When writing package recipes, you should always use the ``spec`` object to query the target platform, os and architecture.\n\nTo see what targets are available in your Spack installation, you can use the following command:\n\n.. command-output:: spack arch --known-targets\n\nReferring to a dependency's prefix, libraries, and headers\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nVery often you need to inform the build system about the location of a dependency.\nThe most common way to do this is to pass the dependency's prefix as a configure argument and let the build system detect the libraries and headers from there.\n\nTo do this, you can obtain the **dependency's spec** by name:\n\n.. code-block:: python\n\n libxml2 = self.spec[\"libxml2\"]\n\nThe ``libxml2`` variable is itself a spec object, and we can refer to its properties:\n\n.. code-block:: python\n\n def configure_args(self):\n return [\n f\"--with-libxml2={self.spec['libxml2'].prefix}\",\n ]\n\nApart from the :ref:`prefix `, you can also access other attributes of the dependency, such as ``libs`` or ``headers``.\nSee :ref:`custom-attributes` for how dependencies define these attributes.\nThese attributes are typically only required if the package is unable to locate the libraries and headers itself, or if you want to be more specific about which libraries or headers to use.\n\nA more advanced example where we explicitly pass libraries and headers to the configure script is shown below.\n\n.. code-block:: python\n\n def configure_args(self):\n return [\n f\"--with-libxml2={self.spec['libxml2'].prefix}\",\n f\"--with-libxml2-libs={self.spec['libxml2'].libs.ld_flags}\",\n f\"--with-libxml2-include={self.spec['libxml2'].headers.include_flags}\",\n ]\n\nThe ``libs`` attribute is a :class:`~spack.package.LibraryList` object that can be used to get a list of libraries by path, but also to get the appropriate linker flags.\nSimilarly, the ``headers`` attribute is a :class:`~spack.package.HeaderList`, which also has methods to get the relevant include flags.\n\n.. _blas_lapack_scalapack:\n\n**Virtual dependencies**.\nYou can also refer to the prefix, libraries and headers of :ref:`virtual dependencies `.\nFor example, suppose we have a package that depends on ``blas`` and ``lapack``.\nWe can get the provider's (e.g. OpenBLAS or Intel MKL) prefixes like this:\n\n.. code-block:: python\n\n class MyPkg(AutotoolPackage):\n depends_on(\"blas\")\n depends_on(\"lapack\")\n\n def configure_args(self):\n return [\n f\"--with-blas={self.spec['blas'].prefix}\",\n f\"--with-lapack={self.spec['lapack'].prefix}\",\n ]\n\nMany build systems struggle to locate the ``blas`` and ``lapack`` libraries during configure, either because they do not know the exact names of the libraries, or because the libraries are not in typical locations --- they may not even know whether ``blas`` and ``lapack`` are a single or separate libraries.\nIn those cases, the build system could use some help, for which we give a few examples below:\n\n1. Space separated list of full paths\n\n .. code-block:: python\n\n lapack_blas = spec[\"lapack\"].libs + spec[\"blas\"].libs\n args.append(f\"--with-blas-lapack-lib={lapack_blas.joined()}\")\n\n2. Names of libraries and directories which contain them\n\n .. code-block:: python\n\n lapack_blas = spec[\"lapack\"].libs + spec[\"blas\"].libs\n args.extend(\n [\n f\"-DMATH_LIBRARY_NAMES={';'.join(lapack_blas.names)}\",\n f\"-DMATH_LIBRARY_DIRS={';'.join(lapack_blas.directories)}\",\n ]\n )\n\n3. Search and link flags\n\n .. code-block:: python\n\n lapack_blas = spec[\"lapack\"].libs + spec[\"blas\"].libs\n args.append(f\"-DMATH_LIBS={lapack_blas.ld_flags}\")\n\n\n.. _before_after_build_phases:\n\nBefore and after build phases\n-----------------------------\n\nTypically the default implementation of the build system's phases is sufficient for most packages.\nHowever, in some cases you may need to complement the default implementation with some custom instructions.\nFor example, some packages do not install all the files they should, and you want to fix this by simply copying the missing files after the normal install phase is done.\nInstead of overriding the entire phase, you can use ``@run_before`` and ``@run_after`` to run custom code before or after a specific phase:\n\n.. code-block:: python\n\n class MyPackage(CMakePackage):\n ...\n\n variant(\"extras\", default=False, description=\"Install extra files\")\n\n @run_before(\"cmake\")\n def run_before_cmake_is_invoked(self) -> None:\n with open(\"custom_file.txt\", \"w\") as f:\n f.write(\"This file is created before cmake is invoked.\")\n\n @run_after(\"install\", when=\"+extras\")\n def custom_post_install_phase(self) -> None:\n # install missing files not covered by the build system\n install_tree(\"extras\", self.prefix.share.extras)\n\nThen ``when=\"+extras\"`` will ensure that the custom post-install phase is only run conditionally.\n\nThe function body should contain the actual instructions you want to run before or after the build phase, which can involve :ref:`running executables ` and creating or copying files to the ``prefix`` directory using convenience functions from :ref:`Spack's Python Package API `.\n\n.. _overriding-phases:\n\nOverriding a build phase\n------------------------\n\nIf a build phase does not do what you need, and you cannot achieve your goal either by implementing the helper methods of the build system, or by using the ``@run_before`` or ``@run_after`` decorators (see :ref:`before_after_build_phases`), you can override the entire build phase.\n\nThe most common scenario is when a package simply does not have a well-defined build system.\nFor example, the installation procedure may just be copying files or running a shell script.\nIn that case, you can use the generic ``Package`` class, which defines only a single ``install()`` phase, to be overridden by the package author:\n\n.. code-block:: python\n\n from spack.package import *\n from spack_repo.builtin.build_systems.generic import Package\n\n\n class MyPkg(Package):\n\n # Override the install phase\n def install(self, spec: Spec, prefix: Prefix) -> None:\n install_tree(\"my_files\", prefix.bin)\n\nWhichever build system is used, **every build phase function has the same set of arguments**.\nThe arguments are:\n\n``self``\n This is the package object, which extends ``CMakePackage``.\n For API docs on Package objects, see :py:class:`Package `.\n\n``spec``\n This is the concrete spec object created by Spack from an abstract spec supplied by the user.\n It describes what should be installed.\n It will be of type :py:class:`Spec `.\n\n``prefix``\n This is where your package should install its files.\n It acts like a string, but it's actually its :ref:`own special type `.\n\nThe function body should contain the actual build instructions, which typically involves:\n\n1. Invoking the build system's commands such as ``make``, ``ninja``, ``python``, et cetera.\n See :ref:`running_build_executables` for how to do this.\n2. Copying files to the ``prefix`` directory, which is where Spack expects the package to be installed.\n This can be done using Spack's built-in functions like ``install_tree()`` or ``install()``.\n See the :ref:`Spack's Python Package API ` for all convenience functions that can be used in the package class.\n\nThe arguments ``spec`` and ``prefix`` are passed only for convenience, as they always correspond to ``self.spec`` and ``self.spec.prefix`` respectively, as we have already seen in :ref:`the previous section `.\n\n.. warning::\n\n When working with :ref:`multiple build systems ` in a single package, the arguments for build phase functions are slightly different.\n\n.. _running_build_executables:\n\nRunning build executables\n-------------------------\n\nWhen you :ref:`override a build phase `, or when you write a :ref:`build phase hook `, you typically need to invoke executables like ``make``, ``cmake``, or ``python`` to kick off the build process.\n\nSpack makes some of these executables available as global functions, making it easy to run them in your package class:\n\n.. code-block:: python\n\n from spack.package import *\n from spack_repo.builtin.build_systems.generic import Package\n\n\n class MyPkg(Package):\n\n depends_on(\"make\", type=\"build\")\n depends_on(\"python\", type=\"build\")\n\n def install(self, spec: Spec, prefix: Prefix) -> None:\n python(\"generate-makefile.py\", \"--output=Makefile\")\n make()\n make(\"install\")\n\nThe ``python()`` and ``make()`` functions in this example invoke the ``python3`` and ``make`` executables, respectively.\nNaturally, you may wonder where these variables come from, since they are not imported from anywhere --- your editor may even underline them in red because they are not defined in the package module.\n\nThe answer lies in the ``python`` and ``make`` dependencies, which implement the :meth:`~spack.package.PackageBase.setup_dependent_package` method in their package classes.\nThis sets up Python variables that can be used in the package class of dependents.\n\nThere is a good reason that it's the *dependency* that sets up these variables, rather than the package itself.\nFor example, the ``make`` package ensures sensible default arguments for the ``make`` executable, such as the ``-j`` flag to enable parallel builds.\nThis means that you do not have to worry about these technical details in your package class; you can just use ``make(\"my_target\")`` and Spack will take care of the rest.\nSee the section about :ref:`parallel builds ` for more details.\n\nNot all dependencies set up such variables for dependent packages, in which case you have two further options:\n\n1. Use the ``command`` attribute of the dependency.\n This is a good option, since it refers to an executable provided by a specific dependency.\n\n .. code-block:: python\n\n def install(self, spec: Spec, prefix: Prefix) -> None:\n cython = self.spec[\"py-cython\"].command\n cython(\"setup.py\", \"build_ext\", \"--inplace\")\n\n2. Use the ``which`` function (from the ``spack.package`` module).\n Do note that this function relies on the order of the ``PATH`` environment variable, which may be less reliable than the first option.\n\n .. code-block:: python\n\n def install(self, spec: Spec, prefix: Prefix) -> None:\n cython = which(\"cython\", required=True)\n cython(\"setup.py\", \"build_ext\", \"--inplace\")\n\nAll executables in Spack are instances of :class:`~spack.package.Executable`, see its API docs for more details.\n\n\n.. _attribute_parallel:\n\nPackage-level parallelism\n-------------------------\n\nMany build tools support parallel builds, including ``make`` and ``ninja``, as well as certain Python build tools.\n\nAs mentioned in :ref:`the previous section `, the ``gmake`` and ``ninja`` packages make their executables available as global functions, which you can use in your package class.\nThey automatically add the ``-j `` when invoked, where ```` is a sensible default for the number of jobs to run in parallel.\nThis exact number :ref:`is determined ` depends on various factors, such as the ``spack install`` command line arguments, configuration options and available CPUs on the system.\nAs a packager, you rarely need to pass the ``-j`` flag when calling ``make()`` or ``ninja()``; it is better to rely on the defaults.\n\nIn certain cases however, you may need to override the default number of jobs for a specific package.\nIf a package does not build properly in parallel, you can simply define ``parallel = False`` in your package class.\nFor example:\n\n.. code-block:: python\n :emphasize-lines: 4\n\n class ExamplePackage(MakefilePackage):\n \"\"\"Example package that does not build in parallel.\"\"\"\n\n parallel = False\n\nThis ensures that any ``make`` or ``ninja`` invocation will *not* set the ``-j `` option, and the build will run sequentially.\n\nYou can also disable parallel builds only for specific make invocation:\n\n.. code-block:: python\n :emphasize-lines: 5\n\n class Libelf(MakefilePackage):\n ...\n\n def install(self, spec: Spec, prefix: Prefix) -> None:\n make(\"install\", parallel=False)\n\nIn this case, the ``build`` phase will still execute in parallel, but the ``install`` phase will run sequentially.\n\nFor packages whose build systems do not run ``make`` or ``ninja``, but have other executables or scripts that support parallel builds, you can control parallelism using the ``make_jobs`` global.\nThis global variable is an integer that specifies the number of jobs to run in parallel during the build process.\n\n.. code-block:: python\n :emphasize-lines: 6\n\n class Xios(Package):\n def install(self, spec: Spec, prefix: Prefix) -> None:\n make_xios = Executable(\"./make_xios\")\n make_xios(\n \"--with-feature\",\n f\"--jobs={make_jobs}\",\n )\n\n.. _python-package-api:\n\nSpack's Python Package API\n--------------------------\n\nWhenever you implement :ref:`overriding phases ` or :ref:`before and after build phases `, you typically need to modify files, work with paths and run executables.\nSpack provides a number of convenience functions and classes of its own to make your life even easier, complementing the Python standard library.\n\nAll of the functionality in this section is made available by importing the :mod:`spack.package` module.\n\n.. code-block:: python\n\n from spack.package import *\n\nThis is already part of the boilerplate for packages created with ``spack create``.\n\n.. _file-filtering:\n\nFile filtering functions\n^^^^^^^^^^^^^^^^^^^^^^^^\n\n:py:func:`filter_file(regex, repl, *filenames, **kwargs) `\n Works like ``sed`` but with Python regular expression syntax.\n Takes a regular expression, a replacement, and a set of files.\n ``repl`` can be a raw string or a callable function.\n If it is a raw string, it can contain ``\\1``, ``\\2``, etc. to refer to capture groups in the regular expression.\n If it is a callable, it is passed the Python ``MatchObject`` and should return a suitable replacement string for the particular match.\n\n Examples:\n\n #. Filtering a Makefile to force it to use Spack's compiler wrappers:\n\n .. code-block:: python\n\n filter_file(r\"^\\s*CC\\s*=.*\", \"CC = \" + spack_cc, \"Makefile\")\n filter_file(r\"^\\s*CXX\\s*=.*\", \"CXX = \" + spack_cxx, \"Makefile\")\n filter_file(r\"^\\s*F77\\s*=.*\", \"F77 = \" + spack_f77, \"Makefile\")\n filter_file(r\"^\\s*FC\\s*=.*\", \"FC = \" + spack_fc, \"Makefile\")\n\n #. Replacing ``#!/usr/bin/perl`` with ``#!/usr/bin/env perl`` in ``bib2xhtml``:\n\n .. code-block:: python\n\n filter_file(r\"#!/usr/bin/perl\", \"#!/usr/bin/env perl\", prefix.bin.bib2xhtml)\n\n #. Switching the compilers used by ``mpich``'s MPI wrapper scripts from ``cc``, etc. to the compilers used by the Spack build:\n\n .. code-block:: python\n\n filter_file(\"CC='cc'\", \"CC='%s'\" % self.compiler.cc, prefix.bin.mpicc)\n filter_file(\"CXX='c++'\", \"CXX='%s'\" % self.compiler.cxx, prefix.bin.mpicxx)\n\n:py:func:`change_sed_delimiter(old_delim, new_delim, *filenames) `\n Some packages, like TAU, have a build system that can't install into directories with, e.g. \"@\" in the name, because they use hard-coded ``sed`` commands in their build.\n\n ``change_sed_delimiter`` finds all ``sed`` search/replace commands and changes the delimiter.\n E.g., if the file contains commands that look like ``s///``, you can use this to change them to ``s@@@``.\n\n Example of changing ``s///`` to ``s@@@`` in TAU:\n\n .. code-block:: python\n\n change_sed_delimiter(\"@\", \";\", \"configure\")\n change_sed_delimiter(\"@\", \";\", \"utils/FixMakefile\")\n change_sed_delimiter(\"@\", \";\", \"utils/FixMakefile.sed.default\")\n\nFile functions\n^^^^^^^^^^^^^^\n\n:py:func:`ancestor(dir, n=1) `\n Get the n\\ :sup:`th` ancestor of the directory ``dir``.\n\n:py:func:`can_access(path) `\n True if we can read and write to the file at ``path``.\n Same as native Python ``os.access(file_name, os.R_OK|os.W_OK)``.\n\n:py:func:`install(src, dest) `\n Install a file to a particular location.\n For example, install a header into the ``include`` directory under the install ``prefix``:\n\n .. code-block:: python\n\n install(\"my-header.h\", prefix.include)\n\n:py:func:`join_path(*paths) `\n An alias for ``os.path.join``.\n This joins paths using the OS path separator.\n\n:py:func:`mkdirp(*paths) `\n Create each of the directories in ``paths``, creating any parent directories if they do not exist.\n\n:py:func:`working_dir(dirname, kwargs) `\n This is a Python `Context Manager `_ that makes it easier to work with subdirectories in builds.\n You use this with the Python ``with`` statement to change into a working directory, and when the with block is done, you change back to the original directory.\n Think of it as a safe ``pushd`` / ``popd`` combination, where ``popd`` is guaranteed to be called at the end, even if exceptions are thrown.\n\n Example usage:\n\n #. The ``libdwarf`` build first runs ``configure`` and ``make`` in a subdirectory called ``libdwarf``.\n It then implements the installation code itself.\n This is natural with ``working_dir``:\n\n .. code-block:: python\n\n with working_dir(\"libdwarf\"):\n configure(\"--prefix=\" + prefix, \"--enable-shared\")\n make()\n install(\"libdwarf.a\", prefix.lib)\n\n #. Many CMake builds require that you build \"out of source\", that is, in a subdirectory.\n You can handle creating and ``cd``'ing to the subdirectory like the LLVM package does:\n\n .. code-block:: python\n\n with working_dir(\"spack-build\", create=True):\n cmake(\n \"..\",\n \"-DLLVM_REQUIRES_RTTI=1\",\n \"-DPYTHON_EXECUTABLE=/usr/bin/python\",\n \"-DPYTHON_INCLUDE_DIR=/usr/include/python2.6\",\n \"-DPYTHON_LIBRARY=/usr/lib64/libpython2.6.so\",\n *std_cmake_args\n )\n make()\n make(\"install\")\n\n The ``create=True`` keyword argument causes the command to create the directory if it does not exist.\n\n:py:func:`touch(path) `\n Create an empty file at ``path``.\n\n\n.. _multimethods:\n\nMultimethods and the ``@when`` decorator\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nThe ``@when`` annotation lets packages declare multiple versions of a method that will be called depending on the package's spec.\nThis can be useful to handle cases where configure options are entirely different depending on the version of the package, or when the package is built for different platforms.\n\n.. code-block:: python\n\n class SomePackage(Package):\n ...\n\n @when(\"@:1\")\n def configure_args(self):\n return [\"--old-flag\"]\n\n @when(\"@2:\")\n def configure_args(self):\n return [\"--new-flag\"]\n\nYou can write multiple ``@when`` specs that satisfy the package's spec, for example:\n\n.. code-block:: python\n\n class SomePackage(Package):\n ...\n depends_on(\"mpi\")\n\n def setup_mpi(self):\n # the default, called when no @when specs match\n pass\n\n @when(\"^mpi@3:\")\n def setup_mpi(self):\n # this will be called when mpi is version 3 or higher\n pass\n\n @when(\"^mpi@2:\")\n def setup_mpi(self):\n # this will be called when mpi is version 2 or higher\n pass\n\n @when(\"^mpi@1:\")\n def setup_mpi(self):\n # this will be called when mpi is version 1 or higher\n pass\n\nIn situations like this, the first matching spec, in declaration order, will be called.\nIf no ``@when`` spec matches, the default method (the one without the ``@when`` decorator) will be called.\n\n.. warning::\n\n The default method (without the ``@when`` decorator) should come first in the declaration order.\n If not, it will erase all ``@when`` methods that precede it in the class.\n This is a limitation of decorators in Python.\n\n\n.. _prefix-objects:\n\nPrefix objects\n^^^^^^^^^^^^^^\n\nYou can find the installation directory of package in Spack by using the ``self.prefix`` attribute of the package object.\nIn :ref:`overriding-phases`, we saw that the ``install()`` method has a ``prefix`` argument, which is the same as ``self.prefix``.\nThis variable behaves like a string, but it is actually an instance of the :py:class:`Prefix ` class, which provides some additional functionality to make it easier to work with file paths in Spack.\n\nIn particular, you can use the ``.`` operator to join paths together, creating nested directory structures:\n\n====================== =======================\nPrefix Attribute Location\n====================== =======================\n``prefix.bin`` ``$prefix/bin``\n``prefix.lib64`` ``$prefix/lib64``\n``prefix.share.man`` ``$prefix/share/man``\n``prefix.foo.bar.baz`` ``$prefix/foo/bar/baz``\n====================== =======================\n\nOf course, this only works if your file or directory is a valid Python variable name.\nIf your file or directory contains dashes or dots, use ``join`` instead:\n\n.. code-block:: python\n\n prefix.lib.join(\"libz.a\")\n\n\n.. _environment-variables:\n\nThe build environment\n---------------------\n\nIn Spack the term **build environment** is used somewhat interchangeably to refer to two things:\n\n1. The set of *environment variables* during the build process\n2. The *process* in which the build is executed\n\nSpack creates a separate process for each package build, and every build has its own environment variables.\nChanges in the build environment do not affect the Spack process itself, and they are not visible to other builds.\n\nSpack manages the build environment in the following ways:\n\n1. It cleans the environment variables that may interfere with the build process (e.g. ``CFLAGS``, ``LD_LIBRARY_PATH``, etc.).\n2. It sets a couple of variables for its own use, prefixed with ``SPACK_*``.\n3. It sets a number of standard environment variables like ``PATH`` to make dependencies available during the build.\n4. It sets custom, package specific environment variables defined in the package class of dependencies.\n\nFor this guide, all that matters is to have a rough understanding of which environments you are supposed to set in your package, and which ones are set by Spack automatically.\n\nThe following variables are considered \"standard\" and are managed by Spack:\n\n===================== ====================================================\n``PATH`` Set to point to ``/bin`` directories of dependencies\n``CMAKE_PREFIX_PATH`` Path to dependency prefixes for CMake\n``PKG_CONFIG_PATH`` Path to any pkgconfig directories for dependencies\n===================== ====================================================\n\nOther typical environment variables such as ``CC``, ``CXX`` and ``FC`` are set by the ``compiler-wrapper`` package.\nIn your package, all you need to specify is language dependencies:\n\n.. code-block:: python\n\n class MyPackage(Package):\n depends_on(\"c\", type=\"build\") # ensures CC is set\n depends_on(\"cxx\", type=\"build\") # ensures CXX is set\n depends_on(\"fortran\", type=\"build\") # ensures FC is set\n\nThe ``compiler-wrapper`` package is an \"injected\" dependency by the compiler package (which provides the ``c``, ``cxx``, and ``fortran`` virtuals).\nIt takes care of setting the ``CC``, ``CXX``, and ``FC`` environment variables to the appropriate compiler executables, so you do not need to set them manually in your package.\n\nFor other compiler related environment variables such as ``CFLAGS`` and ``CXXFLAGS``, see :ref:`compiler flags `.\nThis requires a section of its own, because there are multiple ways to deal with compiler flags, and they can come from different sources.\n\n.. _setup-environment:\n\nPackage specific environment variables\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nSpack provides a few methods to help package authors set environment variables programmatically.\nIn total there are four such methods, distinguishing between the build and run environments, and between the package itself and its dependents:\n\n1. :meth:`setup_build_environment(env, spec) `\n2. :meth:`setup_dependent_build_environment(env, dependent_spec) `\n3. :meth:`setup_run_environment(env) `\n4. :meth:`setup_dependent_run_environment(env, dependent_spec) `\n\nAll these methods take an ``env`` argument, which is an instance of the :class:`EnvironmentModifications ` class.\n\nThe ``setup_build_environment`` method is for certain build systems (e.g. ``PythonPackage``) roughly equivalent to the ``configure_args`` or ``cmake_args`` methods.\nIt allows you to set environment variables that are needed during the build of the package itself, and can be used to inform the build system about the package's configuration and where to find dependencies:\n\n.. code-block:: python\n\n class MyPackage(PythonPackage):\n def setup_build_environment(self, env: EnvironmentModifications) -> None:\n env.set(\"ENABLE_MY_FEATURE\", self.spec.satisfies(\"+my_feature\"))\n env.set(\"HDF5_DIR\", self.spec[\"hdf5\"].prefix)\n\nThe ``setup_dependent_build_environment`` method is similar, but it is called for packages that depend on this package.\nThis is often helpful to avoid repetitive configuration in dependent packages.\nAs an example, a package like ``qt`` may want ``QTDIR`` to be set in the build environment of packages that depend on it.\nThis can be done by overriding the ``setup_dependent_build_environment`` method:\n\n.. code-block:: python\n\n class Qt(Package):\n def setup_dependent_build_environment(\n self, env: EnvironmentModifications, dependent_spec: Spec\n ) -> None:\n env.set(\"QTDIR\", self.prefix)\n\nThe ``setup_run_environment`` and ``setup_dependent_run_environment`` are the counterparts for the run environment, primarily used in commands like ``spack load`` and ``spack env activate``.\nDo note however that these runtime environment variables are *also* relevant during the build process, since Spack effectively creates the runtime environment of build dependencies as part of the build process.\nFor example, if a package ``my-pkg`` depends on ``autoconf`` as a build dependency, and ``autoconf`` needs ``perl`` at runtime, then ``perl``'s runtime environment will be set up during the build of ``my-pkg``.\nThe following diagram will give you an idea when each of these methods is called in a build context:\n\n.. image:: images/setup_env.png\n :align: center\n\nNotice that ``setup_dependent_run_environment`` is called once for each dependent package, whereas ``setup_run_environment`` is called only once for the package itself.\nThis means that the former should only be used if the environment variables depend on the dependent package, whereas the latter should be used if the environment variables depend only on the package itself.\n\n.. _setting-package-module-variables:\n\nSetting package module variables\n--------------------------------\n\nApart from modifying environment variables of the dependent package, you can also define Python variables to be used by the dependent.\nThis is done by implementing :meth:`setup_dependent_package `.\nAn example of this can be found in the ``Python`` package:\n\n.. literalinclude:: .spack/spack-packages/repos/spack_repo/builtin/packages/python/package.py\n :pyobject: Python.setup_dependent_package\n :linenos:\n\nThis allows Python packages to directly use these variables:\n\n.. code-block:: python\n\n def install(self, spec, prefix):\n ...\n install(\"script.py\", python_platlib)\n\n.. note::\n\n We recommend using ``setup_dependent_package`` sparingly, as it is not always clear where global variables are coming from when editing a ``package.py`` file.\n\n\n.. _compiler_flags:\n\nCompiler flags\n--------------\n\nSetting compiler flags is a common task, but there are some subtleties that you should be aware of.\nCompiler flags can be set in three different places:\n\n1. The end user, who can set flags directly from the command line with ``spack install pkg cflags=-O3`` variants or :doc:`compiler configuration `.\n In either case, these flags become part of the :ref:`concrete spec `.\n2. The package author, who defines flags in the package class.\n3. The build system itself, which typically has defaults like ``CFLAGS ?= -O2 -g`` or presets like ``CMAKE_BUILD_TYPE=Release``.\n\nThe main challenge for packagers is to ensure that these flags are combined and applied correctly.\n\n.. warning::\n\n A common pitfall when dealing with compiler flags in ``MakefilePackage`` and ``AutotoolsPackage`` is that the user and package author specified flags override the build system defaults.\n This can inadvertently lead to unoptimized builds.\n For example, suppose a user requests ``spack install pkg cflags=-Wno-unused`` and the build system defaults to ``CFLAGS=-O2 -g``.\n If the package takes the user request literally and sets ``CFLAGS=-Wextra`` as an environment variable, then the user-specified flags may *override* the build system defaults, and the build would not be optimized: the ``-O2`` flag would be lost.\n Whether environment variables like ``CFLAGS`` lead to this problem depends on the build system, and may differ from package to package.\n\nBecause of this pitfall, Spack tries to work around the build system and defaults to **injecting compiler flags** through the compiler wrappers.\nThis means that the build system is unaware of the extra compiler flags added by Spack.\nIt also means that package authors typically do not need to deal with user-specified compiler flags when writing their package classes.\n\nHowever, there are two cases in which you may need to deal with compiler flags in your package class explicitly:\n\n1. You need to pass default compiler flags to make a build work.\n This is typical for packages that do not have a configure phase, and requires *you* to set the appropriate flags per compiler.\n2. The build system *needs to be aware* of the user-specified compiler flags to prevent a build failure.\n This is less common, but there are examples of packages that fail to build when ``-O3`` is used for a specific source file.\n\nIn these cases, you can implement the :meth:`flag_handler ` method in your package class.\nThis method has a curious return type, but once you understand it, it is quite powerful.\n\nHere is a simple example:\n\n.. code-block:: python\n\n class MyPackage(MakefilePackage):\n def flag_handler(self, name: str, flags: List[str]):\n if name in (\"cflags\", \"cxxflags\"):\n # Add optimization flags for C/C++\n flags.append(\"-O3\")\n if name == \"fflags\" and self.spec.satisfies(\"%fortran=gcc@14:\"):\n # Add a specific flag for Fortran when using GCC 14 or higher\n flags.append(\"-fallow-argument-mismatch\")\n # Pass these flags to the compiler wrappers\n return (flags, None, None)\n\nThere are multiple things to unpack in this example, so let's go through them step by step.\nThe ``flag_handler`` method is called by Spack once for each of the compiler flags supported in Spack.\n\nThe ``name`` argument\n The ``name`` parameter is a string that indicates which compiler flag is being processed.\n It can be one of the following:\n\n * ``cppflags``: C preprocessor flags (e.g. ``-DMY_DEFINE=1``)\n * ``cflags``: C compilation flags\n * ``cxxflags``: C++ compilation flags\n * ``fflags``: Fortran compilation flags\n * ``ldflags``: Compiler flags for linking (e.g. ``-Wl,-Bstatic``)\n * ``ldlibs``: Libraries to link against (e.g. ``-lfoo``)\n\nThe ``flags`` argument\n The ``flags`` parameter is a list that already contains the user-specified flags, and you can modify it as needed.\n\nReturn value\n The return value determines *how* the flags are applied in the build process.\n It is a triplet that contains the list of flags:\n\n * ``(flags, None, None)``: inject the flags through the Spack **compiler wrappers**.\n This is the default behavior, and it means that the flags are applied directly to the compiler commands without the build system needing to know about them.\n * ``(None, flags, None)``: set these flags in **environment variables** like ``CFLAGS``, ``CXXFLAGS``, etc.\n This requires the build system to use these environment variables.\n * ``(None, None, flags)``: pass these flags **\"on the command line\"** to the build system.\n This requires the build system to support passing flags in this way.\n An example of a build system that supports this is ``CMakePackage``, and Spack will invoke ``cmake -DCMAKE_C_FLAGS=...`` and similar for the other flags.\n\nSpack also allows you to refer to common compiler flags in a more generic way, using the ``self.compiler`` object.\nThis includes flags to set the C and C++ standard, as well as the compiler specific OpenMP flags, etc.\n\n.. code-block:: python\n\n class MyPackage(MakefilePackage):\n def flag_handler(self, name: str, flags: List[str]):\n if name == \"cflags\":\n # Set the C standard to C11\n flags.append(self.compiler.c11_flag)\n elif name == \"cxxflags\":\n # Set the C++ standard to C++17\n flags.append(self.compiler.cxx17_flag)\n return (flags, None, None)\n\nIf you just want to influence how the flags are passed *without setting additional flags* in your package, Spack provides the following shortcut.\nTo ensure that flags are always set as *environment variables*, you can use:\n\n.. code-block:: python\n\n from spack.package import * # for env_flags\n\n\n class MyPackage(MakefilePackage):\n flag_handler = env_flags # Use environment variables for all flags\n\nTo ensure that flags are always *passed to the build system*, you can use:\n\n.. code-block:: python\n\n from spack.package import * # for build_system_flags\n\n\n class MyPackage(MakefilePackage):\n flag_handler = build_system_flags # Pass flags to the build system\n\n\n.. _compiler-wrappers:\n\nCompiler wrappers and flags\n---------------------------\n\nAs mentioned in the :ref:`build environment ` section, any package that depends on a language virtual (``c``, ``cxx``, or ``fortran``) not only gets a specific compiler package like ``gcc`` or ``llvm`` as a dependency, but also automatically gets the ``compiler-wrapper`` package injected as a dependency.\n\nThe ``compiler-wrapper`` package has several responsibilities:\n\n* It sets the ``CC``, ``CXX``, and ``FC`` environment variables in the :ref:`build environment `.\n These variables point to a wrapper executable in the ``compiler-wrapper``'s bin directory, which is a shell script that ultimately invokes the actual, underlying compiler executable.\n* It ensures that three kinds of compiler flags are passed to the compiler when it is invoked:\n\n 1. Flags requested by the user and package author (see :ref:`compiler flags `)\n 2. Flags needed to locate headers and libraries (during the build as well as at runtime)\n 3. Target specific flags, like ``-march=x86-64-v3``, translated from the spec's ``target=`` variant.\n\nAutomatic search flags\n^^^^^^^^^^^^^^^^^^^^^^\n\nThe flags to locate headers and libraries are the following:\n\n* Compile-time library search paths: ``-L$dep_prefix/lib``, ``-L$dep_prefix/lib64``\n* Runtime library search paths (RPATHs): ``-Wl,-rpath,$dep_prefix/lib``, ``-Wl,-rpath,$dep_prefix/lib64``\n* Include search paths: ``-I$dep_prefix/include``\n\nThese flags are added automatically for *each* link-type dependency (and their transitive dependencies) of the package.\nThe exact format of these flags is determined by the compiler being used.\n\nThese automatic flags are particularly useful in build systems such as ``AutotoolsPackage``, ``MakefilePackage`` and certain ``PythonPackage`` packages that also contain C/C++ code.\nTypically configure scripts and Makefiles just work out of the box: the right headers are included and the right libraries are linked to.\n\nFor example, consider a ``libdwarf`` package that just depends on ``libelf`` and specifies it is written in C:\n\n.. code-block:: python\n\n from spack.package import *\n from spack_repo.builtin.build_systems.autotools import AutotoolsPackage\n\n\n class Libdwarf(AutotoolsPackage):\n url = \"...\"\n version(\"1.0\", sha256=\"...\")\n depends_on(\"c\")\n depends_on(\"libelf\")\n\nYou may not even have to implement :ref:`helper methods ` like ``configure_args`` to make it work.\nIn the ``configure`` stage Spack by default simply :ref:`runs ` ``configure(f\"--prefix={prefix}\")``.\nThe configure script picks up the compiler wrapper from the ``CC`` environment variable, and continues to run tests to find the ``libelf`` headers and libraries.\nBecause the compiler wrapper is set up to automatically include the ``-I/include`` and ``-L/lib`` flags, the configure script succeeds and uses the correct ``libelf.h`` header and the ``libelf.so`` library out of the box.\n\n.. _handling_rpaths:\n\nRuntime library search paths\n----------------------------\nSpack heavily makes use of `RPATHs `_ on Linux and macOS to make executables directly runnable after installation.\nExecutables are able to find their needed libraries *without* any of the infamous environment variables such as ``LD_LIBRARY_PATH`` on Linux or ``DYLD_LIBRARY_PATH`` on macOS.\n\nThe :ref:`compiler wrapper ` is the main component that ensures that all binaries built by Spack have the correct RPATHs set.\nAs a package author, you rarely need to worry about RPATHs: the relevant compiler flags are automatically injected through the compiler wrappers, and the build system is blissfully unaware of them.\n\nThis works for most packages and build systems, with the notable exception of CMake, which has its own RPATH handling.\nCMake has its own RPATH handling, and distinguishes between build and install RPATHs.\nBy default, during the build it registers RPATHs to all libraries it links to, so that just-built executables can be run during the build itself.\nUpon installation, these RPATHs are cleared, unless the user defines the install RPATHs.\nIf you use the ``CMakePackage``, Spack automatically sets the ``CMAKE_INSTALL_RPATH_USE_LINK_PATH`` and ``CMAKE_INSTALL_RPATH`` defines to ensure that the install RPATHs are set correctly.\n\nFor packages that do not fit ``CMakePackage`` but still run ``cmake`` as part of the build, it is recommended to look at :meth:`spack_repo.builtin.build_systems.cmake.CMakeBuilder.std_args` on how to set the install RPATHs correctly.\n\n\nMPI support in Spack\n---------------------\n\n.. note::\n\n The MPI support section is somewhat outdated and will be updated in the future.\n\n.. (This is just a comment not rendered in the docs)\n An attempt to update this section showed that Spack's handling of MPI has various issues.\n 1. MPI provider packages tend to set self.spec.mpicc in setup_dependent_package, which is wrong\n because that function is called for every dependent, meaning that mpi's spec is mutated\n repeatedly with possibly different values if the dependent_spec is used.\n 2. The suggestion to fix this was to make the \"interface\" such that a package class defines\n properties like `mpicc`, and dependents would do `self[\"mpi\"].mpicc` to get the package\n attribute instead of the spec attribute.\n 3. While (2) is cleaner, it simply does not work for all MPI providers, because not all strictly\n adhere to the interface. The `msmpi` package notably does not have mpicc wrappers, and\n currently sets `self.spec.mpicc` in `setup_dependent_package` to the C compiler of the\n dependent, which again is wrong because there are many dependents.\n\nIt is common for high-performance computing software/packages to use the Message Passing Interface ( ``MPI``).\nAs a result of concretization, a given package can be built using different implementations of MPI such as ``OpenMPI``, ``MPICH`` or ``IntelMPI``.\nThat is, when your package declares that it ``depends_on(\"mpi\")``, it can be built with any of these ``mpi`` implementations.\nIn some scenarios, to configure a package, one has to provide it with appropriate MPI compiler wrappers such as ``mpicc``, ``mpic++``.\nHowever, different implementations of ``MPI`` may have different names for those wrappers.\n\nSpack provides an idiomatic way to use MPI compilers in your package.\nTo use MPI wrappers to compile your whole build, do this in your ``install()`` method:\n\n.. code-block:: python\n\n env[\"CC\"] = spec[\"mpi\"].mpicc\n env[\"CXX\"] = spec[\"mpi\"].mpicxx\n env[\"F77\"] = spec[\"mpi\"].mpif77\n env[\"FC\"] = spec[\"mpi\"].mpifc\n\nThat's all.\nA longer explanation of why this works is below.\n\nWe don't try to force any particular build method on packagers.\nThe decision to use MPI wrappers depends on the way the package is written, on common practice, and on \"what works\".\nLoosely, there are three types of MPI builds:\n\n1. Some build systems work well without the wrappers and can treat MPI as an external library, where the person doing the build has to supply includes/libs/etc.\n This is fairly uncommon.\n\n2. Others really want the wrappers and assume you're using an MPI \"compiler\" -- i.e., they have no mechanism to add MPI includes/libraries/etc.\n\n3. CMake's ``FindMPI`` needs the compiler wrappers, but it uses them to extract ``-I`` / ``-L`` / ``-D`` arguments, then treats MPI like a regular library.\n\nNote that some CMake builds fall into case 2 because they either don't know about or don't like CMake's ``FindMPI`` support -- they just assume an MPI compiler.\nAlso, some Autotools builds fall into case 3 (e.g., `here is an autotools version of CMake's FindMPI `_).\n\nGiven all of this, we leave the use of the wrappers up to the packager.\nSpack will support all three ways of building MPI packages.\n\nPackaging Conventions\n^^^^^^^^^^^^^^^^^^^^^\n\nAs mentioned above, in the ``install()`` method, ``CC``, ``CXX``, ``F77``, and ``FC`` point to Spack's wrappers around the chosen compiler.\nSpack's wrappers are not the MPI compiler wrappers, though they do automatically add ``-I``, ``-L``, and ``-Wl,-rpath`` args for dependencies in a similar way.\nThe MPI wrappers are a bit different in that they also add ``-l`` arguments for the MPI libraries, and some add special ``-D`` arguments to trigger build options in MPI programs.\n\nFor case 1 above, you generally don't need to do more than patch your Makefile or add configure args as you normally would.\n\nFor case 3, you don't need to do much of anything, as Spack puts the MPI compiler wrappers in the PATH, and the build will find them and interrogate them.\n\nFor case 2, things are a bit more complicated, as you'll need to tell the build to use the MPI compiler wrappers instead of Spack's compiler wrappers.\nAll it takes is some lines like this:\n\n.. code-block:: python\n\n env[\"CC\"] = spec[\"mpi\"].mpicc\n env[\"CXX\"] = spec[\"mpi\"].mpicxx\n env[\"F77\"] = spec[\"mpi\"].mpif77\n env[\"FC\"] = spec[\"mpi\"].mpifc\n\nOr, if you pass CC, CXX, etc. directly to your build with, e.g., ``--with-cc=``, you'll want to substitute ``spec[\"mpi\"].mpicc`` in there instead, e.g.:\n\n.. code-block:: python\n\n configure(\"--prefix=%s\" % prefix, \"--with-cc=%s\" % spec[\"mpi\"].mpicc)\n\nNow, you may think that doing this will lose the includes, library paths, and RPATHs that Spack's compiler wrappers get you, but we've actually set things up so that the MPI compiler wrappers use Spack's compiler wrappers when run from within Spack.\nSo using the MPI wrappers should really be as simple as the code above.\n\n``spec[\"mpi\"]``\n^^^^^^^^^^^^^^^^^^^^^\n\nOkay, so how does all this work?\n\nIf your package has a virtual dependency like ``mpi``, then referring to ``spec[\"mpi\"]`` within ``install()`` will get you the concrete ``mpi`` implementation in your dependency DAG.\nThat is a spec object just like the one passed to install, only the MPI implementations all set some additional properties on it to help you out.\nE.g., in ``openmpi``, you'll find this:\n\n.. literalinclude:: .spack/spack-packages/repos/spack_repo/builtin/packages/openmpi/package.py\n :pyobject: Openmpi.setup_dependent_package\n\nThat code allows the ``openmpi`` package to associate an ``mpicc`` property with the ``openmpi`` spec in the DAG, so that dependents can access it.\n``mvapich2`` and ``mpich`` do similar things.\nSo, no matter what MPI you're using, ``spec[\"mpi\"].mpicc`` gets you the location of the MPI compilers.\nThis allows us to have a fairly simple polymorphic interface for information about virtual dependencies like MPI.\n\nWrapping wrappers\n^^^^^^^^^^^^^^^^^^^^^\n\nSpack likes to use its own compiler wrappers to make it easy to add ``RPATHs`` to builds, and to try hard to ensure that your builds use the right dependencies.\nThis doesn't play nicely by default with MPI, so we have to do a couple of tricks.\n\n1. If we build MPI with Spack's wrappers, ``mpicc`` and friends will be installed with hard-coded paths to Spack's wrappers, and using them from outside of Spack will fail because they only work within Spack.\n To fix this, we patch ``mpicc`` and friends to use the regular compilers.\n Look at the filter_compilers method in ``mpich``, ``openmpi``, or ``mvapich2`` for details.\n\n2. We still want to use the Spack compiler wrappers when Spack is calling ``mpicc``.\n Luckily, wrappers in all mainstream MPI implementations provide environment variables that allow us to dynamically set the compiler to be used by ``mpicc``, ``mpicxx``, etc.\n Spack's build environment sets ``MPICC``, ``MPICXX``, etc. for MPICH derivatives and ``OMPI_CC``, ``OMPI_CXX``, etc. for OpenMPI.\n This makes the MPI compiler wrappers use the Spack compiler wrappers so that your dependencies still get proper RPATHs even if you use the MPI wrappers.\n\nMPI on Cray machines\n^^^^^^^^^^^^^^^^^^^^^\n\nThe Cray programming environment notably uses its own compiler wrappers, which function like MPI wrappers.\nOn Cray systems, the ``CC``, ``cc``, and ``ftn`` wrappers ARE the MPI compiler wrappers, and it's assumed that you'll use them for all of your builds.\nSo on Cray we don't bother with ``mpicc``, ``mpicxx``, etc., Spack MPI implementations set ``spec[\"mpi\"].mpicc`` to point to Spack's wrappers, which wrap the Cray wrappers, which wrap the regular compilers and include MPI flags.\nThat may seem complicated, but for packagers, that means the same code for using MPI wrappers will work, even on a Cray:\n\n.. code-block:: python\n\n env[\"CC\"] = spec[\"mpi\"].mpicc\n\nThis is because on Cray, ``spec[\"mpi\"].mpicc`` is just ``spack_cc``.\n\n.. _packaging-workflow:\n\nPackaging workflow and commands\n-------------------------------\n\nWhen you are building packages, you will likely not get things completely right the first time.\n\nAfter having :doc:`created a package `, the edit-install loop is a common workflow to get the package building correctly:\n\n.. code-block:: console\n\n $ spack edit mypackage\n $ spack install --verbose mypackage\n\nWhenever a build fails, Spack retains the build directory for you to inspect.\nThe location of the build directory is printed in the build output, but you can also find it with the ``spack locate`` command, or navigate to it directly using ``spack cd``:\n\n.. code-block:: console\n\n $ spack locate mypackage\n /tmp/spack-stage/spack-stage-mypackage-1-2-3-abcdef\n\n $ spack cd mypackage\n $ pwd\n /tmp/spack-stage/spack-stage-mypackage-1-2-3-abcdef\n\nInspecting the build environment\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nOnce you have navigated to the build directory after a failed build, you may also want to manually run build commands to troubleshoot the issue.\nThis requires you to have all environment variables exactly set up as they are in the :ref:`build environment `.\n\nThe command\n\n.. code-block:: console\n\n $ spack build-env mypackage -- /bin/sh\n\nis a convenient way to start a subshell with the build environment variables set up.\n\nKeeping the stage directory on success\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nSometimes a build completes successfully, but you encounter issues only when you try to run the installed package.\nIn such cases, it can be useful to keep the build directory area to find out what went wrong.\n\nBy default, ``spack install`` will delete the staging area once a package has been successfully built and installed.\nUse ``--keep-stage`` to leave the build directory intact:\n\n.. code-block:: console\n\n $ spack install --keep-stage \n\nThis allows you to inspect the build directory and potentially debug the build.\n\nOnce done, you could remove all sources and build directories with:\n\n.. code-block:: console\n\n $ spack clean --stage\n\nKeeping the install prefix on failure\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nConversely, if a build fails but *has* installed some files, you may want to keep the install prefix to diagnose the issue.\n\nBy default, ``spack install`` deletes the install directory if anything fails during build.\n\nThe ``--keep-prefix`` option allows you to keep the install prefix regardless of the build outcome.\n\n.. code-block:: console\n\n $ spack install --keep-prefix \n\n.. _cmd-spack-graph:\n\nUnderstanding the DAG\n^^^^^^^^^^^^^^^^^^^^^\n\nSometimes when you are packaging software, it is useful to have a better understanding of the dependency graph of a package.\nThe ``spack spec `` command gives you a good overview of dependencies right on the command line, but the tree structure may not be entirely clear.\nThe ``spack graph `` command can help you visualize the dependency graph better.\n\nBy default it generates an ASCII rendering of a spec's dependency graph, which can be complementary to the output of ``spack spec``.\n\nMuch more powerful is the set of flags ``spack graph --color --dot ...``, which turns the dependency graph into `Dot `_ format.\nTools such as `Graphviz `_ can render this.\nFor example, you can generate a PDF of the dependency graph of a package with the following command:\n\n.. code-block:: console\n\n $ spack graph --dot hdf5 | dot -Tpdf > hdf5.pdf\n\nThere are several online tools that can render Dot files directly in your browser as well.\n\nAnother useful flag is ``spack graph --deptype=...`` which can reduce the size of the graph, by filtering out certain types of dependencies.\nFor example, supplying ``--deptype=link`` will limit to link type dependencies only.\nThe default is ``--deptype=all``, which is equivalent to ``--deptype=build,link,run,test``.\nOptions for ``deptype`` include:\n\n* Any combination of ``build``, ``link``, ``run``, and ``test`` separated by commas.\n* ``all`` for all types of dependencies.\n" }, { "path": "lib/spack/docs/packaging_guide_creation.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n A guide for developers and administrators on how to package software for Spack, covering the structure of a package, creating and editing packages, and defining dependencies and variants.\n\n.. list-table::\n :widths: 25 25 25 25\n :header-rows: 0\n :width: 100%\n\n * - **1. Creation**\n - :doc:`2. Build `\n - :doc:`3. Testing `\n - :doc:`4. Advanced `\n\nPackaging Guide: defining a package\n===================================\n\nThis packaging guide is intended for developers or administrators who want to package software so that Spack can install it.\nIt assumes that you have at least some familiarity with Python, and that you've read the :ref:`basic usage guide `, especially the part about :ref:`specs `.\n\nThere are two key parts of Spack:\n\n#. **Specs**: expressions for describing builds of software, and\n#. **Packages**: Python modules that describe how to build and test software according to a spec.\n\nSpecs allow a user to describe a *particular* build in a way that a package author can understand.\nPackages allow the packager to encapsulate the build logic for different versions, compilers, options, platforms, and dependency combinations in one place.\nEssentially, a package translates a spec into build logic.\nIt also allows the packager to write spec-specific tests of the installed software.\n\nPackages in Spack are written in pure Python, so you can do anything in Spack that you can do in Python.\nPython was chosen as the implementation language for two reasons.\nFirst, Python is ubiquitous in the scientific software community.\nSecond, it has many powerful features to help make package writing easy.\n\n.. _setting-up-for-package-development:\n\n\nSetting up for package development\n----------------------------------\n\nFor developing new packages or working with existing ones, it's helpful to have the ``spack/spack-packages`` repository in a convenient location like your home directory, rather than the default ``~/.spack/package_repos//``.\n\nIf you plan to contribute changes back to Spack, we recommend creating a fork of the `packages repository `_.\nSee `GitHub's fork documentation `_ for details.\nOnce you have a fork, clone it:\n\n.. code-block:: console\n\n $ git clone --depth=100 git@github.com:YOUR-USERNAME/spack-packages.git ~/spack-packages\n $ cd ~/spack-packages\n $ git remote add --track develop upstream git@github.com:spack/spack-packages.git\n\nThen configure Spack to use your local repository:\n\n.. code-block:: console\n\n $ spack repo set --destination ~/spack-packages builtin\n\nBefore starting work, it's useful to create a new branch in your local repository.\n\n.. code-block:: console\n\n $ git checkout -b add-my-package\n\nLastly, verify that Spack is picking up the right repository by checking the location of a known package, like ``zlib``:\n\n.. code-block:: console\n\n $ spack location --package-dir zlib\n /home/your-username/spack-packages/repos/spack_repo/builtin/packages/zlib\n\nWith this setup, you can conveniently access the package files, and contribute changes back to Spack.\n\nStructure of a package\n----------------------\n\nA Spack package is a Python module ``package.py`` stored in a package repository.\nIt contains a package class and sometimes a builder class that define its metadata and build behavior.\n\nThe typical structure of a package is as follows:\n\n.. code-block:: python\n\n # spack_repo/builtin/packages/example/package.py\n\n # import of package / builder classes\n from spack_repo.builtin.build_systems.cmake import CMakePackage\n\n # import Package API\n from spack.package import *\n\n\n class Example(CMakePackage):\n \"\"\"Example package\"\"\" # package description\n\n # Metadata and Directives\n homepage = \"https://example.com\"\n url = \"https://example.com/example/v2.4.0.tar.gz\"\n\n maintainers(\"github_user1\", \"github_user2\")\n\n license(\"UNKNOWN\", checked_by=\"github_user1\")\n\n # version directives listed in order with the latest first\n version(\"2.4.0\", sha256=\"845ccd79ed915fa2dedf3b2abde3fffe7f9f5673cc51be88e47e6432bd1408be\")\n version(\"2.3.0\", sha256=\"cd3274e0abcbc2dfb678d87595e9d3ab1c6954d7921d57a88a23cf4981af46c9\")\n\n # variant directives expose build options\n variant(\"feature\", default=False, description=\"Enable a specific feature\")\n variant(\"codec\", default=False, description=\"Build the CODEC executables\")\n\n # dependency directives declare required software\n depends_on(\"cxx\", type=\"build\")\n depends_on(\"libfoo\", when=\"+feature\")\n\n # Build Instructions\n def cmake_args(self):\n return [\n self.define_from_variant(\"BUILD_CODEC\", \"codec\"),\n self.define(\"EXAMPLE_OPTIMIZED\", False),\n self.define(\"BUILD_THIRDPARTY\", False),\n ]\n\nThe package class is named after the package, and can roughly be divided into two parts:\n\n* **metadata and directives**: attributes and directives that describe the package, such as its homepage, maintainers, license, variants, and dependencies.\n This is the declarative part of the package.\n* **build instructions**: methods that define how to build and install the package, such as `cmake_args()`.\n This is the imperative part of the package.\n\nIn this part of the packaging guide we will cover the **metadata and directives** in detail.\nIn the :doc:`second part `, we will cover the **build instructions**, including how to write custom build logic for different build systems.\n\nPackage Names and the Package Directory\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nPackages are referred to by their **package names**, whether it's on the command line or in a package recipe.\nPackage names can contain lowercase letters, numbers, and dashes.\nEvery package lives as a ``package.py`` file in a **package directory** inside a :ref:`package repository `.\nUsually the package name coincides with the directory name on the filesystem: the ``libelf`` package corresponds to the ``libelf/package.py`` file.\n\n.. note::\n\n **Package name to directory mapping**.\n There is a one to one mapping between package names and package directories.\n Usually the mapping is trivial: the package name is the same as the directory name.\n However, there are a few exceptions to this rule:\n\n 1. Hyphens in package names are replaced by underscores in directory names.\n For example, the package name ``py-numpy`` maps to ``py_numpy/package.py``.\n 2. Names starting with numbers get an underscore prefix.\n For example, the package name ``7zip`` maps to ``_7zip/package.py``.\n 3. Package names that are reserved keywords in Python are also prefixed with an underscore.\n For example, the package name ``pass`` maps to ``_pass/package.py``.\n\n This ensures that every package directory is a valid Python module name.\n\n\nPackage class names\n^^^^^^^^^^^^^^^^^^^\n\nSpack loads ``package.py`` files dynamically, and it needs to find a special class name in the file for the load to succeed.\nThe **package class** is formed by converting words separated by ``-`` in the package name to CamelCase.\nIf the package name starts with a number, we prefix the class name with ``_``.\nHere are some examples:\n\n================= =================\n Package Name Class Name\n================= =================\n ``foo-bar`` ``FooBar``\n ``3proxy`` ``_3proxy``\n================= =================\n\nIn general, you won't have to remember this naming convention because :ref:`cmd-spack-create` and :ref:`cmd-spack-edit` handle the details for you.\n\n.. _creating-and-editing-packages:\n\nCreating and editing packages\n-----------------------------\n\nSpack has various commands that help you create and edit packages.\nSpack can create the boilerplate for new packages and open them in your editor for you to fill in.\nIt can also help you edit existing packages, so you don't have to navigate to the package directory manually.\n\n\n.. _controlling-the-editor:\n\nControlling the editor\n^^^^^^^^^^^^^^^^^^^^^^\n\nWhen Spack needs to open an editor for you (e.g., for commands like :ref:`cmd-spack-create` or :ref:`cmd-spack-edit`), it looks at several environment variables to figure out what to use.\nThe order of precedence is:\n\n* ``SPACK_EDITOR``: highest precedence, in case you want something specific for Spack;\n* ``VISUAL``: standard environment variable for full-screen editors like ``vim`` or ``emacs``;\n* ``EDITOR``: older environment variable for your editor.\n\nYou can set any of these to the command you want to run, e.g., in ``bash`` you might run one of these:\n\n.. code-block:: console\n\n $ export VISUAL=vim\n $ export EDITOR=\"emacs -nw\"\n $ export SPACK_EDITOR=nano\n\nIf Spack finds none of these variables set, it will look for ``vim``, ``vi``, ``emacs``, ``nano``, and ``notepad``, in that order.\n\n.. _cmd-spack-create:\n\nCreating new packages\n^^^^^^^^^^^^^^^^^^^^^\n\nTo create a new package, Spack provides a command that generates a ``package.py`` file in an existing repository, with a boilerplate package template.\nHere's an example:\n\n.. code-block:: console\n\n $ spack create https://gmplib.org/download/gmp/gmp-6.1.2.tar.bz2\n\nSpack examines the tarball URL and tries to figure out the name of the package to be created.\nIf the name contains uppercase letters, these are automatically converted to lowercase.\nIf the name contains underscores or periods, these are automatically converted to dashes.\n\nSpack also searches for *additional* versions located in the same directory on the website.\nSpack prompts you to tell you how many versions it found and asks you how many you would like to download and checksum:\n\n.. code-block:: console\n\n $ spack create https://gmplib.org/download/gmp/gmp-6.1.2.tar.bz2\n ==> This looks like a URL for gmp\n ==> Found 16 versions of gmp:\n\n 6.1.2 https://gmplib.org/download/gmp/gmp-6.1.2.tar.bz2\n 6.1.1 https://gmplib.org/download/gmp/gmp-6.1.1.tar.bz2\n 6.1.0 https://gmplib.org/download/gmp/gmp-6.1.0.tar.bz2\n ...\n 5.0.0 https://gmplib.org/download/gmp/gmp-5.0.0.tar.bz2\n\n How many would you like to checksum? (default is 1, q to abort)\n\nSpack will automatically download the number of tarballs you specify (starting with the most recent) and checksum each of them.\n\nYou do not *have* to download all of the versions up front.\nYou can always choose to download just one tarball initially, and run :ref:`cmd-spack-checksum` later if you need more versions.\n\nSpack automatically creates a directory in the appropriate repository, generates a boilerplate template for your package, and opens up the new ``package.py`` in your favorite ``$EDITOR`` (see :ref:`controlling-the-editor` for details):\n\n.. code-block:: python\n :linenos:\n\n # Copyright Spack Project Developers. See COPYRIGHT file for details.\n #\n # SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n # ----------------------------------------------------------------------------\n # If you submit this package back to Spack as a pull request,\n # please first remove this boilerplate and all FIXME comments.\n #\n # This is a template package file for Spack. We've put \"FIXME\"\n # next to all the things you'll want to change. Once you've handled\n # them, you can save this file and test your package like this:\n #\n # spack install gmp\n #\n # You can edit this file again by typing:\n #\n # spack edit gmp\n #\n # See the Spack documentation for more information on packaging.\n # ----------------------------------------------------------------------------\n import spack_repo.builtin.build_systems.autotools\n from spack.package import *\n\n\n class Gmp(AutotoolsPackage):\n \"\"\"FIXME: Put a proper description of your package here.\"\"\"\n\n # FIXME: Add a proper url for your package's homepage here.\n homepage = \"https://www.example.com\"\n url = \"https://gmplib.org/download/gmp/gmp-6.1.2.tar.bz2\"\n\n # FIXME: Add a list of GitHub accounts to\n # notify when the package is updated.\n # maintainers(\"github_user1\", \"github_user2\")\n\n # FIXME: Add the SPDX identifier of the project's license below.\n # See https://spdx.org/licenses/ for a list. Upon manually verifying\n # the license, set checked_by to your Github username.\n license(\"UNKNOWN\", checked_by=\"github_user1\")\n\n version(\"6.2.1\", sha256=\"eae9326beb4158c386e39a356818031bd28f3124cf915f8c5b1dc4c7a36b4d7c\")\n\n # FIXME: Add dependencies if required.\n # depends_on(\"foo\")\n\n def configure_args(self):\n # FIXME: Add arguments other than --prefix\n # FIXME: If not needed delete the function\n args = []\n return args\n\nThe tedious stuff (creating the class, checksumming archives) has been done for you.\nSpack correctly detected that ``gmp`` uses the ``autotools`` build system, so it created a new ``Gmp`` package that subclasses the ``AutotoolsPackage`` base class.\n\nThe default installation procedure for a package subclassing the ``AutotoolsPackage`` is to go through the typical process of:\n\n.. code-block:: bash\n\n ./configure --prefix=/path/to/installation/directory\n make\n make check\n make install\n\nFor most Autotools packages, this is sufficient.\nIf you need to add additional arguments to the ``./configure`` call, add them via the ``configure_args`` function.\n\nIn the generated package, the download ``url`` attribute is already set.\nAll the things you still need to change are marked with ``FIXME`` labels.\nYou can delete the commented instructions between the Spack license and the first import statement after reading them.\nThe remaining tasks to complete are as follows:\n\n#. Add a description.\n\n Immediately inside the package class is a *docstring* in triple-quotes (``\"\"\"``).\n It is used to generate the description shown when users run ``spack info``.\n\n#. Change the ``homepage`` to a useful URL.\n\n The ``homepage`` is displayed when users run ``spack info`` so that they can learn more about your package.\n\n#. Add a comma-separated list of maintainers.\n\n Add a list of GitHub accounts of people who want to be notified any time the package is modified.\n See :ref:`maintainers`.\n\n#. Change the ``license`` to the correct license.\n\n The ``license`` is displayed when users run ``spack info`` so that they can learn more about your package.\n See :ref:`package_license`.\n\n#. Add ``depends_on()`` calls for the package's dependencies.\n\n ``depends_on`` tells Spack that other packages need to be built and installed before this one.\n See :ref:`dependencies`.\n\n#. Get the installation working.\n\n Your new package may require specific flags during ``configure``.\n These can be added via ``configure_args``.\n If no arguments are needed at this time, change the implementation to ``return []``.\n Specifics will differ depending on the package and its build system.\n :ref:`installation_process` is covered in detail later.\n\nFurther package creation options\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nIf you do not have a tarball URL, you can still use ``spack create`` to generate the boilerplate for a package.\n\n.. code-block:: console\n\n $ spack create --name intel\n\nThis will create a simple ``intel`` package with an ``install()`` method that you can craft to install your package.\nLikewise, you can force the build system to be used with ``--template`` and, in case it's needed, you can overwrite a package already in the repository with ``--force``:\n\n.. code-block:: console\n\n $ spack create --name gmp https://gmplib.org/download/gmp/gmp-6.1.2.tar.bz2\n $ spack create --force --template autotools https://gmplib.org/download/gmp/gmp-6.1.2.tar.bz2\n\nA complete list of available build system templates can be found by running ``spack create --help``.\n\n.. _cmd-spack-edit:\n\nEditing existing packages\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nOne of the easiest ways to learn how to write packages is to look at existing ones.\nYou can open an existing package in your editor using the ``spack edit`` command:\n\n.. code-block:: console\n\n $ spack edit gmp\n\nIf you used ``spack create`` to create a package, you can get back to it later with ``spack edit``.\nThe ``spack edit`` command saves you the trouble of figuring out the package location and navigating to it.\nIf needed, you can still find the package location using the ``spack location`` command:\n\n.. code-block:: console\n\n $ spack location --package-dir gmp\n ~/spack-packages/repos/spack_repo/builtin/packages/gmp/\n\nand with shell support enabled, you can also enter to the package directory:\n\n.. code-block:: console\n\n $ spack cd --package-dir gmp\n\nIf you want to edit multiple packages at once, you can run\n\n.. code-block:: console\n\n $ spack edit\n\nwithout specifying a package name, which will open the directory containing all the packages in your editor.\n\nFinally, the commands ``spack location --repo`` and ``spack cd --repo`` help you navigate to the root of the package repository.\n\nSource code and versions\n------------------------\n\nSpack packages are designed to be built from source code.\nTypically every package version has a corresponding source code archive, which Spack downloads and verifies before building the package.\n\n.. _versions-and-fetching:\n\nVersions and URLs\n^^^^^^^^^^^^^^^^^\n\nThe most straightforward way to add new versions to your package is to add a line like this in the package class:\n\n.. code-block:: python\n\n class Foo(Package):\n\n url = \"http://example.com/foo-8.2.1.tar.gz\"\n\n version(\"8.2.1\", sha256=\"85f477fdd6f8194ab6a0e7afd1cb34eae46c775278d5db9d7ebc9ddaf50c23b1\")\n version(\"8.2.0\", sha256=\"427b2e244e73385515b8ad4f75358139d44a4c792d9b26ddffe2582835cedd8c\")\n version(\"8.1.2\", sha256=\"67630a20f92ace137e68b67f13010487a03e4f036cdd328e199db85d24a434a4\")\n\n.. note::\n\n By convention, we list versions in descending order, from newest to oldest.\n\n.. note::\n\n :ref:`Bundle packages ` do not have source code so there is nothing to fetch.\n Consequently, their version directives consist solely of the version name (e.g., ``version(\"202309\")``).\n\n\nNotice how you only have to specify the URL once, in the ``url`` field.\nSpack is smart enough to extrapolate the URL for each version based on the version number and download version ``8.2.0`` of the ``Foo`` package above from ``http://example.com/foo-8.2.0.tar.gz``.\n\nIf the URL is particularly complicated or changes based on the release, you can override the default URL generation algorithm by defining your own :py:meth:`~spack.package.PackageBase.url_for_version` function.\nFor example, the download URL for OpenMPI contains the ``major.minor`` version in one spot and the ``major.minor.patch`` version in another:\n\n.. code-block:: text\n\n https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.1.1.tar.bz2\n\nIn order to handle this, you can define a ``url_for_version()`` function like so:\n\n.. literalinclude:: .spack/spack-packages/repos/spack_repo/builtin/packages/openmpi/package.py\n :pyobject: Openmpi.url_for_version\n\nWith the use of this ``url_for_version()``, Spack knows to download OpenMPI ``2.1.1`` from\n\n.. code-block:: text\n\n http://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.1.1.tar.bz2\n\nbut download OpenMPI ``1.10.7`` from\n\n.. code-block:: text\n\n http://www.open-mpi.org/software/ompi/v1.10/downloads/openmpi-1.10.7.tar.bz2\n\nYou'll notice that OpenMPI's ``url_for_version()`` function makes use of a special ``Version`` function called ``up_to()``.\nWhen you call ``version.up_to(2)`` on a version like ``1.10.0``, it returns ``1.10``.\n``version.up_to(1)`` would return ``1``.\nThis can be very useful for packages that place all ``X.Y.*`` versions in a single directory and then places all ``X.Y.Z`` versions in a sub-directory.\n\nThere are a few ``Version`` properties you should be aware of.\nWe generally prefer numeric versions to be separated by dots for uniformity, but not all tarballs are named that way.\nFor example, ``icu4c`` separates its major and minor versions with underscores, like ``icu4c-57_1-src.tgz``.\nThe value ``57_1`` can be obtained with the use of the ``version.underscored`` property.\nThere are other separator properties as well:\n\n=================== ======\nProperty Result\n=================== ======\nversion.dotted 1.2.3\nversion.dashed 1-2-3\nversion.underscored 1_2_3\nversion.joined 123\n=================== ======\n\n.. note::\n\n Python properties don't need parentheses.\n ``version.dashed`` is correct.\n ``version.dashed()`` is incorrect.\n\nIn addition, these version properties can be combined with ``up_to()``.\nFor example:\n\n.. code-block:: pycon\n\n >>> version = Version(\"1.2.3\")\n >>> version.up_to(2).dashed\n Version(\"1-2\")\n >>> version.underscored.up_to(2)\n Version(\"1_2\")\n\n\nAs you can see, order is not important.\nJust keep in mind that ``up_to()`` and the other version properties return ``Version`` objects, not strings.\n\nIf a URL cannot be derived systematically, or there is a special URL for one of its versions, you can add an explicit URL for a particular version:\n\n.. code-block:: python\n\n version(\n \"8.2.1\",\n sha256=\"91ee5e9f42ba3d34e414443b36a27b797a56a47aad6bb1e4c1769e69c77ce0ca\",\n url=\"http://example.com/foo-8.2.1-special-version.tar.gz\",\n )\n\n\nWhen you supply a custom URL for a version, Spack uses that URL *verbatim* and does not perform extrapolation.\nThe order of precedence of these methods is:\n\n#. package-level ``url``\n#. ``url_for_version()``\n#. version-specific ``url``\n\nso if your package contains a ``url_for_version()``, it can be overridden by a version-specific ``url``.\n\nIf your package does not contain a package-level ``url`` or ``url_for_version()``, Spack can determine which URL to download from even if only some of the versions specify their own ``url``.\nSpack will use the nearest URL *before* the requested version.\nThis is useful for packages that have an easy to extrapolate URL, but keep changing their URL format every few releases.\nWith this method, you only need to specify the ``url`` when the URL changes.\n\n.. _checksum-verification:\n\nChecksum verification\n^^^^^^^^^^^^^^^^^^^^^\n\nIn the above example we see that each version is associated with a ``sha256`` checksum.\nSpack uses these checksums to verify that downloaded source code has not been modified, corrupted or compromised.\nTherefore, Spack requires that all URL downloads have a checksum, and refuses to install packages when checksum verification fails.\n\n.. note::\n\n While this requirement can be disabled for development with ``spack install --no-checksum``, it is **not recommended**.\n\n.. warning::\n\n **Trusted Downloads.**\n It is critical from a security and reproducibility standpoint that Spack be able to verify the downloaded source.\n This is accomplished using a hash.\n\n For URL downloads, Spack supports multiple cryptographic hash algorithms, including ``sha256`` (recommended), ``sha384`` and ``sha512``.\n See :ref:`version urls ` for more information.\n\n For repository downloads, which we will cover in more detail later, this is done by specifying a **full commit hash** (e.g., :ref:`git `, :ref:`hg `).\n\n\n.. _cmd-spack-checksum:\n\nAutomatically adding new versions\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nThe ``spack checksum`` command can be used to automate the process of adding new versions to a package, assuming the package's download URLs follow a consistent pattern.\n\n``spack checksum``\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nUsing ``spack checksum`` is straightforward:\n\n.. code-block:: console\n\n $ spack checksum libelf\n ==> Found 16 versions of libelf.\n 0.8.13 http://www.mr511.de/software/libelf-0.8.13.tar.gz\n 0.8.12 http://www.mr511.de/software/libelf-0.8.12.tar.gz\n 0.8.11 http://www.mr511.de/software/libelf-0.8.11.tar.gz\n 0.8.10 http://www.mr511.de/software/libelf-0.8.10.tar.gz\n 0.8.9 http://www.mr511.de/software/libelf-0.8.9.tar.gz\n 0.8.8 http://www.mr511.de/software/libelf-0.8.8.tar.gz\n 0.8.7 http://www.mr511.de/software/libelf-0.8.7.tar.gz\n 0.8.6 http://www.mr511.de/software/libelf-0.8.6.tar.gz\n 0.8.5 http://www.mr511.de/software/libelf-0.8.5.tar.gz\n ...\n 0.5.2 http://www.mr511.de/software/libelf-0.5.2.tar.gz\n\n How many would you like to checksum? (default is 1, q to abort)\n\nThis does the same thing that ``spack create`` does, but it allows you to go back and add new versions easily as you need them (e.g., as they're released).\nIt fetches the tarballs you ask for and prints out a list of ``version`` commands ready to copy/paste into your package file:\n\n.. code-block:: console\n\n ==> Checksummed new versions of libelf:\n version(\"0.8.13\", sha256=\"ec6ddbe4b1ac220244230b040fd6a5a102a96337603e703885848ff64cb582a5\")\n version(\"0.8.12\", sha256=\"46db404a287b3d17210b4183cbc7055d7b8bbcb15957daeb51f2dc06002ca8a3\")\n version(\"0.8.11\", sha256=\"e5be0f5d199ad11fbc74e59a8e120cc8b6fbcadaf1827c4e8e6a133ceaadbc4c\")\n version(\"0.8.10\", sha256=\"f1708dd17a476a7abaf6c395723e0745ba8f6b196115513b6d8922d4b5bfbab4\")\n\n.. note::\n\n ``spack checksum`` assumes that Spack can extrapolate new URLs from an existing URL in the package, and that Spack can find similar URLs on a webpage.\n If that's not possible, e.g., if the package's developers don't name their source archive consistently, you'll need to manually add ``version`` calls yourself.\n\nBy default, Spack will search for new versions by scraping the parent URL component of the source archive you gave it in the ``url`` attribute.\nSo, if the sources are at ``http://example.com/downloads/foo-1.0.tar.gz``, Spack computes a *list URL* from it ``http://example.com/downloads/``, and scans that for links to other versions of the package.\nIf you need to search another path for download links, you can supply some extra attributes that control how your package finds new versions.\nSee the documentation on :ref:`attribute_list_url` and :ref:`attribute_list_depth`.\n\n.. _git_version_provenance:\n\nGit Version Provenance\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nChecksummed assets are preferred but there are a few notable exceptions such as git branches and tags i.e ``pkg@develop``.\nThese versions do not naturally have source provenance because they refer to a range of commits (branches) or can be changed outside the spack packaging infrastructure (tags).\nWithout source provenance we cannot have full provenance.\n\nSpack has a reserved variant to allow users to complete provenance for these cases: ``pkg@develop commit=``.\nThe ``commit`` variant must be supplied using the full 40 character commit SHA.\nUsing a partial commit SHA or assigning the ``commit`` variant to a version that is not using a branch or tag reference will lead to an error during concretization.\n\nSpack will attempt to establish git version provenance by looking up commit SHA's for branch and tag based versions during concretization.\nThere are 3 sources that it uses.\nIn order, they are\n\n1. The local cached downloads (already cached source code for the version needing provenance)\n2. Source mirrors (compressed archives of the source code)\n3. The git url provided in the package definition\n\nIf Spack is unable to determine what the commit should be during concretization a warning will be issued.\nUsers may also specify which commit SHA they want with the spec since it is simply a variant.\nIn this case, or in the case of develop specs (see :ref:`cmd-spack-develop`), Spack will skip attempts to assign the commit SHA automatically.\n\n.. note::\n\n Users wanting to track the latest commits from the internet should utilize ``spack clean --downloads`` prior to concretization to clean out cached downloads that will short-circuit internet queries.\n Disabling source mirrors or ensuring they don't contain branch/tag based versions will also be necessary.\n\n Above all else, the most robust way to ensure binaries have their desired commits is to provide the SHAs via user-specs or config i.e. ``commit=``.\n\n\n\n.. _attribute_list_url:\n\n``list_url``\n\"\"\"\"\"\"\"\"\"\"\"\"\n\nThis optional attribute can be set to tell Spack where to scan for links to other versions of the package.\nFor example, the following package has a ``list_url`` attribute that points to a page listing all available versions of the package:\n\n.. code-block:: python\n :linenos:\n\n class Example(Package):\n homepage = \"http://www.example.com\"\n url = \"http://www.example.com/libexample-1.2.3.tar.gz\"\n list_url = \"http://www.example.com/downloads/all-versions.html\"\n\n.. _attribute_list_depth:\n\n``list_depth``\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nMany packages have a listing of available versions on a single webpage, but not all do.\nFor example, ``mpich`` has a tarball URL that looks like this:\n\n.. code-block:: python\n\n url = \"http://www.mpich.org/static/downloads/3.0.4/mpich-3.0.4.tar.gz\"\n\nBut its downloads are a few clicks away from ``http://www.mpich.org/static/downloads/``.\nSo, we need to add a ``list_url`` *and* a ``list_depth`` attribute:\n\n.. code-block:: python\n :linenos:\n\n class Mpich(Package):\n homepage = \"http://www.mpich.org\"\n url = \"http://www.mpich.org/static/downloads/3.0.4/mpich-3.0.4.tar.gz\"\n list_url = \"http://www.mpich.org/static/downloads/\"\n list_depth = 1\n\nBy default, Spack only looks at the top-level page available at ``list_url``.\n``list_depth = 1`` tells it to follow up to 1 level of links from the top-level page.\nNote that here, this implies 1 level of subdirectories, as the ``mpich`` website is structured much like a filesystem.\nBut ``list_depth`` really refers to link depth when spidering the page.\n\n.. _mirrors-of-the-main-url:\n\nMirrors of the main URL\n^^^^^^^^^^^^^^^^^^^^^^^\n\nSpack supports listing mirrors of the main URL in a package by defining the ``urls`` attribute:\n\n.. code-block:: python\n\n class Foo(Package):\n\n urls = [\"http://example.com/foo-1.0.tar.gz\", \"http://mirror.com/foo-1.0.tar.gz\"]\n\ninstead of just a single ``url``.\nThis attribute is a list of possible URLs that will be tried in order when fetching packages.\nNotice that either one of ``url`` or ``urls`` can be present in a package, but not both at the same time.\n\nA well-known case of packages that can be fetched from multiple mirrors is that of GNU.\nFor that, Spack goes a step further and defines a mixin class that takes care of all of the plumbing and requires packagers to just define a proper ``gnu_mirror_path`` attribute:\n\n.. literalinclude:: .spack/spack-packages/repos/spack_repo/builtin/packages/autoconf/package.py\n :lines: 9-18\n\n\n.. _preferred_versions:\n\nPreferring versions over others\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nWhen users install a package without constraining the versions, Spack will typically pick the latest version available.\nUsually this is the desired behavior, but as a packager you may know that the latest version is not mature enough or has known issues that make it unsuitable for production use.\nIn this case, you can mark an older version as preferred using the ``preferred=True`` argument in the ``version`` directive, so that Spack will default to the latest *preferred* version.\n\n.. code-block:: python\n\n class Foo(Package):\n version(\"2.0.0\", sha256=\"...\")\n version(\"1.2.3\", sha256=\"...\", preferred=True)\n\nSee the section on :ref:`version ordering ` for more details and exceptions on how the latest version is computed.\n\n\n.. _deprecate:\n\nDeprecating old versions\n^^^^^^^^^^^^^^^^^^^^^^^^\n\nThere are many reasons to remove old versions of software:\n\n#. Security vulnerabilities (most serious reason)\n#. No longer available for download (right to be forgotten)\n#. Maintainer/developer inability/unwillingness to support old versions\n#. Changing build systems that increase package complexity\n#. Changing dependencies/patches/resources/flags that increase package complexity\n#. Package or version rename\n\nAt the same time, there are many reasons to keep old versions of software:\n\n#. Reproducibility\n#. Requirements for older packages (e.g., some packages still rely on Qt 3)\n\nIn general, you should not remove old versions from a ``package.py`` directly.\nInstead, you should first deprecate them using the following syntax:\n\n.. code-block:: python\n\n version(\"1.2.3\", sha256=\"...\", deprecated=True)\n\n\nThis has two effects.\nFirst, ``spack info`` will no longer advertise that version.\nSecond, commands like ``spack install`` that fetch the package will require user approval:\n\n.. code-block:: spec\n\n $ spack install openssl@1.0.1e\n ==> Warning: openssl@1.0.1e is deprecated and may be removed in a future Spack release.\n ==> Fetch anyway? [y/N]\n\n\nIf you use ``spack install --deprecated``, this check can be skipped.\n\nThis also applies to package recipes that are renamed or removed.\nYou should first deprecate all versions before removing a package.\nIf you need to rename it, you can deprecate the old package and create a new package at the same time.\n\nVersion deprecations should always last at least one release cycle of the builtin package repository before the version is completely removed.\nNo version should be removed without such a deprecation process.\nThis gives users a chance to complain about the deprecation in case the old version is needed for some application.\nIf you require a deprecated version of a package, simply submit a PR to remove ``deprecated=True`` from the package.\nHowever, you may be asked to help maintain this version of the package if the current maintainers are unwilling to support this older version.\n\n\n.. _version-comparison:\n\nVersion ordering\n^^^^^^^^^^^^^^^^\n\nWithout :ref:`version constraints `, :ref:`preferences ` and :ref:`deprecations `, Spack will always pick *the latest* version as defined in the package.\nWhat latest means is determined by the version comparison rules defined in Spack, *not* the order in which versions are listed in the package file.\n\nSpack imposes a generic total ordering on the set of versions, independently from the package they are associated with.\n\nMost Spack versions are numeric, a tuple of integers; for example, ``0.1``, ``6.96``, or ``1.2.3.1``.\nIn this very basic case, version comparison is lexicographical on the numeric components: ``1.2 < 1.2.1 < 1.2.2 < 1.10``.\n\nOther separators for components are also possible, for example ``2025-03-01 < 2025-06``.\n\nSpack can also support string components such as ``1.1.1a`` and ``1.y.0``.\nString components are considered less than numeric components, so ``1.y.0 < 1.0``.\nThis is for consistency with `RPM `_.\nString components do not have to be separated by dots or any other delimiter.\nSo, the contrived version ``1y0`` is identical to ``1.y.0``.\n\nPre-release suffixes also contain string parts, but they are handled in a special way.\nFor example ``1.2.3alpha1`` is parsed as a pre-release of the version ``1.2.3``.\nThis allows Spack to order it before the actual release: ``1.2.3alpha1 < 1.2.3``.\nSpack supports alpha, beta and release candidate suffixes: ``1.2alpha1 < 1.2beta1 < 1.2rc1 < 1.2``.\nAny suffix not recognized as a pre-release is treated as an ordinary string component, so ``1.2 < 1.2-mysuffix``.\n\nFinally, there are a few special string components that are considered \"infinity versions\".\nThey include ``develop``, ``main``, ``master``, ``head``, ``trunk``, and ``stable``, in descending order.\nFor example: ``1.2 < develop``.\nThese are useful for specifying the most recent development version of a package (often a moving target like a git branch), without assigning a specific version number.\nInfinity versions are not automatically used when determining the latest version of a package unless explicitly required by another package or user.\n\nMore formally, the order on versions is defined as follows.\nA version string is split into a list of components based on delimiters such as ``.``, ``-``, ``_``, and string boundaries.\nThe components are split into the **release** and a possible **pre-release** (if the last component is numeric and the second to last is a string ``alpha``, ``beta`` or ``rc``).\nThe release components are ordered lexicographically, with comparison between different types of components as follows:\n\n#. The following special strings are considered larger than any other numeric or non-numeric version component, and satisfy the following order between themselves: ``develop > main > master > head > trunk > stable``.\n\n#. Numbers are ordered numerically, are less than special strings, and larger than other non-numeric components.\n\n#. All other non-numeric components are less than numeric components, and are ordered alphabetically.\n\nFinally, if the release components are equal, the pre-release components are used to break the tie.\n\nThe logic behind this sort order is two-fold:\n\n#. Non-numeric versions are usually used for special cases while developing or debugging a piece of software.\n Keeping most of them less than numeric versions ensures that Spack chooses numeric versions by default whenever possible.\n\n#. The most-recent development version of a package will usually be newer than any released numeric versions.\n This allows the ``@develop`` version to satisfy dependencies like ``depends_on(abc, when=\"@x.y.z:\")``\n\n\n.. _vcs-fetch:\n\nFetching from code repositories\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nFor some packages, source code is provided in a Version Control System (VCS) repository rather than in a tarball.\nSpack can fetch packages from VCS repositories.\nCurrently, Spack supports fetching with :ref:`Git `, :ref:`Mercurial (hg) `, :ref:`Subversion (svn) `, and :ref:`CVS (cvs) `.\nIn all cases, the destination is the standard stage source path.\n\nTo fetch a package from a source repository, Spack needs to know which VCS to use and where to download from.\nMuch like with ``url``, package authors can specify a class-level ``git``, ``hg``, ``svn``, or ``cvs`` attribute containing the correct download location.\n\nMany packages developed with Git have both a Git repository as well as release tarballs available for download.\nPackages can define both a class-level tarball URL and VCS.\nFor example:\n\n.. code-block:: python\n\n class Trilinos(CMakePackage):\n\n homepage = \"https://trilinos.org/\"\n url = \"https://github.com/trilinos/Trilinos/archive/trilinos-release-12-12-1.tar.gz\"\n git = \"https://github.com/trilinos/Trilinos.git\"\n\n version(\"develop\", branch=\"develop\")\n version(\"master\", branch=\"master\")\n version(\"12.12.1\", sha256=\"87428fc522803d31065e7bce3cf03fe475096631e5e07bbd7a0fde60c4cf25c7\")\n version(\"12.10.1\", sha256=\"0263829989b6fd954f72baaf2fc64bc2e2f01d692d4de72986ea808f6e99813f\")\n version(\"12.8.1\", sha256=\"a3a5e715f0cc574a73c3f9bebb6bc24f32ffd5b67b387244c2c909da779a1478\")\n\nIf a package contains both a ``url`` and ``git`` class-level attribute, Spack decides which to use based on the arguments to the ``version()`` directive.\nVersions containing a specific branch, tag, commit or revision are assumed to be for VCS download methods, while versions containing a checksum are assumed to be for URL download methods.\n\nLike ``url``, if a specific version downloads from a different repository than the default repo, it can be overridden with a version-specific argument.\n\n.. note::\n\n In order to reduce ambiguity, each package can only have a single VCS top-level attribute in addition to ``url``.\n In the rare case that a package uses multiple VCS, a fetch strategy can be specified for each version.\n For example, the ``rockstar`` package contains:\n\n .. code-block:: python\n\n class Rockstar(MakefilePackage):\n\n homepage = \"https://bitbucket.org/gfcstanford/rockstar\"\n\n version(\"develop\", git=\"https://bitbucket.org/gfcstanford/rockstar.git\")\n version(\"yt\", hg=\"https://bitbucket.org/MatthewTurk/rockstar\")\n\n\n.. _git-fetch:\n\nGit\n\"\"\"\"\"\"\"\n\nGit fetching supports the following parameters to the ``version`` directive:\n\n* ``git``: URL of the git repository, if different than the class-level ``git``.\n* ``branch``: Name of a :ref:`branch ` to fetch.\n* ``tag``: Name of a :ref:`tag ` to fetch.\n* ``commit``: SHA hash (or prefix) of a :ref:`commit ` to fetch.\n* ``submodules``: Also fetch :ref:`submodules ` recursively when checking out this repository.\n* ``submodules_delete``: A list of submodules to forcibly delete from the repository after fetching.\n Useful if a version in the repository has submodules that have disappeared/are no longer accessible.\n* ``get_full_repo``: Ensure the full git history is checked out with all remote branch information.\n Normally (``get_full_repo=False``, the default), the git option ``--depth 1`` will be used if the version of git and the specified transport protocol support it, and ``--single-branch`` will be used if the version of git supports it.\n* ``git_sparse_paths``: Only clone the provided :ref:`relative paths `.\n\nThe destination directory for the clone is the standard stage source path.\n\n.. note::\n\n ``tag`` and ``branch`` should not be combined in the version parameters.\n\n We strongly recommend that all ``tag`` entries be paired with ``commit``.\n\n\n.. warning::\n\n **Trusted Downloads.**\n It is critical from a security and reproducibility standpoint that Spack be able to verify the downloaded source.\n\n Providing the full ``commit`` SHA hash allows for Spack to preserve provenance for all binaries since git commits are guaranteed to be unique points in the git history.\n Whereas, the mutable nature of branches and tags cannot provide such a guarantee.\n\n A git download *is trusted* only if the full commit SHA is specified.\n Therefore, it is *the* recommended way to securely download from a Git repository.\n\n\n.. _git-default-branch:\n\nDefault branch\n A version with only a name results in fetching a repository's default branch:\n\n .. code-block:: python\n\n class Example(Package):\n\n git = \"https://github.com/example-project/example.git\"\n\n version(\"develop\")\n\n Aside from use of HTTPS, there is no way to verify that the repository has not been compromised.\n Furthermore, the commit you get when you install the package likely won't be the same commit that was used when the package was first written.\n There is also the risk that the default branch may change.\n\n .. warning::\n\n This download method is **untrusted**, and is **not recommended**.\n\n It is better to specify a branch name (see :ref:`below `).\n\n\n.. _git-branches:\n\nBranches\n To fetch a particular branch, use the ``branch`` parameter, preferably with the same name as the version.\n For example,\n\n .. code-block:: python\n\n version(\"main\", branch=\"main\")\n version(\"experimental\", branch=\"experimental\")\n\n Branches are moving targets, which means the commit you get when you install the package likely won't be the one used when the package was first written.\n\n .. note::\n\n Common branch names are special in terms of how Spack determines the latest version of a package.\n See \"infinity versions\" in :ref:`version ordering ` for more information.\n\n .. warning::\n\n This download method is **untrusted**, and is **not recommended** for production installations.\n\n\n.. _git-tags:\n\nTags\n To fetch from a particular tag, use ``tag`` instead:\n\n .. code-block:: python\n\n version(\"1.0.1\", tag=\"v1.0.1\")\n\n While tags are generally more stable than branches, Git allows tags to be moved.\n Many developers use tags to denote rolling releases, and may move the tag when a bug is fixed.\n\n .. warning::\n\n This download method is **untrusted**, and is **not recommended**.\n\n If you must use a ``tag``, it is recommended to combine it with the ``commit`` option (see :ref:`below `).\n\n\n.. _git-commits:\n\nCommits\n To fetch a particular commit, use the ``commit`` argument:\n\n .. code-block:: python\n\n version(\"2014-10-08\", commit=\"1e6ef73d93a28240f954513bc4c2ed46178fa32b\")\n version(\"1.0.4\", tag=\"v1.0.4\", commit=\"420136f6f1f26050d95138e27cf8bc905bc5e7f52\")\n\n It may be useful to provide a saner version for commits like this, e.g., you might use the date as the version, as done in the first example above.\n Or, if you know the commit at which a release was cut, you can use the release version.\n It is up to the package author to decide which of these options makes the most sense.\n\n .. warning::\n\n A git download is *trusted only if* the **full commit sha** is specified.\n\n\n .. hint::\n\n **Avoid using the commit hash as the version.**\n It is not recommended to use the commit hash as the version itself, since it won't sort properly for :ref:`version ordering ` purposes.\n\n\n.. _git-submodules:\n\nSubmodules\n You can supply ``submodules=True`` to cause Spack to fetch submodules recursively along with the repository.\n\n .. code-block:: python\n\n version(\"1.1.0\", commit=\"907d5f40d653a73955387067799913397807adf3\", submodules=True)\n\n If a package needs more fine-grained control over submodules, define ``submodules`` to be a callable function that takes the package instance as its only argument.\n The function needs to return a list of submodules to be fetched.\n\n .. code-block:: python\n\n def submodules(package):\n submodules = []\n if \"+variant-1\" in package.spec:\n submodules.append(\"submodule_for_variant_1\")\n if \"+variant-2\" in package.spec:\n submodules.append(\"submodule_for_variant_2\")\n return submodules\n\n\n class MyPackage(Package):\n version(\"1.1.0\", commit=\"907d5f40d653a73955387067799913397807adf3\", submodules=submodules)\n\n For more information about git submodules see the man page of git: ``man git-submodule``.\n\n\n.. _git-sparse-checkout:\n\nSparse-Checkout\n If you only want to clone a subset of the contents of a git repository, you can supply ``git_sparse_paths`` at the package or version level to utilize git's sparse-checkout feature.\n The paths can be specified through an attribute, property or callable function.\n This option is useful for large repositories containing separate features that can be built independently.\n\n .. note::\n\n This leverages a newer feature in git that requires version ``2.25.0`` or greater.\n\n If ``git_sparse_paths`` is supplied to a git version that is too old then a warning will be issued before standard cloning operations are performed.\n\n .. note::\n\n Paths to directories result in the cloning of *all* of their contents, including the contents of their subdirectories.\n\n The ``git_sparse_paths`` attribute needs to provide a list of relative paths within the repository.\n If using a property -- a function decorated with ``@property`` -- or an argument that is a callable function, the function needs to return a list of paths.\n\n For example, using the attribute approach:\n\n .. code-block:: python\n\n class MyPackage(package):\n # using an attribute\n git_sparse_paths = [\"doe\", \"rae\"]\n\n version(\"1.0.0\")\n version(\"1.1.0\")\n\n results in the files from the top level directory of the repository and the contents of the ``doe`` and ``rae`` relative paths within the repository to be cloned.\n\n Alternatively, you can provide the paths to the version directive argument using a callable function whose return value is a list for paths.\n For example:\n\n .. code-block:: python\n\n def sparse_path_function(package):\n paths = [\"doe\", \"rae\", \"me/file.cpp\"]\n if package.spec.version > Version(\"1.2.0\"):\n paths.extend([\"fae\"])\n return paths\n\n\n class MyPackage(Package):\n version(\"1.1.5\", git_sparse_paths=sparse_path_function)\n version(\"1.2.0\", git_sparse_paths=sparse_path_function)\n version(\"1.2.5\", git_sparse_paths=sparse_path_function)\n version(\"1.1.5\", git_sparse_paths=sparse_path_function)\n\n results in the cloning of the files from the top level directory of the repository, the contents of the ``doe`` and ``rae`` relative paths, *and* the ``me/file.cpp`` file.\n If the package version is greater than ``1.2.0`` then the contents of the ``fae`` relative path will also be cloned.\n\n .. note::\n\n The version directives in the examples above are simplified to emphasize use of this feature.\n Trusted downloads require a hash, such as a :ref:`sha256 ` or :ref:`commit `.\n\n\n.. _github-fetch:\n\nGitHub\n\"\"\"\"\"\"\n\nIf a project is hosted on GitHub, *any* valid Git branch, tag, or hash may be downloaded as a tarball.\nThis is accomplished simply by constructing an appropriate URL.\nSpack can checksum any package downloaded this way, thereby producing a trusted download.\nFor example, the following downloads a particular hash, and then applies a checksum.\n\n.. code-block:: python\n\n version(\n \"1.9.5.1.1\",\n sha256=\"8d74beec1be996322ad76813bafb92d40839895d6dd7ee808b17ca201eac98be\",\n url=\"https://www.github.com/jswhit/pyproj/tarball/0be612cc9f972e38b50a90c946a9b353e2ab140f\",\n )\n\nAlternatively, you could provide the GitHub ``url`` for one version as a property and Spack will extrapolate the URL for other versions as described in :ref:`Versions and URLs `.\n\n\n.. _hg-fetch:\n\nMercurial\n\"\"\"\"\"\"\"\"\"\n\nFetching with Mercurial works much like :ref:`Git `, but you use the ``hg`` parameter.\nThe destination directory is still the standard stage source path.\n\n.. _hg-default-branch:\n\nDefault branch\n Add the ``hg`` attribute with no ``revision`` passed to ``version``:\n\n .. code-block:: python\n\n class Example(Package):\n\n hg = \"https://bitbucket.org/example-project/example\"\n\n version(\"develop\")\n\n As with Git's default fetching strategy, there is no way to verify the integrity of the download.\n\n .. warning::\n\n This download method is **untrusted**, and is **not recommended**.\n\n\n.. _hg-revisions:\n\nRevisions\n To fetch a particular revision, use the ``revision`` parameter:\n\n .. code-block:: python\n\n version(\"1.0\", revision=\"v1.0\")\n\n Unlike ``git``, which has special parameters for different types of revisions, you can use ``revision`` for branches, tags, and **commits** when you fetch with Mercurial.\n\n .. warning::\n\n Like Git, fetching specific branches or tags is an **untrusted** download method, and is **not recommended**.\n\n The recommended fetch strategy is to specify a particular commit hash as the revision.\n\n\n.. _svn-fetch:\n\nSubversion\n\"\"\"\"\"\"\"\"\"\"\n\nTo fetch with subversion, use the ``svn`` and ``revision`` parameters.\nThe destination directory will be the standard stage source path.\n\nFetching the head\n Simply add an ``svn`` parameter to the package:\n\n .. code-block:: python\n\n class Example(Package):\n\n svn = \"https://outreach.scidac.gov/svn/example/trunk\"\n\n version(\"develop\")\n\n .. warning::\n\n This download method is **untrusted**, and is **not recommended** for the same reasons as mentioned above.\n\n\n.. _svn-revisions:\n\nFetching a revision\n To fetch a particular revision, add a ``revision`` argument to the version directive:\n\n .. code-block:: python\n\n version(\"develop\", revision=128)\n\n Unfortunately, Subversion has no commit hashing scheme like Git and Mercurial do, so there is no way to guarantee that the download you get is the same as the download used when the package was created.\n Use at your own risk.\n\n .. warning::\n\n This download method is **untrusted**, and is **not recommended**.\n\n\nSubversion branches are handled as part of the directory structure, so you can check out a branch or tag by changing the URL.\nIf you want to package multiple branches, simply add a ``svn`` argument to each version directive.\n\n\n.. _cvs-fetch:\n\nCVS\n\"\"\"\"\"\"\"\n\nCVS (Concurrent Versions System) is an old centralized version control system.\nIt is a predecessor of Subversion.\n\nTo fetch with CVS, use the ``cvs``, branch, and ``date`` parameters.\nThe destination directory will be the standard stage source path.\n\n.. _cvs-head:\n\nFetching the head\n Simply add a ``cvs`` parameter to the package:\n\n .. code-block:: python\n\n class Example(Package):\n\n cvs = \":pserver:outreach.scidac.gov/cvsroot%module=modulename\"\n\n version(\"1.1.2.4\")\n\n CVS repository locations are described using an older syntax that is different from today's ubiquitous URL syntax.\n ``:pserver:`` denotes the transport method.\n CVS servers can host multiple repositories (called \"modules\") at the same location, and one needs to specify both the server location and the module name to access.\n Spack combines both into one string using the ``%module=modulename`` suffix shown above.\n\n .. warning::\n\n This download method is **untrusted**.\n\n\n.. _cvs-date:\n\nFetching a date\n Versions in CVS are commonly specified by date.\n To fetch a particular branch or date, add a ``branch`` and/or ``date`` argument to the version directive:\n\n .. code-block:: python\n\n version(\"2021.4.22\", branch=\"branchname\", date=\"2021-04-22\")\n\n Unfortunately, CVS does not identify repository-wide commits via a revision or hash like Subversion, Git, or Mercurial do.\n This makes it impossible to specify an exact commit to check out.\n\n .. warning::\n\n This download method is **untrusted**.\n\n\nCVS has more features, but since CVS is rarely used these days, Spack does not support all of them.\n\nSources that are not archives\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nSpack normally expands archives (e.g., ``*.tar.gz`` and ``*.zip``) automatically into a standard stage source directory (``self.stage.source_path``) after downloading them.\nIf you want to skip this step (e.g., for self-extracting executables and other custom archive types), you can add ``expand=False`` to a ``version`` directive.\n\n.. code-block:: python\n\n version(\n \"8.2.1\",\n sha256=\"a2bbdb2de53523b8099b37013f251546f3d65dbe7a0774fa41af0a4176992fd4\",\n url=\"http://example.com/foo-8.2.1-special-version.sh\",\n expand=False,\n )\n\nWhen ``expand`` is set to ``False``, Spack sets the current working directory to the directory containing the downloaded archive before it calls your ``install`` method.\nWithin ``install``, the path to the downloaded archive is available as ``self.stage.archive_file``.\n\nHere is an example snippet for packages distributed as self-extracting archives.\nThe example sets permissions on the downloaded file to make it executable, then runs it with some arguments.\n\n.. code-block:: python\n\n def install(self, spec, prefix):\n set_executable(self.stage.archive_file)\n installer = Executable(self.stage.archive_file)\n installer(\"--prefix=%s\" % prefix, \"arg1\", \"arg2\", \"etc.\")\n\n\nExtra Resources\n^^^^^^^^^^^^^^^\n\nSome packages (most notably compilers) provide optional features if additional resources are expanded within their source tree before building.\nIn Spack it is possible to describe such a need with the ``resource`` directive:\n\n.. code-block:: python\n\n resource(\n name=\"cargo\",\n git=\"https://github.com/rust-lang/cargo.git\",\n tag=\"0.10.0\",\n destination=\"cargo\",\n )\n\nThe arguments are similar to those of the ``versions`` directive.\nThe keyword ``destination`` is relative to the source root of the package and should point to where the resource is to be expanded.\n\nDownload caching\n^^^^^^^^^^^^^^^^\n\nSpack maintains a cache (described :ref:`here `) which saves files retrieved during package installations to avoid re-downloading in the case that a package is installed with a different specification (but the same version) or reinstalled on account of a change in the hashing scheme.\nIn rare cases, it may be necessary to avoid caching for a particular version by adding ``no_cache=True`` as an option to the ``version()`` directive.\nExample situations would be a \"snapshot\"-like Version Control System (VCS) tag, a VCS branch such as ``v6-16-00-patches``, or a URL specifying a regularly updated snapshot tarball.\n\n\n.. _version_constraints:\n\nSpecifying version constraints\n------------------------------\n\nMany Spack directives allow limiting versions to support features such as :ref:`backward and forward compatibility `.\nThese constraints on :ref:`package specs ` are defined using the ``@`` syntax.\n(See :ref:`version-specifier` for more information.)\n\nFor example, the following:\n\n.. code-block:: python\n\n depends_on(\"foo\")\n depends_on(\"python@3\")\n\n conflicts(\"^foo@1.2.3:\", when=\"@:4.5\")\n\nillustrates, in order, three of four forms of version range constraints: implicit, lower bound and upper bound.\nThe fourth form provides lower *and* upper bounds on the version.\n\nIn this example, the implicit range is used to indicate that the package :ref:`depends on ` *any* ``python`` *with* ``3`` *as the major version number* (e.g., ``3.13.5``).\nThe other two range constraints are shown in the :ref:`conflict ` with the dependency package ``foo``.\nThe conflict with ``foo`` *at version* ``1.2.3`` *or newer* is **triggered** for builds of the package at *any version up to and including* ``4.5``.\nFor an example of the fourth form, suppose the dependency in this example had been ``python@3.6:3``.\nIn this case, the package would depend on *any version of* ``python`` *from* ``3.6`` *on so long as the major version number is* ``3``.\n\nWhile you can constrain the spec to a single version -- using the ``@=`` form of ``specifier`` -- **ranges are preferred** even if they would only match a single version currently defined in the package.\nUsing ranges helps avoid overly constrained dependencies, patches, and conflicts.\nThey also come in handy when, for example, users define versions in :ref:`packages-config` that include custom suffixes.\nFor example, if the package defines the version ``1.2.3``, we know from :ref:`version-comparison`, that a user-defined version ``1.2.3-custom`` will satisfy the version constraint ``@1.2.3``.\n\n.. warning::\n\n Specific ``@=`` versions should only be used in **exceptional cases**, such as when the package has a versioning scheme that omits the zero in the first patch release.\n For example, suppose a package defines versions: ``3.1``, ``3.1.1`` and ``3.1.2``.\n Then the specifier ``@=3.1`` is the correct way to select only ``3.1``, whereas ``@3.1`` would be satisfied by all three versions.\n\n\n.. _variants:\n\nVariants\n--------\n\nMany software packages can be configured to enable optional features, which often come at the expense of additional dependencies or longer build times.\nTo be flexible enough and support a wide variety of use cases, Spack allows you to expose to the end-user the ability to choose which features should be activated in a package at the time it is installed.\nThe mechanism to be employed is the :py:func:`~spack.package.variant` directive.\n\nBoolean variants\n^^^^^^^^^^^^^^^^\n\nIn their simplest form, variants are boolean options specified at the package level:\n\n.. code-block:: python\n\n class Hdf5(AutotoolsPackage):\n ...\n variant(\"shared\", default=True, description=\"Builds a shared version of the library\")\n\nwith a default value and a description of their meaning in the package.\n\nWith this variant defined, users can now run ``spack install hdf5 +shared`` and ``spack install hdf5 ~shared`` to enable or disable the ``shared`` feature, respectively.\nSee also the :ref:`basic-variants` for the spec syntax of variants.\n\nOf course, merely defining a variant in a package does not automatically enable or disable any features in the build system.\nAs a packager, you are responsible for translating variants to build system flags or environment variables, to influence the build process.\nWe will see this in action in the next part of the packaging guide, where we talk about :ref:`configuring the build with spec objects `.\n\nOther than influencing the build process, variants are often used to specify optional :ref:`dependencies of a package `.\nFor example, a package may depend on another package only if a certain variant is enabled:\n\n.. code-block:: python\n\n class Hdf5(AutotoolsPackage):\n ...\n variant(\"szip\", default=False, description=\"Enable szip support\")\n depends_on(\"szip\", when=\"+szip\")\n\nIn this case, ``szip`` is modeled as an optional dependency of ``hdf5``, and users can run ``spack install hdf5 +szip`` to enable it.\n\nSingle-valued variants\n^^^^^^^^^^^^^^^^^^^^^^\n\nOther than boolean variants, Spack supports single- and multi-valued variants that can take one or more *string* values.\n\nTo define a *single-valued* variant, simply pass a tuple of possible values to the ``variant`` directive, together with ``multi=False``:\n\n.. code-block:: python\n\n class Blis(Package):\n ...\n variant(\n \"threads\",\n default=\"none\",\n values=(\"pthreads\", \"openmp\", \"none\"),\n multi=False,\n description=\"Multithreading support\",\n )\n\nThis allows users to ``spack install blis threads=openmp``.\n\nIn the example above the argument ``multi=False`` indicates that only a **single value** can be selected at a time.\nThis constraint is enforced by the solver, and an error is emitted if a user specifies two or more values at the same time:\n\n.. code-block:: spec\n\n $ spack spec blis threads=openmp,pthreads\n Input spec\n --------------------------------\n blis threads=openmp,pthreads\n\n Concretized\n --------------------------------\n ==> Error: multiple values are not allowed for variant \"threads\"\n\n.. hint::\n\n In the example above, the value ``threads=none`` is a variant value like any other, and means that *no value is selected*.\n In Spack, all variants have to have a value, so ``none`` was chosen as a *convention* to indicate that no value is selected.\n\nMulti-valued variants\n^^^^^^^^^^^^^^^^^^^^^\n\nLike single-valued variants, multi-valued variants take one or more *string* values, but allow users to select multiple values at the same time.\n\nTo define a *multi-valued* variant, simply pass ``multi=True`` instead:\n\n.. code-block:: python\n\n class Gcc(AutotoolsPackage):\n ...\n variant(\n \"languages\",\n default=\"c,c++,fortran\",\n values=(\"ada\", \"brig\", \"c\", \"c++\", \"fortran\", \"objc\"),\n multi=True,\n description=\"Compilers and runtime libraries to build\",\n )\n\nThis allows users to run ``spack install languages=c,c++``, where the values are separated by commas.\n\n\nAdvanced validation of multi-valued variants\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nAs noted above, the value ``none`` is a value like any other, which raises the question: what if a variant allows multiple values to be selected, *or* none at all?\nNaively, one might think that this can be achieved by simply creating a multi-valued variant that includes the value ``none``:\n\n.. code-block:: python\n\n class Adios(AutotoolsPackage):\n ...\n variant(\n \"staging\",\n values=(\"dataspaces\", \"flexpath\", \"none\"),\n multi=True,\n description=\"Enable dataspaces and/or flexpath staging transports\",\n )\n\nbut this does not prevent users from selecting the nonsensical option ``staging=dataspaces,none``.\n\nIn these cases, more advanced validation logic is required to prevent ``none`` from being selected along with any other value.\nSpack provides two validator functions to help with this, which can be passed to the ``values=`` argument of the ``variant`` directive.\n\nThe first validator function is :py:func:`~spack.package.any_combination_of`, which can be used as follows:\n\n.. code-block:: python\n\n class Adios(AutotoolsPackage):\n ...\n variant(\n \"staging\",\n values=any_combination_of(\"flexpath\", \"dataspaces\"),\n description=\"Enable dataspaces and/or flexpath staging transports\",\n )\n\nThis solves the issue by allowing the user to select either any combination of the values ``flexpath`` and ``dataspaces``, or ``none``.\nIn other words, users can specify ``staging=none`` to select nothing, or any of ``staging=dataspaces``, ``staging=flexpath``, and ``staging=dataspaces,flexpath``.\n\nThe second validator function :py:func:`~spack.package.disjoint_sets` generalizes this idea further:\n\n.. code-block:: python\n\n class Mvapich2(AutotoolsPackage):\n ...\n variant(\n \"process_managers\",\n description=\"List of the process managers to activate\",\n values=disjoint_sets((\"auto\",), (\"slurm\",), (\"hydra\", \"gforker\", \"remshell\"))\n .prohibit_empty_set()\n .with_error(\"'slurm' or 'auto' cannot be activated along with other process managers\")\n .with_default(\"auto\")\n .with_non_feature_values(\"auto\"),\n )\n\nIn this case, examples of valid options are ``process_managers=auto``, ``process_managers=slurm``, and ``process_managers=hydra,remshell``, whereas ``process_managers=slurm,hydra`` is invalid, as it picks values from two different sets.\n\nBoth validator functions return a :py:class:`~spack.variant.DisjointSetsOfValues` object, which defines chaining methods to further customize the behavior of the variant.\n\n.. _variant-conditional-values:\n\nConditional Possible Values\n^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nThere are cases where a variant may take multiple values, and the list of allowed values expands over time.\nConsider, for instance, the C++ standard with which we might compile Boost, which can take one of multiple possible values with the latest standards only available for more recent versions.\n\nTo model a similar situation we can use *conditional possible values* in the variant declaration:\n\n.. code-block:: python\n\n variant(\n \"cxxstd\",\n default=\"98\",\n values=(\n \"98\",\n \"11\",\n \"14\",\n # C++17 is not supported by Boost < 1.63.0.\n conditional(\"17\", when=\"@1.63.0:\"),\n # C++20/2a is not supported by Boost < 1.73.0\n conditional(\"2a\", \"2b\", when=\"@1.73.0:\"),\n ),\n multi=False,\n description=\"Use the specified C++ standard when building.\",\n )\n\n\nThe snippet above allows ``98``, ``11`` and ``14`` as unconditional possible values for the ``cxxstd`` variant, while ``17`` requires a version greater than or equal to ``1.63.0`` and both ``2a`` and ``2b`` require a version greater than or equal to ``1.73.0``.\n\n\nConditional Variants\n^^^^^^^^^^^^^^^^^^^^\n\nAs new versions of packages are released, optional features may be added and removed.\nSometimes, features are only available for a particular platform or architecture.\n\nTo reduce the visual clutter in specs, packages can define variants *conditionally* using a ``when`` clause.\nThe variant will only be present on specs that satisfy this condition.\n\nFor example, the following package defines a variant ``bar`` that exists only when it is at version 2.0 or higher, and a variant ``baz`` that exists only on the Darwin platform:\n\n.. code-block:: python\n\n class Foo(Package):\n ...\n variant(\"bar\", default=False, when=\"@2.0:\", ...)\n variant(\"baz\", default=True, when=\"platform=darwin\", ...)\n\nDo note that conditional variants can also be a source of confusion.\nIn Spack, the absence of a variant is different from it being disabled.\nFor example, a user might run ``spack install foo ~bar``, expecting it to allow version 1.0 (which does not have the ``bar`` feature) and version 2.0 (with the feature disabled).\nHowever, the constraint ``~bar`` tells Spack that the ``bar`` variant *must exist* and be disabled.\nThis forces Spack to select version 2.0 or higher, where the variant is defined.\n\nSticky Variants\n^^^^^^^^^^^^^^^\n\nThe variant directive can be marked as ``sticky`` by setting the corresponding argument to ``True``:\n\n.. code-block:: python\n\n variant(\"bar\", default=False, sticky=True)\n\nA ``sticky`` variant differs from a regular one in that it is always set to either:\n\n#. An explicit value appearing in a spec literal or\n#. Its default value\n\nThe concretizer thus is not free to pick an alternate value to work around conflicts, but will error out instead.\nSetting this property on a variant is useful in cases where the variant allows some dangerous or controversial options (e.g., using unsupported versions of a compiler for a library) and the packager wants to ensure that allowing these options is done on purpose by the user, rather than automatically by the solver.\n\n\nOverriding Variants\n^^^^^^^^^^^^^^^^^^^\n\nPackages may override variants for several reasons, most often to change the default from a variant defined in a parent class or to change the conditions under which a variant is present on the spec.\n\nWhen a variant is defined multiple times, whether in the same package file or in a subclass and a superclass, the last definition is used for all attributes **except** for the ``when`` clauses.\nThe ``when`` clauses are accumulated through all invocations, and the variant is present on the spec if any of the accumulated conditions are satisfied.\n\nFor example, consider the following package:\n\n.. code-block:: python\n\n class Foo(Package):\n ...\n variant(\"bar\", default=False, when=\"@1.0\", description=\"help1\")\n variant(\"bar\", default=True, when=\"platform=darwin\", description=\"help2\")\n ...\n\nThis package ``foo`` has a variant ``bar`` when the spec satisfies either ``@1.0`` or ``platform=darwin``, but not for other platforms at other versions.\nThe default for this variant, when it is present, is always ``True``, regardless of which condition of the variant is satisfied.\nThis allows packages to override variants in packages or build system classes from which they inherit, by modifying the variant values without modifying the ``when`` clause.\nIt also allows a package to implement ``or`` semantics for a variant ``when`` clause by duplicating the variant definition.\n\n.. _dependencies:\n\nDependencies\n------------\n\nWe've covered how to build a simple package, but what if one package relies on another package to build?\nHow do you express that in a package file?\nAnd how do you refer to the other package in the build script for your own package?\n\nSpack makes this relatively easy.\nLet's take a look at the ``libdwarf`` package to see how it's done:\n\n.. code-block:: python\n :emphasize-lines: 9\n :linenos:\n\n class Libdwarf(Package):\n homepage = \"http://www.prevanders.net/dwarf.html\"\n url = \"http://www.prevanders.net/libdwarf-20130729.tar.gz\"\n list_url = homepage\n\n version(\"20130729\", sha256=\"092fcfbbcfca3b5be7ae1b5e58538e92c35ab273ae13664fed0d67484c8e78a6\")\n ...\n\n depends_on(\"libelf\")\n\n def install(self, spec, prefix): ...\n\n``depends_on()``\n^^^^^^^^^^^^^^^^\n\nThe highlighted ``depends_on(\"libelf\")`` call tells Spack that it needs to build and install the ``libelf`` package before it builds ``libdwarf``.\nThis means that in your ``install()`` method, you are guaranteed that ``libelf`` has been built and installed successfully, so you can rely on it for your ``libdwarf`` build.\n\n.. _dependency_specs:\n\nDependency specs\n^^^^^^^^^^^^^^^^\n\n``depends_on`` doesn't just take the name of another package.\nIt can take a full spec as well.\nThis means that you can restrict the versions or other configuration options of ``libelf`` that ``libdwarf`` will build with.\nFor example, suppose that in the ``libdwarf`` package you write:\n\n.. code-block:: python\n\n depends_on(\"libelf@0.8\")\n\nNow ``libdwarf`` will require ``libelf`` in the range ``0.8``, which includes patch versions ``0.8.1``, ``0.8.2``, etc.\nApart from version restrictions, you can also specify variants if this package requires optional features of the dependency.\n\n.. code-block:: python\n\n depends_on(\"libelf@0.8 +parser +pic\")\n\nBoth users *and* package authors use the same spec syntax to refer to different package configurations.\nUsers use the spec syntax on the command line to find installed packages or to install packages with particular constraints, and package authors can use specs to describe relationships between packages.\n\n.. _version_compatibility:\n\nSpecifying backward and forward compatibility\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nPackages are often compatible with a range of versions of their dependencies.\nThis is typically referred to as backward and forward compatibility.\nSpack allows you to specify this in the ``depends_on`` directive using version ranges.\n\n**Backward compatibility** means that the package requires at least a certain version of its dependency:\n\n.. code-block:: python\n\n depends_on(\"python@3.10:\")\n\nIn this case, the package requires Python 3.10 or newer, as specified in the project's :file:`pyproject.toml`.\n\nCommonly, packages drop support for older versions of a dependency as they release new versions.\nIn Spack you can conveniently add every backward compatibility rule as a separate line:\n\n.. code-block:: python\n\n # backward compatibility with Python\n depends_on(\"python@3.8:\")\n depends_on(\"python@3.9:\", when=\"@1.2:\")\n depends_on(\"python@3.10:\", when=\"@1.4:\")\n\nThis means that in general we need Python 3.8 or newer; from version 1.2 onwards we need Python 3.9 or newer; from version 1.4 onwards we need Python 3.10 or newer.\nNotice that it's fine to have overlapping ranges in the ``when`` clauses.\n\n**Forward compatibility** means that the package requires at most a certain version of its dependency.\nForward compatibility rules are necessary when there are breaking changes in the dependency that the package cannot handle.\nIn Spack we often add forward compatibility bounds only at the time a new, breaking version of a dependency is released.\nAs with backward compatibility, it is typical to see a list of forward compatibility bounds in a package file as separate lines:\n\n.. code-block:: python\n\n # forward compatibility with Python\n depends_on(\"python@:3.12\", when=\"@:1.10\")\n depends_on(\"python@:3.13\", when=\"@:1.12\")\n\nNotice how the ``:`` now appears before the version number both in the dependency and in the ``when`` clause.\nThis tells Spack that in general we need Python 3.13 or older up to version ``1.12.x``, and up to version ``1.10.x`` we need Python 3.12 or older.\nSaid differently, forward compatibility with Python 3.13 was added in version 1.11, while version 1.13 added forward compatibility with Python 3.14.\n\nNotice that a version range ``@:3.12`` includes *any* patch version number ``3.12.x``, which is often useful when specifying forward compatibility bounds.\n\nSo far we have seen open-ended version ranges, which is by far the most common use case.\nIt is also possible to specify both a lower and an upper bound on the version of a dependency, like this:\n\n.. code-block:: python\n\n depends_on(\"python@3.10:3.12\")\n\nThere is short syntax to specify that a package is compatible with say any ``3.x`` version:\n\n.. code-block:: python\n\n depends_on(\"python@3\")\n\nThe above is equivalent to ``depends_on(\"python@3:3\")``, which means at least Python version 3 and at most any ``3.x.y`` version.\n\nIn very rare cases, you may need to specify an exact version, for example if you need to distinguish between ``3.2`` and ``3.2.1``:\n\n.. code-block:: python\n\n depends_on(\"pkg@=3.2\")\n\nBut in general, you should try to use version ranges as much as possible, so that custom suffixes are included too.\nThe above example can be rewritten in terms of ranges as follows:\n\n.. code-block:: python\n\n depends_on(\"pkg@3.2:3.2.0\")\n\nA spec can contain a version list of ranges and individual versions separated by commas.\nFor example, if you need Boost 1.59.0 or newer, but there are known issues with 1.64.0, 1.65.0, and 1.66.0, you can say:\n\n.. code-block:: python\n\n depends_on(\"boost@1.59.0:1.63,1.65.1,1.67.0:\")\n\nor, if those particular versions are excluded due to bugs rather than removed and reintroduced features:\n\n.. code-block:: python\n\n depends_on(\"boost@1.59.0:\")\n conflicts(\"^boost@1.64.0,1.65.0,1.66.0\")\n\nAlways specify version ranges with an open-world assumption:\n\n- all \"ground truths\" about exclusions and inclusions (e.g., versions with features added or removed) must satisfy the range, and\n- no potential but unknown versions are excluded from the range.\n\nThis practice avoids overconstraining version ranges, which can lead to concretization errors, and ensures that every version in a package is *meaningful* and not just *incidental* (i.e., based on the version you happened to test).\nIn the above example, the project has presumably documented (with pyproject.toml, CMakeLists.txt, or release notes) that ``@:1.58`` are incompatible, and it is known from testing that ``@1.67`` is compatible.\nIt is *not* known whether future versions ``@1.68:`` are incompatible, so they must be included by the range.\nIf and when future versions are known incompatible, the version range should be constrained with an upper bound.\n\n.. _dependency-types:\n\nDependency types\n^^^^^^^^^^^^^^^^\n\nNot all dependencies are created equal, and Spack allows you to specify exactly what kind of a dependency you need.\nFor example:\n\n.. code-block:: python\n\n depends_on(\"cmake\", type=\"build\")\n depends_on(\"py-numpy\", type=(\"build\", \"run\"))\n depends_on(\"libelf\", type=(\"build\", \"link\"))\n depends_on(\"py-pytest\", type=\"test\")\n\nThe following dependency types are available:\n\n* **build**: the dependency will be added to the ``PATH`` and ``PYTHONPATH`` at build-time.\n* **link**: the dependency will be added to Spack's compiler wrappers, automatically injecting the appropriate linker flags, including ``-I``, ``-L``, and RPATH/RUNPATH handling.\n* **run**: the dependency will be added to the ``PATH`` and ``PYTHONPATH`` at run-time.\n This is true for both ``spack load`` and the module files Spack writes.\n* **test**: the dependency will be added to the ``PATH`` and ``PYTHONPATH`` at build-time.\n The only difference between \"build\" and \"test\" is that test dependencies are only built if the user requests unit tests with ``spack install --test``.\n\nOne of the advantages of the ``build`` dependency type is that although the dependency needs to be installed in order for the package to be built, it can be uninstalled without concern afterwards.\n``link`` and ``run`` disallow this because uninstalling the dependency would break the package.\n\n``build``, ``link``, and ``run`` dependencies all affect the hash of Spack packages (along with ``sha256`` sums of patches and archives used to build the package, and a `canonical hash `_ of the ``package.py`` recipes).\n``test`` dependencies do not affect the package hash, as they are only used to construct a test environment *after* building and installing a given package installation.\nOlder versions of Spack did not include build dependencies in the hash, but this has been `fixed `_ as of |Spack v0.18|_.\n\n.. |Spack v0.18| replace:: Spack ``v0.18``\n.. _Spack v0.18: https://github.com/spack/spack/releases/tag/v0.18.0\n\nIf the dependency type is not specified, Spack uses a default of ``(\"build\", \"link\")``.\nThis is the common case for compiler languages.\nNon-compiled packages like Python modules commonly use ``(\"build\", \"run\")``.\nThis means that the compiler wrappers don't need to inject the dependency's ``prefix/lib`` directory, but the package needs to be in ``PATH`` and ``PYTHONPATH`` during the build process and later when a user wants to run the package.\n\nConditional dependencies\n^^^^^^^^^^^^^^^^^^^^^^^^\n\nYou may have a package that only requires a dependency under certain conditions.\nFor example, you may have a package with optional MPI support.\nYou would then provide a variant to reflect that the feature is optional and specify the MPI dependency only applies when MPI support is enabled.\nIn that case, you could say something like:\n\n.. code-block:: python\n\n variant(\"mpi\", default=False, description=\"Enable MPI support\")\n\n depends_on(\"mpi\", when=\"+mpi\")\n\n\nSuppose that, starting from version 3, the above package also has optional `Trilinos` support.\nFurthermore, you want to ensure that when `Trilinos` support is enabled, the package can be built both with and without MPI.\nFurther suppose you require a version of `Trilinos` no older than 12.6.\nIn that case, the `trilinos` variant and dependency directives would be:\n\n.. code-block:: python\n\n variant(\"trilinos\", default=False, description=\"Enable Trilinos support\")\n\n depends_on(\"trilinos@12.6:\", when=\"@3: +trilinos\")\n depends_on(\"trilinos@12.6: +mpi\", when=\"@3: +trilinos +mpi\")\n\n\nAlternatively, you could use the `when` context manager to equivalently specify the `trilinos` variant dependencies as follows:\n\n.. code-block:: python\n\n with when(\"@3: +trilinos\"):\n depends_on(\"trilinos@12.6:\")\n depends_on(\"trilinos +mpi\", when=\"+mpi\")\n\n\nThe argument to ``when`` in either case can include any Spec constraints that are supported on the command line using the same :ref:`syntax `.\n\n.. note::\n\n If a dependency isn't typically used, you can save time by making it conditional since Spack will not build the dependency unless it is required for the Spec.\n\n\n.. _dependency_dependency_patching:\n\nDependency patching\n^^^^^^^^^^^^^^^^^^^\n\nSome packages maintain special patches on their dependencies, either to add new features or to fix bugs.\nThis typically makes a package harder to maintain, and we encourage developers to upstream (contribute back) their changes rather than maintaining patches.\nHowever, in some cases it's not possible to upstream.\nMaybe the dependency's developers don't accept changes, or maybe they just haven't had time to integrate them.\n\nFor times like these, Spack's ``depends_on`` directive can optionally take a patch or list of patches:\n\n.. code-block:: python\n\n class SpecialTool(Package):\n ...\n depends_on(\"binutils\", patches=\"special-binutils-feature.patch\")\n ...\n\nHere, the ``special-tool`` package requires a special feature in ``binutils``, so it provides an extra ``patches=`` keyword argument.\nThis is similar to the `patch directive `_, with one small difference.\nHere, ``special-tool`` is responsible for the patch, so it should live in ``special-tool``'s directory in the package repository, not the ``binutils`` directory.\n\nIf you need something more sophisticated, you can nest a ``patch()`` directive inside ``depends_on``:\n\n.. code-block:: python\n\n class SpecialTool(Package):\n ...\n depends_on(\n \"binutils\",\n patches=patch(\n \"special-binutils-feature.patch\", level=3, when=\"@:1.3\" # condition on binutils\n ),\n when=\"@2.0:\", # condition on special-tool\n )\n ...\n\nNote that there are two optional ``when`` conditions here -- one on the ``patch`` directive and the other on ``depends_on``.\nThe condition in the ``patch`` directive applies to ``binutils`` (the package being patched), while the condition in ``depends_on`` applies to ``special-tool``.\nSee `patch directive `_ for details on all the arguments the ``patch`` directive can take.\n\nFinally, if you need *multiple* patches on a dependency, you can provide a list for ``patches``, e.g.:\n\n.. code-block:: python\n\n class SpecialTool(Package):\n ...\n depends_on(\n \"binutils\",\n patches=[\n \"binutils-bugfix1.patch\",\n \"binutils-bugfix2.patch\",\n patch(\n \"https://example.com/special-binutils-feature.patch\",\n sha256=\"252c0af58be3d90e5dc5e0d16658434c9efa5d20a5df6c10bf72c2d77f780866\",\n when=\"@:1.3\",\n ),\n ],\n when=\"@2.0:\",\n )\n ...\n\nAs with ``patch`` directives, patches are applied in the order they appear in the package file (or in this case, in the list).\n\n.. note::\n\n You may wonder whether dependency patching will interfere with other packages that depend on ``binutils``.\n It won't.\n\n As described in :ref:`patching`, Patching a package adds the ``sha256`` of the patch to the package's spec, which means it will have a *different* unique hash than other versions without the patch.\n The patched version coexists with unpatched versions, and Spack's support for :ref:`handling_rpaths` guarantees that each installation finds the right version.\n If two packages depend on ``binutils`` patched *the same* way, they can both use a single installation of ``binutils``.\n\n.. _virtual-dependencies:\n\nVirtual dependencies\n--------------------\n\nIn some cases, more than one package can satisfy another package's dependency.\nOne way this can happen is if a package depends on a particular *interface*, but there are multiple *implementations* of the interface, and the package could be built with any of them.\nA *very* common interface in HPC is the `Message Passing Interface (MPI) `_, which is used in many large-scale parallel applications.\n\nMPI has several different implementations (e.g., `MPICH `_, `OpenMPI `_, and `MVAPICH `_) and scientific applications can be built with any one of them.\nMany package managers handle interfaces like this by requiring many variations of the package recipe for each implementation of MPI, e.g., ``foo``, ``foo-mvapich``, ``foo-mpich``.\nIn Spack every package is defined in a single ``package.py`` file, and avoids the combinatorial explosion through *virtual dependencies*.\n\n``provides``\n^^^^^^^^^^^^\n\nIn Spack, ``mpi`` is handled as a *virtual package*.\nA package like ``mpileaks`` can depend on the virtual ``mpi`` just like any other package, by supplying a ``depends_on`` call in the package definition.\nFor example:\n\n.. code-block:: python\n :linenos:\n :emphasize-lines: 7\n\n class Mpileaks(Package):\n homepage = \"https://github.com/hpc/mpileaks\"\n url = \"https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz\"\n\n version(\"1.0\", sha256=\"768c71d785bf6bbbf8c4d6af6582041f2659027140a962cd0c55b11eddfd5e3d\")\n\n depends_on(\"mpi\")\n depends_on(\"adept-utils\")\n depends_on(\"callpath\")\n\nHere, ``callpath`` and ``adept-utils`` are concrete packages, but there is no actual package for ``mpi``, so we say it is a *virtual* package.\nThe syntax of ``depends_on`` is the same for both.\nIf we look inside the package file of an MPI implementation, say MPICH, we'll see something like this:\n\n.. code-block:: python\n\n class Mpich(Package):\n provides(\"mpi\")\n ...\n\nThe ``provides(\"mpi\")`` call tells Spack that the ``mpich`` package can be used to satisfy the dependency of any package that ``depends_on(\"mpi\")``.\n\nProviding multiple virtuals simultaneously\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nPackages can provide more than one virtual dependency.\nSometimes, due to implementation details, there are subsets of those virtuals that need to be provided together by the same package.\n\nA well-known example is ``openblas``, which provides both the ``lapack`` and ``blas`` API in a single ``libopenblas`` library.\nA package that needs ``lapack`` and ``blas`` must either use ``openblas`` to provide both, or not use ``openblas`` at all.\nIt cannot pick one or the other.\n\nTo express this constraint in a package, the two virtual dependencies must be listed in the same ``provides`` directive:\n\n.. code-block:: python\n\n provides(\"blas\", \"lapack\")\n\nThis makes it impossible to select ``openblas`` as a provider for one of the two virtual dependencies and not for the other.\nIf you try to, Spack will report an error:\n\n.. code-block:: spec\n\n $ spack spec netlib-scalapack ^[virtuals=lapack] openblas ^[virtuals=blas] atlas\n ==> Error: concretization failed for the following reasons:\n\n 1. Package 'openblas' needs to provide both 'lapack' and 'blas' together, but provides only 'lapack'\n\nVersioned Interfaces\n^^^^^^^^^^^^^^^^^^^^\n\nJust as you can pass a spec to ``depends_on``, so can you pass a spec to ``provides`` to add constraints.\nThis allows Spack to support the notion of *versioned interfaces*.\nThe MPI standard has gone through many revisions, each with new functions added, and each revision of the standard has a version number.\nSome packages may require a recent implementation that supports MPI-3 functions, but some MPI versions may only provide up to MPI-2.\nOthers may need MPI 2.1 or higher.\nYou can indicate this by adding a version constraint to the spec passed to ``provides``:\n\n.. code-block:: python\n\n provides(\"mpi@:2\")\n\nSuppose that the above ``provides`` call is in the ``mpich2`` package.\nThis says that ``mpich2`` provides MPI support *up to* version 2, but if a package ``depends_on(\"mpi@3\")``, then Spack will *not* build that package with ``mpich2``.\n\nCurrently, names and versions are the only spec components supported for virtual packages.\n\n``provides when``\n^^^^^^^^^^^^^^^^^\n\nThe same package may provide different versions of an interface depending on *its* version.\nAbove, we simplified the ``provides`` call in ``mpich`` to make the explanation easier.\nIn reality, this is how ``mpich`` calls ``provides``:\n\n.. code-block:: python\n\n provides(\"mpi@:3\", when=\"@3:\")\n provides(\"mpi@:1\", when=\"@1:\")\n\nThe ``when`` argument to ``provides`` allows you to specify optional constraints on the *providing* package, or the *provider*.\nThe provider only provides the declared virtual spec when *it* matches the constraints in the ``when`` clause.\nHere, when ``mpich`` is at version 3 or higher, it provides MPI up to version 3.\nWhen ``mpich`` is at version 1 or higher, it provides the MPI virtual package at version 1.\n\nThe ``when`` qualifier ensures that Spack selects a suitably high version of ``mpich`` to satisfy some other package that ``depends_on`` a particular version of MPI.\nIt will also prevent a user from building with too low a version of ``mpich``.\nFor example, suppose the package ``foo`` declares this:\n\n.. code-block:: python\n\n class Foo(Package):\n ...\n depends_on(\"mpi@2\")\n\nSuppose a user invokes ``spack install`` like this:\n\n.. code-block:: spec\n\n $ spack install foo ^mpich@1.0\n\nSpack will fail with a constraint violation, because the version of MPICH requested is too low for the ``mpi`` requirement in ``foo``.\n\n.. _language-dependencies:\n\nLanguage and compiler dependencies\n----------------------------------\n\nWhenever you use ``spack create`` to create a new package, Spack scans the package's source code and heuristically adds *language dependencies*, which look like this:\n\n.. code-block:: python\n\n depends_on(\"c\", type=\"build\")\n depends_on(\"cxx\", type=\"build\")\n depends_on(\"fortran\", type=\"build\")\n\nThe languages ``c``, ``cxx`` and ``fortran`` are **virtuals provided by compiler packages**, such as ``gcc``, ``llvm``, or ``intel-oneapi-compilers``.\n\nWhen you concretize a package that depends on ``c``, Spack will select a compiler for it that provides the ``c`` virtual package.\n\nTypically one compiler will be used to provide all languages, but Spack is allowed to create a mixed toolchain.\nFor example, the ``c`` compiler could be ``clang`` from the ``llvm`` package, whereas the ``fortran`` compiler could be ``gfortran`` from the ``gcc``.\nThis means that language dependencies translate to one or more compiler packages as build dependencies.\n\n\n.. _packaging_conflicts:\n\nConflicts\n---------\n\nSometimes packages have known bugs, or limitations, that would prevent them from concretizing or building usable software.\nSpack makes it possible to express such constraints with the ``conflicts`` directive, which takes a spec that is known to cause a conflict and optional ``when`` and ``msg`` arguments.\n\nThe ``when`` argument is a spec that triggers the conflict.\n\nThe ``msg`` argument allows you to provide a custom error message that Spack prints when the spec to be installed satisfies the conflict spec and ``when`` trigger.\n\nAdding the following to a package:\n\n.. code-block:: python\n\n conflicts(\n \"%intel-oneapi-compilers@:2024\",\n when=\"@:1.2\",\n msg=\"known bug when using Intel oneAPI compilers through v2024\",\n )\n\nexpresses that the current package *cannot be built* with Intel oneAPI compilers *up through any version* ``2024`` *when trying to install the package with a version up to* ``1.2``.\n\nIf the ``when`` argument is omitted, then the conflict is *always triggered* for specs satisfying the conflict spec.\nFor example,\n\n.. code-block:: python\n\n conflicts(\"+cuda+rocm\", msg=\"Cannot build with both cuda and rocm enabled\")\n\nmeans the package cannot be installed with both variants enabled.\n\nSimilarly, a conflict can be based on where the build is being performed.\nFor example,\n\n.. code-block:: python\n\n for os in [\"ventura\", \"monterey\", \"bigsur\"]:\n conflicts(f\"platform=darwin os={os}\", msg=f\"{os} is not supported\")\n\nmeans the package cannot be built on a Mac running Ventura, Monterey, or Big Sur.\n\n.. note::\n\n These examples illustrate a few of the types of constraints that can be specified.\n Conflict and ``when`` specs can constrain the compiler, :ref:`version `, :ref:`variants `, :ref:`architecture `, :ref:`dependencies `, and more.\n See :ref:`sec-specs` for more information.\n\n\n.. _packaging_requires:\n\nRequires\n--------\n\nSometimes packages can be built only with specific options.\nIn those cases the ``requires`` directive can be used.\nIt allows for complex conditions involving more than a single spec through the ability to specify multiple required specs before keyword arguments.\nThe same optional ``when`` and ``msg`` arguments as ``conflicts`` are supported (see :ref:`packaging_conflicts`).\nThe directive also supports a ``policy`` argument for determining how the multiple required specs apply.\nValues for ``policy`` may be either ``any_of`` or ``one_of`` (default) and have the same semantics described for their equivalents in :ref:`package-requirements`.\n\n.. hint::\n\n We recommend that the ``policy`` argument be explicitly specified when multiple specs are used with the directive.\n\nFor example, suppose a package can only be built with Apple Clang on Darwin.\nThis requirement would be specified as:\n\n.. code-block:: python\n\n requires(\n \"%apple-clang\",\n when=\"platform=darwin\",\n msg=\"builds only with Apple Clang compiler on Darwin\",\n )\n\nSimilarly, suppose a package only builds for the ``x86_64`` target:\n\n.. code-block:: python\n\n requires(\"target=x86_64:\", msg=\"package is only available on x86_64\")\n\nOr the package must be built with a GCC or Clang that supports C++ 20, which you could ensure by adding the following:\n\n.. code-block:: python\n\n requires(\n \"%gcc@10:\",\n \"%clang@16:\",\n policy=\"one_of\",\n msg=\"builds only with a GCC or Clang that support C++ 20\",\n )\n\n.. note::\n\n These examples show only a few of the constraints that can be specified.\n Required and ``when`` specs can constrain the compiler, :ref:`version `, :ref:`variants `, :ref:`architecture `, :ref:`dependencies `, and more.\n See :ref:`sec-specs` for more information.\n\n\n.. _patching:\n\nPatches\n-------\n\nDepending on the host architecture, package version, known bugs, or other issues, you may need to patch your software to get it to build correctly.\nLike many other package systems, Spack allows you to store patches alongside your package files and apply them to source code after it's downloaded.\n\n``patch``\n^^^^^^^^^\n\nYou can specify patches in your package file with the ``patch()`` directive.\nThe first argument can be either the filename or URL of the patch file to be applied to your source.\n\n.. note::\n\n Use of a URL is preferred over maintaining patch files in the package repository.\n This helps reduce the size of the package repository, which can become an issue for those with limited space (or allocations).\n\nFilename patch\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nYou can supply the name of the patch file.\nFor example, a simple conditional ``patch`` based on a file for the ``mvapich2`` package looks like:\n\n.. code-block:: python\n\n class Mvapich2(Package):\n ...\n patch(\"ad_lustre_rwcontig_open_source.patch\", when=\"@1.9:\")\n\nThis patch will only be applied when attempting to install the package at version ``1.9`` or newer.\n\nWhen a filename is provided, the patch needs to live within the Spack source tree.\nThe above patch file lives with the package file within the package repository directory structure in the following location:\n\n.. code-block:: none\n\n spack_repo/builtin/packages/\n mvapich2/\n package.py\n ad_lustre_rwcontig_open_source.patch\n\nURL patch file\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nIf you supply a URL instead of a filename you have two options: patch file URL or commit patch file URL.\nIn either case, you must supply a checksum.\nSpack requires the ``sha256`` hash so that different patches applied to the same package will have unique identifiers.\nPatches will be fetched from their URLs, checked, and applied to your source code.\n\n.. note::\n\n To ensure consistency, a ``sha256`` checksum must be provided for the patch.\n\n You can use the GNU utils ``sha256sum`` or the macOS ``shasum -a 256`` commands to generate a checksum for a patch file.\n\nHere is an example of specifying the unconditional use of a patch file URL:\n\n.. code-block:: python\n\n patch(\n \"http://www.nwchem-sw.org/images/Tddft_mxvec20.patch\",\n sha256=\"252c0af58be3d90e5dc5e0d16658434c9efa5d20a5df6c10bf72c2d77f780866\",\n )\n\nSometimes you can specify the patch file associated with a repository commit.\nFor example, GitHub allows you to reference the commit in the name of the patch file through a URL in the form ``https://github.com///commit/.patch``.\n\nBelow is an example of specifying a conditional commit patch:\n\n.. code-block:: python\n\n patch(\n \"https://github.com/ornladios/ADIOS/commit/17aee8aeed64612cd8cfa0b949147091a5525bbe.patch?full_index=1\",\n sha256=\"aea47e56013b57c2d5d36e23e0ae6010541c3333a84003784437768c2e350b05\",\n when=\"@1.12.0: +mpi\",\n )\n\nIn this case the patch is only processed when attempting to install version ``1.12.0`` or higher of the package when the package's ``mpi`` variant is enabled.\n\n.. note:\n\n Be sure to append ``?full_index=1`` to the GitHub URL to ensure the patch file consistently contains the complete, stable hash information for reproducible patching.\n\n Use the resulting URL to get the patch file contents that you then run through the appropriate utility to get the corresponding ``sha256`` value.\n\nCompressed patches\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nSpack can also handle compressed patches.\nIf you use these, Spack needs a little more help.\nSpecifically, it needs *two* checksums: the ``sha256`` of the patch and ``archive_sha256`` for the compressed archive.\n``archive_sha256`` helps Spack ensure that the downloaded file is not corrupted or malicious, before running it through a tool like ``tar`` or ``zip``.\nThe ``sha256`` of the patch is still required so that it can be included in specs.\nProviding it in the package file ensures that Spack won't have to download and decompress patches it won't end up using at install time.\nBoth the archive and patch checksum are checked when patch archives are downloaded.\n\n.. code-block:: python\n\n patch(\n \"http://www.nwchem-sw.org/images/Tddft_mxvec20.patch.gz\",\n sha256=\"252c0af58be3d90e5dc5e0d16658434c9efa5d20a5df6c10bf72c2d77f780866\",\n archive_sha256=\"4e8092a161ec6c3a1b5253176fcf33ce7ba23ee2ff27c75dbced589dabacd06e\",\n )\n\n``patch`` keyword arguments are described below.\n\n``sha256``, ``archive_sha256``\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nHashes of downloaded patch and compressed archive, respectively.\nOnly needed for patches fetched from URLs.\n\n``when``\n\"\"\"\"\"\"\"\"\n\nIf supplied, this is a spec that tells Spack when to apply the patch.\nIf the installed package spec matches this spec, the patch will be applied.\nIn our example above, the patch is applied when ``mvapich`` is at version ``1.9`` or higher.\n\n``level``\n\"\"\"\"\"\"\"\"\"\n\nThis tells Spack how to run the ``patch`` command.\nBy default, the level is 1 and Spack runs ``patch -p 1``.\nIf level is 2, Spack will run ``patch -p 2``, and so on.\n\nA lot of people are confused by the level, so here's a primer.\nIf you look in your patch file, you may see something like this:\n\n.. code-block:: diff\n :linenos:\n\n --- a/src/mpi/romio/adio/ad_lustre/ad_lustre_rwcontig.c 2013-12-10 12:05:44.806417000 -0800\n +++ b/src/mpi/romio/adio/ad_lustre/ad_lustre_rwcontig.c 2013-12-10 11:53:03.295622000 -0800\n @@ -8,7 +8,7 @@\n * Copyright (C) 2008 Sun Microsystems, Lustre group\n \\*/\n\n -#define _XOPEN_SOURCE 600\n +//#define _XOPEN_SOURCE 600\n #include \n #include \n #include \"ad_lustre.h\"\n\nLines 1-2 show paths with synthetic ``a/`` and ``b/`` prefixes.\nThese are placeholders for the two ``mvapich2`` source directories that ``diff`` compared when it created the patch file.\nThis is git's default behavior when creating patch files, but other programs may behave differently.\n\n``-p1`` strips off the first level of the prefix in both paths, allowing the patch to be applied from the root of an expanded ``mvapich2`` archive.\nIf you set level to ``2``, it would strip off ``src``, and so on.\n\nIt's generally easier to just structure your patch file so that it applies cleanly with ``-p1``, but if you're using a patch you didn't create yourself, ``level`` can be handy.\n\n``working_dir``\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nThis tells Spack where to run the ``patch`` command.\nBy default, the working directory is the source path of the stage (``.``).\nHowever, sometimes patches are made with respect to a subdirectory and this is where the working directory comes in handy.\nInternally, the working directory is given to ``patch`` via the ``-d`` option.\nLet's take the example patch from above and assume for some reason, it can only be downloaded in the following form:\n\n.. code-block:: diff\n :linenos:\n\n --- a/romio/adio/ad_lustre/ad_lustre_rwcontig.c 2013-12-10 12:05:44.806417000 -0800\n +++ b/romio/adio/ad_lustre/ad_lustre_rwcontig.c 2013-12-10 11:53:03.295622000 -0800\n @@ -8,7 +8,7 @@\n * Copyright (C) 2008 Sun Microsystems, Lustre group\n \\*/\n\n -#define _XOPEN_SOURCE 600\n +//#define _XOPEN_SOURCE 600\n #include \n #include \n #include \"ad_lustre.h\"\n\nHence, the patch needs to be applied in the ``src/mpi`` subdirectory, and the ``working_dir=\"src/mpi\"`` option would exactly do that.\n\nPatch functions\n^^^^^^^^^^^^^^^^^^^^^\n\nIn addition to supplying patch files, you can write a custom function to patch a package's source.\nFor example, the ``py-pyside2`` package contains some custom code for tweaking the way the PySide build handles include files:\n\n.. _pyside-patch:\n\n.. literalinclude:: .spack/spack-packages/repos/spack_repo/builtin/packages/py_pyside2/package.py\n :pyobject: PyPyside2.patch\n :linenos:\n\nA ``patch`` function, if present, will be run after patch files are applied and before ``install()`` is run.\n\nYou could put this logic in ``install()``, but putting it in a patch function gives you some benefits.\nFirst, Spack ensures that the ``patch()`` function is run once per code checkout.\nThat means that if you run install, hit ctrl-C, and run install again, the code in the patch function is only run once.\n\n.. _patch_dependency_patching:\n\nDependency patching\n^^^^^^^^^^^^^^^^^^^\n\nSo far we've covered how the ``patch`` directive can be used by a package to patch *its own* source code.\nPackages can *also* specify patches to be applied to their dependencies, if they require special modifications.\nAs with all packages in Spack, a patched dependency library can coexist with other versions of that library.\nSee the `section on depends_on `_ for more details.\n\n.. _patch_inspecting_patches:\n\nInspecting patches\n^^^^^^^^^^^^^^^^^^^\n\nIf you want to better understand the patches that Spack applies to your packages, you can do that using ``spack spec``, ``spack find``, and other query commands.\nLet's look at ``m4``.\nIf you run ``spack spec m4``, you can see the patches that would be applied to ``m4``:\n\n.. code-block:: spec\n\n $ spack spec m4\n Input spec\n --------------------------------\n m4\n\n Concretized\n --------------------------------\n m4@1.4.18%apple-clang@9.0.0 patches=3877ab548f88597ab2327a2230ee048d2d07ace1062efe81fc92e91b7f39cd00,c0a408fbffb7255fcc75e26bd8edab116fc81d216bfd18b473668b7739a4158e,fc9b61654a3ba1a8d6cd78ce087e7c96366c290bc8d2c299f09828d793b853c8 +sigsegv arch=darwin-highsierra-x86_64\n ^libsigsegv@2.11%apple-clang@9.0.0 arch=darwin-highsierra-x86_64\n\nYou can also see patches that have been applied to installed packages with ``spack find -v``:\n\n.. code-block:: spec\n\n $ spack find -v m4\n ==> 1 installed package\n -- darwin-highsierra-x86_64 / apple-clang@9.0.0 -----------------\n m4@1.4.18 patches=3877ab548f88597ab2327a2230ee048d2d07ace1062efe81fc92e91b7f39cd00,c0a408fbffb7255fcc75e26bd8edab116fc81d216bfd18b473668b7739a4158e,fc9b61654a3ba1a8d6cd78ce087e7c96366c290bc8d2c299f09828d793b853c8 +sigsegv\n\n.. _cmd-spack-resource:\n\nIn both cases above, you can see that the patches' sha256 hashes are stored on the spec as a variant.\nAs mentioned above, this means that you can have multiple, differently-patched versions of a package installed at once.\n\nYou can look up a patch by its sha256 hash (or a short version of it) using the ``spack resource show`` command\n\n.. code-block:: console\n\n $ spack resource show 3877ab54\n 3877ab548f88597ab2327a2230ee048d2d07ace1062efe81fc92e91b7f39cd00\n path: .../spack_repo/builtin/packages/m4/gnulib-pgi.patch\n applies to: builtin.m4\n\n``spack resource show`` looks up downloadable resources from package files by hash and prints out information about them.\nAbove, we see that the ``3877ab54`` patch applies to the ``m4`` package.\nThe output also tells us where to find the patch.\n\nThings get more interesting if you want to know about dependency patches.\nFor example, when ``dealii`` is built with ``boost@1.68.0``, it has to patch boost to work correctly.\nIf you didn't know this, you might wonder where the extra boost patches are coming from:\n\n.. code-block:: console\n\n $ spack spec dealii ^boost@1.68.0 ^hdf5+fortran | grep \"\\^boost\"\n ^boost@1.68.0\n ^boost@1.68.0%apple-clang@9.0.0+atomic+chrono~clanglibcpp cxxstd=default +date_time~debug+exception+filesystem+graph~icu+iostreams+locale+log+math~mpi+multithreaded~numpy patches=2ab6c72d03dec6a4ae20220a9dfd5c8c572c5294252155b85c6874d97c323199,b37164268f34f7133cbc9a4066ae98fda08adf51e1172223f6a969909216870f ~pic+program_options~python+random+regex+serialization+shared+signals~singlethreaded+system~taggedlayout+test+thread+timer~versionedlayout+wave arch=darwin-highsierra-x86_64\n $ spack resource show b37164268\n b37164268f34f7133cbc9a4066ae98fda08adf51e1172223f6a969909216870f\n path: .../spack_repo/builtin/packages/dealii/boost_1.68.0.patch\n applies to: builtin.boost\n patched by: builtin.dealii\n\nHere you can see that the patch is applied to ``boost`` by ``dealii``, and that it lives in ``dealii``'s directory in Spack's ``builtin`` package repository.\n\n.. _packaging_extensions:\n\nExtensions\n----------\n\nSpack's support for package extensions is documented extensively in :ref:`extensions`.\nThis section documents how to make your own extendable packages and extensions.\n\nTo support extensions, a package needs to set its ``extendable`` property to ``True``, e.g.:\n\n.. code-block:: python\n\n class Python(Package):\n ...\n extendable = True\n ...\n\nTo make a package into an extension, simply add an ``extends`` call in the package definition, and pass it the name of an extendable package:\n\n.. code-block:: python\n\n class PyNumpy(Package):\n ...\n extends(\"python\")\n ...\n\nThis accomplishes a few things.\nFirstly, the Python package can set special variables such as ``PYTHONPATH`` for all extensions when the run or build environment is set up.\nSecondly, filesystem views can ensure that extensions are put in the same prefix as their extendee.\nThis ensures that Python in a view can always locate its Python packages, even without environment variables set.\n\nA package can only extend one other package at a time.\nTo support packages that may extend one of a list of other packages, Spack supports multiple ``extends`` directives as long as at most one of them is selected as a dependency during concretization.\nFor example, a lua package could extend either ``lua`` or ``lua-luajit``, but not both:\n\n.. code-block:: python\n\n class LuaLpeg(Package):\n ...\n variant(\"use_lua\", default=True)\n extends(\"lua\", when=\"+use_lua\")\n extends(\"lua-luajit\", when=\"~use_lua\")\n ...\n\nNow, a user can install, and activate, the ``lua-lpeg`` package for either lua or ``lua-luajit``.\n\nAdding additional constraints\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nSome packages produce a Python extension, but require a minimum version of Python to work correctly.\nIn those cases, a ``depends_on()`` declaration should be made in addition to the ``extends()`` declaration:\n\n.. code-block:: python\n\n class Icebin(Package):\n extends(\"python\", when=\"+python\")\n depends_on(\"python@3.12:\", when=\"+python\")\n\nMany packages produce Python extensions for *some* variants, but not others: they should extend ``python`` only if the appropriate variant(s) are selected.\nThis may be accomplished with conditional ``extends()`` declarations:\n\n.. code-block:: python\n\n class FooLib(Package):\n variant(\"python\", default=True, description=\"Build the Python extension Module\")\n extends(\"python\", when=\"+python\")\n ...\n\nMixins for common metadata\n--------------------------\n\nSpack's package repository contains a number of mixin classes that can be used to simplify package definitions and to share common metadata and behavior across multiple packages.\n\nFor instance, packages that depend on ``cuda`` typically need variants such as ``+cuda`` and ``cuda_arch``, and conflicts to specify compatibility between architectures, compilers and CUDA versions.\nTo avoid duplicating this metadata in every package that requires CUDA, Spack provides a mixin class called ``CudaPackage`` that can be used to inherit this common metadata and behavior.\n\nOther mixin classes such as ``GNUMirrorPackage`` do not add variants or conflicts, but configure the usual GNU mirror URLs for downloading source code.\n\nThe following table lists the full list of mixin classes available in Spack's builtin package repository.\n\n+----------------------------------------------------------------------------+----------------------------------+\n| **API docs** | **Description** |\n+============================================================================+==================================+\n| :class:`~spack_repo.builtin.build_systems.cuda.CudaPackage` | A helper class for packages that |\n| | use CUDA |\n+----------------------------------------------------------------------------+----------------------------------+\n| :class:`~spack_repo.builtin.build_systems.rocm.ROCmPackage` | A helper class for packages that |\n| | use ROCm |\n+----------------------------------------------------------------------------+----------------------------------+\n| :class:`~spack_repo.builtin.build_systems.gnu.GNUMirrorPackage` | A helper class for GNU packages |\n| | |\n+----------------------------------------------------------------------------+----------------------------------+\n| :class:`~spack_repo.builtin.build_systems.python.PythonExtension` | A helper class for Python |\n| | extensions |\n+----------------------------------------------------------------------------+----------------------------------+\n| :class:`~spack_repo.builtin.build_systems.sourceforge.SourceforgePackage` | A helper class for packages |\n| | from sourceforge.org |\n+----------------------------------------------------------------------------+----------------------------------+\n| :class:`~spack_repo.builtin.build_systems.sourceware.SourcewarePackage` | A helper class for packages |\n| | from sourceware.org |\n+----------------------------------------------------------------------------+----------------------------------+\n| :class:`~spack_repo.builtin.build_systems.xorg.XorgPackage` | A helper class for x.org |\n| | packages |\n+----------------------------------------------------------------------------+----------------------------------+\n\nThese mixins should be used as additional base classes for your package, in addition to the base class that you would normally use (e.g. ``MakefilePackage``, ``AutotoolsPackage``, etc.):\n\n.. code-block:: python\n\n class Cp2k(MakefilePackage, CudaPackage):\n pass\n\nIn the example above ``Cp2k`` inherits the variants and conflicts defined by ``CudaPackage``.\n\n.. _maintainers:\n\nMaintainers\n-----------\n\nEach package in Spack may have one or more GitHub accounts for people who want to be notified whenever the package is modified.\nThe list also provides contacts for people needing help with build errors.\n\nAdding maintainers is easy.\nAfter familiarizing yourself with the responsibilities of the :ref:`Package Maintainers ` role, you simply need to declare their GitHub accounts in the ``maintainers`` directive:\n\n.. code-block:: python\n\n maintainers(\"github_user1\", \"github_user2\")\n\n.. warning::\n\n Please do not add accounts without consent of the owner.\n\nThe final list of maintainers includes accounts declared in the package's base classes.\n\n.. _package_license:\n\nLicense Information\n-------------------\n\nMost of the software in Spack is open source, and most open source software is released under one or more `common open source licenses `_.\nSpecifying the license that a package is released under in a project's ``package.py`` is good practice.\nTo specify a license, find the `SPDX identifier `_ for a project and then add it using the license directive:\n\n.. code-block:: python\n\n license(\"\")\n\nFor example, the SPDX ID for the Apache Software License, version 2.0 is ``Apache-2.0``, so you'd write:\n\n.. code-block:: python\n\n license(\"Apache-2.0\")\n\nOr, for a dual-licensed package like Spack, you would use an `SPDX Expression `_ with both of its licenses:\n\n.. code-block:: python\n\n license(\"Apache-2.0 OR MIT\")\n\nNote that specifying a license without a ``when=`` clause makes it apply to all versions and variants of the package, which might not actually be the case.\nFor example, a project might have switched licenses at some point or have certain build configurations that include files that are licensed differently.\nSpack itself used to be under the ``LGPL-2.1`` license, until it was relicensed in version ``0.12`` in 2018.\n\nYou can specify when a ``license()`` directive applies using a ``when=`` clause, just like other directives.\nFor example, to specify that a specific license identifier should only apply to versions up to ``0.11``, but another license should apply for later versions, you could write:\n\n.. code-block:: python\n\n license(\"LGPL-2.1\", when=\"@:0.11\")\n license(\"Apache-2.0 OR MIT\", when=\"@0.12:\")\n\nNote that unlike for most other directives, the ``when=`` constraints in the ``license()`` directive can't intersect.\nSpack needs to be able to resolve exactly one license identifier expression for any given version.\nTo specify *multiple* licenses, use SPDX expressions and operators as above.\nThe operators you probably care most about are:\n\n* ``OR``: user chooses one license to adhere to; and\n* ``AND``: user has to adhere to all the licenses.\n\nYou may also care about `license exceptions `_ that use the ``WITH`` operator, e.g. ``Apache-2.0 WITH LLVM-exception``.\n\nMany of the licenses that are currently in the spack repositories have been automatically determined.\nWhile this is great for bulk adding license information and is most likely correct, there are sometimes edge cases that require manual intervention.\nTo determine which licenses are validated and which are not, there is the ``checked_by`` parameter in the license directive:\n\n.. code-block:: python\n\n license(\"\", when=\"\", checked_by=\"\")\n\nWhen you have validated a package license, either when doing so explicitly or as part of packaging a new package, please set the ``checked_by`` parameter to your Github username to signal that the license has been manually verified.\n\n.. _license:\n\nProprietary software\n--------------------\n\nIn order to install proprietary software, Spack needs to know a few more details about a package.\nThe following class attributes should be defined.\n\n``license_required``\n^^^^^^^^^^^^^^^^^^^^\n\nBoolean.\nIf set to ``True``, this software requires a license.\nIf set to ``False``, all of the following attributes will be ignored.\nDefaults to ``False``.\n\n``license_comment``\n^^^^^^^^^^^^^^^^^^^\n\nString.\nContains the symbol used by the license manager to denote a comment.\nDefaults to ``#``.\n\n``license_files``\n^^^^^^^^^^^^^^^^^\n\nList of strings.\nThese are files that the software searches for when looking for a license.\nAll file paths must be relative to the installation directory.\nMore complex packages like Intel may require multiple licenses for individual components.\nDefaults to the empty list.\n\n``license_vars``\n^^^^^^^^^^^^^^^^\n\nList of strings.\nEnvironment variables that can be set to tell the software where to look for a license if it is not in the usual location.\nDefaults to the empty list.\n\n``license_url``\n^^^^^^^^^^^^^^^\n\nString.\nA URL pointing to license setup instructions for the software.\nDefaults to the empty string.\n\nFor example, let's take a look at the Arm Forge package.\n\n.. code-block:: python\n\n # Licensing\n license_required = True\n license_comment = \"#\"\n license_files = [\"licences/Licence\"]\n license_vars = [\n \"ALLINEA_LICENSE_DIR\",\n \"ALLINEA_LICENCE_DIR\",\n \"ALLINEA_LICENSE_FILE\",\n \"ALLINEA_LICENCE_FILE\",\n ]\n license_url = \"https://developer.arm.com/documentation/101169/latest/Use-Arm-Licence-Server\"\n\nArm Forge requires a license.\nIts license manager uses the ``#`` symbol to denote a comment.\nIt expects the license file to be named ``License`` and to be located in a ``licenses`` directory in the installation prefix.\n\nIf you would like the installation file to be located elsewhere, simply set ``ALLINEA_LICENSE_DIR`` or one of the other license variables after installation.\nFor further instructions on installation and licensing, see the URL provided.\n\nIf your package requires the license to install, you can reference the location of this global license using ``self.global_license_file``.\nAfter installation, symlinks for all of the files given in ``license_files`` will be created, pointing to this global license.\nIf you install a different version or variant of the package, Spack will automatically detect and reuse the already existing global license.\n\nIf the software you are trying to package doesn't rely on license files, Spack will print a warning message, letting the user know that they need to set an environment variable or pointing them to installation documentation.\n\n\nGrouping directives\n-------------------\n\nWe have seen various directives such as ``depends_on``, ``conflicts``, and ``requires``.\nVery often, these directives share a common argument, which you becomes repetitive and verbose to write.\n\n.. _group_when_spec:\n\nGrouping with ``when()``\n^^^^^^^^^^^^^^^^^^^^^^^^\n\nSpack provides a context manager called ``when()`` that allows you to group directives by a common constraint or condition.\n\n.. code-block:: python\n\n class Gcc(AutotoolsPackage):\n\n with when(\"+nvptx\"):\n depends_on(\"cuda\")\n conflicts(\"@:6\", msg=\"NVPTX only supported in gcc 7 and above\")\n conflicts(\"languages=ada\")\n conflicts(\"languages=brig\")\n conflicts(\"languages=go\")\n\nThe snippet above is equivalent to the more verbose:\n\n.. code-block:: python\n\n class Gcc(AutotoolsPackage):\n\n depends_on(\"cuda\", when=\"+nvptx\")\n conflicts(\"@:6\", when=\"+nvptx\", msg=\"NVPTX only supported in gcc 7 and above\")\n conflicts(\"languages=ada\", when=\"+nvptx\")\n conflicts(\"languages=brig\", when=\"+nvptx\")\n conflicts(\"languages=go\", when=\"+nvptx\")\n\nConstraints from the ``when`` block are composable with ``when`` arguments in directives inside the block.\nFor instance,\n\n.. code-block:: python\n\n with when(\"+elpa\"):\n depends_on(\"elpa+openmp\", when=\"+openmp\")\n\nis equivalent to:\n\n.. code-block:: python\n\n depends_on(\"elpa+openmp\", when=\"+openmp+elpa\")\n\nConstraints from nested context managers are also combined together, but they are rarely needed, and are not recommended.\n\n.. _default_args:\n\nGrouping with ``default_args()``\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nMore generally, if directives have a common set of default arguments, you can group them together in a ``with default_args()`` block:\n\n.. code-block:: python\n\n class PyExample(PythonPackage):\n\n with default_args(type=(\"build\", \"run\")):\n depends_on(\"py-foo\")\n depends_on(\"py-foo@2:\", when=\"@2:\")\n depends_on(\"py-bar\")\n depends_on(\"py-bz\")\n\nThe above is short for:\n\n.. code-block:: python\n\n class PyExample(PythonPackage):\n\n depends_on(\"py-foo\", type=(\"build\", \"run\"))\n depends_on(\"py-foo@2:\", when=\"@2:\", type=(\"build\", \"run\"))\n depends_on(\"py-bar\", type=(\"build\", \"run\"))\n depends_on(\"py-bz\", type=(\"build\", \"run\"))\n\n.. note::\n\n The ``with when()`` context manager is composable, while ``with default_args()`` merely overrides the default.\n For example:\n\n .. code-block:: python\n\n with default_args(when=\"+feature\"):\n depends_on(\"foo\")\n depends_on(\"bar\")\n depends_on(\"baz\", when=\"+baz\")\n\n is equivalent to:\n\n .. code-block:: python\n\n depends_on(\"foo\", when=\"+feature\")\n depends_on(\"bar\", when=\"+feature\")\n depends_on(\"baz\", when=\"+baz\") # Note: not when=\"+feature+baz\"\n\n.. _custom-attributes:\n\n``home``, ``command``, ``headers``, and ``libs``\n------------------------------------------------\n\nOften a package will need to provide attributes for dependents to query various details about what it provides.\nWhile any number of custom defined attributes can be implemented by a package, the four specific attributes described below are always available on every package with default implementations and the ability to customize with alternate implementations in the case of virtual packages provided:\n\n=========== =========================================== =====================\nAttribute Purpose Default\n=========== =========================================== =====================\n``home`` The installation path for the package ``spec.prefix``\n``command`` An executable command for the package | ``spec.name`` found\n in\n | ``.home.bin``\n``headers`` A list of headers provided by the package | All headers\n searched\n | recursively in\n ``.home.include``\n``libs`` A list of libraries provided by the package | ``lib{spec.name}``\n searched\n | recursively in\n ``.home`` starting\n | with ``lib``,\n ``lib64``, then the\n | rest of ``.home``\n=========== =========================================== =====================\n\nEach of these can be customized by implementing the relevant attribute as a ``@property`` in the package's class:\n\n.. code-block:: python\n :linenos:\n\n class Foo(Package):\n ...\n\n @property\n def libs(self):\n # The library provided by Foo is libMyFoo.so\n return find_libraries(\"libMyFoo\", root=self.home, recursive=True)\n\nA package may also provide custom implementations of each attribute for the virtual packages it provides, by implementing the ``_`` property in its package class.\nThe implementation used is the first one found from:\n\n#. Specialized virtual: ``Package._``\n#. Generic package: ``Package.``\n#. Default\n\nThe use of customized attributes is demonstrated in the next example.\n\nExample: Customized attributes for virtual packages\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nConsider a package ``foo`` that can optionally provide two virtual packages ``bar`` and ``baz``.\nWhen both are enabled, the installation tree appears as follows:\n\n.. code-block:: console\n\n include/foo.h\n include/bar/bar.h\n lib64/libFoo.so\n lib64/libFooBar.so\n baz/include/baz/baz.h\n baz/lib/libFooBaz.so\n\nThe install tree shows that ``foo`` provides the header ``include/foo.h`` and library ``lib64/libFoo.so`` in its install prefix.\nThe virtual package ``bar`` provides the header ``include/bar/bar.h`` and library ``lib64/libFooBar.so``, also in ``foo``'s install prefix.\nThe ``baz`` package, however, is provided in the ``baz`` subdirectory of ``foo``'s prefix with the ``include/baz/baz.h`` header and ``lib/libFooBaz.so`` library.\nSuch a package could implement the optional attributes as follows:\n\n.. code-block:: python\n :linenos:\n\n class Foo(Package):\n ...\n variant(\"bar\", default=False, description=\"Enable the Foo implementation of bar\")\n variant(\"baz\", default=False, description=\"Enable the Foo implementation of baz\")\n ...\n provides(\"bar\", when=\"+bar\")\n provides(\"baz\", when=\"+baz\")\n ...\n\n # Just the foo headers\n @property\n def headers(self):\n return find_headers(\"foo\", root=self.home.include, recursive=False)\n\n # Just the foo libraries\n @property\n def libs(self):\n return find_libraries(\"libFoo\", root=self.home, recursive=True)\n\n # The header provided by the bar virtual package\n @property\n def bar_headers(self):\n return find_headers(\"bar/bar.h\", root=self.home.include, recursive=False)\n\n # The library provided by the bar virtual package\n @property\n def bar_libs(self):\n return find_libraries(\"libFooBar\", root=self.home, recursive=True)\n\n # The baz virtual package home\n @property\n def baz_home(self):\n return self.prefix.baz\n\n # The header provided by the baz virtual package\n @property\n def baz_headers(self):\n return find_headers(\"baz/baz\", root=self.baz_home.include, recursive=False)\n\n # The library provided by the baz virtual package\n @property\n def baz_libs(self):\n return find_libraries(\"libFooBaz\", root=self.baz_home, recursive=True)\n\nNow consider another package, ``foo-app``, depending on all three:\n\n.. code-block:: python\n :linenos:\n\n class FooApp(CMakePackage):\n ...\n depends_on(\"foo\")\n depends_on(\"bar\")\n depends_on(\"baz\")\n\nThe resulting spec objects for its dependencies shows the result of the above attribute implementations:\n\n.. code-block:: pycon\n\n # The core headers and libraries of the foo package\n\n >>> spec[\"foo\"]\n foo@1.0/ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6\n >>> spec[\"foo\"].prefix\n \"/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6\"\n\n # home defaults to the package install prefix without an explicit implementation\n >>> spec[\"foo\"].home\n \"/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6\"\n\n # foo headers from the foo prefix\n >>> spec[\"foo\"].headers\n HeaderList([\n \"/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/include/foo.h\",\n ])\n\n # foo include directories from the foo prefix\n >>> spec[\"foo\"].headers.directories\n [\"/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/include\"]\n\n # foo libraries from the foo prefix\n >>> spec[\"foo\"].libs\n LibraryList([\n \"/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/lib64/libFoo.so\",\n ])\n\n # foo library directories from the foo prefix\n >>> spec[\"foo\"].libs.directories\n [\"/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/lib64\"]\n\n.. code-block:: pycon\n\n # The virtual bar package in the same prefix as foo\n\n # bar resolves to the foo package\n >>> spec[\"bar\"]\n foo@1.0/ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6\n >>> spec[\"bar\"].prefix\n \"/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6\"\n\n # home defaults to the foo prefix without either a Foo.bar_home\n # or Foo.home implementation\n >>> spec[\"bar\"].home\n \"/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6\"\n\n # bar header in the foo prefix\n >>> spec[\"bar\"].headers\n HeaderList([\n \"/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/include/bar/bar.h\"\n ])\n\n # bar include dirs from the foo prefix\n >>> spec[\"bar\"].headers.directories\n [\"/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/include\"]\n\n # bar library from the foo prefix\n >>> spec[\"bar\"].libs\n LibraryList([\n \"/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/lib64/libFooBar.so\"\n ])\n\n # bar library directories from the foo prefix\n >>> spec[\"bar\"].libs.directories\n [\"/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/lib64\"]\n\n.. code-block:: pycon\n\n # The virtual baz package in a subdirectory of foo's prefix\n\n # baz resolves to the foo package\n >>> spec[\"baz\"]\n foo@1.0/ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6\n >>> spec[\"baz\"].prefix\n \"/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6\"\n\n # baz_home implementation provides the subdirectory inside the foo prefix\n >>> spec[\"baz\"].home\n \"/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/baz\"\n\n # baz headers in the baz subdirectory of the foo prefix\n >>> spec[\"baz\"].headers\n HeaderList([\n \"/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/baz/include/baz/baz.h\"\n ])\n\n # baz include directories in the baz subdirectory of the foo prefix\n >>> spec[\"baz\"].headers.directories\n [\n \"/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/baz/include\"\n ]\n\n # baz libraries in the baz subdirectory of the foo prefix\n >>> spec[\"baz\"].libs\n LibraryList([\n \"/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/baz/lib/libFooBaz.so\"\n ])\n\n # baz library directories in the baz subdirectory of the foo prefix\n >>> spec[\"baz\"].libs.directories\n [\n \"/opt/spack/linux-fedora35-haswell/gcc-11.3.1/foo-1.0-ca3rczp5omy7dfzoqw4p7oc2yh3u7lt6/baz/lib\"\n ]\n\nStyle guidelines for packages\n-----------------------------\n\nThe following guidelines are provided, in the interests of making Spack packages work in a consistent manner:\n\nVariant Names\n^^^^^^^^^^^^^\n\nSpack packages with variants similar to already-existing Spack packages should use the same name for their variants.\nStandard variant names are:\n\n======= ======== ========================\nName Default Description\n======= ======== ========================\nshared True Build shared libraries\nmpi True Use MPI\npython False Build Python extension\n======= ======== ========================\n\nIf specified in this table, the corresponding default is recommended.\n\nThe semantics of the ``shared`` variant are important.\nWhen a package is built ``~shared``, the package guarantees that no shared libraries are built.\nWhen a package is built ``+shared``, the package guarantees that shared libraries are built, but it makes no guarantee about whether static libraries are built.\n\nVersion definitions\n^^^^^^^^^^^^^^^^^^^\n\nSpack packages should list supported versions with the newest first.\n\nUsing ``home`` vs ``prefix``\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n``home`` and ``prefix`` are both attributes that can be queried on a package's dependencies, often when passing configure arguments pointing to the location of a dependency.\nThe difference is that while ``prefix`` is the location on disk where a concrete package resides, ``home`` is the `logical` location that a package resides, which may be different than ``prefix`` in the case of virtual packages or other special circumstances.\nFor most use cases inside a package, its dependency locations can be accessed via either ``self.spec[\"foo\"].home`` or ``self.spec[\"foo\"].prefix``.\nSpecific packages that should be consumed by dependents via ``.home`` instead of ``.prefix`` should be noted in their respective documentation.\n\nSee :ref:`custom-attributes` for more details and an example implementing a custom ``home`` attribute.\n" }, { "path": "lib/spack/docs/packaging_guide_testing.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n A guide to adding tests to Spack packages to ensure correct installation and functionality.\n\n.. list-table::\n :widths: 25 25 25 25\n :header-rows: 0\n :width: 100%\n\n * - :doc:`1. Creation `\n - :doc:`2. Build `\n - **3. Testing**\n - :doc:`4. Advanced `\n\nPackaging Guide: testing installations\n======================================\n\nIn this part of the packaging guide we will cover how to ensure your package builds correctly by adding tests to it.\n\n\n.. _checking_an_installation:\n\nChecking an installation\n------------------------\n\nA package that *appears* to install successfully does not mean it is actually installed correctly or will continue to work indefinitely.\nThere are a number of possible points of failure so Spack provides features for checking the software along the way.\n\nFailures can occur during and after the installation process.\nThe build may start, but the software may not end up fully installed.\nThe installed software may not work at all, or may not work as expected.\nThe software may work after being installed, but due to changes on the system, may stop working days, weeks, or months after being installed.\n\nThis section describes Spack's support for checks that can be performed during and after its installation.\nThe former checks are referred to as ``build-time tests`` and the latter as ``stand-alone (or smoke) tests``.\n\n.. _build_time-tests:\n\nBuild-time tests\n^^^^^^^^^^^^^^^^\n\nSpack infers the status of a build based on the contents of the install prefix.\nSuccess is assumed if anything (e.g., a file or directory) is written after ``install()`` completes.\nOtherwise, the build is assumed to have failed.\nHowever, the presence of install prefix contents is not a sufficient indicator of success so Spack supports the addition of tests that can be performed during `spack install` processing.\n\nConsider a simple Autotools build using the following commands:\n\n.. code-block:: console\n\n $ ./configure --prefix=/path/to/installation/prefix\n $ make\n $ make install\n\nStandard Autotools and CMake do not write anything to the prefix from the ``configure`` and ``make`` commands.\nFiles are only written from the ``make install`` after the build completes.\n\n.. note::\n\n If you want to learn more about ``Autotools`` and ``CMake`` packages in Spack, refer to :ref:`AutotoolsPackage ` and :ref:`CMakePackage `, respectively.\n\nWhat can you do to check that the build is progressing satisfactorily?\nIf there are specific files and/or directories expected of a successful installation, you can add basic, fast ``sanity checks``.\nYou can also add checks to be performed after one or more installation phases.\n\n.. note::\n\n Build-time tests are performed when the ``--test`` option is passed to ``spack install``.\n\n.. warning::\n\n Build-time test failures result in a failed installation of the software.\n\n\n.. _sanity-checks:\n\nAdding sanity checks\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nUnfortunately, many builds of scientific software modify the installation prefix **before** ``make install``.\nBuilds like this can falsely report success when an error occurs before the installation is complete.\nSimple sanity checks can be used to identify files and/or directories that are required of a successful installation.\nSpack checks for the presence of the files and directories after ``install()`` runs.\n\nIf any of the listed files or directories are missing, then the build will fail and the install prefix will be removed.\nIf they all exist, then Spack considers the build successful from a sanity check perspective and keeps the prefix in place.\n\nFor example, the sanity checks for the ``reframe`` package below specify that eight paths must exist within the installation prefix after the ``install`` method completes.\n\n.. code-block:: python\n\n class Reframe(Package):\n ...\n\n # sanity check\n sanity_check_is_file = [join_path(\"bin\", \"reframe\")]\n sanity_check_is_dir = [\n \"bin\", \n \"config\", \n \"docs\", \n \"reframe\", \n \"tutorials\",\n \"unittests\",\n \"cscs-checks\",\n ]\n\nWhen you run ``spack install`` with tests enabled, Spack will ensure that a successfully installed package has the required files and/or directories.\n\nFor example, running:\n\n.. code-block:: spec\n\n $ spack install --test=root reframe\n\nresults in Spack checking that the installation created the following **file**:\n\n* ``self.prefix.bin.reframe``\n\nand the following **directories**:\n\n* ``self.prefix.bin``\n* ``self.prefix.config``\n* ``self.prefix.docs``\n* ``self.prefix.reframe``\n* ``self.prefix.tutorials``\n* ``self.prefix.unittests``\n* ``self.prefix.cscs-checks``\n\nIf **any** of these paths are missing, then Spack considers the installation to have failed.\n\n.. note::\n\n You **MUST** use ``sanity_check_is_file`` to specify required files and ``sanity_check_is_dir`` for required directories.\n\n.. _install_phase-tests:\n\nAdding installation phase tests\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nSometimes packages appear to build \"correctly\" only to have runtime behavior issues discovered at a later stage, such as after a full software stack relying on them has been built.\nChecks can be performed at different phases of the package installation to possibly avoid these types of problems.\nSome checks are built-in to different build systems, while others will need to be added to the package.\n\nBuilt-in installation phase tests are provided by packages inheriting from select :ref:`build systems `, where naming conventions are used to identify typical test identifiers for those systems.\nIn general, you won't need to add anything to your package to take advantage of these tests if your software's build system complies with the convention; otherwise, you'll want or need to override the post-phase method to perform other checks.\n\n.. list-table:: Built-in installation phase tests\n :header-rows: 1\n\n * - Build System Class\n - Post-Build Phase Method (Runs)\n - Post-Install Phase Method (Runs)\n * - :ref:`AutotoolsPackage `\n - ``check`` (``make test``, ``make check``)\n - ``installcheck`` (``make installcheck``)\n * - :ref:`CachedCMakePackage `\n - ``check`` (``make check``, ``make test``)\n - Not applicable\n * - :ref:`CMakePackage `\n - ``check`` (``make check``, ``make test``)\n - Not applicable\n * - :ref:`MakefilePackage `\n - ``check`` (``make test``, ``make check``)\n - ``installcheck`` (``make installcheck``)\n * - :ref:`MesonPackage `\n - ``check`` (``make test``, ``make check``)\n - Not applicable\n * - :ref:`PerlPackage `\n - ``check`` (``make test``)\n - Not applicable\n * - :ref:`PythonPackage `\n - Not applicable\n - ``test_imports`` (module imports)\n * - :ref:`QMakePackage `\n - ``check`` (``make check``)\n - Not applicable\n * - :ref:`SConsPackage `\n - ``build_test`` (must be overridden)\n - Not applicable\n * - :ref:`SIPPackage `\n - Not applicable\n - ``test_imports`` (module imports)\n * - :ref:`WafPackage `\n - ``build_test`` (must be overridden)\n - ``install_test`` (must be overridden)\n\nFor example, the ``Libelf`` package inherits from ``AutotoolsPackage`` and its ``Makefile`` has a standard ``check`` target.\nSo Spack will automatically run ``make check`` after the ``build`` phase when it is installed using the ``--test`` option, such as:\n\n.. code-block:: spec\n\n $ spack install --test=root libelf\n\nIn addition to overriding any built-in build system installation phase tests, you can write your own install phase tests.\nYou will need to use two decorators for each phase test method:\n\n* ``run_after``\n* ``on_package_attributes``\n\nThe first decorator tells Spack when in the installation process to run your test method installation process; namely *after* the provided installation phase.\nThe second decorator tells Spack to only run the checks when the ``--test`` option is provided on the command line.\n\n.. note::\n\n Be sure to place the directives above your test method in the order ``run_after`` *then* ``on_package_attributes``.\n\n.. note::\n\n You also want to be sure the package supports the phase you use in the ``run_after`` directive.\n For example, ``PackageBase`` only supports the ``install`` phase while the ``AutotoolsPackage`` and ``MakefilePackage`` support both ``install`` and ``build`` phases.\n\nAssuming both ``build`` and ``install`` phases are available, you can add additional checks to be performed after each of those phases based on the skeleton provided below.\n\n.. code-block:: python\n\n class YourMakefilePackage(MakefilePackage):\n ...\n\n @run_after(\"build\")\n @on_package_attributes(run_tests=True)\n def check_build(self):\n # Add your custom post-build phase tests\n pass\n\n @run_after(\"install\")\n @on_package_attributes(run_tests=True)\n def check_install(self):\n # Add your custom post-install phase tests\n pass\n\n.. note::\n\n You could also schedule work to be done **before** a given phase using the ``run_before`` decorator.\n\nBy way of a concrete example, the ``reframe`` package mentioned previously has a simple installation phase check that runs the installed executable.\nThe check is implemented as follows:\n\n.. code-block:: python\n\n class Reframe(Package):\n ...\n\n # check if we can run reframe\n @run_after(\"install\")\n @on_package_attributes(run_tests=True)\n def check_list(self):\n with working_dir(self.stage.source_path):\n reframe = Executable(self.prefix.bin.reframe)\n reframe(\"-l\")\n\nChecking build-time test results\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nChecking the results of these tests after running ``spack install --test`` can be done by viewing the spec's ``install-time-test-log.txt`` file whose location will depend on whether the spec installed successfully.\n\nA successful installation results in the build and stage logs being copied to the ``.spack`` subdirectory of the spec's prefix.\nFor example,\n\n.. code-block:: spec\n\n $ spack install --test=root zlib@1.2.13\n ...\n [+] /home/user/spack/opt/spack/linux-rhel8-broadwell/gcc-10.3.1/zlib-1.2.13-tehu6cbsujufa2tb6pu3xvc6echjstv6\n $ cat /home/user/spack/opt/spack/linux-rhel8-broadwell/gcc-10.3.1/zlib-1.2.13-tehu6cbsujufa2tb6pu3xvc6echjstv6/.spack/install-time-test-log.txt\n\nIf the installation fails due to build-time test failures, then both logs will be left in the build stage directory as illustrated below:\n\n.. code-block:: spec\n\n $ spack install --test=root zlib@1.2.13\n ...\n See build log for details:\n /var/tmp/user/spack-stage/spack-stage-zlib-1.2.13-lxfsivs4htfdewxe7hbi2b3tekj4make/spack-build-out.txt\n\n $ cat /var/tmp/user/spack-stage/spack-stage-zlib-1.2.13-lxfsivs4htfdewxe7hbi2b3tekj4make/install-time-test-log.txt\n\n\n.. _cmd-spack-test:\n\nStand-alone tests\n^^^^^^^^^^^^^^^^^\n\nWhile build-time tests are integrated with the installation process, stand-alone tests are expected to run days, weeks, even months after the software is installed.\nThe goal is to provide a mechanism for gaining confidence that packages work as installed **and** *continue* to work as the underlying software evolves.\nPackages can add and inherit stand-alone tests.\nThe ``spack test`` command is used for stand-alone testing.\n\n.. admonition:: Stand-alone test methods should complete within a few minutes.\n\n Execution speed is important since these tests are intended to quickly assess whether installed specs work on the system.\n Spack cannot spare resources for more extensive testing of packages included in CI stacks.\n\n Consequently, stand-alone tests should run relatively quickly -- as in on the order of at most a few minutes -- while testing at least key aspects of the installed software.\n Save more extensive testing for other tools.\n\nTests are defined in the package using methods with names beginning ``test_``.\nThis allows Spack to support multiple independent checks, or parts.\nFiles needed for testing, such as source, data, and expected outputs, may be saved from the build and/or stored with the package in the repository.\nRegardless of origin, these files are automatically copied to the spec's test stage directory prior to execution of the test method(s).\nSpack also provides helper functions to facilitate common processing.\n\n.. tip::\n\n **The status of stand-alone tests can be used to guide follow-up testing efforts.**\n\n Passing stand-alone tests justifies performing more thorough testing, such as running extensive unit or regression tests or tests that run at scale, when available.\n These tests are outside of the scope of Spack packaging.\n\n Failing stand-alone tests indicate problems with the installation and, therefore, no reason to proceed with more resource-intensive tests until the failures have been investigated.\n\n.. _configure-test-stage:\n\nConfiguring the test stage directory\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nStand-alone tests utilize a test stage directory to build, run, and track tests in the same way Spack uses a build stage directory to install software.\nThe default test stage root directory, ``$HOME/.spack/test``, is defined in :ref:`config.yaml `.\nThis location is customizable by adding or changing the ``test_stage`` path such that:\n\n.. code-block:: yaml\n\n config:\n test_stage: /path/to/test/stage\n\nPackages can use the ``self.test_suite.stage`` property to access the path.\n\n.. admonition:: Each spec being tested has its own test stage directory.\n\n The ``config:test_stage`` option is the path to the root of a **test suite**'s stage directories.\n\n Other package properties that provide paths to spec-specific subdirectories and files are described in :ref:`accessing-files`.\n\n.. _adding-standalone-tests:\n\nAdding stand-alone tests\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nTest recipes are defined in the package using methods with names beginning ``test_``.\nThis allows for the implementation of multiple independent tests.\nEach method has access to the information Spack tracks on the package, such as options, compilers, and dependencies, supporting the customization of tests to the build.\nStandard Python ``assert`` statements and other error reporting mechanisms can be used.\nThese exceptions are automatically caught and reported as test failures.\n\nEach test method is an *implicit test part* named by the method.\nIts purpose is the method's docstring.\nProviding a meaningful purpose for the test gives context that can aid debugging.\nSpack outputs both the name and purpose at the start of test execution so it's also important that the docstring/purpose be brief.\n\n.. tip::\n\n We recommend naming test methods so it is clear *what* is being tested.\n For example, if a test method is building and/or running an executable called ``example``, then call the method ``test_example``.\n This, together with a similarly meaningful test purpose, will aid test comprehension, debugging, and maintainability.\n\nStand-alone tests run in an environment that provides access to information on the installed software, such as build options, dependencies, and compilers.\nBuild options and dependencies are accessed using the same spec checks used by build recipes.\nExamples of checking :ref:`variant settings ` and :ref:`spec constraints ` can be found at the provided links.\n\n.. admonition:: Spack automatically sets up the test stage directory and environment.\n\n Spack automatically creates the test stage directory and copies relevant files *prior to* running tests.\n It can also ensure build dependencies are available **if** necessary.\n\n The path to the test stage is configurable (see :ref:`configure-test-stage`).\n\n Files that Spack knows to copy are those saved from the build (see :ref:`cache_extra_test_sources`) and those added to the package repository (see :ref:`cache_custom_files`).\n\n Spack will use the value of the ``test_requires_compiler`` property to determine whether it needs to also set up build dependencies (see :ref:`test-build-tests`).\n\nThe ``MyPackage`` package below provides two basic test examples: ``test_example`` and ``test_example2``.\nThe first runs the installed ``example`` and ensures its output contains an expected string.\nThe second runs ``example2`` without checking output so is only concerned with confirming the executable runs successfully.\nIf the installed spec is not expected to have ``example2``, then the check at the top of the method will raise a special ``SkipTest`` exception, which is captured to facilitate reporting skipped test parts to tools like CDash.\n\n.. code-block:: python\n\n class MyPackage(Package):\n ...\n\n def test_example(self):\n \"\"\"ensure installed example works\"\"\"\n expected = \"Done.\"\n example = which(self.prefix.bin.example)\n\n # Capture stdout and stderr from running the Executable\n # and check that the expected output was produced.\n out = example(output=str.split, error=str.split)\n assert expected in out, f\"Expected '{expected}' in the output\"\n\n def test_example2(self):\n \"\"\"run installed example2\"\"\"\n if self.spec.satisfies(\"@:1.0\"):\n # Raise SkipTest to ensure flagging the test as skipped for\n # test reporting purposes.\n raise SkipTest(\"Test is only available for v1.1 on\")\n\n example2 = which(self.prefix.bin.example2)\n example2()\n\nOutput showing the identification of each test part after running the tests is illustrated below.\n\n.. code-block:: console\n\n $ spack test run --alias mypackage mypackage@2.0\n ==> Spack test mypackage\n ...\n $ spack test results -l mypackage\n ==> Results for test suite 'mypackage':\n ...\n ==> [2024-03-10-16:03:56.625439] test: test_example: ensure installed example works\n ...\n PASSED: MyPackage::test_example\n ==> [2024-03-10-16:03:56.625439] test: test_example2: run installed example2\n ...\n PASSED: MyPackage::test_example2\n\n.. admonition:: Do NOT implement tests that must run in the installation prefix.\n\n Use of the package spec's installation prefix for building and running tests is **strongly discouraged**.\n Doing so causes permission errors for shared spack instances *and* facilities that install the software in read-only file systems or directories.\n\n Instead, start these test methods by explicitly copying the needed files from the installation prefix to the test stage directory.\n Note the test stage directory is the current directory when the test is executed with the ``spack test run`` command.\n\n.. admonition:: Test methods for library packages should build test executables.\n\n Stand-alone tests for library packages *should* build test executables that utilize the *installed* library.\n Doing so ensures the tests follow a similar build process that users of the library would follow.\n\n For more information on how to do this, see :ref:`test-build-tests`.\n\n.. tip::\n\n If you want to see more examples from packages with stand-alone tests, run ``spack pkg grep \"def\\stest\" | sed \"s/\\/package.py.*//g\" | sort -u`` from the command line to get a list of the packages.\n\n.. _adding-standalone-test-parts:\n\nAdding stand-alone test parts\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nSometimes dependencies between steps of a test lend themselves to being broken into parts.\nTracking the pass/fail status of each part can aid debugging.\nSpack provides a ``test_part`` context manager for use within test methods.\n\nEach test part is independently run, tracked, and reported.\nTest parts are executed in the order they appear.\nIf one fails, subsequent test parts are still performed even if they would also fail.\nThis allows tools like CDash to track and report the status of test parts across runs.\nThe pass/fail status of the enclosing test is derived from the statuses of the embedded test parts.\n\n.. admonition:: Test method and test part names **must** be unique.\n\n Test results reporting requires that test methods and embedded test parts within a package have unique names.\n\nThe signature for ``test_part`` is:\n\n.. code-block:: python\n\n def test_part(pkg, test_name, purpose, work_dir=\".\", verbose=False): ...\n\nwhere each argument has the following meaning:\n\n* ``pkg`` is an instance of the package for the spec under test.\n\n* ``test_name`` is the name of the test part, which must start with ``test_``.\n\n* ``purpose`` is a brief description used as a heading for the test part.\n\n Output from the test is written to a test log file allowing the test name and purpose to be searched for test part confirmation and debugging.\n\n* ``work_dir`` is the path to the directory in which the test will run.\n\n The default of ``None``, or ``\".\"``, corresponds to the spec's test stage (i.e., ``self.test_suite.test_dir_for_spec(self.spec)``).\n\n.. admonition:: Start test part names with the name of the enclosing test.\n\n We **highly recommend** starting the names of test parts with the name of the enclosing test.\n Doing so helps with the comprehension, readability and debugging of test results.\n\nSuppose ``MyPackage`` installs multiple executables that need to run in a specific order since the outputs from one are inputs of others.\nFurther suppose we want to add an integration test that runs the executables in order.\nWe can accomplish this goal by implementing a stand-alone test method consisting of test parts for each executable as follows:\n\n.. code-block:: python\n\n class MyPackage(Package):\n ...\n\n def test_series(self):\n \"\"\"run setup, perform, and report\"\"\"\n\n with test_part(self, \"test_series_setup\", purpose=\"setup operation\"):\n exe = which(self.prefix.bin.setup)\n exe()\n\n with test_part(self, \"test_series_run\", purpose=\"perform operation\"):\n exe = which(self.prefix.bin.run)\n exe()\n\n with test_part(self, \"test_series_report\", purpose=\"generate report\"):\n exe = which(self.prefix.bin.report)\n exe()\n\nThe result is ``test_series`` runs the following executable in order: ``setup``, ``run``, and ``report``.\nIn this case no options are passed to any of the executables and no outputs from running them are checked.\nConsequently, the implementation could be simplified with a for-loop as follows:\n\n.. code-block:: python\n\n class MyPackage(Package):\n ...\n\n def test_series(self):\n \"\"\"execute series setup, run, and report\"\"\"\n\n for exe, reason in [\n (\"setup\", \"setup operation\"),\n (\"run\", \"perform operation\"),\n (\"report\", \"generate report\"),\n ]:\n with test_part(self, f\"test_series_{exe}\", purpose=reason):\n exe = which(self.prefix.bin.join(exe))\n exe()\n\nIn both cases, since we're using a context manager, each test part in ``test_series`` will execute regardless of the status of the other test parts.\n\nNow let's look at the output from running the stand-alone tests where the second test part, ``test_series_run``, fails.\n\n.. code-block:: console\n\n $ spack test run --alias mypackage mypackage@1.0\n ==> Spack test mypackage\n ...\n $ spack test results -l mypackage\n ==> Results for test suite 'mypackage':\n ...\n ==> [2024-03-10-16:03:56.625204] test: test_series: execute series setup, run, and report\n ==> [2024-03-10-16:03:56.625439] test: test_series_setup: setup operation\n ...\n PASSED: MyPackage::test_series_setup\n ==> [2024-03-10-16:03:56.625555] test: test_series_run: perform operation\n ...\n FAILED: MyPackage::test_series_run\n ==> [2024-03-10-16:03:57.003456] test: test_series_report: generate report\n ...\n FAILED: MyPackage::test_series_report\n FAILED: MyPackage::test_series\n ...\n\nSince test parts depended on the success of previous parts, we see that the failure of one results in the failure of subsequent checks and the overall result of the test method, ``test_series``, is failure.\n\n.. tip::\n\n If you want to see more examples from packages using ``test_part``, run ``spack pkg grep \"test_part(\" | sed \"s/\\/package.py.*//g\" | sort -u`` from the command line to get a list of the packages.\n\n.. _test-build-tests:\n\nBuilding and running test executables\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\n.. admonition:: Reuse build-time sources and (small) input data sets when possible.\n\n We **highly recommend** reusing build-time test sources and pared down input files for testing installed software.\n These files are easier to keep synchronized with software capabilities when they reside within the software's repository.\n More information on saving files from the installation process can be found at :ref:`cache_extra_test_sources`.\n\n If that is not possible, you can add test-related files to the package repository (see :ref:`cache_custom_files`).\n It will be important to remember to maintain them so they work across listed or supported versions of the package.\n\nPackages that build libraries are good examples of cases where you'll want to build test executables from the installed software before running them.\nDoing so requires you to let Spack know it needs to load the package's compiler configuration.\nThis is accomplished by setting the package's ``test_requires_compiler`` property to ``True``.\n\n.. admonition:: ``test_requires_compiler = True`` is required to build test executables.\n\n Setting the property to ``True`` ensures access to the compiler through canonical environment variables (e.g., ``CC``, ``CXX``, ``FC``, ``F77``).\n It also gives access to build dependencies like ``cmake`` through their ``spec objects`` (e.g., ``self.spec[\"cmake\"].prefix.bin.cmake`` for the path or ``self.spec[\"cmake\"].command`` for the ``Executable`` instance).\n\n Be sure to add the property at the top of the package class under other properties like the ``homepage``.\n\nThe example below, which ignores how ``cxx-example.cpp`` is acquired, illustrates the basic process of compiling a test executable using the installed library before running it.\n\n.. code-block:: python\n\n class MyLibrary(Package):\n ...\n\n test_requires_compiler = True\n ...\n\n def test_cxx_example(self):\n \"\"\"build and run cxx-example\"\"\"\n exe = \"cxx-example\"\n ...\n cxx = which(os.environ[\"CXX\"])\n cxx(f\"-L{self.prefix.lib}\", f\"-I{self.prefix.include}\", f\"{exe}.cpp\", \"-o\", exe)\n cxx_example = which(exe)\n cxx_example()\n\nTypically the files used to build and/or run test executables are either cached from the installation (see :ref:`cache_extra_test_sources`) or added to the package repository (see :ref:`cache_custom_files`).\nThere is nothing preventing the use of both.\n\n.. _cache_extra_test_sources:\n\nSaving build- and install-time files\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nYou can use the ``cache_extra_test_sources`` helper routine to copy directories and/or files from the source build stage directory to the package's installation directory.\nSpack will automatically copy these files for you when it sets up the test stage directory and before it begins running the tests.\n\nThe signature for ``cache_extra_test_sources`` is:\n\n.. code-block:: python\n\n def cache_extra_test_sources(pkg, srcs): ...\n\nwhere each argument has the following meaning:\n\n* ``pkg`` is an instance of the package for the spec under test.\n\n* ``srcs`` is a string *or* a list of strings corresponding to the paths of subdirectories and/or files needed for stand-alone testing.\n\n.. warning::\n\n Paths provided in the ``srcs`` argument **must be relative** to the staged source directory.\n They will be copied to the equivalent relative location under the test stage directory prior to test execution.\n\nContents of subdirectories and files are copied to a special test cache subdirectory of the installation prefix.\nThey are automatically copied to the appropriate relative paths under the test stage directory prior to executing stand-alone tests.\n\n.. tip::\n\n *Perform test-related conversions once when copying files.*\n\n If one or more of the copied files needs to be modified to reference the installed software, it is recommended that those changes be made to the cached files **once** in the post-``install`` copy method **after** the call to ``cache_extra_test_sources``.\n This will reduce the amount of unnecessary work in the test method **and** avoid problems running stand-alone tests in shared instances and facility deployments.\n\n The ``filter_file`` function can be quite useful for such changes (see :ref:`file-filtering`).\n\nBelow is a basic example of a test that relies on files from the installation.\nThis package method reuses the contents of the ``examples`` subdirectory, which is assumed to have all of the files necessary to allow ``make`` to compile and link ``foo.c`` and ``bar.c`` against the package's installed library.\n\n.. code-block:: python\n\n class MyLibPackage(MakefilePackage):\n ...\n\n @run_after(\"install\")\n def copy_test_files(self):\n cache_extra_test_sources(self, \"examples\")\n\n def test_example(self):\n \"\"\"build and run the examples\"\"\"\n examples_dir = self.test_suite.current_test_cache_dir.examples\n with working_dir(examples_dir):\n make = which(\"make\")\n make()\n\n for program in [\"foo\", \"bar\"]:\n with test_part(self, f\"test_example_{program}\", purpose=f\"ensure {program} runs\"):\n exe = Executable(program)\n exe()\n\nIn this case, ``copy_test_files`` copies the associated files from the build stage to the package's test cache directory under the installation prefix.\nRunning ``spack test run`` for the package results in Spack copying the directory and its contents to the test stage directory.\nThe ``working_dir`` context manager ensures the commands within it are executed from the ``examples_dir``.\nThe test builds the software using ``make`` before running each executable, ``foo`` and ``bar``, as independent test parts.\n\n.. note::\n\n The method name ``copy_test_files`` here is for illustration purposes.\n You are free to use a name that is better suited to your package.\n\n The key to copying files for stand-alone testing at build time is use of the ``run_after`` directive, which ensures the associated files are copied **after** the provided build stage (``install``) when the installation prefix **and** files are available.\n\n The test method uses the path contained in the package's ``self.test_suite.current_test_cache_dir`` property for the root directory of the copied files.\n In this case, that's the ``examples`` subdirectory.\n\n.. tip::\n\n If you want to see more examples from packages that cache build files, run ``spack pkg grep cache_extra_test_sources | sed \"s/\\/package.py.*//g\" | sort -u`` from the command line to get a list of the packages.\n\n.. _cache_custom_files:\n\nAdding custom files\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nSometimes it is helpful or necessary to include custom files for building and/or checking the results of tests as part of the package.\nExamples of the types of files that might be useful are:\n\n- test source files\n- test input files\n- test build scripts\n- expected test outputs\n\nWhile obtaining such files from the software repository is preferred (see :ref:`cache_extra_test_sources`), there are circumstances where doing so is not feasible such as when the software is not being actively maintained.\nWhen test files cannot be obtained from the repository or there is a need to supplement files that can, Spack supports the inclusion of additional files under the ``test`` subdirectory of the package in the Spack repository.\n\nThe following example assumes a ``custom-example.c`` is saved in ``MyLibrary`` package's ``test`` subdirectory.\nIt also assumes the program simply needs to be compiled and linked against the installed ``MyLibrary`` software.\n\n.. code-block:: python\n\n class MyLibrary(Package):\n ...\n\n test_requires_compiler = True\n ...\n\n def test_custom_example(self):\n \"\"\"build and run custom-example\"\"\"\n src_dir = self.test_suite.current_test_data_dir\n exe = \"custom-example\"\n\n with working_dir(src_dir):\n cc = which(os.environ[\"CC\"])\n cc(f\"-L{self.prefix.lib}\", f\"-I{self.prefix.include}\", f\"{exe}.cpp\", \"-o\", exe)\n\n custom_example = Executable(exe)\n custom_example()\n\nIn this case, ``spack test run`` for the package results in Spack copying the contents of the ``test`` subdirectory to the test stage directory path in ``self.test_suite.current_test_data_dir`` before calling ``test_custom_example``.\nUse of the ``working_dir`` context manager ensures the commands to build and run the program are performed from within the appropriate subdirectory of the test stage.\n\n.. _expected_test_output_from_file:\n\nReading expected output from a file\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nThe helper function ``get_escaped_text_output`` is available for packages to retrieve properly formatted text from a file potentially containing special characters.\n\nThe signature for ``get_escaped_text_output`` is:\n\n.. code-block:: python\n\n def get_escaped_text_output(filename): ...\n\nwhere ``filename`` is the path to the file containing the expected output.\n\nThe path provided to ``filename`` for one of the copied custom files (:ref:`custom file `) is in the path rooted at ``self.test_suite.current_test_data_dir``.\n\nThe example below shows how to reference both the custom database (``packages.db``) and expected output (``dump.out``) files Spack copies to the test stage:\n\n.. code-block:: python\n\n import re\n\n\n class Sqlite(AutotoolsPackage):\n ...\n\n def test_example(self):\n \"\"\"check example table dump\"\"\"\n test_data_dir = self.test_suite.current_test_data_dir\n db_filename = test_data_dir.join(\"packages.db\")\n ...\n expected = get_escaped_text_output(test_data_dir.join(\"dump.out\"))\n sqlite3 = which(self.prefix.bin.sqlite3)\n out = sqlite3(db_filename, \".dump\", output=str.split, error=str.split)\n for exp in expected:\n assert re.search(exp, out), f\"Expected '{exp}' in output\"\n\nIf the files were instead cached from installing the software, the paths to the two files would be found under the ``self.test_suite.current_test_cache_dir`` directory as shown below:\n\n.. code-block:: python\n\n def test_example(self):\n \"\"\"check example table dump\"\"\"\n test_cache_dir = self.test_suite.current_test_cache_dir\n db_filename = test_cache_dir.join(\"packages.db\")\n ...\n expected = get_escaped_text_output(test_cache_dir.join(\"dump.out\"))\n ...\n\nAlternatively, if both files had been installed by the software into the ``share/tests`` subdirectory of the installation prefix, the paths to the two files would be referenced as follows:\n\n.. code-block:: python\n\n def test_example(self):\n \"\"\"check example table dump\"\"\"\n db_filename = self.prefix.share.tests.join(\"packages.db\")\n ...\n expected = get_escaped_text_output(self.prefix.share.tests.join(\"dump.out\"))\n ...\n\n.. _check_outputs:\n\nComparing expected to actual outputs\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nThe ``check_outputs`` helper routine is available for packages to ensure multiple expected outputs from running an executable are contained within the actual outputs.\n\nThe signature for ``check_outputs`` is:\n\n.. code-block:: python\n\n def check_outputs(expected, actual): ...\n\nwhere each argument has the expected type and meaning:\n\n* ``expected`` is a string or list of strings containing the expected (raw) output.\n\n* ``actual`` is a string containing the actual output from executing the command.\n\nInvoking the method is the equivalent of:\n\n.. code-block:: python\n\n errors = []\n for check in expected:\n if not re.search(check, actual):\n errors.append(f\"Expected '{check}' in output '{actual}'\")\n if errors:\n raise RuntimeError(\"\\n \".join(errors))\n\n.. tip::\n\n If you want to see more examples from packages that use this helper, run ``spack pkg grep check_outputs | sed \"s/\\/package.py.*//g\" | sort -u`` from the command line to get a list of the packages.\n\n\n.. _accessing-files:\n\nFinding package- and test-related files\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nYou may need to access files from one or more locations when writing stand-alone tests.\nThis can happen if the software's repository does not include test source files or includes them but has no way to build the executables using the installed headers and libraries.\nIn these cases you may need to reference the files relative to one or more root directories.\nThe table below lists relevant path properties and provides additional examples of their use.\nSee :ref:`expected_test_output_from_file` for examples of accessing files saved from the software repository, package repository, and installation.\n\n.. list-table:: Directory-to-property mapping\n :header-rows: 1\n\n * - Root Directory\n - Package Property\n - Example(s)\n * - Package (Spec) Installation\n - ``self.prefix``\n - ``self.prefix.include``, ``self.prefix.lib``\n * - Dependency Installation\n - ``self.spec[\"\"].prefix``\n - ``self.spec[\"trilinos\"].prefix.include``\n * - Test Suite Stage\n - ``self.test_suite.stage``\n - ``join_path(self.test_suite.stage, \"results.txt\")``\n * - Spec's Test Stage\n - ``self.test_suite.test_dir_for_spec()``\n - ``self.test_suite.test_dir_for_spec(self.spec)``\n * - Current Spec's Build-time Files\n - ``self.test_suite.current_test_cache_dir``\n - ``join_path(self.test_suite.current_test_cache_dir.examples, \"foo.c\")``\n * - Current Spec's Custom Test Files\n - ``self.test_suite.current_test_data_dir``\n - ``join_path(self.test_suite.current_test_data_dir, \"hello.f90\")``\n\n.. _inheriting-tests:\n\nInheriting stand-alone tests\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nStand-alone tests defined in parent (e.g., :ref:`build-systems`) and virtual (e.g., :ref:`virtual-dependencies`) packages are executed by packages that inherit from or provide interface implementations for those packages, respectively.\n\nThe table below summarizes the stand-alone tests that will be executed along with those implemented in the package itself.\n\n.. list-table:: Inherited/provided stand-alone tests\n :header-rows: 1\n\n * - Parent/Provider Package\n - Stand-alone Tests\n * - `C\n `_\n - Compiles ``hello.c`` and runs it\n * - `Cxx\n `_\n - Compiles and runs several ``hello`` programs\n * - `Fortran\n `_\n - Compiles and runs ``hello`` programs (``F`` and ``f90``)\n * - `Mpi\n `_\n - Compiles and runs ``mpi_hello`` (``c``, ``fortran``)\n * - :ref:`PythonPackage `\n - Imports modules listed in the ``self.import_modules`` property with defaults derived from the tarball\n * - :ref:`SipPackage `\n - Imports modules listed in the ``self.import_modules`` property with defaults derived from the tarball\n\nThese tests are very basic so it is important that package developers and maintainers provide additional stand-alone tests customized to the package.\n\n.. warning::\n\n Any package that implements a test method with the same name as an inherited method will override the inherited method.\n If that is not the goal and you are not explicitly calling and adding functionality to the inherited method for the test, then make sure that all test methods and embedded test parts have unique test names.\n\nOne example of a package that adds its own stand-alone tests to those \"inherited\" by the virtual package it provides an implementation for is the `OpenMPI package `_.\n\nBelow are snippets from running and viewing the stand-alone test results for ``openmpi``:\n\n.. code-block:: console\n\n $ spack test run --alias openmpi openmpi@4.1.4\n ==> Spack test openmpi\n ==> Testing package openmpi-4.1.4-ubmrigj\n ============================== 1 passed of 1 spec ==============================\n\n $ spack test results -l openmpi\n ==> Results for test suite 'openmpi':\n ==> test specs:\n ==> openmpi-4.1.4-ubmrigj PASSED\n ==> Testing package openmpi-4.1.4-ubmrigj\n ==> [2023-03-10-16:03:56.160361] Installing $spack/opt/spack/linux-rhel7-broadwell/gcc-8.3.1/openmpi-4.1.4-ubmrigjrqcafh3hffqcx7yz2nc5jstra/.spack/test to $test_stage/xez37ekynfbi4e7h4zdndfemzufftnym/openmpi-4.1.4-ubmrigj/cache/openmpi\n ==> [2023-03-10-16:03:56.625204] test: test_bin: test installed binaries\n ==> [2023-03-10-16:03:56.625439] test: test_bin_mpirun: run and check output of mpirun\n ==> [2023-03-10-16:03:56.629807] '$spack/opt/spack/linux-rhel7-broadwell/gcc-8.3.1/openmpi-4.1.4-ubmrigjrqcafh3hffqcx7yz2nc5jstra/bin/mpirun' '-n' '1' 'ls' '..'\n openmpi-4.1.4-ubmrigj repo\n openmpi-4.1.4-ubmrigj-test-out.txt test_suite.lock\n PASSED: test_bin_mpirun\n ...\n ==> [2023-03-10-16:04:01.486977] test: test_version_oshcc: ensure version of oshcc is 8.3.1\n SKIPPED: test_version_oshcc: oshcc is not installed\n ...\n ==> [2023-03-10-16:04:02.215227] Completed testing\n ==> [2023-03-10-16:04:02.215597]\n ======================== SUMMARY: openmpi-4.1.4-ubmrigj ========================\n Openmpi::test_bin_mpirun .. PASSED\n Openmpi::test_bin_ompi_info .. PASSED\n Openmpi::test_bin_oshmem_info .. SKIPPED\n Openmpi::test_bin_oshrun .. SKIPPED\n Openmpi::test_bin_shmemrun .. SKIPPED\n Openmpi::test_bin .. PASSED\n ...\n ============================== 1 passed of 1 spec ==============================\n\n\n.. _cmd-spack-test-list:\n\n``spack test list``\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nPackages available for install testing can be found using the ``spack test list`` command.\nThe command outputs all installed packages that have defined stand-alone test methods.\n\nAlternatively you can use the ``--all`` option to get a list of all packages that have stand-alone test methods even if the packages are not installed.\n\nFor more information, refer to :ref:`spack-test-list`.\n\n.. _cmd-spack-test-run:\n\n``spack test run``\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nInstall tests can be run for one or more installed packages using the ``spack test run`` command.\nA ``test suite`` is created for all of the provided specs.\nThe command accepts the same arguments provided to ``spack install`` (see :ref:`sec-specs`).\nIf no specs are provided the command tests all specs in the active environment or all specs installed in the Spack instance if no environment is active.\n\nTest suites can be named using the ``--alias`` option.\nUnaliased test suites use the content hash of their specs as their name.\n\nSome of the more commonly used debugging options are:\n\n- ``--fail-fast`` stops testing each package after the first failure\n- ``--fail-first`` stops testing packages after the first failure\n\nTest output is written to a text log file by default, though ``junit`` and ``cdash`` are outputs available through the ``--log-format`` option.\n\nFor more information, refer to :ref:`spack-test-run`.\n\n\n.. _cmd-spack-test-results:\n\n``spack test results``\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nThe ``spack test results`` command shows results for all completed test suites by default.\nThe alias or content hash can be provided to limit reporting to the corresponding test suite.\n\nThe ``--logs`` option includes the output generated by the associated test(s) to facilitate debugging.\n\nThe ``--failed`` option limits results shown to that of the failed tests, if any, of matching packages.\n\nFor more information, refer to :ref:`spack-test-results`.\n\n.. _cmd-spack-test-find:\n\n``spack test find``\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nThe ``spack test find`` command lists the aliases or content hashes of all test suites whose results are available.\n\nFor more information, refer to :ref:`spack-test-find`.\n\n.. _cmd-spack-test-remove:\n\n``spack test remove``\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nThe ``spack test remove`` command removes test suites to declutter the test stage directory.\nYou are prompted to confirm the removal of each test suite **unless** you use the ``--yes-to-all`` option.\n\nFor more information, refer to :ref:`spack-test-remove`.\n" }, { "path": "lib/spack/docs/pipelines.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Learn how to generate and run automated build pipelines in Spack for CI instances, enabling the building and deployment of binaries and reporting to CDash.\n\n.. _pipelines:\n\nCI Pipelines\n============\n\nSpack provides commands that support generating and running automated build pipelines in CI instances.\nAt the highest level, it works like this: provide a Spack environment describing the set of packages you care about, and include a description of how those packages should be mapped to Gitlab runners.\nSpack can then generate a ``.gitlab-ci.yml`` file containing job descriptions for all your packages that can be run by a properly configured CI instance.\nWhen run, the generated pipeline will build and deploy binaries, and it can optionally report to a CDash instance regarding the health of the builds as they evolve over time.\n\nGetting started with pipelines\n------------------------------\n\nTo get started with automated build pipelines, a Gitlab instance with version ``>= 12.9`` (more about Gitlab CI `here `_) with at least one `runner `_ configured is required.\nThis can be done quickly by setting up a local Gitlab instance.\n\nIt is possible to set up pipelines on gitlab.com, but the builds there are limited to 60 minutes and generic hardware.\nIt is also possible to `hook up `_ Gitlab to Google Kubernetes Engine (`GKE `_) or Amazon Elastic Kubernetes Service (`EKS `_), though those topics are outside the scope of this document.\n\nAfter setting up a Gitlab instance for running CI, the basic steps for setting up a build pipeline are as follows:\n\n#. Create a repository in the Gitlab instance with CI and a runner enabled.\n#. Add a ``spack.yaml`` at the root containing your pipeline environment\n#. Add a ``.gitlab-ci.yml`` at the root containing two jobs (one to generate the pipeline dynamically, and one to run the generated jobs).\n#. Push a commit containing the ``spack.yaml`` and ``.gitlab-ci.yml`` mentioned above to the GitLab repository\n\nSee the :ref:`functional_example` section for a minimal working example.\nSee also the :ref:`custom_Workflow` section for a link to an example of a custom workflow based on Spack pipelines.\n\nSpack's pipelines are now making use of the `trigger `_ syntax to run dynamically generated `child pipelines `_.\nNote that the use of dynamic child pipelines requires running Gitlab version ``>= 12.9``.\n\n.. _functional_example:\n\nFunctional Example\n------------------\n\nThe simplest fully functional standalone example of a working pipeline can be examined live at this example `project `_ on gitlab.com.\n\nHere's the ``.gitlab-ci.yml`` file from that example that builds and runs the pipeline:\n\n.. code-block:: yaml\n\n stages: [\"generate\", \"build\"]\n\n variables:\n SPACK_REPOSITORY: \"https://github.com/spack/spack.git\"\n SPACK_REF: \"develop-2024-10-06\"\n SPACK_USER_CONFIG_PATH: ${CI_PROJECT_DIR}\n SPACK_BACKTRACE: 1\n\n generate-pipeline:\n tags:\n - saas-linux-small-amd64\n stage: generate\n image:\n name: ghcr.io/spack/ubuntu20.04-runner-x86_64:2023-01-01\n script:\n - git clone ${SPACK_REPOSITORY}\n - cd spack && git checkout ${SPACK_REF} && cd ../\n - . \"./spack/share/spack/setup-env.sh\"\n - spack --version\n - spack env activate --without-view .\n - spack -d -v --color=always ci generate --check-index-only --artifacts-root \"${CI_PROJECT_DIR}/jobs_scratch_dir\" --output-file \"${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml\"\n artifacts:\n paths:\n - \"${CI_PROJECT_DIR}/jobs_scratch_dir\"\n\n build-pipeline:\n stage: build\n trigger:\n include:\n - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml\n job: generate-pipeline\n strategy: depend\n needs:\n - artifacts: true\n job: generate-pipeline\n\n\nThe key thing to note above is that there are two jobs: The first job to run, ``generate-pipeline``, runs the ``spack ci generate`` command to generate a dynamic child pipeline and write it to a YAML file, which is then picked up by the second job, ``build-jobs``, and used to trigger the downstream pipeline.\n\nAnd here's the Spack environment built by the pipeline represented as a ``spack.yaml`` file:\n\n.. code-block:: yaml\n\n spack:\n view: false\n concretizer:\n unify: true\n reuse: false\n\n definitions:\n - pkgs:\n - zlib\n - bzip2 ~debug\n - compiler:\n - \"%gcc\"\n\n specs:\n - matrix:\n - - $pkgs\n - - $compiler\n\n ci:\n target: gitlab\n\n pipeline-gen:\n - any-job:\n tags:\n - saas-linux-small-amd64\n image:\n name: ghcr.io/spack/ubuntu20.04-runner-x86_64:2023-01-01\n before_script:\n - git clone ${SPACK_REPOSITORY}\n - cd spack && git checkout ${SPACK_REF} && cd ../\n - . \"./spack/share/spack/setup-env.sh\"\n - spack --version\n - export SPACK_USER_CONFIG_PATH=${CI_PROJECT_DIR}\n - spack config blame mirrors\n\n\n.. note::\n The use of ``reuse: false`` in Spack environments used for pipelines is almost always what you want, as without it your pipelines will not rebuild packages even if package hashes have changed.\n This is due to the concretizer strongly preferring known hashes when ``reuse: true``.\n\nThe ``ci`` section in the above environment file contains the bare minimum configuration required for ``spack ci generate`` to create a working pipeline.\nThe ``target: gitlab`` tells Spack that the desired pipeline output is for GitLab.\nHowever, this isn't strictly required, as currently, GitLab is the only possible output format for pipelines.\nThe ``pipeline-gen`` section contains the key information needed to specify attributes for the generated jobs.\nNotice that it contains a list which has only a single element in this case.\nIn real pipelines, it will almost certainly have more elements, and in those cases, order is important: Spack starts at the bottom of the list and works upwards when applying attributes.\n\nBut in this simple case, we use only the special key ``any-job`` to indicate that Spack should apply the specified attributes (``tags``, ``image``, and ``before_script``) to any job it generates.\nThis includes jobs for building/pushing all packages, a ``rebuild-index`` job at the end of the pipeline, as well as any ``noop`` jobs that might be needed by GitLab when no rebuilds are required.\n\nSomething to note is that in this simple case, we rely on Spack to generate a reasonable script for the package build jobs (it just creates a script that invokes ``spack ci rebuild``).\n\nAnother thing to note is the use of the ``SPACK_USER_CONFIG_DIR`` environment variable in any generated jobs.\nThe purpose of this is to make Spack aware of one final file in the example, the one that contains the mirror configuration.\nThis file, ``mirrors.yaml`` looks like this:\n\n.. code-block:: yaml\n\n mirrors:\n buildcache-destination:\n url: oci://registry.gitlab.com/spack/pipeline-quickstart\n binary: true\n access_pair:\n id_variable: CI_REGISTRY_USER\n secret_variable: CI_REGISTRY_PASSWORD\n\n\nNote the name of the mirror is ``buildcache-destination``, which is required as of Spack 0.23 (see below for more information).\nThe mirror URL simply points to the container registry associated with the project, while ``id_variable`` and ``secret_variable`` refer to environment variables containing the access credentials for the mirror.\n\nWhen Spack builds packages for this example project, they will be pushed to the project container registry, where they will be available for subsequent jobs to install as dependencies or for other pipelines to use to build runnable container images.\n\nSpack commands supporting pipelines\n-----------------------------------\n\nSpack provides a ``ci`` command with a few sub-commands supporting Spack CI pipelines.\nThese commands are covered in more detail in this section.\n\n.. _cmd-spack-ci:\n\n``spack ci``\n^^^^^^^^^^^^\n\nSuper-command for functionality related to generating pipelines and executing pipeline jobs.\n\n.. _cmd-spack-ci-generate:\n\n``spack ci generate``\n^^^^^^^^^^^^^^^^^^^^^\n\nThroughout this documentation, references to the \"mirror\" mean the target mirror which is checked for the presence of up-to-date specs, and where any scheduled jobs should push built binary packages.\nWhen running ``spack ci generate`` it is required to configure a mirror named ``buildcache-destination`` to be used as the target mirror.\nIt is permitted to configure any number of other mirrors as sources for your pipelines, but only the ``buildcache-destination`` mirror will be used as the destination mirror.\n\nConcretizes the specs in the active environment, stages them (as described in :ref:`staging_algorithm`), and writes the resulting ``.gitlab-ci.yml`` to disk.\nDuring concretization of the environment, ``spack ci generate`` also writes a ``spack.lock`` file which is then provided to generated child jobs and made available in all generated job artifacts to aid in reproducing failed builds in a local environment.\nThis means there are two artifacts that need to be exported in your pipeline generation job (defined in your ``.gitlab-ci.yml``).\nThe first is the output yaml file of ``spack ci generate``, and the other is the directory containing the concrete environment files.\nIn the :ref:`functional_example` section, we only mentioned one path in the ``artifacts`` ``paths`` list because we used ``--artifacts-root`` as the top level directory containing both the generated pipeline yaml and the concrete environment.\n\nUsing ``--prune-dag`` or ``--no-prune-dag`` configures whether or not jobs are generated for specs that are already up to date on the mirror.\nIf enabling DAG pruning using ``--prune-dag``, more information may be required in your ``spack.yaml`` file, see the :ref:`noop_jobs` section below regarding ``noop-job``.\n\nThe optional ``--check-index-only`` argument can be used to speed up pipeline generation by telling Spack to consider only remote build cache indices when checking the remote mirror to determine if each spec in the DAG is up to date or not.\nThe default behavior is for Spack to fetch the index and check it, but if the spec is not found in the index, it also performs a direct check for the spec on the mirror.\nIf the remote build cache index is out of date, which can easily happen if it is not updated frequently, this behavior ensures that Spack has a way to know for certain about the status of any concrete spec on the remote mirror, but can slow down pipeline generation significantly.\n\nThe optional ``--output-file`` argument should be an absolute path (including file name) to the generated pipeline, and if not given, the default is ``./.gitlab-ci.yml``.\n\nWhile optional, the ``--artifacts-root`` argument is used to determine where the concretized environment directory should be located.\nThis directory will be created by ``spack ci generate`` and will contain the ``spack.yaml`` and generated ``spack.lock`` which are then passed to all child jobs as an artifact.\nThis directory will also be the root directory for all artifacts generated by jobs in the pipeline.\n\n.. _cmd-spack-ci-rebuild:\n\n``spack ci rebuild``\n^^^^^^^^^^^^^^^^^^^^\n\nThe purpose of ``spack ci rebuild`` is to take an assigned spec and ensure a binary of a successful build exists on the target mirror.\nIf the binary does not already exist, it is built from source and pushed to the mirror.\nThe associated stand-alone tests are optionally run against the new build.\nAdditionally, files for reproducing the build outside the CI environment are created to facilitate debugging.\n\nIf a binary for the spec does not exist on the target mirror, an install shell script, ``install.sh``, is created and saved in the current working directory.\nThe script is run in a job to install the spec from source.\nThe resulting binary package is pushed to the mirror.\nIf ``cdash`` is configured for the environment, the build results will be uploaded to the site.\n\nEnvironment variables and values in the ``ci::pipeline-gen`` section of the ``spack.yaml`` environment file provide inputs to this process.\nThe two main sources of environment variables are variables written into ``.gitlab-ci.yml`` by ``spack ci generate`` and the GitLab CI runtime.\nSeveral key CI pipeline variables are described in :ref:`ci_environment_variables`.\n\nIf the ``--tests`` option is provided, stand-alone tests are performed but only if the build was successful *and* the package does not appear in the list of ``broken-tests-packages``.\nA shell script, ``test.sh``, is created and run to perform the tests.\nOn completion, test logs are exported as job artifacts for review and to facilitate debugging.\nIf ``cdash`` is configured, test results are also uploaded to the site.\n\nA snippet from an example ``spack.yaml`` file illustrating use of this option *and* specification of a package with broken tests is given below.\nThe inclusion of a spec for building ``gptune`` is not shown here.\nNote that ``--tests`` is passed to ``spack ci rebuild`` as part of the ``build-job`` script.\n\n.. code-block:: yaml\n\n ci:\n pipeline-gen:\n - build-job:\n script:\n - . \"./share/spack/setup-env.sh\"\n - spack --version\n - cd ${SPACK_CONCRETE_ENV_DIR}\n - spack env activate --without-view .\n - spack config add \"config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture.platform}-{architecture.target}/{name}-{version}-{hash}'\"\n - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data\n - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi\n - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi\n - spack -d ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)\n\n broken-tests-packages:\n - gptune\n\nIn this case, even if ``gptune`` is successfully built from source, the pipeline will *not* run its stand-alone tests since the package is listed under ``broken-tests-packages``.\n\nSpack's cloud pipelines provide actual, up-to-date examples of the CI/CD configuration and environment files used by Spack.\nYou can find them under Spack's `stacks `_ repository directory.\n\n.. _cmd-spack-ci-rebuild-index:\n\n``spack ci rebuild-index``\n^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nThis is a convenience command to rebuild the build cache index associated with the mirror in the active, GitLab-enabled environment (specifying the mirror URL or name is not required).\n\n.. _cmd-spack-ci-reproduce-build:\n\n``spack ci reproduce-build``\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nGiven the URL to a GitLab pipeline rebuild job, downloads and unzips the artifacts into a local directory (which can be specified with the optional ``--working-dir`` argument), then finds the target job in the generated pipeline to extract details about how it was run.\nAssuming the job used a docker image, the command prints a ``docker run`` command line and some basic instructions on how to reproduce the build locally.\n\nNote that jobs failing in the pipeline will print messages giving the arguments you can pass to ``spack ci reproduce-build`` in order to reproduce a particular build locally.\n\nJob Types\n------------------------------------\n\nRebuild (build)\n^^^^^^^^^^^^^^^\n\nRebuild jobs, denoted as ``build-job``'s in the ``pipeline-gen`` list, are jobs associated with concrete specs that have been marked for rebuild.\nBy default, a simple script for doing rebuild is generated but may be modified as needed.\n\nThe default script does three main steps: change directories to the pipelines concrete environment, activate the concrete environment, and run the ``spack ci rebuild`` command:\n\n.. code-block:: bash\n\n cd ${concrete_environment_dir}\n spack env activate --without-view .\n spack ci rebuild\n\n.. _rebuild_index:\n\nUpdate Index (reindex)\n^^^^^^^^^^^^^^^^^^^^^^\n\nBy default, while a pipeline job may rebuild a package, create a build cache entry, and push it to the mirror, it does not automatically re-generate the mirror's build cache index afterward.\nBecause the index is not needed by the default rebuild jobs in the pipeline, not updating the index at the end of each job avoids possible race conditions between simultaneous jobs, and it avoids the computational expense of regenerating the index.\nThis potentially saves minutes per job, depending on the number of binary packages in the mirror.\nAs a result, the default is that the mirror's build cache index may not correctly reflect the mirror's contents at the end of a pipeline.\n\nTo make sure the build cache index is up to date at the end of your pipeline, Spack generates a job to update the build cache index of the target mirror at the end of each pipeline by default.\nYou can disable this behavior by adding ``rebuild-index: False`` inside the ``ci`` section of your Spack environment.\n\nReindex jobs do not allow modifying the ``script`` attribute since it is automatically generated using the target mirror listed in the ``mirrors::mirror`` configuration.\n\nSigning (signing)\n^^^^^^^^^^^^^^^^^\n\nThis job is run after all of the rebuild jobs are completed and is intended to be used to sign the package binaries built by a protected CI run.\nSigning jobs are generated only if a signing job ``script`` is specified and the Spack CI job type is protected.\nNote, if an ``any-job`` section contains a script, this will not implicitly create a ``signing`` job; a signing job may only exist if it is explicitly specified in the configuration with a ``script`` attribute.\nSpecifying a signing job without a script does not create a signing job, and the job configuration attributes will be ignored.\nSigning jobs are always assigned the runner tags ``aws``, ``protected``, and ``notary``.\n\n.. _noop_jobs:\n\nNo Op (noop)\n^^^^^^^^^^^^\n\nIf no specs in an environment need to be rebuilt during a given pipeline run (meaning all are already up to date on the mirror), a single successful job (a NO-OP) is still generated to avoid an empty pipeline (which GitLab considers to be an error).\nThe ``noop-job*`` sections can be added to your ``spack.yaml`` where you can provide ``tags`` and ``image`` or ``variables`` for the generated NO-OP job.\nThis section also supports providing ``before_script``, ``script``, and ``after_script``, in case you want to take some custom actions in the case of an empty pipeline.\n\nFollowing is an example of this section added to a ``spack.yaml``:\n\n.. code-block:: yaml\n\n spack:\n ci:\n pipeline-gen:\n - noop-job:\n tags: [\"custom\", \"tag\"]\n image:\n name: \"some.image.registry/custom-image:latest\"\n entrypoint: [\"/bin/bash\"]\n script::\n - echo \"Custom message in a custom script\"\n\nThe example above illustrates how you can provide the attributes used to run the NO-OP job in the case of an empty pipeline.\nThe only field for the NO-OP job that might be generated for you is ``script``, but that will only happen if you do not provide one yourself.\nNotice in this example the ``script`` uses the ``::`` notation to prescribe override behavior.\nWithout this, the ``echo`` command would have been prepended to the automatically generated script rather than replacing it.\n\nci.yaml\n------------------------------------\n\nHere's an example of a Spack configuration file describing a build pipeline:\n\n.. code-block:: yaml\n\n spack:\n ci:\n target: gitlab\n rebuild_index: true\n broken-specs-url: https://broken.specs.url\n broken-tests-packages:\n - gptune\n pipeline-gen:\n - submapping:\n - match:\n - os=ubuntu24.04\n build-job:\n tags:\n - spack-kube\n image: spack/ubuntu-noble\n - match:\n - os=almalinux9\n build-job:\n tags:\n - spack-kube\n image: spack/almalinux9\n\n cdash:\n build-group: Release Testing\n url: https://cdash.spack.io\n project: Spack\n site: Spack AWS Gitlab Instance\n\nThe ``ci`` config section is used to configure how the pipeline workload should be generated, mainly how the jobs for building specs should be assigned to the configured runners on your instance.\nThe main section for configuring pipelines is ``pipeline-gen``, which is a list of job attribute sections that are merged, using the same rules as Spack configs (:ref:`config-scope-precedence`), from the bottom up.\nThe order sections are applied is to be consistent with how Spack orders scope precedence when merging lists.\nThere are two main section types: ``-job`` sections and ``submapping`` sections.\n\n\nJob Attribute Sections\n^^^^^^^^^^^^^^^^^^^^^^\n\nEach type of job may have attributes added or removed via sections in the ``pipeline-gen`` list.\nJob type specific attributes may be specified using the keys ``-job`` to add attributes to all jobs of type ```` or ``-job-remove`` to remove attributes of type ````.\nEach section may only contain one type of job attribute specification, i.e., ``build-job`` and ``noop-job`` may not coexist but ``build-job`` and ``build-job-remove`` may.\n\n.. note::\n The ``*-remove`` specifications are applied before the additive attribute specification.\n For example, in the case where both ``build-job`` and ``build-job-remove`` are listed in the same ``pipeline-gen`` section, the value will still exist in the merged build-job after applying the section.\n\nAll of the attributes specified are forwarded to the generated CI jobs, however special treatment is applied to the attributes ``tags``, ``image``, ``variables``, ``script``, ``before_script``, and ``after_script`` as they are components recognized explicitly by the Spack CI generator.\nFor the ``tags`` attribute, Spack will remove reserved tags (:ref:`reserved_tags`) from all jobs specified in the config.\nIn some cases, such as for ``signing`` jobs, reserved tags will be added back based on the type of CI that is being run.\n\nOnce a runner has been chosen to build a release spec, the ``build-job*`` sections provide information determining details of the job in the context of the runner.\nAt least one of the ``build-job*`` sections must contain a ``tags`` key, which is a list containing at least one tag used to select the runner from among the runners known to the GitLab instance.\nFor Docker executor type runners, the ``image`` key is used to specify the Docker image used to build the release spec (and could also appear as a dictionary with a ``name`` specifying the image name, as well as an ``entrypoint`` to override whatever the default for that image is).\nFor other types of runners the ``variables`` key will be useful to pass any information on to the runner that it needs to do its work (e.g. scheduler parameters, etc.).\nAny ``variables`` provided here will be added, verbatim, to each job.\n\nThe ``build-job`` section also allows users to supply custom ``script``, ``before_script``, and ``after_script`` sections to be applied to every job scheduled on that runner.\nThis allows users to do any custom preparation or cleanup tasks that fit their particular workflow, as well as completely customize the rebuilding of a spec if they so choose.\nSpack will not generate a ``before_script`` or ``after_script`` for jobs, but if you do not provide a custom ``script``, Spack will generate one for you that assumes the concrete environment directory is located within your ``--artifacts-root`` (or if not provided, within your ``$CI_PROJECT_DIR``), activates that environment for you, and invokes ``spack ci rebuild``.\n\nSections that specify scripts (``script``, ``before_script``, ``after_script``) are all read as lists of commands or lists of lists of commands.\nIt is recommended to write scripts as lists of lists if scripts will be composed via merging.\nThe default behavior of merging lists will remove duplicate commands and potentially apply unwanted reordering, whereas merging lists of lists will preserve the local ordering and never removes duplicate commands.\nWhen writing commands to the CI target script, all lists are expanded and flattened into a single list.\n\nSubmapping Sections\n^^^^^^^^^^^^^^^^^^^\n\nA special case of attribute specification is the ``submapping`` section which may be used to apply job attributes to build jobs based on the package spec associated with the rebuild job.\nSubmapping is specified as a list of spec ``match`` lists associated with ``build-job``/``build-job-remove`` sections.\nThere are two options for ``match_behavior``: either ``first`` or ``merge`` may be specified.\nIn either case, the ``submapping`` list is processed from the bottom up, and then each ``match`` list is searched for a string that satisfies the check ``spec.satisfies({match_item})`` for each concrete spec.\n\nIn the case of ``match_behavior: first``, the first ``match`` section in the list of ``submappings`` that contains a string that satisfies the spec will apply its ``build-job*`` attributes to the rebuild job associated with that spec.\nThis is the default behavior and will be the method if no ``match_behavior`` is specified.\n\nIn the case of ``merge`` match, all of the ``match`` sections in the list of ``submappings`` that contain a string that satisfies the spec will have the associated ``build-job*`` attributes applied to the rebuild job associated with that spec.\nAgain, the attributes will be merged starting from the bottom match going up to the top match.\n\nIn the case that no match is found in a submapping section, no additional attributes will be applied.\n\n\nDynamic Mapping Sections\n^^^^^^^^^^^^^^^^^^^^^^^^\n\nFor large scale CI where cost optimization is required, dynamic mapping allows for the use of real-time mapping schemes served by a web service.\nThis type of mapping does not support the ``-remove`` type behavior, but it does follow the rest of the merge rules for configurations.\n\nThe dynamic mapping service needs to implement a single REST API interface for getting requests ``GET [:PORT][/PATH]?spec=``.\n\nexample request.\n\n.. code-block:: text\n\n https://my-dyn-mapping.spack.io/allocation?spec=zlib-ng@2.1.6 +compat+opt+shared+pic+new_strategies arch=linux-ubuntu20.04-x86_64_v3%gcc@12.0.0\n\n\nWith an example response that updates kubernetes request variables, overrides the max retries for GitLab, and prepends a note about the modifications made by the my-dyn-mapping.spack.io service.\n\n.. code-block:: text\n\n 200 OK\n\n {\n \"variables\":\n {\n \"KUBERNETES_CPU_REQUEST\": \"500m\",\n \"KUBERNETES_MEMORY_REQUEST\": \"2G\",\n },\n \"retry\": { \"max:\": \"1\"}\n \"script+:\":\n [\n \"echo \\\"Job modified by my-dyn-mapping.spack.io\\\"\"\n ]\n }\n\n\nThe ci.yaml configuration section takes the URL endpoint as well as a number of options to configure how responses are handled.\n\nIt is possible to specify a list of allowed and ignored configuration attributes under ``allow`` and ``ignore`` respectively.\nIt is also possible to configure required attributes under ``required`` section.\n\nOptions to configure the client timeout and SSL verification using the ``timeout`` and ``verify_ssl`` options.\nBy default, the ``timeout`` is set to the option in ``config:timeout`` and ``verify_ssl`` is set to the option in ``config:verify_ssl``.\n\nPassing header parameters to the request can be achieved through the ``header`` section.\nThe values of the variables passed to the header may be environment variables that are expanded at runtime, such as a private token configured on the runner.\n\nHere is an example configuration pointing to ``my-dyn-mapping.spack.io/allocation``.\n\n\n.. code-block:: yaml\n\n ci:\n pipeline-gen:\n - dynamic-mapping:\n endpoint: my-dyn-mapping.spack.io/allocation\n timeout: 10\n verify_ssl: true\n header:\n PRIVATE_TOKEN: ${MY_PRIVATE_TOKEN}\n MY_CONFIG: \"fuzz_allocation:false\"\n allow:\n - variables\n ignore:\n - script\n require: []\n\n\nBroken Specs URL\n^^^^^^^^^^^^^^^^\n\nThe optional ``broken-specs-url`` key tells Spack to check against a list of specs that are known to be currently broken in ``develop``.\nIf any such specs are found, the ``spack ci generate`` command will fail with an error message informing the user what broken specs were encountered.\nThis allows the pipeline to fail early and avoid wasting compute resources attempting to build packages that will not succeed.\n\nCDash\n^^^^^^\n\nThe optional ``cdash`` section provides information that will be used by the ``spack ci generate`` command (invoked by ``spack ci start``) for reporting to CDash.\nAll the jobs generated from this environment will belong to a \"build group\" within CDash that can be tracked over time.\nAs the release progresses, this build group may have jobs added or removed.\nThe URL, project, and site are used to specify the CDash instance to which build results should be reported.\n\nTake a look at the `schema `_ for the ``ci`` section of the Spack environment file, to see precisely what syntax is allowed there.\n\n.. _reserved_tags:\n\nReserved Tags\n^^^^^^^^^^^^^\n\nSpack has a subset of tags (``public``, ``protected``, and ``notary``) that it reserves for classifying runners that may require special permissions or access.\nThe tags ``public`` and ``protected`` are used to distinguish between runners that use public permissions and runners with protected permissions.\nThe ``notary`` tag is a special tag that is used to indicate runners that have access to the highly protected information used for signing binaries using the ``signing`` job.\n\n.. _staging_algorithm:\n\nSummary of ``.gitlab-ci.yml`` generation algorithm\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nAll specs yielded by the matrix (or all the specs in the environment) have their dependencies computed, and the entire resulting set of specs are staged together before being run through the ``ci/pipeline-gen`` entries, where each staged spec is assigned a runner.\n\"Staging\" is the name given to the process of figuring out in what order the specs should be built, taking into consideration Gitlab CI rules about jobs/stages.\nIn the staging process, the goal is to maximize the number of jobs in any stage of the pipeline, while ensuring that the jobs in any stage only depend on jobs in previous stages (since those jobs are guaranteed to have completed already).\nAs a runner is determined for a job, the information in the merged ``any-job*`` and ``build-job*`` sections is used to populate various parts of the job description that will be used by the target CI pipelines.\nOnce all the jobs have been assigned a runner, the ``.gitlab-ci.yml`` is written to disk.\n\nThe short example provided above would result in the ``readline``, ``ncurses``, and ``pkgconf`` packages getting staged and built on the runner chosen by the ``spack-k8s`` tag.\nIn this example, Spack assumes the runner is a Docker executor type runner, and thus certain jobs will be run in the ``centos7`` container and others in the ``ubuntu-18.04`` container.\nThe resulting ``.gitlab-ci.yml`` will contain 6 jobs in three stages.\nOnce the jobs have been generated, the presence of a ``SPACK_CDASH_AUTH_TOKEN`` environment variable during the ``spack ci generate`` command would result in all of the jobs being put in a build group on CDash called \"Release Testing\" (that group will be created if it didn't already exist).\n\n.. _ci_artifacts:\n\nCI Artifacts Directory Layout\n-----------------------------\n\nWhen running the CI build using the command ``spack ci rebuild`` a number of directories are created for storing data generated during the CI job.\nThe default root directory for artifacts is ``job_scratch_root``.\nThis can be overridden by passing the argument ``--artifacts-root`` to the ``spack ci generate`` command or by setting the ``SPACK_ARTIFACTS_ROOT`` environment variable in the build job scripts.\n\nThe top-level directories under the artifact root are ``concrete_environment``, ``logs``, ``reproduction``, ``tests``, and ``user_data``.\nSpack does not restrict what is written to any of these directories nor does it require user specified files be written to any specific directory.\n\n``concrete_environment``\n^^^^^^^^^^^^^^^^^^^^^^^^\n\nThe directory ``concrete_environment`` is used to communicate the ``spack ci generate`` processed ``spack.yaml`` and the concrete ``spack.lock`` for the CI environment.\n\n``logs``\n^^^^^^^^\n\nThe directory ``logs`` contains the Spack build log, ``spack-build-out.txt``, and the Spack build environment modification file, ``spack-build-mod-env.txt``.\nAdditionally, all files specified by the packages ``Builder`` property ``archive_files`` are also copied here (i.e., ``CMakeCache.txt`` in ``CMakeBuilder``).\n\n``reproduction``\n^^^^^^^^^^^^^^^^\n\nThe directory ``reproduction`` is used to store the files needed by the ``spack ci reproduce-build`` command.\nThis includes ``repro.json``, copies of all of the files in ``concrete_environment``, the concrete spec JSON file for the current spec being built, and all of the files written in the artifacts root directory.\n\nThe ``repro.json`` file is not versioned and is only designed to work with the version that Spack CI was run with.\nAn example of what a ``repro.json`` may look like is here.\n\n.. code-block:: json\n\n {\n \"job_name\": \"adios2@2.9.2 /feaevuj %gcc@11.4.0 arch=linux-ubuntu20.04-x86_64_v3 E4S ROCm External\",\n \"job_spec_json\": \"adios2.json\",\n \"ci_project_dir\": \"/builds/spack/spack\"\n }\n\n``tests``\n^^^^^^^^^\n\nThe directory ``tests`` is used to store output from running ``spack test ``.\nThis may or may not have data in it depending on the package that was built and the availability of tests.\n\n``user_data``\n^^^^^^^^^^^^^\n\nThe directory ``user_data`` is used to store everything else that shouldn't be copied to the ``reproduction`` directory.\nUsers may use this to store additional logs or metrics or other types of files generated by the build job.\n\nUsing a custom Spack in your pipeline\n-------------------------------------\n\nIf your runners will not have a version of Spack ready to invoke, or if for some other reason you want to use a custom version of Spack to run your pipelines, this section provides an example of how you could take advantage of user-provided pipeline scripts to accomplish this fairly simply.\nFirst, consider specifying the source and version of Spack you want to use with variables, either written directly into your ``.gitlab-ci.yml``, or provided by CI variables defined in the GitLab UI or from some upstream pipeline.\nLet's say you choose the variable names ``SPACK_REPO`` and ``SPACK_REF`` to refer to the particular fork of Spack and branch you want for running your pipeline.\nYou can then refer to those in a custom shell script invoked both from your pipeline generation job and your rebuild jobs.\nHere's the ``generate-pipeline`` job from the top of this document, updated to clone and source a custom Spack:\n\n.. code-block:: yaml\n\n generate-pipeline:\n tags:\n - \n before_script:\n - git clone ${SPACK_REPO}\n - pushd spack && git checkout ${SPACK_REF} && popd\n - . \"./spack/share/spack/setup-env.sh\"\n script:\n - spack env activate --without-view .\n - spack ci generate --check-index-only --artifacts-root \"${CI_PROJECT_DIR}/jobs_scratch_dir\" --output-file \"${CI_PROJECT_DIR}/jobs_scratch_dir/pipeline.yml\"\n after_script:\n - rm -rf ./spack\n artifacts:\n paths:\n - \"${CI_PROJECT_DIR}/jobs_scratch_dir\"\n\nThat takes care of getting the desired version of Spack when your pipeline is generated by ``spack ci generate``.\nYou also want your generated rebuild jobs (all of them) to clone that version of Spack, so next you would update your ``spack.yaml`` from above as follows:\n\n.. code-block:: yaml\n\n spack:\n # ...\n ci:\n pipeline-gen:\n - build-job:\n tags:\n - spack-kube\n image: spack/ubuntu-noble\n before_script:\n - git clone ${SPACK_REPO}\n - pushd spack && git checkout ${SPACK_REF} && popd\n - . \"./spack/share/spack/setup-env.sh\"\n script:\n - spack env activate --without-view ${SPACK_CONCRETE_ENV_DIR}\n - spack -d ci rebuild\n after_script:\n - rm -rf ./spack\n\nNow all of the generated rebuild jobs will use the same shell script to clone Spack before running their actual workload.\n\nNow imagine you have long pipelines with many specs to be built, and you are pointing to a Spack repository and branch that has a tendency to change frequently, such as the main repo and its ``develop`` branch.\nIf each child job checks out the ``develop`` branch, that could result in some jobs running with one SHA of Spack, while later jobs run with another.\nTo help avoid this issue, the pipeline generation process saves global variables called ``SPACK_VERSION`` and ``SPACK_CHECKOUT_VERSION`` that capture the version of Spack used to generate the pipeline.\nWhile the ``SPACK_VERSION`` variable simply contains the human-readable value produced by ``spack -V`` at pipeline generation time, the ``SPACK_CHECKOUT_VERSION`` variable can be used in a ``git checkout`` command to make sure all child jobs checkout the same version of Spack used to generate the pipeline.\nTo take advantage of this, you could simply replace ``git checkout ${SPACK_REF}`` in the example ``spack.yaml`` above with ``git checkout ${SPACK_CHECKOUT_VERSION}``.\n\nOn the other hand, if you're pointing to a Spack repository and branch under your control, there may be no benefit in using the captured ``SPACK_CHECKOUT_VERSION``, and you can instead just clone using the variables you define (``SPACK_REPO`` and ``SPACK_REF`` in the example above).\n\n.. _custom_workflow:\n\nCustom Workflow\n---------------\n\nThere are many ways to take advantage of Spack CI pipelines to achieve custom workflows for building packages or other resources.\nOne example of a custom pipelines workflow is the Spack tutorial container `repo `_.\nThis project uses GitHub (for source control), GitLab (for automated Spack CI pipelines), and DockerHub automated builds to build Docker images (complete with fully populated binary mirror) used by instructors and participants of a Spack tutorial.\n\nTake a look at the repo to see how it is accomplished using Spack CI pipelines, and see the following markdown files at the root of the repository for descriptions and documentation describing the workflow: ``DESCRIPTION.md``, ``DOCKERHUB_SETUP.md``, ``GITLAB_SETUP.md``, and ``UPDATING.md``.\n\n.. _ci_environment_variables:\n\nEnvironment variables affecting pipeline operation\n--------------------------------------------------\n\nCertain secrets and some other information should be provided to the pipeline infrastructure via environment variables, usually for reasons of security, but in some cases to support other pipeline use cases such as PR testing.\nThe environment variables used by the pipeline infrastructure are described here.\n\n``AWS_ACCESS_KEY_ID``\n^^^^^^^^^^^^^^^^^^^^^\n\nOptional.\nOnly needed when binary mirror is an S3 bucket.\n\n``AWS_SECRET_ACCESS_KEY``\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nOptional.\nOnly needed when binary mirror is an S3 bucket.\n\n``S3_ENDPOINT_URL``\n^^^^^^^^^^^^^^^^^^^\n\nOptional.\nOnly needed when binary mirror is an S3 bucket that is *not* on AWS.\n\n``CDASH_AUTH_TOKEN``\n^^^^^^^^^^^^^^^^^^^^\n\nOptional.\nOnly needed to report build groups to CDash.\n\n``SPACK_SIGNING_KEY``\n^^^^^^^^^^^^^^^^^^^^^\n\nOptional.\nOnly needed if you want ``spack ci rebuild`` to trust the key you store in this variable, in which case, it will subsequently be used to sign and verify binary packages (when installing or creating build caches).\nYou could also have already trusted a key Spack knows about, or if no key is present anywhere, Spack will install specs using ``--no-check-signature`` and create build caches using ``-u`` (for unsigned binaries).\n\n" }, { "path": "lib/spack/docs/repositories.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Learn how to set up and manage package repositories in Spack, enabling you to maintain custom packages and override built-in ones.\n\n.. _repositories:\n\nPackage Repositories (repos.yaml)\n=================================\n\nSpack comes with thousands of built-in package recipes.\nAs of Spack v1.0, these are hosted in a separate Git repository at `spack/spack-packages `_.\n\nA **package repository** is a directory that Spack searches when it needs to find a package by name.\nYou may need to maintain packages for restricted, proprietary, or experimental software separately from the built-in repository.\nSpack allows you to configure local and remote repositories using either the ``repos.yaml`` configuration file or the ``spack repo`` command.\n\nThis document describes how to set up and manage these package repositories.\n\nStructure of an Individual Package Repository\n---------------------------------------------\n\nAn individual Spack package repository is a directory structured as follows:\n\n.. code-block:: text\n\n /path/to/repos/ # the top-level dir is added to the Python search path\n spack_repo/ # every package repository is part of the spack_repo Python module\n myrepo/ # directory for the 'myrepo' repository (matches namespace)\n repo.yaml # configuration file for this package repository\n packages/ # directory containing package directories\n hdf5/ # directory for the hdf5 package\n package.py # the package recipe file\n mpich/ # directory for the mpich package\n package.py # the package recipe file\n mpich-1.9-bugfix.patch # example patch file\n trilinos/\n package.py\n ...\n\n* ``repo.yaml``.\n This file contains metadata for this specific repository, for example:\n\n .. code-block:: yaml\n\n repo:\n namespace: myrepo\n api: v2.0\n\n It defines primarily:\n\n * ``namespace``.\n A unique identifier for this repository (e.g., ``myrepo``, ``projectx``).\n See the :ref:`Namespaces ` section for more details.\n * ``api``.\n The version of the Spack Package API this repository adheres to (e.g., ``v2.0``).\n Spack itself defines what range of API versions it supports, and will error if it encounters a repository with an unsupported API version.\n\n* ``packages/``.\n This directory contains subdirectories for each package in the repository.\n Each package directory contains a ``package.py`` file and any patches or other files needed to build the package.\n\nPackage repositories allow you to:\n\n1. Maintain your own packages separately from Spack's built-in set.\n2. Share your packages (e.g., by hosting them on a shared file system or in a Git repository) without committing them to the main ``spack/spack-packages`` repository.\n3. Override built-in Spack packages with your own implementations.\n\nPackages in a separate repository can also *depend on* built-in Spack packages, allowing you to leverage existing recipes without re-implementing them.\n\nPackage Names\n^^^^^^^^^^^^^\n\nPackage names are defined by the directory names under ``packages/``.\nIn the example above, the package names are ``hdf5``, ``mpich``, and ``trilinos``.\nPackage names can only contain lowercase characters ``a-z``, digits ``0-9`` and hyphens ``-``.\n\n.. note::\n\n Package names are **derived** from the directory names under ``packages/``.\n Package directories are required to be valid Python module names, which means they cannot contain hyphens or start with a digit.\n This means that a package named ``my-package`` would be stored in a directory named ``my_package/``, and a package named ``7zip`` would be stored in a directory named ``_7zip/`` with an underscore prefix to make it a valid Python module name.\n The mapping between package names and directory names is one-to-one.\n Use ``spack list`` to see how Spack resolves the package names from the directory names.\n\nConfiguring Repositories with ``repos.yaml``\n--------------------------------------------\n\nSpack uses ``repos.yaml`` files found in its :ref:`configuration scopes ` (e.g., ``~/.spack/``, ``etc/spack/``) to discover and prioritize package repositories.\nNote that this ``repos.yaml`` (plural) configuration file is distinct from the ``repo.yaml`` (singular) file within each individual package repository.\n\nSpack supports two main types of repository configurations:\n\nLocal Repositories (Path-based)\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nYou can point Spack to a repository on your local filesystem:\n\n.. code-block:: yaml\n :caption: ``~/.spack/repos.yaml``\n :name: code-example-local-repo\n\n repos:\n my_local_packages: /path/to/my_repository_root\n\nHere, ``/path/to/my_repository_root`` should be the directory containing that repository's ``repo.yaml`` and ``packages/`` subdirectory.\n\nGit-based Repositories\n^^^^^^^^^^^^^^^^^^^^^^\n\nSpack can clone and use repositories directly from Git URLs:\n\n.. code-block:: yaml\n\n repos:\n my_remote_repo: https://github.com/myorg/spack-custom-pkgs.git\n\nAutomatic Cloning\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nWhen Spack first encounters a Git-based repository configuration, it automatically clones it.\nBy default, these repositories are cloned into a subdirectory within ``~/.spack/package_repos/``, named with a hash of the repository URL.\n\nTo change directories to the package repository, you can use ``spack cd --repo [name]``.\nTo find where a repository is cloned, you can use ``spack location --repo [name]`` or ``spack repo list``.\nThe ``name`` argument is optional; if omitted, Spack will use the first package repository in configuration order.\n\nCustomizing Clone Location\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nThe default clone location (``~/.spack/package_repos/``) might not be convenient for package maintainers who want to make changes to packages.\nYou can specify a custom local directory for Spack to clone a Git repository into, or to use if the repository is already cloned there.\nThis is done using the ``destination`` key in ``repos.yaml`` or via the ``spack repo set --destination`` command (see :ref:`cmd-spack-repo-set-destination`).\n\nFor example, to use ``~/custom_packages_clone`` for ``my_remote_repo``:\n\n.. code-block:: yaml\n :caption: ``~/.spack/repos.yaml``\n :name: code-example-location\n\n repos:\n my_remote_repo:\n git: https://github.com/myorg/spack-custom-pkgs.git\n destination: ~/custom_packages_clone\n\nIf the ``git`` URL is defined in a lower-precedence configuration (like Spack's defaults for ``builtin``), you only need to specify the ``destination`` in your user-level ``repos.yaml``.\nSpack can make the configuration changes for you using ``spack repo set --destination ~/spack-packages builtin``, or you can directly edit your ``repos.yaml`` file:\n\n.. code-block:: yaml\n :caption: ``~/.spack/repos.yaml``\n :name: code-example-builtin\n\n repos:\n builtin:\n destination: ~/spack-packages\n\nUpdating and pinning\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nRepos can be pinned to a git branch, tag, or commit.\n\n.. code-block:: yaml\n :caption: ``~/.spack/repos.yaml``\n :name: code-example-branch\n\n repos:\n builtin:\n branch: releases/v2025.07\n # tag: v2025.07.0\n # commit: 6427933daecef74b981d1f773731aeace3b06ede\n\nThe ``spack repo update`` command will update the repo on disk to match the current state of the config.\nIf the repo is pinned to a commit or tag, it will ensure the repo on disk reflects that commit or tag.\nIf the repo is pinned to a branch or unpinned, ``spack repo update`` will pull the most recent state of the branch (the default branch if unpinned).\n\nGit repositories need a package repo index\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nA single Git repository can contain one or more Spack package repositories.\nTo enable Spack to discover these, the root of the Git repository should contain a ``spack-repo-index.yaml`` file.\nThis file lists the relative paths to package repository roots within the git repo.\n\nFor example, assume a Git repository at ``https://example.com/my_org/my_pkgs.git`` has the following structure\n\n.. code-block:: text\n\n my_pkgs.git/\n spack-repo-index.yaml # metadata file at the root of the Git repo\n ...\n spack_pkgs/\n spack_repo/\n my_org/\n comp_sci_packages/ # package repository for computer science packages\n repo.yaml\n packages/\n hdf5/\n package.py\n mpich/\n package.py\n physics_packages/ # package repository for physics packages\n repo.yaml\n packages/\n gromacs/\n package.py\n\nThe ``spack-repo-index.yaml`` in the root of ``https://example.com/my_org/my_pkgs.git`` should look like this:\n\n.. code-block:: yaml\n :caption: ``my_pkgs.git/spack-repo-index.yaml``\n :name: code-example-repo-index\n\n repo_index:\n paths:\n - spack_pkgs/spack_repo/my_org/comp_sci_packages\n - spack_pkgs/spack_repo/my_org/physics_packages\n\nIf ``my_pkgs.git`` is configured in ``repos.yaml`` as follows:\n\n.. code-block:: yaml\n :caption: ``~/.spack/repos.yaml``\n :name: code-example-git-repo\n\n repos:\n example_mono_repo: https://example.com/my_org/my_pkgs.git\n\nSpack will clone ``my_pkgs.git`` and look for ``spack-repo-index.yaml``.\nIt will then register two separate repositories based on the paths found (e.g., ``/spack_pkgs/spack_repo/my_org/comp_sci_packages`` and ``/spack_pkgs/spack_repo/my_org/physics_packages``), each with its own namespace defined in its respective ``repo.yaml`` file.\nThus, one ``repos.yaml`` entry for a Git mono-repo can lead to *multiple repositories* being available to Spack.\n\nIf you want only one of the package repositories from a Git mono-repo, you can override the paths in your user-level ``repos.yaml``.\nFor example, if you only want the computer science packages:\n\n.. code-block:: yaml\n :caption: ``~/.spack/repos.yaml``\n :name: code-example-specific-repo\n\n repos:\n example_mono_repo:\n git: https://example.com/my_org/my_pkgs.git\n paths:\n - spack_pkgs/spack_repo/my_org/comp_sci_packages\n\nThe ``spack repo add`` command can help you set up these configurations easily.\n\nThe ``builtin`` Repository\n^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nSpack's extensive collection of built-in packages resides at `spack/spack-packages `_.\nBy default, Spack is configured to use this as a Git-based repository.\nThe default configuration in ``$spack/etc/spack/defaults/repos.yaml`` looks something like this:\n\n.. code-block:: yaml\n\n repos:\n builtin:\n git: https://github.com/spack/spack-packages.git\n\n.. _namespaces:\n\nNamespaces\n----------\n\nEvery repository in Spack has an associated **namespace** defined in the ``namespace:`` key of its top-level ``repo.yaml`` file.\nFor example, the built-in repository (from ``spack/spack-packages``) has its namespace defined as ``builtin``:\n\n.. code-block:: yaml\n :caption: ``repo.yaml`` of ``spack/spack-packages``\n :name: code-example-repo-yaml\n\n repo:\n namespace: builtin\n api: v2.0 # Or newer\n\nSpack records the repository namespace of each installed package.\nFor example, if you install the ``mpich`` package from the ``builtin`` repo, Spack records its fully qualified name as ``builtin.mpich``.\nThis accomplishes two things:\n\n1. You can have packages with the same name from different namespaces installed simultaneously.\n2. You can easily determine which repository a package came from after it is installed (more :ref:`below `).\n\n.. note::\n\n The ``namespace`` defined in the package repository's ``repo.yaml`` is the **authoritative source** for the namespace.\n It is *not* derived from the local configuration in ``repos.yaml``.\n This means that the namespace is determined by the repository maintainer, not by the user or local configuration.\n\nNested Namespaces for Organizations\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nAs we have already seen in the Git-based package repositories example above, you can create nested namespaces by using periods in the namespace name.\nFor example, a repository for packages related to computation at LLNL might have the namespace ``llnl.comp``, while one for physical and life sciences could be ``llnl.pls``.\nOn the file system, this requires a directory structure like this:\n\n.. code-block:: text\n\n /path/to/repos/\n spack_repo/\n llnl/\n comp/\n repo.yaml # Contains namespace: llnl.comp\n packages/\n mpich/\n package.py\n pls/\n repo.yaml # Contains namespace: llnl.pls\n packages/\n hdf5/\n package.py\n\nUniqueness\n^^^^^^^^^^\n\nSpack cannot ensure global uniqueness of all namespaces, but it will prevent you from registering two repositories with the same namespace *at the same time* in your current configuration.\nIf you try to add a repository that has the same namespace as an already registered one, Spack will print a warning and may ignore the new addition or apply specific override logic depending on the configuration.\n\n.. _namespace-example:\n\nNamespace Example\n^^^^^^^^^^^^^^^^^\n\nSuppose LLNL maintains its own version of ``mpich`` (in a repository with namespace ``llnl.comp``), separate from Spack's built-in ``mpich`` package (namespace ``builtin``).\nIf you've installed both, ``spack find`` alone might be ambiguous:\n\n.. code-block:: console\n\n $ spack find\n ==> 2 installed packages.\n -- linux-rhel6-x86_64 / gcc@4.4.7 -------------\n mpich@3.2 mpich@3.2\n\nUsing ``spack find -N`` displays packages with their namespaces:\n\n.. code-block:: console\n\n $ spack find -N\n ==> 2 installed packages.\n -- linux-rhel6-x86_64 / gcc@4.4.7 -------------\n builtin.mpich@3.2 llnl.comp.mpich@3.2\n\nNow you can distinguish them.\nPackages differing only by namespace will have different hashes:\n\n.. code-block:: console\n\n $ spack find -lN\n ==> 2 installed packages.\n -- linux-rhel6-x86_64 / gcc@4.4.7 -------------\n c35p3gc builtin.mpich@3.2 itoqmox llnl.comp.mpich@3.2\n\nAll Spack commands that take a package :ref:`spec ` also accept a fully qualified spec with a namespace, allowing you to be specific:\n\n.. code-block:: spec\n\n $ spack uninstall llnl.comp.mpich\n\nSearch Order and Overriding Packages\n-------------------------------------\n\nWhen Spack resolves an unqualified package name (e.g., ``mpich`` in ``spack install mpich``), it searches the configured repositories in the order they appear in the *merged* ``repos.yaml`` configuration (from highest to lowest precedence scope, and top to bottom within each file).\nThe first repository found that provides the package will be used.\nFor Git-based mono-repos, the individual repositories listed in its ``spack-repo-index.yaml`` are effectively inserted into this search order based on the mono-repo's position.\n\nThis search order allows you to override built-in packages.\nIf you have your own ``mpich`` in a repository ``my_custom_repo``, and ``my_custom_repo`` is listed before ``builtin`` in your ``repos.yaml``, Spack will use your version of ``mpich`` by default.\n\nSuppose your effective (merged) ``repos.yaml`` implies the following order:\n\n1. ``proto`` (local repo at ``~/my_spack_repos/spack_repo/proto_repo``)\n2. ``llnl`` (local repo at ``/usr/local/repos/spack_repo/llnl_repo``)\n3. ``builtin`` (Spack's default packages from ``spack/spack-packages``)\n\nAnd the packages are:\n\n+--------------+------------------------------------------------+-----------------------------+\n| Namespace | Source | Packages |\n+==============+================================================+=============================+\n| ``proto`` | ``~/my_spack_repos/spack_repo/proto_repo`` | ``mpich`` |\n+--------------+------------------------------------------------+-----------------------------+\n| ``llnl`` | ``/usr/local/repos/spack_repo/llnl_repo`` | ``hdf5`` |\n+--------------+------------------------------------------------+-----------------------------+\n| ``builtin`` | `spack/spack-packages` (Git) | ``mpich``, ``hdf5``, others |\n+--------------+------------------------------------------------+-----------------------------+\n\nIf ``hdf5`` depends on ``mpich``:\n\n* ``spack install hdf5`` will install ``llnl.hdf5 ^proto.mpich``.\n Spack finds ``hdf5`` first in ``llnl``.\n For its dependency ``mpich``, Spack searches again from the top, finding ``mpich`` first in ``proto``.\n\nYou can force a particular repository's package using a fully qualified name:\n\n* ``spack install hdf5 ^builtin.mpich`` will install ``llnl.hdf5 ^builtin.mpich``.\n* ``spack install builtin.hdf5 ^builtin.mpich`` will install ``builtin.hdf5 ^builtin.mpich``.\n\nTo see which repositories will be used for a build *before* installing, use ``spack spec -N``:\n\n.. code-block:: spec\n\n $ spack spec -N hdf5\n llnl.hdf5@1.10.0\n ^proto.mpich@3.2\n ^builtin.zlib@1.2.8\n\n.. warning::\n\n While you *can* use a fully qualified package name in a ``depends_on`` directive within a ``package.py`` file (e.g., ``depends_on(\"proto.hdf5\")``), this is **strongly discouraged**.\n It makes the package non-portable and tightly coupled to a specific repository configuration, hindering sharing and composition of repositories.\n A package will fail to load if the hardcoded namespace's repository is not registered.\n\n.. _cmd-spack-repo:\n\nThe ``spack repo`` Command\n--------------------------\n\nSpack provides commands to manage your repository configurations.\n\n.. _cmd-spack-repo-list:\n\n``spack repo list``\n^^^^^^^^^^^^^^^^^^^^^^\n\nThis command shows all repositories Spack currently knows about, including their namespace, API version, and resolved path (local path or clone directory for Git repos).\n\n.. code-block:: console\n\n $ spack repo list\n [+] my_local v2.0 /path/to/spack_repo/my_local_packages\n [+] comp_sci_packages v2.0 ~/.spack/package_repos//spack_pkgs/spack_repo/comp_sci_packages\n [+] physics_packages v2.0 ~/.spack/package_repos//spack_pkgs/spack_repo/physics_packages # From the same git repo\n [+] builtin v2.0 ~/.spack/package_repos//repos/spack_repo/builtin\n\nSpack shows a green ``[+]`` next to each repository that is available for use.\nIt shows a red ``[-]`` to indicate that package repositories cannot be used due to an error (e.g., unsupported API version, missing ``repo.yaml``, etc.).\nIt can also show just a gray ``-`` if it is a Git-based package repository that has not been cloned yet.\n\nNote that for Git-based package repositories, ``spack repo list`` will show entries for *each* individual package repository registered via ``spack-repo-index.yaml``.\nThis contrasts with ``spack config get repos``, which shows the raw configuration from ``repos.yaml`` files, including just the Git URL for a mono-repo:\n\n.. code-block:: console\n\n $ spack config get repos\n repos:\n my_local_packages: /path/to/spack_repo/my_local_packages\n example_mono_repo: https://example.com/my_org/my_pkgs.git # contains two package repositories\n builtin:\n git: https://github.com/spack/spack-packages.git\n # destination: /my/custom/path (if set by user)\n\n.. _cmd-spack-repo-create:\n\n``spack repo create``\n^^^^^^^^^^^^^^^^^^^^^\n\nTo create the directory structure for a new, empty local repository:\n\n.. code-block:: console\n\n $ spack repo create ~/my_spack_projects myorg.projectx\n ==> Created repo with namespace 'myorg.projectx'.\n ==> To register it with spack, run this command:\n spack repo add ~/my_spack_projects/spack_repo/myorg/projectx\n\nThis command creates the following structure:\n\n.. code-block:: text\n\n ~/my_spack_projects/\n spack_repo/\n myorg/\n projectx/\n repo.yaml # Contains namespace: myorg.projectx\n packages/ # Empty directory for new package.py files\n\nThe ```` is where the ``spack_repo/`` hierarchy will be created.\nThe ```` can be simple (e.g., ``myrepo``) or nested (e.g., ``myorg.projectx``), and Spack will create the corresponding directory structure.\n\n.. _cmd-spack-repo-add:\n\n``spack repo add``\n^^^^^^^^^^^^^^^^^^\n\nTo register package repositories from local paths or a remote Git repositories with Spack:\n\n* **For a local path:** Provide the path to the repository's root directory (the one containing ``repo.yaml`` and ``packages/``).\n\n .. code-block:: console\n\n $ spack repo add ~/my_spack_projects/spack_repo/myorg/projectx\n ==> Added repo to config with name 'myorg.projectx'.\n\n* **For a Git repository:** Provide the Git URL.\n\n .. code-block:: console\n\n $ spack repo add --name my_pkgs https://github.com/spack/spack-packages.git ~/my_pkgs\n Cloning into '/home/user/my_pkgs'...\n ==> Added repo to config with name 'my_pkgs'.\n\n Notice that for Git-based package repositories, you need to specify a configuration name explicitly, which is the key used in your ``repos.yaml`` configuration file.\n The example also shows providing a custom destination path ``~/my_pkgs``.\n You can omit this if you want Spack to use the default clone location (e.g., ``~/.spack/package_repos/``).\n\nAfter adding, packages from this repository should appear in ``spack list`` and be installable.\n\n.. _cmd-spack-repo-remove:\n\n``spack repo remove``\n^^^^^^^^^^^^^^^^^^^^^\n\nTo unregister a repository, use its configuration name (the key in ``repos.yaml``) or its local path.\n\nBy configuration name (e.g., ``projectx`` from the add example):\n\n.. code-block:: console\n\n $ spack repo remove projectx\n ==> Removed repository 'projectx'.\n\nBy path (for a local repo):\n\n.. code-block:: console\n\n $ spack repo remove ~/my_spack_projects/spack_repo/myorg/projectx\n ==> Removed repository '/home/user/my_spack_projects/spack_repo/myorg/projectx'.\n\nThis command removes the corresponding entry from your ``repos.yaml`` configuration.\nIt does *not* delete the local repository files or any cloned Git repositories.\n\n.. _cmd-spack-repo-set-destination:\n\n``spack repo set``\n^^^^^^^^^^^^^^^^^^\n\nFor Git-based repositories, this command allows you to specify a custom local directory where Spack should clone the repository, or use an existing clone.\nThe ```` is the key used in your ``repos.yaml`` file for that Git repository (e.g., ``builtin``, ``my_remote_repo``).\n\n.. code-block:: console\n\n $ spack repo set --destination /my/custom/path/for/spack-packages builtin\n ==> Updated repo 'builtin'\n\nThis updates your user-level ``repos.yaml``, adding or modifying the ``destination:`` key for the specified repository configuration name.\n\n.. code-block:: yaml\n :caption: ``~/.spack/repos.yaml`` after ``spack repo set``\n :name: code-example-specific-destination\n\n repos:\n builtin:\n destination: /my/custom/path/for/spack-packages\n # The 'git:' URL is typically inherited from Spack's default configuration for 'builtin'\n\nSpack will then use ``/my/custom/path/for/spack-packages`` for the ``builtin`` repository.\nIf the directory doesn't exist, Spack will clone into it.\nIf it exists and is a valid Git repository, Spack will use it.\n\nRepository Namespaces and Python\n--------------------------------\n\nPackage repositories in Spack (from ``api: v2.0`` or newer) are structured to integrate smoothly with Python's import system.\nThey are effectively Python namespace packages under the top-level ``spack_repo`` namespace.\n\nThe ``api: v2.0`` repository structure ensures that packages can be imported using a standard Python module path: ``spack_repo..packages..package``.\nFor instance, the ``mpich`` package from the ``builtin`` repository corresponds to the Python module ``spack_repo.builtin.packages.mpich.package``.\n\nThis allows you to easily extend or subclass package classes from other repositories in your own ``package.py`` files:\n\n.. code-block:: python\n\n # In a package file (e.g. my_custom_mpich/package.py) in your custom repo\n # Import the original Mpich class from the 'builtin' repository\n from spack_repo.builtin.packages.mpich.package import Mpich as BuiltinMpich\n\n\n class MyCustomMpich(BuiltinMpich):\n # Override versions, variants, or methods from BuiltinMpich\n version(\"3.5-custom\", sha256=\"...\")\n\n # Add a new variant\n variant(\"custom_feature\", default=False, description=\"Enable my custom feature\")\n\n def install(self, spec, prefix):\n if \"+custom_feature\" in spec:\n # Do custom things\n pass\n super().install(spec, prefix) # Call parent install method\n\nSpack manages Python's ``sys.path`` at runtime to make these imports discoverable across all registered repositories.\nThis capability is powerful for creating derivative packages or slightly modifying existing ones without copying entire package files.\n" }, { "path": "lib/spack/docs/requirements.txt", "content": "sphinx==9.1.0\nsphinxcontrib-programoutput==0.19\nsphinxcontrib-svg2pdfconverter==2.1.0\nsphinx-copybutton==0.5.2\nsphinx-last-updated-by-git==0.3.8\nsphinx-sitemap==2.9.0\nfuro==2025.12.19\ndocutils==0.22.4\npygments==2.20.0\npytest==9.0.3\n" }, { "path": "lib/spack/docs/roles_and_responsibilities.rst", "content": ".. Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n A guide to distinguish the roles and responsibilities associated with managing the Spack Packages repository.\n\n.. _packaging-roles:\n\nPackaging Roles and Responsibilities\n====================================\n\nThere are four roles related to `Spack Package `_ repository Pull Requests (PRs):\n\n#. :ref:`package-contributors`,\n#. :ref:`package-reviewers`,\n#. :ref:`package-maintainers`, and\n#. :ref:`committers`.\n\nOne person can assume multiple roles (e.g., a Package Contributor may also be a Maintainer; a Package Reviewer may also be a Committer).\nThis section defines and describes the responsibilities of each role.\n\n.. _package-contributors:\n\nPackage Contributors\n--------------------\n\nContributors submit changes to packages through PRs `Spack Package `_ repository Pull Requests (PRs).\n\nAs a Contributor, you are **expected** to test your changes on **at least one platform** outside of Spack’s Continuous Integration (CI) checks.\n\n.. note::\n\n We also ask that you include the output from ``spack debug report`` from the platform you used to facilitate PR reviews.\n\n.. _package-reviewers:\n\nPackage Reviewers\n-----------------\n\nAnyone can review a PR so we encourage Spack’s community members to review and comment on those involving software in which they have expertise and/or interest.\n\nAs a Package Reviewer, you are **expected** to assess changes in PRs to the best of your ability and knowledge with special consideration to the information contained in the :ref:`package-review-guide`.\n\n.. _package-maintainers:\n\nMaintainers (Package Owners)\n----------------------------\n\nMaintainers are individuals (technically GitHub accounts) who appear in a package’s :ref:`maintainers` directive.\nThese are people who have agreed to be notified of and given the opportunity to review changes to packages.\nThey are, from a Spack package perspective, `Code Owners `_ of the package, whether or not they “own” or work on the software that the package builds.\n\nAs a Maintainer, you are **expected**, when available, to:\n\n* review PRs in a timely manner (reported in :ref:`committers`) to confirm that the changes made to the package are reasonable;\n* confirm that packages successfully build on at least one platform; and\n* attempt to confirm that any updated or included tests pass.\n\nSee :ref:`build_success_reviews` for acceptable forms of build success confirmation.\n\n.. note::\n\n If at least one maintainer approves a PR -– and there are no objections from others -– then the PR can be merged by any of the :ref:`committers`.\n\n.. _committers:\n\nCommitters\n----------\n\nCommitters are vetted individuals who are allowed to merge PRs into the ``develop`` branch.\n\nAs a Committer, you are **expected** to:\n\n* ensure **at least one review** is performed prior to merging (GitHub rules enforce this);\n* encourage **at least one** :ref:`Package Maintainer ` (if any) to comment and/or review the PR;\n* allow Package Maintainers (if any) **up to one week** to comment or provide a review;\n* determine if the criteria defined in :ref:`package-review-guide` are met; and\n* **merge the reviewed PR** at their discretion.\n\n.. note::\n\n If there are no :ref:`package-maintainers` or the Maintainers have not commented or reviewed the PR within the allotted time, you will need to conduct the review.\n\n.. tip::\n\n The following criteria must be met in order to become a Committer:\n\n * cannot be an anonymous account;\n * must come from a known and trustworthy organization;\n * demonstrated record of contribution to Spack;\n * have an account on the Spack Slack workspace;\n * be approved by the Onboarding subcommittee; and\n * (proposed) be known to at least 3 members of the core development team.\n" }, { "path": "lib/spack/docs/signing.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Understand the Spack package signing process, which ensures data integrity for official packages from automated CI pipelines through cryptographic signing.\n\n\n.. _signing:\n\nSpack Package Signing\n=====================\n\nThe goal of package signing in Spack is to provide data integrity assurances around official packages produced by the automated Spack CI pipelines.\nThese assurances directly address the security of Spack's software supply chain by explaining why a security-conscious user can be reasonably justified in the belief that packages installed via Spack have an uninterrupted auditable trail back to change management decisions judged to be appropriate by the Spack maintainers.\nThis is achieved through cryptographic signing of packages built by Spack CI pipelines based on code that has been transparently reviewed and approved on GitHub.\nThis document describes the signing process for interested users.\n\n.. _risks:\n\nRisks, Impact and Threat Model\n------------------------------\n\nThis document addresses the approach taken to safeguard Spack's reputation with regard to the integrity of the package data produced by Spack's CI pipelines.\nIt does not address issues of data confidentiality (Spack is intended to be largely open source) or availability (efforts are described elsewhere).\nWith that said, the main reputational risk can be broadly categorized as a loss of faith in the data integrity due to a breach of the private key used to sign packages.\nRemediation of a private key breach would require republishing the public key with a revocation certificate, generating a new signing key, an assessment and potential rebuild/resigning of all packages since the key was breached, and finally direct intervention by every spack user to update their copy of Spack's public keys used for local verification.\n\nThe primary threat model used in mitigating the risks of these stated impacts is one of individual error not malicious intent or insider threat.\nThe primary objective is to avoid the above impacts by making a private key breach nearly impossible due to oversight or configuration error.\nObvious and straightforward measures are taken to mitigate issues of malicious interference in data integrity and insider threats but these attack vectors are not systematically addressed.\nIt should be hard to exfiltrate the private key intentionally, and almost impossible to leak the key by accident.\n\n.. _overview:\n\nPipeline Overview\n-----------------\n\nSpack pipelines build software through progressive stages where packages in later stages nominally depend on packages built in earlier stages.\nFor both technical and design reasons these dependencies are not implemented through the default GitLab artifacts mechanism; instead built packages are uploaded to AWS S3 mirrors (buckets) where they are retrieved by subsequent stages in the pipeline.\nTwo broad categories of pipelines exist: Pull Request (PR) pipelines and Develop/Release pipelines.\n\n- PR pipelines are launched in response to pull requests made by trusted and untrusted users.\n Packages built on these pipelines upload code to quarantined AWS S3 locations which cache the built packages for the purposes of review and iteration on the changes proposed in the pull request.\n Packages built on PR pipelines can come from untrusted users so signing of these pipelines is not implemented.\n Jobs in these pipelines are executed via normal GitLab runners both within the AWS GitLab infrastructure and at affiliated institutions.\n- Develop and Release pipelines **sign** the packages they produce and carry strong integrity assurances that trace back to auditable change management decisions.\n These pipelines only run after members from a trusted group of reviewers verify that the proposed changes in a pull request are appropriate.\n Once the PR is merged, or a release is cut, a pipeline is run on protected GitLab runners which provide access to the required signing keys within the job.\n Intermediary keys are used to sign packages in each stage of the pipeline as they are built and a final job officially signs each package external to any specific package's build environment.\n An intermediate key exists in the AWS infrastructure and for each affiliated institution that maintains protected runners.\n The runners that execute these pipelines exclusively accept jobs from protected branches meaning the intermediate keys are never exposed to unreviewed code and the official keys are never exposed to any specific build environment.\n\n.. _key_architecture:\n\nKey Architecture\n----------------\n\nSpack's CI process uses public-key infrastructure (PKI) based on GNU Privacy Guard (gpg) keypairs to sign public releases of spack package metadata, also called specs.\nTwo classes of GPG keys are involved in the process to reduce the impact of an individual private key compromise, these key classes are the *Intermediate CI Key* and *Reputational Key*.\nEach of these keys has signing sub-keys that are used exclusively for signing packages.\nThis can be confusing so for the purpose of this explanation we will refer to Root and Signing keys.\nEach key has a private and a public component as well as one or more identities and zero or more signatures.\n\nIntermediate CI Key\n-------------------\n\nThe Intermediate key class is used to sign and verify packages between stages within a develop or release pipeline.\nAn intermediate key exists for the AWS infrastructure as well as each affiliated institution that maintains protected runners.\nThese intermediate keys are made available to the GitLab execution environment building the package so that the package's dependencies may be verified by the Signing Intermediate CI Public Key and the final package may be signed by the Signing Intermediate CI Private Key.\n\n\n+---------------------------------------------------------------------------------------------------------+\n| **Intermediate CI Key (GPG)** |\n+==================================================+======================================================+\n| Root Intermediate CI Private Key (RSA 4096) | Root Intermediate CI Public Key (RSA 4096) |\n+--------------------------------------------------+------------------------------------------------------+\n| Signing Intermediate CI Private Key (RSA 4096) | Signing Intermediate CI Public Key (RSA 4096) |\n+--------------------------------------------------+------------------------------------------------------+\n| Identity: \"Intermediate CI Key \" |\n+---------------------------------------------------------------------------------------------------------+\n| Signatures: None |\n+---------------------------------------------------------------------------------------------------------+\n\n\nThe *Root intermediate CI Private Key*\\ is stripped out of the GPG key and stored offline completely separate from Spack's infrastructure.\nThis allows the core development team to append revocation certificates to the GPG key and issue new sub-keys for use in the pipeline.\nIt is our expectation that this will happen on a semi-regular basis.\nA corollary of this is that *this key should not be used to verify package integrity outside the internal CI process.*\n\nReputational Key\n----------------\n\nThe Reputational Key is the public facing key used to sign complete groups of development and release packages.\nOnly one key pair exists in this class of keys.\nIn contrast to the Intermediate CI Key the Reputational Key *should* be used to verify package integrity.\nAt the end of develop and release pipelines a final pipeline job pulls down all signed package metadata built by the pipeline, verifies they were signed with an Intermediate CI Key, then strips the Intermediate CI Key signature from the package and re-signs them with the Signing Reputational Private Key.\nThe officially signed packages are then uploaded back to the AWS S3 mirror.\nPlease note that separating use of the reputational key into this final job is done to prevent leakage of the key in a spack package.\nBecause the Signing Reputational Private Key is never exposed to a build job it cannot accidentally end up in any built package.\n\n\n+---------------------------------------------------------------------------------------------------------+\n| **Reputational Key (GPG)** |\n+==================================================+======================================================+\n| Root Reputational Private Key (RSA 4096)# | Root Reputational Public Key (RSA 4096) |\n+--------------------------------------------------+------------------------------------------------------+\n| Signing Reputational Private Key (RSA 4096) | Signing Reputational Public Key (RSA 4096) |\n+--------------------------------------------------+------------------------------------------------------+\n| Identity: \"Spack Project \" |\n+---------------------------------------------------------------------------------------------------------+\n| Signatures: Signed by core development team [#f1]_ |\n+---------------------------------------------------------------------------------------------------------+\n\nThe Root Reputational Private Key is stripped out of the GPG key and stored offline completely separate from Spack's infrastructure.\nThis allows the core development team to append revocation certificates to the GPG key in the unlikely event that the Signing Reputation Private Key is compromised.\nIn general it is the expectation that rotating this key will happen infrequently if at all.\nThis should allow relatively transparent verification for the end-user community without needing deep familiarity with GnuPG or Public Key Infrastructure.\n\n\n.. _build_cache_signing:\n\nBuild Cache Signing\n-------------------\n\nFor an in-depth description of the layout of a binary mirror, see the :ref:`documentation` covering binary caches.\nThe key takeaway from that discussion that applies here is that the entry point to a binary package is its manifest.\nThe manifest refers unambiguously to the spec metadata and compressed archive, which are stored as content-addressed blobs.\n\nThe manifest files can either be signed or unsigned, but are always given a name ending with ``.spec.manifest.json`` regardless.\nThe difference between signed and unsigned manifests is simply that the signed version is wrapped in a gpg cleartext signature, as illustrated below:\n\n.. code-block:: text\n\n -----BEGIN PGP SIGNED MESSAGE-----\n Hash: SHA512\n\n {\n \"version\": 3,\n \"data\": [\n {\n \"contentLength\": 10731083,\n \"mediaType\": \"application/vnd.spack.install.v2.tar+gzip\",\n \"compression\": \"gzip\",\n \"checksumAlgorithm\": \"sha256\",\n \"checksum\": \"0f24aa6b5dd7150067349865217acd3f6a383083f9eca111d2d2fed726c88210\"\n },\n {\n \"contentLength\": 1000,\n \"mediaType\": \"application/vnd.spack.spec.v5+json\",\n \"compression\": \"gzip\",\n \"checksumAlgorithm\": \"sha256\",\n \"checksum\": \"fba751c4796536737c9acbb718dad7429be1fa485f5585d450ab8b25d12ae041\"\n }\n ]\n }\n -----BEGIN PGP SIGNATURE-----\n\n iQGzBAEBCgAdFiEEdbwFKBFJCcB24mB0GAEP+tc8mwcFAmf2rr4ACgkQGAEP+tc8\n mwfefwv+KJs8MsQ5ovFaBdmyx5H/3k4rO4QHBzuSPOB6UaxErA9IyOB31iP6vNTU\n HzYpxz6F5dJCJWmmNEMN/0+vjhMHEOkqd7M1l5reVcxduTF2yc4tBZUO2gienEHL\n W0e+SnUznl1yc/aVpChUiahO2zToCsI8HZRNT4tu6iCnE/OpghqjsSdBOZHmSNDD\n 5wuuCxfDUyWI6ZlLclaaB7RdbCUUJf/iqi711J+wubvnDFhc6Ynwm1xai5laJ1bD\n ev3NrSb2AAroeNFVo4iECA0fZC1OZQYzaRmAEhBXtCideGJ5Zf2Cp9hmCwNK8Hq6\n bNt94JP9LqC3FCCJJOMsPyOOhMSA5MU44zyyzloRwEQpHHLuFzVdbTHA3dmTc18n\n HxNLkZoEMYRc8zNr40g0yb2lCbc+P11TtL1E+5NlE34MX15mPewRCiIFTMwhCnE3\n gFSKtW1MKustZE35/RUwd2mpJRf+mSRVCl1f1RiFjktLjz7vWQq7imIUSam0fPDr\n XD4aDogm\n =RrFX\n -----END PGP SIGNATURE-----\n\nIf a user has trusted the public key associated with the private key used to sign the above manifest file, the signature can be verified with gpg, as follows:\n\n.. code-block:: console\n\n $ gpg --verify gcc-runtime-12.3.0-s2nqujezsce4x6uhtvxscu7jhewqzztx.spec.manifest.json\n\nWhen attempting to install a binary package that has been signed, spack will attempt to verify the signature with one of the trusted keys in its keyring, and will fail if unable to do so.\nWhile not recommended, it is possible to force installation of a signed package without verification by providing the ``--no-check-signature`` argument to ``spack install ...``.\n\n.. _internal_implementation:\n\nInternal Implementation\n-----------------------\n\nThe technical implementation of the pipeline signing process includes components defined in Amazon Web Services, the Kubernetes cluster, at affiliated institutions, and the GitLab/GitLab Runner deployment.\nWe present the technical implementation in two interdependent sections.\nThe first addresses how secrets are managed through the lifecycle of a develop or release pipeline.\nThe second section describes how Gitlab Runner and pipelines are configured and managed to support secure automated signing.\n\nSecrets Management\n^^^^^^^^^^^^^^^^^^\n\nAs stated above the Root Private Keys (intermediate and reputational) are stripped from the GPG keys and stored outside Spack's infrastructure.\n\n.. .. admonition:: TODO\n.. :class: warning\n\n.. - Explanation here about where and how access is handled for these keys.\n.. - Both Root private keys are protected with strong passwords\n.. - Who has access to these and how?\n\nIntermediate CI Key\n^^^^^^^^^^^^^^^^^^^\n\nMultiple intermediate CI signing keys exist, one Intermediate CI Key for jobs run in AWS, and one key for each affiliated institution (e.g. University of Oregon).\nHere we describe how the Intermediate CI Key is managed in AWS:\n\nThe Intermediate CI Key (including the Signing Intermediate CI Private Key) is exported as an ASCII armored file and stored in a Kubernetes secret called ``spack-intermediate-ci-signing-key``.\nFor convenience sake, this same secret contains an ASCII-armored export of just the *public* components of the Reputational Key.\nThis secret also contains the *public* components of each of the affiliated institutions' Intermediate CI Key.\nThese are potentially needed to verify dependent packages which may have been found in the public mirror or built by a protected job running on an affiliated institution's infrastructure in an earlier stage of the pipeline.\n\nProcedurally the ``spack-intermediate-ci-signing-key`` secret is used in the following way:\n\n1. A ``large-arm-prot`` or ``large-x86-prot`` protected runner picks up a job tagged ``protected`` from a protected GitLab branch.\n (See :ref:`protected_runners`).\n2. Based on its configuration, the runner creates a job Pod in the pipeline namespace and mounts the ``spack-intermediate-ci-signing-key`` Kubernetes secret into the build container\n3. The Intermediate CI Key, affiliated institutions' public key and the Reputational Public Key are imported into a keyring by the ``spack gpg ...`` sub-command.\n This is initiated by the job's build script which is created by the generate job at the beginning of the pipeline.\n4. Assuming the package has dependencies those spec manifests are verified using the keyring.\n5. The package is built and the spec manifest is generated\n6. The spec manifest is signed by the keyring and uploaded to the mirror's build cache.\n\nReputational Key\n^^^^^^^^^^^^^^^^\n\nBecause of the increased impact to end users in the case of a private key breach, the Reputational Key is managed separately from the Intermediate CI Keys and has additional controls.\nFirst, the Reputational Key was generated outside of Spack's infrastructure and has been signed by the core development team.\nThe Reputational Key (along with the Signing Reputational Private Key) was then ASCII armor exported to a file.\nUnlike the Intermediate CI Key this exported file is not stored as a base64 encoded secret in Kubernetes.\nInstead\\ *the key file itself*\\ is encrypted and stored in Kubernetes as the ``spack-signing-key-encrypted`` secret in the pipeline namespace.\n\nThe encryption of the exported Reputational Key (including the Signing Reputational Private Key) is handled by `AWS Key Management Store (KMS) data keys `__.\nThe private key material is decrypted and imported at the time of signing into a memory mounted temporary directory holding the keychain.\nThe signing job uses the `AWS Encryption SDK `__ (i.e. ``aws-encryption-cli``) to decrypt the Reputational Key.\nPermission to decrypt the key is granted to the job Pod through a Kubernetes service account specifically used for this, and only this, function.\nFinally, for convenience sake, this same secret contains an ASCII-armored export of the *public* components of the Intermediate CI Keys and the Reputational Key.\nThis allows the signing script to verify that packages were built by the pipeline (both on AWS or at affiliated institutions), or signed previously as a part of a different pipeline.\nThis is done *before* importing decrypting and importing the Signing Reputational Private Key material and officially signing the packages.\n\nProcedurally the ``spack-signing-key-encrypted`` secret is used in the following way:\n\n1. The ``spack-package-signing-gitlab-runner`` protected runner picks up a job tagged ``notary`` from a protected GitLab branch (See :ref:`protected_runners`).\n2. Based on its configuration, the runner creates a job pod in the pipeline namespace.\n The job is run in a stripped down purpose-built image ``ghcr.io/spack/notary:latest`` Docker image.\n The runner is configured to only allow running jobs with this image.\n3. The runner also mounts the ``spack-signing-key-encrypted`` secret to a path on disk.\n Note that this becomes several files on disk, the public components of the Intermediate CI Keys, the public components of the Reputational CI, and an AWS KMS encrypted file containing the Signing Reputational Private Key.\n4. In addition to the secret, the runner creates a tmpfs memory mounted directory where the GnuPG keyring will be created to verify, and then resign the package specs.\n5. The job script syncs all spec manifest files from the build cache to a working directory in the job's execution environment.\n6. The job script then runs the ``sign.sh`` script built into the Notary Docker image.\n7. The ``sign.sh`` script imports the public components of the Reputational and Intermediate CI Keys and uses them to verify good signatures on the spec.manifest.json files.\n If any signed manifest does not verify, the job immediately fails.\n8. Assuming all manifests are verified, the ``sign.sh`` script then unpacks the manifest json data from the signed file in preparation for being re-signed with the Reputational Key.\n9. The private components of the Reputational Key are decrypted to standard out using ``aws-encryption-cli`` directly into a ``gpg --import ...`` statement which imports the key into the keyring mounted in-memory.\n10. The private key is then used to sign each of the manifests and the keyring is removed from disk.\n11. The re-signed manifests are resynced to the AWS S3 Mirror and the public signing of the packages for the develop or release pipeline that created them is complete.\n\nNon service-account access to the private components of the Reputational Key that are managed through access to the symmetric secret in KMS used to encrypt the data key (which in turn is used to encrypt the GnuPG key - See:\\ `Encryption SDK Documentation `__).\nA small trusted subset of the core development team are the only individuals with access to this symmetric key.\n\n.. _protected_runners:\n\nProtected Runners and Reserved Tags\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nSpack has a large number of Gitlab Runners operating in its build farm.\nThese include runners deployed in the AWS Kubernetes cluster as well as runners deployed at affiliated institutions.\nThe majority of runners are shared runners that operate across projects in `gitlab.spack.io `_.\nThese runners pick up jobs primarily from the spack/spack project and execute them in PR pipelines.\n\nA small number of runners operating on AWS and at affiliated institutions are registered as specific *protected* runners on the spack/spack project.\nIn addition to protected runners there are protected branches on the spack/spack project.\nThese are the ``develop`` branch, any release branch (i.e. managed with the ``releases/v*`` wildcard) and any tag branch (managed with the ``v*`` wildcard).\nFinally, Spack's pipeline generation code reserves certain tags to make sure jobs are routed to the correct runners; these tags are ``public``, ``protected``, and ``notary``.\nUnderstanding how all this works together to protect secrets and provide integrity assurances can be a little confusing so lets break these down:\n\nProtected Branches\n Protected branches in Spack prevent anyone other than Maintainers in GitLab from pushing code.\n In the case of Spack, the only Maintainer level entity pushing code to protected branches is Spack bot.\n Protecting branches also marks them in such a way that Protected Runners will only run jobs from those branches\n\nProtected Runners\n Protected Runners only run jobs from protected branches.\n Because protected runners have access to secrets, it's critical that they not run jobs from untrusted code (i.e. PR branches).\n If they did, it would be possible for a PR branch to tag a job in such a way that a protected runner executed that job and mounted secrets into a code execution environment that had not been reviewed by Spack maintainers.\n Note however that in the absence of tagging used to route jobs, public runners *could* run jobs from protected branches.\n No secrets would be at risk of being breached because non-protected runners do not have access to those secrets; lack of secrets would, however, cause the jobs to fail.\n\nReserved Tags\n To mitigate the issue of public runners picking up protected jobs Spack uses a small set of \"reserved\" job tags (Note that these are *job* tags not git tags).\n These tags are \"public\", \"private\", and \"notary.\"\n The majority of jobs executed in Spack's GitLab instance are executed via a ``generate`` job.\n The generate job code systematically ensures that no user defined configuration sets these tags.\n Instead, the ``generate`` job sets these tags based on rules related to the branch where this pipeline originated.\n If the job is a part of a pipeline on a PR branch it sets the ``public`` tag.\n If the job is part of a pipeline on a protected branch it sets the ``protected`` tag.\n Finally if the job is the package signing job and it is running on a pipeline that is part of a protected branch then it sets the ``notary`` tag.\n\nProtected Runners are configured to only run jobs from protected branches.\nOnly jobs running in pipelines on protected branches are tagged with ``protected`` or ``notary`` tags.\nThis tightly couples jobs on protected branches to protected runners that provide access to the secrets required to sign the built packages.\nThe secrets can **only** be accessed via:\n\n1. Runners under direct control of the core development team.\n2. Runners under direct control of trusted maintainers at affiliated institutions.\n3. By code running the automated pipeline that has been reviewed by the Spack maintainers and judged to be appropriate.\n\nOther attempts (either through malicious intent or incompetence) can at worst grab jobs intended for protected runners which will cause those jobs to fail alerting both Spack maintainers and the core development team.\n\n.. [#f1]\n The Reputational Key has also cross signed core development team keys.\n" }, { "path": "lib/spack/docs/spack.yaml", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n#\n# These are requirements for building the documentation. You can run\n# these commands in this directory to install Sphinx and its plugins,\n# then build the docs:\n#\n# spack env activate .\n# spack install\n# make\n#\nspack:\n specs:\n # Sphinx\n - \"py-sphinx@3.4:4.1.1,4.1.3:\"\n - py-sphinxcontrib-programoutput\n - py-docutils@:0.16\n - py-sphinx-design\n - py-sphinx-rtd-theme\n - py-pygments@:2.12\n\n # VCS\n - git\n - mercurial\n - subversion\n # Plotting\n - graphviz\n concretizer:\n unify: true\n" }, { "path": "lib/spack/docs/spec_syntax.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n A detailed guide to the Spack spec syntax for describing package constraints, including versions, variants, and dependencies.\n\n.. _sec-specs:\n\nSpec Syntax\n===========\n\nSpack has a specific syntax to describe package constraints.\nEach constraint is individually referred to as a *spec*.\nSpack uses specs to:\n\n1. Refer to a particular build configuration of a package, or\n2. Express requirements, or preferences, on packages via configuration files, or\n3. Query installed packages, or build caches\n\nSpecs are more than a package name and a version; you can use them to specify the compiler, compiler version, architecture, compile options, and dependency options for a build.\nIn this section, we'll go over the full syntax of specs.\n\nHere is an example of using a complex spec to install a very specific configuration of ``mpileaks``:\n\n.. code-block:: spec\n\n $ spack install mpileaks@1.2:1.4 +debug ~qt target=x86_64_v3 %gcc@15 ^libelf@1.1 %clang@20\n\nThe figure below helps you get a sense of the various parts that compose this spec:\n\n.. figure:: images/spec_anatomy.svg\n :alt: Spack spec with annotations\n :width: 740\n :height: 180\n\nWhen installing this, you will get:\n\n* The ``mpileaks`` package at some version between ``1.2`` and ``1.4`` (inclusive),\n* with ``debug`` options enabled, and without ``qt`` support,\n* optimized for an ``x86_64_v3`` architecture,\n* built using ``gcc`` at version ``15``,\n* depending on ``libelf`` at version ``1.1``, built with ``clang`` at version ``20``.\n\nMost specs will not be as complicated as this one, but this is a good example of what is possible with specs.\nThere are a few general rules that we can already infer from this first example:\n\n1. Users can be as vague, or as specific, as they want about the details of building packages\n2. The spec syntax is recursive, i.e. each dependency after ``%`` or ``^`` is a spec itself\n3. Transitive dependencies come after the ``^`` sigil, and they always refer to the root package\n4. Direct dependencies come after the ``%`` sigil, and they refer either to the root package, or to the last transitive dependency defined\n\nThe flexibility the spec syntax offers in specifying the details of a build makes Spack good for beginners and experts alike.\n\n.. _software-model:\n\nSoftware Model\n--------------\n\nTo really understand what's going on above, we need to think about how software is structured.\nAn executable or a library generally depends on other libraries in order to run.\nWe can represent the relationship between a package and its dependencies as a graph.\nHere is a simplified dependency graph for ``mpileaks``:\n\n.. graphviz::\n\n digraph {\n node[\n fontname=Monaco,\n penwidth=2,\n fontsize=124,\n margin=.4,\n shape=box,\n fillcolor=lightblue,\n style=\"rounded,filled\"\n ]\n\n mpileaks -> { mpich callpath }\n callpath -> { mpich dyninst }\n dyninst -> libdwarf -> libelf\n dyninst -> libelf\n }\n\nEach box above is a package, and each arrow represents a dependency on some other package.\nFor example, we say that the package ``mpileaks`` *depends on* ``callpath`` and ``mpich``.\n``mpileaks`` also depends *indirectly* on ``dyninst``, ``libdwarf``, and ``libelf``, in that these libraries are dependencies of ``callpath``.\nTo install ``mpileaks``, Spack has to build all of these packages.\nDependency graphs in Spack have to be acyclic, and the *depends on* relationship is directional, so this is a *directed, acyclic graph* or *DAG*.\n\nThe package name identifier in the spec is the root of some dependency DAG, and the DAG itself is implicit.\nSpack knows the precise dependencies among packages, but users do not need to know the full DAG structure.\nEach ``^`` in the full spec refers to a *transitive* dependency of the root package.\nEach ``%`` refers to a *direct* dependency, either of the root, or of the last defined transitive dependency .\n\nSpack allows only a single configuration of each package, where that is needed for consistency.\nAbove, both ``mpileaks`` and ``callpath`` depend on ``mpich``, but ``mpich`` appears only once in the DAG.\nYou cannot build an ``mpileaks`` version that depends on one version of ``mpich`` *and* on a ``callpath`` version that depends on some *other* version of ``mpich``.\nIn general, such a configuration would likely behave unexpectedly at runtime, and Spack enforces this to ensure a consistent runtime environment.\n\nThe purpose of specs is to abstract this full DAG away from Spack users.\nA user who does not care about the DAG at all, can refer to ``mpileaks`` by simply writing:\n\n.. code-block:: spec\n\n mpileaks\n\nThe spec becomes only slightly more complicated, if that user knows that ``mpileaks`` indirectly uses ``dyninst`` and wants a particular version of ``dyninst``:\n\n.. code-block:: spec\n\n mpileaks ^dyninst@8.1\n\nSpack will fill in the rest of the details before installing the spec.\nThe user only needs to know package names and minimal details about their relationship.\nYou can put all the same modifiers on dependency specs that you would put on the root spec.\nThat is, you can specify their versions, variants, and architectures just like any other spec.\nSpecifiers are associated with the nearest package name to their left.\n\n.. _sec-virtual-dependencies:\n\nVirtual dependencies\n^^^^^^^^^^^^^^^^^^^^\n\nThe dependency graph for ``mpileaks`` we saw above wasn't *quite* accurate.\n``mpileaks`` uses MPI, which is an interface that has many different implementations.\nAbove, we showed ``mpileaks`` and ``callpath`` depending on ``mpich``, which is one *particular* implementation of MPI.\nHowever, we could build either with another implementation, such as ``openmpi`` or ``mvapich``.\n\nSpack represents interfaces like this using *virtual dependencies*.\nThe real dependency DAG for ``mpileaks`` looks like this:\n\n.. graphviz::\n\n digraph {\n node[\n fontname=Monaco,\n penwidth=2,\n fontsize=124,\n margin=.4,\n shape=box,\n fillcolor=lightblue,\n style=\"rounded,filled\"\n ]\n\n mpi [color=red]\n mpileaks -> mpi\n mpileaks -> callpath -> mpi\n callpath -> dyninst\n dyninst -> libdwarf -> libelf\n dyninst -> libelf\n }\n\nNotice that ``mpich`` has now been replaced with ``mpi``.\nThere is no *real* MPI package, but some packages *provide* the MPI interface, and these packages can be substituted in for ``mpi`` when ``mpileaks`` is built.\n\nSpack is unique in that its virtual packages can be versioned, just like regular packages.\nA particular version of a package may provide a particular version of a virtual package.\nA package can *depend on* a particular version of a virtual package.\nFor instance, if an application needs MPI-2 functions, it can depend on ``mpi@2:`` to indicate that it needs some implementation that provides MPI-2 functions.\n\nBelow are more details about the specifiers that you can add to specs.\n\n.. _version-specifier:\n\nVersion specifier\n-----------------\n\nA version specifier\n\n.. code-block:: spec\n\n pkg@specifier\n\ncomes after a package name and starts with ``@``.\nIt can be something abstract that matches multiple known versions or a specific version.\n\nThe version specifier usually represents *a range of versions*:\n\n.. code-block:: spec\n\n # All versions between v1.0 and v1.5.\n # This includes any v1.5.x version\n @1.0:1.5\n\n # All versions up to and including v3\n # This would include v3.4 etc.\n @:3\n\n # All versions above and including v4.2\n @4.2:\n\nbut can also be *a specific version*:\n\n.. code-block:: spec\n\n # Exactly version v3.2, will NOT match v3.2.1 etc.\n @=3.2\n\n\nAs a shorthand, ``@3`` is equivalent to the range ``@3:3`` and includes any version with major version ``3``.\nVersions are ordered lexicographically by their components.\nFor more details on the order, see :ref:`the packaging guide `.\n\nNotice that you can distinguish between the specific version ``@=3.2`` and the range ``@3.2``.\nThis is useful for packages that follow a versioning scheme that omits the zero patch version number: ``3.2``, ``3.2.1``, ``3.2.2``, etc.\nIn general, it is preferable to use the range syntax ``@3.2``, because ranges also match versions with one-off suffixes, such as ``3.2-custom``.\n\nA version specifier can also be a list of ranges and specific versions, separated by commas.\nFor example:\n\n.. code-block:: spec\n\n @1.0:1.5,=1.7.1\n\nmatches any version in the range ``1.0:1.5`` and the specific version ``1.7.1``.\n\nGit versions\n^^^^^^^^^^^^\n\n.. note::\n Users wanting to just match specific commits for branch or tag based versions should assign the ``commit`` variant (``commit=<40 char sha>``).\n Spack reserves this variant specifically to track provenance of git based versions.\n Spack will attempt to compute this value for you automatically during concretization and raise a warning if it is unable to assign the commit.\n Further details can be found in :ref:`git_version_provenance`.\n\n\nFor packages with a ``git`` attribute, ``git`` references may be specified instead of a numerical version (i.e., branches, tags, and commits).\nSpack will stage and build based off the ``git`` reference provided.\nAcceptable syntaxes for this are:\n\n.. code-block:: spec\n\n # commit hashes\n foo@abcdef1234abcdef1234abcdef1234abcdef1234 # 40 character hashes are automatically treated as git commits\n foo@git.abcdef1234abcdef1234abcdef1234abcdef1234\n\n # branches and tags\n foo@git.develop # use the develop branch\n foo@git.0.19 # use the 0.19 tag\n\nSpack always needs to associate a Spack version with the git reference, which is used for version comparison.\nThis Spack version is heuristically taken from the closest valid git tag among the ancestors of the git ref.\n\nOnce a Spack version is associated with a git ref, it is always printed with the git ref.\nFor example, if the commit ``@git.abcdefg`` is tagged ``0.19``, then the spec will be shown as ``@git.abcdefg=0.19``.\n\nIf the git ref is not exactly a tag, then the distance to the nearest tag is also part of the resolved version.\n``@git.abcdefg=0.19.git.8`` means that the commit is 8 commits away from the ``0.19`` tag.\n\nIn cases where Spack cannot resolve a sensible version from a git ref, users can specify the Spack version to use for the git ref.\nThis is done by appending ``=`` and the Spack version to the git ref.\nFor example:\n\n.. code-block:: spec\n\n foo@git.my_ref=3.2 # use the my_ref tag or branch, but treat it as version 3.2 for version comparisons\n foo@git.abcdef1234abcdef1234abcdef1234abcdef1234=develop # use the given commit, but treat it as develop for version comparisons\n\nDetails about how versions are compared and how Spack determines if one version is less than another are discussed in the developer guide.\n\n.. _basic-variants:\n\nVariants\n--------\n\nVariants are named options associated with a particular package and are typically used to enable or disable certain features at build time.\nThey are optional, as each package must provide default values for each variant it makes available.\n\nThe variants available for a particular package are defined by the package author.\n``spack info `` will provide information on what build variants are available.\n\nThere are different types of variants.\n\nBoolean Variants\n^^^^^^^^^^^^^^^^\n\nTypically used to enable or disable a feature at compile time.\nFor example, a package might have a ``debug`` variant that can be explicitly enabled with:\n\n.. code-block:: spec\n\n +debug\n\nand disabled with\n\n.. code-block:: spec\n\n ~debug\n\nSingle-valued Variants\n^^^^^^^^^^^^^^^^^^^^^^\n\nOften used to set defaults.\nFor example, a package might have a ``compression`` variant that determines the default compression algorithm, which users could set to:\n\n.. code-block:: spec\n\n compression=gzip\n\nor\n\n.. code-block:: spec\n\n compression=zstd\n\nMulti-valued Variants\n^^^^^^^^^^^^^^^^^^^^^\n\nA package might have a ``fabrics`` variant that determines which network fabrics to support.\nUsers could activate multiple values at the same time.\nFor instance:\n\n.. code-block:: spec\n\n fabrics=verbs,ofi\n\nenables both InfiniBand verbs and OpenFabrics interfaces.\nThe values are separated by commas.\n\nThe meaning of ``fabrics=verbs,ofi`` is to enable *at least* the specified fabrics, but other fabrics may be enabled as well.\nIf the intent is to enable *only* the specified fabrics, then the:\n\n.. code-block:: spec\n\n fabrics:=verbs,ofi\n\nsyntax should be used with the ``:=`` operator.\n\n\nVariant propagation to dependencies\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nSpack allows variants to propagate their value to the package's dependencies by using ``++``, ``--``, and ``~~`` for boolean variants.\nFor example, for a ``debug`` variant:\n\n.. code-block:: spec\n\n mpileaks ++debug # enabled debug will be propagated to dependencies\n mpileaks +debug # only mpileaks will have debug enabled\n\nTo propagate the value of non-boolean variants Spack uses ``name==value``.\nFor example, for the ``stackstart`` variant:\n\n.. code-block:: spec\n\n mpileaks stackstart==4 # variant will be propagated to dependencies\n mpileaks stackstart=4 # only mpileaks will have this variant value\n\nSpack also allows variants to be propagated from a package that does not have that variant.\n\nCompiler Flags\n--------------\n\nCompiler flags are specified using the same syntax as non-boolean variants, but fulfill a different purpose.\nWhile the function of a variant is set by the package, compiler flags are used by the compiler wrappers to inject flags into the compile line of the build.\nAdditionally, compiler flags can be inherited by dependencies by using ``==``.\n``spack install libdwarf cppflags==\"-g\"`` will install both libdwarf and libelf with the ``-g`` flag injected into their compile line.\n\nNotice that the value of the compiler flags must be quoted if it contains any spaces.\nAny of ``cppflags=-O3``, ``cppflags=\"-O3\"``, ``cppflags='-O3'``, and ``cppflags=\"-O3 -fPIC\"`` are acceptable, but ``cppflags=-O3 -fPIC`` is not.\nAdditionally, if the value of the compiler flags is not the last thing on the line, it must be followed by a space.\nThe command ``spack install libelf cppflags=\"-O3\"%intel`` will be interpreted as an attempt to set ``cppflags=\"-O3%intel\"``.\n\nThe six compiler flags are injected in the same order as implicit make commands in GNU Autotools.\nIf all flags are set, the order is ``$cppflags $cflags|$cxxflags $ldflags $ldlibs`` for C and C++, and ``$fflags $cppflags $ldflags $ldlibs`` for Fortran.\n\n\n.. _architecture_specifiers:\n\nArchitecture specifiers\n-----------------------\n\nEach node in the dependency graph of a spec has an architecture attribute.\nThis attribute is a triplet of platform, operating system, and processor.\nYou can specify the elements either separately by using the reserved keywords ``platform``, ``os``, and ``target``:\n\n.. code-block:: spec\n\n $ spack install libelf platform=linux\n $ spack install libelf os=ubuntu18.04\n $ spack install libelf target=broadwell\n\nNormally, users don't have to bother specifying the architecture if they are installing software for their current host, as in that case the values will be detected automatically.\nIf you need fine-grained control over which packages use which targets (or over *all* packages' default target), see :ref:`package-preferences`.\n\n\n.. _support-for-microarchitectures:\n.. _cmd-spack-arch:\n\nSupport for specific microarchitectures\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nSpack knows how to detect and optimize for many specific microarchitectures and encodes this information in the ``target`` portion of the architecture specification.\nA complete list of the microarchitectures known to Spack can be obtained in the following way:\n\n.. command-output:: spack arch --known-targets\n\nWhen a spec is installed, Spack matches the compiler being used with the microarchitecture being targeted to inject appropriate optimization flags at compile time.\nGiving a command such as the following:\n\n.. code-block:: spec\n\n $ spack install zlib target=icelake %gcc@14\n\nwill produce compilation lines similar to:\n\n.. code-block:: console\n\n $ /usr/bin/gcc-14 -march=icelake-client -mtune=icelake-client -c ztest10532.c\n $ /usr/bin/gcc-14 -march=icelake-client -mtune=icelake-client -c -fPIC -O2 ztest10532.\n ...\n\nwhere the flags ``-march=icelake-client -mtune=icelake-client`` are injected by Spack based on the requested target and compiler.\n\nIf Spack determines that the requested compiler cannot optimize for the requested target or cannot build binaries for that target at all, it will exit with a meaningful error message:\n\n.. code-block:: spec\n\n $ spack install zlib target=icelake %gcc@5\n ==> Error: cannot produce optimized binary for micro-architecture \"icelake\" with gcc@5.5.0 [supported compiler versions are 8:]\n\nConversely, if an older compiler is selected for a newer microarchitecture, Spack will optimize for the best match instead of failing:\n\n.. code-block:: spec\n\n $ spack arch\n linux-ubuntu18.04-broadwell\n\n $ spack spec zlib%gcc@4.8\n Input spec\n --------------------------------\n zlib%gcc@4.8\n\n Concretized\n --------------------------------\n zlib@1.2.11%gcc@4.8+optimize+pic+shared arch=linux-ubuntu18.04-haswell\n\n $ spack spec zlib%gcc@9.0.1\n Input spec\n --------------------------------\n zlib%gcc@9.0.1\n\n Concretized\n --------------------------------\n zlib@1.2.11%gcc@9.0.1+optimize+pic+shared arch=linux-ubuntu18.04-broadwell\n\nIn the snippet above, for instance, the microarchitecture was demoted to ``haswell`` when compiling with ``gcc@4.8`` because support to optimize for ``broadwell`` starts from ``gcc@4.9:``.\n\nFinally, if Spack has no information to match the compiler and target, it will proceed with the installation but avoid injecting any microarchitecture-specific flags.\n\n\n.. _sec-dependencies:\n\nDependencies\n------------\n\nEach node in a DAG can specify dependencies using either the ``%`` or the ``^`` sigil:\n\n* The ``%`` sigil identifies direct dependencies, which means there must be an edge connecting the dependency to the node they refer to.\n* The ``^`` sigil identifies transitive dependencies, which means the dependency just needs to be in the sub-DAG of the node they refer to.\n\nThe order of transitive dependencies does not matter when writing a spec.\nFor example, these two specs represent exactly the same configuration:\n\n.. code-block:: spec\n\n mpileaks ^callpath@1.0 ^libelf@0.8.3\n mpileaks ^libelf@0.8.3 ^callpath@1.0\n\nDirect dependencies specified with ``%`` apply either to the most recent transitive dependency (``^``), or, if none, to the root package in the spec.\nSo in the spec:\n\n.. code-block:: spec\n\n root %dep1 ^transitive %dep2 %dep3\n\n``dep1`` is a direct dependency of ``root``, while both ``dep2`` and ``dep3`` are direct dependencies of ``transitive``.\n\nConstraining virtual packages\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nWhen installing a package that depends on a virtual package, see :ref:`sec-virtual-dependencies`, you can opt to specify the particular provider you want to use, or you can let Spack pick.\nFor example, if you just type this:\n\n.. code-block:: spec\n\n $ spack install mpileaks\n\nThen Spack will pick an ``mpi`` provider for you according to site policies.\nIf you really want a particular version, say ``mpich``, then you could run this instead:\n\n.. code-block:: spec\n\n $ spack install mpileaks ^mpich\n\nThis forces Spack to use some version of ``mpich`` for its implementation.\nAs always, you can be even more specific and require a particular ``mpich`` version:\n\n.. code-block:: spec\n\n $ spack install mpileaks ^mpich@3\n\nThe ``mpileaks`` package in particular only needs MPI-1 commands, so any MPI implementation will do.\nIf another package depends on ``mpi@2`` and you try to give it an insufficient MPI implementation (e.g., one that provides only ``mpi@:1``), then Spack will raise an error.\nLikewise, if you try to plug in some package that doesn't provide MPI, Spack will raise an error.\n\n.. _explicit-binding-virtuals:\n\nExplicit binding of virtual dependencies\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nThere are packages that provide more than just one virtual dependency.\nWhen interacting with them, users might want to utilize just a subset of what they could provide and use other providers for virtuals they need.\n\nIt is possible to be more explicit and tell Spack which dependency should provide which virtual, using a special syntax:\n\n.. code-block:: spec\n\n $ spack spec strumpack ^mpi=intel-parallel-studio+mkl ^lapack=openblas\n\nConcretizing the spec above produces the following DAG:\n\n.. figure:: images/strumpack_virtuals.svg\n :width: 3044\n :height: 1683\n\nwhere ``intel-parallel-studio`` *could* provide ``mpi``, ``lapack``, and ``blas`` but is used only for the former.\nThe ``lapack`` and ``blas`` dependencies are satisfied by ``openblas``.\n\nDependency edge attributes\n^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nSome specs require additional information about the relationship between a package and its dependency.\nThis information lives on the edge between the two, and can be specified by following the dependency sigil with square-brackets ``[]``.\nEdge attributes are always specified as key-value pairs:\n\n.. code-block:: spec\n\n root ^[key=value] dep\n\nIn the following sections we'll discuss the edge attributes that are currently allowed in the spec syntax.\n\nVirtuals\n\"\"\"\"\"\"\"\"\n\nPackages can provide, or depend on, multiple virtual packages.\nUsers can select which virtuals to use from which dependency by specifying the ``virtuals`` edge attribute:\n\n.. code-block:: spec\n\n $ spack install mpich %[virtuals=c,cxx] clang %[virtuals=fortran] gcc\n\nThe command above tells Spack to use ``clang`` to provide the ``c`` and ``cxx`` virtuals, and ``gcc`` to provide the ``fortran`` virtual.\n\nThe special syntax we have seen in :ref:`explicit-binding-virtuals` is a more compact way to specify the ``virtuals`` edge attribute.\nFor instance, an equivalent formulation of the command above is:\n\n.. code-block:: spec\n\n $ spack install mpich %c,cxx=clang %fortran=gcc\n\nConditional dependencies\n\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n\nConditional dependencies allow dependency constraints to be applied only under certain conditions.\nWe can express conditional constraints by specifying the ``when`` edge attribute:\n\n.. code-block:: spec\n\n $ spack install hdf5 ^[when=+mpi] mpich@3.1\n\nThis tells Spack that hdf5 should depend on ``mpich@3.1`` if it is configured with MPI support.\n\nDependency propagation\n^^^^^^^^^^^^^^^^^^^^^^\n\nThe dependency specifications on a node, can be propagated using a double percent ``%%`` sigil.\nThis is particularly useful when specifying compilers.\nFor instance, the following command:\n\n.. code-block:: spec\n\n $ spack install hdf5+cxx+fortran %%c,cxx=clang %%fortran=gfortran\n\ntells Spack to install ``hdf5`` using Clang as the C and C++ compiler, and GCC as the Fortran compiler.\nIt also tells Spack to propagate the same choices, as :ref:`strong preferences `, to the runtime sub-DAG of ``hdf5``.\nBuild tools are unaffected and can still prefer to use a different compiler.\n\nSpecifying Specs by Hash\n------------------------\n\nComplicated specs can become cumbersome to enter on the command line, especially when many of the qualifications are necessary to distinguish between similar installs.\nTo avoid this, when referencing an existing spec, Spack allows you to reference specs by their hash.\nWe previously discussed the spec hash that Spack computes.\nIn place of a spec in any command, substitute ``/`` where ```` is any amount from the beginning of a spec hash.\n\nFor example, let's say that you accidentally installed two different ``mvapich2`` installations.\nIf you want to uninstall one of them but don't know what the difference is, you can run:\n\n.. code-block:: spec\n\n $ spack find --long mvapich2\n ==> 2 installed packages.\n -- linux-centos7-x86_64 / gcc@6.3.0 ----------\n qmt35td mvapich2@2.2%gcc\n er3die3 mvapich2@2.2%gcc\n\n\nYou can then uninstall the latter installation using:\n\n.. code-block:: spec\n\n $ spack uninstall /er3die3\n\n\nOr, if you want to build with a specific installation as a dependency, you can use:\n\n.. code-block:: spec\n\n $ spack install trilinos ^/er3die3\n\n\nIf the given spec hash is sufficiently long as to be unique, Spack will replace the reference with the spec to which it refers.\nOtherwise, it will prompt for a more qualified hash.\n\nNote that this will not work to reinstall a dependency uninstalled by ``spack uninstall --force``.\n\nSpecs on the command line\n-------------------------\n\nThe characters used in the spec syntax were chosen to work well with most shells.\nHowever, there are cases where the shell may interpret the spec before Spack gets a chance to parse it, leading to unexpected results.\nHere we document two such cases, and how to avoid them.\n\nUnix shells\n^^^^^^^^^^^\n\nOn Unix-like systems, the shell may expand ``~foo`` to the home directory of a user named ``foo``, so Spack won't see it as a :ref:`disabled boolean variant ` ``foo``.\nTo work around this without quoting, you can avoid whitespace between the package name and boolean variants:\n\n.. code-block:: spec\n\n mpileaks ~debug # shell may expand this to `mpileaks /home/debug`\n mpileaks~debug # use this instead\n\nAlternatively, you can use a hyphen ``-`` character to disable a variant, but be aware that this *requires* a space between the package name and the variant:\n\n.. code-block:: spec\n\n mpileaks-debug # wrong: refers to a package named \"mpileaks-debug\"\n mpileaks -debug # right: refers to a package named mpileaks with debug disabled\n\nAs a last resort, ``debug=False`` can also be used to disable a boolean variant.\n\nWindows CMD\n^^^^^^^^^^^\n\nIn Windows CMD, the caret ``^`` is an escape character, and needs itself escaping.\nSimilarly, the equals ``=`` character has special meaning in CMD.\n\nTo use the caret and equals characters in a spec, you can quote and escape them like this:\n\n.. code-block:: console\n\n C:\\> spack install mpileaks \"^^libelf\" \"foo=bar\"\n\nThese issues are not present in PowerShell.\nSee GitHub issue `#42833 `_ and `#43348 `_ for more details.\n" }, { "path": "lib/spack/docs/toolchains_yaml.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n Define named compiler sets (toolchains) in Spack to easily and consistently apply compiler choices for C, C++, and Fortran across different packages.\n\n.. _toolchains:\n\nToolchains (toolchains.yaml)\n=============================\n\nToolchains let you group a set of compiler constraints under a single, user-defined name.\nThis allows you to reference a complex set of compiler choices for C, C++, and Fortran, with a simple spec like ``%my_toolchain``.\nThey are defined under the ``toolchains`` section of the configuration.\n\n.. seealso::\n\n The sections :ref:`language-dependencies` and :ref:`explicit-binding-virtuals` provide more background on how Spack handles languages and compilers.\n\nBasic usage\n-----------\n\nAs an example, the following configuration file defines a toolchain named ``llvm_gfortran``:\n\n.. code-block:: yaml\n :caption: ``~/.spack/toolchains.yaml``\n\n toolchains:\n llvm_gfortran:\n - spec: cflags=-O3\n - spec: \"%c=llvm\"\n when: \"%c\"\n - spec: \"%cxx=llvm\"\n when: \"%cxx\"\n - spec: \"%fortran=gcc\"\n when: \"%fortran\"\n\nThe ``when`` clause in each entry determines if that line's ``spec`` is applied.\nIn this example, it means that ``llvm`` is used as a compiler for the C and C++ languages, and ``gcc`` for Fortran, *whenever the package uses those languages*.\nThe spec ``cflags=-O3`` is *always* applied, because there is no ``when`` clause for that spec.\n\nThe toolchain can be referenced using\n\n.. code-block:: spec\n\n $ spack install my-package %llvm_gfortran\n\nToolchains are useful for three reasons:\n\n1. **They reduce verbosity.**\n Instead of multiple constraints ``%c,cxx=clang %fortran=gcc``, you can simply write ``%llvm_gfortran``.\n2. **They apply conditionally.**\n You can use ``my-package %llvm_gfortran`` even if ``my-package`` is not written in Fortran.\n3. **They apply locally.**\n Toolchains are used at the level of a single spec.\n\n\n.. _pitfalls-without-toolchains:\n\nPitfalls without toolchains\n---------------------------\n\nThe conditional nature of toolchains is important, because it helps you avoid two common pitfalls when specifying compilers.\n\n1. Firstly, when you specify ``my-package %gcc``, your spec is **underconstrained**: Spack has to make ``my-package`` depend on ``gcc``, but the constraint does not rule out mixed compilers, such as ``gcc`` for C and ``llvm`` for C++.\n\n2. Secondly, when you specify ``my-package %c,cxx,fortran=gcc`` to be more explicit, your spec might be **overconstrained**.\n You not only require ``gcc`` for all languages, but *also* that ``my-package`` uses *all* these languages.\n This will cause a concretization error if ``my-package`` is written in C and C++, but not Fortran.\n\nCombining toolchains\n--------------------\n\nDifferent toolchains can be used independently or even in the same spec.\nConsider the following configuration:\n\n.. code-block:: yaml\n :caption: ``~/.spack/toolchains.yaml``\n\n toolchains:\n llvm_gfortran:\n - spec: cflags=-O3\n - spec: \"%c=llvm\"\n when: \"%c\"\n - spec: \"%cxx=llvm\"\n when: \"%cxx\"\n - spec: \"%fortran=gcc\"\n when: \"%fortran\"\n gcc_all:\n - spec: \"%c=gcc\"\n when: \"%c\"\n - spec: \"%cxx=gcc\"\n when: \"%cxx\"\n - spec: \"%fortran=gcc\"\n when: \"%fortran\"\n\n\nNow, you can use these toolchains in a single spec:\n\n.. code-block:: spec\n\n $ spack install hdf5+fortran%llvm_gfortran ^mpich %gcc_all\n\nThis will result in:\n\n* An ``hdf5`` compiled with ``llvm`` for the C/C++ components, but with its Fortran components compiled with ``gfortran``,\n* Built against an MPICH installation compiled entirely with ``gcc`` for C, C++, and Fortran.\n\nToolchains for other dependencies\n---------------------------------\n\nWhile toolchains are typically used to define compiler presets, they can be used for other dependencies as well.\n\nA common use case is to define a toolchain that also picks a specific MPI implementation.\nIn the following example, we define a toolchain that uses ``openmpi@5`` as an MPI provider, and ``llvm@19`` as the compiler for C and C++:\n\n.. code-block:: yaml\n :caption: ``~/.spack/toolchains.yaml``\n\n toolchains:\n clang_openmpi:\n - spec: \"%c=llvm@19\"\n when: \"%c\"\n - spec: \"%cxx=llvm@19\"\n when: \"%cxx\"\n - spec: \"%mpi=openmpi@5\"\n when: \"%mpi\"\n\nThe general pattern in toolchains configuration is to use a ``when`` condition that specifies a direct dependency on a *virtual* package, and a ``spec`` that :ref:`requires a specific provider for that virtual `.\n\nNotice that it's possible to achieve similar configuration with :doc:`packages.yaml `:\n\n.. code-block:: yaml\n :caption: ~/.spack/packages.yaml\n\n packages:\n c:\n require: [llvm@19]\n cxx:\n require: [llvm@19]\n mpi:\n require: [openmpi@5]\n\nThe difference is that the toolchain can be applied **locally** in a spec, while the ``packages.yaml`` configuration is always global.\nThis makes toolchains particularly useful in Spack environments.\n\nToolchains in Spack environments\n--------------------------------\n\nToolchains can be used to simplify the construction of a list of specs for Spack environments using :ref:`spec matrices `, when the list includes packages with different language requirements:\n\n.. code-block:: yaml\n :caption: spack.yaml\n\n spack:\n specs:\n - matrix:\n - [kokkos, hdf5~cxx+fortran, py-scipy]\n - [\"%llvm_gfortran\"]\n\nNote that in this case, we can use a single matrix, and the user doesn't need to know exactly which package requires which language.\nWithout toolchains, it would be difficult to enforce compilers directly, because:\n\n* ``kokkos`` depends on C and C++, but not Fortran\n* ``hdf5~cxx+fortran`` depends on C and Fortran, but not C++\n* ``py-scipy`` depends on C, C++, and Fortran\n\n.. note::\n\n Toolchains are currently limited to using only direct dependencies (``%``) in their definition.\n Transitive dependencies are not allowed.\n" }, { "path": "lib/spack/docs/windows.rst", "content": "..\n Copyright Spack Project Developers. See COPYRIGHT file for details.\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n.. meta::\n :description lang=en:\n A guide to setting up and using Spack on Windows, including installing prerequisites and configuring the environment.\n\n.. _windows_support:\n\nSpack On Windows\n================\n\nWindows support for Spack is currently under development.\nWhile this work is still in an early stage, it is currently possible to set up Spack and perform a few operations on Windows.\nThis section will guide you through the steps needed to install Spack and start running it on a fresh Windows machine.\n\nStep 1: Install prerequisites\n-----------------------------\n\nTo use Spack on Windows, you will need the following packages.\n\nRequired:\n\n* Microsoft Visual Studio\n* Python\n* Git\n* 7z\n\nOptional:\n\n* Intel Fortran (needed for some packages)\n\n.. note::\n\n Currently MSVC is the only compiler tested for C/C++ projects.\n Intel OneAPI provides Fortran support.\n\nMicrosoft Visual Studio\n^^^^^^^^^^^^^^^^^^^^^^^\n\nMicrosoft Visual Studio provides the only Windows C/C++ compiler that is currently supported by Spack.\nSpack additionally requires that the Windows SDK (including WGL) to be installed as part of your Visual Studio installation as it is required to build many packages from source.\n\nWe require several specific components to be included in the Visual Studio installation.\nOne is the C/C++ toolset, which can be selected as \"Desktop development with C++\" or \"C++ build tools,\" depending on installation type (Professional, Build Tools, etc.)\nThe other required component is \"C++ CMake tools for Windows,\" which can be selected from among the optional packages.\nThis provides CMake and Ninja for use during Spack configuration.\n\n\nIf you already have Visual Studio installed, you can make sure these components are installed by rerunning the installer.\nNext to your installation, select \"Modify\" and look at the \"Installation details\" pane on the right.\n\nIntel Fortran\n^^^^^^^^^^^^^\n\nFor Fortran-based packages on Windows, we strongly recommend Intel's oneAPI Fortran compilers.\nThe suite is free to download from Intel's website, located at https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/fortran-compiler.html.\nThe executable of choice for Spack will be Intel's Beta Compiler, ifx, which supports the classic compiler's (ifort's) frontend and runtime libraries by using LLVM.\n\nPython\n^^^^^^\n\nAs Spack is a Python-based package, an installation of Python will be needed to run it.\nPython 3 can be downloaded and installed from the Windows Store, and will be automatically added to your ``PATH`` in this case.\n\n.. note::\n Spack currently supports Python versions later than 3.2 inclusive.\n\nGit\n^^^\n\nA bash console and GUI can be downloaded from https://git-scm.com/downloads.\nIf you are unfamiliar with Git, there are a myriad of resources online to help guide you through checking out repositories and switching development branches.\n\nWhen given the option of adjusting your ``PATH``, choose the ``Git from the command line and also from 3rd-party software`` option.\nThis will automatically update your ``PATH`` variable to include the ``git`` command.\n\nSpack support on Windows is currently dependent on installing the Git for Windows project as the project providing Git support on Windows.\nThis is additionally the recommended method for installing Git on Windows, a link to which can be found above.\nSpack requires the utilities vendored by this project.\n\n7zip\n^^^^\n\nA tool for extracting ``.xz`` files is required for extracting source tarballs.\nThe latest 7-Zip can be located at https://sourceforge.net/projects/sevenzip/.\n\nStep 2: Install and setup Spack\n-------------------------------\n\nWe are now ready to get the Spack environment set up on our machine.\nWe begin by using Git to clone the Spack repo, hosted at https://github.com/spack/spack.git into a desired directory, for our purposes today, called ``spack_install``.\n\nIn order to install Spack with Windows support, run the following one-liner in a Windows CMD prompt.\n\n.. code-block:: console\n\n $ git clone https://github.com/spack/spack.git\n\n.. note::\n If you chose to install Spack into a directory on Windows that is set up to require Administrative Privileges, Spack will require elevated privileges to run.\n Administrative Privileges can be denoted either by default, such as ``C:\\Program Files``, or administrator-applied administrative restrictions on a directory that Spack installs files to such as ``C:\\Users``\n\nStep 3: Run and configure Spack\n-------------------------------\n\nOn Windows, Spack supports both primary native shells, PowerShell and the traditional command prompt.\nTo use Spack, pick your favorite shell, and run ``bin\\spack_cmd.bat`` or ``share/spack/setup-env.ps1`` (you may need to Run as Administrator) from the top-level Spack directory.\nThis will provide a Spack-enabled shell.\nIf you receive a warning message that Python is not in your ``PATH`` (which may happen if you installed Python from the website and not the Windows Store), add the location of the Python executable to your ``PATH`` now.\nYou can permanently add Python to your ``PATH`` variable by using the ``Edit the system environment variables`` utility in Windows Control Panel.\n\nTo configure Spack, first run the following command inside the Spack console:\n\n.. code-block:: console\n\n $ spack compiler find\n\nThis creates a ``.staging`` directory in our Spack prefix, along with a ``windows`` subdirectory containing a ``packages.yaml`` file.\nOn a fresh Windows installation with the above packages installed, this command should only detect Microsoft Visual Studio and the Intel Fortran compiler will be integrated within the first version of MSVC present in the ``packages.yaml`` output.\n\nSpack provides a default ``config.yaml`` file for Windows that it will use unless overridden.\nThis file is located at ``etc\\spack\\defaults\\windows\\config.yaml``.\nYou can read more on how to do this and write your own configuration files in the :ref:`Configuration Files` section of our documentation.\nIf you do this, pay particular attention to the ``build_stage`` block of the file as this specifies the directory that will temporarily hold the source code for the packages to be installed.\nThis path name must be sufficiently short for compliance with CMD, otherwise you will see build errors during installation (particularly with CMake) tied to long path names.\n\nTo allow Spack's use of external tools and dependencies already on your system, the external pieces of software must be described in the ``packages.yaml`` file.\nThere are two methods to populate this file:\n\nThe first and easiest choice is to use Spack to find installations on your system.\nIn the Spack terminal, run the following commands:\n\n.. code-block:: console\n\n $ spack external find cmake\n $ spack external find ninja\n\nThe ``spack external find `` will find executables on your system with the same name given.\nThe command will store the items found in ``packages.yaml`` in the ``.staging\\`` directory.\n\nAssuming that the command found CMake and Ninja executables in the previous step, continue to Step 4.\nIf no executables were found, we may need to manually direct Spack towards the CMake and Ninja installations we set up with Visual Studio.\nTherefore, your ``packages.yaml`` file will look something like this, possibly with slight variations in the paths to CMake and Ninja:\n\n.. code-block:: yaml\n\n packages:\n cmake:\n externals:\n - spec: cmake@3.19\n prefix: 'c:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Professional\\Common7\\IDE\\CommonExtensions\\Microsoft\\CMake\\CMake'\n buildable: false\n ninja:\n externals:\n - spec: ninja@1.8.2\n prefix: 'c:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Professional\\Common7\\IDE\\CommonExtensions\\Microsoft\\CMake\\Ninja'\n buildable: false\n\nYou can also use a separate installation of CMake if you have one and prefer to use it.\nIf you don't have a path to Ninja analogous to the above, then you can obtain it by running the Visual Studio Installer and following the instructions at the start of this section.\nAlso note that YAML files use spaces for indentation and not tabs, so ensure that this is the case when editing one directly.\n\n\n.. note::\n The use of Cygwin is not officially supported by Spack and is not tested.\n However, Spack will not prevent this, so if choosing to use Spack with Cygwin, know that no functionality is guaranteed.\n\nStep 4: Use Spack\n-----------------\n\nOnce the configuration is complete, it is time to give the installation a test.\nInstall a basic package through the Spack console via:\n\n.. code-block:: spec\n\n $ spack install cpuinfo\n\nIf in the previous step, you did not have CMake or Ninja installed, running the command above should install both packages.\n\n.. note::\n Windows has a few idiosyncrasies when it comes to the Spack spec syntax and the use of certain shells See the Spack spec syntax doc for more information\n\n\nFor developers\n--------------\n\nThe intent is to provide a Windows installer that will automatically set up Python, Git, and Spack, instead of requiring the user to do so manually.\nInstructions for creating the installer are at https://github.com/spack/spack/blob/develop/lib/spack/spack/cmd/installer/README.md\n" }, { "path": "lib/spack/llnl/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport warnings\n\nimport spack.error\nimport spack.llnl\n\nwarnings.warn(\n \"The `llnl` module will be removed in Spack v1.1\",\n category=spack.error.SpackAPIWarning,\n stacklevel=2,\n)\n\n\n__path__ = spack.llnl.__path__\n" }, { "path": "lib/spack/spack/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport functools\nimport os\nimport re\nfrom typing import Optional\n\nimport spack.paths\nimport spack.util.git\n\n#: PEP440 canonical ... string\n__version__ = \"1.2.0.dev0\"\nspack_version = __version__\n\n#: The current Package API version implemented by this version of Spack. The Package API defines\n#: the Python interface for packages as well as the layout of package repositories. The minor\n#: version is incremented when the package API is extended in a backwards-compatible way. The major\n#: version is incremented upon breaking changes. This version is changed independently from the\n#: Spack version.\npackage_api_version = (2, 4)\n\n#: The minimum Package API version that this version of Spack is compatible with. This should\n#: always be a tuple of the form ``(major, 0)``, since compatibility with vX.Y implies\n#: compatibility with vX.0.\nmin_package_api_version = (1, 0)\n\n\ndef __try_int(v):\n try:\n return int(v)\n except ValueError:\n return v\n\n\n#: (major, minor, micro, dev release) tuple\nspack_version_info = tuple([__try_int(v) for v in __version__.split(\".\")])\n\n\n@functools.lru_cache(maxsize=None)\ndef get_spack_commit() -> Optional[str]:\n \"\"\"Get the Spack git commit sha.\n\n Returns:\n (str or None) the commit sha if available, otherwise None\n \"\"\"\n git_path = os.path.join(spack.paths.prefix, \".git\")\n if not os.path.exists(git_path):\n return None\n\n git = spack.util.git.git()\n if not git:\n return None\n\n rev = git(\n \"-C\",\n spack.paths.prefix,\n \"rev-parse\",\n \"HEAD\",\n output=str,\n error=os.devnull,\n fail_on_error=False,\n )\n if git.returncode != 0:\n return None\n\n match = re.match(r\"[a-f\\d]{7,}$\", rev)\n return match.group(0) if match else None\n\n\ndef get_version() -> str:\n \"\"\"Get a descriptive version of this instance of Spack.\n\n Outputs ``\" ()\"``.\n\n The commit sha is only added when available.\n \"\"\"\n commit = get_spack_commit()\n if commit:\n return f\"{spack_version} ({commit})\"\n return spack_version\n\n\ndef get_short_version() -> str:\n \"\"\"Short Spack version.\"\"\"\n return f\"{spack_version_info[0]}.{spack_version_info[1]}\"\n\n\n__all__ = [\n \"spack_version_info\",\n \"spack_version\",\n \"get_version\",\n \"get_spack_commit\",\n \"get_short_version\",\n \"package_api_version\",\n \"min_package_api_version\",\n]\n" }, { "path": "lib/spack/spack/aliases.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Alias names to convert legacy compilers to builtin packages and vice-versa\"\"\"\n\nBUILTIN_TO_LEGACY_COMPILER = {\n \"llvm\": \"clang\",\n \"intel-oneapi-compilers\": \"oneapi\",\n \"llvm-amdgpu\": \"rocmcc\",\n \"intel-oneapi-compilers-classic\": \"intel\",\n \"acfl\": \"arm\",\n}\n\nLEGACY_COMPILER_TO_BUILTIN = {\n \"clang\": \"llvm\",\n \"oneapi\": \"intel-oneapi-compilers\",\n \"rocmcc\": \"llvm-amdgpu\",\n \"intel\": \"intel-oneapi-compilers-classic\",\n \"arm\": \"acfl\",\n}\n" }, { "path": "lib/spack/spack/archspec.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Adapter for the archspec library.\"\"\"\n\nimport spack.vendor.archspec.cpu\n\nimport spack.spec\n\n\ndef microarchitecture_flags(spec: spack.spec.Spec, language: str) -> str:\n \"\"\"Get compiler flags for the spec's microarchitecture.\n\n Args:\n spec: The spec defining the target microarchitecture and compiler.\n language: The language (``\"c\"``, ``\"cxx\"``, ``\"fortran\"``) used to select the appropriate\n compiler from the spec.\n\n Example::\n\n >>> spec.format(\"{target}\")\n 'm1'\n >>> spec[\"c\"].format(\"{name}{@version}\")\n 'apple-clang@17.0.0'\n >>> microarchitecture_flags(spec, language=\"c\")\n '-mcpu=apple-m1'\n \"\"\"\n target = spec.target\n\n if not spec.has_virtual_dependency(language):\n raise ValueError(f\"The spec {spec.name} does not depend on {language}\")\n elif target is None:\n raise ValueError(f\"The spec {spec.name} does not have a target defined\")\n\n compiler = spec.dependencies(virtuals=language)[0]\n\n return microarchitecture_flags_from_target(target, compiler)\n\n\ndef microarchitecture_flags_from_target(\n target: spack.vendor.archspec.cpu.Microarchitecture, compiler: spack.spec.Spec\n) -> str:\n \"\"\"Get compiler flags for the spec's microarchitecture. Similar to\n :func:`microarchitecture_flags`, but takes a ``target`` and ``compiler`` directly instead of a\n spec.\n\n Args:\n target: The target microarchitecture.\n compiler: The spec defining the compiler.\n \"\"\"\n # Try to check if the current compiler comes with a version number or has an unexpected suffix.\n # If so, treat it as a compiler with a custom spec.\n version_number, _ = spack.vendor.archspec.cpu.version_components(\n compiler.version.dotted_numeric_string\n )\n try:\n return target.optimization_flags(compiler.package.archspec_name(), version_number)\n except ValueError:\n return \"\"\n\n\n#: The host target family, like x86_64 or aarch64\nHOST_TARGET_FAMILY = spack.vendor.archspec.cpu.host().family\n" }, { "path": "lib/spack/spack/audit.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Classes and functions to register audit checks for various parts of\nSpack and run them on-demand.\n\nTo register a new class of sanity checks (e.g. sanity checks for\ncompilers.yaml), the first action required is to create a new AuditClass\nobject:\n\n.. code-block:: python\n\n audit_cfgcmp = AuditClass(\n tag=\"CFG-COMPILER\",\n description=\"Sanity checks on compilers.yaml\",\n kwargs=()\n )\n\nThis object is to be used as a decorator to register functions\nthat will perform each a single check:\n\n.. code-block:: python\n\n @audit_cfgcmp\n def _search_duplicate_compilers(error_cls):\n pass\n\nThese functions need to take as argument the keywords declared when\ncreating the decorator object plus an ``error_cls`` argument at the\nend, acting as a factory to create Error objects. It should return a\n(possibly empty) list of errors.\n\nCalls to each of these functions are triggered by the ``run`` method of\nthe decorator object, that will forward the keyword arguments passed\nas input.\n\"\"\"\n\nimport ast\nimport collections\nimport collections.abc\nimport glob\nimport inspect\nimport io\nimport itertools\nimport os\nimport pathlib\nimport pickle\nimport re\nimport warnings\nfrom typing import Iterable, List, Set, Tuple\nfrom urllib.request import urlopen\n\nimport spack.builder\nimport spack.config\nimport spack.enums\nimport spack.fetch_strategy\nimport spack.llnl.util.lang\nimport spack.patch\nimport spack.repo\nimport spack.spec\nimport spack.util.crypto\nimport spack.util.spack_yaml as syaml\nimport spack.variant\nfrom spack.llnl.string import plural\n\n#: Map an audit tag to a list of callables implementing checks\nCALLBACKS = {}\n\n#: Map a group of checks to the list of related audit tags\nGROUPS = collections.defaultdict(list)\n\n\nclass Error:\n \"\"\"Information on an error reported in a test.\"\"\"\n\n def __init__(self, summary, details):\n self.summary = summary\n self.details = tuple(details)\n\n def __str__(self):\n if self.details:\n return f\"{self.summary}\\n\" + \"\\n\".join(f\" {detail}\" for detail in self.details)\n return self.summary\n\n def __eq__(self, other):\n if self.summary != other.summary or self.details != other.details:\n return False\n return True\n\n def __hash__(self):\n value = (self.summary, self.details)\n return hash(value)\n\n\nclass AuditClass(collections.abc.Sequence):\n def __init__(self, group, tag, description, kwargs):\n \"\"\"Return an object that acts as a decorator to register functions\n associated with a specific class of sanity checks.\n\n Args:\n group (str): group in which this check is to be inserted\n tag (str): tag uniquely identifying the class of sanity checks\n description (str): description of the sanity checks performed\n by this tag\n kwargs (tuple of str): keyword arguments that each registered\n function needs to accept\n \"\"\"\n if tag in CALLBACKS:\n msg = 'audit class \"{0}\" already registered'\n raise ValueError(msg.format(tag))\n\n self.group = group\n self.tag = tag\n self.description = description\n self.kwargs = kwargs\n self.callbacks = []\n\n # Init the list of hooks\n CALLBACKS[self.tag] = self\n\n # Update the list of tags in the group\n GROUPS[self.group].append(self.tag)\n\n def __call__(self, func):\n self.callbacks.append(func)\n\n def __getitem__(self, item):\n return self.callbacks[item]\n\n def __len__(self):\n return len(self.callbacks)\n\n def run(self, **kwargs):\n msg = 'please pass \"{0}\" as keyword arguments'\n msg = msg.format(\", \".join(self.kwargs))\n assert set(self.kwargs) == set(kwargs), msg\n\n errors = []\n kwargs[\"error_cls\"] = Error\n for fn in self.callbacks:\n errors.extend(fn(**kwargs))\n\n return errors\n\n\ndef run_group(group, **kwargs):\n \"\"\"Run the checks that are part of the group passed as argument.\n\n Args:\n group (str): group of checks to be run\n **kwargs: keyword arguments forwarded to the checks\n\n Returns:\n List of (tag, errors) that failed.\n \"\"\"\n reports = []\n for check in GROUPS[group]:\n errors = run_check(check, **kwargs)\n reports.append((check, errors))\n return reports\n\n\ndef run_check(tag, **kwargs):\n \"\"\"Run the checks associated with a single tag.\n\n Args:\n tag (str): tag of the check\n **kwargs: keyword arguments forwarded to the checks\n\n Returns:\n Errors occurred during the checks\n \"\"\"\n return CALLBACKS[tag].run(**kwargs)\n\n\n# TODO: For the generic check to be useful for end users,\n# TODO: we need to implement hooks like described in\n# TODO: https://github.com/spack/spack/pull/23053/files#r630265011\n#: Generic checks relying on global state\ngeneric = AuditClass(\n group=\"generic\",\n tag=\"GENERIC\",\n description=\"Generic checks relying on global variables\",\n kwargs=(),\n)\n\n\n#: Sanity checks on compilers.yaml\nconfig_compiler = AuditClass(\n group=\"configs\", tag=\"CFG-COMPILER\", description=\"Sanity checks on compilers.yaml\", kwargs=()\n)\n\n\n@config_compiler\ndef _search_duplicate_compilers(error_cls):\n \"\"\"Report compilers with the same spec and two different definitions\"\"\"\n errors = []\n\n compilers = list(sorted(spack.config.get(\"compilers\"), key=lambda x: x[\"compiler\"][\"spec\"]))\n for spec, group in itertools.groupby(compilers, key=lambda x: x[\"compiler\"][\"spec\"]):\n group = list(group)\n if len(group) == 1:\n continue\n\n error_msg = \"Compiler defined multiple times: {0}\"\n try:\n details = [str(x._start_mark).strip() for x in group]\n except Exception:\n details = []\n errors.append(error_cls(summary=error_msg.format(spec), details=details))\n\n return errors\n\n\n#: Sanity checks on packages.yaml\nconfig_packages = AuditClass(\n group=\"configs\", tag=\"CFG-PACKAGES\", description=\"Sanity checks on packages.yaml\", kwargs=()\n)\n\n#: Sanity checks on packages.yaml\nconfig_repos = AuditClass(\n group=\"configs\", tag=\"CFG-REPOS\", description=\"Sanity checks on repositories\", kwargs=()\n)\n\n\n@config_packages\ndef _search_duplicate_specs_in_externals(error_cls):\n \"\"\"Search for duplicate specs declared as externals\"\"\"\n errors, externals = [], collections.defaultdict(list)\n packages_yaml = spack.config.get(\"packages\")\n\n for name, pkg_config in packages_yaml.items():\n # No externals can be declared under all\n if name == \"all\" or \"externals\" not in pkg_config:\n continue\n\n current_externals = pkg_config[\"externals\"]\n for entry in current_externals:\n # Ask for the string representation of the spec to normalize\n # aspects of the spec that may be represented in multiple ways\n # e.g. +foo or foo=true\n key = str(spack.spec.Spec(entry[\"spec\"]))\n externals[key].append(entry)\n\n for spec, entries in sorted(externals.items()):\n # If there's a single external for a spec we are fine\n if len(entries) < 2:\n continue\n\n # Otherwise wwe need to report an error\n error_msg = \"Multiple externals share the same spec: {0}\".format(spec)\n try:\n lines = [str(x._start_mark).strip() for x in entries]\n details = (\n [\"Please remove all but one of the following entries:\"]\n + lines\n + [\"as they might result in non-deterministic hashes\"]\n )\n except (TypeError, AttributeError):\n details = []\n\n errors.append(error_cls(summary=error_msg, details=details))\n\n return errors\n\n\n@config_packages\ndef _avoid_mismatched_variants(error_cls):\n \"\"\"Warns if variant preferences have mismatched types or names.\"\"\"\n errors = []\n packages_yaml = spack.config.CONFIG.get_config(\"packages\")\n\n for pkg_name in packages_yaml:\n # 'all:' must be more forgiving, since it is setting defaults for everything\n if pkg_name == \"all\" or \"variants\" not in packages_yaml[pkg_name]:\n continue\n\n preferences = packages_yaml[pkg_name][\"variants\"]\n if not isinstance(preferences, list):\n preferences = [preferences]\n\n for variants in preferences:\n current_spec = spack.spec.Spec(variants)\n pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)\n for variant in current_spec.variants.values():\n # Variant does not exist at all\n if variant.name not in pkg_cls.variant_names():\n summary = (\n f\"Setting a preference for the '{pkg_name}' package to the \"\n f\"non-existing variant '{variant.name}'\"\n )\n errors.append(_make_config_error(preferences, summary, error_cls=error_cls))\n continue\n\n # Variant cannot accept this value\n try:\n spack.variant.prevalidate_variant_value(pkg_cls, variant, strict=True)\n except Exception:\n summary = (\n f\"Setting the variant '{variant.name}' of the '{pkg_name}' package \"\n f\"to the invalid value '{str(variant)}'\"\n )\n errors.append(_make_config_error(preferences, summary, error_cls=error_cls))\n\n return errors\n\n\n@config_packages\ndef _wrongly_named_spec(error_cls):\n \"\"\"Warns if the wrong name is used for an external spec\"\"\"\n errors = []\n packages_yaml = spack.config.CONFIG.get_config(\"packages\")\n for pkg_name in packages_yaml:\n if pkg_name == \"all\":\n continue\n\n externals = packages_yaml[pkg_name].get(\"externals\", [])\n is_virtual = spack.repo.PATH.is_virtual(pkg_name)\n for entry in externals:\n spec = spack.spec.Spec(entry[\"spec\"])\n regular_pkg_is_wrong = not is_virtual and pkg_name != spec.name\n virtual_pkg_is_wrong = is_virtual and not any(\n p.name == spec.name for p in spack.repo.PATH.providers_for(pkg_name)\n )\n if regular_pkg_is_wrong or virtual_pkg_is_wrong:\n summary = f\"Wrong external spec detected for '{pkg_name}': {spec}\"\n errors.append(_make_config_error(entry, summary, error_cls=error_cls))\n return errors\n\n\n@config_packages\ndef _ensure_all_virtual_packages_have_default_providers(error_cls):\n \"\"\"All virtual packages must have a default provider explicitly set.\"\"\"\n configuration = spack.config.create()\n defaults = configuration.get_config(\"packages\", _merged_scope=\"defaults\")\n default_providers = defaults[\"all\"][\"providers\"]\n virtuals = spack.repo.PATH.provider_index.providers\n default_providers_filename = configuration.scopes[\"defaults\"].get_section_filename(\"packages\")\n\n return [\n error_cls(f\"'{virtual}' must have a default provider in {default_providers_filename}\", [])\n for virtual in virtuals\n if virtual not in default_providers\n ]\n\n\n@config_repos\ndef _ensure_no_folders_without_package_py(error_cls):\n \"\"\"Check that we don't leave any folder without a package.py in repos\"\"\"\n errors = []\n for repository in spack.repo.PATH.repos:\n missing = []\n for entry in os.scandir(repository.packages_path):\n if not entry.is_dir() or entry.name == \"__pycache__\":\n continue\n package_py = pathlib.Path(entry.path) / spack.repo.package_file_name\n if not package_py.exists():\n missing.append(entry.path)\n if missing:\n summary = (\n f\"The '{repository.namespace}' repository misses a package.py file\"\n f\" in the following folders\"\n )\n errors.append(error_cls(summary=summary, details=[f\"{x}\" for x in missing]))\n return errors\n\n\ndef _make_config_error(config_data, summary, error_cls):\n s = io.StringIO()\n s.write(\"Occurring in the following file:\\n\")\n syaml.dump_config(config_data, stream=s, blame=True)\n return error_cls(summary=summary, details=[s.getvalue()])\n\n\n#: Sanity checks on package directives\npackage_directives = AuditClass(\n group=\"packages\",\n tag=\"PKG-DIRECTIVES\",\n description=\"Sanity checks on specs used in directives\",\n kwargs=(\"pkgs\",),\n)\n\npackage_attributes = AuditClass(\n group=\"packages\",\n tag=\"PKG-ATTRIBUTES\",\n description=\"Sanity checks on reserved attributes of packages\",\n kwargs=(\"pkgs\",),\n)\n\npackage_properties = AuditClass(\n group=\"packages\",\n tag=\"PKG-PROPERTIES\",\n description=\"Sanity checks on properties a package should maintain\",\n kwargs=(\"pkgs\",),\n)\n\n\n#: Sanity checks on linting\n# This can take some time, so it's run separately from packages\npackage_https_directives = AuditClass(\n group=\"packages-https\",\n tag=\"PKG-HTTPS-DIRECTIVES\",\n description=\"Sanity checks on https checks of package urls, etc.\",\n kwargs=(\"pkgs\",),\n)\n\n\n@package_properties\ndef _check_build_test_callbacks(pkgs, error_cls):\n \"\"\"Ensure stand-alone test methods are not included in build-time callbacks.\n\n Test methods are for checking the installed software as stand-alone tests.\n They could also be called during the post-install phase of a build.\n \"\"\"\n errors = []\n for pkg_name in pkgs:\n pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)\n test_callbacks = getattr(pkg_cls, \"build_time_test_callbacks\", None)\n\n has_test_method = test_callbacks and any([m.startswith(\"test_\") for m in test_callbacks])\n if has_test_method:\n msg = f\"Package {pkg_name} includes stand-alone test methods in build-time checks.\"\n callbacks = \", \".join(test_callbacks)\n instr = f\"Remove the following from 'build_time_test_callbacks': {callbacks}\"\n errors.append(error_cls(msg.format(pkg_name), [instr]))\n\n return errors\n\n\n@package_directives\ndef _directives_can_be_evaluated(pkgs, error_cls):\n \"\"\"Ensure that all directives in a package can be evaluated.\"\"\"\n errors = []\n for pkg_name in pkgs:\n pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)\n for attr in pkg_cls._dict_to_directives:\n try:\n getattr(pkg_cls, attr)\n except Exception as e:\n error_msg = f\"Package '{pkg_name}' has invalid directive '{attr}'\"\n details = [str(e)]\n errors.append(error_cls(error_msg, details))\n return errors\n\n\n@package_directives\ndef _check_patch_urls(pkgs, error_cls):\n \"\"\"Ensure that patches fetched from GitHub and GitLab have stable sha256\n hashes.\"\"\"\n github_patch_url_re = (\n r\"^https?://(?:patch-diff\\.)?github(?:usercontent)?\\.com/\"\n r\".+/.+/(?:commit|pull)/[a-fA-F0-9]+\\.(?:patch|diff)\"\n )\n github_pull_commits_re = (\n r\"^https?://(?:patch-diff\\.)?github(?:usercontent)?\\.com/\"\n r\".+/.+/pull/\\d+/commits/[a-fA-F0-9]+\\.(?:patch|diff)\"\n )\n # Only .diff URLs have stable/full hashes:\n # https://forum.gitlab.com/t/patches-with-full-index/29313\n gitlab_patch_url_re = (\n r\"^https?://(?:.+)?gitlab(?:.+)/\"\n r\".+/.+/-/(?:commit|merge_requests)/[a-fA-F0-9]+\\.(?:patch|diff)\"\n )\n\n errors = []\n for pkg_name in pkgs:\n pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)\n for condition, patches in pkg_cls.patches.items():\n for patch in patches:\n if not isinstance(patch, spack.patch.UrlPatch):\n continue\n\n if re.match(github_pull_commits_re, patch.url):\n url = re.sub(r\"/pull/\\d+/commits/\", r\"/commit/\", patch.url)\n url = re.sub(r\"^(.*)(? None:\n if not isinstance(node, self.path[self.depth]):\n return\n self.depth += 1\n if self.depth == len(self.path):\n self.in_function = True\n super().generic_visit(node)\n if self.depth == len(self.path):\n self.in_function = False\n self.locals.clear()\n self.depth -= 1\n\n def generic_visit(self, node: ast.AST) -> None:\n # Recurse into function definitions\n if self.depth < len(self.path):\n return self.descend_in_function_def(node)\n elif not self.in_function:\n return\n elif isinstance(node, ast.Global):\n for name in node.names:\n if name in self.magic_globals:\n self.references_to_globals.append((name, node.lineno))\n elif isinstance(node, ast.Assign):\n # visit the rhs before lhs\n super().visit(node.value)\n for target in node.targets:\n super().visit(target)\n elif isinstance(node, ast.Name) and node.id in self.magic_globals:\n if isinstance(node.ctx, ast.Load) and node.id not in self.locals:\n self.references_to_globals.append((node.id, node.lineno))\n elif isinstance(node.ctx, ast.Store):\n self.locals.add(node.id)\n else:\n super().generic_visit(node)\n\n\n@package_properties\ndef _uses_deprecated_globals(pkgs, error_cls):\n \"\"\"Ensure that packages do not use deprecated globals\"\"\"\n errors = []\n\n for pkg_name in pkgs:\n # some packages scheduled to be removed in v0.23 are not worth fixing.\n pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)\n if all(v.get(\"deprecated\", False) for v in pkg_cls.versions.values()):\n continue\n\n file = spack.repo.PATH.filename_for_package_name(pkg_name)\n tree = ast.parse(open(file, \"rb\").read())\n visitor = DeprecatedMagicGlobals((\"std_cmake_args\", \"std_meson_args\", \"std_pip_args\"))\n visitor.visit(tree)\n if visitor.references_to_globals:\n errors.append(\n error_cls(\n f\"Package '{pkg_name}' uses deprecated globals\",\n [\n f\"{file}:{line} references '{name}'\"\n for name, line in visitor.references_to_globals\n ],\n )\n )\n\n return errors\n\n\n@package_properties\ndef _ensure_test_docstring(pkgs, error_cls):\n \"\"\"Ensure stand-alone test methods have a docstring.\n\n The docstring of a test method is implicitly used as the description of\n the corresponding test part during test results reporting.\n \"\"\"\n doc_regex = r'\\s+(\"\"\"[^\"]+\"\"\")'\n\n errors = []\n for pkg_name in pkgs:\n pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)\n methods = inspect.getmembers(pkg_cls, predicate=lambda x: inspect.isfunction(x))\n method_names = []\n for name, test_fn in methods:\n if not name.startswith(\"test_\"):\n continue\n\n # Ensure the test method has a docstring\n source = inspect.getsource(test_fn)\n match = re.search(doc_regex, source)\n if match is None or len(match.group(0).replace('\"', \"\").strip()) == 0:\n method_names.append(name)\n\n num_methods = len(method_names)\n if num_methods > 0:\n methods = plural(num_methods, \"method\", show_n=False)\n docstrings = plural(num_methods, \"docstring\", show_n=False)\n msg = f\"Package {pkg_name} has test {methods} with empty or missing {docstrings}.\"\n names = \", \".join(method_names)\n instr = [\n \"Docstrings are used as descriptions in test outputs.\",\n f\"Add a concise summary to the following {methods} in '{pkg_cls.__module__}':\",\n f\"{names}\",\n ]\n errors.append(error_cls(msg, instr))\n\n return errors\n\n\n@package_properties\ndef _ensure_test_implemented(pkgs, error_cls):\n \"\"\"Ensure stand-alone test methods are implemented.\n\n The test method is also required to be non-empty.\n \"\"\"\n\n def skip(line):\n ln = line.strip()\n return ln.startswith(\"#\") or \"pass\" in ln\n\n doc_regex = r'\\s+(\"\"\"[^\"]+\"\"\")'\n\n errors = []\n for pkg_name in pkgs:\n pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)\n methods = inspect.getmembers(pkg_cls, predicate=lambda x: inspect.isfunction(x))\n method_names = []\n for name, test_fn in methods:\n if not name.startswith(\"test_\"):\n continue\n\n source = inspect.getsource(test_fn)\n\n # Attempt to ensure the test method is implemented.\n impl = re.sub(doc_regex, r\"\", source).splitlines()[1:]\n lines = [ln.strip() for ln in impl if not skip(ln)]\n if not lines:\n method_names.append(name)\n\n num_methods = len(method_names)\n if num_methods > 0:\n methods = plural(num_methods, \"method\", show_n=False)\n msg = f\"Package {pkg_name} has empty or missing test {methods}.\"\n names = \", \".join(method_names)\n instr = [\n f\"Implement or remove the following {methods} from '{pkg_cls.__module__}': {names}\"\n ]\n errors.append(error_cls(msg, instr))\n\n return errors\n\n\n@package_https_directives\ndef _linting_package_file(pkgs, error_cls):\n \"\"\"Check for correctness of links\"\"\"\n errors = []\n for pkg_name in pkgs:\n pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)\n\n homepage = pkg_cls.homepage\n if not homepage:\n continue\n\n # Does the homepage have http, and if so, does https work?\n if homepage.startswith(\"http://\"):\n try:\n with urlopen(f\"https://{homepage[7:]}\") as response:\n if response.getcode() == 200:\n msg = 'Package \"{0}\" uses http but has a valid https endpoint.'\n errors.append(msg.format(pkg_cls.name))\n except Exception as e:\n msg = 'Error with attempting https for \"{0}\": '\n errors.append(error_cls(msg.format(pkg_cls.name), [str(e)]))\n continue\n\n return spack.llnl.util.lang.dedupe(errors)\n\n\n@package_directives\ndef _variant_issues_in_directives(pkgs, error_cls):\n \"\"\"Report unknown, wrong, or propagating variants in directives for this package\"\"\"\n errors = []\n for pkg_name in pkgs:\n pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)\n filename = spack.repo.PATH.filename_for_package_name(pkg_name)\n\n # Check the \"conflicts\" directive\n for trigger, conflicts in pkg_cls.conflicts.items():\n errors.extend(\n _issues_in_directive_constraint(\n pkg_cls,\n spack.spec.Spec(trigger),\n directive=\"conflicts\",\n error_cls=error_cls,\n filename=filename,\n requestor=pkg_name,\n )\n )\n for conflict, _ in conflicts:\n errors.extend(\n _issues_in_directive_constraint(\n pkg_cls,\n spack.spec.Spec(conflict),\n directive=\"conflicts\",\n error_cls=error_cls,\n filename=filename,\n requestor=pkg_name,\n )\n )\n\n # Check \"depends_on\" directive\n for trigger, deps_by_name in pkg_cls.dependencies.items():\n vrn = spack.spec.Spec(trigger)\n errors.extend(\n _issues_in_directive_constraint(\n pkg_cls,\n vrn,\n directive=\"depends_on\",\n error_cls=error_cls,\n filename=filename,\n requestor=pkg_name,\n )\n )\n for dep_name, dep in deps_by_name.items():\n if spack.repo.PATH.is_virtual(dep_name):\n continue\n try:\n dep_pkg_cls = spack.repo.PATH.get_pkg_class(dep_name)\n except spack.repo.UnknownPackageError:\n continue\n errors.extend(\n _issues_in_directive_constraint(\n dep_pkg_cls,\n dep.spec,\n directive=\"depends_on\",\n error_cls=error_cls,\n filename=filename,\n requestor=pkg_name,\n )\n )\n\n # Check \"provides\" directive\n for when_spec in pkg_cls.provided:\n errors.extend(\n _issues_in_directive_constraint(\n pkg_cls,\n when_spec,\n directive=\"provides\",\n error_cls=error_cls,\n filename=filename,\n requestor=pkg_name,\n )\n )\n\n # Check \"resource\" directive\n for vrn in pkg_cls.resources:\n errors.extend(\n _issues_in_directive_constraint(\n pkg_cls,\n vrn,\n directive=\"resource\",\n error_cls=error_cls,\n filename=filename,\n requestor=pkg_name,\n )\n )\n\n return spack.llnl.util.lang.dedupe(errors)\n\n\n@package_directives\ndef _issues_in_depends_on_directive(pkgs, error_cls):\n \"\"\"Reports issues with 'depends_on' directives.\n\n Issues might be unknown dependencies, unknown variants or variant values, or declaration\n of nested dependencies.\n \"\"\"\n errors = []\n for pkg_name in pkgs:\n pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)\n filename = spack.repo.PATH.filename_for_package_name(pkg_name)\n\n for when, deps_by_name in pkg_cls.dependencies.items():\n for dep_name, dep in deps_by_name.items():\n\n def check_virtual_with_variants(spec, msg):\n if not spack.repo.PATH.is_virtual(spec.name) or not spec.variants:\n return\n error = error_cls(\n f\"{pkg_name}: {msg}\",\n [f\"remove variants from '{spec}' in depends_on directive in {filename}\"],\n )\n errors.append(error)\n\n check_virtual_with_variants(dep.spec, \"virtual dependency cannot have variants\")\n check_virtual_with_variants(dep.spec, \"virtual when= spec cannot have variants\")\n\n # No need to analyze virtual packages\n if spack.repo.PATH.is_virtual(dep_name):\n continue\n\n # check for unknown dependencies\n try:\n dependency_pkg_cls = spack.repo.PATH.get_pkg_class(dep_name)\n except spack.repo.UnknownPackageError:\n # This dependency is completely missing, so report\n # and continue the analysis\n summary = f\"{pkg_name}: unknown package '{dep_name}' in 'depends_on' directive\"\n details = [f\" in {filename}\"]\n errors.append(error_cls(summary=summary, details=details))\n continue\n\n # Check for self-referential specs similar to:\n #\n # depends_on(\"foo@X.Y\", when=\"^foo+bar\")\n #\n # That would allow clingo to choose whether to have foo@X.Y+bar in the graph.\n problematic_edges = [\n x for x in when.edges_to_dependencies(dep_name) if not x.virtuals\n ]\n if problematic_edges and not dep.patches:\n summary = (\n f\"{pkg_name}: dependency on '{dep.spec}' when '{when}' is self-referential\"\n )\n details = [\n (\n f\" please specify better using '^[virtuals=...] {dep_name}', or \"\n f\"substitute with an equivalent condition on '{pkg_name}'\"\n ),\n f\" in {filename}\",\n ]\n errors.append(error_cls(summary=summary, details=details))\n continue\n\n # check variants\n dependency_variants = dep.spec.variants\n for name, variant in dependency_variants.items():\n try:\n spack.variant.prevalidate_variant_value(\n dependency_pkg_cls, variant, dep.spec, strict=True\n )\n except Exception as e:\n summary = (\n f\"{pkg_name}: wrong variant used for dependency in 'depends_on()'\"\n )\n\n error_msg = str(e)\n if isinstance(e, KeyError):\n error_msg = (\n f\"variant {str(e).strip()} does not exist in package {dep_name}\"\n f\" in package '{dep_name}'\"\n )\n\n errors.append(\n error_cls(summary=summary, details=[error_msg, f\"in {filename}\"])\n )\n\n return errors\n\n\n@package_directives\ndef _ensure_variant_defaults_are_parsable(pkgs, error_cls):\n \"\"\"Ensures that variant defaults are present and parsable from cli\"\"\"\n\n def check_variant(pkg_cls, variant, vname):\n # bool is a subclass of int in python. Permitting a default that is an instance\n # of 'int' means both foo=false and foo=0 are accepted. Other falsish values are\n # not allowed, since they can't be parsed from CLI ('foo=')\n default_is_parsable = isinstance(variant.default, int) or variant.default\n\n if not default_is_parsable:\n msg = f\"Variant '{vname}' of package '{pkg_cls.name}' has an unparsable default value\"\n return [error_cls(msg, [])]\n\n try:\n vspec = variant.make_default()\n except spack.variant.MultipleValuesInExclusiveVariantError:\n msg = f\"Can't create default value for variant '{vname}' in package '{pkg_cls.name}'\"\n return [error_cls(msg, [])]\n\n try:\n variant.validate_or_raise(vspec, pkg_cls.name)\n except spack.variant.InvalidVariantValueError:\n msg = \"Default value of variant '{vname}' in package '{pkg.name}' is invalid\"\n question = \"Is it among the allowed values?\"\n return [error_cls(msg, [question])]\n\n return []\n\n errors = []\n for pkg_name in pkgs:\n pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)\n for vname in pkg_cls.variant_names():\n for _, variant_def in pkg_cls.variant_definitions(vname):\n errors.extend(check_variant(pkg_cls, variant_def, vname))\n return errors\n\n\n@package_directives\ndef _ensure_variants_have_descriptions(pkgs, error_cls):\n \"\"\"Ensures that all variants have a description.\"\"\"\n errors = []\n for pkg_name in pkgs:\n pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)\n for name in pkg_cls.variant_names():\n for when, variant in pkg_cls.variant_definitions(name):\n if not variant.description:\n msg = f\"Variant '{name}' in package '{pkg_name}' is missing a description\"\n errors.append(error_cls(msg, []))\n\n return errors\n\n\n@package_directives\ndef _version_constraints_are_satisfiable_by_some_version_in_repo(pkgs, error_cls):\n \"\"\"Report if version constraints used in directives are not satisfiable\"\"\"\n errors = []\n for pkg_name in pkgs:\n pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)\n filename = spack.repo.PATH.filename_for_package_name(pkg_name)\n\n dependencies_to_check = []\n\n for _, deps_by_name in pkg_cls.dependencies.items():\n for dep_name, dep in deps_by_name.items():\n # Skip virtual dependencies for the time being, check on\n # their versions can be added later\n if spack.repo.PATH.is_virtual(dep_name):\n continue\n\n dependencies_to_check.append(dep.spec)\n\n host_architecture = spack.spec.ArchSpec.default_arch()\n for s in dependencies_to_check:\n dependency_pkg_cls = None\n try:\n dependency_pkg_cls = spack.repo.PATH.get_pkg_class(s.name)\n # Some packages have hacks that might cause failures on some platform\n # Allow to explicitly set conditions to skip version checks in that case\n skip_conditions = getattr(dependency_pkg_cls, \"skip_version_audit\", [])\n skip_version_check = False\n for condition in skip_conditions:\n if host_architecture.satisfies(spack.spec.Spec(condition).architecture):\n skip_version_check = True\n break\n assert skip_version_check or any(\n v.intersects(s.versions) for v in list(dependency_pkg_cls.versions)\n )\n except Exception:\n summary = (\n \"{0}: dependency on {1} cannot be satisfied by known versions of {1.name}\"\n ).format(pkg_name, s)\n details = [\"happening in \" + filename]\n if dependency_pkg_cls is not None:\n details.append(\n \"known versions of {0.name} are {1}\".format(\n s, \", \".join([str(x) for x in dependency_pkg_cls.versions])\n )\n )\n errors.append(error_cls(summary=summary, details=details))\n\n return errors\n\n\ndef _issues_in_directive_constraint(pkg, constraint, *, directive, error_cls, filename, requestor):\n errors = []\n errors.extend(\n _analyze_variants_in_directive(\n pkg,\n constraint,\n directive=directive,\n error_cls=error_cls,\n filename=filename,\n requestor=requestor,\n )\n )\n errors.extend(\n _analize_propagated_deps_in_directive(\n pkg,\n constraint,\n directive=directive,\n error_cls=error_cls,\n filename=filename,\n requestor=requestor,\n )\n )\n return errors\n\n\ndef _analyze_variants_in_directive(pkg, constraint, *, directive, error_cls, filename, requestor):\n errors = []\n variant_names = pkg.variant_names()\n summary = f\"{requestor}: wrong variant in '{directive}' directive\"\n for name, v in constraint.variants.items():\n if name == \"commit\":\n # Automatic variant\n continue\n\n if name not in variant_names:\n msg = f\"variant {name} does not exist in {pkg.name}\"\n errors.append(error_cls(summary=summary, details=[msg, f\"in {filename}\"]))\n continue\n\n if v.propagate:\n propagation_summary = f\"{requestor}: propagating variant in '{directive}' directive\"\n msg = f\"using {constraint} in a directive, which propagates the '{name}' variant\"\n errors.append(error_cls(summary=propagation_summary, details=[msg, f\"in {filename}\"]))\n\n try:\n spack.variant.prevalidate_variant_value(pkg, v, constraint, strict=True)\n except (\n spack.variant.InconsistentValidationError,\n spack.variant.MultipleValuesInExclusiveVariantError,\n spack.variant.InvalidVariantValueError,\n ) as e:\n msg = str(e).strip()\n errors.append(error_cls(summary=summary, details=[msg, f\"in {filename}\"]))\n\n return errors\n\n\ndef _analize_propagated_deps_in_directive(\n pkg, constraint, *, directive, error_cls, filename, requestor\n):\n errors = []\n summary = f\"{requestor}: dependency propagation ('%%') in '{directive}' directive\"\n for edge in constraint.traverse_edges():\n if edge.propagation != spack.enums.PropagationPolicy.NONE:\n msg = f\"'{edge.spec}' contains a propagated dependency\"\n errors.append(error_cls(summary=summary, details=[msg, f\"in {filename}\"]))\n return errors\n\n\n@package_directives\ndef _named_specs_in_when_arguments(pkgs, error_cls):\n \"\"\"Reports named specs in the 'when=' attribute of a directive.\n\n Note that 'conflicts' is the only directive allowing that.\n \"\"\"\n errors = []\n for pkg_name in pkgs:\n pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)\n\n def _refers_to_pkg(when):\n when_spec = spack.spec.Spec(when)\n return not when_spec.name or when_spec.name == pkg_name\n\n def _error_items(when_dict):\n for when, elts in when_dict.items():\n if not _refers_to_pkg(when):\n yield when, elts, [f\"using '{when}', should be '^{when}'\"]\n\n def _extracts_errors(triggers, summary):\n _errors = []\n for trigger in list(triggers):\n if not _refers_to_pkg(trigger):\n details = [f\"using '{trigger}', should be '^{trigger}'\"]\n _errors.append(error_cls(summary=summary, details=details))\n return _errors\n\n for when, dnames, details in _error_items(pkg_cls.dependencies):\n errors.extend(\n error_cls(f\"{pkg_name}: wrong 'when=' condition for '{dname}' dependency\", details)\n for dname in dnames\n )\n\n for when, variants_by_name in pkg_cls.variants.items():\n for vname, variant in variants_by_name.items():\n summary = f\"{pkg_name}: wrong 'when=' condition for the '{vname}' variant\"\n errors.extend(_extracts_errors([when], summary))\n\n for when, providers, details in _error_items(pkg_cls.provided):\n errors.extend(\n error_cls(f\"{pkg_name}: wrong 'when=' condition for '{provided}' virtual\", details)\n for provided in providers\n )\n\n for when, requirements, details in _error_items(pkg_cls.requirements):\n errors.append(\n error_cls(f\"{pkg_name}: wrong 'when=' condition in 'requires' directive\", details)\n )\n\n for when, _, details in _error_items(pkg_cls.patches):\n errors.append(\n error_cls(f\"{pkg_name}: wrong 'when=' condition in 'patch' directives\", details)\n )\n\n for when, _, details in _error_items(pkg_cls.resources):\n errors.append(\n error_cls(f\"{pkg_name}: wrong 'when=' condition in 'resource' directives\", details)\n )\n\n return spack.llnl.util.lang.dedupe(errors)\n\n\n#: Sanity checks on package directives\nexternal_detection = AuditClass(\n group=\"externals\",\n tag=\"PKG-EXTERNALS\",\n description=\"Sanity checks for external software detection\",\n kwargs=(\"pkgs\", \"debug_log\"),\n)\n\n\ndef packages_with_detection_tests():\n \"\"\"Return the list of packages with a corresponding detection_test.yaml file.\"\"\"\n import spack.config\n import spack.util.path\n\n to_be_tested = []\n for current_repo in spack.repo.PATH.repos:\n namespace = current_repo.namespace\n packages_dir = pathlib.PurePath(current_repo.packages_path)\n pattern = packages_dir / \"**\" / \"detection_test.yaml\"\n pkgs_with_tests = [\n f\"{namespace}.{str(pathlib.PurePath(x).parent.name)}\" for x in glob.glob(str(pattern))\n ]\n to_be_tested.extend(pkgs_with_tests)\n\n return to_be_tested\n\n\n@external_detection\ndef _test_detection_by_executable(pkgs, debug_log, error_cls):\n \"\"\"Test drive external detection for packages\"\"\"\n import spack.detection\n\n errors = []\n\n # Filter the packages and retain only the ones with detection tests\n pkgs_with_tests = packages_with_detection_tests()\n selected_pkgs = []\n for current_package in pkgs_with_tests:\n _, unqualified_name = spack.repo.partition_package_name(current_package)\n # Check for both unqualified name and qualified name\n if unqualified_name in pkgs or current_package in pkgs:\n selected_pkgs.append(current_package)\n selected_pkgs.sort()\n\n if not selected_pkgs:\n summary = \"No detection test to run\"\n details = [f' \"{p}\" has no detection test' for p in pkgs]\n warnings.warn(\"\\n\".join([summary] + details))\n return errors\n\n for pkg_name in selected_pkgs:\n for idx, test_runner in enumerate(\n spack.detection.detection_tests(pkg_name, spack.repo.PATH)\n ):\n debug_log(f\"[{__file__}]: running test {idx} for package {pkg_name}\")\n specs = test_runner.execute()\n expected_specs = test_runner.expected_specs\n\n not_detected = set(expected_specs) - set(specs)\n if not_detected:\n summary = pkg_name + \": cannot detect some specs\"\n details = [f'\"{s}\" was not detected [test_id={idx}]' for s in sorted(not_detected)]\n errors.append(error_cls(summary=summary, details=details))\n\n not_expected = set(specs) - set(expected_specs)\n if not_expected:\n summary = pkg_name + \": detected unexpected specs\"\n msg = '\"{0}\" was detected, but was not expected [test_id={1}]'\n details = [msg.format(s, idx) for s in sorted(not_expected)]\n errors.append(error_cls(summary=summary, details=details))\n\n matched_detection = []\n for candidate in expected_specs:\n try:\n idx = specs.index(candidate)\n matched_detection.append((candidate, specs[idx]))\n except (AttributeError, ValueError):\n pass\n\n def _compare_extra_attribute(_expected, _detected, *, _spec):\n result = []\n # If they are string expected is a regex\n if isinstance(_expected, str) and isinstance(_detected, str):\n try:\n _regex = re.compile(_expected)\n except re.error:\n _summary = f'{pkg_name}: illegal regex in \"{_spec}\" extra attributes'\n _details = [f\"{_expected} is not a valid regex\"]\n return [error_cls(summary=_summary, details=_details)]\n\n if not _regex.match(_detected):\n _summary = (\n f'{pkg_name}: error when trying to match \"{_expected}\" '\n f\"in extra attributes\"\n )\n _details = [f\"{_detected} does not match the regex\"]\n return [error_cls(summary=_summary, details=_details)]\n\n elif isinstance(_expected, dict) and isinstance(_detected, dict):\n _not_detected = set(_expected.keys()) - set(_detected.keys())\n if _not_detected:\n _summary = f\"{pkg_name}: cannot detect some attributes for spec {_spec}\"\n _details = [\n f'\"{_expected}\" was expected',\n f'\"{_detected}\" was detected',\n ] + [f'attribute \"{s}\" was not detected' for s in sorted(_not_detected)]\n result.append(error_cls(summary=_summary, details=_details))\n\n _common = set(_expected.keys()) & set(_detected.keys())\n for _key in _common:\n result.extend(\n _compare_extra_attribute(_expected[_key], _detected[_key], _spec=_spec)\n )\n else:\n _summary = f'{pkg_name}: error when trying to detect \"{_expected}\"'\n _details = [f\"{_detected} was detected instead\"]\n return [error_cls(summary=_summary, details=_details)]\n\n return result\n\n for expected, detected in matched_detection:\n # We might not want to test all attributes, so avoid not_expected\n not_detected = set(expected.extra_attributes) - set(detected.extra_attributes)\n if not_detected:\n summary = f\"{pkg_name}: cannot detect some attributes for spec {expected}\"\n details = [\n f'\"{s}\" was not detected [test_id={idx}]' for s in sorted(not_detected)\n ]\n errors.append(error_cls(summary=summary, details=details))\n\n common = set(expected.extra_attributes) & set(detected.extra_attributes)\n for key in common:\n errors.extend(\n _compare_extra_attribute(\n expected.extra_attributes[key],\n detected.extra_attributes[key],\n _spec=expected,\n )\n )\n\n return errors\n" }, { "path": "lib/spack/spack/binary_distribution.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport collections\nimport concurrent.futures\nimport contextlib\nimport copy\nimport datetime\nimport hashlib\nimport io\nimport itertools\nimport json\nimport os\nimport pathlib\nimport re\nimport shutil\nimport sys\nimport tarfile\nimport tempfile\nimport textwrap\nimport time\nimport urllib.error\nimport urllib.parse\nimport urllib.request\nimport warnings\nfrom collections import defaultdict\nfrom contextlib import closing\nfrom typing import IO, Callable, Dict, Iterable, List, Mapping, Optional, Set, Tuple, Union, cast\n\nimport spack.caches\nimport spack.config\nimport spack.database\nimport spack.deptypes as dt\nimport spack.error\nimport spack.hash_types as ht\nimport spack.hooks\nimport spack.hooks.sbang\nimport spack.llnl.util.filesystem as fsys\nimport spack.llnl.util.lang\nimport spack.llnl.util.tty as tty\nimport spack.mirrors.mirror\nimport spack.oci.image\nimport spack.oci.oci\nimport spack.oci.opener\nimport spack.paths\nimport spack.platforms\nimport spack.relocate as relocate\nimport spack.spec\nimport spack.stage\nimport spack.store\nimport spack.user_environment\nimport spack.util.archive\nimport spack.util.crypto\nimport spack.util.file_cache as file_cache\nimport spack.util.gpg\nimport spack.util.parallel\nimport spack.util.path\nimport spack.util.spack_json as sjson\nimport spack.util.spack_yaml as syaml\nimport spack.util.timer as timer\nimport spack.util.url as url_util\nimport spack.util.web as web_util\nfrom spack import traverse\nfrom spack.llnl.util.filesystem import mkdirp\nfrom spack.oci.image import (\n Digest,\n ImageReference,\n default_config,\n default_manifest,\n ensure_valid_tag,\n)\nfrom spack.oci.oci import (\n copy_missing_layers_with_retry,\n get_manifest_and_config_with_retry,\n list_tags,\n upload_blob_with_retry,\n upload_manifest_with_retry,\n)\nfrom spack.package_prefs import get_package_dir_permissions, get_package_group\nfrom spack.relocate_text import utf8_paths_to_single_binary_regex\nfrom spack.stage import Stage\nfrom spack.util.executable import which\n\nfrom .enums import InstallRecordStatus\nfrom .url_buildcache import (\n CURRENT_BUILD_CACHE_LAYOUT_VERSION,\n BlobRecord,\n BuildcacheComponent,\n BuildcacheEntryError,\n BuildcacheManifest,\n InvalidMetadataFile,\n ListMirrorSpecsError,\n MirrorMetadata,\n URLBuildcacheEntry,\n get_entries_from_cache,\n get_url_buildcache_class,\n get_valid_spec_file,\n)\n\n\nclass BuildCacheDatabase(spack.database.Database):\n \"\"\"A database for binary buildcaches.\n\n A database supports writing buildcache index files, in which case certain fields are not\n needed in each install record, and no locking is required. To use this feature, it provides\n ``lock_cfg=NO_LOCK``, and override the list of ``record_fields``.\n \"\"\"\n\n record_fields = (\"spec\", \"ref_count\", \"in_buildcache\")\n\n def __init__(self, root):\n super().__init__(root, lock_cfg=spack.database.NO_LOCK, layout=None)\n self._write_transaction_impl = spack.llnl.util.lang.nullcontext\n self._read_transaction_impl = spack.llnl.util.lang.nullcontext\n\n def _handle_old_db_versions_read(self, check, db, *, reindex: bool):\n if not self.is_readable():\n raise spack.database.DatabaseNotReadableError(\n f\"cannot read buildcache v{self.db_version} at {self.root}\"\n )\n return self._handle_current_version_read(check, db)\n\n\nclass FetchCacheError(Exception):\n \"\"\"Error thrown when fetching the cache failed, usually a composite error list.\"\"\"\n\n def __init__(self, errors):\n if not isinstance(errors, list):\n raise TypeError(\"Expected a list of errors\")\n self.errors = errors\n if len(errors) > 1:\n msg = \" Error {0}: {1}: {2}\"\n self.message = \"Multiple errors during fetching:\\n\"\n self.message += \"\\n\".join(\n (\n msg.format(i + 1, err.__class__.__name__, str(err))\n for (i, err) in enumerate(errors)\n )\n )\n else:\n err = errors[0]\n self.message = \"{0}: {1}\".format(err.__class__.__name__, str(err))\n super().__init__(self.message)\n\n\nclass BinaryCacheIndex:\n \"\"\"\n The BinaryCacheIndex tracks what specs are available on (usually remote)\n binary caches.\n\n This index is \"best effort\", in the sense that whenever we don't find\n what we're looking for here, we will attempt to fetch it directly from\n configured mirrors anyway. Thus, it has the potential to speed things\n up, but cache misses shouldn't break any spack functionality.\n\n At the moment, everything in this class is initialized as lazily as\n possible, so that it avoids slowing anything in spack down until\n absolutely necessary.\n \"\"\"\n\n def __init__(self, cache_root: Optional[str] = None):\n self._index_cache_root: str = cache_root or binary_index_location()\n\n # the key associated with the serialized _local_index_cache\n self._index_contents_key = \"contents.json\"\n\n # a FileCache instance storing copies of remote binary cache indices\n self._index_file_cache: file_cache.FileCache = file_cache.FileCache(self._index_cache_root)\n self._index_file_cache_initialized = False\n\n # stores a map of mirror URL and version layout to index hash and cache key (index path)\n self._local_index_cache: dict[str, dict] = {}\n\n # hashes of remote indices already ingested into the concrete spec\n # cache (_mirrors_for_spec)\n self._specs_already_associated: Set[str] = set()\n\n # mapping from mirror urls to the time.time() of the last index fetch and a bool indicating\n # whether the fetch succeeded or not.\n self._last_fetch_times: Dict[MirrorMetadata, Tuple[float, bool]] = {}\n\n #: Dictionary mapping DAG hashes of specs to Spec objects\n self._known_specs: Dict[str, spack.spec.Spec] = {}\n #: Dictionary mapping DAG hashes of specs to a list of mirrors where they can be found\n self._mirrors_for_spec: Dict[str, Set[MirrorMetadata]] = defaultdict(set)\n\n def _init_local_index_cache(self):\n if not self._index_file_cache_initialized:\n cache_key = self._index_contents_key\n self._local_index_cache = {}\n with self._index_file_cache.read_transaction(cache_key) as cache_file:\n if cache_file is not None:\n self._local_index_cache = json.load(cache_file)\n\n self._index_file_cache_initialized = True\n\n def _write_local_index_cache(self):\n self._init_local_index_cache()\n cache_key = self._index_contents_key\n with self._index_file_cache.write_transaction(cache_key) as (old, new):\n json.dump(self._local_index_cache, new)\n\n def regenerate_spec_cache(self, clear_existing=False):\n \"\"\"Populate the local cache of concrete specs (``_mirrors_for_spec``)\n from the locally cached buildcache index files. This is essentially a\n no-op if it has already been done, as we keep track of the index\n hashes for which we have already associated the built specs.\"\"\"\n self._init_local_index_cache()\n\n if clear_existing:\n self._specs_already_associated = set()\n self._mirrors_for_spec = defaultdict(set)\n self._known_specs = {}\n\n for mirror_metadata in self._local_index_cache:\n cache_entry = self._local_index_cache[mirror_metadata]\n cached_index_path = cache_entry[\"index_path\"]\n cached_index_hash = cache_entry[\"index_hash\"]\n if cached_index_hash not in self._specs_already_associated:\n self._associate_built_specs_with_mirror(\n cached_index_path, MirrorMetadata.from_string(mirror_metadata)\n )\n self._specs_already_associated.add(cached_index_hash)\n\n def _associate_built_specs_with_mirror(self, cache_key, mirror_metadata: MirrorMetadata):\n with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir:\n db = BuildCacheDatabase(tmpdir)\n\n with self._index_file_cache.read_transaction(cache_key) as f:\n if f is not None:\n try:\n db._read_from_stream(f)\n except spack.database.InvalidDatabaseVersionError as e:\n tty.warn(\n \"you need a newer Spack version to read the buildcache index for the \"\n f\"following v{mirror_metadata.version} mirror: \"\n f\"'{mirror_metadata.url}'. {e.database_version_message}\"\n )\n return\n\n spec_list = [\n s\n for s in db.query_local(installed=InstallRecordStatus.ANY)\n # todo, make it easier to get install records associated with specs\n if s.external or db._data[s.dag_hash()].in_buildcache\n ]\n\n for spec in spec_list:\n dag_hash = spec.dag_hash()\n mirrors = self._mirrors_for_spec[dag_hash]\n\n mirrors.add(mirror_metadata.strip_view())\n if dag_hash not in self._known_specs:\n self._known_specs[dag_hash] = spec\n\n def get_all_built_specs(self) -> List[spack.spec.Spec]:\n \"\"\"Returns a list of all concrete specs known to be available in a binary cache.\"\"\"\n return list(self._known_specs.values())\n\n def find_built_spec(self, spec: spack.spec.Spec) -> List[MirrorMetadata]:\n \"\"\"Returns a list of MirrorMetadata objects indicating which mirrors have the given\n concrete spec.\n\n This method does not trigger reading anything from remote mirrors, but rather just checks\n if the concrete spec is found within the cache.\n\n The cache can be updated by calling ``update()`` on the cache.\n\n Args:\n spec: Concrete spec to find\n \"\"\"\n return self.find_by_hash(spec.dag_hash())\n\n def find_by_hash(self, dag_hash: str) -> List[MirrorMetadata]:\n \"\"\"Same as find_built_spec but uses the hash of a spec.\n\n Args:\n dag_hash: hash of the spec to search\n \"\"\"\n return list(self._mirrors_for_spec.get(dag_hash, []))\n\n def update_spec(self, spec: spack.spec.Spec, found_list: List[MirrorMetadata]) -> None:\n \"\"\"Update the cache with a new list of mirrors for a given spec.\"\"\"\n spec_dag_hash = spec.dag_hash()\n\n if spec_dag_hash not in self._mirrors_for_spec:\n self._mirrors_for_spec[spec_dag_hash] = set(found_list)\n self._known_specs[spec_dag_hash] = spec\n else:\n current_list = self._mirrors_for_spec[spec_dag_hash]\n for new_entry in found_list:\n current_list.add(new_entry.strip_view())\n\n def update(self, with_cooldown: bool = False) -> None:\n \"\"\"Make sure local cache of buildcache index files is up to date.\n If the same mirrors are configured as the last time this was called\n and none of the remote buildcache indices have changed, calling this\n method will only result in fetching the index hash from each mirror\n to confirm it is the same as what is stored locally. Otherwise, the\n buildcache ``index.json`` and ``index.json.hash`` files are retrieved\n from each configured mirror and stored locally (both in memory and\n on disk under ``_index_cache_root``).\"\"\"\n self._init_local_index_cache()\n configured_mirrors = [\n MirrorMetadata(m.fetch_url, layout_version, m.fetch_view)\n for m in spack.mirrors.mirror.MirrorCollection(binary=True).values()\n for layout_version in m.supported_layout_versions\n ]\n items_to_remove = []\n spec_cache_clear_needed = False\n spec_cache_regenerate_needed = not self._mirrors_for_spec\n\n # First compare the mirror urls currently present in the cache to the\n # configured mirrors. If we have a cached index for a mirror which is\n # no longer configured, we should remove it from the cache. For any\n # cached indices corresponding to currently configured mirrors, we need\n # to check if the cache is still good, or needs to be updated.\n # Finally, if there are configured mirrors for which we don't have a\n # cache entry, we need to fetch and cache the indices from those\n # mirrors.\n\n # If, during this process, we find that any mirrors for which we\n # already have entries have either been removed, or their index\n # hash has changed, then our concrete spec cache (_mirrors_for_spec)\n # likely has entries that need to be removed, so we will clear it\n # and regenerate that data structure.\n\n # If, during this process, we find that there are new mirrors for\n # which do not yet have an entry in our index cache, then we simply\n # need to regenerate the concrete spec cache, but do not need to\n # clear it first.\n\n # Otherwise the concrete spec cache should not need to be updated at\n # all.\n\n fetch_errors: List[Exception] = []\n all_methods_failed = True\n ttl = spack.config.get(\"config:binary_index_ttl\", 600)\n now = time.time()\n\n for local_index_cache_key in self._local_index_cache:\n urlAndVersion = MirrorMetadata.from_string(local_index_cache_key)\n cached_mirror_url = urlAndVersion.url\n cache_entry = self._local_index_cache[local_index_cache_key]\n cached_index_path = cache_entry[\"index_path\"]\n if urlAndVersion in configured_mirrors:\n # Only do a fetch if the last fetch was longer than TTL ago\n if (\n with_cooldown\n and ttl > 0\n and cached_mirror_url in self._last_fetch_times\n and now - self._last_fetch_times[urlAndVersion][0] < ttl\n ):\n # We're in the cooldown period, don't try to fetch again\n # If the fetch succeeded last time, consider this update a success, otherwise\n # re-report the error here\n if self._last_fetch_times[urlAndVersion][1]:\n all_methods_failed = False\n else:\n # May need to fetch the index and update the local caches\n needs_regen = False\n try:\n needs_regen = self._fetch_and_cache_index(\n urlAndVersion, cache_entry=cache_entry\n )\n self._last_fetch_times[urlAndVersion] = (now, True)\n all_methods_failed = False\n except FetchIndexError as e:\n fetch_errors.append(e)\n self._last_fetch_times[urlAndVersion] = (now, False)\n except BuildcacheIndexNotExists as e:\n fetch_errors.append(e)\n self._last_fetch_times[urlAndVersion] = (now, False)\n # Binary caches are not required to have an index, don't raise\n # if it doesn't exist.\n all_methods_failed = False\n\n # The need to regenerate implies a need to clear as well.\n spec_cache_clear_needed |= needs_regen\n spec_cache_regenerate_needed |= needs_regen\n else:\n # No longer have this mirror, cached index should be removed\n items_to_remove.append(\n {\n \"url\": local_index_cache_key,\n \"cache_key\": os.path.join(self._index_cache_root, cached_index_path),\n }\n )\n if urlAndVersion in self._last_fetch_times:\n del self._last_fetch_times[urlAndVersion]\n spec_cache_clear_needed = True\n spec_cache_regenerate_needed = True\n\n # Clean up items to be removed, identified above\n for item in items_to_remove:\n url = item[\"url\"]\n cache_key = item[\"cache_key\"]\n self._index_file_cache.remove(cache_key)\n del self._local_index_cache[url]\n\n # Iterate the configured mirrors now. Any mirror urls we do not\n # already have in our cache must be fetched, stored, and represented\n # locally.\n for urlAndVersion in configured_mirrors:\n if str(urlAndVersion) in self._local_index_cache:\n continue\n\n # Need to fetch the index and update the local caches\n needs_regen = False\n try:\n needs_regen = self._fetch_and_cache_index(urlAndVersion)\n self._last_fetch_times[urlAndVersion] = (now, True)\n all_methods_failed = False\n except FetchIndexError as e:\n fetch_errors.append(e)\n self._last_fetch_times[urlAndVersion] = (now, False)\n except BuildcacheIndexNotExists as e:\n fetch_errors.append(e)\n self._last_fetch_times[urlAndVersion] = (now, False)\n # Binary caches are not required to have an index, don't raise\n # if it doesn't exist.\n all_methods_failed = False\n\n # Generally speaking, a new mirror wouldn't imply the need to\n # clear the spec cache, so leave it as is.\n if needs_regen:\n spec_cache_regenerate_needed = True\n\n self._write_local_index_cache()\n\n if configured_mirrors and all_methods_failed:\n raise FetchCacheError(fetch_errors)\n if fetch_errors:\n tty.warn(\n \"The following issues were ignored while updating the indices of binary caches\",\n FetchCacheError(fetch_errors),\n )\n if spec_cache_regenerate_needed:\n self.regenerate_spec_cache(clear_existing=spec_cache_clear_needed)\n\n def _fetch_and_cache_index(self, mirror_metadata: MirrorMetadata, cache_entry={}):\n \"\"\"Fetch a buildcache index file from a remote mirror and cache it.\n\n If we already have a cached index from this mirror, then we first\n check if the hash has changed, and we avoid fetching it if not.\n\n Args:\n mirror_metadata: Contains mirror base url and target binary cache layout version\n cache_entry (dict): Old cache metadata with keys ``index_hash``, ``index_path``,\n ``etag``\n\n Returns:\n True if the local index.json was updated.\n\n Throws:\n FetchIndexError\n BuildcacheIndexNotExists\n \"\"\"\n mirror_url = mirror_metadata.url\n mirror_view = mirror_metadata.view\n layout_version = mirror_metadata.version\n\n # TODO: get rid of this request, handle 404 better\n scheme = urllib.parse.urlparse(mirror_url).scheme\n\n if scheme != \"oci\":\n cache_class = get_url_buildcache_class(layout_version=layout_version)\n index_url = cache_class.get_index_url(mirror_url, mirror_view)\n if not web_util.url_exists(index_url):\n raise BuildcacheIndexNotExists(f\"Index not found in cache {index_url}\")\n\n fetcher: IndexFetcher = get_index_fetcher(scheme, mirror_metadata, cache_entry)\n result = fetcher.conditional_fetch()\n\n # Nothing to do\n if result.fresh:\n return False\n\n # Persist new index.json\n url_hash = compute_hash(str(mirror_metadata))\n cache_key = \"{}_{}.json\".format(url_hash[:10], result.hash[:10])\n with self._index_file_cache.write_transaction(cache_key) as (old, new):\n new.write(result.data)\n\n self._local_index_cache[str(mirror_metadata)] = {\n \"index_hash\": result.hash,\n \"index_path\": cache_key,\n \"etag\": result.etag,\n }\n\n # clean up the old cache_key if necessary\n old_cache_key = cache_entry.get(\"index_path\", None)\n if old_cache_key:\n self._index_file_cache.remove(old_cache_key)\n\n # We fetched an index and updated the local index cache, we should\n # regenerate the spec cache as a result.\n return True\n\n\ndef binary_index_location():\n \"\"\"Set up a BinaryCacheIndex for remote buildcache dbs in the user's homedir.\"\"\"\n cache_root = os.path.join(spack.caches.misc_cache_location(), \"indices\")\n return spack.util.path.canonicalize_path(cache_root)\n\n\n#: Default binary cache index instance\nBINARY_INDEX = cast(BinaryCacheIndex, spack.llnl.util.lang.Singleton(BinaryCacheIndex))\n\n\ndef compute_hash(data):\n if isinstance(data, str):\n data = data.encode(\"utf-8\")\n return hashlib.sha256(data).hexdigest()\n\n\ndef buildinfo_file_name(prefix):\n \"\"\"Filename of the binary package meta-data file\"\"\"\n return os.path.join(prefix, \".spack\", \"binary_distribution\")\n\n\ndef read_buildinfo_file(prefix):\n \"\"\"Read buildinfo file\"\"\"\n with open(buildinfo_file_name(prefix), \"r\", encoding=\"utf-8\") as f:\n return syaml.load(f)\n\n\ndef file_matches(f: IO[bytes], regex: spack.llnl.util.lang.PatternBytes) -> bool:\n try:\n return bool(regex.search(f.read()))\n finally:\n f.seek(0)\n\n\ndef specs_to_relocate(spec: spack.spec.Spec) -> List[spack.spec.Spec]:\n \"\"\"Return the set of specs that may be referenced in the install prefix of the provided spec.\n We currently include non-external transitive link and direct run dependencies.\"\"\"\n specs = [\n s\n for s in itertools.chain(\n spec.traverse(root=True, deptype=\"link\", order=\"breadth\", key=traverse.by_dag_hash),\n spec.dependencies(deptype=\"run\"),\n )\n if not s.external\n ]\n return list(spack.llnl.util.lang.dedupe(specs, key=lambda s: s.dag_hash()))\n\n\ndef get_buildinfo_dict(spec):\n \"\"\"Create metadata for a tarball\"\"\"\n return {\n \"sbang_install_path\": spack.hooks.sbang.sbang_install_path(),\n \"buildpath\": spack.store.STORE.layout.root,\n \"spackprefix\": spack.paths.prefix,\n \"relative_prefix\": os.path.relpath(spec.prefix, spack.store.STORE.layout.root),\n # \"relocate_textfiles\": [],\n # \"relocate_binaries\": [],\n # \"relocate_links\": [],\n \"hardlinks_deduped\": True,\n \"hash_to_prefix\": {d.dag_hash(): str(d.prefix) for d in specs_to_relocate(spec)},\n }\n\n\ndef buildcache_relative_keys_path(layout_version: int = CURRENT_BUILD_CACHE_LAYOUT_VERSION):\n cache_class = get_url_buildcache_class(layout_version=layout_version)\n return os.path.join(*cache_class.get_relative_path_components(BuildcacheComponent.KEY))\n\n\ndef buildcache_relative_keys_url(layout_version: int = CURRENT_BUILD_CACHE_LAYOUT_VERSION):\n cache_class = get_url_buildcache_class(layout_version=layout_version)\n return url_util.join(*cache_class.get_relative_path_components(BuildcacheComponent.KEY))\n\n\ndef buildcache_relative_specs_path(layout_version: int = CURRENT_BUILD_CACHE_LAYOUT_VERSION):\n cache_class = get_url_buildcache_class(layout_version=layout_version)\n return os.path.join(*cache_class.get_relative_path_components(BuildcacheComponent.SPEC))\n\n\ndef buildcache_relative_specs_url(layout_version: int = CURRENT_BUILD_CACHE_LAYOUT_VERSION):\n cache_class = get_url_buildcache_class(layout_version=layout_version)\n return url_util.join(*cache_class.get_relative_path_components(BuildcacheComponent.SPEC))\n\n\ndef buildcache_relative_blobs_path(layout_version: int = CURRENT_BUILD_CACHE_LAYOUT_VERSION):\n cache_class = get_url_buildcache_class(layout_version=layout_version)\n return os.path.join(*cache_class.get_relative_path_components(BuildcacheComponent.BLOB))\n\n\ndef buildcache_relative_blobs_url(layout_version: int = CURRENT_BUILD_CACHE_LAYOUT_VERSION):\n cache_class = get_url_buildcache_class(layout_version=layout_version)\n return url_util.join(*cache_class.get_relative_path_components(BuildcacheComponent.BLOB))\n\n\ndef buildcache_relative_index_path(layout_version: int = CURRENT_BUILD_CACHE_LAYOUT_VERSION):\n cache_class = get_url_buildcache_class(layout_version=layout_version)\n return os.path.join(*cache_class.get_relative_path_components(BuildcacheComponent.INDEX))\n\n\ndef buildcache_relative_index_url(layout_version: int = CURRENT_BUILD_CACHE_LAYOUT_VERSION):\n cache_class = get_url_buildcache_class(layout_version=layout_version)\n return url_util.join(*cache_class.get_relative_path_components(BuildcacheComponent.INDEX))\n\n\n@spack.llnl.util.lang.memoized\ndef warn_v2_layout(mirror_url: str, action: str) -> bool:\n lines = textwrap.wrap(\n f\"{action} from a v2 binary mirror layout, located at \"\n f\"{mirror_url} is deprecated. Support for this will be \"\n \"removed in a future version of spack. \"\n \"If you manage the buildcache please consider running:\",\n width=72,\n subsequent_indent=\" \",\n )\n lines.extend(\n [\n \" 'spack buildcache migrate'\",\n \" or rebuilding the specs in this mirror. Otherwise, consider running:\",\n \" 'spack mirror list'\",\n \" 'spack mirror remove '\",\n \" with the for the mirror url shown in the list.\",\n ]\n )\n tty.warn(\"\\n\".join(lines))\n return True\n\n\ndef select_signing_key() -> str:\n keys = spack.util.gpg.signing_keys()\n num = len(keys)\n if num > 1:\n raise PickKeyException(str(keys))\n elif num == 0:\n raise NoKeyException(\n \"No default key available for signing.\\n\"\n \"Use spack gpg init and spack gpg create\"\n \" to create a default key.\"\n )\n return keys[0]\n\n\ndef _push_index(db: BuildCacheDatabase, temp_dir: str, cache_prefix: str, name: str = \"\"):\n \"\"\"Generate the index, compute its hash, and push the files to the mirror\"\"\"\n index_json_path = os.path.join(temp_dir, spack.database.INDEX_JSON_FILE)\n with open(index_json_path, \"w\", encoding=\"utf-8\") as f:\n db._write_to_file(f)\n\n cache_class = get_url_buildcache_class(layout_version=CURRENT_BUILD_CACHE_LAYOUT_VERSION)\n cache_class.push_local_file_as_blob(\n index_json_path,\n cache_prefix,\n url_util.join(name, \"index\") if name else \"index\",\n BuildcacheComponent.INDEX,\n compression=\"none\",\n )\n cache_class.maybe_push_layout_json(cache_prefix)\n\n\ndef _read_specs_and_push_index(\n file_list: List[str],\n read_method: Callable[[str], URLBuildcacheEntry],\n name: str,\n filter_fn: Callable[[str], bool],\n cache_prefix: str,\n db: BuildCacheDatabase,\n temp_dir: str,\n *,\n timer=timer.NULL_TIMER,\n):\n \"\"\"Read listed specs, generate the index, and push it to the mirror.\n\n Args:\n file_list: List of urls or file paths pointing at spec files to read\n read_method: A function taking a single argument, either a url or a file path,\n and which reads the spec file at that location, and returns the spec.\n cache_prefix: prefix of the build cache on s3 where index should be pushed.\n db: A spack database used for adding specs and then writing the index.\n temp_dir: Location to write index.json and hash for pushing\n \"\"\"\n with timer.measure(\"read\"):\n for file in file_list:\n # All supported versions of build caches put the hash as the last\n # parameter before the extension\n try:\n x = file.split(\"/\")[-1].split(\"-\")[-1].split(\".\")[0]\n except IndexError:\n raise GenerateIndexError(f\"Malformed metadata file name detected {file}\")\n\n if not filter_fn(x):\n continue\n\n cache_entry: Optional[URLBuildcacheEntry] = None\n try:\n cache_entry = read_method(file)\n spec_dict = cache_entry.fetch_metadata()\n fetched_spec = spack.spec.Spec.from_dict(spec_dict)\n except Exception as e:\n tty.warn(f\"Unable to fetch spec for manifest {file} due to: {e}\")\n continue\n finally:\n if cache_entry:\n cache_entry.destroy()\n db.add(fetched_spec)\n db.mark(fetched_spec, \"in_buildcache\", True)\n\n with timer.measure(\"push\"):\n _push_index(db, temp_dir, cache_prefix, name)\n\n\ndef _url_generate_package_index(\n url: str,\n tmpdir: str,\n db: Optional[BuildCacheDatabase] = None,\n name: str = \"\",\n filter_fn: Callable[[str], bool] = lambda x: True,\n *,\n timer=timer.NULL_TIMER,\n):\n \"\"\"Create or replace the build cache index on the given mirror. The\n buildcache index contains an entry for each binary package under the\n cache_prefix.\n\n Args:\n url: Base url of binary mirror.\n\n Return:\n None\n \"\"\"\n with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpspecsdir:\n try:\n with timer.measure(\"list\"):\n filename_to_mtime_mapping, read_fn = get_entries_from_cache(\n url, tmpspecsdir, component_type=BuildcacheComponent.SPEC\n )\n file_list = list(filename_to_mtime_mapping.keys())\n except ListMirrorSpecsError as e:\n raise GenerateIndexError(f\"Unable to generate package index: {e}\") from e\n\n tty.debug(f\"Retrieving spec descriptor files from {url} to build index\")\n\n if not db:\n db = BuildCacheDatabase(tmpdir)\n db._write()\n\n try:\n _read_specs_and_push_index(\n file_list,\n read_fn,\n name,\n filter_fn,\n url,\n db,\n str(db.database_directory),\n timer=timer,\n )\n except Exception as e:\n raise GenerateIndexError(\n f\"Encountered problem pushing package index to {url}: {e}\"\n ) from e\n\n\ndef generate_key_index(mirror_url: str, tmpdir: str) -> None:\n \"\"\"Create the key index page.\n\n Creates (or replaces) the ``index.json`` page at the location given in mirror_url. This page\n contains an entry for each key under mirror_url.\n \"\"\"\n\n tty.debug(f\"Retrieving key.pub files from {url_util.format(mirror_url)} to build key index\")\n\n key_prefix = url_util.join(mirror_url, buildcache_relative_keys_url())\n\n try:\n fingerprints = (\n entry[:-18]\n for entry in web_util.list_url(key_prefix, recursive=False)\n if entry.endswith(\".key.manifest.json\")\n )\n except Exception as e:\n raise CannotListKeys(f\"Encountered problem listing keys at {key_prefix}: {e}\") from e\n\n target = os.path.join(tmpdir, \"index.json\")\n\n index = {\"keys\": dict((fingerprint, {}) for fingerprint in sorted(set(fingerprints)))}\n with open(target, \"w\", encoding=\"utf-8\") as f:\n sjson.dump(index, f)\n\n cache_class = get_url_buildcache_class()\n\n try:\n cache_class.push_local_file_as_blob(\n local_file_path=target,\n mirror_url=mirror_url,\n manifest_name=\"keys\",\n component_type=BuildcacheComponent.KEY_INDEX,\n compression=\"none\",\n )\n cache_class.maybe_push_layout_json(mirror_url)\n except Exception as e:\n raise GenerateIndexError(\n f\"Encountered problem pushing key index to {key_prefix}: {e}\"\n ) from e\n\n\nclass FileTypes:\n BINARY = 0\n TEXT = 1\n UNKNOWN = 2\n\n\nNOT_ISO8859_1_TEXT = re.compile(b\"[\\x00\\x7f-\\x9f]\")\n\n\ndef file_type(f: IO[bytes]) -> int:\n try:\n # first check if this is an ELF or mach-o binary.\n magic = f.read(8)\n if len(magic) < 8:\n return FileTypes.UNKNOWN\n elif relocate.is_elf_magic(magic) or relocate.is_macho_magic(magic):\n return FileTypes.BINARY\n\n f.seek(0)\n\n # Then try utf-8, which has a fast exponential decay in false positive rate with file size.\n # Use chunked reads for fast early exit.\n f_txt = io.TextIOWrapper(f, encoding=\"utf-8\", errors=\"strict\")\n try:\n while f_txt.read(1024):\n pass\n return FileTypes.TEXT\n except UnicodeError:\n f_txt.seek(0)\n pass\n finally:\n f_txt.detach()\n # Finally try iso-8859-1 heuristically. In Python, all possible 256 byte values are valid.\n # We classify it as text if it does not contain any control characters / null bytes.\n data = f.read(1024)\n while data:\n if NOT_ISO8859_1_TEXT.search(data):\n break\n data = f.read(1024)\n else:\n return FileTypes.TEXT\n return FileTypes.UNKNOWN\n finally:\n f.seek(0)\n\n\ndef tarfile_of_spec_prefix(\n tar: tarfile.TarFile, prefix: str, prefixes_to_relocate: List[str]\n) -> dict:\n \"\"\"Create a tarfile of an install prefix of a spec. Skips existing buildinfo file.\n\n Args:\n tar: tarfile object to add files to\n prefix: absolute install prefix of spec\"\"\"\n if not os.path.isabs(prefix) or not os.path.isdir(prefix):\n raise ValueError(f\"prefix '{prefix}' must be an absolute path to a directory\")\n stat_key = lambda stat: (stat.st_dev, stat.st_ino)\n\n try: # skip buildinfo file if it exists\n files_to_skip = [stat_key(os.lstat(buildinfo_file_name(prefix)))]\n skip = lambda entry: stat_key(entry.stat(follow_symlinks=False)) in files_to_skip\n except OSError:\n skip = lambda entry: False\n\n binary_regex = utf8_paths_to_single_binary_regex(prefixes_to_relocate)\n\n relocate_binaries = []\n relocate_links = []\n relocate_textfiles = []\n\n # use callbacks to add files and symlinks, so we can register which files need relocation upon\n # extraction.\n def add_file(tar: tarfile.TarFile, info: tarfile.TarInfo, path: str):\n with open(path, \"rb\") as f:\n relpath = os.path.relpath(path, prefix)\n # no need to relocate anything in the .spack directory\n if relpath.split(os.sep, 1)[0] == \".spack\":\n tar.addfile(info, f)\n return\n f_type = file_type(f)\n if f_type == FileTypes.BINARY:\n relocate_binaries.append(os.path.relpath(path, prefix))\n elif f_type == FileTypes.TEXT and file_matches(f, binary_regex):\n relocate_textfiles.append(os.path.relpath(path, prefix))\n tar.addfile(info, f)\n\n def add_symlink(tar: tarfile.TarFile, info: tarfile.TarInfo, path: str):\n if os.path.isabs(info.linkname) and binary_regex.match(info.linkname.encode(\"utf-8\")):\n relocate_links.append(os.path.relpath(path, prefix))\n tar.addfile(info)\n\n spack.util.archive.reproducible_tarfile_from_prefix(\n tar,\n prefix,\n # Spack <= 0.21 did not include parent directories, leading to issues when tarballs are\n # used in runtimes like AWS lambda.\n include_parent_directories=True,\n skip=skip,\n add_file=add_file,\n add_symlink=add_symlink,\n )\n\n return {\n \"relocate_binaries\": relocate_binaries,\n \"relocate_links\": relocate_links,\n \"relocate_textfiles\": relocate_textfiles,\n }\n\n\ndef create_tarball(spec: spack.spec.Spec, tarfile_path: str) -> Tuple[str, str]:\n \"\"\"Create a tarball of a spec and return the checksums of the compressed tarfile and the\n uncompressed tarfile.\"\"\"\n return _do_create_tarball(\n tarfile_path,\n spec.prefix,\n buildinfo=get_buildinfo_dict(spec),\n prefixes_to_relocate=prefixes_to_relocate(spec),\n )\n\n\ndef _do_create_tarball(\n tarfile_path: str, prefix: str, buildinfo: dict, prefixes_to_relocate: List[str]\n) -> Tuple[str, str]:\n with spack.util.archive.gzip_compressed_tarfile(tarfile_path) as (\n tar,\n tar_gz_checksum,\n tar_checksum,\n ):\n # Tarball the install prefix\n files_to_relocate = tarfile_of_spec_prefix(tar, prefix, prefixes_to_relocate)\n buildinfo.update(files_to_relocate)\n\n # Serialize buildinfo for the tarball\n bstring = syaml.dump(buildinfo, default_flow_style=True).encode(\"utf-8\")\n tarinfo = tarfile.TarInfo(\n name=spack.util.archive.default_path_to_name(buildinfo_file_name(prefix))\n )\n tarinfo.type = tarfile.REGTYPE\n tarinfo.size = len(bstring)\n tarinfo.mode = 0o644\n tar.addfile(tarinfo, io.BytesIO(bstring))\n\n return tar_gz_checksum.hexdigest(), tar_checksum.hexdigest()\n\n\ndef _exists_in_buildcache(\n spec: spack.spec.Spec, out_url: str, allow_unsigned: bool = False\n) -> URLBuildcacheEntry:\n \"\"\"creates and returns (after checking existence) a URLBuildcacheEntry\"\"\"\n cache_type = get_url_buildcache_class(CURRENT_BUILD_CACHE_LAYOUT_VERSION)\n cache_entry = cache_type(out_url, spec, allow_unsigned=allow_unsigned)\n return cache_entry\n\n\ndef prefixes_to_relocate(spec):\n prefixes = [s.prefix for s in specs_to_relocate(spec)]\n prefixes.append(spack.hooks.sbang.sbang_install_path())\n prefixes.append(str(spack.store.STORE.layout.root))\n return prefixes\n\n\ndef _url_upload_tarball_and_specfile(\n spec: spack.spec.Spec, tmpdir: str, cache_entry: URLBuildcacheEntry, signing_key: Optional[str]\n):\n tarball = os.path.join(tmpdir, f\"{spec.dag_hash()}.tar.gz\")\n checksum, _ = create_tarball(spec, tarball)\n\n cache_entry.push_binary_package(spec, tarball, \"sha256\", checksum, tmpdir, signing_key)\n\n\nclass Uploader:\n def __init__(self, mirror: spack.mirrors.mirror.Mirror, force: bool, update_index: bool):\n self.mirror = mirror\n self.force = force\n self.update_index = update_index\n\n self.tmpdir: str\n self.executor: concurrent.futures.Executor\n\n # Verify if the mirror meets the requirements to push\n self.mirror.ensure_mirror_usable(\"push\")\n\n def __enter__(self):\n self._tmpdir = tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root())\n self._executor = spack.util.parallel.make_concurrent_executor()\n\n self.tmpdir = self._tmpdir.__enter__()\n self.executor = self.executor = self._executor.__enter__()\n\n return self\n\n def __exit__(self, *args):\n self._executor.__exit__(*args)\n self._tmpdir.__exit__(*args)\n\n def push_or_raise(self, specs: List[spack.spec.Spec]) -> List[spack.spec.Spec]:\n skipped, errors = self.push(specs)\n if errors:\n raise PushToBuildCacheError(\n f\"Failed to push {len(errors)} specs to {self.mirror.push_url}:\\n\"\n + \"\\n\".join(\n f\"Failed to push {_format_spec(spec)}: {error}\" for spec, error in errors\n )\n )\n return skipped\n\n def push(\n self, specs: List[spack.spec.Spec]\n ) -> Tuple[List[spack.spec.Spec], List[Tuple[spack.spec.Spec, BaseException]]]:\n raise NotImplementedError\n\n def tag(self, tag: str, roots: List[spack.spec.Spec]):\n \"\"\"Make a list of selected specs together available under the given tag\"\"\"\n pass\n\n\nclass OCIUploader(Uploader):\n def __init__(\n self,\n mirror: spack.mirrors.mirror.Mirror,\n force: bool,\n update_index: bool,\n base_image: Optional[str],\n ) -> None:\n super().__init__(mirror, force, update_index)\n self.target_image = spack.oci.oci.image_from_mirror(mirror)\n self.base_image = ImageReference.from_string(base_image) if base_image else None\n\n def push(\n self, specs: List[spack.spec.Spec]\n ) -> Tuple[List[spack.spec.Spec], List[Tuple[spack.spec.Spec, BaseException]]]:\n skipped, base_images, checksums, upload_errors = _oci_push(\n target_image=self.target_image,\n base_image=self.base_image,\n installed_specs_with_deps=specs,\n force=self.force,\n tmpdir=self.tmpdir,\n executor=self.executor,\n )\n\n self._base_images = base_images\n self._checksums = checksums\n\n # only update index if any binaries were uploaded\n if self.update_index and len(skipped) + len(upload_errors) < len(specs):\n _oci_update_index(self.target_image, self.tmpdir, self.executor)\n\n return skipped, upload_errors\n\n def tag(self, tag: str, roots: List[spack.spec.Spec]):\n tagged_image = self.target_image.with_tag(tag)\n\n # _push_oci may not populate self._base_images if binaries were already in the registry\n for spec in roots:\n _oci_update_base_images(\n base_image=self.base_image,\n target_image=self.target_image,\n spec=spec,\n base_image_cache=self._base_images,\n )\n _oci_put_manifest(\n self._base_images, self._checksums, tagged_image, self.tmpdir, None, None, *roots\n )\n\n tty.info(f\"Tagged {tagged_image}\")\n\n\nclass URLUploader(Uploader):\n def __init__(\n self,\n mirror: spack.mirrors.mirror.Mirror,\n force: bool,\n update_index: bool,\n signing_key: Optional[str],\n ) -> None:\n super().__init__(mirror, force, update_index)\n self.url = mirror.push_url\n self.signing_key = signing_key\n\n def push(\n self, specs: List[spack.spec.Spec]\n ) -> Tuple[List[spack.spec.Spec], List[Tuple[spack.spec.Spec, BaseException]]]:\n return _url_push(\n specs,\n out_url=self.url,\n force=self.force,\n update_index=self.update_index,\n signing_key=self.signing_key,\n tmpdir=self.tmpdir,\n executor=self.executor,\n )\n\n\ndef make_uploader(\n mirror: spack.mirrors.mirror.Mirror,\n force: bool = False,\n update_index: bool = False,\n signing_key: Optional[str] = None,\n base_image: Optional[str] = None,\n) -> Uploader:\n \"\"\"Builder for the appropriate uploader based on the mirror type\"\"\"\n if spack.oci.image.is_oci_url(mirror.push_url):\n return OCIUploader(\n mirror=mirror, force=force, update_index=update_index, base_image=base_image\n )\n else:\n return URLUploader(\n mirror=mirror, force=force, update_index=update_index, signing_key=signing_key\n )\n\n\ndef _format_spec(spec: spack.spec.Spec) -> str:\n return spec.cformat(\"{name}{@version}{/hash:7}\")\n\n\nclass FancyProgress:\n def __init__(self, total: int):\n self.n = 0\n self.total = total\n self.running = False\n self.enable = sys.stdout.isatty()\n self.pretty_spec: str = \"\"\n self.pre = \"\"\n\n def _clear(self):\n if self.enable and self.running:\n sys.stdout.write(\"\\033[F\\033[K\")\n\n def _progress(self):\n if self.total > 1:\n digits = len(str(self.total))\n return f\"[{self.n:{digits}}/{self.total}] \"\n return \"\"\n\n def start(self, spec: spack.spec.Spec, running: bool) -> None:\n self.n += 1\n self.running = running\n self.pre = self._progress()\n self.pretty_spec = _format_spec(spec)\n if self.enable and self.running:\n tty.info(f\"{self.pre}Pushing {self.pretty_spec}...\")\n\n def ok(self, msg: Optional[str] = None) -> None:\n self._clear()\n msg = msg or f\"Pushed {self.pretty_spec}\"\n tty.info(f\"{self.pre}{msg}\")\n\n def fail(self) -> None:\n self._clear()\n tty.info(f\"{self.pre}Failed to push {self.pretty_spec}\")\n\n\ndef _url_push(\n specs: List[spack.spec.Spec],\n out_url: str,\n signing_key: Optional[str],\n force: bool,\n update_index: bool,\n tmpdir: str,\n executor: concurrent.futures.Executor,\n) -> Tuple[List[spack.spec.Spec], List[Tuple[spack.spec.Spec, BaseException]]]:\n \"\"\"Pushes to the provided build cache, and returns a list of skipped specs that were already\n present (when force=False), and a list of errors. Does not raise on error.\"\"\"\n skipped: List[spack.spec.Spec] = []\n errors: List[Tuple[spack.spec.Spec, BaseException]] = []\n\n exists_futures = [\n executor.submit(\n _exists_in_buildcache, spec, out_url, allow_unsigned=False if signing_key else True\n )\n for spec in specs\n ]\n\n cache_entries = {\n spec.dag_hash(): exists_future.result()\n for spec, exists_future in zip(specs, exists_futures)\n }\n\n if not force:\n specs_to_upload = []\n\n for spec in specs:\n if cache_entries[spec.dag_hash()].exists(\n [BuildcacheComponent.SPEC, BuildcacheComponent.TARBALL]\n ):\n skipped.append(spec)\n else:\n specs_to_upload.append(spec)\n else:\n specs_to_upload = specs\n\n if not specs_to_upload:\n return skipped, errors\n\n total = len(specs_to_upload)\n\n if total != len(specs):\n tty.info(f\"{total} specs need to be pushed to {out_url}\")\n\n upload_futures = [\n executor.submit(\n _url_upload_tarball_and_specfile,\n spec,\n tmpdir,\n cache_entries[spec.dag_hash()],\n signing_key,\n )\n for spec in specs_to_upload\n ]\n\n uploaded_any = False\n fancy_progress = FancyProgress(total)\n\n for spec, upload_future in zip(specs_to_upload, upload_futures):\n fancy_progress.start(spec, upload_future.running())\n error = upload_future.exception()\n if error is None:\n uploaded_any = True\n fancy_progress.ok()\n else:\n fancy_progress.fail()\n errors.append((spec, error))\n\n # don't bother pushing keys / index if all failed to upload\n if not uploaded_any:\n return skipped, errors\n\n # If the layout.json doesn't yet exist on this mirror, push it\n cache_class = get_url_buildcache_class(layout_version=CURRENT_BUILD_CACHE_LAYOUT_VERSION)\n cache_class.maybe_push_layout_json(out_url)\n\n if signing_key:\n keys_tmpdir = os.path.join(tmpdir, \"keys\")\n os.mkdir(keys_tmpdir)\n _url_push_keys(out_url, keys=[signing_key], update_index=update_index, tmpdir=keys_tmpdir)\n\n if update_index:\n index_tmpdir = os.path.join(tmpdir, \"index\")\n os.mkdir(index_tmpdir)\n _url_generate_package_index(out_url, index_tmpdir)\n\n return skipped, errors\n\n\ndef _oci_upload_success_msg(spec: spack.spec.Spec, digest: Digest, size: int, elapsed: float):\n elapsed = max(elapsed, 0.001) # guard against division by zero\n return (\n f\"Pushed {_format_spec(spec)}: {digest} ({elapsed:.2f}s, \"\n f\"{size / elapsed / 1024 / 1024:.2f} MB/s)\"\n )\n\n\ndef _oci_get_blob_info(image_ref: ImageReference) -> Optional[spack.oci.oci.Blob]:\n \"\"\"Get the spack tarball layer digests and size if it exists\"\"\"\n try:\n manifest, config = get_manifest_and_config_with_retry(image_ref)\n\n return spack.oci.oci.Blob(\n compressed_digest=Digest.from_string(manifest[\"layers\"][-1][\"digest\"]),\n uncompressed_digest=Digest.from_string(config[\"rootfs\"][\"diff_ids\"][-1]),\n size=manifest[\"layers\"][-1][\"size\"],\n )\n except Exception:\n return None\n\n\ndef _oci_push_pkg_blob(\n image_ref: ImageReference, spec: spack.spec.Spec, tmpdir: str\n) -> Tuple[spack.oci.oci.Blob, float]:\n \"\"\"Push a package blob to the registry and return the blob info and the time taken\"\"\"\n filename = os.path.join(tmpdir, f\"{spec.dag_hash()}.tar.gz\")\n\n # Create an oci.image.layer aka tarball of the package\n tar_gz_checksum, tar_checksum = create_tarball(spec, filename)\n\n blob = spack.oci.oci.Blob(\n Digest.from_sha256(tar_gz_checksum),\n Digest.from_sha256(tar_checksum),\n os.path.getsize(filename),\n )\n\n # Upload the blob\n start = time.time()\n upload_blob_with_retry(image_ref, file=filename, digest=blob.compressed_digest)\n elapsed = time.time() - start\n\n # delete the file\n os.unlink(filename)\n\n return blob, elapsed\n\n\ndef _oci_retrieve_env_dict_from_config(config: dict) -> dict:\n \"\"\"Retrieve the environment variables from the image config file.\n Sets a default value for PATH if it is not present.\n\n Args:\n config (dict): The image config file.\n\n Returns:\n dict: The environment variables.\n \"\"\"\n env = {\"PATH\": \"/bin:/usr/bin\"}\n\n if \"Env\" in config.get(\"config\", {}):\n for entry in config[\"config\"][\"Env\"]:\n key, value = entry.split(\"=\", 1)\n env[key] = value\n return env\n\n\ndef _oci_archspec_to_gooarch(spec: spack.spec.Spec) -> str:\n name = spec.target.family.name\n name_map = {\"aarch64\": \"arm64\", \"x86_64\": \"amd64\"}\n return name_map.get(name, name)\n\n\ndef _oci_put_manifest(\n base_images: Dict[str, Tuple[dict, dict]],\n checksums: Dict[str, spack.oci.oci.Blob],\n image_ref: ImageReference,\n tmpdir: str,\n extra_config: Optional[dict],\n annotations: Optional[dict],\n *specs: spack.spec.Spec,\n):\n architecture = _oci_archspec_to_gooarch(specs[0])\n\n expected_blobs: List[spack.spec.Spec] = [\n s\n for s in traverse.traverse_nodes(specs, order=\"topo\", deptype=(\"link\", \"run\"), root=True)\n if not s.external\n ]\n expected_blobs.reverse()\n\n base_manifest, base_config = base_images[architecture]\n env = _oci_retrieve_env_dict_from_config(base_config)\n\n # If the base image uses `vnd.docker.distribution.manifest.v2+json`, then we use that too.\n # This is because Singularity / Apptainer is very strict about not mixing them.\n base_manifest_mediaType = base_manifest.get(\n \"mediaType\", \"application/vnd.oci.image.manifest.v1+json\"\n )\n use_docker_format = (\n base_manifest_mediaType == \"application/vnd.docker.distribution.manifest.v2+json\"\n )\n\n spack.user_environment.environment_modifications_for_specs(*specs).apply_modifications(env)\n\n # Create an oci.image.config file\n config = copy.deepcopy(base_config)\n\n # Add the diff ids of the blobs\n for s in expected_blobs:\n # If a layer for a dependency has gone missing (due to removed manifest in the registry, a\n # failed push, or a local forced uninstall), we cannot create a runnable container image.\n checksum = checksums.get(s.dag_hash())\n if checksum:\n config[\"rootfs\"][\"diff_ids\"].append(str(checksum.uncompressed_digest))\n\n # Set the environment variables\n config[\"config\"][\"Env\"] = [f\"{k}={v}\" for k, v in env.items()]\n\n if extra_config:\n # From the OCI v1.0 spec:\n # > Any extra fields in the Image JSON struct are considered implementation\n # > specific and MUST be ignored by any implementations which are unable to\n # > interpret them.\n config.update(extra_config)\n\n config_file = os.path.join(tmpdir, f\"{specs[0].dag_hash()}.config.json\")\n\n with open(config_file, \"w\", encoding=\"utf-8\") as f:\n json.dump(config, f, separators=(\",\", \":\"))\n\n config_file_checksum = Digest.from_sha256(\n spack.util.crypto.checksum(hashlib.sha256, config_file)\n )\n\n # Upload the config file\n upload_blob_with_retry(image_ref, file=config_file, digest=config_file_checksum)\n\n manifest = {\n \"mediaType\": base_manifest_mediaType,\n \"schemaVersion\": 2,\n \"config\": {\n \"mediaType\": base_manifest[\"config\"][\"mediaType\"],\n \"digest\": str(config_file_checksum),\n \"size\": os.path.getsize(config_file),\n },\n \"layers\": [\n *(layer for layer in base_manifest[\"layers\"]),\n *(\n {\n \"mediaType\": (\n \"application/vnd.docker.image.rootfs.diff.tar.gzip\"\n if use_docker_format\n else \"application/vnd.oci.image.layer.v1.tar+gzip\"\n ),\n \"digest\": str(checksums[s.dag_hash()].compressed_digest),\n \"size\": checksums[s.dag_hash()].size,\n }\n for s in expected_blobs\n if s.dag_hash() in checksums\n ),\n ],\n }\n\n if not use_docker_format and annotations:\n manifest[\"annotations\"] = annotations\n\n # Finally upload the manifest\n upload_manifest_with_retry(image_ref, manifest=manifest)\n\n # delete the config file\n os.unlink(config_file)\n\n\ndef _oci_update_base_images(\n *,\n base_image: Optional[ImageReference],\n target_image: ImageReference,\n spec: spack.spec.Spec,\n base_image_cache: Dict[str, Tuple[dict, dict]],\n):\n \"\"\"For a given spec and base image, copy the missing layers of the base image with matching\n arch to the registry of the target image. If no base image is specified, create a dummy\n manifest and config file.\"\"\"\n architecture = _oci_archspec_to_gooarch(spec)\n if architecture in base_image_cache:\n return\n if base_image is None:\n base_image_cache[architecture] = (\n default_manifest(),\n default_config(architecture, \"linux\"),\n )\n else:\n base_image_cache[architecture] = copy_missing_layers_with_retry(\n base_image, target_image, architecture\n )\n\n\ndef _oci_default_tag(spec: spack.spec.Spec) -> str:\n \"\"\"Return a valid, default image tag for a spec.\"\"\"\n return ensure_valid_tag(f\"{spec.name}-{spec.version}-{spec.dag_hash()}.spack\")\n\n\n#: Default OCI index tag\ndefault_index_tag = \"index.spack\"\n\n\ndef tag_is_spec(tag: str) -> bool:\n \"\"\"Check if a tag is likely a Spec\"\"\"\n return tag.endswith(\".spack\") and tag != default_index_tag\n\n\ndef _oci_push(\n *,\n target_image: ImageReference,\n base_image: Optional[ImageReference],\n installed_specs_with_deps: List[spack.spec.Spec],\n tmpdir: str,\n executor: concurrent.futures.Executor,\n force: bool = False,\n) -> Tuple[\n List[spack.spec.Spec],\n Dict[str, Tuple[dict, dict]],\n Dict[str, spack.oci.oci.Blob],\n List[Tuple[spack.spec.Spec, BaseException]],\n]:\n # Spec dag hash -> blob\n checksums: Dict[str, spack.oci.oci.Blob] = {}\n\n # arch -> (manifest, config)\n base_images: Dict[str, Tuple[dict, dict]] = {}\n\n # Specs not uploaded because they already exist\n skipped: List[spack.spec.Spec] = []\n\n if not force:\n tty.info(\"Checking for existing specs in the buildcache\")\n blobs_to_upload = []\n\n tags_to_check = (\n target_image.with_tag(_oci_default_tag(s)) for s in installed_specs_with_deps\n )\n available_blobs = executor.map(_oci_get_blob_info, tags_to_check)\n\n for spec, maybe_blob in zip(installed_specs_with_deps, available_blobs):\n if maybe_blob is not None:\n checksums[spec.dag_hash()] = maybe_blob\n skipped.append(spec)\n else:\n blobs_to_upload.append(spec)\n else:\n blobs_to_upload = installed_specs_with_deps\n\n if not blobs_to_upload:\n return skipped, base_images, checksums, []\n\n if len(blobs_to_upload) != len(installed_specs_with_deps):\n tty.info(\n f\"{len(blobs_to_upload)} specs need to be pushed to \"\n f\"{target_image.domain}/{target_image.name}\"\n )\n\n blob_progress = FancyProgress(len(blobs_to_upload))\n\n # Upload blobs\n blob_futures = [\n executor.submit(_oci_push_pkg_blob, target_image, spec, tmpdir) for spec in blobs_to_upload\n ]\n\n manifests_to_upload: List[spack.spec.Spec] = []\n errors: List[Tuple[spack.spec.Spec, BaseException]] = []\n\n # And update the spec to blob mapping for successful uploads\n for spec, blob_future in zip(blobs_to_upload, blob_futures):\n blob_progress.start(spec, blob_future.running())\n error = blob_future.exception()\n if error is None:\n blob, elapsed = blob_future.result()\n blob_progress.ok(\n _oci_upload_success_msg(spec, blob.compressed_digest, blob.size, elapsed)\n )\n manifests_to_upload.append(spec)\n checksums[spec.dag_hash()] = blob\n else:\n blob_progress.fail()\n errors.append((spec, error))\n\n # Copy base images if necessary\n for spec in manifests_to_upload:\n _oci_update_base_images(\n base_image=base_image,\n target_image=target_image,\n spec=spec,\n base_image_cache=base_images,\n )\n\n def extra_config(spec: spack.spec.Spec):\n spec_dict = spec.to_dict(hash=ht.dag_hash)\n spec_dict[\"buildcache_layout_version\"] = CURRENT_BUILD_CACHE_LAYOUT_VERSION\n spec_dict[\"binary_cache_checksum\"] = {\n \"hash_algorithm\": \"sha256\",\n \"hash\": checksums[spec.dag_hash()].compressed_digest.digest,\n }\n spec_dict[\"archive_size\"] = checksums[spec.dag_hash()].size\n spec_dict[\"archive_timestamp\"] = datetime.datetime.now().astimezone().isoformat()\n spec_dict[\"archive_compression\"] = \"gzip\"\n return spec_dict\n\n # Upload manifests\n tty.info(\"Uploading manifests\")\n manifest_futures = [\n executor.submit(\n _oci_put_manifest,\n base_images,\n checksums,\n target_image.with_tag(_oci_default_tag(spec)),\n tmpdir,\n extra_config(spec),\n {\"org.opencontainers.image.description\": spec.format()},\n spec,\n )\n for spec in manifests_to_upload\n ]\n\n manifest_progress = FancyProgress(len(manifests_to_upload))\n\n # Print the image names of the top-level specs\n for spec, manifest_future in zip(manifests_to_upload, manifest_futures):\n error = manifest_future.exception()\n manifest_progress.start(spec, manifest_future.running())\n if error is None:\n manifest_progress.ok(\n f\"Tagged {_format_spec(spec)} as {target_image.with_tag(_oci_default_tag(spec))}\"\n )\n else:\n manifest_progress.fail()\n errors.append((spec, error))\n\n return skipped, base_images, checksums, errors\n\n\ndef _oci_config_from_tag(image_ref_and_tag: Tuple[ImageReference, str]) -> Optional[dict]:\n image_ref, tag = image_ref_and_tag\n # Don't allow recursion here, since Spack itself always uploads\n # vnd.oci.image.manifest.v1+json, not vnd.oci.image.index.v1+json\n _, config = get_manifest_and_config_with_retry(image_ref.with_tag(tag), tag, recurse=0)\n\n # Do very basic validation: if \"spec\" is a key in the config, it\n # must be a Spec object too.\n return config if \"spec\" in config else None\n\n\ndef _oci_update_index(\n image_ref: ImageReference, tmpdir: str, pool: concurrent.futures.Executor\n) -> None:\n tags = list_tags(image_ref)\n\n # Fetch all image config files in parallel\n spec_dicts = pool.map(\n _oci_config_from_tag, ((image_ref, tag) for tag in tags if tag_is_spec(tag))\n )\n\n # Populate the database\n db_root_dir = os.path.join(tmpdir, \"db_root\")\n db = BuildCacheDatabase(db_root_dir)\n\n for spec_dict in spec_dicts:\n spec = spack.spec.Spec.from_dict(spec_dict)\n db.add(spec)\n db.mark(spec, \"in_buildcache\", True)\n\n # Create the index.json file\n index_json_path = os.path.join(tmpdir, spack.database.INDEX_JSON_FILE)\n with open(index_json_path, \"w\", encoding=\"utf-8\") as f:\n db._write_to_file(f)\n\n # Create an empty config.json file\n empty_config_json_path = os.path.join(tmpdir, \"config.json\")\n with open(empty_config_json_path, \"wb\") as f:\n f.write(b\"{}\")\n\n # Upload the index.json file\n index_shasum = Digest.from_sha256(spack.util.crypto.checksum(hashlib.sha256, index_json_path))\n upload_blob_with_retry(image_ref, file=index_json_path, digest=index_shasum)\n\n # Upload the config.json file\n empty_config_digest = Digest.from_sha256(\n spack.util.crypto.checksum(hashlib.sha256, empty_config_json_path)\n )\n upload_blob_with_retry(image_ref, file=empty_config_json_path, digest=empty_config_digest)\n\n # Push a manifest file that references the index.json file as a layer\n # Notice that we push this as if it is an image, which it of course is not.\n # When the ORAS spec becomes official, we can use that instead of a fake image.\n # For now we just use the OCI image spec, so that we don't run into issues with\n # automatic garbage collection of blobs that are not referenced by any image manifest.\n oci_manifest = {\n \"mediaType\": \"application/vnd.oci.image.manifest.v1+json\",\n \"schemaVersion\": 2,\n # Config is just an empty {} file for now, and irrelevant\n \"config\": {\n \"mediaType\": \"application/vnd.oci.image.config.v1+json\",\n \"digest\": str(empty_config_digest),\n \"size\": os.path.getsize(empty_config_json_path),\n },\n # The buildcache index is the only layer, and is not a tarball, we lie here.\n \"layers\": [\n {\n \"mediaType\": \"application/vnd.oci.image.layer.v1.tar+gzip\",\n \"digest\": str(index_shasum),\n \"size\": os.path.getsize(index_json_path),\n }\n ],\n }\n\n upload_manifest_with_retry(image_ref.with_tag(default_index_tag), oci_manifest)\n\n\ndef try_fetch(url_to_fetch):\n \"\"\"Utility function to try and fetch a file from a url, stage it\n locally, and return the path to the staged file.\n\n Args:\n url_to_fetch (str): Url pointing to remote resource to fetch\n\n Returns:\n Path to locally staged resource or ``None`` if it could not be fetched.\n \"\"\"\n stage = Stage(url_to_fetch, keep=True)\n stage.create()\n\n try:\n stage.fetch()\n except spack.error.FetchError:\n stage.destroy()\n return None\n\n return stage\n\n\ndef download_tarball(\n spec: spack.spec.Spec,\n unsigned: Optional[bool] = False,\n mirrors_for_spec: Optional[List[MirrorMetadata]] = None,\n) -> Optional[spack.stage.Stage]:\n \"\"\"Download binary tarball for given package\n\n Args:\n spec: a concrete spec\n unsigned: if ``True`` or ``False`` override the mirror signature verification defaults\n mirrors_for_spec: Optional list of mirrors known to have the spec. These will be checked\n in order first before looking in other configured mirrors.\n\n Returns:\n ``None`` if the tarball could not be downloaded, the signature verified\n (if required), and its checksum validated. Otherwise, return the stage\n containing the downloaded tarball.\n \"\"\"\n configured_mirrors: Iterable[spack.mirrors.mirror.Mirror] = (\n spack.mirrors.mirror.MirrorCollection(binary=True).values()\n )\n if not configured_mirrors:\n raise NoConfiguredBinaryMirrors()\n\n # Note on try_first and try_next:\n # mirrors_for_spec mostly likely came from spack caching remote\n # mirror indices locally and adding their specs to a local data\n # structure supporting quick lookup of concrete specs. Those\n # mirrors are likely a subset of all configured mirrors, and\n # we'll probably find what we need in one of them. But we'll\n # look in all configured mirrors if needed, as maybe the spec\n # we need was in an un-indexed mirror. No need to check any\n # mirror for the spec twice though.\n try_first = mirrors_for_spec or []\n try_next = [\n MirrorMetadata(mirror.fetch_url, layout, mirror.fetch_view)\n for mirror in configured_mirrors\n for layout in mirror.supported_layout_versions\n ]\n urls_and_versions = try_first + [uv for uv in try_next if uv not in try_first]\n\n # TODO: turn `mirrors_for_spec` into a list of Mirror instances, instead of doing that here.\n def fetch_url_to_mirror(\n mirror_metadata: MirrorMetadata,\n ) -> Tuple[spack.mirrors.mirror.Mirror, int]:\n url = mirror_metadata.url\n layout_version = mirror_metadata.version\n for mirror in configured_mirrors:\n if mirror.fetch_url == url:\n return mirror, layout_version\n return spack.mirrors.mirror.Mirror(url), layout_version\n\n mirrors = [fetch_url_to_mirror(mirror_metadata) for mirror_metadata in urls_and_versions]\n\n for mirror, layout_version in mirrors:\n # Override mirror's default if\n currently_unsigned = unsigned if unsigned is not None else not mirror.signed\n\n # If it's an OCI index, do things differently, since we cannot compose URLs.\n fetch_url = mirror.fetch_url\n\n # TODO: refactor this to some \"nice\" place.\n if spack.oci.image.is_oci_url(fetch_url):\n ref = ImageReference.from_url(fetch_url).with_tag(_oci_default_tag(spec))\n\n # Fetch the manifest\n try:\n with spack.oci.opener.urlopen(\n urllib.request.Request(\n url=ref.manifest_url(),\n headers={\"Accept\": \", \".join(spack.oci.oci.manifest_content_type)},\n )\n ) as response:\n manifest = json.load(response)\n except Exception:\n continue\n\n # Download the config = spec.json and the relevant tarball\n try:\n spec_digest = spack.oci.image.Digest.from_string(manifest[\"config\"][\"digest\"])\n tarball_digest = spack.oci.image.Digest.from_string(\n manifest[\"layers\"][-1][\"digest\"]\n )\n except Exception:\n continue\n\n with spack.oci.oci.make_stage(\n ref.blob_url(spec_digest), spec_digest, keep=True\n ) as local_specfile_stage:\n try:\n local_specfile_stage.fetch()\n local_specfile_stage.check()\n try:\n get_valid_spec_file(\n local_specfile_stage.save_filename, CURRENT_BUILD_CACHE_LAYOUT_VERSION\n )\n except InvalidMetadataFile as e:\n tty.warn(\n f\"Ignoring binary package for {spec.name}/{spec.dag_hash()[:7]} \"\n f\"from {fetch_url} due to invalid metadata file: {e}\"\n )\n local_specfile_stage.destroy()\n continue\n except Exception:\n continue\n local_specfile_stage.cache_local()\n\n local_specfile_stage.destroy()\n\n with spack.oci.oci.make_stage(\n ref.blob_url(tarball_digest), tarball_digest, keep=True\n ) as tarball_stage:\n try:\n tarball_stage.fetch()\n tarball_stage.check()\n except Exception:\n continue\n tarball_stage.cache_local()\n\n return tarball_stage\n else:\n cache_type = get_url_buildcache_class(layout_version=layout_version)\n cache_entry = cache_type(fetch_url, spec, allow_unsigned=currently_unsigned)\n\n try:\n cache_entry.fetch_archive()\n except Exception as e:\n tty.debug(\n f\"Encountered error attempting to fetch archive for \"\n f\"{spec.name}/{spec.dag_hash()[:7]} from {fetch_url} \"\n f\"(v{layout_version}) due to {e}\"\n )\n cache_entry.destroy()\n continue\n\n if layout_version == 2:\n warn_v2_layout(fetch_url, \"Installing a spec\")\n\n return cache_entry.get_archive_stage()\n\n # Falling through the nested loops means we exhaustively searched\n # for all known kinds of spec files on all mirrors and did not find\n # an acceptable one for which we could download a tarball and (if\n # needed) verify a signature. So at this point, we will proceed to\n # install from source.\n return None\n\n\ndef dedupe_hardlinks_if_necessary(root, buildinfo):\n \"\"\"Updates a buildinfo dict for old archives that did not dedupe hardlinks. De-duping hardlinks\n is necessary when relocating files in parallel and in-place. This means we must preserve inodes\n when relocating.\"\"\"\n\n # New archives don't need this.\n if buildinfo.get(\"hardlinks_deduped\", False):\n return\n\n # Clearly we can assume that an inode is either in the\n # textfile or binary group, but let's just stick to\n # a single set of visited nodes.\n visited = set()\n\n # Note: we do *not* dedupe hardlinked symlinks, since\n # it seems difficult or even impossible to relink\n # symlinks while preserving inode.\n for key in (\"relocate_textfiles\", \"relocate_binaries\"):\n if key not in buildinfo:\n continue\n new_list = []\n for rel_path in buildinfo[key]:\n stat_result = os.lstat(os.path.join(root, rel_path))\n identifier = (stat_result.st_dev, stat_result.st_ino)\n if stat_result.st_nlink > 1:\n if identifier in visited:\n continue\n visited.add(identifier)\n new_list.append(rel_path)\n buildinfo[key] = new_list\n\n\ndef relocate_package(spec: spack.spec.Spec) -> None:\n \"\"\"Relocate binaries and text files in the given spec prefix, based on its buildinfo file.\"\"\"\n spec_prefix = str(spec.prefix)\n buildinfo = read_buildinfo_file(spec_prefix)\n old_layout_root = str(buildinfo[\"buildpath\"])\n\n # Warn about old style tarballs created with the --rel flag (removed in Spack v0.20)\n if buildinfo.get(\"relative_rpaths\", False):\n tty.warn(\n f\"Tarball for {spec} uses relative rpaths, which can cause library loading issues.\"\n )\n\n # In Spack 0.19 and older prefix_to_hash was the default and externals were not dropped, so\n # prefixes were not unique.\n if \"hash_to_prefix\" in buildinfo:\n hash_to_old_prefix = buildinfo[\"hash_to_prefix\"]\n elif \"prefix_to_hash\" in buildinfo:\n hash_to_old_prefix = {v: k for (k, v) in buildinfo[\"prefix_to_hash\"].items()}\n else:\n raise NewLayoutException(\n \"Package tarball was created from an install prefix with a different directory layout \"\n \"and an older buildcache create implementation. It cannot be relocated.\"\n )\n\n prefix_to_prefix: Dict[str, str] = {}\n\n if \"sbang_install_path\" in buildinfo:\n old_sbang_install_path = str(buildinfo[\"sbang_install_path\"])\n prefix_to_prefix[old_sbang_install_path] = spack.hooks.sbang.sbang_install_path()\n\n # First match specific prefix paths. Possibly the *local* install prefix of some dependency is\n # in an upstream, so we cannot assume the original spack store root can be mapped uniformly to\n # the new spack store root.\n\n # If the spec is spliced, we need to handle the simultaneous mapping from the old install_tree\n # to the new install_tree and from the build_spec to the spliced spec. Because foo.build_spec\n # is foo for any non-spliced spec, we can simplify by checking for spliced-in nodes by checking\n # for nodes not in the build_spec without any explicit check for whether the spec is spliced.\n # An analog in this algorithm is any spec that shares a name or provides the same virtuals in\n # the context of the relevant root spec. This ensures that the analog for a spec s is the spec\n # that s replaced when we spliced.\n relocation_specs = specs_to_relocate(spec)\n build_spec_ids = set(id(s) for s in spec.build_spec.traverse(deptype=dt.ALL & ~dt.BUILD))\n for s in relocation_specs:\n analog = s\n if id(s) not in build_spec_ids:\n analogs = [\n d\n for d in spec.build_spec.traverse(deptype=dt.ALL & ~dt.BUILD)\n if s._splice_match(d, self_root=spec, other_root=spec.build_spec)\n ]\n if analogs:\n # Prefer same-name analogs and prefer higher versions\n # This matches the preferences in spack.spec.Spec.splice, so we\n # will find same node\n analog = max(analogs, key=lambda a: (a.name == s.name, a.version))\n\n lookup_dag_hash = analog.dag_hash()\n if lookup_dag_hash in hash_to_old_prefix:\n old_dep_prefix = hash_to_old_prefix[lookup_dag_hash]\n prefix_to_prefix[old_dep_prefix] = str(s.prefix)\n\n # Only then add the generic fallback of install prefix -> install prefix.\n prefix_to_prefix[old_layout_root] = str(spack.store.STORE.layout.root)\n\n # Delete identity mappings from prefix_to_prefix\n prefix_to_prefix = {k: v for k, v in prefix_to_prefix.items() if k != v}\n\n # If there's nothing to relocate, we're done.\n if not prefix_to_prefix:\n return\n\n for old, new in prefix_to_prefix.items():\n tty.debug(f\"Relocating: {old} => {new}.\")\n\n # Old archives may have hardlinks repeated.\n dedupe_hardlinks_if_necessary(spec_prefix, buildinfo)\n\n # Text files containing the prefix text\n textfiles = [os.path.join(spec_prefix, f) for f in buildinfo[\"relocate_textfiles\"]]\n binaries = [os.path.join(spec_prefix, f) for f in buildinfo.get(\"relocate_binaries\")]\n links = [os.path.join(spec_prefix, f) for f in buildinfo.get(\"relocate_links\", [])]\n\n platform = spack.platforms.by_name(spec.platform)\n if \"macho\" in platform.binary_formats:\n relocate.relocate_macho_binaries(binaries, prefix_to_prefix)\n elif \"elf\" in platform.binary_formats:\n relocate.relocate_elf_binaries(binaries, prefix_to_prefix)\n\n relocate.relocate_links(links, prefix_to_prefix)\n relocate.relocate_text(textfiles, prefix_to_prefix)\n changed_files = relocate.relocate_text_bin(binaries, prefix_to_prefix)\n\n # Add ad-hoc signatures to patched macho files when on macOS.\n if \"macho\" in platform.binary_formats and sys.platform == \"darwin\":\n codesign = which(\"codesign\")\n if not codesign:\n return\n for binary in changed_files:\n # preserve the original inode by running codesign on a copy\n with fsys.edit_in_place_through_temporary_file(binary) as tmp_binary:\n codesign(\"-fs-\", tmp_binary)\n\n install_manifest = os.path.join(\n spec.prefix,\n spack.store.STORE.layout.metadata_dir,\n spack.store.STORE.layout.manifest_file_name,\n )\n if not os.path.exists(install_manifest):\n spec_id = spec.format(\"{name}/{hash:7}\")\n tty.warn(\"No manifest file in tarball for spec %s\" % spec_id)\n\n # overwrite old metadata with new\n if spec.spliced:\n # rewrite spec on disk\n spack.store.STORE.layout.write_spec(spec, spack.store.STORE.layout.spec_file_path(spec))\n\n # de-cache the install manifest\n with contextlib.suppress(FileNotFoundError):\n os.unlink(install_manifest)\n\n\ndef _tar_strip_component(tar: tarfile.TarFile, prefix: str):\n \"\"\"Yield all members of tarfile that start with given prefix, and strip that prefix (including\n symlinks)\"\"\"\n # Including trailing /, otherwise we end up with absolute paths.\n regex = re.compile(re.escape(prefix) + \"/*\")\n\n # Only yield members in the package prefix.\n # Note: when a tarfile is created, relative in-prefix symlinks are\n # expanded to matching member names of tarfile entries. So, we have\n # to ensure that those are updated too.\n # Absolute symlinks are copied verbatim -- relocation should take care of\n # them.\n for m in tar.getmembers():\n result = regex.match(m.name)\n if not result:\n continue\n m.name = m.name[result.end() :]\n if m.linkname:\n result = regex.match(m.linkname)\n if result:\n m.linkname = m.linkname[result.end() :]\n yield m\n\n\ndef extract_buildcache_tarball(tarfile_path: str, destination: str) -> None:\n with closing(tarfile.open(tarfile_path, \"r\")) as tar:\n # For consistent behavior across all supported Python versions\n tar.extraction_filter = lambda member, path: member\n # Remove common prefix from tarball entries and directly extract them to the install dir.\n tar.extractall(\n path=destination, members=_tar_strip_component(tar, prefix=_ensure_common_prefix(tar))\n )\n\n\ndef extract_tarball(spec, tarball_stage: spack.stage.Stage, force=False, timer=timer.NULL_TIMER):\n \"\"\"\n extract binary tarball for given package into install area\n \"\"\"\n timer.start(\"extract\")\n\n if os.path.exists(spec.prefix):\n if force:\n shutil.rmtree(spec.prefix)\n else:\n raise NoOverwriteException(str(spec.prefix))\n\n # Create the install prefix\n fsys.mkdirp(\n spec.prefix,\n mode=get_package_dir_permissions(spec),\n group=get_package_group(spec),\n default_perms=\"parents\",\n )\n\n tarfile_path = tarball_stage.save_filename\n\n try:\n extract_buildcache_tarball(tarfile_path, destination=spec.prefix)\n except Exception:\n shutil.rmtree(spec.prefix, ignore_errors=True)\n tarball_stage.destroy()\n raise\n\n timer.stop(\"extract\")\n timer.start(\"relocate\")\n\n try:\n relocate_package(spec)\n except Exception as e:\n shutil.rmtree(spec.prefix, ignore_errors=True)\n raise e\n finally:\n tarball_stage.destroy()\n\n timer.stop(\"relocate\")\n\n\ndef _ensure_common_prefix(tar: tarfile.TarFile) -> str:\n # Find the lowest `binary_distribution` file (hard-coded forward slash is on purpose).\n binary_distribution = min(\n (\n e.name\n for e in tar.getmembers()\n if e.isfile() and e.name.endswith(\".spack/binary_distribution\")\n ),\n key=len,\n default=None,\n )\n\n if binary_distribution is None:\n raise ValueError(\"Tarball is not a Spack package, missing binary_distribution file\")\n\n pkg_path = pathlib.PurePosixPath(binary_distribution).parent.parent\n\n # Even the most ancient Spack version has required to list the dir of the package itself, so\n # guard against broken tarballs where `path.parent.parent` is empty.\n if pkg_path == pathlib.PurePosixPath():\n raise ValueError(\"Invalid tarball, missing package prefix dir\")\n\n pkg_prefix = str(pkg_path)\n\n # Ensure all tar entries are in the pkg_prefix dir, and if they're not, they should be parent\n # dirs of it.\n has_prefix = False\n for member in tar.getmembers():\n stripped = member.name.rstrip(\"/\")\n if not (\n stripped.startswith(pkg_prefix) or member.isdir() and pkg_prefix.startswith(stripped)\n ):\n raise ValueError(f\"Tarball contains file {stripped} outside of prefix {pkg_prefix}\")\n if member.isdir() and stripped == pkg_prefix:\n has_prefix = True\n\n # This is technically not required, but let's be defensive about the existence of the package\n # prefix dir.\n if not has_prefix:\n raise ValueError(f\"Tarball does not contain a common prefix {pkg_prefix}\")\n\n return pkg_prefix\n\n\ndef install_root_node(\n spec: spack.spec.Spec,\n unsigned=False,\n force: bool = False,\n sha256: Optional[str] = None,\n allow_missing: bool = False,\n) -> None:\n \"\"\"Install the root node of a concrete spec from a buildcache.\n\n Checking the sha256 sum of a node before installation is usually needed only\n for software installed during Spack's bootstrapping (since we might not have\n a proper signature verification mechanism available).\n\n Args:\n spec: spec to be installed (note that only the root node will be installed)\n unsigned: if True allows installing unsigned binaries\n force: force installation if the spec is already present in the local store\n sha256: optional sha256 of the binary package, to be checked before installation\n allow_missing: when true, allows installing a node with missing dependencies\n \"\"\"\n # Early termination\n if spec.external or not spec.concrete:\n warnings.warn(\"Skipping external or abstract spec {0}\".format(spec.format()))\n return\n elif spec.installed and not force:\n warnings.warn(\"Package for spec {0} already installed.\".format(spec.format()))\n return\n\n tarball_stage = download_tarball(spec.build_spec, unsigned)\n if not tarball_stage:\n msg = 'download of binary cache file for spec \"{0}\" failed'\n raise RuntimeError(msg.format(spec.build_spec.format()))\n\n # don't print long padded paths while extracting/relocating binaries\n with spack.util.path.filter_padding():\n tty.msg('Installing \"{0}\" from a buildcache'.format(spec.format()))\n extract_tarball(spec, tarball_stage, force)\n spec.package.windows_establish_runtime_linkage()\n spack.hooks.post_install(spec, False)\n spack.store.STORE.db.add(spec, allow_missing=allow_missing)\n\n\ndef install_single_spec(spec, unsigned=False, force=False):\n \"\"\"Install a single concrete spec from a buildcache.\n\n Args:\n spec (spack.spec.Spec): spec to be installed\n unsigned (bool): if True allows installing unsigned binaries\n force (bool): force installation if the spec is already present in the\n local store\n \"\"\"\n for node in spec.traverse(root=True, order=\"post\", deptype=(\"link\", \"run\")):\n install_root_node(node, unsigned=unsigned, force=force)\n\n\ndef try_direct_fetch(spec: spack.spec.Spec) -> List[MirrorMetadata]:\n \"\"\"Try to find the spec directly on the configured mirrors\"\"\"\n found_specs: List[MirrorMetadata] = []\n binary_mirrors = spack.mirrors.mirror.MirrorCollection(binary=True).values()\n\n for mirror in binary_mirrors:\n # TODO: OCI-support\n if spack.oci.image.is_oci_url(mirror.fetch_url):\n continue\n\n for layout_version in mirror.supported_layout_versions:\n # layout_version could eventually come from the mirror config\n cache_class = get_url_buildcache_class(layout_version=layout_version)\n cache_entry = cache_class(mirror.fetch_url, spec)\n\n try:\n spec_dict = cache_entry.fetch_metadata()\n except BuildcacheEntryError:\n continue\n finally:\n cache_entry.destroy()\n\n # All specs in build caches are concrete (as they are built) so we need\n # to mark this spec concrete on read-in.\n fetched_spec = spack.spec.Spec.from_dict(spec_dict)\n fetched_spec._mark_concrete()\n\n found_specs.append(MirrorMetadata(mirror.fetch_url, layout_version, mirror.fetch_view))\n\n return found_specs\n\n\ndef get_mirrors_for_spec(spec: spack.spec.Spec, index_only: bool = False) -> List[MirrorMetadata]:\n \"\"\"\n Check if concrete spec exists on mirrors and return a list indicating the mirrors on which it\n can be found\n\n Args:\n spec: The spec to look for in binary mirrors\n index_only: When ``index_only`` is set to ``True``, only the local cache is checked, no\n requests are made.\n \"\"\"\n if not spack.mirrors.mirror.MirrorCollection(binary=True):\n tty.debug(\"No Spack mirrors are currently configured\")\n return []\n\n results = BINARY_INDEX.find_built_spec(spec)\n\n # The index may be out-of-date. If we aren't only considering indices, try\n # to fetch directly since we know where the file should be.\n if not results and not index_only:\n results = try_direct_fetch(spec)\n # We found a spec by the direct fetch approach, we might as well\n # add it to our mapping.\n if results:\n BINARY_INDEX.update_spec(spec, results)\n\n return results\n\n\ndef update_cache_and_get_specs():\n \"\"\"\n Get all concrete specs for build caches available on configured mirrors.\n Initialization of internal cache data structures is done as lazily as\n possible, so this method will also attempt to initialize and update the\n local index cache (essentially a no-op if it has been done already and\n nothing has changed on the configured mirrors.)\n\n Raises:\n FetchCacheError\n \"\"\"\n BINARY_INDEX.update()\n return BINARY_INDEX.get_all_built_specs()\n\n\ndef get_keys(\n install: bool = False,\n trust: bool = False,\n force: bool = False,\n mirrors: Optional[Mapping[str, spack.mirrors.mirror.Mirror]] = None,\n):\n \"\"\"Get pgp public keys available on mirror with suffix .pub\"\"\"\n mirror_collection = mirrors or spack.mirrors.mirror.MirrorCollection(binary=True)\n\n if not mirror_collection:\n tty.die(\"Please add a spack mirror to allow \" + \"download of build caches.\")\n\n fingerprints = []\n for mirror in mirror_collection.values():\n if not mirror.signed:\n # Don't bother fetching keys for unsigned mirrors\n continue\n for layout_version in mirror.supported_layout_versions:\n fetch_url = mirror.fetch_url\n if layout_version == 2:\n mirror_layout_fingerprints = _get_keys_v2(fetch_url, install, trust, force)\n else:\n mirror_layout_fingerprints = _get_keys(\n fetch_url, layout_version, install, trust, force\n )\n if mirror_layout_fingerprints:\n fingerprints.extend(mirror_layout_fingerprints)\n return fingerprints\n\n\ndef _get_keys(\n mirror_url: str,\n layout_version: int = CURRENT_BUILD_CACHE_LAYOUT_VERSION,\n install: bool = False,\n trust: bool = False,\n force: bool = False,\n) -> Optional[List[str]]:\n cache_class = get_url_buildcache_class(layout_version=layout_version)\n\n tty.debug(\"Finding public keys in {0}\".format(url_util.format(mirror_url)))\n\n keys_prefix = url_util.join(\n mirror_url, *cache_class.get_relative_path_components(BuildcacheComponent.KEY)\n )\n key_index_manifest_url = url_util.join(keys_prefix, \"keys.manifest.json\")\n index_entry = cache_class(mirror_url, allow_unsigned=True)\n\n try:\n index_manifest = index_entry.read_manifest(manifest_url=key_index_manifest_url)\n index_blob_path = index_entry.fetch_blob(index_manifest.data[0])\n except BuildcacheEntryError as e:\n tty.debug(f\"Failed to fetch key index due to: {e}\")\n index_entry.destroy()\n return None\n\n with open(index_blob_path, encoding=\"utf-8\") as fd:\n json_index = json.load(fd)\n index_entry.destroy()\n\n saved_fingerprints = []\n for fingerprint, _ in json_index[\"keys\"].items():\n key_manifest_url = url_util.join(keys_prefix, f\"{fingerprint}.key.manifest.json\")\n key_entry = cache_class(mirror_url, allow_unsigned=True)\n try:\n key_manifest = key_entry.read_manifest(manifest_url=key_manifest_url)\n key_blob_path = key_entry.fetch_blob(key_manifest.data[0])\n except BuildcacheEntryError as e:\n tty.debug(f\"Failed to fetch key {fingerprint} due to: {e}\")\n key_entry.destroy()\n continue\n\n tty.debug(\"Found key {0}\".format(fingerprint))\n if install:\n if trust:\n spack.util.gpg.trust(key_blob_path)\n tty.debug(f\"Added {fingerprint} to trusted keys.\")\n saved_fingerprints.append(fingerprint)\n else:\n tty.debug(\n \"Will not add this key to trusted keys.Use -t to install all downloaded keys\"\n )\n\n key_entry.destroy()\n return saved_fingerprints\n\n\ndef _get_keys_v2(mirror_url, install=False, trust=False, force=False) -> Optional[List[str]]:\n cache_class = get_url_buildcache_class(layout_version=2)\n\n keys_url = url_util.join(\n mirror_url, *cache_class.get_relative_path_components(BuildcacheComponent.KEY)\n )\n keys_index = url_util.join(keys_url, \"index.json\")\n\n tty.debug(\"Finding public keys in {0}\".format(url_util.format(mirror_url)))\n\n try:\n json_index = web_util.read_json(keys_index)\n except (web_util.SpackWebError, OSError, ValueError) as url_err:\n # TODO: avoid repeated request\n if web_util.url_exists(keys_index):\n tty.error(\n f\"Unable to find public keys in {url_util.format(mirror_url)},\"\n f\" caught exception attempting to read from {url_util.format(keys_index)}.\"\n )\n tty.error(url_err)\n return None\n\n saved_fingerprints = []\n for fingerprint, key_attributes in json_index[\"keys\"].items():\n link = os.path.join(keys_url, fingerprint + \".pub\")\n\n with Stage(link, name=\"build_cache\", keep=True) as stage:\n if os.path.exists(stage.save_filename) and force:\n os.remove(stage.save_filename)\n if not os.path.exists(stage.save_filename):\n try:\n stage.fetch()\n except spack.error.FetchError:\n continue\n\n tty.debug(\"Found key {0}\".format(fingerprint))\n if install:\n if trust:\n spack.util.gpg.trust(stage.save_filename)\n tty.debug(\"Added this key to trusted keys.\")\n saved_fingerprints.append(fingerprint)\n else:\n tty.debug(\n \"Will not add this key to trusted keys.Use -t to install all downloaded keys\"\n )\n return saved_fingerprints\n\n\ndef _url_push_keys(\n *mirrors: Union[spack.mirrors.mirror.Mirror, str],\n keys: List[str],\n tmpdir: str,\n update_index: bool = False,\n):\n \"\"\"Upload pgp public keys to the given mirrors\"\"\"\n keys = spack.util.gpg.public_keys(*(keys or ()))\n files = [os.path.join(tmpdir, f\"{key}.pub\") for key in keys]\n\n for key, file in zip(keys, files):\n spack.util.gpg.export_keys(file, [key])\n\n cache_class = get_url_buildcache_class()\n\n for mirror in mirrors:\n push_url = mirror if isinstance(mirror, str) else mirror.push_url\n\n tty.debug(f\"Pushing public keys to {url_util.format(push_url)}\")\n pushed_a_key = False\n\n for key, file in zip(keys, files):\n cache_class.push_local_file_as_blob(\n local_file_path=file,\n mirror_url=push_url,\n manifest_name=f\"{key}.key\",\n component_type=BuildcacheComponent.KEY,\n compression=\"none\",\n )\n pushed_a_key = True\n\n if update_index:\n generate_key_index(push_url, tmpdir=tmpdir)\n\n if pushed_a_key or update_index:\n cache_class.maybe_push_layout_json(push_url)\n\n\ndef needs_rebuild(spec, mirror_url):\n if not spec.concrete:\n raise ValueError(\"spec must be concrete to check against mirror\")\n\n pkg_name = spec.name\n pkg_version = spec.version\n pkg_hash = spec.dag_hash()\n\n tty.debug(\"Checking {0}-{1}, dag_hash = {2}\".format(pkg_name, pkg_version, pkg_hash))\n tty.debug(spec.tree())\n\n # Try to retrieve the specfile directly, based on the known\n # format of the name, in order to determine if the package\n # needs to be rebuilt.\n cache_class = get_url_buildcache_class(layout_version=CURRENT_BUILD_CACHE_LAYOUT_VERSION)\n cache_entry = cache_class(mirror_url, spec, allow_unsigned=True)\n exists = cache_entry.exists([BuildcacheComponent.SPEC, BuildcacheComponent.TARBALL])\n return not exists\n\n\ndef check_specs_against_mirrors(mirrors, specs, output_file=None):\n \"\"\"Check all the given specs against buildcaches on the given mirrors and\n determine if any of the specs need to be rebuilt. Specs need to be rebuilt\n when their hash doesn't exist in the mirror.\n\n Arguments:\n mirrors (dict): Mirrors to check against\n specs (typing.Iterable): Specs to check against mirrors\n output_file (str): Path to output file to be written. If provided,\n mirrors with missing or out-of-date specs will be formatted as a\n JSON object and written to this file.\n\n Returns: 1 if any spec was out-of-date on any mirror, 0 otherwise.\n\n \"\"\"\n rebuilds = {}\n for mirror in spack.mirrors.mirror.MirrorCollection(mirrors, binary=True).values():\n tty.debug(\"Checking for built specs at {0}\".format(mirror.fetch_url))\n\n rebuild_list = []\n\n for spec in specs:\n if needs_rebuild(spec, mirror.fetch_url):\n rebuild_list.append({\"short_spec\": spec.short_spec, \"hash\": spec.dag_hash()})\n\n if rebuild_list:\n rebuilds[mirror.fetch_url] = {\n \"mirrorName\": mirror.name,\n \"mirrorUrl\": mirror.fetch_url,\n \"rebuildSpecs\": rebuild_list,\n }\n\n if output_file:\n with open(output_file, \"w\", encoding=\"utf-8\") as outf:\n outf.write(json.dumps(rebuilds))\n\n return 1 if rebuilds else 0\n\n\ndef download_single_spec(\n concrete_spec,\n destination,\n mirror_url=None,\n layout_version: int = CURRENT_BUILD_CACHE_LAYOUT_VERSION,\n):\n \"\"\"Download the buildcache files for a single concrete spec.\n\n Args:\n concrete_spec: concrete spec to be downloaded\n destination (str): path where to put the downloaded buildcache\n mirror_url (str): url of the mirror from which to download\n \"\"\"\n if not mirror_url and not spack.mirrors.mirror.MirrorCollection(binary=True):\n tty.die(\n \"Please provide or add a spack mirror to allow \" + \"download of buildcache entries.\"\n )\n\n urls = (\n [mirror_url]\n if mirror_url\n else [\n mirror.fetch_url\n for mirror in spack.mirrors.mirror.MirrorCollection(binary=True).values()\n ]\n )\n\n mkdirp(destination)\n\n for url in urls:\n cache_class = get_url_buildcache_class(layout_version=layout_version)\n cache_entry = cache_class(url, concrete_spec, allow_unsigned=True)\n\n try:\n cache_entry.fetch_metadata()\n cache_entry.fetch_archive()\n except BuildcacheEntryError as e:\n tty.warn(f\"Error downloading {concrete_spec.name}/{concrete_spec.dag_hash()[:7]}: {e}\")\n cache_entry.destroy()\n continue\n\n shutil.move(cache_entry.get_local_spec_path(), destination)\n shutil.move(cache_entry.get_local_archive_path(), destination)\n return True\n\n return False\n\n\nclass BinaryCacheQuery:\n \"\"\"Callable object to query if a spec is in a binary cache\"\"\"\n\n def __init__(self, all_architectures):\n \"\"\"\n Args:\n all_architectures (bool): if True consider all the spec for querying,\n otherwise restrict to the current default architecture\n \"\"\"\n self.all_architectures = all_architectures\n\n specs = update_cache_and_get_specs()\n\n if not self.all_architectures:\n arch = spack.spec.Spec.default_arch()\n specs = [s for s in specs if s.satisfies(arch)]\n\n self.possible_specs = specs\n\n def __call__(self, spec: spack.spec.Spec, **kwargs):\n \"\"\"\n Args:\n spec: The spec being searched for\n \"\"\"\n return [s for s in self.possible_specs if s.satisfies(spec)]\n\n\nclass FetchIndexError(Exception):\n def __str__(self):\n if len(self.args) == 1:\n return str(self.args[0])\n else:\n return \"{}, due to: {}\".format(self.args[0], self.args[1])\n\n\nclass BuildcacheIndexError(spack.error.SpackError):\n \"\"\"Raised when a buildcache cannot be read for any reason\"\"\"\n\n\nclass BuildcacheIndexNotExists(Exception):\n \"\"\"Buildcache does not contain an index\"\"\"\n\n\nFetchIndexResult = collections.namedtuple(\"FetchIndexResult\", \"etag hash data fresh\")\n\n\nclass IndexFetcher:\n def conditional_fetch(self) -> FetchIndexResult:\n raise NotImplementedError(f\"{self.__class__.__name__} is abstract\")\n\n def get_index_manifest(self, manifest_response) -> BlobRecord:\n \"\"\"Read the response of the manifest request and return a BlobRecord\"\"\"\n cache_class = get_url_buildcache_class(CURRENT_BUILD_CACHE_LAYOUT_VERSION)\n try:\n result = io.TextIOWrapper(manifest_response, encoding=\"utf-8\").read()\n except (ValueError, OSError) as e:\n raise FetchIndexError(f\"Remote index {manifest_response.url} is invalid\", e) from e\n\n manifest = BuildcacheManifest.from_dict(\n # Currently we do not sign buildcache index, but we could\n cache_class.verify_and_extract_manifest(result, verify=False)\n )\n blob_record = manifest.get_blob_records(\n cache_class.component_to_media_type(BuildcacheComponent.INDEX)\n )[0]\n return blob_record\n\n def fetch_index_blob(\n self, cache_entry: URLBuildcacheEntry, blob_record: BlobRecord\n ) -> Tuple[str, str]:\n \"\"\"Fetch the index blob indicated by the BlobRecord, and return the\n (checksum, contents) of the blob\"\"\"\n try:\n staged_blob_path = cache_entry.fetch_blob(blob_record)\n except BuildcacheEntryError as e:\n cache_entry.destroy()\n raise FetchIndexError(\n f\"Could not fetch index blob from {cache_entry.mirror_url}\"\n ) from e\n\n with open(staged_blob_path, encoding=\"utf-8\") as fd:\n blob_result = fd.read()\n\n computed_hash = compute_hash(blob_result)\n\n if computed_hash != blob_record.checksum:\n cache_entry.destroy()\n raise FetchIndexError(f\"Remote index at {cache_entry.mirror_url} is invalid\")\n\n return (computed_hash, blob_result)\n\n\nclass DefaultIndexFetcherV2(IndexFetcher):\n \"\"\"Fetcher for index.json, using separate index.json.hash as cache invalidation strategy\"\"\"\n\n def __init__(self, url, local_hash, urlopen=web_util.urlopen):\n self.url = url\n self.local_hash = local_hash\n self.urlopen = urlopen\n self.headers = {\"User-Agent\": web_util.SPACK_USER_AGENT}\n\n def get_remote_hash(self):\n # Failure to fetch index.json.hash is not fatal\n url_index_hash = url_util.join(self.url, \"build_cache\", \"index.json.hash\")\n try:\n with self.urlopen(\n urllib.request.Request(url_index_hash, headers=self.headers)\n ) as response:\n remote_hash = response.read(64)\n except OSError:\n return None\n\n # Validate the hash\n if not re.match(rb\"[a-f\\d]{64}$\", remote_hash):\n return None\n return remote_hash.decode(\"utf-8\")\n\n def conditional_fetch(self) -> FetchIndexResult:\n # Do an intermediate fetch for the hash\n # and a conditional fetch for the contents\n\n # Early exit if our cache is up to date.\n if self.local_hash and self.local_hash == self.get_remote_hash():\n return FetchIndexResult(etag=None, hash=None, data=None, fresh=True)\n\n # Otherwise, download index.json\n url_index = url_util.join(self.url, \"build_cache\", spack.database.INDEX_JSON_FILE)\n\n try:\n response = self.urlopen(urllib.request.Request(url_index, headers=self.headers))\n except OSError as e:\n raise FetchIndexError(f\"Could not fetch index from {url_index}\", e) from e\n\n with response:\n try:\n result = io.TextIOWrapper(response, encoding=\"utf-8\").read()\n except (ValueError, OSError) as e:\n raise FetchIndexError(f\"Remote index {url_index} is invalid\") from e\n\n # For now we only handle etags on http(s), since 304 error handling\n # in s3:// is not there yet.\n if urllib.parse.urlparse(self.url).scheme not in (\"http\", \"https\"):\n etag = None\n else:\n etag = web_util.parse_etag(\n response.headers.get(\"Etag\", None) or response.headers.get(\"etag\", None)\n )\n\n computed_hash = compute_hash(result)\n\n # We don't handle computed_hash != remote_hash here, which can happen\n # when remote index.json and index.json.hash are out of sync, or if\n # the hash algorithm changed.\n # The most likely scenario is that we got index.json got updated\n # while we fetched index.json.hash. Warning about an issue thus feels\n # wrong, as it's more of an issue with race conditions in the cache\n # invalidation strategy.\n\n warn_v2_layout(self.url, \"Fetching an index\")\n\n return FetchIndexResult(etag=etag, hash=computed_hash, data=result, fresh=False)\n\n\nclass EtagIndexFetcherV2(IndexFetcher):\n \"\"\"Fetcher for index.json, using ETags headers as cache invalidation strategy\"\"\"\n\n def __init__(self, url, etag, urlopen=web_util.urlopen):\n self.url = url\n self.etag = etag\n self.urlopen = urlopen\n\n def conditional_fetch(self) -> FetchIndexResult:\n # Just do a conditional fetch immediately\n url = url_util.join(self.url, \"build_cache\", spack.database.INDEX_JSON_FILE)\n headers = {\"User-Agent\": web_util.SPACK_USER_AGENT, \"If-None-Match\": f'\"{self.etag}\"'}\n\n try:\n response = self.urlopen(urllib.request.Request(url, headers=headers))\n except urllib.error.HTTPError as e:\n if e.getcode() == 304:\n # Not modified; that means fresh.\n return FetchIndexResult(etag=None, hash=None, data=None, fresh=True)\n raise FetchIndexError(f\"Could not fetch index {url}\", e) from e\n except OSError as e: # URLError, socket.timeout, etc.\n raise FetchIndexError(f\"Could not fetch index {url}\", e) from e\n\n with response:\n try:\n result = io.TextIOWrapper(response, encoding=\"utf-8\").read()\n except (ValueError, OSError) as e:\n raise FetchIndexError(f\"Remote index {url} is invalid\", e) from e\n\n warn_v2_layout(self.url, \"Fetching an index\")\n\n etag_header_value = response.headers.get(\"Etag\", None) or response.headers.get(\n \"etag\", None\n )\n\n return FetchIndexResult(\n etag=web_util.parse_etag(etag_header_value),\n hash=compute_hash(result),\n data=result,\n fresh=False,\n )\n\n\nclass OCIIndexFetcher(IndexFetcher):\n def __init__(self, mirror_metadata: MirrorMetadata, local_hash, urlopen=None) -> None:\n self.local_hash = local_hash\n self.ref = spack.oci.image.ImageReference.from_url(mirror_metadata.url)\n self.urlopen = urlopen or spack.oci.opener.urlopen\n\n def conditional_fetch(self) -> FetchIndexResult:\n \"\"\"Download an index from an OCI registry type mirror.\"\"\"\n url_manifest = self.ref.with_tag(default_index_tag).manifest_url()\n try:\n response = self.urlopen(\n urllib.request.Request(\n url=url_manifest,\n headers={\"Accept\": \"application/vnd.oci.image.manifest.v1+json\"},\n )\n )\n except OSError as e:\n raise FetchIndexError(f\"Could not fetch manifest from {url_manifest}\", e) from e\n\n with response:\n try:\n manifest = json.load(response)\n except Exception as e:\n raise FetchIndexError(f\"Remote index {url_manifest} is invalid\", e) from e\n\n # Get first blob hash, which should be the index.json\n try:\n index_digest = spack.oci.image.Digest.from_string(manifest[\"layers\"][0][\"digest\"])\n except Exception as e:\n raise FetchIndexError(f\"Remote index {url_manifest} is invalid\", e) from e\n\n # Fresh?\n if index_digest.digest == self.local_hash:\n return FetchIndexResult(etag=None, hash=None, data=None, fresh=True)\n\n # Otherwise fetch the blob / index.json\n try:\n with self.urlopen(\n urllib.request.Request(\n url=self.ref.blob_url(index_digest),\n headers={\"Accept\": \"application/vnd.oci.image.layer.v1.tar+gzip\"},\n )\n ) as response:\n result = io.TextIOWrapper(response, encoding=\"utf-8\").read()\n except (OSError, ValueError) as e:\n raise FetchIndexError(f\"Remote index {url_manifest} is invalid\", e) from e\n\n # Make sure the blob we download has the advertised hash\n if compute_hash(result) != index_digest.digest:\n raise FetchIndexError(f\"Remote index {url_manifest} is invalid\")\n\n return FetchIndexResult(etag=None, hash=index_digest.digest, data=result, fresh=False)\n\n\nclass DefaultIndexFetcher(IndexFetcher):\n \"\"\"Fetcher for buildcache index, cache invalidation via manifest contents\"\"\"\n\n def __init__(self, mirror_metadata: MirrorMetadata, local_hash, urlopen=web_util.urlopen):\n self.url = mirror_metadata.url\n self.view = mirror_metadata.view\n self.layout_version = mirror_metadata.version\n self.local_hash = local_hash\n self.urlopen = urlopen\n self.headers = {\"User-Agent\": web_util.SPACK_USER_AGENT}\n\n def conditional_fetch(self) -> FetchIndexResult:\n cache_class = get_url_buildcache_class(layout_version=self.layout_version)\n url_index_manifest = cache_class.get_index_url(self.url, self.view)\n\n try:\n response = self.urlopen(\n urllib.request.Request(url_index_manifest, headers=self.headers)\n )\n except OSError as e:\n raise FetchIndexError(\n f\"Could not read index manifest from {url_index_manifest}\"\n ) from e\n\n with response:\n index_blob_record = self.get_index_manifest(response)\n\n # Early exit if our cache is up to date.\n if self.local_hash and self.local_hash == index_blob_record.checksum:\n return FetchIndexResult(etag=None, hash=None, data=None, fresh=True)\n\n # Otherwise, download the index blob\n cache_entry = cache_class(self.url, allow_unsigned=True)\n computed_hash, result = self.fetch_index_blob(cache_entry, index_blob_record)\n cache_entry.destroy()\n\n # For now we only handle etags on http(s), since 304 error handling\n # in s3:// is not there yet.\n if urllib.parse.urlparse(self.url).scheme not in (\"http\", \"https\"):\n etag = None\n else:\n etag = web_util.parse_etag(\n response.headers.get(\"Etag\", None) or response.headers.get(\"etag\", None)\n )\n\n return FetchIndexResult(etag=etag, hash=computed_hash, data=result, fresh=False)\n\n\nclass EtagIndexFetcher(IndexFetcher):\n \"\"\"Fetcher for buildcache index, cache invalidation via ETags headers\n\n This class differs from the :class:`DefaultIndexFetcher` in the following ways:\n\n 1. It is provided with an etag value on creation, rather than an index checksum value. Note\n that since we never start out with an etag, the default fetcher must have been used initially\n and determined that the etag approach is valid.\n 2. It provides this etag value in the ``If-None-Match`` request header for the\n index manifest.\n 3. It checks for special exception type and response code indicating the index manifest is not\n modified, exiting early and returning ``Fresh``, if encountered.\n 4. If it needs to actually read the manifest, it does not need to do any checks of the url\n scheme to determine whether an etag should be included in the return value.\"\"\"\n\n def __init__(self, mirror_metadata: MirrorMetadata, etag, urlopen=web_util.urlopen):\n self.url = mirror_metadata.url\n self.view = mirror_metadata.view\n self.layout_version = mirror_metadata.version\n self.etag = etag\n self.urlopen = urlopen\n\n def conditional_fetch(self) -> FetchIndexResult:\n # Do a conditional fetch of the index manifest (i.e. using If-None-Match header)\n cache_class = get_url_buildcache_class(layout_version=self.layout_version)\n manifest_url = cache_class.get_index_url(self.url, self.view)\n headers = {\"User-Agent\": web_util.SPACK_USER_AGENT, \"If-None-Match\": f'\"{self.etag}\"'}\n\n try:\n response = self.urlopen(urllib.request.Request(manifest_url, headers=headers))\n except urllib.error.HTTPError as e:\n if e.getcode() == 304:\n # The remote manifest has not been modified, i.e. the index we\n # already have is the freshest there is.\n return FetchIndexResult(etag=None, hash=None, data=None, fresh=True)\n raise FetchIndexError(f\"Could not fetch index manifest {manifest_url}\", e) from e\n except OSError as e: # URLError, socket.timeout, etc.\n raise FetchIndexError(f\"Could not fetch index manifest {manifest_url}\", e) from e\n\n # We need to read the index manifest and fetch the associated blob\n with response:\n index_blob_record = self.get_index_manifest(response)\n etag_header_value = response.headers.get(\"Etag\", None) or response.headers.get(\n \"etag\", None\n )\n\n cache_entry = cache_class(self.url, allow_unsigned=True)\n computed_hash, result = self.fetch_index_blob(cache_entry, index_blob_record)\n cache_entry.destroy()\n\n return FetchIndexResult(\n etag=web_util.parse_etag(etag_header_value),\n hash=computed_hash,\n data=result,\n fresh=False,\n )\n\n\ndef get_index_fetcher(\n scheme: str, mirror_metadata: MirrorMetadata, cache_entry: Dict[str, str]\n) -> IndexFetcher:\n if scheme == \"oci\":\n # TODO: Actually etag and OCI are not mutually exclusive...\n return OCIIndexFetcher(mirror_metadata, cache_entry.get(\"index_hash\", None))\n elif cache_entry.get(\"etag\"):\n if mirror_metadata.version < 3:\n return EtagIndexFetcherV2(mirror_metadata.url, cache_entry[\"etag\"])\n else:\n return EtagIndexFetcher(mirror_metadata, cache_entry[\"etag\"])\n\n else:\n if mirror_metadata.version < 3:\n return DefaultIndexFetcherV2(\n mirror_metadata.url, local_hash=cache_entry.get(\"index_hash\", None)\n )\n else:\n return DefaultIndexFetcher(\n mirror_metadata, local_hash=cache_entry.get(\"index_hash\", None)\n )\n\n\nclass NoOverwriteException(spack.error.SpackError):\n \"\"\"Raised when a file would be overwritten\"\"\"\n\n def __init__(self, file_path):\n super().__init__(f\"Refusing to overwrite the following file: {file_path}\")\n\n\nclass NoGpgException(spack.error.SpackError):\n \"\"\"\n Raised when gpg2 is not in PATH\n \"\"\"\n\n def __init__(self, msg):\n super().__init__(msg)\n\n\nclass NoKeyException(spack.error.SpackError):\n \"\"\"\n Raised when gpg has no default key added.\n \"\"\"\n\n def __init__(self, msg):\n super().__init__(msg)\n\n\nclass PickKeyException(spack.error.SpackError):\n \"\"\"\n Raised when multiple keys can be used to sign.\n \"\"\"\n\n def __init__(self, keys):\n err_msg = \"Multiple keys available for signing\\n%s\\n\" % keys\n err_msg += \"Use spack buildcache create -k to pick a key.\"\n super().__init__(err_msg)\n\n\nclass NewLayoutException(spack.error.SpackError):\n \"\"\"\n Raised if directory layout is different from buildcache.\n \"\"\"\n\n def __init__(self, msg):\n super().__init__(msg)\n\n\nclass UnsignedPackageException(spack.error.SpackError):\n \"\"\"\n Raised if installation of unsigned package is attempted without\n the use of ``--no-check-signature``.\n \"\"\"\n\n\nclass GenerateIndexError(spack.error.SpackError):\n \"\"\"Raised when unable to generate key or package index for mirror\"\"\"\n\n\nclass CannotListKeys(GenerateIndexError):\n \"\"\"Raised when unable to list keys when generating key index\"\"\"\n\n\nclass PushToBuildCacheError(spack.error.SpackError):\n \"\"\"Raised when unable to push objects to binary mirror\"\"\"\n\n\nclass NoConfiguredBinaryMirrors(spack.error.SpackError):\n \"\"\"Raised when no binary mirrors are configured but an operation requires them\"\"\"\n\n def __init__(self):\n super().__init__(\"Please add a spack mirror to allow download of pre-compiled packages.\")\n" }, { "path": "lib/spack/spack/bootstrap/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Function and classes needed to bootstrap Spack itself.\"\"\"\n\nfrom .config import ensure_bootstrap_configuration, is_bootstrapping, store_path\nfrom .core import (\n all_core_root_specs,\n ensure_clingo_importable_or_raise,\n ensure_core_dependencies,\n ensure_gpg_in_path_or_raise,\n ensure_patchelf_in_path_or_raise,\n ensure_winsdk_external_or_raise,\n)\nfrom .environment import BootstrapEnvironment, ensure_environment_dependencies\nfrom .status import status_message\n\n__all__ = [\n \"all_core_root_specs\",\n \"BootstrapEnvironment\",\n \"ensure_bootstrap_configuration\",\n \"ensure_clingo_importable_or_raise\",\n \"ensure_core_dependencies\",\n \"ensure_environment_dependencies\",\n \"ensure_gpg_in_path_or_raise\",\n \"ensure_patchelf_in_path_or_raise\",\n \"ensure_winsdk_external_or_raise\",\n \"is_bootstrapping\",\n \"status_message\",\n \"store_path\",\n]\n" }, { "path": "lib/spack/spack/bootstrap/_common.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Common basic functions used through the spack.bootstrap package\"\"\"\n\nimport fnmatch\nimport glob\nimport importlib\nimport os\nimport re\nimport sys\nimport sysconfig\nimport warnings\nfrom typing import Optional, Sequence, Union\n\nimport spack.vendor.archspec.cpu\nfrom spack.vendor.typing_extensions import TypedDict\n\nimport spack.llnl.util.filesystem as fs\nimport spack.platforms\nimport spack.spec\nimport spack.store\nimport spack.util.environment\nimport spack.util.executable\nfrom spack.llnl.util import tty\n\nfrom .config import spec_for_current_python\n\n\nclass QueryInfo(TypedDict, total=False):\n spec: spack.spec.Spec\n command: spack.util.executable.Executable\n\n\ndef _python_import(module: str) -> bool:\n try:\n importlib.import_module(module)\n except ImportError:\n return False\n return True\n\n\ndef _try_import_from_store(\n module: str, query_spec: Union[str, \"spack.spec.Spec\"], query_info: Optional[QueryInfo] = None\n) -> bool:\n \"\"\"Return True if the module can be imported from an already\n installed spec, False otherwise.\n\n Args:\n module: Python module to be imported\n query_spec: spec that may provide the module\n query_info (dict or None): if a dict is passed it is populated with the\n command found and the concrete spec providing it\n \"\"\"\n # If it is a string assume it's one of the root specs by this module\n if isinstance(query_spec, str):\n # We have to run as part of this python interpreter\n query_spec += \" ^\" + spec_for_current_python()\n\n installed_specs = spack.store.STORE.db.query(query_spec, installed=True)\n\n for candidate_spec in installed_specs:\n # previously bootstrapped specs may not have a python-venv dependency.\n if candidate_spec.dependencies(\"python-venv\"):\n python, *_ = candidate_spec.dependencies(\"python-venv\")\n else:\n python, *_ = candidate_spec.dependencies(\"python\")\n\n # if python is installed, ask it for the layout\n if python.installed:\n module_paths = [\n os.path.join(candidate_spec.prefix, python.package.purelib),\n os.path.join(candidate_spec.prefix, python.package.platlib),\n ]\n # otherwise search for the site-packages directory\n # (clingo from binaries with truncated python-venv runtime)\n else:\n module_paths = glob.glob(\n os.path.join(candidate_spec.prefix, \"lib\", \"python*\", \"site-packages\")\n )\n path_before = list(sys.path)\n\n # NOTE: try module_paths first and last, last allows an existing version in path\n # to be picked up and used, possibly depending on something in the store, first\n # allows the bootstrap version to work when an incompatible version is in\n # sys.path\n orders = [module_paths + sys.path, sys.path + module_paths]\n for path in orders:\n sys.path = path\n try:\n _fix_ext_suffix(candidate_spec)\n if _python_import(module):\n msg = (\n f\"[BOOTSTRAP MODULE {module}] The installed spec \"\n f'\"{query_spec}/{candidate_spec.dag_hash()}\" '\n f'provides the \"{module}\" Python module'\n )\n tty.debug(msg)\n if query_info is not None:\n query_info[\"spec\"] = candidate_spec\n return True\n except Exception as exc: # pylint: disable=broad-except\n msg = (\n \"unexpected error while trying to import module \"\n f'\"{module}\" from spec \"{candidate_spec}\" [error=\"{str(exc)}\"]'\n )\n warnings.warn(msg)\n else:\n msg = \"Spec {0} did not provide module {1}\"\n warnings.warn(msg.format(candidate_spec, module))\n\n sys.path = path_before\n\n return False\n\n\ndef _fix_ext_suffix(candidate_spec: \"spack.spec.Spec\"):\n \"\"\"Fix the external suffixes of Python extensions on the fly for\n platforms that may need it\n\n Args:\n candidate_spec (Spec): installed spec with a Python module\n to be checked.\n \"\"\"\n # Here we map target families to the patterns expected\n # by pristine CPython. Only architectures with known issues\n # are included. Known issues:\n #\n # [RHEL + ppc64le]: https://github.com/spack/spack/issues/25734\n #\n _suffix_to_be_checked = {\n \"ppc64le\": {\n \"glob\": \"*.cpython-*-powerpc64le-linux-gnu.so\",\n \"re\": r\".cpython-[\\w]*-powerpc64le-linux-gnu.so\",\n \"fmt\": r\"{module}.cpython-{major}{minor}m-powerpc64le-linux-gnu.so\",\n }\n }\n\n # If the current architecture is not problematic return\n generic_target = spack.vendor.archspec.cpu.host().family\n if str(generic_target) not in _suffix_to_be_checked:\n return\n\n # If there's no EXT_SUFFIX (Python < 3.5) or the suffix matches\n # the expectations, return since the package is surely good\n ext_suffix = sysconfig.get_config_var(\"EXT_SUFFIX\")\n if ext_suffix is None:\n return\n\n expected = _suffix_to_be_checked[str(generic_target)]\n if fnmatch.fnmatch(ext_suffix, expected[\"glob\"]):\n return\n\n # If we are here it means the current interpreter expects different names\n # than pristine CPython. So:\n # 1. Find what we have installed\n # 2. Create symbolic links for the other names, it they're not there already\n\n # Check if standard names are installed and if we have to create\n # link for this interpreter\n standard_extensions = fs.find(candidate_spec.prefix, expected[\"glob\"])\n link_names = [re.sub(expected[\"re\"], ext_suffix, s) for s in standard_extensions]\n for file_name, link_name in zip(standard_extensions, link_names):\n if os.path.exists(link_name):\n continue\n os.symlink(file_name, link_name)\n\n # Check if this interpreter installed something and we have to create\n # links for a standard CPython interpreter\n non_standard_extensions = fs.find(candidate_spec.prefix, \"*\" + ext_suffix)\n for abs_path in non_standard_extensions:\n directory, filename = os.path.split(abs_path)\n module = filename.split(\".\")[0]\n link_name = os.path.join(\n directory,\n expected[\"fmt\"].format(\n module=module, major=sys.version_info[0], minor=sys.version_info[1]\n ),\n )\n if os.path.exists(link_name):\n continue\n os.symlink(abs_path, link_name)\n\n\ndef _executables_in_store(\n executables: Sequence[str],\n query_spec: Union[\"spack.spec.Spec\", str],\n query_info: Optional[QueryInfo] = None,\n) -> bool:\n \"\"\"Return True if at least one of the executables can be retrieved from\n a spec in store, False otherwise.\n\n The different executables must provide the same functionality and are\n \"alternate\" to each other, i.e. the function will exit True on the first\n executable found.\n\n Args:\n executables: list of executables to be searched\n query_spec: spec that may provide the executable\n query_info (dict or None): if a dict is passed it is populated with the\n command found and the concrete spec providing it\n \"\"\"\n executables_str = \", \".join(executables)\n msg = \"[BOOTSTRAP EXECUTABLES {0}] Try installed specs with query '{1}'\"\n tty.debug(msg.format(executables_str, query_spec))\n installed_specs = spack.store.STORE.db.query(query_spec, installed=True)\n if installed_specs:\n for concrete_spec in installed_specs:\n bin_dir = concrete_spec.prefix.bin\n # IF we have a \"bin\" directory and it contains\n # the executables we are looking for\n if (\n os.path.exists(bin_dir)\n and os.path.isdir(bin_dir)\n and spack.util.executable.which_string(*executables, path=bin_dir)\n ):\n spack.util.environment.path_put_first(\"PATH\", [bin_dir])\n if query_info is not None:\n query_info[\"command\"] = spack.util.executable.which(\n *executables, path=bin_dir, required=True\n )\n query_info[\"spec\"] = concrete_spec\n return True\n return False\n\n\ndef _root_spec(spec_str: str) -> str:\n \"\"\"Add a proper compiler and target to a spec used during bootstrapping.\n\n Args:\n spec_str: spec to be bootstrapped. Must be without compiler and target.\n \"\"\"\n # Add a compiler and platform requirement to the root spec.\n platform = str(spack.platforms.host())\n\n spec_str += f\" platform={platform}\"\n target = spack.vendor.archspec.cpu.host().family\n spec_str += f\" target={target}\"\n\n tty.debug(f\"[BOOTSTRAP ROOT SPEC] {spec_str}\")\n return spec_str\n" }, { "path": "lib/spack/spack/bootstrap/clingo.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Bootstrap concrete specs for clingo\n\nSpack uses clingo to concretize specs. When clingo itself needs to be bootstrapped from sources,\nwe need to rely on another mechanism to get a concrete spec that fits the current host.\n\nThis module contains the logic to get a concrete spec for clingo, starting from a prototype\nJSON file for a similar platform.\n\"\"\"\n\nimport pathlib\nimport sys\nfrom typing import Dict, Optional, Tuple, Type\n\nimport spack.vendor.archspec.cpu\n\nimport spack.compilers.config\nimport spack.compilers.libraries\nimport spack.config\nimport spack.package_base\nimport spack.platforms\nimport spack.repo\nimport spack.spec\nimport spack.traverse\nimport spack.version\n\nfrom .config import spec_for_current_python\n\n\ndef _select_best_version(\n pkg_cls: Type[\"spack.package_base.PackageBase\"], node: spack.spec.Spec, valid_versions: str\n) -> None:\n \"\"\"Try to attach the best known version to a node\"\"\"\n constraint = spack.version.from_string(valid_versions)\n allowed_versions = [v for v in pkg_cls.versions if v.satisfies(constraint)]\n try:\n best_version = spack.package_base.sort_by_pkg_preference(allowed_versions, pkg=pkg_cls)[0]\n except (KeyError, ValueError, IndexError):\n return\n node.versions.versions = [spack.version.from_string(f\"={best_version}\")]\n\n\ndef _add_compilers_if_missing() -> None:\n arch = spack.spec.ArchSpec.default_arch()\n if not spack.compilers.config.compilers_for_arch(arch):\n spack.compilers.config.find_compilers()\n\n\nclass ClingoBootstrapConcretizer:\n def __init__(self, configuration):\n _add_compilers_if_missing()\n self.host_platform = spack.platforms.host()\n self.host_os = self.host_platform.default_operating_system()\n self.host_target = spack.vendor.archspec.cpu.host().family\n self.host_architecture = spack.spec.ArchSpec.default_arch()\n self.host_architecture.target = str(self.host_target)\n self.host_compiler = self._valid_compiler_or_raise()\n self.host_python = self.python_external_spec()\n if str(self.host_platform) == \"linux\":\n self.host_libc = self.libc_external_spec()\n\n self.external_cmake, self.external_bison = self._externals_from_yaml(configuration)\n\n def _valid_compiler_or_raise(self):\n if str(self.host_platform) == \"linux\":\n compiler_name = \"gcc\"\n elif str(self.host_platform) == \"darwin\":\n compiler_name = \"apple-clang\"\n elif str(self.host_platform) == \"windows\":\n compiler_name = \"msvc\"\n elif str(self.host_platform) == \"freebsd\":\n compiler_name = \"llvm\"\n else:\n raise RuntimeError(f\"Cannot bootstrap clingo from sources on {self.host_platform}\")\n\n candidates = [\n x\n for x in spack.compilers.config.CompilerFactory.from_packages_yaml(spack.config.CONFIG)\n if x.name == compiler_name\n ]\n if not candidates:\n raise RuntimeError(\n f\"Cannot find any version of {compiler_name} to bootstrap clingo from sources\"\n )\n candidates.sort(key=lambda x: x.version, reverse=True)\n best = candidates[0]\n # Get compilers for bootstrapping from the 'builtin' repository\n best.namespace = \"builtin\"\n # If the compiler does not support C++ 14, fail with a legible error message\n try:\n _ = best.package.standard_flag(language=\"cxx\", standard=\"14\")\n except RuntimeError as e:\n raise RuntimeError(\n \"cannot find a compiler supporting C++ 14 [needed to bootstrap clingo]\"\n ) from e\n return candidates[0]\n\n def _externals_from_yaml(\n self, configuration: \"spack.config.Configuration\"\n ) -> Tuple[Optional[\"spack.spec.Spec\"], Optional[\"spack.spec.Spec\"]]:\n packages_yaml = configuration.get(\"packages\")\n requirements = {\"cmake\": \"@3.20:\", \"bison\": \"@2.5:\"}\n selected: Dict[str, Optional[\"spack.spec.Spec\"]] = {\"cmake\": None, \"bison\": None}\n for pkg_name in [\"cmake\", \"bison\"]:\n if pkg_name not in packages_yaml:\n continue\n\n candidates = packages_yaml[pkg_name].get(\"externals\", [])\n for candidate in candidates:\n s = spack.spec.Spec(candidate[\"spec\"], external_path=candidate[\"prefix\"])\n if not s.satisfies(requirements[pkg_name]):\n continue\n\n if not s.intersects(f\"arch={self.host_architecture}\"):\n continue\n\n selected[pkg_name] = self._external_spec(s)\n break\n return selected[\"cmake\"], selected[\"bison\"]\n\n def prototype_path(self) -> pathlib.Path:\n \"\"\"Path to a prototype concrete specfile for clingo\"\"\"\n parent_dir = pathlib.Path(__file__).parent\n result = parent_dir / \"prototypes\" / f\"clingo-{self.host_platform}-{self.host_target}.json\"\n if str(self.host_platform) == \"linux\":\n # Using aarch64 as a fallback, since it has gnuconfig (x86_64 doesn't have it)\n if not result.exists():\n result = parent_dir / \"prototypes\" / f\"clingo-{self.host_platform}-aarch64.json\"\n\n elif str(self.host_platform) == \"freebsd\":\n result = parent_dir / \"prototypes\" / f\"clingo-{self.host_platform}-amd64.json\"\n\n elif not result.exists():\n raise RuntimeError(f\"Cannot bootstrap clingo from sources on {self.host_platform}\")\n\n return result\n\n def concretize(self) -> \"spack.spec.Spec\":\n # Read the prototype and mark it NOT concrete\n s = spack.spec.Spec.from_specfile(str(self.prototype_path()))\n s._mark_concrete(False)\n\n # These are nodes in the cmake stack, whose versions are frequently deprecated for\n # security reasons. In case there is no external cmake on this machine, we'll update\n # their versions to the most preferred, within the valid range, according to the\n # repository we know.\n to_be_updated = {\n pkg_name: (spack.repo.PATH.get_pkg_class(pkg_name), valid_versions)\n for pkg_name, valid_versions in {\n \"ca-certificates-mozilla\": \":\",\n \"openssl\": \"3:3\",\n \"curl\": \"8:8\",\n \"cmake\": \"3.16:3\",\n \"libiconv\": \"1:1\",\n \"ncurses\": \"6:6\",\n \"m4\": \"1.4\",\n }.items()\n }\n\n # Tweak it to conform to the host architecture + update the version of a few dependencies\n for node in s.traverse():\n # Clear patches, we'll compute them correctly later\n node.patches.clear()\n if \"patches\" in node.variants:\n del node.variants[\"patches\"]\n\n node.architecture.os = str(self.host_os)\n node.architecture = self.host_architecture\n\n if node.name == \"gcc-runtime\":\n node.versions = self.host_compiler.versions\n\n if node.name in to_be_updated:\n pkg_cls, valid_versions = to_be_updated[node.name]\n _select_best_version(pkg_cls=pkg_cls, node=node, valid_versions=valid_versions)\n\n # Can't use re2c@3.1 with Python 3.6\n if self.host_python.satisfies(\"@3.6\"):\n s[\"re2c\"].versions.versions = [spack.version.from_string(\"=2.2\")]\n\n for edge in spack.traverse.traverse_edges([s], cover=\"edges\"):\n if edge.spec.name == \"python\":\n edge.spec = self.host_python\n\n if edge.spec.name == \"bison\" and self.external_bison:\n edge.spec = self.external_bison\n\n if edge.spec.name == \"cmake\" and self.external_cmake:\n edge.spec = self.external_cmake\n\n if edge.spec.name == self.host_compiler.name:\n edge.spec = self.host_compiler\n\n if \"libc\" in edge.virtuals:\n edge.spec = self.host_libc\n\n spack.spec._inject_patches_variant(s)\n s._finalize_concretization()\n\n # Work around the fact that the installer calls Spec.dependents() and\n # we modified edges inconsistently\n return s.copy()\n\n def python_external_spec(self) -> \"spack.spec.Spec\":\n \"\"\"Python external spec corresponding to the current running interpreter\"\"\"\n result = spack.spec.Spec(spec_for_current_python(), external_path=sys.exec_prefix)\n return self._external_spec(result)\n\n def libc_external_spec(self) -> \"spack.spec.Spec\":\n detector = spack.compilers.libraries.CompilerPropertyDetector(self.host_compiler)\n result = detector.default_libc()\n return self._external_spec(result)\n\n def _external_spec(self, initial_spec) -> \"spack.spec.Spec\":\n initial_spec.namespace = \"builtin\"\n initial_spec.architecture = self.host_architecture\n for flag_type in spack.spec.FlagMap.valid_compiler_flags():\n initial_spec.compiler_flags[flag_type] = []\n return spack.spec.parse_with_version_concrete(initial_spec)\n" }, { "path": "lib/spack/spack/bootstrap/config.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Manage configuration swapping for bootstrapping purposes\"\"\"\n\nimport contextlib\nimport os\nimport sys\nfrom typing import Any, Dict, Generator, MutableSequence, Sequence\n\nimport spack.config\nimport spack.environment\nimport spack.modules\nimport spack.paths\nimport spack.platforms\nimport spack.repo\nimport spack.store\nimport spack.util.path\nfrom spack.llnl.util import tty\n\n#: Reference counter for the bootstrapping configuration context manager\n_REF_COUNT = 0\n\n\ndef is_bootstrapping() -> bool:\n \"\"\"Return True if we are in a bootstrapping context, False otherwise.\"\"\"\n return _REF_COUNT > 0\n\n\ndef spec_for_current_python() -> str:\n \"\"\"For bootstrapping purposes we are just interested in the Python\n minor version (all patches are ABI compatible with the same minor).\n\n See:\n\n * https://www.python.org/dev/peps/pep-0513/\n * https://stackoverflow.com/a/35801395/771663\n \"\"\"\n version_str = \".\".join(str(x) for x in sys.version_info[:2])\n return f\"python@{version_str}\"\n\n\ndef root_path() -> str:\n \"\"\"Root of all the bootstrap related folders\"\"\"\n return spack.util.path.canonicalize_path(\n spack.config.get(\"bootstrap:root\", spack.paths.default_user_bootstrap_path)\n )\n\n\ndef store_path() -> str:\n \"\"\"Path to the store used for bootstrapped software\"\"\"\n enabled = spack.config.get(\"bootstrap:enable\", True)\n if not enabled:\n msg = 'bootstrapping is currently disabled. Use \"spack bootstrap enable\" to enable it'\n raise RuntimeError(msg)\n\n return _store_path()\n\n\n@contextlib.contextmanager\ndef spack_python_interpreter() -> Generator:\n \"\"\"Override the current configuration to set the interpreter under\n which Spack is currently running as the only Python external spec\n available.\n \"\"\"\n python_prefix = sys.exec_prefix\n external_python = spec_for_current_python()\n\n entry = {\n \"buildable\": False,\n \"externals\": [{\"prefix\": python_prefix, \"spec\": str(external_python)}],\n }\n\n with spack.config.override(\"packages:python::\", entry):\n yield\n\n\ndef _store_path() -> str:\n bootstrap_root_path = root_path()\n return spack.util.path.canonicalize_path(os.path.join(bootstrap_root_path, \"store\"))\n\n\ndef _config_path() -> str:\n bootstrap_root_path = root_path()\n return spack.util.path.canonicalize_path(os.path.join(bootstrap_root_path, \"config\"))\n\n\n@contextlib.contextmanager\ndef ensure_bootstrap_configuration() -> Generator:\n \"\"\"Swap the current configuration for the one used to bootstrap Spack.\n\n The context manager is reference counted to ensure we don't swap multiple\n times if there's nested use of it in the stack. One compelling use case\n is bootstrapping patchelf during the bootstrap of clingo.\n \"\"\"\n global _REF_COUNT # pylint: disable=global-statement\n already_swapped = bool(_REF_COUNT)\n _REF_COUNT += 1\n try:\n if already_swapped:\n yield\n else:\n with _ensure_bootstrap_configuration():\n yield\n finally:\n _REF_COUNT -= 1\n\n\ndef _read_and_sanitize_configuration() -> Dict[str, Any]:\n \"\"\"Read the user configuration that needs to be reused for bootstrapping\n and remove the entries that should not be copied over.\n \"\"\"\n # Read the \"config\" section but pop the install tree (the entry will not be\n # considered due to the use_store context manager, so it will be confusing\n # to have it in the configuration).\n config_yaml = spack.config.get(\"config\")\n config_yaml.pop(\"install_tree\", None)\n return {\n \"bootstrap\": spack.config.get(\"bootstrap\"),\n \"config\": config_yaml,\n \"repos\": spack.config.get(\"repos\"),\n }\n\n\ndef _bootstrap_config_scopes() -> Sequence[\"spack.config.ConfigScope\"]:\n tty.debug(\"[BOOTSTRAP CONFIG SCOPE] name=_builtin\")\n config_scopes: MutableSequence[\"spack.config.ConfigScope\"] = [\n spack.config.InternalConfigScope(\"_builtin\", spack.config.CONFIG_DEFAULTS)\n ]\n configuration_paths = (spack.config.CONFIGURATION_DEFAULTS_PATH, (\"bootstrap\", _config_path()))\n for name, path in configuration_paths:\n generic_scope = spack.config.DirectoryConfigScope(name, path)\n config_scopes.append(generic_scope)\n msg = \"[BOOTSTRAP CONFIG SCOPE] name={0}, path={1}\"\n tty.debug(msg.format(generic_scope.name, generic_scope.path))\n return config_scopes\n\n\n@contextlib.contextmanager\ndef _ensure_bootstrap_configuration() -> Generator:\n spack.repo.PATH.repos # ensure this is instantiated from current config.\n spack.store.ensure_singleton_created()\n bootstrap_store_path = store_path()\n user_configuration = _read_and_sanitize_configuration()\n with spack.environment.no_active_environment(), spack.platforms.use_platform(\n spack.platforms.real_host()\n ), spack.config.use_configuration(\n # Default configuration scopes excluding command line and builtin\n *_bootstrap_config_scopes()\n ), spack.store.use_store(bootstrap_store_path, extra_data={\"padded_length\": 0}):\n spack.config.set(\"bootstrap\", user_configuration[\"bootstrap\"])\n spack.config.set(\"config\", user_configuration[\"config\"])\n spack.config.set(\"repos\", user_configuration[\"repos\"])\n with spack.modules.disable_modules(), spack_python_interpreter():\n yield\n" }, { "path": "lib/spack/spack/bootstrap/core.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Bootstrap Spack core dependencies from binaries.\n\nThis module contains logic to bootstrap software required by Spack from binaries served in the\nbootstrapping mirrors. The logic is quite different from an installation done from a Spack user,\nbecause of the following reasons:\n\n1. The binaries are all compiled on the same OS for a given platform (e.g. they are compiled on\n ``centos7`` on ``linux``), but they will be installed and used on the host OS. They are also\n targeted at the most generic architecture possible. That makes the binaries difficult to reuse\n with other specs in an environment without ad-hoc logic.\n2. Bootstrapping has a fallback procedure where we try to install software by default from the\n most recent binaries, and proceed to older versions of the mirror, until we try building from\n sources as a last resort. This allows us not to be blocked on architectures where we don't\n have binaries readily available, but is also not compatible with the working of environments\n (they don't have fallback procedures).\n3. Among the binaries we have clingo, so we can't concretize that with clingo :-)\n4. clingo, GnuPG and patchelf binaries need to be verified by sha256 sum (all the other binaries\n we might add on top of that in principle can be verified with GPG signatures).\n\"\"\"\n\nimport copy\nimport functools\nimport json\nimport os\nimport sys\nimport uuid\nfrom typing import Any, Callable, Dict, List, Optional, Tuple\n\nimport spack.binary_distribution\nimport spack.concretize\nimport spack.config\nimport spack.detection\nimport spack.error\nimport spack.installer_dispatch\nimport spack.mirrors.mirror\nimport spack.platforms\nimport spack.spec\nimport spack.store\nimport spack.user_environment\nimport spack.util.executable\nimport spack.util.path\nimport spack.util.spack_yaml\nimport spack.util.url\nimport spack.version\nfrom spack.llnl.util import tty\nfrom spack.llnl.util.lang import GroupedExceptionHandler\n\nfrom ._common import (\n QueryInfo,\n _executables_in_store,\n _python_import,\n _root_spec,\n _try_import_from_store,\n)\nfrom .clingo import ClingoBootstrapConcretizer\nfrom .config import spack_python_interpreter, spec_for_current_python\n\n#: Name of the file containing metadata about the bootstrapping source\nMETADATA_YAML_FILENAME = \"metadata.yaml\"\n\n#: Whether the current platform is Windows\nIS_WINDOWS = sys.platform == \"win32\"\n\n#: Map a bootstrapper type to the corresponding class\n_bootstrap_methods = {}\n\n\nConfigDictionary = Dict[str, Any]\n\n\ndef bootstrapper(bootstrapper_type: str):\n \"\"\"Decorator to register classes implementing bootstrapping\n methods.\n\n Args:\n bootstrapper_type: string identifying the class\n \"\"\"\n\n def _register(cls):\n _bootstrap_methods[bootstrapper_type] = cls\n return cls\n\n return _register\n\n\nclass Bootstrapper:\n \"\"\"Interface for \"core\" software bootstrappers\"\"\"\n\n config_scope_name = \"\"\n\n def __init__(self, conf: ConfigDictionary) -> None:\n self.conf = conf\n self.name = conf[\"name\"]\n self.metadata_dir = spack.util.path.canonicalize_path(conf[\"metadata\"])\n\n # Check for relative paths, and turn them into absolute paths\n # root is the metadata_dir\n maybe_url = conf[\"info\"][\"url\"]\n if spack.util.url.is_path_instead_of_url(maybe_url) and not os.path.isabs(maybe_url):\n maybe_url = os.path.join(self.metadata_dir, maybe_url)\n self.url = spack.mirrors.mirror.Mirror(maybe_url).fetch_url\n\n @property\n def mirror_scope(self) -> spack.config.InternalConfigScope:\n \"\"\"Mirror scope to be pushed onto the bootstrapping configuration when using\n this bootstrapper.\n \"\"\"\n return spack.config.InternalConfigScope(\n self.config_scope_name, {\"mirrors:\": {self.name: self.url}}\n )\n\n def try_import(self, module: str, abstract_spec_str: str) -> bool:\n \"\"\"Try to import a Python module from a spec satisfying the abstract spec\n passed as argument.\n\n Args:\n module: Python module name to try importing\n abstract_spec_str: abstract spec that can provide the Python module\n\n Return:\n True if the Python module could be imported, False otherwise\n \"\"\"\n return False\n\n def try_search_path(self, executables: Tuple[str], abstract_spec_str: str) -> bool:\n \"\"\"Try to search some executables in the prefix of specs satisfying the abstract\n spec passed as argument.\n\n Args:\n executables: executables to be found\n abstract_spec_str: abstract spec that can provide the Python module\n\n Return:\n True if the executables are found, False otherwise\n \"\"\"\n return False\n\n\n@bootstrapper(bootstrapper_type=\"buildcache\")\nclass BuildcacheBootstrapper(Bootstrapper):\n \"\"\"Install the software needed during bootstrapping from a buildcache.\"\"\"\n\n def __init__(self, conf) -> None:\n super().__init__(conf)\n self.last_search: Optional[QueryInfo] = None\n self.config_scope_name = f\"bootstrap_buildcache-{uuid.uuid4()}\"\n\n @staticmethod\n def _spec_and_platform(\n abstract_spec_str: str,\n ) -> Tuple[spack.spec.Spec, spack.platforms.Platform]:\n \"\"\"Return the spec object and platform we need to use when\n querying the buildcache.\n\n Args:\n abstract_spec_str: abstract spec string we are looking for\n \"\"\"\n # Try to install from an unsigned binary cache\n abstract_spec = spack.spec.Spec(abstract_spec_str)\n # On Cray we want to use Linux binaries if available from mirrors\n bincache_platform = spack.platforms.real_host()\n return abstract_spec, bincache_platform\n\n def _read_metadata(self, package_name: str) -> Any:\n \"\"\"Return metadata about the given package.\"\"\"\n json_filename = f\"{package_name}.json\"\n json_dir = self.metadata_dir\n json_path = os.path.join(json_dir, json_filename)\n with open(json_path, encoding=\"utf-8\") as stream:\n data = json.load(stream)\n return data\n\n def _install_by_hash(\n self, pkg_hash: str, pkg_sha256: str, bincache_platform: spack.platforms.Platform\n ) -> None:\n with spack.platforms.use_platform(bincache_platform):\n query = spack.binary_distribution.BinaryCacheQuery(all_architectures=True)\n for match in spack.store.find([f\"/{pkg_hash}\"], multiple=False, query_fn=query):\n spack.binary_distribution.install_root_node(\n # allow_missing is true since when bootstrapping clingo we truncate runtime\n # deps such as gcc-runtime, since we link libstdc++ statically, and the other\n # further runtime deps are loaded by the Python interpreter. This just silences\n # warnings about missing dependencies.\n match,\n unsigned=True,\n force=True,\n sha256=pkg_sha256,\n allow_missing=True,\n )\n\n def _install_and_test(\n self,\n abstract_spec: spack.spec.Spec,\n bincache_platform: spack.platforms.Platform,\n bincache_data,\n test_fn,\n ) -> bool:\n # Ensure we see only the buildcache being used to bootstrap\n with spack.config.override(self.mirror_scope):\n # This index is currently needed to get the compiler used to build some\n # specs that we know by dag hash.\n spack.binary_distribution.BINARY_INDEX.regenerate_spec_cache()\n index = spack.binary_distribution.update_cache_and_get_specs()\n\n if not index:\n raise RuntimeError(\"The binary index is empty\")\n\n for item in bincache_data[\"verified\"]:\n candidate_spec = item[\"spec\"]\n # This will be None for things that don't depend on python\n python_spec = item.get(\"python\", None)\n # Skip specs which are not compatible\n if not abstract_spec.intersects(candidate_spec):\n continue\n\n if python_spec is not None and not abstract_spec.intersects(f\"^{python_spec}\"):\n continue\n\n for _, pkg_hash, pkg_sha256 in item[\"binaries\"]:\n self._install_by_hash(pkg_hash, pkg_sha256, bincache_platform)\n\n info: QueryInfo = {}\n if test_fn(query_spec=abstract_spec, query_info=info):\n self.last_search = info\n return True\n return False\n\n def try_import(self, module: str, abstract_spec_str: str) -> bool:\n info: QueryInfo\n test_fn, info = functools.partial(_try_import_from_store, module), {}\n if test_fn(query_spec=abstract_spec_str, query_info=info):\n return True\n\n tty.debug(f\"Bootstrapping {module} from pre-built binaries\")\n abstract_spec, bincache_platform = self._spec_and_platform(\n abstract_spec_str + \" ^\" + spec_for_current_python()\n )\n data = self._read_metadata(module)\n return self._install_and_test(abstract_spec, bincache_platform, data, test_fn)\n\n def try_search_path(self, executables: Tuple[str], abstract_spec_str: str) -> bool:\n info: QueryInfo\n test_fn, info = functools.partial(_executables_in_store, executables), {}\n if test_fn(query_spec=abstract_spec_str, query_info=info):\n self.last_search = info\n return True\n\n abstract_spec, bincache_platform = self._spec_and_platform(abstract_spec_str)\n tty.debug(f\"Bootstrapping {abstract_spec.name} from pre-built binaries\")\n data = self._read_metadata(abstract_spec.name)\n return self._install_and_test(abstract_spec, bincache_platform, data, test_fn)\n\n\n@bootstrapper(bootstrapper_type=\"install\")\nclass SourceBootstrapper(Bootstrapper):\n \"\"\"Install the software needed during bootstrapping from sources.\"\"\"\n\n def __init__(self, conf) -> None:\n super().__init__(conf)\n self.last_search: Optional[QueryInfo] = None\n self.config_scope_name = f\"bootstrap_source-{uuid.uuid4()}\"\n\n def try_import(self, module: str, abstract_spec_str: str) -> bool:\n info: QueryInfo = {}\n if _try_import_from_store(module, abstract_spec_str, query_info=info):\n self.last_search = info\n return True\n\n tty.debug(f\"Bootstrapping {module} from sources\")\n\n # If we compile code from sources detecting a few build tools\n # might reduce compilation time by a fair amount\n _add_externals_if_missing()\n\n # Try to build and install from sources\n with spack_python_interpreter():\n if module == \"clingo\":\n bootstrapper = ClingoBootstrapConcretizer(configuration=spack.config.CONFIG)\n concrete_spec = bootstrapper.concretize()\n else:\n abstract_spec = spack.spec.Spec(\n abstract_spec_str + \" ^\" + spec_for_current_python()\n )\n concrete_spec = spack.concretize.concretize_one(abstract_spec)\n\n msg = \"[BOOTSTRAP MODULE {0}] Try installing '{1}' from sources\"\n tty.debug(msg.format(module, abstract_spec_str))\n\n # Install the spec that should make the module importable\n with spack.config.override(self.mirror_scope):\n spack.installer_dispatch.create_installer(\n [concrete_spec.package],\n fail_fast=True,\n root_policy=\"source_only\",\n dependencies_policy=\"source_only\",\n ).install()\n\n if _try_import_from_store(module, query_spec=concrete_spec, query_info=info):\n self.last_search = info\n return True\n return False\n\n def try_search_path(self, executables: Tuple[str], abstract_spec_str: str) -> bool:\n info: QueryInfo = {}\n if _executables_in_store(executables, abstract_spec_str, query_info=info):\n self.last_search = info\n return True\n\n tty.debug(f\"Bootstrapping {abstract_spec_str} from sources\")\n\n # If we compile code from sources detecting a few build tools\n # might reduce compilation time by a fair amount\n _add_externals_if_missing()\n\n concrete_spec = spack.concretize.concretize_one(abstract_spec_str)\n msg = \"[BOOTSTRAP] Try installing '{0}' from sources\"\n tty.debug(msg.format(abstract_spec_str))\n with spack.config.override(self.mirror_scope):\n spack.installer_dispatch.create_installer([concrete_spec.package]).install()\n if _executables_in_store(executables, concrete_spec, query_info=info):\n self.last_search = info\n return True\n return False\n\n\ndef create_bootstrapper(conf: ConfigDictionary):\n \"\"\"Return a bootstrap object built according to the configuration argument\"\"\"\n btype = conf[\"type\"]\n return _bootstrap_methods[btype](conf)\n\n\ndef source_is_enabled(conf: ConfigDictionary) -> bool:\n \"\"\"Returns true if the source is not enabled for bootstrapping\"\"\"\n return spack.config.get(\"bootstrap:trusted\").get(conf[\"name\"], False)\n\n\ndef ensure_module_importable_or_raise(module: str, abstract_spec: Optional[str] = None):\n \"\"\"Make the requested module available for import, or raise.\n\n This function tries to import a Python module in the current interpreter\n using, in order, the methods configured in bootstrap.yaml.\n\n If none of the methods succeed, an exception is raised. The function exits\n on first success.\n\n Args:\n module: module to be imported in the current interpreter\n abstract_spec: abstract spec that might provide the module. If not\n given it defaults to \"module\"\n\n Raises:\n ImportError: if the module couldn't be imported\n \"\"\"\n # If we can import it already, that's great\n tty.debug(f\"[BOOTSTRAP MODULE {module}] Try importing from Python\")\n if _python_import(module):\n return\n\n abstract_spec = abstract_spec or module\n\n exception_handler = GroupedExceptionHandler()\n\n for current_config in bootstrapping_sources():\n if not source_is_enabled(current_config):\n continue\n\n with exception_handler.forward(current_config[\"name\"], Exception):\n if create_bootstrapper(current_config).try_import(module, abstract_spec):\n return\n\n msg = f'cannot bootstrap the \"{module}\" Python module '\n if abstract_spec:\n msg += f'from spec \"{abstract_spec}\" '\n\n if not exception_handler:\n msg += \": no bootstrapping sources are enabled\"\n elif spack.error.debug or spack.error.SHOW_BACKTRACE:\n msg += exception_handler.grouped_message(with_tracebacks=True)\n else:\n msg += exception_handler.grouped_message(with_tracebacks=False)\n msg += \"\\nRun `spack --backtrace ...` for more detailed errors\"\n raise ImportError(msg)\n\n\ndef ensure_executables_in_path_or_raise(\n executables: list,\n abstract_spec: str,\n cmd_check: Optional[Callable[[spack.util.executable.Executable], bool]] = None,\n):\n \"\"\"Ensure that some executables are in path or raise.\n\n Args:\n executables (list): list of executables to be searched in the PATH,\n in order. The function exits on the first one found.\n abstract_spec (str): abstract spec that provides the executables\n cmd_check (object): callable predicate that takes a\n ``spack.util.executable.Executable`` command and validate it. Should return\n ``True`` if the executable is acceptable, ``False`` otherwise.\n Can be used to, e.g., ensure a suitable version of the command before\n accepting for bootstrapping.\n\n Raises:\n RuntimeError: if the executables cannot be ensured to be in PATH\n\n Return:\n Executable object\n\n \"\"\"\n cmd = spack.util.executable.which(*executables)\n if cmd:\n if not cmd_check or cmd_check(cmd):\n return cmd\n\n executables_str = \", \".join(executables)\n\n exception_handler = GroupedExceptionHandler()\n\n for current_config in bootstrapping_sources():\n if not source_is_enabled(current_config):\n continue\n with exception_handler.forward(current_config[\"name\"], Exception):\n current_bootstrapper = create_bootstrapper(current_config)\n if current_bootstrapper.try_search_path(executables, abstract_spec):\n # Additional environment variables needed\n concrete_spec, cmd = (\n current_bootstrapper.last_search[\"spec\"],\n current_bootstrapper.last_search[\"command\"],\n )\n assert cmd is not None, \"expected an Executable\"\n cmd.add_default_envmod(\n spack.user_environment.environment_modifications_for_specs(\n concrete_spec, set_package_py_globals=False\n )\n )\n return cmd\n\n msg = f\"cannot bootstrap any of the {executables_str} executables \"\n if abstract_spec:\n msg += f'from spec \"{abstract_spec}\" '\n\n if not exception_handler:\n msg += \": no bootstrapping sources are enabled\"\n elif spack.error.debug or spack.error.SHOW_BACKTRACE:\n msg += exception_handler.grouped_message(with_tracebacks=True)\n else:\n msg += exception_handler.grouped_message(with_tracebacks=False)\n msg += \"\\nRun `spack --backtrace ...` for more detailed errors\"\n raise RuntimeError(msg)\n\n\ndef _add_externals_if_missing() -> None:\n search_list = [\n # clingo\n \"cmake\",\n \"bison\",\n # GnuPG\n \"gawk\",\n # develop deps\n \"git\",\n ]\n if IS_WINDOWS:\n search_list.append(\"winbison\")\n externals = spack.detection.by_path(search_list)\n # System git is typically deprecated, so mark as non-buildable to force it as external\n non_buildable_externals = {k: externals.pop(k) for k in (\"git\",) if k in externals}\n spack.detection.update_configuration(externals, scope=\"bootstrap\", buildable=True)\n spack.detection.update_configuration(\n non_buildable_externals, scope=\"bootstrap\", buildable=False\n )\n\n\ndef clingo_root_spec() -> str:\n \"\"\"Return the root spec used to bootstrap clingo\"\"\"\n return _root_spec(\"clingo-bootstrap@spack+python\")\n\n\ndef ensure_clingo_importable_or_raise() -> None:\n \"\"\"Ensure that the clingo module is available for import.\"\"\"\n ensure_module_importable_or_raise(module=\"clingo\", abstract_spec=clingo_root_spec())\n\n\ndef gnupg_root_spec() -> str:\n \"\"\"Return the root spec used to bootstrap GnuPG\"\"\"\n root_spec_name = \"win-gpg\" if IS_WINDOWS else \"gnupg\"\n return _root_spec(f\"{root_spec_name}@2.3:\")\n\n\ndef ensure_gpg_in_path_or_raise() -> None:\n \"\"\"Ensure gpg or gpg2 are in the PATH or raise.\"\"\"\n return ensure_executables_in_path_or_raise(\n executables=[\"gpg2\", \"gpg\"], abstract_spec=gnupg_root_spec()\n )\n\n\ndef patchelf_root_spec() -> str:\n \"\"\"Return the root spec used to bootstrap patchelf\"\"\"\n # 0.13.1 is the last version not to require C++17.\n return _root_spec(\"patchelf@0.13.1:\")\n\n\ndef verify_patchelf(patchelf: \"spack.util.executable.Executable\") -> bool:\n \"\"\"Older patchelf versions can produce broken binaries, so we\n verify the version here.\n\n Arguments:\n\n patchelf: patchelf executable\n \"\"\"\n out = patchelf(\"--version\", output=str, error=os.devnull, fail_on_error=False).strip()\n if patchelf.returncode != 0:\n return False\n parts = out.split(\" \")\n if len(parts) < 2:\n return False\n try:\n version = spack.version.Version(parts[1])\n except ValueError:\n return False\n return version >= spack.version.Version(\"0.13.1\")\n\n\ndef ensure_patchelf_in_path_or_raise() -> spack.util.executable.Executable:\n \"\"\"Ensure patchelf is in the PATH or raise.\"\"\"\n # The old concretizer is not smart and we're doing its job: if the latest patchelf\n # does not concretize because the compiler doesn't support C++17, we try to\n # concretize again with an upperbound @:13.\n try:\n return ensure_executables_in_path_or_raise(\n executables=[\"patchelf\"], abstract_spec=patchelf_root_spec(), cmd_check=verify_patchelf\n )\n except RuntimeError:\n return ensure_executables_in_path_or_raise(\n executables=[\"patchelf\"],\n abstract_spec=_root_spec(\"patchelf@0.13.1:0.13\"),\n cmd_check=verify_patchelf,\n )\n\n\ndef ensure_winsdk_external_or_raise() -> None:\n \"\"\"Ensure the Windows SDK + WGL are available on system\n If both of these package are found, the Spack user or bootstrap\n configuration (depending on where Spack is running)\n will be updated to include all versions and variants detected.\n If either the WDK or WSDK are not found, this method will raise\n a RuntimeError.\n\n **NOTE:** This modifies the Spack config in the current scope,\n either user or environment depending on the calling context.\n This is different from all other current bootstrap dependency\n checks.\n \"\"\"\n if set([\"win-sdk\", \"wgl\"]).issubset(spack.config.get(\"packages\").keys()):\n return\n tty.debug(\"Detecting Windows SDK and WGL installations\")\n # find the externals sequentially to avoid subprocesses being spawned\n externals = spack.detection.by_path([\"win-sdk\", \"wgl\"], max_workers=1)\n if not set([\"win-sdk\", \"wgl\"]) == externals.keys():\n missing_packages_lst = []\n if \"wgl\" not in externals:\n missing_packages_lst.append(\"wgl\")\n if \"win-sdk\" not in externals:\n missing_packages_lst.append(\"win-sdk\")\n missing_packages = \" & \".join(missing_packages_lst)\n raise RuntimeError(\n f\"Unable to find the {missing_packages}, please install these packages via the Visual \"\n \"Studio installer before proceeding with Spack or provide the path to a non standard \"\n \"install with 'spack external find --path'\"\n )\n # wgl/sdk are not required for bootstrapping Spack, but\n # are required for building anything non trivial\n # add to user config so they can be used by subsequent Spack ops\n spack.detection.update_configuration(externals, buildable=False)\n\n\ndef ensure_core_dependencies() -> None:\n \"\"\"Ensure the presence of all the core dependencies.\"\"\"\n if sys.platform.lower() == \"linux\":\n ensure_patchelf_in_path_or_raise()\n ensure_gpg_in_path_or_raise()\n ensure_clingo_importable_or_raise()\n\n\ndef all_core_root_specs() -> List[str]:\n \"\"\"Return a list of all the core root specs that may be used to bootstrap Spack\"\"\"\n return [clingo_root_spec(), gnupg_root_spec(), patchelf_root_spec()]\n\n\ndef bootstrapping_sources(scope: Optional[str] = None):\n \"\"\"Return the list of configured sources of software for bootstrapping Spack\n\n Args:\n scope: if a valid configuration scope is given, return the\n list only from that scope\n \"\"\"\n source_configs = spack.config.get(\"bootstrap:sources\", default=None, scope=scope)\n source_configs = source_configs or []\n list_of_sources = []\n for entry in source_configs:\n current = copy.copy(entry)\n metadata_dir = spack.util.path.canonicalize_path(entry[\"metadata\"])\n metadata_yaml = os.path.join(metadata_dir, METADATA_YAML_FILENAME)\n try:\n with open(metadata_yaml, encoding=\"utf-8\") as stream:\n current.update(spack.util.spack_yaml.load(stream))\n list_of_sources.append(current)\n except OSError:\n pass\n return list_of_sources\n" }, { "path": "lib/spack/spack/bootstrap/environment.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Bootstrap non-core Spack dependencies from an environment.\"\"\"\n\nimport contextlib\nimport hashlib\nimport os\nimport pathlib\nimport sys\nfrom typing import Iterable, List\n\nimport spack.vendor.archspec.cpu\n\nimport spack.binary_distribution\nimport spack.config\nimport spack.environment\nimport spack.spec\nimport spack.tengine\nimport spack.util.gpg\nimport spack.util.path\nfrom spack.llnl.util import tty\n\nfrom .config import root_path, spec_for_current_python, store_path\nfrom .core import _add_externals_if_missing\n\n\nclass BootstrapEnvironment(spack.environment.Environment):\n \"\"\"Environment to install dependencies of Spack for a given interpreter and architecture\"\"\"\n\n def __init__(self) -> None:\n if not self.spack_yaml().exists():\n self._write_spack_yaml_file()\n super().__init__(self.environment_root())\n\n # Remove python package roots created before python-venv was introduced\n for s in self.concrete_roots():\n if \"python\" in s.package.extendees and not s.dependencies(\"python-venv\"):\n self.deconcretize_by_hash(s.dag_hash())\n\n @classmethod\n def spack_dev_requirements(cls) -> List[str]:\n \"\"\"Spack development requirements\"\"\"\n return [pytest_root_spec(), ruff_root_spec(), mypy_root_spec()]\n\n @classmethod\n def environment_root(cls) -> pathlib.Path:\n \"\"\"Environment root directory\"\"\"\n bootstrap_root_path = root_path()\n python_part = spec_for_current_python().replace(\"@\", \"\")\n arch_part = spack.vendor.archspec.cpu.host().family\n interpreter_part = hashlib.md5(sys.exec_prefix.encode()).hexdigest()[:5]\n environment_dir = f\"{python_part}-{arch_part}-{interpreter_part}\"\n return pathlib.Path(\n spack.util.path.canonicalize_path(\n os.path.join(bootstrap_root_path, \"environments\", environment_dir)\n )\n )\n\n @classmethod\n def view_root(cls) -> pathlib.Path:\n \"\"\"Location of the view\"\"\"\n return cls.environment_root().joinpath(\"view\")\n\n @classmethod\n def bin_dir(cls) -> pathlib.Path:\n \"\"\"Paths to be added to PATH\"\"\"\n return cls.view_root().joinpath(\"bin\")\n\n def python_dirs(self) -> Iterable[pathlib.Path]:\n python = next(s for s in self.all_specs_generator() if s.name == \"python-venv\").package\n return {self.view_root().joinpath(p) for p in (python.platlib, python.purelib)}\n\n @classmethod\n def spack_yaml(cls) -> pathlib.Path:\n \"\"\"Environment spack.yaml file\"\"\"\n return cls.environment_root().joinpath(\"spack.yaml\")\n\n @contextlib.contextmanager\n def trust_bootstrap_mirror_keys(self):\n with spack.util.gpg.gnupghome_override(os.path.join(root_path(), \".bootstrap-gpg\")):\n spack.binary_distribution.get_keys(install=True, trust=True)\n yield\n\n def update_installations(self) -> None:\n \"\"\"Update the installations of this environment.\"\"\"\n log_enabled = tty.is_debug() or tty.is_verbose()\n with tty.SuppressOutput(msg_enabled=log_enabled, warn_enabled=log_enabled):\n specs = self.concretize()\n if specs:\n colorized_specs = [\n spack.spec.Spec(x).cformat(\"{name}{@version}\")\n for x in self.spack_dev_requirements()\n ]\n tty.msg(f\"[BOOTSTRAPPING] Installing dependencies ({', '.join(colorized_specs)})\")\n self.write(regenerate=False)\n with tty.SuppressOutput(msg_enabled=log_enabled, warn_enabled=log_enabled):\n with self.trust_bootstrap_mirror_keys():\n fetch_policy = (\n \"cache_only\"\n if not spack.config.get(\"bootstrap:dev:enable_source\", False)\n else \"auto\"\n )\n self.install_all(\n fail_fast=True, root_policy=fetch_policy, dependencies_policy=fetch_policy\n )\n self.write(regenerate=True)\n\n def load(self) -> None:\n \"\"\"Update PATH and sys.path.\"\"\"\n # Make executables available (shouldn't need PYTHONPATH)\n os.environ[\"PATH\"] = f\"{self.bin_dir()}{os.pathsep}{os.environ.get('PATH', '')}\"\n\n # Spack itself imports pytest\n sys.path.extend(str(p) for p in self.python_dirs())\n\n def _write_spack_yaml_file(self) -> None:\n tty.msg(\n \"[BOOTSTRAPPING] Spack has missing dependencies, creating a bootstrapping environment\"\n )\n env = spack.tengine.make_environment()\n template = env.get_template(\"bootstrap/spack.yaml\")\n context = {\n \"python_spec\": f\"{spec_for_current_python()}+ctypes\",\n \"python_prefix\": sys.exec_prefix,\n \"architecture\": spack.vendor.archspec.cpu.host().family,\n \"environment_path\": self.environment_root(),\n \"environment_specs\": self.spack_dev_requirements(),\n \"store_path\": store_path(),\n \"bootstrap_mirrors\": dev_bootstrap_mirror_names(),\n }\n self.environment_root().mkdir(parents=True, exist_ok=True)\n self.spack_yaml().write_text(template.render(context), encoding=\"utf-8\")\n\n\ndef mypy_root_spec() -> str:\n \"\"\"Return the root spec used to bootstrap mypy\"\"\"\n return \"py-mypy@0.900: ^py-mypy-extensions@:1.0\"\n\n\ndef pytest_root_spec() -> str:\n \"\"\"Return the root spec used to bootstrap pytest\"\"\"\n return \"py-pytest@6.2.4:\"\n\n\ndef ruff_root_spec() -> str:\n \"\"\"Return the root spec used to bootstrap ruff\"\"\"\n return \"py-ruff@0.15.0\"\n\n\ndef dev_bootstrap_mirror_names() -> List[str]:\n \"\"\"Return the mirror names used for bootstrapping dev\n requirements\"\"\"\n return [\n \"developer-tools-darwin\",\n \"developer-tools-x86_64_v3-linux-gnu\",\n \"developer-tools-aarch64-linux-gnu\",\n ]\n\n\ndef ensure_environment_dependencies() -> None:\n \"\"\"Ensure Spack dependencies from the bootstrap environment are installed and ready to use\"\"\"\n _add_externals_if_missing()\n with BootstrapEnvironment() as env:\n env.update_installations()\n env.load()\n" }, { "path": "lib/spack/spack/bootstrap/prototypes/clingo-darwin-aarch64.json", "content": "{\"spec\":{\"_meta\":{\"version\":5},\"nodes\":[{\"name\":\"clingo-bootstrap\",\"version\":\"spack\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"cmake\",\"build_type\":\"Release\",\"docs\":false,\"generator\":\"make\",\"ipo\":false,\"optimized\":false,\"python\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ozkztarkrp3oet7x2oapc7ehdfyvweap45zb3g44mj6qpblv4l3a====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"bison\",\"hash\":\"mtmvlzy7mfjfrcecjxk6wgjwxlqtmzpj\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"cmake\",\"hash\":\"zlqbht6siyfbw65vi7eg3g2kkjbgeb3s\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"python\",\"hash\":\"ubenpqhxb6v4lsnefcot2naoyufhtvlq\",\"parameters\":{\"deptypes\":[\"build\",\"link\",\"run\"],\"virtuals\":[]}},{\"name\":\"python-venv\",\"hash\":\"4duigy4ujnstkq5c542w4okhszygw72h\",\"parameters\":{\"deptypes\":[\"build\",\"run\"],\"virtuals\":[]}},{\"name\":\"re2c\",\"hash\":\"uhkg474hzahckbth32ydtp24etgavq76\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"lssvl522otfuhyr7zw6rr65fpyd5eicp\"},{\"name\":\"apple-clang\",\"version\":\"16.0.0\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":{\"name\":\"m1\",\"vendor\":\"Apple\",\"features\":[\"aes\",\"asimd\",\"asimddp\",\"asimdfhm\",\"asimdhp\",\"asimdrdm\",\"atomics\",\"cpuid\",\"crc32\",\"dcpodp\",\"dcpop\",\"dit\",\"evtstrm\",\"fcma\",\"flagm\",\"flagm2\",\"fp\",\"fphp\",\"frint\",\"ilrcpc\",\"jscvt\",\"lrcpc\",\"paca\",\"pacg\",\"pmull\",\"sb\",\"sha1\",\"sha2\",\"sha3\",\"sha512\",\"ssbs\",\"uscat\"],\"generation\":0,\"parents\":[\"armv8.4a\"],\"cpupart\":\"0x022\"}},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"bundle\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"external\":{\"path\":\"/usr\",\"module\":null,\"extra_attributes\":{\"compilers\":{\"c\":\"/usr/bin/clang\",\"cxx\":\"/usr/bin/clang++\"}}},\"package_hash\":\"7iabceub7ckyfs2h5g75jxtolk253q6nm3r5hyqbztckky25gnpa====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\"},{\"name\":\"bison\",\"version\":\"3.8.2\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"color\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"d4j62fwvuxqbiez32ltjnhu47ac425wjebsy6fhoptv6saxazcxq====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"diffutils\",\"hash\":\"4p2v6lqdlosuxav6qtrmemodqnp7p7ql\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"rzeea2himrnudsunposb2rlyw6mjhmr7\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"m4\",\"hash\":\"4u7ml457pqh6mzvundcjcv4xzvcjwhw3\",\"parameters\":{\"deptypes\":[\"build\",\"run\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"mtmvlzy7mfjfrcecjxk6wgjwxlqtmzpj\"},{\"name\":\"compiler-wrapper\",\"version\":\"1.0\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":{\"name\":\"m1\",\"vendor\":\"Apple\",\"features\":[\"aes\",\"asimd\",\"asimddp\",\"asimdfhm\",\"asimdhp\",\"asimdrdm\",\"atomics\",\"cpuid\",\"crc32\",\"dcpodp\",\"dcpop\",\"dit\",\"evtstrm\",\"fcma\",\"flagm\",\"flagm2\",\"fp\",\"fphp\",\"frint\",\"ilrcpc\",\"jscvt\",\"lrcpc\",\"paca\",\"pacg\",\"pmull\",\"sb\",\"sha1\",\"sha2\",\"sha3\",\"sha512\",\"ssbs\",\"uscat\"],\"generation\":0,\"parents\":[\"armv8.4a\"],\"cpupart\":\"0x022\"}},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"gogqnfdkxjvnjgj3lndnoncjtdc7ydoc7klkjstywag4oqrvod7a====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\"},{\"name\":\"diffutils\",\"version\":\"3.10\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"kbmzdy7mgklc24qx55cvx7kq7hceby2yav4fnf64gfdo7epdghwa====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"rzeea2himrnudsunposb2rlyw6mjhmr7\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libiconv\",\"hash\":\"hvarrkkr7z5tujmt45xhns2kljvnunof\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"iconv\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"4p2v6lqdlosuxav6qtrmemodqnp7p7ql\"},{\"name\":\"gmake\",\"version\":\"4.4.1\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"guile\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"rpzjfobv7qh3wevti34nlbd2emtw5mnyszqmkyiq5jiq33xm7qzq====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\"},{\"name\":\"gnuconfig\",\"version\":\"2024-07-27\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"aar2tabf35425kgzryprq775xycug7xlbt4rkwvm4aj76dhlychq====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"rzeea2himrnudsunposb2rlyw6mjhmr7\"},{\"name\":\"libiconv\",\"version\":\"1.17\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"libs\":[\"shared\",\"static\"],\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ujsqmcknrabka5mhwwpbaf5rwxgopwoyxkskuwyqlcbynowgdvfa====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"rzeea2himrnudsunposb2rlyw6mjhmr7\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"hvarrkkr7z5tujmt45xhns2kljvnunof\"},{\"name\":\"m4\",\"version\":\"1.4.19\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"patches\":[\"9dc5fbd0d5cb1037ab1e6d0ecc74a30df218d0a94bdd5a02759a97f62daca573\",\"bfdffa7c2eb01021d5849b36972c069693654ad826c1a20b53534009a4ec7a89\"],\"sigsegv\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"patches\":[\"bfdffa7c2eb01021d5849b36972c069693654ad826c1a20b53534009a4ec7a89\",\"9dc5fbd0d5cb1037ab1e6d0ecc74a30df218d0a94bdd5a02759a97f62daca573\"],\"package_hash\":\"npb7a53yz7wqx4nvnasxwgzxaoiks6sdjz2eugrgkjxs4ml24xea====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"diffutils\",\"hash\":\"4p2v6lqdlosuxav6qtrmemodqnp7p7ql\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"rzeea2himrnudsunposb2rlyw6mjhmr7\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libsigsegv\",\"hash\":\"j36atbspivp2gr7gnthqzakoitzzkstp\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"4u7ml457pqh6mzvundcjcv4xzvcjwhw3\"},{\"name\":\"libsigsegv\",\"version\":\"2.14\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"3s645t5rbjrziao47mhgob5xgymot6tf4kalagflbal2jdamdo2a====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"rzeea2himrnudsunposb2rlyw6mjhmr7\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"j36atbspivp2gr7gnthqzakoitzzkstp\"},{\"name\":\"cmake\",\"version\":\"3.31.2\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"build_type\":\"Release\",\"doc\":false,\"ncurses\":true,\"ownlibs\":true,\"qtgui\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"7vk6yhuq2fklcj5kk7bhreqojudugggezq7vntmcsc32cw2avmya====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"curl\",\"hash\":\"jkilt3drjtni4pwxwvujwpasnef4xzqx\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"ncurses\",\"hash\":\"7xdoqvmiu5hzgndg26zn3ne4trd6aq3t\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"7ikuk745stdvecdqbcrmrczgozpfwopt\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"zlqbht6siyfbw65vi7eg3g2kkjbgeb3s\"},{\"name\":\"curl\",\"version\":\"8.10.1\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"gssapi\":false,\"ldap\":false,\"libidn2\":false,\"librtmp\":false,\"libs\":[\"shared\",\"static\"],\"libssh\":false,\"libssh2\":false,\"nghttp2\":true,\"tls\":[\"secure_transport\"],\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ccka5yawqcn2rjbqn3bkhkdjoajlngm5uab7jbyrsl5yqn42ofza====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"rzeea2himrnudsunposb2rlyw6mjhmr7\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"nghttp2\",\"hash\":\"7ypebulcls2zoubgisasufif4lbsvfyv\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"pkgconf\",\"hash\":\"ub6rqfbiqdmmttgtgywljfealg3xnjmh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}},{\"name\":\"zlib-ng\",\"hash\":\"7ikuk745stdvecdqbcrmrczgozpfwopt\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"jkilt3drjtni4pwxwvujwpasnef4xzqx\"},{\"name\":\"nghttp2\",\"version\":\"1.64.0\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"nkykfkj4rxzmysrmoh5mhxrl5ysaemlqh652m3he7pkbgvjhjgba====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"diffutils\",\"hash\":\"4p2v6lqdlosuxav6qtrmemodqnp7p7ql\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"rzeea2himrnudsunposb2rlyw6mjhmr7\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"pkgconf\",\"hash\":\"ub6rqfbiqdmmttgtgywljfealg3xnjmh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"7ypebulcls2zoubgisasufif4lbsvfyv\"},{\"name\":\"pkgconf\",\"version\":\"2.2.0\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"gl6tpyarjlclzsal6wa4dtc7cdzprq36nbibalai4a6wgzblrseq====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"rzeea2himrnudsunposb2rlyw6mjhmr7\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"ub6rqfbiqdmmttgtgywljfealg3xnjmh\"},{\"name\":\"zlib-ng\",\"version\":\"2.2.1\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"compat\":true,\"new_strategies\":true,\"opt\":true,\"pic\":true,\"shared\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"mdxo2xewbdavckgsqlcjywyfssdchgwbzonui22gxww7hqtozurq====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"rzeea2himrnudsunposb2rlyw6mjhmr7\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"7ikuk745stdvecdqbcrmrczgozpfwopt\"},{\"name\":\"ncurses\",\"version\":\"6.5\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"abi\":\"none\",\"build_system\":\"autotools\",\"patches\":[\"7a351bc4953a4ab70dabdbea31c8db0c03d40ce505335f3b6687180dde24c535\"],\"symlinks\":false,\"termlib\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"patches\":[\"7a351bc4953a4ab70dabdbea31c8db0c03d40ce505335f3b6687180dde24c535\"],\"package_hash\":\"cfh76rniab2gnv4jqr77yzz5za4ucfmva2upihvxukn52dybhsvq====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"rzeea2himrnudsunposb2rlyw6mjhmr7\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"pkgconf\",\"hash\":\"ub6rqfbiqdmmttgtgywljfealg3xnjmh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"7xdoqvmiu5hzgndg26zn3ne4trd6aq3t\"},{\"name\":\"python\",\"version\":\"3.13.0\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"bz2\":true,\"ctypes\":true,\"dbm\":true,\"debug\":false,\"libxml2\":true,\"lzma\":true,\"optimizations\":false,\"pic\":true,\"pyexpat\":true,\"pythoncmd\":true,\"readline\":true,\"shared\":true,\"sqlite3\":true,\"ssl\":true,\"tkinter\":false,\"uuid\":true,\"zlib\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"n6v6rt6deysntdggu2gi4zkhqriyba6bgaghxyhluou4ssqf7xfq====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"apple-libuuid\",\"hash\":\"c6v7lfkcgosqqnc2zkzqulybqqg4e22s\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"uuid\"]}},{\"name\":\"bzip2\",\"hash\":\"zv7lxkvykfbn2zaq4lm4bzdso4vjlrnr\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"expat\",\"hash\":\"22rzbasmo5pkkhqri5vplsxujq3fkyzd\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"gdbm\",\"hash\":\"pa6dzar6nbm3muyi5wm7wdqcyjom3rrt\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"gettext\",\"hash\":\"bah7ymgn6hajzicktrj3d26sltksrdyf\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libffi\",\"hash\":\"z533r2ia3xmkjsr2raqwtgnnwqdjhrxf\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"ncurses\",\"hash\":\"7xdoqvmiu5hzgndg26zn3ne4trd6aq3t\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"openssl\",\"hash\":\"7etpau7yq3ctzzymjvlqo64yxykumskl\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"pkgconf\",\"hash\":\"ub6rqfbiqdmmttgtgywljfealg3xnjmh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}},{\"name\":\"readline\",\"hash\":\"vuu3chywd42dhwbavg2bqogsaqb5u2vw\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"sqlite\",\"hash\":\"4g3wvdt4hvdmqqrckn5aqzhebx6cf7t3\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"xz\",\"hash\":\"4aws4yus4ottcqw63gx2ppktuev2z2qw\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"7ikuk745stdvecdqbcrmrczgozpfwopt\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"ubenpqhxb6v4lsnefcot2naoyufhtvlq\"},{\"name\":\"apple-libuuid\",\"version\":\"1353.100.2\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"bundle\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"external\":{\"path\":\"/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk\",\"module\":null,\"extra_attributes\":{}},\"package_hash\":\"rv7eeukm7m2umg6ulafeco2qz2kvaqpx2bjoita6g27hrs6vfmiq====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"c6v7lfkcgosqqnc2zkzqulybqqg4e22s\"},{\"name\":\"bzip2\",\"version\":\"1.0.8\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"debug\":false,\"pic\":false,\"shared\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"jb7yvhkifmvfl3ykmdulsjxkkulker6gqb5tadollyjt2ijg3zsa====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"diffutils\",\"hash\":\"4p2v6lqdlosuxav6qtrmemodqnp7p7ql\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"zv7lxkvykfbn2zaq4lm4bzdso4vjlrnr\"},{\"name\":\"expat\",\"version\":\"2.6.4\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"libbsd\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ei6qyjakl7sgtodwxxbg5brgkp23robfximtpbedkrnpyyyvr3ya====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"rzeea2himrnudsunposb2rlyw6mjhmr7\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"22rzbasmo5pkkhqri5vplsxujq3fkyzd\"},{\"name\":\"gdbm\",\"version\":\"1.23\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"liepxl6phlcxbgfmibxafhewtihlgaa4x3hko37ckqlafhxkrgdq====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"rzeea2himrnudsunposb2rlyw6mjhmr7\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"readline\",\"hash\":\"vuu3chywd42dhwbavg2bqogsaqb5u2vw\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"pa6dzar6nbm3muyi5wm7wdqcyjom3rrt\"},{\"name\":\"readline\",\"version\":\"8.2\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"patches\":[\"bbf97f1ec40a929edab5aa81998c1e2ef435436c597754916e6a5868f273aff7\"],\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"patches\":[\"bbf97f1ec40a929edab5aa81998c1e2ef435436c597754916e6a5868f273aff7\"],\"package_hash\":\"oww6dmr7xqgg6j7iiluonxbcl4irqnnrip4vfkjdwujncwnuhwuq====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"rzeea2himrnudsunposb2rlyw6mjhmr7\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"ncurses\",\"hash\":\"7xdoqvmiu5hzgndg26zn3ne4trd6aq3t\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"vuu3chywd42dhwbavg2bqogsaqb5u2vw\"},{\"name\":\"gettext\",\"version\":\"0.22.5\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"bzip2\":true,\"curses\":true,\"git\":true,\"libunistring\":false,\"libxml2\":true,\"pic\":true,\"shared\":true,\"tar\":true,\"xz\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"d4zxhmw6rownaaokzcolsszrq2cmx44m7qmzopucymoyrhbdfgvq====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"bzip2\",\"hash\":\"zv7lxkvykfbn2zaq4lm4bzdso4vjlrnr\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"rzeea2himrnudsunposb2rlyw6mjhmr7\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libiconv\",\"hash\":\"hvarrkkr7z5tujmt45xhns2kljvnunof\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"iconv\"]}},{\"name\":\"libxml2\",\"hash\":\"zh4o2buvhkcq2ayeoww6bw6p7lpc53lr\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"ncurses\",\"hash\":\"7xdoqvmiu5hzgndg26zn3ne4trd6aq3t\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"tar\",\"hash\":\"4hde5fjjlzaxmazsaeen6cdlejigf4ts\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}},{\"name\":\"xz\",\"hash\":\"4aws4yus4ottcqw63gx2ppktuev2z2qw\",\"parameters\":{\"deptypes\":[\"build\",\"link\",\"run\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"bah7ymgn6hajzicktrj3d26sltksrdyf\"},{\"name\":\"libxml2\",\"version\":\"2.13.4\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"pic\":true,\"python\":false,\"shared\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"j6yob2wgvc2wjzvbs6xdvgyfa3zp3wrm3uxncxzxqfzw6xazzoba====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"rzeea2himrnudsunposb2rlyw6mjhmr7\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libiconv\",\"hash\":\"hvarrkkr7z5tujmt45xhns2kljvnunof\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"iconv\"]}},{\"name\":\"pkgconf\",\"hash\":\"ub6rqfbiqdmmttgtgywljfealg3xnjmh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}},{\"name\":\"xz\",\"hash\":\"4aws4yus4ottcqw63gx2ppktuev2z2qw\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"7ikuk745stdvecdqbcrmrczgozpfwopt\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"zh4o2buvhkcq2ayeoww6bw6p7lpc53lr\"},{\"name\":\"xz\",\"version\":\"5.4.6\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"libs\":[\"shared\",\"static\"],\"pic\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"zt5vu2vph2v2qjwgdbe7btgcz7axpyalorcsqiuxhrg5grwgrrvq====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"rzeea2himrnudsunposb2rlyw6mjhmr7\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"4aws4yus4ottcqw63gx2ppktuev2z2qw\"},{\"name\":\"tar\",\"version\":\"1.35\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"zip\":\"pigz\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"v6a6jvks2setklucxyk622uauxzqlgmsdkrvdijbi3m5jwftmzla====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"bzip2\",\"hash\":\"zv7lxkvykfbn2zaq4lm4bzdso4vjlrnr\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"rzeea2himrnudsunposb2rlyw6mjhmr7\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libiconv\",\"hash\":\"hvarrkkr7z5tujmt45xhns2kljvnunof\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"iconv\"]}},{\"name\":\"pigz\",\"hash\":\"l3lu3os3j4yjdpdvtooaxc74ece64qy6\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}},{\"name\":\"xz\",\"hash\":\"4aws4yus4ottcqw63gx2ppktuev2z2qw\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}},{\"name\":\"zstd\",\"hash\":\"eantpzna3rm5ccxgz3z6p4kdaqcm22lr\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"4hde5fjjlzaxmazsaeen6cdlejigf4ts\"},{\"name\":\"pigz\",\"version\":\"2.8\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"makefile\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"4w67lflje4giekjg4ie2vpyuiunjcumo6geofykvon3hodllp42q====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"7ikuk745stdvecdqbcrmrczgozpfwopt\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"l3lu3os3j4yjdpdvtooaxc74ece64qy6\"},{\"name\":\"zstd\",\"version\":\"1.5.6\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"makefile\",\"compression\":[\"none\"],\"libs\":[\"shared\",\"static\"],\"programs\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"uvmrov4c6unft6o4yd3jk3uqvweua3uhwdli4sw7h5wvklaf5t3q====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"eantpzna3rm5ccxgz3z6p4kdaqcm22lr\"},{\"name\":\"libffi\",\"version\":\"3.4.6\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"umhsnvoj5ooa3glffnkk2hp3txmrsjvqbpfq2hbk4mhcvhza7gaa====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"rzeea2himrnudsunposb2rlyw6mjhmr7\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"z533r2ia3xmkjsr2raqwtgnnwqdjhrxf\"},{\"name\":\"openssl\",\"version\":\"3.4.0\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"certs\":\"mozilla\",\"docs\":false,\"shared\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"5y33vxwjtlrlsyedasvmhukjkk5yfwcri27oceh36iw73xehumfa====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"ca-certificates-mozilla\",\"hash\":\"v5ocihzb43rasxqelwzx4o3htu4xavgu\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"perl\",\"hash\":\"wc57ocdnd6w5f5apre2ywlaca3mcvgks\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"7ikuk745stdvecdqbcrmrczgozpfwopt\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"7etpau7yq3ctzzymjvlqo64yxykumskl\"},{\"name\":\"ca-certificates-mozilla\",\"version\":\"2023-05-30\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"63npvwqwo2x7i6emvnklh4mhcn45gx2qzveorybh5h2inwr55sea====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"v5ocihzb43rasxqelwzx4o3htu4xavgu\"},{\"name\":\"perl\",\"version\":\"5.40.0\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cpanm\":true,\"opcode\":true,\"open\":true,\"shared\":true,\"threads\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"f233ue76vwtkle2r4jwsfe5x27ujx6ea4vdyp6baonfmkgqf5vpa====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"berkeley-db\",\"hash\":\"cwabnmpcy7hllcma4zyjhwi2mhxcrfti\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"bzip2\",\"hash\":\"zv7lxkvykfbn2zaq4lm4bzdso4vjlrnr\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gdbm\",\"hash\":\"pa6dzar6nbm3muyi5wm7wdqcyjom3rrt\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"7ikuk745stdvecdqbcrmrczgozpfwopt\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"wc57ocdnd6w5f5apre2ywlaca3mcvgks\"},{\"name\":\"berkeley-db\",\"version\":\"18.1.40\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cxx\":true,\"docs\":false,\"patches\":[\"26090f418891757af46ac3b89a9f43d6eb5989f7a3dce3d1cfc99fba547203b3\",\"b231fcc4d5cff05e5c3a4814f6a5af0e9a966428dc2176540d2c05aff41de522\"],\"stl\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"patches\":[\"b231fcc4d5cff05e5c3a4814f6a5af0e9a966428dc2176540d2c05aff41de522\",\"26090f418891757af46ac3b89a9f43d6eb5989f7a3dce3d1cfc99fba547203b3\"],\"package_hash\":\"h57ydfn33zevvzctzzioiiwjwe362izbbwncb6a26dfeno4y7tda====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"rzeea2himrnudsunposb2rlyw6mjhmr7\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"cwabnmpcy7hllcma4zyjhwi2mhxcrfti\"},{\"name\":\"sqlite\",\"version\":\"3.46.0\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"column_metadata\":true,\"dynamic_extensions\":true,\"fts\":true,\"functions\":false,\"rtree\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"wm3irnrjil5n275nw2m4x3mpvyg35h7isbmsnuae6vtxbamsrv4q====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"rzeea2himrnudsunposb2rlyw6mjhmr7\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"readline\",\"hash\":\"vuu3chywd42dhwbavg2bqogsaqb5u2vw\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"7ikuk745stdvecdqbcrmrczgozpfwopt\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"4g3wvdt4hvdmqqrckn5aqzhebx6cf7t3\"},{\"name\":\"python-venv\",\"version\":\"1.0\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"j3dgyzp5nei24fbpw22l3gedsk37asrdrjafbnaiwiux3lxasi3a====\",\"dependencies\":[{\"name\":\"python\",\"hash\":\"ubenpqhxb6v4lsnefcot2naoyufhtvlq\",\"parameters\":{\"deptypes\":[\"build\",\"run\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"4duigy4ujnstkq5c542w4okhszygw72h\"},{\"name\":\"re2c\",\"version\":\"3.1\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sonoma\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ebw3m3xkgw2wijfijtzrxt4ldu4tz4haiz6juumq6wn4mjzsuxra====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"nea2oy52arwgstum7vyornhbnk3poj32\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"b3urggrazcghz2ngfudq7ndzyhkjstj4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"3b47stocf6w7bbkc3yqakyrjv72ywszk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"rzeea2himrnudsunposb2rlyw6mjhmr7\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"python\",\"hash\":\"ubenpqhxb6v4lsnefcot2naoyufhtvlq\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"uhkg474hzahckbth32ydtp24etgavq76\"}]}}\n" }, { "path": "lib/spack/spack/bootstrap/prototypes/clingo-darwin-x86_64.json", "content": "{\"spec\":{\"_meta\":{\"version\":5},\"nodes\":[{\"name\":\"clingo-bootstrap\",\"version\":\"spack\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"cmake\",\"build_type\":\"Release\",\"docs\":false,\"generator\":\"make\",\"ipo\":false,\"optimized\":false,\"python\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ozkztarkrp3oet7x2oapc7ehdfyvweap45zb3g44mj6qpblv4l3a====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"bison\",\"hash\":\"e575uqnqgn6zxpyrfphfw35vihuc3af3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"cmake\",\"hash\":\"ev3zcv2blhxx2checfszy6736ya2ve45\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"python\",\"hash\":\"3tktfceps6thsraftda3svkdlypt47vx\",\"parameters\":{\"deptypes\":[\"build\",\"link\",\"run\"],\"virtuals\":[]}},{\"name\":\"python-venv\",\"hash\":\"mg6k4cfdhg6dore5avimwxdc7jn6onzs\",\"parameters\":{\"deptypes\":[\"build\",\"run\"],\"virtuals\":[]}},{\"name\":\"re2c\",\"hash\":\"4ym4yrdx4hfbj5rcevsdidy6zdc77om4\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"qrttp2eu44r35mtyem2njmtdo2tr5xvf\"},{\"name\":\"apple-clang\",\"version\":\"16.0.0\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":{\"name\":\"cannonlake\",\"vendor\":\"GenuineIntel\",\"features\":[\"adx\",\"aes\",\"avx\",\"avx2\",\"avx512bw\",\"avx512cd\",\"avx512dq\",\"avx512f\",\"avx512ifma\",\"avx512vbmi\",\"avx512vl\",\"bmi1\",\"bmi2\",\"clflushopt\",\"f16c\",\"fma\",\"mmx\",\"movbe\",\"pclmulqdq\",\"popcnt\",\"rdrand\",\"rdseed\",\"sha\",\"sse\",\"sse2\",\"sse4_1\",\"sse4_2\",\"ssse3\",\"xsavec\",\"xsaveopt\"],\"generation\":0,\"parents\":[\"skylake\"],\"cpupart\":\"\"}},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"bundle\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"external\":{\"path\":\"/usr\",\"module\":null,\"extra_attributes\":{\"compilers\":{\"c\":\"/usr/bin/clang\",\"cxx\":\"/usr/bin/clang++\"}}},\"package_hash\":\"7iabceub7ckyfs2h5g75jxtolk253q6nm3r5hyqbztckky25gnpa====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\"},{\"name\":\"bison\",\"version\":\"3.8.2\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"color\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"d4j62fwvuxqbiez32ltjnhu47ac425wjebsy6fhoptv6saxazcxq====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"diffutils\",\"hash\":\"acwjalgeefeymuhyv4umstcnz465ar6e\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"m4\",\"hash\":\"d36fz4p3yx77w6b272r5yr74owsvwvfm\",\"parameters\":{\"deptypes\":[\"build\",\"run\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"e575uqnqgn6zxpyrfphfw35vihuc3af3\"},{\"name\":\"compiler-wrapper\",\"version\":\"1.0\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":{\"name\":\"cannonlake\",\"vendor\":\"GenuineIntel\",\"features\":[\"adx\",\"aes\",\"avx\",\"avx2\",\"avx512bw\",\"avx512cd\",\"avx512dq\",\"avx512f\",\"avx512ifma\",\"avx512vbmi\",\"avx512vl\",\"bmi1\",\"bmi2\",\"clflushopt\",\"f16c\",\"fma\",\"mmx\",\"movbe\",\"pclmulqdq\",\"popcnt\",\"rdrand\",\"rdseed\",\"sha\",\"sse\",\"sse2\",\"sse4_1\",\"sse4_2\",\"ssse3\",\"xsavec\",\"xsaveopt\"],\"generation\":0,\"parents\":[\"skylake\"],\"cpupart\":\"\"}},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"gogqnfdkxjvnjgj3lndnoncjtdc7ydoc7klkjstywag4oqrvod7a====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\"},{\"name\":\"diffutils\",\"version\":\"3.10\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"kbmzdy7mgklc24qx55cvx7kq7hceby2yav4fnf64gfdo7epdghwa====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libiconv\",\"hash\":\"2hwokn4ijijiclnl3pyvn3b4a7gbn5ct\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"iconv\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"acwjalgeefeymuhyv4umstcnz465ar6e\"},{\"name\":\"gmake\",\"version\":\"4.4.1\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"guile\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"rpzjfobv7qh3wevti34nlbd2emtw5mnyszqmkyiq5jiq33xm7qzq====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\"},{\"name\":\"libiconv\",\"version\":\"1.17\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"libs\":[\"shared\",\"static\"],\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ujsqmcknrabka5mhwwpbaf5rwxgopwoyxkskuwyqlcbynowgdvfa====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"2hwokn4ijijiclnl3pyvn3b4a7gbn5ct\"},{\"name\":\"m4\",\"version\":\"1.4.19\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"patches\":[\"9dc5fbd0d5cb1037ab1e6d0ecc74a30df218d0a94bdd5a02759a97f62daca573\",\"bfdffa7c2eb01021d5849b36972c069693654ad826c1a20b53534009a4ec7a89\"],\"sigsegv\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"patches\":[\"bfdffa7c2eb01021d5849b36972c069693654ad826c1a20b53534009a4ec7a89\",\"9dc5fbd0d5cb1037ab1e6d0ecc74a30df218d0a94bdd5a02759a97f62daca573\"],\"package_hash\":\"npb7a53yz7wqx4nvnasxwgzxaoiks6sdjz2eugrgkjxs4ml24xea====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"diffutils\",\"hash\":\"acwjalgeefeymuhyv4umstcnz465ar6e\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libsigsegv\",\"hash\":\"mtowncjrriz2jjl6onql3wyacciix4ne\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"d36fz4p3yx77w6b272r5yr74owsvwvfm\"},{\"name\":\"libsigsegv\",\"version\":\"2.14\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"3s645t5rbjrziao47mhgob5xgymot6tf4kalagflbal2jdamdo2a====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"mtowncjrriz2jjl6onql3wyacciix4ne\"},{\"name\":\"cmake\",\"version\":\"3.31.2\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"build_type\":\"Release\",\"doc\":false,\"ncurses\":true,\"ownlibs\":true,\"qtgui\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"7vk6yhuq2fklcj5kk7bhreqojudugggezq7vntmcsc32cw2avmya====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"curl\",\"hash\":\"uugvk6k3zupw4xzto2hwjfe647pqsyyf\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"ncurses\",\"hash\":\"nykye5s3jvzc2zwtpx4xljlos6xnorsw\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"l2fyfx2t7sesnitglbumuds2wqflfir6\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"ev3zcv2blhxx2checfszy6736ya2ve45\"},{\"name\":\"curl\",\"version\":\"8.10.1\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"gssapi\":false,\"ldap\":false,\"libidn2\":false,\"librtmp\":false,\"libs\":[\"shared\",\"static\"],\"libssh\":false,\"libssh2\":false,\"nghttp2\":true,\"tls\":[\"secure_transport\"],\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ccka5yawqcn2rjbqn3bkhkdjoajlngm5uab7jbyrsl5yqn42ofza====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"nghttp2\",\"hash\":\"bm2f7poacyin2wyvgq2axmbynhaslhgb\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"pkgconf\",\"hash\":\"e65fgchge7g22kbiqdpyxu4fmvqehlqb\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}},{\"name\":\"zlib-ng\",\"hash\":\"l2fyfx2t7sesnitglbumuds2wqflfir6\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"uugvk6k3zupw4xzto2hwjfe647pqsyyf\"},{\"name\":\"nghttp2\",\"version\":\"1.64.0\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"nkykfkj4rxzmysrmoh5mhxrl5ysaemlqh652m3he7pkbgvjhjgba====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"diffutils\",\"hash\":\"acwjalgeefeymuhyv4umstcnz465ar6e\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"pkgconf\",\"hash\":\"e65fgchge7g22kbiqdpyxu4fmvqehlqb\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"bm2f7poacyin2wyvgq2axmbynhaslhgb\"},{\"name\":\"pkgconf\",\"version\":\"2.2.0\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"gl6tpyarjlclzsal6wa4dtc7cdzprq36nbibalai4a6wgzblrseq====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"e65fgchge7g22kbiqdpyxu4fmvqehlqb\"},{\"name\":\"zlib-ng\",\"version\":\"2.2.1\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"compat\":true,\"new_strategies\":true,\"opt\":true,\"pic\":true,\"shared\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"mdxo2xewbdavckgsqlcjywyfssdchgwbzonui22gxww7hqtozurq====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"l2fyfx2t7sesnitglbumuds2wqflfir6\"},{\"name\":\"ncurses\",\"version\":\"6.5\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"abi\":\"none\",\"build_system\":\"autotools\",\"patches\":[\"7a351bc4953a4ab70dabdbea31c8db0c03d40ce505335f3b6687180dde24c535\"],\"symlinks\":false,\"termlib\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"patches\":[\"7a351bc4953a4ab70dabdbea31c8db0c03d40ce505335f3b6687180dde24c535\"],\"package_hash\":\"cfh76rniab2gnv4jqr77yzz5za4ucfmva2upihvxukn52dybhsvq====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"pkgconf\",\"hash\":\"e65fgchge7g22kbiqdpyxu4fmvqehlqb\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"nykye5s3jvzc2zwtpx4xljlos6xnorsw\"},{\"name\":\"python\",\"version\":\"3.13.0\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"bz2\":true,\"ctypes\":true,\"dbm\":true,\"debug\":false,\"libxml2\":true,\"lzma\":true,\"optimizations\":false,\"pic\":true,\"pyexpat\":true,\"pythoncmd\":true,\"readline\":true,\"shared\":true,\"sqlite3\":true,\"ssl\":true,\"tkinter\":false,\"uuid\":true,\"zlib\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"n6v6rt6deysntdggu2gi4zkhqriyba6bgaghxyhluou4ssqf7xfq====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"apple-libuuid\",\"hash\":\"m5z7kt64hlhnwisipfs5nqqorpi6u6vm\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"uuid\"]}},{\"name\":\"bzip2\",\"hash\":\"h4bmmb7myvboscsbvlgq46twwacgahk3\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"expat\",\"hash\":\"bqc34odirjhz2jaue7n3dk7sux6hmojn\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"gdbm\",\"hash\":\"h4qbc6g2v5yotoalpyvddbcmqyric4v7\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"gettext\",\"hash\":\"m5uhx7get2eeuftgoadtv7r2vfh7u5ds\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libffi\",\"hash\":\"ofcuigaaxlvvv6tgzetfjwfhabzlkbo7\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"ncurses\",\"hash\":\"nykye5s3jvzc2zwtpx4xljlos6xnorsw\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"openssl\",\"hash\":\"bwnlb7yiy67eabhgaei64susdxgas3to\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"pkgconf\",\"hash\":\"e65fgchge7g22kbiqdpyxu4fmvqehlqb\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}},{\"name\":\"readline\",\"hash\":\"5wymz4fw6majnuwaoopp3m7dmjqbbvrx\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"sqlite\",\"hash\":\"qtxu5k6vkkyr2oii62lz7r4ubs7sz3xq\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"xz\",\"hash\":\"oxzkcdzjdywney64q6tnmmjib33u6ms7\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"l2fyfx2t7sesnitglbumuds2wqflfir6\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"3tktfceps6thsraftda3svkdlypt47vx\"},{\"name\":\"apple-libuuid\",\"version\":\"1353.100.2\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"bundle\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"external\":{\"path\":\"/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk\",\"module\":null,\"extra_attributes\":{}},\"package_hash\":\"rv7eeukm7m2umg6ulafeco2qz2kvaqpx2bjoita6g27hrs6vfmiq====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"m5z7kt64hlhnwisipfs5nqqorpi6u6vm\"},{\"name\":\"bzip2\",\"version\":\"1.0.8\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"debug\":false,\"pic\":false,\"shared\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"jb7yvhkifmvfl3ykmdulsjxkkulker6gqb5tadollyjt2ijg3zsa====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"diffutils\",\"hash\":\"acwjalgeefeymuhyv4umstcnz465ar6e\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"h4bmmb7myvboscsbvlgq46twwacgahk3\"},{\"name\":\"expat\",\"version\":\"2.6.4\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"libbsd\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ei6qyjakl7sgtodwxxbg5brgkp23robfximtpbedkrnpyyyvr3ya====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"bqc34odirjhz2jaue7n3dk7sux6hmojn\"},{\"name\":\"gdbm\",\"version\":\"1.23\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"liepxl6phlcxbgfmibxafhewtihlgaa4x3hko37ckqlafhxkrgdq====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"readline\",\"hash\":\"5wymz4fw6majnuwaoopp3m7dmjqbbvrx\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"h4qbc6g2v5yotoalpyvddbcmqyric4v7\"},{\"name\":\"readline\",\"version\":\"8.2\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"patches\":[\"bbf97f1ec40a929edab5aa81998c1e2ef435436c597754916e6a5868f273aff7\"],\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"patches\":[\"bbf97f1ec40a929edab5aa81998c1e2ef435436c597754916e6a5868f273aff7\"],\"package_hash\":\"oww6dmr7xqgg6j7iiluonxbcl4irqnnrip4vfkjdwujncwnuhwuq====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"ncurses\",\"hash\":\"nykye5s3jvzc2zwtpx4xljlos6xnorsw\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"5wymz4fw6majnuwaoopp3m7dmjqbbvrx\"},{\"name\":\"gettext\",\"version\":\"0.22.5\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"bzip2\":true,\"curses\":true,\"git\":true,\"libunistring\":false,\"libxml2\":true,\"pic\":true,\"shared\":true,\"tar\":true,\"xz\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"d4zxhmw6rownaaokzcolsszrq2cmx44m7qmzopucymoyrhbdfgvq====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"bzip2\",\"hash\":\"h4bmmb7myvboscsbvlgq46twwacgahk3\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libiconv\",\"hash\":\"2hwokn4ijijiclnl3pyvn3b4a7gbn5ct\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"iconv\"]}},{\"name\":\"libxml2\",\"hash\":\"patsnnun4o2w3vupeontcjecxeoyh2js\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"ncurses\",\"hash\":\"nykye5s3jvzc2zwtpx4xljlos6xnorsw\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"tar\",\"hash\":\"tegwze36okijyiui4nbbnkn2ngkqmxlm\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}},{\"name\":\"xz\",\"hash\":\"oxzkcdzjdywney64q6tnmmjib33u6ms7\",\"parameters\":{\"deptypes\":[\"build\",\"link\",\"run\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"m5uhx7get2eeuftgoadtv7r2vfh7u5ds\"},{\"name\":\"libxml2\",\"version\":\"2.13.4\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"pic\":true,\"python\":false,\"shared\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"j6yob2wgvc2wjzvbs6xdvgyfa3zp3wrm3uxncxzxqfzw6xazzoba====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libiconv\",\"hash\":\"2hwokn4ijijiclnl3pyvn3b4a7gbn5ct\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"iconv\"]}},{\"name\":\"pkgconf\",\"hash\":\"e65fgchge7g22kbiqdpyxu4fmvqehlqb\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}},{\"name\":\"xz\",\"hash\":\"oxzkcdzjdywney64q6tnmmjib33u6ms7\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"l2fyfx2t7sesnitglbumuds2wqflfir6\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"patsnnun4o2w3vupeontcjecxeoyh2js\"},{\"name\":\"xz\",\"version\":\"5.4.6\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"libs\":[\"shared\",\"static\"],\"pic\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"zt5vu2vph2v2qjwgdbe7btgcz7axpyalorcsqiuxhrg5grwgrrvq====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"oxzkcdzjdywney64q6tnmmjib33u6ms7\"},{\"name\":\"tar\",\"version\":\"1.35\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"zip\":\"pigz\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"v6a6jvks2setklucxyk622uauxzqlgmsdkrvdijbi3m5jwftmzla====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"bzip2\",\"hash\":\"h4bmmb7myvboscsbvlgq46twwacgahk3\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libiconv\",\"hash\":\"2hwokn4ijijiclnl3pyvn3b4a7gbn5ct\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"iconv\"]}},{\"name\":\"pigz\",\"hash\":\"rv3drbcskvc7snlhqex2byavaddd6xfy\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}},{\"name\":\"xz\",\"hash\":\"oxzkcdzjdywney64q6tnmmjib33u6ms7\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}},{\"name\":\"zstd\",\"hash\":\"isadyc5phrez7pmz4spx4zly5wu5pslt\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"tegwze36okijyiui4nbbnkn2ngkqmxlm\"},{\"name\":\"pigz\",\"version\":\"2.8\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"makefile\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"4w67lflje4giekjg4ie2vpyuiunjcumo6geofykvon3hodllp42q====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"l2fyfx2t7sesnitglbumuds2wqflfir6\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"rv3drbcskvc7snlhqex2byavaddd6xfy\"},{\"name\":\"zstd\",\"version\":\"1.5.6\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"makefile\",\"compression\":[\"none\"],\"libs\":[\"shared\",\"static\"],\"programs\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"uvmrov4c6unft6o4yd3jk3uqvweua3uhwdli4sw7h5wvklaf5t3q====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"isadyc5phrez7pmz4spx4zly5wu5pslt\"},{\"name\":\"libffi\",\"version\":\"3.4.6\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"umhsnvoj5ooa3glffnkk2hp3txmrsjvqbpfq2hbk4mhcvhza7gaa====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"ofcuigaaxlvvv6tgzetfjwfhabzlkbo7\"},{\"name\":\"openssl\",\"version\":\"3.4.0\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"certs\":\"mozilla\",\"docs\":false,\"shared\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"5y33vxwjtlrlsyedasvmhukjkk5yfwcri27oceh36iw73xehumfa====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"ca-certificates-mozilla\",\"hash\":\"xinl4agw3xhagk74cw2pclmlbqoq223j\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"perl\",\"hash\":\"wxl5qpzezncbick5ygjx3fnqwpd3ousb\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"l2fyfx2t7sesnitglbumuds2wqflfir6\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"bwnlb7yiy67eabhgaei64susdxgas3to\"},{\"name\":\"ca-certificates-mozilla\",\"version\":\"2023-05-30\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"63npvwqwo2x7i6emvnklh4mhcn45gx2qzveorybh5h2inwr55sea====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"xinl4agw3xhagk74cw2pclmlbqoq223j\"},{\"name\":\"perl\",\"version\":\"5.40.0\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cpanm\":true,\"opcode\":true,\"open\":true,\"shared\":true,\"threads\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"f233ue76vwtkle2r4jwsfe5x27ujx6ea4vdyp6baonfmkgqf5vpa====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"berkeley-db\",\"hash\":\"wamcmlsv3jtpzy7qvmfful4fabex5q7y\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"bzip2\",\"hash\":\"h4bmmb7myvboscsbvlgq46twwacgahk3\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gdbm\",\"hash\":\"h4qbc6g2v5yotoalpyvddbcmqyric4v7\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"l2fyfx2t7sesnitglbumuds2wqflfir6\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"wxl5qpzezncbick5ygjx3fnqwpd3ousb\"},{\"name\":\"berkeley-db\",\"version\":\"18.1.40\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cxx\":true,\"docs\":false,\"patches\":[\"26090f418891757af46ac3b89a9f43d6eb5989f7a3dce3d1cfc99fba547203b3\",\"b231fcc4d5cff05e5c3a4814f6a5af0e9a966428dc2176540d2c05aff41de522\"],\"stl\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"patches\":[\"b231fcc4d5cff05e5c3a4814f6a5af0e9a966428dc2176540d2c05aff41de522\",\"26090f418891757af46ac3b89a9f43d6eb5989f7a3dce3d1cfc99fba547203b3\"],\"package_hash\":\"h57ydfn33zevvzctzzioiiwjwe362izbbwncb6a26dfeno4y7tda====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"wamcmlsv3jtpzy7qvmfful4fabex5q7y\"},{\"name\":\"sqlite\",\"version\":\"3.46.0\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"column_metadata\":true,\"dynamic_extensions\":true,\"fts\":true,\"functions\":false,\"rtree\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"wm3irnrjil5n275nw2m4x3mpvyg35h7isbmsnuae6vtxbamsrv4q====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"readline\",\"hash\":\"5wymz4fw6majnuwaoopp3m7dmjqbbvrx\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"l2fyfx2t7sesnitglbumuds2wqflfir6\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"qtxu5k6vkkyr2oii62lz7r4ubs7sz3xq\"},{\"name\":\"python-venv\",\"version\":\"1.0\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"j3dgyzp5nei24fbpw22l3gedsk37asrdrjafbnaiwiux3lxasi3a====\",\"dependencies\":[{\"name\":\"python\",\"hash\":\"3tktfceps6thsraftda3svkdlypt47vx\",\"parameters\":{\"deptypes\":[\"build\",\"run\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"mg6k4cfdhg6dore5avimwxdc7jn6onzs\"},{\"name\":\"re2c\",\"version\":\"3.1\",\"arch\":{\"platform\":\"darwin\",\"platform_os\":\"sequoia\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ebw3m3xkgw2wijfijtzrxt4ldu4tz4haiz6juumq6wn4mjzsuxra====\",\"dependencies\":[{\"name\":\"apple-clang\",\"hash\":\"qj3zadkktznahfizazbfvmqvkhzd4bqv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"compiler-wrapper\",\"hash\":\"2uitb26t2s6nfpj244fbsh7gntsiwvge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"fszz4zptmmipakokiufglsphlmdgb6x3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"python\",\"hash\":\"3tktfceps6thsraftda3svkdlypt47vx\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"4ym4yrdx4hfbj5rcevsdidy6zdc77om4\"}]}}\n" }, { "path": "lib/spack/spack/bootstrap/prototypes/clingo-freebsd-amd64.json", "content": "{\"spec\":{\"_meta\":{\"version\":5},\"nodes\":[{\"name\":\"clingo-bootstrap\",\"version\":\"spack\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"cmake\",\"build_type\":\"Release\",\"docs\":false,\"generator\":\"make\",\"ipo\":false,\"optimized\":false,\"python\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ozkztarkrp3oet7x2oapc7ehdfyvweap45zb3g44mj6qpblv4l3a====\",\"dependencies\":[{\"name\":\"bison\",\"hash\":\"l4llzdyliqbeor66ht54qkezfdofmwj6\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"cmake\",\"hash\":\"okz75726c4grndc4kadvpivfbr6546ud\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"python\",\"hash\":\"syeuozebaclogvjl7izswkitiduyniob\",\"parameters\":{\"deptypes\":[\"build\",\"link\",\"run\"],\"virtuals\":[]}},{\"name\":\"python-venv\",\"hash\":\"nw53taerhuinrvwfc6gcg4hztg77dkq5\",\"parameters\":{\"deptypes\":[\"build\",\"run\"],\"virtuals\":[]}},{\"name\":\"re2c\",\"hash\":\"prd7dmeald2bitrpbt6cqdcslfap5aay\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"je2szed32t2zoajsczveb4bokeitrcan\"},{\"name\":\"bison\",\"version\":\"3.8.2\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"color\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"d4j62fwvuxqbiez32ltjnhu47ac425wjebsy6fhoptv6saxazcxq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"diffutils\",\"hash\":\"nk5z5kralivpxqazpvgmxvqdm73mimpx\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"m4\",\"hash\":\"hoq7tejwrsetrepd4kjww3yvxfraycsa\",\"parameters\":{\"deptypes\":[\"build\",\"run\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"l4llzdyliqbeor66ht54qkezfdofmwj6\"},{\"name\":\"compiler-wrapper\",\"version\":\"1.0\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"gogqnfdkxjvnjgj3lndnoncjtdc7ydoc7klkjstywag4oqrvod7a====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\"},{\"name\":\"diffutils\",\"version\":\"3.10\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"kbmzdy7mgklc24qx55cvx7kq7hceby2yav4fnf64gfdo7epdghwa====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libiconv\",\"hash\":\"nnr7brz74vmypk3gfhyykql5rvshhxiu\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"iconv\"]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"nk5z5kralivpxqazpvgmxvqdm73mimpx\"},{\"name\":\"gmake\",\"version\":\"4.4.1\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"guile\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"rpzjfobv7qh3wevti34nlbd2emtw5mnyszqmkyiq5jiq33xm7qzq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\"},{\"name\":\"llvm\",\"version\":\"18.1.5\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"cmake\",\"build_type\":\"Release\",\"clang\":true,\"compiler-rt\":\"runtime\",\"cuda\":false,\"flang\":false,\"generator\":\"ninja\",\"gold\":true,\"libcxx\":\"runtime\",\"libomptarget\":true,\"libomptarget_debug\":false,\"libunwind\":\"runtime\",\"link_llvm_dylib\":false,\"lld\":false,\"lldb\":true,\"llvm_dylib\":true,\"lua\":true,\"mlir\":false,\"openmp\":\"runtime\",\"polly\":true,\"python\":false,\"shlib_symbol_version\":\"none\",\"split_dwarf\":false,\"targets\":[\"all\"],\"version_suffix\":\"none\",\"z3\":false,\"zstd\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"external\":{\"path\":\"/usr\",\"module\":null,\"extra_attributes\":{\"compilers\":{\"c\":\"/usr/bin/clang\",\"cxx\":\"/usr/bin/clang++\"}}},\"package_hash\":\"7iourbijxpsp23e2wj3fel2fmmk23jzyzidcpqdgeux7g7ff2wxq====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\"},{\"name\":\"libiconv\",\"version\":\"1.17\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"libs\":[\"shared\",\"static\"],\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ujsqmcknrabka5mhwwpbaf5rwxgopwoyxkskuwyqlcbynowgdvfa====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"nnr7brz74vmypk3gfhyykql5rvshhxiu\"},{\"name\":\"m4\",\"version\":\"1.4.19\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"patches\":[\"9dc5fbd0d5cb1037ab1e6d0ecc74a30df218d0a94bdd5a02759a97f62daca573\",\"bfdffa7c2eb01021d5849b36972c069693654ad826c1a20b53534009a4ec7a89\"],\"sigsegv\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"patches\":[\"bfdffa7c2eb01021d5849b36972c069693654ad826c1a20b53534009a4ec7a89\",\"9dc5fbd0d5cb1037ab1e6d0ecc74a30df218d0a94bdd5a02759a97f62daca573\"],\"package_hash\":\"npb7a53yz7wqx4nvnasxwgzxaoiks6sdjz2eugrgkjxs4ml24xea====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"diffutils\",\"hash\":\"nk5z5kralivpxqazpvgmxvqdm73mimpx\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libsigsegv\",\"hash\":\"guzz5zr4juvhrq4pqxnibvoma5z3djfi\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"hoq7tejwrsetrepd4kjww3yvxfraycsa\"},{\"name\":\"libsigsegv\",\"version\":\"2.14\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"3s645t5rbjrziao47mhgob5xgymot6tf4kalagflbal2jdamdo2a====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"guzz5zr4juvhrq4pqxnibvoma5z3djfi\"},{\"name\":\"cmake\",\"version\":\"3.31.2\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"build_type\":\"Release\",\"doc\":false,\"ncurses\":true,\"ownlibs\":true,\"qtgui\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"7vk6yhuq2fklcj5kk7bhreqojudugggezq7vntmcsc32cw2avmya====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"curl\",\"hash\":\"rkyymoo7xqnswutyvauf3iv5dddmaygt\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"ncurses\",\"hash\":\"p2m3nzytg5lh6474vclnqtklvk6jpqos\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"c2jgry3yzjofxxjuqjckluoqbcm5exix\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"okz75726c4grndc4kadvpivfbr6546ud\"},{\"name\":\"curl\",\"version\":\"8.10.1\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"gssapi\":false,\"ldap\":false,\"libidn2\":false,\"librtmp\":false,\"libs\":[\"shared\",\"static\"],\"libssh\":false,\"libssh2\":false,\"nghttp2\":true,\"tls\":[\"openssl\"],\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ccka5yawqcn2rjbqn3bkhkdjoajlngm5uab7jbyrsl5yqn42ofza====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"nghttp2\",\"hash\":\"uuslnsztro7in3mxykjmrolg2wfdoyat\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"openssl\",\"hash\":\"c6ojqefenrbxkupgaqznti6q2x3g22qf\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"pkgconf\",\"hash\":\"yc2rz24ll3ulloccgxroltp5243csskb\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}},{\"name\":\"zlib-ng\",\"hash\":\"c2jgry3yzjofxxjuqjckluoqbcm5exix\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"rkyymoo7xqnswutyvauf3iv5dddmaygt\"},{\"name\":\"nghttp2\",\"version\":\"1.64.0\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"nkykfkj4rxzmysrmoh5mhxrl5ysaemlqh652m3he7pkbgvjhjgba====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"diffutils\",\"hash\":\"nk5z5kralivpxqazpvgmxvqdm73mimpx\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"pkgconf\",\"hash\":\"yc2rz24ll3ulloccgxroltp5243csskb\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"uuslnsztro7in3mxykjmrolg2wfdoyat\"},{\"name\":\"pkgconf\",\"version\":\"2.2.0\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"gl6tpyarjlclzsal6wa4dtc7cdzprq36nbibalai4a6wgzblrseq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"yc2rz24ll3ulloccgxroltp5243csskb\"},{\"name\":\"openssl\",\"version\":\"3.4.0\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"certs\":\"mozilla\",\"docs\":false,\"shared\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"5y33vxwjtlrlsyedasvmhukjkk5yfwcri27oceh36iw73xehumfa====\",\"dependencies\":[{\"name\":\"ca-certificates-mozilla\",\"hash\":\"hm3nrr2yydcptn7fvphwvg6bwyo75bwf\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"perl\",\"hash\":\"sadirf62yvikut4yghjhph6o5tztfwao\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"c2jgry3yzjofxxjuqjckluoqbcm5exix\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"c6ojqefenrbxkupgaqznti6q2x3g22qf\"},{\"name\":\"ca-certificates-mozilla\",\"version\":\"2023-05-30\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"63npvwqwo2x7i6emvnklh4mhcn45gx2qzveorybh5h2inwr55sea====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"hm3nrr2yydcptn7fvphwvg6bwyo75bwf\"},{\"name\":\"perl\",\"version\":\"5.40.0\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cpanm\":true,\"opcode\":true,\"open\":true,\"shared\":true,\"threads\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"f233ue76vwtkle2r4jwsfe5x27ujx6ea4vdyp6baonfmkgqf5vpa====\",\"dependencies\":[{\"name\":\"berkeley-db\",\"hash\":\"vncqfho5tjvizrhfpr4vft5nfyawkhw2\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"bzip2\",\"hash\":\"utn5hm325756qkbf3ve5na2qtac7zxc5\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gdbm\",\"hash\":\"ktpz7bar56pafbw2ab5rerdejfwnngjd\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"zlib-ng\",\"hash\":\"c2jgry3yzjofxxjuqjckluoqbcm5exix\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"sadirf62yvikut4yghjhph6o5tztfwao\"},{\"name\":\"berkeley-db\",\"version\":\"18.1.40\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cxx\":true,\"docs\":false,\"patches\":[\"26090f418891757af46ac3b89a9f43d6eb5989f7a3dce3d1cfc99fba547203b3\",\"b231fcc4d5cff05e5c3a4814f6a5af0e9a966428dc2176540d2c05aff41de522\"],\"stl\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"patches\":[\"b231fcc4d5cff05e5c3a4814f6a5af0e9a966428dc2176540d2c05aff41de522\",\"26090f418891757af46ac3b89a9f43d6eb5989f7a3dce3d1cfc99fba547203b3\"],\"package_hash\":\"h57ydfn33zevvzctzzioiiwjwe362izbbwncb6a26dfeno4y7tda====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"vncqfho5tjvizrhfpr4vft5nfyawkhw2\"},{\"name\":\"bzip2\",\"version\":\"1.0.8\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"debug\":false,\"pic\":false,\"shared\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"jb7yvhkifmvfl3ykmdulsjxkkulker6gqb5tadollyjt2ijg3zsa====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"diffutils\",\"hash\":\"nk5z5kralivpxqazpvgmxvqdm73mimpx\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"utn5hm325756qkbf3ve5na2qtac7zxc5\"},{\"name\":\"gdbm\",\"version\":\"1.23\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"liepxl6phlcxbgfmibxafhewtihlgaa4x3hko37ckqlafhxkrgdq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"readline\",\"hash\":\"nixpi6ugx6vmxbxln5ceyqxnu2sypnlx\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"ktpz7bar56pafbw2ab5rerdejfwnngjd\"},{\"name\":\"readline\",\"version\":\"8.2\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"patches\":[\"bbf97f1ec40a929edab5aa81998c1e2ef435436c597754916e6a5868f273aff7\"],\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"patches\":[\"bbf97f1ec40a929edab5aa81998c1e2ef435436c597754916e6a5868f273aff7\"],\"package_hash\":\"oww6dmr7xqgg6j7iiluonxbcl4irqnnrip4vfkjdwujncwnuhwuq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"ncurses\",\"hash\":\"p2m3nzytg5lh6474vclnqtklvk6jpqos\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"nixpi6ugx6vmxbxln5ceyqxnu2sypnlx\"},{\"name\":\"ncurses\",\"version\":\"6.5\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"abi\":\"none\",\"build_system\":\"autotools\",\"patches\":[\"7a351bc4953a4ab70dabdbea31c8db0c03d40ce505335f3b6687180dde24c535\"],\"symlinks\":false,\"termlib\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"patches\":[\"7a351bc4953a4ab70dabdbea31c8db0c03d40ce505335f3b6687180dde24c535\"],\"package_hash\":\"cfh76rniab2gnv4jqr77yzz5za4ucfmva2upihvxukn52dybhsvq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"pkgconf\",\"hash\":\"yc2rz24ll3ulloccgxroltp5243csskb\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"p2m3nzytg5lh6474vclnqtklvk6jpqos\"},{\"name\":\"zlib-ng\",\"version\":\"2.2.1\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"compat\":true,\"new_strategies\":true,\"opt\":true,\"pic\":true,\"shared\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"mdxo2xewbdavckgsqlcjywyfssdchgwbzonui22gxww7hqtozurq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"c2jgry3yzjofxxjuqjckluoqbcm5exix\"},{\"name\":\"python\",\"version\":\"3.13.0\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"bz2\":true,\"ctypes\":true,\"dbm\":true,\"debug\":false,\"libxml2\":true,\"lzma\":true,\"optimizations\":false,\"pic\":true,\"pyexpat\":true,\"pythoncmd\":true,\"readline\":true,\"shared\":true,\"sqlite3\":true,\"ssl\":true,\"tkinter\":false,\"uuid\":true,\"zlib\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"n6v6rt6deysntdggu2gi4zkhqriyba6bgaghxyhluou4ssqf7xfq====\",\"dependencies\":[{\"name\":\"bzip2\",\"hash\":\"utn5hm325756qkbf3ve5na2qtac7zxc5\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"expat\",\"hash\":\"djhfx5nxzsatwcklt743hizybmgvq75l\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"gdbm\",\"hash\":\"ktpz7bar56pafbw2ab5rerdejfwnngjd\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"gettext\",\"hash\":\"nfyjnvifb6n3v55esjgk7rinnq6e7av2\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libffi\",\"hash\":\"657brzxsad4zh6ajeiriuatlxaco5beg\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"ncurses\",\"hash\":\"p2m3nzytg5lh6474vclnqtklvk6jpqos\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"openssl\",\"hash\":\"c6ojqefenrbxkupgaqznti6q2x3g22qf\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"pkgconf\",\"hash\":\"yc2rz24ll3ulloccgxroltp5243csskb\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}},{\"name\":\"readline\",\"hash\":\"nixpi6ugx6vmxbxln5ceyqxnu2sypnlx\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"sqlite\",\"hash\":\"dcqokkasxhtuu7g7htoi2v5btc2b63qf\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"util-linux-uuid\",\"hash\":\"5u5klk6jrayvbilllhrlbszendi5liip\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"uuid\"]}},{\"name\":\"xz\",\"hash\":\"6cqtdj22u47rdbvycoylphh7d6jbrvq4\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"c2jgry3yzjofxxjuqjckluoqbcm5exix\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"syeuozebaclogvjl7izswkitiduyniob\"},{\"name\":\"expat\",\"version\":\"2.6.4\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"libbsd\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ei6qyjakl7sgtodwxxbg5brgkp23robfximtpbedkrnpyyyvr3ya====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"djhfx5nxzsatwcklt743hizybmgvq75l\"},{\"name\":\"gettext\",\"version\":\"0.22.5\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"bzip2\":true,\"curses\":true,\"git\":true,\"libunistring\":false,\"libxml2\":true,\"pic\":true,\"shared\":true,\"tar\":true,\"xz\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"d4zxhmw6rownaaokzcolsszrq2cmx44m7qmzopucymoyrhbdfgvq====\",\"dependencies\":[{\"name\":\"bzip2\",\"hash\":\"utn5hm325756qkbf3ve5na2qtac7zxc5\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libiconv\",\"hash\":\"nnr7brz74vmypk3gfhyykql5rvshhxiu\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"iconv\"]}},{\"name\":\"libxml2\",\"hash\":\"gkoikjiianqwi3r7ynsrj5kczj36mufp\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"ncurses\",\"hash\":\"p2m3nzytg5lh6474vclnqtklvk6jpqos\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"tar\",\"hash\":\"pygs7gph2cxutw2jktsvex3vxb2nl7hl\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}},{\"name\":\"xz\",\"hash\":\"6cqtdj22u47rdbvycoylphh7d6jbrvq4\",\"parameters\":{\"deptypes\":[\"build\",\"link\",\"run\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"nfyjnvifb6n3v55esjgk7rinnq6e7av2\"},{\"name\":\"libxml2\",\"version\":\"2.13.4\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"pic\":true,\"python\":false,\"shared\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"j6yob2wgvc2wjzvbs6xdvgyfa3zp3wrm3uxncxzxqfzw6xazzoba====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libiconv\",\"hash\":\"nnr7brz74vmypk3gfhyykql5rvshhxiu\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"iconv\"]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"pkgconf\",\"hash\":\"yc2rz24ll3ulloccgxroltp5243csskb\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}},{\"name\":\"xz\",\"hash\":\"6cqtdj22u47rdbvycoylphh7d6jbrvq4\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"c2jgry3yzjofxxjuqjckluoqbcm5exix\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"gkoikjiianqwi3r7ynsrj5kczj36mufp\"},{\"name\":\"xz\",\"version\":\"5.4.6\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"libs\":[\"shared\",\"static\"],\"pic\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"zt5vu2vph2v2qjwgdbe7btgcz7axpyalorcsqiuxhrg5grwgrrvq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"6cqtdj22u47rdbvycoylphh7d6jbrvq4\"},{\"name\":\"tar\",\"version\":\"1.35\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"zip\":\"pigz\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"v6a6jvks2setklucxyk622uauxzqlgmsdkrvdijbi3m5jwftmzla====\",\"dependencies\":[{\"name\":\"bzip2\",\"hash\":\"utn5hm325756qkbf3ve5na2qtac7zxc5\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libiconv\",\"hash\":\"nnr7brz74vmypk3gfhyykql5rvshhxiu\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"iconv\"]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"pigz\",\"hash\":\"f5jym2egytrgpubdtunmqolh7ioaaudm\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}},{\"name\":\"xz\",\"hash\":\"6cqtdj22u47rdbvycoylphh7d6jbrvq4\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}},{\"name\":\"zstd\",\"hash\":\"7niz2hlqarxclxncsbbzl7zx5uo3btrq\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"pygs7gph2cxutw2jktsvex3vxb2nl7hl\"},{\"name\":\"pigz\",\"version\":\"2.8\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"makefile\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"4w67lflje4giekjg4ie2vpyuiunjcumo6geofykvon3hodllp42q====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"zlib-ng\",\"hash\":\"c2jgry3yzjofxxjuqjckluoqbcm5exix\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"f5jym2egytrgpubdtunmqolh7ioaaudm\"},{\"name\":\"zstd\",\"version\":\"1.5.6\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"makefile\",\"compression\":[\"none\"],\"libs\":[\"shared\",\"static\"],\"programs\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"uvmrov4c6unft6o4yd3jk3uqvweua3uhwdli4sw7h5wvklaf5t3q====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"7niz2hlqarxclxncsbbzl7zx5uo3btrq\"},{\"name\":\"libffi\",\"version\":\"3.4.6\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"umhsnvoj5ooa3glffnkk2hp3txmrsjvqbpfq2hbk4mhcvhza7gaa====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"657brzxsad4zh6ajeiriuatlxaco5beg\"},{\"name\":\"sqlite\",\"version\":\"3.46.0\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"column_metadata\":true,\"dynamic_extensions\":true,\"fts\":true,\"functions\":false,\"rtree\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"wm3irnrjil5n275nw2m4x3mpvyg35h7isbmsnuae6vtxbamsrv4q====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"readline\",\"hash\":\"nixpi6ugx6vmxbxln5ceyqxnu2sypnlx\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"c2jgry3yzjofxxjuqjckluoqbcm5exix\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"dcqokkasxhtuu7g7htoi2v5btc2b63qf\"},{\"name\":\"util-linux-uuid\",\"version\":\"2.40.2\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"eo6au7zhsz344imzoomhuskbl3cmrqq6ja6mcmrc3li3fnppqs6q====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"pkgconf\",\"hash\":\"yc2rz24ll3ulloccgxroltp5243csskb\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"5u5klk6jrayvbilllhrlbszendi5liip\"},{\"name\":\"python-venv\",\"version\":\"1.0\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"j3dgyzp5nei24fbpw22l3gedsk37asrdrjafbnaiwiux3lxasi3a====\",\"dependencies\":[{\"name\":\"python\",\"hash\":\"syeuozebaclogvjl7izswkitiduyniob\",\"parameters\":{\"deptypes\":[\"build\",\"run\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"nw53taerhuinrvwfc6gcg4hztg77dkq5\"},{\"name\":\"re2c\",\"version\":\"3.1\",\"arch\":{\"platform\":\"freebsd\",\"platform_os\":\"freebsd14.1\",\"target\":\"amd64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ebw3m3xkgw2wijfijtzrxt4ldu4tz4haiz6juumq6wn4mjzsuxra====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"6o4jkave5ri3ooytknfil4p55ifcwxju\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gmake\",\"hash\":\"clafylgtxlepfvfrhjfqgfg2fc52vho3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"llvm\",\"hash\":\"ujjiokwbw55sm7o6zoajb3xtcs65utxg\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"python\",\"hash\":\"syeuozebaclogvjl7izswkitiduyniob\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"prd7dmeald2bitrpbt6cqdcslfap5aay\"}]}}\n" }, { "path": "lib/spack/spack/bootstrap/prototypes/clingo-linux-aarch64.json", "content": "{\"spec\":{\"_meta\":{\"version\":5},\"nodes\":[{\"name\":\"clingo-bootstrap\",\"version\":\"spack\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"cmake\",\"build_type\":\"Release\",\"docs\":false,\"generator\":\"make\",\"ipo\":false,\"optimized\":false,\"python\":true,\"static_libstdcpp\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ozkztarkrp3oet7x2oapc7ehdfyvweap45zb3g44mj6qpblv4l3a====\",\"dependencies\":[{\"name\":\"bison\",\"hash\":\"smnn2cumnp72tnrnnr6igudxyvtriqdk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"cmake\",\"hash\":\"ltrb7aes3hwdnz27nzndzsmbv2vnw6wy\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"python\",\"hash\":\"gygxuzpdf33jg2ya6imlbn4bd5zghbcd\",\"parameters\":{\"deptypes\":[\"build\",\"link\",\"run\"],\"virtuals\":[]}},{\"name\":\"python-venv\",\"hash\":\"hf5bgjk6fsdycb4zovjap4t4g6tjfcvx\",\"parameters\":{\"deptypes\":[\"build\",\"run\"],\"virtuals\":[]}},{\"name\":\"re2c\",\"hash\":\"eavspn7qgilrfiby4v6in34pmjg5le6b\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"fxkanrgnzq7yhegi7z5de6ax7i5dablo\"},{\"name\":\"bison\",\"version\":\"3.8.2\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"color\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"d4j62fwvuxqbiez32ltjnhu47ac425wjebsy6fhoptv6saxazcxq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"diffutils\",\"hash\":\"kntg5epaheq5s2cpiqskcfu3do6nikge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"z2hnkln52bnc5tbjkhtjv7n2av52a5eh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"m4\",\"hash\":\"jjtr2n3inumkcqn26fnznvt3ek5ddknd\",\"parameters\":{\"deptypes\":[\"build\",\"run\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"smnn2cumnp72tnrnnr6igudxyvtriqdk\"},{\"name\":\"compiler-wrapper\",\"version\":\"1.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":{\"name\":\"neoverse_v2\",\"vendor\":\"ARM\",\"features\":[\"aes\",\"asimd\",\"asimddp\",\"asimdfhm\",\"asimdhp\",\"asimdrdm\",\"atomics\",\"bf16\",\"cpuid\",\"crc32\",\"dcpodp\",\"dcpop\",\"evtstrm\",\"fcma\",\"flagm\",\"flagm2\",\"fp\",\"fphp\",\"frint\",\"i8mm\",\"ilrcpc\",\"jscvt\",\"lrcpc\",\"pmull\",\"sb\",\"sha1\",\"sha2\",\"sha3\",\"sha512\",\"sve\",\"sve2\",\"svebf16\",\"svei8mm\",\"uscat\"],\"generation\":0,\"parents\":[\"neoverse_n1\",\"armv9.0a\"],\"cpupart\":\"0xd4f\"}},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"gogqnfdkxjvnjgj3lndnoncjtdc7ydoc7klkjstywag4oqrvod7a====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\"},{\"name\":\"diffutils\",\"version\":\"3.10\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"kbmzdy7mgklc24qx55cvx7kq7hceby2yav4fnf64gfdo7epdghwa====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"z2hnkln52bnc5tbjkhtjv7n2av52a5eh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libiconv\",\"hash\":\"bqcb2qmnv3vsz5u7b3whbrortoieu6bx\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"iconv\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"kntg5epaheq5s2cpiqskcfu3do6nikge\"},{\"name\":\"gcc\",\"version\":\"8.5.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":{\"name\":\"neoverse_v2\",\"vendor\":\"ARM\",\"features\":[\"aes\",\"asimd\",\"asimddp\",\"asimdfhm\",\"asimdhp\",\"asimdrdm\",\"atomics\",\"bf16\",\"cpuid\",\"crc32\",\"dcpodp\",\"dcpop\",\"evtstrm\",\"fcma\",\"flagm\",\"flagm2\",\"fp\",\"fphp\",\"frint\",\"i8mm\",\"ilrcpc\",\"jscvt\",\"lrcpc\",\"pmull\",\"sb\",\"sha1\",\"sha2\",\"sha3\",\"sha512\",\"sve\",\"sve2\",\"svebf16\",\"svei8mm\",\"uscat\"],\"generation\":0,\"parents\":[\"neoverse_n1\",\"armv9.0a\"],\"cpupart\":\"0xd4f\"}},\"namespace\":\"builtin\",\"parameters\":{\"binutils\":false,\"bootstrap\":true,\"build_system\":\"autotools\",\"build_type\":\"RelWithDebInfo\",\"graphite\":false,\"languages\":[\"c\",\"c++\",\"fortran\"],\"nvptx\":false,\"patches\":[\"98a9c96f66ff0264a49bd5e76fd2ba177ceca7c7236f486058a8469c2bcd1b76\",\"d4919d68d5460049d370e79ff78bbc320cfe66a7fdf6dfc92cf7e133152b2d56\"],\"piclibs\":false,\"strip\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"external\":{\"path\":\"/usr\",\"module\":null,\"extra_attributes\":{\"compilers\":{\"c\":\"/usr/bin/gcc\",\"cxx\":\"/usr/bin/g++\"}}},\"patches\":[\"98a9c96f66ff0264a49bd5e76fd2ba177ceca7c7236f486058a8469c2bcd1b76\",\"d4919d68d5460049d370e79ff78bbc320cfe66a7fdf6dfc92cf7e133152b2d56\"],\"package_hash\":\"hnbtowhwympdfoqukgir3chmkqzzasrgcwxbot7im4bncvqtxvxq====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\"},{\"name\":\"gcc-runtime\",\"version\":\"8.5.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"aud4d72goxupc5p3p6mdkwgtshpygn7uuj2ewx3zm6wudcgw4fzq====\",\"dependencies\":[{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\"},{\"name\":\"glibc\",\"version\":\"2.34\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"external\":{\"path\":\"/usr\",\"module\":null,\"extra_attributes\":{}},\"package_hash\":\"4z35ntbdhytzlhaviffrorrqxvspd6k6jf3pqj7gbday4c2hld5q====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\"},{\"name\":\"gmake\",\"version\":\"4.4.1\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"guile\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"rpzjfobv7qh3wevti34nlbd2emtw5mnyszqmkyiq5jiq33xm7qzq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\"},{\"name\":\"gnuconfig\",\"version\":\"2024-07-27\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"aar2tabf35425kgzryprq775xycug7xlbt4rkwvm4aj76dhlychq====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"z2hnkln52bnc5tbjkhtjv7n2av52a5eh\"},{\"name\":\"libiconv\",\"version\":\"1.17\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"libs\":[\"shared\",\"static\"],\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ujsqmcknrabka5mhwwpbaf5rwxgopwoyxkskuwyqlcbynowgdvfa====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"z2hnkln52bnc5tbjkhtjv7n2av52a5eh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"bqcb2qmnv3vsz5u7b3whbrortoieu6bx\"},{\"name\":\"m4\",\"version\":\"1.4.19\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"patches\":[\"9dc5fbd0d5cb1037ab1e6d0ecc74a30df218d0a94bdd5a02759a97f62daca573\",\"bfdffa7c2eb01021d5849b36972c069693654ad826c1a20b53534009a4ec7a89\"],\"sigsegv\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"patches\":[\"bfdffa7c2eb01021d5849b36972c069693654ad826c1a20b53534009a4ec7a89\",\"9dc5fbd0d5cb1037ab1e6d0ecc74a30df218d0a94bdd5a02759a97f62daca573\"],\"package_hash\":\"npb7a53yz7wqx4nvnasxwgzxaoiks6sdjz2eugrgkjxs4ml24xea====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"diffutils\",\"hash\":\"kntg5epaheq5s2cpiqskcfu3do6nikge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"z2hnkln52bnc5tbjkhtjv7n2av52a5eh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libsigsegv\",\"hash\":\"fkqxgj3yfnk4vl3iczancsoq5yc2bgye\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"jjtr2n3inumkcqn26fnznvt3ek5ddknd\"},{\"name\":\"libsigsegv\",\"version\":\"2.14\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"3s645t5rbjrziao47mhgob5xgymot6tf4kalagflbal2jdamdo2a====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"z2hnkln52bnc5tbjkhtjv7n2av52a5eh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"fkqxgj3yfnk4vl3iczancsoq5yc2bgye\"},{\"name\":\"cmake\",\"version\":\"3.31.2\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"build_type\":\"Release\",\"doc\":false,\"ncurses\":true,\"ownlibs\":true,\"qtgui\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"7vk6yhuq2fklcj5kk7bhreqojudugggezq7vntmcsc32cw2avmya====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"curl\",\"hash\":\"ntpxjnhrnsjzadlmrkier3pqoxqpng3t\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"ncurses\",\"hash\":\"d7rkispaw64fota6iabiom2hbawedpgj\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"hvwclyptyu46ird3xmb6gx4ii33rqd53\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"ltrb7aes3hwdnz27nzndzsmbv2vnw6wy\"},{\"name\":\"curl\",\"version\":\"8.10.1\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"gssapi\":false,\"ldap\":false,\"libidn2\":false,\"librtmp\":false,\"libs\":[\"shared\",\"static\"],\"libssh\":false,\"libssh2\":false,\"nghttp2\":true,\"tls\":[\"openssl\"],\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ccka5yawqcn2rjbqn3bkhkdjoajlngm5uab7jbyrsl5yqn42ofza====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"z2hnkln52bnc5tbjkhtjv7n2av52a5eh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"nghttp2\",\"hash\":\"cznkg4nmmy62b3zdogggospnuuy3g5pc\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"openssl\",\"hash\":\"h4j2u76c7rhqompivzi4whe4hjw3cze7\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"pkgconf\",\"hash\":\"mxoabqjj7kluh3md2xo4qyof524orfwl\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}},{\"name\":\"zlib-ng\",\"hash\":\"hvwclyptyu46ird3xmb6gx4ii33rqd53\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"ntpxjnhrnsjzadlmrkier3pqoxqpng3t\"},{\"name\":\"nghttp2\",\"version\":\"1.64.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"nkykfkj4rxzmysrmoh5mhxrl5ysaemlqh652m3he7pkbgvjhjgba====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"diffutils\",\"hash\":\"kntg5epaheq5s2cpiqskcfu3do6nikge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"z2hnkln52bnc5tbjkhtjv7n2av52a5eh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"pkgconf\",\"hash\":\"mxoabqjj7kluh3md2xo4qyof524orfwl\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"cznkg4nmmy62b3zdogggospnuuy3g5pc\"},{\"name\":\"pkgconf\",\"version\":\"2.2.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"gl6tpyarjlclzsal6wa4dtc7cdzprq36nbibalai4a6wgzblrseq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"z2hnkln52bnc5tbjkhtjv7n2av52a5eh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"mxoabqjj7kluh3md2xo4qyof524orfwl\"},{\"name\":\"openssl\",\"version\":\"3.4.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"certs\":\"mozilla\",\"docs\":false,\"shared\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"5y33vxwjtlrlsyedasvmhukjkk5yfwcri27oceh36iw73xehumfa====\",\"dependencies\":[{\"name\":\"ca-certificates-mozilla\",\"hash\":\"qeszxs4rv5nw7zezjc3524ztgkoz33ig\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"perl\",\"hash\":\"4tn6es2ac3gd2dsnvskwle4etlpk6qv3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"hvwclyptyu46ird3xmb6gx4ii33rqd53\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"h4j2u76c7rhqompivzi4whe4hjw3cze7\"},{\"name\":\"ca-certificates-mozilla\",\"version\":\"2023-05-30\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"63npvwqwo2x7i6emvnklh4mhcn45gx2qzveorybh5h2inwr55sea====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"qeszxs4rv5nw7zezjc3524ztgkoz33ig\"},{\"name\":\"perl\",\"version\":\"5.40.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cpanm\":true,\"opcode\":true,\"open\":true,\"shared\":true,\"threads\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"f233ue76vwtkle2r4jwsfe5x27ujx6ea4vdyp6baonfmkgqf5vpa====\",\"dependencies\":[{\"name\":\"berkeley-db\",\"hash\":\"tmpewsx4vcxbciz63y3sjwqld577hzom\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"bzip2\",\"hash\":\"gefii4i45qgge6oeyibc4a6neierycc5\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"gdbm\",\"hash\":\"3tpau5775md4363pqnphjbr2ufir6rno\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"hvwclyptyu46ird3xmb6gx4ii33rqd53\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"4tn6es2ac3gd2dsnvskwle4etlpk6qv3\"},{\"name\":\"berkeley-db\",\"version\":\"18.1.40\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cxx\":true,\"docs\":false,\"patches\":[\"26090f418891757af46ac3b89a9f43d6eb5989f7a3dce3d1cfc99fba547203b3\",\"b231fcc4d5cff05e5c3a4814f6a5af0e9a966428dc2176540d2c05aff41de522\"],\"stl\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"patches\":[\"b231fcc4d5cff05e5c3a4814f6a5af0e9a966428dc2176540d2c05aff41de522\",\"26090f418891757af46ac3b89a9f43d6eb5989f7a3dce3d1cfc99fba547203b3\"],\"package_hash\":\"h57ydfn33zevvzctzzioiiwjwe362izbbwncb6a26dfeno4y7tda====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"z2hnkln52bnc5tbjkhtjv7n2av52a5eh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"tmpewsx4vcxbciz63y3sjwqld577hzom\"},{\"name\":\"bzip2\",\"version\":\"1.0.8\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"debug\":false,\"pic\":false,\"shared\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"jb7yvhkifmvfl3ykmdulsjxkkulker6gqb5tadollyjt2ijg3zsa====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"diffutils\",\"hash\":\"kntg5epaheq5s2cpiqskcfu3do6nikge\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"gefii4i45qgge6oeyibc4a6neierycc5\"},{\"name\":\"gdbm\",\"version\":\"1.23\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"liepxl6phlcxbgfmibxafhewtihlgaa4x3hko37ckqlafhxkrgdq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"z2hnkln52bnc5tbjkhtjv7n2av52a5eh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"readline\",\"hash\":\"6z2sfif7stzpfvb54eoqiiki5edutguc\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"3tpau5775md4363pqnphjbr2ufir6rno\"},{\"name\":\"readline\",\"version\":\"8.2\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"patches\":[\"bbf97f1ec40a929edab5aa81998c1e2ef435436c597754916e6a5868f273aff7\"],\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"patches\":[\"bbf97f1ec40a929edab5aa81998c1e2ef435436c597754916e6a5868f273aff7\"],\"package_hash\":\"oww6dmr7xqgg6j7iiluonxbcl4irqnnrip4vfkjdwujncwnuhwuq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"z2hnkln52bnc5tbjkhtjv7n2av52a5eh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"ncurses\",\"hash\":\"d7rkispaw64fota6iabiom2hbawedpgj\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"6z2sfif7stzpfvb54eoqiiki5edutguc\"},{\"name\":\"ncurses\",\"version\":\"6.5\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"abi\":\"none\",\"build_system\":\"autotools\",\"patches\":[\"7a351bc4953a4ab70dabdbea31c8db0c03d40ce505335f3b6687180dde24c535\"],\"symlinks\":false,\"termlib\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"patches\":[\"7a351bc4953a4ab70dabdbea31c8db0c03d40ce505335f3b6687180dde24c535\"],\"package_hash\":\"cfh76rniab2gnv4jqr77yzz5za4ucfmva2upihvxukn52dybhsvq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"z2hnkln52bnc5tbjkhtjv7n2av52a5eh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"pkgconf\",\"hash\":\"mxoabqjj7kluh3md2xo4qyof524orfwl\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"d7rkispaw64fota6iabiom2hbawedpgj\"},{\"name\":\"zlib-ng\",\"version\":\"2.2.1\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"compat\":true,\"new_strategies\":true,\"opt\":true,\"pic\":true,\"shared\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"mdxo2xewbdavckgsqlcjywyfssdchgwbzonui22gxww7hqtozurq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"z2hnkln52bnc5tbjkhtjv7n2av52a5eh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"hvwclyptyu46ird3xmb6gx4ii33rqd53\"},{\"name\":\"python\",\"version\":\"3.13.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"bz2\":true,\"ctypes\":true,\"dbm\":true,\"debug\":false,\"libxml2\":true,\"lzma\":true,\"optimizations\":false,\"pic\":true,\"pyexpat\":true,\"pythoncmd\":true,\"readline\":true,\"shared\":true,\"sqlite3\":true,\"ssl\":true,\"tkinter\":false,\"uuid\":true,\"zlib\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"n6v6rt6deysntdggu2gi4zkhqriyba6bgaghxyhluou4ssqf7xfq====\",\"dependencies\":[{\"name\":\"bzip2\",\"hash\":\"gefii4i45qgge6oeyibc4a6neierycc5\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"expat\",\"hash\":\"hjwfi4iuk7mecmsuh75z74wycjlw7lzi\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"gdbm\",\"hash\":\"3tpau5775md4363pqnphjbr2ufir6rno\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"gettext\",\"hash\":\"hheyf7ak3sjcfohvfgegvdded4wppbvr\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libffi\",\"hash\":\"qegr7o5zly6cqypzzsm7s6hxcwqsgtqj\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"ncurses\",\"hash\":\"d7rkispaw64fota6iabiom2hbawedpgj\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"openssl\",\"hash\":\"h4j2u76c7rhqompivzi4whe4hjw3cze7\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"pkgconf\",\"hash\":\"mxoabqjj7kluh3md2xo4qyof524orfwl\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}},{\"name\":\"readline\",\"hash\":\"6z2sfif7stzpfvb54eoqiiki5edutguc\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"sqlite\",\"hash\":\"kz4n2vtbxcj72s2teh2g6k6eefy6zxpe\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"util-linux-uuid\",\"hash\":\"rmy7tbekh4lfetlh55swl74gqwlvrm3y\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"uuid\"]}},{\"name\":\"xz\",\"hash\":\"vqzih6qodsu52uopsr42t7h5esj4jd2v\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"hvwclyptyu46ird3xmb6gx4ii33rqd53\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"gygxuzpdf33jg2ya6imlbn4bd5zghbcd\"},{\"name\":\"expat\",\"version\":\"2.6.4\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"libbsd\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ei6qyjakl7sgtodwxxbg5brgkp23robfximtpbedkrnpyyyvr3ya====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"z2hnkln52bnc5tbjkhtjv7n2av52a5eh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libbsd\",\"hash\":\"ukmaajw26pw7xfaklkrklqha4rrrsgra\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"hjwfi4iuk7mecmsuh75z74wycjlw7lzi\"},{\"name\":\"libbsd\",\"version\":\"0.12.2\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"debyg3en7sgggswkdhcyd6lbp7arawzmyujthyyuaiad5jqd5msa====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"z2hnkln52bnc5tbjkhtjv7n2av52a5eh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libmd\",\"hash\":\"il5ykbrdnhlzimhloyrwyymx3aicprt3\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"ukmaajw26pw7xfaklkrklqha4rrrsgra\"},{\"name\":\"libmd\",\"version\":\"1.0.4\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"zs2e7fqr4dzthpj5fascqvfn7xcahf7dtc5bzdwfv6vqkzi7oncq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"z2hnkln52bnc5tbjkhtjv7n2av52a5eh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"il5ykbrdnhlzimhloyrwyymx3aicprt3\"},{\"name\":\"gettext\",\"version\":\"0.22.5\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"bzip2\":true,\"curses\":true,\"git\":true,\"libunistring\":false,\"libxml2\":true,\"pic\":true,\"shared\":true,\"tar\":true,\"xz\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"d4zxhmw6rownaaokzcolsszrq2cmx44m7qmzopucymoyrhbdfgvq====\",\"dependencies\":[{\"name\":\"bzip2\",\"hash\":\"gefii4i45qgge6oeyibc4a6neierycc5\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"z2hnkln52bnc5tbjkhtjv7n2av52a5eh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libiconv\",\"hash\":\"bqcb2qmnv3vsz5u7b3whbrortoieu6bx\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"iconv\"]}},{\"name\":\"libxml2\",\"hash\":\"6r76q5qnwa6ydovyzag7dghcfxsm6rlk\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"ncurses\",\"hash\":\"d7rkispaw64fota6iabiom2hbawedpgj\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"tar\",\"hash\":\"peh5t7ttvsvzqas4gor63twpxwj7ei6i\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}},{\"name\":\"xz\",\"hash\":\"vqzih6qodsu52uopsr42t7h5esj4jd2v\",\"parameters\":{\"deptypes\":[\"build\",\"link\",\"run\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"hheyf7ak3sjcfohvfgegvdded4wppbvr\"},{\"name\":\"libxml2\",\"version\":\"2.13.4\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"pic\":true,\"python\":false,\"shared\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"j6yob2wgvc2wjzvbs6xdvgyfa3zp3wrm3uxncxzxqfzw6xazzoba====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"z2hnkln52bnc5tbjkhtjv7n2av52a5eh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libiconv\",\"hash\":\"bqcb2qmnv3vsz5u7b3whbrortoieu6bx\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"iconv\"]}},{\"name\":\"pkgconf\",\"hash\":\"mxoabqjj7kluh3md2xo4qyof524orfwl\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}},{\"name\":\"xz\",\"hash\":\"vqzih6qodsu52uopsr42t7h5esj4jd2v\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"hvwclyptyu46ird3xmb6gx4ii33rqd53\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"6r76q5qnwa6ydovyzag7dghcfxsm6rlk\"},{\"name\":\"xz\",\"version\":\"5.4.6\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"libs\":[\"shared\",\"static\"],\"pic\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"zt5vu2vph2v2qjwgdbe7btgcz7axpyalorcsqiuxhrg5grwgrrvq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"z2hnkln52bnc5tbjkhtjv7n2av52a5eh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"vqzih6qodsu52uopsr42t7h5esj4jd2v\"},{\"name\":\"tar\",\"version\":\"1.35\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"zip\":\"pigz\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"v6a6jvks2setklucxyk622uauxzqlgmsdkrvdijbi3m5jwftmzla====\",\"dependencies\":[{\"name\":\"bzip2\",\"hash\":\"gefii4i45qgge6oeyibc4a6neierycc5\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"z2hnkln52bnc5tbjkhtjv7n2av52a5eh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libiconv\",\"hash\":\"bqcb2qmnv3vsz5u7b3whbrortoieu6bx\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"iconv\"]}},{\"name\":\"pigz\",\"hash\":\"h5quuhwtol6qrxznml2ffjex2nfndg3e\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}},{\"name\":\"xz\",\"hash\":\"vqzih6qodsu52uopsr42t7h5esj4jd2v\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}},{\"name\":\"zstd\",\"hash\":\"u5inbr2rrtinstce7l5krqqpnsal4vxo\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"peh5t7ttvsvzqas4gor63twpxwj7ei6i\"},{\"name\":\"pigz\",\"version\":\"2.8\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"makefile\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"4w67lflje4giekjg4ie2vpyuiunjcumo6geofykvon3hodllp42q====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"hvwclyptyu46ird3xmb6gx4ii33rqd53\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"h5quuhwtol6qrxznml2ffjex2nfndg3e\"},{\"name\":\"zstd\",\"version\":\"1.5.6\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"makefile\",\"compression\":[\"none\"],\"libs\":[\"shared\",\"static\"],\"programs\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"uvmrov4c6unft6o4yd3jk3uqvweua3uhwdli4sw7h5wvklaf5t3q====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"u5inbr2rrtinstce7l5krqqpnsal4vxo\"},{\"name\":\"libffi\",\"version\":\"3.4.6\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"umhsnvoj5ooa3glffnkk2hp3txmrsjvqbpfq2hbk4mhcvhza7gaa====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"z2hnkln52bnc5tbjkhtjv7n2av52a5eh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"qegr7o5zly6cqypzzsm7s6hxcwqsgtqj\"},{\"name\":\"sqlite\",\"version\":\"3.46.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"column_metadata\":true,\"dynamic_extensions\":true,\"fts\":true,\"functions\":false,\"rtree\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"wm3irnrjil5n275nw2m4x3mpvyg35h7isbmsnuae6vtxbamsrv4q====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"z2hnkln52bnc5tbjkhtjv7n2av52a5eh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"readline\",\"hash\":\"6z2sfif7stzpfvb54eoqiiki5edutguc\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"hvwclyptyu46ird3xmb6gx4ii33rqd53\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"kz4n2vtbxcj72s2teh2g6k6eefy6zxpe\"},{\"name\":\"util-linux-uuid\",\"version\":\"2.40.2\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"eo6au7zhsz344imzoomhuskbl3cmrqq6ja6mcmrc3li3fnppqs6q====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"z2hnkln52bnc5tbjkhtjv7n2av52a5eh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"pkgconf\",\"hash\":\"mxoabqjj7kluh3md2xo4qyof524orfwl\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"rmy7tbekh4lfetlh55swl74gqwlvrm3y\"},{\"name\":\"python-venv\",\"version\":\"1.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"j3dgyzp5nei24fbpw22l3gedsk37asrdrjafbnaiwiux3lxasi3a====\",\"dependencies\":[{\"name\":\"python\",\"hash\":\"gygxuzpdf33jg2ya6imlbn4bd5zghbcd\",\"parameters\":{\"deptypes\":[\"build\",\"run\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"hf5bgjk6fsdycb4zovjap4t4g6tjfcvx\"},{\"name\":\"re2c\",\"version\":\"3.1\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel9\",\"target\":\"aarch64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ebw3m3xkgw2wijfijtzrxt4ldu4tz4haiz6juumq6wn4mjzsuxra====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rximc5jq3c544fhhnloem4mbccot26tv\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vojoispd6oa5kvdlyebgdgddrmhfpkol\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"23zkc4xomaptugrl5ueoh3tv3oyaqjnj\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"2mq2filwjgkrv6j6cxvispjqvtirsssh\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"gjaj5hopp3pqqbupult3vmvokhzzfhld\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"z2hnkln52bnc5tbjkhtjv7n2av52a5eh\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"python\",\"hash\":\"gygxuzpdf33jg2ya6imlbn4bd5zghbcd\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"eavspn7qgilrfiby4v6in34pmjg5le6b\"}]}}\n" }, { "path": "lib/spack/spack/bootstrap/prototypes/clingo-linux-ppc64le.json", "content": "{\"spec\":{\"_meta\":{\"version\":5},\"nodes\":[{\"name\":\"clingo-bootstrap\",\"version\":\"spack\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"cmake\",\"build_type\":\"Release\",\"docs\":false,\"generator\":\"make\",\"ipo\":false,\"optimized\":false,\"python\":true,\"static_libstdcpp\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ozkztarkrp3oet7x2oapc7ehdfyvweap45zb3g44mj6qpblv4l3a====\",\"dependencies\":[{\"name\":\"bison\",\"hash\":\"lgghcjqpoodrawadw7vibeiul7wrnqog\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"cmake\",\"hash\":\"p4cntzqqcfg5a6ymiyjpk6ykqcwwirym\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"python\",\"hash\":\"lk4r47znptvkiszmntnetz6kgen7tgm3\",\"parameters\":{\"deptypes\":[\"build\",\"link\",\"run\"],\"virtuals\":[]}},{\"name\":\"python-venv\",\"hash\":\"dlduozijjwp5o7vnrdghszehqh5j4rim\",\"parameters\":{\"deptypes\":[\"build\",\"run\"],\"virtuals\":[]}},{\"name\":\"re2c\",\"hash\":\"xfl6wrih72mane3eeobwpkyjwtfn2y76\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"7bxhwjxs6euecw5nkz4pi2hoi6lqz6ee\"},{\"name\":\"bison\",\"version\":\"3.8.2\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"color\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"d4j62fwvuxqbiez32ltjnhu47ac425wjebsy6fhoptv6saxazcxq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"diffutils\",\"hash\":\"wly2a7jdclwp6kcz3x3nzhyuqqrhgbjt\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"klycihpzvu77okocxw42le5dbhwduu2z\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"m4\",\"hash\":\"twjk5wpmcqes4w4biqdwwrillznv5qaq\",\"parameters\":{\"deptypes\":[\"build\",\"run\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"lgghcjqpoodrawadw7vibeiul7wrnqog\"},{\"name\":\"compiler-wrapper\",\"version\":\"1.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":{\"name\":\"power9le\",\"vendor\":\"IBM\",\"features\":[],\"generation\":9,\"parents\":[\"power8le\"],\"cpupart\":\"\"}},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"gogqnfdkxjvnjgj3lndnoncjtdc7ydoc7klkjstywag4oqrvod7a====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\"},{\"name\":\"diffutils\",\"version\":\"3.10\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"kbmzdy7mgklc24qx55cvx7kq7hceby2yav4fnf64gfdo7epdghwa====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"klycihpzvu77okocxw42le5dbhwduu2z\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libiconv\",\"hash\":\"jjemqnzesqbyw5tdlrk3nnuuajptekss\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"iconv\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"wly2a7jdclwp6kcz3x3nzhyuqqrhgbjt\"},{\"name\":\"gcc\",\"version\":\"8.5.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":{\"name\":\"power9le\",\"vendor\":\"IBM\",\"features\":[],\"generation\":9,\"parents\":[\"power8le\"],\"cpupart\":\"\"}},\"namespace\":\"builtin\",\"parameters\":{\"binutils\":false,\"bootstrap\":true,\"build_system\":\"autotools\",\"build_type\":\"RelWithDebInfo\",\"graphite\":false,\"languages\":[\"c\",\"c++\",\"fortran\"],\"nvptx\":false,\"patches\":[\"98a9c96f66ff0264a49bd5e76fd2ba177ceca7c7236f486058a8469c2bcd1b76\",\"d4919d68d5460049d370e79ff78bbc320cfe66a7fdf6dfc92cf7e133152b2d56\"],\"piclibs\":false,\"strip\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"external\":{\"path\":\"/usr\",\"module\":null,\"extra_attributes\":{\"compilers\":{\"c\":\"/usr/bin/gcc\",\"cxx\":\"/usr/bin/g++\"}}},\"patches\":[\"98a9c96f66ff0264a49bd5e76fd2ba177ceca7c7236f486058a8469c2bcd1b76\",\"d4919d68d5460049d370e79ff78bbc320cfe66a7fdf6dfc92cf7e133152b2d56\"],\"package_hash\":\"fnrebjvblgu5vg2gnwreotucmf67pkyu6dzgo5afxngtphp66biq====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\"},{\"name\":\"gcc-runtime\",\"version\":\"8.5.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"aud4d72goxupc5p3p6mdkwgtshpygn7uuj2ewx3zm6wudcgw4fzq====\",\"dependencies\":[{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\"},{\"name\":\"glibc\",\"version\":\"2.28\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"external\":{\"path\":\"/usr\",\"module\":null,\"extra_attributes\":{}},\"package_hash\":\"riktbfk2yybad7tgbvdkntk5c5msjcm5pk3x7naszgbvfm57h4rq====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\"},{\"name\":\"gmake\",\"version\":\"4.4.1\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"guile\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"rpzjfobv7qh3wevti34nlbd2emtw5mnyszqmkyiq5jiq33xm7qzq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\"},{\"name\":\"gnuconfig\",\"version\":\"2024-07-27\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"aar2tabf35425kgzryprq775xycug7xlbt4rkwvm4aj76dhlychq====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"klycihpzvu77okocxw42le5dbhwduu2z\"},{\"name\":\"libiconv\",\"version\":\"1.17\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"libs\":[\"shared\",\"static\"],\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ujsqmcknrabka5mhwwpbaf5rwxgopwoyxkskuwyqlcbynowgdvfa====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"klycihpzvu77okocxw42le5dbhwduu2z\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"jjemqnzesqbyw5tdlrk3nnuuajptekss\"},{\"name\":\"m4\",\"version\":\"1.4.19\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"patches\":[\"9dc5fbd0d5cb1037ab1e6d0ecc74a30df218d0a94bdd5a02759a97f62daca573\",\"bfdffa7c2eb01021d5849b36972c069693654ad826c1a20b53534009a4ec7a89\"],\"sigsegv\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"patches\":[\"bfdffa7c2eb01021d5849b36972c069693654ad826c1a20b53534009a4ec7a89\",\"9dc5fbd0d5cb1037ab1e6d0ecc74a30df218d0a94bdd5a02759a97f62daca573\"],\"package_hash\":\"npb7a53yz7wqx4nvnasxwgzxaoiks6sdjz2eugrgkjxs4ml24xea====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"diffutils\",\"hash\":\"wly2a7jdclwp6kcz3x3nzhyuqqrhgbjt\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"klycihpzvu77okocxw42le5dbhwduu2z\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libsigsegv\",\"hash\":\"mgnur44dzhyu7j6gqkqqfaa6odgp4ox2\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"twjk5wpmcqes4w4biqdwwrillznv5qaq\"},{\"name\":\"libsigsegv\",\"version\":\"2.14\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"3s645t5rbjrziao47mhgob5xgymot6tf4kalagflbal2jdamdo2a====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"klycihpzvu77okocxw42le5dbhwduu2z\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"mgnur44dzhyu7j6gqkqqfaa6odgp4ox2\"},{\"name\":\"cmake\",\"version\":\"3.31.2\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"build_type\":\"Release\",\"doc\":false,\"ncurses\":true,\"ownlibs\":true,\"qtgui\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"7vk6yhuq2fklcj5kk7bhreqojudugggezq7vntmcsc32cw2avmya====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"curl\",\"hash\":\"7cw4rfec7mv444ok2avp3qpq62upmims\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"ncurses\",\"hash\":\"s3oz4dsdxhvwkoekfjly6x3q4netali4\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"2q57rihnetmc5erpl6vw3nusqw7ycjqs\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"p4cntzqqcfg5a6ymiyjpk6ykqcwwirym\"},{\"name\":\"curl\",\"version\":\"8.10.1\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"gssapi\":false,\"ldap\":false,\"libidn2\":false,\"librtmp\":false,\"libs\":[\"shared\",\"static\"],\"libssh\":false,\"libssh2\":false,\"nghttp2\":true,\"tls\":[\"openssl\"],\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ccka5yawqcn2rjbqn3bkhkdjoajlngm5uab7jbyrsl5yqn42ofza====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"klycihpzvu77okocxw42le5dbhwduu2z\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"nghttp2\",\"hash\":\"bkgwuueh4jnhdcu6gvtyxldelsp3nrf2\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"openssl\",\"hash\":\"j7xymvpa4nhwhjxb2hhahjcyjvvezyho\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"pkgconf\",\"hash\":\"dylr2p25oj5nqbtq3zhtfkktbocbe4jm\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}},{\"name\":\"zlib-ng\",\"hash\":\"2q57rihnetmc5erpl6vw3nusqw7ycjqs\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"7cw4rfec7mv444ok2avp3qpq62upmims\"},{\"name\":\"nghttp2\",\"version\":\"1.64.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"nkykfkj4rxzmysrmoh5mhxrl5ysaemlqh652m3he7pkbgvjhjgba====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"diffutils\",\"hash\":\"wly2a7jdclwp6kcz3x3nzhyuqqrhgbjt\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"klycihpzvu77okocxw42le5dbhwduu2z\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"pkgconf\",\"hash\":\"dylr2p25oj5nqbtq3zhtfkktbocbe4jm\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"bkgwuueh4jnhdcu6gvtyxldelsp3nrf2\"},{\"name\":\"pkgconf\",\"version\":\"2.2.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"gl6tpyarjlclzsal6wa4dtc7cdzprq36nbibalai4a6wgzblrseq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"klycihpzvu77okocxw42le5dbhwduu2z\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"dylr2p25oj5nqbtq3zhtfkktbocbe4jm\"},{\"name\":\"openssl\",\"version\":\"3.4.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"certs\":\"mozilla\",\"docs\":false,\"shared\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"5y33vxwjtlrlsyedasvmhukjkk5yfwcri27oceh36iw73xehumfa====\",\"dependencies\":[{\"name\":\"ca-certificates-mozilla\",\"hash\":\"6aunhqikyb5jmxkapuhzc43lapta4gaa\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"perl\",\"hash\":\"ly3b5hxhkeavnar35daa3xolmbb7guv2\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"2q57rihnetmc5erpl6vw3nusqw7ycjqs\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"j7xymvpa4nhwhjxb2hhahjcyjvvezyho\"},{\"name\":\"ca-certificates-mozilla\",\"version\":\"2023-05-30\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"63npvwqwo2x7i6emvnklh4mhcn45gx2qzveorybh5h2inwr55sea====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"6aunhqikyb5jmxkapuhzc43lapta4gaa\"},{\"name\":\"perl\",\"version\":\"5.40.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cpanm\":true,\"opcode\":true,\"open\":true,\"shared\":true,\"threads\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"f233ue76vwtkle2r4jwsfe5x27ujx6ea4vdyp6baonfmkgqf5vpa====\",\"dependencies\":[{\"name\":\"berkeley-db\",\"hash\":\"n3eeghdelxrza3mezn7guy6qsqhjcon4\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"bzip2\",\"hash\":\"5myohalomy2tb2s3oxd5zninc6u7v4pr\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"gdbm\",\"hash\":\"bhsabxvim2eymbj3w3chcjwv4boripys\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"2q57rihnetmc5erpl6vw3nusqw7ycjqs\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"ly3b5hxhkeavnar35daa3xolmbb7guv2\"},{\"name\":\"berkeley-db\",\"version\":\"18.1.40\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cxx\":true,\"docs\":false,\"patches\":[\"26090f418891757af46ac3b89a9f43d6eb5989f7a3dce3d1cfc99fba547203b3\",\"b231fcc4d5cff05e5c3a4814f6a5af0e9a966428dc2176540d2c05aff41de522\"],\"stl\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"patches\":[\"b231fcc4d5cff05e5c3a4814f6a5af0e9a966428dc2176540d2c05aff41de522\",\"26090f418891757af46ac3b89a9f43d6eb5989f7a3dce3d1cfc99fba547203b3\"],\"package_hash\":\"h57ydfn33zevvzctzzioiiwjwe362izbbwncb6a26dfeno4y7tda====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"klycihpzvu77okocxw42le5dbhwduu2z\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"n3eeghdelxrza3mezn7guy6qsqhjcon4\"},{\"name\":\"bzip2\",\"version\":\"1.0.8\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"debug\":false,\"pic\":false,\"shared\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"jb7yvhkifmvfl3ykmdulsjxkkulker6gqb5tadollyjt2ijg3zsa====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"diffutils\",\"hash\":\"wly2a7jdclwp6kcz3x3nzhyuqqrhgbjt\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"5myohalomy2tb2s3oxd5zninc6u7v4pr\"},{\"name\":\"gdbm\",\"version\":\"1.23\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"liepxl6phlcxbgfmibxafhewtihlgaa4x3hko37ckqlafhxkrgdq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"klycihpzvu77okocxw42le5dbhwduu2z\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"readline\",\"hash\":\"jxqfcixv66kiwdfxcnbxadbrxmnhpiqf\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"bhsabxvim2eymbj3w3chcjwv4boripys\"},{\"name\":\"readline\",\"version\":\"8.2\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"patches\":[\"bbf97f1ec40a929edab5aa81998c1e2ef435436c597754916e6a5868f273aff7\"],\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"patches\":[\"bbf97f1ec40a929edab5aa81998c1e2ef435436c597754916e6a5868f273aff7\"],\"package_hash\":\"oww6dmr7xqgg6j7iiluonxbcl4irqnnrip4vfkjdwujncwnuhwuq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"klycihpzvu77okocxw42le5dbhwduu2z\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"ncurses\",\"hash\":\"s3oz4dsdxhvwkoekfjly6x3q4netali4\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"jxqfcixv66kiwdfxcnbxadbrxmnhpiqf\"},{\"name\":\"ncurses\",\"version\":\"6.5\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"abi\":\"none\",\"build_system\":\"autotools\",\"patches\":[\"7a351bc4953a4ab70dabdbea31c8db0c03d40ce505335f3b6687180dde24c535\"],\"symlinks\":false,\"termlib\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"patches\":[\"7a351bc4953a4ab70dabdbea31c8db0c03d40ce505335f3b6687180dde24c535\"],\"package_hash\":\"cfh76rniab2gnv4jqr77yzz5za4ucfmva2upihvxukn52dybhsvq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"klycihpzvu77okocxw42le5dbhwduu2z\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"pkgconf\",\"hash\":\"dylr2p25oj5nqbtq3zhtfkktbocbe4jm\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"s3oz4dsdxhvwkoekfjly6x3q4netali4\"},{\"name\":\"zlib-ng\",\"version\":\"2.2.1\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"compat\":true,\"new_strategies\":true,\"opt\":true,\"pic\":true,\"shared\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"mdxo2xewbdavckgsqlcjywyfssdchgwbzonui22gxww7hqtozurq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"klycihpzvu77okocxw42le5dbhwduu2z\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"2q57rihnetmc5erpl6vw3nusqw7ycjqs\"},{\"name\":\"python\",\"version\":\"3.13.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"bz2\":true,\"ctypes\":true,\"dbm\":true,\"debug\":false,\"libxml2\":true,\"lzma\":true,\"optimizations\":false,\"pic\":true,\"pyexpat\":true,\"pythoncmd\":true,\"readline\":true,\"shared\":true,\"sqlite3\":true,\"ssl\":true,\"tkinter\":false,\"uuid\":true,\"zlib\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"n6v6rt6deysntdggu2gi4zkhqriyba6bgaghxyhluou4ssqf7xfq====\",\"dependencies\":[{\"name\":\"bzip2\",\"hash\":\"5myohalomy2tb2s3oxd5zninc6u7v4pr\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"expat\",\"hash\":\"vvi4w4c2ibhcbc653rqnvf2cgkp6lhxm\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"gdbm\",\"hash\":\"bhsabxvim2eymbj3w3chcjwv4boripys\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"gettext\",\"hash\":\"7o3dt5k4qbnr632i3gyiaaexuc3utv4w\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libffi\",\"hash\":\"ibk3s5narlwxsakc4bsawr3npleftvjs\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"ncurses\",\"hash\":\"s3oz4dsdxhvwkoekfjly6x3q4netali4\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"openssl\",\"hash\":\"j7xymvpa4nhwhjxb2hhahjcyjvvezyho\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"pkgconf\",\"hash\":\"dylr2p25oj5nqbtq3zhtfkktbocbe4jm\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}},{\"name\":\"readline\",\"hash\":\"jxqfcixv66kiwdfxcnbxadbrxmnhpiqf\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"sqlite\",\"hash\":\"4q6cdje3u6oxg3eww63oxmoy2dlks3ml\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"util-linux-uuid\",\"hash\":\"dzc4fsrtt5bt5rn3hrq6mguskici66i7\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"uuid\"]}},{\"name\":\"xz\",\"hash\":\"4bw3ito7ggkxzqxl6jryedkokkjdbgjv\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"2q57rihnetmc5erpl6vw3nusqw7ycjqs\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"lk4r47znptvkiszmntnetz6kgen7tgm3\"},{\"name\":\"expat\",\"version\":\"2.6.4\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"libbsd\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ei6qyjakl7sgtodwxxbg5brgkp23robfximtpbedkrnpyyyvr3ya====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"klycihpzvu77okocxw42le5dbhwduu2z\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libbsd\",\"hash\":\"swmg4rlaebhq37ufiskqf3hz5vq76ybj\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"vvi4w4c2ibhcbc653rqnvf2cgkp6lhxm\"},{\"name\":\"libbsd\",\"version\":\"0.12.2\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"debyg3en7sgggswkdhcyd6lbp7arawzmyujthyyuaiad5jqd5msa====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"klycihpzvu77okocxw42le5dbhwduu2z\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libmd\",\"hash\":\"lhb5nmg7qo67plifgcchtgqnjuxa633a\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"swmg4rlaebhq37ufiskqf3hz5vq76ybj\"},{\"name\":\"libmd\",\"version\":\"1.0.4\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"zs2e7fqr4dzthpj5fascqvfn7xcahf7dtc5bzdwfv6vqkzi7oncq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"klycihpzvu77okocxw42le5dbhwduu2z\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"lhb5nmg7qo67plifgcchtgqnjuxa633a\"},{\"name\":\"gettext\",\"version\":\"0.22.5\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"bzip2\":true,\"curses\":true,\"git\":true,\"libunistring\":false,\"libxml2\":true,\"pic\":true,\"shared\":true,\"tar\":true,\"xz\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"d4zxhmw6rownaaokzcolsszrq2cmx44m7qmzopucymoyrhbdfgvq====\",\"dependencies\":[{\"name\":\"bzip2\",\"hash\":\"5myohalomy2tb2s3oxd5zninc6u7v4pr\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"klycihpzvu77okocxw42le5dbhwduu2z\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libiconv\",\"hash\":\"jjemqnzesqbyw5tdlrk3nnuuajptekss\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"iconv\"]}},{\"name\":\"libxml2\",\"hash\":\"2hjspbs3neipsef47zhcjtswkg4x6wzo\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"ncurses\",\"hash\":\"s3oz4dsdxhvwkoekfjly6x3q4netali4\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"tar\",\"hash\":\"p52zhlsdvvqcwgswuev2qkv4lhfk3zpr\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}},{\"name\":\"xz\",\"hash\":\"4bw3ito7ggkxzqxl6jryedkokkjdbgjv\",\"parameters\":{\"deptypes\":[\"build\",\"link\",\"run\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"7o3dt5k4qbnr632i3gyiaaexuc3utv4w\"},{\"name\":\"libxml2\",\"version\":\"2.13.4\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"pic\":true,\"python\":false,\"shared\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"j6yob2wgvc2wjzvbs6xdvgyfa3zp3wrm3uxncxzxqfzw6xazzoba====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"klycihpzvu77okocxw42le5dbhwduu2z\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libiconv\",\"hash\":\"jjemqnzesqbyw5tdlrk3nnuuajptekss\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"iconv\"]}},{\"name\":\"pkgconf\",\"hash\":\"dylr2p25oj5nqbtq3zhtfkktbocbe4jm\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}},{\"name\":\"xz\",\"hash\":\"4bw3ito7ggkxzqxl6jryedkokkjdbgjv\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"2q57rihnetmc5erpl6vw3nusqw7ycjqs\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"2hjspbs3neipsef47zhcjtswkg4x6wzo\"},{\"name\":\"xz\",\"version\":\"5.4.6\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"libs\":[\"shared\",\"static\"],\"pic\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"zt5vu2vph2v2qjwgdbe7btgcz7axpyalorcsqiuxhrg5grwgrrvq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"klycihpzvu77okocxw42le5dbhwduu2z\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"4bw3ito7ggkxzqxl6jryedkokkjdbgjv\"},{\"name\":\"tar\",\"version\":\"1.35\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"zip\":\"pigz\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"v6a6jvks2setklucxyk622uauxzqlgmsdkrvdijbi3m5jwftmzla====\",\"dependencies\":[{\"name\":\"bzip2\",\"hash\":\"5myohalomy2tb2s3oxd5zninc6u7v4pr\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"klycihpzvu77okocxw42le5dbhwduu2z\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libiconv\",\"hash\":\"jjemqnzesqbyw5tdlrk3nnuuajptekss\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"iconv\"]}},{\"name\":\"pigz\",\"hash\":\"7otxklss5g77i5xarpyasp4thet2fqis\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}},{\"name\":\"xz\",\"hash\":\"4bw3ito7ggkxzqxl6jryedkokkjdbgjv\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}},{\"name\":\"zstd\",\"hash\":\"dkk3fhqzamznskkzgijs3dn5p4yqosv3\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"p52zhlsdvvqcwgswuev2qkv4lhfk3zpr\"},{\"name\":\"pigz\",\"version\":\"2.8\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"makefile\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"4w67lflje4giekjg4ie2vpyuiunjcumo6geofykvon3hodllp42q====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"2q57rihnetmc5erpl6vw3nusqw7ycjqs\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"7otxklss5g77i5xarpyasp4thet2fqis\"},{\"name\":\"zstd\",\"version\":\"1.5.6\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"makefile\",\"compression\":[\"none\"],\"libs\":[\"shared\",\"static\"],\"programs\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"uvmrov4c6unft6o4yd3jk3uqvweua3uhwdli4sw7h5wvklaf5t3q====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"dkk3fhqzamznskkzgijs3dn5p4yqosv3\"},{\"name\":\"libffi\",\"version\":\"3.4.6\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"umhsnvoj5ooa3glffnkk2hp3txmrsjvqbpfq2hbk4mhcvhza7gaa====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"klycihpzvu77okocxw42le5dbhwduu2z\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"ibk3s5narlwxsakc4bsawr3npleftvjs\"},{\"name\":\"sqlite\",\"version\":\"3.46.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"column_metadata\":true,\"dynamic_extensions\":true,\"fts\":true,\"functions\":false,\"rtree\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"wm3irnrjil5n275nw2m4x3mpvyg35h7isbmsnuae6vtxbamsrv4q====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"klycihpzvu77okocxw42le5dbhwduu2z\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"readline\",\"hash\":\"jxqfcixv66kiwdfxcnbxadbrxmnhpiqf\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"2q57rihnetmc5erpl6vw3nusqw7ycjqs\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"4q6cdje3u6oxg3eww63oxmoy2dlks3ml\"},{\"name\":\"util-linux-uuid\",\"version\":\"2.40.2\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"eo6au7zhsz344imzoomhuskbl3cmrqq6ja6mcmrc3li3fnppqs6q====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"klycihpzvu77okocxw42le5dbhwduu2z\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"pkgconf\",\"hash\":\"dylr2p25oj5nqbtq3zhtfkktbocbe4jm\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"dzc4fsrtt5bt5rn3hrq6mguskici66i7\"},{\"name\":\"python-venv\",\"version\":\"1.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"j3dgyzp5nei24fbpw22l3gedsk37asrdrjafbnaiwiux3lxasi3a====\",\"dependencies\":[{\"name\":\"python\",\"hash\":\"lk4r47znptvkiszmntnetz6kgen7tgm3\",\"parameters\":{\"deptypes\":[\"build\",\"run\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"dlduozijjwp5o7vnrdghszehqh5j4rim\"},{\"name\":\"re2c\",\"version\":\"3.1\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"ppc64le\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ebw3m3xkgw2wijfijtzrxt4ldu4tz4haiz6juumq6wn4mjzsuxra====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"ktcmkdaifi35awtk4wu3logfsi4nvtai\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"ezexv4wrroazd3i26siktomcoagxii3l\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"gfkhbpchfu2fk7m5yz4dax52d7yt5etp\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"mhe3tozpzp7hwolo3dxeh3zzqh45rlac\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"vjbibbd23up7c3c4cxpgawbz63krxjpk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gnuconfig\",\"hash\":\"klycihpzvu77okocxw42le5dbhwduu2z\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"python\",\"hash\":\"lk4r47znptvkiszmntnetz6kgen7tgm3\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"xfl6wrih72mane3eeobwpkyjwtfn2y76\"}]}}\n" }, { "path": "lib/spack/spack/bootstrap/prototypes/clingo-linux-x86_64.json", "content": "{\"spec\":{\"_meta\":{\"version\":5},\"nodes\":[{\"name\":\"clingo-bootstrap\",\"version\":\"spack\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"cmake\",\"build_type\":\"Release\",\"docs\":false,\"generator\":\"make\",\"ipo\":false,\"optimized\":false,\"python\":true,\"static_libstdcpp\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ozkztarkrp3oet7x2oapc7ehdfyvweap45zb3g44mj6qpblv4l3a====\",\"dependencies\":[{\"name\":\"bison\",\"hash\":\"ipu4y2n34za3lzhgwsqxha3pag2v2dn7\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"cmake\",\"hash\":\"2i4zyafripteq6cssiyrmo67n6tmypfs\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"python\",\"hash\":\"7f76ydmj6f4epvepmak2y5qfllqow5db\",\"parameters\":{\"deptypes\":[\"build\",\"link\",\"run\"],\"virtuals\":[]}},{\"name\":\"python-venv\",\"hash\":\"mvybzkpm37r4xrt3eip5nn2padrhnlrm\",\"parameters\":{\"deptypes\":[\"build\",\"run\"],\"virtuals\":[]}},{\"name\":\"re2c\",\"hash\":\"ketxaszk5wezamuffgkdpie66tkd7rbl\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"5mxtdduhp3wsqlifimjzb53eswxqgd5b\"},{\"name\":\"bison\",\"version\":\"3.8.2\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"color\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"d4j62fwvuxqbiez32ltjnhu47ac425wjebsy6fhoptv6saxazcxq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"diffutils\",\"hash\":\"eso2orwqs33nyzewrf6ccckvkfoxdzn2\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"m4\",\"hash\":\"gb5idois57zldhovt7rx44bd2ou4yiwr\",\"parameters\":{\"deptypes\":[\"build\",\"run\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"ipu4y2n34za3lzhgwsqxha3pag2v2dn7\"},{\"name\":\"compiler-wrapper\",\"version\":\"1.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":{\"name\":\"skylake_avx512\",\"vendor\":\"GenuineIntel\",\"features\":[\"adx\",\"aes\",\"avx\",\"avx2\",\"avx512bw\",\"avx512cd\",\"avx512dq\",\"avx512f\",\"avx512vl\",\"bmi1\",\"bmi2\",\"clflushopt\",\"clwb\",\"f16c\",\"fma\",\"mmx\",\"movbe\",\"pclmulqdq\",\"popcnt\",\"rdrand\",\"rdseed\",\"sse\",\"sse2\",\"sse4_1\",\"sse4_2\",\"ssse3\",\"xsavec\",\"xsaveopt\"],\"generation\":0,\"parents\":[\"skylake\",\"x86_64_v4\"],\"cpupart\":\"\"}},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"gogqnfdkxjvnjgj3lndnoncjtdc7ydoc7klkjstywag4oqrvod7a====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\"},{\"name\":\"diffutils\",\"version\":\"3.10\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"kbmzdy7mgklc24qx55cvx7kq7hceby2yav4fnf64gfdo7epdghwa====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libiconv\",\"hash\":\"3mozqilguvrkepcixf5v5czrvz64sn7a\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"iconv\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"eso2orwqs33nyzewrf6ccckvkfoxdzn2\"},{\"name\":\"gcc\",\"version\":\"8.5.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":{\"name\":\"skylake_avx512\",\"vendor\":\"GenuineIntel\",\"features\":[\"adx\",\"aes\",\"avx\",\"avx2\",\"avx512bw\",\"avx512cd\",\"avx512dq\",\"avx512f\",\"avx512vl\",\"bmi1\",\"bmi2\",\"clflushopt\",\"clwb\",\"f16c\",\"fma\",\"mmx\",\"movbe\",\"pclmulqdq\",\"popcnt\",\"rdrand\",\"rdseed\",\"sse\",\"sse2\",\"sse4_1\",\"sse4_2\",\"ssse3\",\"xsavec\",\"xsaveopt\"],\"generation\":0,\"parents\":[\"skylake\",\"x86_64_v4\"],\"cpupart\":\"\"}},\"namespace\":\"builtin\",\"parameters\":{\"binutils\":false,\"bootstrap\":true,\"build_system\":\"autotools\",\"build_type\":\"RelWithDebInfo\",\"graphite\":false,\"languages\":[\"c\",\"c++\",\"fortran\"],\"nvptx\":false,\"patches\":[\"98a9c96f66ff0264a49bd5e76fd2ba177ceca7c7236f486058a8469c2bcd1b76\",\"d4919d68d5460049d370e79ff78bbc320cfe66a7fdf6dfc92cf7e133152b2d56\"],\"piclibs\":false,\"strip\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"external\":{\"path\":\"/usr\",\"module\":null,\"extra_attributes\":{\"compilers\":{\"c\":\"/usr/bin/gcc\",\"cxx\":\"/usr/bin/g++\"}}},\"patches\":[\"98a9c96f66ff0264a49bd5e76fd2ba177ceca7c7236f486058a8469c2bcd1b76\",\"d4919d68d5460049d370e79ff78bbc320cfe66a7fdf6dfc92cf7e133152b2d56\"],\"package_hash\":\"fnrebjvblgu5vg2gnwreotucmf67pkyu6dzgo5afxngtphp66biq====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\"},{\"name\":\"gcc-runtime\",\"version\":\"8.5.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"aud4d72goxupc5p3p6mdkwgtshpygn7uuj2ewx3zm6wudcgw4fzq====\",\"dependencies\":[{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\"},{\"name\":\"glibc\",\"version\":\"2.28\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"external\":{\"path\":\"/usr\",\"module\":null,\"extra_attributes\":{}},\"package_hash\":\"riktbfk2yybad7tgbvdkntk5c5msjcm5pk3x7naszgbvfm57h4rq====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\"},{\"name\":\"gmake\",\"version\":\"4.4.1\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"guile\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"rpzjfobv7qh3wevti34nlbd2emtw5mnyszqmkyiq5jiq33xm7qzq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\"},{\"name\":\"libiconv\",\"version\":\"1.17\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"libs\":[\"shared\",\"static\"],\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ujsqmcknrabka5mhwwpbaf5rwxgopwoyxkskuwyqlcbynowgdvfa====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"3mozqilguvrkepcixf5v5czrvz64sn7a\"},{\"name\":\"m4\",\"version\":\"1.4.19\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"patches\":[\"9dc5fbd0d5cb1037ab1e6d0ecc74a30df218d0a94bdd5a02759a97f62daca573\",\"bfdffa7c2eb01021d5849b36972c069693654ad826c1a20b53534009a4ec7a89\"],\"sigsegv\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"patches\":[\"bfdffa7c2eb01021d5849b36972c069693654ad826c1a20b53534009a4ec7a89\",\"9dc5fbd0d5cb1037ab1e6d0ecc74a30df218d0a94bdd5a02759a97f62daca573\"],\"package_hash\":\"npb7a53yz7wqx4nvnasxwgzxaoiks6sdjz2eugrgkjxs4ml24xea====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"diffutils\",\"hash\":\"eso2orwqs33nyzewrf6ccckvkfoxdzn2\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libsigsegv\",\"hash\":\"4joh2v5wzpcg5cd5m4fnpwebagp47lai\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"gb5idois57zldhovt7rx44bd2ou4yiwr\"},{\"name\":\"libsigsegv\",\"version\":\"2.14\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"3s645t5rbjrziao47mhgob5xgymot6tf4kalagflbal2jdamdo2a====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"4joh2v5wzpcg5cd5m4fnpwebagp47lai\"},{\"name\":\"cmake\",\"version\":\"3.31.2\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"build_type\":\"Release\",\"doc\":false,\"ncurses\":true,\"ownlibs\":true,\"qtgui\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"7vk6yhuq2fklcj5kk7bhreqojudugggezq7vntmcsc32cw2avmya====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"curl\",\"hash\":\"dp2opcfk3d74hz2nokrdthwa4xc7ghmb\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"ncurses\",\"hash\":\"dm74bntb4otcekmwea6jmevqvhnono72\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"wrvzbh5ldwur22ypf3aa3srtdj77ufe7\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"2i4zyafripteq6cssiyrmo67n6tmypfs\"},{\"name\":\"curl\",\"version\":\"8.10.1\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"gssapi\":false,\"ldap\":false,\"libidn2\":false,\"librtmp\":false,\"libs\":[\"shared\",\"static\"],\"libssh\":false,\"libssh2\":false,\"nghttp2\":true,\"tls\":[\"openssl\"],\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ccka5yawqcn2rjbqn3bkhkdjoajlngm5uab7jbyrsl5yqn42ofza====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"nghttp2\",\"hash\":\"ztbzbssc6u4bylezsl6fc4hou2p3syju\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"openssl\",\"hash\":\"tdkectn77qw2zzxkgwduylz57p7zgi66\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"pkgconf\",\"hash\":\"6lfz6rvu2t7em2fovlh3xfsr6vynzxi2\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}},{\"name\":\"zlib-ng\",\"hash\":\"wrvzbh5ldwur22ypf3aa3srtdj77ufe7\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"dp2opcfk3d74hz2nokrdthwa4xc7ghmb\"},{\"name\":\"nghttp2\",\"version\":\"1.64.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"nkykfkj4rxzmysrmoh5mhxrl5ysaemlqh652m3he7pkbgvjhjgba====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"diffutils\",\"hash\":\"eso2orwqs33nyzewrf6ccckvkfoxdzn2\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"pkgconf\",\"hash\":\"6lfz6rvu2t7em2fovlh3xfsr6vynzxi2\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"ztbzbssc6u4bylezsl6fc4hou2p3syju\"},{\"name\":\"pkgconf\",\"version\":\"2.2.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"gl6tpyarjlclzsal6wa4dtc7cdzprq36nbibalai4a6wgzblrseq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"6lfz6rvu2t7em2fovlh3xfsr6vynzxi2\"},{\"name\":\"openssl\",\"version\":\"3.4.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"certs\":\"mozilla\",\"docs\":false,\"shared\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"5y33vxwjtlrlsyedasvmhukjkk5yfwcri27oceh36iw73xehumfa====\",\"dependencies\":[{\"name\":\"ca-certificates-mozilla\",\"hash\":\"oe3ftgfbeukmc6dzcmqjfgda7cccgx77\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"perl\",\"hash\":\"zjvu7ocv2zwrg4krarhjh3vvi2u3ha2h\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"wrvzbh5ldwur22ypf3aa3srtdj77ufe7\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"tdkectn77qw2zzxkgwduylz57p7zgi66\"},{\"name\":\"ca-certificates-mozilla\",\"version\":\"2023-05-30\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"63npvwqwo2x7i6emvnklh4mhcn45gx2qzveorybh5h2inwr55sea====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"oe3ftgfbeukmc6dzcmqjfgda7cccgx77\"},{\"name\":\"perl\",\"version\":\"5.40.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cpanm\":true,\"opcode\":true,\"open\":true,\"shared\":true,\"threads\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"f233ue76vwtkle2r4jwsfe5x27ujx6ea4vdyp6baonfmkgqf5vpa====\",\"dependencies\":[{\"name\":\"berkeley-db\",\"hash\":\"lfuzet6lgtupoudoympe7rzjb4yndv2d\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"bzip2\",\"hash\":\"mhpnc4vabp2r5fxmq6aakyvofvnnmldt\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"gdbm\",\"hash\":\"5dd4vl5on3dfg6dd6yxy5t5vrpfwaii5\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"wrvzbh5ldwur22ypf3aa3srtdj77ufe7\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"zjvu7ocv2zwrg4krarhjh3vvi2u3ha2h\"},{\"name\":\"berkeley-db\",\"version\":\"18.1.40\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cxx\":true,\"docs\":false,\"patches\":[\"26090f418891757af46ac3b89a9f43d6eb5989f7a3dce3d1cfc99fba547203b3\",\"b231fcc4d5cff05e5c3a4814f6a5af0e9a966428dc2176540d2c05aff41de522\"],\"stl\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"patches\":[\"b231fcc4d5cff05e5c3a4814f6a5af0e9a966428dc2176540d2c05aff41de522\",\"26090f418891757af46ac3b89a9f43d6eb5989f7a3dce3d1cfc99fba547203b3\"],\"package_hash\":\"h57ydfn33zevvzctzzioiiwjwe362izbbwncb6a26dfeno4y7tda====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"lfuzet6lgtupoudoympe7rzjb4yndv2d\"},{\"name\":\"bzip2\",\"version\":\"1.0.8\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"debug\":false,\"pic\":false,\"shared\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"jb7yvhkifmvfl3ykmdulsjxkkulker6gqb5tadollyjt2ijg3zsa====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"diffutils\",\"hash\":\"eso2orwqs33nyzewrf6ccckvkfoxdzn2\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"mhpnc4vabp2r5fxmq6aakyvofvnnmldt\"},{\"name\":\"gdbm\",\"version\":\"1.23\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"liepxl6phlcxbgfmibxafhewtihlgaa4x3hko37ckqlafhxkrgdq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"readline\",\"hash\":\"23lcaxfxq4fy5hchfratqxywajwjgspx\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"5dd4vl5on3dfg6dd6yxy5t5vrpfwaii5\"},{\"name\":\"readline\",\"version\":\"8.2\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"patches\":[\"bbf97f1ec40a929edab5aa81998c1e2ef435436c597754916e6a5868f273aff7\"],\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"patches\":[\"bbf97f1ec40a929edab5aa81998c1e2ef435436c597754916e6a5868f273aff7\"],\"package_hash\":\"oww6dmr7xqgg6j7iiluonxbcl4irqnnrip4vfkjdwujncwnuhwuq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"ncurses\",\"hash\":\"dm74bntb4otcekmwea6jmevqvhnono72\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"23lcaxfxq4fy5hchfratqxywajwjgspx\"},{\"name\":\"ncurses\",\"version\":\"6.5\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"abi\":\"none\",\"build_system\":\"autotools\",\"patches\":[\"7a351bc4953a4ab70dabdbea31c8db0c03d40ce505335f3b6687180dde24c535\"],\"symlinks\":false,\"termlib\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"patches\":[\"7a351bc4953a4ab70dabdbea31c8db0c03d40ce505335f3b6687180dde24c535\"],\"package_hash\":\"cfh76rniab2gnv4jqr77yzz5za4ucfmva2upihvxukn52dybhsvq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"pkgconf\",\"hash\":\"6lfz6rvu2t7em2fovlh3xfsr6vynzxi2\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"dm74bntb4otcekmwea6jmevqvhnono72\"},{\"name\":\"zlib-ng\",\"version\":\"2.2.1\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"compat\":true,\"new_strategies\":true,\"opt\":true,\"pic\":true,\"shared\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"mdxo2xewbdavckgsqlcjywyfssdchgwbzonui22gxww7hqtozurq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"wrvzbh5ldwur22ypf3aa3srtdj77ufe7\"},{\"name\":\"python\",\"version\":\"3.13.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"bz2\":true,\"ctypes\":true,\"dbm\":true,\"debug\":false,\"libxml2\":true,\"lzma\":true,\"optimizations\":false,\"pic\":true,\"pyexpat\":true,\"pythoncmd\":true,\"readline\":true,\"shared\":true,\"sqlite3\":true,\"ssl\":true,\"tkinter\":false,\"uuid\":true,\"zlib\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"n6v6rt6deysntdggu2gi4zkhqriyba6bgaghxyhluou4ssqf7xfq====\",\"dependencies\":[{\"name\":\"bzip2\",\"hash\":\"mhpnc4vabp2r5fxmq6aakyvofvnnmldt\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"expat\",\"hash\":\"cnsgli2fxpinhfywuenoz2t4dhc47hqw\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"gdbm\",\"hash\":\"5dd4vl5on3dfg6dd6yxy5t5vrpfwaii5\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"gettext\",\"hash\":\"fdsw6uskzn4ddgrmdqcseatiziy2pdtx\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libffi\",\"hash\":\"yht6xjipvotkpf3t56t4qhzzng4gbluj\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"ncurses\",\"hash\":\"dm74bntb4otcekmwea6jmevqvhnono72\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"openssl\",\"hash\":\"tdkectn77qw2zzxkgwduylz57p7zgi66\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"pkgconf\",\"hash\":\"6lfz6rvu2t7em2fovlh3xfsr6vynzxi2\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}},{\"name\":\"readline\",\"hash\":\"23lcaxfxq4fy5hchfratqxywajwjgspx\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"sqlite\",\"hash\":\"ypawguzvgqolvimqyrun5r3rfbdphfsg\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"util-linux-uuid\",\"hash\":\"l7pvs6vnv6exgs4uci6ulfrcqb7codqp\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"uuid\"]}},{\"name\":\"xz\",\"hash\":\"ojolxif3gv5pmuc3zveqie7zcbtpgjfd\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"wrvzbh5ldwur22ypf3aa3srtdj77ufe7\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"7f76ydmj6f4epvepmak2y5qfllqow5db\"},{\"name\":\"expat\",\"version\":\"2.6.4\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"libbsd\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ei6qyjakl7sgtodwxxbg5brgkp23robfximtpbedkrnpyyyvr3ya====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libbsd\",\"hash\":\"xlv3vxthk3ra5fsoe7e55pcroy6njci2\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"cnsgli2fxpinhfywuenoz2t4dhc47hqw\"},{\"name\":\"libbsd\",\"version\":\"0.12.2\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"debyg3en7sgggswkdhcyd6lbp7arawzmyujthyyuaiad5jqd5msa====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libmd\",\"hash\":\"w5jj3yfzzxvwjoptrwnna3rbooo44i3b\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"xlv3vxthk3ra5fsoe7e55pcroy6njci2\"},{\"name\":\"libmd\",\"version\":\"1.0.4\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"zs2e7fqr4dzthpj5fascqvfn7xcahf7dtc5bzdwfv6vqkzi7oncq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"w5jj3yfzzxvwjoptrwnna3rbooo44i3b\"},{\"name\":\"gettext\",\"version\":\"0.22.5\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"bzip2\":true,\"curses\":true,\"git\":true,\"libunistring\":false,\"libxml2\":true,\"pic\":true,\"shared\":true,\"tar\":true,\"xz\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"d4zxhmw6rownaaokzcolsszrq2cmx44m7qmzopucymoyrhbdfgvq====\",\"dependencies\":[{\"name\":\"bzip2\",\"hash\":\"mhpnc4vabp2r5fxmq6aakyvofvnnmldt\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libiconv\",\"hash\":\"3mozqilguvrkepcixf5v5czrvz64sn7a\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"iconv\"]}},{\"name\":\"libxml2\",\"hash\":\"xv3omnzedrjqkpn4sda6suxsfeauzkvz\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"ncurses\",\"hash\":\"dm74bntb4otcekmwea6jmevqvhnono72\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"tar\",\"hash\":\"y5e5unbos2j4egc75khytcwtvfmznsxx\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}},{\"name\":\"xz\",\"hash\":\"ojolxif3gv5pmuc3zveqie7zcbtpgjfd\",\"parameters\":{\"deptypes\":[\"build\",\"link\",\"run\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"fdsw6uskzn4ddgrmdqcseatiziy2pdtx\"},{\"name\":\"libxml2\",\"version\":\"2.13.4\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"pic\":true,\"python\":false,\"shared\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"j6yob2wgvc2wjzvbs6xdvgyfa3zp3wrm3uxncxzxqfzw6xazzoba====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libiconv\",\"hash\":\"3mozqilguvrkepcixf5v5czrvz64sn7a\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"iconv\"]}},{\"name\":\"pkgconf\",\"hash\":\"6lfz6rvu2t7em2fovlh3xfsr6vynzxi2\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}},{\"name\":\"xz\",\"hash\":\"ojolxif3gv5pmuc3zveqie7zcbtpgjfd\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"wrvzbh5ldwur22ypf3aa3srtdj77ufe7\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"xv3omnzedrjqkpn4sda6suxsfeauzkvz\"},{\"name\":\"xz\",\"version\":\"5.4.6\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"libs\":[\"shared\",\"static\"],\"pic\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"zt5vu2vph2v2qjwgdbe7btgcz7axpyalorcsqiuxhrg5grwgrrvq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"ojolxif3gv5pmuc3zveqie7zcbtpgjfd\"},{\"name\":\"tar\",\"version\":\"1.35\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"zip\":\"pigz\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"v6a6jvks2setklucxyk622uauxzqlgmsdkrvdijbi3m5jwftmzla====\",\"dependencies\":[{\"name\":\"bzip2\",\"hash\":\"mhpnc4vabp2r5fxmq6aakyvofvnnmldt\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"libiconv\",\"hash\":\"3mozqilguvrkepcixf5v5czrvz64sn7a\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"iconv\"]}},{\"name\":\"pigz\",\"hash\":\"6gltt7sf6leoizgacsyxcvkfjhfajubf\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}},{\"name\":\"xz\",\"hash\":\"ojolxif3gv5pmuc3zveqie7zcbtpgjfd\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}},{\"name\":\"zstd\",\"hash\":\"7hd6zzagnpahpiu46rg2i4ht32mdndmj\",\"parameters\":{\"deptypes\":[\"run\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"y5e5unbos2j4egc75khytcwtvfmznsxx\"},{\"name\":\"pigz\",\"version\":\"2.8\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"makefile\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"4w67lflje4giekjg4ie2vpyuiunjcumo6geofykvon3hodllp42q====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"wrvzbh5ldwur22ypf3aa3srtdj77ufe7\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"6gltt7sf6leoizgacsyxcvkfjhfajubf\"},{\"name\":\"zstd\",\"version\":\"1.5.6\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"makefile\",\"compression\":[\"none\"],\"libs\":[\"shared\",\"static\"],\"programs\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"uvmrov4c6unft6o4yd3jk3uqvweua3uhwdli4sw7h5wvklaf5t3q====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"7hd6zzagnpahpiu46rg2i4ht32mdndmj\"},{\"name\":\"libffi\",\"version\":\"3.4.6\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"umhsnvoj5ooa3glffnkk2hp3txmrsjvqbpfq2hbk4mhcvhza7gaa====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"yht6xjipvotkpf3t56t4qhzzng4gbluj\"},{\"name\":\"sqlite\",\"version\":\"3.46.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"column_metadata\":true,\"dynamic_extensions\":true,\"fts\":true,\"functions\":false,\"rtree\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"wm3irnrjil5n275nw2m4x3mpvyg35h7isbmsnuae6vtxbamsrv4q====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"readline\",\"hash\":\"23lcaxfxq4fy5hchfratqxywajwjgspx\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"zlib-ng\",\"hash\":\"wrvzbh5ldwur22ypf3aa3srtdj77ufe7\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"ypawguzvgqolvimqyrun5r3rfbdphfsg\"},{\"name\":\"util-linux-uuid\",\"version\":\"2.40.2\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"eo6au7zhsz344imzoomhuskbl3cmrqq6ja6mcmrc3li3fnppqs6q====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"pkgconf\",\"hash\":\"6lfz6rvu2t7em2fovlh3xfsr6vynzxi2\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"pkgconfig\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"l7pvs6vnv6exgs4uci6ulfrcqb7codqp\"},{\"name\":\"python-venv\",\"version\":\"1.0\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"j3dgyzp5nei24fbpw22l3gedsk37asrdrjafbnaiwiux3lxasi3a====\",\"dependencies\":[{\"name\":\"python\",\"hash\":\"7f76ydmj6f4epvepmak2y5qfllqow5db\",\"parameters\":{\"deptypes\":[\"build\",\"run\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"mvybzkpm37r4xrt3eip5nn2padrhnlrm\"},{\"name\":\"re2c\",\"version\":\"3.1\",\"arch\":{\"platform\":\"linux\",\"platform_os\":\"rhel8\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"autotools\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ebw3m3xkgw2wijfijtzrxt4ldu4tz4haiz6juumq6wn4mjzsuxra====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"3wjlvksj4tr3qckfozocbeziogwilggn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"xfrx6wio34o7fhpwtv6kjypvxlurblwr\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"hfi7ird7tq2ektlpntoru7znszd7lkbu\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"glibc\",\"hash\":\"z3v4q7z2ksjom7krlru22p27j4mdyw2s\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[\"libc\"]}},{\"name\":\"gmake\",\"hash\":\"l65jstphe3wyvixgkd3lv4dp5boxxjhe\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"python\",\"hash\":\"7f76ydmj6f4epvepmak2y5qfllqow5db\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"ketxaszk5wezamuffgkdpie66tkd7rbl\"}]}}\n" }, { "path": "lib/spack/spack/bootstrap/prototypes/clingo-windows-x86_64.json", "content": "{\"spec\":{\"_meta\":{\"version\":5},\"nodes\":[{\"name\":\"clingo-bootstrap\",\"version\":\"spack\",\"arch\":{\"platform\":\"windows\",\"platform_os\":\"windows10.0.19045\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"cmake\",\"build_type\":\"Release\",\"docs\":false,\"generator\":\"ninja\",\"ipo\":false,\"optimized\":false,\"patches\":[\"311bd2ae3f2f5274d1d36a2d65f887dfdf4c309a3c6bb29a53bbafb82b42ba7a\",\"4ccfd173d439ed1e23eff42d5a01a8fbb21341c632d86b5691242dc270dbf065\",\"c5c4db292a920ded6eecfbb6749d88ce9c4f179500aee6aee3a417b93c7c5c7a\"],\"python\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"patches\":[\"4ccfd173d439ed1e23eff42d5a01a8fbb21341c632d86b5691242dc270dbf065\",\"311bd2ae3f2f5274d1d36a2d65f887dfdf4c309a3c6bb29a53bbafb82b42ba7a\",\"c5c4db292a920ded6eecfbb6749d88ce9c4f179500aee6aee3a417b93c7c5c7a\"],\"package_hash\":\"4c42opkd2w53rbrvk73mrxvy2ynkvq5wj2lang7ov2ptpimldsxa====\",\"dependencies\":[{\"name\":\"cmake\",\"hash\":\"zumu22rfkjg3krutmigxxkx2me42efes\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"rzlyyiuxoojqqm6w2eo5ddyq4psu4ni2\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"msvc\",\"hash\":\"skajkv74f2oyno7p5xp25no66w2mrtrk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"ninja\",\"hash\":\"bqypodje25rvy7ozbsyhzve42m6mcpsx\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"python\",\"hash\":\"j4qa7xsbagk4dex5qo3777lv4jdgbpwn\",\"parameters\":{\"deptypes\":[\"build\",\"link\",\"run\"],\"virtuals\":[]}},{\"name\":\"python-venv\",\"hash\":\"po2f6c4cf4nfwd57jshovkkp6zhsxpuc\",\"parameters\":{\"deptypes\":[\"build\",\"run\"],\"virtuals\":[]}},{\"name\":\"re2c\",\"hash\":\"mf2atm3mtzukanuqcuk6vxmtcnvrjfm6\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"winbison\",\"hash\":\"sjrbf3m2ypcbf2quglw26qfn3kksigyu\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"wzcmsgouevrl3jpzwoh2gh7upehzxta3\"},{\"name\":\"cmake\",\"version\":\"3.31.2\",\"arch\":{\"platform\":\"windows\",\"platform_os\":\"windows10.0.19045\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"build_type\":\"Release\",\"doc\":false,\"ncurses\":false,\"ownlibs\":true,\"qtgui\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"7vk6yhuq2fklcj5kk7bhreqojudugggezq7vntmcsc32cw2avmya====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rzlyyiuxoojqqm6w2eo5ddyq4psu4ni2\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"curl\",\"hash\":\"etpxh45rduqsnd6fap5uj5qzhijabs4g\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"msvc\",\"hash\":\"skajkv74f2oyno7p5xp25no66w2mrtrk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"ninja\",\"hash\":\"bqypodje25rvy7ozbsyhzve42m6mcpsx\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}},{\"name\":\"zlib\",\"hash\":\"sweajh5242hgibn2nsvapphwztahxzpo\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"zumu22rfkjg3krutmigxxkx2me42efes\"},{\"name\":\"compiler-wrapper\",\"version\":\"1.0\",\"arch\":{\"platform\":\"windows\",\"platform_os\":\"windows10.0.19045\",\"target\":{\"name\":\"broadwell\",\"vendor\":\"GenuineIntel\",\"features\":[\"adx\",\"aes\",\"avx\",\"avx2\",\"bmi1\",\"bmi2\",\"f16c\",\"fma\",\"mmx\",\"movbe\",\"pclmulqdq\",\"popcnt\",\"rdrand\",\"rdseed\",\"sse\",\"sse2\",\"sse4_1\",\"sse4_2\",\"ssse3\"],\"generation\":0,\"parents\":[\"haswell\"],\"cpupart\":\"\"}},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"f2cvl7ifstxe4onighf2lrijbckr3wwlzjaqt3yaxtxmepeldkwq====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"rzlyyiuxoojqqm6w2eo5ddyq4psu4ni2\"},{\"name\":\"curl\",\"version\":\"8.10.1\",\"arch\":{\"platform\":\"windows\",\"platform_os\":\"windows10.0.19045\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"nmake\",\"gssapi\":false,\"ldap\":false,\"libidn2\":false,\"librtmp\":false,\"libs\":\"shared\",\"libssh\":false,\"libssh2\":false,\"nghttp2\":false,\"tls\":[\"sspi\"],\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ccka5yawqcn2rjbqn3bkhkdjoajlngm5uab7jbyrsl5yqn42ofza====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rzlyyiuxoojqqm6w2eo5ddyq4psu4ni2\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"msvc\",\"hash\":\"skajkv74f2oyno7p5xp25no66w2mrtrk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"zlib\",\"hash\":\"sweajh5242hgibn2nsvapphwztahxzpo\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[\"zlib-api\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"etpxh45rduqsnd6fap5uj5qzhijabs4g\"},{\"name\":\"msvc\",\"version\":\"19.41.34120\",\"arch\":{\"platform\":\"windows\",\"platform_os\":\"windows10.0.19045\",\"target\":{\"name\":\"broadwell\",\"vendor\":\"GenuineIntel\",\"features\":[\"adx\",\"aes\",\"avx\",\"avx2\",\"bmi1\",\"bmi2\",\"f16c\",\"fma\",\"mmx\",\"movbe\",\"pclmulqdq\",\"popcnt\",\"rdrand\",\"rdseed\",\"sse\",\"sse2\",\"sse4_1\",\"sse4_2\",\"ssse3\"],\"generation\":0,\"parents\":[\"haswell\"],\"cpupart\":\"\"}},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"external\":{\"path\":\"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\VC\\\\Tools\\\\MSVC\\\\14.41.34120\",\"module\":null,\"extra_attributes\":{\"compilers\":{\"c\":\"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\VC\\\\Tools\\\\MSVC\\\\14.41.34120\\\\bin\\\\Hostx64\\\\x64\\\\cl.exe\",\"cxx\":\"C:\\\\Program Files\\\\Microsoft Visual Studio\\\\2022\\\\Community\\\\VC\\\\Tools\\\\MSVC\\\\14.41.34120\\\\bin\\\\Hostx64\\\\x64\\\\cl.exe\"}}},\"package_hash\":\"xywxjwuwneitqkaxzvyewhvhhr4zzuxhewmj6vmvf3cq7nf24k2a====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"skajkv74f2oyno7p5xp25no66w2mrtrk\"},{\"name\":\"zlib\",\"version\":\"1.3.1\",\"arch\":{\"platform\":\"windows\",\"platform_os\":\"windows10.0.19045\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"optimize\":true,\"pic\":true,\"shared\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"7m5x6iihfcayy4fhcdurbffk4krn7ykq2vo6wxbr2ue2pgtetf4a====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rzlyyiuxoojqqm6w2eo5ddyq4psu4ni2\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"msvc\",\"hash\":\"skajkv74f2oyno7p5xp25no66w2mrtrk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"sweajh5242hgibn2nsvapphwztahxzpo\"},{\"name\":\"ninja\",\"version\":\"1.12.1\",\"arch\":{\"platform\":\"windows\",\"platform_os\":\"windows10.0.19045\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"re2c\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"eanqnmavyldorxcgxf6z3j76hehc37sw55hhjbnnjy4gsvrtji3a====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rzlyyiuxoojqqm6w2eo5ddyq4psu4ni2\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"msvc\",\"hash\":\"skajkv74f2oyno7p5xp25no66w2mrtrk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"python\",\"hash\":\"j4qa7xsbagk4dex5qo3777lv4jdgbpwn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"bqypodje25rvy7ozbsyhzve42m6mcpsx\"},{\"name\":\"python\",\"version\":\"3.13.0\",\"arch\":{\"platform\":\"windows\",\"platform_os\":\"windows10.0.19045\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"bz2\":true,\"ctypes\":true,\"dbm\":true,\"debug\":false,\"libxml2\":true,\"lzma\":true,\"optimizations\":false,\"pic\":true,\"pyexpat\":true,\"pythoncmd\":false,\"readline\":false,\"shared\":true,\"sqlite3\":true,\"ssl\":true,\"tkinter\":false,\"uuid\":true,\"zlib\":true,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"n6v6rt6deysntdggu2gi4zkhqriyba6bgaghxyhluou4ssqf7xfq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"rzlyyiuxoojqqm6w2eo5ddyq4psu4ni2\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"msvc\",\"hash\":\"skajkv74f2oyno7p5xp25no66w2mrtrk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"j4qa7xsbagk4dex5qo3777lv4jdgbpwn\"},{\"name\":\"python-venv\",\"version\":\"1.0\",\"arch\":{\"platform\":\"windows\",\"platform_os\":\"windows10.0.19045\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"j3dgyzp5nei24fbpw22l3gedsk37asrdrjafbnaiwiux3lxasi3a====\",\"dependencies\":[{\"name\":\"python\",\"hash\":\"j4qa7xsbagk4dex5qo3777lv4jdgbpwn\",\"parameters\":{\"deptypes\":[\"build\",\"run\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"po2f6c4cf4nfwd57jshovkkp6zhsxpuc\"},{\"name\":\"re2c\",\"version\":\"3.1\",\"arch\":{\"platform\":\"windows\",\"platform_os\":\"windows10.0.19045\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"cmake\",\"build_type\":\"Release\",\"generator\":\"ninja\",\"ipo\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ebw3m3xkgw2wijfijtzrxt4ldu4tz4haiz6juumq6wn4mjzsuxra====\",\"dependencies\":[{\"name\":\"cmake\",\"hash\":\"zumu22rfkjg3krutmigxxkx2me42efes\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"rzlyyiuxoojqqm6w2eo5ddyq4psu4ni2\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"msvc\",\"hash\":\"skajkv74f2oyno7p5xp25no66w2mrtrk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"ninja\",\"hash\":\"bqypodje25rvy7ozbsyhzve42m6mcpsx\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"python\",\"hash\":\"j4qa7xsbagk4dex5qo3777lv4jdgbpwn\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"mf2atm3mtzukanuqcuk6vxmtcnvrjfm6\"},{\"name\":\"winbison\",\"version\":\"2.5.25\",\"arch\":{\"platform\":\"windows\",\"platform_os\":\"windows10.0.19045\",\"target\":\"x86_64\"},\"namespace\":\"builtin\",\"parameters\":{\"build_system\":\"cmake\",\"build_type\":\"Release\",\"generator\":\"ninja\",\"ipo\":false,\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"t3g2slcnnleieqtz66oly6vsfe5ibje6b2wmamxv5chuewwds5la====\",\"dependencies\":[{\"name\":\"cmake\",\"hash\":\"zumu22rfkjg3krutmigxxkx2me42efes\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"compiler-wrapper\",\"hash\":\"rzlyyiuxoojqqm6w2eo5ddyq4psu4ni2\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"msvc\",\"hash\":\"skajkv74f2oyno7p5xp25no66w2mrtrk\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"ninja\",\"hash\":\"bqypodje25rvy7ozbsyhzve42m6mcpsx\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"sjrbf3m2ypcbf2quglw26qfn3kksigyu\"}]}}\n" }, { "path": "lib/spack/spack/bootstrap/status.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Query the status of bootstrapping on this machine\"\"\"\n\nimport sys\nfrom typing import List, Optional, Sequence, Tuple, Union\n\nimport spack.util.executable\n\nfrom ._common import _executables_in_store, _python_import, _try_import_from_store\nfrom .config import ensure_bootstrap_configuration\nfrom .core import clingo_root_spec, gnupg_root_spec, patchelf_root_spec\nfrom .environment import BootstrapEnvironment, mypy_root_spec, pytest_root_spec, ruff_root_spec\n\nExecutablesType = Union[str, Sequence[str]]\nRequiredResponseType = Tuple[bool, Optional[str]]\nSpecLike = Union[\"spack.spec.Spec\", str]\n\n\ndef _required_system_executable(exes: ExecutablesType, msg: str) -> RequiredResponseType:\n \"\"\"Search for an executable is the system path only.\"\"\"\n if isinstance(exes, str):\n exes = (exes,)\n if spack.util.executable.which_string(*exes):\n return True, None\n return False, msg\n\n\ndef _required_executable(\n exes: ExecutablesType, query_spec: SpecLike, msg: str\n) -> RequiredResponseType:\n \"\"\"Search for an executable in the system path or in the bootstrap store.\"\"\"\n if isinstance(exes, str):\n exes = (exes,)\n if spack.util.executable.which_string(*exes) or _executables_in_store(exes, query_spec):\n return True, None\n return False, msg\n\n\ndef _required_python_module(module: str, query_spec: SpecLike, msg: str) -> RequiredResponseType:\n \"\"\"Check if a Python module is available in the current interpreter or\n if it can be loaded from the bootstrap store\n \"\"\"\n if _python_import(module) or _try_import_from_store(module, query_spec):\n return True, None\n return False, msg\n\n\ndef _missing(name: str, purpose: str, system_only: bool = True) -> str:\n \"\"\"Message to be printed if an executable is not found\"\"\"\n msg = '[{2}] MISSING \"{0}\": {1}'\n if not system_only:\n return msg.format(name, purpose, \"@*y{{B}}\")\n return msg.format(name, purpose, \"@*y{{-}}\")\n\n\ndef _core_requirements() -> List[RequiredResponseType]:\n _core_system_exes = {\n \"patch\": _missing(\"patch\", \"required to patch source code before building\"),\n \"tar\": _missing(\"tar\", \"required to manage code archives\"),\n \"gzip\": _missing(\"gzip\", \"required to compress/decompress code archives\"),\n \"unzip\": _missing(\"unzip\", \"required to compress/decompress code archives\"),\n \"bzip2\": _missing(\"bzip2\", \"required to compress/decompress code archives\"),\n \"git\": _missing(\"git\", \"required to fetch/manage git repositories\"),\n }\n if sys.platform == \"linux\":\n _core_system_exes[\"xz\"] = _missing(\"xz\", \"required to compress/decompress code archives\")\n\n # Executables that are not bootstrapped yet\n result = [_required_system_executable(exe, msg) for exe, msg in _core_system_exes.items()]\n # Python modules\n result.append(\n _required_python_module(\n \"clingo\", clingo_root_spec(), _missing(\"clingo\", \"required to concretize specs\", False)\n )\n )\n return result\n\n\ndef _buildcache_requirements() -> List[RequiredResponseType]:\n # Add bootstrappable executables (these can be in PATH or bootstrapped)\n # GPG/GPG2 - used for signing and verifying buildcaches\n result = [\n _required_executable(\n (\"gpg2\", \"gpg\"),\n gnupg_root_spec(),\n _missing(\"gpg2\", \"required to sign/verify buildcaches\", False),\n )\n ]\n\n # Patchelf - only needed on Linux, used for binary relocation\n if sys.platform == \"linux\":\n result.append(\n _required_executable(\n \"patchelf\",\n patchelf_root_spec(),\n _missing(\"patchelf\", \"required to relocate binaries\", False),\n )\n )\n\n return result\n\n\ndef _optional_requirements() -> List[RequiredResponseType]:\n _optional_exes = {\n \"zstd\": _missing(\"zstd\", \"required to compress/decompress code archives\"),\n \"svn\": _missing(\"svn\", \"required to manage subversion repositories\"),\n \"hg\": _missing(\"hg\", \"required to manage mercurial repositories\"),\n }\n # Executables that are not bootstrapped yet\n result = [_required_system_executable(exe, msg) for exe, msg in _optional_exes.items()]\n return result\n\n\ndef _development_requirements() -> List[RequiredResponseType]:\n # Ensure we trigger environment modifications if we have an environment\n if BootstrapEnvironment.spack_yaml().exists():\n with BootstrapEnvironment() as env:\n env.load()\n\n return [\n _required_python_module(\n \"pytest\", pytest_root_spec(), _missing(\"pytest\", \"required to run unit-test\", False)\n ),\n _required_executable(\n \"ruff\",\n ruff_root_spec(),\n _missing(\"ruff\", \"required for code checking/formatting\", False),\n ),\n _required_executable(\n \"mypy\", mypy_root_spec(), _missing(\"mypy\", \"required for type checks\", False)\n ),\n ]\n\n\ndef status_message(section) -> Tuple[str, bool]:\n \"\"\"Return a status message to be printed to screen that refers to the\n section passed as argument and a bool which is True if there are missing\n dependencies.\n\n Args:\n section (str): either 'core' or 'buildcache' or 'optional' or 'develop'\n \"\"\"\n pass_token, fail_token = \"@*g{[PASS]}\", \"@*r{[FAIL]}\"\n\n # Contain the header of the section and a list of requirements\n spack_sections = {\n \"core\": (\"{0} @*{{Core Functionalities}}\", _core_requirements),\n \"buildcache\": (\"{0} @*{{Binary packages}}\", _buildcache_requirements),\n \"optional\": (\"{0} @*{{Optional Features}}\", _optional_requirements),\n \"develop\": (\"{0} @*{{Development Dependencies}}\", _development_requirements),\n }\n msg, required_software = spack_sections[section]\n\n with ensure_bootstrap_configuration():\n missing_software = False\n for found, err_msg in required_software():\n if not found and err_msg:\n missing_software = True\n msg += \"\\n \" + err_msg\n msg += \"\\n\"\n msg = msg.format(pass_token if not missing_software else fail_token)\n return msg, missing_software\n" }, { "path": "lib/spack/spack/build_environment.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"\nThis module contains all routines related to setting up the package\nbuild environment. All of this is set up by package.py just before\ninstall() is called.\n\nThere are two parts to the build environment:\n\n1. Python build environment (i.e. install() method)\n\n This is how things are set up when install() is called. Spack\n takes advantage of each package being in its own module by adding a\n bunch of command-like functions (like configure(), make(), etc.) in\n the package's module scope. This allows package writers to call\n them all directly in Package.install() without writing 'self.'\n everywhere. No, this isn't Pythonic. Yes, it makes the code more\n readable and more like the shell script from which someone is\n likely porting.\n\n2. Build execution environment\n\n This is the set of environment variables, like PATH, CC, CXX,\n etc. that control the build. There are also a number of\n environment variables used to pass information (like RPATHs and\n other information about dependencies) to Spack's compiler wrappers.\n All of these env vars are also set up here.\n\nSkimming this module is a nice way to get acquainted with the types of\ncalls you can make from within the install() function.\n\"\"\"\n\nimport inspect\nimport io\nimport multiprocessing\nimport os\nimport re\nimport signal\nimport sys\nimport traceback\nimport types\nimport warnings\nfrom collections import defaultdict\nfrom enum import Flag, auto\nfrom itertools import chain\nfrom multiprocessing.connection import Connection\nfrom typing import (\n Any,\n BinaryIO,\n Callable,\n Dict,\n List,\n Optional,\n Sequence,\n Set,\n Tuple,\n Type,\n Union,\n overload,\n)\n\nimport spack.vendor.archspec.cpu\n\nimport spack.builder\nimport spack.compilers.libraries\nimport spack.config\nimport spack.deptypes as dt\nimport spack.error\nimport spack.llnl.util.tty as tty\nimport spack.multimethod\nimport spack.package_base\nimport spack.paths\nimport spack.platforms\nimport spack.schema.environment\nimport spack.spec\nimport spack.stage\nimport spack.store\nimport spack.subprocess_context\nimport spack.util.executable\nimport spack.util.module_cmd\nfrom spack import traverse\nfrom spack.context import Context\nfrom spack.error import InstallError, NoHeadersError, NoLibrariesError\nfrom spack.install_test import spack_install_test_log\nfrom spack.llnl.string import plural\nfrom spack.llnl.util.filesystem import join_path, symlink\nfrom spack.llnl.util.lang import dedupe, stable_partition\nfrom spack.llnl.util.tty.color import cescape, colorize\nfrom spack.util.environment import (\n SYSTEM_DIR_CASE_ENTRY,\n EnvironmentModifications,\n ModificationList,\n PrependPath,\n env_flag,\n filter_system_paths,\n get_path,\n is_system_path,\n validate,\n)\nfrom spack.util.executable import Executable\nfrom spack.util.log_parse import make_log_context, parse_log_events\n\n#\n# This can be set by the user to globally disable parallel builds.\n#\nSPACK_NO_PARALLEL_MAKE = \"SPACK_NO_PARALLEL_MAKE\"\n\n#\n# These environment variables are set by\n# set_wrapper_variables and used to pass parameters to\n# Spack's compiler wrappers.\n#\nSPACK_COMPILER_WRAPPER_PATH = \"SPACK_COMPILER_WRAPPER_PATH\"\nSPACK_MANAGED_DIRS = \"SPACK_MANAGED_DIRS\"\nSPACK_INCLUDE_DIRS = \"SPACK_INCLUDE_DIRS\"\nSPACK_LINK_DIRS = \"SPACK_LINK_DIRS\"\nSPACK_RPATH_DIRS = \"SPACK_RPATH_DIRS\"\nSPACK_STORE_INCLUDE_DIRS = \"SPACK_STORE_INCLUDE_DIRS\"\nSPACK_STORE_LINK_DIRS = \"SPACK_STORE_LINK_DIRS\"\nSPACK_STORE_RPATH_DIRS = \"SPACK_STORE_RPATH_DIRS\"\nSPACK_RPATH_DEPS = \"SPACK_RPATH_DEPS\"\nSPACK_LINK_DEPS = \"SPACK_LINK_DEPS\"\nSPACK_PREFIX = \"SPACK_PREFIX\"\nSPACK_INSTALL = \"SPACK_INSTALL\"\nSPACK_DEBUG = \"SPACK_DEBUG\"\nSPACK_SHORT_SPEC = \"SPACK_SHORT_SPEC\"\nSPACK_DEBUG_LOG_ID = \"SPACK_DEBUG_LOG_ID\"\nSPACK_DEBUG_LOG_DIR = \"SPACK_DEBUG_LOG_DIR\"\nSPACK_CCACHE_BINARY = \"SPACK_CCACHE_BINARY\"\nSPACK_SYSTEM_DIRS = \"SPACK_SYSTEM_DIRS\"\n\n# Platform-specific library suffix (deprecated)\nif sys.platform == \"darwin\":\n dso_suffix = \"dylib\"\nelif sys.platform == \"win32\":\n dso_suffix = \"dll\"\nelse:\n dso_suffix = \"so\"\n\nstat_suffix = \"lib\" if sys.platform == \"win32\" else \"a\"\n\n\ndef shared_library_suffix(spec: spack.spec.Spec) -> str:\n \"\"\"Return the shared library suffix for the given spec.\"\"\"\n if spec.platform == \"darwin\":\n return \"dylib\"\n elif spec.platform == \"windows\":\n return \"dll\"\n else:\n return \"so\"\n\n\ndef static_library_suffix(spec: spack.spec.Spec) -> str:\n \"\"\"Return the static library suffix for the given spec.\"\"\"\n if spec.platform == \"windows\":\n return \"lib\"\n else:\n return \"a\"\n\n\ndef jobserver_enabled():\n \"\"\"Returns true if a posix jobserver (make) is detected.\"\"\"\n return \"MAKEFLAGS\" in os.environ and \"--jobserver\" in os.environ[\"MAKEFLAGS\"]\n\n\ndef get_effective_jobs(\n jobs, parallel: bool = True, supports_jobserver: bool = False\n) -> Optional[int]:\n \"\"\"Return the number of jobs, or None if supports_jobserver and a jobserver is detected.\"\"\"\n if not parallel or jobs <= 1 or env_flag(SPACK_NO_PARALLEL_MAKE):\n return 1\n if supports_jobserver and jobserver_enabled():\n return None\n return jobs\n\n\nclass MakeExecutable(Executable):\n \"\"\"Special callable executable object for make so the user can specify parallelism options\n on a per-invocation basis.\n \"\"\"\n\n def __init__(self, name: str, *, jobs: int, supports_jobserver: bool = True) -> None:\n super().__init__(name)\n self.supports_jobserver = supports_jobserver\n self.jobs = jobs\n\n @overload\n def __call__(\n self,\n *args: str,\n parallel: bool = ...,\n jobs_env: Optional[str] = ...,\n jobs_env_supports_jobserver: bool = ...,\n fail_on_error: bool = ...,\n ignore_errors: Union[int, Sequence[int]] = ...,\n ignore_quotes: Optional[bool] = ...,\n timeout: Optional[int] = ...,\n env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ...,\n extra_env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ...,\n input: Optional[BinaryIO] = ...,\n output: Union[Optional[BinaryIO], str] = ...,\n error: Union[Optional[BinaryIO], str] = ...,\n _dump_env: Optional[Dict[str, str]] = ...,\n ) -> None: ...\n\n @overload\n def __call__(\n self,\n *args: str,\n parallel: bool = ...,\n jobs_env: Optional[str] = ...,\n jobs_env_supports_jobserver: bool = ...,\n fail_on_error: bool = ...,\n ignore_errors: Union[int, Sequence[int]] = ...,\n ignore_quotes: Optional[bool] = ...,\n timeout: Optional[int] = ...,\n env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ...,\n extra_env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ...,\n input: Optional[BinaryIO] = ...,\n output: Union[Type[str], Callable] = ...,\n error: spack.util.executable.OutType = ...,\n _dump_env: Optional[Dict[str, str]] = ...,\n ) -> str: ...\n\n @overload\n def __call__(\n self,\n *args: str,\n parallel: bool = ...,\n jobs_env: Optional[str] = ...,\n jobs_env_supports_jobserver: bool = ...,\n fail_on_error: bool = ...,\n ignore_errors: Union[int, Sequence[int]] = ...,\n ignore_quotes: Optional[bool] = ...,\n timeout: Optional[int] = ...,\n env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ...,\n extra_env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ...,\n input: Optional[BinaryIO] = ...,\n output: spack.util.executable.OutType = ...,\n error: Union[Type[str], Callable] = ...,\n _dump_env: Optional[Dict[str, str]] = ...,\n ) -> str: ...\n\n def __call__(\n self,\n *args: str,\n parallel: bool = True,\n jobs_env: Optional[str] = None,\n jobs_env_supports_jobserver: bool = False,\n **kwargs,\n ) -> Optional[str]:\n \"\"\"Runs this ``make`` executable in a subprocess.\n\n Args:\n parallel: if False, parallelism is disabled\n jobs_env: environment variable that will be set to the current level of parallelism\n jobs_env_supports_jobserver: whether the jobs env supports a job server\n\n For all the other ``**kwargs``, refer to :func:`spack.util.executable.Executable.__call__`.\n \"\"\"\n jobs = get_effective_jobs(\n self.jobs, parallel=parallel, supports_jobserver=self.supports_jobserver\n )\n if jobs is not None:\n args = (f\"-j{jobs}\",) + args\n\n if jobs_env:\n # Caller wants us to set an environment variable to control the parallelism\n jobs_env_jobs = get_effective_jobs(\n self.jobs, parallel=parallel, supports_jobserver=jobs_env_supports_jobserver\n )\n if jobs_env_jobs is not None:\n extra_env = kwargs.setdefault(\"extra_env\", {})\n extra_env.update({jobs_env: str(jobs_env_jobs)})\n\n return super().__call__(*args, **kwargs)\n\n\nclass UndeclaredDependencyError(spack.error.SpackError):\n \"\"\"Raised if a dependency is invoking an executable through a module global, without\n declaring a dependency on it.\n \"\"\"\n\n\nclass DeprecatedExecutable:\n def __init__(self, pkg: str, exe: str, exe_pkg: str) -> None:\n self.pkg = pkg\n self.exe = exe\n self.exe_pkg = exe_pkg\n\n def __call__(self, *args, **kwargs):\n raise UndeclaredDependencyError(\n f\"{self.pkg} is using {self.exe} without declaring a dependency on {self.exe_pkg}\"\n )\n\n def add_default_env(self, key: str, value: str):\n self.__call__()\n\n\ndef clean_environment():\n # Stuff in here sanitizes the build environment to eliminate\n # anything the user has set that may interfere. We apply it immediately\n # unlike the other functions so it doesn't overwrite what the modules load.\n env = EnvironmentModifications()\n\n # Remove these vars from the environment during build because they\n # can affect how some packages find libraries. We want to make\n # sure that builds never pull in unintended external dependencies.\n env.unset(\"LD_LIBRARY_PATH\")\n env.unset(\"LD_RUN_PATH\")\n env.unset(\"DYLD_LIBRARY_PATH\")\n env.unset(\"DYLD_FALLBACK_LIBRARY_PATH\")\n\n # These vars affect how the compiler finds libraries and include dirs.\n env.unset(\"LIBRARY_PATH\")\n env.unset(\"CPATH\")\n env.unset(\"C_INCLUDE_PATH\")\n env.unset(\"CPLUS_INCLUDE_PATH\")\n env.unset(\"OBJC_INCLUDE_PATH\")\n\n # prevent configure scripts from sourcing variables from config site file (AC_SITE_LOAD).\n env.set(\"CONFIG_SITE\", os.devnull)\n env.unset(\"CMAKE_PREFIX_PATH\")\n\n env.unset(\"PYTHONPATH\")\n env.unset(\"R_HOME\")\n env.unset(\"R_ENVIRON\")\n env.unset(\"LUA_PATH\")\n env.unset(\"LUA_CPATH\")\n\n # Affects GNU make, can e.g. indirectly inhibit enabling parallel build\n # env.unset('MAKEFLAGS')\n\n # Avoid that libraries of build dependencies get hijacked.\n env.unset(\"LD_PRELOAD\")\n env.unset(\"DYLD_INSERT_LIBRARIES\")\n\n # Avoid _ROOT user variables overriding spack dependencies\n # https://cmake.org/cmake/help/latest/variable/PackageName_ROOT.html\n # Spack needs SPACK_ROOT though, so we need to exclude that\n for varname in os.environ.keys():\n if varname.endswith(\"_ROOT\") and varname != \"SPACK_ROOT\":\n env.unset(varname)\n\n # Unset the following variables because they can affect installation of\n # Autotools and CMake packages.\n build_system_vars = [\n \"CC\",\n \"CFLAGS\",\n \"CPP\",\n \"CPPFLAGS\", # C variables\n \"CXX\",\n \"CCC\",\n \"CXXFLAGS\",\n \"CXXCPP\", # C++ variables\n \"F77\",\n \"FFLAGS\",\n \"FLIBS\", # Fortran77 variables\n \"FC\",\n \"FCFLAGS\",\n \"FCLIBS\", # Fortran variables\n \"LDFLAGS\",\n \"LIBS\", # linker variables\n ]\n for v in build_system_vars:\n env.unset(v)\n\n # Unset mpi environment vars. These flags should only be set by\n # mpi providers for packages with mpi dependencies\n mpi_vars = [\"MPICC\", \"MPICXX\", \"MPIFC\", \"MPIF77\", \"MPIF90\"]\n for v in mpi_vars:\n env.unset(v)\n\n build_lang = spack.config.get(\"config:build_language\")\n if build_lang:\n # Override language-related variables. This can be used to force\n # English compiler messages etc., which allows parse_log_events to\n # show useful matches.\n env.set(\"LC_ALL\", build_lang)\n\n # Remove any macports installs from the PATH. The macports ld can\n # cause conflicts with the built-in linker on el capitan. Solves\n # assembler issues, e.g.:\n # suffix or operands invalid for `movq'\"\n path = get_path(\"PATH\")\n for p in path:\n if \"/macports/\" in p:\n env.remove_path(\"PATH\", p)\n\n return env\n\n\ndef _add_werror_handling(keep_werror, env):\n keep_flags = set()\n # set of pairs\n replace_flags: List[Tuple[str, str]] = []\n if keep_werror == \"all\":\n keep_flags.add(\"-Werror*\")\n else:\n if keep_werror == \"specific\":\n keep_flags.add(\"-Werror-*\")\n keep_flags.add(\"-Werror=*\")\n # This extra case is to handle -Werror-implicit-function-declaration\n replace_flags.append((\"-Werror-\", \"-Wno-error=\"))\n replace_flags.append((\"-Werror\", \"-Wno-error\"))\n env.set(\"SPACK_COMPILER_FLAGS_KEEP\", \"|\".join(keep_flags))\n env.set(\"SPACK_COMPILER_FLAGS_REPLACE\", \" \".join([\"|\".join(item) for item in replace_flags]))\n\n\ndef set_wrapper_environment_variables_for_flags(pkg, env):\n assert pkg.spec.concrete\n spec = pkg.spec\n\n if pkg.keep_werror is not None:\n keep_werror = pkg.keep_werror\n else:\n keep_werror = spack.config.get(\"config:flags:keep_werror\")\n\n _add_werror_handling(keep_werror, env)\n\n # Trap spack-tracked compiler flags as appropriate.\n # env_flags are easy to accidentally override.\n inject_flags = {}\n env_flags = {}\n build_system_flags = {}\n for flag in spack.spec.FlagMap.valid_compiler_flags():\n # Always convert flag_handler to function type.\n # This avoids discrepancies in calling conventions between functions\n # and methods, or between bound and unbound methods in python 2.\n # We cannot effectively convert everything to a bound method, which\n # would be the simpler solution.\n if isinstance(pkg.flag_handler, types.FunctionType):\n handler = pkg.flag_handler\n else:\n handler = pkg.flag_handler.__func__\n\n injf, envf, bsf = handler(pkg, flag, spec.compiler_flags[flag][:])\n inject_flags[flag] = injf or []\n env_flags[flag] = envf or []\n build_system_flags[flag] = bsf or []\n\n # Place compiler flags as specified by flag_handler\n for flag in spack.spec.FlagMap.valid_compiler_flags():\n # Concreteness guarantees key safety here\n if inject_flags[flag]:\n # variables SPACK_ inject flags through wrapper\n var_name = \"SPACK_{0}\".format(flag.upper())\n env.set(var_name, \" \".join(f for f in inject_flags[flag]))\n if env_flags[flag]:\n # implicit variables\n env.set(flag.upper(), \" \".join(f for f in env_flags[flag]))\n pkg.flags_to_build_system_args(build_system_flags)\n env.set(\"SPACK_SYSTEM_DIRS\", SYSTEM_DIR_CASE_ENTRY)\n return env\n\n\ndef optimization_flags(compiler, target):\n # Try to check if the current compiler comes with a version number or\n # has an unexpected suffix. If so, treat it as a compiler with a\n # custom spec.\n version_number, _ = spack.vendor.archspec.cpu.version_components(\n compiler.version.dotted_numeric_string\n )\n try:\n result = target.optimization_flags(compiler.name, version_number)\n except (ValueError, spack.vendor.archspec.cpu.UnsupportedMicroarchitecture):\n result = \"\"\n\n return result\n\n\ndef set_wrapper_variables(pkg, env):\n \"\"\"Set environment variables used by the Spack compiler wrapper (which have the prefix\n ``SPACK_``) and also add the compiler wrappers to PATH.\n\n This determines the injected -L/-I/-rpath options; each of these specifies a search order and\n this function computes these options in a manner that is intended to match the DAG traversal\n order in ``SetupContext``. TODO: this is not the case yet, we're using post order,\n ``SetupContext`` is using topo order.\"\"\"\n # Set compiler flags injected from the spec\n set_wrapper_environment_variables_for_flags(pkg, env)\n\n # Working directory for the spack command itself, for debug logs.\n if spack.config.get(\"config:debug\"):\n env.set(SPACK_DEBUG, \"TRUE\")\n env.set(SPACK_SHORT_SPEC, pkg.spec.short_spec)\n env.set(SPACK_DEBUG_LOG_ID, pkg.spec.format(\"{name}-{hash:7}\"))\n env.set(SPACK_DEBUG_LOG_DIR, spack.paths.spack_working_dir)\n\n if spack.config.get(\"config:ccache\"):\n # Enable ccache in the compiler wrapper\n env.set(SPACK_CCACHE_BINARY, spack.util.executable.which_string(\"ccache\", required=True))\n else:\n # Avoid cache pollution if a build system forces `ccache `.\n env.set(\"CCACHE_DISABLE\", \"1\")\n\n # Gather information about various types of dependencies\n rpath_hashes = set(s.dag_hash() for s in get_rpath_deps(pkg))\n link_deps = pkg.spec.traverse(root=False, order=\"topo\", deptype=dt.LINK)\n external_link_deps, nonexternal_link_deps = stable_partition(link_deps, lambda d: d.external)\n\n link_dirs = []\n include_dirs = []\n rpath_dirs = []\n\n for dep in chain(external_link_deps, nonexternal_link_deps):\n # TODO: is_system_path is wrong, but even if we knew default -L, -I flags from the compiler\n # and default search dirs from the dynamic linker, it's not obvious how to avoid a possibly\n # expensive search in `query.libs.directories` and `query.headers.directories`, which is\n # what this branch is trying to avoid.\n if is_system_path(dep.prefix):\n continue\n # TODO: as of Spack 0.22, multiple instances of the same package may occur among the link\n # deps, so keying by name is wrong. In practice it is not problematic: we obtain the same\n # gcc-runtime / glibc here, and repeatedly add the same dirs that are later deduped.\n query = pkg.spec[dep.name]\n dep_link_dirs = []\n try:\n # Locating libraries can be time consuming, so log start and finish.\n tty.debug(f\"Collecting libraries for {dep.name}\")\n dep_link_dirs.extend(query.libs.directories)\n tty.debug(f\"Libraries for {dep.name} have been collected.\")\n except NoLibrariesError:\n tty.debug(f\"No libraries found for {dep.name}\")\n\n for default_lib_dir in (\"lib\", \"lib64\"):\n default_lib_prefix = os.path.join(dep.prefix, default_lib_dir)\n if os.path.isdir(default_lib_prefix):\n dep_link_dirs.append(default_lib_prefix)\n\n link_dirs[:0] = dep_link_dirs\n if dep.dag_hash() in rpath_hashes:\n rpath_dirs[:0] = dep_link_dirs\n\n try:\n tty.debug(f\"Collecting headers for {dep.name}\")\n include_dirs[:0] = query.headers.directories\n tty.debug(f\"Headers for {dep.name} have been collected.\")\n except NoHeadersError:\n tty.debug(f\"No headers found for {dep.name}\")\n\n # The top-level package is heuristically rpath'ed.\n for libdir in (\"lib64\", \"lib\"):\n lib_path = os.path.join(pkg.prefix, libdir)\n rpath_dirs.insert(0, lib_path)\n\n # TODO: filter_system_paths is again wrong (and probably unnecessary due to the is_system_path\n # branch above). link_dirs should be filtered with entries from _parse_link_paths.\n link_dirs = list(dedupe(filter_system_paths(link_dirs)))\n include_dirs = list(dedupe(filter_system_paths(include_dirs)))\n rpath_dirs = list(dedupe(filter_system_paths(rpath_dirs)))\n\n default_dynamic_linker_filter = spack.compilers.libraries.dynamic_linker_filter_for(pkg.spec)\n if default_dynamic_linker_filter:\n rpath_dirs = default_dynamic_linker_filter(rpath_dirs)\n\n # Spack managed directories include the stage, store and upstream stores. We extend this with\n # their real paths to make it more robust (e.g. /tmp vs /private/tmp on macOS).\n spack_managed_dirs: Set[str] = {\n spack.stage.get_stage_root(),\n spack.store.STORE.db.root,\n *(db.root for db in spack.store.STORE.db.upstream_dbs),\n }\n spack_managed_dirs.update([os.path.realpath(p) for p in spack_managed_dirs])\n\n env.set(SPACK_MANAGED_DIRS, \"|\".join(f'\"{p}/\"*' for p in sorted(spack_managed_dirs)))\n is_spack_managed = lambda p: any(p.startswith(store) for store in spack_managed_dirs)\n link_dirs_spack, link_dirs_system = stable_partition(link_dirs, is_spack_managed)\n include_dirs_spack, include_dirs_system = stable_partition(include_dirs, is_spack_managed)\n rpath_dirs_spack, rpath_dirs_system = stable_partition(rpath_dirs, is_spack_managed)\n env.set(SPACK_LINK_DIRS, \":\".join(link_dirs_system))\n env.set(SPACK_INCLUDE_DIRS, \":\".join(include_dirs_system))\n env.set(SPACK_RPATH_DIRS, \":\".join(rpath_dirs_system))\n env.set(SPACK_STORE_LINK_DIRS, \":\".join(link_dirs_spack))\n env.set(SPACK_STORE_INCLUDE_DIRS, \":\".join(include_dirs_spack))\n env.set(SPACK_STORE_RPATH_DIRS, \":\".join(rpath_dirs_spack))\n\n\ndef set_package_py_globals(pkg, context: Context = Context.BUILD):\n \"\"\"Populate the Python module of a package with some useful global names.\n This makes things easier for package writers.\n \"\"\"\n module = ModuleChangePropagator(pkg)\n\n jobs = spack.config.determine_number_of_jobs(parallel=pkg.parallel)\n module.make_jobs = jobs\n\n module.make = DeprecatedExecutable(pkg.name, \"make\", \"gmake\")\n module.gmake = DeprecatedExecutable(pkg.name, \"gmake\", \"gmake\")\n module.ninja = DeprecatedExecutable(pkg.name, \"ninja\", \"ninja\")\n\n if sys.platform == \"win32\":\n module.nmake = DeprecatedExecutable(pkg.name, \"nmake\", \"msvc\")\n module.msbuild = DeprecatedExecutable(pkg.name, \"msbuild\", \"msvc\")\n # analog to configure for win32\n module.cscript = Executable(\"cscript\")\n\n # Find the configure script in the archive path\n # Don't use which for this; we want to find it in the current dir.\n module.configure = Executable(\"./configure\")\n\n # Useful directories within the prefix are encapsulated in\n # a Prefix object.\n module.prefix = pkg.prefix\n\n # Platform-specific library suffix.\n module.dso_suffix = dso_suffix\n\n def static_to_shared_library(static_lib, shared_lib=None, **kwargs):\n compiler_path = kwargs.get(\"compiler\", module.spack_cc)\n compiler = Executable(compiler_path)\n\n return _static_to_shared_library(\n pkg.spec.architecture, compiler, static_lib, shared_lib, **kwargs\n )\n\n module.static_to_shared_library = static_to_shared_library\n\n module.propagate_changes_to_mro()\n\n\ndef _static_to_shared_library(arch, compiler, static_lib, shared_lib=None, **kwargs):\n \"\"\"\n Converts a static library to a shared library. The static library has to\n be built with PIC for the conversion to work.\n\n Parameters:\n static_lib (str): Path to the static library.\n shared_lib (str): Path to the shared library. Default is to derive\n from the static library's path.\n\n Keyword arguments:\n compiler (str): Path to the compiler. Default is spack_cc.\n compiler_output: Where to print compiler output to.\n arguments (str list): Additional arguments for the compiler.\n version (str): Library version. Default is unspecified.\n compat_version (str): Library compatibility version. Default is\n version.\n \"\"\"\n compiler_output = kwargs.get(\"compiler_output\", None)\n arguments = kwargs.get(\"arguments\", [])\n version = kwargs.get(\"version\", None)\n compat_version = kwargs.get(\"compat_version\", version)\n\n if not shared_lib:\n shared_lib = \"{0}.{1}\".format(os.path.splitext(static_lib)[0], dso_suffix)\n\n compiler_args = []\n\n # TODO: Compiler arguments should not be hardcoded but provided by\n # the different compiler classes.\n if \"linux\" in arch or \"cray\" in arch:\n soname = os.path.basename(shared_lib)\n\n if compat_version:\n soname += \".{0}\".format(compat_version)\n\n compiler_args = [\n \"-shared\",\n \"-Wl,-soname,{0}\".format(soname),\n \"-Wl,--whole-archive\",\n static_lib,\n \"-Wl,--no-whole-archive\",\n ]\n elif \"darwin\" in arch:\n install_name = shared_lib\n\n if compat_version:\n install_name += \".{0}\".format(compat_version)\n\n compiler_args = [\n \"-dynamiclib\",\n \"-install_name\",\n \"{0}\".format(install_name),\n \"-Wl,-force_load,{0}\".format(static_lib),\n ]\n\n if compat_version:\n compiler_args.extend([\"-compatibility_version\", \"{0}\".format(compat_version)])\n\n if version:\n compiler_args.extend([\"-current_version\", \"{0}\".format(version)])\n\n if len(arguments) > 0:\n compiler_args.extend(arguments)\n\n shared_lib_base = shared_lib\n\n if version:\n shared_lib += \".{0}\".format(version)\n elif compat_version:\n shared_lib += \".{0}\".format(compat_version)\n\n compiler_args.extend([\"-o\", shared_lib])\n\n # Create symlinks for version and compat_version\n shared_lib_link = os.path.basename(shared_lib)\n\n if version or compat_version:\n symlink(shared_lib_link, shared_lib_base)\n\n if compat_version and compat_version != version:\n symlink(shared_lib_link, \"{0}.{1}\".format(shared_lib_base, compat_version))\n\n return compiler(*compiler_args, output=compiler_output)\n\n\ndef _get_rpath_deps_from_spec(\n spec: spack.spec.Spec, transitive_rpaths: bool\n) -> List[spack.spec.Spec]:\n if not transitive_rpaths:\n return spec.dependencies(deptype=dt.LINK)\n\n by_name: Dict[str, spack.spec.Spec] = {}\n\n for dep in spec.traverse(root=False, deptype=dt.LINK):\n lookup = by_name.get(dep.name)\n if lookup is None:\n by_name[dep.name] = dep\n elif lookup.version < dep.version:\n by_name[dep.name] = dep\n\n return list(by_name.values())\n\n\ndef get_rpath_deps(pkg: spack.package_base.PackageBase) -> List[spack.spec.Spec]:\n \"\"\"Return immediate or transitive dependencies (depending on the package) that need to be\n rpath'ed. If a package occurs multiple times, the newest version is kept.\"\"\"\n return _get_rpath_deps_from_spec(pkg.spec, pkg.transitive_rpaths)\n\n\ndef get_cmake_prefix_path(pkg: spack.package_base.PackageBase) -> List[str]:\n \"\"\"Obtain the ``CMAKE_PREFIX_PATH`` entries for a package, based on the\n :attr:`~spack.package_base.PackageBase.cmake_prefix_paths` package attribute of direct\n build/test and transitive link dependencies.\"\"\"\n edges = traverse.traverse_topo_edges_generator(\n traverse.with_artificial_edges([pkg.spec]),\n visitor=traverse.MixedDepthVisitor(\n direct=dt.BUILD | dt.TEST, transitive=dt.LINK, key=traverse.by_dag_hash\n ),\n key=traverse.by_dag_hash,\n root=False,\n all_edges=False, # cover all nodes, not all edges\n )\n ordered_specs = [edge.spec for edge in edges]\n # Separate out externals so they do not shadow Spack prefixes\n externals, spack_built = stable_partition((s for s in ordered_specs), lambda x: x.external)\n\n return filter_system_paths(\n path for spec in chain(spack_built, externals) for path in spec.package.cmake_prefix_paths\n )\n\n\ndef setup_package(pkg, dirty, context: Context = Context.BUILD):\n \"\"\"Execute all environment setup routines.\"\"\"\n if context not in (Context.BUILD, Context.TEST):\n raise ValueError(f\"'context' must be Context.BUILD or Context.TEST - got {context}\")\n\n # First populate the package.py's module with the relevant globals that could be used in any\n # of the setup_* functions.\n setup_context = SetupContext(pkg.spec, context=context)\n setup_context.set_all_package_py_globals()\n\n # Keep track of env changes from packages separately, since we want to\n # issue warnings when packages make \"suspicious\" modifications.\n env_base = EnvironmentModifications() if dirty else clean_environment()\n env_mods = EnvironmentModifications()\n\n # setup compilers for build contexts\n need_compiler = context == Context.BUILD or (\n context == Context.TEST and pkg.test_requires_compiler\n )\n if need_compiler:\n set_wrapper_variables(pkg, env_mods)\n\n # Platform specific setup goes before package specific setup. This is for setting\n # defaults like MACOSX_DEPLOYMENT_TARGET on macOS.\n platform = spack.platforms.by_name(pkg.spec.architecture.platform)\n platform.setup_platform_environment(pkg, env_mods)\n\n tty.debug(\"setup_package: grabbing modifications from dependencies\")\n env_mods.extend(setup_context.get_env_modifications())\n tty.debug(\"setup_package: collected all modifications from dependencies\")\n\n tty.debug(\"setup_package: adding compiler wrappers paths\")\n env_by_name = env_mods.group_by_name()\n for x in env_by_name[\"SPACK_COMPILER_WRAPPER_PATH\"]:\n assert isinstance(x, PrependPath), (\n \"unexpected setting used for SPACK_COMPILER_WRAPPER_PATH\"\n )\n env_mods.prepend_path(\"PATH\", x.value)\n\n # Check whether we want to force RPATH or RUNPATH\n enable_var_name, disable_var_name = \"SPACK_ENABLE_NEW_DTAGS\", \"SPACK_DISABLE_NEW_DTAGS\"\n if enable_var_name in env_by_name and disable_var_name in env_by_name:\n enable_new_dtags = _extract_dtags_arg(env_by_name, var_name=enable_var_name)\n disable_new_dtags = _extract_dtags_arg(env_by_name, var_name=disable_var_name)\n if spack.config.CONFIG.get(\"config:shared_linking:type\") == \"rpath\":\n env_mods.set(\"SPACK_DTAGS_TO_STRIP\", enable_new_dtags)\n env_mods.set(\"SPACK_DTAGS_TO_ADD\", disable_new_dtags)\n else:\n env_mods.set(\"SPACK_DTAGS_TO_STRIP\", disable_new_dtags)\n env_mods.set(\"SPACK_DTAGS_TO_ADD\", enable_new_dtags)\n\n if context == Context.TEST:\n env_mods.prepend_path(\"PATH\", \".\")\n elif context == Context.BUILD and not dirty and not env_mods.is_unset(\"CPATH\"):\n tty.debug(\n \"A dependency has updated CPATH, this may lead pkg-config to assume that the package \"\n \"is part of the system includes and omit it when invoked with '--cflags'.\"\n )\n\n # First apply the clean environment changes\n env_base.apply_modifications()\n\n # Load modules on an already clean environment, just before applying Spack's\n # own environment modifications. This ensures Spack controls CC/CXX/... variables.\n load_external_modules(setup_context)\n\n # Make sure nothing's strange about the Spack environment.\n validate(env_mods, tty.warn)\n env_mods.apply_modifications()\n\n # Return all env modifications we controlled (excluding module related ones)\n env_base.extend(env_mods)\n return env_base\n\n\ndef _extract_dtags_arg(env_by_name: Dict[str, ModificationList], *, var_name: str) -> str:\n try:\n enable_new_dtags = env_by_name[var_name][0].value # type: ignore[union-attr]\n except (KeyError, IndexError, AttributeError):\n enable_new_dtags = \"\"\n return enable_new_dtags\n\n\nclass EnvironmentVisitor:\n def __init__(self, *roots: spack.spec.Spec, context: Context):\n # For the roots (well, marked specs) we follow different edges\n # than for their deps, depending on the context.\n self.root_hashes = set(s.dag_hash() for s in roots)\n\n if context == Context.BUILD:\n # Drop direct run deps in build context\n # We don't really distinguish between install and build time test deps,\n # so we include them here as build-time test deps.\n self.root_depflag = dt.BUILD | dt.TEST | dt.LINK\n elif context == Context.TEST:\n # This is more of an extended run environment\n self.root_depflag = dt.TEST | dt.RUN | dt.LINK\n elif context == Context.RUN:\n self.root_depflag = dt.RUN | dt.LINK\n\n def accept(self, item):\n return True\n\n def neighbors(self, item):\n spec = item.edge.spec\n if spec.dag_hash() in self.root_hashes:\n depflag = self.root_depflag\n else:\n depflag = dt.LINK | dt.RUN\n return traverse.sort_edges(spec.edges_to_dependencies(depflag=depflag))\n\n\nclass UseMode(Flag):\n #: Entrypoint spec (a spec to be built; an env root, etc)\n ROOT = auto()\n\n #: A spec used at runtime, but no executables in PATH\n RUNTIME = auto()\n\n #: A spec used at runtime, with executables in PATH\n RUNTIME_EXECUTABLE = auto()\n\n #: A spec that's a direct build or test dep\n BUILDTIME_DIRECT = auto()\n\n #: A spec that should be visible in search paths in a build env.\n BUILDTIME = auto()\n\n #: Flag is set when the (node, mode) is finalized\n ADDED = auto()\n\n\ndef effective_deptypes(\n *specs: spack.spec.Spec, context: Context = Context.BUILD\n) -> List[Tuple[spack.spec.Spec, UseMode]]:\n \"\"\"Given a list of input specs and a context, return a list of tuples of\n all specs that contribute to (environment) modifications, together with\n a flag specifying in what way they do so. The list is ordered topologically\n from root to leaf, meaning that environment modifications should be applied\n in reverse so that dependents override dependencies, not the other way around.\"\"\"\n topo_sorted_edges = traverse.traverse_topo_edges_generator(\n traverse.with_artificial_edges(specs),\n visitor=traverse.CoverEdgesVisitor(\n EnvironmentVisitor(*specs, context=context), key=traverse.by_dag_hash\n ),\n key=traverse.by_dag_hash,\n root=True,\n all_edges=True,\n )\n\n # Dictionary with \"no mode\" as default value, so it's easy to write modes[x] |= flag.\n use_modes = defaultdict(lambda: UseMode(0))\n nodes_with_type = []\n\n for edge in topo_sorted_edges:\n parent, child, depflag = edge.parent, edge.spec, edge.depflag\n\n # Mark the starting point\n if parent is None:\n use_modes[child] = UseMode.ROOT\n continue\n\n parent_mode = use_modes[parent]\n\n # Nothing to propagate.\n if not parent_mode:\n continue\n\n # Depending on the context, include particular deps from the root.\n if UseMode.ROOT & parent_mode:\n if context == Context.BUILD:\n if (dt.BUILD | dt.TEST) & depflag:\n use_modes[child] |= UseMode.BUILDTIME_DIRECT\n if dt.LINK & depflag:\n use_modes[child] |= UseMode.BUILDTIME\n\n elif context == Context.TEST:\n if (dt.RUN | dt.TEST) & depflag:\n use_modes[child] |= UseMode.RUNTIME_EXECUTABLE\n elif dt.LINK & depflag:\n use_modes[child] |= UseMode.RUNTIME\n\n elif context == Context.RUN:\n if dt.RUN & depflag:\n use_modes[child] |= UseMode.RUNTIME_EXECUTABLE\n elif dt.LINK & depflag:\n use_modes[child] |= UseMode.RUNTIME\n\n # Propagate RUNTIME and RUNTIME_EXECUTABLE through link and run deps.\n if (UseMode.RUNTIME | UseMode.RUNTIME_EXECUTABLE | UseMode.BUILDTIME_DIRECT) & parent_mode:\n if dt.LINK & depflag:\n use_modes[child] |= UseMode.RUNTIME\n if dt.RUN & depflag:\n use_modes[child] |= UseMode.RUNTIME_EXECUTABLE\n\n # Propagate BUILDTIME through link deps.\n if UseMode.BUILDTIME & parent_mode:\n if dt.LINK & depflag:\n use_modes[child] |= UseMode.BUILDTIME\n\n # Finalize the spec; the invariant is that all in-edges are processed\n # before out-edges, meaning that parent is done.\n if not (UseMode.ADDED & parent_mode):\n use_modes[parent] |= UseMode.ADDED\n nodes_with_type.append((parent, parent_mode))\n\n # Attach the leaf nodes, since we only added nodes with out-edges.\n for spec, parent_mode in use_modes.items():\n if parent_mode and not (UseMode.ADDED & parent_mode):\n nodes_with_type.append((spec, parent_mode))\n\n return nodes_with_type\n\n\nclass SetupContext:\n \"\"\"This class encapsulates the logic to determine environment modifications, and is used as\n well to set globals in modules of package.py.\"\"\"\n\n def __init__(self, *specs: spack.spec.Spec, context: Context) -> None:\n \"\"\"Construct a ModificationsFromDag object.\n Args:\n specs: single root spec for build/test context, possibly more for run context\n context: build, run, or test\"\"\"\n if (context == Context.BUILD or context == Context.TEST) and not len(specs) == 1:\n raise ValueError(\"Cannot setup build environment for multiple specs\")\n specs_with_type = effective_deptypes(*specs, context=context)\n\n self.specs = specs\n self.context = context\n self.external: List[Tuple[spack.spec.Spec, UseMode]]\n self.nonexternal: List[Tuple[spack.spec.Spec, UseMode]]\n # Reverse so we go from leaf to root\n self.nodes_in_subdag = set(id(s) for s, _ in specs_with_type)\n\n # Split into non-external and external, maintaining topo order per group.\n self.external, self.nonexternal = stable_partition(\n reversed(specs_with_type), lambda t: t[0].external\n )\n self.should_be_runnable = UseMode.BUILDTIME_DIRECT | UseMode.RUNTIME_EXECUTABLE\n self.should_setup_run_env = (\n UseMode.BUILDTIME_DIRECT | UseMode.RUNTIME | UseMode.RUNTIME_EXECUTABLE\n )\n self.should_setup_dependent_build_env = UseMode.BUILDTIME | UseMode.BUILDTIME_DIRECT\n self.should_setup_build_env = UseMode.ROOT if context == Context.BUILD else UseMode(0)\n\n if context == Context.RUN or context == Context.TEST:\n self.should_be_runnable |= UseMode.ROOT\n self.should_setup_run_env |= UseMode.ROOT\n\n # Everything that calls setup_run_environment and setup_dependent_* needs globals set.\n self.should_set_package_py_globals = (\n self.should_setup_dependent_build_env | self.should_setup_run_env | UseMode.ROOT\n )\n # In a build context, the root needs build-specific globals set.\n self.needs_build_context = UseMode.ROOT\n\n def set_all_package_py_globals(self):\n \"\"\"Set the globals in modules of package.py files.\"\"\"\n for dspec, flag in chain(self.external, self.nonexternal):\n pkg = dspec.package\n\n if self.should_set_package_py_globals & flag:\n if self.context == Context.BUILD and self.needs_build_context & flag:\n set_package_py_globals(pkg, context=Context.BUILD)\n else:\n # This includes runtime dependencies, also runtime deps of direct build deps.\n set_package_py_globals(pkg, context=Context.RUN)\n\n # Looping over the set of packages a second time\n # ensures all globals are loaded into the module space prior to\n # any package setup. This guarantees package setup methods have\n # access to expected module level definitions such as \"spack_cc\"\n for dspec, flag in chain(self.external, self.nonexternal):\n pkg = dspec.package\n for spec in dspec.dependents():\n # Note: some specs have dependents that are unreachable from the root, so avoid\n # setting globals for those.\n if id(spec) not in self.nodes_in_subdag:\n continue\n dependent_module = ModuleChangePropagator(spec.package)\n pkg.setup_dependent_package(dependent_module, spec)\n dependent_module.propagate_changes_to_mro()\n\n def get_env_modifications(self) -> EnvironmentModifications:\n \"\"\"Returns the environment variable modifications for the given input specs and context.\n Environment modifications include:\n - Updating PATH for packages that are required at runtime\n - Updating CMAKE_PREFIX_PATH and PKG_CONFIG_PATH so that their respective\n tools can find Spack-built dependencies (when context=build)\n - Running custom package environment modifications: setup_run_environment,\n setup_dependent_run_environment, setup_build_environment,\n setup_dependent_build_environment.\n\n The (partial) order imposed on the specs is externals first, then topological\n from leaf to root. That way externals cannot contribute search paths that would shadow\n Spack's prefixes, and dependents override variables set by dependencies.\"\"\"\n env = EnvironmentModifications()\n for dspec, flag in chain(self.external, self.nonexternal):\n tty.debug(f\"Adding env modifications for {dspec.name}\")\n pkg = dspec.package\n\n if self.should_setup_dependent_build_env & flag:\n self._make_buildtime_detectable(dspec, env)\n\n for root in self.specs: # there is only one root in build context\n spack.builder.create(pkg).setup_dependent_build_environment(env, root)\n\n if self.should_setup_build_env & flag:\n spack.builder.create(pkg).setup_build_environment(env)\n\n if self.should_be_runnable & flag:\n self._make_runnable(dspec, env)\n\n if self.should_setup_run_env & flag:\n run_env_mods = EnvironmentModifications()\n for spec in dspec.dependents(deptype=dt.LINK | dt.RUN):\n if id(spec) in self.nodes_in_subdag:\n pkg.setup_dependent_run_environment(run_env_mods, spec)\n pkg.setup_run_environment(run_env_mods)\n\n external_env = (dspec.extra_attributes or {}).get(\"environment\", {})\n if external_env:\n run_env_mods.extend(spack.schema.environment.parse(external_env))\n\n if self.context == Context.BUILD:\n # Don't let the runtime environment of compiler like dependencies leak into the\n # build env\n run_env_mods.drop(\"CC\", \"CXX\", \"F77\", \"FC\")\n env.extend(run_env_mods)\n\n return env\n\n def _make_buildtime_detectable(self, dep: spack.spec.Spec, env: EnvironmentModifications):\n if is_system_path(dep.prefix):\n return\n\n env.prepend_path(\"CMAKE_PREFIX_PATH\", dep.prefix)\n for d in (\"lib\", \"lib64\", \"share\"):\n pcdir = os.path.join(dep.prefix, d, \"pkgconfig\")\n if os.path.isdir(pcdir):\n env.prepend_path(\"PKG_CONFIG_PATH\", pcdir)\n\n def _make_runnable(self, dep: spack.spec.Spec, env: EnvironmentModifications):\n if is_system_path(dep.prefix):\n return\n\n for d in (\"bin\", \"bin64\"):\n bin_dir = os.path.join(dep.prefix, d)\n if os.path.isdir(bin_dir):\n env.prepend_path(\"PATH\", bin_dir)\n\n\ndef load_external_modules(context: SetupContext) -> None:\n \"\"\"Traverse a package's spec DAG and load any external modules.\n\n Traverse a package's dependencies and load any external modules\n associated with them.\n\n Args:\n context: A populated SetupContext object\n \"\"\"\n for spec, _ in context.external:\n external_modules = spec.external_modules or []\n for external_module in external_modules:\n spack.util.module_cmd.load_module(external_module)\n\n\ndef _setup_pkg_and_run(\n serialized_pkg: \"spack.subprocess_context.PackageInstallContext\",\n function: Callable,\n kwargs: Dict,\n write_pipe: Connection,\n input_pipe: Optional[Connection],\n jsfd1: Optional[Connection],\n jsfd2: Optional[Connection],\n stdout_pipe: Optional[Connection] = None,\n stderr_pipe: Optional[Connection] = None,\n):\n \"\"\"Main entry point in the child process for Spack builds.\n\n ``_setup_pkg_and_run`` is called by the child process created in\n ``start_build_process()``, and its main job is to run ``function()`` on behalf of\n some Spack installation (see :ref:`spack.installer.PackageInstaller._complete_task`).\n\n The child process is passed a ``write_pipe``, on which it's expected to send one of\n the following:\n\n * ``StopPhase``: error raised by a build process indicating it's stopping at a\n particular build phase.\n\n * ``BaseException``: any exception raised by a child build process, which will be\n wrapped in ``ChildError`` (which adds a bunch of debug info and log context) and\n raised in the parent.\n\n * The return value of ``function()``, which can be anything (except an exception).\n This is returned to the caller.\n\n Note: ``jsfd1`` and ``jsfd2`` are passed solely to ensure that the child process\n does not close these file descriptors. Some ``multiprocessing`` backends will close\n them automatically in the child if they are not passed at process creation time.\n\n Arguments:\n serialized_pkg: Spack package install context object (serialized form of the\n package that we'll build in the child process).\n function: function to call in the child process; serialized_pkg is passed to\n this as the first argument.\n kwargs: additional keyword arguments to pass to ``function()``.\n write_pipe: multiprocessing ``Connection`` to the parent process, to which the\n child *must* send a result (or an error) back to parent on.\n input_multiprocess_fd: stdin from the parent (not passed currently on Windows)\n jsfd1: gmake Jobserver file descriptor 1.\n jsfd2: gmake Jobserver file descriptor 2.\n stdout_pipe: pipe to redirect stdout to\n stderr_pipe: pipe to redirect stderr to\n \"\"\"\n\n context: str = kwargs.get(\"context\", \"build\")\n\n try:\n # We are in the child process. Python sets sys.stdin to open(os.devnull) to prevent our\n # process and its parent from simultaneously reading from the original stdin. But, we\n # assume that the parent process is not going to read from it till we are done with the\n # child, so we undo Python's precaution. closefd=False since Connection has ownership.\n if input_pipe is not None:\n sys.stdin = os.fdopen(input_pipe.fileno(), closefd=False)\n if stdout_pipe is not None:\n os.dup2(stdout_pipe.fileno(), sys.stdout.fileno())\n stdout_pipe.close()\n if stderr_pipe is not None:\n os.dup2(stderr_pipe.fileno(), sys.stderr.fileno())\n stderr_pipe.close()\n\n pkg = serialized_pkg.restore()\n\n if not kwargs.get(\"fake\", False):\n kwargs[\"unmodified_env\"] = os.environ.copy()\n kwargs[\"env_modifications\"] = setup_package(\n pkg, dirty=kwargs.get(\"dirty\", False), context=Context.from_string(context)\n )\n return_value = function(pkg, kwargs)\n write_pipe.send(return_value)\n\n except spack.error.StopPhase as e:\n # Do not create a full ChildError from this, it's not an error\n # it's a control statement.\n write_pipe.send(e)\n except BaseException as e:\n # catch ANYTHING that goes wrong in the child process\n\n # Need to unwind the traceback in the child because traceback\n # objects can't be sent to the parent.\n exc_type = type(e)\n tb = e.__traceback__\n tb_string = \"\".join(traceback.format_exception(exc_type, e, tb))\n\n # build up some context from the offending package so we can\n # show that, too.\n package_context = get_package_context(tb)\n\n logfile = None\n if context == \"build\":\n try:\n if hasattr(pkg, \"log_path\"):\n logfile = pkg.log_path\n except NameError:\n # 'pkg' is not defined yet\n pass\n elif context == \"test\":\n logfile = os.path.join(\n pkg.test_suite.stage, # type: ignore[union-attr]\n pkg.test_suite.test_log_name(pkg.spec), # type: ignore[union-attr]\n )\n\n error_msg = str(e)\n if isinstance(e, (spack.multimethod.NoSuchMethodError, AttributeError)):\n process = \"test the installation\" if context == \"test\" else \"build from sources\"\n error_msg = (\n \"The '{}' package cannot find an attribute while trying to {}. You can fix this \"\n \"by updating the {} recipe, and you can also report the issue as a build-error or \"\n \"a bug at https://github.com/spack/spack/issues\"\n ).format(pkg.name, process, context)\n error_msg = colorize(\"@*R{{{}}}\".format(error_msg))\n error_msg = \"{}\\n\\n{}\".format(str(e), error_msg)\n\n # make a pickleable exception to send to parent.\n msg = \"%s: %s\" % (exc_type.__name__, error_msg)\n\n ce = ChildError(\n msg,\n exc_type.__module__,\n exc_type.__name__,\n tb_string,\n logfile,\n context,\n package_context,\n )\n write_pipe.send(ce)\n\n finally:\n write_pipe.close()\n if input_pipe is not None:\n input_pipe.close()\n\n\nclass BuildProcess:\n \"\"\"Class used to manage builds launched by Spack.\n\n Each build is launched in its own child process, and the main Spack process\n tracks each child with a ``BuildProcess`` object. ``BuildProcess`` is used to:\n - Start and monitor an active child process.\n - Clean up its processes and resources when the child process completes.\n - Kill the child process if needed.\n\n See also ``start_build_process()`` and ``complete_build_process()``.\n \"\"\"\n\n def __init__(\n self,\n *,\n target: Callable,\n args: Tuple[Any, ...],\n pkg: \"spack.package_base.PackageBase\",\n read_pipe: Connection,\n timeout: Optional[int],\n ) -> None:\n self.p = multiprocessing.Process(target=target, args=args)\n self.pkg = pkg\n self.read_pipe = read_pipe\n self.timeout = timeout\n\n def start(self) -> None:\n self.p.start()\n\n def poll(self) -> bool:\n \"\"\"Check if there is data available to receive from the read pipe.\"\"\"\n return self.read_pipe.poll()\n\n def complete(self):\n \"\"\"Wait (if needed) for child process to complete\n and return its exit status.\n\n See ``complete_build_process()``.\n \"\"\"\n return complete_build_process(self)\n\n def is_alive(self) -> bool:\n return self.p.is_alive()\n\n def join(self, *, timeout: Optional[int] = None):\n self.p.join(timeout=timeout)\n\n def terminate(self):\n # Opportunity for graceful termination\n self.p.terminate()\n self.p.join(timeout=1)\n\n # If the process didn't gracefully terminate, forcefully kill\n if self.p.is_alive():\n # TODO (python 3.6 removal): use self.p.kill() instead, consider removing this class\n assert isinstance(self.p.pid, int), f\"unexpected value for PID: {self.p.pid}\"\n os.kill(self.p.pid, signal.SIGKILL)\n self.p.join()\n\n @property\n def pid(self):\n return self.p.pid\n\n @property\n def exitcode(self):\n return self.p.exitcode\n\n\ndef start_build_process(\n pkg: \"spack.package_base.PackageBase\",\n function: Callable,\n kwargs: Dict[str, Any],\n *,\n timeout: Optional[int] = None,\n) -> BuildProcess:\n \"\"\"Create a child process to do part of a spack build.\n\n Args:\n pkg: package whose environment we should set up the\n child process for.\n function: argless function to run in the child\n process.\n kwargs: additional keyword arguments to pass to ``function()``\n timeout: maximum time allowed to finish the execution of function\n\n Usage::\n\n def child_fun():\n # do stuff\n process = build_env.start_build_process(pkg, child_fun)\n complete_build_process(process)\n\n The child process is run with the build environment set up by\n spack.build_environment. This allows package authors to have full\n control over the environment, etc. without affecting other builds\n that might be executed in the same spack call.\n\n \"\"\"\n read_pipe, write_pipe = multiprocessing.Pipe(duplex=False)\n input_fd = None\n stdout_fd = None\n stderr_fd = None\n jobserver_fd1 = None\n jobserver_fd2 = None\n\n serialized_pkg = spack.subprocess_context.PackageInstallContext(pkg)\n\n try:\n # Forward sys.stdin when appropriate, to allow toggling verbosity\n if sys.platform != \"win32\" and sys.stdin.isatty() and hasattr(sys.stdin, \"fileno\"):\n input_fd = Connection(os.dup(sys.stdin.fileno()))\n\n # If our process has redirected stdout/stderr after the forkserver was started, we need to\n # make the forked processes use the new file descriptors.\n if multiprocessing.get_start_method() == \"forkserver\":\n try:\n stdout_fd = Connection(os.dup(sys.stdout.fileno()))\n stderr_fd = Connection(os.dup(sys.stderr.fileno()))\n except Exception:\n pass\n\n mflags = os.environ.get(\"MAKEFLAGS\")\n if mflags is not None:\n m = re.search(r\"--jobserver-[^=]*=(\\d),(\\d)\", mflags)\n if m:\n jobserver_fd1 = Connection(int(m.group(1)))\n jobserver_fd2 = Connection(int(m.group(2)))\n\n p = BuildProcess(\n target=_setup_pkg_and_run,\n args=(\n serialized_pkg,\n function,\n kwargs,\n write_pipe,\n input_fd,\n jobserver_fd1,\n jobserver_fd2,\n stdout_fd,\n stderr_fd,\n ),\n read_pipe=read_pipe,\n timeout=timeout,\n pkg=pkg,\n )\n\n p.start()\n\n # We close the writable end of the pipe now to be sure that p is the\n # only process which owns a handle for it. This ensures that when p\n # closes its handle for the writable end, read_pipe.recv() will\n # promptly report the readable end as being ready.\n write_pipe.close()\n\n except InstallError as e:\n e.pkg = pkg\n raise\n\n finally:\n # Close the input stream in the parent process\n if input_fd is not None:\n input_fd.close()\n if stdout_fd is not None:\n stdout_fd.close()\n if stderr_fd is not None:\n stderr_fd.close()\n\n return p\n\n\ndef complete_build_process(process: BuildProcess):\n \"\"\"\n Wait for the child process to complete and handles its exit status.\n\n If something goes wrong, the child process catches the error and\n passes it to the parent wrapped in a ChildError. The parent is\n expected to handle (or re-raise) the ChildError.\n \"\"\"\n\n def exitcode_msg(process):\n typ = \"exit\" if process.exitcode >= 0 else \"signal\"\n return f\"{typ} {abs(process.exitcode)}\"\n\n try:\n # Check if information from the read pipe has been received.\n child_result = process.read_pipe.recv()\n except EOFError:\n raise InstallError(f\"The process has stopped unexpectedly ({exitcode_msg(process)})\")\n finally:\n timeout = process.timeout\n process.join(timeout=timeout)\n if process.is_alive():\n warnings.warn(f\"Terminating process, since the timeout of {timeout}s was exceeded\")\n process.terminate()\n # If returns a StopPhase, raise it\n if isinstance(child_result, spack.error.StopPhase):\n raise child_result\n\n # let the caller know which package went wrong.\n if isinstance(child_result, InstallError):\n child_result.pkg = process.pkg\n\n if isinstance(child_result, ChildError):\n # If the child process raised an error, print its output here rather\n # than waiting until the call to SpackError.die() in main(). This\n # allows exception handling output to be logged from within Spack.\n # see spack.main.SpackCommand.\n child_result.print_context()\n raise child_result\n\n # Fallback. Usually caught beforehand in EOFError above.\n if process.exitcode != 0:\n raise InstallError(f\"The process failed unexpectedly ({exitcode_msg(process)})\")\n\n return child_result\n\n\nCONTEXT_BASES = (spack.package_base.PackageBase, spack.builder.BaseBuilder)\n\n\ndef get_package_context(traceback, context=3):\n \"\"\"Return some context for an error message when the build fails.\n\n Args:\n traceback: A traceback from some exception raised during\n install\n\n context (int): Lines of context to show before and after the line\n where the error happened\n\n This function inspects the stack to find where we failed in the\n package file, and it adds detailed context to the long_message\n from there.\n\n \"\"\"\n\n def make_stack(tb, stack=None):\n \"\"\"Tracebacks come out of the system in caller -> callee order. Return\n an array in callee -> caller order so we can traverse it.\"\"\"\n if stack is None:\n stack = []\n if tb is not None:\n make_stack(tb.tb_next, stack)\n stack.append(tb)\n return stack\n\n stack = make_stack(traceback)\n\n basenames = tuple(base.__name__ for base in CONTEXT_BASES)\n for tb in stack:\n frame = tb.tb_frame\n if \"self\" in frame.f_locals:\n # Find the first proper subclass of the PackageBase or BaseBuilder, but\n # don't provide context if the code is actually in the base classes.\n obj = frame.f_locals[\"self\"]\n func = getattr(obj, tb.tb_frame.f_code.co_name, \"\")\n if func and hasattr(func, \"__qualname__\"):\n typename, *_ = func.__qualname__.partition(\".\")\n if isinstance(obj, CONTEXT_BASES) and typename not in basenames:\n break\n else:\n return None\n\n # We found obj, the Package implementation we care about.\n # Point out the location in the install method where we failed.\n filename = inspect.getfile(frame.f_code)\n lines = [f\"{filename}:{frame.f_lineno}, in {frame.f_code.co_name}:\"]\n\n # Build a message showing context in the install method.\n sourcelines, start = inspect.getsourcelines(frame)\n\n # Calculate lineno of the error relative to the start of the function.\n fun_lineno = frame.f_lineno - start\n start_ctx = max(0, fun_lineno - context)\n sourcelines = sourcelines[start_ctx : fun_lineno + context + 1]\n\n for i, line in enumerate(sourcelines):\n is_error = start_ctx + i == fun_lineno\n # Add start to get lineno relative to start of file, not function.\n marked = f\" {'>> ' if is_error else ' '}{start + start_ctx + i:-6d}{line.rstrip()}\"\n if is_error:\n marked = colorize(\"@R{%s}\" % cescape(marked))\n lines.append(marked)\n\n return lines\n\n\nclass ChildError(InstallError):\n \"\"\"Special exception class for wrapping exceptions from child processes\n in Spack's build environment.\n\n The main features of a ChildError are:\n\n 1. They're serializable, so when a child build fails, we can send one\n of these to the parent and let the parent report what happened.\n\n 2. They have a ``traceback`` field containing a traceback generated\n on the child immediately after failure. Spack will print this on\n failure in lieu of trying to run sys.excepthook on the parent\n process, so users will see the correct stack trace from a child.\n\n 3. They also contain context, which shows context in the Package\n implementation where the error happened. This helps people debug\n Python code in their packages. To get it, Spack searches the\n stack trace for the deepest frame where ``self`` is in scope and\n is an instance of PackageBase. This will generally find a useful\n spot in the ``package.py`` file.\n\n The long_message of a ChildError displays one of two things:\n\n 1. If the original error was a ProcessError, indicating a command\n died during the build, we'll show context from the build log.\n\n 2. If the original error was any other type of error, we'll show\n context from the Python code.\n\n SpackError handles displaying the special traceback if we're in debug\n mode with spack -d.\n\n \"\"\"\n\n # List of errors considered \"build errors\", for which we'll show log\n # context instead of Python context.\n build_errors = [(\"spack.util.executable\", \"ProcessError\")]\n\n def __init__(self, msg, module, classname, traceback_string, log_name, log_type, context):\n super().__init__(msg)\n self.module = module\n self.name = classname\n self.traceback = traceback_string\n self.log_name = log_name\n self.log_type = log_type\n self.context = context\n\n @property\n def long_message(self):\n out = io.StringIO()\n out.write(self._long_message if self._long_message else \"\")\n\n have_log = self.log_name and os.path.exists(self.log_name)\n\n if (self.module, self.name) in ChildError.build_errors:\n # The error happened in some external executed process. Show\n # the log with errors or warnings highlighted.\n if have_log:\n write_log_summary(out, self.log_type, self.log_name)\n\n else:\n # The error happened in the Python code, so try to show\n # some context from the Package itself.\n if self.context:\n out.write(\"\\n\")\n out.write(\"\\n\".join(self.context))\n out.write(\"\\n\")\n\n if out.getvalue():\n out.write(\"\\n\")\n\n if have_log:\n out.write(\"See {0} log for details:\\n\".format(self.log_type))\n out.write(\" {0}\\n\".format(self.log_name))\n\n # Also output the test log path IF it exists\n if self.context != \"test\" and have_log:\n test_log = join_path(os.path.dirname(self.log_name), spack_install_test_log)\n if os.path.isfile(test_log):\n out.write(\"\\nSee test log for details:\\n\")\n out.write(\" {0}\\n\".format(test_log))\n\n return out.getvalue()\n\n def __str__(self):\n return self.message\n\n def __reduce__(self):\n \"\"\"__reduce__ is used to serialize (pickle) ChildErrors.\n\n Return a function to reconstruct a ChildError, along with the\n salient properties we'll need.\n \"\"\"\n return _make_child_error, (\n self.message,\n self.module,\n self.name,\n self.traceback,\n self.log_name,\n self.log_type,\n self.context,\n )\n\n\ndef _make_child_error(msg, module, name, traceback, log, log_type, context):\n \"\"\"Used by __reduce__ in ChildError to reconstruct pickled errors.\"\"\"\n return ChildError(msg, module, name, traceback, log, log_type, context)\n\n\ndef write_log_summary(out, log_type, log, last=None):\n errors, warnings = parse_log_events(log)\n nerr = len(errors)\n nwar = len(warnings)\n\n if nerr > 0:\n if last and nerr > last:\n errors = errors[-last:]\n nerr = last\n\n # If errors are found, only display errors\n out.write(\"\\n%s found in %s log:\\n\" % (plural(nerr, \"error\"), log_type))\n out.write(make_log_context(errors))\n elif nwar > 0:\n if last and nwar > last:\n warnings = warnings[-last:]\n nwar = last\n\n # If no errors are found but warnings are, display warnings\n out.write(\"\\n%s found in %s log:\\n\" % (plural(nwar, \"warning\"), log_type))\n out.write(make_log_context(warnings))\n\n\nclass ModuleChangePropagator:\n \"\"\"The function :meth:`spack.package_base.PackageBase.setup_dependent_package` receives\n an instance of this class for the ``module`` argument. It's used to set global variables in the\n module of a package, and propagate those globals to the modules of all classes in the\n inheritance hierarchy of the package. It's reminiscent of\n :class:`spack.util.environment.EnvironmentModifications`, but sets Python variables instead\n of environment variables. This class should typically not be instantiated in packages directly.\n \"\"\"\n\n _PROTECTED_NAMES = (\"package\", \"current_module\", \"modules_in_mro\", \"_set_attributes\")\n\n def __init__(self, package: spack.package_base.PackageBase) -> None:\n self._set_self_attributes(\"package\", package)\n self._set_self_attributes(\"current_module\", package.module)\n\n #: Modules for the classes in the MRO up to PackageBase\n modules_in_mro = []\n for cls in package.__class__.__mro__:\n module = getattr(cls, \"module\", None)\n\n if module is None or module is spack.package_base:\n break\n\n if module is self.current_module:\n continue\n\n modules_in_mro.append(module)\n self._set_self_attributes(\"modules_in_mro\", modules_in_mro)\n self._set_self_attributes(\"_set_attributes\", {})\n\n def _set_self_attributes(self, key, value):\n super().__setattr__(key, value)\n\n def __getattr__(self, item):\n return getattr(self.current_module, item)\n\n def __setattr__(self, key, value):\n if key in ModuleChangePropagator._PROTECTED_NAMES:\n msg = f'Cannot set attribute \"{key}\" in ModuleMonkeyPatcher'\n return AttributeError(msg)\n\n setattr(self.current_module, key, value)\n self._set_attributes[key] = value\n\n def propagate_changes_to_mro(self):\n for module_in_mro in self.modules_in_mro:\n module_in_mro.__dict__.update(self._set_attributes)\n" }, { "path": "lib/spack/spack/buildcache_migrate.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport json\nimport os\nimport pathlib\nimport tempfile\nfrom typing import NamedTuple\n\nimport spack.binary_distribution\nimport spack.database as spack_db\nimport spack.error\nimport spack.llnl.util.tty as tty\nimport spack.mirrors.mirror\nimport spack.spec\nimport spack.stage\nimport spack.util.crypto\nimport spack.util.parallel\nimport spack.util.url as url_util\nimport spack.util.web as web_util\n\nfrom .enums import InstallRecordStatus\nfrom .url_buildcache import (\n BlobRecord,\n BuildcacheComponent,\n compressed_json_from_dict,\n get_url_buildcache_class,\n sign_file,\n try_verify,\n)\n\n\ndef v2_tarball_directory_name(spec):\n \"\"\"\n Return name of the tarball directory according to the convention\n -//-/\n \"\"\"\n return spec.format_path(\"{architecture}/{compiler.name}-{compiler.version}/{name}-{version}\")\n\n\ndef v2_tarball_name(spec, ext):\n \"\"\"\n Return the name of the tarfile according to the convention\n ---\n \"\"\"\n spec_formatted = spec.format_path(\n \"{architecture}-{compiler.name}-{compiler.version}-{name}-{version}-{hash}\"\n )\n return f\"{spec_formatted}{ext}\"\n\n\ndef v2_tarball_path_name(spec, ext):\n \"\"\"\n Return the full path+name for a given spec according to the convention\n /\n \"\"\"\n return os.path.join(v2_tarball_directory_name(spec), v2_tarball_name(spec, ext))\n\n\nclass MigrateSpecResult(NamedTuple):\n success: bool\n message: str\n\n\nclass MigrationException(spack.error.SpackError):\n \"\"\"\n Raised when migration fails irrevocably\n \"\"\"\n\n def __init__(self, msg):\n super().__init__(msg)\n\n\ndef _migrate_spec(\n s: spack.spec.Spec, mirror_url: str, tmpdir: str, unsigned: bool = False, signing_key: str = \"\"\n) -> MigrateSpecResult:\n \"\"\"Parallelizable function to migrate a single spec\"\"\"\n print_spec = f\"{s.name}/{s.dag_hash()[:7]}\"\n\n # Check if the spec file exists in the new location and exit early if so\n\n v3_cache_class = get_url_buildcache_class(layout_version=3)\n v3_cache_entry = v3_cache_class(mirror_url, s, allow_unsigned=unsigned)\n exists = v3_cache_entry.exists([BuildcacheComponent.SPEC, BuildcacheComponent.TARBALL])\n v3_cache_entry.destroy()\n\n if exists:\n msg = f\"No need to migrate {print_spec}\"\n return MigrateSpecResult(True, msg)\n\n # Try to fetch the spec metadata\n v2_metadata_urls = [\n url_util.join(mirror_url, \"build_cache\", v2_tarball_name(s, \".spec.json.sig\"))\n ]\n\n if unsigned:\n v2_metadata_urls.append(\n url_util.join(mirror_url, \"build_cache\", v2_tarball_name(s, \".spec.json\"))\n )\n\n spec_contents = None\n\n for meta_url in v2_metadata_urls:\n try:\n spec_contents = web_util.read_text(meta_url)\n v2_spec_url = meta_url\n break\n except (web_util.SpackWebError, OSError):\n pass\n else:\n msg = f\"Unable to read metadata for {print_spec}\"\n return MigrateSpecResult(False, msg)\n\n spec_dict = {}\n\n if unsigned:\n # User asked for unsigned, if we found a signed specfile, just ignore\n # the signature\n if v2_spec_url.endswith(\".sig\"):\n spec_dict = spack.spec.Spec.extract_json_from_clearsig(spec_contents)\n else:\n spec_dict = json.loads(spec_contents)\n else:\n # User asked for signed, we must successfully verify the signature\n local_signed_pre_verify = os.path.join(\n tmpdir, f\"{s.name}_{s.dag_hash()}_verify.spec.json.sig\"\n )\n with open(local_signed_pre_verify, \"w\", encoding=\"utf-8\") as fd:\n fd.write(spec_contents)\n if not try_verify(local_signed_pre_verify):\n return MigrateSpecResult(False, f\"Failed to verify signature of {print_spec}\")\n with open(local_signed_pre_verify, encoding=\"utf-8\") as fd:\n spec_dict = spack.spec.Spec.extract_json_from_clearsig(fd.read())\n\n # Read out and remove the bits needed to rename and position the archive\n bcc = spec_dict.pop(\"binary_cache_checksum\", None)\n if not bcc:\n msg = \"Cannot migrate a spec that does not have 'binary_cache_checksum'\"\n return MigrateSpecResult(False, msg)\n\n algorithm = bcc[\"hash_algorithm\"]\n checksum = bcc[\"hash\"]\n\n # TODO: Remove this key once oci buildcache no longer uses it\n spec_dict[\"buildcache_layout_version\"] = 2\n\n v2_archive_url = url_util.join(mirror_url, \"build_cache\", v2_tarball_path_name(s, \".spack\"))\n\n # spacks web utilities do not include direct copying of s3 objects, so we\n # need to download the archive locally, and then push it back to the target\n # location\n archive_stage_path = os.path.join(tmpdir, f\"archive_stage_{s.name}_{s.dag_hash()}\")\n archive_stage = spack.stage.Stage(v2_archive_url, path=archive_stage_path)\n\n try:\n archive_stage.create()\n archive_stage.fetch()\n except spack.error.FetchError:\n return MigrateSpecResult(False, f\"Unable to fetch archive for {print_spec}\")\n\n local_tarfile_path = archive_stage.save_filename\n\n # As long as we have to download the tarball anyway, we might as well compute the\n # checksum locally and check it against the expected value\n local_checksum = spack.util.crypto.checksum(\n spack.util.crypto.hash_fun_for_algo(algorithm), local_tarfile_path\n )\n\n if local_checksum != checksum:\n return MigrateSpecResult(\n False, f\"Checksum mismatch for {print_spec}: expected {checksum}, got {local_checksum}\"\n )\n\n spec_dict[\"archive_size\"] = os.stat(local_tarfile_path).st_size\n\n # Compress the spec dict and compute its checksum\n metadata_checksum_algo = \"sha256\"\n spec_json_path = os.path.join(tmpdir, f\"{s.name}_{s.dag_hash()}.spec.json\")\n metadata_checksum, metadata_size = compressed_json_from_dict(\n spec_json_path, spec_dict, metadata_checksum_algo\n )\n\n tarball_blob_record = BlobRecord(\n spec_dict[\"archive_size\"], v3_cache_class.TARBALL_MEDIATYPE, \"gzip\", algorithm, checksum\n )\n\n metadata_blob_record = BlobRecord(\n metadata_size,\n v3_cache_class.SPEC_MEDIATYPE,\n \"gzip\",\n metadata_checksum_algo,\n metadata_checksum,\n )\n\n # Compute the urls to the new blobs\n v3_archive_url = v3_cache_class.get_blob_url(mirror_url, tarball_blob_record)\n v3_spec_url = v3_cache_class.get_blob_url(mirror_url, metadata_blob_record)\n\n # First push the tarball\n tty.debug(f\"Pushing {local_tarfile_path} to {v3_archive_url}\")\n\n try:\n web_util.push_to_url(local_tarfile_path, v3_archive_url, keep_original=True)\n except Exception:\n return MigrateSpecResult(False, f\"Failed to push archive for {print_spec}\")\n\n # Then push the spec file\n tty.debug(f\"Pushing {spec_json_path} to {v3_spec_url}\")\n\n try:\n web_util.push_to_url(spec_json_path, v3_spec_url, keep_original=True)\n except Exception:\n return MigrateSpecResult(False, f\"Failed to push spec metadata for {print_spec}\")\n\n # Generate the manifest and write it to a temporary location\n manifest = {\n \"version\": v3_cache_class.get_layout_version(),\n \"data\": [tarball_blob_record.to_dict(), metadata_blob_record.to_dict()],\n }\n\n manifest_path = os.path.join(tmpdir, f\"{s.dag_hash()}.manifest.json\")\n with open(manifest_path, \"w\", encoding=\"utf-8\") as f:\n json.dump(manifest, f, indent=0, separators=(\",\", \":\"))\n # Note: when using gpg clear sign, we need to avoid long lines (19995\n # chars). If lines are longer, they are truncated without error. So,\n # here we still add newlines, but no indent, so save on file size and\n # line length.\n\n # Possibly sign the manifest\n if not unsigned:\n manifest_path = sign_file(signing_key, manifest_path)\n\n v3_manifest_url = v3_cache_class.get_manifest_url(s, mirror_url)\n\n # Push the manifest\n try:\n web_util.push_to_url(manifest_path, v3_manifest_url, keep_original=True)\n except Exception:\n return MigrateSpecResult(False, f\"Failed to push manifest for {print_spec}\")\n\n return MigrateSpecResult(True, f\"Successfully migrated {print_spec}\")\n\n\ndef migrate(\n mirror: spack.mirrors.mirror.Mirror, unsigned: bool = False, delete_existing: bool = False\n) -> None:\n \"\"\"Perform migration of the given mirror\n\n If unsigned is True, signatures on signed specs will be ignored, and specs\n will not be re-signed before pushing to the new location. Otherwise, spack\n will attempt to verify signatures and re-sign specs, and will fail if not\n able to do so. If delete_existing is True, spack will delete the original\n contents of the mirror once the migration is complete.\"\"\"\n signing_key = \"\"\n if not unsigned:\n try:\n signing_key = spack.binary_distribution.select_signing_key()\n except (\n spack.binary_distribution.NoKeyException,\n spack.binary_distribution.PickKeyException,\n ):\n raise MigrationException(\n \"Signed migration requires exactly one secret key in keychain\"\n )\n\n delete_action = \"deleting\" if delete_existing else \"keeping\"\n sign_action = \"an unsigned\" if unsigned else \"a signed\"\n mirror_url = mirror.fetch_url\n\n tty.msg(\n f\"Performing {sign_action} migration of {mirror.push_url} \"\n f\"and {delete_action} existing contents\"\n )\n\n index_url = url_util.join(mirror_url, \"build_cache\", spack_db.INDEX_JSON_FILE)\n contents = None\n\n try:\n contents = web_util.read_text(index_url)\n except (web_util.SpackWebError, OSError):\n raise MigrationException(\"Buildcache migration requires a buildcache index\")\n\n with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir:\n index_path = os.path.join(tmpdir, \"_tmp_index.json\")\n with open(index_path, \"w\", encoding=\"utf-8\") as fd:\n fd.write(contents)\n\n db = spack.binary_distribution.BuildCacheDatabase(tmpdir)\n db._read_from_file(pathlib.Path(index_path))\n\n specs_to_migrate = [\n s\n for s in db.query_local(installed=InstallRecordStatus.ANY)\n # todo, make it easier to get install records associated with specs\n if not s.external and db._data[s.dag_hash()].in_buildcache\n ]\n\n # Run the tasks in parallel if possible\n executor = spack.util.parallel.make_concurrent_executor()\n migrate_futures = [\n executor.submit(_migrate_spec, spec, mirror_url, tmpdir, unsigned, signing_key)\n for spec in specs_to_migrate\n ]\n\n success_count = 0\n\n tty.msg(\"Migration summary:\")\n for spec, migrate_future in zip(specs_to_migrate, migrate_futures):\n result = migrate_future.result()\n msg = f\" {spec.name}/{spec.dag_hash()[:7]}: {result.message}\"\n if result.success:\n success_count += 1\n tty.msg(msg)\n else:\n tty.error(msg)\n # The migrated index should have the same specs as the original index,\n # modulo any specs that we failed to migrate for whatever reason. So\n # to avoid having to re-fetch all the spec files now, just mark them\n # appropriately in the existing database and push that.\n db.mark(spec, \"in_buildcache\", result.success)\n\n if success_count > 0:\n tty.msg(\"Updating index and pushing keys\")\n\n # If the layout.json doesn't yet exist on this mirror, push it\n v3_cache_class = get_url_buildcache_class(layout_version=3)\n v3_cache_class.maybe_push_layout_json(mirror_url)\n\n # Push the migrated mirror index\n index_tmpdir = os.path.join(tmpdir, \"rebuild_index\")\n os.mkdir(index_tmpdir)\n spack.binary_distribution._push_index(db, index_tmpdir, mirror_url)\n\n # Push the public part of the signing key\n if not unsigned:\n keys_tmpdir = os.path.join(tmpdir, \"keys\")\n os.mkdir(keys_tmpdir)\n spack.binary_distribution._url_push_keys(\n mirror_url, keys=[signing_key], update_index=True, tmpdir=keys_tmpdir\n )\n else:\n tty.warn(\"No specs migrated, did you mean to perform an unsigned migration instead?\")\n\n # Delete the old layout if the user requested it\n if delete_existing:\n delete_prefix = url_util.join(mirror_url, \"build_cache\")\n tty.msg(f\"Recursively deleting {delete_prefix}\")\n web_util.remove_url(delete_prefix, recursive=True)\n\n tty.msg(\"Migration complete\")\n" }, { "path": "lib/spack/spack/buildcache_prune.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport os\nimport pathlib\nimport re\nimport tempfile\nimport uuid\nfrom concurrent.futures import Future, as_completed\nfrom fnmatch import fnmatch\nfrom pathlib import Path\nfrom typing import Callable, Dict, Iterable, Iterator, List, Optional, Set, Tuple, cast\n\nimport spack.binary_distribution\nimport spack.error\nimport spack.llnl.util.tty as tty\nimport spack.stage\nimport spack.util.parallel\nimport spack.util.url as url_util\nimport spack.util.web as web_util\nfrom spack.util.executable import which\n\nfrom .mirrors.mirror import Mirror\nfrom .url_buildcache import (\n CURRENT_BUILD_CACHE_LAYOUT_VERSION,\n BuildcacheComponent,\n URLBuildcacheEntry,\n get_entries_from_cache,\n get_url_buildcache_class,\n)\n\n\ndef _fetch_manifests(\n mirror: Mirror, tmpspecsdir: str\n) -> Tuple[Dict[str, float], Callable[[str], URLBuildcacheEntry], List[str]]:\n \"\"\"\n Fetch all manifests from the buildcache for a given mirror.\n\n This function retrieves all the manifest files from the buildcache of the specified\n mirror and returns a list of tuples containing the file names and a callable to read\n each manifest.\n\n :param mirror: The mirror from which to fetch the manifests.\n :return: A tuple with three elements - a list of manifest files in the mirror, a\n callable to read each manifest, and a list of blobs in the mirror.\n \"\"\"\n manifest_file_to_mtime_mapping, read_fn = get_entries_from_cache(\n mirror.fetch_url, tmpspecsdir, BuildcacheComponent.MANIFEST\n )\n url_to_list = url_util.join(\n mirror.fetch_url, spack.binary_distribution.buildcache_relative_blobs_path()\n )\n tty.debug(f\"Listing blobs in {url_to_list}\")\n blobs = web_util.list_url(url_to_list, recursive=True) or []\n if not blobs:\n tty.warn(f\"Unable to list blobs in {url_to_list}\")\n blobs = [\n url_util.join(\n mirror.fetch_url, spack.binary_distribution.buildcache_relative_blobs_path(), blob_name\n )\n for blob_name in blobs\n ]\n return manifest_file_to_mtime_mapping, read_fn, blobs\n\n\ndef _delete_manifests_from_cache_aws(\n url: str, tmpspecsdir: str, urls_to_delete: Set[str]\n) -> Optional[int]:\n aws = which(\"aws\")\n\n if not aws:\n tty.warn(\"AWS CLI not found, skipping deletion of cache entries.\")\n return None\n\n cache_class = get_url_buildcache_class(layout_version=CURRENT_BUILD_CACHE_LAYOUT_VERSION)\n\n include_pattern = cache_class.get_buildcache_component_include_pattern(\n BuildcacheComponent.MANIFEST\n )\n\n file_count_before_deletion = len(list(pathlib.Path(tmpspecsdir).rglob(include_pattern)))\n\n tty.debug(f\"Deleting {len(urls_to_delete)} entries from cache at {url}\")\n deleted = _delete_entries_from_cache_manual(tmpspecsdir, urls_to_delete)\n tty.debug(f\"Deleted {deleted} entries from cache at {url}\")\n\n sync_command_args = [\n \"s3\",\n \"sync\",\n \"--delete\",\n \"--exclude\",\n \"*\",\n \"--include\",\n include_pattern,\n tmpspecsdir,\n url,\n ]\n\n try:\n aws(*sync_command_args, output=os.devnull, error=os.devnull)\n # `aws s3 sync` doesn't return the number of deleted files,\n # but we can calculate it based on the local file count from\n # before and after the deletion.\n return file_count_before_deletion - len(\n list(pathlib.Path(tmpspecsdir).rglob(include_pattern))\n )\n except Exception:\n tty.warn(\n \"Failed to use aws s3 sync to delete manifests, falling back to parallel deletion.\"\n )\n\n return None\n\n\ndef _delete_entries_from_cache_manual(url: str, urls_to_delete: Set[str]) -> int:\n pruned_objects = 0\n futures: List[Future] = []\n\n with spack.util.parallel.make_concurrent_executor() as executor:\n for url in urls_to_delete:\n futures.append(executor.submit(_delete_object, url))\n\n for manifest_or_blob_future in as_completed(futures):\n pruned_objects += manifest_or_blob_future.result()\n\n return pruned_objects\n\n\ndef _delete_entries_from_cache(\n mirror: Mirror, tmpspecsdir: str, manifests_to_delete: Set[str], blobs_to_delete: Set[str]\n) -> int:\n pruned_manifests: Optional[int] = None\n\n if mirror.fetch_url.startswith(\"s3://\"):\n pruned_manifests = _delete_manifests_from_cache_aws(\n url=mirror.fetch_url, tmpspecsdir=tmpspecsdir, urls_to_delete=manifests_to_delete\n )\n\n if pruned_manifests is None:\n # If the AWS CLI deletion failed, we fall back to deleting both manifests\n # and blobs with the fallback method.\n objects_to_delete = blobs_to_delete.union(manifests_to_delete)\n pruned_objects = 0\n else:\n # If the AWS CLI deletion succeeded, we only need to worry about\n # deleting the blobs, since the manifests have already been deleted.\n objects_to_delete = blobs_to_delete\n pruned_objects = pruned_manifests\n\n return pruned_objects + _delete_entries_from_cache_manual(\n url=mirror.fetch_url, urls_to_delete=objects_to_delete\n )\n\n\ndef _delete_object(url: str) -> int:\n try:\n web_util.remove_url(url=url)\n tty.info(f\"Removed object {url}\")\n return 1\n except Exception as e:\n tty.warn(f\"Unable to remove object {url} due to: {e}\")\n return 0\n\n\ndef _object_has_prunable_mtime(url: str, pruning_started_at: float) -> Tuple[str, bool]:\n \"\"\"Check if an object's modification time makes it eligible for pruning.\n\n Objects modified after pruning started should not be pruned to avoid\n race conditions with concurrent uploads.\n \"\"\"\n stat_result = web_util.stat_url(url)\n assert stat_result is not None\n if stat_result[1] > pruning_started_at:\n tty.verbose(f\"Skipping deletion of {url} because it was modified after pruning started\")\n return url, False\n return url, True\n\n\ndef _filter_new_specs(urls: Iterable[str], pruning_started_at: float) -> Iterator[str]:\n \"\"\"Filter out URLs that were modified after pruning started.\n\n Runs parallel modification time checks on all URLs and yields only\n those that are old enough to be safely pruned.\n \"\"\"\n with spack.util.parallel.make_concurrent_executor() as executor:\n futures = []\n for url in urls:\n futures.append(executor.submit(_object_has_prunable_mtime, url, pruning_started_at))\n\n for manifest_or_blob_future in as_completed(futures):\n url, has_prunable_mtime = manifest_or_blob_future.result()\n if has_prunable_mtime:\n yield url\n\n\ndef _prune_orphans(\n mirror: Mirror,\n manifests: List[str],\n read_fn: Callable[[str], URLBuildcacheEntry],\n blobs: List[str],\n pruning_started_at: float,\n tmpspecsdir: str,\n dry_run: bool,\n) -> int:\n \"\"\"\n Prune orphaned manifests and blobs from the buildcache.\n\n This function crawls the buildcache for a given mirror and identifies orphaned\n manifests and blobs. An \"orphaned manifest\" is one that references blobs that\n are not present in the cache, while an \"orphaned blob\" is one that is present in\n the cache but not referenced in any manifest.\n\n It uses the following steps to identify and prune orphaned objects:\n\n 1. Fetch all the manifests in the cache and build up a list of all the blobs that they\n reference.\n 2. List all the blobs in the buildcache, resulting in a list of all the blobs that\n *actually* exist in the cache.\n 3. Compare the two lists and use the difference to determine which objects are orphaned.\n - If a blob is listed in the cache but not in any manifest, that blob is orphaned.\n - If a blob is listed in a manifest but not in the cache, that manifest is orphaned.\n \"\"\"\n\n # As part of the pruning process, we need to keep track of the mapping between\n # blob URLs and their corresponding manifest URLs. Once we start computing\n # which blobs are referenced by a manifest but not present in the cache,\n # we will need to know which manifest to prune.\n blob_to_manifest_mapping: Dict[str, str] = {}\n\n for manifest in manifests:\n cache_entry: Optional[URLBuildcacheEntry] = None\n try:\n cache_entry = cast(URLBuildcacheEntry, read_fn(manifest))\n assert cache_entry.manifest is not None # to satisfy type checker\n blob_to_manifest_mapping.update(\n {\n cache_entry.get_blob_url(mirror_url=mirror.fetch_url, record=data): manifest\n for data in cache_entry.manifest.data\n }\n )\n except Exception as e:\n tty.warn(f\"Unable to fetch manifest {manifest} due to: {e}\")\n continue\n finally:\n if cache_entry:\n cache_entry.destroy()\n\n # Blobs that are referenced in a manifest file (but not necessarily present in the cache)\n blob_urls_referenced_by_manifest = set(blob_to_manifest_mapping.keys())\n\n # Blobs that are actually present in the cache (but not necessarily referenced in any manifest)\n blob_urls_present_in_cache: Set[str] = set(blobs)\n\n # Compute set of blobs that are present in the cache but not referenced in any manifest\n orphaned_blobs = blob_urls_present_in_cache - blob_urls_referenced_by_manifest\n\n # Compute set of blobs that are referenced in a manifest but not present in the cache\n nonexisting_referenced_blobs = blob_urls_referenced_by_manifest - blob_urls_present_in_cache\n\n # Compute set of manifests that are orphaned (i.e., they reference blobs that are not\n # present in the cache)\n orphaned_manifests = {\n blob_to_manifest_mapping[blob_url] for blob_url in nonexisting_referenced_blobs\n }\n\n if not orphaned_blobs and not orphaned_manifests:\n return 0\n\n # Filter out any new specs that have been uploaded since the pruning started\n orphaned_blobs = set(_filter_new_specs(orphaned_blobs, pruning_started_at))\n orphaned_manifests = set(_filter_new_specs(orphaned_manifests, pruning_started_at))\n\n if orphaned_blobs:\n tty.info(f\"Found {len(orphaned_blobs)} blob(s) with no manifest\")\n if orphaned_manifests:\n tty.info(f\"Found {len(orphaned_manifests)} manifest(s) that are missing blobs\")\n\n # If dry run, just print the manifests and blobs that would be deleted\n # and exit early.\n if dry_run:\n pruned_object_count = len(orphaned_blobs) + len(orphaned_manifests)\n for manifest in orphaned_manifests:\n manifests.remove(manifest)\n tty.info(f\" Would prune manifest: {manifest}\")\n for blob in orphaned_blobs:\n blobs.remove(blob)\n tty.info(f\" Would prune blob: {blob}\")\n return pruned_object_count\n\n # Otherwise, perform the deletions.\n pruned_object_count = _delete_entries_from_cache(\n mirror=mirror,\n tmpspecsdir=tmpspecsdir,\n manifests_to_delete=orphaned_manifests,\n blobs_to_delete=orphaned_blobs,\n )\n\n for manifest in orphaned_manifests:\n manifests.remove(manifest)\n for blob in orphaned_blobs:\n blobs.remove(blob)\n\n return pruned_object_count\n\n\ndef prune_direct(\n mirror: Mirror, keeplist_file: pathlib.Path, pruning_started_at: float, dry_run: bool\n) -> None:\n \"\"\"\n Execute direct pruning for a given mirror using a keeplist file.\n\n This function reads a file containing spec hashes to keep, then deletes\n all other spec manifests from the buildcache.\n Note that this function does *not* prune the blobs associated with the manifests;\n to do that, `prune_orphan` must be invoked to clean up the now-orphaned blobs.\n\n Args:\n mirror: Mirror to prune\n keeplist_file: Path to file containing newline-delimited hashes to keep\n pruning_started_at: Timestamp of when the pruning started\n dry_run: Whether to perform a dry run without actually deleting\n \"\"\"\n tty.info(\"Running Direct Pruning\")\n tty.debug(f\"Direct pruning mirror: {mirror.fetch_url}\" + (\" (dry run)\" if dry_run else \"\"))\n\n keep_hashes: Set[str] = set()\n for line in keeplist_file.read_text().splitlines():\n keep_hash = line.strip().lstrip(\"/\")\n if len(keep_hash) != 32:\n raise MalformedKeepListException(f\"Found malformed hash in keeplist: {line}\")\n keep_hashes.add(keep_hash)\n\n if not keep_hashes:\n raise BuildcachePruningException(f\"No hashes found in keeplist file: {keeplist_file}\")\n\n tty.info(f\"Loaded {len(keep_hashes)} hashes to keep from {keeplist_file}\")\n total_pruned: Optional[int] = None\n with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpspecsdir:\n try:\n manifest_to_mtime_mapping, read_fn, blob_list = _fetch_manifests(mirror, tmpspecsdir)\n except Exception as e:\n raise BuildcachePruningException(\"Error getting entries from buildcache\") from e\n\n # Determine which manifests correspond to specs we want to prune\n manifests_to_prune: List[str] = []\n specs_to_prune: List[str] = []\n\n for manifest in manifest_to_mtime_mapping.keys():\n if not fnmatch(\n manifest,\n URLBuildcacheEntry.get_buildcache_component_include_pattern(\n BuildcacheComponent.SPEC\n ),\n ):\n tty.info(f\"Found a non-spec manifest at {manifest}, skipping...\")\n continue\n\n # Attempt to regex match the manifest name in order to extract the name, version,\n # and hash for the spec.\n manifest_name = manifest.split(\"/\")[-1] # strip off parent directories\n regex_match = re.match(r\"([^ ]+)-([^- ]+)[-_]([^-_\\. ]+)\", manifest_name)\n\n if regex_match is None:\n # This should never happen, unless the buildcache is somehow corrupted\n # and/or there is a bug.\n raise BuildcachePruningException(\n \"Unable to extract spec name, version, and hash from \"\n f'the manifest named \"{manifest_name}\"'\n )\n\n spec_name, spec_version, spec_hash = regex_match.groups()\n\n # Chop off any prefix/parent file path to get just the name\n spec_name = pathlib.Path(spec_name).name\n\n if spec_hash not in keep_hashes:\n manifests_to_prune.append(manifest)\n specs_to_prune.append(f\"{spec_name}/{spec_hash[:7]}\")\n\n if not manifests_to_prune:\n tty.info(\"No specs to prune - all specs are in the keeplist\")\n return\n\n tty.info(f\"Found {len(manifests_to_prune)} spec(s) to prune\")\n\n if dry_run:\n for spec_name in specs_to_prune:\n tty.info(f\" Would prune: {spec_name}\")\n total_pruned = len(manifests_to_prune)\n else:\n manifests_to_delete = set(_filter_new_specs(manifests_to_prune, pruning_started_at))\n\n total_pruned = _delete_entries_from_cache(\n mirror=mirror,\n tmpspecsdir=tmpspecsdir,\n manifests_to_delete=manifests_to_delete,\n blobs_to_delete=set(),\n )\n\n if dry_run:\n tty.info(f\"Would have pruned {total_pruned} objects from mirror: {mirror.fetch_url}\")\n else:\n tty.info(f\"Pruned {total_pruned} objects from mirror: {mirror.fetch_url}\")\n if total_pruned > 0:\n tty.info(\n \"As a consequence of pruning, the buildcache index is now likely out of date.\"\n )\n tty.info(\"Run `spack buildcache update-index` to update the index for this mirror.\")\n\n\ndef prune_orphan(mirror: Mirror, pruning_started_at: float, dry_run: bool) -> None:\n \"\"\"\n Execute the pruning process for a given mirror.\n\n Currently, this function only performs the pruning of orphaned manifests and blobs.\n \"\"\"\n tty.info(\"=== Orphan Pruning Phase ===\")\n tty.debug(f\"Pruning mirror: {mirror.fetch_url}\" + (\" (dry run)\" if dry_run else \"\"))\n\n total_pruned = 0\n with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpspecsdir:\n try:\n manifest_to_mtime_mapping, read_fn, blob_list = _fetch_manifests(mirror, tmpspecsdir)\n manifests = list(manifest_to_mtime_mapping.keys())\n except Exception as e:\n raise BuildcachePruningException(\"Error getting entries from buildcache\") from e\n while True:\n # Continue pruning until no more orphaned objects are found\n pruned = _prune_orphans(\n mirror=mirror,\n manifests=manifests,\n read_fn=read_fn,\n blobs=blob_list,\n pruning_started_at=pruning_started_at,\n tmpspecsdir=tmpspecsdir,\n dry_run=dry_run,\n )\n if pruned == 0:\n break\n total_pruned += pruned\n\n if dry_run:\n tty.info(\n f\"Would have pruned {total_pruned} orphaned objects from mirror: \"\n + mirror.fetch_url\n )\n else:\n tty.info(f\"Pruned {total_pruned} orphaned objects from mirror: {mirror.fetch_url}\")\n if total_pruned > 0:\n # If we pruned any objects, the buildcache index is likely out of date.\n # Inform the user about this.\n tty.info(\n \"As a consequence of pruning, the buildcache index is now likely out of date.\"\n )\n tty.info(\n \"Run `spack buildcache update-index` to update the index for this mirror.\"\n )\n\n\ndef get_buildcache_normalized_time(mirror: Mirror) -> float:\n \"\"\"\n Get the current time as reported by the buildcache.\n\n This is necessary because different buildcache implementations may use different\n time formats/time zones. This function creates a temporary file, calls `stat_url`\n on it, and then deletes it. This guarantees that the time used for the beginning\n of the pruning is consistent across all buildcache implementations.\n \"\"\"\n with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as f:\n tmpdir = Path(f)\n touch_file = tmpdir / f\".spack-prune-marker-{uuid.uuid4()}\"\n touch_file.touch()\n remote_path = url_util.join(mirror.push_url, touch_file.name)\n\n web_util.push_to_url(\n local_file_path=str(touch_file), remote_path=remote_path, keep_original=True\n )\n\n stat_info = web_util.stat_url(remote_path)\n assert stat_info is not None\n start_time = stat_info[1]\n\n web_util.remove_url(remote_path)\n\n return start_time\n\n\ndef prune_buildcache(mirror: Mirror, keeplist: Optional[str] = None, dry_run: bool = False):\n \"\"\"\n Runs buildcache pruning for a given mirror.\n\n Args:\n mirror: Mirror to prune\n keeplist_file: Path to file containing newline-delimited hashes to keep\n dry_run: Whether to perform a dry run without actually deleting\n \"\"\"\n # Determine the time to use as the \"started at\" time for pruning.\n # If a cache index exists, use that time. Otherwise, use the current time (normalized\n # to the buildcache's time zone).\n cache_index_url = URLBuildcacheEntry.get_index_url(mirror_url=mirror.fetch_url)\n stat_result = web_util.stat_url(cache_index_url)\n if stat_result is not None:\n started_at = stat_result[1]\n else:\n started_at = get_buildcache_normalized_time(mirror)\n\n if keeplist:\n prune_direct(mirror, pathlib.Path(keeplist), started_at, dry_run)\n\n prune_orphan(mirror, started_at, dry_run)\n\n\nclass BuildcachePruningException(spack.error.SpackError):\n \"\"\"\n Raised when pruning fails irrevocably\n \"\"\"\n\n pass\n\n\nclass MalformedKeepListException(BuildcachePruningException):\n \"\"\"\n Raised when the keeplist passed to the direct pruner\n is invalid or malformed in some way\n \"\"\"\n\n pass\n" }, { "path": "lib/spack/spack/builder.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport collections\nimport collections.abc\nimport copy\nimport functools\nimport os\nfrom typing import Callable, Dict, List, Optional, Tuple, Type\n\nimport spack.directives\nimport spack.error\nimport spack.multimethod\nimport spack.package_base\nimport spack.phase_callbacks\nimport spack.relocate\nimport spack.repo\nimport spack.spec\nimport spack.util.environment\nfrom spack.error import SpackError\nfrom spack.util.prefix import Prefix\n\n#: Builder classes, as registered by the ``builder`` decorator\nBUILDER_CLS: Dict[str, Type[\"Builder\"]] = {}\n\n#: Map id(pkg) to a builder, to avoid creating multiple\n#: builders for the same package object.\n_BUILDERS: Dict[int, \"Builder\"] = {}\n\n\ndef register_builder(build_system_name: str):\n \"\"\"Class decorator used to register the default builder for a given build system. The name\n corresponds to the ``build_system`` variant value of the package.\n\n Example::\n\n @register_builder(\"cmake\")\n class CMakeBuilder(BuilderWithDefaults):\n pass\n\n\n Args:\n build_system_name: name of the build system\n \"\"\"\n\n def _decorator(cls):\n cls.build_system = build_system_name\n BUILDER_CLS[build_system_name] = cls\n return cls\n\n return _decorator\n\n\ndef create(pkg: spack.package_base.PackageBase) -> \"Builder\":\n \"\"\"Given a package object with an associated concrete spec, return the builder object that can\n install it.\"\"\"\n if id(pkg) not in _BUILDERS:\n _BUILDERS[id(pkg)] = _create(pkg)\n return _BUILDERS[id(pkg)]\n\n\nclass _PhaseAdapter:\n def __init__(self, builder, phase_fn):\n self.builder = builder\n self.phase_fn = phase_fn\n\n def __call__(self, spec, prefix):\n return self.phase_fn(self.builder.pkg, spec, prefix)\n\n\ndef get_builder_class(pkg, name: str) -> Optional[Type[\"Builder\"]]:\n \"\"\"Return the builder class if a package module defines it.\"\"\"\n for current_cls in type(pkg).__mro__:\n if not hasattr(current_cls, \"module\"):\n continue\n maybe_builder = getattr(current_cls.module, name, None)\n if maybe_builder and spack.repo.is_package_module(maybe_builder.__module__):\n return maybe_builder\n return None\n\n\ndef _create(pkg: spack.package_base.PackageBase) -> \"Builder\":\n \"\"\"Return a new builder object for the package object being passed as argument.\n\n The function inspects the build-system used by the package object and try to:\n\n 1. Return a custom builder, if any is defined in the same ``package.py`` file.\n 2. Return a customization of more generic builders, if any is defined in the\n class hierarchy (look at AspellDictPackage for an example of that)\n 3. Return a run-time generated adapter builder otherwise\n\n The run-time generated adapter builder is capable of adapting an old-style package\n to the new architecture, where the installation procedure has been extracted from\n the ``*Package`` hierarchy into a ``*Builder`` hierarchy. This means that the\n adapter looks for attribute or method overrides preferably in the ``*Package``\n before using the default builder implementation.\n\n Note that in case a builder is explicitly coded in ``package.py``, no attempt is made\n to look for build-related methods in the ``*Package``.\n\n Args:\n pkg: package object for which we need a builder\n \"\"\"\n package_buildsystem = buildsystem_name(pkg)\n default_builder_cls = BUILDER_CLS[package_buildsystem]\n builder_cls_name = default_builder_cls.__name__\n builder_class = get_builder_class(pkg, builder_cls_name)\n\n if builder_class:\n return builder_class(pkg)\n\n # Specialized version of a given buildsystem can subclass some\n # base classes and specialize certain phases or methods or attributes.\n # In that case they can store their builder class as a class level attribute.\n # See e.g. AspellDictPackage as an example.\n base_cls = getattr(pkg, builder_cls_name, default_builder_cls)\n\n # From here on we define classes to construct a special builder that adapts to the\n # old, single class, package format. The adapter forwards any call or access to an\n # attribute related to the installation procedure to a package object wrapped in\n # a class that falls-back on calling the base builder if no override is found on the\n # package. The semantic should be the same as the method in the base builder were still\n # present in the base class of the package.\n\n class _ForwardToBaseBuilder:\n def __init__(self, wrapped_pkg_object, root_builder):\n self.wrapped_package_object = wrapped_pkg_object\n self.root_builder = root_builder\n\n package_cls = type(wrapped_pkg_object)\n wrapper_cls = type(self)\n bases = (package_cls, wrapper_cls)\n new_cls_name = package_cls.__name__ + \"Wrapper\"\n # Forward attributes that might be monkey patched later\n new_cls = type(\n new_cls_name,\n bases,\n {\n \"__module__\": package_cls.__module__,\n \"run_tests\": property(lambda x: x.wrapped_package_object.run_tests),\n \"test_requires_compiler\": property(\n lambda x: x.wrapped_package_object.test_requires_compiler\n ),\n \"test_suite\": property(lambda x: x.wrapped_package_object.test_suite),\n \"tester\": property(lambda x: x.wrapped_package_object.tester),\n },\n )\n self.__class__ = new_cls\n self.__dict__.update(wrapped_pkg_object.__dict__)\n\n def __getattr__(self, item):\n result = getattr(super(type(self.root_builder), self.root_builder), item)\n if item in super(type(self.root_builder), self.root_builder).phases:\n result = _PhaseAdapter(self.root_builder, result)\n return result\n\n def forward_method_to_getattr(fn_name):\n def __forward(self, *args, **kwargs):\n return self.__getattr__(fn_name)(*args, **kwargs)\n\n return __forward\n\n # Add fallback methods for the Package object to refer to the builder. If a method\n # with the same name is defined in the Package, it will override this definition\n # (when _ForwardToBaseBuilder is initialized)\n for method_name in (\n base_cls.phases # type: ignore\n + package_methods(base_cls) # type: ignore\n + package_long_methods(base_cls) # type: ignore\n + (\"setup_build_environment\", \"setup_dependent_build_environment\")\n ):\n setattr(_ForwardToBaseBuilder, method_name, forward_method_to_getattr(method_name))\n\n def forward_property_to_getattr(property_name):\n def __forward(self):\n return self.__getattr__(property_name)\n\n return __forward\n\n for attribute_name in package_attributes(base_cls): # type: ignore\n setattr(\n _ForwardToBaseBuilder,\n attribute_name,\n property(forward_property_to_getattr(attribute_name)),\n )\n\n class Adapter(base_cls, metaclass=_PackageAdapterMeta): # type: ignore\n def __init__(self, pkg):\n # Deal with custom phases in packages here\n if hasattr(pkg, \"phases\"):\n self.phases = pkg.phases\n for phase in self.phases:\n setattr(Adapter, phase, _PackageAdapterMeta.phase_method_adapter(phase))\n\n # Attribute containing the package wrapped in dispatcher with a `__getattr__`\n # method that will forward certain calls to the default builder.\n self.pkg_with_dispatcher = _ForwardToBaseBuilder(pkg, root_builder=self)\n super().__init__(pkg)\n\n # These two methods don't follow the (self, spec, prefix) signature of phases nor\n # the (self) signature of methods, so they are added explicitly to avoid using a\n # catch-all (*args, **kwargs)\n def setup_build_environment(\n self, env: spack.util.environment.EnvironmentModifications\n ) -> None:\n return self.pkg_with_dispatcher.setup_build_environment(env)\n\n def setup_dependent_build_environment(\n self,\n env: spack.util.environment.EnvironmentModifications,\n dependent_spec: spack.spec.Spec,\n ) -> None:\n return self.pkg_with_dispatcher.setup_dependent_build_environment(env, dependent_spec)\n\n return Adapter(pkg)\n\n\ndef buildsystem_name(pkg: spack.package_base.PackageBase) -> str:\n \"\"\"Given a package object with an associated concrete spec,\n return the name of its build system.\"\"\"\n try:\n return pkg.spec.variants[\"build_system\"].value\n except KeyError as e:\n # We are reading an old spec without the build_system variant\n if hasattr(pkg, \"default_buildsystem\"):\n # Package API v2.2\n return pkg.default_buildsystem\n elif hasattr(pkg, \"legacy_buildsystem\"):\n return pkg.legacy_buildsystem\n\n raise SpackError(f\"Package {pkg.name} does not define a build system.\") from e\n\n\nclass BuilderMeta(\n spack.phase_callbacks.PhaseCallbacksMeta,\n spack.multimethod.MultiMethodMeta,\n type(collections.abc.Sequence), # type: ignore\n):\n pass\n\n\nclass _PackageAdapterMeta(BuilderMeta):\n \"\"\"Metaclass to adapt old-style packages to the new architecture based on builders\n for the installation phase.\n\n This class does the necessary mangling to function argument so that a call to a\n builder object can delegate to a package object.\n \"\"\"\n\n @staticmethod\n def phase_method_adapter(phase_name):\n def _adapter(self, pkg, spec, prefix):\n phase_fn = getattr(self.pkg_with_dispatcher, phase_name)\n return phase_fn(spec, prefix)\n\n return _adapter\n\n @staticmethod\n def legacy_long_method_adapter(method_name):\n def _adapter(self, spec, prefix):\n bind_method = getattr(self.pkg_with_dispatcher, method_name)\n return bind_method(spec, prefix)\n\n return _adapter\n\n @staticmethod\n def legacy_method_adapter(method_name):\n def _adapter(self):\n bind_method = getattr(self.pkg_with_dispatcher, method_name)\n return bind_method()\n\n return _adapter\n\n @staticmethod\n def legacy_attribute_adapter(attribute_name):\n def _adapter(self):\n return getattr(self.pkg_with_dispatcher, attribute_name)\n\n return property(_adapter)\n\n @staticmethod\n def combine_callbacks(pipeline_attribute_name):\n \"\"\"This function combines callbacks from old-style packages with callbacks that might\n be registered for the default builder.\n\n It works by:\n 1. Extracting the callbacks from the old-style package\n 2. Transforming those callbacks by adding an adapter that receives a builder as argument\n and calls the wrapped function with ``builder.pkg``\n 3. Combining the list of transformed callbacks with those that might be present in the\n default builder\n \"\"\"\n\n def _adapter(self):\n def unwrap_pkg(fn):\n @functools.wraps(fn)\n def _wrapped(builder):\n return fn(builder.pkg_with_dispatcher)\n\n return _wrapped\n\n # Concatenate the current list with the one from package\n callbacks_from_package = getattr(self.pkg, pipeline_attribute_name, [])\n callbacks_from_package = [(key, unwrap_pkg(x)) for key, x in callbacks_from_package]\n callbacks_from_builder = getattr(super(type(self), self), pipeline_attribute_name, [])\n return callbacks_from_package + callbacks_from_builder\n\n return property(_adapter)\n\n def __new__(mcs, name, bases, attr_dict):\n # Add ways to intercept methods and attribute calls and dispatch\n # them first to a package object\n default_builder_cls = bases[0]\n for phase_name in default_builder_cls.phases:\n attr_dict[phase_name] = _PackageAdapterMeta.phase_method_adapter(phase_name)\n\n for method_name in package_methods(default_builder_cls):\n attr_dict[method_name] = _PackageAdapterMeta.legacy_method_adapter(method_name)\n\n # These exist e.g. for Python, see discussion in https://github.com/spack/spack/pull/32068\n for method_name in package_long_methods(default_builder_cls):\n attr_dict[method_name] = _PackageAdapterMeta.legacy_long_method_adapter(method_name)\n\n for attribute_name in package_attributes(default_builder_cls):\n attr_dict[attribute_name] = _PackageAdapterMeta.legacy_attribute_adapter(\n attribute_name\n )\n\n combine_callbacks = _PackageAdapterMeta.combine_callbacks\n attr_dict[spack.phase_callbacks._RUN_BEFORE.attribute_name] = combine_callbacks(\n spack.phase_callbacks._RUN_BEFORE.attribute_name\n )\n attr_dict[spack.phase_callbacks._RUN_AFTER.attribute_name] = combine_callbacks(\n spack.phase_callbacks._RUN_AFTER.attribute_name\n )\n\n return super(_PackageAdapterMeta, mcs).__new__(mcs, name, bases, attr_dict)\n\n\nclass InstallationPhase:\n \"\"\"Manages a single phase of the installation.\n\n This descriptor stores at creation time the name of the method it should\n search for execution. The method is retrieved at __get__ time, so that\n it can be overridden by subclasses of whatever class declared the phases.\n\n It also provides hooks to execute arbitrary callbacks before and after\n the phase.\n \"\"\"\n\n def __init__(self, name, builder):\n self.name = name\n self.builder = builder\n self.phase_fn = self._select_phase_fn()\n self.run_before = self._make_callbacks(spack.phase_callbacks._RUN_BEFORE.attribute_name)\n self.run_after = self._make_callbacks(spack.phase_callbacks._RUN_AFTER.attribute_name)\n\n def _make_callbacks(self, callbacks_attribute):\n result = []\n callbacks = getattr(self.builder, callbacks_attribute, [])\n for (phase, condition), fn in callbacks:\n # Same if it is for another phase\n if phase != self.name:\n continue\n\n # If we have no condition or the callback satisfies a condition, register it\n if condition is None or self.builder.pkg.spec.satisfies(condition):\n result.append(fn)\n return result\n\n def __str__(self):\n msg = '{0}: executing \"{1}\" phase'\n return msg.format(self.builder, self.name)\n\n def execute(self):\n pkg = self.builder.pkg\n self._on_phase_start(pkg)\n\n for callback in self.run_before:\n callback(self.builder)\n\n self.phase_fn(pkg, pkg.spec, pkg.prefix)\n\n for callback in self.run_after:\n callback(self.builder)\n\n self._on_phase_exit(pkg)\n\n def _select_phase_fn(self):\n phase_fn = getattr(self.builder, self.name, None)\n\n if not phase_fn:\n msg = (\n 'unexpected error: package \"{0.fullname}\" must implement an '\n '\"{1}\" phase for the \"{2}\" build system'\n )\n raise RuntimeError(msg.format(self.builder.pkg, self.name, self.builder.build_system))\n\n return phase_fn\n\n def _on_phase_start(self, instance):\n # If a phase has a matching stop_before_phase attribute,\n # stop the installation process raising a StopPhase\n if getattr(instance, \"stop_before_phase\", None) == self.name:\n raise spack.error.StopPhase(\"Stopping before '{0}' phase\".format(self.name))\n\n def _on_phase_exit(self, instance):\n # If a phase has a matching last_phase attribute,\n # stop the installation process raising a StopPhase\n if getattr(instance, \"last_phase\", None) == self.name:\n raise spack.error.StopPhase(\"Stopping at '{0}' phase\".format(self.name))\n\n def copy(self):\n return copy.deepcopy(self)\n\n\nclass BaseBuilder(metaclass=BuilderMeta):\n \"\"\"An interface for builders, without any phases defined. This class is exposed in the package\n API, so that packagers can create a single class to define :meth:`setup_build_environment` and\n :func:`spack.phase_callbacks.run_before` and :func:`spack.phase_callbacks.run_after`\n callbacks that can be shared among different builders.\n\n Example:\n\n .. code-block:: python\n\n class AnyBuilder(BaseBuilder):\n @run_after(\"install\")\n def fixup_install(self):\n # do something after the package is installed\n pass\n\n def setup_build_environment(self, env: EnvironmentModifications) -> None:\n env.set(\"MY_ENV_VAR\", \"my_value\")\n\n\n class CMakeBuilder(cmake.CMakeBuilder, AnyBuilder):\n pass\n\n\n class AutotoolsBuilder(autotools.AutotoolsBuilder, AnyBuilder):\n pass\n \"\"\"\n\n def __init__(self, pkg: spack.package_base.PackageBase) -> None:\n self.pkg = pkg\n\n @property\n def spec(self) -> spack.spec.Spec:\n return self.pkg.spec\n\n @property\n def stage(self):\n return self.pkg.stage\n\n @property\n def prefix(self):\n return self.pkg.prefix\n\n def setup_build_environment(\n self, env: spack.util.environment.EnvironmentModifications\n ) -> None:\n \"\"\"Sets up the build environment for a package.\n\n This method will be called before the current package prefix exists in\n Spack's store.\n\n Args:\n env: environment modifications to be applied when the package is built. Package authors\n can call methods on it to alter the build environment.\n \"\"\"\n if not hasattr(super(), \"setup_build_environment\"):\n return\n super().setup_build_environment(env) # type: ignore\n\n def setup_dependent_build_environment(\n self, env: spack.util.environment.EnvironmentModifications, dependent_spec: spack.spec.Spec\n ) -> None:\n \"\"\"Sets up the build environment of a package that depends on this one.\n\n This is similar to ``setup_build_environment``, but it is used to modify the build\n environment of a package that *depends* on this one.\n\n This gives packages the ability to set environment variables for the build of the\n dependent, which can be useful to provide search hints for headers or libraries if they are\n not in standard locations.\n\n This method will be called before the dependent package prefix exists in Spack's store.\n\n Args:\n env: environment modifications to be applied when the dependent package is built.\n Package authors can call methods on it to alter the build environment.\n\n dependent_spec: the spec of the dependent package about to be built. This allows the\n extendee (self) to query the dependent's state. Note that *this* package's spec is\n available as ``self.spec``\n \"\"\"\n if not hasattr(super(), \"setup_dependent_build_environment\"):\n return\n super().setup_dependent_build_environment(env, dependent_spec) # type: ignore\n\n def __repr__(self):\n fmt = \"{name}{/hash:7}\"\n return f\"{self.__class__.__name__}({self.spec.format(fmt)})\"\n\n def __str__(self):\n fmt = \"{name}{/hash:7}\"\n return f'\"{self.__class__.__name__}\" builder for \"{self.spec.format(fmt)}\"'\n\n\nclass Builder(BaseBuilder, collections.abc.Sequence):\n \"\"\"A builder is a class that, given a package object (i.e. associated with concrete spec),\n knows how to install it.\n\n The builder behaves like a sequence, and when iterated over return the ``phases`` of the\n installation in the correct order.\n \"\"\"\n\n #: Sequence of phases. Must be defined in derived classes\n phases: Tuple[str, ...] = ()\n #: Build system name. Must also be defined in derived classes.\n build_system: Optional[str] = None\n\n #: Methods, with no arguments, that the adapter can find in Package classes,\n #: if a builder is not defined.\n package_methods: Tuple[str, ...]\n # Use :attr:`package_methods` instead of this attribute, which is deprecated\n legacy_methods: Tuple[str, ...] = ()\n\n #: Methods with the same signature as phases, that the adapter can find in Package classes,\n #: if a builder is not defined.\n package_long_methods: Tuple[str, ...]\n # Use :attr:`package_long_methods` instead of this attribute, which is deprecated\n legacy_long_methods: Tuple[str, ...]\n\n #: Attributes that the adapter can find in Package classes, if a builder is not defined\n package_attributes: Tuple[str, ...]\n # Use :attr:`package_attributes` instead of this attribute, which is deprecated\n legacy_attributes: Tuple[str, ...] = ()\n\n # type hints for some of the legacy methods\n build_time_test_callbacks: List[str]\n install_time_test_callbacks: List[str]\n\n #: List of glob expressions. Each expression must either be absolute or relative to the package\n #: source path. Matching artifacts found at the end of the build process will be copied in the\n #: same directory tree as _spack_build_logfile and _spack_build_envfile.\n @property\n def archive_files(self) -> List[str]:\n return []\n\n def __init__(self, pkg: spack.package_base.PackageBase) -> None:\n super().__init__(pkg)\n self.callbacks = {}\n for phase in self.phases:\n self.callbacks[phase] = InstallationPhase(phase, self)\n\n def __getitem__(self, idx):\n key = self.phases[idx]\n return self.callbacks[key]\n\n def __len__(self):\n return len(self.phases)\n\n\ndef package_methods(builder: Type[Builder]) -> Tuple[str, ...]:\n \"\"\"Returns the list of methods, taking no arguments, that are defined in the package\n class and are associated with the builder.\n \"\"\"\n if hasattr(builder, \"package_methods\"):\n # Package API v2.2\n return builder.package_methods\n\n return builder.legacy_methods\n\n\ndef package_attributes(builder: Type[Builder]) -> Tuple[str, ...]:\n \"\"\"Returns the list of attributes that are defined in the package class and are associated\n with the builder.\n \"\"\"\n if hasattr(builder, \"package_attributes\"):\n # Package API v2.2\n return builder.package_attributes\n\n return builder.legacy_attributes\n\n\ndef package_long_methods(builder: Type[Builder]) -> Tuple[str, ...]:\n \"\"\"Returns the list of methods, with the same signature as phases, that are defined in\n the package class and are associated with the builder.\n \"\"\"\n if hasattr(builder, \"package_long_methods\"):\n # Package API v2.2\n return builder.package_long_methods\n\n return getattr(builder, \"legacy_long_methods\", tuple())\n\n\ndef sanity_check_prefix(builder: Builder):\n \"\"\"Check that specific directories and files are created after installation.\n\n The files to be checked are in the ``sanity_check_is_file`` attribute of the\n package object, while the directories are in the ``sanity_check_is_dir``.\n\n Args:\n builder: builder that installed the package\n \"\"\"\n pkg = builder.pkg\n\n def check_paths(path_list: List[str], filetype: str, predicate: Callable[[str], bool]) -> None:\n if isinstance(path_list, str):\n path_list = [path_list]\n\n for path in path_list:\n if not predicate(os.path.join(pkg.prefix, path)):\n raise spack.error.InstallError(\n f\"Install failed for {pkg.name}. No such {filetype} in prefix: {path}\"\n )\n\n check_paths(pkg.sanity_check_is_file, \"file\", os.path.isfile)\n check_paths(pkg.sanity_check_is_dir, \"directory\", os.path.isdir)\n\n # Check that the prefix is not empty apart from the .spack/ directory\n with os.scandir(pkg.prefix) as entries:\n f = next(\n (f for f in entries if not (f.name == \".spack\" and f.is_dir(follow_symlinks=False))),\n None,\n )\n\n if f is None:\n raise spack.error.InstallError(f\"Install failed for {pkg.name}. Nothing was installed!\")\n\n\nclass BuilderWithDefaults(Builder):\n \"\"\"Base class for all specific builders with common callbacks registered.\"\"\"\n\n # Check that self.prefix is there after installation\n spack.phase_callbacks.run_after(\"install\")(sanity_check_prefix)\n\n\ndef apply_macos_rpath_fixups(builder: Builder):\n \"\"\"On Darwin, make installed libraries more easily relocatable.\n\n Some build systems (handrolled, autotools, makefiles) can set their own rpaths that are\n duplicated by spack's compiler wrapper. This fixup interrogates, and postprocesses if\n necessary, all libraries installed by the code.\n\n It should be added as a :func:`~spack.phase_callbacks.run_after` to packaging systems (or\n individual packages) that do not install relocatable libraries by default.\n\n Example::\n\n run_after(\"install\", when=\"platform=darwin\")(apply_macos_rpath_fixups)\n\n Args:\n builder: builder that installed the package\n \"\"\"\n spack.relocate.fixup_macos_rpaths(builder.spec)\n\n\ndef execute_install_time_tests(builder: Builder):\n \"\"\"Execute the install-time tests prescribed by builder.\n\n Args:\n builder: builder prescribing the test callbacks. The name of the callbacks is\n stored as a list of strings in the ``install_time_test_callbacks`` attribute.\n \"\"\"\n if not builder.pkg.run_tests or not builder.install_time_test_callbacks:\n return\n\n builder.pkg.tester.phase_tests(builder, \"install\", builder.install_time_test_callbacks)\n\n\nclass Package(spack.package_base.PackageBase):\n \"\"\"Build system base class for packages that do not use a specific build system. It adds the\n ``build_system=generic`` variant to the package.\n\n This is the only build system base class defined in Spack core. All other build systems\n are defined in the builtin package repository :mod:`spack_repo.builtin.build_systems`.\n\n The associated builder is :class:`GenericBuilder`, which is only necessary when the package\n has multiple build systems.\n\n Example::\n\n from spack.package import *\n\n class MyPackage(Package):\n \\\"\\\"\\\"A package that does not use a specific build system.\\\"\\\"\\\"\n\n homepage = \"https://example.com/mypackage\"\n url = \"https://example.com/mypackage-1.0.tar.gz\"\n\n version(\"1.0\", sha256=\"...\")\n\n def install(self, spec: Spec, prefix: Prefix) -> None:\n # Custom installation logic here\n pass\n\n .. note::\n\n The difference between :class:`Package` and :class:`~spack.package_base.PackageBase` is that\n :class:`~spack.package_base.PackageBase` is the universal base class for all package\n classes, no matter their build system.\n\n The :class:`Package` class is a *build system base class*, similar to\n ``CMakePackage``, and ``AutotoolsPackage``. It is called ``Package`` and not\n ``GenericPackage`` for legacy reasons.\n\n \"\"\"\n\n #: This attribute is used in UI queries that require to know which\n #: build-system class we are using\n build_system_class = \"Package\"\n\n #: Legacy buildsystem attribute used to deserialize and install old specs\n default_buildsystem = \"generic\"\n\n spack.directives.build_system(\"generic\")\n\n\n@register_builder(\"generic\")\nclass GenericBuilder(BuilderWithDefaults):\n \"\"\"The associated builder for the :class:`Package` base class. This class is typically only\n used in ``package.py`` files when a package has multiple build systems. Packagers need to\n implement the :meth:`install` phase to define how the package is installed.\n\n This is the only builder that is defined in the Spack core, all other builders are defined\n in the builtin package repository :mod:`spack_repo.builtin.build_systems`.\n\n Example::\n\n from spack.package import *\n\n class MyPackage(Package):\n \\\"\\\"\\\"A package that does not use a specific build system.\\\"\\\"\\\"\n homepage = \"https://example.com/mypackage\"\n url = \"https://example.com/mypackage-1.0.tar.gz\"\n\n version(\"1.0\", sha256=\"...\")\n\n class GenericBuilder(GenericBuilder):\n def install(self, pkg: Package, spec: Spec, prefix: Prefix) -> None:\n pass\n \"\"\"\n\n #: A generic package has only the ``install`` phase\n phases = (\"install\",)\n\n #: Names associated with package methods in the old build-system format\n package_methods: Tuple[str, ...] = ()\n\n #: Names associated with package attributes in the old build-system format\n package_attributes: Tuple[str, ...] = (\"archive_files\", \"install_time_test_callbacks\")\n\n #: Callback names for post-install phase tests\n install_time_test_callbacks = []\n\n # On macOS, force rpaths for shared library IDs and remove duplicate rpaths\n spack.phase_callbacks.run_after(\"install\", when=\"platform=darwin\")(apply_macos_rpath_fixups)\n\n # unconditionally perform any post-install phase tests\n spack.phase_callbacks.run_after(\"install\")(execute_install_time_tests)\n\n def install(self, pkg: Package, spec: spack.spec.Spec, prefix: Prefix) -> None:\n \"\"\"Install phase for the generic builder, to be implemented by packagers.\"\"\"\n raise NotImplementedError\n" }, { "path": "lib/spack/spack/caches.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Caches used by Spack to store data\"\"\"\n\nimport os\nfrom typing import cast\n\nimport spack.config\nimport spack.fetch_strategy\nimport spack.llnl.util.lang\nimport spack.paths\nimport spack.util.file_cache\nimport spack.util.path\nfrom spack.llnl.util.filesystem import mkdirp\n\n\ndef misc_cache_location():\n \"\"\"The ``MISC_CACHE`` is Spack's cache for small data.\n\n Currently the ``MISC_CACHE`` stores indexes for virtual dependency\n providers and for which packages provide which tags.\n \"\"\"\n path = spack.config.get(\"config:misc_cache\", spack.paths.default_misc_cache_path)\n return spack.util.path.canonicalize_path(path)\n\n\ndef _misc_cache():\n path = misc_cache_location()\n return spack.util.file_cache.FileCache(path)\n\n\n#: Spack's cache for small data\nMISC_CACHE = cast(spack.util.file_cache.FileCache, spack.llnl.util.lang.Singleton(_misc_cache))\n\n\ndef fetch_cache_location():\n \"\"\"Filesystem cache of downloaded archives.\n\n This prevents Spack from repeatedly fetch the same files when\n building the same package different ways or multiple times.\n \"\"\"\n path = spack.config.get(\"config:source_cache\")\n if not path:\n path = spack.paths.default_fetch_cache_path\n path = spack.util.path.canonicalize_path(path)\n return path\n\n\ndef _fetch_cache():\n path = fetch_cache_location()\n return spack.fetch_strategy.FsCache(path)\n\n\nclass MirrorCache:\n def __init__(self, root, skip_unstable_versions):\n self.root = os.path.abspath(root)\n self.skip_unstable_versions = skip_unstable_versions\n\n def store(self, fetcher, relative_dest):\n \"\"\"Fetch and relocate the fetcher's target into our mirror cache.\"\"\"\n\n # Note this will archive package sources even if they would not\n # normally be cached (e.g. the current tip of an hg/git branch)\n dst = os.path.join(self.root, relative_dest)\n mkdirp(os.path.dirname(dst))\n fetcher.archive(dst)\n\n\n#: Spack's local cache for downloaded source archives\nFETCH_CACHE = cast(spack.fetch_strategy.FsCache, spack.llnl.util.lang.Singleton(_fetch_cache))\n" }, { "path": "lib/spack/spack/ci/README.md", "content": "# Spack CI generators\n\nThis document describes how the ci module can be extended to provide novel\nci generators. The module currently has only a single generator for gitlab.\nThe unit-tests for the ci module define a small custom generator for testing\npurposes as well.\n\nThe process of generating a pipeline involves creating a ci-enabled spack\nenvironment, activating it, and running `spack ci generate`, possibly with\narguments describing things like where the output should be written.\n\nInternally pipeline generation is broken into two components: general and\nci platform specific.\n\n## General pipeline functionality\n\nGeneral pipeline functionality includes building a pipeline graph (really,\na forest), pruning it in a variety of ways, and gathering attributes for all\nthe generated spec build jobs from the spack configuration.\n\nAll of the above functionality is defined in the `__init__.py` of the top-level\nci module, and should be roughly the same for pipelines generated for any\nplatform.\n\n## CI platform specific functionality\n\nFunctionality specific to CI platforms (e.g. gitlab, gha, etc.) should be\ndefined in a dedicated module. In order to define a generator for a new\nplatform, there are only a few requirements:\n\n1. add a file under `ci` in which you define a generator method decorated with\nthe `@generator` attribute. .\n\n1. import it from `lib/spack/spack/ci/__init__.py`, so that your new generator\nis registered.\n\n1. the generator method must take as arguments PipelineDag, SpackCIConfig,\nand PipelineOptions objects, in that order.\n\n1. the generator method must produce an output file containing the\ngenerated pipeline.\n" }, { "path": "lib/spack/spack/ci/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport base64\nimport json\nimport os\nimport pathlib\nimport re\nimport shutil\nimport stat\nimport subprocess\nimport tempfile\nimport zipfile\nfrom collections import namedtuple\nfrom typing import Callable, Dict, Iterable, List, Optional, Set, Tuple\nfrom urllib.request import Request\n\nimport spack\nimport spack.binary_distribution\nimport spack.builder\nimport spack.config as cfg\nimport spack.environment as ev\nimport spack.llnl.path\nimport spack.llnl.util.filesystem as fs\nimport spack.llnl.util.tty as tty\nimport spack.main\nimport spack.mirrors.mirror\nimport spack.paths\nimport spack.repo\nimport spack.spec\nimport spack.stage\nimport spack.store\nimport spack.util.git\nimport spack.util.gpg as gpg_util\nimport spack.util.spack_yaml as syaml\nimport spack.util.url as url_util\nimport spack.util.web as web_util\nfrom spack import traverse\nfrom spack.error import SpackError\nfrom spack.llnl.util.tty.color import cescape, colorize\nfrom spack.reporters.cdash import SPACK_CDASH_TIMEOUT\n\nfrom .common import (\n IS_WINDOWS,\n CDashHandler,\n PipelineDag,\n PipelineOptions,\n PipelineType,\n SpackCIConfig,\n SpackCIError,\n copy_files_to_artifacts,\n)\nfrom .generator_registry import UnknownGeneratorException, get_generator\n\n# Import any modules with generator functions from here, so they get\n# registered without introducing any import cycles.\nfrom .gitlab import generate_gitlab_yaml # noqa: F401\n\nspack_gpg = spack.main.SpackCommand(\"gpg\")\nspack_compiler = spack.main.SpackCommand(\"compiler\")\n\nPushResult = namedtuple(\"PushResult\", \"success url\")\n\nurlopen = web_util.urlopen # alias for mocking in tests\n\n\ndef get_git_root(path: str) -> Optional[str]:\n git = spack.util.git.git(required=True)\n try:\n with fs.working_dir(path):\n # Raises SpackError on command failure\n git_dir = git(\"rev-parse\", \"--show-toplevel\", fail_on_error=True, output=str).strip()\n tty.debug(f\"{path} git toplevel at {git_dir}\")\n return git_dir\n except SpackError:\n return None\n\n\ndef get_change_revisions(path: str) -> Tuple[Optional[str], Optional[str]]:\n \"\"\"If this is a git repo get the revisions to use when checking\n for changed packages and spack core modules.\"\"\"\n\n if get_git_root(path):\n # TODO: This will only find changed packages from the last\n # TODO: commit. While this may work for single merge commits\n # TODO: when merging the topic branch into the base, it will\n # TODO: require more thought outside of that narrow case.\n return \"HEAD^\", \"HEAD\"\n else:\n return None, None\n\n\ndef filter_added_checksums(\n checksums: Iterable[str], path: str, from_ref: str = \"HEAD~1\", to_ref: str = \"HEAD\"\n) -> List[str]:\n \"\"\"Get a list of the version checksums added between ``from_ref`` and ``to_ref``.\n\n Args:\n checksums: an iterable of checksums to look for in the diff\n path: path to the package.py\n from_ref: oldest git ref, defaults to ``HEAD~1``\n to_ref: newer git ref, defaults to ``HEAD``\n Returns: list of version checksums added between refs\n \"\"\"\n git_exe = spack.util.git.git(required=True)\n\n # Gather git diff\n diff_lines = git_exe(\"diff\", from_ref, to_ref, \"--\", path, output=str).split(\"\\n\")\n\n # Store added and removed versions\n # Removed versions are tracked here to determine when versions are moved in a file\n # and show up as both added and removed in a git diff.\n added_checksums: Set[str] = set()\n removed_checksums: Set[str] = set()\n\n # Scrape diff for modified versions and prune added versions if they show up\n # as also removed (which means they've actually just moved in the file and\n # we shouldn't need to rechecksum them)\n for checksum in checksums:\n for line in diff_lines:\n if checksum in line:\n if line.startswith(\"+\"):\n added_checksums.add(checksum)\n if line.startswith(\"-\"):\n removed_checksums.add(checksum)\n\n return list(added_checksums - removed_checksums)\n\n\ndef stack_changed(env_path: str) -> bool:\n \"\"\"Given an environment manifest path, return whether or not the stack was changed.\n Returns True iff the environment manifest changed between the provided revisions (or\n additionally if the ``.gitlab-ci.yml`` file itself changed).\"\"\"\n # git returns posix paths always, normalize input to be compatible with that\n env_path = spack.llnl.path.convert_to_posix_path(os.path.dirname(env_path))\n\n git = spack.util.git.git(required=True)\n git_dir = get_git_root(env_path)\n\n if git_dir is None:\n return False\n\n with fs.working_dir(git_dir):\n diff = git(\n \"diff\",\n \"--name-only\",\n \"HEAD^\",\n \"HEAD\",\n output=str,\n error=os.devnull,\n fail_on_error=False,\n ).strip()\n\n if not diff:\n return False\n\n for path in diff.split():\n if \".gitlab-ci.yml\" in path or path in env_path:\n tty.debug(f\"env represented by {env_path} changed\")\n tty.debug(f\"touched file: {path}\")\n return True\n return False\n\n\ndef compute_affected_packages(\n repo: spack.repo.Repo, rev1: str = \"HEAD^\", rev2: str = \"HEAD\"\n) -> Set[str]:\n \"\"\"Determine which packages were added, removed or changed\n between rev1 and rev2, and return the names as a set\"\"\"\n return spack.repo.get_all_package_diffs(\"ARC\", repo, rev1=rev1, rev2=rev2)\n\n\ndef get_spec_filter_list(\n env: ev.Environment, affected_pkgs: Set[str], dependent_traverse_depth: Optional[int] = None\n) -> Set[spack.spec.Spec]:\n \"\"\"Given a list of package names and an active/concretized environment, return the set of all\n concrete specs from the environment that could have been affected by changing the list of\n packages.\n\n If a ``dependent_traverse_depth`` is given, it is used to limit upward (in the parent\n direction) traversal of specs of touched packages. E.g. if 1 is provided, then only direct\n dependents of touched package specs are traversed to produce specs that could have been\n affected by changing the package, while if 0 is provided, only the changed specs themselves\n are traversed. If ``None`` is given, upward traversal of touched package specs is done all the\n way to the environment roots. Providing a negative number results in no traversals at all,\n yielding an empty set.\n\n Arguments:\n env: Active concrete environment\n affected_pkgs: Affected package names\n dependent_traverse_depth: Integer to limit dependent traversal, None means no limit\n\n Returns:\n A set of concrete specs from the active environment including those associated with\n affected packages, their dependencies and dependents, as well as their dependents\n dependencies.\n \"\"\"\n affected_specs: Set[spack.spec.Spec] = set()\n all_concrete_specs = env.all_specs()\n env_matches = [s for s in all_concrete_specs if s.name in affected_pkgs]\n visited: Set[str] = set()\n for depth, parent in traverse.traverse_nodes(\n env_matches, direction=\"parents\", key=traverse.by_dag_hash, depth=True, order=\"breadth\"\n ):\n if dependent_traverse_depth is not None and depth > dependent_traverse_depth:\n break\n affected_specs.update(\n parent.traverse(direction=\"children\", visited=visited, key=traverse.by_dag_hash)\n )\n return affected_specs\n\n\n# Pruning functions should take a spack.spec.Spec object and\n# return a RebuildDecision containing the pruners opinion on\n# whether or not to keep (rebuild) the spec and a message\n# containing the reason for the decision.\n\n\nclass RebuildDecision:\n def __init__(self, rebuild: bool = True, reason: str = \"\"):\n self.rebuild = rebuild\n self.reason = reason\n\n\nPrunerCallback = Callable[[spack.spec.Spec], RebuildDecision]\n\n\ndef create_unaffected_pruner(affected_specs: Set[spack.spec.Spec]) -> PrunerCallback:\n \"\"\"Given a set of \"affected\" specs, return a filter that prunes specs\n not in the set.\"\"\"\n\n def rebuild_filter(s: spack.spec.Spec) -> RebuildDecision:\n if s in affected_specs:\n return RebuildDecision(True, \"affected by change\")\n return RebuildDecision(False, \"unaffected by change\")\n\n return rebuild_filter\n\n\ndef create_already_built_pruner(check_index_only: bool = True) -> PrunerCallback:\n \"\"\"Return a filter that prunes specs already present on any configured\n mirrors\"\"\"\n try:\n spack.binary_distribution.BINARY_INDEX.update()\n except spack.binary_distribution.FetchCacheError as e:\n tty.warn(e)\n\n def rebuild_filter(s: spack.spec.Spec) -> RebuildDecision:\n spec_locations = spack.binary_distribution.get_mirrors_for_spec(\n spec=s, index_only=check_index_only\n )\n\n if not spec_locations:\n return RebuildDecision(True, \"not found anywhere\")\n\n urls = \",\".join(f\"{loc.url}@v{loc.version}\" for loc in spec_locations)\n message = f\"up-to-date [{urls}]\"\n return RebuildDecision(False, message)\n\n return rebuild_filter\n\n\ndef create_external_pruner() -> PrunerCallback:\n \"\"\"Return a filter that prunes external specs\"\"\"\n\n def rebuild_filter(s: spack.spec.Spec) -> RebuildDecision:\n if not s.external:\n return RebuildDecision(True, \"not external\")\n return RebuildDecision(False, \"external spec\")\n\n return rebuild_filter\n\n\ndef _format_pruning_message(spec: spack.spec.Spec, prune: bool, reasons: List[str]) -> str:\n reason_msg = \", \".join(reasons)\n spec_fmt = \"{name}{@version}{/hash:7}{compilers}\"\n\n if not prune:\n status = colorize(\"@*g{[x]} \")\n return f\" {status}{spec.cformat(spec_fmt)} ({reason_msg})\"\n\n msg = f\"{spec.format(spec_fmt)} ({reason_msg})\"\n return colorize(f\" @K - {cescape(msg)}@.\")\n\n\ndef prune_pipeline(\n pipeline: PipelineDag, pruning_filters: List[PrunerCallback], print_summary: bool = False\n) -> None:\n \"\"\"Given a PipelineDag and a list of pruning filters, return a modified\n PipelineDag containing only the nodes that survive pruning by all of the\n filters.\"\"\"\n keys_to_prune = set()\n keys_to_rebuild = set()\n specs: Dict[str, spack.spec.Spec] = {}\n reasons: Dict[str, List[str]] = {}\n\n for _, node in pipeline.traverse_nodes(direction=\"children\"):\n filter_results = [keepSpec(node.spec) for keepSpec in pruning_filters]\n\n reasons[node.key] = [r.reason for r in filter_results]\n specs[node.key] = node.spec\n\n if not all(r.rebuild for r in filter_results):\n keys_to_prune.add(node.key)\n else:\n keys_to_rebuild.add(node.key)\n\n for key in keys_to_prune:\n pipeline.prune(key)\n\n if print_summary:\n sort_key = lambda k: f\"{specs[k].name}/{specs[k].dag_hash(7)}\"\n tty.msg(\"Pipeline pruning summary:\")\n if keys_to_rebuild:\n tty.msg(\" Rebuild list:\")\n for key in sorted(keys_to_rebuild, key=sort_key):\n tty.msg(_format_pruning_message(specs[key], False, reasons[key]))\n if keys_to_prune:\n tty.msg(\" Prune list:\")\n for key in sorted(keys_to_prune, key=sort_key):\n tty.msg(_format_pruning_message(specs[key], True, reasons[key]))\n\n\ndef check_for_broken_specs(pipeline_specs: List[spack.spec.Spec], broken_specs_url: str) -> bool:\n \"\"\"Check the pipeline specs against the list of known broken specs and return\n True if there were any matches, False otherwise.\"\"\"\n if broken_specs_url.startswith(\"http\"):\n # To make checking each spec against the list faster, we require\n # a url protocol that allows us to iterate the url in advance.\n tty.msg(\"Cannot use an http(s) url for broken specs, ignoring\")\n return False\n\n broken_spec_urls = web_util.list_url(broken_specs_url)\n\n if broken_spec_urls is None:\n return False\n\n known_broken_specs_encountered = []\n for release_spec in pipeline_specs:\n release_spec_dag_hash = release_spec.dag_hash()\n if release_spec_dag_hash in broken_spec_urls:\n known_broken_specs_encountered.append(release_spec_dag_hash)\n\n if known_broken_specs_encountered:\n tty.error(\"This pipeline generated hashes known to be broken on develop:\")\n display_broken_spec_messages(broken_specs_url, known_broken_specs_encountered)\n return True\n\n return False\n\n\ndef collect_pipeline_options(env: ev.Environment, args) -> PipelineOptions:\n \"\"\"Gather pipeline options from cli args, spack environment, and\n os environment variables\"\"\"\n pipeline_mirrors = spack.mirrors.mirror.MirrorCollection(binary=True)\n if \"buildcache-destination\" not in pipeline_mirrors:\n raise SpackCIError(\"spack ci generate requires a mirror named 'buildcache-destination'\")\n\n buildcache_destination = pipeline_mirrors[\"buildcache-destination\"]\n options = PipelineOptions(env, buildcache_destination)\n\n options.env = env\n options.artifacts_root = args.artifacts_root\n options.output_file = args.output_file\n options.prune_up_to_date = args.prune_dag\n options.prune_unaffected = args.prune_unaffected\n options.prune_external = args.prune_externals\n options.check_index_only = args.index_only\n options.forward_variables = args.forward_variable or []\n\n ci_config = cfg.get(\"ci\")\n\n cdash_config = cfg.get(\"cdash\")\n if \"build-group\" in cdash_config:\n options.cdash_handler = CDashHandler(cdash_config)\n\n dependent_depth = os.environ.get(\"SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH\", None)\n if dependent_depth is not None:\n try:\n options.untouched_pruning_dependent_depth = int(dependent_depth)\n except (TypeError, ValueError):\n tty.warn(\n f\"Unrecognized value ({dependent_depth}) \"\n \"provided for SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH, \"\n \"ignoring it.\"\n )\n\n spack_prune_untouched = str(os.environ.get(\"SPACK_PRUNE_UNTOUCHED\", options.prune_unaffected))\n options.prune_untouched = (\n spack_prune_untouched is not None and spack_prune_untouched.lower() == \"true\"\n )\n\n # Allow overriding --prune-dag cli opt with environment variable\n prune_dag_override = os.environ.get(\"SPACK_PRUNE_UP_TO_DATE\", None)\n if prune_dag_override is not None:\n options.prune_up_to_date = True if prune_dag_override.lower() == \"true\" else False\n\n options.stack_name = os.environ.get(\"SPACK_CI_STACK_NAME\", None)\n require_signing = os.environ.get(\"SPACK_REQUIRE_SIGNING\", None)\n options.require_signing = (\n True if require_signing and require_signing.lower() == \"true\" else False\n )\n\n # Get the type of pipeline, which is optional\n spack_pipeline_type = os.environ.get(\"SPACK_PIPELINE_TYPE\", None)\n if spack_pipeline_type:\n try:\n options.pipeline_type = PipelineType[spack_pipeline_type]\n except KeyError:\n options.pipeline_type = None\n\n if \"broken-specs-url\" in ci_config:\n options.broken_specs_url = ci_config[\"broken-specs-url\"]\n\n if \"rebuild-index\" in ci_config and ci_config[\"rebuild-index\"] is False:\n options.rebuild_index = False\n\n return options\n\n\ndef get_unaffected_pruners(\n env: ev.Environment, untouched_pruning_dependent_depth: Optional[int]\n) -> Optional[PrunerCallback]:\n\n # If the stack env has changed, do not apply unaffected pruning\n if stack_changed(env.manifest_path):\n tty.info(\"Skipping unaffected pruning: stack environment changed\")\n return None\n\n # TODO: This should be configurable to only check for changed packages\n # in specific configured repos that are being tested with CI. For now\n # it assumes all configured repos are merge commits that contain relevant\n # changes to run CI on.\n affected_pkgs: Set[str] = set()\n for repo in spack.repo.PATH.repos:\n rev1, rev2 = get_change_revisions(repo.root)\n if not (rev1 and rev2):\n continue\n\n tty.debug(f\"repo {repo.namespace}: revisions rev1={rev1}, rev2={rev2}\")\n\n repo_affected_pkgs = compute_affected_packages(repo, rev1=rev1, rev2=rev2)\n tty.debug(f\"repo {repo.namespace}: affected pkgs\")\n for p in repo_affected_pkgs:\n tty.debug(f\" {p}\")\n\n affected_pkgs.update(repo_affected_pkgs)\n\n if not affected_pkgs:\n tty.info(\"Skipping unaffected pruning: no package changes were detected\")\n return None\n\n affected_specs = get_spec_filter_list(\n env, affected_pkgs, dependent_traverse_depth=untouched_pruning_dependent_depth\n )\n tty.debug(f\"dependent_traverse_depth={untouched_pruning_dependent_depth}, affected specs:\")\n for s in affected_specs:\n tty.debug(f\" {PipelineDag.key(s)}\")\n\n return create_unaffected_pruner(affected_specs)\n\n\ndef generate_pipeline(env: ev.Environment, args) -> None:\n \"\"\"Given an environment and the command-line args, generate a pipeline.\n\n Arguments:\n env (spack.environment.Environment): Activated environment object\n which must contain a ci section describing attributes for\n all jobs and a target which should specify an existing\n pipeline generator.\n args: (spack.main.SpackArgumentParser): Parsed arguments from the command\n line.\n \"\"\"\n with env.write_transaction():\n env.concretize()\n env.write()\n\n options = collect_pipeline_options(env, args)\n\n # Get the joined \"ci\" config with all of the current scopes resolved\n ci_config = cfg.get(\"ci\")\n if not ci_config:\n raise SpackCIError(\"Environment does not have a `ci` configuration\")\n\n # Get the target platform we should generate a pipeline for\n ci_target = ci_config.get(\"target\", \"gitlab\")\n try:\n generate_method = get_generator(ci_target)\n except UnknownGeneratorException:\n raise SpackCIError(f\"Spack CI module cannot generate a pipeline for format {ci_target}\")\n\n # If we are not doing any kind of pruning, we are rebuilding everything\n rebuild_everything = not options.prune_up_to_date and not options.prune_untouched\n\n # Build a pipeline from the specs in the concrete environment\n pipeline = PipelineDag([env.specs_by_hash[x.hash] for x in env.concretized_roots])\n\n # Optionally add various pruning filters\n pruning_filters = []\n\n # Possibly prune specs that were unaffected by the change\n if options.prune_untouched:\n # If we don't have two revisions to compare, or if either the spack.yaml\n # associated with the active env or the .gitlab-ci.yml files changed\n # between the provided revisions, then don't do any \"untouched spec\"\n # pruning. Otherwise, list the names of all packages touched between\n # rev1 and rev2, and prune from the pipeline any node whose spec has a\n # packagen name not in that list.\n unaffected_pruner = get_unaffected_pruners(env, options.untouched_pruning_dependent_depth)\n if unaffected_pruner:\n tty.info(\"Enabling Unaffected Pruner\")\n pruning_filters.append(unaffected_pruner)\n\n # Possibly prune specs that are already built on some configured mirror\n if options.prune_up_to_date:\n tty.info(\"Enabling Up-to-date Pruner\")\n pruning_filters.append(\n create_already_built_pruner(check_index_only=options.check_index_only)\n )\n\n # Possibly prune specs that are external\n if options.prune_external:\n tty.info(\"Enabling Externals Pruner\")\n pruning_filters.append(create_external_pruner())\n\n # Do all the pruning\n prune_pipeline(pipeline, pruning_filters, options.print_summary)\n\n # List all specs remaining after any pruning\n pipeline_specs = [n.spec for _, n in pipeline.traverse_nodes(direction=\"children\")]\n\n # If this is configured, spack will fail \"spack ci generate\" if it\n # generates any hash which exists under the broken specs url.\n if options.broken_specs_url and not options.pipeline_type == PipelineType.COPY_ONLY:\n broken = check_for_broken_specs(pipeline_specs, options.broken_specs_url)\n if broken and not rebuild_everything:\n raise SpackCIError(\"spack ci generate failed broken specs check\")\n\n spack_ci_config = SpackCIConfig(ci_config)\n spack_ci_config.init_pipeline_jobs(pipeline)\n\n # Format the pipeline using the formatter specified in the configs\n generate_method(pipeline, spack_ci_config, options)\n\n # Use all unpruned specs to populate the build group for this set\n cdash_config = cfg.get(\"cdash\")\n if options.cdash_handler and options.cdash_handler.auth_token:\n options.cdash_handler.create_buildgroup()\n elif cdash_config:\n # warn only if there was actually a CDash configuration.\n tty.warn(\"Unable to populate buildgroup without CDash credentials\")\n\n\ndef import_signing_key(base64_signing_key: str) -> None:\n \"\"\"Given Base64-encoded gpg key, decode and import it to use for signing packages.\n\n Arguments:\n base64_signing_key:\n A gpg key including the secret key, armor-exported and base64 encoded, so it can be\n stored in a gitlab CI variable. For an example of how to generate such a key, see\n https://github.com/spack/spack-infrastructure/blob/main/gitlab-docker/files/gen-key.\n \"\"\"\n if not base64_signing_key:\n tty.warn(\"No key found for signing/verifying packages\")\n return\n\n tty.debug(\"ci.import_signing_key() will attempt to import a key\")\n\n # This command has the side-effect of creating the directory referred\n # to as GNUPGHOME in setup_environment()\n list_output = spack_gpg(\"list\")\n\n tty.debug(\"spack gpg list:\")\n tty.debug(list_output)\n\n decoded_key = base64.b64decode(base64_signing_key).decode(\"utf-8\")\n\n with tempfile.TemporaryDirectory() as tmpdir:\n sign_key_path = os.path.join(tmpdir, \"signing_key\")\n with open(sign_key_path, \"w\", encoding=\"utf-8\") as fd:\n fd.write(decoded_key)\n\n key_import_output = spack_gpg(\"trust\", sign_key_path)\n tty.debug(f\"spack gpg trust {sign_key_path}\")\n tty.debug(key_import_output)\n\n # Now print the keys we have for verifying and signing\n trusted_keys_output = spack_gpg(\"list\", \"--trusted\")\n signing_keys_output = spack_gpg(\"list\", \"--signing\")\n\n tty.debug(\"spack gpg list --trusted\")\n tty.debug(trusted_keys_output)\n tty.debug(\"spack gpg list --signing\")\n tty.debug(signing_keys_output)\n\n\ndef can_sign_binaries():\n \"\"\"Utility method to determine if this spack instance is capable of\n signing binary packages. This is currently only possible if the\n spack gpg keystore contains exactly one secret key.\"\"\"\n return len(gpg_util.signing_keys()) == 1\n\n\ndef can_verify_binaries():\n \"\"\"Utility method to determine if this spack instance is capable (at\n least in theory) of verifying signed binaries.\"\"\"\n return len(gpg_util.public_keys()) >= 1\n\n\ndef push_to_build_cache(spec: spack.spec.Spec, mirror_url: str, sign_binaries: bool) -> bool:\n \"\"\"Push one or more binary packages to the mirror.\n\n Arguments:\n\n spec: Installed spec to push\n mirror_url: URL of target mirror\n sign_binaries: If True, spack will attempt to sign binary package before pushing.\n \"\"\"\n tty.debug(f\"Pushing to build cache ({'signed' if sign_binaries else 'unsigned'})\")\n signing_key = spack.binary_distribution.select_signing_key() if sign_binaries else None\n mirror = spack.mirrors.mirror.Mirror.from_url(mirror_url)\n try:\n with spack.binary_distribution.make_uploader(mirror, signing_key=signing_key) as uploader:\n uploader.push_or_raise([spec])\n return True\n except spack.binary_distribution.PushToBuildCacheError as e:\n tty.error(f\"Problem writing to {mirror_url}: {e}\")\n return False\n\n\ndef copy_stage_logs_to_artifacts(job_spec: spack.spec.Spec, job_log_dir: str) -> None:\n \"\"\"Copy selected build stage file(s) to the given artifacts directory\n\n Looks for build logs in the stage directory of the given\n job_spec, and attempts to copy the files into the directory given\n by job_log_dir.\n\n Parameters:\n job_spec: spec associated with spack install log\n job_log_dir: path into which build log should be copied\n \"\"\"\n tty.debug(f\"job spec: {job_spec}\")\n if not job_spec.concrete:\n tty.warn(\"Cannot copy artifacts for non-concrete specs\")\n return\n\n package_metadata_root = pathlib.Path(spack.store.STORE.layout.metadata_path(job_spec))\n if not os.path.isdir(package_metadata_root):\n # Fallback to using the stage directory\n job_pkg = job_spec.package\n\n package_metadata_root = pathlib.Path(job_pkg.stage.path)\n archive_files = spack.builder.create(job_pkg).archive_files\n tty.warn(\"Package not installed, falling back to use stage dir\")\n tty.debug(f\"stage dir: {package_metadata_root}\")\n else:\n # Get the package's archived files\n archive_files = []\n archive_root = package_metadata_root / \"archived-files\"\n if os.path.isdir(archive_root):\n archive_files = [str(f) for f in archive_root.rglob(\"*\") if os.path.isfile(f)]\n else:\n tty.debug(f\"No archived files detected at {archive_root}\")\n\n # Try zipped and unzipped versions of the build log\n build_log_zipped = package_metadata_root / \"spack-build-out.txt.gz\"\n build_log = package_metadata_root / \"spack-build-out.txt\"\n build_env_mods = package_metadata_root / \"spack-build-env.txt\"\n\n for f in [build_log_zipped, build_log, build_env_mods, *archive_files]:\n copy_files_to_artifacts(str(f), job_log_dir, compress_artifacts=True)\n\n\ndef copy_test_logs_to_artifacts(test_stage, job_test_dir):\n \"\"\"\n Copy test log file(s) to the given artifacts directory\n\n Parameters:\n test_stage (str): test stage path\n job_test_dir (str): the destination artifacts test directory\n \"\"\"\n tty.debug(f\"test stage: {test_stage}\")\n if not os.path.exists(test_stage):\n tty.error(f\"Cannot copy test logs: job test stage ({test_stage}) does not exist\")\n return\n\n copy_files_to_artifacts(\n os.path.join(test_stage, \"*\", \"*.txt\"), job_test_dir, compress_artifacts=True\n )\n\n\ndef download_and_extract_artifacts(url: str, work_dir: str) -> str:\n \"\"\"Look for gitlab artifacts.zip at the given url, and attempt to download\n and extract the contents into the given work_dir\n\n Arguments:\n url: Complete url to artifacts.zip file\n work_dir: Path to destination where artifacts should be extracted\n\n Returns:\n Artifacts root path relative to the archive root\n \"\"\"\n tty.msg(f\"Fetching artifacts from: {url}\")\n\n headers = {\"Content-Type\": \"application/zip\"}\n\n token = os.environ.get(\"GITLAB_PRIVATE_TOKEN\", None)\n if token:\n headers[\"PRIVATE-TOKEN\"] = token\n\n request = Request(url, headers=headers, method=\"GET\")\n artifacts_zip_path = os.path.join(work_dir, \"artifacts.zip\")\n os.makedirs(work_dir, exist_ok=True)\n\n try:\n with urlopen(request, timeout=SPACK_CDASH_TIMEOUT) as response:\n with open(artifacts_zip_path, \"wb\") as out_file:\n shutil.copyfileobj(response, out_file)\n\n with zipfile.ZipFile(artifacts_zip_path) as zip_file:\n zip_file.extractall(work_dir)\n # Get the artifact root\n artifact_root = \"\"\n for f in zip_file.filelist:\n if \"spack.lock\" in f.filename:\n artifact_root = os.path.dirname(os.path.dirname(f.filename))\n break\n except OSError as e:\n raise SpackError(f\"Error fetching artifacts: {e}\")\n finally:\n try:\n os.remove(artifacts_zip_path)\n except FileNotFoundError:\n # If the file doesn't exist we are already raising\n pass\n\n return artifact_root\n\n\ndef get_spack_info():\n \"\"\"If spack is running from a git repo, return the most recent git log\n entry, otherwise, return a string containing the spack version.\"\"\"\n git_path = os.path.join(spack.paths.prefix, \".git\")\n if os.path.exists(git_path):\n git = spack.util.git.git()\n if git:\n with fs.working_dir(spack.paths.prefix):\n git_log = git(\"log\", \"-1\", output=str, error=os.devnull, fail_on_error=False)\n\n return git_log\n\n return f\"no git repo, use spack {spack.spack_version}\"\n\n\ndef setup_spack_repro_version(\n repro_dir: str, checkout_commit: str, merge_commit: Optional[str] = None\n) -> bool:\n \"\"\"Look in the local spack clone to find the checkout_commit, and if provided, the\n merge_commit given as arguments. If those commits can be found locally, then clone spack and\n attempt to recreate a merge commit with the same parent commits as tested in gitlab. This looks\n something like\n\n 1. ``git clone repo && cd repo``\n 2. ``git checkout ``\n 3. ``git merge ``\n\n If there is no merge_commit provided, then skip step (3).\n\n Arguments:\n\n repro_dir: Location where spack should be cloned\n checkout_commit: SHA of PR branch commit\n merge_commit: SHA of target branch parent\n\n Returns: True iff the git repo state was successfully recreated\n \"\"\"\n # figure out the path to the spack git version being used for the\n # reproduction\n tty.info(f\"checkout_commit: {checkout_commit}\")\n tty.info(f\"merge_commit: {merge_commit}\")\n\n dot_git_path = os.path.join(spack.paths.prefix, \".git\")\n if not os.path.exists(dot_git_path):\n tty.error(\"Unable to find the path to your local spack clone\")\n return False\n\n spack_git_path = spack.paths.prefix\n\n git = spack.util.git.git()\n if not git:\n tty.error(\"reproduction of pipeline job requires git\")\n return False\n\n # Check if we can find the tested commits in your local spack repo\n with fs.working_dir(spack_git_path):\n git(\"log\", \"-1\", checkout_commit, output=str, error=os.devnull, fail_on_error=False)\n\n if git.returncode != 0:\n tty.error(f\"Missing commit: {checkout_commit}\")\n return False\n\n if merge_commit:\n git(\"log\", \"-1\", merge_commit, output=str, error=os.devnull, fail_on_error=False)\n\n if git.returncode != 0:\n tty.error(f\"Missing commit: {merge_commit}\")\n return False\n\n # Next attempt to clone your local spack repo into the repro dir\n with fs.working_dir(repro_dir):\n clone_out = git(\n \"clone\", spack_git_path, \"spack\", output=str, error=os.devnull, fail_on_error=False\n )\n\n if git.returncode != 0:\n tty.error(\"Unable to clone your local spack repo:\")\n tty.msg(clone_out)\n return False\n\n # Finally, attempt to put the cloned repo into the same state used during\n # the pipeline build job\n repro_spack_path = os.path.join(repro_dir, \"spack\")\n with fs.working_dir(repro_spack_path):\n co_out = git(\n \"checkout\", checkout_commit, output=str, error=os.devnull, fail_on_error=False\n )\n\n if git.returncode != 0:\n tty.error(f\"Unable to checkout {checkout_commit}\")\n tty.msg(co_out)\n return False\n\n if merge_commit:\n merge_out = git(\n \"-c\",\n \"user.name=cirepro\",\n \"-c\",\n \"user.email=user@email.org\",\n \"merge\",\n \"--no-edit\",\n merge_commit,\n output=str,\n error=os.devnull,\n fail_on_error=False,\n )\n\n if git.returncode != 0:\n tty.error(f\"Unable to merge {merge_commit}\")\n tty.msg(merge_out)\n return False\n\n return True\n\n\ndef reproduce_ci_job(url, work_dir, autostart, gpg_url, runtime, use_local_head):\n \"\"\"Given a url to gitlab artifacts.zip from a failed ``spack ci rebuild`` job,\n attempt to setup an environment in which the failure can be reproduced\n locally. This entails the following:\n\n First download and extract artifacts. Then look through those artifacts\n to glean some information needed for the reproduer (e.g. one of the\n artifacts contains information about the version of spack tested by\n gitlab, another is the generated pipeline yaml containing details\n of the job like the docker image used to run it). The output of this\n function is a set of printed instructions for running docker and then\n commands to run to reproduce the build once inside the container.\n \"\"\"\n work_dir = os.path.realpath(work_dir)\n if os.path.exists(work_dir) and os.listdir(work_dir):\n raise SpackError(f\"Cannot run reproducer in non-empty working dir:\\n {work_dir}\")\n\n platform_script_ext = \"ps1\" if IS_WINDOWS else \"sh\"\n artifact_root = download_and_extract_artifacts(url, work_dir)\n\n gpg_path = None\n if gpg_url:\n gpg_path = web_util.fetch_url_text(gpg_url, dest_dir=os.path.join(work_dir, \"_pgp\"))\n rel_gpg_path = gpg_path.replace(work_dir, \"\").lstrip(os.path.sep)\n\n lock_file = fs.find(work_dir, \"spack.lock\")[0]\n repro_lock_dir = os.path.dirname(lock_file)\n\n tty.debug(f\"Found lock file in: {repro_lock_dir}\")\n\n yaml_files = fs.find(work_dir, [\"*.yaml\", \"*.yml\"])\n\n tty.debug(\"yaml files:\")\n for yaml_file in yaml_files:\n tty.debug(f\" {yaml_file}\")\n\n pipeline_yaml = None\n\n # Try to find the dynamically generated pipeline yaml file in the\n # reproducer. If the user did not put it in the artifacts root,\n # but rather somewhere else and exported it as an artifact from\n # that location, we won't be able to find it.\n for yf in yaml_files:\n with open(yf, encoding=\"utf-8\") as y_fd:\n yaml_obj = syaml.load(y_fd)\n if \"variables\" in yaml_obj and \"stages\" in yaml_obj:\n pipeline_yaml = yaml_obj\n\n if pipeline_yaml:\n tty.debug(f\"\\n{yf} is likely your pipeline file\")\n\n relative_concrete_env_dir = pipeline_yaml[\"variables\"][\"SPACK_CONCRETE_ENV_DIR\"]\n tty.debug(f\"Relative environment path used by cloud job: {relative_concrete_env_dir}\")\n\n # Using the relative concrete environment path found in the generated\n # pipeline variable above, copy the spack environment files so they'll\n # be found in the same location as when the job ran in the cloud.\n concrete_env_dir = os.path.join(work_dir, relative_concrete_env_dir)\n os.makedirs(concrete_env_dir, exist_ok=True)\n copy_lock_path = os.path.join(concrete_env_dir, \"spack.lock\")\n orig_yaml_path = os.path.join(repro_lock_dir, \"spack.yaml\")\n copy_yaml_path = os.path.join(concrete_env_dir, \"spack.yaml\")\n shutil.copyfile(lock_file, copy_lock_path)\n shutil.copyfile(orig_yaml_path, copy_yaml_path)\n\n # Find the install script in the unzipped artifacts and make it executable\n install_script = fs.find(work_dir, f\"install.{platform_script_ext}\")[0]\n if not IS_WINDOWS:\n # pointless on Windows\n st = os.stat(install_script)\n os.chmod(install_script, st.st_mode | stat.S_IEXEC)\n # Find the repro details file. This just includes some values we wrote\n # during `spack ci rebuild` to make reproduction easier. E.g. the job\n # name is written here so we can easily find the configuration of the\n # job from the generated pipeline file.\n repro_file = fs.find(work_dir, \"repro.json\")[0]\n repro_details = None\n with open(repro_file, encoding=\"utf-8\") as fd:\n repro_details = json.load(fd)\n\n spec_file = fs.find(work_dir, repro_details[\"job_spec_json\"])[0]\n reproducer_spec = spack.spec.Spec.from_specfile(spec_file)\n\n repro_dir = os.path.dirname(repro_file)\n rel_repro_dir = repro_dir.replace(work_dir, \"\").lstrip(os.path.sep)\n\n # Find the spack info text file that should contain the git log\n # of the HEAD commit used during the CI build\n spack_info_file = fs.find(work_dir, \"spack_info.txt\")[0]\n with open(spack_info_file, encoding=\"utf-8\") as fd:\n spack_info = fd.read()\n\n # Access the specific job configuration\n job_name = repro_details[\"job_name\"]\n job_yaml = None\n\n if job_name in pipeline_yaml:\n job_yaml = pipeline_yaml[job_name]\n\n if job_yaml:\n tty.debug(\"Found job:\")\n tty.debug(job_yaml)\n\n job_image = None\n setup_result = False\n if \"image\" in job_yaml:\n job_image_elt = job_yaml[\"image\"]\n if \"name\" in job_image_elt:\n job_image = job_image_elt[\"name\"]\n else:\n job_image = job_image_elt\n tty.msg(f\"Job ran with the following image: {job_image}\")\n\n # Because we found this job was run with a docker image, so we will try\n # to print a \"docker run\" command that bind-mounts the directory where\n # we extracted the artifacts.\n\n # Destination of bind-mounted reproduction directory. It makes for a\n # more faithful reproducer if everything appears to run in the same\n # absolute path used during the CI build.\n mount_as_dir = \"/work\"\n mounted_workdir = \"/reproducer\"\n if repro_details:\n mount_as_dir = repro_details[\"ci_project_dir\"]\n mounted_repro_dir = os.path.join(mount_as_dir, rel_repro_dir)\n mounted_env_dir = os.path.join(mount_as_dir, relative_concrete_env_dir)\n if gpg_path:\n mounted_gpg_path = os.path.join(mounted_workdir, rel_gpg_path)\n\n # We will also try to clone spack from your local checkout and\n # reproduce the state present during the CI build, and put that into\n # the bind-mounted reproducer directory.\n\n # Regular expressions for parsing that HEAD commit. If the pipeline\n # was on the gitlab spack mirror, it will have been a merge commit made by\n # github and pushed by the sync script. If the pipeline was run on some\n # environment repo, then the tested spack commit will likely have been\n # a regular commit.\n commit_1 = None\n commit_2 = None\n commit_regex = re.compile(r\"commit\\s+([^\\s]+)\")\n merge_commit_regex = re.compile(r\"Merge\\s+([^\\s]+)\\s+into\\s+([^\\s]+)\")\n\n if use_local_head:\n commit_1 = \"HEAD\"\n else:\n # Try the more specific merge commit regex first\n m = merge_commit_regex.search(spack_info)\n if m:\n # This was a merge commit and we captured the parents\n commit_1 = m.group(1)\n commit_2 = m.group(2)\n else:\n # Not a merge commit, just get the commit sha\n m = commit_regex.search(spack_info)\n if m:\n commit_1 = m.group(1)\n\n setup_result = False\n if commit_1:\n if commit_2:\n setup_result = setup_spack_repro_version(work_dir, commit_2, merge_commit=commit_1)\n else:\n setup_result = setup_spack_repro_version(work_dir, commit_1)\n\n if not setup_result:\n setup_msg = \"\"\"\n This can happen if the spack you are using to run this command is not a git\n repo, or if it is a git repo, but it does not have the commits needed to\n recreate the tested merge commit. If you are trying to reproduce a spack\n PR pipeline job failure, try fetching the latest develop commits from\n mainline spack and make sure you have the most recent commit of the PR\n branch in your local spack repo. Then run this command again.\n Alternatively, you can also manually clone spack if you know the version\n you want to test.\n \"\"\"\n tty.error(\n \"Failed to automatically setup the tested version of spack \"\n \"in your local reproduction directory.\"\n )\n tty.info(setup_msg)\n\n # In cases where CI build was run on a shell runner, it might be useful\n # to see what tags were applied to the job so the user knows what shell\n # runner was used. But in that case in general, we cannot do nearly as\n # much to set up the reproducer.\n job_tags = None\n if \"tags\" in job_yaml:\n job_tags = job_yaml[\"tags\"]\n tty.msg(f\"Job ran with the following tags: {job_tags}\")\n\n entrypoint_script = [\n [\"git\", \"config\", \"--global\", \"--add\", \"safe.directory\", mount_as_dir],\n [\n \".\",\n os.path.join(\n mount_as_dir if job_image else work_dir,\n f\"share/spack/setup-env.{platform_script_ext}\",\n ),\n ],\n [\"spack\", \"gpg\", \"trust\", mounted_gpg_path if job_image else gpg_path] if gpg_path else [],\n [\"spack\", \"env\", \"activate\", mounted_env_dir if job_image else repro_dir],\n [\n (\n os.path.join(mounted_repro_dir, f\"install.{platform_script_ext}\")\n if job_image\n else install_script\n )\n ],\n ]\n entry_script = os.path.join(mounted_workdir, f\"entrypoint.{platform_script_ext}\")\n inst_list = []\n # Finally, print out some instructions to reproduce the build\n if job_image:\n # Allow interactive\n install_mechanism = (\n os.path.join(mounted_repro_dir, f\"install.{platform_script_ext}\")\n if job_image\n else install_script\n )\n entrypoint_script.append([\"echo\", f\"Re-run install script using:\\n\\t{install_mechanism}\"])\n # Allow interactive\n if IS_WINDOWS:\n entrypoint_script.append([\"&\", \"($args -Join ' ')\", \"-NoExit\"])\n else:\n entrypoint_script.append([\"exec\", \"$@\"])\n\n process_command(\n \"entrypoint\", entrypoint_script, work_dir, run=False, exit_on_failure=False\n )\n\n # Attempt to create a unique name for the reproducer container\n container_suffix = \"_\" + reproducer_spec.dag_hash() if reproducer_spec else \"\"\n docker_command = [\n runtime,\n \"run\",\n \"-i\",\n \"-t\",\n \"--rm\",\n \"--name\",\n f\"spack_reproducer{container_suffix}\",\n \"-v\",\n \":\".join([work_dir, mounted_workdir, \"Z\"]),\n \"-v\",\n \":\".join(\n [\n os.path.join(work_dir, artifact_root),\n os.path.join(mount_as_dir, artifact_root),\n \"Z\",\n ]\n ),\n \"-v\",\n \":\".join([os.path.join(work_dir, \"spack\"), mount_as_dir, \"Z\"]),\n \"--entrypoint\",\n ]\n if IS_WINDOWS:\n docker_command.extend([\"powershell.exe\", job_image, entry_script, \"powershell.exe\"])\n else:\n docker_command.extend([entry_script, job_image, \"bash\"])\n docker_command = [docker_command]\n autostart = autostart and setup_result\n process_command(\"start\", docker_command, work_dir, run=autostart)\n\n if not autostart:\n inst_list.append(\"\\nTo run the docker reproducer:\\n\\n\")\n inst_list.extend(\n [\n \" - Start the docker container install\",\n f\" $ {work_dir}/start.{platform_script_ext}\",\n ]\n )\n else:\n autostart = autostart and setup_result\n process_command(\"reproducer\", entrypoint_script, work_dir, run=autostart)\n\n inst_list.append(\"\\nOnce on the tagged runner:\\n\\n\")\n inst_list.extend(\n [\n \" - Run the reproducer script\",\n f\" $ {work_dir}/reproducer.{platform_script_ext}\",\n ]\n )\n\n if not setup_result:\n inst_list.append(\"\\n - Clone spack and acquire tested commit\")\n inst_list.append(f\"\\n {spack_info}\\n\")\n inst_list.append(\"\\n\")\n inst_list.append(f\"\\n Path to clone spack: {work_dir}/spack\\n\\n\")\n\n tty.msg(\"\".join(inst_list))\n\n\ndef process_command(name, commands, repro_dir, run=True, exit_on_failure=True):\n \"\"\"\n Create a script for and run the command. Copy the script to the\n reproducibility directory.\n\n Arguments:\n name (str): name of the command being processed\n commands (list): list of arguments for single command or list of lists of\n arguments for multiple commands. No shell escape is performed.\n repro_dir (str): Job reproducibility directory\n run (bool): Run the script and return the exit code if True\n\n Returns: the exit code from processing the command\n \"\"\"\n\n tty.debug(f\"spack {name} arguments: {commands}\")\n if len(commands) == 0 or isinstance(commands[0], str):\n commands = [commands]\n\n def compose_command_err_handling(args):\n if not IS_WINDOWS:\n args = [f'\"{arg}\"' for arg in args]\n arg_str = \" \".join(args)\n result = arg_str + \"\\n\"\n # ErrorActionPreference will handle PWSH commandlets (Spack calls),\n # but we need to handle EXEs (git, etc) ourselves\n catch_exe_failure = (\n \"\"\"\nif ($LASTEXITCODE -ne 0){{\n throw 'Command {} has failed'\n}}\n\"\"\"\n if IS_WINDOWS\n else \"\"\n )\n if exit_on_failure and catch_exe_failure:\n result += catch_exe_failure.format(arg_str)\n return result\n\n # Create a string [command 1] \\n [command 2] \\n ... \\n [command n] with\n # commands composed into a platform dependent shell script, pwsh on Windows,\n full_command = \"\\n\".join(map(compose_command_err_handling, commands))\n # Write the command to a python script\n if IS_WINDOWS:\n script = f\"{name}.ps1\"\n script_content = [f\"\\n# spack {name} command\\n\"]\n if exit_on_failure:\n script_content.append('$ErrorActionPreference = \"Stop\"\\n')\n if os.environ.get(\"SPACK_VERBOSE_SCRIPT\"):\n script_content.append(\"Set-PSDebug -Trace 2\\n\")\n else:\n script = f\"{name}.sh\"\n script_content = [\"#!/bin/sh\\n\\n\", f\"\\n# spack {name} command\\n\"]\n if exit_on_failure:\n script_content.append(\"set -e\\n\")\n if os.environ.get(\"SPACK_VERBOSE_SCRIPT\"):\n script_content.append(\"set -x\\n\")\n script_content.append(full_command)\n script_content.append(\"\\n\")\n\n with open(script, \"w\", encoding=\"utf-8\") as fd:\n for line in script_content:\n fd.write(line)\n\n copy_path = os.path.join(repro_dir, script)\n shutil.copyfile(script, copy_path)\n if not IS_WINDOWS:\n st = os.stat(copy_path)\n os.chmod(copy_path, st.st_mode | stat.S_IEXEC)\n\n # Run the generated shell script as if it were being run in\n # a login shell.\n exit_code = None\n if run:\n try:\n # We use sh as executor on Linux like platforms, pwsh on Windows\n interpreter = \"powershell.exe\" if IS_WINDOWS else \"/bin/sh\"\n cmd_process = subprocess.Popen([interpreter, f\"./{script}\"])\n cmd_process.wait()\n exit_code = cmd_process.returncode\n except (ValueError, subprocess.CalledProcessError, OSError) as err:\n tty.error(f\"Encountered error running {name} script\")\n tty.error(err)\n exit_code = 1\n\n tty.debug(f\"spack {name} exited {exit_code}\")\n else:\n # Delete the script, it is copied to the destination dir\n os.remove(script)\n\n return exit_code\n\n\ndef create_buildcache(\n input_spec: spack.spec.Spec, *, destination_mirror_urls: List[str], sign_binaries: bool = False\n) -> List[PushResult]:\n \"\"\"Create the buildcache at the provided mirror(s).\n\n Arguments:\n input_spec: Installed spec to package and push\n destination_mirror_urls: List of urls to push to\n sign_binaries: Whether or not to sign buildcache entry\n\n Returns: A list of PushResults, indicating success or failure.\n \"\"\"\n results = []\n\n for mirror_url in destination_mirror_urls:\n results.append(\n PushResult(\n success=push_to_build_cache(input_spec, mirror_url, sign_binaries), url=mirror_url\n )\n )\n\n return results\n\n\ndef write_broken_spec(url, pkg_name, stack_name, job_url, pipeline_url, spec_dict):\n \"\"\"Given a url to write to and the details of the failed job, write an entry\n in the broken specs list.\n \"\"\"\n with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir:\n file_path = os.path.join(tmpdir, \"broken.txt\")\n\n broken_spec_details = {\n \"broken-spec\": {\n \"job-name\": pkg_name,\n \"job-stack\": stack_name,\n \"job-url\": job_url,\n \"pipeline-url\": pipeline_url,\n \"concrete-spec-dict\": spec_dict,\n }\n }\n\n try:\n with open(file_path, \"w\", encoding=\"utf-8\") as fd:\n syaml.dump(broken_spec_details, fd)\n web_util.push_to_url(\n file_path, url, keep_original=False, extra_args={\"ContentType\": \"text/plain\"}\n )\n except Exception as err:\n # If there is an S3 error (e.g., access denied or connection\n # error), the first non boto-specific class in the exception\n # hierarchy is Exception. Just print a warning and return\n msg = f\"Error writing to broken specs list {url}: {err}\"\n tty.warn(msg)\n\n\ndef read_broken_spec(broken_spec_url):\n \"\"\"Read data from broken specs file located at the url, return as a yaml\n object.\n \"\"\"\n try:\n broken_spec_contents = web_util.read_text(broken_spec_url)\n except web_util.SpackWebError:\n tty.warn(f\"Unable to read broken spec from {broken_spec_url}\")\n return None\n\n return syaml.load(broken_spec_contents)\n\n\ndef display_broken_spec_messages(base_url, hashes):\n \"\"\"Fetch the broken spec file for each of the hashes under the base_url and\n print a message with some details about each one.\n \"\"\"\n broken_specs = [(h, read_broken_spec(url_util.join(base_url, h))) for h in hashes]\n for spec_hash, broken_spec in [tup for tup in broken_specs if tup[1]]:\n details = broken_spec[\"broken-spec\"]\n if \"job-name\" in details:\n item_name = f\"{details['job-name']}/{spec_hash[:7]}\"\n else:\n item_name = spec_hash\n\n if \"job-stack\" in details:\n item_name = f\"{item_name} (in stack {details['job-stack']})\"\n\n msg = f\" {item_name} was reported broken here: {details['job-url']}\"\n tty.msg(msg)\n\n\ndef run_standalone_tests(\n *,\n cdash: Optional[CDashHandler] = None,\n fail_fast: bool = False,\n log_file: Optional[str] = None,\n job_spec: Optional[spack.spec.Spec] = None,\n repro_dir: Optional[str] = None,\n timeout: Optional[int] = None,\n):\n \"\"\"Run stand-alone tests on the current spec.\n\n Args:\n cdash: cdash handler instance\n fail_fast: terminate tests after the first failure\n log_file: test log file name if NOT CDash reporting\n job_spec: spec that was built\n repro_dir: reproduction directory\n timeout: maximum time (in seconds) that tests are allowed to run\n \"\"\"\n if cdash and log_file:\n tty.msg(f\"The test log file {log_file} option is ignored with CDash reporting\")\n log_file = None\n\n # Error out but do NOT terminate if there are missing required arguments.\n if not job_spec:\n tty.error(\"Job spec is required to run stand-alone tests\")\n return\n\n if not repro_dir:\n tty.error(\"Reproduction directory is required for stand-alone tests\")\n return\n\n test_args = [\"spack\", \"--color=always\", \"--backtrace\", \"--verbose\", \"test\", \"run\"]\n if fail_fast:\n test_args.append(\"--fail-fast\")\n\n if timeout is not None:\n test_args.extend([\"--timeout\", str(timeout)])\n\n if cdash:\n test_args.extend(cdash.args())\n else:\n test_args.extend([\"--log-format\", \"junit\"])\n if log_file:\n test_args.extend([\"--log-file\", log_file])\n test_args.append(job_spec.name)\n\n tty.debug(f\"Running {job_spec.name} stand-alone tests\")\n exit_code = process_command(\"test\", test_args, repro_dir)\n\n tty.debug(f\"spack test exited {exit_code}\")\n" }, { "path": "lib/spack/spack/ci/common.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport copy\nimport errno\nimport glob\nimport gzip\nimport json\nimport os\nimport re\nimport shutil\nimport sys\nimport time\nfrom collections import deque\nfrom enum import Enum\nfrom typing import Dict, Generator, List, Optional, Set, Tuple\nfrom urllib.parse import quote, urlencode, urlparse\nfrom urllib.request import Request\n\nimport spack.binary_distribution\nimport spack.config as cfg\nimport spack.deptypes as dt\nimport spack.environment as ev\nimport spack.error\nimport spack.llnl.util.filesystem as fs\nimport spack.llnl.util.tty as tty\nimport spack.mirrors.mirror\nimport spack.schema\nimport spack.spec\nimport spack.util.compression as compression\nimport spack.util.web as web_util\nfrom spack import traverse\nfrom spack.llnl.util.lang import memoized\nfrom spack.reporters import CDash, CDashConfiguration\nfrom spack.reporters.cdash import SPACK_CDASH_TIMEOUT\nfrom spack.reporters.cdash import build_stamp as cdash_build_stamp\nfrom spack.url_buildcache import get_url_buildcache_class\n\nIS_WINDOWS = sys.platform == \"win32\"\nSPACK_RESERVED_TAGS = [\"public\", \"protected\", \"notary\"]\n\n# this exists purely for testing purposes\n_urlopen = web_util.urlopen\n\n\ndef copy_gzipped(glob_or_path: str, dest: str) -> None:\n \"\"\"Copy all of the files in the source glob/path to the destination.\n\n Args:\n glob_or_path: path to file to test\n dest: destination path to copy to\n \"\"\"\n\n files = glob.glob(glob_or_path)\n if not files:\n raise OSError(\"No such file or directory: '{0}'\".format(glob_or_path), errno.ENOENT)\n if len(files) > 1 and not os.path.isdir(dest):\n raise ValueError(\n \"'{0}' matches multiple files but '{1}' is not a directory\".format(glob_or_path, dest)\n )\n\n def is_gzipped(path):\n with open(path, \"rb\") as fd:\n return compression.GZipFileType().matches_magic(fd)\n\n for src in files:\n if is_gzipped(src):\n fs.copy(src, dest)\n else:\n # Compress and copy in one step\n src_name = os.path.basename(src)\n if os.path.isdir(dest):\n zipped = os.path.join(dest, f\"{src_name}.gz\")\n elif not dest.endswith(\".gz\"):\n zipped = f\"{dest}.gz\"\n else:\n zipped = dest\n\n with open(src, \"rb\") as fin, gzip.open(zipped, \"wb\") as fout:\n shutil.copyfileobj(fin, fout)\n\n\ndef copy_files_to_artifacts(\n src: str, artifacts_dir: str, *, compress_artifacts: bool = False\n) -> None:\n \"\"\"\n Copy file(s) to the given artifacts directory\n\n Args:\n src (str): the glob-friendly path expression for the file(s) to copy\n artifacts_dir (str): the destination directory\n compress_artifacts (bool): option to compress copied artifacts using Gzip\n \"\"\"\n try:\n if compress_artifacts:\n copy_gzipped(src, artifacts_dir)\n else:\n fs.copy(src, artifacts_dir)\n except Exception as err:\n tty.warn(\n (\n f\"Unable to copy files ({src}) to artifacts {artifacts_dir} due to \"\n f\"exception: {str(err)}\"\n )\n )\n\n\ndef win_quote(quote_str: str) -> str:\n if IS_WINDOWS:\n quote_str = f'\"{quote_str}\"'\n return quote_str\n\n\ndef _spec_matches(spec, match_string):\n return spec.intersects(match_string)\n\n\ndef _noop(x):\n return x\n\n\ndef unpack_script(script_section, op=_noop):\n script = []\n for cmd in script_section:\n if isinstance(cmd, list):\n for subcmd in cmd:\n script.append(op(subcmd))\n else:\n script.append(op(cmd))\n\n return script\n\n\ndef ensure_expected_target_path(path: str) -> str:\n \"\"\"Returns passed paths with all Windows path separators exchanged\n for posix separators\n\n TODO (johnwparent): Refactor config + cli read/write to deal only in posix style paths\n \"\"\"\n if path:\n return path.replace(\"\\\\\", \"/\")\n return path\n\n\ndef write_pipeline_manifest(specs, src_prefix, dest_prefix, output_file):\n \"\"\"Write out the file describing specs that should be copied\"\"\"\n buildcache_copies = {}\n\n for release_spec in specs:\n release_spec_dag_hash = release_spec.dag_hash()\n cache_class = get_url_buildcache_class(\n layout_version=spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION\n )\n buildcache_copies[release_spec_dag_hash] = {\n \"src\": cache_class.get_manifest_url(release_spec, src_prefix),\n \"dest\": cache_class.get_manifest_url(release_spec, dest_prefix),\n }\n\n target_dir = os.path.dirname(output_file)\n\n if not os.path.exists(target_dir):\n os.makedirs(target_dir)\n\n with open(output_file, \"w\", encoding=\"utf-8\") as fd:\n fd.write(json.dumps(buildcache_copies))\n\n\nclass CDashHandler:\n \"\"\"\n Class for managing CDash data and processing.\n \"\"\"\n\n def __init__(self, ci_cdash):\n # start with the gitlab ci configuration\n self.url = ci_cdash.get(\"url\")\n self.build_group = ci_cdash.get(\"build-group\")\n self.project = ci_cdash.get(\"project\")\n self.site = ci_cdash.get(\"site\")\n\n # grab the authorization token when available\n self.auth_token = os.environ.get(\"SPACK_CDASH_AUTH_TOKEN\")\n if self.auth_token:\n tty.verbose(\"Using CDash auth token from environment\")\n\n # append runner description to the site if available\n runner = os.environ.get(\"CI_RUNNER_DESCRIPTION\")\n if runner:\n self.site += f\" ({runner})\"\n\n def args(self):\n return [\n \"--cdash-upload-url\",\n win_quote(self.upload_url),\n \"--cdash-build\",\n win_quote(self.build_name()),\n \"--cdash-site\",\n win_quote(self.site),\n \"--cdash-buildstamp\",\n win_quote(self.build_stamp),\n ]\n\n def build_name(self, spec: Optional[spack.spec.Spec] = None) -> Optional[str]:\n \"\"\"Returns the CDash build name.\n\n A name will be generated if the ``spec`` is provided,\n otherwise, the value will be retrieved from the environment\n through the ``SPACK_CDASH_BUILD_NAME`` variable.\n\n Returns: (str) given spec's CDash build name.\"\"\"\n if spec:\n spec_str = spec.format(\"{name}{@version}{%compiler} hash={hash} arch={architecture}\")\n build_name = f\"{spec_str} ({self.build_group})\"\n tty.debug(f\"Generated CDash build name ({build_name}) from the {spec.name}\")\n return build_name\n\n env_build_name = os.environ.get(\"SPACK_CDASH_BUILD_NAME\")\n tty.debug(f\"Using CDash build name ({env_build_name}) from the environment\")\n return env_build_name\n\n @property # type: ignore\n def build_stamp(self):\n \"\"\"Returns the CDash build stamp.\n\n The one defined by SPACK_CDASH_BUILD_STAMP environment variable\n is preferred due to the representation of timestamps; otherwise,\n one will be built.\n\n Returns: (str) current CDash build stamp\"\"\"\n build_stamp = os.environ.get(\"SPACK_CDASH_BUILD_STAMP\")\n if build_stamp:\n tty.debug(f\"Using build stamp ({build_stamp}) from the environment\")\n return build_stamp\n\n build_stamp = cdash_build_stamp(self.build_group, time.time())\n tty.debug(f\"Generated new build stamp ({build_stamp})\")\n return build_stamp\n\n @property # type: ignore\n @memoized\n def project_enc(self):\n tty.debug(f\"Encoding project ({type(self.project)}): {self.project})\")\n encode = urlencode({\"project\": self.project})\n index = encode.find(\"=\") + 1\n return encode[index:]\n\n @property\n def upload_url(self):\n url_format = f\"{self.url}/submit.php?project={self.project_enc}\"\n return url_format\n\n def copy_test_results(self, source, dest):\n \"\"\"Copy test results to artifacts directory.\"\"\"\n reports = fs.join_path(source, \"*_Test*.xml\")\n copy_files_to_artifacts(reports, dest)\n\n def create_buildgroup(self):\n \"\"\"Create the CDash buildgroup if it does not already exist.\"\"\"\n headers = {\n \"Authorization\": f\"Bearer {self.auth_token}\",\n \"Content-Type\": \"application/json\",\n }\n data = {\"newbuildgroup\": self.build_group, \"project\": self.project, \"type\": \"Daily\"}\n enc_data = json.dumps(data).encode(\"utf-8\")\n request = Request(f\"{self.url}/api/v1/buildgroup.php\", data=enc_data, headers=headers)\n\n response_text = None\n group_id = None\n\n try:\n with _urlopen(request, timeout=SPACK_CDASH_TIMEOUT) as response:\n response_text = response.read()\n except OSError as e:\n tty.warn(f\"Failed to create CDash buildgroup: {e}\")\n\n if response_text:\n try:\n response_json = json.loads(response_text)\n group_id = response_json[\"id\"]\n except (json.JSONDecodeError, KeyError) as e:\n tty.warn(f\"Failed to parse CDash response: {e}\")\n\n if not group_id:\n tty.warn(f\"Failed to create or retrieve buildgroup for {self.build_group}\")\n\n def report_skipped(self, spec: spack.spec.Spec, report_dir: str, reason: Optional[str]):\n \"\"\"Explicitly report skipping testing of a spec (e.g., it's CI\n configuration identifies it as known to have broken tests or\n the CI installation failed).\n\n Args:\n spec: spec being tested\n report_dir: directory where the report will be written\n reason: reason the test is being skipped\n \"\"\"\n configuration = CDashConfiguration(\n upload_url=self.upload_url,\n packages=[spec.name],\n build=self.build_name(),\n site=self.site,\n buildstamp=self.build_stamp,\n track=None,\n )\n reporter = CDash(configuration=configuration)\n reporter.test_skipped_report(report_dir, spec, reason)\n\n\nclass PipelineType(Enum):\n COPY_ONLY = 1\n spack_copy_only = 1\n PROTECTED_BRANCH = 2\n spack_protected_branch = 2\n PULL_REQUEST = 3\n spack_pull_request = 3\n\n\nclass PipelineOptions:\n \"\"\"A container for all pipeline options that can be specified (whether\n via cli, config/yaml, or environment variables)\"\"\"\n\n def __init__(\n self,\n env: ev.Environment,\n buildcache_destination: spack.mirrors.mirror.Mirror,\n artifacts_root: str = \"jobs_scratch_dir\",\n print_summary: bool = True,\n output_file: Optional[str] = None,\n check_index_only: bool = False,\n broken_specs_url: Optional[str] = None,\n rebuild_index: bool = True,\n untouched_pruning_dependent_depth: Optional[int] = None,\n prune_untouched: bool = False,\n prune_up_to_date: bool = True,\n prune_unaffected: bool = True,\n prune_external: bool = True,\n stack_name: Optional[str] = None,\n pipeline_type: Optional[PipelineType] = None,\n require_signing: bool = False,\n cdash_handler: Optional[\"CDashHandler\"] = None,\n ):\n \"\"\"\n Args:\n env: Active spack environment\n buildcache_destination: The mirror where built binaries should be pushed\n artifacts_root: Path to location where artifacts should be stored\n print_summary: Print a summary of the scheduled pipeline\n output_file: Path where output file should be written\n check_index_only: Only fetch the index or fetch all spec files\n broken_specs_url: URL where broken specs (on develop) should be reported\n rebuild_index: Generate a job to rebuild mirror index after rebuilds\n untouched_pruning_dependent_depth: How many parents to traverse from changed pkg specs\n prune_untouched: Prune jobs for specs that were unchanged in git history\n prune_up_to_date: Prune specs from pipeline if binary exists on the mirror\n prune_external: Prune specs from pipeline if they are external\n stack_name: Name of spack stack\n pipeline_type: Type of pipeline running (optional)\n require_signing: Require buildcache to be signed (fail w/out signing key)\n cdash_handler: Object for communicating build information with CDash\n \"\"\"\n self.env = env\n self.buildcache_destination = buildcache_destination\n self.artifacts_root = artifacts_root\n self.print_summary = print_summary\n self.output_file = output_file\n self.check_index_only = check_index_only\n self.broken_specs_url = broken_specs_url\n self.rebuild_index = rebuild_index\n self.untouched_pruning_dependent_depth = untouched_pruning_dependent_depth\n self.prune_untouched = prune_untouched\n self.prune_up_to_date = prune_up_to_date\n self.prune_unaffected = prune_unaffected\n self.prune_external = prune_external\n self.stack_name = stack_name\n self.pipeline_type = pipeline_type\n self.require_signing = require_signing\n self.cdash_handler = cdash_handler\n self.forward_variables: List[str] = []\n\n\nclass PipelineNode:\n spec: spack.spec.Spec\n parents: Set[str]\n children: Set[str]\n\n def __init__(self, spec: spack.spec.Spec):\n self.spec = spec\n self.parents = set()\n self.children = set()\n\n @property\n def key(self):\n \"\"\"Return key of the stored spec\"\"\"\n return PipelineDag.key(self.spec)\n\n\nclass PipelineDag:\n \"\"\"Turn a list of specs into a simple directed graph, that doesn't keep track\n of edge types.\"\"\"\n\n @classmethod\n def key(cls, spec: spack.spec.Spec) -> str:\n return spec.dag_hash()\n\n def __init__(self, specs: List[spack.spec.Spec]) -> None:\n # Build dictionary of nodes\n self.nodes: Dict[str, PipelineNode] = {\n PipelineDag.key(s): PipelineNode(s)\n for s in traverse.traverse_nodes(specs, deptype=dt.ALL_TYPES, root=True)\n }\n\n # Create edges\n for edge in traverse.traverse_edges(\n specs, deptype=dt.ALL_TYPES, root=False, cover=\"edges\"\n ):\n parent_key = PipelineDag.key(edge.parent)\n child_key = PipelineDag.key(edge.spec)\n\n self.nodes[parent_key].children.add(child_key)\n self.nodes[child_key].parents.add(parent_key)\n\n def prune(self, node_key: str):\n \"\"\"Remove a node from the graph, and reconnect its parents and children\"\"\"\n node = self.nodes[node_key]\n for parent in node.parents:\n self.nodes[parent].children.remove(node_key)\n self.nodes[parent].children |= node.children\n for child in node.children:\n self.nodes[child].parents.remove(node_key)\n self.nodes[child].parents |= node.parents\n del self.nodes[node_key]\n\n def traverse_nodes(\n self, direction: str = \"children\"\n ) -> Generator[Tuple[int, PipelineNode], None, None]:\n \"\"\"Yields (depth, node) from the pipeline graph. Traversal is topologically\n ordered from the roots if ``direction`` is ``children``, or from the leaves\n if ``direction`` is ``parents``. The yielded depth is the length of the\n longest path from the starting point to the yielded node.\"\"\"\n if direction == \"children\":\n get_in_edges = lambda node: node.parents\n get_out_edges = lambda node: node.children\n else:\n get_in_edges = lambda node: node.children\n get_out_edges = lambda node: node.parents\n\n sort_key = lambda k: self.nodes[k].spec.name\n\n out_edges = {k: sorted(get_out_edges(n), key=sort_key) for k, n in self.nodes.items()}\n num_in_edges = {k: len(get_in_edges(n)) for k, n in self.nodes.items()}\n\n # Populate a queue with all the nodes that have no incoming edges\n nodes = deque(\n sorted(\n [(0, key) for key in self.nodes.keys() if num_in_edges[key] == 0],\n key=lambda item: item[1],\n )\n )\n\n while nodes:\n # Remove the next node, n, from the queue and yield it\n depth, n_key = nodes.pop()\n yield (depth, self.nodes[n_key])\n\n # Remove an in-edge from every node, m, pointed to by an\n # out-edge from n. If any of those nodes are left with\n # 0 remaining in-edges, add them to the queue.\n for m in out_edges[n_key]:\n num_in_edges[m] -= 1\n if num_in_edges[m] == 0:\n nodes.appendleft((depth + 1, m))\n\n def get_dependencies(self, node: PipelineNode) -> List[PipelineNode]:\n \"\"\"Returns a list of nodes corresponding to the direct dependencies\n of the given node.\"\"\"\n return [self.nodes[k] for k in node.children]\n\n\nclass SpackCIConfig:\n \"\"\"Spack CI object used to generate intermediate representation\n used by the CI generator(s).\n \"\"\"\n\n def __init__(self, ci_config):\n \"\"\"Given the information from the ci section of the config\n and the staged jobs, set up meta data needed for generating Spack\n CI IR.\n \"\"\"\n\n self.ci_config = ci_config\n self.named_jobs = [\"any\", \"build\", \"copy\", \"cleanup\", \"noop\", \"reindex\", \"signing\"]\n\n self.ir = {\n \"jobs\": {},\n \"rebuild-index\": self.ci_config.get(\"rebuild-index\", True),\n \"broken-specs-url\": self.ci_config.get(\"broken-specs-url\", None),\n \"broken-tests-packages\": self.ci_config.get(\"broken-tests-packages\", []),\n \"target\": self.ci_config.get(\"target\", \"gitlab\"),\n }\n jobs = self.ir[\"jobs\"]\n\n for name in self.named_jobs:\n # Skip the special named jobs\n if name not in [\"any\", \"build\"]:\n jobs[name] = self.__init_job(\"\")\n\n def __init_job(self, release_spec):\n \"\"\"Initialize job object\"\"\"\n job_object = {\"spec\": release_spec, \"attributes\": {}}\n if release_spec:\n job_vars = job_object[\"attributes\"].setdefault(\"variables\", {})\n job_vars[\"SPACK_JOB_SPEC_DAG_HASH\"] = release_spec.dag_hash()\n job_vars[\"SPACK_JOB_SPEC_PKG_NAME\"] = release_spec.name\n job_vars[\"SPACK_JOB_SPEC_PKG_VERSION\"] = release_spec.format(\"{version}\")\n job_vars[\"SPACK_JOB_SPEC_COMPILER_NAME\"] = release_spec.format(\"{compiler.name}\")\n job_vars[\"SPACK_JOB_SPEC_COMPILER_VERSION\"] = release_spec.format(\"{compiler.version}\")\n job_vars[\"SPACK_JOB_SPEC_ARCH\"] = release_spec.format(\"{architecture}\")\n job_vars[\"SPACK_JOB_SPEC_VARIANTS\"] = release_spec.format(\"{variants}\")\n return job_object\n\n def __is_named(self, section):\n \"\"\"Check if a pipeline-gen configuration section is for a named job,\n and if so return the name otherwise return none.\n \"\"\"\n for _name in self.named_jobs:\n keys = [f\"{_name}-job\", f\"{_name}-job-remove\"]\n if any([key for key in keys if key in section]):\n return _name\n\n return None\n\n @staticmethod\n def __job_name(name, suffix=\"\"):\n \"\"\"Compute the name of a named job with appropriate suffix.\n Valid suffixes are either '-remove' or empty string or None\n \"\"\"\n assert isinstance(name, str)\n\n jname = name\n if suffix:\n jname = f\"{name}-job{suffix}\"\n else:\n jname = f\"{name}-job\"\n\n return jname\n\n def __apply_submapping(self, dest, spec, section):\n \"\"\"Apply submapping section to the IR dict\"\"\"\n matched = False\n only_first = section.get(\"match_behavior\", \"first\") == \"first\"\n\n for match_attrs in reversed(section[\"submapping\"]):\n attrs = cfg.InternalConfigScope._process_dict_keyname_overrides(match_attrs)\n for match_string in match_attrs[\"match\"]:\n if _spec_matches(spec, match_string):\n matched = True\n if \"build-job-remove\" in match_attrs:\n cfg.remove_yaml(dest, attrs[\"build-job-remove\"])\n if \"build-job\" in match_attrs:\n spack.schema.merge_yaml(dest, attrs[\"build-job\"])\n break\n if matched and only_first:\n break\n\n return dest\n\n # Create jobs for all the pipeline specs\n def init_pipeline_jobs(self, pipeline: PipelineDag):\n for _, node in pipeline.traverse_nodes():\n dag_hash = node.spec.dag_hash()\n self.ir[\"jobs\"][dag_hash] = self.__init_job(node.spec)\n\n # Generate IR from the configs\n def generate_ir(self):\n \"\"\"Generate the IR from the Spack CI configurations.\"\"\"\n\n jobs = self.ir[\"jobs\"]\n\n # Implicit job defaults\n defaults = [\n {\n \"build-job\": {\n \"script\": [\n \"cd {env_dir}\",\n \"spack env activate --without-view .\",\n \"spack spec /$SPACK_JOB_SPEC_DAG_HASH\",\n \"spack ci rebuild\",\n ]\n }\n },\n {\"noop-job\": {\"script\": ['echo \"All specs already up to date, nothing to rebuild.\"']}},\n ]\n\n # Job overrides\n overrides = [\n # Reindex script\n {\n \"reindex-job\": {\n \"script:\": [\"spack -v buildcache update-index --keys {index_target_mirror}\"]\n }\n },\n # Cleanup script\n {\n \"cleanup-job\": {\n \"script:\": [\"spack -d mirror destroy {mirror_prefix}/$CI_PIPELINE_ID\"]\n }\n },\n # Add signing job tags\n {\"signing-job\": {\"tags\": [\"aws\", \"protected\", \"notary\"]}},\n # Remove reserved tags\n {\"any-job-remove\": {\"tags\": SPACK_RESERVED_TAGS}},\n ]\n\n pipeline_gen = overrides + self.ci_config.get(\"pipeline-gen\", []) + defaults\n\n for section in reversed(pipeline_gen):\n name = self.__is_named(section)\n has_submapping = \"submapping\" in section\n has_dynmapping = \"dynamic-mapping\" in section\n section = cfg.InternalConfigScope._process_dict_keyname_overrides(section)\n\n if name:\n remove_job_name = self.__job_name(name, suffix=\"-remove\")\n merge_job_name = self.__job_name(name)\n do_remove = remove_job_name in section\n do_merge = merge_job_name in section\n\n def _apply_section(dest, src):\n if do_remove:\n dest = cfg.remove_yaml(dest, src[remove_job_name])\n if do_merge:\n dest = copy.copy(spack.schema.merge_yaml(dest, src[merge_job_name]))\n\n if name == \"build\":\n # Apply attributes to all build jobs\n for _, job in jobs.items():\n if job[\"spec\"]:\n _apply_section(job[\"attributes\"], section)\n elif name == \"any\":\n # Apply section attributes too all jobs\n for _, job in jobs.items():\n _apply_section(job[\"attributes\"], section)\n else:\n # Create a signing job if there is script and the job hasn't\n # been initialized yet\n if name == \"signing\" and name not in jobs:\n if \"signing-job\" in section:\n if \"script\" not in section[\"signing-job\"]:\n continue\n else:\n jobs[name] = self.__init_job(\"\")\n # Apply attributes to named job\n _apply_section(jobs[name][\"attributes\"], section)\n\n elif has_submapping:\n # Apply section jobs with specs to match\n for _, job in jobs.items():\n if job[\"spec\"]:\n job[\"attributes\"] = self.__apply_submapping(\n job[\"attributes\"], job[\"spec\"], section\n )\n elif has_dynmapping:\n mapping = section[\"dynamic-mapping\"]\n\n dynmap_name = mapping.get(\"name\")\n\n # Check if this section should be skipped\n dynmap_skip = os.environ.get(\"SPACK_CI_SKIP_DYNAMIC_MAPPING\")\n if dynmap_name and dynmap_skip:\n if re.match(dynmap_skip, dynmap_name):\n continue\n\n # Get the endpoint\n endpoint = mapping[\"endpoint\"]\n endpoint_url = urlparse(endpoint)\n\n # Configure the request header\n header = {\"User-Agent\": web_util.SPACK_USER_AGENT}\n header.update(mapping.get(\"header\", {}))\n\n # Expand header environment variables\n # ie. if tokens are passed\n for value in header.values():\n value = os.path.expandvars(value)\n\n required = mapping.get(\"require\", [])\n allowed = mapping.get(\"allow\", [])\n ignored = mapping.get(\"ignore\", [])\n\n # required keys are implicitly allowed\n allowed = sorted(set(allowed + required))\n ignored = sorted(set(ignored))\n required = sorted(set(required))\n\n # Make sure required things are not also ignored\n assert not any([ikey in required for ikey in ignored])\n\n def job_query(job):\n job_vars = job[\"attributes\"][\"variables\"]\n query = (\n \"{SPACK_JOB_SPEC_PKG_NAME}@{SPACK_JOB_SPEC_PKG_VERSION}\"\n # The preceding spaces are required (ref. https://github.com/spack/spack-gantry/blob/develop/docs/api.md#allocation)\n \" {SPACK_JOB_SPEC_VARIANTS}\"\n \" arch={SPACK_JOB_SPEC_ARCH}\"\n \"%{SPACK_JOB_SPEC_COMPILER_NAME}@{SPACK_JOB_SPEC_COMPILER_VERSION}\"\n ).format_map(job_vars)\n return f\"spec={quote(query)}\"\n\n for job in jobs.values():\n if not job[\"spec\"]:\n continue\n\n # Create request for this job\n query = job_query(job)\n request = Request(\n endpoint_url._replace(query=query).geturl(), headers=header, method=\"GET\"\n )\n try:\n with _urlopen(request) as response:\n config = json.load(response)\n except Exception as e:\n # For now just ignore any errors from dynamic mapping and continue\n # This is still experimental, and failures should not stop CI\n # from running normally\n tty.warn(f\"Failed to fetch dynamic mapping for query:\\n\\t{query}: {e}\")\n continue\n\n # Strip ignore keys\n if ignored:\n for key in ignored:\n if key in config:\n config.pop(key)\n\n # Only keep allowed keys\n clean_config = {}\n if allowed:\n for key in allowed:\n if key in config:\n clean_config[key] = config[key]\n else:\n clean_config = config\n\n # Verify all of the required keys are present\n if required:\n missing_keys = []\n for key in required:\n if key not in clean_config.keys():\n missing_keys.append(key)\n\n if missing_keys:\n tty.warn(f\"Response missing required keys: {missing_keys}\")\n\n if clean_config:\n job[\"attributes\"] = spack.schema.merge_yaml(\n job.get(\"attributes\", {}), clean_config\n )\n\n for _, job in jobs.items():\n if job[\"spec\"]:\n job[\"spec\"] = job[\"spec\"].name\n\n return self.ir\n\n\nclass SpackCIError(spack.error.SpackError):\n def __init__(self, msg):\n super().__init__(msg)\n" }, { "path": "lib/spack/spack/ci/generator_registry.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n# Holds all known formatters\n\"\"\"Generators that support writing out pipelines for various CI platforms,\nusing a common pipeline graph definition.\n\"\"\"\n\nimport spack.error\n\n_generators = {}\n\n\ndef generator(name):\n \"\"\"Decorator to register a pipeline generator method.\n A generator method should take PipelineDag, SpackCIConfig, and\n PipelineOptions arguments, and should produce a pipeline file.\n \"\"\"\n\n def _decorator(generate_method):\n _generators[name] = generate_method\n return generate_method\n\n return _decorator\n\n\ndef get_generator(name):\n try:\n return _generators[name]\n except KeyError:\n raise UnknownGeneratorException(name)\n\n\nclass UnknownGeneratorException(spack.error.SpackError):\n def __init__(self, generator_name):\n super().__init__(f\"No registered generator for {generator_name}\")\n" }, { "path": "lib/spack/spack/ci/gitlab.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport copy\nimport os\nimport shutil\nimport urllib\nfrom typing import List, Optional\n\nimport spack.vendor.ruamel.yaml\n\nimport spack\nimport spack.binary_distribution\nimport spack.config\nimport spack.llnl.util.tty as tty\nimport spack.mirrors.mirror\nimport spack.schema\nimport spack.spec\nimport spack.util.path as path_util\nimport spack.util.spack_yaml as syaml\n\nfrom .common import (\n SPACK_RESERVED_TAGS,\n PipelineDag,\n PipelineOptions,\n PipelineType,\n SpackCIConfig,\n SpackCIError,\n ensure_expected_target_path,\n unpack_script,\n write_pipeline_manifest,\n)\nfrom .generator_registry import generator\n\n# See https://docs.gitlab.com/ee/ci/yaml/#retry for descriptions of conditions\nJOB_RETRY_CONDITIONS = [\n # \"always\",\n \"unknown_failure\",\n \"script_failure\",\n \"api_failure\",\n \"stuck_or_timeout_failure\",\n \"runner_system_failure\",\n \"runner_unsupported\",\n \"stale_schedule\",\n # \"job_execution_timeout\",\n \"archived_failure\",\n \"unmet_prerequisites\",\n \"scheduler_failure\",\n \"data_integrity_failure\",\n]\nJOB_NAME_FORMAT = \"{name}{@version} {/hash}\"\n\n\ndef _remove_reserved_tags(tags):\n \"\"\"Convenience function to strip reserved tags from jobs\"\"\"\n return [tag for tag in tags if tag not in SPACK_RESERVED_TAGS]\n\n\ndef get_job_name(spec: spack.spec.Spec, build_group: Optional[str] = None) -> str:\n \"\"\"Given a spec and possibly a build group, return the job name. If the\n resulting name is longer than 255 characters, it will be truncated.\n\n Arguments:\n spec: Spec job will build\n build_group: Name of build group this job belongs to (a CDash notion)\n\n Returns: The job name\n \"\"\"\n job_name = spec.format(JOB_NAME_FORMAT)\n\n if build_group:\n job_name = f\"{job_name} {build_group}\"\n\n return job_name[:255]\n\n\ndef maybe_generate_manifest(pipeline: PipelineDag, options: PipelineOptions, manifest_path):\n # TODO: Consider including only hashes of rebuilt specs in the manifest,\n # instead of full source and destination urls. Also, consider renaming\n # the variable that controls whether or not to write the manifest from\n # \"SPACK_COPY_BUILDCACHE\" to \"SPACK_WRITE_PIPELINE_MANIFEST\" or similar.\n spack_buildcache_copy = os.environ.get(\"SPACK_COPY_BUILDCACHE\", None)\n if spack_buildcache_copy:\n buildcache_copy_src_prefix = options.buildcache_destination.fetch_url\n buildcache_copy_dest_prefix = spack_buildcache_copy\n\n if options.pipeline_type == PipelineType.COPY_ONLY:\n manifest_specs = [s for s in options.env.all_specs() if not s.external]\n else:\n manifest_specs = [n.spec for _, n in pipeline.traverse_nodes(direction=\"children\")]\n\n write_pipeline_manifest(\n manifest_specs, buildcache_copy_src_prefix, buildcache_copy_dest_prefix, manifest_path\n )\n\n\n@generator(\"gitlab\")\ndef generate_gitlab_yaml(pipeline: PipelineDag, spack_ci: SpackCIConfig, options: PipelineOptions):\n \"\"\"Given a pipeline graph, job attributes, and pipeline options,\n write a pipeline that can be consumed by GitLab to the given output file.\n\n Arguments:\n pipeline: An already pruned graph of jobs representing all the specs to build\n spack_ci: An object containing the configured attributes of all jobs in the pipeline\n options: An object containing all the pipeline options gathered from yaml, env, etc...\n \"\"\"\n ci_project_dir = os.environ.get(\"CI_PROJECT_DIR\") or os.getcwd()\n generate_job_name = os.environ.get(\"CI_JOB_NAME\", \"job-does-not-exist\")\n generate_pipeline_id = os.environ.get(\"CI_PIPELINE_ID\", \"pipeline-does-not-exist\")\n artifacts_root = options.artifacts_root\n if artifacts_root.startswith(ci_project_dir):\n artifacts_root = os.path.relpath(artifacts_root, ci_project_dir)\n pipeline_artifacts_dir = os.path.join(ci_project_dir, artifacts_root)\n output_file = options.output_file\n\n if not output_file:\n output_file = os.path.abspath(\".gitlab-ci.yml\")\n else:\n output_file_path = os.path.abspath(output_file)\n gen_ci_dir = os.path.dirname(output_file_path)\n if not os.path.exists(gen_ci_dir):\n os.makedirs(gen_ci_dir)\n\n spack_ci_ir = spack_ci.generate_ir()\n\n concrete_env_dir = os.path.join(pipeline_artifacts_dir, \"concrete_environment\")\n\n # Now that we've added the mirrors we know about, they should be properly\n # reflected in the environment manifest file, so copy that into the\n # concrete environment directory, along with the spack.lock file.\n if not os.path.exists(concrete_env_dir):\n os.makedirs(concrete_env_dir)\n\n # Copy the manifest and handle relative included paths\n with open(options.env.manifest_path, \"r\", encoding=\"utf-8\") as fin, open(\n os.path.join(concrete_env_dir, \"spack.yaml\"), \"w\", encoding=\"utf-8\"\n ) as fout:\n data = syaml.load(fin)\n if \"spack\" not in data:\n raise spack.config.ConfigSectionError(\n 'Missing top level \"spack\" section in environment'\n )\n\n def _rewrite_include(path, orig_root, new_root):\n expanded_path = path_util.substitute_path_variables(path)\n\n # Skip non-local paths\n parsed = urllib.parse.urlparse(expanded_path)\n file_schemes = [\"\", \"file\"]\n if parsed.scheme not in file_schemes:\n return path\n\n if os.path.isabs(expanded_path):\n return path\n abs_path = path_util.canonicalize_path(path, orig_root)\n return os.path.relpath(abs_path, start=new_root)\n\n # If there are no includes, just copy\n if \"include\" in data[\"spack\"]:\n includes = data[\"spack\"][\"include\"]\n # If there are includes in the config, then we need to fix the relative paths\n # to be relative from the concrete env dir used by downstream pipelines\n env_root_path = os.path.dirname(os.path.abspath(options.env.manifest_path))\n fixed_includes = []\n for inc in includes:\n if isinstance(inc, dict):\n inc[\"path\"] = _rewrite_include(inc[\"path\"], env_root_path, concrete_env_dir)\n else:\n inc = _rewrite_include(inc, env_root_path, concrete_env_dir)\n\n fixed_includes.append(inc)\n\n data[\"spack\"][\"include\"] = fixed_includes\n\n os.makedirs(concrete_env_dir, exist_ok=True)\n syaml.dump(data, fout)\n\n shutil.copyfile(options.env.lock_path, os.path.join(concrete_env_dir, \"spack.lock\"))\n\n job_log_dir = os.path.join(pipeline_artifacts_dir, \"logs\")\n job_repro_dir = os.path.join(pipeline_artifacts_dir, \"reproduction\")\n job_test_dir = os.path.join(pipeline_artifacts_dir, \"tests\")\n user_artifacts_dir = os.path.join(pipeline_artifacts_dir, \"user_data\")\n\n # We communicate relative paths to the downstream jobs to avoid issues in\n # situations where the CI_PROJECT_DIR varies between the pipeline\n # generation job and the rebuild jobs. This can happen when gitlab\n # checks out the project into a runner-specific directory, for example,\n # and different runners are picked for generate and rebuild jobs.\n\n rel_concrete_env_dir = os.path.relpath(concrete_env_dir, ci_project_dir)\n rel_job_log_dir = os.path.relpath(job_log_dir, ci_project_dir)\n rel_job_repro_dir = os.path.relpath(job_repro_dir, ci_project_dir)\n rel_job_test_dir = os.path.relpath(job_test_dir, ci_project_dir)\n rel_user_artifacts_dir = os.path.relpath(user_artifacts_dir, ci_project_dir)\n\n def main_script_replacements(cmd):\n return cmd.replace(\"{env_dir}\", rel_concrete_env_dir)\n\n output_object = {}\n job_id = 0\n stage_id = 0\n stages: List[List] = []\n stage_names = []\n\n max_length_needs = 0\n max_needs_job = \"\"\n\n if not options.pipeline_type == PipelineType.COPY_ONLY:\n for level, node in pipeline.traverse_nodes(direction=\"parents\"):\n stage_id = level\n if len(stages) == stage_id:\n stages.append([])\n stages[stage_id].append(node.spec)\n stage_name = f\"stage-{level}\"\n\n if stage_name not in stage_names:\n stage_names.append(stage_name)\n\n release_spec = node.spec\n release_spec_dag_hash = release_spec.dag_hash()\n\n job_object = spack_ci_ir[\"jobs\"][release_spec_dag_hash][\"attributes\"]\n\n if not job_object:\n tty.warn(f\"No match found for {release_spec}, skipping it\")\n continue\n\n if options.pipeline_type is not None:\n # For spack pipelines \"public\" and \"protected\" are reserved tags\n job_object[\"tags\"] = _remove_reserved_tags(job_object.get(\"tags\", []))\n if options.pipeline_type == PipelineType.PROTECTED_BRANCH:\n job_object[\"tags\"].extend([\"protected\"])\n elif options.pipeline_type == PipelineType.PULL_REQUEST:\n job_object[\"tags\"].extend([\"public\"])\n\n if \"script\" not in job_object:\n raise AttributeError\n\n job_object[\"script\"] = unpack_script(job_object[\"script\"], op=main_script_replacements)\n\n if \"before_script\" in job_object:\n job_object[\"before_script\"] = unpack_script(job_object[\"before_script\"])\n\n if \"after_script\" in job_object:\n job_object[\"after_script\"] = unpack_script(job_object[\"after_script\"])\n\n build_group = options.cdash_handler.build_group if options.cdash_handler else None\n job_name = get_job_name(release_spec, build_group)\n\n dep_nodes = pipeline.get_dependencies(node)\n job_object[\"needs\"] = [\n {\"job\": get_job_name(dep_node.spec, build_group), \"artifacts\": False}\n for dep_node in dep_nodes\n ]\n\n job_object[\"needs\"].append(\n {\"job\": generate_job_name, \"pipeline\": f\"{generate_pipeline_id}\"}\n )\n\n job_vars = job_object[\"variables\"]\n\n # Let downstream jobs know whether the spec needed rebuilding, regardless\n # whether DAG pruning was enabled or not.\n already_built = spack.binary_distribution.get_mirrors_for_spec(\n spec=release_spec, index_only=True\n )\n job_vars[\"SPACK_SPEC_NEEDS_REBUILD\"] = \"False\" if already_built else \"True\"\n\n if options.cdash_handler:\n build_name = options.cdash_handler.build_name(release_spec)\n job_vars[\"SPACK_CDASH_BUILD_NAME\"] = build_name\n build_stamp = options.cdash_handler.build_stamp\n job_vars[\"SPACK_CDASH_BUILD_STAMP\"] = build_stamp\n\n job_object[\"artifacts\"] = spack.schema.merge_yaml(\n job_object.get(\"artifacts\", {}),\n {\n \"when\": \"always\",\n \"paths\": [\n rel_job_log_dir,\n rel_job_repro_dir,\n rel_job_test_dir,\n rel_user_artifacts_dir,\n ],\n },\n )\n\n job_object[\"stage\"] = stage_name\n job_object[\"retry\"] = spack.schema.merge_yaml(\n {\"max\": 2, \"when\": JOB_RETRY_CONDITIONS}, job_object.get(\"retry\", {})\n )\n job_object[\"interruptible\"] = True\n\n length_needs = len(job_object[\"needs\"])\n if length_needs > max_length_needs:\n max_length_needs = length_needs\n max_needs_job = job_name\n\n output_object[job_name] = job_object\n job_id += 1\n\n tty.debug(f\"{job_id} build jobs generated in {stage_id} stages\")\n\n if job_id > 0:\n tty.debug(f\"The max_needs_job is {max_needs_job}, with {max_length_needs} needs\")\n\n service_job_retries = {\n \"max\": 2,\n \"when\": [\"runner_system_failure\", \"stuck_or_timeout_failure\", \"script_failure\"],\n }\n\n # In some cases, pipeline generation should write a manifest. Currently\n # the only purpose is to specify a list of sources and destinations for\n # everything that should be copied.\n distinguish_stack = options.stack_name if options.stack_name else \"rebuilt\"\n manifest_path = os.path.join(\n pipeline_artifacts_dir, \"specs_to_copy\", f\"copy_{distinguish_stack}_specs.json\"\n )\n maybe_generate_manifest(pipeline, options, manifest_path)\n\n relative_specs_url = spack.binary_distribution.buildcache_relative_specs_url()\n relative_keys_url = spack.binary_distribution.buildcache_relative_keys_url()\n\n if options.pipeline_type == PipelineType.COPY_ONLY:\n stage_names.append(\"copy\")\n sync_job = copy.deepcopy(spack_ci_ir[\"jobs\"][\"copy\"][\"attributes\"])\n sync_job[\"stage\"] = \"copy\"\n sync_job[\"needs\"] = [{\"job\": generate_job_name, \"pipeline\": f\"{generate_pipeline_id}\"}]\n\n if \"variables\" not in sync_job:\n sync_job[\"variables\"] = {}\n\n sync_job[\"variables\"].update(\n {\n \"SPACK_COPY_ONLY_DESTINATION\": options.buildcache_destination.fetch_url,\n \"SPACK_BUILDCACHE_RELATIVE_KEYS_URL\": relative_keys_url,\n }\n )\n\n pipeline_mirrors = spack.mirrors.mirror.MirrorCollection(binary=True)\n if \"buildcache-source\" not in pipeline_mirrors:\n raise SpackCIError(\"Copy-only pipelines require a mirror named 'buildcache-source'\")\n\n buildcache_source = pipeline_mirrors[\"buildcache-source\"].fetch_url\n sync_job[\"variables\"][\"SPACK_BUILDCACHE_SOURCE\"] = buildcache_source\n sync_job[\"dependencies\"] = []\n\n output_object[\"copy\"] = sync_job\n job_id += 1\n\n if job_id > 0:\n if (\n \"script\" in spack_ci_ir[\"jobs\"][\"signing\"][\"attributes\"]\n and options.pipeline_type == PipelineType.PROTECTED_BRANCH\n ):\n # External signing: generate a job to check and sign binary pkgs\n stage_names.append(\"stage-sign-pkgs\")\n signing_job = spack_ci_ir[\"jobs\"][\"signing\"][\"attributes\"]\n\n signing_job[\"script\"] = unpack_script(signing_job[\"script\"])\n\n signing_job[\"stage\"] = \"stage-sign-pkgs\"\n signing_job[\"when\"] = \"always\"\n signing_job[\"retry\"] = {\"max\": 2, \"when\": [\"always\"]}\n signing_job[\"interruptible\"] = True\n if \"variables\" not in signing_job:\n signing_job[\"variables\"] = {}\n signing_job[\"variables\"].update(\n {\n \"SPACK_BUILDCACHE_DESTINATION\": options.buildcache_destination.push_url,\n \"SPACK_BUILDCACHE_RELATIVE_SPECS_URL\": relative_specs_url,\n \"SPACK_BUILDCACHE_RELATIVE_KEYS_URL\": relative_keys_url,\n }\n )\n signing_job[\"dependencies\"] = []\n\n output_object[\"sign-pkgs\"] = signing_job\n\n if options.rebuild_index:\n # Add a final job to regenerate the index\n stage_names.append(\"stage-rebuild-index\")\n final_job = spack_ci_ir[\"jobs\"][\"reindex\"][\"attributes\"]\n\n final_job[\"stage\"] = \"stage-rebuild-index\"\n target_mirror = options.buildcache_destination.push_url\n final_job[\"script\"] = unpack_script(\n final_job[\"script\"],\n op=lambda cmd: cmd.replace(\"{index_target_mirror}\", target_mirror),\n )\n\n final_job[\"when\"] = \"always\"\n final_job[\"retry\"] = service_job_retries\n final_job[\"interruptible\"] = True\n final_job[\"dependencies\"] = []\n\n output_object[\"rebuild-index\"] = final_job\n\n output_object[\"stages\"] = stage_names\n\n # Capture the version of Spack used to generate the pipeline, that can be\n # passed to `git checkout` for version consistency. If we aren't in a Git\n # repository, presume we are a Spack release and use the Git tag instead.\n spack_version = spack.get_version()\n version_to_clone = spack.get_spack_commit() or f\"v{spack.spack_version}\"\n\n rebuild_everything = not options.prune_up_to_date and not options.prune_untouched\n\n output_object[\"variables\"] = {\n \"SPACK_ARTIFACTS_ROOT\": artifacts_root,\n \"SPACK_CONCRETE_ENV_DIR\": rel_concrete_env_dir,\n \"SPACK_VERSION\": spack_version,\n \"SPACK_CHECKOUT_VERSION\": version_to_clone,\n \"SPACK_JOB_LOG_DIR\": rel_job_log_dir,\n \"SPACK_JOB_REPRO_DIR\": rel_job_repro_dir,\n \"SPACK_JOB_TEST_DIR\": rel_job_test_dir,\n \"SPACK_PIPELINE_TYPE\": options.pipeline_type.name if options.pipeline_type else \"None\",\n \"SPACK_CI_STACK_NAME\": os.environ.get(\"SPACK_CI_STACK_NAME\", \"None\"),\n \"SPACK_REBUILD_CHECK_UP_TO_DATE\": str(options.prune_up_to_date),\n \"SPACK_REBUILD_EVERYTHING\": str(rebuild_everything),\n \"SPACK_REQUIRE_SIGNING\": str(options.require_signing),\n }\n output_object[\"variables\"].update(\n dict([(v, os.environ[v]) for v in options.forward_variables if v in os.environ])\n )\n\n if options.stack_name:\n output_object[\"variables\"][\"SPACK_CI_STACK_NAME\"] = options.stack_name\n\n output_vars = output_object[\"variables\"]\n for item, val in output_vars.items():\n output_vars[item] = ensure_expected_target_path(val)\n\n else:\n # No jobs were generated\n noop_job = spack_ci_ir[\"jobs\"][\"noop\"][\"attributes\"]\n # If this job fails ignore the status and carry on\n noop_job[\"retry\"] = 0\n noop_job[\"allow_failure\"] = True\n\n tty.debug(\"No specs to rebuild, generating no-op job\")\n output_object = {\"no-specs-to-rebuild\": noop_job}\n\n # Ensure the child pipeline always runs\n output_object[\"workflow\"] = {\"rules\": [{\"when\": \"always\"}]}\n\n sorted_output = {}\n for output_key, output_value in sorted(output_object.items()):\n sorted_output[output_key] = output_value\n\n # Minimize yaml output size through use of anchors\n syaml.anchorify(sorted_output)\n\n with open(output_file, \"w\", encoding=\"utf-8\") as f:\n spack.vendor.ruamel.yaml.YAML().dump(sorted_output, f)\n" }, { "path": "lib/spack/spack/cmd/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport difflib\nimport importlib\nimport os\nimport re\nimport subprocess\nimport sys\nimport textwrap\nfrom collections import Counter\nfrom typing import Generator, List, Optional, Sequence, Union\n\nimport spack.concretize\nimport spack.config\nimport spack.environment as ev\nimport spack.error\nimport spack.extensions\nimport spack.llnl.string\nimport spack.llnl.util.tty as tty\nimport spack.paths\nimport spack.repo\nimport spack.spec\nimport spack.spec_parser\nimport spack.store\nimport spack.traverse as traverse\nimport spack.user_environment as uenv\nimport spack.util.spack_json as sjson\nimport spack.util.spack_yaml as syaml\nfrom spack.llnl.util.filesystem import join_path\nfrom spack.llnl.util.lang import attr_setdefault, index_by\nfrom spack.llnl.util.tty.colify import colify\nfrom spack.llnl.util.tty.color import colorize\n\nfrom ..enums import InstallRecordStatus\n\n# cmd has a submodule called \"list\" so preserve the python list module\npython_list = list\n\n# Patterns to ignore in the commands directory when looking for commands.\nignore_files = r\"^\\.|^__init__.py$|^#\"\n\nSETUP_PARSER = \"setup_parser\"\nDESCRIPTION = \"description\"\n\n\ndef python_name(cmd_name):\n \"\"\"Convert ``-`` to ``_`` in command name, to make a valid identifier.\"\"\"\n return cmd_name.replace(\"-\", \"_\")\n\n\ndef require_python_name(pname):\n \"\"\"Require that the provided name is a valid python name (per\n python_name()). Useful for checking parameters for function\n prerequisites.\"\"\"\n if python_name(pname) != pname:\n raise PythonNameError(pname)\n\n\ndef cmd_name(python_name):\n \"\"\"Convert module name (with ``_``) to command name (with ``-``).\"\"\"\n return python_name.replace(\"_\", \"-\")\n\n\ndef require_cmd_name(cname):\n \"\"\"Require that the provided name is a valid command name (per\n cmd_name()). Useful for checking parameters for function\n prerequisites.\n \"\"\"\n if cmd_name(cname) != cname:\n raise CommandNameError(cname)\n\n\n#: global, cached list of all commands -- access through all_commands()\n_all_commands = None\n\n\ndef all_commands():\n \"\"\"Get a sorted list of all spack commands.\n\n This will list the lib/spack/spack/cmd directory and find the\n commands there to construct the list. It does not actually import\n the python files -- just gets the names.\n \"\"\"\n global _all_commands\n if _all_commands is None:\n _all_commands = []\n command_paths = [spack.paths.command_path] # Built-in commands\n command_paths += spack.extensions.get_command_paths() # Extensions\n for path in command_paths:\n for file in os.listdir(path):\n if file.endswith(\".py\") and not re.search(ignore_files, file):\n cmd = re.sub(r\".py$\", \"\", file)\n _all_commands.append(cmd_name(cmd))\n\n _all_commands.sort()\n\n return _all_commands\n\n\ndef remove_options(parser, *options):\n \"\"\"Remove some options from a parser.\"\"\"\n for option in options:\n for action in parser._actions:\n if vars(action)[\"option_strings\"][0] == option:\n parser._handle_conflict_resolve(None, [(option, action)])\n break\n\n\ndef get_module(cmd_name):\n \"\"\"Imports the module for a particular command name and returns it.\n\n Args:\n cmd_name (str): name of the command for which to get a module\n (contains ``-``, not ``_``).\n \"\"\"\n require_cmd_name(cmd_name)\n pname = python_name(cmd_name)\n\n try:\n # Try to import the command from the built-in directory\n module_name = f\"{__name__}.{pname}\"\n module = importlib.import_module(module_name)\n tty.debug(\"Imported {0} from built-in commands\".format(pname))\n except ImportError:\n module = spack.extensions.get_module(cmd_name)\n if not module:\n raise CommandNotFoundError(cmd_name)\n\n attr_setdefault(module, SETUP_PARSER, lambda *args: None) # null-op\n attr_setdefault(module, DESCRIPTION, \"\")\n\n if not hasattr(module, pname):\n tty.die(\n \"Command module %s (%s) must define function '%s'.\"\n % (module.__name__, module.__file__, pname)\n )\n\n return module\n\n\ndef get_command(cmd_name):\n \"\"\"Imports the command function associated with cmd_name.\n\n The function's name is derived from cmd_name using python_name().\n\n Args:\n cmd_name (str): name of the command (contains ``-``, not ``_``).\n \"\"\"\n require_cmd_name(cmd_name)\n pname = python_name(cmd_name)\n return getattr(get_module(cmd_name), pname)\n\n\ndef quote_kvp(string: str) -> str:\n \"\"\"For strings like ``name=value`` or ``name==value``, quote and escape the value if needed.\n\n This is a compromise to respect quoting of key-value pairs on the CLI. The shell\n strips quotes from quoted arguments, so we cannot know *exactly* how CLI arguments\n were quoted. To compensate, we re-add quotes around anything staritng with ``name=``\n or ``name==``, and we assume the rest of the argument is the value. This covers the\n common cases of passign flags, e.g., ``cflags=\"-O2 -g\"`` on the command line.\n \"\"\"\n match = spack.spec_parser.SPLIT_KVP.match(string)\n if not match:\n return string\n\n key, delim, value = match.groups()\n return f\"{key}{delim}{spack.spec_parser.quote_if_needed(value)}\"\n\n\ndef parse_specs(\n args: Union[str, List[str]],\n concretize: bool = False,\n tests: spack.concretize.TestsType = False,\n) -> List[spack.spec.Spec]:\n \"\"\"Convenience function for parsing arguments from specs. Handles common\n exceptions and dies if there are errors.\n \"\"\"\n args = [args] if isinstance(args, str) else args\n arg_string = \" \".join([quote_kvp(arg) for arg in args])\n\n toolchains = spack.config.CONFIG.get(\"toolchains\", {})\n specs = spack.spec_parser.parse(arg_string, toolchains=toolchains)\n if not concretize:\n return specs\n\n to_concretize: List[spack.concretize.SpecPairInput] = [(s, None) for s in specs]\n return _concretize_spec_pairs(to_concretize, tests=tests)\n\n\ndef _concretize_spec_pairs(\n to_concretize: List[spack.concretize.SpecPairInput], tests: spack.concretize.TestsType = False\n) -> List[spack.spec.Spec]:\n \"\"\"Helper method that concretizes abstract specs from a list of abstract,concrete pairs.\n\n Any spec with a concrete spec associated with it will concretize to that spec. Any spec\n with ``None`` for its concrete spec will be newly concretized. This method respects unification\n rules from config.\"\"\"\n unify = spack.config.get(\"concretizer:unify\", False)\n\n # Special case for concretizing a single spec\n if len(to_concretize) == 1:\n abstract, concrete = to_concretize[0]\n return [concrete or spack.concretize.concretize_one(abstract, tests=tests)]\n\n # Special case if every spec is either concrete or has an abstract hash\n if all(\n concrete or abstract.concrete or abstract.abstract_hash\n for abstract, concrete in to_concretize\n ):\n # Get all the concrete specs\n ret = [\n concrete or (abstract if abstract.concrete else abstract.lookup_hash())\n for abstract, concrete in to_concretize\n ]\n\n # If unify: true, check that specs don't conflict\n # Since all concrete, \"when_possible\" is not relevant\n if unify is True: # True, \"when_possible\", False are possible values\n runtimes = spack.repo.PATH.packages_with_tags(\"runtime\")\n specs_per_name = Counter(\n spec.name\n for spec in traverse.traverse_nodes(\n ret, deptype=(\"link\", \"run\"), key=traverse.by_dag_hash\n )\n if spec.name not in runtimes # runtimes are allowed multiple times\n )\n\n conflicts = sorted(name for name, count in specs_per_name.items() if count > 1)\n if conflicts:\n raise spack.error.SpecError(\n \"Specs conflict and `concretizer:unify` is configured true.\",\n f\" specs depend on multiple versions of {', '.join(conflicts)}\",\n )\n return ret\n\n # Standard case\n concretize_method = spack.concretize.concretize_separately # unify: false\n if unify is True:\n concretize_method = spack.concretize.concretize_together\n elif unify == \"when_possible\":\n concretize_method = spack.concretize.concretize_together_when_possible\n\n concretized = concretize_method(to_concretize, tests=tests)\n return [concrete for _, concrete in concretized]\n\n\ndef matching_spec_from_env(spec):\n \"\"\"\n Returns a concrete spec, matching what is available in the environment.\n If no matching spec is found in the environment (or if no environment is\n active), this will return the given spec but concretized.\n \"\"\"\n env = ev.active_environment()\n if env:\n return env.matching_spec(spec) or spack.concretize.concretize_one(spec)\n else:\n return spack.concretize.concretize_one(spec)\n\n\ndef matching_specs_from_env(specs):\n \"\"\"\n Same as ``matching_spec_from_env`` but respects spec unification rules.\n\n For each spec, if there is a matching spec in the environment it is used. If no\n matching spec is found, this will return the given spec but concretized in the\n context of the active environment and other given specs, with unification rules applied.\n \"\"\"\n env = ev.active_environment()\n spec_pairs = [(spec, env.matching_spec(spec) if env else None) for spec in specs]\n additional_concrete_specs = (\n [(concrete, concrete) for _, concrete in env.concretized_specs()] if env else []\n )\n return _concretize_spec_pairs(spec_pairs + additional_concrete_specs)[: len(spec_pairs)]\n\n\ndef disambiguate_spec(\n spec: spack.spec.Spec,\n env: Optional[ev.Environment],\n local: bool = False,\n installed: Union[bool, InstallRecordStatus] = True,\n first: bool = False,\n) -> spack.spec.Spec:\n \"\"\"Given a spec, figure out which installed package it refers to.\n\n Args:\n spec: a spec to disambiguate\n env: a spack environment, if one is active, or None if no environment is active\n local: do not search chained spack instances\n installed: install status argument passed to database query.\n first: returns the first matching spec, even if more than one match is found\n \"\"\"\n hashes = env.all_hashes() if env else None\n return disambiguate_spec_from_hashes(spec, hashes, local, installed, first)\n\n\ndef disambiguate_spec_from_hashes(\n spec: spack.spec.Spec,\n hashes: Optional[List[str]],\n local: bool = False,\n installed: Union[bool, InstallRecordStatus] = True,\n first: bool = False,\n) -> spack.spec.Spec:\n \"\"\"Given a spec and a list of hashes, get concrete spec the spec refers to.\n\n Arguments:\n spec: a spec to disambiguate\n hashes: a set of hashes of specs among which to disambiguate\n local: if True, do not search chained spack instances\n installed: install status argument passed to database query.\n first: returns the first matching spec, even if more than one match is found\n \"\"\"\n if local:\n matching_specs = spack.store.STORE.db.query_local(spec, hashes=hashes, installed=installed)\n else:\n matching_specs = spack.store.STORE.db.query(spec, hashes=hashes, installed=installed)\n if not matching_specs:\n tty.die(f\"Spec '{spec}' matches no installed packages.\")\n\n elif first:\n return matching_specs[0]\n\n ensure_single_spec_or_die(spec, matching_specs)\n\n return matching_specs[0]\n\n\ndef ensure_single_spec_or_die(spec, matching_specs):\n if len(matching_specs) <= 1:\n return\n\n format_string = (\n \"{name}{@version}\"\n \"{ platform=architecture.platform}{ os=architecture.os}{ target=architecture.target}\"\n \"{%compiler.name}{@compiler.version}\"\n )\n args = [\"%s matches multiple packages.\" % spec, \"Matching packages:\"]\n args += [\n colorize(\" @K{%s} \" % s.dag_hash(7)) + s.cformat(format_string) for s in matching_specs\n ]\n args += [\"Use a more specific spec (e.g., prepend '/' to the hash).\"]\n tty.die(*args)\n\n\ndef gray_hash(spec, length):\n if not length:\n # default to maximum hash length\n length = 32\n h = spec.dag_hash(length) if spec.concrete else \"-\" * length\n return colorize(\"@K{%s}\" % h)\n\n\ndef display_specs_as_json(specs, deps=False):\n \"\"\"Convert specs to a list of json records.\"\"\"\n seen = set()\n records = []\n for spec in specs:\n dag_hash = spec.dag_hash()\n if dag_hash in seen:\n continue\n records.append(spec.node_dict_with_hashes())\n seen.add(dag_hash)\n\n if deps:\n for dep in spec.traverse():\n dep_dag_hash = dep.dag_hash()\n if dep_dag_hash in seen:\n continue\n records.append(dep.node_dict_with_hashes())\n seen.add(dep_dag_hash)\n\n sjson.dump(records, sys.stdout)\n\n\ndef iter_groups(specs, indent, all_headers):\n \"\"\"Break a list of specs into groups indexed by arch/compilers.\"\"\"\n # Make a dict with specs keyed by architecture and compilers.\n index = index_by(specs, (\"architecture\", \"compilers\"))\n ispace = indent * \" \"\n\n def _key(item):\n if item is None:\n return \"\"\n return str(item)\n\n # Traverse the index and print out each package\n for i, (architecture, compilers) in enumerate(sorted(index, key=_key)):\n if i > 0:\n print()\n\n # Drop the leading space from compilers to clean up output and aid checks.\n compilers_info = compilers.strip() or \"no compilers\"\n header = \"%s{%s} / %s{%s}\" % (\n spack.spec.ARCHITECTURE_COLOR,\n architecture if architecture else \"no arch\",\n spack.spec.COMPILER_COLOR,\n compilers_info,\n )\n\n # Sometimes we want to display specs that are not yet concretized.\n # If they don't have compilers / architecture attached to them,\n # then skip the header\n if all_headers or (architecture is not None or compilers_info):\n sys.stdout.write(ispace)\n tty.hline(colorize(header), char=\"-\")\n\n specs = index[(architecture, compilers)]\n specs.sort()\n yield specs\n\n\ndef display_specs(specs, args=None, **kwargs):\n \"\"\"Display human readable specs with customizable formatting.\n\n Prints the supplied specs to the screen, formatted according to the\n arguments provided.\n\n Specs are grouped by architecture and compiler, and columnized if\n possible.\n\n Options can add more information to the default display. Options can\n be provided either as keyword arguments or as an argparse namespace.\n Keyword arguments take precedence over settings in the argparse\n namespace.\n\n Args:\n specs (list): the specs to display\n args (argparse.Namespace or None): namespace containing formatting arguments\n\n Keyword Args:\n paths (bool): Show paths with each displayed spec\n deps (bool): Display dependencies with specs\n long (bool): Display short hashes with specs\n very_long (bool): Display full hashes with specs (supersedes ``long``)\n namespaces (bool): Print namespaces along with names\n show_flags (bool): Show compiler flags with specs\n variants (bool): Show variants with specs\n indent (int): indent each line this much\n groups (bool): display specs grouped by arch/compiler (default True)\n decorator (typing.Callable): function to call to decorate specs\n all_headers (bool): show headers even when arch/compiler aren't defined\n status_fn (typing.Callable): if provided, prepend install-status info\n output (typing.IO): A file object to write to. Default is ``sys.stdout``\n specfile_format (bool): specfile format of the current spec\n \"\"\"\n\n def get_arg(name, default=None):\n \"\"\"Prefer kwargs, then args, then default.\"\"\"\n if name in kwargs:\n return kwargs.get(name)\n elif args is not None:\n return getattr(args, name, default)\n else:\n return default\n\n paths = get_arg(\"paths\", False)\n deps = get_arg(\"deps\", False)\n hashes = get_arg(\"long\", False)\n namespaces = get_arg(\"namespaces\", False)\n flags = get_arg(\"show_flags\", False)\n variants = get_arg(\"variants\", False)\n groups = get_arg(\"groups\", True)\n all_headers = get_arg(\"all_headers\", False)\n output = get_arg(\"output\", sys.stdout)\n status_fn = get_arg(\"status_fn\", None)\n specfile_format = get_arg(\"specfile_format\", False)\n\n decorator = get_arg(\"decorator\", None)\n if decorator is None:\n decorator = lambda s, f: f\n\n indent = get_arg(\"indent\", 0)\n\n hlen = 7\n if get_arg(\"very_long\", False):\n hashes = True\n hlen = None\n\n format_string = get_arg(\"format\", None)\n if format_string is None:\n nfmt = \"{fullname}\" if namespaces else \"{name}\"\n ffmt = \"\"\n if flags:\n ffmt += \" {compiler_flags}\"\n vfmt = \"{variants}\" if variants else \"\"\n hfmt = \"{/abstract_hash}\"\n format_string = nfmt + \"{@version}\" + vfmt + ffmt + hfmt\n\n if specfile_format:\n format_string = \"[{specfile_version}] \" + format_string\n\n def fmt(s, depth=0):\n \"\"\"Formatter function for all output specs\"\"\"\n string = \"\"\n\n if status_fn:\n # This was copied from spec.tree's colorization logic\n # then shortened because it seems like status_fn should\n # always return an InstallStatus\n string += colorize(status_fn(s).value)\n\n if hashes:\n string += gray_hash(s, hlen) + \" \"\n string += depth * \" \"\n string += decorator(s, s.cformat(format_string))\n return string\n\n def format_list(specs):\n \"\"\"Display a single list of specs, with no groups\"\"\"\n # create the final, formatted versions of all specs\n formatted = []\n for spec in specs:\n if deps:\n for depth, dep in traverse.traverse_tree([spec], depth_first=False):\n formatted.append((fmt(dep.spec, depth), dep.spec))\n formatted.append((\"\", None)) # mark newlines\n else:\n formatted.append((fmt(spec), spec))\n\n # unless any of these are set, we can just colify and be done.\n if not any((deps, paths)):\n colify((f[0] for f in formatted), indent=indent, output=output)\n return \"\"\n\n # otherwise, we'll print specs one by one\n max_width = max(len(f[0]) for f in formatted)\n path_fmt = \"%%-%ds%%s\" % (max_width + 2)\n\n out = \"\"\n # getting lots of prefixes requires DB lookups. Ensure\n # all spec.prefix calls are in one transaction.\n with spack.store.STORE.db.read_transaction():\n for string, spec in formatted:\n if not string:\n # print newline from above\n out += \"\\n\"\n continue\n\n if paths:\n out += path_fmt % (string, spec.prefix) + \"\\n\"\n else:\n out += string + \"\\n\"\n\n return out\n\n out = \"\"\n if groups:\n for specs in iter_groups(specs, indent, all_headers):\n output.write(format_list(specs))\n else:\n out = format_list(sorted(specs))\n\n output.write(out)\n output.flush()\n\n\ndef filter_loaded_specs(specs):\n \"\"\"Filter a list of specs returning only those that are\n currently loaded.\"\"\"\n hashes = os.environ.get(uenv.spack_loaded_hashes_var, \"\").split(os.pathsep)\n return [x for x in specs if x.dag_hash() in hashes]\n\n\ndef print_how_many_pkgs(specs, pkg_type=\"\", suffix=\"\"):\n \"\"\"Given a list of specs, this will print a message about how many\n specs are in that list.\n\n Args:\n specs (list): depending on how many items are in this list, choose\n the plural or singular form of the word \"package\"\n pkg_type (str): the output string will mention this provided\n category, e.g. if pkg_type is \"installed\" then the message\n would be \"3 installed packages\"\n \"\"\"\n tty.msg(\"%s\" % spack.llnl.string.plural(len(specs), pkg_type + \" package\") + suffix)\n\n\ndef spack_is_git_repo():\n \"\"\"Ensure that this instance of Spack is a git clone.\"\"\"\n return is_git_repo(spack.paths.prefix)\n\n\ndef is_git_repo(path):\n dotgit_path = join_path(path, \".git\")\n if os.path.isdir(dotgit_path):\n # we are in a regular git repo\n return True\n if os.path.isfile(dotgit_path):\n # we might be in a git worktree\n try:\n with open(dotgit_path, \"rb\") as f:\n dotgit_content = syaml.load(f)\n return os.path.isdir(dotgit_content.get(\"gitdir\", dotgit_path))\n except syaml.SpackYAMLError:\n pass\n return False\n\n\nclass PythonNameError(spack.error.SpackError):\n \"\"\"Exception class thrown for impermissible python names\"\"\"\n\n def __init__(self, name):\n self.name = name\n super().__init__(\"{0} is not a permissible Python name.\".format(name))\n\n\nclass CommandNameError(spack.error.SpackError):\n \"\"\"Exception class thrown for impermissible command names\"\"\"\n\n def __init__(self, name):\n self.name = name\n super().__init__(\"{0} is not a permissible Spack command name.\".format(name))\n\n\nclass MultipleSpecsMatch(Exception):\n \"\"\"Raised when multiple specs match a constraint, in a context where\n this is not allowed.\n \"\"\"\n\n\nclass NoSpecMatches(Exception):\n \"\"\"Raised when no spec matches a constraint, in a context where\n this is not allowed.\n \"\"\"\n\n\n########################################\n# argparse types for argument validation\n########################################\ndef extant_file(f):\n \"\"\"\n Argparse type for files that exist.\n \"\"\"\n if not os.path.isfile(f):\n raise argparse.ArgumentTypeError(\"%s does not exist\" % f)\n return f\n\n\ndef require_active_env(cmd_name):\n \"\"\"Used by commands to get the active environment\n\n If an environment is not found, print an error message that says the calling\n command *needs* an active environment.\n\n Arguments:\n cmd_name (str): name of calling command\n\n Returns:\n (spack.environment.Environment): the active environment\n \"\"\"\n env = ev.active_environment()\n\n if env:\n return env\n\n tty.die(\n \"`spack %s` requires an environment\" % cmd_name,\n \"activate an environment first:\",\n \" spack env activate ENV\",\n \"or use:\",\n \" spack -e ENV %s ...\" % cmd_name,\n )\n\n\ndef find_environment(args: argparse.Namespace) -> Optional[ev.Environment]:\n \"\"\"Find active environment from args or environment variable.\n\n Check for an environment in this order:\n\n 1. via ``spack -e ENV`` or ``spack -D DIR`` (arguments)\n 2. via a path in the spack.environment.spack_env_var environment variable.\n\n If an environment is found, read it in. If not, return None.\n\n Arguments:\n args: argparse namespace with command arguments\n\n Returns: a found environment, or ``None``\n \"\"\"\n\n # treat env as a name\n env = args.env\n if env:\n if ev.exists(env):\n return ev.read(env)\n\n else:\n # if env was specified, see if it is a directory otherwise, look\n # at env_dir (env and env_dir are mutually exclusive)\n env = args.env_dir\n\n # if no argument, look for the environment variable\n if not env:\n env = os.environ.get(ev.spack_env_var)\n\n # nothing was set; there's no active environment\n if not env:\n return None\n\n # if we get here, env isn't the name of a spack environment; it has\n # to be a path to an environment, or there is something wrong.\n if ev.is_env_dir(env):\n return ev.Environment(env)\n\n raise ev.SpackEnvironmentError(\"no environment in %s\" % env)\n\n\ndef doc_first_line(function: object) -> Optional[str]:\n \"\"\"Return the first line of the docstring.\"\"\"\n return function.__doc__.split(\"\\n\", 1)[0].strip() if function.__doc__ else None\n\n\nif sys.version_info >= (3, 13):\n # indent of __doc__ is automatically removed in 3.13+\n # see https://github.com/python/cpython/commit/2566b74b26bcce24199427acea392aed644f4b17\n def doc_dedented(function: object) -> Optional[str]:\n \"\"\"Return the docstring with leading indentation removed.\"\"\"\n return function.__doc__\n\nelse:\n\n def doc_dedented(function: object) -> Optional[str]:\n \"\"\"Return the docstring with leading indentation removed.\"\"\"\n return textwrap.dedent(function.__doc__) if function.__doc__ else None\n\n\ndef converted_arg_length(arg: str):\n if sys.platform == \"win32\":\n # An argument may have extra characters inserted for a command\n # line invocation (e.g. on Windows, an argument with a space\n # is quoted)\n return len(subprocess.list2cmdline([arg]))\n else:\n return len(arg)\n\n\ndef group_arguments(\n args: Sequence[str],\n *,\n max_group_size: int = 500,\n prefix_length: int = 0,\n max_group_length: Optional[int] = None,\n) -> Generator[List[str], None, None]:\n \"\"\"Splits the supplied list of arguments into groups for passing to CLI tools.\n\n When passing CLI arguments, we need to ensure that argument lists are no longer than\n the system command line size limit, and we may also need to ensure that groups are\n no more than some number of arguments long.\n\n This returns an iterator over lists of arguments that meet these constraints.\n Arguments are in the same order they appeared in the original argument list.\n\n If any argument's length is greater than the max_group_length, this will raise a\n ``ValueError``.\n\n Arguments:\n args: list of arguments to split into groups\n max_group_size: max number of elements in any group (default 500)\n prefix_length: length of any additional arguments (including spaces) to be passed before\n the groups from args; default is 0 characters\n max_group_length: max length of characters that if a group of args is joined by ``\" \"``\n On unix, this defaults to SC_ARG_MAX from sysconf. On Windows the default is\n the max usable for CreateProcess (32,768 chars)\n\n \"\"\"\n if max_group_length is None:\n # Windows limit is 32767, including null terminator (not measured by len)\n # so max length is 32766\n max_group_length = 32766\n if hasattr(os, \"sysconf\"): # sysconf is only on unix\n try:\n # returns -1 if an option isn't present (some older POSIXes)\n sysconf_max = os.sysconf(\"SC_ARG_MAX\")\n max_group_length = sysconf_max if sysconf_max != -1 else max_group_length\n except (ValueError, OSError):\n pass # keep windows default if SC_ARG_MAX isn't in sysconf_names\n\n group: List[str] = []\n grouplen, space = prefix_length, 0\n for arg in args:\n arglen = converted_arg_length(arg)\n if arglen > max_group_length:\n raise ValueError(f\"Argument is longer than max command line size: '{arg}'\")\n if arglen + prefix_length > max_group_length:\n raise ValueError(f\"Argument with prefix is longer than max command line size: '{arg}'\")\n\n next_grouplen = grouplen + arglen + space\n if len(group) == max_group_size or next_grouplen > max_group_length:\n yield group\n group, grouplen, space = [], prefix_length, 0\n\n group.append(arg)\n grouplen += arglen + space\n space = 1 # add a space for elements 1, 2, etc. but not 0\n\n if group:\n yield group\n\n\nclass CommandNotFoundError(spack.error.SpackError):\n \"\"\"Exception class thrown when a requested command is not recognized as\n such.\n \"\"\"\n\n def __init__(self, cmd_name):\n msg = (\n f\"{cmd_name} is not a recognized Spack command or extension command; \"\n \"check with `spack commands`.\"\n )\n long_msg = None\n\n similar = difflib.get_close_matches(cmd_name, all_commands())\n\n if 1 <= len(similar) <= 5:\n long_msg = \"\\nDid you mean one of the following commands?\\n \"\n long_msg += \"\\n \".join(similar)\n\n super().__init__(msg, long_msg)\n" }, { "path": "lib/spack/spack/cmd/add.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\n\nimport spack.cmd\nimport spack.llnl.util.tty as tty\nfrom spack.cmd.common import arguments\n\ndescription = \"add a spec to an environment\"\nsection = \"environments\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n subparser.add_argument(\n \"-l\",\n \"--list-name\",\n dest=\"list_name\",\n default=\"specs\",\n help=\"name of the list to add specs to\",\n )\n arguments.add_common_arguments(subparser, [\"specs\"])\n\n\ndef add(parser, args):\n env = spack.cmd.require_active_env(cmd_name=\"add\")\n\n with env.write_transaction():\n for spec in spack.cmd.parse_specs(args.specs):\n if not env.add(spec, args.list_name):\n tty.msg(\"Package {0} was already added to {1}\".format(spec.name, env.name))\n else:\n tty.msg(\"Adding %s to environment %s\" % (spec, env.name))\n env.write()\n" }, { "path": "lib/spack/spack/cmd/arch.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport collections\nimport warnings\n\nimport spack.vendor.archspec.cpu\n\nimport spack.llnl.util.tty.colify as colify\nimport spack.llnl.util.tty.color as color\nimport spack.platforms\nimport spack.spec\n\ndescription = \"print architecture information about this machine\"\nsection = \"config\"\nlevel = \"short\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n # DEPRECATED: equivalent to --generic --target\n subparser.add_argument(\n \"-g\",\n \"--generic-target\",\n action=\"store_true\",\n help=\"show the best generic target (deprecated)\",\n )\n subparser.add_argument(\n \"--known-targets\", action=\"store_true\", help=\"show a list of all known targets and exit\"\n )\n target_type = subparser.add_mutually_exclusive_group()\n target_type.add_argument(\n \"--family\", action=\"store_true\", help=\"print generic ISA (x86_64, aarch64, ppc64le, ...)\"\n )\n target_type.add_argument(\n \"--generic\", action=\"store_true\", help=\"print feature level (x86_64_v3, armv8.4a, ...)\"\n )\n parts = subparser.add_mutually_exclusive_group()\n parts2 = subparser.add_mutually_exclusive_group()\n parts.add_argument(\n \"-p\", \"--platform\", action=\"store_true\", default=False, help=\"print only the platform\"\n )\n parts.add_argument(\n \"-o\",\n \"--operating-system\",\n action=\"store_true\",\n default=False,\n help=\"print only the operating system\",\n )\n parts.add_argument(\n \"-t\", \"--target\", action=\"store_true\", default=False, help=\"print only the target\"\n )\n parts2.add_argument(\n \"-f\", \"--frontend\", action=\"store_true\", default=False, help=\"print frontend (DEPRECATED)\"\n )\n parts2.add_argument(\n \"-b\", \"--backend\", action=\"store_true\", default=False, help=\"print backend (DEPRECATED)\"\n )\n\n\ndef display_targets(targets):\n \"\"\"Prints a human readable list of the targets passed as argument.\"\"\"\n by_vendor = collections.defaultdict(list)\n for _, target in targets.items():\n by_vendor[target.vendor].append(target)\n\n def display_target_group(header, target_group):\n print(header)\n colify.colify(target_group, indent=4)\n print(\"\")\n\n generic_architectures = by_vendor.pop(\"generic\", None)\n if generic_architectures:\n header = color.colorize(r\"@*B{Generic architectures (families)}\")\n group = sorted(generic_architectures, key=lambda x: str(x))\n display_target_group(header, group)\n\n for vendor, vendor_targets in by_vendor.items():\n by_family = collections.defaultdict(list)\n for t in vendor_targets:\n by_family[str(t.family)].append(t)\n\n for family, group in by_family.items():\n vendor = color.colorize(r\"@*B{\" + vendor + r\"}\")\n family = color.colorize(r\"@*B{\" + family + r\"}\")\n header = \" - \".join([vendor, family])\n group = sorted(group, key=lambda x: len(x.ancestors))\n display_target_group(header, group)\n\n\ndef arch(parser, args):\n if args.generic_target:\n # TODO: add deprecation warning in 0.24\n print(spack.vendor.archspec.cpu.host().generic)\n return\n\n if args.known_targets:\n display_targets(spack.vendor.archspec.cpu.TARGETS)\n return\n\n if args.frontend:\n warnings.warn(\"the argument --frontend is deprecated, and will be removed in Spack v1.0\")\n elif args.backend:\n warnings.warn(\"the argument --backend is deprecated, and will be removed in Spack v1.0\")\n\n host_platform = spack.platforms.host()\n host_os = host_platform.default_operating_system()\n host_target = host_platform.default_target()\n if args.family:\n host_target = host_target.family\n elif args.generic:\n host_target = host_target.generic\n architecture = spack.spec.ArchSpec((str(host_platform), str(host_os), str(host_target)))\n\n if args.platform:\n print(architecture.platform)\n elif args.operating_system:\n print(architecture.os)\n elif args.target:\n print(architecture.target)\n else:\n print(architecture)\n" }, { "path": "lib/spack/spack/cmd/audit.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport argparse\nimport warnings\n\nimport spack.audit\nimport spack.llnl.util.tty as tty\nimport spack.llnl.util.tty.colify\nimport spack.llnl.util.tty.color as cl\nimport spack.repo\n\ndescription = \"audit configuration files, packages, etc.\"\nsection = \"packaging\"\nlevel = \"short\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n # Top level flags, valid for every audit class\n sp = subparser.add_subparsers(metavar=\"SUBCOMMAND\", dest=\"subcommand\")\n\n # Audit configuration files\n sp.add_parser(\"configs\", help=\"audit configuration files\")\n\n # Audit package recipes\n external_parser = sp.add_parser(\"externals\", help=\"check external detection in packages\")\n external_parser.add_argument(\n \"--list\",\n action=\"store_true\",\n dest=\"list_externals\",\n help=\"if passed, list which packages have detection tests\",\n )\n\n # Https and other linting\n https_parser = sp.add_parser(\"packages-https\", help=\"check https in packages\")\n https_parser.add_argument(\n \"--all\", action=\"store_true\", default=False, dest=\"check_all\", help=\"audit all packages\"\n )\n\n # Audit package recipes\n pkg_parser = sp.add_parser(\"packages\", help=\"audit package recipes\")\n\n for group in [pkg_parser, https_parser, external_parser]:\n group.add_argument(\n \"name\",\n metavar=\"PKG\",\n nargs=\"*\",\n help=\"package to be analyzed (if none all packages will be processed)\",\n )\n\n # List all checks\n sp.add_parser(\"list\", help=\"list available checks and exits\")\n\n\ndef configs(parser, args):\n with warnings.catch_warnings():\n warnings.simplefilter(\"ignore\")\n reports = spack.audit.run_group(args.subcommand)\n _process_reports(reports)\n\n\ndef packages(parser, args):\n pkgs = args.name or spack.repo.PATH.all_package_names()\n reports = spack.audit.run_group(args.subcommand, pkgs=pkgs)\n _process_reports(reports)\n\n\ndef packages_https(parser, args):\n # Since packages takes a long time, --all is required without name\n if not args.check_all and not args.name:\n tty.die(\"Please specify one or more packages to audit, or --all.\")\n\n pkgs = args.name or spack.repo.PATH.all_package_names()\n reports = spack.audit.run_group(args.subcommand, pkgs=pkgs)\n _process_reports(reports)\n\n\ndef externals(parser, args):\n if args.list_externals:\n msg = \"@*{The following packages have detection tests:}\"\n tty.msg(cl.colorize(msg))\n spack.llnl.util.tty.colify.colify(spack.audit.packages_with_detection_tests(), indent=2)\n return\n\n pkgs = args.name or spack.repo.PATH.all_package_names()\n reports = spack.audit.run_group(args.subcommand, pkgs=pkgs, debug_log=tty.debug)\n _process_reports(reports)\n\n\ndef list(parser, args):\n for subcommand, check_tags in spack.audit.GROUPS.items():\n print(cl.colorize(\"@*b{\" + subcommand + \"}:\"))\n for tag in check_tags:\n audit_obj = spack.audit.CALLBACKS[tag]\n print(\" \" + audit_obj.description)\n if args.verbose:\n for idx, fn in enumerate(audit_obj.callbacks):\n print(\" {0}. \".format(idx + 1) + fn.__doc__)\n print()\n print()\n\n\ndef audit(parser, args):\n subcommands = {\n \"configs\": configs,\n \"externals\": externals,\n \"packages\": packages,\n \"packages-https\": packages_https,\n \"list\": list,\n }\n subcommands[args.subcommand](parser, args)\n\n\ndef _process_reports(reports):\n for check, errors in reports:\n if errors:\n status = f\"{len(errors)} issue{'' if len(errors) == 1 else 's'} found\"\n print(cl.colorize(f\"{check}: @*r{{{status}}}\"))\n numdigits = len(str(len(errors)))\n for idx, error in enumerate(errors):\n print(f\"{idx + 1:>{numdigits}}. {error}\")\n raise SystemExit(1)\n else:\n print(cl.colorize(f\"{check}: @*g{{passed}}\"))\n" }, { "path": "lib/spack/spack/cmd/blame.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport os\nimport pathlib\nimport re\nimport sys\nfrom typing import Optional, Union\n\nimport spack.config\nimport spack.llnl.util.tty as tty\nimport spack.repo\nimport spack.util.git\nimport spack.util.spack_json as sjson\nfrom spack.cmd import spack_is_git_repo\nfrom spack.llnl.util.filesystem import working_dir\nfrom spack.llnl.util.lang import pretty_date\nfrom spack.llnl.util.tty.colify import colify_table\nfrom spack.util.executable import ProcessError\n\ndescription = \"show contributors to packages\"\nsection = \"query\"\nlevel = \"long\"\n\ngit = spack.util.git.git(required=True)\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n view_group = subparser.add_mutually_exclusive_group()\n view_group.add_argument(\n \"-t\",\n \"--time\",\n dest=\"view\",\n action=\"store_const\",\n const=\"time\",\n default=\"time\",\n help=\"sort by last modification date (default)\",\n )\n view_group.add_argument(\n \"-p\",\n \"--percent\",\n dest=\"view\",\n action=\"store_const\",\n const=\"percent\",\n help=\"sort by percent of code\",\n )\n view_group.add_argument(\n \"-g\",\n \"--git\",\n dest=\"view\",\n action=\"store_const\",\n const=\"git\",\n help=\"show git blame output instead of summary\",\n )\n subparser.add_argument(\n \"--json\",\n action=\"store_true\",\n default=False,\n help=\"output blame as machine-readable json records\",\n )\n\n subparser.add_argument(\n \"package_or_file\",\n help=\"name of package to show contributions for, or path to a file in the spack repo\",\n )\n\n\ndef print_table(rows, last_mod, total_lines, emails):\n \"\"\"\n Given a set of rows with authors and lines, print a table.\n \"\"\"\n table = [[\"LAST_COMMIT\", \"LINES\", \"%\", \"AUTHOR\", \"EMAIL\"]]\n for author, nlines in rows:\n table += [\n [\n pretty_date(last_mod[author]),\n nlines,\n round(nlines / float(total_lines) * 100, 1),\n author,\n emails[author],\n ]\n ]\n\n table += [[\"\"] * 5]\n table += [[pretty_date(max(last_mod.values())), total_lines, \"100.0\"] + [\"\"] * 3]\n\n colify_table(table)\n\n\ndef dump_json(rows, last_mod, total_lines, emails):\n \"\"\"\n Dump the blame as a json object to the terminal.\n \"\"\"\n result = {}\n authors = []\n for author, nlines in rows:\n authors.append(\n {\n \"last_commit\": pretty_date(last_mod[author]),\n \"lines\": nlines,\n \"percentage\": round(nlines / float(total_lines) * 100, 1),\n \"author\": author,\n \"email\": emails[author],\n }\n )\n\n result[\"authors\"] = authors\n result[\"totals\"] = {\n \"last_commit\": pretty_date(max(last_mod.values())),\n \"lines\": total_lines,\n \"percentage\": \"100.0\",\n }\n\n sjson.dump(result, sys.stdout)\n\n\ndef git_prefix(path: Union[str, pathlib.Path]) -> Optional[pathlib.Path]:\n \"\"\"Return the top level directory if path is under a git repository.\n\n Args:\n path: path of the item presumably under a git repository\n\n Returns: path to the root of the git repository\n \"\"\"\n if not os.path.exists(path):\n return None\n\n work_dir = path if os.path.isdir(path) else os.path.dirname(path)\n with working_dir(work_dir):\n try:\n result = git(\"rev-parse\", \"--show-toplevel\", output=str, error=str)\n return pathlib.Path(result.split(\"\\n\")[0])\n except ProcessError:\n tty.die(f\"'{path}' is not in a git repository.\")\n\n\ndef package_repo_root(path: Union[str, pathlib.Path]) -> Optional[pathlib.Path]:\n \"\"\"Find the appropriate package repository's git root directory.\n\n Provides a warning for a remote package repository since there is a risk that\n the blame results are inaccurate.\n\n Args:\n path: path to an arbitrary file presumably in one of the spack package repos\n\n Returns: path to the package repository's git root directory or None\n \"\"\"\n descriptors = spack.repo.RepoDescriptors.from_config(\n lock=spack.repo.package_repository_lock(), config=spack.config.CONFIG\n )\n path = pathlib.Path(path)\n prefix: Optional[pathlib.Path] = None\n for _, desc in descriptors.items():\n # Handle the remote case, whose destination is by definition the git root\n if hasattr(desc, \"destination\"):\n repo_dest = pathlib.Path(desc.destination)\n if (repo_dest / \".git\").exists():\n prefix = repo_dest\n\n # TODO: replace check with `is_relative_to` once supported\n if prefix and str(path).startswith(str(prefix)):\n return prefix\n\n # Handle the local repository case, making sure it's a spack repository.\n if hasattr(desc, \"path\"):\n repo_path = pathlib.Path(desc.path)\n if \"spack_repo\" in repo_path.parts:\n prefix = git_prefix(repo_path)\n\n # TODO: replace check with `is_relative_to` once supported\n if prefix and str(path).startswith(str(prefix)):\n return prefix\n\n return None\n\n\ndef git_supports_unshallow() -> bool:\n output = git(\"fetch\", \"--help\", output=str, error=str)\n return \"--unshallow\" in output\n\n\ndef ensure_full_history(prefix: str, path: str) -> None:\n \"\"\"Ensure the git repository at the prefix has its full history.\n\n Args:\n prefix: the root directory of the git repository\n path: the package or file name under consideration (for messages)\n \"\"\"\n assert os.path.isdir(prefix)\n\n with working_dir(prefix):\n shallow_dir = os.path.join(prefix, \".git\", \"shallow\")\n if os.path.isdir(shallow_dir):\n if git_supports_unshallow():\n try:\n # Capture the error output (e.g., irrelevant for full repo)\n # to ensure the output is clean.\n git(\"fetch\", \"--unshallow\", error=str)\n except ProcessError as e:\n tty.die(\n f\"Cannot report blame for {path}.\\n\"\n \"Unable to retrieve the full git history for \"\n f'{prefix} due to \"{str(e)}\" error.'\n )\n else:\n tty.die(\n f\"Cannot report blame for {path}.\\n\"\n f\"Unable to retrieve the full git history for {prefix}. \"\n \"Use a newer 'git' that supports 'git fetch --unshallow'.\"\n )\n\n\ndef blame(parser, args):\n # make sure this is a git repo\n if not spack_is_git_repo():\n tty.die(\"This spack is not a git clone. You cannot use 'spack blame'.\")\n\n # Get the name of the path to blame and its repository prefix\n # so we can honor any .git-blame-ignore-revs that may be present.\n blame_file = None\n prefix = None\n if os.path.exists(args.package_or_file):\n blame_file = os.path.realpath(args.package_or_file)\n prefix = package_repo_root(blame_file)\n\n # Get path to what we assume is a package (including to a cached version\n # of a remote package repository.)\n if not blame_file:\n try:\n blame_file = spack.repo.PATH.filename_for_package_name(args.package_or_file)\n except spack.repo.UnknownNamespaceError:\n # the argument is not a package (or does not exist)\n pass\n\n if blame_file and os.path.isfile(blame_file):\n prefix = package_repo_root(blame_file)\n\n if not blame_file or not os.path.exists(blame_file):\n tty.die(f\"'{args.package_or_file}' does not exist.\")\n\n if prefix is None:\n tty.msg(f\"'{args.package_or_file}' is not within a spack package repository\")\n\n path_prefix = git_prefix(blame_file)\n if path_prefix != prefix:\n # You are attempting to get 'blame' for a path outside of a configured\n # package repository (e.g., within a spack/spack clone). We'll use the\n # path's prefix instead to ensure working under the proper git\n # repository.\n prefix = path_prefix\n\n # Make sure we can get the full/known blame even when the repository\n # is remote.\n ensure_full_history(prefix, args.package_or_file)\n\n # Get blame information for the path EVEN when it is located in a different\n # spack repository (e.g., spack/spack-packages) or a different git\n # repository.\n with working_dir(prefix):\n # Now we can get the blame results.\n options = [\"blame\"]\n\n # ignore the great black reformatting of 2022\n ignore_file = prefix / \".git-blame-ignore-revs\"\n if ignore_file.exists():\n options.extend([\"--ignore-revs-file\", str(ignore_file)])\n\n try:\n if args.view == \"git\":\n options.append(str(blame_file))\n git(*options)\n return\n else:\n options.extend([\"--line-porcelain\", str(blame_file)])\n output = git(*options, output=str, error=str)\n lines = output.split(\"\\n\")\n except ProcessError as err:\n # e.g., blame information is not tracked if the path is a directory\n tty.die(f\"Blame information is not tracked for '{blame_file}':\\n{err.long_message}\")\n\n # Histogram authors\n counts = {}\n emails = {}\n last_mod = {}\n total_lines = 0\n for line in lines:\n match = re.match(r\"^author (.*)\", line)\n if match:\n author = match.group(1)\n\n match = re.match(r\"^author-mail (.*)\", line)\n if match:\n email = match.group(1)\n\n match = re.match(r\"^author-time (.*)\", line)\n if match:\n mod = int(match.group(1))\n last_mod[author] = max(last_mod.setdefault(author, 0), mod)\n\n # ignore comments\n if re.match(r\"^\\t[^#]\", line):\n counts[author] = counts.setdefault(author, 0) + 1\n emails.setdefault(author, email)\n total_lines += 1\n\n if args.view == \"time\":\n rows = sorted(counts.items(), key=lambda t: last_mod[t[0]], reverse=True)\n else: # args.view == 'percent'\n rows = sorted(counts.items(), key=lambda t: t[1], reverse=True)\n\n # Dump as json\n if args.json:\n dump_json(rows, last_mod, total_lines, emails)\n\n # Print a nice table with authors and emails\n else:\n print_table(rows, last_mod, total_lines, emails)\n" }, { "path": "lib/spack/spack/cmd/bootstrap.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport argparse\nimport os\nimport pathlib\nimport shutil\nimport sys\nimport tempfile\n\nimport spack\nimport spack.bootstrap\nimport spack.bootstrap.config\nimport spack.bootstrap.core\nimport spack.cmd.mirror\nimport spack.concretize\nimport spack.config\nimport spack.llnl.util.filesystem\nimport spack.llnl.util.tty\nimport spack.llnl.util.tty.color\nimport spack.stage\nimport spack.util.path\nimport spack.util.spack_yaml\nfrom spack.cmd.common import arguments\n\ndescription = \"manage bootstrap configuration\"\nsection = \"admin\"\nlevel = \"long\"\n\n\n# Tarball to be downloaded if binary packages are requested in a local mirror\nBINARY_TARBALL = \"https://github.com/spack/spack-bootstrap-mirrors/releases/download/v2.2/bootstrap-buildcache.tar.gz\"\n\n#: Subdirectory where to create the mirror\nLOCAL_MIRROR_DIR = \"bootstrap_cache\"\n\n# Metadata for a generated binary mirror\nBINARY_METADATA = {\n \"type\": \"buildcache\",\n \"description\": (\n \"Buildcache copied from a public tarball available on Github.\"\n \"The sha256 checksum of binaries is checked before installation.\"\n ),\n \"info\": {\n # This is a mis-nomer since it's not a URL; but file urls cannot\n # represent relative paths, so we have to live with it for now.\n \"url\": os.path.join(\"..\", \"..\", LOCAL_MIRROR_DIR),\n \"homepage\": \"https://github.com/spack/spack-bootstrap-mirrors\",\n \"releases\": \"https://github.com/spack/spack-bootstrap-mirrors/releases\",\n \"tarball\": BINARY_TARBALL,\n },\n}\n\nCLINGO_JSON = \"$spack/share/spack/bootstrap/github-actions-v2/clingo.json\"\nGNUPG_JSON = \"$spack/share/spack/bootstrap/github-actions-v2/gnupg.json\"\nPATCHELF_JSON = \"$spack/share/spack/bootstrap/github-actions-v2/patchelf.json\"\n\n# Metadata for a generated source mirror\nSOURCE_METADATA = {\n \"type\": \"install\",\n \"description\": \"Mirror with software needed to bootstrap Spack\",\n \"info\": {\n # This is a mis-nomer since it's not a URL; but file urls cannot\n # represent relative paths, so we have to live with it for now.\n \"url\": os.path.join(\"..\", \"..\", LOCAL_MIRROR_DIR)\n },\n}\n\n\ndef _add_scope_option(parser):\n parser.add_argument(\n \"--scope\", action=arguments.ConfigScope, help=\"configuration scope to read/modify\"\n )\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n sp = subparser.add_subparsers(dest=\"subcommand\")\n\n now = sp.add_parser(\"now\", help=\"Spack ready, right now!\")\n now.add_argument(\"--dev\", action=\"store_true\", help=\"bootstrap dev dependencies too\")\n\n status = sp.add_parser(\"status\", help=\"get the status of Spack\")\n status.add_argument(\n \"--optional\",\n action=\"store_true\",\n default=False,\n help=\"show the status of rarely used optional dependencies\",\n )\n status.add_argument(\n \"--dev\",\n action=\"store_true\",\n default=False,\n help=\"show the status of dependencies needed to develop Spack\",\n )\n\n enable = sp.add_parser(\"enable\", help=\"enable bootstrapping\")\n _add_scope_option(enable)\n enable.add_argument(\"name\", help=\"name of the source to be enabled\", nargs=\"?\", default=None)\n\n disable = sp.add_parser(\"disable\", help=\"disable bootstrapping\")\n _add_scope_option(disable)\n disable.add_argument(\"name\", help=\"name of the source to be disabled\", nargs=\"?\", default=None)\n\n reset = sp.add_parser(\"reset\", help=\"reset bootstrapping configuration to Spack defaults\")\n arguments.add_common_arguments(reset, [\"yes_to_all\"])\n\n root = sp.add_parser(\"root\", help=\"get/set the root bootstrap directory\")\n _add_scope_option(root)\n root.add_argument(\n \"path\", nargs=\"?\", default=None, help=\"set the bootstrap directory to this value\"\n )\n\n list = sp.add_parser(\"list\", help=\"list all the sources of software to bootstrap Spack\")\n list.add_argument(\n \"--scope\",\n action=arguments.ConfigScope,\n type=arguments.config_scope_readable_validator,\n help=\"configuration scope to read/modify\",\n )\n\n add = sp.add_parser(\"add\", help=\"add a new source for bootstrapping\")\n _add_scope_option(add)\n add.add_argument(\n \"--trust\", action=\"store_true\", help=\"enable the source immediately upon addition\"\n )\n add.add_argument(\"name\", help=\"name of the new source of software\")\n add.add_argument(\"metadata_dir\", help=\"directory where to find metadata files\")\n\n remove = sp.add_parser(\"remove\", help=\"remove a bootstrapping source\")\n remove.add_argument(\"name\", help=\"name of the source to be removed\")\n\n mirror = sp.add_parser(\"mirror\", help=\"create a local mirror to bootstrap Spack\")\n mirror.add_argument(\n \"--binary-packages\", action=\"store_true\", help=\"download public binaries in the mirror\"\n )\n mirror.add_argument(\"--dev\", action=\"store_true\", help=\"download dev dependencies too\")\n mirror.add_argument(\n metavar=\"DIRECTORY\",\n dest=\"root_dir\",\n help=\"root directory in which to create the mirror and metadata\",\n )\n\n\ndef _enable_or_disable(args):\n value = args.subcommand == \"enable\"\n if args.name is None:\n # Set to True if we called \"enable\", otherwise set to false\n old_value = spack.config.get(\"bootstrap:enable\", scope=args.scope)\n if old_value == value:\n spack.llnl.util.tty.msg(\"Bootstrapping is already {}d\".format(args.subcommand))\n else:\n spack.config.set(\"bootstrap:enable\", value, scope=args.scope)\n spack.llnl.util.tty.msg(\"Bootstrapping has been {}d\".format(args.subcommand))\n return\n\n if value is True:\n _enable_source(args)\n else:\n _disable_source(args)\n\n\ndef _reset(args):\n if not args.yes_to_all:\n msg = [\n \"Bootstrapping configuration is being reset to Spack's defaults. \"\n \"Current configuration will be lost.\\n\",\n \"Do you want to continue?\",\n ]\n ok_to_continue = spack.llnl.util.tty.get_yes_or_no(\"\".join(msg), default=True)\n if not ok_to_continue:\n raise RuntimeError(\"Aborting\")\n\n for scope in spack.config.CONFIG.writable_scopes:\n # The default scope should stay untouched\n if scope.name == \"defaults\":\n continue\n\n # If we are in an env scope we can't delete a file, but the best we\n # can do is nullify the corresponding configuration\n if scope.name.startswith(\"env\") and spack.config.get(\"bootstrap\", scope=scope.name):\n spack.config.set(\"bootstrap\", {}, scope=scope.name)\n continue\n\n # If we are outside of an env scope delete the bootstrap.yaml file\n bootstrap_yaml = os.path.join(scope.path, \"bootstrap.yaml\")\n backup_file = bootstrap_yaml + \".bkp\"\n if os.path.exists(bootstrap_yaml):\n shutil.move(bootstrap_yaml, backup_file)\n\n spack.config.CONFIG.clear_caches()\n\n\ndef _root(args):\n if args.path:\n spack.config.set(\"bootstrap:root\", args.path, scope=args.scope)\n elif args.scope:\n if args.scope not in spack.config.existing_scope_names():\n spack.llnl.util.tty.die(\n f\"The argument --scope={args.scope} must refer to an existing scope.\"\n )\n\n root = spack.config.get(\"bootstrap:root\", default=None, scope=args.scope)\n if root:\n root = spack.util.path.canonicalize_path(root)\n print(root)\n\n\ndef _list(args):\n sources = spack.bootstrap.core.bootstrapping_sources(scope=args.scope)\n if not sources:\n spack.llnl.util.tty.msg(\"No method available for bootstrapping Spack's dependencies\")\n return\n\n def _print_method(source, trusted):\n color = spack.llnl.util.tty.color\n\n def fmt(header, content):\n header_fmt = \"@*b{{{0}:}} {1}\"\n color.cprint(header_fmt.format(header, content))\n\n trust_str = \"@*y{DISABLED}\"\n if trusted is True:\n trust_str = \"@*g{ENABLED}\"\n elif trusted is False:\n trust_str = \"@*r{DISABLED}\"\n\n fmt(\"Name\", source[\"name\"] + \" \" + trust_str)\n print()\n if trusted is True or args.verbose:\n fmt(\" Type\", source[\"type\"])\n print()\n\n info_lines = [\"\\n\"]\n for key, value in source.get(\"info\", {}).items():\n info_lines.append(\" \" * 4 + \"@*{{{0}}}: {1}\\n\".format(key, value))\n if len(info_lines) > 1:\n fmt(\" Info\", \"\".join(info_lines))\n\n description_lines = [\"\\n\"]\n for line in source[\"description\"].split(\"\\n\"):\n description_lines.append(\" \" * 4 + line + \"\\n\")\n\n fmt(\" Description\", \"\".join(description_lines))\n\n trusted = spack.config.get(\"bootstrap:trusted\", {})\n\n def sort_fn(x):\n x_trust = trusted.get(x[\"name\"], None)\n if x_trust is True:\n return 0\n elif x_trust is None:\n return 1\n return 2\n\n sources = sorted(sources, key=sort_fn)\n for s in sources:\n _print_method(s, trusted.get(s[\"name\"], None))\n\n\ndef _write_bootstrapping_source_status(name, enabled, scope=None):\n \"\"\"Write if a bootstrapping source is enable or disabled to config file.\n\n Args:\n name (str): name of the bootstrapping source.\n enabled (bool): True if the source is enabled, False if it is disabled.\n scope (None or str): configuration scope to modify. If none use the default scope.\n \"\"\"\n sources = spack.config.get(\"bootstrap:sources\")\n\n matches = [s for s in sources if s[\"name\"] == name]\n if not matches:\n names = [s[\"name\"] for s in sources]\n msg = 'there is no bootstrapping method named \"{0}\". Valid method names are: {1}'.format(\n name, \", \".join(names)\n )\n raise RuntimeError(msg)\n\n if len(matches) > 1:\n msg = (\n 'there is more than one bootstrapping method named \"{0}\". '\n \"Please delete all methods but one from bootstrap.yaml \"\n \"before proceeding\"\n ).format(name)\n raise RuntimeError(msg)\n\n # Setting the scope explicitly is needed to not copy over to a new scope\n # the entire default configuration for bootstrap.yaml\n scope = scope or spack.config.default_modify_scope(\"bootstrap\")\n spack.config.add(\"bootstrap:trusted:{0}:{1}\".format(name, str(enabled)), scope=scope)\n\n\ndef _enable_source(args):\n _write_bootstrapping_source_status(args.name, enabled=True, scope=args.scope)\n msg = '\"{0}\" is now enabled for bootstrapping'\n spack.llnl.util.tty.msg(msg.format(args.name))\n\n\ndef _disable_source(args):\n _write_bootstrapping_source_status(args.name, enabled=False, scope=args.scope)\n msg = '\"{0}\" is now disabled and will not be used for bootstrapping'\n spack.llnl.util.tty.msg(msg.format(args.name))\n\n\ndef _status(args):\n sections = [\"core\", \"buildcache\"]\n if args.optional:\n sections.append(\"optional\")\n if args.dev:\n sections.append(\"develop\")\n\n header = \"@*b{{Spack v{0} - {1}}}\".format(\n spack.spack_version, spack.bootstrap.config.spec_for_current_python()\n )\n print(spack.llnl.util.tty.color.colorize(header))\n print()\n # Use the context manager here to avoid swapping between user and\n # bootstrap config many times\n missing = False\n with spack.bootstrap.ensure_bootstrap_configuration():\n for current_section in sections:\n status_msg, fail = spack.bootstrap.status_message(section=current_section)\n missing = missing or fail\n if status_msg:\n print(spack.llnl.util.tty.color.colorize(status_msg))\n print()\n legend = (\n \"Spack will take care of bootstrapping any missing dependency marked\"\n \" as [@*y{B}]. Dependencies marked as [@*y{-}] are instead required\"\n \" to be found on the system.\"\n )\n if missing:\n print(spack.llnl.util.tty.color.colorize(legend))\n print()\n sys.exit(1)\n\n\ndef _add(args):\n initial_sources = spack.bootstrap.core.bootstrapping_sources()\n names = [s[\"name\"] for s in initial_sources]\n\n # If the name is already used error out\n if args.name in names:\n msg = 'a source named \"{0}\" already exist. Please choose a different name'\n raise RuntimeError(msg.format(args.name))\n\n # Check that the metadata file exists\n metadata_dir = spack.util.path.canonicalize_path(args.metadata_dir)\n if not os.path.exists(metadata_dir) or not os.path.isdir(metadata_dir):\n raise RuntimeError('the directory \"{0}\" does not exist'.format(args.metadata_dir))\n\n file = os.path.join(metadata_dir, \"metadata.yaml\")\n if not os.path.exists(file):\n raise RuntimeError('the file \"{0}\" does not exist'.format(file))\n\n # Insert the new source as the highest priority one\n write_scope = args.scope or spack.config.default_modify_scope(section=\"bootstrap\")\n sources = spack.config.get(\"bootstrap:sources\", scope=write_scope) or []\n sources = [{\"name\": args.name, \"metadata\": args.metadata_dir}] + sources\n spack.config.set(\"bootstrap:sources\", sources, scope=write_scope)\n\n msg = 'New bootstrapping source \"{0}\" added in the \"{1}\" configuration scope'\n spack.llnl.util.tty.msg(msg.format(args.name, write_scope))\n if args.trust:\n _enable_source(args)\n\n\ndef _remove(args):\n initial_sources = spack.bootstrap.core.bootstrapping_sources()\n names = [s[\"name\"] for s in initial_sources]\n if args.name not in names:\n msg = (\n 'cannot find any bootstrapping source named \"{0}\". '\n \"Run `spack bootstrap list` to see available sources.\"\n )\n raise RuntimeError(msg.format(args.name))\n\n for current_scope in spack.config.scopes():\n sources = spack.config.get(\"bootstrap:sources\", scope=current_scope) or []\n if args.name in [s[\"name\"] for s in sources]:\n sources = [s for s in sources if s[\"name\"] != args.name]\n spack.config.set(\"bootstrap:sources\", sources, scope=current_scope)\n msg = (\n 'Removed the bootstrapping source named \"{0}\" from the \"{1}\" configuration scope.'\n )\n spack.llnl.util.tty.msg(msg.format(args.name, current_scope))\n trusted = spack.config.get(\"bootstrap:trusted\", scope=current_scope) or []\n if args.name in trusted:\n trusted.pop(args.name)\n spack.config.set(\"bootstrap:trusted\", trusted, scope=current_scope)\n msg = 'Deleting information on \"{0}\" from list of trusted sources'\n spack.llnl.util.tty.msg(msg.format(args.name))\n\n\ndef _mirror(args):\n mirror_dir = spack.util.path.canonicalize_path(os.path.join(args.root_dir, LOCAL_MIRROR_DIR))\n\n # TODO: Here we are adding gnuconfig manually, but this can be fixed\n # TODO: as soon as we have an option to add to a mirror all the possible\n # TODO: dependencies of a spec\n root_specs = spack.bootstrap.all_core_root_specs() + [\"gnuconfig\"]\n if args.dev:\n root_specs += spack.bootstrap.BootstrapEnvironment.spack_dev_requirements()\n\n for spec_str in root_specs:\n msg = 'Adding \"{0}\" and dependencies to the mirror at {1}'\n spack.llnl.util.tty.msg(msg.format(spec_str, mirror_dir))\n # Suppress tty from the call below for terser messages\n spack.llnl.util.tty.set_msg_enabled(False)\n spec = spack.concretize.concretize_one(spec_str)\n for node in spec.traverse():\n if node.external:\n continue\n spack.cmd.mirror.create(mirror_dir, [node])\n spack.llnl.util.tty.set_msg_enabled(True)\n\n if args.binary_packages:\n msg = 'Adding binary packages from \"{0}\" to the mirror at {1}'\n spack.llnl.util.tty.msg(msg.format(BINARY_TARBALL, mirror_dir))\n spack.llnl.util.tty.set_msg_enabled(False)\n stage = spack.stage.Stage(BINARY_TARBALL, path=tempfile.mkdtemp())\n stage.create()\n stage.fetch()\n stage.expand_archive()\n stage_dir = pathlib.Path(stage.source_path)\n for entry in stage_dir.iterdir():\n shutil.move(str(entry), mirror_dir)\n spack.llnl.util.tty.set_msg_enabled(True)\n\n def write_metadata(subdir, metadata):\n metadata_rel_dir = os.path.join(\"metadata\", subdir)\n metadata_yaml = os.path.join(args.root_dir, metadata_rel_dir, \"metadata.yaml\")\n spack.llnl.util.filesystem.mkdirp(os.path.dirname(metadata_yaml))\n with open(metadata_yaml, mode=\"w\", encoding=\"utf-8\") as f:\n spack.util.spack_yaml.dump(metadata, stream=f)\n return os.path.dirname(metadata_yaml), metadata_rel_dir\n\n instructions = (\n \"\\nTo register the mirror on the platform where it's supposed \"\n 'to be used, move \"{0}\" to its final location and run the '\n \"following command(s):\\n\\n\"\n ).format(args.root_dir)\n cmd = \" % spack bootstrap add --trust {0} /{1}\\n\"\n _, rel_directory = write_metadata(subdir=\"sources\", metadata=SOURCE_METADATA)\n instructions += cmd.format(\"local-sources\", rel_directory)\n if args.binary_packages:\n abs_directory, rel_directory = write_metadata(subdir=\"binaries\", metadata=BINARY_METADATA)\n shutil.copy(spack.util.path.canonicalize_path(CLINGO_JSON), abs_directory)\n shutil.copy(spack.util.path.canonicalize_path(GNUPG_JSON), abs_directory)\n shutil.copy(spack.util.path.canonicalize_path(PATCHELF_JSON), abs_directory)\n instructions += cmd.format(\"local-binaries\", rel_directory)\n print(instructions)\n\n\ndef _now(args):\n with spack.bootstrap.ensure_bootstrap_configuration():\n spack.bootstrap.ensure_core_dependencies()\n if args.dev:\n spack.bootstrap.ensure_environment_dependencies()\n\n\ndef bootstrap(parser, args):\n callbacks = {\n \"status\": _status,\n \"enable\": _enable_or_disable,\n \"disable\": _enable_or_disable,\n \"reset\": _reset,\n \"root\": _root,\n \"list\": _list,\n \"add\": _add,\n \"remove\": _remove,\n \"mirror\": _mirror,\n \"now\": _now,\n }\n callbacks[args.subcommand](args)\n" }, { "path": "lib/spack/spack/cmd/build_env.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport spack.cmd.common.env_utility as env_utility\nfrom spack.context import Context\n\ndescription = \"dump the install environment for a spec,\\nor run a command in that environment\"\nsection = \"build\"\nlevel = \"long\"\n\nsetup_parser = env_utility.setup_parser\n\n\ndef build_env(parser, args):\n env_utility.emulate_env_utility(\"build-env\", Context.BUILD, args)\n" }, { "path": "lib/spack/spack/cmd/buildcache.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport argparse\nimport enum\nimport glob\nimport json\nimport os\nimport sys\nimport tempfile\nfrom typing import List, Optional, Tuple\n\nimport spack.binary_distribution\nimport spack.cmd\nimport spack.concretize\nimport spack.config\nimport spack.deptypes as dt\nimport spack.environment as ev\nimport spack.error\nimport spack.llnl.util.tty as tty\nimport spack.mirrors.mirror\nimport spack.oci.image\nimport spack.oci.oci\nimport spack.spec\nimport spack.stage\nimport spack.store\nimport spack.util.parallel\nimport spack.util.timer as timer_mod\nimport spack.util.web as web_util\nfrom spack import traverse\nfrom spack.binary_distribution import BINARY_INDEX\nfrom spack.cmd import display_specs\nfrom spack.cmd.common import arguments\nfrom spack.llnl.string import plural\nfrom spack.llnl.util.lang import elide_list, stable_partition\nfrom spack.spec import Spec, save_dependency_specfiles\n\nfrom ..buildcache_migrate import migrate\nfrom ..buildcache_prune import prune_buildcache\nfrom ..enums import InstallRecordStatus\nfrom ..url_buildcache import (\n BuildcacheComponent,\n BuildcacheEntryError,\n URLBuildcacheEntry,\n check_mirror_for_layout,\n get_entries_from_cache,\n get_url_buildcache_class,\n)\n\ndescription = \"create, download and install binary packages\"\nsection = \"packaging\"\nlevel = \"long\"\n\n\nclass ViewUpdateMode(enum.Enum):\n CREATE = enum.auto()\n OVERWRITE = enum.auto()\n APPEND = enum.auto()\n\n\ndef setup_parser(subparser: argparse.ArgumentParser):\n setattr(setup_parser, \"parser\", subparser)\n subparsers = subparser.add_subparsers(help=\"buildcache sub-commands\")\n\n push = subparsers.add_parser(\"push\", aliases=[\"create\"], help=push_fn.__doc__)\n push.add_argument(\"-f\", \"--force\", action=\"store_true\", help=\"overwrite tarball if it exists\")\n push_sign = push.add_mutually_exclusive_group(required=False)\n push_sign.add_argument(\n \"--unsigned\",\n \"-u\",\n action=\"store_false\",\n dest=\"signed\",\n default=None,\n help=\"push unsigned buildcache tarballs\",\n )\n push_sign.add_argument(\n \"--signed\",\n action=\"store_true\",\n dest=\"signed\",\n default=None,\n help=\"push signed buildcache tarballs\",\n )\n push_sign.add_argument(\n \"--key\", \"-k\", metavar=\"key\", type=str, default=None, help=\"key for signing\"\n )\n push.add_argument(\n \"mirror\", type=arguments.mirror_name_or_url, help=\"mirror name, path, or URL\"\n )\n push.add_argument(\n \"--update-index\",\n \"--rebuild-index\",\n action=\"store_true\",\n default=False,\n help=\"regenerate buildcache index after building package(s)\",\n )\n push.add_argument(\n \"--only\",\n default=\"package,dependencies\",\n dest=\"things_to_install\",\n choices=[\"package\", \"dependencies\"],\n help=\"select the buildcache mode. \"\n \"The default is to build a cache for the package along with all its dependencies. \"\n \"Alternatively, one can decide to build a cache for only the package or only the \"\n \"dependencies\",\n )\n with_or_without_build_deps = push.add_mutually_exclusive_group()\n with_or_without_build_deps.add_argument(\n \"--with-build-dependencies\",\n action=\"store_true\",\n help=\"include build dependencies in the buildcache\",\n )\n with_or_without_build_deps.add_argument(\n \"--without-build-dependencies\",\n action=\"store_true\",\n help=\"exclude build dependencies from the buildcache\",\n )\n push.add_argument(\n \"--fail-fast\",\n action=\"store_true\",\n help=\"stop pushing on first failure (default is best effort)\",\n )\n push.add_argument(\n \"--base-image\", default=None, help=\"specify the base image for the buildcache\"\n )\n push.add_argument(\n \"--tag\",\n \"-t\",\n default=None,\n help=\"when pushing to an OCI registry, tag an image containing all root specs and their \"\n \"runtime dependencies\",\n )\n push.add_argument(\n \"--private\",\n action=\"store_true\",\n help=\"for a private mirror, include non-redistributable packages\",\n )\n push.add_argument(\n \"--group\",\n action=\"append\",\n default=None,\n dest=\"groups\",\n metavar=\"GROUP\",\n help=\"push only specs from the given environment group \"\n \"(can be specified multiple times, requires an active environment)\",\n )\n arguments.add_common_arguments(push, [\"specs\", \"jobs\"])\n push.set_defaults(func=push_fn)\n\n install = subparsers.add_parser(\"install\", help=install_fn.__doc__)\n install.add_argument(\n \"-f\", \"--force\", action=\"store_true\", help=\"overwrite install directory if it exists\"\n )\n install.add_argument(\n \"-m\", \"--multiple\", action=\"store_true\", help=\"allow all matching packages\"\n )\n install.add_argument(\n \"-u\",\n \"--unsigned\",\n action=\"store_true\",\n help=\"install unsigned buildcache tarballs for testing\",\n )\n install.add_argument(\n \"-o\",\n \"--otherarch\",\n action=\"store_true\",\n help=\"install specs from other architectures instead of default platform and OS\",\n )\n\n arguments.add_common_arguments(install, [\"specs\"])\n install.set_defaults(func=install_fn)\n\n listcache = subparsers.add_parser(\"list\", help=list_fn.__doc__)\n arguments.add_common_arguments(listcache, [\"long\", \"very_long\", \"namespaces\"])\n listcache.add_argument(\n \"-v\",\n \"--variants\",\n action=\"store_true\",\n dest=\"variants\",\n help=\"show variants in output (can be long)\",\n )\n listcache.add_argument(\n \"-a\",\n \"--allarch\",\n action=\"store_true\",\n help=\"list specs for all available architectures instead of default platform and OS\",\n )\n arguments.add_common_arguments(listcache, [\"specs\"])\n listcache.set_defaults(func=list_fn)\n\n keys = subparsers.add_parser(\"keys\", help=keys_fn.__doc__)\n keys.add_argument(\n \"-i\", \"--install\", action=\"store_true\", help=\"install Keys pulled from mirror\"\n )\n keys.add_argument(\"-t\", \"--trust\", action=\"store_true\", help=\"trust all downloaded keys\")\n keys.add_argument(\"-f\", \"--force\", action=\"store_true\", help=\"force new download of keys\")\n keys.set_defaults(func=keys_fn)\n\n # Check if binaries need to be rebuilt on remote mirror\n check = subparsers.add_parser(\"check\", help=check_fn.__doc__)\n check.add_argument(\n \"-m\",\n \"--mirror-url\",\n default=None,\n help=\"override any configured mirrors with this mirror URL\",\n )\n\n check.add_argument(\n \"-o\", \"--output-file\", default=None, help=\"file where rebuild info should be written\"\n )\n\n # used to construct scope arguments below\n check.add_argument(\n \"--scope\",\n action=arguments.ConfigScope,\n type=arguments.config_scope_readable_validator,\n default=lambda: spack.config.default_modify_scope(),\n help=\"configuration scope containing mirrors to check\",\n )\n\n arguments.add_common_arguments(check, [\"specs\"])\n\n check.set_defaults(func=check_fn)\n\n # Download tarball and specfile\n download = subparsers.add_parser(\"download\", help=download_fn.__doc__)\n download.add_argument(\"-s\", \"--spec\", help=\"download built tarball for spec from mirror\")\n download.add_argument(\n \"-p\",\n \"--path\",\n required=True,\n default=None,\n help=\"path to directory where tarball should be downloaded\",\n )\n download.set_defaults(func=download_fn)\n\n prune = subparsers.add_parser(\"prune\", help=prune_fn.__doc__)\n prune.add_argument(\n \"mirror\", type=arguments.mirror_name_or_url, help=\"mirror name, path, or URL\"\n )\n prune.add_argument(\n \"-k\",\n \"--keeplist\",\n default=None,\n help=\"file containing newline-delimited list of package hashes to keep (optional)\",\n )\n prune.add_argument(\n \"--dry-run\",\n action=\"store_true\",\n help=\"do not actually delete anything from the buildcache, but log what would be deleted\",\n )\n prune.set_defaults(func=prune_fn)\n\n # Given the root spec, save the yaml of the dependent spec to a file\n savespecfile = subparsers.add_parser(\"save-specfile\", help=save_specfile_fn.__doc__)\n savespecfile_spec_or_specfile = savespecfile.add_mutually_exclusive_group(required=True)\n savespecfile_spec_or_specfile.add_argument(\"--root-spec\", help=\"root spec of dependent spec\")\n savespecfile.add_argument(\n \"-s\",\n \"--specs\",\n required=True,\n help=\"list of dependent specs for which saved yaml is desired\",\n )\n savespecfile.add_argument(\n \"--specfile-dir\", required=True, help=\"path to directory where spec yamls should be saved\"\n )\n savespecfile.set_defaults(func=save_specfile_fn)\n\n # Sync buildcache entries from one mirror to another\n sync = subparsers.add_parser(\"sync\", help=sync_fn.__doc__)\n\n sync_manifest_source = sync.add_argument_group(\n \"Manifest Source\",\n \"Specify a list of build cache objects to sync using manifest file(s).\"\n 'This option takes the place of the \"source mirror\" for synchronization'\n 'and optionally takes a \"destination mirror\" ',\n )\n sync_manifest_source.add_argument(\n \"--manifest-glob\", help=\"a quoted glob pattern identifying CI rebuild manifest files\"\n )\n sync_source_mirror = sync.add_argument_group(\n \"Named Source\",\n \"Specify a single registered source mirror to synchronize from. This option requires\"\n \"the specification of a destination mirror.\",\n )\n sync_source_mirror.add_argument(\n \"src_mirror\",\n metavar=\"source mirror\",\n nargs=\"?\",\n type=arguments.mirror_name_or_url,\n help=\"source mirror name, path, or URL\",\n )\n\n sync.add_argument(\n \"dest_mirror\",\n metavar=\"destination mirror\",\n nargs=\"?\",\n type=arguments.mirror_name_or_url,\n help=\"destination mirror name, path, or URL\",\n )\n\n sync.set_defaults(func=sync_fn)\n\n # Check the validity of a buildcache\n check_index = subparsers.add_parser(\"check-index\", help=check_index_fn.__doc__)\n check_index.add_argument(\n \"--verify\",\n nargs=\"+\",\n choices=[\"exists\", \"manifests\", \"blobs\", \"all\"],\n default=[\"exists\"],\n help=\"List of items to verify along along with the index.\",\n )\n check_index.add_argument(\n \"--name\", \"-n\", action=\"store\", help=\"Name of the view index to check\"\n )\n check_index.add_argument(\n \"--output\", \"-o\", action=\"store\", help=\"File to write check details to\"\n )\n check_index.add_argument(\n \"mirror\", type=arguments.mirror_name_or_url, help=\"mirror name, path, or URL\"\n )\n check_index.set_defaults(func=check_index_fn)\n\n # Update buildcache index without copying any additional packages\n update_index = subparsers.add_parser(\n \"update-index\", aliases=[\"rebuild-index\"], help=update_index_fn.__doc__\n )\n update_index.add_argument(\n \"mirror\", type=arguments.mirror_name_or_url, help=\"destination mirror name, path, or URL\"\n )\n update_index_view_args = update_index.add_argument_group(\"view arguments\")\n update_index_view_args.add_argument(\n \"sources\", nargs=\"*\", help=\"List of environments names or paths\"\n )\n update_index_view_args.add_argument(\n \"--name\", \"-n\", action=\"store\", help=\"Name of the view index to update\"\n )\n update_index_view_mode_args = update_index_view_args.add_mutually_exclusive_group(\n required=False\n )\n update_index_view_mode_args.add_argument(\n \"--append\",\n \"-a\",\n action=\"store_true\",\n help=\"Append the listed specs to the current view index if it already exists. \"\n \"This operation does not guarantee atomic write and should be run with care.\",\n )\n update_index_view_mode_args.add_argument(\n \"--force\",\n \"-f\",\n action=\"store_true\",\n help=\"If a view index already exists, overwrite it and \"\n \"suppress warnings (this is the default for non-view indices)\",\n )\n update_index.add_argument(\n \"-k\",\n \"--keys\",\n default=False,\n action=\"store_true\",\n help=\"if provided, key index will be updated as well as package index\",\n )\n arguments.add_common_arguments(update_index, [\"yes_to_all\"])\n update_index.set_defaults(func=update_index_fn)\n\n # Migrate a buildcache from layout_version 2 to version 3\n migrate = subparsers.add_parser(\"migrate\", help=migrate_fn.__doc__)\n migrate.add_argument(\"mirror\", type=arguments.mirror_name, help=\"name of a configured mirror\")\n migrate.add_argument(\n \"-u\",\n \"--unsigned\",\n default=False,\n action=\"store_true\",\n help=\"Ignore signatures and do not resign, default is False\",\n )\n migrate.add_argument(\n \"-d\",\n \"--delete-existing\",\n default=False,\n action=\"store_true\",\n help=\"Delete the previous layout, the default is to keep it.\",\n )\n arguments.add_common_arguments(migrate, [\"yes_to_all\"])\n # TODO: add -y argument to prompt if user really means to delete existing\n migrate.set_defaults(func=migrate_fn)\n\n\ndef _matching_specs(specs: List[Spec]) -> List[Spec]:\n \"\"\"Disambiguate specs and return a list of matching specs\"\"\"\n return [\n spack.cmd.disambiguate_spec(s, ev.active_environment(), installed=InstallRecordStatus.ANY)\n for s in specs\n ]\n\n\ndef _format_spec(spec: Spec) -> str:\n return spec.cformat(\"{name}{@version}{/hash:7}\")\n\n\ndef _skip_no_redistribute_for_public(specs):\n remaining_specs = list()\n removed_specs = list()\n for spec in specs:\n if spec.package.redistribute_binary:\n remaining_specs.append(spec)\n else:\n removed_specs.append(spec)\n if removed_specs:\n colified_output = tty.colify.colified(list(s.name for s in removed_specs), indent=4)\n tty.debug(\n \"The following specs will not be added to the binary cache\"\n \" because they cannot be redistributed:\\n\"\n f\"{colified_output}\\n\"\n \"You can use `--private` to include them.\"\n )\n return remaining_specs\n\n\nclass PackagesAreNotInstalledError(spack.error.SpackError):\n \"\"\"Raised when a list of specs is not installed but picked to be packaged.\"\"\"\n\n def __init__(self, specs: List[Spec]):\n super().__init__(\n \"Cannot push non-installed packages\",\n \", \".join(elide_list([_format_spec(s) for s in specs], 5)),\n )\n\n\nclass PackageNotInstalledError(spack.error.SpackError):\n \"\"\"Raised when a spec is not installed but picked to be packaged.\"\"\"\n\n\ndef _specs_to_be_packaged(\n requested: List[Spec], things_to_install: str, build_deps: bool\n) -> List[Spec]:\n \"\"\"Collect all non-external with or without roots and dependencies\"\"\"\n if \"dependencies\" not in things_to_install:\n deptype = dt.NONE\n elif build_deps:\n deptype = dt.ALL\n else:\n deptype = dt.RUN | dt.LINK | dt.TEST\n specs = [\n s\n for s in traverse.traverse_nodes(\n requested,\n root=\"package\" in things_to_install,\n deptype=deptype,\n order=\"breadth\",\n key=traverse.by_dag_hash,\n )\n if not s.external\n ]\n specs.reverse()\n return specs\n\n\ndef push_fn(args):\n \"\"\"create a binary package and push it to a mirror\"\"\"\n if args.specs and args.groups:\n tty.die(\"--group and explicit specs are mutually exclusive\")\n\n if args.groups:\n env = spack.cmd.require_active_env(cmd_name=\"buildcache push\")\n available_groups = env.manifest.groups()\n if any(g not in available_groups for g in args.groups):\n tty.die(\n f\"Some of the groups do not exist in the environment. \"\n f\"Available groups are: {', '.join(sorted(available_groups))}\"\n )\n\n roots = [c for g in args.groups for _, c in env.concretized_specs_by(group=g)]\n elif args.specs:\n roots = _matching_specs(spack.cmd.parse_specs(args.specs))\n else:\n roots = spack.cmd.require_active_env(cmd_name=\"buildcache push\").concrete_roots()\n\n mirror = args.mirror\n assert isinstance(mirror, spack.mirrors.mirror.Mirror)\n\n push_url = mirror.push_url\n\n # When neither --signed, --unsigned nor --key are specified, use the mirror's default.\n if args.signed is None and not args.key:\n unsigned = not mirror.signed\n else:\n unsigned = not (args.key or args.signed)\n\n # For OCI images, we require dependencies to be pushed for now.\n if spack.oci.image.is_oci_url(mirror.push_url) and not unsigned:\n tty.warn(\n \"Code signing is currently not supported for OCI images. \"\n \"Use --unsigned to silence this warning.\"\n )\n unsigned = True\n\n # Select a signing key, or None if unsigned.\n signing_key = (\n None if unsigned else (args.key or spack.binary_distribution.select_signing_key())\n )\n\n specs = _specs_to_be_packaged(\n roots,\n things_to_install=args.things_to_install,\n build_deps=args.with_build_dependencies or not args.without_build_dependencies,\n )\n\n if not args.private:\n specs = _skip_no_redistribute_for_public(specs)\n\n if len(specs) > 1:\n tty.info(f\"Selected {len(specs)} specs to push to {push_url}\")\n\n # Pushing not installed specs is an error. Either fail fast or populate the error list and\n # push installed package in best effort mode.\n failed: List[Tuple[Spec, BaseException]] = []\n with spack.store.STORE.db.read_transaction():\n if any(not s.installed for s in specs):\n specs, not_installed = stable_partition(specs, lambda s: s.installed)\n if args.fail_fast:\n raise PackagesAreNotInstalledError(not_installed)\n else:\n failed.extend(\n (s, PackageNotInstalledError(\"package not installed\")) for s in not_installed\n )\n\n # Warn about possible old binary mirror layout\n if not spack.oci.image.is_oci_url(mirror.push_url):\n check_mirror_for_layout(mirror)\n\n with spack.binary_distribution.make_uploader(\n mirror=mirror,\n force=args.force,\n update_index=args.update_index,\n signing_key=signing_key,\n base_image=args.base_image,\n ) as uploader:\n skipped, upload_errors = uploader.push(specs=specs)\n failed.extend(upload_errors)\n\n if skipped:\n if len(specs) == 1:\n tty.info(\"The spec is already in the buildcache. Use --force to overwrite it.\")\n elif len(skipped) == len(specs):\n tty.info(\"All specs are already in the buildcache. Use --force to overwrite them.\")\n else:\n tty.info(\n \"The following {} specs were skipped as they already exist in the \"\n \"buildcache:\\n\"\n \" {}\\n\"\n \" Use --force to overwrite them.\".format(\n len(skipped), \", \".join(elide_list([_format_spec(s) for s in skipped], 5))\n )\n )\n\n if failed:\n if len(failed) == 1:\n raise failed[0][1]\n\n raise spack.error.SpackError(\n f\"The following {len(failed)} errors occurred while pushing specs to the \"\n \"buildcache\",\n \"\\n\".join(\n elide_list(\n [\n f\" {_format_spec(spec)}: {e.__class__.__name__}: {e}\"\n for spec, e in failed\n ],\n 5,\n )\n ),\n )\n\n # Finally tag all roots as a single image if requested.\n if args.tag:\n uploader.tag(args.tag, roots)\n\n\ndef install_fn(args):\n \"\"\"install from a binary package\"\"\"\n if not args.specs:\n tty.die(\"a spec argument is required to install from a buildcache\")\n\n query = spack.binary_distribution.BinaryCacheQuery(all_architectures=args.otherarch)\n matches = spack.store.find(args.specs, multiple=args.multiple, query_fn=query)\n for match in matches:\n spack.binary_distribution.install_single_spec(\n match, unsigned=args.unsigned, force=args.force\n )\n\n\ndef list_fn(args):\n \"\"\"list binary packages available from mirrors\"\"\"\n try:\n specs = spack.binary_distribution.update_cache_and_get_specs()\n except spack.binary_distribution.FetchCacheError as e:\n tty.die(e)\n\n if not args.allarch:\n arch = spack.spec.Spec.default_arch()\n specs = [s for s in specs if s.intersects(arch)]\n\n if args.specs:\n constraints = set(args.specs)\n specs = [s for s in specs if any(s.intersects(c) for c in constraints)]\n if sys.stdout.isatty():\n builds = len(specs)\n tty.msg(\"%s.\" % plural(builds, \"cached build\"))\n if not builds and not args.allarch:\n tty.msg(\n \"You can query all available architectures with:\",\n \"spack buildcache list --allarch\",\n )\n display_specs(specs, args, all_headers=True)\n\n\ndef keys_fn(args):\n \"\"\"get public keys available on mirrors\"\"\"\n spack.binary_distribution.get_keys(args.install, args.trust, args.force)\n\n\ndef check_fn(args: argparse.Namespace):\n \"\"\"check specs against remote binary mirror(s) to see if any need to be rebuilt\n\n this command uses the process exit code to indicate its result, specifically, if the\n exit code is non-zero, then at least one of the indicated specs needs to be rebuilt\n \"\"\"\n specs_arg = args.specs\n\n if specs_arg:\n specs = _matching_specs(spack.cmd.parse_specs(specs_arg))\n else:\n specs = spack.cmd.require_active_env(\"buildcache check\").all_specs()\n\n if not specs:\n tty.msg(\"No specs provided, exiting.\")\n return\n\n specs = [spack.concretize.concretize_one(s) for s in specs]\n\n # Next see if there are any configured binary mirrors\n configured_mirrors = spack.config.get(\"mirrors\", scope=args.scope)\n\n if args.mirror_url:\n configured_mirrors = {\"additionalMirrorUrl\": args.mirror_url}\n\n if not configured_mirrors:\n tty.msg(\"No mirrors provided, exiting.\")\n return\n\n if (\n spack.binary_distribution.check_specs_against_mirrors(\n configured_mirrors, specs, args.output_file\n )\n == 1\n ):\n sys.exit(1)\n\n\ndef download_fn(args):\n \"\"\"download buildcache entry from a remote mirror to local folder\n\n this command uses the process exit code to indicate its result, specifically, a non-zero exit\n code indicates that the command failed to download at least one of the required buildcache\n components\n \"\"\"\n specs = _matching_specs(spack.cmd.parse_specs(args.spec))\n\n if len(specs) != 1:\n tty.die(\"a single spec argument is required to download from a buildcache\")\n\n spack.binary_distribution.download_single_spec(specs[0], args.path)\n\n\ndef save_specfile_fn(args):\n \"\"\"get full spec for dependencies and write them to files in the specified output directory\n\n uses exit code to signal success or failure. an exit code of zero means the command was likely\n successful. if any errors or exceptions are encountered, or if expected command-line arguments\n are not provided, then the exit code will be non-zero\n \"\"\"\n specs = spack.cmd.parse_specs(args.root_spec)\n\n if len(specs) != 1:\n tty.die(\"a single spec argument is required to save specfile\")\n\n root = specs[0]\n\n if not root.concrete:\n root = spack.concretize.concretize_one(root)\n\n save_dependency_specfiles(\n root, args.specfile_dir, dependencies=spack.cmd.parse_specs(args.specs)\n )\n\n\ndef copy_buildcache_entry(cache_entry: URLBuildcacheEntry, destination_url: str):\n \"\"\"Download buildcache entry and copy it to the destination_url\"\"\"\n try:\n spec_dict = cache_entry.fetch_metadata()\n cache_entry.fetch_archive()\n except spack.binary_distribution.BuildcacheEntryError as e:\n tty.warn(f\"Failed to retrieve buildcache for copying due to {e}\")\n cache_entry.destroy()\n return\n\n spec_blob_record = cache_entry.get_blob_record(BuildcacheComponent.SPEC)\n local_spec_path = cache_entry.get_local_spec_path()\n tarball_blob_record = cache_entry.get_blob_record(BuildcacheComponent.TARBALL)\n local_tarball_path = cache_entry.get_local_archive_path()\n\n target_spec = spack.spec.Spec.from_dict(spec_dict)\n spec_label = f\"{target_spec.name}/{target_spec.dag_hash()[:7]}\"\n\n if not tarball_blob_record:\n cache_entry.destroy()\n raise BuildcacheEntryError(f\"No source tarball blob record, failed to sync {spec_label}\")\n\n # Try to push the tarball\n tarball_dest_url = cache_entry.get_blob_url(destination_url, tarball_blob_record)\n\n try:\n web_util.push_to_url(local_tarball_path, tarball_dest_url, keep_original=True)\n except Exception as e:\n tty.warn(f\"Failed to push {local_tarball_path} to {tarball_dest_url} due to {e}\")\n cache_entry.destroy()\n return\n\n if not spec_blob_record:\n cache_entry.destroy()\n raise BuildcacheEntryError(f\"No source spec blob record, failed to sync {spec_label}\")\n\n # Try to push the spec file\n spec_dest_url = cache_entry.get_blob_url(destination_url, spec_blob_record)\n\n try:\n web_util.push_to_url(local_spec_path, spec_dest_url, keep_original=True)\n except Exception as e:\n tty.warn(f\"Failed to push {local_spec_path} to {spec_dest_url} due to {e}\")\n cache_entry.destroy()\n return\n\n # Stage the manifest locally, since if it's signed, we don't want to try to\n # to reproduce that here. Instead just push the locally staged manifest to\n # the expected path at the destination url.\n manifest_src_url = cache_entry.remote_manifest_url\n manifest_dest_url = cache_entry.get_manifest_url(target_spec, destination_url)\n\n manifest_stage = spack.stage.Stage(manifest_src_url)\n\n try:\n manifest_stage.create()\n manifest_stage.fetch()\n except Exception as e:\n tty.warn(f\"Failed to fetch manifest from {manifest_src_url} due to {e}\")\n manifest_stage.destroy()\n cache_entry.destroy()\n return\n\n local_manifest_path = manifest_stage.save_filename\n\n try:\n web_util.push_to_url(local_manifest_path, manifest_dest_url, keep_original=True)\n except Exception as e:\n tty.warn(f\"Failed to push manifest to {manifest_dest_url} due to {e}\")\n\n manifest_stage.destroy()\n cache_entry.destroy()\n\n\ndef sync_fn(args):\n \"\"\"sync binaries (and associated metadata) from one mirror to another\n\n requires an active environment in order to know which specs to sync\n \"\"\"\n if args.manifest_glob:\n # Passing the args.src_mirror here because it is not possible to\n # have the destination be required when specifying a named source\n # mirror and optional for the --manifest-glob argument. In the case\n # of manifest glob sync, the source mirror positional argument is the\n # destination mirror if it is specified. If there are two mirrors\n # specified, the second is ignored and the first is the override\n # destination.\n if args.dest_mirror:\n tty.warn(f\"Ignoring unused argument: {args.dest_mirror.name}\")\n\n manifest_copy(glob.glob(args.manifest_glob), args.src_mirror)\n return 0\n\n if args.src_mirror is None or args.dest_mirror is None:\n tty.die(\"Provide mirrors to sync from and to.\")\n\n src_mirror = args.src_mirror\n dest_mirror = args.dest_mirror\n\n src_mirror_url = src_mirror.fetch_url\n dest_mirror_url = dest_mirror.push_url\n\n # Get the active environment\n env = spack.cmd.require_active_env(cmd_name=\"buildcache sync\")\n\n tty.msg(\n \"Syncing environment buildcache files from {0} to {1}\".format(\n src_mirror_url, dest_mirror_url\n )\n )\n\n tty.debug(\"Syncing the following specs:\")\n specs_to_sync = [s for s in env.all_specs() if not s.external]\n for s in specs_to_sync:\n tty.debug(\" {0}{1}: {2}\".format(\"* \" if s in env.roots() else \" \", s.name, s.dag_hash()))\n cache_class = get_url_buildcache_class(\n layout_version=spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION\n )\n src_cache_entry = cache_class(src_mirror_url, s, allow_unsigned=True)\n src_cache_entry.read_manifest()\n copy_buildcache_entry(src_cache_entry, dest_mirror_url)\n\n\ndef manifest_copy(\n manifest_file_list: List[str], dest_mirror: Optional[spack.mirrors.mirror.Mirror] = None\n):\n \"\"\"Read manifest files containing information about specific specs to copy\n from source to destination, remove duplicates since any binary package for\n a given hash should be the same as any other, and copy all files specified\n in the manifest files.\"\"\"\n deduped_manifest = {}\n\n for manifest_path in manifest_file_list:\n with open(manifest_path, encoding=\"utf-8\") as fd:\n manifest = json.loads(fd.read())\n for spec_hash, copy_obj in manifest.items():\n # Last duplicate hash wins\n deduped_manifest[spec_hash] = copy_obj\n\n for spec_hash, copy_obj in deduped_manifest.items():\n cache_class = get_url_buildcache_class(\n layout_version=spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION\n )\n src_cache_entry = cache_class(\n cache_class.get_base_url(copy_obj[\"src\"]), allow_unsigned=True\n )\n src_cache_entry.read_manifest(manifest_url=copy_obj[\"src\"])\n if dest_mirror:\n destination_url = dest_mirror.push_url\n else:\n destination_url = cache_class.get_base_url(copy_obj[\"dest\"])\n tty.debug(\"copying {0} to {1}\".format(copy_obj[\"src\"], destination_url))\n copy_buildcache_entry(src_cache_entry, destination_url)\n\n\ndef update_index(\n mirror: spack.mirrors.mirror.Mirror, update_keys=False, timer=timer_mod.NULL_TIMER\n):\n timer.start()\n # Special case OCI images for now.\n try:\n image_ref = spack.oci.oci.image_from_mirror(mirror)\n except ValueError:\n image_ref = None\n\n if image_ref:\n with tempfile.TemporaryDirectory(\n dir=spack.stage.get_stage_root()\n ) as tmpdir, spack.util.parallel.make_concurrent_executor() as executor:\n spack.binary_distribution._oci_update_index(image_ref, tmpdir, executor)\n return\n\n # Otherwise, assume a normal mirror.\n url = mirror.push_url\n\n with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir:\n spack.binary_distribution._url_generate_package_index(url, tmpdir, timer=timer)\n\n if update_keys:\n mirror_update_keys(mirror)\n\n\ndef mirror_update_keys(mirror: spack.mirrors.mirror.Mirror):\n url = mirror.push_url\n try:\n with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir:\n spack.binary_distribution.generate_key_index(url, tmpdir)\n except spack.binary_distribution.CannotListKeys as e:\n # Do not error out if listing keys went wrong. This usually means that the _gpg path\n # does not exist. TODO: distinguish between this and other errors.\n tty.warn(f\"did not update the key index: {e}\")\n\n\ndef update_view(\n mirror: spack.mirrors.mirror.Mirror,\n update_mode: ViewUpdateMode,\n *sources: str,\n name: Optional[str] = None,\n update_keys: bool = False,\n yes_to_all: bool = False,\n):\n \"\"\"update a buildcache view index\"\"\"\n # OCI images do not support views.\n try:\n spack.oci.oci.image_from_mirror(mirror)\n raise spack.error.SpackError(\"OCI build caches do not support index views\")\n except ValueError:\n pass\n\n if update_mode == ViewUpdateMode.APPEND and not yes_to_all:\n tty.warn(\n \"Appending to a view index does not guarantee idempotent write when contending \"\n \"with multiple writers. This feature is meant to be used by a single process.\"\n )\n tty.get_yes_or_no(\"Do you want to proceed?\", default=False)\n\n # Otherwise, assume a normal mirror.\n url = mirror.push_url\n\n if (name and mirror.push_view) and not name == mirror.push_view:\n tty.warn(\n (\n f\"Updating index view with name ({name}), which is different than \"\n f\"the configured name ({mirror.push_view}) for the mirror {mirror.name}\"\n )\n )\n\n name = name or mirror.push_view\n if not name:\n tty.die(\n \"Attempting to update a view but could not determine the view name.\\n\"\n \" Either pass --name or configure the view name in mirrors.yaml\"\n )\n\n mirror_metadata = spack.binary_distribution.MirrorMetadata(\n url, spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION, name\n )\n\n # Check if the index already exists, if it does make sure there is a copy in the\n # local cache.\n index_exists = True\n try:\n BINARY_INDEX._fetch_and_cache_index(mirror_metadata)\n except spack.binary_distribution.BuildcacheIndexNotExists:\n index_exists = False\n\n if index_exists and update_mode == ViewUpdateMode.CREATE:\n raise spack.error.SpackError(\n \"Index already exists. To overwrite or update pass --force or --append respectively\"\n )\n\n hashes = []\n if sources:\n for source in sources:\n tty.debug(f\"reading specs from source: {source}\")\n env = ev.environment_from_name_or_dir(source)\n hashes.extend(env.all_hashes())\n else:\n # Get hashes in the current active environment\n hashes = spack.cmd.require_active_env(cmd_name=\"buildcache update-view\").all_hashes()\n\n if not hashes:\n tty.warn(\"No specs found for view, creating an empty index\")\n\n filter_fn = lambda x: x in hashes\n\n with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir:\n # Initialize a database\n db = spack.binary_distribution.BuildCacheDatabase(tmpdir)\n db._write()\n\n if update_mode == ViewUpdateMode.APPEND:\n # Load the current state of the view index from the cache into the database\n cache_index = BINARY_INDEX._local_index_cache.get(str(mirror_metadata))\n if cache_index:\n cache_key = cache_index[\"index_path\"]\n with BINARY_INDEX._index_file_cache.read_transaction(cache_key) as f:\n if f is not None:\n db._read_from_stream(f)\n\n spack.binary_distribution._url_generate_package_index(url, tmpdir, db, name, filter_fn)\n\n if update_keys:\n mirror_update_keys(mirror)\n\n\ndef check_index_fn(args):\n \"\"\"Check if a build cache index, manifests, and blobs are consistent\"\"\"\n mirror = args.mirror\n verify = set(args.verify)\n\n checking_view_index = (args.name or mirror.fetch_view) is not None\n\n if \"all\" in verify:\n verify.update([\"exists\", \"manifests\", \"blobs\"])\n\n try:\n spack.oci.oci.image_from_mirror(mirror)\n raise spack.error.SpackError(\"OCI build caches do not support index views\")\n except ValueError:\n pass\n\n # Check if the index exists, and cache it locally for next operations\n mirror_metadata = spack.binary_distribution.MirrorMetadata(\n mirror.fetch_url,\n spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION,\n args.name or mirror.fetch_view,\n )\n index_exists = True\n missing_index_blob = False\n try:\n BINARY_INDEX._fetch_and_cache_index(mirror_metadata)\n except spack.binary_distribution.BuildcacheIndexNotExists:\n index_exists = False\n except spack.binary_distribution.FetchIndexError:\n # Here the index manifest exists, but the index blob did not\n # We can still run some of the other validations here, so let's try\n index_exists = False\n missing_index_blob = True\n\n missing_specs = []\n unindexed_specs = []\n missing_blobs = {}\n cache_hash_list = []\n index_hash_list = []\n # List the manifests and verify\n with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir:\n # Get listing of spec manifests in mirror\n manifest_files = []\n if \"manifests\" in verify or \"blobs\" in verify:\n manifest_files, read_fn = get_entries_from_cache(\n mirror.fetch_url, tmpdir, BuildcacheComponent.SPEC\n )\n if \"manifests\" in verify and index_exists:\n # Read the index file\n db = spack.binary_distribution.BuildCacheDatabase(tmpdir)\n cache_entry = BINARY_INDEX._local_index_cache[str(mirror_metadata)]\n cache_key = cache_entry[\"index_path\"]\n with BINARY_INDEX._index_file_cache.read_transaction(cache_key) as f:\n if f is not None:\n db._read_from_stream(f)\n\n index_hash_list = set(\n [\n s.dag_hash()\n for s in db.query_local(installed=InstallRecordStatus.ANY)\n if db._data[s.dag_hash()].in_buildcache\n ]\n )\n\n for spec_manifest in manifest_files:\n # Spec manifests have a naming format\n # --.spec.manifest.json\n spec_hash = spec_manifest.rsplit(\"-\", 1)[1].split(\".\", 1)[0]\n if checking_view_index and spec_hash not in index_hash_list:\n continue\n\n cache_hash_list.append(spec_hash)\n if spec_hash not in index_hash_list:\n unindexed_specs.append(spec_hash)\n\n if \"blobs\" in verify:\n entry = read_fn(spec_manifest)\n entry.read_manifest()\n for record in entry.manifest.data:\n if not entry.check_blob_exists(record):\n blobs = missing_blobs.get(spec_hash, [])\n blobs.append(record)\n missing_blobs[spec_hash] = blobs\n\n for h in index_hash_list:\n if h not in cache_hash_list:\n missing_specs.append(h)\n\n # Print summary\n summary_msg = \"Build cache check:\\n\\t\"\n if \"exists\" in verify:\n if index_exists:\n summary_msg = f\"Index exists in mirror: {mirror.name}\"\n else:\n summary_msg = f\"Index does not exist in mirror: {mirror.name}\"\n if mirror.fetch_view:\n summary_msg += f\"@{mirror.fetch_view}\"\n summary_msg += \"\\n\"\n if missing_index_blob:\n tty.warn(\"The index blob is missing\")\n\n if \"manifests\" in verify:\n if checking_view_index:\n count = \"n/a\"\n else:\n count = len(unindexed_specs)\n summary_msg += f\"\\tUnindexed specs: {count}\\n\"\n\n if \"manifests\" in verify:\n summary_msg += f\"\\tMissing specs: {len(missing_specs)}\\n\"\n\n if \"blobs\" in verify:\n summary_msg += f\"\\tMissing blobs: {len(missing_blobs)}\\n\"\n\n if args.output:\n os.makedirs(os.path.dirname(args.output), exist_ok=True)\n with open(args.output, \"w\", encoding=\"utf-8\") as fd:\n json.dump(\n {\n \"exists\": index_exists,\n \"manifests\": {\"missing\": missing_specs, \"unindexed\": unindexed_specs},\n \"blobs\": {\"missing\": missing_blobs},\n },\n fd,\n )\n\n tty.info(summary_msg)\n\n\ndef update_index_fn(args):\n \"\"\"update a buildcache index or index view if extra arguments are provided.\"\"\"\n\n t = timer_mod.Timer() if tty.is_verbose() else timer_mod.NullTimer()\n\n update_view_index = (\n args.append or args.force or args.name or args.sources or args.mirror.push_view\n )\n\n if update_view_index:\n update_mode = ViewUpdateMode.CREATE\n if args.force:\n update_mode = ViewUpdateMode.OVERWRITE\n elif args.append:\n update_mode = ViewUpdateMode.APPEND\n\n return update_view(\n args.mirror,\n update_mode,\n *args.sources,\n name=args.name,\n update_keys=args.keys,\n yes_to_all=args.yes_to_all,\n )\n else:\n update_index(args.mirror, update_keys=args.keys, timer=t)\n\n if tty.is_verbose():\n tty.msg(\"Timing summary:\")\n t.stop()\n t.write_tty()\n\n\ndef migrate_fn(args):\n \"\"\"perform in-place binary mirror migration (2 to 3)\n\n A mirror can contain both layout version 2 and version 3 simultaneously without\n interference. This command performs in-place migration of a binary mirror laid\n out according to version 2, to a binary mirror laid out according to layout\n version 3. Only indexed specs will be migrated, so consider updating the mirror\n index before running this command. Re-run the command to migrate any missing\n items.\n\n The default mode of operation is to perform a signed migration, that is, spack\n will attempt to verify the signatures on specs, and then re-sign them before\n migration, using whatever keys are already installed in your key ring. You can\n migrate a mirror of unsigned binaries (or convert a mirror of signed binaries\n to unsigned) by providing the ``--unsigned`` argument.\n\n By default spack will leave the original mirror contents (in the old layout) in\n place after migration. You can have spack remove the old contents by providing\n the ``--delete-existing`` argument. Because migrating a mostly-already-migrated\n mirror should be fast, consider a workflow where you perform a default migration,\n (i.e. preserve the existing layout rather than deleting it) then evaluate the\n state of the migrated mirror by attempting to install from it, and finally\n running the migration again with ``--delete-existing``.\"\"\"\n target_mirror = args.mirror\n unsigned = args.unsigned\n assert isinstance(target_mirror, spack.mirrors.mirror.Mirror)\n delete_existing = args.delete_existing\n\n proceed = True\n if delete_existing and not args.yes_to_all:\n msg = (\n \"Using --delete-existing will delete the entire contents \\n\"\n \" of the old layout within the mirror. Because migrating a mirror \\n\"\n \" that has already been migrated should be fast, consider a workflow \\n\"\n \" where you perform a default migration (i.e. preserve the existing \\n\"\n \" layout rather than deleting it), then evaluate the state of the \\n\"\n \" migrated mirror by attempting to install from it, and finally, \\n\"\n \" run the migration again with --delete-existing.\"\n )\n tty.warn(msg)\n proceed = tty.get_yes_or_no(\"Do you want to proceed?\", default=False)\n\n if not proceed:\n tty.die(\"Migration aborted.\")\n\n migrate(target_mirror, unsigned=unsigned, delete_existing=delete_existing)\n\n\ndef prune_fn(args):\n \"\"\"prune buildcache entries from the mirror\n\n If a keeplist file is provided, performs direct pruning (deletes packages not in keeplist)\n followed by orphan pruning. If no keeplist is provided, only performs orphan pruning.\n \"\"\"\n mirror: spack.mirrors.mirror.Mirror = args.mirror\n keeplist: Optional[str] = args.keeplist\n dry_run: bool = args.dry_run\n assert isinstance(mirror, spack.mirrors.mirror.Mirror)\n\n prune_buildcache(mirror=mirror, keeplist=keeplist, dry_run=dry_run)\n\n\ndef buildcache(parser, args):\n return args.func(args)\n" }, { "path": "lib/spack/spack/cmd/cd.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\n\nimport spack.cmd.common\nimport spack.cmd.location\n\ndescription = \"cd to spack directories in the shell\"\nsection = \"user environment\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n \"\"\"This is for decoration -- spack cd is used through spack's\n shell support. This allows spack cd to print a descriptive\n help message when called with -h.\"\"\"\n spack.cmd.location.setup_parser(subparser)\n\n\ndef cd(parser, args):\n spec = \" \".join(args.spec) if args.spec else \"SPEC\"\n spack.cmd.common.shell_init_instructions(\n \"spack cd\", \"cd `spack location --install-dir %s`\" % spec\n )\n" }, { "path": "lib/spack/spack/cmd/change.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport warnings\n\nimport spack.cmd\nimport spack.environment\nimport spack.spec\nfrom spack.cmd.common import arguments\n\ndescription = \"change an existing spec in an environment\"\nsection = \"environments\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n subparser.add_argument(\n \"-l\",\n \"--list-name\",\n dest=\"list_name\",\n default=\"specs\",\n help=\"name of the list to remove abstract specs from\",\n )\n subparser.add_argument(\n \"--match-spec\",\n dest=\"match_spec\",\n help=\"change all specs matching match-spec (default is match by spec name)\",\n )\n subparser.add_argument(\n \"-a\",\n \"--all\",\n action=\"store_true\",\n help=\"change all matching abstract specs (allow changing more than one abstract spec)\",\n )\n subparser.add_argument(\n \"-c\",\n \"--concrete\",\n action=\"store_true\",\n default=False,\n help=\"change concrete specs in the environment\",\n )\n subparser.add_argument(\n \"-C\",\n \"--concrete-only\",\n action=\"store_true\",\n default=False,\n help=\"change only concrete specs in the environment\",\n )\n arguments.add_common_arguments(subparser, [\"specs\"])\n\n\ndef change(parser, args):\n if args.all and args.concrete_only:\n warnings.warn(\"'spack change --all' argument is ignored with '--concrete-only'\")\n if args.list_name != \"specs\" and args.concrete_only:\n warnings.warn(\"'spack change --list-name' argument is ignored with '--concrete-only'\")\n\n env = spack.cmd.require_active_env(cmd_name=\"change\")\n\n match_spec = None\n if args.match_spec:\n match_spec = spack.cmd.parse_specs([args.match_spec])[0]\n specs = spack.cmd.parse_specs(args.specs)\n\n with env.write_transaction():\n if not args.concrete_only:\n try:\n for spec in specs:\n env.change_existing_spec(\n spec,\n list_name=args.list_name,\n match_spec=match_spec,\n allow_changing_multiple_specs=args.all,\n )\n except (ValueError, spack.environment.SpackEnvironmentError) as e:\n msg = \"Cannot change abstract specs.\"\n msg += \" Try again with '--concrete-only' to change concrete specs only.\"\n raise ValueError(msg) from e\n\n if args.concrete or args.concrete_only:\n for spec in specs:\n env.mutate(selector=match_spec or spack.spec.Spec(spec.name), mutator=spec)\n\n env.write()\n" }, { "path": "lib/spack/spack/cmd/checksum.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport re\nimport sys\nfrom typing import Dict, Optional, Tuple\n\nimport spack.llnl.string\nimport spack.llnl.util.lang\nimport spack.repo\nimport spack.spec\nimport spack.stage\nimport spack.util.web as web_util\nfrom spack.cmd.common import arguments\nfrom spack.llnl.util import tty\nfrom spack.package_base import (\n ManualDownloadRequiredError,\n PackageBase,\n deprecated_version,\n preferred_version,\n)\nfrom spack.util.editor import editor\nfrom spack.util.format import get_version_lines\nfrom spack.version import StandardVersion, Version\n\ndescription = \"checksum available versions of a package\"\nsection = \"packaging\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n subparser.add_argument(\n \"--keep-stage\",\n action=\"store_true\",\n default=False,\n help=\"don't clean up staging area when command completes\",\n )\n subparser.add_argument(\n \"--batch\",\n \"-b\",\n action=\"store_true\",\n default=False,\n help=\"don't ask which versions to checksum\",\n )\n subparser.add_argument(\n \"--latest\",\n \"-l\",\n action=\"store_true\",\n default=False,\n help=\"checksum the latest available version\",\n )\n subparser.add_argument(\n \"--preferred\",\n \"-p\",\n action=\"store_true\",\n default=False,\n help=\"checksum the known Spack preferred version\",\n )\n modes_parser = subparser.add_mutually_exclusive_group()\n modes_parser.add_argument(\n \"--add-to-package\",\n \"-a\",\n action=\"store_true\",\n default=False,\n help=\"add new versions to package\",\n )\n modes_parser.add_argument(\n \"--verify\", action=\"store_true\", default=False, help=\"verify known package checksums\"\n )\n subparser.add_argument(\"package\", help=\"name or spec (e.g. ``cmake`` or ``cmake@3.18``)\")\n subparser.add_argument(\n \"versions\",\n nargs=\"*\",\n help=\"checksum these specific versions (if omitted, Spack searches for remote versions)\",\n )\n arguments.add_common_arguments(subparser, [\"jobs\"])\n subparser.epilog = (\n \"examples:\\n\"\n \" `spack checksum zlib@1.2` autodetects versions 1.2.0 to 1.2.13 from the remote\\n\"\n \" `spack checksum zlib 1.2.13` checksums exact version 1.2.13 directly without search\\n\"\n )\n\n\ndef checksum(parser, args):\n spec = spack.spec.Spec(args.package)\n\n # Get the package we're going to generate checksums for\n pkg: PackageBase = spack.repo.PATH.get_pkg_class(spec.name)(spec)\n\n # Skip manually downloaded packages\n if pkg.manual_download:\n raise ManualDownloadRequiredError(pkg.download_instr)\n\n versions = [StandardVersion.from_string(v) for v in args.versions]\n\n # Define placeholder for remote versions. This'll help reduce redundant work if we need to\n # check for the existence of remote versions more than once.\n remote_versions: Optional[Dict[StandardVersion, str]] = None\n\n # Add latest version if requested\n if args.latest:\n remote_versions = pkg.fetch_remote_versions(concurrency=args.jobs)\n if len(remote_versions) > 0:\n versions.append(max(remote_versions.keys()))\n\n # Add preferred version if requested (todo: exclude git versions)\n if args.preferred:\n versions.append(preferred_version(pkg))\n\n # Store a dict of the form version -> URL\n url_dict: Dict[StandardVersion, str] = {}\n\n for version in versions:\n if deprecated_version(pkg, version):\n tty.warn(f\"Version {version} is deprecated\")\n\n url = pkg.find_valid_url_for_version(version)\n if url is not None:\n url_dict[version] = url\n continue\n # If we get here, it's because no valid url was provided by the package. Do expensive\n # fallback to try to recover\n if remote_versions is None:\n remote_versions = pkg.fetch_remote_versions(concurrency=args.jobs)\n if version in remote_versions:\n url_dict[version] = remote_versions[version]\n\n if len(versions) <= 0:\n if remote_versions is None:\n remote_versions = pkg.fetch_remote_versions(concurrency=args.jobs)\n url_dict = remote_versions\n\n # A spidered URL can differ from the package.py *computed* URL, pointing to different tarballs.\n # For example, GitHub release pages sometimes have multiple tarballs with different shasum:\n # - releases/download/1.0/-1.0.tar.gz (uploaded tarball)\n # - archive/refs/tags/1.0.tar.gz (generated tarball)\n # We wanna ensure that `spack checksum` and `spack install` ultimately use the same URL, so\n # here we check whether the crawled and computed URLs disagree, and if so, prioritize the\n # former if that URL exists (just sending a HEAD request that is).\n url_changed_for_version = set()\n for version, url in url_dict.items():\n possible_urls = pkg.all_urls_for_version(version)\n if url not in possible_urls:\n for possible_url in possible_urls:\n if web_util.url_exists(possible_url):\n url_dict[version] = possible_url\n break\n else:\n url_changed_for_version.add(version)\n\n if not url_dict:\n tty.die(f\"Could not find any remote versions for {pkg.name}\")\n elif len(url_dict) > 1 and not args.batch and sys.stdin.isatty():\n filtered_url_dict = spack.stage.interactive_version_filter(\n url_dict,\n pkg.versions,\n url_changes=url_changed_for_version,\n initial_verion_filter=spec.versions,\n )\n if not filtered_url_dict:\n exit(0)\n url_dict = filtered_url_dict\n else:\n tty.info(f\"Found {spack.llnl.string.plural(len(url_dict), 'version')} of {pkg.name}\")\n\n version_hashes = spack.stage.get_checksums_for_versions(\n url_dict, pkg.name, keep_stage=args.keep_stage, fetch_options=pkg.fetch_options\n )\n\n if args.verify:\n print_checksum_status(pkg, version_hashes)\n sys.exit(0)\n\n # convert dict into package.py version statements\n version_lines = get_version_lines(version_hashes)\n print()\n print(version_lines)\n print()\n\n if args.add_to_package:\n path = spack.repo.PATH.filename_for_package_name(pkg.name)\n num_versions_added = add_versions_to_pkg(path, version_lines)\n tty.msg(f\"Added {num_versions_added} new versions to {pkg.name} in {path}\")\n if not args.batch and sys.stdin.isatty():\n editor(path)\n\n\ndef print_checksum_status(pkg: PackageBase, version_hashes: dict):\n \"\"\"\n Verify checksums present in version_hashes against those present\n in the package's instructions.\n\n Args:\n pkg (spack.package_base.PackageBase): A package class for a given package in Spack.\n version_hashes (dict): A dictionary of the form: version -> checksum.\n\n \"\"\"\n results = []\n num_verified = 0\n failed = False\n\n max_len = max(len(str(v)) for v in version_hashes)\n num_total = len(version_hashes)\n\n for version, sha in version_hashes.items():\n if version not in pkg.versions:\n msg = \"No previous checksum\"\n status = \"-\"\n\n elif sha == pkg.versions[version][\"sha256\"]:\n msg = \"Correct\"\n status = \"=\"\n num_verified += 1\n\n else:\n msg = sha\n status = \"x\"\n failed = True\n\n results.append(\"{0:{1}} {2} {3}\".format(str(version), max_len, f\"[{status}]\", msg))\n\n # Display table of checksum results.\n tty.msg(\n f\"Verified {num_verified} of {num_total}\",\n \"\",\n *spack.llnl.util.lang.elide_list(results),\n \"\",\n )\n\n # Terminate at the end of function to prevent additional output.\n if failed:\n print()\n tty.die(\"Invalid checksums found.\")\n\n\ndef _update_version_statements(package_src: str, version_lines: str) -> Tuple[int, str]:\n \"\"\"Returns a tuple of number of versions added and the package's modified contents.\"\"\"\n num_versions_added = 0\n version_statement_re = re.compile(r\"([\\t ]+version\\([^\\)]*\\))\")\n version_re = re.compile(r'[\\t ]+version\\(\\s*\"([^\"]+)\"[^\\)]*\\)')\n\n # Split rendered version lines into tuple of (version, version_line)\n # We reverse sort here to make sure the versions match the version_lines\n new_versions = []\n for ver_line in version_lines.split(\"\\n\"):\n match = version_re.match(ver_line)\n if match:\n new_versions.append((Version(match.group(1)), ver_line))\n\n split_contents = version_statement_re.split(package_src)\n\n for i, subsection in enumerate(split_contents):\n # If there are no more versions to add we should exit\n if len(new_versions) <= 0:\n break\n\n # Check if the section contains a version\n contents_version = version_re.match(subsection)\n if contents_version is not None:\n parsed_version = Version(contents_version.group(1))\n\n if parsed_version < new_versions[0][0]:\n split_contents[i:i] = [new_versions.pop(0)[1], \" # FIXME\", \"\\n\"]\n num_versions_added += 1\n\n elif parsed_version == new_versions[0][0]:\n new_versions.pop(0)\n\n return num_versions_added, \"\".join(split_contents)\n\n\ndef add_versions_to_pkg(path: str, version_lines: str) -> int:\n \"\"\"Add new versions to a package.py file. Returns the number of versions added.\"\"\"\n with open(path, \"r\", encoding=\"utf-8\") as f:\n package_src = f.read()\n num_versions_added, package_src = _update_version_statements(package_src, version_lines)\n if num_versions_added > 0:\n with open(path, \"w\", encoding=\"utf-8\") as f:\n f.write(package_src)\n return num_versions_added\n" }, { "path": "lib/spack/spack/cmd/ci.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport json\nimport os\nimport shutil\nimport sys\nfrom typing import Dict, List\nfrom urllib.parse import urlparse, urlunparse\n\nimport spack.binary_distribution\nimport spack.ci as spack_ci\nimport spack.cmd\nimport spack.cmd.buildcache as buildcache\nimport spack.cmd.common.arguments\nimport spack.config as cfg\nimport spack.environment as ev\nimport spack.error\nimport spack.fetch_strategy\nimport spack.hash_types as ht\nimport spack.llnl.util.filesystem as fs\nimport spack.llnl.util.tty.color as clr\nimport spack.mirrors.mirror\nimport spack.package_base\nimport spack.repo\nimport spack.spec\nimport spack.stage\nimport spack.util.git\nimport spack.util.gpg as gpg_util\nimport spack.util.timer as timer\nimport spack.util.url as url_util\nimport spack.util.web as web_util\nfrom spack.llnl.util import tty\nfrom spack.version import StandardVersion\n\nfrom . import doc_dedented, doc_first_line\n\ndescription = \"manage continuous integration pipelines\"\nsection = \"build\"\nlevel = \"long\"\n\nSPACK_COMMAND = \"spack\"\nINSTALL_FAIL_CODE = 1\nFAILED_CREATE_BUILDCACHE_CODE = 100\n\n\ndef deindent(desc):\n return desc.replace(\" \", \"\")\n\n\ndef unicode_escape(path: str) -> str:\n \"\"\"Returns transformed path with any unicode\n characters replaced with their corresponding escapes\"\"\"\n return path.encode(\"unicode-escape\").decode(\"utf-8\")\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n setattr(setup_parser, \"parser\", subparser)\n subparsers = subparser.add_subparsers(help=\"CI sub-commands\")\n\n # Dynamic generation of the jobs yaml from a spack environment\n generate = subparsers.add_parser(\n \"generate\", description=doc_dedented(ci_generate), help=doc_first_line(ci_generate)\n )\n generate.add_argument(\n \"--output-file\",\n default=None,\n help=\"pathname for the generated gitlab ci yaml file\\n\\n\"\n \"path to the file where generated jobs file should be written. \"\n \"default is .gitlab-ci.yml in the root of the repository\",\n )\n prune_dag_group = generate.add_mutually_exclusive_group()\n prune_dag_group.add_argument(\n \"--prune-dag\",\n action=\"store_true\",\n dest=\"prune_dag\",\n default=True,\n help=\"skip up-to-date specs\\n\\n\"\n \"do not generate jobs for specs that are up-to-date on the mirror\",\n )\n prune_dag_group.add_argument(\n \"--no-prune-dag\",\n action=\"store_false\",\n dest=\"prune_dag\",\n default=True,\n help=\"process up-to-date specs\\n\\n\"\n \"generate jobs for specs even when they are up-to-date on the mirror\",\n )\n prune_unaffected_group = generate.add_mutually_exclusive_group()\n prune_unaffected_group.add_argument(\n \"--prune-unaffected\",\n action=\"store_true\",\n dest=\"prune_unaffected\",\n default=False,\n help=\"skip up-to-date specs\\n\\n\"\n \"do not generate jobs for specs that are up-to-date on the mirror\",\n )\n prune_unaffected_group.add_argument(\n \"--no-prune-unaffected\",\n action=\"store_false\",\n dest=\"prune_unaffected\",\n default=False,\n help=\"process up-to-date specs\\n\\n\"\n \"generate jobs for specs even when they are up-to-date on the mirror\",\n )\n prune_ext_group = generate.add_mutually_exclusive_group()\n prune_ext_group.add_argument(\n \"--prune-externals\",\n action=\"store_true\",\n dest=\"prune_externals\",\n default=True,\n help=\"skip external specs\\n\\ndo not generate jobs for specs that are marked as external\",\n )\n prune_ext_group.add_argument(\n \"--no-prune-externals\",\n action=\"store_false\",\n dest=\"prune_externals\",\n default=True,\n help=\"process external specs\\n\\n\"\n \"generate jobs for specs even when they are marked as external\",\n )\n generate.add_argument(\n \"--check-index-only\",\n action=\"store_true\",\n dest=\"index_only\",\n default=False,\n help=\"only check spec state from buildcache indices\\n\\n\"\n \"Spack always checks specs against configured binary mirrors, regardless of the DAG \"\n \"pruning option. if enabled, Spack will assume all remote buildcache indices are \"\n \"up-to-date when assessing whether the spec on the mirror, if present, is up-to-date. \"\n \"this has the benefit of reducing pipeline generation time but at the potential cost of \"\n \"needlessly rebuilding specs when the indices are outdated. if not enabled, Spack will \"\n \"fetch remote spec files directly to assess whether the spec on the mirror is up-to-date\",\n )\n generate.add_argument(\n \"--artifacts-root\",\n default=\"jobs_scratch_dir\",\n help=\"path to the root of the artifacts directory\\n\\n\"\n \"The spack ci module assumes it will normally be run from within your project \"\n \"directory, wherever that is checked out to run your ci. The artifacts root directory \"\n \"should specify a name that can safely be used for artifacts within your project \"\n \"directory.\",\n )\n generate.add_argument(\n \"--forward-variable\",\n action=\"append\",\n help=\"Environment variables to forward from the generate environment \"\n \"to the generated jobs.\",\n )\n generate.set_defaults(func=ci_generate)\n\n spack.cmd.common.arguments.add_concretizer_args(generate)\n spack.cmd.common.arguments.add_common_arguments(generate, [\"jobs\"])\n\n # Rebuild the buildcache index associated with the mirror in the\n # active, gitlab-enabled environment.\n index = subparsers.add_parser(\n \"rebuild-index\", description=doc_dedented(ci_reindex), help=doc_first_line(ci_reindex)\n )\n index.set_defaults(func=ci_reindex)\n\n # Handle steps of a ci build/rebuild\n rebuild = subparsers.add_parser(\n \"rebuild\", description=doc_dedented(ci_rebuild), help=doc_first_line(ci_rebuild)\n )\n rebuild.add_argument(\n \"-t\",\n \"--tests\",\n action=\"store_true\",\n default=False,\n help=\"run stand-alone tests after the build\",\n )\n rebuild_ff_group = rebuild.add_mutually_exclusive_group()\n rebuild_ff_group.add_argument(\n \"--no-fail-fast\",\n action=\"store_false\",\n default=True,\n dest=\"fail_fast\",\n help=\"continue build/stand-alone tests after the first failure\",\n )\n rebuild_ff_group.add_argument(\n \"--fail-fast\",\n action=\"store_true\",\n dest=\"fail_fast\",\n help=\"stop build/stand-alone tests after the first failure\",\n )\n rebuild.add_argument(\n \"--timeout\",\n type=int,\n default=None,\n help=\"maximum time (in seconds) that tests are allowed to run\",\n )\n rebuild.set_defaults(func=ci_rebuild)\n spack.cmd.common.arguments.add_common_arguments(rebuild, [\"jobs\"])\n\n # Facilitate reproduction of a failed CI build job\n reproduce = subparsers.add_parser(\n \"reproduce-build\",\n description=doc_dedented(ci_reproduce),\n help=doc_first_line(ci_reproduce),\n )\n reproduce.add_argument(\n \"job_url\", help=\"URL of GitLab job web page or artifact\", type=_gitlab_artifacts_url\n )\n reproduce.add_argument(\n \"--runtime\",\n help=\"Container runtime to use.\",\n default=\"docker\",\n choices=[\"docker\", \"podman\"],\n )\n reproduce.add_argument(\n \"--working-dir\",\n help=\"where to unpack artifacts\",\n default=os.path.join(os.getcwd(), \"ci_reproduction\"),\n )\n reproduce.add_argument(\n \"-s\", \"--autostart\", help=\"Run docker reproducer automatically\", action=\"store_true\"\n )\n reproduce.add_argument(\n \"--use-local-head\",\n help=\"Use the HEAD of the local Spack instead of reproducing a commit\",\n action=\"store_true\",\n )\n gpg_group = reproduce.add_mutually_exclusive_group(required=False)\n gpg_group.add_argument(\n \"--gpg-file\", help=\"Path to public GPG key for validating binary cache installs\"\n )\n gpg_group.add_argument(\n \"--gpg-url\", help=\"URL to public GPG key for validating binary cache installs\"\n )\n\n reproduce.set_defaults(func=ci_reproduce)\n\n # Verify checksums inside of ci workflows\n verify_versions = subparsers.add_parser(\n \"verify-versions\",\n description=doc_dedented(ci_verify_versions),\n help=doc_first_line(ci_verify_versions),\n )\n verify_versions.add_argument(\"from_ref\", help=\"git ref from which start looking at changes\")\n verify_versions.add_argument(\"to_ref\", help=\"git ref to end looking at changes\")\n verify_versions.set_defaults(func=ci_verify_versions)\n\n\ndef ci_generate(args):\n \"\"\"\\\n generate jobs file from a CI-aware spack file\n\n if you want to report the results on CDash, you will need to set the SPACK_CDASH_AUTH_TOKEN\n before invoking this command. the value must be the CDash authorization token needed to create\n a build group and register all generated jobs under it\n \"\"\"\n env = spack.cmd.require_active_env(cmd_name=\"ci generate\")\n spack_ci.generate_pipeline(env, args)\n\n\ndef ci_reindex(args):\n \"\"\"\\\n rebuild the buildcache index for the remote mirror\n\n use the active, gitlab-enabled environment to rebuild the buildcache index for the associated\n mirror\n \"\"\"\n env = spack.cmd.require_active_env(cmd_name=\"ci rebuild-index\")\n yaml_root = env.manifest[ev.TOP_LEVEL_KEY]\n\n if \"mirrors\" not in yaml_root or len(yaml_root[\"mirrors\"].values()) < 1:\n tty.die(\"spack ci rebuild-index requires an env containing a mirror\")\n\n ci_mirrors = yaml_root[\"mirrors\"]\n mirror_urls = [url for url in ci_mirrors.values()]\n remote_mirror_url = mirror_urls[0]\n mirror = spack.mirrors.mirror.Mirror(remote_mirror_url)\n\n buildcache.update_index(mirror, update_keys=True)\n\n\ndef ci_rebuild(args):\n \"\"\"\\\n rebuild a spec if it is not on the remote mirror\n\n check a single spec against the remote mirror, and rebuild it from source if the mirror does\n not contain the hash\n \"\"\"\n rebuild_timer = timer.Timer()\n\n env = spack.cmd.require_active_env(cmd_name=\"ci rebuild\")\n\n # Make sure the environment is \"gitlab-enabled\", or else there's nothing\n # to do.\n ci_config = cfg.get(\"ci\")\n if not ci_config:\n tty.die(\"spack ci rebuild requires an env containing ci cfg\")\n\n # Grab the environment variables we need. These either come from the\n # pipeline generation step (\"spack ci generate\"), where they were written\n # out as variables, or else provided by GitLab itself.\n pipeline_artifacts_dir = os.environ.get(\"SPACK_ARTIFACTS_ROOT\")\n job_log_dir = os.environ.get(\"SPACK_JOB_LOG_DIR\")\n job_test_dir = os.environ.get(\"SPACK_JOB_TEST_DIR\")\n repro_dir = os.environ.get(\"SPACK_JOB_REPRO_DIR\")\n concrete_env_dir = os.environ.get(\"SPACK_CONCRETE_ENV_DIR\")\n ci_job_name = os.environ.get(\"CI_JOB_NAME\")\n signing_key = os.environ.get(\"SPACK_SIGNING_KEY\")\n job_spec_pkg_name = os.environ.get(\"SPACK_JOB_SPEC_PKG_NAME\")\n job_spec_dag_hash = os.environ.get(\"SPACK_JOB_SPEC_DAG_HASH\")\n spack_pipeline_type = os.environ.get(\"SPACK_PIPELINE_TYPE\")\n spack_ci_stack_name = os.environ.get(\"SPACK_CI_STACK_NAME\")\n rebuild_everything = os.environ.get(\"SPACK_REBUILD_EVERYTHING\")\n require_signing = os.environ.get(\"SPACK_REQUIRE_SIGNING\")\n\n # If signing key was provided via \"SPACK_SIGNING_KEY\", then try to import it.\n if signing_key:\n spack_ci.import_signing_key(signing_key)\n\n # Fail early if signing is required but we don't have a signing key\n sign_binaries = require_signing is not None and require_signing.lower() == \"true\"\n if sign_binaries and not spack_ci.can_sign_binaries():\n gpg_util.list(False, True)\n tty.die(\"SPACK_REQUIRE_SIGNING=True => spack must have exactly one signing key\")\n\n # Construct absolute paths relative to current $CI_PROJECT_DIR\n ci_project_dir = os.environ.get(\"CI_PROJECT_DIR\")\n pipeline_artifacts_dir = os.path.join(ci_project_dir, pipeline_artifacts_dir)\n job_log_dir = os.path.join(ci_project_dir, job_log_dir)\n job_test_dir = os.path.join(ci_project_dir, job_test_dir)\n repro_dir = os.path.join(ci_project_dir, repro_dir)\n concrete_env_dir = os.path.join(ci_project_dir, concrete_env_dir)\n\n # Debug print some of the key environment variables we should have received\n tty.debug(\"pipeline_artifacts_dir = {0}\".format(pipeline_artifacts_dir))\n tty.debug(\"job_spec_pkg_name = {0}\".format(job_spec_pkg_name))\n\n # Query the environment manifest to find out whether we're reporting to a\n # CDash instance, and if so, gather some information from the manifest to\n # support that task.\n cdash_config = cfg.get(\"cdash\")\n cdash_handler = None\n if \"build-group\" in cdash_config:\n cdash_handler = spack_ci.CDashHandler(cdash_config)\n tty.debug(\"cdash url = {0}\".format(cdash_handler.url))\n tty.debug(\"cdash project = {0}\".format(cdash_handler.project))\n tty.debug(\"cdash project_enc = {0}\".format(cdash_handler.project_enc))\n tty.debug(\"cdash build_name = {0}\".format(cdash_handler.build_name))\n tty.debug(\"cdash build_stamp = {0}\".format(cdash_handler.build_stamp))\n tty.debug(\"cdash site = {0}\".format(cdash_handler.site))\n tty.debug(\"cdash build_group = {0}\".format(cdash_handler.build_group))\n\n # Is this a pipeline run on a spack PR or a merge to develop? It might\n # be neither, e.g. a pipeline run on some environment repository.\n spack_is_pr_pipeline = spack_pipeline_type == \"spack_pull_request\"\n spack_is_develop_pipeline = spack_pipeline_type == \"spack_protected_branch\"\n\n tty.debug(\n \"Pipeline type - PR: {0}, develop: {1}\".format(\n spack_is_pr_pipeline, spack_is_develop_pipeline\n )\n )\n\n full_rebuild = True if rebuild_everything and rebuild_everything.lower() == \"true\" else False\n\n pipeline_mirrors = spack.mirrors.mirror.MirrorCollection(binary=True)\n buildcache_destination = None\n if \"buildcache-destination\" not in pipeline_mirrors:\n tty.die(\"spack ci rebuild requires a mirror named 'buildcache-destination\")\n\n buildcache_destination = pipeline_mirrors[\"buildcache-destination\"]\n\n # Get the concrete spec to be built by this job.\n try:\n job_spec = env.get_one_by_hash(job_spec_dag_hash)\n except AssertionError:\n tty.die(\"Could not find environment spec with hash {0}\".format(job_spec_dag_hash))\n\n job_spec_json_file = \"{0}.json\".format(job_spec_pkg_name)\n job_spec_json_path = os.path.join(repro_dir, job_spec_json_file)\n\n # To provide logs, cdash reports, etc for developer download/perusal,\n # these things have to be put into artifacts. This means downstream\n # jobs that \"need\" this job will get those artifacts too. So here we\n # need to clean out the artifacts we may have got from upstream jobs.\n\n cdash_report_dir = os.path.join(pipeline_artifacts_dir, \"cdash_report\")\n if os.path.exists(cdash_report_dir):\n shutil.rmtree(cdash_report_dir)\n\n if os.path.exists(job_log_dir):\n shutil.rmtree(job_log_dir)\n\n if os.path.exists(job_test_dir):\n shutil.rmtree(job_test_dir)\n\n if os.path.exists(repro_dir):\n shutil.rmtree(repro_dir)\n\n # Now that we removed them if they existed, create the directories we\n # need for storing artifacts. The cdash_report directory will be\n # created internally if needed.\n os.makedirs(job_log_dir)\n os.makedirs(job_test_dir)\n os.makedirs(repro_dir)\n\n # Copy the concrete environment files to the repro directory so we can\n # expose them as artifacts and not conflict with the concrete environment\n # files we got as artifacts from the upstream pipeline generation job.\n # Try to cast a slightly wider net too, and hopefully get the generated\n # pipeline yaml. If we miss it, the user will still be able to go to the\n # pipeline generation job and get it from there.\n target_dirs = [concrete_env_dir, pipeline_artifacts_dir]\n\n for dir_to_list in target_dirs:\n for file_name in os.listdir(dir_to_list):\n src_file = os.path.join(dir_to_list, file_name)\n if os.path.isfile(src_file):\n dst_file = os.path.join(repro_dir, file_name)\n shutil.copyfile(src_file, dst_file)\n\n # Write this job's spec json into the reproduction directory, and it will\n # also be used in the generated \"spack install\" command to install the spec\n tty.debug(\"job concrete spec path: {0}\".format(job_spec_json_path))\n with open(job_spec_json_path, \"w\", encoding=\"utf-8\") as fd:\n fd.write(job_spec.to_json(hash=ht.dag_hash))\n\n # Write some other details to aid in reproduction into an artifact\n repro_file = os.path.join(repro_dir, \"repro.json\")\n repro_details = {\n \"job_name\": ci_job_name,\n \"job_spec_json\": job_spec_json_file,\n \"ci_project_dir\": ci_project_dir,\n }\n with open(repro_file, \"w\", encoding=\"utf-8\") as fd:\n fd.write(json.dumps(repro_details))\n\n # Write information about spack into an artifact in the repro dir\n spack_info = spack_ci.get_spack_info()\n spack_info_file = os.path.join(repro_dir, \"spack_info.txt\")\n with open(spack_info_file, \"wb\") as fd:\n fd.write(b\"\\n\")\n fd.write(spack_info.encode(\"utf8\"))\n fd.write(b\"\\n\")\n\n matches = (\n None\n if full_rebuild\n else spack.binary_distribution.get_mirrors_for_spec(job_spec, index_only=False)\n )\n\n if matches:\n # Got a hash match on at least one configured mirror. All\n # matches represent the fully up-to-date spec, so should all be\n # equivalent. If artifacts mirror is enabled, we just pick one\n # of the matches and download the buildcache files from there to\n # the artifacts, so they're available to be used by dependent\n # jobs in subsequent stages.\n tty.msg(\"No need to rebuild {0}, found hash match at: \".format(job_spec_pkg_name))\n for match in matches:\n tty.msg(\" {0}\".format(match.url))\n\n # Now we are done and successful\n return 0\n\n # No hash match anywhere means we need to rebuild spec\n\n # Start with spack arguments\n spack_cmd = [SPACK_COMMAND, \"--color=always\", \"install\"]\n\n config = cfg.get(\"config\")\n if not config[\"verify_ssl\"]:\n spack_cmd.append(\"-k\")\n\n install_args = [\n f\"--use-buildcache={spack_ci.common.win_quote('package:never,dependencies:only')}\"\n ]\n\n can_verify = spack_ci.can_verify_binaries()\n verify_binaries = can_verify and spack_is_pr_pipeline is False\n if not verify_binaries:\n install_args.append(\"--no-check-signature\")\n\n if args.jobs:\n install_args.append(f\"-j{args.jobs}\")\n\n fail_fast = bool(os.environ.get(\"SPACK_CI_FAIL_FAST\", str(args.fail_fast)))\n if fail_fast:\n install_args.append(\"--fail-fast\")\n\n slash_hash = spack_ci.common.win_quote(\"/\" + job_spec.dag_hash())\n\n # Arguments when installing the root from sources\n deps_install_args = install_args + [\"--only=dependencies\"]\n root_install_args = install_args + [\"--verbose\", \"--keep-stage\", \"--only=package\"]\n\n if cdash_handler:\n # Add additional arguments to `spack install` for CDash reporting.\n root_install_args.extend(cdash_handler.args())\n\n commands = [\n # apparently there's a race when spack bootstraps? do it up front once\n [SPACK_COMMAND, \"-e\", unicode_escape(env.path), \"bootstrap\", \"now\"],\n spack_cmd + deps_install_args + [slash_hash],\n spack_cmd + root_install_args + [slash_hash],\n ]\n tty.debug(\"Installing {0} from source\".format(job_spec.name))\n install_exit_code = spack_ci.process_command(\"install\", commands, repro_dir)\n\n # Now do the post-install tasks\n tty.debug(\"spack install exited {0}\".format(install_exit_code))\n\n # If a spec fails to build in a spack develop pipeline, we add it to a\n # list of known broken hashes. This allows spack PR pipelines to\n # avoid wasting compute cycles attempting to build those hashes.\n if install_exit_code == INSTALL_FAIL_CODE and spack_is_develop_pipeline:\n tty.debug(\"Install failed on develop\")\n if \"broken-specs-url\" in ci_config:\n broken_specs_url = ci_config[\"broken-specs-url\"]\n dev_fail_hash = job_spec.dag_hash()\n broken_spec_path = url_util.join(broken_specs_url, dev_fail_hash)\n tty.msg(\"Reporting broken develop build as: {0}\".format(broken_spec_path))\n spack_ci.write_broken_spec(\n broken_spec_path,\n job_spec_pkg_name,\n spack_ci_stack_name,\n os.environ.get(\"CI_JOB_URL\"),\n os.environ.get(\"CI_PIPELINE_URL\"),\n job_spec.to_dict(hash=ht.dag_hash),\n )\n\n # Copy logs and archived files from the install metadata (.spack) directory to artifacts now\n spack_ci.copy_stage_logs_to_artifacts(job_spec, job_log_dir)\n\n # Clear the stage directory\n spack.stage.purge()\n\n # If the installation succeeded and we're running stand-alone tests for\n # the package, run them and copy the output. Failures of any kind should\n # *not* terminate the build process or preclude creating the build cache.\n broken_tests = (\n \"broken-tests-packages\" in ci_config\n and job_spec.name in ci_config[\"broken-tests-packages\"]\n )\n reports_dir = fs.join_path(os.getcwd(), \"cdash_report\")\n if args.tests and broken_tests:\n tty.warn(\"Unable to run stand-alone tests since listed in ci's 'broken-tests-packages'\")\n if cdash_handler:\n msg = \"Package is listed in ci's broken-tests-packages\"\n cdash_handler.report_skipped(job_spec, reports_dir, reason=msg)\n cdash_handler.copy_test_results(reports_dir, job_test_dir)\n elif args.tests:\n if install_exit_code == 0:\n try:\n # First ensure we will use a reasonable test stage directory\n stage_root = os.path.dirname(str(job_spec.package.stage.path))\n test_stage = fs.join_path(stage_root, \"spack-standalone-tests\")\n tty.debug(\"Configuring test_stage to {0}\".format(test_stage))\n config_test_path = \"config:test_stage:{0}\".format(test_stage)\n cfg.add(config_test_path, scope=cfg.default_modify_scope())\n\n # Run the tests, resorting to junit results if not using cdash\n log_file = (\n None if cdash_handler else fs.join_path(test_stage, \"ci-test-results.xml\")\n )\n spack_ci.run_standalone_tests(\n cdash=cdash_handler,\n job_spec=job_spec,\n fail_fast=fail_fast,\n log_file=log_file,\n repro_dir=repro_dir,\n timeout=args.timeout,\n )\n\n except Exception as err:\n # If there is any error, just print a warning.\n msg = \"Error processing stand-alone tests: {0}\".format(str(err))\n tty.warn(msg)\n\n finally:\n # Copy the test log/results files\n spack_ci.copy_test_logs_to_artifacts(test_stage, job_test_dir)\n if cdash_handler:\n cdash_handler.copy_test_results(reports_dir, job_test_dir)\n elif log_file:\n spack_ci.copy_files_to_artifacts(log_file, job_test_dir)\n else:\n tty.warn(\"No recognized test results reporting option\")\n\n else:\n tty.warn(\"Unable to run stand-alone tests due to unsuccessful installation\")\n if cdash_handler:\n msg = \"Failed to install the package\"\n cdash_handler.report_skipped(job_spec, reports_dir, reason=msg)\n cdash_handler.copy_test_results(reports_dir, job_test_dir)\n\n if install_exit_code == 0:\n # If the install succeeded, push it to the buildcache destination. Failure to push\n # will result in a non-zero exit code. Pushing is best-effort.\n for result in spack_ci.create_buildcache(\n input_spec=job_spec,\n destination_mirror_urls=[buildcache_destination.push_url],\n sign_binaries=spack_ci.can_sign_binaries(),\n ):\n if not result.success:\n install_exit_code = FAILED_CREATE_BUILDCACHE_CODE\n (tty.msg if result.success else tty.error)(\n f\"{'Pushed' if result.success else 'Failed to push'} \"\n f\"{job_spec.format('{name}{@version}{/hash:7}', color=clr.get_color_when())} \"\n f\"to {result.url}\"\n )\n\n # If this is a develop pipeline, check if the spec that we just built is\n # on the broken-specs list. If so, remove it.\n if spack_is_develop_pipeline and \"broken-specs-url\" in ci_config:\n broken_specs_url = ci_config[\"broken-specs-url\"]\n just_built_hash = job_spec.dag_hash()\n broken_spec_path = url_util.join(broken_specs_url, just_built_hash)\n if web_util.url_exists(broken_spec_path):\n tty.msg(\"Removing {0} from the list of broken specs\".format(broken_spec_path))\n try:\n web_util.remove_url(broken_spec_path)\n except Exception as err:\n # If there is an S3 error (e.g., access denied or connection\n # error), the first non boto-specific class in the exception\n # hierarchy is Exception. Just print a warning and return.\n msg = \"Error removing {0} from broken specs list: {1}\"\n tty.warn(msg.format(broken_spec_path, err))\n\n else:\n # If the install did not succeed, print out some instructions on how to reproduce this\n # build failure outside of the pipeline environment.\n tty.debug(\"spack install exited non-zero, will not create buildcache\")\n\n api_root_url = os.environ.get(\"CI_API_V4_URL\")\n ci_project_id = os.environ.get(\"CI_PROJECT_ID\")\n ci_job_id = os.environ.get(\"CI_JOB_ID\")\n\n repro_job_url = f\"{api_root_url}/projects/{ci_project_id}/jobs/{ci_job_id}/artifacts\"\n # Control characters cause this to be printed in blue so it stands out\n print(\n f\"\"\"\n\n\\033[34mTo reproduce this build locally, run:\n\n spack ci reproduce-build {repro_job_url} [--working-dir ] [--autostart]\n\nIf this project does not have public pipelines, you will need to first:\n\n export GITLAB_PRIVATE_TOKEN=\n\n... then follow the printed instructions.\\033[0;0m\n\n\"\"\"\n )\n\n rebuild_timer.stop()\n try:\n with open(\"install_timers.json\", \"w\", encoding=\"utf-8\") as timelog:\n extra_attributes = {\"name\": \".ci-rebuild\"}\n rebuild_timer.write_json(timelog, extra_attributes=extra_attributes)\n except Exception as e:\n tty.debug(str(e))\n\n # Tie job success/failure to the success/failure of building the spec\n return install_exit_code\n\n\ndef ci_reproduce(args):\n \"\"\"\\\n generate instructions for reproducing the spec rebuild job\n\n artifacts of the provided gitlab pipeline rebuild job's URL will be used to derive\n instructions for reproducing the build locally\n \"\"\"\n # Allow passing GPG key for reprocuding protected CI jobs\n if args.gpg_file:\n gpg_key_url = url_util.path_to_file_url(args.gpg_file)\n elif args.gpg_url:\n gpg_key_url = args.gpg_url\n else:\n gpg_key_url = None\n\n return spack_ci.reproduce_ci_job(\n args.job_url,\n args.working_dir,\n args.autostart,\n gpg_key_url,\n args.runtime,\n args.use_local_head,\n )\n\n\ndef _gitlab_artifacts_url(url: str) -> str:\n \"\"\"Take a URL either to the URL of the job in the GitLab UI, or to the artifacts zip file,\n and output the URL to the artifacts zip file.\"\"\"\n parsed = urlparse(url)\n\n if not parsed.scheme or not parsed.netloc:\n raise ValueError(url)\n\n parts = parsed.path.split(\"/\")\n\n if len(parts) < 2:\n raise ValueError(url)\n\n # Just use API endpoints verbatim, they're probably generated by Spack.\n if parts[1] == \"api\":\n return url\n\n # If it's a URL to the job in the Gitlab UI, we may need to append the artifacts path.\n minus_idx = parts.index(\"-\")\n\n # Remove repeated slashes in the remainder\n rest = [p for p in parts[minus_idx + 1 :] if p]\n\n # Now the format is jobs/X or jobs/X/artifacts/download\n if len(rest) < 2 or rest[0] != \"jobs\":\n raise ValueError(url)\n\n if len(rest) == 2:\n # replace jobs/X with jobs/X/artifacts/download\n rest.extend((\"artifacts\", \"download\"))\n\n # Replace the parts and unparse.\n parts[minus_idx + 1 :] = rest\n\n # Don't allow fragments / queries\n return urlunparse(parsed._replace(path=\"/\".join(parts), fragment=\"\", query=\"\"))\n\n\ndef validate_standard_versions(\n pkg: spack.package_base.PackageBase, versions: List[StandardVersion]\n) -> bool:\n \"\"\"Get and test the checksum of a package version based on a tarball.\n Args:\n pkg: Spack package for which to validate a version checksum\n versions: list of package versions to validate\n Returns: True if all versions are valid, False if any version is invalid.\n \"\"\"\n url_dict: Dict[StandardVersion, str] = {}\n\n for version in versions:\n url = pkg.find_valid_url_for_version(version)\n assert url is not None, (\n f\"Package {pkg.name} does not have a valid URL for version {version}\"\n )\n url_dict[version] = url\n\n version_hashes = spack.stage.get_checksums_for_versions(\n url_dict, pkg.name, fetch_options=pkg.fetch_options\n )\n\n valid_checksums = True\n for version, sha in version_hashes.items():\n if sha != pkg.versions[version][\"sha256\"]:\n tty.error(\n f\"Invalid checksum found {pkg.name}@{version}\\n\"\n f\" [package.py] {pkg.versions[version]['sha256']}\\n\"\n f\" [Downloaded] {sha}\"\n )\n valid_checksums = False\n continue\n\n tty.info(f\"Validated {pkg.name}@{version} --> {sha}\")\n\n return valid_checksums\n\n\ndef validate_git_versions(\n pkg: spack.package_base.PackageBase, versions: List[StandardVersion]\n) -> bool:\n \"\"\"Get and test the commit and tag of a package version based on a git repository.\n Args:\n pkg: Spack package for which to validate a version\n versions: list of package versions to validate\n Returns: True if all versions are valid, False if any version is invalid.\n \"\"\"\n valid_commit = True\n for version in versions:\n fetcher = spack.fetch_strategy.for_package_version(pkg, version)\n assert isinstance(fetcher, spack.fetch_strategy.GitFetchStrategy)\n with spack.stage.Stage(fetcher) as stage:\n known_commit = pkg.versions[version][\"commit\"]\n try:\n stage.fetch()\n except spack.error.FetchError:\n tty.error(\n f\"Invalid commit for {pkg.name}@{version}\\n\"\n f\" {known_commit} could not be checked out in the git repository.\"\n )\n valid_commit = False\n continue\n\n # Test if the specified tag matches the commit in the package.py\n # We retrieve the commit associated with a tag and compare it to the\n # commit that is located in the package.py file.\n if \"tag\" in pkg.versions[version]:\n tag = pkg.versions[version][\"tag\"]\n url = pkg.version_or_package_attr(\"git\", version)\n found_commit = spack.util.git.get_commit_sha(url, tag)\n if not found_commit:\n tty.error(\n f\"Invalid tag for {pkg.name}@{version}\\n\"\n f\" {tag} could not be found in the git repository.\"\n )\n valid_commit = False\n continue\n\n if found_commit != known_commit:\n tty.error(\n f\"Mismatched tag <-> commit found for {pkg.name}@{version}\\n\"\n f\" [package.py] {known_commit}\\n\"\n f\" [Downloaded] {found_commit}\"\n )\n valid_commit = False\n continue\n\n # If we have downloaded the repository, found the commit, and compared\n # the tag (if specified) we can conclude that the version is pointing\n # at what we would expect.\n tty.info(f\"Validated {pkg.name}@{version} --> {known_commit}\")\n\n return valid_commit\n\n\ndef ci_verify_versions(args):\n \"\"\"\\\n validate version checksum & commits between git refs\n This command takes a from_ref and to_ref arguments and\n then parses the git diff between the two to determine which packages\n have been modified verifies the new checksums inside of them.\n \"\"\"\n # Get a list of all packages that have been changed or added\n # between from_ref and to_ref\n pkgs = spack.repo.get_all_package_diffs(\n \"AC\", spack.repo.builtin_repo(), args.from_ref, args.to_ref\n )\n\n success = True\n for pkg_name in pkgs:\n spec = spack.spec.Spec(pkg_name)\n pkg = spack.repo.PATH.get_pkg_class(spec.name)(spec)\n path = spack.repo.PATH.package_path(pkg_name)\n\n # Skip checking manual download packages and trust the maintainers\n if pkg.manual_download:\n tty.warn(f\"Skipping manual download package: {pkg_name}\")\n continue\n\n # Store versions checksums / commits for future loop\n url_version_to_checksum: Dict[StandardVersion, str] = {}\n git_version_to_checksum: Dict[StandardVersion, str] = {}\n for version in pkg.versions:\n # If the package version defines a sha256 we'll use that as the high entropy\n # string to detect which versions have been added between from_ref and to_ref\n if \"sha256\" in pkg.versions[version]:\n url_version_to_checksum[version] = pkg.versions[version][\"sha256\"]\n\n # If a package version instead defines a commit we'll use that as a\n # high entropy string to detect new versions.\n elif \"commit\" in pkg.versions[version]:\n git_version_to_checksum[version] = pkg.versions[version][\"commit\"]\n\n # TODO: enforce every version have a commit or a sha256 defined if not\n # an infinite version (there are a lot of packages where this doesn't work yet.)\n\n def filter_added_versions(versions: Dict[StandardVersion, str]) -> List[StandardVersion]:\n added_checksums = spack_ci.filter_added_checksums(\n versions.values(), path, from_ref=args.from_ref, to_ref=args.to_ref\n )\n return [v for v, c in versions.items() if c in added_checksums]\n\n with fs.working_dir(os.path.dirname(path)):\n new_url_versions = filter_added_versions(url_version_to_checksum)\n new_git_versions = filter_added_versions(git_version_to_checksum)\n\n if new_url_versions:\n success &= validate_standard_versions(pkg, new_url_versions)\n\n if new_git_versions:\n success &= validate_git_versions(pkg, new_git_versions)\n\n if not success:\n sys.exit(1)\n\n\ndef ci(parser, args):\n if args.func:\n return args.func(args)\n" }, { "path": "lib/spack/spack/cmd/clean.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport os\nimport shutil\n\nimport spack.caches\nimport spack.cmd\nimport spack.config\nimport spack.llnl.util.filesystem\nimport spack.llnl.util.tty as tty\nimport spack.stage\nimport spack.store\nimport spack.util.path\nfrom spack.cmd.common import arguments\nfrom spack.paths import lib_path, var_path\n\ndescription = \"remove temporary build files and/or downloaded archives\"\nsection = \"build\"\nlevel = \"long\"\n\n\nclass AllClean(argparse.Action):\n \"\"\"Activates flags -s -d -f -m -p and -b simultaneously\"\"\"\n\n def __call__(self, parser, namespace, values, option_string=None):\n parser.parse_args([\"-sdfmpb\"], namespace=namespace)\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n subparser.add_argument(\n \"-s\", \"--stage\", action=\"store_true\", help=\"remove all temporary build stages (default)\"\n )\n subparser.add_argument(\n \"-d\", \"--downloads\", action=\"store_true\", help=\"remove cached downloads\"\n )\n subparser.add_argument(\n \"-f\",\n \"--failures\",\n action=\"store_true\",\n help=\"force removal of all install failure tracking markers\",\n )\n subparser.add_argument(\n \"-m\",\n \"--misc-cache\",\n action=\"store_true\",\n help=\"remove long-lived caches, like the virtual package index\",\n )\n subparser.add_argument(\n \"-p\",\n \"--python-cache\",\n action=\"store_true\",\n help=\"remove .pyc, .pyo files and __pycache__ folders\",\n )\n subparser.add_argument(\n \"-b\",\n \"--bootstrap\",\n action=\"store_true\",\n help=\"remove software and configuration needed to bootstrap Spack\",\n )\n subparser.add_argument(\n \"-a\", \"--all\", action=AllClean, help=\"equivalent to ``-sdfmpb``\", nargs=0\n )\n arguments.add_common_arguments(subparser, [\"specs\"])\n\n\ndef remove_python_cache():\n for directory in [lib_path, var_path]:\n for root, dirs, files in os.walk(directory):\n for f in files:\n if f.endswith(\".pyc\") or f.endswith(\".pyo\"):\n fname = os.path.join(root, f)\n tty.debug(\"Removing {0}\".format(fname))\n os.remove(fname)\n for d in dirs:\n if d == \"__pycache__\":\n dname = os.path.join(root, d)\n tty.debug(\"Removing {0}\".format(dname))\n shutil.rmtree(dname)\n\n\ndef clean(parser, args):\n # If nothing was set, activate the default\n if not any(\n [\n args.specs,\n args.stage,\n args.downloads,\n args.failures,\n args.misc_cache,\n args.python_cache,\n args.bootstrap,\n ]\n ):\n args.stage = True\n\n # Then do the cleaning falling through the cases\n if args.specs:\n specs = spack.cmd.parse_specs(args.specs, concretize=False)\n specs = spack.cmd.matching_specs_from_env(specs)\n\n for spec in specs:\n msg = \"Cleaning build stage [{0}]\"\n tty.msg(msg.format(spec.short_spec))\n spec.package.do_clean()\n\n if args.stage:\n tty.msg(\"Removing all temporary build stages\")\n spack.stage.purge()\n\n if args.downloads:\n tty.msg(\"Removing cached downloads\")\n spack.caches.FETCH_CACHE.destroy()\n\n if args.failures:\n tty.msg(\"Removing install failure marks\")\n spack.store.STORE.failure_tracker.clear_all()\n\n if args.misc_cache:\n tty.msg(\"Removing cached information on repositories\")\n spack.caches.MISC_CACHE.destroy()\n\n if args.python_cache:\n tty.msg(\"Removing python cache files\")\n remove_python_cache()\n\n if args.bootstrap:\n bootstrap_prefix = spack.util.path.canonicalize_path(spack.config.get(\"bootstrap:root\"))\n msg = 'Removing bootstrapped software and configuration in \"{0}\"'\n tty.msg(msg.format(bootstrap_prefix))\n spack.llnl.util.filesystem.remove_directory_contents(bootstrap_prefix)\n" }, { "path": "lib/spack/spack/cmd/commands.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport copy\nimport os\nimport re\nimport shlex\nimport sys\nfrom argparse import ArgumentParser, Namespace\nfrom typing import IO, Any, Callable, Dict, Iterable, List, Optional, Sequence, Set, Tuple, Union\n\nimport spack.cmd\nimport spack.config\nimport spack.llnl.util.tty as tty\nimport spack.main\nimport spack.paths\nimport spack.platforms\nfrom spack.llnl.util.argparsewriter import ArgparseRstWriter, ArgparseWriter, Command\nfrom spack.llnl.util.tty.colify import colify\nfrom spack.main import SpackArgumentParser, section_descriptions\n\ndescription = \"list available spack commands\"\nsection = \"config\"\nlevel = \"long\"\n\n\n#: list of command formatters\nformatters: Dict[str, Callable[[Namespace, IO], None]] = {}\n\n\n#: standard arguments for updating completion scripts\n#: we iterate through these when called with ``--update-completion``\nupdate_completion_args: Dict[str, Dict[str, Any]] = {\n \"bash\": {\n \"aliases\": True,\n \"format\": \"bash\",\n \"header\": os.path.join(spack.paths.share_path, \"bash\", \"spack-completion.bash\"),\n \"update\": os.path.join(spack.paths.share_path, \"spack-completion.bash\"),\n },\n \"fish\": {\n \"aliases\": True,\n \"format\": \"fish\",\n \"header\": os.path.join(spack.paths.share_path, \"fish\", \"spack-completion.fish\"),\n \"update\": os.path.join(spack.paths.share_path, \"spack-completion.fish\"),\n },\n}\n\n\ndef formatter(func: Callable[[Namespace, IO], None]) -> Callable[[Namespace, IO], None]:\n \"\"\"Decorator used to register formatters.\n\n Args:\n func: Formatting function.\n\n Returns:\n The same function.\n \"\"\"\n formatters[func.__name__] = func\n return func\n\n\ndef setup_parser(subparser: ArgumentParser) -> None:\n \"\"\"Set up the argument parser.\n\n Args:\n subparser: Preliminary argument parser.\n \"\"\"\n subparser.add_argument(\n \"--update-completion\",\n action=\"store_true\",\n default=False,\n help=\"regenerate spack's tab completion scripts\",\n )\n\n subparser.add_argument(\n \"-a\", \"--aliases\", action=\"store_true\", default=False, help=\"include command aliases\"\n )\n subparser.add_argument(\n \"--format\",\n default=\"names\",\n choices=formatters,\n help=\"format to be used to print the output (default: names)\",\n )\n subparser.add_argument(\n \"--header\",\n metavar=\"FILE\",\n default=None,\n action=\"store\",\n help=\"prepend contents of FILE to the output (useful for rst format)\",\n )\n subparser.add_argument(\n \"--update\",\n metavar=\"FILE\",\n default=None,\n action=\"store\",\n help=\"write output to the specified file, if any command is newer\",\n )\n subparser.add_argument(\n \"rst_files\",\n nargs=argparse.REMAINDER,\n help=\"list of rst files to search for `_cmd-spack-` cross-refs\",\n )\n\n\nclass SpackArgparseRstWriter(ArgparseRstWriter):\n \"\"\"RST writer tailored for spack documentation.\"\"\"\n\n def __init__(\n self,\n prog: str,\n out: IO = sys.stdout,\n aliases: bool = False,\n documented_commands: Set[str] = set(),\n rst_levels: Sequence[str] = [\"-\", \"-\", \"^\", \"~\", \":\", \"`\"],\n ):\n \"\"\"Initialize a new SpackArgparseRstWriter instance.\n\n Args:\n prog: Program name.\n out: File object to write to.\n aliases: Whether or not to include subparsers for aliases.\n documented_commands: Set of commands with additional documentation.\n rst_levels: List of characters for rst section headings.\n \"\"\"\n super().__init__(prog, out, aliases, rst_levels)\n self.documented = documented_commands\n\n def usage(self, usage: str) -> str:\n \"\"\"Example usage of a command.\n\n Args:\n usage: Command usage.\n\n Returns:\n Usage of a command.\n \"\"\"\n string = super().usage(usage)\n\n cmd = self.parser.prog.replace(\" \", \"-\")\n if cmd in self.documented:\n string = f\"{string}\\n:ref:`More documentation `\\n\"\n\n return string\n\n\nclass SubcommandWriter(ArgparseWriter):\n \"\"\"Write argparse output as a list of subcommands.\"\"\"\n\n def format(self, cmd: Command) -> str:\n \"\"\"Return the string representation of a single node in the parser tree.\n\n Args:\n cmd: Parsed information about a command or subcommand.\n\n Returns:\n String representation of this subcommand.\n \"\"\"\n return \" \" * self.level + cmd.prog + \"\\n\"\n\n\n_positional_to_subroutine: Dict[str, str] = {\n \"package\": \"_all_packages\",\n \"spec\": \"_all_packages\",\n \"filter\": \"_all_packages\",\n \"installed\": \"_installed_packages\",\n \"compiler\": \"_installed_compilers\",\n \"section\": \"_config_sections\",\n \"env\": \"_environments\",\n \"extendable\": \"_extensions\",\n \"keys\": \"_keys\",\n \"help_command\": \"_subcommands\",\n \"mirror\": \"_mirrors\",\n \"virtual\": \"_providers\",\n \"namespace\": \"_repos\",\n \"hash\": \"_all_resource_hashes\",\n \"pytest\": \"_unit_tests\",\n}\n\n\nclass BashCompletionWriter(ArgparseWriter):\n \"\"\"Write argparse output as bash programmable tab completion.\"\"\"\n\n def format(self, cmd: Command) -> str:\n \"\"\"Return the string representation of a single node in the parser tree.\n\n Args:\n cmd: Parsed information about a command or subcommand.\n\n Returns:\n String representation of this subcommand.\n \"\"\"\n\n assert cmd.optionals # we should always at least have -h, --help\n assert not (cmd.positionals and cmd.subcommands) # one or the other\n\n # We only care about the arguments/flags, not the help messages\n positionals = cmd.positionals or ()\n optionals, _, _, _, _ = zip(*cmd.optionals)\n subcommands: Tuple[str, ...] = ()\n if cmd.subcommands:\n _, subcommands, _ = zip(*cmd.subcommands)\n\n # Flatten lists of lists\n optionals = [x for xx in optionals for x in xx]\n\n return (\n self.start_function(cmd.prog)\n + self.body(positionals, optionals, subcommands)\n + self.end_function(cmd.prog)\n )\n\n def start_function(self, prog: str) -> str:\n \"\"\"Return the syntax needed to begin a function definition.\n\n Args:\n prog: Program name.\n\n Returns:\n Function definition beginning.\n \"\"\"\n name = prog.replace(\"-\", \"_\").replace(\" \", \"_\")\n return \"\\n_{0}() {{\".format(name)\n\n def end_function(self, prog: str) -> str:\n \"\"\"Return the syntax needed to end a function definition.\n\n Args:\n prog: Program name\n\n Returns:\n Function definition ending.\n \"\"\"\n return \"}\\n\"\n\n def body(\n self, positionals: Sequence, optionals: Sequence[str], subcommands: Sequence[str]\n ) -> str:\n \"\"\"Return the body of the function.\n\n Args:\n positionals: List of positional argument tuples (name, choices, nargs, help).\n optionals: List of optional arguments.\n subcommands: List of subcommand parsers.\n\n Returns:\n Function body.\n \"\"\"\n if positionals:\n return f\"\"\"\n if $list_options\n then\n {self.optionals(optionals)}\n else\n {self.positionals(positionals)}\n fi\n\"\"\"\n elif subcommands:\n return f\"\"\"\n if $list_options\n then\n {self.optionals(optionals)}\n else\n {self.subcommands(subcommands)}\n fi\n\"\"\"\n else:\n return f\"\"\"\n {self.optionals(optionals)}\n\"\"\"\n\n def positionals(self, positionals: Sequence) -> str:\n \"\"\"Return the syntax for reporting positional arguments.\n\n Args:\n positionals: List of positional argument tuples (name, choices, nargs, help).\n\n Returns:\n Syntax for positional arguments.\n \"\"\"\n for name, choices, nargs, help in positionals:\n # Check for a predefined subroutine mapping\n for key, value in _positional_to_subroutine.items():\n if name.startswith(key):\n return value\n\n # Use choices if available\n if choices is not None:\n if isinstance(choices, dict):\n choices = sorted(choices.keys())\n elif isinstance(choices, (set, frozenset)):\n choices = sorted(choices)\n else:\n choices = sorted(choices)\n return 'SPACK_COMPREPLY=\"{}\"'.format(\" \".join(str(c) for c in choices))\n\n # If no matches found, return empty list\n return 'SPACK_COMPREPLY=\"\"'\n\n def optionals(self, optionals: Sequence[str]) -> str:\n \"\"\"Return the syntax for reporting optional flags.\n\n Args:\n optionals: List of optional arguments.\n\n Returns:\n Syntax for optional flags.\n \"\"\"\n return f'SPACK_COMPREPLY=\"{\" \".join(optionals)}\"'\n\n def subcommands(self, subcommands: Sequence[str]) -> str:\n \"\"\"Return the syntax for reporting subcommands.\n\n Args:\n subcommands: List of subcommand parsers.\n\n Returns:\n Syntax for subcommand parsers\n \"\"\"\n return f'SPACK_COMPREPLY=\"{\" \".join(subcommands)}\"'\n\n\n# Map argument destination names to their complete commands\n# Earlier items in the list have higher precedence\n_dest_to_fish_complete = {\n (\"activate\", \"view\"): \"-f -a '(__fish_complete_directories)'\",\n (\"bootstrap root\", \"path\"): \"-f -a '(__fish_complete_directories)'\",\n (\"mirror add\", \"mirror\"): \"-f\",\n (\"repo add\", \"path\"): \"-f -a '(__fish_complete_directories)'\",\n (\"test find\", \"filter\"): \"-f -a '(__fish_spack_tests)'\",\n (\"bootstrap\", \"name\"): \"-f -a '(__fish_spack_bootstrap_names)'\",\n (\"buildcache create\", \"key\"): \"-f -a '(__fish_spack_gpg_keys)'\",\n (\"build-env\", r\"spec \\[--\\].*\"): \"-f -a '(__fish_spack_build_env_spec)'\",\n (\"checksum\", \"package\"): \"-f -a '(__fish_spack_packages)'\",\n (\n \"checksum\",\n \"versions\",\n ): \"-f -a '(__fish_spack_package_versions $__fish_spack_argparse_argv[1])'\",\n (\"config\", \"path\"): \"-f -a '(__fish_spack_colon_path)'\",\n (\"config\", \"section\"): \"-f -a '(__fish_spack_config_sections)'\",\n (\"develop\", \"specs?\"): \"-f -k -a '(__fish_spack_specs_or_id)'\",\n (\"diff\", \"specs?\"): \"-f -a '(__fish_spack_installed_specs)'\",\n (\"gpg sign\", \"output\"): \"-f -a '(__fish_complete_directories)'\",\n (\"gpg\", \"keys?\"): \"-f -a '(__fish_spack_gpg_keys)'\",\n (\"graph\", \"specs?\"): \"-f -k -a '(__fish_spack_specs_or_id)'\",\n (\"help\", \"help_command\"): \"-f -a '(__fish_spack_commands)'\",\n (\"list\", \"filter\"): \"-f -a '(__fish_spack_packages)'\",\n (\"mirror\", \"mirror\"): \"-f -a '(__fish_spack_mirrors)'\",\n (\"pkg\", \"package\"): \"-f -a '(__fish_spack_pkg_packages)'\",\n (\"remove\", \"specs?\"): \"-f -a '(__fish_spack_installed_specs)'\",\n (\"repo\", \"namespace_or_path\"): \"$__fish_spack_force_files -a '(__fish_spack_repos)'\",\n (\"restage\", \"specs?\"): \"-f -k -a '(__fish_spack_specs_or_id)'\",\n (\"rm\", \"specs?\"): \"-f -a '(__fish_spack_installed_specs)'\",\n (\"solve\", \"specs?\"): \"-f -k -a '(__fish_spack_specs_or_id)'\",\n (\"spec\", \"specs?\"): \"-f -k -a '(__fish_spack_specs_or_id)'\",\n (\"stage\", \"specs?\"): \"-f -k -a '(__fish_spack_specs_or_id)'\",\n (\"test-env\", r\"spec \\[--\\].*\"): \"-f -a '(__fish_spack_build_env_spec)'\",\n (\"test\", r\"\\[?name.*\"): \"-f -a '(__fish_spack_tests)'\",\n (\"undevelop\", \"specs?\"): \"-f -k -a '(__fish_spack_specs_or_id)'\",\n (\"verify\", \"specs_or_files\"): \"$__fish_spack_force_files -a '(__fish_spack_installed_specs)'\",\n (\"view\", \"path\"): \"-f -a '(__fish_complete_directories)'\",\n (\"\", \"comment\"): \"-f\",\n (\"\", \"compiler_spec\"): \"-f -a '(__fish_spack_installed_compilers)'\",\n (\"\", \"config_scopes\"): \"-f -a '(__fish_complete_directories)'\",\n (\"\", \"extendable\"): \"-f -a '(__fish_spack_extensions)'\",\n (\"\", \"installed_specs?\"): \"-f -a '(__fish_spack_installed_specs)'\",\n (\"\", \"job_url\"): \"-f\",\n (\"\", \"location_env\"): \"-f -a '(__fish_complete_directories)'\",\n (\"\", \"pytest_args\"): \"-f -a '(__fish_spack_unit_tests)'\",\n (\"\", \"package_or_file\"): \"$__fish_spack_force_files -a '(__fish_spack_packages)'\",\n (\"\", \"package_or_user\"): \"-f -a '(__fish_spack_packages)'\",\n (\"\", \"package\"): \"-f -a '(__fish_spack_packages)'\",\n (\"\", \"PKG\"): \"-f -a '(__fish_spack_packages)'\",\n (\"\", \"prefix\"): \"-f -a '(__fish_complete_directories)'\",\n (\"\", r\"rev\\d?\"): \"-f -a '(__fish_spack_git_rev)'\",\n (\"\", \"specs?\"): \"-f -k -a '(__fish_spack_specs)'\",\n (\"\", \"tags?\"): \"-f -a '(__fish_spack_tags)'\",\n (\"\", \"virtual_package\"): \"-f -a '(__fish_spack_providers)'\",\n (\"\", \"working_dir\"): \"-f -a '(__fish_complete_directories)'\",\n (\"\", r\"(\\w*_)?env\"): \"-f -a '(__fish_spack_environments)'\",\n (\"\", r\"(\\w*_)?dir(ectory)?\"): \"-f -a '(__fish_spack_environments)'\",\n (\"\", r\"(\\w*_)?mirror_name\"): \"-f -a '(__fish_spack_mirrors)'\",\n}\n\n\ndef _fish_dest_get_complete(prog: str, dest: str) -> Optional[str]:\n \"\"\"Map from subcommand to autocompletion argument.\n\n Args:\n prog: Program name.\n dest: Destination.\n\n Returns:\n Autocompletion argument.\n \"\"\"\n s = prog.split(None, 1)\n subcmd = s[1] if len(s) == 2 else \"\"\n\n for (prog_key, pos_key), value in _dest_to_fish_complete.items():\n if subcmd.startswith(prog_key) and re.match(f\"^{pos_key}$\", dest):\n return value\n return None\n\n\nclass FishCompletionWriter(ArgparseWriter):\n \"\"\"Write argparse output as bash programmable tab completion.\"\"\"\n\n def format(self, cmd: Command) -> str:\n \"\"\"Return the string representation of a single node in the parser tree.\n\n Args:\n cmd: Parsed information about a command or subcommand.\n\n Returns:\n String representation of a node.\n \"\"\"\n assert cmd.optionals # we should always at least have -h, --help\n assert not (cmd.positionals and cmd.subcommands) # one or the other\n\n # We also need help messages and how arguments are used\n # So we pass everything to completion writer\n positionals = cmd.positionals\n optionals = cmd.optionals\n subcommands = cmd.subcommands\n\n return (\n self.prog_comment(cmd.prog)\n + self.optspecs(cmd.prog, optionals)\n + self.complete(cmd.prog, positionals, optionals, subcommands)\n )\n\n def optspecs(\n self,\n prog: str,\n optionals: List[Tuple[Sequence[str], List[str], str, Union[int, str, None], str]],\n ) -> str:\n \"\"\"Read the optionals and return the command to set optspec.\n\n Args:\n prog: Program name.\n optionals: List of optional arguments.\n\n Returns:\n Command to set optspec variable.\n \"\"\"\n # Variables of optspecs\n optspec_var = \"__fish_spack_optspecs_\" + prog.replace(\" \", \"_\").replace(\"-\", \"_\")\n\n if optionals is None:\n return f\"set -g {optspec_var}\\n\"\n\n # Build optspec by iterating over options\n args = []\n\n for flags, dest, _, nargs, _ in optionals:\n if len(flags) == 0:\n continue\n\n required = \"\"\n\n # Because nargs '?' is treated differently in fish, we treat it as required.\n # Because multi-argument options are not supported, we treat it like one argument.\n required = \"=\"\n if nargs == 0:\n required = \"\"\n\n # Pair short options with long options\n\n # We need to do this because fish doesn't support multiple short\n # or long options.\n # However, since we are paring options only, this is fine\n\n short = [f[1:] for f in flags if f.startswith(\"-\") and len(f) == 2]\n long = [f[2:] for f in flags if f.startswith(\"--\")]\n\n while len(short) > 0 and len(long) > 0:\n arg = f\"{short.pop()}/{long.pop()}{required}\"\n while len(short) > 0:\n arg = f\"{short.pop()}/{required}\"\n while len(long) > 0:\n arg = f\"{long.pop()}{required}\"\n\n args.append(arg)\n\n # Even if there is no option, we still set variable.\n # In fish such variable is an empty array, we use it to\n # indicate that such subcommand exists.\n args = \" \".join(args)\n\n return f\"set -g {optspec_var} {args}\\n\"\n\n @staticmethod\n def complete_head(\n prog: str, index: Optional[int] = None, nargs: Optional[Union[int, str]] = None\n ) -> str:\n \"\"\"Return the head of the completion command.\n\n Args:\n prog: Program name.\n index: Index of positional argument.\n nargs: Number of arguments.\n\n Returns:\n Head of the completion command.\n \"\"\"\n # Split command and subcommand\n s = prog.split(None, 1)\n subcmd = s[1] if len(s) == 2 else \"\"\n\n if index is None:\n return f\"complete -c {s[0]} -n '__fish_spack_using_command {subcmd}'\"\n elif nargs in [argparse.ZERO_OR_MORE, argparse.ONE_OR_MORE, argparse.REMAINDER]:\n return (\n f\"complete -c {s[0]} -n '__fish_spack_using_command_pos_remainder \"\n f\"{index} {subcmd}'\"\n )\n else:\n return f\"complete -c {s[0]} -n '__fish_spack_using_command_pos {index} {subcmd}'\"\n\n def complete(\n self,\n prog: str,\n positionals: List[Tuple[str, Optional[Iterable[Any]], Union[int, str, None], str]],\n optionals: List[Tuple[Sequence[str], List[str], str, Union[int, str, None], str]],\n subcommands: List[Tuple[ArgumentParser, str, str]],\n ) -> str:\n \"\"\"Return all the completion commands.\n\n Args:\n prog: Program name.\n positionals: List of positional arguments.\n optionals: List of optional arguments.\n subcommands: List of subcommand parsers.\n\n Returns:\n Completion command.\n \"\"\"\n commands = []\n\n if positionals:\n commands.append(self.positionals(prog, positionals))\n\n if subcommands:\n commands.append(self.subcommands(prog, subcommands))\n\n if optionals:\n commands.append(self.optionals(prog, optionals))\n\n return \"\".join(commands)\n\n def positionals(\n self,\n prog: str,\n positionals: List[Tuple[str, Optional[Iterable[Any]], Union[int, str, None], str]],\n ) -> str:\n \"\"\"Return the completion for positional arguments.\n\n Args:\n prog: Program name.\n positionals: List of positional arguments.\n\n Returns:\n Completion command.\n \"\"\"\n commands = []\n\n for idx, (args, choices, nargs, help) in enumerate(positionals):\n # Make sure we always get same order of output\n if isinstance(choices, dict):\n choices = sorted(choices.keys())\n elif isinstance(choices, (set, frozenset)):\n choices = sorted(choices)\n\n # Remove platform-specific choices to avoid hard-coding the platform.\n if choices is not None:\n valid_choices = []\n for choice in choices:\n if spack.platforms.host().name not in choice:\n valid_choices.append(choice)\n choices = valid_choices\n\n head = self.complete_head(prog, idx, nargs)\n\n if choices is not None:\n # If there are choices, we provide a completion for all possible values.\n commands.append(f\"{head} -f -a {shlex.quote(' '.join(choices))}\")\n else:\n # Otherwise, we try to find a predefined completion for it\n value = _fish_dest_get_complete(prog, args)\n if value is not None:\n commands.append(f\"{head} {value}\")\n\n return \"\\n\".join(commands) + \"\\n\"\n\n def prog_comment(self, prog: str) -> str:\n \"\"\"Return a comment line for the command.\"\"\"\n return f\"\\n# {prog}\\n\"\n\n def optionals(\n self,\n prog: str,\n optionals: List[Tuple[Sequence[str], List[str], str, Union[int, str, None], str]],\n ) -> str:\n \"\"\"Return the completion for optional arguments.\n\n Args:\n prog: Program name.\n optionals: List of optional arguments.\n\n Returns:\n Completion command.\n \"\"\"\n commands = []\n head = self.complete_head(prog)\n\n for flags, dest, _, nargs, help in optionals:\n # Make sure we always get same order of output\n if isinstance(dest, dict):\n dest = sorted(dest.keys())\n elif isinstance(dest, (set, frozenset)):\n dest = sorted(dest)\n\n # Remove platform-specific choices to avoid hard-coding the platform.\n if dest is not None:\n valid_choices = []\n for choice in dest:\n if spack.platforms.host().name not in choice:\n valid_choices.append(choice)\n dest = valid_choices\n\n # To provide description for optionals, and also possible values,\n # we need to use two split completion command.\n # Otherwise, each option will have same description.\n prefix = head\n\n # Add all flags to the completion\n for f in flags:\n if f.startswith(\"--\"):\n long = f[2:]\n prefix = f\"{prefix} -l {long}\"\n elif f.startswith(\"-\"):\n short = f[1:]\n assert len(short) == 1\n prefix = f\"{prefix} -s {short}\"\n\n # Check if option require argument.\n # Currently multi-argument options are not supported, so we treat it like one argument.\n if nargs != 0:\n prefix = f\"{prefix} -r\"\n\n if dest is not None:\n # If there are choices, we provide a completion for all possible values.\n commands.append(f\"{prefix} -f -a {shlex.quote(' '.join(dest))}\")\n else:\n # Otherwise, we try to find a predefined completion for it\n value = _fish_dest_get_complete(prog, dest)\n if value is not None:\n commands.append(f\"{prefix} {value}\")\n\n if help:\n commands.append(f\"{prefix} -d {shlex.quote(help)}\")\n\n return \"\\n\".join(commands) + \"\\n\"\n\n def subcommands(self, prog: str, subcommands: List[Tuple[ArgumentParser, str, str]]) -> str:\n \"\"\"Return the completion for subcommands.\n\n Args:\n prog: Program name.\n subcommands: List of subcommand parsers.\n\n Returns:\n Completion command.\n \"\"\"\n commands = []\n head = self.complete_head(prog, 0)\n\n for _, subcommand, help in subcommands:\n command = f\"{head} -f -a {shlex.quote(subcommand)}\"\n\n if help is not None and len(help) > 0:\n help = help.split(\"\\n\")[0]\n command = f\"{command} -d {shlex.quote(help)}\"\n\n commands.append(command)\n\n return \"\\n\".join(commands) + \"\\n\"\n\n\n@formatter\ndef subcommands(args: Namespace, out: IO) -> None:\n \"\"\"Hierarchical tree of subcommands.\n\n args:\n args: Command-line arguments.\n out: File object to write to.\n \"\"\"\n parser = get_all_spack_commands(out)\n writer = SubcommandWriter(parser.prog, out, args.aliases)\n writer.write(parser)\n\n\ndef rst_index(out: IO) -> None:\n \"\"\"Generate an index of all commands.\n\n Args:\n out: File object to write to.\n \"\"\"\n out.write(\"\\n\")\n\n index = spack.main.index_commands()\n sections = index[\"long\"]\n\n dmax = max(len(section_descriptions.get(s, s)) for s in sections) + 2\n cmax = max(len(c) for _, c in sections.items()) + 60\n\n row = \"%s %s\\n\" % (\"=\" * dmax, \"=\" * cmax)\n line = \"%%-%ds %%s\\n\" % dmax\n\n out.write(row)\n out.write(line % (\" Category \", \" Commands \"))\n out.write(row)\n for section, commands in sorted(sections.items()):\n description = section_descriptions.get(section, section)\n\n for i, cmd in enumerate(sorted(commands)):\n description = description.capitalize() if i == 0 else \"\"\n ref = f\":ref:`{cmd} `\"\n comma = \",\" if i != len(commands) - 1 else \"\"\n bar = \"| \" if i % 8 == 0 else \" \"\n out.write(line % (description, bar + ref + comma))\n out.write(row)\n\n\n@formatter\ndef rst(args: Namespace, out: IO) -> None:\n \"\"\"ReStructuredText documentation of subcommands.\n\n args:\n args: Command-line arguments.\n out: File object to write to.\n \"\"\"\n # create a parser with all commands\n parser = get_all_spack_commands(out)\n\n # extract cross-refs of the form `_cmd-spack-:` from rst files\n documented_commands: Set[str] = set()\n for filename in args.rst_files:\n with open(filename, encoding=\"utf-8\") as f:\n for line in f:\n match = re.match(r\"\\.\\. _cmd-(spack-.*):\", line)\n if match:\n documented_commands.add(match.group(1).strip())\n\n # print an index to each command\n rst_index(out)\n out.write(\"\\n\")\n\n # print sections for each command and subcommand\n writer = SpackArgparseRstWriter(parser.prog, out, args.aliases, documented_commands)\n writer.write(parser)\n\n\n@formatter\ndef names(args: Namespace, out: IO) -> None:\n \"\"\"Simple list of top-level commands.\n\n args:\n args: Command-line arguments.\n out: File object to write to.\n \"\"\"\n commands = copy.copy(spack.cmd.all_commands())\n\n if args.aliases:\n aliases = spack.config.get(\"config:aliases\")\n if aliases:\n commands.extend(aliases.keys())\n\n colify(commands, output=out)\n\n\ndef get_all_spack_commands(out: IO) -> SpackArgumentParser:\n is_tty = hasattr(out, \"isatty\") and out.isatty()\n # Argparse python 3.14 adds a default color argument that\n # adds color control characters to argparse output\n # that breaks expected output format from spack formatters\n # when written to non tty IO\n # If 3.14 and newer and not tty, disable color\n parser = spack.main.make_argument_parser(\n **({\"color\": False} if sys.version_info[:2] >= (3, 14) and not is_tty else {})\n )\n spack.main.add_all_commands(parser)\n return parser\n\n\n@formatter\ndef bash(args: Namespace, out: IO) -> None:\n \"\"\"Bash tab-completion script.\n\n args:\n args: Command-line arguments.\n out: File object to write to.\n \"\"\"\n parser = get_all_spack_commands(out)\n aliases_config = spack.config.get(\"config:aliases\")\n if aliases_config:\n aliases = \";\".join(f\"{key}:{val}\" for key, val in aliases_config.items())\n out.write(f'SPACK_ALIASES=\"{aliases}\"\\n\\n')\n\n writer = BashCompletionWriter(parser.prog, out, args.aliases)\n writer.write(parser)\n\n\n@formatter\ndef fish(args, out):\n parser = get_all_spack_commands(out)\n writer = FishCompletionWriter(parser.prog, out, args.aliases)\n writer.write(parser)\n\n\ndef prepend_header(args: Namespace, out: IO) -> None:\n \"\"\"Prepend header text at the beginning of a file.\n\n Args:\n args: Command-line arguments.\n out: File object to write to.\n \"\"\"\n if not args.header:\n return\n\n with open(args.header, encoding=\"utf-8\") as header:\n out.write(header.read())\n\n\ndef _commands(parser: ArgumentParser, args: Namespace) -> None:\n \"\"\"This is the 'regular' command, which can be called multiple times.\n\n See ``commands()`` below for ``--update-completion`` handling.\n\n Args:\n parser: Argument parser.\n args: Command-line arguments.\n \"\"\"\n formatter = formatters[args.format]\n\n # check header first so we don't open out files unnecessarily\n if args.header and not os.path.exists(args.header):\n tty.die(f\"No such file: '{args.header}'\")\n\n if args.update:\n tty.msg(f\"Updating file: {args.update}\")\n with open(args.update, \"w\", encoding=\"utf-8\") as f:\n prepend_header(args, f)\n formatter(args, f)\n\n else:\n prepend_header(args, sys.stdout)\n formatter(args, sys.stdout)\n\n\ndef update_completion(parser: ArgumentParser, args: Namespace) -> None:\n \"\"\"Iterate through the shells and update the standard completion files.\n\n This is a convenience method to avoid calling this command many\n times, and to simplify completion update for developers.\n\n Args:\n parser: Argument parser.\n args: Command-line arguments.\n \"\"\"\n for shell, shell_args in update_completion_args.items():\n for attr, value in shell_args.items():\n setattr(args, attr, value)\n _commands(parser, args)\n\n\ndef commands(parser: ArgumentParser, args: Namespace) -> None:\n \"\"\"Main function that calls formatter functions.\n\n Args:\n parser: Argument parser.\n args: Command-line arguments.\n \"\"\"\n if args.update_completion:\n if args.format != \"names\" or any([args.aliases, args.update, args.header]):\n tty.die(\"--update-completion can only be specified alone.\")\n\n # this runs the command multiple times with different arguments\n update_completion(parser, args)\n\n else:\n # run commands normally\n _commands(parser, args)\n" }, { "path": "lib/spack/spack/cmd/common/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport spack.llnl.util.tty as tty\nimport spack.llnl.util.tty.color as color\nimport spack.paths\n\n\ndef shell_init_instructions(cmd, equivalent):\n \"\"\"Print out instructions for users to initialize shell support.\n\n Arguments:\n cmd (str): the command the user tried to run that requires\n shell support in order to work\n equivalent (str): a command they can run instead, without\n enabling shell support\n \"\"\"\n\n shell_specific = \"{sh_arg}\" in equivalent\n\n msg = [\n \"`%s` requires Spack's shell support.\" % cmd,\n \"\",\n \"To set up shell support, run the command below for your shell.\",\n \"\",\n color.colorize(\"@*c{For bash/zsh/sh:}\"),\n \" . %s/setup-env.sh\" % spack.paths.share_path,\n \"\",\n color.colorize(\"@*c{For csh/tcsh:}\"),\n \" source %s/setup-env.csh\" % spack.paths.share_path,\n \"\",\n color.colorize(\"@*c{For fish:}\"),\n \" source %s/setup-env.fish\" % spack.paths.share_path,\n \"\",\n color.colorize(\"@*c{For Windows batch:}\"),\n \" %s\\\\spack_cmd.bat\" % spack.paths.bin_path,\n \"\",\n color.colorize(\"@*c{For PowerShell:}\"),\n \" %s\\\\setup-env.ps1\" % spack.paths.share_path,\n \"\",\n \"Or, if you do not want to use shell support, run \"\n + (\"one of these\" if shell_specific else \"this\")\n + \" instead:\",\n \"\",\n ]\n\n if shell_specific:\n msg += [\n equivalent.format(sh_arg=\"--sh \") + \" # bash/zsh/sh\",\n equivalent.format(sh_arg=\"--csh \") + \" # csh/tcsh\",\n equivalent.format(sh_arg=\"--fish\") + \" # fish\",\n equivalent.format(sh_arg=\"--bat \") + \" # batch\",\n equivalent.format(sh_arg=\"--pwsh\") + \" # powershell\",\n ]\n else:\n msg += [\" \" + equivalent]\n\n msg += [\n \"\",\n \"If you have already set up Spack's shell support but still receive\",\n \"this message, please make sure to call Spack via the `spack` command\",\n \"without any path components (such as `bin/spack`).\",\n ]\n\n msg += [\"\"]\n tty.error(*msg)\n" }, { "path": "lib/spack/spack/cmd/common/arguments.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport os\nimport textwrap\nfrom typing import Any, Optional\n\nimport spack.cmd\nimport spack.config\nimport spack.deptypes as dt\nimport spack.environment as ev\nimport spack.llnl.util.tty as tty\nimport spack.mirrors.mirror\nimport spack.mirrors.utils\nimport spack.reporters\nimport spack.spec\nimport spack.store\nfrom spack.llnl.util.lang import stable_partition\nfrom spack.util.pattern import Args\n\n__all__ = [\"add_common_arguments\"]\n\n#: dictionary of argument-generating functions, keyed by name\n_arguments = {}\n\n\ndef arg(fn):\n \"\"\"Decorator for a function that generates a common argument.\n\n This ensures that argument bunches are created lazily. Decorate\n argument-generating functions below with @arg so that\n ``add_common_arguments()`` can find them.\n\n \"\"\"\n _arguments[fn.__name__] = fn\n return fn\n\n\ndef add_common_arguments(parser, list_of_arguments):\n \"\"\"Extend a parser with extra arguments\n\n Args:\n parser: parser to be extended\n list_of_arguments: arguments to be added to the parser\n \"\"\"\n for argument in list_of_arguments:\n if argument not in _arguments:\n message = 'Trying to add non existing argument \"{0}\" to a command'\n raise KeyError(message.format(argument))\n\n x = _arguments[argument]()\n parser.add_argument(*x.flags, **x.kwargs)\n\n\nclass ConstraintAction(argparse.Action):\n \"\"\"Constructs a list of specs based on constraints from the command line\n\n An instance of this class is supposed to be used as an argument action\n in a parser. It will read a constraint and will attach a function to the\n arguments that accepts optional keyword arguments.\n\n To obtain the specs from a command the function must be called.\n \"\"\"\n\n def __call__(self, parser, namespace, values, option_string=None):\n # Query specs from command line\n self.constraint = namespace.constraint = values\n self.constraint_specs = namespace.constraint_specs = []\n namespace.specs = self._specs\n\n def _specs(self, **kwargs):\n # store parsed specs in spec.constraint after a call to specs()\n self.constraint_specs[:] = spack.cmd.parse_specs(self.constraint)\n\n # If an environment is provided, we'll restrict the search to\n # only its installed packages.\n env = ev.active_environment()\n if env:\n kwargs[\"hashes\"] = set(env.all_hashes())\n\n # return everything for an empty query.\n if not self.constraint_specs:\n return spack.store.STORE.db.query(**kwargs)\n\n # Return only matching stuff otherwise.\n specs = {}\n for spec in self.constraint_specs:\n for s in spack.store.STORE.db.query(spec, **kwargs):\n # This is fast for already-concrete specs\n specs[s.dag_hash()] = s\n\n return sorted(specs.values())\n\n\nclass SetParallelJobs(argparse.Action):\n \"\"\"Sets the correct value for parallel build jobs.\n\n The value is set in the command line configuration scope so that\n it can be retrieved using the spack.config API.\n \"\"\"\n\n def __call__(self, parser, namespace, jobs, option_string):\n # Jobs is a single integer, type conversion is already applied\n # see https://docs.python.org/3/library/argparse.html#action-classes\n if jobs < 1:\n msg = 'invalid value for argument \"{0}\" [expected a positive integer, got \"{1}\"]'\n raise ValueError(msg.format(option_string, jobs))\n\n spack.config.set(\"config:build_jobs\", jobs, scope=\"command_line\")\n\n setattr(namespace, \"jobs\", jobs)\n\n\nclass SetConcurrentPackages(argparse.Action):\n \"\"\"Sets the value for maximum number of concurrent package builds\n\n The value is set in the command line configuration scope so that\n it can be retrieved using the spack.config API.\n \"\"\"\n\n def __call__(self, parser, namespace, concurrent_packages, option_string):\n if concurrent_packages < 1:\n msg = 'invalid value for argument \"{0}\" [expected a positive integer, got \"{1}\"]'\n raise ValueError(msg.format(option_string, concurrent_packages))\n\n spack.config.set(\"config:concurrent_packages\", concurrent_packages, scope=\"command_line\")\n\n setattr(namespace, \"concurrent_packages\", concurrent_packages)\n\n\nclass DeprecatedStoreTrueAction(argparse.Action):\n \"\"\"Like the builtin store_true, but prints a deprecation warning.\"\"\"\n\n def __init__(\n self,\n option_strings,\n dest: str,\n default: Optional[Any] = False,\n required: bool = False,\n help: Optional[str] = None,\n removed_in: Optional[str] = None,\n instructions: Optional[str] = None,\n ):\n super().__init__(\n option_strings=option_strings,\n dest=dest,\n nargs=0,\n const=True,\n required=required,\n help=help,\n default=default,\n )\n self.removed_in = removed_in\n self.instructions = instructions\n\n def __call__(self, parser, namespace, value, option_string=None):\n instructions = [] if not self.instructions else [self.instructions]\n tty.warn(\n f\"{option_string} is deprecated and will be removed in {self.removed_in}.\",\n *instructions,\n )\n setattr(namespace, self.dest, self.const)\n\n\nclass DeptypeAction(argparse.Action):\n \"\"\"Creates a flag of valid dependency types from a deptype argument.\"\"\"\n\n def __call__(self, parser, namespace, values, option_string=None):\n if not values or values == \"all\":\n deptype = dt.ALL\n else:\n deptype = dt.canonicalize(values.split(\",\"))\n setattr(namespace, self.dest, deptype)\n\n\nclass ConfigScope(argparse.Action):\n \"\"\"Pick the currently configured config scopes.\"\"\"\n\n def __init__(self, *args, **kwargs) -> None:\n kwargs.setdefault(\"metavar\", spack.config.SCOPES_METAVAR)\n super().__init__(*args, **kwargs)\n\n @property\n def default(self):\n return self._default() if callable(self._default) else self._default\n\n @default.setter\n def default(self, value):\n self._default = value\n\n @property\n def choices(self):\n return spack.config.scopes().keys()\n\n @choices.setter\n def choices(self, value):\n pass\n\n def __call__(self, parser, namespace, values, option_string=None):\n setattr(namespace, self.dest, values)\n\n\ndef config_scope_readable_validator(value):\n if value not in spack.config.existing_scope_names():\n raise ValueError(\n f\"Invalid scope argument {value} \"\n \"for config read operation, scope context does not exist\"\n )\n return value\n\n\ndef _cdash_reporter(namespace):\n \"\"\"Helper function to create a CDash reporter. This function gets an early reference to the\n argparse namespace under construction, so it can later use it to create the object.\n \"\"\"\n\n def _factory():\n def installed_specs(args):\n packages = []\n\n if getattr(args, \"spec\", \"\"):\n packages = args.spec\n elif getattr(args, \"specs\", \"\"):\n packages = args.specs\n elif getattr(args, \"package\", \"\"):\n # Ensure CI 'spack test run' can output CDash results\n packages = args.package\n\n return [str(spack.spec.Spec(s)) for s in packages]\n\n configuration = spack.reporters.CDashConfiguration(\n upload_url=namespace.cdash_upload_url,\n packages=installed_specs(namespace),\n build=namespace.cdash_build,\n site=namespace.cdash_site,\n buildstamp=namespace.cdash_buildstamp,\n track=namespace.cdash_track,\n )\n\n return spack.reporters.CDash(configuration=configuration)\n\n return _factory\n\n\nclass CreateReporter(argparse.Action):\n \"\"\"Create the correct object to generate reports for installation and testing.\"\"\"\n\n def __call__(self, parser, namespace, values, option_string=None):\n setattr(namespace, self.dest, values)\n if values == \"junit\":\n setattr(namespace, \"reporter\", spack.reporters.JUnit)\n elif values == \"cdash\":\n setattr(namespace, \"reporter\", _cdash_reporter(namespace))\n\n\n@arg\ndef log_format():\n return Args(\n \"--log-format\",\n default=None,\n action=CreateReporter,\n choices=(\"junit\", \"cdash\"),\n help=\"format to be used for log files\",\n )\n\n\n# TODO: merge constraint and installed_specs\n@arg\ndef constraint():\n return Args(\n \"constraint\",\n nargs=argparse.REMAINDER,\n action=ConstraintAction,\n help=\"constraint to select a subset of installed packages\",\n metavar=\"installed_specs\",\n )\n\n\n@arg\ndef package():\n return Args(\"package\", help=\"package name\")\n\n\n@arg\ndef packages():\n return Args(\"packages\", nargs=\"+\", help=\"one or more package names\", metavar=\"package\")\n\n\n# Specs must use `nargs=argparse.REMAINDER` because a single spec can\n# contain spaces, and contain variants like '-mpi' that argparse thinks\n# are a collection of optional flags.\n@arg\ndef spec():\n return Args(\"spec\", nargs=argparse.REMAINDER, help=\"package spec\")\n\n\n@arg\ndef specs():\n return Args(\"specs\", nargs=argparse.REMAINDER, help=\"one or more package specs\")\n\n\n@arg\ndef installed_spec():\n return Args(\n \"spec\", nargs=argparse.REMAINDER, help=\"installed package spec\", metavar=\"installed_spec\"\n )\n\n\n@arg\ndef installed_specs():\n return Args(\n \"specs\",\n nargs=argparse.REMAINDER,\n help=\"one or more installed package specs\",\n metavar=\"installed_specs\",\n )\n\n\n@arg\ndef yes_to_all():\n return Args(\n \"-y\",\n \"--yes-to-all\",\n action=\"store_true\",\n dest=\"yes_to_all\",\n help='assume \"yes\" is the answer to every confirmation request',\n )\n\n\n@arg\ndef recurse_dependencies():\n return Args(\n \"-r\",\n \"--dependencies\",\n action=\"store_true\",\n dest=\"recurse_dependencies\",\n help=\"recursively traverse spec dependencies\",\n )\n\n\n@arg\ndef recurse_dependents():\n return Args(\n \"-R\",\n \"--dependents\",\n action=\"store_true\",\n dest=\"dependents\",\n help=\"also uninstall any packages that depend on the ones given via command line\",\n )\n\n\n@arg\ndef clean():\n return Args(\n \"--clean\",\n action=\"store_false\",\n default=spack.config.get(\"config:dirty\"),\n dest=\"dirty\",\n help=\"unset harmful variables in the build environment (default)\",\n )\n\n\n@arg\ndef deptype():\n return Args(\n \"--deptype\",\n action=DeptypeAction,\n default=dt.ALL,\n help=\"comma-separated list of deptypes to traverse (default=%s)\" % \",\".join(dt.ALL_TYPES),\n )\n\n\n@arg\ndef dirty():\n return Args(\n \"--dirty\",\n action=\"store_true\",\n default=spack.config.get(\"config:dirty\"),\n dest=\"dirty\",\n help=\"preserve user environment in spack's build environment (danger!)\",\n )\n\n\n@arg\ndef long():\n return Args(\n \"-l\", \"--long\", action=\"store_true\", help=\"show dependency hashes as well as versions\"\n )\n\n\n@arg\ndef very_long():\n return Args(\n \"-L\",\n \"--very-long\",\n action=\"store_true\",\n help=\"show full dependency hashes as well as versions\",\n )\n\n\n@arg\ndef tags():\n return Args(\n \"-t\",\n \"--tag\",\n action=\"append\",\n dest=\"tags\",\n metavar=\"TAG\",\n help=\"filter a package query by tag (multiple use allowed)\",\n )\n\n\n@arg\ndef namespaces():\n return Args(\n \"-N\",\n \"--namespaces\",\n action=\"store_true\",\n default=False,\n help=\"show fully qualified package names\",\n )\n\n\n@arg\ndef jobs():\n return Args(\n \"-j\",\n \"--jobs\",\n action=SetParallelJobs,\n type=int,\n dest=\"jobs\",\n help=\"explicitly set number of parallel jobs\",\n )\n\n\n@arg\ndef concurrent_packages():\n return Args(\n \"-p\",\n \"--concurrent-packages\",\n action=SetConcurrentPackages,\n type=int,\n default=None,\n help=\"maximum number of packages to build concurrently\",\n )\n\n\n@arg\ndef install_status():\n return Args(\n \"-I\",\n \"--install-status\",\n action=\"store_true\",\n default=True,\n help=(\n \"show install status of packages\\n\"\n \"[+] installed [^] installed in an upstream\\n\"\n \" - not installed [-] missing dep of installed package\\n\"\n ),\n )\n\n\n@arg\ndef no_install_status():\n return Args(\n \"--no-install-status\",\n dest=\"install_status\",\n action=\"store_false\",\n default=True,\n help=\"do not show install status annotations\",\n )\n\n\n@arg\ndef show_non_defaults():\n return Args(\n \"--non-defaults\",\n action=\"store_true\",\n default=False,\n help=\"highlight non-default versions or variants\",\n )\n\n\n@arg\ndef no_checksum():\n return Args(\n \"-n\",\n \"--no-checksum\",\n action=\"store_true\",\n default=False,\n help=\"do not use checksums to verify downloaded files (unsafe)\",\n )\n\n\n@arg\ndef deprecated():\n return Args(\n \"--deprecated\",\n action=\"store_true\",\n default=False,\n help=\"fetch deprecated versions without warning\",\n )\n\n\ndef add_cdash_args(subparser, add_help):\n cdash_help = {}\n if add_help:\n cdash_help[\"upload-url\"] = \"CDash URL where reports will be uploaded\"\n cdash_help[\"build\"] = (\n \"name of the build that will be reported to CDash\\n\\n\"\n \"defaults to spec of the package to operate on\"\n )\n cdash_help[\"site\"] = (\n \"site name that will be reported to CDash\\n\\ndefaults to current system hostname\"\n )\n cdash_help[\"track\"] = (\n \"results will be reported to this group on CDash\\n\\ndefaults to Experimental\"\n )\n cdash_help[\"buildstamp\"] = (\n \"use custom buildstamp\\n\\n\"\n \"instead of letting the CDash reporter prepare the \"\n \"buildstamp which, when combined with build name, site and project, \"\n \"uniquely identifies the build, provide this argument to identify \"\n \"the build yourself. format: %%Y%%m%%d-%%H%%M-[cdash-track]\"\n )\n else:\n cdash_help[\"upload-url\"] = argparse.SUPPRESS\n cdash_help[\"build\"] = argparse.SUPPRESS\n cdash_help[\"site\"] = argparse.SUPPRESS\n cdash_help[\"track\"] = argparse.SUPPRESS\n cdash_help[\"buildstamp\"] = argparse.SUPPRESS\n\n subparser.add_argument(\"--cdash-upload-url\", default=None, help=cdash_help[\"upload-url\"])\n subparser.add_argument(\"--cdash-build\", default=None, help=cdash_help[\"build\"])\n subparser.add_argument(\"--cdash-site\", default=None, help=cdash_help[\"site\"])\n\n cdash_subgroup = subparser.add_mutually_exclusive_group()\n cdash_subgroup.add_argument(\"--cdash-track\", default=\"Experimental\", help=cdash_help[\"track\"])\n cdash_subgroup.add_argument(\"--cdash-buildstamp\", default=None, help=cdash_help[\"buildstamp\"])\n\n\ndef print_cdash_help():\n parser = argparse.ArgumentParser(\n formatter_class=argparse.RawDescriptionHelpFormatter,\n epilog=textwrap.dedent(\n \"\"\"\\\nenvironment variables:\nSPACK_CDASH_AUTH_TOKEN\n authentication token to present to CDash\n \"\"\"\n ),\n )\n add_cdash_args(parser, True)\n parser.print_help()\n\n\ndef sanitize_reporter_options(namespace: argparse.Namespace):\n \"\"\"Sanitize options that affect generation and configuration of reports, like\n CDash or JUnit.\n\n Args:\n namespace: options parsed from cli\n \"\"\"\n has_any_cdash_option = (\n namespace.cdash_upload_url or namespace.cdash_build or namespace.cdash_site\n )\n if namespace.log_format == \"junit\" and has_any_cdash_option:\n raise argparse.ArgumentTypeError(\"cannot pass any cdash option when --log-format=junit\")\n\n # If any CDash option is passed, assume --log-format=cdash is implied\n if namespace.log_format is None and has_any_cdash_option:\n namespace.log_format = \"cdash\"\n namespace.reporter = _cdash_reporter(namespace)\n\n\nclass ConfigSetAction(argparse.Action):\n \"\"\"Generic action for setting spack config options from CLI.\n\n This works like a ``store_const`` action but you can set the\n ``dest`` to some Spack configuration path (like ``concretizer:reuse``)\n and the ``const`` will be stored there using ``spack.config.set()``\n \"\"\"\n\n def __init__(\n self,\n option_strings,\n dest,\n const,\n default=None,\n required=False,\n help=None,\n metavar=None,\n require_environment=False,\n ):\n # save the config option we're supposed to set\n self.config_path = dest\n\n # save whether the option requires an active env\n self.require_environment = require_environment\n\n # destination is translated to a legal python identifier by\n # substituting '_' for ':'.\n dest = dest.replace(\":\", \"_\")\n\n super().__init__(\n option_strings=option_strings,\n dest=dest,\n nargs=0,\n const=const,\n default=default,\n required=required,\n help=help,\n )\n\n def __call__(self, parser, namespace, values, option_string):\n if self.require_environment and not ev.active_environment():\n raise argparse.ArgumentTypeError(\n f\"argument '{self.option_strings[-1]}' requires an environment\"\n )\n\n # Retrieve the name of the config option and set it to\n # the const from the constructor or a value from the CLI.\n # Note that this is only called if the argument is actually\n # specified on the command line.\n spack.config.set(self.config_path, self.const, scope=\"command_line\")\n\n\ndef add_concretizer_args(subparser):\n \"\"\"Add a subgroup of arguments for controlling concretization.\n\n These will appear in a separate group called 'concretizer arguments'.\n There's no need to handle them in your command logic -- they all use\n ``ConfigSetAction``, which automatically handles setting configuration\n options.\n\n If you *do* need to access a value passed on the command line, you can\n get at, e.g., the ``concretizer:reuse`` via ``args.concretizer_reuse``.\n Just substitute ``_`` for ``:``.\n \"\"\"\n subgroup = subparser.add_argument_group(\"concretizer arguments\")\n subgroup.add_argument(\n \"-f\",\n \"--force\",\n action=ConfigSetAction,\n require_environment=True,\n dest=\"concretizer:force\",\n const=True,\n default=False,\n help=\"allow changes to concretized specs in spack.lock (in an env)\",\n )\n subgroup.add_argument(\n \"-U\",\n \"--fresh\",\n action=ConfigSetAction,\n dest=\"concretizer:reuse\",\n const=False,\n default=None,\n help=\"do not reuse installed deps; build newest configuration\",\n )\n subgroup.add_argument(\n \"--reuse\",\n action=ConfigSetAction,\n dest=\"concretizer:reuse\",\n const=True,\n default=None,\n help=\"reuse installed packages/buildcaches when possible\",\n )\n subgroup.add_argument(\n \"--fresh-roots\",\n \"--reuse-deps\",\n action=ConfigSetAction,\n dest=\"concretizer:reuse\",\n const=\"dependencies\",\n default=None,\n help=\"concretize with fresh roots and reused dependencies\",\n )\n subgroup.add_argument(\n \"--deprecated\",\n action=ConfigSetAction,\n dest=\"config:deprecated\",\n const=True,\n default=None,\n help=\"allow concretizer to select deprecated versions\",\n )\n\n\ndef add_connection_args(subparser, add_help):\n def add_argument_string_or_variable(parser, arg: str, *, deprecate_str: bool = True, **kwargs):\n group = parser.add_mutually_exclusive_group()\n group.add_argument(arg, **kwargs)\n # Update help string\n if \"help\" in kwargs:\n kwargs[\"help\"] = \"environment variable containing \" + kwargs[\"help\"]\n group.add_argument(arg + \"-variable\", **kwargs)\n\n s3_connection_parser = subparser.add_argument_group(\"S3 Connection\")\n\n add_argument_string_or_variable(\n s3_connection_parser,\n \"--s3-access-key-id\",\n help=\"ID string to use to connect to this S3 mirror\",\n )\n s3_connection_parser.add_argument(\n \"--s3-access-key-secret-variable\",\n help=\"environment variable containing secret string to use to connect to this S3 mirror\",\n )\n s3_connection_parser.add_argument(\n \"--s3-access-token-variable\",\n help=\"environment variable containing access token to use to connect to this S3 mirror\",\n )\n s3_connection_parser.add_argument(\n \"--s3-profile\", help=\"S3 profile name to use to connect to this S3 mirror\", default=None\n )\n s3_connection_parser.add_argument(\n \"--s3-endpoint-url\", help=\"endpoint URL to use to connect to this S3 mirror\"\n )\n\n oci_connection_parser = subparser.add_argument_group(\"OCI Connection\")\n\n add_argument_string_or_variable(\n oci_connection_parser,\n \"--oci-username\",\n deprecate_str=False,\n help=\"username to use to connect to this OCI mirror\",\n )\n oci_connection_parser.add_argument(\n \"--oci-password-variable\",\n help=\"environment variable containing password to use to connect to this OCI mirror\",\n )\n\n\ndef use_buildcache(cli_arg_value):\n \"\"\"Translate buildcache related command line arguments into a pair of strings,\n representing whether the root or its dependencies can use buildcaches.\n\n Argument type that accepts comma-separated subargs:\n\n 1. auto|only|never\n 2. package:auto|only|never\n 3. dependencies:auto|only|never\n\n Args:\n cli_arg_value (str): command line argument value to be translated\n\n Return:\n Tuple of two strings\n \"\"\"\n valid_keys = frozenset([\"package\", \"dependencies\"])\n valid_values = frozenset([\"only\", \"never\", \"auto\"])\n\n # Split in args, split in key/value, and trim whitespace\n args = [tuple(map(lambda x: x.strip(), part.split(\":\"))) for part in cli_arg_value.split(\",\")]\n\n # Verify keys and values\n def is_valid(arg):\n if len(arg) == 1:\n return arg[0] in valid_values\n if len(arg) == 2:\n return arg[0] in valid_keys and arg[1] in valid_values\n return False\n\n valid, invalid = stable_partition(args, is_valid)\n\n # print first error\n if invalid:\n raise argparse.ArgumentTypeError(\"invalid argument `{}`\".format(\":\".join(invalid[0])))\n\n # Default values\n package = \"auto\"\n dependencies = \"auto\"\n\n # Override in order.\n for arg in valid:\n if len(arg) == 1:\n package = dependencies = arg[0]\n continue\n key, val = arg\n if key == \"package\":\n package = val\n else:\n dependencies = val\n\n return package, dependencies\n\n\ndef mirror_name_or_url(m):\n # Look up mirror by name or use anonymous mirror with path/url.\n # We want to guard against typos in mirror names, to avoid pushing\n # accidentally to a dir in the current working directory.\n\n # If there's a \\ or / in the name, it's interpreted as a path or url.\n if \"/\" in m or \"\\\\\" in m or m in (\".\", \"..\"):\n return spack.mirrors.mirror.Mirror(m)\n\n # Otherwise, the named mirror is required to exist.\n try:\n return spack.mirrors.utils.require_mirror_name(m)\n except ValueError as e:\n raise argparse.ArgumentTypeError(f\"{e}. Did you mean {os.path.join('.', m)}?\") from e\n\n\ndef mirror_url(url):\n try:\n return spack.mirrors.mirror.Mirror.from_url(url)\n except ValueError as e:\n raise argparse.ArgumentTypeError(str(e)) from e\n\n\ndef mirror_directory(path):\n try:\n return spack.mirrors.mirror.Mirror.from_local_path(path)\n except ValueError as e:\n raise argparse.ArgumentTypeError(str(e)) from e\n\n\ndef mirror_name(name):\n try:\n return spack.mirrors.utils.require_mirror_name(name)\n except ValueError as e:\n raise argparse.ArgumentTypeError(str(e)) from e\n" }, { "path": "lib/spack/spack/cmd/common/confirmation.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport sys\nfrom typing import List\n\nimport spack.cmd\nimport spack.llnl.util.tty as tty\nimport spack.spec\n\ndisplay_args = {\"long\": True, \"show_flags\": False, \"variants\": False, \"indent\": 4}\n\n\ndef confirm_action(specs: List[spack.spec.Spec], participle: str, noun: str):\n \"\"\"Display the list of specs to be acted on and ask for confirmation.\n\n Args:\n specs: specs to be removed\n participle: action expressed as a participle, e.g. \"uninstalled\"\n noun: action expressed as a noun, e.g. \"uninstallation\"\n \"\"\"\n spack.cmd.display_specs(specs, **display_args)\n print()\n answer = tty.get_yes_or_no(\n f\"{len(specs)} packages will be {participle}. Do you want to proceed?\", default=False\n )\n if not answer:\n tty.msg(f\"Aborting {noun}\")\n sys.exit(0)\n" }, { "path": "lib/spack/spack/cmd/common/env_utility.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport argparse\nimport os\n\nimport spack.cmd\nimport spack.deptypes as dt\nimport spack.error\nimport spack.llnl.util.tty as tty\nimport spack.spec\nimport spack.store\nfrom spack import build_environment, traverse\nfrom spack.cmd.common import arguments\nfrom spack.context import Context\nfrom spack.util.environment import dump_environment, pickle_environment\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n arguments.add_common_arguments(subparser, [\"clean\", \"dirty\"])\n arguments.add_concretizer_args(subparser)\n\n subparser.add_argument(\"--dump\", metavar=\"FILE\", help=\"dump a source-able environment to FILE\")\n subparser.add_argument(\n \"--pickle\", metavar=\"FILE\", help=\"dump a pickled source-able environment to FILE\"\n )\n subparser.add_argument(\n \"spec\",\n nargs=argparse.REMAINDER,\n metavar=\"spec [--] [cmd]...\",\n help=\"specs of package environment to emulate\",\n )\n subparser.epilog = (\n \"If a command is not specified, the environment will be printed \"\n \"to standard output (cf /usr/bin/env) unless --dump and/or --pickle \"\n \"are specified.\\n\\nIf a command is specified and spec is \"\n \"multi-word, then the -- separator is obligatory.\"\n )\n\n\nclass AreDepsInstalledVisitor:\n def __init__(self, context: Context = Context.BUILD):\n if context == Context.BUILD:\n # TODO: run deps shouldn't be required for build env.\n self.direct_deps = dt.BUILD | dt.LINK | dt.RUN\n elif context == Context.TEST:\n self.direct_deps = dt.BUILD | dt.TEST | dt.LINK | dt.RUN\n else:\n raise ValueError(\"context can only be Context.BUILD or Context.TEST\")\n\n self.has_uninstalled_deps = False\n\n def accept(self, item):\n # The root may be installed or uninstalled.\n if item.depth == 0:\n return True\n\n # Early exit after we've seen an uninstalled dep.\n if self.has_uninstalled_deps:\n return False\n\n spec = item.edge.spec\n if not spec.external and not spec.installed:\n self.has_uninstalled_deps = True\n return False\n\n return True\n\n def neighbors(self, item):\n # Direct deps: follow build & test edges.\n # Transitive deps: follow link / run.\n depflag = self.direct_deps if item.depth == 0 else dt.LINK | dt.RUN\n return item.edge.spec.edges_to_dependencies(depflag=depflag)\n\n\ndef emulate_env_utility(cmd_name, context: Context, args):\n if not args.spec:\n tty.die(\"spack %s requires a spec.\" % cmd_name)\n\n # Specs may have spaces in them, so if they do, require that the\n # caller put a '--' between the spec and the command to be\n # executed. If there is no '--', assume that the spec is the\n # first argument.\n sep = \"--\"\n if sep in args.spec:\n s = args.spec.index(sep)\n spec = args.spec[:s]\n cmd = args.spec[s + 1 :]\n else:\n spec = args.spec[0]\n cmd = args.spec[1:]\n\n if not spec:\n tty.die(\"spack %s requires a spec.\" % cmd_name)\n\n specs = spack.cmd.parse_specs(spec, concretize=False)\n if len(specs) > 1:\n tty.die(\"spack %s only takes one spec.\" % cmd_name)\n spec = specs[0]\n\n spec = spack.cmd.matching_spec_from_env(spec)\n\n # Require that dependencies are installed.\n visitor = AreDepsInstalledVisitor(context=context)\n\n # Mass install check needs read transaction.\n with spack.store.STORE.db.read_transaction():\n traverse.traverse_breadth_first_with_visitor([spec], traverse.CoverNodesVisitor(visitor))\n\n if visitor.has_uninstalled_deps:\n raise spack.error.SpackError(\n f\"Not all dependencies of {spec.name} are installed. \"\n f\"Cannot setup {context} environment:\",\n spec.tree(\n status_fn=spack.spec.Spec.install_status,\n hashlen=7,\n hashes=True,\n # This shows more than necessary, but we cannot dynamically change deptypes\n # in Spec.tree(...).\n deptypes=\"all\" if context == Context.BUILD else (\"build\", \"test\", \"link\", \"run\"),\n ),\n )\n\n build_environment.setup_package(spec.package, args.dirty, context)\n\n if args.dump:\n # Dump a source-able environment to a text file.\n tty.msg(\"Dumping a source-able environment to {0}\".format(args.dump))\n dump_environment(args.dump)\n\n if args.pickle:\n # Dump a source-able environment to a pickle file.\n tty.msg(\"Pickling a source-able environment to {0}\".format(args.pickle))\n pickle_environment(args.pickle)\n\n if cmd:\n # Execute the command with the new environment\n os.execvp(cmd[0], cmd)\n\n elif not bool(args.pickle or args.dump):\n # If no command or dump/pickle option then act like the \"env\" command\n # and print out env vars.\n for key, val in os.environ.items():\n print(\"%s=%s\" % (key, val))\n" }, { "path": "lib/spack/spack/cmd/common/spec_strings.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport ast\nimport os\nimport re\nimport sys\nimport warnings\nfrom typing import Callable, List, Optional\n\nimport spack.llnl.util.tty as tty\nimport spack.util.spack_yaml\nfrom spack.spec_parser import NAME, VERSION_LIST, SpecTokens\nfrom spack.tokenize import Token, TokenBase, Tokenizer\n\nIS_PROBABLY_COMPILER = re.compile(r\"%[a-zA-Z_][a-zA-Z0-9\\-]\")\n\n\nclass _LegacySpecTokens(TokenBase):\n \"\"\"Reconstructs the tokens for previous specs, so we can reuse code to rotate them\"\"\"\n\n # Dependency\n START_EDGE_PROPERTIES = r\"(?:\\^\\[)\"\n END_EDGE_PROPERTIES = r\"(?:\\])\"\n DEPENDENCY = r\"(?:\\^)\"\n # Version\n VERSION_HASH_PAIR = SpecTokens.VERSION_HASH_PAIR.regex\n GIT_VERSION = SpecTokens.GIT_VERSION.regex\n VERSION = SpecTokens.VERSION.regex\n # Variants\n PROPAGATED_BOOL_VARIANT = SpecTokens.PROPAGATED_BOOL_VARIANT.regex\n BOOL_VARIANT = SpecTokens.BOOL_VARIANT.regex\n PROPAGATED_KEY_VALUE_PAIR = SpecTokens.PROPAGATED_KEY_VALUE_PAIR.regex\n KEY_VALUE_PAIR = SpecTokens.KEY_VALUE_PAIR.regex\n # Compilers\n COMPILER_AND_VERSION = rf\"(?:%\\s*(?:{NAME})(?:[\\s]*)@\\s*(?:{VERSION_LIST}))\"\n COMPILER = rf\"(?:%\\s*(?:{NAME}))\"\n # FILENAME\n FILENAME = SpecTokens.FILENAME.regex\n # Package name\n FULLY_QUALIFIED_PACKAGE_NAME = SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME.regex\n UNQUALIFIED_PACKAGE_NAME = SpecTokens.UNQUALIFIED_PACKAGE_NAME.regex\n # DAG hash\n DAG_HASH = SpecTokens.DAG_HASH.regex\n # White spaces\n WS = SpecTokens.WS.regex\n # Unexpected character(s)\n UNEXPECTED = SpecTokens.UNEXPECTED.regex\n\n\ndef _spec_str_reorder_compiler(idx: int, blocks: List[List[Token]]) -> None:\n # only move the compiler to the back if it exists and is not already at the end\n if not 0 <= idx < len(blocks) - 1:\n return\n # if there's only whitespace after the compiler, don't move it\n if all(token.kind == _LegacySpecTokens.WS for block in blocks[idx + 1 :] for token in block):\n return\n # rotate left and always add at least one WS token between compiler and previous token\n compiler_block = blocks.pop(idx)\n if compiler_block[0].kind != _LegacySpecTokens.WS:\n compiler_block.insert(0, Token(_LegacySpecTokens.WS, \" \"))\n # delete the WS tokens from the new first block if it was at the very start, to prevent leading\n # WS tokens.\n while idx == 0 and blocks[0][0].kind == _LegacySpecTokens.WS:\n blocks[0].pop(0)\n blocks.append(compiler_block)\n\n\ndef _spec_str_format(spec_str: str) -> Optional[str]:\n \"\"\"Given any string, try to parse as spec string, and rotate the compiler token to the end\n of each spec instance. Returns the formatted string if it was changed, otherwise None.\"\"\"\n # We parse blocks of tokens that include leading whitespace, and move the compiler block to\n # the end when we hit a dependency ^... or the end of a string.\n # [@3.1][ +foo][ +bar][ %gcc@3.1][ +baz]\n # [@3.1][ +foo][ +bar][ +baz][ %gcc@3.1]\n\n current_block: List[Token] = []\n blocks: List[List[Token]] = []\n compiler_block_idx = -1\n in_edge_attr = False\n\n legacy_tokenizer = Tokenizer(_LegacySpecTokens)\n\n for token in legacy_tokenizer.tokenize(spec_str):\n if token.kind == _LegacySpecTokens.UNEXPECTED:\n # parsing error, we cannot fix this string.\n return None\n elif token.kind in (_LegacySpecTokens.COMPILER, _LegacySpecTokens.COMPILER_AND_VERSION):\n # multiple compilers are not supported in Spack v0.x, so early return\n if compiler_block_idx != -1:\n return None\n current_block.append(token)\n blocks.append(current_block)\n current_block = []\n compiler_block_idx = len(blocks) - 1\n elif token.kind in (\n _LegacySpecTokens.START_EDGE_PROPERTIES,\n _LegacySpecTokens.DEPENDENCY,\n _LegacySpecTokens.UNQUALIFIED_PACKAGE_NAME,\n _LegacySpecTokens.FULLY_QUALIFIED_PACKAGE_NAME,\n ):\n _spec_str_reorder_compiler(compiler_block_idx, blocks)\n compiler_block_idx = -1\n if token.kind == _LegacySpecTokens.START_EDGE_PROPERTIES:\n in_edge_attr = True\n current_block.append(token)\n blocks.append(current_block)\n current_block = []\n elif token.kind == _LegacySpecTokens.END_EDGE_PROPERTIES:\n in_edge_attr = False\n current_block.append(token)\n blocks.append(current_block)\n current_block = []\n elif in_edge_attr:\n current_block.append(token)\n elif token.kind in (\n _LegacySpecTokens.VERSION_HASH_PAIR,\n _LegacySpecTokens.GIT_VERSION,\n _LegacySpecTokens.VERSION,\n _LegacySpecTokens.PROPAGATED_BOOL_VARIANT,\n _LegacySpecTokens.BOOL_VARIANT,\n _LegacySpecTokens.PROPAGATED_KEY_VALUE_PAIR,\n _LegacySpecTokens.KEY_VALUE_PAIR,\n _LegacySpecTokens.DAG_HASH,\n ):\n current_block.append(token)\n blocks.append(current_block)\n current_block = []\n elif token.kind == _LegacySpecTokens.WS:\n current_block.append(token)\n else:\n raise ValueError(f\"unexpected token {token}\")\n\n if current_block:\n blocks.append(current_block)\n _spec_str_reorder_compiler(compiler_block_idx, blocks)\n\n new_spec_str = \"\".join(token.value for block in blocks for token in block)\n return new_spec_str if spec_str != new_spec_str else None\n\n\nSpecStrHandler = Callable[[str, int, int, str, str], None]\n\n\ndef _spec_str_default_handler(path: str, line: int, col: int, old: str, new: str):\n \"\"\"A SpecStrHandler that prints formatted spec strings and their locations.\"\"\"\n print(f\"{path}:{line}:{col}: `{old}` -> `{new}`\")\n\n\ndef _spec_str_fix_handler(path: str, line: int, col: int, old: str, new: str):\n \"\"\"A SpecStrHandler that updates formatted spec strings in files.\"\"\"\n with open(path, \"r\", encoding=\"utf-8\") as f:\n lines = f.readlines()\n new_line = lines[line - 1].replace(old, new)\n if new_line == lines[line - 1]:\n tty.warn(f\"{path}:{line}:{col}: could not apply fix: `{old}` -> `{new}`\")\n return\n lines[line - 1] = new_line\n print(f\"{path}:{line}:{col}: fixed `{old}` -> `{new}`\")\n with open(path, \"w\", encoding=\"utf-8\") as f:\n f.writelines(lines)\n\n\ndef _spec_str_ast(path: str, tree: ast.AST, handler: SpecStrHandler) -> None:\n \"\"\"Walk the AST of a Python file and apply handler to formatted spec strings.\"\"\"\n for node in ast.walk(tree):\n if sys.version_info >= (3, 8):\n if isinstance(node, ast.Constant) and isinstance(node.value, str):\n current_str = node.value\n else:\n continue\n elif isinstance(node, ast.Str):\n current_str = node.s\n else:\n continue\n if not IS_PROBABLY_COMPILER.search(current_str):\n continue\n new = _spec_str_format(current_str)\n if new is not None:\n handler(path, node.lineno, node.col_offset, current_str, new)\n\n\ndef _spec_str_json_and_yaml(path: str, data: dict, handler: SpecStrHandler) -> None:\n \"\"\"Walk a YAML or JSON data structure and apply handler to formatted spec strings.\"\"\"\n queue = [data]\n seen = set()\n\n while queue:\n current = queue.pop(0)\n if id(current) in seen:\n continue\n seen.add(id(current))\n if isinstance(current, dict):\n queue.extend(current.values())\n queue.extend(current.keys())\n elif isinstance(current, list):\n queue.extend(current)\n elif isinstance(current, str) and IS_PROBABLY_COMPILER.search(current):\n new = _spec_str_format(current)\n if new is not None:\n mark = getattr(current, \"_start_mark\", None)\n if mark:\n line, col = mark.line + 1, mark.column + 1\n else:\n line, col = 0, 0\n handler(path, line, col, current, new)\n\n\ndef _check_spec_strings(\n paths: List[str], handler: SpecStrHandler = _spec_str_default_handler\n) -> None:\n \"\"\"Open Python, JSON and YAML files, and format their string literals that look like spec\n strings. A handler is called for each formatting, which can be used to print or apply fixes.\"\"\"\n for path in paths:\n is_json_or_yaml = path.endswith(\".json\") or path.endswith(\".yaml\") or path.endswith(\".yml\")\n is_python = path.endswith(\".py\")\n if not is_json_or_yaml and not is_python:\n continue\n\n try:\n with open(path, \"r\", encoding=\"utf-8\") as f:\n # skip files that are likely too large to be user code or config\n if os.fstat(f.fileno()).st_size > 1024 * 1024:\n warnings.warn(f\"skipping {path}: too large.\")\n continue\n if is_json_or_yaml:\n _spec_str_json_and_yaml(path, spack.util.spack_yaml.load_config(f), handler)\n elif is_python:\n _spec_str_ast(path, ast.parse(f.read()), handler)\n except (OSError, spack.util.spack_yaml.SpackYAMLError, SyntaxError, ValueError):\n warnings.warn(f\"skipping {path}\")\n continue\n" }, { "path": "lib/spack/spack/cmd/compiler.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport sys\nfrom typing import List, Optional\n\nimport spack.binary_distribution\nimport spack.compilers.config\nimport spack.config\nimport spack.llnl.util.tty as tty\nimport spack.spec\nimport spack.store\nfrom spack.cmd.common import arguments\nfrom spack.llnl.util.lang import index_by\nfrom spack.llnl.util.tty.colify import colify\nfrom spack.llnl.util.tty.color import colorize\nfrom spack.spec import Spec\n\ndescription = \"manage compilers\"\nsection = \"config\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n sp = subparser.add_subparsers(metavar=\"SUBCOMMAND\", dest=\"compiler_command\")\n\n # Find\n find_parser = sp.add_parser(\n \"find\",\n aliases=[\"add\"],\n help=\"search the system for compilers to add to Spack configuration\",\n )\n find_parser.add_argument(\"add_paths\", nargs=argparse.REMAINDER)\n find_parser.add_argument(\n \"--scope\",\n action=arguments.ConfigScope,\n default=lambda: spack.config.default_modify_scope(\"packages\"),\n help=\"configuration scope to modify\",\n )\n arguments.add_common_arguments(find_parser, [\"jobs\"])\n\n # Remove\n remove_parser = sp.add_parser(\"remove\", aliases=[\"rm\"], help=\"remove compiler by spec\")\n remove_parser.add_argument(\n \"-a\", \"--all\", action=\"store_true\", help=\"remove ALL compilers that match spec\"\n )\n remove_parser.add_argument(\"compiler_spec\")\n remove_parser.add_argument(\n \"--scope\", action=arguments.ConfigScope, default=None, help=\"configuration scope to modify\"\n )\n\n # List\n list_parser = sp.add_parser(\"list\", aliases=[\"ls\"], help=\"list available compilers\")\n list_parser.add_argument(\n \"--scope\",\n action=arguments.ConfigScope,\n type=arguments.config_scope_readable_validator,\n help=\"configuration scope to read from\",\n )\n list_parser.add_argument(\n \"--remote\", action=\"store_true\", help=\"list also compilers from registered buildcaches\"\n )\n\n # Info\n info_parser = sp.add_parser(\"info\", help=\"show compiler paths\")\n info_parser.add_argument(\"compiler_spec\")\n info_parser.add_argument(\n \"--scope\",\n action=arguments.ConfigScope,\n type=arguments.config_scope_readable_validator,\n help=\"configuration scope to read from\",\n )\n info_parser.add_argument(\n \"--remote\", action=\"store_true\", help=\"list also compilers from registered buildcaches\"\n )\n\n\ndef compiler_find(args):\n \"\"\"Search either $PATH or a list of paths OR MODULES for compilers and\n add them to Spack's configuration.\n \"\"\"\n paths = args.add_paths or None\n new_compilers = spack.compilers.config.find_compilers(\n path_hints=paths, scope=args.scope, max_workers=args.jobs\n )\n if new_compilers:\n n = len(new_compilers)\n s = \"s\" if n > 1 else \"\"\n filename = spack.config.CONFIG.get_config_filename(args.scope, \"packages\")\n tty.msg(f\"Added {n:d} new compiler{s} to {filename}\")\n compiler_strs = sorted(f\"{spec.name}@{spec.versions}\" for spec in new_compilers)\n colify(reversed(compiler_strs), indent=4)\n else:\n tty.msg(\"Found no new compilers\")\n tty.msg(\"Compilers are defined in the following files:\")\n colify(spack.compilers.config.compiler_config_files(), indent=4)\n\n\ndef compiler_remove(args):\n remover = spack.compilers.config.CompilerRemover(spack.config.CONFIG)\n candidates = remover.mark_compilers(match=args.compiler_spec, scope=args.scope)\n if not candidates:\n tty.die(f\"No compiler matches '{args.compiler_spec}'\")\n\n compiler_strs = reversed(sorted(f\"{spec.name}@{spec.versions}\" for spec in candidates))\n\n if not args.all and len(candidates) > 1:\n tty.error(f\"multiple compilers match the spec '{args.compiler_spec}':\")\n print()\n colify(compiler_strs, indent=4)\n print()\n print(\n \"Either use a stricter spec to select only one, or use `spack compiler remove -a`\"\n \" to remove all of them.\"\n )\n sys.exit(1)\n\n remover.flush()\n tty.msg(\"The following compilers have been removed:\")\n print()\n colify(compiler_strs, indent=4)\n print()\n\n\ndef compiler_info(args):\n \"\"\"Print info about all compilers matching a spec.\"\"\"\n all_compilers = _all_available_compilers(scope=args.scope, remote=args.remote)\n query = spack.spec.Spec(args.compiler_spec)\n compilers = [x for x in all_compilers if x.satisfies(query)]\n\n if not compilers:\n tty.die(f\"No compilers match spec {query.cformat()}\")\n\n compilers.sort(key=lambda x: (not x.external, x.name, x.version))\n\n for c in compilers:\n exes = {\n cname: getattr(c.package, cname)\n for cname in (\"cc\", \"cxx\", \"fortran\")\n if hasattr(c.package, cname)\n }\n if not exes:\n tty.debug(\n f\"{__name__}: skipping {c.format()} from compiler list, \"\n f\"since it has no executables\"\n )\n continue\n\n print(f\"{c.tree(recurse_dependencies=False, status_fn=spack.spec.Spec.install_status)}\")\n print(f\" prefix: {c.prefix}\")\n print(\" compilers:\")\n for language, exe in exes.items():\n print(f\" {language}: {exe}\")\n\n extra_attributes = getattr(c, \"extra_attributes\", {})\n if \"flags\" in extra_attributes:\n print(\" flags:\")\n for flag, flag_value in extra_attributes[\"flags\"].items():\n print(f\" {flag} = {flag_value}\")\n if \"environment\" in extra_attributes:\n environment = extra_attributes[\"environment\"]\n if len(environment.get(\"set\", {})) != 0:\n print(\"\\tenvironment:\")\n print(\"\\t set:\")\n for key, value in environment[\"set\"].items():\n print(f\"\\t {key} = {value}\")\n if \"extra_rpaths\" in extra_attributes:\n print(\" extra rpaths:\")\n for extra_rpath in extra_attributes[\"extra_rpaths\"]:\n print(f\" {extra_rpath}\")\n if getattr(c, \"external_modules\", []):\n print(\" modules: \")\n for module in c.external_modules:\n print(f\" {module}\")\n print()\n\n\ndef compiler_list(args):\n compilers = _all_available_compilers(scope=args.scope, remote=args.remote)\n\n if not sys.stdout.isatty():\n for c in sorted(compilers): # type: ignore\n print(c.format(\"{name}@{version}\"))\n return\n\n # If there are no compilers in any scope, and we're outputting to a tty, give a\n # hint to the user.\n if len(compilers) == 0:\n msg = \"No compilers available\"\n if args.scope is None:\n msg += \". Run `spack compiler find` to autodetect compilers\"\n tty.msg(msg)\n return\n\n index = index_by(compilers, spack.compilers.config.name_os_target)\n\n tty.msg(\"Available compilers\")\n\n # For a container, take each element which does not evaluate to false and\n # convert it to a string. For elements which evaluate to False (e.g. None)\n # convert them to '' (in which case it still evaluates to False but is a\n # string type). Tuples produced by this are guaranteed to be comparable in\n # Python 3\n convert_str = lambda tuple_container: tuple(str(x) if x else \"\" for x in tuple_container)\n\n index_str_keys = list((convert_str(x), y) for x, y in index.items())\n ordered_sections = sorted(index_str_keys, key=lambda item: item[0])\n for i, (key, compilers) in enumerate(ordered_sections):\n if i >= 1:\n print()\n name, os, target = key\n os_str = os\n if target:\n os_str += f\"-{target}\"\n cname = f\"{spack.spec.COMPILER_COLOR}{{{name}}} {os_str}\"\n tty.hline(colorize(cname), char=\"-\")\n result = {\n colorize(c.install_status().value) + c.format(\"{name}@{version}\") for c in compilers\n }\n colify(reversed(sorted(result)))\n\n\ndef _all_available_compilers(scope: Optional[str], remote: bool) -> List[Spec]:\n supported_compilers = spack.compilers.config.supported_compilers()\n\n def _is_compiler(x):\n return x.name in supported_compilers and x.package.supported_languages and not x.external\n\n compilers_from_store = [x for x in spack.store.STORE.db.query() if _is_compiler(x)]\n compilers_from_yaml = spack.compilers.config.all_compilers(scope=scope, init_config=False)\n compilers = compilers_from_yaml + compilers_from_store\n\n if remote:\n compilers.extend(\n [x for x in spack.binary_distribution.update_cache_and_get_specs() if _is_compiler(x)]\n )\n return compilers\n\n\ndef compiler(parser, args):\n action = {\n \"add\": compiler_find,\n \"find\": compiler_find,\n \"remove\": compiler_remove,\n \"rm\": compiler_remove,\n \"info\": compiler_info,\n \"list\": compiler_list,\n \"ls\": compiler_list,\n }\n action[args.compiler_command](args)\n" }, { "path": "lib/spack/spack/cmd/compilers.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\n\nfrom spack.cmd.common import arguments\nfrom spack.cmd.compiler import compiler_list\n\ndescription = \"list available compilers\"\nsection = \"config\"\nlevel = \"short\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n subparser.add_argument(\n \"--scope\",\n action=arguments.ConfigScope,\n type=arguments.config_scope_readable_validator,\n help=\"configuration scope to read/modify\",\n )\n subparser.add_argument(\n \"--remote\", action=\"store_true\", help=\"list also compilers from registered buildcaches\"\n )\n\n\ndef compilers(parser, args):\n compiler_list(args)\n" }, { "path": "lib/spack/spack/cmd/concretize.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\n\nimport spack.cmd\nimport spack.cmd.common.arguments\nimport spack.environment as ev\nimport spack.llnl.util.tty as tty\nfrom spack.llnl.string import plural\n\ndescription = \"concretize an environment and write a lockfile\"\nsection = \"environments\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n subparser.add_argument(\n \"--test\",\n default=None,\n choices=[\"root\", \"all\"],\n help=\"concretize with test dependencies of only root packages or all packages\",\n )\n subparser.add_argument(\n \"-q\", \"--quiet\", action=\"store_true\", help=\"don't print concretized specs\"\n )\n\n spack.cmd.common.arguments.add_concretizer_args(subparser)\n spack.cmd.common.arguments.add_common_arguments(subparser, [\"jobs\", \"show_non_defaults\"])\n\n\ndef concretize(parser, args):\n env = spack.cmd.require_active_env(cmd_name=\"concretize\")\n\n if args.test == \"all\":\n tests = True\n elif args.test == \"root\":\n tests = [spec.name for spec in env.user_specs]\n else:\n tests = False\n\n with env.write_transaction():\n concretized_specs = env.concretize(tests=tests)\n if not args.quiet:\n if concretized_specs:\n tty.msg(f\"Concretized {plural(len(concretized_specs), 'spec')}:\")\n ev.display_specs(\n [concrete for _, concrete in concretized_specs],\n highlight_non_defaults=args.non_defaults,\n )\n else:\n tty.msg(\"No new specs to concretize.\")\n env.write()\n" }, { "path": "lib/spack/spack/cmd/config.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport argparse\nimport collections\nimport os\nimport shutil\nimport sys\nfrom typing import List\n\nimport spack.config\nimport spack.environment as ev\nimport spack.error\nimport spack.llnl.util.filesystem as fs\nimport spack.llnl.util.tty as tty\nimport spack.llnl.util.tty.color as color\nimport spack.schema\nimport spack.schema.env\nimport spack.spec\nimport spack.store\nimport spack.util.spack_json as sjson\nimport spack.util.spack_yaml as syaml\nfrom spack.cmd.common import arguments\nfrom spack.llnl.util.tty.colify import colify_table\nfrom spack.util.editor import editor\n\ndescription = \"get and set configuration options\"\nsection = \"config\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n # User can only choose one\n subparser.add_argument(\n \"--scope\", action=arguments.ConfigScope, help=\"configuration scope to read/modify\"\n )\n\n sp = subparser.add_subparsers(metavar=\"SUBCOMMAND\", dest=\"config_command\")\n\n get_parser = sp.add_parser(\"get\", help=\"print configuration values\")\n get_parser.add_argument(\n \"section\",\n help=\"configuration section to print\\n\\noptions: %(choices)s\",\n nargs=\"?\",\n metavar=\"section\",\n choices=spack.config.SECTION_SCHEMAS,\n )\n get_parser.add_argument(\"--json\", action=\"store_true\", help=\"output configuration as JSON\")\n get_parser.add_argument(\n \"--group\",\n metavar=\"group\",\n default=None,\n help=\"show configuration as seen by this environment spec group (requires active env)\",\n )\n\n blame_parser = sp.add_parser(\n \"blame\", help=\"print configuration annotated with source file:line\"\n )\n blame_parser.add_argument(\n \"section\",\n help=\"configuration section to print\\n\\noptions: %(choices)s\",\n nargs=\"?\",\n metavar=\"section\",\n choices=spack.config.SECTION_SCHEMAS,\n )\n blame_parser.add_argument(\n \"--group\",\n metavar=\"group\",\n default=None,\n help=\"show configuration as seen by this environment spec group (requires active env)\",\n )\n\n edit_parser = sp.add_parser(\"edit\", help=\"edit configuration file\")\n edit_parser.add_argument(\n \"section\",\n help=\"configuration section to edit\\n\\noptions: %(choices)s\",\n metavar=\"section\",\n nargs=\"?\",\n choices=spack.config.SECTION_SCHEMAS,\n )\n edit_parser.add_argument(\n \"--print-file\", action=\"store_true\", help=\"print the file name that would be edited\"\n )\n\n sp.add_parser(\"list\", help=\"list configuration sections\")\n\n scopes_parser = sp.add_parser(\n \"scopes\", help=\"list defined scopes in descending order of precedence\"\n )\n scopes_parser.add_argument(\n \"-p\",\n \"--paths\",\n action=\"store_true\",\n default=False,\n help=\"show associated paths for appropriate scopes\",\n )\n scopes_parser.add_argument(\n \"-t\",\n \"--type\",\n default=[\"all\"],\n metavar=\"scope-type\",\n nargs=\"+\",\n choices=(\"all\", \"env\", \"include\", \"internal\", \"path\"),\n help=\"list only scopes of the specified type(s)\\n\\noptions: %(choices)s\",\n )\n scopes_parser.add_argument(\n \"-v\",\n \"--verbose\",\n dest=\"scopes_verbose\", # spack has -v as well\n action=\"store_true\",\n default=False,\n help=\"show scope types and whether scopes are overridden\",\n )\n scopes_parser.add_argument(\n \"section\",\n help=\"tailor scope path information to the specified section (implies ``--paths``)\"\n \"\\n\\noptions: %(choices)s\",\n metavar=\"section\",\n nargs=\"?\",\n choices=spack.config.SECTION_SCHEMAS,\n )\n\n add_parser = sp.add_parser(\"add\", help=\"add configuration parameters\")\n add_parser.add_argument(\n \"path\",\n nargs=\"?\",\n help=\"colon-separated path to config that should be added, e.g. 'config:default:true'\",\n )\n add_parser.add_argument(\"-f\", \"--file\", help=\"file from which to set all config values\")\n\n change_parser = sp.add_parser(\"change\", help=\"swap variants etc. on specs in config\")\n change_parser.add_argument(\"path\", help=\"colon-separated path to config section with specs\")\n change_parser.add_argument(\"--match-spec\", help=\"only change constraints that match this\")\n\n prefer_upstream_parser = sp.add_parser(\n \"prefer-upstream\", help=\"set package preferences from upstream\"\n )\n\n prefer_upstream_parser.add_argument(\n \"--local\",\n action=\"store_true\",\n default=False,\n help=\"set packages preferences based on local installs, rather than upstream\",\n )\n\n remove_parser = sp.add_parser(\"remove\", aliases=[\"rm\"], help=\"remove configuration parameters\")\n remove_parser.add_argument(\n \"path\",\n help=\"colon-separated path to config that should be removed, e.g. 'config:default:true'\",\n )\n\n # Make the add parser available later\n setattr(setup_parser, \"add_parser\", add_parser)\n\n update = sp.add_parser(\"update\", help=\"update configuration files to the latest format\")\n arguments.add_common_arguments(update, [\"yes_to_all\"])\n update.add_argument(\"section\", help=\"section to update\")\n\n revert = sp.add_parser(\n \"revert\", help=\"revert configuration files to their state before update\"\n )\n arguments.add_common_arguments(revert, [\"yes_to_all\"])\n revert.add_argument(\"section\", help=\"section to update\")\n\n\ndef _get_scope_and_section(args):\n \"\"\"Extract config scope and section from arguments.\"\"\"\n scope = args.scope\n section = getattr(args, \"section\", None)\n path = getattr(args, \"path\", None)\n\n # w/no args and an active environment, point to env manifest\n if not section and not scope:\n env = ev.active_environment()\n if env:\n scope = env.scope_name\n\n # set scope defaults\n elif not scope:\n scope = spack.config.default_modify_scope(section)\n\n # special handling for commands that take value instead of section\n if path:\n section = path[: path.find(\":\")] if \":\" in path else path\n if not scope:\n scope = spack.config.default_modify_scope(section)\n\n return scope, section\n\n\ndef print_configuration(args, *, blame: bool) -> None:\n if args.scope and args.scope not in spack.config.existing_scope_names():\n tty.die(f\"the argument --scope={args.scope} must refer to an existing scope.\")\n if args.scope and args.section is None:\n tty.die(f\"the argument --scope={args.scope} requires specifying a section.\")\n\n group = getattr(args, \"group\", None)\n if group is not None:\n env = ev.active_environment()\n if env is None:\n tty.die(\"the argument --group requires an active environment\")\n try:\n with env.config_override_for_group(group=group):\n _print_configuration_helper(args, blame=blame)\n except ValueError as e:\n tty.die(str(e))\n return\n\n _print_configuration_helper(args, blame=blame)\n\n\ndef _print_configuration_helper(args, *, blame: bool) -> None:\n yaml = blame or not args.json\n\n if args.section is not None:\n spack.config.CONFIG.print_section(args.section, yaml=yaml, blame=blame, scope=args.scope)\n return\n\n print_flattened_configuration(blame=blame, yaml=yaml)\n\n\ndef print_flattened_configuration(*, blame: bool, yaml: bool) -> None:\n \"\"\"Prints to stdout a flattened version of the configuration.\n\n Args:\n blame: if True, shows file provenance for each entry in the configuration.\n \"\"\"\n env = ev.active_environment()\n if env is not None:\n pristine = env.manifest.yaml_content\n flattened = pristine.copy()\n flattened[spack.schema.env.TOP_LEVEL_KEY] = pristine[spack.schema.env.TOP_LEVEL_KEY].copy()\n else:\n flattened = syaml.syaml_dict()\n flattened[spack.schema.env.TOP_LEVEL_KEY] = syaml.syaml_dict()\n\n for config_section in spack.config.SECTION_SCHEMAS:\n current = spack.config.get(config_section)\n flattened[spack.schema.env.TOP_LEVEL_KEY][config_section] = current\n if blame or yaml:\n syaml.dump_config(flattened, stream=sys.stdout, default_flow_style=False, blame=blame)\n else:\n sjson.dump(flattened, sys.stdout)\n sys.stdout.write(\"\\n\")\n\n\ndef config_get(args):\n \"\"\"Dump merged YAML configuration for a specific section.\n\n With no arguments and an active environment, print the contents of\n the environment's manifest file (spack.yaml).\n \"\"\"\n print_configuration(args, blame=False)\n\n\ndef config_blame(args):\n \"\"\"Print out line-by-line blame of merged YAML.\"\"\"\n print_configuration(args, blame=True)\n\n\ndef config_edit(args):\n \"\"\"Edit the configuration file for a specific scope and config section.\n\n With no arguments and an active environment, edit the spack.yaml for\n the active environment.\n \"\"\"\n spack_env = os.environ.get(ev.spack_env_var)\n env_error = ev.environment._active_environment_error\n\n if env_error and args.scope:\n # Cannot use scopes beyond the environment itself with a failed environment\n raise env_error\n elif env_error:\n # The rest of the config system wasn't set up fully, but spack.main was allowed\n # to progress so the user can open the malformed environment file\n config_file = env_error.filename\n elif spack_env and not args.scope:\n # Don't use the scope object for envs, as `config edit` can be called\n # for a malformed environment. Use SPACK_ENV to find spack.yaml.\n config_file = ev.manifest_file(spack_env)\n else:\n # If we aren't editing a spack.yaml file, get config path from scope.\n scope, section = _get_scope_and_section(args)\n if not scope and not section:\n tty.die(\"`spack config edit` requires a section argument or an active environment.\")\n config_file = spack.config.CONFIG.get_config_filename(scope, section)\n\n if args.print_file:\n print(config_file)\n else:\n editor(config_file)\n\n\ndef config_list(args):\n \"\"\"List the possible configuration sections.\n\n Used primarily for shell tab completion scripts.\n \"\"\"\n print(\" \".join(list(spack.config.SECTION_SCHEMAS)))\n\n\ndef _config_scope_info(args, scope, active, included):\n result = [scope.name] # always print the name\n\n if args.scopes_verbose:\n result.append(\",\".join(_config_basic_scope_types(scope, included)))\n if scope.name not in active:\n scope_status = \"override\"\n elif args.section and not spack.config.CONFIG.get_config(args.section, scope=scope.name):\n scope_status = \"absent\"\n else:\n scope_status = \"active\"\n result.append(scope_status)\n\n section_path = None\n if args.section or args.paths:\n if hasattr(scope, \"path\"):\n section_path = scope.get_section_filename(args.section) if args.section else None\n result.append(\n section_path\n if section_path and os.path.exists(section_path)\n else f\"{scope.path}{'' if os.path.isfile(scope.path) else os.sep}\"\n )\n else:\n result.append(\" \")\n\n if args.scopes_verbose and scope_status in (\"absent\", \"override\"):\n result = [color.colorize(f\"@k{{{elt}}}\") for elt in result]\n\n return result\n\n\ndef _config_basic_scope_types(scope, included):\n types = []\n if isinstance(scope, spack.config.InternalConfigScope):\n types.append(\"internal\")\n if hasattr(scope, \"yaml_path\") and scope.yaml_path == [spack.schema.env.TOP_LEVEL_KEY]:\n types.append(\"env\")\n if hasattr(scope, \"path\"):\n types.append(\"path\")\n if scope.name in included:\n types.append(\"include\")\n return sorted(types)\n\n\ndef config_scopes(args):\n \"\"\"List configured scopes in descending order of precedence.\"\"\"\n included = list(i.name for s in spack.config.scopes().values() for i in s.included_scopes)\n active = [s.name for s in spack.config.CONFIG.active_scopes]\n scopes = [\n s\n for s in spack.config.scopes().reversed_values()\n if (\n \"include\" in args.type\n and s.name in included\n or any(i in (\"all\", *_config_basic_scope_types(s, included)) for i in args.type)\n )\n and (s.name in active or args.scopes_verbose)\n ]\n\n if scopes:\n headers = [\"Scope\"]\n if args.scopes_verbose:\n headers += [\"Type\", \"Status\"]\n if args.section or args.paths:\n headers += [\"Path\"]\n\n table = [_config_scope_info(args, s, active, included) for s in scopes]\n\n # add headers if we have > 1 column\n if len(headers) > 1:\n table = [[color.colorize(f\"@*C{{{colname}}}\") for colname in headers]] + table\n\n colify_table(table)\n\n\ndef config_add(args):\n \"\"\"Add the given configuration to the specified config scope\n\n This is a stateful operation that edits the config files.\"\"\"\n if not (args.file or args.path):\n tty.error(\"No changes requested. Specify a file or value.\")\n setup_parser.add_parser.print_help()\n exit(1)\n\n scope, section = _get_scope_and_section(args)\n\n if args.file:\n spack.config.add_from_file(args.file, scope=scope)\n\n if args.path:\n spack.config.add(args.path, scope=scope)\n\n\ndef config_remove(args):\n \"\"\"Remove the given configuration from the specified config scope\n\n This is a stateful operation that edits the config files.\"\"\"\n scope, _ = _get_scope_and_section(args)\n\n path, _, value = args.path.rpartition(\":\")\n existing = spack.config.get(path, scope=scope)\n\n if not isinstance(existing, (list, dict)):\n path, _, value = path.rpartition(\":\")\n existing = spack.config.get(path, scope=scope)\n\n value = syaml.load(value)\n\n if isinstance(existing, list):\n values = value if isinstance(value, list) else [value]\n for v in values:\n existing.remove(v)\n elif isinstance(existing, dict):\n existing.pop(value, None)\n else:\n # This should be impossible to reach\n raise spack.error.ConfigError(\"Config has nested non-dict values\")\n\n spack.config.set(path, existing, scope)\n\n\ndef _can_update_config_file(scope: spack.config.ConfigScope, cfg_file):\n if isinstance(scope, spack.config.SingleFileScope):\n return fs.can_access(cfg_file)\n elif isinstance(scope, spack.config.DirectoryConfigScope):\n return fs.can_write_to_dir(scope.path) and fs.can_access(cfg_file)\n return False\n\n\ndef _config_change_requires_scope(path, spec, scope, match_spec=None):\n \"\"\"Return whether or not anything changed.\"\"\"\n require = spack.config.get(path, scope=scope)\n if not require:\n return False\n\n changed = False\n\n def override_cfg_spec(spec_str):\n nonlocal changed\n\n init_spec = spack.spec.Spec(spec_str)\n # Overridden spec cannot be anonymous\n init_spec.name = spec.name\n if match_spec and not init_spec.satisfies(match_spec):\n # If there is a match_spec, don't change constraints that\n # don't match it\n return spec_str\n elif not init_spec.intersects(spec):\n changed = True\n return str(spack.spec.Spec.override(init_spec, spec))\n else:\n # Don't override things if they intersect, otherwise we'd\n # be e.g. attaching +debug to every single version spec\n return spec_str\n\n if isinstance(require, str):\n new_require = override_cfg_spec(require)\n else:\n new_require = []\n for item in require:\n if \"one_of\" in item:\n item[\"one_of\"] = [override_cfg_spec(x) for x in item[\"one_of\"]]\n elif \"any_of\" in item:\n item[\"any_of\"] = [override_cfg_spec(x) for x in item[\"any_of\"]]\n elif \"spec\" in item:\n item[\"spec\"] = override_cfg_spec(item[\"spec\"])\n elif isinstance(item, str):\n item = override_cfg_spec(item)\n else:\n raise ValueError(f\"Unexpected requirement: ({type(item)}) {str(item)}\")\n new_require.append(item)\n\n spack.config.set(path, new_require, scope=scope)\n return changed\n\n\ndef _config_change(config_path, match_spec_str=None):\n all_components = spack.config.process_config_path(config_path)\n key_components = all_components[:-1]\n key_path = \":\".join(key_components)\n\n spec = spack.spec.Spec(syaml.syaml_str(all_components[-1]))\n\n match_spec = None\n if match_spec_str:\n match_spec = spack.spec.Spec(match_spec_str)\n\n if key_components[-1] == \"require\":\n # Extract the package name from the config path, which allows\n # args.spec to be anonymous if desired\n pkg_name = key_components[1]\n spec.name = pkg_name\n\n changed = False\n for scope in spack.config.writable_scope_names():\n changed |= _config_change_requires_scope(key_path, spec, scope, match_spec=match_spec)\n\n if not changed:\n existing_requirements = spack.config.get(key_path)\n if isinstance(existing_requirements, str):\n raise spack.error.ConfigError(\n \"'config change' needs to append a requirement,\"\n \" but existing require: config is not a list\"\n )\n\n ideal_scope_to_modify = None\n for scope in spack.config.writable_scope_names():\n if spack.config.get(key_path, scope=scope):\n ideal_scope_to_modify = scope\n break\n # If we find our key in a specific scope, that's the one we want\n # to modify. Otherwise we use the default write scope.\n write_scope = ideal_scope_to_modify or spack.config.default_modify_scope()\n\n update_path = f\"{key_path}:[{str(spec)}]\"\n spack.config.add(update_path, scope=write_scope)\n else:\n raise ValueError(\"'config change' can currently only change 'require' sections\")\n\n\ndef config_change(args):\n _config_change(args.path, args.match_spec)\n\n\ndef config_update(args):\n # Read the configuration files\n spack.config.CONFIG.get_config(args.section, scope=args.scope)\n updates: List[spack.config.ConfigScope] = [\n x\n for x in spack.config.CONFIG.updated_scopes_by_section[args.section]\n if not isinstance(x, spack.config.InternalConfigScope) and x.writable\n ]\n\n cannot_overwrite, skip_system_scope = [], False\n for scope in updates:\n cfg_file = spack.config.CONFIG.get_config_filename(scope.name, args.section)\n can_be_updated = _can_update_config_file(scope, cfg_file)\n if not can_be_updated:\n if scope.name == \"system\":\n skip_system_scope = True\n tty.warn(\n 'Not enough permissions to write to \"system\" scope. '\n f\"Skipping update at that location [cfg={cfg_file}]\"\n )\n continue\n cannot_overwrite.append((scope, cfg_file))\n\n if cannot_overwrite:\n msg = \"Detected permission issues with the following scopes:\\n\\n\"\n for scope, cfg_file in cannot_overwrite:\n msg += \"\\t[scope={0}, cfg={1}]\\n\".format(scope.name, cfg_file)\n msg += (\n \"\\nEither ensure that you have sufficient permissions to \"\n \"modify these files or do not include these scopes in the \"\n \"update.\"\n )\n tty.die(msg)\n\n if skip_system_scope:\n updates = [x for x in updates if x.name != \"system\"]\n\n # Report if there are no updates to be done\n if not updates:\n msg = 'No updates needed for \"{0}\" section.'\n tty.msg(msg.format(args.section))\n return\n\n proceed = True\n if not args.yes_to_all:\n msg = (\n \"The following configuration files are going to be updated to\"\n \" the latest schema format:\\n\\n\"\n )\n for scope in updates:\n cfg_file = spack.config.CONFIG.get_config_filename(scope.name, args.section)\n msg += \"\\t[scope={0}, file={1}]\\n\".format(scope.name, cfg_file)\n msg += (\n \"\\nIf the configuration files are updated, versions of Spack \"\n \"that are older than this version may not be able to read \"\n \"them. Spack stores backups of the updated files which can \"\n 'be retrieved with \"spack config revert\"'\n )\n tty.msg(msg)\n proceed = tty.get_yes_or_no(\"Do you want to proceed?\", default=False)\n\n if not proceed:\n tty.die(\"Operation aborted.\")\n\n # Get a function to update the format\n update_fn = spack.config.ensure_latest_format_fn(args.section)\n for scope in updates:\n cfg_file = spack.config.CONFIG.get_config_filename(scope.name, args.section)\n data = scope.get_section(args.section)\n assert data is not None, f\"Cannot find section {args.section} in {scope.name} scope\"\n update_fn(data)\n\n # Make a backup copy and rewrite the file\n bkp_file = cfg_file + \".bkp\"\n shutil.copy(cfg_file, bkp_file)\n spack.config.CONFIG.update_config(\n args.section, data[args.section], scope=scope.name, force=True\n )\n tty.msg(f'File \"{cfg_file}\" update [backup={bkp_file}]')\n\n\ndef _can_revert_update(scope_dir, cfg_file, bkp_file):\n dir_ok = fs.can_write_to_dir(scope_dir)\n cfg_ok = not os.path.exists(cfg_file) or fs.can_access(cfg_file)\n bkp_ok = fs.can_access(bkp_file)\n return dir_ok and cfg_ok and bkp_ok\n\n\ndef config_revert(args):\n scopes = [args.scope] if args.scope else [x.name for x in spack.config.CONFIG.writable_scopes]\n\n # Search for backup files in the configuration scopes\n Entry = collections.namedtuple(\"Entry\", [\"scope\", \"cfg\", \"bkp\"])\n to_be_restored, cannot_overwrite = [], []\n for scope in scopes:\n cfg_file = spack.config.CONFIG.get_config_filename(scope, args.section)\n bkp_file = cfg_file + \".bkp\"\n\n # If the backup files doesn't exist move to the next scope\n if not os.path.exists(bkp_file):\n continue\n\n # If it exists and we don't have write access in this scope\n # keep track of it and report a comprehensive error later\n entry = Entry(scope, cfg_file, bkp_file)\n scope_dir = os.path.dirname(bkp_file)\n can_be_reverted = _can_revert_update(scope_dir, cfg_file, bkp_file)\n if not can_be_reverted:\n cannot_overwrite.append(entry)\n continue\n\n to_be_restored.append(entry)\n\n # Report errors if we can't revert a configuration\n if cannot_overwrite:\n msg = \"Detected permission issues with the following scopes:\\n\\n\"\n for e in cannot_overwrite:\n msg += \"\\t[scope={0.scope}, cfg={0.cfg}, bkp={0.bkp}]\\n\".format(e)\n msg += (\n \"\\nEither ensure to have the right permissions before retrying\"\n \" or be more specific on the scope to revert.\"\n )\n tty.die(msg)\n\n proceed = True\n if not args.yes_to_all:\n msg = \"The following scopes will be restored from the corresponding backup files:\\n\"\n for entry in to_be_restored:\n msg += \"\\t[scope={0.scope}, bkp={0.bkp}]\\n\".format(entry)\n msg += \"This operation cannot be undone.\"\n tty.msg(msg)\n proceed = tty.get_yes_or_no(\"Do you want to proceed?\", default=False)\n\n if not proceed:\n tty.die(\"Operation aborted.\")\n\n for _, cfg_file, bkp_file in to_be_restored:\n shutil.copy(bkp_file, cfg_file)\n os.unlink(bkp_file)\n msg = 'File \"{0}\" reverted to old state'\n tty.msg(msg.format(cfg_file))\n\n\ndef config_prefer_upstream(args):\n \"\"\"Generate a packages config based on the configuration of all upstream\n installs.\"\"\"\n\n scope = args.scope\n if scope is None:\n scope = spack.config.default_modify_scope(\"packages\")\n\n all_specs = set(spack.store.STORE.db.query(installed=True))\n local_specs = set(spack.store.STORE.db.query_local(installed=True))\n pref_specs = local_specs if args.local else all_specs - local_specs\n\n conflicting_variants = set()\n\n pkgs = {}\n for spec in pref_specs:\n # Collect all the upstream compilers and versions for this package.\n pkg = pkgs.get(spec.name, {\"version\": []})\n pkgs[spec.name] = pkg\n\n # We have no existing variant if this is our first added version.\n existing_variants = pkg.get(\"variants\", None if not pkg[\"version\"] else \"\")\n\n version = spec.version.string\n if version not in pkg[\"version\"]:\n pkg[\"version\"].append(version)\n\n # Get and list all the variants that differ from the default.\n variants = []\n for var_name, variant in spec.variants.items():\n if var_name in [\"patches\"] or not spec.package.has_variant(var_name):\n continue\n\n vdef = spec.package.get_variant(var_name)\n if variant.value != vdef.default:\n variants.append(str(variant))\n variants.sort()\n variants = \" \".join(variants)\n\n if spec.name not in conflicting_variants:\n # Only specify the variants if there's a single variant\n # set across all versions/compilers.\n if existing_variants is not None and existing_variants != variants:\n conflicting_variants.add(spec.name)\n pkg.pop(\"variants\", None)\n elif variants:\n pkg[\"variants\"] = variants\n\n if conflicting_variants:\n tty.warn(\n \"The following packages have multiple conflicting upstream \"\n \"specs. You may have to specify, by \"\n \"concretized hash, which spec you want when building \"\n \"packages that depend on them:\\n - {0}\".format(\n \"\\n - \".join(sorted(conflicting_variants))\n )\n )\n\n # Simply write the config to the specified file.\n existing = spack.config.get(\"packages\", scope=scope)\n new = spack.schema.merge_yaml(existing, pkgs)\n spack.config.set(\"packages\", new, scope)\n config_file = spack.config.CONFIG.get_config_filename(scope, section)\n\n tty.msg(\"Updated config at {0}\".format(config_file))\n\n\ndef config(parser, args):\n action = {\n \"get\": config_get,\n \"blame\": config_blame,\n \"edit\": config_edit,\n \"list\": config_list,\n \"scopes\": config_scopes,\n \"add\": config_add,\n \"rm\": config_remove,\n \"remove\": config_remove,\n \"update\": config_update,\n \"revert\": config_revert,\n \"prefer-upstream\": config_prefer_upstream,\n \"change\": config_change,\n }\n action[args.config_command](args)\n" }, { "path": "lib/spack/spack/cmd/containerize.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport argparse\nimport os\n\nimport spack.container\nimport spack.container.images\nimport spack.llnl.util.tty\n\ndescription = \"create a container build recipe from an environment\"\nsection = \"environments\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n subparser.add_argument(\n \"--list-os\",\n action=\"store_true\",\n default=False,\n help=\"list all the OS that can be used in the bootstrap phase and exit\",\n )\n subparser.add_argument(\n \"--last-stage\",\n choices=(\"bootstrap\", \"build\", \"final\"),\n default=\"final\",\n help=\"last stage in the container recipe\",\n )\n\n\ndef containerize(parser, args):\n if args.list_os:\n possible_os = spack.container.images.all_bootstrap_os()\n msg = \"The following operating systems can be used to bootstrap Spack:\"\n msg += \"\\n{0}\".format(\" \".join(possible_os))\n spack.llnl.util.tty.msg(msg)\n return\n\n config_dir = args.env_dir or os.getcwd()\n config_file = os.path.abspath(os.path.join(config_dir, \"spack.yaml\"))\n if not os.path.exists(config_file):\n msg = \"file not found: {0}\"\n raise ValueError(msg.format(config_file))\n\n config = spack.container.validate(config_file)\n recipe = spack.container.recipe(config, last_phase=args.last_stage)\n print(recipe)\n" }, { "path": "lib/spack/spack/cmd/create.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport argparse\nimport os\nimport re\nimport sys\nimport urllib.parse\nfrom typing import List, Optional, Tuple\n\nimport spack.llnl.util.tty as tty\nimport spack.repo\nimport spack.stage\nfrom spack.llnl.util.filesystem import mkdirp\nfrom spack.spec import Spec\nfrom spack.url import (\n UndetectableNameError,\n UndetectableVersionError,\n find_versions_of_archive,\n parse_name,\n parse_version,\n)\nfrom spack.util.editor import editor\nfrom spack.util.executable import which\nfrom spack.util.format import get_version_lines\nfrom spack.util.naming import pkg_name_to_class_name, simplify_name\n\ndescription = \"create a new package file\"\nsection = \"packaging\"\nlevel = \"short\"\n\n\npackage_template = '''\\\n# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n# ----------------------------------------------------------------------------\n# If you submit this package back to Spack as a pull request,\n# please first remove this boilerplate and all FIXME comments.\n#\n# This is a template package file for Spack. We've put \"FIXME\"\n# next to all the things you'll want to change. Once you've handled\n# them, you can save this file and test your package like this:\n#\n# spack install {name}\n#\n# You can edit this file again by typing:\n#\n# spack edit {name}\n#\n# See the Spack documentation for more information on packaging.\n# ----------------------------------------------------------------------------\n\n{package_class_import}\n\nfrom spack.package import *\n\n\nclass {class_name}({base_class_name}):\n \"\"\"FIXME: Put a proper description of your package here.\"\"\"\n\n # FIXME: Add a proper url for your package's homepage here.\n homepage = \"https://www.example.com\"\n{url_def}\n\n # FIXME: Add a list of GitHub accounts to\n # notify when the package is updated.\n # maintainers(\"github_user1\", \"github_user2\")\n\n # FIXME: Add the SPDX identifier of the project's license below.\n # See https://spdx.org/licenses/ for a list. Upon manually verifying\n # the license, set checked_by to your Github username.\n license(\"UNKNOWN\", checked_by=\"github_user1\")\n\n{versions}\n\n{dependencies}\n\n{body_def}\n'''\n\n\nclass BundlePackageTemplate:\n \"\"\"\n Provides the default values to be used for a bundle package file template.\n \"\"\"\n\n base_class_name = \"BundlePackage\"\n package_class_import = \"from spack_repo.builtin.build_systems.bundle import BundlePackage\"\n\n dependencies = \"\"\"\\\n # FIXME: Add dependencies if required.\n # depends_on(\"foo\")\"\"\"\n\n url_def = \" # There is no URL since there is no code to download.\"\n body_def = \" # There is no need for install() since there is no code.\"\n\n def __init__(self, name: str, versions, languages: List[str]):\n self.name = name\n self.class_name = pkg_name_to_class_name(name)\n self.versions = versions\n self.languages = languages\n\n def write(self, pkg_path):\n \"\"\"Writes the new package file.\"\"\"\n\n all_deps = [f' depends_on(\"{lang}\", type=\"build\")' for lang in self.languages]\n if all_deps and self.dependencies:\n all_deps.append(\"\")\n all_deps.append(self.dependencies)\n\n # Write out a template for the file\n with open(pkg_path, \"w\", encoding=\"utf-8\") as pkg_file:\n pkg_file.write(\n package_template.format(\n name=self.name,\n class_name=self.class_name,\n base_class_name=self.base_class_name,\n package_class_import=self.package_class_import,\n url_def=self.url_def,\n versions=self.versions,\n dependencies=\"\\n\".join(all_deps),\n body_def=self.body_def,\n )\n )\n\n\nclass PackageTemplate(BundlePackageTemplate):\n \"\"\"Provides the default values to be used for the package file template\"\"\"\n\n base_class_name = \"Package\"\n package_class_import = \"from spack_repo.builtin.build_systems.generic import Package\"\n\n body_def = \"\"\"\\\n def install(self, spec, prefix):\n # FIXME: Unknown build system\n make()\n make(\"install\")\"\"\"\n\n url_line = ' url = \"{url}\"'\n\n def __init__(self, name, url, versions, languages: List[str]):\n super().__init__(name, versions, languages)\n\n self.url_def = self.url_line.format(url=url)\n\n\nclass AutotoolsPackageTemplate(PackageTemplate):\n \"\"\"Provides appropriate overrides for Autotools-based packages\n that *do* come with a ``configure`` script\"\"\"\n\n base_class_name = \"AutotoolsPackage\"\n package_class_import = (\n \"from spack_repo.builtin.build_systems.autotools import AutotoolsPackage\"\n )\n\n body_def = \"\"\"\\\n def configure_args(self):\n # FIXME: Add arguments other than --prefix\n # FIXME: If not needed delete this function\n args = []\n return args\"\"\"\n\n\nclass AutoreconfPackageTemplate(PackageTemplate):\n \"\"\"Provides appropriate overrides for Autotools-based packages\n that *do not* come with a ``configure`` script\"\"\"\n\n base_class_name = \"AutotoolsPackage\"\n package_class_import = (\n \"from spack_repo.builtin.build_systems.autotools import AutotoolsPackage\"\n )\n\n dependencies = \"\"\"\\\n with default_args(type=\"build\"):\n depends_on(\"autoconf\")\n depends_on(\"automake\")\n depends_on(\"libtool\")\n depends_on(\"m4\")\n\n # FIXME: Add additional dependencies if required.\n # depends_on(\"foo\")\"\"\"\n\n body_def = \"\"\"\\\n def autoreconf(self, spec, prefix):\n # FIXME: Modify the autoreconf method as necessary\n autoreconf(\"--install\", \"--verbose\", \"--force\")\n\n def configure_args(self):\n # FIXME: Add arguments other than --prefix\n # FIXME: If not needed delete this function\n args = []\n return args\"\"\"\n\n\nclass CargoPackageTemplate(PackageTemplate):\n \"\"\"Provides appropriate overrides for cargo-based packages\"\"\"\n\n base_class_name = \"CargoPackage\"\n package_class_import = \"from spack_repo.builtin.build_systems.cargo import CargoPackage\"\n\n body_def = \"\"\n\n\nclass CMakePackageTemplate(PackageTemplate):\n \"\"\"Provides appropriate overrides for CMake-based packages\"\"\"\n\n base_class_name = \"CMakePackage\"\n package_class_import = \"from spack_repo.builtin.build_systems.cmake import CMakePackage\"\n\n body_def = \"\"\"\\\n def cmake_args(self):\n # FIXME: Add arguments other than\n # FIXME: CMAKE_INSTALL_PREFIX and CMAKE_BUILD_TYPE\n # FIXME: If not needed delete this function\n args = []\n return args\"\"\"\n\n\nclass GoPackageTemplate(PackageTemplate):\n \"\"\"Provides appropriate overrides for Go-module-based packages\"\"\"\n\n base_class_name = \"GoPackage\"\n package_class_import = \"from spack_repo.builtin.build_systems.go import GoPackage\"\n\n body_def = \"\"\n\n\nclass LuaPackageTemplate(PackageTemplate):\n \"\"\"Provides appropriate overrides for LuaRocks-based packages\"\"\"\n\n base_class_name = \"LuaPackage\"\n package_class_import = \"from spack_repo.builtin.build_systems.lua import LuaPackage\"\n\n body_def = \"\"\"\\\n def luarocks_args(self):\n # FIXME: Add arguments to `luarocks make` other than rockspec path\n # FIXME: If not needed delete this function\n args = []\n return args\"\"\"\n\n def __init__(self, name, url, versions, languages: List[str]):\n # If the user provided `--name lua-lpeg`, don't rename it lua-lua-lpeg\n if not name.startswith(\"lua-\"):\n # Make it more obvious that we are renaming the package\n tty.msg(\"Changing package name from {0} to lua-{0}\".format(name))\n name = \"lua-{0}\".format(name)\n super().__init__(name, url, versions, languages)\n\n\nclass MesonPackageTemplate(PackageTemplate):\n \"\"\"Provides appropriate overrides for meson-based packages\"\"\"\n\n base_class_name = \"MesonPackage\"\n package_class_import = \"from spack_repo.builtin.build_systems.meson import MesonPackage\"\n\n body_def = \"\"\"\\\n def meson_args(self):\n # FIXME: If not needed delete this function\n args = []\n return args\"\"\"\n\n\nclass QMakePackageTemplate(PackageTemplate):\n \"\"\"Provides appropriate overrides for QMake-based packages\"\"\"\n\n base_class_name = \"QMakePackage\"\n package_class_import = \"from spack_repo.builtin.build_systems.qmake import QMakePackage\"\n\n body_def = \"\"\"\\\n def qmake_args(self):\n # FIXME: If not needed delete this function\n args = []\n return args\"\"\"\n\n\nclass MavenPackageTemplate(PackageTemplate):\n \"\"\"Provides appropriate overrides for Maven-based packages\"\"\"\n\n base_class_name = \"MavenPackage\"\n package_class_import = \"from spack_repo.builtin.build_systems.maven import MavenPackage\"\n\n body_def = \"\"\"\\\n def build(self, spec, prefix):\n # FIXME: If not needed delete this function\n pass\"\"\"\n\n\nclass SconsPackageTemplate(PackageTemplate):\n \"\"\"Provides appropriate overrides for SCons-based packages\"\"\"\n\n base_class_name = \"SConsPackage\"\n package_class_import = \"from spack_repo.builtin.build_systems.scons import SConsPackage\"\n\n body_def = \"\"\"\\\n def build_args(self, spec, prefix):\n # FIXME: Add arguments to pass to build.\n # FIXME: If not needed delete this function\n args = []\n return args\"\"\"\n\n\nclass WafPackageTemplate(PackageTemplate):\n \"\"\"Provides appropriate override for Waf-based packages\"\"\"\n\n base_class_name = \"WafPackage\"\n package_class_import = \"from spack_repo.builtin.build_systems.waf import WafPackage\"\n\n body_def = \"\"\"\\\n # FIXME: Override configure_args(), build_args(),\n # or install_args() if necessary.\"\"\"\n\n\nclass BazelPackageTemplate(PackageTemplate):\n \"\"\"Provides appropriate overrides for Bazel-based packages\"\"\"\n\n dependencies = \"\"\"\\\n # FIXME: Add additional dependencies if required.\n with default_args(type=\"build\"):\n depends_on(\"bazel\")\"\"\"\n\n body_def = \"\"\"\\\n def install(self, spec, prefix):\n # FIXME: Add logic to build and install here.\n bazel()\"\"\"\n\n\nclass RacketPackageTemplate(PackageTemplate):\n \"\"\"Provides appropriate overrides for Racket extensions\"\"\"\n\n base_class_name = \"RacketPackage\"\n package_class_import = \"from spack_repo.builtin.build_systems.racket import RacketPackage\"\n\n url_line = \"\"\"\\\n # FIXME: set the proper location from which to fetch your package\n git = \"git@github.com:example/example.git\"\n \"\"\"\n\n dependencies = \"\"\"\\\n # FIXME: Add dependencies if required. Only add the racket dependency\n # if you need specific versions. A generic racket dependency is\n # added implicitly by the RacketPackage class.\n # with default_args(type=(\"build\", \"run\")):\n # depends_on(\"racket@8.3:\")\"\"\"\n\n body_def = \"\"\"\\\n # FIXME: specify the name of the package,\n # as it should appear to ``raco pkg install``\n name = \"{0}\"\n # FIXME: set to true if published on pkgs.racket-lang.org\n # pkgs = False\n # FIXME: specify path to the root directory of the\n # package, if not the base directory\n # subdirectory = None\n \"\"\"\n\n def __init__(self, name, url, versions, languages: List[str]):\n # If the user provided `--name rkt-scribble`, don't rename it rkt-rkt-scribble\n if not name.startswith(\"rkt-\"):\n # Make it more obvious that we are renaming the package\n tty.msg(\"Changing package name from {0} to rkt-{0}\".format(name))\n name = \"rkt-{0}\".format(name)\n self.body_def = self.body_def.format(name[4:])\n super().__init__(name, url, versions, languages)\n\n\nclass PythonPackageTemplate(PackageTemplate):\n \"\"\"Provides appropriate overrides for python extensions\"\"\"\n\n base_class_name = \"PythonPackage\"\n package_class_import = \"from spack_repo.builtin.build_systems.python import PythonPackage\"\n\n dependencies = \"\"\"\\\n # FIXME: Only add the python/pip/wheel dependencies if you need specific versions\n # or need to change the dependency type. Generic python/pip/wheel dependencies are\n # added implicitly by the PythonPackage base class.\n # depends_on(\"python@2.X:2.Y,3.Z:\", type=(\"build\", \"run\"))\n # depends_on(\"py-pip@X.Y:\", type=\"build\")\n # depends_on(\"py-wheel@X.Y:\", type=\"build\")\n\n # FIXME: Add a build backend, usually defined in pyproject.toml. If no such file\n # exists, use setuptools.\n # with default_args(type=\"build\"):\n # depends_on(\"py-setuptools\")\n # depends_on(\"py-hatchling\")\n # depends_on(\"py-flit-core\")\n # depends_on(\"py-poetry-core\")\n\n # FIXME: Add additional dependencies if required.\n # with default_args(type=(\"build\", \"run\")):\n # depends_on(\"py-foo\")\"\"\"\n\n body_def = \"\"\"\\\n def config_settings(self, spec, prefix):\n # FIXME: Add configuration settings to be passed to the build backend\n # FIXME: If not needed, delete this function\n settings = {}\n return settings\"\"\"\n\n def __init__(self, name, url, versions, languages: List[str]):\n # If the user provided `--name py-numpy`, don't rename it py-py-numpy\n if not name.startswith(\"py-\"):\n # Make it more obvious that we are renaming the package\n tty.msg(\"Changing package name from {0} to py-{0}\".format(name))\n name = \"py-{0}\".format(name)\n\n # Simple PyPI URLs:\n # https:///packages////\n # e.g. https://pypi.io/packages/source/n/numpy/numpy-1.19.4.zip\n # e.g. https://www.pypi.io/packages/source/n/numpy/numpy-1.19.4.zip\n # e.g. https://pypi.org/packages/source/n/numpy/numpy-1.19.4.zip\n # e.g. https://pypi.python.org/packages/source/n/numpy/numpy-1.19.4.zip\n # e.g. https://files.pythonhosted.org/packages/source/n/numpy/numpy-1.19.4.zip\n\n # PyPI URLs containing hash:\n # https:///packages//// # noqa: E501\n # e.g. https://pypi.io/packages/c5/63/a48648ebc57711348420670bb074998f79828291f68aebfff1642be212ec/numpy-1.19.4.zip\n # e.g. https://files.pythonhosted.org/packages/c5/63/a48648ebc57711348420670bb074998f79828291f68aebfff1642be212ec/numpy-1.19.4.zip\n # e.g. https://files.pythonhosted.org/packages/c5/63/a48648ebc57711348420670bb074998f79828291f68aebfff1642be212ec/numpy-1.19.4.zip#sha256=141ec3a3300ab89c7f2b0775289954d193cc8edb621ea05f99db9cb181530512\n\n # PyPI URLs for wheels:\n # https://pypi.io/packages/py3/a/azureml_core/azureml_core-1.11.0-py3-none-any.whl\n # https://pypi.io/packages/py3/d/dotnetcore2/dotnetcore2-2.1.14-py3-none-macosx_10_9_x86_64.whl\n # https://pypi.io/packages/py3/d/dotnetcore2/dotnetcore2-2.1.14-py3-none-manylinux1_x86_64.whl\n # https://files.pythonhosted.org/packages/cp35.cp36.cp37.cp38.cp39/s/shiboken2/shiboken2-5.15.2-5.15.2-cp35.cp36.cp37.cp38.cp39-abi3-manylinux1_x86_64.whl\n # https://files.pythonhosted.org/packages/f4/99/ad2ef1aeeb395ee2319bb981ea08dbbae878d30dd28ebf27e401430ae77a/azureml_core-1.36.0.post2-py3-none-any.whl#sha256=60bcad10b4380d78a8280deb7365de2c2cd66527aacdcb4a173f613876cbe739\n\n match = re.search(r\"(?:pypi|pythonhosted)[^/]+/packages\" + \"/([^/#]+)\" * 4, url)\n if match:\n # PyPI URLs for wheels are too complicated, ignore them for now\n # https://www.python.org/dev/peps/pep-0427/#file-name-convention\n if not match.group(4).endswith(\".whl\"):\n if len(match.group(2)) == 1:\n # Simple PyPI URL\n url = \"/\".join(match.group(3, 4))\n else:\n # PyPI URL containing hash\n # Project name doesn't necessarily match download name, but it\n # usually does, so this is the best we can do\n project = parse_name(url)\n url = \"/\".join([project, match.group(4)])\n\n self.url_line = ' pypi = \"{url}\"'\n else:\n # Add a reminder about spack preferring PyPI URLs\n self.url_line = (\n \"\"\"\n # FIXME: ensure the package is not available through PyPI. If it is,\n # re-run `spack create --force` with the PyPI URL.\n\"\"\"\n + self.url_line\n )\n\n super().__init__(name, url, versions, languages)\n\n\nclass RPackageTemplate(PackageTemplate):\n \"\"\"Provides appropriate overrides for R extensions\"\"\"\n\n base_class_name = \"RPackage\"\n package_class_import = \"from spack_repo.builtin.build_systems.r import RPackage\"\n\n dependencies = \"\"\"\\\n # FIXME: Add dependencies if required.\n # with default_args(type=(\"build\", \"run\")):\n # depends_on(\"r-foo\")\"\"\"\n\n body_def = \"\"\"\\\n def configure_args(self):\n # FIXME: Add arguments to pass to install via --configure-args\n # FIXME: If not needed delete this function\n args = []\n return args\"\"\"\n\n def __init__(self, name, url, versions, languages: List[str]):\n # If the user provided `--name r-rcpp`, don't rename it r-r-rcpp\n if not name.startswith(\"r-\"):\n # Make it more obvious that we are renaming the package\n tty.msg(\"Changing package name from {0} to r-{0}\".format(name))\n name = \"r-{0}\".format(name)\n\n r_name = parse_name(url)\n\n cran = re.search(r\"(?:r-project|rstudio)[^/]+/src\" + \"/([^/]+)\" * 2, url)\n\n if cran:\n url = r_name\n self.url_line = ' cran = \"{url}\"'\n\n bioc = re.search(r\"(?:bioconductor)[^/]+/packages\" + \"/([^/]+)\" * 5, url)\n\n if bioc:\n self.url_line = ' url = \"{0}\"\\n bioc = \"{1}\"'.format(url, r_name)\n\n super().__init__(name, url, versions, languages)\n\n\nclass PerlmakePackageTemplate(PackageTemplate):\n \"\"\"Provides appropriate overrides for Perl extensions\n that come with a Makefile.PL\"\"\"\n\n base_class_name = \"PerlPackage\"\n package_class_import = \"from spack_repo.builtin.build_systems.perl import PerlPackage\"\n\n dependencies = \"\"\"\\\n # FIXME: Add dependencies if required:\n # with default_args(type=(\"build\", \"run\")):\n # depends_on(\"perl-foo\")\"\"\"\n\n body_def = \"\"\"\\\n def configure_args(self):\n # FIXME: Add non-standard arguments\n # FIXME: If not needed delete this function\n args = []\n return args\"\"\"\n\n def __init__(self, name, url, versions, languages: List[str]):\n # If the user provided `--name perl-cpp`, don't rename it perl-perl-cpp\n if not name.startswith(\"perl-\"):\n # Make it more obvious that we are renaming the package\n tty.msg(\"Changing package name from {0} to perl-{0}\".format(name))\n name = \"perl-{0}\".format(name)\n\n super().__init__(name, url, versions, languages)\n\n\nclass PerlbuildPackageTemplate(PerlmakePackageTemplate):\n \"\"\"Provides appropriate overrides for Perl extensions\n that come with a Build.PL instead of a Makefile.PL\"\"\"\n\n dependencies = \"\"\"\\\n depends_on(\"perl-module-build\", type=\"build\")\n\n # FIXME: Add additional dependencies if required:\n # with default_args(type=(\"build\", \"run\")):\n # depends_on(\"perl-foo\")\"\"\"\n\n\nclass OctavePackageTemplate(PackageTemplate):\n \"\"\"Provides appropriate overrides for octave packages\"\"\"\n\n base_class_name = \"OctavePackage\"\n package_class_import = \"from spack_repo.builtin.build_systems.octave import OctavePackage\"\n\n dependencies = \"\"\"\\\n extends(\"octave\")\n\n # FIXME: Add additional dependencies if required.\n # with default_args(type=(\"build\", \"run\")):\n # depends_on(\"octave-foo\")\"\"\"\n\n def __init__(self, name, url, versions, languages: List[str]):\n # If the user provided `--name octave-splines`, don't rename it\n # octave-octave-splines\n if not name.startswith(\"octave-\"):\n # Make it more obvious that we are renaming the package\n tty.msg(\"Changing package name from {0} to octave-{0}\".format(name))\n name = \"octave-{0}\".format(name)\n\n super().__init__(name, url, versions, languages)\n\n\nclass RubyPackageTemplate(PackageTemplate):\n \"\"\"Provides appropriate overrides for Ruby packages\"\"\"\n\n base_class_name = \"RubyPackage\"\n package_class_import = \"from spack_repo.builtin.build_systems.ruby import RubyPackage\"\n\n dependencies = \"\"\"\\\n # FIXME: Add dependencies if required. Only add the ruby dependency\n # if you need specific versions. A generic ruby dependency is\n # added implicitly by the RubyPackage class.\n # with default_args(type=(\"build\", \"run\")):\n # depends_on(\"ruby@X.Y.Z:\")\n # depends_on(\"ruby-foo\")\"\"\"\n\n body_def = \"\"\"\\\n def build(self, spec, prefix):\n # FIXME: If not needed delete this function\n pass\"\"\"\n\n def __init__(self, name, url, versions, languages: List[str]):\n # If the user provided `--name ruby-numpy`, don't rename it\n # ruby-ruby-numpy\n if not name.startswith(\"ruby-\"):\n # Make it more obvious that we are renaming the package\n tty.msg(\"Changing package name from {0} to ruby-{0}\".format(name))\n name = \"ruby-{0}\".format(name)\n\n super().__init__(name, url, versions, languages)\n\n\nclass MakefilePackageTemplate(PackageTemplate):\n \"\"\"Provides appropriate overrides for Makefile packages\"\"\"\n\n base_class_name = \"MakefilePackage\"\n package_class_import = \"from spack_repo.builtin.build_systems.makefile import MakefilePackage\"\n\n body_def = \"\"\"\\\n def edit(self, spec, prefix):\n # FIXME: Edit the Makefile if necessary\n # FIXME: If not needed delete this function\n # makefile = FileFilter(\"Makefile\")\n # makefile.filter(\"CC = .*\", \"CC = cc\")\n pass\"\"\"\n\n\nclass IntelPackageTemplate(PackageTemplate):\n \"\"\"Provides appropriate overrides for licensed Intel software\"\"\"\n\n base_class_name = \"IntelOneApiPackage\"\n package_class_import = \"from spack_repo.builtin.build_systems.oneapi import IntelOneApiPackage\"\n\n body_def = \"\"\"\\\n # FIXME: Override `setup_environment` if necessary.\"\"\"\n\n\nclass SIPPackageTemplate(PackageTemplate):\n \"\"\"Provides appropriate overrides for SIP packages.\"\"\"\n\n base_class_name = \"SIPPackage\"\n package_class_import = \"from spack_repo.builtin.build_systems.sip import SIPPackage\"\n\n body_def = \"\"\"\\\n def configure_args(self, spec, prefix):\n # FIXME: Add arguments other than --bindir and --destdir\n # FIXME: If not needed delete this function\n args = []\n return args\"\"\"\n\n def __init__(self, name, url, versions, languages: List[str]):\n # If the user provided `--name py-pyqt4`, don't rename it py-py-pyqt4\n if not name.startswith(\"py-\"):\n # Make it more obvious that we are renaming the package\n tty.msg(\"Changing package name from {0} to py-{0}\".format(name))\n name = \"py-{0}\".format(name)\n\n super().__init__(name, url, versions, languages)\n\n\ntemplates = {\n \"autoreconf\": AutoreconfPackageTemplate,\n \"autotools\": AutotoolsPackageTemplate,\n \"bazel\": BazelPackageTemplate,\n \"bundle\": BundlePackageTemplate,\n \"cargo\": CargoPackageTemplate,\n \"cmake\": CMakePackageTemplate,\n \"generic\": PackageTemplate,\n \"go\": GoPackageTemplate,\n \"intel\": IntelPackageTemplate,\n \"lua\": LuaPackageTemplate,\n \"makefile\": MakefilePackageTemplate,\n \"maven\": MavenPackageTemplate,\n \"meson\": MesonPackageTemplate,\n \"octave\": OctavePackageTemplate,\n \"perlbuild\": PerlbuildPackageTemplate,\n \"perlmake\": PerlmakePackageTemplate,\n \"python\": PythonPackageTemplate,\n \"qmake\": QMakePackageTemplate,\n \"r\": RPackageTemplate,\n \"racket\": RacketPackageTemplate,\n \"ruby\": RubyPackageTemplate,\n \"scons\": SconsPackageTemplate,\n \"sip\": SIPPackageTemplate,\n \"waf\": WafPackageTemplate,\n}\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n subparser.add_argument(\"url\", nargs=\"?\", help=\"url of package archive\")\n subparser.add_argument(\n \"--keep-stage\",\n action=\"store_true\",\n help=\"don't clean up staging area when command completes\",\n )\n subparser.add_argument(\"-n\", \"--name\", help=\"name of the package to create\")\n subparser.add_argument(\n \"-t\",\n \"--template\",\n metavar=\"TEMPLATE\",\n choices=sorted(templates.keys()),\n help=\"build system template to use\\n\\noptions: %(choices)s\",\n )\n subparser.add_argument(\n \"-r\", \"--repo\", help=\"path to a repository where the package should be created\"\n )\n subparser.add_argument(\n \"-N\",\n \"--namespace\",\n help=\"specify a namespace for the package\\n\\nmust be the namespace of \"\n \"a repository registered with Spack\",\n )\n subparser.add_argument(\n \"-f\",\n \"--force\",\n action=\"store_true\",\n help=\"overwrite any existing package file with the same name\",\n )\n subparser.add_argument(\n \"--skip-editor\",\n action=\"store_true\",\n help=\"skip the edit session for the package (e.g., automation)\",\n )\n subparser.add_argument(\n \"-b\", \"--batch\", action=\"store_true\", help=\"don't ask which versions to checksum\"\n )\n\n\n#: C file extensions\nC_EXT = {\".c\"}\n\n#: C++ file extensions\nCXX_EXT = {\n \".C\",\n \".c++\",\n \".cc\",\n \".ccm\",\n \".cpp\",\n \".CPP\",\n \".cxx\",\n \".h++\",\n \".hh\",\n \".hpp\",\n \".hxx\",\n \".inl\",\n \".ipp\",\n \".ixx\",\n \".tcc\",\n \".tpp\",\n}\n\n#: Fortran file extensions\nFORTRAN_EXT = {\n \".f77\",\n \".F77\",\n \".f90\",\n \".F90\",\n \".f95\",\n \".F95\",\n \".f\",\n \".F\",\n \".for\",\n \".FOR\",\n \".ftn\",\n \".FTN\",\n}\n\n\nclass BuildSystemAndLanguageGuesser:\n \"\"\"An instance of BuildSystemAndLanguageGuesser provides a callable object to be used\n during ``spack create``. By passing this object to ``spack checksum``, we\n can take a peek at the fetched tarball and discern the build system it uses\n \"\"\"\n\n def __init__(self):\n \"\"\"Sets the default build system.\"\"\"\n self.build_system = \"generic\"\n self._c = False\n self._cxx = False\n self._fortran = False\n\n # List of files in the archive ordered by their depth in the directory tree.\n self._file_entries: List[str] = []\n\n def __call__(self, archive: str, url: str) -> None:\n \"\"\"Try to guess the type of build system used by a project based on\n the contents of its archive or the URL it was downloaded from.\"\"\"\n\n # Peek inside the compressed file.\n if archive.endswith(\".zip\") or \".zip#\" in archive:\n try:\n unzip = which(\"unzip\")\n assert unzip is not None\n output = unzip(\"-lq\", archive, output=str)\n except Exception:\n output = \"\"\n else:\n try:\n tar = which(\"tar\")\n assert tar is not None\n output = tar(\"tf\", archive, output=str)\n except Exception:\n output = \"\"\n self._file_entries[:] = output.splitlines()\n\n # Files closest to the root should be considered first when determining build system.\n self._file_entries.sort(key=lambda p: p.count(\"/\"))\n\n self._determine_build_system(url)\n self._determine_language()\n\n def _determine_build_system(self, url: str) -> None:\n # Most octave extensions are hosted on Octave-Forge:\n # https://octave.sourceforge.net/index.html\n # They all have the same base URL.\n if \"downloads.sourceforge.net/octave/\" in url:\n self.build_system = \"octave\"\n elif url.endswith(\".gem\"):\n self.build_system = \"ruby\"\n elif url.endswith(\".whl\") or \".whl#\" in url:\n self.build_system = \"python\"\n elif url.endswith(\".rock\"):\n self.build_system = \"lua\"\n elif self._file_entries:\n # A list of clues that give us an idea of the build system a package\n # uses. If the regular expression matches a file contained in the\n # archive, the corresponding build system is assumed.\n # NOTE: Order is important here. If a package supports multiple\n # build systems, we choose the first match in this list.\n clues = [\n (re.compile(pattern), build_system)\n for pattern, build_system in (\n (r\"/CMakeLists\\.txt$\", \"cmake\"),\n (r\"/NAMESPACE$\", \"r\"),\n (r\"/Cargo\\.toml$\", \"cargo\"),\n (r\"/go\\.mod$\", \"go\"),\n (r\"/configure$\", \"autotools\"),\n (r\"/configure\\.(in|ac)$\", \"autoreconf\"),\n (r\"/Makefile\\.am$\", \"autoreconf\"),\n (r\"/pom\\.xml$\", \"maven\"),\n (r\"/SConstruct$\", \"scons\"),\n (r\"/waf$\", \"waf\"),\n (r\"/pyproject.toml\", \"python\"),\n (r\"/setup\\.(py|cfg)$\", \"python\"),\n (r\"/WORKSPACE$\", \"bazel\"),\n (r\"/Build\\.PL$\", \"perlbuild\"),\n (r\"/Makefile\\.PL$\", \"perlmake\"),\n (r\"/.*\\.gemspec$\", \"ruby\"),\n (r\"/Rakefile$\", \"ruby\"),\n (r\"/setup\\.rb$\", \"ruby\"),\n (r\"/.*\\.pro$\", \"qmake\"),\n (r\"/.*\\.rockspec$\", \"lua\"),\n (r\"/(GNU)?[Mm]akefile$\", \"makefile\"),\n (r\"/DESCRIPTION$\", \"octave\"),\n (r\"/meson\\.build$\", \"meson\"),\n (r\"/configure\\.py$\", \"sip\"),\n )\n ]\n\n # Determine the build system based on the files contained in the archive.\n for file in self._file_entries:\n for pattern, build_system in clues:\n if pattern.search(file):\n self.build_system = build_system\n return\n\n def _determine_language(self):\n for entry in self._file_entries:\n _, ext = os.path.splitext(entry)\n\n if not self._c and ext in C_EXT:\n self._c = True\n elif not self._cxx and ext in CXX_EXT:\n self._cxx = True\n elif not self._fortran and ext in FORTRAN_EXT:\n self._fortran = True\n\n if self._c and self._cxx and self._fortran:\n return\n\n @property\n def languages(self) -> List[str]:\n langs: List[str] = []\n if self._c:\n langs.append(\"c\")\n if self._cxx:\n langs.append(\"cxx\")\n if self._fortran:\n langs.append(\"fortran\")\n return langs\n\n\ndef get_name(name, url):\n \"\"\"Get the name of the package based on the supplied arguments.\n\n If a name was provided, always use that. Otherwise, if a URL was\n provided, extract the name from that. Otherwise, use a default.\n\n Args:\n name (str): explicit ``--name`` argument given to ``spack create``\n url (str): ``url`` argument given to ``spack create``\n\n Returns:\n str: The name of the package\n \"\"\"\n\n # Default package name\n result = \"example\"\n\n if name is not None:\n # Use a user-supplied name if one is present\n result = name\n if len(name.strip()) > 0:\n tty.msg(\"Using specified package name: '{0}'\".format(result))\n else:\n tty.die(\"A package name must be provided when using the option.\")\n elif url is not None:\n # Try to guess the package name based on the URL\n try:\n result = parse_name(url)\n if result != url:\n desc = \"URL\"\n else:\n desc = \"package name\"\n tty.msg(\"This looks like a {0} for {1}\".format(desc, result))\n except UndetectableNameError:\n tty.die(\n \"Couldn't guess a name for this package.\",\n \" Please report this bug. In the meantime, try running:\",\n \" `spack create --name `\",\n )\n\n result = simplify_name(result)\n\n if not re.match(r\"^[a-z0-9-]+$\", result):\n tty.die(\"Package name can only contain a-z, 0-9, and '-'\")\n\n return result\n\n\ndef get_url(url: Optional[str]) -> str:\n \"\"\"Get the URL to use.\n\n Use a default URL if none is provided.\n\n Args:\n url: ``url`` argument to ``spack create``\n\n Returns: The URL of the package\n \"\"\"\n\n # Use the user-supplied URL or a default URL if none is present.\n return url or \"https://www.example.com/example-1.2.3.tar.gz\"\n\n\ndef get_versions(args: argparse.Namespace, name: str) -> Tuple[str, BuildSystemAndLanguageGuesser]:\n \"\"\"Returns a list of versions and hashes for a package.\n\n Also returns a BuildSystemAndLanguageGuesser object.\n\n Returns default values if no URL is provided.\n\n Args:\n args: The arguments given to ``spack create``\n name: The name of the package\n\n Returns: Tuple of versions and hashes, and a BuildSystemAndLanguageGuesser object\n \"\"\"\n\n # Default version with hash\n hashed_versions = \"\"\"\\\n # FIXME: Add proper versions and checksums here.\n # version(\"1.2.3\", md5=\"0123456789abcdef0123456789abcdef\")\"\"\"\n\n # Default version without hash\n unhashed_versions = \"\"\"\\\n # FIXME: Add proper versions here.\n # version(\"1.2.4\")\"\"\"\n\n # Default guesser\n guesser = BuildSystemAndLanguageGuesser()\n\n valid_url = True\n try:\n parsed = urllib.parse.urlparse(args.url)\n if not parsed.scheme or parsed.scheme == \"file\":\n valid_url = False # No point in spidering these\n except (ValueError, TypeError):\n valid_url = False\n\n if args.url is not None and args.template != \"bundle\" and valid_url:\n # Find available versions\n try:\n url_dict = find_versions_of_archive(args.url)\n if len(url_dict) > 1 and not args.batch and sys.stdin.isatty():\n url_dict_filtered = spack.stage.interactive_version_filter(url_dict)\n if url_dict_filtered is None:\n exit(0)\n url_dict = url_dict_filtered\n except UndetectableVersionError:\n # Use fake versions\n tty.warn(\"Couldn't detect version in: {0}\".format(args.url))\n return hashed_versions, guesser\n\n if not url_dict:\n # If no versions were found, revert to what the user provided\n version = parse_version(args.url)\n url_dict = {version: args.url}\n\n version_hashes = spack.stage.get_checksums_for_versions(\n url_dict, name, first_stage_function=guesser, keep_stage=args.keep_stage\n )\n\n versions = get_version_lines(version_hashes)\n else:\n versions = unhashed_versions\n\n return versions, guesser\n\n\ndef get_build_system(\n template: Optional[str], url: str, guesser: BuildSystemAndLanguageGuesser\n) -> str:\n \"\"\"Determine the build system template.\n\n If a template is specified, always use that. Otherwise, if a URL\n is provided, download the tarball and peek inside to guess what\n build system it uses. Otherwise, use a generic template by default.\n\n Args:\n template: ``--template`` argument given to ``spack create``\n url: ``url`` argument given to ``spack create``\n guesser: The first_stage_function given to ``spack checksum`` which records the build\n system it detects\n\n Returns:\n str: The name of the build system template to use\n \"\"\"\n # Default template\n selected_template = \"generic\"\n\n if template is not None:\n selected_template = template\n # Use a user-supplied template if one is present\n tty.msg(\"Using specified package template: '{0}'\".format(selected_template))\n elif url is not None:\n # Use whatever build system the guesser detected\n selected_template = guesser.build_system\n if selected_template == \"generic\":\n tty.warn(\"Unable to detect a build system. Using a generic package template.\")\n else:\n msg = \"This package looks like it uses the {0} build system\"\n tty.msg(msg.format(selected_template))\n\n return selected_template\n\n\ndef get_repository(args: argparse.Namespace, name: str) -> spack.repo.Repo:\n \"\"\"Returns a Repo object that will allow us to determine the path where\n the new package file should be created.\n\n Args:\n args: The arguments given to ``spack create``\n name: The name of the package to create\n\n Returns:\n A Repo object capable of determining the path to the package file\n \"\"\"\n spec = Spec(name)\n # Figure out namespace for spec\n if spec.namespace and args.namespace and spec.namespace != args.namespace:\n tty.die(\"Namespaces '{0}' and '{1}' do not match.\".format(spec.namespace, args.namespace))\n\n if not spec.namespace and args.namespace:\n spec.namespace = args.namespace\n\n # Figure out where the new package should live\n repo_path = args.repo\n if repo_path is not None:\n repo = spack.repo.from_path(repo_path)\n if spec.namespace and spec.namespace != repo.namespace:\n tty.die(\n \"Can't create package with namespace {0} in repo with namespace {1}\".format(\n spec.namespace, repo.namespace\n )\n )\n else:\n if spec.namespace:\n repo = spack.repo.PATH.get_repo(spec.namespace)\n else:\n _repo = spack.repo.PATH.first_repo()\n assert _repo is not None, \"No package repository found\"\n repo = _repo\n\n # Set the namespace on the spec if it's not there already\n if not spec.namespace:\n spec.namespace = repo.namespace\n\n return repo\n\n\ndef create(parser, args):\n # Gather information about the package to be created\n name = get_name(args.name, args.url)\n url = get_url(args.url)\n versions, guesser = get_versions(args, name)\n build_system = get_build_system(args.template, url, guesser)\n\n # Create the package template object\n constr_args = {\"name\": name, \"versions\": versions, \"languages\": guesser.languages}\n package_class = templates[build_system]\n if package_class != BundlePackageTemplate:\n constr_args[\"url\"] = url\n package = package_class(**constr_args)\n tty.msg(\"Created template for {0} package\".format(package.name))\n\n # Create a directory for the new package\n repo = get_repository(args, name)\n pkg_path = repo.filename_for_package_name(package.name)\n if os.path.exists(pkg_path) and not args.force:\n tty.die(\n \"{0} already exists.\".format(pkg_path),\n \" Try running `spack create --force` to overwrite it.\",\n )\n else:\n mkdirp(os.path.dirname(pkg_path))\n\n # Write the new package file\n package.write(pkg_path)\n tty.msg(\"Created package file: {0}\".format(pkg_path))\n\n # Optionally open up the new package file in your $EDITOR\n if not args.skip_editor:\n editor(pkg_path)\n" }, { "path": "lib/spack/spack/cmd/debug.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport os\nimport platform\nimport re\nfrom typing import Optional\n\nimport spack\nimport spack.config\nimport spack.platforms\nimport spack.repo\nimport spack.spec\nimport spack.util.git\n\ndescription = \"debugging commands for troubleshooting Spack\"\nsection = \"developer\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n sp = subparser.add_subparsers(metavar=\"SUBCOMMAND\", dest=\"debug_command\")\n sp.add_parser(\"report\", help=\"print information useful for bug reports\")\n\n\ndef _format_repo_info(source, commit):\n if source.endswith(\".git\"):\n return f\"{source[:-4]}/commit/{commit}\"\n\n return f\"{source} ({commit[:7]})\"\n\n\ndef _get_builtin_repo_info() -> Optional[str]:\n \"\"\"Get the builtin package repository git commit sha.\"\"\"\n # Get builtin from config\n descriptors = spack.repo.RepoDescriptors.from_config(\n spack.repo.package_repository_lock(), spack.config.CONFIG\n )\n if \"builtin\" not in descriptors:\n return None\n\n builtin = descriptors[\"builtin\"]\n\n source = None\n if isinstance(builtin, spack.repo.RemoteRepoDescriptor) and builtin.fetched():\n destination = builtin.destination\n source = builtin.repository\n elif isinstance(builtin, spack.repo.LocalRepoDescriptor):\n destination = builtin.path\n source = builtin.path\n else:\n return None # no git info\n\n git = spack.util.git.git(required=False)\n if not git:\n return None\n\n rev = git(\n \"-C\", destination, \"rev-parse\", \"HEAD\", output=str, error=os.devnull, fail_on_error=False\n )\n if git.returncode != 0:\n return None\n\n match = re.match(r\"[a-f\\d]{7,}$\", rev)\n return _format_repo_info(source, match.group(0)) if match else None\n\n\ndef _get_spack_repo_info() -> str:\n \"\"\"Get the spack package repository git info.\"\"\"\n commit = spack.get_spack_commit()\n if not commit:\n return spack.spack_version\n\n repo_info = _format_repo_info(\"https://github.com/spack/spack.git\", commit)\n return f\"{spack.spack_version} ({repo_info})\"\n\n\ndef report(args):\n host_platform = spack.platforms.host()\n host_os = host_platform.default_operating_system()\n host_target = host_platform.default_target()\n architecture = spack.spec.ArchSpec((str(host_platform), str(host_os), str(host_target)))\n print(\"* **Spack:**\", _get_spack_repo_info())\n print(\"* **Builtin repo:**\", _get_builtin_repo_info() or \"not available\")\n print(\"* **Python:**\", platform.python_version())\n print(\"* **Platform:**\", architecture)\n\n\ndef debug(parser, args):\n if args.debug_command == \"report\":\n report(args)\n" }, { "path": "lib/spack/spack/cmd/deconcretize.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport sys\nfrom typing import List\n\nimport spack.cmd\nimport spack.cmd.common.confirmation as confirmation\nimport spack.environment as ev\nimport spack.llnl.util.tty as tty\nimport spack.spec\nfrom spack.cmd.common import arguments\n\ndescription = \"remove specs from the lockfile of an environment\"\nsection = \"environments\"\nlevel = \"long\"\n\n# Arguments for display_specs when we find ambiguity\ndisplay_args = {\"long\": True, \"show_flags\": False, \"variants\": False, \"indent\": 4}\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n subparser.add_argument(\n \"--root\", action=\"store_true\", help=\"deconcretize only specific environment roots\"\n )\n arguments.add_common_arguments(subparser, [\"yes_to_all\", \"specs\"])\n subparser.add_argument(\n \"-a\",\n \"--all\",\n action=\"store_true\",\n dest=\"all\",\n help=\"deconcretize ALL specs that match each supplied spec\",\n )\n\n\ndef get_deconcretize_list(\n args: argparse.Namespace, specs: List[spack.spec.Spec], env: ev.Environment\n) -> List[spack.spec.Spec]:\n \"\"\"\n Get list of environment roots to deconcretize\n \"\"\"\n env_specs = [s for _, s in env.concretized_specs()]\n to_deconcretize = []\n errors = []\n\n for s in specs:\n if args.root:\n # find all roots matching given spec\n to_deconc = [e for e in env_specs if e.satisfies(s)]\n else:\n # find all roots matching or depending on a matching spec\n to_deconc = [e for e in env_specs if any(d.satisfies(s) for d in e.traverse())]\n\n if len(to_deconc) < 1:\n tty.warn(f\"No matching specs to deconcretize for {s}\")\n\n elif len(to_deconc) > 1 and not args.all:\n errors.append((s, to_deconc))\n\n to_deconcretize.extend(to_deconc)\n\n if errors:\n for spec, matching in errors:\n tty.error(f\"{spec} matches multiple concrete specs:\")\n sys.stderr.write(\"\\n\")\n spack.cmd.display_specs(matching, output=sys.stderr, **display_args)\n sys.stderr.write(\"\\n\")\n sys.stderr.flush()\n tty.die(\"Use '--all' to deconcretize all matching specs, or be more specific\")\n\n return to_deconcretize\n\n\ndef deconcretize_specs(args, specs):\n env = spack.cmd.require_active_env(cmd_name=\"deconcretize\")\n\n if args.specs:\n deconcretize_list = get_deconcretize_list(args, specs, env)\n else:\n deconcretize_list = [s for _, s in env.concretized_specs()]\n\n if not args.yes_to_all:\n confirmation.confirm_action(deconcretize_list, \"deconcretized\", \"deconcretization\")\n\n with env.write_transaction():\n for spec in deconcretize_list:\n env.deconcretize_by_hash(spec.dag_hash())\n env.write()\n\n\ndef deconcretize(parser, args):\n if not args.specs and not args.all:\n tty.die(\n \"deconcretize requires at least one spec argument.\",\n \" Use `spack deconcretize --all` to deconcretize ALL specs.\",\n )\n\n specs = spack.cmd.parse_specs(args.specs) if args.specs else [None]\n deconcretize_specs(args, specs)\n" }, { "path": "lib/spack/spack/cmd/dependencies.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport sys\n\nimport spack.cmd\nimport spack.environment as ev\nimport spack.llnl.util.tty as tty\nimport spack.store\nfrom spack.cmd.common import arguments\nfrom spack.llnl.util.tty.colify import colify\nfrom spack.solver.input_analysis import create_graph_analyzer\n\ndescription = \"show dependencies of a package\"\nsection = \"query\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n subparser.add_argument(\n \"-i\",\n \"--installed\",\n action=\"store_true\",\n default=False,\n help=\"list installed dependencies of an installed spec \"\n \"instead of possible dependencies of a package\",\n )\n subparser.add_argument(\n \"-t\",\n \"--transitive\",\n action=\"store_true\",\n default=False,\n help=\"show all transitive dependencies\",\n )\n arguments.add_common_arguments(subparser, [\"deptype\"])\n subparser.add_argument(\n \"-V\",\n \"--no-expand-virtuals\",\n action=\"store_false\",\n default=True,\n dest=\"expand_virtuals\",\n help=\"do not expand virtual dependencies\",\n )\n arguments.add_common_arguments(subparser, [\"spec\"])\n\n\ndef dependencies(parser, args):\n specs = spack.cmd.parse_specs(args.spec)\n if len(specs) != 1:\n tty.die(\"spack dependencies takes only one spec.\")\n\n if args.installed:\n env = ev.active_environment()\n spec = spack.cmd.disambiguate_spec(specs[0], env)\n\n format_string = \"{name}{@version}{/hash:7}{%compiler}\"\n if sys.stdout.isatty():\n tty.msg(\"Dependencies of %s\" % spec.format(format_string, color=True))\n deps = spack.store.STORE.db.installed_relatives(\n spec, \"children\", args.transitive, deptype=args.deptype\n )\n if deps:\n spack.cmd.display_specs(deps, long=True)\n else:\n print(\"No dependencies\")\n\n else:\n spec = specs[0]\n dependencies, virtuals, _ = create_graph_analyzer().possible_dependencies(\n spec,\n transitive=args.transitive,\n expand_virtuals=args.expand_virtuals,\n allowed_deps=args.deptype,\n )\n if not args.expand_virtuals:\n dependencies.update(virtuals)\n\n if spec.name in dependencies:\n dependencies.remove(spec.name)\n\n if dependencies:\n colify(sorted(dependencies))\n else:\n print(\"No dependencies\")\n" }, { "path": "lib/spack/spack/cmd/dependents.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport collections\nimport sys\n\nimport spack.cmd\nimport spack.environment as ev\nimport spack.llnl.util.tty as tty\nimport spack.repo\nimport spack.store\nfrom spack.cmd.common import arguments\nfrom spack.llnl.util.tty.colify import colify\n\ndescription = \"show packages that depend on another\"\nsection = \"query\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n subparser.add_argument(\n \"-i\",\n \"--installed\",\n action=\"store_true\",\n default=False,\n help=\"list installed dependents of an installed spec \"\n \"instead of possible dependents of a package\",\n )\n subparser.add_argument(\n \"-t\",\n \"--transitive\",\n action=\"store_true\",\n default=False,\n help=\"show all transitive dependents\",\n )\n arguments.add_common_arguments(subparser, [\"spec\"])\n\n\ndef inverted_dependencies():\n \"\"\"Iterate through all packages and return a dictionary mapping package\n names to possible dependencies.\n\n Virtual packages are included as sources, so that you can query\n dependents of, e.g., ``mpi``, but virtuals are not included as\n actual dependents.\n \"\"\"\n dag = collections.defaultdict(set)\n for pkg_cls in spack.repo.PATH.all_package_classes():\n for _, deps_by_name in pkg_cls.dependencies.items():\n for dep in deps_by_name:\n deps = [dep]\n\n # expand virtuals if necessary\n if spack.repo.PATH.is_virtual(dep):\n deps += [s.name for s in spack.repo.PATH.providers_for(dep)]\n\n for d in deps:\n dag[d].add(pkg_cls.name)\n return dag\n\n\ndef get_dependents(pkg_name, ideps, transitive=False, dependents=None):\n \"\"\"Get all dependents for a package.\n\n Args:\n pkg_name (str): name of the package whose dependents should be returned\n ideps (dict): dictionary of dependents, from inverted_dependencies()\n transitive (bool or None): return transitive dependents when True\n \"\"\"\n if dependents is None:\n dependents = set()\n\n if pkg_name in dependents:\n return set()\n dependents.add(pkg_name)\n\n direct = ideps[pkg_name]\n if transitive:\n for dep_name in direct:\n get_dependents(dep_name, ideps, transitive, dependents)\n dependents.update(direct)\n return dependents\n\n\ndef dependents(parser, args):\n specs = spack.cmd.parse_specs(args.spec)\n if len(specs) != 1:\n tty.die(\"spack dependents takes only one spec.\")\n\n if args.installed:\n env = ev.active_environment()\n spec = spack.cmd.disambiguate_spec(specs[0], env)\n\n format_string = \"{name}{@version}{/hash:7}{%compiler}\"\n if sys.stdout.isatty():\n tty.msg(\"Dependents of %s\" % spec.cformat(format_string))\n deps = spack.store.STORE.db.installed_relatives(spec, \"parents\", args.transitive)\n if deps:\n spack.cmd.display_specs(deps, long=True)\n else:\n print(\"No dependents\")\n\n else:\n spec = specs[0]\n ideps = inverted_dependencies()\n\n dependents = get_dependents(spec.name, ideps, args.transitive)\n dependents.remove(spec.name)\n if dependents:\n colify(sorted(dependents))\n else:\n print(\"No dependents\")\n" }, { "path": "lib/spack/spack/cmd/deprecate.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Deprecate one Spack install in favor of another\n\nSpack packages of different configurations cannot be installed to the same\nlocation. However, in some circumstances (e.g. security patches) old\ninstallations should never be used again. In these cases, we will mark the old\ninstallation as deprecated, remove it, and link another installation into its\nplace.\n\nIt is up to the user to ensure binary compatibility between the deprecated\ninstallation and its deprecator.\n\"\"\"\n\nimport argparse\n\nimport spack.cmd\nimport spack.concretize\nimport spack.environment as ev\nimport spack.installer\nimport spack.llnl.util.tty as tty\nimport spack.store\nfrom spack.cmd.common import arguments\nfrom spack.error import SpackError\nfrom spack.llnl.util.filesystem import symlink\n\nfrom ..enums import InstallRecordStatus\n\ndescription = \"replace one package with another via symlinks\"\nsection = \"admin\"\nlevel = \"long\"\n\n# Arguments for display_specs when we find ambiguity\ndisplay_args = {\"long\": True, \"show_flags\": True, \"variants\": True, \"indent\": 4}\n\n\ndef setup_parser(sp: argparse.ArgumentParser) -> None:\n setattr(setup_parser, \"parser\", sp)\n\n arguments.add_common_arguments(sp, [\"yes_to_all\"])\n\n deps = sp.add_mutually_exclusive_group()\n deps.add_argument(\n \"-d\",\n \"--dependencies\",\n action=\"store_true\",\n default=True,\n dest=\"dependencies\",\n help=\"deprecate dependencies (default)\",\n )\n deps.add_argument(\n \"-D\",\n \"--no-dependencies\",\n action=\"store_false\",\n default=True,\n dest=\"dependencies\",\n help=\"do not deprecate dependencies\",\n )\n\n install = sp.add_mutually_exclusive_group()\n install.add_argument(\n \"-i\",\n \"--install-deprecator\",\n action=\"store_true\",\n default=False,\n dest=\"install\",\n help=\"concretize and install deprecator spec\",\n )\n install.add_argument(\n \"-I\",\n \"--no-install-deprecator\",\n action=\"store_false\",\n default=False,\n dest=\"install\",\n help=\"deprecator spec must already be installed (default)\",\n )\n\n sp.add_argument(\n \"-l\", \"--link-type\", type=str, default=None, choices=[\"soft\", \"hard\"], help=\"(deprecated)\"\n )\n\n sp.add_argument(\n \"specs\", nargs=argparse.REMAINDER, help=\"spec to deprecate and spec to use as deprecator\"\n )\n\n\ndef deprecate(parser, args):\n \"\"\"Deprecate one spec in favor of another\"\"\"\n if args.link_type is not None:\n tty.warn(\"The --link-type option is deprecated and will be removed in a future release.\")\n\n env = ev.active_environment()\n specs = spack.cmd.parse_specs(args.specs)\n\n if len(specs) != 2:\n raise SpackError(\"spack deprecate requires exactly two specs\")\n\n deprecate = spack.cmd.disambiguate_spec(\n specs[0],\n env,\n local=True,\n installed=(InstallRecordStatus.INSTALLED | InstallRecordStatus.DEPRECATED),\n )\n\n if args.install:\n deprecator = spack.concretize.concretize_one(specs[1])\n else:\n deprecator = spack.cmd.disambiguate_spec(specs[1], env, local=True)\n\n # calculate all deprecation pairs for errors and warning message\n all_deprecate = []\n all_deprecators = []\n\n generator = (\n deprecate.traverse(order=\"post\", deptype=\"link\", root=True)\n if args.dependencies\n else [deprecate]\n )\n for spec in generator:\n all_deprecate.append(spec)\n all_deprecators.append(deprecator[spec.name])\n # This will throw a key error if deprecator does not have a dep\n # that matches the name of a dep of the spec\n\n if not args.yes_to_all:\n tty.msg(\"The following packages will be deprecated:\\n\")\n spack.cmd.display_specs(all_deprecate, **display_args)\n tty.msg(\"In favor of (respectively):\\n\")\n spack.cmd.display_specs(all_deprecators, **display_args)\n print()\n\n already_deprecated = []\n already_deprecated_for = []\n for spec in all_deprecate:\n deprecated_for = spack.store.STORE.db.deprecator(spec)\n if deprecated_for:\n already_deprecated.append(spec)\n already_deprecated_for.append(deprecated_for)\n\n tty.msg(\"The following packages are already deprecated:\\n\")\n spack.cmd.display_specs(already_deprecated, **display_args)\n tty.msg(\"In favor of (respectively):\\n\")\n spack.cmd.display_specs(already_deprecated_for, **display_args)\n\n answer = tty.get_yes_or_no(\"Do you want to proceed?\", default=False)\n if not answer:\n tty.die(\"Will not deprecate any packages.\")\n\n for dcate, dcator in zip(all_deprecate, all_deprecators):\n spack.installer.deprecate(dcate, dcator, symlink)\n" }, { "path": "lib/spack/spack/cmd/dev_build.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport os\nimport sys\n\nimport spack.build_environment\nimport spack.cmd\nimport spack.cmd.common.arguments\nimport spack.concretize\nimport spack.config\nimport spack.installer_dispatch\nimport spack.llnl.util.tty as tty\nimport spack.repo\nfrom spack.cmd.common import arguments\n\ndescription = \"build package from code in current working directory\"\nsection = \"build\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n arguments.add_common_arguments(subparser, [\"jobs\", \"no_checksum\", \"spec\"])\n subparser.add_argument(\n \"-d\",\n \"--source-path\",\n dest=\"source_path\",\n default=None,\n help=\"path to source directory (defaults to the current directory)\",\n )\n subparser.add_argument(\n \"-i\",\n \"--ignore-dependencies\",\n action=\"store_true\",\n dest=\"ignore_deps\",\n help=\"do not try to install dependencies of requested packages\",\n )\n subparser.add_argument(\n \"--keep-prefix\",\n action=\"store_true\",\n help=\"do not remove the install prefix if installation fails\",\n )\n subparser.add_argument(\n \"--skip-patch\", action=\"store_true\", help=\"skip patching for the developer build\"\n )\n subparser.add_argument(\n \"-q\",\n \"--quiet\",\n action=\"store_true\",\n dest=\"quiet\",\n help=\"do not display verbose build output while installing\",\n )\n subparser.add_argument(\n \"--drop-in\",\n type=str,\n dest=\"shell\",\n default=None,\n help=\"drop into a build environment in a new shell, e.g., bash\",\n )\n subparser.add_argument(\n \"--test\",\n default=None,\n choices=[\"root\", \"all\"],\n help=\"run tests on only root packages or all packages\",\n )\n\n stop_group = subparser.add_mutually_exclusive_group()\n stop_group.add_argument(\n \"-b\",\n \"--before\",\n type=str,\n dest=\"before\",\n default=None,\n help=\"phase to stop before when installing (default None)\",\n )\n stop_group.add_argument(\n \"-u\",\n \"--until\",\n type=str,\n dest=\"until\",\n default=None,\n help=\"phase to stop after when installing (default None)\",\n )\n\n cd_group = subparser.add_mutually_exclusive_group()\n arguments.add_common_arguments(cd_group, [\"clean\", \"dirty\"])\n\n spack.cmd.common.arguments.add_concretizer_args(subparser)\n\n\ndef dev_build(self, args):\n if not args.spec:\n tty.die(\"spack dev-build requires a package spec argument.\")\n\n specs = spack.cmd.parse_specs(args.spec)\n if len(specs) > 1:\n tty.die(\"spack dev-build only takes one spec.\")\n\n spec = specs[0]\n if not spack.repo.PATH.exists(spec.name):\n raise spack.repo.UnknownPackageError(spec.name)\n\n if not spec.versions.concrete_range_as_version:\n tty.die(\n \"spack dev-build spec must have a single, concrete version. \"\n \"Did you forget a package version number?\"\n )\n\n source_path = args.source_path\n if source_path is None:\n source_path = os.getcwd()\n source_path = os.path.abspath(source_path)\n\n # Forces the build to run out of the source directory.\n spec.constrain(f'dev_path=\"{source_path}\"')\n spec = spack.concretize.concretize_one(spec)\n\n if spec.installed:\n tty.error(\"Already installed in %s\" % spec.prefix)\n tty.msg(\"Uninstall or try adding a version suffix for this dev build.\")\n sys.exit(1)\n\n # disable checksumming if requested\n if args.no_checksum:\n spack.config.set(\"config:checksum\", False, scope=\"command_line\")\n\n tests = False\n if args.test == \"all\":\n tests = True\n elif args.test == \"root\":\n tests = [spec.name for spec in specs]\n\n spack.installer_dispatch.create_installer(\n [spec.package],\n tests=tests,\n keep_prefix=args.keep_prefix,\n install_deps=not args.ignore_deps,\n verbose=not args.quiet,\n dirty=args.dirty,\n stop_before=args.before,\n skip_patch=args.skip_patch,\n stop_at=args.until,\n ).install()\n\n # drop into the build environment of the package?\n if args.shell is not None:\n spack.build_environment.setup_package(spec.package, dirty=False)\n os.execvp(args.shell, [args.shell])\n" }, { "path": "lib/spack/spack/cmd/develop.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport argparse\nimport os\nimport shutil\nfrom typing import Optional\n\nimport spack.cmd\nimport spack.config\nimport spack.environment\nimport spack.fetch_strategy\nimport spack.llnl.util.tty as tty\nimport spack.repo\nimport spack.spec\nimport spack.stage\nimport spack.util.path\nimport spack.version\nfrom spack.cmd.common import arguments\nfrom spack.error import SpackError\n\ndescription = \"add a spec to an environment's dev-build information\"\nsection = \"environments\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n subparser.add_argument(\"-p\", \"--path\", help=\"source location of package\")\n subparser.add_argument(\"-b\", \"--build-directory\", help=\"build directory for the package\")\n\n clone_group = subparser.add_mutually_exclusive_group()\n clone_group.add_argument(\n \"--no-clone\",\n action=\"store_false\",\n dest=\"clone\",\n help=\"do not clone, the package already exists at the source path\",\n )\n clone_group.add_argument(\n \"--clone\",\n action=\"store_true\",\n dest=\"clone\",\n default=True,\n help=(\n \"(default) clone the package unless the path already exists, \"\n \"use ``--force`` to overwrite\"\n ),\n )\n\n subparser.add_argument(\n \"--no-modify-concrete-specs\",\n action=\"store_false\",\n default=True,\n dest=\"apply_changes\",\n help=(\n \"do not mutate concrete specs to have dev_path provenance.\"\n \" This requires a later `spack concretize --force` command to use develop specs\"\n ),\n )\n\n subparser.add_argument(\n \"-f\",\n \"--force\",\n action=\"store_true\",\n default=False,\n help=\"remove any files or directories that block cloning source code\",\n )\n\n subparser.add_argument(\n \"-r\",\n \"--recursive\",\n action=\"store_true\",\n help=\"traverse nodes of the graph to mark everything up to the root as a develop spec\",\n )\n\n arguments.add_common_arguments(subparser, [\"spec\"])\n\n\ndef _retrieve_develop_source(spec: spack.spec.Spec, abspath: str) -> None:\n # \"steal\" the source code via staging API. We ask for a stage\n # to be created, then copy it afterwards somewhere else. It would be\n # better if we can create the `source_path` directly into its final\n # destination.\n pkg_cls = spack.repo.PATH.get_pkg_class(spec.name)\n # We construct a package class ourselves, rather than asking for\n # Spec.package, since Spec only allows this when it is concrete\n package = pkg_cls(spec)\n source_stage: spack.stage.Stage = package.stage[0]\n if isinstance(source_stage.fetcher, spack.fetch_strategy.GitFetchStrategy):\n source_stage.fetcher.get_full_repo = True\n # If we retrieved this version before and cached it, we may have\n # done so without cloning the full git repo; likewise, any\n # mirror might store an instance with truncated history.\n source_stage.default_fetcher_only = True\n\n source_stage.fetcher.set_package(package)\n package.stage.steal_source(abspath)\n\n\ndef assure_concrete_spec(env: spack.environment.Environment, spec: spack.spec.Spec):\n version = spec.versions.concrete_range_as_version\n if not version:\n # first check environment for a matching concrete spec\n matching_specs = env.all_matching_specs(spec)\n if matching_specs:\n version = matching_specs[0].version\n test_spec = spack.spec.Spec(f\"{spec}@{version}\")\n for m_spec in matching_specs:\n if not m_spec.satisfies(test_spec):\n raise SpackError(\n f\"{spec.name}: has multiple concrete instances in the graph that can't be\"\n \" satisfied by a single develop spec. To use `spack develop` ensure one\"\n \" of the following:\"\n f\"\\n a) {spec.name} nodes can satisfy the same develop spec (minimally \"\n \"this means they all share the same version)\"\n f\"\\n b) Provide a concrete develop spec ({spec.name}@[version]) to clearly\"\n \" indicate what should be developed\"\n )\n else:\n # look up the maximum version so infintiy versions are preferred for develop\n version = max(spack.repo.PATH.get_pkg_class(spec.fullname).versions.keys())\n tty.msg(f\"Defaulting to highest version: {spec.name}@{version}\")\n spec.versions = spack.version.VersionList([version])\n\n\ndef setup_src_code(spec: spack.spec.Spec, src_path: str, clone: bool = True, force: bool = False):\n \"\"\"\n Handle checking, cloning or overwriting source code\n \"\"\"\n assert spec.versions\n\n if clone:\n _clone(spec, src_path, force)\n\n if not clone and not os.path.exists(src_path):\n raise SpackError(f\"Provided path {src_path} does not exist\")\n\n version = spec.versions.concrete_range_as_version\n if not version:\n # look up the maximum version so infintiy versions are preferred for develop\n version = max(spack.repo.PATH.get_pkg_class(spec.fullname).versions.keys())\n tty.msg(f\"Defaulting to highest version: {spec.name}@{version}\")\n spec.versions = spack.version.VersionList([version])\n\n\ndef _update_config(spec, path):\n find_fn = lambda section: spec.name in section\n\n entry = {\"spec\": str(spec)}\n if path and path != spec.name:\n entry[\"path\"] = path\n\n def change_fn(section):\n section[spec.name] = entry\n\n spack.config.change_or_add(\"develop\", find_fn, change_fn)\n\n\ndef update_env(\n env: spack.environment.Environment,\n spec: spack.spec.Spec,\n specified_path: Optional[str] = None,\n build_dir: Optional[str] = None,\n apply_changes: bool = True,\n):\n \"\"\"\n Update the spack.yaml file with additions or changes from a develop call\n \"\"\"\n tty.debug(f\"Updating develop config for {env.name} transactionally\")\n\n if not specified_path:\n dev_entry = env.dev_specs.get(spec.name)\n if dev_entry:\n specified_path = dev_entry.get(\"path\", None)\n\n with env.write_transaction():\n if build_dir is not None:\n spack.config.add(\n f\"packages:{spec.name}:package_attributes:build_directory:{build_dir}\",\n env.scope_name,\n )\n # add develop spec and update path\n _update_config(spec, specified_path)\n\n # If we are automatically mutating the concrete specs for dev provenance, do so\n if apply_changes:\n env.apply_develop(spec, _abs_code_path(env, spec, specified_path))\n\n\ndef _clone(spec: spack.spec.Spec, abspath: str, force: bool = False):\n if os.path.exists(abspath):\n if force:\n shutil.rmtree(abspath)\n else:\n msg = f\"Skipping developer download of {spec.name}\"\n msg += f\" because its path {abspath} already exists.\"\n tty.msg(msg)\n return\n\n # cloning can take a while and it's nice to get a message for the longer clones\n tty.msg(f\"Cloning source code for {spec}\")\n _retrieve_develop_source(spec, abspath)\n\n\ndef _abs_code_path(\n env: spack.environment.Environment, spec: spack.spec.Spec, path: Optional[str] = None\n):\n src_path = path if path else spec.name\n return spack.util.path.canonicalize_path(src_path, default_wd=env.path)\n\n\ndef _dev_spec_generator(args, env):\n \"\"\"\n Generator function to loop over all the develop specs based on how the command is called\n If no specs are supplied then loop over the develop specs listed in the environment.\n \"\"\"\n if not args.spec:\n if args.clone is False:\n raise SpackError(\"No spec provided to spack develop command\")\n\n for name, entry in env.dev_specs.items():\n path = entry.get(\"path\", name)\n abspath = spack.util.path.canonicalize_path(path, default_wd=env.path)\n # Both old syntax `spack develop pkg@x` and new syntax `spack develop pkg@=x`\n # are currently supported.\n spec = spack.spec.parse_with_version_concrete(entry[\"spec\"])\n yield spec, abspath\n else:\n specs = spack.cmd.parse_specs(args.spec)\n if (args.path or args.build_directory) and len(specs) > 1:\n raise SpackError(\n \"spack develop requires at most one named spec when using the --path or\"\n \" --build-directory arguments\"\n )\n\n for spec in specs:\n if args.recursive:\n concrete_specs = env.all_matching_specs(spec)\n if not concrete_specs:\n tty.warn(\n f\"{spec.name} has no matching concrete specs in the environment and \"\n \"will be skipped. `spack develop --recursive` requires a concretized\"\n \" environment\"\n )\n else:\n for s in concrete_specs:\n for node_spec in s.traverse(direction=\"parents\", root=True):\n tty.debug(f\"Recursive develop for {node_spec.name}\")\n dev_spec = spack.spec.Spec(node_spec.format(\"{name}@{versions}\"))\n yield dev_spec, _abs_code_path(env, node_spec, args.path)\n else:\n yield spec, _abs_code_path(env, spec, args.path)\n\n\ndef develop(parser, args):\n env = spack.cmd.require_active_env(cmd_name=\"develop\")\n\n for spec, abspath in _dev_spec_generator(args, env):\n assure_concrete_spec(env, spec)\n setup_src_code(spec, abspath, clone=args.clone, force=args.force)\n update_env(env, spec, args.path, args.build_directory, args.apply_changes)\n" }, { "path": "lib/spack/spack/cmd/diff.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\nimport argparse\nimport sys\n\nimport spack.cmd\nimport spack.environment as ev\nimport spack.llnl.util.tty as tty\nimport spack.solver.asp as asp\nimport spack.util.spack_json as sjson\nfrom spack.cmd.common import arguments\nfrom spack.llnl.util.tty.color import cprint, get_color_when\n\ndescription = \"compare two specs\"\nsection = \"query\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n arguments.add_common_arguments(subparser, [\"specs\"])\n\n subparser.add_argument(\n \"--json\",\n action=\"store_true\",\n default=False,\n dest=\"dump_json\",\n help=\"dump json output instead of pretty printing\",\n )\n subparser.add_argument(\n \"--first\",\n action=\"store_true\",\n default=False,\n dest=\"load_first\",\n help=\"load the first match if multiple packages match the spec\",\n )\n subparser.add_argument(\n \"-a\",\n \"--attribute\",\n action=\"append\",\n help=\"select the attributes to show (defaults to all)\",\n )\n subparser.add_argument(\n \"--ignore\", action=\"append\", help=\"omit diffs related to these dependencies\"\n )\n\n\ndef shift(asp_function: asp.AspFunction) -> asp.AspFunction:\n \"\"\"Transforms ``attr(\"foo\", \"bar\")`` into ``foo(\"bar\")``.\"\"\"\n args = asp_function.args\n if not args:\n raise ValueError(f\"Can't shift ASP function with no arguments: {str(asp_function)}\")\n return asp.AspFunction(args[0], args[1:])\n\n\ndef compare_specs(a, b, to_string=False, color=None, ignore_packages=None):\n \"\"\"\n Generate a comparison, including diffs (for each side) and an intersection.\n\n We can either print the result to the console, or parse\n into a json object for the user to save. We return an object that shows\n the differences, intersection, and names for a pair of specs a and b.\n\n Arguments:\n a (spack.spec.Spec): the first spec to compare\n b (spack.spec.Spec): the second spec to compare\n a_name (str): the name of spec a\n b_name (str): the name of spec b\n to_string (bool): return an object that can be json dumped\n color (bool): whether to format the names for the console\n \"\"\"\n if color is None:\n color = get_color_when()\n\n a = a.copy()\n b = b.copy()\n\n if ignore_packages:\n for pkg_name in ignore_packages:\n a.trim(pkg_name)\n b.trim(pkg_name)\n\n # Prepare a solver setup to parse differences\n setup = asp.SpackSolverSetup()\n\n # get facts for specs, making sure to include build dependencies of concrete\n # specs and to descend into dependency hashes so we include all facts.\n a_facts = set(\n shift(func)\n for func in setup.spec_clauses(\n a, body=True, expand_hashes=True, concrete_build_deps=True, include_runtimes=True\n )\n if func.name == \"attr\"\n )\n b_facts = set(\n shift(func)\n for func in setup.spec_clauses(\n b, body=True, expand_hashes=True, concrete_build_deps=True, include_runtimes=True\n )\n if func.name == \"attr\"\n )\n\n # We want to present them to the user as simple key: values\n intersect = sorted(a_facts.intersection(b_facts))\n spec1_not_spec2 = sorted(a_facts.difference(b_facts))\n spec2_not_spec1 = sorted(b_facts.difference(a_facts))\n\n # Format the spec names to be colored\n fmt = \"{name}{@version}{/hash}\"\n a_name = a.format(fmt, color=color)\n b_name = b.format(fmt, color=color)\n\n # We want to show what is the same, and then difference for each\n return {\n \"intersect\": flatten(intersect) if to_string else intersect,\n \"a_not_b\": flatten(spec1_not_spec2) if to_string else spec1_not_spec2,\n \"b_not_a\": flatten(spec2_not_spec1) if to_string else spec2_not_spec1,\n \"a_name\": a_name,\n \"b_name\": b_name,\n }\n\n\ndef flatten(functions):\n \"\"\"\n Given a list of ASP functions, convert into a list of key: value tuples.\n\n We are squashing whatever is after the first index into one string for\n easier parsing in the interface\n \"\"\"\n updated = []\n for fun in functions:\n updated.append([fun.name, \" \".join(str(a) for a in fun.args)])\n return updated\n\n\ndef print_difference(c, attributes=\"all\", out=None):\n \"\"\"\n Print the difference.\n\n Given a diffset for A and a diffset for B, print red/green diffs to show\n the differences.\n \"\"\"\n # Default to standard out unless another stream is provided\n out = out or sys.stdout\n\n A = c[\"b_not_a\"]\n B = c[\"a_not_b\"]\n\n cprint(\"@R{--- %s}\" % c[\"a_name\"]) # bright red\n cprint(\"@G{+++ %s}\" % c[\"b_name\"]) # bright green\n\n # Cut out early if we don't have any differences!\n if not A and not B:\n print(\"No differences\\n\")\n return\n\n def group_by_type(diffset):\n grouped = {}\n for entry in diffset:\n if entry[0] not in grouped:\n grouped[entry[0]] = []\n grouped[entry[0]].append(entry[1])\n\n # Sort by second value to make comparison slightly closer\n for key, values in grouped.items():\n values.sort()\n return grouped\n\n A = group_by_type(A)\n B = group_by_type(B)\n\n # print a directionally relevant diff\n keys = list(A) + list(B)\n\n category = None\n for key in keys:\n if \"all\" not in attributes and key not in attributes:\n continue\n\n # Write the attribute, B is subtraction A is addition\n subtraction = [] if key not in B else B[key]\n addition = [] if key not in A else A[key]\n\n # Bail out early if we don't have any entries\n if not subtraction and not addition:\n continue\n\n # If we have a new category, create a new section\n if category != key:\n category = key\n\n # print category in bold, colorized\n cprint(\"@*b{@@ %s @@}\" % category) # bold blue\n\n # Print subtractions first\n while subtraction:\n cprint(\"@R{- %s}\" % subtraction.pop(0)) # bright red\n if addition:\n cprint(\"@G{+ %s}\" % addition.pop(0)) # bright green\n\n # Any additions left?\n while addition:\n cprint(\"@G{+ %s}\" % addition.pop(0))\n\n\ndef diff(parser, args):\n env = ev.active_environment()\n\n if len(args.specs) != 2:\n tty.die(\"You must provide two specs to diff.\")\n\n specs = []\n for spec in spack.cmd.parse_specs(args.specs):\n # If the spec has a hash, check it before disambiguating\n spec.replace_hash()\n if spec.concrete:\n specs.append(spec)\n else:\n specs.append(spack.cmd.disambiguate_spec(spec, env, first=args.load_first))\n\n # Calculate the comparison (c)\n color = False if args.dump_json else get_color_when()\n c = compare_specs(specs[0], specs[1], to_string=True, color=color, ignore_packages=args.ignore)\n\n # Default to all attributes\n attributes = args.attribute or [\"all\"]\n\n if args.dump_json:\n print(sjson.dump(c))\n else:\n tty.warn(\"This interface is subject to change.\\n\")\n print_difference(c, attributes)\n" }, { "path": "lib/spack/spack/cmd/docs.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport webbrowser\n\ndescription = \"open spack documentation in a web browser\"\nsection = \"help\"\nlevel = \"short\"\n\n\ndef docs(parser, args):\n webbrowser.open(\"https://spack.readthedocs.io\")\n" }, { "path": "lib/spack/spack/cmd/edit.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport errno\nimport glob\nimport os\nfrom typing import Optional, Union\n\nimport spack.cmd\nimport spack.llnl.util.tty as tty\nimport spack.paths\nimport spack.repo\nimport spack.util.editor\n\ndescription = \"open package files in ``$EDITOR``\"\nsection = \"packaging\"\nlevel = \"short\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n excl_args = subparser.add_mutually_exclusive_group()\n\n # Various types of Spack files that can be edited\n # Edits package files by default\n # build systems require separate logic to find\n excl_args.add_argument(\n \"-b\",\n \"--build-system\",\n dest=\"path\",\n action=\"store_const\",\n const=\"BUILD_SYSTEM\", # placeholder for path that requires computing late\n help=\"edit the build system with the supplied name or fullname\",\n )\n excl_args.add_argument(\n \"-c\",\n \"--command\",\n dest=\"path\",\n action=\"store_const\",\n const=spack.paths.command_path,\n help=\"edit the command with the supplied name\",\n )\n excl_args.add_argument(\n \"-d\",\n \"--docs\",\n dest=\"path\",\n action=\"store_const\",\n const=os.path.join(spack.paths.lib_path, \"docs\"),\n help=\"edit the docs with the supplied name\",\n )\n excl_args.add_argument(\n \"-t\",\n \"--test\",\n dest=\"path\",\n action=\"store_const\",\n const=spack.paths.test_path,\n help=\"edit the test with the supplied name\",\n )\n excl_args.add_argument(\n \"-m\",\n \"--module\",\n dest=\"path\",\n action=\"store_const\",\n const=spack.paths.module_path,\n help=\"edit the main spack module with the supplied name\",\n )\n\n # Options for editing packages and build systems\n subparser.add_argument(\n \"-r\", \"--repo\", default=None, help=\"path to repo to edit package or build system in\"\n )\n subparser.add_argument(\n \"-N\", \"--namespace\", default=None, help=\"namespace of package or build system to edit\"\n )\n\n subparser.add_argument(\"package\", nargs=\"*\", default=None, help=\"package name\")\n\n\ndef locate_package(name: str, repo: Optional[spack.repo.Repo]) -> str:\n # if not given a repo, use the full repo path to choose one\n repo_like: Union[spack.repo.Repo, spack.repo.RepoPath] = repo or spack.repo.PATH\n path: str = repo_like.filename_for_package_name(name)\n\n try:\n with open(path, \"r\", encoding=\"utf-8\"):\n return path\n except OSError as e:\n if e.errno == errno.ENOENT:\n raise spack.repo.UnknownPackageError(name) from e\n tty.die(f\"Cannot edit package: {e}\")\n\n\ndef locate_build_system(name: str, repo: Optional[spack.repo.Repo]) -> str:\n # If given a fullname for a build system, split it into namespace and name\n namespace = None\n if \".\" in name:\n namespace, name = name.rsplit(\".\", 1)\n\n # If given a namespace and a repo, they better match\n if namespace and repo:\n if repo.namespace != namespace:\n msg = f\"{namespace}.{name}: namespace conflicts with repo '{repo.namespace}'\"\n msg += \" specified from --repo or --namespace argument\"\n raise ValueError(msg)\n\n if namespace:\n repo = spack.repo.PATH.get_repo(namespace)\n\n # If not given a namespace, use the default\n if not repo:\n repo = spack.repo.PATH.first_repo()\n\n assert repo\n return locate_file(name, repo.build_systems_path)\n\n\ndef locate_file(name: str, path: str) -> str:\n # convert command names to python module name\n if path == spack.paths.command_path:\n name = spack.cmd.python_name(name)\n\n file_path = os.path.join(path, name)\n\n # Try to open direct match.\n try:\n with open(file_path, \"r\", encoding=\"utf-8\"):\n return file_path\n except OSError as e:\n if e.errno != errno.ENOENT:\n tty.die(f\"Cannot edit file: {e}\")\n pass\n\n # Otherwise try to find a file that starts with the name\n candidates = glob.glob(file_path + \"*\")\n exclude_list = [\".pyc\", \"~\"] # exclude binaries and backups\n files = [f for f in candidates if not any(f.endswith(ext) for ext in exclude_list)]\n if len(files) > 1:\n tty.die(\n f\"Multiple files start with `{name}`:\\n\"\n + \"\\n\".join(f\" {os.path.basename(f)}\" for f in files)\n )\n elif not files:\n tty.die(f\"No file for '{name}' was found in {path}\")\n return files[0]\n\n\ndef edit(parser, args):\n names = args.package\n\n # If `--command`, `--test`, `--docs`, or `--module` is chosen, edit those instead\n if args.path and args.path != \"BUILD_SYSTEM\":\n paths = [locate_file(name, args.path) for name in names] if names else [args.path]\n spack.util.editor.editor(*paths)\n return\n\n # Cannot set repo = spack.repo.PATH.first_repo() as default because packages and build_systems\n # can include repo information as part of their fullname\n repo = None\n if args.namespace:\n repo = spack.repo.PATH.get_repo(args.namespace)\n elif args.repo:\n repo = spack.repo.from_path(args.repo)\n # default_repo used when no name provided\n default_repo = repo or spack.repo.PATH.first_repo()\n\n if args.path == \"BUILD_SYSTEM\":\n if names:\n paths = [locate_build_system(n, repo) for n in names]\n else:\n paths = [default_repo.build_systems_path]\n spack.util.editor.editor(*paths)\n return\n\n paths = [locate_package(n, repo) for n in names] if names else [default_repo.packages_path]\n spack.util.editor.editor(*paths)\n" }, { "path": "lib/spack/spack/cmd/env.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport os\nimport shlex\nimport shutil\nimport sys\nimport tempfile\nfrom pathlib import Path\nfrom typing import List, Optional, Set, Tuple, Union\n\nimport spack.cmd\nimport spack.cmd.common\nimport spack.cmd.common.arguments\nimport spack.cmd.modules\nimport spack.config\nimport spack.environment as ev\nimport spack.environment.depfile as depfile\nimport spack.environment.environment\nimport spack.environment.shell\nimport spack.llnl.string as string\nimport spack.llnl.util.filesystem as fs\nimport spack.llnl.util.tty as tty\nimport spack.tengine\nfrom spack.cmd.common import arguments\nfrom spack.llnl.util.filesystem import islink, symlink\nfrom spack.llnl.util.tty.colify import colify\nfrom spack.llnl.util.tty.color import cescape, colorize\nfrom spack.traverse import traverse_nodes\nfrom spack.util.environment import EnvironmentModifications\n\ndescription = \"manage environments\"\nsection = \"environments\"\nlevel = \"short\"\n\n\n#: List of subcommands of ``spack env``\nsubcommands: List[Tuple[str, ...]] = [\n (\"activate\",),\n (\"deactivate\",),\n (\"create\",),\n (\"remove\", \"rm\"),\n (\"rename\", \"mv\"),\n (\"list\", \"ls\"),\n (\"status\", \"st\"),\n (\"loads\",),\n (\"view\",),\n (\"update\",),\n (\"revert\",),\n (\"depfile\",),\n (\"track\",),\n (\"untrack\",),\n]\n\n\n#\n# env create\n#\ndef env_create_setup_parser(subparser):\n \"\"\"\\\n create a new environment\n\n create a new environment or, optionally, copy an existing environment\n\n a manifest file results in a new abstract environment while a lock file\n creates a new concrete environment\n \"\"\"\n subparser.add_argument(\n \"env_name\", metavar=\"env\", help=\"name or directory of the new environment\"\n )\n subparser.add_argument(\n \"-d\", \"--dir\", action=\"store_true\", help=\"create an environment in a specific directory\"\n )\n subparser.add_argument(\n \"--keep-relative\",\n action=\"store_true\",\n help=\"copy envfile's relative develop paths verbatim\",\n )\n view_opts = subparser.add_mutually_exclusive_group()\n view_opts.add_argument(\n \"--without-view\", action=\"store_true\", help=\"do not maintain a view for this environment\"\n )\n view_opts.add_argument(\n \"--with-view\", help=\"maintain view at WITH_VIEW (vs. environment's directory)\"\n )\n subparser.add_argument(\n \"envfile\",\n nargs=\"?\",\n default=None,\n help=\"manifest or lock file (ends with '.json' or '.lock') or an environment name or path\",\n )\n subparser.add_argument(\n \"--include-concrete\",\n action=\"append\",\n help=\"copy concrete specs from INCLUDE_CONCRETE's environment\",\n )\n\n\ndef env_create(args):\n if args.with_view:\n # Expand relative paths provided on the command line to the current working directory\n # This way we interpret `spack env create --with-view ./view --dir ./env` as\n # a view in $PWD/view, not $PWD/env/view. This is different from specifying a relative\n # path in the manifest, which is resolved relative to the manifest file's location.\n with_view = os.path.abspath(args.with_view)\n elif args.without_view:\n with_view = False\n else:\n # Note that 'None' means unspecified, in which case the Environment\n # object could choose to enable a view by default. False means that\n # the environment should not include a view.\n with_view = None\n\n include_concrete = None\n if hasattr(args, \"include_concrete\"):\n include_concrete = args.include_concrete\n\n env = _env_create(\n args.env_name,\n init_file=args.envfile,\n dir=args.dir or os.path.sep in args.env_name or args.env_name in (\".\", \"..\"),\n with_view=with_view,\n keep_relative=args.keep_relative,\n include_concrete=include_concrete,\n )\n\n # Generate views, only really useful for environments created from spack.lock files.\n if args.envfile:\n env.regenerate_views()\n\n\ndef _env_create(\n name_or_path: str,\n *,\n init_file: Optional[str] = None,\n dir: bool = False,\n with_view: Optional[Union[bool, str]] = None,\n keep_relative: bool = False,\n include_concrete: Optional[List[str]] = None,\n):\n \"\"\"Create a new environment, with an optional yaml description.\n\n Arguments:\n name_or_path: name of the environment to create, or path to it\n init_file: optional initialization file -- can be a JSON lockfile\n (*.lock, *.json), YAML manifest file, or env dir\n dir: if True, create an environment in a directory instead of a named\n environment\n keep_relative: if True, develop paths are copied verbatim into the new\n environment file, otherwise they may be made absolute if the new\n environment is in a different location\n include_concrete: list of the included concrete environments\n \"\"\"\n if not dir:\n env = ev.create(\n name_or_path,\n init_file=init_file,\n with_view=with_view,\n keep_relative=keep_relative,\n include_concrete=include_concrete,\n )\n tty.msg(\n colorize(\n f\"Created environment @c{{{cescape(name_or_path)}}} in: @c{{{cescape(env.path)}}}\"\n )\n )\n else:\n env = ev.create_in_dir(\n name_or_path,\n init_file=init_file,\n with_view=with_view,\n keep_relative=keep_relative,\n include_concrete=include_concrete,\n )\n tty.msg(colorize(f\"Created independent environment in: @c{{{cescape(env.path)}}}\"))\n tty.msg(f\"Activate with: {colorize(f'@c{{spack env activate {cescape(name_or_path)}}}')}\")\n return env\n\n\n#\n# env activate\n#\ndef env_activate_setup_parser(subparser):\n \"\"\"set the active environment\"\"\"\n shells = subparser.add_mutually_exclusive_group()\n shells.add_argument(\n \"--sh\",\n action=\"store_const\",\n dest=\"shell\",\n const=\"sh\",\n help=\"print sh commands to activate the environment\",\n )\n shells.add_argument(\n \"--csh\",\n action=\"store_const\",\n dest=\"shell\",\n const=\"csh\",\n help=\"print csh commands to activate the environment\",\n )\n shells.add_argument(\n \"--fish\",\n action=\"store_const\",\n dest=\"shell\",\n const=\"fish\",\n help=\"print fish commands to activate the environment\",\n )\n shells.add_argument(\n \"--bat\",\n action=\"store_const\",\n dest=\"shell\",\n const=\"bat\",\n help=\"print bat commands to activate the environment\",\n )\n shells.add_argument(\n \"--pwsh\",\n action=\"store_const\",\n dest=\"shell\",\n const=\"pwsh\",\n help=\"print powershell commands to activate environment\",\n )\n\n view_options = subparser.add_mutually_exclusive_group()\n view_options.add_argument(\n \"-v\",\n \"--with-view\",\n metavar=\"name\",\n help=\"set runtime environment variables for the named view\",\n )\n view_options.add_argument(\n \"-V\",\n \"--without-view\",\n action=\"store_true\",\n help=\"do not set runtime environment variables for any view\",\n )\n\n subparser.add_argument(\n \"-p\",\n \"--prompt\",\n action=\"store_true\",\n default=False,\n help=\"add the active environment to the command line prompt\",\n )\n\n subparser.add_argument(\n \"--temp\",\n action=\"store_true\",\n default=False,\n help=\"create and activate in a temporary directory\",\n )\n subparser.add_argument(\n \"--create\",\n action=\"store_true\",\n default=False,\n help=\"create and activate the environment if it doesn't exist\",\n )\n subparser.add_argument(\n \"--envfile\",\n nargs=\"?\",\n default=None,\n help=\"manifest or lock file (ends with '.json' or '.lock')\",\n )\n subparser.add_argument(\n \"--keep-relative\",\n action=\"store_true\",\n help=\"copy envfile's relative develop paths verbatim when create\",\n )\n subparser.add_argument(\n \"-d\",\n \"--dir\",\n default=False,\n action=\"store_true\",\n help=\"activate environment based on the directory supplied\",\n )\n subparser.add_argument(\n metavar=\"env\",\n dest=\"env_name\",\n nargs=\"?\",\n default=None,\n help=(\"name or directory of the environment being activated\"),\n )\n\n\ndef create_temp_env_directory():\n \"\"\"\n Returns the path of a temporary directory in which to\n create an environment\n \"\"\"\n return tempfile.mkdtemp(prefix=\"spack-\")\n\n\ndef _tty_info(msg):\n \"\"\"tty.info like function that prints the equivalent printf statement for eval.\"\"\"\n decorated = f\"{colorize('@*b{==>}')} {msg}\\n\"\n executor = \"echo\" if sys.platform == \"win32\" else \"printf\"\n print(f\"{executor} {shlex.quote(decorated)};\")\n\n\ndef env_activate(args):\n if not args.shell:\n spack.cmd.common.shell_init_instructions(\n \"spack env activate\", \" eval `spack env activate {sh_arg} [...]`\"\n )\n return 1\n\n # Error out when -e, -E, -D flags are given, cause they are ambiguous.\n if args.env or args.no_env or args.env_dir:\n tty.die(\"Calling spack env activate with --env, --env-dir and --no-env is ambiguous\")\n\n # special parser error handling relative to the --temp flag\n temp_conflicts = iter([args.keep_relative, args.dir, args.env_name, args.with_view])\n if args.temp and any(temp_conflicts):\n tty.die(\n \"spack env activate --temp cannot be combined with managed environments, --with-view,\"\n \" --keep-relative, or --dir.\"\n )\n\n # When executing `spack env activate` without further arguments, activate\n # the default environment. It's created when it doesn't exist yet.\n if not args.env_name and not args.temp:\n short_name = \"default\"\n if not ev.exists(short_name):\n ev.create(short_name)\n action = \"Created and activated\"\n else:\n action = \"Activated\"\n env_path = ev.root(short_name)\n _tty_info(f\"{action} default environment in {env_path}\")\n\n # Temporary environment\n elif args.temp:\n env = create_temp_env_directory()\n env_path = os.path.abspath(env)\n short_name = os.path.basename(env_path)\n view = not args.without_view\n ev.create_in_dir(env, with_view=view).write(regenerate=False)\n _tty_info(f\"Created and activated temporary environment in {env_path}\")\n\n # Managed environment\n elif ev.exists(args.env_name) and not args.dir:\n env_path = ev.root(args.env_name)\n short_name = args.env_name\n\n # Environment directory\n elif ev.is_env_dir(args.env_name):\n env_path = os.path.abspath(args.env_name)\n short_name = os.path.basename(env_path)\n\n # create if user requested, and then recall recursively\n elif args.create:\n tty.set_msg_enabled(False)\n env_create(args)\n tty.set_msg_enabled(True)\n env_activate(args)\n return\n\n else:\n tty.die(\"No such environment: '%s'\" % args.env_name)\n\n env_prompt = \"[%s]\" % short_name\n\n # We only support one active environment at a time, so deactivate the current one.\n if ev.active_environment() is None:\n cmds = \"\"\n env_mods = EnvironmentModifications()\n else:\n cmds = spack.environment.shell.deactivate_header(shell=args.shell)\n env_mods = spack.environment.shell.deactivate()\n\n # Activate new environment\n active_env = ev.Environment(env_path)\n\n # Check if runtime environment variables are requested, and if so, for what view.\n view: Optional[str] = None\n if args.with_view:\n view = args.with_view\n if not active_env.has_view(view):\n tty.die(f\"The environment does not have a view named '{view}'\")\n elif not args.without_view and active_env.has_view(ev.default_view_name):\n view = ev.default_view_name\n\n cmds += spack.environment.shell.activate_header(\n env=active_env, shell=args.shell, prompt=env_prompt if args.prompt else None, view=view\n )\n env_mods.extend(spack.environment.shell.activate(env=active_env, view=view))\n cmds += env_mods.shell_modifications(args.shell)\n sys.stdout.write(cmds)\n\n\n#\n# env deactivate\n#\ndef env_deactivate_setup_parser(subparser):\n \"\"\"deactivate the active environment\"\"\"\n shells = subparser.add_mutually_exclusive_group()\n shells.add_argument(\n \"--sh\",\n action=\"store_const\",\n dest=\"shell\",\n const=\"sh\",\n help=\"print sh commands to deactivate the environment\",\n )\n shells.add_argument(\n \"--csh\",\n action=\"store_const\",\n dest=\"shell\",\n const=\"csh\",\n help=\"print csh commands to deactivate the environment\",\n )\n shells.add_argument(\n \"--fish\",\n action=\"store_const\",\n dest=\"shell\",\n const=\"fish\",\n help=\"print fish commands to activate the environment\",\n )\n shells.add_argument(\n \"--bat\",\n action=\"store_const\",\n dest=\"shell\",\n const=\"bat\",\n help=\"print bat commands to activate the environment\",\n )\n shells.add_argument(\n \"--pwsh\",\n action=\"store_const\",\n dest=\"shell\",\n const=\"pwsh\",\n help=\"print pwsh commands to activate the environment\",\n )\n\n\ndef env_deactivate(args):\n if not args.shell:\n spack.cmd.common.shell_init_instructions(\n \"spack env deactivate\", \" eval `spack env deactivate {sh_arg}`\"\n )\n return 1\n\n # Error out when -e, -E, -D flags are given, cause they are ambiguous.\n if args.env or args.no_env or args.env_dir:\n tty.die(\"Calling spack env deactivate with --env, --env-dir and --no-env is ambiguous\")\n\n if ev.active_environment() is None:\n tty.die(\"No environment is currently active.\")\n\n cmds = spack.environment.shell.deactivate_header(args.shell)\n env_mods = spack.environment.shell.deactivate()\n cmds += env_mods.shell_modifications(args.shell)\n sys.stdout.write(cmds)\n\n\n#\n# env track\n#\ndef env_track_setup_parser(subparser):\n \"\"\"track an environment from a directory in Spack\"\"\"\n subparser.add_argument(\"-n\", \"--name\", help=\"custom environment name\")\n subparser.add_argument(\"dir\", help=\"path to environment\")\n arguments.add_common_arguments(subparser, [\"yes_to_all\"])\n\n\ndef env_track(args):\n src_path = os.path.abspath(args.dir)\n if not ev.is_env_dir(src_path):\n tty.die(\"Cannot track environment. Path doesn't contain an environment\")\n\n if args.name:\n name = args.name\n else:\n name = os.path.basename(src_path)\n\n try:\n dst_path = ev.environment_dir_from_name(name, exists_ok=False)\n except ev.SpackEnvironmentError:\n tty.die(\n f\"An environment named {name} already exists. Set a name with:\"\n \"\\n\\n\"\n f\" spack env track --name NAME {src_path}\\n\"\n )\n\n symlink(src_path, dst_path)\n\n tty.msg(f\"Tracking environment in {src_path}\")\n tty.msg(\n \"You can now activate this environment with the following command:\\n\\n\"\n f\" spack env activate {name}\\n\"\n )\n\n\n#\n# env remove & untrack helpers\n#\ndef filter_managed_env_names(env_names: Set[str]) -> Set[str]:\n tracked_env_names = {e for e in env_names if islink(ev.environment_dir_from_name(e))}\n managed_env_names = env_names - set(tracked_env_names)\n\n num_managed_envs = len(managed_env_names)\n managed_envs_str = \" \".join(managed_env_names)\n if num_managed_envs >= 2:\n tty.error(\n f\"The following are not tracked environments. \"\n \"To remove them completely run,\"\n \"\\n\\n\"\n f\" spack env rm {managed_envs_str}\\n\"\n )\n\n elif num_managed_envs > 0:\n tty.error(\n f\"'{managed_envs_str}' is not a tracked env. \"\n \"To remove it completely run,\"\n \"\\n\\n\"\n f\" spack env rm {managed_envs_str}\\n\"\n )\n\n return tracked_env_names\n\n\ndef get_valid_envs(env_names: Set[str]) -> Set[ev.Environment]:\n valid_envs = set()\n for env_name in env_names:\n try:\n env = ev.read(env_name)\n valid_envs.add(env)\n\n except (spack.config.ConfigFormatError, ev.SpackEnvironmentConfigError):\n pass\n\n return valid_envs\n\n\ndef _env_untrack_or_remove(\n env_names: List[str], remove: bool = False, force: bool = False, yes_to_all: bool = False\n):\n all_env_names = set(ev.all_environment_names())\n known_env_names = set(env_names).intersection(all_env_names)\n unknown_env_names = set(env_names) - known_env_names\n\n # print error for unknown environments\n for env_name in unknown_env_names:\n tty.error(f\"Environment '{env_name}' does not exist\")\n\n # if only unlinking is allowed, remove all environments\n # which do not point internally at symlinks\n if not remove:\n env_names_to_remove = filter_managed_env_names(known_env_names)\n else:\n env_names_to_remove = known_env_names\n\n # initialize all environments with valid spack.yaml configs\n all_valid_envs = get_valid_envs(all_env_names)\n\n # build a task list of environments and bad env names to remove\n envs_to_remove = [e for e in all_valid_envs if e.name in env_names_to_remove]\n bad_env_names_to_remove = env_names_to_remove - {e.name for e in envs_to_remove}\n for remove_env in envs_to_remove:\n for env in all_valid_envs:\n # don't check if an environment is included to itself\n if env.name == remove_env.name:\n continue\n\n # check if an environment is included in another\n if remove_env.path in env.included_concrete_env_root_dirs:\n msg = f\"Environment '{remove_env.name}' is used by environment '{env.name}'\"\n if force:\n tty.warn(msg)\n else:\n tty.error(msg)\n envs_to_remove.remove(remove_env)\n\n # ask the user if they really want to remove the known environments\n # force should do the same as yes to all here following the semantics of rm\n if not (yes_to_all or force) and (envs_to_remove or bad_env_names_to_remove):\n environments = string.plural(len(env_names_to_remove), \"environment\", show_n=False)\n envs = string.comma_and(list(env_names_to_remove))\n answer = tty.get_yes_or_no(\n f\"Really {'remove' if remove else 'untrack'} {environments} {envs}?\", default=False\n )\n if not answer:\n tty.msg(f\"Will not remove environment(s) {envs}\")\n return\n\n # keep track of the environments we remove for later printing the exit code\n removed_env_names = []\n for env in envs_to_remove:\n name = env.name\n if not force and env.active:\n tty.error(\n f\"Environment '{name}' can't be \"\n f\"{'removed' if remove else 'untracked'} while activated.\"\n )\n continue\n # Get path to check if environment is a tracked / symlinked environment\n if islink(env.path):\n real_env_path = os.path.realpath(env.path)\n os.unlink(env.path)\n tty.msg(\n f\"Successfully untracked environment '{name}', \"\n \"but it can still be found at:\\n\\n\"\n f\" {real_env_path}\\n\"\n )\n else:\n env.destroy()\n tty.msg(f\"Successfully removed environment '{name}'\")\n\n removed_env_names.append(env.name)\n\n for bad_env_name in bad_env_names_to_remove:\n shutil.rmtree(\n spack.environment.environment.environment_dir_from_name(bad_env_name, exists_ok=True)\n )\n tty.msg(f\"Successfully removed environment '{bad_env_name}'\")\n removed_env_names.append(bad_env_name)\n\n # Following the design of linux rm we should exit with a status of 1\n # anytime we cannot delete every environment the user asks for.\n # However, we should still process all the environments we know about\n # and delete them instead of failing on the first unknown environment.\n if len(removed_env_names) < len(known_env_names):\n sys.exit(1)\n\n\n#\n# env untrack\n#\ndef env_untrack_setup_parser(subparser):\n \"\"\"untrack an environment from a directory in Spack\"\"\"\n subparser.add_argument(\"env\", nargs=\"+\", help=\"tracked environment name\")\n subparser.add_argument(\n \"-f\", \"--force\", action=\"store_true\", help=\"force unlink even when environment is active\"\n )\n arguments.add_common_arguments(subparser, [\"yes_to_all\"])\n\n\ndef env_untrack(args):\n _env_untrack_or_remove(\n env_names=args.env, force=args.force, yes_to_all=args.yes_to_all, remove=False\n )\n\n\n#\n# env remove\n#\ndef env_remove_setup_parser(subparser):\n \"\"\"\\\n remove managed environment(s)\n\n remove existing environment(s) managed by Spack\n\n directory environments and manifests embedded in repositories must be\n removed manually\n \"\"\"\n subparser.add_argument(\n \"rm_env\", metavar=\"env\", nargs=\"+\", help=\"name(s) of the environment(s) being removed\"\n )\n arguments.add_common_arguments(subparser, [\"yes_to_all\"])\n subparser.add_argument(\n \"-f\",\n \"--force\",\n action=\"store_true\",\n help=\"force removal even when included in other environment(s)\",\n )\n\n\ndef env_remove(args):\n \"\"\"remove existing environment(s)\"\"\"\n _env_untrack_or_remove(\n env_names=args.rm_env, remove=True, force=args.force, yes_to_all=args.yes_to_all\n )\n\n\n#\n# env rename\n#\ndef env_rename_setup_parser(subparser):\n \"\"\"\\\n rename an existing environment\n\n rename a managed environment or move an independent/directory environment\n\n operation cannot be performed to or from an active environment\n \"\"\"\n subparser.add_argument(\n \"mv_from\", metavar=\"from\", help=\"current name or directory of the environment\"\n )\n subparser.add_argument(\"mv_to\", metavar=\"to\", help=\"new name or directory for the environment\")\n subparser.add_argument(\n \"-d\",\n \"--dir\",\n action=\"store_true\",\n help=\"positional arguments are environment directory paths\",\n )\n subparser.add_argument(\n \"-f\",\n \"--force\",\n action=\"store_true\",\n help=\"force renaming even if overwriting an existing environment\",\n )\n\n\ndef env_rename(args):\n \"\"\"rename or move an existing environment\"\"\"\n\n # Directory option has been specified\n if args.dir:\n if not ev.is_env_dir(args.mv_from):\n tty.die(\"The specified path does not correspond to a valid spack environment\")\n from_path = Path(args.mv_from)\n if not args.force:\n if ev.is_env_dir(args.mv_to):\n tty.die(\n \"The new path corresponds to an existing environment;\"\n \" specify the --force flag to overwrite it.\"\n )\n if Path(args.mv_to).exists():\n tty.die(\"The new path already exists; specify the --force flag to overwrite it.\")\n to_path = Path(args.mv_to)\n\n # Name option being used\n elif ev.exists(args.mv_from):\n from_path = ev.environment.environment_dir_from_name(args.mv_from)\n if not args.force and ev.exists(args.mv_to):\n tty.die(\n \"The new name corresponds to an existing environment;\"\n \" specify the --force flag to overwrite it.\"\n )\n to_path = ev.environment.root(args.mv_to)\n\n # Neither\n else:\n tty.die(\"The specified name does not correspond to a managed spack environment\")\n\n # Guard against renaming from or to an active environment\n active_env = ev.active_environment()\n if active_env:\n from_env = ev.Environment(from_path)\n if from_env.path == active_env.path:\n tty.die(\"Cannot rename active environment\")\n if to_path == active_env.path:\n tty.die(f\"{args.mv_to} is an active environment\")\n\n shutil.rmtree(to_path, ignore_errors=True)\n fs.rename(from_path, to_path)\n tty.msg(f\"Successfully renamed environment {args.mv_from} to {args.mv_to}\")\n\n\n#\n# env list\n#\ndef env_list_setup_parser(subparser):\n \"\"\"list all managed environments\"\"\"\n\n\ndef env_list(args):\n names = ev.all_environment_names()\n\n color_names = []\n for name in names:\n if ev.active(name):\n name = colorize(\"@*g{%s}\" % name)\n color_names.append(name)\n\n # say how many there are if writing to a tty\n if sys.stdout.isatty():\n if not names:\n tty.msg(\"No environments\")\n else:\n tty.msg(\"%d environments\" % len(names))\n\n colify(color_names, indent=4)\n\n\nclass ViewAction:\n regenerate = \"regenerate\"\n enable = \"enable\"\n disable = \"disable\"\n\n @staticmethod\n def actions():\n return [ViewAction.regenerate, ViewAction.enable, ViewAction.disable]\n\n\n#\n# env view\n#\ndef env_view_setup_parser(subparser):\n \"\"\"\\\n manage the environment's view\n\n provide the path when enabling a view with a non-default path\n \"\"\"\n subparser.add_argument(\n \"action\", choices=ViewAction.actions(), help=\"action to take for the environment's view\"\n )\n subparser.add_argument(\"view_path\", nargs=\"?\", help=\"view's non-default path when enabling it\")\n\n\ndef env_view(args):\n env = ev.active_environment()\n\n if not env:\n tty.msg(\"No active environment\")\n return\n\n if args.action == ViewAction.regenerate:\n env.regenerate_views()\n elif args.action == ViewAction.enable:\n if args.view_path:\n view_path = args.view_path\n else:\n view_path = env.view_path_default\n env.update_default_view(view_path)\n env.write()\n elif args.action == ViewAction.disable:\n env.update_default_view(path_or_bool=False)\n env.write()\n\n\n#\n# env status\n#\ndef env_status_setup_parser(subparser):\n \"\"\"print active environment status\"\"\"\n\n\ndef env_status(args):\n env = ev.active_environment()\n if env:\n if env.path == os.getcwd():\n tty.msg(\"Using %s in current directory: %s\" % (ev.manifest_name, env.path))\n else:\n tty.msg(\"In environment %s\" % env.name)\n\n # Check if environment views can be safely activated\n env.check_views()\n else:\n tty.msg(\"No active environment\")\n\n\n#\n# env loads\n#\ndef env_loads_setup_parser(subparser):\n \"\"\"list modules for an installed environment '(see spack module loads)'\"\"\"\n subparser.add_argument(\n \"-n\",\n \"--module-set-name\",\n default=\"default\",\n help=\"module set for which to generate load operations\",\n )\n subparser.add_argument(\n \"-m\",\n \"--module-type\",\n choices=(\"tcl\", \"lmod\"),\n help=\"type of module system to generate loads for\",\n )\n spack.cmd.modules.add_loads_arguments(subparser)\n\n\ndef env_loads(args):\n env = spack.cmd.require_active_env(cmd_name=\"env loads\")\n\n # Set the module types that have been selected\n module_type = args.module_type\n if module_type is None:\n # If no selection has been made select all of them\n module_type = \"tcl\"\n\n recurse_dependencies = args.recurse_dependencies\n args.recurse_dependencies = False\n\n loads_file = fs.join_path(env.path, \"loads\")\n with open(loads_file, \"w\", encoding=\"utf-8\") as f:\n if not recurse_dependencies:\n specs = [env.specs_by_hash[x.hash] for x in env.concretized_roots]\n else:\n specs = list(traverse_nodes(env.concrete_roots(), deptype=(\"link\", \"run\")))\n spack.cmd.modules.loads(module_type, specs, args, f)\n\n print(\"To load this environment, type:\")\n print(\" source %s\" % loads_file)\n\n\ndef env_update_setup_parser(subparser):\n \"\"\"\\\n update the environment manifest to the latest schema format\n\n update the environment to the latest schema format, which may not be\n readable by older versions of spack\n\n a backup copy of the manifest is retained in case there is a need to revert\n this operation\n \"\"\"\n subparser.add_argument(\n metavar=\"env\", dest=\"update_env\", help=\"name or directory of the environment\"\n )\n spack.cmd.common.arguments.add_common_arguments(subparser, [\"yes_to_all\"])\n\n\ndef env_update(args):\n \"\"\"update the manifest to the latest format\"\"\"\n manifest_file = ev.manifest_file(args.update_env)\n backup_file = manifest_file + \".bkp\"\n\n needs_update = not ev.is_latest_format(manifest_file)\n if not needs_update:\n tty.msg('No update needed for the environment \"{0}\"'.format(args.update_env))\n return\n\n proceed = True\n if not args.yes_to_all:\n msg = (\n 'The environment \"{0}\" is going to be updated to the latest '\n \"schema format.\\nIf the environment is updated, versions of \"\n \"Spack that are older than this version may not be able to \"\n \"read it. Spack stores backups of the updated environment \"\n 'which can be retrieved with \"spack env revert\"'\n )\n tty.msg(msg.format(args.update_env))\n proceed = tty.get_yes_or_no(\"Do you want to proceed?\", default=False)\n\n if not proceed:\n tty.die(\"Operation aborted.\")\n\n ev.update_yaml(manifest_file, backup_file=backup_file)\n msg = 'Environment \"{0}\" has been updated [backup={1}]'\n tty.msg(msg.format(args.update_env, backup_file))\n\n\ndef env_revert_setup_parser(subparser):\n \"\"\"\\\n restore the environment manifest to its previous format\n\n revert the environment's manifest to the schema format from its last\n 'spack env update'\n\n the current manifest will be overwritten by the backup copy and the backup\n copy will be removed\n \"\"\"\n subparser.add_argument(\n metavar=\"env\", dest=\"revert_env\", help=\"name or directory of the environment\"\n )\n spack.cmd.common.arguments.add_common_arguments(subparser, [\"yes_to_all\"])\n\n\ndef env_revert(args):\n \"\"\"restore the environment manifest to its previous format\"\"\"\n manifest_file = ev.manifest_file(args.revert_env)\n backup_file = manifest_file + \".bkp\"\n\n # Check that both the spack.yaml and the backup exist, the inform user\n # on what is going to happen and ask for confirmation\n if not os.path.exists(manifest_file):\n msg = \"cannot find the manifest file of the environment [file={0}]\"\n tty.die(msg.format(manifest_file))\n if not os.path.exists(backup_file):\n msg = \"cannot find the old manifest file to be restored [file={0}]\"\n tty.die(msg.format(backup_file))\n\n proceed = True\n if not args.yes_to_all:\n msg = (\n \"Spack is going to overwrite the current manifest file\"\n \" with a backup copy [manifest={0}, backup={1}]\"\n )\n tty.msg(msg.format(manifest_file, backup_file))\n proceed = tty.get_yes_or_no(\"Do you want to proceed?\", default=False)\n\n if not proceed:\n tty.die(\"Operation aborted.\")\n\n shutil.copy(backup_file, manifest_file)\n os.remove(backup_file)\n msg = 'Environment \"{0}\" reverted to old state'\n tty.msg(msg.format(manifest_file))\n\n\ndef env_depfile_setup_parser(subparser):\n \"\"\"\\\n generate a depfile to exploit parallel builds across specs\n\n requires the active environment to be concrete\n \"\"\"\n subparser.add_argument(\n \"--make-prefix\",\n \"--make-target-prefix\",\n default=None,\n metavar=\"TARGET\",\n help=\"prefix Makefile targets/variables with /,\\n\"\n \"which can be an empty string (--make-prefix '')\\n\"\n \"defaults to the absolute path of the environment's makedeps\\n\"\n \"environment metadata dir\\n\",\n )\n subparser.add_argument(\n \"--make-disable-jobserver\",\n default=True,\n action=\"store_false\",\n dest=\"jobserver\",\n help=\"disable POSIX jobserver support\",\n )\n subparser.add_argument(\n \"--use-buildcache\",\n dest=\"use_buildcache\",\n type=arguments.use_buildcache,\n default=\"package:auto,dependencies:auto\",\n metavar=\"[{auto,only,never},][package:{auto,only,never},][dependencies:{auto,only,never}]\",\n help=\"use `only` to prune redundant build dependencies\\n\"\n \"option is also passed to generated spack install commands\",\n )\n subparser.add_argument(\n \"-o\",\n \"--output\",\n default=None,\n metavar=\"FILE\",\n help=\"write the depfile to FILE rather than to stdout\",\n )\n subparser.add_argument(\n \"-G\",\n \"--generator\",\n default=\"make\",\n choices=(\"make\",),\n help=\"specify the depfile type (only supports `make`)\",\n )\n subparser.add_argument(\n metavar=\"specs\",\n dest=\"specs\",\n nargs=argparse.REMAINDER,\n default=None,\n help=\"limit the generated file to matching specs\",\n )\n\n\ndef env_depfile(args):\n # Currently only make is supported.\n spack.cmd.require_active_env(cmd_name=\"env depfile\")\n\n env = ev.active_environment()\n\n # What things do we build when running make? By default, we build the\n # root specs. If specific specs are provided as input, we build those.\n filter_specs = spack.cmd.parse_specs(args.specs) if args.specs else None\n template = spack.tengine.make_environment().get_template(os.path.join(\"depfile\", \"Makefile\"))\n model = depfile.MakefileModel.from_env(\n env,\n filter_specs=filter_specs,\n pkg_buildcache=depfile.UseBuildCache.from_string(args.use_buildcache[0]),\n dep_buildcache=depfile.UseBuildCache.from_string(args.use_buildcache[1]),\n make_prefix=args.make_prefix,\n jobserver=args.jobserver,\n )\n\n # Warn in case we're generating a depfile for an empty environment. We don't automatically\n # concretize; the user should do that explicitly. Could be changed in the future if requested.\n if model.empty:\n if not env.user_specs:\n tty.warn(\"no specs in the environment\")\n elif filter_specs is not None:\n tty.warn(\"no concrete matching specs found in environment\")\n else:\n tty.warn(\"environment is not concretized. Run `spack concretize` first\")\n\n makefile = template.render(model.to_dict())\n\n # Finally write to stdout/file.\n if args.output:\n with open(args.output, \"w\", encoding=\"utf-8\") as f:\n f.write(makefile)\n else:\n sys.stdout.write(makefile)\n\n\n#: Dictionary mapping subcommand names and aliases to functions\nsubcommand_functions = {}\n\n\n#\n# spack env\n#\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n sp = subparser.add_subparsers(metavar=\"SUBCOMMAND\", dest=\"env_command\")\n\n _globals = globals()\n\n for name_and_aliases in subcommands:\n name, aliases = name_and_aliases[0], name_and_aliases[1:]\n\n # add commands to subcommands dict\n for alias in name_and_aliases:\n subcommand_functions[alias] = _globals[f\"env_{name}\"]\n\n # make a subparser and run the command's setup function on it\n setup_parser_cmd = _globals[f\"env_{name}_setup_parser\"]\n\n subsubparser = sp.add_parser(\n name,\n aliases=aliases,\n description=spack.cmd.doc_dedented(setup_parser_cmd),\n help=spack.cmd.doc_first_line(setup_parser_cmd),\n )\n setup_parser_cmd(subsubparser)\n\n\ndef env(parser, args):\n \"\"\"Look for a function called environment_ and call it.\"\"\"\n action = subcommand_functions[args.env_command]\n action(args)\n" }, { "path": "lib/spack/spack/cmd/extensions.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport sys\n\nimport spack.cmd as cmd\nimport spack.environment as ev\nimport spack.llnl.util.tty as tty\nimport spack.repo\nimport spack.store\nfrom spack.cmd.common import arguments\nfrom spack.llnl.util.tty.colify import colify\n\ndescription = \"list extensions for package\"\nsection = \"query\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n subparser.epilog = (\n \"If called without argument returns the list of all valid extendable packages\"\n )\n arguments.add_common_arguments(subparser, [\"long\", \"very_long\"])\n subparser.add_argument(\n \"-d\", \"--deps\", action=\"store_true\", help=\"output dependencies along with found specs\"\n )\n\n subparser.add_argument(\n \"-p\", \"--paths\", action=\"store_true\", help=\"show paths to package install directories\"\n )\n subparser.add_argument(\n \"-s\",\n \"--show\",\n action=\"store\",\n default=\"all\",\n choices=(\"packages\", \"installed\", \"all\"),\n help=\"show only part of output\",\n )\n\n subparser.add_argument(\n \"spec\",\n nargs=argparse.REMAINDER,\n help=\"spec of package to list extensions for\",\n metavar=\"extendable\",\n )\n\n\ndef extensions(parser, args):\n if not args.spec:\n # If called without arguments, list all the extendable packages\n isatty = sys.stdout.isatty()\n if isatty:\n tty.info(\"Extendable packages:\")\n\n extendable_pkgs = []\n for name in spack.repo.all_package_names():\n pkg_cls = spack.repo.PATH.get_pkg_class(name)\n if pkg_cls.extendable:\n extendable_pkgs.append(name)\n\n colify(extendable_pkgs, indent=4)\n return\n\n # Checks\n spec = cmd.parse_specs(args.spec)\n if len(spec) > 1:\n tty.die(\"Can only list extensions for one package.\")\n\n env = ev.active_environment()\n spec = cmd.disambiguate_spec(spec[0], env)\n\n if not spec.package.extendable:\n tty.die(\"%s is not an extendable package.\" % spec.name)\n\n if not spec.package.extendable:\n tty.die(\"%s does not have extensions.\" % spec.short_spec)\n\n if args.show in (\"packages\", \"all\"):\n # List package names of extensions\n extensions = spack.repo.PATH.extensions_for(spec)\n if not extensions:\n tty.msg(\"%s has no extensions.\" % spec.cshort_spec)\n else:\n tty.msg(spec.cshort_spec)\n tty.msg(\"%d extensions:\" % len(extensions))\n colify(ext.name for ext in extensions)\n\n if args.show in (\"installed\", \"all\"):\n # List specs of installed extensions.\n installed = [s.spec for s in spack.store.STORE.db.installed_extensions_for(spec)]\n\n if args.show == \"all\":\n print\n if not installed:\n tty.msg(\"None installed.\")\n else:\n tty.msg(\"%d installed:\" % len(installed))\n cmd.display_specs(installed, args)\n" }, { "path": "lib/spack/spack/cmd/external.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport argparse\nimport errno\nimport os\nimport re\nimport sys\nfrom typing import List, Optional, Set\n\nimport spack\nimport spack.cmd\nimport spack.config\nimport spack.cray_manifest as cray_manifest\nimport spack.detection\nimport spack.error\nimport spack.llnl.util.tty as tty\nimport spack.llnl.util.tty.colify as colify\nimport spack.package_base\nimport spack.repo\nimport spack.spec\nfrom spack.cmd.common import arguments\n\ndescription = \"manage external packages in Spack configuration\"\nsection = \"config\"\nlevel = \"short\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n sp = subparser.add_subparsers(metavar=\"SUBCOMMAND\", dest=\"external_command\")\n\n find_parser = sp.add_parser(\"find\", help=\"add external packages to packages.yaml\")\n find_parser.add_argument(\n \"--not-buildable\",\n action=\"store_true\",\n default=False,\n help=\"packages with detected externals won't be built with Spack\",\n )\n find_parser.add_argument(\"--exclude\", action=\"append\", help=\"packages to exclude from search\")\n find_parser.add_argument(\n \"-p\",\n \"--path\",\n default=None,\n action=\"append\",\n help=\"one or more alternative search paths for finding externals\",\n )\n find_parser.add_argument(\n \"--scope\",\n action=arguments.ConfigScope,\n default=lambda: spack.config.default_modify_scope(\"packages\"),\n help=\"configuration scope to modify\",\n )\n find_parser.add_argument(\n \"--all\", action=\"store_true\", help=\"search for all packages that Spack knows about\"\n )\n arguments.add_common_arguments(find_parser, [\"tags\", \"jobs\"])\n find_parser.add_argument(\"packages\", nargs=argparse.REMAINDER)\n find_parser.epilog = (\n 'The search is by default on packages tagged with the \"build-tools\" or '\n '\"core-packages\" tags. Use the --all option to search for every possible '\n \"package Spack knows how to find.\"\n )\n\n sp.add_parser(\"list\", aliases=[\"ls\"], help=\"list detectable packages, by repository and name\")\n\n read_cray_manifest = sp.add_parser(\n \"read-cray-manifest\",\n help=\"consume a Spack-compatible description of externally-installed packages, including \"\n \"dependency relationships\",\n )\n read_cray_manifest.add_argument(\n \"--file\", default=None, help=\"specify a location other than the default\"\n )\n read_cray_manifest.add_argument(\n \"--directory\", default=None, help=\"specify a directory storing a group of manifest files\"\n )\n read_cray_manifest.add_argument(\n \"--ignore-default-dir\",\n action=\"store_true\",\n default=False,\n help=\"ignore the default directory of manifest files\",\n )\n read_cray_manifest.add_argument(\n \"--dry-run\",\n action=\"store_true\",\n default=False,\n help=\"don't modify DB with files that are read\",\n )\n read_cray_manifest.add_argument(\n \"--fail-on-error\",\n action=\"store_true\",\n help=\"if a manifest file cannot be parsed, fail and report the full stack trace\",\n )\n\n\ndef external_find(args):\n if args.all or not (args.tags or args.packages):\n # If the user calls 'spack external find' with no arguments, and\n # this system has a description of installed packages, then we should\n # consume it automatically.\n try:\n _collect_and_consume_cray_manifest_files()\n except NoManifestFileError:\n # It's fine to not find any manifest file if we are doing the\n # search implicitly (i.e. as part of 'spack external find')\n pass\n except Exception as e:\n # For most exceptions, just print a warning and continue.\n # Note that KeyboardInterrupt does not subclass Exception\n # (so CTRL-C will terminate the program as expected).\n skip_msg = \"Skipping manifest and continuing with other external checks\"\n if isinstance(e, OSError) and e.errno in (errno.EPERM, errno.EACCES):\n # The manifest file does not have sufficient permissions enabled:\n # print a warning and keep going\n tty.warn(\"Unable to read manifest due to insufficient permissions.\", skip_msg)\n else:\n tty.warn(\"Unable to read manifest, unexpected error: {0}\".format(str(e)), skip_msg)\n\n # Outside the Cray manifest, the search is done by tag for performance reasons,\n # since tags are cached.\n\n # If the user specified both --all and --tag, then --all has precedence\n if args.all or args.packages:\n # Each detectable package has at least the detectable tag\n args.tags = [\"detectable\"]\n elif not args.tags:\n # If the user didn't specify anything, search for build tools by default\n args.tags = [\"core-packages\", \"build-tools\"]\n\n candidate_packages = packages_to_search_for(\n names=args.packages, tags=args.tags, exclude=args.exclude\n )\n detected_packages = spack.detection.by_path(\n candidate_packages, path_hints=args.path, max_workers=args.jobs\n )\n\n new_specs = spack.detection.update_configuration(\n detected_packages, scope=args.scope, buildable=not args.not_buildable\n )\n\n # If the user runs `spack external find --not-buildable mpich` we also mark `mpi` non-buildable\n # to avoid that the concretizer picks a different mpi provider.\n if new_specs and args.not_buildable:\n virtuals: Set[str] = {\n virtual.name\n for new_spec in new_specs\n for virtual_specs in spack.repo.PATH.get_pkg_class(new_spec.name).provided.values()\n for virtual in virtual_specs\n }\n new_virtuals = spack.detection.set_virtuals_nonbuildable(virtuals, scope=args.scope)\n new_specs.extend(spack.spec.Spec(name) for name in new_virtuals)\n\n if new_specs:\n path = spack.config.CONFIG.get_config_filename(args.scope, \"packages\")\n tty.msg(f\"The following specs have been detected on this system and added to {path}\")\n spack.cmd.display_specs(new_specs)\n else:\n tty.msg(\"No new external packages detected\")\n\n\ndef packages_to_search_for(\n *, names: Optional[List[str]], tags: List[str], exclude: Optional[List[str]]\n):\n result = list(\n {pkg for tag in tags for pkg in spack.repo.PATH.packages_with_tags(tag, full=True)}\n )\n\n if names:\n # Match both fully qualified and unqualified\n parts = [rf\"(^{x}$|[.]{x}$)\" for x in names]\n select_re = re.compile(\"|\".join(parts))\n result = [x for x in result if select_re.search(x)]\n\n if exclude:\n # Match both fully qualified and unqualified\n parts = [rf\"(^{x}$|[.]{x}$)\" for x in exclude]\n select_re = re.compile(\"|\".join(parts))\n result = [x for x in result if not select_re.search(x)]\n\n return result\n\n\ndef external_read_cray_manifest(args):\n _collect_and_consume_cray_manifest_files(\n manifest_file=args.file,\n manifest_directory=args.directory,\n dry_run=args.dry_run,\n fail_on_error=args.fail_on_error,\n ignore_default_dir=args.ignore_default_dir,\n )\n\n\ndef _collect_and_consume_cray_manifest_files(\n manifest_file=None,\n manifest_directory=None,\n dry_run=False,\n fail_on_error=False,\n ignore_default_dir=False,\n):\n manifest_files = []\n if manifest_file:\n manifest_files.append(manifest_file)\n\n manifest_dirs = []\n if manifest_directory:\n manifest_dirs.append(manifest_directory)\n\n if not ignore_default_dir and os.path.isdir(cray_manifest.default_path):\n tty.debug(\n \"Cray manifest path {0} exists: collecting all files to read.\".format(\n cray_manifest.default_path\n )\n )\n manifest_dirs.append(cray_manifest.default_path)\n else:\n tty.debug(\n \"Default Cray manifest directory {0} does not exist.\".format(\n cray_manifest.default_path\n )\n )\n\n for directory in manifest_dirs:\n for fname in os.listdir(directory):\n if fname.endswith(\".json\"):\n fpath = os.path.join(directory, fname)\n tty.debug(\"Adding manifest file: {0}\".format(fpath))\n manifest_files.append(os.path.join(directory, fpath))\n\n if not manifest_files:\n raise NoManifestFileError(\n \"--file/--directory not specified, and no manifest found at {0}\".format(\n cray_manifest.default_path\n )\n )\n\n for path in manifest_files:\n tty.debug(\"Reading manifest file: \" + path)\n try:\n cray_manifest.read(path, not dry_run)\n except spack.error.SpackError as e:\n if fail_on_error:\n raise\n else:\n tty.warn(\"Failure reading manifest file: {0}\\n\\t{1}\".format(path, str(e)))\n\n\ndef external_list(args):\n # Trigger a read of all packages, might take a long time.\n list(spack.repo.PATH.all_package_classes())\n # Print all the detectable packages\n tty.msg(\"Detectable packages per repository\")\n for namespace, pkgs in sorted(spack.package_base.detectable_packages.items()):\n print(\"Repository:\", namespace)\n colify.colify(pkgs, indent=4, output=sys.stdout)\n\n\ndef external(parser, args):\n action = {\n \"find\": external_find,\n \"list\": external_list,\n \"ls\": external_list,\n \"read-cray-manifest\": external_read_cray_manifest,\n }\n action[args.external_command](args)\n\n\nclass NoManifestFileError(spack.error.SpackError):\n pass\n" }, { "path": "lib/spack/spack/cmd/fetch.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\n\nimport spack.cmd\nimport spack.config\nimport spack.environment as ev\nimport spack.llnl.util.tty as tty\nimport spack.traverse\nfrom spack.cmd.common import arguments\n\ndescription = \"fetch archives for packages\"\nsection = \"build\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n arguments.add_common_arguments(subparser, [\"no_checksum\", \"specs\"])\n subparser.add_argument(\n \"-m\",\n \"--missing\",\n action=\"store_true\",\n help=\"fetch only missing (not yet installed) dependencies\",\n )\n subparser.add_argument(\n \"-D\", \"--dependencies\", action=\"store_true\", help=\"also fetch all dependencies\"\n )\n arguments.add_concretizer_args(subparser)\n subparser.epilog = (\n \"With an active environment, the specs \"\n \"parameter can be omitted. In this case all (uninstalled\"\n \", in case of --missing) specs from the environment are fetched\"\n )\n\n\ndef fetch(parser, args):\n if args.no_checksum:\n spack.config.set(\"config:checksum\", False, scope=\"command_line\")\n\n if args.specs:\n specs = spack.cmd.parse_specs(args.specs, concretize=True)\n else:\n # No specs were given explicitly, check if we are in an\n # environment. If yes, check the missing argument, if yes\n # fetch all uninstalled specs from it otherwise fetch all.\n # If we are also not in an environment, complain to the\n # user that we don't know what to do.\n env = ev.active_environment()\n if env:\n if args.missing:\n specs = env.uninstalled_specs()\n else:\n specs = env.all_specs()\n if specs == []:\n tty.die(\"No uninstalled specs in environment. Did you run `spack concretize` yet?\")\n else:\n tty.die(\"fetch requires at least one spec argument\")\n\n if args.dependencies or args.missing:\n to_be_fetched = spack.traverse.traverse_nodes(specs, key=spack.traverse.by_dag_hash)\n else:\n to_be_fetched = specs\n\n for spec in to_be_fetched:\n if args.missing and spec.installed:\n continue\n\n pkg = spec.package\n\n pkg.stage.keep = True\n with pkg.stage:\n pkg.do_fetch()\n" }, { "path": "lib/spack/spack/cmd/find.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport copy\nimport sys\n\nimport spack.cmd as cmd\nimport spack.config\nimport spack.environment as ev\nimport spack.llnl.util.lang\nimport spack.llnl.util.tty as tty\nimport spack.llnl.util.tty.color as color\nimport spack.repo\nimport spack.solver.reuse\nimport spack.spec\nimport spack.store\nfrom spack.cmd.common import arguments\nfrom spack.llnl.util.tty.color import colorize\nfrom spack.solver.reuse import create_external_parser\nfrom spack.solver.runtimes import external_config_with_implicit_externals\n\nfrom ..enums import InstallRecordStatus\n\ndescription = \"list and search installed packages\"\nsection = \"query\"\nlevel = \"short\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n format_group = subparser.add_mutually_exclusive_group()\n format_group.add_argument(\n \"--format\",\n action=\"store\",\n default=None,\n help=\"output specs with the specified format string\",\n )\n format_group.add_argument(\n \"-H\",\n \"--hashes\",\n action=\"store_const\",\n dest=\"format\",\n const=\"{/hash}\",\n help=\"same as ``--format {/hash}``; use with ``xargs`` or ``$()``\",\n )\n format_group.add_argument(\n \"--json\",\n action=\"store_true\",\n default=False,\n help=\"output specs as machine-readable json records\",\n )\n\n subparser.add_argument(\n \"-I\", \"--install-status\", action=\"store_true\", help=\"show install status of packages\"\n )\n\n subparser.add_argument(\n \"--specfile-format\",\n action=\"store_true\",\n help=\"show the specfile format for installed deps \",\n )\n\n subparser.add_argument(\n \"-d\", \"--deps\", action=\"store_true\", help=\"output dependencies along with found specs\"\n )\n\n subparser.add_argument(\n \"-p\", \"--paths\", action=\"store_true\", help=\"show paths to package install directories\"\n )\n subparser.add_argument(\n \"--groups\",\n action=\"store_true\",\n default=None,\n dest=\"groups\",\n help=\"display specs in arch/compiler groups (default on)\",\n )\n subparser.add_argument(\n \"--no-groups\",\n action=\"store_false\",\n default=None,\n dest=\"groups\",\n help=\"do not group specs by arch/compiler\",\n )\n\n arguments.add_common_arguments(subparser, [\"long\", \"very_long\", \"tags\", \"namespaces\"])\n\n subparser.add_argument(\n \"-r\",\n \"--only-roots\",\n action=\"store_true\",\n help=\"don't show full list of installed specs in an environment\",\n )\n concretized_vs_packages = subparser.add_mutually_exclusive_group()\n concretized_vs_packages.add_argument(\n \"-c\",\n \"--show-concretized\",\n action=\"store_true\",\n help=\"show concretized specs in an environment\",\n )\n concretized_vs_packages.add_argument(\n \"--show-configured-externals\",\n action=\"store_true\",\n help=\"show externals defined in the 'packages' section of the configuration\",\n )\n subparser.add_argument(\n \"-f\",\n \"--show-flags\",\n action=\"store_true\",\n dest=\"show_flags\",\n help=\"show spec compiler flags\",\n )\n subparser.add_argument(\n \"--show-full-compiler\",\n action=\"store_true\",\n dest=\"show_full_compiler\",\n help=\"(DEPRECATED) show full compiler specs. Currently it's a no-op\",\n )\n implicit_explicit = subparser.add_mutually_exclusive_group()\n implicit_explicit.add_argument(\n \"-x\",\n \"--explicit\",\n action=\"store_true\",\n help=\"show only specs that were installed explicitly\",\n )\n implicit_explicit.add_argument(\n \"-X\",\n \"--implicit\",\n action=\"store_true\",\n help=\"show only specs that were installed as dependencies\",\n )\n subparser.add_argument(\n \"-e\",\n \"--external\",\n action=\"store_true\",\n help=\"show only specs that are marked as externals\",\n )\n subparser.add_argument(\n \"-u\",\n \"--unknown\",\n action=\"store_true\",\n dest=\"unknown\",\n help=\"show only specs Spack does not have a package for\",\n )\n subparser.add_argument(\n \"-m\",\n \"--missing\",\n action=\"store_true\",\n dest=\"missing\",\n help=\"show missing dependencies as well as installed specs\",\n )\n subparser.add_argument(\n \"-v\",\n \"--variants\",\n action=\"store_true\",\n dest=\"variants\",\n help=\"show variants in output (can be long)\",\n )\n subparser.add_argument(\n \"--loaded\", action=\"store_true\", help=\"show only packages loaded in the user environment\"\n )\n only_missing_or_deprecated = subparser.add_mutually_exclusive_group()\n only_missing_or_deprecated.add_argument(\n \"-M\",\n \"--only-missing\",\n action=\"store_true\",\n dest=\"only_missing\",\n help=\"show only missing dependencies\",\n )\n only_missing_or_deprecated.add_argument(\n \"--only-deprecated\", action=\"store_true\", help=\"show only deprecated packages\"\n )\n subparser.add_argument(\n \"--deprecated\",\n action=\"store_true\",\n help=\"show deprecated packages as well as installed specs\",\n )\n subparser.add_argument(\n \"--install-tree\",\n action=\"store\",\n default=\"all\",\n help=\"Install trees to query: 'all' (default), 'local', 'upstream', upstream name or path\",\n )\n\n subparser.add_argument(\"--start-date\", help=\"earliest date of installation [YYYY-MM-DD]\")\n subparser.add_argument(\"--end-date\", help=\"latest date of installation [YYYY-MM-DD]\")\n arguments.add_common_arguments(subparser, [\"constraint\"])\n\n\ndef query_arguments(args):\n if args.only_missing and (args.deprecated or args.missing):\n raise RuntimeError(\"cannot use --only-missing with --deprecated, or --missing\")\n\n if args.only_deprecated and (args.deprecated or args.missing):\n raise RuntimeError(\"cannot use --only-deprecated with --deprecated, or --missing\")\n\n installed = InstallRecordStatus.INSTALLED\n if args.only_missing:\n installed = InstallRecordStatus.MISSING\n elif args.only_deprecated:\n installed = InstallRecordStatus.DEPRECATED\n\n if args.missing:\n installed |= InstallRecordStatus.MISSING\n\n if args.deprecated:\n installed |= InstallRecordStatus.DEPRECATED\n\n predicate_fn = None\n if args.unknown:\n predicate_fn = lambda x: not spack.repo.PATH.exists(x.spec.name)\n\n explicit = None\n if args.explicit:\n explicit = True\n if args.implicit:\n explicit = False\n\n q_args = {\"installed\": installed, \"predicate_fn\": predicate_fn, \"explicit\": explicit}\n\n install_tree = args.install_tree\n upstreams = spack.config.get(\"upstreams\", {})\n if install_tree in upstreams.keys():\n install_tree = upstreams[install_tree][\"install_tree\"]\n q_args[\"install_tree\"] = install_tree\n\n # Time window of installation\n for attribute in (\"start_date\", \"end_date\"):\n date = getattr(args, attribute)\n if date:\n q_args[attribute] = spack.llnl.util.lang.pretty_string_to_date(date)\n\n return q_args\n\n\ndef make_env_decorator(env):\n \"\"\"Create a function for decorating specs when in an environment.\"\"\"\n\n roots = set(env.roots())\n removed = set(env.removed_specs())\n\n def decorator(spec, fmt):\n # add +/-/* to show added/removed/root specs\n if any(spec.dag_hash() == r.dag_hash() for r in roots):\n return color.colorize(f\"@*{{{fmt}}}\")\n elif spec in removed:\n return color.colorize(f\"@K{{{fmt}}}\")\n else:\n return fmt\n\n return decorator\n\n\ndef display_env(env, args, decorator, results):\n \"\"\"Display extra find output when running in an environment.\n\n In an environment, ``spack find`` outputs a preliminary section\n showing the root specs of the environment (this is in addition\n to the section listing out specs matching the query parameters).\n \"\"\"\n total_roots = sum(len(env.user_specs_by(group=g)) for g in env.manifest.groups())\n root_spec_str = f\"{total_roots or 'no'} root {'spec' if total_roots == 1 else 'specs'}\"\n tty.msg(f\"In environment {env.name} ({root_spec_str})\")\n\n concrete_specs = {x.root: env.specs_by_hash[x.hash] for x in env.concretized_roots}\n\n def root_decorator(spec, string):\n \"\"\"Decorate root specs with their install status if needed\"\"\"\n concrete = concrete_specs.get(spec)\n if concrete:\n status = color.colorize(concrete.install_status().value)\n hash = concrete.dag_hash()\n else:\n status = color.colorize(spack.spec.InstallStatus.absent.value)\n hash = \"-\" * 32\n\n # TODO: status has two extra spaces on the end of it, but fixing this and other spec\n # TODO: space format idiosyncrasies is complicated. Fix this eventually\n status = status[:-2]\n\n if args.long or args.very_long:\n hash = color.colorize(f\"@K{{{hash[: 7 if args.long else None]}}}\")\n return f\"{status} {hash} {string}\"\n else:\n return f\"{status} {string}\"\n\n with spack.store.STORE.db.read_transaction():\n for group in env.manifest.groups():\n group_specs = env.user_specs_by(group=group)\n if not group_specs:\n continue\n\n if env.has_groups():\n header = (\n f\"{spack.spec.ARCHITECTURE_COLOR}{{root specs}} / \"\n f\"{spack.spec.COMPILER_COLOR}{{{group}}}\"\n )\n tty.hline(colorize(header), char=\"-\")\n\n cmd.display_specs(\n group_specs,\n args,\n # these are overrides of CLI args\n paths=False,\n long=False,\n very_long=False,\n # these enforce details in the root specs to show what the user asked for\n groups=False,\n namespaces=True,\n show_flags=True,\n decorator=root_decorator,\n variants=True,\n specfile_format=args.specfile_format,\n )\n print()\n\n if env.included_concrete_env_root_dirs:\n tty.msg(\"Included specs\")\n\n # Root specs cannot be displayed with prefixes, since those are not\n # set for abstract specs. Same for hashes\n root_args = copy.copy(args)\n root_args.paths = False\n\n # Roots are displayed with variants, etc. so that we can see\n # specifically what the user asked for.\n cmd.display_specs(\n env.included_user_specs,\n root_args,\n decorator=lambda s, f: color.colorize(\"@*{%s}\" % f),\n namespace=True,\n show_flags=True,\n variants=True,\n specfile_format=args.specfile_format,\n )\n print()\n\n\ndef _find_query(args, env):\n q_args = query_arguments(args)\n concretized_but_not_installed = []\n if args.show_configured_externals:\n packages_with_externals = external_config_with_implicit_externals(spack.config.CONFIG)\n completion_mode = spack.config.CONFIG.get(\"concretizer:externals:completion\")\n results = spack.solver.reuse.spec_filter_from_packages_yaml(\n external_parser=create_external_parser(packages_with_externals, completion_mode),\n packages_with_externals=packages_with_externals,\n include=[],\n exclude=[],\n ).selected_specs()\n elif env:\n all_env_specs = env.all_specs()\n if args.constraint:\n init_specs = cmd.parse_specs(args.constraint)\n env_specs = env.all_matching_specs(*init_specs)\n else:\n env_specs = all_env_specs\n\n spec_hashes = set(x.dag_hash() for x in env_specs)\n specs_meeting_q_args = set(spack.store.STORE.db.query(hashes=spec_hashes, **q_args))\n\n results = list()\n with spack.store.STORE.db.read_transaction():\n for spec in env_specs:\n if not spec.installed:\n concretized_but_not_installed.append(spec)\n if spec in specs_meeting_q_args:\n results.append(spec)\n else:\n results = args.specs(**q_args)\n\n if args.external:\n results = [s for s in results if s.external]\n\n # use groups by default except with format.\n if args.groups is None:\n args.groups = not args.format\n\n # Exit early with an error code if no package matches the constraint\n if concretized_but_not_installed and args.show_concretized:\n pass\n elif results:\n pass\n elif args.constraint:\n raise cmd.NoSpecMatches()\n\n # If tags have been specified on the command line, filter by tags\n if args.tags:\n packages_with_tags = spack.repo.PATH.packages_with_tags(*args.tags)\n results = [x for x in results if x.name in packages_with_tags]\n concretized_but_not_installed = [\n x for x in concretized_but_not_installed if x.name in packages_with_tags\n ]\n\n if args.loaded:\n results = cmd.filter_loaded_specs(results)\n\n return results, concretized_but_not_installed\n\n\ndef find(parser, args):\n env = ev.active_environment()\n\n if not env and args.only_roots:\n tty.die(\"-r / --only-roots requires an active environment\")\n if not env and args.show_concretized:\n tty.die(\"-c / --show-concretized requires an active environment\")\n\n try:\n results, concretized_but_not_installed = _find_query(args, env)\n except cmd.NoSpecMatches:\n # Note: this uses args.constraint vs. args.constraint_specs because\n # the latter only exists if you call args.specs()\n tty.die(f\"No package matches the query: {' '.join(args.constraint)}\")\n\n if args.install_status or args.show_concretized:\n status_fn = spack.spec.Spec.install_status\n else:\n status_fn = None\n\n # Display the result\n if args.json:\n cmd.display_specs_as_json(results, deps=args.deps)\n else:\n decorator = make_env_decorator(env) if env else lambda s, f: f\n\n if not args.format:\n if env:\n display_env(env, args, decorator, results)\n\n if not args.only_roots:\n display_results = list(results)\n if args.show_concretized:\n display_results += concretized_but_not_installed\n cmd.display_specs(\n display_results,\n args,\n decorator=decorator,\n all_headers=True,\n status_fn=status_fn,\n specfile_format=args.specfile_format,\n )\n\n # print number of installed packages last (as the list may be long)\n if sys.stdout.isatty() and args.groups:\n installed_suffix = \"\"\n concretized_suffix = \" to be installed\"\n\n if args.only_roots:\n installed_suffix += \" (not shown)\"\n concretized_suffix += \" (not shown)\"\n else:\n if env and not args.show_concretized:\n concretized_suffix += \" (show with `spack find -c`)\"\n\n pkg_type = \"loaded\" if args.loaded else \"installed\"\n cmd.print_how_many_pkgs(results, pkg_type, suffix=installed_suffix)\n\n if env:\n cmd.print_how_many_pkgs(\n concretized_but_not_installed, \"concretized\", suffix=concretized_suffix\n )\n" }, { "path": "lib/spack/spack/cmd/gc.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\n\nimport spack.cmd.common.arguments\nimport spack.cmd.common.confirmation\nimport spack.cmd.uninstall\nimport spack.deptypes as dt\nimport spack.environment as ev\nimport spack.llnl.util.tty as tty\nimport spack.store\n\ndescription = \"remove specs that are now no longer needed\"\nsection = \"build\"\nlevel = \"short\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n subparser.add_argument(\n \"-E\",\n \"--except-any-environment\",\n action=\"store_true\",\n help=\"remove everything unless needed by an environment\",\n )\n subparser.add_argument(\n \"-e\",\n \"--except-environment\",\n metavar=\"ENV\",\n action=\"append\",\n default=[],\n help=\"remove everything unless needed by specified environment\\n\"\n \"you can list multiple environments, or specify directory\\n\"\n \"environments by path.\",\n )\n subparser.add_argument(\n \"-b\",\n \"--keep-build-dependencies\",\n action=\"store_true\",\n help=\"do not remove installed build-only dependencies of roots\\n\"\n \"(default is to keep only link & run dependencies)\",\n )\n spack.cmd.common.arguments.add_common_arguments(subparser, [\"yes_to_all\", \"constraint\"])\n\n\ndef roots_from_environments(args, active_env):\n # if we're using -E or -e, make a list of environments whose roots we should consider.\n all_environments = []\n\n # -E will garbage collect anything not needed by any env, including the current one\n if args.except_any_environment:\n all_environments += list(ev.all_environments())\n if active_env:\n all_environments.append(active_env)\n\n # -e says \"also preserve things needed by this particular env\"\n for env_name_or_dir in args.except_environment:\n if ev.exists(env_name_or_dir):\n env = ev.read(env_name_or_dir)\n elif ev.is_env_dir(env_name_or_dir):\n env = ev.Environment(env_name_or_dir)\n else:\n tty.die(f\"No such environment: '{env_name_or_dir}'\")\n all_environments.append(env)\n\n # add root hashes from all considered environments to list of roots\n root_hashes = set()\n for env in all_environments:\n root_hashes |= {x.hash for x in env.explicit_roots()}\n\n return root_hashes\n\n\ndef gc(parser, args):\n deptype = dt.LINK | dt.RUN\n if args.keep_build_dependencies:\n deptype |= dt.BUILD\n\n active_env = ev.active_environment()\n\n # wrap the whole command with a read transaction to avoid multiple\n with spack.store.STORE.db.read_transaction():\n if args.except_environment or args.except_any_environment:\n # if either of these is specified, we ignore the active environment and garbage\n # collect anything NOT in specified environments.\n root_hashes = roots_from_environments(args, active_env)\n\n elif active_env:\n # only gc what's in current environment\n tty.msg(f\"Restricting garbage collection to environment '{active_env.name}'\")\n root_hashes = set(spack.store.STORE.db.all_hashes()) # keep everything\n root_hashes -= set(active_env.all_hashes()) # except this env\n # but keep its explicit roots\n root_hashes |= {x.hash for x in active_env.explicit_roots()}\n else:\n # consider all explicit specs roots (the default for db.unused_specs())\n root_hashes = None\n\n specs = spack.store.STORE.db.unused_specs(root_hashes=root_hashes, deptype=deptype)\n\n # limit search to constraint specs if provided\n if args.constraint:\n hashes = set(spec.dag_hash() for spec in args.specs())\n specs = [spec for spec in specs if spec.dag_hash() in hashes]\n\n if not specs:\n tty.msg(\"There are no unused specs. Spack's store is clean.\")\n return\n\n if not args.yes_to_all:\n spack.cmd.common.confirmation.confirm_action(specs, \"uninstalled\", \"uninstall\")\n\n spack.cmd.uninstall.do_uninstall(specs, force=False)\n" }, { "path": "lib/spack/spack/cmd/gpg.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport os\nimport tempfile\n\nimport spack.binary_distribution\nimport spack.mirrors.mirror\nimport spack.paths\nimport spack.stage\nimport spack.util.gpg\nimport spack.util.url\nfrom spack.cmd.common import arguments\n\ndescription = \"handle GPG actions for spack\"\nsection = \"packaging\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n setattr(setup_parser, \"parser\", subparser)\n subparsers = subparser.add_subparsers(help=\"GPG sub-commands\")\n\n verify = subparsers.add_parser(\"verify\", help=gpg_verify.__doc__)\n arguments.add_common_arguments(verify, [\"installed_spec\"])\n verify.add_argument(\"signature\", type=str, nargs=\"?\", help=\"the signature file\")\n verify.set_defaults(func=gpg_verify)\n\n trust = subparsers.add_parser(\"trust\", help=gpg_trust.__doc__)\n trust.add_argument(\"keyfile\", type=str, help=\"add a key to the trust store\")\n trust.set_defaults(func=gpg_trust)\n\n untrust = subparsers.add_parser(\"untrust\", help=gpg_untrust.__doc__)\n untrust.add_argument(\"--signing\", action=\"store_true\", help=\"allow untrusting signing keys\")\n untrust.add_argument(\"keys\", nargs=\"+\", type=str, help=\"remove keys from the trust store\")\n untrust.set_defaults(func=gpg_untrust)\n\n sign = subparsers.add_parser(\"sign\", help=gpg_sign.__doc__)\n sign.add_argument(\n \"--output\", metavar=\"DEST\", type=str, help=\"the directory to place signatures\"\n )\n sign.add_argument(\"--key\", metavar=\"KEY\", type=str, help=\"the key to use for signing\")\n sign.add_argument(\n \"--clearsign\", action=\"store_true\", help=\"if specified, create a clearsign signature\"\n )\n arguments.add_common_arguments(sign, [\"installed_spec\"])\n sign.set_defaults(func=gpg_sign)\n\n create = subparsers.add_parser(\"create\", help=gpg_create.__doc__)\n create.add_argument(\"name\", type=str, help=\"the name to use for the new key\")\n create.add_argument(\"email\", type=str, help=\"the email address to use for the new key\")\n create.add_argument(\n \"--comment\",\n metavar=\"COMMENT\",\n type=str,\n default=\"GPG created for Spack\",\n help=\"a description for the intended use of the key\",\n )\n create.add_argument(\n \"--expires\", metavar=\"EXPIRATION\", type=str, default=\"0\", help=\"when the key should expire\"\n )\n create.add_argument(\n \"--export\", metavar=\"DEST\", type=str, help=\"export the public key to a file\"\n )\n create.add_argument(\n \"--export-secret\",\n metavar=\"DEST\",\n type=str,\n dest=\"secret\",\n help=\"export the private key to a file\",\n )\n create.set_defaults(func=gpg_create)\n\n list = subparsers.add_parser(\"list\", help=gpg_list.__doc__)\n list.add_argument(\"--trusted\", action=\"store_true\", default=True, help=\"list trusted keys\")\n list.add_argument(\n \"--signing\", action=\"store_true\", help=\"list keys which may be used for signing\"\n )\n list.set_defaults(func=gpg_list)\n\n init = subparsers.add_parser(\"init\", help=gpg_init.__doc__)\n init.add_argument(\"--from\", metavar=\"DIR\", type=str, dest=\"import_dir\", help=argparse.SUPPRESS)\n init.set_defaults(func=gpg_init)\n\n export = subparsers.add_parser(\"export\", help=gpg_export.__doc__)\n export.add_argument(\"location\", type=str, help=\"where to export keys\")\n export.add_argument(\n \"keys\", nargs=\"*\", help=\"the keys to export (all public keys if unspecified)\"\n )\n export.add_argument(\"--secret\", action=\"store_true\", help=\"export secret keys\")\n export.set_defaults(func=gpg_export)\n\n publish = subparsers.add_parser(\"publish\", help=gpg_publish.__doc__)\n\n output = publish.add_mutually_exclusive_group(required=True)\n output.add_argument(\n \"-d\",\n \"--directory\",\n metavar=\"directory\",\n type=str,\n help=\"local directory where keys will be published\",\n )\n output.add_argument(\n \"-m\",\n \"--mirror-name\",\n metavar=\"mirror-name\",\n type=str,\n help=\"name of the mirror where keys will be published\",\n )\n output.add_argument(\n \"--mirror-url\",\n metavar=\"mirror-url\",\n type=str,\n help=\"URL of the mirror where keys will be published\",\n )\n publish.add_argument(\n \"--update-index\",\n \"--rebuild-index\",\n action=\"store_true\",\n default=False,\n help=\"regenerate buildcache key index after publishing key(s)\",\n )\n publish.add_argument(\n \"keys\", nargs=\"*\", help=\"keys to publish (all public keys if unspecified)\"\n )\n publish.set_defaults(func=gpg_publish)\n\n\ndef gpg_create(args):\n \"\"\"create a new key\"\"\"\n if args.export or args.secret:\n old_sec_keys = spack.util.gpg.signing_keys()\n\n # Create the new key\n spack.util.gpg.create(\n name=args.name, email=args.email, comment=args.comment, expires=args.expires\n )\n if args.export or args.secret:\n new_sec_keys = set(spack.util.gpg.signing_keys())\n new_keys = new_sec_keys.difference(old_sec_keys)\n\n if args.export:\n spack.util.gpg.export_keys(args.export, new_keys)\n if args.secret:\n spack.util.gpg.export_keys(args.secret, new_keys, secret=True)\n\n\ndef gpg_export(args):\n \"\"\"export a gpg key, optionally including secret key\"\"\"\n keys = args.keys\n if not keys:\n keys = spack.util.gpg.signing_keys()\n spack.util.gpg.export_keys(args.location, keys, args.secret)\n\n\ndef gpg_list(args):\n \"\"\"list keys available in the keyring\"\"\"\n spack.util.gpg.list(args.trusted, args.signing)\n\n\ndef gpg_sign(args):\n \"\"\"sign a package\"\"\"\n key = args.key\n if key is None:\n keys = spack.util.gpg.signing_keys()\n if len(keys) == 1:\n key = keys[0]\n elif not keys:\n raise RuntimeError(\"no signing keys are available\")\n else:\n raise RuntimeError(\"multiple signing keys are available; please choose one\")\n output = args.output\n if not output:\n output = args.spec[0] + \".asc\"\n # TODO: Support the package format Spack creates.\n spack.util.gpg.sign(key, \" \".join(args.spec), output, args.clearsign)\n\n\ndef gpg_trust(args):\n \"\"\"add a key to the keyring\"\"\"\n spack.util.gpg.trust(args.keyfile)\n\n\ndef gpg_init(args):\n \"\"\"add the default keys to the keyring\"\"\"\n import_dir = args.import_dir\n if import_dir is None:\n import_dir = spack.paths.gpg_keys_path\n\n for root, _, filenames in os.walk(import_dir):\n for filename in filenames:\n if not filename.endswith(\".key\"):\n continue\n spack.util.gpg.trust(os.path.join(root, filename))\n\n\ndef gpg_untrust(args):\n \"\"\"remove a key from the keyring\"\"\"\n spack.util.gpg.untrust(args.signing, *args.keys)\n\n\ndef gpg_verify(args):\n \"\"\"verify a signed package\"\"\"\n # TODO: Support the package format Spack creates.\n signature = args.signature\n if signature is None:\n signature = args.spec[0] + \".asc\"\n spack.util.gpg.verify(signature, \" \".join(args.spec))\n\n\ndef gpg_publish(args):\n \"\"\"publish public keys to a build cache\"\"\"\n\n mirror = None\n if args.directory:\n url = spack.util.url.path_to_file_url(args.directory)\n mirror = spack.mirrors.mirror.Mirror(url, url)\n elif args.mirror_name:\n mirror = spack.mirrors.mirror.MirrorCollection(binary=True).lookup(args.mirror_name)\n elif args.mirror_url:\n mirror = spack.mirrors.mirror.Mirror(args.mirror_url, args.mirror_url)\n\n with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir:\n spack.binary_distribution._url_push_keys(\n mirror, keys=args.keys, tmpdir=tmpdir, update_index=args.update_index\n )\n\n\ndef gpg(parser, args):\n if args.func:\n args.func(args)\n" }, { "path": "lib/spack/spack/cmd/graph.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport argparse\n\nimport spack.cmd\nimport spack.config\nimport spack.environment as ev\nimport spack.store\nfrom spack.cmd.common import arguments\nfrom spack.graph import DAGWithDependencyTypes, SimpleDAG, graph_ascii, graph_dot, static_graph_dot\nfrom spack.llnl.util import tty\n\ndescription = \"generate graphs of package dependency relationships\"\nsection = \"query\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n setattr(setup_parser, \"parser\", subparser)\n subparser.epilog = \"\"\"\nOutside of an environment, the command concretizes specs and graphs them, unless the\n--installed option is given. In that case specs are matched from the current DB.\n\nIf an environment is active, specs are matched from the currently available concrete specs\nin the lockfile.\n\n\"\"\"\n method = subparser.add_mutually_exclusive_group()\n method.add_argument(\n \"-a\", \"--ascii\", action=\"store_true\", help=\"draw graph as ascii to stdout (default)\"\n )\n method.add_argument(\n \"-d\", \"--dot\", action=\"store_true\", help=\"generate graph in dot format and print to stdout\"\n )\n\n subparser.add_argument(\n \"-s\",\n \"--static\",\n action=\"store_true\",\n help=\"graph static (possible) deps, don't concretize (implies ``--dot``)\",\n )\n subparser.add_argument(\n \"-c\",\n \"--color\",\n action=\"store_true\",\n help=\"use different colors for different dependency types\",\n )\n\n subparser.add_argument(\n \"-i\", \"--installed\", action=\"store_true\", help=\"graph specs from the DB\"\n )\n\n arguments.add_common_arguments(subparser, [\"deptype\", \"specs\"])\n\n\ndef graph(parser, args):\n env = ev.active_environment()\n if args.installed and env:\n tty.die(\"cannot use --installed with an active environment\")\n\n if args.color and not args.dot:\n tty.die(\"the --color option can be used only with --dot\")\n\n if args.installed:\n if not args.specs:\n specs = spack.store.STORE.db.query()\n else:\n result = []\n for item in args.specs:\n result.extend(spack.store.STORE.db.query(item))\n specs = list(set(result))\n elif env:\n specs = env.concrete_roots()\n if args.specs:\n specs = env.all_matching_specs(*args.specs)\n\n else:\n specs = spack.cmd.parse_specs(args.specs, concretize=not args.static)\n\n if not specs:\n tty.die(\"no spec matching the query\")\n\n if args.static:\n static_graph_dot(specs, depflag=args.deptype)\n return\n\n if args.dot:\n builder = SimpleDAG()\n if args.color:\n builder = DAGWithDependencyTypes()\n graph_dot(specs, builder=builder, depflag=args.deptype)\n return\n\n # ascii is default: user doesn't need to provide it explicitly\n debug = spack.config.get(\"config:debug\")\n graph_ascii(specs[0], debug=debug, depflag=args.deptype)\n for spec in specs[1:]:\n print() # extra line bt/w independent graphs\n graph_ascii(spec, debug=debug)\n" }, { "path": "lib/spack/spack/cmd/help.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport sys\n\nfrom spack.llnl.util.tty.color import colorize\n\ndescription = \"get help on spack and its commands\"\nsection = \"help\"\nlevel = \"short\"\n\n#\n# These are longer guides on particular aspects of Spack. Currently there\n# is only one on spec syntax.\n#\nspec_guide = \"\"\"\\\n@*B{spec expression syntax:}\n\n package [constraints] [^dependency [constraints] ...]\n\n package any package from 'spack list', or\n @K{/hash} unique prefix or full hash of\n installed package\n\n @*B{constraints:}\n @*c{versions:}\n @c{@version} single version\n @c{@min:max} version range (inclusive)\n @c{@min:} version or higher\n @c{@:max} up to version (inclusive)\n @c{@=version} exact version\n\n @*c{compilers:}\n @g{%compiler} build with \n @g{%compiler@version} build with specific compiler version\n @g{%compiler@min:max} specific version range (see above)\n\n @*c{compiler flags:}\n @g{cflags=\"flags\"} cppflags, cflags, cxxflags,\n fflags, ldflags, ldlibs\n @g{==} propagate flags to package dependencies\n\n @*c{variants:}\n @B{+variant} enable \n @r{-variant} or @r{~variant} disable \n @B{variant=value} set non-boolean to \n @B{variant=value1,value2,value3} set multi-value values\n @B{++}, @r{--}, @r{~~}, @B{==} propagate variants to package dependencies\n\n @*c{architecture variants:}\n @m{platform=platform} linux, darwin, freebsd, windows\n @m{os=operating_system} specific \n @m{target=target} specific processor\n @m{arch=platform-os-target} shortcut for all three above\n\n @*c{dependencies:}\n ^dependency [constraints] specify constraints on dependencies\n ^@K{/hash} build with a specific installed\n dependency\n\n @*B{examples:}\n hdf5 any hdf5 configuration\n hdf5 @c{@1.10.1} hdf5 version 1.10.1\n hdf5 @c{@1.8:} hdf5 1.8 or higher\n hdf5 @c{@1.8:} @g{%gcc} hdf5 1.8 or higher built with gcc\n hdf5 @B{+mpi} hdf5 with mpi enabled\n hdf5 @r{~mpi} hdf5 with mpi disabled\n hdf5 @B{++mpi} hdf5 with mpi enabled and propagates\n hdf5 @r{~~mpi} hdf5 with mpi disabled and propagates\n hdf5 @B{+mpi} ^mpich hdf5 with mpi, using mpich\n hdf5 @B{+mpi} ^openmpi@c{@1.7} hdf5 with mpi, using openmpi 1.7\n boxlib @B{dim=2} boxlib built for 2 dimensions\n libdwarf @g{%intel} ^libelf@g{%gcc}\n libdwarf, built with intel compiler, linked to libelf built with gcc\n mvapich2 @B{fabrics=psm,mrail,sock} @g{%gcc}\n mvapich2, built with gcc compiler, with support for multiple fabrics\n\"\"\"\n\n\nguides = {\"spec\": spec_guide}\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n help_cmd_group = subparser.add_mutually_exclusive_group()\n help_cmd_group.add_argument(\n \"help_command\", nargs=\"?\", default=None, help=\"command to get help on\"\n )\n\n help_all_group = subparser.add_mutually_exclusive_group()\n help_all_group.add_argument(\n \"-a\",\n \"--all\",\n action=\"store_const\",\n const=\"long\",\n default=\"short\",\n help=\"list all available commands and options\",\n )\n\n help_spec_group = subparser.add_mutually_exclusive_group()\n help_spec_group.add_argument(\n \"--spec\",\n action=\"store_const\",\n dest=\"guide\",\n const=\"spec\",\n default=None,\n help=\"help on the package specification syntax\",\n )\n\n\ndef help(parser, args):\n if args.guide:\n print(colorize(guides[args.guide]))\n return 0\n\n if args.help_command:\n parser.add_command(args.help_command)\n parser.parse_args([args.help_command, \"-h\"])\n else:\n sys.stdout.write(parser.format_help(level=args.all))\n" }, { "path": "lib/spack/spack/cmd/info.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n# mypy: disallow-untyped-defs\n\nimport argparse\nimport collections\nimport shutil\nimport sys\nimport textwrap\nfrom argparse import Namespace\nfrom typing import Any, Callable, Dict, Iterable, List, Optional, TextIO, Tuple\n\nimport spack.builder\nimport spack.cmd\nimport spack.dependency\nimport spack.deptypes as dt\nimport spack.fetch_strategy as fs\nimport spack.install_test\nimport spack.llnl.util.tty as tty\nimport spack.llnl.util.tty.color as color\nimport spack.package_base\nimport spack.repo\nimport spack.spec\nimport spack.variant\nimport spack.version\nfrom spack.cmd.common import arguments\nfrom spack.llnl.util.tty.colify import colify\nfrom spack.package_base import PackageBase\nfrom spack.util.typing import SupportsRichComparison\n\ndescription = \"get detailed information on a particular package\"\nsection = \"query\"\nlevel = \"short\"\n\nheader_color = \"@*b\"\nplain_format = \"@.\"\n\n#: Allow at least this much room for values when formatting definitions\n#: Wrap after a long variant name/condition if we need to do so to preserve this width.\nMIN_VALUES_WIDTH = 30\n\n\nclass Formatter:\n \"\"\"Generic formatter for elements displayed by `spack info`.\n\n Elements have four parts: name, values, when condition, and description. They can\n be formatted two ways (shown here for variants):\n\n Grouped by when (default)::\n\n when +cuda\n cuda_arch [none] none, 10, 100, 100a, 101,\n 101a, 11, 12, 120, 120a, 13\n CUDA architecture\n\n Or, by name (each name has a when nested under it)::\n\n cuda_arch [none] none, 10, 100, 100a, 101,\n 101a, 11, 12, 120, 120a, 13\n when +cuda\n CUDA architecture\n\n The values and description will be wrapped if needed. the name (and any additional info)\n will not (so they should be kept short).\n\n Subclasses are responsible for generating colorized text, but not wrapping,\n indentation, or other formatting, for the name, values, and description.\n\n \"\"\"\n\n def format_name(self, element: Any) -> str:\n return str(element)\n\n def format_values(self, element: Any) -> str:\n return \"\"\n\n def format_description(self, element: Any) -> str:\n return \"\"\n\n\ndef padder(str_list: Iterable, extra: int = 0) -> Callable:\n \"\"\"Return a function to pad elements of a list.\"\"\"\n length = max(len(str(s)) for s in str_list) + extra\n\n def pad(string: str) -> str:\n string = str(string)\n padding = max(0, length - len(string))\n return string + (padding * \" \")\n\n return pad\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n subparser.add_argument(\n \"-a\", \"--all\", action=\"store_true\", default=False, help=\"output all package information\"\n )\n\n by = subparser.add_mutually_exclusive_group()\n by.add_argument(\n \"--by-name\",\n dest=\"by_name\",\n action=\"store_true\",\n default=True,\n help=\"list variants, dependency, etc. in name order, then by when condition\",\n )\n by.add_argument(\n \"--by-when\",\n dest=\"by_name\",\n action=\"store_false\",\n default=False,\n help=\"group variants, dependencies, etc. first by when condition, then by name\",\n )\n\n options = [\n (\"--detectable\", print_detectable.__doc__),\n (\"--maintainers\", print_maintainers.__doc__),\n (\"--namespace\", print_namespace.__doc__),\n (\"--no-dependencies\", f\"do not {print_dependencies.__doc__}\"),\n (\"--no-variants\", f\"do not {print_variants.__doc__}\"),\n (\"--no-versions\", f\"do not {print_versions.__doc__}\"),\n (\"--phases\", print_phases.__doc__),\n (\"--tags\", print_tags.__doc__),\n (\"--tests\", print_tests.__doc__),\n (\"--virtuals\", print_virtuals.__doc__),\n ]\n for opt, help_comment in options:\n subparser.add_argument(opt, action=\"store_true\", help=help_comment)\n\n # deprecated for the more generic --by-name, but still here until we can remove it\n subparser.add_argument(\n \"--variants-by-name\",\n dest=\"by_name\",\n action=arguments.DeprecatedStoreTrueAction,\n help=argparse.SUPPRESS,\n removed_in=\"a future Spack release\",\n instructions=\"use --by-name instead\",\n )\n arguments.add_common_arguments(subparser, [\"spec\"])\n\n\ndef section_title(s: str) -> str:\n return header_color + s + plain_format\n\n\ndef version(s: str) -> str:\n return spack.spec.VERSION_COLOR + s + plain_format\n\n\ndef format_deptype(depflag: int) -> str:\n color_flags = zip(\"gcbm\", dt.ALL_FLAGS)\n return \", \".join(\n color.colorize(f\"@{c}{{{dt.flag_to_string(depflag & flag)}}}\")\n for c, flag in color_flags\n if depflag & flag\n )\n\n\nclass DependencyFormatter(Formatter):\n def format_name(self, dep: spack.dependency.Dependency) -> str:\n return dep.spec._long_spec(color=color.get_color_when())\n\n def format_values(self, dep: spack.dependency.Dependency) -> str:\n return str(format_deptype(dep.depflag))\n\n\ndef count_bool_variant_conditions(\n when_indexed_dictionary: Dict[spack.spec.Spec, Any],\n) -> List[Tuple[int, Tuple[str, bool]]]:\n \"\"\"Counts boolean variants in whens in a dictionary.\n\n Returns a list of the most used when conditions for boolean variants along with their value.\n \"\"\"\n top: Dict = collections.defaultdict(int)\n for when, _ in when_indexed_dictionary.items():\n for v, variant in when.variants.items():\n if type(variant.value) is bool:\n top[(variant.name, variant.value)] += 1\n\n # sorted by frequency, highest first\n return list(reversed(sorted((n, t) for t, n in top.items())))\n\n\ndef print_dependencies(pkg: PackageBase, args: Namespace) -> None:\n \"\"\"output build, link, and run package dependencies\"\"\"\n print_definitions(pkg, \"Dependencies\", pkg.dependencies, DependencyFormatter(), args.by_name)\n\n\ndef print_dependency_suggestion(pkg: PackageBase) -> None:\n variant_counts = count_bool_variant_conditions(pkg.dependencies)\n big_variants = [\n (name, val)\n for n, (name, val) in variant_counts\n # make a note of variants with large counts that aren't already toggled by the user.\n if n >= 20 and not (name in pkg.spec.variants and pkg.spec.variants[name].value != val)\n ]\n\n if big_variants:\n spec = spack.spec.Spec(pkg.name)\n for name, val in big_variants:\n # skip if user specified, or already saw a value (e.g. many +mpi and ~mpi)\n if name in spec.variants or name in pkg.spec.variants:\n continue\n spec.variants[name] = spack.variant.BoolValuedVariant(name, not val)\n\n # if there is new stuff to add beyond the input\n if spec.variants:\n spec.constrain(pkg.spec) # include already specified constraints\n print()\n tty.info(\n f\"{pkg.name} has many complex dependencies; consider this for a simpler view:\",\n f\"spack info {spec.format(color=tty.color.get_color_when())}\",\n format=\"y\",\n )\n\n\ndef print_detectable(pkg: PackageBase, args: Namespace) -> None:\n \"\"\"output information on external detection\"\"\"\n\n color.cprint(\"\")\n color.cprint(section_title(\"Externally Detectable:\"))\n\n # If the package has an 'executables' of 'libraries' field, it\n # can detect an installation\n if hasattr(pkg, \"executables\") or hasattr(pkg, \"libraries\"):\n find_attributes = []\n if hasattr(pkg, \"determine_version\"):\n find_attributes.append(\"version\")\n\n if hasattr(pkg, \"determine_variants\"):\n find_attributes.append(\"variants\")\n\n # If the package does not define 'determine_version' nor\n # 'determine_variants', then it must use some custom detection\n # mechanism. In this case, just inform the user it's detectable somehow.\n color.cprint(\n \" True{0}\".format(\n \" (\" + \", \".join(find_attributes) + \")\" if find_attributes else \"\"\n )\n )\n else:\n color.cprint(\" False\")\n\n\ndef print_maintainers(pkg: PackageBase, args: Namespace) -> None:\n \"\"\"output package maintainers\"\"\"\n\n if len(pkg.maintainers) > 0:\n mnt = \" \".join([\"@@\" + m for m in pkg.maintainers])\n color.cprint(\"\")\n color.cprint(section_title(\"Maintainers: \") + mnt)\n\n\ndef print_namespace(pkg: PackageBase, args: Namespace) -> None:\n \"\"\"output package namespace\"\"\"\n\n repo = spack.repo.PATH.get_repo(pkg.namespace)\n color.cprint(\"\")\n color.cprint(section_title(\"Namespace:\"))\n color.cprint(f\" @c{{{repo.namespace}}} at {repo.root}\")\n\n\ndef print_phases(pkg: PackageBase, args: Namespace) -> None:\n \"\"\"output installation phases\"\"\"\n\n builder = spack.builder.create(pkg)\n\n if hasattr(builder, \"phases\") and builder.phases:\n color.cprint(\"\")\n color.cprint(section_title(\"Installation Phases:\"))\n phase_str = \"\"\n for phase in builder.phases:\n phase_str += \" {0}\".format(phase)\n color.cprint(phase_str)\n\n\ndef print_tags(pkg: PackageBase, args: Namespace) -> None:\n \"\"\"output package tags\"\"\"\n\n color.cprint(\"\")\n color.cprint(section_title(\"Tags: \"))\n if hasattr(pkg, \"tags\"):\n tags = sorted(pkg.tags)\n colify(tags, indent=4)\n else:\n color.cprint(\" None\")\n\n\ndef print_tests(pkg: PackageBase, args: Namespace) -> None:\n \"\"\"output relevant build-time and stand-alone tests\"\"\"\n\n # Some built-in base packages (e.g., Autotools) define callback (e.g.,\n # check) inherited by descendant packages. These checks may not result\n # in build-time testing if the package's build does not implement the\n # expected functionality (e.g., a 'check' or 'test' targets).\n #\n # So the presence of a callback in Spack does not necessarily correspond\n # to the actual presence of built-time tests for a package.\n for callbacks, phase in [\n (getattr(pkg, \"build_time_test_callbacks\", None), \"Build\"),\n (getattr(pkg, \"install_time_test_callbacks\", None), \"Install\"),\n ]:\n color.cprint(\"\")\n color.cprint(section_title(\"Available {0} Phase Test Methods:\".format(phase)))\n names = []\n if callbacks:\n for name in callbacks:\n if getattr(pkg, name, False):\n names.append(name)\n\n if names:\n colify(sorted(names), indent=4)\n else:\n color.cprint(\" None\")\n\n # PackageBase defines an empty install/smoke test but we want to know\n # if it has been overridden and, therefore, assumed to be implemented.\n color.cprint(\"\")\n color.cprint(section_title(\"Stand-Alone/Smoke Test Methods:\"))\n names = spack.install_test.test_function_names(pkg, add_virtuals=True)\n if names:\n colify(sorted(names), indent=4)\n else:\n color.cprint(\" None\")\n\n\ndef _fmt_when(when: \"spack.spec.Spec\", indent: int) -> str:\n return color.colorize(\n f\"{indent * ' '}@B{{when}} {color.cescape(when._long_spec(color=color.get_color_when()))}\"\n )\n\n\ndef _fmt_variant_value(v: Any) -> str:\n return str(v).lower() if v is None or isinstance(v, bool) else str(v)\n\n\ndef _print_definition(\n name_field: str,\n values_field: str,\n description: str,\n max_name_len: int,\n indent: int,\n when: Optional[spack.spec.Spec] = None,\n out: Optional[TextIO] = None,\n) -> None:\n \"\"\"Print a definition entry for `spack info` output.\n\n Arguments:\n name_field: name and optional info, e.g. a default; should be short.\n values_field: possible values for the entry; Wrapped if long.\n description: description of the field (wrapped if overly long)\n max_name_len: max length of any definition to be printed\n indent: size of leading indent for entry\n when: optional when condition\n out: stream to print to\n\n Caller is expected to calculate the max name length in advance and pass it to\n ``_print_definition``.\n\n \"\"\"\n out = out or sys.stdout\n cols = shutil.get_terminal_size().columns\n\n # prevent values from being compressed by really long names\n name_col_width = min(max_name_len, cols - MIN_VALUES_WIDTH - indent)\n name_len = color.clen(name_field)\n\n pad = 4 # min padding between name and values\n value_indent = (indent + name_col_width + pad) * \" \" # left edge of values\n\n formatted_name_and_values = f\"{indent * ' '}{name_field}\"\n if values_field:\n formatted_values = \"\\n\".join(\n color.cwrap(\n values_field,\n width=cols - 2,\n initial_indent=value_indent,\n subsequent_indent=value_indent,\n )\n )\n\n if name_len > name_col_width:\n # for overlong names, values appear aligned on next line\n formatted_name_and_values += f\"\\n{formatted_values}\"\n else:\n # for regular names, trim indentation to make room for name on same line\n formatted_values = formatted_values[indent + name_len + pad :]\n\n # e.g,. name [default] value1, value2, value3, ...\n formatted_name_and_values += f\"{pad * ' '}{formatted_values}\"\n\n out.write(f\"{formatted_name_and_values}\\n\")\n\n # when \n description_indent = indent + 4\n if when is not None and when != spack.spec.Spec():\n out.write(_fmt_when(when, description_indent - 2))\n out.write(\"\\n\")\n\n # description, preserving explicit line breaks from the way it's written in the\n # package file, but still wrapoing long lines for small terminals. This allows\n # descriptions to provide detailed help in descriptions (see, e.g., gasnet's variants).\n if description:\n formatted_description = \"\\n\".join(\n textwrap.fill(\n line,\n width=cols - 2,\n initial_indent=description_indent * \" \",\n subsequent_indent=description_indent * \" \",\n )\n for line in description.split(\"\\n\")\n )\n out.write(formatted_description)\n out.write(\"\\n\")\n\n\ndef print_header(header: str, when_indexed_dictionary: Dict, formatter: Formatter) -> bool:\n color.cprint(\"\")\n color.cprint(section_title(f\"{header}:\"))\n\n if not when_indexed_dictionary:\n print(\" None\")\n return False\n return True\n\n\ndef max_name_length(when_indexed_dictionary: Dict, formatter: Formatter) -> int:\n # Calculate the max length of the first field of the definition. Lets us know how\n # much to pad other fields on the first line.\n return max(\n color.clen(formatter.format_name(definition))\n for subkey in spack.package_base._subkeys(when_indexed_dictionary)\n for _, definition in spack.package_base._definitions(when_indexed_dictionary, subkey)\n )\n\n\ndef print_grouped_by_when(\n pkg: PackageBase, header: str, when_indexed_dictionary: Dict, formatter: Formatter\n) -> None:\n \"\"\"Generic method to print metadata grouped by when conditions.\"\"\"\n if not print_header(header, when_indexed_dictionary, formatter):\n return\n\n max_name_len = max_name_length(when_indexed_dictionary, formatter)\n\n # ensure that items without conditions come first\n unconditional_first = lambda item: (item[0] != spack.spec.Spec(), item)\n\n indent = 4\n for when, by_name in sorted(when_indexed_dictionary.items(), key=unconditional_first):\n if not pkg.intersects(when):\n continue\n\n start_indent = indent\n values_indent = max_name_len + 4\n\n if when != spack.spec.Spec():\n sys.stdout.write(\"\\n\")\n sys.stdout.write(_fmt_when(when, indent))\n sys.stdout.write(\"\\n\")\n\n # indent names slightly inside 'when', but line up values\n start_indent += 2\n values_indent -= 2\n\n for subkey, definition in sorted(by_name.items()):\n _print_definition(\n formatter.format_name(definition),\n formatter.format_values(definition),\n formatter.format_description(definition),\n values_indent,\n start_indent,\n when=None,\n out=sys.stdout,\n )\n\n\ndef print_by_name(\n pkg: PackageBase, header: str, when_indexed_dictionary: Dict, formatter: Formatter\n) -> None:\n if not print_header(header, when_indexed_dictionary, formatter):\n return\n\n max_name_len = max_name_length(when_indexed_dictionary, formatter)\n max_name_len += 4\n\n indent = 4\n\n def unconditional_first(definition: Any) -> SupportsRichComparison:\n spec = getattr(definition, \"spec\", None)\n if spec:\n return (spec != spack.spec.Spec(spec.name), spec)\n else:\n return getattr(definition, \"name\", None) # type: ignore[return-value]\n\n for subkey in spack.package_base._subkeys(when_indexed_dictionary):\n for when, definition in sorted(\n spack.package_base._definitions(when_indexed_dictionary, subkey),\n key=lambda t: unconditional_first(t[1]),\n ):\n if not pkg.intersects(when):\n continue\n\n _print_definition(\n formatter.format_name(definition),\n formatter.format_values(definition),\n formatter.format_description(definition),\n max_name_len,\n indent,\n when=when,\n out=sys.stdout,\n )\n sys.stdout.write(\"\\n\")\n\n\ndef print_definitions(\n pkg: PackageBase,\n header: str,\n when_indexed_dictionary: Dict,\n formatter: Formatter,\n by_name: bool,\n) -> None:\n # convert simple dictionaries to dicts of dicts before formatting.\n # subkeys are ignored in formatting, so use stringified numbers.\n values = when_indexed_dictionary.values()\n if when_indexed_dictionary and not isinstance(next(iter(values)), dict):\n when_indexed_dictionary = {\n when: {str(i): element}\n for i, (when, element) in enumerate(when_indexed_dictionary.items())\n }\n\n if by_name:\n print_by_name(pkg, header, when_indexed_dictionary, formatter)\n else:\n print_grouped_by_when(pkg, header, when_indexed_dictionary, formatter)\n\n\nclass VariantFormatter(Formatter):\n def format_name(self, variant: spack.variant.Variant) -> str:\n return color.colorize(\n f\"@c{{{variant.name}}} @C{{[{_fmt_variant_value(variant.default)}]}}\"\n )\n\n def format_values(self, variant: spack.variant.Variant) -> str:\n values = (\n [variant.values]\n if not isinstance(variant.values, (tuple, list, spack.variant.DisjointSetsOfValues))\n else variant.values\n )\n\n # put 'none' first, sort the rest by value\n sorted_values = sorted(values, key=lambda v: (v != \"none\", v))\n\n return color.colorize(f\"@c{{{', '.join(_fmt_variant_value(v) for v in sorted_values)}}}\")\n\n def format_description(self, variant: spack.variant.Variant) -> str:\n return variant.description\n\n\ndef print_variants(pkg: PackageBase, args: Namespace) -> None:\n \"\"\"output variants\"\"\"\n print_definitions(pkg, \"Variants\", pkg.variants, VariantFormatter(), args.by_name)\n\n\ndef print_licenses(pkg: PackageBase, args: Namespace) -> None:\n \"\"\"Output the licenses of the project.\"\"\"\n print_definitions(pkg, \"Licenses\", pkg.licenses, Formatter(), args.by_name)\n\n\ndef print_versions(pkg: PackageBase, args: Namespace) -> None:\n \"\"\"output versions\"\"\"\n\n color.cprint(\"\")\n color.cprint(section_title(\"Preferred version: \"))\n\n versions = [v for v in pkg.versions if pkg.spec.versions.intersects(v)]\n\n if not versions:\n color.cprint(version(\" None\"))\n color.cprint(\"\")\n color.cprint(section_title(\"Safe versions: \"))\n color.cprint(version(\" None\"))\n color.cprint(\"\")\n color.cprint(section_title(\"Deprecated versions: \"))\n color.cprint(version(\" None\"))\n else:\n pad = padder(versions, 4)\n\n preferred = spack.package_base.preferred_version(pkg)\n\n def get_url(version: spack.version.VersionType) -> str:\n try:\n return str(fs.for_package_version(pkg, version))\n except fs.InvalidArgsError:\n return \"No URL\"\n\n url = get_url(preferred) if pkg.has_code else \"\"\n line = version(\" {0}\".format(pad(preferred))) + color.cescape(str(url))\n color.cwrite(line)\n\n print()\n\n safe = []\n deprecated = []\n for v in reversed(sorted(versions)):\n if pkg.has_code:\n url = get_url(v)\n if pkg.versions[v].get(\"deprecated\", False):\n deprecated.append((v, url))\n else:\n safe.append((v, url))\n\n for title, vers in [(\"Safe\", safe), (\"Deprecated\", deprecated)]:\n color.cprint(\"\")\n color.cprint(section_title(\"{0} versions: \".format(title)))\n if not vers:\n color.cprint(version(\" None\"))\n continue\n\n for v, url in vers:\n line = version(\" {0}\".format(pad(v))) + color.cescape(str(url))\n color.cprint(line)\n\n\ndef print_virtuals(pkg: PackageBase, args: Namespace) -> None:\n \"\"\"output virtual packages\"\"\"\n\n color.cprint(\"\")\n color.cprint(section_title(\"Virtual Packages: \"))\n if pkg.provided:\n for when, specs in reversed(sorted(pkg.provided.items())):\n line = \" %s provides %s\" % (when.cformat(), \", \".join(s.cformat() for s in specs))\n print(line)\n\n else:\n color.cprint(\" None\")\n\n\ndef info(parser: argparse.ArgumentParser, args: Namespace) -> None:\n specs = spack.cmd.parse_specs(args.spec)\n if len(specs) > 1:\n tty.die(f\"`spack info` requires exactly one spec. Parsed {len(specs)}\")\n if len(specs) == 0:\n tty.die(\"`spack info` requires a spec.\")\n\n spec = specs[0]\n pkg_cls = spack.repo.PATH.get_pkg_class(spec.fullname)\n pkg_cls.validate_variant_names(spec)\n pkg = pkg_cls(spec)\n\n # Output core package information\n header = section_title(\"{0}: \").format(pkg.build_system_class) + pkg.name\n color.cprint(header)\n\n color.cprint(\"\")\n color.cprint(section_title(\"Description:\"))\n if pkg.__doc__:\n color.cprint(color.cescape(pkg.format_doc(indent=4)))\n else:\n color.cprint(\" None\")\n\n if getattr(pkg, \"homepage\"):\n color.cprint(section_title(\"Homepage: \") + str(pkg.homepage))\n\n # Now output optional information in expected order\n sections = [\n (args.all or args.maintainers, print_maintainers),\n (args.all or args.namespace, print_namespace),\n (args.all or args.detectable, print_detectable),\n (args.all or args.tags, print_tags),\n (args.all or not args.no_versions, print_versions),\n (args.all or not args.no_variants, print_variants),\n (args.all or args.phases, print_phases),\n (args.all or not args.no_dependencies, print_dependencies),\n (args.all or args.virtuals, print_virtuals),\n (args.all or args.tests, print_tests),\n (True, print_licenses),\n ]\n for print_it, func in sections:\n if print_it:\n func(pkg, args)\n\n print_dependency_suggestion(pkg)\n\n color.cprint(\"\")\n" }, { "path": "lib/spack/spack/cmd/install.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport os\nimport shutil\nimport sys\nfrom typing import List\n\nimport spack.cmd\nimport spack.config\nimport spack.environment as ev\nimport spack.installer_dispatch\nimport spack.llnl.util.filesystem as fs\nimport spack.paths\nimport spack.spec\nimport spack.store\nfrom spack.cmd.common import arguments\nfrom spack.error import InstallError, SpackError\nfrom spack.installer import InstallPolicy\nfrom spack.llnl.string import plural\nfrom spack.llnl.util import tty\n\ndescription = \"build and install packages\"\nsection = \"build\"\nlevel = \"short\"\n\n\ndef cache_opt(use_buildcache: str, default: InstallPolicy) -> InstallPolicy:\n if use_buildcache == \"only\":\n return \"cache_only\"\n elif use_buildcache == \"never\":\n return \"source_only\"\n return default\n\n\ndef install_kwargs_from_args(args):\n \"\"\"Translate command line arguments into a dictionary that will be passed\n to the package installer.\n \"\"\"\n pkg_use_bc, dep_use_bc = args.use_buildcache\n if args.cache_only:\n default = \"cache_only\"\n elif args.use_cache:\n default = \"auto\"\n else:\n default = \"source_only\"\n\n return {\n \"fail_fast\": args.fail_fast,\n \"keep_prefix\": args.keep_prefix,\n \"keep_stage\": args.keep_stage,\n \"restage\": not args.dont_restage,\n \"install_source\": args.install_source,\n \"verbose\": args.verbose or args.install_verbose,\n \"fake\": args.fake,\n \"dirty\": args.dirty,\n \"root_policy\": cache_opt(pkg_use_bc, default),\n \"dependencies_policy\": cache_opt(dep_use_bc, default),\n \"include_build_deps\": args.include_build_deps,\n \"stop_at\": args.until,\n \"unsigned\": args.unsigned,\n \"install_deps\": (\"dependencies\" in args.things_to_install),\n \"install_package\": (\"package\" in args.things_to_install),\n \"concurrent_packages\": args.concurrent_packages,\n }\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n subparser.add_argument(\n \"--only\",\n default=\"package,dependencies\",\n dest=\"things_to_install\",\n choices=[\"package\", \"dependencies\"],\n help=\"select the mode of installation\\n\\n\"\n \"default is to install the package along with all its dependencies. \"\n \"alternatively, one can decide to install only the package or only the dependencies\",\n )\n subparser.add_argument(\n \"-u\",\n \"--until\",\n type=str,\n dest=\"until\",\n default=None,\n help=\"phase to stop after when installing (default None)\",\n )\n arguments.add_common_arguments(subparser, [\"concurrent_packages\"])\n arguments.add_common_arguments(subparser, [\"jobs\"])\n subparser.add_argument(\n \"--overwrite\",\n action=\"store_true\",\n help=\"reinstall an existing spec, even if it has dependents\",\n )\n subparser.add_argument(\n \"--fail-fast\",\n action=\"store_true\",\n help=\"stop all builds if any build fails (default is best effort)\",\n )\n subparser.add_argument(\n \"--keep-prefix\",\n action=\"store_true\",\n help=\"don't remove the install prefix if installation fails\",\n )\n subparser.add_argument(\n \"--keep-stage\",\n action=\"store_true\",\n help=\"don't remove the build stage if installation succeeds\",\n )\n subparser.add_argument(\n \"--dont-restage\",\n action=\"store_true\",\n help=\"if a partial install is detected, don't delete prior state\",\n )\n\n cache_group = subparser.add_mutually_exclusive_group()\n cache_group.add_argument(\n \"--use-cache\",\n action=\"store_true\",\n dest=\"use_cache\",\n default=True,\n help=\"check for pre-built Spack packages in mirrors (default)\",\n )\n cache_group.add_argument(\n \"--no-cache\",\n action=\"store_false\",\n dest=\"use_cache\",\n default=True,\n help=\"do not check for pre-built Spack packages in mirrors\",\n )\n cache_group.add_argument(\n \"--cache-only\",\n action=\"store_true\",\n dest=\"cache_only\",\n default=False,\n help=\"only install package from binary mirrors\",\n )\n cache_group.add_argument(\n \"--use-buildcache\",\n dest=\"use_buildcache\",\n type=arguments.use_buildcache,\n default=\"package:auto,dependencies:auto\",\n metavar=\"[{auto,only,never},][package:{auto,only,never},][dependencies:{auto,only,never}]\",\n help=\"select the mode of buildcache for the 'package' and 'dependencies'\\n\\n\"\n \"default: package:auto,dependencies:auto\\n\\n\"\n \"- `auto` behaves like --use-cache\\n\"\n \"- `only` behaves like --cache-only\\n\"\n \"- `never` behaves like --no-cache\",\n )\n\n subparser.add_argument(\n \"--include-build-deps\",\n action=\"store_true\",\n dest=\"include_build_deps\",\n default=False,\n help=\"include build deps when installing from cache, \"\n \"useful for CI pipeline troubleshooting\",\n )\n\n subparser.add_argument(\n \"--no-check-signature\",\n action=\"store_true\",\n dest=\"unsigned\",\n default=None,\n help=\"do not check signatures of binary packages (override mirror config)\",\n )\n subparser.add_argument(\n \"--show-log-on-error\",\n action=\"store_true\",\n help=\"print full build log to stderr if build fails\",\n )\n subparser.add_argument(\n \"--source\",\n action=\"store_true\",\n dest=\"install_source\",\n help=\"install source files in prefix\",\n )\n arguments.add_common_arguments(subparser, [\"no_checksum\"])\n subparser.add_argument(\n \"-v\",\n \"--verbose\",\n action=\"store_true\",\n dest=\"install_verbose\",\n help=\"display verbose build output while installing\",\n )\n subparser.add_argument(\"--fake\", action=\"store_true\", help=\"fake install for debug purposes\")\n subparser.add_argument(\n \"--only-concrete\",\n action=\"store_true\",\n default=False,\n help=\"(with environment) only install already concretized specs\",\n )\n\n updateenv_group = subparser.add_mutually_exclusive_group()\n updateenv_group.add_argument(\n \"--add\",\n action=\"store_true\",\n default=False,\n help=\"(with environment) add spec to the environment as a root\",\n )\n updateenv_group.add_argument(\n \"--no-add\",\n action=\"store_false\",\n dest=\"add\",\n help=\"(with environment) do not add spec to the environment as a root\",\n )\n\n cd_group = subparser.add_mutually_exclusive_group()\n arguments.add_common_arguments(cd_group, [\"clean\", \"dirty\"])\n\n testing = subparser.add_mutually_exclusive_group()\n testing.add_argument(\n \"--test\",\n default=None,\n choices=[\"root\", \"all\"],\n help=\"run tests on only root packages or all packages\",\n )\n arguments.add_common_arguments(subparser, [\"log_format\"])\n subparser.add_argument(\"--log-file\", default=None, help=\"filename for the log file\")\n subparser.add_argument(\n \"--help-cdash\", action=\"store_true\", help=\"show usage instructions for CDash reporting\"\n )\n arguments.add_cdash_args(subparser, False)\n arguments.add_common_arguments(subparser, [\"yes_to_all\", \"spec\"])\n arguments.add_concretizer_args(subparser)\n\n\ndef default_log_file(spec):\n \"\"\"Computes the default filename for the log file and creates\n the corresponding directory if not present\n \"\"\"\n basename = spec.format_path(\"test-{name}-{version}-{hash}.xml\")\n dirname = fs.os.path.join(spack.paths.reports_path, \"junit\")\n fs.mkdirp(dirname)\n return fs.os.path.join(dirname, basename)\n\n\ndef report_filename(args: argparse.Namespace, specs: List[spack.spec.Spec]) -> str:\n \"\"\"Return the filename to be used for reporting to JUnit or CDash format.\"\"\"\n result = args.log_file or default_log_file(specs[0])\n return result\n\n\ndef compute_tests_install_kwargs(specs, cli_test_arg):\n \"\"\"Translate the test cli argument into the proper install argument\"\"\"\n if cli_test_arg == \"all\":\n return True\n elif cli_test_arg == \"root\":\n return [spec.name for spec in specs]\n return False\n\n\ndef require_user_confirmation_for_overwrite(concrete_specs, args):\n if args.yes_to_all:\n return\n\n installed = list(filter(lambda x: x, map(spack.store.STORE.db.query_one, concrete_specs)))\n display_args = {\"long\": True, \"show_flags\": True, \"variants\": True}\n\n if installed:\n tty.msg(\"The following package specs will be reinstalled:\\n\")\n spack.cmd.display_specs(installed, **display_args)\n\n not_installed = list(filter(lambda x: x not in installed, concrete_specs))\n if not_installed:\n tty.msg(\n \"The following package specs are not installed and\"\n \" the --overwrite flag was given. The package spec\"\n \" will be newly installed:\\n\"\n )\n spack.cmd.display_specs(not_installed, **display_args)\n\n # We have some specs, so one of the above must have been true\n answer = tty.get_yes_or_no(\"Do you want to proceed?\", default=False)\n if not answer:\n tty.die(\"Reinstallation aborted.\")\n\n\ndef _dump_log_on_error(e: InstallError):\n e.print_context()\n assert e.pkg, \"Expected InstallError to include the associated package\"\n if not os.path.exists(e.pkg.log_path):\n tty.error(\"'spack install' created no log.\")\n else:\n sys.stderr.write(\"Full build log:\\n\")\n with open(e.pkg.log_path, errors=\"replace\", encoding=\"utf-8\") as log:\n shutil.copyfileobj(log, sys.stderr)\n\n\ndef _die_require_env():\n msg = \"install requires a package argument or active environment\"\n if \"spack.yaml\" in os.listdir(os.getcwd()):\n # There's a spack.yaml file in the working dir, the user may\n # have intended to use that\n msg += (\n \"\\n\\n\"\n \"Did you mean to install using the `spack.yaml`\"\n \" in this directory? Try: \\n\"\n \" spack env activate .\\n\"\n \" spack install\\n\"\n \" OR\\n\"\n \" spack --env . install\"\n )\n tty.die(msg)\n\n\ndef install(parser, args):\n # TODO: unify args.verbose?\n tty.set_verbose(args.verbose or args.install_verbose)\n\n if args.help_cdash:\n arguments.print_cdash_help()\n return\n\n if args.no_checksum:\n spack.config.set(\"config:checksum\", False, scope=\"command_line\")\n\n if args.log_file and not args.log_format:\n msg = \"the '--log-format' must be specified when using '--log-file'\"\n tty.die(msg)\n\n arguments.sanitize_reporter_options(args)\n\n reporter = args.reporter() if args.log_format else None\n install_kwargs = install_kwargs_from_args(args)\n env = ev.active_environment()\n\n if not env and not args.spec:\n _die_require_env()\n\n try:\n if env:\n install_with_active_env(env, args, install_kwargs, reporter)\n else:\n install_without_active_env(args, install_kwargs, reporter)\n except InstallError as e:\n if args.show_log_on_error:\n _dump_log_on_error(e)\n raise\n\n\ndef _maybe_add_and_concretize(args, env, specs):\n \"\"\"Handle the overloaded spack install behavior of adding\n and automatically concretizing specs\"\"\"\n\n # Users can opt out of accidental concretizations with --only-concrete\n if args.only_concrete:\n return\n\n # Otherwise, we will modify the environment.\n with env.write_transaction():\n # `spack add` adds these specs.\n if args.add:\n for spec in specs:\n env.add(spec)\n\n # `spack concretize`\n tests = compute_tests_install_kwargs(env.user_specs, args.test)\n concretized_specs = env.concretize(tests=tests)\n if concretized_specs:\n tty.msg(f\"Concretized {plural(len(concretized_specs), 'spec')}\")\n ev.display_specs([concrete for _, concrete in concretized_specs])\n\n # save view regeneration for later, so that we only do it\n # once, as it can be slow.\n env.write(regenerate=False)\n\n\ndef install_with_active_env(env: ev.Environment, args, install_kwargs, reporter):\n specs = spack.cmd.parse_specs(args.spec)\n\n # The following two commands are equivalent:\n # 1. `spack install --add x y z`\n # 2. `spack add x y z && spack concretize && spack install --only-concrete`\n # here we do the `add` and `concretize` part.\n _maybe_add_and_concretize(args, env, specs)\n\n # Now we're doing `spack install --only-concrete`.\n if args.add or not specs:\n specs_to_install = env.concrete_roots()\n if not specs_to_install:\n tty.msg(f\"{env.name} environment has no specs to install\")\n return\n\n # `spack install x y z` without --add is installing matching specs in the env.\n else:\n specs_to_install = env.all_matching_specs(*specs)\n if not specs_to_install:\n msg = (\n \"Cannot install '{0}' because no matching specs are in the current environment.\\n\"\n \" Specs can be added to the environment with 'spack add {0}',\\n\"\n \" or as part of the install command with 'spack install --add {0}'\"\n ).format(\" \".join(args.spec))\n tty.die(msg)\n\n install_kwargs[\"tests\"] = compute_tests_install_kwargs(specs_to_install, args.test)\n\n if args.overwrite:\n require_user_confirmation_for_overwrite(specs_to_install, args)\n install_kwargs[\"overwrite\"] = [spec.dag_hash() for spec in specs_to_install]\n\n try:\n report_file = report_filename(args, specs_to_install)\n install_kwargs[\"report_file\"] = report_file\n install_kwargs[\"reporter\"] = reporter\n env.install_specs(specs_to_install, **install_kwargs)\n finally:\n if env.views:\n with env.write_transaction():\n env.write(regenerate=True)\n\n\ndef concrete_specs_from_cli(args, install_kwargs):\n \"\"\"Return abstract and concrete spec parsed from the command line.\"\"\"\n abstract_specs = spack.cmd.parse_specs(args.spec)\n install_kwargs[\"tests\"] = compute_tests_install_kwargs(abstract_specs, args.test)\n try:\n concrete_specs = spack.cmd.parse_specs(\n args.spec, concretize=True, tests=install_kwargs[\"tests\"]\n )\n except SpackError as e:\n tty.debug(e)\n if args.log_format is not None:\n reporter = args.reporter()\n reporter.concretization_report(report_filename(args, abstract_specs), e.message)\n raise\n return concrete_specs\n\n\ndef install_without_active_env(args, install_kwargs, reporter):\n concrete_specs = concrete_specs_from_cli(args, install_kwargs)\n\n if len(concrete_specs) == 0:\n tty.die(\"The `spack install` command requires a spec to install.\")\n\n if args.overwrite:\n require_user_confirmation_for_overwrite(concrete_specs, args)\n install_kwargs[\"overwrite\"] = [spec.dag_hash() for spec in concrete_specs]\n\n installs = [s.package for s in concrete_specs]\n install_kwargs[\"explicit\"] = [s.dag_hash() for s in concrete_specs]\n\n try:\n builder = spack.installer_dispatch.create_installer(\n installs, create_reports=reporter is not None, **install_kwargs\n )\n builder.install()\n finally:\n if reporter:\n report_file = report_filename(args, concrete_specs)\n reporter.build_report(report_file, list(builder.reports.values()))\n" }, { "path": "lib/spack/spack/cmd/installer/CMakeLists.txt", "content": "cmake_minimum_required (VERSION 3.13)\nproject(spack_installer NONE)\n\nset(PYTHON_VERSION \"3.9.0\" CACHE STRING \"Version of Python to build.\")\nset(PY_DOWNLOAD_LINK \"https://www.paraview.org/files/dependencies\")\nset(PY_FILENAME \"Python-${PYTHON_VERSION}-win64.tar.xz\")\nset(PYTHON_DIR \"Python-${PYTHON_VERSION}\")\n\nif (SPACK_VERSION)\n set(SPACK_DL \"https://github.com/spack/spack/releases/download/v${SPACK_VERSION}\")\n set(SPACK_FILENAME \"spack-${SPACK_VERSION}.tar.gz\")\n set(SPACK_DIR \"spack-${SPACK_VERSION}\")\n\n # SPACK DOWNLOAD AND EXTRACTION-----------------------------------\n file(DOWNLOAD \"${SPACK_DL}/${SPACK_FILENAME}\"\n \"${CMAKE_CURRENT_BINARY_DIR}/${SPACK_FILENAME}\"\n STATUS download_status\n )\n list(GET download_status 0 res)\n if(res)\n list(GET download_status 1 err)\n message(FATAL_ERROR \"Failed to download ${SPACK_FILENAME} ${err}\")\n endif()\n message(STATUS \"Successfully downloaded ${SPACK_FILENAME}\")\n\n execute_process(COMMAND ${CMAKE_COMMAND} -E tar xfz\n \"${CMAKE_CURRENT_BINARY_DIR}/${SPACK_FILENAME}\"\n WORKING_DIRECTORY \"${CMAKE_CURRENT_BINARY_DIR}\"\n RESULT_VARIABLE res)\n if(NOT res EQUAL 0)\n message(FATAL_ERROR \"Extraction of ${SPACK_FILENAME} failed.\")\n endif()\n message(STATUS \"Extracted ${SPACK_DIR}\")\n SET(SPACK_SOURCE \"${CMAKE_CURRENT_BINARY_DIR}/${SPACK_DIR}\")\nelseif(SPACK_SOURCE)\n get_filename_component(SPACK_DIR ${SPACK_SOURCE} NAME)\nelse()\n message(FATAL_ERROR \"Must specify SPACK_VERSION or SPACK_SOURCE\")\nendif()\n\n\n# GIT DOWNLOAD----------------------------------------------------\nset(GIT_FILENAME \"Git-2.31.1-64-bit.exe\")\nfile(DOWNLOAD \"https://github.com/git-for-windows/git/releases/download/v2.31.1.windows.1/Git-2.31.1-64-bit.exe\"\n \"${CMAKE_CURRENT_BINARY_DIR}/${GIT_FILENAME}\"\n STATUS download_status\n EXPECTED_HASH \"SHA256=c43611eb73ad1f17f5c8cc82ae51c3041a2e7279e0197ccf5f739e9129ce426e\"\n)\nlist(GET download_status 0 res)\nif(res)\n list(GET download_status 1 err)\n message(FATAL_ERROR \"Failed to download ${GIT_FILENAME} ${err}\")\nendif()\nmessage(STATUS \"Successfully downloaded ${GIT_FILENAME}\")\n\n\n# PYTHON DOWNLOAD AND EXTRACTION-----------------------------------\nfile(DOWNLOAD \"${PY_DOWNLOAD_LINK}/${PY_FILENAME}\"\n \"${CMAKE_CURRENT_BINARY_DIR}/${PY_FILENAME}\"\n STATUS download_status\n EXPECTED_HASH \"SHA256=f6aeebc6d1ff77418678ed5612b64ce61be6bc9ef3ab9c291ac557abb1783420\"\n)\nlist(GET download_status 0 res)\nif(res)\n list(GET download_status 1 err)\n message(FATAL_ERROR \"Failed to download ${PY_FILENAME} ${err}\")\nendif()\nmessage(STATUS \"Successfully downloaded ${PY_FILENAME}\")\n\nexecute_process(COMMAND ${CMAKE_COMMAND} -E tar xfz\n \"${CMAKE_CURRENT_BINARY_DIR}/${PY_FILENAME}\"\n WORKING_DIRECTORY \"${CMAKE_CURRENT_BINARY_DIR}\"\n RESULT_VARIABLE res)\nif(NOT res EQUAL 0)\n message(FATAL_ERROR \"Extraction of ${PY_FILENAME} failed.\")\nendif()\nmessage(STATUS \"Extracted ${PY_FILENAME}.\")\n\n# license must be a .txt or .rtf file\nconfigure_file(\"${SPACK_LICENSE}\" \"${CMAKE_CURRENT_BINARY_DIR}/LICENSE.rtf\" COPYONLY)\n\n\n#INSTALLATION COMMANDS---------------------------------------------------\ninstall(DIRECTORY \"${SPACK_SOURCE}/\"\n DESTINATION \"${SPACK_DIR}\")\ninstall(DIRECTORY \"${CMAKE_CURRENT_BINARY_DIR}/Python-${PYTHON_VERSION}-win64/\"\n DESTINATION \"${PYTHON_DIR}\")\n\n# CPACK Installer Instructions\nset(CPACK_PACKAGE_NAME \"Spack\")\nset(CPACK_PACKAGE_VENDOR \"Lawrence Livermore National Laboratories\")\nset(CPACK_PACKAGE_VERSION \"0.16.0\")\nset(CPACK_PACKAGE_DESCRIPTION \"A flexible package manager designed to support multiple versions, configurations, platforms, and compilers.\")\nset(CPACK_PACKAGE_HOMEPAGE_URL \"https://spack.io\")\nset(CPACK_PACKAGE_FILE_NAME \"${CPACK_PACKAGE_NAME}\")\nset(CPACK_PACKAGE_ICON \"${SPACK_LOGO}\")\nset(CPACK_RESOURCE_FILE_README \"${CMAKE_CURRENT_SOURCE_DIR}/README.md\")\nset(CPACK_RESOURCE_FILE_LICENSE \"${CMAKE_CURRENT_BINARY_DIR}/LICENSE.rtf\")\n#set(CPACK_RESOURCE_FILE_WELCOME \"${CMAKE_CURRENT_SOURCE_DIR}/NOTICE\")\n# WIX options (the default)\n\nset(CPACK_GENERATOR \"WIX\")\nset(CPACK_WIX_PRODUCT_ICON \"${SPACK_LOGO}\")\nset(CPACK_WIX_UI_BANNER \"${CMAKE_CURRENT_SOURCE_DIR}/banner493x58.bmp\")\nset(CPACK_WIX_PATCH_FILE \"${CMAKE_CURRENT_SOURCE_DIR}/patch.xml\")\nset(CPACK_WIX_UPGRADE_GUID \"D2C703E4-721D-44EC-8016-BCB96BB64E0B\")\nset(CPACK_WIX_SKIP_PROGRAM_FOLDER TRUE)\n\nset(SHORTCUT_GUID \"099213BC-0D37-4F29-B758-60CA2A7E6DDA\")\n# Set full path to icon, shortcut in spack.wxs\nset(SPACK_SHORTCUT \"spack_cmd.bat\")\nconfigure_file(\"spack.wxs.in\" \"${CMAKE_CURRENT_BINARY_DIR}/spack.wxs\")\nconfigure_file(\"bundle.wxs.in\" \"${CMAKE_CURRENT_BINARY_DIR}/bundle.wxs\")\nset(CPACK_WIX_EXTRA_SOURCES \"${CMAKE_CURRENT_BINARY_DIR}/spack.wxs\")\n\ninclude(CPack)\n" }, { "path": "lib/spack/spack/cmd/installer/README.md", "content": "This README is a guide for creating a Spack installer for Windows using the\n``make-installer`` command. The installer is an executable file that users\ncan run to install Spack like any other Windows binary.\n\nBefore proceeding, follow the setup instructions in Steps 1 and 2 of\n[Getting Started on Windows](https://spack.readthedocs.io/en/latest/getting_started.html#windows_support).\n\n# Step 1: Install prerequisites\n\nThe only additional prerequisite for making the installer is Wix. Wix is a\nutility used for .msi creation and can be downloaded and installed at\nhttps://wixtoolset.org/releases/. The Visual Studio extensions are not\nnecessary.\n\n# Step 2: Make the installer\n\nTo use Spack, run ``spack_cmd.bat``. This will provide a Windows command\nprompt with an environment properly set up with Spack and its prerequisites.\n\nEnsure that Python and CMake are on your PATH. If needed, you may add the\nCMake executable provided by Visual Studio to your path, which will look\nsomething like:\n\n``C:\\Program Files (x86)\\Microsoft Visual Studio\\\\\\Common7\\IDE\\CommonExtensions\\Microsoft\\CMake\\CMake``\n\n**IMPORTANT**: If you use Tab to complete any part of this path, the console\nwill automatically add quotation marks to the start and the end since it will\nsee the spaces and want to parse the whole of it as a string. This is\nincorrect for our purposes so before submitting the command, ensure that the\nquotes are removed. You will encounter configuration errors if you fail to do\nthis.\n\nThere are two ways to create the installer using Spack's ``make-installer``\ncommand. The recommended method is to build the installer using a local\ncheckout of Spack source (release or development), using the\n`-s` flag to specify the directory where the local checkout is. For\nexample, if the local checkout is in a directory called ``spack-develop``\nand want to generate an installer with the source there, you can use:\n\n``spack make-installer -s spack-develop tmp``\n\nBoth the Spack source directory (e.g. ``spack-develop``) and installer\ndestination directory (e.g. ``tmp``) may be an absolute path or relative to\nthe current working directory. The entire contents of the specified\ndirectory will be included in the installer (e.g. .git files or local\nchanges).\n\nAlternatively, if you would like to create an installer from a release version\nof Spack, say, 0.16.0, and store it in ``tmp``, you can use the following\ncommand:\n\n``spack make-installer -v 0.16.0 tmp``\n\n**IMPORTANT**: Windows features are not currently supported in Spack's\nofficial release branches, so an installer created using this method will\n*not* run on Windows.\n\nRegardless of your method, a file called ``Spack.exe`` will be created\ninside the destination directory. This executable bundles the Spack installer\n(``Spack.msi`` also located in destination directory) and the git installer.\n\n# Step 3: Run the installer\n\nAfter accepting the terms of service, select where on your computer you would\nlike Spack installed, and after a few minutes Spack, Python and git will be\ninstalled and ready for use.\n\n**IMPORTANT**: To avoid permissions issues, it is recommended to select an\ninstall location other than ``C:\\Program Files``.\n\n**IMPORTANT**: There is a specific option that must be chosen when letting Git\ninstall. When given the option of adjusting your ``PATH``, choose the\n``Git from the command line and also from 3rd-party software`` option. This will\nautomatically update your ``PATH`` variable to include the ``git`` command.\nCertain Spack commands expect ``git`` to be part of the ``PATH``. If this step\nis not performed properly, certain Spack commands will not work.\n\nIf your Spack installation needs to be modified, repaired, or uninstalled,\nyou can do any of these things by rerunning ``Spack.exe``.\n\nRunning the installer creates a shortcut on your desktop that, when\nlaunched, will run ``spack_cmd.bat`` and launch a console with its initial\ndirectory being wherever Spack was installed on your computer. If Python is\nfound on your PATH, that will be used. If not, the Python included with the\ninstaller will be used when running Spack.\n" }, { "path": "lib/spack/spack/cmd/installer/bundle.wxs.in", "content": "\n\n \n \n\t\t\n \n \n \n \n \n \n\n" }, { "path": "lib/spack/spack/cmd/installer/patch.xml", "content": "\n\n\t\n\t\t\n\t\n\t\n\t\t\n\t\t\n\t\n" }, { "path": "lib/spack/spack/cmd/installer/spack.wxs.in", "content": "\n\n\n \n \n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n\n \n \n \n \n\n\n" }, { "path": "lib/spack/spack/cmd/license.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport enum\nimport os\nimport re\nfrom collections import defaultdict\nfrom typing import Dict, Generator\n\nimport spack.llnl.util.tty as tty\nimport spack.paths\n\ndescription = \"list and check license headers on files in spack\"\nsection = \"query\"\nlevel = \"long\"\n\n#: SPDX license id must appear in the first lines of a file\nlicense_lines = 6\n\n#: Spack's license identifier\napache2_mit_spdx = \"(Apache-2.0 OR MIT)\"\n\nsubdirs = (\"bin\", \"lib\", \"share\", \".github\")\n\n#: regular expressions for licensed files.\nlicensed_files_patterns = [\n # spack scripts\n r\"^bin/spack$\",\n r\"^bin/spack\\.bat$\",\n r\"^bin/spack\\.ps1$\",\n r\"^bin/spack_pwsh\\.ps1$\",\n r\"^bin/sbang$\",\n r\"^bin/spack-python$\",\n r\"^bin/haspywin\\.py$\",\n # all of spack core except unparse\n r\"^lib/spack/spack/(?!vendor/|util/unparse|util/ctest_log_parser|test/util/unparse).*\\.py$\",\n r\"^lib/spack/spack/.*\\.sh$\",\n r\"^lib/spack/spack/.*\\.lp$\",\n r\"^lib/spack/llnl/.*\\.py$\",\n # 1 file in vendored packages\n r\"^lib/spack/spack/vendor/__init__.py$\",\n # special case some test data files that have license headers\n r\"^lib/spack/spack/test/data/style/broken.dummy\",\n r\"^lib/spack/spack/test/data/unparse/.*\\.txt\",\n # rst files in documentation\n r\"^lib/spack/docs/(?!command_index|spack).*\\.rst$\",\n r\"^lib/spack/docs/(?!\\.spack/|\\.spack-env/).*\\.py$\",\n r\"^lib/spack/docs/spack.yaml$\",\n # shell scripts in share\n r\"^share/spack/.*\\.sh$\",\n r\"^share/spack/.*\\.bash$\",\n r\"^share/spack/.*\\.csh$\",\n r\"^share/spack/.*\\.fish$\",\n r\"^share/spack/setup-env\\.ps1$\",\n r\"^share/spack/qa/run-[^/]*$\",\n r\"^share/spack/qa/*.py$\",\n r\"^share/spack/bash/spack-completion.in$\",\n # action workflows\n r\"^.github/actions/.*\\.py$\",\n]\n\n\ndef _licensed_files(root: str = spack.paths.prefix) -> Generator[str, None, None]:\n \"\"\"Generates paths of licensed files.\"\"\"\n licensed_files = re.compile(\"|\".join(f\"(?:{pattern})\" for pattern in licensed_files_patterns))\n dirs = [\n os.path.join(root, subdir)\n for subdir in subdirs\n if os.path.isdir(os.path.join(root, subdir))\n ]\n\n while dirs:\n with os.scandir(dirs.pop()) as it:\n for entry in it:\n if entry.is_dir(follow_symlinks=False):\n dirs.append(entry.path)\n elif entry.is_file(follow_symlinks=False):\n relpath = os.path.relpath(entry.path, root)\n if licensed_files.match(relpath):\n yield relpath\n\n\ndef list_files(args):\n \"\"\"list files in spack that should have license headers\"\"\"\n for relpath in sorted(_licensed_files(args.root)):\n print(os.path.join(spack.paths.spack_root, relpath))\n\n\n# Error codes for license verification. All values are chosen such that\n# bool(value) evaluates to True\nclass ErrorType(enum.Enum):\n SPDX_MISMATCH = 1\n NOT_IN_FIRST_N_LINES = 2\n GENERAL_MISMATCH = 3\n\n\n#: regexes for valid license lines at tops of files\nlicense_line_regexes = [\n r\"Copyright (Spack|sbang) [Pp]roject [Dd]evelopers\\. See COPYRIGHT file for details.\",\n r\"\",\n r\"SPDX-License-Identifier: \\(Apache-2\\.0 OR MIT\\)\",\n]\n\n\nclass LicenseError:\n error_counts: Dict[ErrorType, int]\n\n def __init__(self):\n self.error_counts = defaultdict(int)\n\n def add_error(self, error):\n self.error_counts[error] += 1\n\n def has_errors(self):\n return sum(self.error_counts.values()) > 0\n\n def error_messages(self):\n total = sum(self.error_counts.values())\n missing = self.error_counts[ErrorType.GENERAL_MISMATCH]\n lines = self.error_counts[ErrorType.NOT_IN_FIRST_N_LINES]\n spdx_mismatch = self.error_counts[ErrorType.SPDX_MISMATCH]\n return (\n f\"{total} improperly licensed files\",\n f\"files with wrong SPDX-License-Identifier: {spdx_mismatch}\",\n f\"files without license in first {license_lines} lines: {lines}\",\n f\"files not containing expected license: {missing}\",\n )\n\n\ndef _check_license(lines, path):\n def sanitize(line):\n return re.sub(r\"^[\\s#\\%\\.\\:]*\", \"\", line).rstrip()\n\n for i, line in enumerate(lines):\n if all(\n re.match(regex, sanitize(lines[i + j])) for j, regex in enumerate(license_line_regexes)\n ):\n return\n\n if i >= (license_lines - len(license_line_regexes)):\n print(f\"{path}: License not found in first {license_lines} lines\")\n return ErrorType.NOT_IN_FIRST_N_LINES\n\n # If the SPDX identifier is present, then there is a mismatch (since it\n # did not match the above regex)\n def wrong_spdx_identifier(line, path):\n m = re.search(r\"SPDX-License-Identifier: ([^\\n]*)\", line)\n if m and m.group(1) != apache2_mit_spdx:\n print(\n f\"{path}: SPDX license identifier mismatch \"\n f\"(expecting {apache2_mit_spdx}, found {m.group(1)})\"\n )\n return ErrorType.SPDX_MISMATCH\n\n checks = [wrong_spdx_identifier]\n\n for line in lines:\n for check in checks:\n error = check(line, path)\n if error:\n return error\n\n print(f\"{path}: the license header at the top of the file does not match the expected format\")\n return ErrorType.GENERAL_MISMATCH\n\n\ndef verify(args):\n \"\"\"verify that files in spack have the right license header\"\"\"\n\n license_errors = LicenseError()\n\n for relpath in _licensed_files(args.root):\n path = os.path.join(args.root, relpath)\n with open(path, encoding=\"utf-8\") as f:\n lines = [line for line in f][:license_lines]\n\n error = _check_license(lines, path)\n if error:\n license_errors.add_error(error)\n\n if license_errors.has_errors():\n tty.die(*license_errors.error_messages())\n else:\n tty.msg(\"No license issues found.\")\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n subparser.add_argument(\n \"--root\",\n action=\"store\",\n default=spack.paths.prefix,\n help=\"scan a different prefix for license issues\",\n )\n\n sp = subparser.add_subparsers(metavar=\"SUBCOMMAND\", dest=\"license_command\")\n sp.add_parser(\"list-files\", help=list_files.__doc__)\n sp.add_parser(\"verify\", help=verify.__doc__)\n\n\ndef license(parser, args):\n commands = {\"list-files\": list_files, \"verify\": verify}\n return commands[args.license_command](args)\n" }, { "path": "lib/spack/spack/cmd/list.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport fnmatch\nimport json\nimport math\nimport os\nimport re\nimport sys\nfrom html import escape\nfrom typing import Optional, Type\n\nimport spack.deptypes as dt\nimport spack.llnl.util.tty as tty\nimport spack.package_base\nimport spack.repo\nimport spack.util.git\nfrom spack.cmd.common import arguments\nfrom spack.llnl.util.filesystem import working_dir\nfrom spack.llnl.util.tty.colify import colify\nfrom spack.util.url import path_to_file_url\nfrom spack.version import VersionList\n\ndescription = \"list and search available packages\"\nsection = \"query\"\nlevel = \"short\"\n\n\nformatters = {}\n\n\ndef formatter(func):\n \"\"\"Decorator used to register formatters\"\"\"\n formatters[func.__name__] = func\n return func\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n subparser.add_argument(\n \"filter\",\n nargs=argparse.REMAINDER,\n help=\"optional case-insensitive glob patterns to filter results\",\n )\n subparser.add_argument(\n \"-r\",\n \"--repo\",\n \"-N\",\n \"--namespace\",\n dest=\"repos\",\n action=\"append\",\n default=[],\n help=\"only list packages from the specified repo/namespace\",\n )\n subparser.add_argument(\n \"-d\",\n \"--search-description\",\n action=\"store_true\",\n default=False,\n help=\"filtering will also search the description for a match\",\n )\n subparser.add_argument(\n \"--format\",\n default=\"name_only\",\n choices=formatters,\n help=\"format to be used to print the output [default: name_only]\",\n )\n subparser.add_argument(\n \"-v\",\n \"--virtuals\",\n action=\"store_true\",\n default=False,\n help=\"include virtual packages in list\",\n )\n arguments.add_common_arguments(subparser, [\"tags\"])\n\n # Doesn't really make sense to update in count mode.\n count_or_update = subparser.add_mutually_exclusive_group()\n count_or_update.add_argument(\n \"--count\",\n action=\"store_true\",\n default=False,\n help=\"display the number of packages that would be listed\",\n )\n count_or_update.add_argument(\n \"--update\",\n metavar=\"FILE\",\n default=None,\n action=\"store\",\n help=\"write output to the specified file, if any package is newer\",\n )\n\n\ndef filter_by_name(pkgs, args):\n \"\"\"\n Filters the sequence of packages according to user prescriptions\n\n Args:\n pkgs: sequence of packages\n args: parsed command line arguments\n\n Returns:\n filtered and sorted list of packages\n \"\"\"\n if args.filter:\n res = []\n for f in args.filter:\n if \"*\" not in f and \"?\" not in f:\n r = fnmatch.translate(\"*\" + f + \"*\")\n else:\n r = fnmatch.translate(f)\n\n rc = re.compile(r, flags=re.IGNORECASE)\n res.append(rc)\n\n if args.search_description:\n\n def match(p, f):\n if f.match(p):\n return True\n\n pkg_cls = spack.repo.PATH.get_pkg_class(p)\n if pkg_cls.__doc__:\n return f.match(pkg_cls.__doc__)\n return False\n\n else:\n\n def match(p, f):\n return f.match(p)\n\n pkgs = [p for p in pkgs if any(match(p, f) for f in res)]\n\n return sorted(pkgs, key=lambda s: s.lower())\n\n\n@formatter\ndef name_only(pkgs, out):\n indent = 0\n colify(pkgs, indent=indent, output=out)\n if out.isatty():\n tty.msg(\"%d packages\" % len(pkgs))\n\n\ndef github_url(pkg: Type[spack.package_base.PackageBase]) -> Optional[str]:\n \"\"\"Link to a package file in spack package's github or the path to the file.\n\n Args:\n pkg: package instance\n\n Returns: URL to the package file on github or the local file path; otherwise, ``None``.\n \"\"\"\n git = None\n module_path = f\"{pkg.__module__.replace('.', '/')}.py\"\n for repo in spack.repo.PATH.repos:\n if not repo.python_path:\n continue\n\n path = os.path.join(repo.python_path, module_path)\n if not os.path.exists(path):\n continue\n\n git = git or spack.util.git.git()\n if not git:\n tty.debug(\"Cannot determine package URL for {pkg} without 'git', using path URL\")\n return path_to_file_url(path)\n\n tty.debug(f\"Checking git for repository path '{path}'\")\n with working_dir(os.path.dirname(path)):\n origin_url = git(\n \"config\",\n \"--get\",\n \"remote.origin.url\",\n output=str,\n error=os.devnull,\n fail_on_error=False,\n )\n\n if not origin_url:\n tty.debug(\"Cannot determine remote origin url, using path URL\")\n return path_to_file_url(path)\n\n # Handle spack repositories cloned with any scheme (e.g., ssh) by\n # ignoring the scheme designation.\n if any([name in origin_url for name in [\"spack.git\", \"spack-packages.git\"]]):\n git_repo = (origin_url.split(\"/\")[-1]).replace(\".git\", \"\").strip()\n prefix = git(\n \"rev-parse\", \"--show-prefix\", output=str, error=os.devnull, fail_on_error=False\n )\n return (\n f\"https://github.com/spack/{git_repo}/blob/develop/{prefix.strip()}package.py\"\n )\n\n tty.debug(f\"Unrecognized repository for {pkg}, using path URL\")\n return path_to_file_url(path)\n\n tty.debug(f\"Unable to determine the package repository URL for {pkg}\")\n return None\n\n\ndef rows_for_ncols(elts, ncols):\n \"\"\"Print out rows in a table with ncols of elts laid out vertically.\"\"\"\n clen = int(math.ceil(len(elts) / ncols))\n for r in range(clen):\n row = []\n for c in range(ncols):\n i = c * clen + r\n row.append(elts[i] if i < len(elts) else None)\n yield row\n\n\ndef get_dependencies(pkg):\n all_deps = {}\n for deptype in dt.ALL_TYPES:\n deps = pkg.dependencies_of_type(dt.flag_from_string(deptype))\n all_deps[deptype] = [d for d in deps]\n\n return all_deps\n\n\n@formatter\ndef version_json(pkg_names, out):\n \"\"\"Print all packages with their latest versions.\"\"\"\n pkg_classes = [spack.repo.PATH.get_pkg_class(name) for name in pkg_names]\n\n out.write(\"[\\n\")\n\n # output name and latest version for each package\n pkg_latest = \",\\n\".join(\n [\n ' {{\"name\": \"{0}\",\\n'\n ' \"latest_version\": \"{1}\",\\n'\n ' \"versions\": {2},\\n'\n ' \"homepage\": \"{3}\",\\n'\n ' \"file\": \"{4}\",\\n'\n ' \"maintainers\": {5},\\n'\n ' \"dependencies\": {6}'\n \"}}\".format(\n pkg_cls.name,\n VersionList(pkg_cls.versions).preferred(),\n json.dumps([str(v) for v in reversed(sorted(pkg_cls.versions))]),\n pkg_cls.homepage,\n github_url(pkg_cls),\n json.dumps(pkg_cls.maintainers),\n json.dumps(get_dependencies(pkg_cls)),\n )\n for pkg_cls in pkg_classes\n ]\n )\n out.write(pkg_latest)\n # important: no trailing comma in JSON arrays\n out.write(\"\\n]\\n\")\n\n\n@formatter\ndef html(pkg_names, out):\n \"\"\"Print out information on all packages in Sphinx HTML.\n\n This is intended to be inlined directly into Sphinx documentation.\n We write HTML instead of RST for speed; generating RST from *all*\n packages causes the Sphinx build to take forever. Including this as\n raw HTML is much faster.\n \"\"\"\n\n # Read in all packages\n pkg_classes = [spack.repo.PATH.get_pkg_class(name) for name in pkg_names]\n\n # Start at 2 because the title of the page from Sphinx is id1.\n span_id = 2\n\n # HTML header with an increasing id span\n def head(n, span_id, title, anchor=None):\n if anchor is None:\n anchor = title\n out.write(\n (\n ''\n '

%s'\n \"

\\n\"\n )\n % (span_id, title, anchor)\n )\n\n # Start with the number of packages, skipping the title and intro\n # blurb, which we maintain in the RST file.\n out.write(\"

\\n\")\n out.write(\"Spack currently has %d mainline packages:\\n\" % len(pkg_classes))\n out.write(\"

\\n\")\n\n # Table of links to all packages\n out.write('\\n')\n out.write('\\n')\n for i, row in enumerate(rows_for_ncols(pkg_names, 3)):\n out.write('\\n' if i % 2 == 0 else '\\n')\n for name in row:\n out.write(\"\\n' % (name, name))\n out.write(\"\\n\")\n out.write(\"\\n\")\n out.write(\"\\n\")\n out.write(\"
\\n\")\n out.write('%s
\\n\")\n out.write('
\\n')\n\n # Output some text for each package.\n for pkg_cls in pkg_classes:\n out.write('
\\n' % pkg_cls.name)\n head(2, span_id, pkg_cls.name)\n span_id += 1\n\n out.write('
\\n')\n\n out.write(\"
Homepage:
\\n\")\n out.write('
    \\n')\n\n if pkg_cls.homepage:\n out.write(\n ('
  • %s
    • \\n')\n % (pkg_cls.homepage, escape(pkg_cls.homepage, True))\n )\n else:\n out.write(\"No homepage\\n\")\n out.write(\"
\\n\")\n\n out.write(\"
Spack package:
\\n\")\n out.write('
    \\n')\n out.write(\n ('
  • %s/package.py
    • \\n')\n % (github_url(pkg_cls), pkg_cls.name)\n )\n out.write(\"
\\n\")\n\n if pkg_cls.versions:\n out.write(\"
Versions:
\\n\")\n out.write(\"
\\n\")\n out.write(\", \".join(str(v) for v in reversed(sorted(pkg_cls.versions))))\n out.write(\"\\n\")\n out.write(\"
\\n\")\n\n for deptype in dt.ALL_TYPES:\n deps = pkg_cls.dependencies_of_type(dt.flag_from_string(deptype))\n if deps:\n out.write(\"
%s Dependencies:
\\n\" % deptype.capitalize())\n out.write(\"
\\n\")\n out.write(\n \", \".join(\n (\n d\n if d not in pkg_names\n else '%s' % (d, d)\n )\n for d in deps\n )\n )\n out.write(\"\\n\")\n out.write(\"
\\n\")\n\n out.write(\"
Description:
\\n\")\n out.write(\"
\\n\")\n out.write(escape(pkg_cls.format_doc(indent=2), True))\n out.write(\"\\n\")\n out.write(\"
\\n\")\n out.write(\"
\\n\")\n\n out.write('
\\n')\n out.write(\"
\\n\")\n\n\ndef list(parser, args):\n # retrieve the formatter to use from args\n formatter = formatters[args.format]\n\n # Retrieve the names of all the packages\n repos = [spack.repo.PATH]\n if args.repos:\n repos = [spack.repo.PATH.get_repo(name) for name in args.repos]\n\n pkgs = {name for repo in repos for name in repo.all_package_names(args.virtuals)}\n\n # Filter the set appropriately\n sorted_packages = filter_by_name(pkgs, args)\n\n # If tags have been specified on the command line, filter by tags\n if args.tags:\n packages_with_tags = spack.repo.PATH.packages_with_tags(*args.tags)\n sorted_packages = [p for p in sorted_packages if p in packages_with_tags]\n\n if args.update:\n # change output stream if user asked for update\n if os.path.exists(args.update):\n if os.path.getmtime(args.update) > spack.repo.PATH.last_mtime():\n tty.msg(\"File is up to date: %s\" % args.update)\n return\n\n tty.msg(\"Updating file: %s\" % args.update)\n with open(args.update, \"w\", encoding=\"utf-8\") as f:\n formatter(sorted_packages, f)\n\n elif args.count:\n # just print the number of packages in the result\n print(len(sorted_packages))\n else:\n # print formatted package list\n formatter(sorted_packages, sys.stdout)\n" }, { "path": "lib/spack/spack/cmd/load.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport sys\n\nimport spack.cmd\nimport spack.cmd.common\nimport spack.environment as ev\nimport spack.store\nimport spack.user_environment as uenv\nfrom spack.cmd.common import arguments\n\ndescription = \"add package to the user environment\"\nsection = \"user environment\"\nlevel = \"short\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n \"\"\"Parser is only constructed so that this prints a nice help\n message with -h.\"\"\"\n arguments.add_common_arguments(subparser, [\"constraint\"])\n\n shells = subparser.add_mutually_exclusive_group()\n shells.add_argument(\n \"--sh\",\n action=\"store_const\",\n dest=\"shell\",\n const=\"sh\",\n help=\"print sh commands to load the package\",\n )\n shells.add_argument(\n \"--csh\",\n action=\"store_const\",\n dest=\"shell\",\n const=\"csh\",\n help=\"print csh commands to load the package\",\n )\n shells.add_argument(\n \"--fish\",\n action=\"store_const\",\n dest=\"shell\",\n const=\"fish\",\n help=\"print fish commands to load the package\",\n )\n shells.add_argument(\n \"--bat\",\n action=\"store_const\",\n dest=\"shell\",\n const=\"bat\",\n help=\"print bat commands to load the package\",\n )\n shells.add_argument(\n \"--pwsh\",\n action=\"store_const\",\n dest=\"shell\",\n const=\"pwsh\",\n help=\"print pwsh commands to load the package\",\n )\n\n subparser.add_argument(\n \"--first\",\n action=\"store_true\",\n default=False,\n dest=\"load_first\",\n help=\"load the first match if multiple packages match the spec\",\n )\n\n subparser.add_argument(\n \"--list\",\n action=\"store_true\",\n default=False,\n help=\"show loaded packages: same as ``spack find --loaded``\",\n )\n\n\ndef load(parser, args):\n env = ev.active_environment()\n\n if args.list:\n results = spack.cmd.filter_loaded_specs(args.specs())\n if sys.stdout.isatty():\n spack.cmd.print_how_many_pkgs(results, \"loaded\")\n spack.cmd.display_specs(results)\n return\n\n constraint_specs = spack.cmd.parse_specs(args.constraint)\n specs = [\n spack.cmd.disambiguate_spec(spec, env, first=args.load_first) for spec in constraint_specs\n ]\n\n if not args.shell:\n specs_str = \" \".join(str(s) for s in constraint_specs) or \"SPECS\"\n spack.cmd.common.shell_init_instructions(\n \"spack load\", f\" eval `spack load {{sh_arg}} {specs_str}`\"\n )\n return 1\n\n with spack.store.STORE.db.read_transaction():\n env_mod = uenv.environment_modifications_for_specs(*specs)\n for spec in specs:\n env_mod.prepend_path(uenv.spack_loaded_hashes_var, spec.dag_hash())\n cmds = env_mod.shell_modifications(args.shell)\n\n sys.stdout.write(cmds)\n" }, { "path": "lib/spack/spack/cmd/location.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport os\n\nimport spack.builder\nimport spack.cmd\nimport spack.environment as ev\nimport spack.llnl.util.tty as tty\nimport spack.paths\nimport spack.repo\nimport spack.stage\nfrom spack.cmd.common import arguments\n\ndescription = \"print out locations of packages and spack directories\"\nsection = \"query\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n directories = subparser.add_mutually_exclusive_group()\n\n directories.add_argument(\n \"-m\", \"--module-dir\", action=\"store_true\", help=\"spack python module directory\"\n )\n directories.add_argument(\n \"-r\", \"--spack-root\", action=\"store_true\", help=\"spack installation root\"\n )\n\n directories.add_argument(\n \"-i\",\n \"--install-dir\",\n action=\"store_true\",\n help=\"install prefix for spec (spec need not be installed)\",\n )\n directories.add_argument(\n \"-p\",\n \"--package-dir\",\n action=\"store_true\",\n help=\"directory enclosing a spec's package.py file\",\n )\n directories.add_argument(\n \"--repo\",\n # for backwards compatibility\n \"--packages\",\n \"-P\",\n nargs=\"?\",\n default=False,\n metavar=\"repo\",\n help=\"package repository root (defaults to first configured repository)\",\n )\n directories.add_argument(\n \"-s\", \"--stage-dir\", action=\"store_true\", help=\"stage directory for a spec\"\n )\n directories.add_argument(\n \"-S\", \"--stages\", action=\"store_true\", help=\"top level stage directory\"\n )\n directories.add_argument(\n \"-c\",\n \"--source-dir\",\n action=\"store_true\",\n help=\"source directory for a spec (requires it to be staged first)\",\n )\n directories.add_argument(\n \"-b\",\n \"--build-dir\",\n action=\"store_true\",\n help=\"build directory for a spec (requires it to be staged first)\",\n )\n directories.add_argument(\n \"-e\",\n \"--env\",\n action=\"store\",\n dest=\"location_env\",\n nargs=\"?\",\n metavar=\"name\",\n default=False,\n help=\"location of the named or current environment\",\n )\n\n subparser.add_argument(\n \"--first\",\n action=\"store_true\",\n default=False,\n dest=\"find_first\",\n help=\"use the first match if multiple packages match the spec\",\n )\n\n arguments.add_common_arguments(subparser, [\"spec\"])\n\n\ndef location(parser, args):\n if args.module_dir:\n print(spack.paths.module_path)\n return\n\n if args.spack_root:\n print(spack.paths.prefix)\n return\n\n # no -e corresponds to False, -e without arg to None, -e name to the string name.\n if args.location_env is not False:\n if args.location_env is None:\n # Get current environment path\n spack.cmd.require_active_env(\"location -e\")\n path = ev.active_environment().path\n else:\n # Get path of requested environment\n if not ev.exists(args.location_env):\n tty.die(\"no such environment: '%s'\" % args.location_env)\n path = ev.root(args.location_env)\n print(path)\n return\n\n if args.repo is not False:\n if args.repo is None:\n print(spack.repo.PATH.first_repo().root)\n return\n try:\n print(spack.repo.PATH.get_repo(args.repo).root)\n except spack.repo.UnknownNamespaceError:\n tty.die(f\"no such repository: '{args.repo}'\")\n return\n\n if args.stages:\n print(spack.stage.get_stage_root())\n return\n\n specs = spack.cmd.parse_specs(args.spec)\n\n if not specs:\n tty.die(\"You must supply a spec.\")\n\n if len(specs) != 1:\n tty.die(\"Too many specs. Supply only one.\")\n\n # install_dir command matches against installed specs.\n if args.install_dir:\n env = ev.active_environment()\n spec = spack.cmd.disambiguate_spec(specs[0], env, first=args.find_first)\n print(spec.prefix)\n return\n\n spec = specs[0]\n\n # Package dir just needs the spec name\n if args.package_dir:\n print(spack.repo.PATH.dirname_for_package_name(spec.name))\n return\n\n # Either concretize or filter from already concretized environment\n spec = spack.cmd.matching_spec_from_env(spec)\n pkg = spec.package\n builder = spack.builder.create(pkg)\n\n if args.stage_dir:\n print(pkg.stage.path)\n return\n\n if args.build_dir:\n # Out of source builds have build_directory defined\n if hasattr(builder, \"build_directory\"):\n # build_directory can be either absolute or relative to the stage path\n # in either case os.path.join makes it absolute\n print(os.path.normpath(os.path.join(pkg.stage.path, builder.build_directory)))\n return\n\n # Otherwise assume in-source builds\n print(pkg.stage.source_path)\n return\n\n # source dir remains, which requires the spec to be staged\n if not pkg.stage.expanded:\n tty.die(\n \"Source directory does not exist yet. Run this to create it:\",\n \"spack stage \" + \" \".join(args.spec),\n )\n\n # Default to source dir.\n print(pkg.stage.source_path)\n" }, { "path": "lib/spack/spack/cmd/log_parse.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport io\nimport sys\nimport warnings\n\nimport spack.llnl.util.tty as tty\nfrom spack.util.log_parse import make_log_context, parse_log_events\n\ndescription = \"filter errors and warnings from build logs\"\nsection = \"developer\"\nlevel = \"long\"\n\nevent_types = (\"errors\", \"warnings\")\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n subparser.add_argument(\n \"--show\",\n action=\"store\",\n default=\"errors\",\n help=\"comma-separated list of what to show; options: errors, warnings\",\n )\n subparser.add_argument(\n \"-c\",\n \"--context\",\n action=\"store\",\n type=int,\n default=3,\n help=\"lines of context to show around lines of interest\",\n )\n subparser.add_argument(\n \"-p\",\n \"--profile\",\n action=\"store_true\",\n help=\"print out a profile of time spent in regexes during parse\",\n )\n subparser.add_argument(\n \"-w\", \"--width\", action=\"store\", type=int, default=None, help=argparse.SUPPRESS\n )\n subparser.add_argument(\n \"-j\", \"--jobs\", action=\"store\", type=int, default=None, help=argparse.SUPPRESS\n )\n\n subparser.add_argument(\"file\", help=\"a log file containing build output, or - for stdin\")\n\n\ndef log_parse(parser, args):\n input = args.file\n if args.file == \"-\":\n input = io.TextIOWrapper(\n sys.stdin.buffer, encoding=\"utf-8\", errors=\"replace\", closefd=False\n )\n\n if args.width is not None:\n warnings.warn(\"The --width option is deprecated and will be removed in Spack v1.3\")\n if args.jobs is not None:\n warnings.warn(\"The --jobs option is deprecated and will be removed in Spack v1.3\")\n\n log_errors, log_warnings = parse_log_events(input, args.context, args.profile)\n if args.profile:\n return\n\n types = [s.strip() for s in args.show.split(\",\")]\n for e in types:\n if e not in event_types:\n tty.die(\"Invalid event type: %s\" % e)\n\n events = []\n if \"errors\" in types:\n events.extend(log_errors)\n print(\"%d errors\" % len(log_errors))\n if \"warnings\" in types:\n events.extend(log_warnings)\n print(\"%d warnings\" % len(log_warnings))\n\n print(make_log_context(events), end=\"\")\n" }, { "path": "lib/spack/spack/cmd/logs.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport errno\nimport gzip\nimport io\nimport os\nimport shutil\nimport sys\n\nimport spack.cmd\nimport spack.spec\nimport spack.util.compression as compression\nfrom spack.cmd.common import arguments\nfrom spack.error import SpackError\n\ndescription = \"print out logs for packages\"\nsection = \"query\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n arguments.add_common_arguments(subparser, [\"spec\"])\n\n\ndef _dump_byte_stream_to_stdout(instream: io.BufferedIOBase) -> None:\n # Reopen stdout in binary mode so we don't have to worry about encoding\n outstream = os.fdopen(sys.stdout.fileno(), \"wb\", closefd=False)\n shutil.copyfileobj(instream, outstream)\n\n\ndef _logs(cmdline_spec: spack.spec.Spec, concrete_spec: spack.spec.Spec):\n if concrete_spec.installed:\n log_path = concrete_spec.package.install_log_path\n elif os.path.exists(concrete_spec.package.stage.path):\n # TODO: `spack logs` can currently not show the logs while a package is being built, as the\n # combined log file is only written after the build is finished.\n log_path = concrete_spec.package.log_path\n else:\n raise SpackError(f\"{cmdline_spec} is not installed or staged\")\n\n try:\n stream = open(log_path, \"rb\")\n except OSError as e:\n if e.errno == errno.ENOENT:\n raise SpackError(f\"No logs are available for {cmdline_spec}\") from e\n raise SpackError(f\"Error reading logs for {cmdline_spec}: {e}\") from e\n\n with stream as f:\n ext = compression.extension_from_magic_numbers_by_stream(f, decompress=False)\n if ext and ext != \"gz\":\n raise SpackError(f\"Unsupported storage format for {log_path}: {ext}\")\n\n # If the log file is gzip compressed, wrap it with a decompressor\n _dump_byte_stream_to_stdout(gzip.GzipFile(fileobj=f) if ext == \"gz\" else f)\n\n\ndef logs(parser, args):\n specs = spack.cmd.parse_specs(args.spec)\n\n if not specs:\n raise SpackError(\"You must supply a spec.\")\n\n if len(specs) != 1:\n raise SpackError(\"Too many specs. Supply only one.\")\n\n concrete_spec = spack.cmd.matching_spec_from_env(specs[0])\n\n _logs(specs[0], concrete_spec)\n" }, { "path": "lib/spack/spack/cmd/maintainers.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nfrom collections import defaultdict\n\nimport spack.llnl.util.tty as tty\nimport spack.llnl.util.tty.color as color\nimport spack.repo\nfrom spack.llnl.util.tty.colify import colify\n\ndescription = \"get information about package maintainers\"\nsection = \"query\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n maintained_group = subparser.add_mutually_exclusive_group()\n maintained_group.add_argument(\n \"--maintained\",\n action=\"store_true\",\n default=False,\n help=\"show names of maintained packages\",\n )\n\n maintained_group.add_argument(\n \"--unmaintained\",\n action=\"store_true\",\n default=False,\n help=\"show names of unmaintained packages\",\n )\n\n subparser.add_argument(\n \"-a\", \"--all\", action=\"store_true\", default=False, help=\"show maintainers for all packages\"\n )\n\n subparser.add_argument(\n \"--by-user\",\n action=\"store_true\",\n default=False,\n help=\"show packages for users instead of users for packages\",\n )\n\n # options for commands that take package arguments\n subparser.add_argument(\n \"package_or_user\",\n nargs=argparse.REMAINDER,\n help=\"names of packages or users to get info for\",\n )\n\n\ndef packages_to_maintainers(package_names=None):\n if not package_names:\n package_names = spack.repo.PATH.all_package_names()\n\n pkg_to_users = defaultdict(lambda: set())\n for name in package_names:\n cls = spack.repo.PATH.get_pkg_class(name)\n for user in cls.maintainers:\n pkg_to_users[name].add(user)\n\n return pkg_to_users\n\n\ndef maintainers_to_packages(users=None):\n user_to_pkgs = defaultdict(lambda: [])\n for name in spack.repo.PATH.all_package_names():\n cls = spack.repo.PATH.get_pkg_class(name)\n for user in cls.maintainers:\n lower_users = [u.lower() for u in users]\n if not users or user.lower() in lower_users:\n user_to_pkgs[user].append(cls.name)\n\n return user_to_pkgs\n\n\ndef maintained_packages():\n maintained = []\n unmaintained = []\n for name in spack.repo.PATH.all_package_names():\n cls = spack.repo.PATH.get_pkg_class(name)\n if cls.maintainers:\n maintained.append(name)\n else:\n unmaintained.append(name)\n\n return maintained, unmaintained\n\n\ndef union_values(dictionary):\n \"\"\"Given a dictionary with values that are Collections, return their union.\n\n Arguments:\n dictionary (dict): dictionary whose values are all collections.\n\n Return:\n (set): the union of all collections in the dictionary's values.\n \"\"\"\n sets = [set(p) for p in dictionary.values()]\n return sorted(set.union(*sets)) if sets else set()\n\n\ndef maintainers(parser, args):\n if args.maintained or args.unmaintained:\n maintained, unmaintained = maintained_packages()\n pkgs = maintained if args.maintained else unmaintained\n colify(pkgs)\n return 0 if pkgs else 1\n\n if args.all:\n if args.by_user:\n maintainers = maintainers_to_packages(args.package_or_user)\n for user, packages in sorted(maintainers.items()):\n color.cprint(\"@c{%s}: %s\" % (user, \", \".join(sorted(packages))))\n return 0 if maintainers else 1\n\n else:\n packages = packages_to_maintainers(args.package_or_user)\n for pkg, maintainers in sorted(packages.items()):\n color.cprint(\"@c{%s}: %s\" % (pkg, \", \".join(sorted(maintainers))))\n return 0 if packages else 1\n\n if args.by_user:\n if not args.package_or_user:\n tty.die(\"spack maintainers --by-user requires a user or --all\")\n\n packages = union_values(maintainers_to_packages(args.package_or_user))\n colify(packages)\n return 0 if packages else 1\n\n else:\n if not args.package_or_user:\n tty.die(\"spack maintainers requires a package or --all\")\n\n users = union_values(packages_to_maintainers(args.package_or_user))\n colify(users)\n return 0 if users else 1\n" }, { "path": "lib/spack/spack/cmd/make_installer.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport argparse\nimport os\nimport posixpath\nimport sys\n\nimport spack.concretize\nimport spack.paths\nimport spack.util.executable\nfrom spack.llnl.path import convert_to_posix_path\n\ndescription = \"generate Windows installer\"\nsection = \"admin\"\nlevel = \"long\"\n\n\ndef txt_to_rtf(file_path):\n rtf_header = r\"\"\"{{\\rtf1\\ansi\\deff0\\nouicompat\n {{\\fonttbl{{\\f0\\\\fnil\\fcharset0 Courier New;}}}}\n {{\\colortbl ;\\red0\\green0\\blue255;}}\n {{\\*\\generator Riched20 10.0.19041}}\\viewkind4\\uc1\n \\f0\\fs22\\lang1033\n {}\n }}\n \"\"\"\n\n def line_to_rtf(str):\n return str.replace(\"\\n\", \"\\\\par\")\n\n contents = \"\"\n with open(file_path, \"r+\", encoding=\"utf-8\") as f:\n for line in f.readlines():\n contents += line_to_rtf(line)\n return rtf_header.format(contents)\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n spack_source_group = subparser.add_mutually_exclusive_group(required=True)\n spack_source_group.add_argument(\n \"-v\", \"--spack-version\", default=\"\", help=\"download given spack version\"\n )\n spack_source_group.add_argument(\n \"-s\", \"--spack-source\", default=\"\", help=\"full path to spack source\"\n )\n\n subparser.add_argument(\n \"-g\",\n \"--git-installer-verbosity\",\n default=\"\",\n choices=[\"SILENT\", \"VERYSILENT\"],\n help=\"level of verbosity provided by bundled git installer (default is fully verbose)\",\n required=False,\n action=\"store\",\n dest=\"git_verbosity\",\n )\n\n subparser.add_argument(\"output_dir\", help=\"output directory\")\n\n\ndef make_installer(parser, args):\n \"\"\"\n Use CMake to generate WIX installer in newly created build directory\n \"\"\"\n if sys.platform == \"win32\":\n output_dir = args.output_dir\n cmake_spec = spack.concretize.concretize_one(\"cmake\")\n cmake_path = os.path.join(cmake_spec.prefix, \"bin\", \"cmake.exe\")\n cpack_path = os.path.join(cmake_spec.prefix, \"bin\", \"cpack.exe\")\n spack_source = args.spack_source\n git_verbosity = \"\"\n if args.git_verbosity:\n git_verbosity = \"/\" + args.git_verbosity\n\n if spack_source:\n if not os.path.exists(spack_source):\n print(\"%s does not exist\" % spack_source)\n return\n else:\n if not os.path.isabs(spack_source):\n spack_source = posixpath.abspath(spack_source)\n spack_source = convert_to_posix_path(spack_source)\n\n spack_version = args.spack_version\n\n here = os.path.dirname(os.path.abspath(__file__))\n source_dir = os.path.join(here, \"installer\")\n posix_root = convert_to_posix_path(spack.paths.spack_root)\n spack_license = posixpath.join(posix_root, \"LICENSE-APACHE\")\n rtf_spack_license = txt_to_rtf(spack_license)\n spack_license = posixpath.join(source_dir, \"LICENSE.rtf\")\n\n with open(spack_license, \"w\", encoding=\"utf-8\") as rtf_license:\n written = rtf_license.write(rtf_spack_license)\n if written == 0:\n raise RuntimeError(\"Failed to generate properly formatted license file\")\n spack_logo = posixpath.join(posix_root, \"share/spack/logo/favicon.ico\")\n\n try:\n spack.util.executable.Executable(cmake_path)(\n \"-S\",\n source_dir,\n \"-B\",\n output_dir,\n \"-DSPACK_VERSION=%s\" % spack_version,\n \"-DSPACK_SOURCE=%s\" % spack_source,\n \"-DSPACK_LICENSE=%s\" % spack_license,\n \"-DSPACK_LOGO=%s\" % spack_logo,\n \"-DSPACK_GIT_VERBOSITY=%s\" % git_verbosity,\n )\n except spack.util.executable.ProcessError:\n print(\"Failed to generate installer\")\n return spack.util.executable.ProcessError.returncode\n\n try:\n spack.util.executable.Executable(cpack_path)(\n \"--config\", \"%s/CPackConfig.cmake\" % output_dir, \"-B\", \"%s/\" % output_dir\n )\n except spack.util.executable.ProcessError:\n print(\"Failed to generate installer\")\n return spack.util.executable.ProcessError.returncode\n try:\n spack.util.executable.Executable(os.environ.get(\"WIX\") + \"/bin/candle.exe\")(\n \"-ext\",\n \"WixBalExtension\",\n \"%s/bundle.wxs\" % output_dir,\n \"-out\",\n \"%s/bundle.wixobj\" % output_dir,\n )\n except spack.util.executable.ProcessError:\n print(\"Failed to generate installer chain\")\n return spack.util.executable.ProcessError.returncode\n try:\n spack.util.executable.Executable(os.environ.get(\"WIX\") + \"/bin/light.exe\")(\n \"-sw1134\",\n \"-ext\",\n \"WixBalExtension\",\n \"%s/bundle.wixobj\" % output_dir,\n \"-out\",\n \"%s/Spack.exe\" % output_dir,\n )\n except spack.util.executable.ProcessError:\n print(\"Failed to generate installer chain\")\n return spack.util.executable.ProcessError.returncode\n print(\"Successfully generated Spack.exe in %s\" % (output_dir))\n else:\n print(\"The make-installer command is currently only supported on Windows.\")\n" }, { "path": "lib/spack/spack/cmd/mark.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport sys\nfrom typing import List, Union\n\nimport spack.cmd\nimport spack.spec\nimport spack.store\nfrom spack.cmd.common import arguments\nfrom spack.llnl.util import tty\n\nfrom ..enums import InstallRecordStatus\n\ndescription = \"mark packages as explicitly or implicitly installed\"\nsection = \"build\"\nlevel = \"long\"\n\nerror_message = \"\"\"You can either:\n a) use a more specific spec, or\n b) use `spack mark --all` to mark ALL matching specs.\n\"\"\"\n\n# Arguments for display_specs when we find ambiguity\ndisplay_args = {\"long\": True, \"show_flags\": False, \"variants\": False, \"indent\": 4}\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n arguments.add_common_arguments(subparser, [\"installed_specs\"])\n subparser.add_argument(\n \"-a\",\n \"--all\",\n action=\"store_true\",\n dest=\"all\",\n help=\"mark ALL installed packages that match each supplied spec\",\n )\n exim = subparser.add_mutually_exclusive_group(required=True)\n exim.add_argument(\n \"-e\",\n \"--explicit\",\n action=\"store_true\",\n dest=\"explicit\",\n help=\"mark packages as explicitly installed\",\n )\n exim.add_argument(\n \"-i\",\n \"--implicit\",\n action=\"store_true\",\n dest=\"implicit\",\n help=\"mark packages as implicitly installed\",\n )\n\n\ndef find_matching_specs(\n specs: List[Union[str, spack.spec.Spec]], allow_multiple_matches: bool = False\n) -> List[spack.spec.Spec]:\n \"\"\"Returns a list of specs matching the not necessarily concretized specs given from cli\n\n Args:\n specs: list of specs to be matched against installed packages\n allow_multiple_matches: if True multiple matches are admitted\n \"\"\"\n # List of specs that match expressions given via command line\n specs_from_cli = []\n has_errors = False\n\n for spec in specs:\n matching = spack.store.STORE.db.query_local(spec, installed=InstallRecordStatus.INSTALLED)\n # For each spec provided, make sure it refers to only one package.\n # Fail and ask user to be unambiguous if it doesn't\n if not allow_multiple_matches and len(matching) > 1:\n tty.error(\"{0} matches multiple packages:\".format(spec))\n sys.stderr.write(\"\\n\")\n spack.cmd.display_specs(matching, output=sys.stderr, **display_args)\n sys.stderr.write(\"\\n\")\n sys.stderr.flush()\n has_errors = True\n\n # No installed package matches the query\n if len(matching) == 0 and spec is not None:\n tty.die(f\"{spec} does not match any installed packages.\")\n\n specs_from_cli.extend(matching)\n\n if has_errors:\n tty.die(error_message)\n\n return specs_from_cli\n\n\ndef do_mark(specs, explicit):\n \"\"\"Marks all the specs in a list.\n\n Args:\n specs (list): list of specs to be marked\n explicit (bool): whether to mark specs as explicitly installed\n \"\"\"\n with spack.store.STORE.db.write_transaction():\n for spec in specs:\n spack.store.STORE.db.mark(spec, \"explicit\", explicit)\n\n\ndef mark_specs(args, specs):\n mark_list = find_matching_specs(specs, args.all)\n\n # Mark everything on the list\n do_mark(mark_list, args.explicit)\n\n\ndef mark(parser, args):\n if not args.specs and not args.all:\n tty.die(\n \"mark requires at least one package argument.\",\n \" Use `spack mark --all` to mark ALL packages.\",\n )\n\n # [None] here handles the --all case by forcing all specs to be returned\n specs = spack.cmd.parse_specs(args.specs) if args.specs else [None]\n mark_specs(args, specs)\n" }, { "path": "lib/spack/spack/cmd/mirror.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport sys\nfrom concurrent.futures import as_completed\n\nimport spack.caches\nimport spack.cmd\nimport spack.concretize\nimport spack.config\nimport spack.environment as ev\nimport spack.llnl.util.lang as lang\nimport spack.llnl.util.tty as tty\nimport spack.llnl.util.tty.colify as colify\nimport spack.mirrors.mirror\nimport spack.mirrors.utils\nimport spack.repo\nimport spack.spec\nimport spack.util.parallel\nimport spack.util.web as web_util\nfrom spack.cmd.common import arguments\nfrom spack.error import SpackError\nfrom spack.llnl.string import comma_or\n\ndescription = \"manage mirrors (source and binary)\"\nsection = \"config\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n arguments.add_common_arguments(subparser, [\"no_checksum\"])\n\n sp = subparser.add_subparsers(metavar=\"SUBCOMMAND\", dest=\"mirror_command\")\n\n # Create\n create_parser = sp.add_parser(\"create\", help=mirror_create.__doc__)\n create_parser.add_argument(\n \"-d\", \"--directory\", default=None, help=\"directory in which to create mirror\"\n )\n create_parser.add_argument(\n \"-a\",\n \"--all\",\n action=\"store_true\",\n help=\"mirror all versions of all packages in Spack, or all packages\"\n \" in the current environment if there is an active environment\"\n \" (this requires significant time and space)\",\n )\n create_parser.add_argument(\n \"-j\",\n \"--jobs\",\n type=int,\n default=None,\n help=\"Use a given number of workers to make the mirror (used in combination with -a)\",\n )\n create_parser.add_argument(\"--file\", help=\"file with specs of packages to put in mirror\")\n create_parser.add_argument(\n \"--exclude-file\",\n help=\"specs which Spack should not try to add to a mirror\"\n \" (listed in a file, one per line)\",\n )\n create_parser.add_argument(\n \"--exclude-specs\",\n help=\"specs which Spack should not try to add to a mirror (specified on command line)\",\n )\n\n create_parser.add_argument(\n \"--skip-unstable-versions\",\n action=\"store_true\",\n help=\"don't cache versions unless they identify a stable (unchanging) source code\",\n )\n create_parser.add_argument(\n \"-D\", \"--dependencies\", action=\"store_true\", help=\"also fetch all dependencies\"\n )\n create_parser.add_argument(\n \"-n\",\n \"--versions-per-spec\",\n help=\"the number of versions to fetch for each spec, choose 'all' to\"\n \" retrieve all versions of each package\",\n )\n create_parser.add_argument(\n \"--private\",\n action=\"store_true\",\n help=\"for a private mirror, include non-redistributable packages\",\n )\n arguments.add_common_arguments(create_parser, [\"specs\"])\n arguments.add_concretizer_args(create_parser)\n\n # Destroy\n destroy_parser = sp.add_parser(\"destroy\", help=mirror_destroy.__doc__)\n\n destroy_target = destroy_parser.add_mutually_exclusive_group(required=True)\n destroy_target.add_argument(\n \"-m\",\n \"--mirror-name\",\n metavar=\"mirror_name\",\n type=str,\n help=\"find mirror to destroy by name\",\n )\n destroy_target.add_argument(\n \"--mirror-url\", metavar=\"mirror_url\", type=str, help=\"find mirror to destroy by url\"\n )\n\n # Add\n add_parser = sp.add_parser(\"add\", help=mirror_add.__doc__)\n add_parser.add_argument(\"name\", help=\"mnemonic name for mirror\", metavar=\"mirror\")\n add_parser.add_argument(\"url\", help=\"url of mirror directory from 'spack mirror create'\")\n add_parser.add_argument(\n \"--scope\",\n action=arguments.ConfigScope,\n default=lambda: spack.config.default_modify_scope(),\n help=\"configuration scope to modify\",\n )\n add_parser.add_argument(\n \"--type\",\n action=\"append\",\n choices=(\"binary\", \"source\"),\n help=(\n \"specify the mirror type: for both binary \"\n \"and source use ``--type binary --type source`` (default)\"\n ),\n )\n add_parser.add_argument(\n \"--autopush\",\n action=\"store_true\",\n help=(\"set mirror to push automatically after installation\"),\n )\n add_parser_signed = add_parser.add_mutually_exclusive_group(required=False)\n add_parser_signed.add_argument(\n \"--unsigned\",\n help=\"do not require signing and signature verification when pushing and installing from \"\n \"this build cache\",\n action=\"store_false\",\n default=None,\n dest=\"signed\",\n )\n add_parser_signed.add_argument(\n \"--signed\",\n help=\"require signing and signature verification when pushing and installing from this \"\n \"build cache\",\n action=\"store_true\",\n default=None,\n dest=\"signed\",\n )\n add_parser.add_argument(\n \"--name\",\n \"-n\",\n action=\"store\",\n dest=\"view_name\",\n help=\"Name of the index view for a binary mirror\",\n )\n arguments.add_connection_args(add_parser, False)\n # Remove\n remove_parser = sp.add_parser(\"remove\", aliases=[\"rm\"], help=mirror_remove.__doc__)\n remove_parser.add_argument(\"name\", help=\"mnemonic name for mirror\", metavar=\"mirror\")\n remove_parser.add_argument(\n \"--scope\", action=arguments.ConfigScope, default=None, help=\"configuration scope to modify\"\n )\n remove_parser.add_argument(\n \"--all-scopes\",\n action=\"store_true\",\n default=False,\n help=\"remove from all config scopes (default: highest scope with matching mirror)\",\n )\n # Set-Url\n set_url_parser = sp.add_parser(\"set-url\", help=mirror_set_url.__doc__)\n set_url_parser.add_argument(\"name\", help=\"mnemonic name for mirror\", metavar=\"mirror\")\n set_url_parser.add_argument(\"url\", help=\"url of mirror directory from 'spack mirror create'\")\n set_url_push_or_fetch = set_url_parser.add_mutually_exclusive_group(required=False)\n set_url_push_or_fetch.add_argument(\n \"--push\", action=\"store_true\", help=\"set only the URL used for uploading\"\n )\n set_url_push_or_fetch.add_argument(\n \"--fetch\", action=\"store_true\", help=\"set only the URL used for downloading\"\n )\n set_url_parser.add_argument(\n \"--scope\",\n action=arguments.ConfigScope,\n default=lambda: spack.config.default_modify_scope(),\n help=\"configuration scope to modify\",\n )\n arguments.add_connection_args(set_url_parser, False)\n\n # Set\n set_parser = sp.add_parser(\"set\", help=mirror_set.__doc__)\n set_parser.add_argument(\"name\", help=\"mnemonic name for mirror\", metavar=\"mirror\")\n set_parser_push_or_fetch = set_parser.add_mutually_exclusive_group(required=False)\n set_parser_push_or_fetch.add_argument(\n \"--push\", action=\"store_true\", help=\"modify just the push connection details\"\n )\n set_parser_push_or_fetch.add_argument(\n \"--fetch\", action=\"store_true\", help=\"modify just the fetch connection details\"\n )\n set_parser.add_argument(\n \"--type\",\n action=\"append\",\n choices=(\"binary\", \"source\"),\n help=(\n \"specify the mirror type: for both binary \"\n \"and source use ``--type binary --type source``\"\n ),\n )\n set_parser.add_argument(\"--url\", help=\"url of mirror directory from 'spack mirror create'\")\n set_parser_autopush = set_parser.add_mutually_exclusive_group(required=False)\n set_parser_autopush.add_argument(\n \"--autopush\",\n help=\"set mirror to push automatically after installation\",\n action=\"store_true\",\n default=None,\n dest=\"autopush\",\n )\n set_parser_autopush.add_argument(\n \"--no-autopush\",\n help=\"set mirror to not push automatically after installation\",\n action=\"store_false\",\n default=None,\n dest=\"autopush\",\n )\n set_parser_unsigned = set_parser.add_mutually_exclusive_group(required=False)\n set_parser_unsigned.add_argument(\n \"--unsigned\",\n help=\"do not require signing and signature verification when pushing and installing from \"\n \"this build cache\",\n action=\"store_false\",\n default=None,\n dest=\"signed\",\n )\n set_parser_unsigned.add_argument(\n \"--signed\",\n help=\"require signing and signature verification when pushing and installing from this \"\n \"build cache\",\n action=\"store_true\",\n default=None,\n dest=\"signed\",\n )\n set_parser.add_argument(\n \"--scope\",\n action=arguments.ConfigScope,\n default=lambda: spack.config.default_modify_scope(),\n help=\"configuration scope to modify\",\n )\n arguments.add_connection_args(set_parser, False)\n\n # List\n list_parser = sp.add_parser(\"list\", aliases=[\"ls\"], help=mirror_list.__doc__)\n list_parser.add_argument(\n \"--scope\", action=arguments.ConfigScope, help=\"configuration scope to read from\"\n )\n\n\ndef _configure_access_pair(args, id_tok, id_variable_tok, secret_variable_tok, default=None):\n \"\"\"Configure the access_pair options\"\"\"\n\n # Check if any of the arguments are set to update this access_pair.\n # If none are set, then skip computing the new access pair\n args_id = getattr(args, id_tok)\n args_id_variable = getattr(args, id_variable_tok)\n args_secret_variable = getattr(args, secret_variable_tok)\n if not any([args_id, args_id_variable, args_secret_variable]):\n return None\n\n def _default_value(id_):\n if isinstance(default, list):\n return default[0] if id_ == \"id\" else default[1]\n elif isinstance(default, dict):\n return default.get(id_)\n else:\n return None\n\n def _default_variable(id_):\n if isinstance(default, dict):\n return default.get(id_ + \"_variable\")\n else:\n return None\n\n id_ = None\n id_variable = None\n secret_variable = None\n\n # Get the value/default value if the argument of the inverse\n if not args_id_variable:\n id_ = getattr(args, id_tok) or _default_value(\"id\")\n if not args_id:\n id_variable = getattr(args, id_variable_tok) or _default_variable(\"id\")\n secret_variable = getattr(args, secret_variable_tok) or _default_variable(\"secret\")\n\n if (id_ or id_variable) and secret_variable:\n return dict(\n [\n ((\"id\", id_) if id_ else (\"id_variable\", id_variable)),\n (\"secret_variable\", secret_variable),\n ]\n )\n else:\n if id_ or id_variable or secret_variable is not None:\n id_arg_tok = id_tok.replace(\"_\", \"-\")\n secret_variable_arg_tok = secret_variable_tok.replace(\"_\", \"-\")\n tty.warn(\n \"Expected both parts of the access pair to be specified. \"\n f\"(i.e. --{id_arg_tok} and --{secret_variable_arg_tok})\"\n )\n\n return None\n\n\ndef mirror_add(args):\n \"\"\"add a mirror to Spack\"\"\"\n if (\n args.s3_access_key_id\n or args.s3_access_key_id_variable\n or args.s3_access_key_secret_variable\n or args.s3_access_token_variable\n or args.s3_profile\n or args.s3_endpoint_url\n or args.view_name\n or args.type\n or args.oci_username\n or args.oci_username_variable\n or args.oci_password_variable\n or args.autopush\n or args.signed is not None\n ):\n connection = {\"url\": args.url}\n # S3 Connection\n access_pair = _configure_access_pair(\n args, \"s3_access_key_id\", \"s3_access_key_id_variable\", \"s3_access_key_secret_variable\"\n )\n if access_pair:\n connection[\"access_pair\"] = access_pair\n\n if args.s3_access_token_variable:\n connection[\"access_token_variable\"] = args.s3_access_token_variable\n\n if args.s3_profile:\n connection[\"profile\"] = args.s3_profile\n\n if args.s3_endpoint_url:\n connection[\"endpoint_url\"] = args.s3_endpoint_url\n\n # OCI Connection\n access_pair = _configure_access_pair(\n args, \"oci_username\", \"oci_username_variable\", \"oci_password_variable\"\n )\n if access_pair:\n connection[\"access_pair\"] = access_pair\n\n if args.type:\n connection[\"binary\"] = \"binary\" in args.type\n connection[\"source\"] = \"source\" in args.type\n if args.autopush:\n connection[\"autopush\"] = args.autopush\n if args.signed is not None:\n connection[\"signed\"] = args.signed\n if args.view_name:\n connection[\"view\"] = args.view_name\n\n mirror = spack.mirrors.mirror.Mirror(connection, name=args.name)\n else:\n mirror = spack.mirrors.mirror.Mirror(args.url, name=args.name)\n spack.mirrors.utils.add(mirror, args.scope)\n\n\ndef mirror_remove(args):\n \"\"\"remove a mirror by name\"\"\"\n name = args.name\n scopes = [args.scope] if args.scope else reversed(list(spack.config.CONFIG.scopes.keys()))\n\n removed = False\n for scope in scopes:\n removed_from_this_scope = spack.mirrors.utils.remove(name, scope)\n if removed_from_this_scope:\n tty.msg(f\"Removed mirror {name} from {scope} scope\")\n\n removed |= removed_from_this_scope\n if removed and not args.all_scopes:\n return\n\n if not removed:\n tty.die(f\"No mirror with name {name} in {comma_or(scopes)} scope\")\n\n\ndef _configure_mirror(args):\n mirrors = spack.config.get(\"mirrors\", scope=args.scope)\n\n if args.name not in mirrors:\n tty.die(f\"No mirror found with name {args.name}.\")\n\n entry = spack.mirrors.mirror.Mirror(mirrors[args.name], args.name)\n direction = \"fetch\" if args.fetch else \"push\" if args.push else None\n changes = {}\n if args.url:\n changes[\"url\"] = args.url\n\n default_access_pair = entry._get_value(\"access_pair\", direction or \"fetch\")\n # TODO: Init access_pair args with the fetch/push/base values in the current mirror state\n access_pair = _configure_access_pair(\n args,\n \"s3_access_key_id\",\n \"s3_access_key_id_variable\",\n \"s3_access_key_secret_variable\",\n default=default_access_pair,\n )\n if access_pair:\n changes[\"access_pair\"] = access_pair\n if getattr(args, \"s3_access_token_variable\", None):\n changes[\"access_token_variable\"] = args.s3_access_token_variable\n if args.s3_profile:\n changes[\"profile\"] = args.s3_profile\n if args.s3_endpoint_url:\n changes[\"endpoint_url\"] = args.s3_endpoint_url\n access_pair = _configure_access_pair(\n args,\n \"oci_username\",\n \"oci_username_variable\",\n \"oci_password_variable\",\n default=default_access_pair,\n )\n if access_pair:\n changes[\"access_pair\"] = access_pair\n if getattr(args, \"signed\", None) is not None:\n changes[\"signed\"] = args.signed\n if getattr(args, \"autopush\", None) is not None:\n changes[\"autopush\"] = args.autopush\n\n # argparse cannot distinguish between --binary and --no-binary when same dest :(\n # notice that set-url does not have these args, so getattr\n if getattr(args, \"type\", None):\n changes[\"binary\"] = \"binary\" in args.type\n changes[\"source\"] = \"source\" in args.type\n\n changed = entry.update(changes, direction)\n\n if changed:\n mirrors[args.name] = entry.to_dict()\n spack.config.set(\"mirrors\", mirrors, scope=args.scope)\n else:\n tty.msg(\"No changes made to mirror %s.\" % args.name)\n\n\ndef mirror_set(args):\n \"\"\"configure the connection details of a mirror\"\"\"\n _configure_mirror(args)\n\n\ndef mirror_set_url(args):\n \"\"\"change the URL of a mirror\"\"\"\n _configure_mirror(args)\n\n\ndef mirror_list(args):\n \"\"\"print out available mirrors to the console\"\"\"\n\n mirrors = spack.mirrors.mirror.MirrorCollection(scope=args.scope)\n if not mirrors:\n tty.msg(\"No mirrors configured.\")\n return\n\n mirrors.display()\n\n\ndef specs_from_text_file(filename, concretize=False):\n \"\"\"Return a list of specs read from a text file.\n\n The file should contain one spec per line.\n\n Args:\n filename (str): name of the file containing the abstract specs.\n concretize (bool): if True concretize the specs before returning\n the list.\n \"\"\"\n with open(filename, \"r\", encoding=\"utf-8\") as f:\n specs_in_file = f.readlines()\n specs_in_file = [s.strip() for s in specs_in_file]\n return spack.cmd.parse_specs(\" \".join(specs_in_file), concretize=concretize)\n\n\ndef concrete_specs_from_user(args):\n \"\"\"Return the list of concrete specs that the user wants to mirror. The list\n is passed either from command line or from a text file.\n \"\"\"\n specs = concrete_specs_from_cli_or_file(args)\n specs = extend_with_additional_versions(specs, num_versions=versions_per_spec(args))\n if args.dependencies:\n specs = extend_with_dependencies(specs)\n specs = filter_externals(specs)\n specs = list(set(specs))\n specs.sort(key=lambda s: (s.name, s.version))\n return specs\n\n\ndef extend_with_additional_versions(specs, num_versions):\n if num_versions == \"all\":\n mirror_specs = spack.mirrors.utils.get_all_versions(specs)\n else:\n mirror_specs = spack.mirrors.utils.get_matching_versions(specs, num_versions=num_versions)\n mirror_specs = [spack.concretize.concretize_one(x) for x in mirror_specs]\n return mirror_specs\n\n\ndef filter_externals(specs):\n specs, external_specs = lang.stable_partition(specs, predicate_fn=lambda x: not x.external)\n for spec in external_specs:\n msg = \"Skipping {0} as it is an external spec.\"\n tty.msg(msg.format(spec.cshort_spec))\n return specs\n\n\ndef extend_with_dependencies(specs):\n \"\"\"Extend the input list by adding all the dependencies explicitly.\"\"\"\n result = set()\n for spec in specs:\n for s in spec.traverse():\n result.add(s)\n return list(result)\n\n\ndef concrete_specs_from_cli_or_file(args):\n if args.specs:\n specs = spack.cmd.parse_specs(args.specs, concretize=False)\n if not specs:\n raise SpackError(\"unable to parse specs from command line\")\n\n if args.file:\n specs = specs_from_text_file(args.file, concretize=False)\n if not specs:\n raise SpackError(\"unable to parse specs from file '{}'\".format(args.file))\n\n concrete_specs = spack.cmd.matching_specs_from_env(specs)\n return concrete_specs\n\n\nclass IncludeFilter:\n def __init__(self, args):\n self.exclude_specs = []\n if args.exclude_file:\n self.exclude_specs.extend(specs_from_text_file(args.exclude_file, concretize=False))\n if args.exclude_specs:\n self.exclude_specs.extend(spack.cmd.parse_specs(str(args.exclude_specs).split()))\n self.private = args.private\n\n def __call__(self, x):\n return all([self._not_license_excluded(x), self._not_cmdline_excluded(x)])\n\n def _not_license_excluded(self, x):\n \"\"\"True if the spec is for a private mirror, or as long as the\n package does not explicitly forbid redistributing source.\"\"\"\n if self.private:\n return True\n elif spack.repo.PATH.get_pkg_class(x.fullname).redistribute_source(x):\n return True\n else:\n tty.debug(\n \"Skip adding {0} to mirror: the package.py file\"\n \" indicates that a public mirror should not contain\"\n \" it.\".format(x.name)\n )\n return False\n\n def _not_cmdline_excluded(self, x):\n \"\"\"True if a spec was not explicitly excluded by the user.\"\"\"\n return not any(x.satisfies(y) for y in self.exclude_specs)\n\n\ndef concrete_specs_from_environment():\n env = ev.active_environment()\n assert env, \"an active environment is required\"\n mirror_specs = env.all_specs()\n mirror_specs = filter_externals(mirror_specs)\n return mirror_specs\n\n\ndef all_specs_with_all_versions():\n specs = [spack.spec.Spec(n) for n in spack.repo.all_package_names()]\n mirror_specs = spack.mirrors.utils.get_all_versions(specs)\n mirror_specs.sort(key=lambda s: (s.name, s.version))\n return mirror_specs\n\n\ndef versions_per_spec(args):\n \"\"\"Return how many versions should be mirrored per spec.\"\"\"\n if not args.versions_per_spec:\n num_versions = 1\n elif args.versions_per_spec == \"all\":\n num_versions = \"all\"\n else:\n try:\n num_versions = int(args.versions_per_spec)\n except ValueError:\n raise SpackError(\n \"'--versions-per-spec' must be a number or 'all', got '{0}'\".format(\n args.versions_per_spec\n )\n )\n return num_versions\n\n\ndef process_mirror_stats(present, mirrored, error):\n p, m, e = len(present), len(mirrored), len(error)\n tty.msg(\n \"Archive stats:\",\n \" %-4d already present\" % p,\n \" %-4d added\" % m,\n \" %-4d failed to fetch.\" % e,\n )\n if error:\n tty.error(\"Failed downloads:\")\n colify.colify(s.cformat(\"{name}{@version}\") for s in error)\n sys.exit(1)\n\n\ndef mirror_create(args):\n \"\"\"create a directory to be used as a spack mirror, and fill it with package archives\"\"\"\n if args.file and args.all:\n raise SpackError(\n \"cannot specify specs with a file if you chose to mirror all specs with '--all'\"\n )\n\n if args.file and args.specs:\n raise SpackError(\"cannot specify specs with a file AND on command line\")\n\n if not args.specs and not args.file and not args.all:\n raise SpackError(\n \"no packages were specified.\",\n \"To mirror all packages, use the '--all' option \"\n \"(this will require significant time and space).\",\n )\n\n if args.versions_per_spec and args.all:\n raise SpackError(\n \"cannot specify '--versions_per-spec' and '--all' together\",\n \"The option '--all' already implies mirroring all versions for each package.\",\n )\n\n # When no directory is provided, the source dir is used\n path = args.directory or spack.caches.fetch_cache_location()\n\n mirror_specs = _specs_to_mirror(args)\n workers = args.jobs\n if workers is None:\n if args.all:\n workers = min(\n 16, spack.config.determine_number_of_jobs(parallel=True), len(mirror_specs)\n )\n else:\n workers = 1\n\n create_mirror_for_all_specs(\n mirror_specs,\n path=path,\n skip_unstable_versions=args.skip_unstable_versions,\n workers=workers,\n )\n\n\ndef _specs_to_mirror(args):\n include_fn = IncludeFilter(args)\n\n if args.all and not ev.active_environment():\n mirror_specs = all_specs_with_all_versions()\n elif args.all and ev.active_environment():\n mirror_specs = concrete_specs_from_environment()\n else:\n mirror_specs = concrete_specs_from_user(args)\n\n mirror_specs, _ = lang.stable_partition(mirror_specs, predicate_fn=include_fn)\n return mirror_specs\n\n\ndef create_mirror_for_one_spec(candidate, mirror_cache):\n pkg_cls = spack.repo.PATH.get_pkg_class(candidate.name)\n pkg_obj = pkg_cls(spack.spec.Spec(candidate))\n mirror_stats = spack.mirrors.utils.MirrorStatsForOneSpec(candidate)\n spack.mirrors.utils.create_mirror_from_package_object(pkg_obj, mirror_cache, mirror_stats)\n mirror_stats.finalize()\n return mirror_stats\n\n\ndef create_mirror_for_all_specs(mirror_specs, path, skip_unstable_versions, workers):\n mirror_cache = spack.mirrors.utils.get_mirror_cache(\n path, skip_unstable_versions=skip_unstable_versions\n )\n mirror_stats = spack.mirrors.utils.MirrorStatsForAllSpecs()\n with spack.util.parallel.make_concurrent_executor(jobs=workers) as executor:\n # Submit tasks to the process pool\n futures = [\n executor.submit(create_mirror_for_one_spec, candidate, mirror_cache)\n for candidate in mirror_specs\n ]\n for mirror_future in as_completed(futures):\n ext_mirror_stats = mirror_future.result()\n mirror_stats.merge(ext_mirror_stats)\n\n process_mirror_stats(*mirror_stats.stats())\n return mirror_stats\n\n\ndef create(path, specs, skip_unstable_versions=False):\n \"\"\"Create a directory to be used as a spack mirror, and fill it with\n package archives.\n\n Arguments:\n path: Path to create a mirror directory hierarchy in.\n specs: Any package versions matching these specs will be added \\\n to the mirror.\n skip_unstable_versions: if true, this skips adding resources when\n they do not have a stable archive checksum (as determined by\n ``fetch_strategy.stable_target``)\n\n Returns:\n A tuple of lists, each containing specs\n\n * present: Package specs that were already present.\n * mirrored: Package specs that were successfully mirrored.\n * error: Package specs that failed to mirror due to some error.\n \"\"\"\n # automatically spec-ify anything in the specs array.\n specs = [s if isinstance(s, spack.spec.Spec) else spack.spec.Spec(s) for s in specs]\n mirror_stats = create_mirror_for_all_specs(specs, path, skip_unstable_versions, workers=1)\n return mirror_stats.stats()\n\n\ndef mirror_destroy(args):\n \"\"\"given a url, recursively delete everything under it\"\"\"\n mirror_url = None\n\n if args.mirror_name:\n result = spack.mirrors.mirror.MirrorCollection().lookup(args.mirror_name)\n mirror_url = result.push_url\n elif args.mirror_url:\n mirror_url = args.mirror_url\n\n web_util.remove_url(mirror_url, recursive=True)\n\n\ndef mirror(parser, args):\n action = {\n \"create\": mirror_create,\n \"destroy\": mirror_destroy,\n \"add\": mirror_add,\n \"remove\": mirror_remove,\n \"rm\": mirror_remove,\n \"set-url\": mirror_set_url,\n \"set\": mirror_set,\n \"list\": mirror_list,\n \"ls\": mirror_list,\n }\n\n if args.no_checksum:\n spack.config.set(\"config:checksum\", False, scope=\"command_line\")\n\n action[args.mirror_command](args)\n" }, { "path": "lib/spack/spack/cmd/module.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nfrom typing import Callable, Dict\n\nimport spack.cmd.modules.lmod\nimport spack.cmd.modules.tcl\n\ndescription = \"generate/manage module files\"\nsection = \"user environment\"\nlevel = \"short\"\n\n\n_subcommands: Dict[str, Callable] = {}\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n sp = subparser.add_subparsers(metavar=\"SUBCOMMAND\", dest=\"module_command\")\n spack.cmd.modules.lmod.add_command(sp, _subcommands)\n spack.cmd.modules.tcl.add_command(sp, _subcommands)\n\n\ndef module(parser, args):\n _subcommands[args.module_command](parser, args)\n" }, { "path": "lib/spack/spack/cmd/modules/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Implementation details of the ``spack module`` command.\"\"\"\n\nimport collections\nimport os\nimport shutil\nimport sys\n\nimport spack.cmd\nimport spack.config\nimport spack.error\nimport spack.modules\nimport spack.modules.common\nimport spack.repo\nfrom spack.cmd import MultipleSpecsMatch, NoSpecMatches\nfrom spack.cmd.common import arguments\nfrom spack.llnl.util import filesystem, tty\nfrom spack.llnl.util.tty import color\n\ndescription = \"manipulate module files\"\nsection = \"environment\"\nlevel = \"short\"\n\n\ndef setup_parser(subparser):\n subparser.add_argument(\n \"-n\",\n \"--name\",\n action=\"store\",\n dest=\"module_set_name\",\n default=\"default\",\n help=\"named module set to use from modules configuration\",\n )\n sp = subparser.add_subparsers(metavar=\"SUBCOMMAND\", dest=\"subparser_name\")\n\n refresh_parser = sp.add_parser(\"refresh\", help=\"regenerate module files\")\n refresh_parser.add_argument(\n \"--delete-tree\", help=\"delete the module file tree before refresh\", action=\"store_true\"\n )\n refresh_parser.add_argument(\n \"--upstream-modules\",\n help=\"generate modules for packages installed upstream\",\n action=\"store_true\",\n )\n arguments.add_common_arguments(refresh_parser, [\"constraint\", \"yes_to_all\"])\n\n find_parser = sp.add_parser(\"find\", help=\"find module files for packages\")\n find_parser.add_argument(\n \"--full-path\", help=\"display full path to module file\", action=\"store_true\"\n )\n arguments.add_common_arguments(find_parser, [\"constraint\", \"recurse_dependencies\"])\n\n rm_parser = sp.add_parser(\"rm\", help=\"remove module files\")\n arguments.add_common_arguments(rm_parser, [\"constraint\", \"yes_to_all\"])\n\n loads_parser = sp.add_parser(\n \"loads\", help=\"prompt the list of modules associated with a constraint\"\n )\n add_loads_arguments(loads_parser)\n arguments.add_common_arguments(loads_parser, [\"constraint\"])\n\n return sp\n\n\ndef add_loads_arguments(subparser):\n subparser.add_argument(\n \"--input-only\",\n action=\"store_false\",\n dest=\"shell\",\n help=\"generate input for module command (instead of a shell script)\",\n )\n subparser.add_argument(\n \"-p\",\n \"--prefix\",\n dest=\"prefix\",\n default=\"\",\n help=\"prepend to module names when issuing module load commands\",\n )\n subparser.add_argument(\n \"-x\",\n \"--exclude\",\n dest=\"exclude\",\n action=\"append\",\n default=[],\n help=\"exclude package from output; may be specified multiple times\",\n )\n arguments.add_common_arguments(subparser, [\"recurse_dependencies\"])\n\n\ndef one_spec_or_raise(specs):\n \"\"\"Ensures exactly one spec has been selected, or raises the appropriate\n exception.\n \"\"\"\n # Ensure a single spec matches the constraint\n if len(specs) == 0:\n raise NoSpecMatches()\n if len(specs) > 1:\n raise MultipleSpecsMatch()\n\n # Get the spec and module type\n return specs[0]\n\n\ndef check_module_set_name(name):\n modules = spack.config.get(\"modules\")\n if name != \"prefix_inspections\" and name in modules:\n return\n\n names = [k for k in modules if k != \"prefix_inspections\"]\n\n if not names:\n raise spack.error.ConfigError(\n f\"Module set configuration is missing. Cannot use module set '{name}'\"\n )\n\n pretty_names = \"', '\".join(names)\n\n raise spack.error.ConfigError(\n f\"Cannot use invalid module set '{name}'.\",\n f\"Valid module set names are: '{pretty_names}'.\",\n )\n\n\n_missing_modules_warning = (\n \"Modules have been omitted for one or more specs, either\"\n \" because they were excluded or because the spec is\"\n \" associated with a package that is installed upstream and\"\n \" that installation has not generated a module file. Rerun\"\n \" this command with debug output enabled for more details.\"\n)\n\n\ndef loads(module_type, specs, args, out=None):\n \"\"\"Prompt the list of modules associated with a list of specs\"\"\"\n check_module_set_name(args.module_set_name)\n out = sys.stdout if out is None else out\n\n # Get a comprehensive list of specs\n if args.recurse_dependencies:\n specs_from_user_constraint = specs[:]\n specs = []\n # FIXME : during module file creation nodes seem to be visited\n # FIXME : multiple times even if cover='nodes' is given. This\n # FIXME : work around permits to get a unique list of spec anyhow.\n # FIXME : (same problem as in spack/modules.py)\n seen = set()\n seen_add = seen.add\n for spec in specs_from_user_constraint:\n specs.extend(\n [\n item\n for item in spec.traverse(order=\"post\", cover=\"nodes\")\n if not (item in seen or seen_add(item))\n ]\n )\n\n modules = list(\n (\n spec,\n spack.modules.get_module(\n module_type,\n spec,\n get_full_path=False,\n module_set_name=args.module_set_name,\n required=False,\n ),\n )\n for spec in specs\n )\n\n module_commands = {\"tcl\": \"module load \", \"lmod\": \"module load \"}\n\n d = {\"command\": \"\" if not args.shell else module_commands[module_type], \"prefix\": args.prefix}\n\n exclude_set = set(args.exclude)\n load_template = \"{comment}{exclude}{command}{prefix}{name}\"\n for spec, mod in modules:\n if not mod:\n module_output_for_spec = \"## excluded or missing from upstream: {0}\".format(\n spec.format()\n )\n else:\n d[\"exclude\"] = \"## \" if spec.name in exclude_set else \"\"\n d[\"comment\"] = \"\" if not args.shell else \"# {0}\\n\".format(spec.format())\n d[\"name\"] = mod\n module_output_for_spec = load_template.format(**d)\n out.write(module_output_for_spec)\n out.write(\"\\n\")\n\n if not all(mod for _, mod in modules):\n tty.warn(_missing_modules_warning)\n\n\ndef find(module_type, specs, args):\n \"\"\"Retrieve paths or use names of module files\"\"\"\n check_module_set_name(args.module_set_name)\n\n single_spec = one_spec_or_raise(specs)\n\n if args.recurse_dependencies:\n dependency_specs_to_retrieve = list(\n single_spec.traverse(root=False, order=\"post\", cover=\"nodes\", deptype=(\"link\", \"run\"))\n )\n else:\n dependency_specs_to_retrieve = []\n\n try:\n modules = [\n spack.modules.get_module(\n module_type,\n spec,\n args.full_path,\n module_set_name=args.module_set_name,\n required=False,\n )\n for spec in dependency_specs_to_retrieve\n ]\n\n modules.append(\n spack.modules.get_module(\n module_type,\n single_spec,\n args.full_path,\n module_set_name=args.module_set_name,\n required=True,\n )\n )\n except spack.modules.common.ModuleNotFoundError as e:\n tty.die(e.message)\n\n if not all(modules):\n tty.warn(_missing_modules_warning)\n modules = list(x for x in modules if x)\n print(\" \".join(modules))\n\n\ndef rm(module_type, specs, args):\n \"\"\"Deletes the module files associated with every spec in specs, for every\n module type in module types.\n \"\"\"\n check_module_set_name(args.module_set_name)\n\n module_cls = spack.modules.module_types[module_type]\n module_exist = lambda x: os.path.exists(module_cls(x, args.module_set_name).layout.filename)\n\n specs_with_modules = [spec for spec in specs if module_exist(spec)]\n\n modules = [module_cls(spec, args.module_set_name) for spec in specs_with_modules]\n\n if not modules:\n tty.die(\"No module file matches your query\")\n\n # Ask for confirmation\n if not args.yes_to_all:\n msg = \"You are about to remove {0} module files for:\\n\"\n tty.msg(msg.format(module_type))\n spack.cmd.display_specs(specs_with_modules, long=True)\n print(\"\")\n answer = tty.get_yes_or_no(\"Do you want to proceed?\")\n if not answer:\n tty.die(\"Will not remove any module files\")\n\n # Remove the module files\n for s in modules:\n s.remove()\n\n\ndef refresh(module_type, specs, args):\n \"\"\"Regenerates the module files for every spec in specs and every module\n type in module types.\n \"\"\"\n check_module_set_name(args.module_set_name)\n\n # Prompt a message to the user about what is going to change\n if not specs:\n tty.msg(\"No package matches your query\")\n return\n\n if not args.upstream_modules:\n specs = list(s for s in specs if not s.installed_upstream)\n\n if not args.yes_to_all:\n msg = \"You are about to regenerate {types} module files for:\\n\"\n tty.msg(msg.format(types=module_type))\n spack.cmd.display_specs(specs, long=True)\n print(\"\")\n answer = tty.get_yes_or_no(\"Do you want to proceed?\")\n if not answer:\n tty.die(\"Module file regeneration aborted.\")\n\n # Cycle over the module types and regenerate module files\n\n cls = spack.modules.module_types[module_type]\n\n # Skip unknown packages.\n writers = [\n cls(spec, args.module_set_name) for spec in specs if spack.repo.PATH.exists(spec.name)\n ]\n\n # Filter excluded packages early\n writers = [x for x in writers if not x.conf.excluded]\n\n # Detect name clashes in module files\n file2writer = collections.defaultdict(list)\n for item in writers:\n file2writer[item.layout.filename].append(item)\n\n if len(file2writer) != len(writers):\n spec_fmt_str = \"{name}@={version}%{compiler}/{hash:7} {variants} arch={arch}\"\n message = \"Name clashes detected in module files:\\n\"\n for filename, writer_list in file2writer.items():\n if len(writer_list) > 1:\n message += \"\\nfile: {0}\\n\".format(filename)\n for x in writer_list:\n message += \"spec: {0}\\n\".format(x.spec.format(spec_fmt_str))\n tty.error(message)\n tty.error(\"Operation aborted\")\n raise SystemExit(1)\n\n if len(writers) == 0:\n msg = \"Nothing to be done for {0} module files.\"\n tty.msg(msg.format(module_type))\n return\n # If we arrived here we have at least one writer\n module_type_root = writers[0].layout.dirname()\n\n # Proceed regenerating module files\n tty.msg(\"Regenerating {name} module files\".format(name=module_type))\n if os.path.isdir(module_type_root) and args.delete_tree:\n shutil.rmtree(module_type_root, ignore_errors=False)\n filesystem.mkdirp(module_type_root)\n\n # Dump module index after potentially removing module tree\n spack.modules.common.generate_module_index(\n module_type_root, writers, overwrite=args.delete_tree\n )\n errors = []\n for x in writers:\n try:\n x.write(overwrite=True)\n except spack.error.SpackError as e:\n msg = f\"{x.layout.filename}: {e.message}\"\n errors.append(msg)\n except Exception as e:\n msg = f\"{x.layout.filename}: {str(e)}\"\n errors.append(msg)\n\n if errors:\n errors.insert(0, color.colorize(\"@*{some module files could not be written}\"))\n tty.warn(\"\\n\".join(errors))\n\n\n#: Dictionary populated with the list of sub-commands.\n#: Each sub-command must be callable and accept 3 arguments:\n#:\n#: - module_type: the type of module it refers to\n#: - specs : the list of specs to be processed\n#: - args : namespace containing the parsed command line arguments\ncallbacks = {\"refresh\": refresh, \"rm\": rm, \"find\": find, \"loads\": loads}\n\n\ndef modules_cmd(parser, args, module_type, callbacks=callbacks):\n # Qualifiers to be used when querying the db for specs\n constraint_qualifiers = {\n \"refresh\": {\n \"installed\": True,\n \"predicate_fn\": lambda x: spack.repo.PATH.exists(x.spec.name),\n }\n }\n query_args = constraint_qualifiers.get(args.subparser_name, {})\n\n # Get the specs that match the query from the DB\n specs = args.specs(**query_args)\n\n try:\n callbacks[args.subparser_name](module_type, specs, args)\n\n except MultipleSpecsMatch:\n query = \" \".join(str(s) for s in args.constraint_specs)\n msg = f\"the constraint '{query}' matches multiple packages:\\n\"\n for s in specs:\n spec_fmt = (\n \"{hash:7} {name}{@version}{compiler_flags}{variants}\"\n \"{ platform=architecture.platform}{ os=architecture.os}\"\n \"{ target=architecture.target}\"\n \"{%compiler}\"\n )\n msg += \"\\t\" + s.cformat(spec_fmt) + \"\\n\"\n tty.die(msg, \"In this context exactly *one* match is needed.\")\n\n except NoSpecMatches:\n query = \" \".join(str(s) for s in args.constraint_specs)\n msg = f\"the constraint '{query}' matches no package.\"\n tty.die(msg, \"In this context exactly *one* match is needed.\")\n" }, { "path": "lib/spack/spack/cmd/modules/lmod.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport functools\n\nimport spack.cmd.common.arguments\nimport spack.cmd.modules\nimport spack.config\nimport spack.modules\nimport spack.modules.lmod\n\n\ndef add_command(parser, command_dict):\n lmod_parser = parser.add_parser(\"lmod\", help=\"manipulate hierarchical module files\")\n sp = spack.cmd.modules.setup_parser(lmod_parser)\n\n # Set default module file for a package\n setdefault_parser = sp.add_parser(\n \"setdefault\", help=\"set the default module file for a package\"\n )\n spack.cmd.common.arguments.add_common_arguments(setdefault_parser, [\"constraint\"])\n\n callbacks = dict(spack.cmd.modules.callbacks.items())\n callbacks[\"setdefault\"] = setdefault\n\n command_dict[\"lmod\"] = functools.partial(\n spack.cmd.modules.modules_cmd, module_type=\"lmod\", callbacks=callbacks\n )\n\n\ndef setdefault(module_type, specs, args):\n \"\"\"set the default module file, when multiple are present\"\"\"\n # For details on the underlying mechanism see:\n #\n # https://lmod.readthedocs.io/en/latest/060_locating.html#marking-a-version-as-default\n #\n spack.cmd.modules.one_spec_or_raise(specs)\n spec = specs[0]\n data = {\"modules\": {args.module_set_name: {\"lmod\": {\"defaults\": [str(spec)]}}}}\n # Need to clear the cache if a SpackCommand is called during scripting\n spack.modules.lmod.configuration_registry = {}\n scope = spack.config.InternalConfigScope(\"lmod-setdefault\", data)\n with spack.config.override(scope):\n writer = spack.modules.module_types[\"lmod\"](spec, args.module_set_name)\n writer.update_module_defaults()\n" }, { "path": "lib/spack/spack/cmd/modules/tcl.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport functools\n\nimport spack.cmd.common.arguments\nimport spack.cmd.modules\nimport spack.config\nimport spack.modules\nimport spack.modules.tcl\n\n\ndef add_command(parser, command_dict):\n tcl_parser = parser.add_parser(\"tcl\", help=\"manipulate non-hierarchical module files\")\n sp = spack.cmd.modules.setup_parser(tcl_parser)\n\n # Set default module file for a package\n setdefault_parser = sp.add_parser(\n \"setdefault\", help=\"set the default module file for a package\"\n )\n spack.cmd.common.arguments.add_common_arguments(setdefault_parser, [\"constraint\"])\n\n callbacks = dict(spack.cmd.modules.callbacks.items())\n callbacks[\"setdefault\"] = setdefault\n\n command_dict[\"tcl\"] = functools.partial(\n spack.cmd.modules.modules_cmd, module_type=\"tcl\", callbacks=callbacks\n )\n\n\ndef setdefault(module_type, specs, args):\n \"\"\"set the default module file, when multiple are present\"\"\"\n # Currently, accepts only a single matching spec\n spack.cmd.modules.one_spec_or_raise(specs)\n spec = specs[0]\n data = {\"modules\": {args.module_set_name: {\"tcl\": {\"defaults\": [str(spec)]}}}}\n spack.modules.tcl.configuration_registry = {}\n scope = spack.config.InternalConfigScope(\"tcl-setdefault\", data)\n with spack.config.override(scope):\n writer = spack.modules.module_types[\"tcl\"](spec, args.module_set_name)\n writer.update_module_defaults()\n" }, { "path": "lib/spack/spack/cmd/patch.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\n\nimport spack.cmd\nimport spack.config\nimport spack.environment as ev\nimport spack.llnl.util.tty as tty\nimport spack.package_base\nimport spack.traverse\nfrom spack.cmd.common import arguments\n\ndescription = \"patch expanded sources in preparation for install\"\nsection = \"build\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n arguments.add_common_arguments(subparser, [\"no_checksum\", \"specs\"])\n arguments.add_concretizer_args(subparser)\n\n\ndef patch(parser, args):\n if not args.specs:\n env = ev.active_environment()\n if not env:\n tty.die(\"`spack patch` requires a spec or an active environment\")\n return _patch_env(env)\n\n if args.no_checksum:\n spack.config.set(\"config:checksum\", False, scope=\"command_line\")\n\n specs = spack.cmd.parse_specs(args.specs, concretize=False)\n specs = spack.cmd.matching_specs_from_env(specs)\n for spec in specs:\n _patch(spec.package)\n\n\ndef _patch_env(env: ev.Environment):\n tty.msg(f\"Patching specs from environment {env.name}\")\n for spec in spack.traverse.traverse_nodes(env.concrete_roots()):\n _patch(spec.package)\n\n\ndef _patch(pkg: spack.package_base.PackageBase):\n pkg.stage.keep = True\n with pkg.stage:\n pkg.do_patch()\n tty.msg(f\"Patched {pkg.name} in {pkg.stage.path}\")\n" }, { "path": "lib/spack/spack/cmd/pkg.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport os\nimport sys\n\nimport spack.cmd\nimport spack.llnl.util.tty as tty\nimport spack.repo\nimport spack.util.executable as exe\nimport spack.util.package_hash as ph\nfrom spack.cmd.common import arguments\nfrom spack.llnl.util.tty.colify import colify\n\ndescription = \"query packages associated with particular git revisions\"\nsection = \"developer\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n sp = subparser.add_subparsers(metavar=\"SUBCOMMAND\", dest=\"pkg_command\")\n\n add_parser = sp.add_parser(\"add\", help=pkg_add.__doc__)\n arguments.add_common_arguments(add_parser, [\"packages\"])\n\n list_parser = sp.add_parser(\"list\", help=pkg_list.__doc__)\n list_parser.add_argument(\n \"rev\", default=\"HEAD\", nargs=\"?\", help=\"revision to list packages for\"\n )\n\n diff_parser = sp.add_parser(\"diff\", help=pkg_diff.__doc__)\n diff_parser.add_argument(\n \"rev1\", nargs=\"?\", default=\"HEAD^\", help=\"revision to compare against\"\n )\n diff_parser.add_argument(\n \"rev2\", nargs=\"?\", default=\"HEAD\", help=\"revision to compare to rev1 (default is HEAD)\"\n )\n\n add_parser = sp.add_parser(\"added\", help=pkg_added.__doc__)\n add_parser.add_argument(\"rev1\", nargs=\"?\", default=\"HEAD^\", help=\"revision to compare against\")\n add_parser.add_argument(\n \"rev2\", nargs=\"?\", default=\"HEAD\", help=\"revision to compare to rev1 (default is HEAD)\"\n )\n\n add_parser = sp.add_parser(\"changed\", help=pkg_changed.__doc__)\n add_parser.add_argument(\"rev1\", nargs=\"?\", default=\"HEAD^\", help=\"revision to compare against\")\n add_parser.add_argument(\n \"rev2\", nargs=\"?\", default=\"HEAD\", help=\"revision to compare to rev1 (default is HEAD)\"\n )\n add_parser.add_argument(\n \"-t\",\n \"--type\",\n action=\"store\",\n default=\"C\",\n help=\"types of changes to show (A: added, R: removed, C: changed); default is 'C'\",\n )\n\n rm_parser = sp.add_parser(\"removed\", help=pkg_removed.__doc__)\n rm_parser.add_argument(\"rev1\", nargs=\"?\", default=\"HEAD^\", help=\"revision to compare against\")\n rm_parser.add_argument(\n \"rev2\", nargs=\"?\", default=\"HEAD\", help=\"revision to compare to rev1 (default is HEAD)\"\n )\n\n # explicitly add help for `spack pkg grep` with just `--help` and NOT `-h`. This is so\n # that the very commonly used -h (no filename) argument can be passed through to grep\n grep_parser = sp.add_parser(\"grep\", help=pkg_grep.__doc__, add_help=False)\n grep_parser.add_argument(\n \"grep_args\", nargs=argparse.REMAINDER, default=None, help=\"arguments for grep\"\n )\n grep_parser.add_argument(\"--help\", action=\"help\", help=\"show this help message and exit\")\n\n source_parser = sp.add_parser(\"source\", help=pkg_source.__doc__)\n source_parser.add_argument(\n \"-c\",\n \"--canonical\",\n action=\"store_true\",\n default=False,\n help=\"dump canonical source as used by package hash\",\n )\n arguments.add_common_arguments(source_parser, [\"spec\"])\n\n hash_parser = sp.add_parser(\"hash\", help=pkg_hash.__doc__)\n arguments.add_common_arguments(hash_parser, [\"spec\"])\n\n\ndef pkg_add(args):\n \"\"\"add a package to the git stage with ``git add``\"\"\"\n spack.repo.add_package_to_git_stage(args.packages, spack.repo.builtin_repo())\n\n\ndef pkg_list(args):\n \"\"\"list packages associated with a particular spack git revision\"\"\"\n colify(spack.repo.list_packages(args.rev, spack.repo.builtin_repo()))\n\n\ndef pkg_diff(args):\n \"\"\"compare packages available in two different git revisions\"\"\"\n u1, u2 = spack.repo.diff_packages(args.rev1, args.rev2, spack.repo.builtin_repo())\n\n if u1:\n print(\"%s:\" % args.rev1)\n colify(sorted(u1), indent=4)\n if u1:\n print()\n\n if u2:\n print(\"%s:\" % args.rev2)\n colify(sorted(u2), indent=4)\n\n\ndef pkg_removed(args):\n \"\"\"show packages removed since a commit\"\"\"\n u1, u2 = spack.repo.diff_packages(args.rev1, args.rev2, spack.repo.builtin_repo())\n if u1:\n colify(sorted(u1))\n\n\ndef pkg_added(args):\n \"\"\"show packages added since a commit\"\"\"\n u1, u2 = spack.repo.diff_packages(args.rev1, args.rev2, spack.repo.builtin_repo())\n if u2:\n colify(sorted(u2))\n\n\ndef pkg_changed(args):\n \"\"\"show packages changed since a commit\"\"\"\n packages = spack.repo.get_all_package_diffs(\n args.type, spack.repo.builtin_repo(), args.rev1, args.rev2\n )\n\n if packages:\n colify(sorted(packages))\n\n\ndef pkg_source(args):\n \"\"\"dump source code for a package\"\"\"\n specs = spack.cmd.parse_specs(args.spec, concretize=False)\n if len(specs) != 1:\n tty.die(\"spack pkg source requires exactly one spec\")\n\n spec = specs[0]\n filename = spack.repo.PATH.filename_for_package_name(spec.name)\n\n # regular source dump -- just get the package and print its contents\n if args.canonical:\n message = \"Canonical source for %s:\" % filename\n content = ph.canonical_source(spec)\n else:\n message = \"Source for %s:\" % filename\n with open(filename, encoding=\"utf-8\") as f:\n content = f.read()\n\n if sys.stdout.isatty():\n tty.msg(message)\n sys.stdout.write(content)\n\n\ndef pkg_hash(args):\n \"\"\"dump canonical source code hash for a package spec\"\"\"\n specs = spack.cmd.parse_specs(args.spec, concretize=False)\n\n for spec in specs:\n print(ph.package_hash(spec))\n\n\ndef get_grep(required=False):\n \"\"\"Get a grep command to use with ``spack pkg grep``.\"\"\"\n grep = exe.which(os.environ.get(\"SPACK_GREP\") or \"grep\", required=required)\n if grep:\n grep.ignore_quotes = True # allow `spack pkg grep '\"quoted string\"'` without warning\n return grep\n\n\ndef pkg_grep(args, unknown_args):\n \"\"\"grep for strings in package.py files from all repositories\"\"\"\n grep = get_grep(required=True)\n\n # add a little color to the output if we can\n if \"GNU\" in grep(\"--version\", output=str):\n grep.add_default_arg(\"--color=auto\")\n\n all_paths = spack.repo.PATH.all_package_paths()\n if not all_paths:\n return 0 # no packages to search\n\n # these args start every command invocation (grep arg1 arg2 ...)\n all_prefix_args = grep.exe + args.grep_args + unknown_args\n prefix_length = sum(spack.cmd.converted_arg_length(arg) for arg in all_prefix_args) + len(\n all_prefix_args\n )\n\n # set up iterator and save the first group to ensure we don't end up with a group of size 1\n groups = spack.cmd.group_arguments(all_paths, prefix_length=prefix_length)\n\n # You can force GNU grep to show filenames on every line with -H, but not POSIX grep.\n # POSIX grep only shows filenames when you're grepping 2 or more files. Since we\n # don't know which one we're running, we ensure there are always >= 2 files by\n # saving the prior group of paths and adding it to a straggling group of 1 if needed.\n # This works unless somehow there is only one package in all of Spack.\n prior_paths = next(groups)\n\n # grep returns 1 for nothing found, 0 for something found, and > 1 for error\n return_code = 1\n\n # assemble args and run grep on a group of paths\n def grep_group(paths):\n all_args = args.grep_args + unknown_args + paths\n grep(*all_args, fail_on_error=False)\n return grep.returncode\n\n for paths in groups:\n if len(paths) == 1:\n # Only the very last group can have length 1. If it does, combine\n # it with the prior group to ensure more than one path is grepped.\n prior_paths += paths\n else:\n # otherwise run grep on the prior group\n error = grep_group(prior_paths)\n if error != 1:\n return_code = error\n if error > 1: # fail fast on error\n return error\n\n prior_paths = paths\n\n # Handle the last remaining group after the loop\n error = grep_group(prior_paths)\n if error != 1:\n return_code = error\n\n return return_code\n\n\ndef pkg(parser, args, unknown_args):\n if not spack.cmd.spack_is_git_repo():\n tty.die(\"This spack is not a git clone. Can't use 'spack pkg'\")\n\n action = {\n \"add\": pkg_add,\n \"added\": pkg_added,\n \"changed\": pkg_changed,\n \"diff\": pkg_diff,\n \"hash\": pkg_hash,\n \"list\": pkg_list,\n \"removed\": pkg_removed,\n \"source\": pkg_source,\n }\n\n # grep is special as it passes unknown arguments through\n if args.pkg_command == \"grep\":\n return pkg_grep(args, unknown_args)\n elif unknown_args:\n tty.die(\"unrecognized arguments: %s\" % \" \".join(unknown_args))\n else:\n return action[args.pkg_command](args)\n" }, { "path": "lib/spack/spack/cmd/providers.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport io\nimport sys\n\nimport spack.cmd\nimport spack.llnl.util.tty.colify as colify\nimport spack.repo\n\ndescription = \"list packages that provide a particular virtual package\"\nsection = \"query\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n subparser.epilog = \"If called without argument returns the list of all valid virtual packages\"\n subparser.add_argument(\n \"virtual_package\", nargs=\"*\", help=\"find packages that provide this virtual package\"\n )\n\n\ndef providers(parser, args):\n valid_virtuals = sorted(spack.repo.PATH.provider_index.providers.keys())\n\n buffer = io.StringIO()\n isatty = sys.stdout.isatty()\n if isatty:\n buffer.write(\"Virtual packages:\\n\")\n colify.colify(valid_virtuals, output=buffer, tty=isatty, indent=4)\n valid_virtuals_str = buffer.getvalue()\n\n # If called without arguments, list all the virtual packages\n if not args.virtual_package:\n print(valid_virtuals_str)\n return\n\n # Otherwise, parse the specs from command line\n specs = spack.cmd.parse_specs(args.virtual_package)\n\n # Check prerequisites\n non_virtual = [\n str(s)\n for s in specs\n if not spack.repo.PATH.is_virtual(s.name) or s.name not in valid_virtuals\n ]\n if non_virtual:\n msg = \"non-virtual specs cannot be part of the query \"\n msg += \"[{0}]\\n\".format(\", \".join(non_virtual))\n msg += valid_virtuals_str\n raise ValueError(msg)\n\n # Display providers\n for spec in specs:\n if sys.stdout.isatty():\n print(\"{0}:\".format(spec))\n spack.cmd.display_specs(sorted(spack.repo.PATH.providers_for(spec)))\n print(\"\")\n" }, { "path": "lib/spack/spack/cmd/pydoc.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\nimport argparse\n\ndescription = \"run pydoc from within spack\"\nsection = \"developer\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n subparser.add_argument(\"entity\", help=\"run pydoc help on entity\")\n\n\ndef pydoc(parser, args):\n help(args.entity)\n" }, { "path": "lib/spack/spack/cmd/python.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport code\nimport os\nimport platform\nimport runpy\nimport sys\n\nimport spack\nimport spack.llnl.util.tty as tty\nimport spack.repo\n\ndescription = \"launch an interpreter as spack would launch a command\"\nsection = \"developer\"\nlevel = \"long\"\n\nIS_WINDOWS = sys.platform == \"win32\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n subparser.add_argument(\n \"-V\",\n \"--version\",\n action=\"store_true\",\n dest=\"python_version\",\n help=\"print the Python version number and exit\",\n )\n subparser.add_argument(\"-c\", dest=\"python_command\", help=\"command to execute\")\n subparser.add_argument(\n \"-u\",\n dest=\"unbuffered\",\n action=\"store_true\",\n help=\"for compatibility with xdist, do not use without adding -u to the interpreter\",\n )\n subparser.add_argument(\n \"-i\",\n dest=\"python_interpreter\",\n help=\"python interpreter\",\n choices=[\"python\", \"ipython\"],\n default=\"python\",\n )\n subparser.add_argument(\n \"-m\", dest=\"module\", action=\"store\", help=\"run library module as a script\"\n )\n subparser.add_argument(\n \"--path\",\n action=\"store_true\",\n dest=\"show_path\",\n help=\"show path to python interpreter that spack uses\",\n )\n subparser.add_argument(\n \"python_args\", nargs=argparse.REMAINDER, help=\"file to run plus arguments\"\n )\n\n\ndef python(parser, args, unknown_args):\n if args.python_version:\n print(\"Python\", platform.python_version())\n return\n\n if args.show_path:\n print(sys.executable)\n return\n\n if args.module:\n sys.argv = [\"spack-python\"] + unknown_args + args.python_args\n runpy.run_module(args.module, run_name=\"__main__\", alter_sys=True)\n return\n\n if unknown_args:\n tty.die(\"Unknown arguments:\", \" \".join(unknown_args))\n\n # Unexpected behavior from supplying both\n if args.python_command and args.python_args:\n tty.die(\"You can only specify a command OR script, but not both.\")\n\n # Ensure that spack.repo.PATH is initialized\n spack.repo.PATH.repos\n\n # Run user choice of interpreter\n if args.python_interpreter == \"ipython\":\n return ipython_interpreter(args)\n return python_interpreter(args)\n\n\ndef ipython_interpreter(args):\n \"\"\"An ipython interpreter is intended to be interactive, so it doesn't\n support running a script or arguments\n \"\"\"\n try:\n import IPython # type: ignore[import]\n except ImportError:\n tty.die(\"ipython is not installed, install and try again.\")\n\n if \"PYTHONSTARTUP\" in os.environ:\n startup_file = os.environ[\"PYTHONSTARTUP\"]\n if os.path.isfile(startup_file):\n with open(startup_file, encoding=\"utf-8\") as startup:\n exec(startup.read())\n\n # IPython can also support running a script OR command, not both\n if args.python_args:\n IPython.start_ipython(argv=args.python_args)\n elif args.python_command:\n IPython.start_ipython(argv=[\"-c\", args.python_command])\n else:\n header = \"Spack version %s\\nPython %s, %s %s\" % (\n spack.spack_version,\n platform.python_version(),\n platform.system(),\n platform.machine(),\n )\n\n __name__ = \"__main__\" # noqa: F841\n IPython.embed(module=\"__main__\", header=header)\n\n\ndef python_interpreter(args):\n \"\"\"A python interpreter is the default interpreter\"\"\"\n\n if args.python_args and not args.python_command:\n sys.argv = args.python_args\n runpy.run_path(args.python_args[0], run_name=\"__main__\")\n else:\n # Fake a main python shell by setting __name__ to __main__.\n console = code.InteractiveConsole({\"__name__\": \"__main__\", \"spack\": spack})\n if \"PYTHONSTARTUP\" in os.environ:\n startup_file = os.environ[\"PYTHONSTARTUP\"]\n if os.path.isfile(startup_file):\n with open(startup_file, encoding=\"utf-8\") as startup:\n console.runsource(startup.read(), startup_file, \"exec\")\n if args.python_command:\n propagate_exceptions_from(console)\n console.runsource(args.python_command)\n else:\n # no readline module on Windows\n if not IS_WINDOWS:\n # Provides readline support, allowing user to use arrow keys\n console.push(\"import readline\")\n # Provide tabcompletion\n console.push(\"from rlcompleter import Completer\")\n console.push(\"readline.set_completer(Completer(locals()).complete)\")\n console.push('readline.parse_and_bind(\"tab: complete\")')\n\n console.interact(\n \"Spack version %s\\nPython %s, %s %s\"\n % (\n spack.spack_version,\n platform.python_version(),\n platform.system(),\n platform.machine(),\n )\n )\n\n\ndef propagate_exceptions_from(console):\n \"\"\"Set sys.excepthook to let uncaught exceptions return 1 to the shell.\n\n Args:\n console (code.InteractiveConsole): the console that needs a change in sys.excepthook\n \"\"\"\n console.push(\"import sys\")\n console.push(\"_wrapped_hook = sys.excepthook\")\n console.push(\"def _hook(exc_type, exc_value, exc_tb):\")\n console.push(\" _wrapped_hook(exc_type, exc_value, exc_tb)\")\n console.push(\" sys.exit(1)\")\n console.push(\"\")\n console.push(\"sys.excepthook = _hook\")\n" }, { "path": "lib/spack/spack/cmd/reindex.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport os\nimport shutil\n\nimport spack.database\nimport spack.store\nfrom spack.llnl.util import tty\n\ndescription = \"rebuild Spack's package database\"\nsection = \"admin\"\nlevel = \"long\"\n\n\ndef reindex(parser, args):\n current_index = spack.store.STORE.db._index_path\n needs_backup = os.path.isfile(current_index)\n\n if needs_backup:\n backup = f\"{current_index}.bkp\"\n shutil.copy(current_index, backup)\n tty.msg(\"Created a backup copy of the DB at\", backup)\n\n spack.store.STORE.reindex()\n\n extra = [\"If you need to restore, replace it with the backup.\"] if needs_backup else []\n tty.msg(\n f\"The DB at {current_index} has been reindexed to v{spack.database._DB_VERSION}\", *extra\n )\n" }, { "path": "lib/spack/spack/cmd/remove.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\n\nimport spack.cmd\nimport spack.llnl.util.tty as tty\nfrom spack.cmd.common import arguments\n\ndescription = \"remove specs from an environment\"\nsection = \"environments\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n subparser.add_argument(\n \"-a\", \"--all\", action=\"store_true\", help=\"remove all specs from (clear) the environment\"\n )\n subparser.add_argument(\n \"-l\",\n \"--list-name\",\n dest=\"list_name\",\n default=\"specs\",\n help=\"name of the list to remove specs from\",\n )\n subparser.add_argument(\n \"-f\", \"--force\", action=\"store_true\", help=\"remove concretized spec (if any) immediately\"\n )\n arguments.add_common_arguments(subparser, [\"specs\"])\n\n\ndef remove(parser, args):\n env = spack.cmd.require_active_env(cmd_name=\"remove\")\n\n with env.write_transaction():\n if args.all:\n env.clear()\n else:\n for spec in spack.cmd.parse_specs(args.specs):\n env.remove(spec, args.list_name, force=args.force)\n tty.msg(f\"{spec} has been removed from {env.manifest}\")\n env.write()\n" }, { "path": "lib/spack/spack/cmd/repo.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport os\nimport shlex\nimport sys\nimport tempfile\nfrom typing import Any, Dict, Generator, List, Optional, Tuple, Union\n\nimport spack\nimport spack.caches\nimport spack.ci\nimport spack.config\nimport spack.llnl.util.filesystem as fs\nimport spack.llnl.util.tty as tty\nimport spack.repo\nimport spack.spec\nimport spack.util.executable\nimport spack.util.git\nimport spack.util.path\nimport spack.util.spack_json as sjson\nimport spack.util.spack_yaml\nfrom spack.cmd.common import arguments\nfrom spack.error import SpackError\nfrom spack.llnl.util.tty import color\nfrom spack.version import StandardVersion\n\nfrom . import doc_dedented, doc_first_line\n\ndescription = \"manage package source repositories\"\nsection = \"config\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser):\n sp = subparser.add_subparsers(metavar=\"SUBCOMMAND\", dest=\"repo_command\")\n\n # Create\n create_parser = sp.add_parser(\n \"create\", description=doc_dedented(repo_create), help=doc_first_line(repo_create)\n )\n create_parser.add_argument(\"directory\", help=\"directory to create the repo in\")\n create_parser.add_argument(\n \"namespace\", help=\"name or namespace to identify packages in the repository\"\n )\n create_parser.add_argument(\n \"-d\",\n \"--subdirectory\",\n action=\"store\",\n dest=\"subdir\",\n default=spack.repo.packages_dir_name,\n help=\"subdirectory to store packages in the repository\\n\\n\"\n \"default 'packages'. use an empty string for no subdirectory\",\n )\n\n # List\n list_parser = sp.add_parser(\n \"list\", aliases=[\"ls\"], description=doc_dedented(repo_list), help=doc_first_line(repo_list)\n )\n list_parser.add_argument(\n \"--scope\",\n action=arguments.ConfigScope,\n type=arguments.config_scope_readable_validator,\n help=\"configuration scope to read from\",\n )\n output_group = list_parser.add_mutually_exclusive_group()\n output_group.add_argument(\"--names\", action=\"store_true\", help=\"show configuration names only\")\n output_group.add_argument(\n \"--namespaces\", action=\"store_true\", help=\"show repository namespaces only\"\n )\n output_group.add_argument(\n \"--json\", action=\"store_true\", help=\"output repositories as machine-readable json records\"\n )\n\n # Add\n add_parser = sp.add_parser(\n \"add\", description=doc_dedented(repo_add), help=doc_first_line(repo_add)\n )\n add_parser.add_argument(\n \"path_or_repo\", help=\"path or git repository of a Spack package repository\"\n )\n # optional positional argument for destination name in case of git repository\n add_parser.add_argument(\n \"destination\",\n nargs=\"?\",\n default=None,\n help=\"destination to clone git repository into (defaults to cache directory)\",\n )\n add_parser.add_argument(\n \"--name\",\n action=\"store\",\n help=\"config name for the package repository, defaults to the namespace of the repository\",\n )\n add_parser.add_argument(\n \"--path\",\n help=\"relative path to the Spack package repository inside a git repository. Can be \"\n \"repeated to add multiple package repositories in case of a monorepo\",\n action=\"append\",\n default=[],\n )\n add_parser.add_argument(\n \"--scope\",\n action=arguments.ConfigScope,\n default=lambda: spack.config.default_modify_scope(),\n help=\"configuration scope to modify\",\n )\n\n # Set (modify existing repository configuration)\n set_parser = sp.add_parser(\n \"set\", description=doc_dedented(repo_set), help=doc_first_line(repo_set)\n )\n set_parser.add_argument(\"namespace\", help=\"namespace of a Spack package repository\")\n set_parser.add_argument(\n \"--destination\", help=\"destination to clone git repository into\", action=\"store\"\n )\n set_parser.add_argument(\n \"--path\",\n help=\"relative path to the Spack package repository inside a git repository. Can be \"\n \"repeated to add multiple package repositories in case of a monorepo\",\n action=\"append\",\n default=[],\n )\n set_parser.add_argument(\n \"--scope\",\n action=arguments.ConfigScope,\n default=lambda: spack.config.default_modify_scope(),\n help=\"configuration scope to modify\",\n )\n\n # Remove\n remove_parser = sp.add_parser(\n \"remove\",\n description=doc_dedented(repo_remove),\n help=doc_first_line(repo_remove),\n aliases=[\"rm\"],\n )\n remove_parser.add_argument(\n \"namespace_or_path\", help=\"namespace or path of a Spack package repository\"\n )\n remove_parser.add_argument(\n \"--scope\", action=arguments.ConfigScope, default=None, help=\"configuration scope to modify\"\n )\n remove_parser.add_argument(\n \"--all-scopes\",\n action=\"store_true\",\n default=False,\n help=\"remove from all config scopes (default: highest scope with matching repo)\",\n )\n\n # Migrate\n migrate_parser = sp.add_parser(\n \"migrate\", description=doc_dedented(repo_migrate), help=doc_first_line(repo_migrate)\n )\n migrate_parser.add_argument(\n \"namespace_or_path\", help=\"path to a Spack package repository directory\"\n )\n patch_or_fix = migrate_parser.add_mutually_exclusive_group(required=True)\n patch_or_fix.add_argument(\n \"--dry-run\",\n action=\"store_true\",\n help=\"do not modify the repository, but dump a patch file\",\n )\n patch_or_fix.add_argument(\n \"--fix\",\n action=\"store_true\",\n help=\"automatically migrate the repository to the latest Package API\",\n )\n\n # Update\n update_parser = sp.add_parser(\n \"update\", description=doc_dedented(repo_update), help=doc_first_line(repo_update)\n )\n update_parser.add_argument(\"names\", nargs=\"*\", default=[], help=\"repositories to update\")\n update_parser.add_argument(\n \"--remote\",\n \"-r\",\n default=\"origin\",\n nargs=\"?\",\n help=\"name of remote to check for branches, tags, or commits\",\n )\n update_parser.add_argument(\n \"--scope\",\n action=arguments.ConfigScope,\n default=lambda: spack.config.default_modify_scope(),\n help=\"configuration scope to modify\",\n )\n update_parser.add_argument(\n \"--branch\", \"-b\", nargs=\"?\", default=None, help=\"name of a branch to change to\"\n )\n refspec = update_parser.add_mutually_exclusive_group(required=False)\n refspec.add_argument(\"--tag\", \"-t\", nargs=\"?\", default=None, help=\"name of a tag to change to\")\n refspec.add_argument(\n \"--commit\", \"-c\", nargs=\"?\", default=None, help=\"name of a commit to change to\"\n )\n\n # Show updates\n show_version_updates_parser = sp.add_parser(\n \"show-version-updates\", help=repo_show_version_updates.__doc__\n )\n show_version_updates_parser.add_argument(\n \"--no-manual-packages\", action=\"store_true\", help=\"exclude manual packages\"\n )\n show_version_updates_parser.add_argument(\n \"--no-git-versions\", action=\"store_true\", help=\"exclude versions from git\"\n )\n show_version_updates_parser.add_argument(\n \"--only-redistributable\", action=\"store_true\", help=\"exclude non-redistributable packages\"\n )\n show_version_updates_parser.add_argument(\n \"repository\", help=\"name or path of the repository to analyze\"\n )\n show_version_updates_parser.add_argument(\n \"from_ref\", help=\"git ref from which to start looking at changes\"\n )\n show_version_updates_parser.add_argument(\"to_ref\", help=\"git ref to end looking at changes\")\n\n\ndef repo_create(args):\n \"\"\"create a new package repository\"\"\"\n full_path, namespace = spack.repo.create_repo(args.directory, args.namespace, args.subdir)\n tty.msg(\"Created repo with namespace '%s'.\" % namespace)\n tty.msg(\"To register it with spack, run this command:\", \"spack repo add %s\" % full_path)\n\n\ndef _add_repo(\n path_or_repo: str,\n name: Optional[str],\n scope: Optional[str],\n paths: List[str],\n destination: Optional[str],\n config: Optional[spack.config.Configuration] = None,\n) -> str:\n config = config or spack.config.CONFIG\n\n existing: Dict[str, Any] = config.get(\"repos\", default={}, scope=scope)\n\n if name and name in existing:\n raise SpackError(f\"A repository with the name '{name}' already exists.\")\n\n # Interpret as a git URL when it contains a colon at index 2 or more, not preceded by a\n # forward slash. That allows C:/ windows paths, while following git's convention to distinguish\n # between local paths on the one hand and URLs and SCP like syntax on the other.\n entry: Union[str, Dict[str, Any]]\n colon_idx = path_or_repo.find(\":\")\n\n if colon_idx > 1 and \"/\" not in path_or_repo[:colon_idx]: # git URL\n entry = {\"git\": path_or_repo}\n if len(paths) >= 1:\n entry[\"paths\"] = paths\n if destination:\n entry[\"destination\"] = destination\n else: # local path\n if destination:\n raise SpackError(\"The 'destination' argument is only valid for git repositories\")\n elif paths:\n raise SpackError(\"The --paths flag is only valid for git repositories\")\n entry = spack.util.path.canonicalize_path(path_or_repo)\n\n descriptor = spack.repo.parse_config_descriptor(\n name or \"\", entry, lock=spack.repo.package_repository_lock()\n )\n descriptor.initialize(git=spack.util.executable.which(\"git\"))\n\n packages_repos = descriptor.construct(cache=spack.caches.MISC_CACHE)\n\n usable_repos: Dict[str, spack.repo.Repo] = {}\n\n for _path, _repo_or_err in packages_repos.items():\n if isinstance(_repo_or_err, Exception):\n tty.warn(f\"Skipping package repository '{_path}' due to: {_repo_or_err}\")\n else:\n usable_repos[_path] = _repo_or_err\n\n if not usable_repos:\n raise SpackError(f\"No package repository could be constructed from {path_or_repo}\")\n\n # For the config key, default to --name, then to the namespace if there's only one repo.\n # Otherwise, the name is unclear and we require the user to specify it.\n if name:\n key = name\n elif len(usable_repos) == 1:\n key = next(iter(usable_repos.values())).namespace\n else:\n raise SpackError(\"Multiple package repositories found, please specify a name with --name.\")\n\n if key in existing:\n raise SpackError(f\"A repository with the name '{key}' already exists.\")\n\n # Prepend the new repository\n config.set(\"repos\", spack.util.spack_yaml.syaml_dict({key: entry, **existing}), scope)\n return key\n\n\ndef repo_add(args):\n \"\"\"add package repositories to Spack's configuration\"\"\"\n name = _add_repo(\n path_or_repo=args.path_or_repo,\n name=args.name,\n scope=args.scope,\n paths=args.path,\n destination=args.destination,\n )\n tty.msg(f\"Added repo to config with name '{name}'.\")\n\n\ndef repo_remove(args):\n \"\"\"remove a repository from Spack's configuration\"\"\"\n scopes = [args.scope] if args.scope else reversed(list(spack.config.CONFIG.scopes.keys()))\n found_and_removed = False\n for scope in scopes:\n found_and_removed |= _remove_repo(args.namespace_or_path, scope)\n if found_and_removed and not args.all_scopes:\n return\n if not found_and_removed:\n tty.die(f\"No repository with path or namespace: {args.namespace_or_path}\")\n\n\ndef _remove_repo(namespace_or_path, scope):\n repos: Dict[str, str] = spack.config.get(\"repos\", scope=scope)\n\n if namespace_or_path in repos:\n # delete by name (from config)\n key = namespace_or_path\n else:\n # delete by namespace or path (requires constructing the repo)\n canon_path = spack.util.path.canonicalize_path(namespace_or_path)\n descriptors = spack.repo.RepoDescriptors.from_config(\n spack.repo.package_repository_lock(), spack.config.CONFIG, scope=scope\n )\n for name, descriptor in descriptors.items():\n descriptor.initialize(fetch=False)\n\n # For now you cannot delete monorepos with multiple package repositories from config,\n # hence \"all\" and not \"any\". We can improve this later if needed.\n if all(\n r.namespace == namespace_or_path or r.root == canon_path\n for r in descriptor.construct(cache=spack.caches.MISC_CACHE).values()\n if isinstance(r, spack.repo.Repo)\n ):\n key = name\n break\n else:\n return False\n\n del repos[key]\n spack.config.set(\"repos\", repos, scope)\n tty.msg(f\"Removed repository '{namespace_or_path}' from scope '{scope}'.\")\n return True\n\n\ndef repo_list(args):\n \"\"\"show registered repositories and their namespaces\n\n List all package repositories known to Spack. Repositories\n can be local directories or remote git repositories.\n \"\"\"\n descriptors = spack.repo.RepoDescriptors.from_config(\n lock=spack.repo.package_repository_lock(), config=spack.config.CONFIG, scope=args.scope\n )\n\n # --names: just print config names\n if args.names:\n for name in descriptors:\n print(name)\n return\n\n # --namespaces: print all repo namespaces\n if args.namespaces:\n for name, path, maybe_repo in _iter_repos_from_descriptors(descriptors):\n if isinstance(maybe_repo, spack.repo.Repo):\n print(maybe_repo.namespace)\n return\n\n # Collect all repository information\n repo_info = []\n\n for name, path, maybe_repo in _iter_repos_from_descriptors(descriptors):\n if isinstance(maybe_repo, spack.repo.Repo):\n status = \"installed\"\n namespace = maybe_repo.namespace\n api = maybe_repo.package_api_str\n repo_path = maybe_repo.root\n elif maybe_repo is None: # Uninitialized Git-based repo case\n status = \"uninitialized\"\n namespace = name\n api = \"\"\n repo_path = path\n else: # Exception/error case\n status = \"error\"\n namespace = name\n api = \"\"\n repo_path = path\n\n # Add the repo info to our list\n repo_info.append(\n {\n \"name\": name,\n \"namespace\": namespace,\n \"path\": repo_path,\n \"api_version\": api,\n \"status\": status,\n \"error\": str(maybe_repo) if isinstance(maybe_repo, Exception) else None,\n }\n )\n\n # Output in JSON format if requested\n if args.json:\n sjson.dump(repo_info, sys.stdout)\n return\n\n # Default table format with aligned output\n formatted_repo_info = []\n for repo in repo_info:\n if repo[\"status\"] == \"installed\":\n status = \"@g{[+]}\"\n elif repo[\"status\"] == \"uninitialized\":\n status = \"@K{ - }\"\n else: # error\n status = \"@r{[-]}\"\n\n formatted_repo_info.append((status, repo[\"namespace\"], repo[\"api_version\"], repo[\"path\"]))\n\n if formatted_repo_info:\n max_namespace_width = max(len(namespace) for _, namespace, _, _ in formatted_repo_info) + 3\n max_api_width = max(len(api) for _, _, api, _ in formatted_repo_info) + 3\n\n # Print aligned output\n for status, namespace, api, path in formatted_repo_info:\n cpath = color.cescape(path)\n color.cprint(\n f\"{status} {namespace:<{max_namespace_width}} {api:<{max_api_width}} {cpath}\"\n )\n\n\ndef _get_repo(name_or_path: str) -> Optional[spack.repo.Repo]:\n \"\"\"get a repo by path or namespace\"\"\"\n try:\n return spack.repo.from_path(name_or_path)\n except spack.repo.RepoError:\n pass\n\n descriptors = spack.repo.RepoDescriptors.from_config(\n spack.repo.package_repository_lock(), spack.config.CONFIG\n )\n\n repo_path, _ = descriptors.construct(cache=spack.caches.MISC_CACHE, fetch=False)\n\n for repo in repo_path.repos:\n if repo.namespace == name_or_path:\n return repo\n\n return None\n\n\ndef repo_migrate(args: Any) -> int:\n \"\"\"migrate a package repository to the latest Package API\"\"\"\n from spack.repo_migrate import migrate_v1_to_v2, migrate_v2_imports\n\n repo = _get_repo(args.namespace_or_path)\n\n if repo is None:\n tty.die(f\"No such repository: {args.namespace_or_path}\")\n\n if args.dry_run:\n fd, patch_file_path = tempfile.mkstemp(\n suffix=\".patch\", prefix=\"repo-migrate-\", dir=os.getcwd()\n )\n patch_file = os.fdopen(fd, \"bw\")\n tty.msg(f\"Patch file will be written to {patch_file_path}\")\n else:\n patch_file_path = None\n patch_file = None\n\n try:\n if (1, 0) <= repo.package_api < (2, 0):\n success, repo_v2 = migrate_v1_to_v2(repo, patch_file=patch_file)\n exit_code = 0 if success else 1\n elif (2, 0) <= repo.package_api < (3, 0):\n repo_v2 = None\n exit_code = (\n 0\n if migrate_v2_imports(repo.packages_path, repo.root, patch_file=patch_file)\n else 1\n )\n else:\n repo_v2 = None\n exit_code = 0\n finally:\n if patch_file is not None:\n patch_file.flush()\n patch_file.close()\n\n if patch_file_path:\n tty.warn(\n f\"No changes were made to the '{repo.namespace}' repository with. Review \"\n f\"the changes written to {patch_file_path}. Run \\n\\n\"\n f\" spack repo migrate --fix {args.namespace_or_path}\\n\\n\"\n \"to upgrade the repo.\"\n )\n\n elif exit_code == 1:\n tty.error(\n f\"Repository '{repo.namespace}' could not be migrated to the latest Package API. \"\n \"Please check the error messages above.\"\n )\n\n elif isinstance(repo_v2, spack.repo.Repo):\n tty.info(\n f\"Repository '{repo_v2.namespace}' was successfully migrated from \"\n f\"package API {repo.package_api_str} to {repo_v2.package_api_str}.\"\n )\n tty.warn(\n \"Remove the old repository from Spack's configuration and add the new one using:\\n\"\n f\" spack repo remove {shlex.quote(repo.root)}\\n\"\n f\" spack repo add {shlex.quote(repo_v2.root)}\"\n )\n\n else:\n tty.info(f\"Repository '{repo.namespace}' was successfully migrated\")\n\n return exit_code\n\n\ndef repo_set(args):\n \"\"\"modify an existing repository configuration\"\"\"\n namespace = args.namespace\n\n # First, check if the repository exists across all scopes for validation\n all_repos: Dict[str, Any] = spack.config.get(\"repos\", default={})\n\n if namespace not in all_repos:\n raise SpackError(f\"No repository with namespace '{namespace}' found in configuration.\")\n\n # Validate that it's a git repository\n if not isinstance(all_repos[namespace], dict):\n raise SpackError(\n f\"Repository '{namespace}' is not a git repository. \"\n \"The 'set' command only works with git repositories.\"\n )\n\n # Now get the repos for the specific scope we're modifying\n scope_repos: Dict[str, Any] = spack.config.get(\"repos\", default={}, scope=args.scope)\n\n updated_entry = scope_repos[namespace] if namespace in scope_repos else {}\n\n if args.destination:\n updated_entry[\"destination\"] = args.destination\n\n if args.path:\n updated_entry[\"paths\"] = args.path\n\n scope_repos[namespace] = updated_entry\n spack.config.set(\"repos\", scope_repos, args.scope)\n\n tty.msg(f\"Updated repo '{namespace}'\")\n\n\ndef _iter_repos_from_descriptors(\n descriptors: spack.repo.RepoDescriptors,\n) -> Generator[Tuple[str, str, Union[spack.repo.Repo, Exception, None]], None, None]:\n \"\"\"Iterate through repository descriptors and yield (name, path, maybe_repo) tuples.\n\n Yields:\n Tuple of (config_name, path, maybe_repo) where maybe_repo is a Repo instance if it could\n be instantiated, an Exception if it could not be instantiated, or None if it was not\n initialized yet.\n \"\"\"\n for name, descriptor in descriptors.items():\n descriptor.initialize(fetch=False)\n repos_for_descriptor = descriptor.construct(cache=spack.caches.MISC_CACHE)\n\n for path, maybe_repo in repos_for_descriptor.items():\n yield name, path, maybe_repo\n\n # If there are no repos, it means it's not yet cloned; yield descriptor info\n if not repos_for_descriptor and isinstance(descriptor, spack.repo.RemoteRepoDescriptor):\n yield name, descriptor.repository, None # None indicates remote descriptor\n\n\ndef repo_update(args):\n \"\"\"update one or more package repositories\"\"\"\n descriptors = spack.repo.RepoDescriptors.from_config(\n spack.repo.package_repository_lock(), spack.config.CONFIG\n )\n\n git_flags = [\"commit\", \"tag\", \"branch\"]\n active_flag = next((attr for attr in git_flags if getattr(args, attr)), None)\n if active_flag and len(args.names) != 1:\n raise SpackError(\n f\"Unable to set --{active_flag} because more than one namespace was given.\"\n if len(args.names) > 1\n else f\"Unable to apply --{active_flag} without a namespace\"\n )\n\n for name in args.names:\n if name not in descriptors:\n raise SpackError(f\"{name} is not a known repository name.\")\n\n # filter descriptors when namespaces are provided as arguments\n descriptors = spack.repo.RepoDescriptors(\n {name: descriptor for name, descriptor in descriptors.items() if name in args.names}\n )\n\n # Get the repos for the specific scope we're modifying\n scope_repos: Dict[str, Any] = spack.config.get(\"repos\", default={}, scope=args.scope)\n\n for name, descriptor in descriptors.items():\n if not isinstance(descriptor, spack.repo.RemoteRepoDescriptor):\n continue\n\n if active_flag:\n # update the git commit, tag, or branch of the descriptor\n setattr(descriptor, active_flag, getattr(args, active_flag))\n\n updated_entry = scope_repos[name] if name in scope_repos else {}\n\n # prune previous values of git fields\n for entry in {\"commit\", \"tag\"} - {active_flag}:\n setattr(descriptor, entry, None)\n updated_entry.pop(entry, None)\n\n updated_entry[active_flag] = args.commit or args.tag or args.branch\n scope_repos[name] = updated_entry\n\n git = spack.util.git.git(required=True)\n\n previous_commit = descriptor.get_commit(git=git)\n descriptor.update(git=git, remote=args.remote)\n new_commit = descriptor.get_commit(git=git)\n\n if previous_commit == new_commit:\n tty.msg(f\"{name}: Already up to date.\")\n else:\n fails = [\n r\n for r in descriptor.construct(cache=spack.caches.MISC_CACHE).values()\n if type(r) is spack.repo.BadRepoVersionError\n ]\n if fails:\n min_ver = \".\".join(str(n) for n in spack.min_package_api_version)\n max_ver = \".\".join(str(n) for n in spack.package_api_version)\n tty.error(\n f\"{name}: repo is too new for this version of Spack. \",\n f\" Spack supports API v{min_ver} to v{max_ver}, but repo is {fails[0].api}\",\n \" Please upgrade Spack or revert with:\\n\",\n f\" spack repo update --commit {previous_commit}\\n\",\n )\n\n else:\n tty.msg(f\"{name}: Updated successfully.\")\n\n if active_flag:\n spack.config.set(\"repos\", scope_repos, args.scope)\n\n\ndef repo_show_version_updates(args):\n \"\"\"show version specs that were added between two commits\"\"\"\n # Get the repository by name or path\n repo = _get_repo(args.repository)\n\n if repo is None:\n tty.die(f\"No such repository: {args.repository}\")\n\n # Get packages that were changed or added between the refs\n pkgs = spack.repo.get_all_package_diffs(\"AC\", repo, args.from_ref, args.to_ref)\n\n # Filter out manual packages if requested\n if args.no_manual_packages:\n pkgs = {\n pkg_name\n for pkg_name in pkgs\n if not spack.repo.PATH.get_pkg_class(pkg_name).manual_download\n }\n\n if not pkgs:\n tty.info(\"No packages were added or changed between the specified refs\", stream=sys.stderr)\n return 0\n\n # Collect version specs that were added\n specs_to_output = []\n\n for pkg_name in pkgs:\n pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)\n path = spack.repo.PATH.package_path(pkg_name)\n\n # Get all versions with checksums or commits\n version_to_checksum: Dict[StandardVersion, str] = {}\n for version in pkg_cls.versions:\n version_dict = pkg_cls.versions[version]\n if \"sha256\" in version_dict:\n version_to_checksum[version] = version_dict[\"sha256\"]\n elif \"commit\" in version_dict:\n version_to_checksum[version] = version_dict[\"commit\"]\n\n # Find versions added between the refs\n with fs.working_dir(os.path.dirname(path)):\n added_checksums = spack.ci.filter_added_checksums(\n version_to_checksum.values(), path, from_ref=args.from_ref, to_ref=args.to_ref\n )\n new_versions = [v for v, c in version_to_checksum.items() if c in added_checksums]\n\n # Create specs for new versions\n for version in new_versions:\n version_spec = spack.spec.Spec(pkg_name)\n version_spec.constrain(f\"@={version}\")\n specs_to_output.append(version_spec)\n\n # Filter out git versions if requested\n if args.no_git_versions:\n specs_to_output = [\n spec\n for spec in specs_to_output\n if \"commit\" not in spack.repo.PATH.get_pkg_class(spec.name).versions[spec.version]\n ]\n\n # Filter out non-redistributable packages if requested\n if args.only_redistributable:\n specs_to_output = [\n spec\n for spec in specs_to_output\n if spack.repo.PATH.get_pkg_class(spec.name).redistribute_source(spec)\n ]\n\n if not specs_to_output:\n tty.info(\"No new package versions found between the specified refs\", stream=sys.stderr)\n return 0\n\n # Output specs one per line\n for spec in specs_to_output:\n print(spec)\n\n\ndef repo(parser, args):\n return {\n \"create\": repo_create,\n \"list\": repo_list,\n \"ls\": repo_list,\n \"add\": repo_add,\n \"set\": repo_set,\n \"remove\": repo_remove,\n \"rm\": repo_remove,\n \"migrate\": repo_migrate,\n \"update\": repo_update,\n \"show-version-updates\": repo_show_version_updates,\n }[args.repo_command](args)\n" }, { "path": "lib/spack/spack/cmd/resource.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport os\n\nimport spack.llnl.util.tty as tty\nimport spack.llnl.util.tty.color as color\nimport spack.repo\n\ndescription = \"list downloadable resources (tarballs, repos, patches)\"\nsection = \"query\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n sp = subparser.add_subparsers(metavar=\"SUBCOMMAND\", dest=\"resource_command\")\n\n list_parser = sp.add_parser(\"list\", help=resource_list.__doc__)\n list_parser.add_argument(\n \"--only-hashes\", action=\"store_true\", help=\"only print sha256 hashes of resources\"\n )\n\n show_parser = sp.add_parser(\"show\", help=resource_show.__doc__)\n show_parser.add_argument(\"hash\", action=\"store\")\n\n\ndef _show_patch(sha256):\n \"\"\"Show a record from the patch index.\"\"\"\n patches = spack.repo.PATH.get_patch_index().index\n data = patches.get(sha256)\n\n if not data:\n candidates = [k for k in patches if k.startswith(sha256)]\n if not candidates:\n tty.die(\"no such resource: %s\" % sha256)\n elif len(candidates) > 1:\n tty.die(\"%s: ambiguous hash prefix. Options are:\", *candidates)\n\n sha256 = candidates[0]\n data = patches.get(sha256)\n\n color.cprint(\"@c{%s}\" % sha256)\n for package, rec in data.items():\n owner = rec[\"owner\"]\n\n if \"relative_path\" in rec:\n pkg_dir = spack.repo.PATH.get_pkg_class(owner).package_dir\n path = os.path.join(pkg_dir, rec[\"relative_path\"])\n print(\" path: %s\" % path)\n else:\n print(\" url: %s\" % rec[\"url\"])\n\n print(\" applies to: %s\" % package)\n if owner != package:\n print(\" patched by: %s\" % owner)\n\n\ndef resource_list(args):\n \"\"\"list all resources known to spack (currently just patches)\"\"\"\n patches = spack.repo.PATH.get_patch_index().index\n for sha256 in patches:\n if args.only_hashes:\n print(sha256)\n else:\n _show_patch(sha256)\n\n\ndef resource_show(args):\n \"\"\"show a resource, identified by its checksum\"\"\"\n _show_patch(args.hash)\n\n\ndef resource(parser, args):\n action = {\"list\": resource_list, \"show\": resource_show}\n action[args.resource_command](args)\n" }, { "path": "lib/spack/spack/cmd/restage.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\n\nimport spack.cmd\nimport spack.llnl.util.tty as tty\nfrom spack.cmd.common import arguments\n\ndescription = \"revert checked out package source code\"\nsection = \"build\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n arguments.add_common_arguments(subparser, [\"specs\"])\n\n\ndef restage(parser, args):\n if not args.specs:\n tty.die(\"spack restage requires at least one package spec.\")\n\n specs = spack.cmd.parse_specs(args.specs, concretize=True)\n for spec in specs:\n spec.package.do_restage()\n" }, { "path": "lib/spack/spack/cmd/solve.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport re\nimport sys\n\nimport spack\nimport spack.cmd\nimport spack.cmd.spec\nimport spack.config\nimport spack.environment\nimport spack.hash_types as ht\nimport spack.llnl.util.tty as tty\nimport spack.llnl.util.tty.color as color\nimport spack.package_base\nimport spack.solver.asp as asp\nimport spack.spec\n\ndescription = \"concretize a specs using an ASP solver\"\nsection = \"developer\"\nlevel = \"long\"\n\n#: output options\nshow_options = (\"asp\", \"opt\", \"output\", \"solutions\")\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n # Solver arguments\n subparser.add_argument(\n \"--show\",\n action=\"store\",\n default=\"opt,solutions\",\n help=\"select outputs\\n\\ncomma-separated list of:\\n\"\n \" asp asp program text\\n\"\n \" opt optimization criteria for best model\\n\"\n \" output raw clingo output\\n\"\n \" solutions models found by asp program\\n\"\n \" all all of the above\",\n )\n subparser.add_argument(\n \"--timers\",\n action=\"store_true\",\n default=False,\n help=\"print out timers for different solve phases\",\n )\n subparser.add_argument(\n \"--stats\", action=\"store_true\", default=False, help=\"print out statistics from clingo\"\n )\n\n spack.cmd.spec.setup_parser(subparser)\n\n\ndef _process_result(result, show, required_format, kwargs):\n opt, _, _ = min(result.answers)\n if (\"opt\" in show) and (not required_format):\n tty.msg(\"Best of %d considered solutions.\" % result.nmodels)\n\n print()\n maxlen = max(len(s.name) for s in result.criteria)\n color.cprint(\"@*{ Priority Value Criterion}\")\n\n for i, criterion in enumerate(result.criteria, 1):\n value = f\"@K{{{criterion.value:>5}}}\"\n grey_out = True\n if criterion.value > 0:\n value = f\"@*{{{criterion.value:>5}}}\"\n grey_out = False\n\n if grey_out:\n lc = \"@K\"\n elif criterion.kind == asp.OptimizationKind.CONCRETE:\n lc = \"@b\"\n elif criterion.kind == asp.OptimizationKind.BUILD:\n lc = \"@g\"\n else:\n lc = \"@y\"\n\n color.cprint(f\" @K{{{i:8}}} {value} {lc}{{{criterion.name:<{maxlen}}}}\")\n print()\n print()\n color.cprint(\" @*{Legend:}\")\n color.cprint(\" @g{Specs to be built}\")\n color.cprint(\" @b{Reused specs}\")\n color.cprint(\" @y{Other criteria}\")\n print()\n\n # dump the solutions as concretized specs\n if \"solutions\" in show:\n if required_format:\n for spec in result.specs:\n # With -y, just print YAML to output.\n if required_format == \"yaml\":\n # use write because to_yaml already has a newline.\n sys.stdout.write(spec.to_yaml(hash=ht.dag_hash))\n elif required_format == \"json\":\n sys.stdout.write(spec.to_json(hash=ht.dag_hash))\n else:\n sys.stdout.write(spack.spec.tree(result.specs, color=sys.stdout.isatty(), **kwargs))\n print()\n\n if result.unsolved_specs and \"solutions\" in show:\n tty.msg(asp.Result.format_unsolved(result.unsolved_specs))\n\n\ndef solve(parser, args):\n # these are the same options as `spack spec`\n install_status_fn = spack.spec.Spec.install_status\n\n fmt = spack.spec.DISPLAY_FORMAT\n if args.namespaces:\n fmt = \"{namespace}.\" + fmt\n\n kwargs = {\n \"cover\": args.cover,\n \"format\": fmt,\n \"hashlen\": None if args.very_long else 7,\n \"show_types\": args.types,\n \"status_fn\": install_status_fn if args.install_status else None,\n \"hashes\": args.long or args.very_long,\n \"highlight_version_fn\": (\n spack.package_base.non_preferred_version if args.non_defaults else None\n ),\n \"highlight_variant_fn\": (\n spack.package_base.non_default_variant if args.non_defaults else None\n ),\n }\n\n # process output options\n show = re.split(r\"\\s*,\\s*\", args.show)\n if \"all\" in show:\n show = show_options\n for d in show:\n if d not in show_options:\n raise ValueError(\n \"Invalid option for '--show': '%s'\\nchoose from: (%s)\"\n % (d, \", \".join(show_options + (\"all\",)))\n )\n\n # Format required for the output (JSON, YAML or None)\n required_format = args.format\n\n # If we have an active environment, pick the specs from there\n env = spack.environment.active_environment()\n if args.specs:\n specs = spack.cmd.parse_specs(args.specs)\n elif env:\n specs = list(env.user_specs)\n else:\n tty.die(\"spack solve requires at least one spec or an active environment\")\n\n solver = asp.Solver()\n output = sys.stdout if \"asp\" in show else None\n setup_only = set(show) == {\"asp\"}\n unify = spack.config.get(\"concretizer:unify\")\n allow_deprecated = spack.config.get(\"config:deprecated\", False)\n if unify == \"when_possible\":\n for idx, result in enumerate(\n solver.solve_in_rounds(\n specs,\n out=output,\n timers=args.timers,\n stats=args.stats,\n allow_deprecated=allow_deprecated,\n )\n ):\n if \"solutions\" in show:\n tty.msg(\"ROUND {0}\".format(idx))\n tty.msg(\"\")\n else:\n print(\"% END ROUND {0}\\n\".format(idx))\n if not setup_only:\n _process_result(result, show, required_format, kwargs)\n elif unify:\n # set up solver parameters\n # Note: reuse and other concretizer prefs are passed as configuration\n result = solver.solve(\n specs,\n out=output,\n timers=args.timers,\n stats=args.stats,\n setup_only=setup_only,\n allow_deprecated=allow_deprecated,\n )\n if not setup_only:\n _process_result(result, show, required_format, kwargs)\n else:\n for spec in specs:\n tty.msg(\"SOLVING SPEC:\", spec)\n result = solver.solve(\n [spec],\n out=output,\n timers=args.timers,\n stats=args.stats,\n setup_only=setup_only,\n allow_deprecated=allow_deprecated,\n )\n if not setup_only:\n _process_result(result, show, required_format, kwargs)\n" }, { "path": "lib/spack/spack/cmd/spec.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport sys\n\nimport spack\nimport spack.cmd\nimport spack.environment as ev\nimport spack.hash_types as ht\nimport spack.llnl.util.lang as lang\nimport spack.llnl.util.tty as tty\nimport spack.package_base\nimport spack.spec\nimport spack.store\nimport spack.traverse\nfrom spack.cmd.common import arguments\n\ndescription = \"show what would be installed, given a spec\"\nsection = \"build\"\nlevel = \"short\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n subparser.epilog = \"\"\"\\\nwhen an environment is active and no specs are provided, the environment root \\\nspecs are used instead\n\nfor further documentation regarding the spec syntax, see:\n spack help --spec\n\"\"\"\n arguments.add_common_arguments(subparser, [\"long\", \"very_long\", \"namespaces\"])\n\n install_status_group = subparser.add_mutually_exclusive_group()\n arguments.add_common_arguments(install_status_group, [\"install_status\", \"no_install_status\"])\n\n format_group = subparser.add_mutually_exclusive_group()\n format_group.add_argument(\n \"-y\",\n \"--yaml\",\n action=\"store_const\",\n dest=\"format\",\n default=None,\n const=\"yaml\",\n help=\"print concrete spec as YAML\",\n )\n format_group.add_argument(\n \"-j\",\n \"--json\",\n action=\"store_const\",\n dest=\"format\",\n default=None,\n const=\"json\",\n help=\"print concrete spec as JSON\",\n )\n format_group.add_argument(\n \"--format\",\n action=\"store\",\n default=None,\n help=\"print concrete spec with the specified format string\",\n )\n arguments.add_common_arguments(format_group, [\"show_non_defaults\"])\n\n subparser.add_argument(\n \"-c\",\n \"--cover\",\n action=\"store\",\n default=\"nodes\",\n choices=[\"nodes\", \"edges\", \"paths\"],\n help=\"how extensively to traverse the DAG (default: nodes)\",\n )\n subparser.add_argument(\n \"-t\", \"--types\", action=\"store_true\", default=False, help=\"show dependency types\"\n )\n arguments.add_common_arguments(subparser, [\"specs\"])\n arguments.add_concretizer_args(subparser)\n\n\ndef spec(parser, args):\n install_status_fn = spack.spec.Spec.install_status\n\n fmt = spack.spec.DISPLAY_FORMAT\n if args.namespaces:\n fmt = \"{namespace}.\" + fmt\n\n # use a read transaction if we are getting install status for every\n # spec in the DAG. This avoids repeatedly querying the DB.\n tree_context = lang.nullcontext\n if args.install_status:\n tree_context = spack.store.STORE.db.read_transaction\n\n env = ev.active_environment()\n\n if args.specs:\n concrete_specs = spack.cmd.parse_specs(args.specs, concretize=True)\n elif env:\n env.concretize()\n concrete_specs = env.concrete_roots()\n else:\n tty.die(\"spack spec requires at least one spec or an active environment\")\n\n # With --yaml, --json, or --format, just print the raw specs to output\n if args.format:\n for spec in concrete_specs:\n if args.format == \"yaml\":\n # use write because to_yaml already has a newline.\n sys.stdout.write(spec.to_yaml(hash=ht.dag_hash))\n elif args.format == \"json\":\n print(spec.to_json(hash=ht.dag_hash))\n else:\n print(spec.format(args.format))\n return\n\n with tree_context():\n print(\n spack.spec.tree(\n concrete_specs,\n cover=args.cover,\n format=fmt,\n hashlen=None if args.very_long else 7,\n show_types=args.types,\n status_fn=install_status_fn if args.install_status else None,\n hashes=args.long or args.very_long,\n key=spack.traverse.by_dag_hash,\n highlight_version_fn=(\n spack.package_base.non_preferred_version if args.non_defaults else None\n ),\n highlight_variant_fn=(\n spack.package_base.non_default_variant if args.non_defaults else None\n ),\n )\n )\n" }, { "path": "lib/spack/spack/cmd/stage.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport os\n\nimport spack.cmd\nimport spack.config\nimport spack.environment as ev\nimport spack.llnl.util.tty as tty\nimport spack.package_base\nimport spack.traverse\nfrom spack.cmd.common import arguments\n\ndescription = \"expand downloaded archive in preparation for install\"\nsection = \"build\"\nlevel = \"long\"\n\n\nclass StageFilter:\n \"\"\"\n Encapsulation of reasons to skip staging\n \"\"\"\n\n def __init__(self, exclusions, skip_installed):\n \"\"\"\n :param exclusions: A list of specs to skip if satisfied.\n :param skip_installed: A boolean indicating whether to skip already installed specs.\n \"\"\"\n self.exclusions = exclusions\n self.skip_installed = skip_installed\n\n def __call__(self, spec):\n \"\"\"filter action, true means spec should be filtered\"\"\"\n if spec.external:\n return True\n\n if self.skip_installed and spec.installed:\n return True\n\n if any(spec.satisfies(exclude) for exclude in self.exclusions):\n return True\n\n return False\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n arguments.add_common_arguments(subparser, [\"no_checksum\", \"specs\"])\n subparser.add_argument(\n \"-p\", \"--path\", dest=\"path\", help=\"path to stage package, does not add to spack tree\"\n )\n subparser.add_argument(\n \"-e\",\n \"--exclude\",\n action=\"append\",\n default=[],\n help=\"exclude packages that satisfy the specified specs\",\n )\n subparser.add_argument(\n \"-s\", \"--skip-installed\", action=\"store_true\", help=\"dont restage already installed specs\"\n )\n arguments.add_concretizer_args(subparser)\n\n\ndef stage(parser, args):\n if args.no_checksum:\n spack.config.set(\"config:checksum\", False, scope=\"command_line\")\n\n exclusion_specs = spack.cmd.parse_specs(args.exclude, concretize=False)\n filter = StageFilter(exclusion_specs, args.skip_installed)\n\n if not args.specs:\n env = ev.active_environment()\n if not env:\n tty.die(\"`spack stage` requires a spec or an active environment\")\n return _stage_env(env, filter)\n\n specs = spack.cmd.parse_specs(args.specs, concretize=False)\n\n # We temporarily modify the working directory when setting up a stage, so we need to\n # convert this to an absolute path here in order for it to remain valid later.\n custom_path = os.path.abspath(args.path) if args.path else None\n\n # prevent multiple specs from extracting in the same folder\n if len(specs) > 1 and custom_path:\n tty.die(\"`--path` requires a single spec, but multiple were provided\")\n\n specs = spack.cmd.matching_specs_from_env(specs)\n for spec in specs:\n spec = spack.cmd.matching_spec_from_env(spec)\n\n if filter(spec):\n continue\n\n pkg = spec.package\n\n if custom_path:\n pkg.path = custom_path\n\n _stage(pkg)\n\n\ndef _stage_env(env: ev.Environment, filter):\n tty.msg(f\"Staging specs from environment {env.name}\")\n for spec in spack.traverse.traverse_nodes(env.concrete_roots()):\n if filter(spec):\n continue\n\n _stage(spec.package)\n\n\ndef _stage(pkg: spack.package_base.PackageBase):\n # Use context manager to ensure we don't restage while an installation is in progress\n # keep = True ensures that the stage is not removed after exiting the context manager\n pkg.stage.keep = True\n with pkg.stage:\n pkg.do_stage()\n tty.msg(f\"Staged {pkg.name} in {pkg.stage.path}\")\n" }, { "path": "lib/spack/spack/cmd/style.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport argparse\nimport ast\nimport os\nimport re\nimport sys\nfrom pathlib import Path\nfrom typing import Dict, List, Optional, Set, Union\n\nimport spack.llnl.util.tty as tty\nimport spack.llnl.util.tty.color as color\nimport spack.paths\nimport spack.repo\nimport spack.util.git\nfrom spack.cmd.common.spec_strings import (\n _check_spec_strings,\n _spec_str_default_handler,\n _spec_str_fix_handler,\n)\nfrom spack.llnl.util.filesystem import working_dir\nfrom spack.util.executable import Executable, which\n\ndescription = \"runs source code style checks on spack\"\nsection = \"developer\"\nlevel = \"long\"\n\n#: List of paths to exclude from checks -- relative to spack root\nexclude_paths = [os.path.relpath(spack.paths.vendor_path, spack.paths.prefix)]\n\n#: Order in which tools should be run.\n#: The list maps an executable name to a method to ensure the tool is\n#: bootstrapped or present in the environment.\ntool_names = [\"import\", \"ruff-format\", \"ruff-check\", \"mypy\"]\n\n#: warnings to ignore in mypy\nmypy_ignores = [\n # same as `disable_error_code = \"annotation-unchecked\"` in pyproject.toml, which\n # doesn't exist in mypy 0.971 for Python 3.6\n \"[annotation-unchecked]\"\n]\n\n\n#: decorator for adding tools to the list\nclass tool:\n def __init__(\n self, name: str, cmd: Optional[str] = None, required: bool = False, external: bool = True\n ) -> None:\n self.name = name\n self.external = external\n self.required = required\n self.cmd = cmd if cmd else name\n\n def __call__(self, fun):\n self.fun = fun\n tools[self.name] = self\n return fun\n\n @property\n def installed(self) -> bool:\n return bool(which(self.cmd)) if self.external else True\n\n @property\n def executable(self) -> Optional[Executable]:\n return which(self.cmd) if self.external else None\n\n\n#: tools we run in spack style\ntools: Dict[str, tool] = {}\n\n\ndef changed_files(base=\"develop\", untracked=True, all_files=False, root=None) -> List[Path]:\n \"\"\"Get list of changed files in the Spack repository.\n\n Arguments:\n base (str): name of base branch to evaluate differences with.\n untracked (bool): include untracked files in the list.\n all_files (bool): list all files in the repository.\n root (str): use this directory instead of the Spack prefix.\n \"\"\"\n if root is None:\n root = spack.paths.prefix\n\n git = spack.util.git.git(required=True)\n\n # ensure base is in the repo\n base_sha = git(\n \"rev-parse\", \"--quiet\", \"--verify\", \"--revs-only\", base, fail_on_error=False, output=str\n )\n if git.returncode != 0:\n tty.die(\n \"This repository does not have a '%s' revision.\" % base,\n \"spack style needs this branch to determine which files changed.\",\n \"Ensure that '%s' exists, or specify files to check explicitly.\" % base,\n )\n\n range = \"{0}...\".format(base_sha.strip())\n\n git_args = [\n # Add changed files committed since branching off of develop\n [\"diff\", \"--name-only\", \"--diff-filter=ACMR\", range],\n # Add changed files that have been staged but not yet committed\n [\"diff\", \"--name-only\", \"--diff-filter=ACMR\", \"--cached\"],\n # Add changed files that are unstaged\n [\"diff\", \"--name-only\", \"--diff-filter=ACMR\"],\n ]\n\n # Add new files that are untracked\n if untracked:\n git_args.append([\"ls-files\", \"--exclude-standard\", \"--other\"])\n\n # add everything if the user asked for it\n if all_files:\n git_args.append([\"ls-files\", \"--exclude-standard\"])\n\n excludes = [os.path.realpath(os.path.join(root, f)) for f in exclude_paths]\n changed = set()\n\n for arg_list in git_args:\n files = git(*arg_list, output=str).split(\"\\n\")\n\n for f in files:\n # Ignore non-Python files\n if not (f.endswith(\".py\") or f == \"bin/spack\"):\n continue\n\n # Ignore files in the exclude locations\n if any(os.path.realpath(f).startswith(e) for e in excludes):\n continue\n\n changed.add(Path(f))\n\n return sorted(changed)\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n subparser.add_argument(\n \"-b\",\n \"--base\",\n action=\"store\",\n default=\"develop\",\n help=\"branch to compare against to determine changed files (default: develop)\",\n )\n subparser.add_argument(\n \"-a\",\n \"--all\",\n action=\"store_true\",\n help=\"check all files, not just changed files (applies only to Import Check)\",\n )\n subparser.add_argument(\n \"-r\",\n \"--root-relative\",\n action=\"store_true\",\n default=False,\n help=\"print root-relative paths (default: cwd-relative)\",\n )\n subparser.add_argument(\n \"-U\",\n \"--no-untracked\",\n dest=\"untracked\",\n action=\"store_false\",\n default=True,\n help=\"exclude untracked files from checks\",\n )\n subparser.add_argument(\n \"-f\",\n \"--fix\",\n action=\"store_true\",\n default=False,\n help=\"format automatically if possible (e.g., with isort, black)\",\n )\n subparser.add_argument(\n \"--root\", action=\"store\", default=None, help=\"style check a different spack instance\"\n )\n\n tool_group = subparser.add_mutually_exclusive_group()\n tool_group.add_argument(\n \"-t\",\n \"--tool\",\n action=\"append\",\n help=\"specify which tools to run (default: %s)\" % \", \".join(tool_names),\n )\n tool_group.add_argument(\n \"-s\",\n \"--skip\",\n metavar=\"TOOL\",\n action=\"append\",\n help=\"specify tools to skip (choose from %s)\" % \", \".join(tool_names),\n )\n subparser.add_argument(\n \"--spec-strings\",\n action=\"store_true\",\n help=\"upgrade spec strings in Python, JSON and YAML files for compatibility with Spack \"\n \"v1.0 and v0.x. Example: spack style ``--spec-strings $(git ls-files)``. Note: must be \"\n \"used only on specs from spack v0.X.\",\n )\n\n subparser.add_argument(\"files\", nargs=argparse.REMAINDER, help=\"specific files to check\")\n\n\ndef cwd_relative(path: Path, root: Union[Path, str], initial_working_dir: Path) -> Path:\n \"\"\"Translate prefix-relative path to current working directory-relative.\"\"\"\n if path.is_absolute():\n return path\n return Path(os.path.relpath((root / path), initial_working_dir))\n\n\ndef rewrite_and_print_output(\n output,\n root,\n working_dir,\n root_relative,\n re_obj=re.compile(r\"^(.+):([0-9]+):\"),\n replacement=r\"{0}:{1}:\",\n):\n \"\"\"rewrite output with :: format to respect path args\"\"\"\n\n # print results relative to current working directory\n def translate(match):\n return replacement.format(\n cwd_relative(Path(match.group(1)), root, working_dir), *list(match.groups()[1:])\n )\n\n for line in output.split(\"\\n\"):\n if not line:\n continue\n if any(ignore in line for ignore in mypy_ignores):\n # some mypy annotations can't be disabled in older mypys (e.g. .971, which\n # is the only mypy that supports python 3.6), so we filter them here.\n continue\n if not root_relative and re_obj:\n line = re_obj.sub(translate, line)\n print(line)\n\n\ndef print_tool_result(tool, returncode):\n if returncode == 0:\n color.cprint(\" @g{%s checks were clean}\" % tool)\n else:\n color.cprint(\" @r{%s found errors}\" % tool)\n\n\n@tool(\"ruff-check\", cmd=\"ruff\")\ndef ruff_check(file_list, args):\n \"\"\"Run the ruff-check command. Handles config and non generic ruff argument logic\"\"\"\n cmd_args = [\"--config\", os.path.join(spack.paths.prefix, \"pyproject.toml\"), \"--quiet\"]\n if args.fix:\n cmd_args += [\"--fix\", \"--no-unsafe-fixes\"]\n else:\n cmd_args += [\"--no-fix\"]\n return run_ruff(\n file_list, \"check\", cmd_args, args.root, args.initial_working_dir, args.root_relative\n )\n\n\n@tool(\"ruff-format\", cmd=\"ruff\")\ndef ruff_format(file_list, args):\n \"\"\"Run the ruff format command\"\"\"\n cmd_args = [\"--config\", os.path.join(spack.paths.prefix, \"pyproject.toml\"), \"--quiet\"]\n if not args.fix:\n cmd_args += [\"--check\", \"--diff\"]\n return run_ruff(\n file_list, \"format\", cmd_args, args.root, args.initial_working_dir, args.root_relative\n )\n\n\ndef run_ruff(\n file_list: List[Path],\n cmd: str,\n args: List[str],\n root: Path,\n working_dir: Path,\n root_relative: bool,\n):\n \"\"\"Run the ruff tool\"\"\"\n ruff_cmd = tools[f\"ruff-{cmd}\"].executable\n if not ruff_cmd:\n tty.warn(\"Cannot execute requested tool: ruff\\nCannot find tool\")\n return -1\n\n files = (str(x) for x in file_list)\n if color.get_color_when():\n args += (\"--color\", \"auto\")\n pat = re.compile(\"would reformat +(.*)\")\n replacement = \"would reformat {0}\"\n\n packed_args = (cmd,) + (*args,) + tuple(files)\n output = ruff_cmd(*packed_args, fail_on_error=False, output=str, error=str)\n returncode = ruff_cmd.returncode\n rewrite_and_print_output(output, root, working_dir, root_relative, pat, replacement)\n\n print_tool_result(f\"ruff-{cmd}\", returncode)\n return returncode\n\n\n@tool(\"mypy\")\ndef run_mypy(file_list, args):\n mypy_cmd = tools[\"mypy\"].executable\n if not mypy_cmd:\n tty.warn(\"Cannot execute requested tool: mypy\\nCannot find tool\")\n return -1\n # always run with config from running spack prefix\n common_mypy_args = [\n \"--config-file\",\n os.path.join(spack.paths.prefix, \"pyproject.toml\"),\n \"--show-error-codes\",\n ]\n mypy_arg_sets = [common_mypy_args + [\"--package\", \"spack\", \"--package\", \"llnl\"]]\n if \"SPACK_MYPY_CHECK_PACKAGES\" in os.environ:\n mypy_arg_sets.append(\n common_mypy_args + [\"--package\", \"packages\", \"--disable-error-code\", \"no-redef\"]\n )\n\n returncode = 0\n for mypy_args in mypy_arg_sets:\n output = mypy_cmd(*mypy_args, fail_on_error=False, output=str)\n returncode |= mypy_cmd.returncode\n\n rewrite_and_print_output(output, args.root, args.initial_working_dir, args.root_relative)\n\n print_tool_result(\"mypy\", returncode)\n return returncode\n\n\ndef _module_part(root: Path, expr: str):\n parts = expr.split(\".\")\n # spack.pkg is for repositories, don't try to resolve it here.\n if expr.startswith(spack.repo.PKG_MODULE_PREFIX_V1) or expr == \"spack.pkg\":\n return None\n while parts:\n f1 = (root / \"lib\" / \"spack\").joinpath(*parts).with_suffix(\".py\")\n f2 = (root / \"lib\" / \"spack\").joinpath(*parts, \"__init__.py\")\n\n if (\n f1.exists()\n # ensure case sensitive match\n and any(p.name == f\"{parts[-1]}.py\" for p in f1.parent.iterdir())\n or f2.exists()\n ):\n return \".\".join(parts)\n parts.pop()\n return None\n\n\ndef _run_import_check(\n file_list: List[Path],\n *,\n fix: bool,\n root_relative: bool,\n root: Path,\n working_dir: Path,\n out=sys.stdout,\n base=\"develop\",\n all=False,\n):\n if sys.version_info < (3, 9):\n print(\"import check requires Python 3.9 or later\")\n return 0\n\n is_use = re.compile(r\"(? List[str]:\n return [t for t in tools_to_run if not tools[t].installed]\n\n\ndef _bootstrap_dev_dependencies():\n import spack.bootstrap\n\n with spack.bootstrap.ensure_bootstrap_configuration():\n spack.bootstrap.ensure_environment_dependencies()\n\n\ndef style(parser, args):\n if args.spec_strings:\n if not args.files:\n tty.die(\"No files provided to check spec strings.\")\n handler = _spec_str_fix_handler if args.fix else _spec_str_default_handler\n return _check_spec_strings(args.files, handler)\n\n # save initial working directory for relativizing paths later\n args.initial_working_dir = Path.cwd()\n\n # ensure that the config files we need actually exist in the spack prefix.\n # assertions b/c users should not ever see these errors -- they're checked in CI.\n assert (Path(spack.paths.prefix) / \"pyproject.toml\").is_file()\n\n # validate spack root if the user provided one\n args.root = Path(args.root).resolve() if args.root else Path(spack.paths.prefix)\n spack_script = args.root / \"bin\" / \"spack\"\n if not spack_script.exists():\n tty.die(\"This does not look like a valid spack root.\", \"No such file: '%s'\" % spack_script)\n\n def prefix_relative(path: Union[Path, str]) -> Path:\n return Path(os.path.relpath(os.path.abspath(os.path.realpath(path)), args.root))\n\n file_list = [prefix_relative(file) for file in args.files]\n\n # process --tool and --skip arguments\n selected = set(tool_names)\n if args.tool is not None:\n selected = validate_toolset(args.tool)\n if args.skip is not None:\n selected -= validate_toolset(args.skip)\n\n if not selected:\n tty.msg(\"Nothing to run.\")\n return\n\n tools_to_run = [t for t in tool_names if t in selected]\n if missing_tools(tools_to_run):\n _bootstrap_dev_dependencies()\n\n return_code = 0\n with working_dir(str(args.root)):\n print_style_header(file_list, args, tools_to_run)\n for tool_name in tools_to_run:\n tool = tools[tool_name]\n tty.msg(f\"Running {tool.name} checks\")\n return_code |= tool.fun(file_list, args)\n if return_code == 0:\n tty.msg(color.colorize(\"@*{spack style checks were clean}\"))\n else:\n tty.error(color.colorize(\"@*{spack style found errors}\"))\n\n return return_code\n" }, { "path": "lib/spack/spack/cmd/tags.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport argparse\nimport io\nimport sys\nfrom typing import Dict, Iterable, List\n\nimport spack.environment\nimport spack.llnl.string\nimport spack.llnl.util.tty as tty\nimport spack.llnl.util.tty.colify as colify\nimport spack.repo\n\ndescription = \"show package tags and associated packages\"\nsection = \"query\"\nlevel = \"long\"\n\n\ndef report_tags(category, tags):\n buffer = io.StringIO()\n isatty = sys.stdout.isatty()\n\n if isatty:\n num = len(tags)\n fmt = \"{0} package tag\".format(category)\n buffer.write(\"{0}:\\n\".format(spack.llnl.string.plural(num, fmt)))\n\n if tags:\n colify.colify(tags, output=buffer, tty=isatty, indent=4)\n else:\n buffer.write(\" None\\n\")\n print(buffer.getvalue())\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n subparser.epilog = (\n \"Tags from known packages will be used if no tags are provided on \"\n \"the command\\nline. If tags are provided, packages with at least one \"\n \"will be reported.\\n\\nYou are not allowed to provide tags and use \"\n \"'--all' at the same time.\"\n )\n subparser.add_argument(\n \"-i\",\n \"--installed\",\n action=\"store_true\",\n default=False,\n help=\"show information for installed packages only\",\n )\n subparser.add_argument(\n \"-a\",\n \"--all\",\n action=\"store_true\",\n default=False,\n help=\"show packages for all available tags\",\n )\n subparser.add_argument(\"tag\", nargs=\"*\", help=\"show packages with the specified tag\")\n\n\ndef tags(parser, args):\n # Disallow combining all option with (positional) tags to avoid confusion\n if args.all and args.tag:\n tty.die(\"Use the '--all' option OR provide tag(s) on the command line\")\n\n # Provide a nice, simple message if database is empty\n if args.installed and not spack.environment.installed_specs():\n tty.msg(\"No installed packages\")\n return\n\n # unique list of available tags\n available_tags = sorted(spack.repo.PATH.tag_index.tags)\n if not available_tags:\n tty.msg(\"No tagged packages\")\n return\n\n show_packages = args.tag or args.all\n\n # Only report relevant, available tags if no packages are to be shown\n if not show_packages:\n if not args.installed:\n report_tags(\"available\", available_tags)\n else:\n tag_pkgs = packages_with_tags(available_tags, True, True)\n tags = tag_pkgs.keys() if tag_pkgs else []\n report_tags(\"installed\", tags)\n return\n\n # Report packages associated with tags\n buffer = io.StringIO()\n isatty = sys.stdout.isatty()\n\n tags = args.tag if args.tag else available_tags\n tag_pkgs = packages_with_tags(tags, args.installed, False)\n missing = \"No installed packages\" if args.installed else \"None\"\n for tag in sorted(tag_pkgs):\n # TODO: Remove the sorting once we're sure no one has an old\n # TODO: tag cache since it can accumulate duplicates.\n packages = sorted(list(set(tag_pkgs[tag])))\n if isatty:\n buffer.write(\"{0}:\\n\".format(tag))\n\n if packages:\n colify.colify(packages, output=buffer, tty=isatty, indent=4)\n else:\n buffer.write(\" {0}\\n\".format(missing))\n buffer.write(\"\\n\")\n print(buffer.getvalue())\n\n\ndef packages_with_tags(\n tags: Iterable[str], installed: bool, skip_empty: bool\n) -> Dict[str, List[str]]:\n \"\"\"\n Returns a dict, indexed by tag, containing lists of names of packages\n containing the tag or, if no tags, for all available tags.\n\n Arguments:\n tags: list of tags of interest or None for all\n installed: True if want names of packages that are installed;\n otherwise, False if want all packages with the tag\n skip_empty: True if exclude tags with no associated packages;\n otherwise, False if want entries for all tags even when no such\n tagged packages\n \"\"\"\n tag_pkgs: Dict[str, List[str]] = {}\n name_filter = {x.name for x in spack.environment.installed_specs()} if installed else None\n for tag in tags:\n packages = spack.repo.PATH.tag_index.get_packages(tag)\n if name_filter is not None:\n packages = [p for p in packages if p in name_filter]\n if packages or not skip_empty:\n tag_pkgs[tag] = packages\n return tag_pkgs\n" }, { "path": "lib/spack/spack/cmd/test.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport fnmatch\nimport os\nimport re\nimport shutil\nimport sys\nfrom collections import Counter\n\nimport spack.cmd\nimport spack.config\nimport spack.environment as ev\nimport spack.install_test\nimport spack.repo\nimport spack.store\nfrom spack.cmd.common import arguments\nfrom spack.llnl.util import tty\nfrom spack.llnl.util.tty import colify\n\nfrom . import doc_dedented, doc_first_line\n\ndescription = \"run spack's tests for an install\"\nsection = \"build\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n sp = subparser.add_subparsers(metavar=\"SUBCOMMAND\", dest=\"test_command\")\n\n # Run\n run_parser = sp.add_parser(\n \"run\", description=doc_dedented(test_run), help=doc_first_line(test_run)\n )\n\n run_parser.add_argument(\n \"--alias\", help=\"provide an alias for this test-suite for subsequent access\"\n )\n\n run_parser.add_argument(\n \"--fail-fast\",\n action=\"store_true\",\n help=\"stop tests for each package after the first failure\",\n )\n run_parser.add_argument(\n \"--fail-first\", action=\"store_true\", help=\"stop after the first failed package\"\n )\n run_parser.add_argument(\n \"--externals\", action=\"store_true\", help=\"test packages that are externally installed\"\n )\n run_parser.add_argument(\n \"-x\",\n \"--explicit\",\n action=\"store_true\",\n help=\"only test packages that are explicitly installed\",\n )\n run_parser.add_argument(\n \"--keep-stage\", action=\"store_true\", help=\"keep testing directory for debugging\"\n )\n arguments.add_common_arguments(run_parser, [\"log_format\"])\n run_parser.add_argument(\"--log-file\", default=None, help=\"filename for the log file\")\n arguments.add_cdash_args(run_parser, False)\n run_parser.add_argument(\n \"--help-cdash\", action=\"store_true\", help=\"show usage instructions for CDash reporting\"\n )\n run_parser.add_argument(\n \"--timeout\",\n type=int,\n default=None,\n help=\"maximum time (in seconds) that tests are allowed to run\",\n )\n\n cd_group = run_parser.add_mutually_exclusive_group()\n arguments.add_common_arguments(cd_group, [\"clean\", \"dirty\"])\n\n arguments.add_common_arguments(run_parser, [\"installed_specs\"])\n\n # List\n list_parser = sp.add_parser(\n \"list\", description=doc_dedented(test_list), help=doc_first_line(test_list)\n )\n list_parser.add_argument(\n \"-a\",\n \"--all\",\n action=\"store_true\",\n dest=\"list_all\",\n help=\"list all packages with tests (not just installed)\",\n )\n\n list_parser.add_argument(\"tag\", nargs=\"*\", help=\"limit packages to those with all listed tags\")\n\n # Find\n find_parser = sp.add_parser(\n \"find\", description=doc_dedented(test_find), help=doc_first_line(test_find)\n )\n find_parser.add_argument(\n \"filter\",\n nargs=argparse.REMAINDER,\n help=\"optional case-insensitive glob patterns to filter results\",\n )\n\n # Status\n status_parser = sp.add_parser(\n \"status\", description=doc_dedented(test_status), help=doc_first_line(test_status)\n )\n status_parser.add_argument(\n \"names\", nargs=argparse.REMAINDER, help=\"test suites for which to print status\"\n )\n\n # Results\n results_parser = sp.add_parser(\n \"results\", description=doc_dedented(test_results), help=doc_first_line(test_results)\n )\n results_parser.add_argument(\n \"-l\", \"--logs\", action=\"store_true\", help=\"print the test log for each matching package\"\n )\n results_parser.add_argument(\n \"-f\",\n \"--failed\",\n action=\"store_true\",\n help=\"only show results for failed tests of matching packages\",\n )\n results_parser.add_argument(\n \"names\",\n nargs=argparse.REMAINDER,\n metavar=\"[name(s)] [-- installed_specs]...\",\n help=\"suite names and installed package constraints\",\n )\n results_parser.epilog = (\n \"Test results will be filtered by space-\"\n \"separated suite name(s) and installed\\nspecs when provided. \"\n \"If names are provided, then only results for those test\\nsuites \"\n \"will be shown. If installed specs are provided, then only results\"\n \"\\nmatching those specs will be shown.\"\n )\n\n # Remove\n remove_parser = sp.add_parser(\n \"remove\", description=doc_dedented(test_remove), help=doc_first_line(test_remove)\n )\n arguments.add_common_arguments(remove_parser, [\"yes_to_all\"])\n remove_parser.add_argument(\n \"names\", nargs=argparse.REMAINDER, help=\"test suites to remove from test stage\"\n )\n\n\ndef test_run(args):\n \"\"\"\\\n run tests for the specified installed packages\n\n if no specs are listed, run tests for all packages in the current\n environment or all installed packages if there is no active environment\n \"\"\"\n if args.alias:\n suites = spack.install_test.get_named_test_suites(args.alias)\n if suites:\n tty.die('Test suite \"{0}\" already exists. Try another alias.'.format(args.alias))\n\n # cdash help option\n if args.help_cdash:\n arguments.print_cdash_help()\n return\n\n arguments.sanitize_reporter_options(args)\n\n # set config option for fail-fast\n if args.fail_fast:\n spack.config.set(\"config:fail_fast\", True, scope=\"command_line\")\n\n explicit = args.explicit or None\n explicit_str = \"explicitly \" if args.explicit else \"\"\n\n # Get specs to test\n env = ev.active_environment()\n hashes = env.all_hashes() if env else None\n\n specs = spack.cmd.parse_specs(args.specs) if args.specs else [None]\n specs_to_test = []\n for spec in specs:\n matching = spack.store.STORE.db.query_local(spec, hashes=hashes, explicit=explicit)\n if spec and not matching:\n tty.warn(f\"No {explicit_str}installed packages match spec {spec}\")\n\n # TODO: Need to write out a log message and/or CDASH Testing\n # output that package not installed IF continue to process\n # these issues here.\n\n # if args.log_format:\n # # Proceed with the spec assuming the test process\n # # to ensure report package as skipped (e.g., for CI)\n # specs_to_test.append(spec)\n\n specs_to_test.extend(matching)\n\n # test_stage_dir\n test_suite = spack.install_test.TestSuite(specs_to_test, args.alias)\n test_suite.ensure_stage()\n tty.msg(f\"Spack test {test_suite.name}\")\n\n # Set up reporter\n reporter = args.reporter() if args.log_format else None\n try:\n test_suite(\n remove_directory=not args.keep_stage,\n dirty=args.dirty,\n fail_first=args.fail_first,\n externals=args.externals,\n timeout=args.timeout,\n )\n finally:\n if reporter:\n report_file = report_filename(args, test_suite)\n reporter.test_report(report_file, test_suite.reports)\n\n\ndef report_filename(args, test_suite):\n return os.path.abspath(args.log_file or \"test-{}\".format(test_suite.name))\n\n\ndef test_list(args):\n \"\"\"list installed packages with available tests\"\"\"\n tagged = spack.repo.PATH.packages_with_tags(*args.tag) if args.tag else set()\n\n def has_test_and_tags(pkg_class):\n tests = spack.install_test.test_functions(pkg_class)\n return len(tests) and (not args.tag or pkg_class.name in tagged)\n\n if args.list_all:\n report_packages = [\n pkg_class.name\n for pkg_class in spack.repo.PATH.all_package_classes()\n if has_test_and_tags(pkg_class)\n ]\n\n if sys.stdout.isatty():\n filtered = \" tagged\" if args.tag else \"\"\n tty.msg(\"{0}{1} packages with tests.\".format(len(report_packages), filtered))\n colify.colify(report_packages)\n return\n\n # TODO: This can be extended to have all of the output formatting options\n # from `spack find`.\n env = ev.active_environment()\n hashes = env.all_hashes() if env else None\n\n specs = spack.store.STORE.db.query(hashes=hashes)\n specs = list(\n filter(lambda s: has_test_and_tags(spack.repo.PATH.get_pkg_class(s.fullname)), specs)\n )\n\n spack.cmd.display_specs(specs, long=True)\n\n\ndef test_find(args): # TODO: merge with status (noargs)\n \"\"\"\\\n find tests that are running or have available results\n\n displays aliases for tests that have them, otherwise test suite content hashes\n \"\"\"\n test_suites = spack.install_test.get_all_test_suites()\n\n # Filter tests by filter argument\n if args.filter:\n\n def create_filter(f):\n raw = fnmatch.translate(\"f\" if \"*\" in f or \"?\" in f else \"*\" + f + \"*\")\n return re.compile(raw, flags=re.IGNORECASE)\n\n filters = [create_filter(f) for f in args.filter]\n\n def match(t, f):\n return f.match(t)\n\n test_suites = [\n t\n for t in test_suites\n if any(match(t.alias, f) for f in filters) and os.path.isdir(t.stage)\n ]\n\n names = [t.name for t in test_suites]\n\n if names:\n # TODO: Make these specify results vs active\n msg = \"Spack test results available for the following tests:\\n\"\n msg += \" %s\\n\" % \" \".join(names)\n msg += \" Run `spack test remove` to remove all tests\"\n tty.msg(msg)\n else:\n msg = \"No test results match the query\\n\"\n msg += \" Tests may have been removed using `spack test remove`\"\n tty.msg(msg)\n\n\ndef test_status(args):\n \"\"\"get the current status for the specified Spack test suite(s)\"\"\"\n if args.names:\n test_suites = []\n for name in args.names:\n test_suite = spack.install_test.get_test_suite(name)\n if test_suite:\n test_suites.append(test_suite)\n else:\n tty.msg(\"No test suite %s found in test stage\" % name)\n else:\n test_suites = spack.install_test.get_all_test_suites()\n if not test_suites:\n tty.msg(\"No test suites with status to report\")\n\n for test_suite in test_suites:\n # TODO: Make this handle capability tests too\n # TODO: Make this handle tests running in another process\n tty.msg(\"Test suite %s completed\" % test_suite.name)\n\n\ndef _report_suite_results(test_suite, args, constraints):\n \"\"\"Report the relevant test suite results.\"\"\"\n\n # TODO: Make this handle capability tests too\n # The results file may turn out to be a placeholder for future work\n\n if constraints:\n # TBD: Should I be refactoring or re-using ConstraintAction?\n qspecs = spack.cmd.parse_specs(constraints)\n specs = {}\n for spec in qspecs:\n for s in spack.store.STORE.db.query(spec, installed=True):\n specs[s.dag_hash()] = s\n specs = sorted(specs.values())\n test_specs = dict((test_suite.test_pkg_id(s), s) for s in test_suite.specs if s in specs)\n else:\n test_specs = dict((test_suite.test_pkg_id(s), s) for s in test_suite.specs)\n\n if not test_specs:\n return\n\n if os.path.exists(test_suite.results_file):\n results_desc = \"Failing results\" if args.failed else \"Results\"\n matching = \", spec matching '{0}'\".format(\" \".join(constraints)) if constraints else \"\"\n tty.msg(\"{0} for test suite '{1}'{2}:\".format(results_desc, test_suite.name, matching))\n\n results = {}\n with open(test_suite.results_file, \"r\", encoding=\"utf-8\") as f:\n for line in f:\n pkg_id, status = line.split()\n results[pkg_id] = status\n\n tty.msg(\"test specs:\")\n\n counts = Counter()\n for pkg_id in test_specs:\n if pkg_id in results:\n status = results[pkg_id]\n # Backward-compatibility: NO-TESTS => NO_TESTS\n status = \"NO_TESTS\" if status == \"NO-TESTS\" else status\n\n status = spack.install_test.TestStatus[status]\n counts[status] += 1\n\n if args.failed and status != spack.install_test.TestStatus.FAILED:\n continue\n\n msg = \" {0} {1}\".format(pkg_id, status)\n if args.logs:\n spec = test_specs[pkg_id]\n log_file = test_suite.log_file_for_spec(spec)\n if os.path.isfile(log_file):\n with open(log_file, \"r\", encoding=\"utf-8\") as f:\n msg += \"\\n{0}\".format(\"\".join(f.readlines()))\n tty.msg(msg)\n\n spack.install_test.write_test_summary(counts)\n else:\n msg = \"Test %s has no results.\\n\" % test_suite.name\n msg += \" Check if it is running with \"\n msg += \"`spack test status %s`\" % test_suite.name\n tty.msg(msg)\n\n\ndef test_results(args):\n \"\"\"get the results from Spack test suite(s) (default all)\"\"\"\n if args.names:\n try:\n sep_index = args.names.index(\"--\")\n names = args.names[:sep_index]\n constraints = args.names[sep_index + 1 :]\n except ValueError:\n names = args.names\n constraints = None\n else:\n names, constraints = None, None\n\n if names:\n test_suites = [spack.install_test.get_test_suite(name) for name in names]\n test_suites = list(filter(lambda ts: ts is not None, test_suites))\n if not test_suites:\n tty.msg(\"No test suite(s) found in test stage: {0}\".format(\", \".join(names)))\n else:\n test_suites = spack.install_test.get_all_test_suites()\n if not test_suites:\n tty.msg(\"No test suites with results to report\")\n\n for test_suite in test_suites:\n _report_suite_results(test_suite, args, constraints)\n\n\ndef test_remove(args):\n \"\"\"\\\n remove results from Spack test suite(s) (default all)\n\n if no test suite is listed, remove results for all suites.\n\n removed tests can no longer be accessed for results or status, and will not\n appear in ``spack test list`` results\n \"\"\"\n if args.names:\n test_suites = []\n for name in args.names:\n test_suite = spack.install_test.get_test_suite(name)\n if test_suite:\n test_suites.append(test_suite)\n else:\n tty.msg(\"No test suite %s found in test stage\" % name)\n else:\n test_suites = spack.install_test.get_all_test_suites()\n\n if not test_suites:\n tty.msg(\"No test suites to remove\")\n return\n\n if not args.yes_to_all:\n msg = \"The following test suites will be removed:\\n\\n\"\n msg += \" \" + \" \".join(test.name for test in test_suites) + \"\\n\"\n tty.msg(msg)\n answer = tty.get_yes_or_no(\"Do you want to proceed?\", default=False)\n if not answer:\n tty.msg(\"Aborting removal of test suites\")\n return\n\n for test_suite in test_suites:\n shutil.rmtree(test_suite.stage)\n\n\ndef test(parser, args):\n globals()[\"test_%s\" % args.test_command](args)\n" }, { "path": "lib/spack/spack/cmd/test_env.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport spack.cmd.common.env_utility as env_utility\nfrom spack.context import Context\n\ndescription = (\n \"run a command in a spec's test environment,\\nor dump its environment to screen or file\"\n)\nsection = \"developer\"\nlevel = \"long\"\n\nsetup_parser = env_utility.setup_parser\n\n\ndef test_env(parser, args):\n env_utility.emulate_env_utility(\"test-env\", Context.TEST, args)\n" }, { "path": "lib/spack/spack/cmd/tutorial.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport os\nimport shutil\n\nimport spack\nimport spack.cmd\nimport spack.config\nimport spack.llnl.util.tty as tty\nimport spack.paths\nimport spack.util.git\nimport spack.util.gpg\nfrom spack.cmd.common import arguments\nfrom spack.llnl.util.filesystem import working_dir\nfrom spack.util.spack_yaml import syaml_dict\n\ndescription = \"set up spack for our tutorial (WARNING: modifies config!)\"\nsection = \"config\"\nlevel = \"long\"\n\n\n# tutorial configuration parameters\ntutorial_branch = \"releases/v1.1\"\ntutorial_mirror = \"file:///mirror\"\ntutorial_key = os.path.join(spack.paths.share_path, \"keys\", \"tutorial.pub\")\n\n# configs to remove\nrm_configs = [\n \"~/.spack/linux/compilers.yaml\",\n \"~/.spack/packages.yaml\",\n \"~/.spack/mirrors.yaml\",\n \"~/.spack/modules.yaml\",\n \"~/.spack/config.yaml\",\n]\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n arguments.add_common_arguments(subparser, [\"yes_to_all\"])\n\n\ndef tutorial(parser, args):\n if not spack.cmd.spack_is_git_repo():\n tty.die(\"This command requires a git installation of Spack!\")\n\n if not args.yes_to_all:\n tty.msg(\n \"This command will set up Spack for the tutorial at \"\n \"https://spack-tutorial.readthedocs.io.\",\n \"\",\n )\n tty.warn(\n \"This will modify your Spack configuration by:\",\n \" - deleting some configuration in ~/.spack\",\n \" - adding a mirror and trusting its public key\",\n \" - checking out a particular branch of Spack\",\n \"\",\n )\n if not tty.get_yes_or_no(\"Are you sure you want to proceed?\"):\n tty.die(\"Aborted\")\n\n rm_cmds = [f\"rm -f {f}\" for f in rm_configs]\n tty.msg(\"Reverting compiler and repository configuration\", *rm_cmds)\n for path in rm_configs:\n if os.path.exists(path):\n shutil.rmtree(path, ignore_errors=True)\n\n tty.msg(\n \"Ensuring that the tutorial binary mirror is configured:\",\n f\"spack mirror add tutorial {tutorial_mirror}\",\n )\n mirror_config = syaml_dict()\n mirror_config[\"tutorial\"] = tutorial_mirror\n spack.config.set(\"mirrors\", mirror_config, scope=\"user\")\n\n tty.msg(\"Ensuring that we trust tutorial binaries\", f\"spack gpg trust {tutorial_key}\")\n spack.util.gpg.trust(tutorial_key)\n\n # Note that checkout MUST be last. It changes Spack under our feet.\n # If you don't put this last, you'll get import errors for the code\n # that follows (exacerbated by the various lazy singletons we use)\n tty.msg(f\"Ensuring we're on the {tutorial_branch} branch\")\n git = spack.util.git.git(required=True)\n with working_dir(spack.paths.prefix):\n git(\"checkout\", tutorial_branch)\n # NO CODE BEYOND HERE\n" }, { "path": "lib/spack/spack/cmd/undevelop.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\n\nimport spack.cmd\nimport spack.config\nimport spack.llnl.util.tty as tty\nimport spack.spec\nfrom spack.cmd.common import arguments\n\ndescription = \"remove specs from an environment\"\nsection = \"environments\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n subparser.add_argument(\n \"--no-modify-concrete-specs\",\n action=\"store_false\",\n dest=\"apply_changes\",\n help=(\n \"do not mutate concrete specs to remove dev_path provenance.\"\n \" This requires running `spack concretize -f` later to apply changes to concrete specs\"\n ),\n )\n\n subparser.add_argument(\n \"-a\", \"--all\", action=\"store_true\", help=\"remove all specs from (clear) the environment\"\n )\n\n arguments.add_common_arguments(subparser, [\"specs\"])\n\n\ndef _update_config(specs_to_remove):\n def change_fn(dev_config):\n modified = False\n for spec in specs_to_remove:\n if spec.name in dev_config:\n tty.msg(\"Undevelop: removing {0}\".format(spec.name))\n del dev_config[spec.name]\n modified = True\n return modified\n\n spack.config.update_all(\"develop\", change_fn)\n\n\ndef undevelop(parser, args):\n # TODO: when https://github.com/spack/spack/pull/35307 is merged,\n # an active env is not required if a scope is specified\n env = spack.cmd.require_active_env(cmd_name=\"undevelop\")\n\n if args.all:\n remove_specs = [spack.spec.Spec(s) for s in env.dev_specs]\n else:\n remove_specs = spack.cmd.parse_specs(args.specs)\n\n with env.write_transaction():\n _update_config(remove_specs)\n if args.apply_changes:\n for spec in remove_specs:\n env.apply_develop(spec, path=None)\n\n updated_all_dev_specs = set(spack.config.get(\"develop\"))\n\n remove_spec_names = set(x.name for x in remove_specs)\n not_fully_removed = updated_all_dev_specs & remove_spec_names\n\n if not_fully_removed:\n tty.msg(\n \"The following specs could not be removed as develop specs\"\n \" - see `spack config blame develop` to locate files requiring\"\n f\" manual edits: {', '.join(not_fully_removed)}\"\n )\n" }, { "path": "lib/spack/spack/cmd/uninstall.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport sys\nfrom typing import Dict, List, Optional\n\nimport spack.cmd\nimport spack.cmd.common.confirmation as confirmation\nimport spack.environment as ev\nimport spack.package_base\nimport spack.spec\nimport spack.store\nimport spack.traverse as traverse\nfrom spack.cmd.common import arguments\nfrom spack.llnl.util import tty\nfrom spack.llnl.util.tty.colify import colify\n\nfrom ..enums import InstallRecordStatus\n\ndescription = \"remove installed packages\"\nsection = \"build\"\nlevel = \"short\"\n\nerror_message = \"\"\"You can either:\n a) use a more specific spec, or\n b) specify the spec by its hash (e.g. `spack uninstall /hash`), or\n c) use `spack uninstall --all` to uninstall ALL matching specs.\n\"\"\"\n\n# Arguments for display_specs when we find ambiguity\ndisplay_args = {\"long\": True, \"show_flags\": False, \"variants\": False, \"indent\": 4}\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n epilog_msg = (\n \"Specs to be uninstalled are specified using the spec syntax\"\n \" (`spack help --spec`) and can be identified by their \"\n \"hashes. To remove packages that are needed only at build \"\n \"time and were not explicitly installed see `spack gc -h`.\"\n \"\\n\\nWhen using the --all option ALL packages matching the \"\n \"supplied specs will be uninstalled. For instance, \"\n \"`spack uninstall --all libelf` uninstalls all the versions \"\n \"of `libelf` currently present in Spack's store. If no spec \"\n \"is supplied, all installed packages will be uninstalled. \"\n \"If used in an environment, all packages in the environment \"\n \"will be uninstalled.\"\n )\n subparser.epilog = epilog_msg\n subparser.add_argument(\n \"-f\",\n \"--force\",\n action=\"store_true\",\n dest=\"force\",\n help=\"remove regardless of whether other packages or environments depend on this one\",\n )\n subparser.add_argument(\n \"--remove\",\n action=\"store_true\",\n dest=\"remove\",\n help=\"if in an environment, then the spec should also be removed from \"\n \"the environment description\",\n )\n arguments.add_common_arguments(\n subparser, [\"recurse_dependents\", \"yes_to_all\", \"installed_specs\"]\n )\n subparser.add_argument(\n \"-a\",\n \"--all\",\n action=\"store_true\",\n dest=\"all\",\n help=\"remove ALL installed packages that match each supplied spec\",\n )\n subparser.add_argument(\n \"--origin\", dest=\"origin\", help=\"only remove DB records with the specified origin\"\n )\n\n\ndef find_matching_specs(\n env: Optional[ev.Environment],\n specs: List[spack.spec.Spec],\n allow_multiple_matches: bool = False,\n origin=None,\n) -> List[spack.spec.Spec]:\n \"\"\"Returns a list of specs matching the not necessarily concretized specs given from cli\n\n Args:\n env: optional active environment\n specs: list of specs to be matched against installed packages\n allow_multiple_matches: if True multiple matches are admitted\n origin: origin of the spec\n \"\"\"\n # constrain uninstall resolution to current environment if one is active\n hashes = env.all_hashes() if env else None\n\n # List of specs that match expressions given via command line\n specs_from_cli: List[spack.spec.Spec] = []\n has_errors = False\n for spec in specs:\n matching = spack.store.STORE.db.query_local(\n spec,\n hashes=hashes,\n installed=(InstallRecordStatus.INSTALLED | InstallRecordStatus.DEPRECATED),\n origin=origin,\n )\n # For each spec provided, make sure it refers to only one package.\n # Fail and ask user to be unambiguous if it doesn't\n if not allow_multiple_matches and len(matching) > 1:\n tty.error(\"{0} matches multiple packages:\".format(spec))\n sys.stderr.write(\"\\n\")\n spack.cmd.display_specs(matching, output=sys.stderr, **display_args)\n sys.stderr.write(\"\\n\")\n sys.stderr.flush()\n has_errors = True\n\n # No installed package matches the query\n if len(matching) == 0 and spec is not None:\n if env:\n pkg_type = \"packages in environment '%s'\" % env.name\n else:\n pkg_type = \"installed packages\"\n tty.die(\"{0} does not match any {1}.\".format(spec, pkg_type))\n\n specs_from_cli.extend(matching)\n\n if has_errors:\n tty.die(error_message)\n\n return specs_from_cli\n\n\ndef installed_dependents(specs: List[spack.spec.Spec]) -> List[spack.spec.Spec]:\n # Note: the combination of arguments (in particular order=breadth\n # and root=False) ensures dependents and matching_specs are non-overlapping;\n # In the extreme case of \"spack uninstall --all\" we get the entire database as\n # input; in that case we return an empty list.\n\n def is_installed(spec):\n record = spack.store.STORE.db.query_local_by_spec_hash(spec.dag_hash())\n return record and record.installed\n\n all_specs = traverse.traverse_nodes(\n specs,\n root=False,\n order=\"breadth\",\n cover=\"nodes\",\n deptype=(\"link\", \"run\"),\n direction=\"parents\",\n key=lambda s: s.dag_hash(),\n )\n\n with spack.store.STORE.db.read_transaction():\n return [spec for spec in all_specs if is_installed(spec)]\n\n\ndef dependent_environments(\n specs: List[spack.spec.Spec], current_env: Optional[ev.Environment] = None\n) -> Dict[ev.Environment, List[spack.spec.Spec]]:\n # For each tracked environment, get the specs we would uninstall from it.\n # Don't instantiate current environment twice.\n env_names = ev.all_environment_names()\n if current_env:\n env_names = (name for name in env_names if name != current_env.name)\n\n # Mapping from Environment -> non-zero list of specs contained in it.\n other_envs_to_specs: Dict[ev.Environment, List[spack.spec.Spec]] = {}\n for other_env in (ev.Environment(ev.root(name)) for name in env_names):\n specs_in_other_env = all_specs_in_env(other_env, specs)\n if specs_in_other_env:\n other_envs_to_specs[other_env] = specs_in_other_env\n\n return other_envs_to_specs\n\n\ndef all_specs_in_env(env: ev.Environment, specs: List[spack.spec.Spec]) -> List[spack.spec.Spec]:\n \"\"\"Given a list of specs, return those that are in the env\"\"\"\n hashes = set(env.all_hashes())\n return [s for s in specs if s.dag_hash() in hashes]\n\n\ndef _remove_from_env(spec, env):\n \"\"\"Remove a spec from an environment if it is a root.\"\"\"\n try:\n # try removing the spec from the current active\n # environment. this will fail if the spec is not a root\n env.remove(spec, force=True)\n except ev.SpackEnvironmentError:\n pass # ignore non-root specs\n\n\ndef do_uninstall(specs: List[spack.spec.Spec], force: bool = False):\n # TODO: get rid of the call-sites that use this function,\n # so that we don't have to do a dance of list -> set -> list -> set\n hashes_to_remove = set(s.dag_hash() for s in specs)\n\n for s in traverse.traverse_nodes(\n specs, order=\"topo\", direction=\"children\", root=True, cover=\"nodes\", deptype=\"all\"\n ):\n if s.dag_hash() in hashes_to_remove:\n spack.package_base.PackageBase.uninstall_by_spec(s, force=force)\n\n\ndef get_uninstall_list(args, specs: List[spack.spec.Spec], env: Optional[ev.Environment]):\n \"\"\"Returns unordered uninstall_list and remove_list: these may overlap (some things\n may be both uninstalled and removed from the current environment).\n\n It is assumed we are in an environment if ``--remove`` is specified (this\n method raises an exception otherwise).\"\"\"\n if args.remove and not env:\n raise ValueError(\"Can only use --remove when in an environment\")\n\n # Gets the list of installed specs that match the ones given via cli\n # args.all takes care of the case where '-a' is given in the cli\n matching_specs = find_matching_specs(env, specs, args.all, origin=args.origin)\n dependent_specs = installed_dependents(matching_specs)\n all_uninstall_specs = matching_specs + dependent_specs if args.dependents else matching_specs\n other_dependent_envs = dependent_environments(all_uninstall_specs, current_env=env)\n\n # There are dependents and we didn't ask to remove dependents\n dangling_dependents = dependent_specs and not args.dependents\n\n # An environment different than the current env depends on\n # one or more of the list of all specs to be uninstalled.\n dangling_environments = not args.remove and other_dependent_envs\n\n has_error = not args.force and (dangling_dependents or dangling_environments)\n\n if has_error:\n msgs = []\n tty.info(\"Refusing to uninstall the following specs\")\n spack.cmd.display_specs(matching_specs, **display_args)\n if dangling_dependents:\n print()\n tty.info(\"The following dependents are still installed:\")\n spack.cmd.display_specs(dependent_specs, **display_args)\n msgs.append(\"use `spack uninstall --dependents` to remove dependents too\")\n if dangling_environments:\n print()\n tty.info(\"The following environments still reference these specs:\")\n colify([e.name for e in other_dependent_envs.keys()], indent=4)\n if env:\n msgs.append(\"use `spack remove` to remove the spec from the current environment\")\n msgs.append(\"use `spack env remove` to remove environments\")\n msgs.append(\"use `spack uninstall --force` to override\")\n print()\n tty.die(\"There are still dependents.\", *msgs)\n\n # If we are in an environment, this will track specs in this environment\n # which should only be removed from the environment rather than uninstalled\n remove_only = []\n if args.remove and not args.force:\n for specs_in_other_env in other_dependent_envs.values():\n remove_only.extend(specs_in_other_env)\n\n if remove_only:\n tty.info(\n \"The following specs will be removed but not uninstalled because\"\n \" they are also used by another environment: {speclist}\".format(\n speclist=\", \".join(x.name for x in remove_only)\n )\n )\n\n # Compute the set of specs that should be removed from the current env.\n # This may overlap (some specs may be uninstalled and also removed from\n # the current environment).\n remove_specs = all_specs_in_env(env, all_uninstall_specs) if env and args.remove else []\n\n return list(set(all_uninstall_specs) - set(remove_only)), remove_specs\n\n\ndef uninstall_specs(args, specs):\n env = ev.active_environment()\n\n uninstall_list, remove_list = get_uninstall_list(args, specs, env)\n\n if not uninstall_list:\n tty.warn(\"There are no package to uninstall.\")\n return\n\n if not args.yes_to_all:\n confirmation.confirm_action(uninstall_list, \"uninstalled\", \"uninstall\")\n\n # Uninstall everything on the list\n do_uninstall(uninstall_list, args.force)\n\n if env:\n with env.write_transaction():\n for spec in remove_list:\n _remove_from_env(spec, env)\n env.write()\n\n env.regenerate_views()\n\n\ndef uninstall(parser, args):\n if not args.specs and not args.all:\n tty.die(\n \"uninstall requires at least one package argument.\",\n \" Use `spack uninstall --all` to uninstall ALL packages.\",\n )\n\n # [None] here handles the --all case by forcing all specs to be returned\n specs = spack.cmd.parse_specs(args.specs) if args.specs else [None]\n uninstall_specs(args, specs)\n" }, { "path": "lib/spack/spack/cmd/unit_test.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport collections\nimport io\nimport os\nimport re\nimport sys\n\nimport spack.extensions\n\ntry:\n import pytest\nexcept ImportError:\n pytest = None # type: ignore\n\nimport spack.llnl.util.filesystem\nimport spack.llnl.util.tty as tty\nimport spack.llnl.util.tty.color as color\nimport spack.paths\nfrom spack.llnl.util.tty.colify import colify\n\ndescription = \"run spack's unit tests (wrapper around pytest)\"\nsection = \"developer\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n subparser.add_argument(\n \"-H\",\n \"--pytest-help\",\n action=\"store_true\",\n default=False,\n help=\"show full pytest help, with advanced options\",\n )\n subparser.add_argument(\n \"-n\",\n \"--numprocesses\",\n type=int,\n default=1,\n help=\"run tests in parallel up to this wide, default 1 for sequential\",\n )\n\n # extra spack arguments to list tests\n list_group = subparser.add_argument_group(\"listing tests\")\n list_mutex = list_group.add_mutually_exclusive_group()\n list_mutex.add_argument(\n \"-l\",\n \"--list\",\n action=\"store_const\",\n default=None,\n dest=\"list\",\n const=\"list\",\n help=\"list test filenames\",\n )\n list_mutex.add_argument(\n \"-L\",\n \"--list-long\",\n action=\"store_const\",\n default=None,\n dest=\"list\",\n const=\"long\",\n help=\"list all test functions\",\n )\n list_mutex.add_argument(\n \"-N\",\n \"--list-names\",\n action=\"store_const\",\n default=None,\n dest=\"list\",\n const=\"names\",\n help=\"list full names of all tests\",\n )\n\n # use tests for extension\n subparser.add_argument(\n \"--extension\", default=None, help=\"run test for a given spack extension\"\n )\n\n # spell out some common pytest arguments, so they'll show up in help\n pytest_group = subparser.add_argument_group(\n \"common pytest arguments (spack unit-test --pytest-help for more)\"\n )\n pytest_group.add_argument(\n \"-s\",\n action=\"append_const\",\n dest=\"parsed_args\",\n const=\"-s\",\n help=\"print output while tests run (disable capture)\",\n )\n pytest_group.add_argument(\n \"-k\",\n action=\"store\",\n metavar=\"EXPRESSION\",\n dest=\"expression\",\n help=\"filter tests by keyword (can also use w/list options)\",\n )\n pytest_group.add_argument(\n \"--showlocals\",\n action=\"append_const\",\n dest=\"parsed_args\",\n const=\"--showlocals\",\n help=\"show local variable values in tracebacks\",\n )\n\n # remainder is just passed to pytest\n subparser.add_argument(\"pytest_args\", nargs=argparse.REMAINDER, help=\"arguments for pytest\")\n\n\ndef do_list(args, extra_args):\n \"\"\"Print a lists of tests than what pytest offers.\"\"\"\n\n def colorize(c, prefix):\n if isinstance(prefix, tuple):\n return \"::\".join(color.colorize(\"@%s{%s}\" % (c, p)) for p in prefix if p != \"()\")\n return color.colorize(\"@%s{%s}\" % (c, prefix))\n\n # To list the files we just need to inspect the filesystem,\n # which doesn't need to wait for pytest collection and doesn't\n # require parsing pytest output\n files = spack.llnl.util.filesystem.find(\n root=spack.paths.test_path, files=\"*.py\", recursive=True\n )\n files = [\n os.path.relpath(f, start=spack.paths.spack_root)\n for f in files\n if not f.endswith((\"conftest.py\", \"__init__.py\"))\n ]\n\n old_output = sys.stdout\n try:\n sys.stdout = output = io.StringIO()\n pytest.main([\"--collect-only\"] + extra_args)\n finally:\n sys.stdout = old_output\n\n lines = output.getvalue().split(\"\\n\")\n tests = collections.defaultdict(set)\n\n # collect tests into sections\n node_regexp = re.compile(r\"(\\s*)<([^ ]*) ['\\\"]?([^']*)['\\\"]?>\")\n key_parts, name_parts = [], []\n for line in lines:\n match = node_regexp.match(line)\n if not match:\n continue\n indent, nodetype, name = match.groups()\n\n # strip parametrized tests\n if \"[\" in name:\n name = name[: name.index(\"[\")]\n\n len_indent = len(indent)\n if os.path.isabs(name):\n name = os.path.relpath(name, start=spack.paths.spack_root)\n\n item = (len_indent, name, nodetype)\n\n # Reduce the parts to the scopes that are of interest\n name_parts = [x for x in name_parts if x[0] < len_indent]\n key_parts = [x for x in key_parts if x[0] < len_indent]\n\n # From version 3.X to version 6.X the output format\n # changed a lot in pytest, and probably will change\n # in the future - so this manipulation might be fragile\n if nodetype.lower() == \"function\":\n name_parts.append(item)\n key_end = os.path.join(*key_parts[-1][1].split(\"/\"))\n key = next(f for f in files if f.endswith(key_end))\n tests[key].add(tuple(x[1] for x in name_parts))\n elif nodetype.lower() == \"class\":\n name_parts.append(item)\n elif nodetype.lower() in (\"package\", \"module\"):\n key_parts.append(item)\n\n if args.list == \"list\":\n files = set(tests.keys())\n color_files = [colorize(\"B\", file) for file in sorted(files)]\n colify(color_files)\n\n elif args.list == \"long\":\n for prefix, functions in sorted(tests.items()):\n path = colorize(\"*B\", prefix) + \"::\"\n functions = [colorize(\"c\", f) for f in sorted(functions)]\n color.cprint(path)\n colify(functions, indent=4)\n print()\n\n else: # args.list == \"names\"\n all_functions = [\n colorize(\"*B\", prefix) + \"::\" + colorize(\"c\", f)\n for prefix, functions in sorted(tests.items())\n for f in sorted(functions)\n ]\n colify(all_functions)\n\n\ndef add_back_pytest_args(args, unknown_args):\n \"\"\"Add parsed pytest args, unknown args, and remainder together.\n\n We add some basic pytest arguments to the Spack parser to ensure that\n they show up in the short help, so we have to reassemble things here.\n \"\"\"\n result = args.parsed_args or []\n result += unknown_args or []\n result += args.pytest_args or []\n if args.expression:\n result += [\"-k\", args.expression]\n return result\n\n\ndef unit_test(parser, args, unknown_args):\n global pytest\n import spack.bootstrap\n\n # Ensure clingo is available before switching to the\n # mock configuration used by unit tests\n with spack.bootstrap.ensure_bootstrap_configuration():\n spack.bootstrap.ensure_clingo_importable_or_raise()\n if pytest is None:\n spack.bootstrap.ensure_environment_dependencies()\n import pytest\n\n if args.pytest_help:\n # make the pytest.main help output more accurate\n sys.argv[0] = \"spack unit-test\"\n return pytest.main([\"-h\"])\n\n # add back any parsed pytest args we need to pass to pytest\n pytest_args = add_back_pytest_args(args, unknown_args)\n\n # The default is to test the core of Spack. If the option `--extension`\n # has been used, then test that extension.\n pytest_root = spack.paths.spack_root\n if args.extension:\n pytest_root = spack.extensions.load_extension(args.extension)\n\n if args.numprocesses is not None and args.numprocesses > 1:\n try:\n import xdist # noqa: F401\n except ImportError:\n tty.error(\"parallel unit-test requires pytest-xdist module\")\n return 1\n\n pytest_args.extend(\n [\n \"--dist\",\n \"loadfile\",\n \"--tx\",\n f\"{args.numprocesses}*popen//python=spack-tmpconfig spack python\",\n ]\n )\n\n # pytest.ini lives in the root of the spack repository.\n with spack.llnl.util.filesystem.working_dir(pytest_root):\n if args.list:\n do_list(args, pytest_args)\n return\n\n return pytest.main(pytest_args)\n" }, { "path": "lib/spack/spack/cmd/unload.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport os\nimport sys\n\nimport spack.cmd\nimport spack.cmd.common\nimport spack.error\nimport spack.store\nimport spack.user_environment as uenv\nfrom spack.cmd.common import arguments\n\ndescription = \"remove package from the user environment\"\nsection = \"user environment\"\nlevel = \"short\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n \"\"\"Parser is only constructed so that this prints a nice help\n message with -h.\"\"\"\n arguments.add_common_arguments(subparser, [\"installed_specs\"])\n\n shells = subparser.add_mutually_exclusive_group()\n shells.add_argument(\n \"--sh\",\n action=\"store_const\",\n dest=\"shell\",\n const=\"sh\",\n help=\"print sh commands to activate the environment\",\n )\n shells.add_argument(\n \"--csh\",\n action=\"store_const\",\n dest=\"shell\",\n const=\"csh\",\n help=\"print csh commands to activate the environment\",\n )\n shells.add_argument(\n \"--fish\",\n action=\"store_const\",\n dest=\"shell\",\n const=\"fish\",\n help=\"print fish commands to load the package\",\n )\n shells.add_argument(\n \"--bat\",\n action=\"store_const\",\n dest=\"shell\",\n const=\"bat\",\n help=\"print bat commands to load the package\",\n )\n shells.add_argument(\n \"--pwsh\",\n action=\"store_const\",\n dest=\"shell\",\n const=\"pwsh\",\n help=\"print pwsh commands to load the package\",\n )\n\n subparser.add_argument(\n \"-a\", \"--all\", action=\"store_true\", help=\"unload all loaded Spack packages\"\n )\n\n\ndef unload(parser, args):\n \"\"\"unload spack packages from the user environment\"\"\"\n if args.specs and args.all:\n raise spack.error.SpackError(\n \"Cannot specify specs on command line when unloading all specs with '--all'\"\n )\n\n hashes = os.environ.get(uenv.spack_loaded_hashes_var, \"\").split(os.pathsep)\n if args.specs:\n specs = [\n spack.cmd.disambiguate_spec_from_hashes(spec, hashes)\n for spec in spack.cmd.parse_specs(args.specs)\n ]\n else:\n specs = spack.store.STORE.db.query(hashes=hashes)\n\n if not args.shell:\n specs_str = \" \".join(args.specs) or \"SPECS\"\n\n spack.cmd.common.shell_init_instructions(\n \"spack unload\", \" eval `spack unload {sh_arg}` %s\" % specs_str\n )\n return 1\n\n env_mod = uenv.environment_modifications_for_specs(*specs).reversed()\n for spec in specs:\n env_mod.remove_path(uenv.spack_loaded_hashes_var, spec.dag_hash())\n cmds = env_mod.shell_modifications(args.shell)\n\n sys.stdout.write(cmds)\n" }, { "path": "lib/spack/spack/cmd/url.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport urllib.parse\nfrom collections import defaultdict\n\nimport spack.fetch_strategy as fs\nimport spack.llnl.util.tty.color as color\nimport spack.repo\nimport spack.spec\nimport spack.url\nimport spack.util.crypto as crypto\nfrom spack.llnl.util import tty\nfrom spack.url import (\n UndetectableNameError,\n UndetectableVersionError,\n UrlParseError,\n color_url,\n parse_name,\n parse_name_offset,\n parse_version,\n parse_version_offset,\n substitute_version,\n substitution_offsets,\n)\nfrom spack.util.naming import simplify_name\n\ndescription = \"debugging tool for url parsing\"\nsection = \"developer\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n sp = subparser.add_subparsers(metavar=\"SUBCOMMAND\", dest=\"subcommand\")\n\n # Parse\n parse_parser = sp.add_parser(\"parse\", help=\"attempt to parse a url\")\n\n parse_parser.add_argument(\"url\", help=\"url to parse\")\n parse_parser.add_argument(\n \"-s\", \"--spider\", action=\"store_true\", help=\"spider the source page for versions\"\n )\n\n # List\n list_parser = sp.add_parser(\"list\", help=\"list urls in all packages\")\n\n list_parser.add_argument(\n \"-c\",\n \"--color\",\n action=\"store_true\",\n help=\"color the parsed version and name in the urls shown \"\n \"(versions will be cyan, name red)\",\n )\n list_parser.add_argument(\n \"-e\",\n \"--extrapolation\",\n action=\"store_true\",\n help=\"color the versions used for extrapolation as well \"\n \"(additional versions will be green, names magenta)\",\n )\n\n excl_args = list_parser.add_mutually_exclusive_group()\n\n excl_args.add_argument(\n \"-n\",\n \"--incorrect-name\",\n action=\"store_true\",\n help=\"only list urls for which the name was incorrectly parsed\",\n )\n excl_args.add_argument(\n \"-N\",\n \"--correct-name\",\n action=\"store_true\",\n help=\"only list urls for which the name was correctly parsed\",\n )\n excl_args.add_argument(\n \"-v\",\n \"--incorrect-version\",\n action=\"store_true\",\n help=\"only list urls for which the version was incorrectly parsed\",\n )\n excl_args.add_argument(\n \"-V\",\n \"--correct-version\",\n action=\"store_true\",\n help=\"only list urls for which the version was correctly parsed\",\n )\n\n # Summary\n sp.add_parser(\"summary\", help=\"print a summary of how well we are parsing package urls\")\n\n # Stats\n stats_parser = sp.add_parser(\n \"stats\", help=\"print statistics on versions and checksums for all packages\"\n )\n stats_parser.add_argument(\n \"--show-issues\",\n action=\"store_true\",\n help=\"show packages with issues (md5 hashes, http urls)\",\n )\n\n\ndef url(parser, args):\n action = {\"parse\": url_parse, \"list\": url_list, \"summary\": url_summary, \"stats\": url_stats}\n\n action[args.subcommand](args)\n\n\ndef url_parse(args):\n url = args.url\n\n tty.msg(\"Parsing URL: {0}\".format(url))\n print()\n\n ver, vs, vl, vi, vregex = parse_version_offset(url)\n tty.msg(\"Matched version regex {0:>2}: r{1!r}\".format(vi, vregex))\n\n name, ns, nl, ni, nregex = parse_name_offset(url, ver)\n tty.msg(\"Matched name regex {0:>2}: r{1!r}\".format(ni, nregex))\n\n print()\n tty.msg(\"Detected:\")\n try:\n print_name_and_version(url)\n except UrlParseError as e:\n tty.error(str(e))\n\n print(\" name: {0}\".format(name))\n print(\" version: {0}\".format(ver))\n print()\n\n tty.msg(\"Substituting version 9.9.9b:\")\n newurl = substitute_version(url, \"9.9.9b\")\n print_name_and_version(newurl)\n\n if args.spider:\n print()\n tty.msg(\"Spidering for versions:\")\n versions = spack.url.find_versions_of_archive(url)\n\n if not versions:\n print(\" Found no versions for {0}\".format(name))\n return\n\n max_len = max(len(str(v)) for v in versions)\n\n for v in sorted(versions):\n print(\"{0:{1}} {2}\".format(v, max_len, versions[v]))\n\n\ndef url_list(args):\n urls = set()\n\n # Gather set of URLs from all packages\n for pkg_cls in spack.repo.PATH.all_package_classes():\n url = getattr(pkg_cls, \"url\", None)\n urls = url_list_parsing(args, urls, url, pkg_cls)\n\n for params in pkg_cls.versions.values():\n url = params.get(\"url\", None)\n urls = url_list_parsing(args, urls, url, pkg_cls)\n\n # Print URLs\n for url in sorted(urls):\n if args.color or args.extrapolation:\n print(color_url(url, subs=args.extrapolation, errors=True))\n else:\n print(url)\n\n # Return the number of URLs that were printed, only for testing purposes\n return len(urls)\n\n\ndef url_summary(args):\n # Collect statistics on how many URLs were correctly parsed\n total_urls = 0\n correct_names = 0\n correct_versions = 0\n\n # Collect statistics on which regexes were matched and how often\n name_regex_dict = dict()\n right_name_count = defaultdict(int)\n wrong_name_count = defaultdict(int)\n\n version_regex_dict = dict()\n right_version_count = defaultdict(int)\n wrong_version_count = defaultdict(int)\n\n tty.msg(\"Generating a summary of URL parsing in Spack...\")\n\n # Loop through all packages\n for pkg_cls in spack.repo.PATH.all_package_classes():\n urls = set()\n pkg = pkg_cls(spack.spec.Spec(pkg_cls.name))\n\n url = getattr(pkg, \"url\", None)\n if url:\n urls.add(url)\n\n for params in pkg.versions.values():\n url = params.get(\"url\", None)\n if url:\n urls.add(url)\n\n # Calculate statistics\n for url in urls:\n total_urls += 1\n\n # Parse versions\n version = None\n try:\n version, vs, vl, vi, vregex = parse_version_offset(url)\n version_regex_dict[vi] = vregex\n if version_parsed_correctly(pkg, version):\n correct_versions += 1\n right_version_count[vi] += 1\n else:\n wrong_version_count[vi] += 1\n except UndetectableVersionError:\n pass\n\n # Parse names\n try:\n name, ns, nl, ni, nregex = parse_name_offset(url, version)\n name_regex_dict[ni] = nregex\n if name_parsed_correctly(pkg, name):\n correct_names += 1\n right_name_count[ni] += 1\n else:\n wrong_name_count[ni] += 1\n except UndetectableNameError:\n pass\n\n print()\n print(\" Total URLs found: {0}\".format(total_urls))\n print(\n \" Names correctly parsed: {0:>4}/{1:>4} ({2:>6.2%})\".format(\n correct_names, total_urls, correct_names / total_urls\n )\n )\n print(\n \" Versions correctly parsed: {0:>4}/{1:>4} ({2:>6.2%})\".format(\n correct_versions, total_urls, correct_versions / total_urls\n )\n )\n print()\n\n tty.msg(\"Statistics on name regular expressions:\")\n\n print()\n print(\" Index Right Wrong Total Regular Expression\")\n for ni in sorted(name_regex_dict.keys()):\n print(\n \" {0:>5} {1:>5} {2:>5} {3:>5} r{4!r}\".format(\n ni,\n right_name_count[ni],\n wrong_name_count[ni],\n right_name_count[ni] + wrong_name_count[ni],\n name_regex_dict[ni],\n )\n )\n print()\n\n tty.msg(\"Statistics on version regular expressions:\")\n\n print()\n print(\" Index Right Wrong Total Regular Expression\")\n for vi in sorted(version_regex_dict.keys()):\n print(\n \" {0:>5} {1:>5} {2:>5} {3:>5} r{4!r}\".format(\n vi,\n right_version_count[vi],\n wrong_version_count[vi],\n right_version_count[vi] + wrong_version_count[vi],\n version_regex_dict[vi],\n )\n )\n print()\n\n # Return statistics, only for testing purposes\n return (total_urls, correct_names, correct_versions, right_name_count, right_version_count)\n\n\ndef url_stats(args):\n # dictionary of issue type -> package -> descriptions\n issues = defaultdict(lambda: defaultdict(lambda: []))\n\n class UrlStats:\n def __init__(self):\n self.total = 0\n self.schemes = defaultdict(lambda: 0)\n self.checksums = defaultdict(lambda: 0)\n self.url_type = defaultdict(lambda: 0)\n self.git_type = defaultdict(lambda: 0)\n\n def add(self, pkg_name, fetcher):\n self.total += 1\n\n url_type = fetcher.url_attr\n self.url_type[url_type or \"no code\"] += 1\n\n if url_type == \"url\":\n digest = getattr(fetcher, \"digest\", None)\n if digest:\n algo = crypto.hash_algo_for_digest(digest)\n else:\n algo = \"no checksum\"\n self.checksums[algo] += 1\n\n if algo == \"md5\":\n md5_hashes = issues[\"md5 hashes\"]\n md5_hashes[pkg_name].append(fetcher.url)\n\n # parse out the URL scheme (https/http/ftp/etc.)\n urlinfo = urllib.parse.urlparse(fetcher.url)\n self.schemes[urlinfo.scheme] += 1\n\n if urlinfo.scheme == \"http\":\n http_urls = issues[\"http urls\"]\n http_urls[pkg_name].append(fetcher.url)\n\n elif url_type == \"git\":\n if getattr(fetcher, \"commit\", None):\n self.git_type[\"commit\"] += 1\n elif getattr(fetcher, \"branch\", None):\n self.git_type[\"branch\"] += 1\n elif getattr(fetcher, \"tag\", None):\n self.git_type[\"tag\"] += 1\n else:\n self.git_type[\"no ref\"] += 1\n\n npkgs = 0\n version_stats = UrlStats()\n resource_stats = UrlStats()\n\n for pkg_cls in spack.repo.PATH.all_package_classes():\n npkgs += 1\n\n for v in list(pkg_cls.versions):\n try:\n pkg = pkg_cls(spack.spec.Spec(pkg_cls.name))\n fetcher = fs.for_package_version(pkg, v)\n except (fs.InvalidArgsError, fs.FetcherConflict):\n continue\n version_stats.add(pkg_cls.name, fetcher)\n\n for _, resources in pkg_cls.resources.items():\n for resource in resources:\n resource_stats.add(pkg_cls.name, resource.fetcher)\n\n # print a nice summary table\n tty.msg(\"URL stats for %d packages:\" % npkgs)\n\n def print_line():\n print(\"-\" * 62)\n\n def print_stat(indent, name, stat_name=None):\n width = 20 - indent\n fmt = \" \" * indent\n fmt += \"%%-%ds\" % width\n if stat_name is None:\n print(fmt % name)\n else:\n fmt += \"%12d%8.1f%%%12d%8.1f%%\"\n v = getattr(version_stats, stat_name).get(name, 0)\n r = getattr(resource_stats, stat_name).get(name, 0)\n print(\n fmt % (name, v, v / version_stats.total * 100, r, r / resource_stats.total * 100)\n )\n\n print_line()\n print(\"%-20s%12s%9s%12s%9s\" % (\"stat\", \"versions\", \"%\", \"resources\", \"%\"))\n print_line()\n print_stat(0, \"url\", \"url_type\")\n\n print_stat(4, \"schemes\")\n schemes = set(version_stats.schemes) | set(resource_stats.schemes)\n for scheme in schemes:\n print_stat(8, scheme, \"schemes\")\n\n print_stat(4, \"checksums\")\n checksums = set(version_stats.checksums) | set(resource_stats.checksums)\n for checksum in checksums:\n print_stat(8, checksum, \"checksums\")\n print_line()\n\n types = set(version_stats.url_type) | set(resource_stats.url_type)\n types -= set([\"url\", \"git\"])\n for url_type in sorted(types):\n print_stat(0, url_type, \"url_type\")\n print_line()\n\n print_stat(0, \"git\", \"url_type\")\n git_types = set(version_stats.git_type) | set(resource_stats.git_type)\n for git_type in sorted(git_types):\n print_stat(4, git_type, \"git_type\")\n print_line()\n\n if args.show_issues:\n total_issues = sum(\n len(issues) for _, pkg_issues in issues.items() for _, issues in pkg_issues.items()\n )\n print()\n tty.msg(\"Found %d issues.\" % total_issues)\n for issue_type, pkgs in issues.items():\n tty.msg(\"Package URLs with %s\" % issue_type)\n for pkg_cls, pkg_issues in pkgs.items():\n color.cprint(\" @*C{%s}\" % pkg_cls)\n for issue in pkg_issues:\n print(\" %s\" % issue)\n\n\ndef print_name_and_version(url):\n \"\"\"Prints a URL. Underlines the detected name with dashes and\n the detected version with tildes.\n\n Args:\n url (str): The url to parse\n \"\"\"\n name, ns, nl, ntup, ver, vs, vl, vtup = substitution_offsets(url)\n underlines = [\" \"] * max(ns + nl, vs + vl)\n for i in range(ns, ns + nl):\n underlines[i] = \"-\"\n for i in range(vs, vs + vl):\n underlines[i] = \"~\"\n\n print(\" {0}\".format(url))\n print(\" {0}\".format(\"\".join(underlines)))\n\n\ndef url_list_parsing(args, urls, url, pkg):\n \"\"\"Helper function for :func:`url_list`.\n\n Args:\n args (argparse.Namespace): The arguments given to ``spack url list``\n urls (set): List of URLs that have already been added\n url (str or None): A URL to potentially add to ``urls`` depending on\n ``args``\n pkg (spack.package_base.PackageBase): The Spack package\n\n Returns:\n set: The updated set of ``urls``\n \"\"\"\n if url:\n if args.correct_name or args.incorrect_name:\n # Attempt to parse the name\n try:\n name = parse_name(url)\n if args.correct_name and name_parsed_correctly(pkg, name):\n # Add correctly parsed URLs\n urls.add(url)\n elif args.incorrect_name and not name_parsed_correctly(pkg, name):\n # Add incorrectly parsed URLs\n urls.add(url)\n except UndetectableNameError:\n if args.incorrect_name:\n # Add incorrectly parsed URLs\n urls.add(url)\n elif args.correct_version or args.incorrect_version:\n # Attempt to parse the version\n try:\n version = parse_version(url)\n if args.correct_version and version_parsed_correctly(pkg, version):\n # Add correctly parsed URLs\n urls.add(url)\n elif args.incorrect_version and not version_parsed_correctly(pkg, version):\n # Add incorrectly parsed URLs\n urls.add(url)\n except UndetectableVersionError:\n if args.incorrect_version:\n # Add incorrectly parsed URLs\n urls.add(url)\n else:\n urls.add(url)\n\n return urls\n\n\ndef name_parsed_correctly(pkg, name):\n \"\"\"Determine if the name of a package was correctly parsed.\n\n Args:\n pkg (spack.package_base.PackageBase): The Spack package\n name (str): The name that was extracted from the URL\n\n Returns:\n bool: True if the name was correctly parsed, else False\n \"\"\"\n pkg_name = remove_prefix(pkg.name)\n\n name = simplify_name(name)\n\n return name == pkg_name\n\n\ndef version_parsed_correctly(pkg, version):\n \"\"\"Determine if the version of a package was correctly parsed.\n\n Args:\n pkg (spack.package_base.PackageBase): The Spack package\n version (str): The version that was extracted from the URL\n\n Returns:\n bool: True if the name was correctly parsed, else False\n \"\"\"\n version = remove_separators(version)\n\n # If the version parsed from the URL is listed in a version()\n # directive, we assume it was correctly parsed\n for pkg_version in pkg.versions:\n pkg_version = remove_separators(pkg_version)\n if pkg_version == version:\n return True\n return False\n\n\ndef remove_prefix(pkg_name):\n \"\"\"Remove build system prefix (``'py-'``, ``'perl-'``, etc.) from a package name.\n\n After determining a name, ``spack create`` determines a build system.\n Some build systems prepend a special string to the front of the name.\n Since this can't be guessed from the URL, it would be unfair to say\n that these names are incorrectly parsed, so we remove them.\n\n Args:\n pkg_name (str): the name of the package\n\n Returns:\n str: the name of the package with any build system prefix removed\n \"\"\"\n prefixes = [\n \"r-\",\n \"py-\",\n \"tcl-\",\n \"lua-\",\n \"perl-\",\n \"ruby-\",\n \"llvm-\",\n \"intel-\",\n \"votca-\",\n \"octave-\",\n \"gtkorvo-\",\n ]\n\n prefix = next((p for p in prefixes if pkg_name.startswith(p)), \"\")\n\n return pkg_name[len(prefix) :]\n\n\ndef remove_separators(version):\n \"\"\"Remove separator characters (``.``, ``_``, and ``-``) from a version.\n\n A version like 1.2.3 may be displayed as 1_2_3 in the URL.\n Make sure 1.2.3, 1-2-3, 1_2_3, and 123 are considered equal.\n Unfortunately, this also means that 1.23 and 12.3 are equal.\n\n Args:\n version (str or spack.version.Version): A version\n\n Returns:\n str: The version with all separator characters removed\n \"\"\"\n version = str(version)\n\n version = version.replace(\".\", \"\")\n version = version.replace(\"_\", \"\")\n version = version.replace(\"-\", \"\")\n\n return version\n" }, { "path": "lib/spack/spack/cmd/verify.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport argparse\nimport io\nfrom typing import List, Optional\n\nimport spack.cmd\nimport spack.environment as ev\nimport spack.llnl.util.tty as tty\nimport spack.spec\nimport spack.store\nimport spack.verify\nimport spack.verify_libraries\nfrom spack.cmd.common import arguments\nfrom spack.llnl.string import plural\nfrom spack.llnl.util.filesystem import visit_directory_tree\n\ndescription = \"verify spack installations on disk\"\nsection = \"admin\"\nlevel = \"long\"\n\nMANIFEST_SUBPARSER: Optional[argparse.ArgumentParser] = None\n\n\ndef setup_parser(subparser: argparse.ArgumentParser):\n global MANIFEST_SUBPARSER\n sp = subparser.add_subparsers(metavar=\"SUBCOMMAND\", dest=\"verify_command\")\n\n MANIFEST_SUBPARSER = sp.add_parser(\n \"manifest\", help=verify_manifest.__doc__, description=verify_manifest.__doc__\n )\n MANIFEST_SUBPARSER.add_argument(\n \"-l\", \"--local\", action=\"store_true\", help=\"verify only locally installed packages\"\n )\n MANIFEST_SUBPARSER.add_argument(\n \"-j\", \"--json\", action=\"store_true\", help=\"output json-formatted errors\"\n )\n MANIFEST_SUBPARSER.add_argument(\"-a\", \"--all\", action=\"store_true\", help=\"verify all packages\")\n MANIFEST_SUBPARSER.add_argument(\n \"specs_or_files\", nargs=argparse.REMAINDER, help=\"specs or files to verify\"\n )\n\n manifest_sp_type = MANIFEST_SUBPARSER.add_mutually_exclusive_group()\n manifest_sp_type.add_argument(\n \"-s\",\n \"--specs\",\n action=\"store_const\",\n const=\"specs\",\n dest=\"type\",\n default=\"specs\",\n help=\"treat entries as specs (default)\",\n )\n manifest_sp_type.add_argument(\n \"-f\",\n \"--files\",\n action=\"store_const\",\n const=\"files\",\n dest=\"type\",\n default=\"specs\",\n help=\"treat entries as absolute filenames\\n\\ncannot be used with '-a'\",\n )\n\n libraries_subparser = sp.add_parser(\n \"libraries\", help=verify_libraries.__doc__, description=verify_libraries.__doc__\n )\n\n arguments.add_common_arguments(libraries_subparser, [\"constraint\"])\n\n versions_subparser = sp.add_parser(\n \"versions\", help=verify_versions.__doc__, description=verify_versions.__doc__\n )\n arguments.add_common_arguments(versions_subparser, [\"constraint\"])\n\n\ndef verify(parser, args):\n cmd = args.verify_command\n if cmd == \"libraries\":\n return verify_libraries(args)\n elif cmd == \"manifest\":\n return verify_manifest(args)\n elif cmd == \"versions\":\n return verify_versions(args)\n parser.error(\"invalid verify subcommand\")\n\n\ndef verify_versions(args):\n \"\"\"Check that all versions of installed packages are known to Spack and non-deprecated.\n\n Reports errors for any of the following:\n\n 1. Installed package not loadable from the repo\n 2. Installed package version not known by the package recipe\n 3. Installed package version deprecated in the package recipe\n \"\"\"\n specs = args.specs(installed=True)\n\n msg_lines = _verify_version(specs)\n if msg_lines:\n tty.die(\"\\n\".join(msg_lines))\n\n\ndef _verify_version(specs):\n \"\"\"Helper method for verify_versions.\"\"\"\n missing_package = []\n unknown_version = []\n deprecated_version = []\n\n for spec in specs:\n try:\n pkg = spec.package\n except Exception as e:\n tty.debug(str(e))\n missing_package.append(spec)\n continue\n\n if spec.version not in pkg.versions:\n unknown_version.append(spec)\n continue\n\n if pkg.versions[spec.version].get(\"deprecated\", False):\n deprecated_version.append(spec)\n\n msg_lines = []\n if missing_package or unknown_version or deprecated_version:\n errors = len(missing_package) + len(unknown_version) + len(deprecated_version)\n msg_lines = [f\"{errors} installed packages have unknown/deprecated versions\\n\"]\n\n msg_lines += [\n f\" Cannot check version for {spec} at {spec.prefix}. Cannot load package.\"\n for spec in missing_package\n ]\n msg_lines += [\n f\" Spec {spec} at {spec.prefix} has version {spec.version} unknown to Spack.\"\n for spec in unknown_version\n ]\n msg_lines += [\n f\" Spec {spec} at {spec.prefix} has deprecated version {spec.version}.\"\n for spec in deprecated_version\n ]\n\n return msg_lines\n\n\ndef verify_libraries(args):\n \"\"\"verify that shared libraries of install packages can be located in rpaths (Linux only)\"\"\"\n specs_from_db = [s for s in args.specs(installed=True) if not s.external]\n\n tty.info(f\"Checking {len(specs_from_db)} packages for shared library resolution\")\n\n errors = 0\n for spec in specs_from_db:\n try:\n pkg = spec.package\n except Exception:\n tty.warn(f\"Skipping {spec.cformat('{name}{@version}{/hash}')} due to missing package\")\n error_msg = _verify_libraries(spec, pkg.unresolved_libraries)\n if error_msg is not None:\n errors += 1\n tty.error(error_msg)\n\n if errors:\n tty.error(f\"Cannot resolve shared libraries in {plural(errors, 'package')}\")\n return 1\n\n\ndef _verify_libraries(spec: spack.spec.Spec, unresolved_libraries: List[str]) -> Optional[str]:\n \"\"\"Go over the prefix of the installed spec and verify its shared libraries can be resolved.\"\"\"\n visitor = spack.verify_libraries.ResolveSharedElfLibDepsVisitor(\n [*spack.verify_libraries.ALLOW_UNRESOLVED, *unresolved_libraries]\n )\n visit_directory_tree(spec.prefix, visitor)\n\n if not visitor.problems:\n return None\n\n output = io.StringIO()\n visitor.write(output, indent=4, brief=True)\n message = output.getvalue().rstrip()\n return f\"{spec.cformat('{name}{@version}{/hash}')}: {spec.prefix}:\\n{message}\"\n\n\ndef verify_manifest(args):\n \"\"\"verify that install directories have not been modified since installation\"\"\"\n local = args.local\n\n if args.type == \"files\":\n if args.all:\n MANIFEST_SUBPARSER.error(\"cannot use --all with --files\")\n\n for file in args.specs_or_files:\n results = spack.verify.check_file_manifest(file)\n if results.has_errors():\n if args.json:\n print(results.json_string())\n else:\n print(results)\n\n return 0\n else:\n spec_args = spack.cmd.parse_specs(args.specs_or_files)\n\n if args.all:\n query = spack.store.STORE.db.query_local if local else spack.store.STORE.db.query\n\n # construct spec list\n if spec_args:\n spec_list = spack.cmd.parse_specs(args.specs_or_files)\n specs = []\n for spec in spec_list:\n specs += query(spec, installed=True)\n else:\n specs = query(installed=True)\n\n elif args.specs_or_files:\n # construct disambiguated spec list\n env = ev.active_environment()\n specs = list(map(lambda x: spack.cmd.disambiguate_spec(x, env, local=local), spec_args))\n else:\n MANIFEST_SUBPARSER.error(\"use --all or specify specs to verify\")\n\n for spec in specs:\n tty.debug(\"Verifying package %s\")\n results = spack.verify.check_spec_manifest(spec)\n if results.has_errors():\n if args.json:\n print(results.json_string())\n else:\n tty.msg(\"In package %s\" % spec.format(\"{name}/{hash:7}\"))\n print(results)\n return 1\n else:\n tty.debug(results)\n" }, { "path": "lib/spack/spack/cmd/versions.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport sys\n\nimport spack.llnl.util.tty as tty\nimport spack.repo\nimport spack.spec\nfrom spack.cmd.common import arguments\nfrom spack.llnl.util.tty.colify import colify\nfrom spack.version import infinity_versions, ver\n\ndescription = \"list available versions of a package\"\nsection = \"query\"\nlevel = \"long\"\n\n\ndef setup_parser(subparser: argparse.ArgumentParser) -> None:\n output = subparser.add_mutually_exclusive_group()\n output.add_argument(\n \"-s\", \"--safe\", action=\"store_true\", help=\"only list safe versions of the package\"\n )\n output.add_argument(\n \"-r\", \"--remote\", action=\"store_true\", help=\"only list remote versions of the package\"\n )\n output.add_argument(\n \"-n\",\n \"--new\",\n action=\"store_true\",\n help=\"only list remote versions newer than the latest checksummed version\",\n )\n arguments.add_common_arguments(subparser, [\"package\", \"jobs\"])\n\n\ndef versions(parser, args):\n spec = spack.spec.Spec(args.package)\n pkg_cls = spack.repo.PATH.get_pkg_class(spec.name)\n pkg = pkg_cls(spec)\n\n safe_versions = pkg.versions\n\n if not (args.remote or args.new):\n if sys.stdout.isatty():\n tty.msg(\"Safe versions (already checksummed):\")\n\n if not safe_versions:\n if sys.stdout.isatty():\n tty.warn(f\"Found no versions for {pkg.name}\")\n tty.debug(\"Manually add versions to the package.\")\n else:\n colify(sorted(safe_versions, reverse=True), indent=2)\n\n if args.safe:\n return\n\n fetched_versions = pkg.fetch_remote_versions(args.jobs)\n\n if args.new:\n if sys.stdout.isatty():\n tty.msg(\"New remote versions (not yet checksummed):\")\n numeric_safe_versions = list(\n filter(lambda v: str(v) not in infinity_versions, safe_versions)\n )\n highest_safe_version = max(numeric_safe_versions)\n remote_versions = set([ver(v) for v in set(fetched_versions) if v > highest_safe_version])\n else:\n if sys.stdout.isatty():\n tty.msg(\"Remote versions (not yet checksummed):\")\n remote_versions = set(fetched_versions).difference(safe_versions)\n\n if not remote_versions:\n if sys.stdout.isatty():\n if not fetched_versions:\n tty.warn(f\"Found no versions for {pkg.name}\")\n tty.debug(\n \"Check the list_url and list_depth attributes of \"\n \"the package to help Spack find versions.\"\n )\n else:\n tty.warn(f\"Found no unchecksummed versions for {pkg.name}\")\n else:\n colify(sorted(remote_versions, reverse=True), indent=2)\n" }, { "path": "lib/spack/spack/cmd/view.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Produce a \"view\" of a Spack DAG.\n\nA \"view\" is file hierarchy representing the union of a number of\nSpack-installed package file hierarchies. The union is formed from:\n\n- specs resolved from the package names given by the user (the seeds)\n\n- all dependencies of the seeds unless user specifies ``--no-dependencies``\n\n- less any specs with names matching the regular expressions given by\n ``--exclude``\n\nThe ``view`` can be built and tore down via a number of methods (the \"actions\"):\n\n- symlink :: a file system view which is a directory hierarchy that is\n the union of the hierarchies of the installed packages in the DAG\n where installed files are referenced via symlinks.\n\n- hardlink :: like the symlink view but hardlinks are used.\n\n- statlink :: a view producing a status report of a symlink or\n hardlink view.\n\nThe file system view concept is inspired by Nix, implemented by\nBrett Viren ca 2016.\n\nAll operations on views are performed via proxy objects such as\nYamlFilesystemView.\n\n\"\"\"\n\nimport argparse\nimport sys\n\nimport spack.cmd\nimport spack.environment as ev\nimport spack.filesystem_view as fsv\nimport spack.llnl.util.tty as tty\nimport spack.schema.projections\nimport spack.store\nfrom spack.config import validate\nfrom spack.llnl.util.link_tree import MergeConflictError\nfrom spack.util import spack_yaml as s_yaml\n\ndescription = \"manipulate view directories in the filesystem\"\nsection = \"environments\"\nlevel = \"short\"\n\nactions_link = [\"symlink\", \"add\", \"soft\", \"hardlink\", \"hard\", \"copy\", \"relocate\"]\nactions_remove = [\"remove\", \"rm\"]\nactions_status = [\"statlink\", \"status\", \"check\"]\n\n\ndef disambiguate_in_view(specs, view):\n \"\"\"\n When dealing with querying actions (remove/status) we only need to\n disambiguate among specs in the view\n \"\"\"\n view_specs = set(view.get_all_specs())\n\n def squash(matching_specs):\n if not matching_specs:\n tty.die(\"Spec matches no installed packages.\")\n\n matching_in_view = [ms for ms in matching_specs if ms in view_specs]\n spack.cmd.ensure_single_spec_or_die(\"Spec\", matching_in_view)\n\n return matching_in_view[0] if matching_in_view else matching_specs[0]\n\n # make function always return a list to keep consistency between py2/3\n return list(map(squash, map(spack.store.STORE.db.query, specs)))\n\n\ndef setup_parser(sp: argparse.ArgumentParser) -> None:\n setattr(setup_parser, \"parser\", sp)\n\n sp.add_argument(\n \"-v\",\n \"--verbose\",\n action=\"store_true\",\n default=False,\n help=\"if not verbose only warnings/errors will be printed\",\n )\n sp.add_argument(\n \"-e\",\n \"--exclude\",\n action=\"append\",\n default=[],\n help=\"exclude packages with names matching the given regex pattern\",\n )\n sp.add_argument(\n \"-d\",\n \"--dependencies\",\n choices=[\"true\", \"false\", \"yes\", \"no\"],\n default=\"true\",\n help=\"link/remove/list dependencies\",\n )\n\n ssp = sp.add_subparsers(metavar=\"ACTION\", dest=\"action\")\n\n # The action parameterizes the command but in keeping with Spack\n # patterns we make it a subcommand.\n file_system_view_actions = {\n \"symlink\": ssp.add_parser(\n \"symlink\",\n aliases=[\"add\", \"soft\"],\n help=\"add package files to a filesystem view via symbolic links\",\n ),\n \"hardlink\": ssp.add_parser(\n \"hardlink\",\n aliases=[\"hard\"],\n help=\"add packages files to a filesystem view via hard links\",\n ),\n \"copy\": ssp.add_parser(\n \"copy\",\n aliases=[\"relocate\"],\n help=\"add package files to a filesystem view via copy/relocate\",\n ),\n \"remove\": ssp.add_parser(\n \"remove\", aliases=[\"rm\"], help=\"remove packages from a filesystem view\"\n ),\n \"statlink\": ssp.add_parser(\n \"statlink\",\n aliases=[\"status\", \"check\"],\n help=\"check status of packages in a filesystem view\",\n ),\n }\n\n # All these options and arguments are common to every action.\n for cmd, act in file_system_view_actions.items():\n act.add_argument(\"path\", nargs=1, help=\"path to file system view directory\")\n\n if cmd in (\"symlink\", \"hardlink\", \"copy\"):\n # invalid for remove/statlink, for those commands the view needs to\n # already know its own projections.\n act.add_argument(\n \"--projection-file\",\n dest=\"projection_file\",\n type=spack.cmd.extant_file,\n help=\"initialize view using projections from file\",\n )\n\n if cmd == \"remove\":\n grp = act.add_mutually_exclusive_group(required=True)\n act.add_argument(\n \"--no-remove-dependents\",\n action=\"store_true\",\n help=\"do not remove dependents of specified specs\",\n )\n\n # with all option, spec is an optional argument\n grp.add_argument(\n \"specs\",\n nargs=\"*\",\n default=[],\n metavar=\"spec\",\n action=\"store\",\n help=\"seed specs of the packages to view\",\n )\n grp.add_argument(\"-a\", \"--all\", action=\"store_true\", help=\"act on all specs in view\")\n\n elif cmd == \"statlink\":\n act.add_argument(\n \"specs\",\n nargs=\"*\",\n metavar=\"spec\",\n action=\"store\",\n help=\"seed specs of the packages to view\",\n )\n\n else:\n # without all option, spec is required\n act.add_argument(\n \"specs\",\n nargs=\"+\",\n metavar=\"spec\",\n action=\"store\",\n help=\"seed specs of the packages to view\",\n )\n\n for cmd in (\"symlink\", \"hardlink\", \"copy\"):\n act = file_system_view_actions[cmd]\n act.add_argument(\"-i\", \"--ignore-conflicts\", action=\"store_true\")\n\n\ndef view(parser, args):\n \"\"\"Produce a view of a set of packages.\"\"\"\n\n if sys.platform == \"win32\" and args.action in (\"hardlink\", \"hard\"):\n # Hard-linked views are not yet allowed on Windows.\n # See https://github.com/spack/spack/pull/46335#discussion_r1757411915\n tty.die(\"Hard linking is not supported on Windows. Please use symlinks or copy methods.\")\n\n specs = spack.cmd.parse_specs(args.specs)\n path = args.path[0]\n\n if args.action in actions_link and args.projection_file:\n # argparse confirms file exists\n with open(args.projection_file, \"r\", encoding=\"utf-8\") as f:\n projections_data = s_yaml.load(f)\n validate(projections_data, spack.schema.projections.schema)\n ordered_projections = projections_data[\"projections\"]\n else:\n ordered_projections = {}\n\n # What method are we using for this view\n link_type = args.action if args.action in actions_link else \"symlink\"\n view = fsv.YamlFilesystemView(\n path,\n spack.store.STORE.layout,\n projections=ordered_projections,\n ignore_conflicts=getattr(args, \"ignore_conflicts\", False),\n link_type=link_type,\n verbose=args.verbose,\n )\n\n # Process common args and specs\n if getattr(args, \"all\", False):\n specs = view.get_all_specs()\n if len(specs) == 0:\n tty.warn(\"Found no specs in %s\" % path)\n\n elif args.action in actions_link:\n # only link commands need to disambiguate specs\n env = ev.active_environment()\n specs = [spack.cmd.disambiguate_spec(s, env) for s in specs]\n\n elif args.action in actions_status:\n # no specs implies all\n if len(specs) == 0:\n specs = view.get_all_specs()\n else:\n specs = disambiguate_in_view(specs, view)\n\n else:\n # status and remove can map a partial spec to packages in view\n specs = disambiguate_in_view(specs, view)\n\n with_dependencies = args.dependencies.lower() in [\"true\", \"yes\"]\n\n # Map action to corresponding functionality\n if args.action in actions_link:\n try:\n view.add_specs(*specs, with_dependencies=with_dependencies, exclude=args.exclude)\n except MergeConflictError:\n tty.info(\n \"Some file blocked the merge, adding the '-i' flag will \"\n \"ignore this conflict. For more information see e.g. \"\n \"https://github.com/spack/spack/issues/9029\"\n )\n raise\n\n elif args.action in actions_remove:\n view.remove_specs(\n *specs,\n with_dependencies=with_dependencies,\n exclude=args.exclude,\n with_dependents=not args.no_remove_dependents,\n )\n\n elif args.action in actions_status:\n view.print_status(*specs, with_dependencies=with_dependencies)\n\n else:\n tty.error('Unknown action: \"%s\"' % args.action)\n" }, { "path": "lib/spack/spack/compilers/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n" }, { "path": "lib/spack/spack/compilers/adaptor.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport enum\nfrom typing import Dict, List\n\nimport spack.spec\nfrom spack.llnl.util import lang\n\nfrom .libraries import CompilerPropertyDetector\n\n\nclass Languages(enum.Enum):\n C = \"c\"\n CXX = \"cxx\"\n FORTRAN = \"fortran\"\n\n\nclass CompilerAdaptor:\n \"\"\"Provides access to compiler attributes via ``Package.compiler``. Useful for\n packages which do not yet access compiler properties via ``self.spec[language]``.\n \"\"\"\n\n def __init__(\n self, compiled_spec: spack.spec.Spec, compilers: Dict[Languages, spack.spec.Spec]\n ) -> None:\n if not compilers:\n raise AttributeError(f\"{compiled_spec} has no 'compiler' attribute\")\n\n self.compilers = compilers\n self.compiled_spec = compiled_spec\n\n def _lang_exists_or_raise(self, name: str, *, lang: Languages) -> None:\n if lang not in self.compilers:\n raise AttributeError(\n f\"'{self.compiled_spec}' has no {lang.value} compiler, so the \"\n f\"'{name}' property cannot be retrieved\"\n )\n\n def _maybe_return_attribute(self, name: str, *, lang: Languages) -> str:\n self._lang_exists_or_raise(name, lang=lang)\n return getattr(self.compilers[lang].package, name)\n\n @property\n def cc_rpath_arg(self) -> str:\n self._lang_exists_or_raise(\"cc_rpath_arg\", lang=Languages.C)\n return self.compilers[Languages.C].package.rpath_arg\n\n @property\n def cxx_rpath_arg(self) -> str:\n self._lang_exists_or_raise(\"cxx_rpath_arg\", lang=Languages.CXX)\n return self.compilers[Languages.CXX].package.rpath_arg\n\n @property\n def fc_rpath_arg(self) -> str:\n self._lang_exists_or_raise(\"fc_rpath_arg\", lang=Languages.FORTRAN)\n return self.compilers[Languages.FORTRAN].package.rpath_arg\n\n @property\n def f77_rpath_arg(self) -> str:\n self._lang_exists_or_raise(\"f77_rpath_arg\", lang=Languages.FORTRAN)\n return self.compilers[Languages.FORTRAN].package.rpath_arg\n\n @property\n def linker_arg(self) -> str:\n return self._maybe_return_attribute(\"linker_arg\", lang=Languages.C)\n\n @property\n def name(self):\n return next(iter(self.compilers.values())).name\n\n @property\n def version(self):\n return next(iter(self.compilers.values())).version\n\n def implicit_rpaths(self) -> List[str]:\n result, seen = [], set()\n for compiler in self.compilers.values():\n if compiler in seen:\n continue\n seen.add(compiler)\n result.extend(CompilerPropertyDetector(compiler).implicit_rpaths())\n return result\n\n @property\n def opt_flags(self) -> List[str]:\n return next(iter(self.compilers.values())).package.opt_flags\n\n @property\n def debug_flags(self) -> List[str]:\n return next(iter(self.compilers.values())).package.debug_flags\n\n @property\n def openmp_flag(self) -> str:\n return next(iter(self.compilers.values())).package.openmp_flag\n\n @property\n def cxx98_flag(self) -> str:\n return self.compilers[Languages.CXX].package.standard_flag(\n language=Languages.CXX.value, standard=\"98\"\n )\n\n @property\n def cxx11_flag(self) -> str:\n return self.compilers[Languages.CXX].package.standard_flag(\n language=Languages.CXX.value, standard=\"11\"\n )\n\n @property\n def cxx14_flag(self) -> str:\n return self.compilers[Languages.CXX].package.standard_flag(\n language=Languages.CXX.value, standard=\"14\"\n )\n\n @property\n def cxx17_flag(self) -> str:\n return self.compilers[Languages.CXX].package.standard_flag(\n language=Languages.CXX.value, standard=\"17\"\n )\n\n @property\n def cxx20_flag(self) -> str:\n return self.compilers[Languages.CXX].package.standard_flag(\n language=Languages.CXX.value, standard=\"20\"\n )\n\n @property\n def cxx23_flag(self) -> str:\n return self.compilers[Languages.CXX].package.standard_flag(\n language=Languages.CXX.value, standard=\"23\"\n )\n\n @property\n def c99_flag(self) -> str:\n return self.compilers[Languages.C].package.standard_flag(\n language=Languages.C.value, standard=\"99\"\n )\n\n @property\n def c11_flag(self) -> str:\n return self.compilers[Languages.C].package.standard_flag(\n language=Languages.C.value, standard=\"11\"\n )\n\n @property\n def c17_flag(self) -> str:\n return self.compilers[Languages.C].package.standard_flag(\n language=Languages.C.value, standard=\"17\"\n )\n\n @property\n def c23_flag(self) -> str:\n return self.compilers[Languages.C].package.standard_flag(\n language=Languages.C.value, standard=\"23\"\n )\n\n @property\n def cc_pic_flag(self) -> str:\n self._lang_exists_or_raise(\"cc_pic_flag\", lang=Languages.C)\n return self.compilers[Languages.C].package.pic_flag\n\n @property\n def cxx_pic_flag(self) -> str:\n self._lang_exists_or_raise(\"cxx_pic_flag\", lang=Languages.CXX)\n return self.compilers[Languages.CXX].package.pic_flag\n\n @property\n def fc_pic_flag(self) -> str:\n self._lang_exists_or_raise(\"fc_pic_flag\", lang=Languages.FORTRAN)\n return self.compilers[Languages.FORTRAN].package.pic_flag\n\n @property\n def f77_pic_flag(self) -> str:\n self._lang_exists_or_raise(\"f77_pic_flag\", lang=Languages.FORTRAN)\n return self.compilers[Languages.FORTRAN].package.pic_flag\n\n @property\n def prefix(self) -> str:\n return next(iter(self.compilers.values())).prefix\n\n @property\n def extra_rpaths(self) -> List[str]:\n compiler = next(iter(self.compilers.values()))\n return getattr(compiler, \"extra_attributes\", {}).get(\"extra_rpaths\", [])\n\n @property\n def cc(self):\n return self._maybe_return_attribute(\"cc\", lang=Languages.C)\n\n @property\n def cxx(self):\n return self._maybe_return_attribute(\"cxx\", lang=Languages.CXX)\n\n @property\n def fc(self):\n self._lang_exists_or_raise(\"fc\", lang=Languages.FORTRAN)\n return self.compilers[Languages.FORTRAN].package.fortran\n\n @property\n def f77(self):\n self._lang_exists_or_raise(\"f77\", lang=Languages.FORTRAN)\n return self.compilers[Languages.FORTRAN].package.fortran\n\n @property\n def stdcxx_libs(self):\n return self._maybe_return_attribute(\"stdcxx_libs\", lang=Languages.CXX)\n\n\nclass DeprecatedCompiler(lang.DeprecatedProperty):\n def __init__(self) -> None:\n super().__init__(name=\"compiler\")\n\n def factory(self, instance, owner) -> CompilerAdaptor:\n spec = instance.spec\n if not spec.concrete:\n raise ValueError(\"Can only get a compiler for a concrete package.\")\n\n compilers = {}\n for language in Languages:\n deps = spec.dependencies(virtuals=[language.value])\n if deps:\n compilers[language] = deps[0]\n\n return CompilerAdaptor(instance, compilers)\n" }, { "path": "lib/spack/spack/compilers/config.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"This module contains functions related to finding compilers on the system,\nand configuring Spack to use multiple compilers.\n\"\"\"\n\nimport os\nimport re\nimport sys\nimport warnings\nfrom typing import Any, Dict, List, Optional, Tuple\n\nimport spack.config\nimport spack.detection\nimport spack.detection.path\nimport spack.error\nimport spack.llnl.util.filesystem as fs\nimport spack.llnl.util.lang\nimport spack.llnl.util.tty as tty\nimport spack.platforms\nimport spack.repo\nimport spack.spec\nfrom spack.externals import ExternalSpecsParser, external_spec, extract_dicts_from_configuration\nfrom spack.operating_systems import windows_os\nfrom spack.util.environment import get_path\n\n#: Tag used to identify packages providing a compiler\nCOMPILER_TAG = \"compiler\"\n\n\ndef compiler_config_files():\n config_files = []\n configuration = spack.config.CONFIG\n for scope in configuration.writable_scopes:\n name = scope.name\n\n from_packages_yaml = CompilerFactory.from_packages_yaml(configuration, scope=name)\n if from_packages_yaml:\n config_files.append(configuration.get_config_filename(name, \"packages\"))\n\n return config_files\n\n\ndef add_compiler_to_config(new_compilers, *, scope=None) -> None:\n \"\"\"Add a Compiler object to the configuration, at the required scope.\"\"\"\n by_name: Dict[str, List[spack.spec.Spec]] = {}\n for x in new_compilers:\n by_name.setdefault(x.name, []).append(x)\n\n spack.detection.update_configuration(by_name, buildable=True, scope=scope)\n\n\ndef find_compilers(\n path_hints: Optional[List[str]] = None,\n *,\n scope: Optional[str] = None,\n max_workers: Optional[int] = None,\n) -> List[spack.spec.Spec]:\n \"\"\"Searches for compiler in the paths given as argument. If any new compiler is found, the\n configuration is updated, and the list of new compiler objects is returned.\n\n Args:\n path_hints: list of path hints where to look for. A sensible default based on the ``PATH``\n environment variable will be used if the value is None\n scope: configuration scope to modify\n max_workers: number of processes used to search for compilers\n \"\"\"\n if path_hints is None:\n path_hints = get_path(\"PATH\")\n default_paths = fs.search_paths_for_executables(*path_hints)\n if sys.platform == \"win32\":\n default_paths.extend(windows_os.WindowsOs().compiler_search_paths)\n compiler_pkgs = spack.repo.PATH.packages_with_tags(COMPILER_TAG, full=True)\n\n detected_packages = spack.detection.by_path(\n compiler_pkgs, path_hints=default_paths, max_workers=max_workers\n )\n\n new_compilers = spack.detection.update_configuration(\n detected_packages, buildable=True, scope=scope\n )\n return new_compilers\n\n\ndef select_new_compilers(\n candidates: List[spack.spec.Spec], *, scope: Optional[str] = None\n) -> List[spack.spec.Spec]:\n \"\"\"Given a list of compilers, remove those that are already defined in\n the configuration.\n \"\"\"\n compilers_in_config = all_compilers_from(configuration=spack.config.CONFIG, scope=scope)\n return [c for c in candidates if c not in compilers_in_config]\n\n\ndef supported_compilers() -> List[str]:\n \"\"\"Returns all the currently supported compiler packages\"\"\"\n return sorted(spack.repo.PATH.packages_with_tags(COMPILER_TAG))\n\n\ndef all_compilers(scope: Optional[str] = None, init_config: bool = True) -> List[spack.spec.Spec]:\n \"\"\"Returns all the compilers from the current global configuration.\n\n Args:\n scope: configuration scope from which to extract the compilers. If None, the merged\n configuration is used.\n init_config: if True, search for compilers if none is found in configuration.\n \"\"\"\n compilers = all_compilers_from(configuration=spack.config.CONFIG, scope=scope)\n\n if not compilers and init_config:\n _init_packages_yaml(spack.config.CONFIG, scope=scope)\n compilers = all_compilers_from(configuration=spack.config.CONFIG, scope=scope)\n\n return compilers\n\n\ndef _init_packages_yaml(\n configuration: spack.config.Configuration, *, scope: Optional[str]\n) -> None:\n # Try importing from compilers.yaml\n legacy_compilers = CompilerFactory.from_compilers_yaml(configuration, scope=scope)\n if legacy_compilers:\n by_name: Dict[str, List[spack.spec.Spec]] = {}\n for legacy in legacy_compilers:\n by_name.setdefault(legacy.name, []).append(legacy)\n spack.detection.update_configuration(by_name, buildable=True, scope=scope)\n tty.info(\n \"Compilers have been converted from 'compilers.yaml' and written to \"\n \"'packages.yaml'. Use of 'compilers.yaml' is deprecated, and will be \"\n \"ignored in future versions of Spack\"\n )\n return\n\n # Look for compilers in PATH\n new_compilers = find_compilers(scope=scope)\n if not new_compilers:\n raise NoAvailableCompilerError(\n \"no compiler configured, and Spack cannot find working compilers in PATH\"\n )\n tty.info(\"Compilers have been configured automatically from PATH inspection\")\n\n\ndef all_compilers_from(\n configuration: spack.config.Configuration, scope: Optional[str] = None\n) -> List[spack.spec.Spec]:\n \"\"\"Returns all the compilers from the current global configuration.\n\n Args:\n configuration: configuration to be queried\n scope: configuration scope from which to extract the compilers. If None, the merged\n configuration is used.\n \"\"\"\n compilers = CompilerFactory.from_packages_yaml(configuration, scope=scope)\n return compilers\n\n\nclass CompilerRemover:\n \"\"\"Removes compiler from configuration.\"\"\"\n\n def __init__(self, configuration: spack.config.Configuration) -> None:\n self.configuration = configuration\n self.marked_packages_yaml: List[Tuple[str, Any]] = []\n\n def mark_compilers(self, *, match: str, scope: Optional[str] = None) -> List[spack.spec.Spec]:\n \"\"\"Marks compilers to be removed in configuration, and returns a corresponding list\n of specs.\n\n Args:\n match: constraint that the compiler must match to be removed.\n scope: scope where to remove the compiler. If None, all writeable scopes are checked.\n \"\"\"\n self.marked_packages_yaml = []\n candidate_scopes = [scope]\n if scope is None:\n candidate_scopes = [x.name for x in self.configuration.writable_scopes]\n\n return self._mark_in_packages_yaml(match, candidate_scopes)\n\n def _mark_in_packages_yaml(self, match, candidate_scopes):\n compiler_package_names = supported_compilers()\n all_removals = []\n for current_scope in candidate_scopes:\n packages_yaml = self.configuration.get(\"packages\", scope=current_scope)\n if not packages_yaml:\n continue\n\n removed_from_scope = []\n for name, entry in packages_yaml.items():\n if name not in compiler_package_names:\n continue\n\n externals_config = entry.get(\"externals\", None)\n if not externals_config:\n continue\n\n def _partition_match(external_yaml):\n return not external_spec(external_yaml).satisfies(match)\n\n to_keep, to_remove = spack.llnl.util.lang.stable_partition(\n externals_config, _partition_match\n )\n if not to_remove:\n continue\n\n removed_from_scope.extend(to_remove)\n entry[\"externals\"] = to_keep\n\n if not removed_from_scope:\n continue\n\n self.marked_packages_yaml.append((current_scope, packages_yaml))\n all_removals.extend([external_spec(x) for x in removed_from_scope])\n return all_removals\n\n def flush(self):\n \"\"\"Removes from configuration the specs that have been marked by the previous call\n of ``remove_compilers``.\n \"\"\"\n for scope, packages_yaml in self.marked_packages_yaml:\n self.configuration.set(\"packages\", packages_yaml, scope=scope)\n\n\ndef compilers_for_arch(\n arch_spec: spack.spec.ArchSpec, *, scope: Optional[str] = None\n) -> List[spack.spec.Spec]:\n \"\"\"Returns the compilers that can be used on the input architecture\"\"\"\n compilers = all_compilers_from(spack.config.CONFIG, scope=scope)\n query = f\"platform={arch_spec.platform} target=:{arch_spec.target}\"\n return [x for x in compilers if x.satisfies(query)]\n\n\n_EXTRA_ATTRIBUTES_KEY = \"extra_attributes\"\n\n\ndef name_os_target(spec: spack.spec.Spec) -> Tuple[str, str, str]:\n if not spec.architecture:\n host_platform = spack.platforms.host()\n operating_system = host_platform.operating_system(\"default_os\")\n target = host_platform.target(\"default_target\")\n else:\n target = spec.architecture.target\n if not target:\n target = spack.platforms.host().target(\"default_target\")\n target = target.family\n\n operating_system = spec.os\n if not operating_system:\n host_platform = spack.platforms.host()\n operating_system = host_platform.operating_system(\"default_os\")\n\n return spec.name, str(operating_system), str(target)\n\n\nclass CompilerFactory:\n \"\"\"Class aggregating all ways of constructing a list of compiler specs from config entries.\"\"\"\n\n @staticmethod\n def from_packages_yaml(\n configuration: spack.config.Configuration, *, scope: Optional[str] = None\n ) -> List[spack.spec.Spec]:\n \"\"\"Returns the compiler specs defined in the \"packages\" section of the configuration\"\"\"\n compiler_package_names = supported_compilers()\n packages_yaml = configuration.deepcopy_as_builtin(\"packages\", scope=scope)\n\n init_external_dicts = extract_dicts_from_configuration(packages_yaml)\n init_external_dicts = list(\n x\n for x in init_external_dicts\n if spack.spec.Spec(x[\"spec\"]).name in compiler_package_names\n )\n\n externals_dicts = []\n for current in init_external_dicts:\n if _EXTRA_ATTRIBUTES_KEY not in current:\n header = f\"The external spec '{current['spec']}' cannot be used as a compiler\"\n tty.debug(f\"[{__file__}] {header}: missing the '{_EXTRA_ATTRIBUTES_KEY}' key\")\n continue\n\n externals_dicts.append(current)\n\n external_parser = ExternalSpecsParser(externals_dicts)\n return external_parser.all_specs()\n\n @staticmethod\n def from_legacy_yaml(compiler_dict: Dict[str, Any]) -> List[spack.spec.Spec]:\n \"\"\"Returns a list of external specs, corresponding to a compiler entry\n from compilers.yaml.\n \"\"\"\n result = []\n candidate_paths = [x for x in compiler_dict[\"paths\"].values() if x is not None]\n finder = spack.detection.path.ExecutablesFinder()\n\n for pkg_name in spack.repo.PATH.packages_with_tags(\"compiler\"):\n pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)\n pattern = re.compile(r\"|\".join(finder.search_patterns(pkg=pkg_cls)))\n filtered_paths = [x for x in candidate_paths if pattern.search(os.path.basename(x))]\n try:\n detected = finder.detect_specs(\n pkg=pkg_cls, paths=filtered_paths, repo_path=spack.repo.PATH\n )\n except Exception:\n warnings.warn(\n f\"[{__name__}] cannot detect {pkg_name} from the \"\n f\"following paths: {', '.join(filtered_paths)}\"\n )\n continue\n\n for s in detected:\n for key in (\"flags\", \"environment\", \"extra_rpaths\"):\n if key in compiler_dict:\n s.extra_attributes[key] = compiler_dict[key]\n\n if \"modules\" in compiler_dict:\n s.external_modules = list(compiler_dict[\"modules\"])\n\n result.extend(detected)\n\n return result\n\n @staticmethod\n def from_compilers_yaml(\n configuration: spack.config.Configuration, *, scope: Optional[str] = None\n ) -> List[spack.spec.Spec]:\n \"\"\"Returns the compiler specs defined in the \"compilers\" section of the configuration\"\"\"\n result: List[spack.spec.Spec] = []\n for item in configuration.get(\"compilers\", scope=scope):\n result.extend(CompilerFactory.from_legacy_yaml(item[\"compiler\"]))\n return result\n\n\nclass UnknownCompilerError(spack.error.SpackError):\n def __init__(self, compiler_name):\n super().__init__(f\"Spack doesn't support the requested compiler: {compiler_name}\")\n\n\nclass NoAvailableCompilerError(spack.error.SpackError):\n pass\n" }, { "path": "lib/spack/spack/compilers/error.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport warnings\n\nfrom ..error import SpackAPIWarning, SpackError\n\n\nclass CompilerAccessError(SpackError):\n def __init__(self, compiler, paths):\n super().__init__(\n f\"Compiler '{compiler.spec}' has executables that are missing\"\n f\" or are not executable: {paths}\"\n )\n\n\nclass UnsupportedCompilerFlag(SpackError):\n \"\"\"Raised when a compiler does not support a flag type (e.g. a flag to enforce a\n language standard).\n \"\"\"\n\n def __init__(self, message, long_message=None):\n warnings.warn(\n \"UnsupportedCompilerFlag is deprecated, use CompilerError instead\",\n SpackAPIWarning,\n stacklevel=2,\n )\n" }, { "path": "lib/spack/spack/compilers/flags.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nfrom typing import List, Tuple\n\n\ndef tokenize_flags(flags_values: str, propagate: bool = False) -> List[Tuple[str, bool]]:\n \"\"\"Given a compiler flag specification as a string, this returns a list\n where the entries are the flags. For compiler options which set values\n using the syntax ``-flag value``, this function groups flags and their\n values together. Any token not preceded by a ``-`` is considered the\n value of a prior flag.\"\"\"\n tokens = flags_values.split()\n if not tokens:\n return []\n flag = tokens[0]\n flags_with_propagation = []\n for token in tokens[1:]:\n if not token.startswith(\"-\"):\n flag += \" \" + token\n else:\n flags_with_propagation.append((flag, propagate))\n flag = token\n flags_with_propagation.append((flag, propagate))\n return flags_with_propagation\n" }, { "path": "lib/spack/spack/compilers/libraries.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport contextlib\nimport hashlib\nimport json\nimport os\nimport re\nimport shutil\nimport stat\nimport sys\nimport tempfile\nfrom typing import Dict, List, Optional, Set, Tuple, cast\n\nimport spack.caches\nimport spack.llnl.path\nimport spack.llnl.util.lang\nimport spack.schema.environment\nimport spack.spec\nimport spack.util.executable\nimport spack.util.libc\nimport spack.util.module_cmd\nfrom spack.llnl.util import tty\nfrom spack.llnl.util.filesystem import path_contains_subdirectory, paths_containing_libs\nfrom spack.util.environment import filter_system_paths\nfrom spack.util.file_cache import FileCache\n\n#: regex for parsing linker lines\n_LINKER_LINE = re.compile(r\"^( *|.*[/\\\\])\" r\"(link|ld|([^/\\\\]+-)?ld|collect2)\" r\"[^/\\\\]*( |$)\")\n\n#: components of linker lines to ignore\n_LINKER_LINE_IGNORE = re.compile(r\"(collect2 version|^[A-Za-z0-9_]+=|/ldfe )\")\n\n#: regex to match linker search paths\n_LINK_DIR_ARG = re.compile(r\"^-L(.:)?(?P[/\\\\].*)\")\n\n#: regex to match linker library path arguments\n_LIBPATH_ARG = re.compile(r\"^[-/](LIBPATH|libpath):(?P.*)\")\n\n\n@spack.llnl.path.system_path_filter\ndef parse_non_system_link_dirs(compiler_debug_output: str) -> List[str]:\n \"\"\"Parses link paths out of compiler debug output.\n\n Args:\n compiler_debug_output: compiler debug output as a string\n\n Returns:\n Implicit link paths parsed from the compiler output\n \"\"\"\n link_dirs = _parse_link_paths(compiler_debug_output)\n\n # Remove directories that do not exist. Some versions of the Cray compiler\n # report nonexistent directories\n link_dirs = filter_non_existing_dirs(link_dirs)\n\n # Return set of directories containing needed compiler libs, minus\n # system paths. Note that 'filter_system_paths' only checks for an\n # exact match, while 'in_system_subdirectory' checks if a path contains\n # a system directory as a subdirectory\n link_dirs = filter_system_paths(link_dirs)\n return list(p for p in link_dirs if not in_system_subdirectory(p))\n\n\ndef filter_non_existing_dirs(dirs):\n return [d for d in dirs if os.path.isdir(d)]\n\n\ndef in_system_subdirectory(path):\n system_dirs = [\n \"/lib/\",\n \"/lib64/\",\n \"/usr/lib/\",\n \"/usr/lib64/\",\n \"/usr/local/lib/\",\n \"/usr/local/lib64/\",\n ]\n return any(path_contains_subdirectory(path, x) for x in system_dirs)\n\n\ndef _parse_link_paths(string):\n \"\"\"Parse implicit link paths from compiler debug output.\n\n This gives the compiler runtime library paths that we need to add to\n the RPATH of generated binaries and libraries. It allows us to\n ensure, e.g., that codes load the right libstdc++ for their compiler.\n \"\"\"\n lib_search_paths = False\n raw_link_dirs = []\n for line in string.splitlines():\n if lib_search_paths:\n if line.startswith(\"\\t\"):\n raw_link_dirs.append(line[1:])\n continue\n else:\n lib_search_paths = False\n elif line.startswith(\"Library search paths:\"):\n lib_search_paths = True\n\n if not _LINKER_LINE.match(line):\n continue\n if _LINKER_LINE_IGNORE.match(line):\n continue\n tty.debug(f\"implicit link dirs: link line: {line}\")\n\n next_arg = False\n for arg in line.split():\n if arg in (\"-L\", \"-Y\"):\n next_arg = True\n continue\n\n if next_arg:\n raw_link_dirs.append(arg)\n next_arg = False\n continue\n\n link_dir_arg = _LINK_DIR_ARG.match(arg)\n if link_dir_arg:\n link_dir = link_dir_arg.group(\"dir\")\n raw_link_dirs.append(link_dir)\n\n link_dir_arg = _LIBPATH_ARG.match(arg)\n if link_dir_arg:\n link_dir = link_dir_arg.group(\"dir\")\n raw_link_dirs.append(link_dir)\n\n implicit_link_dirs = list()\n visited = set()\n for link_dir in raw_link_dirs:\n normalized_path = os.path.abspath(link_dir)\n if normalized_path not in visited:\n implicit_link_dirs.append(normalized_path)\n visited.add(normalized_path)\n\n tty.debug(f\"implicit link dirs: result: {', '.join(implicit_link_dirs)}\")\n return implicit_link_dirs\n\n\nclass CompilerPropertyDetector:\n \"\"\"Detects compiler properties of a given compiler spec. Useful for compiler wrappers.\"\"\"\n\n def __init__(self, compiler_spec: spack.spec.Spec):\n assert compiler_spec.concrete, \"only concrete compiler specs are allowed\"\n self.spec = compiler_spec\n self.cache = COMPILER_CACHE\n\n @contextlib.contextmanager\n def compiler_environment(self):\n \"\"\"Sets the environment to run this compiler\"\"\"\n\n # No modifications for Spack managed compilers\n if not self.spec.external:\n yield\n return\n\n # Avoid modifying os.environ if possible.\n environment = self.spec.extra_attributes.get(\"environment\", {})\n modules = self.spec.external_modules or []\n if not self.spec.external_modules and not environment:\n yield\n return\n\n # store environment to replace later\n backup_env = os.environ.copy()\n\n try:\n # load modules and set env variables\n for module in modules:\n spack.util.module_cmd.load_module(module)\n\n # apply other compiler environment changes\n spack.schema.environment.parse(environment).apply_modifications()\n\n yield\n finally:\n # Restore environment regardless of whether inner code succeeded\n os.environ.clear()\n os.environ.update(backup_env)\n\n def _compile_dummy_c_source(self) -> Optional[str]:\n compiler_pkg = self.spec.package\n if getattr(compiler_pkg, \"cc\"):\n cc = compiler_pkg.cc\n ext = \"c\"\n else:\n cc = compiler_pkg.cxx\n ext = \"cc\"\n\n if not cc or not self.spec.package.verbose_flags:\n return None\n\n try:\n tmpdir = tempfile.mkdtemp(prefix=\"spack-implicit-link-info\")\n fout = os.path.join(tmpdir, \"output\")\n fin = os.path.join(tmpdir, f\"main.{ext}\")\n\n with open(fin, \"w\", encoding=\"utf-8\") as csource:\n csource.write(\n \"int main(int argc, char* argv[]) { (void)argc; (void)argv; return 0; }\\n\"\n )\n cc_exe = spack.util.executable.Executable(cc)\n\n if self.spec.external:\n compiler_flags = self.spec.extra_attributes.get(\"flags\", {})\n for flag_type in [\n \"cflags\" if cc == compiler_pkg.cc else \"cxxflags\",\n \"cppflags\",\n \"ldflags\",\n ]:\n current_flags = compiler_flags.get(flag_type, \"\").strip()\n if current_flags:\n cc_exe.add_default_arg(*current_flags.split(\" \"))\n\n with self.compiler_environment():\n return cc_exe(\"-v\", fin, \"-o\", fout, output=str, error=str)\n except spack.util.executable.ProcessError as pe:\n tty.debug(f\"ProcessError: Command exited with non-zero status: {pe.long_message}\")\n return None\n finally:\n shutil.rmtree(tmpdir, ignore_errors=True)\n\n def compiler_verbose_output(self) -> Optional[str]:\n \"\"\"Get the compiler verbose output from the cache or by compiling a dummy C source.\"\"\"\n return self.cache.get(self.spec).c_compiler_output\n\n def default_dynamic_linker(self) -> Optional[str]:\n \"\"\"Determine the default dynamic linker path from the compiler verbose output.\"\"\"\n output = self.compiler_verbose_output()\n\n if not output:\n return None\n\n return spack.util.libc.parse_dynamic_linker(output)\n\n def default_libc(self) -> Optional[spack.spec.Spec]:\n \"\"\"Determine libc targeted by the compiler from link line\"\"\"\n # technically this should be testing the target platform of the compiler, but we don't have\n # that, so stick to host platform for now.\n if sys.platform in (\"darwin\", \"win32\"):\n return None\n\n dynamic_linker = self.default_dynamic_linker()\n\n if dynamic_linker is None:\n return None\n\n return spack.util.libc.libc_from_dynamic_linker(dynamic_linker)\n\n def implicit_rpaths(self) -> List[str]:\n \"\"\"Obtain the implicit rpaths to be added from the default ``-L`` link directories,\n excluding system directories.\"\"\"\n output = self.compiler_verbose_output()\n if output is None:\n return []\n\n link_dirs = parse_non_system_link_dirs(output)\n all_required_libs = list(self.spec.package.implicit_rpath_libs) + [\n \"libc\",\n \"libc++\",\n \"libstdc++\",\n ]\n dynamic_linker = self.default_dynamic_linker()\n result = DefaultDynamicLinkerFilter(dynamic_linker)(\n paths_containing_libs(link_dirs, all_required_libs)\n )\n return list(result)\n\n\nclass DefaultDynamicLinkerFilter:\n \"\"\"Remove rpaths to directories that are default search paths of the dynamic linker.\"\"\"\n\n _CACHE: Dict[Optional[str], Set[Tuple[int, int]]] = {}\n\n def __init__(self, dynamic_linker: Optional[str]) -> None:\n if dynamic_linker not in DefaultDynamicLinkerFilter._CACHE:\n # Identify directories by (inode, device) tuple, which handles symlinks too.\n default_path_identifiers: Set[Tuple[int, int]] = set()\n if not dynamic_linker:\n self.default_path_identifiers = None\n return\n for path in spack.util.libc.default_search_paths_from_dynamic_linker(dynamic_linker):\n try:\n s = os.stat(path)\n if stat.S_ISDIR(s.st_mode):\n default_path_identifiers.add((s.st_ino, s.st_dev))\n except OSError:\n continue\n\n DefaultDynamicLinkerFilter._CACHE[dynamic_linker] = default_path_identifiers\n\n self.default_path_identifiers = DefaultDynamicLinkerFilter._CACHE[dynamic_linker]\n\n def is_dynamic_loader_default_path(self, p: str) -> bool:\n if self.default_path_identifiers is None:\n return False\n try:\n s = os.stat(p)\n return (s.st_ino, s.st_dev) in self.default_path_identifiers\n except OSError:\n return False\n\n def __call__(self, dirs: List[str]) -> List[str]:\n if not self.default_path_identifiers:\n return dirs\n return [p for p in dirs if not self.is_dynamic_loader_default_path(p)]\n\n\ndef dynamic_linker_filter_for(node: spack.spec.Spec) -> Optional[DefaultDynamicLinkerFilter]:\n compiler = compiler_spec(node)\n if compiler is None:\n return None\n detector = CompilerPropertyDetector(compiler)\n dynamic_linker = detector.default_dynamic_linker()\n if dynamic_linker is None:\n return None\n return DefaultDynamicLinkerFilter(dynamic_linker)\n\n\ndef compiler_spec(node: spack.spec.Spec) -> Optional[spack.spec.Spec]:\n \"\"\"Returns a compiler :class:`~spack.spec.Spec` associated with the node passed as argument.\n\n The function looks for a ``c``, ``cxx``, and ``fortran`` compiler in that order,\n and returns the first found. If the node does not depend on any of these languages,\n it returns :obj:`None`.\n\n Use of this function is *discouraged*, because a single spec can have multiple compilers\n associated with it, and this function only returns one of them. It can be better to refer to\n compilers on a per-language basis, through the language virtuals: ``spec[\"c\"]``,\n ``spec[\"cxx\"]``, and ``spec[\"fortran\"]``.\n \"\"\"\n for language in (\"c\", \"cxx\", \"fortran\"):\n candidates = node.dependencies(virtuals=[language])\n if candidates:\n break\n else:\n return None\n\n return candidates[0]\n\n\nclass CompilerCacheEntry:\n \"\"\"Deserialized cache entry for a compiler\"\"\"\n\n __slots__ = (\"c_compiler_output\",)\n\n def __init__(self, c_compiler_output: Optional[str]):\n self.c_compiler_output = c_compiler_output\n\n @property\n def empty(self) -> bool:\n \"\"\"Sometimes the compiler is temporarily broken, preventing us from getting output. The\n call site determines if that is a problem.\"\"\"\n return self.c_compiler_output is None\n\n @classmethod\n def from_dict(cls, data: Dict[str, Optional[str]]):\n if not isinstance(data, dict):\n raise ValueError(f\"Invalid {cls.__name__} data\")\n c_compiler_output = data.get(\"c_compiler_output\")\n if not isinstance(c_compiler_output, (str, type(None))):\n raise ValueError(f\"Invalid {cls.__name__} data\")\n return cls(c_compiler_output)\n\n\nclass CompilerCache:\n \"\"\"Base class for compiler output cache. Default implementation does not cache anything.\"\"\"\n\n def value(self, compiler: spack.spec.Spec) -> Dict[str, Optional[str]]:\n return {\"c_compiler_output\": CompilerPropertyDetector(compiler)._compile_dummy_c_source()}\n\n def get(self, compiler: spack.spec.Spec) -> CompilerCacheEntry:\n return CompilerCacheEntry.from_dict(self.value(compiler))\n\n\nclass FileCompilerCache(CompilerCache):\n \"\"\"Cache for compiler output, which is used to determine implicit link paths, the default libc\n version, and the compiler version.\"\"\"\n\n name = os.path.join(\"compilers\", \"compilers.json\")\n\n def __init__(self, cache: \"FileCache\") -> None:\n self.cache = cache\n self._data: Dict[str, Dict[str, Optional[str]]] = {}\n\n def _get_entry(self, key: str, *, allow_empty: bool) -> Optional[CompilerCacheEntry]:\n try:\n entry = CompilerCacheEntry.from_dict(self._data[key])\n return entry if allow_empty or not entry.empty else None\n except ValueError:\n del self._data[key]\n except KeyError:\n pass\n return None\n\n def get(self, compiler: spack.spec.Spec) -> CompilerCacheEntry:\n # Cache hit\n with self.cache.read_transaction(self.name) as f:\n if f is not None:\n try:\n self._data = json.loads(f.read())\n if not isinstance(self._data, dict):\n self._data = {}\n except json.JSONDecodeError:\n self._data = {}\n else:\n self._data = {}\n\n key = self._key(compiler)\n value = self._get_entry(key, allow_empty=False)\n if value is not None:\n return value\n\n # Cache miss\n with self.cache.write_transaction(self.name) as (old, new):\n if old is not None:\n try:\n self._data = json.loads(old.read())\n if not isinstance(self._data, dict):\n self._data = {}\n except json.JSONDecodeError:\n self._data = {}\n else:\n self._data = {}\n\n # Use cache entry that may have been created by another process in the meantime.\n entry = self._get_entry(key, allow_empty=True)\n\n # Finally compute the cache entry\n if entry is None:\n self._data[key] = self.value(compiler)\n entry = CompilerCacheEntry.from_dict(self._data[key])\n\n new.write(json.dumps(self._data, separators=(\",\", \":\")))\n\n return entry\n\n def _key(self, compiler: spack.spec.Spec) -> str:\n as_bytes = json.dumps(compiler.to_dict(), separators=(\",\", \":\")).encode(\"utf-8\")\n return hashlib.sha256(as_bytes).hexdigest()\n\n\ndef _make_compiler_cache():\n return FileCompilerCache(spack.caches.MISC_CACHE)\n\n\nCOMPILER_CACHE = cast(CompilerCache, spack.llnl.util.lang.Singleton(_make_compiler_cache))\n" }, { "path": "lib/spack/spack/concretize.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"High-level functions to concretize list of specs\"\"\"\n\nimport importlib\nimport sys\nimport time\nfrom typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Sequence, Tuple, Union\n\nimport spack.compilers\nimport spack.compilers.config\nimport spack.config\nimport spack.error\nimport spack.llnl.util.tty as tty\nimport spack.repo\nimport spack.util.parallel\nfrom spack.spec import ArchSpec, CompilerSpec, Spec\n\nSpecPairInput = Tuple[Spec, Optional[Spec]]\nSpecPair = Tuple[Spec, Spec]\nTestsType = Union[bool, Iterable[str]]\n\nif TYPE_CHECKING:\n from spack.solver.reuse import SpecFiltersFactory\n\n\ndef _concretize_specs_together(\n abstract_specs: Sequence[Spec],\n *,\n tests: TestsType = False,\n factory: Optional[\"SpecFiltersFactory\"] = None,\n) -> List[Spec]:\n \"\"\"Given a number of specs as input, tries to concretize them together.\n\n Args:\n abstract_specs: abstract specs to be concretized\n tests: list of package names for which to consider tests dependencies. If True, all nodes\n will have test dependencies. If False, test dependencies will be disregarded.\n factory: optional factory to produce a list of specs to be reused\n \"\"\"\n from spack.solver.asp import Solver\n\n allow_deprecated = spack.config.get(\"config:deprecated\", False)\n result = Solver(specs_factory=factory).solve(\n abstract_specs, tests=tests, allow_deprecated=allow_deprecated\n )\n return [s.copy() for s in result.specs]\n\n\ndef concretize_together(\n spec_list: Sequence[SpecPairInput],\n *,\n tests: TestsType = False,\n factory: Optional[\"SpecFiltersFactory\"] = None,\n) -> List[SpecPair]:\n \"\"\"Given a number of specs as input, tries to concretize them together.\n\n Args:\n spec_list: list of tuples to concretize. First entry is abstract spec, second entry is\n already concrete spec or None if not yet concretized\n tests: list of package names for which to consider tests dependencies. If True, all nodes\n will have test dependencies. If False, test dependencies will be disregarded.\n factory: optional factory to produce a list of specs to be reused\n \"\"\"\n to_concretize = [concrete if concrete else abstract for abstract, concrete in spec_list]\n abstract_specs = [abstract for abstract, _ in spec_list]\n concrete_specs = _concretize_specs_together(to_concretize, tests=tests, factory=factory)\n return list(zip(abstract_specs, concrete_specs))\n\n\ndef concretize_together_when_possible(\n spec_list: Sequence[SpecPairInput],\n *,\n tests: TestsType = False,\n factory: Optional[\"SpecFiltersFactory\"] = None,\n) -> List[SpecPair]:\n \"\"\"Given a number of specs as input, tries to concretize them together to the extent possible.\n\n See documentation for ``unify: when_possible`` concretization for the precise definition of\n \"to the extent possible\".\n\n Args:\n spec_list: list of tuples to concretize. First entry is abstract spec, second entry is\n already concrete spec or None if not yet concretized\n tests: list of package names for which to consider tests dependencies. If True, all nodes\n will have test dependencies. If False, test dependencies will be disregarded.\n factory: optional factory to produce a list of specs to be reused\n \"\"\"\n from spack.solver.asp import Solver\n\n to_concretize = [concrete if concrete else abstract for abstract, concrete in spec_list]\n old_concrete_to_abstract = {\n concrete: abstract for (abstract, concrete) in spec_list if concrete\n }\n\n result_by_user_spec: Dict[Spec, Spec] = {}\n allow_deprecated = spack.config.get(\"config:deprecated\", False)\n j = 0\n start = time.monotonic()\n for result in Solver(specs_factory=factory).solve_in_rounds(\n to_concretize, tests=tests, allow_deprecated=allow_deprecated\n ):\n now = time.monotonic()\n duration = now - start\n percentage = int((j + 1) / len(to_concretize) * 100)\n for abstract, concrete in result.specs_by_input.items():\n tty.verbose(\n f\"{duration:6.1f}s [{percentage:3d}%] {concrete.cformat('{hash:7}')} \"\n f\"{abstract.colored_str}\"\n )\n j += 1\n sys.stdout.flush()\n result_by_user_spec.update(result.specs_by_input)\n start = now\n\n # If the \"abstract\" spec is a concrete spec from the previous concretization\n # translate it back to an abstract spec. Otherwise, keep the abstract spec\n return [\n (old_concrete_to_abstract.get(abstract, abstract), concrete)\n for abstract, concrete in sorted(result_by_user_spec.items())\n ]\n\n\ndef concretize_separately(\n spec_list: Sequence[SpecPairInput],\n *,\n tests: TestsType = False,\n factory: Optional[\"SpecFiltersFactory\"] = None,\n) -> List[SpecPair]:\n \"\"\"Concretizes the input specs separately from each other.\n\n Args:\n spec_list: list of tuples to concretize. First entry is abstract spec, second entry is\n already concrete spec or None if not yet concretized\n tests: list of package names for which to consider tests dependencies. If True, all nodes\n will have test dependencies. If False, test dependencies will be disregarded.\n factory: optional factory to produce a list of specs to be reused\n \"\"\"\n from spack.bootstrap import (\n ensure_bootstrap_configuration,\n ensure_clingo_importable_or_raise,\n ensure_winsdk_external_or_raise,\n )\n\n to_concretize = [abstract for abstract, concrete in spec_list if not concrete]\n args = [\n (i, str(abstract), tests, factory)\n for i, abstract in enumerate(to_concretize)\n if not abstract.concrete\n ]\n ret = [(i, abstract) for i, abstract in enumerate(to_concretize) if abstract.concrete]\n try:\n # Ensure we don't try to bootstrap clingo in parallel\n importlib.import_module(\"clingo\")\n except ImportError:\n with ensure_bootstrap_configuration():\n ensure_clingo_importable_or_raise()\n\n # ensure we don't try to detect winsdk in parallel\n if sys.platform == \"win32\":\n ensure_winsdk_external_or_raise()\n\n # Ensure all the indexes have been built or updated, since\n # otherwise the processes in the pool may timeout on waiting\n # for a write lock. We do this indirectly by retrieving the\n # provider index, which should in turn trigger the update of\n # all the indexes if there's any need for that.\n _ = spack.repo.PATH.provider_index\n\n # Ensure we have compilers in packages.yaml to avoid that\n # processes try to write the config file in parallel\n _ = spack.compilers.config.all_compilers()\n\n # Early return if there is nothing to do\n if len(args) == 0:\n # Still have to combine the things that were passed in as abstract with the things\n # that were passed in as pairs\n return [(abstract, concrete) for abstract, (_, concrete) in zip(to_concretize, ret)] + [\n (abstract, concrete) for abstract, concrete in spec_list if concrete\n ]\n\n # Solve the environment in parallel on Linux\n num_procs = min(len(args), spack.config.determine_number_of_jobs(parallel=True))\n\n msg = \"Starting concretization\"\n # no parallel conc on Windows\n if not sys.platform == \"win32\" and num_procs > 1:\n msg += f\" pool with {num_procs} processes\"\n tty.msg(msg)\n\n for j, (i, concrete, duration) in enumerate(\n spack.util.parallel.imap_unordered(\n _concretize_task, args, processes=num_procs, debug=tty.is_debug(), maxtaskperchild=1\n )\n ):\n ret.append((i, concrete))\n percentage = int((j + 1) / len(args) * 100)\n tty.verbose(\n f\"{duration:6.1f}s [{percentage:3d}%] {concrete.cformat('{hash:7}')} \"\n f\"{to_concretize[i].colored_str}\"\n )\n sys.stdout.flush()\n\n # Add specs in original order\n ret.sort(key=lambda x: x[0])\n\n return [(abstract, concrete) for abstract, (_, concrete) in zip(to_concretize, ret)] + [\n (abstract, concrete) for abstract, concrete in spec_list if concrete\n ]\n\n\ndef _concretize_task(\n packed_arguments: Tuple[int, str, TestsType, Optional[\"SpecFiltersFactory\"]],\n) -> Tuple[int, Spec, float]:\n index, spec_str, tests, factory = packed_arguments\n with tty.SuppressOutput(msg_enabled=False):\n start = time.time()\n spec = concretize_one(Spec(spec_str), tests=tests, factory=factory)\n return index, spec, time.time() - start\n\n\ndef concretize_one(\n spec: Union[str, Spec],\n *,\n tests: TestsType = False,\n factory: Optional[\"SpecFiltersFactory\"] = None,\n) -> Spec:\n \"\"\"Return a concretized copy of the given spec.\n\n Args:\n tests: if False disregard test dependencies, if a list of names activate them for\n the packages in the list, if True activate test dependencies for all packages.\n \"\"\"\n from spack.solver.asp import Solver, SpecBuilder\n\n if isinstance(spec, str):\n spec = Spec(spec)\n spec = spec.lookup_hash()\n\n if spec.concrete:\n return spec.copy()\n\n for node in spec.traverse():\n if not node.name:\n raise spack.error.SpecError(\n f\"Spec {node} has no name; cannot concretize an anonymous spec\"\n )\n\n allow_deprecated = spack.config.get(\"config:deprecated\", False)\n result = Solver(specs_factory=factory).solve(\n [spec], tests=tests, allow_deprecated=allow_deprecated\n )\n\n # take the best answer\n opt, i, answer = min(result.answers)\n name = spec.name\n # TODO: Consolidate this code with similar code in solve.py\n if spack.repo.PATH.is_virtual(spec.name):\n providers = [s.name for s in answer.values() if s.package.provides(name)]\n name = providers[0]\n\n node = SpecBuilder.make_node(pkg=name)\n assert node in answer, (\n f\"cannot find {name} in the list of specs {','.join([n.pkg for n in answer.keys()])}\"\n )\n\n concretized = answer[node]\n return concretized\n\n\nclass UnavailableCompilerVersionError(spack.error.SpackError):\n \"\"\"Raised when there is no available compiler that satisfies a\n compiler spec.\"\"\"\n\n def __init__(self, compiler_spec: CompilerSpec, arch: Optional[ArchSpec] = None) -> None:\n err_msg = f\"No compilers with spec {compiler_spec} found\"\n if arch:\n err_msg += f\" for operating system {arch.os} and target {arch.target}.\"\n\n super().__init__(\n err_msg,\n \"Run 'spack compiler find' to add compilers or \"\n \"'spack compilers' to see which compilers are already recognized\"\n \" by spack.\",\n )\n" }, { "path": "lib/spack/spack/config.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"This module implements Spack's configuration file handling.\n\nThis implements Spack's configuration system, which handles merging\nmultiple scopes with different levels of precedence. See the\ndocumentation on :ref:`configuration-scopes` for details on how Spack's\nconfiguration system behaves. The scopes set up here are:\n\n#. ``spack`` in ``$spack/etc/spack`` - controls all built-in spack scopes,\n except default\n#. ``defaults`` in ``$spack/etc/spack/defaults`` - defaults that Spack\n needs to function\n\nImportant functions in this module are:\n\n* :func:`~spack.config.Configuration.get_config`\n* :func:`~spack.config.Configuration.update_config`\n\n``get_config`` reads in YAML data for a particular scope and returns\nit. Callers can then modify the data and write it back with\n``update_config``.\n\nWhen read in, Spack validates configurations with jsonschemas. The\nschemas are in submodules of :py:mod:`spack.schema`.\n\n\"\"\"\n\nimport contextlib\nimport copy\nimport functools\nimport os\nimport os.path\nimport pathlib\nimport re\nimport sys\nimport tempfile\nfrom collections import defaultdict\nfrom itertools import chain\nfrom typing import Any, Callable, Dict, Generator, List, Optional, Set, Tuple, Union, cast\n\nfrom spack.vendor import jsonschema\n\nimport spack.error\nimport spack.paths\nimport spack.schema\nimport spack.schema.bootstrap\nimport spack.schema.cdash\nimport spack.schema.ci\nimport spack.schema.compilers\nimport spack.schema.concretizer\nimport spack.schema.config\nimport spack.schema.definitions\nimport spack.schema.develop\nimport spack.schema.env\nimport spack.schema.env_vars\nimport spack.schema.include\nimport spack.schema.merged\nimport spack.schema.mirrors\nimport spack.schema.modules\nimport spack.schema.packages\nimport spack.schema.repos\nimport spack.schema.toolchains\nimport spack.schema.upstreams\nimport spack.schema.view\nimport spack.util.executable\nimport spack.util.git\nimport spack.util.hash\nimport spack.util.remote_file_cache as rfc_util\nimport spack.util.spack_json as sjson\nimport spack.util.spack_yaml as syaml\nfrom spack.llnl.util import filesystem, lang, tty\nfrom spack.util.cpus import cpus_available\nfrom spack.util.spack_yaml import get_mark_from_yaml_data\n\nfrom .enums import ConfigScopePriority\n\n#: Dict from section names -> schema for that section\nSECTION_SCHEMAS: Dict[str, Any] = {\n \"compilers\": spack.schema.compilers.schema,\n \"concretizer\": spack.schema.concretizer.schema,\n \"definitions\": spack.schema.definitions.schema,\n \"env_vars\": spack.schema.env_vars.schema,\n \"include\": spack.schema.include.schema,\n \"view\": spack.schema.view.schema,\n \"develop\": spack.schema.develop.schema,\n \"mirrors\": spack.schema.mirrors.schema,\n \"repos\": spack.schema.repos.schema,\n \"packages\": spack.schema.packages.schema,\n \"modules\": spack.schema.modules.schema,\n \"config\": spack.schema.config.schema,\n \"upstreams\": spack.schema.upstreams.schema,\n \"bootstrap\": spack.schema.bootstrap.schema,\n \"ci\": spack.schema.ci.schema,\n \"cdash\": spack.schema.cdash.schema,\n \"toolchains\": spack.schema.toolchains.schema,\n}\n\n# Same as above, but including keys for environments\n# this allows us to unify config reading between configs and environments\n_ALL_SCHEMAS: Dict[str, Any] = {\n **SECTION_SCHEMAS,\n spack.schema.env.TOP_LEVEL_KEY: spack.schema.env.schema,\n}\n\n#: Path to the main configuration scope\nCONFIGURATION_DEFAULTS_PATH = (\"defaults\", os.path.join(spack.paths.etc_path, \"defaults\"))\n\n#: Hard-coded default values for some key configuration options.\n#: This ensures that Spack will still work even if config.yaml in\n#: the defaults scope is removed.\nCONFIG_DEFAULTS = {\n \"config\": {\n \"debug\": False,\n \"connect_timeout\": 10,\n \"verify_ssl\": True,\n \"checksum\": True,\n \"dirty\": False,\n \"build_jobs\": min(16, cpus_available()),\n \"build_stage\": \"$tempdir/spack-stage\",\n \"license_dir\": spack.paths.default_license_dir,\n },\n \"concretizer\": {\"externals\": {\"completion\": \"default_variants\"}},\n}\n\n#: metavar to use for commands that accept scopes\n#: this is shorter and more readable than listing all choices\nSCOPES_METAVAR = \"{defaults,system,site,user,command_line} or env:ENVIRONMENT\"\n\n#: Base name for the (internal) overrides scope.\n_OVERRIDES_BASE_NAME = \"overrides-\"\n\n#: Type used for raw YAML configuration\nYamlConfigDict = Dict[str, Any]\n\n#: safeguard for recursive includes -- maximum include depth\nMAX_RECURSIVE_INCLUDES = 100\n\n\nclass ConfigScope:\n def __init__(self, name: str, included: bool = False) -> None:\n self.name = name\n self.writable = False\n self.sections = syaml.syaml_dict()\n self.prefer_modify = False\n self.included = included\n\n #: included configuration scopes\n self._included_scopes: Optional[List[\"ConfigScope\"]] = None\n\n @property\n def included_scopes(self) -> List[\"ConfigScope\"]:\n \"\"\"Memoized list of included scopes, in the order they appear in this scope.\"\"\"\n if self._included_scopes is None:\n self._included_scopes = []\n\n includes = self.get_section(\"include\")\n if includes:\n include_paths = [included_path(data) for data in includes[\"include\"]]\n included_scopes = chain(*[include.scopes(self) for include in include_paths])\n\n # Do not include duplicate scopes\n for included_scope in included_scopes:\n if any([included_scope.name == scope.name for scope in self._included_scopes]):\n tty.warn(f\"Ignoring duplicate included scope: {included_scope.name}\")\n continue\n\n if included_scope not in self._included_scopes:\n self._included_scopes.append(included_scope)\n\n return self._included_scopes\n\n @property\n def exists(self) -> bool:\n \"\"\"Whether the config object indicated by the scope can be read\"\"\"\n return True\n\n def override_include(self):\n \"\"\"Whether the ``include::`` section of this scope should override lower scopes.\"\"\"\n include = self.sections.get(\"include\")\n if not include:\n return False\n\n # override if this has an include section and there is an override attribute on\n # the include key in the dict and it is set to True.\n return getattr(next(iter(include.keys()), None), \"override\", False)\n\n def transitive_includes(self, _names: Optional[Set[str]] = None) -> Set[str]:\n \"\"\"Get name of this scope and names of its transitively included scopes.\"\"\"\n if _names is None:\n _names = _set()\n _names.add(self.name)\n for scope in self.included_scopes:\n _names |= scope.transitive_includes(_names=_names)\n return _names\n\n def get_section_filename(self, section: str) -> str:\n raise NotImplementedError\n\n def get_section(self, section: str) -> Optional[YamlConfigDict]:\n raise NotImplementedError\n\n def _write_section(self, section: str) -> None:\n raise NotImplementedError\n\n def clear(self) -> None:\n \"\"\"Empty cached config information.\"\"\"\n self.sections = syaml.syaml_dict()\n\n def __repr__(self) -> str:\n return f\"\"\n\n\nclass DirectoryConfigScope(ConfigScope):\n \"\"\"Config scope backed by a directory containing one file per section.\"\"\"\n\n def __init__(\n self,\n name: str,\n path: str,\n *,\n writable: bool = True,\n prefer_modify: bool = True,\n included: bool = False,\n ) -> None:\n super().__init__(name, included)\n self.path = path\n self.writable = writable\n self.prefer_modify = prefer_modify\n\n @property\n def exists(self) -> bool:\n return os.path.exists(self.path)\n\n def get_section_filename(self, section: str) -> str:\n \"\"\"Returns the filename associated with a given section\"\"\"\n _validate_section_name(section)\n return os.path.join(self.path, f\"{section}.yaml\")\n\n def get_section(self, section: str) -> Optional[YamlConfigDict]:\n \"\"\"Returns the data associated with a given section if the scope exists\"\"\"\n if not self.exists:\n tty.debug(f\"Attempting to read from missing scope: {self} at {self.path}\")\n return {}\n return self._get_section(section)\n\n def _get_section(self, section: str) -> Optional[YamlConfigDict]:\n \"\"\"get_section but without the existence check\"\"\"\n if section not in self.sections:\n path = self.get_section_filename(section)\n schema = SECTION_SCHEMAS[section]\n data = read_config_file(path, schema)\n self.sections[section] = data\n return self.sections[section]\n\n def _write_section(self, section: str) -> None:\n if not self.writable:\n raise spack.error.ConfigError(f\"Cannot write to immutable scope {self}\")\n\n filename = self.get_section_filename(section)\n data = self._get_section(section)\n if data is None:\n return\n\n validate(data, SECTION_SCHEMAS[section])\n\n try:\n filesystem.mkdirp(self.path)\n fd, tmp = tempfile.mkstemp(dir=self.path, suffix=\".tmp\")\n try:\n with os.fdopen(fd, \"w\", encoding=\"utf-8\") as f:\n syaml.dump_config(data, stream=f, default_flow_style=False)\n filesystem.rename(tmp, filename)\n except Exception:\n os.unlink(tmp)\n raise\n except (syaml.SpackYAMLError, OSError) as e:\n raise ConfigFileError(f\"cannot write to '{filename}'\") from e\n\n\nclass SingleFileScope(ConfigScope):\n \"\"\"This class represents a configuration scope in a single YAML file.\"\"\"\n\n def __init__(\n self,\n name: str,\n path: str,\n schema: YamlConfigDict,\n *,\n yaml_path: Optional[List[str]] = None,\n writable: bool = True,\n prefer_modify: bool = True,\n included: bool = False,\n ) -> None:\n \"\"\"Similar to ``ConfigScope`` but can be embedded in another schema.\n\n Arguments:\n schema (dict): jsonschema for the file to read\n yaml_path (list): path in the schema where config data can be\n found.\n\n If the schema accepts the following yaml data, the yaml_path\n would be ['outer', 'inner']\n\n .. code-block:: yaml\n\n outer:\n inner:\n config:\n install_tree: $spack/opt/spack\n \"\"\"\n super().__init__(name, included)\n self._raw_data: Optional[YamlConfigDict] = None\n self.schema = schema\n self.path = path\n self.writable = writable\n self.prefer_modify = prefer_modify\n self.yaml_path = yaml_path or []\n\n @property\n def exists(self) -> bool:\n return os.path.exists(self.path)\n\n def get_section_filename(self, section) -> str:\n return self.path\n\n def get_section(self, section: str) -> Optional[YamlConfigDict]:\n # read raw data from the file, which looks like:\n # {\n # 'config': {\n # ... data ...\n # },\n # 'packages': {\n # ... data ...\n # },\n # }\n #\n # To preserve overrides up to the section level (e.g. to override\n # the \"packages\" section with the \"::\" syntax), data in self.sections\n # looks like this:\n # {\n # 'config': {\n # 'config': {\n # ... data ...\n # }\n # },\n # 'packages': {\n # 'packages': {\n # ... data ...\n # }\n # }\n # }\n\n if not self.exists:\n tty.debug(f\"Attempting to read from missing scope: {self} at {self.path}\")\n return {}\n\n # This bit ensures we have read the file and have\n # the raw data in memory\n if self._raw_data is None:\n self._raw_data = read_config_file(self.path, self.schema)\n if self._raw_data is None:\n return None\n\n # Here we know we have the raw data and ensure we\n # populate the sections dictionary, which may be\n # cleared by the clear() method\n if not self.sections:\n section_data = self._raw_data\n for key in self.yaml_path:\n if section_data is None:\n return None\n section_data = section_data[key]\n\n for section_key, data in section_data.items():\n self.sections[section_key] = {section_key: data}\n\n return self.sections.get(section, None)\n\n def _write_section(self, section: str) -> None:\n if not self.writable:\n raise spack.error.ConfigError(f\"Cannot write to immutable scope {self}\")\n data_to_write: Optional[YamlConfigDict] = self._raw_data\n\n # If there is no existing data, this section SingleFileScope has never\n # been written to disk. We need to construct the portion of the data\n # from the root of self._raw_data to the level at which the config\n # sections are defined. That requires creating keys for every entry in\n # self.yaml_path\n if not data_to_write:\n data_to_write = {}\n # reverse because we construct it from the inside out\n for key in reversed(self.yaml_path):\n data_to_write = {key: data_to_write}\n\n # data_update_pointer is a pointer to the part of data_to_write\n # that we are currently updating.\n # We start by traversing into the data to the point at which the\n # config sections are defined. This means popping the keys from\n # self.yaml_path\n data_update_pointer = data_to_write\n for key in self.yaml_path:\n data_update_pointer = data_update_pointer[key]\n\n # For each section, update the data at the level of our pointer\n # with the data from the section\n for key, data in self.sections.items():\n data_update_pointer[key] = data[key]\n\n validate(data_to_write, self.schema)\n try:\n parent = os.path.dirname(self.path)\n filesystem.mkdirp(parent)\n fd, tmp = tempfile.mkstemp(dir=parent, suffix=\".tmp\")\n try:\n with os.fdopen(fd, \"w\", encoding=\"utf-8\") as f:\n syaml.dump_config(data_to_write, stream=f, default_flow_style=False)\n filesystem.rename(tmp, self.path)\n except Exception:\n os.unlink(tmp)\n raise\n except (syaml.SpackYAMLError, OSError) as e:\n raise ConfigFileError(f\"cannot write to config file {str(e)}\") from e\n\n def __repr__(self) -> str:\n return f\"\"\n\n\nclass InternalConfigScope(ConfigScope):\n \"\"\"An internal configuration scope that is not persisted to a file.\n\n This is for spack internal use so that command-line options and\n config file settings are accessed the same way, and Spack can easily\n override settings from files.\n \"\"\"\n\n def __init__(self, name: str, data: Optional[YamlConfigDict] = None) -> None:\n super().__init__(name)\n self.sections = syaml.syaml_dict()\n\n if data is not None:\n data = InternalConfigScope._process_dict_keyname_overrides(data)\n for section in data:\n dsec = data[section]\n validate({section: dsec}, SECTION_SCHEMAS[section])\n self.sections[section] = _mark_internal(syaml.syaml_dict({section: dsec}), name)\n\n def get_section(self, section: str) -> Optional[YamlConfigDict]:\n \"\"\"Just reads from an internal dictionary.\"\"\"\n if section not in self.sections:\n self.sections[section] = None\n return self.sections[section]\n\n def _write_section(self, section: str) -> None:\n \"\"\"This only validates, as the data is already in memory.\"\"\"\n data = self.get_section(section)\n if data is not None:\n validate(data, SECTION_SCHEMAS[section])\n self.sections[section] = _mark_internal(data, self.name)\n\n def __repr__(self) -> str:\n return f\"\"\n\n def clear(self) -> None:\n # no cache to clear here.\n pass\n\n @staticmethod\n def _process_dict_keyname_overrides(data: YamlConfigDict) -> YamlConfigDict:\n \"\"\"Turn a trailing `:' in a key name into an override attribute.\"\"\"\n # Below we have a lot of type directives, since we hack on types and monkey-patch them\n # by adding attributes that otherwise they won't have.\n result: YamlConfigDict = {}\n for sk, sv in data.items():\n if sk.endswith(\":\"):\n key = syaml.syaml_str(sk[:-1])\n key.override = True # type: ignore[attr-defined]\n elif sk.endswith(\"+\"):\n key = syaml.syaml_str(sk[:-1])\n key.prepend = True # type: ignore[attr-defined]\n elif sk.endswith(\"-\"):\n key = syaml.syaml_str(sk[:-1])\n key.append = True # type: ignore[attr-defined]\n else:\n key = sk # type: ignore[assignment]\n\n if isinstance(sv, dict):\n result[key] = InternalConfigScope._process_dict_keyname_overrides(sv)\n else:\n result[key] = copy.copy(sv)\n\n return result\n\n\ndef _config_mutator(method):\n \"\"\"Decorator to mark all the methods in the Configuration class\n that mutate the underlying configuration. Used to clear the\n memoization cache.\n \"\"\"\n\n @functools.wraps(method)\n def _method(self, *args, **kwargs):\n self._get_config_memoized.cache_clear()\n return method(self, *args, **kwargs)\n\n return _method\n\n\nScopeWithOptionalPriority = Union[ConfigScope, Tuple[int, ConfigScope]]\nScopeWithPriority = Tuple[int, ConfigScope]\n\n\nclass Configuration:\n \"\"\"A hierarchical configuration, merging a number of scopes at different priorities.\"\"\"\n\n # convert to typing.OrderedDict when we drop 3.6, or OrderedDict when we reach 3.9\n scopes: lang.PriorityOrderedMapping[str, ConfigScope]\n\n def __init__(self) -> None:\n self.scopes = lang.PriorityOrderedMapping()\n self.updated_scopes_by_section: Dict[str, List[ConfigScope]] = defaultdict(list)\n\n def ensure_unwrapped(self) -> \"Configuration\":\n \"\"\"Ensure we unwrap this object from any dynamic wrapper (like Singleton)\"\"\"\n return self\n\n def highest(self) -> ConfigScope:\n \"\"\"Scope with the highest precedence\"\"\"\n return next(self.scopes.reversed_values()) # type: ignore\n\n @_config_mutator\n def push_scope_incremental(\n self, scope: ConfigScope, priority: Optional[int] = None, _depth: int = 0\n ) -> Generator[\"Configuration\", None, None]:\n \"\"\"Adds a scope to the Configuration, at a given priority.\n\n ``push_scope_incremental`` yields included scopes incrementally, so that their\n data can be used by higher priority scopes during config initialization. If you\n push a scope that includes other, low-priority scopes, they will be pushed on\n first, before the scope that included them.\n\n If a priority is not given, it is assumed to be the current highest priority.\n\n Args:\n scope: scope to be added\n priority: priority of the scope\n\n \"\"\"\n # TODO: As a follow on to #48784, change this to create a graph of the\n # TODO: includes AND ensure properly sorted such that the order included\n # TODO: at the highest level is reflected in the value of an option that\n # TODO: is set in multiple included files.\n # before pushing the scope itself, push included scopes recursively, at the same priority\n for included_scope in reversed(scope.included_scopes):\n if _depth + 1 > MAX_RECURSIVE_INCLUDES: # make sure we're not recursing endlessly\n mark = \"\"\n if hasattr(included_scope, \"path\") and syaml.marked(included_scope.path):\n mark = included_scope.path._start_mark # type: ignore\n raise RecursiveIncludeError(\n f\"Maximum include recursion exceeded in {included_scope.name}\", str(mark)\n )\n\n # record this inclusion so that remove_scope() can use it\n self.push_scope(included_scope, priority=priority, _depth=_depth + 1)\n yield self\n\n tty.debug(f\"[CONFIGURATION: PUSH SCOPE]: {str(scope)}, priority={priority}\", level=2)\n self.scopes.add(scope.name, value=scope, priority=priority)\n yield self\n\n @_config_mutator\n def push_scope(\n self, scope: ConfigScope, priority: Optional[int] = None, _depth: int = 0\n ) -> None:\n \"\"\"Add a scope to the Configuration, at a given priority.\n\n If a priority is not given, it is assumed to be the current highest priority.\n\n Args:\n scope: scope to be added\n priority: priority of the scope\n\n \"\"\"\n # Use push_scope_incremental to do the real work. It returns a generator, which needs\n # to be consumed to get each of the yielded scopes added to the scope stack.\n # It will usually yield one scope, but if there are includes it will yield those first,\n # before the scope we're actually pushing.\n for _ in self.push_scope_incremental(scope=scope, priority=priority, _depth=_depth):\n pass\n\n @_config_mutator\n def remove_scope(self, scope_name: str) -> Optional[ConfigScope]:\n \"\"\"Removes a scope by name, and returns it. If the scope does not exist, returns None.\"\"\"\n\n try:\n scope = self.scopes.remove(scope_name)\n tty.debug(f\"[CONFIGURATION: REMOVE SCOPE]: {str(scope)}\", level=2)\n except KeyError as e:\n tty.debug(f\"[CONFIGURATION: REMOVE SCOPE]: {e}\", level=2)\n return None\n\n # transitively remove included scopes\n for included_scope in scope.included_scopes:\n assert included_scope.name in self.scopes, (\n f\"Included scope '{included_scope.name}' was never added to configuration!\"\n )\n self.remove_scope(included_scope.name)\n\n return scope\n\n @property\n def writable_scopes(self) -> Generator[ConfigScope, None, None]:\n \"\"\"Generator of writable scopes with an associated file.\"\"\"\n return (s for s in self.scopes.values() if s.writable)\n\n @property\n def existing_scopes(self) -> Generator[ConfigScope, None, None]:\n \"\"\"Generator of existing scopes. These are self.scopes where the\n scope has a representation on the filesystem or is internal\"\"\"\n return (s for s in self.scopes.values() if s.exists)\n\n def highest_precedence_scope(self) -> ConfigScope:\n \"\"\"Writable scope with the highest precedence.\"\"\"\n scope = next(s for s in self.scopes.reversed_values() if s.writable)\n\n # if a scope prefers that we edit another, respect that.\n while scope:\n preferred = scope\n scope = next(\n (s for s in scope.included_scopes if s.writable and s.prefer_modify), None\n )\n\n return preferred\n\n def matching_scopes(self, reg_expr) -> List[ConfigScope]:\n \"\"\"\n List of all scopes whose names match the provided regular expression.\n\n For example, ``matching_scopes(r'^command')`` will return all scopes\n whose names begin with ``command``.\n \"\"\"\n return [s for s in self.scopes.values() if re.search(reg_expr, s.name)]\n\n def _validate_scope(self, scope: Optional[str]) -> ConfigScope:\n \"\"\"Ensure that scope is valid in this configuration.\n\n This should be used by routines in ``config.py`` to validate\n scope name arguments, and to determine a default scope where no\n scope is specified.\n\n Raises:\n ValueError: if ``scope`` is not valid\n\n Returns:\n ConfigScope: a valid ConfigScope if ``scope`` is ``None`` or valid\n \"\"\"\n if scope is None:\n # default to the scope with highest precedence.\n return self.highest_precedence_scope()\n\n elif scope in self.scopes:\n return self.scopes[scope]\n\n else:\n raise ValueError(\n f\"Invalid config scope: '{scope}'. Must be one of \"\n f\"{[k for k in self.scopes.keys()]}\"\n )\n\n def get_config_filename(self, scope: str, section: str) -> str:\n \"\"\"For some scope and section, get the name of the configuration file.\"\"\"\n scope = self._validate_scope(scope)\n return scope.get_section_filename(section)\n\n @_config_mutator\n def clear_caches(self) -> None:\n \"\"\"Clears the caches for configuration files,\n\n This will cause files to be re-read upon the next request.\"\"\"\n for scope in self.scopes.values():\n scope.clear()\n\n @_config_mutator\n def update_config(\n self, section: str, update_data: Dict, scope: Optional[str] = None, force: bool = False\n ) -> None:\n \"\"\"Update the configuration file for a particular scope.\n\n Overwrites contents of a section in a scope with update_data,\n then writes out the config file.\n\n update_data should have the top-level section name stripped off\n (it will be re-added). Data itself can be a list, dict, or any\n other yaml-ish structure.\n\n Configuration scopes that are still written in an old schema\n format will fail to update unless ``force`` is True.\n\n Args:\n section: section of the configuration to be updated\n update_data: data to be used for the update\n scope: scope to be updated\n force: force the update\n \"\"\"\n if self.updated_scopes_by_section.get(section) and not force:\n msg = (\n 'The \"{0}\" section of the configuration needs to be written'\n \" to disk, but is currently using a deprecated format. \"\n \"Please update it using:\\n\\n\"\n \"\\tspack config [--scope=] update {0}\\n\\n\"\n \"Note that previous versions of Spack will not be able to \"\n \"use the updated configuration.\"\n )\n msg = msg.format(section)\n raise RuntimeError(msg)\n\n _validate_section_name(section) # validate section name\n scope = self._validate_scope(scope) # get ConfigScope object\n\n # manually preserve comments\n need_comment_copy = section in scope.sections and scope.sections[section]\n if need_comment_copy:\n comments = syaml.extract_comments(scope.sections[section][section])\n\n # read only the requested section's data.\n scope.sections[section] = syaml.syaml_dict({section: update_data})\n if need_comment_copy and comments:\n syaml.set_comments(scope.sections[section][section], data_comments=comments)\n\n scope._write_section(section)\n\n def get_config(\n self, section: str, scope: Optional[str] = None, _merged_scope: Optional[str] = None\n ) -> YamlConfigDict:\n \"\"\"Get configuration settings for a section.\n\n If ``scope`` is ``None`` or not provided, return the merged contents\n of all of Spack's configuration scopes. If ``scope`` is provided,\n return only the configuration as specified in that scope.\n\n This off the top-level name from the YAML section. That is, for a\n YAML config file that looks like this::\n\n config:\n install_tree:\n root: $spack/opt/spack\n build_stage:\n - $tmpdir/$user/spack-stage\n\n ``get_config('config')`` will return::\n\n { 'install_tree': {\n 'root': '$spack/opt/spack',\n }\n 'build_stage': ['$tmpdir/$user/spack-stage']\n }\n\n \"\"\"\n return self._get_config_memoized(section, scope=scope, _merged_scope=_merged_scope)\n\n def deepcopy_as_builtin(\n self, section: str, scope: Optional[str] = None, *, line_info: bool = False\n ) -> Dict[str, Any]:\n \"\"\"Get a deep copy of a section with native Python types, excluding YAML metadata.\"\"\"\n return syaml.deepcopy_as_builtin(\n self.get_config(section, scope=scope), line_info=line_info\n )\n\n def _filter_overridden(self, scopes: List[ConfigScope], includes: bool = False):\n \"\"\"Filter out overridden scopes.\n\n NOTE: this does not yet handle diamonds or nested `include::` in lists. It is\n sufficient for include::[] in an env, which allows isolation.\n\n The ``includes`` option controls whether to return all active scopes (``includes=False``)\n or all scopes whose includes have not been overridden (``includes=True``).\n \"\"\"\n # find last override in scopes\n i = next((i for i, s in reversed(list(enumerate(scopes))) if s.override_include()), -1)\n if i < 0:\n return scopes # no overrides\n\n keep = _set(s.name for s in scopes[i:])\n keep |= _set(s.name for s in self.scopes.priority_values(ConfigScopePriority.DEFAULTS))\n\n if not includes:\n # For all sections except for the include section:\n # non-included scopes are still active, as are scopes included\n # from the overriding scope\n # Transitive scopes from the overriding scope are not included\n keep |= _set([s.name for s in scopes[i].included_scopes])\n keep |= _set([s.name for s in scopes if not s.included])\n\n # return scopes to keep, with order preserved\n return [s for s in scopes if s.name in keep]\n\n @property\n def active_include_section_scopes(self) -> List[ConfigScope]:\n \"\"\"Return a list of all scopes whose includes have not been overridden by include::.\n\n This is different from the active scopes because the ``spack`` scope can be active\n while its includes are overwritten, as can the transitive includes from the overriding\n scope.\"\"\"\n return self._filter_overridden([s for s in self.scopes.values()], includes=True)\n\n @property\n def active_scopes(self) -> List[ConfigScope]:\n \"\"\"Return a list of scopes that have not been overridden by include::.\"\"\"\n return self._filter_overridden([s for s in self.scopes.values()])\n\n @lang.memoized\n def _get_config_memoized(\n self, section: str, scope: Optional[str], _merged_scope: Optional[str]\n ) -> YamlConfigDict:\n \"\"\"Memoized helper for ``get_config()``.\n\n Note that the memoization cache for this function is cleared whenever\n any function decorated with ``@_config_mutator`` is called.\n \"\"\"\n _validate_section_name(section)\n\n if scope is not None and _merged_scope is not None:\n raise ValueError(\"Cannot specify both scope and _merged_scope\")\n elif scope is not None:\n scopes = [self._validate_scope(scope)]\n elif _merged_scope is not None:\n scope_stack = list(self.scopes.values())\n merge_idx = next(i for i, s in enumerate(scope_stack) if s.name == _merged_scope)\n scopes = scope_stack[: merge_idx + 1]\n else:\n scopes = list(self.scopes.values())\n\n # filter any scopes overridden by `include::`\n scopes = self._filter_overridden(scopes)\n\n merged_section: Dict[str, Any] = syaml.syaml_dict()\n updated_scopes = []\n for config_scope in scopes:\n if section == \"include\" and config_scope not in self.active_include_section_scopes:\n continue\n\n # read potentially cached data from the scope.\n data = config_scope.get_section(section)\n\n if data and section == \"include\":\n # Include overrides are handled by `_filter_overridden` above. Any remaining\n # includes at this point are *not* actually overridden -- they're scopes with\n # ConfigScopePriority.DEFAULT, which we currently do *not* remove with\n # `include::`, because these scopes are needed for Spack to function correctly.\n # So, we ignore :: here.\n data = data.copy()\n data[\"include\"] = data.pop(\"include\") # strip override\n\n # Skip empty configs\n if not isinstance(data, dict) or section not in data:\n continue\n\n # If configuration is in an old format, transform it and keep track of the scope that\n # may need to be written out to disk.\n if _update_in_memory(data, section):\n updated_scopes.append(config_scope)\n\n merged_section = spack.schema.merge_yaml(merged_section, data)\n\n self.updated_scopes_by_section[section] = updated_scopes\n\n # no config files -- empty config.\n if section not in merged_section:\n return syaml.syaml_dict()\n\n # take the top key off before returning.\n ret = merged_section[section]\n if isinstance(ret, dict):\n ret = syaml.syaml_dict(ret)\n return ret\n\n def get(self, path: str, default: Optional[Any] = None, scope: Optional[str] = None) -> Any:\n \"\"\"Get a config section or a single value from one.\n\n Accepts a path syntax that allows us to grab nested config map\n entries. Getting the ``config`` section would look like::\n\n spack.config.get(\"config\")\n\n and the ``dirty`` section in the ``config`` scope would be::\n\n spack.config.get(\"config:dirty\")\n\n We use ``:`` as the separator, like YAML objects.\n \"\"\"\n parts = process_config_path(path)\n section = parts.pop(0)\n\n value = self.get_config(section, scope=scope)\n\n while parts:\n key = parts.pop(0)\n # cannot use value.get(key, default) in case there is another part\n # and default is not a dict\n if key not in value:\n return default\n value = value[key]\n\n return value\n\n @_config_mutator\n def set(self, path: str, value: Any, scope: Optional[str] = None) -> None:\n \"\"\"Convenience function for setting single values in config files.\n\n Accepts the path syntax described in ``get()``.\n \"\"\"\n if \":\" not in path:\n # handle bare section name as path\n self.update_config(path, value, scope=scope)\n return\n\n parts = process_config_path(path)\n section = parts.pop(0)\n\n section_data = self.get_config(section, scope=scope)\n\n data = section_data\n while len(parts) > 1:\n key = parts.pop(0)\n\n if spack.schema.override(key):\n new = type(data[key])()\n del data[key]\n else:\n new = data[key]\n\n if isinstance(new, dict):\n # Make it an ordered dict\n new = syaml.syaml_dict(new)\n # reattach to parent object\n data[key] = new\n data = new\n\n if spack.schema.override(parts[0]):\n data.pop(parts[0], None)\n\n # update new value\n data[parts[0]] = value\n\n self.update_config(section, section_data, scope=scope)\n\n def __iter__(self):\n \"\"\"Iterate over scopes in this configuration.\"\"\"\n yield from self.scopes.values()\n\n def print_section(\n self, section: str, yaml: bool = True, blame: bool = False, *, scope: Optional[str] = None\n ) -> None:\n \"\"\"Print a configuration to stdout.\n\n Arguments:\n section: The configuration section to print.\n yaml: If True, output in YAML format, otherwise JSON (ignored when blame is True).\n blame: Whether to include source locations for each entry.\n scope: The configuration scope to use.\n \"\"\"\n try:\n data = syaml.syaml_dict()\n data[section] = self.get_config(section, scope=scope)\n if yaml or blame:\n syaml.dump_config(data, stream=sys.stdout, default_flow_style=False, blame=blame)\n else:\n sjson.dump(data, sys.stdout)\n sys.stdout.write(\"\\n\")\n\n except (syaml.SpackYAMLError, OSError) as e:\n raise spack.error.ConfigError(f\"cannot read '{section}' configuration\") from e\n\n\n@contextlib.contextmanager\ndef override(\n path_or_scope: Union[ConfigScope, str], value: Optional[Any] = None\n) -> Generator[Configuration, None, None]:\n \"\"\"Simple way to override config settings within a context.\n\n Arguments:\n path_or_scope (ConfigScope or str): scope or single option to override\n value (object or None): value for the single option\n\n Temporarily push a scope on the current configuration, then remove it\n after the context completes. If a single option is provided, create\n an internal config scope for it and push/pop that scope.\n\n \"\"\"\n if isinstance(path_or_scope, ConfigScope):\n overrides = path_or_scope\n CONFIG.push_scope(path_or_scope, priority=None)\n else:\n base_name = _OVERRIDES_BASE_NAME\n # Ensure the new override gets a unique scope name\n current_overrides = [s.name for s in CONFIG.matching_scopes(rf\"^{base_name}\")]\n num_overrides = len(current_overrides)\n while True:\n scope_name = f\"{base_name}{num_overrides}\"\n if scope_name in current_overrides:\n num_overrides += 1\n else:\n break\n\n overrides = InternalConfigScope(scope_name)\n CONFIG.push_scope(overrides, priority=None)\n CONFIG.set(path_or_scope, value, scope=scope_name)\n\n try:\n yield CONFIG\n finally:\n scope = CONFIG.remove_scope(overrides.name)\n assert scope is overrides\n\n\n#: Class for the relevance of an optional path conditioned on a limited\n#: python code that evaluates to a boolean and or explicit specification\n#: as optional.\nclass OptionalInclude:\n \"\"\"Base properties for all includes.\"\"\"\n\n name: str\n when: str\n optional: bool\n prefer_modify: bool\n remote: bool\n _scopes: List[ConfigScope]\n\n def __init__(self, entry: dict):\n self.name = entry.get(\"name\", \"\")\n self.when = entry.get(\"when\", \"\")\n self.optional = entry.get(\"optional\", False)\n self.prefer_modify = entry.get(\"prefer_modify\", False)\n self.remote = False\n self._scopes = []\n\n @staticmethod\n def _parent_scope_directory(parent_scope: Optional[ConfigScope]) -> Optional[str]:\n \"\"\"Return the directory of the parent scope, or ``None`` if unavailable.\n\n Normalizes ``SingleFileScope`` to its containing directory.\n \"\"\"\n path = getattr(parent_scope, \"path\", \"\") if parent_scope else \"\"\n if not path:\n return None\n return os.path.dirname(path) if os.path.isfile(path) else path\n\n def base_directory(\n self, path_or_url: str, parent_scope: Optional[ConfigScope] = None\n ) -> Optional[str]:\n \"\"\"Return the local directory to use for this include.\n\n For remote includes this is the cache destination directory.\n For local relative includes this is the working directory from which to resolve the path.\n\n Args:\n path_or_url: path or URL of the include\n parent_scope: including scope\n\n Returns: ``None`` for a local include without an enclosing parent scope;\n an appropriate subdirectory of the enclosing (parent) scope's writable\n directory (when available); otherwise a stable temporary directory.\n \"\"\"\n scope_dir = self._parent_scope_directory(parent_scope)\n if not self.remote:\n return scope_dir\n\n def _subdir():\n # Prefer the provided include name over the git repository name.\n # If neither, use a hash of the url or path for uniqueness.\n if self.name:\n return self.name\n\n match = re.search(r\"/([^/]+?)(\\.git)?$\", path_or_url)\n if match:\n if not os.path.splitext(match.group(1))[1]:\n return match.group(1)\n\n return spack.util.hash.b32_hash(path_or_url)[-7:]\n\n # For remote includes, prefer a writable subdirectory of the parent scope.\n if scope_dir and filesystem.can_write_to_dir(scope_dir):\n assert parent_scope is not None\n subdir = os.path.join(\"includes\", _subdir())\n if parent_scope.name.startswith(\"env:\"):\n subdir = os.path.join(\".spack-env\", subdir)\n return os.path.join(scope_dir, subdir)\n\n # Fall back to a stable, unique, temporary directory, logging the reason.\n tmpdir = tempfile.gettempdir()\n if path_or_url:\n pre = self.name or getattr(parent_scope, \"name\", \"\")\n subdir = f\"{pre}:{path_or_url}\" if pre else path_or_url\n tmpdir = os.path.join(tmpdir, spack.util.hash.b32_hash(subdir)[-7:])\n\n if not scope_dir:\n tty.debug(f\"No parent scope directory for include ({self}). Using {tmpdir}.\")\n else:\n assert parent_scope is not None\n tty.debug(\n f\"Parent scope {parent_scope.name}'s directory ({scope_dir}) is not writable. \"\n f\"Using {tmpdir}.\"\n )\n return tmpdir\n\n def _scope(\n self, path: str, config_path: str, parent_scope: ConfigScope\n ) -> Optional[ConfigScope]:\n \"\"\"Instantiate a configuration scope for the configuration path.\n\n Args:\n path: raw include path\n config_path: configuration path\n parent_scope: including scope\n\n Returns: configuration scopes\n\n Raises:\n ValueError: the required configuration path does not exist\n \"\"\"\n # circular dependencies\n import spack.util.path\n\n # Ignore included concrete environment files (i.e., ``spack.lock``)\n # since they are not normal configuration (scope) files and their\n # processing is handled when the environment is processed.\n if path and os.path.basename(path) == \"spack.lock\":\n tty.debug(\n f\"Ignoring inclusion of '{path}' since environment lock files \"\n \"are processed elsewhere\"\n )\n return None\n\n # Ensure the parent scope is valid\n self._validate_parent_scope(parent_scope)\n\n # Determine the configuration scope name\n config_name = self.name or parent_scope.name\n\n # But ensure that name is unique if there are multiple paths.\n if not self.name or len(getattr(self, \"paths\", [])) > 1:\n parent_path = pathlib.Path(getattr(parent_scope, \"path\", \"\"))\n real_path = pathlib.Path(spack.util.path.substitute_path_variables(path))\n\n try:\n included_name = real_path.relative_to(parent_path)\n except ValueError:\n included_name = real_path\n\n if sys.platform == \"win32\":\n # Clean windows path for use in config name that looks nicer\n # ie. The path: C:\\\\some\\\\path\\\\to\\\\a\\\\file\n # becomes C/some/path/to/a/file\n included_name = included_name.as_posix().replace(\":\", \"\")\n\n config_name = f\"{config_name}:{included_name}\"\n\n _, ext = os.path.splitext(config_path)\n ext_is_yaml = ext == \".yaml\" or ext == \".yml\"\n is_dir = os.path.isdir(config_path)\n exists = os.path.exists(config_path)\n\n if not exists and not self.optional:\n dest = f\" at ({config_path})\" if config_path != os.path.normpath(path) else \"\"\n raise ValueError(f\"Required path ({path}) does not exist{dest}\")\n\n if (exists and not is_dir) or ext_is_yaml:\n tty.debug(f\"Creating SingleFileScope {config_name} for '{config_path}'\")\n return SingleFileScope(\n config_name,\n config_path,\n spack.schema.merged.schema,\n prefer_modify=self.prefer_modify,\n included=True,\n )\n\n if ext and not is_dir:\n raise ValueError(\n f\"File-based scope does not exist yet: should have a .yaml/.yml extension \\\nfor file scopes, or no extension for directory scopes (currently {ext})\"\n )\n\n # directories are treated as regular ConfigScopes\n # assign by \"default\"\n tty.debug(f\"Creating DirectoryConfigScope {config_name} for '{config_path}'\")\n return DirectoryConfigScope(\n config_name, config_path, prefer_modify=self.prefer_modify, included=True\n )\n\n def _validate_parent_scope(self, parent_scope: ConfigScope):\n \"\"\"Validates that a parent scope is a valid configuration object\"\"\"\n # enforced by type checking but those can always be # type: ignore'd\n assert isinstance(parent_scope, ConfigScope), (\n f\"Includes must be within a configuration scope (ConfigScope), not {type(parent_scope)}\" # noqa: E501\n )\n\n assert parent_scope.name.strip(), \"Parent scope of an include must have a name\"\n\n def evaluate_condition(self) -> bool:\n \"\"\"Evaluate the include condition:\n\n Returns: ``True`` if the include condition is satisfied; else ``False``.\n \"\"\"\n # circular dependencies\n import spack.spec\n\n return (not self.when) or spack.spec.eval_conditional(self.when)\n\n def scopes(self, parent_scope: ConfigScope) -> List[ConfigScope]:\n \"\"\"Instantiate configuration scopes.\n\n Args:\n parent_scope: including scope\n\n Returns: configuration scopes for configuration files IF the when\n condition is satisfied; otherwise, an empty list.\n\n Raises:\n ValueError: the required configuration path does not exist\n \"\"\"\n raise NotImplementedError(\"must be implemented in derived classes\")\n\n @property\n def paths(self) -> List[str]:\n \"\"\"Path(s) associated with the include.\"\"\"\n\n raise NotImplementedError(\"must be implemented in derived classes\")\n\n\nclass IncludePath(OptionalInclude):\n path: str\n sha256: str\n destination: Optional[str]\n\n def __init__(self, entry: dict):\n # circular dependencies\n import spack.util.path\n\n super().__init__(entry)\n path_override_env_var = entry.get(\"path_override_env_var\", \"\")\n if path_override_env_var and path_override_env_var in os.environ:\n path = os.environ[path_override_env_var]\n else:\n path = entry.get(\"path\", \"\")\n self.path = spack.util.path.substitute_path_variables(path)\n\n self.sha256 = entry.get(\"sha256\", \"\")\n self.remote = \"sha256\" in entry\n self.destination = None\n\n def __repr__(self):\n return (\n f\"IncludePath({self.path}, sha256={self.sha256}, \"\n f\"when='{self.when}', optional={self.optional})\"\n )\n\n def scopes(self, parent_scope: ConfigScope) -> List[ConfigScope]:\n \"\"\"Instantiate a configuration scope for the included path.\n\n Args:\n parent_scope: including scope\n\n Returns: configuration scopes IF the when condition is satisfied;\n otherwise, an empty list.\n\n Raises:\n ConfigFileError: unable to access remote configuration file\n ValueError: included path has an unsupported URL scheme, is required\n but does not exist; configuration stage directory argument is missing\n \"\"\"\n if not self.evaluate_condition():\n tty.debug(f\"Include condition is not satisfied in {self}\")\n return []\n\n if self._scopes:\n tty.debug(f\"Using existing scopes: {[s.name for s in self._scopes]}\")\n return self._scopes\n\n # An absolute path does not need a local base directory.\n if os.path.isabs(self.path):\n tty.debug(f\"The included path ({self}) is absolute so needs no base directory\")\n base = None\n else:\n base = self.base_directory(self.path, parent_scope)\n\n # Make sure to use a proper working directory when obtaining the local\n # path for a local (or remote) file.\n tty.debug(f\"Local base directory for {self.path} is {base}\")\n\n config_path = rfc_util.local_path(self.path, self.sha256, base)\n assert config_path\n self.destination = config_path\n\n scope = self._scope(self.path, self.destination, parent_scope)\n if scope is not None:\n self._scopes = [scope]\n\n return self._scopes\n\n @property\n def paths(self) -> List[str]:\n \"\"\"Path(s) associated with the include.\"\"\"\n\n return [self.path]\n\n\nclass GitIncludePaths(OptionalInclude):\n git: str\n branch: str\n commit: str\n tag: str\n _paths: List[str]\n destination: Optional[str]\n\n def __init__(self, entry: dict):\n # circular dependencies\n import spack.util.path\n\n super().__init__(entry)\n self.git = spack.util.path.substitute_path_variables(entry.get(\"git\", \"\"))\n\n self.branch = entry.get(\"branch\", \"\")\n self.commit = entry.get(\"commit\", \"\")\n self.tag = entry.get(\"tag\", \"\")\n self._paths = [\n spack.util.path.substitute_path_variables(path) for path in entry.get(\"paths\", [])\n ]\n self.destination = None\n self.remote = True\n\n if not self.branch and not self.commit and not self.tag:\n raise spack.error.ConfigError(\n \"Git include paths ({self}) must specify one or more of: branch, commit, tag\"\n )\n\n if not self._paths:\n raise spack.error.ConfigError(\n \"Git include paths ({self}) must include one or more relative paths\"\n )\n\n def __repr__(self):\n if self.branch:\n identifier = f\"branch={self.branch}\"\n else:\n identifier = f\"commit={self.commit}, tag={self.tag}\"\n\n return (\n f\"GitIncludePaths('{self.name}', {self.git}, paths={self._paths}, \"\n f\"{identifier}, when='{self.when}', optional={self.optional})\"\n )\n\n def _clone(self, parent_scope: ConfigScope) -> Optional[str]:\n \"\"\"Clone the repository.\n\n Args:\n parent_scope: enclosing scope\n\n Returns: destination path if cloned or ``None``\n \"\"\"\n if self.fetched():\n tty.debug(f\"Repository ({self.git}) already cloned to {self.destination}\")\n return self.destination\n\n # environment includes should be located under the environment\n destination = self.base_directory(self.git, parent_scope)\n assert destination, f\"{self} requires a local cache directory\"\n tty.debug(f\"Cloning {self.git} into {destination}\")\n\n with filesystem.working_dir(destination, create=True):\n if not os.path.exists(\".git\"):\n try:\n tty.debug(\"Initializing the git repository\")\n spack.util.git.init_git_repo(self.git)\n except spack.util.executable.ProcessError as e:\n raise spack.error.ConfigError(\n f\"Unable to initialize repository ({self.git}) under {destination}: {e}\"\n )\n\n try:\n if self.commit:\n tty.debug(f\"Pulling commit {self.commit}\")\n spack.util.git.pull_checkout_commit(self.commit)\n elif self.tag:\n tty.debug(f\"Pulling tag {self.tag}\")\n spack.util.git.pull_checkout_tag(self.tag)\n elif self.branch:\n # if the branch already exists we should use the\n # previously configured remote\n tty.debug(f\"Pulling branch {self.branch}\")\n try:\n git = spack.util.git.git(required=True)\n output = git(\"config\", f\"branch.{self.branch}.remote\", output=str)\n remote = output.strip()\n except spack.util.executable.ProcessError:\n remote = \"origin\"\n spack.util.git.pull_checkout_branch(self.branch, remote=remote)\n else:\n raise spack.error.ConfigError(f\"Missing or unsupported options in {self}\")\n\n except spack.util.executable.ProcessError as e:\n raise spack.error.ConfigError(\n f\"Unable to check out repository ({self}) in {destination}: {e}\"\n )\n\n # only set the destination on successful clone/checkout\n self.destination = destination\n return self.destination\n\n def fetched(self) -> bool:\n return bool(self.destination) and os.path.exists(\n os.path.join(self.destination, \".git\") # type: ignore[arg-type]\n )\n\n def scopes(self, parent_scope: ConfigScope) -> List[ConfigScope]:\n \"\"\"Instantiate configuration scopes for the included paths.\n\n Args:\n parent_scope: including scope\n\n Returns: configuration scopes IF the when condition is satisfied;\n otherwise, an empty list.\n\n Raises:\n ConfigFileError: unable to access remote configuration file(s)\n ValueError: included path has an unsupported URL scheme, is required\n but does not exist; configuration stage directory argument is missing\n \"\"\"\n if not self.evaluate_condition():\n tty.debug(f\"Include condition is not satisfied in {self}\")\n return []\n\n if self._scopes:\n tty.debug(f\"Using existing scopes: {[s.name for s in self._scopes]}\")\n return self._scopes\n\n destination = self._clone(parent_scope)\n if not destination:\n raise spack.error.ConfigError(f\"Unable to cache the include: {self}\")\n\n scopes: List[ConfigScope] = []\n for path in self.paths:\n config_path = str(pathlib.Path(destination) / path)\n scope = self._scope(path, config_path, parent_scope)\n if scope is not None:\n scopes.append(scope)\n\n # cache the scopes if successfully able to process all of them\n if scopes:\n self._scopes = scopes\n return self._scopes\n\n @property\n def paths(self) -> List[str]:\n \"\"\"Path(s) associated with the include.\"\"\"\n\n return self._paths\n\n\ndef included_path(entry: Union[str, pathlib.Path, dict]) -> Union[IncludePath, GitIncludePaths]:\n \"\"\"Convert the included paths entry into the appropriate optional include.\n\n Args:\n entry: include configuration entry\n\n Returns: converted entry, where an empty ``when`` means the path is not conditionally included\n \"\"\"\n if isinstance(entry, (str, pathlib.Path)):\n return IncludePath({\"path\": str(entry)})\n\n if entry.get(\"path\", \"\"):\n return IncludePath(entry)\n\n return GitIncludePaths(entry)\n\n\ndef paths_from_includes(includes: List[Union[str, dict]]) -> List[str]:\n \"\"\"The path(s) from the configured includes.\n\n Args:\n includes: include configuration information\n\n Returns: list of path or an empty list if there are none\n \"\"\"\n\n paths = []\n for entry in includes:\n include = included_path(entry)\n paths.extend(include.paths)\n return paths\n\n\ndef config_paths_from_entry_points() -> List[Tuple[str, str]]:\n \"\"\"Load configuration paths from entry points\n\n A python package can register entry point metadata so that Spack can find\n its configuration by adding the following to the project's pyproject.toml:\n\n .. code-block:: toml\n\n [project.entry-points.\"spack.config\"]\n baz = \"baz:get_spack_config_path\"\n\n The function ``get_spack_config_path`` returns the path to the package's\n spack configuration scope\n\n \"\"\"\n config_paths: List[Tuple[str, str]] = []\n for entry_point in lang.get_entry_points(group=\"spack.config\"):\n hook = entry_point.load()\n if callable(hook):\n config_path = hook()\n if config_path and os.path.exists(config_path):\n config_paths.append((\"plugin-%s\" % entry_point.name, str(config_path)))\n return config_paths\n\n\ndef create_incremental() -> Generator[Configuration, None, None]:\n \"\"\"Singleton Configuration instance.\n\n This constructs one instance associated with this module and returns\n it. It is bundled inside a function so that configuration can be\n initialized lazily.\n \"\"\"\n # Default scopes are builtins and the default scope within the Spack instance.\n # These are versioned with Spack and can be overridden by systems, sites or user scopes.\n cfg = create_from(\n (ConfigScopePriority.DEFAULTS, InternalConfigScope(\"_builtin\", CONFIG_DEFAULTS)),\n (ConfigScopePriority.DEFAULTS, DirectoryConfigScope(*CONFIGURATION_DEFAULTS_PATH)),\n )\n yield cfg\n\n # Initial topmost scope is spack (the config scope in the spack instance).\n # It includes the user, site, and system scopes. Environments and command\n # line scopes go above this.\n configuration_paths = [(\"spack\", os.path.join(spack.paths.etc_path))]\n\n # Python packages can register configuration scopes via entry_points\n configuration_paths.extend(config_paths_from_entry_points())\n\n # add each scope\n for name, path in configuration_paths:\n # yield the config incrementally so that each config level's init code can get\n # data from the one below. This can be tricky, but it enables us to have a\n # single unified config system.\n #\n # TODO: think about whether we want to restrict what types of config can be used\n # at each level. e.g., we may want to just more forcibly disallow remote\n # config (which uses ssl and other config options) for some of the scopes,\n # to make the bootstrap issues more explicit, even if allowing config scope\n # init to reference lower scopes is more flexible.\n yield from cfg.push_scope_incremental(\n DirectoryConfigScope(name, path), priority=ConfigScopePriority.CONFIG_FILES\n )\n\n\ndef create() -> Configuration:\n \"\"\"Create a configuration using create_incremental(), return the last yielded result.\"\"\"\n return list(create_incremental())[-1]\n\n\n#: This is the singleton configuration instance for Spack.\nCONFIG = cast(Configuration, lang.Singleton(create_incremental))\n\n\ndef add_from_file(filename: str, scope: Optional[str] = None) -> None:\n \"\"\"Add updates to a config from a filename\"\"\"\n # Extract internal attributes, if we are dealing with an environment\n data = read_config_file(filename)\n if data is None:\n return\n\n if spack.schema.env.TOP_LEVEL_KEY in data:\n data = data[spack.schema.env.TOP_LEVEL_KEY]\n\n msg = (\n \"unexpected 'None' value when retrieving configuration. \"\n \"Please submit a bug-report at https://github.com/spack/spack/issues\"\n )\n assert data is not None, msg\n\n # update all sections from config dict\n # We have to iterate on keys to keep overrides from the file\n for section in data.keys():\n if section in SECTION_SCHEMAS.keys():\n # Special handling for compiler scope difference\n # Has to be handled after we choose a section\n if scope is None:\n scope = default_modify_scope(section)\n\n value = data[section]\n existing = get(section, scope=scope)\n new = spack.schema.merge_yaml(existing, value)\n\n # We cannot call config.set directly (set is a type)\n CONFIG.set(section, new, scope)\n\n\ndef add(fullpath: str, scope: Optional[str] = None) -> None:\n \"\"\"Add the given configuration to the specified config scope.\n Add accepts a path. If you want to add from a filename, use add_from_file\"\"\"\n components = process_config_path(fullpath)\n\n has_existing_value = True\n path = \"\"\n override = False\n value = components[-1]\n if not isinstance(value, syaml.syaml_str):\n value = syaml.load_config(value)\n for idx, name in enumerate(components[:-1]):\n # First handle double colons in constructing path\n colon = \"::\" if override else \":\" if path else \"\"\n path += colon + name\n if getattr(name, \"override\", False):\n override = True\n else:\n override = False\n\n # Test whether there is an existing value at this level\n existing = get(path, scope=scope)\n\n if existing is None:\n has_existing_value = False\n # We've nested further than existing config, so we need the\n # type information for validation to know how to handle bare\n # values appended to lists.\n existing = get_valid_type(path)\n\n # construct value from this point down\n for component in reversed(components[idx + 1 : -1]):\n value: Dict[str, str] = {component: value} # type: ignore[no-redef]\n break\n\n if override:\n path += \"::\"\n\n if has_existing_value:\n existing = get(path, scope=scope)\n\n # append values to lists\n if isinstance(existing, list) and not isinstance(value, list):\n value: List[str] = [value] # type: ignore[no-redef]\n\n # merge value into existing\n new = spack.schema.merge_yaml(existing, value)\n CONFIG.set(path, new, scope)\n\n\ndef get(path: str, default: Optional[Any] = None, scope: Optional[str] = None) -> Any:\n \"\"\"Module-level wrapper for ``Configuration.get()``.\"\"\"\n return CONFIG.get(path, default, scope)\n\n\n_set = set #: save this before defining set -- maybe config.set was ill-advised :)\n\n\ndef set(path: str, value: Any, scope: Optional[str] = None) -> None:\n \"\"\"Convenience function for setting single values in config files.\n\n Accepts the path syntax described in ``get()``.\n \"\"\"\n result = CONFIG.set(path, value, scope)\n return result\n\n\ndef scopes() -> lang.PriorityOrderedMapping[str, ConfigScope]:\n \"\"\"Convenience function to get list of configuration scopes.\"\"\"\n return CONFIG.scopes\n\n\ndef writable_scopes() -> List[ConfigScope]:\n \"\"\"Return list of writable scopes. Higher-priority scopes come first in the list.\"\"\"\n scopes = [x for x in CONFIG.scopes.values() if x.writable]\n scopes.reverse()\n return scopes\n\n\ndef existing_scopes() -> List[ConfigScope]:\n \"\"\"Return list of existing scopes. Scopes where Spack is\n aware of said scope, and the scope has a representation\n on the filesystem or are internal scopes.\n Higher-priority scopes come first in the list.\"\"\"\n scopes = [x for x in CONFIG.scopes.values() if x.exists]\n scopes.reverse()\n return scopes\n\n\ndef writable_scope_names() -> List[str]:\n return list(x.name for x in writable_scopes())\n\n\ndef existing_scope_names() -> List[str]:\n return list(x.name for x in existing_scopes())\n\n\ndef matched_config(cfg_path: str) -> List[Tuple[str, Any]]:\n return [(scope, get(cfg_path, scope=scope)) for scope in writable_scope_names()]\n\n\ndef change_or_add(\n section_name: str, find_fn: Callable[[str], bool], update_fn: Callable[[str], None]\n) -> None:\n \"\"\"Change or add a subsection of config, with additional logic to\n select a reasonable scope where the change is applied.\n\n Search through config scopes starting with the highest priority:\n the first matching a criteria (determined by ``find_fn``) is updated;\n if no such config exists, find the first config scope that defines\n any config for the named section; if no scopes define any related\n config, then update the highest-priority config scope.\n \"\"\"\n configs_by_section = matched_config(section_name)\n\n found = False\n for scope, section in configs_by_section:\n found = find_fn(section)\n if found:\n break\n\n if found:\n update_fn(section)\n CONFIG.set(section_name, section, scope=scope)\n return\n\n # If no scope meets the criteria specified by ``find_fn``,\n # then look for a scope that has any content (for the specified\n # section name)\n for scope, section in configs_by_section:\n if section:\n update_fn(section)\n found = True\n break\n\n if found:\n CONFIG.set(section_name, section, scope=scope)\n return\n\n # If no scopes define any config for the named section, then\n # modify the highest-priority scope.\n scope, section = configs_by_section[0]\n update_fn(section)\n CONFIG.set(section_name, section, scope=scope)\n\n\ndef update_all(section_name: str, change_fn: Callable[[str], bool]) -> None:\n \"\"\"Change a config section, which may have details duplicated\n across multiple scopes.\n \"\"\"\n configs_by_section = matched_config(\"develop\")\n\n for scope, section in configs_by_section:\n modified = change_fn(section)\n if modified:\n CONFIG.set(section_name, section, scope=scope)\n\n\ndef _validate_section_name(section: str) -> None:\n \"\"\"Exit if the section is not a valid section.\"\"\"\n if section not in SECTION_SCHEMAS:\n raise ConfigSectionError(\n f\"Invalid config section: '{section}'. Options are: {' '.join(SECTION_SCHEMAS.keys())}\"\n )\n\n\ndef validate(\n data: YamlConfigDict, schema: YamlConfigDict, filename: Optional[str] = None\n) -> YamlConfigDict:\n \"\"\"Validate data read in from a Spack YAML file.\n\n Arguments:\n data: data read from a Spack YAML file\n schema: jsonschema to validate data\n\n This leverages the line information (start_mark, end_mark) stored\n on Spack YAML structures.\n \"\"\"\n try:\n spack.schema.Validator(schema).validate(data)\n except jsonschema.ValidationError as e:\n if hasattr(e.instance, \"lc\"):\n line_number = e.instance.lc.line + 1\n else:\n line_number = None\n raise ConfigFormatError(e, data, filename, line_number) from e\n # return the validated data so that we can access the raw data\n # mostly relevant for environments\n return data\n\n\ndef read_config_file(\n path: str, schema: Optional[YamlConfigDict] = None\n) -> Optional[YamlConfigDict]:\n \"\"\"Read a YAML configuration file.\n\n User can provide a schema for validation. If no schema is provided,\n we will infer the schema from the top-level key.\"\"\"\n # Dev: Inferring schema and allowing it to be provided directly allows us\n # to preserve flexibility in calling convention (don't need to provide\n # schema when it's not necessary) while allowing us to validate against a\n # known schema when the top-level key could be incorrect.\n try:\n with open(path, encoding=\"utf-8\") as f:\n tty.debug(f\"Reading config from file {path}\")\n data = syaml.load_config(f)\n\n if data:\n if schema is None:\n key = next(iter(data))\n schema = _ALL_SCHEMAS[key]\n validate(data, schema)\n\n return data\n\n except FileNotFoundError:\n # Ignore nonexistent files.\n tty.debug(f\"Skipping nonexistent config path {path}\", level=3)\n return None\n\n except OSError as e:\n raise ConfigFileError(f\"Path is not a file or is not readable: {path}: {str(e)}\") from e\n\n except StopIteration as e:\n raise ConfigFileError(f\"Config file is empty or is not a valid YAML dict: {path}\") from e\n\n except syaml.SpackYAMLError as e:\n raise ConfigFileError(str(e)) from e\n\n\ndef _mark_internal(data, name):\n \"\"\"Add a simple name mark to raw YAML/JSON data.\n\n This is used by `spack config blame` to show where config lines came from.\n \"\"\"\n if isinstance(data, dict):\n d = syaml.syaml_dict(\n (_mark_internal(k, name), _mark_internal(v, name)) for k, v in data.items()\n )\n elif isinstance(data, list):\n d = syaml.syaml_list(_mark_internal(e, name) for e in data)\n else:\n d = syaml.syaml_type(data)\n\n if syaml.markable(d):\n d._start_mark = syaml.name_mark(name)\n d._end_mark = syaml.name_mark(name)\n\n return d\n\n\ndef get_valid_type(path):\n \"\"\"Returns an instance of a type that will pass validation for path.\n\n The instance is created by calling the constructor with no arguments.\n If multiple types will satisfy validation for data at the configuration\n path given, the priority order is ``list``, ``dict``, ``str``, ``bool``,\n ``int``, ``float``.\n \"\"\"\n types = {\n \"array\": list,\n \"object\": syaml.syaml_dict,\n \"string\": str,\n \"boolean\": bool,\n \"integer\": int,\n \"number\": float,\n }\n\n components = process_config_path(path)\n section = components[0]\n\n # Use None to construct the test data\n test_data = None\n for component in reversed(components):\n test_data = {component: test_data}\n\n try:\n validate(test_data, SECTION_SCHEMAS[section])\n except (ConfigFormatError, AttributeError) as e:\n jsonschema_error = e.validation_error\n if jsonschema_error.validator == \"type\":\n return types[jsonschema_error.validator_value]()\n elif jsonschema_error.validator in (\"anyOf\", \"oneOf\"):\n for subschema in jsonschema_error.validator_value:\n schema_type = subschema.get(\"type\")\n if schema_type is not None:\n return types[schema_type]()\n else:\n return type(None)\n raise spack.error.ConfigError(f\"Cannot determine valid type for path '{path}'.\")\n\n\ndef remove_yaml(dest, source):\n \"\"\"UnMerges source from dest; entries in source take precedence over dest.\n\n This routine may modify dest and should be assigned to dest, in\n case dest was None to begin with, e.g.::\n\n dest = remove_yaml(dest, source)\n\n In the result, elements from lists from ``source`` will not appear\n as elements of lists from ``dest``. Likewise, when iterating over keys\n or items in merged ``OrderedDict`` objects, keys from ``source`` will not\n appear as keys in ``dest``.\n\n Config file authors can optionally end any attribute in a dict\n with ``::`` instead of ``:``, and the key will remove the entire section\n from ``dest``\n \"\"\"\n\n def they_are(t):\n return isinstance(dest, t) and isinstance(source, t)\n\n # If source is None, overwrite with source.\n if source is None:\n return dest\n\n # Source list is prepended (for precedence)\n if they_are(list):\n # Make sure to copy ruamel comments\n dest[:] = [x for x in dest if x not in source]\n return dest\n\n # Source dict is merged into dest.\n elif they_are(dict):\n for sk, sv in source.items():\n # always remove the dest items. Python dicts do not overwrite\n # keys on insert, so this ensures that source keys are copied\n # into dest along with mark provenance (i.e., file/line info).\n unmerge = sk in dest\n old_dest_value = dest.pop(sk, None)\n\n if unmerge and not spack.schema.override(sk):\n dest[sk] = remove_yaml(old_dest_value, sv)\n\n return dest\n\n # If we reach here source and dest are either different types or are\n # not both lists or dicts: replace with source.\n return dest\n\n\nclass ConfigPath:\n quoted_string = \"(?:\\\"[^\\\"]+\\\")|(?:'[^']+')\"\n unquoted_string = \"[^:'\\\"]+\"\n element = rf\"(?:(?:{quoted_string})|(?:{unquoted_string}))\"\n next_key_pattern = rf\"({element}[+-]?)(?:\\:|$)\"\n\n @staticmethod\n def _split_front(string, extract):\n m = re.match(extract, string)\n if not m:\n return None, None\n token = m.group(1)\n return token, string[len(token) :]\n\n @staticmethod\n def _validate(path):\n \"\"\"Example valid config paths:\n\n x:y:z\n x:\"y\":z\n x:y+:z\n x:y::z\n x:y+::z\n x:y:\n x:y::\n \"\"\"\n first_key, path = ConfigPath._split_front(path, ConfigPath.next_key_pattern)\n if not first_key:\n raise ValueError(f\"Config path does not start with a parse-able key: {path}\")\n path_elements = [first_key]\n path_index = 1\n while path:\n separator, path = ConfigPath._split_front(path, r\"(\\:+)\")\n if not separator:\n raise ValueError(f\"Expected separator for {path}\")\n\n path_elements[path_index - 1] += separator\n if not path:\n break\n\n element, remainder = ConfigPath._split_front(path, ConfigPath.next_key_pattern)\n if not element:\n # If we can't parse something as a key, then it must be a\n # value (if it's valid).\n try:\n syaml.load_config(path)\n except syaml.SpackYAMLError as e:\n raise ValueError(\n \"Remainder of path is not a valid key\"\n f\" and does not parse as a value {path}\"\n ) from e\n element = path\n path = None # The rest of the path was consumed into the value\n else:\n path = remainder\n\n path_elements.append(element)\n path_index += 1\n\n return path_elements\n\n @staticmethod\n def process(path):\n result = []\n quote = \"['\\\"]\"\n seen_override_in_path = False\n\n path_elements = ConfigPath._validate(path)\n last_element_idx = len(path_elements) - 1\n for i, element in enumerate(path_elements):\n override = False\n append = False\n prepend = False\n quoted = False\n if element.endswith(\"::\") or (element.endswith(\":\") and i == last_element_idx):\n if seen_override_in_path:\n raise syaml.SpackYAMLError(\n \"Meaningless second override indicator `::' in path `{0}'\".format(path), \"\"\n )\n override = True\n seen_override_in_path = True\n element = element.rstrip(\":\")\n\n if element.endswith(\"+\"):\n prepend = True\n elif element.endswith(\"-\"):\n append = True\n element = element.rstrip(\"+-\")\n\n if re.match(f\"^{quote}\", element):\n quoted = True\n element = element.strip(\"'\\\"\")\n\n if append or prepend or override or quoted:\n element = syaml.syaml_str(element)\n if append:\n element.append = True\n if prepend:\n element.prepend = True\n if override:\n element.override = True\n\n result.append(element)\n\n return result\n\n\ndef process_config_path(path: str) -> List[str]:\n \"\"\"Process a path argument to config.set() that may contain overrides (``::`` or\n trailing ``:``)\n\n Colons will be treated as static strings if inside of quotes,\n e.g. ``this:is:a:path:'value:with:colon'`` will yield:\n\n .. code-block:: text\n\n [this, is, a, path, value:with:colon]\n\n The path may consist only of keys (e.g. for a ``get``) or may end in a value.\n Keys are always strings: if a user encloses a key in quotes, the quotes\n should be removed. Values with quotes should be treated as strings,\n but without quotes, may be parsed as a different yaml object (e.g.\n ``'{}'`` is a dict, but ``'\"{}\"'`` is a string).\n\n This function does not know whether the final element of the path is a\n key or value, so:\n\n * It must strip the quotes, in case it is a key (so we look for ``key`` and\n not ``\"key\"``)\n * It must indicate somehow that the quotes were stripped, in case it is a\n value (so that we don't process ``\"{}\"`` as a YAML dict)\n\n Therefore, all elements with quotes are stripped, and then also converted\n to ``syaml_str`` (if treating the final element as a value, the caller\n should not parse it in this case).\n \"\"\"\n return ConfigPath.process(path)\n\n\n#\n# Settings for commands that modify configuration\n#\ndef default_modify_scope(section: str = \"config\") -> str:\n \"\"\"Return the config scope that commands should modify by default.\n\n Commands that modify configuration by default modify the *highest*\n priority scope.\n\n Arguments:\n section (bool): Section for which to get the default scope.\n \"\"\"\n return CONFIG.highest_precedence_scope().name\n\n\ndef _update_in_memory(data: YamlConfigDict, section: str) -> bool:\n \"\"\"Update the format of the configuration data in memory.\n\n This function assumes the section is valid (i.e. validation\n is responsibility of the caller)\n\n Args:\n data: configuration data\n section: section of the configuration to update\n\n Returns:\n True if the data was changed, False otherwise\n \"\"\"\n return ensure_latest_format_fn(section)(data)\n\n\ndef ensure_latest_format_fn(section: str) -> Callable[[YamlConfigDict], bool]:\n \"\"\"Return a function that takes a config dictionary and update it to the latest format.\n\n The function returns True iff there was any update.\n\n Args:\n section: section of the configuration e.g. \"packages\", \"config\", etc.\n \"\"\"\n # Every module we need is already imported at the top level, so getattr should not raise\n return getattr(getattr(spack.schema, section), \"update\", lambda _: False)\n\n\n@contextlib.contextmanager\ndef use_configuration(\n *scopes_or_paths: Union[ScopeWithOptionalPriority, str],\n) -> Generator[Configuration, None, None]:\n \"\"\"Use the configuration scopes passed as arguments within the context manager.\n\n This function invalidates caches, and is therefore very slow.\n\n Args:\n *scopes_or_paths: scope objects or paths to be used\n\n Returns:\n Configuration object associated with the scopes passed as arguments\n \"\"\"\n global CONFIG\n\n # Normalize input and construct a Configuration object\n configuration = create_from(*scopes_or_paths)\n CONFIG.clear_caches(), configuration.clear_caches()\n\n saved_config, CONFIG = CONFIG, configuration\n\n try:\n yield configuration\n finally:\n CONFIG = saved_config\n\n\ndef _normalize_input(entry: Union[ScopeWithOptionalPriority, str]) -> ScopeWithPriority:\n if isinstance(entry, tuple):\n return entry\n\n default_priority = ConfigScopePriority.CONFIG_FILES\n if isinstance(entry, ConfigScope):\n return default_priority, entry\n\n # Otherwise we need to construct it\n path = os.path.normpath(entry)\n assert os.path.isdir(path), f'\"{path}\" must be a directory'\n name = os.path.basename(path)\n return default_priority, DirectoryConfigScope(name, path)\n\n\n@lang.memoized\ndef create_from(*scopes_or_paths: Union[ScopeWithOptionalPriority, str]) -> Configuration:\n \"\"\"Creates a configuration object from the scopes passed in input.\n\n Args:\n *scopes_or_paths: either a tuple of (priority, ConfigScope), or a ConfigScope, or a string\n If priority is not given, it is assumed to be ConfigScopePriority.CONFIG_FILES. If a\n string is given, a DirectoryConfigScope is created from it.\n\n Examples:\n\n >>> builtin_scope = InternalConfigScope(\"_builtin\", {\"config\": {\"build_jobs\": 1}})\n >>> cl_scope = InternalConfigScope(\"command_line\", {\"config\": {\"build_jobs\": 10}})\n >>> cfg = create_from(\n ... (ConfigScopePriority.COMMAND_LINE, cl_scope),\n ... (ConfigScopePriority.BUILTIN, builtin_scope)\n ... )\n \"\"\"\n scopes_with_priority = [_normalize_input(x) for x in scopes_or_paths]\n result = Configuration()\n for priority, scope in scopes_with_priority:\n result.push_scope(scope, priority=priority)\n return result\n\n\ndef determine_number_of_jobs(\n *,\n parallel: bool = False,\n max_cpus: int = cpus_available(),\n config: Optional[Configuration] = None,\n) -> int:\n \"\"\"\n Packages that require sequential builds need 1 job. Otherwise we use the\n number of jobs set on the command line. If not set, then we use the config\n defaults (which is usually set through the builtin config scope), but we\n cap to the number of CPUs available to avoid oversubscription.\n\n Parameters:\n parallel: true when package supports parallel builds\n max_cpus: maximum number of CPUs to use (defaults to cpus_available())\n config: configuration object (defaults to global config)\n \"\"\"\n if not parallel:\n return 1\n\n cfg = config or CONFIG\n\n # Command line overrides all\n try:\n command_line = cfg.get(\"config:build_jobs\", default=None, scope=\"command_line\")\n if command_line is not None:\n return command_line\n except ValueError:\n pass\n\n return min(max_cpus, cfg.get(\"config:build_jobs\", 16))\n\n\nclass ConfigSectionError(spack.error.ConfigError):\n \"\"\"Error for referring to a bad config section name in a configuration.\"\"\"\n\n\nclass ConfigFileError(spack.error.ConfigError):\n \"\"\"Issue reading or accessing a configuration file.\"\"\"\n\n\nclass ConfigFormatError(spack.error.ConfigError):\n \"\"\"Raised when a configuration format does not match its schema.\"\"\"\n\n def __init__(\n self,\n validation_error,\n data: YamlConfigDict,\n filename: Optional[str] = None,\n line: Optional[int] = None,\n ) -> None:\n # spack yaml has its own file/line marks -- try to find them\n # we prioritize these over the inputs\n self.validation_error = validation_error\n mark = self._get_mark(validation_error, data)\n if mark:\n filename = mark.name\n line = mark.line + 1\n\n self.filename = filename # record this for ruamel.yaml\n\n # construct location\n location = \"\"\n if filename:\n location = f\"{filename}\"\n if line is not None:\n location += f\":{line:d}\"\n\n message = f\"{location}: {validation_error.message}\"\n super().__init__(message)\n\n def _get_mark(self, validation_error, data):\n \"\"\"Get the file/line mark for a validation error from a Spack YAML file.\"\"\"\n\n # Try various places, starting with instance and parent\n for obj in (validation_error.instance, validation_error.parent):\n mark = get_mark_from_yaml_data(obj)\n if mark:\n return mark\n\n def get_path(path, data):\n if path:\n return get_path(path[1:], data[path[0]])\n else:\n return data\n\n # Try really hard to get the parent (which sometimes is not\n # set) This digs it out of the validated structure if it's not\n # on the validation_error.\n path = validation_error.path\n if path:\n parent = get_path(list(path)[:-1], data)\n if path[-1] in parent:\n if isinstance(parent, dict):\n keylist = list(parent.keys())\n elif isinstance(parent, list):\n keylist = parent\n idx = keylist.index(path[-1])\n mark = getattr(keylist[idx], \"_start_mark\", None)\n if mark:\n return mark\n\n # give up and return None if nothing worked\n return None\n\n\nclass RecursiveIncludeError(spack.error.SpackError):\n \"\"\"Too many levels of recursive includes.\"\"\"\n" }, { "path": "lib/spack/spack/container/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Package that provides functions and classes to\ngenerate container recipes from a Spack environment\n\"\"\"\n\nimport warnings\n\nimport spack.vendor.jsonschema\n\nimport spack.environment as ev\nimport spack.schema.env as env\nimport spack.util.spack_yaml as syaml\n\nfrom .writers import recipe\n\n__all__ = [\"validate\", \"recipe\"]\n\n\ndef validate(configuration_file):\n \"\"\"Validate a Spack environment YAML file that is being used to generate a\n recipe for a container.\n\n Since a few attributes of the configuration must have specific values for\n the container recipe, this function returns a sanitized copy of the\n configuration in the input file. If any modification is needed, a warning\n will be issued.\n\n Args:\n configuration_file (str): path to the Spack environment YAML file\n\n Returns:\n A sanitized copy of the configuration stored in the input file\n \"\"\"\n with open(configuration_file, encoding=\"utf-8\") as f:\n config = syaml.load(f)\n\n # Ensure we have a \"container\" attribute with sensible defaults set\n env_dict = config[ev.TOP_LEVEL_KEY]\n env_dict.setdefault(\n \"container\", {\"format\": \"docker\", \"images\": {\"os\": \"ubuntu:22.04\", \"spack\": \"develop\"}}\n )\n env_dict[\"container\"].setdefault(\"format\", \"docker\")\n env_dict[\"container\"].setdefault(\"images\", {\"os\": \"ubuntu:22.04\", \"spack\": \"develop\"})\n\n # Remove attributes that are not needed / allowed in the\n # container recipe\n for subsection in (\"cdash\", \"gitlab_ci\", \"modules\"):\n if subsection in env_dict:\n msg = (\n 'the subsection \"{0}\" in \"{1}\" is not used when generating'\n \" container recipes and will be discarded\"\n )\n warnings.warn(msg.format(subsection, configuration_file))\n env_dict.pop(subsection)\n\n # Set the default value of the concretization strategy to unify and\n # warn if the user explicitly set another value\n env_dict.setdefault(\"concretizer\", {\"unify\": True})\n if env_dict[\"concretizer\"][\"unify\"] is not True:\n warnings.warn(\n '\"concretizer:unify\" is not set to \"true\", which means the '\n \"generated image may contain different variants of the same \"\n 'packages. Set to \"true\" to get a consistent set of packages.'\n )\n\n # Check if the install tree was explicitly set to a custom value and warn\n # that it will be overridden\n environment_config = env_dict.get(\"config\", {})\n if environment_config.get(\"install_tree\", None):\n msg = (\n 'the \"config:install_tree\" attribute has been set explicitly '\n \"and will be overridden in the container image\"\n )\n warnings.warn(msg)\n\n # Likewise for the view\n environment_view = env_dict.get(\"view\", None)\n if environment_view:\n msg = (\n 'the \"view\" attribute has been set explicitly '\n \"and will be overridden in the container image\"\n )\n warnings.warn(msg)\n\n spack.vendor.jsonschema.validate(config, schema=env.schema)\n return config\n" }, { "path": "lib/spack/spack/container/images.json", "content": "{\n \"images\": {\n \"alpine:3\": {\n \"bootstrap\": {\n \"template\": \"container/alpine_3.dockerfile\"\n },\n \"os_package_manager\": \"apk\"\n },\n \"amazonlinux:2\": {\n \"bootstrap\": {\n \"template\": \"container/amazonlinux_2.dockerfile\"\n },\n \"os_package_manager\": \"yum_amazon\"\n },\n \"fedora:40\": {\n \"bootstrap\": {\n \"template\": \"container/fedora.dockerfile\",\n \"image\": \"docker.io/fedora:40\"\n },\n \"os_package_manager\": \"dnf\",\n \"build\": \"spack/fedora40\",\n \"final\": {\n \"image\": \"docker.io/fedora:40\"\n }\n },\n \"fedora:39\": {\n \"bootstrap\": {\n \"template\": \"container/fedora.dockerfile\",\n \"image\": \"docker.io/fedora:39\"\n },\n \"os_package_manager\": \"dnf\",\n \"build\": \"spack/fedora39\",\n \"final\": {\n \"image\": \"docker.io/fedora:39\"\n }\n },\n \"rockylinux:9\": {\n \"bootstrap\": {\n \"template\": \"container/rockylinux_9.dockerfile\",\n \"image\": \"docker.io/rockylinux:9\"\n },\n \"os_package_manager\": \"dnf_epel\",\n \"build\": \"spack/rockylinux9\",\n \"final\": {\n \"image\": \"docker.io/rockylinux:9\"\n }\n },\n \"rockylinux:8\": {\n \"bootstrap\": {\n \"template\": \"container/rockylinux_8.dockerfile\",\n \"image\": \"docker.io/rockylinux:8\"\n },\n \"os_package_manager\": \"dnf_epel\",\n \"build\": \"spack/rockylinux8\",\n \"final\": {\n \"image\": \"docker.io/rockylinux:8\"\n }\n },\n \"almalinux:9\": {\n \"bootstrap\": {\n \"template\": \"container/almalinux_9.dockerfile\",\n \"image\": \"quay.io/almalinuxorg/almalinux:9\"\n },\n \"os_package_manager\": \"dnf_epel\",\n \"build\": \"spack/almalinux9\",\n \"final\": {\n \"image\": \"quay.io/almalinuxorg/almalinux:9\"\n }\n },\n \"almalinux:8\": {\n \"bootstrap\": {\n \"template\": \"container/almalinux_8.dockerfile\",\n \"image\": \"quay.io/almalinuxorg/almalinux:8\"\n },\n \"os_package_manager\": \"dnf_epel\",\n \"build\": \"spack/almalinux8\",\n \"final\": {\n \"image\": \"quay.io/almalinuxorg/almalinux:8\"\n }\n },\n \"centos:stream9\": {\n \"bootstrap\": {\n \"template\": \"container/centos_stream9.dockerfile\",\n \"image\": \"quay.io/centos/centos:stream9\"\n },\n \"os_package_manager\": \"dnf_epel\",\n \"build\": \"spack/centos-stream9\",\n \"final\": {\n \"image\": \"quay.io/centos/centos:stream9\"\n }\n },\n \"opensuse/leap:15\": {\n \"bootstrap\": {\n \"template\": \"container/leap-15.dockerfile\"\n },\n \"os_package_manager\": \"zypper\",\n \"build\": \"spack/leap15\",\n \"final\": {\n \"image\": \"opensuse/leap:latest\"\n }\n },\n \"nvidia/cuda:11.2.1\": {\n \"bootstrap\": {\n \"template\": \"container/cuda_11_2_1.dockerfile\",\n \"image\": \"nvidia/cuda:11.2.1-devel\"\n },\n \"final\": {\n \"image\": \"nvidia/cuda:11.2.1-base\"\n },\n \"os_package_manager\": \"apt\"\n },\n \"ubuntu:24.04\": {\n \"bootstrap\": {\n \"template\": \"container/ubuntu_2404.dockerfile\"\n },\n \"os_package_manager\": \"apt\",\n \"build\": \"spack/ubuntu-noble\"\n },\n \"ubuntu:22.04\": {\n \"bootstrap\": {\n \"template\": \"container/ubuntu_2204.dockerfile\"\n },\n \"os_package_manager\": \"apt\",\n \"build\": \"spack/ubuntu-jammy\"\n },\n \"ubuntu:20.04\": {\n \"bootstrap\": {\n \"template\": \"container/ubuntu_2004.dockerfile\"\n },\n \"build\": \"spack/ubuntu-focal\",\n \"os_package_manager\": \"apt\"\n }\n },\n \"os_package_managers\": {\n \"apk\": {\n \"update\": \"apk update\",\n \"install\": \"apk add --no-cache\",\n \"clean\": \"true\"\n },\n \"apt\": {\n \"update\": \"apt-get -yqq update && apt-get -yqq upgrade\",\n \"install\": \"apt-get -yqq install\",\n \"clean\": \"rm -rf /var/lib/apt/lists/*\"\n },\n \"dnf\": {\n \"update\": \"dnf update -y\",\n \"install\": \"dnf install -y\",\n \"clean\": \"rm -rf /var/cache/dnf && dnf clean all\"\n },\n \"dnf_epel\": {\n \"update\": \"dnf update -y && dnf install -y epel-release && dnf update -y\",\n \"install\": \"dnf install -y\",\n \"clean\": \"rm -rf /var/cache/dnf && dnf clean all\"\n },\n \"yum\": {\n \"update\": \"yum update -y && yum install -y epel-release && yum update -y\",\n \"install\": \"yum install -y\",\n \"clean\": \"rm -rf /var/cache/yum && yum clean all\"\n },\n \"yum_amazon\": {\n \"update\": \"yum update -y && amazon-linux-extras install epel -y\",\n \"install\": \"yum install -y\",\n \"clean\": \"rm -rf /var/cache/yum && yum clean all\"\n },\n \"zypper\": {\n \"update\": \"zypper update -y\",\n \"install\": \"zypper install -y\",\n \"clean\": \"rm -rf /var/cache/zypp && zypper clean -a\"\n }\n }\n}\n" }, { "path": "lib/spack/spack/container/images.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Manages the details on the images used in the various stages.\"\"\"\n\nimport json\nimport os\nimport shlex\nimport sys\n\nimport spack.llnl.util.filesystem as fs\nimport spack.llnl.util.tty as tty\nimport spack.util.git\n\n#: Global variable used to cache in memory the content of images.json\n_data = None\n\n\ndef data():\n \"\"\"Returns a dictionary with the static data on the images.\n\n The dictionary is read from a JSON file lazily the first time\n this function is called.\n \"\"\"\n global _data\n if not _data:\n json_dir = os.path.abspath(os.path.dirname(__file__))\n json_file = os.path.join(json_dir, \"images.json\")\n with open(json_file, encoding=\"utf-8\") as f:\n _data = json.load(f)\n return _data\n\n\ndef build_info(image, spack_version):\n \"\"\"Returns the name of the build image and its tag.\n\n Args:\n image (str): image to be used at run-time. Should be of the form\n : e.g. ``\"ubuntu:18.04\"``\n spack_version (str): version of Spack that we want to use to build\n\n Returns:\n A tuple with (image_name, image_tag) for the build image\n \"\"\"\n # Don't handle error here, as a wrong image should have been\n # caught by the JSON schema\n image_data = data()[\"images\"][image]\n build_image = image_data.get(\"build\", None)\n if not build_image:\n return None, None\n\n return build_image, spack_version\n\n\ndef os_package_manager_for(image):\n \"\"\"Returns the name of the OS package manager for the image\n passed as argument.\n\n Args:\n image (str): image to be used at run-time. Should be of the form\n : e.g. ``\"ubuntu:18.04\"``\n\n Returns:\n Name of the package manager, e.g. ``\"apt\"`` or ``\"yum\"``\n \"\"\"\n name = data()[\"images\"][image][\"os_package_manager\"]\n return name\n\n\ndef all_bootstrap_os():\n \"\"\"Return a list of all the OS that can be used to bootstrap Spack\"\"\"\n return list(data()[\"images\"])\n\n\ndef commands_for(package_manager):\n \"\"\"Returns the commands used to update system repositories, install\n system packages and clean afterwards.\n\n Args:\n package_manager (str): package manager to be used\n\n Returns:\n A tuple of (update, install, clean) commands.\n \"\"\"\n info = data()[\"os_package_managers\"][package_manager]\n return info[\"update\"], info[\"install\"], info[\"clean\"]\n\n\ndef bootstrap_template_for(image):\n return data()[\"images\"][image][\"bootstrap\"][\"template\"]\n\n\ndef _verify_ref(url, ref, enforce_sha):\n # Do a checkout in a temporary directory\n msg = 'Cloning \"{0}\" to verify ref \"{1}\"'.format(url, ref)\n tty.info(msg, stream=sys.stderr)\n git = spack.util.git.git(required=True)\n with fs.temporary_dir():\n git(\"clone\", \"-q\", url, \".\")\n sha = git(\n \"rev-parse\", \"-q\", ref + \"^{commit}\", output=str, error=os.devnull, fail_on_error=False\n )\n if git.returncode:\n msg = '\"{0}\" is not a valid reference for \"{1}\"'\n raise RuntimeError(msg.format(sha, url))\n\n if enforce_sha:\n ref = sha.strip()\n\n return ref\n\n\ndef checkout_command(url, ref, enforce_sha, verify):\n \"\"\"Return the checkout command to be used in the bootstrap phase.\n\n Args:\n url (str): url of the Spack repository\n ref (str): either a branch name, a tag or a commit sha\n enforce_sha (bool): if true turns every\n verify (bool):\n \"\"\"\n url = url or \"https://github.com/spack/spack.git\"\n ref = ref or \"develop\"\n enforce_sha, verify = bool(enforce_sha), bool(verify)\n # If we want to enforce a sha or verify the ref we need\n # to checkout the repository locally\n if enforce_sha or verify:\n ref = _verify_ref(url, ref, enforce_sha)\n\n return \" && \".join(\n [\n \"git init --quiet\",\n f\"git remote add origin {shlex.quote(url)}\",\n f\"git fetch --depth=1 origin {shlex.quote(ref)}\",\n \"git checkout --detach FETCH_HEAD\",\n ]\n )\n" }, { "path": "lib/spack/spack/container/writers.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Writers for different kind of recipes and related\nconvenience functions.\n\"\"\"\n\nimport copy\nimport shlex\nfrom collections import namedtuple\nfrom typing import Optional\n\nimport spack.vendor.jsonschema\n\nimport spack.environment as ev\nimport spack.error\nimport spack.schema.env\nimport spack.tengine as tengine\nimport spack.util.spack_yaml as syaml\n\nfrom .images import (\n bootstrap_template_for,\n build_info,\n checkout_command,\n commands_for,\n data,\n os_package_manager_for,\n)\n\n#: Caches all the writers that are currently supported\n_writer_factory = {}\n\n\ndef writer(name):\n \"\"\"Decorator to register a factory for a recipe writer.\n\n Each factory should take a configuration dictionary and return a\n properly configured writer that, when called, prints the\n corresponding recipe.\n \"\"\"\n\n def _decorator(factory):\n _writer_factory[name] = factory\n return factory\n\n return _decorator\n\n\ndef create(configuration, last_phase=None):\n \"\"\"Returns a writer that conforms to the configuration passed as input.\n\n Args:\n configuration (dict): how to generate the current recipe\n last_phase (str): last phase to be printed or None to print them all\n \"\"\"\n name = configuration[ev.TOP_LEVEL_KEY][\"container\"][\"format\"]\n return _writer_factory[name](configuration, last_phase)\n\n\ndef recipe(configuration, last_phase=None):\n \"\"\"Returns a recipe that conforms to the configuration passed as input.\n\n Args:\n configuration (dict): how to generate the current recipe\n last_phase (str): last phase to be printed or None to print them all\n \"\"\"\n return create(configuration, last_phase)()\n\n\ndef _stage_base_images(images_config):\n \"\"\"Return a tuple with the base images to be used at the various stages.\n\n Args:\n images_config (dict): configuration under container:images\n \"\"\"\n # If we have custom base images, just return them verbatim.\n build_stage = images_config.get(\"build\", None)\n if build_stage:\n final_stage = images_config[\"final\"]\n return None, build_stage, final_stage\n\n # Check the operating system: this will be the base of the bootstrap\n # stage, if there, and of the final stage.\n operating_system = images_config.get(\"os\", None)\n\n # Check the OS is mentioned in the internal data stored in a JSON file\n images_json = data()[\"images\"]\n if not any(os_name == operating_system for os_name in images_json):\n msg = 'invalid operating system name \"{0}\". [Allowed values are {1}]'\n msg = msg.format(operating_system, \", \".join(data()[\"images\"]))\n raise ValueError(msg)\n\n # Retrieve the build stage\n spack_info = images_config[\"spack\"]\n if isinstance(spack_info, dict):\n build_stage = \"bootstrap\"\n else:\n spack_version = images_config[\"spack\"]\n image_name, tag = build_info(operating_system, spack_version)\n build_stage = \"bootstrap\"\n if image_name:\n build_stage = \":\".join([image_name, tag])\n\n # Retrieve the bootstrap stage\n bootstrap_stage = None\n if build_stage == \"bootstrap\":\n bootstrap_stage = images_json[operating_system][\"bootstrap\"].get(\"image\", operating_system)\n\n # Retrieve the final stage\n final_stage = images_json[operating_system].get(\"final\", {\"image\": operating_system})[\"image\"]\n\n return bootstrap_stage, build_stage, final_stage\n\n\ndef _spack_checkout_config(images_config):\n spack_info = images_config[\"spack\"]\n\n url = \"https://github.com/spack/spack.git\"\n ref = \"develop\"\n resolve_sha, verify = False, False\n\n # Config specific values may override defaults\n if isinstance(spack_info, dict):\n url = spack_info.get(\"url\", url)\n ref = spack_info.get(\"ref\", ref)\n resolve_sha = spack_info.get(\"resolve_sha\", resolve_sha)\n verify = spack_info.get(\"verify\", verify)\n else:\n ref = spack_info\n\n return url, ref, resolve_sha, verify\n\n\nclass PathContext(tengine.Context):\n \"\"\"Generic context used to instantiate templates of recipes that\n install software in a common location and make it available\n directly via PATH.\n \"\"\"\n\n # Must be set by derived classes\n template_name: Optional[str] = None\n\n def __init__(self, config, last_phase):\n self.config = config[ev.TOP_LEVEL_KEY]\n self.container_config = self.config[\"container\"]\n\n # Operating system tag as written in the configuration file\n self.operating_system_key = self.container_config[\"images\"].get(\"os\")\n # Get base images and verify the OS\n bootstrap, build, final = _stage_base_images(self.container_config[\"images\"])\n self.bootstrap_image = bootstrap\n self.build_image = build\n self.final_image = final\n\n # Record the last phase\n self.last_phase = last_phase\n\n @tengine.context_property\n def depfile(self):\n return self.container_config.get(\"depfile\", False)\n\n @tengine.context_property\n def run(self):\n \"\"\"Information related to the run image.\"\"\"\n Run = namedtuple(\"Run\", [\"image\"])\n return Run(image=self.final_image)\n\n @tengine.context_property\n def build(self):\n \"\"\"Information related to the build image.\"\"\"\n Build = namedtuple(\"Build\", [\"image\"])\n return Build(image=self.build_image)\n\n @tengine.context_property\n def strip(self):\n \"\"\"Whether or not to strip binaries in the image\"\"\"\n return self.container_config.get(\"strip\", True)\n\n @tengine.context_property\n def paths(self):\n \"\"\"Important paths in the image\"\"\"\n Paths = namedtuple(\"Paths\", [\"environment\", \"store\", \"view_parent\", \"view\", \"former_view\"])\n return Paths(\n environment=\"/opt/spack-environment\",\n store=\"/opt/software\",\n view_parent=\"/opt/views\",\n view=\"/opt/views/view\",\n former_view=\"/opt/view\", # /opt/view -> /opt/views/view for backward compatibility\n )\n\n @tengine.context_property\n def manifest(self):\n \"\"\"The spack.yaml file that should be used in the image\"\"\"\n # Copy in the part of spack.yaml prescribed in the configuration file\n manifest = copy.deepcopy(self.config)\n manifest.pop(\"container\")\n\n # Ensure that a few paths are where they need to be\n manifest.setdefault(\"config\", syaml.syaml_dict())\n manifest[\"config\"][\"install_tree\"] = {\"root\": self.paths.store}\n manifest[\"view\"] = self.paths.view\n manifest = {\"spack\": manifest}\n\n # Validate the manifest file\n spack.vendor.jsonschema.validate(manifest, schema=spack.schema.env.schema)\n\n return syaml.dump(manifest, default_flow_style=False).strip()\n\n @tengine.context_property\n def os_packages_final(self):\n \"\"\"Additional system packages that are needed at run-time.\"\"\"\n try:\n return self._os_packages_for_stage(\"final\")\n except Exception as e:\n msg = f\"an error occurred while rendering the 'final' stage of the image: {e}\"\n raise spack.error.SpackError(msg) from e\n\n @tengine.context_property\n def os_packages_build(self):\n \"\"\"Additional system packages that are needed at build-time.\"\"\"\n try:\n return self._os_packages_for_stage(\"build\")\n except Exception as e:\n msg = f\"an error occurred while rendering the 'build' stage of the image: {e}\"\n raise spack.error.SpackError(msg) from e\n\n @tengine.context_property\n def os_package_update(self):\n \"\"\"Whether or not to update the OS package manager cache.\"\"\"\n os_packages = self.container_config.get(\"os_packages\", {})\n return os_packages.get(\"update\", True)\n\n def _os_packages_for_stage(self, stage):\n os_packages = self.container_config.get(\"os_packages\", {})\n package_list = os_packages.get(stage, None)\n return self._package_info_from(package_list)\n\n def _package_info_from(self, package_list):\n \"\"\"Helper method to pack a list of packages with the additional\n information required by the template.\n\n Args:\n package_list: list of packages\n\n Returns:\n Enough information to know how to update the cache, install\n a list of packages, and clean in the end.\n \"\"\"\n if not package_list:\n return package_list\n\n image_config = self.container_config[\"images\"]\n image = image_config.get(\"build\", None)\n\n if image is None:\n os_pkg_manager = os_package_manager_for(image_config[\"os\"])\n else:\n os_pkg_manager = self._os_pkg_manager()\n\n update, install, clean = commands_for(os_pkg_manager)\n\n Packages = namedtuple(\"Packages\", [\"update\", \"install\", \"list\", \"clean\"])\n return Packages(update=update, install=install, list=package_list, clean=clean)\n\n def _os_pkg_manager(self):\n try:\n os_pkg_manager = self.container_config[\"os_packages\"][\"command\"]\n except KeyError:\n msg = (\n \"cannot determine the OS package manager to use.\\n\\n\\tPlease add an \"\n \"appropriate 'os_packages:command' entry to the spack.yaml manifest file\\n\"\n )\n raise spack.error.SpackError(msg)\n return os_pkg_manager\n\n @tengine.context_property\n def labels(self):\n return self.container_config.get(\"labels\", {})\n\n @tengine.context_property\n def bootstrap(self):\n \"\"\"Information related to the build image.\"\"\"\n images_config = self.container_config[\"images\"]\n bootstrap_recipe = None\n if self.bootstrap_image:\n config_args = _spack_checkout_config(images_config)\n command = checkout_command(*config_args)\n template_path = bootstrap_template_for(self.operating_system_key)\n env = tengine.make_environment()\n context = {\"bootstrap\": {\"image\": self.bootstrap_image, \"spack_checkout\": command}}\n bootstrap_recipe = env.get_template(template_path).render(**context)\n\n Bootstrap = namedtuple(\"Bootstrap\", [\"image\", \"recipe\"])\n return Bootstrap(image=self.bootstrap_image, recipe=bootstrap_recipe)\n\n @tengine.context_property\n def render_phase(self):\n render_bootstrap = bool(self.bootstrap_image)\n render_build = not (self.last_phase == \"bootstrap\")\n render_final = self.last_phase in (None, \"final\")\n Render = namedtuple(\"Render\", [\"bootstrap\", \"build\", \"final\"])\n return Render(bootstrap=render_bootstrap, build=render_build, final=render_final)\n\n def __call__(self):\n \"\"\"Returns the recipe as a string\"\"\"\n env = tengine.make_environment()\n template_name = self.container_config.get(\"template\", self.template_name)\n t = env.get_template(template_name)\n return t.render(**self.to_dict())\n\n\n@writer(\"docker\")\nclass DockerContext(PathContext):\n \"\"\"Context used to instantiate a Dockerfile\"\"\"\n\n #: Name of the template used for Dockerfiles\n template_name = \"container/Dockerfile\"\n\n @tengine.context_property\n def manifest(self):\n manifest_str = super().manifest\n # Docker doesn't support HEREDOC, so we need to resort to\n # a horrible echo trick to have the manifest in the Dockerfile\n echoed_lines = []\n for idx, line in enumerate(manifest_str.split(\"\\n\")):\n quoted_line = shlex.quote(line)\n if idx == 0:\n echoed_lines.append(\"&& (echo \" + quoted_line + \" \\\\\")\n continue\n echoed_lines.append(\"&& echo \" + quoted_line + \" \\\\\")\n\n echoed_lines[-1] = echoed_lines[-1].replace(\" \\\\\", \")\")\n\n return \"\\n\".join(echoed_lines)\n\n\n@writer(\"singularity\")\nclass SingularityContext(PathContext):\n \"\"\"Context used to instantiate a Singularity definition file\"\"\"\n\n #: Name of the template used for Singularity definition files\n template_name = \"container/singularity.def\"\n\n @property\n def singularity_config(self):\n return self.container_config.get(\"singularity\", {})\n\n @tengine.context_property\n def runscript(self):\n return self.singularity_config.get(\"runscript\", \"\")\n\n @tengine.context_property\n def startscript(self):\n return self.singularity_config.get(\"startscript\", \"\")\n\n @tengine.context_property\n def test(self):\n return self.singularity_config.get(\"test\", \"\")\n\n @tengine.context_property\n def help(self):\n return self.singularity_config.get(\"help\", \"\")\n" }, { "path": "lib/spack/spack/context.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"This module provides classes used in user and build environment\"\"\"\n\nfrom enum import Enum\n\n\nclass Context(Enum):\n \"\"\"Enum used to indicate the context in which an environment has to be setup: build,\n run or test.\"\"\"\n\n BUILD = 1\n RUN = 2\n TEST = 3\n\n def __str__(self):\n return (\"build\", \"run\", \"test\")[self.value - 1]\n\n @classmethod\n def from_string(cls, s: str):\n if s == \"build\":\n return Context.BUILD\n elif s == \"run\":\n return Context.RUN\n elif s == \"test\":\n return Context.TEST\n raise ValueError(f\"context should be one of 'build', 'run', 'test', got {s}\")\n" }, { "path": "lib/spack/spack/cray_manifest.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport json\nimport os\nimport traceback\nimport warnings\nfrom typing import Any, Dict, Iterable, List, Optional\n\nfrom spack.vendor import jsonschema\nfrom spack.vendor.jsonschema import exceptions\n\nimport spack.cmd\nimport spack.compilers.config\nimport spack.deptypes as dt\nimport spack.error\nimport spack.hash_types as hash_types\nimport spack.llnl.util.tty as tty\nimport spack.platforms\nimport spack.repo\nimport spack.spec\nimport spack.store\nfrom spack.detection.path import ExecutablesFinder\nfrom spack.schema.cray_manifest import schema as manifest_schema\n\n#: Cray systems can store a Spack-compatible description of system\n#: packages here.\ndefault_path = \"/opt/cray/pe/cpe-descriptive-manifest/\"\n\nCOMPILER_NAME_TRANSLATION = {\"nvidia\": \"nvhpc\", \"rocm\": \"llvm-amdgpu\", \"clang\": \"llvm\"}\n\n\ndef translated_compiler_name(manifest_compiler_name):\n \"\"\"\n When creating a Compiler object, Spack expects a name matching\n one of the classes in :mod:`spack.compilers.config`. Names in the Cray manifest\n may differ; for cases where we know the name refers to a compiler in\n Spack, this function translates it automatically.\n\n This function will raise an error if there is no recorded translation\n and the name doesn't match a known compiler name.\n \"\"\"\n if manifest_compiler_name in COMPILER_NAME_TRANSLATION:\n return COMPILER_NAME_TRANSLATION[manifest_compiler_name]\n elif manifest_compiler_name in spack.compilers.config.supported_compilers():\n return manifest_compiler_name\n else:\n raise spack.compilers.config.UnknownCompilerError(\n f\"[CRAY MANIFEST] unknown compiler: {manifest_compiler_name}\"\n )\n\n\ndef compiler_from_entry(entry: dict, *, manifest_path: str) -> Optional[spack.spec.Spec]:\n # Note that manifest_path is only passed here to compose a\n # useful warning message when paths appear to be missing.\n compiler_name = translated_compiler_name(entry[\"name\"])\n paths = extract_compiler_paths(entry)\n\n # Do a check for missing paths. Note that this isn't possible for\n # all compiler entries, since their \"paths\" might actually be\n # exe names like \"cc\" that depend on modules being loaded. Cray\n # manifest entries are always paths though.\n missing_paths = [x for x in paths if not os.path.exists(x)]\n if missing_paths:\n warnings.warn(\n \"Manifest entry refers to nonexistent paths:\\n\\t\"\n + \"\\n\\t\".join(missing_paths)\n + f\"\\nfor {entry['name']}@{entry['version']}\"\n + f\"\\nin {manifest_path}\"\n + \"\\nPlease report this issue\"\n )\n\n try:\n compiler_spec = compiler_spec_from_paths(pkg_name=compiler_name, compiler_paths=paths)\n except spack.error.SpackError as e:\n tty.debug(f\"[CRAY MANIFEST] {e}\")\n return None\n\n compiler_spec.constrain(\n f\"platform=linux os={entry['arch']['os']} target={entry['arch']['target']}\"\n )\n return compiler_spec\n\n\ndef compiler_spec_from_paths(*, pkg_name: str, compiler_paths: Iterable[str]) -> spack.spec.Spec:\n \"\"\"Returns the external spec associated with a series of compilers, if any.\"\"\"\n pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)\n finder = ExecutablesFinder()\n specs = finder.detect_specs(pkg=pkg_cls, paths=compiler_paths, repo_path=spack.repo.PATH)\n\n if not specs or len(specs) > 1:\n raise CrayCompilerDetectionError(\n message=f\"cannot detect a single {pkg_name} compiler for Cray manifest entry\",\n long_message=f\"Analyzed paths are: {', '.join(compiler_paths)}\",\n )\n\n return specs[0]\n\n\ndef extract_compiler_paths(entry: Dict[str, Any]) -> List[str]:\n \"\"\"Returns the paths to compiler executables, from a dictionary entry in the Cray manifest.\"\"\"\n paths = list(entry[\"executables\"].values())\n if \"prefix\" in entry:\n paths = [os.path.join(entry[\"prefix\"], relpath) for relpath in paths]\n return paths\n\n\ndef spec_from_entry(entry):\n arch_str = \"\"\n if \"arch\" in entry:\n local_platform = spack.platforms.host()\n spec_platform = entry[\"arch\"][\"platform\"]\n # Note that Cray systems are now treated as Linux. Specs\n # in the manifest which specify \"cray\" as the platform\n # should be registered in the DB as \"linux\"\n if local_platform.name == \"linux\" and spec_platform.lower() == \"cray\":\n spec_platform = \"linux\"\n arch_format = \"arch={platform}-{os}-{target}\"\n arch_str = arch_format.format(\n platform=spec_platform,\n os=entry[\"arch\"][\"platform_os\"],\n target=entry[\"arch\"][\"target\"][\"name\"],\n )\n\n compiler_str = \"\"\n if \"compiler\" in entry:\n compiler_format = \"%{name}@={version}\"\n compiler_str = compiler_format.format(\n name=translated_compiler_name(entry[\"compiler\"][\"name\"]),\n version=entry[\"compiler\"][\"version\"],\n )\n\n spec_format = \"{name}@={version} {arch}\"\n spec_str = spec_format.format(\n name=entry[\"name\"], version=entry[\"version\"], compiler=compiler_str, arch=arch_str\n )\n\n pkg_cls = spack.repo.PATH.get_pkg_class(entry[\"name\"])\n\n if \"parameters\" in entry:\n variant_strs = list()\n for name, value in entry[\"parameters\"].items():\n # TODO: also ensure that the variant value is valid?\n if not pkg_cls.has_variant(name):\n tty.debug(\n \"Omitting variant {0} for entry {1}/{2}\".format(\n name, entry[\"name\"], entry[\"hash\"][:7]\n )\n )\n continue\n\n # Value could be a list (of strings), boolean, or string\n if isinstance(value, str):\n variant_strs.append(\"{0}={1}\".format(name, value))\n else:\n try:\n iter(value)\n variant_strs.append(\"{0}={1}\".format(name, \",\".join(value)))\n continue\n except TypeError:\n # Not an iterable\n pass\n # At this point not a string or collection, check for boolean\n if value in [True, False]:\n bool_symbol = \"+\" if value else \"~\"\n variant_strs.append(\"{0}{1}\".format(bool_symbol, name))\n else:\n raise ValueError(\n \"Unexpected value for {0} ({1}): {2}\".format(\n name, str(type(value)), str(value)\n )\n )\n spec_str += \" \" + \" \".join(variant_strs)\n\n (spec,) = spack.cmd.parse_specs(spec_str.split())\n\n for ht in [hash_types.dag_hash, hash_types.build_hash, hash_types.full_hash]:\n setattr(spec, ht.attr, entry[\"hash\"])\n\n spec._concrete = True\n spec._hashes_final = True\n spec.external_path = entry[\"prefix\"]\n spec.origin = \"external-db\"\n spec.namespace = pkg_cls.namespace\n spack.spec.Spec.ensure_valid_variants(spec)\n\n return spec\n\n\ndef entries_to_specs(entries):\n spec_dict = {}\n for entry in entries:\n try:\n spec = spec_from_entry(entry)\n assert spec.concrete, f\"{spec} is not concrete\"\n spec_dict[spec._hash] = spec\n except spack.repo.UnknownPackageError:\n tty.debug(\"Omitting package {0}: no corresponding repo package\".format(entry[\"name\"]))\n except spack.error.SpackError:\n raise\n except Exception:\n tty.warn(\"Could not parse entry: \" + str(entry))\n\n for entry in filter(lambda x: \"dependencies\" in x, entries):\n dependencies = entry[\"dependencies\"]\n for name, properties in dependencies.items():\n dep_hash = properties[\"hash\"]\n depflag = dt.canonicalize(properties[\"type\"])\n if dep_hash in spec_dict:\n if entry[\"hash\"] not in spec_dict:\n continue\n parent_spec = spec_dict[entry[\"hash\"]]\n dep_spec = spec_dict[dep_hash]\n parent_spec._add_dependency(dep_spec, depflag=depflag, virtuals=())\n\n for spec in spec_dict.values():\n spack.spec.reconstruct_virtuals_on_edges(spec)\n\n return spec_dict\n\n\ndef read(path, apply_updates):\n decode_exception_type = json.decoder.JSONDecodeError\n try:\n with open(path, \"r\", encoding=\"utf-8\") as json_file:\n json_data = json.load(json_file)\n\n jsonschema.validate(json_data, manifest_schema)\n except (exceptions.ValidationError, decode_exception_type) as e:\n raise ManifestValidationError(\"error parsing manifest JSON:\", str(e)) from e\n\n specs = entries_to_specs(json_data[\"specs\"])\n tty.debug(\"{0}: {1} specs read from manifest\".format(path, str(len(specs))))\n compilers = []\n if \"compilers\" in json_data:\n for x in json_data[\"compilers\"]:\n # We don't want to fail reading the manifest, if a single compiler fails\n try:\n candidate = compiler_from_entry(x, manifest_path=path)\n except Exception:\n candidate = None\n\n if candidate is None:\n continue\n\n compilers.append(candidate)\n tty.debug(f\"{path}: {str(len(compilers))} compilers read from manifest\")\n # Filter out the compilers that already appear in the configuration\n compilers = spack.compilers.config.select_new_compilers(compilers)\n if apply_updates and compilers:\n try:\n spack.compilers.config.add_compiler_to_config(compilers)\n except Exception:\n warnings.warn(\n f\"Could not add compilers from manifest: {path}\"\n \"\\nPlease reexecute with 'spack -d' and include the stack trace\"\n )\n tty.debug(f\"Include this\\n{traceback.format_exc()}\")\n if apply_updates:\n for spec in specs.values():\n assert spec.concrete, f\"{spec} is not concrete\"\n spack.store.STORE.db.add(spec)\n\n\nclass ManifestValidationError(spack.error.SpackError):\n def __init__(self, msg, long_msg=None):\n super().__init__(msg, long_msg)\n\n\nclass CrayCompilerDetectionError(spack.error.SpackError):\n \"\"\"Raised if a compiler, listed in the Cray manifest, cannot be detected correctly based on\n the paths provided.\n \"\"\"\n" }, { "path": "lib/spack/spack/database.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Spack's installation tracking database.\n\nThe database serves two purposes:\n\n1. It implements a cache on top of a potentially very large Spack\n directory hierarchy, speeding up many operations that would\n otherwise require filesystem access.\n2. It will allow us to track external installations as well as lost\n packages and their dependencies.\n\nPrior to the implementation of this store, a directory layout served\nas the authoritative database of packages in Spack. This module\nprovides a cache and a sanity checking mechanism for what is in the\nfilesystem.\n\"\"\"\n\nimport contextlib\nimport datetime\nimport os\nimport pathlib\nimport sys\nimport time\nfrom json import JSONDecoder\nfrom typing import (\n IO,\n Any,\n Callable,\n Container,\n Dict,\n Generator,\n Iterable,\n List,\n NamedTuple,\n Optional,\n Set,\n Tuple,\n Type,\n Union,\n)\n\nimport spack\nimport spack.repo\n\ntry:\n import uuid\n\n _use_uuid = True\nexcept ImportError:\n _use_uuid = False\n pass\n\nimport spack.deptypes as dt\nimport spack.hash_types as ht\nimport spack.llnl.util.filesystem as fs\nimport spack.llnl.util.tty as tty\nimport spack.spec\nimport spack.traverse as tr\nimport spack.util.lock as lk\nimport spack.util.spack_json as sjson\nimport spack.version as vn\nfrom spack.directory_layout import (\n DirectoryLayout,\n DirectoryLayoutError,\n InconsistentInstallDirectoryError,\n)\nfrom spack.error import SpackError\nfrom spack.util.crypto import bit_length\nfrom spack.util.socket import _gethostname\n\nfrom .enums import InstallRecordStatus\n\n# TODO: Provide an API automatically retrying a build after detecting and\n# TODO: clearing a failure.\n\n#: DB goes in this directory underneath the root\n_DB_DIRNAME = \".spack-db\"\n\n#: DB version. This is stuck in the DB file to track changes in format.\n#: Increment by one when the database format changes.\n#: Versions before 5 were not integers.\n_DB_VERSION = vn.Version(\"8\")\n\n#: For any version combinations here, skip reindex when upgrading.\n#: Reindexing can take considerable time and is not always necessary.\n_REINDEX_NOT_NEEDED_ON_READ = [\n # reindexing takes a significant amount of time, and there's\n # no reason to do it from DB version 0.9.3 to version 5. The\n # only difference is that v5 can contain \"deprecated_for\"\n # fields. So, skip the reindex for this transition. The new\n # version is saved to disk the first time the DB is written.\n (vn.Version(\"0.9.3\"), vn.Version(\"5\")),\n (vn.Version(\"5\"), vn.Version(\"6\")),\n (vn.Version(\"6\"), vn.Version(\"7\")),\n (vn.Version(\"6\"), vn.Version(\"8\")),\n (vn.Version(\"7\"), vn.Version(\"8\")),\n]\n\n#: Default timeout for spack database locks in seconds or None (no timeout).\n#: A balance needs to be struck between quick turnaround for parallel installs\n#: (to avoid excess delays) and waiting long enough when the system is busy\n#: (to ensure the database is updated).\n_DEFAULT_DB_LOCK_TIMEOUT = 120\n\n#: Default timeout for spack package locks in seconds or None (no timeout).\n#: A balance needs to be struck between quick turnaround for parallel installs\n#: (to avoid excess delays when performing a parallel installation) and waiting\n#: long enough for the next possible spec to install (to avoid excessive\n#: checking of the last high priority package) or holding on to a lock (to\n#: ensure a failed install is properly tracked).\n_DEFAULT_PKG_LOCK_TIMEOUT = None\n\n#: Types of dependencies tracked by the database\n#: We store by DAG hash, so we track the dependencies that the DAG hash includes.\n_TRACKED_DEPENDENCIES = ht.dag_hash.depflag\n\n#: Default list of fields written for each install record\nDEFAULT_INSTALL_RECORD_FIELDS = (\n \"spec\",\n \"ref_count\",\n \"path\",\n \"installed\",\n \"explicit\",\n \"installation_time\",\n \"deprecated_for\",\n)\n\n#: File where the database is written\nINDEX_JSON_FILE = \"index.json\"\n\n# Verifier file to check last modification of the DB\n_INDEX_VERIFIER_FILE = \"index_verifier\"\n\n# Lockfile for the database\n_LOCK_FILE = \"lock\"\n\n\ndef reader(version: vn.StandardVersion) -> Type[\"spack.spec.SpecfileReaderBase\"]:\n reader_cls = {\n vn.StandardVersion.from_string(\"5\"): spack.spec.SpecfileV1,\n vn.StandardVersion.from_string(\"6\"): spack.spec.SpecfileV3,\n vn.StandardVersion.from_string(\"7\"): spack.spec.SpecfileV4,\n vn.StandardVersion.from_string(\"8\"): spack.spec.SpecfileV5,\n }\n return reader_cls[version]\n\n\ndef _now() -> float:\n \"\"\"Returns the time since the epoch\"\"\"\n return time.time()\n\n\ndef _autospec(function):\n \"\"\"Decorator that automatically converts the argument of a single-arg\n function to a Spec.\"\"\"\n\n def converter(self, spec_like, *args, **kwargs):\n if not isinstance(spec_like, spack.spec.Spec):\n spec_like = spack.spec.Spec(spec_like)\n return function(self, spec_like, *args, **kwargs)\n\n return converter\n\n\ndef normalize_query(installed: Union[bool, InstallRecordStatus]) -> InstallRecordStatus:\n if installed is True:\n installed = InstallRecordStatus.INSTALLED\n elif installed is False:\n installed = InstallRecordStatus.MISSING\n return installed\n\n\nclass InstallRecord:\n \"\"\"A record represents one installation in the DB.\n\n The record keeps track of the spec for the installation, its\n install path, AND whether or not it is installed. We need the\n installed flag in case a user either:\n\n 1. blew away a directory, or\n 2. used spack uninstall -f to get rid of it\n\n If, in either case, the package was removed but others still\n depend on it, we still need to track its spec, so we don't\n actually remove from the database until a spec has no installed\n dependents left.\n\n Args:\n spec: spec tracked by the install record\n path: path where the spec has been installed\n installed: whether or not the spec is currently installed\n ref_count (int): number of specs that depend on this one\n explicit (bool or None): whether or not this spec was explicitly\n installed, or pulled-in as a dependency of something else\n installation_time (datetime.datetime or None): time of the installation\n \"\"\"\n\n def __init__(\n self,\n spec: \"spack.spec.Spec\",\n path: Optional[str],\n installed: bool,\n ref_count: int = 0,\n explicit: bool = False,\n installation_time: Optional[float] = None,\n deprecated_for: Optional[str] = None,\n in_buildcache: bool = False,\n origin: Optional[str] = None,\n ) -> None:\n self.spec = spec\n self.path = str(path) if path else None\n self.installed = bool(installed)\n self.ref_count = ref_count\n self.explicit = explicit\n self.installation_time = installation_time or _now()\n self.deprecated_for = deprecated_for\n self.in_buildcache = in_buildcache\n self.origin = origin\n\n def install_type_matches(self, installed: InstallRecordStatus) -> bool:\n if self.installed:\n return InstallRecordStatus.INSTALLED in installed\n elif self.deprecated_for:\n return InstallRecordStatus.DEPRECATED in installed\n return InstallRecordStatus.MISSING in installed\n\n def to_dict(self, include_fields=DEFAULT_INSTALL_RECORD_FIELDS):\n rec_dict = {}\n\n for field_name in include_fields:\n if field_name == \"spec\":\n rec_dict.update({\"spec\": self.spec.node_dict_with_hashes()})\n elif field_name == \"deprecated_for\" and self.deprecated_for:\n rec_dict.update({\"deprecated_for\": self.deprecated_for})\n else:\n rec_dict.update({field_name: getattr(self, field_name)})\n\n if self.origin:\n rec_dict[\"origin\"] = self.origin\n\n return rec_dict\n\n @classmethod\n def from_dict(cls, spec, dictionary):\n d = dict(dictionary.items())\n d.pop(\"spec\", None)\n\n # Old databases may have \"None\" for path for externals\n if \"path\" not in d or d[\"path\"] == \"None\":\n d[\"path\"] = None\n\n if \"installed\" not in d:\n d[\"installed\"] = False\n\n return InstallRecord(spec, **d)\n\n\nclass ForbiddenLockError(SpackError):\n \"\"\"Raised when an upstream DB attempts to acquire a lock\"\"\"\n\n\nclass ForbiddenLock:\n def __getattr__(self, name):\n raise ForbiddenLockError(f\"Cannot access attribute '{name}' of lock\")\n\n def __reduce__(self):\n return ForbiddenLock, tuple()\n\n\nclass LockConfiguration(NamedTuple):\n \"\"\"Data class to configure locks in Database objects\n\n Args:\n enable: whether to enable locks or not.\n database_timeout: timeout for the database lock\n package_timeout: timeout for the package lock\n \"\"\"\n\n enable: bool\n database_timeout: Optional[int]\n package_timeout: Optional[int]\n\n\n#: Configure a database to avoid using locks\nNO_LOCK: LockConfiguration = LockConfiguration(\n enable=False, database_timeout=None, package_timeout=None\n)\n\n\n#: Configure the database to use locks without a timeout\nNO_TIMEOUT: LockConfiguration = LockConfiguration(\n enable=True, database_timeout=None, package_timeout=None\n)\n\n#: Default configuration for database locks\nDEFAULT_LOCK_CFG: LockConfiguration = LockConfiguration(\n enable=True,\n database_timeout=_DEFAULT_DB_LOCK_TIMEOUT,\n package_timeout=_DEFAULT_PKG_LOCK_TIMEOUT,\n)\n\n\ndef lock_configuration(configuration):\n \"\"\"Return a LockConfiguration from a spack.config.Configuration object.\"\"\"\n return LockConfiguration(\n enable=configuration.get(\"config:locks\", True),\n database_timeout=configuration.get(\"config:db_lock_timeout\"),\n package_timeout=configuration.get(\"config:package_lock_timeout\"),\n )\n\n\ndef prefix_lock_path(root_dir: Union[str, pathlib.Path]) -> pathlib.Path:\n \"\"\"Returns the path of the prefix lock file, given the root directory.\n\n Args:\n root_dir: root directory containing the database directory\n \"\"\"\n return pathlib.Path(root_dir) / _DB_DIRNAME / \"prefix_lock\"\n\n\ndef failures_lock_path(root_dir: Union[str, pathlib.Path]) -> pathlib.Path:\n \"\"\"Returns the path of the failures lock file, given the root directory.\n\n Args:\n root_dir: root directory containing the database directory\n \"\"\"\n return pathlib.Path(root_dir) / _DB_DIRNAME / \"prefix_failures\"\n\n\nclass SpecLocker:\n \"\"\"Manages acquiring and releasing read or write locks on concrete specs.\"\"\"\n\n def __init__(self, lock_path: Union[str, pathlib.Path], default_timeout: Optional[float]):\n self.lock_path = pathlib.Path(lock_path)\n self.default_timeout = default_timeout\n\n # Maps (spec.dag_hash(), spec.name) to the corresponding lock object\n self.locks: Dict[Tuple[str, str], lk.Lock] = {}\n\n def lock(self, spec: \"spack.spec.Spec\", timeout: Optional[float] = None) -> lk.Lock:\n \"\"\"Returns a lock on a concrete spec.\n\n The lock is a byte range lock on the nth byte of a file.\n\n The lock file is ``self.lock_path``.\n\n n is the sys.maxsize-bit prefix of the DAG hash. This makes likelihood of collision is\n very low AND it gives us readers-writer lock semantics with just a single lockfile, so\n no cleanup required.\n \"\"\"\n assert spec.concrete, \"cannot lock a non-concrete spec\"\n timeout = timeout or self.default_timeout\n key = self._lock_key(spec)\n\n if key not in self.locks:\n self.locks[key] = self.raw_lock(spec, timeout=timeout)\n else:\n self.locks[key].default_timeout = timeout\n\n return self.locks[key]\n\n def raw_lock(self, spec: \"spack.spec.Spec\", timeout: Optional[float] = None) -> lk.Lock:\n \"\"\"Returns a raw lock for a Spec, but doesn't keep track of it.\"\"\"\n return lk.Lock(\n str(self.lock_path),\n start=spec.dag_hash_bit_prefix(bit_length(sys.maxsize)),\n length=1,\n default_timeout=timeout,\n desc=spec.name,\n )\n\n def has_lock(self, spec: \"spack.spec.Spec\") -> bool:\n \"\"\"Returns True if the spec is already managed by this spec locker\"\"\"\n return self._lock_key(spec) in self.locks\n\n def _lock_key(self, spec: \"spack.spec.Spec\") -> Tuple[str, str]:\n return (spec.dag_hash(), spec.name)\n\n @contextlib.contextmanager\n def write_lock(self, spec: \"spack.spec.Spec\") -> Generator[\"SpecLocker\", None, None]:\n lock = self.lock(spec)\n lock.acquire_write()\n\n try:\n yield self\n except lk.LockError:\n # This addresses the case where a nested lock attempt fails inside\n # of this context manager\n raise\n except (Exception, KeyboardInterrupt):\n lock.release_write()\n raise\n else:\n lock.release_write()\n\n def clear(self, spec: \"spack.spec.Spec\") -> Tuple[bool, Optional[lk.Lock]]:\n key = self._lock_key(spec)\n lock = self.locks.pop(key, None)\n return bool(lock), lock\n\n def clear_all(self, clear_fn: Optional[Callable[[lk.Lock], Any]] = None) -> None:\n if clear_fn is not None:\n for lock in self.locks.values():\n clear_fn(lock)\n self.locks.clear()\n\n\nclass FailureTracker:\n \"\"\"Tracks installation failures.\n\n Prefix failure marking takes the form of a byte range lock on the nth\n byte of a file for coordinating between concurrent parallel build\n processes and a persistent file, named with the full hash and\n containing the spec, in a subdirectory of the database to enable\n persistence across overlapping but separate related build processes.\n\n The failure lock file lives alongside the install DB.\n\n ``n`` is the sys.maxsize-bit prefix of the associated DAG hash to make\n the likelihood of collision very low with no cleanup required.\n \"\"\"\n\n #: root directory of the failure tracker\n dir: pathlib.Path\n\n #: File for locking particular concrete spec hashes\n locker: SpecLocker\n\n def __init__(self, root_dir: Union[str, pathlib.Path], default_timeout: Optional[float]):\n #: Ensure a persistent location for dealing with parallel installation\n #: failures (e.g., across near-concurrent processes).\n self.dir = pathlib.Path(root_dir) / _DB_DIRNAME / \"failures\"\n self.locker = SpecLocker(failures_lock_path(root_dir), default_timeout=default_timeout)\n\n def _ensure_parent_directories(self) -> None:\n \"\"\"Ensure that parent directories of the FailureTracker exist.\n\n Accesses the filesystem only once, the first time it's called on a given FailureTracker.\n \"\"\"\n self.dir.mkdir(parents=True, exist_ok=True)\n\n def clear(self, spec: \"spack.spec.Spec\", force: bool = False) -> None:\n \"\"\"Removes any persistent and cached failure tracking for the spec.\n\n see :meth:`mark`.\n\n Args:\n spec: the spec whose failure indicators are being removed\n force: True if the failure information should be cleared when a failure lock\n exists for the file, or False if the failure should not be cleared (e.g.,\n it may be associated with a concurrent build)\n \"\"\"\n locked = self.lock_taken(spec)\n if locked and not force:\n tty.msg(f\"Retaining failure marking for {spec.name} due to lock\")\n return\n\n if locked:\n tty.warn(f\"Removing failure marking despite lock for {spec.name}\")\n\n succeeded, lock = self.locker.clear(spec)\n if succeeded and lock is not None:\n lock.release_write()\n\n if self.persistent_mark(spec):\n path = self._path(spec)\n tty.debug(f\"Removing failure marking for {spec.name}\")\n try:\n path.unlink()\n except OSError as err:\n tty.warn(\n f\"Unable to remove failure marking for {spec.name} ({str(path)}): {str(err)}\"\n )\n\n def clear_all(self) -> None:\n \"\"\"Force remove install failure tracking files.\"\"\"\n tty.debug(\"Releasing prefix failure locks\")\n self.locker.clear_all(\n clear_fn=lambda x: x.release_write() if x.is_write_locked() else True\n )\n\n tty.debug(\"Removing prefix failure tracking files\")\n try:\n marks = os.listdir(str(self.dir))\n except FileNotFoundError:\n return # directory doesn't exist yet\n except OSError as exc:\n tty.warn(f\"Unable to remove failure marking files: {str(exc)}\")\n return\n\n for fail_mark in marks:\n try:\n (self.dir / fail_mark).unlink()\n except OSError as exc:\n tty.warn(f\"Unable to remove failure marking file {fail_mark}: {str(exc)}\")\n\n def mark(self, spec: \"spack.spec.Spec\") -> lk.Lock:\n \"\"\"Marks a spec as failing to install.\n\n Args:\n spec: spec that failed to install\n \"\"\"\n self._ensure_parent_directories()\n\n # Dump the spec to the failure file for (manual) debugging purposes\n path = self._path(spec)\n path.write_text(spec.to_json())\n\n # Also ensure a failure lock is taken to prevent cleanup removal\n # of failure status information during a concurrent parallel build.\n if not self.locker.has_lock(spec):\n try:\n mark = self.locker.lock(spec)\n mark.acquire_write()\n except lk.LockTimeoutError:\n # Unlikely that another process failed to install at the same\n # time but log it anyway.\n tty.debug(f\"PID {os.getpid()} failed to mark install failure for {spec.name}\")\n tty.warn(f\"Unable to mark {spec.name} as failed.\")\n\n return self.locker.lock(spec)\n\n def has_failed(self, spec: \"spack.spec.Spec\") -> bool:\n \"\"\"Return True if the spec is marked as failed.\"\"\"\n # The failure was detected in this process.\n if self.locker.has_lock(spec):\n return True\n\n # The failure was detected by a concurrent process (e.g., an srun),\n # which is expected to be holding a write lock if that is the case.\n if self.lock_taken(spec):\n return True\n\n # Determine if the spec may have been marked as failed by a separate\n # spack build process running concurrently.\n return self.persistent_mark(spec)\n\n def lock_taken(self, spec: \"spack.spec.Spec\") -> bool:\n \"\"\"Return True if another process has a failure lock on the spec.\"\"\"\n check = self.locker.raw_lock(spec)\n return check.is_write_locked()\n\n def persistent_mark(self, spec: \"spack.spec.Spec\") -> bool:\n \"\"\"Determine if the spec has a persistent failure marking.\"\"\"\n return self._path(spec).exists()\n\n def _path(self, spec: \"spack.spec.Spec\") -> pathlib.Path:\n \"\"\"Return the path to the spec's failure file, which may not exist.\"\"\"\n assert spec.concrete, \"concrete spec required for failure path\"\n return self.dir / f\"{spec.name}-{spec.dag_hash()}\"\n\n\nSelectType = Callable[[InstallRecord], bool]\n\n\nclass Database:\n #: Fields written for each install record\n record_fields: Tuple[str, ...] = DEFAULT_INSTALL_RECORD_FIELDS\n\n def __init__(\n self,\n root: str,\n *,\n upstream_dbs: Optional[List[\"Database\"]] = None,\n is_upstream: bool = False,\n lock_cfg: LockConfiguration = DEFAULT_LOCK_CFG,\n layout: Optional[DirectoryLayout] = None,\n ) -> None:\n \"\"\"Database for Spack installations.\n\n A Database is a cache of Specs data from ``$prefix/spec.yaml`` files\n in Spack installation directories.\n\n Database files (data and lock files) are stored under ``root/.spack-db``, which is\n created if it does not exist. This is the \"database directory\".\n\n The database will attempt to read an ``index.json`` file in the database directory.\n If that does not exist, it will create a database when needed by scanning the entire\n store root for ``spec.json`` files according to Spack's directory layout.\n\n Args:\n root: root directory where to create the database directory.\n upstream_dbs: upstream databases for this repository.\n is_upstream: whether this repository is an upstream.\n lock_cfg: configuration for the locks to be used by this repository.\n Relevant only if the repository is not an upstream.\n \"\"\"\n self.root = root\n self.database_directory = pathlib.Path(self.root) / _DB_DIRNAME\n self.layout = layout\n\n # Set up layout of database files within the db dir\n self._index_path = self.database_directory / INDEX_JSON_FILE\n self._verifier_path = self.database_directory / _INDEX_VERIFIER_FILE\n self._lock_path = self.database_directory / _LOCK_FILE\n\n self.is_upstream = is_upstream\n self.last_seen_verifier = \"\"\n # Failed write transactions (interrupted by exceptions) will alert\n # _write. When that happens, we set this flag to indicate that\n # future read/write transactions should re-read the DB. Normally it\n # would make more sense to resolve this at the end of the transaction\n # but typically a failed transaction will terminate the running\n # instance of Spack and we don't want to incur an extra read in that\n # case, so we defer the cleanup to when we begin the next transaction\n self._state_is_inconsistent = False\n\n # initialize rest of state.\n self.db_lock_timeout = lock_cfg.database_timeout\n tty.debug(f\"DATABASE LOCK TIMEOUT: {str(self.db_lock_timeout)}s\")\n\n self.lock: Union[ForbiddenLock, lk.Lock]\n if self.is_upstream:\n self.lock = ForbiddenLock()\n else:\n self.lock = lk.Lock(\n str(self._lock_path),\n default_timeout=self.db_lock_timeout,\n desc=\"database\",\n enable=lock_cfg.enable,\n )\n self._data: Dict[str, InstallRecord] = {}\n\n # For every installed spec we keep track of its install prefix, so that\n # we can answer the simple query whether a given path is already taken\n # before installing a different spec.\n self._installed_prefixes: Set[str] = set()\n\n self.upstream_dbs = list(upstream_dbs) if upstream_dbs else []\n\n self._write_transaction_impl = lk.WriteTransaction\n self._read_transaction_impl = lk.ReadTransaction\n self._db_version: Optional[vn.ConcreteVersion] = None\n\n @property\n def db_version(self) -> vn.ConcreteVersion:\n if self._db_version is None:\n raise AttributeError(\"version not set -- DB has not been read yet\")\n return self._db_version\n\n @db_version.setter\n def db_version(self, value: vn.ConcreteVersion):\n self._db_version = value\n\n def _ensure_parent_directories(self):\n \"\"\"Create the parent directory for the DB, if necessary.\"\"\"\n if not self.is_upstream:\n self.database_directory.mkdir(parents=True, exist_ok=True)\n\n def write_transaction(self):\n \"\"\"Get a write lock context manager for use in a ``with`` block.\"\"\"\n return self._write_transaction_impl(self.lock, acquire=self._read, release=self._write)\n\n def read_transaction(self):\n \"\"\"Get a read lock context manager for use in a ``with`` block.\"\"\"\n return self._read_transaction_impl(self.lock, acquire=self._read)\n\n def _write_to_file(self, stream):\n \"\"\"Write out the database in JSON format to the stream passed\n as argument.\n\n This function does not do any locking or transactions.\n \"\"\"\n self._ensure_parent_directories()\n\n # map from per-spec hash code to installation record.\n installs = dict(\n (k, v.to_dict(include_fields=self.record_fields)) for k, v in self._data.items()\n )\n\n # database includes installation list and version.\n\n # NOTE: this DB version does not handle multiple installs of\n # the same spec well. If there are 2 identical specs with\n # different paths, it can't differentiate.\n # TODO: fix this before we support multiple install locations.\n database = {\n \"database\": {\n # TODO: move this to a top-level _meta section if we ever\n # TODO: bump the DB version to 7\n \"version\": str(_DB_VERSION),\n # dictionary of installation records, keyed by DAG hash\n \"installs\": installs,\n }\n }\n\n try:\n sjson.dump(database, stream)\n except (TypeError, ValueError) as e:\n raise sjson.SpackJSONError(\"error writing JSON database:\", str(e))\n\n def _read_spec_from_dict(self, spec_reader, hash_key, installs, hash=ht.dag_hash):\n \"\"\"Recursively construct a spec from a hash in a YAML database.\n\n Does not do any locking.\n \"\"\"\n spec_dict = installs[hash_key][\"spec\"]\n\n # Install records don't include hash with spec, so we add it in here\n # to ensure it is read properly.\n if \"name\" not in spec_dict.keys():\n # old format, can't update format here\n for name in spec_dict:\n spec_dict[name][\"hash\"] = hash_key\n else:\n # new format, already a singleton\n spec_dict[hash.name] = hash_key\n\n # Build spec from dict first.\n return spec_reader.from_node_dict(spec_dict)\n\n def db_for_spec_hash(self, hash_key):\n with self.read_transaction():\n if hash_key in self._data:\n return self\n\n for db in self.upstream_dbs:\n if hash_key in db._data:\n return db\n\n def query_by_spec_hash(\n self, hash_key: str, data: Optional[Dict[str, InstallRecord]] = None\n ) -> Tuple[bool, Optional[InstallRecord]]:\n \"\"\"Get a spec for hash, and whether it's installed upstream.\n\n Return:\n Tuple of bool and optional InstallRecord. The bool tells us whether the record is from\n an upstream. Its InstallRecord is also returned if available (the record must be\n checked to know whether the hash is installed).\n\n If the record is available locally, this function will always have\n a preference for returning that, even if it is not installed locally\n and is installed upstream.\n \"\"\"\n if data and hash_key in data:\n return False, data[hash_key]\n if not data:\n with self.read_transaction():\n if hash_key in self._data:\n return False, self._data[hash_key]\n for db in self.upstream_dbs:\n if hash_key in db._data:\n return True, db._data[hash_key]\n return False, None\n\n def query_local_by_spec_hash(self, hash_key: str) -> Optional[InstallRecord]:\n \"\"\"Get a spec by hash in the local database\n\n Return:\n InstallRecord when installed locally, otherwise None.\"\"\"\n with self.read_transaction():\n return self._data.get(hash_key, None)\n\n def _assign_build_spec(\n self,\n spec_reader: Type[\"spack.spec.SpecfileReaderBase\"],\n hash_key: str,\n installs: dict,\n data: Dict[str, InstallRecord],\n ):\n # Add dependencies from other records in the install DB to\n # form a full spec.\n spec = data[hash_key].spec\n spec_node_dict = installs[hash_key][\"spec\"]\n if \"name\" not in spec_node_dict:\n # old format\n spec_node_dict = spec_node_dict[spec.name]\n if \"build_spec\" in spec_node_dict:\n assert spec_reader.SPEC_VERSION >= 2, \"SpecfileV1 spec cannot have build_spec\"\n _, bhash, _ = spec_reader.extract_build_spec_info_from_node_dict(spec_node_dict)\n _, build_spec = self.query_by_spec_hash(bhash, data=data)\n assert build_spec is not None, f\"build_spec with hash {bhash} not found in database\"\n spec._build_spec = build_spec.spec\n\n def _assign_dependencies(\n self,\n spec_reader: Type[\"spack.spec.SpecfileReaderBase\"],\n hash_key: str,\n installs: dict,\n data: Dict[str, InstallRecord],\n ):\n # Add dependencies from other records in the install DB to\n # form a full spec.\n spec = data[hash_key].spec\n spec_node_dict = installs[hash_key][\"spec\"]\n if \"name\" not in spec_node_dict:\n # old format\n spec_node_dict = spec_node_dict[spec.name]\n if \"dependencies\" in spec_node_dict:\n yaml_deps = spec_node_dict[\"dependencies\"]\n for dname, dhash, dtypes, _, virtuals, direct in spec_reader.read_specfile_dep_specs(\n yaml_deps\n ):\n # It is important that we always check upstream installations in the same order,\n # and that we always check the local installation first: if a downstream Spack\n # installs a package then dependents in that installation could be using it. If a\n # hash is installed locally and upstream, there isn't enough information to\n # determine which one a local package depends on, so the convention ensures that\n # this isn't an issue.\n _, record = self.query_by_spec_hash(dhash, data=data)\n child = record.spec if record else None\n\n if not child:\n tty.warn(\n f\"Missing dependency not in database: \"\n f\"{spec.cformat('{name}{/hash:7}')} needs {dname}-{dhash[:7]}\"\n )\n continue\n\n spec._add_dependency(\n child, depflag=dt.canonicalize(dtypes), virtuals=virtuals, direct=direct\n )\n\n def _read_from_file(self, filename: pathlib.Path, *, reindex: bool = False) -> None:\n \"\"\"Fill database from file, do not maintain old data.\n\n Does not do any locking.\n \"\"\"\n with filename.open(\"r\", encoding=\"utf-8\") as f:\n self._read_from_stream(f, reindex=reindex)\n\n def _read_from_stream(self, stream: IO[str], *, reindex: bool = False) -> None:\n \"\"\"Fill database from a text stream, do not maintain old data.\n Translate the spec portions from node-dict form to spec form.\n\n Does not do any locking.\n \"\"\"\n source = getattr(stream, \"name\", None) or self._index_path\n try:\n # In the future we may use a stream of JSON objects, hence `raw_decode` for compat.\n fdata, _ = JSONDecoder().raw_decode(stream.read())\n except Exception as e:\n raise CorruptDatabaseError(f\"error parsing database at {source}:\", str(e)) from e\n\n if fdata is None:\n return\n\n def check(cond, msg):\n if not cond:\n raise CorruptDatabaseError(f\"Spack database is corrupt: {msg}\", str(source))\n\n check(\"database\" in fdata, \"no 'database' attribute in JSON DB.\")\n\n # High-level file checks\n db = fdata[\"database\"]\n check(\"version\" in db, \"no 'version' in JSON DB.\")\n\n self.db_version = vn.StandardVersion.from_string(db[\"version\"])\n if self.db_version > _DB_VERSION:\n raise InvalidDatabaseVersionError(self, _DB_VERSION, self.db_version)\n elif self.db_version < _DB_VERSION:\n installs = self._handle_old_db_versions_read(check, db, reindex=reindex)\n else:\n installs = self._handle_current_version_read(check, db)\n\n spec_reader = reader(self.db_version)\n\n def invalid_record(hash_key, error):\n return CorruptDatabaseError(\n f\"Invalid record in Spack database: hash: {hash_key}, cause: \"\n f\"{type(error).__name__}: {error}\",\n str(source),\n )\n\n # Build up the database in three passes:\n #\n # 1. Read in all specs without dependencies.\n # 2. Hook dependencies up among specs.\n # 3. Mark all specs concrete.\n #\n # The database is built up so that ALL specs in it share nodes\n # (i.e., its specs are a true Merkle DAG, unlike most specs.)\n\n # Pass 1: Iterate through database and build specs w/o dependencies\n data: Dict[str, InstallRecord] = {}\n installed_prefixes: Set[str] = set()\n for hash_key, rec in installs.items():\n try:\n # This constructs a spec DAG from the list of all installs\n spec = self._read_spec_from_dict(spec_reader, hash_key, installs)\n\n # Insert the brand new spec in the database. Each\n # spec has its own copies of its dependency specs.\n # TODO: would a more immmutable spec implementation simplify\n # this?\n data[hash_key] = InstallRecord.from_dict(spec, rec)\n\n if not spec.external and \"installed\" in rec and rec[\"installed\"]:\n installed_prefixes.add(rec[\"path\"])\n except Exception as e:\n raise invalid_record(hash_key, e) from e\n\n # Pass 2: Assign dependencies once all specs are created.\n for hash_key in data:\n try:\n self._assign_build_spec(spec_reader, hash_key, installs, data)\n self._assign_dependencies(spec_reader, hash_key, installs, data)\n except MissingDependenciesError:\n raise\n except Exception as e:\n raise invalid_record(hash_key, e) from e\n\n # Pass 3: Mark all specs concrete. Specs representing real\n # installations must be explicitly marked.\n # We do this *after* all dependencies are connected because if we\n # do it *while* we're constructing specs,it causes hashes to be\n # cached prematurely.\n for hash_key, rec in data.items():\n rec.spec._mark_root_concrete()\n\n self._data = data\n self._installed_prefixes = installed_prefixes\n\n def _handle_current_version_read(self, check, db):\n check(\"installs\" in db, \"no 'installs' in JSON DB.\")\n installs = db[\"installs\"]\n return installs\n\n def _handle_old_db_versions_read(self, check, db, *, reindex: bool):\n if reindex is False and not self.is_upstream:\n self.raise_explicit_database_upgrade_error()\n\n if not self.is_readable():\n raise DatabaseNotReadableError(\n f\"cannot read database v{self.db_version} at {self.root}\"\n )\n\n return self._handle_current_version_read(check, db)\n\n def is_readable(self) -> bool:\n \"\"\"Returns true if this DB can be read without reindexing\"\"\"\n return (self.db_version, _DB_VERSION) in _REINDEX_NOT_NEEDED_ON_READ\n\n def raise_explicit_database_upgrade_error(self):\n \"\"\"Raises an ExplicitDatabaseUpgradeError with an appropriate message\"\"\"\n raise ExplicitDatabaseUpgradeError(\n f\"database is v{self.db_version}, but Spack v{spack.__version__} needs v{_DB_VERSION}\",\n long_message=(\n f\"You will need to either:\"\n f\"\\n\"\n f\"\\n 1. Migrate the database to v{_DB_VERSION}, or\"\n f\"\\n 2. Use a new database by changing config:install_tree:root.\"\n f\"\\n\"\n f\"\\nTo migrate the database at {self.root} \"\n f\"\\nto version {_DB_VERSION}, run:\"\n f\"\\n\"\n f\"\\n spack reindex\"\n f\"\\n\"\n f\"\\nNOTE that if you do this, older Spack versions will no longer\"\n f\"\\nbe able to read the database. However, `spack reindex` will create a backup,\"\n f\"\\nin case you want to revert.\"\n f\"\\n\"\n f\"\\nIf you still need your old database, you can instead run\"\n f\"\\n`spack config edit config` and set install_tree:root to a new location.\"\n ),\n )\n\n def reindex(self):\n \"\"\"Build database index from scratch based on a directory layout.\n\n Locks the DB if it isn't locked already.\n \"\"\"\n if self.is_upstream:\n raise UpstreamDatabaseLockingError(\"Cannot reindex an upstream database\")\n\n self._ensure_parent_directories()\n\n # Special transaction to avoid recursive reindex calls and to\n # ignore errors if we need to rebuild a corrupt database.\n def _read_suppress_error():\n try:\n with self._index_path.open(\"r\", encoding=\"utf-8\") as f:\n self._read_from_stream(f, reindex=True)\n except FileNotFoundError:\n pass\n except (CorruptDatabaseError, DatabaseNotReadableError):\n self._data = {}\n self._installed_prefixes = set()\n\n with lk.WriteTransaction(self.lock, acquire=_read_suppress_error, release=self._write):\n old_installed_prefixes, self._installed_prefixes = self._installed_prefixes, set()\n old_data, self._data = self._data, {}\n try:\n self._reindex(old_data)\n except BaseException:\n # If anything explodes, restore old data, skip write.\n self._data = old_data\n self._installed_prefixes = old_installed_prefixes\n raise\n\n def _reindex(self, old_data: Dict[str, InstallRecord]):\n # Specs on the file system are the source of truth for record.spec. The old database values\n # if available are the source of truth for the rest of the record.\n assert self.layout, \"Database layout must be set to reindex\"\n\n specs_from_fs = self.layout.all_specs()\n deprecated_for = self.layout.deprecated_for(specs_from_fs)\n\n known_specs: List[spack.spec.Spec] = [\n *specs_from_fs,\n *(deprecated for _, deprecated in deprecated_for),\n *(rec.spec for rec in old_data.values()),\n ]\n\n upstream_hashes = {\n dag_hash for upstream in self.upstream_dbs for dag_hash in upstream._data\n }\n upstream_hashes.difference_update(spec.dag_hash() for spec in known_specs)\n\n def create_node(edge: spack.spec.DependencySpec, is_upstream: bool):\n if is_upstream:\n return\n\n self._data[edge.spec.dag_hash()] = InstallRecord(\n spec=edge.spec.copy(deps=False),\n path=edge.spec.external_path if edge.spec.external else None,\n installed=edge.spec.external,\n )\n\n # Store all nodes of known specs, excluding ones found in upstreams\n tr.traverse_breadth_first_with_visitor(\n known_specs,\n tr.CoverNodesVisitor(\n NoUpstreamVisitor(upstream_hashes, create_node), key=tr.by_dag_hash\n ),\n )\n\n # Store the prefix and other information for specs were found on the file system\n for s in specs_from_fs:\n record = self._data[s.dag_hash()]\n record.path = s.prefix\n record.installed = True\n record.explicit = True # conservative assumption\n record.installation_time = os.stat(s.prefix).st_ctime\n\n # Deprecate specs\n for new, old in deprecated_for:\n self._data[old.dag_hash()].deprecated_for = new.dag_hash()\n\n # Copy data we have from the old database\n for old_record in old_data.values():\n record = self._data[old_record.spec.dag_hash()]\n record.explicit = old_record.explicit\n record.installation_time = old_record.installation_time\n record.origin = old_record.origin\n record.deprecated_for = old_record.deprecated_for\n\n # Warn when the spec has been removed from the file system (i.e. it was not detected)\n if not record.installed and old_record.installed:\n tty.warn(\n f\"Spec {old_record.spec.short_spec} was marked installed in the database \"\n \"but was not found on the file system. It is now marked as missing.\"\n )\n\n def create_edge(edge: spack.spec.DependencySpec, is_upstream: bool):\n if not edge.parent:\n return\n parent_record = self._data[edge.parent.dag_hash()]\n if is_upstream:\n upstream, child_record = self.query_by_spec_hash(edge.spec.dag_hash())\n assert upstream and child_record, \"Internal error: upstream spec not found\"\n else:\n child_record = self._data[edge.spec.dag_hash()]\n parent_record.spec._add_dependency(\n child_record.spec, depflag=edge.depflag, virtuals=edge.virtuals\n )\n\n # Then store edges\n tr.traverse_breadth_first_with_visitor(\n known_specs,\n tr.CoverEdgesVisitor(\n NoUpstreamVisitor(upstream_hashes, create_edge), key=tr.by_dag_hash\n ),\n )\n\n # Finally update the ref counts\n for record in self._data.values():\n for dep in record.spec.dependencies(deptype=_TRACKED_DEPENDENCIES):\n dep_record = self._data.get(dep.dag_hash())\n if dep_record: # dep might be upstream\n dep_record.ref_count += 1\n if record.deprecated_for:\n self._data[record.deprecated_for].ref_count += 1\n\n self._check_ref_counts()\n\n def _check_ref_counts(self):\n \"\"\"Ensure consistency of reference counts in the DB.\n\n Raise an AssertionError if something is amiss.\n\n Does no locking.\n \"\"\"\n counts: Dict[str, int] = {}\n for key, rec in self._data.items():\n counts.setdefault(key, 0)\n for dep in rec.spec.dependencies(deptype=_TRACKED_DEPENDENCIES):\n dep_key = dep.dag_hash()\n counts.setdefault(dep_key, 0)\n counts[dep_key] += 1\n\n if rec.deprecated_for:\n counts.setdefault(rec.deprecated_for, 0)\n counts[rec.deprecated_for] += 1\n\n for rec in self._data.values():\n key = rec.spec.dag_hash()\n expected = counts[key]\n found = rec.ref_count\n if not expected == found:\n raise AssertionError(\n \"Invalid ref_count: %s: %d (expected %d), in DB %s\"\n % (key, found, expected, self._index_path)\n )\n\n def _write(self, type=None, value=None, traceback=None):\n \"\"\"Write the in-memory database index to its file path.\n\n This is a helper function called by the WriteTransaction context\n manager. If there is an exception while the write lock is active,\n nothing will be written to the database file, but the in-memory\n database *may* be left in an inconsistent state. It will be consistent\n after the start of the next transaction, when it read from disk again.\n\n This routine does no locking.\n \"\"\"\n self._ensure_parent_directories()\n\n # Do not write if exceptions were raised\n if type is not None:\n # A failure interrupted a transaction, so we should record that\n # the Database is now in an inconsistent state: we should\n # restore it in the next transaction\n self._state_is_inconsistent = True\n return\n\n temp_file = str(self._index_path) + (\".%s.%s.temp\" % (_gethostname(), os.getpid()))\n\n # Write a temporary database file them move it into place\n try:\n with open(temp_file, \"w\", encoding=\"utf-8\") as f:\n self._write_to_file(f)\n fs.rename(temp_file, str(self._index_path))\n\n if _use_uuid:\n with self._verifier_path.open(\"w\", encoding=\"utf-8\") as f:\n new_verifier = str(uuid.uuid4())\n f.write(new_verifier)\n self.last_seen_verifier = new_verifier\n except BaseException as e:\n tty.debug(e)\n # Clean up temp file if something goes wrong.\n if os.path.exists(temp_file):\n os.remove(temp_file)\n raise\n\n def _read(self):\n \"\"\"Re-read Database from the data in the set location. This does no locking.\"\"\"\n try:\n index_file = self._index_path.open(\"r\", encoding=\"utf-8\")\n except FileNotFoundError:\n if self.is_upstream:\n tty.warn(f\"upstream not found: {self._index_path}\")\n return\n\n with index_file as f:\n current_verifier = \"\"\n if _use_uuid:\n try:\n with self._verifier_path.open(\"r\", encoding=\"utf-8\") as vf:\n current_verifier = vf.read()\n except BaseException:\n pass\n if (current_verifier != self.last_seen_verifier) or (current_verifier == \"\"):\n self.last_seen_verifier = current_verifier\n # Read from file if a database exists\n self._read_from_stream(f)\n elif self._state_is_inconsistent:\n self._read_from_stream(f)\n self._state_is_inconsistent = False\n\n def _add(\n self,\n spec: \"spack.spec.Spec\",\n explicit: bool = False,\n installation_time: Optional[float] = None,\n allow_missing: bool = False,\n ):\n \"\"\"Add an install record for this spec to the database.\n\n Also ensures dependencies are present and updated in the DB as either installed or missing.\n\n Args:\n spec: spec to be added\n explicit:\n Possible values: True, False, any\n\n A spec that was installed following a specific user request is marked as explicit.\n If instead it was pulled-in as a dependency of a user requested spec it's\n considered implicit.\n\n installation_time:\n Date and time of installation\n allow_missing: if True, don't warn when installation is not found on on disk\n This is useful when installing specs without build/test deps.\n \"\"\"\n if not spec.concrete:\n raise NonConcreteSpecAddError(\"Specs added to DB must be concrete.\")\n\n key = spec.dag_hash()\n spec_pkg_hash = spec._package_hash # type: ignore[attr-defined]\n upstream, record = self.query_by_spec_hash(key)\n if upstream and record and record.installed:\n return\n\n installation_time = installation_time or _now()\n\n for edge in spec.edges_to_dependencies(depflag=_TRACKED_DEPENDENCIES):\n if edge.spec.dag_hash() in self._data:\n continue\n self._add(\n edge.spec,\n explicit=False,\n installation_time=installation_time,\n # allow missing build / test only deps\n allow_missing=allow_missing or edge.depflag & (dt.BUILD | dt.TEST) == edge.depflag,\n )\n\n if spec.spliced:\n self._add(spec.build_spec, explicit=False, allow_missing=True)\n\n # Make sure the directory layout agrees whether the spec is installed\n if not spec.external and self.layout:\n path = self.layout.path_for_spec(spec)\n installed = False\n try:\n self.layout.ensure_installed(spec)\n installed = True\n self._installed_prefixes.add(path)\n except DirectoryLayoutError as e:\n if not (allow_missing and isinstance(e, InconsistentInstallDirectoryError)):\n action = \"updated\" if key in self._data else \"registered\"\n tty.warn(\n f\"{spec.short_spec} is being {action} in the database with prefix {path}, \"\n \"but this directory does not contain an installation of \"\n f\"the spec, due to: {e}\"\n )\n elif spec.external_path:\n path = spec.external_path\n installed = True\n else:\n path = None\n installed = True\n\n if key not in self._data:\n # Create a new install record with no deps initially.\n new_spec = spec.copy(deps=False)\n self._data[key] = InstallRecord(\n new_spec,\n path=path,\n installed=installed,\n ref_count=0,\n explicit=explicit,\n installation_time=installation_time,\n origin=None if not hasattr(spec, \"origin\") else spec.origin,\n )\n\n # Connect dependencies from the DB to the new copy.\n for dep in spec.edges_to_dependencies(depflag=_TRACKED_DEPENDENCIES):\n dkey = dep.spec.dag_hash()\n upstream, record = self.query_by_spec_hash(dkey)\n assert record, f\"Missing dependency {dep.spec.short_spec} in DB\"\n new_spec._add_dependency(record.spec, depflag=dep.depflag, virtuals=dep.virtuals)\n if not upstream:\n record.ref_count += 1\n\n # Mark concrete once everything is built, and preserve the original hashes of concrete\n # specs.\n new_spec._mark_concrete()\n new_spec._hash = key\n new_spec._package_hash = spec_pkg_hash\n\n else:\n # It is already in the database\n self._data[key].installed = installed\n self._data[key].installation_time = _now()\n\n self._data[key].explicit = explicit\n\n @_autospec\n def add(self, spec: \"spack.spec.Spec\", *, explicit: bool = False, allow_missing=False) -> None:\n \"\"\"Add spec at path to database, locking and reading DB to sync.\n\n ``add()`` will lock and read from the DB on disk.\n\n \"\"\"\n # TODO: ensure that spec is concrete?\n # Entire add is transactional.\n with self.write_transaction():\n self._add(spec, explicit=explicit, allow_missing=allow_missing)\n\n def _get_matching_spec_key(self, spec: \"spack.spec.Spec\", **kwargs) -> str:\n \"\"\"Get the exact spec OR get a single spec that matches.\"\"\"\n key = spec.dag_hash()\n _, record = self.query_by_spec_hash(key)\n if not record:\n match = self.query_one(spec, **kwargs)\n if match:\n return match.dag_hash()\n raise NoSuchSpecError(spec)\n return key\n\n @_autospec\n def get_record(self, spec: \"spack.spec.Spec\", **kwargs) -> Optional[InstallRecord]:\n key = self._get_matching_spec_key(spec, **kwargs)\n _, record = self.query_by_spec_hash(key)\n return record\n\n def _decrement_ref_count(self, spec: \"spack.spec.Spec\") -> None:\n key = spec.dag_hash()\n\n if key not in self._data:\n # TODO: print something here? DB is corrupt, but\n # not much we can do.\n return\n\n rec = self._data[key]\n rec.ref_count -= 1\n\n if rec.ref_count == 0 and not rec.installed:\n del self._data[key]\n\n for dep in spec.dependencies(deptype=_TRACKED_DEPENDENCIES):\n self._decrement_ref_count(dep)\n\n def _increment_ref_count(self, spec: \"spack.spec.Spec\") -> None:\n key = spec.dag_hash()\n\n if key not in self._data:\n return\n\n rec = self._data[key]\n rec.ref_count += 1\n\n def _remove(self, spec: \"spack.spec.Spec\") -> \"spack.spec.Spec\":\n \"\"\"Non-locking version of remove(); does real work.\"\"\"\n key = self._get_matching_spec_key(spec)\n rec = self._data[key]\n\n # This install prefix is now free for other specs to use, even if the\n # spec is only marked uninstalled.\n if not rec.spec.external and rec.installed and rec.path:\n self._installed_prefixes.remove(rec.path)\n\n if rec.ref_count > 0:\n rec.installed = False\n return rec.spec\n\n del self._data[key]\n\n # Remove any reference to this node from dependencies and\n # decrement the reference count\n rec.spec.detach(deptype=_TRACKED_DEPENDENCIES)\n for dep in rec.spec.dependencies(deptype=_TRACKED_DEPENDENCIES):\n self._decrement_ref_count(dep)\n\n if rec.deprecated_for:\n new_spec = self._data[rec.deprecated_for].spec\n self._decrement_ref_count(new_spec)\n\n # Returns the concrete spec so we know it in the case where a\n # query spec was passed in.\n return rec.spec\n\n @_autospec\n def remove(self, spec: \"spack.spec.Spec\") -> \"spack.spec.Spec\":\n \"\"\"Removes a spec from the database. To be called on uninstall.\n\n Reads the database, then:\n\n 1. Marks the spec as not installed.\n 2. Removes the spec if it has no more dependents.\n 3. If removed, recursively updates dependencies' ref counts\n and removes them if they are no longer needed.\n\n \"\"\"\n # Take a lock around the entire removal.\n with self.write_transaction():\n return self._remove(spec)\n\n def deprecator(self, spec: \"spack.spec.Spec\") -> Optional[\"spack.spec.Spec\"]:\n \"\"\"Return the spec that the given spec is deprecated for, or None\"\"\"\n with self.read_transaction():\n spec_key = self._get_matching_spec_key(spec)\n spec_rec = self._data[spec_key]\n\n if spec_rec.deprecated_for:\n return self._data[spec_rec.deprecated_for].spec\n else:\n return None\n\n def specs_deprecated_by(self, spec: \"spack.spec.Spec\") -> List[\"spack.spec.Spec\"]:\n \"\"\"Return all specs deprecated in favor of the given spec\"\"\"\n with self.read_transaction():\n return [\n rec.spec for rec in self._data.values() if rec.deprecated_for == spec.dag_hash()\n ]\n\n def _deprecate(self, spec: \"spack.spec.Spec\", deprecator: \"spack.spec.Spec\") -> None:\n spec_key = self._get_matching_spec_key(spec)\n spec_rec = self._data[spec_key]\n\n deprecator_key = self._get_matching_spec_key(deprecator)\n\n self._increment_ref_count(deprecator)\n\n # If spec was already deprecated, update old deprecator's ref count\n if spec_rec.deprecated_for:\n old_repl_rec = self._data[spec_rec.deprecated_for]\n self._decrement_ref_count(old_repl_rec.spec)\n\n spec_rec.deprecated_for = deprecator_key\n spec_rec.installed = False\n self._data[spec_key] = spec_rec\n\n @_autospec\n def mark(self, spec: \"spack.spec.Spec\", key: str, value: Any) -> None:\n \"\"\"Mark an arbitrary record on a spec.\"\"\"\n with self.write_transaction():\n return self._mark(spec, key, value)\n\n def _mark(self, spec: \"spack.spec.Spec\", key, value) -> None:\n record = self._data[self._get_matching_spec_key(spec)]\n setattr(record, key, value)\n\n @_autospec\n def deprecate(self, spec: \"spack.spec.Spec\", deprecator: \"spack.spec.Spec\") -> None:\n \"\"\"Marks a spec as deprecated in favor of its deprecator\"\"\"\n with self.write_transaction():\n return self._deprecate(spec, deprecator)\n\n @_autospec\n def installed_relatives(\n self,\n spec: \"spack.spec.Spec\",\n direction: tr.DirectionType = \"children\",\n transitive: bool = True,\n deptype: Union[dt.DepFlag, dt.DepTypes] = dt.ALL,\n ) -> Set[\"spack.spec.Spec\"]:\n \"\"\"Return installed specs related to this one.\"\"\"\n if direction not in (\"parents\", \"children\"):\n raise ValueError(\"Invalid direction: %s\" % direction)\n\n relatives: Set[spack.spec.Spec] = set()\n for spec in self.query(spec):\n if transitive:\n to_add = spec.traverse(direction=direction, root=False, deptype=deptype)\n elif direction == \"parents\":\n to_add = spec.dependents(deptype=deptype)\n else: # direction == 'children'\n to_add = spec.dependencies(deptype=deptype)\n\n for relative in to_add:\n hash_key = relative.dag_hash()\n _, record = self.query_by_spec_hash(hash_key)\n if not record:\n tty.warn(\n f\"Inconsistent state: \"\n f\"{'dependent' if direction == 'parents' else 'dependency'} {hash_key} of \"\n f\"{spec.dag_hash()} not in DB\"\n )\n continue\n\n if not record.installed:\n continue\n\n relatives.add(relative)\n return relatives\n\n @_autospec\n def installed_extensions_for(self, extendee_spec: \"spack.spec.Spec\"):\n \"\"\"Returns the specs of all packages that extend the given spec\"\"\"\n for spec in self.query():\n if spec.package.extends(extendee_spec):\n yield spec.package\n\n def _get_by_hash_local(\n self,\n dag_hash: str,\n default: Optional[List[\"spack.spec.Spec\"]] = None,\n installed: Union[bool, InstallRecordStatus] = InstallRecordStatus.ANY,\n ) -> Optional[List[\"spack.spec.Spec\"]]:\n installed = normalize_query(installed)\n # hash is a full hash and is in the data somewhere\n if dag_hash in self._data:\n rec = self._data[dag_hash]\n if rec.install_type_matches(installed):\n return [rec.spec]\n else:\n return default\n\n # check if hash is a prefix of some installed (or previously installed) spec.\n matches = [\n record.spec\n for h, record in self._data.items()\n if h.startswith(dag_hash) and record.install_type_matches(installed)\n ]\n if matches:\n return matches\n\n # nothing found\n return default\n\n def get_by_hash_local(\n self,\n dag_hash: str,\n default: Optional[List[\"spack.spec.Spec\"]] = None,\n installed: Union[bool, InstallRecordStatus] = InstallRecordStatus.ANY,\n ) -> Optional[List[\"spack.spec.Spec\"]]:\n \"\"\"Look up a spec in *this DB* by DAG hash, or by a DAG hash prefix.\n\n Args:\n dag_hash: hash (or hash prefix) to look up\n default: default value to return if dag_hash is not in the DB\n installed: if ``True``, includes only installed specs in the search; if ``False``\n only missing specs. Otherwise, a InstallRecordStatus flag.\n\n ``installed`` defaults to ``InstallRecordStatus.ANY`` so we can refer to any known hash.\n\n ``query()`` and ``query_one()`` differ in that they only return installed specs by default.\n \"\"\"\n with self.read_transaction():\n return self._get_by_hash_local(dag_hash, default=default, installed=installed)\n\n def get_by_hash(\n self,\n dag_hash: str,\n default: Optional[List[\"spack.spec.Spec\"]] = None,\n installed: Union[bool, InstallRecordStatus] = InstallRecordStatus.ANY,\n ) -> Optional[List[\"spack.spec.Spec\"]]:\n \"\"\"Look up a spec by DAG hash, or by a DAG hash prefix.\n\n Args:\n dag_hash: hash (or hash prefix) to look up\n default: default value to return if dag_hash is not in the DB\n installed: if ``True``, includes only installed specs in the search; if ``False``\n only missing specs. Otherwise, a InstallRecordStatus flag.\n\n ``installed`` defaults to ``InstallRecordStatus.ANY`` so we can refer to any known hash.\n ``query()`` and ``query_one()`` differ in that they only return installed specs by default.\n\n \"\"\"\n\n spec = self.get_by_hash_local(dag_hash, default=default, installed=installed)\n if spec is not None:\n return spec\n\n for upstream_db in self.upstream_dbs:\n spec = upstream_db._get_by_hash_local(dag_hash, default=default, installed=installed)\n if spec is not None:\n return spec\n\n return default\n\n def _query(\n self,\n query_spec: Optional[Union[str, \"spack.spec.Spec\"]] = None,\n *,\n predicate_fn: Optional[SelectType] = None,\n installed: Union[bool, InstallRecordStatus] = True,\n explicit: Optional[bool] = None,\n start_date: Optional[datetime.datetime] = None,\n end_date: Optional[datetime.datetime] = None,\n hashes: Optional[Iterable[str]] = None,\n in_buildcache: Optional[bool] = None,\n origin: Optional[str] = None,\n ) -> List[\"spack.spec.Spec\"]:\n installed = normalize_query(installed)\n\n # Restrict the set of records over which we iterate first\n matching_hashes = self._data\n if hashes is not None:\n matching_hashes = {h: self._data[h] for h in hashes if h in self._data}\n\n if isinstance(query_spec, str):\n query_spec = spack.spec.Spec(query_spec)\n\n if query_spec is not None and query_spec.concrete:\n hash_key = query_spec.dag_hash()\n if hash_key not in matching_hashes:\n return []\n matching_hashes = {hash_key: matching_hashes[hash_key]}\n\n results = []\n start_date = start_date or datetime.datetime.min\n end_date = end_date or datetime.datetime.max\n\n deferred = []\n for rec in matching_hashes.values():\n if origin and not (origin == rec.origin):\n continue\n\n if not rec.install_type_matches(installed):\n continue\n\n if in_buildcache is not None and rec.in_buildcache != in_buildcache:\n continue\n\n if explicit is not None and rec.explicit != explicit:\n continue\n\n if predicate_fn is not None and not predicate_fn(rec):\n continue\n\n if start_date or end_date:\n inst_date = datetime.datetime.fromtimestamp(rec.installation_time)\n if not (start_date < inst_date < end_date):\n continue\n\n if query_spec is None or query_spec.concrete:\n results.append(rec.spec)\n continue\n\n # check anon specs and exact name matches first\n if not query_spec.name or rec.spec.name == query_spec.name:\n if rec.spec.satisfies(query_spec):\n results.append(rec.spec)\n\n # save potential virtual matches for later, but not if we already found a match\n elif not results:\n deferred.append(rec.spec)\n\n # Checking for virtuals is expensive, so we save it for last and only if needed.\n # If we get here, we didn't find anything in the DB that matched by name.\n # If we did fine something, the query spec can't be virtual b/c we matched an actual\n # package installation, so skip the virtual check entirely. If we *didn't* find anything,\n # check all the deferred specs *if* the query is virtual.\n if (\n not results\n and query_spec is not None\n and deferred\n and spack.repo.PATH.is_virtual(query_spec.name)\n ):\n results = [spec for spec in deferred if spec.satisfies(query_spec)]\n\n return results\n\n def query_local(\n self,\n query_spec: Optional[Union[str, \"spack.spec.Spec\"]] = None,\n *,\n predicate_fn: Optional[SelectType] = None,\n installed: Union[bool, InstallRecordStatus] = True,\n explicit: Optional[bool] = None,\n start_date: Optional[datetime.datetime] = None,\n end_date: Optional[datetime.datetime] = None,\n hashes: Optional[List[str]] = None,\n in_buildcache: Optional[bool] = None,\n origin: Optional[str] = None,\n ) -> List[\"spack.spec.Spec\"]:\n \"\"\"Queries the local Spack database.\n\n This function doesn't guarantee any sorting of the returned data for performance reason,\n since comparing specs for __lt__ may be an expensive operation.\n\n Args:\n query_spec: if query_spec is ``None``, match all specs in the database.\n If it is a spec, return all specs matching ``spec.satisfies(query_spec)``.\n\n predicate_fn: optional predicate taking an InstallRecord as argument, and returning\n whether that record is selected for the query. It can be used to craft criteria\n that need some data for selection not provided by the Database itself.\n\n installed: if ``True``, includes only installed specs in the search. If ``False`` only\n missing specs, and if ``any``, all specs in database. If an InstallStatus or\n iterable of InstallStatus, returns specs whose install status matches at least\n one of the InstallStatus.\n\n explicit: a spec that was installed following a specific user request is marked as\n explicit. If instead it was pulled-in as a dependency of a user requested spec\n it's considered implicit.\n\n start_date: if set considers only specs installed from the starting date.\n\n end_date: if set considers only specs installed until the ending date.\n\n in_buildcache: specs that are marked in this database as part of an associated binary\n cache are ``in_buildcache``. All other specs are not. This field is used for\n querying mirror indices. By default, it does not check this status.\n\n hashes: list of hashes used to restrict the search\n\n origin: origin of the spec\n \"\"\"\n with self.read_transaction():\n return self._query(\n query_spec,\n predicate_fn=predicate_fn,\n installed=installed,\n explicit=explicit,\n start_date=start_date,\n end_date=end_date,\n hashes=hashes,\n in_buildcache=in_buildcache,\n origin=origin,\n )\n\n def query(\n self,\n query_spec: Optional[Union[str, \"spack.spec.Spec\"]] = None,\n *,\n predicate_fn: Optional[SelectType] = None,\n installed: Union[bool, InstallRecordStatus] = True,\n explicit: Optional[bool] = None,\n start_date: Optional[datetime.datetime] = None,\n end_date: Optional[datetime.datetime] = None,\n in_buildcache: Optional[bool] = None,\n hashes: Optional[List[str]] = None,\n origin: Optional[str] = None,\n install_tree: str = \"all\",\n ) -> List[\"spack.spec.Spec\"]:\n \"\"\"Queries the Spack database including all upstream databases.\n\n Args:\n query_spec: if query_spec is ``None``, match all specs in the database.\n If it is a spec, return all specs matching ``spec.satisfies(query_spec)``.\n\n predicate_fn: optional predicate taking an InstallRecord as argument, and returning\n whether that record is selected for the query. It can be used to craft criteria\n that need some data for selection not provided by the Database itself.\n\n installed: if ``True``, includes only installed specs in the search. If ``False`` only\n missing specs, and if ``any``, all specs in database. If an InstallStatus or\n iterable of InstallStatus, returns specs whose install status matches at least\n one of the InstallStatus.\n\n explicit: a spec that was installed following a specific user request is marked as\n explicit. If instead it was pulled-in as a dependency of a user requested spec\n it's considered implicit.\n\n start_date: if set considers only specs installed from the starting date.\n\n end_date: if set considers only specs installed until the ending date.\n\n in_buildcache: specs that are marked in this database as part of an associated binary\n cache are ``in_buildcache``. All other specs are not. This field is used for\n querying mirror indices. By default, it does not check this status.\n\n hashes: list of hashes used to restrict the search\n\n install_tree: query ``\"all\"`` (default), ``\"local\"``, ``\"upstream\"``, or upstream path\n\n origin: origin of the spec\n \"\"\"\n valid_trees = [\"all\", \"upstream\", \"local\", self.root] + [u.root for u in self.upstream_dbs]\n if install_tree not in valid_trees:\n msg = \"Invalid install_tree argument to Database.query()\\n\"\n msg += f\"Try one of {', '.join(valid_trees)}\"\n tty.error(msg)\n return []\n\n upstream_results = []\n upstreams = self.upstream_dbs\n if install_tree not in (\"all\", \"upstream\"):\n upstreams = [u for u in self.upstream_dbs if u.root == install_tree]\n for upstream_db in upstreams:\n # queries for upstream DBs need to *not* lock - we may not\n # have permissions to do this and the upstream DBs won't know about\n # us anyway (so e.g. they should never uninstall specs)\n upstream_results.extend(\n upstream_db._query(\n query_spec,\n predicate_fn=predicate_fn,\n installed=installed,\n explicit=explicit,\n start_date=start_date,\n end_date=end_date,\n hashes=hashes,\n in_buildcache=in_buildcache,\n origin=origin,\n )\n or []\n )\n\n local_results: Set[\"spack.spec.Spec\"] = set()\n if install_tree in (\"all\", \"local\") or self.root == install_tree:\n local_results = set(\n self.query_local(\n query_spec,\n predicate_fn=predicate_fn,\n installed=installed,\n explicit=explicit,\n start_date=start_date,\n end_date=end_date,\n hashes=hashes,\n in_buildcache=in_buildcache,\n origin=origin,\n )\n )\n\n results = list(local_results) + list(x for x in upstream_results if x not in local_results)\n results.sort() # type: ignore[call-arg,call-overload]\n return results\n\n def query_one(\n self,\n query_spec: Optional[Union[str, \"spack.spec.Spec\"]],\n predicate_fn: Optional[SelectType] = None,\n installed: Union[bool, InstallRecordStatus] = True,\n ) -> Optional[\"spack.spec.Spec\"]:\n \"\"\"Query for exactly one spec that matches the query spec.\n\n Returns None if no installed package matches.\n\n Raises:\n AssertionError: if more than one spec matches the query.\n \"\"\"\n concrete_specs = self.query(query_spec, predicate_fn=predicate_fn, installed=installed)\n assert len(concrete_specs) <= 1\n return concrete_specs[0] if concrete_specs else None\n\n def missing(self, spec):\n key = spec.dag_hash()\n _, record = self.query_by_spec_hash(key)\n return record and not record.installed\n\n def is_occupied_install_prefix(self, path):\n with self.read_transaction():\n return path in self._installed_prefixes\n\n def all_hashes(self):\n \"\"\"Return dag hash of every spec in the database.\"\"\"\n with self.read_transaction():\n return list(self._data.keys())\n\n def unused_specs(\n self,\n root_hashes: Optional[Container[str]] = None,\n deptype: Union[dt.DepFlag, dt.DepTypes] = dt.LINK | dt.RUN,\n ) -> List[\"spack.spec.Spec\"]:\n \"\"\"Return all specs that are currently installed but not needed by root specs.\n\n By default, roots are all explicit specs in the database. If a set of root\n hashes are passed in, they are instead used as the roots.\n\n Arguments:\n root_hashes: optional list of roots to consider when evaluating needed installations.\n deptype: if a spec is reachable from a root via these dependency types, it is\n considered needed. By default only link and run dependency types are considered.\n \"\"\"\n\n def root(key, record):\n \"\"\"Whether a DB record is a root for garbage collection.\"\"\"\n return key in root_hashes if root_hashes is not None else record.explicit\n\n with self.read_transaction():\n roots = [rec.spec for key, rec in self._data.items() if root(key, rec)]\n needed = set(id(spec) for spec in tr.traverse_nodes(roots, deptype=deptype))\n return [\n rec.spec\n for rec in self._data.values()\n if id(rec.spec) not in needed and rec.installed\n ]\n\n\nclass NoUpstreamVisitor:\n \"\"\"Gives edges to upstream specs, but does follow edges from upstream specs.\"\"\"\n\n def __init__(\n self,\n upstream_hashes: Set[str],\n on_visit: Callable[[\"spack.spec.DependencySpec\", bool], None],\n ):\n self.upstream_hashes = upstream_hashes\n self.on_visit = on_visit\n\n def accept(self, item: tr.EdgeAndDepth) -> bool:\n self.on_visit(item.edge, self.is_upstream(item))\n return True\n\n def is_upstream(self, item: tr.EdgeAndDepth) -> bool:\n return item.edge.spec.dag_hash() in self.upstream_hashes\n\n def neighbors(self, item: tr.EdgeAndDepth):\n # Prune edges from upstream nodes, only follow database tracked dependencies\n return (\n []\n if self.is_upstream(item)\n else item.edge.spec.edges_to_dependencies(depflag=_TRACKED_DEPENDENCIES)\n )\n\n\nclass UpstreamDatabaseLockingError(SpackError):\n \"\"\"Raised when an operation would need to lock an upstream database\"\"\"\n\n\nclass CorruptDatabaseError(SpackError):\n \"\"\"Raised when errors are found while reading the database.\"\"\"\n\n\nclass NonConcreteSpecAddError(SpackError):\n \"\"\"Raised when attempting to add non-concrete spec to DB.\"\"\"\n\n\nclass MissingDependenciesError(SpackError):\n \"\"\"Raised when DB cannot find records for dependencies\"\"\"\n\n\nclass InvalidDatabaseVersionError(SpackError):\n \"\"\"Exception raised when the database metadata is newer than current Spack.\"\"\"\n\n def __init__(self, database, expected, found):\n self.expected = expected\n self.found = found\n msg = (\n f\"you need a newer Spack version to read the DB in '{database.root}'. \"\n f\"{self.database_version_message}\"\n )\n super().__init__(msg)\n\n @property\n def database_version_message(self):\n return f\"The expected DB version is '{self.expected}', but '{self.found}' was found.\"\n\n\nclass ExplicitDatabaseUpgradeError(SpackError):\n \"\"\"Raised to request an explicit DB upgrade to the user\"\"\"\n\n\nclass DatabaseNotReadableError(SpackError):\n \"\"\"Raised to signal Database.reindex that the reindex should happen via spec.json\"\"\"\n\n\nclass NoSuchSpecError(KeyError):\n \"\"\"Raised when a spec is not found in the database.\"\"\"\n\n def __init__(self, spec):\n self.spec = spec\n super().__init__(spec)\n\n def __str__(self):\n # This exception is raised frequently, and almost always\n # caught, so ensure we don't pay the cost of Spec.__str__\n # unless the exception is actually printed.\n return f\"No such spec in database: {self.spec}\"\n" }, { "path": "lib/spack/spack/dependency.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Data structures that represent Spack's dependency relationships.\"\"\"\n\nfrom typing import TYPE_CHECKING, Dict, List, Type\n\nimport spack.deptypes as dt\nimport spack.spec\n\nif TYPE_CHECKING:\n import spack.package_base\n import spack.patch\n\n\nclass Dependency:\n \"\"\"Class representing metadata for a dependency on a package.\n\n This class differs from ``spack.spec.DependencySpec`` because it\n represents metadata at the ``Package`` level.\n ``spack.spec.DependencySpec`` is a descriptor for an actual package\n configuration, while ``Dependency`` is a descriptor for a package's\n dependency *requirements*.\n\n A dependency is a requirement for a configuration of another package\n that satisfies a particular spec. The dependency can have *types*,\n which determine *how* that package configuration is required,\n e.g. whether it is required for building the package, whether it\n needs to be linked to, or whether it is needed at runtime so that\n Spack can call commands from it.\n\n A package can also depend on another package with *patches*. This is\n for cases where the maintainers of one package also maintain special\n patches for their dependencies. If one package depends on another\n with patches, a special version of that dependency with patches\n applied will be built for use by the dependent package. The patches\n are included in the new version's spec hash to differentiate it from\n unpatched versions of the same package, so that unpatched versions of\n the dependency package can coexist with the patched version.\n\n \"\"\"\n\n __slots__ = \"pkg\", \"spec\", \"patches\", \"depflag\"\n\n def __init__(\n self,\n pkg: Type[\"spack.package_base.PackageBase\"],\n spec: spack.spec.Spec,\n depflag: dt.DepFlag = dt.DEFAULT,\n ):\n \"\"\"Create a new Dependency.\n\n Args:\n pkg: Package that has this dependency\n spec: Spec indicating dependency requirements\n type: strings describing dependency relationship\n \"\"\"\n self.pkg = pkg\n self.spec = spec\n\n # This dict maps condition specs to lists of Patch objects, just\n # as the patches dict on packages does.\n self.patches: Dict[spack.spec.Spec, List[\"spack.patch.Patch\"]] = {}\n self.depflag = depflag\n\n @property\n def name(self) -> str:\n \"\"\"Get the name of the dependency package.\"\"\"\n return self.spec.name\n\n def __repr__(self) -> str:\n types = dt.flag_to_chars(self.depflag)\n if self.patches:\n return f\" {self.spec} [{types}, {self.patches}]>\"\n else:\n return f\" {self.spec} [{types}]>\"\n" }, { "path": "lib/spack/spack/deptypes.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Data structures that represent Spack's edge types.\"\"\"\n\nfrom typing import Iterable, List, Tuple, Union\n\nfrom spack.vendor.typing_extensions import Literal\n\n#: Type hint for the low-level dependency input (enum.Flag is too slow)\nDepFlag = int\n\n#: Type hint for the high-level dependency input\nDepTypes = Union[str, List[str], Tuple[str, ...]]\n\n#: Individual dependency types\nDepType = Literal[\"build\", \"link\", \"run\", \"test\"]\n\n# Flag values. NOTE: these values are not arbitrary, since hash computation imposes\n# the order (link, run, build, test) when depending on the same package multiple times,\n# and we rely on default integer comparison to sort dependency types.\n# New dependency types should be appended.\nLINK = 0b0001\nRUN = 0b0010\nBUILD = 0b0100\nTEST = 0b1000\n\n#: The types of dependency relationships that Spack understands.\nALL_TYPES: Tuple[DepType, ...] = (\"build\", \"link\", \"run\", \"test\")\n\n#: Default dependency type if none is specified\nDEFAULT_TYPES: Tuple[DepType, ...] = (\"build\", \"link\")\n\n#: A flag with all dependency types set\nALL: DepFlag = BUILD | LINK | RUN | TEST\n\n#: Default dependency type if none is specified\nDEFAULT: DepFlag = BUILD | LINK\n\n#: A flag with no dependency types set\nNONE: DepFlag = 0\n\n#: An iterator of all flag components\nALL_FLAGS: Tuple[DepFlag, DepFlag, DepFlag, DepFlag] = (BUILD, LINK, RUN, TEST)\n\n\ndef compatible(flag1: DepFlag, flag2: DepFlag) -> bool:\n \"\"\"Returns True if two depflags can be dependencies from a Spec to deps of the same name.\n\n The only allowable separated dependencies are a build-only dependency, combined with a\n non-build dependency. This separates our two process spaces, build time and run time.\n\n These dependency combinations are allowed:\n\n * single dep on name: ``[b]``, ``[l]``, ``[r]``, ``[bl]``, ``[br]``, ``[blr]``\n * two deps on name: ``[b, l]``, ``[b, r]``, ``[b, lr]``\n\n but none of these make any sense:\n\n * two build deps: ``[b, b]``, ``[b, br]``, ``[b, bl]``, ``[b, blr]``\n * any two deps that both have an ``l`` or an ``r``, i.e. ``[l, l]``, ``[r, r]``, ``[l, r]``,\n ``[bl, l]``, ``[bl, r]``\"\"\"\n # Cannot have overlapping build types to two different dependencies\n if flag1 & flag2:\n return False\n\n # Cannot have two different link/run dependencies for the same name\n link_run = LINK | RUN\n if flag1 & link_run and flag2 & link_run:\n return False\n\n return True\n\n\ndef flag_from_string(s: str) -> DepFlag:\n if s == \"build\":\n return BUILD\n elif s == \"link\":\n return LINK\n elif s == \"run\":\n return RUN\n elif s == \"test\":\n return TEST\n else:\n raise ValueError(f\"Invalid dependency type: {s}\")\n\n\ndef flag_from_strings(deptype: Iterable[str]) -> DepFlag:\n \"\"\"Transform an iterable of deptype strings into a flag.\"\"\"\n flag = 0\n for deptype_str in deptype:\n flag |= flag_from_string(deptype_str)\n return flag\n\n\ndef canonicalize(deptype: DepTypes) -> DepFlag:\n \"\"\"Convert deptype user input to a DepFlag, or raise ValueError.\n\n Args:\n deptype: string representing dependency type, or a list/tuple of such strings.\n Can also be the builtin function ``all`` or the string 'all', which result in\n a tuple of all dependency types known to Spack.\n \"\"\"\n if deptype in (\"all\", all):\n return ALL\n\n if isinstance(deptype, str):\n return flag_from_string(deptype)\n\n if isinstance(deptype, (tuple, list, set)):\n return flag_from_strings(deptype)\n\n raise ValueError(f\"Invalid dependency type: {deptype!r}\")\n\n\ndef flag_to_tuple(x: DepFlag) -> Tuple[DepType, ...]:\n deptype: List[DepType] = []\n if x & BUILD:\n deptype.append(\"build\")\n if x & LINK:\n deptype.append(\"link\")\n if x & RUN:\n deptype.append(\"run\")\n if x & TEST:\n deptype.append(\"test\")\n return tuple(deptype)\n\n\ndef flag_to_string(x: DepFlag) -> DepType:\n if x == BUILD:\n return \"build\"\n elif x == LINK:\n return \"link\"\n elif x == RUN:\n return \"run\"\n elif x == TEST:\n return \"test\"\n else:\n raise ValueError(f\"Invalid dependency type flag: {x}\")\n\n\ndef flag_to_chars(depflag: DepFlag) -> str:\n \"\"\"Create a string representing deptypes for many dependencies.\n\n The string will be some subset of ``blrt``, like ``bl ``, ``b t``, or\n `` lr `` where each letter in ``blrt`` stands for ``build``, ``link``,\n ``run``, and ``test`` (the dependency types).\n\n For a single dependency, this just indicates that the dependency has\n the indicated deptypes. For a list of dependnecies, this shows\n whether ANY dependency in the list has the deptypes (so the deptypes\n are merged).\"\"\"\n return \"\".join(\n t_str[0] if t_flag & depflag else \" \" for t_str, t_flag in zip(ALL_TYPES, ALL_FLAGS)\n )\n" }, { "path": "lib/spack/spack/detection/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nfrom .common import executable_prefix, set_virtuals_nonbuildable, update_configuration\nfrom .path import by_path, executables_in_path\nfrom .test import detection_tests\n\n__all__ = [\n \"by_path\",\n \"executables_in_path\",\n \"executable_prefix\",\n \"update_configuration\",\n \"set_virtuals_nonbuildable\",\n \"detection_tests\",\n]\n" }, { "path": "lib/spack/spack/detection/common.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Define a common data structure to represent external packages and a\nfunction to update packages.yaml given a list of detected packages.\n\nIdeally, each detection method should be placed in a specific subpackage\nand implement at least a function that returns a list of specs.\n\nThe update in packages.yaml can then be done using the function provided here.\n\nThe module also contains other functions that might be useful across different\ndetection mechanisms.\n\"\"\"\n\nimport glob\nimport itertools\nimport os\nimport pathlib\nimport re\nimport sys\nfrom typing import Dict, List, Optional, Set, Tuple, Union\n\nimport spack.config\nimport spack.error\nimport spack.operating_systems.windows_os as winOs\nimport spack.schema\nimport spack.spec\nimport spack.util.environment\nimport spack.util.spack_yaml\nimport spack.util.windows_registry\nfrom spack.llnl.util import tty\n\n\ndef _externals_in_packages_yaml() -> Set[spack.spec.Spec]:\n \"\"\"Returns all the specs mentioned as externals in packages.yaml\"\"\"\n packages_yaml = spack.config.get(\"packages\")\n already_defined_specs = set()\n for pkg_name, package_configuration in packages_yaml.items():\n for item in package_configuration.get(\"externals\", []):\n already_defined_specs.add(spack.spec.Spec(item[\"spec\"]))\n return already_defined_specs\n\n\nExternalEntryType = Union[str, Dict[str, str]]\n\n\ndef _pkg_config_dict(\n external_pkg_entries: List[\"spack.spec.Spec\"],\n) -> Dict[str, Union[bool, List[Dict[str, ExternalEntryType]]]]:\n \"\"\"Generate a package specific config dict according to the packages.yaml schema.\n\n This does not generate the entire packages.yaml. For example, given some\n external entries for the CMake package, this could return::\n\n {\n 'externals': [{\n 'spec': 'cmake@3.17.1',\n 'prefix': '/opt/cmake-3.17.1/'\n }, {\n 'spec': 'cmake@3.16.5',\n 'prefix': '/opt/cmake-3.16.5/'\n }]\n }\n \"\"\"\n pkg_dict = spack.util.spack_yaml.syaml_dict()\n pkg_dict[\"externals\"] = []\n for e in external_pkg_entries:\n if not _spec_is_valid(e):\n continue\n\n external_items: List[Tuple[str, ExternalEntryType]] = [\n (\"spec\", str(e)),\n (\"prefix\", pathlib.Path(e.external_path).as_posix()),\n ]\n if e.external_modules:\n external_items.append((\"modules\", e.external_modules))\n\n if e.extra_attributes:\n external_items.append(\n (\"extra_attributes\", spack.util.spack_yaml.syaml_dict(e.extra_attributes.items()))\n )\n\n # external_items.extend(e.spec.extra_attributes.items())\n pkg_dict[\"externals\"].append(spack.util.spack_yaml.syaml_dict(external_items))\n\n return pkg_dict\n\n\ndef _spec_is_valid(spec: spack.spec.Spec) -> bool:\n try:\n str(spec)\n except spack.error.SpackError:\n # It is assumed here that we can at least extract the package name from the spec so we\n # can look up the implementation of determine_spec_details\n tty.warn(f\"Constructed spec for {spec.name} does not have a string representation\")\n return False\n\n try:\n spack.spec.Spec(str(spec))\n except spack.error.SpackError:\n tty.warn(\n \"Constructed spec has a string representation but the string\"\n \" representation does not evaluate to a valid spec: {0}\".format(str(spec))\n )\n return False\n\n return True\n\n\ndef path_to_dict(search_paths: List[str]) -> Dict[str, str]:\n \"\"\"Return dictionary[fullpath]: basename from list of paths\"\"\"\n path_to_lib: Dict[str, str] = {}\n # Reverse order of search directories so that a lib in the first\n # entry overrides later entries\n for search_path in reversed(search_paths):\n try:\n dir_iter = os.scandir(search_path)\n except OSError as e:\n tty.debug(f\"cannot scan '{search_path}' for external software: {e}\")\n continue\n with dir_iter as entries:\n for entry in entries:\n try:\n if entry.is_file():\n path_to_lib[entry.path] = entry.name\n except OSError as e:\n tty.debug(f\"cannot scan '{search_path}' for external software: {e}\")\n\n return path_to_lib\n\n\ndef is_executable(file_path: str) -> bool:\n \"\"\"Return True if the path passed as argument is that of an executable\"\"\"\n return os.path.isfile(file_path) and os.access(file_path, os.X_OK)\n\n\ndef _convert_to_iterable(single_val_or_multiple):\n x = single_val_or_multiple\n if x is None:\n return []\n elif isinstance(x, str):\n return [x]\n elif isinstance(x, spack.spec.Spec):\n # Specs are iterable, but a single spec should be converted to a list\n return [x]\n\n try:\n iter(x)\n return x\n except TypeError:\n return [x]\n\n\ndef executable_prefix(executable_dir: str) -> str:\n \"\"\"Given a directory where an executable is found, guess the prefix\n (i.e. the \"root\" directory of that installation) and return it.\n\n Args:\n executable_dir: directory where an executable is found\n \"\"\"\n # Given a prefix where an executable is found, assuming that prefix\n # contains /bin/, strip off the 'bin' directory to get a Spack-compatible\n # prefix\n assert os.path.isdir(executable_dir)\n\n components = executable_dir.split(os.sep)\n # convert to lower to match Bin, BIN, bin\n lowered_components = executable_dir.lower().split(os.sep)\n if \"bin\" not in lowered_components:\n return executable_dir\n idx = lowered_components.index(\"bin\")\n return os.sep.join(components[:idx])\n\n\ndef library_prefix(library_dir: str) -> str:\n \"\"\"Given a directory where a library is found, guess the prefix\n (i.e. the \"root\" directory of that installation) and return it.\n\n Args:\n library_dir: directory where a library is found\n \"\"\"\n # Given a prefix where an library is found, assuming that prefix\n # contains /lib/ or /lib64/, strip off the 'lib' or 'lib64' directory\n # to get a Spack-compatible prefix\n assert os.path.isdir(library_dir)\n\n components = library_dir.split(os.sep)\n # convert to lowercase to match lib, LIB, Lib, etc.\n lowered_components = library_dir.lower().split(os.sep)\n if \"lib64\" in lowered_components:\n idx = lowered_components.index(\"lib64\")\n return os.sep.join(components[:idx])\n elif \"lib\" in lowered_components:\n idx = lowered_components.index(\"lib\")\n return os.sep.join(components[:idx])\n elif sys.platform == \"win32\" and \"bin\" in lowered_components:\n idx = lowered_components.index(\"bin\")\n return os.sep.join(components[:idx])\n else:\n return library_dir\n\n\ndef update_configuration(\n detected_packages: Dict[str, List[\"spack.spec.Spec\"]],\n scope: Optional[str] = None,\n buildable: bool = True,\n) -> List[spack.spec.Spec]:\n \"\"\"Add the packages passed as arguments to packages.yaml\n\n Args:\n detected_packages: list of specs to be added\n scope: configuration scope where to add the detected packages\n buildable: whether the detected packages are buildable or not\n \"\"\"\n predefined_external_specs = _externals_in_packages_yaml()\n pkg_to_cfg, all_new_specs = {}, []\n for package_name, entries in detected_packages.items():\n new_entries = [s for s in entries if s not in predefined_external_specs]\n\n pkg_config = _pkg_config_dict(new_entries)\n external_entries = pkg_config.get(\"externals\", [])\n assert not isinstance(external_entries, bool), \"unexpected value for external entry\"\n\n all_new_specs.extend(new_entries)\n if buildable is False:\n pkg_config[\"buildable\"] = False\n pkg_to_cfg[package_name] = pkg_config\n\n scope = scope or spack.config.default_modify_scope()\n pkgs_cfg = spack.config.get(\"packages\", scope=scope)\n pkgs_cfg = spack.schema.merge_yaml(pkgs_cfg, pkg_to_cfg)\n spack.config.set(\"packages\", pkgs_cfg, scope=scope)\n\n return all_new_specs\n\n\ndef set_virtuals_nonbuildable(virtuals: Set[str], scope: Optional[str] = None) -> List[str]:\n \"\"\"Update packages:virtual:buildable:False for the provided virtual packages, if the property\n is not set by the user. Returns the list of virtual packages that have been updated.\"\"\"\n packages = spack.config.get(\"packages\")\n new_config = {}\n for virtual in virtuals:\n # If the user has set the buildable prop do not override it\n if virtual in packages and \"buildable\" in packages[virtual]:\n continue\n new_config[virtual] = {\"buildable\": False}\n\n # Update the provided scope\n spack.config.set(\n \"packages\",\n spack.schema.merge_yaml(spack.config.get(\"packages\", scope=scope), new_config),\n scope=scope,\n )\n\n return list(new_config.keys())\n\n\ndef _windows_drive() -> str:\n \"\"\"Return Windows drive string extracted from the PROGRAMFILES environment variable,\n which is guaranteed to be defined for all logins.\n \"\"\"\n match = re.match(r\"([a-zA-Z]:)\", os.environ[\"PROGRAMFILES\"])\n if match is None:\n raise RuntimeError(\"cannot read the PROGRAMFILES environment variable\")\n return match.group(1)\n\n\nclass WindowsCompilerExternalPaths:\n @staticmethod\n def find_windows_compiler_root_paths() -> List[str]:\n \"\"\"Helper for Windows compiler installation root discovery\n\n At the moment simply returns location of VS install paths from VSWhere\n But should be extended to include more information as relevant\"\"\"\n return list(winOs.WindowsOs().vs_install_paths)\n\n @staticmethod\n def find_windows_compiler_cmake_paths() -> List[str]:\n \"\"\"Semi hard-coded search path for cmake bundled with MSVC\"\"\"\n return [\n os.path.join(\n path, \"Common7\", \"IDE\", \"CommonExtensions\", \"Microsoft\", \"CMake\", \"CMake\", \"bin\"\n )\n for path in WindowsCompilerExternalPaths.find_windows_compiler_root_paths()\n ]\n\n @staticmethod\n def find_windows_compiler_ninja_paths() -> List[str]:\n \"\"\"Semi hard-coded search heuristic for locating ninja bundled with MSVC\"\"\"\n return [\n os.path.join(path, \"Common7\", \"IDE\", \"CommonExtensions\", \"Microsoft\", \"CMake\", \"Ninja\")\n for path in WindowsCompilerExternalPaths.find_windows_compiler_root_paths()\n ]\n\n @staticmethod\n def find_windows_compiler_bundled_packages() -> List[str]:\n \"\"\"Return all MSVC compiler bundled packages\"\"\"\n return (\n WindowsCompilerExternalPaths.find_windows_compiler_cmake_paths()\n + WindowsCompilerExternalPaths.find_windows_compiler_ninja_paths()\n )\n\n\nclass WindowsKitExternalPaths:\n @staticmethod\n def find_windows_kit_roots() -> List[str]:\n \"\"\"Return Windows kit root, typically %programfiles%\\\\Windows Kits\\\\10|11\\\\\"\"\"\n if sys.platform != \"win32\":\n return []\n program_files = os.environ[\"PROGRAMFILES(x86)\"]\n kit_base = os.path.join(program_files, \"Windows Kits\", \"**\")\n return glob.glob(kit_base)\n\n @staticmethod\n def find_windows_kit_bin_paths(\n kit_base: Union[Optional[str], Optional[list]] = None,\n ) -> List[str]:\n \"\"\"Returns Windows kit bin directory per version\"\"\"\n kit_base = WindowsKitExternalPaths.find_windows_kit_roots() if not kit_base else kit_base\n assert kit_base, \"Unexpectedly empty value for Windows kit base path\"\n if isinstance(kit_base, str):\n kit_base = kit_base.split(\";\")\n kit_paths = []\n for kit in kit_base:\n kit_bin = os.path.join(kit, \"bin\")\n kit_paths.extend(glob.glob(os.path.join(kit_bin, \"[0-9]*\", \"*\\\\\")))\n return kit_paths\n\n @staticmethod\n def find_windows_kit_lib_paths(\n kit_base: Union[Optional[str], Optional[list]] = None,\n ) -> List[str]:\n \"\"\"Returns Windows kit lib directory per version\"\"\"\n kit_base = WindowsKitExternalPaths.find_windows_kit_roots() if not kit_base else kit_base\n assert kit_base, \"Unexpectedly empty value for Windows kit base path\"\n if isinstance(kit_base, str):\n kit_base = kit_base.split(\";\")\n kit_paths = []\n for kit in kit_base:\n kit_lib = os.path.join(kit, \"Lib\")\n kit_paths.extend(glob.glob(os.path.join(kit_lib, \"[0-9]*\", \"*\", \"*\\\\\")))\n return kit_paths\n\n @staticmethod\n def find_windows_driver_development_kit_paths() -> List[str]:\n \"\"\"Provides a list of all installation paths\n for the WDK by version and architecture\n \"\"\"\n wdk_content_root = os.getenv(\"WDKContentRoot\")\n return WindowsKitExternalPaths.find_windows_kit_lib_paths(wdk_content_root)\n\n @staticmethod\n def find_windows_kit_reg_installed_roots_paths() -> List[str]:\n reg = spack.util.windows_registry.WindowsRegistryView(\n \"SOFTWARE\\\\Microsoft\\\\Windows Kits\\\\Installed Roots\",\n root_key=spack.util.windows_registry.HKEY.HKEY_LOCAL_MACHINE,\n )\n if not reg:\n # couldn't find key, return empty list\n return []\n kit_root_reg = re.compile(r\"KitsRoot[0-9]+\")\n root_paths = []\n for kit_root in filter(kit_root_reg.match, reg.get_values().keys()):\n root_paths.extend(\n WindowsKitExternalPaths.find_windows_kit_lib_paths(reg.get_value(kit_root).value)\n )\n return root_paths\n\n @staticmethod\n def find_windows_kit_reg_sdk_paths() -> List[str]:\n sdk_paths = []\n sdk_regex = re.compile(r\"v[0-9]+.[0-9]+\")\n windows_reg = spack.util.windows_registry.WindowsRegistryView(\n \"SOFTWARE\\\\WOW6432Node\\\\Microsoft\\\\Microsoft SDKs\\\\Windows\",\n root_key=spack.util.windows_registry.HKEY.HKEY_LOCAL_MACHINE,\n )\n for key in filter(sdk_regex.match, [x.name for x in windows_reg.get_subkeys()]):\n reg = windows_reg.get_subkey(key)\n sdk_paths.extend(\n WindowsKitExternalPaths.find_windows_kit_lib_paths(\n reg.get_value(\"InstallationFolder\").value\n )\n )\n return sdk_paths\n\n\ndef find_win32_additional_install_paths() -> List[str]:\n \"\"\"Not all programs on Windows live on the PATH\n Return a list of other potential install locations.\n \"\"\"\n drive_letter = _windows_drive()\n windows_search_ext = []\n cuda_re = r\"CUDA_PATH[a-zA-Z1-9_]*\"\n # The list below should be expanded with other\n # common Windows install locations as necessary\n path_ext_keys = [\"I_MPI_ONEAPI_ROOT\", \"MSMPI_BIN\", \"MLAB_ROOT\", \"NUGET_PACKAGES\"]\n user = os.environ[\"USERPROFILE\"]\n add_path = lambda key: re.search(cuda_re, key) or key in path_ext_keys\n windows_search_ext.extend([os.environ[key] for key in os.environ.keys() if add_path(key)])\n # note windows paths are fine here as this method should only ever be invoked\n # to interact with Windows\n # Add search path for default Chocolatey (https://github.com/chocolatey/choco)\n # install directory\n windows_search_ext.append(\"%s\\\\ProgramData\\\\chocolatey\\\\bin\" % drive_letter)\n # Add search path for NuGet package manager default install location\n windows_search_ext.append(os.path.join(user, \".nuget\", \"packages\"))\n windows_search_ext.extend(\n spack.config.get(\"config:additional_external_search_paths\", default=[])\n )\n windows_search_ext.extend(spack.util.environment.get_path(\"PATH\"))\n return windows_search_ext\n\n\ndef compute_windows_program_path_for_package(pkg: \"spack.package_base.PackageBase\") -> List[str]:\n \"\"\"Given a package, attempts to compute its Windows program files location,\n and returns the list of best guesses.\n\n Args:\n pkg: package for which Program Files location is to be computed\n \"\"\"\n if sys.platform != \"win32\":\n return []\n # note windows paths are fine here as this method should only ever be invoked\n # to interact with Windows\n program_files = \"{}\\\\Program Files{}\\\\{}\"\n drive_letter = _windows_drive()\n\n return [\n program_files.format(drive_letter, arch, name)\n for arch, name in itertools.product((\"\", \" (x86)\"), (pkg.name, pkg.name.capitalize()))\n ]\n\n\ndef compute_windows_user_path_for_package(pkg: \"spack.package_base.PackageBase\") -> List[str]:\n \"\"\"Given a package attempt to compute its user scoped\n install location, return list of potential locations based\n on common heuristics. For more info on Windows user specific\n installs see:\n https://learn.microsoft.com/en-us/dotnet/api/system.environment.specialfolder?view=netframework-4.8\n \"\"\"\n if sys.platform != \"win32\":\n return []\n\n # Current user directory\n user = os.environ[\"USERPROFILE\"]\n app_data = \"AppData\"\n app_data_locations = [\"Local\", \"Roaming\"]\n user_appdata_install_stubs = [os.path.join(app_data, x) for x in app_data_locations]\n return [\n os.path.join(user, app_data, name)\n for app_data, name in list(\n itertools.product(user_appdata_install_stubs, (pkg.name, pkg.name.capitalize()))\n )\n ] + [os.path.join(user, name) for name in (pkg.name, pkg.name.capitalize())]\n" }, { "path": "lib/spack/spack/detection/path.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Detection of software installed in the system, based on paths inspections\nand running executables.\n\"\"\"\n\nimport collections\nimport concurrent.futures\nimport os\nimport pathlib\nimport re\nimport sys\nimport traceback\nimport warnings\nfrom typing import Dict, Iterable, List, Optional, Set, Tuple, Type\n\nimport spack.error\nimport spack.llnl.util.filesystem\nimport spack.llnl.util.lang\nimport spack.llnl.util.tty\nimport spack.spec\nimport spack.util.elf as elf_utils\nimport spack.util.environment\nimport spack.util.environment as environment\nimport spack.util.ld_so_conf\nimport spack.util.parallel\n\nfrom .common import (\n WindowsCompilerExternalPaths,\n WindowsKitExternalPaths,\n _convert_to_iterable,\n compute_windows_program_path_for_package,\n compute_windows_user_path_for_package,\n executable_prefix,\n find_win32_additional_install_paths,\n library_prefix,\n path_to_dict,\n)\n\n#: Timeout used for package detection (seconds)\nDETECTION_TIMEOUT = 60\nif sys.platform == \"win32\":\n DETECTION_TIMEOUT = 120\n\n\ndef common_windows_package_paths(pkg_cls=None) -> List[str]:\n \"\"\"Get the paths for common package installation location on Windows\n that are outside the PATH\n Returns [] on unix\n \"\"\"\n if sys.platform != \"win32\":\n return []\n paths = WindowsCompilerExternalPaths.find_windows_compiler_bundled_packages()\n paths.extend(find_win32_additional_install_paths())\n paths.extend(WindowsKitExternalPaths.find_windows_kit_bin_paths())\n paths.extend(WindowsKitExternalPaths.find_windows_kit_reg_installed_roots_paths())\n paths.extend(WindowsKitExternalPaths.find_windows_kit_reg_sdk_paths())\n if pkg_cls:\n paths.extend(compute_windows_user_path_for_package(pkg_cls))\n paths.extend(compute_windows_program_path_for_package(pkg_cls))\n return paths\n\n\ndef file_identifier(path):\n s = os.stat(path)\n return s.st_dev, s.st_ino\n\n\ndef dedupe_paths(paths: List[str]) -> List[str]:\n \"\"\"Deduplicate paths based on inode and device number. In case the list contains first a\n symlink and then the directory it points to, the symlink is replaced with the directory path.\n This ensures that we pick for example ``/usr/bin`` over ``/bin`` if the latter is a symlink to\n the former.\"\"\"\n seen: Dict[Tuple[int, int], str] = {}\n\n linked_parent_check = lambda x: any(\n [spack.llnl.util.filesystem.islink(str(y)) for y in pathlib.Path(x).parents]\n )\n\n for path in paths:\n identifier = file_identifier(path)\n if identifier not in seen:\n seen[identifier] = path\n # we also want to deprioritize paths if they contain a symlink in any parent\n # (not just the basedir): e.g. oneapi has \"latest/bin\",\n # where \"latest\" is a symlink to 2025.0\"\n elif not (spack.llnl.util.filesystem.islink(path) or linked_parent_check(path)):\n seen[identifier] = path\n return list(seen.values())\n\n\ndef executables_in_path(path_hints: List[str]) -> Dict[str, str]:\n \"\"\"Get the paths of all executables available from the current PATH.\n\n For convenience, this is constructed as a dictionary where the keys are\n the executable paths and the values are the names of the executables\n (i.e. the basename of the executable path).\n\n There may be multiple paths with the same basename. In this case it is\n assumed there are two different instances of the executable.\n\n Args:\n path_hints: list of paths to be searched. If None the list will be\n constructed based on the PATH environment variable.\n \"\"\"\n search_paths = spack.llnl.util.filesystem.search_paths_for_executables(*path_hints)\n # Make use we don't doubly list /usr/lib and /lib etc\n return path_to_dict(dedupe_paths(search_paths))\n\n\ndef accept_elf(path, host_compat):\n \"\"\"Accept an ELF file if the header matches the given compat triplet. In case it's not an ELF\n (e.g. static library, or some arbitrary file, fall back to is_readable_file).\"\"\"\n # Fast path: assume libraries at least have .so in their basename.\n # Note: don't replace with splitext, because of libsmth.so.1.2.3 file names.\n if \".so\" not in os.path.basename(path):\n return spack.llnl.util.filesystem.is_readable_file(path)\n try:\n return host_compat == elf_utils.get_elf_compat(path)\n except (OSError, elf_utils.ElfParsingError):\n return spack.llnl.util.filesystem.is_readable_file(path)\n\n\ndef libraries_in_ld_and_system_library_path(\n path_hints: Optional[List[str]] = None,\n) -> Dict[str, str]:\n \"\"\"Get the paths of all libraries available from ``path_hints`` or the\n following defaults:\n\n - Environment variables (Linux: ``LD_LIBRARY_PATH``, Darwin: ``DYLD_LIBRARY_PATH``,\n and ``DYLD_FALLBACK_LIBRARY_PATH``)\n - Dynamic linker default paths (glibc: ld.so.conf, musl: ld-musl-.path)\n - Default system library paths.\n\n For convenience, this is constructed as a dictionary where the keys are\n the library paths and the values are the names of the libraries\n (i.e. the basename of the library path).\n\n There may be multiple paths with the same basename. In this case it is\n assumed there are two different instances of the library.\n\n Args:\n path_hints: list of paths to be searched. If None the list will be\n constructed based on the set of LD_LIBRARY_PATH, LIBRARY_PATH,\n DYLD_LIBRARY_PATH, and DYLD_FALLBACK_LIBRARY_PATH environment\n variables as well as the standard system library paths.\n path_hints (list): list of paths to be searched. If ``None``, the default\n system paths are used.\n \"\"\"\n if path_hints:\n search_paths = spack.llnl.util.filesystem.search_paths_for_libraries(*path_hints)\n else:\n search_paths = []\n\n # Environment variables\n if sys.platform == \"darwin\":\n search_paths.extend(environment.get_path(\"DYLD_LIBRARY_PATH\"))\n search_paths.extend(environment.get_path(\"DYLD_FALLBACK_LIBRARY_PATH\"))\n elif sys.platform.startswith(\"linux\"):\n search_paths.extend(environment.get_path(\"LD_LIBRARY_PATH\"))\n\n # Dynamic linker paths\n search_paths.extend(spack.util.ld_so_conf.host_dynamic_linker_search_paths())\n\n # Drop redundant paths\n search_paths = list(filter(os.path.isdir, search_paths))\n\n # Make use we don't doubly list /usr/lib and /lib etc\n search_paths = dedupe_paths(search_paths)\n\n try:\n host_compat = elf_utils.get_elf_compat(sys.executable)\n accept = lambda path: accept_elf(path, host_compat)\n except (OSError, elf_utils.ElfParsingError):\n accept = spack.llnl.util.filesystem.is_readable_file\n\n path_to_lib = {}\n # Reverse order of search directories so that a lib in the first\n # search path entry overrides later entries\n for search_path in reversed(search_paths):\n for lib in os.listdir(search_path):\n lib_path = os.path.join(search_path, lib)\n if accept(lib_path):\n path_to_lib[lib_path] = lib\n return path_to_lib\n\n\ndef libraries_in_windows_paths(path_hints: Optional[List[str]] = None) -> Dict[str, str]:\n \"\"\"Get the paths of all libraries available from the system PATH paths.\n\n For more details, see ``libraries_in_ld_and_system_library_path`` regarding\n return type and contents.\n\n Args:\n path_hints: list of paths to be searched. If None the list will be\n constructed based on the set of PATH environment\n variables as well as the standard system library paths.\n \"\"\"\n search_hints = (\n path_hints if path_hints is not None else spack.util.environment.get_path(\"PATH\")\n )\n search_paths = spack.llnl.util.filesystem.search_paths_for_libraries(*search_hints)\n # on Windows, some libraries (.dlls) are found in the bin directory or sometimes\n # at the search root. Add both of those options to the search scheme\n search_paths.extend(spack.llnl.util.filesystem.search_paths_for_executables(*search_hints))\n if path_hints is None:\n # if no user provided path was given, add defaults to the search\n search_paths.extend(WindowsKitExternalPaths.find_windows_kit_lib_paths())\n # SDK and WGL should be handled by above, however on occasion the WDK is in an atypical\n # location, so we handle that case specifically.\n search_paths.extend(WindowsKitExternalPaths.find_windows_driver_development_kit_paths())\n return path_to_dict(search_paths)\n\n\ndef _group_by_prefix(paths: List[str]) -> Dict[str, Set[str]]:\n groups = collections.defaultdict(set)\n for p in paths:\n groups[os.path.dirname(p)].add(p)\n return groups\n\n\nclass Finder:\n \"\"\"Inspects the file-system looking for packages. Guesses places where to look using PATH.\"\"\"\n\n def default_path_hints(self) -> List[str]:\n return []\n\n def search_patterns(self, *, pkg: Type[\"spack.package_base.PackageBase\"]) -> List[str]:\n \"\"\"Returns the list of patterns used to match candidate files.\n\n Args:\n pkg: package being detected\n \"\"\"\n raise NotImplementedError(\"must be implemented by derived classes\")\n\n def candidate_files(self, *, patterns: List[str], paths: List[str]) -> List[str]:\n \"\"\"Returns a list of candidate files found on the system.\n\n Args:\n patterns: search patterns to be used for matching files\n paths: paths where to search for files\n \"\"\"\n raise NotImplementedError(\"must be implemented by derived classes\")\n\n def prefix_from_path(self, *, path: str) -> str:\n \"\"\"Given a path where a file was found, returns the corresponding prefix.\n\n Args:\n path: path of a detected file\n \"\"\"\n raise NotImplementedError(\"must be implemented by derived classes\")\n\n def detect_specs(\n self, *, pkg: Type[\"spack.package_base.PackageBase\"], paths: Iterable[str], repo_path\n ) -> List[\"spack.spec.Spec\"]:\n \"\"\"Given a list of files matching the search patterns, returns a list of detected specs.\n\n Args:\n pkg: package being detected\n paths: files matching the package search patterns\n \"\"\"\n if not hasattr(pkg, \"determine_spec_details\"):\n warnings.warn(\n f\"{pkg.name} must define 'determine_spec_details' in order\"\n f\" for Spack to detect externally-provided instances\"\n f\" of the package.\"\n )\n return []\n\n result = []\n resolved_specs: Dict[spack.spec.Spec, str] = {} # spec -> prefix of first detection\n for candidate_path, items_in_prefix in _group_by_prefix(\n spack.llnl.util.lang.dedupe(paths)\n ).items():\n # TODO: multiple instances of a package can live in the same\n # prefix, and a package implementation can return multiple specs\n # for one prefix, but without additional details (e.g. about the\n # naming scheme which differentiates them), the spec won't be\n # usable.\n try:\n specs = _convert_to_iterable(\n pkg.determine_spec_details(candidate_path, items_in_prefix)\n )\n except Exception as e:\n specs = []\n if spack.error.SHOW_BACKTRACE:\n details = traceback.format_exc()\n else:\n details = f\"[{e.__class__.__name__}: {e}]\"\n warnings.warn(\n f'error detecting \"{pkg.name}\" from prefix {candidate_path}: {details}'\n )\n\n if not specs:\n files = \", \".join(_convert_to_iterable(items_in_prefix))\n spack.llnl.util.tty.debug(\n f\"The following files in {candidate_path} were decidedly not \"\n f\"part of the package {pkg.name}: {files}\"\n )\n\n for spec in specs:\n prefix = self.prefix_from_path(path=candidate_path)\n if not prefix:\n continue\n\n if spec in resolved_specs:\n prior_prefix = resolved_specs[spec]\n warnings.warn(\n f'\"{spec}\" detected in \"{prefix}\" was already detected in \"{prior_prefix}\"'\n )\n continue\n\n resolved_specs[spec] = prefix\n try:\n # Validate the spec calling a package specific method\n pkg_cls = repo_path.get_pkg_class(spec.name)\n validate_fn = getattr(pkg_cls, \"validate_detected_spec\", lambda x, y: None)\n validate_fn(spec, spec.extra_attributes)\n except Exception as e:\n msg = (\n f'\"{spec}\" has been detected on the system but will '\n f\"not be added to packages.yaml [reason={str(e)}]\"\n )\n warnings.warn(msg)\n continue\n\n if not spec.external_path:\n spec.external_path = prefix\n\n result.append(spec)\n\n return result\n\n def find(\n self, *, pkg_name: str, repository, initial_guess: Optional[List[str]] = None\n ) -> List[\"spack.spec.Spec\"]:\n \"\"\"For a given package, returns a list of detected specs.\n\n Args:\n pkg_name: package being detected\n repository: repository to retrieve the package\n initial_guess: initial list of paths to search from the caller if None, default paths\n are searched. If this is an empty list, nothing will be searched.\n \"\"\"\n pkg_cls = repository.get_pkg_class(pkg_name)\n patterns = self.search_patterns(pkg=pkg_cls)\n if not patterns:\n return []\n if initial_guess is None:\n initial_guess = self.default_path_hints()\n initial_guess.extend(common_windows_package_paths(pkg_cls))\n candidates = self.candidate_files(patterns=patterns, paths=initial_guess)\n return self.detect_specs(pkg=pkg_cls, paths=candidates, repo_path=repository)\n\n\nclass ExecutablesFinder(Finder):\n def default_path_hints(self) -> List[str]:\n return spack.util.environment.get_path(\"PATH\")\n\n def search_patterns(self, *, pkg: Type[\"spack.package_base.PackageBase\"]) -> List[str]:\n result = []\n if hasattr(pkg, \"executables\") and hasattr(pkg, \"platform_executables\"):\n result = pkg.platform_executables()\n return result\n\n def candidate_files(self, *, patterns: List[str], paths: List[str]) -> List[str]:\n executables_by_path = executables_in_path(path_hints=paths)\n joined_pattern = re.compile(r\"|\".join(patterns))\n result = [path for path, exe in executables_by_path.items() if joined_pattern.search(exe)]\n result.sort()\n return result\n\n def prefix_from_path(self, *, path: str) -> str:\n result = executable_prefix(path)\n if not result:\n msg = f\"no bin/ dir found in {path}. Cannot add it as a Spack package\"\n spack.llnl.util.tty.debug(msg)\n return result\n\n\nclass LibrariesFinder(Finder):\n \"\"\"Finds libraries on the system, searching by LD_LIBRARY_PATH, LIBRARY_PATH,\n DYLD_LIBRARY_PATH, DYLD_FALLBACK_LIBRARY_PATH, and standard system library paths\n \"\"\"\n\n def search_patterns(self, *, pkg: Type[\"spack.package_base.PackageBase\"]) -> List[str]:\n result = []\n if hasattr(pkg, \"libraries\"):\n result = pkg.libraries\n return result\n\n def candidate_files(self, *, patterns: List[str], paths: List[str]) -> List[str]:\n libraries_by_path = (\n libraries_in_ld_and_system_library_path(path_hints=paths)\n if sys.platform != \"win32\"\n else libraries_in_windows_paths(path_hints=paths)\n )\n patterns = [re.compile(x) for x in patterns]\n result = []\n for compiled_re in patterns:\n for path, exe in libraries_by_path.items():\n if compiled_re.search(exe):\n result.append(path)\n return result\n\n def prefix_from_path(self, *, path: str) -> str:\n result = library_prefix(path)\n if not result:\n msg = f\"no lib/ or lib64/ dir found in {path}. Cannot add it as a Spack package\"\n spack.llnl.util.tty.debug(msg)\n return result\n\n\ndef by_path(\n packages_to_search: Iterable[str],\n *,\n path_hints: Optional[List[str]] = None,\n max_workers: Optional[int] = None,\n) -> Dict[str, List[\"spack.spec.Spec\"]]:\n \"\"\"Return the list of packages that have been detected on the system, keyed by\n unqualified package name.\n\n Args:\n packages_to_search: list of packages to be detected. Each package can be either unqualified\n of fully qualified\n path_hints: initial list of paths to be searched\n max_workers: maximum number of workers to search for packages in parallel\n \"\"\"\n from spack.repo import PATH, partition_package_name\n\n # TODO: Packages should be able to define both .libraries and .executables in the future\n # TODO: determine_spec_details should get all relevant libraries and executables in one call\n executables_finder, libraries_finder = ExecutablesFinder(), LibrariesFinder()\n detected_specs_by_package: Dict[str, Tuple[concurrent.futures.Future, ...]] = {}\n\n result = collections.defaultdict(list)\n repository = PATH.ensure_unwrapped()\n\n executor: concurrent.futures.Executor\n if max_workers == 1:\n executor = spack.util.parallel.SequentialExecutor()\n else:\n executor = spack.util.parallel.make_concurrent_executor(max_workers, require_fork=False)\n with executor:\n for pkg in packages_to_search:\n executable_future = executor.submit(\n executables_finder.find,\n pkg_name=pkg,\n initial_guess=path_hints,\n repository=repository,\n )\n library_future = executor.submit(\n libraries_finder.find,\n pkg_name=pkg,\n initial_guess=path_hints,\n repository=repository,\n )\n detected_specs_by_package[pkg] = executable_future, library_future\n\n for pkg_name, futures in detected_specs_by_package.items():\n for future in futures:\n try:\n detected = future.result(timeout=DETECTION_TIMEOUT)\n if detected:\n _, unqualified_name = partition_package_name(pkg_name)\n result[unqualified_name].extend(detected)\n except concurrent.futures.TimeoutError:\n spack.llnl.util.tty.debug(\n f\"[EXTERNAL DETECTION] Skipping {pkg_name}: timeout reached\"\n )\n except Exception:\n spack.llnl.util.tty.debug(\n f\"[EXTERNAL DETECTION] Skipping {pkg_name}: {traceback.format_exc()}\"\n )\n\n return result\n" }, { "path": "lib/spack/spack/detection/test.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Create and run mock e2e tests for package detection.\"\"\"\n\nimport collections\nimport contextlib\nimport pathlib\nimport tempfile\nfrom typing import Any, Deque, Dict, Generator, List, NamedTuple, Tuple\n\nimport spack.platforms\nimport spack.repo\nimport spack.spec\nfrom spack.llnl.util import filesystem\nfrom spack.util import spack_yaml\n\nfrom .path import by_path\n\n\nclass MockExecutables(NamedTuple):\n \"\"\"Mock executables to be used in detection tests\"\"\"\n\n #: Relative paths for mock executables to be created\n executables: List[str]\n #: Shell script for the mock executable\n script: str\n\n\nclass ExpectedTestResult(NamedTuple):\n \"\"\"Data structure to model assertions on detection tests\"\"\"\n\n #: Spec to be detected\n spec: str\n #: Attributes expected in the external spec\n extra_attributes: Dict[str, str]\n\n\nclass DetectionTest(NamedTuple):\n \"\"\"Data structure to construct detection tests by PATH inspection.\n\n Packages may have a YAML file containing the description of one or more detection tests\n to be performed. Each test creates a few mock executable scripts in a temporary folder,\n and checks that detection by PATH gives the expected results.\n \"\"\"\n\n pkg_name: str\n layout: List[MockExecutables]\n results: List[ExpectedTestResult]\n\n\nclass Runner:\n \"\"\"Runs an external detection test\"\"\"\n\n def __init__(self, *, test: DetectionTest, repository: spack.repo.RepoPath) -> None:\n self.test = test\n self.repository = repository\n self.tmpdir = tempfile.TemporaryDirectory()\n\n def execute(self) -> List[spack.spec.Spec]:\n \"\"\"Executes a test and returns the specs that have been detected.\n\n This function sets-up a test in a temporary directory, according to the prescriptions\n in the test layout, then performs a detection by executables and returns the specs that\n have been detected.\n \"\"\"\n with self._mock_layout() as path_hints:\n entries = by_path([self.test.pkg_name], path_hints=path_hints)\n _, unqualified_name = spack.repo.partition_package_name(self.test.pkg_name)\n specs = set(entries[unqualified_name])\n return list(specs)\n\n @contextlib.contextmanager\n def _mock_layout(self) -> Generator[List[str], None, None]:\n hints = set()\n try:\n for entry in self.test.layout:\n exes = self._create_executable_scripts(entry)\n\n for mock_executable in exes:\n hints.add(str(mock_executable.parent))\n\n yield list(hints)\n finally:\n self.tmpdir.cleanup()\n\n def _create_executable_scripts(self, mock_executables: MockExecutables) -> List[pathlib.Path]:\n import spack.vendor.jinja2\n\n relative_paths = mock_executables.executables\n script = mock_executables.script\n script_template = spack.vendor.jinja2.Template(\"#!/bin/bash\\n{{ script }}\\n\")\n result = []\n for mock_exe_path in relative_paths:\n rel_path = pathlib.Path(mock_exe_path)\n abs_path = pathlib.Path(self.tmpdir.name) / rel_path\n abs_path.parent.mkdir(parents=True, exist_ok=True)\n abs_path.write_text(script_template.render(script=script))\n filesystem.set_executable(abs_path)\n result.append(abs_path)\n return result\n\n @property\n def expected_specs(self) -> List[spack.spec.Spec]:\n return [\n spack.spec.Spec.from_detection(\n item.spec, external_path=self.tmpdir.name, extra_attributes=item.extra_attributes\n )\n for item in self.test.results\n ]\n\n\ndef detection_tests(pkg_name: str, repository: spack.repo.RepoPath) -> List[Runner]:\n \"\"\"Returns a list of test runners for a given package.\n\n Currently, detection tests are specified in a YAML file, called ``detection_test.yaml``,\n alongside the ``package.py`` file.\n\n This function reads that file to create a bunch of ``Runner`` objects.\n\n Args:\n pkg_name: name of the package to test\n repository: repository where the package lives\n \"\"\"\n result = []\n detection_tests_content = read_detection_tests(pkg_name, repository)\n current_platform = str(spack.platforms.host())\n\n tests_by_path = detection_tests_content.get(\"paths\", [])\n for single_test_data in tests_by_path:\n if current_platform not in single_test_data.get(\"platforms\", [current_platform]):\n continue\n\n mock_executables = []\n for layout in single_test_data[\"layout\"]:\n mock_executables.append(\n MockExecutables(executables=layout[\"executables\"], script=layout[\"script\"])\n )\n expected_results = []\n for assertion in single_test_data[\"results\"]:\n expected_results.append(\n ExpectedTestResult(\n spec=assertion[\"spec\"], extra_attributes=assertion.get(\"extra_attributes\", {})\n )\n )\n\n current_test = DetectionTest(\n pkg_name=pkg_name, layout=mock_executables, results=expected_results\n )\n result.append(Runner(test=current_test, repository=repository))\n\n return result\n\n\ndef read_detection_tests(pkg_name: str, repository: spack.repo.RepoPath) -> Dict[str, Any]:\n \"\"\"Returns the normalized content of the detection_tests.yaml associated with the package\n passed in input.\n\n The content is merged with that of any package that is transitively included using the\n \"includes\" attribute.\n\n Args:\n pkg_name: name of the package to test\n repository: repository in which to search for packages\n \"\"\"\n content_stack, seen = [], set()\n included_packages: Deque[str] = collections.deque()\n\n root_detection_yaml, result = _detection_tests_yaml(pkg_name, repository)\n included_packages.extend(result.get(\"includes\", []))\n seen |= set(result.get(\"includes\", []))\n\n while included_packages:\n current_package = included_packages.popleft()\n try:\n current_detection_yaml, content = _detection_tests_yaml(current_package, repository)\n except FileNotFoundError as e:\n msg = (\n f\"cannot read the detection tests from the '{current_package}' package, \"\n f\"included by {root_detection_yaml}\"\n )\n raise FileNotFoundError(msg + f\"\\n\\n\\t{e}\\n\")\n\n content_stack.append((current_package, content))\n included_packages.extend(x for x in content.get(\"includes\", []) if x not in seen)\n seen |= set(content.get(\"includes\", []))\n\n result.setdefault(\"paths\", [])\n for pkg_name, content in content_stack:\n result[\"paths\"].extend(content.get(\"paths\", []))\n\n return result\n\n\ndef _detection_tests_yaml(\n pkg_name: str, repository: spack.repo.RepoPath\n) -> Tuple[pathlib.Path, Dict[str, Any]]:\n pkg_dir = pathlib.Path(repository.filename_for_package_name(pkg_name)).parent\n detection_tests_yaml = pkg_dir / \"detection_test.yaml\"\n with open(str(detection_tests_yaml), encoding=\"utf-8\") as f:\n content = spack_yaml.load(f)\n return detection_tests_yaml, content\n" }, { "path": "lib/spack/spack/directives.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"This package contains directives that can be used within a package.\n\nDirectives are functions that can be called inside a package\ndefinition to modify the package, for example::\n\n class OpenMpi(Package):\n depends_on(\"hwloc\")\n provides(\"mpi\")\n ...\n\n``provides`` and ``depends_on`` are spack directives.\n\nThe available directives are:\n\n* ``build_system``\n* ``conflicts``\n* ``depends_on``\n* ``extends``\n* ``license``\n* ``patch``\n* ``provides``\n* ``resource``\n* ``variant``\n* ``version``\n* ``requires``\n* ``redistribute``\n\nThey're implemented as functions that return partial functions that are later executed with a\npackage class as first argument::\n\n @directive(\"example\")\n def example_directive(arg1, arg2):\n return partial(_execute_example_directive, arg1=arg1, arg2=arg2)\n\n def _execute_example_directive(pkg, arg1, arg2):\n # modify pkg.example based on arg1 and arg2\n\"\"\"\n\nimport collections\nimport collections.abc\nimport os\nimport re\nimport warnings\nfrom functools import partial\nfrom typing import Any, Callable, List, Optional, Tuple, Type, Union\n\nimport spack.deptypes as dt\nimport spack.error\nimport spack.fetch_strategy\nimport spack.llnl.util.tty.color\nimport spack.package_base\nimport spack.patch\nimport spack.spec\nimport spack.util.crypto\nimport spack.variant\nfrom spack.dependency import Dependency\nfrom spack.directives_meta import DirectiveError, directive, get_spec\nfrom spack.resource import Resource\nfrom spack.spec import EMPTY_SPEC\nfrom spack.version import StandardVersion, VersionChecksumError, VersionError\n\n__all__ = [\n \"DirectiveError\",\n \"version\",\n \"conditional\",\n \"conflicts\",\n \"depends_on\",\n \"extends\",\n \"maintainers\",\n \"license\",\n \"provides\",\n \"patch\",\n \"variant\",\n \"resource\",\n \"build_system\",\n \"requires\",\n \"redistribute\",\n \"can_splice\",\n]\n\n_patch_order_index = 0\n\n\nSpecType = str\nDepType = Union[Tuple[str, ...], str]\nWhenType = Optional[Union[spack.spec.Spec, str, bool]]\nPackageType = Type[spack.package_base.PackageBase]\nPatcher = Callable[[Union[PackageType, Dependency]], None]\nPatchesType = Union[Patcher, str, List[Union[Patcher, str]]]\n\n\ndef _make_when_spec(value: Union[WhenType, Tuple[str, ...]]) -> Optional[spack.spec.Spec]:\n \"\"\"Create a ``Spec`` that indicates when a directive should be applied.\n\n Directives with ``when`` specs, e.g.:\n\n patch('foo.patch', when='@4.5.1:')\n depends_on('mpi', when='+mpi')\n depends_on('readline', when=sys.platform() != 'darwin')\n\n are applied conditionally depending on the value of the ``when``\n keyword argument. Specifically:\n\n 1. If the ``when`` argument is ``True``, the directive is always applied\n 2. If it is ``False``, the directive is never applied\n 3. If it is a ``Spec`` string, it is applied when the package's\n concrete spec satisfies the ``when`` spec.\n\n The first two conditions are useful for the third example case above.\n It allows package authors to include directives that are conditional\n at package definition time, in additional to ones that are evaluated\n as part of concretization.\n\n Arguments:\n value: a conditional Spec, constant ``bool``, or None if not supplied\n value indicating when a directive should be applied. It can also be a tuple of when\n conditions (as strings) to be combined together.\n\n \"\"\"\n # This branch is never taken, but our WhenType type annotation allows it, so handle it too.\n if isinstance(value, spack.spec.Spec):\n return value\n\n if isinstance(value, tuple):\n assert value, \"when stack cannot be empty\"\n # avoid a copy when there's only one condition\n if len(value) == 1:\n return get_spec(value[0])\n # reduce the when-stack to a single spec by combining all constraints.\n combined_spec = spack.spec.Spec(value[0])\n for cond in value[1:]:\n combined_spec._constrain_symbolically(get_spec(cond))\n return combined_spec\n\n # Unsatisfiable conditions are discarded by the caller, and never\n # added to the package class\n if value is False:\n return None\n\n # If there is no constraint, the directive should always apply;\n # represent this by returning the unconstrained `Spec()`, which is\n # always satisfied.\n if value is None or value is True:\n return EMPTY_SPEC\n\n # This is conditional on the spec\n return get_spec(value)\n\n\nSubmoduleCallback = Callable[[spack.package_base.PackageBase], Union[str, List[str], bool]]\n\n\n@directive(\"versions\", supports_when=False)\ndef version(\n ver: Union[str, int],\n # this positional argument is deprecated, use sha256=... instead\n checksum: Optional[str] = None,\n *,\n # generic version options\n preferred: Optional[bool] = None,\n deprecated: Optional[bool] = None,\n no_cache: Optional[bool] = None,\n # url fetch options\n url: Optional[str] = None,\n extension: Optional[str] = None,\n expand: Optional[bool] = None,\n fetch_options: Optional[dict] = None,\n # url archive verification options\n md5: Optional[str] = None,\n sha1: Optional[str] = None,\n sha224: Optional[str] = None,\n sha256: Optional[str] = None,\n sha384: Optional[str] = None,\n sha512: Optional[str] = None,\n # git fetch options\n git: Optional[str] = None,\n commit: Optional[str] = None,\n tag: Optional[str] = None,\n branch: Optional[str] = None,\n get_full_repo: Optional[bool] = None,\n git_sparse_paths: Optional[\n Union[List[str], Callable[[spack.package_base.PackageBase], List[str]]]\n ] = None,\n submodules: Union[SubmoduleCallback, Optional[bool]] = None,\n submodules_delete: Optional[bool] = None,\n # other version control\n svn: Optional[str] = None,\n hg: Optional[str] = None,\n cvs: Optional[str] = None,\n revision: Optional[str] = None,\n date: Optional[str] = None,\n):\n \"\"\"Declare a version for a package with optional metadata for fetching its code.\n\n Example::\n\n version(\"2.1\", sha256=\"...\")\n version(\"2.0\", sha256=\"...\", preferred=True)\n\n .. versionchanged:: v2.3\n\n The ``git_sparse_paths`` parameter was added.\n \"\"\"\n kwargs: dict = {\n key: value\n for key, value in (\n (\"sha256\", sha256),\n (\"sha384\", sha384),\n (\"sha512\", sha512),\n (\"preferred\", preferred),\n (\"deprecated\", deprecated),\n (\"expand\", expand),\n (\"url\", url),\n (\"extension\", extension),\n (\"no_cache\", no_cache),\n (\"fetch_options\", fetch_options),\n (\"git\", git),\n (\"svn\", svn),\n (\"hg\", hg),\n (\"cvs\", cvs),\n (\"get_full_repo\", get_full_repo),\n (\"git_sparse_paths\", git_sparse_paths),\n (\"branch\", branch),\n (\"submodules\", submodules),\n (\"submodules_delete\", submodules_delete),\n (\"commit\", commit),\n (\"tag\", tag),\n (\"revision\", revision),\n (\"date\", date),\n (\"md5\", md5),\n (\"sha1\", sha1),\n (\"sha224\", sha224),\n (\"checksum\", checksum),\n )\n if value is not None\n }\n return partial(_execute_version, ver=ver, kwargs=kwargs)\n\n\ndef _execute_version(pkg: PackageType, ver: Union[str, int], kwargs: dict):\n if (\n (any(s in kwargs for s in spack.util.crypto.hashes) or \"checksum\" in kwargs)\n and hasattr(pkg, \"has_code\")\n and not pkg.has_code\n ):\n raise VersionChecksumError(\n f\"{pkg.name}: Checksums not allowed in no-code packages (see '{ver}' version).\"\n )\n\n if not isinstance(ver, (int, str)):\n raise VersionError(\n f\"{pkg.name}: declared version '{ver!r}' in package should be a string or int.\"\n )\n\n version = StandardVersion.from_string(str(ver))\n\n # Store kwargs for the package to later with a fetch_strategy.\n pkg.versions[version] = kwargs\n\n\n@directive(\"conflicts\")\ndef conflicts(conflict_spec: SpecType, when: WhenType = None, msg: Optional[str] = None):\n \"\"\"Declare a conflict for a package.\n\n A conflict is a spec that is known to be invalid. For example, a package that cannot build\n with GCC 14 and above can declare::\n\n conflicts(\"%gcc@14:\")\n\n To express the same constraint only when the ``foo`` variant is activated::\n\n conflicts(\"%gcc@14:\", when=\"+foo\")\n\n Args:\n conflict_spec: constraint defining the known conflict\n when: optional condition that triggers the conflict\n msg: optional user defined message\n \"\"\"\n return partial(_execute_conflicts, conflict_spec=conflict_spec, when=when, msg=msg)\n\n\ndef _execute_conflicts(pkg: PackageType, conflict_spec, when, msg):\n # If when is not specified the conflict always holds\n when_spec = _make_when_spec(when)\n if not when_spec:\n return\n\n # Save in a list the conflicts and the associated custom messages\n conflict_spec_list = pkg.conflicts.setdefault(when_spec, [])\n msg_with_name = f\"{pkg.name}: {msg}\" if msg is not None else msg\n conflict_spec_list.append((get_spec(conflict_spec), msg_with_name))\n\n\n@directive(\"dependencies\", can_patch_dependencies=True)\ndef depends_on(\n spec: SpecType,\n when: WhenType = None,\n type: DepType = dt.DEFAULT_TYPES,\n *,\n patches: Optional[PatchesType] = None,\n):\n \"\"\"Declare a dependency on another package.\n\n Example::\n\n depends_on(\"hwloc@2:\", when=\"@1:\", type=\"link\")\n\n Args:\n spec: dependency spec\n when: condition when this dependency applies\n type: One or more of ``\"build\"``, ``\"run\"``, ``\"test\"``, or ``\"link\"`` (either a string or\n tuple). Defaults to ``(\"build\", \"link\")``.\n patches: single result of :py:func:`patch` directive, a\n ``str`` to be passed to ``patch``, or a list of these\n \"\"\"\n return partial(_execute_depends_on, spec=spec, when=when, type=type, patches=patches)\n\n\ndef _execute_depends_on(\n pkg: PackageType,\n spec: Union[str, spack.spec.Spec],\n *,\n when: WhenType = None,\n type: DepType = dt.DEFAULT_TYPES,\n patches: Optional[PatchesType] = None,\n):\n spec = get_spec(spec) if isinstance(spec, str) else spec\n when_spec = _make_when_spec(when)\n if not when_spec:\n return\n\n if not spec.name:\n raise DependencyError(\n f\"Invalid dependency specification in package '{pkg.name}':\", str(spec)\n )\n if pkg.name == spec.name:\n raise CircularReferenceError(f\"Package '{pkg.name}' cannot depend on itself.\")\n\n depflag = dt.canonicalize(type)\n\n # call this patches here for clarity -- we want patch to be a list,\n # but the caller doesn't have to make it one.\n\n # Note: we cannot check whether a package is virtual in a directive\n # because directives are run as part of class instantiation, and specs\n # instantiate the package class as part of the `virtual` check.\n # To be technical, specs only instantiate the package class as part of the\n # virtual check if the provider index hasn't been created yet.\n # TODO: There could be a cache warming strategy that would allow us to\n # ensure `Spec.virtual` is a valid thing to call in a directive.\n # For now, we comment out the following check to allow for virtual packages\n # with package files.\n # if patches and spec.virtual:\n # raise DependencyPatchError(\"Cannot patch a virtual dependency.\")\n\n # ensure patches is a list\n if patches is None:\n patches = []\n elif not isinstance(patches, (list, tuple)):\n patches = [patches]\n\n # this is where we actually add the dependency to this package\n deps_by_name = pkg.dependencies.setdefault(when_spec, {})\n dependency = deps_by_name.get(spec.name)\n\n edges = spec.edges_to_dependencies()\n if edges and not all(x.direct for x in edges):\n raise DirectiveError(\n f\"the '^' sigil cannot be used in 'depends_on' directives. Please reformulate \"\n f\"the directive below as multiple directives:\\n\\n\"\n f'\\tdepends_on(\"{spec}\", when=\"{when_spec}\")\\n'\n )\n\n if not dependency:\n dependency = Dependency(pkg, spec, depflag=depflag)\n deps_by_name[spec.name] = dependency\n else:\n copy = dependency.spec.copy()\n copy.constrain(spec, deps=False)\n dependency.spec = copy\n dependency.depflag |= depflag\n\n # apply patches to the dependency\n for patch in patches:\n if isinstance(patch, str):\n _execute_patch(dependency, url_or_filename=patch)\n else:\n assert callable(patch), f\"Invalid patch argument: {patch!r}\"\n patch(dependency)\n\n\n@directive(\"disable_redistribute\")\ndef redistribute(\n source: Optional[bool] = None, binary: Optional[bool] = None, when: WhenType = None\n):\n \"\"\"Declare that the package source and/or compiled binaries should not be redistributed.\n\n By default, packages allow source/binary distribution (in mirrors/build caches resp.).\n This directive allows users to explicitly disable redistribution for specs.\n \"\"\"\n return partial(_execute_redistribute, source=source, binary=binary, when=when)\n\n\ndef _execute_redistribute(\n pkg: PackageType, source: Optional[bool], binary: Optional[bool], when: WhenType\n):\n if source is None and binary is None:\n return\n elif (source is True) or (binary is True):\n raise DirectiveError(\n \"Source/binary distribution are true by default, they can only be explicitly disabled.\"\n )\n\n if source is None:\n source = True\n if binary is None:\n binary = True\n\n when_spec = _make_when_spec(when)\n if not when_spec:\n return\n if source is False:\n max_constraint = get_spec(f\"{pkg.name}@{when_spec.versions}\")\n if not max_constraint.satisfies(when_spec):\n raise DirectiveError(\"Source distribution can only be disabled for versions\")\n\n if when_spec in pkg.disable_redistribute:\n disable = pkg.disable_redistribute[when_spec]\n if not source:\n disable.source = True\n if not binary:\n disable.binary = True\n else:\n pkg.disable_redistribute[when_spec] = spack.package_base.DisableRedistribute(\n source=not source, binary=not binary\n )\n\n\n@directive((\"extendees\", \"dependencies\"), can_patch_dependencies=True)\ndef extends(\n spec: str,\n when: WhenType = None,\n type: DepType = (\"build\", \"run\"),\n *,\n patches: Optional[PatchesType] = None,\n):\n \"\"\"Same as :func:`depends_on`, but also adds this package to the extendee list.\n In case of Python, also adds a dependency on ``python-venv``.\n\n .. note::\n\n Notice that the default ``type`` is ``(\"build\", \"run\")``, which is different from\n :func:`depends_on` where the default is ``(\"build\", \"link\")``.\"\"\"\n\n return partial(_execute_extends, spec=spec, when=when, type=type, patches=patches)\n\n\ndef _execute_extends(\n pkg: PackageType, spec: str, when: WhenType, type: DepType, patches: Optional[PatchesType]\n):\n when_spec = _make_when_spec(when)\n if not when_spec:\n return\n\n dep_spec = get_spec(spec)\n\n _execute_depends_on(pkg, dep_spec, when=when, type=type, patches=patches)\n\n # When extending python, also add a dependency on python-venv. This is done so that\n # Spack environment views are Python virtual environments.\n if dep_spec.name == \"python\" and not pkg.name == \"python-venv\":\n _execute_depends_on(pkg, \"python-venv\", when=when, type=(\"build\", \"run\"))\n\n pkg.extendees[dep_spec.name] = (dep_spec, when_spec)\n\n\n@directive((\"provided\", \"provided_together\"))\ndef provides(*specs: SpecType, when: WhenType = None):\n \"\"\"Declare that this package provides a virtual dependency.\n\n If a package provides ``mpi``, other packages can declare that they depend on ``mpi``,\n and spack can use the providing package to satisfy the dependency.\n\n Args:\n *specs: virtual specs provided by this package\n when: condition when this provides clause needs to be considered\n \"\"\"\n\n return partial(_execute_provides, specs=specs, when=when)\n\n\ndef _execute_provides(pkg: PackageType, specs: Tuple[SpecType, ...], when: WhenType):\n when_spec = _make_when_spec(when)\n if not when_spec:\n return\n\n spec_objs = [get_spec(x) for x in specs]\n spec_names = [x.name for x in spec_objs]\n if len(spec_names) > 1:\n pkg.provided_together.setdefault(when_spec, []).append(set(spec_names))\n\n for provided_spec in spec_objs:\n if pkg.name == provided_spec.name:\n raise CircularReferenceError(f\"Package '{pkg.name}' cannot provide itself.\")\n\n pkg.provided.setdefault(when_spec, set()).add(provided_spec)\n\n\n@directive(\"splice_specs\")\ndef can_splice(\n target: SpecType, *, when: SpecType, match_variants: Union[None, str, List[str]] = None\n):\n \"\"\"Declare whether the package is ABI-compatible with another package and thus can be spliced\n into concrete versions of that package.\n\n Args:\n target: The spec that the current package is ABI-compatible with.\n\n when: An anonymous spec constraining current package for when it is ABI-compatible with\n target.\n\n match_variants: A list of variants that must match between target spec and current package,\n with special value ``*`` which matches all variants. Example: a ``json`` variant is\n defined on two packages, and they are ABI-compatible whenever they agree on\n the json variant (regardless of whether it is turned on or off). Note that this cannot\n be applied to multi-valued variants and multi-valued variants will be skipped by ``*``.\n \"\"\"\n\n return partial(_execute_can_splice, target=target, when=when, match_variants=match_variants)\n\n\ndef _execute_can_splice(\n pkg: PackageType, target: SpecType, when: SpecType, match_variants: Union[None, str, List[str]]\n):\n when_spec = _make_when_spec(when)\n if isinstance(match_variants, str) and match_variants != \"*\":\n raise ValueError(\n \"* is the only valid string for match_variants \"\n \"if looking to provide a single variant, use \"\n f\"[{match_variants}] instead\"\n )\n if when_spec is None:\n return\n pkg.splice_specs[when_spec] = (get_spec(target), match_variants)\n\n\n@directive(\"patches\")\ndef patch(\n url_or_filename: str,\n level: int = 1,\n when: WhenType = None,\n working_dir: str = \".\",\n reverse: bool = False,\n sha256: Optional[str] = None,\n archive_sha256: Optional[str] = None,\n) -> Patcher:\n \"\"\"Declare a patch to apply to package sources. A when spec can be provided to indicate that a\n particular patch should only be applied when the package's spec meets certain conditions.\n\n Example::\n\n patch(\"foo.patch\", when=\"@1.0.0:\")\n patch(\"https://example.com/foo.patch\", sha256=\"...\")\n\n Args:\n url_or_filename: url or relative filename of the patch\n level: patch level (as in the patch shell command)\n when: optional anonymous spec that specifies when to apply the patch\n working_dir: dir to change to before applying\n reverse: reverse the patch\n sha256: sha256 sum of the patch, used to verify the patch (only required for URL patches)\n archive_sha256: sha256 sum of the *archive*, if the patch is compressed (only required for\n compressed URL patches)\n \"\"\"\n\n return partial(\n _execute_patch,\n when=when,\n url_or_filename=url_or_filename,\n level=level,\n working_dir=working_dir,\n reverse=reverse,\n sha256=sha256,\n archive_sha256=archive_sha256,\n )\n\n\ndef _execute_patch(\n pkg_or_dep: Union[PackageType, Dependency],\n url_or_filename: str,\n level: int = 1,\n when: WhenType = None,\n working_dir: str = \".\",\n reverse: bool = False,\n sha256: Optional[str] = None,\n archive_sha256: Optional[str] = None,\n) -> None:\n pkg = pkg_or_dep.pkg if isinstance(pkg_or_dep, Dependency) else pkg_or_dep\n\n if hasattr(pkg, \"has_code\") and not pkg.has_code:\n raise UnsupportedPackageDirective(\n \"Patches are not allowed in {0}: package has no code.\".format(pkg.name)\n )\n\n when_spec = _make_when_spec(when)\n if not when_spec:\n return\n\n # If this spec is identical to some other, then append this\n # patch to the existing list.\n cur_patches = pkg_or_dep.patches.setdefault(when_spec, [])\n\n global _patch_order_index\n ordering_key = (pkg.name, _patch_order_index)\n _patch_order_index += 1\n\n patch: spack.patch.Patch\n if \"://\" in url_or_filename:\n if sha256 is None:\n raise ValueError(\"patch() with a url requires a sha256\")\n\n patch = spack.patch.UrlPatch(\n pkg,\n url_or_filename,\n level,\n working_dir=working_dir,\n reverse=reverse,\n ordering_key=ordering_key,\n sha256=sha256,\n archive_sha256=archive_sha256,\n )\n else:\n patch = spack.patch.FilePatch(\n pkg, url_or_filename, level, working_dir, reverse, ordering_key=ordering_key\n )\n\n cur_patches.append(patch)\n\n\ndef conditional(*values: Union[str, bool], when: Optional[WhenType] = None):\n \"\"\"Conditional values that can be used in variant declarations.\"\"\"\n # _make_when_spec returns None when the condition is statically false.\n when = _make_when_spec(when)\n return spack.variant.ConditionalVariantValues(\n spack.variant.ConditionalValue(x, when=when) for x in values\n )\n\n\n@directive(\"variants\")\ndef variant(\n name: str,\n default: Optional[Union[bool, str, Tuple[str, ...]]] = None,\n description: str = \"\",\n values: Optional[Union[collections.abc.Sequence, Callable[[Any], bool]]] = None,\n multi: Optional[bool] = None,\n validator: Optional[Callable[[str, str, Tuple[Any, ...]], None]] = None,\n when: Optional[Union[str, bool]] = None,\n sticky: bool = False,\n):\n \"\"\"Declare a variant for a package.\n\n Packager can specify a default value as well as a text description.\n\n Args:\n name: Name of the variant\n default: Default value for the variant, if not specified otherwise the default will be\n False for a boolean variant and 'nothing' for a multi-valued variant\n description: Description of the purpose of the variant\n values: Either a tuple of strings containing the allowed values, or a callable accepting\n one value and returning True if it is valid\n multi: If False only one value per spec is allowed for this variant\n validator: Optional group validator to enforce additional logic. It receives the package\n name, the variant name and a tuple of values and should raise an instance of SpackError\n if the group doesn't meet the additional constraints\n when: Optional condition on which the variant applies\n sticky: The variant should not be changed by the concretizer to find a valid concrete spec\n\n Raises:\n spack.directives_meta.DirectiveError: If arguments passed to the directive are invalid\n \"\"\"\n return partial(\n _execute_variant,\n name=name,\n default=default,\n description=description,\n values=values,\n multi=multi,\n validator=validator,\n when=when,\n sticky=sticky,\n )\n\n\ndef _format_error(msg, pkg, name):\n msg += \" @*r{{[{0}, variant '{1}']}}\"\n return spack.llnl.util.tty.color.colorize(msg.format(pkg.name, name))\n\n\ndef _execute_variant(\n pkg: PackageType,\n name: str,\n default: Optional[Union[bool, str, Tuple[str, ...]]],\n description: str,\n values: Optional[Union[collections.abc.Sequence, Callable[[Any], bool]]],\n multi: Optional[bool],\n validator: Optional[Callable[[str, str, Tuple[Any, ...]], None]],\n when: Optional[Union[str, bool]],\n sticky: bool,\n):\n\n # This validation can be removed at runtime and enforced with an audit in Spack v1.0.\n # For now it's a warning to let people migrate faster.\n if not (\n default is None\n or type(default) in (bool, str)\n or (type(default) is tuple and all(type(x) is str for x in default))\n ):\n if isinstance(default, (list, tuple)):\n did_you_mean = f\"default={','.join(str(x) for x in default)!r}\"\n else:\n did_you_mean = f\"default={str(default)!r}\"\n warnings.warn(\n f\"default value for variant '{name}' is not a boolean or string: default={default!r}. \"\n f\"Did you mean {did_you_mean}?\",\n stacklevel=3,\n category=spack.error.SpackAPIWarning,\n )\n\n if name in spack.variant.RESERVED_NAMES:\n raise DirectiveError(_format_error(f\"The name '{name}' is reserved by Spack\", pkg, name))\n\n # Ensure we have a sequence of allowed variant values, or a\n # predicate for it.\n if values is None:\n if (\n default in (True, False)\n or type(default) is str\n and default.upper() in (\"TRUE\", \"FALSE\")\n ):\n values = (True, False)\n else:\n values = lambda x: True\n\n # The object defining variant values might supply its own defaults for\n # all the other arguments. Ensure we have no conflicting definitions\n # in place.\n for argument in (\"default\", \"multi\", \"validator\"):\n # TODO: we can consider treating 'default' differently from other\n # TODO: attributes and let a packager decide whether to use the fluent\n # TODO: interface or the directive argument\n if hasattr(values, argument) and locals()[argument] is not None:\n raise DirectiveError(\n _format_error(\n f\"Remove specification of {argument} argument: it is handled \"\n \"by an attribute of the 'values' argument\",\n pkg,\n name,\n )\n )\n\n # Allow for the object defining the allowed values to supply its own\n # default value and group validator, say if it supports multiple values.\n default = getattr(values, \"default\", default)\n validator = getattr(values, \"validator\", validator)\n multi = getattr(values, \"multi\", bool(multi))\n\n # Here we sanitize against a default value being either None\n # or the empty string, as the former indicates that a default\n # was not set while the latter will make the variant unparsable\n # from the command line\n if isinstance(default, tuple):\n default = \",\".join(default)\n\n if default is None or default == \"\":\n if default is None:\n msg = \"either a default was not explicitly set, or 'None' was used\"\n else:\n msg = \"the default cannot be an empty string\"\n raise DirectiveError(_format_error(msg, pkg, name))\n\n description = str(description).strip()\n when_spec = _make_when_spec(when)\n\n if not re.match(spack.spec.IDENTIFIER_RE, name):\n raise DirectiveError(\"variant\", f\"Invalid variant name in {pkg.name}: '{name}'\")\n\n # variants are stored by condition then by name (so only the last variant of a\n # given name takes precedence *per condition*).\n # NOTE: variant defaults and values can conflict if when conditions overlap.\n variants_by_name = pkg.variants.setdefault(when_spec, {}) # type: ignore[arg-type]\n variants_by_name[name] = spack.variant.Variant(\n name=name,\n default=default,\n description=description,\n values=values,\n multi=multi,\n validator=validator,\n sticky=sticky,\n precedence=pkg.num_variant_definitions(),\n )\n\n\n@directive(\"resources\")\ndef resource(\n *,\n name: Optional[str] = None,\n destination: str = \"\",\n placement: Optional[str] = None,\n when: WhenType = None,\n # additional kwargs are as for `version()`\n **kwargs,\n):\n \"\"\"Declare an external resource to be fetched and staged when building the package.\n Based on the keywords present in the dictionary the appropriate FetchStrategy will\n be used for the resource. Resources are fetched and staged in their own folder\n inside spack stage area, and then moved into the stage area of the package that\n needs them.\n\n Keyword Arguments:\n name: name for the resource\n when: condition defining when the resource is needed\n destination: path, relative to the package stage area, to which resource should be moved\n placement: optionally rename the expanded resource inside the destination directory\n\n \"\"\"\n\n return partial(\n _execute_resource,\n name=name,\n destination=destination,\n placement=placement,\n when=when,\n kwargs=kwargs,\n )\n\n\ndef _execute_resource(\n pkg: PackageType,\n name: Optional[str],\n destination: str,\n placement: Optional[str],\n when: WhenType,\n # additional kwargs are as for `version()`\n kwargs: dict,\n):\n when_spec = _make_when_spec(when)\n if not when_spec:\n return\n\n # Check if the path is relative\n if os.path.isabs(destination):\n msg = \"The destination keyword of a resource directive can't be an absolute path.\\n\"\n msg += f\"\\tdestination : '{destination}\\n'\"\n raise RuntimeError(msg)\n\n # Check if the path falls within the main package stage area\n test_path = \"stage_folder_root\"\n\n # Normalized absolute path\n normalized_destination = os.path.normpath(os.path.join(test_path, destination))\n\n if test_path not in normalized_destination:\n msg = \"Destination of a resource must be within the package stage directory.\\n\"\n msg += f\"\\tdestination : '{destination}'\\n\"\n raise RuntimeError(msg)\n\n resources = pkg.resources.setdefault(when_spec, [])\n resources.append(\n Resource(name, spack.fetch_strategy.from_kwargs(**kwargs), destination, placement)\n )\n\n\ndef build_system(*values, **kwargs):\n \"\"\"Define the build system used by the package. This defines the ``build_system`` variant.\n\n Example::\n\n build_system(\"cmake\", \"autotools\", \"meson\", default=\"cmake\")\n \"\"\"\n default = kwargs.get(\"default\", None) or values[0]\n return variant(\n \"build_system\",\n values=tuple(values),\n description=\"Build systems supported by the package\",\n default=default,\n multi=False,\n )\n\n\n@directive(dicts=())\ndef maintainers(*names: str):\n \"\"\"Declare the maintainers of a package.\n\n Args:\n names: GitHub username for the maintainer\n \"\"\"\n return partial(_execute_maintainer, names=names)\n\n\ndef _execute_maintainer(pkg: PackageType, names: Tuple[str, ...]):\n maintainers = set(pkg.maintainers)\n maintainers.update(names)\n pkg.maintainers = sorted(maintainers)\n\n\n@directive(\"licenses\")\ndef license(\n license_identifier: str,\n checked_by: Optional[Union[str, List[str]]] = None,\n when: Optional[Union[str, bool]] = None,\n):\n \"\"\"Declare the license(s) the software is distributed under.\n\n Args:\n license_identifiers: SPDX identifier specifying the license(s) the software\n is distributed under.\n checked_by: string or list of strings indicating which github user checked the\n license (if any).\n when: A spec specifying when the license applies.\n \"\"\"\n\n return partial(_execute_license, license_identifier=license_identifier, when=when)\n\n\ndef _execute_license(pkg: PackageType, license_identifier: str, when: Optional[Union[str, bool]]):\n # If when is not specified the license always holds\n when_spec = _make_when_spec(when)\n if not when_spec:\n return\n\n for other_when_spec in pkg.licenses:\n if when_spec.intersects(other_when_spec):\n when_message = \"\"\n if when_spec != EMPTY_SPEC:\n when_message = f\"when {when_spec}\"\n other_when_message = \"\"\n if other_when_spec != EMPTY_SPEC:\n other_when_message = f\"when {other_when_spec}\"\n err_msg = (\n f\"{pkg.name} is specified as being licensed as {license_identifier} \"\n f\"{when_message}, but it is also specified as being licensed under \"\n f\"{pkg.licenses[other_when_spec]} {other_when_message}, which conflict.\"\n )\n raise OverlappingLicenseError(err_msg)\n\n pkg.licenses[when_spec] = license_identifier\n\n\n@directive(\"requirements\")\ndef requires(\n *requirement_specs: str,\n policy: str = \"one_of\",\n when: Optional[str] = None,\n msg: Optional[str] = None,\n):\n \"\"\"Declare that a spec must be satisfied for a package.\n\n For instance, a package whose Fortran code can only be compiled with GCC can declare::\n\n requires(\"%fortran=gcc\")\n\n A package that requires Apple-Clang on Darwin can declare instead::\n\n requires(\"%apple-clang\", when=\"platform=darwin\", msg=\"Apple Clang is required on Darwin\")\n\n Args:\n requirement_specs: spec expressing the requirement\n policy: either ``\"one_of\"`` or ``\"any_of\"``. If ``\"one_of\"``, exactly one of the\n requirements must be satisfied. If ``\"any_of\"``, at least one of the requirements must\n be satisfied. Defaults to ``\"one_of\"``.\n when: optional constraint that triggers the requirement. If None the requirement\n is applied unconditionally.\n msg: optional user defined message\n \"\"\"\n\n return partial(\n _execute_requires, requirement_specs=requirement_specs, policy=policy, when=when, msg=msg\n )\n\n\ndef _execute_requires(\n pkg: PackageType,\n requirement_specs: Tuple[str, ...],\n policy: str,\n when: Optional[str],\n msg: Optional[str],\n):\n if policy not in (\"one_of\", \"any_of\"):\n err_msg = (\n f\"the 'policy' argument of the 'requires' directive in {pkg.name} is set \"\n f\"to a wrong value (only 'one_of' or 'any_of' are allowed)\"\n )\n raise DirectiveError(err_msg)\n\n when_spec = _make_when_spec(when)\n if not when_spec:\n return\n\n # Save in a list the requirements and the associated custom messages\n requirement_list = pkg.requirements.setdefault(when_spec, [])\n msg_with_name = f\"{pkg.name}: {msg}\" if msg is not None else msg\n requirements = tuple(get_spec(s) for s in requirement_specs)\n requirement_list.append((requirements, policy, msg_with_name))\n\n\nclass DependencyError(DirectiveError):\n \"\"\"This is raised when a dependency specification is invalid.\"\"\"\n\n\nclass CircularReferenceError(DependencyError):\n \"\"\"This is raised when something depends on itself.\"\"\"\n\n\nclass DependencyPatchError(DirectiveError):\n \"\"\"Raised for errors with patching dependencies.\"\"\"\n\n\nclass UnsupportedPackageDirective(DirectiveError):\n \"\"\"Raised when an invalid or unsupported package directive is specified.\"\"\"\n\n\nclass OverlappingLicenseError(DirectiveError):\n \"\"\"Raised when two licenses are declared that apply on overlapping specs.\"\"\"\n" }, { "path": "lib/spack/spack/directives_meta.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport collections\nimport functools\nfrom typing import Any, Callable, Dict, List, Set, Tuple, Type, TypeVar, Union\n\nfrom spack.vendor.typing_extensions import ParamSpec\n\nimport spack.error\nimport spack.repo\nimport spack.spec\nfrom spack.llnl.util.lang import dedupe\n\nP = ParamSpec(\"P\")\nR = TypeVar(\"R\")\n\n#: Names of possible directives. This list is mostly populated using the @directive decorator.\n#: Some directives leverage others and in that case are not automatically added.\ndirective_names = [\"build_system\"]\n\nSPEC_CACHE: Dict[str, spack.spec.Spec] = {}\n\n\ndef get_spec(spec_str: str) -> spack.spec.Spec:\n \"\"\"Get a spec from the cache, or create it if not present.\"\"\"\n if spec_str not in SPEC_CACHE:\n SPEC_CACHE[spec_str] = spack.spec._ImmutableSpec(spec_str)\n return SPEC_CACHE[spec_str]\n\n\nclass DirectiveMeta(type):\n \"\"\"Flushes the directives that were temporarily stored in the staging\n area into the package.\n \"\"\"\n\n #: Registry of {directive_name: [list_of_dicts_it_modifies]} populated by @directive\n _directive_to_dicts: Dict[str, Tuple[str, ...]] = {}\n #: Inverted index of {dict_name: [list_of_directives_modifying_it]}\n _dict_to_directives: Dict[str, List[str]] = collections.defaultdict(list)\n #: Maps dictionary name to its descriptor instance\n _descriptor_cache: Dict[str, \"DirectiveDictDescriptor\"] = {}\n #: Set of all known directive dictionary names from `@directive(dicts=...)`\n _directive_dict_names: Set[str] = set()\n #: Lists of directives to be executed for the class being defined, grouped by directive\n #: function name (e.g. \"depends_on\", \"version\", etc.)\n _directives_to_be_executed: Dict[str, List[Callable]] = collections.defaultdict(list)\n #: Stack of when constraints from `with when(...)` context managers\n _when_constraints_stack: List[str] = []\n #: Stack of default args from `with default_args(...)` context managers\n _default_args_stack: List[dict] = []\n #: This property is set *automatically* during class definition as directives are invoked,\n #: if any ``depends_on`` or ``extends`` calls include patches for dependencies. This flag can\n #: be used as an optimization to detect whether a package provides patches for dependencies,\n #: without triggering the expensive deferred execution of those directives (without populating\n #: the ``dependencies`` dictionary).\n _patches_dependencies: bool = False\n\n def __new__(\n cls: Type[\"DirectiveMeta\"], name: str, bases: tuple, attr_dict: dict\n ) -> \"DirectiveMeta\":\n attr_dict[\"_patches_dependencies\"] = DirectiveMeta._patches_dependencies\n # Initialize the attribute containing the list of directives to be executed. Here we go\n # reversed because we want to execute commands in the order they were defined, following\n # the MRO.\n merged: Dict[str, List[Callable]] = {}\n sources = [getattr(b, \"_directives_to_be_executed\", None) or {} for b in reversed(bases)]\n for source in sources:\n for key, directive_list in source.items():\n merged.setdefault(key, []).extend(directive_list)\n\n merged = {key: list(dedupe(directive_list)) for key, directive_list in merged.items()}\n\n # Add current class's directives (no deduplication needed here)\n for key, directive_list in DirectiveMeta._directives_to_be_executed.items():\n merged.setdefault(key, []).extend(directive_list)\n\n attr_dict[\"_directives_to_be_executed\"] = merged\n\n DirectiveMeta._directives_to_be_executed.clear()\n DirectiveMeta._patches_dependencies = False\n\n # Add descriptors for all known directive dictionaries\n for dict_name in DirectiveMeta._directive_dict_names:\n # Where the actual data will be stored\n attr_dict[f\"_{dict_name}\"] = None\n # Descriptor to lazily initialize and populate the dictionary\n attr_dict[dict_name] = DirectiveMeta._get_descriptor(dict_name)\n\n return super(DirectiveMeta, cls).__new__(cls, name, bases, attr_dict)\n\n def __init__(cls: \"DirectiveMeta\", name: str, bases: tuple, attr_dict: dict):\n if spack.repo.is_package_module(cls.__module__):\n # Historically, maintainers was not a directive. They were simply set as class\n # attributes `maintainers = [\"alice\", \"bob\"]`. Therefore, we execute these directives\n # eagerly.\n for directive in cls._directives_to_be_executed.get(\"maintainers\", ()):\n directive(cls)\n super(DirectiveMeta, cls).__init__(name, bases, attr_dict)\n\n @staticmethod\n def register_directive(name: str, dicts: Tuple[str, ...]) -> None:\n \"\"\"Called by @directive to register relationships.\"\"\"\n DirectiveMeta._directive_to_dicts[name] = dicts\n for d in dicts:\n DirectiveMeta._dict_to_directives[d].append(name)\n\n @staticmethod\n def _get_descriptor(name: str) -> \"DirectiveDictDescriptor\":\n \"\"\"Returns a singleton descriptor for the given dictionary name.\"\"\"\n if name not in DirectiveMeta._descriptor_cache:\n DirectiveMeta._descriptor_cache[name] = DirectiveDictDescriptor(name)\n return DirectiveMeta._descriptor_cache[name]\n\n @staticmethod\n def push_when_constraint(when_spec: str) -> None:\n \"\"\"Add a spec to the context constraints.\"\"\"\n DirectiveMeta._when_constraints_stack.append(when_spec)\n\n @staticmethod\n def pop_when_constraint() -> str:\n \"\"\"Pop the last constraint from the context\"\"\"\n return DirectiveMeta._when_constraints_stack.pop()\n\n @staticmethod\n def push_default_args(default_args: Dict[str, Any]) -> None:\n \"\"\"Push default arguments\"\"\"\n DirectiveMeta._default_args_stack.append(default_args)\n\n @staticmethod\n def pop_default_args() -> dict:\n \"\"\"Pop default arguments\"\"\"\n return DirectiveMeta._default_args_stack.pop()\n\n @staticmethod\n def _remove_kwarg_value_directives_from_queue(value) -> None:\n \"\"\"Remove directives found in a kwarg value from the execution queue.\"\"\"\n # Certain keyword argument values of directives may themselves be (lists of) directives. An\n # example of this is ``depends_on(..., patches=[patch(...), ...])``. In that case, we\n # should not execute those directives as part of the current package, but let the called\n # directive handle them. This function removes such directives from the execution queue.\n if isinstance(value, (list, tuple)):\n for item in value:\n DirectiveMeta._remove_kwarg_value_directives_from_queue(item)\n elif callable(value): # directives are always callable\n # Remove directives args from the exec queue\n for lst in DirectiveMeta._directives_to_be_executed.values():\n for directive in lst:\n if value is directive:\n lst.remove(directive) # iterations ends, so mutation is fine\n break\n\n @staticmethod\n def _get_execution_plan(target_dict: str) -> Tuple[List[str], List[str]]:\n \"\"\"Calculates the closure of dicts and directives needed to populate target_dict.\"\"\"\n dicts_involved = {target_dict}\n directives_involved = set()\n stack = [target_dict]\n\n while stack:\n current_dict = stack.pop()\n\n for directive_name in DirectiveMeta._dict_to_directives.get(current_dict, ()):\n if directive_name in directives_involved:\n continue\n\n directives_involved.add(directive_name)\n\n for other_dict in DirectiveMeta._directive_to_dicts[directive_name]:\n if other_dict not in dicts_involved:\n dicts_involved.add(other_dict)\n stack.append(other_dict)\n\n return sorted(dicts_involved), sorted(directives_involved)\n\n\nclass DirectiveDictDescriptor:\n \"\"\"A descriptor that lazily executes directives on first access.\"\"\"\n\n def __init__(self, name: str):\n self.name = name\n self.private_name = f\"_{name}\"\n self.dicts_to_init, self.directives_to_run = DirectiveMeta._get_execution_plan(name)\n\n def __get__(self, obj, objtype=None):\n val = getattr(objtype, self.private_name)\n if val is not None:\n return val\n\n # The None value is a sentinel for \"not yet initialized\".\n for dictionary in self.dicts_to_init:\n if getattr(objtype, f\"_{dictionary}\") is None:\n setattr(objtype, f\"_{dictionary}\", {})\n\n # Populate these dictionaries by running all directives that modify them\n for directive_name in self.directives_to_run:\n directives = objtype._directives_to_be_executed.get(directive_name)\n if directives:\n for directive in directives:\n directive(objtype)\n\n return getattr(objtype, self.private_name)\n\n\nclass directive:\n def __init__(\n self,\n dicts: Union[Tuple[str, ...], str] = (),\n supports_when: bool = True,\n can_patch_dependencies: bool = False,\n ) -> None:\n \"\"\"Decorator for Spack directives.\n\n Spack directives allow you to modify a package while it is being defined, e.g. to add\n version or dependency information. Directives are one of the key pieces of Spack's\n package \"language\", which is embedded in python.\n\n Here's an example directive::\n\n @directive(dicts=\"versions\")\n def version(pkg, ...):\n ...\n\n This directive allows you write::\n\n class Foo(Package):\n version(...)\n\n The ``@directive`` decorator handles a couple things for you:\n\n 1. Adds the class scope (pkg) as an initial parameter when called, like a class method\n would. This allows you to modify a package from within a directive, while the package is\n still being defined.\n\n 2. It automatically adds a dictionary called ``versions`` to the package so that you can\n refer to pkg.versions.\n\n Arguments:\n dicts: A tuple of names of dictionaries to add to the package class if they don't\n already exist.\n supports_when: If True, the directive can be used within a ``with when(...)`` context\n manager. (To be removed when all directives support ``when=`` arguments.)\n can_patch_dependencies: If True, the directive can patch dependencies. This is used to\n identify nested directives so they can be removed from the execution queue, and to\n mark the package as patching dependencies.\n \"\"\"\n\n if isinstance(dicts, str):\n dicts = (dicts,)\n\n # Add the dictionary names if not already there\n DirectiveMeta._directive_dict_names.update(dicts)\n\n self.supports_when = supports_when\n self.can_patch_dependencies = can_patch_dependencies\n self.dicts = tuple(dicts)\n\n def __call__(self, decorated_function: Callable[P, R]) -> Callable[P, R]:\n directive_names.append(decorated_function.__name__)\n DirectiveMeta.register_directive(decorated_function.__name__, self.dicts)\n\n @functools.wraps(decorated_function)\n def _wrapper(*args, **_kwargs):\n # First merge default args with kwargs\n if DirectiveMeta._default_args_stack:\n kwargs = {}\n for default_args in DirectiveMeta._default_args_stack:\n kwargs.update(default_args)\n kwargs.update(_kwargs)\n else:\n kwargs = _kwargs\n\n # Inject when arguments from the `with when(...)` stack.\n if DirectiveMeta._when_constraints_stack:\n if not self.supports_when:\n raise DirectiveError(\n f'directive \"{decorated_function.__name__}\" cannot be used within a '\n '\"when\" context since it does not support a \"when=\" argument'\n )\n if \"when\" in kwargs:\n kwargs[\"when\"] = (*DirectiveMeta._when_constraints_stack, kwargs[\"when\"])\n else:\n kwargs[\"when\"] = tuple(DirectiveMeta._when_constraints_stack)\n\n # Remove directives passed as arguments, so they are not executed as part of this\n # class's directive execution, but handled by the called directive instead\n if self.can_patch_dependencies and \"patches\" in kwargs:\n DirectiveMeta._remove_kwarg_value_directives_from_queue(kwargs[\"patches\"])\n DirectiveMeta._patches_dependencies = True\n\n result = decorated_function(*args, **kwargs)\n\n DirectiveMeta._directives_to_be_executed[decorated_function.__name__].append(result)\n\n # wrapped function returns same result as original so that we can nest directives\n return result\n\n return _wrapper\n\n\nclass DirectiveError(spack.error.SpackError):\n \"\"\"This is raised when something is wrong with a package directive.\"\"\"\n" }, { "path": "lib/spack/spack/directory_layout.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport errno\nimport os\nimport re\nimport shutil\nimport sys\nfrom pathlib import Path\nfrom typing import Dict, List, Optional, Tuple\n\nimport spack.config\nimport spack.hash_types as ht\nimport spack.llnl.util.filesystem as fs\nimport spack.projections\nimport spack.spec\nimport spack.util.spack_json as sjson\nfrom spack.error import SpackError\nfrom spack.llnl.util.filesystem import readlink\n\ndefault_projections = {\n \"all\": \"{architecture.platform}-{architecture.target}/{name}-{version}-{hash}\"\n}\n\n\ndef _check_concrete(spec: \"spack.spec.Spec\") -> None:\n \"\"\"If the spec is not concrete, raise a ValueError\"\"\"\n if not spec.concrete:\n raise ValueError(\"Specs passed to a DirectoryLayout must be concrete!\")\n\n\ndef _get_spec(prefix: str) -> Optional[\"spack.spec.Spec\"]:\n \"\"\"Returns a spec if the prefix contains a spec file in the .spack subdir\"\"\"\n for f in (\"spec.json\", \"spec.yaml\"):\n try:\n return spack.spec.Spec.from_specfile(os.path.join(prefix, \".spack\", f))\n except Exception:\n continue\n return None\n\n\ndef specs_from_metadata_dirs(root: str) -> List[\"spack.spec.Spec\"]:\n stack = [root]\n specs = []\n\n while stack:\n prefix = stack.pop()\n\n spec = _get_spec(prefix)\n\n if spec:\n spec.set_prefix(prefix)\n specs.append(spec)\n continue\n\n try:\n scandir = os.scandir(prefix)\n except OSError:\n continue\n\n with scandir as entries:\n for entry in entries:\n if entry.is_dir(follow_symlinks=False):\n stack.append(entry.path)\n return specs\n\n\nclass DirectoryLayout:\n \"\"\"A directory layout is used to associate unique paths with specs. Different installations are\n going to want different layouts for their install, and they can use this to customize the\n nesting structure of spack installs. The default layout is:\n\n * /\n\n * /\n\n * -/\n\n * --\n\n The installation directory projections can be modified with the projections argument.\"\"\"\n\n def __init__(\n self,\n root: str,\n *,\n projections: Optional[Dict[str, str]] = None,\n hash_length: Optional[int] = None,\n ) -> None:\n self.root = root\n projections = projections or default_projections\n self.projections = {key: projection.lower() for key, projection in projections.items()}\n\n # apply hash length as appropriate\n self.hash_length = hash_length\n if self.hash_length is not None:\n for when_spec, projection in self.projections.items():\n if \"{hash}\" not in projection:\n raise InvalidDirectoryLayoutParametersError(\n \"Conflicting options for installation layout hash length\"\n if \"{hash\" in projection\n else \"Cannot specify hash length when the hash is not part of all \"\n \"install_tree projections\"\n )\n self.projections[when_spec] = projection.replace(\n \"{hash}\", \"{hash:%d}\" % self.hash_length\n )\n\n # If any of these paths change, downstream databases may not be able to\n # locate files in older upstream databases\n self.metadata_dir = \".spack\"\n self.deprecated_dir = \"deprecated\"\n self.spec_file_name = \"spec.json\"\n # Use for checking yaml and deprecated types\n self._spec_file_name_yaml = \"spec.yaml\"\n self.extension_file_name = \"extensions.yaml\"\n self.packages_dir = \"repos\" # archive of package.py files\n self.manifest_file_name = \"install_manifest.json\"\n\n @property\n def hidden_file_regexes(self) -> Tuple[str]:\n return (\"^{0}$\".format(re.escape(self.metadata_dir)),)\n\n def relative_path_for_spec(self, spec: \"spack.spec.Spec\") -> str:\n _check_concrete(spec)\n\n projection = spack.projections.get_projection(self.projections, spec)\n path = spec.format_path(projection)\n return str(Path(path))\n\n def write_spec(self, spec: \"spack.spec.Spec\", path: str) -> None:\n \"\"\"Write a spec out to a file.\"\"\"\n _check_concrete(spec)\n with open(path, \"w\", encoding=\"utf-8\") as f:\n # The hash of the projection is the DAG hash which contains\n # the full provenance, so it's available if we want it later\n spec.to_json(f, hash=ht.dag_hash)\n\n def write_host_environment(self, spec: \"spack.spec.Spec\") -> None:\n \"\"\"The host environment is a json file with os, kernel, and spack\n versioning. We use it in the case that an analysis later needs to\n easily access this information.\n \"\"\"\n env_file = self.env_metadata_path(spec)\n environ = spack.spec.get_host_environment_metadata()\n with open(env_file, \"w\", encoding=\"utf-8\") as fd:\n sjson.dump(environ, fd)\n\n def read_spec(self, path: str) -> \"spack.spec.Spec\":\n \"\"\"Read the contents of a file and parse them as a spec\"\"\"\n try:\n with open(path, encoding=\"utf-8\") as f:\n extension = os.path.splitext(path)[-1].lower()\n if extension == \".json\":\n spec = spack.spec.Spec.from_json(f)\n elif extension == \".yaml\":\n # Too late for conversion; spec_file_path() already called.\n spec = spack.spec.Spec.from_yaml(f)\n else:\n raise SpecReadError(f\"Did not recognize spec file extension: {extension}\")\n except Exception as e:\n if spack.config.get(\"config:debug\"):\n raise\n raise SpecReadError(f\"Unable to read file: {path}\", f\"Cause: {e}\")\n\n # Specs read from actual installations are always concrete\n spec._mark_concrete()\n return spec\n\n def spec_file_path(self, spec: \"spack.spec.Spec\") -> str:\n \"\"\"Gets full path to spec file\"\"\"\n _check_concrete(spec)\n yaml_path = os.path.join(self.metadata_path(spec), self._spec_file_name_yaml)\n json_path = os.path.join(self.metadata_path(spec), self.spec_file_name)\n return yaml_path if os.path.exists(yaml_path) else json_path\n\n def deprecated_file_path(\n self,\n deprecated_spec: \"spack.spec.Spec\",\n deprecator_spec: Optional[\"spack.spec.Spec\"] = None,\n ) -> str:\n \"\"\"Gets full path to spec file for deprecated spec\n\n If the deprecator_spec is provided, use that. Otherwise, assume\n deprecated_spec is already deprecated and its prefix links to the\n prefix of its deprecator.\"\"\"\n _check_concrete(deprecated_spec)\n if deprecator_spec:\n _check_concrete(deprecator_spec)\n\n # If deprecator spec is None, assume deprecated_spec already deprecated\n # and use its link to find the file.\n base_dir = (\n self.path_for_spec(deprecator_spec)\n if deprecator_spec\n else readlink(deprecated_spec.prefix)\n )\n\n yaml_path = os.path.join(\n base_dir,\n self.metadata_dir,\n self.deprecated_dir,\n deprecated_spec.dag_hash() + \"_\" + self._spec_file_name_yaml,\n )\n\n json_path = os.path.join(\n base_dir,\n self.metadata_dir,\n self.deprecated_dir,\n deprecated_spec.dag_hash() + \"_\" + self.spec_file_name,\n )\n\n return yaml_path if os.path.exists(yaml_path) else json_path\n\n def metadata_path(self, spec: \"spack.spec.Spec\") -> str:\n return os.path.join(spec.prefix, self.metadata_dir)\n\n def env_metadata_path(self, spec: \"spack.spec.Spec\") -> str:\n return os.path.join(self.metadata_path(spec), \"install_environment.json\")\n\n def build_packages_path(self, spec: \"spack.spec.Spec\") -> str:\n return os.path.join(self.metadata_path(spec), self.packages_dir)\n\n def create_install_directory(self, spec: \"spack.spec.Spec\") -> None:\n _check_concrete(spec)\n\n # Create install directory with properly configured permissions\n # Cannot import at top of file\n from spack.package_prefs import get_package_dir_permissions, get_package_group\n\n # Each package folder can have its own specific permissions, while\n # intermediate folders (arch/compiler) are set with access permissions\n # equivalent to the root permissions of the layout.\n group = get_package_group(spec)\n perms = get_package_dir_permissions(spec)\n\n fs.mkdirp(spec.prefix, mode=perms, group=group, default_perms=\"parents\")\n fs.mkdirp(self.metadata_path(spec), mode=perms, group=group) # in prefix\n\n self.write_spec(spec, self.spec_file_path(spec))\n\n def ensure_installed(self, spec: \"spack.spec.Spec\") -> None:\n \"\"\"\n Throws InconsistentInstallDirectoryError if:\n 1. spec prefix does not exist\n 2. spec prefix does not contain a spec file, or\n 3. We read a spec with the wrong DAG hash out of an existing install directory.\n \"\"\"\n _check_concrete(spec)\n path = self.path_for_spec(spec)\n spec_file_path = self.spec_file_path(spec)\n\n if not os.path.isdir(path):\n raise InconsistentInstallDirectoryError(\n \"Install prefix {0} does not exist.\".format(path)\n )\n\n if not os.path.isfile(spec_file_path):\n raise InconsistentInstallDirectoryError(\n \"Install prefix exists but contains no spec.json:\", \" \" + path\n )\n\n installed_spec = self.read_spec(spec_file_path)\n if installed_spec.dag_hash() != spec.dag_hash():\n raise InconsistentInstallDirectoryError(\n \"Spec file in %s does not match hash!\" % spec_file_path\n )\n\n def path_for_spec(self, spec: \"spack.spec.Spec\") -> str:\n \"\"\"Return absolute path from the root to a directory for the spec.\"\"\"\n _check_concrete(spec)\n\n if spec.external:\n return spec.external_path\n\n path = self.relative_path_for_spec(spec)\n assert not path.startswith(self.root)\n return os.path.join(self.root, path)\n\n def remove_install_directory(self, spec: \"spack.spec.Spec\", deprecated: bool = False) -> None:\n \"\"\"Removes a prefix and any empty parent directories from the root.\n Raised RemoveFailedError if something goes wrong.\n \"\"\"\n path = self.path_for_spec(spec)\n assert path.startswith(self.root), (\n \"Attempted to remove dir outside Spack's install tree. PATH: {path}, ROOT: {self.root}\"\n )\n\n if deprecated:\n if os.path.exists(path):\n try:\n metapath = self.deprecated_file_path(spec)\n os.unlink(path)\n os.remove(metapath)\n except OSError as e:\n raise RemoveFailedError(spec, path, e) from e\n elif os.path.exists(path):\n try:\n if sys.platform == \"win32\":\n # Windows readonly files cannot be removed by Python\n # directly, change permissions before attempting to remove\n shutil.rmtree(\n path,\n ignore_errors=False,\n onerror=fs.readonly_file_handler(ignore_errors=False),\n )\n else:\n shutil.rmtree(path)\n except OSError as e:\n raise RemoveFailedError(spec, path, e) from e\n\n path = os.path.dirname(path)\n while path != self.root:\n if os.path.isdir(path):\n try:\n os.rmdir(path)\n except OSError as e:\n if e.errno == errno.ENOENT:\n # already deleted, continue with parent\n pass\n elif e.errno == errno.ENOTEMPTY:\n # directory wasn't empty, done\n return\n else:\n raise e\n path = os.path.dirname(path)\n\n def all_specs(self) -> List[\"spack.spec.Spec\"]:\n \"\"\"Returns a list of all specs detected in self.root, detected by ``.spack`` directories.\n Their prefix is set to the directory containing the ``.spack`` directory. Note that these\n specs may follow a different layout than the current layout if it was changed after\n installation.\"\"\"\n return specs_from_metadata_dirs(self.root)\n\n def deprecated_for(\n self, specs: List[\"spack.spec.Spec\"]\n ) -> List[Tuple[\"spack.spec.Spec\", \"spack.spec.Spec\"]]:\n \"\"\"Returns a list of tuples of specs (new, old) where new is deprecated for old\"\"\"\n spec_with_deprecated = []\n for spec in specs:\n try:\n deprecated = os.scandir(\n os.path.join(str(spec.prefix), self.metadata_dir, self.deprecated_dir)\n )\n except OSError:\n continue\n\n with deprecated as entries:\n for entry in entries:\n try:\n deprecated_spec = spack.spec.Spec.from_specfile(entry.path)\n spec_with_deprecated.append((spec, deprecated_spec))\n except Exception:\n continue\n return spec_with_deprecated\n\n\nclass DirectoryLayoutError(SpackError):\n \"\"\"Superclass for directory layout errors.\"\"\"\n\n def __init__(self, message, long_msg=None):\n super().__init__(message, long_msg)\n\n\nclass RemoveFailedError(DirectoryLayoutError):\n \"\"\"Raised when a DirectoryLayout cannot remove an install prefix.\"\"\"\n\n def __init__(self, installed_spec, prefix, error):\n super().__init__(\n \"Could not remove prefix %s for %s : %s\" % (prefix, installed_spec.short_spec, error)\n )\n self.cause = error\n\n\nclass InconsistentInstallDirectoryError(DirectoryLayoutError):\n \"\"\"Raised when a package seems to be installed to the wrong place.\"\"\"\n\n def __init__(self, message, long_msg=None):\n super().__init__(message, long_msg)\n\n\nclass SpecReadError(DirectoryLayoutError):\n \"\"\"Raised when directory layout can't read a spec.\"\"\"\n\n\nclass InvalidDirectoryLayoutParametersError(DirectoryLayoutError):\n \"\"\"Raised when a invalid directory layout parameters are supplied\"\"\"\n\n def __init__(self, message, long_msg=None):\n super().__init__(message, long_msg)\n\n\nclass InvalidExtensionSpecError(DirectoryLayoutError):\n \"\"\"Raised when an extension file has a bad spec in it.\"\"\"\n\n\nclass ExtensionAlreadyInstalledError(DirectoryLayoutError):\n \"\"\"Raised when an extension is added to a package that already has it.\"\"\"\n\n def __init__(self, spec, ext_spec):\n super().__init__(\"%s is already installed in %s\" % (ext_spec.short_spec, spec.short_spec))\n\n\nclass ExtensionConflictError(DirectoryLayoutError):\n \"\"\"Raised when an extension is added to a package that already has it.\"\"\"\n\n def __init__(self, spec, ext_spec, conflict):\n super().__init__(\n \"%s cannot be installed in %s because it conflicts with %s\"\n % (ext_spec.short_spec, spec.short_spec, conflict.short_spec)\n )\n" }, { "path": "lib/spack/spack/enums.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Enumerations used throughout Spack\"\"\"\n\nimport enum\n\n\nclass InstallRecordStatus(enum.Flag):\n \"\"\"Enum flag to facilitate querying status from the DB\"\"\"\n\n INSTALLED = enum.auto()\n DEPRECATED = enum.auto()\n MISSING = enum.auto()\n ANY = INSTALLED | DEPRECATED | MISSING\n\n\nclass ConfigScopePriority(enum.IntEnum):\n \"\"\"Priorities of the different kind of config scopes used by Spack\"\"\"\n\n DEFAULTS = 0\n CONFIG_FILES = 1\n ENVIRONMENT = 2\n CUSTOM = 3\n COMMAND_LINE = 4\n # Topmost scope reserved for internal use\n ENVIRONMENT_SPEC_GROUPS = 5\n\n\nclass PropagationPolicy(enum.Enum):\n \"\"\"Enum to specify the behavior of a propagated dependency\"\"\"\n\n NONE = enum.auto()\n PREFERENCE = enum.auto()\n" }, { "path": "lib/spack/spack/environment/__init__.py", "content": "# -*- coding: utf-8 -*-\n# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"This package implements Spack environments.\n\n.. _lockfile-format:\n\n``spack.lock`` format\n=====================\n\nSpack environments have existed since Spack ``v0.12.0``, and there have been different\n``spack.lock`` formats since then. The formats are documented here.\n\nThe high-level format of a Spack lockfile hasn't changed much between versions, but the\ncontents have. Lockfiles are JSON-formatted and their top-level sections are:\n\n1. ``_meta`` (object): this contains details about the file format, including:\n\n * ``file-type``: always ``\"spack-lockfile\"``\n * ``lockfile-version``: an integer representing the lockfile format version\n * ``specfile-version``: an integer representing the spec format version (since\n ``v0.17``)\n2. ``spack`` (object): optional, this identifies information about Spack\n used to concretize the environment:\n\n * ``type``: required, identifies form Spack version took (e.g., ``git``, ``release``)\n * ``commit``: the commit if the version is from git\n * ``version``: the Spack version\n3. ``roots`` (list): an ordered list of records representing the roots of the Spack\n environment. Each has two fields:\n\n * ``hash``: a Spack spec hash uniquely identifying the concrete root spec\n * ``spec``: a string representation of the abstract spec that was concretized\n4. ``concrete_specs``: a dictionary containing the specs in the environment.\n5. ``include_concrete`` (dictionary): an optional dictionary that includes the roots\n and concrete specs from the included environments, keyed by the path to that\n environment\n\nCompatibility\n-------------\n\nNew versions of Spack can (so far) read all old lockfile formats -- they are\nbackward-compatible. Old versions cannot read new lockfile formats, and you'll need to\nupgrade Spack to use them.\n\n.. list-table:: Lockfile version compatibility across Spack versions\n :header-rows: 1\n\n * - Spack version\n - ``v1``\n - ``v2``\n - ``v3``\n - ``v4``\n - ``v5``\n - ``v6``\n - ``v7``\n * - ``v0.12:0.14``\n - ✅\n -\n -\n -\n -\n -\n -\n * - ``v0.15:0.16``\n - ✅\n - ✅\n -\n -\n -\n -\n -\n * - ``v0.17``\n - ✅\n - ✅\n - ✅\n -\n -\n -\n -\n * - ``v0.18:``\n - ✅\n - ✅\n - ✅\n - ✅\n -\n -\n -\n * - ``v0.22:v0.23``\n - ✅\n - ✅\n - ✅\n - ✅\n - ✅\n -\n -\n * - ``v1.0:1.1``\n - ✅\n - ✅\n - ✅\n - ✅\n - ✅\n - ✅\n -\n * - ``v1.2:``\n - ✅\n - ✅\n - ✅\n - ✅\n - ✅\n - ✅\n - ✅\n\nVersion 1\n---------\n\nWhen lockfiles were first created, there was only one hash in Spack: the DAG hash. This\nDAG hash (we'll call it the old DAG hash) did *not* include build dependencies -- it\nonly included transitive link and run dependencies.\n\nThe spec format at this time was keyed by name. Each spec started with a key for its\nname, whose value was a dictionary of other spec attributes. The lockfile put these\nname-keyed specs into dictionaries keyed by their DAG hash, and the spec records did not\nactually have a \"hash\" field in the lockfile -- you have to associate the hash from the\nkey with the spec record after the fact.\n\nDependencies in original lockfiles were keyed by ``\"hash\"``, i.e. the old DAG hash.\n\n.. code-block:: json\n\n {\n \"_meta\": {\n \"file-type\": \"spack-lockfile\",\n \"lockfile-version\": 1\n },\n \"roots\": [\n {\n \"hash\": \"\",\n \"spec\": \"\"\n },\n {\n \"hash\": \"\",\n \"spec\": \"\"\n }\n ],\n \"concrete_specs\": {\n \"\": {\n \"... ...\": { },\n \"dependencies\": {\n \"depname_1\": {\n \"hash\": \"\",\n \"type\": [\"build\", \"link\"]\n },\n \"depname_2\": {\n \"hash\": \"\",\n \"type\": [\"build\", \"link\"]\n }\n },\n \"hash\": \"\"\n },\n \"\": {\n \"... ...\": { },\n \"dependencies\": {\n \"depname_3\": {\n \"hash\": \"\",\n \"type\": [\"build\", \"link\"]\n },\n \"depname_4\": {\n \"hash\": \"\",\n \"type\": [\"build\", \"link\"]\n },\n },\n \"hash\": \"\"\n },\n }\n }\n\n\nVersion 2\n---------\n\nVersion 2 changes one thing: specs in the lockfile are now keyed by ``build_hash``\ninstead of the old ``dag_hash``. Specs have a ``hash`` attribute with their real DAG\nhash, so you can't go by the dictionary key anymore to identify a spec -- you have to\nread it in and look at ``\"hash\"``. Dependencies are still keyed by old DAG hash.\n\nEven though we key lockfiles by ``build_hash``, specs in Spack were still deployed with\nthe old, coarser DAG hash. This means that in v2 and v3 lockfiles (which are keyed by\nbuild hash), there may be multiple versions of the same spec with different build\ndependencies, which means they will have different build hashes but the same DAG hash.\nSpack would only have been able to actually install one of these.\n\n.. code-block:: json\n\n {\n \"_meta\": {\n \"file-type\": \"spack-lockfile\",\n \"lockfile-version\": 2\n },\n \"roots\": [\n {\n \"hash\": \"\",\n \"spec\": \"\"\n },\n {\n \"hash\": \"\",\n \"spec\": \"\"\n }\n ],\n \"concrete_specs\": {\n \"\": {\n \"... ...\": { },\n \"dependencies\": {\n \"depname_1\": {\n \"hash\": \"\",\n \"type\": [\"build\", \"link\"]\n },\n \"depname_2\": {\n \"hash\": \"\",\n \"type\": [\"build\", \"link\"]\n }\n },\n \"hash\": \"\",\n },\n \"\": {\n \"... ...\": { },\n \"dependencies\": {\n \"depname_3\": {\n \"hash\": \"\",\n \"type\": [\"build\", \"link\"]\n },\n \"depname_4\": {\n \"hash\": \"\",\n \"type\": [\"build\", \"link\"]\n }\n },\n \"hash\": \"\"\n }\n }\n }\n\n\nVersion 3\n---------\n\nVersion 3 doesn't change the top-level lockfile format, but this was when we changed the\nspecfile format. Specs in ``concrete_specs`` are now keyed by the build hash, with no\ninner dictionary keyed by their package name. The package name is in a ``name`` field\ninside each spec dictionary. The ``dependencies`` field in the specs is a list instead\nof a dictionary, and each element of the list is a record with the name, dependency\ntypes, and hash of the dependency. Instead of a key called ``hash``, dependencies are\nkeyed by ``build_hash``. Each spec still has a ``hash`` attribute.\n\nVersion 3 adds the ``specfile_version`` field to ``_meta`` and uses the new JSON spec\nformat.\n\n.. code-block:: json\n\n {\n \"_meta\": {\n \"file-type\": \"spack-lockfile\",\n \"lockfile-version\": 3,\n \"specfile-version\": 2\n },\n \"roots\": [\n {\n \"hash\": \"\",\n \"spec\": \"\"\n },\n {\n \"hash\": \"\",\n \"spec\": \"\"\n },\n ],\n \"concrete_specs\": {\n \"\": {\n \"... ...\": { },\n \"dependencies\": [\n {\n \"name\": \"depname_1\",\n \"build_hash\": \"\",\n \"type\": [\"build\", \"link\"]\n },\n {\n \"name\": \"depname_2\",\n \"build_hash\": \"\",\n \"type\": [\"build\", \"link\"]\n },\n ],\n \"hash\": \"\",\n },\n \"\": {\n \"... ...\": { },\n \"dependencies\": [\n {\n \"name\": \"depname_3\",\n \"build_hash\": \"\",\n \"type\": [\"build\", \"link\"]\n },\n {\n \"name\": \"depname_4\",\n \"build_hash\": \"\",\n \"type\": [\"build\", \"link\"]\n },\n ],\n \"hash\": \"\"\n }\n }\n }\n\n\nVersion 4\n---------\n\nVersion 4 removes build hashes and is keyed by the new DAG hash (``hash``). The ``hash``\nnow includes build dependencies and a canonical hash of the ``package.py`` file.\nDependencies are keyed by ``hash`` (DAG hash) as well. There are no more ``build_hash``\nfields in the specs, and there are no more issues with lockfiles being able to store\nmultiple specs with the same DAG hash (because the DAG hash is now finer-grained).\nAn optional ``spack`` property may be included to track version information, such as\nthe commit or version.\n\n\n.. code-block:: json\n\n {\n \"_meta\": {\n \"file-type\": \"spack-lockfile\",\n \"lockfile-version\": 4,\n \"specfile-version\": 3\n },\n \"roots\": [\n {\n \"hash\": \"\",\n \"spec\": \"\"\n },\n {\n \"hash\": \"\",\n \"spec\": \"\"\n }\n ],\n \"concrete_specs\": {\n \"\": {\n \"... ...\": { },\n \"dependencies\": [\n {\n \"name\": \"depname_1\",\n \"hash\": \"\",\n \"type\": [\"build\", \"link\"]\n },\n {\n \"name\": \"depname_2\",\n \"hash\": \"\",\n \"type\": [\"build\", \"link\"]\n }\n ],\n \"hash\": \"\",\n },\n \"\": {\n \"... ...\": { },\n \"dependencies\": [\n {\n \"name\": \"depname_3\",\n \"hash\": \"\",\n \"type\": [\"build\", \"link\"]\n },\n {\n \"name\": \"depname_4\",\n \"hash\": \"\",\n \"type\": [\"build\", \"link\"]\n }\n ],\n \"hash\": \"\"\n }\n }\n }\n\n\nVersion 5\n---------\n\nVersion 5 doesn't change the top-level lockfile format, but an optional dictionary is\nadded. The dictionary has the ``root`` and ``concrete_specs`` of the included\nenvironments, which are keyed by the path to that environment. Since this is optional\nif the environment does not have any included environments ``include_concrete`` will\nnot be a part of the lockfile.\n\n.. code-block:: json\n\n {\n \"_meta\": {\n \"file-type\": \"spack-lockfile\",\n \"lockfile-version\": 5,\n \"specfile-version\": 3\n },\n \"roots\": [\n {\n \"hash\": \"\",\n \"spec\": \"\"\n },\n {\n \"hash\": \"\",\n \"spec\": \"\"\n }\n ],\n \"concrete_specs\": {\n \"\": {\n \"... ...\": { },\n \"dependencies\": [\n {\n \"name\": \"depname_1\",\n \"hash\": \"\",\n \"type\": [\"build\", \"link\"]\n },\n {\n \"name\": \"depname_2\",\n \"hash\": \"\",\n \"type\": [\"build\", \"link\"]\n }\n ],\n \"hash\": \"\",\n },\n \"\": {\n \"... ...\": { },\n \"dependencies\": [\n {\n \"name\": \"depname_3\",\n \"hash\": \"\",\n \"type\": [\"build\", \"link\"]\n },\n {\n \"name\": \"depname_4\",\n \"hash\": \"\",\n \"type\": [\"build\", \"link\"]\n }\n ],\n \"hash\": \"\"\n }\n }\n \"include_concrete\": {\n \"\": {\n \"roots\": [\n {\n \"hash\": \"\",\n \"spec\": \"\"\n },\n {\n \"hash\": \"\",\n \"spec\": \"\"\n }\n ],\n \"concrete_specs\": {\n \"\": {\n \"... ...\": { },\n \"dependencies\": [\n {\n \"name\": \"depname_1\",\n \"hash\": \"\",\n \"type\": [\"build\", \"link\"]\n },\n {\n \"name\": \"depname_2\",\n \"hash\": \"\",\n \"type\": [\"build\", \"link\"]\n }\n ],\n \"hash\": \"\",\n },\n \"\": {\n \"... ...\": { },\n \"dependencies\": [\n {\n \"name\": \"depname_3\",\n \"hash\": \"\",\n \"type\": [\"build\", \"link\"]\n },\n {\n \"name\": \"depname_4\",\n \"hash\": \"\",\n \"type\": [\"build\", \"link\"]\n }\n ],\n \"hash\": \"\"\n }\n }\n }\n }\n }\n\n\nVersion 6\n---------\n\nVersion 6 uses specs where compilers are modeled as real dependencies, and not as a node attribute.\nIt doesn't change the top-level lockfile format.\n\nAs part of Spack v1.0, compilers stopped being a node attribute, and became a build-only\ndependency. Packages may declare a dependency on the c, cxx, or fortran languages, which are now\ntreated as virtuals, and compilers would be providers for one or more of those languages. Compilers\ncan also inject runtime dependency, on the node being compiled. The compiler-wrapper is explicitly\nrepresented as a node in the DAG, and enters the hash.\n\n.. code-block:: json\n\n {\n \"_meta\": {\n \"file-type\": \"spack-lockfile\",\n \"lockfile-version\": 6,\n \"specfile-version\": 5\n },\n \"spack\": {\n \"version\": \"1.0.0.dev0\",\n \"type\": \"git\",\n \"commit\": \"395b34f17417132389a6a8ee4dbf831c4a04f642\"\n },\n \"roots\": [\n {\n \"hash\": \"tivmbe3xjw7oqv4c3jv3v4jw42a7cajq\",\n \"spec\": \"zlib-ng\"\n }\n ],\n \"concrete_specs\": {\n \"tivmbe3xjw7oqv4c3jv3v4jw42a7cajq\": {\n \"name\": \"zlib-ng\",\n \"version\": \"2.2.3\",\n \"\": {}\n }\n \"dependencies\": [\n {\n \"name\": \"compiler-wrapper\",\n \"hash\": \"n5lamxu36f4cx4sm7m7gocalctve4mcx\",\n \"parameters\": {\n \"deptypes\": [\n \"build\"\n ],\n \"virtuals\": []\n }\n },\n {\n \"name\": \"gcc\",\n \"hash\": \"b375mbpprxze4vxy4ho7aixhuchsime2\",\n \"parameters\": {\n \"deptypes\": [\n \"build\"\n ],\n \"virtuals\": [\n \"c\",\n \"cxx\"\n ]\n }\n },\n {\n \"\": {}\n }\n ],\n \"annotations\": {\n \"original_specfile_version\": 5\n },\n }\n }\n\nVersion 7\n---------\n\nVersion 7 adds the additional attribute ``group`` to ``roots``.\n\nAs part of Spack v1.2 each environment can define multiple groups of specs, and fine-tune their\nconcretization separately. This attribute is needed to associate each root spec with the\ncorresponding group.\n\n.. code-block:: json\n\n {\n \"_meta\": {\n \"file-type\": \"spack-lockfile\",\n \"lockfile-version\": 7,\n \"specfile-version\": 5\n },\n \"spack\": {\n \"version\": \"1.2.0.dev0\",\n \"type\": \"git\",\n \"commit\": \"94b055476f874f424f20e3c0f33b0f22de29220a\"\n },\n \"roots\": [\n {\n \"hash\": \"o72mlpqvb5xijyqg4iyubpnvd5bfcomb\",\n \"spec\": \"hdf5\",\n \"group\": \"default\"\n }\n ],\n \"concrete_specs\": {\n }\n }\n\n\"\"\"\n\nfrom .environment import (\n TOP_LEVEL_KEY,\n Environment,\n SpackEnvironmentConfigError,\n SpackEnvironmentDevelopError,\n SpackEnvironmentError,\n SpackEnvironmentViewError,\n activate,\n active,\n active_environment,\n all_environment_names,\n all_environments,\n as_env_dir,\n create,\n create_in_dir,\n deactivate,\n default_manifest_yaml,\n default_view_name,\n display_specs,\n environment_dir_from_name,\n environment_from_name_or_dir,\n environment_path_scope,\n exists,\n initialize_environment_dir,\n installed_specs,\n is_env_dir,\n is_latest_format,\n lockfile_include_key,\n lockfile_name,\n manifest_file,\n manifest_include_name,\n manifest_name,\n no_active_environment,\n read,\n root,\n spack_env_var,\n spack_env_view_var,\n update_yaml,\n)\n\n__all__ = [\n \"TOP_LEVEL_KEY\",\n \"Environment\",\n \"SpackEnvironmentConfigError\",\n \"SpackEnvironmentDevelopError\",\n \"SpackEnvironmentError\",\n \"SpackEnvironmentViewError\",\n \"activate\",\n \"active\",\n \"active_environment\",\n \"all_environment_names\",\n \"all_environments\",\n \"as_env_dir\",\n \"create\",\n \"create_in_dir\",\n \"deactivate\",\n \"default_manifest_yaml\",\n \"default_view_name\",\n \"display_specs\",\n \"environment_dir_from_name\",\n \"environment_from_name_or_dir\",\n \"environment_path_scope\",\n \"exists\",\n \"initialize_environment_dir\",\n \"installed_specs\",\n \"is_env_dir\",\n \"is_latest_format\",\n \"lockfile_include_key\",\n \"lockfile_name\",\n \"manifest_file\",\n \"manifest_include_name\",\n \"manifest_name\",\n \"no_active_environment\",\n \"read\",\n \"root\",\n \"spack_env_var\",\n \"spack_env_view_var\",\n \"update_yaml\",\n]\n" }, { "path": "lib/spack/spack/environment/depfile.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"\nThis module contains the traversal logic and models that can be used to generate\ndepfiles from an environment.\n\"\"\"\n\nimport os\nimport re\nimport shlex\nfrom enum import Enum\nfrom typing import List, Optional\n\nimport spack.deptypes as dt\nimport spack.environment.environment as ev\nimport spack.paths\nimport spack.spec\nimport spack.traverse as traverse\n\n\nclass UseBuildCache(Enum):\n ONLY = 1\n NEVER = 2\n AUTO = 3\n\n @staticmethod\n def from_string(s: str) -> \"UseBuildCache\":\n if s == \"only\":\n return UseBuildCache.ONLY\n elif s == \"never\":\n return UseBuildCache.NEVER\n elif s == \"auto\":\n return UseBuildCache.AUTO\n raise ValueError(f\"invalid value for UseBuildCache: {s}\")\n\n\ndef _deptypes(use_buildcache: UseBuildCache):\n \"\"\"What edges should we follow for a given node? If it's a cache-only\n node, then we can drop build type deps.\"\"\"\n return (\n dt.LINK | dt.RUN if use_buildcache == UseBuildCache.ONLY else dt.BUILD | dt.LINK | dt.RUN\n )\n\n\nclass DepfileNode:\n \"\"\"Contains a spec, a subset of its dependencies, and a flag whether it should be\n buildcache only/never/auto.\"\"\"\n\n def __init__(\n self, target: spack.spec.Spec, prereqs: List[spack.spec.Spec], buildcache: UseBuildCache\n ):\n self.target = MakefileSpec(target)\n self.prereqs = list(MakefileSpec(x) for x in prereqs)\n if buildcache == UseBuildCache.ONLY:\n self.buildcache_flag = \"--use-buildcache=only\"\n elif buildcache == UseBuildCache.NEVER:\n self.buildcache_flag = \"--use-buildcache=never\"\n else:\n self.buildcache_flag = \"\"\n\n\nclass DepfileSpecVisitor:\n \"\"\"This visitor produces an adjacency list of a (reduced) DAG, which\n is used to generate depfile targets with their prerequisites. Currently\n it only drops build deps when using buildcache only mode.\n\n Note that the DAG could be reduced even more by dropping build edges of specs\n installed at the moment the depfile is generated, but that would produce\n stateful depfiles that would not fail when the database is wiped later.\"\"\"\n\n def __init__(self, pkg_buildcache: UseBuildCache, deps_buildcache: UseBuildCache):\n self.adjacency_list: List[DepfileNode] = []\n self.pkg_buildcache = pkg_buildcache\n self.deps_buildcache = deps_buildcache\n self.depflag_root = _deptypes(pkg_buildcache)\n self.depflag_deps = _deptypes(deps_buildcache)\n\n def neighbors(self, node):\n \"\"\"Produce a list of spec to follow from node\"\"\"\n depflag = self.depflag_root if node.depth == 0 else self.depflag_deps\n return traverse.sort_edges(node.edge.spec.edges_to_dependencies(depflag=depflag))\n\n def accept(self, node):\n self.adjacency_list.append(\n DepfileNode(\n target=node.edge.spec,\n prereqs=[edge.spec for edge in self.neighbors(node)],\n buildcache=self.pkg_buildcache if node.depth == 0 else self.deps_buildcache,\n )\n )\n\n # We already accepted this\n return True\n\n\nclass MakefileSpec(object):\n \"\"\"Limited interface to spec to help generate targets etc. without\n introducing unwanted special characters.\n \"\"\"\n\n _pattern = None\n\n def __init__(self, spec):\n self.spec = spec\n\n def safe_name(self):\n return self.safe_format(\"{name}-{version}-{hash}\")\n\n def spec_hash(self):\n return self.spec.dag_hash()\n\n def safe_format(self, format_str):\n unsafe_result = self.spec.format(format_str)\n if not MakefileSpec._pattern:\n MakefileSpec._pattern = re.compile(r\"[^A-Za-z0-9_.-]\")\n return MakefileSpec._pattern.sub(\"_\", unsafe_result)\n\n def unsafe_format(self, format_str):\n return self.spec.format(format_str)\n\n\nclass MakefileModel:\n \"\"\"This class produces all data to render a makefile for specs of an environment.\"\"\"\n\n def __init__(\n self,\n env: ev.Environment,\n roots: List[spack.spec.Spec],\n adjacency_list: List[DepfileNode],\n make_prefix: Optional[str],\n jobserver: bool,\n ):\n \"\"\"\n Args:\n env: environment to generate the makefile for\n roots: specs that get built in the default target\n adjacency_list: list of DepfileNode, mapping specs to their dependencies\n make_prefix: prefix for makefile targets\n jobserver: when enabled, make will invoke Spack with jobserver support. For\n dry-run this should be disabled.\n \"\"\"\n # Currently we can only use depfile with an environment since Spack needs to\n # find the concrete specs somewhere.\n self.env_path = env.path\n\n # These specs are built in the default target.\n self.roots = list(MakefileSpec(x) for x in roots)\n\n # The SPACK_PACKAGE_IDS variable is \"exported\", which can be used when including\n # generated makefiles to add post-install hooks, like pushing to a buildcache,\n # running tests, etc.\n if make_prefix is None:\n self.make_prefix = os.path.join(env.env_subdir_path, \"makedeps\")\n self.pkg_identifier_variable = \"SPACK_PACKAGE_IDS\"\n else:\n # NOTE: GNU Make allows directory separators in variable names, so for consistency\n # we can namespace this variable with the same prefix as targets.\n self.make_prefix = make_prefix\n self.pkg_identifier_variable = os.path.join(make_prefix, \"SPACK_PACKAGE_IDS\")\n\n # And here we collect a tuple of (target, prereqs, dag_hash, nice_name, buildcache_flag)\n self.make_adjacency_list = [\n (\n item.target.safe_name(),\n \" \".join(self._install_target(s.safe_name()) for s in item.prereqs),\n item.target.spec_hash(),\n item.target.unsafe_format(\n \"{name}{@version}{variants}\"\n \"{ platform=architecture.platform}{ os=architecture.os}\"\n \"{ target=architecture.target}\"\n ),\n item.buildcache_flag,\n )\n for item in adjacency_list\n ]\n\n # Root specs without deps are the prereqs for the environment target\n self.root_install_targets = [self._install_target(s.safe_name()) for s in self.roots]\n\n self.jobserver_support = \"+\" if jobserver else \"\"\n\n # All package identifiers, used to generate the SPACK_PACKAGE_IDS variable\n self.all_pkg_identifiers: List[str] = []\n\n # All install and install-deps targets\n self.all_install_related_targets: List[str] = []\n\n # Convenience shortcuts: ensure that `make install/pkg-version-hash` triggers\n # /.spack-env/makedeps/install/pkg-version-hash in case\n # we don't have a custom make target prefix.\n self.phony_convenience_targets: List[str] = []\n\n for node in adjacency_list:\n tgt = node.target.safe_name()\n self.all_pkg_identifiers.append(tgt)\n self.all_install_related_targets.append(self._install_target(tgt))\n self.all_install_related_targets.append(self._install_deps_target(tgt))\n if make_prefix is None:\n self.phony_convenience_targets.append(os.path.join(\"install\", tgt))\n self.phony_convenience_targets.append(os.path.join(\"install-deps\", tgt))\n\n def _target(self, name: str) -> str:\n # The `all` and `clean` targets are phony. It doesn't make sense to\n # have /abs/path/to/env/metadir/{all,clean} targets. But it *does* make\n # sense to have a prefix like `env/all`, `env/clean` when they are\n # supposed to be included\n if name in (\"all\", \"clean\") and os.path.isabs(self.make_prefix):\n return name\n else:\n return os.path.join(self.make_prefix, name)\n\n def _install_target(self, name: str) -> str:\n return os.path.join(self.make_prefix, \"install\", name)\n\n def _install_deps_target(self, name: str) -> str:\n return os.path.join(self.make_prefix, \"install-deps\", name)\n\n def to_dict(self):\n return {\n \"all_target\": self._target(\"all\"),\n \"env_target\": self._target(\"env\"),\n \"clean_target\": self._target(\"clean\"),\n \"all_install_related_targets\": \" \".join(self.all_install_related_targets),\n \"root_install_targets\": \" \".join(self.root_install_targets),\n \"dirs_target\": self._target(\"dirs\"),\n \"environment\": self.env_path,\n \"install_target\": self._target(\"install\"),\n \"install_deps_target\": self._target(\"install-deps\"),\n \"any_hash_target\": self._target(\"%\"),\n \"jobserver_support\": self.jobserver_support,\n \"spack_script\": shlex.quote(spack.paths.spack_script),\n \"adjacency_list\": self.make_adjacency_list,\n \"phony_convenience_targets\": \" \".join(self.phony_convenience_targets),\n \"pkg_ids_variable\": self.pkg_identifier_variable,\n \"pkg_ids\": \" \".join(self.all_pkg_identifiers),\n }\n\n @property\n def empty(self):\n return len(self.roots) == 0\n\n @staticmethod\n def from_env(\n env: ev.Environment,\n *,\n filter_specs: Optional[List[spack.spec.Spec]] = None,\n pkg_buildcache: UseBuildCache = UseBuildCache.AUTO,\n dep_buildcache: UseBuildCache = UseBuildCache.AUTO,\n make_prefix: Optional[str] = None,\n jobserver: bool = True,\n ) -> \"MakefileModel\":\n \"\"\"Produces a MakefileModel from an environment and a list of specs.\n\n Args:\n env: the environment to use\n filter_specs: if provided, only these specs will be built from the environment,\n otherwise the environment roots are used.\n pkg_buildcache: whether to only use the buildcache for top-level specs.\n dep_buildcache: whether to only use the buildcache for non-top-level specs.\n make_prefix: the prefix for the makefile targets\n jobserver: when enabled, make will invoke Spack with jobserver support. For\n dry-run this should be disabled.\n \"\"\"\n roots = env.all_matching_specs(*filter_specs) if filter_specs else env.concrete_roots()\n visitor = DepfileSpecVisitor(pkg_buildcache, dep_buildcache)\n traverse.traverse_breadth_first_with_visitor(\n roots, traverse.CoverNodesVisitor(visitor, key=lambda s: s.dag_hash())\n )\n\n return MakefileModel(env, roots, visitor.adjacency_list, make_prefix, jobserver)\n" }, { "path": "lib/spack/spack/environment/environment.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport collections\nimport collections.abc\nimport contextlib\nimport errno\nimport glob\nimport os\nimport pathlib\nimport re\nimport shutil\nimport stat\nimport warnings\nfrom collections.abc import KeysView\nfrom typing import (\n Any,\n Callable,\n Dict,\n Iterable,\n List,\n Mapping,\n Optional,\n Sequence,\n Set,\n Tuple,\n Union,\n)\n\nimport spack\nimport spack.config\nimport spack.deptypes as dt\nimport spack.error\nimport spack.filesystem_view as fsv\nimport spack.hash_types as ht\nimport spack.installer_dispatch\nimport spack.llnl.util.filesystem as fs\nimport spack.llnl.util.tty as tty\nimport spack.llnl.util.tty.color as clr\nimport spack.package_base\nimport spack.paths\nimport spack.repo\nimport spack.schema.env\nimport spack.spec\nimport spack.store\nimport spack.user_environment as uenv\nimport spack.util.environment\nimport spack.util.hash\nimport spack.util.lock as lk\nimport spack.util.path\nimport spack.util.spack_json as sjson\nimport spack.util.spack_yaml as syaml\nimport spack.variant as vt\nfrom spack import traverse\nfrom spack.enums import ConfigScopePriority\nfrom spack.llnl.util.filesystem import copy_tree, islink, readlink, symlink\nfrom spack.llnl.util.lang import stable_partition\nfrom spack.llnl.util.link_tree import ConflictingSpecsError\nfrom spack.schema.env import TOP_LEVEL_KEY\nfrom spack.spec import Spec\nfrom spack.spec_filter import SpecFilter\nfrom spack.util.path import substitute_path_variables\n\nfrom .list import SpecList, SpecListError, SpecListParser\n\nSpecPair = Tuple[Spec, Spec]\n\nDEFAULT_USER_SPEC_GROUP = \"default\"\n\n#: environment variable used to indicate the active environment\nspack_env_var = \"SPACK_ENV\"\n\n#: environment variable used to indicate the active environment view\nspack_env_view_var = \"SPACK_ENV_VIEW\"\n\n#: currently activated environment\n_active_environment: Optional[\"Environment\"] = None\n\n# This is used in spack.main to bypass env failures if the command is `spack config edit`\n# It is used in spack.cmd.config to get the path to a failed env for `spack config edit`\n#: Validation error for a currently activate environment that failed to parse\n_active_environment_error: Optional[spack.config.ConfigFormatError] = None\n\n#: default path where environments are stored in the spack tree\ndefault_env_path = os.path.join(spack.paths.var_path, \"environments\")\n\n\n#: Name of the input yaml file for an environment\nmanifest_name = \"spack.yaml\"\n\n\n#: Name of the input yaml file for an environment\nlockfile_name = \"spack.lock\"\n\n\n#: Name of the directory where environments store repos, logs, views, configs\nenv_subdir_name = \".spack-env\"\n\n\ndef env_root_path() -> str:\n \"\"\"Override default root path if the user specified it\"\"\"\n return spack.util.path.canonicalize_path(\n spack.config.get(\"config:environments_root\", default=default_env_path)\n )\n\n\ndef environment_name(path: Union[str, pathlib.Path]) -> str:\n \"\"\"Human-readable representation of the environment.\n\n This is the path for independent environments, and just the name\n for managed environments.\n \"\"\"\n env_root = pathlib.Path(env_root_path()).resolve()\n path_path = pathlib.Path(path)\n\n # For a managed environment created in Spack, env.path is ENV_ROOT/NAME\n # For a tracked environment from `spack env track`, the path is symlinked to ENV_ROOT/NAME\n # So if ENV_ROOT/NAME resolves to env.path we know the environment is tracked/managed.\n # Otherwise, it is an independent environment and we return the path.\n #\n # We resolve both paths fully because the env_root itself could also be a symlink,\n # and any directory in env.path could be a symlink.\n if (env_root / path_path.name).resolve() == path_path.resolve():\n return path_path.name\n else:\n return str(path)\n\n\ndef ensure_no_disallowed_env_config_mods(scope: spack.config.ConfigScope) -> None:\n config = scope.get_section(\"config\")\n if config and \"environments_root\" in config[\"config\"]:\n raise SpackEnvironmentError(\n \"Spack environments are prohibited from modifying 'config:environments_root' \"\n \"because it can make the definition of the environment ill-posed. Please \"\n \"remove from your environment and place it in a permanent scope such as \"\n \"defaults, system, site, etc.\"\n )\n\n\ndef default_manifest_yaml():\n \"\"\"default spack.yaml file to put in new environments\"\"\"\n return \"\"\"\\\n# This is a Spack Environment file.\n#\n# It describes a set of packages to be installed, along with\n# configuration settings.\nspack:\n # add package specs to the `specs` list\n specs: []\n view: true\n concretizer:\n unify: {}\n\"\"\".format(\"true\" if spack.config.get(\"concretizer:unify\") else \"false\")\n\n\nsep_re = re.escape(os.sep)\n\n#: regex for validating environment names\nvalid_environment_name_re = rf\"^\\w[{sep_re}\\w-]*$\"\n\n#: version of the lockfile format. Must increase monotonically.\nCURRENT_LOCKFILE_VERSION = 7\n\n\nREADER_CLS = {\n 1: spack.spec.SpecfileV1,\n 2: spack.spec.SpecfileV1,\n 3: spack.spec.SpecfileV2,\n 4: spack.spec.SpecfileV3,\n 5: spack.spec.SpecfileV4,\n 6: spack.spec.SpecfileV5,\n 7: spack.spec.SpecfileV5,\n}\n\n\n# Magic names\n# The name of the standalone spec list in the manifest yaml\nUSER_SPECS_KEY = \"specs\"\n# The name of the default view (the view loaded on env.activate)\ndefault_view_name = \"default\"\n# Default behavior to link all packages into views (vs. only root packages)\ndefault_view_link = \"all\"\n\n# (DEPRECATED) Use as the heading/name in the manifest is deprecated.\n# The key for any concrete specs included in a lockfile.\nlockfile_include_key = \"include_concrete\"\n\n# The name/heading for include paths in the manifest file.\nmanifest_include_name = \"include\"\n\n\ndef installed_specs():\n \"\"\"\n Returns the specs of packages installed in the active environment or None\n if no packages are installed.\n \"\"\"\n env = active_environment()\n hashes = env.all_hashes() if env else None\n return spack.store.STORE.db.query(hashes=hashes)\n\n\ndef valid_env_name(name):\n return re.match(valid_environment_name_re, name)\n\n\ndef validate_env_name(name):\n if not valid_env_name(name):\n raise ValueError(\n f\"{name}: names may only contain letters, numbers, _, and -, and may not start with -.\"\n )\n return name\n\n\ndef activate(env, use_env_repo=False):\n \"\"\"Activate an environment.\n\n To activate an environment, we add its manifest's configuration scope to the\n existing Spack configuration, and we set active to the current environment.\n\n Arguments:\n env (Environment): the environment to activate\n use_env_repo (bool): use the packages exactly as they appear in the\n environment's repository\n \"\"\"\n global _active_environment\n\n try:\n _active_environment = env\n\n # Fail early to avoid ending in an invalid state\n if not isinstance(env, Environment):\n raise TypeError(\"`env` should be of type {0}\".format(Environment.__name__))\n\n # Check if we need to reinitialize spack.store.STORE and spack.repo.REPO due to\n # config changes.\n install_tree_before = spack.config.get(\"config:install_tree\")\n upstreams_before = spack.config.get(\"upstreams\")\n repos_before = spack.config.get(\"repos\")\n env.manifest.prepare_config_scope()\n install_tree_after = spack.config.get(\"config:install_tree\")\n upstreams_after = spack.config.get(\"upstreams\")\n repos_after = spack.config.get(\"repos\")\n\n if install_tree_before != install_tree_after or upstreams_before != upstreams_after:\n setattr(env, \"store_token\", spack.store.reinitialize())\n\n if repos_before != repos_after:\n setattr(env, \"repo_token\", spack.repo.PATH)\n spack.repo.PATH.disable()\n new_repo = spack.repo.RepoPath.from_config(spack.config.CONFIG)\n if use_env_repo:\n new_repo.put_first(env.repo)\n spack.repo.enable_repo(new_repo)\n\n tty.debug(f\"Using environment '{env.name}'\")\n except Exception:\n _active_environment = None\n raise\n\n\ndef deactivate():\n \"\"\"Undo any configuration or repo settings modified by ``activate()``.\"\"\"\n global _active_environment\n\n if not _active_environment:\n return\n\n # If any config changes affected spack.store.STORE or spack.repo.PATH, undo them.\n store = getattr(_active_environment, \"store_token\", None)\n if store is not None:\n spack.store.restore(store)\n delattr(_active_environment, \"store_token\")\n\n repo = getattr(_active_environment, \"repo_token\", None)\n\n if repo is not None:\n spack.repo.PATH.disable()\n spack.repo.enable_repo(repo)\n\n _active_environment.manifest.deactivate_config_scope()\n\n tty.debug(f\"Deactivated environment '{_active_environment.name}'\")\n\n _active_environment = None\n\n\ndef active_environment() -> Optional[\"Environment\"]:\n \"\"\"Returns the active environment when there is any\"\"\"\n return _active_environment\n\n\ndef _root(name):\n \"\"\"Non-validating version of root(), to be used internally.\"\"\"\n return os.path.join(env_root_path(), name)\n\n\ndef root(name):\n \"\"\"Get the root directory for an environment by name.\"\"\"\n validate_env_name(name)\n return _root(name)\n\n\ndef exists(name):\n \"\"\"Whether an environment with this name exists or not.\"\"\"\n return valid_env_name(name) and os.path.lexists(os.path.join(_root(name), manifest_name))\n\n\ndef active(name):\n \"\"\"True if the named environment is active.\"\"\"\n return _active_environment and name == _active_environment.name\n\n\ndef is_env_dir(path):\n \"\"\"Whether a directory contains a spack environment.\"\"\"\n return os.path.isdir(path) and os.path.exists(os.path.join(path, manifest_name))\n\n\ndef as_env_dir(name_or_dir):\n \"\"\"Translate an environment name or directory to the environment directory\"\"\"\n if is_env_dir(name_or_dir):\n return name_or_dir\n else:\n validate_env_name(name_or_dir)\n if not exists(name_or_dir):\n raise SpackEnvironmentError(\"no such environment '%s'\" % name_or_dir)\n return _root(name_or_dir)\n\n\ndef environment_from_name_or_dir(name_or_dir):\n \"\"\"Get an environment with the supplied name.\"\"\"\n return Environment(as_env_dir(name_or_dir))\n\n\ndef read(name):\n \"\"\"Get an environment with the supplied name.\"\"\"\n validate_env_name(name)\n if not exists(name):\n raise SpackEnvironmentError(\"no such environment '%s'\" % name)\n return Environment(root(name))\n\n\ndef create(\n name: str,\n init_file: Optional[Union[str, pathlib.Path]] = None,\n with_view: Optional[Union[str, pathlib.Path, bool]] = None,\n keep_relative: bool = False,\n include_concrete: Optional[List[str]] = None,\n) -> \"Environment\":\n \"\"\"Create a managed environment in Spack and returns it.\n\n A managed environment is created in a root directory managed by this Spack instance, so that\n Spack can keep track of them.\n\n Files with suffix ``.json`` or ``.lock`` are considered lockfiles. Files with any other name\n are considered manifest files.\n\n Args:\n name: name of the managed environment\n init_file: either a lockfile, a manifest file, or None\n with_view: whether a view should be maintained for the environment. If the value is a\n string, it specifies the path to the view\n keep_relative: if True, develop paths are copied verbatim into the new environment file,\n otherwise they are made absolute\n include_concrete: concrete environment names/paths to be included\n \"\"\"\n environment_dir = environment_dir_from_name(name, exists_ok=False)\n return create_in_dir(\n environment_dir,\n init_file=init_file,\n with_view=with_view,\n keep_relative=keep_relative,\n include_concrete=include_concrete,\n )\n\n\ndef create_in_dir(\n root: Union[str, pathlib.Path],\n init_file: Optional[Union[str, pathlib.Path]] = None,\n with_view: Optional[Union[str, pathlib.Path, bool]] = None,\n keep_relative: bool = False,\n include_concrete: Optional[List[str]] = None,\n) -> \"Environment\":\n \"\"\"Create an environment in the directory passed as input and returns it.\n\n Files with suffix ``.json`` or ``.lock`` are considered lockfiles. Files with any other name\n are considered manifest files.\n\n Args:\n root: directory where to create the environment.\n init_file: either a lockfile, a manifest file, an env directory, or None\n with_view: whether a view should be maintained for the environment. If the value is a\n string, it specifies the path to the view\n keep_relative: if True, develop paths are copied verbatim into the new environment file,\n otherwise they are made absolute\n include_concrete: concrete environment names/paths to be included\n \"\"\"\n # If the initfile is a named environment, get its path\n if init_file and exists(str(init_file)):\n init_file = read(str(init_file)).path\n initialize_environment_dir(root, envfile=init_file)\n\n if with_view is None and keep_relative:\n return Environment(root)\n\n try:\n manifest = EnvironmentManifestFile(root)\n\n if with_view is not None:\n manifest.set_default_view(with_view)\n\n if include_concrete is not None:\n set_included_envs_to_env_paths(include_concrete)\n validate_included_envs_exists(include_concrete)\n validate_included_envs_concrete(include_concrete)\n manifest.set_include_concrete(include_concrete)\n\n manifest.flush()\n\n except (spack.config.ConfigFormatError, SpackEnvironmentConfigError) as e:\n shutil.rmtree(root)\n raise e\n\n env = Environment(root)\n\n if init_file:\n if os.path.isdir(init_file):\n init_file_dir = init_file\n copied = True\n else:\n init_file_dir = os.path.abspath(os.path.dirname(init_file))\n copied = False\n\n if not keep_relative:\n if env.path != init_file_dir:\n # If we are here, we are creating an environment based on an\n # spack.yaml file in another directory, and moreover we want\n # dev paths in this environment to refer to their original\n # locations.\n # If the full env was copied including internal files, only rewrite\n # relative paths outside of env\n _rewrite_relative_dev_paths_on_relocation(env, init_file_dir, copied_env=copied)\n _rewrite_relative_repos_paths_on_relocation(env, init_file_dir, copied_env=copied)\n\n return env\n\n\ndef _rewrite_relative_dev_paths_on_relocation(env, init_file_dir, copied_env=False):\n \"\"\"When initializing the environment from a manifest file and we plan\n to store the environment in a different directory, we have to rewrite\n relative paths to absolute ones.\"\"\"\n with env:\n dev_specs = spack.config.get(\"develop\", default={}, scope=env.scope_name)\n if not dev_specs:\n return\n for name, entry in dev_specs.items():\n dev_path = substitute_path_variables(entry[\"path\"])\n expanded_path = spack.util.path.canonicalize_path(dev_path, default_wd=init_file_dir)\n\n # Skip if the substituted and expanded path is the same (e.g. when absolute)\n if entry[\"path\"] == expanded_path:\n continue\n\n # If copied and it's inside the env, we copied it and don't need to relativize\n if copied_env and expanded_path.startswith(init_file_dir):\n continue\n\n tty.debug(\"Expanding develop path for {0} to {1}\".format(name, expanded_path))\n\n dev_specs[name][\"path\"] = expanded_path\n\n spack.config.set(\"develop\", dev_specs, scope=env.scope_name)\n\n env._dev_specs = None\n # If we changed the environment's spack.yaml scope, that will not be reflected\n # in the manifest that we read\n env._re_read()\n\n\ndef _rewrite_relative_repos_paths_on_relocation(env, init_file_dir, copied_env=False):\n \"\"\"When initializing the environment from a manifest file and we plan\n to store the environment in a different directory, we have to rewrite\n relative repo paths to absolute ones and expand environment variables.\"\"\"\n with env:\n repos_specs = spack.config.get(\"repos\", default={}, scope=env.scope_name)\n if not repos_specs:\n return\n for name, entry in list(repos_specs.items()):\n # only rewrite when we have a path-based repository\n if not isinstance(entry, str):\n continue\n repo_path = substitute_path_variables(entry)\n expanded_path = spack.util.path.canonicalize_path(repo_path, default_wd=init_file_dir)\n\n # Skip if the substituted and expanded path is the same (e.g. when absolute)\n if entry == expanded_path:\n continue\n\n # If copied and it's inside the env, we copied it and don't need to relativize\n if copied_env and expanded_path.startswith(init_file_dir):\n continue\n\n tty.debug(\"Expanding repo path for {0} to {1}\".format(entry, expanded_path))\n\n repos_specs[name] = expanded_path\n\n spack.config.set(\"repos\", repos_specs, scope=env.scope_name)\n\n env.repos_specs = None\n # If we changed the environment's spack.yaml scope, that will not be reflected\n # in the manifest that we read\n env._re_read()\n\n\ndef environment_dir_from_name(name: str, exists_ok: bool = True) -> str:\n \"\"\"Returns the directory associated with a named environment.\n\n Args:\n name: name of the environment\n exists_ok: if False, raise an error if the environment exists already\n\n Raises:\n SpackEnvironmentError: if exists_ok is False and the environment exists already\n \"\"\"\n if not exists_ok and exists(name):\n raise SpackEnvironmentError(f\"'{name}': environment already exists at {root(name)}\")\n\n ensure_env_root_path_exists()\n validate_env_name(name)\n return root(name)\n\n\ndef ensure_env_root_path_exists():\n if not os.path.isdir(env_root_path()):\n fs.mkdirp(env_root_path())\n\n\ndef set_included_envs_to_env_paths(include_concrete: List[str]) -> None:\n \"\"\"If the included environment(s) is the environment name\n it is replaced by the path to the environment\n\n Args:\n include_concrete: list of env name or path to env\"\"\"\n\n for i, env_name in enumerate(include_concrete):\n if is_env_dir(env_name):\n include_concrete[i] = env_name\n elif exists(env_name):\n include_concrete[i] = root(env_name)\n\n\ndef validate_included_envs_exists(include_concrete: List[str]) -> None:\n \"\"\"Checks that all of the included environments exist\n\n Args:\n include_concrete: list of already existing concrete environments to include\n\n Raises:\n SpackEnvironmentError: if any of the included environments do not exist\n \"\"\"\n\n missing_envs = set()\n\n for i, env_name in enumerate(include_concrete):\n if not is_env_dir(env_name):\n missing_envs.add(env_name)\n\n if missing_envs:\n msg = \"The following environment(s) are missing: {0}\".format(\", \".join(missing_envs))\n raise SpackEnvironmentError(msg)\n\n\ndef validate_included_envs_concrete(include_concrete: List[str]) -> None:\n \"\"\"Checks that all of the included environments are concrete\n\n Args:\n include_concrete: list of already existing concrete environments to include\n\n Raises:\n SpackEnvironmentError: if any of the included environments are not concrete\n \"\"\"\n\n non_concrete_envs = set()\n\n for env_path in include_concrete:\n if not os.path.exists(os.path.join(env_path, lockfile_name)):\n non_concrete_envs.add(environment_name(env_path))\n\n if non_concrete_envs:\n msg = \"The following environment(s) are not concrete: {0}\\nPlease run:\".format(\n \", \".join(non_concrete_envs)\n )\n for env in non_concrete_envs:\n msg += f\"\\n\\t`spack -e {env} concretize`\"\n\n raise SpackEnvironmentError(msg)\n\n\ndef all_environment_names():\n \"\"\"List the names of environments that currently exist.\"\"\"\n # just return empty if the env path does not exist. A read-only\n # operation like list should not try to create a directory.\n if not os.path.exists(env_root_path()):\n return []\n\n env_root = pathlib.Path(env_root_path()).resolve()\n\n def yaml_paths():\n for root, dirs, files in os.walk(env_root, topdown=True, followlinks=True):\n dirs[:] = [\n d\n for d in dirs\n if not d.startswith(\".\") and not env_root.samefile(os.path.join(root, d))\n ]\n if manifest_name in files:\n yield os.path.join(root, manifest_name)\n\n names = []\n for yaml_path in yaml_paths():\n candidate = str(pathlib.Path(yaml_path).relative_to(env_root).parent)\n if valid_env_name(candidate):\n names.append(candidate)\n return names\n\n\ndef all_environments():\n \"\"\"Generator for all managed Environments.\"\"\"\n for name in all_environment_names():\n yield read(name)\n\n\ndef _read_yaml(str_or_file):\n \"\"\"Read YAML from a file for round-trip parsing.\"\"\"\n try:\n data = syaml.load_config(str_or_file)\n except syaml.SpackYAMLError as e:\n raise SpackEnvironmentConfigError(\n f\"Invalid environment configuration detected: {e.message}\", e.filename\n )\n\n filename = getattr(str_or_file, \"name\", None)\n spack.config.validate(data, spack.schema.env.schema, filename)\n return data\n\n\ndef _write_yaml(data, str_or_file):\n \"\"\"Write YAML to a file preserving comments and dict order.\"\"\"\n filename = getattr(str_or_file, \"name\", None)\n spack.config.validate(data, spack.schema.env.schema, filename)\n syaml.dump_config(data, str_or_file, default_flow_style=False)\n\n\ndef _is_dev_spec_and_has_changed(spec):\n \"\"\"Check if the passed spec is a dev build and whether it has changed since the\n last installation\"\"\"\n # First check if this is a dev build and in the process already try to get\n # the dev_path\n if not spec.variants.get(\"dev_path\", None):\n return False\n\n # Now we can check whether the code changed since the last installation\n if not spec.installed:\n # Not installed -> nothing to compare against\n return False\n\n # hook so packages can use to write their own method for checking the dev_path\n # use package so attributes about concretization such as variant state can be\n # utilized\n return spec.package.detect_dev_src_change()\n\n\ndef _error_on_nonempty_view_dir(new_root):\n \"\"\"Defensively error when the target view path already exists and is not an\n empty directory. This usually happens when the view symlink was removed, but\n not the directory it points to. In those cases, it's better to just error when\n the new view dir is non-empty, since it indicates the user removed part but not\n all of the view, and it likely in an inconsistent state.\"\"\"\n # Check if the target path lexists\n try:\n st = os.lstat(new_root)\n except OSError:\n return\n\n # Empty directories are fine\n if stat.S_ISDIR(st.st_mode) and len(os.listdir(new_root)) == 0:\n return\n\n # Anything else is an error\n raise SpackEnvironmentViewError(\n \"Failed to generate environment view, because the target {} already \"\n \"exists or is not empty. To update the view, remove this path, and run \"\n \"`spack env view regenerate`\".format(new_root)\n )\n\n\nclass ViewDescriptor:\n def __init__(\n self,\n base_path: str,\n root: str,\n *,\n projections: Optional[Dict[str, str]] = None,\n select: Optional[List[str]] = None,\n exclude: Optional[List[str]] = None,\n link: str = default_view_link,\n link_type: fsv.LinkType = \"symlink\",\n groups: Optional[Union[str, List[str]]] = None,\n ) -> None:\n self.base = base_path\n self.raw_root = root\n self.root = spack.util.path.canonicalize_path(root, default_wd=base_path)\n self.projections = projections or {}\n self.select = select or []\n self.exclude = exclude or []\n self.link_type: fsv.LinkType = fsv.canonicalize_link_type(link_type)\n self.link = link\n if isinstance(groups, str):\n groups = [groups]\n self.groups: Optional[List[str]] = groups\n\n def select_fn(self, spec: Spec) -> bool:\n return any(spec.satisfies(s) for s in self.select)\n\n def exclude_fn(self, spec: Spec) -> bool:\n return not any(spec.satisfies(e) for e in self.exclude)\n\n def update_root(self, new_path: str) -> None:\n self.raw_root = new_path\n self.root = spack.util.path.canonicalize_path(new_path, default_wd=self.base)\n\n def __eq__(self, other: object) -> bool:\n return isinstance(other, ViewDescriptor) and all(\n [\n self.root == other.root,\n self.projections == other.projections,\n self.select == other.select,\n self.exclude == other.exclude,\n self.link == other.link,\n self.link_type == other.link_type,\n ]\n )\n\n def to_dict(self):\n ret = syaml.syaml_dict([(\"root\", self.raw_root)])\n if self.projections:\n ret[\"projections\"] = self.projections\n if self.select:\n ret[\"select\"] = self.select\n if self.exclude:\n ret[\"exclude\"] = self.exclude\n if self.link_type:\n ret[\"link_type\"] = self.link_type\n if self.link != default_view_link:\n ret[\"link\"] = self.link\n return ret\n\n @staticmethod\n def from_dict(base_path: str, d) -> \"ViewDescriptor\":\n return ViewDescriptor(\n base_path,\n d[\"root\"],\n projections=d.get(\"projections\", {}),\n select=d.get(\"select\", []),\n exclude=d.get(\"exclude\", []),\n link=d.get(\"link\", default_view_link),\n link_type=d.get(\"link_type\", \"symlink\"),\n groups=d.get(\"group\", None),\n )\n\n @property\n def _current_root(self) -> Optional[str]:\n if not islink(self.root):\n return None\n\n root = readlink(self.root)\n if os.path.isabs(root):\n return root\n\n root_dir = os.path.dirname(self.root)\n return os.path.join(root_dir, root)\n\n def _next_root(self, specs):\n content_hash = self.content_hash(specs)\n root_dir = os.path.dirname(self.root)\n root_name = os.path.basename(self.root)\n return os.path.join(root_dir, \"._%s\" % root_name, content_hash)\n\n def content_hash(self, specs):\n d = syaml.syaml_dict(\n [\n (\"descriptor\", self.to_dict()),\n (\"specs\", [(spec.dag_hash(), spec.prefix) for spec in sorted(specs)]),\n ]\n )\n contents = sjson.dump(d)\n return spack.util.hash.b32_hash(contents)\n\n def get_projection_for_spec(self, spec):\n \"\"\"Get projection for spec. This function does not require the view\n to exist on the filesystem.\"\"\"\n return self._view(self.root).get_projection_for_spec(spec)\n\n def view(self, new: Optional[str] = None) -> fsv.SimpleFilesystemView:\n \"\"\"\n Returns a view object for the *underlying* view directory. This means that the\n self.root symlink is followed, and that the view has to exist on the filesystem\n (unless ``new``). This function is useful when writing to the view.\n\n Raise if new is None and there is no current view\n\n Arguments:\n new: If a string, create a FilesystemView rooted at that path. Default None. This\n should only be used to regenerate the view, and cannot be used to access specs.\n \"\"\"\n path = new if new else self._current_root\n if not path:\n # This can only be hit if we write a future bug\n raise SpackEnvironmentViewError(\n f\"Attempting to get nonexistent view from environment. View root is at {self.root}\"\n )\n return self._view(path)\n\n def _view(self, root: str) -> fsv.SimpleFilesystemView:\n \"\"\"Returns a view object for a given root dir.\"\"\"\n return fsv.SimpleFilesystemView(\n root,\n spack.store.STORE.layout,\n ignore_conflicts=True,\n projections=self.projections,\n link_type=self.link_type,\n )\n\n def __contains__(self, spec):\n \"\"\"Is the spec described by the view descriptor\n\n Note: This does not claim the spec is already linked in the view.\n It merely checks that the spec is selected if a select operation is\n specified and is not excluded if an exclude operator is specified.\n \"\"\"\n if self.select:\n if not self.select_fn(spec):\n return False\n\n if self.exclude:\n if not self.exclude_fn(spec):\n return False\n\n return True\n\n def specs_for_view(self, concrete_roots: List[Spec]) -> List[Spec]:\n \"\"\"Flatten the DAGs of the concrete roots, keep only unique, selected, and installed specs\n in topological order from root to leaf.\"\"\"\n if self.link == \"all\":\n deptype = dt.LINK | dt.RUN\n elif self.link == \"run\":\n deptype = dt.RUN\n else:\n deptype = dt.NONE\n\n specs = traverse.traverse_nodes(\n concrete_roots, order=\"topo\", deptype=deptype, key=traverse.by_dag_hash\n )\n\n # Filter selected, installed specs\n with spack.store.STORE.db.read_transaction():\n result = [s for s in specs if s in self and s.installed]\n\n return self._exclude_duplicate_runtimes(result)\n\n def regenerate(self, env: \"Environment\") -> None:\n if self.groups is None:\n concrete_roots = env.concrete_roots()\n else:\n concrete_roots = [c for g in self.groups for _, c in env.concretized_specs_by(group=g)]\n\n specs = self.specs_for_view(concrete_roots)\n\n # To ensure there are no conflicts with packages being installed\n # that cannot be resolved or have repos that have been removed\n # we always regenerate the view from scratch.\n # We will do this by hashing the view contents and putting the view\n # in a directory by hash, and then having a symlink to the real\n # view in the root. The real root for a view at /dirname/basename\n # will be /dirname/._basename_.\n # This allows for atomic swaps when we update the view\n\n # cache the roots because the way we determine which is which does\n # not work while we are updating\n new_root = self._next_root(specs)\n old_root = self._current_root\n\n if new_root == old_root:\n tty.debug(f\"View at {self.root} does not need regeneration.\")\n return\n\n _error_on_nonempty_view_dir(new_root)\n\n # construct view at new_root\n if specs:\n tty.msg(f\"Updating view at {self.root}\")\n\n view = self.view(new=new_root)\n\n root_dirname = os.path.dirname(self.root)\n tmp_symlink_name = os.path.join(root_dirname, \"._view_link\")\n\n # Remove self.root if is it an empty dir, since we need a symlink there. Note that rmdir\n # fails if self.root is a symlink.\n try:\n os.rmdir(self.root)\n except (FileNotFoundError, NotADirectoryError):\n pass\n except OSError as e:\n if e.errno == errno.ENOTEMPTY:\n msg = \"it is a non-empty directory\"\n elif e.errno == errno.EACCES:\n msg = \"of insufficient permissions\"\n else:\n raise\n raise SpackEnvironmentViewError(\n f\"The environment view in {self.root} cannot not be created because {msg}.\"\n ) from e\n\n # Create a new view\n try:\n fs.mkdirp(new_root)\n view.add_specs(*specs)\n\n # create symlink from tmp_symlink_name to new_root\n if os.path.exists(tmp_symlink_name):\n os.unlink(tmp_symlink_name)\n symlink(new_root, tmp_symlink_name)\n\n # mv symlink atomically over root symlink to old_root\n fs.rename(tmp_symlink_name, self.root)\n except Exception as e:\n # Clean up new view and temporary symlink on any failure.\n try:\n shutil.rmtree(new_root, ignore_errors=True)\n os.unlink(tmp_symlink_name)\n except OSError:\n pass\n\n # Give an informative error message for the typical error case: two specs, same package\n # project to same prefix.\n if isinstance(e, ConflictingSpecsError):\n spec_a = e.args[0].format(color=clr.get_color_when())\n spec_b = e.args[1].format(color=clr.get_color_when())\n raise SpackEnvironmentViewError(\n f\"The environment view in {self.root} could not be created, \"\n \"because the following two specs project to the same prefix:\\n\"\n f\" {spec_a}, and\\n\"\n f\" {spec_b}.\\n\"\n \" To resolve this issue:\\n\"\n \" a. use `concretization:unify:true` to ensure there is only one \"\n \"package per spec in the environment, or\\n\"\n \" b. disable views with `view:false`, or\\n\"\n \" c. create custom view projections.\"\n ) from e\n raise\n\n # Remove the old root when it's in the same folder as the new root. This guards\n # against removal of an arbitrary path when the original symlink in self.root\n # was not created by the environment, but by the user.\n if (\n old_root\n and os.path.exists(old_root)\n and os.path.samefile(os.path.dirname(new_root), os.path.dirname(old_root))\n ):\n try:\n shutil.rmtree(old_root)\n except OSError as e:\n msg = \"Failed to remove old view at %s\\n\" % old_root\n msg += str(e)\n tty.warn(msg)\n\n def _exclude_duplicate_runtimes(self, specs: List[Spec]) -> List[Spec]:\n \"\"\"Stably filter out duplicates of \"runtime\" tagged packages, keeping only latest.\"\"\"\n # Maps packages tagged \"runtime\" to the spec with latest version.\n latest: Dict[str, Spec] = {}\n for s in specs:\n if \"runtime\" not in getattr(s.package, \"tags\", ()):\n continue\n elif s.name not in latest or latest[s.name].version < s.version:\n latest[s.name] = s\n\n return [x for x in specs if x.name not in latest or latest[x.name] is x]\n\n\ndef env_subdir_path(manifest_dir: Union[str, pathlib.Path]) -> str:\n \"\"\"Path to where the environment stores repos, logs, views, configs.\n\n Args:\n manifest_dir: directory containing the environment manifest file\n\n Returns: directory the environment uses to manage its files\n \"\"\"\n return os.path.join(str(manifest_dir), env_subdir_name)\n\n\nclass ConcretizedRootInfo:\n \"\"\"Data on root specs that have been concretized\"\"\"\n\n __slots__ = (\"root\", \"hash\", \"new\", \"group\")\n\n def __init__(\n self, *, root_spec: spack.spec.Spec, root_hash: str, new: bool = False, group: str\n ):\n self.root = root_spec\n self.hash = root_hash\n self.new = new\n self.group = group\n\n def __str__(self):\n return f\"{self.root} -> {self.hash} [new={self.new}]\"\n\n def __eq__(self, other: object) -> bool:\n return (\n isinstance(other, ConcretizedRootInfo)\n and self.root == other.root\n and self.hash == other.hash\n and self.new == other.new\n and self.group == other.group\n )\n\n def __hash__(self) -> int:\n return hash((self.root, self.hash, self.new, self.group))\n\n @staticmethod\n def from_info_dict(info_dict: Dict[str, str]) -> \"ConcretizedRootInfo\":\n # Lockfile versions < 7 don't have the \"group\" attribute\n return ConcretizedRootInfo(\n root_spec=Spec(info_dict[\"spec\"]),\n root_hash=info_dict[\"hash\"],\n new=False,\n group=info_dict.get(\"group\", DEFAULT_USER_SPEC_GROUP),\n )\n\n\nclass Environment:\n \"\"\"A Spack environment, which bundles together configuration and a list of specs.\"\"\"\n\n def __init__(self, manifest_dir: Union[str, pathlib.Path]) -> None:\n \"\"\"An environment can be constructed from a directory containing a \"spack.yaml\" file, and\n optionally a consistent \"spack.lock\" file.\n\n Args:\n manifest_dir: directory with the \"spack.yaml\" associated with the environment\n \"\"\"\n self.path = os.path.abspath(str(manifest_dir))\n self.name = environment_name(self.path)\n self.env_subdir_path = env_subdir_path(self.path)\n\n self.txlock = lk.Lock(self._transaction_lock_path)\n\n self._unify = None\n self.views: Dict[str, ViewDescriptor] = {}\n\n #: Parser for spec lists\n self._spec_lists_parser = SpecListParser()\n #: Specs from \"spack.yaml\"\n self.spec_lists: Dict[str, SpecList] = {}\n #: Information on concretized roots\n self.concretized_roots: List[ConcretizedRootInfo] = []\n #: Concretized specs by hash\n self.specs_by_hash: Dict[str, Spec] = {}\n #: Repository for this environment (memoized)\n self._repo = None\n\n #: Environment root dirs for concrete (lockfile) included environments\n self.included_concrete_env_root_dirs: List[str] = []\n #: First-level included concretized spec data from/to the lockfile.\n self.included_concrete_spec_data: Dict[str, Dict[str, List[str]]] = {}\n #: Roots from included environments from the last concretization, keyed by env path\n self.included_concretized_roots: Dict[str, List[ConcretizedRootInfo]] = {}\n #: Concretized specs by hash from the included environments\n self.included_specs_by_hash: Dict[str, Dict[str, Spec]] = {}\n\n #: Previously active environment\n self._previous_active = None\n self._dev_specs = None\n\n # Load the manifest file contents into memory\n self._load_manifest_file()\n\n def _load_manifest_file(self):\n \"\"\"Instantiate and load the manifest file contents into memory.\"\"\"\n with lk.ReadTransaction(self.txlock):\n self.manifest = EnvironmentManifestFile(self.path, self.name)\n with self.manifest.use_config():\n self._read()\n\n @contextlib.contextmanager\n def config_override_for_group(self, *, group: str):\n key = self.manifest._ensure_group_exists(group=group)\n internal_scope = self.manifest.config_override(group=key)\n if internal_scope is None:\n # No internal scope\n tty.debug(\n f\"[{__name__}] No configuration override necessary for the '{group}' group \"\n f\"in the environment at {self.manifest_path}\"\n )\n yield\n return\n\n try:\n tty.debug(\n f\"[{__name__}] Overriding the configuration for the '{group}' group defined \"\n f\"in {self.manifest_path} before concretization\"\n )\n spack.config.CONFIG.push_scope(\n internal_scope, priority=ConfigScopePriority.ENVIRONMENT_SPEC_GROUPS\n )\n yield\n finally:\n spack.config.CONFIG.remove_scope(internal_scope.name)\n\n def __getstate__(self):\n state = self.__dict__.copy()\n state.pop(\"txlock\", None)\n state.pop(\"_repo\", None)\n state.pop(\"repo_token\", None)\n state.pop(\"store_token\", None)\n return state\n\n def __setstate__(self, state):\n self.__dict__.update(state)\n self.txlock = lk.Lock(self._transaction_lock_path)\n self._repo = None\n\n def _re_read(self):\n \"\"\"Reinitialize the environment object.\"\"\"\n self.clear()\n self._load_manifest_file()\n\n def _read(self):\n self._construct_state_from_manifest()\n\n if os.path.exists(self.lock_path):\n with open(self.lock_path, encoding=\"utf-8\") as f:\n read_lock_version = self._read_lockfile(f)[\"_meta\"][\"lockfile-version\"]\n\n if read_lock_version == 1:\n tty.debug(f\"Storing backup of {self.lock_path} at {self._lock_backup_v1_path}\")\n shutil.copy(self.lock_path, self._lock_backup_v1_path)\n\n def write_transaction(self):\n \"\"\"Get a write lock context manager for use in a ``with`` block.\"\"\"\n return lk.WriteTransaction(self.txlock, acquire=self._re_read)\n\n def _process_view(self, env_view: Optional[Union[bool, str, Dict]]):\n \"\"\"Process view option(s), which can be boolean, string, or None.\n\n A boolean environment view option takes precedence over any that may\n be included. So ``view: True`` results in the default view only. And\n ``view: False`` means the environment will have no view.\n\n Args:\n env_view: view option provided in the manifest or configuration\n \"\"\"\n\n def add_view(name, values):\n \"\"\"Add the view with the name and the string or dict values.\"\"\"\n if isinstance(values, str):\n self.views[name] = ViewDescriptor(self.path, values)\n elif isinstance(values, dict):\n self.views[name] = ViewDescriptor.from_dict(self.path, values)\n else:\n tty.error(f\"Cannot add view named {name} for {type(values)} values {values}\")\n\n # If the configuration specifies 'view: False' then we are done\n # processing views. If this is called with the environment's view\n # view (versus an included view), then there are to be NO views.\n if env_view is False:\n return\n\n # If the configuration specifies 'view: True' then only the default\n # view will be created for the environment and we are done processing\n # views.\n if env_view is True:\n add_view(default_view_name, self.view_path_default)\n return\n\n # Otherwise, the configuration has a subdirectory or dictionary.\n if isinstance(env_view, str):\n add_view(default_view_name, env_view)\n elif env_view:\n for name, values in env_view.items():\n add_view(name, values)\n\n # If we reach this point without an explicit view option then we\n # provide the default view.\n if self.views == dict():\n self.views[default_view_name] = ViewDescriptor(self.path, self.view_path_default)\n\n def _load_concrete_include_data(self):\n \"\"\"Load concrete include specs data from included concrete directories.\"\"\"\n if self.included_concrete_env_root_dirs:\n if os.path.exists(self.lock_path):\n with open(self.lock_path, encoding=\"utf-8\") as f:\n data = self._read_lockfile(f)\n\n if lockfile_include_key in data:\n self.included_concrete_spec_data = data[lockfile_include_key]\n else:\n self.include_concrete_envs()\n\n def _process_included_lockfiles(self):\n \"\"\"Extract and load into memory included lock file data.\"\"\"\n includes = self.manifest[TOP_LEVEL_KEY].get(lockfile_include_key, [])\n if includes:\n tty.warn(\n f\"Use of '{lockfile_include_key}' in manifest files \"\n f\"is deprecated. The key should be '{manifest_include_name}' \"\n f\"and the path should end with '{lockfile_name}'. Run \"\n f\"'spack env update {self.name}' to update the manifest.\"\n )\n includes = [os.path.join(inc, lockfile_name) for inc in includes]\n includes += self.manifest[TOP_LEVEL_KEY].get(manifest_include_name, [])\n if not includes:\n return\n\n # Expand config and environment variables for concrete environments,\n # indicated by the inclusion of lock files.\n self.included_concrete_env_root_dirs = []\n\n for entry in includes:\n include = spack.config.included_path(entry)\n if isinstance(include, spack.config.GitIncludePaths):\n # Git includes must be cloned first; paths are relative to the\n # clone destination, not to the manifest directory.\n destination = include._clone(self.manifest.env_config_scope)\n if destination is None:\n continue\n resolved = [os.path.join(destination, p) for p in include.paths]\n else:\n resolved = [\n spack.util.path.canonicalize_path(p, default_wd=self.path)\n for p in include.paths\n ]\n\n for path in resolved:\n if os.path.basename(path) != lockfile_name:\n continue\n\n tty.debug(f\"Adding {path} to the concrete environment root directories\")\n self.included_concrete_env_root_dirs.append(os.path.dirname(path))\n\n # Cache concrete environments for required lock files.\n self._load_concrete_include_data()\n\n def _construct_state_from_manifest(self):\n \"\"\"Set up user specs and views from the manifest file.\"\"\"\n self.views = {}\n self._sync_speclists()\n self._process_view(spack.config.get(\"view\", True))\n self._process_included_lockfiles()\n\n def _sync_speclists(self):\n self._spec_lists_parser = SpecListParser(\n toolchains=spack.config.CONFIG.get(\"toolchains\", {})\n )\n self.spec_lists = {}\n self.spec_lists.update(\n self._spec_lists_parser.parse_definitions(\n data=spack.config.CONFIG.get(\"definitions\", [])\n )\n )\n for group in self.manifest.groups():\n tty.debug(f\"[{__name__}]: Synchronizing user specs from the '{group}' group\", level=2)\n key = self._user_specs_key(group=group)\n self.spec_lists[key] = self._spec_lists_parser.parse_user_specs(\n name=key, yaml_list=self.manifest.user_specs(group=group)\n )\n\n def _user_specs_key(self, *, group: Optional[str] = None) -> str:\n if group is None or group == DEFAULT_USER_SPEC_GROUP:\n return USER_SPECS_KEY\n return f\"{USER_SPECS_KEY}:{group}\"\n\n @property\n def user_specs(self) -> SpecList:\n return self.user_specs_by(group=DEFAULT_USER_SPEC_GROUP)\n\n def user_specs_by(self, *, group: Optional[str]) -> SpecList:\n \"\"\"Returns a dictionary of user specs keyed by their group.\"\"\"\n key = self._user_specs_key(group=group)\n return self.spec_lists[key]\n\n def explicit_roots(self):\n for x in self.concretized_roots:\n if self.manifest.is_explicit(group=x.group):\n yield x\n\n @property\n def dev_specs(self):\n dev_specs = {}\n dev_config = spack.config.get(\"develop\", {})\n for name, entry in dev_config.items():\n local_entry = {\"spec\": str(entry[\"spec\"])}\n # default path is the spec name\n if \"path\" not in entry:\n local_entry[\"path\"] = name\n else:\n local_entry[\"path\"] = entry[\"path\"]\n dev_specs[name] = local_entry\n return dev_specs\n\n @property\n def included_user_specs(self) -> SpecList:\n \"\"\"Included concrete user (or root) specs from last concretization.\"\"\"\n spec_list = SpecList()\n\n if not self.included_concrete_env_root_dirs:\n return spec_list\n\n def add_root_specs(included_concrete_specs):\n # add specs from the include *and* any nested includes it may have\n for env, info in included_concrete_specs.items():\n for root_list in info[\"roots\"]:\n spec_list.add(root_list[\"spec\"])\n\n if lockfile_include_key in info:\n add_root_specs(info[lockfile_include_key])\n\n add_root_specs(self.included_concrete_spec_data)\n return spec_list\n\n def clear(self):\n \"\"\"Clear the contents of the environment\"\"\"\n self.spec_lists = {}\n self._dev_specs = {}\n self.concretized_roots = []\n self.specs_by_hash = {} # concretized specs by hash\n\n self.included_concrete_spec_data = {} # concretized specs from lockfile of included envs\n self.included_concretized_roots = {} # root specs of the included envs, keyed by env path\n self.included_specs_by_hash = {} # concretized specs by hash from the included envs\n\n self.invalidate_repository_cache()\n self._previous_active = None # previously active environment\n\n self.manifest.clear()\n\n @property\n def active(self):\n \"\"\"True if this environment is currently active.\"\"\"\n return _active_environment and self.path == _active_environment.path\n\n @property\n def manifest_path(self):\n \"\"\"Path to spack.yaml file in this environment.\"\"\"\n return os.path.join(self.path, manifest_name)\n\n @property\n def _transaction_lock_path(self):\n \"\"\"The location of the lock file used to synchronize multiple\n processes updating the same environment.\n \"\"\"\n return os.path.join(self.env_subdir_path, \"transaction_lock\")\n\n @property\n def lock_path(self):\n \"\"\"Path to spack.lock file in this environment.\"\"\"\n return os.path.join(self.path, lockfile_name)\n\n @property\n def _lock_backup_v1_path(self):\n \"\"\"Path to backup of v1 lockfile before conversion to v2\"\"\"\n return self.lock_path + \".backup.v1\"\n\n @property\n def repos_path(self):\n return os.path.join(self.env_subdir_path, \"repos\")\n\n @property\n def view_path_default(self) -> str:\n # default path for environment views\n return os.path.join(self.env_subdir_path, \"view\")\n\n @property\n def repo(self):\n if self._repo is None:\n self._repo = make_repo_path(self.repos_path)\n return self._repo\n\n @property\n def scope_name(self):\n \"\"\"Name of the config scope of this environment's manifest file.\"\"\"\n return self.manifest.scope_name\n\n def include_concrete_envs(self):\n \"\"\"Copy and save the included environments' specs internally.\"\"\"\n\n root_hash_seen = set()\n concrete_hash_seen = set()\n self.included_concrete_spec_data = {}\n\n for env_path in self.included_concrete_env_root_dirs:\n # Check that the environment (lockfile) exists\n if not is_env_dir(env_path):\n raise SpackEnvironmentError(f\"Unable to find env at {env_path}\")\n\n env = Environment(env_path)\n self.included_concrete_spec_data[env_path] = {\"roots\": [], \"concrete_specs\": {}}\n\n # Copy unique root specs from env\n for root_dict in env._concrete_roots_dict():\n if root_dict[\"hash\"] not in root_hash_seen:\n self.included_concrete_spec_data[env_path][\"roots\"].append(root_dict)\n root_hash_seen.add(root_dict[\"hash\"])\n\n # Copy unique concrete specs from env\n for dag_hash, spec_details in env._concrete_specs_dict().items():\n if dag_hash not in concrete_hash_seen:\n self.included_concrete_spec_data[env_path][\"concrete_specs\"].update(\n {dag_hash: spec_details}\n )\n concrete_hash_seen.add(dag_hash)\n\n # Copy transitive include data\n transitive = env.included_concrete_spec_data\n if transitive:\n self.included_concrete_spec_data[env_path][lockfile_include_key] = transitive\n\n self.unify_specs()\n self.write()\n\n def destroy(self):\n \"\"\"Remove this environment from Spack entirely.\"\"\"\n shutil.rmtree(self.path)\n\n def add(self, user_spec, list_name=USER_SPECS_KEY) -> bool:\n \"\"\"Add a single user_spec (non-concretized) to the Environment\n\n Returns:\n True if the spec was added, False if it was already present and did not need to be\n added\n\n \"\"\"\n spec = Spec(user_spec)\n\n if list_name not in self.spec_lists:\n raise SpackEnvironmentError(f\"No list {list_name} exists in environment {self.name}\")\n\n if list_name == USER_SPECS_KEY:\n if spec.anonymous:\n raise SpackEnvironmentError(\"cannot add anonymous specs to an environment\")\n elif not spack.repo.PATH.exists(spec.name) and not spec.abstract_hash:\n virtuals = spack.repo.PATH.provider_index.providers.keys()\n if spec.name not in virtuals:\n raise SpackEnvironmentError(f\"no such package: {spec.name}\")\n\n list_to_change = self.spec_lists[list_name]\n existing = str(spec) in list_to_change.yaml_list\n if not existing:\n list_to_change.add(spec)\n if list_name == USER_SPECS_KEY:\n self.manifest.add_user_spec(str(user_spec))\n else:\n self.manifest.add_definition(str(user_spec), list_name=list_name)\n self._sync_speclists()\n\n return bool(not existing)\n\n def change_existing_spec(\n self,\n change_spec: Spec,\n list_name: str = USER_SPECS_KEY,\n match_spec: Optional[Spec] = None,\n allow_changing_multiple_specs=False,\n ):\n \"\"\"\n Find the spec identified by ``match_spec`` and change it to ``change_spec``.\n\n Arguments:\n change_spec: defines the spec properties that\n need to be changed. This will not change attributes of the\n matched spec unless they conflict with ``change_spec``.\n list_name: identifies the spec list in the environment that\n should be modified\n match_spec: if set, this identifies the spec\n that should be changed. If not set, it is assumed we are\n looking for a spec with the same name as ``change_spec``.\n \"\"\"\n if not (change_spec.name or match_spec):\n raise ValueError(\n \"Must specify a spec name or match spec to identify a single spec\"\n \" in the environment that will be changed (or multiple with '--all')\"\n )\n match_spec = match_spec or Spec(change_spec.name)\n\n list_to_change = self.spec_lists[list_name]\n if list_to_change.is_matrix:\n raise SpackEnvironmentError(\n \"Cannot directly change specs in matrices:\"\n \" specify a named list that is not a matrix\"\n )\n\n matches = list((idx, x) for idx, x in enumerate(list_to_change) if x.satisfies(match_spec))\n if len(matches) == 0:\n raise ValueError(\n \"There are no specs named {0} in {1}\".format(match_spec.name, list_name)\n )\n elif len(matches) > 1 and not allow_changing_multiple_specs:\n raise ValueError(f\"{str(match_spec)} matches multiple specs\")\n\n for idx, spec in matches:\n override_spec = Spec.override(spec, change_spec)\n if list_name == USER_SPECS_KEY:\n self.manifest.override_user_spec(str(override_spec), idx=idx)\n else:\n self.manifest.override_definition(\n str(spec), override=str(override_spec), list_name=list_name\n )\n self._sync_speclists()\n\n def remove(self, query_spec, list_name=USER_SPECS_KEY, force=False):\n \"\"\"Remove specs from an environment that match a query_spec\"\"\"\n err_msg_header = (\n f\"Cannot remove '{query_spec}' from '{list_name}' definition \"\n f\"in {self.manifest.manifest_file}\"\n )\n query_spec = Spec(query_spec)\n try:\n list_to_change = self.spec_lists[list_name]\n except KeyError as e:\n msg = f\"{err_msg_header}, since '{list_name}' does not exist\"\n raise SpackEnvironmentError(msg) from e\n\n if not query_spec.concrete:\n matches = [s for s in list_to_change if s.satisfies(query_spec)]\n\n else:\n # concrete specs match against concrete specs in the env by dag hash.\n matches = [x.root for x in self.concretized_roots if query_spec.dag_hash() == x.hash]\n\n if not matches:\n raise SpackEnvironmentError(f\"{err_msg_header}, no spec matches\")\n\n old_specs = set(self.user_specs)\n\n # Remove specs from the appropriate spec list\n for spec in matches:\n if spec not in list_to_change:\n continue\n try:\n list_to_change.remove(spec)\n except SpecListError as e:\n msg = str(e)\n if force:\n msg += \" It will be removed from the concrete specs.\"\n tty.warn(msg)\n else:\n if list_name == USER_SPECS_KEY:\n self.manifest.remove_user_spec(str(spec))\n else:\n self.manifest.remove_definition(str(spec), list_name=list_name)\n\n # Recompute \"definitions\" and user specs\n self._sync_speclists()\n new_specs = set(self.user_specs)\n\n # If 'force', update stale concretized specs\n if force:\n stale_specs = old_specs - new_specs\n self.concretized_roots, removed = stable_partition(\n self.concretized_roots, lambda x: x.root not in stale_specs\n )\n for x in removed:\n del self.specs_by_hash[x.hash]\n\n def is_develop(self, spec):\n \"\"\"Returns true when the spec is built from local sources\"\"\"\n return spec.name in self.dev_specs\n\n def apply_develop(self, spec: spack.spec.Spec, path: Optional[str] = None):\n \"\"\"Mutate concrete specs to include dev_path provenance pointing to path.\n\n This will fail if any existing concrete spec for the same package does not satisfy the\n\n given develop spec.\"\"\"\n selector = spack.spec.Spec(spec.name)\n\n mutator = spack.spec.Spec()\n if path:\n variant = vt.SingleValuedVariant(\"dev_path\", path)\n else:\n variant = vt.VariantValueRemoval(\"dev_path\")\n mutator.variants[\"dev_path\"] = variant\n\n msg = (\n f\"Develop spec '{spec}' conflicts with concrete specs in environment.\"\n \" Try again with 'spack develop --no-modify-concrete-specs'\"\n \" and run 'spack concretize --force' to apply your changes.\"\n )\n self.mutate(selector, mutator, validator=spec, msg=msg)\n\n def mutate(\n self,\n selector: spack.spec.Spec,\n mutator: spack.spec.Spec,\n validator: Optional[spack.spec.Spec] = None,\n msg: Optional[str] = None,\n ):\n \"\"\"Mutate concrete specs of an environment\n\n Mutate any spec that matches ``selector``. Invalidate caches on parents of mutated specs.\n If a validator spec is supplied, throw an error if a selected spec does not satisfy the\n validator.\n \"\"\"\n # Find all specs that this mutation applies to\n modify_specs = []\n modified_specs = []\n for dep in self.all_specs_generator():\n if dep.satisfies(selector):\n if not dep.satisfies(validator or selector):\n if not msg:\n msg = f\"spec {dep} satisfies selector {selector}\"\n msg += f\" but not validator {validator}\"\n raise SpackEnvironmentDevelopError(msg)\n modify_specs.append(dep)\n\n # Manipulate selected specs\n for s in modify_specs:\n modified = s.mutate(mutator, rehash=False)\n if modified:\n modified_specs.append(s)\n\n # Identify roots modified and invalidate all dependent hashes\n modified_roots = []\n for parent in traverse.traverse_nodes(modified_specs, direction=\"parents\"):\n # record whether this parent is a root before we modify the hash\n if parent.dag_hash() in self.specs_by_hash:\n modified_roots.append((parent, parent.dag_hash()))\n # modify the parent to invalidate hashes\n parent._mark_root_concrete(False)\n parent.clear_caches()\n\n # Compute new hashes and update the env list of specs\n hash_mutations = {}\n for root, old_hash in modified_roots:\n # New hash must be computed after we finalize concretization\n root._finalize_concretization()\n new_hash = root.dag_hash()\n self.specs_by_hash.pop(old_hash)\n self.specs_by_hash[new_hash] = root\n hash_mutations[old_hash] = new_hash\n\n for x in self.concretized_roots:\n if x.hash in hash_mutations:\n x.hash = hash_mutations[x.hash]\n\n if modified_roots:\n self.write()\n\n def concretize(\n self, *, force: Optional[bool] = None, tests: Union[bool, Sequence[str]] = False\n ) -> Sequence[SpecPair]:\n \"\"\"Concretize user_specs in this environment.\n\n Only concretizes specs that haven't been concretized yet unless force is ``True``.\n\n This only modifies the environment in memory. ``write()`` will write out a lockfile\n containing concretized specs.\n\n Arguments:\n force: re-concretize ALL specs, even those that were already concretized;\n defaults to ``spack.config.get(\"concretizer:force\")``\n tests: False to run no tests, True to test all packages, or a list of\n package names to run tests for some\n\n Returns:\n List of specs that have been concretized. Each entry is a tuple of\n the user spec and the corresponding concretized spec.\n \"\"\"\n return EnvironmentConcretizer(self).concretize(force=force, tests=tests)\n\n def sync_concretized_specs(self) -> None:\n \"\"\"Removes concrete specs that no longer correlate to a user spec\"\"\"\n if not self.concretized_roots:\n return\n\n to_deconcretize, user_specs = [], self._all_user_specs_with_group()\n for x in self.concretized_roots:\n if (x.group, x.root) not in user_specs:\n to_deconcretize.append(x)\n for x in to_deconcretize:\n self.deconcretize_by_user_spec(x.root, group=x.group)\n\n def _all_user_specs_with_group(self) -> Set[Tuple[str, Spec]]:\n result = set()\n for group in self.manifest.groups():\n result.update([(group, x) for x in self.user_specs_by(group=group)])\n return result\n\n def clear_concretized_specs(self) -> None:\n \"\"\"Clears the currently concretized specs\"\"\"\n self.concretized_roots = []\n self.specs_by_hash = {}\n\n def deconcretize_by_hash(self, dag_hash: str) -> None:\n \"\"\"Removes a concrete spec from the environment concretization\"\"\"\n self.concretized_roots = [x for x in self.concretized_roots if x.hash != dag_hash]\n self._maybe_remove_dag_hash(dag_hash)\n\n def deconcretize_by_user_spec(\n self, spec: spack.spec.Spec, *, group: Optional[str] = None\n ) -> None:\n \"\"\"Removes a user spec from the environment concretization\n\n Arguments:\n spec: user spec to deconcretize\n group: group of the spec to remove. If not specified, the spec is removed from\n the default group\n \"\"\"\n group = group or DEFAULT_USER_SPEC_GROUP\n # spec has to be a root of the environment\n discarded, self.concretized_roots = stable_partition(\n self.concretized_roots, lambda x: x.group == group and x.root == spec\n )\n assert len({x.hash for x in discarded}) == 1, (\n \"More than one hash associated with a single user spec\"\n )\n dag_hash = discarded[0].hash\n self._maybe_remove_dag_hash(dag_hash)\n\n def _maybe_remove_dag_hash(self, dag_hash: str):\n # If this was the only user spec that concretized to this concrete spec, remove it\n if not self.user_spec_with_hash(dag_hash) and dag_hash in self.specs_by_hash:\n # if we deconcretized a dependency that doesn't correspond to a root, it won't be here.\n del self.specs_by_hash[dag_hash]\n\n def user_spec_with_hash(self, dag_hash: str) -> bool:\n \"\"\"Returns True if any user spec is associated with a concrete spec with the given hash\"\"\"\n return any(x.hash == dag_hash for x in self.concretized_roots)\n\n def unify_specs(self) -> None:\n # Keep the information on new specs by copying the concretized roots\n old_concretized_roots = self.concretized_roots\n self._read_lockfile_dict(self._to_lockfile_dict())\n self.concretized_roots = old_concretized_roots\n\n @property\n def default_view(self):\n if not self.has_view(default_view_name):\n raise SpackEnvironmentError(f\"{self.name} does not have a default view enabled\")\n\n return self.views[default_view_name]\n\n def has_view(self, view_name: str) -> bool:\n return view_name in self.views\n\n def update_default_view(self, path_or_bool: Union[str, bool]) -> None:\n \"\"\"Updates the path of the default view.\n\n If the argument passed as input is False the default view is deleted, if present. The\n manifest will have an entry ``view: false``.\n\n If the argument passed as input is True a default view is created, if not already present.\n The manifest will have an entry ``view: true``. If a default view is already declared, it\n will be left untouched.\n\n If the argument passed as input is a path a default view pointing to that path is created,\n if not present already. If a default view is already declared, only its \"root\" will be\n changed.\n\n Args:\n path_or_bool: either True, or False or a path\n \"\"\"\n view_path = self.view_path_default if path_or_bool is True else path_or_bool\n\n # We don't have the view, and we want to remove it\n if default_view_name not in self.views and path_or_bool is False:\n return\n\n # We want to enable the view, but we have it already\n if default_view_name in self.views and path_or_bool is True:\n return\n\n # We have the view, and we want to set it to the same path\n if default_view_name in self.views and self.default_view.root == view_path:\n return\n\n self.delete_default_view()\n if path_or_bool is False:\n self.views.pop(default_view_name, None)\n self.manifest.remove_default_view()\n return\n\n # If we had a default view already just update its path,\n # else create a new one and add it to views\n if default_view_name in self.views:\n self.default_view.update_root(view_path)\n else:\n assert isinstance(view_path, str), f\"expected str for 'view_path', but got {view_path}\"\n self.views[default_view_name] = ViewDescriptor(self.path, view_path)\n\n self.manifest.set_default_view(self._default_view_as_yaml())\n\n def delete_default_view(self) -> None:\n \"\"\"Deletes the default view associated with this environment.\"\"\"\n if default_view_name not in self.views:\n return\n\n try:\n view = pathlib.Path(self.default_view.root)\n shutil.rmtree(view.resolve())\n view.unlink()\n except FileNotFoundError as e:\n msg = f\"[ENVIRONMENT] error trying to delete the default view: {str(e)}\"\n tty.debug(msg)\n\n def regenerate_views(self):\n if not self.views:\n tty.debug(\"Skip view update, this environment does not maintain a view\")\n return\n\n for view in self.views.values():\n view.regenerate(self)\n\n def check_views(self):\n \"\"\"Checks if the environments default view can be activated.\"\"\"\n try:\n # This is effectively a no-op, but it touches all packages in the\n # default view if they are installed.\n for view_name, view in self.views.items():\n for spec in self.concrete_roots():\n if spec in view and spec.package and spec.installed:\n msg = '{0} in view \"{1}\"'\n tty.debug(msg.format(spec.name, view_name))\n\n except (spack.repo.UnknownPackageError, spack.repo.UnknownNamespaceError) as e:\n tty.warn(e)\n tty.warn(\n \"Environment %s includes out of date packages or repos. \"\n \"Loading the environment view will require reconcretization.\" % self.name\n )\n\n def _env_modifications_for_view(\n self, view: ViewDescriptor, reverse: bool = False\n ) -> spack.util.environment.EnvironmentModifications:\n try:\n with spack.store.STORE.db.read_transaction():\n installed_roots = [s for s in self.concrete_roots() if s.installed]\n mods = uenv.environment_modifications_for_specs(*installed_roots, view=view)\n except Exception as e:\n # Failing to setup spec-specific changes shouldn't be a hard error.\n tty.warn(\n f\"could not {'unload' if reverse else 'load'} runtime environment due \"\n f\"to {e.__class__.__name__}: {e}\"\n )\n return spack.util.environment.EnvironmentModifications()\n return mods.reversed() if reverse else mods\n\n def add_view_to_env(\n self, env_mod: spack.util.environment.EnvironmentModifications, view: str\n ) -> spack.util.environment.EnvironmentModifications:\n \"\"\"Collect the environment modifications to activate an environment using the provided\n view. Removes duplicate paths.\n\n Args:\n env_mod: the environment modifications object that is modified.\n view: the name of the view to activate.\"\"\"\n descriptor = self.views.get(view)\n if not descriptor:\n return env_mod\n\n env_mod.extend(uenv.unconditional_environment_modifications(descriptor))\n env_mod.extend(self._env_modifications_for_view(descriptor))\n\n # deduplicate paths from specs mapped to the same location\n for env_var in env_mod.group_by_name():\n env_mod.prune_duplicate_paths(env_var)\n\n return env_mod\n\n def rm_view_from_env(\n self, env_mod: spack.util.environment.EnvironmentModifications, view: str\n ) -> spack.util.environment.EnvironmentModifications:\n \"\"\"Collect the environment modifications to deactivate an environment using the provided\n view. Reverses the action of ``add_view_to_env``.\n\n Args:\n env_mod: the environment modifications object that is modified.\n view: the name of the view to deactivate.\"\"\"\n descriptor = self.views.get(view)\n if not descriptor:\n return env_mod\n\n env_mod.extend(uenv.unconditional_environment_modifications(descriptor).reversed())\n env_mod.extend(self._env_modifications_for_view(descriptor, reverse=True))\n\n return env_mod\n\n def add_concrete_spec(\n self,\n spec: spack.spec.Spec,\n concrete: spack.spec.Spec,\n *,\n new: bool = True,\n group: Optional[str] = None,\n ):\n \"\"\"Called when a new concretized spec is added to the environment.\n\n This ensures that all internal data structures are kept in sync.\n\n Arguments:\n spec: user spec that resulted in the concrete spec\n concrete: spec concretized within this environment\n new: whether to write this spec's package to the env repo on write()\n \"\"\"\n assert concrete.concrete\n h = concrete.dag_hash()\n group = group or DEFAULT_USER_SPEC_GROUP\n self.concretized_roots.append(\n ConcretizedRootInfo(root_spec=spec, root_hash=h, new=new, group=group)\n )\n self.specs_by_hash[h] = concrete\n\n def _dev_specs_that_need_overwrite(self):\n \"\"\"Return the hashes of all specs that need to be reinstalled due to source code change.\"\"\"\n changed_dev_specs = [\n s\n for s in traverse.traverse_nodes(\n self.concrete_roots(), order=\"breadth\", key=traverse.by_dag_hash\n )\n if _is_dev_spec_and_has_changed(s)\n ]\n\n # Collect their hashes, and the hashes of their installed parents.\n # Notice: with order=breadth all changed dev specs are at depth 0,\n # even if they occur as parents of one another.\n return [\n spec.dag_hash()\n for depth, spec in traverse.traverse_nodes(\n changed_dev_specs,\n root=True,\n order=\"breadth\",\n depth=True,\n direction=\"parents\",\n key=traverse.by_dag_hash,\n )\n if depth == 0 or spec.installed\n ]\n\n def _partition_roots_by_install_status(self):\n \"\"\"Partition root specs into those that do not have to be passed to the\n installer, and those that should be, taking into account development\n specs. This is done in a single read transaction per environment instead\n of per spec.\"\"\"\n with spack.store.STORE.db.read_transaction():\n uninstalled, installed = stable_partition(self.concrete_roots(), _is_uninstalled)\n return installed, uninstalled\n\n def uninstalled_specs(self):\n \"\"\"Return root specs that are not installed, or are installed, but\n are development specs themselves or have those among their dependencies.\"\"\"\n return self._partition_roots_by_install_status()[1]\n\n def install_all(self, **install_args):\n \"\"\"Install all concretized specs in an environment.\n\n Note: this does not regenerate the views for the environment;\n that needs to be done separately with a call to write().\n\n Args:\n install_args (dict): keyword install arguments\n \"\"\"\n self.install_specs(None, **install_args)\n\n def install_specs(self, specs: Optional[List[Spec]] = None, **install_args):\n roots = self.concrete_roots()\n specs = specs if specs is not None else roots\n\n # Extract reporter arguments\n reporter = install_args.pop(\"reporter\", None)\n report_file = install_args.pop(\"report_file\", None)\n\n # Extend the set of specs to overwrite with modified dev specs and their parents\n install_args[\"overwrite\"] = {\n *install_args.get(\"overwrite\", ()),\n *self._dev_specs_that_need_overwrite(),\n }\n\n # Only environment roots in explicit groups are marked explicit\n install_args[\"explicit\"] = {\n *install_args.get(\"explicit\", ()),\n *(x.hash for x in self.explicit_roots()),\n }\n\n builder = spack.installer_dispatch.create_installer(\n [spec.package for spec in specs], create_reports=reporter is not None, **install_args\n )\n\n try:\n builder.install()\n finally:\n if reporter:\n if isinstance(builder.reports, dict):\n reporter.build_report(report_file, list(builder.reports.values()))\n elif isinstance(builder.reports, list):\n reporter.build_report(report_file, builder.reports)\n else:\n raise TypeError(\"builder.reports must be either a dictionary or a list\")\n\n def all_specs_generator(self) -> Iterable[Spec]:\n \"\"\"Returns a generator for all concrete specs\"\"\"\n return traverse.traverse_nodes(self.concrete_roots(), key=traverse.by_dag_hash)\n\n def all_specs(self) -> List[Spec]:\n \"\"\"Returns a list of all concrete specs\"\"\"\n return list(self.all_specs_generator())\n\n def all_hashes(self):\n \"\"\"Return hashes of all specs.\"\"\"\n return [s.dag_hash() for s in self.all_specs_generator()]\n\n def roots(self):\n \"\"\"Specs explicitly requested by the user *in this environment*.\n\n Yields both added and installed specs that have user specs in\n ``spack.yaml``.\n \"\"\"\n concretized = dict(self.concretized_specs())\n for spec in self.user_specs:\n concrete = concretized.get(spec)\n yield concrete if concrete else spec\n\n def added_specs(self):\n \"\"\"Specs that are not yet installed.\n\n Yields the user spec for non-concretized specs, and the concrete\n spec for already concretized but not yet installed specs.\n \"\"\"\n # use a transaction to avoid overhead of repeated calls\n # to `package.spec.installed`\n with spack.store.STORE.db.read_transaction():\n concretized = dict(self.concretized_specs())\n for spec in self.user_specs:\n concrete = concretized.get(spec)\n if not concrete:\n yield spec\n elif not concrete.installed:\n yield concrete\n\n def concretized_specs(self):\n \"\"\"Tuples of (user spec, concrete spec) for all concrete specs.\"\"\"\n for x in self.concretized_roots:\n yield x.root, self.specs_by_hash[x.hash]\n\n yield from self.concretized_specs_from_all_included_environments()\n\n def concretized_specs_from_all_included_environments(self):\n seen = {(x.root, x.hash) for x in self.concretized_roots}\n for included_env in self.included_concretized_roots:\n yield from self.concretized_specs_from_included_environment(included_env, _seen=seen)\n\n def concretized_specs_from_included_environment(\n self, included_env: str, *, _seen: Optional[Set[Tuple[spack.spec.Spec, str]]] = None\n ):\n _seen = set() if _seen is None else _seen\n for x in self.included_concretized_roots[included_env]:\n if (x.root, x.hash) in _seen:\n continue\n _seen.add((x.root, x.hash))\n yield x.root, self.included_specs_by_hash[included_env][x.hash]\n\n def concrete_roots(self):\n \"\"\"Same as concretized_specs, except it returns the list of concrete\n roots *without* associated user spec\"\"\"\n return [root for _, root in self.concretized_specs()]\n\n def concretized_specs_by(self, *, group: str) -> Iterable[Tuple[Spec, Spec]]:\n \"\"\"Generates all the (abstract, concrete) spec pairs for a given group\"\"\"\n for x in self.concretized_roots:\n if x.group != group:\n continue\n yield x.root, self.specs_by_hash[x.hash]\n\n def get_by_hash(self, dag_hash: str) -> List[Spec]:\n # If it's not a partial hash prefix we can early exit\n early_exit = len(dag_hash) == 32\n matches = []\n for spec in traverse.traverse_nodes(\n self.concrete_roots(), key=traverse.by_dag_hash, order=\"breadth\"\n ):\n if spec.dag_hash().startswith(dag_hash):\n matches.append(spec)\n if early_exit:\n break\n return matches\n\n def get_one_by_hash(self, dag_hash):\n \"\"\"Returns the single spec from the environment which matches the\n provided hash. Raises an AssertionError if no specs match or if\n more than one spec matches.\"\"\"\n hash_matches = self.get_by_hash(dag_hash)\n assert len(hash_matches) == 1\n return hash_matches[0]\n\n def all_matching_specs(self, *specs: spack.spec.Spec) -> List[Spec]:\n \"\"\"Returns all concretized specs in the environment satisfying any of the input specs\"\"\"\n return [\n s\n for s in traverse.traverse_nodes(self.concrete_roots(), key=traverse.by_dag_hash)\n if any(s.satisfies(t) for t in specs)\n ]\n\n @spack.repo.autospec\n def matching_spec(self, spec):\n \"\"\"\n Given a spec (likely not concretized), find a matching concretized\n spec in the environment.\n\n The matching spec does not have to be installed in the environment,\n but must be concrete (specs added with ``spack add`` without an\n intervening ``spack concretize`` will not be matched).\n\n If there is a single root spec that matches the provided spec or a\n single dependency spec that matches the provided spec, then the\n concretized instance of that spec will be returned.\n\n If multiple root specs match the provided spec, or no root specs match\n and multiple dependency specs match, then this raises an error\n and reports all matching specs.\n \"\"\"\n env_root_to_user = {root.dag_hash(): user for user, root in self.concretized_specs()}\n root_matches, dep_matches = [], []\n\n for env_spec in traverse.traverse_nodes(\n specs=[root for _, root in self.concretized_specs()],\n key=traverse.by_dag_hash,\n order=\"breadth\",\n ):\n if not env_spec.satisfies(spec):\n continue\n\n # If the spec is concrete, then there is no possibility of multiple matches,\n # and we immediately return the single match\n if spec.concrete:\n return env_spec\n\n # Distinguish between environment roots and deps. Specs that are both\n # are classified as environment roots.\n user_spec = env_root_to_user.get(env_spec.dag_hash())\n if user_spec:\n root_matches.append((env_spec, user_spec))\n else:\n dep_matches.append(env_spec)\n\n # No matching spec\n if not root_matches and not dep_matches:\n return None\n\n # Single root spec, any number of dep specs => return root spec.\n if len(root_matches) == 1:\n return root_matches[0][0]\n\n if not root_matches and len(dep_matches) == 1:\n return dep_matches[0]\n\n # More than one spec matched, and either multiple roots matched or\n # none of the matches were roots\n # If multiple root specs match, it is assumed that the abstract\n # spec will most-succinctly summarize the difference between them\n # (and the user can enter one of these to disambiguate)\n fmt_str = \"{hash:7} \" + spack.spec.DEFAULT_FORMAT\n color = clr.get_color_when()\n match_strings = [\n f\"Root spec {abstract.format(color=color)}\\n {concrete.format(fmt_str, color=color)}\"\n for concrete, abstract in root_matches\n ]\n match_strings.extend(\n f\"Dependency spec\\n {s.format(fmt_str, color=color)}\" for s in dep_matches\n )\n matches_str = \"\\n\".join(match_strings)\n\n raise SpackEnvironmentError(\n f\"{spec} matches multiple specs in the environment {self.name}: \\n{matches_str}\"\n )\n\n def removed_specs(self):\n \"\"\"Tuples of (user spec, concrete spec) for all specs that will be\n removed on next concretize.\"\"\"\n needed = set()\n for s, c in self.concretized_specs():\n if s in self.user_specs:\n for d in c.traverse():\n needed.add(d)\n\n for s, c in self.concretized_specs():\n for d in c.traverse():\n if d not in needed:\n yield d\n\n def _concrete_specs_dict(self):\n concrete_specs = {}\n for s in traverse.traverse_nodes(self.specs_by_hash.values(), key=traverse.by_dag_hash):\n spec_dict = s.node_dict_with_hashes(hash=ht.dag_hash)\n # Assumes no legacy formats, since this was just created.\n spec_dict[ht.dag_hash.name] = s.dag_hash()\n concrete_specs[s.dag_hash()] = spec_dict\n\n if s.build_spec is not s:\n for d in s.build_spec.traverse():\n build_spec_dict = d.node_dict_with_hashes(hash=ht.dag_hash)\n build_spec_dict[ht.dag_hash.name] = d.dag_hash()\n concrete_specs[d.dag_hash()] = build_spec_dict\n\n return concrete_specs\n\n def _concrete_roots_dict(self):\n if not self.has_groups():\n return [{\"hash\": x.hash, \"spec\": str(x.root)} for x in self.concretized_roots]\n\n return [\n {\"hash\": x.hash, \"spec\": str(x.root), \"group\": x.group} for x in self.concretized_roots\n ]\n\n def has_groups(self) -> bool:\n groups = self.manifest.groups()\n # True if groups != {DEFAULT_USER_SPEC_GROUP}\n return len(groups) != 1 or DEFAULT_USER_SPEC_GROUP not in groups\n\n def _to_lockfile_dict(self):\n \"\"\"Create a dictionary to store a lockfile for this environment.\"\"\"\n lockfile_version = CURRENT_LOCKFILE_VERSION if self.has_groups() else 6\n concrete_specs = self._concrete_specs_dict()\n root_specs = self._concrete_roots_dict()\n\n spack_dict = {\"version\": spack.spack_version}\n spack_commit = spack.get_spack_commit()\n if spack_commit:\n spack_dict[\"type\"] = \"git\"\n spack_dict[\"commit\"] = spack_commit\n else:\n spack_dict[\"type\"] = \"release\"\n\n # this is the lockfile we'll write out\n data = {\n # metadata about the format\n \"_meta\": {\n \"file-type\": \"spack-lockfile\",\n \"lockfile-version\": lockfile_version,\n \"specfile-version\": spack.spec.SPECFILE_FORMAT_VERSION,\n },\n # spack version information\n \"spack\": spack_dict,\n # users specs + hashes are the 'roots' of the environment\n \"roots\": root_specs,\n # Concrete specs by hash, including dependencies\n \"concrete_specs\": concrete_specs,\n }\n\n if self.included_concrete_env_root_dirs:\n data[lockfile_include_key] = self.included_concrete_spec_data\n\n return data\n\n def _read_lockfile(self, file_or_json):\n \"\"\"Read a lockfile from a file or from a raw string.\"\"\"\n lockfile_dict = sjson.load(file_or_json)\n self._read_lockfile_dict(lockfile_dict)\n return lockfile_dict\n\n def _set_included_env_roots(\n self,\n env_name: str,\n env_info: Dict[str, Dict[str, Any]],\n included_json_specs_by_hash: Dict[str, Dict[str, Any]],\n ) -> Dict[str, Dict[str, Any]]:\n \"\"\"Populates included_concretized_roots from included environment data,\n including any transitively nested included environments.\n\n Args:\n env_name: the path of the included environment\n env_info: included concrete environment data\n included_json_specs_by_hash: concrete spec data keyed by hash\n\n Returns: updated specs_by_hash\n \"\"\"\n self.included_concretized_roots[env_name] = []\n\n def add_specs(name, info, specs_by_hash):\n # Add specs from the environment as well as any of its nested\n # environments.\n for root_info in info[\"roots\"]:\n self.included_concretized_roots[name].append(\n ConcretizedRootInfo.from_info_dict(root_info)\n )\n if \"concrete_specs\" in info:\n specs_by_hash.update(info[\"concrete_specs\"])\n\n if lockfile_include_key in info:\n for included_name, included_info in info[lockfile_include_key].items():\n if included_name not in self.included_concretized_roots:\n self.included_concretized_roots[included_name] = []\n add_specs(included_name, included_info, specs_by_hash)\n\n add_specs(env_name, env_info, included_json_specs_by_hash)\n return included_json_specs_by_hash\n\n def _read_lockfile_dict(self, d):\n \"\"\"Read a lockfile dictionary into this environment.\"\"\"\n self.specs_by_hash = {}\n self.included_specs_by_hash = {}\n self.included_concretized_roots = {}\n\n roots = d[\"roots\"]\n self.concretized_roots = [ConcretizedRootInfo.from_info_dict(r) for r in roots]\n\n json_specs_by_hash = d[\"concrete_specs\"]\n included_json_specs_by_hash = {}\n\n if lockfile_include_key in d:\n for env_name, env_info in d[lockfile_include_key].items():\n included_json_specs_by_hash.update(\n self._set_included_env_roots(env_name, env_info, included_json_specs_by_hash)\n )\n\n current_lockfile_format = d[\"_meta\"][\"lockfile-version\"]\n try:\n reader = READER_CLS[current_lockfile_format]\n except KeyError:\n msg = (\n f\"Spack {spack.__version__} cannot read the lockfile '{self.lock_path}', using \"\n f\"the v{current_lockfile_format} format.\"\n )\n if CURRENT_LOCKFILE_VERSION < current_lockfile_format:\n msg += \" You need to use a newer Spack version.\"\n raise SpackEnvironmentError(msg)\n\n concretized_order = [x.hash for x in self.concretized_roots]\n first_seen, concretized_order = self._filter_specs(\n reader, json_specs_by_hash, concretized_order\n )\n for idx, spec_dag_hash in enumerate(concretized_order):\n self.concretized_roots[idx].hash = spec_dag_hash\n self.specs_by_hash[spec_dag_hash] = first_seen[spec_dag_hash]\n\n if any(self.included_concretized_roots.values()):\n first_seen = {}\n\n for env_name, roots in self.included_concretized_roots.items():\n order = [x.hash for x in roots]\n filtered_spec, new_order = self._filter_specs(\n reader, included_json_specs_by_hash, order\n )\n first_seen.update(filtered_spec)\n for idx, spec_dag_hash in enumerate(new_order):\n roots[idx].hash = spec_dag_hash\n\n for env_path, roots in self.included_concretized_roots.items():\n self.included_specs_by_hash[env_path] = {x.hash: first_seen[x.hash] for x in roots}\n\n def _filter_specs(self, reader, json_specs_by_hash, order_concretized):\n # Track specs by their lockfile key. Currently, spack uses the finest\n # grained hash as the lockfile key, while older formats used the build\n # hash or a previous incarnation of the DAG hash (one that did not\n # include build deps or package hash).\n specs_by_hash = {}\n\n # Track specs by their DAG hash, allows handling DAG hash collisions\n first_seen = {}\n\n # First pass: Put each spec in the map ignoring dependencies\n for lockfile_key, node_dict in json_specs_by_hash.items():\n spec = reader.from_node_dict(node_dict)\n if not spec._hash:\n # in v1 lockfiles, the hash only occurs as a key\n spec._hash = lockfile_key\n specs_by_hash[lockfile_key] = spec\n\n # Second pass: For each spec, get its dependencies from the node dict\n # and add them to the spec, including build specs\n for lockfile_key, node_dict in json_specs_by_hash.items():\n name, data = reader.name_and_data(node_dict)\n for _, dep_hash, deptypes, _, virtuals, direct in reader.dependencies_from_node_dict(\n data\n ):\n specs_by_hash[lockfile_key]._add_dependency(\n specs_by_hash[dep_hash],\n depflag=dt.canonicalize(deptypes),\n virtuals=virtuals,\n direct=direct,\n )\n\n if \"build_spec\" in node_dict:\n _, bhash, _ = reader.extract_build_spec_info_from_node_dict(node_dict)\n specs_by_hash[lockfile_key]._build_spec = specs_by_hash[bhash]\n\n # Traverse the root specs one at a time in the order they appear.\n # The first time we see each DAG hash, that's the one we want to\n # keep. This is only required as long as we support older lockfile\n # formats where the mapping from DAG hash to lockfile key is possibly\n # one-to-many.\n\n for lockfile_key in order_concretized:\n for s in specs_by_hash[lockfile_key].traverse():\n if s.dag_hash() not in first_seen:\n first_seen[s.dag_hash()] = s\n\n # Now make sure concretized_order and our internal specs dict\n # contains the keys used by modern spack (i.e. the dag_hash\n # that includes build deps and package hash).\n\n order_concretized = [specs_by_hash[h_key].dag_hash() for h_key in order_concretized]\n\n return first_seen, order_concretized\n\n def write(self, regenerate: bool = True) -> None:\n \"\"\"Writes an in-memory environment to its location on disk.\n\n Write out package files for each newly concretized spec. Also\n regenerate any views associated with the environment and run post-write\n hooks, if regenerate is True.\n\n Args:\n regenerate: regenerate views and run post-write hooks as well as writing if True.\n \"\"\"\n self.manifest_uptodate_or_warn()\n if self.specs_by_hash or self.included_concrete_env_root_dirs:\n self.ensure_env_directory_exists(dot_env=True)\n self.update_environment_repository()\n self.manifest.flush()\n # Write the lock file last. This is useful for Makefiles\n # with `spack.lock: spack.yaml` rules, where the target\n # should be newer than the prerequisite to avoid\n # redundant re-concretization.\n self.update_lockfile()\n else:\n self.ensure_env_directory_exists(dot_env=False)\n with fs.safe_remove(self.lock_path):\n self.manifest.flush()\n\n if regenerate:\n self.regenerate_views()\n\n for x in self.concretized_roots:\n x.new = False\n\n def update_lockfile(self) -> None:\n with fs.write_tmp_and_move(self.lock_path, encoding=\"utf-8\") as f:\n sjson.dump(self._to_lockfile_dict(), stream=f)\n\n def ensure_env_directory_exists(self, dot_env: bool = False) -> None:\n \"\"\"Ensure that the root directory of the environment exists\n\n Args:\n dot_env: if True also ensures that the /.env directory exists\n \"\"\"\n fs.mkdirp(self.path)\n if dot_env:\n fs.mkdirp(self.env_subdir_path)\n\n def update_environment_repository(self) -> None:\n \"\"\"Updates the repository associated with the environment.\"\"\"\n new_specs = [self.specs_by_hash[x.hash] for x in self.concretized_roots if x.new]\n for spec in traverse.traverse_nodes(new_specs):\n if not spec.concrete:\n raise ValueError(\"specs passed to environment.write() must be concrete!\")\n\n self._add_to_environment_repository(spec)\n\n def _add_to_environment_repository(self, spec_node: Spec) -> None:\n \"\"\"Add the root node of the spec to the environment repository\"\"\"\n namespace: str = spec_node.namespace\n repository = spack.repo.create_or_construct(\n root=os.path.join(self.repos_path, namespace),\n namespace=namespace,\n package_api=spack.repo.PATH.get_repo(namespace).package_api,\n )\n pkg_dir = repository.dirname_for_package_name(spec_node.name)\n fs.mkdirp(pkg_dir)\n spack.repo.PATH.dump_provenance(spec_node, pkg_dir)\n\n def manifest_uptodate_or_warn(self):\n \"\"\"Emits a warning if the manifest file is not up-to-date.\"\"\"\n if not is_latest_format(self.manifest_path):\n ver = \".\".join(str(s) for s in spack.spack_version_info[:2])\n msg = (\n 'The environment \"{}\" is written to disk in a deprecated format. '\n \"Please update it using:\\n\\n\"\n \"\\tspack env update {}\\n\\n\"\n \"Note that versions of Spack older than {} may not be able to \"\n \"use the updated configuration.\"\n )\n warnings.warn(msg.format(self.name, self.name, ver))\n\n def _default_view_as_yaml(self):\n \"\"\"This internal function assumes the default view is set\"\"\"\n path = self.default_view.raw_root\n if (\n self.default_view == ViewDescriptor(self.path, self.view_path_default)\n and len(self.views) == 1\n ):\n return True\n\n if self.default_view == ViewDescriptor(self.path, path) and len(self.views) == 1:\n return path\n\n return self.default_view.to_dict()\n\n def invalidate_repository_cache(self):\n self._repo = None\n\n def __enter__(self):\n self._previous_active = _active_environment\n if self._previous_active:\n deactivate()\n activate(self)\n return self\n\n def __exit__(self, exc_type, exc_val, exc_tb):\n deactivate()\n if self._previous_active:\n activate(self._previous_active)\n\n\ndef _is_uninstalled(spec):\n return not spec.installed or (spec.satisfies(\"dev_path=*\") or spec.satisfies(\"^dev_path=*\"))\n\n\nclass ReusableSpecsFactory:\n \"\"\"Creates a list of SpecFilters to generate the reusable specs for the environment\"\"\"\n\n def __init__(self, *, env: Environment, group: str):\n self.env = env\n self.group = group\n\n @staticmethod\n def _const(specs: List[Spec]) -> Callable[[], List[Spec]]:\n \"\"\"Returns a zero-argument callable that always returns the given list.\"\"\"\n return lambda: specs\n\n def __call__(\n self, is_usable: Callable[[Spec], bool], configuration: spack.config.Configuration\n ) -> List[SpecFilter]:\n result = []\n # Specs from group dependencies _must_ be reused, regardless of configuration\n dependencies = self.env.manifest.needs(group=self.group)\n necessary_specs = []\n for d in dependencies:\n necessary_specs.extend([x for _, x in self.env.concretized_specs_by(group=d)])\n\n # Specs from groups listed as dependencies\n if necessary_specs:\n necessary_specs = list(\n traverse.traverse_nodes(necessary_specs, deptype=(\"link\", \"run\"))\n )\n result.append(\n SpecFilter(\n self._const(necessary_specs), include=[], exclude=[], is_usable=is_usable\n )\n )\n\n # Included environments and _this_ group, instead, are subject to configuration\n concretizer_yaml = configuration.get_config(\"concretizer\")\n reuse_yaml = concretizer_yaml.get(\"reuse\", False)\n\n # With no reuse don't account for previously concretized specs in _this_ group\n if reuse_yaml is False:\n return result\n\n this_group_specs = [x for _, x in self.env.concretized_specs_by(group=self.group)]\n included_specs = [\n x for _, x in self.env.concretized_specs_from_all_included_environments()\n ]\n additional_specs = list(traverse.traverse_nodes(this_group_specs + included_specs))\n if not isinstance(reuse_yaml, Mapping):\n result.append(\n SpecFilter(\n self._const(additional_specs), include=[], exclude=[], is_usable=is_usable\n )\n )\n return result\n\n # Here we know we have a complex reuse configuration\n default_include = reuse_yaml.get(\"include\", [])\n default_exclude = reuse_yaml.get(\"exclude\", [])\n for source in reuse_yaml.get(\"from\", []):\n # We just need to take care of the environment-related parts\n if source[\"type\"] != \"environment\":\n continue\n\n include = source.get(\"include\", default_include)\n exclude = source.get(\"exclude\", default_exclude)\n if \"path\" not in source:\n result.append(\n SpecFilter(\n self._const(additional_specs),\n include=include,\n exclude=exclude,\n is_usable=is_usable,\n )\n )\n continue\n\n env_dir = as_env_dir(source[\"path\"])\n if env_dir in self.env.included_concrete_env_root_dirs:\n spec_pairs_from_included_envs = [\n x for _, x in self.env.concretized_specs_from_included_environment(env_dir)\n ]\n included_specs = list(traverse.traverse_nodes(spec_pairs_from_included_envs))\n result.append(\n SpecFilter(\n self._const(included_specs),\n include=include,\n exclude=exclude,\n is_usable=is_usable,\n )\n )\n\n return result\n\n\nclass EnvironmentConcretizer:\n def __init__(self, env: Environment):\n self.env = env\n\n def concretize(\n self, *, force: Optional[bool] = None, tests: Union[bool, Sequence[str]] = False\n ) -> List[SpecPair]:\n if force is None:\n force = spack.config.get(\"concretizer:force\")\n self._prepare_environment_for_concretization(force=force)\n\n result = []\n # Sort so that the ordering is deterministic, and \"default\" specs are first\n for current_group in self._order_groups():\n with self.env.config_override_for_group(group=current_group):\n partial_result = self._concretize_single_group(group=current_group, tests=tests)\n result.extend(partial_result)\n\n # Unify the specs objects, so we get correct references to all parents\n if result:\n self.env.unify_specs()\n return result\n\n def _concretize_single_group(\n self, *, group: str, tests: Union[bool, Sequence[str]]\n ) -> List[SpecPair]:\n # Exit early if the set of concretized specs is the set of user specs\n new_user_specs, kept_user_specs = self._partition_user_specs(group=group)\n if not new_user_specs:\n return []\n\n # Pick the right concretization strategy\n if group != DEFAULT_USER_SPEC_GROUP:\n tty.msg(f\"Concretizing the '{group}' group of specs\")\n unify = spack.config.CONFIG.get_config(\"concretizer\").get(\"unify\", False)\n factory = ReusableSpecsFactory(env=self.env, group=group)\n if unify == \"when_possible\":\n partial_result = self._concretize_together_where_possible(\n new_user_specs, kept_user_specs, tests=tests, group=group, factory=factory\n )\n\n elif unify is True:\n partial_result = self._concretize_together(\n new_user_specs, kept_user_specs, tests=tests, group=group, factory=factory\n )\n\n elif unify is False:\n partial_result = self._concretize_separately(\n new_user_specs, kept_user_specs, tests=tests, group=group, factory=factory\n )\n else:\n raise SpackEnvironmentError(f\"concretization strategy not implemented [{unify}]\")\n\n return partial_result\n\n def _prepare_environment_for_concretization(self, *, force: bool):\n \"\"\"Reset the environment concrete state and ensure consistency with user specs.\"\"\"\n if force:\n self.env.clear_concretized_specs()\n else:\n self.env.sync_concretized_specs()\n\n # If a combined env, check updated spec is in the linked envs\n if self.env.included_concrete_env_root_dirs:\n self.env.include_concrete_envs()\n\n def _partition_user_specs(\n self, *, group: str\n ) -> Tuple[List[spack.spec.Spec], List[spack.spec.Spec]]:\n \"\"\"Splits the users specs in the list of the ones to be computed, and the list of\n the ones to retain.\n \"\"\"\n concretized_user_specs = {x.root for x in self.env.concretized_roots if x.group == group}\n kept_user_specs, new_user_specs = stable_partition(\n self.env.user_specs_by(group=group), lambda x: x in concretized_user_specs\n )\n kept_user_specs += self.env.included_user_specs\n return new_user_specs, kept_user_specs\n\n def _order_groups(self) -> List[str]:\n done, result = {DEFAULT_USER_SPEC_GROUP}, [DEFAULT_USER_SPEC_GROUP]\n all_groups = self.env.manifest.groups()\n remaining = all_groups - {DEFAULT_USER_SPEC_GROUP}\n\n # Validate upfront that all 'needs' references point to defined groups\n for group in remaining:\n for dep in self.env.manifest.needs(group=group):\n if dep not in all_groups:\n raise SpackEnvironmentConfigError(\n f\"group '{group}' needs '{dep}', but '{dep}' is not a defined group\",\n self.env.manifest.manifest_file,\n )\n\n while remaining:\n # Check we have groups that are \"ready\"\n ready = []\n for current in remaining:\n deps = self.env.manifest.needs(group=current)\n if all(d in done for d in deps):\n ready.append(current)\n\n # Check we can progress - if nothing is ready, there is a cycle\n if not ready:\n raise SpackEnvironmentConfigError(\n f\"cyclic dependency detected among groups: {', '.join(sorted(remaining))}\",\n self.env.manifest.manifest_file,\n )\n\n result.extend(ready)\n done.update(ready)\n remaining.difference_update(ready)\n return result\n\n def _user_spec_pairs(\n self, user_specs_to_compute: List[Spec], user_specs_to_keep: List[Spec]\n ) -> List[SpecPair]:\n specs_to_concretize = [(s, None) for s in user_specs_to_compute] + [\n (abstract, concrete)\n for abstract, concrete in self.env.concretized_specs()\n if abstract in user_specs_to_keep\n ]\n return specs_to_concretize\n\n def _concretize_together_where_possible(\n self,\n to_compute: List[Spec],\n to_keep: List[Spec],\n *,\n group: Optional[str] = None,\n tests: Union[bool, Sequence] = False,\n factory: ReusableSpecsFactory,\n ) -> List[SpecPair]:\n import spack.concretize\n\n specs_to_concretize = self._user_spec_pairs(to_compute, to_keep)\n result = spack.concretize.concretize_together_when_possible(\n specs_to_concretize, tests=tests, factory=factory\n )\n result = [x for x in result if x[0] in to_compute]\n for abstract, concrete in result:\n self.env.add_concrete_spec(abstract, concrete, new=True, group=group)\n\n return result\n\n def _concretize_together(\n self,\n to_compute: List[Spec],\n to_keep: List[Spec],\n *,\n group: Optional[str] = None,\n tests: Union[bool, Sequence] = False,\n factory: ReusableSpecsFactory,\n ) -> List[SpecPair]:\n import spack.concretize\n\n to_concretize = self._user_spec_pairs(to_compute, to_keep)\n try:\n concrete_pairs = spack.concretize.concretize_together(\n to_concretize, tests=tests, factory=factory\n )\n except spack.error.UnsatisfiableSpecError as e:\n # \"Enhance\" the error message for multiple root specs, suggest a less strict\n # form of concretization.\n if len(self.env.user_specs_by(group=group)) > 1:\n e.message += \". \"\n if to_keep:\n e.message += (\n \"Couldn't concretize without changing the existing environment. \"\n \"If you are ok with changing it, try `spack concretize --force`. \"\n )\n e.message += (\n \"You could consider setting `concretizer:unify` to `when_possible` \"\n \"or `false` to allow multiple versions of some packages.\"\n )\n raise\n\n # Return the portion of the return value that is new\n result = concrete_pairs[: len(to_compute)]\n for abstract, concrete in result:\n self.env.add_concrete_spec(abstract, concrete, new=True, group=group)\n return result\n\n def _concretize_separately(\n self,\n to_compute: List[Spec],\n to_keep: List[Spec],\n *,\n group: Optional[str] = None,\n tests: Union[bool, Sequence] = False,\n factory: ReusableSpecsFactory,\n ) -> List[SpecPair]:\n \"\"\"Concretization strategy that concretizes separately one user spec after the other\"\"\"\n import spack.concretize\n\n to_concretize = [(x, None) for x in to_compute]\n concrete_pairs = spack.concretize.concretize_separately(\n to_concretize, tests=tests, factory=factory\n )\n\n for abstract, concrete in concrete_pairs:\n self.env.add_concrete_spec(abstract, concrete, new=True, group=group)\n\n return concrete_pairs\n\n\ndef yaml_equivalent(first, second) -> bool:\n \"\"\"Returns whether two spack yaml items are equivalent, including overrides\"\"\"\n # YAML has timestamps and dates, but we don't use them yet in schemas\n if isinstance(first, dict):\n return isinstance(second, dict) and _equiv_dict(first, second)\n elif isinstance(first, list):\n return isinstance(second, list) and _equiv_list(first, second)\n elif isinstance(first, bool):\n return isinstance(second, bool) and first is second\n elif isinstance(first, int):\n return isinstance(second, int) and first == second\n elif first is None:\n return second is None\n else: # it's a string\n return isinstance(second, str) and first == second\n\n\ndef _equiv_list(first, second):\n \"\"\"Returns whether two spack yaml lists are equivalent, including overrides\"\"\"\n if len(first) != len(second):\n return False\n return all(yaml_equivalent(f, s) for f, s in zip(first, second))\n\n\ndef _equiv_dict(first, second):\n \"\"\"Returns whether two spack yaml dicts are equivalent, including overrides\"\"\"\n if len(first) != len(second):\n return False\n same_values = all(yaml_equivalent(fv, sv) for fv, sv in zip(first.values(), second.values()))\n same_keys_with_same_overrides = all(\n fk == sk and getattr(fk, \"override\", False) == getattr(sk, \"override\", False)\n for fk, sk in zip(first.keys(), second.keys())\n )\n return same_values and same_keys_with_same_overrides\n\n\ndef display_specs(specs: List[spack.spec.Spec], *, highlight_non_defaults: bool = False) -> None:\n \"\"\"Displays a list of specs traversed breadth-first, covering nodes, with install status.\n\n Args:\n specs: list of specs to be displayed\n highlight_non_defaults: if True, highlights non-default versions and variants in the specs\n being displayed\n \"\"\"\n tree_string = spack.spec.tree(\n specs,\n format=spack.spec.DISPLAY_FORMAT,\n hashes=True,\n hashlen=7,\n status_fn=spack.spec.Spec.install_status,\n highlight_version_fn=(\n spack.package_base.non_preferred_version if highlight_non_defaults else None\n ),\n highlight_variant_fn=(\n spack.package_base.non_default_variant if highlight_non_defaults else None\n ),\n key=traverse.by_dag_hash,\n )\n print(tree_string)\n\n\ndef make_repo_path(root):\n \"\"\"Make a RepoPath from the repo subdirectories in an environment.\"\"\"\n repos = (\n spack.repo.from_path(os.path.dirname(p))\n for p in glob.glob(os.path.join(root, \"**\", \"repo.yaml\"), recursive=True)\n )\n return spack.repo.RepoPath(*repos)\n\n\ndef manifest_file(env_name_or_dir):\n \"\"\"Return the absolute path to a manifest file given the environment\n name or directory.\n\n Args:\n env_name_or_dir (str): either the name of a valid environment\n or a directory where a manifest file resides\n\n Raises:\n AssertionError: if the environment is not found\n \"\"\"\n env_dir = None\n if is_env_dir(env_name_or_dir):\n env_dir = os.path.abspath(env_name_or_dir)\n elif exists(env_name_or_dir):\n env_dir = os.path.abspath(root(env_name_or_dir))\n\n assert env_dir, \"environment not found [env={0}]\".format(env_name_or_dir)\n return os.path.join(env_dir, manifest_name)\n\n\ndef update_yaml(manifest, backup_file):\n \"\"\"Update a manifest file from an old format to the current one.\n\n Args:\n manifest (str): path to a manifest file\n backup_file (str): file where to copy the original manifest\n\n Returns:\n True if the manifest was updated, False otherwise.\n\n Raises:\n AssertionError: in case anything goes wrong during the update\n \"\"\"\n # Check if the environment needs update\n with open(manifest, encoding=\"utf-8\") as f:\n data = syaml.load(f)\n\n top_level_key = _top_level_key(data)\n needs_update = spack.schema.env.update(data[top_level_key])\n if not needs_update:\n msg = \"No update needed [manifest={0}]\".format(manifest)\n tty.debug(msg)\n return False\n\n # Copy environment to a backup file and update it\n msg = (\n 'backup file \"{0}\" already exists on disk. Check its content '\n \"and remove it before trying to update again.\"\n )\n assert not os.path.exists(backup_file), msg.format(backup_file)\n\n shutil.copy(manifest, backup_file)\n with open(manifest, \"w\", encoding=\"utf-8\") as f:\n syaml.dump_config(data, f)\n return True\n\n\ndef _top_level_key(data):\n \"\"\"Return the top level key used in this environment\n\n Args:\n data (dict): raw yaml data of the environment\n\n Returns:\n Either 'spack' or 'env'\n \"\"\"\n msg = 'cannot find top level attribute \"spack\" or \"env\" in the environment'\n assert any(x in data for x in (\"spack\", \"env\")), msg\n if \"spack\" in data:\n return \"spack\"\n return \"env\"\n\n\ndef is_latest_format(manifest):\n \"\"\"Return False if the manifest file exists and is not in the latest schema format.\n\n Args:\n manifest (str): manifest file to be analyzed\n \"\"\"\n try:\n with open(manifest, encoding=\"utf-8\") as f:\n data = syaml.load(f)\n except OSError:\n return True\n top_level_key = _top_level_key(data)\n changed = spack.schema.env.update(data[top_level_key])\n return not changed\n\n\n@contextlib.contextmanager\ndef no_active_environment():\n \"\"\"Deactivate the active environment for the duration of the context. Has no\n effect when there is no active environment.\"\"\"\n env = active_environment()\n try:\n deactivate()\n yield\n finally:\n # TODO: we don't handle `use_env_repo` here.\n if env:\n activate(env)\n\n\ndef initialize_environment_dir(\n environment_dir: Union[str, pathlib.Path], envfile: Optional[Union[str, pathlib.Path]]\n) -> None:\n \"\"\"Initialize an environment directory starting from an envfile.\n\n Files with suffix .json or .lock are considered lockfiles. Files with any other name\n are considered manifest files.\n\n Args:\n environment_dir: directory where the environment should be placed\n envfile: manifest file or lockfile used to initialize the environment\n\n Raises:\n SpackEnvironmentError: if the directory can't be initialized\n \"\"\"\n environment_dir = pathlib.Path(environment_dir)\n target_lockfile = environment_dir / lockfile_name\n target_manifest = environment_dir / manifest_name\n if target_manifest.exists():\n msg = f\"cannot initialize environment, {target_manifest} already exists\"\n raise SpackEnvironmentError(msg)\n\n if target_lockfile.exists():\n msg = f\"cannot initialize environment, {target_lockfile} already exists\"\n raise SpackEnvironmentError(msg)\n\n def _ensure_env_dir():\n try:\n environment_dir.mkdir(parents=True, exist_ok=True)\n except FileExistsError as e:\n msg = f\"cannot initialize the environment, '{environment_dir}' already exists\"\n raise SpackEnvironmentError(msg) from e\n\n if envfile is None:\n _ensure_env_dir()\n target_manifest.write_text(default_manifest_yaml())\n return\n\n envfile = pathlib.Path(envfile)\n if not envfile.exists():\n msg = f\"cannot initialize environment, {envfile} is not a valid file\"\n raise SpackEnvironmentError(msg)\n\n if envfile.is_dir():\n # initialization file is an entire env directory\n if not (envfile / \"spack.yaml\").is_file():\n msg = f\"cannot initialize environment, {envfile} is not a valid environment\"\n raise SpackEnvironmentError(msg)\n copy_tree(str(envfile), str(environment_dir))\n return\n\n _ensure_env_dir()\n\n # When we have a lockfile we should copy that and produce a consistent default manifest\n if str(envfile).endswith(\".lock\") or str(envfile).endswith(\".json\"):\n shutil.copy(envfile, target_lockfile)\n # This constructor writes a spack.yaml which is consistent with the root\n # specs in the spack.lock\n try:\n EnvironmentManifestFile.from_lockfile(environment_dir)\n except Exception as e:\n msg = f\"cannot initialize environment, '{environment_dir}' from lockfile\"\n raise SpackEnvironmentError(msg) from e\n return\n\n shutil.copy(envfile, target_manifest)\n\n # Copy relative path includes that live inside the environment dir\n try:\n manifest = EnvironmentManifestFile(environment_dir)\n except Exception:\n # error handling for bad manifests is handled on other code paths\n return\n\n # TODO: make this recursive\n includes = manifest[TOP_LEVEL_KEY].get(manifest_include_name, [])\n paths = spack.config.paths_from_includes(includes)\n for path in paths:\n if os.path.isabs(path):\n continue\n\n abspath = pathlib.Path(os.path.normpath(environment_dir / path))\n common_path = pathlib.Path(os.path.commonpath([environment_dir, abspath]))\n if common_path != environment_dir:\n tty.debug(f\"Will not copy relative include file from outside environment: {path}\")\n continue\n\n orig_abspath = os.path.normpath(envfile.parent / path)\n if os.path.isfile(orig_abspath):\n fs.touchp(abspath)\n shutil.copy(orig_abspath, abspath)\n continue\n\n if not os.path.exists(orig_abspath):\n tty.warn(f\"Skipping copy of non-existent include path: '{path}'\")\n continue\n\n if os.path.exists(abspath):\n tty.warn(f\"Skipping copy of directory over existing path: {path}\")\n continue\n\n shutil.copytree(orig_abspath, abspath, symlinks=True)\n\n\nclass EnvironmentManifestFile(collections.abc.Mapping):\n \"\"\"Manages the in-memory representation of a manifest file, and its synchronization\n with the actual manifest on disk.\n \"\"\"\n\n @staticmethod\n def from_lockfile(manifest_dir: Union[pathlib.Path, str]) -> \"EnvironmentManifestFile\":\n \"\"\"Returns an environment manifest file compatible with the lockfile already present in\n the environment directory.\n\n This function also writes a spack.yaml file that is consistent with the spack.lock\n already existing in the directory.\n\n Args:\n manifest_dir: directory containing the manifest and lockfile\n \"\"\"\n # TBD: Should this be the abspath?\n manifest_dir = pathlib.Path(manifest_dir)\n lockfile = manifest_dir / lockfile_name\n with lockfile.open(\"r\", encoding=\"utf-8\") as f:\n data = sjson.load(f)\n roots = data[\"roots\"]\n\n user_specs_by_group: Dict[str, List[str]] = {}\n for item in roots:\n # \"group\" is not there for Lockfile v6 and lower\n group = item.get(\"group\", DEFAULT_USER_SPEC_GROUP)\n user_specs_by_group.setdefault(group, []).append(item[\"spec\"])\n\n default_content = manifest_dir / manifest_name\n default_content.write_text(default_manifest_yaml())\n manifest = EnvironmentManifestFile(manifest_dir)\n\n for group, specs in user_specs_by_group.items():\n for spec in specs:\n manifest.add_user_spec(spec, group=group)\n\n manifest.flush()\n return manifest\n\n def __init__(self, manifest_dir: Union[pathlib.Path, str], name: Optional[str] = None) -> None:\n self.manifest_dir = pathlib.Path(manifest_dir)\n self.name = name or str(manifest_dir)\n self.manifest_file = self.manifest_dir / manifest_name\n self.scope_name = f\"env:{self.name}\"\n self.config_stage_dir = os.path.join(env_subdir_path(manifest_dir), \"config\")\n\n #: Configuration scope associated with this environment. Note that this is not\n #: invalidated by a re-read of the manifest file.\n self._env_config_scope: Optional[spack.config.ConfigScope] = None\n\n if not self.manifest_file.exists():\n msg = f\"cannot find '{manifest_name}' in {self.manifest_dir}\"\n raise SpackEnvironmentError(msg)\n\n with self.manifest_file.open(encoding=\"utf-8\") as f:\n self.yaml_content = _read_yaml(f)\n\n # Maps groups to their dependencies\n self._groups: Dict[str, Tuple[str, ...]] = {DEFAULT_USER_SPEC_GROUP: tuple()}\n # Raw YAML definitions of the user specs for each group\n self._user_specs: Dict[str, List] = {DEFAULT_USER_SPEC_GROUP: []}\n # Configuration overrides for each group\n self._config_override: Dict[str, Any] = {DEFAULT_USER_SPEC_GROUP: None}\n # Whether specs in each group are marked explicit\n self._explicit: Dict[str, bool] = {DEFAULT_USER_SPEC_GROUP: True}\n self._init_user_specs()\n\n self.changed = False\n\n def _init_user_specs(self):\n specs_yaml = self.configuration.get(USER_SPECS_KEY, [])\n for item in specs_yaml:\n if isinstance(item, str):\n self._user_specs[DEFAULT_USER_SPEC_GROUP].append(item)\n elif isinstance(item, dict):\n group = item.get(\"group\", DEFAULT_USER_SPEC_GROUP)\n\n # Error if a group is defined more than once\n if group != DEFAULT_USER_SPEC_GROUP and group in self._groups:\n raise SpackEnvironmentConfigError(\n f\"group '{group}' defined more than once\", self.manifest_file\n )\n\n # Add an entry for the user specs and store group dependencies\n if group not in self._user_specs:\n self._user_specs[group] = []\n self._groups[group] = tuple(item.get(\"needs\", ()))\n self._config_override[group] = item.get(\"override\", None)\n self._explicit[group] = item.get(\"explicit\", True)\n\n if \"matrix\" in item:\n # Short form if the group is composed of only one matrix\n self._user_specs[group].append({\"matrix\": item[\"matrix\"]})\n elif \"specs\" in item:\n self._user_specs[group].extend(item[\"specs\"])\n\n def _clear_user_specs(self) -> None:\n self._user_specs = {DEFAULT_USER_SPEC_GROUP: []}\n self._groups = {DEFAULT_USER_SPEC_GROUP: tuple()}\n self._config_override = {DEFAULT_USER_SPEC_GROUP: None}\n self._explicit = {DEFAULT_USER_SPEC_GROUP: True}\n\n def _all_matches(self, user_spec: str) -> List[str]:\n \"\"\"Maps the input string to the first equivalent user spec in the manifest,\n and returns it.\n\n Args:\n user_spec: user spec to be found\n\n Raises:\n ValueError: if no equivalent match is found\n \"\"\"\n result = []\n for yaml_spec_str in self.configuration[\"specs\"]:\n if Spec(yaml_spec_str) == Spec(user_spec):\n result.append(yaml_spec_str)\n\n if not result:\n raise ValueError(f\"cannot find a spec equivalent to {user_spec}\")\n\n return result\n\n def user_specs(self, *, group: Optional[str] = None) -> List:\n group = self._ensure_group_exists(group)\n return self._user_specs[group]\n\n def config_override(\n self, *, group: Optional[str] = None\n ) -> Optional[spack.config.InternalConfigScope]:\n group = self._ensure_group_exists(group)\n data = self._config_override[group]\n if data is None:\n return None\n return spack.config.InternalConfigScope(f\"env:groups:{group}\", data)\n\n def groups(self) -> KeysView:\n \"\"\"Returns the list of groups defined in the manifest\"\"\"\n return self._groups.keys()\n\n def needs(self, *, group: Optional[str] = None) -> Tuple[str, ...]:\n \"\"\"Returns the dependencies of a group of user specs.\"\"\"\n group = self._ensure_group_exists(group)\n return self._groups[group]\n\n def is_explicit(self, *, group: Optional[str] = None) -> bool:\n \"\"\"Returns whether specs in a group are marked explicit.\n\n When False, specs in the group are installed as implicit dependencies\n and are eligible for garbage collection once no other spec depends on them.\n \"\"\"\n group = self._ensure_group_exists(group)\n return self._explicit[group]\n\n def _ensure_group_exists(self, group: Optional[str]) -> str:\n group = DEFAULT_USER_SPEC_GROUP if group is None else group\n if group not in self._groups:\n raise ValueError(f\"user specs group '{group}' not found in {self.manifest_file}\")\n return group\n\n def add_user_spec(self, user_spec: str, *, group: Optional[str] = None) -> None:\n \"\"\"Appends the user spec passed as input to the list of root specs for the given group.\n\n Args:\n user_spec: user spec to be appended\n group: group where the spec should be added. If None, the default group is used.\n \"\"\"\n group = group or DEFAULT_USER_SPEC_GROUP\n\n if group == DEFAULT_USER_SPEC_GROUP:\n # Append to top-most specs: attribute\n specs_yaml = self.configuration.setdefault(\"specs\", [])\n specs_yaml.append(user_spec)\n else:\n # Append to specs: attribute within a group\n group_in_yaml = self._get_group(group)\n group_in_yaml.setdefault(\"specs\", []).append(user_spec)\n\n self._user_specs[group].append(user_spec)\n self.changed = True\n\n def _get_group(self, group: str) -> Dict:\n \"\"\"Find or create the group entry in the manifest\"\"\"\n specs_yaml = self.configuration.setdefault(\"specs\", [])\n group_entry = None\n for item in specs_yaml:\n if isinstance(item, dict) and item.get(\"group\") == group:\n group_entry = item\n break\n\n if group_entry is None:\n group_entry = {\"group\": group, \"specs\": []}\n specs_yaml.append(group_entry)\n self._groups[group] = tuple()\n self._config_override[group] = None\n self._user_specs[group] = []\n self._explicit[group] = True\n\n return group_entry\n\n def remove_user_spec(self, user_spec: str) -> None:\n \"\"\"Removes the user spec passed as input from the default list of root specs\n\n Args:\n user_spec: user spec to be removed\n\n Raises:\n SpackEnvironmentError: when the user spec is not in the list\n \"\"\"\n try:\n for key in self._all_matches(user_spec):\n self.configuration[\"specs\"].remove(key)\n self._user_specs[DEFAULT_USER_SPEC_GROUP].remove(key)\n except ValueError as e:\n msg = f\"cannot remove {user_spec} from {self}, no such spec exists\"\n raise SpackEnvironmentError(msg) from e\n self.changed = True\n\n def clear(self) -> None:\n \"\"\"Clear all user specs from the list of root specs\"\"\"\n self.configuration[\"specs\"] = []\n self._clear_user_specs()\n self.changed = True\n\n def override_user_spec(self, user_spec: str, idx: int) -> None:\n \"\"\"Overrides the user spec at index idx with the one passed as input.\n\n Args:\n user_spec: new user spec\n idx: index of the spec to be overridden\n\n Raises:\n SpackEnvironmentError: when the user spec cannot be overridden\n \"\"\"\n try:\n self.configuration[\"specs\"][idx] = user_spec\n self._clear_user_specs()\n self._init_user_specs()\n except ValueError as e:\n msg = f\"cannot override {user_spec} from {self}\"\n raise SpackEnvironmentError(msg) from e\n self.changed = True\n\n def set_include_concrete(self, include_concrete: List[str]) -> None:\n \"\"\"Sets the included concrete environments in the manifest to the value(s) passed as input.\n\n Args:\n include_concrete: list of already existing concrete environments to include\n \"\"\"\n self.configuration[lockfile_include_key] = list(include_concrete)\n self.changed = True\n\n def add_definition(self, user_spec: str, list_name: str) -> None:\n \"\"\"Appends a user spec to the first active definition matching the name passed as argument.\n\n Args:\n user_spec: user spec to be appended\n list_name: name of the definition where to append\n\n Raises:\n SpackEnvironmentError: is no valid definition exists already\n \"\"\"\n defs = self.configuration.get(\"definitions\", [])\n msg = f\"cannot add {user_spec} to the '{list_name}' definition, no valid list exists\"\n\n for idx, item in self._iterate_on_definitions(defs, list_name=list_name, err_msg=msg):\n item[list_name].append(user_spec)\n break\n\n # \"definitions\" can be remote, so we need to update the global config too\n spack.config.CONFIG.set(\"definitions\", defs, scope=self.scope_name)\n self.changed = True\n\n def remove_definition(self, user_spec: str, list_name: str) -> None:\n \"\"\"Removes a user spec from an active definition that matches the name passed as argument.\n\n Args:\n user_spec: user spec to be removed\n list_name: name of the definition where to remove the spec from\n\n Raises:\n SpackEnvironmentError: if the user spec cannot be removed from the list,\n or the list does not exist\n \"\"\"\n defs = self.configuration.get(\"definitions\", [])\n msg = f\"cannot remove {user_spec} from the '{list_name}' definition, no valid list exists\"\n\n for idx, item in self._iterate_on_definitions(defs, list_name=list_name, err_msg=msg):\n try:\n item[list_name].remove(user_spec)\n break\n except ValueError:\n pass\n\n # \"definitions\" can be remote, so we need to update the global config too\n spack.config.CONFIG.set(\"definitions\", defs, scope=self.scope_name)\n self.changed = True\n\n def override_definition(self, user_spec: str, *, override: str, list_name: str) -> None:\n \"\"\"Overrides a user spec from an active definition that matches the name passed\n as argument.\n\n Args:\n user_spec: user spec to be overridden\n override: new spec to be used\n list_name: name of the definition where to override the spec\n\n Raises:\n SpackEnvironmentError: if the user spec cannot be overridden\n \"\"\"\n defs = self.configuration.get(\"definitions\", [])\n msg = f\"cannot override {user_spec} with {override} in the '{list_name}' definition\"\n\n for idx, item in self._iterate_on_definitions(defs, list_name=list_name, err_msg=msg):\n try:\n sub_index = item[list_name].index(user_spec)\n item[list_name][sub_index] = override\n break\n except ValueError:\n pass\n\n # \"definitions\" can be remote, so we need to update the global config too\n spack.config.CONFIG.set(\"definitions\", defs, scope=self.scope_name)\n self.changed = True\n\n def _iterate_on_definitions(self, definitions, *, list_name, err_msg):\n \"\"\"Iterates on definitions, returning the active ones matching a given name.\"\"\"\n\n def extract_name(_item):\n names = list(x for x in _item if x != \"when\")\n assert len(names) == 1, f\"more than one name in {_item}\"\n return names[0]\n\n for idx, item in enumerate(definitions):\n name = extract_name(item)\n if name != list_name:\n continue\n\n condition_str = item.get(\"when\", \"True\")\n if not spack.spec.eval_conditional(condition_str):\n continue\n\n yield idx, item\n else:\n raise SpackEnvironmentError(err_msg)\n\n def set_default_view(self, view: Union[bool, str, pathlib.Path, Dict[str, str]]) -> None:\n \"\"\"Sets the default view root in the manifest to the value passed as input.\n\n Args:\n view: If the value is a string or a path, it specifies the path to the view. If\n True the default view is used for the environment, if False there's no view.\n \"\"\"\n if isinstance(view, dict):\n self.configuration[\"view\"][default_view_name].update(view)\n self.changed = True\n return\n\n if not isinstance(view, bool):\n view = str(view)\n\n self.configuration[\"view\"] = view\n self.changed = True\n\n def remove_default_view(self) -> None:\n \"\"\"Removes the default view from the manifest file\"\"\"\n view_data = self.configuration.get(\"view\")\n if isinstance(view_data, collections.abc.Mapping):\n self.configuration[\"view\"].pop(default_view_name)\n self.changed = True\n return\n\n self.set_default_view(view=False)\n\n def flush(self) -> None:\n \"\"\"Synchronizes the object with the manifest file on disk.\"\"\"\n if not self.changed:\n return\n\n with fs.write_tmp_and_move(os.path.realpath(self.manifest_file)) as f:\n _write_yaml(self.yaml_content, f)\n self.changed = False\n\n @property\n def configuration(self):\n \"\"\"Return the dictionaries in the pristine YAML, without the top level attribute\"\"\"\n return self.yaml_content[TOP_LEVEL_KEY]\n\n def __len__(self):\n return len(self.yaml_content)\n\n def __getitem__(self, key):\n return self.yaml_content[key]\n\n def __iter__(self):\n return iter(self.yaml_content)\n\n def __str__(self):\n return str(self.manifest_file)\n\n @property\n def env_config_scope(self) -> spack.config.ConfigScope:\n \"\"\"The configuration scope for the environment manifest\"\"\"\n if self._env_config_scope is None:\n self._env_config_scope = spack.config.SingleFileScope(\n self.scope_name,\n str(self.manifest_file),\n spack.schema.env.schema,\n yaml_path=[TOP_LEVEL_KEY],\n )\n ensure_no_disallowed_env_config_mods(self._env_config_scope)\n return self._env_config_scope\n\n def prepare_config_scope(self) -> None:\n \"\"\"Add the manifest's scope to the global configuration search path.\"\"\"\n spack.config.CONFIG.push_scope(\n self.env_config_scope, priority=ConfigScopePriority.ENVIRONMENT\n )\n\n def deactivate_config_scope(self) -> None:\n \"\"\"Remove the manifest's scope from the global config path.\"\"\"\n spack.config.CONFIG.remove_scope(self.env_config_scope.name)\n\n @contextlib.contextmanager\n def use_config(self):\n \"\"\"Ensure only the manifest's configuration scopes are global.\"\"\"\n with no_active_environment():\n self.prepare_config_scope()\n yield\n self.deactivate_config_scope()\n\n\ndef environment_path_scope(name: str, path: str) -> Optional[spack.config.ConfigScope]:\n \"\"\"Retrieve the suitably named environment path scope\n\n Arguments:\n name: configuration scope name\n path: path to configuration file(s)\n\n Returns: list of environment scopes, if any, or None\n \"\"\"\n if exists(path): # managed environment\n manifest = EnvironmentManifestFile(root(path))\n elif is_env_dir(path): # anonymous environment\n manifest = EnvironmentManifestFile(path)\n else:\n return None\n\n manifest.env_config_scope.name = f\"{name}:{manifest.env_config_scope.name}\"\n manifest.env_config_scope.writable = False\n return manifest.env_config_scope\n\n\nclass SpackEnvironmentError(spack.error.SpackError):\n \"\"\"Superclass for all errors to do with Spack environments.\"\"\"\n\n\nclass SpackEnvironmentViewError(SpackEnvironmentError):\n \"\"\"Class for errors regarding view generation.\"\"\"\n\n\nclass SpackEnvironmentConfigError(SpackEnvironmentError):\n \"\"\"Class for Spack environment-specific configuration errors.\"\"\"\n\n def __init__(self, msg, filename):\n super().__init__(f\"{msg} in {filename}\")\n\n\nclass SpackEnvironmentDevelopError(SpackEnvironmentError):\n \"\"\"Class for errors in applying develop information to an environment.\"\"\"\n" }, { "path": "lib/spack/spack/environment/list.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport itertools\nfrom typing import Any, Dict, List, NamedTuple, Optional, Union\n\nimport spack.spec\nimport spack.util.spack_yaml\nimport spack.variant\nfrom spack.error import SpackError\nfrom spack.spec import Spec\nfrom spack.spec_parser import expand_toolchains\n\n\nclass SpecList:\n def __init__(\n self, *, name: str = \"specs\", yaml_list=None, expanded_list=None, toolchains=None\n ):\n self.name = name\n self.yaml_list = yaml_list[:] if yaml_list is not None else []\n # Expansions can be expensive to compute and difficult to keep updated\n # We cache results and invalidate when self.yaml_list changes\n self.specs_as_yaml_list = expanded_list or []\n self._constraints = None\n self._specs: Optional[List[Spec]] = None\n self._toolchains = toolchains\n\n @property\n def is_matrix(self):\n for item in self.specs_as_yaml_list:\n if isinstance(item, dict):\n return True\n return False\n\n @property\n def specs_as_constraints(self):\n if self._constraints is None:\n constraints = []\n for item in self.specs_as_yaml_list:\n if isinstance(item, dict): # matrix of specs\n constraints.extend(_expand_matrix_constraints(item))\n else: # individual spec\n constraints.append([Spec(item)])\n self._constraints = constraints\n\n return self._constraints\n\n @property\n def specs(self) -> List[Spec]:\n if self._specs is None:\n specs: List[Spec] = []\n # This could be slightly faster done directly from yaml_list,\n # but this way is easier to maintain.\n for constraint_list in self.specs_as_constraints:\n spec = constraint_list[0].copy()\n for const in constraint_list[1:]:\n spec.constrain(const)\n if self._toolchains:\n expand_toolchains(spec, self._toolchains)\n specs.append(spec)\n self._specs = specs\n\n return self._specs\n\n def add(self, spec: Spec):\n spec_str = str(spec)\n self.yaml_list.append(spec_str)\n\n # expanded list can be updated without invalidation\n if self.specs_as_yaml_list is not None:\n self.specs_as_yaml_list.append(spec_str)\n\n # Invalidate cache variables when we change the list\n self._constraints = None\n self._specs = None\n\n def remove(self, spec):\n # Get spec to remove from list\n remove = [\n s\n for s in self.yaml_list\n if (isinstance(s, str) and not s.startswith(\"$\")) and Spec(s) == Spec(spec)\n ]\n if not remove:\n msg = f\"Cannot remove {spec} from SpecList {self.name}.\\n\"\n msg += f\"Either {spec} is not in {self.name} or {spec} is \"\n msg += \"expanded from a matrix and cannot be removed directly.\"\n raise SpecListError(msg)\n\n # Remove may contain more than one string representation of the same spec\n for item in remove:\n self.yaml_list.remove(item)\n self.specs_as_yaml_list.remove(item)\n\n # invalidate cache variables when we change the list\n self._constraints = None\n self._specs = None\n\n def extend(self, other: \"SpecList\", copy_reference=True) -> None:\n self.yaml_list.extend(other.yaml_list)\n self.specs_as_yaml_list.extend(other.specs_as_yaml_list)\n self._constraints = None\n self._specs = None\n\n def __len__(self):\n return len(self.specs)\n\n def __getitem__(self, key):\n return self.specs[key]\n\n def __iter__(self):\n return iter(self.specs)\n\n\ndef _expand_matrix_constraints(matrix_config):\n # recurse so we can handle nested matrices\n expanded_rows = []\n for row in matrix_config[\"matrix\"]:\n new_row = []\n for r in row:\n if isinstance(r, dict):\n # Flatten the nested matrix into a single row of constraints\n new_row.extend(\n [\n [\" \".join([str(c) for c in expanded_constraint_list])]\n for expanded_constraint_list in _expand_matrix_constraints(r)\n ]\n )\n else:\n new_row.append([r])\n expanded_rows.append(new_row)\n\n excludes = matrix_config.get(\"exclude\", []) # only compute once\n sigil = matrix_config.get(\"sigil\", \"\")\n\n results = []\n for combo in itertools.product(*expanded_rows):\n # Construct a combined spec to test against excludes\n flat_combo = [Spec(constraint) for constraints in combo for constraint in constraints]\n\n test_spec = flat_combo[0].copy()\n for constraint in flat_combo[1:]:\n test_spec.constrain(constraint)\n\n # Abstract variants don't have normal satisfaction semantics\n # Convert all variants to concrete types.\n # This method is best effort, so all existing variants will be\n # converted before any error is raised.\n # Catch exceptions because we want to be able to operate on\n # abstract specs without needing package information\n try:\n spack.spec.substitute_abstract_variants(test_spec)\n except spack.variant.UnknownVariantError:\n pass\n\n # Resolve abstract hashes for exclusion criteria\n if any(test_spec.lookup_hash().satisfies(x) for x in excludes):\n continue\n\n if sigil:\n flat_combo[0] = Spec(sigil + str(flat_combo[0]))\n\n # Add to list of constraints\n results.append(flat_combo)\n\n return results\n\n\ndef _sigilify(item, sigil):\n if isinstance(item, dict):\n if sigil:\n item[\"sigil\"] = sigil\n return item\n else:\n return sigil + item\n\n\nclass Definition(NamedTuple):\n name: str\n yaml_list: List[Union[str, Dict]]\n when: Optional[str]\n\n\nclass SpecListParser:\n \"\"\"Parse definitions and user specs from data in environments\"\"\"\n\n def __init__(self, *, toolchains=None):\n self.definitions: Dict[str, SpecList] = {}\n self._toolchains = toolchains\n\n def parse_definitions(self, *, data: List[Dict[str, Any]]) -> Dict[str, SpecList]:\n definitions_from_yaml: Dict[str, List[Definition]] = {}\n for item in data:\n value = self._parse_yaml_definition(item)\n definitions_from_yaml.setdefault(value.name, []).append(value)\n\n self.definitions = {}\n self._build_definitions(definitions_from_yaml)\n return self.definitions\n\n def parse_user_specs(self, *, name, yaml_list) -> SpecList:\n definition = Definition(name=name, yaml_list=yaml_list, when=None)\n return self._speclist_from_definitions(name, [definition])\n\n def _parse_yaml_definition(self, yaml_entry) -> Definition:\n when_string = yaml_entry.get(\"when\")\n\n if (when_string and len(yaml_entry) > 2) or (not when_string and len(yaml_entry) > 1):\n mark = spack.util.spack_yaml.get_mark_from_yaml_data(yaml_entry)\n attributes = \", \".join(x for x in yaml_entry if x != \"when\")\n error_msg = f\"definition must have a single attribute, got many: {attributes}\"\n raise SpecListError(f\"{mark.name}:{mark.line + 1}: {error_msg}\")\n\n for name, yaml_list in yaml_entry.items():\n if name == \"when\":\n continue\n return Definition(name=name, yaml_list=yaml_list, when=when_string)\n\n # If we are here, it means only \"when\" is in the entry\n mark = spack.util.spack_yaml.get_mark_from_yaml_data(yaml_entry)\n error_msg = \"definition must have a single attribute, got none\"\n raise SpecListError(f\"{mark.name}:{mark.line + 1}: {error_msg}\")\n\n def _build_definitions(self, definitions_from_yaml: Dict[str, List[Definition]]):\n for name, definitions in definitions_from_yaml.items():\n self.definitions[name] = self._speclist_from_definitions(name, definitions)\n\n def _speclist_from_definitions(self, name, definitions) -> SpecList:\n combined_yaml_list = []\n for def_part in definitions:\n if def_part.when is not None and not spack.spec.eval_conditional(def_part.when):\n continue\n combined_yaml_list.extend(def_part.yaml_list)\n expanded_list = self._expand_yaml_list(combined_yaml_list)\n return SpecList(\n name=name,\n yaml_list=combined_yaml_list,\n expanded_list=expanded_list,\n toolchains=self._toolchains,\n )\n\n def _expand_yaml_list(self, raw_yaml_list):\n result = []\n for item in raw_yaml_list:\n if isinstance(item, str) and item.startswith(\"$\"):\n result.extend(self._expand_reference(item))\n continue\n\n value = item\n if isinstance(item, dict):\n value = self._expand_yaml_matrix(item)\n result.append(value)\n return result\n\n def _expand_reference(self, item: str):\n sigil, name = \"\", item[1:]\n if name.startswith(\"^\") or name.startswith(\"%\"):\n sigil, name = name[0], name[1:]\n\n if name not in self.definitions:\n mark = spack.util.spack_yaml.get_mark_from_yaml_data(item)\n error_msg = f\"trying to expand the name '{name}', which is not defined yet\"\n raise UndefinedReferenceError(f\"{mark.name}:{mark.line + 1}: {error_msg}\")\n\n value = self.definitions[name].specs_as_yaml_list\n if not sigil:\n return value\n return [_sigilify(x, sigil) for x in value]\n\n def _expand_yaml_matrix(self, matrix_yaml):\n extra_attributes = set(matrix_yaml) - {\"matrix\", \"exclude\"}\n if extra_attributes:\n mark = spack.util.spack_yaml.get_mark_from_yaml_data(matrix_yaml)\n error_msg = f\"extra attributes in spec matrix: {','.join(sorted(extra_attributes))}\"\n raise SpecListError(f\"{mark.name}:{mark.line + 1}: {error_msg}\")\n\n if \"matrix\" not in matrix_yaml:\n mark = spack.util.spack_yaml.get_mark_from_yaml_data(matrix_yaml)\n error_msg = \"matrix is missing the 'matrix' attribute\"\n raise SpecListError(f\"{mark.name}:{mark.line + 1}: {error_msg}\")\n\n # Assume data has been validated against the YAML schema\n result = {\"matrix\": [self._expand_yaml_list(row) for row in matrix_yaml[\"matrix\"]]}\n if \"exclude\" in matrix_yaml:\n result[\"exclude\"] = matrix_yaml[\"exclude\"]\n return result\n\n\nclass SpecListError(SpackError):\n \"\"\"Error class for all errors related to SpecList objects.\"\"\"\n\n\nclass UndefinedReferenceError(SpecListError):\n \"\"\"Error class for undefined references in Spack stacks.\"\"\"\n\n\nclass InvalidSpecConstraintError(SpecListError):\n \"\"\"Error class for invalid spec constraints at concretize time.\"\"\"\n" }, { "path": "lib/spack/spack/environment/shell.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport os\nimport textwrap\nfrom typing import Optional\n\nimport spack.config\nimport spack.environment as ev\nimport spack.llnl.util.tty as tty\nimport spack.repo\nimport spack.schema.environment\nimport spack.store\nfrom spack.llnl.util.tty.color import colorize\nfrom spack.util.environment import EnvironmentModifications\n\n\ndef activate_header(env, shell, prompt=None, view: Optional[str] = None):\n # Construct the commands to run\n cmds = \"\"\n if shell == \"csh\":\n # TODO: figure out how to make color work for csh\n cmds += \"setenv SPACK_ENV %s;\\n\" % env.path\n if view:\n cmds += \"setenv SPACK_ENV_VIEW %s;\\n\" % view\n cmds += 'alias despacktivate \"spack env deactivate\";\\n'\n if prompt:\n cmds += \"if (! $?SPACK_OLD_PROMPT ) \"\n cmds += 'setenv SPACK_OLD_PROMPT \"${prompt}\";\\n'\n cmds += 'set prompt=\"%s ${prompt}\";\\n' % prompt\n elif shell == \"fish\":\n if \"color\" in os.getenv(\"TERM\", \"\") and prompt:\n prompt = colorize(\"@G{%s} \" % prompt, color=True)\n\n cmds += \"set -gx SPACK_ENV %s;\\n\" % env.path\n if view:\n cmds += \"set -gx SPACK_ENV_VIEW %s;\\n\" % view\n cmds += \"function despacktivate;\\n\"\n cmds += \" spack env deactivate;\\n\"\n cmds += \"end;\\n\"\n #\n # NOTE: We're not changing the fish_prompt function (which is fish's\n # solution to the PS1 variable) here. This is a bit fiddly, and easy to\n # screw up => spend time reasearching a solution. Feedback welcome.\n #\n elif shell == \"bat\":\n # TODO: Color\n cmds += 'set \"SPACK_ENV=%s\"\\n' % env.path\n if view:\n cmds += 'set \"SPACK_ENV_VIEW=%s\"\\n' % view\n if prompt:\n old_prompt = os.environ.get(\"SPACK_OLD_PROMPT\")\n if not old_prompt:\n old_prompt = os.environ.get(\"PROMPT\")\n cmds += f'set \"SPACK_OLD_PROMPT={old_prompt}\"\\n'\n cmds += f'set \"PROMPT={prompt} $P$G\"\\n'\n elif shell == \"pwsh\":\n cmds += \"$Env:SPACK_ENV='%s'\\n\" % env.path\n if view:\n cmds += \"$Env:SPACK_ENV_VIEW='%s'\\n\" % view\n if prompt:\n cmds += (\n \"function global:prompt { $pth = $(Convert-Path $(Get-Location))\"\n ' | Split-Path -leaf; if(!\"$Env:SPACK_OLD_PROMPT\") '\n '{$Env:SPACK_OLD_PROMPT=\"[spack] PS $pth>\"}; '\n '\"%s PS $pth>\"}\\n' % prompt\n )\n else:\n bash_color_prompt = colorize(f\"@G{{{prompt}}}\", color=True, enclose=True)\n zsh_color_prompt = colorize(f\"@G{{{prompt}}}\", color=True, enclose=False, zsh=True)\n\n cmds += \"export SPACK_ENV=%s;\\n\" % env.path\n if view:\n cmds += \"export SPACK_ENV_VIEW=%s;\\n\" % view\n cmds += \"alias despacktivate='spack env deactivate';\\n\"\n if prompt:\n cmds += textwrap.dedent(\n rf\"\"\"\n if [ -z ${{SPACK_OLD_PS1+x}} ]; then\n if [ -z ${{PS1+x}} ]; then\n PS1='$$$$';\n fi;\n export SPACK_OLD_PS1=\"${{PS1}}\";\n fi;\n if [ -n \"${{TERM:-}}\" ] && [ \"${{TERM#*color}}\" != \"${{TERM}}\" ] && \\\n [ -n \"${{BASH:-}}\" ];\n then\n export PS1=\"{bash_color_prompt} ${{PS1}}\";\n elif [ -n \"${{TERM:-}}\" ] && [ \"${{TERM#*color}}\" != \"${{TERM}}\" ] && \\\n [ -n \"${{ZSH_NAME:-}}\" ];\n then\n export PS1=\"{zsh_color_prompt} ${{PS1}}\";\n else\n export PS1=\"{prompt} ${{PS1}}\";\n fi\n \"\"\"\n ).lstrip(\"\\n\")\n return cmds\n\n\ndef deactivate_header(shell):\n cmds = \"\"\n if shell == \"csh\":\n cmds += \"unsetenv SPACK_ENV;\\n\"\n cmds += \"unsetenv SPACK_ENV_VIEW;\\n\"\n cmds += \"if ( $?SPACK_OLD_PROMPT ) \"\n cmds += ' eval \\'set prompt=\"$SPACK_OLD_PROMPT\" &&'\n cmds += \" unsetenv SPACK_OLD_PROMPT';\\n\"\n cmds += \"unalias despacktivate;\\n\"\n elif shell == \"fish\":\n cmds += \"set -e SPACK_ENV;\\n\"\n cmds += \"set -e SPACK_ENV_VIEW;\\n\"\n cmds += \"functions -e despacktivate;\\n\"\n #\n # NOTE: Not changing fish_prompt (above) => no need to restore it here.\n #\n elif shell == \"bat\":\n # TODO: Color\n cmds += 'set \"SPACK_ENV=\"\\n'\n cmds += 'set \"SPACK_ENV_VIEW=\"\\n'\n # TODO: despacktivate\n old_prompt = os.environ.get(\"SPACK_OLD_PROMPT\")\n if old_prompt:\n cmds += f'set \"PROMPT={old_prompt}\"\\n'\n cmds += 'set \"SPACK_OLD_PROMPT=\"\\n'\n elif shell == \"pwsh\":\n cmds += \"Set-Item -Path Env:SPACK_ENV\\n\"\n cmds += \"Set-Item -Path Env:SPACK_ENV_VIEW\\n\"\n cmds += (\n \"function global:prompt { $pth = $(Convert-Path $(Get-Location))\"\n ' | Split-Path -leaf; $spack_prompt = \"[spack] $pth >\"; '\n 'if(\"$Env:SPACK_OLD_PROMPT\") {$spack_prompt=$Env:SPACK_OLD_PROMPT};'\n \" $spack_prompt}\\n\"\n )\n else:\n cmds += \"if [ ! -z ${SPACK_ENV+x} ]; then\\n\"\n cmds += \"unset SPACK_ENV; export SPACK_ENV;\\n\"\n cmds += \"fi;\\n\"\n cmds += \"if [ ! -z ${SPACK_ENV_VIEW+x} ]; then\\n\"\n cmds += \"unset SPACK_ENV_VIEW; export SPACK_ENV_VIEW;\\n\"\n cmds += \"fi;\\n\"\n cmds += \"alias despacktivate > /dev/null 2>&1 && unalias despacktivate;\\n\"\n cmds += \"if [ ! -z ${SPACK_OLD_PS1+x} ]; then\\n\"\n cmds += \" if [ \\\"$SPACK_OLD_PS1\\\" = '$$$$' ]; then\\n\"\n cmds += \" unset PS1; export PS1;\\n\"\n cmds += \" else\\n\"\n cmds += ' export PS1=\"$SPACK_OLD_PS1\";\\n'\n cmds += \" fi;\\n\"\n cmds += \" unset SPACK_OLD_PS1; export SPACK_OLD_PS1;\\n\"\n cmds += \"fi;\\n\"\n\n return cmds\n\n\ndef activate(\n env: ev.Environment, use_env_repo=False, view: Optional[str] = \"default\"\n) -> EnvironmentModifications:\n \"\"\"Activate an environment and append environment modifications\n\n To activate an environment, we add its configuration scope to the\n existing Spack configuration, and we set active to the current\n environment.\n\n Arguments:\n env: the environment to activate\n use_env_repo: use the packages exactly as they appear in the environment's repository\n view: generate commands to add runtime environment variables for named view\n\n Returns:\n spack.util.environment.EnvironmentModifications: Environment variables\n modifications to activate environment.\"\"\"\n ev.activate(env, use_env_repo=use_env_repo)\n\n env_mods = EnvironmentModifications()\n\n #\n # NOTE in the fish-shell: Path variables are a special kind of variable\n # used to support colon-delimited path lists including PATH, CDPATH,\n # MANPATH, PYTHONPATH, etc. All variables that end in PATH (case-sensitive)\n # become PATH variables.\n #\n\n env_vars_yaml = spack.config.get(\"env_vars\", None)\n if env_vars_yaml:\n env_mods.extend(spack.schema.environment.parse(env_vars_yaml))\n\n try:\n if view and env.has_view(view):\n with spack.store.STORE.db.read_transaction():\n env.add_view_to_env(env_mods, view)\n except (spack.repo.UnknownPackageError, spack.repo.UnknownNamespaceError) as e:\n tty.error(e)\n tty.die(\n \"Environment view is broken due to a missing package or repo.\\n\",\n \" To activate without views enabled, activate with:\\n\",\n \" spack env activate -V {0}\\n\".format(env.name),\n \" To remove it and resolve the issue, force concretize with the command:\\n\",\n \" spack -e {0} concretize --force\".format(env.name),\n )\n\n return env_mods\n\n\ndef deactivate() -> EnvironmentModifications:\n \"\"\"Deactivate an environment and collect corresponding environment modifications.\n\n Note: unloads the environment in its current state, not in the state it was\n loaded in, meaning that specs that were removed from the spack environment\n after activation are not unloaded.\n\n Returns:\n Environment variables modifications to activate environment.\n \"\"\"\n env_mods = EnvironmentModifications()\n active = ev.active_environment()\n\n if active is None:\n return env_mods\n\n with active.manifest.use_config():\n env_vars_yaml = spack.config.get(\"env_vars\", None)\n if env_vars_yaml:\n env_mods.extend(spack.schema.environment.parse(env_vars_yaml).reversed())\n\n active_view = os.getenv(ev.spack_env_view_var)\n\n if active_view and active.has_view(active_view):\n try:\n with spack.store.STORE.db.read_transaction():\n active.rm_view_from_env(env_mods, active_view)\n except (spack.repo.UnknownPackageError, spack.repo.UnknownNamespaceError) as e:\n tty.warn(e)\n tty.warn(\n \"Could not fully deactivate view due to missing package \"\n \"or repo, shell environment may be corrupt.\"\n )\n\n ev.deactivate()\n\n return env_mods\n" }, { "path": "lib/spack/spack/error.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport sys\nfrom typing import Optional\n\nimport spack.llnl.util.tty as tty\n\n#: at what level we should write stack traces or short error messages\n#: this is module-scoped because it needs to be set very early\ndebug = 0\n\n#: whether to show a backtrace when an error is printed, enabled with ``--backtrace``.\nSHOW_BACKTRACE = False\n\n\nclass SpackAPIWarning(UserWarning):\n \"\"\"Warning that formats with file and line number.\"\"\"\n\n\nclass SpackError(Exception):\n \"\"\"This is the superclass for all Spack errors.\n Subclasses can be found in the modules they have to do with.\n \"\"\"\n\n def __init__(self, message: str, long_message: Optional[str] = None) -> None:\n super().__init__()\n self.message = message\n self._long_message = long_message\n\n # for exceptions raised from child build processes, we save the\n # traceback as a string and print it in the parent.\n self.traceback = None\n\n # we allow exceptions to print debug info via print_context()\n # before they are caught at the top level. If they *haven't*\n # printed context early, we do it by default when die() is\n # called, so we need to remember whether it's been called.\n self.printed = False\n\n @property\n def long_message(self):\n return self._long_message\n\n def print_context(self):\n \"\"\"Print extended debug information about this exception.\n\n This is usually printed when the top-level Spack error handler\n calls ``die()``, but it can be called separately beforehand if a\n lower-level error handler needs to print error context and\n continue without raising the exception to the top level.\n \"\"\"\n if self.printed:\n return\n\n # basic debug message\n tty.error(self.message)\n if self.long_message:\n sys.stderr.write(self.long_message)\n sys.stderr.write(\"\\n\")\n\n # stack trace, etc. in debug mode.\n if debug:\n if self.traceback:\n # exception came from a build child, already got\n # traceback in child, so print it.\n sys.stderr.write(self.traceback)\n else:\n # run parent exception hook.\n sys.excepthook(*sys.exc_info())\n\n sys.stderr.flush()\n self.printed = True\n\n def die(self):\n self.print_context()\n sys.exit(1)\n\n def __str__(self):\n if self._long_message:\n return f\"{self.message}\\n {self._long_message}\"\n return self.message\n\n def __repr__(self):\n qualified_name = type(self).__module__ + \".\" + type(self).__name__\n return f\"{qualified_name}({repr(self.message)}, {repr(self.long_message)})\"\n\n def __reduce__(self):\n return type(self), (self.message, self.long_message)\n\n\nclass UnsupportedPlatformError(SpackError):\n \"\"\"Raised by packages when a platform is not supported\"\"\"\n\n def __init__(self, message):\n super().__init__(message)\n\n\nclass NoLibrariesError(SpackError):\n \"\"\"Raised when package libraries are requested but cannot be found\"\"\"\n\n def __init__(self, message_or_name, prefix=None):\n super().__init__(\n message_or_name\n if prefix is None\n else \"Unable to locate {0} libraries in {1}\".format(message_or_name, prefix)\n )\n\n\nclass NoHeadersError(SpackError):\n \"\"\"Raised when package headers are requested but cannot be found\"\"\"\n\n\nclass SpecError(SpackError):\n \"\"\"Superclass for all errors that occur while constructing specs.\"\"\"\n\n\nclass UnsatisfiableSpecError(SpecError):\n \"\"\"\n Raised when a spec conflicts with package constraints.\n\n For original concretizer, provide the requirement that was violated when\n raising.\n \"\"\"\n\n def __init__(self, provided, required, constraint_type):\n # This is only the entrypoint for old concretizer errors\n super().__init__(\"%s does not satisfy %s\" % (provided, required))\n\n self.provided = provided\n self.required = required\n self.constraint_type = constraint_type\n\n\nclass FetchError(SpackError):\n \"\"\"Superclass for fetch-related errors.\"\"\"\n\n\nclass NoSuchPatchError(SpackError):\n \"\"\"Raised when a patch file doesn't exist.\"\"\"\n\n\nclass PatchDirectiveError(SpackError):\n \"\"\"Raised when the wrong arguments are suppled to the patch directive.\"\"\"\n\n\nclass PatchLookupError(NoSuchPatchError):\n \"\"\"Raised when a patch file cannot be located from sha256.\"\"\"\n\n\nclass SpecSyntaxError(Exception):\n \"\"\"Base class for Spec syntax errors\"\"\"\n\n\nclass PackageError(SpackError):\n \"\"\"Raised when something is wrong with a package definition.\"\"\"\n\n def __init__(self, message, long_msg=None):\n super().__init__(message, long_msg)\n\n\nclass NoURLError(PackageError):\n \"\"\"Raised when someone tries to build a URL for a package with no URLs.\"\"\"\n\n def __init__(self, cls):\n super().__init__(\"Package %s has no version with a URL.\" % cls.__name__)\n\n\nclass InstallError(SpackError):\n \"\"\"Raised when something goes wrong during install or uninstall.\n\n The error can be annotated with a ``pkg`` attribute to allow the\n caller to get the package for which the exception was raised.\n \"\"\"\n\n def __init__(self, message, long_msg=None, pkg=None):\n super().__init__(message, long_msg)\n self.pkg = pkg\n\n\nclass ConfigError(SpackError):\n \"\"\"Superclass for all Spack config related errors.\"\"\"\n\n\nclass StopPhase(SpackError):\n \"\"\"Pickle-able exception to control stopped builds.\"\"\"\n\n def __reduce__(self):\n return _make_stop_phase, (self.message, self.long_message)\n\n\ndef _make_stop_phase(msg, long_msg):\n return StopPhase(msg, long_msg)\n\n\nclass MirrorError(SpackError):\n \"\"\"Superclass of all mirror-creation related errors.\"\"\"\n\n def __init__(self, msg, long_msg=None):\n super().__init__(msg, long_msg)\n\n\nclass NoChecksumException(SpackError):\n \"\"\"\n Raised if file fails checksum verification.\n \"\"\"\n\n def __init__(self, path, size, contents, algorithm, expected, computed):\n super().__init__(\n f\"{algorithm} checksum failed for {path}\",\n f\"Expected {expected} but got {computed}. \"\n f\"File size = {size} bytes. Contents = {contents!r}\",\n )\n\n\nclass CompilerError(SpackError):\n \"\"\"Raised if something goes wrong when probing or querying a compiler.\"\"\"\n\n\nclass SpecFilenameError(SpecError):\n \"\"\"Raised when a spec file name is invalid.\"\"\"\n\n\nclass NoSuchSpecFileError(SpecFilenameError):\n \"\"\"Raised when a spec file doesn't exist.\"\"\"\n" }, { "path": "lib/spack/spack/extensions.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Service functions and classes to implement the hooks\nfor Spack's command extensions.\n\"\"\"\n\nimport glob\nimport importlib\nimport os\nimport re\nimport sys\nimport types\nfrom pathlib import Path\nfrom typing import List\n\nimport spack.config\nimport spack.error\nimport spack.llnl.util.lang\nimport spack.util.path\n\n_extension_regexp = re.compile(r\"spack-(\\w[-\\w]*)$\")\n\n\ndef _python_name(cmd_name):\n return cmd_name.replace(\"-\", \"_\")\n\n\ndef extension_name(path):\n \"\"\"Returns the name of the extension in the path passed as argument.\n\n Args:\n path (str): path where the extension resides\n\n Returns:\n The extension name.\n\n Raises:\n ExtensionNamingError: if path does not match the expected format\n for a Spack command extension.\n \"\"\"\n regexp_match = re.search(_extension_regexp, os.path.basename(os.path.normpath(path)))\n if not regexp_match:\n raise ExtensionNamingError(path)\n return regexp_match.group(1)\n\n\ndef load_command_extension(command, path):\n \"\"\"Loads a command extension from the path passed as argument.\n\n Args:\n command (str): name of the command (contains ``-``, not ``_``).\n path (str): base path of the command extension\n\n Returns:\n A valid module if found and loadable; None if not found. Module\n loading exceptions are passed through.\n \"\"\"\n extension = _python_name(extension_name(path))\n\n # Compute the name of the module we search, exit early if already imported\n cmd_package = \"{0}.{1}.cmd\".format(__name__, extension)\n python_name = _python_name(command)\n module_name = \"{0}.{1}\".format(cmd_package, python_name)\n if module_name in sys.modules:\n return sys.modules[module_name]\n\n # Compute the absolute path of the file to be loaded, along with the\n # name of the python module where it will be stored\n cmd_path = os.path.join(path, extension, \"cmd\", python_name + \".py\")\n\n # Short circuit if the command source file does not exist\n if not os.path.exists(cmd_path):\n return None\n\n ensure_extension_loaded(extension, path=path)\n\n module = importlib.import_module(module_name)\n sys.modules[module_name] = module\n\n return module\n\n\ndef ensure_extension_loaded(extension, *, path):\n def ensure_package_creation(name):\n package_name = \"{0}.{1}\".format(__name__, name)\n if package_name in sys.modules:\n return\n\n parts = [path] + name.split(\".\") + [\"__init__.py\"]\n init_file = os.path.join(*parts)\n if os.path.exists(init_file):\n m = spack.llnl.util.lang.load_module_from_file(package_name, init_file)\n else:\n m = types.ModuleType(package_name)\n\n # Setting __path__ to give spack extensions the\n # ability to import from their own tree, see:\n #\n # https://docs.python.org/3/reference/import.html#package-path-rules\n #\n m.__path__ = [os.path.dirname(init_file)]\n sys.modules[package_name] = m\n\n # Create a searchable package for both the root folder of the extension\n # and the subfolder containing the commands\n ensure_package_creation(extension)\n ensure_package_creation(extension + \".cmd\")\n\n\ndef load_extension(name: str) -> str:\n \"\"\"Loads a single extension into the ``spack.extensions`` package.\n\n Args:\n name: name of the extension\n \"\"\"\n extension_root = path_for_extension(name, paths=get_extension_paths())\n ensure_extension_loaded(name, path=extension_root)\n commands = glob.glob(\n os.path.join(extension_root, extension_name(extension_root), \"cmd\", \"*.py\")\n )\n commands = [os.path.basename(x).rstrip(\".py\") for x in commands]\n for command in commands:\n load_command_extension(command, extension_root)\n return extension_root\n\n\ndef get_extension_paths():\n \"\"\"Return the list of canonicalized extension paths from config:extensions.\"\"\"\n extension_paths = spack.config.get(\"config:extensions\") or []\n extension_paths.extend(extension_paths_from_entry_points())\n paths = [spack.util.path.canonicalize_path(p) for p in extension_paths]\n return paths\n\n\n@spack.llnl.util.lang.memoized\ndef extension_paths_from_entry_points() -> List[str]:\n \"\"\"Load extensions from a Python package's entry points.\n\n A python package can register entry point metadata so that Spack can find\n its extensions by adding the following to the project's pyproject.toml:\n\n .. code-block:: toml\n\n [project.entry-points.\"spack.extensions\"]\n baz = \"baz:get_spack_extensions\"\n\n The function ``get_spack_extensions`` returns paths to the package's\n spack extensions\n\n This function assumes that the state of entry points doesn't change from the first time it's\n called. E.g., it doesn't support any new installation of packages between two calls.\n \"\"\"\n extension_paths: List[str] = []\n for entry_point in spack.llnl.util.lang.get_entry_points(group=\"spack.extensions\"):\n hook = entry_point.load()\n if callable(hook):\n paths = hook() or []\n if isinstance(paths, (Path, str)):\n extension_paths.append(str(paths))\n else:\n extension_paths.extend(paths)\n return extension_paths\n\n\ndef get_command_paths():\n \"\"\"Return the list of paths where to search for command files.\"\"\"\n command_paths = []\n extension_paths = get_extension_paths()\n\n for path in extension_paths:\n extension = _python_name(extension_name(path))\n command_paths.append(os.path.join(path, extension, \"cmd\"))\n\n return command_paths\n\n\ndef path_for_extension(target_name: str, *, paths: List[str]) -> str:\n \"\"\"Return the test root dir for a given extension.\n\n Args:\n target_name (str): name of the extension to test\n *paths: paths where the extensions reside\n\n Returns:\n Root directory where tests should reside or None\n \"\"\"\n for path in paths:\n name = extension_name(path)\n if name == target_name:\n return path\n else:\n raise OSError('extension \"{0}\" not found'.format(target_name))\n\n\ndef get_module(cmd_name):\n \"\"\"Imports the extension module for a particular command name\n and returns it.\n\n Args:\n cmd_name (str): name of the command for which to get a module\n (contains ``-``, not ``_``).\n \"\"\"\n # If built-in failed the import search the extension\n # directories in order\n extensions = get_extension_paths()\n for folder in extensions:\n module = load_command_extension(cmd_name, folder)\n if module:\n return module\n return None\n\n\ndef get_template_dirs():\n \"\"\"Returns the list of directories where to search for templates\n in extensions.\n \"\"\"\n extension_dirs = get_extension_paths()\n extensions = [os.path.join(x, \"templates\") for x in extension_dirs]\n return extensions\n\n\nclass ExtensionNamingError(spack.error.SpackError):\n \"\"\"Exception class thrown when a configured extension does not follow\n the expected naming convention.\n \"\"\"\n\n def __init__(self, path):\n super().__init__(\"{0} does not match the format for a Spack extension path.\".format(path))\n" }, { "path": "lib/spack/spack/externals.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"\nThis module turns the configuration data in the ``packages`` section into a list of concrete specs.\n\nThis is mainly done by the ``ExternalSpecsParser`` class, which is responsible for:\n\n 1. Transforming an intermediate representation of the YAML configuration into a set of nodes\n 2. Ensuring the dependency specifications are not ambiguous\n 3. Inferring missing information about the external specs (e.g. architecture, deptypes)\n 4. Wiring up the external specs to their dependencies\n\nThe helper function ``extract_dicts_from_configuration`` is used to transform the configuration\ninto the intermediate representation.\n\"\"\"\n\nimport re\nimport uuid\nimport warnings\nfrom typing import Any, Callable, Dict, List, NamedTuple, Tuple, Union\n\nfrom spack.vendor.typing_extensions import TypedDict\n\nimport spack.archspec\nimport spack.deptypes\nimport spack.repo\nimport spack.spec\nfrom spack.error import SpackError\nfrom spack.llnl.util import tty\n\n\nclass DependencyDict(TypedDict, total=False):\n id: str\n spec: str\n deptypes: spack.deptypes.DepTypes\n virtuals: str\n\n\nclass ExternalDict(TypedDict, total=False):\n \"\"\"Dictionary representation of an external spec.\n\n This representation mostly follows the one used in the configuration files, with a few\n exceptions needed to support specific features.\n \"\"\"\n\n spec: str\n prefix: str\n modules: List[str]\n extra_attributes: Dict[str, Any]\n id: str\n dependencies: List[DependencyDict]\n # Target requirement from configuration. Not in the external schema\n required_target: str\n\n\ndef node_from_dict(external_dict: ExternalDict) -> spack.spec.Spec:\n \"\"\"Returns an external spec node from a dictionary representation.\"\"\"\n extra_attributes = external_dict.get(\"extra_attributes\", {})\n result = spack.spec.Spec(\n # Allow `@x.y.z` instead of `@=x.y.z`\n str(spack.spec.parse_with_version_concrete(external_dict[\"spec\"])),\n external_path=external_dict.get(\"prefix\"),\n external_modules=external_dict.get(\"modules\"),\n )\n if not result.versions.concrete:\n raise ExternalSpecError(\n f\"The external spec '{external_dict['spec']}' doesn't have a concrete version\"\n )\n\n result.extra_attributes = extra_attributes\n if \"required_target\" in external_dict:\n result.constrain(f\"target={external_dict['required_target']}\")\n return result\n\n\ndef complete_architecture(node: spack.spec.Spec) -> None:\n \"\"\"Completes a node with architecture information.\n\n Undefined targets are set to the default host target family (e.g. ``x86_64``).\n The operating system and platform are set based on the current host.\n \"\"\"\n if node.architecture:\n if not node.architecture.target:\n node.architecture.target = spack.archspec.HOST_TARGET_FAMILY\n node.architecture.complete_with_defaults()\n else:\n node.constrain(spack.spec.Spec.default_arch())\n node.architecture.target = spack.archspec.HOST_TARGET_FAMILY\n\n node.namespace = spack.repo.PATH.repo_for_pkg(node.name).namespace\n for flag_type in spack.spec.FlagMap.valid_compiler_flags():\n node.compiler_flags.setdefault(flag_type, [])\n\n\ndef complete_variants_and_architecture(node: spack.spec.Spec) -> None:\n \"\"\"Completes a node with variants and architecture information.\n\n Architecture is completed first, delegating to ``complete_architecture``.\n Variants are then added to the node, using their default value.\n \"\"\"\n complete_architecture(node)\n pkg_class = spack.repo.PATH.get_pkg_class(node.name)\n variants_dict = pkg_class.variants.copy()\n changed = True\n\n while variants_dict and changed:\n changed = False\n items = list(variants_dict.items()) # copy b/c loop modifies dict\n\n for when, variants_by_name in items:\n if not node.satisfies(when):\n continue\n variants_dict.pop(when)\n for name, vdef in variants_by_name.items():\n if name not in node.variants:\n # Cannot use Spec.constrain, because we lose information on the variant type\n node.variants[name] = vdef.make_default()\n elif (\n node.variants[name].type != vdef.variant_type\n and len(node.variants[name].values) == 1\n ):\n # Spec parsing defaults to MULTI for non-boolean variants. Correct the type\n # using the package definition, preserving the user-specified value.\n existing = node.variants[name]\n corrected = vdef.make_variant(*existing.values)\n node.variants.substitute(corrected)\n changed = True\n\n\ndef extract_dicts_from_configuration(packages_yaml) -> List[ExternalDict]:\n \"\"\"Transforms the packages.yaml configuration into a list of external dictionaries.\n\n The default required target is extracted from ``packages:all:require``, if present.\n Any package-specific required target overrides the default.\n \"\"\"\n result = []\n default_required_target = \"\"\n if \"all\" in packages_yaml:\n default_required_target = _required_target(packages_yaml[\"all\"])\n\n for name, entry in packages_yaml.items():\n pkg_required_target = _required_target(entry) or default_required_target\n partial_result = [current for current in entry.get(\"externals\", [])]\n if pkg_required_target:\n for partial in partial_result:\n partial[\"required_target\"] = pkg_required_target\n result.extend(partial_result)\n return result\n\n\ndef _line_info(config_dict: Any) -> str:\n result = getattr(config_dict, \"line_info\", \"\")\n return \"\" if not result else f\" [{result}]\"\n\n\n_TARGET_RE = re.compile(r\"target=([^\\s:]+)\")\n\n\ndef _required_target(entry) -> str:\n \"\"\"Parses the YAML configuration for a single external spec and returns the required target\n if defined. Returns an empty string otherwise.\n \"\"\"\n if \"require\" not in entry:\n return \"\"\n\n requirements = entry[\"require\"]\n if not isinstance(requirements, list):\n requirements = [requirements]\n\n results = []\n for requirement in requirements:\n if not isinstance(requirement, str):\n continue\n\n matches = _TARGET_RE.match(requirement)\n if matches:\n results.append(matches.group(1))\n\n if len(results) == 1:\n return results[0]\n\n return \"\"\n\n\nclass ExternalSpecAndConfig(NamedTuple):\n spec: spack.spec.Spec\n config: ExternalDict\n\n\nclass ExternalSpecsParser:\n \"\"\"Transforms a list of external dicts into a list of specs.\"\"\"\n\n def __init__(\n self,\n external_dicts: List[ExternalDict],\n *,\n complete_node: Callable[[spack.spec.Spec], None] = complete_variants_and_architecture,\n allow_nonexisting: bool = True,\n ):\n \"\"\"Initializes a class to manage and process external specifications in ``packages.yaml``.\n\n Args:\n external_dicts: list of ExternalDict objects to provide external specifications.\n complete_node: a callable that completes a node with missing variants, targets, etc.\n Defaults to `complete_architecture`.\n allow_nonexisting: whether to allow non-existing packages. Defaults to True.\n\n Raises:\n spack.repo.UnknownPackageError: if a package does not exist,\n and allow_nonexisting is False.\n \"\"\"\n self.external_dicts = external_dicts\n self.specs_by_external_id: Dict[str, ExternalSpecAndConfig] = {}\n self.specs_by_name: Dict[str, List[ExternalSpecAndConfig]] = {}\n self.nodes: List[spack.spec.Spec] = []\n self.allow_nonexisting = allow_nonexisting\n # Fill the data structures above (can be done lazily)\n self.complete_node = complete_node\n self._parse()\n\n def _parse(self) -> None:\n # Parse all nodes without creating edges among them\n self._parse_all_nodes()\n # Map dependencies specified as specs to a single id\n self._ensure_dependencies_have_single_id()\n # Attach dependencies to externals\n self._create_edges()\n # Mark the specs as concrete\n for node in self.nodes:\n node._finalize_concretization()\n\n def _create_edges(self):\n for eid, entry in self.specs_by_external_id.items():\n current_node, current_dict = entry.spec, entry.config\n line_info = _line_info(current_dict)\n spec_str = current_dict[\"spec\"]\n\n # Compute the dependency types for this spec\n pkg_class, deptypes_by_package = spack.repo.PATH.get_pkg_class(current_node.name), {}\n for when, by_name in pkg_class.dependencies.items():\n if not current_node.satisfies(when):\n continue\n for name, dep in by_name.items():\n if name not in deptypes_by_package:\n deptypes_by_package[name] = dep.depflag\n deptypes_by_package[name] |= dep.depflag\n\n for dependency_dict in current_dict.get(\"dependencies\", []):\n dependency_id = dependency_dict.get(\"id\")\n if not dependency_id:\n raise ExternalDependencyError(\n f\"A dependency for {spec_str} does not have an external id{line_info}\"\n )\n elif dependency_id not in self.specs_by_external_id:\n raise ExternalDependencyError(\n f\"A dependency for {spec_str} has an external id \"\n f\"{dependency_id} that cannot be found in packages.yaml{line_info}\"\n )\n\n dependency_node = self.specs_by_external_id[dependency_id].spec\n\n # Compute dependency types and virtuals\n depflag = spack.deptypes.NONE\n if \"deptypes\" in dependency_dict:\n depflag = spack.deptypes.canonicalize(dependency_dict[\"deptypes\"])\n\n virtuals: Tuple[str, ...] = ()\n if \"virtuals\" in dependency_dict:\n virtuals = tuple(dependency_dict[\"virtuals\"].split(\",\"))\n\n # Infer dependency types and virtuals if the user didn't specify them\n if depflag == spack.deptypes.NONE and not virtuals:\n # Infer the deptype if only '%' was used in the spec\n inferred_virtuals = []\n for name, current_flag in deptypes_by_package.items():\n if not dependency_node.intersects(name):\n continue\n depflag |= current_flag\n if spack.repo.PATH.is_virtual(name):\n inferred_virtuals.append(name)\n virtuals = tuple(inferred_virtuals)\n elif depflag == spack.deptypes.NONE:\n depflag = spack.deptypes.DEFAULT\n\n current_node._add_dependency(dependency_node, depflag=depflag, virtuals=virtuals)\n\n def _ensure_dependencies_have_single_id(self):\n for eid, entry in self.specs_by_external_id.items():\n current_node, current_dict = entry.spec, entry.config\n spec_str = current_dict[\"spec\"]\n line_info = _line_info(current_dict)\n\n if current_node.dependencies() and \"dependencies\" in current_dict:\n raise ExternalSpecError(\n f\"the spec {spec_str} cannot specify dependencies both in the root spec and\"\n f\"in the 'dependencies' field{line_info}\"\n )\n\n # Transform inline entries like 'mpich %gcc' to a canonical form using 'dependencies'\n for edge in current_node.edges_to_dependencies():\n entry: DependencyDict = {\"spec\": str(edge.spec)}\n\n # Handle entries with more options specified\n if edge.depflag != 0:\n entry[\"deptypes\"] = spack.deptypes.flag_to_tuple(edge.depflag)\n\n if edge.virtuals:\n entry[\"virtuals\"] = \",\".join(edge.virtuals)\n\n current_dict.setdefault(\"dependencies\", []).append(entry)\n current_node.clear_edges()\n\n # Map a spec: to id:\n for dependency_dict in current_dict.get(\"dependencies\", []):\n if \"id\" in dependency_dict:\n continue\n\n if \"spec\" not in dependency_dict:\n raise ExternalDependencyError(\n f\"the spec {spec_str} needs to specify either the id or the spec \"\n f\"of its dependencies{line_info}\"\n )\n\n query_spec = spack.spec.Spec(dependency_dict[\"spec\"])\n candidates = [\n x\n for x in self.specs_by_name.get(query_spec.name, [])\n if x.spec.satisfies(query_spec)\n ]\n if len(candidates) == 0:\n raise ExternalDependencyError(\n f\"the spec '{spec_str}' depends on '{query_spec}', but there is no such \"\n f\"external spec in packages.yaml{line_info}\"\n )\n elif len(candidates) > 1:\n candidates_str = (\n f\" [candidates are {', '.join([str(x.spec) for x in candidates])}]\"\n )\n raise ExternalDependencyError(\n f\"the spec '{spec_str}' depends on '{query_spec}', but there are multiple \"\n f\"external specs that could satisfy the request{candidates_str}{line_info}\"\n )\n\n dependency_dict[\"id\"] = candidates[0].config[\"id\"]\n\n def _parse_all_nodes(self) -> None:\n \"\"\"Parses all the nodes from the external dicts but doesn't add any edge.\"\"\"\n for external_dict in self.external_dicts:\n line_info = _line_info(external_dict)\n try:\n node = node_from_dict(external_dict)\n except spack.spec.UnsatisfiableArchitectureSpecError:\n spec_str, target_str = external_dict[\"spec\"], external_dict[\"required_target\"]\n tty.debug(\n f\"[{__name__}]{line_info} Skipping external spec '{spec_str}' because it \"\n f\"cannot be constrained with the required target '{target_str}'.\"\n )\n continue\n except ExternalSpecError as e:\n warnings.warn(f\"{e}{line_info}\")\n continue\n\n package_exists = spack.repo.PATH.exists(node.name)\n\n # If we allow non-existing packages, just continue\n if not package_exists and self.allow_nonexisting:\n continue\n\n if not package_exists and not self.allow_nonexisting:\n raise ExternalSpecError(f\"Package '{node.name}' does not exist{line_info}\")\n\n eid = external_dict.setdefault(\"id\", str(uuid.uuid4()))\n if eid in self.specs_by_external_id:\n other_node = self.specs_by_external_id[eid]\n other_line_info = _line_info(other_node.config)\n raise DuplicateExternalError(\n f\"Specs {node} and {other_node.spec} cannot have the same external id {eid}\"\n f\"{line_info}{other_line_info}\"\n )\n\n self.complete_node(node)\n\n # Add a Python dependency to Python extensions that don't specify it\n pkg_class = spack.repo.PATH.get_pkg_class(node.name)\n if (\n \"dependencies\" not in external_dict\n and not node.dependencies()\n and any([c.__name__ == \"PythonExtension\" for c in pkg_class.__mro__])\n ):\n warnings.warn(\n f\"Spack is trying attach a Python dependency to '{node}'. This feature is \"\n f\"deprecated, and will be removed in v1.2. Please make the dependency \"\n f\"explicit in your configuration.\"\n )\n external_dict.setdefault(\"dependencies\", []).append({\"spec\": \"python\"})\n\n # Normalize internally so that each node has a unique id\n spec_and_config = ExternalSpecAndConfig(spec=node, config=external_dict)\n self.specs_by_external_id[eid] = spec_and_config\n self.specs_by_name.setdefault(node.name, []).append(spec_and_config)\n self.nodes.append(node)\n\n def get_specs_for_package(self, package_name: str) -> List[spack.spec.Spec]:\n \"\"\"Returns the external specs for a given package name.\"\"\"\n result = self.specs_by_name.get(package_name, [])\n return [x.spec for x in result]\n\n def all_specs(self) -> List[spack.spec.Spec]:\n \"\"\"Returns all the external specs.\"\"\"\n return self.nodes\n\n def query(self, query: Union[str, spack.spec.Spec]) -> List[spack.spec.Spec]:\n \"\"\"Returns the external specs matching a query spec.\"\"\"\n result = []\n for node in self.nodes:\n if node.satisfies(query):\n result.append(node)\n return result\n\n\ndef external_spec(config: ExternalDict) -> spack.spec.Spec:\n \"\"\"Returns an external spec from a dictionary representation.\"\"\"\n return ExternalSpecsParser([config]).all_specs()[0]\n\n\nclass DuplicateExternalError(SpackError):\n \"\"\"Raised when a duplicate external is detected.\"\"\"\n\n\nclass ExternalDependencyError(SpackError):\n \"\"\"Raised when a dependency on an external package is specified wrongly.\"\"\"\n\n\nclass ExternalSpecError(SpackError):\n \"\"\"Raised when a dependency on an external package is specified wrongly.\"\"\"\n" }, { "path": "lib/spack/spack/fetch_strategy.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"\nFetch strategies are used to download source code into a staging area\nin order to build it. They need to define the following methods:\n\n``fetch()``\n This should attempt to download/check out source from somewhere.\n\n``check()``\n Apply a checksum to the downloaded source code, e.g. for an archive.\n May not do anything if the fetch method was safe to begin with.\n\n``expand()``\n Expand (e.g., an archive) downloaded file to source, with the\n standard stage source path as the destination directory.\n\n``reset()``\n Restore original state of downloaded code. Used by clean commands.\n This may just remove the expanded source and re-expand an archive,\n or it may run something like git reset ``--hard``.\n\n``archive()``\n Archive a source directory, e.g. for creating a mirror.\n\"\"\"\n\nimport copy\nimport functools\nimport hashlib\nimport http.client\nimport os\nimport re\nimport shutil\nimport sys\nimport time\nimport urllib.parse\nimport urllib.request\nfrom pathlib import PurePath\nfrom typing import Callable, List, Mapping, Optional, Type\n\nimport spack.config\nimport spack.error\nimport spack.llnl.url\nimport spack.llnl.util.filesystem as fs\nimport spack.llnl.util.tty as tty\nimport spack.oci.opener\nimport spack.util.archive\nimport spack.util.crypto as crypto\nimport spack.util.executable\nimport spack.util.git\nimport spack.util.url as url_util\nimport spack.util.web as web_util\nimport spack.version\nfrom spack.llnl.string import comma_and, quote\nfrom spack.llnl.util.filesystem import get_single_file, mkdirp, symlink, temp_cwd, working_dir\nfrom spack.util.compression import decompressor_for\nfrom spack.util.executable import CommandNotFoundError, Executable, which\n\n#: List of all fetch strategies, created by FetchStrategy metaclass.\nall_strategies: List[Type[\"FetchStrategy\"]] = []\n\n\ndef _needs_stage(fun):\n \"\"\"Many methods on fetch strategies require a stage to be set\n using set_stage(). This decorator adds a check for self.stage.\"\"\"\n\n @functools.wraps(fun)\n def wrapper(self, *args, **kwargs):\n if not self.stage:\n raise NoStageError(fun)\n return fun(self, *args, **kwargs)\n\n return wrapper\n\n\ndef _ensure_one_stage_entry(stage_path):\n \"\"\"Ensure there is only one stage entry in the stage path.\"\"\"\n stage_entries = os.listdir(stage_path)\n assert len(stage_entries) == 1\n return os.path.join(stage_path, stage_entries[0])\n\n\ndef fetcher(cls):\n \"\"\"Decorator used to register fetch strategies.\"\"\"\n all_strategies.append(cls)\n return cls\n\n\nclass FetchStrategy:\n \"\"\"Superclass of all fetch strategies.\"\"\"\n\n #: The URL attribute must be specified either at the package class\n #: level, or as a keyword argument to ``version()``. It is used to\n #: distinguish fetchers for different versions in the package DSL.\n url_attr: Optional[str] = None\n\n #: Optional attributes can be used to distinguish fetchers when :\n #: classes have multiple ``url_attrs`` at the top-level.\n # optional attributes in version() args.\n optional_attrs: List[str] = []\n\n def __init__(self, **kwargs):\n # The stage is initialized late, so that fetch strategies can be\n # constructed at package construction time. This is where things\n # will be fetched.\n self.stage = None\n # Enable or disable caching for this strategy based on\n # 'no_cache' option from version directive.\n self.cache_enabled = not kwargs.pop(\"no_cache\", False)\n\n self.package = None\n\n def set_package(self, package):\n self.package = package\n\n # Subclasses need to implement these methods\n def fetch(self):\n \"\"\"Fetch source code archive or repo.\n\n Returns:\n bool: True on success, False on failure.\n \"\"\"\n\n def check(self):\n \"\"\"Checksum the archive fetched by this FetchStrategy.\"\"\"\n\n def expand(self):\n \"\"\"Expand the downloaded archive into the stage source path.\"\"\"\n\n def reset(self):\n \"\"\"Revert to freshly downloaded state.\n\n For archive files, this may just re-expand the archive.\n \"\"\"\n\n def archive(self, destination):\n \"\"\"Create an archive of the downloaded data for a mirror.\n\n For downloaded files, this should preserve the checksum of the\n original file. For repositories, it should just create an\n expandable tarball out of the downloaded repository.\n \"\"\"\n\n @property\n def cachable(self):\n \"\"\"Whether fetcher is capable of caching the resource it retrieves.\n\n This generally is determined by whether the resource is\n identifiably associated with a specific package version.\n\n Returns:\n bool: True if can cache, False otherwise.\n \"\"\"\n\n def source_id(self):\n \"\"\"A unique ID for the source.\n\n It is intended that a human could easily generate this themselves using\n the information available to them in the Spack package.\n\n The returned value is added to the content which determines the full\n hash for a package using :class:`str`.\n \"\"\"\n raise NotImplementedError\n\n def mirror_id(self):\n \"\"\"This is a unique ID for a source that is intended to help identify\n reuse of resources across packages.\n\n It is unique like source-id, but it does not include the package name\n and is not necessarily easy for a human to create themselves.\n \"\"\"\n raise NotImplementedError\n\n def __str__(self): # Should be human readable URL.\n return \"FetchStrategy.__str___\"\n\n @classmethod\n def matches(cls, args):\n \"\"\"Predicate that matches fetch strategies to arguments of\n the version directive.\n\n Args:\n args: arguments of the version directive\n \"\"\"\n return cls.url_attr in args\n\n\n@fetcher\nclass BundleFetchStrategy(FetchStrategy):\n \"\"\"\n Fetch strategy associated with bundle, or no-code, packages.\n\n Having a basic fetch strategy is a requirement for executing post-install\n hooks. Consequently, this class provides the API but does little more\n than log messages.\n\n TODO: Remove this class by refactoring resource handling and the link\n between composite stages and composite fetch strategies (see #11981).\n \"\"\"\n\n #: There is no associated URL keyword in ``version()`` for no-code\n #: packages but this property is required for some strategy-related\n #: functions (e.g., check_pkg_attributes).\n url_attr = \"\"\n\n def fetch(self):\n \"\"\"Simply report success -- there is no code to fetch.\"\"\"\n return True\n\n @property\n def cachable(self):\n \"\"\"Report False as there is no code to cache.\"\"\"\n return False\n\n def source_id(self):\n \"\"\"BundlePackages don't have a source id.\"\"\"\n return \"\"\n\n def mirror_id(self):\n \"\"\"BundlePackages don't have a mirror id.\"\"\"\n\n\ndef _format_speed(total_bytes: int, elapsed: float) -> str:\n \"\"\"Return a human-readable average download speed string.\"\"\"\n elapsed = 1 if elapsed <= 0 else elapsed # avoid divide by zero\n speed = total_bytes / elapsed\n if speed >= 1e9:\n return f\"{speed / 1e9:6.1f} GB/s\"\n elif speed >= 1e6:\n return f\"{speed / 1e6:6.1f} MB/s\"\n elif speed >= 1e3:\n return f\"{speed / 1e3:6.1f} KB/s\"\n return f\"{speed:6.1f} B/s\"\n\n\ndef _format_bytes(total_bytes: int) -> str:\n \"\"\"Return a human-readable total bytes string.\"\"\"\n if total_bytes >= 1e9:\n return f\"{total_bytes / 1e9:7.2f} GB\"\n elif total_bytes >= 1e6:\n return f\"{total_bytes / 1e6:7.2f} MB\"\n elif total_bytes >= 1e3:\n return f\"{total_bytes / 1e3:7.2f} KB\"\n return f\"{total_bytes:7.2f} B\"\n\n\nclass FetchProgress:\n #: Characters to rotate in the spinner.\n spinner = [\"|\", \"/\", \"-\", \"\\\\\"]\n\n def __init__(\n self,\n total_bytes: Optional[int] = None,\n enabled: bool = True,\n get_time: Callable[[], float] = time.time,\n ) -> None:\n \"\"\"Initialize a FetchProgress instance.\n Args:\n total_bytes: Total number of bytes to download, if known.\n enabled: Whether to print progress information.\n get_time: Function to get the current time.\"\"\"\n #: Number of bytes downloaded so far.\n self.current_bytes = 0\n #: Delta time between progress prints\n self.delta = 0.1\n #: Whether to print progress information.\n self.enabled = enabled\n #: Function to get the current time.\n self.get_time = get_time\n #: Time of last progress print to limit output\n self.last_printed = 0.0\n #: Time of start of download\n self.start_time = get_time() if enabled else 0.0\n #: Total number of bytes to download, if known.\n self.total_bytes = total_bytes if total_bytes and total_bytes > 0 else 0\n #: Index of spinner character to print (used if total bytes is unknown)\n self.index = 0\n\n @classmethod\n def from_headers(\n cls,\n headers: Mapping[str, str],\n enabled: bool = True,\n get_time: Callable[[], float] = time.time,\n ) -> \"FetchProgress\":\n \"\"\"Create a FetchProgress instance from HTTP headers.\"\"\"\n # headers.get is case-insensitive if it's from a HTTPResponse object.\n content_length = headers.get(\"Content-Length\")\n try:\n total_bytes = int(content_length) if content_length else None\n except ValueError:\n total_bytes = None\n return cls(total_bytes=total_bytes, enabled=enabled, get_time=get_time)\n\n def advance(self, num_bytes: int, out=sys.stdout) -> None:\n if not self.enabled:\n return\n self.current_bytes += num_bytes\n self.print(out=out)\n\n def print(self, final: bool = False, out=sys.stdout) -> None:\n if not self.enabled:\n return\n current_time = self.get_time()\n if self.last_printed + self.delta < current_time or final:\n self.last_printed = current_time\n # print a newline if this is the final update\n maybe_newline = \"\\n\" if final else \"\"\n # if we know the total bytes, show a percentage, otherwise a spinner\n if self.total_bytes > 0:\n percentage = min(100 * self.current_bytes / self.total_bytes, 100.0)\n percent_or_spinner = f\"[{percentage:3.0f}%] \"\n else:\n # only show the spinner if we are not at 100%\n if final:\n percent_or_spinner = \"[100%] \"\n else:\n percent_or_spinner = f\"[ {self.spinner[self.index]} ] \"\n self.index = (self.index + 1) % len(self.spinner)\n\n print(\n f\"\\r {percent_or_spinner}{_format_bytes(self.current_bytes)} \"\n f\"@ {_format_speed(self.current_bytes, current_time - self.start_time)}\"\n f\"{maybe_newline}\",\n end=\"\",\n flush=True,\n file=out,\n )\n\n\n@fetcher\nclass URLFetchStrategy(FetchStrategy):\n \"\"\"URLFetchStrategy pulls source code from a URL for an archive, check the\n archive against a checksum, and decompresses the archive.\n\n The destination for the resulting file(s) is the standard stage path.\n \"\"\"\n\n url_attr = \"url\"\n\n # these are checksum types. The generic 'checksum' is deprecated for\n # specific hash names, but we need it for backward compatibility\n optional_attrs = [*crypto.hashes.keys(), \"checksum\"]\n\n def __init__(self, *, url: str, checksum: Optional[str] = None, **kwargs) -> None:\n super().__init__(**kwargs)\n\n self.url = url\n self.mirrors = kwargs.get(\"mirrors\", [])\n\n # digest can be set as the first argument, or from an explicit\n # kwarg by the hash name.\n self.digest: Optional[str] = checksum\n for h in self.optional_attrs:\n if h in kwargs:\n self.digest = kwargs[h]\n\n self.expand_archive: bool = kwargs.get(\"expand\", True)\n self.extra_options: dict = kwargs.get(\"fetch_options\", {})\n self._curl: Optional[Executable] = None\n self.extension: Optional[str] = kwargs.get(\"extension\", None)\n self._effective_url: Optional[str] = None\n\n @property\n def curl(self) -> Executable:\n if not self._curl:\n self._curl = web_util.require_curl()\n return self._curl\n\n def source_id(self):\n return self.digest\n\n def mirror_id(self):\n if not self.digest:\n return None\n # The filename is the digest. A directory is also created based on\n # truncating the digest to avoid creating a directory with too many\n # entries\n return os.path.sep.join([\"archive\", self.digest[:2], self.digest])\n\n @property\n def candidate_urls(self):\n return [self.url] + (self.mirrors or [])\n\n @_needs_stage\n def fetch(self):\n if self.archive_file:\n tty.debug(f\"Already downloaded {self.archive_file}\")\n return\n\n errors: List[Exception] = []\n for url in self.candidate_urls:\n try:\n self._fetch_from_url(url)\n break\n except FailedDownloadError as e:\n errors.extend(e.exceptions)\n else:\n raise FailedDownloadError(*errors)\n\n if not self.archive_file:\n raise FailedDownloadError(\n RuntimeError(f\"Missing archive {self.archive_file} after fetching\")\n )\n\n def _fetch_from_url(self, url):\n fetch_method = spack.config.get(\"config:url_fetch_method\", \"urllib\")\n if fetch_method.startswith(\"curl\"):\n return self._fetch_curl(url, config_args=fetch_method.split()[1:])\n else:\n return self._fetch_urllib(url)\n\n def _check_headers(self, headers):\n # Check if we somehow got an HTML file rather than the archive we\n # asked for. We only look at the last content type, to handle\n # redirects properly.\n content_types = re.findall(r\"Content-Type:[^\\r\\n]+\", headers, flags=re.IGNORECASE)\n if content_types and \"text/html\" in content_types[-1]:\n msg = (\n f\"The contents of {self.archive_file or 'the archive'} fetched from {self.url} \"\n \" looks like HTML. This can indicate a broken URL, or an internet gateway issue.\"\n )\n if self._effective_url != self.url:\n msg += f\" The URL redirected to {self._effective_url}.\"\n tty.warn(msg)\n\n @_needs_stage\n def _fetch_urllib(self, url, chunk_size=65536, retries=5):\n \"\"\"Fetch a URL using urllib, with retries on transient errors and progress reporting.\"\"\"\n save_file = self.stage.save_filename\n part_file = save_file + \".part\"\n\n request = urllib.request.Request(\n url, headers={\"User-Agent\": web_util.SPACK_USER_AGENT, \"Accept\": \"*/*\"}\n )\n\n response_headers_str = None\n for attempt in range(retries):\n try:\n with web_util.urlopen(request) as response:\n tty.verbose(f\"Fetching {url}\")\n progress = FetchProgress.from_headers(\n response.headers, enabled=sys.stdout.isatty()\n )\n with open(part_file, \"wb\") as f:\n while True:\n chunk = response.read(chunk_size)\n if not chunk:\n break\n f.write(chunk)\n progress.advance(len(chunk))\n progress.print(final=True)\n # Capture metadata before context manager closes the connection\n if isinstance(response, http.client.HTTPResponse):\n self._effective_url = response.geturl()\n response_headers_str = str(response.headers)\n os.replace(part_file, save_file)\n break # success: exit retry loop\n except Exception as e:\n # clean up archive on failure.\n if self.archive_file:\n os.remove(self.archive_file)\n if os.path.lexists(part_file):\n os.remove(part_file)\n # Raise if this was the last attempt, or if the error was not transient.\n if (attempt + 1 == retries) or not web_util.is_transient_error(e):\n raise FailedDownloadError(e) from e\n tty.debug(f\"Retrying fetch (attempt {attempt + 1}): {e}\")\n time.sleep(2**attempt)\n\n # Save the redirected URL for error messages. Sometimes we're redirected to an arbitrary\n # mirror that is broken, leading to spurious download failures. In that case it's helpful\n # for users to know which URL was actually fetched.\n self._check_headers(response_headers_str)\n\n @_needs_stage\n def _fetch_curl(self, url, config_args=[]):\n save_file = None\n partial_file = None\n if self.stage.save_filename:\n save_file = self.stage.save_filename\n partial_file = self.stage.save_filename + \".part\"\n tty.verbose(f\"Fetching {url}\")\n if partial_file:\n save_args = [\n \"-C\",\n \"-\", # continue partial downloads\n \"-o\",\n partial_file,\n ] # use a .part file\n else:\n save_args = [\"-O\"]\n\n timeout = 0\n cookie_args = []\n if self.extra_options:\n cookie = self.extra_options.get(\"cookie\")\n if cookie:\n cookie_args.append(\"-j\") # junk cookies\n cookie_args.append(\"-b\") # specify cookie\n cookie_args.append(cookie)\n\n timeout = self.extra_options.get(\"timeout\")\n\n base_args = web_util.base_curl_fetch_args(url, timeout)\n curl_args = config_args + save_args + base_args + cookie_args\n\n # Run curl but grab the mime type from the http headers\n curl = self.curl\n with working_dir(self.stage.path):\n headers = curl(*curl_args, output=str, fail_on_error=False)\n\n if curl.returncode != 0:\n # clean up archive on failure.\n if self.archive_file:\n os.remove(self.archive_file)\n\n if partial_file and os.path.lexists(partial_file):\n os.remove(partial_file)\n\n try:\n web_util.check_curl_code(curl.returncode)\n except spack.error.FetchError as e:\n raise FailedDownloadError(e) from e\n\n self._check_headers(headers)\n\n if save_file and (partial_file is not None):\n fs.rename(partial_file, save_file)\n\n @property # type: ignore # decorated properties unsupported in mypy\n @_needs_stage\n def archive_file(self):\n \"\"\"Path to the source archive within this stage directory.\"\"\"\n return self.stage.archive_file\n\n @property\n def cachable(self):\n return self.cache_enabled and bool(self.digest)\n\n @_needs_stage\n def expand(self):\n if not self.expand_archive:\n tty.debug(\n \"Staging unexpanded archive {0} in {1}\".format(\n self.archive_file, self.stage.source_path\n )\n )\n if not self.stage.expanded:\n mkdirp(self.stage.source_path)\n dest = os.path.join(self.stage.source_path, os.path.basename(self.archive_file))\n shutil.move(self.archive_file, dest)\n return\n\n tty.debug(\"Staging archive: {0}\".format(self.archive_file))\n\n if not self.archive_file:\n raise NoArchiveFileError(\n \"Couldn't find archive file\", \"Failed on expand() for URL %s\" % self.url\n )\n\n # TODO: replace this by mime check.\n if not self.extension:\n self.extension = spack.llnl.url.determine_url_file_extension(self.url)\n\n if self.stage.expanded:\n tty.debug(\"Source already staged to %s\" % self.stage.source_path)\n return\n\n decompress = decompressor_for(self.archive_file, self.extension)\n\n # Below we assume that the command to decompress expand the\n # archive in the current working directory\n with fs.exploding_archive_catch(self.stage):\n decompress(self.archive_file)\n\n def archive(self, destination):\n \"\"\"Just moves this archive to the destination.\"\"\"\n if not self.archive_file:\n raise NoArchiveFileError(\"Cannot call archive() before fetching.\")\n\n web_util.push_to_url(\n self.archive_file, url_util.path_to_file_url(destination), keep_original=True\n )\n\n @_needs_stage\n def check(self):\n \"\"\"Check the downloaded archive against a checksum digest.\n No-op if this stage checks code out of a repository.\"\"\"\n if not self.digest:\n raise NoDigestError(f\"Attempt to check {self.__class__.__name__} with no digest.\")\n\n verify_checksum(self.archive_file, self.digest, self.url, self._effective_url)\n\n @_needs_stage\n def reset(self):\n \"\"\"\n Removes the source path if it exists, then re-expands the archive.\n \"\"\"\n if not self.archive_file:\n raise NoArchiveFileError(\n f\"Tried to reset {self.__class__.__name__} before fetching\",\n f\"Failed on reset() for URL{self.url}\",\n )\n\n # Remove everything but the archive from the stage\n for filename in os.listdir(self.stage.path):\n abspath = os.path.join(self.stage.path, filename)\n if abspath != self.archive_file:\n shutil.rmtree(abspath, ignore_errors=True)\n\n # Expand the archive again\n self.expand()\n\n def __repr__(self):\n return f\"{self.__class__.__name__}<{self.url}>\"\n\n def __str__(self):\n return self.url\n\n\n@fetcher\nclass CacheURLFetchStrategy(URLFetchStrategy):\n \"\"\"The resource associated with a cache URL may be out of date.\"\"\"\n\n @_needs_stage\n def fetch(self):\n path = url_util.file_url_string_to_path(self.url)\n\n # check whether the cache file exists.\n if not os.path.isfile(path):\n raise NoCacheError(f\"No cache of {path}\")\n\n # remove old symlink if one is there.\n filename = self.stage.save_filename\n if os.path.lexists(filename):\n os.remove(filename)\n\n # Symlink to local cached archive.\n symlink(path, filename)\n\n # Remove link if checksum fails, or subsequent fetchers will assume they don't need to\n # download.\n if self.digest:\n try:\n self.check()\n except ChecksumError:\n os.remove(self.archive_file)\n raise\n\n # Notify the user how we fetched.\n tty.msg(f\"Using cached archive: {path}\")\n\n\nclass OCIRegistryFetchStrategy(URLFetchStrategy):\n def __init__(self, *, url: str, checksum: Optional[str] = None, **kwargs):\n super().__init__(url=url, checksum=checksum, **kwargs)\n\n self._urlopen = kwargs.get(\"_urlopen\", spack.oci.opener.urlopen)\n\n @_needs_stage\n def fetch(self):\n file = self.stage.save_filename\n\n if os.path.lexists(file):\n os.remove(file)\n\n try:\n response = self._urlopen(self.url)\n tty.verbose(f\"Fetching {self.url}\")\n with open(file, \"wb\") as f:\n shutil.copyfileobj(response, f)\n except OSError as e:\n # clean up archive on failure.\n if self.archive_file:\n os.remove(self.archive_file)\n if os.path.lexists(file):\n os.remove(file)\n raise FailedDownloadError(e) from e\n\n\nclass VCSFetchStrategy(FetchStrategy):\n \"\"\"Superclass for version control system fetch strategies.\n\n Like all fetchers, VCS fetchers are identified by the attributes\n passed to the ``version`` directive. The optional_attrs for a VCS\n fetch strategy represent types of revisions, e.g. tags, branches,\n commits, etc.\n\n The required attributes (git, svn, etc.) are used to specify the URL\n and to distinguish a VCS fetch strategy from a URL fetch strategy.\n\n \"\"\"\n\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n\n # Set a URL based on the type of fetch strategy.\n self.url = kwargs.get(self.url_attr, None)\n if not self.url:\n raise ValueError(f\"{self.__class__} requires {self.url_attr} argument.\")\n\n for attr in self.optional_attrs:\n setattr(self, attr, kwargs.get(attr, None))\n\n @_needs_stage\n def check(self):\n tty.debug(f\"No checksum needed when fetching with {self.url_attr}\")\n\n @_needs_stage\n def expand(self):\n tty.debug(f\"Source fetched with {self.url_attr} is already expanded.\")\n\n @_needs_stage\n def archive(self, destination, *, exclude: Optional[str] = None):\n assert spack.llnl.url.extension_from_path(destination) == \"tar.gz\"\n assert self.stage.source_path.startswith(self.stage.path)\n # We need to prepend this dir name to every entry of the tarfile\n top_level_dir = PurePath(self.stage.srcdir or os.path.basename(self.stage.source_path))\n\n with working_dir(self.stage.source_path), spack.util.archive.gzip_compressed_tarfile(\n destination\n ) as (tar, _, _):\n spack.util.archive.reproducible_tarfile_from_prefix(\n tar=tar,\n prefix=\".\",\n skip=lambda entry: entry.name == exclude,\n path_to_name=lambda path: (top_level_dir / PurePath(path)).as_posix(),\n )\n\n def __str__(self):\n return f\"VCS: {self.url}\"\n\n def __repr__(self):\n return f\"{self.__class__}<{self.url}>\"\n\n\n@fetcher\nclass GoFetchStrategy(VCSFetchStrategy):\n \"\"\"Fetch strategy that employs the ``go get`` infrastructure.\n\n Use like this in a package::\n\n version(\"name\", go=\"github.com/monochromegane/the_platinum_searcher/...\")\n\n Go get does not natively support versions, they can be faked with git.\n\n The fetched source will be moved to the standard stage sourcepath directory\n during the expand step.\n \"\"\"\n\n url_attr = \"go\"\n\n def __init__(self, **kwargs):\n # Discards the keywords in kwargs that may conflict with the next\n # call to __init__\n forwarded_args = copy.copy(kwargs)\n forwarded_args.pop(\"name\", None)\n super().__init__(**forwarded_args)\n\n self._go = None\n\n @property\n def go_version(self):\n vstring = self.go(\"version\", output=str).split(\" \")[2]\n return spack.version.Version(vstring)\n\n @property\n def go(self):\n if not self._go:\n self._go = which(\"go\", required=True)\n return self._go\n\n @_needs_stage\n def fetch(self):\n tty.debug(\"Getting go resource: {0}\".format(self.url))\n\n with working_dir(self.stage.path):\n try:\n os.mkdir(\"go\")\n except OSError:\n pass\n env = dict(os.environ)\n env[\"GOPATH\"] = os.path.join(os.getcwd(), \"go\")\n self.go(\"get\", \"-v\", \"-d\", self.url, env=env)\n\n def archive(self, destination):\n super().archive(destination, exclude=\".git\")\n\n @_needs_stage\n def expand(self):\n tty.debug(\"Source fetched with %s is already expanded.\" % self.url_attr)\n\n # Move the directory to the well-known stage source path\n repo_root = _ensure_one_stage_entry(self.stage.path)\n shutil.move(repo_root, self.stage.source_path)\n\n @_needs_stage\n def reset(self):\n with working_dir(self.stage.source_path):\n self.go(\"clean\")\n\n def __str__(self):\n return \"[go] %s\" % self.url\n\n\n@fetcher\nclass GitFetchStrategy(VCSFetchStrategy):\n \"\"\"\n Fetch strategy that gets source code from a git repository.\n Use like this in a package::\n\n version(\"name\", git=\"https://github.com/project/repo.git\")\n\n Optionally, you can provide a branch, or commit to check out, e.g.::\n\n version(\"1.1\", git=\"https://github.com/project/repo.git\", tag=\"v1.1\")\n\n You can use these three optional attributes in addition to ``git``:\n\n * ``branch``: Particular branch to build from (default is the repository's default branch)\n * ``tag``: Particular tag to check out\n * ``commit``: Particular commit hash in the repo\n\n Repositories are cloned into the standard stage source path directory.\n \"\"\"\n\n url_attr = \"git\"\n optional_attrs = [\n \"tag\",\n \"branch\",\n \"commit\",\n \"submodules\",\n \"get_full_repo\",\n \"submodules_delete\",\n \"git_sparse_paths\",\n \"skip_checkout\",\n ]\n\n def __init__(self, **kwargs):\n\n self.commit: Optional[str] = None\n self.tag: Optional[str] = None\n self.branch: Optional[str] = None\n\n # Discards the keywords in kwargs that may conflict with the next call\n # to __init__\n forwarded_args = copy.copy(kwargs)\n forwarded_args.pop(\"name\", None)\n super().__init__(**forwarded_args)\n\n self._git = None\n self.submodules = kwargs.get(\"submodules\", False)\n self.submodules_delete = kwargs.get(\"submodules_delete\", False)\n self.get_full_repo = kwargs.get(\"get_full_repo\", False)\n self.git_sparse_paths = kwargs.get(\"git_sparse_paths\", None)\n # skipping checkout with a blobless clone is an efficient way to traverse meta-data\n # see https://bhupesh.me/minimalist-guide-git-clone/\n self.skip_checkout = kwargs.get(\"skip_checkout\", False)\n\n @property\n def git_version(self):\n return GitFetchStrategy.version_from_git(self.git)\n\n @staticmethod\n def version_from_git(git_exe):\n \"\"\"Given a git executable, return the Version (this will fail if\n the output cannot be parsed into a valid Version).\n \"\"\"\n version_string = \".\".join(map(str, git_exe.version))\n return spack.version.Version(version_string)\n\n @property\n def git(self):\n if not self._git:\n try:\n self._git = spack.util.git.git(required=True)\n except CommandNotFoundError as exc:\n tty.error(str(exc))\n raise\n\n # Disable advice for a quieter fetch\n # https://github.com/git/git/blob/master/Documentation/RelNotes/1.7.2.txt\n if self.git_version >= spack.version.Version(\"1.7.2\"):\n self._git.add_default_arg(\"-c\", \"advice.detachedHead=false\")\n\n # If the user asked for insecure fetching, make that work\n # with git as well.\n if not spack.config.get(\"config:verify_ssl\"):\n self._git.add_default_env(\"GIT_SSL_NO_VERIFY\", \"true\")\n\n return self._git\n\n @property\n def cachable(self):\n return self.cache_enabled and bool(self.commit)\n\n def source_id(self):\n # TODO: tree-hash would secure download cache and mirrors, commit only secures checkouts.\n # TODO(psakiev): Tree-hash is part of the commit SHA computation, question comment validity\n return self.commit\n\n def mirror_id(self):\n if self.commit:\n provenance_id = self.commit\n repo_path = urllib.parse.urlparse(self.url).path\n if self.git_sparse_paths:\n sparse_paths = []\n if callable(self.git_sparse_paths):\n sparse_paths.extend(self.git_sparse_paths())\n else:\n sparse_paths.extend(self.git_sparse_paths)\n sparse_string = \"_\".join(sparse_paths)\n sparse_hash = hashlib.sha1(sparse_string.encode(\"utf-8\")).hexdigest()\n provenance_id = f\"{provenance_id}_{sparse_hash}\"\n result = os.path.sep.join([\"git\", repo_path, provenance_id])\n return result\n\n def _repo_info(self):\n args = \"\"\n if self.commit:\n args = f\" at commit {self.commit}\"\n elif self.tag:\n args = f\" at tag {self.tag}\"\n elif self.branch:\n args = f\" on branch {self.branch}\"\n\n return f\"{self.url}{args}\"\n\n @_needs_stage\n def fetch(self):\n if self.stage.expanded:\n tty.debug(f\"Already fetched {self.stage.source_path}\")\n return\n\n self._clone_src()\n self.submodule_operations()\n\n def bare_clone(self, dest: str) -> None:\n \"\"\"\n Execute a bare clone for metadata only\n\n Requires a destination since bare cloning does not provide source\n and shouldn't be used for staging.\n \"\"\"\n # Default to spack source path\n tty.debug(f\"Cloning git repository: {self._repo_info()}\")\n\n git = self.git\n debug = spack.config.get(\"config:debug\")\n\n # We don't need to worry about which commit/branch/tag is checked out\n clone_args = [\"clone\", \"--bare\"]\n if not debug:\n clone_args.append(\"--quiet\")\n clone_args.extend([self.url, dest])\n git(*clone_args)\n\n def _clone_src(self) -> None:\n \"\"\"Clone a repository to a path using git.\"\"\"\n # Default to spack source path\n dest = self.stage.source_path\n tty.debug(f\"Cloning git repository: {self._repo_info()}\")\n\n depth = None if self.get_full_repo else 1\n name = self.package.name if self.package else None\n checkout_ref = self.commit or self.tag or self.branch\n fetch_ref = self.tag or self.branch\n\n kwargs = {\"debug\": spack.config.get(\"config:debug\"), \"git_exe\": self.git, \"dest\": name}\n\n # TODO(psakievich) The use of the minimal clone need clearer justification via package API\n # or something. There is a trade space of storage minimization vs available git information\n # that grows to non-trivial proportions for larger projects\n minimal_clone = self.commit and name and not self.get_full_repo\n\n with temp_cwd(ignore_cleanup_errors=True):\n if minimal_clone:\n try:\n spack.util.git.git_init_fetch(self.url, self.commit, depth, **kwargs)\n except spack.util.executable.ProcessError:\n spack.util.git.git_clone(\n self.url, fetch_ref, self.get_full_repo, depth, **kwargs\n )\n else:\n spack.util.git.git_clone(self.url, fetch_ref, self.get_full_repo, depth, **kwargs)\n repo_name = get_single_file(\".\")\n kwargs[\"dest\"] = repo_name\n if not self.skip_checkout:\n spack.util.git.git_checkout(checkout_ref, self.git_sparse_paths, **kwargs)\n\n if self.stage:\n self.stage.srcdir = repo_name\n shutil.copytree(repo_name, dest, symlinks=True)\n return\n\n def submodule_operations(self):\n dest = self.stage.source_path\n git = self.git\n\n if self.submodules_delete:\n with working_dir(dest):\n for submodule_to_delete in self.submodules_delete:\n args = [\"rm\", submodule_to_delete]\n if not spack.config.get(\"config:debug\"):\n args.insert(1, \"--quiet\")\n git(*args)\n\n # Init submodules if the user asked for them.\n git_commands = []\n submodules = self.submodules\n if callable(submodules):\n submodules = submodules(self.package)\n if submodules:\n if isinstance(submodules, str):\n submodules = [submodules]\n git_commands.append([\"submodule\", \"init\", \"--\"] + submodules)\n git_commands.append([\"submodule\", \"update\", \"--recursive\"])\n elif submodules:\n git_commands.append([\"submodule\", \"update\", \"--init\", \"--recursive\"])\n\n if not git_commands:\n return\n\n with working_dir(dest):\n for args in git_commands:\n if not spack.config.get(\"config:debug\"):\n args.insert(1, \"--quiet\")\n git(*args)\n\n @_needs_stage\n def reset(self):\n with working_dir(self.stage.source_path):\n co_args = [\"checkout\", \".\"]\n clean_args = [\"clean\", \"-f\"]\n if spack.config.get(\"config:debug\"):\n co_args.insert(1, \"--quiet\")\n clean_args.insert(1, \"--quiet\")\n\n self.git(*co_args)\n self.git(*clean_args)\n\n def __str__(self):\n return f\"[git] {self._repo_info()}\"\n\n\n@fetcher\nclass CvsFetchStrategy(VCSFetchStrategy):\n \"\"\"Fetch strategy that gets source code from a CVS repository.\n Use like this in a package::\n\n version(\"name\", cvs=\":pserver:anonymous@www.example.com:/cvsroot%module=modulename\")\n\n Optionally, you can provide a branch and/or a date for the URL::\n\n version(\n \"name\",\n cvs=\":pserver:anonymous@www.example.com:/cvsroot%module=modulename\",\n branch=\"branchname\", date=\"date\"\n )\n\n Repositories are checked out into the standard stage source path directory.\n \"\"\"\n\n url_attr = \"cvs\"\n optional_attrs = [\"branch\", \"date\"]\n\n def __init__(self, **kwargs):\n # Discards the keywords in kwargs that may conflict with the next call\n # to __init__\n forwarded_args = copy.copy(kwargs)\n forwarded_args.pop(\"name\", None)\n super().__init__(**forwarded_args)\n\n self._cvs = None\n if self.branch is not None:\n self.branch = str(self.branch)\n if self.date is not None:\n self.date = str(self.date)\n\n @property\n def cvs(self):\n if not self._cvs:\n self._cvs = which(\"cvs\", required=True)\n return self._cvs\n\n @property\n def cachable(self):\n return self.cache_enabled and (bool(self.branch) or bool(self.date))\n\n def source_id(self):\n if not (self.branch or self.date):\n # We need a branch or a date to make a checkout reproducible\n return None\n id = \"id\"\n if self.branch:\n id += \"-branch=\" + self.branch\n if self.date:\n id += \"-date=\" + self.date\n return id\n\n def mirror_id(self):\n if not (self.branch or self.date):\n # We need a branch or a date to make a checkout reproducible\n return None\n # Special-case handling because this is not actually a URL\n elements = self.url.split(\":\")\n final = elements[-1]\n elements = final.split(\"/\")\n # Everything before the first slash is a port number\n elements = elements[1:]\n result = os.path.sep.join([\"cvs\"] + elements)\n if self.branch:\n result += \"%branch=\" + self.branch\n if self.date:\n result += \"%date=\" + self.date\n return result\n\n @_needs_stage\n def fetch(self):\n if self.stage.expanded:\n tty.debug(\"Already fetched {0}\".format(self.stage.source_path))\n return\n\n tty.debug(\"Checking out CVS repository: {0}\".format(self.url))\n\n with temp_cwd():\n url, module = self.url.split(\"%module=\")\n # Check out files\n args = [\"-z9\", \"-d\", url, \"checkout\"]\n if self.branch is not None:\n args.extend([\"-r\", self.branch])\n if self.date is not None:\n args.extend([\"-D\", self.date])\n args.append(module)\n self.cvs(*args)\n # Rename repo\n repo_name = get_single_file(\".\")\n self.stage.srcdir = repo_name\n shutil.move(repo_name, self.stage.source_path)\n\n def _remove_untracked_files(self):\n \"\"\"Removes untracked files in a CVS repository.\"\"\"\n with working_dir(self.stage.source_path):\n status = self.cvs(\"-qn\", \"update\", output=str)\n for line in status.split(\"\\n\"):\n if re.match(r\"^[?]\", line):\n path = line[2:].strip()\n if os.path.isfile(path):\n os.unlink(path)\n\n def archive(self, destination):\n super().archive(destination, exclude=\"CVS\")\n\n @_needs_stage\n def reset(self):\n self._remove_untracked_files()\n with working_dir(self.stage.source_path):\n self.cvs(\"update\", \"-C\", \".\")\n\n def __str__(self):\n return \"[cvs] %s\" % self.url\n\n\n@fetcher\nclass SvnFetchStrategy(VCSFetchStrategy):\n \"\"\"Fetch strategy that gets source code from a subversion repository.\n Use like this in a package::\n\n version(\"name\", svn=\"http://www.example.com/svn/trunk\")\n\n Optionally, you can provide a revision for the URL::\n\n version(\"name\", svn=\"http://www.example.com/svn/trunk\", revision=\"1641\")\n\n Repositories are checked out into the standard stage source path directory.\n \"\"\"\n\n url_attr = \"svn\"\n optional_attrs = [\"revision\"]\n\n def __init__(self, **kwargs):\n # Discards the keywords in kwargs that may conflict with the next call\n # to __init__\n forwarded_args = copy.copy(kwargs)\n forwarded_args.pop(\"name\", None)\n super().__init__(**forwarded_args)\n\n self._svn = None\n if self.revision is not None:\n self.revision = str(self.revision)\n\n @property\n def svn(self):\n if not self._svn:\n self._svn = which(\"svn\", required=True)\n return self._svn\n\n @property\n def cachable(self):\n return self.cache_enabled and bool(self.revision)\n\n def source_id(self):\n return self.revision\n\n def mirror_id(self):\n if self.revision:\n repo_path = urllib.parse.urlparse(self.url).path\n result = os.path.sep.join([\"svn\", repo_path, self.revision])\n return result\n\n @_needs_stage\n def fetch(self):\n if self.stage.expanded:\n tty.debug(\"Already fetched {0}\".format(self.stage.source_path))\n return\n\n tty.debug(\"Checking out subversion repository: {0}\".format(self.url))\n\n args = [\"checkout\", \"--force\", \"--quiet\"]\n if self.revision:\n args += [\"-r\", self.revision]\n args.extend([self.url])\n\n with temp_cwd():\n self.svn(*args)\n repo_name = get_single_file(\".\")\n self.stage.srcdir = repo_name\n shutil.move(repo_name, self.stage.source_path)\n\n def _remove_untracked_files(self):\n \"\"\"Removes untracked files in an svn repository.\"\"\"\n with working_dir(self.stage.source_path):\n status = self.svn(\"status\", \"--no-ignore\", output=str)\n self.svn(\"status\", \"--no-ignore\")\n for line in status.split(\"\\n\"):\n if not re.match(\"^[I?]\", line):\n continue\n path = line[8:].strip()\n if os.path.isfile(path):\n os.unlink(path)\n elif os.path.isdir(path):\n shutil.rmtree(path, ignore_errors=True)\n\n def archive(self, destination):\n super().archive(destination, exclude=\".svn\")\n\n @_needs_stage\n def reset(self):\n self._remove_untracked_files()\n with working_dir(self.stage.source_path):\n self.svn(\"revert\", \".\", \"-R\")\n\n def __str__(self):\n return \"[svn] %s\" % self.url\n\n\n@fetcher\nclass HgFetchStrategy(VCSFetchStrategy):\n \"\"\"\n Fetch strategy that gets source code from a Mercurial repository.\n Use like this in a package::\n\n version(\"name\", hg=\"https://jay.grs.rwth-aachen.de/hg/lwm2\")\n\n Optionally, you can provide a branch, or revision to check out, e.g.::\n\n version(\"torus\", hg=\"https://jay.grs.rwth-aachen.de/hg/lwm2\", branch=\"torus\")\n\n You can use the optional ``revision`` attribute to check out a\n branch, tag, or particular revision in hg. To prevent\n non-reproducible builds, using a moving target like a branch is\n discouraged.\n\n * ``revision``: Particular revision, branch, or tag.\n\n Repositories are cloned into the standard stage source path directory.\n \"\"\"\n\n url_attr = \"hg\"\n optional_attrs = [\"revision\"]\n\n def __init__(self, **kwargs):\n # Discards the keywords in kwargs that may conflict with the next call\n # to __init__\n forwarded_args = copy.copy(kwargs)\n forwarded_args.pop(\"name\", None)\n super().__init__(**forwarded_args)\n\n self._hg = None\n\n @property\n def hg(self):\n \"\"\"\n Returns:\n Executable: the hg executable\n \"\"\"\n if not self._hg:\n self._hg = which(\"hg\", required=True)\n\n # When building PythonPackages, Spack automatically sets\n # PYTHONPATH. This can interfere with hg, which is a Python\n # script. Unset PYTHONPATH while running hg.\n self._hg.add_default_env(\"PYTHONPATH\", \"\")\n\n return self._hg\n\n @property\n def cachable(self):\n return self.cache_enabled and bool(self.revision)\n\n def source_id(self):\n return self.revision\n\n def mirror_id(self):\n if self.revision:\n repo_path = urllib.parse.urlparse(self.url).path\n result = os.path.sep.join([\"hg\", repo_path, self.revision])\n return result\n\n @_needs_stage\n def fetch(self):\n if self.stage.expanded:\n tty.debug(\"Already fetched {0}\".format(self.stage.source_path))\n return\n\n args = []\n if self.revision:\n args.append(\"at revision %s\" % self.revision)\n tty.debug(\"Cloning mercurial repository: {0} {1}\".format(self.url, args))\n\n args = [\"clone\"]\n\n if not spack.config.get(\"config:verify_ssl\"):\n args.append(\"--insecure\")\n\n if self.revision:\n args.extend([\"-r\", self.revision])\n\n args.extend([self.url])\n\n with temp_cwd():\n self.hg(*args)\n repo_name = get_single_file(\".\")\n self.stage.srcdir = repo_name\n shutil.move(repo_name, self.stage.source_path)\n\n def archive(self, destination):\n super().archive(destination, exclude=\".hg\")\n\n @_needs_stage\n def reset(self):\n with working_dir(self.stage.path):\n source_path = self.stage.source_path\n scrubbed = \"scrubbed-source-tmp\"\n\n args = [\"clone\"]\n if self.revision:\n args += [\"-r\", self.revision]\n args += [source_path, scrubbed]\n self.hg(*args)\n\n shutil.rmtree(source_path, ignore_errors=True)\n shutil.move(scrubbed, source_path)\n\n def __str__(self):\n return f\"[hg] {self.url}\"\n\n\n@fetcher\nclass S3FetchStrategy(URLFetchStrategy):\n \"\"\"FetchStrategy that pulls from an S3 bucket.\"\"\"\n\n url_attr = \"s3\"\n\n @_needs_stage\n def fetch(self):\n if not self.url.startswith(\"s3://\"):\n raise spack.error.FetchError(\n f\"{self.__class__.__name__} can only fetch from s3:// urls.\"\n )\n if self.archive_file:\n tty.debug(f\"Already downloaded {self.archive_file}\")\n return\n self._fetch_urllib(self.url)\n if not self.archive_file:\n raise FailedDownloadError(\n RuntimeError(f\"Missing archive {self.archive_file} after fetching\")\n )\n\n\n@fetcher\nclass GCSFetchStrategy(URLFetchStrategy):\n \"\"\"FetchStrategy that pulls from a GCS bucket.\"\"\"\n\n url_attr = \"gs\"\n\n @_needs_stage\n def fetch(self):\n if not self.url.startswith(\"gs\"):\n raise spack.error.FetchError(\n f\"{self.__class__.__name__} can only fetch from gs:// urls.\"\n )\n if self.archive_file:\n tty.debug(f\"Already downloaded {self.archive_file}\")\n return\n\n self._fetch_urllib(self.url)\n\n if not self.archive_file:\n raise FailedDownloadError(\n RuntimeError(f\"Missing archive {self.archive_file} after fetching\")\n )\n\n\n@fetcher\nclass FetchAndVerifyExpandedFile(URLFetchStrategy):\n \"\"\"Fetch strategy that verifies the content digest during fetching,\n as well as after expanding it.\"\"\"\n\n def __init__(self, url, archive_sha256: str, expanded_sha256: str):\n super().__init__(url=url, checksum=archive_sha256)\n self.expanded_sha256 = expanded_sha256\n\n def expand(self):\n \"\"\"Verify checksum after expanding the archive.\"\"\"\n\n # Expand the archive\n super().expand()\n\n # Ensure a single patch file.\n src_dir = self.stage.source_path\n files = os.listdir(src_dir)\n\n if len(files) != 1:\n raise ChecksumError(self, f\"Expected a single file in {src_dir}.\")\n\n verify_checksum(\n os.path.join(src_dir, files[0]), self.expanded_sha256, self.url, self._effective_url\n )\n\n\ndef verify_checksum(file: str, digest: str, url: str, effective_url: Optional[str]) -> None:\n checker = crypto.Checker(digest)\n if not checker.check(file):\n # On failure, provide some information about the file size and\n # contents, so that we can quickly see what the issue is (redirect\n # was not followed, empty file, text instead of binary, ...)\n size, contents = fs.filesummary(file)\n long_msg = (\n f\"Expected {digest} but got {checker.sum}. \"\n f\"File size = {size} bytes. Contents = {contents!r}. \"\n f\"URL = {url}\"\n )\n if effective_url and effective_url != url:\n long_msg += f\", redirected to = {effective_url}\"\n raise ChecksumError(f\"{checker.hash_name} checksum failed for {file}\", long_msg)\n\n\ndef stable_target(fetcher):\n \"\"\"Returns whether the fetcher target is expected to have a stable\n checksum. This is only true if the target is a preexisting archive\n file.\"\"\"\n if isinstance(fetcher, URLFetchStrategy) and fetcher.cachable:\n return True\n return False\n\n\ndef from_url(url: str) -> URLFetchStrategy:\n \"\"\"Given a URL, find an appropriate fetch strategy for it.\n Currently just gives you a URLFetchStrategy that uses curl.\n\n TODO: make this return appropriate fetch strategies for other types of URLs.\n \"\"\"\n return URLFetchStrategy(url=url)\n\n\ndef from_kwargs(**kwargs) -> FetchStrategy:\n \"\"\"Construct an appropriate FetchStrategy from the given keyword arguments.\n\n Args:\n **kwargs: dictionary of keyword arguments, e.g. from a ``version()`` directive in a\n package.\n\n Returns:\n The fetch strategy that matches the args, based on attribute names (e.g., ``git``, ``hg``,\n etc.)\n\n Raises:\n spack.error.FetchError: If no ``fetch_strategy`` matches the args.\n \"\"\"\n for fetcher in all_strategies:\n if fetcher.matches(kwargs):\n return fetcher(**kwargs)\n\n raise InvalidArgsError(**kwargs)\n\n\ndef check_pkg_attributes(pkg):\n \"\"\"Find ambiguous top-level fetch attributes in a package.\n\n Currently this only ensures that two or more VCS fetch strategies are\n not specified at once.\n \"\"\"\n # a single package cannot have URL attributes for multiple VCS fetch\n # strategies *unless* they are the same attribute.\n conflicts = set([s.url_attr for s in all_strategies if hasattr(pkg, s.url_attr)])\n\n # URL isn't a VCS fetch method. We can use it with a VCS method.\n conflicts -= set([\"url\"])\n\n if len(conflicts) > 1:\n raise FetcherConflict(\n \"Package %s cannot specify %s together. Pick at most one.\"\n % (pkg.name, comma_and(quote(conflicts)))\n )\n\n\ndef _check_version_attributes(fetcher, pkg, version):\n \"\"\"Ensure that the fetcher for a version is not ambiguous.\n\n This assumes that we have already determined the fetcher for the\n specific version using ``for_package_version()``\n \"\"\"\n all_optionals = set(a for s in all_strategies for a in s.optional_attrs)\n\n args = pkg.versions[version]\n extra = set(args) - set(fetcher.optional_attrs) - set([fetcher.url_attr, \"no_cache\"])\n extra.intersection_update(all_optionals)\n\n if extra:\n legal_attrs = [fetcher.url_attr] + list(fetcher.optional_attrs)\n raise FetcherConflict(\n \"%s version '%s' has extra arguments: %s\"\n % (pkg.name, version, comma_and(quote(extra))),\n \"Valid arguments for a %s fetcher are: \\n %s\"\n % (fetcher.url_attr, comma_and(quote(legal_attrs))),\n )\n\n\ndef _extrapolate(pkg, version):\n \"\"\"Create a fetcher from an extrapolated URL for this version.\"\"\"\n try:\n return URLFetchStrategy(url=pkg.url_for_version(version), fetch_options=pkg.fetch_options)\n except spack.error.NoURLError:\n raise ExtrapolationError(\n f\"Can't extrapolate a URL for version {version} because \"\n f\"package {pkg.name} defines no URLs\"\n )\n\n\ndef _from_merged_attrs(fetcher, pkg, version):\n \"\"\"Create a fetcher from merged package and version attributes.\"\"\"\n if fetcher.url_attr == \"url\":\n mirrors = pkg.all_urls_for_version(version)\n url = mirrors[0]\n mirrors = mirrors[1:]\n attrs = {fetcher.url_attr: url, \"mirrors\": mirrors}\n else:\n url = getattr(pkg, fetcher.url_attr)\n attrs = {fetcher.url_attr: url}\n\n attrs[\"fetch_options\"] = pkg.fetch_options\n attrs.update(pkg.versions[version])\n\n if fetcher.url_attr == \"git\":\n pkg_attr_list = [\"submodules\", \"git_sparse_paths\"]\n for pkg_attr in pkg_attr_list:\n if hasattr(pkg, pkg_attr):\n attrs.setdefault(pkg_attr, getattr(pkg, pkg_attr))\n\n return fetcher(**attrs)\n\n\ndef for_package_version(pkg, version=None):\n saved_versions = None\n if version is not None:\n saved_versions = pkg.spec.versions\n\n try:\n return _for_package_version(pkg, version)\n finally:\n if saved_versions is not None:\n pkg.spec.versions = saved_versions\n\n\ndef _for_package_version(pkg, version=None):\n \"\"\"Determine a fetch strategy based on the arguments supplied to\n version() in the package description.\"\"\"\n\n # No-code packages have a custom fetch strategy to work around issues\n # with resource staging.\n if not pkg.has_code:\n return BundleFetchStrategy()\n\n check_pkg_attributes(pkg)\n\n if version is not None:\n assert not pkg.spec.concrete, \"concrete specs should not pass the 'version=' argument\"\n # Specs are initialized with the universe range, if no version information is given,\n # so here we make sure we always match the version passed as argument\n if not isinstance(version, spack.version.StandardVersion):\n version = spack.version.Version(version)\n\n version_list = spack.version.VersionList()\n version_list.add(version)\n pkg.spec.versions = version_list\n else:\n version = pkg.version\n\n # if it's a commit, we must use a GitFetchStrategy\n commit_var = pkg.spec.variants.get(\"commit\", None)\n commit = commit_var.value if commit_var else None\n tag = None\n if isinstance(version, spack.version.GitVersion) or commit:\n git_url = pkg.version_or_package_attr(\"git\", version)\n if not git_url:\n raise spack.error.FetchError(\n f\"Cannot fetch git version for {pkg.name}. Package has no 'git' attribute\"\n )\n if isinstance(version, spack.version.GitVersion):\n # Populate the version with comparisons to other commits\n from spack.version.git_ref_lookup import GitRefLookup\n\n version.attach_lookup(GitRefLookup(pkg.name))\n\n if not commit and version.is_commit:\n commit = version.ref\n version_meta_data = pkg.versions.get(version.std_version)\n else:\n version_meta_data = pkg.versions.get(version)\n\n # For GitVersion, we have no way to determine whether a ref is a branch or tag\n # Fortunately, we handle branches and tags identically, except tags are\n # handled slightly more conservatively for older versions of git.\n # We call all non-commit refs tags in this context, at the cost of a slight\n # performance hit for branches on older versions of git.\n # Branches cannot be cached, so we tell the fetcher not to cache tags/branches\n\n # TODO(psakiev) eventually we should only need to clone based on the commit\n\n # commit stashed on version\n if version_meta_data:\n if not commit:\n commit = version_meta_data.get(\"commit\")\n tag = version_meta_data.get(\"tag\") or version_meta_data.get(\"branch\")\n\n kwargs = {\"commit\": commit, \"tag\": tag, \"no_cache\": bool(not commit)}\n kwargs[\"git\"] = git_url\n kwargs[\"submodules\"] = pkg.version_or_package_attr(\"submodules\", version, False)\n kwargs[\"git_sparse_paths\"] = pkg.version_or_package_attr(\"git_sparse_paths\", version, None)\n kwargs[\"get_full_repo\"] = pkg.version_or_package_attr(\"get_full_repo\", version, False)\n\n # if the ref_version is a known version from the package, use that version's\n # attributes\n ref_version = getattr(pkg.version, \"ref_version\", None)\n if ref_version:\n kwargs[\"git\"] = pkg.version_or_package_attr(\"git\", ref_version)\n kwargs[\"submodules\"] = pkg.version_or_package_attr(\"submodules\", ref_version, False)\n\n fetcher = GitFetchStrategy(**kwargs)\n return fetcher\n\n # If it's not a known version, try to extrapolate one by URL\n if version not in pkg.versions:\n return _extrapolate(pkg, version)\n\n # Set package args first so version args can override them\n args = {\"fetch_options\": pkg.fetch_options}\n # Grab a dict of args out of the package version dict\n args.update(pkg.versions[version])\n\n # If the version specifies a `url_attr` directly, use that.\n for fetcher in all_strategies:\n if fetcher.url_attr in args:\n _check_version_attributes(fetcher, pkg, version)\n if fetcher.url_attr == \"git\" and hasattr(pkg, \"submodules\"):\n args.setdefault(\"submodules\", pkg.submodules)\n return fetcher(**args)\n\n # if a version's optional attributes imply a particular fetch\n # strategy, and we have the `url_attr`, then use that strategy.\n for fetcher in all_strategies:\n if hasattr(pkg, fetcher.url_attr) or fetcher.url_attr == \"url\":\n optionals = fetcher.optional_attrs\n if optionals and any(a in args for a in optionals):\n _check_version_attributes(fetcher, pkg, version)\n return _from_merged_attrs(fetcher, pkg, version)\n\n # if the optional attributes tell us nothing, then use any `url_attr`\n # on the package. This prefers URL vs. VCS, b/c URLFetchStrategy is\n # defined first in this file.\n for fetcher in all_strategies:\n if hasattr(pkg, fetcher.url_attr):\n _check_version_attributes(fetcher, pkg, version)\n return _from_merged_attrs(fetcher, pkg, version)\n\n raise InvalidArgsError(pkg, version, **args)\n\n\ndef from_url_scheme(url: str, **kwargs) -> FetchStrategy:\n \"\"\"Finds a suitable FetchStrategy by matching its url_attr with the scheme\n in the given url.\"\"\"\n parsed_url = urllib.parse.urlparse(url, scheme=\"file\")\n\n scheme_mapping = kwargs.get(\"scheme_mapping\") or {\n \"file\": \"url\",\n \"http\": \"url\",\n \"https\": \"url\",\n \"ftp\": \"url\",\n \"ftps\": \"url\",\n }\n\n scheme = parsed_url.scheme\n scheme = scheme_mapping.get(scheme, scheme)\n\n for fetcher in all_strategies:\n url_attr = getattr(fetcher, \"url_attr\", None)\n if url_attr and url_attr == scheme:\n return fetcher(url=url, **kwargs)\n\n raise ValueError(f'No FetchStrategy found for url with scheme: \"{parsed_url.scheme}\"')\n\n\ndef from_list_url(pkg):\n \"\"\"If a package provides a URL which lists URLs for resources by\n version, this can can create a fetcher for a URL discovered for\n the specified package's version.\"\"\"\n\n if pkg.list_url:\n try:\n versions = pkg.fetch_remote_versions()\n try:\n # get a URL, and a checksum if we have it\n url_from_list = versions[pkg.version]\n checksum = None\n\n # try to find a known checksum for version, from the package\n version = pkg.version\n if version in pkg.versions:\n args = pkg.versions[version]\n checksum = next(\n (v for k, v in args.items() if k in crypto.hashes), args.get(\"checksum\")\n )\n\n # construct a fetcher\n return URLFetchStrategy(\n url=url_from_list, checksum=checksum, fetch_options=pkg.fetch_options\n )\n except KeyError as e:\n tty.debug(e)\n tty.msg(\"Cannot find version %s in url_list\" % pkg.version)\n\n except BaseException as e:\n # TODO: Don't catch BaseException here! Be more specific.\n tty.debug(e)\n tty.msg(\"Could not determine url from list_url.\")\n\n\nclass FsCache:\n def __init__(self, root):\n self.root = os.path.abspath(root)\n\n def store(self, fetcher, relative_dest):\n # skip fetchers that aren't cachable\n if not fetcher.cachable:\n return\n\n # Don't store things that are already cached.\n if isinstance(fetcher, CacheURLFetchStrategy):\n return\n\n dst = os.path.join(self.root, relative_dest)\n mkdirp(os.path.dirname(dst))\n fetcher.archive(dst)\n\n def fetcher(self, target_path: str, digest: Optional[str], **kwargs) -> CacheURLFetchStrategy:\n path = os.path.join(self.root, target_path)\n url = url_util.path_to_file_url(path)\n return CacheURLFetchStrategy(url=url, checksum=digest, **kwargs)\n\n def destroy(self):\n shutil.rmtree(self.root, ignore_errors=True)\n\n\nclass NoCacheError(spack.error.FetchError):\n \"\"\"Raised when there is no cached archive for a package.\"\"\"\n\n\nclass FailedDownloadError(spack.error.FetchError):\n \"\"\"Raised when a download fails.\"\"\"\n\n def __init__(self, *exceptions: Exception):\n super().__init__(\"Failed to download\")\n self.exceptions = exceptions\n\n\nclass NoArchiveFileError(spack.error.FetchError):\n \"\"\"Raised when an archive file is expected but none exists.\"\"\"\n\n\nclass NoDigestError(spack.error.FetchError):\n \"\"\"Raised after attempt to checksum when URL has no digest.\"\"\"\n\n\nclass ExtrapolationError(spack.error.FetchError):\n \"\"\"Raised when we can't extrapolate a version for a package.\"\"\"\n\n\nclass FetcherConflict(spack.error.FetchError):\n \"\"\"Raised for packages with invalid fetch attributes.\"\"\"\n\n\nclass InvalidArgsError(spack.error.FetchError):\n \"\"\"Raised when a version can't be deduced from a set of arguments.\"\"\"\n\n def __init__(self, pkg=None, version=None, **args):\n msg = \"Could not guess a fetch strategy\"\n if pkg:\n msg += \" for {pkg}\".format(pkg=pkg)\n if version:\n msg += \"@{version}\".format(version=version)\n long_msg = \"with arguments: {args}\".format(args=args)\n super().__init__(msg, long_msg)\n\n\nclass ChecksumError(spack.error.FetchError):\n \"\"\"Raised when archive fails to checksum.\"\"\"\n\n\nclass NoStageError(spack.error.FetchError):\n \"\"\"Raised when fetch operations are called before set_stage().\"\"\"\n\n def __init__(self, method):\n super().__init__(\"Must call FetchStrategy.set_stage() before calling %s\" % method.__name__)\n" }, { "path": "lib/spack/spack/filesystem_view.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport functools as ft\nimport itertools\nimport os\nimport re\nimport shutil\nimport stat\nimport sys\nimport tempfile\nfrom typing import Callable, Dict, List, Optional\n\nfrom spack.vendor.typing_extensions import Literal\n\nimport spack.config\nimport spack.directory_layout\nimport spack.projections\nimport spack.relocate\nimport spack.schema.projections\nimport spack.spec\nimport spack.store\nimport spack.util.spack_json as s_json\nimport spack.util.spack_yaml as s_yaml\nfrom spack.error import SpackError\nfrom spack.llnl.string import comma_or\nfrom spack.llnl.util import tty\nfrom spack.llnl.util.filesystem import (\n mkdirp,\n remove_dead_links,\n remove_empty_directories,\n symlink,\n visit_directory_tree,\n)\nfrom spack.llnl.util.lang import index_by, match_predicate\nfrom spack.llnl.util.link_tree import (\n ConflictingSpecsError,\n DestinationMergeVisitor,\n LinkTree,\n MergeConflictSummary,\n SingleMergeConflictError,\n SourceMergeVisitor,\n)\nfrom spack.llnl.util.tty.color import colorize\n\n__all__ = [\"FilesystemView\", \"YamlFilesystemView\"]\n\n\n_projections_path = \".spack/projections.yaml\"\n\n\nLinkCallbackType = Callable[[str, str, \"FilesystemView\", Optional[spack.spec.Spec]], None]\n\n\ndef view_symlink(src: str, dst: str, *args, **kwargs) -> None:\n symlink(src, dst)\n\n\ndef view_hardlink(src: str, dst: str, *args, **kwargs) -> None:\n os.link(src, dst)\n\n\ndef view_copy(\n src: str, dst: str, view: \"FilesystemView\", spec: Optional[spack.spec.Spec] = None\n) -> None:\n \"\"\"\n Copy a file from src to dst.\n\n Use spec and view to generate relocations\n \"\"\"\n shutil.copy2(src, dst, follow_symlinks=False)\n\n # No need to relocate if no metadata or external.\n if not spec or spec.external:\n return\n\n # Order of this dict is somewhat irrelevant\n prefix_to_projection = {\n str(s.prefix): view.get_projection_for_spec(s)\n for s in spec.traverse(root=True, order=\"breadth\")\n if not s.external\n }\n\n src_stat = os.lstat(src)\n\n # TODO: change this into a bulk operation instead of a per-file operation\n\n if stat.S_ISLNK(src_stat.st_mode):\n spack.relocate.relocate_links(links=[dst], prefix_to_prefix=prefix_to_projection)\n elif spack.relocate.is_binary(dst):\n spack.relocate.relocate_text_bin(binaries=[dst], prefix_to_prefix=prefix_to_projection)\n else:\n prefix_to_projection[spack.store.STORE.layout.root] = view._root\n spack.relocate.relocate_text(files=[dst], prefix_to_prefix=prefix_to_projection)\n\n # The os module on Windows does not have a chown function.\n if sys.platform != \"win32\":\n try:\n os.chown(dst, src_stat.st_uid, src_stat.st_gid)\n except OSError:\n tty.debug(f\"Can't change the permissions for {dst}\")\n\n\n#: Type alias for link types\nLinkType = Literal[\"hardlink\", \"hard\", \"copy\", \"relocate\", \"add\", \"symlink\", \"soft\"]\nCanonicalLinkType = Literal[\"hardlink\", \"copy\", \"symlink\"]\n\n\n#: supported string values for `link_type` in an env, mapped to canonical values\n_LINK_TYPES: Dict[LinkType, CanonicalLinkType] = {\n \"hardlink\": \"hardlink\",\n \"hard\": \"hardlink\",\n \"copy\": \"copy\",\n \"relocate\": \"copy\",\n \"add\": \"symlink\",\n \"symlink\": \"symlink\",\n \"soft\": \"symlink\",\n}\n\n_VALID_LINK_TYPES = sorted(set(_LINK_TYPES.values()))\n\n\ndef canonicalize_link_type(link_type: LinkType) -> CanonicalLinkType:\n \"\"\"Return canonical\"\"\"\n canonical = _LINK_TYPES.get(link_type)\n if not canonical:\n raise ValueError(\n f\"Invalid link type: '{link_type}. Must be one of {comma_or(_VALID_LINK_TYPES)}'\"\n )\n return canonical\n\n\ndef function_for_link_type(link_type: LinkType) -> LinkCallbackType:\n link_type = canonicalize_link_type(link_type)\n if link_type == \"hardlink\":\n return view_hardlink\n elif link_type == \"symlink\":\n return view_symlink\n elif link_type == \"copy\":\n return view_copy\n\n assert False, \"invalid link type\"\n\n\nclass FilesystemView:\n \"\"\"\n Governs a filesystem view that is located at certain root-directory.\n\n Packages are linked from their install directories into a common file\n hierarchy.\n\n In distributed filesystems, loading each installed package separately\n can lead to slow-downs due to too many directories being traversed.\n This can be circumvented by loading all needed modules into a common\n directory structure.\n \"\"\"\n\n def __init__(\n self,\n root: str,\n layout: spack.directory_layout.DirectoryLayout,\n *,\n projections: Optional[Dict] = None,\n ignore_conflicts: bool = False,\n verbose: bool = False,\n link_type: LinkType = \"symlink\",\n ):\n \"\"\"\n Initialize a filesystem view under the given ``root`` directory with\n corresponding directory ``layout``.\n\n Files are linked by method ``link`` (spack.llnl.util.filesystem.symlink by default).\n \"\"\"\n self._root = root\n self.layout = layout\n self.projections = {} if projections is None else projections\n\n self.ignore_conflicts = ignore_conflicts\n self.verbose = verbose\n\n # Setup link function to include view\n self.link_type = link_type\n self._link = function_for_link_type(link_type)\n\n def link(self, src: str, dst: str, spec: Optional[spack.spec.Spec] = None) -> None:\n self._link(src, dst, self, spec)\n\n def add_specs(self, *specs: spack.spec.Spec, **kwargs) -> None:\n \"\"\"\n Add given specs to view.\n\n Should accept ``with_dependencies`` as keyword argument (default\n True) to indicate whether or not dependencies should be activated as\n well.\n\n Should except an ``exclude`` keyword argument containing a list of\n regexps that filter out matching spec names.\n\n This method should make use of ``activate_standalone``.\n \"\"\"\n raise NotImplementedError\n\n def add_standalone(self, spec: spack.spec.Spec) -> bool:\n \"\"\"\n Add (link) a standalone package into this view.\n \"\"\"\n raise NotImplementedError\n\n def check_added(self, spec: spack.spec.Spec) -> bool:\n \"\"\"\n Check if the given concrete spec is active in this view.\n \"\"\"\n raise NotImplementedError\n\n def remove_specs(self, *specs: spack.spec.Spec, **kwargs) -> None:\n \"\"\"\n Removes given specs from view.\n\n Should accept ``with_dependencies`` as keyword argument (default\n True) to indicate whether or not dependencies should be deactivated\n as well.\n\n Should accept ``with_dependents`` as keyword argument (default True)\n to indicate whether or not dependents on the deactivated specs\n should be removed as well.\n\n Should except an ``exclude`` keyword argument containing a list of\n regexps that filter out matching spec names.\n\n This method should make use of ``deactivate_standalone``.\n \"\"\"\n raise NotImplementedError\n\n def remove_standalone(self, spec: spack.spec.Spec) -> None:\n \"\"\"\n Remove (unlink) a standalone package from this view.\n \"\"\"\n raise NotImplementedError\n\n def get_projection_for_spec(self, spec: spack.spec.Spec) -> str:\n \"\"\"\n Get the projection in this view for a spec.\n \"\"\"\n raise NotImplementedError\n\n def get_all_specs(self) -> List[spack.spec.Spec]:\n \"\"\"\n Get all specs currently active in this view.\n \"\"\"\n raise NotImplementedError\n\n def get_spec(self, spec: spack.spec.Spec) -> Optional[spack.spec.Spec]:\n \"\"\"\n Return the actual spec linked in this view (i.e. do not look it up\n in the database by name).\n\n ``spec`` can be a name or a spec from which the name is extracted.\n\n As there can only be a single version active for any spec the name\n is enough to identify the spec in the view.\n\n If no spec is present, returns None.\n \"\"\"\n raise NotImplementedError\n\n def print_status(self, *specs: spack.spec.Spec, **kwargs) -> None:\n \"\"\"\n Print a short summary about the given specs, detailing whether..\n\n * ..they are active in the view.\n * ..they are active but the activated version differs.\n * ..they are not active in the view.\n\n Takes ``with_dependencies`` keyword argument so that the status of\n dependencies is printed as well.\n \"\"\"\n raise NotImplementedError\n\n\nclass YamlFilesystemView(FilesystemView):\n \"\"\"\n Filesystem view to work with a yaml based directory layout.\n \"\"\"\n\n def __init__(\n self,\n root: str,\n layout: spack.directory_layout.DirectoryLayout,\n *,\n projections: Optional[Dict] = None,\n ignore_conflicts: bool = False,\n verbose: bool = False,\n link_type: LinkType = \"symlink\",\n ):\n super().__init__(\n root,\n layout,\n projections=projections,\n ignore_conflicts=ignore_conflicts,\n verbose=verbose,\n link_type=link_type,\n )\n\n # Super class gets projections from the kwargs\n # YAML specific to get projections from YAML file\n self.projections_path = os.path.join(self._root, _projections_path)\n if not self.projections:\n # Read projections file from view\n self.projections = self.read_projections()\n elif not os.path.exists(self.projections_path):\n # Write projections file to new view\n self.write_projections()\n else:\n # Ensure projections are the same from each source\n # Read projections file from view\n if self.projections != self.read_projections():\n raise ConflictingProjectionsError(\n f\"View at {self._root} has projections file\"\n \" which does not match projections passed manually.\"\n )\n\n self._croot = colorize_root(self._root) + \" \"\n\n def write_projections(self):\n if self.projections:\n mkdirp(os.path.dirname(self.projections_path))\n with open(self.projections_path, \"w\", encoding=\"utf-8\") as f:\n f.write(s_yaml.dump_config({\"projections\": self.projections}))\n\n def read_projections(self):\n if os.path.exists(self.projections_path):\n with open(self.projections_path, \"r\", encoding=\"utf-8\") as f:\n projections_data = s_yaml.load(f)\n spack.config.validate(projections_data, spack.schema.projections.schema)\n return projections_data[\"projections\"]\n else:\n return {}\n\n def add_specs(self, *specs, **kwargs):\n assert all((s.concrete for s in specs))\n specs = set(specs)\n\n if kwargs.get(\"with_dependencies\", True):\n specs.update(get_dependencies(specs))\n\n if kwargs.get(\"exclude\", None):\n specs = set(filter_exclude(specs, kwargs[\"exclude\"]))\n\n conflicts = self.get_conflicts(*specs)\n\n if conflicts:\n for s, v in conflicts:\n self.print_conflict(v, s)\n return\n\n for s in specs:\n self.add_standalone(s)\n\n def add_standalone(self, spec):\n if spec.external:\n tty.warn(f\"{self._croot}Skipping external package: {colorize_spec(spec)}\")\n return True\n\n if self.check_added(spec):\n tty.warn(f\"{self._croot}Skipping already linked package: {colorize_spec(spec)}\")\n return True\n\n self.merge(spec)\n\n self.link_meta_folder(spec)\n\n if self.verbose:\n tty.info(f\"{self._croot}Linked package: {colorize_spec(spec)}\")\n return True\n\n def merge(self, spec, ignore=None):\n pkg = spec.package\n view_source = pkg.view_source()\n view_dst = pkg.view_destination(self)\n\n tree = LinkTree(view_source)\n\n ignore = ignore or (lambda f: False)\n ignore_file = match_predicate(self.layout.hidden_file_regexes, ignore)\n\n # check for dir conflicts\n conflicts = tree.find_dir_conflicts(view_dst, ignore_file)\n\n merge_map = tree.get_file_map(view_dst, ignore_file)\n if not self.ignore_conflicts:\n conflicts.extend(pkg.view_file_conflicts(self, merge_map))\n\n if conflicts:\n raise SingleMergeConflictError(conflicts[0])\n\n # merge directories with the tree\n tree.merge_directories(view_dst, ignore_file)\n\n pkg.add_files_to_view(self, merge_map)\n\n def unmerge(self, spec, ignore=None):\n pkg = spec.package\n view_source = pkg.view_source()\n view_dst = pkg.view_destination(self)\n\n tree = LinkTree(view_source)\n\n ignore = ignore or (lambda f: False)\n ignore_file = match_predicate(self.layout.hidden_file_regexes, ignore)\n\n merge_map = tree.get_file_map(view_dst, ignore_file)\n pkg.remove_files_from_view(self, merge_map)\n\n # now unmerge the directory tree\n tree.unmerge_directories(view_dst, ignore_file)\n\n def remove_files(self, files):\n def needs_file(spec, file):\n # convert the file we want to remove to a source in this spec\n projection = self.get_projection_for_spec(spec)\n relative_path = os.path.relpath(file, projection)\n test_path = os.path.join(spec.prefix, relative_path)\n\n # check if this spec owns a file of that name (through the\n # manifest in the metadata dir, which we have in the view).\n manifest_file = os.path.join(\n self.get_path_meta_folder(spec), spack.store.STORE.layout.manifest_file_name\n )\n try:\n with open(manifest_file, \"r\", encoding=\"utf-8\") as f:\n manifest = s_json.load(f)\n except OSError:\n # if we can't load it, assume it doesn't know about the file.\n manifest = {}\n return test_path in manifest\n\n specs = self.get_all_specs()\n\n for file in files:\n if not os.path.lexists(file):\n tty.warn(f\"Tried to remove {file} which does not exist\")\n continue\n\n # remove if file is not owned by any other package in the view\n # This will only be false if two packages are merged into a prefix\n # and have a conflicting file\n\n # check all specs for whether they own the file. That include the spec\n # we are currently removing, as we remove files before unlinking the\n # metadata directory.\n if len([s for s in specs if needs_file(s, file)]) <= 1:\n tty.debug(f\"Removing file {file}\")\n os.remove(file)\n\n def check_added(self, spec):\n assert spec.concrete\n return spec == self.get_spec(spec)\n\n def remove_specs(self, *specs, **kwargs):\n assert all((s.concrete for s in specs))\n with_dependents = kwargs.get(\"with_dependents\", True)\n with_dependencies = kwargs.get(\"with_dependencies\", False)\n\n # caller can pass this in, as get_all_specs() is expensive\n all_specs = kwargs.get(\"all_specs\", None) or set(self.get_all_specs())\n\n specs = set(specs)\n\n if with_dependencies:\n specs = get_dependencies(specs)\n\n if kwargs.get(\"exclude\", None):\n specs = set(filter_exclude(specs, kwargs[\"exclude\"]))\n\n to_deactivate = specs\n to_keep = all_specs - to_deactivate\n\n dependents = find_dependents(to_keep, to_deactivate)\n\n if with_dependents:\n # remove all packages depending on the ones to remove\n if len(dependents) > 0:\n tty.warn(\n self._croot\n + \"The following dependents will be removed: %s\"\n % \", \".join((s.name for s in dependents))\n )\n to_deactivate.update(dependents)\n elif len(dependents) > 0:\n tty.warn(\n self._croot\n + \"The following packages will be unusable: %s\"\n % \", \".join((s.name for s in dependents))\n )\n\n # Determine the order that packages should be removed from the view;\n # dependents come before their dependencies.\n to_deactivate_sorted = list()\n depmap = dict()\n for spec in to_deactivate:\n depmap[spec] = set(d for d in spec.traverse(root=False) if d in to_deactivate)\n\n while depmap:\n for spec in [s for s, d in depmap.items() if not d]:\n to_deactivate_sorted.append(spec)\n for s in depmap.keys():\n depmap[s].discard(spec)\n depmap.pop(spec)\n to_deactivate_sorted.reverse()\n\n # Ensure that the sorted list contains all the packages\n assert set(to_deactivate_sorted) == to_deactivate\n\n # Remove the packages from the view\n for spec in to_deactivate_sorted:\n self.remove_standalone(spec)\n\n self._purge_empty_directories()\n\n def remove_standalone(self, spec):\n \"\"\"\n Remove (unlink) a standalone package from this view.\n \"\"\"\n if not self.check_added(spec):\n tty.warn(f\"{self._croot}Skipping package not linked in view: {spec.name}\")\n return\n\n self.unmerge(spec)\n self.unlink_meta_folder(spec)\n\n if self.verbose:\n tty.info(f\"{self._croot}Removed package: {colorize_spec(spec)}\")\n\n def get_projection_for_spec(self, spec):\n \"\"\"\n Return the projection for a spec in this view.\n\n Relies on the ordering of projections to avoid ambiguity.\n \"\"\"\n spec = spack.spec.Spec(spec)\n locator_spec = spec\n\n if spec.package.extendee_spec:\n locator_spec = spec.package.extendee_spec\n\n proj = spack.projections.get_projection(self.projections, locator_spec)\n if proj:\n return os.path.join(self._root, locator_spec.format_path(proj))\n return self._root\n\n def get_all_specs(self):\n md_dirs = []\n for root, dirs, files in os.walk(self._root):\n if spack.store.STORE.layout.metadata_dir in dirs:\n md_dirs.append(os.path.join(root, spack.store.STORE.layout.metadata_dir))\n\n specs = []\n for md_dir in md_dirs:\n if os.path.exists(md_dir):\n for name_dir in os.listdir(md_dir):\n filename = os.path.join(\n md_dir, name_dir, spack.store.STORE.layout.spec_file_name\n )\n spec = get_spec_from_file(filename)\n if spec:\n specs.append(spec)\n return specs\n\n def get_conflicts(self, *specs):\n \"\"\"\n Return list of tuples (, ) where the spec\n active in the view differs from the one to be activated.\n \"\"\"\n in_view = map(self.get_spec, specs)\n return [(s, v) for s, v in zip(specs, in_view) if v is not None and s != v]\n\n def get_path_meta_folder(self, spec):\n \"Get path to meta folder for either spec or spec name.\"\n return os.path.join(\n self.get_projection_for_spec(spec),\n spack.store.STORE.layout.metadata_dir,\n getattr(spec, \"name\", spec),\n )\n\n def get_spec(self, spec):\n dotspack = self.get_path_meta_folder(spec)\n filename = os.path.join(dotspack, spack.store.STORE.layout.spec_file_name)\n\n return get_spec_from_file(filename)\n\n def link_meta_folder(self, spec):\n src = spack.store.STORE.layout.metadata_path(spec)\n tgt = self.get_path_meta_folder(spec)\n\n tree = LinkTree(src)\n # there should be no conflicts when linking the meta folder\n tree.merge(tgt, link=self.link)\n\n def print_conflict(self, spec_active, spec_specified, level=\"error\"):\n \"Singular print function for spec conflicts.\"\n cprint = getattr(tty, level)\n color = sys.stdout.isatty()\n linked = tty.color.colorize(\" (@gLinked@.)\", color=color)\n specified = tty.color.colorize(\"(@rSpecified@.)\", color=color)\n cprint(\n f\"{self._croot}Package conflict detected:\\n\"\n f\"{linked} {colorize_spec(spec_active)}\\n\"\n f\"{specified} {colorize_spec(spec_specified)}\"\n )\n\n def print_status(self, *specs, **kwargs):\n if kwargs.get(\"with_dependencies\", False):\n specs = set(get_dependencies(specs))\n\n specs = sorted(specs, key=lambda s: s.name)\n in_view = list(map(self.get_spec, specs))\n\n for s, v in zip(specs, in_view):\n if not v:\n tty.error(f\"{self._croot}Package not linked: {s.name}\")\n elif s != v:\n self.print_conflict(v, s, level=\"warn\")\n\n in_view = list(filter(None, in_view))\n\n if len(specs) > 0:\n tty.msg(f\"Packages linked in {self._croot[:-1]}:\")\n\n # Make a dict with specs keyed by architecture and compiler.\n index = index_by(specs, (\"architecture\", \"compiler\"))\n\n # Traverse the index and print out each package\n for i, (architecture, compiler) in enumerate(sorted(index)):\n if i > 0:\n print()\n\n header = (\n f\"{spack.spec.ARCHITECTURE_COLOR}{{{architecture}}} \"\n f\"/ {spack.spec.COMPILER_COLOR}{{{compiler}}}\"\n )\n tty.hline(colorize(header), char=\"-\")\n\n specs = index[(architecture, compiler)]\n specs.sort()\n\n abbreviated = [\n s.cformat(\"{name}{@version}{compiler_flags}{variants}{%compiler}\")\n for s in specs\n ]\n\n # Print one spec per line along with prefix path\n width = max(len(s) for s in abbreviated)\n width += 2\n format = \" %%-%ds%%s\" % width\n\n for abbrv, s in zip(abbreviated, specs):\n prefix = \"\"\n if self.verbose:\n prefix = colorize(\"@K{%s}\" % s.dag_hash(7))\n print(prefix + (format % (abbrv, self.get_projection_for_spec(s))))\n else:\n tty.warn(self._croot + \"No packages found.\")\n\n def _purge_empty_directories(self):\n remove_empty_directories(self._root)\n\n def _purge_broken_links(self):\n remove_dead_links(self._root)\n\n def clean(self):\n self._purge_broken_links()\n self._purge_empty_directories()\n\n def unlink_meta_folder(self, spec):\n path = self.get_path_meta_folder(spec)\n assert os.path.exists(path)\n shutil.rmtree(path)\n\n\nclass SimpleFilesystemView(FilesystemView):\n \"\"\"A simple and partial implementation of FilesystemView focused on performance and immutable\n views, where specs cannot be removed after they were added.\"\"\"\n\n def _sanity_check_view_projection(self, specs):\n \"\"\"A very common issue is that we end up with two specs of the same package, that project\n to the same prefix. We want to catch that as early as possible and give a sensible error to\n the user. Here we use the metadata dir (.spack) projection as a quick test to see whether\n two specs in the view are going to clash. The metadata dir is used because it's always\n added by Spack with identical files, so a guaranteed clash that's easily verified.\"\"\"\n seen = {}\n for current_spec in specs:\n metadata_dir = self.relative_metadata_dir_for_spec(current_spec)\n conflicting_spec = seen.get(metadata_dir)\n if conflicting_spec:\n raise ConflictingSpecsError(current_spec, conflicting_spec)\n seen[metadata_dir] = current_spec\n\n def add_specs(self, *specs, **kwargs) -> None:\n \"\"\"Link a root-to-leaf topologically ordered list of specs into the view.\"\"\"\n assert all((s.concrete for s in specs))\n if len(specs) == 0:\n return\n\n # Drop externals\n specs = [s for s in specs if not s.external]\n\n self._sanity_check_view_projection(specs)\n\n # Ignore spack meta data folder.\n def skip_list(file):\n return os.path.basename(file) == spack.store.STORE.layout.metadata_dir\n\n # Determine if the root is on a case-insensitive filesystem\n normalize_paths = is_folder_on_case_insensitive_filesystem(self._root)\n\n visitor = SourceMergeVisitor(ignore=skip_list, normalize_paths=normalize_paths)\n\n # Gather all the directories to be made and files to be linked\n for spec in specs:\n src_prefix = spec.package.view_source()\n visitor.set_projection(self.get_relative_projection_for_spec(spec))\n visit_directory_tree(src_prefix, visitor)\n\n # Check for conflicts in destination dir.\n visit_directory_tree(self._root, DestinationMergeVisitor(visitor))\n\n # Throw on fatal dir-file conflicts.\n if visitor.fatal_conflicts:\n raise MergeConflictSummary(visitor.fatal_conflicts)\n\n # Inform about file-file conflicts.\n if visitor.file_conflicts:\n if self.ignore_conflicts:\n tty.debug(f\"{len(visitor.file_conflicts)} file conflicts\")\n else:\n raise MergeConflictSummary(visitor.file_conflicts)\n\n tty.debug(f\"Creating {len(visitor.directories)} dirs and {len(visitor.files)} links\")\n\n # Make the directory structure\n for dst in visitor.directories:\n os.mkdir(os.path.join(self._root, dst))\n\n # Link the files using a \"merge map\": full src => full dst\n merge_map_per_prefix = self._source_merge_visitor_to_merge_map(visitor)\n for spec in specs:\n merge_map = merge_map_per_prefix.get(spec.package.view_source(), None)\n if not merge_map:\n # Not every spec may have files to contribute.\n continue\n spec.package.add_files_to_view(self, merge_map, skip_if_exists=False)\n\n # Finally create the metadata dirs.\n self.link_metadata(specs)\n\n def _source_merge_visitor_to_merge_map(self, visitor: SourceMergeVisitor):\n # For compatibility with add_files_to_view, we have to create a\n # merge_map of the form join(src_root, src_rel) => join(dst_root, dst_rel),\n # but our visitor.files format is dst_rel => (src_root, src_rel).\n # We exploit that visitor.files is an ordered dict, and files per source\n # prefix are contiguous.\n source_root = lambda item: item[1][0]\n per_source = itertools.groupby(visitor.files.items(), key=source_root)\n return {\n src_root: {\n os.path.join(src_root, src_rel): os.path.join(self._root, dst_rel)\n for dst_rel, (_, src_rel) in group\n }\n for src_root, group in per_source\n }\n\n def relative_metadata_dir_for_spec(self, spec):\n return os.path.join(\n self.get_relative_projection_for_spec(spec),\n spack.store.STORE.layout.metadata_dir,\n spec.name,\n )\n\n def link_metadata(self, specs):\n metadata_visitor = SourceMergeVisitor()\n\n for spec in specs:\n src_prefix = os.path.join(\n spec.package.view_source(), spack.store.STORE.layout.metadata_dir\n )\n proj = self.relative_metadata_dir_for_spec(spec)\n metadata_visitor.set_projection(proj)\n visit_directory_tree(src_prefix, metadata_visitor)\n\n # Check for conflicts in destination dir.\n visit_directory_tree(self._root, DestinationMergeVisitor(metadata_visitor))\n\n # Throw on dir-file conflicts -- unlikely, but who knows.\n if metadata_visitor.fatal_conflicts:\n raise MergeConflictSummary(metadata_visitor.fatal_conflicts)\n\n # We are strict here for historical reasons\n if metadata_visitor.file_conflicts:\n raise MergeConflictSummary(metadata_visitor.file_conflicts)\n\n for dst in metadata_visitor.directories:\n os.mkdir(os.path.join(self._root, dst))\n\n for dst_relpath, (src_root, src_relpath) in metadata_visitor.files.items():\n self.link(os.path.join(src_root, src_relpath), os.path.join(self._root, dst_relpath))\n\n def get_relative_projection_for_spec(self, spec):\n # Extensions are placed by their extendee, not by their own spec\n if spec.package.extendee_spec:\n spec = spec.package.extendee_spec\n\n p = spack.projections.get_projection(self.projections, spec)\n return spec.format_path(p) if p else \"\"\n\n def get_projection_for_spec(self, spec):\n \"\"\"\n Return the projection for a spec in this view.\n\n Relies on the ordering of projections to avoid ambiguity.\n \"\"\"\n spec = spack.spec.Spec(spec)\n\n if spec.package.extendee_spec:\n spec = spec.package.extendee_spec\n\n proj = spack.projections.get_projection(self.projections, spec)\n if proj:\n return os.path.join(self._root, spec.format_path(proj))\n return self._root\n\n\n#####################\n# utility functions #\n#####################\ndef get_spec_from_file(filename) -> Optional[spack.spec.Spec]:\n try:\n with open(filename, \"r\", encoding=\"utf-8\") as f:\n return spack.spec.Spec.from_yaml(f)\n except OSError:\n return None\n\n\ndef colorize_root(root):\n colorize = ft.partial(tty.color.colorize, color=sys.stdout.isatty())\n pre, post = map(colorize, \"@M[@. @M]@.\".split())\n return \"\".join([pre, root, post])\n\n\ndef colorize_spec(spec):\n \"Colorize spec output if in TTY.\"\n if sys.stdout.isatty():\n return spec.cshort_spec\n else:\n return spec.short_spec\n\n\ndef find_dependents(all_specs, providers, deptype=\"run\"):\n \"\"\"\n Return a set containing all those specs from all_specs that depend on\n providers at the given dependency type.\n \"\"\"\n dependents = set()\n for s in all_specs:\n for dep in s.traverse(deptype=deptype):\n if dep in providers:\n dependents.add(s)\n return dependents\n\n\ndef filter_exclude(specs, exclude):\n \"Filter specs given sequence of exclude regex\"\n to_exclude = [re.compile(e) for e in exclude]\n\n def keep(spec):\n for e in to_exclude:\n if e.match(spec.name):\n return False\n return True\n\n return filter(keep, specs)\n\n\ndef get_dependencies(specs):\n \"Get set of dependencies (includes specs)\"\n retval = set()\n set(map(retval.update, (set(s.traverse()) for s in specs)))\n return retval\n\n\nclass ConflictingProjectionsError(SpackError):\n \"\"\"Raised when a view has a projections file and is given one manually.\"\"\"\n\n\ndef is_folder_on_case_insensitive_filesystem(path: str) -> bool:\n with tempfile.NamedTemporaryFile(dir=path, prefix=\".sentinel\") as sentinel:\n return os.path.exists(os.path.join(path, os.path.basename(sentinel.name).upper()))\n" }, { "path": "lib/spack/spack/graph.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nr\"\"\"Functions for graphing DAGs of dependencies.\n\nThis file contains code for graphing DAGs of software packages\n(i.e. Spack specs). There are two main functions you probably care\nabout:\n\n:func:`graph_ascii` will output a colored graph of a spec in ascii format,\nkind of like the graph git shows with ``git log --graph``, e.g.\n\n.. code-block:: text\n\n o mpileaks\n |\\\n | |\\\n | o | callpath\n |/| |\n | |\\|\n | |\\ \\\n | | |\\ \\\n | | | | o adept-utils\n | |_|_|/|\n |/| | | |\n o | | | | mpi\n / / / /\n | | o | dyninst\n | |/| |\n |/|/| |\n | | |/\n | o | libdwarf\n |/ /\n o | libelf\n /\n o boost\n\n:func:`graph_dot` will output a graph of a spec (or multiple specs) in dot format.\n\"\"\"\n\nimport enum\nimport sys\nfrom typing import List, Optional, Set, TextIO, Tuple\n\nimport spack.deptypes as dt\nimport spack.llnl.util.tty.color\nimport spack.spec\nimport spack.tengine\nimport spack.traverse\nfrom spack.solver.input_analysis import create_graph_analyzer\n\n\ndef find(seq, predicate):\n \"\"\"Find index in seq for which predicate is True.\n\n Searches the sequence and returns the index of the element for\n which the predicate evaluates to True. Returns -1 if the\n predicate does not evaluate to True for any element in seq.\n\n \"\"\"\n for i, elt in enumerate(seq):\n if predicate(elt):\n return i\n return -1\n\n\nclass _GraphLineState(enum.Enum):\n \"\"\"Names of different graph line states.\"\"\"\n\n NODE = enum.auto()\n COLLAPSE = enum.auto()\n MERGE_RIGHT = enum.auto()\n EXPAND_RIGHT = enum.auto()\n BACK_EDGE = enum.auto()\n\n\nclass AsciiGraph:\n def __init__(self):\n # These can be set after initialization or after a call to\n # graph() to change behavior.\n self.node_character = \"o\"\n self.debug = False\n self.indent = 0\n self.depflag = dt.ALL\n\n # These are colors in the order they'll be used for edges.\n # See spack.llnl.util.tty.color for details on color characters.\n self.colors = \"rgbmcyRGBMCY\"\n\n # Internal vars are used in the graph() function and are initialized there\n self._name_to_color = None # Node name to color\n self._out = None # Output stream\n self._frontier = None # frontier\n self._prev_state = None # State of previous line\n self._prev_index = None # Index of expansion point of prev line\n self._pos = None\n\n def _indent(self):\n self._out.write(self.indent * \" \")\n\n def _write_edge(self, string, index, sub=0):\n \"\"\"Write a colored edge to the output stream.\"\"\"\n # Ignore empty frontier entries (they're just collapsed)\n if not self._frontier[index]:\n return\n name = self._frontier[index][sub]\n edge = f\"@{self._name_to_color[name]}{{{string}}}\"\n self._out.write(edge)\n\n def _connect_deps(self, i, deps, label=None):\n \"\"\"Connect dependencies to existing edges in the frontier.\n\n ``deps`` are to be inserted at position i in the\n frontier. This routine determines whether other open edges\n should be merged with (if there are other open edges\n pointing to the same place) or whether they should just be\n inserted as a completely new open edge.\n\n Open edges that are not fully expanded (i.e. those that point\n at multiple places) are left intact.\n\n Parameters:\n\n label -- optional debug label for the connection.\n\n Returns: True if the deps were connected to another edge\n (i.e. the frontier did not grow) and False if the deps were\n NOT already in the frontier (i.e. they were inserted and the\n frontier grew).\n\n \"\"\"\n if len(deps) == 1 and deps in self._frontier:\n j = self._frontier.index(deps)\n\n # convert a right connection into a left connection\n if i < j:\n self._frontier.pop(j)\n self._frontier.insert(i, deps)\n return self._connect_deps(j, deps, label)\n\n collapse = True\n if self._prev_state == _GraphLineState.EXPAND_RIGHT:\n # Special case where previous line expanded and i is off by 1.\n self._back_edge_line([], j, i + 1, True, label + \"-1.5 \" + str((i + 1, j)))\n collapse = False\n\n else:\n # Previous node also expanded here, so i is off by one.\n if self._prev_state == _GraphLineState.NODE and self._prev_index < i:\n i += 1\n\n if i - j > 1:\n # We need two lines to connect if distance > 1\n self._back_edge_line([], j, i, True, label + \"-1 \" + str((i, j)))\n collapse = False\n\n self._back_edge_line([j], -1, -1, collapse, label + \"-2 \" + str((i, j)))\n return True\n\n if deps:\n self._frontier.insert(i, deps)\n return False\n\n return False\n\n def _set_state(self, state, index, label=None):\n self._prev_state = state\n self._prev_index = index\n\n if self.debug:\n self._out.write(\" \" * 20)\n self._out.write(f\"{str(self._prev_state) if self._prev_state else '':<20}\")\n self._out.write(f\"{str(label) if label else '':<20}\")\n self._out.write(f\"{self._frontier}\")\n\n def _back_edge_line(self, prev_ends, end, start, collapse, label=None):\n \"\"\"Write part of a backwards edge in the graph.\n\n Writes single- or multi-line backward edges in an ascii graph.\n For example, a single line edge::\n\n | | | | o |\n | | | |/ / <-- single-line edge connects two nodes.\n | | | o |\n\n Or a multi-line edge (requires two calls to back_edge)::\n\n | | | | o |\n | |_|_|/ / <-- multi-line edge crosses vertical edges.\n |/| | | |\n o | | | |\n\n Also handles \"pipelined\" edges, where the same line contains\n parts of multiple edges::\n\n o start\n | |_|_|_|/|\n |/| | |_|/| <-- this line has parts of 2 edges.\n | | |/| | |\n o o\n\n Arguments:\n\n prev_ends -- indices in frontier of previous edges that need\n to be finished on this line.\n\n end -- end of the current edge on this line.\n\n start -- start index of the current edge.\n\n collapse -- whether the graph will be collapsing (i.e. whether\n to slant the end of the line or keep it straight)\n\n label -- optional debug label to print after the line.\n\n \"\"\"\n\n def advance(to_pos, edges):\n \"\"\"Write edges up to .\"\"\"\n for i in range(self._pos, to_pos):\n for e in edges():\n self._write_edge(*e)\n self._pos += 1\n\n flen = len(self._frontier)\n self._pos = 0\n self._indent()\n\n for p in prev_ends:\n advance(p, lambda: [(\"| \", self._pos)])\n advance(p + 1, lambda: [(\"|/\", self._pos)])\n\n if end >= 0:\n advance(end + 1, lambda: [(\"| \", self._pos)])\n advance(start - 1, lambda: [(\"|\", self._pos), (\"_\", end)])\n else:\n advance(start - 1, lambda: [(\"| \", self._pos)])\n\n if start >= 0:\n advance(start, lambda: [(\"|\", self._pos), (\"/\", end)])\n\n if collapse:\n advance(flen, lambda: [(\" /\", self._pos)])\n else:\n advance(flen, lambda: [(\"| \", self._pos)])\n\n self._set_state(_GraphLineState.BACK_EDGE, end, label)\n self._out.write(\"\\n\")\n\n def _node_label(self, node):\n return node.format(\"{name}@@{version}{/hash:7}\")\n\n def _node_line(self, index, node):\n \"\"\"Writes a line with a node at index.\"\"\"\n self._indent()\n for c in range(index):\n self._write_edge(\"| \", c)\n\n self._out.write(f\"{self.node_character} \")\n\n for c in range(index + 1, len(self._frontier)):\n self._write_edge(\"| \", c)\n\n self._out.write(self._node_label(node))\n self._set_state(_GraphLineState.NODE, index)\n self._out.write(\"\\n\")\n\n def _collapse_line(self, index):\n \"\"\"Write a collapsing line after a node was added at index.\"\"\"\n self._indent()\n for c in range(index):\n self._write_edge(\"| \", c)\n for c in range(index, len(self._frontier)):\n self._write_edge(\" /\", c)\n\n self._set_state(_GraphLineState.COLLAPSE, index)\n self._out.write(\"\\n\")\n\n def _merge_right_line(self, index):\n \"\"\"Edge at index is same as edge to right. Merge directly with '\\'\"\"\"\n self._indent()\n for c in range(index):\n self._write_edge(\"| \", c)\n self._write_edge(\"|\", index)\n self._write_edge(\"\\\\\", index + 1)\n for c in range(index + 1, len(self._frontier)):\n self._write_edge(\"| \", c)\n\n self._set_state(_GraphLineState.MERGE_RIGHT, index)\n self._out.write(\"\\n\")\n\n def _expand_right_line(self, index):\n self._indent()\n for c in range(index):\n self._write_edge(\"| \", c)\n\n self._write_edge(\"|\", index)\n self._write_edge(\"\\\\\", index + 1)\n\n for c in range(index + 2, len(self._frontier)):\n self._write_edge(\" \\\\\", c)\n\n self._set_state(_GraphLineState.EXPAND_RIGHT, index)\n self._out.write(\"\\n\")\n\n def write(self, spec, color=None, out=None):\n \"\"\"Write out an ascii graph of the provided spec.\n\n Arguments:\n spec: spec to graph. This only handles one spec at a time.\n out: file object to write out to (default is sys.stdout)\n color: whether to write in color. Default is to autodetect\n based on output file.\n\n \"\"\"\n if out is None:\n out = sys.stdout\n\n if color is None:\n color = out.isatty()\n\n self._out = spack.llnl.util.tty.color.ColorStream(out, color=color)\n\n # We'll traverse the spec in topological order as we graph it.\n nodes_in_topological_order = list(spec.traverse(order=\"topo\", deptype=self.depflag))\n nodes_in_topological_order.reverse()\n\n # Work on a copy to be nondestructive\n spec = spec.copy()\n\n # Colors associated with each node in the DAG.\n # Edges are colored by the node they point to.\n self._name_to_color = {\n spec.dag_hash(): self.colors[i % len(self.colors)]\n for i, spec in enumerate(nodes_in_topological_order)\n }\n\n # Frontier tracks open edges of the graph as it's written out.\n self._frontier = [[spec.dag_hash()]]\n while self._frontier:\n # Find an unexpanded part of frontier\n i = find(self._frontier, lambda f: len(f) > 1)\n\n if i >= 0:\n # Expand frontier until there are enough columns for all children.\n\n # Figure out how many back connections there are and\n # sort them so we do them in order\n back = []\n for d in self._frontier[i]:\n b = find(self._frontier[:i], lambda f: f == [d])\n if b != -1:\n back.append((b, d))\n\n # Do all back connections in sorted order so we can\n # pipeline them and save space.\n if back:\n back.sort()\n prev_ends = []\n collapse_l1 = False\n for j, (b, d) in enumerate(back):\n self._frontier[i].remove(d)\n if i - b > 1:\n collapse_l1 = any(not e for e in self._frontier)\n self._back_edge_line(prev_ends, b, i, collapse_l1, \"left-1\")\n del prev_ends[:]\n prev_ends.append(b)\n\n # Check whether we did ALL the deps as back edges,\n # in which case we're done.\n pop = not self._frontier[i]\n collapse_l2 = pop\n if collapse_l1:\n collapse_l2 = False\n if pop:\n self._frontier.pop(i)\n self._back_edge_line(prev_ends, -1, -1, collapse_l2, \"left-2\")\n\n elif len(self._frontier[i]) > 1:\n # Expand forward after doing all back connections\n\n if (\n i + 1 < len(self._frontier)\n and len(self._frontier[i + 1]) == 1\n and self._frontier[i + 1][0] in self._frontier[i]\n ):\n # We need to connect to the element to the right.\n # Keep lines straight by connecting directly and\n # avoiding unnecessary expand/contract.\n name = self._frontier[i + 1][0]\n self._frontier[i].remove(name)\n self._merge_right_line(i)\n\n else:\n # Just allow the expansion here.\n dep_hash = self._frontier[i].pop(0)\n deps = [dep_hash]\n self._frontier.insert(i, deps)\n self._expand_right_line(i)\n\n self._frontier.pop(i)\n self._connect_deps(i, deps, \"post-expand\")\n\n # Handle any remaining back edges to the right\n j = i + 1\n while j < len(self._frontier):\n deps = self._frontier.pop(j)\n if not self._connect_deps(j, deps, \"back-from-right\"):\n j += 1\n\n else:\n # Nothing to expand; add dependencies for a node.\n node = nodes_in_topological_order.pop()\n\n # Find the named node in the frontier and draw it.\n i = find(self._frontier, lambda f: node.dag_hash() in f)\n self._node_line(i, node)\n\n # Replace node with its dependencies\n self._frontier.pop(i)\n edges = sorted(node.edges_to_dependencies(depflag=self.depflag), reverse=True)\n if edges:\n deps = [e.spec.dag_hash() for e in edges]\n self._connect_deps(i, deps, \"new-deps\") # anywhere.\n\n elif self._frontier:\n self._collapse_line(i)\n\n\ndef graph_ascii(\n spec, node=\"o\", out=None, debug=False, indent=0, color=None, depflag: dt.DepFlag = dt.ALL\n):\n graph = AsciiGraph()\n graph.debug = debug\n graph.indent = indent\n graph.node_character = node\n graph.depflag = depflag\n\n graph.write(spec, color=color, out=out)\n\n\nclass DotGraphBuilder:\n \"\"\"Visit edges of a graph a build DOT options for nodes and edges\"\"\"\n\n def __init__(self):\n self.nodes: Set[Tuple[str, str]] = set()\n self.edges: Set[Tuple[str, str, str]] = set()\n\n def visit(self, edge: spack.spec.DependencySpec):\n \"\"\"Visit an edge and builds up entries to render the graph\"\"\"\n if edge.parent is None:\n self.nodes.add(self.node_entry(edge.spec))\n return\n\n self.nodes.add(self.node_entry(edge.parent))\n self.nodes.add(self.node_entry(edge.spec))\n self.edges.add(self.edge_entry(edge))\n\n def node_entry(self, node: spack.spec.Spec) -> Tuple[str, str]:\n \"\"\"Return a tuple of (node_id, node_options)\"\"\"\n raise NotImplementedError(\"Need to be implemented by derived classes\")\n\n def edge_entry(self, edge: spack.spec.DependencySpec) -> Tuple[str, str, str]:\n \"\"\"Return a tuple of (parent_id, child_id, edge_options)\"\"\"\n raise NotImplementedError(\"Need to be implemented by derived classes\")\n\n def context(self):\n \"\"\"Return the context to be used to render the DOT graph template\"\"\"\n result = {\"nodes\": self.nodes, \"edges\": self.edges}\n return result\n\n def render(self) -> str:\n \"\"\"Return a string with the output in DOT format\"\"\"\n environment = spack.tengine.make_environment()\n template = environment.get_template(\"misc/graph.dot\")\n return template.render(self.context())\n\n\nclass SimpleDAG(DotGraphBuilder):\n \"\"\"Simple DOT graph, with nodes colored uniformly and edges without properties\"\"\"\n\n def node_entry(self, node):\n format_option = \"{name}{@version}{/hash:7}{%compiler}\"\n return node.dag_hash(), f'[label=\"{node.format(format_option)}\"]'\n\n def edge_entry(self, edge):\n return edge.parent.dag_hash(), edge.spec.dag_hash(), None\n\n\nclass StaticDag(DotGraphBuilder):\n \"\"\"DOT graph for possible dependencies\"\"\"\n\n def node_entry(self, node):\n return node.name, f'[label=\"{node.name}\"]'\n\n def edge_entry(self, edge):\n return edge.parent.name, edge.spec.name, None\n\n\nclass DAGWithDependencyTypes(DotGraphBuilder):\n \"\"\"DOT graph with link,run nodes grouped together and edges colored according to\n the dependency types.\n \"\"\"\n\n def __init__(self):\n super().__init__()\n self.main_unified_space: Set[str] = set()\n\n def visit(self, edge):\n if edge.parent is None:\n for node in spack.traverse.traverse_nodes([edge.spec], deptype=dt.LINK | dt.RUN):\n self.main_unified_space.add(node.dag_hash())\n super().visit(edge)\n\n def node_entry(self, node):\n node_str = node.format(\"{name}{@version}{/hash:7}{%compiler}\")\n options = f'[label=\"{node_str}\", group=\"build_dependencies\", fillcolor=\"coral\"]'\n if node.dag_hash() in self.main_unified_space:\n options = f'[label=\"{node_str}\", group=\"main_psid\"]'\n return node.dag_hash(), options\n\n def edge_entry(self, edge):\n colormap = {\"build\": \"dodgerblue\", \"link\": \"crimson\", \"run\": \"goldenrod\"}\n label = \"\"\n if edge.virtuals:\n label = f' xlabel=\"virtuals={\",\".join(edge.virtuals)}\"'\n return (\n edge.parent.dag_hash(),\n edge.spec.dag_hash(),\n f'[color=\"{\":\".join(colormap[x] for x in dt.flag_to_tuple(edge.depflag))}\"'\n + label\n + \"]\",\n )\n\n\ndef _static_edges(specs, depflag):\n for spec in specs:\n *_, edges = create_graph_analyzer().possible_dependencies(\n spec.name, expand_virtuals=True, allowed_deps=depflag\n )\n\n for parent_name, dependencies in edges.items():\n for dependency_name in dependencies:\n yield spack.spec.DependencySpec(\n spack.spec.Spec(parent_name),\n spack.spec.Spec(dependency_name),\n depflag=depflag,\n virtuals=(),\n )\n\n\ndef static_graph_dot(\n specs: List[spack.spec.Spec], depflag: dt.DepFlag = dt.ALL, out: Optional[TextIO] = None\n):\n \"\"\"Static DOT graph with edges to all possible dependencies.\n\n Args:\n specs: abstract specs to be represented\n depflag: dependency types to consider\n out: optional output stream. If None sys.stdout is used\n \"\"\"\n out = out or sys.stdout\n builder = StaticDag()\n for edge in _static_edges(specs, depflag):\n builder.visit(edge)\n out.write(builder.render())\n\n\ndef graph_dot(\n specs: List[spack.spec.Spec],\n builder: Optional[DotGraphBuilder] = None,\n depflag: dt.DepFlag = dt.ALL,\n out: Optional[TextIO] = None,\n):\n \"\"\"DOT graph of the concrete specs passed as input.\n\n Args:\n specs: specs to be represented\n builder: builder to use to render the graph\n depflag: dependency types to consider\n out: optional output stream. If None sys.stdout is used\n \"\"\"\n if not specs:\n raise ValueError(\"Must provide specs to graph_dot\")\n\n if out is None:\n out = sys.stdout\n\n builder = builder or SimpleDAG()\n for edge in spack.traverse.traverse_edges(\n specs, cover=\"edges\", order=\"breadth\", deptype=depflag\n ):\n builder.visit(edge)\n\n out.write(builder.render())\n" }, { "path": "lib/spack/spack/hash_types.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Definitions that control how Spack creates Spec hashes.\"\"\"\n\nfrom typing import TYPE_CHECKING, Callable, List, Optional\n\nimport spack.deptypes as dt\nimport spack.repo\n\nif TYPE_CHECKING:\n import spack.spec\n\n\nclass SpecHashDescriptor:\n \"\"\"This class defines how hashes are generated on Spec objects.\n\n Spec hashes in Spack are generated from a serialized (e.g., with\n YAML) representation of the Spec graph. The representation may only\n include certain dependency types, and it may optionally include a\n canonicalized hash of the package.py for each node in the graph.\n\n We currently use different hashes for different use cases.\"\"\"\n\n __slots__ = \"depflag\", \"package_hash\", \"name\", \"attr\", \"override\"\n\n def __init__(\n self,\n depflag: dt.DepFlag,\n package_hash: bool,\n name: str,\n override: Optional[Callable[[\"spack.spec.Spec\"], str]] = None,\n ) -> None:\n self.depflag = depflag\n self.package_hash = package_hash\n self.name = name\n self.attr = f\"_{name}\"\n # Allow spec hashes to have an alternate computation method\n self.override = override\n\n def __call__(self, spec: \"spack.spec.Spec\") -> str:\n \"\"\"Run this hash on the provided spec.\"\"\"\n return spec.spec_hash(self)\n\n def __repr__(self) -> str:\n return (\n f\"SpecHashDescriptor(depflag={self.depflag!r}, \"\n f\"package_hash={self.package_hash!r}, name={self.name!r}, override={self.override!r})\"\n )\n\n\n#: The DAG hash includes all inputs that can affect how a package is built.\ndag_hash = SpecHashDescriptor(\n depflag=dt.BUILD | dt.LINK | dt.RUN | dt.TEST, package_hash=True, name=\"hash\"\n)\n\n\ndef _content_hash_override(spec: \"spack.spec.Spec\") -> str:\n pkg_cls = spack.repo.PATH.get_pkg_class(spec.name)\n pkg = pkg_cls(spec)\n return pkg.content_hash()\n\n\n#: Package hash used as part of dag hash\npackage_hash = SpecHashDescriptor(\n depflag=0, package_hash=True, name=\"package_hash\", override=_content_hash_override\n)\n\n\n# Deprecated hash types, no longer used, but needed to understand old serialized\n# spec formats\n\nfull_hash = SpecHashDescriptor(\n depflag=dt.BUILD | dt.LINK | dt.RUN, package_hash=True, name=\"full_hash\"\n)\n\n\nbuild_hash = SpecHashDescriptor(\n depflag=dt.BUILD | dt.LINK | dt.RUN, package_hash=False, name=\"build_hash\"\n)\n\nHASHES: List[\"SpecHashDescriptor\"] = [dag_hash, package_hash, full_hash, build_hash]\n" }, { "path": "lib/spack/spack/hooks/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"This package contains modules with hooks for various stages in the\nSpack install process. You can add modules here and they'll be\nexecuted by package at various times during the package lifecycle.\n\nEach hook is just a function that takes a package as a parameter.\nHooks are not executed in any particular order.\n\nCurrently the following hooks are supported:\n\n* ``pre_install(spec)``\n* ``post_install(spec, explicit)``\n* ``pre_uninstall(spec)``\n* ``post_uninstall(spec)``\n\nThis can be used to implement support for things like module\nsystems (e.g. modules, lmod, etc.) or to add other custom\nfeatures.\n\"\"\"\n\nimport importlib\nimport types\nfrom typing import List, Optional\n\n\nclass _HookRunner:\n #: Order in which hooks are executed\n HOOK_ORDER = [\n \"spack.hooks.module_file_generation\",\n \"spack.hooks.licensing\",\n \"spack.hooks.sbang\",\n \"spack.hooks.windows_runtime_linkage\",\n \"spack.hooks.drop_redundant_rpaths\",\n \"spack.hooks.absolutify_elf_sonames\",\n \"spack.hooks.permissions_setters\",\n \"spack.hooks.resolve_shared_libraries\",\n # after all mutations to the install prefix, write metadata\n \"spack.hooks.write_install_manifest\",\n # after all metadata is written\n \"spack.hooks.autopush\",\n ]\n\n #: Contains all hook modules after first call, shared among all HookRunner objects\n _hooks: Optional[List[types.ModuleType]] = None\n\n def __init__(self, hook_name):\n self.hook_name = hook_name\n\n @property\n def hooks(self) -> List[types.ModuleType]:\n if not self._hooks:\n self._hooks = [importlib.import_module(module_name) for module_name in self.HOOK_ORDER]\n return self._hooks\n\n def __call__(self, *args, **kwargs):\n for module in self.hooks:\n if hasattr(module, self.hook_name):\n hook = getattr(module, self.hook_name)\n if hasattr(hook, \"__call__\"):\n hook(*args, **kwargs)\n\n\n# pre/post install and run by the install subprocess\npre_install = _HookRunner(\"pre_install\")\npost_install = _HookRunner(\"post_install\")\n\npre_uninstall = _HookRunner(\"pre_uninstall\")\npost_uninstall = _HookRunner(\"post_uninstall\")\n" }, { "path": "lib/spack/spack/hooks/absolutify_elf_sonames.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\n\nimport spack.bootstrap\nimport spack.config\nimport spack.llnl.util.tty as tty\nimport spack.relocate\nfrom spack.llnl.util.filesystem import BaseDirectoryVisitor, visit_directory_tree\nfrom spack.llnl.util.lang import elide_list\nfrom spack.util.elf import ElfParsingError, parse_elf\n\n\ndef is_shared_library_elf(filepath):\n \"\"\"Return true if filepath is most a shared library.\n Our definition of a shared library for ELF requires:\n 1. a dynamic section,\n 2. a soname OR lack of interpreter.\n The problem is that PIE objects (default on Ubuntu) are\n ET_DYN too, and not all shared libraries have a soname...\n no interpreter is typically the best indicator then.\"\"\"\n try:\n with open(filepath, \"rb\") as f:\n elf = parse_elf(f, interpreter=True, dynamic_section=True)\n return elf.has_pt_dynamic and (elf.has_soname or not elf.has_pt_interp)\n except (OSError, ElfParsingError):\n return False\n\n\nclass SharedLibrariesVisitor(BaseDirectoryVisitor):\n \"\"\"Visitor that collects all shared libraries in a prefix, with the\n exception of an exclude list.\"\"\"\n\n def __init__(self, exclude_list):\n # List of file and directory names to be excluded\n self.exclude_list = frozenset(exclude_list)\n\n # Map from (ino, dev) -> path. We need 1 path per file, if there are hardlinks,\n # we don't need to store the path multiple times.\n self.libraries = dict()\n\n # Set of (ino, dev) pairs (excluded by symlinks).\n self.excluded_through_symlink = set()\n\n def visit_file(self, root, rel_path, depth):\n # Check if excluded\n basename = os.path.basename(rel_path)\n if basename in self.exclude_list:\n return\n\n filepath = os.path.join(root, rel_path)\n s = os.lstat(filepath)\n identifier = (s.st_ino, s.st_dev)\n\n # We're hitting a hardlink or symlink of an excluded lib, no need to parse.\n if identifier in self.libraries or identifier in self.excluded_through_symlink:\n return\n\n # Register the file if it's a shared lib that needs to be patched.\n if is_shared_library_elf(filepath):\n self.libraries[identifier] = rel_path\n\n def visit_symlinked_file(self, root, rel_path, depth):\n # We don't need to follow the symlink and parse the file, since we will hit\n # it by recursing the prefix anyways. We only need to check if the target\n # should be excluded based on the filename of the symlink. E.g. when excluding\n # libf.so, which is a symlink to libf.so.1.2.3, we keep track of the stat data\n # of the latter.\n basename = os.path.basename(rel_path)\n if basename not in self.exclude_list:\n return\n\n # Register the (ino, dev) pair as ignored (if the symlink is not dangling)\n filepath = os.path.join(root, rel_path)\n try:\n s = os.stat(filepath)\n except OSError:\n return\n self.excluded_through_symlink.add((s.st_ino, s.st_dev))\n\n def before_visit_dir(self, root, rel_path, depth):\n # Allow skipping over directories. E.g. `/lib/stubs` can be skipped by\n # adding `\"stubs\"` to the exclude list.\n return os.path.basename(rel_path) not in self.exclude_list\n\n def before_visit_symlinked_dir(self, root, rel_path, depth):\n # Never enter symlinked dirs, since we don't want to leave the prefix, and\n # we'll enter the target dir inside the prefix anyways since we're recursing\n # everywhere.\n return False\n\n def get_shared_libraries_relative_paths(self):\n \"\"\"Get the libraries that should be patched, with the excluded libraries\n removed.\"\"\"\n for identifier in self.excluded_through_symlink:\n self.libraries.pop(identifier, None)\n\n return [rel_path for rel_path in self.libraries.values()]\n\n\ndef patch_sonames(patchelf, root, rel_paths):\n \"\"\"Set the soname to the file's own path for a list of\n given shared libraries.\"\"\"\n fixed = []\n for rel_path in rel_paths:\n filepath = os.path.join(root, rel_path)\n normalized = os.path.normpath(filepath)\n args = [\"--set-soname\", normalized, normalized]\n output = patchelf(*args, output=str, error=str, fail_on_error=False)\n if patchelf.returncode == 0:\n fixed.append(rel_path)\n else:\n # Note: treat as warning to avoid (long) builds to fail post-install.\n tty.warn(\"patchelf: failed to set soname of {}: {}\".format(normalized, output.strip()))\n return fixed\n\n\ndef find_and_patch_sonames(prefix, exclude_list, patchelf):\n # Locate all shared libraries in the prefix dir of the spec, excluding\n # the ones set in the non_bindable_shared_objects property.\n visitor = SharedLibrariesVisitor(exclude_list)\n visit_directory_tree(prefix, visitor)\n\n # Patch all sonames.\n relative_paths = visitor.get_shared_libraries_relative_paths()\n return patch_sonames(patchelf, prefix, relative_paths)\n\n\ndef post_install(spec, explicit=None):\n # Skip if disabled\n if not spack.config.get(\"config:shared_linking:bind\", False):\n return\n\n # Skip externals\n if spec.external:\n return\n\n # Only enable on platforms using ELF.\n if not spec.satisfies(\"platform=linux\"):\n return\n\n # Disable this hook when bootstrapping, to avoid recursion.\n if spack.bootstrap.is_bootstrapping():\n return\n\n # Should failing to locate patchelf be a hard error?\n patchelf = spack.relocate._patchelf()\n if not patchelf:\n return\n\n fixes = find_and_patch_sonames(spec.prefix, spec.package.non_bindable_shared_objects, patchelf)\n\n if not fixes:\n return\n\n # Unfortunately this does not end up in the build logs.\n tty.info(\n \"{}: Patched {} {}: {}\".format(\n spec.name,\n len(fixes),\n \"soname\" if len(fixes) == 1 else \"sonames\",\n \", \".join(elide_list(fixes, max_num=5)),\n )\n )\n" }, { "path": "lib/spack/spack/hooks/autopush.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport spack.binary_distribution\nimport spack.llnl.util.tty as tty\nimport spack.mirrors.mirror\n\n\ndef post_install(spec, explicit):\n # Push package to all buildcaches with autopush==True\n\n # Do nothing if spec is an external package\n if spec.external:\n return\n\n # Do nothing if package was not installed from source\n pkg = spec.package\n if pkg.installed_from_binary_cache:\n return\n\n # Push the package to all autopush mirrors\n for mirror in spack.mirrors.mirror.MirrorCollection(binary=True, autopush=True).values():\n signing_key = spack.binary_distribution.select_signing_key() if mirror.signed else None\n with spack.binary_distribution.make_uploader(\n mirror=mirror, force=True, signing_key=signing_key\n ) as uploader:\n uploader.push_or_raise([spec])\n tty.msg(f\"{spec.name}: Pushed to build cache: '{mirror.name}'\")\n" }, { "path": "lib/spack/spack/hooks/drop_redundant_rpaths.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nfrom typing import BinaryIO, Optional, Tuple\n\nimport spack.llnl.util.tty as tty\nfrom spack.llnl.util.filesystem import BaseDirectoryVisitor, visit_directory_tree\nfrom spack.util.elf import ElfParsingError, parse_elf\n\n\ndef should_keep(path: bytes) -> bool:\n \"\"\"Return True iff path starts with $ (typically for $ORIGIN/${ORIGIN}) or is\n absolute and exists.\"\"\"\n return path.startswith(b\"$\") or (os.path.isabs(path) and os.path.lexists(path))\n\n\ndef _drop_redundant_rpaths(f: BinaryIO) -> Optional[Tuple[bytes, bytes]]:\n \"\"\"Drop redundant entries from rpath.\n\n Args:\n f: File object to patch opened in r+b mode.\n\n Returns:\n A tuple of the old and new rpath if the rpath was patched, None otherwise.\n \"\"\"\n try:\n elf = parse_elf(f, interpreter=False, dynamic_section=True)\n except ElfParsingError:\n return None\n\n # Nothing to do.\n if not elf.has_rpath:\n return None\n\n old_rpath_str = elf.dt_rpath_str\n new_rpath_str = b\":\".join(p for p in old_rpath_str.split(b\":\") if should_keep(p))\n\n # Nothing to write.\n if old_rpath_str == new_rpath_str:\n return None\n\n # Pad with 0 bytes.\n pad = len(old_rpath_str) - len(new_rpath_str)\n\n # This can never happen since we're filtering, but lets be defensive.\n if pad < 0:\n return None\n\n # The rpath is at a given offset in the string table used by the\n # dynamic section.\n rpath_offset = elf.pt_dynamic_strtab_offset + elf.rpath_strtab_offset\n\n f.seek(rpath_offset)\n f.write(new_rpath_str + b\"\\x00\" * pad)\n return old_rpath_str, new_rpath_str\n\n\ndef drop_redundant_rpaths(path: str) -> Optional[Tuple[bytes, bytes]]:\n \"\"\"Drop redundant entries from rpath.\n\n Args:\n path: Path to a potential ELF file to patch.\n\n Returns:\n A tuple of the old and new rpath if the rpath was patched, None otherwise.\n \"\"\"\n try:\n with open(path, \"r+b\") as f:\n return _drop_redundant_rpaths(f)\n except OSError:\n return None\n\n\nclass ElfFilesWithRPathVisitor(BaseDirectoryVisitor):\n \"\"\"Visitor that collects all elf files that have an rpath\"\"\"\n\n def __init__(self):\n # Keep track of what hardlinked files we've already visited.\n self.visited = set()\n\n def visit_file(self, root, rel_path, depth):\n filepath = os.path.join(root, rel_path)\n s = os.lstat(filepath)\n identifier = (s.st_ino, s.st_dev)\n\n # We're hitting a hardlink or symlink of an excluded lib, no need to parse.\n if s.st_nlink > 1:\n if identifier in self.visited:\n return\n self.visited.add(identifier)\n\n result = drop_redundant_rpaths(filepath)\n\n if result is not None:\n old, new = result\n tty.debug(f\"Patched rpath in {rel_path} from {old!r} to {new!r}\")\n\n def visit_symlinked_file(self, root, rel_path, depth):\n pass\n\n def before_visit_dir(self, root, rel_path, depth):\n # Always enter dirs\n return True\n\n def before_visit_symlinked_dir(self, root, rel_path, depth):\n # Never enter symlinked dirs\n return False\n\n\ndef post_install(spec, explicit=None):\n # Skip externals\n if spec.external:\n return\n\n # Only enable on platforms using ELF.\n if not spec.satisfies(\"platform=linux\"):\n return\n\n visit_directory_tree(spec.prefix, ElfFilesWithRPathVisitor())\n" }, { "path": "lib/spack/spack/hooks/licensing.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\n\nimport spack.llnl.util.tty as tty\nimport spack.util.editor as ed\nfrom spack.llnl.util.filesystem import mkdirp, symlink\n\n\ndef pre_install(spec):\n \"\"\"This hook handles global license setup for licensed software.\"\"\"\n pkg = spec.package\n if pkg.license_required and not pkg.spec.external:\n set_up_license(pkg)\n\n\ndef set_up_license(pkg):\n \"\"\"Prompt the user, letting them know that a license is required.\n\n For packages that rely on license files, a global license file is\n created and opened for editing.\n\n For packages that rely on environment variables to point to a\n license, a warning message is printed.\n\n For all other packages, documentation on how to set up a license\n is printed.\"\"\"\n\n # If the license can be stored in a file, create one\n if pkg.license_files:\n license_path = pkg.global_license_file\n if not os.path.exists(license_path):\n # Create a new license file\n write_license_file(pkg, license_path)\n\n # use spack.util.executable so the editor does not hang on return here\n ed.editor(license_path, exec_fn=ed.executable)\n else:\n # Use already existing license file\n tty.msg(\"Found already existing license %s\" % license_path)\n\n # If not a file, what about an environment variable?\n elif pkg.license_vars:\n tty.warn(\n \"A license is required to use %s. Please set %s to the \"\n \"full pathname to the license file, or port@host if you\"\n \" store your license keys on a dedicated license server\"\n % (pkg.name, \" or \".join(pkg.license_vars))\n )\n\n # If not a file or variable, suggest a website for further info\n elif pkg.license_url:\n tty.warn(\n \"A license is required to use %s. See %s for details\" % (pkg.name, pkg.license_url)\n )\n\n # If all else fails, you're on your own\n else:\n tty.warn(\"A license is required to use %s\" % pkg.name)\n\n\ndef write_license_file(pkg, license_path):\n \"\"\"Writes empty license file.\n\n Comments give suggestions on alternative methods of\n installing a license.\"\"\"\n\n # License files\n linktargets = \"\"\n for f in pkg.license_files:\n linktargets += \"\\t%s\\n\" % f\n\n # Environment variables\n envvars = \"\"\n if pkg.license_vars:\n for varname in pkg.license_vars:\n envvars += \"\\t%s\\n\" % varname\n\n # Documentation\n url = \"\"\n if pkg.license_url:\n url += \"\\t%s\\n\" % pkg.license_url\n\n # Assemble. NB: pkg.license_comment will be prepended upon output.\n txt = \"\"\"\n A license is required to use package '{0}'.\n\n * If your system is already properly configured for such a license, save this\n file UNCHANGED. The system may be configured if:\n\n - A license file is installed in a default location.\n\"\"\".format(pkg.name)\n\n if envvars:\n txt += \"\"\"\\\n - One of the following environment variable(s) is set for you, possibly via\n a module file:\n\n{0}\n\"\"\".format(envvars)\n\n txt += \"\"\"\\\n * Otherwise, depending on the license you have, enter AT THE BEGINNING of\n this file:\n\n - the contents of your license file, or\n - the address(es) of your license server.\n\n After installation, the following symlink(s) will be added to point to\n this Spack-global file (relative to the installation prefix).\n\n{0}\n\"\"\".format(linktargets)\n\n if url:\n txt += \"\"\"\\\n * For further information on licensing, see:\n\n{0}\n\"\"\".format(url)\n\n txt += \"\"\"\\\n Recap:\n - You may not need to modify this file at all.\n - Otherwise, enter your license or server address AT THE BEGINNING.\n\"\"\"\n # Global license directory may not already exist\n if not os.path.exists(os.path.dirname(license_path)):\n os.makedirs(os.path.dirname(license_path))\n\n # Output\n with open(license_path, \"w\", encoding=\"utf-8\") as f:\n for line in txt.splitlines():\n f.write(\"{0}{1}\\n\".format(pkg.license_comment, line))\n f.close()\n\n\ndef post_install(spec, explicit=None):\n \"\"\"This hook symlinks local licenses to the global license for\n licensed software.\n \"\"\"\n pkg = spec.package\n if pkg.license_required and not pkg.spec.external:\n symlink_license(pkg)\n\n\ndef symlink_license(pkg):\n \"\"\"Create local symlinks that point to the global license file.\"\"\"\n target = pkg.global_license_file\n for filename in pkg.license_files:\n link_name = os.path.join(pkg.prefix, filename)\n link_name = os.path.abspath(link_name)\n license_dir = os.path.dirname(link_name)\n if not os.path.exists(license_dir):\n mkdirp(license_dir)\n\n # If example file already exists, overwrite it with a symlink\n if os.path.lexists(link_name):\n os.remove(link_name)\n\n if os.path.exists(target):\n symlink(target, link_name)\n tty.msg(\"Added local symlink %s to global license file\" % link_name)\n" }, { "path": "lib/spack/spack/hooks/module_file_generation.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nfrom typing import Optional, Set\n\nimport spack.config\nimport spack.modules\nimport spack.spec\nfrom spack.llnl.util import tty\n\n\ndef _for_each_enabled(\n spec: spack.spec.Spec, method_name: str, explicit: Optional[bool] = None\n) -> None:\n \"\"\"Calls a method for each enabled module\"\"\"\n set_names: Set[str] = set(spack.config.get(\"modules\", {}).keys())\n for name in set_names:\n enabled = spack.config.get(f\"modules:{name}:enable\")\n if not enabled:\n tty.debug(\"NO MODULE WRITTEN: list of enabled module files is empty\")\n continue\n\n for module_type in enabled:\n generator = spack.modules.module_types[module_type](spec, name, explicit)\n try:\n getattr(generator, method_name)()\n except RuntimeError as e:\n msg = \"cannot perform the requested {0} operation on module files\"\n msg += \" [{1}]\"\n tty.warn(msg.format(method_name, str(e)))\n\n\ndef post_install(spec, explicit: bool):\n _for_each_enabled(spec, \"write\", explicit)\n\n\ndef post_uninstall(spec):\n _for_each_enabled(spec, \"remove\")\n" }, { "path": "lib/spack/spack/hooks/permissions_setters.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\n\nimport spack.util.file_permissions as fp\n\n\ndef post_install(spec, explicit=None):\n if not spec.external:\n fp.set_permissions_by_spec(spec.prefix, spec)\n\n # os.walk explicitly set not to follow links\n for root, dirs, files in os.walk(spec.prefix, followlinks=False):\n for d in dirs:\n if not os.path.islink(os.path.join(root, d)):\n fp.set_permissions_by_spec(os.path.join(root, d), spec)\n for f in files:\n if not os.path.islink(os.path.join(root, f)):\n fp.set_permissions_by_spec(os.path.join(root, f), spec)\n" }, { "path": "lib/spack/spack/hooks/resolve_shared_libraries.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport io\n\nimport spack.config\nimport spack.error\nimport spack.llnl.util.tty as tty\nimport spack.verify_libraries\nfrom spack.llnl.util.filesystem import visit_directory_tree\n\n\ndef post_install(spec, explicit):\n \"\"\"Check whether shared libraries can be resolved in RPATHs.\"\"\"\n policy = spack.config.get(\"config:shared_linking:missing_library_policy\", \"ignore\")\n\n # Currently only supported for ELF files.\n if policy == \"ignore\" or spec.external or spec.platform not in (\"linux\", \"freebsd\"):\n return\n\n visitor = spack.verify_libraries.ResolveSharedElfLibDepsVisitor(\n [*spack.verify_libraries.ALLOW_UNRESOLVED, *spec.package.unresolved_libraries]\n )\n visit_directory_tree(spec.prefix, visitor)\n\n if not visitor.problems:\n return\n\n output = io.StringIO(\"not all executables and libraries can resolve their dependencies:\\n\")\n visitor.write(output)\n message = output.getvalue().strip()\n\n if policy == \"error\":\n raise CannotLocateSharedLibraries(message)\n\n tty.warn(message)\n\n\nclass CannotLocateSharedLibraries(spack.error.SpackError):\n pass\n" }, { "path": "lib/spack/spack/hooks/sbang.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport filecmp\nimport os\nimport re\nimport shutil\nimport stat\nimport sys\nimport tempfile\n\nimport spack.error\nimport spack.llnl.util.filesystem as fs\nimport spack.llnl.util.tty as tty\nimport spack.package_prefs\nimport spack.paths\nimport spack.spec\nimport spack.store\nfrom spack.util.socket import _gethostname\n\n#: OS-imposed character limit for shebang line: 127 for Linux; 511 for Mac.\n#: Different Linux distributions have different limits, but 127 is the\n#: smallest among all modern versions.\nif sys.platform == \"darwin\":\n system_shebang_limit = 511\nelse:\n system_shebang_limit = 127\n try:\n # searching for line '#define BINPRM_BUF_SIZE 256' in /usr/include/linux/binfmts.h\n # the nbr-1 is the sbang limit on the linux platform\n sbang_limit_re = re.compile(\"#define BINPRM_BUF_SIZE ([0-9]+)\")\n with open(\"/usr/include/linux/binfmts.h\", \"r\", encoding=\"utf-8\") as f:\n for line in f:\n m = sbang_limit_re.match(line)\n if m:\n system_shebang_limit = int(m.group(1)) - 1\n except Exception:\n # ignore any error a sane default is set already\n pass\n\n#: Groupdb does not exist on Windows, prevent imports\n#: on supported systems\nif sys.platform != \"win32\":\n import grp\n\n#: Spack itself also limits the shebang line to at most 4KB, which should be plenty.\nspack_shebang_limit = 4096\n\ninterpreter_regex = re.compile(b\"#![ \\t]*?([^ \\t\\0\\n]+)\")\n\n\ndef sbang_install_path():\n \"\"\"Location sbang is installed within the install tree.\"\"\"\n sbang_root = str(spack.store.STORE.unpadded_root)\n install_path = os.path.join(sbang_root, \"bin\", \"sbang\")\n path_length = len(install_path)\n if path_length > system_shebang_limit:\n msg = (\n \"Install tree root is too long. Spack cannot patch shebang lines\"\n \" when script path length ({0}) exceeds limit ({1}).\\n {2}\"\n )\n msg = msg.format(path_length, system_shebang_limit, install_path)\n raise SbangPathError(msg)\n return install_path\n\n\ndef sbang_shebang_line():\n \"\"\"Full shebang line that should be prepended to files to use sbang.\n\n The line returned does not have a final newline (caller should add it\n if needed).\n\n This should be the only place in Spack that knows about what\n interpreter we use for ``sbang``.\n \"\"\"\n return \"#!/bin/sh %s\" % sbang_install_path()\n\n\ndef get_interpreter(binary_string):\n # The interpreter may be preceded with ' ' and \\t, is itself any byte that\n # follows until the first occurrence of ' ', \\t, \\0, \\n or end of file.\n match = interpreter_regex.match(binary_string)\n return None if match is None else match.group(1)\n\n\ndef filter_shebang(path):\n \"\"\"\n Adds a second shebang line, using sbang, at the beginning of a file, if necessary.\n Note: Spack imposes a relaxed shebang line limit, meaning that a newline or end of\n file must occur before ``spack_shebang_limit`` bytes. If not, the file is not\n patched.\n \"\"\"\n with open(path, \"rb\") as original:\n # If there is no shebang, we shouldn't replace anything.\n old_shebang_line = original.read(2)\n if old_shebang_line != b\"#!\":\n return False\n\n # Stop reading after b'\\n'. Note that old_shebang_line includes the first b'\\n'.\n old_shebang_line += original.readline(spack_shebang_limit - 2)\n\n # If the shebang line is short, we don't have to do anything.\n if len(old_shebang_line) <= system_shebang_limit:\n return False\n\n # Whenever we can't find a newline within the maximum number of bytes, we will\n # not attempt to rewrite it. In principle we could still get the interpreter if\n # only the arguments are truncated, but note that for PHP we need the full line\n # since we have to append `?>` to it. Since our shebang limit is already very\n # generous, it's unlikely to happen, and it should be fine to ignore.\n if len(old_shebang_line) == spack_shebang_limit and old_shebang_line[-1] != b\"\\n\":\n return False\n\n # This line will be prepended to file\n new_sbang_line = (sbang_shebang_line() + \"\\n\").encode(\"utf-8\")\n\n # Skip files that are already using sbang.\n if old_shebang_line == new_sbang_line:\n return\n\n interpreter = get_interpreter(old_shebang_line)\n\n # If there was only whitespace we don't have to do anything.\n if not interpreter:\n return False\n\n # Store the file permissions, the patched version needs the same.\n saved_mode = os.stat(path).st_mode\n\n # Change non-writable files to be writable if needed.\n if not os.access(path, os.W_OK):\n os.chmod(path, saved_mode | stat.S_IWUSR)\n\n # No need to delete since we'll move it and overwrite the original.\n patched = tempfile.NamedTemporaryFile(\"wb\", delete=False)\n patched.write(new_sbang_line)\n\n # Note that in Python this does not go out of bounds even if interpreter is a\n # short byte array.\n # Note: if the interpreter string was encoded with UTF-16, there would have\n # been a \\0 byte between all characters of lua, node, php; meaning that it would\n # lead to truncation of the interpreter. So we don't have to worry about weird\n # encodings here, and just looking at bytes is justified.\n if interpreter[-4:] == b\"/lua\" or interpreter[-7:] == b\"/luajit\":\n # Use --! instead of #! on second line for lua.\n patched.write(b\"--!\" + old_shebang_line[2:])\n elif interpreter[-5:] == b\"/node\":\n # Use //! instead of #! on second line for node.js.\n patched.write(b\"//!\" + old_shebang_line[2:])\n elif interpreter[-4:] == b\"/php\":\n # Use instead of #!... on second line for php.\n patched.write(b\"\")\n else:\n patched.write(old_shebang_line)\n\n # After copying the remainder of the file, we can close the original\n shutil.copyfileobj(original, patched)\n\n # And close the temporary file so we can move it.\n patched.close()\n\n # Overwrite original file with patched file, and keep the original mode\n shutil.move(patched.name, path)\n os.chmod(path, saved_mode)\n return True\n\n\ndef filter_shebangs_in_directory(directory, filenames=None):\n if filenames is None:\n filenames = os.listdir(directory)\n\n is_exe = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH\n\n for file in filenames:\n path = os.path.join(directory, file)\n\n # Only look at executable, non-symlink files.\n try:\n st = os.lstat(path)\n except OSError:\n continue\n\n if stat.S_ISLNK(st.st_mode) or stat.S_ISDIR(st.st_mode) or not st.st_mode & is_exe:\n continue\n\n # test the file for a long shebang, and filter\n if filter_shebang(path):\n tty.debug(\"Patched overlong shebang in %s\" % path)\n\n\ndef install_sbang():\n \"\"\"Ensure that ``sbang`` is installed in the root of Spack's install_tree.\n\n This is the shortest known publicly accessible path, and installing\n ``sbang`` here ensures that users can access the script and that\n ``sbang`` itself is in a short path.\n \"\"\"\n # copy in a new version of sbang if it differs from what's in spack\n sbang_path = sbang_install_path()\n if os.path.exists(sbang_path) and filecmp.cmp(spack.paths.sbang_script, sbang_path):\n return\n\n # make $install_tree/bin\n sbang_bin_dir = os.path.dirname(sbang_path)\n fs.mkdirp(sbang_bin_dir)\n\n # get permissions for bin dir from configuration files\n group_name = spack.package_prefs.get_package_group(spack.spec.Spec(\"all\"))\n config_mode = spack.package_prefs.get_package_dir_permissions(spack.spec.Spec(\"all\"))\n\n if group_name:\n os.chmod(sbang_bin_dir, config_mode) # Use package directory permissions\n else:\n fs.set_install_permissions(sbang_bin_dir)\n\n # set group on sbang_bin_dir if not already set (only if set in configuration)\n # TODO: after we drop python2 support, use shutil.chown to avoid gid lookups that\n # can fail for remote groups\n if group_name and os.stat(sbang_bin_dir).st_gid != grp.getgrnam(group_name).gr_gid:\n os.chown(sbang_bin_dir, os.stat(sbang_bin_dir).st_uid, grp.getgrnam(group_name).gr_gid)\n\n # copy over the fresh copy of `sbang`\n sbang_tmp_path = os.path.join(sbang_bin_dir, f\".sbang.{_gethostname()}.{os.getpid()}.tmp\")\n shutil.copy(spack.paths.sbang_script, sbang_tmp_path)\n\n # set permissions on `sbang` (including group if set in configuration)\n os.chmod(sbang_tmp_path, config_mode)\n if group_name:\n os.chown(sbang_tmp_path, os.stat(sbang_tmp_path).st_uid, grp.getgrnam(group_name).gr_gid)\n\n # Finally, move the new `sbang` into place atomically\n os.rename(sbang_tmp_path, sbang_path)\n\n\ndef post_install(spec, explicit=None):\n \"\"\"This hook edits scripts so that they call /bin/bash\n $spack_prefix/bin/sbang instead of something longer than the\n shebang limit.\n \"\"\"\n if sys.platform == \"win32\":\n return\n if spec.external:\n tty.debug(\"SKIP: shebang filtering [external package]\")\n return\n\n install_sbang()\n\n for directory, _, filenames in os.walk(spec.prefix):\n filter_shebangs_in_directory(directory, filenames)\n\n\nclass SbangPathError(spack.error.SpackError):\n \"\"\"Raised when the install tree root is too long for sbang to work.\"\"\"\n" }, { "path": "lib/spack/spack/hooks/windows_runtime_linkage.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\ndef post_install(spec, explicit=None):\n spec.package.windows_establish_runtime_linkage()\n" }, { "path": "lib/spack/spack/hooks/write_install_manifest.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport spack.verify\n\n\ndef post_install(spec, explicit=None):\n if not spec.external:\n spack.verify.write_manifest(spec)\n" }, { "path": "lib/spack/spack/install_test.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport base64\nimport contextlib\nimport enum\nimport hashlib\nimport inspect\nimport io\nimport os\nimport re\nimport shutil\nimport sys\nfrom collections import Counter, OrderedDict\nfrom typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Optional, Tuple, Type, Union\n\nimport spack.config\nimport spack.error\nimport spack.llnl.util.filesystem as fs\nimport spack.llnl.util.tty as tty\nimport spack.llnl.util.tty.log\nimport spack.paths\nimport spack.repo\nimport spack.report\nimport spack.spec\nimport spack.util.executable\nimport spack.util.path\nimport spack.util.spack_json as sjson\nfrom spack.error import InstallError\nfrom spack.llnl.string import plural\nfrom spack.llnl.util.lang import nullcontext\nfrom spack.llnl.util.tty.color import colorize\nfrom spack.spec import Spec\nfrom spack.util.prefix import Prefix\n\nif TYPE_CHECKING:\n import spack.package_base\n\n#: Stand-alone test failure info type\nTestFailureType = Tuple[BaseException, str]\n\n#: Name of the test suite's (JSON) lock file\ntest_suite_filename = \"test_suite.lock\"\n\n#: Name of the test suite results (summary) file\nresults_filename = \"results.txt\"\n\n#: Name of the Spack install phase-time test log file\nspack_install_test_log = \"install-time-test-log.txt\"\n\n\nListOrStringType = Union[str, List[str]]\nLogType = Union[spack.llnl.util.tty.log.nixlog, spack.llnl.util.tty.log.winlog]\n\nPackageObjectOrClass = Union[\n \"spack.package_base.PackageBase\", Type[\"spack.package_base.PackageBase\"]\n]\n\n\nclass TestStatus(enum.Enum):\n \"\"\"Names of different stand-alone test states.\"\"\"\n\n NO_TESTS = -1\n SKIPPED = 0\n FAILED = 1\n PASSED = 2\n\n def __str__(self):\n return f\"{self.name}\"\n\n def lower(self):\n name = f\"{self.name}\"\n return name.lower()\n\n\ndef get_escaped_text_output(filename: str) -> List[str]:\n \"\"\"Retrieve and escape the expected text output from the file\n\n Args:\n filename: path to the file\n\n Returns:\n escaped text lines read from the file\n \"\"\"\n with open(filename, encoding=\"utf-8\") as f:\n # Ensure special characters are escaped as needed\n expected = f.read()\n\n # Split the lines to make it easier to debug failures when there is\n # a lot of output\n return [re.escape(ln) for ln in expected.split(\"\\n\")]\n\n\ndef get_test_stage_dir() -> str:\n \"\"\"Retrieves the ``config:test_stage`` path to the configured test stage\n root directory\n\n Returns:\n absolute path to the configured test stage root or, if none, the default test stage path\n \"\"\"\n return spack.util.path.canonicalize_path(\n spack.config.get(\"config:test_stage\", spack.paths.default_test_path)\n )\n\n\ndef cache_extra_test_sources(pkg: \"spack.package_base.PackageBase\", srcs: ListOrStringType):\n \"\"\"Copy relative source paths to the corresponding install test subdir\n\n This routine is intended as an optional install test setup helper for\n grabbing source files/directories during the installation process and\n copying them to the installation test subdirectory for subsequent use\n during install testing.\n\n Args:\n pkg: package being tested\n srcs: relative path for file(s) and or subdirectory(ies) located in\n the staged source path that are to be copied to the corresponding\n location(s) under the install testing directory.\n\n Raises:\n spack.error.InstallError: if any of the source paths are absolute\n or do not exist\n under the build stage\n \"\"\"\n errors = []\n paths = [srcs] if isinstance(srcs, str) else srcs\n for path in paths:\n pre = f\"Source path ('{path}')\"\n src_path = os.path.join(pkg.stage.source_path, path)\n dest_path = os.path.join(install_test_root(pkg), path)\n if os.path.isabs(path):\n errors.append(f\"{pre} must be relative to the build stage directory.\")\n continue\n\n if os.path.isdir(src_path):\n fs.install_tree(src_path, dest_path)\n elif os.path.exists(src_path):\n fs.mkdirp(os.path.dirname(dest_path))\n fs.copy(src_path, dest_path)\n else:\n errors.append(f\"{pre} for the copy does not exist\")\n\n if errors:\n raise InstallError(\"\\n\".join(errors), pkg=pkg)\n\n\ndef check_outputs(expected: Union[list, set, str], actual: str):\n \"\"\"Ensure the expected outputs are contained in the actual outputs.\n\n Args:\n expected: expected raw output string(s)\n actual: actual output string\n\n Raises:\n RuntimeError: the expected output is not found in the actual output\n \"\"\"\n expected = expected if isinstance(expected, (list, set)) else [expected]\n errors = []\n for check in expected:\n if not re.search(check, actual):\n errors.append(f\"Expected '{check}' in output '{actual}'\")\n if errors:\n raise RuntimeError(\"\\n \".join(errors))\n\n\ndef find_required_file(\n root: str, filename: str, expected: int = 1, recursive: bool = True\n) -> ListOrStringType:\n \"\"\"Find the required file(s) under the root directory.\n\n Args:\n root: root directory for the search\n filename: name of the file being located\n expected: expected number of files to be found under the directory\n (default is 1)\n recursive: ``True`` if subdirectories are to be recursively searched,\n else ``False`` (default is ``True``)\n\n Returns: the path(s), relative to root, to the required file(s)\n\n Raises:\n Exception: SkipTest when number of files detected does not match expected\n \"\"\"\n paths = fs.find(root, filename, recursive=recursive)\n num_paths = len(paths)\n if num_paths != expected:\n files = \": {}\".format(\", \".join(paths)) if num_paths else \"\"\n raise SkipTest(\n \"Expected {} of {} under {} but {} found{}\".format(\n plural(expected, \"copy\", \"copies\"),\n filename,\n root,\n plural(num_paths, \"copy\", \"copies\"),\n files,\n )\n )\n\n return paths[0] if expected == 1 else paths\n\n\ndef install_test_root(pkg: \"spack.package_base.PackageBase\") -> str:\n \"\"\"The install test root directory.\"\"\"\n return os.path.join(pkg.metadata_dir, \"test\")\n\n\ndef print_message(logger: LogType, msg: str, verbose: bool = False):\n \"\"\"Print the message to the log, optionally echoing.\n\n Args:\n logger: instance of the output logger (e.g. nixlog or winlog)\n msg: message being output\n verbose: ``True`` displays verbose output, ``False`` suppresses\n it (``False`` is default)\n \"\"\"\n if verbose:\n with logger.force_echo():\n tty.info(msg, format=\"g\")\n else:\n tty.info(msg, format=\"g\")\n\n\ndef overall_status(current_status: \"TestStatus\", substatuses: List[\"TestStatus\"]) -> \"TestStatus\":\n \"\"\"Determine the overall status based on the current and associated sub status values.\n\n Args:\n current_status: current overall status, assumed to default to PASSED\n substatuses: status of each test part or overall status of each test spec\n Returns:\n test status encompassing the main test and all subtests\n \"\"\"\n if current_status in [TestStatus.SKIPPED, TestStatus.NO_TESTS, TestStatus.FAILED]:\n return current_status\n\n skipped = 0\n for status in substatuses:\n if status == TestStatus.FAILED:\n return status\n elif status == TestStatus.SKIPPED:\n skipped += 1\n\n if skipped and skipped == len(substatuses):\n return TestStatus.SKIPPED\n\n return current_status\n\n\nclass PackageTest:\n \"\"\"The class that manages stand-alone (post-install) package tests.\"\"\"\n\n def __init__(self, pkg: \"spack.package_base.PackageBase\") -> None:\n \"\"\"\n Args:\n pkg: package being tested\n\n Raises:\n ValueError: if the package is not concrete\n \"\"\"\n if not pkg.spec.concrete:\n raise ValueError(\"Stand-alone tests require a concrete package\")\n\n self.counts: \"Counter\" = Counter() # type: ignore[attr-defined]\n self.pkg = pkg\n self.test_failures: List[TestFailureType] = []\n self.test_parts: OrderedDict[str, \"TestStatus\"] = OrderedDict()\n self.test_log_file: str\n self.pkg_id: str\n\n if pkg.test_suite:\n # Running stand-alone tests\n self.test_log_file = pkg.test_suite.log_file_for_spec(pkg.spec)\n self.tested_file = pkg.test_suite.tested_file_for_spec(pkg.spec)\n self.pkg_id = pkg.test_suite.test_pkg_id(pkg.spec)\n else:\n # Running phase-time tests for a single package whose results are\n # retained in the package's stage directory.\n pkg.test_suite = TestSuite([pkg.spec])\n self.test_log_file = fs.join_path(pkg.stage.path, spack_install_test_log)\n self.pkg_id = pkg.spec.format(\"{name}-{version}-{hash:7}\")\n\n # Internal logger for test part processing\n self._logger = None\n\n @property\n def logger(self) -> Optional[LogType]:\n \"\"\"The current logger or, if none, sets to one.\"\"\"\n if not self._logger:\n self._logger = spack.llnl.util.tty.log.log_output(self.test_log_file)\n\n return self._logger\n\n @contextlib.contextmanager\n def test_logger(self, verbose: bool = False, externals: bool = False):\n \"\"\"Context manager for setting up the test logger\n\n Args:\n verbose: Display verbose output, including echoing to stdout,\n otherwise suppress it\n externals: ``True`` for performing tests if external package,\n ``False`` to skip them\n \"\"\"\n fs.touch(self.test_log_file) # Otherwise log_parse complains\n fs.set_install_permissions(self.test_log_file)\n\n with spack.llnl.util.tty.log.log_output(\n self.test_log_file, verbose, append=True\n ) as self._logger:\n with self.logger.force_echo(): # type: ignore[union-attr]\n tty.msg(\"Testing package \" + colorize(r\"@*g{\" + self.pkg_id + r\"}\"))\n\n # use debug print levels for log file to record commands\n old_debug = tty.is_debug()\n tty.set_debug(True)\n\n try:\n yield self.logger\n finally:\n # reset debug level\n tty.set_debug(old_debug)\n\n @property\n def archived_install_test_log(self) -> str:\n return fs.join_path(self.pkg.metadata_dir, spack_install_test_log)\n\n def archive_install_test_log(self, dest_dir: str):\n if os.path.exists(self.test_log_file):\n fs.install(self.test_log_file, self.archived_install_test_log)\n\n def add_failure(self, exception: Exception, msg: str):\n \"\"\"Add the failure details to the current list.\"\"\"\n self.test_failures.append((exception, msg))\n\n def status(self, name: str, status: \"TestStatus\", msg: Optional[str] = None):\n \"\"\"Track and print the test status for the test part name.\"\"\"\n part_name = f\"{self.pkg.__class__.__name__}::{name}\"\n extra = \"\" if msg is None else f\": {msg}\"\n\n # Handle the special case of a test part consisting of subparts.\n # The containing test part can be PASSED while sub-parts (assumed\n # to start with the same name) may not have PASSED. This extra\n # check is used to ensure the containing test part is not claiming\n # to have passed when at least one subpart failed.\n substatuses = []\n for pname, substatus in self.test_parts.items():\n if pname != part_name and pname.startswith(part_name):\n substatuses.append(substatus)\n if substatuses:\n status = overall_status(status, substatuses)\n\n print(f\"{status}: {part_name}{extra}\")\n self.test_parts[part_name] = status\n self.counts[status] += 1\n\n def phase_tests(self, builder, phase_name: str, method_names: List[str]):\n \"\"\"Execute the builder's package phase-time tests.\n\n Args:\n builder: builder for package being tested\n phase_name: the name of the build-time phase (e.g., ``build``, ``install``)\n method_names: phase-specific callback method names\n \"\"\"\n verbose = tty.is_verbose()\n fail_fast = spack.config.get(\"config:fail_fast\", False)\n\n with self.test_logger(verbose=verbose, externals=False) as logger:\n # Report running each of the methods in the build log\n print_message(logger, f\"Running {phase_name}-time tests\", verbose)\n builder.pkg.test_suite.current_test_spec = builder.pkg.spec\n builder.pkg.test_suite.current_base_spec = builder.pkg.spec\n\n have_tests = any(name.startswith(\"test_\") for name in method_names)\n if have_tests:\n copy_test_files(builder.pkg, builder.pkg.spec)\n\n for name in method_names:\n try:\n fn = getattr(builder, name, None) or getattr(builder.pkg, name)\n except AttributeError as e:\n print_message(logger, f\"RUN-TESTS: method not implemented [{name}]\", verbose)\n self.add_failure(e, f\"RUN-TESTS: method not implemented [{name}]\")\n if fail_fast:\n break\n continue\n\n print_message(logger, f\"RUN-TESTS: {phase_name}-time tests [{name}]\", verbose)\n fn()\n\n if have_tests:\n print_message(logger, \"Completed testing\", verbose)\n\n # Raise any collected failures here\n if self.test_failures:\n raise TestFailure(self.test_failures)\n\n def stand_alone_tests(self, kwargs, timeout: Optional[int] = None) -> None:\n \"\"\"Run the package's stand-alone tests.\n\n Args:\n kwargs (dict): arguments to be used by the test process\n \"\"\"\n import spack.build_environment # avoid circular dependency\n\n process = spack.build_environment.start_build_process(\n self.pkg, test_process, kwargs, timeout=timeout\n )\n process.complete()\n\n def parts(self) -> int:\n \"\"\"The total number of (checked) test parts.\"\"\"\n try:\n # New in Python 3.10\n total = self.counts.total() # type: ignore[attr-defined]\n except AttributeError:\n nums = [n for _, n in self.counts.items()]\n total = sum(nums)\n return total\n\n def print_log_path(self):\n \"\"\"Print the test log file path.\"\"\"\n log = self.archived_install_test_log\n if not os.path.isfile(log):\n log = self.test_log_file\n if not (log and os.path.isfile(log)):\n tty.debug(\"There is no test log file (staged or installed)\")\n return\n\n print(f\"\\nSee test results at:\\n {log}\")\n\n def ran_tests(self) -> bool:\n \"\"\"``True`` if ran tests, ``False`` otherwise.\"\"\"\n return self.parts() > self.counts[TestStatus.NO_TESTS]\n\n def summarize(self):\n \"\"\"Collect test results summary lines for this spec.\"\"\"\n lines = []\n lines.append(\"{:=^80}\".format(f\" SUMMARY: {self.pkg_id} \"))\n for name, status in self.test_parts.items():\n msg = f\"{name} .. {status}\"\n lines.append(msg)\n\n summary = [f\"{n} {s.lower()}\" for s, n in self.counts.items() if n > 0]\n totals = \" {} of {} parts \".format(\", \".join(summary), self.parts())\n lines.append(f\"{totals:=^80}\")\n return lines\n\n def write_tested_status(self):\n \"\"\"Write the overall status to the tested file.\n\n If there any test part failures, then the tests failed. If all test\n parts are skipped, then the tests were skipped. If any tests passed\n then the tests passed; otherwise, there were not tests executed.\n \"\"\"\n status = TestStatus.NO_TESTS\n if self.counts[TestStatus.FAILED] > 0:\n status = TestStatus.FAILED\n else:\n skipped = self.counts[TestStatus.SKIPPED]\n if skipped and self.parts() == skipped:\n status = TestStatus.SKIPPED\n elif self.counts[TestStatus.PASSED] > 0:\n status = TestStatus.PASSED\n\n with open(self.tested_file, \"w\", encoding=\"utf-8\") as f:\n f.write(f\"{status.value}\\n\")\n\n\n@contextlib.contextmanager\ndef test_part(\n pkg: \"spack.package_base.PackageBase\",\n test_name: str,\n purpose: str,\n work_dir: str = \".\",\n verbose: bool = False,\n):\n # avoid circular dependency\n from spack.build_environment import get_package_context, write_log_summary\n\n wdir = \".\" if work_dir is None else work_dir\n tester = pkg.tester\n assert test_name and test_name.startswith(\"test_\"), (\n f\"Test name must start with 'test_' but {test_name} was provided\"\n )\n\n title = \"test: {}: {}\".format(test_name, purpose or \"unspecified purpose\")\n with fs.working_dir(wdir, create=True):\n try:\n context = tester.logger.force_echo if verbose else nullcontext\n with context():\n tty.info(title, format=\"g\")\n yield\n tester.status(test_name, TestStatus.PASSED)\n\n except SkipTest as e:\n tester.status(test_name, TestStatus.SKIPPED, str(e))\n\n except (AssertionError, BaseException) as e:\n # print a summary of the error to the log file\n # so that cdash and junit reporters know about it\n exc_type, _, tb = sys.exc_info()\n tester.status(test_name, TestStatus.FAILED, str(e))\n\n import traceback\n\n # remove the current call frame to exclude the extract_stack\n # call from the error\n stack = traceback.extract_stack()[:-1]\n\n # Format and print the stack\n out = traceback.format_list(stack)\n for line in out:\n print(line.rstrip(\"\\n\"))\n\n if exc_type is spack.util.executable.ProcessError or exc_type is TypeError:\n iostr = io.StringIO()\n write_log_summary(iostr, \"test\", tester.test_log_file, last=1) # type: ignore[assignment]\n m = iostr.getvalue()\n else:\n # We're below the package context, so get context from\n # stack instead of from traceback.\n # The traceback is truncated here, so we can't use it to\n # traverse the stack.\n m = \"\\n\".join(get_package_context(tb) or \"\")\n\n exc = e # e is deleted after this block\n\n # If we fail fast, raise another error\n if spack.config.get(\"config:fail_fast\", False):\n raise TestFailure([(exc, m)])\n else:\n tester.add_failure(exc, m)\n\n\ndef copy_test_files(pkg: \"spack.package_base.PackageBase\", test_spec: spack.spec.Spec):\n \"\"\"Copy the spec's cached and custom test files to the test stage directory.\n\n Args:\n pkg: package being tested\n test_spec: spec being tested, where the spec may be virtual\n\n Raises:\n TestSuiteError: package must be part of an active test suite\n \"\"\"\n if pkg is None or pkg.test_suite is None:\n base = \"Cannot copy test files\"\n msg = (\n f\"{base} without a package\"\n if pkg is None\n else f\"{pkg.name}: {base}: test suite is missing\"\n )\n raise TestSuiteError(msg)\n\n # copy installed test sources cache into test stage dir\n if test_spec.concrete:\n cache_source = install_test_root(test_spec.package)\n cache_dir = pkg.test_suite.current_test_cache_dir\n if os.path.isdir(cache_source) and not os.path.exists(cache_dir):\n fs.install_tree(cache_source, cache_dir)\n\n # copy test data into test stage data dir\n try:\n pkg_cls = spack.repo.PATH.get_pkg_class(test_spec.fullname)\n except spack.repo.UnknownPackageError:\n tty.debug(f\"{test_spec.name}: skipping test data copy since no package class found\")\n return\n\n data_source = Prefix(pkg_cls.package_dir).test\n data_dir = pkg.test_suite.current_test_data_dir\n if os.path.isdir(data_source) and not os.path.exists(data_dir):\n # We assume data dir is used read-only\n # maybe enforce this later\n shutil.copytree(data_source, data_dir)\n\n\ndef test_function_names(pkg: \"PackageObjectOrClass\", add_virtuals: bool = False) -> List[str]:\n \"\"\"Grab the names of all non-empty test functions.\n\n Args:\n pkg: package or package class of interest\n add_virtuals: ``True`` adds test methods of provided package\n virtual, ``False`` only returns test functions of the package\n\n Returns:\n names of non-empty test functions\n\n Raises:\n ValueError: occurs if pkg is not a package class\n \"\"\"\n fns = test_functions(pkg, add_virtuals)\n return [f\"{cls_name}.{fn.__name__}\" for (cls_name, fn) in fns]\n\n\ndef test_functions(\n pkg: \"PackageObjectOrClass\", add_virtuals: bool = False\n) -> List[Tuple[str, Callable]]:\n \"\"\"Grab all non-empty test functions.\n\n Args:\n pkg: package or package class of interest\n add_virtuals: ``True`` adds test methods of provided package\n virtual, ``False`` only returns test functions of the package\n\n Returns:\n list of non-empty test functions' (name, function)\n\n Raises:\n ValueError: occurs if pkg is not a package class\n \"\"\"\n classes = [pkg if isinstance(pkg, type) else pkg.__class__]\n if add_virtuals:\n vpkgs = virtuals(pkg)\n for vname in vpkgs:\n try:\n classes.append(spack.repo.PATH.get_pkg_class(vname))\n except spack.repo.UnknownPackageError:\n tty.debug(f\"{vname}: virtual does not appear to have a package file\")\n\n tests = []\n for clss in classes:\n methods = inspect.getmembers(clss, predicate=lambda x: inspect.isfunction(x))\n for name, test_fn in methods:\n if not name.startswith(\"test_\"):\n continue\n\n tests.append((clss.__name__, test_fn)) # type: ignore[union-attr]\n\n return tests\n\n\ndef process_test_parts(\n pkg: \"spack.package_base.PackageBase\", test_specs: List[spack.spec.Spec], verbose: bool = False\n):\n \"\"\"Process test parts associated with the package.\n\n Args:\n pkg: package being tested\n test_specs: list of test specs\n verbose: Display verbose output (suppress by default)\n\n Raises:\n TestSuiteError: package must be part of an active test suite\n \"\"\"\n if pkg is None or pkg.test_suite is None:\n base = \"Cannot process tests\"\n msg = (\n f\"{base} without a package\"\n if pkg is None\n else f\"{pkg.name}: {base}: test suite is missing\"\n )\n raise TestSuiteError(msg)\n\n test_suite = pkg.test_suite\n tester = pkg.tester\n try:\n work_dir = test_suite.test_dir_for_spec(pkg.spec)\n for spec in test_specs:\n test_suite.current_test_spec = spec\n\n # grab test functions associated with the spec, which may be virtual\n try:\n tests = test_functions(spack.repo.PATH.get_pkg_class(spec.fullname))\n except spack.repo.UnknownPackageError:\n # Some virtuals don't have a package so we don't want to report\n # them as not having tests when that isn't appropriate.\n continue\n\n if len(tests) == 0:\n tester.status(spec.name, TestStatus.NO_TESTS)\n continue\n\n # copy custom and cached test files to the test stage directory\n copy_test_files(pkg, spec)\n\n # Run the tests\n for _, test_fn in tests:\n with test_part(\n pkg,\n test_fn.__name__,\n purpose=getattr(test_fn, \"__doc__\"),\n work_dir=work_dir,\n verbose=verbose,\n ):\n test_fn(pkg)\n\n # If fail-fast was on, we error out above\n # If we collect errors, raise them in batch here\n if tester.test_failures:\n raise TestFailure(tester.test_failures)\n\n finally:\n if tester.ran_tests():\n tester.write_tested_status()\n\n # log one more test message to provide a completion timestamp\n # for CDash reporting\n tty.msg(\"Completed testing\")\n\n lines = tester.summarize()\n tty.msg(\"\\n{}\".format(\"\\n\".join(lines)))\n\n if tester.test_failures:\n # Print the test log file path\n tty.msg(f\"\\n\\nSee test results at:\\n {tester.test_log_file}\")\n else:\n tty.msg(\"No tests to run\")\n\n\ndef test_process(pkg: \"spack.package_base.PackageBase\", kwargs):\n verbose = kwargs.get(\"verbose\", True)\n externals = kwargs.get(\"externals\", False)\n\n with pkg.tester.test_logger(verbose, externals) as logger:\n if pkg.spec.external and not externals:\n print_message(logger, \"Skipped tests for external package\", verbose)\n pkg.tester.status(pkg.spec.name, TestStatus.SKIPPED)\n return\n\n if not pkg.spec.installed:\n print_message(logger, \"Skipped not installed package\", verbose)\n pkg.tester.status(pkg.spec.name, TestStatus.SKIPPED)\n return\n\n # Make sure properly named build-time test methods actually run as\n # stand-alone tests.\n pkg.run_tests = True\n\n # run test methods from the package and all virtuals it provides\n v_names = virtuals(pkg)\n test_specs = [pkg.spec] + [spack.spec.Spec(v_name) for v_name in sorted(v_names)]\n process_test_parts(pkg, test_specs, verbose)\n\n\ndef virtuals(pkg):\n \"\"\"Return a list of unique virtuals for the package.\n\n Args:\n pkg: package of interest\n\n Returns: names of unique virtual packages\n \"\"\"\n # provided virtuals have to be deduped by name\n v_names = list({vspec.name for vspec in pkg.virtuals_provided})\n\n # hack for compilers that are not dependencies (yet)\n # TODO: this all eventually goes away\n c_names = (\"gcc\", \"intel\", \"intel-parallel-studio\")\n if pkg.name in c_names:\n v_names.extend([\"c\", \"cxx\", \"fortran\"])\n if pkg.spec.satisfies(\"llvm+clang\"):\n v_names.extend([\"c\", \"cxx\"])\n return v_names\n\n\ndef get_all_test_suites():\n \"\"\"Retrieves all validly staged TestSuites\n\n Returns:\n list: a list of TestSuite objects, which may be empty if there are none\n \"\"\"\n stage_root = get_test_stage_dir()\n if not os.path.isdir(stage_root):\n return []\n\n def valid_stage(d):\n dirpath = os.path.join(stage_root, d)\n return os.path.isdir(dirpath) and test_suite_filename in os.listdir(dirpath)\n\n candidates = [\n os.path.join(stage_root, d, test_suite_filename)\n for d in os.listdir(stage_root)\n if valid_stage(d)\n ]\n\n test_suites = [TestSuite.from_file(c) for c in candidates]\n return test_suites\n\n\ndef get_named_test_suites(name):\n \"\"\"Retrieves test suites with the provided name.\n\n Returns:\n list: a list of matching TestSuite instances, which may be empty if none\n\n Raises:\n Exception: TestSuiteNameError if no name is provided\n \"\"\"\n if not name:\n raise TestSuiteNameError(\"Test suite name is required.\")\n\n test_suites = get_all_test_suites()\n return [ts for ts in test_suites if ts.name == name]\n\n\ndef get_test_suite(name: str) -> Optional[\"TestSuite\"]:\n \"\"\"Ensure there is only one matching test suite with the provided name.\n\n Returns:\n the name if one matching test suite, else None\n\n Raises:\n TestSuiteNameError: If there are more than one matching TestSuites\n \"\"\"\n suites = get_named_test_suites(name)\n if len(suites) > 1:\n raise TestSuiteNameError(f\"Too many suites named '{name}'. May shadow hash.\")\n\n if not suites:\n return None\n return suites[0]\n\n\ndef write_test_suite_file(suite):\n \"\"\"Write the test suite to its (JSON) lock file.\"\"\"\n with open(suite.stage.join(test_suite_filename), \"w\", encoding=\"utf-8\") as f:\n sjson.dump(suite.to_dict(), stream=f)\n\n\ndef write_test_summary(counts: \"Counter\"):\n \"\"\"Write summary of the totals for each relevant status category.\n\n Args:\n counts: counts of the occurrences of relevant test status types\n \"\"\"\n summary = [f\"{n} {s.lower()}\" for s, n in counts.items() if n > 0]\n try:\n # New in Python 3.10\n total = counts.total() # type: ignore[attr-defined]\n except AttributeError:\n nums = [n for _, n in counts.items()]\n total = sum(nums)\n\n if total:\n print(\"{:=^80}\".format(\" {} of {} \".format(\", \".join(summary), plural(total, \"spec\"))))\n\n\nclass TestSuite:\n \"\"\"The class that manages specs for ``spack test run`` execution.\"\"\"\n\n def __init__(self, specs: Iterable[Spec], alias: Optional[str] = None) -> None:\n # copy so that different test suites have different package objects\n # even if they contain the same spec\n self.specs = [spec.copy() for spec in specs]\n self.current_test_spec = None # spec currently tested, can be virtual\n self.current_base_spec = None # spec currently running do_test\n\n self.alias = alias\n self._hash: Optional[str] = None\n self._stage: Optional[Prefix] = None\n\n self.counts: \"Counter\" = Counter()\n\n self.reports: List[spack.report.RequestRecord] = []\n\n @property\n def name(self) -> str:\n \"\"\"The name (alias or, if none, hash) of the test suite.\"\"\"\n return self.alias if self.alias else self.content_hash\n\n @property\n def content_hash(self) -> str:\n \"\"\"The hash used to uniquely identify the test suite.\"\"\"\n if not self._hash:\n json_text = sjson.dump(self.to_dict())\n assert json_text is not None, f\"{__name__} unexpected value for 'json_text'\"\n sha = hashlib.sha1(json_text.encode(\"utf-8\"))\n b32_hash = base64.b32encode(sha.digest()).lower()\n b32_hash = b32_hash.decode(\"utf-8\")\n self._hash = b32_hash\n return self._hash\n\n def __call__(\n self,\n *,\n remove_directory: bool = True,\n dirty: bool = False,\n fail_first: bool = False,\n externals: bool = False,\n timeout: Optional[int] = None,\n ):\n self.write_reproducibility_data()\n for spec in self.specs:\n # Setup cdash/junit/etc reports\n report = spack.report.RequestRecord(spec)\n self.reports.append(report)\n\n record = spack.report.TestRecord(spec, self.stage)\n report.append_record(record)\n record.start()\n\n try:\n if spec.package.test_suite:\n raise TestSuiteSpecError(\n f\"Package {spec.package.name} cannot be run in two test suites at once\"\n )\n\n # Set up the test suite to know which test is running\n spec.package.test_suite = self\n self.current_base_spec = spec\n self.current_test_spec = spec\n\n # setup per-test directory in the stage dir\n test_dir = self.test_dir_for_spec(spec)\n if os.path.exists(test_dir):\n shutil.rmtree(test_dir)\n fs.mkdirp(test_dir)\n\n # run the package tests\n spec.package.do_test(dirty=dirty, externals=externals, timeout=timeout)\n\n # Clean up on success\n if remove_directory:\n shutil.rmtree(test_dir)\n\n status = self.test_status(spec, externals)\n self.counts[status] += 1\n self.write_test_result(spec, status)\n record.succeed(externals)\n\n except SkipTest:\n record.skip(msg=\"Test marked to skip\")\n status = TestStatus.SKIPPED\n self.counts[status] += 1\n self.write_test_result(spec, TestStatus.SKIPPED)\n\n except BaseException as exc:\n record.fail(exc)\n\n status = TestStatus.FAILED\n self.counts[status] += 1\n tty.debug(f\"Test failure: {str(exc)}\")\n\n if isinstance(exc, (SyntaxError, TestSuiteSpecError)):\n # Create the test log file and report the error.\n self.ensure_stage()\n msg = f\"Testing package {self.test_pkg_id(spec)}\\n{str(exc)}\"\n _add_msg_to_file(self.log_file_for_spec(spec), msg)\n\n msg = f\"Test failure: {str(exc)}\"\n _add_msg_to_file(self.log_file_for_spec(spec), msg)\n self.write_test_result(spec, TestStatus.FAILED)\n if fail_first:\n break\n\n finally:\n spec.package.test_suite = None\n self.current_test_spec = None\n self.current_base_spec = None\n\n write_test_summary(self.counts)\n\n if self.counts[TestStatus.FAILED]:\n for spec in self.specs:\n print(\n \"\\nSee {} test results at:\\n {}\".format(\n spec.format(\"{name}-{version}-{hash:7}\"), self.log_file_for_spec(spec)\n )\n )\n\n failures = self.counts[TestStatus.FAILED]\n if failures:\n raise TestSuiteFailure(failures)\n\n def test_status(self, spec: spack.spec.Spec, externals: bool) -> TestStatus:\n \"\"\"Returns the overall test results status for the spec.\n\n Args:\n spec: instance of the spec under test\n externals: ``True`` if externals are to be tested, else ``False``\n \"\"\"\n tests_status_file = self.tested_file_for_spec(spec)\n if not os.path.exists(tests_status_file):\n self.ensure_stage()\n if spec.external and not externals:\n status = TestStatus.SKIPPED\n elif not spec.installed:\n status = TestStatus.SKIPPED\n else:\n status = TestStatus.NO_TESTS\n return status\n\n with open(tests_status_file, \"r\", encoding=\"utf-8\") as f:\n value = (f.read()).strip(\"\\n\")\n return TestStatus(int(value)) if value else TestStatus.NO_TESTS\n\n def ensure_stage(self) -> None:\n \"\"\"Ensure the test suite stage directory exists.\"\"\"\n if not os.path.exists(self.stage):\n fs.mkdirp(self.stage)\n\n @property\n def stage(self) -> Prefix:\n \"\"\"The root test suite stage directory\"\"\"\n if not self._stage:\n self._stage = Prefix(fs.join_path(get_test_stage_dir(), self.content_hash))\n return self._stage\n\n @stage.setter\n def stage(self, value: Union[Prefix, str]) -> None:\n \"\"\"Set the value of a non-default stage directory.\"\"\"\n self._stage = value if isinstance(value, Prefix) else Prefix(value)\n\n @property\n def results_file(self) -> Prefix:\n \"\"\"The path to the results summary file.\"\"\"\n return self.stage.join(results_filename)\n\n @classmethod\n def test_pkg_id(cls, spec: Spec) -> str:\n \"\"\"The standard install test package identifier.\n\n Args:\n spec: instance of the spec under test\n \"\"\"\n return spec.format_path(\"{name}-{version}-{hash:7}\")\n\n @classmethod\n def test_log_name(cls, spec: Spec) -> str:\n \"\"\"The standard log filename for a spec.\n\n Args:\n spec: instance of the spec under test\n \"\"\"\n return f\"{cls.test_pkg_id(spec)}-test-out.txt\"\n\n def log_file_for_spec(self, spec: Spec) -> Prefix:\n \"\"\"The test log file path for the provided spec.\n\n Args:\n spec: instance of the spec under test\n \"\"\"\n return self.stage.join(self.test_log_name(spec))\n\n def test_dir_for_spec(self, spec: Spec) -> Prefix:\n \"\"\"The path to the test stage directory for the provided spec.\n\n Args:\n spec: instance of the spec under test\n \"\"\"\n return Prefix(self.stage.join(self.test_pkg_id(spec)))\n\n @classmethod\n def tested_file_name(cls, spec: Spec) -> str:\n \"\"\"The standard test status filename for the spec.\n\n Args:\n spec: instance of the spec under test\n \"\"\"\n return \"%s-tested.txt\" % cls.test_pkg_id(spec)\n\n def tested_file_for_spec(self, spec: Spec) -> str:\n \"\"\"The test status file path for the spec.\n\n Args:\n spec: instance of the spec under test\n \"\"\"\n return fs.join_path(self.stage, self.tested_file_name(spec))\n\n @property\n def current_test_cache_dir(self) -> str:\n \"\"\"Path to the test stage directory where the current spec's cached\n build-time files were automatically copied.\n\n Raises:\n TestSuiteSpecError: If there is no spec being tested\n \"\"\"\n if not (self.current_test_spec and self.current_base_spec):\n raise TestSuiteSpecError(\"Unknown test cache directory: no specs being tested\")\n\n test_spec = self.current_test_spec\n base_spec = self.current_base_spec\n return self.test_dir_for_spec(base_spec).cache.join(test_spec.name)\n\n @property\n def current_test_data_dir(self) -> str:\n \"\"\"Path to the test stage directory where the current spec's custom\n package (data) files were automatically copied.\n\n Raises:\n TestSuiteSpecError: If there is no spec being tested\n \"\"\"\n if not (self.current_test_spec and self.current_base_spec):\n raise TestSuiteSpecError(\"Unknown test data directory: no specs being tested\")\n\n test_spec = self.current_test_spec\n base_spec = self.current_base_spec\n return self.test_dir_for_spec(base_spec).data.join(test_spec.name)\n\n def write_test_result(self, spec: Spec, result: TestStatus) -> None:\n \"\"\"Write the spec's test result to the test suite results file.\n\n Args:\n spec: instance of the spec under test\n result: result from the spec's test execution (e.g, PASSED)\n \"\"\"\n msg = f\"{self.test_pkg_id(spec)} {result}\"\n _add_msg_to_file(self.results_file, msg)\n\n def write_reproducibility_data(self) -> None:\n for spec in self.specs:\n repo_cache_path = self.stage.repo.join(spec.name)\n spack.repo.PATH.dump_provenance(spec, repo_cache_path)\n for vspec in spec.package.virtuals_provided:\n repo_cache_path = self.stage.repo.join(vspec.name)\n if not os.path.exists(repo_cache_path):\n try:\n spack.repo.PATH.dump_provenance(vspec, repo_cache_path)\n except spack.repo.UnknownPackageError:\n pass # not all virtuals have package files\n\n write_test_suite_file(self)\n\n def to_dict(self) -> Dict[str, Any]:\n \"\"\"Build a dictionary for the test suite.\n\n Returns:\n The dictionary contains entries for up to two keys.\n\n * specs: list of the test suite's specs in dictionary form\n * alias: the alias, or name, given to the test suite if provided\n \"\"\"\n specs = [s.to_dict() for s in self.specs]\n d: Dict[str, Any] = {\"specs\": specs}\n if self.alias:\n d[\"alias\"] = self.alias\n return d\n\n @staticmethod\n def from_dict(d):\n \"\"\"Instantiates a TestSuite based on a dictionary specs and an\n optional alias:\n\n * specs: list of the test suite's specs in dictionary form\n * alias: the test suite alias\n\n Returns:\n TestSuite: Instance created from the specs\n \"\"\"\n specs = [Spec.from_dict(spec_dict) for spec_dict in d[\"specs\"]]\n alias = d.get(\"alias\", None)\n return TestSuite(specs, alias)\n\n @staticmethod\n def from_file(filename: str) -> \"TestSuite\":\n \"\"\"Instantiate a TestSuite using the specs and optional alias\n provided in the given file.\n\n Args:\n filename: The path to the JSON file containing the test\n suite specs and optional alias.\n\n Raises:\n BaseException: sjson.SpackJSONError if problem parsing the file\n \"\"\"\n try:\n with open(filename, encoding=\"utf-8\") as f:\n data = sjson.load(f)\n test_suite = TestSuite.from_dict(data)\n content_hash = os.path.basename(os.path.dirname(filename))\n test_suite._hash = content_hash\n return test_suite\n except Exception as e:\n raise sjson.SpackJSONError(\"error parsing JSON TestSuite:\", e)\n\n\ndef _add_msg_to_file(filename, msg):\n \"\"\"Append the message to the specified file.\n\n Args:\n filename (str): path to the file\n msg (str): message to be appended to the file\n \"\"\"\n with open(filename, \"a+\", encoding=\"utf-8\") as f:\n f.write(f\"{msg}\\n\")\n\n\nclass SkipTest(Exception):\n \"\"\"Raised when a test (part) is being skipped.\"\"\"\n\n\nclass TestFailure(spack.error.SpackError):\n \"\"\"Raised when package tests have failed for an installation.\"\"\"\n\n def __init__(self, failures: List[TestFailureType]):\n # Failures are all exceptions\n num = len(failures)\n msg = \"{} failed.\\n\".format(plural(num, \"test\"))\n for failure, message in failures:\n msg += \"\\n\\n%s\\n\" % str(failure)\n msg += \"\\n%s\\n\" % message\n\n super().__init__(msg)\n\n\nclass TestSuiteError(spack.error.SpackError):\n \"\"\"Raised when there is an error with the test suite.\"\"\"\n\n\nclass TestSuiteFailure(spack.error.SpackError):\n \"\"\"Raised when one or more tests in a suite have failed.\"\"\"\n\n def __init__(self, num_failures):\n msg = \"%d test(s) in the suite failed.\\n\" % num_failures\n\n super().__init__(msg)\n\n\nclass TestSuiteSpecError(spack.error.SpackError):\n \"\"\"Raised when there is an issue associated with the spec being tested.\"\"\"\n\n\nclass TestSuiteNameError(spack.error.SpackError):\n \"\"\"Raised when there is an issue with the naming of the test suite.\"\"\"\n" }, { "path": "lib/spack/spack/installer.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"This module encapsulates package installation functionality.\n\nThe PackageInstaller coordinates concurrent builds of packages for the same\nSpack instance by leveraging the dependency DAG and file system locks. It\nalso proceeds with the installation of non-dependent packages of failed\ndependencies in order to install as many dependencies of a package as possible.\n\nBottom-up traversal of the dependency DAG while prioritizing packages with no\nuninstalled dependencies allows multiple processes to perform concurrent builds\nof separate packages associated with a spec.\n\nFile system locks enable coordination such that no two processes attempt to\nbuild the same or a failed dependency package.\n\nIf a dependency package fails to install, its dependents' tasks will be\nremoved from the installing process's queue. A failure file is also written\nand locked. Other processes use this file to detect the failure and dequeue\nits dependents.\n\nThis module supports the coordination of local and distributed concurrent\ninstallations of packages in a Spack instance.\n\n\"\"\"\n\nimport copy\nimport enum\nimport glob\nimport heapq\nimport io\nimport itertools\nimport os\nimport shutil\nimport sys\nimport tempfile\nimport time\nfrom collections import defaultdict\nfrom gzip import GzipFile\nfrom typing import TYPE_CHECKING, Dict, Iterator, List, Optional, Set, Tuple, Union\n\nfrom spack.vendor.typing_extensions import Literal\n\nimport spack.binary_distribution as binary_distribution\nimport spack.build_environment\nimport spack.builder\nimport spack.config\nimport spack.database\nimport spack.deptypes as dt\nimport spack.error\nimport spack.hooks\nimport spack.llnl.util.filesystem as fs\nimport spack.llnl.util.lock as lk\nimport spack.llnl.util.tty as tty\nimport spack.mirrors.mirror\nimport spack.package_base\nimport spack.package_prefs as prefs\nimport spack.repo\nimport spack.report\nimport spack.rewiring\nimport spack.store\nimport spack.util.path\nimport spack.util.timer as timer\nfrom spack.llnl.string import ordinal\nfrom spack.llnl.util.lang import pretty_seconds\nfrom spack.llnl.util.tty.color import colorize\nfrom spack.llnl.util.tty.log import log_output, preserve_terminal_settings\nfrom spack.url_buildcache import BuildcacheEntryError\nfrom spack.util.environment import EnvironmentModifications, dump_environment\n\nif TYPE_CHECKING:\n import spack.spec\n\n#: Counter to support unique spec sequencing that is used to ensure packages\n#: with the same priority are (initially) processed in the order in which they\n#: were added (see https://docs.python.org/2/library/heapq.html).\n_counter = itertools.count(0)\n\n_FAIL_FAST_ERR = \"Terminating after first install failure\"\n\n#: Type for specifying installation source modes\nInstallPolicy = Literal[\"auto\", \"cache_only\", \"source_only\"]\n\n\nclass BuildStatus(enum.Enum):\n \"\"\"Different build (task) states.\"\"\"\n\n #: Build status indicating task has been added/queued.\n QUEUED = enum.auto()\n\n #: Build status indicating the spec failed to install\n FAILED = enum.auto()\n\n #: Build status indicating the spec is being installed (possibly by another\n #: process)\n INSTALLING = enum.auto()\n\n #: Build status indicating the spec was successfully installed\n INSTALLED = enum.auto()\n\n #: Build status indicating the task has been popped from the queue\n DEQUEUED = enum.auto()\n\n #: Build status indicating task has been removed (to maintain priority\n #: queue invariants).\n REMOVED = enum.auto()\n\n def __str__(self):\n return f\"{self.name.lower()}\"\n\n\ndef _write_timer_json(pkg, timer, cache):\n extra_attributes = {\"name\": pkg.name, \"cache\": cache, \"hash\": pkg.spec.dag_hash()}\n try:\n with open(pkg.times_log_path, \"w\", encoding=\"utf-8\") as timelog:\n timer.write_json(timelog, extra_attributes=extra_attributes)\n except Exception as e:\n tty.debug(str(e))\n return\n\n\nclass ExecuteResult(enum.Enum):\n # Task succeeded\n SUCCESS = enum.auto()\n # Task failed\n FAILED = enum.auto()\n # Task is missing build spec and will be requeued\n MISSING_BUILD_SPEC = enum.auto()\n # Task is installed upstream/external or\n # task is not ready for installation (locked by another process)\n NO_OP = enum.auto()\n\n\nclass InstallAction(enum.Enum):\n #: Don't perform an install\n NONE = enum.auto()\n #: Do a standard install\n INSTALL = enum.auto()\n #: Do an overwrite install\n OVERWRITE = enum.auto()\n\n\nclass InstallStatus:\n def __init__(self, pkg_count: int):\n # Counters used for showing status information\n self.pkg_num: int = 0\n self.pkg_count: int = pkg_count\n self.pkg_ids: Set[str] = set()\n\n def next_pkg(self, pkg: \"spack.package_base.PackageBase\"):\n pkg_id = package_id(pkg.spec)\n\n if pkg_id not in self.pkg_ids:\n self.pkg_num += 1\n self.pkg_ids.add(pkg_id)\n\n def set_term_title(self, text: str):\n if not spack.config.get(\"config:install_status\", True):\n return\n\n if not sys.stdout.isatty():\n return\n\n status = f\"{text} {self.get_progress()}\"\n sys.stdout.write(f\"\\x1b]0;Spack: {status}\\x07\")\n sys.stdout.flush()\n\n def get_progress(self) -> str:\n return f\"[{self.pkg_num}/{self.pkg_count}]\"\n\n\nclass TermStatusLine:\n \"\"\"\n This class is used in distributed builds to inform the user that other packages are\n being installed by another process.\n \"\"\"\n\n def __init__(self, enabled: bool):\n self.enabled: bool = enabled\n self.pkg_set: Set[str] = set()\n self.pkg_list: List[str] = []\n\n def add(self, pkg_id: str):\n \"\"\"Add a package to the waiting list, and if it is new, update the status line.\"\"\"\n if not self.enabled or pkg_id in self.pkg_set:\n return\n\n self.pkg_set.add(pkg_id)\n self.pkg_list.append(pkg_id)\n tty.msg(colorize(\"@*{Waiting for} @*g{%s}\" % pkg_id))\n sys.stdout.flush()\n\n def clear(self):\n \"\"\"Clear the status line.\"\"\"\n if not self.enabled:\n return\n\n lines = len(self.pkg_list)\n\n if lines == 0:\n return\n\n self.pkg_set.clear()\n self.pkg_list = []\n\n # Move the cursor to the beginning of the first \"Waiting for\" message and clear\n # everything after it.\n sys.stdout.write(f\"\\x1b[{lines}F\\x1b[J\")\n sys.stdout.flush()\n\n\ndef _check_last_phase(pkg: \"spack.package_base.PackageBase\") -> None:\n \"\"\"\n Ensures the specified package has a valid last phase before proceeding\n with its installation.\n\n The last phase is also set to None if it is the last phase of the\n package already.\n\n Args:\n pkg: the package being installed\n\n Raises:\n ``BadInstallPhase`` if stop_before or last phase is invalid\n \"\"\"\n phases = spack.builder.create(pkg).phases # type: ignore[attr-defined]\n if pkg.stop_before_phase and pkg.stop_before_phase not in phases: # type: ignore[attr-defined]\n raise BadInstallPhase(pkg.name, pkg.stop_before_phase) # type: ignore[attr-defined]\n\n if pkg.last_phase and pkg.last_phase not in phases: # type: ignore[attr-defined]\n raise BadInstallPhase(pkg.name, pkg.last_phase) # type: ignore[attr-defined]\n\n\ndef _handle_external_and_upstream(pkg: \"spack.package_base.PackageBase\", explicit: bool) -> bool:\n \"\"\"\n Determine if the package is external or upstream and register it in the\n database if it is external package.\n\n Args:\n pkg: the package whose installation is under consideration\n explicit: the package was explicitly requested by the user\n Return:\n ``True`` if the package is not to be installed locally, otherwise ``False``\n \"\"\"\n # For external packages the workflow is simplified, and basically\n # consists in module file generation and registration in the DB.\n if pkg.spec.external:\n _process_external_package(pkg, explicit)\n _print_installed_pkg(f\"{pkg.prefix} (external {package_id(pkg.spec)})\")\n return True\n\n if pkg.spec.installed_upstream:\n tty.verbose(\n f\"{package_id(pkg.spec)} is installed in an upstream Spack instance at \"\n f\"{pkg.spec.prefix}\"\n )\n _print_installed_pkg(pkg.prefix)\n\n # This will result in skipping all post-install hooks. In the case\n # of modules this is considered correct because we want to retrieve\n # the module from the upstream Spack instance.\n return True\n\n return False\n\n\ndef _do_fake_install(pkg: \"spack.package_base.PackageBase\") -> None:\n \"\"\"Make a fake install directory with fake executables, headers, and libraries.\"\"\"\n command = pkg.name\n header = pkg.name\n library = pkg.name\n\n # Avoid double 'lib' for packages whose names already start with lib\n if not pkg.name.startswith(\"lib\"):\n library = \"lib\" + library\n\n plat_shared = \".dll\" if sys.platform == \"win32\" else \".so\"\n plat_static = \".lib\" if sys.platform == \"win32\" else \".a\"\n dso_suffix = \".dylib\" if sys.platform == \"darwin\" else plat_shared\n\n # Install fake command\n fs.mkdirp(pkg.prefix.bin)\n executable = lambda path, flags: os.open(path, flags, 0o700)\n open(os.path.join(pkg.prefix.bin, command), \"wb\", opener=executable).close()\n\n # Install fake header file\n fs.mkdirp(pkg.prefix.include)\n fs.touch(os.path.join(pkg.prefix.include, header + \".h\"))\n\n # Install fake shared and static libraries\n fs.mkdirp(pkg.prefix.lib)\n for suffix in [dso_suffix, plat_static]:\n fs.touch(os.path.join(pkg.prefix.lib, library + suffix))\n\n # Install fake man page\n fs.mkdirp(pkg.prefix.man.man1)\n\n packages_dir = spack.store.STORE.layout.build_packages_path(pkg.spec)\n dump_packages(pkg.spec, packages_dir)\n\n\ndef _hms(seconds: int) -> str:\n \"\"\"\n Convert seconds to hours, minutes, seconds\n\n Args:\n seconds: time to be converted in seconds\n\n Return: String representation of the time as #h #m #.##s\n \"\"\"\n m, s = divmod(seconds, 60)\n h, m = divmod(m, 60)\n\n parts = []\n if h:\n parts.append(\"%dh\" % h)\n if m:\n parts.append(\"%dm\" % m)\n if s:\n parts.append(f\"{s:.2f}s\")\n return \" \".join(parts)\n\n\ndef _log_prefix(pkg_name) -> str:\n \"\"\"Prefix of the form \"[pid]: [pkg name]: ...\" when printing a status update during\n the build.\"\"\"\n pid = f\"{os.getpid()}: \" if tty.show_pid() else \"\"\n return f\"{pid}{pkg_name}:\"\n\n\ndef _print_installed_pkg(message: str) -> None:\n \"\"\"\n Output a message with a package icon.\n\n Args:\n message (str): message to be output\n \"\"\"\n if tty.msg_enabled():\n print(colorize(\"@*g{[+]} \") + spack.util.path.debug_padded_filter(message))\n\n\ndef print_install_test_log(pkg: \"spack.package_base.PackageBase\") -> None:\n \"\"\"Output install test log file path but only if have test failures.\n\n Args:\n pkg: instance of the package under test\n \"\"\"\n if not pkg.run_tests or not (pkg.tester and pkg.tester.test_failures):\n # The tests were not run or there were no test failures\n return\n\n pkg.tester.print_log_path()\n\n\ndef _print_timer(pre: str, pkg_id: str, timer: timer.BaseTimer) -> None:\n phases = [f\"{p.capitalize()}: {_hms(timer.duration(p))}.\" for p in timer.phases]\n phases.append(f\"Total: {_hms(timer.duration())}\")\n tty.msg(f\"{pre} Successfully installed {pkg_id}\", \" \".join(phases))\n\n\ndef _install_from_cache(\n pkg: \"spack.package_base.PackageBase\", explicit: bool, unsigned: Optional[bool] = False\n) -> bool:\n \"\"\"\n Install the package from binary cache\n\n Args:\n pkg: package to install from the binary cache\n explicit: ``True`` if installing the package was explicitly\n requested by the user, otherwise, ``False``\n unsigned: if ``True`` or ``False`` override the mirror signature verification defaults\n\n Return: ``True`` if the package was extract from binary cache, ``False`` otherwise\n \"\"\"\n t = timer.Timer()\n installed_from_cache = _try_install_from_binary_cache(\n pkg, explicit, unsigned=unsigned, timer=t\n )\n if not installed_from_cache:\n return False\n t.stop()\n\n pkg_id = package_id(pkg.spec)\n tty.debug(f\"Successfully extracted {pkg_id} from binary cache\")\n\n _write_timer_json(pkg, t, True)\n _print_timer(pre=_log_prefix(pkg.name), pkg_id=pkg_id, timer=t)\n _print_installed_pkg(pkg.spec.prefix)\n spack.hooks.post_install(pkg.spec, explicit)\n return True\n\n\ndef _process_external_package(pkg: \"spack.package_base.PackageBase\", explicit: bool) -> None:\n \"\"\"\n Helper function to run post install hooks and register external packages.\n\n Args:\n pkg: the external package\n explicit: if the package was requested explicitly by the user,\n ``False`` if it was pulled in as a dependency of an explicit\n package.\n \"\"\"\n assert pkg.spec.external, \"Expected to post-install/register an external package.\"\n\n pre = f\"{pkg.spec.name}@{pkg.spec.version} :\"\n spec = pkg.spec\n\n if spec.external_modules:\n tty.msg(f\"{pre} has external module in {spec.external_modules}\")\n tty.debug(f\"{pre} is actually installed in {spec.external_path}\")\n else:\n tty.debug(f\"{pre} externally installed in {spec.external_path}\")\n\n try:\n # Check if the package was already registered in the DB.\n # If this is the case, then only make explicit if required.\n tty.debug(f\"{pre} already registered in DB\")\n record = spack.store.STORE.db.get_record(spec)\n if explicit and not record.explicit:\n spack.store.STORE.db.mark(spec, \"explicit\", True)\n\n except KeyError:\n # If not, register it and generate the module file.\n # For external packages we just need to run\n # post-install hooks to generate module files.\n tty.debug(f\"{pre} generating module file\")\n spack.hooks.post_install(spec, explicit)\n\n # Add to the DB\n tty.debug(f\"{pre} registering into DB\")\n spack.store.STORE.db.add(spec, explicit=explicit)\n\n\ndef _process_binary_cache_tarball(\n pkg: \"spack.package_base.PackageBase\",\n explicit: bool,\n unsigned: Optional[bool],\n mirrors_for_spec: Optional[list] = None,\n timer: timer.BaseTimer = timer.NULL_TIMER,\n) -> bool:\n \"\"\"\n Process the binary cache tarball.\n\n Args:\n pkg: the package being installed\n explicit: the package was explicitly requested by the user\n unsigned: if ``True`` or ``False`` override the mirror signature verification defaults\n mirrors_for_spec: Optional list of mirrors to look for the spec.\n obtained by calling binary_distribution.get_mirrors_for_spec().\n timer: timer to keep track of binary install phases.\n\n Return:\n bool: ``True`` if the package was extracted from binary cache,\n else ``False``\n \"\"\"\n with timer.measure(\"fetch\"):\n tarball_stage = binary_distribution.download_tarball(\n pkg.spec.build_spec, unsigned, mirrors_for_spec\n )\n\n if tarball_stage is None:\n return False\n\n tty.msg(f\"Extracting {package_id(pkg.spec)} from binary cache\")\n\n with timer.measure(\"install\"), spack.util.path.filter_padding():\n binary_distribution.extract_tarball(pkg.spec, tarball_stage, force=False, timer=timer)\n\n if pkg.spec.spliced: # overwrite old metadata with new\n spack.store.STORE.layout.write_spec(\n pkg.spec, spack.store.STORE.layout.spec_file_path(pkg.spec)\n )\n\n if hasattr(pkg, \"_post_buildcache_install_hook\"):\n pkg._post_buildcache_install_hook()\n\n pkg.installed_from_binary_cache = True\n spack.store.STORE.db.add(pkg.spec, explicit=explicit)\n return True\n\n\ndef _try_install_from_binary_cache(\n pkg: \"spack.package_base.PackageBase\",\n explicit: bool,\n unsigned: Optional[bool] = None,\n timer: timer.BaseTimer = timer.NULL_TIMER,\n) -> bool:\n \"\"\"\n Try to extract the package from binary cache.\n\n Args:\n pkg: package to be extracted from binary cache\n explicit: the package was explicitly requested by the user\n unsigned: if ``True`` or ``False`` override the mirror signature verification defaults\n timer: timer to keep track of binary install phases.\n \"\"\"\n # Early exit if no binary mirrors are configured.\n if not spack.mirrors.mirror.MirrorCollection(binary=True):\n return False\n\n tty.debug(f\"Searching for binary cache of {package_id(pkg.spec)}\")\n\n with timer.measure(\"search\"):\n mirrors = binary_distribution.get_mirrors_for_spec(pkg.spec, index_only=True)\n\n return _process_binary_cache_tarball(\n pkg, explicit, unsigned, mirrors_for_spec=mirrors, timer=timer\n )\n\n\ndef combine_phase_logs(phase_log_files: List[str], log_path: str) -> None:\n \"\"\"\n Read set or list of logs and combine them into one file.\n\n Each phase will produce it's own log, so this function aims to cat all the\n separate phase log output files into the pkg.log_path. It is written\n generally to accept some list of files, and a log path to combine them to.\n\n Args:\n phase_log_files: a list or iterator of logs to combine\n log_path: the path to combine them to\n \"\"\"\n with open(log_path, \"bw\") as log_file:\n for phase_log_file in phase_log_files:\n with open(phase_log_file, \"br\") as phase_log:\n shutil.copyfileobj(phase_log, log_file)\n\n\ndef dump_packages(spec: \"spack.spec.Spec\", path: str) -> None:\n \"\"\"\n Dump all package information for a spec and its dependencies.\n\n This creates a package repository within path for every namespace in the\n spec DAG, and fills the repos with package files and patch files for every\n node in the DAG.\n\n Args:\n spec: the Spack spec whose package information is to be dumped\n path: the path to the build packages directory\n \"\"\"\n fs.mkdirp(path)\n\n # Copy in package.py files from any dependencies.\n # Note that we copy them in as they are in the *install* directory\n # NOT as they are in the repository, because we want a snapshot of\n # how *this* particular build was done.\n for node in spec.traverse(deptype=\"all\"):\n if node is not spec:\n # Locate the dependency package in the install tree and find\n # its provenance information.\n source = spack.store.STORE.layout.build_packages_path(node)\n source_repo_root = os.path.join(source, node.namespace)\n\n # If there's no provenance installed for the package, skip it.\n # If it's external, skip it because it either:\n # 1) it wasn't built with Spack, so it has no Spack metadata\n # 2) it was built by another Spack instance, and we do not\n # (currently) use Spack metadata to associate repos with externals\n # built by other Spack instances.\n # Spack can always get something current from the builtin repo.\n if node.external or not os.path.isdir(source_repo_root):\n continue\n\n # Create a source repo and get the pkg directory out of it.\n try:\n source_repo = spack.repo.from_path(source_repo_root)\n source_pkg_dir = source_repo.dirname_for_package_name(node.name)\n except spack.repo.RepoError as err:\n tty.debug(f\"Failed to create source repo for {node.name}: {str(err)}\")\n source_pkg_dir = None\n tty.warn(f\"Warning: Couldn't copy in provenance for {node.name}\")\n\n # Create a destination repository\n pkg_api = spack.repo.PATH.get_repo(node.namespace).package_api\n repo_root = os.path.join(path, node.namespace) if pkg_api < (2, 0) else path\n repo = spack.repo.create_or_construct(\n repo_root, namespace=node.namespace, package_api=pkg_api\n )\n\n # Get the location of the package in the dest repo.\n dest_pkg_dir = repo.dirname_for_package_name(node.name)\n if node is spec:\n spack.repo.PATH.dump_provenance(node, dest_pkg_dir)\n elif source_pkg_dir:\n fs.install_tree(source_pkg_dir, dest_pkg_dir)\n\n\ndef get_dependent_ids(spec: \"spack.spec.Spec\") -> List[str]:\n \"\"\"\n Return a list of package ids for the spec's dependents\n\n Args:\n spec: Concretized spec\n\n Returns: list of package ids\n \"\"\"\n return [package_id(d) for d in spec.dependents()]\n\n\ndef install_msg(name: str, pid: int, install_status: InstallStatus) -> str:\n \"\"\"\n Colorize the name/id of the package being installed\n\n Args:\n name: Name/id of the package being installed\n pid: id of the installer process\n\n Return: Colorized installing message\n \"\"\"\n pre = f\"{pid}: \" if tty.show_pid() else \"\"\n post = (\n \" @*{%s}\" % install_status.get_progress()\n if install_status and spack.config.get(\"config:install_status\", True)\n else \"\"\n )\n return pre + colorize(\"@*{Installing} @*g{%s}%s\" % (name, post))\n\n\ndef archive_install_logs(pkg: \"spack.package_base.PackageBase\", phase_log_dir: str) -> None:\n \"\"\"\n Copy install logs to their destination directory(ies)\n Args:\n pkg: the package that was built and installed\n phase_log_dir: path to the archive directory\n \"\"\"\n # Copy a compressed version of the install log\n with open(pkg.log_path, \"rb\") as f, open(pkg.install_log_path, \"wb\") as g:\n # Use GzipFile directly so we can omit filename / mtime in header\n gzip_file = GzipFile(filename=\"\", mode=\"wb\", compresslevel=6, mtime=0, fileobj=g)\n shutil.copyfileobj(f, gzip_file)\n gzip_file.close()\n\n # Archive the install-phase test log, if present\n pkg.archive_install_test_log()\n\n\ndef log(pkg: \"spack.package_base.PackageBase\") -> None:\n \"\"\"\n Copy provenance into the install directory on success\n\n Args:\n pkg: the package that was built and installed\n \"\"\"\n packages_dir = spack.store.STORE.layout.build_packages_path(pkg.spec)\n\n # Remove first if we're overwriting another build\n try:\n # log and env install paths are inside this\n shutil.rmtree(packages_dir)\n except Exception as e:\n # FIXME : this potentially catches too many things...\n tty.debug(e)\n\n archive_install_logs(pkg, os.path.dirname(packages_dir))\n\n # Archive the environment modifications for the build.\n fs.install(pkg.env_mods_path, pkg.install_env_path)\n\n if os.path.exists(pkg.configure_args_path):\n # Archive the args used for the build\n fs.install(pkg.configure_args_path, pkg.install_configure_args_path)\n\n # Finally, archive files that are specific to each package\n with fs.working_dir(pkg.stage.path):\n errors = io.StringIO()\n target_dir = os.path.join(\n spack.store.STORE.layout.metadata_path(pkg.spec), \"archived-files\"\n )\n\n for glob_expr in spack.builder.create(pkg).archive_files:\n # Check that we are trying to copy things that are\n # in the stage tree (not arbitrary files)\n abs_expr = os.path.realpath(glob_expr)\n if os.path.realpath(pkg.stage.path) not in abs_expr:\n errors.write(f\"[OUTSIDE SOURCE PATH]: {glob_expr}\\n\")\n continue\n # Now that we are sure that the path is within the correct\n # folder, make it relative and check for matches\n if os.path.isabs(glob_expr):\n glob_expr = os.path.relpath(glob_expr, pkg.stage.path)\n files = glob.glob(glob_expr)\n for f in files:\n try:\n target = os.path.join(target_dir, f)\n # We must ensure that the directory exists before\n # copying a file in\n fs.mkdirp(os.path.dirname(target))\n fs.install(f, target)\n except Exception as e:\n tty.debug(e)\n\n # Here try to be conservative, and avoid discarding\n # the whole install procedure because of copying a\n # single file failed\n errors.write(f\"[FAILED TO ARCHIVE]: {f}\")\n\n if errors.getvalue():\n error_file = os.path.join(target_dir, \"errors.txt\")\n fs.mkdirp(target_dir)\n with open(error_file, \"w\", encoding=\"utf-8\") as err:\n err.write(errors.getvalue())\n tty.warn(f\"Errors occurred when archiving files.\\n\\tSee: {error_file}\")\n\n dump_packages(pkg.spec, packages_dir)\n\n\ndef package_id(spec: \"spack.spec.Spec\") -> str:\n \"\"\"A \"unique\" package identifier for installation purposes\n\n The identifier is used to track tasks, locks, install, and\n failure statuses.\n\n The identifier needs to distinguish between combinations of compilers\n and packages for combinatorial environments.\n\n Args:\n pkg: the package from which the identifier is derived\n \"\"\"\n if not spec.concrete:\n raise ValueError(\"Cannot provide a unique, readable id when the spec is not concretized.\")\n\n return f\"{spec.name}-{spec.version}-{spec.dag_hash()}\"\n\n\nclass BuildRequest:\n \"\"\"Class for representing an installation request.\"\"\"\n\n def __init__(self, pkg: \"spack.package_base.PackageBase\", install_args: dict):\n \"\"\"\n Instantiate a build request for a package.\n\n Args:\n pkg: the package to be built and installed\n install_args: the install arguments associated with ``pkg``\n \"\"\"\n # Ensure dealing with a package that has a concrete spec\n if not isinstance(pkg, spack.package_base.PackageBase):\n raise ValueError(f\"{str(pkg)} must be a package\")\n\n self.pkg = pkg\n if not self.pkg.spec.concrete:\n raise ValueError(f\"{self.pkg.name} must have a concrete spec\")\n\n self.pkg.stop_before_phase = install_args.get(\"stop_before\") # type: ignore[attr-defined] # noqa: E501\n self.pkg.last_phase = install_args.get(\"stop_at\") # type: ignore[attr-defined]\n\n # Cache the package id for convenience\n self.pkg_id = package_id(pkg.spec)\n\n # Save off the original install arguments plus standard defaults\n # since they apply to the requested package *and* dependencies.\n self.install_args = install_args if install_args else {}\n self._add_default_args()\n\n # Cache overwrite information\n self.overwrite = set(self.install_args.get(\"overwrite\", []))\n self.overwrite_time = time.time()\n\n # Save off dependency package ids for quick checks since traversals\n # are not able to return full dependents for all packages across\n # environment specs.\n self.dependencies = set(\n package_id(d)\n for d in self.pkg.spec.dependencies(deptype=self.get_depflags(self.pkg))\n if package_id(d) != self.pkg_id\n )\n\n def __repr__(self) -> str:\n \"\"\"Return a formal representation of the build request.\"\"\"\n rep = f\"{self.__class__.__name__}(\"\n for attr, value in self.__dict__.items():\n rep += f\"{attr}={value.__repr__()}, \"\n return f\"{rep.strip(', ')})\"\n\n def __str__(self) -> str:\n \"\"\"Return a printable version of the build request.\"\"\"\n return f\"package={self.pkg.name}, install_args={self.install_args}\"\n\n def _add_default_args(self) -> None:\n \"\"\"Ensure standard install options are set to at least the default.\"\"\"\n for arg, default in [\n (\"context\", \"build\"), # installs *always* build\n (\"dependencies_policy\", \"auto\"),\n (\"dirty\", False),\n (\"fail_fast\", False),\n (\"fake\", False),\n (\"install_deps\", True),\n (\"install_package\", True),\n (\"install_source\", False),\n (\"root_policy\", \"auto\"),\n (\"keep_prefix\", False),\n (\"keep_stage\", False),\n (\"restage\", False),\n (\"skip_patch\", False),\n (\"tests\", False),\n (\"unsigned\", None),\n (\"verbose\", False),\n ]:\n _ = self.install_args.setdefault(arg, default)\n\n def get_depflags(self, pkg: \"spack.package_base.PackageBase\") -> int:\n \"\"\"Determine the required dependency types for the associated package.\n\n Args:\n pkg: explicit or implicit package being installed\n\n Returns:\n tuple: required dependency type(s) for the package\n \"\"\"\n depflag = dt.LINK | dt.RUN\n include_build_deps = self.install_args.get(\"include_build_deps\")\n\n if self.pkg_id == package_id(pkg.spec):\n policy = self.install_args.get(\"root_policy\", \"auto\")\n else:\n policy = self.install_args.get(\"dependencies_policy\", \"auto\")\n\n # Include build dependencies if pkg is going to be built from sources, or\n # if build deps are explicitly requested.\n if include_build_deps or not (\n policy == \"cache_only\"\n or pkg.spec.installed\n and pkg.spec.dag_hash() not in self.overwrite\n ):\n depflag |= dt.BUILD\n if self.run_tests(pkg):\n depflag |= dt.TEST\n return depflag\n\n def has_dependency(self, dep_id) -> bool:\n \"\"\"Returns ``True`` if the package id represents a known dependency\n of the requested package, ``False`` otherwise.\"\"\"\n return dep_id in self.dependencies\n\n def run_tests(self, pkg: \"spack.package_base.PackageBase\") -> bool:\n \"\"\"Determine if the tests should be run for the provided packages\n\n Args:\n pkg: explicit or implicit package being installed\n\n Returns:\n bool: ``True`` if they should be run; ``False`` otherwise\n \"\"\"\n tests = self.install_args.get(\"tests\", False)\n return tests is True or (tests and pkg.name in tests)\n\n @property\n def spec(self) -> \"spack.spec.Spec\":\n \"\"\"The specification associated with the package.\"\"\"\n return self.pkg.spec\n\n def traverse_dependencies(self, spec=None, visited=None) -> Iterator[\"spack.spec.Spec\"]:\n \"\"\"Yield any dependencies of the appropriate type(s)\"\"\"\n # notice: deptype is not constant across nodes, so we cannot use\n # spec.traverse_edges(deptype=...).\n\n if spec is None:\n spec = self.spec\n if visited is None:\n visited = set()\n\n for dep in spec.dependencies(deptype=self.get_depflags(spec.package)):\n hash = dep.dag_hash()\n if hash in visited:\n continue\n visited.add(hash)\n # In Python 3: yield from self.traverse_dependencies(dep, visited)\n for s in self.traverse_dependencies(dep, visited):\n yield s\n yield dep\n\n\nclass Task:\n \"\"\"Base class for representing a task for a package.\"\"\"\n\n success_result: Optional[ExecuteResult] = None\n error_result: Optional[BaseException] = None\n no_op: bool = False\n\n def __init__(\n self,\n pkg: \"spack.package_base.PackageBase\",\n request: BuildRequest,\n *,\n compiler: bool = False,\n start_time: float = 0.0,\n attempts: int = 0,\n status: BuildStatus = BuildStatus.QUEUED,\n installed: Set[str] = set(),\n ):\n \"\"\"\n Instantiate a task for a package.\n\n Args:\n pkg: the package to be built and installed\n request: the associated install request\n start_time: the initial start time for the package, in seconds\n attempts: the number of attempts to install the package, which\n should be 0 when the task is initially instantiated\n status: the installation status\n installed: the (string) identifiers of packages that have\n been installed so far\n\n Raises:\n ``InstallError`` if the build status is incompatible with the task\n ``TypeError`` if provided an argument of the wrong type\n ``ValueError`` if provided an argument with the wrong value or state\n \"\"\"\n\n # Ensure dealing with a package that has a concrete spec\n if not isinstance(pkg, spack.package_base.PackageBase):\n raise TypeError(f\"{str(pkg)} must be a package\")\n\n self.pkg = pkg\n if not self.pkg.spec.concrete:\n raise ValueError(f\"{self.pkg.name} must have a concrete spec\")\n\n # The \"unique\" identifier for the task's package\n self.pkg_id = package_id(self.pkg.spec)\n\n # The explicit build request associated with the package\n if not isinstance(request, BuildRequest):\n raise TypeError(f\"{request} is not a valid build request\")\n self.request = request\n\n # Report for tracking install success/failure\n record_cls = self.request.install_args.get(\"record_cls\", spack.report.InstallRecord)\n self.record = record_cls(self.pkg.spec)\n\n # Initialize the status to an active state. The status is used to\n # ensure priority queue invariants when tasks are \"removed\" from the\n # queue.\n if not isinstance(status, BuildStatus):\n raise TypeError(f\"{status} is not a valid build status\")\n\n # The initial build task cannot have status \"removed\".\n if attempts == 0 and status == BuildStatus.REMOVED:\n raise spack.error.InstallError(\n f\"Cannot create a task for {self.pkg_id} with status '{status}'\", pkg=pkg\n )\n self.status = status\n\n # cache the PID, which is used for distributed build messages in self.execute\n self.pid = os.getpid()\n\n # The initial start time for processing the spec\n self.start_time = start_time\n\n if not isinstance(installed, set):\n raise TypeError(\n f\"BuildTask constructor requires 'installed' be a 'set', \"\n f\"not '{installed.__class__.__name__}'.\"\n )\n\n # Set of dependents, which needs to include the requesting package\n # to support tracking of parallel, multi-spec, environment installs.\n self.dependents = set(get_dependent_ids(self.pkg.spec))\n\n tty.debug(f\"Pkg id {self.pkg_id} has the following dependents:\")\n for dep_id in self.dependents:\n tty.debug(f\"- {dep_id}\")\n\n # Set of dependencies\n #\n # Be consistent wrt use of dependents and dependencies. That is,\n # if use traverse for transitive dependencies, then must remove\n # transitive dependents on failure.\n self.dependencies = set(\n package_id(d)\n for d in self.pkg.spec.dependencies(deptype=self.request.get_depflags(self.pkg))\n if package_id(d) != self.pkg_id\n )\n\n # List of uninstalled dependencies, which is used to establish\n # the priority of the task.\n self.uninstalled_deps = set(\n pkg_id for pkg_id in self.dependencies if pkg_id not in installed\n )\n\n # Ensure key sequence-related properties are updated accordingly.\n self.attempts = attempts\n self._update()\n\n # initialize cache variables\n self._install_action = None\n\n def start(self):\n \"\"\"Start the work of this task.\"\"\"\n raise NotImplementedError\n\n def poll(self) -> bool:\n \"\"\"Check if child process has information ready to receive.\"\"\"\n raise NotImplementedError\n\n def complete(self) -> ExecuteResult:\n \"\"\"Complete the work of this task.\"\"\"\n raise NotImplementedError\n\n def __eq__(self, other):\n return self.key == other.key\n\n def __ge__(self, other):\n return self.key >= other.key\n\n def __gt__(self, other):\n return self.key > other.key\n\n def __le__(self, other):\n return self.key <= other.key\n\n def __lt__(self, other):\n return self.key < other.key\n\n def __ne__(self, other):\n return self.key != other.key\n\n def __repr__(self) -> str:\n \"\"\"Returns a formal representation of the task.\"\"\"\n rep = f\"{self.__class__.__name__}(\"\n for attr, value in self.__dict__.items():\n rep += f\"{attr}={value.__repr__()}, \"\n return f\"{rep.strip(', ')})\"\n\n def __str__(self) -> str:\n \"\"\"Returns a printable version of the task.\"\"\"\n dependencies = f\"#dependencies={len(self.dependencies)}\"\n return \"priority={0}, status={1}, start_time={2}, {3}\".format(\n self.priority, self.status, self.start_time, dependencies\n )\n\n def _update(self) -> None:\n \"\"\"Update properties associated with a new instance of a task.\"\"\"\n # Number of times the task has/will be queued\n self.attempts = self.attempts + 1\n\n # Ensure the task gets a unique sequence number to preserve the\n # order in which it is added.\n self.sequence = next(_counter)\n\n def add_dependent(self, pkg_id: str) -> None:\n \"\"\"\n Ensure the package is in this task's ``dependents`` list.\n\n Args:\n pkg_id: package identifier of the dependent package\n \"\"\"\n if pkg_id != self.pkg_id and pkg_id not in self.dependents:\n tty.debug(f\"Adding {pkg_id} as a dependent of {self.pkg_id}\")\n self.dependents.add(pkg_id)\n\n def add_dependency(self, pkg_id, installed=False):\n \"\"\"\n Ensure the package is in this task's ``dependencies`` list.\n\n Args:\n pkg_id (str): package identifier of the dependency package\n installed (bool): install status of the dependency package\n \"\"\"\n if pkg_id != self.pkg_id and pkg_id not in self.dependencies:\n tty.debug(f\"Adding {pkg_id} as a dependency of {self.pkg_id}\")\n self.dependencies.add(pkg_id)\n if not installed:\n self.uninstalled_deps.add(pkg_id)\n\n def flag_installed(self, installed: List[str]) -> None:\n \"\"\"\n Ensure the dependency is not considered to still be uninstalled.\n\n Args:\n installed: the identifiers of packages that have been installed so far\n \"\"\"\n now_installed = self.uninstalled_deps & set(installed)\n for pkg_id in now_installed:\n self.uninstalled_deps.remove(pkg_id)\n tty.debug(\n f\"{self.pkg_id}: Removed {pkg_id} from uninstalled deps list: \"\n f\"{self.uninstalled_deps}\",\n level=2,\n )\n\n def _setup_install_dir(self, pkg: \"spack.package_base.PackageBase\") -> None:\n \"\"\"\n Create and ensure proper access controls for the install directory.\n Write a small metadata file with the current spack environment.\n\n Args:\n pkg: the package to be built and installed\n \"\"\"\n # Move to a module level method.\n if not os.path.exists(pkg.spec.prefix):\n path = spack.util.path.debug_padded_filter(pkg.spec.prefix)\n tty.debug(f\"Creating the installation directory {path}\")\n spack.store.STORE.layout.create_install_directory(pkg.spec)\n else:\n # Set the proper group for the prefix\n group = prefs.get_package_group(pkg.spec)\n if group:\n fs.chgrp(pkg.spec.prefix, group)\n\n # Set the proper permissions.\n # This has to be done after group because changing groups blows\n # away the sticky group bit on the directory\n mode = os.stat(pkg.spec.prefix).st_mode\n perms = prefs.get_package_dir_permissions(pkg.spec)\n if mode != perms:\n os.chmod(pkg.spec.prefix, perms)\n\n # Ensure the metadata path exists as well\n fs.mkdirp(spack.store.STORE.layout.metadata_path(pkg.spec), mode=perms)\n\n # Always write host environment - we assume this can change\n spack.store.STORE.layout.write_host_environment(pkg.spec)\n\n @property\n def install_action(self):\n if not self._install_action:\n self._install_action = self.get_install_action()\n return self._install_action\n\n def get_install_action(self: \"Task\") -> InstallAction:\n \"\"\"\n Determine whether the installation should be overwritten (if it already\n exists) or skipped (if has been handled by another process).\n\n If the package has not been installed yet, this will indicate that the\n installation should proceed as normal (i.e. no need to transactionally\n preserve the old prefix).\n \"\"\"\n # If we don't have to overwrite, do a normal install\n if self.pkg.spec.dag_hash() not in self.request.overwrite:\n return InstallAction.INSTALL\n\n # If it's not installed, do a normal install as well\n rec, installed = check_db(self.pkg.spec)\n\n if not installed:\n return InstallAction.INSTALL\n\n # Ensure install_tree projections have not changed.\n assert rec and self.pkg.prefix == rec.path\n\n # If another process has overwritten this, we shouldn't install at all\n if rec.installation_time >= self.request.overwrite_time:\n return InstallAction.NONE\n\n # If the install prefix is missing, warn about it, and proceed with\n # normal install.\n if not os.path.exists(self.pkg.prefix):\n tty.debug(\"Missing installation to overwrite\")\n return InstallAction.INSTALL\n\n # Otherwise, do an actual overwrite install. We backup the original\n # install directory, put the old prefix\n # back on failure\n return InstallAction.OVERWRITE\n\n @property\n def explicit(self) -> bool:\n return self.pkg.spec.dag_hash() in self.request.install_args.get(\"explicit\", [])\n\n @property\n def is_build_request(self) -> bool:\n \"\"\"The package was requested directly\"\"\"\n return self.pkg == self.request.pkg\n\n @property\n def install_policy(self) -> InstallPolicy:\n if self.is_build_request:\n return self.request.install_args.get(\"root_policy\", \"auto\")\n else:\n return self.request.install_args.get(\"dependencies_policy\", \"auto\")\n\n @property\n def key(self) -> Tuple[int, int]:\n \"\"\"The key is the tuple (# uninstalled dependencies, sequence).\"\"\"\n return (self.priority, self.sequence)\n\n def next_attempt(self, installed) -> \"Task\":\n \"\"\"Create a new, updated task for the next installation attempt.\"\"\"\n task = copy.copy(self)\n task._update()\n task.start_time = self.start_time or time.time()\n task.flag_installed(installed)\n return task\n\n @property\n def priority(self):\n \"\"\"The priority is based on the remaining uninstalled dependencies.\"\"\"\n return len(self.uninstalled_deps)\n\n def terminate(self) -> None:\n \"\"\"End any processes and clean up any resources allocated by this Task.\n\n By default this is a no-op.\n \"\"\"\n\n\ndef check_db(spec: \"spack.spec.Spec\") -> Tuple[Optional[spack.database.InstallRecord], bool]:\n \"\"\"Determine if the spec is flagged as installed in the database\n\n Args:\n spec: spec whose database install status is being checked\n\n Return:\n Tuple of optional database record, and a boolean installed_in_db that's ``True`` iff the\n spec is considered installed\n \"\"\"\n try:\n rec = spack.store.STORE.db.get_record(spec)\n installed_in_db = rec.installed if rec else False\n except KeyError:\n # KeyError is raised if there is no matching spec in the database\n # (versus no matching specs that are installed).\n rec = None\n installed_in_db = False\n return rec, installed_in_db\n\n\nclass BuildTask(Task):\n \"\"\"Class for representing a build task for a package.\"\"\"\n\n process_handle: Optional[\"spack.build_environment.BuildProcess\"] = None\n started: bool = False\n no_op: bool = False\n tmpdir = None\n backup_dir = None\n\n def start(self):\n \"\"\"Attempt to use the binary cache to install\n requested spec and/or dependency if requested.\n\n Otherwise, start a process for of the requested spec and/or\n dependency represented by the BuildTask.\"\"\"\n self.record.start()\n\n if self.install_action == InstallAction.OVERWRITE:\n self.tmpdir = tempfile.mkdtemp(dir=os.path.dirname(self.pkg.prefix), prefix=\".backup\")\n self.backup_dir = os.path.join(self.tmpdir, \"backup\")\n os.rename(self.pkg.prefix, self.backup_dir)\n\n assert not self.started, \"Cannot start a task that has already been started.\"\n self.started = True\n self.start_time = self.start_time or time.time()\n\n install_args = self.request.install_args\n unsigned = install_args.get(\"unsigned\")\n pkg, pkg_id = self.pkg, self.pkg_id\n\n tests = install_args.get(\"tests\")\n pkg.run_tests = tests is True or tests and pkg.name in tests\n\n # Use the binary cache to install if requested,\n # save result to be handled in BuildTask.complete()\n # TODO: change binary installs to occur in subprocesses rather than the main Spack process\n policy = self.install_policy\n if policy != \"source_only\":\n if _install_from_cache(pkg, self.explicit, unsigned):\n self.success_result = ExecuteResult.SUCCESS\n return\n elif policy == \"cache_only\":\n self.error_result = spack.error.InstallError(\n \"No binary found when cache-only was specified\", pkg=pkg\n )\n return\n else:\n tty.msg(f\"No binary for {pkg_id} found: installing from source\")\n\n # if there's an error result, don't start a new process, and leave\n if self.error_result is not None:\n return\n\n # Create stage object now and let it be serialized for the child process. That\n # way monkeypatch in tests works correctly.\n pkg.stage\n self._setup_install_dir(pkg)\n\n # Create a child process to do the actual installation.\n self._start_build_process()\n\n def _start_build_process(self):\n self.process_handle = spack.build_environment.start_build_process(\n self.pkg, build_process, self.request.install_args\n )\n\n # Identify the child process\n self.child_pid = self.process_handle.pid\n\n def poll(self):\n \"\"\"Check if task has successfully executed, caused an InstallError,\n or the child process has information ready to receive.\"\"\"\n assert self.started or self.no_op, (\n \"Can't call `poll()` before `start()` or identified no-operation task\"\n )\n return self.no_op or self.success_result or self.error_result or self.process_handle.poll()\n\n def succeed(self):\n self.record.succeed()\n\n # delete the temporary backup for an overwrite\n # see spack.llnl.util.filesystem.restore_directory_transaction\n if self.install_action == InstallAction.OVERWRITE:\n shutil.rmtree(self.tmpdir, ignore_errors=True)\n\n def fail(self, inner_exception):\n self.record.fail(inner_exception)\n\n if self.install_action != InstallAction.OVERWRITE:\n raise inner_exception\n\n # restore the overwrite directory from backup\n # see spack.llnl.util.filesystem.restore_directory_transaction\n try:\n if os.path.exists(self.pkg.prefix):\n shutil.rmtree(self.pkg.prefix)\n os.rename(self.backup_dir, self.pkg.prefix)\n except Exception as outer_exception:\n raise fs.CouldNotRestoreDirectoryBackup(inner_exception, outer_exception)\n\n raise inner_exception\n\n def complete(self):\n \"\"\"\n Complete the installation of the requested spec and/or dependency\n represented by the build task.\n \"\"\"\n assert self.started or self.no_op, (\n \"Can't call `complete()` before `start()` or identified no-operation task\"\n )\n pkg = self.pkg\n\n self.status = BuildStatus.INSTALLING\n\n # If task has been identified as a no operation,\n # return ExecuteResult.NOOP\n if self.no_op:\n # This is one exit point that does not need to call\n # self.succeed/fail. Job is either a no_op (external, upstream)\n # or requeued.\n return ExecuteResult.NO_OP\n\n # If installing a package from binary cache is successful,\n # return ExecuteResult.SUCCESS\n if self.success_result is not None:\n self.succeed()\n return self.success_result\n\n # If an error arises from installing a package,\n # raise spack.error.InstallError\n if self.error_result is not None:\n self.fail(self.error_result)\n\n # hook that allows tests to inspect the Package before installation\n # see _unit_test_check() docs.\n if not pkg._unit_test_check():\n self.succeed()\n return ExecuteResult.FAILED\n\n try:\n # Check if the task's child process has completed\n spack.package_base.PackageBase._verbose = self.process_handle.complete()\n # Note: PARENT of the build process adds the new package to\n # the database, so that we don't need to re-read from file.\n spack.store.STORE.db.add(pkg.spec, explicit=self.explicit)\n except spack.error.StopPhase as e:\n # A StopPhase exception means that do_install was asked to\n # stop early from clients, and is not an error at this point\n pid = f\"{self.pid}: \" if tty.show_pid() else \"\"\n tty.debug(f\"{pid}{str(e)}\")\n tty.debug(f\"Package stage directory: {pkg.stage.source_path}\")\n except (Exception, KeyboardInterrupt, SystemExit) as e:\n self.fail(e)\n\n self.succeed()\n return ExecuteResult.SUCCESS\n\n def terminate(self) -> None:\n \"\"\"Terminate any processes this task still has running.\"\"\"\n if self.process_handle:\n self.process_handle.terminate()\n\n\nclass MockBuildProcess:\n def complete(self) -> bool:\n return True\n\n def terminate(self) -> None:\n pass\n\n\nclass FakeBuildTask(BuildTask):\n \"\"\"Blocking BuildTask executed directly in the main thread. Used for --fake installs.\"\"\"\n\n process_handle = MockBuildProcess() # type: ignore[assignment]\n\n def _start_build_process(self):\n build_process(self.pkg, self.request.install_args)\n\n def poll(self):\n return True\n\n\nclass RewireTask(Task):\n \"\"\"Class for representing a rewire task for a package.\"\"\"\n\n def start(self):\n self.record.start()\n\n def poll(self):\n return True\n\n def complete(self):\n \"\"\"Execute rewire task\n\n Rewire tasks are executed by either rewiring self.package.spec.build_spec that is already\n installed or downloading and rewiring a binary for the it.\n\n If not available installed or as binary, return ExecuteResult.MISSING_BUILD_SPEC.\n This will prompt the Installer to requeue the task with a dependency on the BuildTask\n to install self.pkg.spec.build_spec\n \"\"\"\n oldstatus = self.status\n self.status = BuildStatus.INSTALLING\n if not self.pkg.spec.build_spec.installed:\n try:\n install_args = self.request.install_args\n unsigned = install_args.get(\"unsigned\")\n success = _process_binary_cache_tarball(\n self.pkg, explicit=self.explicit, unsigned=unsigned\n )\n\n if not success:\n tty.msg(\n \"Failed to find binary for build spec, requeuing {self.pkg.spec} with\"\n \"dependency install task for its build spec\"\n )\n self.status = oldstatus\n return ExecuteResult.MISSING_BUILD_SPEC\n\n _print_installed_pkg(self.pkg.prefix)\n self.record.succeed()\n return ExecuteResult.SUCCESS\n except BaseException as e:\n tty.error(f\"Failed to rewire {self.pkg.spec} from binary. {e}\")\n self.status = oldstatus\n return ExecuteResult.MISSING_BUILD_SPEC\n try:\n spack.rewiring.rewire_node(self.pkg.spec, self.explicit)\n _print_installed_pkg(self.pkg.prefix)\n self.record.succeed()\n return ExecuteResult.SUCCESS\n except BaseException as e:\n self.record.fail(e)\n\n\nclass PackageInstaller:\n \"\"\"\n Class for managing the install process for a Spack instance based on a bottom-up DAG approach.\n\n This installer can coordinate concurrent batch and interactive, local and distributed (on a\n shared file system) builds for the same Spack instance.\n \"\"\"\n\n def __init__(\n self,\n packages: List[\"spack.package_base.PackageBase\"],\n *,\n dirty: bool = False,\n explicit: Union[Set[str], bool] = False,\n overwrite: Optional[Union[List[str], Set[str]]] = None,\n fail_fast: bool = False,\n fake: bool = False,\n include_build_deps: bool = False,\n install_deps: bool = True,\n install_package: bool = True,\n install_source: bool = False,\n keep_prefix: bool = False,\n keep_stage: bool = False,\n restage: bool = False,\n skip_patch: bool = False,\n stop_at: Optional[str] = None,\n stop_before: Optional[str] = None,\n tests: Union[bool, List[str], Set[str]] = False,\n unsigned: Optional[bool] = None,\n verbose: bool = False,\n concurrent_packages: Optional[int] = None,\n root_policy: InstallPolicy = \"auto\",\n dependencies_policy: InstallPolicy = \"auto\",\n create_reports: bool = False,\n ) -> None:\n \"\"\"\n Arguments:\n explicit: Set of package hashes to be marked as installed explicitly in the db. If\n True, the specs from ``packages`` are marked explicit, while their dependencies are\n not.\n fail_fast: Fail if any dependency fails to install; otherwise, the default is to\n install as many dependencies as possible (i.e., best effort installation).\n fake: Don't really build; install fake stub files instead.\n install_deps: Install dependencies before installing this package\n install_source: By default, source is not installed, but for debugging it might be\n useful to keep it around.\n keep_prefix: Keep install prefix on failure. By default, destroys it.\n keep_stage: By default, stage is destroyed only if there are no exceptions during\n build. Set to True to keep the stage even with exceptions.\n restage: Force spack to restage the package source.\n skip_patch: Skip patch stage of build if True.\n stop_before: stop execution before this installation phase (or None)\n stop_at: last installation phase to be executed (or None)\n tests: False to run no tests, True to test all packages, or a list of package names to\n run tests for some\n verbose: Display verbose build output (by default, suppresses it)\n concurrent_packages: Max packages to be built concurrently\n root_policy: ``\"auto\"``, ``\"cache_only\"``, ``\"source_only\"``.\n dependencies_policy: ``\"auto\"``, ``\"cache_only\"``, ``\"source_only\"``.\n create_reports: whether to generate reports for each install\n \"\"\"\n if sys.platform == \"win32\":\n # No locks on Windows, we should always use 1 process\n # TODO: perhaps raise an error instead and update cmd-line interface\n # to omit this option on Windows for now\n concurrent_packages = 1\n\n if isinstance(explicit, bool):\n explicit = {pkg.spec.dag_hash() for pkg in packages} if explicit else set()\n\n if concurrent_packages is None:\n concurrent_packages = spack.config.get(\"config:concurrent_packages\", default=1)\n # The value 0 means no concurrency in the old installer.\n if concurrent_packages == 0:\n concurrent_packages = 1\n self.concurrent_packages = concurrent_packages\n\n install_args = {\n \"dependencies_policy\": dependencies_policy,\n \"dirty\": dirty,\n \"explicit\": explicit,\n \"fail_fast\": fail_fast,\n \"fake\": fake,\n \"include_build_deps\": include_build_deps,\n \"install_deps\": install_deps,\n \"install_package\": install_package,\n \"install_source\": install_source,\n \"keep_prefix\": keep_prefix,\n \"keep_stage\": keep_stage,\n \"overwrite\": overwrite or [],\n \"root_policy\": root_policy,\n \"restage\": restage,\n \"skip_patch\": skip_patch,\n \"stop_at\": stop_at,\n \"stop_before\": stop_before,\n \"tests\": tests,\n \"unsigned\": unsigned,\n \"verbose\": verbose,\n \"concurrent_packages\": self.concurrent_packages,\n }\n\n # List of build requests\n self.build_requests = [BuildRequest(pkg, install_args) for pkg in packages]\n\n # When no reporter is configured, use NullInstallRecord to skip log file reads.\n if not create_reports:\n for br in self.build_requests:\n br.install_args[\"record_cls\"] = spack.report.NullInstallRecord\n\n # Priority queue of tasks\n self.build_pq: List[Tuple[Tuple[int, int], Task]] = []\n\n # Mapping of unique package ids to task\n self.build_tasks: Dict[str, Task] = {}\n\n # Cache of package locks for failed packages, keyed on package's ids\n self.failed: Dict[str, Optional[lk.Lock]] = {}\n\n # Cache the PID for distributed build messaging\n self.pid: int = os.getpid()\n\n # Cache of installed packages' unique ids\n self.installed: Set[str] = set()\n\n # Data store layout\n self.layout = spack.store.STORE.layout\n\n # Locks on specs being built, keyed on the package's unique id\n self.locks: Dict[str, Tuple[str, Optional[lk.Lock]]] = {}\n\n # Cache fail_fast option to ensure if one build request asks to fail\n # fast then that option applies to all build requests.\n self.fail_fast = False\n\n # Initializing all_dependencies to empty. This will be set later in _init_queue.\n self.all_dependencies: Dict[str, Set[str]] = {}\n\n # Maximum number of concurrent packages to build\n self.max_active_tasks = self.concurrent_packages\n\n # Reports on install success/failure\n if create_reports:\n self.reports: Dict[str, spack.report.RequestRecord] = {}\n for build_request in self.build_requests:\n # Skip reporting for already installed specs\n request_record = spack.report.RequestRecord(build_request.pkg.spec)\n request_record.skip_installed()\n self.reports[build_request.pkg_id] = request_record\n else:\n self.reports = {\n br.pkg_id: spack.report.NullRequestRecord() for br in self.build_requests\n }\n\n def __repr__(self) -> str:\n \"\"\"Returns a formal representation of the package installer.\"\"\"\n rep = f\"{self.__class__.__name__}(\"\n for attr, value in self.__dict__.items():\n rep += f\"{attr}={value.__repr__()}, \"\n return f\"{rep.strip(', ')})\"\n\n def __str__(self) -> str:\n \"\"\"Returns a printable version of the package installer.\"\"\"\n requests = f\"#requests={len(self.build_requests)}\"\n tasks = f\"#tasks={len(self.build_tasks)}\"\n failed = f\"failed ({len(self.failed)}) = {self.failed}\"\n installed = f\"installed ({len(self.installed)}) = {self.installed}\"\n return f\"{self.pid}: {requests}; {tasks}; {installed}; {failed}\"\n\n def _add_init_task(\n self,\n pkg: \"spack.package_base.PackageBase\",\n request: BuildRequest,\n all_deps: Dict[str, Set[str]],\n ) -> None:\n \"\"\"\n Creates and queues the initial task for the package.\n\n Args:\n pkg: the package to be built and installed\n request: the associated install request\n all_deps: dictionary of all dependencies and associated dependents\n \"\"\"\n cls: type[Task] = BuildTask\n if pkg.spec.spliced:\n cls = RewireTask\n elif request.install_args.get(\"fake\"):\n cls = FakeBuildTask\n\n task = cls(pkg, request=request, status=BuildStatus.QUEUED, installed=self.installed)\n for dep_id in task.dependencies:\n all_deps[dep_id].add(package_id(pkg.spec))\n\n self._push_task(task)\n\n def _check_deps_status(self, request: BuildRequest) -> None:\n \"\"\"Check the install status of the requested package\n\n Args:\n request: the associated install request\n \"\"\"\n err = \"Cannot proceed with {0}: {1}\"\n for dep in request.traverse_dependencies():\n dep_pkg = dep.package\n dep_id = package_id(dep)\n\n # Check for failure since a prefix lock is not required\n if spack.store.STORE.failure_tracker.has_failed(dep):\n action = \"'spack install' the dependency\"\n msg = f\"{dep_id} is marked as an install failure: {action}\"\n raise spack.error.InstallError(err.format(request.pkg_id, msg), pkg=dep_pkg)\n\n # Attempt to get a read lock to ensure another process does not\n # uninstall the dependency while the requested spec is being\n # installed\n ltype, lock = self._ensure_locked(\"read\", dep_pkg)\n if lock is None:\n msg = f\"{dep_id} is write locked by another process\"\n raise spack.error.InstallError(err.format(request.pkg_id, msg), pkg=request.pkg)\n\n # Flag external and upstream packages as being installed\n if dep_pkg.spec.external or dep_pkg.spec.installed_upstream:\n self._flag_installed(dep_pkg)\n continue\n\n # Check the database to see if the dependency has been installed\n # and flag as such if appropriate\n rec, installed_in_db = check_db(dep)\n if (\n rec\n and installed_in_db\n and (\n dep.dag_hash() not in request.overwrite\n or rec.installation_time > request.overwrite_time\n )\n ):\n tty.debug(f\"Flagging {dep_id} as installed per the database\")\n self._flag_installed(dep_pkg)\n else:\n lock.release_read()\n\n def _prepare_for_install(self, task: Task) -> None:\n \"\"\"\n Check the database and leftover installation directories/files and\n prepare for a new install attempt for an uninstalled package.\n Preparation includes cleaning up installation and stage directories\n and ensuring the database is up-to-date.\n\n Args:\n task: the task whose associated package is\n being checked\n \"\"\"\n install_args = task.request.install_args\n keep_prefix = install_args.get(\"keep_prefix\")\n\n # Make sure the package is ready to be locally installed.\n self._ensure_install_ready(task.pkg)\n\n # Skip file system operations if we've already gone through them for\n # this spec.\n if task.pkg_id in self.installed:\n # Already determined the spec has been installed\n return\n\n # Determine if the spec is flagged as installed in the database\n rec, installed_in_db = check_db(task.pkg.spec)\n\n if not installed_in_db:\n # Ensure there is no other installed spec with the same prefix dir\n if spack.store.STORE.db.is_occupied_install_prefix(task.pkg.spec.prefix):\n task.error_result = spack.error.InstallError(\n f\"Install prefix collision for {task.pkg_id}\",\n long_msg=f\"Prefix directory {task.pkg.spec.prefix} already \"\n \"used by another installed spec.\",\n pkg=task.pkg,\n )\n return\n\n # Make sure the installation directory is in the desired state\n # for uninstalled specs.\n if os.path.isdir(task.pkg.spec.prefix):\n if not keep_prefix and task.install_action != InstallAction.OVERWRITE:\n task.pkg.remove_prefix()\n else:\n tty.debug(f\"{task.pkg_id} is partially installed\")\n\n if (\n rec\n and installed_in_db\n and (\n rec.spec.dag_hash() not in task.request.overwrite\n or rec.installation_time > task.request.overwrite_time\n )\n ):\n self._update_installed(task)\n\n # Only update the explicit entry once for the explicit package\n if task.explicit and not rec.explicit:\n spack.store.STORE.db.mark(task.pkg.spec, \"explicit\", True)\n\n def _cleanup_all_tasks(self) -> None:\n \"\"\"Cleanup all tasks to include releasing their locks.\"\"\"\n for pkg_id in self.locks:\n self._release_lock(pkg_id)\n\n for pkg_id in self.failed:\n self._cleanup_failed(pkg_id)\n\n ids = list(self.build_tasks)\n for pkg_id in ids:\n try:\n self._remove_task(pkg_id)\n except Exception:\n pass\n\n def _cleanup_failed(self, pkg_id: str) -> None:\n \"\"\"\n Cleanup any failed markers for the package\n\n Args:\n pkg_id (str): identifier for the failed package\n \"\"\"\n lock = self.failed.get(pkg_id, None)\n if lock is not None:\n err = \"{0} exception when removing failure tracking for {1}: {2}\"\n try:\n tty.verbose(f\"Removing failure mark on {pkg_id}\")\n lock.release_write()\n except Exception as exc:\n tty.warn(err.format(exc.__class__.__name__, pkg_id, str(exc)))\n\n def _cleanup_task(self, pkg: \"spack.package_base.PackageBase\") -> None:\n \"\"\"\n Cleanup the task for the spec\n\n Args:\n pkg: the package being installed\n \"\"\"\n self._remove_task(package_id(pkg.spec))\n\n # Ensure we have a read lock to prevent others from uninstalling the\n # spec during our installation.\n self._ensure_locked(\"read\", pkg)\n\n def _ensure_install_ready(self, pkg: \"spack.package_base.PackageBase\") -> None:\n \"\"\"\n Ensure the package is ready to install locally, which includes\n already locked.\n\n Args:\n pkg: the package being locally installed\n \"\"\"\n pkg_id = package_id(pkg.spec)\n pre = f\"{pkg_id} cannot be installed locally:\"\n\n # External packages cannot be installed locally.\n if pkg.spec.external:\n raise ExternalPackageError(f\"{pre} is external\")\n\n # Upstream packages cannot be installed locally.\n if pkg.spec.installed_upstream:\n raise UpstreamPackageError(f\"{pre} is upstream\")\n\n # The package must have a prefix lock at this stage.\n if pkg_id not in self.locks:\n raise InstallLockError(f\"{pre} not locked\")\n\n def _ensure_locked(\n self, lock_type: str, pkg: \"spack.package_base.PackageBase\"\n ) -> Tuple[str, Optional[lk.Lock]]:\n \"\"\"\n Add a prefix lock of the specified type for the package spec\n\n If the lock exists, then adjust accordingly. That is, read locks\n will be upgraded to write locks if a write lock is requested and\n write locks will be downgraded to read locks if a read lock is\n requested.\n\n The lock timeout for write locks is deliberately near zero seconds in\n order to ensure the current process proceeds as quickly as possible to\n the next spec.\n\n Args:\n lock_type: 'read' for a read lock, 'write' for a write lock\n pkg: the package whose spec is being installed\n\n Return:\n (lock_type, lock) tuple where lock will be None if it could not be obtained\n \"\"\"\n assert lock_type in [\"read\", \"write\"], (\n f'\"{lock_type}\" is not a supported package management lock type'\n )\n\n pkg_id = package_id(pkg.spec)\n ltype, lock = self.locks.get(pkg_id, (lock_type, None))\n if lock and ltype == lock_type:\n return ltype, lock\n\n desc = f\"{lock_type} lock\"\n msg = \"{0} a {1} on {2} with timeout {3}\"\n err = \"Failed to {0} a {1} for {2} due to {3}: {4}\"\n\n if lock_type == \"read\":\n # Wait until the other process finishes if there are no more\n # tasks with priority 0 (i.e., with no uninstalled\n # dependencies).\n no_p0 = len(self.build_tasks) == 0 or not self._next_is_pri0()\n timeout = None if no_p0 else 3.0\n else:\n timeout = 1e-9 # Near 0 to iterate through install specs quickly\n\n try:\n if lock is None:\n tty.debug(msg.format(\"Acquiring\", desc, pkg_id, pretty_seconds(timeout or 0)))\n op = \"acquire\"\n lock = spack.store.STORE.prefix_locker.lock(pkg.spec, timeout)\n if timeout != lock.default_timeout:\n tty.warn(f\"Expected prefix lock timeout {timeout}, not {lock.default_timeout}\")\n if lock_type == \"read\":\n lock.acquire_read()\n else:\n lock.acquire_write()\n\n elif lock_type == \"read\": # write -> read\n # Only get here if the current lock is a write lock, which\n # must be downgraded to be a read lock\n # Retain the original lock timeout, which is in the lock's\n # default_timeout setting.\n tty.debug(\n msg.format(\n \"Downgrading to\", desc, pkg_id, pretty_seconds(lock.default_timeout or 0)\n )\n )\n op = \"downgrade to\"\n lock.downgrade_write_to_read()\n\n else: # read -> write\n # Only get here if the current lock is a read lock, which\n # must be upgraded to be a write lock\n tty.debug(msg.format(\"Upgrading to\", desc, pkg_id, pretty_seconds(timeout or 0)))\n op = \"upgrade to\"\n lock.upgrade_read_to_write(timeout)\n tty.debug(f\"{pkg_id} is now {lock_type} locked\")\n\n except (lk.LockDowngradeError, lk.LockTimeoutError) as exc:\n tty.debug(err.format(op, desc, pkg_id, exc.__class__.__name__, str(exc)))\n return (lock_type, None)\n\n except (Exception, KeyboardInterrupt, SystemExit) as exc:\n tty.error(err.format(op, desc, pkg_id, exc.__class__.__name__, str(exc)))\n self._cleanup_all_tasks()\n raise\n\n self.locks[pkg_id] = (lock_type, lock)\n return self.locks[pkg_id]\n\n def _requeue_with_build_spec_tasks(self, task):\n \"\"\"Requeue the task and its missing build spec dependencies\"\"\"\n # Full install of the build_spec is necessary because it didn't already exist somewhere\n spec = task.pkg.spec\n for dep in spec.build_spec.traverse():\n dep_pkg = dep.package\n\n dep_id = package_id(dep)\n if dep_id not in self.build_tasks:\n self._add_init_task(dep_pkg, task.request, self.all_dependencies)\n\n # Clear any persistent failure markings _unless_ they are\n # associated with another process in this parallel build\n # of the spec.\n spack.store.STORE.failure_tracker.clear(dep, force=False)\n\n # Queue the build spec.\n build_pkg_id = package_id(spec.build_spec)\n build_spec_task = self.build_tasks[build_pkg_id]\n spec_pkg_id = package_id(spec)\n spec_task = task.next_attempt(self.installed)\n spec_task.status = BuildStatus.QUEUED\n # Convey a build spec as a dependency of a deployed spec.\n build_spec_task.add_dependent(spec_pkg_id)\n spec_task.add_dependency(build_pkg_id)\n self._push_task(spec_task)\n\n def _add_tasks(self, request: BuildRequest, all_deps):\n \"\"\"Add tasks to the priority queue for the given build request.\n\n It also tracks all dependents associated with each dependency in\n order to ensure proper tracking of uninstalled dependencies.\n\n Args:\n request (BuildRequest): the associated install request\n all_deps (defaultdict(set)): dictionary of all dependencies and\n associated dependents\n \"\"\"\n tty.debug(f\"Initializing the build queue for {request.pkg.name}\")\n\n # Ensure not attempting to perform an installation when user didn't\n # want to go that far for the requested package.\n try:\n _check_last_phase(request.pkg)\n except BadInstallPhase as err:\n tty.warn(f\"Installation request refused: {str(err)}\")\n return\n\n install_deps = request.install_args.get(\"install_deps\")\n\n if install_deps:\n for dep in request.traverse_dependencies():\n dep_pkg = dep.package\n\n dep_id = package_id(dep)\n if dep_id not in self.build_tasks:\n self._add_init_task(dep_pkg, request, all_deps=all_deps)\n\n # Clear any persistent failure markings _unless_ they are\n # associated with another process in this parallel build\n # of the spec.\n spack.store.STORE.failure_tracker.clear(dep, force=False)\n\n install_package = request.install_args.get(\"install_package\")\n if install_package and request.pkg_id not in self.build_tasks:\n # Be sure to clear any previous failure\n spack.store.STORE.failure_tracker.clear(request.spec, force=True)\n\n # If not installing dependencies, then determine their\n # installation status before proceeding\n if not install_deps:\n self._check_deps_status(request)\n\n # Now add the package itself, if appropriate\n self._add_init_task(request.pkg, request, all_deps=all_deps)\n\n # Ensure if one request is to fail fast then all requests will.\n fail_fast = bool(request.install_args.get(\"fail_fast\"))\n self.fail_fast = self.fail_fast or fail_fast\n\n def _complete_task(self, task: Task, install_status: InstallStatus) -> None:\n \"\"\"\n Complete the installation of the requested spec and/or dependency\n represented by the task.\n\n Args:\n task: the installation task for a package\n install_status: the installation status for the package\n \"\"\"\n try:\n rc = task.complete()\n except BaseException:\n self.reports[task.request.pkg_id].append_record(task.record)\n raise\n if rc == ExecuteResult.MISSING_BUILD_SPEC:\n self._requeue_with_build_spec_tasks(task)\n elif rc == ExecuteResult.NO_OP:\n pass\n else: # if rc == ExecuteResult.SUCCESS or rc == ExecuteResult.FAILED\n self._update_installed(task)\n self.reports[task.request.pkg_id].append_record(task.record)\n\n def _next_is_pri0(self) -> bool:\n \"\"\"\n Determine if the next task has priority 0\n\n Return:\n True if it does, False otherwise\n \"\"\"\n # Leverage the fact that the first entry in the queue is the next\n # one that will be processed\n task = self.build_pq[0][1]\n return task.priority == 0\n\n def _clear_removed_tasks(self):\n \"\"\"Get rid of any tasks in the queue with status 'BuildStatus.REMOVED'\"\"\"\n while self.build_pq and self.build_pq[0][1].status == BuildStatus.REMOVED:\n heapq.heappop(self.build_pq)\n\n def _peek_ready_task(self) -> Optional[Task]:\n \"\"\"\n Return the first ready task in the queue, or None if there are no ready tasks.\n \"\"\"\n self._clear_removed_tasks()\n if not self.build_pq:\n return None\n\n task = self.build_pq[0][1]\n return task if task.priority == 0 else None\n\n def _tasks_installing_in_other_spack(self) -> bool:\n \"\"\"Whether any tasks in the build queue are installing in other spack processes.\"\"\"\n return any(task.status == BuildStatus.INSTALLING for _, task in self.build_pq)\n\n def _pop_task(self) -> Task:\n \"\"\"Pop the first task off the queue and return it.\n\n Raise an index error if the queue is empty.\"\"\"\n self._clear_removed_tasks()\n if not self.build_pq:\n raise IndexError(\"Attempt to pop empty queue\")\n _, task = heapq.heappop(self.build_pq)\n del self.build_tasks[task.pkg_id]\n task.status = BuildStatus.DEQUEUED\n return task\n\n def _pop_ready_task(self) -> Optional[Task]:\n \"\"\"\n Pop the first ready task off the queue and return it.\n\n Return None if no ready task.\n \"\"\"\n if self._peek_ready_task():\n return self._pop_task()\n return None\n\n def _push_task(self, task: Task) -> None:\n \"\"\"\n Push (or queue) the specified task for the package.\n\n Source: Customization of \"add_task\" function at\n docs.python.org/2/library/heapq.html\n\n Args:\n task: the installation task for a package\n \"\"\"\n msg = \"{0} a task for {1} with status '{2}'\"\n skip = \"Skipping requeue of task for {0}: {1}\"\n\n # Ensure do not (re-)queue installed or failed packages whose status\n # may have been determined by a separate process.\n if task.pkg_id in self.installed:\n tty.debug(skip.format(task.pkg_id, \"installed\"))\n return\n\n if task.pkg_id in self.failed:\n tty.debug(skip.format(task.pkg_id, \"failed\"))\n return\n\n # Remove any associated task since its sequence will change\n self._remove_task(task.pkg_id)\n desc = (\n \"Queueing\" if task.attempts == 1 else f\"Requeuing ({ordinal(task.attempts)} attempt)\"\n )\n tty.debug(msg.format(desc, task.pkg_id, task.status))\n\n # Now add the new task to the queue with a new sequence number to\n # ensure it is the last entry popped with the same priority. This\n # is necessary in case we are re-queueing a task whose priority\n # was decremented due to the installation of one of its dependencies.\n self.build_tasks[task.pkg_id] = task\n heapq.heappush(self.build_pq, (task.key, task))\n\n def _release_lock(self, pkg_id: str) -> None:\n \"\"\"\n Release any lock on the package\n\n Args:\n pkg_id (str): identifier for the package whose lock is be released\n \"\"\"\n if pkg_id in self.locks:\n err = \"{0} exception when releasing {1} lock for {2}: {3}\"\n msg = \"Releasing {0} lock on {1}\"\n ltype, lock = self.locks[pkg_id]\n if lock is not None:\n try:\n tty.debug(msg.format(ltype, pkg_id))\n if ltype == \"read\":\n lock.release_read()\n else:\n lock.release_write()\n except Exception as exc:\n tty.warn(err.format(exc.__class__.__name__, ltype, pkg_id, str(exc)))\n\n def _remove_task(self, pkg_id: str) -> Optional[Task]:\n \"\"\"\n Mark the existing package task as being removed and return it.\n Raises KeyError if not found.\n\n Source: Variant of function at docs.python.org/2/library/heapq.html\n\n Args:\n pkg_id: identifier for the package to be removed\n \"\"\"\n if pkg_id in self.build_tasks:\n tty.debug(f\"Removing task for {pkg_id} from list\")\n task = self.build_tasks.pop(pkg_id)\n task.status = BuildStatus.REMOVED\n return task\n else:\n return None\n\n def _requeue_task(self, task: Task, install_status: InstallStatus) -> None:\n \"\"\"\n Requeues a task that appears to be in progress by another process.\n\n Args:\n task (Task): the installation task for a package\n \"\"\"\n if task.status not in [BuildStatus.INSTALLED, BuildStatus.INSTALLING]:\n tty.debug(\n f\"{install_msg(task.pkg_id, self.pid, install_status)} \"\n \"in progress by another process\"\n )\n\n new_task = task.next_attempt(self.installed)\n new_task.status = BuildStatus.INSTALLING\n self._push_task(new_task)\n\n def _update_failed(\n self, task: Task, mark: bool = False, exc: Optional[BaseException] = None\n ) -> None:\n \"\"\"\n Update the task and transitive dependents as failed; optionally mark\n externally as failed; and remove associated tasks.\n\n Args:\n task: the task for the failed package\n mark: ``True`` if the package and its dependencies are to\n be marked as \"failed\", otherwise, ``False``\n exc: optional exception if associated with the failure\n \"\"\"\n pkg_id = task.pkg_id\n err = \"\" if exc is None else f\": {str(exc)}\"\n tty.debug(f\"Flagging {pkg_id} as failed{err}\")\n if mark:\n self.failed[pkg_id] = spack.store.STORE.failure_tracker.mark(task.pkg.spec)\n else:\n self.failed[pkg_id] = None\n task.status = BuildStatus.FAILED\n\n for dep_id in task.dependents:\n if dep_id in self.build_tasks:\n tty.warn(f\"Skipping build of {dep_id} since {pkg_id} failed\")\n # Ensure the dependent's uninstalled dependents are\n # up-to-date and their tasks removed.\n dep_task = self.build_tasks[dep_id]\n self._update_failed(dep_task, mark)\n self._remove_task(dep_id)\n else:\n tty.debug(f\"No task for {dep_id} to skip since {pkg_id} failed\")\n\n def _update_installed(self, task: Task) -> None:\n \"\"\"\n Mark the task as installed and ensure dependent tasks are aware.\n\n Args:\n task: the task for the installed package\n \"\"\"\n task.status = BuildStatus.INSTALLED\n self._flag_installed(task.pkg, task.dependents)\n\n def _flag_installed(\n self, pkg: \"spack.package_base.PackageBase\", dependent_ids: Optional[Set[str]] = None\n ) -> None:\n \"\"\"\n Flag the package as installed and ensure known by all tasks of\n known dependents.\n\n Args:\n pkg: Package that has been installed locally, externally or upstream\n dependent_ids: set of the package's dependent ids, or None if the dependent ids are\n limited to those maintained in the package (dependency DAG)\n \"\"\"\n pkg_id = package_id(pkg.spec)\n\n if pkg_id in self.installed:\n # Already determined the package has been installed\n return\n\n tty.debug(f\"Flagging {pkg_id} as installed\")\n\n self.installed.add(pkg_id)\n\n # Update affected dependents\n dependent_ids = dependent_ids or get_dependent_ids(pkg.spec)\n for dep_id in set(dependent_ids):\n tty.debug(f\"Removing {pkg_id} from {dep_id}'s uninstalled dependencies.\")\n if dep_id in self.build_tasks:\n # Ensure the dependent's uninstalled dependencies are\n # up-to-date. This will require requeuing the task.\n dep_task = self.build_tasks[dep_id]\n self._push_task(dep_task.next_attempt(self.installed))\n else:\n tty.debug(f\"{dep_id} has no task to update for {pkg_id}'s success\")\n\n def _init_queue(self) -> None:\n \"\"\"Initialize the build queue from the list of build requests.\"\"\"\n all_dependencies: Dict[str, Set[str]] = defaultdict(set)\n\n tty.debug(\"Initializing the build queue from the build requests\")\n for request in self.build_requests:\n self._add_tasks(request, all_dependencies)\n\n # Add any missing dependents to ensure proper uninstalled dependency\n # tracking when installing multiple specs\n tty.debug(\"Ensure all dependencies know all dependents across specs\")\n for dep_id in all_dependencies:\n if dep_id in self.build_tasks:\n dependents = all_dependencies[dep_id]\n task = self.build_tasks[dep_id]\n for dependent_id in dependents.difference(task.dependents):\n task.add_dependent(dependent_id)\n self.all_dependencies = all_dependencies\n\n def start_task(\n self, task: Task, install_status: InstallStatus, term_status: TermStatusLine\n ) -> None:\n \"\"\"Attempt to start a package installation.\"\"\"\n pkg, pkg_id, spec = task.pkg, task.pkg_id, task.pkg.spec\n install_status.next_pkg(pkg)\n # install_status.set_term_title(f\"Processing {task.pkg.name}\")\n tty.debug(f\"Processing {pkg_id}: task={task}\")\n\n # Debug\n task.record.start()\n\n # Skip the installation if the spec is not being installed locally\n # (i.e., if external or upstream) BUT flag it as installed since\n # some package likely depends on it.\n if _handle_external_and_upstream(pkg, task.explicit):\n term_status.clear()\n self._flag_installed(pkg, task.dependents)\n task.no_op = True\n return\n\n # Flag a failed spec. Do not need an (install) prefix lock since\n # assume using a separate (failed) prefix lock file.\n if pkg_id in self.failed or spack.store.STORE.failure_tracker.has_failed(spec):\n term_status.clear()\n tty.warn(f\"{pkg_id} failed to install\")\n self._update_failed(task)\n\n if self.fail_fast:\n task.error_result = spack.error.InstallError(_FAIL_FAST_ERR, pkg=pkg)\n\n # Attempt to get a write lock. If we can't get the lock then\n # another process is likely (un)installing the spec or has\n # determined the spec has already been installed (though the\n # other process may be hung).\n install_status.set_term_title(f\"Acquiring lock for {task.pkg.name}\")\n term_status.add(pkg_id)\n ltype, lock = self._ensure_locked(\"write\", pkg)\n if lock is None:\n # Attempt to get a read lock instead. If this fails then\n # another process has a write lock so must be (un)installing\n # the spec (or that process is hung).\n ltype, lock = self._ensure_locked(\"read\", pkg)\n # Requeue the spec if we cannot get at least a read lock so we\n # can check the status presumably established by another process\n # -- failed, installed, or uninstalled -- on the next pass.\n if lock is None:\n self._requeue_task(task, install_status)\n task.no_op = True\n return\n\n term_status.clear()\n\n # Take a timestamp with the overwrite argument to allow checking\n # whether another process has already overridden the package.\n if task.request.overwrite and task.explicit:\n task.request.overwrite_time = time.time()\n\n # install_status.set_term_title(f\"Preparing {task.pkg.name}\")\n self._prepare_for_install(task)\n\n # Flag an already installed package\n if pkg_id in self.installed:\n # Downgrade to a read lock to preclude other processes from\n # uninstalling the package until we're done installing its\n # dependents.\n ltype, lock = self._ensure_locked(\"read\", pkg)\n if lock is not None:\n self._update_installed(task)\n path = spack.util.path.debug_padded_filter(pkg.prefix)\n _print_installed_pkg(path)\n else:\n # At this point we've failed to get a write or a read\n # lock, which means another process has taken a write\n # lock between our releasing the write and acquiring the\n # read.\n #\n # Requeue the task so we can re-check the status\n # established by the other process -- failed, installed,\n # or uninstalled -- on the next pass.\n self.installed.remove(pkg_id)\n self._requeue_task(task, install_status)\n task.no_op = True\n return\n\n # Having a read lock on an uninstalled pkg may mean another\n # process completed an uninstall of the software between the\n # time we failed to acquire the write lock and the time we\n # took the read lock.\n #\n # Requeue the task so we can check the status presumably\n # established by the other process -- failed, installed, or\n # uninstalled -- on the next pass.\n if ltype == \"read\":\n lock.release_read()\n self._requeue_task(task, install_status)\n task.no_op = True\n return\n\n # Proceed with the installation since we have an exclusive write\n # lock on the package.\n install_status.set_term_title(f\"Installing {task.pkg.name}\")\n action = task.install_action\n\n if action in (InstallAction.INSTALL, InstallAction.OVERWRITE):\n # Start a child process for a task that's ready to be installed.\n task.start()\n tty.msg(install_msg(pkg_id, self.pid, install_status))\n\n def complete_task(self, task: Task, install_status: InstallStatus) -> Optional[Tuple]:\n \"\"\"Attempts to complete a package installation.\"\"\"\n pkg, pkg_id = task.pkg, task.pkg_id\n install_args = task.request.install_args\n keep_prefix = install_args.get(\"keep_prefix\")\n action = task.install_action\n try:\n self._complete_task(task, install_status)\n\n # If we installed then we should keep the prefix\n stop_before_phase = getattr(pkg, \"stop_before_phase\", None)\n last_phase = getattr(pkg, \"last_phase\", None)\n keep_prefix = keep_prefix or (stop_before_phase is None and last_phase is None)\n\n except KeyboardInterrupt as exc:\n # The build has been terminated with a Ctrl-C so terminate\n # regardless of the number of remaining specs.\n tty.error(f\"Failed to install {pkg.name} due to {exc.__class__.__name__}: {str(exc)}\")\n raise\n\n except BuildcacheEntryError as exc:\n if task.install_policy == \"cache_only\":\n raise\n\n # Checking hash on downloaded binary failed.\n tty.error(\n f\"Failed to install {pkg.name} from binary cache due \"\n f\"to {str(exc)}: Requeuing to install from source.\"\n )\n # this overrides a full method, which is ugly.\n task.install_policy = \"source_only\" # type: ignore[misc]\n self._requeue_task(task, install_status)\n return None\n\n # Overwrite install exception handling\n except fs.CouldNotRestoreDirectoryBackup as e:\n spack.store.STORE.db.remove(task.pkg.spec)\n tty.error(\n f\"Recovery of install dir of {task.pkg.name} failed due to \"\n f\"{e.outer_exception.__class__.__name__}: {str(e.outer_exception)}. \"\n \"The spec is now uninstalled.\"\n )\n\n # Unwrap the actual installation exception.\n raise e.inner_exception\n\n except (Exception, SystemExit) as exc:\n # Overwrite process exception handling\n self._update_failed(task, True, exc)\n\n # Best effort installs suppress the exception and mark the\n # package as a failure.\n if not isinstance(exc, spack.error.SpackError) or not exc.printed: # type: ignore[union-attr] # noqa: E501\n exc.printed = True # type: ignore[union-attr]\n # SpackErrors can be printed by the build process or at\n # lower levels -- skip printing if already printed.\n # TODO: sort out this and SpackError.print_context()\n tty.error(\n f\"Failed to install {pkg.name} due to {exc.__class__.__name__}: {str(exc)}\"\n )\n # Terminate if requested to do so on the first failure.\n if self.fail_fast:\n raise spack.error.InstallError(f\"{_FAIL_FAST_ERR}: {str(exc)}\", pkg=pkg) from exc\n\n # Terminate when a single build request has failed, or summarize errors later.\n if task.is_build_request:\n if len(self.build_requests) == 1:\n raise\n return (pkg, pkg_id, str(exc))\n\n finally:\n # Remove the install prefix if anything went wrong during\n # install.\n if not keep_prefix and action != InstallAction.OVERWRITE:\n pkg.remove_prefix()\n\n # Perform basic task cleanup for the installed spec to\n # include downgrading the write to a read lock\n if pkg.spec.installed:\n self._cleanup_task(pkg)\n # mark installed if we haven't yet - may be discovering installed for the first time\n self._update_installed(task)\n\n return None\n\n def install(self) -> None:\n \"\"\"Install the requested package(s) and/or associated dependencies.\"\"\"\n # ensure that build processes do not permanently bork terminal settings\n with preserve_terminal_settings(sys.stdin):\n self._install()\n\n def _install(self) -> None:\n \"\"\"Helper with main implementation of ``install()``.\n\n We need to wrap the installation routine with a context manager for preserving\n keyboard sanity. Wrappers go in ``install()``. This does the real work.\n\n \"\"\"\n\n self._init_queue()\n failed_build_requests = []\n install_status = InstallStatus(len(self.build_pq))\n active_tasks: List[Task] = []\n\n # Only enable the terminal status line when we're in a tty without debug info\n # enabled, so that the output does not get cluttered.\n term_status = TermStatusLine(\n enabled=sys.stdout.isatty() and tty.msg_enabled() and not tty.is_debug()\n )\n\n # While a task is ready or tasks are running\n while self._peek_ready_task() or active_tasks or self._tasks_installing_in_other_spack():\n # While there's space for more active tasks to start\n while len(active_tasks) < self.max_active_tasks:\n task = self._pop_ready_task()\n if not task:\n # no ready tasks\n break\n\n active_tasks.append(task)\n try:\n # Attempt to start the task's package installation\n self.start_task(task, install_status, term_status)\n except BaseException as e:\n # Delegating any exception that happens in start_task() to be\n # handled in complete_task()\n task.error_result = e\n\n # 10 ms to avoid busy waiting\n time.sleep(0.01)\n # Check if any tasks have completed and add to list\n done = [task for task in active_tasks if task.poll()]\n try:\n # Iterate through the done tasks and complete them\n for task in done:\n # If complete_task does not return None, the build request failed\n failure = self.complete_task(task, install_status)\n if failure:\n failed_build_requests.append(failure)\n active_tasks.remove(task)\n except Exception:\n # Terminate any active child processes if there's an installation error\n for task in active_tasks:\n task.terminate()\n active_tasks.clear() # they're all done now\n raise\n\n self._clear_removed_tasks()\n if self.build_pq:\n task = self._pop_task()\n assert task.priority != 0, \"Found ready task after _peek_ready_task returned None\"\n # If the spec has uninstalled dependencies\n # and no active tasks running, then there must be\n # a bug in the code (e.g., priority queue or uninstalled\n # dependencies handling). So terminate under the assumption\n # that all subsequent task will have non-zero priorities or may\n # be dependencies of this task.\n term_status.clear()\n tty.error(\n f\"Detected uninstalled dependencies for {task.pkg_id}: {task.uninstalled_deps}\"\n )\n left = [dep_id for dep_id in task.uninstalled_deps if dep_id not in self.installed]\n if not left:\n tty.warn(f\"{task.pkg_id} does NOT actually have any uninstalled deps left\")\n dep_str = \"dependencies\" if task.priority > 1 else \"dependency\"\n\n raise spack.error.InstallError(\n f\"Cannot proceed with {task.pkg_id}: {task.priority} uninstalled \"\n f\"{dep_str}: {','.join(task.uninstalled_deps)}\",\n pkg=task.pkg,\n )\n\n # Cleanup, which includes releasing all of the read locks\n self._cleanup_all_tasks()\n\n # Ensure we properly report if one or more explicit specs failed\n # or were not installed when should have been.\n missing = [\n (request.pkg, request.pkg_id)\n for request in self.build_requests\n if request.install_args.get(\"install_package\") and request.pkg_id not in self.installed\n ]\n\n if failed_build_requests or missing:\n for _, pkg_id, err in failed_build_requests:\n tty.error(f\"{pkg_id}: {err}\")\n\n for _, pkg_id in missing:\n tty.error(f\"{pkg_id}: Package was not installed\")\n\n if len(failed_build_requests) > 0:\n pkg = failed_build_requests[0][0]\n ids = [pkg_id for _, pkg_id, _ in failed_build_requests]\n tty.debug(\n \"Associating installation failure with first failed \"\n f\"explicit package ({ids[0]}) from {', '.join(ids)}\"\n )\n\n elif len(missing) > 0:\n pkg = missing[0][0]\n ids = [pkg_id for _, pkg_id in missing]\n tty.debug(\n \"Associating installation failure with first \"\n f\"missing package ({ids[0]}) from {', '.join(ids)}\"\n )\n\n raise spack.error.InstallError(\n \"Installation request failed. Refer to reported errors for failing package(s).\",\n pkg=pkg,\n )\n\n\nclass BuildProcessInstaller:\n \"\"\"This class implements the part installation that happens in the child process.\"\"\"\n\n def __init__(self, pkg: \"spack.package_base.PackageBase\", install_args: dict):\n \"\"\"Create a new BuildProcessInstaller.\n\n It is assumed that the lifecycle of this object is the same as the child\n process in the build.\n\n Arguments:\n pkg: the package being installed.\n install_args: arguments to the installer from parent process.\n\n \"\"\"\n self.pkg = pkg\n\n # whether to do a fake install\n self.fake = install_args.get(\"fake\", False)\n\n # whether to install source code with the package\n self.install_source = install_args.get(\"install_source\", False)\n\n is_develop = pkg.spec.is_develop\n # whether to keep the build stage after installation\n # Note: user commands do not have an explicit choice to disable\n # keeping stages (i.e., we have a --keep-stage option, but not\n # a --destroy-stage option), so we can override a default choice\n # to destroy\n self.keep_stage = is_develop or install_args.get(\"keep_stage\", False)\n # whether to restage\n self.restage = (not is_develop) and install_args.get(\"restage\", False)\n\n # whether to skip the patch phase\n self.skip_patch = install_args.get(\"skip_patch\", False)\n\n # whether to enable echoing of build output initially or not\n self.verbose = bool(install_args.get(\"verbose\", False))\n\n # whether installation was explicitly requested by the user\n self.explicit = pkg.spec.dag_hash() in install_args.get(\"explicit\", [])\n\n # env before starting installation\n self.unmodified_env = install_args.get(\"unmodified_env\", {})\n\n # env modifications by Spack\n self.env_mods = install_args.get(\"env_modifications\", EnvironmentModifications())\n\n # timer for build phases\n self.timer = timer.Timer()\n\n # If we are using a padded path, filter the output to compress padded paths\n padding = spack.config.get(\"config:install_tree:padded_length\", None)\n self.filter_fn = spack.util.path.padding_filter if padding else None\n\n # info/debug information\n self.pre = _log_prefix(pkg.name)\n self.pkg_id = package_id(pkg.spec)\n\n def run(self) -> bool:\n \"\"\"Main entry point from ``build_process`` to kick off install in child.\"\"\"\n\n stage = self.pkg.stage\n stage.keep = self.keep_stage\n\n with stage:\n if self.restage:\n stage.destroy()\n\n self.timer.start(\"stage\")\n\n if not self.fake:\n if not self.skip_patch:\n self.pkg.do_patch()\n else:\n self.pkg.do_stage()\n\n self.timer.stop(\"stage\")\n\n tty.debug(\n f\"{self.pre} Building {self.pkg_id} [{self.pkg.build_system_class}]\" # type: ignore[attr-defined] # noqa: E501\n )\n\n # get verbosity from install parameter or saved value\n self.echo = self.verbose\n if spack.package_base.PackageBase._verbose is not None:\n self.echo = spack.package_base.PackageBase._verbose\n\n # Run the pre-install hook in the child process after\n # the directory is created.\n spack.hooks.pre_install(self.pkg.spec)\n if self.fake:\n _do_fake_install(self.pkg)\n else:\n if self.install_source:\n self._install_source()\n\n self._real_install()\n\n # Run post install hooks before build stage is removed.\n self.timer.start(\"post-install\")\n spack.hooks.post_install(self.pkg.spec, self.explicit)\n self.timer.stop(\"post-install\")\n\n # Stop the timer and save results\n self.timer.stop()\n _write_timer_json(self.pkg, self.timer, False)\n\n print_install_test_log(self.pkg)\n _print_timer(pre=self.pre, pkg_id=self.pkg_id, timer=self.timer)\n _print_installed_pkg(self.pkg.prefix)\n\n # preserve verbosity across runs\n return self.echo\n\n def _install_source(self) -> None:\n \"\"\"Install source code from stage into share/pkg/src if necessary.\"\"\"\n pkg = self.pkg\n if not os.path.isdir(pkg.stage.source_path):\n return\n\n src_target = os.path.join(pkg.spec.prefix, \"share\", pkg.name, \"src\")\n tty.debug(f\"{self.pre} Copying source to {src_target}\")\n\n fs.install_tree(pkg.stage.source_path, src_target)\n\n def _real_install(self) -> None:\n\n pkg = self.pkg\n\n # Do the real install in the source directory.\n with fs.working_dir(pkg.stage.source_path):\n # Save the build environment in a file before building.\n dump_environment(pkg.env_path)\n\n # Save just the changes to the environment. This file can be\n # safely installed, since it does not contain secret variables.\n with open(pkg.env_mods_path, \"w\", encoding=\"utf-8\") as env_mods_file:\n mods = self.env_mods.shell_modifications(explicit=True, env=self.unmodified_env)\n env_mods_file.write(mods)\n\n for attr in (\"configure_args\", \"cmake_args\"):\n try:\n configure_args = getattr(pkg, attr)()\n configure_args = \" \".join(configure_args)\n\n with open(pkg.configure_args_path, \"w\", encoding=\"utf-8\") as args_file:\n args_file.write(configure_args)\n\n break\n except Exception:\n pass\n\n # cache debug settings\n debug_level = tty.debug_level()\n\n # Spawn a daemon that reads from a pipe and redirects\n # everything to log_path, and provide the phase for logging\n builder = spack.builder.create(pkg)\n for i, phase_fn in enumerate(builder):\n # Keep a log file for each phase\n log_dir = os.path.dirname(pkg.log_path)\n log_file = \"spack-build-%02d-%s-out.txt\" % (i + 1, phase_fn.name.lower())\n log_file = os.path.join(log_dir, log_file)\n\n try:\n # DEBUGGING TIP - to debug this section, insert an IPython\n # embed here, and run the sections below without log capture\n log_contextmanager = log_output(\n log_file, self.echo, debug=True, filter_fn=self.filter_fn\n )\n\n with log_contextmanager as logger:\n # Redirect stdout and stderr to daemon pipe\n with logger.force_echo():\n inner_debug_level = tty.debug_level()\n tty.set_debug(debug_level)\n tty.msg(f\"{self.pre} Executing phase: '{phase_fn.name}'\")\n tty.set_debug(inner_debug_level)\n\n # Catch any errors to report to logging\n self.timer.start(phase_fn.name)\n phase_fn.execute()\n self.timer.stop(phase_fn.name)\n\n except BaseException:\n combine_phase_logs(pkg.phase_log_files, pkg.log_path)\n raise\n\n # We assume loggers share echo True/False\n self.echo = logger.echo\n\n # After log, we can get all output/error files from the package stage\n combine_phase_logs(pkg.phase_log_files, pkg.log_path)\n log(pkg)\n\n\ndef build_process(pkg: \"spack.package_base.PackageBase\", install_args: dict) -> bool:\n \"\"\"Perform the installation/build of the package.\n\n This runs in a separate child process, and has its own process and\n python module space set up by build_environment.start_build_process().\n\n This essentially wraps an instance of ``BuildProcessInstaller`` so that we can\n more easily create one in a subprocess.\n\n This function's return value is returned to the parent process.\n\n Arguments:\n pkg: the package being installed.\n install_args: arguments to installer from parent process.\n\n \"\"\"\n installer = BuildProcessInstaller(pkg, install_args)\n\n # don't print long padded paths in executable debug output.\n with spack.util.path.filter_padding():\n return installer.run()\n\n\ndef deprecate(spec: \"spack.spec.Spec\", deprecator: \"spack.spec.Spec\", link_fn) -> None:\n \"\"\"Deprecate this package in favor of deprecator spec\"\"\"\n # Here we assume we don't deprecate across different stores, and that same hash\n # means same binary artifacts\n if spec.dag_hash() == deprecator.dag_hash():\n return\n\n # We can't really have control over external specs, and cannot link anything in their place\n if spec.external:\n return\n\n # Install deprecator if it isn't installed already\n if not spack.store.STORE.db.query(deprecator):\n PackageInstaller([deprecator.package], explicit=True).install()\n\n old_deprecator = spack.store.STORE.db.deprecator(spec)\n if old_deprecator:\n # Find this spec file from its old deprecation\n specfile = spack.store.STORE.layout.deprecated_file_path(spec, old_deprecator)\n else:\n specfile = spack.store.STORE.layout.spec_file_path(spec)\n\n # copy spec metadata to \"deprecated\" dir of deprecator\n depr_specfile = spack.store.STORE.layout.deprecated_file_path(spec, deprecator)\n fs.mkdirp(os.path.dirname(depr_specfile))\n shutil.copy2(specfile, depr_specfile)\n\n # Any specs deprecated in favor of this spec are re-deprecated in favor of its new deprecator\n for deprecated in spack.store.STORE.db.specs_deprecated_by(spec):\n deprecate(deprecated, deprecator, link_fn)\n\n # Now that we've handled metadata, uninstall and replace with link\n spack.package_base.PackageBase.uninstall_by_spec(spec, force=True, deprecator=deprecator)\n link_fn(deprecator.prefix, spec.prefix)\n\n\nclass BadInstallPhase(spack.error.InstallError):\n def __init__(self, pkg_name, phase):\n super().__init__(f\"'{phase}' is not a valid phase for package {pkg_name}\")\n\n\nclass ExternalPackageError(spack.error.InstallError):\n \"\"\"Raised by install() when a package is only for external use.\"\"\"\n\n\nclass InstallLockError(spack.error.InstallError):\n \"\"\"Raised during install when something goes wrong with package locking.\"\"\"\n\n\nclass UpstreamPackageError(spack.error.InstallError):\n \"\"\"Raised during install when something goes wrong with an upstream\n package.\"\"\"\n" }, { "path": "lib/spack/spack/installer_dispatch.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport sys\nfrom typing import TYPE_CHECKING, List, Optional, Set, Union\n\nfrom spack.vendor.typing_extensions import Literal\n\nimport spack.config\nimport spack.traverse\n\nif TYPE_CHECKING:\n import spack.installer\n import spack.new_installer\n import spack.package_base\n\n\ndef create_installer(\n packages: List[\"spack.package_base.PackageBase\"],\n *,\n dirty: bool = False,\n explicit: Union[Set[str], bool] = False,\n overwrite: Optional[Union[List[str], Set[str]]] = None,\n fail_fast: bool = False,\n fake: bool = False,\n include_build_deps: bool = False,\n install_deps: bool = True,\n install_package: bool = True,\n install_source: bool = False,\n keep_prefix: bool = False,\n keep_stage: bool = False,\n restage: bool = True,\n skip_patch: bool = False,\n stop_at: Optional[str] = None,\n stop_before: Optional[str] = None,\n tests: Union[bool, List[str], Set[str]] = False,\n unsigned: Optional[bool] = None,\n verbose: bool = False,\n concurrent_packages: Optional[int] = None,\n root_policy: Literal[\"auto\", \"cache_only\", \"source_only\"] = \"auto\",\n dependencies_policy: Literal[\"auto\", \"cache_only\", \"source_only\"] = \"auto\",\n create_reports: bool = False,\n) -> Union[\"spack.installer.PackageInstaller\", \"spack.new_installer.PackageInstaller\"]:\n \"\"\"Create an installer based on the current configuration and feature support.\"\"\"\n use_old_installer = (\n sys.platform == \"win32\" or spack.config.get(\"config:installer\", \"new\") == \"old\"\n )\n\n # Use the old installer if splicing is used.\n if not use_old_installer:\n specs = [pkg.spec for pkg in packages]\n for s in spack.traverse.traverse_nodes(specs):\n if s.build_spec is not s:\n use_old_installer = True\n break\n if use_old_installer:\n from spack.installer import PackageInstaller # type: ignore\n else:\n from spack.new_installer import PackageInstaller # type: ignore\n\n return PackageInstaller(\n packages,\n dirty=dirty,\n explicit=explicit,\n overwrite=overwrite,\n fail_fast=fail_fast,\n fake=fake,\n include_build_deps=include_build_deps,\n install_deps=install_deps,\n install_package=install_package,\n install_source=install_source,\n keep_prefix=keep_prefix,\n keep_stage=keep_stage,\n restage=restage,\n skip_patch=skip_patch,\n stop_at=stop_at,\n stop_before=stop_before,\n tests=tests,\n unsigned=unsigned,\n verbose=verbose,\n concurrent_packages=concurrent_packages,\n root_policy=root_policy,\n dependencies_policy=dependencies_policy,\n create_reports=create_reports,\n )\n" }, { "path": "lib/spack/spack/llnl/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n" }, { "path": "lib/spack/spack/llnl/path.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Path primitives that just require Python standard library.\"\"\"\n\nimport functools\nimport sys\nfrom typing import List, Optional\nfrom urllib.parse import urlparse\n\n\nclass Path:\n \"\"\"Enum to identify the path-style.\"\"\"\n\n unix: int = 0\n windows: int = 1\n platform_path: int = windows if sys.platform == \"win32\" else unix\n\n\ndef format_os_path(path: str, mode: int = Path.unix) -> str:\n \"\"\"Formats the input path to use consistent, platform specific separators.\n\n Absolute paths are converted between drive letters and a prepended ``/`` as per platform\n requirement.\n\n Parameters:\n path: the path to be normalized, must be a string or expose the replace method.\n mode: the path file separator style to normalize the passed path to.\n Default is unix style, i.e. ``/``\n \"\"\"\n if not path:\n return path\n if mode == Path.windows:\n path = path.replace(\"/\", \"\\\\\")\n else:\n path = path.replace(\"\\\\\", \"/\")\n return path\n\n\ndef convert_to_posix_path(path: str) -> str:\n \"\"\"Converts the input path to POSIX style.\"\"\"\n return format_os_path(path, mode=Path.unix)\n\n\ndef convert_to_platform_path(path: str) -> str:\n \"\"\"Converts the input path to the current platform's native style.\"\"\"\n return format_os_path(path, mode=Path.platform_path)\n\n\ndef path_to_os_path(*parameters: str) -> List[str]:\n \"\"\"Takes an arbitrary number of positional parameters, converts each argument of type\n string to use a normalized filepath separator, and returns a list of all values.\n \"\"\"\n\n def _is_url(path_or_url: str) -> bool:\n if \"\\\\\" in path_or_url:\n return False\n url_tuple = urlparse(path_or_url)\n return bool(url_tuple.scheme) and len(url_tuple.scheme) > 1\n\n result = []\n for item in parameters:\n if isinstance(item, str) and not _is_url(item):\n item = convert_to_platform_path(item)\n result.append(item)\n return result\n\n\ndef _system_path_filter(_func=None, arg_slice: Optional[slice] = None):\n \"\"\"Filters function arguments to account for platform path separators.\n Optional slicing range can be specified to select specific arguments\n\n This decorator takes all (or a slice) of a method's positional arguments\n and normalizes usage of filepath separators on a per platform basis.\n\n Note: `**kwargs`, urls, and any type that is not a string are ignored\n so in such cases where path normalization is required, that should be\n handled by calling path_to_os_path directly as needed.\n\n Parameters:\n arg_slice: a slice object specifying the slice of arguments\n in the decorated method over which filepath separators are\n normalized\n \"\"\"\n\n def holder_func(func):\n @functools.wraps(func)\n def path_filter_caller(*args, **kwargs):\n args = list(args)\n if arg_slice:\n args[arg_slice] = path_to_os_path(*args[arg_slice])\n else:\n args = path_to_os_path(*args)\n return func(*args, **kwargs)\n\n return path_filter_caller\n\n if _func:\n return holder_func(_func)\n return holder_func\n\n\ndef _noop_decorator(_func=None, arg_slice: Optional[slice] = None):\n return _func if _func else lambda x: x\n\n\nif sys.platform == \"win32\":\n system_path_filter = _system_path_filter\nelse:\n system_path_filter = _noop_decorator\n\n\ndef sanitize_win_longpath(path: str) -> str:\n \"\"\"Strip Windows extended path prefix from strings\n Returns sanitized string.\n no-op if extended path prefix is not present\"\"\"\n return path.lstrip(\"\\\\\\\\?\\\\\")\n" }, { "path": "lib/spack/spack/llnl/string.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"String manipulation functions that do not have other dependencies than Python\nstandard library\n\"\"\"\n\nfrom typing import List, Optional, Sequence\n\n\ndef comma_list(sequence: Sequence[str], article: str = \"\") -> str:\n if type(sequence) is not list:\n sequence = list(sequence)\n\n if not sequence:\n return \"\"\n if len(sequence) == 1:\n return sequence[0]\n\n out = \", \".join(str(s) for s in sequence[:-1])\n if len(sequence) != 2:\n out += \",\" # oxford comma\n out += \" \"\n if article:\n out += article + \" \"\n out += str(sequence[-1])\n return out\n\n\ndef comma_or(sequence: Sequence[str]) -> str:\n \"\"\"Return a string with all the elements of the input joined by comma, but the last\n one (which is joined by ``\"or\"``).\n \"\"\"\n return comma_list(sequence, \"or\")\n\n\ndef comma_and(sequence: List[str]) -> str:\n \"\"\"Return a string with all the elements of the input joined by comma, but the last\n one (which is joined by ``\"and\"``).\n \"\"\"\n return comma_list(sequence, \"and\")\n\n\ndef ordinal(number: int) -> str:\n \"\"\"Return the ordinal representation (1st, 2nd, 3rd, etc.) for the provided number.\n\n Args:\n number: int to convert to ordinal number\n\n Returns: number's corresponding ordinal\n \"\"\"\n idx = (number % 10) << 1\n tens = number % 100 // 10\n suffix = \"th\" if tens == 1 or idx > 6 else \"thstndrd\"[idx : idx + 2]\n return f\"{number}{suffix}\"\n\n\ndef quote(sequence: List[str], q: str = \"'\") -> List[str]:\n \"\"\"Quotes each item in the input list with the quote character passed as second argument.\"\"\"\n return [f\"{q}{e}{q}\" for e in sequence]\n\n\ndef plural(n: int, singular: str, plural: Optional[str] = None, show_n: bool = True) -> str:\n \"\"\"Pluralize word by adding an s if n != 1.\n\n Arguments:\n n: number of things there are\n singular: singular form of word\n plural: optional plural form, for when it's not just singular + 's'\n show_n: whether to include n in the result string (default True)\n\n Returns:\n \"1 thing\" if n == 1 or \"n things\" if n != 1\n \"\"\"\n number = f\"{n} \" if show_n else \"\"\n if n == 1:\n return f\"{number}{singular}\"\n elif plural is not None:\n return f\"{number}{plural}\"\n else:\n return f\"{number}{singular}s\"\n" }, { "path": "lib/spack/spack/llnl/url.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"URL primitives that just require Python standard library.\"\"\"\n\nimport itertools\nimport os\nimport re\nfrom typing import Optional, Set, Tuple\nfrom urllib.parse import urlsplit, urlunsplit\n\n# Archive extensions allowed in Spack\nPREFIX_EXTENSIONS = (\"tar\", \"TAR\")\nEXTENSIONS = (\"gz\", \"bz2\", \"xz\", \"Z\")\nNO_TAR_EXTENSIONS = (\"zip\", \"tgz\", \"tbz2\", \"tbz\", \"txz\", \"whl\")\n\n# Add PREFIX_EXTENSIONS and EXTENSIONS last so that .tar.gz is matched *before* .tar or .gz\nALLOWED_ARCHIVE_TYPES = (\n tuple(\".\".join(ext) for ext in itertools.product(PREFIX_EXTENSIONS, EXTENSIONS))\n + PREFIX_EXTENSIONS\n + EXTENSIONS\n + NO_TAR_EXTENSIONS\n)\nCONTRACTION_MAP = {\"tgz\": \"tar.gz\", \"txz\": \"tar.xz\", \"tbz\": \"tar.bz2\", \"tbz2\": \"tar.bz2\"}\n\n\ndef find_list_urls(url: str) -> Set[str]:\n r\"\"\"Find good list URLs for the supplied URL.\n\n By default, returns the dirname of the archive path.\n\n Provides special treatment for the following websites, which have a\n unique list URL different from the dirname of the download URL:\n\n ========= =======================================================\n GitHub ``https://github.com///releases``\n GitLab ``https://gitlab.\\*///tags``\n BitBucket ``https://bitbucket.org///downloads/?tab=tags``\n CRAN ``https://\\*.r-project.org/src/contrib/Archive/``\n PyPI ``https://pypi.org/simple//``\n LuaRocks ``https://luarocks.org/modules//``\n ========= =======================================================\n\n Note: this function is called by ``spack versions``, ``spack checksum``,\n and ``spack create``, but not by ``spack fetch`` or ``spack install``.\n\n Parameters:\n url (str): The download URL for the package\n\n Returns:\n set: One or more list URLs for the package\n \"\"\"\n\n url_types = [\n # GitHub\n # e.g. https://github.com/llnl/callpath/archive/v1.0.1.tar.gz\n (r\"(.*github\\.com/[^/]+/[^/]+)\", lambda m: m.group(1) + \"/releases\"),\n # GitLab API endpoint\n # e.g. https://gitlab.dkrz.de/api/v4/projects/k202009%2Flibaec/repository/archive.tar.gz?sha=v1.0.2\n (\n r\"(.*gitlab[^/]+)/api/v4/projects/([^/]+)%2F([^/]+)\",\n lambda m: m.group(1) + \"/\" + m.group(2) + \"/\" + m.group(3) + \"/tags\",\n ),\n # GitLab non-API endpoint\n # e.g. https://gitlab.dkrz.de/k202009/libaec/uploads/631e85bcf877c2dcaca9b2e6d6526339/libaec-1.0.0.tar.gz\n (r\"(.*gitlab[^/]+/(?!api/v4/projects)[^/]+/[^/]+)\", lambda m: m.group(1) + \"/tags\"),\n # BitBucket\n # e.g. https://bitbucket.org/eigen/eigen/get/3.3.3.tar.bz2\n (r\"(.*bitbucket.org/[^/]+/[^/]+)\", lambda m: m.group(1) + \"/downloads/?tab=tags\"),\n # CRAN\n # e.g. https://cran.r-project.org/src/contrib/Rcpp_0.12.9.tar.gz\n # e.g. https://cloud.r-project.org/src/contrib/rgl_0.98.1.tar.gz\n (\n r\"(.*\\.r-project\\.org/src/contrib)/([^_]+)\",\n lambda m: m.group(1) + \"/Archive/\" + m.group(2),\n ),\n # PyPI\n # e.g. https://pypi.io/packages/source/n/numpy/numpy-1.19.4.zip\n # e.g. https://www.pypi.io/packages/source/n/numpy/numpy-1.19.4.zip\n # e.g. https://pypi.org/packages/source/n/numpy/numpy-1.19.4.zip\n # e.g. https://pypi.python.org/packages/source/n/numpy/numpy-1.19.4.zip\n # e.g. https://files.pythonhosted.org/packages/source/n/numpy/numpy-1.19.4.zip\n # e.g. https://pypi.io/packages/py2.py3/o/opencensus-context/opencensus_context-0.1.1-py2.py3-none-any.whl\n (\n r\"(?:pypi|pythonhosted)[^/]+/packages/[^/]+/./([^/]+)\",\n lambda m: \"https://pypi.org/simple/\" + m.group(1) + \"/\",\n ),\n # LuaRocks\n # e.g. https://luarocks.org/manifests/gvvaughan/lpeg-1.0.2-1.src.rock\n # e.g. https://luarocks.org/manifests/openresty/lua-cjson-2.1.0-1.src.rock\n (\n r\"luarocks[^/]+/(?:modules|manifests)/(?P[^/]+)/\"\n + r\"(?P.+?)-[0-9.-]*\\.src\\.rock\",\n lambda m: (\n \"https://luarocks.org/modules/\" + m.group(\"org\") + \"/\" + m.group(\"name\") + \"/\"\n ),\n ),\n ]\n\n list_urls = {os.path.dirname(url)}\n\n for pattern, fun in url_types:\n match = re.search(pattern, url)\n if match:\n list_urls.add(fun(match))\n\n return list_urls\n\n\ndef strip_query_and_fragment(url: str) -> Tuple[str, str]:\n \"\"\"Strips query and fragment from a url, then returns the base url and the suffix.\n\n Args:\n url: URL to be stripped\n\n Raises:\n ValueError: when there is any error parsing the URL\n \"\"\"\n components = urlsplit(url)\n stripped = components[:3] + (None, None)\n\n query, frag = components[3:5]\n suffix = \"\"\n if query:\n suffix += \"?\" + query\n if frag:\n suffix += \"#\" + frag\n\n return urlunsplit(stripped), suffix\n\n\nSOURCEFORGE_RE = re.compile(r\"(.*(?:sourceforge\\.net|sf\\.net)/.*)(/download)$\")\n\n\ndef split_url_on_sourceforge_suffix(url: str) -> Tuple[str, ...]:\n \"\"\"If the input is a sourceforge URL, returns base URL and ``/download`` suffix. Otherwise,\n returns the input URL and an empty string.\n \"\"\"\n match = SOURCEFORGE_RE.search(url)\n if match is not None:\n return match.groups()\n return url, \"\"\n\n\ndef has_extension(path_or_url: str, ext: str) -> bool:\n \"\"\"Returns true if the extension in input is present in path, false otherwise.\"\"\"\n prefix, _ = split_url_on_sourceforge_suffix(path_or_url)\n if not ext.startswith(r\"\\.\"):\n ext = rf\"\\.{ext}$\"\n\n if re.search(ext, prefix):\n return True\n return False\n\n\ndef extension_from_path(path_or_url: Optional[str]) -> Optional[str]:\n \"\"\"Tries to match an allowed archive extension to the input. Returns the first match,\n or None if no match was found.\n\n Raises:\n ValueError: if the input is None\n \"\"\"\n if path_or_url is None:\n raise ValueError(\"Can't call extension() on None\")\n\n for t in ALLOWED_ARCHIVE_TYPES:\n if has_extension(path_or_url, t):\n return t\n return None\n\n\ndef remove_extension(path_or_url: str, *, extension: str) -> str:\n \"\"\"Returns the input with the extension removed\"\"\"\n suffix = rf\"\\.{extension}$\"\n return re.sub(suffix, \"\", path_or_url)\n\n\ndef check_and_remove_ext(path: str, *, extension: str) -> str:\n \"\"\"Returns the input path with the extension removed, if the extension is present in path.\n Otherwise, returns the input unchanged.\n \"\"\"\n if not has_extension(path, extension):\n return path\n path, _ = split_url_on_sourceforge_suffix(path)\n return remove_extension(path, extension=extension)\n\n\ndef strip_extension(path_or_url: str, *, extension: Optional[str] = None) -> str:\n \"\"\"If a path contains the extension in input, returns the path stripped of the extension.\n Otherwise, returns the input path.\n\n If extension is None, attempts to strip any allowed extension from path.\n \"\"\"\n if extension is None:\n for t in ALLOWED_ARCHIVE_TYPES:\n if has_extension(path_or_url, ext=t):\n extension = t\n break\n else:\n return path_or_url\n\n return check_and_remove_ext(path_or_url, extension=extension)\n\n\ndef split_url_extension(url: str) -> Tuple[str, ...]:\n \"\"\"Some URLs have a query string, e.g.:\n\n 1. ``https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true``\n 2. ``http://www.apache.org/dyn/closer.cgi?path=/cassandra/1.2.0/apache-cassandra-1.2.0-rc2-bin.tar.gz``\n 3. ``https://gitlab.kitware.com/vtk/vtk/repository/archive.tar.bz2?ref=v7.0.0``\n\n In (1), the query string needs to be stripped to get at the\n extension, but in (2) & (3), the filename is IN a single final query\n argument.\n\n This strips the URL into three pieces: ``prefix``, ``ext``, and ``suffix``.\n The suffix contains anything that was stripped off the URL to\n get at the file extension. In (1), it will be ``'?raw=true'``, but\n in (2), it will be empty. In (3) the suffix is a parameter that follows\n after the file extension, e.g.:\n\n 1. ``('https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7', '.tgz', '?raw=true')``\n 2. ``('http://www.apache.org/dyn/closer.cgi?path=/cassandra/1.2.0/apache-cassandra-1.2.0-rc2-bin', '.tar.gz', None)``\n 3. ``('https://gitlab.kitware.com/vtk/vtk/repository/archive', '.tar.bz2', '?ref=v7.0.0')``\n \"\"\" # noqa: E501\n # Strip off sourceforge download suffix.\n # e.g. https://sourceforge.net/projects/glew/files/glew/2.0.0/glew-2.0.0.tgz/download\n prefix, suffix = split_url_on_sourceforge_suffix(url)\n\n ext = extension_from_path(prefix)\n if ext is not None:\n prefix = strip_extension(prefix)\n return prefix, ext, suffix\n\n try:\n prefix, suf = strip_query_and_fragment(prefix)\n except ValueError:\n # FIXME: tty.debug(\"Got error parsing path %s\" % path)\n # Ignore URL parse errors here\n return url, \"\"\n\n ext = extension_from_path(prefix)\n prefix = strip_extension(prefix)\n suffix = suf + suffix\n if ext is None:\n ext = \"\"\n\n return prefix, ext, suffix\n\n\ndef strip_version_suffixes(path_or_url: str) -> str:\n \"\"\"Some tarballs contain extraneous information after the version:\n\n * ``bowtie2-2.2.5-source``\n * ``libevent-2.0.21-stable``\n * ``cuda_8.0.44_linux.run``\n\n These strings are not part of the version number and should be ignored.\n This function strips those suffixes off and returns the remaining string.\n The goal is that the version is always the last thing in ``path``:\n\n * ``bowtie2-2.2.5``\n * ``libevent-2.0.21``\n * ``cuda_8.0.44``\n\n Args:\n path_or_url: The filename or URL for the package\n\n Returns:\n The ``path`` with any extraneous suffixes removed\n \"\"\"\n # NOTE: This could be done with complicated regexes in parse_version_offset\n # NOTE: The problem is that we would have to add these regexes to the end\n # NOTE: of every single version regex. Easier to just strip them off\n # NOTE: permanently\n\n suffix_regexes = [\n # Download type\n r\"[Ii]nstall\",\n r\"all\",\n r\"code\",\n r\"[Ss]ources?\",\n r\"file\",\n r\"full\",\n r\"single\",\n r\"with[a-zA-Z_-]+\",\n r\"rock\",\n r\"src(_0)?\",\n r\"public\",\n r\"bin\",\n r\"binary\",\n r\"run\",\n r\"[Uu]niversal\",\n r\"jar\",\n r\"complete\",\n r\"dynamic\",\n r\"oss\",\n r\"gem\",\n r\"tar\",\n r\"sh\",\n # Download version\n r\"release\",\n r\"bin\",\n r\"stable\",\n r\"[Ff]inal\",\n r\"rel\",\n r\"orig\",\n r\"dist\",\n r\"\\+\",\n # License\n r\"gpl\",\n # Arch\n # Needs to come before and after OS, appears in both orders\n r\"ia32\",\n r\"intel\",\n r\"amd64\",\n r\"linux64\",\n r\"x64\",\n r\"64bit\",\n r\"x86[_-]64\",\n r\"i586_64\",\n r\"x86\",\n r\"i[36]86\",\n r\"ppc64(le)?\",\n r\"armv?(7l|6l|64)\",\n # Other\n r\"cpp\",\n r\"gtk\",\n r\"incubating\",\n # OS\n r\"[Ll]inux(_64)?\",\n r\"LINUX\",\n r\"[Uu]ni?x\",\n r\"[Ss]un[Oo][Ss]\",\n r\"[Mm]ac[Oo][Ss][Xx]?\",\n r\"[Oo][Ss][Xx]\",\n r\"[Dd]arwin(64)?\",\n r\"[Aa]pple\",\n r\"[Ww]indows\",\n r\"[Ww]in(64|32)?\",\n r\"[Cc]ygwin(64|32)?\",\n r\"[Mm]ingw\",\n r\"centos\",\n # Arch\n # Needs to come before and after OS, appears in both orders\n r\"ia32\",\n r\"intel\",\n r\"amd64\",\n r\"linux64\",\n r\"x64\",\n r\"64bit\",\n r\"x86[_-]64\",\n r\"i586_64\",\n r\"x86\",\n r\"i[36]86\",\n r\"ppc64(le)?\",\n r\"armv?(7l|6l|64)?\",\n # PyPI wheels\n r\"-(?:py|cp)[23].*\",\n ]\n\n for regex in suffix_regexes:\n # Remove the suffix from the end of the path\n # This may be done multiple times\n path_or_url = re.sub(r\"[._-]?\" + regex + \"$\", \"\", path_or_url)\n\n return path_or_url\n\n\ndef expand_contracted_extension(extension: str) -> str:\n \"\"\"Returns the expanded version of a known contracted extension.\n\n This function maps extensions like ``.tgz`` to ``.tar.gz``. On unknown extensions,\n return the input unmodified.\n \"\"\"\n extension = extension.strip(\".\")\n return CONTRACTION_MAP.get(extension, extension)\n\n\ndef expand_contracted_extension_in_path(\n path_or_url: str, *, extension: Optional[str] = None\n) -> str:\n \"\"\"Returns the input path or URL with any contraction extension expanded.\n\n Args:\n path_or_url: path or URL to be expanded\n extension: if specified, only attempt to expand that extension\n \"\"\"\n extension = extension or extension_from_path(path_or_url)\n if extension is None:\n return path_or_url\n\n expanded = expand_contracted_extension(extension)\n if expanded != extension:\n return re.sub(rf\"{extension}\", rf\"{expanded}\", path_or_url)\n return path_or_url\n\n\ndef compression_ext_from_compressed_archive(extension: str) -> Optional[str]:\n \"\"\"Returns compression extension for a compressed archive\"\"\"\n extension = expand_contracted_extension(extension)\n for ext in EXTENSIONS:\n if ext in extension:\n return ext\n return None\n\n\ndef strip_compression_extension(path_or_url: str, ext: Optional[str] = None) -> str:\n \"\"\"Strips the compression extension from the input, and returns it. For instance,\n ``\"foo.tgz\"`` becomes ``\"foo.tar\"``.\n\n If no extension is given, try a default list of extensions.\n\n Args:\n path_or_url: input to be stripped\n ext: if given, extension to be stripped\n \"\"\"\n if not extension_from_path(path_or_url):\n return path_or_url\n\n expanded_path = expand_contracted_extension_in_path(path_or_url)\n candidates = [ext] if ext is not None else EXTENSIONS\n for current_extension in candidates:\n modified_path = check_and_remove_ext(expanded_path, extension=current_extension)\n if modified_path != expanded_path:\n return modified_path\n return expanded_path\n\n\ndef allowed_archive(path_or_url: str) -> bool:\n \"\"\"Returns true if the input is a valid archive, False otherwise.\"\"\"\n return (\n False if not path_or_url else any(path_or_url.endswith(t) for t in ALLOWED_ARCHIVE_TYPES)\n )\n\n\ndef determine_url_file_extension(path: str) -> str:\n \"\"\"This returns the type of archive a URL refers to. This is\n sometimes confusing because of URLs like:\n\n (1) ``https://github.com/petdance/ack/tarball/1.93_02``\n\n Where the URL doesn't actually contain the filename. We need\n to know what type it is so that we can appropriately name files\n in mirrors.\n \"\"\"\n match = re.search(r\"github.com/.+/(zip|tar)ball/\", path)\n if match:\n if match.group(1) == \"zip\":\n return \"zip\"\n elif match.group(1) == \"tar\":\n return \"tar.gz\"\n\n prefix, ext, suffix = split_url_extension(path)\n return ext\n" }, { "path": "lib/spack/spack/llnl/util/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n" }, { "path": "lib/spack/spack/llnl/util/argparsewriter.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport abc\nimport argparse\nimport io\nimport re\nimport sys\nfrom argparse import ArgumentParser\nfrom typing import IO, Any, Iterable, List, Optional, Sequence, Tuple, Union\n\n\nclass Command:\n \"\"\"Parsed representation of a command from argparse.\n\n This is a single command from an argparse parser. ``ArgparseWriter`` creates these and returns\n them from ``parse()``, and it passes one of these to each call to ``format()`` so that we can\n take an action for a single command.\n \"\"\"\n\n def __init__(\n self,\n prog: str,\n description: Optional[str],\n usage: str,\n positionals: List[Tuple[str, Optional[Iterable[Any]], Union[int, str, None], str]],\n optionals: List[Tuple[Sequence[str], List[str], str, Union[int, str, None], str]],\n subcommands: List[Tuple[ArgumentParser, str, str]],\n ) -> None:\n \"\"\"Initialize a new Command instance.\n\n Args:\n prog: Program name.\n description: Command description.\n usage: Command usage.\n positionals: List of positional arguments.\n optionals: List of optional arguments.\n subcommands: List of subcommand parsers.\n \"\"\"\n self.prog = prog\n self.description = description\n self.usage = usage\n self.positionals = positionals\n self.optionals = optionals\n self.subcommands = subcommands\n\n\n# NOTE: The only reason we subclass argparse.HelpFormatter is to get access to self._expand_help(),\n# ArgparseWriter is not intended to be used as a formatter_class.\nclass ArgparseWriter(argparse.HelpFormatter, abc.ABC):\n \"\"\"Analyze an argparse ArgumentParser for easy generation of help.\"\"\"\n\n def __init__(self, prog: str, out: IO = sys.stdout, aliases: bool = False) -> None:\n \"\"\"Initialize a new ArgparseWriter instance.\n\n Args:\n prog: Program name.\n out: File object to write to.\n aliases: Whether or not to include subparsers for aliases.\n \"\"\"\n super().__init__(prog)\n self.level = 0\n self.prog = prog\n self.out = out\n self.aliases = aliases\n\n def parse(self, parser: ArgumentParser, prog: str) -> Command:\n \"\"\"Parse the parser object and return the relevant components.\n\n Args:\n parser: Command parser.\n prog: Program name.\n\n Returns:\n Information about the command from the parser.\n \"\"\"\n self.parser = parser\n\n split_prog = parser.prog.split(\" \")\n split_prog[-1] = prog\n prog = \" \".join(split_prog)\n description = parser.description\n\n fmt = parser._get_formatter()\n actions = parser._actions\n groups = parser._mutually_exclusive_groups\n usage = fmt._format_usage(None, actions, groups, \"\").strip()\n\n # Go through actions and split them into optionals, positionals, and subcommands\n optionals = []\n positionals = []\n subcommands = []\n for action in actions:\n if action.option_strings:\n flags = action.option_strings\n dest_flags = fmt._format_action_invocation(action)\n nargs = action.nargs\n help = (\n self._expand_help(action)\n if action.help and action.help != argparse.SUPPRESS\n else \"\"\n )\n help = help.split(\"\\n\")[0]\n\n if action.choices is not None:\n dest = [str(choice) for choice in action.choices]\n else:\n dest = [action.dest]\n\n optionals.append((flags, dest, dest_flags, nargs, help))\n elif isinstance(action, argparse._SubParsersAction):\n for subaction in action._choices_actions:\n subparser = action._name_parser_map[subaction.dest]\n help = (\n self._expand_help(subaction)\n if subaction.help and action.help != argparse.SUPPRESS\n else \"\"\n )\n help = help.split(\"\\n\")[0]\n subcommands.append((subparser, subaction.dest, help))\n\n # Look for aliases of the form 'name (alias, ...)'\n if self.aliases and isinstance(subaction.metavar, str):\n match = re.match(r\"(.*) \\((.*)\\)\", subaction.metavar)\n if match:\n aliases = match.group(2).split(\", \")\n for alias in aliases:\n subparser = action._name_parser_map[alias]\n help = (\n self._expand_help(subaction)\n if subaction.help and action.help != argparse.SUPPRESS\n else \"\"\n )\n help = help.split(\"\\n\")[0]\n subcommands.append((subparser, alias, help))\n else:\n args = fmt._format_action_invocation(action)\n help = (\n self._expand_help(action)\n if action.help and action.help != argparse.SUPPRESS\n else \"\"\n )\n help = help.split(\"\\n\")[0]\n positionals.append((args, action.choices, action.nargs, help))\n\n return Command(prog, description, usage, positionals, optionals, subcommands)\n\n @abc.abstractmethod\n def format(self, cmd: Command) -> str:\n \"\"\"Return the string representation of a single node in the parser tree.\n\n Override this in subclasses to define how each subcommand should be displayed.\n\n Args:\n cmd: Parsed information about a command or subcommand.\n\n Returns:\n String representation of this subcommand.\n \"\"\"\n\n def _write(self, parser: ArgumentParser, prog: str, level: int = 0) -> None:\n \"\"\"Recursively write a parser.\n\n Args:\n parser: Command parser.\n prog: Program name.\n level: Current level.\n \"\"\"\n self.level = level\n\n cmd = self.parse(parser, prog)\n self.out.write(self.format(cmd))\n\n for subparser, prog, help in cmd.subcommands:\n self._write(subparser, prog, level=level + 1)\n\n def write(self, parser: ArgumentParser) -> None:\n \"\"\"Write out details about an ArgumentParser.\n\n Args:\n parser: Command parser.\n \"\"\"\n try:\n self._write(parser, self.prog)\n except BrokenPipeError:\n # Swallow pipe errors\n pass\n\n\n_rst_levels = [\"=\", \"-\", \"^\", \"~\", \":\", \"`\"]\n\n\nclass ArgparseRstWriter(ArgparseWriter):\n \"\"\"Write argparse output as rst sections.\"\"\"\n\n def __init__(\n self,\n prog: str,\n out: IO = sys.stdout,\n aliases: bool = False,\n rst_levels: Sequence[str] = _rst_levels,\n ) -> None:\n \"\"\"Initialize a new ArgparseRstWriter instance.\n\n Args:\n prog: Program name.\n out: File object to write to.\n aliases: Whether or not to include subparsers for aliases.\n rst_levels: List of characters for rst section headings.\n \"\"\"\n super().__init__(prog, out, aliases)\n self.rst_levels = rst_levels\n\n def format(self, cmd: Command) -> str:\n \"\"\"Return the string representation of a single node in the parser tree.\n\n Args:\n cmd: Parsed information about a command or subcommand.\n\n Returns:\n String representation of a node.\n \"\"\"\n string = io.StringIO()\n string.write(self.begin_command(cmd.prog))\n\n if cmd.description:\n string.write(self.description(cmd.description))\n\n string.write(self.usage(cmd.usage))\n\n if cmd.positionals:\n string.write(self.begin_positionals())\n for args, choices, nargs, help in cmd.positionals:\n string.write(self.positional(args, help))\n string.write(self.end_positionals())\n\n if cmd.optionals:\n string.write(self.begin_optionals())\n for flags, dest, dest_flags, nargs, help in cmd.optionals:\n string.write(self.optional(dest_flags, help))\n string.write(self.end_optionals())\n\n if cmd.subcommands:\n string.write(self.begin_subcommands(cmd.subcommands))\n\n return string.getvalue()\n\n def begin_command(self, prog: str) -> str:\n \"\"\"Text to print before a command.\n\n Args:\n prog: Program name.\n\n Returns:\n Text before a command.\n \"\"\"\n return \"\"\"\n----\n\n.. _{0}:\n\n{1}\n{2}\n\n\"\"\".format(prog.replace(\" \", \"-\"), prog, self.rst_levels[self.level] * len(prog))\n\n def description(self, description: str) -> str:\n \"\"\"Description of a command.\n\n Args:\n description: Command description.\n\n Returns:\n Description of a command.\n \"\"\"\n return description + \"\\n\\n\"\n\n def usage(self, usage: str) -> str:\n \"\"\"Example usage of a command.\n\n Args:\n usage: Command usage.\n\n Returns:\n Usage of a command.\n \"\"\"\n return \"\"\"\\\n.. code-block:: console\n\n {0}\n\n\"\"\".format(usage)\n\n def begin_positionals(self) -> str:\n \"\"\"Text to print before positional arguments.\n\n Returns:\n Positional arguments header.\n \"\"\"\n return \"\\n**Positional arguments**\\n\\n\"\n\n def positional(self, name: str, help: str) -> str:\n \"\"\"Description of a positional argument.\n\n Args:\n name: Argument name.\n help: Help text.\n\n Returns:\n Positional argument description.\n \"\"\"\n return \"\"\"\\\n``{0}``\n {1}\n\n\"\"\".format(name, help)\n\n def end_positionals(self) -> str:\n \"\"\"Text to print after positional arguments.\n\n Returns:\n Positional arguments footer.\n \"\"\"\n return \"\"\n\n def begin_optionals(self) -> str:\n \"\"\"Text to print before optional arguments.\n\n Returns:\n Optional arguments header.\n \"\"\"\n return \"\\n**Optional arguments**\\n\\n\"\n\n def optional(self, opts: str, help: str) -> str:\n \"\"\"Description of an optional argument.\n\n Args:\n opts: Optional argument.\n help: Help text.\n\n Returns:\n Optional argument description.\n \"\"\"\n return \"\"\"\\\n``{0}``\n {1}\n\n\"\"\".format(opts, help)\n\n def end_optionals(self) -> str:\n \"\"\"Text to print after optional arguments.\n\n Returns:\n Optional arguments footer.\n \"\"\"\n return \"\"\n\n def begin_subcommands(self, subcommands: List[Tuple[ArgumentParser, str, str]]) -> str:\n \"\"\"Table with links to other subcommands.\n\n Arguments:\n subcommands: List of subcommands.\n\n Returns:\n Subcommand linking text.\n \"\"\"\n string = \"\"\"\n**Subcommands**\n\n.. hlist::\n :columns: 4\n\n\"\"\"\n\n for cmd, _, _ in subcommands:\n prog = re.sub(r\"^[^ ]* \", \"\", cmd.prog)\n string += \" * :ref:`{0} <{1}>`\\n\".format(prog, cmd.prog.replace(\" \", \"-\"))\n\n return string + \"\\n\"\n" }, { "path": "lib/spack/spack/llnl/util/filesystem.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport collections\nimport collections.abc\nimport errno\nimport fnmatch\nimport glob\nimport hashlib\nimport io\nimport itertools\nimport numbers\nimport os\nimport pathlib\nimport posixpath\nimport re\nimport shutil\nimport stat\nimport subprocess\nimport sys\nimport tempfile\nfrom contextlib import contextmanager\nfrom itertools import accumulate\nfrom typing import (\n IO,\n Callable,\n Deque,\n Dict,\n Generator,\n Iterable,\n List,\n Match,\n Optional,\n Sequence,\n Set,\n Tuple,\n Union,\n)\n\nfrom spack.llnl.path import path_to_os_path, sanitize_win_longpath, system_path_filter\nfrom spack.llnl.util import lang, tty\nfrom spack.llnl.util.lang import dedupe, fnmatch_translate_multiple, memoized\n\nif sys.platform != \"win32\":\n import grp\n import pwd\nelse:\n import win32security\n from win32file import CreateHardLink\n\n\n__all__ = [\n \"FileFilter\",\n \"FileList\",\n \"HeaderList\",\n \"LibraryList\",\n \"ancestor\",\n \"can_access\",\n \"change_sed_delimiter\",\n \"copy_mode\",\n \"filter_file\",\n \"find\",\n \"find_first\",\n \"find_headers\",\n \"find_all_headers\",\n \"find_libraries\",\n \"find_system_libraries\",\n \"force_remove\",\n \"force_symlink\",\n \"getuid\",\n \"chgrp\",\n \"chmod_x\",\n \"copy\",\n \"install\",\n \"copy_tree\",\n \"install_tree\",\n \"is_exe\",\n \"join_path\",\n \"library_extensions\",\n \"mkdirp\",\n \"partition_path\",\n \"prefixes\",\n \"remove_dead_links\",\n \"remove_directory_contents\",\n \"remove_if_dead_link\",\n \"remove_linked_tree\",\n \"rename\",\n \"set_executable\",\n \"set_install_permissions\",\n \"touch\",\n \"touchp\",\n \"traverse_tree\",\n \"unset_executable_mode\",\n \"working_dir\",\n \"keep_modification_time\",\n \"BaseDirectoryVisitor\",\n \"visit_directory_tree\",\n]\n\nPath = Union[str, pathlib.Path]\n\nif sys.version_info < (3, 7, 4):\n # monkeypatch shutil.copystat to fix PermissionError when copying read-only\n # files on Lustre when using Python < 3.7.4\n\n def copystat(src, dst, follow_symlinks=True):\n \"\"\"Copy file metadata\n Copy the permission bits, last access time, last modification time, and\n flags from `src` to `dst`. On Linux, copystat() also copies the \"extended\n attributes\" where possible. The file contents, owner, and group are\n unaffected. `src` and `dst` are path names given as strings.\n If the optional flag `follow_symlinks` is not set, symlinks aren't\n followed if and only if both `src` and `dst` are symlinks.\n \"\"\"\n\n def _nop(args, ns=None, follow_symlinks=None):\n pass\n\n # follow symlinks (aka don't not follow symlinks)\n follow = follow_symlinks or not (islink(src) and islink(dst))\n if follow:\n # use the real function if it exists\n def lookup(name):\n return getattr(os, name, _nop)\n\n else:\n # use the real function only if it exists\n # *and* it supports follow_symlinks\n def lookup(name):\n fn = getattr(os, name, _nop)\n if sys.version_info >= (3, 3):\n if fn in os.supports_follow_symlinks: # novermin\n return fn\n return _nop\n\n st = lookup(\"stat\")(src, follow_symlinks=follow)\n mode = stat.S_IMODE(st.st_mode)\n lookup(\"utime\")(dst, ns=(st.st_atime_ns, st.st_mtime_ns), follow_symlinks=follow)\n\n # We must copy extended attributes before the file is (potentially)\n # chmod()'ed read-only, otherwise setxattr() will error with -EACCES.\n shutil._copyxattr(src, dst, follow_symlinks=follow)\n\n try:\n lookup(\"chmod\")(dst, mode, follow_symlinks=follow)\n except NotImplementedError:\n # if we got a NotImplementedError, it's because\n # * follow_symlinks=False,\n # * lchown() is unavailable, and\n # * either\n # * fchownat() is unavailable or\n # * fchownat() doesn't implement AT_SYMLINK_NOFOLLOW.\n # (it returned ENOSUP.)\n # therefore we're out of options--we simply cannot chown the\n # symlink. give up, suppress the error.\n # (which is what shutil always did in this circumstance.)\n pass\n if hasattr(st, \"st_flags\"):\n try:\n lookup(\"chflags\")(dst, st.st_flags, follow_symlinks=follow)\n except OSError as why:\n for err in \"EOPNOTSUPP\", \"ENOTSUP\":\n if hasattr(errno, err) and why.errno == getattr(errno, err):\n break\n else:\n raise\n\n shutil.copystat = copystat\n\n\ndef polite_path(components: Iterable[str]):\n \"\"\"\n Given a list of strings which are intended to be path components,\n generate a path, and format each component to avoid generating extra\n path entries.\n\n For example all \"/\", \"\\\", and \":\" characters will be replaced with\n \"_\". Other characters like \"=\" will also be replaced.\n \"\"\"\n return os.path.join(*[polite_filename(x) for x in components])\n\n\n@memoized\ndef _polite_antipattern():\n # A regex of all the characters we don't want in a filename\n return re.compile(r\"[^A-Za-z0-9_+.-]\")\n\n\ndef polite_filename(filename: str) -> str:\n \"\"\"\n Replace generally problematic filename characters with underscores.\n\n This differs from sanitize_filename in that it is more aggressive in\n changing characters in the name. For example it removes \"=\" which can\n confuse path parsing in external tools.\n \"\"\"\n # This character set applies for both Windows and Linux. It does not\n # account for reserved filenames in Windows.\n return _polite_antipattern().sub(\"_\", filename)\n\n\nif sys.platform == \"win32\":\n\n def _getuid_win32() -> Union[str, int]:\n \"\"\"Returns os getuid on non Windows\n On Windows returns 0 for admin users, login string otherwise\n This is in line with behavior from get_owner_uid which\n always returns the login string on Windows\n \"\"\"\n import ctypes\n\n # If not admin, use the string name of the login as a unique ID\n if ctypes.windll.shell32.IsUserAnAdmin() == 0:\n return os.getlogin()\n return 0\n\n getuid = _getuid_win32\nelse:\n getuid = os.getuid\n\n\n@system_path_filter\ndef _win_rename(src, dst):\n # On Windows, os.rename will fail if the destination file already exists\n # os.replace is the same as os.rename on POSIX and is MoveFileExW w/\n # the MOVEFILE_REPLACE_EXISTING flag on Windows\n # Windows invocation is abstracted behind additional logic handling\n # remaining cases of divergent behavior across platforms\n # os.replace will still fail if on Windows (but not POSIX) if the dst\n # is a symlink to a directory (all other cases have parity Windows <-> Posix)\n if os.path.islink(dst) and os.path.isdir(os.path.realpath(dst)):\n if os.path.samefile(src, dst):\n # src and dst are the same\n # do nothing and exit early\n return\n # If dst exists and is a symlink to a directory\n # we need to remove dst and then perform rename/replace\n # this is safe to do as there's no chance src == dst now\n os.remove(dst)\n os.replace(src, dst)\n\n\n@system_path_filter\ndef msdos_escape_parens(path):\n \"\"\"MS-DOS interprets parens as grouping parameters even in a quoted string\"\"\"\n if sys.platform == \"win32\":\n return path.replace(\"(\", \"^(\").replace(\")\", \"^)\")\n else:\n return path\n\n\n@system_path_filter\ndef path_contains_subdirectory(path: str, root: str) -> bool:\n \"\"\"Check if the path is a subdirectory of the root directory. Note: this is a symbolic check,\n and does not resolve symlinks.\"\"\"\n norm_root = os.path.abspath(root).rstrip(os.path.sep) + os.path.sep\n norm_path = os.path.abspath(path).rstrip(os.path.sep) + os.path.sep\n return norm_path.startswith(norm_root)\n\n\n#: This generates the library filenames that may appear on any OS.\nlibrary_extensions = [\"a\", \"la\", \"so\", \"tbd\", \"dylib\"]\n\n\ndef possible_library_filenames(library_names):\n \"\"\"Given a collection of library names like 'libfoo', generate the set of\n library filenames that may be found on the system (e.g. libfoo.so).\n \"\"\"\n lib_extensions = library_extensions\n return set(\n \".\".join((lib, extension))\n for lib, extension in itertools.product(library_names, lib_extensions)\n )\n\n\ndef paths_containing_libs(paths, library_names):\n \"\"\"Given a collection of filesystem paths, return the list of paths that\n which include one or more of the specified libraries.\n \"\"\"\n required_lib_fnames = possible_library_filenames(library_names)\n\n rpaths_to_include = []\n paths = path_to_os_path(*paths)\n for path in paths:\n fnames = set(os.listdir(path))\n if fnames & required_lib_fnames:\n rpaths_to_include.append(path)\n\n return rpaths_to_include\n\n\ndef filter_file(\n regex: str,\n repl: Union[str, Callable[[Match], str]],\n *filenames: str,\n string: bool = False,\n backup: bool = False,\n ignore_absent: bool = False,\n start_at: Optional[str] = None,\n stop_at: Optional[str] = None,\n encoding: Optional[str] = \"utf-8\",\n) -> None:\n r\"\"\"Like ``sed``, but uses Python regular expressions.\n\n Filters every line of each file through regex and replaces the file with a filtered version.\n Preserves mode of filtered files.\n\n As with :func:`re.sub`, ``repl`` can be either a string or a callable. If it is a callable, it\n is passed the match object and should return a suitable replacement string. If it is a string,\n it can contain ``\\1``, ``\\2``, etc. to represent back-substitution as sed would allow.\n\n Args:\n regex: The regular expression to search for\n repl: The string to replace matches with\n *filenames: One or more files to search and replace string: Treat regex as a plain string.\n Default it False backup: Make backup file(s) suffixed with ``~``. Default is False\n ignore_absent: Ignore any files that don't exist. Default is False\n start_at: Marker used to start applying the replacements. If a text line matches this\n marker filtering is started at the next line. All contents before the marker and the\n marker itself are copied verbatim. Default is to start filtering from the first line of\n the file.\n stop_at: Marker used to stop scanning the file further. If a text line matches this marker\n filtering is stopped and the rest of the file is copied verbatim. Default is to filter\n until the end of the file.\n encoding: The encoding to use when reading and writing the files. Default is None, which\n uses the system's default encoding.\n \"\"\"\n # Allow strings to use \\1, \\2, etc. for replacement, like sed\n if not callable(repl):\n unescaped = repl.replace(r\"\\\\\", \"\\\\\")\n\n def replace_groups_with_groupid(m: Match) -> str:\n def groupid_to_group(x):\n return m.group(int(x.group(1)))\n\n return re.sub(r\"\\\\([1-9])\", groupid_to_group, unescaped)\n\n repl = replace_groups_with_groupid\n\n if string:\n regex = re.escape(regex)\n regex_compiled = re.compile(regex)\n for path in path_to_os_path(*filenames):\n if ignore_absent and not os.path.exists(path):\n tty.debug(f'FILTER FILE: file \"{path}\" not found. Skipping to next file.')\n continue\n else:\n tty.debug(f'FILTER FILE: {path} [replacing \"{regex}\"]')\n\n fd, temp_path = tempfile.mkstemp(\n prefix=f\"{os.path.basename(path)}.\", dir=os.path.dirname(path)\n )\n os.close(fd)\n\n shutil.copy(path, temp_path)\n errored = False\n\n try:\n # Open as a text file and filter until the end of the file is reached, or we found a\n # marker in the line if it was specified. To avoid translating line endings (\\n to\n # \\r\\n and vice-versa) use newline=\"\".\n with open(\n temp_path, mode=\"r\", errors=\"surrogateescape\", newline=\"\", encoding=encoding\n ) as input_file, open(\n path, mode=\"w\", errors=\"surrogateescape\", newline=\"\", encoding=encoding\n ) as output_file:\n if start_at is None and stop_at is None: # common case, avoids branching in loop\n for line in input_file:\n output_file.write(re.sub(regex_compiled, repl, line))\n else:\n # state is -1 before start_at; 0 between; 1 after stop_at\n state = 0 if start_at is None else -1\n for line in input_file:\n if state == 0:\n if stop_at == line.strip():\n state = 1\n else:\n line = re.sub(regex_compiled, repl, line)\n elif state == -1 and start_at == line.strip():\n state = 0\n output_file.write(line)\n\n except BaseException:\n # restore the original file\n os.rename(temp_path, path)\n errored = True\n raise\n\n finally:\n if not errored and not backup:\n os.unlink(temp_path)\n\n\nclass FileFilter:\n \"\"\"\n Convenience class for repeatedly applying :func:`filter_file` to one or more files.\n\n This class allows you to specify a set of filenames and then call :meth:`filter`\n multiple times to perform search-and-replace operations using Python regular expressions,\n similar to ``sed``.\n\n Example usage::\n\n foo_c = FileFilter(\"foo.c\")\n foo_c.filter(r\"#define FOO\", \"#define BAR\")\n foo_c.filter(r\"old_func\", \"new_func\")\n \"\"\"\n\n def __init__(self, *filenames):\n self.filenames = filenames\n\n def filter(\n self,\n regex: str,\n repl: Union[str, Callable[[Match], str]],\n string: bool = False,\n backup: bool = False,\n ignore_absent: bool = False,\n start_at: Optional[str] = None,\n stop_at: Optional[str] = None,\n ) -> None:\n return filter_file(\n regex,\n repl,\n *self.filenames,\n string=string,\n backup=backup,\n ignore_absent=ignore_absent,\n start_at=start_at,\n stop_at=stop_at,\n )\n\n\ndef change_sed_delimiter(old_delim: str, new_delim: str, *filenames: str) -> None:\n \"\"\"Find all sed search/replace commands and change the delimiter.\n\n e.g., if the file contains seds that look like ``'s///'``, you can\n call ``change_sed_delimiter('/', '@', file)`` to change the\n delimiter to ``'@'``.\n\n Note that this routine will fail if the delimiter is ``'`` or ``\"``.\n Handling those is left for future work.\n\n Parameters:\n old_delim: The delimiter to search for\n new_delim: The delimiter to replace with\n *filenames: One or more files to search and replace\n \"\"\"\n assert len(old_delim) == 1\n assert len(new_delim) == 1\n\n # TODO: handle these cases one day?\n assert old_delim != '\"'\n assert old_delim != \"'\"\n assert new_delim != '\"'\n assert new_delim != \"'\"\n\n whole_lines = \"^s@([^@]*)@(.*)@[gIp]$\"\n whole_lines = whole_lines.replace(\"@\", old_delim)\n\n single_quoted = r\"'s@((?:\\\\'|[^@'])*)@((?:\\\\'|[^'])*)@[gIp]?'\"\n single_quoted = single_quoted.replace(\"@\", old_delim)\n\n double_quoted = r'\"s@((?:\\\\\"|[^@\"])*)@((?:\\\\\"|[^\"])*)@[gIp]?\"'\n double_quoted = double_quoted.replace(\"@\", old_delim)\n\n repl = r\"s@\\1@\\2@g\"\n repl = repl.replace(\"@\", new_delim)\n filenames = path_to_os_path(*filenames)\n for f in filenames:\n filter_file(whole_lines, repl, f)\n filter_file(single_quoted, \"'%s'\" % repl, f)\n filter_file(double_quoted, '\"%s\"' % repl, f)\n\n\n@contextmanager\ndef exploding_archive_catch(stage):\n # Check for an exploding tarball, i.e. one that doesn't expand to\n # a single directory. If the tarball *didn't* explode, move its\n # contents to the staging source directory & remove the container\n # directory. If the tarball did explode, just rename the tarball\n # directory to the staging source directory.\n #\n # NOTE: The tar program on Mac OS X will encode HFS metadata in\n # hidden files, which can end up *alongside* a single top-level\n # directory. We initially ignore presence of hidden files to\n # accommodate these \"semi-exploding\" tarballs but ensure the files\n # are copied to the source directory.\n\n # Expand all tarballs in their own directory to contain\n # exploding tarballs.\n tarball_container = os.path.join(stage.path, \"spack-expanded-archive\")\n mkdirp(tarball_container)\n orig_dir = os.getcwd()\n os.chdir(tarball_container)\n try:\n yield\n # catch an exploding archive on successful extraction\n os.chdir(orig_dir)\n exploding_archive_handler(tarball_container, stage)\n except Exception as e:\n # return current directory context to previous on failure\n os.chdir(orig_dir)\n raise e\n\n\n@system_path_filter\ndef exploding_archive_handler(tarball_container, stage):\n \"\"\"\n Args:\n tarball_container: where the archive was expanded to\n stage: Stage object referencing filesystem location\n where archive is being expanded\n \"\"\"\n files = os.listdir(tarball_container)\n non_hidden = [f for f in files if not f.startswith(\".\")]\n if len(non_hidden) == 1:\n src = os.path.join(tarball_container, non_hidden[0])\n if os.path.isdir(src):\n stage.srcdir = non_hidden[0]\n shutil.move(src, stage.source_path)\n if len(files) > 1:\n files.remove(non_hidden[0])\n for f in files:\n src = os.path.join(tarball_container, f)\n dest = os.path.join(stage.path, f)\n shutil.move(src, dest)\n os.rmdir(tarball_container)\n else:\n # This is a non-directory entry (e.g., a patch file) so simply\n # rename the tarball container to be the source path.\n shutil.move(tarball_container, stage.source_path)\n else:\n shutil.move(tarball_container, stage.source_path)\n\n\n@system_path_filter(arg_slice=slice(1))\ndef get_owner_uid(path, err_msg=None) -> Union[str, int]:\n \"\"\"Returns owner UID of path destination\n On non Windows this is the value of st_uid\n On Windows this is the login string associated with the\n owning user.\n\n \"\"\"\n if not os.path.exists(path):\n mkdirp(path, mode=stat.S_IRWXU)\n\n p_stat = os.stat(path)\n if p_stat.st_mode & stat.S_IRWXU != stat.S_IRWXU:\n tty.error(\n \"Expected {0} to support mode {1}, but it is {2}\".format(\n path, stat.S_IRWXU, p_stat.st_mode\n )\n )\n\n raise OSError(errno.EACCES, err_msg.format(path, path) if err_msg else \"\")\n else:\n p_stat = os.stat(path)\n\n if sys.platform != \"win32\":\n owner_uid = p_stat.st_uid\n else:\n sid = win32security.GetFileSecurity(\n path, win32security.OWNER_SECURITY_INFORMATION\n ).GetSecurityDescriptorOwner()\n owner_uid = win32security.LookupAccountSid(None, sid)[0]\n return owner_uid\n\n\n@system_path_filter\ndef set_install_permissions(path):\n \"\"\"Set appropriate permissions on the installed file.\"\"\"\n # If this points to a file maintained in a Spack prefix, it is assumed that\n # this function will be invoked on the target. If the file is outside a\n # Spack-maintained prefix, the permissions should not be modified.\n if islink(path):\n return\n if os.path.isdir(path):\n os.chmod(path, 0o755)\n else:\n os.chmod(path, 0o644)\n\n\ndef group_ids(uid: Optional[int] = None) -> List[int]:\n \"\"\"Get group ids that a uid is a member of.\n\n Arguments:\n uid: id of user, or None for current user\n\n Returns: gids of groups the user is a member of\n \"\"\"\n if sys.platform == \"win32\":\n tty.warn(\"Function is not supported on Windows\")\n return []\n\n if uid is None:\n uid = getuid()\n\n pwd_entry = pwd.getpwuid(uid)\n user = pwd_entry.pw_name\n\n # user's primary group id may not be listed in grp (i.e. /etc/group)\n # you have to check pwd for that, so start the list with that\n gids = [pwd_entry.pw_gid]\n\n return sorted(set(gids + [g.gr_gid for g in grp.getgrall() if user in g.gr_mem]))\n\n\n@system_path_filter(arg_slice=slice(1))\ndef chgrp(path, group, follow_symlinks=True):\n \"\"\"Implement the bash chgrp function on a single path\"\"\"\n if sys.platform == \"win32\":\n raise OSError(\"Function 'chgrp' is not supported on Windows\")\n\n if isinstance(group, str):\n gid = grp.getgrnam(group).gr_gid\n else:\n gid = group\n if os.stat(path).st_gid == gid:\n return\n if follow_symlinks:\n os.chown(path, -1, gid)\n else:\n os.lchown(path, -1, gid)\n\n\n@system_path_filter(arg_slice=slice(1))\ndef chmod_x(entry, perms):\n \"\"\"Implements chmod, treating all executable bits as set using the chmod\n utility's ``+X`` option.\n \"\"\"\n mode = os.stat(entry).st_mode\n if os.path.isfile(entry):\n if not mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH):\n perms &= ~stat.S_IXUSR\n perms &= ~stat.S_IXGRP\n perms &= ~stat.S_IXOTH\n os.chmod(entry, perms)\n\n\n@system_path_filter\ndef copy_mode(src, dest):\n \"\"\"Set the mode of dest to that of src unless it is a link.\"\"\"\n if islink(dest):\n return\n src_mode = os.stat(src).st_mode\n dest_mode = os.stat(dest).st_mode\n if src_mode & stat.S_IXUSR:\n dest_mode |= stat.S_IXUSR\n if src_mode & stat.S_IXGRP:\n dest_mode |= stat.S_IXGRP\n if src_mode & stat.S_IXOTH:\n dest_mode |= stat.S_IXOTH\n os.chmod(dest, dest_mode)\n\n\n@system_path_filter\ndef unset_executable_mode(path):\n mode = os.stat(path).st_mode\n mode &= ~stat.S_IXUSR\n mode &= ~stat.S_IXGRP\n mode &= ~stat.S_IXOTH\n os.chmod(path, mode)\n\n\n@system_path_filter\ndef copy(src: str, dest: str, _permissions: bool = False) -> None:\n \"\"\"Copy the file(s) ``src`` to the file or directory ``dest``.\n\n If ``dest`` specifies a directory, the file will be copied into ``dest``\n using the base filename from ``src``.\n\n ``src`` may contain glob characters.\n\n Parameters:\n src: the file(s) to copy\n dest: the destination file or directory\n _permissions: for internal use only\n\n Raises:\n OSError: if ``src`` does not match any files or directories\n ValueError: if ``src`` matches multiple files but ``dest`` is not a directory\n \"\"\"\n if _permissions:\n tty.debug(f\"Installing {src} to {dest}\")\n else:\n tty.debug(f\"Copying {src} to {dest}\")\n\n files = glob.glob(src)\n if not files:\n raise OSError(f\"No such file or directory: '{src}'\")\n if len(files) > 1 and not os.path.isdir(dest):\n raise ValueError(f\"'{src}' matches multiple files but '{dest}' is not a directory\")\n\n for src in files:\n # Expand dest to its eventual full path if it is a directory.\n dst = dest\n if os.path.isdir(dest):\n dst = join_path(dest, os.path.basename(src))\n\n shutil.copy(src, dst)\n\n if _permissions:\n set_install_permissions(dst)\n copy_mode(src, dst)\n\n\n@system_path_filter\ndef install(src: str, dest: str) -> None:\n \"\"\"Install the file(s) ``src`` to the file or directory ``dest``.\n\n Same as :py:func:`copy` with the addition of setting proper\n permissions on the installed file.\n\n Parameters:\n src: the file(s) to install\n dest: the destination file or directory\n\n Raises:\n OSError: if ``src`` does not match any files or directories\n ValueError: if ``src`` matches multiple files but ``dest`` is not a directory\n \"\"\"\n copy(src, dest, _permissions=True)\n\n\n@system_path_filter\ndef copy_tree(\n src: str,\n dest: str,\n symlinks: bool = True,\n ignore: Optional[Callable[[str], bool]] = None,\n _permissions: bool = False,\n):\n \"\"\"Recursively copy an entire directory tree rooted at ``src``.\n\n If the destination directory ``dest`` does not already exist, it will\n be created as well as missing parent directories.\n\n ``src`` may contain glob characters.\n\n If *symlinks* is true, symbolic links in the source tree are represented\n as symbolic links in the new tree and the metadata of the original links\n will be copied as far as the platform allows; if false, the contents and\n metadata of the linked files are copied to the new tree.\n\n If *ignore* is set, then each path relative to ``src`` will be passed to\n this function; the function returns whether that path should be skipped.\n\n Parameters:\n src: the directory to copy\n dest: the destination directory\n symlinks: whether or not to preserve symlinks\n ignore: function indicating which files to ignore\n _permissions: for internal use only\n\n Raises:\n OSError: if ``src`` does not match any files or directories\n ValueError: if ``src`` is a parent directory of ``dest``\n \"\"\"\n if _permissions:\n tty.debug(\"Installing {0} to {1}\".format(src, dest))\n else:\n tty.debug(\"Copying {0} to {1}\".format(src, dest))\n\n abs_dest = os.path.abspath(dest)\n if not abs_dest.endswith(os.path.sep):\n abs_dest += os.path.sep\n\n files = glob.glob(src)\n if not files:\n raise OSError(\"No such file or directory: '{0}'\".format(src), errno.ENOENT)\n\n # For Windows hard-links and junctions, the source path must exist to make a symlink. Add\n # all symlinks to this list while traversing the tree, then when finished, make all\n # symlinks at the end.\n links = []\n\n for src in files:\n abs_src = os.path.abspath(src)\n if not abs_src.endswith(os.path.sep):\n abs_src += os.path.sep\n\n # Stop early to avoid unnecessary recursion if being asked to copy\n # from a parent directory.\n if abs_dest.startswith(abs_src):\n raise ValueError(\n \"Cannot copy ancestor directory {0} into {1}\".format(abs_src, abs_dest)\n )\n\n mkdirp(abs_dest)\n\n for s, d in traverse_tree(\n abs_src,\n abs_dest,\n order=\"pre\",\n follow_links=not symlinks,\n ignore=ignore,\n follow_nonexisting=True,\n ):\n if islink(s):\n link_target = resolve_link_target_relative_to_the_link(s)\n if symlinks:\n target = readlink(s)\n if os.path.isabs(target):\n\n def escaped_path(path):\n return path.replace(\"\\\\\", r\"\\\\\")\n\n new_target = re.sub(escaped_path(abs_src), escaped_path(abs_dest), target)\n if new_target != target:\n tty.debug(\"Redirecting link {0} to {1}\".format(target, new_target))\n target = new_target\n\n links.append((target, d, s))\n continue\n\n elif os.path.isdir(link_target):\n mkdirp(d)\n else:\n shutil.copyfile(s, d)\n else:\n if os.path.isdir(s):\n mkdirp(d)\n else:\n shutil.copy2(s, d)\n\n if _permissions:\n set_install_permissions(d)\n copy_mode(s, d)\n\n for target, d, s in links:\n symlink(target, d)\n if _permissions:\n set_install_permissions(d)\n copy_mode(s, d)\n\n\n@system_path_filter\ndef install_tree(\n src: str, dest: str, symlinks: bool = True, ignore: Optional[Callable[[str], bool]] = None\n):\n \"\"\"Recursively install an entire directory tree rooted at ``src``.\n\n Same as :py:func:`copy_tree` with the addition of setting proper\n permissions on the installed files and directories.\n\n Parameters:\n src: the directory to install\n dest: the destination directory\n symlinks: whether or not to preserve symlinks\n ignore: function indicating which files to ignore\n\n Raises:\n OSError: if ``src`` does not match any files or directories\n ValueError: if ``src`` is a parent directory of ``dest``\n \"\"\"\n copy_tree(src, dest, symlinks=symlinks, ignore=ignore, _permissions=True)\n\n\n@system_path_filter\ndef is_exe(path) -> bool:\n \"\"\"Returns :obj:`True` iff the specified path exists, is a regular file, and has executable\n permissions for the current process.\"\"\"\n return os.path.isfile(path) and os.access(path, os.X_OK)\n\n\ndef has_shebang(path) -> bool:\n \"\"\"Returns whether a path has a shebang line. Returns False if the file cannot be opened.\"\"\"\n try:\n with open(path, \"rb\") as f:\n return f.read(2) == b\"#!\"\n except OSError:\n return False\n\n\n@system_path_filter\ndef is_nonsymlink_exe_with_shebang(path):\n \"\"\"Returns whether the path is an executable regular file with a shebang. Returns False too\n when the path is a symlink to a script, and also when the file cannot be opened.\"\"\"\n try:\n st = os.lstat(path)\n except OSError:\n return False\n\n # Should not be a symlink\n if stat.S_ISLNK(st.st_mode):\n return False\n\n # Should be executable\n if not st.st_mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH):\n return False\n\n return has_shebang(path)\n\n\n@system_path_filter(arg_slice=slice(1))\ndef chgrp_if_not_world_writable(path, group):\n \"\"\"chgrp path to group if path is not world writable\"\"\"\n mode = os.stat(path).st_mode\n if not mode & stat.S_IWOTH:\n chgrp(path, group)\n\n\ndef mkdirp(\n *paths: str,\n mode: Optional[int] = None,\n group: Optional[Union[str, int]] = None,\n default_perms: Optional[str] = None,\n):\n \"\"\"Creates a directory, as well as parent directories if needed.\n\n Arguments:\n paths: paths to create with mkdirp\n mode: optional permissions to set on the created directory -- use OS default\n if not provided\n group: optional group for permissions of final created directory -- use OS\n default if not provided. Only used if world write permissions are not set\n default_perms: one of ``\"parents\"`` or ``\"args\"``. The default permissions that are set for\n directories that are not themselves an argument for mkdirp. ``\"parents\"`` means\n intermediate directories get the permissions of their direct parent directory,\n ``\"args\"`` means intermediate get the same permissions specified in the arguments to\n mkdirp -- default value is ``\"args\"``\n \"\"\"\n default_perms = default_perms or \"args\"\n paths = path_to_os_path(*paths)\n for path in paths:\n if not os.path.exists(path):\n try:\n last_parent, intermediate_folders = longest_existing_parent(path)\n\n # create folders\n os.makedirs(path)\n\n # leaf folder permissions\n if mode is not None:\n os.chmod(path, mode)\n if group:\n chgrp_if_not_world_writable(path, group)\n if mode is not None:\n os.chmod(path, mode) # reset sticky grp bit post chgrp\n\n # for intermediate folders, change mode just for newly created\n # ones and if mode_intermediate has been specified, otherwise\n # intermediate folders list is not populated at all and default\n # OS mode will be used\n if default_perms == \"args\":\n intermediate_mode = mode\n intermediate_group = group\n elif default_perms == \"parents\":\n stat_info = os.stat(last_parent)\n intermediate_mode = stat_info.st_mode\n intermediate_group = stat_info.st_gid\n else:\n msg = \"Invalid value: '%s'. \" % default_perms\n msg += \"Choose from 'args' or 'parents'.\"\n raise ValueError(msg)\n\n for intermediate_path in reversed(intermediate_folders):\n if intermediate_mode is not None:\n os.chmod(intermediate_path, intermediate_mode)\n if intermediate_group is not None:\n chgrp_if_not_world_writable(intermediate_path, intermediate_group)\n if intermediate_mode is not None:\n os.chmod(\n intermediate_path, intermediate_mode\n ) # reset sticky bit after\n\n except OSError as e:\n if e.errno != errno.EEXIST or not os.path.isdir(path):\n raise e\n elif not os.path.isdir(path):\n raise OSError(errno.EEXIST, \"File already exists\", path)\n\n\ndef longest_existing_parent(path: str) -> Tuple[str, List[str]]:\n \"\"\"Return the last existing parent and a list of all intermediate directories\n to be created for the directory passed as input.\n\n Args:\n path: directory to be created\n \"\"\"\n # detect missing intermediate folders\n intermediate_folders = []\n last_parent = \"\"\n intermediate_path = os.path.dirname(path)\n while intermediate_path:\n if os.path.lexists(intermediate_path):\n last_parent = intermediate_path\n break\n\n intermediate_folders.append(intermediate_path)\n intermediate_path = os.path.dirname(intermediate_path)\n return last_parent, intermediate_folders\n\n\n@system_path_filter\ndef force_remove(*paths: str) -> None:\n \"\"\"Remove files without printing errors. Like ``rm -f``, does NOT\n remove directories.\"\"\"\n for path in paths:\n try:\n os.remove(path)\n except OSError:\n pass\n\n\n@contextmanager\n@system_path_filter\ndef working_dir(dirname: str, *, create: bool = False):\n \"\"\"Context manager to change the current working directory to ``dirname``.\n\n Args:\n dirname: the directory to change to\n create: if :obj:`True`, create the directory if it does not exist\n\n Example usage::\n\n with working_dir(\"/path/to/dir\"):\n # do something in /path/to/dir\n pass\n \"\"\"\n if create:\n mkdirp(dirname)\n\n orig_dir = os.getcwd()\n os.chdir(dirname)\n try:\n yield\n finally:\n os.chdir(orig_dir)\n\n\nclass CouldNotRestoreDirectoryBackup(RuntimeError):\n def __init__(self, inner_exception, outer_exception):\n self.inner_exception = inner_exception\n self.outer_exception = outer_exception\n\n\n@contextmanager\n@system_path_filter\ndef replace_directory_transaction(directory_name):\n \"\"\"Temporarily renames a directory in the same parent dir. If the operations\n executed within the context manager don't raise an exception, the renamed directory\n is deleted. If there is an exception, the move is undone.\n\n Args:\n directory_name (path): absolute path of the directory name\n\n Returns:\n temporary directory where ``directory_name`` has been moved\n \"\"\"\n\n # Check the input is indeed a directory with absolute path.\n # Raise before anything is done to avoid moving the wrong directory\n directory_name = os.path.abspath(directory_name)\n assert os.path.isdir(directory_name), \"Not a directory: \" + directory_name\n\n # Note: directory_name is normalized here, meaning the trailing slash is dropped,\n # so dirname is the directory's parent not the directory itself.\n tmpdir = tempfile.mkdtemp(dir=os.path.dirname(directory_name), prefix=\".backup\")\n\n # We have to jump through hoops to support Windows, since\n # os.rename(directory_name, tmpdir) errors there.\n backup_dir = os.path.join(tmpdir, \"backup\")\n os.rename(directory_name, backup_dir)\n tty.debug(\"Directory moved [src={0}, dest={1}]\".format(directory_name, backup_dir))\n\n try:\n yield backup_dir\n except (Exception, KeyboardInterrupt, SystemExit) as inner_exception:\n # Try to recover the original directory, if this fails, raise a\n # composite exception.\n try:\n # Delete what was there, before copying back the original content\n if os.path.exists(directory_name):\n shutil.rmtree(directory_name)\n os.rename(backup_dir, directory_name)\n except Exception as outer_exception:\n raise CouldNotRestoreDirectoryBackup(inner_exception, outer_exception)\n\n tty.debug(\"Directory recovered [{0}]\".format(directory_name))\n raise\n else:\n # Otherwise delete the temporary directory\n shutil.rmtree(tmpdir, ignore_errors=True)\n tty.debug(\"Temporary directory deleted [{0}]\".format(tmpdir))\n\n\n@system_path_filter\ndef hash_directory(directory, ignore=[]):\n \"\"\"Hashes recursively the content of a directory.\n\n Args:\n directory (path): path to a directory to be hashed\n\n Returns:\n hash of the directory content\n \"\"\"\n assert os.path.isdir(directory), '\"directory\" must be a directory!'\n\n md5_hash = hashlib.md5()\n\n # Adapted from https://stackoverflow.com/a/3431835/771663\n for root, dirs, files in os.walk(directory):\n for name in sorted(files):\n filename = os.path.join(root, name)\n if filename not in ignore:\n # TODO: if caching big files becomes an issue, convert this to\n # TODO: read in chunks. Currently it's used only for testing\n # TODO: purposes.\n with open(filename, \"rb\") as f:\n md5_hash.update(f.read())\n\n return md5_hash.hexdigest()\n\n\n@contextmanager\n@system_path_filter\ndef write_tmp_and_move(filename: str, *, encoding: Optional[str] = None):\n \"\"\"Write to a temporary file, then move into place.\"\"\"\n dirname = os.path.dirname(filename)\n basename = os.path.basename(filename)\n tmp = os.path.join(dirname, \".%s.tmp\" % basename)\n with open(tmp, \"w\", encoding=encoding) as f:\n yield f\n shutil.move(tmp, filename)\n\n\n@system_path_filter\ndef touch(path):\n \"\"\"Creates an empty file at the specified path.\"\"\"\n if sys.platform == \"win32\":\n perms = os.O_WRONLY | os.O_CREAT\n else:\n perms = os.O_WRONLY | os.O_CREAT | os.O_NONBLOCK | os.O_NOCTTY\n fd = None\n try:\n fd = os.open(path, perms)\n os.utime(path, None)\n finally:\n if fd is not None:\n os.close(fd)\n\n\n@system_path_filter\ndef touchp(path):\n \"\"\"Like ``touch``, but creates any parent directories needed for the file.\"\"\"\n mkdirp(os.path.dirname(path))\n touch(path)\n\n\n@system_path_filter\ndef force_symlink(src: str, dest: str) -> None:\n \"\"\"Create a symlink at ``dest`` pointing to ``src``. Similar to ``ln -sf``.\"\"\"\n try:\n symlink(src, dest)\n except OSError:\n os.remove(dest)\n symlink(src, dest)\n\n\n@system_path_filter\ndef join_path(prefix, *args) -> str:\n \"\"\"Alias for :func:`os.path.join`\"\"\"\n path = str(prefix)\n for elt in args:\n path = os.path.join(path, str(elt))\n return path\n\n\n@system_path_filter\ndef ancestor(dir, n=1):\n \"\"\"Get the nth ancestor of a directory.\"\"\"\n parent = os.path.abspath(dir)\n for i in range(n):\n parent = os.path.dirname(parent)\n return parent\n\n\n@system_path_filter\ndef get_single_file(directory):\n fnames = os.listdir(directory)\n if len(fnames) != 1:\n raise ValueError(\"Expected exactly 1 file, got {0}\".format(str(len(fnames))))\n return fnames[0]\n\n\n@system_path_filter\ndef windows_sfn(path: os.PathLike):\n \"\"\"Returns 8.3 Filename (SFN) representation of\n path\n\n 8.3 Filenames (SFN or short filename) is a file\n naming convention used prior to Win95 that Windows\n still (and will continue to) support. This convention\n caps filenames at 8 characters, and most importantly\n does not allow for spaces in addition to other specifications.\n The scheme is generally the same as a normal Windows\n file scheme, but all spaces are removed and the filename\n is capped at 6 characters. The remaining characters are\n replaced with ~N where N is the number file in a directory\n that a given file represents i.e. Program Files and Program Files (x86)\n would be PROGRA~1 and PROGRA~2 respectively.\n Further, all file/directory names are all caps (although modern Windows\n is case insensitive in practice).\n Conversion is accomplished by fileapi.h GetShortPathNameW\n\n Returns paths in 8.3 Filename form\n\n Note: this method is a no-op on Linux\n\n Args:\n path: Path to be transformed into SFN (8.3 filename) format\n \"\"\"\n # This should not be run-able on linux/macos\n if sys.platform != \"win32\":\n return path\n path = str(path)\n import ctypes\n\n k32 = ctypes.WinDLL(\"kernel32\", use_last_error=True)\n # Method with null values returns size of short path name\n sz = k32.GetShortPathNameW(path, None, 0)\n # stub Windows types TCHAR[LENGTH]\n TCHAR_arr = ctypes.c_wchar * sz\n ret_str = TCHAR_arr()\n k32.GetShortPathNameW(path, ctypes.byref(ret_str), sz)\n return ret_str.value\n\n\n@contextmanager\ndef temp_cwd(ignore_cleanup_errors=False):\n tmp_dir = tempfile.mkdtemp()\n try:\n with working_dir(tmp_dir):\n yield tmp_dir\n finally:\n kwargs = {}\n if sys.platform == \"win32\" or ignore_cleanup_errors:\n kwargs[\"ignore_errors\"] = False\n kwargs[\"onerror\"] = readonly_file_handler(ignore_errors=True)\n shutil.rmtree(tmp_dir, **kwargs)\n\n\n@system_path_filter\ndef can_access(file_name):\n \"\"\"True if the current process has read and write access to the file.\"\"\"\n return os.access(file_name, os.R_OK | os.W_OK)\n\n\n@system_path_filter\ndef traverse_tree(\n source_root: str,\n dest_root: str,\n rel_path: str = \"\",\n *,\n order: str = \"pre\",\n ignore: Optional[Callable[[str], bool]] = None,\n follow_nonexisting: bool = True,\n follow_links: bool = False,\n):\n \"\"\"Traverse two filesystem trees simultaneously.\n\n Walks the LinkTree directory in pre or post order. Yields each\n file in the source directory with a matching path from the dest\n directory, along with whether the file is a directory.\n e.g., for this tree::\n\n root/\n a/\n file1\n file2\n b/\n file3\n\n When called on dest, this yields::\n\n (\"root\", \"dest\")\n (\"root/a\", \"dest/a\")\n (\"root/a/file1\", \"dest/a/file1\")\n (\"root/a/file2\", \"dest/a/file2\")\n (\"root/b\", \"dest/b\")\n (\"root/b/file3\", \"dest/b/file3\")\n\n Keyword Arguments:\n order (str): Whether to do pre- or post-order traversal. Accepted\n values are ``\"pre\"`` and ``\"post\"``\n ignore (typing.Callable): function indicating which files to ignore. This will also\n ignore symlinks if they point to an ignored file (regardless of whether the symlink\n is explicitly ignored); note this only supports one layer of indirection (i.e. if\n you have x -> y -> z, and z is ignored but x/y are not, then y would be ignored\n but not x). To avoid this, make sure the ignore function also ignores the symlink\n paths too.\n follow_nonexisting (bool): Whether to descend into directories in\n ``src`` that do not exit in ``dest``. Default is True\n follow_links (bool): Whether to descend into symlinks in ``src``\n \"\"\"\n if order not in (\"pre\", \"post\"):\n raise ValueError(\"Order must be 'pre' or 'post'.\")\n\n # List of relative paths to ignore under the src root.\n ignore = ignore or (lambda filename: False)\n\n # Don't descend into ignored directories\n if ignore(rel_path):\n return\n\n source_path = os.path.join(source_root, rel_path)\n dest_path = os.path.join(dest_root, rel_path)\n\n # preorder yields directories before children\n if order == \"pre\":\n yield (source_path, dest_path)\n\n for f in os.listdir(source_path):\n source_child = os.path.join(source_path, f)\n dest_child = os.path.join(dest_path, f)\n rel_child = os.path.join(rel_path, f)\n\n # If the source path is a link and the link's source is ignored, then ignore the link too,\n # but only do this if the ignore is defined.\n if ignore is not None:\n if islink(source_child) and not follow_links:\n target = readlink(source_child)\n all_parents = accumulate(target.split(os.sep), lambda x, y: os.path.join(x, y))\n if any(map(ignore, all_parents)):\n tty.warn(\n f\"Skipping {source_path} because the source or a part of the source's \"\n f\"path is included in the ignores.\"\n )\n continue\n\n # Treat as a directory\n # TODO: for symlinks, os.path.isdir looks for the link target. If the\n # target is relative to the link, then that may not resolve properly\n # relative to our cwd - see resolve_link_target_relative_to_the_link\n if os.path.isdir(source_child) and (follow_links or not islink(source_child)):\n # When follow_nonexisting isn't set, don't descend into dirs\n # in source that do not exist in dest\n if follow_nonexisting or os.path.exists(dest_child):\n tuples = traverse_tree(\n source_root,\n dest_root,\n rel_child,\n order=order,\n ignore=ignore,\n follow_nonexisting=follow_nonexisting,\n follow_links=follow_links,\n )\n for t in tuples:\n yield t\n\n # Treat as a file.\n elif not ignore(os.path.join(rel_path, f)):\n yield (source_child, dest_child)\n\n if order == \"post\":\n yield (source_path, dest_path)\n\n\nclass BaseDirectoryVisitor:\n \"\"\"Base class and interface for :py:func:`visit_directory_tree`.\"\"\"\n\n def visit_file(self, root: str, rel_path: str, depth: int) -> None:\n \"\"\"Handle the non-symlink file at ``os.path.join(root, rel_path)``\n\n Parameters:\n root: root directory\n rel_path: relative path to current file from ``root``\n depth (int): depth of current file from the ``root`` directory\"\"\"\n pass\n\n def visit_symlinked_file(self, root: str, rel_path: str, depth) -> None:\n \"\"\"Handle the symlink to a file at ``os.path.join(root, rel_path)``. Note: ``rel_path`` is\n the location of the symlink, not to what it is pointing to. The symlink may be dangling.\n\n Parameters:\n root: root directory\n rel_path: relative path to current symlink from ``root``\n depth: depth of current symlink from the ``root`` directory\"\"\"\n pass\n\n def before_visit_dir(self, root: str, rel_path: str, depth: int) -> bool:\n \"\"\"Return True from this function to recurse into the directory at\n os.path.join(root, rel_path). Return False in order not to recurse further.\n\n Parameters:\n root: root directory\n rel_path: relative path to current directory from ``root``\n depth: depth of current directory from the ``root`` directory\n\n Returns:\n bool: ``True`` when the directory should be recursed into. ``False`` when\n not\"\"\"\n return False\n\n def before_visit_symlinked_dir(self, root: str, rel_path: str, depth: int) -> bool:\n \"\"\"Return ``True`` to recurse into the symlinked directory and ``False`` in order not to.\n Note: ``rel_path`` is the path to the symlink itself. Following symlinked directories\n blindly can cause infinite recursion due to cycles.\n\n Parameters:\n root: root directory\n rel_path: relative path to current symlink from ``root``\n depth: depth of current symlink from the ``root`` directory\n\n Returns:\n bool: ``True`` when the directory should be recursed into. ``False`` when\n not\"\"\"\n return False\n\n def after_visit_dir(self, root: str, rel_path: str, depth: int) -> None:\n \"\"\"Called after recursion into ``rel_path`` finished. This function is not called when\n ``rel_path`` was not recursed into.\n\n Parameters:\n root: root directory\n rel_path: relative path to current directory from ``root``\n depth: depth of current directory from the ``root`` directory\"\"\"\n pass\n\n def after_visit_symlinked_dir(self, root: str, rel_path: str, depth: int) -> None:\n \"\"\"Called after recursion into ``rel_path`` finished. This function is not called when\n ``rel_path`` was not recursed into.\n\n Parameters:\n root: root directory\n rel_path: relative path to current symlink from ``root``\n depth: depth of current symlink from the ``root`` directory\"\"\"\n pass\n\n\ndef visit_directory_tree(\n root: str, visitor: BaseDirectoryVisitor, rel_path: str = \"\", depth: int = 0\n):\n \"\"\"Recurses the directory root depth-first through a visitor pattern using the interface from\n :py:class:`BaseDirectoryVisitor`\n\n Parameters:\n root: path of directory to recurse into\n visitor: what visitor to use\n rel_path: current relative path from the root\n depth: current depth from the root\n \"\"\"\n dir = os.path.join(root, rel_path)\n dir_entries = sorted(os.scandir(dir), key=lambda d: d.name)\n\n for f in dir_entries:\n rel_child = os.path.join(rel_path, f.name)\n islink = f.is_symlink()\n # On Windows, symlinks to directories are distinct from symlinks to files, and it is\n # possible to create a broken symlink to a directory (e.g. using os.symlink without\n # `target_is_directory=True`), invoking `isdir` on a symlink on Windows that is broken in\n # this manner will result in an error. In this case we can work around the issue by reading\n # the target and resolving the directory ourselves\n try:\n isdir = f.is_dir()\n except OSError as e:\n if sys.platform == \"win32\" and e.errno == errno.EACCES and islink:\n # if path is a symlink, determine destination and evaluate file vs directory\n link_target = resolve_link_target_relative_to_the_link(f)\n # link_target might be relative but resolve_link_target_relative_to_the_link\n # will ensure that if so, that it is relative to the CWD and therefore makes sense\n isdir = os.path.isdir(link_target)\n else:\n raise e\n\n if not isdir and not islink:\n # handle non-symlink files\n visitor.visit_file(root, rel_child, depth)\n elif not isdir:\n visitor.visit_symlinked_file(root, rel_child, depth)\n elif not islink and visitor.before_visit_dir(root, rel_child, depth):\n # Handle ordinary directories\n visit_directory_tree(root, visitor, rel_child, depth + 1)\n visitor.after_visit_dir(root, rel_child, depth)\n elif islink and visitor.before_visit_symlinked_dir(root, rel_child, depth):\n # Handle symlinked directories\n visit_directory_tree(root, visitor, rel_child, depth + 1)\n visitor.after_visit_symlinked_dir(root, rel_child, depth)\n\n\n@system_path_filter\ndef set_executable(path):\n \"\"\"Set the executable bit on a file or directory.\"\"\"\n mode = os.stat(path).st_mode\n if mode & stat.S_IRUSR:\n mode |= stat.S_IXUSR\n if mode & stat.S_IRGRP:\n mode |= stat.S_IXGRP\n if mode & stat.S_IROTH:\n mode |= stat.S_IXOTH\n os.chmod(path, mode)\n\n\n@system_path_filter\ndef recursive_mtime_greater_than(path: str, time: float) -> bool:\n \"\"\"Returns true if any file or dir recursively under `path` has mtime greater than `time`.\"\"\"\n # use bfs order to increase likelihood of early return\n queue: Deque[str] = collections.deque([path])\n\n if os.stat(path).st_mtime > time:\n return True\n\n while queue:\n current = queue.popleft()\n\n try:\n entries = os.scandir(current)\n except OSError:\n continue\n\n with entries:\n for entry in entries:\n try:\n st = entry.stat(follow_symlinks=False)\n except OSError:\n continue\n\n if st.st_mtime > time:\n return True\n\n if entry.is_dir(follow_symlinks=False):\n queue.append(entry.path)\n\n return False\n\n\n@system_path_filter\ndef remove_empty_directories(root):\n \"\"\"Ascend up from the leaves accessible from `root` and remove empty\n directories.\n\n Parameters:\n root (str): path where to search for empty directories\n \"\"\"\n for dirpath, subdirs, files in os.walk(root, topdown=False):\n for sd in subdirs:\n sdp = os.path.join(dirpath, sd)\n try:\n os.rmdir(sdp)\n except OSError:\n pass\n\n\n@system_path_filter\ndef remove_dead_links(root):\n \"\"\"Recursively removes any dead link that is present in root.\n\n Parameters:\n root (str): path where to search for dead links\n \"\"\"\n for dirpath, subdirs, files in os.walk(root, topdown=False):\n for f in files:\n path = join_path(dirpath, f)\n remove_if_dead_link(path)\n\n\n@system_path_filter\ndef remove_if_dead_link(path):\n \"\"\"Removes the argument if it is a dead link.\n\n Parameters:\n path (str): The potential dead link\n \"\"\"\n if islink(path) and not os.path.exists(path):\n os.unlink(path)\n\n\ndef readonly_file_handler(ignore_errors=False):\n # TODO: generate stages etc. with write permissions wherever\n # so this callback is no-longer required\n \"\"\"\n Generate callback for shutil.rmtree to handle permissions errors on\n Windows. Some files may unexpectedly lack write permissions even\n though they were generated by Spack on behalf of the user (e.g. the\n stage), so this callback will detect such cases and modify the\n permissions if that is the issue. For other errors, the fallback\n is either to raise (if ignore_errors is False) or ignore (if\n ignore_errors is True). This is only intended for Windows systems\n and will raise a separate error if it is ever invoked (by accident)\n on a non-Windows system.\n \"\"\"\n\n def error_remove_readonly(func, path, exc):\n if sys.platform != \"win32\":\n raise RuntimeError(\"This method should only be invoked on Windows\")\n excvalue = exc[1]\n if (\n sys.platform == \"win32\"\n and func in (os.rmdir, os.remove, os.unlink)\n and excvalue.errno == errno.EACCES\n ):\n # change the file to be readable,writable,executable: 0777\n os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)\n # retry\n func(path)\n elif not ignore_errors:\n raise\n\n return error_remove_readonly\n\n\n@system_path_filter\ndef remove_linked_tree(path: str) -> None:\n \"\"\"Removes a directory and its contents.\n\n If the directory is a symlink, follows the link and removes the real\n directory before removing the link.\n\n This method will force-delete files on Windows\n\n Parameters:\n path: Directory to be removed\n \"\"\"\n kwargs: dict = {\"ignore_errors\": True}\n\n # Windows readonly files cannot be removed by Python\n # directly.\n if sys.platform == \"win32\":\n kwargs[\"ignore_errors\"] = False\n kwargs[\"onerror\"] = readonly_file_handler(ignore_errors=True)\n\n if os.path.exists(path):\n if islink(path):\n shutil.rmtree(os.path.realpath(path), **kwargs)\n os.unlink(path)\n else:\n if sys.platform == \"win32\":\n # Adding this prefix allows shutil to remove long paths on windows\n # https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=registry\n long_path_pfx = \"\\\\\\\\?\\\\\"\n if not path.startswith(long_path_pfx):\n path = long_path_pfx + path\n shutil.rmtree(path, **kwargs)\n\n\n@contextmanager\n@system_path_filter\ndef safe_remove(*files_or_dirs):\n \"\"\"Context manager to remove the files passed as input, but restore\n them in case any exception is raised in the context block.\n\n Args:\n *files_or_dirs: glob expressions for files or directories\n to be removed\n\n Returns:\n Dictionary that maps deleted files to their temporary copy\n within the context block.\n \"\"\"\n # Find all the files or directories that match\n glob_matches = [glob.glob(x) for x in files_or_dirs]\n # Sort them so that shorter paths like \"/foo/bar\" come before\n # nested paths like \"/foo/bar/baz.yaml\". This simplifies the\n # handling of temporary copies below\n sorted_matches = sorted([os.path.abspath(x) for x in itertools.chain(*glob_matches)], key=len)\n\n # Copy files and directories in a temporary location\n removed, dst_root = {}, tempfile.mkdtemp()\n try:\n for id, file_or_dir in enumerate(sorted_matches):\n # The glob expression at the top ensures that the file/dir exists\n # at the time we enter the loop. Double check here since it might\n # happen that a previous iteration of the loop already removed it.\n # This is the case, for instance, if we remove the directory\n # \"/foo/bar\" before the file \"/foo/bar/baz.yaml\".\n if not os.path.exists(file_or_dir):\n continue\n # The monotonic ID is a simple way to make the filename\n # or directory name unique in the temporary folder\n basename = os.path.basename(file_or_dir) + \"-{0}\".format(id)\n temporary_path = os.path.join(dst_root, basename)\n shutil.move(file_or_dir, temporary_path)\n removed[file_or_dir] = temporary_path\n yield removed\n except BaseException:\n # Restore the files that were removed\n for original_path, temporary_path in removed.items():\n shutil.move(temporary_path, original_path)\n raise\n\n\ndef find_first(root: str, files: Union[Iterable[str], str], bfs_depth: int = 2) -> Optional[str]:\n \"\"\"Find the first file matching a pattern.\n\n The following\n\n .. code-block:: console\n\n $ find /usr -name 'abc*' -o -name 'def*' -quit\n\n is equivalent to:\n\n >>> find_first(\"/usr\", [\"abc*\", \"def*\"])\n\n Any glob pattern supported by fnmatch can be used.\n\n The search order of this method is breadth-first over directories,\n until depth bfs_depth, after which depth-first search is used.\n\n Parameters:\n root: The root directory to start searching from\n files: File pattern(s) to search for\n bfs_depth: (advanced) parameter that specifies at which\n depth to switch to depth-first search.\n\n Returns:\n The matching file or :data:`None` when no file is found.\n \"\"\"\n if isinstance(files, str):\n files = [files]\n return FindFirstFile(root, *files, bfs_depth=bfs_depth).find()\n\n\ndef find(\n root: Union[Path, Sequence[Path]],\n files: Union[str, Sequence[str]],\n recursive: bool = True,\n max_depth: Optional[int] = None,\n) -> List[str]:\n \"\"\"Finds all files matching the patterns from ``files`` starting from ``root``. This function\n returns a deterministic result for the same input and directory structure when run multiple\n times. Symlinked directories are followed, and unique directories are searched only once. Each\n matching file is returned only once at lowest depth in case multiple paths exist due to\n symlinked directories.\n\n Accepts any glob characters accepted by :py:func:`fnmatch.fnmatch`:\n\n ========== ====================================\n Pattern Meaning\n ========== ====================================\n ``*`` matches one or more characters\n ``?`` matches any single character\n ``[seq]`` matches any character in ``seq``\n ``[!seq]`` matches any character not in ``seq``\n ========== ====================================\n\n Examples:\n\n >>> find(\"/usr\", \"*.txt\", recursive=True, max_depth=2)\n\n finds all files with the extension ``.txt`` in the directory ``/usr`` and subdirectories up to\n depth 2.\n\n >>> find([\"/usr\", \"/var\"], [\"*.txt\", \"*.log\"], recursive=True)\n\n finds all files with the extension ``.txt`` or ``.log`` in the directories ``/usr`` and\n ``/var`` at any depth.\n\n >>> find(\"/usr\", \"GL/*.h\", recursive=True)\n\n finds all header files in a directory GL at any depth in the directory ``/usr``.\n\n Parameters:\n root: One or more root directories to start searching from\n files: One or more filename patterns to search for\n recursive: if False search only root, if True descends from roots. Defaults to True.\n max_depth: if set, don't search below this depth. Cannot be set if recursive is False\n\n Returns a list of absolute, matching file paths.\n \"\"\"\n if isinstance(root, (str, pathlib.Path)):\n root = [root]\n elif not isinstance(root, collections.abc.Sequence):\n raise TypeError(f\"'root' arg must be a path or a sequence of paths, not '{type(root)}']\")\n\n if isinstance(files, str):\n files = [files]\n elif not isinstance(files, collections.abc.Sequence):\n raise TypeError(f\"'files' arg must be str or a sequence of str, not '{type(files)}']\")\n\n # If recursive is false, max_depth can only be None or 0\n if max_depth and not recursive:\n raise ValueError(f\"max_depth ({max_depth}) cannot be set if recursive is False\")\n\n tty.debug(f\"Find (max depth = {max_depth}): {root} {files}\")\n if not recursive:\n max_depth = 0\n elif max_depth is None:\n max_depth = sys.maxsize\n result = _find_max_depth(root, files, max_depth)\n tty.debug(f\"Find complete: {root} {files}\")\n return result\n\n\ndef _log_file_access_issue(e: OSError, path: str) -> None:\n tty.debug(f\"find must skip {path}: {e}\")\n\n\ndef _file_id(s: os.stat_result) -> Tuple[int, int]:\n # Note: on windows, st_ino is the file index and st_dev is the volume serial number. See\n # https://github.com/python/cpython/blob/3.9/Python/fileutils.c\n return (s.st_ino, s.st_dev)\n\n\ndef _dedupe_files(paths: List[str]) -> List[str]:\n \"\"\"Deduplicate files by inode and device, dropping files that cannot be accessed.\"\"\"\n unique_files: List[str] = []\n # tuple of (inode, device) for each file without following symlinks\n visited: Set[Tuple[int, int]] = set()\n for path in paths:\n try:\n stat_info = os.lstat(path)\n except OSError as e:\n _log_file_access_issue(e, path)\n continue\n file_id = _file_id(stat_info)\n if file_id not in visited:\n unique_files.append(path)\n visited.add(file_id)\n return unique_files\n\n\ndef _find_max_depth(\n roots: Sequence[Path], globs: Sequence[str], max_depth: int = sys.maxsize\n) -> List[str]:\n \"\"\"See ``find`` for the public API.\"\"\"\n # We optimize for the common case of simple filename only patterns: a single, combined regex\n # is used. For complex patterns that include path components, we use a slower glob call from\n # every directory we visit within max_depth.\n filename_only_patterns = {\n f\"pattern_{i}\": os.path.normcase(x) for i, x in enumerate(globs) if \"/\" not in x\n }\n complex_patterns = {f\"pattern_{i}\": x for i, x in enumerate(globs) if \"/\" in x}\n regex = re.compile(fnmatch_translate_multiple(filename_only_patterns))\n # Ordered dictionary that keeps track of what pattern found which files\n matched_paths: Dict[str, List[str]] = {f\"pattern_{i}\": [] for i, _ in enumerate(globs)}\n # Ensure returned paths are always absolute\n roots = [os.path.abspath(r) for r in roots]\n # Breadth-first search queue. Each element is a tuple of (depth, dir)\n dir_queue: Deque[Tuple[int, str]] = collections.deque()\n # Set of visited directories. Each element is a tuple of (inode, device)\n visited_dirs: Set[Tuple[int, int]] = set()\n\n for root in roots:\n try:\n stat_root = os.stat(root)\n except OSError as e:\n _log_file_access_issue(e, root)\n continue\n dir_id = _file_id(stat_root)\n if dir_id not in visited_dirs:\n dir_queue.appendleft((0, root))\n visited_dirs.add(dir_id)\n\n while dir_queue:\n depth, curr_dir = dir_queue.pop()\n try:\n dir_iter = os.scandir(curr_dir)\n except OSError as e:\n _log_file_access_issue(e, curr_dir)\n continue\n\n # Use glob.glob for complex patterns.\n for pattern_name, pattern in complex_patterns.items():\n matched_paths[pattern_name].extend(\n path for path in glob.glob(os.path.join(curr_dir, pattern))\n )\n\n # List of subdirectories by path and (inode, device) tuple\n subdirs: List[Tuple[str, Tuple[int, int]]] = []\n\n with dir_iter:\n for dir_entry in dir_iter:\n # Match filename only patterns\n if filename_only_patterns:\n m = regex.match(os.path.normcase(dir_entry.name))\n if m:\n for pattern_name in filename_only_patterns:\n if m.group(pattern_name):\n matched_paths[pattern_name].append(dir_entry.path)\n break\n\n # Collect subdirectories\n if depth >= max_depth:\n continue\n\n try:\n if not dir_entry.is_dir(follow_symlinks=True):\n continue\n if sys.platform == \"win32\":\n # Note: st_ino/st_dev on DirEntry.stat are not set on Windows, so we have\n # to call os.stat\n stat_info = os.stat(dir_entry.path, follow_symlinks=True)\n else:\n stat_info = dir_entry.stat(follow_symlinks=True)\n except OSError as e:\n # Possible permission issue, or a symlink that cannot be resolved (ELOOP).\n _log_file_access_issue(e, dir_entry.path)\n continue\n\n subdirs.append((dir_entry.path, _file_id(stat_info)))\n\n # Enqueue subdirectories in a deterministic order\n if subdirs:\n subdirs.sort(key=lambda s: os.path.basename(s[0]))\n for subdir, subdir_id in subdirs:\n if subdir_id not in visited_dirs:\n dir_queue.appendleft((depth + 1, subdir))\n visited_dirs.add(subdir_id)\n\n # Sort the matched paths for deterministic output\n for paths in matched_paths.values():\n paths.sort()\n all_matching_paths = [path for paths in matched_paths.values() for path in paths]\n\n # We only dedupe files if we have any complex patterns, since only they can match the same file\n # multiple times\n return _dedupe_files(all_matching_paths) if complex_patterns else all_matching_paths\n\n\n# Utilities for libraries and headers\n\n\nclass FileList(collections.abc.Sequence):\n \"\"\"Sequence of absolute paths to files.\n\n Provides a few convenience methods to manipulate file paths.\n \"\"\"\n\n def __init__(self, files: Union[str, Iterable[str]]) -> None:\n if isinstance(files, str):\n files = [files]\n\n self.files = list(dedupe(files))\n\n @property\n def directories(self) -> List[str]:\n \"\"\"Stable de-duplication of the directories where the files reside.\n\n >>> l = LibraryList([\"/dir1/liba.a\", \"/dir2/libb.a\", \"/dir1/libc.a\"])\n >>> l.directories\n [\"/dir1\", \"/dir2\"]\n >>> h = HeaderList([\"/dir1/a.h\", \"/dir1/b.h\", \"/dir2/c.h\"])\n >>> h.directories\n [\"/dir1\", \"/dir2\"]\n\n Returns:\n A list of directories\n \"\"\"\n return list(dedupe(os.path.dirname(x) for x in self.files if os.path.dirname(x)))\n\n @property\n def basenames(self) -> List[str]:\n \"\"\"Stable de-duplication of the base-names in the list\n\n >>> l = LibraryList([\"/dir1/liba.a\", \"/dir2/libb.a\", \"/dir3/liba.a\"])\n >>> l.basenames\n [\"liba.a\", \"libb.a\"]\n >>> h = HeaderList([\"/dir1/a.h\", \"/dir2/b.h\", \"/dir3/a.h\"])\n >>> h.basenames\n [\"a.h\", \"b.h\"]\n\n Returns:\n A list of base-names\n \"\"\"\n return list(dedupe(os.path.basename(x) for x in self.files))\n\n def __getitem__(self, item):\n cls = type(self)\n if isinstance(item, numbers.Integral):\n return self.files[item]\n return cls(self.files[item])\n\n def __add__(self, other):\n return self.__class__(dedupe(self.files + list(other)))\n\n def __radd__(self, other):\n return self.__add__(other)\n\n def __eq__(self, other):\n return self.files == other.files\n\n def __len__(self):\n return len(self.files)\n\n def joined(self, separator: str = \" \") -> str:\n return separator.join(self.files)\n\n def __repr__(self):\n return self.__class__.__name__ + \"(\" + repr(self.files) + \")\"\n\n def __str__(self):\n return self.joined()\n\n\nclass HeaderList(FileList):\n \"\"\"Sequence of absolute paths to headers.\n\n Provides a few convenience methods to manipulate header paths and get\n commonly used compiler flags or names.\n \"\"\"\n\n # Make sure to only match complete words, otherwise path components such\n # as \"xinclude\" will cause false matches.\n # Avoid matching paths such as /include/something/detail/include,\n # e.g. in the CUDA Toolkit which ships internal libc++ headers.\n include_regex = re.compile(r\"(.*?)(\\binclude\\b)(.*)\")\n\n def __init__(self, files):\n super().__init__(files)\n\n self._macro_definitions = []\n self._directories = None\n\n @property\n def directories(self) -> List[str]:\n \"\"\"Directories to be searched for header files.\"\"\"\n values = self._directories\n if values is None:\n values = self._default_directories()\n return list(dedupe(values))\n\n @directories.setter\n def directories(self, value):\n value = value or []\n # Accept a single directory as input\n if isinstance(value, str):\n value = [value]\n\n self._directories = [path_to_os_path(os.path.normpath(x))[0] for x in value]\n\n def _default_directories(self):\n \"\"\"Default computation of directories based on the list of\n header files.\n \"\"\"\n dir_list = super().directories\n values = []\n for d in dir_list:\n # If the path contains a subdirectory named 'include' then stop\n # there and don't add anything else to the path.\n m = self.include_regex.match(d)\n value = os.path.join(*m.group(1, 2)) if m else d\n values.append(value)\n return values\n\n @property\n def headers(self) -> List[str]:\n \"\"\"Stable de-duplication of the headers.\n\n Returns:\n A list of header files\n \"\"\"\n return self.files\n\n @property\n def names(self) -> List[str]:\n \"\"\"Stable de-duplication of header names in the list without extensions\n\n >>> h = HeaderList([\"/dir1/a.h\", \"/dir2/b.h\", \"/dir3/a.h\"])\n >>> h.names\n [\"a\", \"b\"]\n\n Returns:\n A list of files without extensions\n \"\"\"\n names = []\n\n for x in self.basenames:\n name = x\n\n # Valid extensions include: [\".cuh\", \".hpp\", \".hh\", \".h\"]\n for ext in [\".cuh\", \".hpp\", \".hh\", \".h\"]:\n i = name.rfind(ext)\n if i != -1:\n names.append(name[:i])\n break\n else:\n # No valid extension, should we still include it?\n names.append(name)\n\n return list(dedupe(names))\n\n @property\n def include_flags(self) -> str:\n \"\"\"Include flags\n\n >>> h = HeaderList([\"/dir1/a.h\", \"/dir1/b.h\", \"/dir2/c.h\"])\n >>> h.include_flags\n \"-I/dir1 -I/dir2\"\n\n Returns:\n A joined list of include flags\n \"\"\"\n return \" \".join([\"-I\" + x for x in self.directories])\n\n @property\n def macro_definitions(self) -> str:\n \"\"\"Macro definitions\n\n >>> h = HeaderList([\"/dir1/a.h\", \"/dir1/b.h\", \"/dir2/c.h\"])\n >>> h.add_macro(\"-DBOOST_LIB_NAME=boost_regex\")\n >>> h.add_macro(\"-DBOOST_DYN_LINK\")\n >>> h.macro_definitions\n \"-DBOOST_LIB_NAME=boost_regex -DBOOST_DYN_LINK\"\n\n Returns:\n A joined list of macro definitions\n \"\"\"\n return \" \".join(self._macro_definitions)\n\n @property\n def cpp_flags(self) -> str:\n \"\"\"Include flags + macro definitions\n\n >>> h = HeaderList([\"/dir1/a.h\", \"/dir1/b.h\", \"/dir2/c.h\"])\n >>> h.cpp_flags\n \"-I/dir1 -I/dir2\"\n >>> h.add_macro(\"-DBOOST_DYN_LINK\")\n >>> h.cpp_flags\n \"-I/dir1 -I/dir2 -DBOOST_DYN_LINK\"\n\n Returns:\n A joined list of include flags and macro definitions\n \"\"\"\n cpp_flags = self.include_flags\n if self.macro_definitions:\n cpp_flags += \" \" + self.macro_definitions\n return cpp_flags\n\n def add_macro(self, macro: str) -> None:\n \"\"\"Add a macro definition\n\n Parameters:\n macro: The macro to add\n \"\"\"\n self._macro_definitions.append(macro)\n\n\ndef find_headers(headers: Union[str, List[str]], root: str, recursive: bool = False) -> HeaderList:\n \"\"\"Returns an iterable object containing a list of full paths to\n headers if found.\n\n Accepts any glob characters accepted by :py:func:`fnmatch.fnmatch`:\n\n ========== ====================================\n Pattern Meaning\n ========== ====================================\n ``*`` matches one or more characters\n ``?`` matches any single character\n ``[seq]`` matches any character in ``seq``\n ``[!seq]`` matches any character not in ``seq``\n ========== ====================================\n\n Parameters:\n headers: Header name(s) to search for\n root: The root directory to start searching from\n recursive: if :data:`False` search only root folder,\n if :data:`True` descends top-down from the root. Defaults to :data:`False`.\n\n Returns:\n The headers that have been found\n \"\"\"\n if isinstance(headers, str):\n headers = [headers]\n elif not isinstance(headers, collections.abc.Sequence):\n message = \"{0} expects a string or sequence of strings as the \"\n message += \"first argument [got {1} instead]\"\n message = message.format(find_headers.__name__, type(headers))\n raise TypeError(message)\n\n # Construct the right suffix for the headers\n suffixes = [\n # C\n \"h\",\n # C++\n \"hpp\",\n \"hxx\",\n \"hh\",\n \"H\",\n \"txx\",\n \"tcc\",\n \"icc\",\n # Fortran\n \"mod\",\n \"inc\",\n ]\n\n # List of headers we are searching with suffixes\n headers = [\"{0}.{1}\".format(header, suffix) for header in headers for suffix in suffixes]\n\n return HeaderList(find(root, headers, recursive))\n\n\n@system_path_filter\ndef find_all_headers(root: str) -> HeaderList:\n \"\"\"Convenience function that returns the list of all headers found\n in the directory passed as argument.\n\n Args:\n root: directory where to look recursively for header files\n\n Returns:\n List of all headers found in ``root`` and subdirectories.\n \"\"\"\n return find_headers(\"*\", root=root, recursive=True)\n\n\nclass LibraryList(FileList):\n \"\"\"Sequence of absolute paths to libraries\n\n Provides a few convenience methods to manipulate library paths and get\n commonly used compiler flags or names\n \"\"\"\n\n @property\n def libraries(self) -> List[str]:\n \"\"\"Stable de-duplication of library files.\n\n Returns:\n A list of library files\n \"\"\"\n return self.files\n\n @property\n def names(self) -> List[str]:\n \"\"\"Stable de-duplication of library names in the list\n\n >>> l = LibraryList([\"/dir1/liba.a\", \"/dir2/libb.a\", \"/dir3/liba.so\"])\n >>> l.names\n [\"a\", \"b\"]\n\n Returns:\n A list of library names\n \"\"\"\n names = []\n\n for x in self.basenames:\n name = x\n if x.startswith(\"lib\"):\n name = x[3:]\n\n # Valid extensions include: ['.dylib', '.so', '.a']\n # on non Windows platform\n # Windows valid library extensions are:\n # ['.dll', '.lib']\n valid_exts = [\".dll\", \".lib\"] if sys.platform == \"win32\" else [\".dylib\", \".so\", \".a\"]\n for ext in valid_exts:\n i = name.rfind(ext)\n if i != -1:\n names.append(name[:i])\n break\n else:\n # No valid extension, should we still include it?\n names.append(name)\n\n return list(dedupe(names))\n\n @property\n def search_flags(self) -> str:\n \"\"\"Search flags for the libraries\n\n >>> l = LibraryList([\"/dir1/liba.a\", \"/dir2/libb.a\", \"/dir1/liba.so\"])\n >>> l.search_flags\n \"-L/dir1 -L/dir2\"\n\n Returns:\n A joined list of search flags\n \"\"\"\n return \" \".join([\"-L\" + x for x in self.directories])\n\n @property\n def link_flags(self) -> str:\n \"\"\"Link flags for the libraries\n\n >>> l = LibraryList([\"/dir1/liba.a\", \"/dir2/libb.a\", \"/dir1/liba.so\"])\n >>> l.link_flags\n \"-la -lb\"\n\n Returns:\n A joined list of link flags\n \"\"\"\n return \" \".join([\"-l\" + name for name in self.names])\n\n @property\n def ld_flags(self) -> str:\n \"\"\"Search flags + link flags\n\n >>> l = LibraryList([\"/dir1/liba.a\", \"/dir2/libb.a\", \"/dir1/liba.so\"])\n >>> l.ld_flags\n \"-L/dir1 -L/dir2 -la -lb\"\n\n Returns:\n A joined list of search flags and link flags\n \"\"\"\n return self.search_flags + \" \" + self.link_flags\n\n\ndef find_system_libraries(libraries: Union[str, List[str]], shared: bool = True) -> LibraryList:\n \"\"\"Searches the usual system library locations for ``libraries``.\n\n Search order is as follows:\n\n 1. ``/lib64``\n 2. ``/lib``\n 3. ``/usr/lib64``\n 4. ``/usr/lib``\n 5. ``/usr/local/lib64``\n 6. ``/usr/local/lib``\n\n Accepts any glob characters accepted by :py:func:`fnmatch.fnmatch`:\n\n ========== ====================================\n Pattern Meaning\n ========== ====================================\n ``*`` matches one or more characters\n ``?`` matches any single character\n ``[seq]`` matches any character in ``seq``\n ``[!seq]`` matches any character not in ``seq``\n ========== ====================================\n\n Parameters:\n libraries: Library name(s) to search for\n shared: if :data:`True` searches for shared libraries,\n otherwise for static. Defaults to :data:`True`.\n\n Returns:\n The libraries that have been found\n \"\"\"\n if isinstance(libraries, str):\n libraries = [libraries]\n elif not isinstance(libraries, collections.abc.Sequence):\n message = \"{0} expects a string or sequence of strings as the \"\n message += \"first argument [got {1} instead]\"\n message = message.format(find_system_libraries.__name__, type(libraries))\n raise TypeError(message)\n\n libraries_found = LibraryList([])\n search_locations = [\n \"/lib64\",\n \"/lib\",\n \"/usr/lib64\",\n \"/usr/lib\",\n \"/usr/local/lib64\",\n \"/usr/local/lib\",\n ]\n\n for library in libraries:\n for root in search_locations:\n result = find_libraries(library, root, shared, recursive=True)\n if result:\n libraries_found += result\n break\n\n return libraries_found\n\n\ndef find_libraries(\n libraries: Union[str, List[str]],\n root: str,\n shared: bool = True,\n recursive: bool = False,\n runtime: bool = True,\n max_depth: Optional[int] = None,\n) -> LibraryList:\n \"\"\"Returns an iterable of full paths to libraries found in a root dir.\n\n Accepts any glob characters accepted by :py:func:`fnmatch.fnmatch`:\n\n ========== ====================================\n Pattern Meaning\n ========== ====================================\n ``*`` matches one or more characters\n ``?`` matches any single character\n ``[seq]`` matches any character in ``seq``\n ``[!seq]`` matches any character not in ``seq``\n ========== ====================================\n\n Parameters:\n libraries: Library name(s) to search for\n root: The root directory to start searching from\n shared: if :data:`True` searches for shared libraries,\n otherwise for static. Defaults to :data:`True`.\n recursive: if :data:`False` search only root folder,\n if :data:`True` descends top-down from the root. Defaults to :data:`False`.\n max_depth: if set, don't search below this depth. Cannot be set\n if recursive is :data:`False`\n runtime: Windows only option, no-op elsewhere. If :data:`True`,\n search for runtime shared libs (``.DLL``), otherwise, search\n for ``.Lib`` files. If ``shared`` is :data:`False`, this has no meaning.\n Defaults to :data:`True`.\n\n Returns:\n The libraries that have been found\n \"\"\"\n\n if isinstance(libraries, str):\n libraries = [libraries]\n elif not isinstance(libraries, collections.abc.Sequence):\n message = \"{0} expects a string or sequence of strings as the \"\n message += \"first argument [got {1} instead]\"\n message = message.format(find_libraries.__name__, type(libraries))\n raise TypeError(message)\n\n if sys.platform == \"win32\":\n static_ext = \"lib\"\n # For linking (runtime=False) you need the .lib files regardless of\n # whether you are doing a shared or static link\n shared_ext = \"dll\" if runtime else \"lib\"\n else:\n # Used on both Linux and macOS\n static_ext = \"a\"\n shared_ext = \"so\"\n\n # Construct the right suffix for the library\n if shared:\n # Used on both Linux and macOS\n suffixes = [shared_ext]\n if sys.platform == \"darwin\":\n # Only used on macOS\n suffixes.append(\"dylib\")\n else:\n suffixes = [static_ext]\n\n # List of libraries we are searching with suffixes\n libraries = [\"{0}.{1}\".format(lib, suffix) for lib in libraries for suffix in suffixes]\n\n if not recursive:\n if max_depth:\n raise ValueError(f\"max_depth ({max_depth}) cannot be set if recursive is False\")\n # If not recursive, look for the libraries directly in root\n return LibraryList(find(root, libraries, recursive=False))\n\n # To speedup the search for external packages configured e.g. in /usr,\n # perform first non-recursive search in root/lib then in root/lib64 and\n # finally search all of root recursively. The search stops when the first\n # match is found.\n common_lib_dirs = [\"lib\", \"lib64\"]\n if sys.platform == \"win32\":\n common_lib_dirs.extend([\"bin\", \"Lib\"])\n\n for subdir in common_lib_dirs:\n dirname = join_path(root, subdir)\n if not os.path.isdir(dirname):\n continue\n found_libs = find(dirname, libraries, False)\n if found_libs:\n break\n else:\n found_libs = find(root, libraries, recursive=True, max_depth=max_depth)\n\n return LibraryList(found_libs)\n\n\ndef find_all_shared_libraries(\n root: str, recursive: bool = False, runtime: bool = True\n) -> LibraryList:\n \"\"\"Convenience function that returns the list of all shared libraries found\n in the directory passed as argument.\n\n See documentation for :py:func:`find_libraries` for more information\n \"\"\"\n return find_libraries(\"*\", root=root, shared=True, recursive=recursive, runtime=runtime)\n\n\ndef find_all_static_libraries(root: str, recursive: bool = False) -> LibraryList:\n \"\"\"Convenience function that returns the list of all static libraries found\n in the directory passed as argument.\n\n See documentation for :py:func:`find_libraries` for more information\n \"\"\"\n return find_libraries(\"*\", root=root, shared=False, recursive=recursive)\n\n\ndef find_all_libraries(root: str, recursive: bool = False) -> LibraryList:\n \"\"\"Convenience function that returns the list of all libraries found\n in the directory passed as argument.\n\n See documentation for :py:func:`find_libraries` for more information\n \"\"\"\n\n return find_all_shared_libraries(root, recursive=recursive) + find_all_static_libraries(\n root, recursive=recursive\n )\n\n\n@system_path_filter\n@memoized\ndef can_access_dir(path):\n \"\"\"Returns True if the argument is an accessible directory.\n\n Args:\n path: path to be tested\n\n Returns:\n True if ``path`` is an accessible directory, else False\n \"\"\"\n return os.path.isdir(path) and os.access(path, os.R_OK | os.X_OK)\n\n\n@system_path_filter\n@memoized\ndef can_write_to_dir(path):\n \"\"\"Return True if the argument is a directory in which we can write.\n\n Args:\n path: path to be tested\n\n Returns:\n True if ``path`` is an writeable directory, else False\n \"\"\"\n return os.path.isdir(path) and os.access(path, os.R_OK | os.X_OK | os.W_OK)\n\n\n@system_path_filter\n@memoized\ndef files_in(*search_paths):\n \"\"\"Returns all the files in paths passed as arguments.\n\n Caller must ensure that each path in ``search_paths`` is a directory.\n\n Args:\n *search_paths: directories to be searched\n\n Returns:\n List of (file, full_path) tuples with all the files found.\n \"\"\"\n files = []\n for d in filter(can_access_dir, search_paths):\n files.extend(\n filter(\n lambda x: os.path.isfile(x[1]), [(f, os.path.join(d, f)) for f in os.listdir(d)]\n )\n )\n return files\n\n\ndef is_readable_file(file_path):\n \"\"\"Return True if the path passed as argument is readable\"\"\"\n return os.path.isfile(file_path) and os.access(file_path, os.R_OK)\n\n\n@system_path_filter\ndef search_paths_for_executables(*path_hints):\n \"\"\"Given a list of path hints returns a list of paths where\n to search for an executable.\n\n Args:\n *path_hints (list of paths): list of paths taken into\n consideration for a search\n\n Returns:\n A list containing the real path of every existing directory\n in `path_hints` and its `bin` subdirectory if it exists.\n \"\"\"\n executable_paths = []\n for path in path_hints:\n if not os.path.isdir(path):\n continue\n\n path = os.path.abspath(path)\n executable_paths.append(path)\n\n bin_dir = os.path.join(path, \"bin\")\n if os.path.isdir(bin_dir):\n executable_paths.append(bin_dir)\n\n return executable_paths\n\n\n@system_path_filter\ndef search_paths_for_libraries(*path_hints):\n \"\"\"Given a list of path hints returns a list of paths where\n to search for a shared library.\n\n Args:\n *path_hints (list of paths): list of paths taken into\n consideration for a search\n\n Returns:\n A list containing the real path of every existing directory\n in `path_hints` and its `lib` and `lib64` subdirectory if it exists.\n \"\"\"\n library_paths = []\n for path in path_hints:\n if not os.path.isdir(path):\n continue\n\n path = os.path.abspath(path)\n library_paths.append(path)\n\n lib_dir = os.path.join(path, \"lib\")\n if os.path.isdir(lib_dir):\n library_paths.append(lib_dir)\n\n lib64_dir = os.path.join(path, \"lib64\")\n if os.path.isdir(lib64_dir):\n library_paths.append(lib64_dir)\n\n return library_paths\n\n\n@system_path_filter\ndef partition_path(path, entry=None):\n \"\"\"\n Split the prefixes of the path at the first occurrence of entry and\n return a 3-tuple containing a list of the prefixes before the entry, a\n string of the prefix ending with the entry, and a list of the prefixes\n after the entry.\n\n If the entry is not a node in the path, the result will be the prefix list\n followed by an empty string and an empty list.\n \"\"\"\n paths = prefixes(path)\n\n if entry is not None:\n # Derive the index of entry within paths, which will correspond to\n # the location of the entry in within the path.\n try:\n sep = os.sep\n entries = path.split(sep)\n if entries[0].endswith(\":\"):\n # Handle drive letters e.g. C:/ on Windows\n entries[0] = entries[0] + sep\n i = entries.index(entry)\n if \"\" in entries:\n i -= 1\n return paths[:i], paths[i], paths[i + 1 :]\n except ValueError:\n pass\n\n return paths, \"\", []\n\n\n@system_path_filter\ndef prefixes(path):\n \"\"\"\n Returns a list containing the path and its ancestors, top-to-bottom.\n\n The list for an absolute path will not include an ``os.sep`` entry.\n For example, assuming ``os.sep`` is ``/``, given path ``/ab/cd/efg``\n the resulting paths will be, in order: ``/ab``, ``/ab/cd``, and\n ``/ab/cd/efg``\n\n The list for a relative path starting ``./`` will not include ``.``.\n For example, path ``./hi/jkl/mn`` results in a list with the following\n paths, in order: ``./hi``, ``./hi/jkl``, and ``./hi/jkl/mn``.\n\n On Windows, paths will be normalized to use ``/`` and ``/`` will always\n be used as the separator instead of ``os.sep``.\n\n Parameters:\n path (str): the string used to derive ancestor paths\n\n Returns:\n A list containing ancestor paths in order and ending with the path\n \"\"\"\n if not path:\n return []\n sep = os.sep\n parts = path.strip(sep).split(sep)\n if path.startswith(sep):\n parts.insert(0, sep)\n elif parts[0].endswith(\":\"):\n # Handle drive letters e.g. C:/ on Windows\n parts[0] = parts[0] + sep\n paths = [os.path.join(*parts[: i + 1]) for i in range(len(parts))]\n\n try:\n paths.remove(sep)\n except ValueError:\n pass\n\n try:\n paths.remove(\".\")\n except ValueError:\n pass\n\n return paths\n\n\n@system_path_filter\ndef remove_directory_contents(dir):\n \"\"\"Remove all contents of a directory.\"\"\"\n if os.path.exists(dir):\n for entry in [os.path.join(dir, entry) for entry in os.listdir(dir)]:\n if os.path.isfile(entry) or islink(entry):\n os.unlink(entry)\n else:\n shutil.rmtree(entry)\n\n\n@contextmanager\n@system_path_filter\ndef keep_modification_time(*filenames: str) -> Generator[None, None, None]:\n \"\"\"\n Context manager to keep the modification timestamps of the input files.\n Tolerates and has no effect on non-existent files and files that are\n deleted by the nested code.\n\n Example::\n\n with keep_modification_time(\"file1.txt\", \"file2.txt\"):\n # do something that modifies file1.txt and file2.txt\n\n Parameters:\n *filenames: one or more files that must have their modification\n timestamps unchanged\n \"\"\"\n mtimes = {}\n for f in filenames:\n if os.path.exists(f):\n mtimes[f] = os.path.getmtime(f)\n yield\n for f, mtime in mtimes.items():\n if os.path.exists(f):\n os.utime(f, (os.path.getatime(f), mtime))\n\n\n@contextmanager\ndef temporary_file_position(stream):\n orig_pos = stream.tell()\n yield\n stream.seek(orig_pos)\n\n\n@contextmanager\ndef current_file_position(stream: IO, loc: int, relative_to=io.SEEK_CUR):\n with temporary_file_position(stream):\n stream.seek(loc, relative_to)\n yield\n\n\n@contextmanager\ndef temporary_dir(\n suffix: Optional[str] = None, prefix: Optional[str] = None, dir: Optional[str] = None\n):\n \"\"\"Create a temporary directory and cd's into it. Delete the directory\n on exit.\n\n Takes the same arguments as tempfile.mkdtemp()\n \"\"\"\n tmp_dir = tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=dir)\n try:\n with working_dir(tmp_dir):\n yield tmp_dir\n finally:\n remove_directory_contents(tmp_dir)\n\n\n@contextmanager\ndef edit_in_place_through_temporary_file(file_path: str) -> Generator[str, None, None]:\n \"\"\"Context manager for modifying ``file_path`` in place, preserving its inode and hardlinks,\n for functions or external tools that do not support in-place editing. Notice that this function\n is unsafe in that it works with paths instead of a file descriptors, but this is by design,\n since we assume the call site will create a new inode at the same path.\"\"\"\n tmp_fd, tmp_path = tempfile.mkstemp(\n dir=os.path.dirname(file_path), prefix=f\"{os.path.basename(file_path)}.\"\n )\n # windows cannot replace a file with open fds, so close since the call site needs to replace.\n os.close(tmp_fd)\n try:\n shutil.copyfile(file_path, tmp_path, follow_symlinks=True)\n yield tmp_path\n shutil.copyfile(tmp_path, file_path, follow_symlinks=True)\n finally:\n os.unlink(tmp_path)\n\n\ndef filesummary(path, print_bytes=16) -> Tuple[int, bytes]:\n \"\"\"Create a small summary of the given file. Does not error\n when file does not exist.\n\n Args:\n print_bytes (int): Number of bytes to print from start/end of file\n\n Returns:\n Tuple of size and byte string containing first n .. last n bytes.\n Size is 0 if file cannot be read.\"\"\"\n try:\n n = print_bytes\n with open(path, \"rb\") as f:\n size = os.fstat(f.fileno()).st_size\n if size <= 2 * n:\n short_contents = f.read(2 * n)\n else:\n short_contents = f.read(n)\n f.seek(-n, 2)\n short_contents += b\"...\" + f.read(n)\n return size, short_contents\n except OSError:\n return 0, b\"\"\n\n\nclass FindFirstFile:\n \"\"\"Uses hybrid iterative deepening to locate the first matching\n file. Up to depth ``bfs_depth`` it uses iterative deepening, which\n mimics breadth-first with the same memory footprint as depth-first\n search, after which it switches to ordinary depth-first search using\n ``os.walk``.\"\"\"\n\n def __init__(self, root: str, *file_patterns: str, bfs_depth: int = 2):\n \"\"\"Create a small summary of the given file. Does not error\n when file does not exist.\n\n Args:\n root (str): directory in which to recursively search\n file_patterns (str): glob file patterns understood by fnmatch\n bfs_depth (int): until this depth breadth-first traversal is used,\n when no match is found, the mode is switched to depth-first search.\n \"\"\"\n self.root = root\n self.bfs_depth = bfs_depth\n self.match: Callable\n\n # normcase is trivial on posix\n regex = re.compile(\"|\".join(fnmatch.translate(os.path.normcase(p)) for p in file_patterns))\n\n # On case sensitive filesystems match against normcase'd paths.\n if os.path is posixpath:\n self.match = regex.match\n else:\n self.match = lambda p: regex.match(os.path.normcase(p))\n\n def find(self) -> Optional[str]:\n \"\"\"Run the file search\n\n Returns:\n str or None: path of the matching file\n \"\"\"\n self.file = None\n\n # First do iterative deepening (i.e. bfs through limited depth dfs)\n for i in range(self.bfs_depth + 1):\n if self._find_at_depth(self.root, i):\n return self.file\n\n # Then fall back to depth-first search\n return self._find_dfs()\n\n def _find_at_depth(self, path, max_depth, depth=0) -> bool:\n \"\"\"Returns True when done. Notice search can be done\n either because a file was found, or because it recursed\n through all directories.\"\"\"\n try:\n entries = os.scandir(path)\n except OSError:\n return True\n\n done = True\n\n with entries:\n # At max depth we look for matching files.\n if depth == max_depth:\n for f in entries:\n # Exit on match\n if self.match(f.name):\n self.file = os.path.join(path, f.name)\n return True\n\n # is_dir should not require a stat call, so it's a good optimization.\n if self._is_dir(f):\n done = False\n return done\n\n # At lower depth only recurse into subdirs\n for f in entries:\n if not self._is_dir(f):\n continue\n\n # If any subdir is not fully traversed, we're not done yet.\n if not self._find_at_depth(os.path.join(path, f.name), max_depth, depth + 1):\n done = False\n\n # Early exit when we've found something.\n if self.file:\n return True\n\n return done\n\n def _is_dir(self, f: os.DirEntry) -> bool:\n \"\"\"Returns True when f is dir we can enter (and not a symlink).\"\"\"\n try:\n return f.is_dir(follow_symlinks=False)\n except OSError:\n return False\n\n def _find_dfs(self) -> Optional[str]:\n \"\"\"Returns match or None\"\"\"\n for dirpath, _, filenames in os.walk(self.root):\n for file in filenames:\n if self.match(file):\n return os.path.join(dirpath, file)\n return None\n\n\ndef _windows_symlink(\n src: str, dst: str, target_is_directory: bool = False, *, dir_fd: Union[int, None] = None\n):\n \"\"\"On Windows with System Administrator privileges this will be a normal symbolic link via\n os.symlink. On Windows without privileges the link will be a junction for a directory and a\n hardlink for a file. On Windows the various link types are:\n\n Symbolic Link: A link to a file or directory on the same or different volume (drive letter) or\n even to a remote file or directory (using UNC in its path). Need System Administrator\n privileges to make these.\n\n Hard Link: A link to a file on the same volume (drive letter) only. Every file (file's data)\n has at least 1 hard link (file's name). But when this method creates a new hard link there will\n be 2. Deleting all hard links effectively deletes the file. Don't need System Administrator\n privileges.\n\n Junction: A link to a directory on the same or different volume (drive letter) but not to a\n remote directory. Don't need System Administrator privileges.\"\"\"\n source_path = os.path.normpath(src)\n win_source_path = source_path\n link_path = os.path.normpath(dst)\n\n # Perform basic checks to make sure symlinking will succeed\n if os.path.lexists(link_path):\n raise AlreadyExistsError(f\"Link path ({link_path}) already exists. Cannot create link.\")\n\n if not os.path.exists(source_path):\n if os.path.isabs(source_path):\n # An absolute source path that does not exist will result in a broken link.\n raise SymlinkError(\n f\"Source path ({source_path}) is absolute but does not exist. Resulting \"\n f\"link would be broken so not making link.\"\n )\n else:\n # os.symlink can create a link when the given source path is relative to\n # the link path. Emulate this behavior and check to see if the source exists\n # relative to the link path ahead of link creation to prevent broken\n # links from being made.\n link_parent_dir = os.path.dirname(link_path)\n relative_path = os.path.join(link_parent_dir, source_path)\n if os.path.exists(relative_path):\n # In order to work on windows, the source path needs to be modified to be\n # relative because hardlink/junction dont resolve relative paths the same\n # way as os.symlink. This is ignored on other operating systems.\n win_source_path = relative_path\n else:\n raise SymlinkError(\n f\"The source path ({source_path}) is not relative to the link path \"\n f\"({link_path}). Resulting link would be broken so not making link.\"\n )\n\n # Create the symlink\n if not _windows_can_symlink():\n _windows_create_link(win_source_path, link_path)\n else:\n os.symlink(source_path, link_path, target_is_directory=os.path.isdir(source_path))\n\n\ndef _windows_islink(path: str) -> bool:\n \"\"\"Override os.islink to give correct answer for spack logic.\n\n For Non-Windows: a link can be determined with the os.path.islink method.\n Windows-only methods will return false for other operating systems.\n\n For Windows: spack considers symlinks, hard links, and junctions to\n all be links, so if any of those are True, return True.\n\n Args:\n path (str): path to check if it is a link.\n\n Returns:\n bool - whether the path is any kind link or not.\n \"\"\"\n return any([os.path.islink(path), _windows_is_junction(path), _windows_is_hardlink(path)])\n\n\ndef _windows_is_hardlink(path: str) -> bool:\n \"\"\"Determines if a path is a windows hard link. This is accomplished\n by looking at the number of links using os.stat. A non-hard-linked file\n will have a st_nlink value of 1, whereas a hard link will have a value\n larger than 1. Note that both the original and hard-linked file will\n return True because they share the same inode.\n\n Args:\n path (str): Windows path to check for a hard link\n\n Returns:\n bool - Whether the path is a hard link or not.\n \"\"\"\n if sys.platform != \"win32\" or os.path.islink(path) or not os.path.exists(path):\n return False\n\n return os.stat(path).st_nlink > 1\n\n\ndef _windows_is_junction(path: str) -> bool:\n \"\"\"Determines if a path is a windows junction. A junction can be\n determined using a bitwise AND operation between the file's\n attribute bitmask and the known junction bitmask (0x400).\n\n Args:\n path (str): A non-file path\n\n Returns:\n bool - whether the path is a junction or not.\n \"\"\"\n if sys.platform != \"win32\" or os.path.islink(path) or os.path.isfile(path):\n return False\n\n import ctypes.wintypes\n\n get_file_attributes = ctypes.windll.kernel32.GetFileAttributesW # type: ignore[attr-defined]\n get_file_attributes.argtypes = (ctypes.wintypes.LPWSTR,)\n get_file_attributes.restype = ctypes.wintypes.DWORD\n\n invalid_file_attributes = 0xFFFFFFFF\n reparse_point = 0x400\n file_attr = get_file_attributes(str(path))\n\n if file_attr == invalid_file_attributes:\n return False\n\n return file_attr & reparse_point > 0\n\n\n@lang.memoized\ndef _windows_can_symlink() -> bool:\n \"\"\"\n Determines if windows is able to make a symlink depending on\n the system configuration and the level of the user's permissions.\n \"\"\"\n if sys.platform != \"win32\":\n tty.warn(\"windows_can_symlink method can't be used on non-Windows OS.\")\n return False\n\n tempdir = tempfile.mkdtemp()\n\n dpath = os.path.join(tempdir, \"dpath\")\n fpath = os.path.join(tempdir, \"fpath.txt\")\n\n dlink = os.path.join(tempdir, \"dlink\")\n flink = os.path.join(tempdir, \"flink.txt\")\n\n touchp(fpath)\n mkdirp(dpath)\n\n try:\n os.symlink(dpath, dlink)\n can_symlink_directories = os.path.islink(dlink)\n except OSError:\n can_symlink_directories = False\n\n try:\n os.symlink(fpath, flink)\n can_symlink_files = os.path.islink(flink)\n except OSError:\n can_symlink_files = False\n\n # Cleanup the test directory\n shutil.rmtree(tempdir)\n\n return can_symlink_directories and can_symlink_files\n\n\ndef _windows_create_link(source: str, link: str):\n \"\"\"\n Attempts to create a Hard Link or Junction as an alternative\n to a symbolic link. This is called when symbolic links cannot\n be created.\n \"\"\"\n if sys.platform != \"win32\":\n raise SymlinkError(\"windows_create_link method can't be used on non-Windows OS.\")\n elif os.path.isdir(source):\n _windows_create_junction(source=source, link=link)\n elif os.path.isfile(source):\n _windows_create_hard_link(path=source, link=link)\n else:\n raise SymlinkError(\n f\"Cannot create link from {source}. It is neither a file nor a directory.\"\n )\n\n\ndef _windows_create_junction(source: str, link: str):\n \"\"\"Duly verify that the path and link are eligible to create a junction,\n then create the junction.\n \"\"\"\n if sys.platform != \"win32\":\n raise SymlinkError(\"windows_create_junction method can't be used on non-Windows OS.\")\n elif not os.path.exists(source):\n raise SymlinkError(\"Source path does not exist, cannot create a junction.\")\n elif os.path.lexists(link):\n raise AlreadyExistsError(\"Link path already exists, cannot create a junction.\")\n elif not os.path.isdir(source):\n raise SymlinkError(\"Source path is not a directory, cannot create a junction.\")\n\n import subprocess\n\n cmd = [\"cmd\", \"/C\", \"mklink\", \"/J\", link, source]\n proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n out, err = proc.communicate()\n tty.debug(out.decode())\n if proc.returncode != 0:\n err_str = err.decode()\n tty.error(err_str)\n raise SymlinkError(\"Make junction command returned a non-zero return code.\", err_str)\n\n\ndef _windows_create_hard_link(path: str, link: str):\n \"\"\"Duly verify that the path and link are eligible to create a hard\n link, then create the hard link.\n \"\"\"\n if sys.platform != \"win32\":\n raise SymlinkError(\"windows_create_hard_link method can't be used on non-Windows OS.\")\n elif not os.path.exists(path):\n raise SymlinkError(f\"File path {path} does not exist. Cannot create hard link.\")\n elif os.path.lexists(link):\n raise AlreadyExistsError(f\"Link path ({link}) already exists. Cannot create hard link.\")\n elif not os.path.isfile(path):\n raise SymlinkError(f\"File path ({link}) is not a file. Cannot create hard link.\")\n else:\n tty.debug(f\"Creating hard link {link} pointing to {path}\")\n CreateHardLink(link, path)\n\n\ndef _windows_readlink(path: str, *, dir_fd=None):\n \"\"\"Spack utility to override of os.readlink method to work cross platform\"\"\"\n if _windows_is_hardlink(path):\n return _windows_read_hard_link(path)\n elif _windows_is_junction(path):\n return _windows_read_junction(path)\n else:\n return sanitize_win_longpath(os.readlink(path, dir_fd=dir_fd))\n\n\ndef _windows_read_hard_link(link: str) -> str:\n \"\"\"Find all of the files that point to the same inode as the link\"\"\"\n if sys.platform != \"win32\":\n raise SymlinkError(\"Can't read hard link on non-Windows OS.\")\n link = os.path.abspath(link)\n fsutil_cmd = [\"fsutil\", \"hardlink\", \"list\", link]\n proc = subprocess.Popen(fsutil_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)\n out, err = proc.communicate()\n if proc.returncode != 0:\n raise SymlinkError(f\"An error occurred while reading hard link: {err.decode()}\")\n\n # fsutil response does not include the drive name, so append it back to each linked file.\n drive, link_tail = os.path.splitdrive(os.path.abspath(link))\n links = set([os.path.join(drive, p) for p in out.decode().splitlines()])\n links.remove(link)\n if len(links) == 1:\n return links.pop()\n elif len(links) > 1:\n # TODO: How best to handle the case where 3 or more paths point to a single inode?\n raise SymlinkError(f\"Found multiple paths pointing to the same inode {links}\")\n else:\n raise SymlinkError(\"Cannot determine hard link source path.\")\n\n\ndef _windows_read_junction(link: str):\n \"\"\"Find the path that a junction points to.\"\"\"\n if sys.platform != \"win32\":\n raise SymlinkError(\"Can't read junction on non-Windows OS.\")\n\n link = os.path.abspath(link)\n link_basename = os.path.basename(link)\n link_parent = os.path.dirname(link)\n fsutil_cmd = [\"dir\", \"/a:l\", link_parent]\n proc = subprocess.Popen(fsutil_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)\n out, err = proc.communicate()\n if proc.returncode != 0:\n raise SymlinkError(f\"An error occurred while reading junction: {err.decode()}\")\n matches = re.search(rf\"\\s+{link_basename} \\[(.*)]\", out.decode())\n if matches:\n return matches.group(1)\n else:\n raise SymlinkError(\"Could not find junction path.\")\n\n\n@system_path_filter\ndef resolve_link_target_relative_to_the_link(link):\n \"\"\"\n os.path.isdir uses os.path.exists, which for links will check\n the existence of the link target. If the link target is relative to\n the link, we need to construct a pathname that is valid from\n our cwd (which may not be the same as the link's directory)\n \"\"\"\n target = readlink(link)\n if os.path.isabs(target):\n return target\n link_dir = os.path.dirname(os.path.abspath(link))\n return os.path.join(link_dir, target)\n\n\nif sys.platform == \"win32\":\n symlink = _windows_symlink\n readlink = _windows_readlink\n islink = _windows_islink\n rename = _win_rename\nelse:\n symlink = os.symlink\n readlink = os.readlink\n islink = os.path.islink\n rename = os.rename\n\n\nclass SymlinkError(OSError):\n \"\"\"Exception class for errors raised while creating symlinks,\n junctions and hard links\n \"\"\"\n\n\nclass AlreadyExistsError(SymlinkError):\n \"\"\"Link path already exists.\"\"\"\n" }, { "path": "lib/spack/spack/llnl/util/lang.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport collections.abc\nimport contextlib\nimport fnmatch\nimport functools\nimport itertools\nimport os\nimport re\nimport sys\nimport traceback\nimport types\nimport typing\nimport warnings\nfrom datetime import datetime, timedelta\nfrom typing import (\n Any,\n Callable,\n Dict,\n Generic,\n Iterable,\n Iterator,\n List,\n Mapping,\n Optional,\n Tuple,\n TypeVar,\n Union,\n)\n\n# Ignore emacs backups when listing modules\nignore_modules = r\"^\\.#|~$\"\n\n\ndef index_by(objects, *funcs):\n \"\"\"Create a hierarchy of dictionaries by splitting the supplied\n set of objects on unique values of the supplied functions.\n\n Values are used as keys. For example, suppose you have four\n objects with attributes that look like this::\n\n a = Spec(\"boost %gcc target=skylake\")\n b = Spec(\"mrnet %intel target=zen2\")\n c = Spec(\"libelf %xlc target=skylake\")\n d = Spec(\"libdwarf %intel target=zen2\")\n\n list_of_specs = [a,b,c,d]\n index1 = index_by(list_of_specs, lambda s: str(s.target),\n lambda s: s.compiler)\n index2 = index_by(list_of_specs, lambda s: s.compiler)\n\n ``index1`` now has two levels of dicts, with lists at the\n leaves, like this::\n\n { 'zen2' : { 'gcc' : [a], 'xlc' : [c] },\n 'skylake' : { 'intel' : [b, d] }\n }\n\n And ``index2`` is a single level dictionary of lists that looks\n like this::\n\n { 'gcc' : [a],\n 'intel' : [b,d],\n 'xlc' : [c]\n }\n\n If any elements in funcs is a string, it is treated as the name\n of an attribute, and acts like ``getattr(object, name)``. So\n shorthand for the above two indexes would be::\n\n index1 = index_by(list_of_specs, 'arch', 'compiler')\n index2 = index_by(list_of_specs, 'compiler')\n\n You can also index by tuples by passing tuples::\n\n index1 = index_by(list_of_specs, ('target', 'compiler'))\n\n Keys in the resulting dict will look like ``('gcc', 'skylake')``.\n\n \"\"\"\n if not funcs:\n return objects\n\n f = funcs[0]\n if isinstance(f, str):\n f = lambda x: getattr(x, funcs[0])\n elif isinstance(f, tuple):\n f = lambda x: tuple(getattr(x, p, None) for p in funcs[0])\n\n result = {}\n for o in objects:\n key = f(o)\n result.setdefault(key, []).append(o)\n\n for key, objects in result.items():\n result[key] = index_by(objects, *funcs[1:])\n\n return result\n\n\ndef attr_setdefault(obj, name, value):\n \"\"\"Like dict.setdefault, but for objects.\"\"\"\n if not hasattr(obj, name):\n setattr(obj, name, value)\n return getattr(obj, name)\n\n\ndef memoized(func):\n \"\"\"Decorator that caches the results of a function, storing them in\n an attribute of that function.\n\n Example::\n\n @memoized\n def expensive_computation(x):\n # Some expensive computation\n return result\n \"\"\"\n return functools.lru_cache(maxsize=None)(func)\n\n\ndef list_modules(directory, **kwargs):\n \"\"\"Lists all of the modules, excluding ``__init__.py``, in a\n particular directory. Listed packages have no particular\n order.\"\"\"\n list_directories = kwargs.setdefault(\"directories\", True)\n\n ignore = re.compile(ignore_modules)\n\n with os.scandir(directory) as it:\n for entry in it:\n if entry.name == \"__init__.py\" or entry.name == \"__pycache__\":\n continue\n\n if (\n list_directories\n and entry.is_dir()\n and os.path.isfile(os.path.join(entry.path, \"__init__.py\"))\n ):\n yield entry.name\n\n elif entry.name.endswith(\".py\") and entry.is_file() and not ignore.search(entry.name):\n yield entry.name[:-3] # strip .py\n\n\ndef decorator_with_or_without_args(decorator):\n \"\"\"Allows a decorator to be used with or without arguments, e.g.::\n\n # Calls the decorator function some args\n @decorator(with, arguments, and=kwargs)\n\n or::\n\n # Calls the decorator function with zero arguments\n @decorator\n\n \"\"\"\n\n # See https://stackoverflow.com/questions/653368 for more on this\n @functools.wraps(decorator)\n def new_dec(*args, **kwargs):\n if len(args) == 1 and len(kwargs) == 0 and callable(args[0]):\n # actual decorated function\n return decorator(args[0])\n else:\n # decorator arguments\n return lambda realf: decorator(realf, *args, **kwargs)\n\n return new_dec\n\n\ndef key_ordering(cls):\n \"\"\"Decorates a class with extra methods that implement rich comparison\n operations and ``__hash__``. The decorator assumes that the class\n implements a function called ``_cmp_key()``. The rich comparison\n operations will compare objects using this key, and the ``__hash__``\n function will return the hash of this key.\n\n If a class already has ``__eq__``, ``__ne__``, ``__lt__``, ``__le__``,\n ``__gt__``, or ``__ge__`` defined, this decorator will overwrite them.\n\n Raises:\n TypeError: If the class does not have a ``_cmp_key`` method\n \"\"\"\n\n def setter(name, value):\n value.__name__ = name\n setattr(cls, name, value)\n\n if not hasattr(cls, \"_cmp_key\"):\n raise TypeError(f\"'{cls.__name__}' doesn't define _cmp_key().\")\n\n setter(\"__eq__\", lambda s, o: (s is o) or (o is not None and s._cmp_key() == o._cmp_key()))\n setter(\"__lt__\", lambda s, o: o is not None and s._cmp_key() < o._cmp_key())\n setter(\"__le__\", lambda s, o: o is not None and s._cmp_key() <= o._cmp_key())\n\n setter(\"__ne__\", lambda s, o: (s is not o) and (o is None or s._cmp_key() != o._cmp_key()))\n setter(\"__gt__\", lambda s, o: o is None or s._cmp_key() > o._cmp_key())\n setter(\"__ge__\", lambda s, o: o is None or s._cmp_key() >= o._cmp_key())\n\n setter(\"__hash__\", lambda self: hash(self._cmp_key()))\n\n return cls\n\n\n#: sentinel for testing that iterators are done in lazy_lexicographic_ordering\ndone = object()\n\n\ndef tuplify(seq):\n \"\"\"Helper for lazy_lexicographic_ordering().\"\"\"\n return tuple([(tuplify(x) if callable(x) else x) for x in seq()])\n\n\ndef lazy_eq(lseq, rseq):\n \"\"\"Equality comparison for two lazily generated sequences.\n\n See ``lazy_lexicographic_ordering``.\n \"\"\"\n liter = lseq() # call generators\n riter = rseq()\n\n # zip_longest is implemented in native code, so use it for speed.\n # use zip_longest instead of zip because it allows us to tell\n # which iterator was longer.\n for left, right in itertools.zip_longest(liter, riter, fillvalue=done):\n if (left is done) or (right is done):\n return False\n\n # recursively enumerate any generators, otherwise compare\n equal = lazy_eq(left, right) if callable(left) else left == right\n if not equal:\n return False\n\n return True\n\n\ndef lazy_lt(lseq, rseq):\n \"\"\"Less-than comparison for two lazily generated sequences.\n\n See ``lazy_lexicographic_ordering``.\n \"\"\"\n liter = lseq()\n riter = rseq()\n\n for left, right in itertools.zip_longest(liter, riter, fillvalue=done):\n if (left is done) or (right is done):\n return left is done # left was shorter than right\n\n sequence = callable(left)\n equal = lazy_eq(left, right) if sequence else left == right\n if equal:\n continue\n\n if sequence:\n return lazy_lt(left, right)\n if left is None:\n return True\n if right is None:\n return False\n\n return left < right\n\n return False # if equal, return False\n\n\n@decorator_with_or_without_args\ndef lazy_lexicographic_ordering(cls, set_hash=True):\n \"\"\"Decorates a class with extra methods that implement rich comparison.\n\n This is a lazy version of the tuple comparison used frequently to\n implement comparison in Python. Given some objects with fields, you\n might use tuple keys to implement comparison, e.g.:\n\n .. code-block:: python\n\n class Widget:\n def _cmp_key(self):\n return (self.a, self.b, (self.c, self.d), self.e)\n\n def __eq__(self, other):\n return self._cmp_key() == other._cmp_key()\n\n def __lt__(self):\n return self._cmp_key() < other._cmp_key()\n\n # etc.\n\n Python would compare ``Widgets`` lexicographically based on their\n tuples. The issue there for simple comparators is that we have to\n build the tuples *and* we have to generate all the values in them up\n front. When implementing comparisons for large data structures, this\n can be costly.\n\n Lazy lexicographic comparison maps the tuple comparison shown above\n to generator functions. Instead of comparing based on pre-constructed\n tuple keys, users of this decorator can compare using elements from a\n generator. So, you'd write:\n\n .. code-block:: python\n\n @lazy_lexicographic_ordering\n class Widget:\n def _cmp_iter(self):\n yield a\n yield b\n def cd_fun():\n yield c\n yield d\n yield cd_fun\n yield e\n\n # operators are added by decorator\n\n There are no tuples preconstructed, and the generator does not have\n to complete. Instead of tuples, we simply make functions that lazily\n yield what would've been in the tuple. The\n ``@lazy_lexicographic_ordering`` decorator handles the details of\n implementing comparison operators, and the ``Widget`` implementor\n only has to worry about writing ``_cmp_iter``, and making sure the\n elements in it are also comparable.\n\n In some cases, you may have a fast way to determine whether two\n objects are equal, e.g. the ``is`` function or an already-computed\n cryptographic hash. For this, you can implement your own\n ``_cmp_fast_eq`` function:\n\n .. code-block:: python\n\n @lazy_lexicographic_ordering\n class Widget:\n def _cmp_iter(self):\n yield a\n yield b\n\n def cd_fun():\n yield c\n yield d\n\n yield cd_fun\n yield e\n\n def _cmp_fast_eq(self, other):\n return self is other or None\n\n ``_cmp_fast_eq`` should return:\n\n * ``True`` if ``self`` is equal to ``other``,\n * ``False`` if ``self`` is not equal to ``other``, and\n * ``None`` if it's not known whether they are equal, and the full\n comparison should be done.\n\n ``lazy_lexicographic_ordering`` uses ``_cmp_fast_eq`` to short-circuit\n the comparison if the answer can be determined quickly. If you do not\n implement it, it defaults to ``self is other or None``.\n\n Some things to note:\n\n * If a class already has ``__eq__``, ``__ne__``, ``__lt__``,\n ``__le__``, ``__gt__``, ``__ge__``, or ``__hash__`` defined, this\n decorator will overwrite them.\n\n * If ``set_hash`` is ``False``, this will not overwrite\n ``__hash__``.\n\n * This class uses Python 2 None-comparison semantics. If you yield\n None and it is compared to a non-None type, None will always be\n less than the other object.\n\n Raises:\n TypeError: If the class does not have a ``_cmp_iter`` method\n\n \"\"\"\n if not hasattr(cls, \"_cmp_iter\"):\n raise TypeError(f\"'{cls.__name__}' doesn't define _cmp_iter().\")\n\n # get an equal operation that allows us to short-circuit comparison\n # if it's not provided, default to `is`\n _cmp_fast_eq = getattr(cls, \"_cmp_fast_eq\", lambda x, y: x is y or None)\n\n # comparison operators are implemented in terms of lazy_eq and lazy_lt\n def eq(self, other):\n fast_eq = _cmp_fast_eq(self, other)\n if fast_eq is not None:\n return fast_eq\n return (other is not None) and lazy_eq(self._cmp_iter, other._cmp_iter)\n\n def lt(self, other):\n if _cmp_fast_eq(self, other) is True:\n return False\n return (other is not None) and lazy_lt(self._cmp_iter, other._cmp_iter)\n\n def gt(self, other):\n if _cmp_fast_eq(self, other) is True:\n return False\n return (other is None) or lazy_lt(other._cmp_iter, self._cmp_iter)\n\n def ne(self, other):\n return not (self == other)\n\n def le(self, other):\n return not (self > other)\n\n def ge(self, other):\n return not (self < other)\n\n def h(self):\n return hash(tuplify(self._cmp_iter))\n\n def add_func_to_class(name, func):\n \"\"\"Add a function to a class with a particular name.\"\"\"\n func.__name__ = name\n setattr(cls, name, func)\n\n add_func_to_class(\"__eq__\", eq)\n add_func_to_class(\"__lt__\", lt)\n add_func_to_class(\"__gt__\", gt)\n add_func_to_class(\"__ne__\", ne)\n add_func_to_class(\"__le__\", le)\n add_func_to_class(\"__ge__\", ge)\n if set_hash:\n add_func_to_class(\"__hash__\", h)\n\n return cls\n\n\nK = TypeVar(\"K\")\nV = TypeVar(\"V\")\n\n\n@lazy_lexicographic_ordering\nclass HashableMap(typing.MutableMapping[K, V]):\n \"\"\"This is a hashable, comparable dictionary. Hash is performed on\n a tuple of the values in the dictionary.\"\"\"\n\n __slots__ = (\"dict\",)\n\n def __init__(self):\n self.dict: Dict[K, V] = {}\n\n def __getitem__(self, key: K) -> V:\n return self.dict[key]\n\n def __setitem__(self, key: K, value: V) -> None:\n self.dict[key] = value\n\n def __iter__(self) -> Iterator[K]:\n return iter(self.dict)\n\n def __len__(self) -> int:\n return len(self.dict)\n\n def __delitem__(self, key: K) -> None:\n del self.dict[key]\n\n def _cmp_iter(self):\n for _, v in sorted(self.dict.items()):\n yield v\n\n\ndef match_predicate(*args):\n \"\"\"Utility function for making string matching predicates.\n\n Each arg can be a:\n * regex\n * list or tuple of regexes\n * predicate that takes a string.\n\n This returns a predicate that is true if:\n * any arg regex matches\n * any regex in a list or tuple of regexes matches.\n * any predicate in args matches.\n \"\"\"\n\n def match(string):\n for arg in args:\n if isinstance(arg, str):\n if re.search(arg, string):\n return True\n elif isinstance(arg, list) or isinstance(arg, tuple):\n if any(re.search(i, string) for i in arg):\n return True\n elif callable(arg):\n if arg(string):\n return True\n else:\n raise ValueError(\n \"args to match_predicate must be regex, list of regexes, or callable.\"\n )\n return False\n\n return match\n\n\ndef dedupe(sequence, key=None):\n \"\"\"Yields a stable de-duplication of an hashable sequence by key\n\n Args:\n sequence: hashable sequence to be de-duplicated\n key: callable applied on values before uniqueness test; identity\n by default.\n\n Returns:\n stable de-duplication of the sequence\n\n Examples:\n\n Dedupe a list of integers::\n\n [x for x in dedupe([1, 2, 1, 3, 2])] == [1, 2, 3]\n\n [x for x in spack.llnl.util.lang.dedupe([1,-2,1,3,2], key=abs)] == [1, -2, 3]\n \"\"\"\n seen = set()\n for x in sequence:\n x_key = x if key is None else key(x)\n if x_key not in seen:\n yield x\n seen.add(x_key)\n\n\ndef pretty_date(time: Union[datetime, int], now: Optional[datetime] = None) -> str:\n \"\"\"Convert a datetime or timestamp to a pretty, relative date.\n\n Args:\n time: date to print prettily\n now: the date the pretty date is relative to (default is ``datetime.now()``)\n\n Returns:\n pretty string like \"an hour ago\", \"Yesterday\", \"3 months ago\", \"just now\", etc.\n\n Adapted from https://stackoverflow.com/questions/1551382.\n\n \"\"\"\n if now is None:\n now = datetime.now()\n\n if type(time) is int:\n diff = now - datetime.fromtimestamp(time)\n elif isinstance(time, datetime):\n diff = now - time\n else:\n raise ValueError(\"pretty_date requires a timestamp or datetime\")\n\n second_diff = diff.seconds\n day_diff = diff.days\n\n if day_diff < 0:\n return \"\"\n\n if day_diff == 0:\n if second_diff < 10:\n return \"just now\"\n if second_diff < 60:\n return f\"{second_diff} seconds ago\"\n if second_diff < 120:\n return \"a minute ago\"\n if second_diff < 3600:\n return f\"{second_diff // 60} minutes ago\"\n if second_diff < 7200:\n return \"an hour ago\"\n if second_diff < 86400:\n return f\"{second_diff // 3600} hours ago\"\n if day_diff == 1:\n return \"yesterday\"\n if day_diff < 7:\n return f\"{day_diff} days ago\"\n if day_diff < 28:\n weeks = day_diff // 7\n if weeks == 1:\n return \"a week ago\"\n else:\n return f\"{day_diff // 7} weeks ago\"\n if day_diff < 365:\n months = day_diff // 30\n if months == 1:\n return \"a month ago\"\n elif months == 12:\n months -= 1\n return f\"{months} months ago\"\n\n year_diff = day_diff // 365\n if year_diff == 1:\n return \"a year ago\"\n return f\"{year_diff} years ago\"\n\n\ndef pretty_string_to_date(date_str: str, now: Optional[datetime] = None) -> datetime:\n \"\"\"Parses a string representing a date and returns a datetime object.\n\n Args:\n date_str: string representing a date. This string might be\n in different format (like ``YYYY``, ``YYYY-MM``, ``YYYY-MM-DD``,\n ``YYYY-MM-DD HH:MM``, ``YYYY-MM-DD HH:MM:SS``)\n or be a *pretty date* (like ``yesterday`` or ``two months ago``)\n\n Returns: datetime object corresponding to ``date_str``\n \"\"\"\n\n pattern = {}\n\n now = now or datetime.now()\n\n # datetime formats\n pattern[re.compile(r\"^\\d{4}$\")] = lambda x: datetime.strptime(x, \"%Y\")\n pattern[re.compile(r\"^\\d{4}-\\d{2}$\")] = lambda x: datetime.strptime(x, \"%Y-%m\")\n pattern[re.compile(r\"^\\d{4}-\\d{2}-\\d{2}$\")] = lambda x: datetime.strptime(x, \"%Y-%m-%d\")\n pattern[re.compile(r\"^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}$\")] = lambda x: datetime.strptime(\n x, \"%Y-%m-%d %H:%M\"\n )\n pattern[re.compile(r\"^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$\")] = lambda x: datetime.strptime(\n x, \"%Y-%m-%d %H:%M:%S\"\n )\n\n pretty_regex = re.compile(r\"(a|\\d+)\\s*(year|month|week|day|hour|minute|second)s?\\s*ago\")\n\n def _n_xxx_ago(x):\n how_many, time_period = pretty_regex.search(x).groups()\n\n how_many = 1 if how_many == \"a\" else int(how_many)\n\n # timedelta natively supports time periods up to 'weeks'.\n # To apply month or year we convert to 30 and 365 days\n if time_period == \"month\":\n how_many *= 30\n time_period = \"day\"\n elif time_period == \"year\":\n how_many *= 365\n time_period = \"day\"\n\n kwargs = {(time_period + \"s\"): how_many}\n return now - timedelta(**kwargs)\n\n pattern[pretty_regex] = _n_xxx_ago\n\n # yesterday\n callback = lambda x: now - timedelta(days=1)\n pattern[re.compile(\"^yesterday$\")] = callback\n\n for regexp, parser in pattern.items():\n if bool(regexp.match(date_str)):\n return parser(date_str)\n\n raise ValueError(f'date \"{date_str}\" does not match any valid format')\n\n\ndef pretty_seconds_formatter(seconds):\n if seconds >= 1:\n multiplier, unit = 1, \"s\"\n elif seconds >= 1e-3:\n multiplier, unit = 1e3, \"ms\"\n elif seconds >= 1e-6:\n multiplier, unit = 1e6, \"us\"\n else:\n multiplier, unit = 1e9, \"ns\"\n return lambda s: \"%.3f%s\" % (multiplier * s, unit)\n\n\ndef pretty_seconds(seconds):\n \"\"\"Seconds to string with appropriate units\n\n Arguments:\n seconds (float): Number of seconds\n\n Returns:\n str: Time string with units\n \"\"\"\n return pretty_seconds_formatter(seconds)(seconds)\n\n\ndef pretty_duration(seconds: float) -> str:\n \"\"\"Format a duration in seconds as a compact human-readable string (e.g. \"1h02m\", \"3m05s\",\n \"45s\").\"\"\"\n s = int(seconds)\n if s < 60:\n return f\"{s}s\"\n m, s = divmod(s, 60)\n if m < 60:\n return f\"{m}m{s:02d}s\"\n h, m = divmod(m, 60)\n return f\"{h}h{m:02d}m\"\n\n\nclass ObjectWrapper:\n \"\"\"Base class that wraps an object. Derived classes can add new behavior\n while staying undercover.\n\n This class is modeled after the stackoverflow answer:\n * http://stackoverflow.com/a/1445289/771663\n \"\"\"\n\n def __init__(self, wrapped_object):\n wrapped_cls = type(wrapped_object)\n wrapped_name = wrapped_cls.__name__\n\n # If the wrapped object is already an ObjectWrapper, or a derived class\n # of it, adding type(self) in front of type(wrapped_object)\n # results in an inconsistent MRO.\n #\n # TODO: the implementation below doesn't account for the case where we\n # TODO: have different base classes of ObjectWrapper, say A and B, and\n # TODO: we want to wrap an instance of A with B.\n if type(self) not in wrapped_cls.__mro__:\n self.__class__ = type(wrapped_name, (type(self), wrapped_cls), {})\n else:\n self.__class__ = type(wrapped_name, (wrapped_cls,), {})\n\n self.__dict__ = wrapped_object.__dict__\n\n\nclass Singleton:\n \"\"\"Wrapper for lazily initialized singleton objects.\"\"\"\n\n def __init__(self, factory: Callable[[], object]):\n \"\"\"Create a new singleton to be inited with the factory function.\n\n Most factories will simply create the object to be initialized and\n return it.\n\n In some cases, e.g. when bootstrapping some global state, the singleton\n may need to be initialized incrementally. If the factory returns a generator\n instead of a regular object, the singleton will assign each result yielded by\n the generator to the singleton instance. This allows methods called by\n the factory in later stages to refer back to the singleton.\n\n Args:\n factory (function): function taking no arguments that creates the\n singleton instance.\n\n \"\"\"\n self.factory = factory\n self._instance = None\n\n @property\n def instance(self):\n if self._instance is None:\n try:\n instance = self.factory()\n\n if isinstance(instance, types.GeneratorType):\n # if it's a generator, assign every value\n for value in instance:\n self._instance = value\n else:\n # if not, just assign the result like a normal singleton\n self._instance = instance\n except AttributeError as e:\n # getattr will \"absorb\" an AttributeError that occurs\n # during the execution of the factory method: we'd like\n # to show that so wrap it in something that isn't absorbed\n raise SingletonInstantiationError(\n \"AttrbuteError during creation of Singleton instance\"\n ) from e\n return self._instance\n\n def __getattr__(self, name):\n # When unpickling Singleton objects, the 'instance' attribute may be\n # requested but not yet set. The final 'getattr' line here requires\n # 'instance'/'_instance' to be defined or it will enter an infinite\n # loop, so protect against that here.\n if name in [\"_instance\", \"instance\"]:\n raise AttributeError(f\"cannot create {name}\")\n return getattr(self.instance, name)\n\n def __getitem__(self, name):\n return self.instance[name]\n\n def __contains__(self, element):\n return element in self.instance\n\n def __call__(self, *args, **kwargs):\n return self.instance(*args, **kwargs)\n\n def __iter__(self):\n return iter(self.instance)\n\n def __str__(self):\n return str(self.instance)\n\n def __repr__(self):\n return repr(self.instance)\n\n\nclass SingletonInstantiationError(Exception):\n \"\"\"Error that indicates a singleton that cannot instantiate.\"\"\"\n\n\ndef get_entry_points(*, group: str):\n \"\"\"Wrapper for ``importlib.metadata.entry_points``\n\n Args:\n group: entry points to select\n\n Returns:\n EntryPoints for ``group`` or empty list if unsupported\n \"\"\"\n\n try:\n import importlib.metadata # type: ignore # novermin\n except ImportError:\n return []\n\n try:\n return importlib.metadata.entry_points(group=group)\n except TypeError:\n # Prior to Python 3.10, entry_points accepted no parameters and always\n # returned a dictionary of entry points, keyed by group. See\n # https://docs.python.org/3/library/importlib.metadata.html#entry-points\n return importlib.metadata.entry_points().get(group, [])\n\n\ndef load_module_from_file(module_name, module_path):\n \"\"\"Loads a python module from the path of the corresponding file.\n\n If the module is already in ``sys.modules`` it will be returned as\n is and not reloaded.\n\n Args:\n module_name (str): namespace where the python module will be loaded,\n e.g. ``foo.bar``\n module_path (str): path of the python file containing the module\n\n Returns:\n A valid module object\n\n Raises:\n ImportError: when the module can't be loaded\n FileNotFoundError: when module_path doesn't exist\n \"\"\"\n import importlib.util\n\n if module_name in sys.modules:\n return sys.modules[module_name]\n\n # This recipe is adapted from https://stackoverflow.com/a/67692/771663\n\n spec = importlib.util.spec_from_file_location(module_name, module_path)\n module = importlib.util.module_from_spec(spec)\n # The module object needs to exist in sys.modules before the\n # loader executes the module code.\n #\n # See https://docs.python.org/3/reference/import.html#loading\n sys.modules[spec.name] = module\n try:\n spec.loader.exec_module(module)\n except BaseException:\n try:\n del sys.modules[spec.name]\n except KeyError:\n pass\n raise\n return module\n\n\ndef uniq(sequence):\n \"\"\"Remove strings of duplicate elements from a list.\n\n This works like the command-line ``uniq`` tool. It filters strings\n of duplicate elements in a list. Adjacent matching elements are\n merged into the first occurrence.\n\n For example::\n\n uniq([1, 1, 1, 1, 2, 2, 2, 3, 3]) == [1, 2, 3]\n uniq([1, 1, 1, 1, 2, 2, 2, 1, 1]) == [1, 2, 1]\n\n \"\"\"\n if not sequence:\n return []\n\n uniq_list = [sequence[0]]\n last = sequence[0]\n for element in sequence[1:]:\n if element != last:\n uniq_list.append(element)\n last = element\n return uniq_list\n\n\ndef elide_list(line_list: List[str], max_num: int = 10) -> List[str]:\n \"\"\"Takes a long list and limits it to a smaller number of elements,\n replacing intervening elements with ``\"...\"``. For example::\n\n elide_list([\"1\", \"2\", \"3\", \"4\", \"5\", \"6\"], 4)\n\n gives::\n\n [\"1\", \"2\", \"3\", \"...\", \"6\"]\n \"\"\"\n if len(line_list) > max_num:\n return [*line_list[: max_num - 1], \"...\", line_list[-1]]\n return line_list\n\n\nif sys.version_info >= (3, 9):\n PatternStr = re.Pattern[str]\n PatternBytes = re.Pattern[bytes]\nelse:\n PatternStr = typing.Pattern[str]\n PatternBytes = typing.Pattern[bytes]\n\n\ndef fnmatch_translate_multiple(named_patterns: Dict[str, str]) -> str:\n \"\"\"Similar to ``fnmatch.translate``, but takes an ordered dictionary where keys are pattern\n names, and values are filename patterns. The output is a regex that matches any of the\n patterns in order, and named capture groups are used to identify which pattern matched.\"\"\"\n return \"|\".join(f\"(?P<{n}>{fnmatch.translate(p)})\" for n, p in named_patterns.items())\n\n\n@contextlib.contextmanager\ndef nullcontext(*args, **kwargs):\n \"\"\"Empty context manager.\n TODO: replace with contextlib.nullcontext() if we ever require python 3.7.\n \"\"\"\n yield\n\n\nclass UnhashableArguments(TypeError):\n \"\"\"Raise when an @memoized function receives unhashable arg or kwarg values.\"\"\"\n\n\nT = TypeVar(\"T\")\n\n\ndef stable_partition(\n input_iterable: Iterable[T], predicate_fn: Callable[[T], bool]\n) -> Tuple[List[T], List[T]]:\n \"\"\"Partition the input iterable according to a custom predicate.\n\n Args:\n input_iterable: input iterable to be partitioned.\n predicate_fn: predicate function accepting an iterable item\n as argument.\n\n Return:\n Tuple of the list of elements evaluating to True, and\n list of elements evaluating to False.\n \"\"\"\n true_items: List[T] = []\n false_items: List[T] = []\n for item in input_iterable:\n if predicate_fn(item):\n true_items.append(item)\n else:\n false_items.append(item)\n return true_items, false_items\n\n\ndef ensure_last(lst, *elements):\n \"\"\"Performs a stable partition of lst, ensuring that ``elements``\n occur at the end of ``lst`` in specified order. Mutates ``lst``.\n Raises ``ValueError`` if any ``elements`` are not already in ``lst``.\"\"\"\n for elt in elements:\n lst.append(lst.pop(lst.index(elt)))\n\n\nclass Const:\n \"\"\"Class level constant, raises when trying to set the attribute\"\"\"\n\n __slots__ = [\"value\"]\n\n def __init__(self, value):\n self.value = value\n\n def __get__(self, instance, owner):\n return self.value\n\n def __set__(self, instance, value):\n raise TypeError(f\"Const value does not support assignment [value={self.value}]\")\n\n\nclass TypedMutableSequence(collections.abc.MutableSequence):\n \"\"\"Base class that behaves like a list, just with a different type.\n\n Client code can inherit from this base class::\n\n class Foo(TypedMutableSequence):\n pass\n\n and later perform checks based on types::\n\n if isinstance(l, Foo):\n # do something\n \"\"\"\n\n def __init__(self, iterable):\n self.data = list(iterable)\n\n def __getitem__(self, item):\n return self.data[item]\n\n def __setitem__(self, key, value):\n self.data[key] = value\n\n def __delitem__(self, key):\n del self.data[key]\n\n def __len__(self):\n return len(self.data)\n\n def insert(self, index, item):\n self.data.insert(index, item)\n\n def __repr__(self):\n return repr(self.data)\n\n def __str__(self):\n return str(self.data)\n\n\nclass GroupedExceptionHandler:\n \"\"\"A generic mechanism to coalesce multiple exceptions and preserve tracebacks.\"\"\"\n\n def __init__(self):\n self.exceptions: List[Tuple[str, Exception, List[str]]] = []\n\n def __bool__(self):\n \"\"\"Whether any exceptions were handled.\"\"\"\n return bool(self.exceptions)\n\n def forward(self, context: str, base: type = BaseException) -> \"GroupedExceptionForwarder\":\n \"\"\"Return a contextmanager which extracts tracebacks and prefixes a message.\"\"\"\n return GroupedExceptionForwarder(context, self, base)\n\n def _receive_forwarded(self, context: str, exc: Exception, tb: List[str]):\n self.exceptions.append((context, exc, tb))\n\n def grouped_message(self, with_tracebacks: bool = True) -> str:\n \"\"\"Print out an error message coalescing all the forwarded errors.\"\"\"\n each_exception_message = [\n \"\\n\\t{0} raised {1}: {2}\\n{3}\".format(\n context, exc.__class__.__name__, exc, f\"\\n{''.join(tb)}\" if with_tracebacks else \"\"\n )\n for context, exc, tb in self.exceptions\n ]\n return \"due to the following failures:\\n{0}\".format(\"\\n\".join(each_exception_message))\n\n\nclass GroupedExceptionForwarder:\n \"\"\"A contextmanager to capture exceptions and forward them to a\n GroupedExceptionHandler.\"\"\"\n\n def __init__(self, context: str, handler: GroupedExceptionHandler, base: type):\n self._context = context\n self._handler = handler\n self._base = base\n\n def __enter__(self):\n return None\n\n def __exit__(self, exc_type, exc_value, tb):\n if exc_value is not None:\n if not issubclass(exc_type, self._base):\n return False\n self._handler._receive_forwarded(self._context, exc_value, traceback.format_tb(tb))\n\n # Suppress any exception from being re-raised:\n # https://docs.python.org/3/reference/datamodel.html#object.__exit__.\n return True\n\n\nClassPropertyType = TypeVar(\"ClassPropertyType\")\n\n\nclass classproperty(Generic[ClassPropertyType]):\n \"\"\"Non-data descriptor to evaluate a class-level property. The function that performs\n the evaluation is injected at creation time and takes an owner (i.e., the class that\n originated the instance).\n \"\"\"\n\n def __init__(self, callback: Callable[[Any], ClassPropertyType]) -> None:\n self.callback = callback\n self.__doc__ = callback.__doc__\n\n def __get__(self, instance, owner) -> ClassPropertyType:\n return self.callback(owner)\n\n\n#: A type alias that represents either a classproperty descriptor or a constant value of the same\n#: type. This allows derived classes to override a computed class-level property with a constant\n#: value while retaining type compatibility.\nClassProperty = Union[ClassPropertyType, classproperty[ClassPropertyType]]\n\n\nclass DeprecatedProperty:\n \"\"\"Data descriptor to error or warn when a deprecated property is accessed.\n\n Derived classes must define a factory method to return an adaptor for the deprecated\n property, if the descriptor is not set to error.\n \"\"\"\n\n __slots__ = [\"name\"]\n\n #: 0 - Nothing\n #: 1 - Warning\n #: 2 - Error\n error_lvl = 0\n\n def __init__(self, name: str) -> None:\n self.name = name\n\n def __get__(self, instance, owner):\n if instance is None:\n return self\n\n if self.error_lvl == 1:\n warnings.warn(\n f\"accessing the '{self.name}' property of '{instance}', which is deprecated\"\n )\n elif self.error_lvl == 2:\n raise AttributeError(f\"cannot access the '{self.name}' attribute of '{instance}'\")\n\n return self.factory(instance, owner)\n\n def __set__(self, instance, value):\n raise TypeError(\n f\"the deprecated property '{self.name}' of '{instance}' does not support assignment\"\n )\n\n def factory(self, instance, owner):\n raise NotImplementedError(\"must be implemented by derived classes\")\n\n\nKT = TypeVar(\"KT\")\nVT = TypeVar(\"VT\")\n\n\nclass PriorityOrderedMapping(Mapping[KT, VT]):\n \"\"\"Mapping that iterates over key according to an integer priority. If the priority is\n the same for two keys, insertion order is what matters.\n\n The priority is set when the key/value pair is added. If not set, the highest current priority\n is used.\n \"\"\"\n\n _data: Dict[KT, VT]\n _priorities: List[Tuple[int, KT]]\n\n def __init__(self) -> None:\n self._data = {}\n # Tuple of (priority, key)\n self._priorities = []\n\n def __getitem__(self, key: KT) -> VT:\n return self._data[key]\n\n def __len__(self) -> int:\n return len(self._data)\n\n def __iter__(self):\n yield from (key for _, key in self._priorities)\n\n def __reversed__(self):\n yield from (key for _, key in reversed(self._priorities))\n\n def reversed_keys(self):\n \"\"\"Iterates over keys from the highest priority, to the lowest.\"\"\"\n return reversed(self)\n\n def reversed_values(self):\n \"\"\"Iterates over values from the highest priority, to the lowest.\"\"\"\n yield from (self._data[key] for _, key in reversed(self._priorities))\n\n def priority_values(self, priority: int):\n \"\"\"Iterate over values of a given priority.\"\"\"\n if not any(p == priority for p, _ in self._priorities):\n raise KeyError(f\"No such priority in PriorityOrderedMapping: {priority}\")\n yield from (self._data[k] for p, k in self._priorities if p == priority)\n\n def _highest_priority(self) -> int:\n if not self._priorities:\n return 0\n result, _ = self._priorities[-1]\n return result\n\n def add(self, key: KT, *, value: VT, priority: Optional[int] = None) -> None:\n \"\"\"Adds a key/value pair to the mapping, with a specific priority.\n\n If the priority is None, then it is assumed to be the highest priority value currently\n in the container.\n\n Raises:\n ValueError: when the same priority is already in the mapping\n \"\"\"\n if priority is None:\n priority = self._highest_priority()\n\n if key in self._data:\n self.remove(key)\n\n self._priorities.append((priority, key))\n # We rely on sort being stable\n self._priorities.sort(key=lambda x: x[0])\n self._data[key] = value\n assert len(self._data) == len(self._priorities)\n\n def remove(self, key: KT) -> VT:\n \"\"\"Removes a key from the mapping.\n\n Returns:\n The value associated with the key being removed\n\n Raises:\n KeyError: if the key is not in the mapping\n \"\"\"\n if key not in self._data:\n raise KeyError(f\"cannot find {key}\")\n\n popped_item = self._data.pop(key)\n self._priorities = [(p, k) for p, k in self._priorities if k != key]\n assert len(self._data) == len(self._priorities)\n return popped_item\n" }, { "path": "lib/spack/spack/llnl/util/link_tree.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"LinkTree class for setting up trees of symbolic links.\"\"\"\n\nimport filecmp\nimport os\nimport shutil\nfrom typing import Callable, Dict, List, Optional, Tuple\n\nimport spack.llnl.util.filesystem as fs\nimport spack.llnl.util.tty as tty\n\n__all__ = [\"LinkTree\"]\n\nempty_file_name = \".spack-empty\"\n\n\ndef remove_link(src, dest):\n if not fs.islink(dest):\n raise ValueError(\"%s is not a link tree!\" % dest)\n # remove if dest is a hardlink/symlink to src; this will only\n # be false if two packages are merged into a prefix and have a\n # conflicting file\n if filecmp.cmp(src, dest, shallow=True):\n os.remove(dest)\n\n\nclass MergeConflict:\n \"\"\"\n The invariant here is that src_a and src_b are both mapped\n to dst:\n\n project(src_a) == project(src_b) == dst\n \"\"\"\n\n def __init__(self, dst, src_a=None, src_b=None):\n self.dst = dst\n self.src_a = src_a\n self.src_b = src_b\n\n def __repr__(self) -> str:\n return f\"MergeConflict(dst={self.dst!r}, src_a={self.src_a!r}, src_b={self.src_b!r})\"\n\n\ndef _samefile(a: str, b: str):\n try:\n return os.path.samefile(a, b)\n except OSError:\n return False\n\n\nclass SourceMergeVisitor(fs.BaseDirectoryVisitor):\n \"\"\"\n Visitor that produces actions:\n - An ordered list of directories to create in dst\n - A list of files to link in dst\n - A list of merge conflicts in dst/\n \"\"\"\n\n def __init__(\n self, ignore: Optional[Callable[[str], bool]] = None, normalize_paths: bool = False\n ):\n self.ignore = ignore if ignore is not None else lambda f: False\n\n # On case-insensitive filesystems, normalize paths to detect duplications\n self.normalize_paths = normalize_paths\n\n # When mapping to /, we need to prepend the \n # bit to the relative path in the destination dir.\n self.projection: str = \"\"\n\n # Two files f and g conflict if they are not os.path.samefile(f, g) and they are both\n # projected to the same destination file. These conflicts are not necessarily fatal, and\n # can be resolved or ignored. For example /LICENSE or\n # //__init__.py conflicts can be ignored).\n self.file_conflicts: List[MergeConflict] = []\n\n # When we have to create a dir where a file is, or a file where a dir is, we have fatal\n # errors, listed here.\n self.fatal_conflicts: List[MergeConflict] = []\n\n # What directories we have to make; this is an ordered dict, so that we have a fast lookup\n # and can run mkdir in order.\n self.directories: Dict[str, Tuple[str, str]] = {}\n\n # If the visitor is configured to normalize paths, keep a map of\n # normalized path to: original path, root directory + relative path\n self._directories_normalized: Dict[str, Tuple[str, str, str]] = {}\n\n # Files to link. Maps dst_rel to (src_root, src_rel). This is an ordered dict, where files\n # are guaranteed to be grouped by src_root in the order they were visited.\n self.files: Dict[str, Tuple[str, str]] = {}\n\n # If the visitor is configured to normalize paths, keep a map of\n # normalized path to: original path, root directory + relative path\n self._files_normalized: Dict[str, Tuple[str, str, str]] = {}\n\n def _in_directories(self, proj_rel_path: str) -> bool:\n \"\"\"\n Check if a path is already in the directory list\n \"\"\"\n if self.normalize_paths:\n return proj_rel_path.lower() in self._directories_normalized\n else:\n return proj_rel_path in self.directories\n\n def _directory(self, proj_rel_path: str) -> Tuple[str, str, str]:\n \"\"\"\n Get the directory that is mapped to a path\n \"\"\"\n if self.normalize_paths:\n return self._directories_normalized[proj_rel_path.lower()]\n else:\n return (proj_rel_path, *self.directories[proj_rel_path])\n\n def _del_directory(self, proj_rel_path: str):\n \"\"\"\n Remove a directory from the list of directories\n \"\"\"\n del self.directories[proj_rel_path]\n if self.normalize_paths:\n del self._directories_normalized[proj_rel_path.lower()]\n\n def _add_directory(self, proj_rel_path: str, root: str, rel_path: str):\n \"\"\"\n Add a directory to the list of directories.\n Also stores the normalized version for later lookups\n \"\"\"\n self.directories[proj_rel_path] = (root, rel_path)\n if self.normalize_paths:\n self._directories_normalized[proj_rel_path.lower()] = (proj_rel_path, root, rel_path)\n\n def _in_files(self, proj_rel_path: str) -> bool:\n \"\"\"\n Check if a path is already in the files list\n \"\"\"\n if self.normalize_paths:\n return proj_rel_path.lower() in self._files_normalized\n else:\n return proj_rel_path in self.files\n\n def _file(self, proj_rel_path: str) -> Tuple[str, str, str]:\n \"\"\"\n Get the file that is mapped to a path\n \"\"\"\n if self.normalize_paths:\n return self._files_normalized[proj_rel_path.lower()]\n else:\n return (proj_rel_path, *self.files[proj_rel_path])\n\n def _del_file(self, proj_rel_path: str):\n \"\"\"\n Remove a file from the list of files\n \"\"\"\n del self.files[proj_rel_path]\n if self.normalize_paths:\n del self._files_normalized[proj_rel_path.lower()]\n\n def _add_file(self, proj_rel_path: str, root: str, rel_path: str):\n \"\"\"\n Add a file to the list of files\n Also stores the normalized version for later lookups\n \"\"\"\n self.files[proj_rel_path] = (root, rel_path)\n if self.normalize_paths:\n self._files_normalized[proj_rel_path.lower()] = (proj_rel_path, root, rel_path)\n\n def before_visit_dir(self, root: str, rel_path: str, depth: int) -> bool:\n \"\"\"\n Register a directory if dst / rel_path is not blocked by a file or ignored.\n \"\"\"\n proj_rel_path = os.path.join(self.projection, rel_path)\n\n if self.ignore(rel_path):\n # Don't recurse when dir is ignored.\n return False\n elif self._in_files(proj_rel_path):\n # A file-dir conflict is fatal except if they're the same file (symlinked dir).\n src_a = os.path.join(*self._file(proj_rel_path))\n src_b = os.path.join(root, rel_path)\n\n if not _samefile(src_a, src_b):\n self.fatal_conflicts.append(\n MergeConflict(dst=proj_rel_path, src_a=src_a, src_b=src_b)\n )\n return False\n\n # Remove the link in favor of the dir.\n existing_proj_rel_path, _, _ = self._file(proj_rel_path)\n self._del_file(existing_proj_rel_path)\n self._add_directory(proj_rel_path, root, rel_path)\n return True\n elif self._in_directories(proj_rel_path):\n # No new directory, carry on.\n return True\n else:\n # Register new directory.\n self._add_directory(proj_rel_path, root, rel_path)\n return True\n\n def before_visit_symlinked_dir(self, root: str, rel_path: str, depth: int) -> bool:\n \"\"\"\n Replace symlinked dirs with actual directories when possible in low depths,\n otherwise handle it as a file (i.e. we link to the symlink).\n\n Transforming symlinks into dirs makes it more likely we can merge directories,\n e.g. when /lib -> /subdir/lib.\n\n We only do this when the symlink is pointing into a subdirectory from the\n symlink's directory, to avoid potential infinite recursion; and only at a\n constant level of nesting, to avoid potential exponential blowups in file\n duplication.\n \"\"\"\n if self.ignore(rel_path):\n return False\n\n # Only follow symlinked dirs in /**/**/*\n if depth > 1:\n handle_as_dir = False\n else:\n # Only follow symlinked dirs when pointing deeper\n src = os.path.join(root, rel_path)\n real_parent = os.path.realpath(os.path.dirname(src))\n real_child = os.path.realpath(src)\n handle_as_dir = real_child.startswith(real_parent)\n\n if handle_as_dir:\n return self.before_visit_dir(root, rel_path, depth)\n\n self.visit_file(root, rel_path, depth, symlink=True)\n return False\n\n def visit_file(self, root: str, rel_path: str, depth: int, *, symlink: bool = False) -> None:\n proj_rel_path = os.path.join(self.projection, rel_path)\n\n if self.ignore(rel_path):\n pass\n elif self._in_directories(proj_rel_path):\n # Can't create a file where a dir is, unless they are the same file (symlinked dir),\n # in which case we simply drop the symlink in favor of the actual dir.\n src_a = os.path.join(*self._directory(proj_rel_path))\n src_b = os.path.join(root, rel_path)\n if not symlink or not _samefile(src_a, src_b):\n self.fatal_conflicts.append(\n MergeConflict(dst=proj_rel_path, src_a=src_a, src_b=src_b)\n )\n elif self._in_files(proj_rel_path):\n # When two files project to the same path, they conflict iff they are distinct.\n # If they are the same (i.e. one links to the other), register regular files rather\n # than symlinks. The reason is that in copy-type views, we need a copy of the actual\n # file, not the symlink.\n src_a = os.path.join(*self._file(proj_rel_path))\n src_b = os.path.join(root, rel_path)\n if not _samefile(src_a, src_b):\n # Distinct files produce a conflict.\n self.file_conflicts.append(\n MergeConflict(dst=proj_rel_path, src_a=src_a, src_b=src_b)\n )\n return\n\n if not symlink:\n # Remove the link in favor of the actual file. The del is necessary to maintain the\n # order of the files dict, which is grouped by root.\n existing_proj_rel_path, _, _ = self._file(proj_rel_path)\n self._del_file(existing_proj_rel_path)\n self._add_file(proj_rel_path, root, rel_path)\n else:\n # Otherwise register this file to be linked.\n self._add_file(proj_rel_path, root, rel_path)\n\n def visit_symlinked_file(self, root: str, rel_path: str, depth: int) -> None:\n # Treat symlinked files as ordinary files (without \"dereferencing\")\n self.visit_file(root, rel_path, depth, symlink=True)\n\n def set_projection(self, projection: str) -> None:\n self.projection = os.path.normpath(projection)\n\n # Todo, is this how to check in general for empty projection?\n if self.projection == \".\":\n self.projection = \"\"\n return\n\n # If there is a projection, we'll also create the directories\n # it consists of, and check whether that's causing conflicts.\n path = \"\"\n for part in self.projection.split(os.sep):\n path = os.path.join(path, part)\n if not self._in_files(path):\n self._add_directory(path, \"\", path)\n else:\n # Can't create a dir where a file is.\n _, src_a_root, src_a_relpath = self._file(path)\n self.fatal_conflicts.append(\n MergeConflict(\n dst=path,\n src_a=os.path.join(src_a_root, src_a_relpath),\n src_b=os.path.join(\"\", path),\n )\n )\n\n\nclass DestinationMergeVisitor(fs.BaseDirectoryVisitor):\n \"\"\"DestinationMergeVisitor takes a SourceMergeVisitor and:\n\n a. registers additional conflicts when merging to the destination prefix\n b. removes redundant mkdir operations when directories already exist in the destination prefix.\n\n This also makes sure that symlinked directories in the target prefix will never be merged with\n directories in the sources directories.\n \"\"\"\n\n def __init__(self, source_merge_visitor: SourceMergeVisitor):\n self.src = source_merge_visitor\n\n def before_visit_dir(self, root: str, rel_path: str, depth: int) -> bool:\n # If destination dir is a file in a src dir, add a conflict,\n # and don't traverse deeper\n if self.src._in_files(rel_path):\n _, src_a_root, src_a_relpath = self.src._file(rel_path)\n self.src.fatal_conflicts.append(\n MergeConflict(\n rel_path, os.path.join(src_a_root, src_a_relpath), os.path.join(root, rel_path)\n )\n )\n return False\n\n # If destination dir was also a src dir, remove the mkdir\n # action, and traverse deeper.\n if self.src._in_directories(rel_path):\n existing_proj_rel_path, _, _ = self.src._directory(rel_path)\n self.src._del_directory(existing_proj_rel_path)\n return True\n\n # If the destination dir does not appear in the src dir,\n # don't descend into it.\n return False\n\n def before_visit_symlinked_dir(self, root: str, rel_path: str, depth: int) -> bool:\n \"\"\"\n Symlinked directories in the destination prefix should\n be seen as files; we should not accidentally merge\n source dir with a symlinked dest dir.\n \"\"\"\n\n self.visit_file(root, rel_path, depth)\n\n # Never descend into symlinked target dirs.\n return False\n\n def visit_file(self, root: str, rel_path: str, depth: int) -> None:\n # Can't merge a file if target already exists\n if self.src._in_directories(rel_path):\n _, src_a_root, src_a_relpath = self.src._directory(rel_path)\n self.src.fatal_conflicts.append(\n MergeConflict(\n rel_path, os.path.join(src_a_root, src_a_relpath), os.path.join(root, rel_path)\n )\n )\n\n elif self.src._in_files(rel_path):\n _, src_a_root, src_a_relpath = self.src._file(rel_path)\n self.src.fatal_conflicts.append(\n MergeConflict(\n rel_path, os.path.join(src_a_root, src_a_relpath), os.path.join(root, rel_path)\n )\n )\n\n def visit_symlinked_file(self, root: str, rel_path: str, depth: int) -> None:\n # Treat symlinked files as ordinary files (without \"dereferencing\")\n self.visit_file(root, rel_path, depth)\n\n\nclass LinkTree:\n \"\"\"Class to create trees of symbolic links from a source directory.\n\n LinkTree objects are constructed with a source root. Their\n methods allow you to create and delete trees of symbolic links\n back to the source tree in specific destination directories.\n Trees comprise symlinks only to files; directories are never\n symlinked to, to prevent the source directory from ever being\n modified.\n \"\"\"\n\n def __init__(self, source_root):\n if not os.path.exists(source_root):\n raise OSError(\"No such file or directory: '%s'\", source_root)\n\n self._root = source_root\n\n def find_conflict(self, dest_root, ignore=None, ignore_file_conflicts=False):\n \"\"\"Returns the first file in dest that conflicts with src\"\"\"\n ignore = ignore or (lambda x: False)\n conflicts = self.find_dir_conflicts(dest_root, ignore)\n\n if not ignore_file_conflicts:\n conflicts.extend(\n dst\n for src, dst in self.get_file_map(dest_root, ignore).items()\n if os.path.exists(dst)\n )\n\n if conflicts:\n return conflicts[0]\n\n def find_dir_conflicts(self, dest_root, ignore):\n conflicts = []\n kwargs = {\"follow_nonexisting\": False, \"ignore\": ignore}\n for src, dest in fs.traverse_tree(self._root, dest_root, **kwargs):\n if os.path.isdir(src):\n if os.path.exists(dest) and not os.path.isdir(dest):\n conflicts.append(\"File blocks directory: %s\" % dest)\n elif os.path.exists(dest) and os.path.isdir(dest):\n conflicts.append(\"Directory blocks directory: %s\" % dest)\n return conflicts\n\n def get_file_map(self, dest_root, ignore):\n merge_map = {}\n kwargs = {\"follow_nonexisting\": True, \"ignore\": ignore}\n for src, dest in fs.traverse_tree(self._root, dest_root, **kwargs):\n if not os.path.isdir(src):\n merge_map[src] = dest\n return merge_map\n\n def merge_directories(self, dest_root, ignore):\n for src, dest in fs.traverse_tree(self._root, dest_root, ignore=ignore):\n if os.path.isdir(src):\n if not os.path.exists(dest):\n fs.mkdirp(dest)\n continue\n\n if not os.path.isdir(dest):\n raise ValueError(\"File blocks directory: %s\" % dest)\n\n # mark empty directories so they aren't removed on unmerge.\n if not os.listdir(dest):\n marker = os.path.join(dest, empty_file_name)\n fs.touch(marker)\n\n def unmerge_directories(self, dest_root, ignore):\n for src, dest in fs.traverse_tree(self._root, dest_root, ignore=ignore, order=\"post\"):\n if os.path.isdir(src):\n if not os.path.exists(dest):\n continue\n elif not os.path.isdir(dest):\n raise ValueError(\"File blocks directory: %s\" % dest)\n\n # remove directory if it is empty.\n if not os.listdir(dest):\n shutil.rmtree(dest, ignore_errors=True)\n\n # remove empty dir marker if present.\n marker = os.path.join(dest, empty_file_name)\n if os.path.exists(marker):\n os.remove(marker)\n\n def merge(\n self,\n dest_root,\n ignore_conflicts: bool = False,\n ignore: Optional[Callable[[str], bool]] = None,\n link: Callable = fs.symlink,\n relative: bool = False,\n ):\n \"\"\"Link all files in src into dest, creating directories if necessary.\n\n Arguments:\n\n ignore_conflicts: if True, do not break when the target exists; return a list of files\n that could not be linked\n\n ignore: callable that returns True if a file is to be ignored in the merge (by default\n ignore nothing)\n\n link: function to create links with (defaults to\n ``spack.llnl.util.filesystem.symlink``)\n\n relative: create all symlinks relative to the target (default False)\n \"\"\"\n if ignore is None:\n ignore = lambda x: False\n\n conflict = self.find_conflict(\n dest_root, ignore=ignore, ignore_file_conflicts=ignore_conflicts\n )\n if conflict:\n raise SingleMergeConflictError(conflict)\n\n self.merge_directories(dest_root, ignore)\n existing = []\n for src, dst in self.get_file_map(dest_root, ignore).items():\n if os.path.exists(dst):\n existing.append(dst)\n elif relative:\n abs_src = os.path.abspath(src)\n dst_dir = os.path.dirname(os.path.abspath(dst))\n rel = os.path.relpath(abs_src, dst_dir)\n link(rel, dst)\n else:\n link(src, dst)\n\n for c in existing:\n tty.warn(\"Could not merge: %s\" % c)\n\n def unmerge(self, dest_root, ignore=None, remove_file=remove_link):\n \"\"\"Unlink all files in dest that exist in src.\n\n Unlinks directories in dest if they are empty.\n \"\"\"\n if ignore is None:\n ignore = lambda x: False\n\n for src, dst in self.get_file_map(dest_root, ignore).items():\n remove_file(src, dst)\n self.unmerge_directories(dest_root, ignore)\n\n\nclass MergeConflictError(Exception):\n pass\n\n\nclass ConflictingSpecsError(MergeConflictError):\n def __init__(self, spec_1, spec_2):\n super().__init__(spec_1, spec_2)\n\n\nclass SingleMergeConflictError(MergeConflictError):\n def __init__(self, path):\n super().__init__(\"Package merge blocked by file: %s\" % path)\n\n\nclass MergeConflictSummary(MergeConflictError):\n def __init__(self, conflicts):\n \"\"\"\n A human-readable summary of file system view merge conflicts (showing only the\n first 3 issues.)\n \"\"\"\n msg = \"{0} fatal error(s) when merging prefixes:\".format(len(conflicts))\n # show the first 3 merge conflicts.\n for conflict in conflicts[:3]:\n msg += \"\\n `{0}` and `{1}` both project to `{2}`\".format(\n conflict.src_a, conflict.src_b, conflict.dst\n )\n super().__init__(msg)\n" }, { "path": "lib/spack/spack/llnl/util/lock.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport errno\nimport os\nimport socket\nimport sys\nimport time\nfrom datetime import datetime\nfrom types import TracebackType\nfrom typing import IO, Callable, Dict, Generator, Optional, Tuple, Type\n\nfrom spack.llnl.util import lang, tty\n\nfrom ..string import plural\n\nif sys.platform != \"win32\":\n import fcntl\n\n\n__all__ = [\n \"Lock\",\n \"LockDowngradeError\",\n \"LockUpgradeError\",\n \"LockTransaction\",\n \"WriteTransaction\",\n \"ReadTransaction\",\n \"LockError\",\n \"LockTimeoutError\",\n \"LockPermissionError\",\n \"LockROFileError\",\n \"CantCreateLockError\",\n]\n\n\nExitFnType = Callable[\n [Optional[Type[BaseException]], Optional[BaseException], Optional[TracebackType]],\n Optional[bool],\n]\nReleaseFnType = Optional[Callable[[], Optional[bool]]]\nDevIno = Tuple[int, int] # (st_dev, st_ino) from os.stat_result\n\n\ndef true_fn() -> bool:\n \"\"\"A function that always returns True.\"\"\"\n return True\n\n\nclass OpenFile:\n \"\"\"Record for keeping track of open lockfiles (with reference counting).\"\"\"\n\n __slots__ = (\"fh\", \"key\", \"refs\")\n\n def __init__(self, fh: IO[bytes], key: DevIno):\n self.fh = fh\n self.key = key # (dev, ino)\n self.refs = 0\n\n\nclass OpenFileTracker:\n \"\"\"Track open lockfiles by inode, to minimize the number of open file descriptors.\n\n ``fcntl`` locks are associated with an inode. If a process closes *any* file descriptor for an\n inode, all fcntl locks the process holds on that inode are released, even if other descriptors\n for the same inode are still open.\n\n To avoid accidentally dropping locks we keep at most one open file descriptor per inode and\n reference-count it. The descriptor is only closed when the reference count reaches zero (i.e.\n no ``Lock`` in this process still needs it).\n\n Descriptors are *not* released on unlock; they are kept alive across lock/unlock cycles so that\n the next lock operation can skip re-opening the file. ``Lock._ensure_valid_handle``\n re-validates the on-disk inode before each lock operation and drops a stale descriptor when\n the file was deleted and replaced.\n \"\"\"\n\n def __init__(self):\n self._descriptors: Dict[DevIno, OpenFile] = {}\n\n def get_ref_for_inode(self, key: DevIno) -> Optional[OpenFile]:\n \"\"\"Fast lookup: do we already have this inode open?\"\"\"\n return self._descriptors.get(key)\n\n def create_and_track(self, path: str) -> OpenFile:\n \"\"\"Slow path: Open file, handle directory creation, track it.\"\"\"\n # Open the file and create it if it doesn't exist (incl. directories).\n try:\n try:\n fd = os.open(path, os.O_RDWR | os.O_CREAT)\n mode = \"rb+\"\n except PermissionError:\n fd = os.open(path, os.O_RDONLY)\n mode = \"rb\"\n except OSError as e:\n if e.errno != errno.ENOENT:\n raise\n # Directory missing, create and retry\n try:\n os.makedirs(os.path.dirname(path), exist_ok=True)\n fd = os.open(path, os.O_RDWR | os.O_CREAT)\n except OSError:\n raise CantCreateLockError(path)\n mode = \"rb+\"\n\n # Get file identifier (device, inode) for tracking.\n stat = os.fstat(fd)\n key = (stat.st_dev, stat.st_ino)\n\n # Did we open a file we already track, e.g. a symlink to existing tracker file.\n if key in self._descriptors:\n os.close(fd)\n existing = self._descriptors[key]\n existing.refs += 1\n return existing\n\n # Track the new file.\n fh = os.fdopen(fd, mode)\n obj = OpenFile(fh, key)\n obj.refs += 1\n self._descriptors[key] = obj\n return obj\n\n def release(self, open_file: OpenFile):\n \"\"\"Decrement the reference count and close the file handle when it reaches zero.\"\"\"\n open_file.refs -= 1\n if open_file.refs <= 0:\n if self._descriptors.get(open_file.key) is open_file:\n del self._descriptors[open_file.key]\n open_file.fh.close()\n\n def purge(self):\n \"\"\"Close all tracked file descriptors and clear the cache.\"\"\"\n for open_file in self._descriptors.values():\n open_file.fh.close()\n self._descriptors.clear()\n\n\n#: Open file descriptors for locks in this process. Used to prevent one process\n#: from opening the sam file many times for different byte range locks\nFILE_TRACKER = OpenFileTracker()\n\n\ndef _attempts_str(wait_time, nattempts):\n # Don't print anything if we succeeded on the first try\n if nattempts <= 1:\n return \"\"\n\n attempts = plural(nattempts, \"attempt\")\n return \" after {} and {}\".format(lang.pretty_seconds(wait_time), attempts)\n\n\nclass LockType:\n READ = 0\n WRITE = 1\n\n @staticmethod\n def to_str(tid):\n ret = \"READ\"\n if tid == LockType.WRITE:\n ret = \"WRITE\"\n return ret\n\n @staticmethod\n def to_module(tid):\n lock = fcntl.LOCK_SH\n if tid == LockType.WRITE:\n lock = fcntl.LOCK_EX\n return lock\n\n @staticmethod\n def is_valid(op: int) -> bool:\n return op == LockType.READ or op == LockType.WRITE\n\n\nclass Lock:\n \"\"\"This is an implementation of a filesystem lock using Python's lockf.\n\n In Python, ``lockf`` actually calls ``fcntl``, so this should work with any filesystem\n implementation that supports locking through the fcntl calls. This includes distributed\n filesystems like Lustre (when flock is enabled) and recent NFS versions.\n\n Note that this is for managing contention over resources *between* processes and not for\n managing contention between threads in a process: the functions of this object are not\n thread-safe. A process also must not maintain multiple locks on the same file (or, more\n specifically, on overlapping byte ranges in the same file).\n \"\"\"\n\n def __init__(\n self,\n path: str,\n *,\n start: int = 0,\n length: int = 0,\n default_timeout: Optional[float] = None,\n debug: bool = False,\n desc: str = \"\",\n ) -> None:\n \"\"\"Construct a new lock on the file at ``path``.\n\n By default, the lock applies to the whole file. Optionally, caller can specify a byte\n range beginning ``start`` bytes from the start of the file and extending ``length`` bytes\n from there.\n\n This exposes a subset of fcntl locking functionality. It does not currently expose the\n ``whence`` parameter -- ``whence`` is always ``os.SEEK_SET`` and ``start`` is always\n evaluated from the beginning of the file.\n\n Args:\n path: path to the lock\n start: optional byte offset at which the lock starts\n length: optional number of bytes to lock\n default_timeout: seconds to wait for lock attempts, where None means to wait\n indefinitely\n debug: debug mode specific to locking\n desc: optional debug message lock description, which is helpful for distinguishing\n between different Spack locks.\n \"\"\"\n self.path = path\n self._reads = 0\n self._writes = 0\n self._file_ref: Optional[OpenFile] = None\n self._cached_key: Optional[DevIno] = None\n\n # byte range parameters\n self._start = start\n self._length = length\n\n # enable debug mode\n self.debug = debug\n\n # optional debug description\n self.desc = f\" ({desc})\" if desc else \"\"\n\n # If the user doesn't set a default timeout, or if they choose\n # None, 0, etc. then lock attempts will not time out (unless the\n # user sets a timeout for each attempt)\n self.default_timeout = default_timeout or None\n\n # PID and host of lock holder (only used in debug mode)\n self.pid: Optional[int] = None\n self.old_pid: Optional[int] = None\n self.host: Optional[str] = None\n self.old_host: Optional[str] = None\n\n def _ensure_valid_handle(self) -> IO[bytes]:\n \"\"\"Return a valid file handle for the lock file, opening or re-opening as needed.\n\n On the happy path this costs a single ``os.stat`` syscall: if the inode on disk matches\n ``_cached_key``, the already-open file handle is returned immediately.\n\n If the inode changed (the lock file was deleted and replaced by another process), the stale\n reference is released and a fresh one is obtained. If the file does not exist yet it is\n created (along with any missing parent directories).\n \"\"\"\n try:\n # Check what is currently on disk. This is the only syscall in the happy path.\n stat_res = os.stat(self.path)\n current_key = (stat_res.st_dev, stat_res.st_ino)\n\n # Double-check that our cache corresponds the file on disk.\n if self._file_ref and not self._file_ref.fh.closed:\n if self._cached_key == current_key:\n return self._file_ref.fh\n\n # Stale path: file was deleted and replaced on disk.\n FILE_TRACKER.release(self._file_ref)\n self._file_ref = None\n\n # Get reference to the verified inode from the tracker if it exist, or a new one.\n existing_ref = FILE_TRACKER.get_ref_for_inode(current_key)\n if existing_ref:\n self._file_ref = existing_ref\n self._file_ref.refs += 1\n else:\n # We don't have it tracked, so we need to open and track it ourselves.\n self._file_ref = FILE_TRACKER.create_and_track(self.path)\n except OSError as e:\n # Re-raise all errors except for \"file not found\".\n if e.errno != errno.ENOENT:\n raise\n\n # File was not found, so remove it from our cache.\n if self._file_ref:\n FILE_TRACKER.release(self._file_ref)\n self._file_ref = None\n\n self._file_ref = FILE_TRACKER.create_and_track(self.path)\n\n # Update our local cache of what we hold\n self._cached_key = self._file_ref.key\n\n return self._file_ref.fh\n\n @staticmethod\n def _poll_interval_generator(\n _wait_times: Optional[Tuple[float, float, float]] = None,\n ) -> Generator[float, None, None]:\n \"\"\"This implements a backoff scheme for polling a contended resource by suggesting a\n succession of wait times between polls.\n\n It suggests a poll interval of .1s until 2 seconds have passed, then a poll interval of\n .2s until 10 seconds have passed, and finally (for all requests after 10s) suggests a poll\n interval of .5s.\n\n This doesn't actually track elapsed time, it estimates the waiting time as though the\n caller always waits for the full length of time suggested by this function.\n \"\"\"\n num_requests = 0\n stage1, stage2, stage3 = _wait_times or (1e-1, 2e-1, 5e-1)\n wait_time = stage1\n while True:\n if num_requests >= 60: # 40 * .2 = 8\n wait_time = stage3\n elif num_requests >= 20: # 20 * .1 = 2\n wait_time = stage2\n num_requests += 1\n yield wait_time\n\n def __repr__(self) -> str:\n \"\"\"Formal representation of the lock.\"\"\"\n rep = f\"{self.__class__.__name__}(\"\n for attr, value in self.__dict__.items():\n rep += f\"{attr}={value.__repr__()}, \"\n return f\"{rep.strip(', ')})\"\n\n def __str__(self) -> str:\n \"\"\"Readable string (with key fields) of the lock.\"\"\"\n location = f\"{self.path}[{self._start}:{self._length}]\"\n timeout = f\"timeout={self.default_timeout}\"\n activity = f\"#reads={self._reads}, #writes={self._writes}\"\n return f\"({location}, {timeout}, {activity})\"\n\n def __getstate__(self):\n \"\"\"Don't include file handles or counts in pickled state.\"\"\"\n state = self.__dict__.copy()\n del state[\"_file_ref\"]\n del state[\"_reads\"]\n del state[\"_writes\"]\n return state\n\n def __setstate__(self, state):\n self.__dict__.update(state)\n self._file_ref = None\n self._reads = 0\n self._writes = 0\n\n def _lock(self, op: int, timeout: Optional[float] = None) -> Tuple[float, int]:\n \"\"\"This takes a lock using POSIX locks (``fcntl.lockf``).\n\n The lock is implemented as a spin lock using a nonblocking call to ``lockf()``.\n\n If the lock times out, it raises a ``LockError``. If the lock is successfully acquired, the\n total wait time and the number of attempts is returned.\n \"\"\"\n assert LockType.is_valid(op)\n op_str = LockType.to_str(op)\n\n self._log_acquiring(\"{0} LOCK\".format(op_str))\n timeout = timeout or self.default_timeout\n\n fh = self._ensure_valid_handle()\n\n if LockType.to_module(op) == fcntl.LOCK_EX and fh.mode == \"rb\":\n # Attempt to upgrade to write lock w/a read-only file.\n # If the file were writable, we'd have opened it rb+\n raise LockROFileError(self.path)\n\n self._log_debug(\n \"{} locking [{}:{}]: timeout {}\".format(\n op_str.lower(), self._start, self._length, lang.pretty_seconds(timeout or 0)\n )\n )\n\n start_time = time.monotonic()\n end_time = float(\"inf\") if not timeout else start_time + timeout\n num_attempts = 1\n poll_intervals = Lock._poll_interval_generator()\n\n while True:\n if self._poll_lock(op):\n return time.monotonic() - start_time, num_attempts\n if time.monotonic() >= end_time:\n break\n time.sleep(next(poll_intervals))\n num_attempts += 1\n\n raise LockTimeoutError(op, self.path, time.monotonic() - start_time, num_attempts)\n\n def _poll_lock(self, op: int) -> bool:\n \"\"\"Attempt to acquire the lock in a non-blocking manner. Return whether\n the locking attempt succeeds\n \"\"\"\n assert self._file_ref is not None, \"cannot poll a lock without the file being set\"\n fh = self._file_ref.fh.fileno()\n module_op = LockType.to_module(op)\n try:\n # Try to get the lock (will raise if not available.)\n fcntl.lockf(fh, module_op | fcntl.LOCK_NB, self._length, self._start, os.SEEK_SET)\n\n # help for debugging distributed locking\n if self.debug:\n # All locks read the owner PID and host\n self._read_log_debug_data()\n self._log_debug(\n \"{0} locked {1} [{2}:{3}] (owner={4})\".format(\n LockType.to_str(op), self.path, self._start, self._length, self.pid\n )\n )\n\n # Exclusive locks write their PID/host\n if module_op == fcntl.LOCK_EX:\n self._write_log_debug_data()\n\n return True\n\n except OSError as e:\n # EAGAIN and EACCES == locked by another process (so try again)\n if e.errno not in (errno.EAGAIN, errno.EACCES):\n raise\n\n return False\n\n def _read_log_debug_data(self) -> None:\n \"\"\"Read PID and host data out of the file if it is there.\"\"\"\n assert self._file_ref is not None, \"cannot read debug log without the file being set\"\n self.old_pid = self.pid\n self.old_host = self.host\n\n self._file_ref.fh.seek(0)\n line = self._file_ref.fh.read()\n if line:\n pid, host = line.decode(\"utf-8\").strip().split(\",\")\n _, _, pid = pid.rpartition(\"=\")\n _, _, self.host = host.rpartition(\"=\")\n self.pid = int(pid)\n\n def _write_log_debug_data(self) -> None:\n \"\"\"Write PID and host data to the file, recording old values.\"\"\"\n assert self._file_ref is not None, \"cannot write debug log without the file being set\"\n self.old_pid = self.pid\n self.old_host = self.host\n\n self.pid = os.getpid()\n self.host = socket.gethostname()\n\n # write pid, host to disk to sync over FS\n self._file_ref.fh.seek(0)\n self._file_ref.fh.write(f\"pid={self.pid},host={self.host}\".encode(\"utf-8\"))\n self._file_ref.fh.truncate()\n self._file_ref.fh.flush()\n os.fsync(self._file_ref.fh.fileno())\n\n def _unlock(self) -> None:\n \"\"\"Releases a lock using POSIX locks (``fcntl.lockf``)\n\n Releases the lock regardless of mode. Note that read locks may be masquerading as write\n locks, but this removes either.\n \"\"\"\n assert self._file_ref is not None, \"cannot unlock without the file being set\"\n fcntl.lockf(\n self._file_ref.fh.fileno(), fcntl.LOCK_UN, self._length, self._start, os.SEEK_SET\n )\n self._reads = 0\n self._writes = 0\n\n def acquire_read(self, timeout: Optional[float] = None) -> bool:\n \"\"\"Acquires a recursive, shared lock for reading.\n\n Read and write locks can be acquired and released in arbitrary order, but the POSIX lock is\n held until all local read and write locks are released.\n\n Returns True if it is the first acquire and actually acquires the POSIX lock, False if it\n is a nested transaction.\n \"\"\"\n timeout = timeout or self.default_timeout\n\n if self._reads == 0 and self._writes == 0:\n # can raise LockError.\n wait_time, nattempts = self._lock(LockType.READ, timeout=timeout)\n self._reads += 1\n # Log if acquired, which includes counts when verbose\n self._log_acquired(\"READ LOCK\", wait_time, nattempts)\n return True\n else:\n # Increment the read count for nested lock tracking\n self._reaffirm_lock()\n self._reads += 1\n return False\n\n def acquire_write(self, timeout: Optional[float] = None) -> bool:\n \"\"\"Acquires a recursive, exclusive lock for writing.\n\n Read and write locks can be acquired and released in arbitrary order, but the POSIX lock\n is held until all local read and write locks are released.\n\n Returns True if it is the first acquire and actually acquires the POSIX lock, False if it\n is a nested transaction.\n \"\"\"\n timeout = timeout or self.default_timeout\n\n if self._writes == 0:\n # can raise LockError.\n wait_time, nattempts = self._lock(LockType.WRITE, timeout=timeout)\n self._writes += 1\n # Log if acquired, which includes counts when verbose\n self._log_acquired(\"WRITE LOCK\", wait_time, nattempts)\n\n # return True only if we weren't nested in a read lock.\n # TODO: we may need to return two values: whether we got\n # the write lock, and whether this is acquiring a read OR\n # write lock for the first time. Now it returns the latter.\n return self._reads == 0\n else:\n # Increment the write count for nested lock tracking\n self._reaffirm_lock()\n self._writes += 1\n return False\n\n def _reaffirm_lock(self) -> None:\n \"\"\"Fork-safety: always re-affirm the lock with one non-blocking attempt. In the same\n process, re-locking an already-held byte range succeeds instantly (POSIX). In a forked\n child that doesn't own the POSIX lock, the call fails immediately and we raise. Use WRITE\n if we hold an exclusive lock so we don't accidentally downgrade it.\"\"\"\n if self._writes > 0:\n op = LockType.WRITE\n elif self._reads > 0:\n op = LockType.READ\n else:\n return\n self._ensure_valid_handle()\n if not self._poll_lock(op):\n raise LockTimeoutError(op, self.path, time=0, attempts=1)\n\n def try_acquire_read(self) -> bool:\n \"\"\"Non-blocking attempt to acquire a shared read lock.\n\n Returns True if the lock was acquired, False if it would block.\n \"\"\"\n if self._reads == 0 and self._writes == 0:\n self._ensure_valid_handle()\n if not self._poll_lock(LockType.READ):\n return False\n self._reads += 1\n self._log_acquired(\"READ LOCK\", 0, 1)\n return True\n else:\n self._reaffirm_lock()\n self._reads += 1\n return True\n\n def try_acquire_write(self) -> bool:\n \"\"\"Non-blocking attempt to acquire an exclusive write lock.\n\n Returns True if the lock was acquired, False if it would block.\n \"\"\"\n if self._writes == 0:\n fh = self._ensure_valid_handle()\n if LockType.to_module(LockType.WRITE) == fcntl.LOCK_EX and fh.mode == \"rb\":\n raise LockROFileError(self.path)\n if not self._poll_lock(LockType.WRITE):\n return False\n self._writes += 1\n self._log_acquired(\"WRITE LOCK\", 0, 1)\n return True\n else:\n self._reaffirm_lock()\n self._writes += 1\n return True\n\n def is_write_locked(self) -> bool:\n \"\"\"Returns ``True`` if the path is write locked, otherwise, ``False``\"\"\"\n try:\n self.acquire_read()\n\n # If we have a read lock then no other process has a write lock.\n self.release_read()\n except LockTimeoutError:\n # Another process is holding a write lock on the file\n return True\n\n return False\n\n def downgrade_write_to_read(self, timeout: Optional[float] = None) -> None:\n \"\"\"Downgrade from an exclusive write lock to a shared read.\n\n Raises:\n LockDowngradeError: if this is an attempt at a nested transaction\n \"\"\"\n timeout = timeout or self.default_timeout\n\n if self._writes == 1 and self._reads == 0:\n self._log_downgrading()\n # can raise LockError.\n wait_time, nattempts = self._lock(LockType.READ, timeout=timeout)\n self._reads = 1\n self._writes = 0\n self._log_downgraded(wait_time, nattempts)\n else:\n raise LockDowngradeError(self.path)\n\n def upgrade_read_to_write(self, timeout: Optional[float] = None) -> None:\n \"\"\"Attempts to upgrade from a shared read lock to an exclusive write.\n\n Raises:\n LockUpgradeError: if this is an attempt at a nested transaction\n \"\"\"\n timeout = timeout or self.default_timeout\n\n if self._reads == 1 and self._writes == 0:\n self._log_upgrading()\n # can raise LockError.\n wait_time, nattempts = self._lock(LockType.WRITE, timeout=timeout)\n self._reads = 0\n self._writes = 1\n self._log_upgraded(wait_time, nattempts)\n else:\n raise LockUpgradeError(self.path)\n\n def release_read(self, release_fn: ReleaseFnType = None) -> bool:\n \"\"\"Releases a read lock.\n\n Arguments:\n release_fn: function to call *before* the last recursive lock (read or write) is\n released.\n\n If the last recursive lock will be released, then this will call release_fn and return its\n result (if provided), or return True (if release_fn was not provided).\n\n Otherwise, we are still nested inside some other lock, so do not call the release_fn and,\n return False.\n\n Does limited correctness checking: if a read lock is released when none are held, this\n will raise an assertion error.\n \"\"\"\n assert self._reads > 0\n\n locktype = \"READ LOCK\"\n if self._reads == 1 and self._writes == 0:\n self._log_releasing(locktype)\n\n # we need to call release_fn before releasing the lock\n release_fn = release_fn or true_fn\n result = release_fn()\n\n self._unlock() # can raise LockError.\n self._reads = 0\n self._log_released(locktype)\n return bool(result)\n else:\n self._reads -= 1\n return False\n\n def release_write(self, release_fn: ReleaseFnType = None) -> bool:\n \"\"\"Releases a write lock.\n\n Arguments:\n release_fn: function to call before the last recursive write is released.\n\n If the last recursive *write* lock will be released, then this will call release_fn and\n return its result (if provided), or return True (if release_fn was not provided).\n Otherwise, we are still nested inside some other write lock, so do not call the release_fn,\n and return False.\n\n Does limited correctness checking: if a read lock is released when none are held, this\n will raise an assertion error.\n \"\"\"\n assert self._writes > 0\n release_fn = release_fn or true_fn\n\n locktype = \"WRITE LOCK\"\n if self._writes == 1:\n self._log_releasing(locktype)\n\n # we need to call release_fn before releasing the lock\n result = release_fn()\n\n if self._reads > 0:\n self._lock(LockType.READ)\n else:\n self._unlock() # can raise LockError.\n\n self._writes = 0\n self._log_released(locktype)\n return bool(result)\n else:\n self._writes -= 1\n return False\n\n def cleanup(self) -> None:\n if self._reads == 0 and self._writes == 0:\n os.unlink(self.path)\n else:\n raise LockError(\"Attempting to cleanup active lock.\")\n\n def _get_counts_desc(self) -> str:\n return (\n \"(reads {0}, writes {1})\".format(self._reads, self._writes) if tty.is_verbose() else \"\"\n )\n\n def _log_acquired(self, locktype, wait_time, nattempts) -> None:\n attempts_part = _attempts_str(wait_time, nattempts)\n now = datetime.now()\n desc = \"Acquired at %s\" % now.strftime(\"%H:%M:%S.%f\")\n self._log_debug(self._status_msg(locktype, \"{0}{1}\".format(desc, attempts_part)))\n\n def _log_acquiring(self, locktype) -> None:\n self._log_debug(self._status_msg(locktype, \"Acquiring\"), level=3)\n\n def _log_debug(self, *args, **kwargs) -> None:\n \"\"\"Output lock debug messages.\"\"\"\n kwargs[\"level\"] = kwargs.get(\"level\", 2)\n tty.debug(*args, **kwargs)\n\n def _log_downgraded(self, wait_time, nattempts) -> None:\n attempts_part = _attempts_str(wait_time, nattempts)\n now = datetime.now()\n desc = \"Downgraded at %s\" % now.strftime(\"%H:%M:%S.%f\")\n self._log_debug(self._status_msg(\"READ LOCK\", \"{0}{1}\".format(desc, attempts_part)))\n\n def _log_downgrading(self) -> None:\n self._log_debug(self._status_msg(\"WRITE LOCK\", \"Downgrading\"), level=3)\n\n def _log_released(self, locktype) -> None:\n now = datetime.now()\n desc = \"Released at %s\" % now.strftime(\"%H:%M:%S.%f\")\n self._log_debug(self._status_msg(locktype, desc))\n\n def _log_releasing(self, locktype) -> None:\n self._log_debug(self._status_msg(locktype, \"Releasing\"), level=3)\n\n def _log_upgraded(self, wait_time, nattempts) -> None:\n attempts_part = _attempts_str(wait_time, nattempts)\n now = datetime.now()\n desc = \"Upgraded at %s\" % now.strftime(\"%H:%M:%S.%f\")\n self._log_debug(self._status_msg(\"WRITE LOCK\", \"{0}{1}\".format(desc, attempts_part)))\n\n def _log_upgrading(self) -> None:\n self._log_debug(self._status_msg(\"READ LOCK\", \"Upgrading\"), level=3)\n\n def _status_msg(self, locktype: str, status: str) -> str:\n status_desc = \"[{0}] {1}\".format(status, self._get_counts_desc())\n return \"{0}{1.desc}: {1.path}[{1._start}:{1._length}] {2}\".format(\n locktype, self, status_desc\n )\n\n\nclass LockTransaction:\n \"\"\"Simple nested transaction context manager that uses a file lock.\n\n Arguments:\n lock: underlying lock for this transaction to be acquired on enter and released on exit\n acquire: function to be called after lock is acquired\n release: function to be called before release, with ``(exc_type, exc_value, traceback)``\n timeout: number of seconds to set for the timeout when acquiring the lock (default no\n timeout)\n \"\"\"\n\n def __init__(\n self,\n lock: Lock,\n acquire: Optional[Callable[[], None]] = None,\n release: Optional[ExitFnType] = None,\n timeout: Optional[float] = None,\n ) -> None:\n self._lock = lock\n self._timeout = timeout\n self._acquire_fn = acquire\n self._release_fn = release\n\n def __enter__(self):\n if self._enter() and self._acquire_fn:\n return self._acquire_fn()\n\n def __exit__(\n self,\n exc_type: Optional[Type[BaseException]],\n exc_value: Optional[BaseException],\n traceback: Optional[TracebackType],\n ) -> bool:\n def release_fn():\n if self._release_fn is not None:\n return self._release_fn(exc_type, exc_value, traceback)\n\n return bool(self._exit(release_fn))\n\n def _enter(self) -> bool:\n raise NotImplementedError\n\n def _exit(self, release_fn: ReleaseFnType) -> bool:\n raise NotImplementedError\n\n\nclass ReadTransaction(LockTransaction):\n \"\"\"LockTransaction context manager that does a read and releases it.\"\"\"\n\n def _enter(self):\n return self._lock.acquire_read(self._timeout)\n\n def _exit(self, release_fn):\n return self._lock.release_read(release_fn)\n\n\nclass WriteTransaction(LockTransaction):\n \"\"\"LockTransaction context manager that does a write and releases it.\"\"\"\n\n def _enter(self):\n return self._lock.acquire_write(self._timeout)\n\n def _exit(self, release_fn):\n return self._lock.release_write(release_fn)\n\n\nclass LockError(Exception):\n \"\"\"Raised for any errors related to locks.\"\"\"\n\n\nclass LockDowngradeError(LockError):\n \"\"\"Raised when unable to downgrade from a write to a read lock.\"\"\"\n\n def __init__(self, path: str) -> None:\n msg = \"Cannot downgrade lock from write to read on file: %s\" % path\n super().__init__(msg)\n\n\nclass LockTimeoutError(LockError):\n \"\"\"Raised when an attempt to acquire a lock times out.\"\"\"\n\n def __init__(self, lock_type: int, path: str, time: float, attempts: int) -> None:\n lock_type_str = LockType.to_str(lock_type).lower()\n fmt = \"Timed out waiting for a {} lock after {}.\\n Made {} {} on file: {}\"\n super().__init__(\n fmt.format(\n lock_type_str,\n lang.pretty_seconds(time),\n attempts,\n \"attempt\" if attempts == 1 else \"attempts\",\n path,\n )\n )\n\n\nclass LockUpgradeError(LockError):\n \"\"\"Raised when unable to upgrade from a read to a write lock.\"\"\"\n\n def __init__(self, path: str) -> None:\n msg = \"Cannot upgrade lock from read to write on file: %s\" % path\n super().__init__(msg)\n\n\nclass LockPermissionError(LockError):\n \"\"\"Raised when there are permission issues with a lock.\"\"\"\n\n\nclass LockROFileError(LockPermissionError):\n \"\"\"Tried to take an exclusive lock on a read-only file.\"\"\"\n\n def __init__(self, path: str) -> None:\n msg = \"Can't take write lock on read-only file: %s\" % path\n super().__init__(msg)\n\n\nclass CantCreateLockError(LockPermissionError):\n \"\"\"Attempt to create a lock in an unwritable location.\"\"\"\n\n def __init__(self, path: str) -> None:\n msg = \"cannot create lock '%s': \" % path\n msg += \"file does not exist and location is not writable\"\n super().__init__(msg)\n" }, { "path": "lib/spack/spack/llnl/util/symlink.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport warnings\n\nimport spack.error\nimport spack.llnl.util.filesystem\n\nwarnings.warn(\n \"The `spack.llnl.util.symlink` module will be removed in Spack v1.1\",\n category=spack.error.SpackAPIWarning,\n stacklevel=2,\n)\n\nreadlink = spack.llnl.util.filesystem.readlink\nislink = spack.llnl.util.filesystem.islink\nsymlink = spack.llnl.util.filesystem.symlink\n" }, { "path": "lib/spack/spack/llnl/util/tty/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport contextlib\nimport io\nimport os\nimport shutil\nimport sys\nimport textwrap\nimport traceback\nfrom datetime import datetime\nfrom types import TracebackType\nfrom typing import IO, Callable, Iterator, NoReturn, Optional, Type, Union\n\nfrom .color import cescape, clen, cprint, cwrite\n\n# Globals\n_debug = 0\n_verbose = False\n_stacktrace = False\n_timestamp = False\n_msg_enabled = True\n_warn_enabled = True\n_error_enabled = True\n_output_filter: Callable[[str], str] = lambda s: s\nindent = \" \"\n\n\ndef debug_level() -> int:\n return _debug\n\n\ndef is_verbose() -> bool:\n return _verbose\n\n\ndef is_debug(level: int = 1) -> bool:\n return _debug >= level\n\n\ndef set_debug(level: int = 0) -> None:\n global _debug\n assert level >= 0, \"Debug level must be a positive value\"\n _debug = level\n\n\ndef set_verbose(flag: bool) -> None:\n global _verbose\n _verbose = flag\n\n\ndef set_timestamp(flag: bool) -> None:\n global _timestamp\n _timestamp = flag\n\n\ndef set_msg_enabled(flag: bool) -> None:\n global _msg_enabled\n _msg_enabled = flag\n\n\ndef set_warn_enabled(flag: bool) -> None:\n global _warn_enabled\n _warn_enabled = flag\n\n\ndef set_error_enabled(flag: bool) -> None:\n global _error_enabled\n _error_enabled = flag\n\n\ndef msg_enabled() -> bool:\n return _msg_enabled\n\n\ndef warn_enabled() -> bool:\n return _warn_enabled\n\n\ndef error_enabled() -> bool:\n return _error_enabled\n\n\n@contextlib.contextmanager\ndef output_filter(filter_fn: Callable[[str], str]) -> Iterator[None]:\n \"\"\"Context manager that applies a filter to all output.\"\"\"\n global _output_filter\n saved_filter = _output_filter\n try:\n _output_filter = filter_fn\n yield\n finally:\n _output_filter = saved_filter\n\n\nclass SuppressOutput:\n \"\"\"Class for disabling output in a scope using ``with`` keyword\"\"\"\n\n def __init__(\n self, msg_enabled: bool = True, warn_enabled: bool = True, error_enabled: bool = True\n ) -> None:\n self._msg_enabled_initial = _msg_enabled\n self._warn_enabled_initial = _warn_enabled\n self._error_enabled_initial = _error_enabled\n\n self._msg_enabled = msg_enabled\n self._warn_enabled = warn_enabled\n self._error_enabled = error_enabled\n\n def __enter__(self) -> None:\n set_msg_enabled(self._msg_enabled)\n set_warn_enabled(self._warn_enabled)\n set_error_enabled(self._error_enabled)\n\n def __exit__(\n self,\n exc_type: Optional[Type[BaseException]],\n exc_val: Optional[BaseException],\n exc_tb: Optional[TracebackType],\n ) -> None:\n set_msg_enabled(self._msg_enabled_initial)\n set_warn_enabled(self._warn_enabled_initial)\n set_error_enabled(self._error_enabled_initial)\n\n\ndef set_stacktrace(flag: bool) -> None:\n global _stacktrace\n _stacktrace = flag\n\n\ndef process_stacktrace(countback: int) -> str:\n \"\"\"Gives file and line frame ``countback`` frames from the bottom\"\"\"\n st = traceback.extract_stack()\n # Not all entries may be spack files, we have to remove those that aren't.\n file_list = []\n for frame in st:\n # Check that the file is a spack file\n if frame[0].find(os.path.sep + \"spack\") >= 0:\n file_list.append(frame[0])\n # We use commonprefix to find what the spack 'root' directory is.\n root_dir = os.path.commonprefix(file_list)\n root_len = len(root_dir)\n st_idx = len(st) - countback - 1\n st_text = f\"{st[st_idx][0][root_len:]}:{st[st_idx][1]:d} \"\n return st_text\n\n\ndef show_pid() -> bool:\n return is_debug(2)\n\n\ndef get_timestamp(force: bool = False) -> str:\n \"\"\"Get a string timestamp\"\"\"\n if _debug or _timestamp or force:\n # Note the inclusion of the PID is useful for parallel builds.\n pid = f\", {os.getpid()}\" if show_pid() else \"\"\n return f\"[{datetime.now().strftime('%Y-%m-%d-%H:%M:%S.%f')}{pid}] \"\n else:\n return \"\"\n\n\ndef msg(message: Union[Exception, str], *args: str, newline: bool = True) -> None:\n \"\"\"Print a message to the console.\"\"\"\n if not msg_enabled():\n return\n\n if isinstance(message, Exception):\n message = f\"{message.__class__.__name__}: {message}\"\n else:\n message = str(message)\n\n st_text = \"\"\n if _stacktrace:\n st_text = process_stacktrace(2)\n\n nl = \"\\n\" if newline else \"\"\n cwrite(f\"@*b{{{st_text}==>}} {get_timestamp()}{cescape(_output_filter(message))}{nl}\")\n\n for arg in args:\n print(indent + _output_filter(str(arg)))\n\n\ndef info(\n message: Union[Exception, str],\n *args,\n format: str = \"*b\",\n stream: Optional[IO[str]] = None,\n wrap: bool = False,\n break_long_words: bool = False,\n countback: int = 3,\n) -> None:\n \"\"\"Print an informational message.\"\"\"\n if isinstance(message, Exception):\n message = f\"{message.__class__.__name__}: {str(message)}\"\n\n stream = stream or sys.stdout\n st_text = \"\"\n if _stacktrace:\n st_text = process_stacktrace(countback)\n cprint(\n \"@%s{%s==>} %s%s\"\n % (format, st_text, get_timestamp(), cescape(_output_filter(str(message)))),\n stream=stream,\n )\n for arg in args:\n if wrap:\n lines = textwrap.wrap(\n _output_filter(str(arg)),\n initial_indent=indent,\n subsequent_indent=indent,\n break_long_words=break_long_words,\n )\n for line in lines:\n stream.write(line + \"\\n\")\n else:\n stream.write(indent + _output_filter(str(arg)) + \"\\n\")\n stream.flush()\n\n\ndef verbose(message, *args, format: str = \"c\", **kwargs) -> None:\n \"\"\"Print a verbose message if the verbose flag is set.\"\"\"\n if _verbose:\n info(message, *args, format=format, **kwargs)\n\n\ndef debug(\n message, *args, level: int = 1, format: str = \"g\", stream: Optional[IO[str]] = None, **kwargs\n) -> None:\n \"\"\"Print a debug message if the debug level is set.\"\"\"\n if is_debug(level):\n stream_arg = stream or sys.stderr\n info(message, *args, format=format, stream=stream_arg, **kwargs)\n\n\ndef error(message, *args, format: str = \"*r\", stream: Optional[IO[str]] = None, **kwargs) -> None:\n \"\"\"Print an error message.\"\"\"\n if not error_enabled():\n return\n\n stream = stream or sys.stderr\n info(f\"Error: {message}\", *args, format=format, stream=stream, **kwargs)\n\n\ndef warn(message, *args, format: str = \"*Y\", stream: Optional[IO[str]] = None, **kwargs) -> None:\n \"\"\"Print a warning message.\"\"\"\n if not warn_enabled():\n return\n\n stream = stream or sys.stderr\n info(f\"Warning: {message}\", *args, format=format, stream=stream, **kwargs)\n\n\ndef die(message, *args, countback: int = 4, **kwargs) -> NoReturn:\n error(message, *args, countback=countback, **kwargs)\n sys.exit(1)\n\n\ndef get_yes_or_no(prompt: str, default: Optional[bool] = None) -> Optional[bool]:\n if default is None:\n prompt += \" [y/n] \"\n elif default is True:\n prompt += \" [Y/n] \"\n elif default is False:\n prompt += \" [y/N] \"\n else:\n raise ValueError(\"default for get_yes_no() must be True, False, or None.\")\n\n result = None\n while result is None:\n msg(prompt, newline=False)\n ans = input().lower()\n if not ans:\n result = default\n if result is None:\n print(\"Please enter yes or no.\")\n else:\n if ans == \"y\" or ans == \"yes\":\n result = True\n elif ans == \"n\" or ans == \"no\":\n result = False\n return result\n\n\ndef hline(label: Optional[str] = None, *, char: str = \"-\", max_width: int = 64) -> None:\n \"\"\"Draw a labeled horizontal line.\n\n Args:\n char: char to draw the line with\n max_width: maximum width of the line\n \"\"\"\n cols = shutil.get_terminal_size().columns\n if not cols:\n cols = max_width\n else:\n cols -= 2\n cols = min(max_width, cols)\n\n label = str(label)\n prefix = char * 2 + \" \"\n suffix = \" \" + (cols - len(prefix) - clen(label)) * char\n\n out = io.StringIO()\n out.write(prefix)\n out.write(label)\n out.write(suffix)\n\n print(out.getvalue())\n" }, { "path": "lib/spack/spack/llnl/util/tty/colify.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"\nRoutines for printing columnar output. See ``colify()`` for more information.\n\"\"\"\n\nimport io\nimport os\nimport shutil\nimport sys\nfrom typing import IO, Any, List, Optional\n\nfrom spack.llnl.util.tty.color import cextra, clen\n\n\nclass ColumnConfig:\n def __init__(self, cols: int) -> None:\n self.cols = cols\n self.line_length = 0\n self.valid = True\n self.widths = [0] * cols # does not include ansi colors\n\n def __repr__(self) -> str:\n attrs = [(a, getattr(self, a)) for a in dir(self) if not a.startswith(\"__\")]\n return f\"\"\n\n\ndef config_variable_cols(\n elts: List[str], console_width: int, padding: int, cols: int = 0\n) -> ColumnConfig:\n \"\"\"Variable-width column fitting algorithm.\n\n This function determines the most columns that can fit in the\n screen width. Unlike uniform fitting, where all columns take\n the width of the longest element in the list, each column takes\n the width of its own longest element. This packs elements more\n efficiently on screen.\n\n If cols is nonzero, force the table to use that many columns and\n just add minimal padding between the columns.\n \"\"\"\n if cols < 0:\n raise ValueError(\"cols must be non-negative.\")\n\n # Get a bound on the most columns we could possibly have.\n # 'clen' ignores length of ansi color sequences.\n lengths = [clen(e) for e in elts]\n max_cols = max(1, console_width // (min(lengths) + padding))\n max_cols = min(len(elts), max_cols)\n\n # Range of column counts to try. If forced, use the supplied value.\n col_range = [cols] if cols else range(1, max_cols + 1)\n\n # Determine the most columns possible for the console width.\n configs = [ColumnConfig(c) for c in col_range]\n for i, length in enumerate(lengths):\n for conf in configs:\n if conf.valid:\n rows = (len(elts) + conf.cols - 1) // conf.cols\n col = i // rows\n p = padding if col < (conf.cols - 1) else 0\n\n if conf.widths[col] < (length + p):\n conf.line_length += length + p - conf.widths[col]\n conf.widths[col] = length + p\n conf.valid = conf.line_length < console_width\n\n try:\n # take the last valid config in the list (the one with most columns)\n config = next(conf for conf in reversed(configs) if conf.valid)\n except StopIteration:\n # If nothing was valid, the screen was too narrow -- use 1 col if cols was not\n # specified, otherwise, use the requested columns and overflow.\n config = configs[0]\n if cols:\n rows = (len(lengths) + cols - 1) // cols\n config.widths = [\n max(length for i, length in enumerate(lengths) if i // rows == c)\n + (padding if c < cols - 1 else 0)\n for c in range(cols)\n ]\n\n # trim off any columns with nothing in them\n config.widths = [w for w in config.widths if w != 0]\n config.cols = len(config.widths)\n return config\n\n\ndef config_uniform_cols(\n elts: List[str], console_width: int, padding: int, cols: int = 0\n) -> ColumnConfig:\n \"\"\"Uniform-width column fitting algorithm.\n\n Determines the longest element in the list, and determines how\n many columns of that width will fit on screen. Returns a\n corresponding column config.\n \"\"\"\n if cols < 0:\n raise ValueError(\"cols must be non-negative.\")\n\n # 'clen' ignores length of ansi color sequences.\n max_len = max(clen(e) for e in elts) + padding\n if cols == 0:\n cols = max(1, console_width // max_len)\n cols = min(len(elts), cols)\n\n config = ColumnConfig(cols)\n config.widths = [max_len] * cols\n\n return config\n\n\ndef colify(\n elts: List[Any],\n *,\n cols: int = 0,\n output: Optional[IO] = None,\n indent: int = 0,\n padding: int = 2,\n tty: Optional[bool] = None,\n method: str = \"variable\",\n console_cols: Optional[int] = None,\n):\n \"\"\"Takes a list of elements as input and finds a good columnization\n of them, similar to how gnu ls does. This supports both\n uniform-width and variable-width (tighter) columns.\n\n If elts is not a list of strings, each element is first converted\n using :class:`str`.\n\n Keyword Arguments:\n output: A file object to write to. Default is ``sys.stdout``\n indent: Optionally indent all columns by some number of spaces\n padding: Spaces between columns. Default is 2\n width: Width of the output. Default is 80 if tty not detected\n cols: Force number of columns. Default is to size to terminal, or\n single-column if no tty\n tty: Whether to attempt to write to a tty. Default is to autodetect a\n tty. Set to False to force single-column output\n method: Method to use to fit columns. Options are variable or uniform.\n Variable-width columns are tighter, uniform columns are all the same width\n and fit less data on the screen\n console_cols: number of columns on this console (default: autodetect)\n \"\"\"\n if output is None:\n output = sys.stdout\n\n # elts needs to be an array of strings so we can count the elements\n elts = [str(elt) for elt in elts]\n if not elts:\n return (0, ())\n\n # environment size is of the form \"x\"\n env_size = os.environ.get(\"COLIFY_SIZE\")\n if env_size:\n try:\n console_cols = int(env_size.partition(\"x\")[2])\n tty = True\n except ValueError:\n pass\n\n # Use only one column if not a tty, unless cols specified explicitly\n if not cols and not tty:\n if tty is False or not output.isatty():\n cols = 1\n\n # Specify the number of character columns to use.\n if console_cols is None:\n console_cols = shutil.get_terminal_size().columns\n elif not isinstance(console_cols, int):\n raise ValueError(\"Number of columns must be an int\")\n\n console_cols = max(1, console_cols - indent)\n\n # Choose a method. Variable-width columns vs uniform-width.\n if method == \"variable\":\n config = config_variable_cols(elts, console_cols, padding, cols)\n elif method == \"uniform\":\n config = config_uniform_cols(elts, console_cols, padding, cols)\n else:\n raise ValueError(\"method must be either 'variable' or 'uniform'\")\n\n cols = config.cols\n rows = (len(elts) + cols - 1) // cols\n rows_last_col = len(elts) % rows\n\n for row in range(rows):\n output.write(\" \" * indent)\n for col in range(cols):\n elt = col * rows + row\n width = config.widths[col] + cextra(elts[elt])\n if col < cols - 1:\n fmt = \"%%-%ds\" % width\n output.write(fmt % elts[elt])\n else:\n # Don't pad the rightmost column (spaces can wrap on\n # small teriminals if one line is overlong)\n output.write(elts[elt])\n\n output.write(\"\\n\")\n row += 1\n if row == rows_last_col:\n cols -= 1\n\n return (config.cols, tuple(config.widths))\n\n\ndef colify_table(\n table: List[List[Any]],\n *,\n output: Optional[IO] = None,\n indent: int = 0,\n padding: int = 2,\n console_cols: Optional[int] = None,\n):\n \"\"\"Version of ``colify()`` for data expressed in rows, (list of lists).\n\n Same as regular colify but:\n\n 1. This takes a list of lists, where each sub-list must be the\n same length, and each is interpreted as a row in a table.\n Regular colify displays a sequential list of values in columns.\n\n 2. Regular colify will always print with 1 column when the output\n is not a tty. This will always print with same dimensions of\n the table argument.\n\n \"\"\"\n if table is None:\n raise TypeError(\"Can't call colify_table on NoneType\")\n elif not table or not table[0]:\n raise ValueError(\"Table is empty in colify_table!\")\n\n columns = len(table[0])\n\n def transpose():\n for i in range(columns):\n for row in table:\n yield row[i]\n\n colify(\n transpose(),\n cols=columns, # this is always the number of cols in the table\n tty=True, # don't reduce to 1 column for non-tty\n output=output,\n indent=indent,\n padding=padding,\n console_cols=console_cols,\n )\n\n\ndef colified(\n elts: List[Any],\n *,\n cols: int = 0,\n indent: int = 0,\n padding: int = 2,\n tty: Optional[bool] = None,\n method: str = \"variable\",\n console_cols: Optional[int] = None,\n):\n \"\"\"Invokes the ``colify()`` function but returns the result as a string\n instead of writing it to an output string.\"\"\"\n sio = io.StringIO()\n colify(\n elts,\n cols=cols,\n output=sio,\n indent=indent,\n padding=padding,\n tty=tty,\n method=method,\n console_cols=console_cols,\n )\n return sio.getvalue()\n" }, { "path": "lib/spack/spack/llnl/util/tty/color.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"\nThis file implements an expression syntax, similar to ``printf``, for adding\nANSI colors to text.\n\nSee :func:`colorize`, :func:`cwrite`, and :func:`cprint` for routines that can\ngenerate colored output.\n\n:func:`colorize` will take a string and replace all color expressions with\nANSI control codes. If the ``isatty`` keyword arg is set to False, then\nthe color expressions will be converted to null strings, and the\nreturned string will have no color.\n\n:func:`cwrite` and :func:`cprint` are equivalent to ``write()`` and ``print()``\ncalls in python, but they colorize their output. If the ``stream`` argument is\nnot supplied, they write to ``sys.stdout``.\n\nHere are some example color expressions:\n\n============== ============================================================\nExpression Meaning\n============== ============================================================\n``@r`` Turn on red coloring\n``@R`` Turn on bright red coloring\n``@*{foo}`` Bold foo, but don't change text color\n``@_{bar}`` Underline bar, but don't change text color\n``@*b`` Turn on bold, blue text\n``@_B`` Turn on bright blue text with an underline\n``@.`` Revert to plain formatting\n``@*g{green}`` Print out 'green' in bold, green text, then reset to plain.\n``@*ggreen@.`` Print out 'green' in bold, green text, then reset to plain.\n============== ============================================================\n\nThe syntax consists of:\n\n========== =====================================================\ncolor-expr ``'@' [style] color-code '{' text '}' | '@.' | '@@'``\nstyle ``'*' | '_'``\ncolor-code ``[krgybmcwKRGYBMCW]``\ntext ``.*``\n========== =====================================================\n\n``@`` indicates the start of a color expression. It can be followed\nby an optional ``*`` or ``_`` that indicates whether the font should be bold or\nunderlined. If ``*`` or ``_`` is not provided, the text will be plain. Then\nan optional color code is supplied. This can be ``[krgybmcw]`` or ``[KRGYBMCW]``,\nwhere the letters map to ``black(k)``, ``red(r)``, ``green(g)``, ``yellow(y)``, ``blue(b)``,\n``magenta(m)``, ``cyan(c)``, and ``white(w)``. Lowercase letters denote normal ANSI\ncolors and capital letters denote bright ANSI colors.\n\nFinally, the color expression can be followed by text enclosed in ``{}``. If\nbraces are present, only the text in braces is colored. If the braces are\nNOT present, then just the control codes to enable the color will be output.\nThe console can be reset later to plain text with ``@.``.\n\nTo output an ``@``, use ``@@``. To output a ``}`` inside braces, use ``}}``.\n\"\"\"\n\nimport io\nimport os\nimport re\nimport sys\nimport textwrap\nfrom contextlib import contextmanager\nfrom typing import IO, Iterator, List, NamedTuple, Optional, Tuple, Union\n\n\nclass ColorParseError(Exception):\n \"\"\"Raised when a color format fails to parse.\"\"\"\n\n def __init__(self, message: str) -> None:\n super().__init__(message)\n\n\n# Text styles for ansi codes\nstyles = {\"*\": \"1\", \"_\": \"4\", None: \"0\"} # bold # underline # plain\n\n# Dim and bright ansi colors\ncolors = {\n \"k\": 30,\n \"K\": 90, # black\n \"r\": 31,\n \"R\": 91, # red\n \"g\": 32,\n \"G\": 92, # green\n \"y\": 33,\n \"Y\": 93, # yellow\n \"b\": 34,\n \"B\": 94, # blue\n \"m\": 35,\n \"M\": 95, # magenta\n \"c\": 36,\n \"C\": 96, # cyan\n \"w\": 37,\n \"W\": 97,\n} # white\n\n# Regex to be used for color formatting\nCOLOR_RE = re.compile(r\"@(?:(@)|(\\.)|([*_])?([a-zA-Z])?(?:{((?:[^}]|}})*)})?)\")\n\n# Mapping from color arguments to values for tty.set_color\ncolor_when_values = {\"always\": True, \"auto\": None, \"never\": False}\n\n\ndef _color_when_value(when: Union[str, bool, None]) -> Optional[bool]:\n \"\"\"Raise a ValueError for an invalid color setting.\n\n Valid values are 'always', 'never', and 'auto', or equivalently,\n True, False, and None.\n \"\"\"\n if isinstance(when, bool) or when is None:\n return when\n\n elif when not in color_when_values:\n raise ValueError(f\"Invalid color setting: {when}\")\n\n return color_when_values[when]\n\n\ndef _color_from_environ() -> Optional[bool]:\n try:\n return _color_when_value(os.environ.get(\"SPACK_COLOR\", \"auto\"))\n except ValueError:\n return None\n\n\n#: When `None` colorize when stdout is tty, when `True` or `False` always or never colorize resp.\n_force_color = _color_from_environ()\n\n\ndef try_enable_terminal_color_on_windows() -> None:\n \"\"\"Turns coloring in Windows terminal by enabling VTP in Windows consoles (CMD/PWSH/CONHOST)\n Method based on the link below\n https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences#example-of-enabling-virtual-terminal-processing\n\n Note: No-op on non windows platforms\n \"\"\"\n if sys.platform == \"win32\":\n import ctypes\n import msvcrt\n from ctypes import wintypes\n\n try:\n ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004\n DISABLE_NEWLINE_AUTO_RETURN = 0x0008\n kernel32 = ctypes.WinDLL(\"kernel32\")\n\n def _err_check(result, func, args):\n if not result:\n raise ctypes.WinError(ctypes.get_last_error())\n return args\n\n kernel32.GetConsoleMode.errcheck = _err_check\n kernel32.GetConsoleMode.argtypes = (\n wintypes.HANDLE, # hConsoleHandle, i.e. GetStdHandle output type\n ctypes.POINTER(wintypes.DWORD), # result of GetConsoleHandle\n )\n kernel32.SetConsoleMode.errcheck = _err_check\n kernel32.SetConsoleMode.argtypes = (\n wintypes.HANDLE, # hConsoleHandle, i.e. GetStdHandle output type\n wintypes.DWORD, # result of GetConsoleHandle\n )\n # Use conout$ here to handle a redirectired stdout/get active console associated\n # with spack\n with open(r\"\\\\.\\CONOUT$\", \"w\", encoding=\"utf-8\") as conout:\n # Link above would use kernel32.GetStdHandle(-11) however this would not handle\n # a redirected stdout appropriately, so we always refer to the current CONSOLE out\n # which is defined as conout$ on Windows.\n # linked example is follow more or less to the letter beyond this point\n con_handle = msvcrt.get_osfhandle(conout.fileno())\n dw_orig_mode = wintypes.DWORD()\n kernel32.GetConsoleMode(con_handle, ctypes.byref(dw_orig_mode))\n dw_new_mode_request = (\n ENABLE_VIRTUAL_TERMINAL_PROCESSING | DISABLE_NEWLINE_AUTO_RETURN\n )\n dw_new_mode = dw_new_mode_request | dw_orig_mode.value\n kernel32.SetConsoleMode(con_handle, wintypes.DWORD(dw_new_mode))\n except OSError:\n # We failed to enable color support for associated console\n # report and move on but spack will no longer attempt to\n # color\n global _force_color\n _force_color = False\n\n\ndef get_color_when(stdout=None) -> bool:\n \"\"\"Return whether commands should print color or not.\"\"\"\n if _force_color is not None:\n return _force_color\n if stdout is None:\n stdout = sys.stdout\n return stdout.isatty()\n\n\ndef set_color_when(when: Union[str, bool, None]) -> None:\n \"\"\"Set when color should be applied. Options are:\n\n * True or ``\"always\"``: always print color\n * False or ``\"never\"``: never print color\n * None or ``\"auto\"``: only print color if sys.stdout is a tty.\n \"\"\"\n global _force_color\n _force_color = _color_when_value(when)\n\n\n@contextmanager\ndef color_when(value: Union[str, bool, None]) -> Iterator[None]:\n \"\"\"Context manager to temporarily use a particular color setting.\"\"\"\n old_value = _force_color\n set_color_when(value)\n yield\n set_color_when(old_value)\n\n\n_ConvertibleToStr = Union[str, int, bool, None]\n\n\ndef _escape(s: _ConvertibleToStr, color: bool, enclose: bool, zsh: bool) -> str:\n \"\"\"Returns a TTY escape sequence for a color\"\"\"\n if not color:\n return \"\"\n elif zsh:\n return f\"\\033[0;{s}m\"\n\n result = f\"\\033[{s}m\"\n\n if enclose:\n result = rf\"\\[{result}\\]\"\n\n return result\n\n\ndef colorize(\n string: str, color: Optional[bool] = None, enclose: bool = False, zsh: bool = False\n) -> str:\n \"\"\"Replace all color expressions in a string with ANSI control codes.\n\n Args:\n string: The string to replace\n color: If False, output will be plain text without control codes, for output to\n non-console devices (default: automatically choose color or not)\n enclose: If True, enclose ansi color sequences with square brackets to prevent\n misestimation of terminal width.\n zsh: If True, use zsh ansi codes instead of bash ones (for variables like PS1)\n \"\"\"\n if color is None:\n color = get_color_when()\n\n def match_to_ansi(match) -> str:\n \"\"\"Convert a match object generated by ``COLOR_RE`` into an ansi\n color code. This can be used as a handler in ``re.sub``.\n \"\"\"\n escaped_at, dot, style, color_code, text = match.groups()\n\n if escaped_at:\n return \"@\"\n elif dot:\n return _escape(0, color, enclose, zsh)\n elif not (style or color_code):\n raise ColorParseError(\n f\"Incomplete color format: '{match.group(0)}' in '{match.string}'\"\n )\n\n color_number = colors.get(color_code, \"\")\n semi = \";\" if color_number else \"\"\n ansi_code = _escape(f\"{styles[style]}{semi}{color_number}\", color, enclose, zsh)\n if text:\n # must be here, not in the final return: top-level @@ is already handled by\n # the regex, and its @-results could form new @@ pairs.\n text = text.replace(\"@@\", \"@\")\n return f\"{ansi_code}{text}{_escape(0, color, enclose, zsh)}\"\n else:\n return ansi_code\n\n return COLOR_RE.sub(match_to_ansi, string).replace(\"}}\", \"}\")\n\n\n#: matches a standard ANSI color code\nANSI_CODE_RE = re.compile(r\"\\033[^m]*m\")\n\n\ndef csub(string: str) -> str:\n \"\"\"Return the string with ANSI color sequences removed.\"\"\"\n return ANSI_CODE_RE.sub(\"\", string)\n\n\nclass ColorMapping(NamedTuple):\n color: str #: color string\n colors: List[str] #: ANSI color codes in the color string, in order\n offsets: List[Tuple[int, int]] #: map indices in plain string to offsets in color string\n\n def plain_to_color(self, index: int) -> int:\n \"\"\"Convert plain string index to color index.\"\"\"\n offset = 0\n for i, off in self.offsets:\n if i > index:\n break\n offset = off\n return index + offset\n\n\ndef cmapping(string: str) -> ColorMapping:\n \"\"\"Return a mapping for translating indices in a plain string to indices in colored text.\n\n The returned dictionary maps indices in the plain string to the offset of the corresponding\n indices in the colored string.\n\n \"\"\"\n colors = []\n offsets = []\n color_offset = 0\n\n for m in ANSI_CODE_RE.finditer(string):\n start, end = m.start(), m.end()\n start_offset = color_offset\n color_offset += end - start\n offsets.append((start - start_offset, color_offset))\n colors.append(m.group())\n\n return ColorMapping(string, colors, offsets)\n\n\ndef cwrap(\n string: str, *, initial_indent: str = \"\", subsequent_indent: str = \"\", **kwargs\n) -> List[str]:\n \"\"\"Wrapper around ``textwrap.wrap()`` that handles ANSI color codes.\"\"\"\n plain = csub(string)\n lines = textwrap.wrap(\n plain, initial_indent=initial_indent, subsequent_indent=subsequent_indent, **kwargs\n )\n\n # do nothing if string has no ANSI codes\n if plain == string:\n return lines\n\n # otherwise add colors back to lines after wrapping plain text\n cmap = cmapping(string)\n\n clines = []\n start = 0\n for i, line in enumerate(lines):\n # scan to find the actual start, skipping any whitespace from a prior line break\n # can assume this b/c textwrap only collapses whitespace at line breaks\n while start < len(plain) and plain[start].isspace():\n start += 1\n\n # map the start and end positions in the plain string to the color string\n cstart = cmap.plain_to_color(start)\n\n # rewind to include any color codes before cstart\n while cstart and string[cstart - 1] == \"m\":\n cstart = string.rfind(\"\\033\", 0, cstart - 1)\n\n indent = initial_indent if i == 0 else subsequent_indent\n end = start + len(line) - len(indent)\n cend = cmap.plain_to_color(end)\n\n # append the color line to the result\n clines.append(indent + string[cstart:cend])\n start = end\n\n return clines\n\n\ndef clen(string: str) -> int:\n \"\"\"Return the length of a string, excluding ansi color sequences.\"\"\"\n return len(csub(string))\n\n\ndef cextra(string: str) -> int:\n \"\"\"Length of extra color characters in a string\"\"\"\n return len(\"\".join(re.findall(r\"\\033[^m]*m\", string)))\n\n\ndef cwrite(string: str, stream: Optional[IO[str]] = None, color: Optional[bool] = None) -> None:\n \"\"\"Replace all color expressions in string with ANSI control\n codes and write the result to the stream. If color is\n False, this will write plain text with no color. If True,\n then it will always write colored output. If not supplied,\n then it will be set based on stream.isatty().\n \"\"\"\n stream = sys.stdout if stream is None else stream\n if color is None:\n color = get_color_when()\n stream.write(colorize(string, color=color))\n\n\ndef cprint(string: str, stream: Optional[IO[str]] = None, color: Optional[bool] = None) -> None:\n \"\"\"Same as cwrite, but writes a trailing newline to the stream.\"\"\"\n cwrite(string + \"\\n\", stream, color)\n\n\ndef cescape(string: str) -> str:\n \"\"\"Escapes special characters needed for color codes.\n\n Replaces the following symbols with their equivalent literal forms:\n\n ===== ======\n ``@`` ``@@``\n ``}`` ``}}``\n ===== ======\n\n Parameters:\n string (str): the string to escape\n\n Returns:\n (str): the string with color codes escaped\n \"\"\"\n return string.replace(\"@\", \"@@\").replace(\"}\", \"}}\")\n\n\nclass ColorStream:\n def __init__(self, stream: io.IOBase, color: Optional[bool] = None) -> None:\n self._stream = stream\n self._color = color\n\n def write(self, string: str, *, raw: bool = False) -> None:\n raw_write = getattr(self._stream, \"write\")\n\n color = self._color\n if self._color is None:\n if raw:\n color = True\n else:\n color = get_color_when()\n raw_write(colorize(string, color=color))\n" }, { "path": "lib/spack/spack/llnl/util/tty/log.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Utility classes for logging the output of blocks of code.\"\"\"\n\nimport atexit\nimport ctypes\nimport errno\nimport io\nimport multiprocessing\nimport os\nimport re\nimport select\nimport signal\nimport sys\nimport threading\nimport traceback\nfrom contextlib import contextmanager\nfrom multiprocessing.connection import Connection\nfrom threading import Thread\nfrom typing import IO, Callable, Optional, Tuple\n\nimport spack.llnl.util.tty as tty\n\ntry:\n import termios\nexcept ImportError:\n termios = None # type: ignore[assignment]\n\n\nesc, bell, lbracket, bslash, newline = r\"\\x1b\", r\"\\x07\", r\"\\[\", r\"\\\\\", r\"\\n\"\n# Ansi Control Sequence Introducers (CSI) are a well-defined format\n# Standard ECMA-48: Control Functions for Character-Imaging I/O Devices, section 5.4\n# https://www.ecma-international.org/wp-content/uploads/ECMA-48_5th_edition_june_1991.pdf\ncsi_pre = f\"{esc}{lbracket}\"\ncsi_param, csi_inter, csi_post = r\"[0-?]\", r\"[ -/]\", r\"[@-~]\"\nansi_csi = f\"{csi_pre}{csi_param}*{csi_inter}*{csi_post}\"\n# General ansi escape sequences have well-defined prefixes,\n# but content and suffixes are less reliable.\n# Conservatively assume they end with either \"\\\" or \"\",\n# with no intervening \"\"/\"\" keys or newlines\nesc_pre = f\"{esc}[@-_]\"\nesc_content = f\"[^{esc}{bell}{newline}]\"\nesc_post = f\"(?:{esc}{bslash}|{bell})\"\nansi_esc = f\"{esc_pre}{esc_content}*{esc_post}\"\n# Use this to strip escape sequences\n_escape = re.compile(f\"{ansi_csi}|{ansi_esc}\")\n\n# control characters for enabling/disabling echo\n#\n# We use control characters to ensure that echo enable/disable are inline\n# with the other output. We always follow these with a newline to ensure\n# one per line the following newline is ignored in output.\nxon, xoff = \"\\x11\\n\", \"\\x13\\n\"\ncontrol = re.compile(\"(\\x11\\n|\\x13\\n)\")\n\n\n@contextmanager\ndef ignore_signal(signum):\n \"\"\"Context manager to temporarily ignore a signal.\"\"\"\n old_handler = signal.signal(signum, signal.SIG_IGN)\n try:\n yield\n finally:\n signal.signal(signum, old_handler)\n\n\ndef _is_background_tty(stdin: IO[str]) -> bool:\n \"\"\"True if the stream is a tty and calling process is in the background.\"\"\"\n return stdin.isatty() and os.getpgrp() != os.tcgetpgrp(stdin.fileno())\n\n\ndef _strip(line: str) -> str:\n \"\"\"Strip color and control characters from a line.\"\"\"\n return _escape.sub(\"\", line)\n\n\nclass preserve_terminal_settings:\n \"\"\"Context manager to preserve terminal settings on a stream.\n\n Stores terminal settings before the context and ensures they are restored after.\n Ensures that things like echo and canonical line mode are not left disabled if\n terminal settings in the context are not properly restored.\n \"\"\"\n\n def __init__(self, stdin: Optional[IO[str]]) -> None:\n \"\"\"Create a context manager that preserves terminal settings on a stream.\n\n Args:\n stream: keyboard input stream, typically sys.stdin\n \"\"\"\n self.stdin = stdin\n\n def _restore_default_terminal_settings(self) -> None:\n \"\"\"Restore the original input configuration on ``self.stdin``.\"\"\"\n # Can be called in foreground or background. When called in the background, tcsetattr\n # triggers SIGTTOU, which we must ignore, or the process will be stopped.\n assert self.stdin is not None and self.old_cfg is not None and termios is not None\n with ignore_signal(signal.SIGTTOU):\n termios.tcsetattr(self.stdin, termios.TCSANOW, self.old_cfg)\n\n def __enter__(self) -> \"preserve_terminal_settings\":\n \"\"\"Store terminal settings.\"\"\"\n self.old_cfg = None\n\n # Ignore all this if the input stream is not a tty.\n if not self.stdin or not self.stdin.isatty() or not termios:\n return self\n\n # save old termios settings to restore later\n self.old_cfg = termios.tcgetattr(self.stdin)\n\n # add an atexit handler to ensure the terminal is restored\n atexit.register(self._restore_default_terminal_settings)\n\n return self\n\n def __exit__(self, exc_type, exception, traceback):\n \"\"\"If termios was available, restore old settings.\"\"\"\n if self.old_cfg:\n self._restore_default_terminal_settings()\n atexit.unregister(self._restore_default_terminal_settings)\n\n\nclass keyboard_input(preserve_terminal_settings):\n \"\"\"Context manager to disable line editing and echoing.\n\n Use this with ``sys.stdin`` for keyboard input, e.g.::\n\n with keyboard_input(sys.stdin) as kb:\n while True:\n kb.check_fg_bg()\n r, w, x = select.select([sys.stdin], [], [])\n # ... do something with keypresses ...\n\n The ``keyboard_input`` context manager disables canonical\n (line-based) input and echoing, so that keypresses are available on\n the stream immediately, and they are not printed to the\n terminal. Typically, standard input is line-buffered, which means\n keypresses won't be sent until the user hits return. In this mode, a\n user can hit, e.g., ``v``, and it will be read on the other end of the\n pipe immediately but not printed.\n\n The handler takes care to ensure that terminal changes only take\n effect when the calling process is in the foreground. If the process\n is backgrounded, canonical mode and echo are re-enabled. They are\n disabled again when the calling process comes back to the foreground.\n\n This context manager works through a single signal handler for\n ``SIGTSTP``, along with a poolling routine called ``check_fg_bg()``.\n Here are the relevant states, transitions, and POSIX signals::\n\n [Running] -------- Ctrl-Z sends SIGTSTP ------------.\n [ in FG ] <------- fg sends SIGCONT --------------. |\n ^ | |\n | fg (no signal) | |\n | | v\n [Running] <------- bg sends SIGCONT ---------- [Stopped]\n [ in BG ] [ in BG ]\n\n We handle all transitions except for ``SIGTSTP`` generated by Ctrl-Z\n by periodically calling ``check_fg_bg()``. This routine notices if\n we are in the background with canonical mode or echo disabled, or if\n we are in the foreground without canonical disabled and echo enabled,\n and it fixes the terminal settings in response.\n\n ``check_fg_bg()`` works *except* for when the process is stopped with\n ``SIGTSTP``. We cannot rely on a periodic timer in this case, as it\n may not rrun before the process stops. We therefore restore terminal\n settings in the ``SIGTSTP`` handler.\n\n Additional notes:\n\n * We mostly use polling here instead of a SIGARLM timer or a\n thread. This is to avoid the complexities of many interrupts, which\n seem to make system calls (like I/O) unreliable in older Python\n versions (2.6 and 2.7). See these issues for details:\n\n 1. https://www.python.org/dev/peps/pep-0475/\n 2. https://bugs.python.org/issue8354\n\n There are essentially too many ways for asynchronous signals to go\n wrong if we also have to support older Python versions, so we opt\n not to use them.\n\n * ``SIGSTOP`` can stop a process (in the foreground or background),\n but it can't be caught. Because of this, we can't fix any terminal\n settings on ``SIGSTOP``, and the terminal will be left with\n ``ICANON`` and ``ECHO`` disabled until it is resumes execution.\n\n * Technically, a process *could* be sent ``SIGTSTP`` while running in\n the foreground, without the shell backgrounding that process. This\n doesn't happen in practice, and we assume that ``SIGTSTP`` always\n means that defaults should be restored.\n\n * We rely on ``termios`` support. Without it, or if the stream isn't\n a TTY, ``keyboard_input`` has no effect.\n\n \"\"\"\n\n def __init__(self, stdin: Optional[IO[str]]) -> None:\n \"\"\"Create a context manager that will enable keyboard input on stream.\n\n Args:\n stdin: text io wrapper of stdin (keyboard input)\n\n Note that stdin can be None, in which case ``keyboard_input`` will do nothing.\n \"\"\"\n super().__init__(stdin)\n\n def _is_background(self) -> bool:\n \"\"\"True iff calling process is in the background.\"\"\"\n assert self.stdin is not None, \"stdin should be available\"\n return _is_background_tty(self.stdin)\n\n def _get_canon_echo_flags(self) -> Tuple[bool, bool]:\n \"\"\"Get current termios canonical and echo settings.\"\"\"\n assert termios is not None and self.stdin is not None\n cfg = termios.tcgetattr(self.stdin)\n return (bool(cfg[3] & termios.ICANON), bool(cfg[3] & termios.ECHO))\n\n def _enable_keyboard_input(self) -> None:\n \"\"\"Disable canonical input and echoing on ``self.stdin``.\"\"\"\n # \"enable\" input by disabling canonical mode and echo\n assert termios is not None and self.stdin is not None\n new_cfg = termios.tcgetattr(self.stdin)\n new_cfg[3] &= ~termios.ICANON\n new_cfg[3] &= ~termios.ECHO\n\n # Apply new settings for terminal\n with ignore_signal(signal.SIGTTOU):\n termios.tcsetattr(self.stdin, termios.TCSANOW, new_cfg)\n\n def _tstp_handler(self, signum, frame):\n self._restore_default_terminal_settings()\n os.kill(os.getpid(), signal.SIGSTOP)\n\n def check_fg_bg(self) -> None:\n # old_cfg is set up in __enter__ and indicates that we have\n # termios and a valid stream.\n if not self.old_cfg:\n return\n\n # query terminal flags and fg/bg status\n flags = self._get_canon_echo_flags()\n bg = self._is_background()\n\n # restore sanity if flags are amiss -- see diagram in class docs\n if not bg and any(flags): # fg, but input not enabled\n self._enable_keyboard_input()\n elif bg and not all(flags): # bg, but input enabled\n self._restore_default_terminal_settings()\n\n def __enter__(self) -> \"keyboard_input\":\n \"\"\"Enable immediate keypress input, while this process is foreground.\n\n If the stream is not a TTY or the system doesn't support termios,\n do nothing.\n \"\"\"\n super().__enter__()\n self.old_handlers = {}\n\n # Ignore all this if the input stream is not a tty.\n if not self.stdin or not self.stdin.isatty():\n return self\n\n if termios:\n # Install a signal handler to disable/enable keyboard input\n # when the process moves between foreground and background.\n self.old_handlers[signal.SIGTSTP] = signal.signal(signal.SIGTSTP, self._tstp_handler)\n\n # enable keyboard input initially (if foreground)\n if not self._is_background():\n self._enable_keyboard_input()\n\n return self\n\n def __exit__(self, exc_type, exception, traceback):\n \"\"\"If termios was available, restore old settings.\"\"\"\n super().__exit__(exc_type, exception, traceback)\n\n # restore SIGSTP and SIGCONT handlers\n if self.old_handlers:\n for signum, old_handler in self.old_handlers.items():\n signal.signal(signum, old_handler)\n\n\nclass Unbuffered:\n \"\"\"Wrapper for Python streams that forces them to be unbuffered.\n\n This is implemented by forcing a flush after each write.\n \"\"\"\n\n def __init__(self, stream):\n self.stream = stream\n\n def write(self, data):\n self.stream.write(data)\n self.stream.flush()\n\n def writelines(self, datas):\n self.stream.writelines(datas)\n self.stream.flush()\n\n def __getattr__(self, attr):\n return getattr(self.stream, attr)\n\n\ndef log_output(*args, **kwargs):\n \"\"\"Context manager that logs its output to a file.\n\n In the simplest case, the usage looks like this::\n\n with log_output('logfile.txt'):\n # do things ... output will be logged\n\n Any output from the with block will be redirected to ``logfile.txt``.\n If you also want the output to be echoed to ``stdout``, use the\n ``echo`` parameter::\n\n with log_output('logfile.txt', echo=True):\n # do things ... output will be logged and printed out\n\n The following is available on Unix only. No-op on Windows.\n And, if you just want to echo *some* stuff from the parent, use\n ``force_echo``::\n\n with log_output('logfile.txt', echo=False) as logger:\n # do things ... output will be logged\n\n with logger.force_echo():\n # things here will be echoed *and* logged\n\n See individual log classes for more information.\n\n\n This method is actually a factory serving a per platform\n (unix vs windows) log_output class\n \"\"\"\n if sys.platform == \"win32\":\n return winlog(*args, **kwargs)\n else:\n return nixlog(*args, **kwargs)\n\n\nclass nixlog:\n \"\"\"\n Under the hood, we spawn a daemon and set up a pipe between this\n process and the daemon. The daemon writes our output to both the\n file and to stdout (if echoing). The parent process can communicate\n with the daemon to tell it when and when not to echo; this is what\n force_echo does. You can also enable/disable echoing by typing ``v``.\n\n We use OS-level file descriptors to do the redirection, which\n redirects output for subprocesses and system calls.\n \"\"\"\n\n def __init__(\n self,\n filename: str,\n echo=False,\n debug=0,\n buffer=False,\n env=None,\n filter_fn=None,\n append=False,\n ):\n \"\"\"Create a new output log context manager.\n\n Args:\n filename (str): path to file where output should be logged\n echo (bool): whether to echo output in addition to logging it\n debug (int): positive to enable tty debug mode during logging\n buffer (bool): pass buffer=True to skip unbuffering output; note\n this doesn't set up any *new* buffering\n filter_fn (callable, optional): Callable[str] -> str to filter each\n line of output\n append (bool): whether to append to file ('a' mode)\n\n The filename will be opened and closed entirely within ``__enter__``\n and ``__exit__``.\n\n By default, we unbuffer sys.stdout and sys.stderr because the\n logger will include output from executed programs and from python\n calls. If stdout and stderr are buffered, their output won't be\n printed in the right place w.r.t. output from commands.\n\n Logger daemon is not started until ``__enter__()``.\n\n \"\"\"\n self.filename = filename\n self.echo = echo\n self.debug = debug\n self.buffer = buffer\n self.filter_fn = filter_fn\n self.append = append\n\n self._active = False # used to prevent re-entry\n\n def __enter__(self):\n if self._active:\n raise RuntimeError(\"Can't re-enter the same log_output!\")\n\n # record parent color settings before redirecting. We do this\n # because color output depends on whether the *original* stdout\n # is a TTY. New stdout won't be a TTY so we force colorization.\n self._saved_color = tty.color._force_color\n forced_color = tty.color.get_color_when()\n\n # also record parent debug settings -- in case the logger is\n # forcing debug output.\n self._saved_debug = tty._debug\n\n # Pipe for redirecting output to logger\n read_fd, self.write_fd = multiprocessing.Pipe(duplex=False)\n\n # Pipe for communication back from the daemon\n # Currently only used to save echo value between uses\n self.parent_pipe, child_pipe = multiprocessing.Pipe(duplex=False)\n\n stdin_fd = None\n stdout_fd = None\n try:\n # need to pass this b/c multiprocessing closes stdin in child.\n try:\n if sys.stdin.isatty():\n stdin_fd = Connection(os.dup(sys.stdin.fileno()))\n except BaseException:\n # just don't forward input if this fails\n pass\n\n # If our process has redirected stdout after the forkserver was started, we need to\n # make the forked processes use the new file descriptors.\n if multiprocessing.get_start_method() == \"forkserver\":\n stdout_fd = Connection(os.dup(sys.stdout.fileno()))\n\n self.process = multiprocessing.Process(\n target=_writer_daemon,\n args=(\n stdin_fd,\n stdout_fd,\n read_fd,\n self.write_fd,\n self.echo,\n self.filename,\n self.append,\n child_pipe,\n self.filter_fn,\n ),\n )\n self.process.daemon = True # must set before start()\n self.process.start()\n\n finally:\n if stdin_fd:\n stdin_fd.close()\n if stdout_fd:\n stdout_fd.close()\n read_fd.close()\n\n # Flush immediately before redirecting so that anything buffered\n # goes to the original stream\n sys.stdout.flush()\n sys.stderr.flush()\n\n # Now do the actual output redirection.\n # We use OS-level file descriptors, as this\n # redirects output for subprocesses and system calls.\n self._redirected_fds = {}\n\n # sys.stdout and sys.stderr may have been replaced with file objects under pytest, so\n # redirect their file descriptors in addition to the original fds 1 and 2.\n fds = {sys.stdout.fileno(), sys.stderr.fileno(), 1, 2}\n for fd in fds:\n self._redirected_fds[fd] = os.dup(fd)\n os.dup2(self.write_fd.fileno(), fd)\n\n self.write_fd.close()\n\n # Unbuffer stdout and stderr at the Python level\n if not self.buffer:\n sys.stdout = Unbuffered(sys.stdout)\n sys.stderr = Unbuffered(sys.stderr)\n\n # Force color and debug settings now that we have redirected.\n tty.color.set_color_when(forced_color)\n tty._debug = self.debug\n\n # track whether we're currently inside this log_output\n self._active = True\n\n # return this log_output object so that the user can do things\n # like temporarily echo some output.\n return self\n\n def __exit__(self, exc_type, exc_val, exc_tb):\n # Flush any buffered output to the logger daemon.\n sys.stdout.flush()\n sys.stderr.flush()\n\n # restore previous output settings using the OS-level way\n for fd, saved_fd in self._redirected_fds.items():\n os.dup2(saved_fd, fd)\n os.close(saved_fd)\n\n # recover and store echo settings from the child before it dies\n try:\n self.echo = self.parent_pipe.recv()\n except EOFError:\n # This may occur if some exception prematurely terminates the\n # _writer_daemon. An exception will have already been generated.\n pass\n\n # now that the write pipe is closed (in this __exit__, when we restore\n # stdout with dup2), the logger daemon process loop will terminate. We\n # wait for that here.\n self.process.join()\n\n # restore old color and debug settings\n tty.color._force_color = self._saved_color\n tty._debug = self._saved_debug\n\n self._active = False # safe to enter again\n\n @contextmanager\n def force_echo(self):\n \"\"\"Context manager to force local echo, even if echo is off.\"\"\"\n if not self._active:\n raise RuntimeError(\"Can't call force_echo() outside log_output region!\")\n\n # This uses the xon/xoff to highlight regions to be echoed in the\n # output. We us these control characters rather than, say, a\n # separate pipe, because they're in-band and assured to appear\n # exactly before and after the text we want to echo.\n sys.stdout.write(xon)\n sys.stdout.flush()\n try:\n yield\n finally:\n sys.stdout.write(xoff)\n sys.stdout.flush()\n\n\nclass StreamWrapper:\n \"\"\"Wrapper class to handle redirection of io streams\"\"\"\n\n def __init__(self, sys_attr):\n self.sys_attr = sys_attr\n self.saved_stream = None\n if sys.platform.startswith(\"win32\"):\n if hasattr(sys, \"gettotalrefcount\"): # debug build\n libc = ctypes.CDLL(\"ucrtbased\")\n else:\n libc = ctypes.CDLL(\"api-ms-win-crt-stdio-l1-1-0\")\n\n kernel32 = ctypes.WinDLL(\"kernel32\")\n\n # https://docs.microsoft.com/en-us/windows/console/getstdhandle\n if self.sys_attr == \"stdout\":\n STD_HANDLE = -11\n elif self.sys_attr == \"stderr\":\n STD_HANDLE = -12\n else:\n raise KeyError(self.sys_attr)\n\n c_stdout = kernel32.GetStdHandle(STD_HANDLE)\n self.libc = libc\n self.c_stream = c_stdout\n else:\n self.libc = ctypes.CDLL(None)\n self.c_stream = ctypes.c_void_p.in_dll(self.libc, self.sys_attr)\n self.sys_stream = getattr(sys, self.sys_attr)\n self.orig_stream_fd = self.sys_stream.fileno()\n # Save a copy of the original stdout fd in saved_stream\n self.saved_stream = os.dup(self.orig_stream_fd)\n\n def redirect_stream(self, to_fd):\n \"\"\"Redirect stdout to the given file descriptor.\"\"\"\n # Flush the C-level buffer stream\n if sys.platform.startswith(\"win32\"):\n self.libc.fflush(None)\n else:\n self.libc.fflush(self.c_stream)\n # Flush and close sys_stream - also closes the file descriptor (fd)\n sys_stream = getattr(sys, self.sys_attr)\n sys_stream.flush()\n sys_stream.close()\n # Make orig_stream_fd point to the same file as to_fd\n os.dup2(to_fd, self.orig_stream_fd)\n # Set sys_stream to a new stream that points to the redirected fd\n new_buffer = open(self.orig_stream_fd, \"wb\")\n new_stream = io.TextIOWrapper(new_buffer)\n setattr(sys, self.sys_attr, new_stream)\n self.sys_stream = getattr(sys, self.sys_attr)\n\n def flush(self):\n if sys.platform.startswith(\"win32\"):\n self.libc.fflush(None)\n else:\n self.libc.fflush(self.c_stream)\n self.sys_stream.flush()\n\n def close(self):\n \"\"\"Redirect back to the original system stream, and close stream\"\"\"\n try:\n if self.saved_stream is not None:\n self.redirect_stream(self.saved_stream)\n finally:\n if self.saved_stream is not None:\n os.close(self.saved_stream)\n\n\nclass winlog:\n \"\"\"\n Similar to nixlog, with underlying\n functionality ported to support Windows.\n\n Does not support the use of ``v`` toggling as nixlog does.\n \"\"\"\n\n def __init__(\n self, filename: str, echo=False, debug=0, buffer=False, filter_fn=None, append=False\n ):\n self.debug = debug\n self.echo = echo\n self.logfile = filename\n self.stdout = StreamWrapper(\"stdout\")\n self.stderr = StreamWrapper(\"stderr\")\n self._active = False\n self.old_stdout = sys.stdout\n self.old_stderr = sys.stderr\n self.append = append\n\n def __enter__(self):\n if self._active:\n raise RuntimeError(\"Can't re-enter the same log_output!\")\n\n # Open both write and reading on logfile\n write_mode = \"ab+\" if self.append else \"wb+\"\n self.writer = open(self.logfile, mode=write_mode)\n self.reader = open(self.logfile, mode=\"rb+\")\n\n # Dup stdout so we can still write to it after redirection\n self.echo_writer = open(os.dup(sys.stdout.fileno()), \"w\", encoding=sys.stdout.encoding)\n # Redirect stdout and stderr to write to logfile\n self.stderr.redirect_stream(self.writer.fileno())\n self.stdout.redirect_stream(self.writer.fileno())\n self._kill = threading.Event()\n\n def background_reader(reader, echo_writer, _kill):\n # for each line printed to logfile, read it\n # if echo: write line to user\n try:\n while True:\n is_killed = _kill.wait(0.1)\n # Flush buffered build output to file\n # stdout/err fds refer to log file\n self.stderr.flush()\n self.stdout.flush()\n\n line = reader.readline()\n if self.echo and line:\n echo_writer.write(\"{0}\".format(line.decode()))\n echo_writer.flush()\n\n if is_killed:\n break\n finally:\n reader.close()\n\n self._active = True\n self._thread = Thread(\n target=background_reader, args=(self.reader, self.echo_writer, self._kill)\n )\n self._thread.start()\n return self\n\n def __exit__(self, exc_type, exc_val, exc_tb):\n self.writer.close()\n self.echo_writer.flush()\n self.stdout.flush()\n self.stderr.flush()\n self._kill.set()\n self._thread.join()\n self.stdout.close()\n self.stderr.close()\n self._active = False\n\n @contextmanager\n def force_echo(self):\n \"\"\"Context manager to force local echo, even if echo is off.\"\"\"\n if not self._active:\n raise RuntimeError(\"Can't call force_echo() outside log_output region!\")\n yield\n\n\ndef _writer_daemon(\n stdin_fd: Optional[Connection],\n stdout_fd: Optional[Connection],\n read_fd: Connection,\n write_fd: Connection,\n echo: bool,\n log_filename: str,\n append: bool,\n control_fd: Connection,\n filter_fn: Optional[Callable[[str], str]],\n) -> None:\n \"\"\"Daemon used by ``log_output`` to write to a log file and to ``stdout``.\n\n The daemon receives output from the parent process and writes it both\n to a log and, optionally, to ``stdout``. The relationship looks like\n this::\n\n Terminal\n |\n | +-------------------------+\n | | Parent Process |\n +--------> | with log_output(): |\n | stdin | ... |\n | +-------------------------+\n | ^ | write_fd (parent's redirected stdout)\n | | control |\n | | pipe |\n | | v read_fd\n | +-------------------------+ stdout\n | | Writer daemon |------------>\n +--------> | read from read_fd | log_file\n stdin | write to out and log |------------>\n +-------------------------+\n\n Within the ``log_output`` handler, the parent's output is redirected\n to a pipe from which the daemon reads. The daemon writes each line\n from the pipe to a log file and (optionally) to ``stdout``. The user\n can hit ``v`` to toggle output on ``stdout``.\n\n In addition to the input and output file descriptors, the daemon\n interacts with the parent via ``control_pipe``. It reports whether\n ``stdout`` was enabled or disabled when it finished.\n\n Arguments:\n stdin_fd: optional input from the terminal\n read_fd: pipe for reading from parent's redirected stdout\n echo: initial echo setting -- controlled by user and preserved across multiple writer\n daemons\n log_filename: filename where output should be logged\n append: whether to append to the file or overwrite it\n control_pipe: multiprocessing pipe on which to send control information to the parent\n filter_fn: optional function to filter each line of output\n\n \"\"\"\n # This process depends on closing all instances of write_pipe to terminate the reading loop\n write_fd.close()\n\n # 1. Use line buffering (3rd param = 1) since Python 3 has a bug\n # that prevents unbuffered text I/O. [needs citation]\n # 2. Enforce a UTF-8 interpretation of build process output with errors replaced by '?'.\n # The downside is that the log file will not contain the exact output of the build process.\n # 3. closefd=False because Connection has \"ownership\"\n read_file = os.fdopen(\n read_fd.fileno(), \"r\", 1, encoding=\"utf-8\", errors=\"replace\", closefd=False\n )\n\n if stdin_fd:\n stdin_file = os.fdopen(stdin_fd.fileno(), closefd=False)\n else:\n stdin_file = None\n\n if stdout_fd:\n os.dup2(stdout_fd.fileno(), sys.stdout.fileno())\n stdout_fd.close()\n\n # list of streams to select from\n istreams = [read_file, stdin_file] if stdin_file else [read_file]\n force_echo = False # parent can force echo for certain output\n log_file = open(log_filename, mode=\"a\" if append else \"w\", encoding=\"utf-8\")\n\n try:\n with keyboard_input(stdin_file) as kb:\n while True:\n # fix the terminal settings if we recently came to\n # the foreground\n kb.check_fg_bg()\n\n # wait for input from any stream. use a coarse timeout to\n # allow other checks while we wait for input\n rlist, _, _ = select.select(istreams, [], [], 0.1)\n\n # Allow user to toggle echo with 'v' key.\n # Currently ignores other chars.\n # only read stdin if we're in the foreground\n if stdin_file and stdin_file in rlist and not _is_background_tty(stdin_file):\n # it's possible to be backgrounded between the above\n # check and the read, so we ignore SIGTTIN here.\n with ignore_signal(signal.SIGTTIN):\n try:\n if stdin_file.read(1) == \"v\":\n echo = not echo\n except OSError as e:\n # If SIGTTIN is ignored, the system gives EIO\n # to let the caller know the read failed b/c it\n # was in the bg. Ignore that too.\n if e.errno != errno.EIO:\n raise\n\n if read_file in rlist:\n line_count = 0\n try:\n while line_count < 100:\n # Handle output from the calling process.\n line = read_file.readline()\n\n if not line:\n return\n line_count += 1\n\n # find control characters and strip them.\n clean_line, num_controls = control.subn(\"\", line)\n\n # Echo to stdout if requested or forced.\n if echo or force_echo:\n output_line = clean_line\n if filter_fn:\n output_line = filter_fn(clean_line)\n enc = sys.stdout.encoding\n if enc != \"utf-8\":\n # On Python 3.6 and 3.7-3.14 with non-{utf-8,C} locale stdout\n # may not be able to handle utf-8 output. We do an inefficient\n # dance of re-encoding with errors replaced, so stdout.write\n # does not raise.\n output_line = output_line.encode(enc, \"replace\").decode(enc)\n sys.stdout.write(output_line)\n\n # Stripped output to log file.\n log_file.write(_strip(clean_line))\n\n if num_controls > 0:\n controls = control.findall(line)\n if xon in controls:\n force_echo = True\n if xoff in controls:\n force_echo = False\n\n if not _input_available(read_file):\n break\n finally:\n if line_count > 0:\n if echo or force_echo:\n sys.stdout.flush()\n log_file.flush()\n\n except BaseException:\n tty.error(\"Exception occurred in writer daemon!\")\n traceback.print_exc()\n\n finally:\n log_file.close()\n read_fd.close()\n if stdin_fd:\n stdin_fd.close()\n\n # send echo value back to the parent so it can be preserved.\n control_fd.send(echo)\n\n\ndef _input_available(f):\n return f in select.select([f], [], [], 0)[0]\n" }, { "path": "lib/spack/spack/main.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"This is the implementation of the Spack command line executable.\n\nIn a normal Spack installation, this is invoked from the bin/spack script\nafter the system path is set up.\n\"\"\"\n\nimport argparse\nimport gc\nimport inspect\nimport multiprocessing\nimport operator\nimport os\nimport pstats\nimport re\nimport shlex\nimport signal\nimport sys\nimport tempfile\nimport textwrap\nimport traceback\nimport warnings\nfrom contextlib import contextmanager\nfrom typing import Any, List, Optional, Set, Tuple\n\nimport spack.vendor.archspec.cpu\n\nimport spack\nimport spack.cmd\nimport spack.config\nimport spack.environment\nimport spack.environment as ev\nimport spack.environment.environment\nimport spack.error\nimport spack.llnl.util.lang\nimport spack.llnl.util.tty as tty\nimport spack.llnl.util.tty.colify\nimport spack.llnl.util.tty.color as color\nimport spack.paths\nimport spack.platforms\nimport spack.solver.asp\nimport spack.spec\nimport spack.util.environment\nimport spack.util.lock\n\nfrom .enums import ConfigScopePriority\n\n#: names of profile statistics\nstat_names = pstats.Stats.sort_arg_dict_default\n\n#: help levels in order of detail (i.e., number of commands shown)\nlevels = [\"short\", \"long\"]\n\n#: intro text for help at different levels\nintro_by_level = {\"short\": \"Common spack commands:\", \"long\": \"Commands:\"}\n\n#: control top-level spack options shown in basic vs. advanced help\noptions_by_level = {\"short\": [\"e\", \"h\", \"k\", \"V\", \"color\"], \"long\": \"all\"}\n\n#: Longer text for each section, to show in help\nsection_descriptions = {\n \"query\": \"query packages\",\n \"build\": \"build, install, and test packages\",\n \"environment\": \"environment\",\n \"config\": \"configuration\",\n \"packaging\": \"create packages\",\n \"admin\": \"administration\",\n \"developer\": \"spack development\",\n}\n\n#: preferential command order for some sections (e.g., build pipeline is\n#: in execution order, not alphabetical)\nsection_order = {\n \"basic\": [\"list\", \"info\", \"find\"],\n \"build\": [\n \"fetch\",\n \"stage\",\n \"patch\",\n \"configure\",\n \"build\",\n \"restage\",\n \"install\",\n \"uninstall\",\n \"clean\",\n ],\n \"packaging\": [\"create\", \"edit\"],\n}\n\n#: Properties that commands are required to set.\nrequired_command_properties = [\"level\", \"section\", \"description\"]\n\nspack_ld_library_path = os.environ.get(\"LD_LIBRARY_PATH\", \"\")\n\n\ndef add_all_commands(parser):\n \"\"\"Add all spack subcommands to the parser.\"\"\"\n for cmd in spack.cmd.all_commands():\n parser.add_command(cmd)\n\n\ndef index_commands():\n \"\"\"create an index of commands by section for this help level\"\"\"\n index = {}\n for command in spack.cmd.all_commands():\n cmd_module = spack.cmd.get_module(command)\n\n # make sure command modules have required properties\n for p in required_command_properties:\n prop = getattr(cmd_module, p, None)\n if not prop:\n tty.die(\"Command doesn't define a property '%s': %s\" % (p, command))\n\n # add commands to lists for their level and higher levels\n for level in reversed(levels):\n level_sections = index.setdefault(level, {})\n commands = level_sections.setdefault(cmd_module.section, [])\n commands.append(command)\n if level == cmd_module.level:\n break\n\n return index\n\n\nclass SpackHelpFormatter(argparse.RawTextHelpFormatter):\n def _format_actions_usage(self, actions, groups):\n \"\"\"Formatter with more concise usage strings.\"\"\"\n usage = super()._format_actions_usage(actions, groups)\n\n # Eliminate any occurrence of two or more consecutive spaces\n usage = re.sub(r\"[ ]{2,}\", \" \", usage)\n\n # compress single-character flags that are not mutually exclusive\n # at the beginning of the usage string\n chars = \"\".join(re.findall(r\"\\[-(.)\\]\", usage))\n usage = re.sub(r\"\\[-.\\] ?\", \"\", usage)\n if chars:\n usage = \"[-%s] %s\" % (chars, usage)\n return usage.strip()\n\n def start_section(self, heading):\n return super().start_section(color.colorize(f\"@*B{{{heading}}}\"))\n\n def _format_usage(self, usage, actions, groups, prefix=None):\n # if no optionals or positionals are available, usage is just prog\n if usage is None and not actions:\n return super()._format_usage(usage, actions, groups, prefix)\n\n # add color *after* argparse aligns the text, so as not to interfere\n result = super()._format_usage(usage, actions, groups, prefix)\n escaped = color.cescape(result)\n escaped = escaped.replace(self._prog, f\"@.@*C{{{self._prog}}}@c\")\n return color.colorize(f\"@B{escaped}@.\")\n\n def add_argument(self, action):\n if action.help is not argparse.SUPPRESS:\n # find all invocations\n get_invocation = self._format_action_invocation\n invocation_lengths = [color.clen(get_invocation(action)) + self._current_indent]\n for subaction in self._iter_indented_subactions(action):\n invocation_lengths.append(\n color.clen(get_invocation(subaction)) + self._current_indent\n )\n\n # update the maximum item length\n action_length = max(invocation_lengths)\n self._action_max_length = max(self._action_max_length, action_length)\n\n # add the item to the list\n self._add_item(self._format_action, [action])\n\n def _format_action(self, action):\n # this is where argparse aligns the help text next to each option\n help_position = min(self._action_max_length + 2, self._max_help_position)\n\n result = super()._format_action(action)\n\n # add color *after* argparse aligns the text, so we don't interfere with lengths\n if len(result) <= help_position:\n header, rest = result, \"\"\n elif result[help_position - 1] == \" \":\n header, rest = result[:help_position], result[help_position:]\n else:\n first_newline = result.index(\"\\n\")\n header, rest = result[:first_newline], result[first_newline:]\n\n return color.colorize(f\"@c{{{color.cescape(header)}}}{color.cescape(rest)}\")\n\n def add_arguments(self, actions):\n actions = sorted(actions, key=operator.attrgetter(\"option_strings\"))\n super().add_arguments(actions)\n\n\nclass SpackArgumentParser(argparse.ArgumentParser):\n def format_help_sections(self, level):\n \"\"\"Format help on sections for a particular verbosity level.\n\n Args:\n level (str): ``\"short\"`` or ``\"long\"`` (more commands shown for long)\n \"\"\"\n if level not in levels:\n raise ValueError(\"level must be one of: %s\" % levels)\n\n # lazily add all commands to the parser when needed.\n add_all_commands(self)\n\n # Print help on subcommands in neatly formatted sections.\n formatter = self._get_formatter()\n\n # Create a list of subcommand actions. Argparse internals are nasty!\n # Note: you can only call _get_subactions() once. Even nastier!\n if not hasattr(self, \"actions\"):\n self.actions = self._subparsers._actions[-1]._get_subactions()\n\n # make a set of commands not yet added.\n remaining = set(spack.cmd.all_commands())\n\n def add_group(group):\n formatter.start_section(group.title)\n formatter.add_text(group.description)\n formatter.add_arguments(group._group_actions)\n formatter.end_section()\n\n def add_subcommand_group(title, commands):\n \"\"\"Add informational help group for a specific subcommand set.\"\"\"\n cmd_set = set(c for c in commands)\n\n # make a dict of commands of interest\n cmds = dict((a.dest, a) for a in self.actions if a.dest in cmd_set)\n\n # add commands to a group in order, and add the group\n group = argparse._ArgumentGroup(self, title=title)\n for name in commands:\n group._add_action(cmds[name])\n if name in remaining:\n remaining.remove(name)\n add_group(group)\n\n # select only the options for the particular level we're showing.\n show_options = options_by_level[level]\n options = [\n opt\n for group in self._action_groups\n for opt in group._group_actions\n if group.title not in [\"positional arguments\"]\n ]\n opts = {opt.option_strings[0].strip(\"-\"): opt for opt in options}\n actions = [o for o in opts.values()]\n if show_options != \"all\":\n actions = [opts[letter] for letter in show_options]\n\n # custom, more concise usage for top level\n help_options = actions + [self._positionals._group_actions[-1]]\n formatter.add_usage(self.usage, help_options, self._mutually_exclusive_groups)\n\n # description\n formatter.add_text(self.description)\n\n # start subcommands\n formatter.add_text(color.colorize(f\"@*C{{{intro_by_level[level]}}}\"))\n\n # add argument groups based on metadata in commands\n index = index_commands()\n sections = index[level]\n\n for section in sorted(sections):\n if section == \"help\":\n continue # Cover help in the epilog.\n\n group_description = section_descriptions.get(section, section)\n\n to_display = sections[section]\n commands = []\n\n # add commands whose order we care about first.\n if section in section_order:\n commands.extend(cmd for cmd in section_order[section] if cmd in to_display)\n\n # add rest in alphabetical order.\n commands.extend(cmd for cmd in sorted(sections[section]) if cmd not in commands)\n\n # add the group to the parser\n add_subcommand_group(group_description, commands)\n\n # start subcommands\n formatter.add_text(color.colorize(\"@*C{Options:}\"))\n\n # optionals and user-defined groups\n for group in sorted(\n self._action_groups, key=lambda g: (g.title == \"help\", g.title != \"general\", g.title)\n ):\n if group.title == \"positional arguments\":\n continue # handled by subcommand help above\n\n filtered_actions = [action for action in group._group_actions if action in actions]\n if not filtered_actions:\n continue\n\n formatter.start_section(group.title)\n formatter.add_text(group.description)\n\n formatter.add_arguments(filtered_actions)\n formatter.end_section()\n\n # epilog\n help_section = textwrap.dedent(\n \"\"\"\\\n @*C{More help}:\n @c{spack help --all} list all commands and options\n @c{spack help } help on a specific command\n @c{spack help --spec} help on the package specification syntax\n @c{spack docs} open https://spack.rtfd.io/ in a browser\n \"\"\"\n )\n formatter.add_text(color.colorize(help_section))\n\n # determine help from format above\n return formatter.format_help()\n\n def add_subparsers(self, **kwargs):\n \"\"\"Ensure that sensible defaults are propagated to subparsers\"\"\"\n kwargs.setdefault(\"metavar\", \"SUBCOMMAND\")\n\n # From Python 3.7 we can require a subparser, earlier versions\n # of argparse will error because required=True is unknown\n if sys.version_info[:2] > (3, 6):\n kwargs.setdefault(\"required\", True)\n\n sp = super().add_subparsers(**kwargs)\n # This monkey patching is needed for Python 3.6, which supports\n # having a required subparser but don't expose the API used above\n if sys.version_info[:2] == (3, 6):\n sp.required = True\n\n old_add_parser = sp.add_parser\n\n def add_parser(name, **kwargs):\n kwargs.setdefault(\"formatter_class\", SpackHelpFormatter)\n return old_add_parser(name, **kwargs)\n\n sp.add_parser = add_parser\n return sp\n\n def add_command(self, cmd_name):\n \"\"\"Add one subcommand to this parser.\"\"\"\n # lazily initialize any subparsers\n if not hasattr(self, \"subparsers\"):\n # remove the dummy \"command\" argument.\n if self._actions[-1].dest == \"command\":\n self._remove_action(self._actions[-1])\n self.subparsers = self.add_subparsers(metavar=\"COMMAND\", dest=\"command\")\n\n if cmd_name not in self.subparsers._name_parser_map:\n # each command module implements a parser() function, to which we\n # pass its subparser for setup.\n module = spack.cmd.get_module(cmd_name)\n\n # build a list of aliases\n alias_list = []\n aliases = spack.config.get(\"config:aliases\")\n if aliases:\n alias_list = [k for k, v in aliases.items() if shlex.split(v)[0] == cmd_name]\n\n subparser = self.subparsers.add_parser(\n cmd_name,\n aliases=alias_list,\n help=module.description,\n description=module.description,\n )\n module.setup_parser(subparser)\n\n # return the callable function for the command\n return spack.cmd.get_command(cmd_name)\n\n def format_help(self, level=\"short\"):\n if self.prog == \"spack\":\n # use format_help_sections for the main spack parser, but not\n # for subparsers\n return self.format_help_sections(level)\n else:\n # in subparsers, self.prog is, e.g., 'spack install'\n return super().format_help()\n\n def _check_value(self, action, value):\n # converted value must be one of the choices (if specified)\n if action.choices is not None and value not in action.choices:\n cols = spack.llnl.util.tty.colify.colified(sorted(action.choices), indent=4, tty=True)\n msg = \"invalid choice: %r choose from:\\n%s\" % (value, cols)\n raise argparse.ArgumentError(action, msg)\n\n\ndef make_argument_parser(**kwargs):\n \"\"\"Create an basic argument parser without any subcommands added.\"\"\"\n parser = SpackArgumentParser(\n prog=\"spack\",\n formatter_class=SpackHelpFormatter,\n add_help=False,\n description=(\n \"A flexible package manager that supports multiple versions,\\n\"\n \"configurations, platforms, and compilers.\"\n ),\n **kwargs,\n )\n\n general = parser.add_argument_group(\"general\")\n general.add_argument(\n \"--color\",\n action=\"store\",\n default=None,\n choices=(\"always\", \"never\", \"auto\"),\n help=\"when to colorize output (default: auto)\",\n )\n general.add_argument(\n \"-v\", \"--verbose\", action=\"store_true\", help=\"print additional output during builds\"\n )\n general.add_argument(\n \"-k\",\n \"--insecure\",\n action=\"store_true\",\n help=\"do not check ssl certificates when downloading\",\n )\n general.add_argument(\n \"-b\", \"--bootstrap\", action=\"store_true\", help=\"use bootstrap config, store, and externals\"\n )\n general.add_argument(\n \"-V\", \"--version\", action=\"store_true\", help=\"show version number and exit\"\n )\n general.add_argument(\n \"-h\",\n \"--help\",\n dest=\"help\",\n action=\"store_const\",\n const=\"short\",\n default=None,\n help=\"show this help message and exit\",\n )\n general.add_argument(\n \"-H\",\n \"--all-help\",\n dest=\"help\",\n action=\"store_const\",\n const=\"long\",\n default=None,\n help=\"show help for all commands (same as `spack help --all`)\",\n )\n\n config = parser.add_argument_group(\"configuration and environments\")\n config.add_argument(\n \"-c\",\n \"--config\",\n default=None,\n action=\"append\",\n dest=\"config_vars\",\n help=\"add one or more custom, one-off config settings\",\n )\n config.add_argument(\n \"-C\",\n \"--config-scope\",\n dest=\"config_scopes\",\n action=\"append\",\n metavar=\"DIR|ENV\",\n help=\"add directory or environment as read-only config scope\",\n )\n envs = config # parser.add_argument_group(\"environments\")\n env_mutex = envs.add_mutually_exclusive_group()\n env_mutex.add_argument(\n \"-e\", \"--env\", dest=\"env\", metavar=\"ENV\", action=\"store\", help=\"run with an environment\"\n )\n env_mutex.add_argument(\n \"-D\",\n \"--env-dir\",\n dest=\"env_dir\",\n metavar=\"DIR\",\n action=\"store\",\n help=\"run with environment in directory (ignore managed envs)\",\n )\n env_mutex.add_argument(\n \"-E\",\n \"--no-env\",\n dest=\"no_env\",\n action=\"store_true\",\n help=\"run without any environments activated (see spack env)\",\n )\n envs.add_argument(\n \"--use-env-repo\",\n action=\"store_true\",\n help=\"when in an environment, use its package repository\",\n )\n\n debug = parser.add_argument_group(\"debug\")\n debug.add_argument(\n \"-d\",\n \"--debug\",\n action=\"count\",\n default=0,\n help=\"write out debug messages\\n\\n(more d's for more verbosity: -d, -dd, -ddd, etc.)\",\n )\n debug.add_argument(\n \"-t\",\n \"--backtrace\",\n action=\"store_true\",\n default=\"SPACK_BACKTRACE\" in os.environ,\n help=\"always show backtraces for exceptions\",\n )\n debug.add_argument(\"--pdb\", action=\"store_true\", help=argparse.SUPPRESS)\n debug.add_argument(\"--timestamp\", action=\"store_true\", help=\"add a timestamp to tty output\")\n debug.add_argument(\n \"-m\", \"--mock\", action=\"store_true\", help=\"use mock packages instead of real ones\"\n )\n debug.add_argument(\n \"--print-shell-vars\", action=\"store\", help=\"print info needed by setup-env.*sh\"\n )\n debug.add_argument(\n \"--stacktrace\",\n action=\"store_true\",\n default=\"SPACK_STACKTRACE\" in os.environ,\n help=\"add stacktraces to all printed statements\",\n )\n\n locks = general\n lock_mutex = locks.add_mutually_exclusive_group()\n lock_mutex.add_argument(\n \"-l\",\n \"--enable-locks\",\n action=\"store_true\",\n dest=\"locks\",\n default=None,\n help=\"use filesystem locking (default)\",\n )\n lock_mutex.add_argument(\n \"-L\",\n \"--disable-locks\",\n action=\"store_false\",\n dest=\"locks\",\n help=\"do not use filesystem locking (unsafe)\",\n )\n\n debug.add_argument(\n \"-p\", \"--profile\", action=\"store_true\", dest=\"spack_profile\", help=argparse.SUPPRESS\n )\n debug.add_argument(\"--profile-file\", default=None, help=argparse.SUPPRESS)\n debug.add_argument(\"--sorted-profile\", default=None, metavar=\"STAT\", help=argparse.SUPPRESS)\n debug.add_argument(\"--lines\", default=20, action=\"store\", help=argparse.SUPPRESS)\n\n return parser\n\n\ndef showwarning(message, category, filename, lineno, file=None, line=None):\n \"\"\"Redirects messages to tty.warn.\"\"\"\n if category is spack.error.SpackAPIWarning:\n tty.warn(f\"{filename}:{lineno}: {message}\")\n else:\n tty.warn(message)\n\n\ndef setup_main_options(args):\n \"\"\"Configure spack globals based on the basic options.\"\"\"\n # Set up environment based on args.\n tty.set_verbose(args.verbose)\n tty.set_debug(args.debug)\n tty.set_stacktrace(args.stacktrace)\n\n # debug must be set first so that it can even affect behavior of\n # errors raised by spack.config.\n\n if args.debug or args.backtrace:\n spack.error.debug = True\n spack.error.SHOW_BACKTRACE = True\n\n if args.debug:\n spack.config.set(\"config:debug\", True, scope=\"command_line\")\n spack.util.environment.TRACING_ENABLED = True\n\n if args.timestamp:\n tty.set_timestamp(True)\n\n # override lock configuration if passed on command line\n if args.locks is not None:\n if args.locks is False:\n spack.util.lock.check_lock_safety(spack.paths.prefix)\n spack.config.set(\"config:locks\", args.locks, scope=\"command_line\")\n\n if args.mock:\n import spack.util.spack_yaml as syaml\n\n key = syaml.syaml_str(\"repos\")\n key.override = True\n spack.config.CONFIG.scopes[\"command_line\"].sections[\"repos\"] = syaml.syaml_dict(\n [(key, [spack.paths.mock_packages_path])]\n )\n\n # If the user asked for it, don't check ssl certs.\n if args.insecure:\n tty.warn(\"You asked for --insecure. Will NOT check SSL certificates.\")\n spack.config.set(\"config:verify_ssl\", False, scope=\"command_line\")\n\n # Use the spack config command to handle parsing the config strings\n for config_var in args.config_vars or []:\n spack.config.add(fullpath=config_var, scope=\"command_line\")\n\n # On Windows10 console handling for ASCI/VT100 sequences is not\n # on by default. Turn on before we try to write to console\n # with color\n color.try_enable_terminal_color_on_windows()\n # when to use color (takes always, auto, or never)\n if args.color is not None:\n color.set_color_when(args.color)\n\n\ndef allows_unknown_args(command):\n \"\"\"Implements really simple argument injection for unknown arguments.\n\n Commands may add an optional argument called \"unknown args\" to\n indicate they can handle unknown args, and we'll pass the unknown\n args in.\n \"\"\"\n info = dict(inspect.getmembers(command))\n varnames = info[\"__code__\"].co_varnames\n argcount = info[\"__code__\"].co_argcount\n return argcount == 3 and varnames[2] == \"unknown_args\"\n\n\ndef _invoke_command(command, parser, args, unknown_args):\n \"\"\"Run a spack command *without* setting spack global options.\"\"\"\n if allows_unknown_args(command):\n return_val = command(parser, args, unknown_args)\n else:\n if unknown_args:\n tty.die(\"unrecognized arguments: %s\" % \" \".join(unknown_args))\n return_val = command(parser, args)\n\n # Allow commands to return and error code if they want\n return 0 if return_val is None else return_val\n\n\nclass SpackCommand:\n \"\"\"Callable object that invokes a Spack command (for testing).\n\n Example usage::\n\n install = SpackCommand(\"install\")\n install(\"-v\", \"mpich\")\n\n Use this to invoke Spack commands directly from Python and check their output.\"\"\"\n\n def __init__(self, command_name: str) -> None:\n \"\"\"Create a new SpackCommand that invokes ``command_name`` when called.\n\n Args:\n command_name: name of the command to invoke\n \"\"\"\n self.parser = make_argument_parser()\n self.command_name = command_name\n #: Return code of the last command invocation\n self.returncode: Any = None\n #: Error raised during the last command invocation, if any\n self.error: Optional[BaseException] = None\n #: Binary output captured from the last command invocation\n self.binary_output = b\"\"\n #: Decoded output captured from the last command invocation\n self.output = \"\"\n\n def __call__(self, *argv: str, capture: bool = True, fail_on_error: bool = True) -> str:\n \"\"\"Invoke this SpackCommand. Returns the combined stdout/stderr.\n\n Args:\n argv: command line arguments.\n\n Keyword Args:\n capture: Capture output from the command\n fail_on_error: Don't raise an exception on error\n\n On return, if ``fail_on_error`` is False, return value of command is set in ``returncode``\n property, and the error is set in the ``error`` property. Otherwise, raise an error.\"\"\"\n self.returncode = None\n self.error = None\n self.binary_output = b\"\"\n self.output = \"\"\n\n try:\n with self.capture_output(enable=capture):\n command = self.parser.add_command(self.command_name)\n args, unknown = self.parser.parse_known_args([self.command_name, *argv])\n setup_main_options(args)\n self.returncode = _invoke_command(command, self.parser, args, unknown)\n except SystemExit as e:\n # When the command calls sys.exit instead of returning an exit code\n self.error = e\n self.returncode = e.code\n except BaseException as e:\n # For other exceptions, raise the original exception if fail_on_error is True\n self.error = e\n if fail_on_error:\n raise\n finally:\n self.output = self.binary_output.decode(\"utf-8\", errors=\"replace\")\n\n if fail_on_error and self.returncode not in (0, None):\n raise SpackCommandError(self.returncode, self.output) from self.error\n\n return self.output\n\n @contextmanager\n def capture_output(self, enable: bool = True):\n \"\"\"Captures stdout and stderr from the current process and all subprocesses. This uses a\n temporary file and os.dup2 to redirect file descriptors.\"\"\"\n if not enable:\n yield self\n return\n with tempfile.TemporaryFile(mode=\"w+b\") as tmp_file:\n # sys.stdout and sys.stderr may have been replaced with file objects under pytest, so\n # redirect their file descriptors in addition to the original fds 1 and 2.\n fds: Set[int] = {sys.stdout.fileno(), sys.stderr.fileno(), 1, 2}\n saved_fds = {fd: os.dup(fd) for fd in fds}\n sys.stdout.flush()\n sys.stderr.flush()\n for fd in fds:\n os.dup2(tmp_file.fileno(), fd)\n try:\n yield self\n finally:\n sys.stdout.flush()\n sys.stderr.flush()\n for fd, saved_fd in saved_fds.items():\n os.dup2(saved_fd, fd)\n os.close(saved_fd)\n tmp_file.seek(0)\n self.binary_output = tmp_file.read()\n\n\ndef _profile_wrapper(command, main_args, parser, args, unknown_args):\n import cProfile\n\n try:\n nlines = int(main_args.lines)\n except ValueError:\n if main_args.lines != \"all\":\n tty.die(\"Invalid number for --lines: %s\" % main_args.lines)\n nlines = -1\n\n # allow comma-separated list of fields\n sortby = [\"time\"]\n if main_args.sorted_profile:\n sortby = main_args.sorted_profile.split(\",\")\n for stat in sortby:\n if stat not in stat_names:\n tty.die(\"Invalid sort field: %s\" % stat)\n\n try:\n # make a profiler and run the code.\n pr = cProfile.Profile()\n pr.enable()\n return _invoke_command(command, parser, args, unknown_args)\n\n finally:\n pr.disable()\n\n if main_args.profile_file:\n pr.dump_stats(main_args.profile_file)\n\n # print out profile stats.\n stats = pstats.Stats(pr, stream=sys.stderr)\n stats.sort_stats(*sortby)\n stats.print_stats(nlines)\n\n\n@spack.llnl.util.lang.memoized\ndef _compatible_sys_types():\n \"\"\"Return a list of all the platform-os-target tuples compatible\n with the current host.\n \"\"\"\n host_platform = spack.platforms.host()\n host_os = str(host_platform.default_operating_system())\n host_target = spack.vendor.archspec.cpu.host()\n compatible_targets = [host_target] + host_target.ancestors\n\n compatible_archs = [\n str(spack.spec.ArchSpec((str(host_platform), host_os, str(target))))\n for target in compatible_targets\n ]\n return compatible_archs\n\n\ndef print_setup_info(*info):\n \"\"\"Print basic information needed by setup-env.[c]sh.\n\n Args:\n info (list): list of things to print: comma-separated list\n of ``\"csh\"``, ``\"sh\"``, or ``\"modules\"``\n\n This is in ``main.py`` to make it fast; the setup scripts need to\n invoke spack in login scripts, and it needs to be quick.\n \"\"\"\n from spack.modules.common import root_path\n\n shell = \"csh\" if \"csh\" in info else \"sh\"\n\n def shell_set(var, value):\n if shell == \"sh\":\n print(\"%s='%s'\" % (var, value))\n elif shell == \"csh\":\n print(\"set %s = '%s'\" % (var, value))\n else:\n tty.die(\"shell must be sh or csh\")\n\n # print sys type\n shell_set(\"_sp_sys_type\", str(spack.spec.ArchSpec.default_arch()))\n shell_set(\"_sp_compatible_sys_types\", \":\".join(_compatible_sys_types()))\n # print roots for all module systems\n module_to_roots = {\"tcl\": list(), \"lmod\": list()}\n for name in module_to_roots.keys():\n path = root_path(name, \"default\")\n module_to_roots[name].append(path)\n\n other_spack_instances = spack.config.get(\"upstreams\") or {}\n for install_properties in other_spack_instances.values():\n upstream_module_roots = install_properties.get(\"modules\", {})\n upstream_module_roots = dict(\n (k, v) for k, v in upstream_module_roots.items() if k in module_to_roots\n )\n for module_type, root in upstream_module_roots.items():\n module_to_roots[module_type].append(root)\n\n for name, paths in module_to_roots.items():\n # Environment setup prepends paths, so the order is reversed here to\n # preserve the intended priority: the modules of the local Spack\n # instance are the highest-precedence.\n roots_val = \":\".join(reversed(paths))\n shell_set(\"_sp_%s_roots\" % name, roots_val)\n\n\ndef restore_macos_dyld_vars():\n \"\"\"\n Spack mutates ``DYLD_*`` variables in ``spack load`` and ``spack env activate``.\n Unlike Linux, macOS SIP clears these variables in new processes, meaning\n that ``os.environ[\"DYLD_*\"]`` in our Python process is not the same as the user's\n shell. Therefore, we store the user's ``DYLD_*`` variables in ``SPACK_DYLD_*`` and\n restore them here.\n \"\"\"\n if not sys.platform == \"darwin\":\n return\n\n for dyld_var in (\"DYLD_LIBRARY_PATH\", \"DYLD_FALLBACK_LIBRARY_PATH\"):\n stored_var_name = f\"SPACK_{dyld_var}\"\n if stored_var_name in os.environ:\n os.environ[dyld_var] = os.environ[stored_var_name]\n\n\ndef resolve_alias(cmd_name: str, cmd: List[str]) -> Tuple[str, List[str]]:\n \"\"\"Resolves aliases in the given command.\n\n Args:\n cmd_name: command name.\n cmd: command line arguments.\n\n Returns:\n new command name and arguments.\n \"\"\"\n all_commands = spack.cmd.all_commands()\n aliases = spack.config.get(\"config:aliases\")\n\n if aliases:\n for key, value in aliases.items():\n if \" \" in key:\n tty.warn(\n f\"Alias '{key}' (mapping to '{value}') contains a space\"\n \", which is not supported.\"\n )\n if key in all_commands:\n tty.warn(\n f\"Alias '{key}' (mapping to '{value}') attempts to override built-in command.\"\n )\n\n if cmd_name not in all_commands:\n alias = None\n\n if aliases:\n alias = aliases.get(cmd_name)\n\n if alias is not None:\n alias_parts = shlex.split(alias)\n cmd_name = alias_parts[0]\n cmd = alias_parts + cmd[1:]\n\n return cmd_name, cmd\n\n\n# sentinel scope marker for environments passed on the command line\n_ENV = object()\n\n\ndef add_command_line_scopes(\n cfg: spack.config.Configuration, command_line_scopes: List[str]\n) -> None:\n \"\"\"Add additional scopes from the ``--config-scope`` argument, either envs or dirs.\n\n Args:\n cfg: configuration instance\n command_line_scopes: list of configuration scope paths\n\n Raises:\n spack.error.ConfigError: if the path is an invalid configuration scope\n \"\"\"\n for i, path in enumerate(command_line_scopes):\n name = f\"cmd_scope_{i}\"\n scope = ev.environment_path_scope(name, path)\n if scope is None:\n if os.path.isdir(path): # directory with config files\n cfg.push_scope(\n spack.config.DirectoryConfigScope(name, path, writable=False),\n priority=ConfigScopePriority.CUSTOM,\n )\n continue\n else:\n raise spack.error.ConfigError(f\"Invalid configuration scope: {path}\")\n\n cfg.push_scope(scope, priority=ConfigScopePriority.CUSTOM)\n\n\ndef _main(argv=None):\n \"\"\"Logic for the main entry point for the Spack command.\n\n ``main()`` calls ``_main()`` and catches any errors that emerge.\n\n ``_main()`` handles:\n\n 1. Parsing arguments;\n 2. Setting up configuration; and\n 3. Finding and executing a Spack command.\n\n Args:\n argv (list or None): command line arguments, NOT including\n the executable name. If None, parses from ``sys.argv``.\n\n \"\"\"\n # ------------------------------------------------------------------------\n # main() is tricky to get right, so be careful where you put things.\n #\n # Things in this first part of `main()` should *not* require any\n # configuration. This doesn't include much -- setting up the parser,\n # restoring some key environment variables, very simple CLI options, etc.\n # ------------------------------------------------------------------------\n warnings.showwarning = showwarning\n\n # Create a parser with a simple positional argument first. We'll\n # lazily load the subcommand(s) we need later. This allows us to\n # avoid loading all the modules from spack.cmd when we don't need\n # them, which reduces startup latency.\n parser = make_argument_parser()\n parser.add_argument(\"command\", nargs=argparse.REMAINDER)\n args = parser.parse_args(argv)\n\n # Just print help and exit if run with no arguments at all\n no_args = (len(sys.argv) == 1) if argv is None else (len(argv) == 0)\n if no_args:\n parser.print_help()\n return 1\n\n # version is special as it does not require a command or loading and additional infrastructure\n if args.version:\n print(spack.get_version())\n return 0\n\n # ------------------------------------------------------------------------\n # This part of the `main()` sets up Spack's configuration.\n #\n # We set command line options (like --debug), then command line config\n # scopes, then environment configuration here.\n # ------------------------------------------------------------------------\n\n # Make spack load / env activate work on macOS\n restore_macos_dyld_vars()\n\n # store any error that occurred loading an env\n env_format_error = None\n env = None\n\n # try to find an active environment here, so that we can activate it later\n if not args.no_env:\n try:\n env = spack.cmd.find_environment(args)\n except (spack.config.ConfigFormatError, ev.SpackEnvironmentConfigError) as e:\n # print the context but delay this exception so that commands like\n # `spack config edit` can still work with a bad environment.\n e.print_context()\n env_format_error = e\n\n def add_environment_scope():\n if env_format_error:\n # Allow command to continue without env in case it is `spack config edit`\n # All other cases will raise in `finish_parse_and_run`\n spack.environment.environment._active_environment_error = env_format_error\n return\n # do not call activate here, as it has a lot of expensive function calls to deal\n # with mutation of spack.config.CONFIG -- but we are still building the config.\n env.manifest.prepare_config_scope()\n spack.environment.environment._active_environment = env\n\n # add the environment\n if env:\n add_environment_scope()\n\n # Push scopes from the command line last\n if args.config_scopes:\n add_command_line_scopes(spack.config.CONFIG, args.config_scopes)\n spack.config.CONFIG.push_scope(\n spack.config.InternalConfigScope(\"command_line\"), priority=ConfigScopePriority.COMMAND_LINE\n )\n setup_main_options(args)\n\n # ------------------------------------------------------------------------\n # Things that require configuration should go below here\n # ------------------------------------------------------------------------\n if args.print_shell_vars:\n print_setup_info(*args.print_shell_vars.split(\",\"))\n return 0\n\n # -h and -H are special as they do not require a command, but\n # all the other options do nothing without a command.\n if args.help:\n sys.stdout.write(parser.format_help(level=args.help))\n return 0\n\n # At this point we've considered all the options to spack itself, so we\n # need a command or we're done.\n if not args.command:\n parser.print_help()\n return 1\n\n # Try to load the particular command the caller asked for.\n cmd_name = args.command[0]\n cmd_name, args.command = resolve_alias(cmd_name, args.command)\n\n # set up a bootstrap context, if asked.\n # bootstrap context needs to include parsing the command, b/c things\n # like `ConstraintAction` and `ConfigSetAction` happen at parse time.\n bootstrap_context = spack.llnl.util.lang.nullcontext()\n if args.bootstrap:\n import spack.bootstrap as bootstrap # avoid circular imports\n\n bootstrap_context = bootstrap.ensure_bootstrap_configuration()\n\n with bootstrap_context:\n return finish_parse_and_run(parser, cmd_name, args, env_format_error)\n\n\ndef finish_parse_and_run(parser, cmd_name, main_args, env_format_error):\n \"\"\"Finish parsing after we know the command to run.\"\"\"\n # add the found command to the parser and re-run then re-parse\n command = parser.add_command(cmd_name)\n args, unknown = parser.parse_known_args(main_args.command)\n # we need to inherit verbose since the install command checks for it\n args.verbose = main_args.verbose\n args.lines = main_args.lines\n\n # Now that we know what command this is and what its args are, determine\n # whether we can continue with a bad environment and raise if not.\n if env_format_error:\n subcommand = getattr(args, \"config_command\", None)\n if (cmd_name, subcommand) != (\"config\", \"edit\"):\n raise env_format_error\n\n # many operations will fail without a working directory.\n spack.paths.set_working_dir()\n\n # now we can actually execute the command.\n if main_args.spack_profile or main_args.sorted_profile or main_args.profile_file:\n new_args = [sys.executable, \"-m\", \"cProfile\"]\n if main_args.sorted_profile:\n new_args.extend([\"-s\", main_args.sorted_profile])\n if main_args.profile_file:\n new_args.extend([\"-o\", main_args.profile_file])\n new_args.append(spack.paths.spack_script)\n skip_next = False\n for arg in sys.argv[1:]:\n if skip_next:\n skip_next = False\n continue\n if arg in (\"--sorted-profile\", \"--profile-file\", \"--lines\"):\n skip_next = True\n continue\n if arg.startswith((\"--sorted-profile=\", \"--profile-file=\", \"--lines=\")):\n continue\n if arg in (\"--profile\", \"-p\"):\n continue\n new_args.append(arg)\n formatted_args = \" \".join(shlex.quote(a) for a in new_args)\n tty.warn(\n \"The --profile flag is deprecated and will be removed in Spack v1.3. \"\n f\"Use `{formatted_args}` instead.\"\n )\n _profile_wrapper(command, main_args, parser, args, unknown)\n elif main_args.pdb:\n new_args = [sys.executable, \"-m\", \"pdb\", spack.paths.spack_script]\n new_args.extend(arg for arg in sys.argv[1:] if arg != \"--pdb\")\n formatted_args = \" \".join(shlex.quote(arg) for arg in new_args)\n tty.warn(\n \"The --pdb flag is deprecated and will be removed in Spack v1.3. \"\n f\"Use `{formatted_args}` instead.\"\n )\n import pdb\n\n pdb.runctx(\"_invoke_command(command, parser, args, unknown)\", globals(), locals())\n return 0\n else:\n return _invoke_command(command, parser, args, unknown)\n\n\ndef main(argv=None):\n \"\"\"This is the entry point for the Spack command.\n\n ``main()`` itself is just an error handler -- it handles errors for\n everything in Spack that makes it to the top level.\n\n The logic is all in ``_main()``.\n\n Args:\n argv (list or None): command line arguments, NOT including\n the executable name. If None, parses from sys.argv.\n\n \"\"\"\n # When using the forkserver start method, preload the following modules to improve startup\n # time of child processes.\n multiprocessing.set_forkserver_preload([\"spack.main\", \"spack.package\", \"spack.new_installer\"])\n try:\n g0, g1, g2 = gc.get_threshold()\n gc.set_threshold(50 * g0, g1, g2)\n return _main(argv)\n\n except spack.solver.asp.OutputDoesNotSatisfyInputError as e:\n _handle_solver_bug(e)\n return 1\n\n except spack.error.SpackError as e:\n tty.debug(e)\n e.die() # gracefully die on any SpackErrors\n\n except KeyboardInterrupt:\n if spack.config.get(\"config:debug\") or spack.error.SHOW_BACKTRACE:\n raise\n sys.stderr.write(\"\\n\")\n tty.error(\"Keyboard interrupt.\")\n return signal.SIGINT.value\n\n except SystemExit as e:\n if spack.config.get(\"config:debug\") or spack.error.SHOW_BACKTRACE:\n traceback.print_exc()\n return e.code\n\n except Exception as e:\n if spack.config.get(\"config:debug\") or spack.error.SHOW_BACKTRACE:\n raise\n tty.error(e)\n return 3\n\n\ndef _handle_solver_bug(\n e: spack.solver.asp.OutputDoesNotSatisfyInputError, out=sys.stderr, root=None\n) -> None:\n # when the solver outputs specs that do not satisfy the input and spack is used as a command\n # line tool, we dump the incorrect output specs to json so users can upload them in bug reports\n wrong_output = [(input, output) for input, output in e.input_to_output if output is not None]\n no_output = [input for input, output in e.input_to_output if output is None]\n if no_output:\n tty.error(\n \"internal solver error: the following specs were not solved:\\n - \"\n + \"\\n - \".join(str(s) for s in no_output),\n stream=out,\n )\n if wrong_output:\n msg = \"internal solver error: the following specs were concretized, but do not satisfy \"\n msg += \"the input:\\n\"\n for in_spec, out_spec in wrong_output:\n msg += f\" - input: {in_spec}\\n\"\n msg += f\" output: {out_spec.long_spec}\\n\"\n msg += \"\\n Please report a bug at https://github.com/spack/spack/issues\"\n\n # try to write the input/output specs to a temporary directory for bug reports\n try:\n tmpdir = tempfile.mkdtemp(prefix=\"spack-asp-\", dir=root)\n files = []\n for i, (input, output) in enumerate(wrong_output, start=1):\n in_file = os.path.join(tmpdir, f\"input-{i}.json\")\n out_file = os.path.join(tmpdir, f\"output-{i}.json\")\n files.append(in_file)\n files.append(out_file)\n with open(in_file, \"w\", encoding=\"utf-8\") as f:\n input.to_json(f)\n with open(out_file, \"w\", encoding=\"utf-8\") as f:\n output.to_json(f)\n\n msg += \" and attach the following files:\\n - \" + \"\\n - \".join(files)\n except Exception:\n msg += \".\"\n tty.error(msg, stream=out)\n\n\nclass SpackCommandError(Exception):\n \"\"\"Raised when SpackCommand execution fails, replacing SystemExit.\"\"\"\n\n def __init__(self, code, output):\n self.code = code\n self.output = output\n super().__init__(f\"Spack command failed with exit code {code}\")\n" }, { "path": "lib/spack/spack/mirrors/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n" }, { "path": "lib/spack/spack/mirrors/layout.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport os\nfrom typing import TYPE_CHECKING, Optional\n\nimport spack.fetch_strategy\nimport spack.llnl.url\nimport spack.oci.image\nimport spack.repo\nfrom spack.error import MirrorError\nfrom spack.llnl.util.filesystem import mkdirp, symlink\n\nif TYPE_CHECKING:\n import spack.spec\n\n\nclass MirrorLayout:\n \"\"\"A ``MirrorLayout`` object describes the relative path of a mirror entry.\"\"\"\n\n def __init__(self, path: str) -> None:\n self.path = path\n\n def __iter__(self):\n \"\"\"Yield all paths including aliases where the resource can be found.\"\"\"\n yield self.path\n\n def make_alias(self, root: str) -> None:\n \"\"\"Make the entry ``root / self.path`` available under a human readable alias\"\"\"\n pass\n\n\nclass DefaultLayout(MirrorLayout):\n def __init__(self, alias_path: str, digest_path: Optional[str] = None) -> None:\n # When we have a digest, it is used as the primary storage location. If not, then we use\n # the human-readable alias. In case of mirrors of a VCS checkout, we currently do not have\n # a digest, that's why an alias is required and a digest optional.\n super().__init__(path=digest_path or alias_path)\n self.alias = alias_path\n self.digest_path = digest_path\n\n def make_alias(self, root: str) -> None:\n \"\"\"Symlink a human readable path in our mirror to the actual storage location.\"\"\"\n # We already use the human-readable path as the main storage location.\n if not self.digest_path:\n return\n\n alias, digest = os.path.join(root, self.alias), os.path.join(root, self.digest_path)\n\n alias_dir = os.path.dirname(alias)\n relative_dst = os.path.relpath(digest, start=alias_dir)\n\n mkdirp(alias_dir)\n tmp = f\"{alias}.tmp\"\n symlink(relative_dst, tmp)\n\n try:\n os.rename(tmp, alias)\n except OSError:\n # Clean up the temporary if possible\n try:\n os.unlink(tmp)\n except OSError:\n pass\n raise\n\n def __iter__(self):\n if self.digest_path:\n yield self.digest_path\n yield self.alias\n\n\nclass OCILayout(MirrorLayout):\n \"\"\"Follow the OCI Image Layout Specification to archive blobs where paths are of the form\n ``blobs//``\"\"\"\n\n def __init__(self, digest: spack.oci.image.Digest) -> None:\n super().__init__(os.path.join(\"blobs\", digest.algorithm, digest.digest))\n\n\ndef _determine_extension(fetcher):\n if isinstance(fetcher, spack.fetch_strategy.URLFetchStrategy):\n if fetcher.expand_archive:\n # If we fetch with a URLFetchStrategy, use URL's archive type\n ext = spack.llnl.url.determine_url_file_extension(fetcher.url)\n\n if ext:\n # Remove any leading dots\n ext = ext.lstrip(\".\")\n else:\n msg = \"\"\"\\\nUnable to parse extension from {0}.\n\nIf this URL is for a tarball but does not include the file extension\nin the name, you can explicitly declare it with the following syntax:\n\n version('1.2.3', 'hash', extension='tar.gz')\n\nIf this URL is for a download like a .jar or .whl that does not need\nto be expanded, or an uncompressed installation script, you can tell\nSpack not to expand it with the following syntax:\n\n version('1.2.3', 'hash', expand=False)\n\"\"\"\n raise MirrorError(msg.format(fetcher.url))\n else:\n # If the archive shouldn't be expanded, don't check extension.\n ext = None\n else:\n # Otherwise we'll make a .tar.gz ourselves\n ext = \"tar.gz\"\n\n return ext\n\n\ndef default_mirror_layout(\n fetcher: \"spack.fetch_strategy.FetchStrategy\",\n per_package_ref: str,\n spec: Optional[\"spack.spec.Spec\"] = None,\n) -> MirrorLayout:\n \"\"\"Returns a ``MirrorReference`` object which keeps track of the relative\n storage path of the resource associated with the specified ``fetcher``.\"\"\"\n ext = None\n if spec:\n pkg_cls = spack.repo.PATH.get_pkg_class(spec.name)\n versions = pkg_cls.versions.get(spec.version, {})\n ext = versions.get(\"extension\", None)\n # If the spec does not explicitly specify an extension (the default case),\n # then try to determine it automatically. An extension can only be\n # specified for the primary source of the package (e.g. the source code\n # identified in the 'version' declaration). Resources/patches don't have\n # an option to specify an extension, so it must be inferred for those.\n ext = ext or _determine_extension(fetcher)\n\n if ext:\n per_package_ref += \".%s\" % ext\n\n global_ref = fetcher.mirror_id()\n if global_ref:\n global_ref = os.path.join(\"_source-cache\", global_ref)\n if global_ref and ext:\n global_ref += \".%s\" % ext\n\n return DefaultLayout(per_package_ref, global_ref)\n" }, { "path": "lib/spack/spack/mirrors/mirror.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport operator\nimport os\nimport urllib.parse\nfrom typing import Any, Dict, List, Mapping, Optional, Tuple, Union\n\nimport spack.config\nimport spack.llnl.util.tty as tty\nimport spack.util.path\nimport spack.util.spack_json as sjson\nimport spack.util.spack_yaml as syaml\nimport spack.util.url as url_util\nfrom spack.error import MirrorError\nfrom spack.oci.image import is_oci_url\n\n#: What schemes do we support\nsupported_url_schemes = (\"file\", \"http\", \"https\", \"sftp\", \"ftp\", \"s3\", \"gs\", \"oci\", \"oci+http\")\n\n#: The layout version spack can current install\nSUPPORTED_LAYOUT_VERSIONS = (3, 2)\n\n\ndef _url_or_path_to_url(url_or_path: str) -> str:\n \"\"\"For simplicity we allow mirror URLs in config files to be local, relative paths.\n This helper function takes care of distinguishing between URLs and paths, and\n canonicalizes paths before transforming them into file:// URLs.\"\"\"\n # Is it a supported URL already? Then don't do path-related canonicalization.\n parsed = urllib.parse.urlparse(url_or_path)\n if parsed.scheme in supported_url_schemes:\n return url_or_path\n\n # Otherwise we interpret it as path, and we should promote it to file:// URL.\n return url_util.path_to_file_url(spack.util.path.canonicalize_path(url_or_path))\n\n\nclass Mirror:\n \"\"\"Represents a named location for storing source tarballs and binary\n packages.\n\n Mirrors have a fetch_url that indicate where and how artifacts are fetched\n from them, and a push_url that indicate where and how artifacts are pushed\n to them. These two URLs are usually the same.\n \"\"\"\n\n def __init__(self, data: Union[str, dict], name: Optional[str] = None):\n self._data = data\n self._name = name\n\n @staticmethod\n def from_yaml(stream, name=None):\n return Mirror(syaml.load(stream), name)\n\n @staticmethod\n def from_json(stream, name=None):\n try:\n return Mirror(sjson.load(stream), name)\n except Exception as e:\n raise sjson.SpackJSONError(\"error parsing JSON mirror:\", str(e)) from e\n\n @staticmethod\n def from_local_path(path: str):\n return Mirror(url_util.path_to_file_url(path))\n\n @staticmethod\n def from_url(url: str):\n \"\"\"Create an anonymous mirror by URL. This method validates the URL.\"\"\"\n if urllib.parse.urlparse(url).scheme not in supported_url_schemes:\n raise ValueError(\n f'\"{url}\" is not a valid mirror URL. '\n f\"Scheme must be one of {supported_url_schemes}.\"\n )\n return Mirror(url)\n\n def __eq__(self, other):\n if not isinstance(other, Mirror):\n return NotImplemented\n return self._data == other._data and self._name == other._name\n\n def __str__(self):\n return f\"{self._name}: {self.push_url} {self.fetch_url}\"\n\n def __repr__(self):\n return f\"Mirror(name={self._name!r}, data={self._data!r})\"\n\n def to_json(self, stream=None):\n return sjson.dump(self.to_dict(), stream)\n\n def to_yaml(self, stream=None):\n return syaml.dump(self.to_dict(), stream)\n\n def to_dict(self):\n return self._data\n\n def display(self, max_len=0):\n fetch, push = self.fetch_url, self.push_url\n # don't print the same URL twice\n url = fetch if fetch == push else f\"fetch: {fetch} push: {push}\"\n source = \"s\" if self.source else \" \"\n binary = \"b\" if self.binary else \" \"\n print(f\"{self.name: <{max_len}} [{source}{binary}] {url}\")\n\n @property\n def name(self):\n return self._name or \"\"\n\n @property\n def binary(self):\n return isinstance(self._data, str) or self._data.get(\"binary\", True)\n\n @property\n def source(self):\n return isinstance(self._data, str) or self._data.get(\"source\", True)\n\n @property\n def signed(self) -> bool:\n # TODO: OCI support signing\n # Only checking for fetch, push is handled by OCI implementation\n if is_oci_url(self.fetch_url):\n return False\n\n return isinstance(self._data, str) or self._data.get(\"signed\", True)\n\n @property\n def autopush(self) -> bool:\n if isinstance(self._data, str):\n return False\n return self._data.get(\"autopush\", False)\n\n @property\n def fetch_url(self):\n \"\"\"Get the valid, canonicalized fetch URL\"\"\"\n return self.get_url(\"fetch\")\n\n @property\n def push_url(self):\n \"\"\"Get the valid, canonicalized fetch URL\"\"\"\n return self.get_url(\"push\")\n\n @property\n def fetch_view(self):\n \"\"\"Get the valid, canonicalized fetch URL\"\"\"\n return self.get_view(\"fetch\")\n\n @property\n def push_view(self):\n \"\"\"Get the valid, canonicalized fetch URL\"\"\"\n return self.get_view(\"push\")\n\n def ensure_mirror_usable(self, direction: str = \"push\"):\n access_pair = self._get_value(\"access_pair\", direction)\n access_token_variable = self._get_value(\"access_token_variable\", direction)\n\n errors = []\n\n # Verify that the credentials that are variables expand\n if access_pair and isinstance(access_pair, dict):\n if \"id_variable\" in access_pair and access_pair[\"id_variable\"] not in os.environ:\n errors.append(f\"id_variable {access_pair['id_variable']} not set in environment\")\n if \"secret_variable\" in access_pair:\n if access_pair[\"secret_variable\"] not in os.environ:\n errors.append(\n f\"environment variable `{access_pair['secret_variable']}` \"\n \"(secret_variable) not set\"\n )\n\n if access_token_variable:\n if access_token_variable not in os.environ:\n errors.append(\n f\"environment variable `{access_pair['access_token_variable']}` \"\n \"(access_token_variable) not set\"\n )\n\n if errors:\n msg = f\"invalid {direction} configuration for mirror {self.name}: \"\n msg += \"\\n \".join(errors)\n raise MirrorError(msg)\n\n @property\n def supported_layout_versions(self) -> List[int]:\n \"\"\"List all of the supported layouts a mirror can fetch from\"\"\"\n # Only check the fetch configuration, the push configuration is whatever the latest\n # mirror version is which should support all configurable features.\n\n # All configured mirrors support the latest version\n supported_versions = [SUPPORTED_LAYOUT_VERSIONS[0]]\n has_view = self.fetch_view is not None\n\n # Check if the mirror supports older layout versions\n # OCI - Only return the newest version, the layout version is a dummy version since OCI\n # has its own layout.\n # Views - Only versions >=3 support the views feature\n if not is_oci_url(self.fetch_url) and not has_view:\n supported_versions.extend(SUPPORTED_LAYOUT_VERSIONS[1:])\n\n return supported_versions\n\n def _update_connection_dict(self, current_data: dict, new_data: dict, top_level: bool):\n # Only allow one to exist in the config\n if \"access_token\" in current_data and \"access_token_variable\" in new_data:\n current_data.pop(\"access_token\")\n elif \"access_token_variable\" in current_data and \"access_token\" in new_data:\n current_data.pop(\"access_token_variable\")\n\n # If updating to a new access_pair that is the deprecated list, warn\n warn_deprecated_access_pair = False\n if \"access_pair\" in new_data:\n warn_deprecated_access_pair = isinstance(new_data[\"access_pair\"], list)\n # If the not updating the current access_pair, and it is the deprecated list, warn\n elif \"access_pair\" in current_data:\n warn_deprecated_access_pair = isinstance(current_data[\"access_pair\"], list)\n\n if warn_deprecated_access_pair:\n tty.warn(\n f\"in mirror {self.name}: support for plain text secrets in config files \"\n \"(access_pair: [id, secret]) is deprecated and will be removed in a future Spack \"\n \"version. Use environment variables instead (access_pair: \"\n \"{id: ..., secret_variable: ...})\"\n )\n\n keys = [\n \"url\",\n \"access_pair\",\n \"access_token\",\n \"access_token_variable\",\n \"profile\",\n \"endpoint_url\",\n ]\n if top_level:\n keys += [\"binary\", \"source\", \"signed\", \"autopush\"]\n changed = False\n for key in keys:\n if key in new_data and current_data.get(key) != new_data[key]:\n current_data[key] = new_data[key]\n changed = True\n return changed\n\n def update(self, data: dict, direction: Optional[str] = None) -> bool:\n \"\"\"Modify the mirror with the given data. This takes care\n of expanding trivial mirror definitions by URL to something more\n rich with a dict if necessary\n\n Args:\n data (dict): The data to update the mirror with.\n direction (str): The direction to update the mirror in (fetch\n or push or None for top-level update)\n\n Returns:\n bool: True if the mirror was updated, False otherwise.\"\"\"\n\n # Modify the top-level entry when no direction is given.\n if not data:\n return False\n\n # If we only update a URL, there's typically no need to expand things to a dict.\n set_url = data[\"url\"] if len(data) == 1 and \"url\" in data else None\n\n if direction is None:\n # First deal with the case where the current top-level entry is just a string.\n if isinstance(self._data, str):\n # Can we replace that string with something new?\n if set_url:\n if self._data == set_url:\n return False\n self._data = set_url\n return True\n\n # Otherwise promote to a dict\n self._data = {\"url\": self._data}\n\n # And update the dictionary accordingly.\n return self._update_connection_dict(self._data, data, top_level=True)\n\n # Otherwise, update the fetch / push entry; turn top-level\n # url string into a dict if necessary.\n if isinstance(self._data, str):\n self._data = {\"url\": self._data}\n\n # Create a new fetch / push entry if necessary\n if direction not in self._data:\n # Keep config minimal if we're just setting the URL.\n if set_url:\n self._data[direction] = set_url\n return True\n self._data[direction] = {}\n\n entry = self._data[direction]\n\n # Keep the entry simple if we're just swapping out the URL.\n if isinstance(entry, str):\n if set_url:\n if entry == set_url:\n return False\n self._data[direction] = set_url\n return True\n\n # Otherwise promote to a dict\n self._data[direction] = {\"url\": entry}\n\n return self._update_connection_dict(self._data[direction], data, top_level=False)\n\n def _get_value(self, attribute: str, direction: str):\n \"\"\"Returns the most specific value for a given attribute (either push/fetch or global)\"\"\"\n if direction not in (\"fetch\", \"push\"):\n raise ValueError(f\"direction must be either 'fetch' or 'push', not {direction}\")\n\n if isinstance(self._data, str):\n return None\n\n # Either a string (url) or a dictionary, we care about the dict here.\n value = self._data.get(direction, {})\n\n # Return top-level entry if only a URL was set.\n if isinstance(value, str) or attribute not in value:\n return self._data.get(attribute)\n\n return value[attribute]\n\n def get_url(self, direction: str) -> str:\n if direction not in (\"fetch\", \"push\"):\n raise ValueError(f\"direction must be either 'fetch' or 'push', not {direction}\")\n\n # Whole mirror config is just a url.\n if isinstance(self._data, str):\n return _url_or_path_to_url(self._data)\n\n # Default value\n url = self._data.get(\"url\")\n\n # Override it with a direction-specific value\n if direction in self._data:\n # Either a url as string or a dict with url key\n info = self._data[direction]\n if isinstance(info, str):\n url = info\n elif \"url\" in info:\n url = info[\"url\"]\n\n if not url:\n raise ValueError(f\"Mirror {self.name} has no URL configured\")\n\n return _url_or_path_to_url(url)\n\n def get_view(self, direction: str):\n return self._get_value(\"view\", direction)\n\n def get_credentials(self, direction: str) -> Dict[str, Any]:\n \"\"\"Get the mirror credentials from the mirror config\n\n Args:\n direction: fetch or push mirror config\n\n Returns:\n Dictionary from credential type string to value\n\n Credential Type Map:\n\n * ``access_token``: ``str``\n * ``access_pair``: ``Tuple[str, str]``\n * ``profile``: ``str``\n \"\"\"\n creddict: Dict[str, Any] = {}\n access_token = self.get_access_token(direction)\n if access_token:\n creddict[\"access_token\"] = access_token\n\n access_pair = self.get_access_pair(direction)\n if access_pair:\n creddict.update({\"access_pair\": access_pair})\n\n profile = self.get_profile(direction)\n if profile:\n creddict[\"profile\"] = profile\n\n return creddict\n\n def get_access_token(self, direction: str) -> Optional[str]:\n tok = self._get_value(\"access_token_variable\", direction)\n if tok:\n return os.environ.get(tok)\n else:\n return self._get_value(\"access_token\", direction)\n return None\n\n def get_access_pair(self, direction: str) -> Optional[Tuple[str, str]]:\n pair = self._get_value(\"access_pair\", direction)\n if isinstance(pair, (tuple, list)) and len(pair) == 2:\n return (pair[0], pair[1]) if all(pair) else None\n elif isinstance(pair, dict):\n id_ = os.environ.get(pair[\"id_variable\"]) if \"id_variable\" in pair else pair[\"id\"]\n secret = os.environ.get(pair[\"secret_variable\"])\n return (id_, secret) if id_ and secret else None\n else:\n return None\n\n def get_profile(self, direction: str) -> Optional[str]:\n return self._get_value(\"profile\", direction)\n\n def get_endpoint_url(self, direction: str) -> Optional[str]:\n return self._get_value(\"endpoint_url\", direction)\n\n\nclass MirrorCollection(Mapping[str, Mirror]):\n \"\"\"A mapping of mirror names to mirrors.\"\"\"\n\n def __init__(\n self,\n mirrors=None,\n scope=None,\n binary: Optional[bool] = None,\n source: Optional[bool] = None,\n autopush: Optional[bool] = None,\n ):\n \"\"\"Initialize a mirror collection.\n\n Args:\n mirrors: A name-to-mirror mapping to initialize the collection with.\n scope: The scope to use when looking up mirrors from the config.\n binary: If True, only include binary mirrors.\n If False, omit binary mirrors.\n If None, do not filter on binary mirrors.\n source: If True, only include source mirrors.\n If False, omit source mirrors.\n If None, do not filter on source mirrors.\n autopush: If True, only include mirrors that have autopush enabled.\n If False, omit mirrors that have autopush enabled.\n If None, do not filter on autopush.\"\"\"\n mirrors_data = (\n mirrors.items()\n if mirrors is not None\n else spack.config.CONFIG.get_config(\"mirrors\", scope=scope).items()\n )\n mirrors = (Mirror(data=mirror, name=name) for name, mirror in mirrors_data)\n\n def _filter(m: Mirror):\n if source is not None and m.source != source:\n return False\n if binary is not None and m.binary != binary:\n return False\n if autopush is not None and m.autopush != autopush:\n return False\n return True\n\n self._mirrors = {m.name: m for m in mirrors if _filter(m)}\n\n def __eq__(self, other):\n return self._mirrors == other._mirrors\n\n def to_json(self, stream=None):\n return sjson.dump(self.to_dict(True), stream)\n\n def to_yaml(self, stream=None):\n return syaml.dump(self.to_dict(True), stream)\n\n # TODO: this isn't called anywhere\n @staticmethod\n def from_yaml(stream, name=None):\n data = syaml.load(stream)\n return MirrorCollection(data)\n\n @staticmethod\n def from_json(stream, name=None):\n try:\n d = sjson.load(stream)\n return MirrorCollection(d)\n except Exception as e:\n raise sjson.SpackJSONError(\"error parsing JSON mirror collection:\", str(e)) from e\n\n def to_dict(self, recursive=False):\n return syaml.syaml_dict(\n sorted(\n ((k, (v.to_dict() if recursive else v)) for (k, v) in self._mirrors.items()),\n key=operator.itemgetter(0),\n )\n )\n\n @staticmethod\n def from_dict(d):\n return MirrorCollection(d)\n\n def __getitem__(self, item):\n return self._mirrors[item]\n\n def display(self):\n max_len = max(len(mirror.name) for mirror in self._mirrors.values())\n for mirror in self._mirrors.values():\n mirror.display(max_len)\n\n def lookup(self, name_or_url):\n \"\"\"Looks up and returns a Mirror.\n\n If this MirrorCollection contains a named Mirror under the name\n [name_or_url], then that mirror is returned. Otherwise, [name_or_url]\n is assumed to be a mirror URL, and an anonymous mirror with the given\n URL is returned.\n \"\"\"\n result = self.get(name_or_url)\n\n if result is None:\n result = Mirror(fetch=name_or_url)\n\n return result\n\n def __iter__(self):\n return iter(self._mirrors)\n\n def __len__(self):\n return len(self._mirrors)\n" }, { "path": "lib/spack/spack/mirrors/utils.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport os\nimport traceback\nfrom collections import Counter\n\nimport spack.caches\nimport spack.config\nimport spack.llnl.util.tty as tty\nimport spack.repo\nimport spack.spec\nimport spack.util.spack_yaml as syaml\nimport spack.version\nfrom spack.error import MirrorError\nfrom spack.llnl.util.filesystem import mkdirp\nfrom spack.mirrors.mirror import Mirror, MirrorCollection\nfrom spack.package import InstallError\n\n\ndef get_all_versions(specs):\n \"\"\"Given a set of initial specs, return a new set of specs that includes\n each version of each package in the original set.\n\n Note that if any spec in the original set specifies properties other than\n version, this information will be omitted in the new set; for example; the\n new set of specs will not include variant settings.\n \"\"\"\n version_specs = []\n for spec in specs:\n pkg_cls = spack.repo.PATH.get_pkg_class(spec.name)\n # Skip any package that has no known versions.\n if not pkg_cls.versions:\n tty.msg(\"No safe (checksummed) versions for package %s\" % pkg_cls.name)\n continue\n\n for version in pkg_cls.versions:\n version_spec = spack.spec.Spec(pkg_cls.name)\n version_spec.versions = spack.version.VersionList([version])\n version_specs.append(version_spec)\n\n return version_specs\n\n\ndef get_matching_versions(specs, num_versions=1):\n \"\"\"Get a spec for EACH known version matching any spec in the list.\n For concrete specs, this retrieves the concrete version and, if more\n than one version per spec is requested, retrieves the latest versions\n of the package.\n \"\"\"\n matching = []\n for spec in specs:\n pkg = spec.package\n\n # Skip any package that has no known versions.\n if not pkg.versions:\n tty.msg(\"No safe (checksummed) versions for package %s\" % pkg.name)\n continue\n\n pkg_versions = num_versions\n\n version_order = list(reversed(sorted(pkg.versions)))\n matching_spec = []\n if spec.concrete:\n matching_spec.append(spec)\n pkg_versions -= 1\n if spec.version in version_order:\n version_order.remove(spec.version)\n\n for v in version_order:\n # Generate no more than num_versions versions for each spec.\n if pkg_versions < 1:\n break\n\n # Generate only versions that satisfy the spec.\n if spec.concrete or v.intersects(spec.versions):\n s = spack.spec.Spec(pkg.name)\n s.versions = spack.version.VersionList([v])\n s.variants = spec.variants.copy()\n # This is needed to avoid hanging references during the\n # concretization phase\n s.variants.spec = s\n matching_spec.append(s)\n pkg_versions -= 1\n\n if not matching_spec:\n tty.warn(\"No known version matches spec: %s\" % spec)\n matching.extend(matching_spec)\n\n return matching\n\n\ndef get_mirror_cache(path, skip_unstable_versions=False):\n \"\"\"Returns a mirror cache, starting from the path where a mirror ought to be created.\n\n Args:\n path (str): path to create a mirror directory hierarchy in.\n skip_unstable_versions: if true, this skips adding resources when\n they do not have a stable archive checksum (as determined by\n ``fetch_strategy.stable_target``).\n\n Returns:\n spack.caches.MirrorCache: mirror cache object for the given path.\n \"\"\"\n # Get the absolute path of the root before we start jumping around.\n if not os.path.isdir(path):\n try:\n mkdirp(path)\n except OSError as e:\n raise MirrorError(\"Cannot create directory '%s':\" % path, str(e))\n mirror_cache = spack.caches.MirrorCache(path, skip_unstable_versions=skip_unstable_versions)\n return mirror_cache\n\n\ndef add(mirror: Mirror, scope=None):\n \"\"\"Add a named mirror in the given scope\"\"\"\n mirrors = spack.config.get(\"mirrors\", scope=scope)\n if not mirrors:\n mirrors = syaml.syaml_dict()\n\n if mirror.name in mirrors:\n tty.die(\"Mirror with name {} already exists.\".format(mirror.name))\n\n items = [(n, u) for n, u in mirrors.items()]\n items.insert(0, (mirror.name, mirror.to_dict()))\n mirrors = syaml.syaml_dict(items)\n spack.config.set(\"mirrors\", mirrors, scope=scope)\n\n\ndef remove(name, scope):\n \"\"\"Remove the named mirror in the given scope\"\"\"\n mirrors = spack.config.get(\"mirrors\", scope=scope)\n if not mirrors:\n mirrors = syaml.syaml_dict()\n\n removed = mirrors.pop(name, False)\n spack.config.set(\"mirrors\", mirrors, scope=scope)\n return bool(removed)\n\n\nclass MirrorStatsForOneSpec:\n def __init__(self, spec):\n self.present = Counter()\n self.new = Counter()\n self.errors = Counter()\n self.spec = spec\n self.added_resources = set()\n self.existing_resources = set()\n\n def finalize(self):\n if self.spec:\n if self.added_resources:\n self.new[self.spec] = len(self.added_resources)\n if self.existing_resources:\n self.present[self.spec] = len(self.existing_resources)\n self.added_resources = set()\n self.existing_resources = set()\n\n def already_existed(self, resource):\n # If an error occurred after caching a subset of a spec's\n # resources, a secondary attempt may consider them already added\n if resource not in self.added_resources:\n self.existing_resources.add(resource)\n\n def added(self, resource):\n self.added_resources.add(resource)\n\n def error(self):\n if self.spec:\n self.errors[self.spec] += 1\n\n\nclass MirrorStatsForAllSpecs:\n def __init__(self):\n # Counter is used to easily merge mirror stats for one spec into mirror stats for all specs\n self.present = Counter()\n self.new = Counter()\n self.errors = Counter()\n\n def merge(self, ext_mirror_stat: MirrorStatsForOneSpec):\n # For the sake of parallelism we need a way to reduce/merge different\n # MirrorStats objects.\n self.present.update(ext_mirror_stat.present)\n self.new.update(ext_mirror_stat.new)\n self.errors.update(ext_mirror_stat.errors)\n\n def stats(self):\n # Convert dictionary to list\n present_list = list(self.present.keys())\n new_list = list(self.new.keys())\n errors_list = list(self.errors.keys())\n\n return present_list, new_list, errors_list\n\n\ndef create_mirror_from_package_object(\n pkg_obj, mirror_cache: \"spack.caches.MirrorCache\", mirror_stats: MirrorStatsForOneSpec\n) -> bool:\n \"\"\"Add a single package object to a mirror.\n\n The package object is only required to have an associated spec\n with a concrete version.\n\n Args:\n pkg_obj (spack.package_base.PackageBase): package object with to be added.\n mirror_cache: mirror where to add the spec.\n mirror_stats: statistics on the current mirror\n\n Return:\n True if the spec was added successfully, False otherwise\n \"\"\"\n tty.msg(\"Adding package {} to mirror\".format(pkg_obj.spec.format(\"{name}{@version}\")))\n # Skip placeholder packages\n try:\n pkg_obj.fetcher\n except InstallError:\n return False\n max_retries = 3\n for num_retries in range(max_retries):\n try:\n # Includes patches and resources\n with pkg_obj.stage as pkg_stage:\n pkg_stage.cache_mirror(mirror_cache, mirror_stats)\n break\n except Exception as e:\n pkg_obj.stage.destroy()\n if num_retries + 1 == max_retries:\n if spack.config.get(\"config:debug\"):\n traceback.print_exc()\n else:\n tty.warn(\n \"Error while fetching %s\" % pkg_obj.spec.format(\"{name}{@version}\"), str(e)\n )\n mirror_stats.error()\n return False\n return True\n\n\ndef require_mirror_name(mirror_name):\n \"\"\"Find a mirror by name and raise if it does not exist\"\"\"\n mirror = MirrorCollection().get(mirror_name)\n if not mirror:\n raise ValueError(f'no mirror named \"{mirror_name}\"')\n return mirror\n" }, { "path": "lib/spack/spack/mixins.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"This module contains additional behavior that can be attached to any given package.\"\"\"\n\nimport os\nfrom typing import Optional\n\nimport spack.llnl.util.filesystem\nimport spack.phase_callbacks\n\n\ndef filter_compiler_wrappers(\n *files: str,\n after: str = \"install\",\n relative_root: Optional[str] = None,\n ignore_absent: bool = True,\n backup: bool = False,\n recursive: bool = False,\n **kwargs, # for compatibility with package api v2.0\n) -> None:\n \"\"\"Registers a phase callback (e.g. post-install) to look for references to Spack's compiler\n wrappers in the given files and replace them with the underlying compilers.\n\n Example usage::\n\n class MyPackage(Package):\n\n filter_compiler_wrappers(\"mpicc\", \"mpicxx\", relative_root=\"bin\")\n\n This is useful for packages that register the path to the compiler they are built with to be\n used later at runtime. Spack's compiler wrappers cannot be used at runtime, as they require\n Spack's build environment to be set up. Using this function, the compiler wrappers are replaced\n with the actual compilers, so that the package works correctly at runtime.\n\n Args:\n *files: files to be filtered relative to the search root (install prefix by default).\n after: specifies after which phase the files should be filtered (defaults to\n ``\"install\"``).\n relative_root: path relative to install prefix where to start searching for the files to be\n filtered. If not set the install prefix will be used as the search root. It is *highly\n recommended* to set this, as searching recursively from the installation prefix can be\n very slow.\n ignore_absent: if present, will be forwarded to\n :func:`~spack.llnl.util.filesystem.filter_file`\n backup: if present, will be forwarded to\n :func:`~spack.llnl.util.filesystem.filter_file`\n recursive: if present, will be forwarded to :func:`~spack.llnl.util.filesystem.find`\n \"\"\"\n\n def _filter_compiler_wrappers_impl(pkg_or_builder):\n pkg = getattr(pkg_or_builder, \"pkg\", pkg_or_builder)\n # Compute the absolute path of the search root\n root = os.path.join(pkg.prefix, relative_root) if relative_root else pkg.prefix\n\n # Compute the absolute path of the files to be filtered and remove links from the list.\n abs_files = spack.llnl.util.filesystem.find(root, files, recursive=recursive)\n abs_files = [x for x in abs_files if not os.path.islink(x)]\n\n x = spack.llnl.util.filesystem.FileFilter(*abs_files)\n\n compiler_vars = []\n if \"c\" in pkg.spec:\n compiler_vars.append((\"CC\", pkg.spec[\"c\"].package.cc))\n\n if \"cxx\" in pkg.spec:\n compiler_vars.append((\"CXX\", pkg.spec[\"cxx\"].package.cxx))\n\n if \"fortran\" in pkg.spec:\n compiler_vars.append((\"FC\", pkg.spec[\"fortran\"].package.fortran))\n compiler_vars.append((\"F77\", pkg.spec[\"fortran\"].package.fortran))\n\n # Some paths to the compiler wrappers might be substrings of the others.\n # For example:\n # CC=/path/to/spack/lib/spack/env/cc (realpath to the wrapper)\n # FC=/path/to/spack/lib/spack/env/cce/ftn\n # Therefore, we perform the filtering in the reversed sorted order of the substituted\n # strings. If, however, the strings are identical (e.g. both CC and FC are set using\n # realpath), the filtering is done according to the order in compiler_vars. To achieve\n # that, we populate the following array with tuples of three elements: path to the wrapper,\n # negated index of the variable in compiler_vars, path to the real compiler. This way, the\n # reversed sorted order of the resulting array is the order of replacements that we need.\n replacements = []\n\n for idx, (env_var, compiler_path) in enumerate(compiler_vars):\n if env_var in os.environ and compiler_path is not None:\n # filter spack wrapper and links to spack wrapper in case\n # build system runs realpath\n wrapper = os.environ[env_var]\n for wrapper_path in (wrapper, os.path.realpath(wrapper)):\n replacements.append((wrapper_path, -idx, compiler_path))\n\n for wrapper_path, _, compiler_path in sorted(replacements, reverse=True):\n x.filter(\n wrapper_path,\n compiler_path,\n ignore_absent=ignore_absent,\n backup=backup,\n string=True,\n )\n\n # Remove this linking flag if present (it turns RPATH into RUNPATH)\n for compiler_lang in (\"c\", \"cxx\", \"fortran\"):\n if compiler_lang not in pkg.spec:\n continue\n compiler_pkg = pkg.spec[compiler_lang].package\n x.filter(\n f\"{compiler_pkg.linker_arg}--enable-new-dtags\",\n \"\",\n ignore_absent=ignore_absent,\n backup=backup,\n string=True,\n )\n\n # NAG compiler is usually mixed with GCC, which has a different\n # prefix for linker arguments.\n if pkg.compiler.name == \"nag\":\n x.filter(\n \"-Wl,--enable-new-dtags\",\n \"\",\n ignore_absent=ignore_absent,\n backup=backup,\n string=True,\n )\n\n spack.phase_callbacks.run_after(after)(_filter_compiler_wrappers_impl)\n" }, { "path": "lib/spack/spack/modules/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"This package contains code for creating environment modules, which can\ninclude Tcl non-hierarchical modules, Lua hierarchical modules, and others.\n\"\"\"\n\nimport os\nfrom typing import Dict, Type\n\nimport spack.llnl.util.tty as tty\nimport spack.repo\nimport spack.spec\nimport spack.store\n\nfrom . import common\nfrom .common import BaseModuleFileWriter, disable_modules\nfrom .lmod import LmodModulefileWriter\nfrom .tcl import TclModulefileWriter\n\n__all__ = [\"TclModulefileWriter\", \"LmodModulefileWriter\", \"disable_modules\"]\n\nmodule_types: Dict[str, Type[BaseModuleFileWriter]] = {\n \"tcl\": TclModulefileWriter,\n \"lmod\": LmodModulefileWriter,\n}\n\n\ndef get_module(\n module_type, spec: spack.spec.Spec, get_full_path, module_set_name=\"default\", required=True\n):\n \"\"\"Retrieve the module file for a given spec and module type.\n\n Retrieve the module file for the given spec if it is available. If the\n module is not available, this will raise an exception unless the module\n is excluded or if the spec is installed upstream.\n\n Args:\n module_type: the type of module we want to retrieve (e.g. lmod)\n spec: refers to the installed package that we want to retrieve a module\n for\n required: if the module is required but excluded, this function will\n print a debug message. If a module is missing but not excluded,\n then an exception is raised (regardless of whether it is required)\n get_full_path: if ``True``, this returns the full path to the module.\n Otherwise, this returns the module name.\n module_set_name: the named module configuration set from modules.yaml\n for which to retrieve the module.\n\n Returns:\n The module name or path. May return ``None`` if the module is not\n available.\n \"\"\"\n try:\n upstream = spec.installed_upstream\n except spack.repo.UnknownPackageError:\n upstream, record = spack.store.STORE.db.query_by_spec_hash(spec.dag_hash())\n if upstream:\n module = common.upstream_module_index.upstream_module(spec, module_type)\n if not module:\n return None\n\n if get_full_path:\n return module.path\n else:\n return module.use_name\n else:\n writer = module_types[module_type](spec, module_set_name)\n if not os.path.isfile(writer.layout.filename):\n fmt_str = \"{name}{@version}{/hash:7}\"\n if not writer.conf.excluded:\n raise common.ModuleNotFoundError(\n \"The module for package {} should be at {}, but it does not exist\".format(\n spec.format(fmt_str), writer.layout.filename\n )\n )\n elif required:\n tty.debug(\n \"The module configuration has excluded {}: omitting it\".format(\n spec.format(fmt_str)\n )\n )\n else:\n return None\n\n if get_full_path:\n return writer.layout.filename\n else:\n return writer.layout.use_name\n" }, { "path": "lib/spack/spack/modules/common.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Here we consolidate the logic for creating an abstract description\nof the information that module systems need.\n\nThis information maps **a single spec** to:\n\n* a unique module filename\n* the module file content\n\nand is divided among four classes:\n\n* a configuration class that provides a convenient interface to query\n details about the configuration for the spec under consideration.\n* a layout class that provides the information associated with module\n file names and directories\n* a context class that provides the dictionary used by the template engine\n to generate the module file\n* a writer that collects and uses the information above to either write\n or remove the module file\n\nEach of the four classes needs to be sub-classed when implementing a new\nmodule type.\n\"\"\"\n\nimport collections\nimport contextlib\nimport copy\nimport datetime\nimport inspect\nimport os\nimport re\nimport string\nfrom typing import List, Optional\n\nimport spack.vendor.jinja2\n\nimport spack.build_environment\nimport spack.config\nimport spack.deptypes as dt\nimport spack.environment\nimport spack.error\nimport spack.llnl.util.filesystem\nimport spack.llnl.util.tty as tty\nimport spack.paths\nimport spack.projections as proj\nimport spack.schema\nimport spack.schema.environment\nimport spack.spec\nimport spack.store\nimport spack.tengine as tengine\nimport spack.user_environment\nimport spack.util.environment\nimport spack.util.file_permissions as fp\nimport spack.util.path\nimport spack.util.spack_yaml as syaml\nfrom spack.context import Context\nfrom spack.llnl.util.lang import Singleton, dedupe, memoized\n\n\n#: config section for this file\ndef configuration(module_set_name):\n config_path = f\"modules:{module_set_name}\"\n return spack.config.get(config_path, {})\n\n\n#: Valid tokens for naming scheme and env variable names\n_valid_tokens = (\n \"name\",\n \"version\",\n \"compiler\",\n \"compiler.name\",\n \"compiler.version\",\n \"architecture\",\n # tokens from old-style format strings\n \"package\",\n \"compilername\",\n \"compilerver\",\n)\n\n\n_FORMAT_STRING_RE = re.compile(r\"({[^}]*})\")\n\n\ndef _format_env_var_name(spec, var_name_fmt):\n \"\"\"Format the variable name, but uppercase any formatted fields.\"\"\"\n fmt_parts = _FORMAT_STRING_RE.split(var_name_fmt)\n return \"\".join(\n spec.format(part).upper() if _FORMAT_STRING_RE.match(part) else part for part in fmt_parts\n )\n\n\ndef _check_tokens_are_valid(format_string, message):\n \"\"\"Checks that the tokens used in the format string are valid in\n the context of module file and environment variable naming.\n\n Args:\n format_string (str): string containing the format to be checked. This\n is supposed to be a 'template' for ``Spec.format``\n\n message (str): first sentence of the error message in case invalid\n tokens are found\n\n \"\"\"\n named_tokens = re.findall(r\"{(\\w*)}\", format_string)\n invalid_tokens = [x for x in named_tokens if x.lower() not in _valid_tokens]\n if invalid_tokens:\n raise RuntimeError(\n f\"{message} [{', '.join(invalid_tokens)}]. \"\n f\"Did you check your 'modules.yaml' configuration?\"\n )\n\n\ndef update_dictionary_extending_lists(target, update):\n \"\"\"Updates a dictionary, but extends lists instead of overriding them.\n\n Args:\n target: dictionary to be updated\n update: update to be applied\n \"\"\"\n for key in update:\n value = target.get(key, None)\n if isinstance(value, list):\n target[key].extend(update[key])\n elif isinstance(value, dict):\n update_dictionary_extending_lists(target[key], update[key])\n else:\n target[key] = update[key]\n\n\ndef dependencies(spec: spack.spec.Spec, request: str = \"all\") -> List[spack.spec.Spec]:\n \"\"\"Returns the list of dependent specs for a given spec.\n\n Args:\n spec: spec to be analyzed\n request: one of ``\"none\"``, ``\"run\"``, ``\"direct\"``, ``\"all\"``\n\n Returns:\n list of requested dependencies\n \"\"\"\n if request == \"none\":\n return []\n elif request == \"run\":\n return spec.dependencies(deptype=dt.RUN)\n elif request == \"direct\":\n return spec.dependencies(deptype=dt.RUN | dt.LINK)\n elif request == \"all\":\n return list(spec.traverse(order=\"topo\", deptype=dt.LINK | dt.RUN, root=False))\n\n raise ValueError(f'request \"{request}\" is not one of \"none\", \"direct\", \"run\", \"all\"')\n\n\ndef merge_config_rules(configuration, spec):\n \"\"\"Parses the module specific part of a configuration and returns a\n dictionary containing the actions to be performed on the spec passed as\n an argument.\n\n Args:\n configuration: module specific configuration (e.g. entries under\n the top-level 'tcl' key)\n spec: spec for which we need to generate a module file\n\n Returns:\n dict: actions to be taken on the spec passed as an argument\n \"\"\"\n # The keyword 'all' is always evaluated first, all the others are\n # evaluated in order of appearance in the module file\n spec_configuration = copy.deepcopy(configuration.get(\"all\", {}))\n for constraint, action in configuration.items():\n if spec.satisfies(constraint):\n if hasattr(constraint, \"override\") and constraint.override:\n spec_configuration = {}\n update_dictionary_extending_lists(spec_configuration, copy.deepcopy(action))\n\n # Transform keywords for dependencies or prerequisites into a list of spec\n\n # Which modulefiles we want to autoload\n autoload_strategy = spec_configuration.get(\"autoload\", \"direct\")\n spec_configuration[\"autoload\"] = dependencies(spec, autoload_strategy)\n\n # Which instead we want to mark as prerequisites\n prerequisite_strategy = spec_configuration.get(\"prerequisites\", \"none\")\n spec_configuration[\"prerequisites\"] = dependencies(spec, prerequisite_strategy)\n\n # Attach options that are spec-independent to the spec-specific\n # configuration\n\n # Hash length in module files\n hash_length = configuration.get(\"hash_length\", 7)\n spec_configuration[\"hash_length\"] = hash_length\n\n verbose = configuration.get(\"verbose\", False)\n spec_configuration[\"verbose\"] = verbose\n\n # module defaults per-package\n defaults = configuration.get(\"defaults\", [])\n spec_configuration[\"defaults\"] = defaults\n\n return spec_configuration\n\n\ndef root_path(name, module_set_name):\n \"\"\"Returns the root folder for module file installation.\n\n Args:\n name: name of the module system to be used (``\"tcl\"`` or ``\"lmod\"``)\n module_set_name: name of the set of module configs to use\n\n Returns:\n root folder for module file installation\n \"\"\"\n defaults = {\"lmod\": \"$spack/share/spack/lmod\", \"tcl\": \"$spack/share/spack/modules\"}\n # Root folders where the various module files should be written\n roots = spack.config.get(f\"modules:{module_set_name}:roots\", {})\n\n # Merge config values into the defaults so we prefer configured values\n roots = spack.schema.merge_yaml(defaults, roots)\n\n path = roots.get(name, os.path.join(spack.paths.share_path, name))\n return spack.util.path.canonicalize_path(path)\n\n\ndef generate_module_index(root, modules, overwrite=False):\n index_path = os.path.join(root, \"module-index.yaml\")\n if overwrite or not os.path.exists(index_path):\n entries = syaml.syaml_dict()\n else:\n with open(index_path, encoding=\"utf-8\") as index_file:\n yaml_content = syaml.load(index_file)\n entries = yaml_content[\"module_index\"]\n\n for m in modules:\n entry = {\"path\": m.layout.filename, \"use_name\": m.layout.use_name}\n entries[m.spec.dag_hash()] = entry\n index = {\"module_index\": entries}\n spack.llnl.util.filesystem.mkdirp(root)\n with open(index_path, \"w\", encoding=\"utf-8\") as index_file:\n syaml.dump(index, default_flow_style=False, stream=index_file)\n\n\ndef _generate_upstream_module_index():\n module_indices = read_module_indices()\n\n return UpstreamModuleIndex(spack.store.STORE.db, module_indices)\n\n\nupstream_module_index = Singleton(_generate_upstream_module_index)\n\n\nModuleIndexEntry = collections.namedtuple(\"ModuleIndexEntry\", [\"path\", \"use_name\"])\n\n\ndef read_module_index(root):\n index_path = os.path.join(root, \"module-index.yaml\")\n if not os.path.exists(index_path):\n return {}\n with open(index_path, encoding=\"utf-8\") as index_file:\n return _read_module_index(index_file)\n\n\ndef _read_module_index(str_or_file):\n \"\"\"Read in the mapping of spec hash to module location/name. For a given\n Spack installation there is assumed to be (at most) one such mapping\n per module type.\"\"\"\n yaml_content = syaml.load(str_or_file)\n index = {}\n yaml_index = yaml_content[\"module_index\"]\n for dag_hash, module_properties in yaml_index.items():\n index[dag_hash] = ModuleIndexEntry(\n module_properties[\"path\"], module_properties[\"use_name\"]\n )\n return index\n\n\ndef read_module_indices():\n other_spack_instances = spack.config.get(\"upstreams\") or {}\n\n module_indices = []\n\n for install_properties in other_spack_instances.values():\n module_type_to_index = {}\n module_type_to_root = install_properties.get(\"modules\", {})\n for module_type, root in module_type_to_root.items():\n module_type_to_index[module_type] = read_module_index(root)\n module_indices.append(module_type_to_index)\n\n return module_indices\n\n\nclass UpstreamModuleIndex:\n \"\"\"This is responsible for taking the individual module indices of all\n upstream Spack installations and locating the module for a given spec\n based on which upstream install it is located in.\"\"\"\n\n def __init__(self, local_db, module_indices):\n self.local_db = local_db\n self.upstream_dbs = local_db.upstream_dbs\n self.module_indices = module_indices\n\n def upstream_module(self, spec, module_type):\n db_for_spec = self.local_db.db_for_spec_hash(spec.dag_hash())\n if db_for_spec in self.upstream_dbs:\n db_index = self.upstream_dbs.index(db_for_spec)\n elif db_for_spec:\n raise spack.error.SpackError(f\"Unexpected: {spec} is installed locally\")\n else:\n raise spack.error.SpackError(f\"Unexpected: no install DB found for {spec}\")\n module_index = self.module_indices[db_index]\n module_type_index = module_index.get(module_type, {})\n if not module_type_index:\n tty.debug(\n f\"No {module_type} modules associated with the Spack instance \"\n f\"where {spec} is installed\"\n )\n return None\n if spec.dag_hash() in module_type_index:\n return module_type_index[spec.dag_hash()]\n else:\n tty.debug(f\"No module is available for upstream package {spec}\")\n return None\n\n\nclass BaseConfiguration:\n \"\"\"Manipulates the information needed to generate a module file to make\n querying easier. It needs to be sub-classed for specific module types.\n \"\"\"\n\n default_projections = {\"all\": \"{name}/{version}-{compiler.name}-{compiler.version}\"}\n\n def __init__(self, spec: spack.spec.Spec, module_set_name: str, explicit: bool) -> None:\n # Spec for which we want to generate a module file\n self.spec = spec\n self.name = module_set_name\n self.explicit = explicit\n # Dictionary of configuration options that should be applied to the spec\n self.conf = merge_config_rules(self.module.configuration(self.name), self.spec)\n\n @property\n def module(self):\n return inspect.getmodule(self)\n\n @property\n def projections(self):\n \"\"\"Projection from specs to module names\"\"\"\n # backwards compatibility for naming_scheme key\n conf = self.module.configuration(self.name)\n if \"naming_scheme\" in conf:\n default = {\"all\": conf[\"naming_scheme\"]}\n else:\n default = self.default_projections\n projections = conf.get(\"projections\", default)\n\n # Ensure the named tokens we are expanding are allowed, see\n # issue #2884 for reference\n msg = \"some tokens cannot be part of the module naming scheme\"\n for projection in projections.values():\n _check_tokens_are_valid(projection, message=msg)\n\n return projections\n\n @property\n def template(self):\n \"\"\"Returns the name of the template to use for the module file\n or None if not specified in the configuration.\n \"\"\"\n return self.conf.get(\"template\", None)\n\n @property\n def defaults(self):\n \"\"\"Returns the specs configured as defaults or [].\"\"\"\n return self.conf.get(\"defaults\", [])\n\n @property\n def env(self):\n \"\"\"List of environment modifications that should be done in the\n module.\n \"\"\"\n return spack.schema.environment.parse(self.conf.get(\"environment\", {}))\n\n @property\n def suffixes(self):\n \"\"\"List of suffixes that should be appended to the module\n file name.\n \"\"\"\n suffixes = []\n for constraint, suffix in self.conf.get(\"suffixes\", {}).items():\n if constraint in self.spec:\n suffixes.append(suffix)\n suffixes = list(dedupe(suffixes))\n # For hidden modules we can always add a fixed length hash as suffix, since it guards\n # against file name clashes, and the module is not exposed to the user anyways.\n if self.hidden:\n suffixes.append(self.spec.dag_hash(length=7))\n elif self.hash:\n suffixes.append(self.hash)\n return suffixes\n\n @property\n def hash(self):\n \"\"\"Hash tag for the module or None\"\"\"\n hash_length = self.conf.get(\"hash_length\", 7)\n if hash_length != 0:\n return self.spec.dag_hash(length=hash_length)\n return None\n\n @property\n def conflicts(self):\n \"\"\"Conflicts for this module file\"\"\"\n return self.conf.get(\"conflict\", [])\n\n @property\n def excluded(self):\n \"\"\"Returns True if the module has been excluded, False otherwise.\"\"\"\n\n # A few variables for convenience of writing the method\n spec = self.spec\n conf = self.module.configuration(self.name)\n\n # Compute the list of matching include / exclude rules, and whether excluded as implicit\n include_matches = [x for x in conf.get(\"include\", []) if spec.satisfies(x)]\n exclude_matches = [x for x in conf.get(\"exclude\", []) if spec.satisfies(x)]\n excluded_as_implicit = not self.explicit and conf.get(\"exclude_implicits\", False)\n\n def debug_info(line_header, match_list):\n if match_list:\n tty.debug(f\"\\t{line_header} : {spec.cshort_spec}\")\n for rule in match_list:\n tty.debug(f\"\\t\\tmatches rule: {rule}\")\n\n debug_info(\"INCLUDE\", include_matches)\n debug_info(\"EXCLUDE\", exclude_matches)\n\n if excluded_as_implicit:\n tty.debug(f\"\\tEXCLUDED_AS_IMPLICIT : {spec.cshort_spec}\")\n\n return not include_matches and (exclude_matches or excluded_as_implicit)\n\n @property\n def hidden(self):\n \"\"\"Returns True if the module has been hidden, False otherwise.\"\"\"\n\n conf = self.module.configuration(self.name)\n\n hidden_as_implicit = not self.explicit and conf.get(\"hide_implicits\", False)\n\n if hidden_as_implicit:\n tty.debug(f\"\\tHIDDEN_AS_IMPLICIT : {self.spec.cshort_spec}\")\n\n return hidden_as_implicit\n\n @property\n def context(self):\n return self.conf.get(\"context\", {})\n\n @property\n def specs_to_load(self):\n \"\"\"List of specs that should be loaded in the module file.\"\"\"\n return self._create_list_for(\"autoload\")\n\n @property\n def literals_to_load(self):\n \"\"\"List of literal modules to be loaded.\"\"\"\n return self.conf.get(\"load\", [])\n\n @property\n def specs_to_prereq(self):\n \"\"\"List of specs that should be prerequisite of the module file.\"\"\"\n return self._create_list_for(\"prerequisites\")\n\n @property\n def exclude_env_vars(self):\n \"\"\"List of variables that should be left unmodified.\"\"\"\n filter_subsection = self.conf.get(\"filter\", {})\n return filter_subsection.get(\"exclude_env_vars\", {})\n\n def _create_list_for(self, what):\n include = []\n for item in self.conf[what]:\n if not self.module.make_configuration(item, self.name).excluded:\n include.append(item)\n return include\n\n @property\n def verbose(self):\n \"\"\"Returns True if the module file needs to be verbose, False\n otherwise\n \"\"\"\n return self.conf.get(\"verbose\")\n\n\nclass BaseFileLayout:\n \"\"\"Provides information on the layout of module files. Needs to be\n sub-classed for specific module types.\n \"\"\"\n\n #: This needs to be redefined\n extension: Optional[str] = None\n\n def __init__(self, configuration):\n self.conf = configuration\n\n @property\n def spec(self):\n \"\"\"Spec under consideration\"\"\"\n return self.conf.spec\n\n def dirname(self):\n \"\"\"Root folder for module files of this type.\"\"\"\n module_system = str(self.conf.module.__name__).split(\".\")[-1]\n return root_path(module_system, self.conf.name)\n\n @property\n def use_name(self):\n \"\"\"Returns the 'use' name of the module i.e. the name you have to type\n to console to use it. This implementation fits the needs of most\n non-hierarchical layouts.\n \"\"\"\n projection = proj.get_projection(self.conf.projections, self.spec)\n if not projection:\n projection = self.conf.default_projections[\"all\"]\n\n name = self.spec.format_path(projection)\n # Not everybody is working on linux...\n parts = name.split(\"/\")\n name = os.path.join(*parts)\n # Add optional suffixes based on constraints\n path_elements = [name]\n path_elements.extend(map(self.spec.format, self.conf.suffixes))\n return \"-\".join(path_elements)\n\n @property\n def filename(self):\n \"\"\"Name of the module file for the current spec.\"\"\"\n # Just the name of the file\n filename = self.use_name\n if self.extension:\n filename = f\"{self.use_name}.{self.extension}\"\n # Architecture sub-folder\n arch_folder_conf = spack.config.get(\"modules:%s:arch_folder\" % self.conf.name, True)\n if arch_folder_conf:\n # include an arch specific folder between root and filename\n arch_folder = str(self.spec.architecture)\n filename = os.path.join(arch_folder, filename)\n # Return the absolute path\n return os.path.join(self.dirname(), filename)\n\n\nclass BaseContext(tengine.Context):\n \"\"\"Provides the base context needed for template rendering.\n\n This class needs to be sub-classed for specific module types. The\n following attributes need to be implemented:\n\n - fields\n\n \"\"\"\n\n def __init__(self, configuration):\n self.conf = configuration\n\n @tengine.context_property\n def spec(self):\n return self.conf.spec\n\n @tengine.context_property\n def tags(self):\n if not hasattr(self.spec.package, \"tags\"):\n return []\n return self.spec.package.tags\n\n @tengine.context_property\n def timestamp(self):\n return datetime.datetime.now()\n\n @tengine.context_property\n def category(self):\n return getattr(self.spec, \"category\", \"spack\")\n\n @tengine.context_property\n def short_description(self):\n # If we have a valid docstring return the first paragraph.\n docstring = type(self.spec.package).__doc__\n if docstring:\n value = docstring.split(\"\\n\\n\")[0]\n # Transform tabs and friends into spaces\n value = re.sub(r\"\\s+\", \" \", value)\n # Turn double quotes into single quotes (double quotes are needed\n # to start and end strings)\n value = re.sub(r'\"', \"'\", value)\n return value\n # Otherwise the short description is just the package + version\n return self.spec.format(\"{name} {@version}\")\n\n @tengine.context_property\n def long_description(self):\n # long description is the docstring with reduced whitespace.\n if self.spec.package.__doc__:\n return re.sub(r\"\\s+\", \" \", self.spec.package.__doc__)\n return None\n\n @tengine.context_property\n def configure_options(self):\n pkg = self.spec.package\n\n # If the spec is external Spack doesn't know its configure options\n if self.spec.external:\n msg = \"unknown, software installed outside of Spack\"\n return msg\n\n if os.path.exists(pkg.install_configure_args_path):\n with open(pkg.install_configure_args_path, encoding=\"utf-8\") as args_file:\n return spack.util.path.padding_filter(args_file.read())\n\n # Returning a false-like value makes the default templates skip\n # the configure option section\n return None\n\n def modification_needs_formatting(self, modification):\n \"\"\"Returns True if environment modification entry needs to be formatted.\"\"\"\n return (\n not isinstance(modification, (spack.util.environment.SetEnv)) or not modification.raw\n )\n\n @tengine.context_property\n @memoized\n def environment_modifications(self):\n \"\"\"List of environment modifications to be processed.\"\"\"\n # Modifications guessed by inspecting the spec prefix\n prefix_inspections = syaml.syaml_dict()\n spack.schema.merge_yaml(\n prefix_inspections, spack.config.get(\"modules:prefix_inspections\", {})\n )\n spack.schema.merge_yaml(\n prefix_inspections,\n spack.config.get(f\"modules:{self.conf.name}:prefix_inspections\", {}),\n )\n\n use_view = spack.config.get(f\"modules:{self.conf.name}:use_view\", False)\n\n assert isinstance(use_view, (bool, str))\n\n if use_view:\n env = spack.environment.active_environment()\n if not env:\n raise spack.environment.SpackEnvironmentViewError(\n \"Module generation with views requires active environment\"\n )\n\n view_name = spack.environment.default_view_name if use_view is True else use_view\n\n if not env.has_view(view_name):\n raise spack.environment.SpackEnvironmentViewError(\n f\"View {view_name} not found in environment {env.name} when generating modules\"\n )\n\n view = env.views[view_name]\n else:\n view = None\n\n env = spack.util.environment.inspect_path(\n self.spec.prefix, prefix_inspections, exclude=spack.util.environment.is_system_path\n )\n\n # Let the extendee/dependency modify their extensions/dependencies\n\n # The only thing we care about is `setup_dependent_run_environment`, but\n # for that to work, globals have to be set on the package modules, and the\n # whole chain of setup_dependent_package has to be followed from leaf to spec.\n # So: just run it here, but don't collect env mods.\n spack.build_environment.SetupContext(\n self.spec, context=Context.RUN\n ).set_all_package_py_globals()\n\n # Then run setup_dependent_run_environment before setup_run_environment.\n for dep in self.spec.dependencies(deptype=(\"link\", \"run\")):\n dep.package.setup_dependent_run_environment(env, self.spec)\n self.spec.package.setup_run_environment(env)\n\n # Project the environment variables from prefix to view if needed\n if view and self.spec in view:\n spack.user_environment.project_env_mods(\n *self.spec.traverse(deptype=dt.LINK | dt.RUN), view=view, env=env\n )\n\n # Modifications required from modules.yaml\n env.extend(self.conf.env)\n\n # List of variables that are excluded in modules.yaml\n exclude = self.conf.exclude_env_vars\n\n # We may have tokens to substitute in environment commands\n for x in env:\n # Ensure all the tokens are valid in this context\n msg = \"some tokens cannot be expanded in an environment variable name\"\n\n _check_tokens_are_valid(x.name, message=msg)\n x.name = _format_env_var_name(self.spec, x.name)\n if self.modification_needs_formatting(x):\n try:\n # Not every command has a value\n x.value = self.spec.format(x.value)\n except AttributeError:\n pass\n x.name = str(x.name).replace(\"-\", \"_\")\n\n return [(type(x).__name__, x) for x in env if x.name not in exclude]\n\n @tengine.context_property\n def has_manpath_modifications(self):\n \"\"\"True if MANPATH environment variable is modified.\"\"\"\n for modification_type, cmd in self.environment_modifications:\n if not isinstance(\n cmd, (spack.util.environment.PrependPath, spack.util.environment.AppendPath)\n ):\n continue\n if cmd.name == \"MANPATH\":\n return True\n else:\n return False\n\n @tengine.context_property\n def conflicts(self):\n \"\"\"List of conflicts for the module file.\"\"\"\n fmts = []\n projection = proj.get_projection(self.conf.projections, self.spec)\n for item in self.conf.conflicts:\n self._verify_conflict_naming_consistency_or_raise(item, projection)\n item = self.spec.format(item)\n fmts.append(item)\n return fmts\n\n def _verify_conflict_naming_consistency_or_raise(self, item, projection):\n f = string.Formatter()\n errors = []\n if len([x for x in f.parse(item)]) > 1:\n for naming_dir, conflict_dir in zip(projection.split(\"/\"), item.split(\"/\")):\n if naming_dir != conflict_dir:\n errors.extend(\n [\n f\"spec={self.spec.cshort_spec}\",\n f\"conflict_scheme={item}\",\n f\"naming_scheme={projection}\",\n ]\n )\n if errors:\n raise ModulesError(\n message=\"conflict scheme does not match naming scheme\",\n long_message=\"\\n \".join(errors),\n )\n\n @tengine.context_property\n def autoload(self):\n \"\"\"List of modules that needs to be loaded automatically.\"\"\"\n # From 'autoload' configuration option\n specs = self._create_module_list_of(\"specs_to_load\")\n # From 'load' configuration option\n literals = self.conf.literals_to_load\n return specs + literals\n\n def _create_module_list_of(self, what):\n m = self.conf.module\n name = self.conf.name\n return [m.make_layout(x, name).use_name for x in getattr(self.conf, what)]\n\n @tengine.context_property\n def verbose(self):\n \"\"\"Verbosity level.\"\"\"\n return self.conf.verbose\n\n\nclass BaseModuleFileWriter:\n default_template: str\n hide_cmd_format: str\n modulerc_header: List[str]\n\n def __init__(\n self, spec: spack.spec.Spec, module_set_name: str, explicit: Optional[bool] = None\n ) -> None:\n self.spec = spec\n\n m = self.module\n\n # Create the triplet of configuration/layout/context\n self.conf = m.make_configuration(spec, module_set_name, explicit)\n self.layout = m.make_layout(spec, module_set_name, explicit)\n self.context = m.make_context(spec, module_set_name, explicit)\n\n # Check if a default template has been defined,\n # throw if not found\n try:\n self.default_template\n except AttributeError:\n msg = \"'{0}' object has no attribute 'default_template'\\n\"\n msg += \"Did you forget to define it in the class?\"\n name = type(self).__name__\n raise DefaultTemplateNotDefined(msg.format(name))\n\n # Check if format for module hide command has been defined,\n # throw if not found\n try:\n self.hide_cmd_format\n except AttributeError:\n msg = \"'{0}' object has no attribute 'hide_cmd_format'\\n\"\n msg += \"Did you forget to define it in the class?\"\n name = type(self).__name__\n raise HideCmdFormatNotDefined(msg.format(name))\n\n # Check if modulerc header content has been defined,\n # throw if not found\n try:\n self.modulerc_header\n except AttributeError:\n msg = \"'{0}' object has no attribute 'modulerc_header'\\n\"\n msg += \"Did you forget to define it in the class?\"\n name = type(self).__name__\n raise ModulercHeaderNotDefined(msg.format(name))\n\n @property\n def module(self):\n return inspect.getmodule(self)\n\n def _get_template(self):\n \"\"\"Gets the template that will be rendered for this spec.\"\"\"\n # Get templates and put them in the order of importance:\n # 1. template specified in \"modules.yaml\"\n # 2. template specified in a package directly\n # 3. default template (must be defined, check in __init__)\n module_system_name = str(self.module.__name__).split(\".\")[-1]\n package_attribute = f\"{module_system_name}_template\"\n choices = [\n self.conf.template,\n getattr(self.spec.package, package_attribute, None),\n self.default_template, # This is always defined at this point\n ]\n # Filter out false-ish values\n choices = list(filter(lambda x: bool(x), choices))\n # ... and return the first match\n return choices.pop(0)\n\n def write(self, overwrite=False):\n \"\"\"Writes the module file.\n\n Args:\n overwrite (bool): if True it is fine to overwrite an already\n existing file. If False the operation is skipped an we print\n a warning to the user.\n \"\"\"\n # Return immediately if the module is excluded\n if self.conf.excluded:\n msg = \"\\tNOT WRITING: {0} [EXCLUDED]\"\n tty.debug(msg.format(self.spec.cshort_spec))\n return\n\n # Print a warning in case I am accidentally overwriting\n # a module file that is already there (name clash)\n if not overwrite and os.path.exists(self.layout.filename):\n message = \"Module file {0.filename} exists and will not be overwritten\"\n tty.warn(message.format(self.layout))\n return\n\n # If we are here it means it's ok to write the module file\n msg = \"\\tWRITE: {0} [{1}]\"\n tty.debug(msg.format(self.spec.cshort_spec, self.layout.filename))\n\n # If the directory where the module should reside does not exist\n # create it\n module_dir = os.path.dirname(self.layout.filename)\n if not os.path.exists(module_dir):\n spack.llnl.util.filesystem.mkdirp(module_dir)\n\n # Get the template for the module\n template_name = self._get_template()\n\n try:\n env = tengine.make_environment()\n template = env.get_template(template_name)\n except spack.vendor.jinja2.TemplateNotFound:\n # If the template was not found raise an exception with a little\n # more information\n msg = \"template '{0}' was not found for '{1}'\"\n name = type(self).__name__\n msg = msg.format(template_name, name)\n raise ModulesTemplateNotFoundError(msg)\n\n # Construct the context following the usual hierarchy of updates:\n # 1. start with the default context from the module writer class\n # 2. update with package specific context\n # 3. update with 'modules.yaml' specific context\n\n context = self.context.to_dict()\n\n # Attribute from package\n module_name = str(self.module.__name__).split(\".\")[-1]\n attr_name = f\"{module_name}_context\"\n pkg_update = getattr(self.spec.package, attr_name, {})\n context.update(pkg_update)\n\n # Context key in modules.yaml\n conf_update = self.conf.context\n context.update(conf_update)\n\n # Render the template\n text = template.render(context)\n # Write it to file\n with open(self.layout.filename, \"w\", encoding=\"utf-8\") as f:\n f.write(text)\n\n # Set the file permissions of the module to match that of the package\n if os.path.exists(self.layout.filename):\n fp.set_permissions_by_spec(self.layout.filename, self.spec)\n\n # Symlink defaults if needed\n self.update_module_defaults()\n\n # record module hiddenness if implicit\n self.update_module_hiddenness()\n\n def update_module_defaults(self):\n if any(self.spec.satisfies(default) for default in self.conf.defaults):\n # This spec matches a default, it needs to be symlinked to default\n # Symlink to a tmp location first and move, so that existing\n # symlinks do not cause an error.\n default_path = os.path.join(os.path.dirname(self.layout.filename), \"default\")\n default_tmp = os.path.join(os.path.dirname(self.layout.filename), \".tmp_spack_default\")\n os.symlink(self.layout.filename, default_tmp)\n os.rename(default_tmp, default_path)\n\n def update_module_hiddenness(self, remove=False):\n \"\"\"Update modulerc file corresponding to module to add or remove\n command that hides module depending on its hidden state.\n\n Args:\n remove (bool): if True, hiddenness information for module is\n removed from modulerc.\n \"\"\"\n modulerc_path = self.layout.modulerc\n hide_module_cmd = self.hide_cmd_format % self.layout.use_name\n hidden = self.conf.hidden and not remove\n modulerc_exists = os.path.exists(modulerc_path)\n updated = False\n\n if modulerc_exists:\n # retrieve modulerc content\n with open(modulerc_path, encoding=\"utf-8\") as f:\n content = f.readlines()\n content = \"\".join(content).split(\"\\n\")\n # remove last empty item if any\n if len(content[-1]) == 0:\n del content[-1]\n already_hidden = hide_module_cmd in content\n\n # remove hide command if module not hidden\n if already_hidden and not hidden:\n content.remove(hide_module_cmd)\n updated = True\n\n # add hide command if module is hidden\n elif not already_hidden and hidden:\n if len(content) == 0:\n content = self.modulerc_header.copy()\n content.append(hide_module_cmd)\n updated = True\n else:\n content = self.modulerc_header.copy()\n if hidden:\n content.append(hide_module_cmd)\n updated = True\n\n # no modulerc file change if no content update\n if updated:\n is_empty = content == self.modulerc_header or len(content) == 0\n # remove existing modulerc if empty\n if modulerc_exists and is_empty:\n os.remove(modulerc_path)\n # create or update modulerc\n elif content != self.modulerc_header:\n # ensure file ends with a newline character\n content.append(\"\")\n with open(modulerc_path, \"w\", encoding=\"utf-8\") as f:\n f.write(\"\\n\".join(content))\n\n def remove(self):\n \"\"\"Deletes the module file.\"\"\"\n mod_file = self.layout.filename\n if os.path.exists(mod_file):\n try:\n os.remove(mod_file) # Remove the module file\n self.remove_module_defaults() # Remove default targeting module file\n self.update_module_hiddenness(remove=True) # Remove hide cmd in modulerc\n os.removedirs(\n os.path.dirname(mod_file)\n ) # Remove all the empty directories from the leaf up\n except OSError:\n # removedirs throws OSError on first non-empty directory found\n pass\n\n def remove_module_defaults(self):\n if not any(self.spec.satisfies(default) for default in self.conf.defaults):\n return\n\n # This spec matches a default, symlink needs to be removed as we remove the module\n # file it targets.\n default_symlink = os.path.join(os.path.dirname(self.layout.filename), \"default\")\n try:\n os.unlink(default_symlink)\n except OSError:\n pass\n\n\n@contextlib.contextmanager\ndef disable_modules():\n \"\"\"Disable the generation of modulefiles within the context manager.\"\"\"\n data = {\"modules:\": {\"default\": {\"enable\": []}}}\n disable_scope = spack.config.InternalConfigScope(\"disable_modules\", data=data)\n with spack.config.override(disable_scope):\n yield\n\n\nclass ModulesError(spack.error.SpackError):\n \"\"\"Base error for modules.\"\"\"\n\n\nclass ModuleNotFoundError(ModulesError):\n \"\"\"Raised when a module cannot be found for a spec\"\"\"\n\n\nclass DefaultTemplateNotDefined(AttributeError, ModulesError):\n \"\"\"Raised if the attribute ``default_template`` has not been specified\n in the derived classes.\n \"\"\"\n\n\nclass HideCmdFormatNotDefined(AttributeError, ModulesError):\n \"\"\"Raised if the attribute ``hide_cmd_format`` has not been specified\n in the derived classes.\n \"\"\"\n\n\nclass ModulercHeaderNotDefined(AttributeError, ModulesError):\n \"\"\"Raised if the attribute ``modulerc_header`` has not been specified\n in the derived classes.\n \"\"\"\n\n\nclass ModulesTemplateNotFoundError(ModulesError, RuntimeError):\n \"\"\"Raised if the template for a module file was not found.\"\"\"\n" }, { "path": "lib/spack/spack/modules/lmod.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport collections\nimport itertools\nimport os\nimport pathlib\nimport warnings\nfrom typing import Dict, List, Optional, Tuple\n\nimport spack.compilers.config\nimport spack.config\nimport spack.error\nimport spack.llnl.util.filesystem as fs\nimport spack.llnl.util.lang as lang\nimport spack.spec\nimport spack.tengine as tengine\nimport spack.util.environment\nfrom spack.aliases import BUILTIN_TO_LEGACY_COMPILER\n\nfrom .common import BaseConfiguration, BaseContext, BaseFileLayout, BaseModuleFileWriter\n\n\n#: lmod specific part of the configuration\ndef configuration(module_set_name: str) -> dict:\n return spack.config.get(f\"modules:{module_set_name}:lmod\", {})\n\n\n# Caches the configuration {spec_hash: configuration}\nconfiguration_registry: Dict[Tuple[str, str, bool], BaseConfiguration] = {}\n\n\ndef make_configuration(\n spec: spack.spec.Spec, module_set_name: str, explicit: Optional[bool] = None\n) -> BaseConfiguration:\n \"\"\"Returns the lmod configuration for spec\"\"\"\n explicit = bool(spec._installed_explicitly()) if explicit is None else explicit\n key = (spec.dag_hash(), module_set_name, explicit)\n try:\n return configuration_registry[key]\n except KeyError:\n return configuration_registry.setdefault(\n key, LmodConfiguration(spec, module_set_name, explicit)\n )\n\n\ndef make_layout(\n spec: spack.spec.Spec, module_set_name: str, explicit: Optional[bool] = None\n) -> BaseFileLayout:\n \"\"\"Returns the layout information for spec\"\"\"\n return LmodFileLayout(make_configuration(spec, module_set_name, explicit))\n\n\ndef make_context(\n spec: spack.spec.Spec, module_set_name: str, explicit: Optional[bool] = None\n) -> BaseContext:\n \"\"\"Returns the context information for spec\"\"\"\n return LmodContext(make_configuration(spec, module_set_name, explicit))\n\n\ndef guess_core_compilers(name, store=False) -> List[spack.spec.Spec]:\n \"\"\"Guesses the list of core compilers installed in the system.\n\n Args:\n store (bool): if True writes the core compilers to the\n modules.yaml configuration file\n\n Returns:\n List of found core compilers\n \"\"\"\n core_compilers = []\n for compiler in spack.compilers.config.all_compilers(init_config=False):\n try:\n cc_dir = pathlib.Path(compiler.package.cc).parent\n is_system_compiler = str(cc_dir) in spack.util.environment.SYSTEM_DIRS\n if is_system_compiler:\n core_compilers.append(compiler)\n except (KeyError, TypeError, AttributeError):\n continue\n\n if store and core_compilers:\n # If we asked to store core compilers, update the entry\n # in the default modify scope (i.e. within the directory hierarchy\n # of Spack itself)\n modules_cfg = spack.config.get(\n \"modules:\" + name, {}, scope=spack.config.default_modify_scope()\n )\n modules_cfg.setdefault(\"lmod\", {})[\"core_compilers\"] = [str(x) for x in core_compilers]\n spack.config.set(\"modules:\" + name, modules_cfg, scope=spack.config.default_modify_scope())\n\n return core_compilers\n\n\nclass LmodConfiguration(BaseConfiguration):\n \"\"\"Configuration class for lmod module files.\"\"\"\n\n default_projections = {\"all\": \"{name}/{version}\"}\n\n compiler: Optional[spack.spec.Spec]\n\n def __init__(self, spec: spack.spec.Spec, module_set_name: str, explicit: bool) -> None:\n super().__init__(spec, module_set_name, explicit)\n\n candidates = collections.defaultdict(list)\n language_virtuals = (\"c\", \"cxx\", \"fortran\")\n\n for node in spec.traverse(deptype=(\"link\", \"run\")):\n for language in language_virtuals:\n candidates[language].extend(node.dependencies(virtuals=(language,)))\n\n self.compiler = None\n\n for language in language_virtuals:\n if candidates[language]:\n self.compiler = candidates[language][0]\n if len(set(candidates[language])) > 1:\n warnings.warn(\n f\"{spec.short_spec} uses more than one compiler, and might not fit the \"\n f\"LMod hierarchy. Using {self.compiler.short_spec} as the LMod compiler.\"\n )\n break\n\n @property\n def core_compilers(self) -> List[spack.spec.Spec]:\n \"\"\"Returns the list of \"Core\" compilers\n\n Raises:\n CoreCompilersNotFoundError: if the key was not specified in the configuration file or\n the sequence is empty\n \"\"\"\n compilers = []\n for c in configuration(self.name).get(\"core_compilers\", []):\n compilers.extend(spack.spec.Spec(f\"%{c}\").dependencies())\n\n if not compilers:\n compilers = guess_core_compilers(self.name, store=True)\n\n if not compilers:\n msg = 'the key \"core_compilers\" must be set in modules.yaml'\n raise CoreCompilersNotFoundError(msg)\n\n return compilers\n\n @property\n def core_specs(self):\n \"\"\"Returns the list of \"Core\" specs\"\"\"\n return configuration(self.name).get(\"core_specs\", [])\n\n @property\n def filter_hierarchy_specs(self):\n \"\"\"Returns the dict of specs with modified hierarchies\"\"\"\n return configuration(self.name).get(\"filter_hierarchy_specs\", {})\n\n @property\n @lang.memoized\n def hierarchy_tokens(self):\n \"\"\"Returns the list of tokens that are part of the modulefile\n hierarchy. ``compiler`` is always present.\n \"\"\"\n tokens = configuration(self.name).get(\"hierarchy\", [])\n\n # Append 'compiler' which is always implied\n tokens.append(\"compiler\")\n\n # Deduplicate tokens in case duplicates have been coded\n tokens = list(lang.dedupe(tokens))\n\n return tokens\n\n @property\n @lang.memoized\n def requires(self):\n \"\"\"Returns a dictionary mapping all the requirements of this spec to the actual provider.\n\n The ``compiler`` key is always present among the requirements.\n \"\"\"\n # If it's a core_spec, lie and say it requires a core compiler\n if any(self.spec.satisfies(core_spec) for core_spec in self.core_specs):\n return {\"compiler\": self.core_compilers[0]}\n\n hierarchy_filter_list = []\n for spec, filter_list in self.filter_hierarchy_specs.items():\n if self.spec.satisfies(spec):\n hierarchy_filter_list = filter_list\n break\n\n # Keep track of the requirements that this package has in terms\n # of virtual packages that participate in the hierarchical structure\n requirements = {\"compiler\": self.compiler or self.core_compilers[0]}\n\n # For each dependency in the hierarchy\n for x in self.hierarchy_tokens:\n # Skip anything filtered for this spec\n if x in hierarchy_filter_list:\n continue\n\n # If I depend on it\n if x in self.spec and not (self.spec.name == x or self.spec.package.provides(x)):\n requirements[x] = self.spec[x] # record the actual provider\n\n return requirements\n\n @property\n def provides(self):\n \"\"\"Returns a dictionary mapping all the services provided by this\n spec to the spec itself.\n \"\"\"\n provides = {}\n\n # Treat the 'compiler' case in a special way, as compilers are not\n # virtual dependencies in spack\n\n # If it is in the list of supported compilers family -> compiler\n if self.spec.name in spack.compilers.config.supported_compilers():\n provides[\"compiler\"] = spack.spec.Spec(self.spec.format(\"{name}{@versions}\"))\n elif self.spec.name in BUILTIN_TO_LEGACY_COMPILER:\n # If it is the package for a supported compiler, but of a different name\n cname = BUILTIN_TO_LEGACY_COMPILER[self.spec.name]\n provides[\"compiler\"] = spack.spec.Spec(cname, self.spec.versions)\n\n # All the other tokens in the hierarchy must be virtual dependencies\n for x in self.hierarchy_tokens:\n if self.spec.name == x or self.spec.package.provides(x):\n provides[x] = self.spec\n return provides\n\n @property\n def available(self):\n \"\"\"Returns a dictionary of the services that are currently\n available.\n \"\"\"\n available = {}\n # What is available is what I require plus what I provide.\n # 'compiler' is the only key that may be overridden.\n available.update(self.requires)\n available.update(self.provides)\n return available\n\n @property\n @lang.memoized\n def missing(self):\n \"\"\"Returns the list of tokens that are not available.\"\"\"\n return [x for x in self.hierarchy_tokens if x not in self.available]\n\n @property\n def hidden(self):\n # Never hide a module that opens a hierarchy\n if any(\n self.spec.name == x or self.spec.package.provides(x) for x in self.hierarchy_tokens\n ):\n return False\n return super().hidden\n\n\nclass LmodFileLayout(BaseFileLayout):\n \"\"\"File layout for lmod module files.\"\"\"\n\n #: file extension of lua module files\n extension = \"lua\"\n\n @property\n def arch_dirname(self):\n \"\"\"Returns the root folder for THIS architecture\"\"\"\n # Architecture sub-folder\n arch_folder_conf = spack.config.get(\"modules:%s:arch_folder\" % self.conf.name, True)\n if arch_folder_conf:\n # include an arch specific folder between root and filename\n arch_folder = \"-\".join(\n [str(self.spec.platform), str(self.spec.os), str(self.spec.target.family)]\n )\n return os.path.join(self.dirname(), arch_folder)\n return self.dirname()\n\n @property\n def filename(self):\n \"\"\"Returns the filename for the current module file\"\"\"\n\n # Get the list of requirements and build an **ordered**\n # list of the path parts\n requires = self.conf.requires\n hierarchy = self.conf.hierarchy_tokens\n path_parts = lambda x: self.token_to_path(x, requires[x])\n parts = [path_parts(x) for x in hierarchy if x in requires]\n\n # My relative path if just a join of all the parts\n hierarchy_name = os.path.join(*parts)\n\n # Compute the absolute path\n return os.path.join(\n self.arch_dirname, # root for lmod files on this architecture\n hierarchy_name, # relative path\n f\"{self.use_name}.{self.extension}\", # file name\n )\n\n @property\n def modulerc(self):\n \"\"\"Returns the modulerc file associated with current module file\"\"\"\n return os.path.join(os.path.dirname(self.filename), f\".modulerc.{self.extension}\")\n\n def token_to_path(self, name, value):\n \"\"\"Transforms a hierarchy token into the corresponding path part.\n\n Args:\n name (str): name of the service in the hierarchy\n value: actual provider of the service\n\n Returns:\n str: part of the path associated with the service\n \"\"\"\n\n # General format for the path part\n def path_part_fmt(token):\n return fs.polite_path([f\"{token.name}\", f\"{token.version}\"])\n\n # If we are dealing with a core compiler, return 'Core'\n core_compilers = self.conf.core_compilers\n if name == \"compiler\" and any(spack.spec.Spec(value).satisfies(c) for c in core_compilers):\n return \"Core\"\n\n # Spec does not have a hash, as we are not allowed to\n # use different flavors of the same compiler\n if name == \"compiler\":\n return path_part_fmt(token=value)\n\n # In case the hierarchy token refers to a virtual provider\n # we need to append a hash to the version to distinguish\n # among flavors of the same library (e.g. openblas~openmp vs.\n # openblas+openmp)\n return f\"{path_part_fmt(token=value)}-{value.dag_hash(length=7)}\"\n\n @property\n def available_path_parts(self):\n \"\"\"List of path parts that are currently available. Needed to\n construct the file name.\n \"\"\"\n # List of available services\n available = self.conf.available\n # List of services that are part of the hierarchy\n hierarchy = self.conf.hierarchy_tokens\n # Tokenize each part that is both in the hierarchy and available\n return [self.token_to_path(x, available[x]) for x in hierarchy if x in available]\n\n @property\n @lang.memoized\n def unlocked_paths(self):\n \"\"\"Returns a dictionary mapping conditions to a list of unlocked\n paths.\n\n The paths that are unconditionally unlocked are under the\n key 'None'. The other keys represent the list of services you need\n loaded to unlock the corresponding paths.\n \"\"\"\n\n unlocked = collections.defaultdict(list)\n\n # Get the list of services we require and we provide\n requires_key = list(self.conf.requires)\n provides_key = list(self.conf.provides)\n\n # A compiler is always required. To avoid duplication pop the\n # 'compiler' item from required if we also **provide** one\n if \"compiler\" in provides_key:\n requires_key.remove(\"compiler\")\n\n # Compute the unique combinations of the services we provide\n combinations = []\n for ii in range(len(provides_key)):\n combinations += itertools.combinations(provides_key, ii + 1)\n\n # Attach the services required to each combination\n to_be_processed = [x + tuple(requires_key) for x in combinations]\n\n # Compute the paths that are unconditionally added\n # and append them to the dictionary (key = None)\n available_combination = []\n for item in to_be_processed:\n hierarchy = self.conf.hierarchy_tokens\n available = self.conf.available\n ac = [x for x in hierarchy if x in item]\n available_combination.append(tuple(ac))\n parts = [self.token_to_path(x, available[x]) for x in ac]\n unlocked[None].append(tuple([self.arch_dirname] + parts))\n\n # Deduplicate the list\n unlocked[None] = list(lang.dedupe(unlocked[None]))\n\n # Compute the combination of missing requirements: this will lead to\n # paths that are unlocked conditionally\n missing = self.conf.missing\n\n missing_combinations = []\n for ii in range(len(missing)):\n missing_combinations += itertools.combinations(missing, ii + 1)\n\n # Attach the services required to each combination\n for m in missing_combinations:\n to_be_processed = [m + x for x in available_combination]\n for item in to_be_processed:\n hierarchy = self.conf.hierarchy_tokens\n available = self.conf.available\n token2path = lambda x: self.token_to_path(x, available[x])\n parts = []\n for x in hierarchy:\n if x not in item:\n continue\n value = token2path(x) if x in available else x\n parts.append(value)\n unlocked[m].append(tuple([self.arch_dirname] + parts))\n # Deduplicate the list\n unlocked[m] = list(lang.dedupe(unlocked[m]))\n return unlocked\n\n\nclass LmodContext(BaseContext):\n \"\"\"Context class for lmod module files.\"\"\"\n\n @tengine.context_property\n def has_modulepath_modifications(self):\n \"\"\"True if this module modifies MODULEPATH, False otherwise.\"\"\"\n return bool(self.conf.provides)\n\n @tengine.context_property\n def has_conditional_modifications(self):\n \"\"\"True if this module modifies MODULEPATH conditionally to the\n presence of other services in the environment, False otherwise.\n \"\"\"\n # In general we have conditional modifications if we have modifications\n # and we are not providing **only** a compiler\n provides = self.conf.provides\n provide_compiler_only = \"compiler\" in provides and len(provides) == 1\n has_modifications = self.has_modulepath_modifications\n return has_modifications and not provide_compiler_only\n\n @tengine.context_property\n def name_part(self):\n \"\"\"Name of this provider.\"\"\"\n return self.spec.name\n\n @tengine.context_property\n def version_part(self):\n \"\"\"Version of this provider.\"\"\"\n s = self.spec\n return \"-\".join([str(s.version), s.dag_hash(length=7)])\n\n @tengine.context_property\n def provides(self):\n \"\"\"Returns the dictionary of provided services.\"\"\"\n return self.conf.provides\n\n @tengine.context_property\n def missing(self):\n \"\"\"Returns a list of missing services.\"\"\"\n return self.conf.missing\n\n @tengine.context_property\n @lang.memoized\n def unlocked_paths(self):\n \"\"\"Returns the list of paths that are unlocked unconditionally.\"\"\"\n layout = make_layout(self.spec, self.conf.name)\n return [os.path.join(*parts) for parts in layout.unlocked_paths[None]]\n\n @tengine.context_property\n def conditionally_unlocked_paths(self):\n \"\"\"Returns the list of paths that are unlocked conditionally.\n Each item in the list is a tuple with the structure (condition, path).\n \"\"\"\n layout = make_layout(self.spec, self.conf.name)\n value = []\n conditional_paths = layout.unlocked_paths\n conditional_paths.pop(None)\n for services_needed, list_of_path_parts in conditional_paths.items():\n condition = \" and \".join([x + \"_name\" for x in services_needed])\n for parts in list_of_path_parts:\n\n def manipulate_path(token):\n if token in self.conf.hierarchy_tokens:\n return \"{0}_name, {0}_version\".format(token)\n return '\"' + token + '\"'\n\n path = \", \".join([manipulate_path(x) for x in parts])\n\n value.append((condition, path))\n return value\n\n\nclass LmodModulefileWriter(BaseModuleFileWriter):\n \"\"\"Writer class for lmod module files.\"\"\"\n\n default_template = \"modules/modulefile.lua\"\n\n modulerc_header = []\n\n hide_cmd_format = 'hide_version(\"%s\")'\n\n\nclass CoreCompilersNotFoundError(spack.error.SpackError, KeyError):\n \"\"\"Error raised if the key ``core_compilers`` has not been specified\n in the configuration file.\n \"\"\"\n" }, { "path": "lib/spack/spack/modules/tcl.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"This module implements the classes necessary to generate Tcl\nnon-hierarchical modules.\n\"\"\"\n\nimport os\nfrom typing import Dict, Optional, Tuple\n\nimport spack.config\nimport spack.spec\nimport spack.tengine as tengine\n\nfrom .common import BaseConfiguration, BaseContext, BaseFileLayout, BaseModuleFileWriter\n\n\n#: Tcl specific part of the configuration\ndef configuration(module_set_name: str) -> dict:\n return spack.config.get(f\"modules:{module_set_name}:tcl\", {})\n\n\n# Caches the configuration {spec_hash: configuration}\nconfiguration_registry: Dict[Tuple[str, str, bool], BaseConfiguration] = {}\n\n\ndef make_configuration(\n spec: spack.spec.Spec, module_set_name: str, explicit: Optional[bool] = None\n) -> BaseConfiguration:\n \"\"\"Returns the tcl configuration for spec\"\"\"\n explicit = bool(spec._installed_explicitly()) if explicit is None else explicit\n key = (spec.dag_hash(), module_set_name, explicit)\n try:\n return configuration_registry[key]\n except KeyError:\n return configuration_registry.setdefault(\n key, TclConfiguration(spec, module_set_name, explicit)\n )\n\n\ndef make_layout(\n spec: spack.spec.Spec, module_set_name: str, explicit: Optional[bool] = None\n) -> BaseFileLayout:\n \"\"\"Returns the layout information for spec\"\"\"\n return TclFileLayout(make_configuration(spec, module_set_name, explicit))\n\n\ndef make_context(\n spec: spack.spec.Spec, module_set_name: str, explicit: Optional[bool] = None\n) -> BaseContext:\n \"\"\"Returns the context information for spec\"\"\"\n return TclContext(make_configuration(spec, module_set_name, explicit))\n\n\nclass TclConfiguration(BaseConfiguration):\n \"\"\"Configuration class for tcl module files.\"\"\"\n\n\nclass TclFileLayout(BaseFileLayout):\n \"\"\"File layout for tcl module files.\"\"\"\n\n @property\n def modulerc(self):\n \"\"\"Returns the modulerc file associated with current module file\"\"\"\n return os.path.join(os.path.dirname(self.filename), \".modulerc\")\n\n\nclass TclContext(BaseContext):\n \"\"\"Context class for tcl module files.\"\"\"\n\n @tengine.context_property\n def prerequisites(self):\n \"\"\"List of modules that needs to be loaded automatically.\"\"\"\n return self._create_module_list_of(\"specs_to_prereq\")\n\n\nclass TclModulefileWriter(BaseModuleFileWriter):\n \"\"\"Writer class for tcl module files.\"\"\"\n\n default_template = \"modules/modulefile.tcl\"\n\n modulerc_header = [\"#%Module4.7\"]\n\n hide_cmd_format = \"module-hide --soft --hidden-loaded %s\"\n" }, { "path": "lib/spack/spack/multimethod.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"This module contains utilities for using multi-methods in\nspack. You can think of multi-methods like overloaded methods --\nthey're methods with the same name, and we need to select a version\nof the method based on some criteria. e.g., for overloaded\nmethods, you would select a version of the method to call based on\nthe types of its arguments.\n\nIn spack, multi-methods are used to ease the life of package\nauthors. They allow methods like install() (or other methods\ncalled by install()) to declare multiple versions to be called when\nthe package is instantiated with different specs. e.g., if the\npackage is built with OpenMPI on x86_64,, you might want to call a\ndifferent install method than if it was built for mpich2 on\nBlueGene/Q. Likewise, you might want to do a different type of\ninstall for different versions of the package.\n\nMulti-methods provide a simple decorator-based syntax for this that\navoids overly complicated rat nests of if statements. Obviously,\ndepending on the scenario, regular old conditionals might be clearer,\nso package authors should use their judgement.\n\"\"\"\n\nimport functools\nfrom contextlib import contextmanager\nfrom typing import Optional, Union\n\nimport spack.directives_meta\nimport spack.error\nimport spack.spec\n\n\nclass MultiMethodMeta(type):\n \"\"\"This allows us to track the class's dict during instantiation.\"\"\"\n\n #: saved dictionary of attrs on the class being constructed\n _locals = None\n\n @classmethod\n def __prepare__(cls, name, bases, **kwargs):\n \"\"\"Save the dictionary that will be used for the class namespace.\"\"\"\n MultiMethodMeta._locals = dict()\n return MultiMethodMeta._locals\n\n def __init__(cls, name, bases, attr_dict):\n \"\"\"Clear out the cached locals dict once the class is built.\"\"\"\n MultiMethodMeta._locals = None\n super(MultiMethodMeta, cls).__init__(name, bases, attr_dict)\n\n\nclass SpecMultiMethod:\n \"\"\"This implements a multi-method for Spack specs. Packages are\n instantiated with a particular spec, and you may want to\n execute different versions of methods based on what the spec\n looks like. For example, you might want to call a different\n version of install() for one platform than you call on another.\n\n The SpecMultiMethod class implements a callable object that\n handles method dispatch. When it is called, it looks through\n registered methods and their associated specs, and it tries\n to find one that matches the package's spec. If it finds one\n (and only one), it will call that method.\n\n This is intended for use with decorators (see below). The\n decorator (see docs below) creates SpecMultiMethods and\n registers method versions with them.\n\n To register a method, you can do something like this::\n\n mm = SpecMultiMethod()\n mm.register(\"^chaos_5_x86_64_ib\", some_method)\n\n The object registered needs to be a Spec or some string that\n will parse to be a valid spec.\n\n When the ``mm`` is actually called, it selects a version of the\n method to call based on the ``sys_type`` of the object it is\n called on.\n\n See the docs for decorators below for more details.\n \"\"\"\n\n def __init__(self, default=None):\n self.method_list = []\n self.default = default\n if default:\n functools.update_wrapper(self, default)\n\n def register(self, spec, method):\n \"\"\"Register a version of a method for a particular spec.\"\"\"\n self.method_list.append((spec, method))\n\n if not hasattr(self, \"__name__\"):\n functools.update_wrapper(self, method)\n else:\n assert self.__name__ == method.__name__\n\n def __get__(self, obj, objtype):\n \"\"\"This makes __call__ support instance methods.\"\"\"\n # Method_list is a list of tuples (constraint, method)\n # Here we are going to assume that we have at least one\n # element in the list. The first registered function\n # will be the one 'wrapped'.\n wrapped_method = self.method_list[0][1]\n\n # Call functools.wraps manually to get all the attributes\n # we need to be disguised as the wrapped_method\n func = functools.wraps(wrapped_method)(functools.partial(self.__call__, obj))\n return func\n\n def _get_method_by_spec(self, spec):\n \"\"\"Find the method of this SpecMultiMethod object that satisfies the\n given spec, if one exists\n \"\"\"\n for condition, method in self.method_list:\n if spec.satisfies(condition):\n return method\n return self.default or None\n\n def __call__(self, package_or_builder_self, *args, **kwargs):\n \"\"\"Find the first method with a spec that matches the\n package's spec. If none is found, call the default\n or if there is none, then raise a NoSuchMethodError.\n \"\"\"\n spec_method = self._get_method_by_spec(package_or_builder_self.spec)\n if spec_method:\n return spec_method(package_or_builder_self, *args, **kwargs)\n # Unwrap the MRO of `package_self by hand. Note that we can't\n # use `super()` here, because using `super()` recursively\n # requires us to know the class of `package_self`, as well as\n # its superclasses for successive calls. We don't have that\n # information within `SpecMultiMethod`, because it is not\n # associated with the package class.\n for cls in package_or_builder_self.__class__.__mro__[1:]:\n superself = cls.__dict__.get(self.__name__, None)\n\n if isinstance(superself, SpecMultiMethod):\n # Check parent multimethod for method for spec.\n superself_method = superself._get_method_by_spec(package_or_builder_self.spec)\n if superself_method:\n return superself_method(package_or_builder_self, *args, **kwargs)\n elif superself:\n return superself(package_or_builder_self, *args, **kwargs)\n\n raise NoSuchMethodError(\n type(package_or_builder_self),\n self.__name__,\n package_or_builder_self.spec,\n [m[0] for m in self.method_list],\n )\n\n\nclass when:\n \"\"\"This is a multi-purpose class, which can be used\n\n 1. As a context manager to **group directives together** that share the same ``when=``\n argument.\n 2. As a **decorator** for defining multi-methods (multiple methods with the same name are\n defined, but the version that is called depends on the condition of the package's spec)\n\n As a **context manager** it groups directives together. It allows you to write::\n\n with when(\"+nvptx\"):\n conflicts(\"@:6\", msg=\"NVPTX only supported from gcc 7\")\n conflicts(\"languages=ada\")\n conflicts(\"languages=brig\")\n\n instead of the more repetitive::\n\n conflicts(\"@:6\", when=\"+nvptx\", msg=\"NVPTX only supported from gcc 7\")\n conflicts(\"languages=ada\", when=\"+nvptx\")\n conflicts(\"languages=brig\", when=\"+nvptx\")\n\n This context manager is composable both with nested ``when`` contexts and with other ``when=``\n arguments in directives. For example::\n\n with when(\"+foo\"):\n with when(\"+bar\"):\n depends_on(\"dependency\", when=\"+baz\")\n\n is equilavent to::\n\n depends_on(\"dependency\", when=\"+foo +bar +baz\")\n\n As a **decorator**, it allows packages to declare multiple versions of methods like\n ``install()`` that depend on the package's spec. For example::\n\n class SomePackage(Package):\n ...\n\n def install(self, spec: Spec, prefix: Prefix):\n # Do default install\n\n @when(\"target=x86_64:\")\n def install(self, spec: Spec, prefix: Prefix):\n # This will be executed instead of the default install if\n # the package's target is in the x86_64 family.\n\n @when(\"target=aarch64:\")\n def install(self, spec: Spec, prefix: Prefix):\n # This will be executed if the package's target is in\n # the aarch64 family\n\n This allows each package to have a default version of ``install()`` AND\n specialized versions for particular platforms. The version that is\n called depends on the architecture of the instantiated package.\n\n Note that this works for methods other than install, as well. So,\n if you only have part of the install that is platform specific, you\n could do this:\n\n .. code-block:: python\n\n class SomePackage(Package):\n ...\n # virtual dependence on MPI.\n # could resolve to mpich, mpich2, OpenMPI\n depends_on(\"mpi\")\n\n def setup(self):\n # do nothing in the default case\n pass\n\n @when(\"^openmpi\")\n def setup(self):\n # do something special when this is built with OpenMPI for its MPI implementations.\n pass\n\n def install(self, prefix):\n # Do common install stuff\n self.setup()\n # Do more common install stuff\n\n Note that the default version of decorated methods must *always* come first. Otherwise it will\n override all of the decorated versions. This is a limitation of the Python language.\n \"\"\"\n\n spec: Optional[spack.spec.Spec]\n\n def __init__(self, condition: Union[str, bool]):\n \"\"\"Can be used both as a decorator, for multimethods, or as a context\n manager to group ``when=`` arguments together.\n\n\n Args:\n condition (str): condition to be met\n \"\"\"\n self.when = condition\n\n def __call__(self, method):\n assert MultiMethodMeta._locals is not None, (\n \"cannot use multimethod, missing MultiMethodMeta metaclass?\"\n )\n\n # Create a multimethod with this name if there is not one already\n original_method = MultiMethodMeta._locals.get(method.__name__)\n if not isinstance(original_method, SpecMultiMethod):\n original_method = SpecMultiMethod(original_method)\n\n if self.when is True:\n original_method.register(spack.spec.EMPTY_SPEC, method)\n elif self.when is not False:\n original_method.register(spack.directives_meta.get_spec(self.when), method)\n\n return original_method\n\n def __enter__(self):\n # TODO: support when=False.\n if isinstance(self.when, str):\n spack.directives_meta.DirectiveMeta.push_when_constraint(self.when)\n\n def __exit__(self, exc_type, exc_val, exc_tb):\n if isinstance(self.when, str):\n spack.directives_meta.DirectiveMeta.pop_when_constraint()\n\n\n@contextmanager\ndef default_args(**kwargs):\n \"\"\"Context manager to override the default arguments of directives.\n\n Example::\n\n with default_args(type=(\"build\", \"run\")):\n depends_on(\"py-foo\")\n depends_on(\"py-bar\")\n depends_on(\"py-baz\")\n\n Notice that unlike then :func:`when` context manager, this one is *not* composable, as it\n merely overrides the default argument values for the duration of the context. For example::\n\n with default_args(when=\"+foo\"):\n depends_on(\"pkg-a\")\n depends_on(\"pkg-b\", when=\"+bar\")\n\n is equivalent to::\n\n depends_on(\"pkg-a\", when=\"+foo\")\n depends_on(\"pkg-b\", when=\"+bar\")\n \"\"\"\n spack.directives_meta.DirectiveMeta.push_default_args(kwargs)\n yield\n spack.directives_meta.DirectiveMeta.pop_default_args()\n\n\nclass MultiMethodError(spack.error.SpackError):\n \"\"\"Superclass for multimethod dispatch errors\"\"\"\n\n def __init__(self, message):\n super().__init__(message)\n\n\nclass NoSuchMethodError(spack.error.SpackError):\n \"\"\"Raised when we can't find a version of a multi-method.\"\"\"\n\n def __init__(self, cls, method_name, spec, possible_specs):\n super().__init__(\n \"Package %s does not support %s called with %s. Options are: %s\"\n % (cls.__name__, method_name, spec, \", \".join(str(s) for s in possible_specs))\n )\n" }, { "path": "lib/spack/spack/new_installer.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"New installer that will ultimately replace installer.py. It features an event loop, non-blocking\nI/O, and a POSIX jobserver to limit concurrency. It also has a more advanced terminal UI. It's\nmostly self-contained to avoid interfering with the rest of Spack too much while it's being\ndeveloped and tested.\n\nThe installer consists of a UI process that manages multiple build processes and handles updates\nto the database. It detects or creates a jobserver, and then kicks off an event loop in which it\nruns through a build queue, always running at least one build. Concurrent builds run as jobserver\ntokens are obtained. This means only one -j flag is needed to control concurrency.\n\nThe UI process has two modes: an overview mode where it shows the status of all builds, and a\nmode where it follows the logs of a specific build. It listens to keyboard input to switch between\nmodes.\n\nThe build process does an ordinary install, but also spawns a \"tee\" thread that forwards its build\noutput to both a log file and the UI process (if the UI process has requested it). This thread also\nruns an event loop to listen for control messages from the UI process (to enable/disable echoing\nof logs), and for output from the build process.\"\"\"\n\nimport codecs\nimport fcntl\nimport glob\nimport io\nimport json\nimport multiprocessing\nimport os\nimport re\nimport selectors\nimport shlex\nimport shutil\nimport signal\nimport sys\nimport tempfile\nimport termios\nimport threading\nimport time\nimport traceback\nimport tty\nimport warnings\nfrom gzip import GzipFile\nfrom multiprocessing import Pipe, Process\nfrom multiprocessing.connection import Connection\nfrom typing import (\n TYPE_CHECKING,\n Callable,\n Dict,\n FrozenSet,\n Generator,\n List,\n NamedTuple,\n Optional,\n Set,\n Tuple,\n Union,\n)\n\nfrom spack.vendor.typing_extensions import Literal\n\nimport spack.binary_distribution\nimport spack.build_environment\nimport spack.builder\nimport spack.config\nimport spack.database\nimport spack.deptypes as dt\nimport spack.error\nimport spack.hooks\nimport spack.llnl.util.filesystem as fs\nimport spack.llnl.util.tty\nimport spack.llnl.util.tty.color\nimport spack.paths\nimport spack.report\nimport spack.spec\nimport spack.stage\nimport spack.store\nimport spack.subprocess_context\nimport spack.traverse\nimport spack.url_buildcache\nimport spack.util.environment\nimport spack.util.lock\nfrom spack.installer import _do_fake_install, dump_packages\nfrom spack.llnl.util.lang import pretty_duration\nfrom spack.llnl.util.tty.log import _is_background_tty, ignore_signal\nfrom spack.util.path import padding_filter, padding_filter_bytes\n\nif TYPE_CHECKING:\n import spack.package_base\n\n#: Type for specifying installation source modes\nInstallPolicy = Literal[\"auto\", \"cache_only\", \"source_only\"]\n\n#: How often to update a spinner in seconds\nSPINNER_INTERVAL = 0.1\n\n#: How often to wake up in headless mode to check for background->foreground transition (seconds)\nHEADLESS_WAKE_INTERVAL = 1.0\n\n#: How long to display finished packages before graying them out\nCLEANUP_TIMEOUT = 2.0\n\n#: How often to flush completed builds to the database\nDATABASE_WRITE_INTERVAL = 5.0\n\n#: Size of the output buffer for child processes\nOUTPUT_BUFFER_SIZE = 32768\n\n#: Suffix for temporary backup during overwrite install\nOVERWRITE_BACKUP_SUFFIX = \".old\"\n\n#: Suffix for temporary cleanup during failed install\nOVERWRITE_GARBAGE_SUFFIX = \".garbage\"\n\n#: Exit code used by the child process to signal that the build was stopped at a phase boundary\nEXIT_STOPPED_AT_PHASE = 3\n\n\nclass DatabaseAction:\n \"\"\"Base class for objects that need to be persisted to the database.\"\"\"\n\n __slots__ = (\"spec\", \"prefix_lock\")\n\n spec: \"spack.spec.Spec\"\n prefix_lock: Optional[spack.util.lock.Lock]\n\n def save_to_db(self, db: spack.database.Database) -> None: ...\n\n def release_lock(self) -> None:\n if self.prefix_lock is not None:\n try:\n self.prefix_lock.release_write()\n except Exception:\n pass\n self.prefix_lock = None\n\n\nclass MarkExplicitAction(DatabaseAction):\n \"\"\"Action to mark an already installed spec as explicitly installed. Similar to ChildInfo, but\n used when no build process was needed.\"\"\"\n\n __slots__ = ()\n\n def __init__(self, spec: \"spack.spec.Spec\") -> None:\n self.spec = spec\n self.prefix_lock = None\n\n def save_to_db(self, db: spack.database.Database) -> None:\n db._mark(self.spec, \"explicit\", True)\n\n\nclass ChildInfo(DatabaseAction):\n \"\"\"Information about a child process.\"\"\"\n\n __slots__ = (\"proc\", \"output_r_conn\", \"state_r_conn\", \"control_w_conn\", \"explicit\", \"log_path\")\n\n def __init__(\n self,\n proc: Process,\n spec: spack.spec.Spec,\n output_r_conn: Connection,\n state_r_conn: Connection,\n control_w_conn: Connection,\n log_path: str,\n explicit: bool = False,\n ) -> None:\n self.proc = proc\n self.spec = spec\n self.output_r_conn = output_r_conn\n self.state_r_conn = state_r_conn\n self.control_w_conn = control_w_conn\n self.log_path = log_path\n self.explicit = explicit\n self.prefix_lock: Optional[spack.util.lock.Lock] = None\n\n def save_to_db(self, db: spack.database.Database) -> None:\n return db._add(self.spec, explicit=self.explicit)\n\n def cleanup(self, selector: selectors.BaseSelector) -> None:\n \"\"\"Unregister and close file descriptors, and join the child process.\"\"\"\n try:\n selector.unregister(self.output_r_conn.fileno())\n except KeyError:\n pass\n try:\n selector.unregister(self.state_r_conn.fileno())\n except KeyError:\n pass\n try:\n selector.unregister(self.proc.sentinel)\n except (KeyError, ValueError):\n pass\n self.output_r_conn.close()\n self.state_r_conn.close()\n self.control_w_conn.close()\n self.proc.join()\n\n\ndef send_state(state: str, state_pipe: io.TextIOWrapper) -> None:\n \"\"\"Send a state update message.\"\"\"\n json.dump({\"state\": state}, state_pipe, separators=(\",\", \":\"))\n state_pipe.write(\"\\n\")\n\n\ndef send_progress(current: int, total: int, state_pipe: io.TextIOWrapper) -> None:\n \"\"\"Send a progress update message.\"\"\"\n json.dump({\"progress\": current, \"total\": total}, state_pipe, separators=(\",\", \":\"))\n state_pipe.write(\"\\n\")\n\n\ndef send_installed_from_binary_cache(state_pipe: io.TextIOWrapper) -> None:\n \"\"\"Send a notification that the package was installed from binary cache.\"\"\"\n json.dump({\"installed_from_binary_cache\": True}, state_pipe, separators=(\",\", \":\"))\n state_pipe.write(\"\\n\")\n\n\ndef tee(control_r: int, log_r: int, log_path: str, parent_w: int) -> None:\n \"\"\"Forward log_r to file_w and parent_w (if echoing is enabled).\n Echoing is enabled and disabled by reading from control_r.\"\"\"\n echo_on = False\n selector = selectors.DefaultSelector()\n selector.register(log_r, selectors.EVENT_READ)\n selector.register(control_r, selectors.EVENT_READ)\n\n try:\n with open(log_path, \"wb\") as log_file, open(parent_w, \"wb\", closefd=False) as parent:\n while True:\n for key, _ in selector.select():\n if key.fd == log_r:\n data = os.read(log_r, OUTPUT_BUFFER_SIZE)\n if not data: # EOF: exit the thread\n return\n log_file.write(data)\n log_file.flush()\n if echo_on:\n parent.write(data)\n parent.flush()\n\n elif key.fd == control_r:\n control_data = os.read(control_r, 1)\n if not control_data:\n return\n else:\n echo_on = control_data == b\"1\"\n except OSError: # do not raise\n pass\n finally:\n os.close(log_r)\n\n\nclass Tee:\n \"\"\"Emulates ./build 2>&1 | tee build.log. The output is sent both to a log file and the parent\n process (if echoing is enabled). The control_fd is used to enable/disable echoing.\"\"\"\n\n def __init__(self, control: Connection, parent: Connection, log_path: str) -> None:\n self.control = control\n self.parent = parent\n # sys.stdout and sys.stderr may have been replaced with file objects under pytest, so\n # redirect their file descriptors in addition to the original fds 1 and 2.\n fds = {sys.stdout.fileno(), sys.stderr.fileno(), 1, 2}\n self.saved_fds = {fd: os.dup(fd) for fd in fds}\n #: The path of the log file\n self.log_path = log_path\n r, w = os.pipe()\n self.tee_thread = threading.Thread(\n target=tee,\n args=(self.control.fileno(), r, self.log_path, self.parent.fileno()),\n daemon=True,\n )\n self.tee_thread.start()\n for fd in fds:\n os.dup2(w, fd)\n os.close(w)\n\n def close(self) -> None:\n # Closing stdout and stderr should close the last reference to the write end of the pipe,\n # causing the tee thread to wake up, flush the last data, and exit. We restore stdout and\n # stderr, because between sys.exit and the actual process exit buffers may be flushed, and\n # can cause exit code 120 (witnessed under pytest+coverage on macOS).\n sys.stdout.flush()\n sys.stderr.flush()\n for fd, saved_fd in self.saved_fds.items():\n os.dup2(saved_fd, fd)\n os.close(saved_fd)\n self.tee_thread.join()\n # Only then close the other fds.\n self.control.close()\n self.parent.close()\n\n\ndef install_from_buildcache(\n mirrors: List[spack.url_buildcache.MirrorMetadata],\n spec: spack.spec.Spec,\n unsigned: Optional[bool],\n state_stream: io.TextIOWrapper,\n) -> bool:\n send_state(\"fetching from build cache\", state_stream)\n try:\n tarball_stage = spack.binary_distribution.download_tarball(\n spec.build_spec, unsigned, mirrors\n )\n except spack.binary_distribution.NoConfiguredBinaryMirrors:\n return False\n\n if tarball_stage is None:\n return False\n\n send_state(\"relocating\", state_stream)\n spack.binary_distribution.extract_tarball(spec, tarball_stage, force=False)\n\n if spec.spliced: # overwrite old metadata with new\n spack.store.STORE.layout.write_spec(spec, spack.store.STORE.layout.spec_file_path(spec))\n\n # now a block of curious things follow that should be fixed.\n pkg = spec.package\n if hasattr(pkg, \"_post_buildcache_install_hook\"):\n pkg._post_buildcache_install_hook()\n pkg.installed_from_binary_cache = True\n\n # inform also the parent that this package was installed from binary cache.\n send_installed_from_binary_cache(state_stream)\n\n return True\n\n\nclass GlobalState:\n \"\"\"Global state needed in a build subprocess. This is similar to spack.subprocess_context,\n but excludes the Spack environment, which is slow to serialize and should not be needed\n during the build.\"\"\"\n\n __slots__ = (\"store\", \"config\", \"monkey_patches\", \"spack_working_dir\", \"repo_cache\")\n\n def __init__(self):\n if multiprocessing.get_start_method() == \"fork\":\n return\n self.config = spack.config.CONFIG.ensure_unwrapped()\n self.store = spack.store.STORE\n self.monkey_patches = spack.subprocess_context.TestPatches.create()\n self.spack_working_dir = spack.paths.spack_working_dir\n\n def restore(self):\n if multiprocessing.get_start_method() == \"fork\":\n # In the forking case we must erase SSL contexts.\n from spack.oci import opener\n from spack.util import web\n from spack.util.s3 import s3_client_cache\n\n web.urlopen._instance = None\n opener.urlopen._instance = None\n s3_client_cache.clear()\n return\n spack.store.STORE = self.store\n spack.config.CONFIG = self.config\n self.monkey_patches.restore()\n spack.paths.spack_working_dir = self.spack_working_dir\n\n\nclass PrefixPivoter:\n \"\"\"Manages the installation prefix of a build.\"\"\"\n\n def __init__(self, prefix: str, keep_prefix: bool = False) -> None:\n \"\"\"Initialize the prefix pivoter.\n\n Args:\n prefix: The installation prefix path\n keep_prefix: Whether to keep a failed installation prefix\n \"\"\"\n self.prefix = prefix\n #: Whether to keep a failed installation prefix\n self.keep_prefix = keep_prefix\n #: Temporary location for the original prefix\n self.tmp_prefix: Optional[str] = None\n self.parent = os.path.dirname(prefix)\n\n def __enter__(self) -> \"PrefixPivoter\":\n \"\"\"Enter the context: move existing prefix to temporary location if needed.\"\"\"\n if not self._lexists(self.prefix):\n return self\n # Move the existing prefix to a temporary location so the build starts fresh\n self.tmp_prefix = self._mkdtemp(\n dir=self.parent, prefix=\".\", suffix=OVERWRITE_BACKUP_SUFFIX\n )\n self._rename(self.prefix, self.tmp_prefix)\n return self\n\n def __exit__(\n self, exc_type: Optional[type], exc_val: Optional[BaseException], exc_tb: Optional[object]\n ) -> None:\n \"\"\"Exit the context: cleanup on success, restore on failure.\"\"\"\n if exc_type is None:\n # Success: remove the backup\n if self.tmp_prefix is not None:\n self._rmtree_ignore_errors(self.tmp_prefix)\n return\n\n # Failure handling:\n if self.keep_prefix:\n # Leave the failed prefix in place, discard the backup\n if self.tmp_prefix is not None:\n self._rmtree_ignore_errors(self.tmp_prefix)\n elif self.tmp_prefix is not None:\n # There was a pre-existing prefix: pivot back to it and discard the failed build\n garbage = self._mkdtemp(dir=self.parent, prefix=\".\", suffix=OVERWRITE_GARBAGE_SUFFIX)\n try:\n self._rename(self.prefix, garbage)\n has_failed_prefix = True\n except FileNotFoundError: # build never created the prefix dir\n has_failed_prefix = False\n self._rename(self.tmp_prefix, self.prefix)\n if has_failed_prefix:\n self._rmtree_ignore_errors(garbage)\n elif self._lexists(self.prefix):\n # No backup, just remove the failed installation\n garbage = self._mkdtemp(dir=self.parent, prefix=\".\", suffix=OVERWRITE_GARBAGE_SUFFIX)\n self._rename(self.prefix, garbage)\n self._rmtree_ignore_errors(garbage)\n\n def _lexists(self, path: str) -> bool:\n return os.path.lexists(path)\n\n def _rename(self, src: str, dst: str) -> None:\n os.rename(src, dst)\n\n def _mkdtemp(self, dir: str, prefix: str, suffix: str) -> str:\n return tempfile.mkdtemp(dir=dir, prefix=prefix, suffix=suffix)\n\n def _rmtree_ignore_errors(self, path: str) -> None:\n shutil.rmtree(path, ignore_errors=True)\n\n\ndef worker_function(\n spec: spack.spec.Spec,\n explicit: bool,\n mirrors: List[spack.url_buildcache.MirrorMetadata],\n unsigned: Optional[bool],\n install_policy: InstallPolicy,\n dirty: bool,\n keep_stage: bool,\n restage: bool,\n keep_prefix: bool,\n skip_patch: bool,\n fake: bool,\n install_source: bool,\n run_tests: bool,\n state: Connection,\n parent: Connection,\n echo_control: Connection,\n makeflags: str,\n js1: Optional[Connection],\n js2: Optional[Connection],\n log_path: str,\n global_state: GlobalState,\n stop_before: Optional[str] = None,\n stop_at: Optional[str] = None,\n):\n \"\"\"\n Function run in the build child process. Installs the specified spec, sending state updates\n and build output back to the parent process.\n\n Args:\n spec: Spec to install\n explicit: Whether the spec was explicitly requested by the user\n mirrors: List of buildcache mirrors to try\n unsigned: Whether to allow unsigned buildcache entries\n install_policy: ``\"auto\"``, ``\"cache_only\"``, or ``\"source_only\"``\n dirty: Whether to preserve user environment in the build environment\n keep_stage: Whether to keep the build stage after installation\n restage: Whether to restage the source before building\n keep_prefix: Whether to keep a failed installation prefix\n skip_patch: Whether to skip the patch phase\n run_tests: Whether to run install-time tests for this package\n state: Connection to send state updates to\n parent: Connection to send build output to\n echo_control: Connection to receive echo control messages from\n makeflags: MAKEFLAGS to set, so that the build process uses the POSIX jobserver\n js1: Connection for old style jobserver read fd (if any). Unused, just to inherit fd.\n js2: Connection for old style jobserver write fd (if any). Unused, just to inherit fd.\n log_path: Path to the log file to write build output to\n global_state: Global state to restore\n \"\"\"\n\n # TODO: don't start a build for external packages\n if spec.external:\n return\n\n global_state.restore()\n\n # Isolate the process group to shield against Ctrl+C and enable safe killpg() cleanup. In\n # constrast to setsid(), this keeps a neat process group hierarchy for utils like pstree.\n os.setpgid(0, 0)\n\n # Reset SIGTSTP to default in case the parent had a custom handler.\n signal.signal(signal.SIGTSTP, signal.SIG_DFL)\n\n def handle_sigterm(signum, frame):\n # This SIGTERM handler forwards the signal to child processes (cmake, make, etc). We wait\n # for all child processes to exit before raising KeyboardInterrupt. This ensures all\n # __exit__ and finally blocks run after the child processes have stopped, meaning that we\n # get to clean up the prefix without risking that the child process writes to it\n # afterwards.\n signal.signal(signal.SIGTERM, signal.SIG_IGN)\n os.killpg(0, signal.SIGTERM)\n\n try:\n while True:\n os.waitpid(-1, 0)\n except ChildProcessError:\n pass\n\n raise KeyboardInterrupt(\"Installation interrupted\")\n\n signal.signal(signal.SIGTERM, handle_sigterm)\n\n os.environ[\"MAKEFLAGS\"] = makeflags\n\n # Force line buffering for Python's textio wrappers of stdout/stderr. We're not going to print\n # much ourselves, but what we print should appear before output from `make` and other build\n # tools.\n sys.stdout = os.fdopen(\n sys.stdout.fileno(), \"w\", buffering=1, encoding=sys.stdout.encoding, closefd=False\n )\n sys.stderr = os.fdopen(\n sys.stderr.fileno(), \"w\", buffering=1, encoding=sys.stderr.encoding, closefd=False\n )\n\n # Detach stdin from the terminal like `./build < /dev/null`. This would not be necessary if we\n # used os.setsid() instead of os.setpgid(), but that would \"break\" pstree output.\n devnull_fd = os.open(os.devnull, os.O_RDONLY)\n os.dup2(devnull_fd, 0)\n os.close(devnull_fd)\n sys.stdin = open(os.devnull, \"r\", encoding=sys.stdin.encoding)\n\n # Start the tee thread to forward output to the log file and parent process.\n tee = Tee(echo_control, parent, log_path)\n\n # Use closedfd=false because of the connection objects. Use line buffering.\n state_stream = os.fdopen(state.fileno(), \"w\", buffering=1, closefd=False)\n exit_code = 0\n\n try:\n with PrefixPivoter(spec.prefix, keep_prefix):\n _install(\n spec,\n explicit,\n mirrors,\n unsigned,\n install_policy,\n dirty,\n keep_stage,\n restage,\n skip_patch,\n fake,\n install_source,\n state_stream,\n log_path,\n spack.store.STORE,\n run_tests,\n stop_before,\n stop_at,\n )\n except spack.error.StopPhase:\n exit_code = EXIT_STOPPED_AT_PHASE\n except BaseException:\n traceback.print_exc() # log the traceback to the log file\n exit_code = 1\n finally:\n tee.close()\n state_stream.close()\n\n if exit_code == 0:\n # Try to install the compressed log file\n if not os.path.lexists(spec.package.install_log_path):\n try:\n with open(log_path, \"rb\") as f, open(spec.package.install_log_path, \"wb\") as g:\n # Use GzipFile directly so we can omit filename / mtime in header\n gzip_file = GzipFile(\n filename=\"\", mode=\"wb\", compresslevel=6, mtime=0, fileobj=g\n )\n shutil.copyfileobj(f, gzip_file)\n gzip_file.close()\n except Exception:\n pass # don't fail the build just because log compression failed\n\n # Remove the uncompressed log file from the stage dir on successful install.\n if not keep_stage:\n try:\n os.unlink(log_path)\n except OSError:\n pass\n\n sys.exit(exit_code)\n\n\ndef _archive_build_metadata(pkg: \"spack.package_base.PackageBase\") -> None:\n \"\"\"Copy build metadata from stage to install prefix .spack directory.\n\n Mirrors what the old installer's log() function does in the parent process.\n Only called after a successful source build (not for binary cache installs).\n Errors are suppressed to avoid failing the build over metadata archiving.\"\"\"\n\n try:\n if os.path.lexists(pkg.env_mods_path):\n shutil.copy2(pkg.env_mods_path, pkg.install_env_path)\n except OSError as e:\n spack.llnl.util.tty.debug(e)\n try:\n if os.path.lexists(pkg.configure_args_path):\n shutil.copy2(pkg.configure_args_path, pkg.install_configure_args_path)\n except OSError as e:\n spack.llnl.util.tty.debug(e)\n\n # Archive install-phase test log if present\n try:\n pkg.archive_install_test_log()\n except Exception as e:\n spack.llnl.util.tty.debug(e)\n\n # Archive package-specific files matched by archive_files glob patterns\n try:\n with fs.working_dir(pkg.stage.path):\n target_dir = os.path.join(\n spack.store.STORE.layout.metadata_path(pkg.spec), \"archived-files\"\n )\n errors = io.StringIO()\n for glob_expr in spack.builder.create(pkg).archive_files:\n abs_expr = os.path.realpath(glob_expr)\n if os.path.realpath(pkg.stage.path) not in abs_expr:\n errors.write(f\"[OUTSIDE SOURCE PATH]: {glob_expr}\\n\")\n continue\n if os.path.isabs(glob_expr):\n glob_expr = os.path.relpath(glob_expr, pkg.stage.path)\n for f in glob.glob(glob_expr):\n try:\n target = os.path.join(target_dir, f)\n fs.mkdirp(os.path.dirname(target))\n fs.install(f, target)\n except Exception as e:\n spack.llnl.util.tty.debug(e)\n errors.write(f\"[FAILED TO ARCHIVE]: {f}\")\n if errors.getvalue():\n error_file = os.path.join(target_dir, \"errors.txt\")\n fs.mkdirp(target_dir)\n with open(error_file, \"w\", encoding=\"utf-8\") as err:\n err.write(errors.getvalue())\n spack.llnl.util.tty.warn(\n f\"Errors occurred when archiving files.\\n\\tSee: {error_file}\"\n )\n except Exception as e:\n spack.llnl.util.tty.debug(e)\n\n try:\n packages_dir = spack.store.STORE.layout.build_packages_path(pkg.spec)\n dump_packages(pkg.spec, packages_dir)\n except Exception as e:\n spack.llnl.util.tty.debug(e)\n\n try:\n spack.store.STORE.layout.write_host_environment(pkg.spec)\n except Exception as e:\n spack.llnl.util.tty.debug(e)\n\n\ndef _install(\n spec: spack.spec.Spec,\n explicit: bool,\n mirrors: List[spack.url_buildcache.MirrorMetadata],\n unsigned: Optional[bool],\n install_policy: InstallPolicy,\n dirty: bool,\n keep_stage: bool,\n restage: bool,\n skip_patch: bool,\n fake: bool,\n install_source: bool,\n state_stream: io.TextIOWrapper,\n log_path: str,\n store: spack.store.Store = spack.store.STORE,\n run_tests: bool = False,\n stop_before: Optional[str] = None,\n stop_at: Optional[str] = None,\n) -> None:\n \"\"\"Install a spec from build cache or source.\"\"\"\n\n # Create the stage and log file before starting the tee thread.\n pkg = spec.package\n pkg.run_tests = run_tests\n\n if fake:\n store.layout.create_install_directory(spec)\n _do_fake_install(pkg)\n spack.hooks.post_install(spec, explicit)\n return\n\n # Try to install from buildcache, unless user asked for source only\n if install_policy != \"source_only\":\n if install_from_buildcache(mirrors, spec, unsigned, state_stream):\n spack.hooks.post_install(spec, explicit)\n return\n elif install_policy == \"cache_only\":\n # Binary required but not available\n send_state(\"no binary available\", state_stream)\n raise spack.error.InstallError(f\"No binary available for {spec}\")\n\n unmodified_env = os.environ.copy()\n env_mods = spack.build_environment.setup_package(pkg, dirty=dirty)\n store.layout.create_install_directory(spec)\n\n stage = pkg.stage\n stage.keep = keep_stage\n\n # Then try a source build.\n with stage:\n if restage:\n stage.destroy()\n stage.create()\n\n # Write build environment and env-mods to stage\n spack.util.environment.dump_environment(pkg.env_path)\n with open(pkg.env_mods_path, \"w\", encoding=\"utf-8\") as f:\n f.write(env_mods.shell_modifications(explicit=True, env=unmodified_env))\n\n # Try to snapshot configure/cmake args before phases run\n for attr in (\"configure_args\", \"cmake_args\"):\n try:\n args = getattr(pkg, attr)()\n with open(pkg.configure_args_path, \"w\", encoding=\"utf-8\") as f:\n f.write(\" \".join(shlex.quote(a) for a in args))\n break\n except Exception:\n pass\n\n # For develop packages or non-develop packages with --keep-stage there may be a\n # pre-existing symlink at pkg.log_path which would cause the new symlink to fail.\n # Try removing it if it exists.\n try:\n os.unlink(pkg.log_path)\n except OSError:\n pass\n os.symlink(log_path, pkg.log_path)\n\n send_state(\"staging\", state_stream)\n\n if not skip_patch:\n pkg.do_patch()\n else:\n pkg.do_stage()\n\n os.chdir(stage.source_path)\n\n if install_source and os.path.isdir(stage.source_path):\n src_target = os.path.join(spec.prefix, \"share\", spec.name, \"src\")\n fs.install_tree(stage.source_path, src_target)\n\n spack.hooks.pre_install(spec)\n\n builder = spack.builder.create(pkg)\n if stop_before is not None and stop_before not in builder.phases:\n raise spack.error.InstallError(f\"'{stop_before}' is not a valid phase for {pkg.name}\")\n if stop_at is not None and stop_at not in builder.phases:\n raise spack.error.InstallError(f\"'{stop_at}' is not a valid phase for {pkg.name}\")\n\n for phase in builder:\n if stop_before is not None and phase.name == stop_before:\n send_state(f\"stopped before {stop_before}\", state_stream)\n raise spack.error.StopPhase(f\"Stopping before '{stop_before}'\")\n send_state(phase.name, state_stream)\n spack.llnl.util.tty.msg(f\"{pkg.name}: Executing phase: '{phase.name}'\")\n # Run the install phase with debug output enabled.\n old_debug = spack.llnl.util.tty.debug_level()\n spack.llnl.util.tty.set_debug(1)\n try:\n phase.execute()\n finally:\n spack.llnl.util.tty.set_debug(old_debug)\n if stop_at is not None and phase.name == stop_at:\n send_state(f\"stopped after {stop_at}\", state_stream)\n raise spack.error.StopPhase(f\"Stopping at '{stop_at}'\")\n\n _archive_build_metadata(pkg)\n spack.hooks.post_install(spec, explicit)\n\n\nclass JobServer:\n \"\"\"Attach to an existing POSIX jobserver or create a FIFO-based one.\"\"\"\n\n def __init__(self, num_jobs: int) -> None:\n #: Keep track of how many tokens Spack itself has acquired, which is used to release them.\n self.tokens_acquired = 0\n #: The number of jobs to run concurrently. This translates to `num_jobs - 1` tokens in the\n #: jobserver.\n self.num_jobs = num_jobs\n #: The target number of jobs to run concurrently, which may differ from num_jobs if the\n #: user has requested a decrease in parallelism, but we haven't consumed enough tokens to\n #: reflect that yet. This value is used in the UI. The invariant is that self.target_jobs\n #: can only be modified if self.created is True.\n self.target_jobs = num_jobs\n self.fifo_path: Optional[str] = None\n self.created = False\n self._setup()\n # Ensure that Executable()(...) in build processes ultimately inherit jobserver fds.\n os.set_inheritable(self.r, True)\n os.set_inheritable(self.w, True)\n # r_conn and w_conn are used to make build processes inherit the jobserver fds if needed.\n # Connection objects close the fd as they are garbage collected, so store them.\n self.r_conn = Connection(self.r)\n self.w_conn = Connection(self.w)\n\n def _setup(self) -> None:\n\n fifo_config = get_jobserver_config()\n\n if type(fifo_config) is str:\n # FIFO-based jobserver. Try to open the FIFO.\n open_attempt = open_existing_jobserver_fifo(fifo_config)\n if open_attempt:\n self.r, self.w = open_attempt\n self.fifo_path = fifo_config\n return\n elif type(fifo_config) is tuple:\n # Old style pipe-based jobserver. Validate the fds before using them.\n r, w = fifo_config\n if fcntl.fcntl(r, fcntl.F_GETFD) != -1 and fcntl.fcntl(w, fcntl.F_GETFD) != -1:\n self.r, self.w = r, w\n return\n\n # No existing jobserver we can connect to: create a FIFO-based one.\n self.r, self.w, self.fifo_path = create_jobserver_fifo(self.num_jobs)\n self.created = True\n\n def makeflags(self, gmake: Optional[spack.spec.Spec]) -> str:\n \"\"\"Return the MAKEFLAGS for a build process, depending on its gmake build dependency.\"\"\"\n if self.fifo_path and (not gmake or gmake.satisfies(\"@4.4:\")):\n return f\" -j{self.num_jobs} --jobserver-auth=fifo:{self.fifo_path}\"\n elif not gmake or gmake.satisfies(\"@4.0:\"):\n return f\" -j{self.num_jobs} --jobserver-auth={self.r},{self.w}\"\n else:\n return f\" -j{self.num_jobs} --jobserver-fds={self.r},{self.w}\"\n\n def has_target_parallelism(self) -> bool:\n return self.num_jobs == self.target_jobs\n\n def increase_parallelism(self) -> None:\n \"\"\"Add one token to the jobserver to increase parallelism; this should always work.\"\"\"\n if not self.created:\n return\n os.write(self.w, b\"+\")\n self.target_jobs += 1\n self.num_jobs += 1\n\n def decrease_parallelism(self) -> None:\n \"\"\"Request an eventual concurrency decrease by 1.\"\"\"\n if not self.created or self.target_jobs <= 1:\n return\n self.target_jobs -= 1\n self.maybe_discard_tokens()\n\n def maybe_discard_tokens(self) -> None:\n \"\"\"Try to get reduce parallelism by discarding tokens.\"\"\"\n to_discard = self.num_jobs - self.target_jobs\n if to_discard <= 0:\n return\n try:\n # The read may return zero or just fewer bytes than requested; we'll try again later.\n self.num_jobs -= len(os.read(self.r, to_discard))\n except BlockingIOError:\n pass\n\n def acquire(self, jobs: int) -> int:\n \"\"\"Try and acquire at most 'jobs' tokens from the jobserver. Returns the number of\n tokens actually acquired (may be less than requested, or zero).\"\"\"\n try:\n num_acquired = len(os.read(self.r, jobs))\n self.tokens_acquired += num_acquired\n return num_acquired\n except BlockingIOError:\n return 0\n\n def release(self) -> None:\n \"\"\"Release a token back to the jobserver.\"\"\"\n # The last job to quit has an implicit token, so don't release if we have none.\n if self.tokens_acquired == 0:\n return\n self.tokens_acquired -= 1\n if self.target_jobs < self.num_jobs:\n # If a decrease in parallelism is requested, discard a token instead of releasing it.\n self.num_jobs -= 1\n else:\n os.write(self.w, b\"+\")\n\n def close(self) -> None:\n if self.created and self.num_jobs > 1:\n if self.tokens_acquired != 0:\n # It's a non-fatal internal error to close the jobserver with acquired tokens.\n warnings.warn(\"Spack failed to release jobserver tokens\", stacklevel=2)\n else:\n # Verify that all build processes released the tokens they acquired.\n total = self.num_jobs - 1\n drained = self.acquire(total)\n if drained != total:\n n = total - drained\n warnings.warn(\n f\"{n} jobserver {'token was' if n == 1 else 'tokens were'} not released \"\n \"by the build processes. This can indicate that the build ran with \"\n \"limited parallelism.\",\n stacklevel=2,\n )\n\n self.r_conn.close()\n self.w_conn.close()\n\n # Remove the FIFO if we created it.\n if self.created and self.fifo_path:\n try:\n os.unlink(self.fifo_path)\n except OSError:\n pass\n try:\n os.rmdir(os.path.dirname(self.fifo_path))\n except OSError:\n pass\n\n\ndef start_build(\n spec: spack.spec.Spec,\n explicit: bool,\n mirrors: List[spack.url_buildcache.MirrorMetadata],\n unsigned: Optional[bool],\n install_policy: InstallPolicy,\n dirty: bool,\n keep_stage: bool,\n restage: bool,\n keep_prefix: bool,\n skip_patch: bool,\n fake: bool,\n install_source: bool,\n run_tests: bool,\n jobserver: JobServer,\n stop_before: Optional[str] = None,\n stop_at: Optional[str] = None,\n) -> ChildInfo:\n \"\"\"Start a new build.\"\"\"\n # Create pipes for the child's output, state reporting, and control.\n state_r_conn, state_w_conn = Pipe(duplex=False)\n output_r_conn, output_w_conn = Pipe(duplex=False)\n control_r_conn, control_w_conn = Pipe(duplex=False)\n\n # Obtain the MAKEFLAGS to be set in the child process, and determine whether it's necessary\n # for the child process to inherit our jobserver fds.\n gmake = next(iter(spec.dependencies(\"gmake\")), None)\n makeflags = jobserver.makeflags(gmake)\n fifo = \"--jobserver-auth=fifo:\" in makeflags\n\n # TODO: remove once external specs do not create a build process\n if spec.external:\n log_path = os.devnull\n else:\n log_fd, log_path = tempfile.mkstemp(\n prefix=f\"spack-stage-{spec.name}-{spec.version}-{spec.dag_hash()}-\",\n suffix=\".log\",\n dir=spack.stage.get_stage_root(),\n )\n os.close(log_fd) # child will open it\n\n proc = Process(\n target=worker_function,\n args=(\n spec,\n explicit,\n mirrors,\n unsigned,\n install_policy,\n dirty,\n keep_stage,\n restage,\n keep_prefix,\n skip_patch,\n fake,\n install_source,\n run_tests,\n state_w_conn,\n output_w_conn,\n control_r_conn,\n makeflags,\n None if fifo else jobserver.r_conn,\n None if fifo else jobserver.w_conn,\n log_path,\n GlobalState(),\n stop_before,\n stop_at,\n ),\n )\n proc.start()\n\n # The parent process does not need the write ends of the main pipes or the read end of control.\n state_w_conn.close()\n output_w_conn.close()\n control_r_conn.close()\n\n # Set the read ends to non-blocking: in principle redundant with epoll/kqueue, but safer.\n os.set_blocking(output_r_conn.fileno(), False)\n os.set_blocking(state_r_conn.fileno(), False)\n\n return ChildInfo(proc, spec, output_r_conn, state_r_conn, control_w_conn, log_path, explicit)\n\n\ndef get_jobserver_config(makeflags: Optional[str] = None) -> Optional[Union[str, Tuple[int, int]]]:\n \"\"\"Parse MAKEFLAGS for jobserver. Either it's a FIFO or (r, w) pair of file descriptors.\n\n Args:\n makeflags: MAKEFLAGS string to parse. If None, reads from os.environ.\n \"\"\"\n makeflags = os.environ.get(\"MAKEFLAGS\", \"\") if makeflags is None else makeflags\n if not makeflags:\n return None\n # We can have the following flags:\n # --jobserver-fds=R,W (before GNU make 4.2)\n # --jobserver-auth=fifo:PATH or --jobserver-auth=R,W (after GNU make 4.2)\n # In case of multiple, the last one wins.\n matches = re.findall(r\" --jobserver-[^=]+=([^ ]+)\", makeflags)\n if not matches:\n return None\n last_match: str = matches[-1]\n assert isinstance(last_match, str)\n if last_match.startswith(\"fifo:\"):\n return last_match[5:]\n parts = last_match.split(\",\", 1)\n if len(parts) != 2:\n return None\n try:\n return int(parts[0]), int(parts[1])\n except ValueError:\n return None\n\n\ndef create_jobserver_fifo(num_jobs: int) -> Tuple[int, int, str]:\n \"\"\"Create a new jobserver FIFO with the specified number of job tokens.\"\"\"\n tmpdir = tempfile.mkdtemp()\n fifo_path = os.path.join(tmpdir, \"jobserver_fifo\")\n\n try:\n os.mkfifo(fifo_path, 0o600)\n read_fd = os.open(fifo_path, os.O_RDONLY | os.O_NONBLOCK)\n write_fd = os.open(fifo_path, os.O_WRONLY)\n # write num_jobs - 1 tokens, because the first job is implicit\n os.write(write_fd, b\"+\" * (num_jobs - 1))\n return read_fd, write_fd, fifo_path\n except Exception:\n try:\n os.unlink(fifo_path)\n except OSError as e:\n spack.llnl.util.tty.debug(f\"Failed to remove POSIX jobserver FIFO: {e}\", level=3)\n pass\n try:\n os.rmdir(tmpdir)\n except OSError as e:\n spack.llnl.util.tty.debug(f\"Failed to remove POSIX jobserver FIFO dir: {e}\", level=3)\n pass\n raise\n\n\ndef open_existing_jobserver_fifo(fifo_path: str) -> Optional[Tuple[int, int]]:\n \"\"\"Open an existing jobserver FIFO for reading and writing.\"\"\"\n try:\n read_fd = os.open(fifo_path, os.O_RDONLY | os.O_NONBLOCK)\n write_fd = os.open(fifo_path, os.O_WRONLY)\n return read_fd, write_fd\n except OSError:\n return None\n\n\nclass FdInfo:\n \"\"\"Information about a file descriptor mapping.\"\"\"\n\n __slots__ = (\"pid\", \"name\")\n\n def __init__(self, pid: int, name: str) -> None:\n self.pid = pid\n self.name = name\n\n\nclass BuildInfo:\n \"\"\"Information about a package being built.\"\"\"\n\n __slots__ = (\n \"state\",\n \"explicit\",\n \"version\",\n \"hash\",\n \"name\",\n \"external\",\n \"prefix\",\n \"finished_time\",\n \"start_time\",\n \"duration\",\n \"progress_percent\",\n \"control_w_conn\",\n \"log_path\",\n \"log_summary\",\n )\n\n def __init__(\n self,\n spec: spack.spec.Spec,\n explicit: bool,\n control_w_conn: Optional[Connection],\n log_path: Optional[str] = None,\n start_time: float = 0.0,\n ) -> None:\n self.state: str = \"starting\"\n self.explicit: bool = explicit\n self.version: str = str(spec.version)\n self.hash: str = spec.dag_hash(7)\n self.name: str = spec.name\n self.external: bool = spec.external\n self.prefix: str = spec.prefix\n self.finished_time: Optional[float] = None\n self.start_time: float = start_time\n self.duration: Optional[float] = None\n self.progress_percent: Optional[int] = None\n self.control_w_conn = control_w_conn\n self.log_path: Optional[str] = log_path\n self.log_summary: Optional[str] = None\n\n\nclass BuildStatus:\n \"\"\"Tracks the build status display for terminal output.\"\"\"\n\n def __init__(\n self,\n total: int,\n stdout: io.TextIOWrapper = sys.stdout, # type: ignore[assignment]\n get_terminal_size: Callable[[], os.terminal_size] = os.get_terminal_size,\n get_time: Callable[[], float] = time.monotonic,\n is_tty: Optional[bool] = None,\n color: Optional[bool] = None,\n verbose: bool = False,\n filter_padding: bool = False,\n ) -> None:\n #: Ordered dict of build ID -> info\n self.total = total\n self.completed = 0\n self.builds: Dict[str, BuildInfo] = {}\n self.finished_builds: List[BuildInfo] = []\n self.spinner_chars = [\"|\", \"/\", \"-\", \"\\\\\"]\n self.spinner_index = 0\n self.dirty = True # Start dirty to draw initial state\n self.active_area_rows = 0\n self.total_lines = 0\n self.next_spinner_update = 0.0\n self.next_update = 0.0\n self.overview_mode = True # Whether to draw the package overview\n self.tracked_build_id = \"\" # identifier of the package whose logs we follow\n self.search_term = \"\"\n self.search_mode = False\n self.log_ends_with_newline = True\n self.actual_jobs: int = 0\n self.target_jobs: int = 0\n\n self.stdout = stdout\n self.get_terminal_size = get_terminal_size\n self.terminal_size = os.terminal_size((0, 0))\n self.terminal_size_changed: bool = True\n self.get_time = get_time\n self.is_tty = is_tty if is_tty is not None else stdout.isatty()\n if color is not None:\n self.color = color\n else:\n self.color = spack.llnl.util.tty.color.get_color_when(stdout)\n #: Verbose mode only applies to non-TTY where we want to track a single build log.\n self.verbose = verbose and not self.is_tty\n self.filter_padding = filter_padding\n #: When True, suppress all terminal output (process is in background).\n #: Controlling code is responsible for modifying this variable based on process state\n self.headless = False\n\n def on_resize(self) -> None:\n \"\"\"Refresh cached terminal size and trigger a redraw.\"\"\"\n self.terminal_size_changed = True\n self.dirty = True\n\n def add_build(\n self,\n spec: spack.spec.Spec,\n explicit: bool,\n control_w_conn: Optional[Connection] = None,\n log_path: Optional[str] = None,\n ) -> None:\n \"\"\"Add a new build to the display and mark the display as dirty.\"\"\"\n build_info = BuildInfo(spec, explicit, control_w_conn, log_path, int(self.get_time()))\n self.builds[spec.dag_hash()] = build_info\n self.dirty = True\n # Track the new build's logs when we're not already following another build. This applies\n # only in non-TTY verbose mode.\n if self.verbose and not self.tracked_build_id and control_w_conn is not None:\n self.tracked_build_id = spec.dag_hash()\n try:\n os.write(control_w_conn.fileno(), b\"1\")\n except OSError:\n pass\n\n def toggle(self) -> None:\n \"\"\"Toggle between overview mode and following a specific build.\"\"\"\n if self.overview_mode:\n self.next()\n else:\n if not self.log_ends_with_newline:\n self.stdout.buffer.write(b\"\\n\")\n self.log_ends_with_newline = True\n self.active_area_rows = 0\n self.search_term = \"\"\n self.search_mode = False\n self.overview_mode = True\n self.dirty = True\n try:\n conn = self.builds[self.tracked_build_id].control_w_conn\n if conn is not None:\n os.write(conn.fileno(), b\"0\")\n except (KeyError, OSError):\n pass\n self.tracked_build_id = \"\"\n\n def search_input(self, input: str) -> None:\n \"\"\"Handle keyboard input when in search mode\"\"\"\n if input in (\"\\r\", \"\\n\"):\n self.log_ends_with_newline = False\n self.next(1)\n elif input == \"\\x1b\": # Escape\n self.search_mode = False\n self.search_term = \"\"\n self.dirty = True\n elif input in (\"\\x7f\", \"\\b\"): # Backspace\n self.search_term = self.search_term[:-1]\n self.dirty = True\n elif input.isprintable():\n self.search_term += input\n self.dirty = True\n\n def enter_search(self) -> None:\n self.search_mode = True\n self.dirty = True\n\n def _is_displayed(self, build: BuildInfo) -> bool:\n \"\"\"Returns true if the build matches the search term, or when no search term is set.\"\"\"\n # When not in search mode, the search_term is \"\", which always evaluates to True below\n return self.search_term in build.name or build.hash.startswith(self.search_term)\n\n def _get_next(self, direction: int) -> Optional[str]:\n \"\"\"Returns the next or previous unfinished build ID matching the search term, or None if\n none found. Direction should be 1 for next, -1 for previous.\"\"\"\n matching = [\n build_id\n for build_id, build in self.builds.items()\n if (build.finished_time is None or build.state == \"failed\")\n and self._is_displayed(build)\n ]\n if not matching:\n return None\n try:\n idx = matching.index(self.tracked_build_id)\n except ValueError:\n return matching[0] if direction == 1 else matching[-1]\n\n return matching[(idx + direction) % len(matching)]\n\n def next(self, direction: int = 1) -> None:\n \"\"\"Follow the logs of the next build in the list.\"\"\"\n new_build_id = self._get_next(direction)\n\n if not new_build_id or self.tracked_build_id == new_build_id:\n return\n\n new_build = self.builds[new_build_id]\n\n if self.overview_mode:\n self.overview_mode = False\n\n # Stop following the previous and start following the new build.\n if self.tracked_build_id:\n try:\n conn = self.builds[self.tracked_build_id].control_w_conn\n if conn is not None:\n os.write(conn.fileno(), b\"0\")\n except (KeyError, OSError):\n pass\n\n self.tracked_build_id = new_build_id\n\n version_str = (\n f\"\\033[0;36m@{new_build.version}\\033[0m\" if self.color else f\"@{new_build.version}\"\n )\n prefix = \"\" if self.log_ends_with_newline else \"\\n\"\n\n if new_build.state == \"failed\":\n # For failed builds, show the stored log summary instead of following live logs.\n self.stdout.write(f\"{prefix}==> Log summary of {new_build.name}{version_str}\\n\")\n self.log_ends_with_newline = True\n if new_build.log_summary:\n self.stdout.write(new_build.log_summary)\n if new_build.log_path:\n if not new_build.log_summary:\n self.stdout.write(\"No errors parsed from log, see full log: \")\n else:\n self.stdout.write(\"Full log: \")\n self.stdout.write(f\"{new_build.log_path}\\n\")\n self.stdout.flush()\n else:\n # Tell the user we're following new logs, and instruct the child to start sending.\n self.stdout.write(f\"{prefix}==> Following logs of {new_build.name}{version_str}\\n\")\n self.log_ends_with_newline = True\n self.stdout.flush()\n try:\n conn = new_build.control_w_conn\n if conn is not None:\n os.write(conn.fileno(), b\"1\")\n except (KeyError, OSError):\n pass\n\n def set_jobs(self, actual: int, target: int) -> None:\n \"\"\"Set the actual and target number of jobs to run concurrently.\"\"\"\n if actual == self.actual_jobs and target == self.target_jobs:\n return\n self.actual_jobs = actual\n self.target_jobs = target\n self.dirty = True\n\n def update_state(self, build_id: str, state: str) -> None:\n \"\"\"Update the state of a package and mark the display as dirty.\"\"\"\n build_info = self.builds[build_id]\n build_info.state = state\n build_info.progress_percent = None\n\n if state in (\"finished\", \"failed\"):\n self.completed += 1\n now = self.get_time()\n build_info.duration = now - build_info.start_time\n build_info.finished_time = now + CLEANUP_TIMEOUT\n\n # Stop tracking the finished build's logs.\n if build_id == self.tracked_build_id:\n if not self.overview_mode:\n self.toggle()\n if self.verbose:\n self.tracked_build_id = \"\"\n\n self.dirty = True\n\n # For non-TTY output, print state changes immediately\n if not self.is_tty and not self.headless:\n line = \"\".join(\n self._generate_line_components(build_info, static=True, now=self.get_time())\n )\n self.stdout.write(line + \"\\n\")\n self.stdout.flush()\n\n def parse_log_summary(self, build_id: str) -> None:\n \"\"\"Parse the build log for errors/warnings and store the summary.\"\"\"\n build_info = self.builds[build_id]\n if not build_info.log_path or not os.path.exists(build_info.log_path):\n return\n buf = io.StringIO()\n spack.build_environment.write_log_summary(\n buf, f\"{build_info.name}@{build_info.version} build\", build_info.log_path\n )\n summary = buf.getvalue()\n if summary:\n build_info.log_summary = summary\n\n def update_progress(self, build_id: str, current: int, total: int) -> None:\n \"\"\"Update the progress of a package and mark the display as dirty.\"\"\"\n percent = int((current / total) * 100)\n build_info = self.builds[build_id]\n if build_info.progress_percent != percent:\n build_info.progress_percent = percent\n self.dirty = True\n\n def update(self, finalize: bool = False) -> None:\n \"\"\"Redraw the interactive display.\"\"\"\n if self.headless or not self.is_tty or not self.overview_mode:\n return\n\n now = self.get_time()\n\n # Avoid excessive redraws\n if not finalize and now < self.next_update:\n return\n\n # Only update the spinner if there are still running packages\n if now >= self.next_spinner_update and any(\n pkg.finished_time is None for pkg in self.builds.values()\n ):\n self.spinner_index = (self.spinner_index + 1) % len(self.spinner_chars)\n self.dirty = True\n self.next_spinner_update = now + SPINNER_INTERVAL\n\n for build_id in list(self.builds):\n build_info = self.builds[build_id]\n if build_info.state == \"failed\" or build_info.finished_time is None:\n continue\n\n if finalize or now >= build_info.finished_time:\n self.finished_builds.append(build_info)\n del self.builds[build_id]\n self.dirty = True\n\n if not self.dirty and not finalize:\n return\n\n # Build the overview output in a buffer and print all at once to avoid flickering.\n buffer = io.StringIO()\n\n # Move cursor up to the start of the display area assuming the same terminal width. If the\n # terminal resized, lines may have wrapped, and we should've moved up further. We do not\n # try to track that (would require keeping track of each line's width).\n if self.active_area_rows > 0:\n buffer.write(f\"\\033[{self.active_area_rows}A\\r\")\n\n if self.terminal_size_changed:\n self.terminal_size = self.get_terminal_size()\n self.terminal_size_changed = False\n # After resize, active_area_rows is invalidated due to possible line wrapping. Set to\n # 0 to force newlines instead of cursor movement.\n self.active_area_rows = 0\n max_width, max_height = self.terminal_size\n\n # First flush the finished builds. These are \"persisted\" in terminal history.\n if self.finished_builds:\n for build in self.finished_builds:\n self._render_build(build, buffer, now=now)\n self._println(buffer, force_newline=True) # should scroll the terminal\n self.finished_builds.clear()\n # Finished builds can span multiple lines, overlapping our \"active area\", invalidating\n # active_area_rows. Set to 0 to force newlines instead of cursor movement.\n self.active_area_rows = 0\n\n # Then a header followed by the active builds. This is the \"mutable\" part of the display.\n self.total_lines = 0\n\n if not finalize:\n if self.color:\n bold = \"\\033[1m\"\n reset = \"\\033[0m\"\n cyan = \"\\033[36m\"\n else:\n bold = reset = cyan = \"\"\n\n if self.actual_jobs != self.target_jobs:\n jobs_str = f\"{self.actual_jobs}=>{self.target_jobs}\"\n else:\n jobs_str = str(self.target_jobs)\n long_header_len = len(\n f\"Progress: {self.completed}/{self.total} +/-: {jobs_str} jobs\"\n \" /: filter v: logs n/p: next/prev\"\n )\n if long_header_len < max_width:\n self._println(\n buffer,\n f\"{bold}Progress:{reset} {self.completed}/{self.total}\"\n f\" {cyan}+{reset}/{cyan}-{reset}: \"\n f\"{jobs_str} jobs\"\n f\" {cyan}/{reset}: filter {cyan}v{reset}: logs\"\n f\" {cyan}n{reset}/{cyan}p{reset}: next/prev\",\n )\n else:\n self._println(buffer, f\"{bold}Progress:{reset} {self.completed}/{self.total}\")\n\n displayed_builds = (\n [b for b in self.builds.values() if self._is_displayed(b)]\n if self.search_term\n else self.builds.values()\n )\n len_builds = len(displayed_builds)\n\n # Truncate if we have more builds than fit on the screen. In that case we have to reserve\n # an additional line for the \"N more...\" message.\n truncate_at = max_height - 3 if len_builds + 2 > max_height else len_builds\n\n for i, build in enumerate(displayed_builds, 1):\n if i > truncate_at:\n self._println(buffer, f\"{len_builds - i + 1} more...\")\n break\n self._render_build(build, buffer, max_width, now=now)\n self._println(buffer)\n\n if self.search_mode:\n buffer.write(f\"filter> {self.search_term}\\033[K\")\n\n # Clear any remaining lines from previous display\n buffer.write(\"\\033[0J\")\n\n # Print everything at once to avoid flickering\n self.stdout.write(buffer.getvalue())\n self.stdout.flush()\n\n # Update the number of lines drawn for next time. It reflects the number of active builds.\n self.active_area_rows = self.total_lines\n self.dirty = False\n\n # Schedule next UI update\n self.next_update = now + SPINNER_INTERVAL / 2\n\n def _println(self, buffer: io.StringIO, line: str = \"\", force_newline: bool = False) -> None:\n \"\"\"Print a line to the buffer, handling line clearing and cursor movement.\"\"\"\n self.total_lines += 1\n if line:\n buffer.write(line)\n if self.total_lines > self.active_area_rows or force_newline:\n buffer.write(\"\\033[0m\\033[K\\n\") # reset, clear to EOL, newline\n else:\n buffer.write(\"\\033[0m\\033[K\\033[1B\\r\") # reset, clear to EOL, move to next line\n\n def print_logs(self, build_id: str, data: bytes) -> None:\n if self.headless:\n return\n # Discard logs we are not following. Generally this should not happen as we tell the child\n # to only send logs when we are following it. It could maybe happen while transitioning\n # between builds.\n if build_id != self.tracked_build_id:\n return\n if self.filter_padding:\n data = padding_filter_bytes(data)\n self.stdout.buffer.write(data)\n self.stdout.flush()\n self.log_ends_with_newline = data.endswith(b\"\\n\")\n\n def _render_build(\n self, build_info: BuildInfo, buffer: io.StringIO, max_width: int = 0, now: float = 0.0\n ) -> None:\n \"\"\"Print a single build line to the buffer, truncating to max_width (if > 0).\"\"\"\n line_width = 0\n for component in self._generate_line_components(build_info, now=now):\n # ANSI escape sequence(s), does not contribute to width\n if not component.startswith(\"\\033\") and max_width > 0:\n line_width += len(component)\n if line_width > max_width:\n break\n buffer.write(component)\n\n def _generate_line_components(\n self, build_info: BuildInfo, static: bool = False, now: float = 0.0\n ) -> Generator[str, None, None]:\n \"\"\"Yield formatted line components for a package. Escape sequences are yielded as separate\n strings so they do not contribute to the line width.\"\"\"\n if build_info.external:\n indicator = \"[e]\"\n elif build_info.state == \"finished\":\n indicator = \"[+]\"\n elif build_info.state == \"failed\":\n indicator = \"[x]\"\n elif static:\n indicator = \"[ ]\"\n else:\n indicator = f\"[{self.spinner_chars[self.spinner_index]}]\"\n\n if self.color:\n if build_info.state == \"failed\":\n yield \"\\033[31m\" # red\n elif build_info.state == \"finished\":\n yield \"\\033[32m\" # green\n\n yield indicator\n if self.color:\n yield \"\\033[0m\" # reset\n yield \" \"\n if self.color:\n yield \"\\033[0;90m\" # dark gray\n yield build_info.hash\n if self.color:\n yield \"\\033[0m\" # reset\n yield \" \"\n\n # Package name in bold white if explicit, default otherwise\n if build_info.explicit:\n if self.color:\n yield \"\\033[1;37m\" # bold white\n yield build_info.name\n if self.color:\n yield \"\\033[0m\" # reset\n else:\n yield build_info.name\n\n if self.color:\n yield \"\\033[0;36m\" # cyan\n yield f\"@{build_info.version}\"\n if self.color:\n yield \"\\033[0m\" # reset\n\n # progress or state\n if build_info.progress_percent is not None:\n yield \" fetching\"\n yield f\": {build_info.progress_percent}%\"\n elif build_info.state == \"finished\":\n prefix = build_info.prefix\n yield f\" {padding_filter(prefix) if self.filter_padding else prefix}\"\n elif build_info.state == \"failed\":\n yield \" failed\"\n if build_info.log_path:\n yield f\": {build_info.log_path}\"\n else:\n yield f\" {build_info.state}\"\n\n # Duration\n elapsed = (\n build_info.duration\n if build_info.duration is not None\n else (now - build_info.start_time)\n )\n if elapsed > 0:\n if self.color:\n yield \"\\033[0;90m\" # dark gray\n yield f\" ({pretty_duration(elapsed)})\"\n if self.color:\n yield \"\\033[0m\"\n\n\nNodes = Dict[str, spack.spec.Spec]\nEdges = Dict[str, Set[str]]\n\n\nclass BuildGraph:\n \"\"\"Represents the dependency graph for package installation.\"\"\"\n\n def __init__(\n self,\n specs: List[spack.spec.Spec],\n root_policy: InstallPolicy,\n dependencies_policy: InstallPolicy,\n include_build_deps: bool,\n install_package: bool,\n install_deps: bool,\n database: spack.database.Database,\n overwrite_set: Optional[Set[str]] = None,\n tests: Union[bool, List[str], Set[str]] = False,\n explicit_set: Optional[Set[str]] = None,\n ):\n \"\"\"Construct a build graph from the given specs. This includes only packages that need to\n be installed. Installed packages are pruned from the graph, and build dependencies are only\n included when necessary.\"\"\"\n self.roots = {s.dag_hash() for s in specs}\n self.nodes = {s.dag_hash(): s for s in specs}\n self.parent_to_child: Dict[str, Set[str]] = {}\n self.child_to_parent: Dict[str, Set[str]] = {}\n overwrite_set = overwrite_set or set()\n explicit_set = explicit_set or set()\n self.pruned: Set[str] = set()\n stack: List[Tuple[spack.spec.Spec, InstallPolicy]] = [\n (s, root_policy) for s in self.nodes.values()\n ]\n\n with database.read_transaction():\n # Set the install prefix for each spec based on the db record or store layout\n for s in spack.traverse.traverse_nodes(specs):\n _, record = database.query_by_spec_hash(s.dag_hash())\n if record and record.path:\n s.set_prefix(record.path)\n else:\n s.set_prefix(spack.store.STORE.layout.path_for_spec(s))\n\n # Build the graph and determine which specs to prune\n while stack:\n spec, install_policy = stack.pop()\n key = spec.dag_hash()\n _, record = database.query_by_spec_hash(key)\n\n # Conditionally include build dependencies. Don't prune installed specs\n # that need to be marked explicit so they flow through the DB write path.\n if record and record.installed and key not in overwrite_set:\n # Installed spec only needs link/run deps traversed.\n dependencies = spec.dependencies(deptype=dt.LINK | dt.RUN)\n # If it needs to be marked explicit, keep it in the graph (don't prune).\n if not (key in explicit_set and not record.explicit):\n self.pruned.add(key)\n elif install_policy == \"cache_only\" and not include_build_deps:\n dependencies = spec.dependencies(deptype=dt.LINK | dt.RUN)\n else:\n deptype = dt.BUILD | dt.LINK | dt.RUN\n if tests is True or (tests and spec.name in tests):\n deptype |= dt.TEST\n dependencies = spec.dependencies(deptype=deptype)\n\n self.parent_to_child[key] = {d.dag_hash() for d in dependencies}\n\n # Enqueue new dependencies\n for d in dependencies:\n if d.dag_hash() in self.nodes:\n continue\n self.nodes[d.dag_hash()] = d\n stack.append((d, dependencies_policy))\n\n # Construct reverse lookup from child to parent\n for parent, children in self.parent_to_child.items():\n for child in children:\n if child in self.child_to_parent:\n self.child_to_parent[child].add(parent)\n else:\n self.child_to_parent[child] = {parent}\n\n # If we're not installing the package itself, mark root specs for pruning too\n if not install_package:\n self.pruned.update(s.dag_hash() for s in specs)\n\n # Prune specs from the build graph. Their parents become parents of their children and\n # their children become children of their parents.\n for key in self.pruned:\n for parent in self.child_to_parent.get(key, ()):\n self.parent_to_child[parent].remove(key)\n self.parent_to_child[parent].update(self.parent_to_child.get(key, ()))\n for child in self.parent_to_child.get(key, ()):\n self.child_to_parent[child].remove(key)\n self.child_to_parent[child].update(self.child_to_parent.get(key, ()))\n self.parent_to_child.pop(key, None)\n self.child_to_parent.pop(key, None)\n self.nodes.pop(key, None)\n\n # Check that all prefixes to be created are unique.\n prefixes = [s.prefix for s in self.nodes.values() if not s.external]\n if len(prefixes) != len(set(prefixes)):\n raise spack.error.InstallError(\n \"Install prefix collision: \"\n + \", \".join(p for p in prefixes if prefixes.count(p) > 1)\n )\n\n # If we're not installing dependencies, verify that all remaining nodes in the build graph\n # after pruning are roots. If there are any non-root nodes, it means there are uninstalled\n # dependencies that we're not supposed to install.\n if not install_deps:\n non_root_spec = next((v for k, v in self.nodes.items() if k not in self.roots), None)\n if non_root_spec is not None:\n raise spack.error.InstallError(\n f\"Failed to install in package only mode: dependency {non_root_spec} is not \"\n \"installed\"\n )\n\n def enqueue_parents(self, dag_hash: str, pending_builds: List[str]) -> None:\n \"\"\"After a spec is installed, remove it from the graph and enqueue any parents that are\n now ready to install.\n\n Args:\n dag_hash: The dag_hash of the spec that was just installed\n pending_builds: List to append parent specs that are ready to build\n \"\"\"\n # Remove node and edges from the node in the build graph\n self.parent_to_child.pop(dag_hash, None)\n self.nodes.pop(dag_hash, None)\n parents = self.child_to_parent.pop(dag_hash, None)\n\n if not parents:\n return\n\n # Enqueue any parents and remove edges to the installed child\n for parent in parents:\n children = self.parent_to_child[parent]\n children.remove(dag_hash)\n if not children:\n pending_builds.append(parent)\n\n\nclass ScheduleResult(NamedTuple):\n \"\"\"Return value of :func:`schedule_builds`.\"\"\"\n\n #: True if any pending builds were blocked on locks held by other processes.\n blocked: bool\n #: ``(dag_hash, lock)`` pairs where the write lock is held and the caller must start the build\n #: and eventually release the lock.\n to_start: List[Tuple[str, spack.util.lock.Lock]]\n #: ``(dag_hash, spec, lock)`` triples found already installed by another process; the read lock\n #: is held and the caller must add it to retained_read_locks.\n newly_installed: List[Tuple[str, spack.spec.Spec, spack.util.lock.Lock]]\n #: Actions to mark already installed specs explicit in the DB.\n to_mark_explicit: List[MarkExplicitAction]\n\n\ndef schedule_builds(\n pending: List[str],\n build_graph: BuildGraph,\n db: spack.database.Database,\n prefix_locker: spack.database.SpecLocker,\n overwrite: Set[str],\n overwrite_time: float,\n capacity: int,\n needs_jobserver_token: bool,\n jobserver: JobServer,\n explicit: Set[str],\n) -> ScheduleResult:\n \"\"\"Try to schedule as many pending builds as possible.\n\n For each pending spec, attempts to acquire a non-blocking per-spec write lock. If the write\n lock times out, a read lock is tried as a fallback: a successful read lock means the first\n process finished and downgraded its write lock. If the DB confirms the spec is installed, it\n is captured as newly_installed; if the DB says it is not installed, the concurrent process was\n likely killed mid-build, and the spec is retried next iteration. Under both the DB read lock\n and the prefix lock, checks whether another process has already installed the spec. If so,\n captures it as newly_installed (caller enqueues parents) and keeps a read lock on the prefix\n to prevent concurrent uninstall. Otherwise, acquires a jobserver token if needed and adds the\n (dag_hash, lock) pair to to_start (caller launches the build).\n\n Args:\n pending: List of dag hashes pending installation; modified in-place.\n build_graph: The build dependency graph; used for node lookup and parent enqueueing.\n db: Package database; used for read lock and installed-status queries.\n prefix_locker: Per-spec write locker.\n overwrite: Set of dag hashes to overwrite even if already installed.\n overwrite_time: Timestamp (from time.time()) at which the overwrite install was requested.\n A spec in ``overwrite`` whose DB installation_time >= overwrite_time was installed by\n a concurrent process after our request started and should be treated as done.\n capacity: Maximum number of new builds to add to to_start in this call.\n needs_jobserver_token: True if a jobserver token is required for the first new build.\n jobserver: Jobserver for acquiring tokens.\n explicit: Set of dag hashes to mark explicit in the DB if found already installed.\n\n Returns:\n A :class:`ScheduleResult` with ``blocked``, ``to_start``, and ``newly_installed``\n fields; see :class:`ScheduleResult` for field semantics.\n \"\"\"\n to_start: List[Tuple[str, spack.util.lock.Lock]] = []\n newly_installed: List[Tuple[str, spack.spec.Spec, spack.util.lock.Lock]] = []\n to_mark_explicit: List[MarkExplicitAction] = []\n blocked = True\n\n # Acquire the DB read lock non-blocking; hold it throughout the loop so the in-memory snapshot\n # stays consistent while we acquire per-spec prefix locks.\n if not db.lock.try_acquire_read():\n return ScheduleResult(blocked, to_start, newly_installed, to_mark_explicit)\n\n try:\n db._read() # refresh in-memory snapshot under the read lock\n\n idx = 0\n while capacity and idx < len(pending):\n dag_hash = pending[idx]\n spec = build_graph.nodes[dag_hash]\n lock = prefix_locker.lock(spec)\n\n if lock.try_acquire_write():\n blocked = False\n have_write = True\n elif lock.try_acquire_read():\n have_write = False\n else:\n idx += 1\n continue\n\n # Check installed status under the DB read lock and prefix lock.\n upstream, record = db.query_by_spec_hash(dag_hash)\n\n # If the spec is already installed, treat it as done regardless of lock type.\n # A spec in the overwrite set is also treated as done if another process installed it\n # after our overwrite request was created (installation_time >= overwrite_time).\n if (\n record\n and record.installed\n and (dag_hash not in overwrite or record.installation_time >= overwrite_time)\n ):\n if have_write:\n lock.downgrade_write_to_read()\n # keep the read lock (either downgraded or already a read lock)\n del pending[idx]\n newly_installed.append((dag_hash, spec, lock))\n # It's already installed, but needs to be marked as explicitly installed in the DB.\n if dag_hash in explicit and not record.explicit:\n to_mark_explicit.append(MarkExplicitAction(spec))\n build_graph.enqueue_parents(dag_hash, pending)\n continue\n\n if not have_write:\n # If have to install but only got a read lock, try it in next iteration of the\n # event loop.\n lock.release_read()\n idx += 1\n continue\n\n # Write lock acquired: proceed with scheduling.\n # Don't schedule builds for specs from upstream databases.\n if upstream and record and not record.installed:\n lock.release_write()\n raise spack.error.InstallError(\n f\"Cannot install {spec}: it is uninstalled in an upstream database.\"\n )\n\n # Defensively assert prefix invariants\n if not spec.external:\n if (\n dag_hash in overwrite\n and record\n and record.installed\n and record.path != spec.prefix\n ):\n # Cannot do an overwrite install to a different prefix.\n lock.release_write()\n raise spack.error.InstallError(\n f\"Prefix mismatch in overwrite of {spec}: expected {record.path}, \"\n f\"got {spec.prefix}\"\n )\n elif dag_hash not in overwrite and spec.prefix in db._installed_prefixes:\n # Prevent install prefix collision with other specs.\n lock.release_write()\n raise spack.error.InstallError(\n f\"Cannot install {spec}: prefix {spec.prefix} already exists\"\n )\n\n # Acquire a jobserver token if needed. The first (implicit) job needs no token.\n if needs_jobserver_token and not jobserver.acquire(1):\n lock.release_write()\n break # no tokens available right now; stop scheduling\n\n del pending[idx]\n to_start.append((dag_hash, lock))\n capacity -= 1\n needs_jobserver_token = True # all subsequent jobs need a token\n\n finally:\n db.lock.release_read()\n\n return ScheduleResult(blocked, to_start, newly_installed, to_mark_explicit)\n\n\ndef _node_to_roots(roots: List[spack.spec.Spec]) -> Dict[str, FrozenSet[str]]:\n \"\"\"Map each node in a graph to the set of root node DAG hashes that can reach it.\n\n Args:\n roots: List of root specs.\n\n Returns:\n A dictionary mapping each node's dag_hash to a frozenset of root dag_hashes.\n \"\"\"\n node_to_roots: Dict[str, FrozenSet[str]] = {\n s.dag_hash(): frozenset([s.dag_hash()]) for s in roots\n }\n\n for edge in spack.traverse.traverse_edges(\n roots, order=\"topo\", cover=\"edges\", root=False, key=spack.traverse.by_dag_hash\n ):\n parent_roots = node_to_roots[edge.parent.dag_hash()]\n child_hash = edge.spec.dag_hash()\n existing = node_to_roots.get(child_hash)\n\n if existing is None:\n node_to_roots[child_hash] = parent_roots # keep a reference if no mutation is needed\n elif not parent_roots.issubset(existing):\n node_to_roots[child_hash] = existing | parent_roots\n\n return node_to_roots\n\n\nclass ReportData:\n \"\"\"Data collected for reports during installation.\"\"\"\n\n def __init__(self, roots: List[spack.spec.Spec]):\n self.roots = roots\n self.build_records: Dict[str, spack.report.InstallRecord] = {}\n\n def start_record(self, spec: spack.spec.Spec) -> None:\n \"\"\"Begin an InstallRecord for a spec that is about to be built.\"\"\"\n if spec.external:\n return\n record = spack.report.InstallRecord(spec)\n record.start()\n self.build_records[spec.dag_hash()] = record\n\n def finish_record(self, spec: spack.spec.Spec, exitcode: int) -> None:\n \"\"\"Mark the InstallRecord for a spec as succeeded or failed.\"\"\"\n record = self.build_records.get(spec.dag_hash())\n if record is None or spec.external:\n return\n if exitcode == 0:\n record.succeed()\n else:\n record.fail(\n spack.error.InstallError(\n f\"Installation of {spec.name} failed; see log for details\"\n )\n )\n\n def finalize(\n self, reports: Dict[str, spack.report.RequestRecord], build_graph: BuildGraph\n ) -> None:\n \"\"\"Finalize InstallRecords and append them to RequestRecords after all builds finish.\n\n Args:\n reports: Map of root dag_hash to RequestRecord to append to.\n build_graph: The build graph containing all nodes and their states.\n \"\"\"\n node_to_roots = _node_to_roots(self.roots)\n\n for spec in spack.traverse.traverse_nodes(self.roots):\n h = spec.dag_hash()\n if h in self.build_records:\n record = self.build_records[h]\n else:\n record = spack.report.InstallRecord(spec)\n if spec.external:\n msg = \"Spec is external\"\n elif h in build_graph.pruned:\n msg = \"Spec was not scheduled for installation\"\n elif h in build_graph.nodes:\n msg = \"Dependencies failed to install\"\n else:\n # If not installed or failed (build_records), not statically pruned ahead of\n # time (build_graph.pruned), and also not scheduled (build_graph.nodes), it\n # means it was in pending_builds or running_builds but never started/finished.\n # This branch is followed on KeyboardInterrupt and --fail-fast.\n msg = \"Installation was interrupted\"\n record.skip(msg=msg)\n\n for root_hash in node_to_roots[h]:\n reports[root_hash].append_record(record)\n\n\nclass NullReportData(ReportData):\n \"\"\"No-op drop-in for ReportData when no reporter is configured.\n\n Avoids creating InstallRecords and reading log files on every completed build.\"\"\"\n\n def __init__(self) -> None:\n pass\n\n def start_record(self, spec: spack.spec.Spec) -> None:\n pass\n\n def finish_record(self, spec: spack.spec.Spec, exitcode: int) -> None:\n pass\n\n def finalize(\n self, reports: Dict[str, spack.report.RequestRecord], build_graph: \"BuildGraph\"\n ) -> None:\n pass\n\n\nclass TerminalState:\n \"\"\"Manages terminal settings, stdin selector registration, and suspend/resume signals.\n\n Installs a SIGTSTP handler that restores the terminal before suspending and re-applies it\n on resume. After waking up it checks whether the process is in the foreground or background\n and enables or suppresses interactive output accordingly.\n\n Optional ``on_suspend`` / ``on_resume`` hooks are called just before the process suspends\n and just after it wakes, allowing callers to pause and resume child processes.\"\"\"\n\n def __init__(\n self,\n selector: selectors.BaseSelector,\n build_status: BuildStatus,\n on_suspend: Optional[Callable[[], None]] = None,\n on_resume: Optional[Callable[[], None]] = None,\n ) -> None:\n self.selector = selector\n self.build_status = build_status\n self.on_suspend = on_suspend\n self.on_resume = on_resume\n self.old_stdin_settings = termios.tcgetattr(sys.stdin)\n self.sigwinch_r = -1\n self.sigwinch_w = -1\n\n def setup(self) -> None:\n \"\"\"Set cbreak mode, register stdin and signal pipes in the selector.\"\"\"\n\n # SIGWINCH self-pipe (stdout must be a tty too)\n if sys.stdout.isatty():\n self.sigwinch_r, self.sigwinch_w = os.pipe()\n os.set_blocking(self.sigwinch_r, False)\n os.set_blocking(self.sigwinch_w, False)\n self.selector.register(self.sigwinch_r, selectors.EVENT_READ, \"sigwinch\")\n self.old_sigwinch = signal.signal(signal.SIGWINCH, self._handle_sigwinch)\n else:\n self.old_sigwinch = None\n\n self.old_sigtstp = signal.signal(signal.SIGTSTP, self._handle_sigtstp)\n\n # Start correctly depending on whether we're foregrounded or backgrounded\n self.build_status.headless = True\n if not _is_background_tty(sys.stdin):\n self.enter_foreground()\n\n def teardown(self) -> None:\n \"\"\"Restore terminal settings and signal handlers, close pipes.\"\"\"\n with ignore_signal(signal.SIGTTOU):\n termios.tcsetattr(sys.stdin, termios.TCSADRAIN, self.old_stdin_settings)\n\n for sig, old in ((signal.SIGTSTP, self.old_sigtstp), (signal.SIGWINCH, self.old_sigwinch)):\n if old is not None:\n try:\n signal.signal(sig, old)\n except Exception as e:\n spack.llnl.util.tty.debug(f\"Failed to restore signal handler for {sig}: {e}\")\n\n if sys.stdin.fileno() in self.selector.get_map():\n self.selector.unregister(sys.stdin.fileno())\n\n for fd in (self.sigwinch_r, self.sigwinch_w):\n if fd < 0:\n continue\n if fd in self.selector.get_map():\n self.selector.unregister(fd)\n try:\n os.close(fd)\n except Exception as e:\n spack.llnl.util.tty.debug(f\"Failed to close sigwinch pipe {fd}: {e}\")\n\n def _handle_sigtstp(self, signum: int, frame: object) -> None:\n \"\"\"Restore terminal before suspending, then re-install handler after resume.\"\"\"\n\n # Reset so the first redraw after resume doesn't overwrite the shell's\n # prompt / \"$ fg\" line.\n self.build_status.active_area_rows = 0\n\n # Restore terminal so the user's shell works normally while we're stopped.\n with ignore_signal(signal.SIGTTOU):\n termios.tcsetattr(sys.stdin, termios.TCSANOW, self.old_stdin_settings)\n\n # Force headless mode before suspending so that enter_foreground() doesn't\n # exit early when we resume, ensuring terminal settings are re-applied.\n self.build_status.headless = True\n\n # Actually suspend: reset to default handler then re-send SIGTSTP.\n if self.on_suspend is not None:\n self.on_suspend()\n signal.signal(signal.SIGTSTP, signal.SIG_DFL)\n os.kill(os.getpid(), signal.SIGTSTP)\n\n # Execution resumes here after SIGCONT. Re-install our handler.\n signal.signal(signal.SIGTSTP, self._handle_sigtstp)\n\n if self.on_resume is not None:\n self.on_resume()\n self.handle_continue()\n\n def _handle_sigwinch(self, signum: int, frame: object) -> None:\n try:\n os.write(self.sigwinch_w, b\"\\x00\")\n except OSError:\n pass\n\n def enter_foreground(self) -> None:\n \"\"\"Restore interactive terminal mode.\"\"\"\n if not self.build_status.headless:\n return\n\n # We save old settings right before applying cbreak.\n # If we started in the background, bash may have had the terminal in its own\n # readline (raw) mode when __init__ ran. Waiting until we are foregrounded\n # ensures we capture the shell's exported 'sane' configuration for this job.\n self.old_stdin_settings = termios.tcgetattr(sys.stdin)\n\n with ignore_signal(signal.SIGTTOU):\n tty.setcbreak(sys.stdin.fileno())\n\n if sys.stdin.fileno() not in self.selector.get_map():\n self.selector.register(sys.stdin.fileno(), selectors.EVENT_READ, \"stdin\")\n self.build_status.headless = False\n self.build_status.dirty = True\n\n def enter_background(self) -> None:\n \"\"\"Suppress output and stop reading stdin to avoid SIGTTIN/SIGTTOU.\"\"\"\n if sys.stdin.fileno() in self.selector.get_map():\n self.selector.unregister(sys.stdin.fileno())\n self.build_status.headless = True\n\n def handle_continue(self) -> None:\n \"\"\"Detect whether the process is in the foreground or background and adjust accordingly.\"\"\"\n if _is_background_tty(sys.stdin):\n self.enter_background()\n else:\n self.enter_foreground()\n\n\ndef _signal_children(running_builds: Dict[int, ChildInfo], sig: signal.Signals) -> None:\n \"\"\"Send a signal to the process group of each running build.\"\"\"\n for child in running_builds.values():\n try:\n pid = child.proc.pid\n if pid is not None:\n os.killpg(pid, sig)\n except OSError:\n pass\n\n\nclass StdinReader:\n \"\"\"Helper class to do non-blocking, incremental decoding of stdin, stripping ANSI escape\n sequences. The input is the backing file descriptor for stdin (instead of the TextIOWrapper) to\n avoid double buffering issues: the event loop triggers when the fd is ready to read, and if we\n do a partial read from the TextIOWrapper, it will likely drain the fd and buffer the remainder\n internally, which the event loop is not aware of, and user input doesn't come through.\"\"\"\n\n def __init__(self, fd: int) -> None:\n self.fd = fd\n #: Handle multi-byte UTF-8 characters\n self.decoder = codecs.getincrementaldecoder(\"utf-8\")(errors=\"replace\")\n #: For stripping out arrow and navigation keys\n self.ansi_escape_re = re.compile(r\"\\x1b\\[[0-9;]*[A-Za-z~]\")\n\n def read(self) -> str:\n try:\n chars = self.decoder.decode(os.read(self.fd, 1024))\n return self.ansi_escape_re.sub(\"\", chars)\n except OSError:\n return \"\"\n\n\nclass PackageInstaller:\n explicit: Set[str]\n\n def __init__(\n self,\n packages: List[\"spack.package_base.PackageBase\"],\n *,\n dirty: bool = False,\n explicit: Union[Set[str], bool] = False,\n overwrite: Optional[Union[List[str], Set[str]]] = None,\n fail_fast: bool = False,\n fake: bool = False,\n include_build_deps: bool = False,\n install_deps: bool = True,\n install_package: bool = True,\n install_source: bool = False,\n keep_prefix: bool = False,\n keep_stage: bool = False,\n restage: bool = True,\n skip_patch: bool = False,\n stop_at: Optional[str] = None,\n stop_before: Optional[str] = None,\n tests: Union[bool, List[str], Set[str]] = False,\n unsigned: Optional[bool] = None,\n verbose: bool = False,\n concurrent_packages: Optional[int] = None,\n root_policy: InstallPolicy = \"auto\",\n dependencies_policy: InstallPolicy = \"auto\",\n create_reports: bool = False,\n ) -> None:\n assert install_package or install_deps, \"Must install package, dependencies or both\"\n\n self.install_source = install_source\n self.stop_at = stop_at\n self.stop_before = stop_before\n self.tests: Union[bool, List[str], Set[str]] = tests\n\n self.db = spack.store.STORE.db\n\n specs = [pkg.spec for pkg in packages]\n\n self.root_policy: InstallPolicy = root_policy\n self.dependencies_policy: InstallPolicy = dependencies_policy\n self.include_build_deps = include_build_deps\n #: Set of DAG hashes to overwrite (if already installed)\n self.overwrite: Set[str] = set(overwrite) if overwrite else set()\n #: Time at which the overwrite install was requested; used to detect concurrent overwrites.\n self.overwrite_time: float = time.time()\n self.keep_prefix = keep_prefix\n self.fail_fast = fail_fast\n\n # Buffer for incoming, partially received state data from child processes\n self.state_buffers: Dict[int, str] = {}\n\n if explicit is True:\n self.explicit = {spec.dag_hash() for spec in specs}\n elif explicit is False:\n self.explicit = set()\n else:\n self.explicit = explicit\n\n # Build the dependency graph\n self.build_graph = BuildGraph(\n specs,\n root_policy,\n dependencies_policy,\n include_build_deps,\n install_package,\n install_deps,\n self.db,\n self.overwrite,\n tests,\n self.explicit,\n )\n\n #: check what specs we could fetch from binaries (checks against cache, not remotely)\n try:\n spack.binary_distribution.BINARY_INDEX.update()\n except spack.binary_distribution.FetchCacheError:\n pass\n\n self.binary_cache_for_spec = {\n s.dag_hash(): spack.binary_distribution.BINARY_INDEX.find_by_hash(s.dag_hash())\n for s in self.build_graph.nodes.values()\n }\n self.unsigned = unsigned\n self.dirty = dirty\n self.fake = fake\n self.restage = restage\n self.keep_stage = keep_stage\n self.skip_patch = skip_patch\n\n #: queue of packages ready to install (no children)\n self.pending_builds = [\n parent for parent, children in self.build_graph.parent_to_child.items() if not children\n ]\n\n self.verbose = verbose\n self.running_builds: Dict[int, ChildInfo] = {}\n self.log_paths: Dict[str, str] = {}\n self.build_status = BuildStatus(\n len(self.build_graph.nodes),\n verbose=verbose,\n filter_padding=spack.store.STORE.has_padding(),\n )\n self.jobs = spack.config.determine_number_of_jobs(parallel=True)\n self.build_status.actual_jobs = self.jobs\n self.build_status.target_jobs = self.jobs\n if concurrent_packages is None:\n concurrent_packages_config = spack.config.get(\"config:concurrent_packages\", 0)\n # The value 0 in config means no limit (other than self.jobs)\n if concurrent_packages_config == 0:\n self.capacity = sys.maxsize\n else:\n self.capacity = concurrent_packages_config\n else:\n self.capacity = concurrent_packages\n\n # The reports property is what the old installer has and used as public interface.\n if create_reports:\n self.reports = {spec.dag_hash(): spack.report.RequestRecord(spec) for spec in specs}\n self.report_data = ReportData(specs)\n else:\n self.reports = {}\n self.report_data = NullReportData()\n\n self.next_database_write = 0.0\n\n def install(self) -> None:\n self._installer()\n\n def _installer(self) -> None:\n jobserver = JobServer(self.jobs)\n selector = selectors.DefaultSelector()\n\n # Set up terminal handling (cbreak, signals, stdin registration)\n terminal: Optional[TerminalState] = None\n stdin_reader: Optional[StdinReader] = None\n if sys.stdin.isatty():\n stdin_reader = StdinReader(sys.stdin.fileno())\n terminal = TerminalState(\n selector,\n self.build_status,\n on_suspend=lambda: _signal_children(self.running_builds, signal.SIGSTOP),\n on_resume=lambda: _signal_children(self.running_builds, signal.SIGCONT),\n )\n terminal.setup()\n\n # Finished builds that have not yet been written to the database.\n database_actions: List[DatabaseAction] = []\n # Prefix read locks retained after DB flush (downgraded from write locks in _save_to_db).\n retained_read_locks: List[spack.util.lock.Lock] = []\n\n failures: List[spack.spec.Spec] = []\n finished_pids: List[int] = []\n\n try:\n # Try to schedule builds immediately. The first job does not require a token.\n blocked = self._schedule_builds(\n selector, jobserver, retained_read_locks, database_actions\n )\n\n while self.pending_builds or self.running_builds or database_actions:\n # Monitor the jobserver when we have pending builds, capacity, and at least one\n # spec is not locked by another process. Also listen if the target parallelism is\n # reduced.\n wake_on_jobserver = (\n self.pending_builds\n and self.capacity\n and not blocked\n or not jobserver.has_target_parallelism()\n )\n if wake_on_jobserver and jobserver.r not in selector.get_map():\n selector.register(jobserver.r, selectors.EVENT_READ, \"jobserver\")\n elif not wake_on_jobserver and jobserver.r in selector.get_map():\n selector.unregister(jobserver.r)\n\n stdin_ready = False\n\n if self.build_status.headless:\n # no UI to update, but check background to foreground transition periodically\n timeout = HEADLESS_WAKE_INTERVAL\n elif self.build_status.is_tty:\n timeout = SPINNER_INTERVAL\n else:\n # when not in interactive mode, wake least often (no spinner/terminal updates)\n timeout = DATABASE_WRITE_INTERVAL\n events = selector.select(timeout=timeout)\n\n finished_pids.clear()\n\n # The transition \"suspended to foreground/background\" is handled in the signal\n # handler, but there's no SIGCONT event in the transition of background to\n # foreground, so we conditionally poll for that here (headless case). In the\n # headless case the event loop only fires once per second, so this is cheap enough.\n if terminal and self.build_status.headless and not _is_background_tty(sys.stdin):\n terminal.enter_foreground()\n\n for key, _ in events:\n data = key.data\n if isinstance(data, FdInfo):\n # Child output (logs and state updates)\n child_info = self.running_builds[data.pid]\n if data.name == \"output\":\n self._handle_child_logs(key.fd, child_info, selector)\n elif data.name == \"state\":\n self._handle_child_state(key.fd, child_info, selector)\n elif data.name == \"sentinel\":\n finished_pids.append(data.pid)\n elif data == \"stdin\":\n stdin_ready = True\n elif data == \"sigwinch\":\n assert terminal is not None\n os.read(terminal.sigwinch_r, 64) # drain the pipe\n self.build_status.on_resize()\n elif data == \"jobserver\" and not jobserver.has_target_parallelism():\n jobserver.maybe_discard_tokens()\n self.build_status.set_jobs(jobserver.num_jobs, jobserver.target_jobs)\n\n if finished_pids:\n self._handle_finished_builds(\n finished_pids, selector, jobserver, database_actions, failures\n )\n\n if failures and self.fail_fast:\n # Terminate other builds to actually fail fast. We continue in the event loop\n # waiting for child processes to finish, which may take a little while.\n for child in self.running_builds.values():\n child.proc.terminate()\n self.pending_builds.clear()\n\n if stdin_ready and stdin_reader is not None:\n for char in stdin_reader.read():\n overview = self.build_status.overview_mode\n if overview and self.build_status.search_mode:\n self.build_status.search_input(char)\n elif overview and char == \"/\":\n self.build_status.enter_search()\n elif char == \"v\" or char in (\"q\", \"\\x1b\") and not overview:\n self.build_status.toggle()\n elif char == \"n\":\n self.build_status.next(1)\n elif char == \"p\" or char == \"N\":\n self.build_status.next(-1)\n elif char == \"+\":\n jobserver.increase_parallelism()\n self.build_status.set_jobs(jobserver.num_jobs, jobserver.target_jobs)\n elif char == \"-\":\n jobserver.decrease_parallelism()\n self.build_status.set_jobs(jobserver.num_jobs, jobserver.target_jobs)\n\n # Insert into the database if we have any finished builds, and either the delay\n # interval has passed, or we're done with all builds. The database save is not\n # guaranteed; it fails if another process holds the lock. We'll try again next\n # iteration of the event loop in that case.\n if (\n database_actions\n and (\n time.monotonic() >= self.next_database_write\n or not (self.pending_builds or self.running_builds)\n )\n and self._save_to_db(database_actions, retained_read_locks)\n ):\n database_actions.clear()\n\n # Try to schedule more builds, acquiring per-spec locks and jobserver tokens.\n if self.capacity and self.pending_builds:\n blocked = self._schedule_builds(\n selector, jobserver, retained_read_locks, database_actions\n )\n\n # Finally update the UI\n self.build_status.update()\n finally:\n # First ensure that the user's terminal state is restored.\n if terminal is not None:\n terminal.teardown()\n\n # Flush any not-yet-written successful builds to the DB; save the exception on error\n # to be re-raised after best-effort cleanup.\n db_exc = None\n if database_actions:\n try:\n with self.db.write_transaction():\n for action in database_actions:\n action.save_to_db(self.db)\n except Exception as e:\n db_exc = e\n\n # Send SIGTERM to running builds; this is a no-op in the successful case.\n for child in self.running_builds.values():\n try:\n child.proc.terminate()\n except Exception:\n pass\n\n # Release our jobserver token for each terminated build and then join.\n for child in self.running_builds.values():\n try:\n jobserver.release()\n child.proc.join(timeout=30)\n if child.proc.is_alive():\n child.proc.kill()\n child.proc.join()\n except Exception:\n pass\n\n # Release all held locks best-effort, so that one failure does not prevent the others\n # from being released.\n for child in self.running_builds.values():\n child.release_lock()\n\n for lock in retained_read_locks:\n try:\n lock.release_read()\n except Exception:\n pass\n for action in database_actions:\n action.release_lock()\n\n try:\n self.build_status.overview_mode = True\n self.build_status.update(finalize=True)\n selector.close()\n jobserver.close()\n except Exception:\n pass\n\n # Re-raise the DB exception if any.\n if db_exc is not None:\n raise db_exc\n\n try:\n self.report_data.finalize(self.reports, build_graph=self.build_graph)\n except Exception as e:\n spack.llnl.util.tty.debug(f\"[{__name__}]: Failed to finalize reports: {e}]\")\n\n if failures:\n for s in failures:\n build_info = self.build_status.builds[s.dag_hash()]\n if build_info and build_info.log_summary:\n sys.stderr.write(build_info.log_summary)\n lines = [f\"{s}: {self.log_paths[s.dag_hash()]}\" for s in failures]\n raise spack.error.InstallError(\n \"The following packages failed to install:\\n\" + \"\\n\".join(lines)\n )\n\n def _handle_finished_builds(\n self,\n finished_pids: List[int],\n selector: selectors.BaseSelector,\n jobserver: JobServer,\n database_actions: List[DatabaseAction],\n failures: List[spack.spec.Spec],\n ) -> None:\n \"\"\"Handle builds that finished since the last event loop iteration.\"\"\"\n current_time = time.monotonic()\n for pid in finished_pids:\n build = self.running_builds.pop(pid)\n self.capacity += 1\n jobserver.release()\n self.build_status.set_jobs(jobserver.num_jobs, jobserver.target_jobs)\n self._drain_child_output(build, selector)\n self._drain_child_state(build, selector)\n build.cleanup(selector)\n exitcode = build.proc.exitcode\n assert exitcode is not None, \"Finished build should have exit code set\"\n self.report_data.finish_record(build.spec, exitcode)\n if exitcode == 0:\n # Add successful builds for database insertion (after a short delay)\n database_actions.append(build)\n self.build_graph.enqueue_parents(build.spec.dag_hash(), self.pending_builds)\n self.next_database_write = current_time + DATABASE_WRITE_INTERVAL\n self.build_status.update_state(build.spec.dag_hash(), \"finished\")\n elif exitcode == EXIT_STOPPED_AT_PHASE:\n # Partial build: neither failure nor success. Should not be persisted in\n # the database, but also not treated as a failure in the UI. Just release\n # locks and move on.\n build.release_lock()\n elif not self.fail_fast or not failures:\n # In fail-fast mode, only record the first failure. Subsequent failures may\n # be a consequence of us terminating other builds, and should not be\n # reported as failures in the UI.\n failures.append(build.spec)\n self.build_status.update_state(build.spec.dag_hash(), \"failed\")\n self.build_status.parse_log_summary(build.spec.dag_hash())\n\n def _save_to_db(\n self,\n database_actions: List[DatabaseAction],\n retained_read_locks: List[spack.util.lock.Lock],\n ) -> bool:\n if not self.db.lock.try_acquire_write():\n return False\n try:\n self.db._read()\n for action in database_actions:\n action.save_to_db(self.db)\n finally:\n self.db.lock.release_write(self.db._write)\n\n # DB has been written and flushed; downgrade per-spec prefix write locks to read locks so\n # other processes can see the specs are installed, while preventing concurrent uninstalls.\n for action in database_actions:\n if action.prefix_lock is not None:\n try:\n action.prefix_lock.downgrade_write_to_read()\n retained_read_locks.append(action.prefix_lock)\n except Exception:\n action.prefix_lock.release_write()\n raise\n finally:\n action.prefix_lock = None\n\n return True\n\n def _schedule_builds(\n self,\n selector: selectors.BaseSelector,\n jobserver: JobServer,\n retained_read_locks: List[spack.util.lock.Lock],\n database_actions: List[DatabaseAction],\n ) -> bool:\n \"\"\"Try to schedule as many pending builds as possible.\n\n Delegates to the module-level schedule_builds() function and then performs the\n side-effects that require the selector and running-build state: updating build_status for\n specs that were found already installed, and launching new builds via _start().\n\n Preconditions: self.capacity > 0 and self.pending_builds is not empty.\n\n Returns True if we had capacity to schedule, but were blocked by locks held by other\n processes. In that case we should not monitor the jobserver for new tokens, since we'd end\n up in a busy wait loop until the locks are released.\n \"\"\"\n result = schedule_builds(\n pending=self.pending_builds,\n build_graph=self.build_graph,\n db=self.db,\n prefix_locker=spack.store.STORE.prefix_locker,\n overwrite=self.overwrite,\n overwrite_time=self.overwrite_time,\n capacity=self.capacity,\n needs_jobserver_token=bool(self.running_builds),\n jobserver=jobserver,\n explicit=self.explicit,\n )\n blocked = result.blocked\n database_actions.extend(result.to_mark_explicit)\n # Specs installed by another process.\n for dag_hash, spec, lock in result.newly_installed:\n retained_read_locks.append(lock)\n explicit = dag_hash in self.explicit\n self.build_status.add_build(spec, explicit=explicit)\n self.build_status.update_state(dag_hash, \"finished\")\n # Specs we can start building ourselves.\n for dag_hash, lock in result.to_start:\n self._start(selector, jobserver, dag_hash, lock)\n return blocked\n\n def _start(\n self,\n selector: selectors.BaseSelector,\n jobserver: JobServer,\n dag_hash: str,\n prefix_lock: spack.util.lock.Lock,\n ) -> None:\n self.capacity -= 1\n explicit = dag_hash in self.explicit\n spec = self.build_graph.nodes[dag_hash]\n is_develop = spec.is_develop\n tests = self.tests\n run_tests = tests is True or bool(tests and spec.name in tests)\n is_root = dag_hash in self.build_graph.roots\n child_info = start_build(\n spec,\n explicit=explicit,\n mirrors=self.binary_cache_for_spec[dag_hash],\n unsigned=self.unsigned,\n install_policy=self.root_policy if is_root else self.dependencies_policy,\n dirty=self.dirty,\n # keep_stage/restage logic taken from installer.py\n keep_stage=self.keep_stage or is_develop,\n restage=self.restage and not is_develop,\n keep_prefix=self.keep_prefix,\n skip_patch=self.skip_patch,\n fake=self.fake,\n install_source=self.install_source,\n run_tests=run_tests,\n jobserver=jobserver,\n stop_before=self.stop_before if is_root else None,\n stop_at=self.stop_at if is_root else None,\n )\n self.log_paths[dag_hash] = child_info.log_path\n child_info.prefix_lock = prefix_lock\n pid = child_info.proc.pid\n assert type(pid) is int\n self.running_builds[pid] = child_info\n selector.register(\n child_info.output_r_conn.fileno(), selectors.EVENT_READ, FdInfo(pid, \"output\")\n )\n selector.register(\n child_info.state_r_conn.fileno(), selectors.EVENT_READ, FdInfo(pid, \"state\")\n )\n selector.register(child_info.proc.sentinel, selectors.EVENT_READ, FdInfo(pid, \"sentinel\"))\n self.build_status.add_build(\n child_info.spec,\n explicit=explicit,\n control_w_conn=child_info.control_w_conn,\n log_path=child_info.log_path,\n )\n self.report_data.start_record(spec)\n\n def _handle_child_logs(\n self, r_fd: int, child_info: ChildInfo, selector: selectors.BaseSelector\n ) -> None:\n \"\"\"Handle reading output logs from a child process pipe.\"\"\"\n try:\n # There might be more data than OUTPUT_BUFFER_SIZE, but we will read that in the next\n # iteration of the event loop to keep things responsive.\n data = os.read(r_fd, OUTPUT_BUFFER_SIZE)\n except BlockingIOError:\n return\n except OSError:\n data = None\n\n if not data: # EOF or error\n try:\n selector.unregister(r_fd)\n except KeyError:\n pass\n return\n\n self.build_status.print_logs(child_info.spec.dag_hash(), data)\n\n def _drain_child_output(self, child_info: ChildInfo, selector: selectors.BaseSelector) -> None:\n \"\"\"Read and print any remaining output from a finished child's pipe.\"\"\"\n r_fd = child_info.output_r_conn.fileno()\n while r_fd in selector.get_map():\n self._handle_child_logs(r_fd, child_info, selector)\n\n def _drain_child_state(self, child_info: ChildInfo, selector: selectors.BaseSelector) -> None:\n \"\"\"Read and process any remaining state messages from a finished child's pipe.\"\"\"\n r_fd = child_info.state_r_conn.fileno()\n while r_fd in selector.get_map():\n self._handle_child_state(r_fd, child_info, selector)\n\n def _handle_child_state(\n self, r_fd: int, child_info: ChildInfo, selector: selectors.BaseSelector\n ) -> None:\n \"\"\"Handle reading state updates from a child process pipe.\"\"\"\n try:\n # There might be more data than OUTPUT_BUFFER_SIZE, but we will read that in the next\n # iteration of the event loop to keep things responsive.\n data = os.read(r_fd, OUTPUT_BUFFER_SIZE)\n except BlockingIOError:\n return\n except OSError:\n data = None\n\n if not data: # EOF or error\n try:\n selector.unregister(r_fd)\n except KeyError:\n pass\n self.state_buffers.pop(r_fd, None)\n return\n\n # Append new data to the buffer for this fd and process it\n buffer = self.state_buffers.get(r_fd, \"\") + data.decode(errors=\"replace\")\n lines = buffer.split(\"\\n\")\n\n # The last element of split() will be a partial line or an empty string.\n # We store it back in the buffer for the next read.\n self.state_buffers[r_fd] = lines.pop()\n\n for line in lines:\n if not line:\n continue\n try:\n message = json.loads(line)\n except json.JSONDecodeError:\n continue\n if \"state\" in message:\n self.build_status.update_state(child_info.spec.dag_hash(), message[\"state\"])\n elif \"progress\" in message and \"total\" in message:\n self.build_status.update_progress(\n child_info.spec.dag_hash(), message[\"progress\"], message[\"total\"]\n )\n elif \"installed_from_binary_cache\" in message:\n child_info.spec.package.installed_from_binary_cache = True\n" }, { "path": "lib/spack/spack/oci/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n" }, { "path": "lib/spack/spack/oci/image.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport re\nimport urllib.parse\nfrom typing import Optional, Union\n\n# notice: Docker is more strict (no uppercase allowed). We parse image names *with* uppercase\n# and normalize, so: example.com/Organization/Name -> example.com/organization/name. Tags are\n# case sensitive though.\nalphanumeric_with_uppercase = r\"[a-zA-Z0-9]+\"\nseparator = r\"(?:[._]|__|[-]+)\"\nlocalhost = r\"localhost\"\ndomainNameComponent = r\"(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])\"\noptionalPort = r\"(?::[0-9]+)?\"\ntag = r\"[\\w][\\w.-]{0,127}\"\ndigestPat = r\"[A-Za-z][A-Za-z0-9]*(?:[-_+.][A-Za-z][A-Za-z0-9]*)*[:][0-9a-fA-F]{32,}\"\nipv6address = r\"\\[(?:[a-fA-F0-9:]+)\\]\"\n\n# domain name\ndomainName = rf\"{domainNameComponent}(?:\\.{domainNameComponent})*\"\nhost = rf\"(?:{domainName}|{ipv6address})\"\ndomainAndPort = rf\"{host}{optionalPort}\"\n\n# image name\npathComponent = rf\"{alphanumeric_with_uppercase}(?:{separator}{alphanumeric_with_uppercase})*\"\nremoteName = rf\"{pathComponent}(?:\\/{pathComponent})*\"\nnamePat = rf\"(?:{domainAndPort}\\/)?{remoteName}\"\n\n# Regex for a full image reference, with 3 groups: name, tag, digest\nreferencePat = re.compile(rf\"^({namePat})(?::({tag}))?(?:@({digestPat}))?$\")\n\n# Regex for splitting the name into domain and path components\nanchoredNameRegexp = re.compile(rf\"^(?:({domainAndPort})\\/)?({remoteName})$\")\n\n\ndef ensure_sha256_checksum(oci_blob: str):\n \"\"\"Validate that the reference is of the format sha256:\n Return the checksum if valid, raise ValueError otherwise.\"\"\"\n if \":\" not in oci_blob:\n raise ValueError(f\"Invalid OCI blob format: {oci_blob}\")\n alg, checksum = oci_blob.split(\":\", 1)\n if alg != \"sha256\":\n raise ValueError(f\"Unsupported OCI blob checksum algorithm: {alg}\")\n if len(checksum) != 64:\n raise ValueError(f\"Invalid OCI blob checksum length: {len(checksum)}\")\n return checksum\n\n\ndef is_oci_url(url: str) -> bool:\n \"\"\"Check if the URL is an OCI URL.\"\"\"\n return url.startswith(\"oci://\") or url.startswith(\"oci+http://\")\n\n\nclass Digest:\n \"\"\"Represents a digest in the format :.\n Currently only supports sha256 digests.\"\"\"\n\n __slots__ = [\"algorithm\", \"digest\"]\n\n def __init__(self, *, algorithm: str, digest: str) -> None:\n self.algorithm = algorithm\n self.digest = digest\n\n def __eq__(self, __value: object) -> bool:\n if not isinstance(__value, Digest):\n return NotImplemented\n return self.algorithm == __value.algorithm and self.digest == __value.digest\n\n @classmethod\n def from_string(cls, string: str) -> \"Digest\":\n return cls(algorithm=\"sha256\", digest=ensure_sha256_checksum(string))\n\n @classmethod\n def from_sha256(cls, digest: str) -> \"Digest\":\n return cls(algorithm=\"sha256\", digest=digest)\n\n def __str__(self) -> str:\n return f\"{self.algorithm}:{self.digest}\"\n\n\nclass ImageReference:\n \"\"\"A parsed image of the form domain/name:tag[@digest].\n The digest is optional, and domain and tag are automatically\n filled out with defaults when parsed from string.\"\"\"\n\n __slots__ = [\"scheme\", \"domain\", \"name\", \"tag\", \"digest\"]\n\n def __init__(\n self,\n *,\n domain: str,\n name: str,\n tag: str = \"latest\",\n digest: Optional[Digest] = None,\n scheme: str = \"https\",\n ):\n self.scheme = scheme\n self.domain = domain\n self.name = name\n self.tag = tag\n self.digest = digest\n\n @classmethod\n def from_string(cls, string: str, *, scheme: str = \"https\") -> \"ImageReference\":\n match = referencePat.match(string)\n if not match:\n raise ValueError(f\"Invalid image reference: {string}\")\n\n image, tag, digest = match.groups()\n\n assert isinstance(image, str)\n assert isinstance(tag, (str, type(None)))\n assert isinstance(digest, (str, type(None)))\n\n match = anchoredNameRegexp.match(image)\n\n # This can never happen, since the regex is implied\n # by the regex above. It's just here to make mypy happy.\n assert match, f\"Invalid image reference: {string}\"\n\n domain, name = match.groups()\n\n assert isinstance(domain, (str, type(None)))\n assert isinstance(name, str)\n\n # Fill out defaults like docker would do...\n # Based on github.com/distribution/distribution: allow short names like \"ubuntu\"\n # and \"user/repo\" to be interpreted as \"library/ubuntu\" and \"user/repo:latest\n # Not sure if Spack should follow Docker, but it's what people expect...\n if not domain:\n domain = \"index.docker.io\"\n name = f\"library/{name}\"\n elif (\n \".\" not in domain\n and \":\" not in domain\n and domain != \"localhost\"\n and domain == domain.lower()\n ):\n name = f\"{domain}/{name}\"\n domain = \"index.docker.io\"\n\n # Lowercase the image name. This is enforced by Docker, although the OCI spec isn't clear?\n # We do this anyways, cause for example in Github Actions the /\n # part can have uppercase, and may be interpolated when specifying the relevant OCI image.\n name = name.lower()\n\n if not tag:\n tag = \"latest\"\n\n # sha256 is currently the only algorithm that\n # we implement, even though the spec allows for more\n if isinstance(digest, str):\n digest = Digest.from_string(digest)\n\n return cls(domain=domain, name=name, tag=tag, digest=digest, scheme=scheme)\n\n @classmethod\n def from_url(cls, url: str) -> \"ImageReference\":\n \"\"\"Parse an OCI URL into an ImageReference, either oci:// or oci+http://.\"\"\"\n if url.startswith(\"oci://\"):\n img = url[6:]\n scheme = \"https\"\n elif url.startswith(\"oci+http://\"):\n img = url[11:]\n scheme = \"http\"\n else:\n raise ValueError(f\"Invalid OCI URL: {url}\")\n return cls.from_string(img, scheme=scheme)\n\n def manifest_url(self) -> str:\n digest_or_tag = self.digest or self.tag\n return f\"{self.scheme}://{self.domain}/v2/{self.name}/manifests/{digest_or_tag}\"\n\n def blob_url(self, digest: Union[str, Digest]) -> str:\n if isinstance(digest, str):\n digest = Digest.from_string(digest)\n return f\"{self.scheme}://{self.domain}/v2/{self.name}/blobs/{digest}\"\n\n def with_digest(self, digest: Union[str, Digest]) -> \"ImageReference\":\n if isinstance(digest, str):\n digest = Digest.from_string(digest)\n return ImageReference(\n domain=self.domain, name=self.name, tag=self.tag, digest=digest, scheme=self.scheme\n )\n\n def with_tag(self, tag: str) -> \"ImageReference\":\n return ImageReference(\n domain=self.domain, name=self.name, tag=tag, digest=self.digest, scheme=self.scheme\n )\n\n def uploads_url(self, digest: Optional[Digest] = None) -> str:\n url = f\"{self.scheme}://{self.domain}/v2/{self.name}/blobs/uploads/\"\n if digest:\n url += f\"?digest={digest}\"\n return url\n\n def tags_url(self) -> str:\n return f\"{self.scheme}://{self.domain}/v2/{self.name}/tags/list\"\n\n def endpoint(self, path: str = \"\") -> str:\n return urllib.parse.urljoin(f\"{self.scheme}://{self.domain}/v2/\", path)\n\n def __str__(self) -> str:\n s = f\"{self.domain}/{self.name}\"\n if self.tag:\n s += f\":{self.tag}\"\n if self.digest:\n s += f\"@{self.digest}\"\n return s\n\n def __eq__(self, __value: object) -> bool:\n if not isinstance(__value, ImageReference):\n return NotImplemented\n return (\n self.domain == __value.domain\n and self.name == __value.name\n and self.tag == __value.tag\n and self.digest == __value.digest\n and self.scheme == __value.scheme\n )\n\n\ndef ensure_valid_tag(tag: str) -> str:\n \"\"\"Ensure a tag is valid for an OCI registry.\"\"\"\n sanitized = re.sub(r\"[^\\w.-]\", \"_\", tag)\n if len(sanitized) > 128:\n return sanitized[:64] + sanitized[-64:]\n return sanitized\n\n\ndef default_config(architecture: str, os: str):\n return {\n \"architecture\": architecture,\n \"os\": os,\n \"rootfs\": {\"type\": \"layers\", \"diff_ids\": []},\n \"config\": {\"Env\": []},\n }\n\n\ndef default_manifest():\n return {\n \"mediaType\": \"application/vnd.oci.image.manifest.v1+json\",\n \"schemaVersion\": 2,\n \"config\": {\"mediaType\": \"application/vnd.oci.image.config.v1+json\"},\n \"layers\": [],\n }\n" }, { "path": "lib/spack/spack/oci/oci.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport hashlib\nimport json\nimport os\nimport urllib.error\nimport urllib.parse\nfrom typing import List, NamedTuple, Tuple\nfrom urllib.request import Request\n\nimport spack.fetch_strategy\nimport spack.llnl.util.tty as tty\nimport spack.mirrors.layout\nimport spack.mirrors.mirror\nimport spack.oci.opener\nimport spack.stage\nimport spack.util.url\n\nfrom .image import Digest, ImageReference\n\n\nclass Blob(NamedTuple):\n compressed_digest: Digest\n uncompressed_digest: Digest\n size: int\n\n\ndef with_query_param(url: str, param: str, value: str) -> str:\n \"\"\"Add a query parameter to a URL\n\n Args:\n url: The URL to add the parameter to.\n param: The parameter name.\n value: The parameter value.\n\n Returns:\n The URL with the parameter added.\n \"\"\"\n parsed = urllib.parse.urlparse(url)\n query = urllib.parse.parse_qs(parsed.query)\n if param in query:\n query[param].append(value)\n else:\n query[param] = [value]\n return urllib.parse.urlunparse(\n parsed._replace(query=urllib.parse.urlencode(query, doseq=True))\n )\n\n\ndef list_tags(ref: ImageReference, _urlopen: spack.oci.opener.MaybeOpen = None) -> List[str]:\n \"\"\"Retrieves the list of tags associated with an image, handling pagination.\"\"\"\n _urlopen = _urlopen or spack.oci.opener.urlopen\n tags = set()\n fetch_url = ref.tags_url()\n\n while True:\n # Fetch tags\n request = Request(url=fetch_url)\n with _urlopen(request) as response:\n spack.oci.opener.ensure_status(request, response, 200)\n tags.update(json.load(response)[\"tags\"])\n\n # Check for pagination\n link_header = response.headers[\"Link\"]\n\n if link_header is None:\n break\n\n tty.debug(f\"OCI tag pagination: {link_header}\")\n\n rel_next_value = spack.util.url.parse_link_rel_next(link_header)\n\n if rel_next_value is None:\n break\n\n rel_next = urllib.parse.urlparse(rel_next_value)\n\n if rel_next.scheme not in (\"https\", \"\"):\n break\n\n fetch_url = ref.endpoint(rel_next_value)\n\n return sorted(tags)\n\n\ndef upload_blob(\n ref: ImageReference,\n file: str,\n digest: Digest,\n force: bool = False,\n small_file_size: int = 0,\n _urlopen: spack.oci.opener.MaybeOpen = None,\n) -> bool:\n \"\"\"Uploads a blob to an OCI registry\n\n We only do monolithic uploads, even though it's very simple to do chunked.\n Observed problems with chunked uploads:\n (1) it's slow, many sequential requests, (2) some registries set an *unknown*\n max chunk size, and the spec doesn't say how to obtain it\n\n Args:\n ref: The image reference.\n file: The file to upload.\n digest: The digest of the file.\n force: Whether to force upload the blob, even if it already exists.\n small_file_size: For files at most this size, attempt\n to do a single POST request instead of POST + PUT.\n Some registries do no support single requests, and others\n do not specify what size they support in single POST.\n For now this feature is disabled by default (0KB)\n\n Returns:\n True if the blob was uploaded, False if it already existed.\n \"\"\"\n _urlopen = _urlopen or spack.oci.opener.urlopen\n\n # Test if the blob already exists, if so, early exit.\n if not force and blob_exists(ref, digest, _urlopen):\n return False\n\n with open(file, \"rb\") as f:\n file_size = os.fstat(f.fileno()).st_size\n\n # For small blobs, do a single POST request.\n # The spec says that registries MAY support this\n if file_size <= small_file_size:\n request = Request(\n url=ref.uploads_url(digest),\n method=\"POST\",\n data=f,\n headers={\n \"Content-Type\": \"application/octet-stream\",\n \"Content-Length\": str(file_size),\n },\n )\n else:\n request = Request(\n url=ref.uploads_url(), method=\"POST\", headers={\"Content-Length\": \"0\"}\n )\n\n with _urlopen(request) as response:\n # Created the blob in one go.\n if response.status == 201:\n return True\n\n # Otherwise, do another PUT request.\n spack.oci.opener.ensure_status(request, response, 202)\n assert \"Location\" in response.headers\n\n # Can be absolute or relative, joining handles both\n upload_url = with_query_param(\n ref.endpoint(response.headers[\"Location\"]), \"digest\", str(digest)\n )\n\n f.seek(0)\n\n request = Request(\n url=upload_url,\n method=\"PUT\",\n data=f,\n headers={\"Content-Type\": \"application/octet-stream\", \"Content-Length\": str(file_size)},\n )\n\n with _urlopen(request) as response:\n spack.oci.opener.ensure_status(request, response, 201)\n\n return True\n\n\ndef upload_manifest(\n ref: ImageReference,\n manifest: dict,\n tag: bool = True,\n _urlopen: spack.oci.opener.MaybeOpen = None,\n):\n \"\"\"Uploads a manifest/index to a registry\n\n Args:\n ref: The image reference.\n manifest: The manifest or index.\n tag: When true, use the tag, otherwise use the digest,\n this is relevant for multi-arch images, where the\n tag is an index, referencing the manifests by digest.\n\n Returns:\n The digest and size of the uploaded manifest.\n \"\"\"\n _urlopen = _urlopen or spack.oci.opener.urlopen\n\n data = json.dumps(manifest, separators=(\",\", \":\")).encode()\n digest = Digest.from_sha256(hashlib.sha256(data).hexdigest())\n size = len(data)\n\n if not tag:\n ref = ref.with_digest(digest)\n\n request = Request(\n url=ref.manifest_url(),\n method=\"PUT\",\n data=data,\n headers={\"Content-Type\": manifest[\"mediaType\"]},\n )\n\n with _urlopen(request) as response:\n spack.oci.opener.ensure_status(request, response, 201)\n return digest, size\n\n\ndef image_from_mirror(mirror: spack.mirrors.mirror.Mirror) -> ImageReference:\n \"\"\"Given an OCI based mirror, extract the URL and image name from it\"\"\"\n return ImageReference.from_url(mirror.push_url)\n\n\ndef blob_exists(\n ref: ImageReference, digest: Digest, _urlopen: spack.oci.opener.MaybeOpen = None\n) -> bool:\n \"\"\"Checks if a blob exists in an OCI registry\"\"\"\n try:\n _urlopen = _urlopen or spack.oci.opener.urlopen\n with _urlopen(Request(url=ref.blob_url(digest), method=\"HEAD\")) as response:\n return response.status == 200\n except urllib.error.HTTPError as e:\n if e.getcode() == 404:\n return False\n raise\n\n\ndef copy_missing_layers(\n src: ImageReference,\n dst: ImageReference,\n architecture: str,\n _urlopen: spack.oci.opener.MaybeOpen = None,\n) -> Tuple[dict, dict]:\n \"\"\"Copy image layers from src to dst for given architecture.\n\n Args:\n src: The source image reference.\n dst: The destination image reference.\n architecture: The architecture (when referencing an index)\n\n Returns:\n Tuple of manifest and config of the base image.\n \"\"\"\n _urlopen = _urlopen or spack.oci.opener.urlopen\n manifest, config = get_manifest_and_config(src, architecture, _urlopen=_urlopen)\n\n # Get layer digests\n digests = [Digest.from_string(layer[\"digest\"]) for layer in manifest[\"layers\"]]\n\n # Filter digests that are don't exist in the registry\n missing_digests = [\n digest for digest in digests if not blob_exists(dst, digest, _urlopen=_urlopen)\n ]\n\n if not missing_digests:\n return manifest, config\n\n # Pull missing blobs, push them to the registry\n with spack.stage.StageComposite.from_iterable(\n make_stage(url=src.blob_url(digest), digest=digest, _urlopen=_urlopen)\n for digest in missing_digests\n ) as stages:\n stages.fetch()\n stages.check()\n stages.cache_local()\n\n for stage, digest in zip(stages, missing_digests):\n # No need to check existence again, force=True.\n upload_blob(\n dst, file=stage.save_filename, force=True, digest=digest, _urlopen=_urlopen\n )\n\n return manifest, config\n\n\n#: OCI manifest content types (including docker type)\nmanifest_content_type = [\n \"application/vnd.oci.image.manifest.v1+json\",\n \"application/vnd.docker.distribution.manifest.v2+json\",\n]\n\n#: OCI index content types (including docker type)\nindex_content_type = [\n \"application/vnd.oci.image.index.v1+json\",\n \"application/vnd.docker.distribution.manifest.list.v2+json\",\n]\n\n#: All OCI manifest / index content types\nall_content_type = manifest_content_type + index_content_type\n\n\ndef get_manifest_and_config(\n ref: ImageReference,\n architecture=\"amd64\",\n recurse=3,\n _urlopen: spack.oci.opener.MaybeOpen = None,\n) -> Tuple[dict, dict]:\n \"\"\"Recursively fetch manifest and config for a given image reference\n with a given architecture.\n\n Args:\n ref: The image reference.\n architecture: The architecture (when referencing an index)\n recurse: How many levels of index to recurse into.\n\n Returns:\n A tuple of (manifest, config)\"\"\"\n\n _urlopen = _urlopen or spack.oci.opener.urlopen\n\n # Get manifest\n with _urlopen(\n Request(url=ref.manifest_url(), headers={\"Accept\": \", \".join(all_content_type)})\n ) as response:\n # Recurse when we find an index\n if response.headers[\"Content-Type\"] in index_content_type:\n if recurse == 0:\n raise Exception(\"Maximum recursion depth reached while fetching OCI manifest\")\n\n index = json.load(response)\n manifest_meta = next(\n manifest\n for manifest in index[\"manifests\"]\n if manifest[\"platform\"][\"architecture\"] == architecture\n )\n\n return get_manifest_and_config(\n ref.with_digest(manifest_meta[\"digest\"]),\n architecture=architecture,\n recurse=recurse - 1,\n _urlopen=_urlopen,\n )\n\n # Otherwise, require a manifest\n if response.headers[\"Content-Type\"] not in manifest_content_type:\n raise Exception(f\"Unknown content type {response.headers['Content-Type']}\")\n\n manifest = json.load(response)\n\n # Download, verify and cache config file\n config_digest = Digest.from_string(manifest[\"config\"][\"digest\"])\n with make_stage(ref.blob_url(config_digest), config_digest, _urlopen=_urlopen) as stage:\n stage.fetch()\n stage.check()\n stage.cache_local()\n with open(stage.save_filename, \"rb\") as f:\n config = json.load(f)\n\n return manifest, config\n\n\n#: Same as upload_manifest, but with retry wrapper\nupload_manifest_with_retry = spack.oci.opener.default_retry(upload_manifest)\n\n#: Same as upload_blob, but with retry wrapper\nupload_blob_with_retry = spack.oci.opener.default_retry(upload_blob)\n\n#: Same as get_manifest_and_config, but with retry wrapper\nget_manifest_and_config_with_retry = spack.oci.opener.default_retry(get_manifest_and_config)\n\n#: Same as copy_missing_layers, but with retry wrapper\ncopy_missing_layers_with_retry = spack.oci.opener.default_retry(copy_missing_layers)\n\n\ndef make_stage(\n url: str, digest: Digest, keep: bool = False, _urlopen: spack.oci.opener.MaybeOpen = None\n) -> spack.stage.Stage:\n _urlopen = _urlopen or spack.oci.opener.urlopen\n fetch_strategy = spack.fetch_strategy.OCIRegistryFetchStrategy(\n url=url, checksum=digest.digest, _urlopen=_urlopen\n )\n # Use blobs// as the cache path, which follows\n # the OCI Image Layout Specification. What's missing though,\n # is the `oci-layout` and `index.json` files, which are\n # required by the spec.\n return spack.stage.Stage(\n fetch_strategy,\n mirror_paths=spack.mirrors.layout.OCILayout(digest),\n name=digest.digest,\n keep=keep,\n )\n" }, { "path": "lib/spack/spack/oci/opener.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"All the logic for OCI fetching and authentication\"\"\"\n\nimport base64\nimport json\nimport re\nimport urllib.error\nimport urllib.parse\nimport urllib.request\nfrom enum import Enum, auto\nfrom http.client import HTTPResponse\nfrom typing import Callable, Dict, Iterable, List, NamedTuple, Optional, Tuple\nfrom urllib.request import Request\n\nimport spack.config\nimport spack.llnl.util.lang\nimport spack.mirrors.mirror\nimport spack.tokenize\nimport spack.util.web\n\nfrom .image import ImageReference\n\n\ndef _urlopen():\n opener = create_opener()\n\n def dispatch_open(fullurl, data=None, timeout=None):\n timeout = timeout or spack.config.get(\"config:connect_timeout\", 10)\n return opener.open(fullurl, data, timeout)\n\n return dispatch_open\n\n\nOpenType = Callable[..., HTTPResponse]\nMaybeOpen = Optional[OpenType]\n\n#: Opener that automatically uses OCI authentication based on mirror config\nurlopen: OpenType = spack.llnl.util.lang.Singleton(_urlopen)\n\n\nSP = r\" \"\nOWS = r\"[ \\t]*\"\nBWS = OWS\nHTAB = r\"\\t\"\nVCHAR = r\"\\x21-\\x7E\"\ntchar = r\"[!#$%&'*+\\-.^_`|~0-9A-Za-z]\"\ntoken = rf\"{tchar}+\"\nobs_text = r\"\\x80-\\xFF\"\nqdtext = rf\"[{HTAB}{SP}\\x21\\x23-\\x5B\\x5D-\\x7E{obs_text}]\"\nquoted_pair = rf\"\\\\([{HTAB}{SP}{VCHAR}{obs_text}])\"\nquoted_string = rf'\"(?:({qdtext}*)|{quoted_pair})*\"'\n\n\nclass WwwAuthenticateTokens(spack.tokenize.TokenBase):\n AUTH_PARAM = rf\"({token}){BWS}={BWS}({token}|{quoted_string})\"\n # TOKEN68 = r\"([A-Za-z0-9\\-._~+/]+=*)\" # todo... support this?\n TOKEN = rf\"{tchar}+\"\n EQUALS = rf\"{BWS}={BWS}\"\n COMMA = rf\"{OWS},{OWS}\"\n SPACE = r\" +\"\n EOF = r\"$\"\n ANY = r\".\"\n\n\nWWW_AUTHENTICATE_TOKENIZER = spack.tokenize.Tokenizer(WwwAuthenticateTokens)\n\n\nclass State(Enum):\n CHALLENGE = auto()\n AUTH_PARAM_LIST_START = auto()\n AUTH_PARAM = auto()\n NEXT_IN_LIST = auto()\n AUTH_PARAM_OR_SCHEME = auto()\n\n\nclass Challenge:\n __slots__ = [\"scheme\", \"params\"]\n\n def __init__(\n self, scheme: Optional[str] = None, params: Optional[List[Tuple[str, str]]] = None\n ) -> None:\n self.scheme = scheme or \"\"\n self.params = params or []\n\n def __repr__(self) -> str:\n return f\"Challenge({self.scheme}, {self.params})\"\n\n def __eq__(self, other: object) -> bool:\n return (\n isinstance(other, Challenge)\n and self.scheme == other.scheme\n and self.params == other.params\n )\n\n def matches_scheme(self, scheme: str) -> bool:\n \"\"\"Checks whether the challenge matches the given scheme, case-insensitive.\"\"\"\n return self.scheme == scheme.lower()\n\n def get_param(self, key: str) -> Optional[str]:\n \"\"\"Get the value of an auth param by key, or None if not found.\"\"\"\n return next((v for k, v in self.params if k == key.lower()), None)\n\n\ndef parse_www_authenticate(input: str):\n \"\"\"Very basic parsing of www-authenticate parsing (RFC7235 section 4.1)\n Notice: this omits token68 support.\"\"\"\n\n # auth-scheme = token\n # auth-param = token BWS \"=\" BWS ( token / quoted-string )\n # challenge = auth-scheme [ 1*SP ( token68 / #auth-param ) ]\n # WWW-Authenticate = 1#challenge\n\n challenges: List[Challenge] = []\n\n _unquote = re.compile(quoted_pair).sub\n unquote = lambda s: _unquote(r\"\\1\", s[1:-1])\n\n mode: State = State.CHALLENGE\n tokens = WWW_AUTHENTICATE_TOKENIZER.tokenize(input)\n\n current_challenge = Challenge()\n\n def extract_auth_param(input: str) -> Tuple[str, str]:\n key, value = input.split(\"=\", 1)\n key = key.rstrip().lower()\n value = value.lstrip()\n if value.startswith('\"'):\n value = unquote(value)\n return key, value\n\n while True:\n token: spack.tokenize.Token = next(tokens)\n\n if mode == State.CHALLENGE:\n if token.kind == WwwAuthenticateTokens.EOF:\n raise ValueError(token)\n elif token.kind == WwwAuthenticateTokens.TOKEN:\n current_challenge.scheme = token.value.lower()\n mode = State.AUTH_PARAM_LIST_START\n else:\n raise ValueError(token)\n\n elif mode == State.AUTH_PARAM_LIST_START:\n if token.kind == WwwAuthenticateTokens.EOF:\n challenges.append(current_challenge)\n break\n elif token.kind == WwwAuthenticateTokens.COMMA:\n # Challenge without param list, followed by another challenge.\n challenges.append(current_challenge)\n current_challenge = Challenge()\n mode = State.CHALLENGE\n elif token.kind == WwwAuthenticateTokens.SPACE:\n # A space means it must be followed by param list\n mode = State.AUTH_PARAM\n else:\n raise ValueError(token)\n\n elif mode == State.AUTH_PARAM:\n if token.kind == WwwAuthenticateTokens.EOF:\n raise ValueError(token)\n elif token.kind == WwwAuthenticateTokens.AUTH_PARAM:\n key, value = extract_auth_param(token.value)\n current_challenge.params.append((key, value))\n mode = State.NEXT_IN_LIST\n else:\n raise ValueError(token)\n\n elif mode == State.NEXT_IN_LIST:\n if token.kind == WwwAuthenticateTokens.EOF:\n challenges.append(current_challenge)\n break\n elif token.kind == WwwAuthenticateTokens.COMMA:\n mode = State.AUTH_PARAM_OR_SCHEME\n else:\n raise ValueError(token)\n\n elif mode == State.AUTH_PARAM_OR_SCHEME:\n if token.kind == WwwAuthenticateTokens.EOF:\n raise ValueError(token)\n elif token.kind == WwwAuthenticateTokens.TOKEN:\n challenges.append(current_challenge)\n current_challenge = Challenge(token.value.lower())\n mode = State.AUTH_PARAM_LIST_START\n elif token.kind == WwwAuthenticateTokens.AUTH_PARAM:\n key, value = extract_auth_param(token.value)\n current_challenge.params.append((key, value))\n mode = State.NEXT_IN_LIST\n\n return challenges\n\n\nclass RealmServiceScope(NamedTuple):\n realm: str\n service: str\n scope: str\n\n\nclass UsernamePassword(NamedTuple):\n username: str\n password: str\n\n @property\n def basic_auth_header(self) -> str:\n encoded = base64.b64encode(f\"{self.username}:{self.password}\".encode(\"utf-8\")).decode(\n \"utf-8\"\n )\n return f\"Basic {encoded}\"\n\n\ndef _get_bearer_challenge(challenges: List[Challenge]) -> Optional[RealmServiceScope]:\n \"\"\"Return the realm/service/scope for a Bearer auth challenge, or None if not found.\"\"\"\n challenge = next((c for c in challenges if c.matches_scheme(\"Bearer\")), None)\n\n if challenge is None:\n return None\n\n # Get realm / service / scope from challenge\n realm = challenge.get_param(\"realm\")\n service = challenge.get_param(\"service\")\n scope = challenge.get_param(\"scope\")\n\n if realm is None or service is None or scope is None:\n return None\n\n return RealmServiceScope(realm, service, scope)\n\n\ndef _get_basic_challenge(challenges: List[Challenge]) -> Optional[str]:\n \"\"\"Return the realm for a Basic auth challenge, or None if not found.\"\"\"\n challenge = next((c for c in challenges if c.matches_scheme(\"Basic\")), None)\n\n if challenge is None:\n return None\n\n return challenge.get_param(\"realm\")\n\n\nclass OCIAuthHandler(urllib.request.BaseHandler):\n def __init__(self, credentials_provider: Callable[[str], Optional[UsernamePassword]]):\n \"\"\"\n Args:\n credentials_provider: A function that takes a domain and may return a UsernamePassword.\n \"\"\"\n self.credentials_provider = credentials_provider\n\n # Cached authorization headers for a given domain.\n self.cached_auth_headers: Dict[str, str] = {}\n\n def https_request(self, req: Request):\n # Eagerly add the bearer token to the request if no\n # auth header is set yet, to avoid 401s in multiple\n # requests to the same registry.\n\n # Use has_header, not .headers, since there are two\n # types of headers (redirected and unredirected)\n if req.has_header(\"Authorization\"):\n return req\n\n parsed = urllib.parse.urlparse(req.full_url)\n auth_header = self.cached_auth_headers.get(parsed.netloc)\n\n if not auth_header:\n return req\n\n req.add_unredirected_header(\"Authorization\", auth_header)\n return req\n\n def _try_bearer_challenge(\n self,\n challenges: List[Challenge],\n credentials: Optional[UsernamePassword],\n timeout: Optional[float],\n ) -> Optional[str]:\n # Check whether a Bearer challenge is present in the WWW-Authenticate header\n challenge = _get_bearer_challenge(challenges)\n if not challenge:\n return None\n\n # Get the token from the auth handler\n query = urllib.parse.urlencode(\n {\"service\": challenge.service, \"scope\": challenge.scope, \"client_id\": \"spack\"}\n )\n parsed = urllib.parse.urlparse(challenge.realm)._replace(\n query=query, fragment=\"\", params=\"\"\n )\n\n # Don't send credentials over insecure transport.\n if parsed.scheme != \"https\":\n raise ValueError(f\"Cannot login over insecure {parsed.scheme} connection\")\n\n request = Request(urllib.parse.urlunparse(parsed), method=\"GET\")\n\n if credentials is not None:\n request.add_unredirected_header(\"Authorization\", credentials.basic_auth_header)\n\n # Do a GET request.\n response = self.parent.open(request, timeout=timeout)\n try:\n response_json = json.load(response)\n token = response_json.get(\"token\")\n if token is None:\n token = response_json.get(\"access_token\")\n assert type(token) is str\n except Exception as e:\n raise ValueError(f\"Malformed token response from {challenge.realm}\") from e\n return f\"Bearer {token}\"\n\n def _try_basic_challenge(\n self, challenges: List[Challenge], credentials: UsernamePassword\n ) -> Optional[str]:\n # Check whether a Basic challenge is present in the WWW-Authenticate header\n # A realm is required for Basic auth, although we don't use it here. Leave this as a\n # validation step.\n realm = _get_basic_challenge(challenges)\n if not realm:\n return None\n return credentials.basic_auth_header\n\n def http_error_401(self, req: Request, fp, code, msg, headers):\n # Login failed, avoid infinite recursion where we go back and\n # forth between auth server and registry\n if hasattr(req, \"login_attempted\"):\n raise spack.util.web.DetailedHTTPError(\n req, code, f\"Failed to login: {msg}\", headers, fp\n )\n\n # On 401 Unauthorized, parse the WWW-Authenticate header\n # to determine what authentication is required\n if \"WWW-Authenticate\" not in headers:\n raise spack.util.web.DetailedHTTPError(\n req, code, \"Cannot login to registry, missing WWW-Authenticate header\", headers, fp\n )\n\n www_auth_str = headers[\"WWW-Authenticate\"]\n\n try:\n challenges = parse_www_authenticate(www_auth_str)\n except ValueError as e:\n raise spack.util.web.DetailedHTTPError(\n req,\n code,\n f\"Cannot login to registry, malformed WWW-Authenticate header: {www_auth_str}\",\n headers,\n fp,\n ) from e\n\n registry = urllib.parse.urlparse(req.get_full_url()).netloc\n\n credentials = self.credentials_provider(registry)\n\n # First try Bearer, then Basic\n try:\n auth_header = self._try_bearer_challenge(challenges, credentials, req.timeout)\n if not auth_header and credentials:\n auth_header = self._try_basic_challenge(challenges, credentials)\n except Exception as e:\n raise spack.util.web.DetailedHTTPError(\n req, code, f\"Cannot login to registry: {e}\", headers, fp\n ) from e\n\n if not auth_header:\n raise spack.util.web.DetailedHTTPError(\n req,\n code,\n f\"Cannot login to registry, unsupported authentication scheme: {www_auth_str}\",\n headers,\n fp,\n )\n\n self.cached_auth_headers[registry] = auth_header\n\n # Add the authorization header to the request\n req.add_unredirected_header(\"Authorization\", auth_header)\n setattr(req, \"login_attempted\", True)\n\n return self.parent.open(req, timeout=req.timeout)\n\n\ndef credentials_from_mirrors(\n domain: str, *, mirrors: Optional[Iterable[spack.mirrors.mirror.Mirror]] = None\n) -> Optional[UsernamePassword]:\n \"\"\"Filter out OCI registry credentials from a list of mirrors.\"\"\"\n\n mirrors = mirrors or spack.mirrors.mirror.MirrorCollection().values()\n\n for mirror in mirrors:\n # Prefer push credentials over fetch. Unlikely that those are different\n # but our config format allows it.\n for direction in (\"push\", \"fetch\"):\n pair = mirror.get_credentials(direction).get(\"access_pair\")\n if not pair:\n continue\n\n url = mirror.get_url(direction)\n try:\n parsed = ImageReference.from_url(url)\n except ValueError:\n continue\n if parsed.domain == domain:\n return UsernamePassword(*pair)\n return None\n\n\ndef create_opener():\n \"\"\"Create an opener that can handle OCI authentication.\"\"\"\n opener = urllib.request.OpenerDirector()\n for handler in [\n urllib.request.ProxyHandler(),\n urllib.request.UnknownHandler(),\n urllib.request.HTTPHandler(),\n spack.util.web.SpackHTTPSHandler(context=spack.util.web.ssl_create_default_context()),\n spack.util.web.SpackHTTPDefaultErrorHandler(),\n urllib.request.HTTPRedirectHandler(),\n urllib.request.HTTPErrorProcessor(),\n OCIAuthHandler(credentials_from_mirrors),\n ]:\n opener.add_handler(handler)\n return opener\n\n\ndef ensure_status(request: urllib.request.Request, response: HTTPResponse, status: int):\n \"\"\"Raise an error if the response status is not the expected one.\"\"\"\n if response.status == status:\n return\n\n raise spack.util.web.DetailedHTTPError(\n request, response.status, response.reason, response.info(), None\n )\n\n\ndefault_retry = spack.util.web.retry_on_transient_error\n" }, { "path": "lib/spack/spack/operating_systems/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nfrom ._operating_system import OperatingSystem\nfrom .freebsd import FreeBSDOs\nfrom .linux_distro import LinuxDistro\nfrom .mac_os import MacOs\nfrom .windows_os import WindowsOs\n\n__all__ = [\"OperatingSystem\", \"LinuxDistro\", \"MacOs\", \"WindowsOs\", \"FreeBSDOs\"]\n\n#: List of all the Operating Systems known to Spack\noperating_systems = [LinuxDistro, MacOs, WindowsOs, FreeBSDOs]\n" }, { "path": "lib/spack/spack/operating_systems/_operating_system.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport spack.llnl.util.lang\n\n\n@spack.llnl.util.lang.lazy_lexicographic_ordering\nclass OperatingSystem:\n \"\"\"Base class for all the Operating Systems.\n\n On a multiple architecture machine, the architecture spec field can be set to\n build a package against any target and operating system that is present on the\n platform. On Cray platforms or any other architecture that has different front\n and back end environments, the operating system will determine the method of\n compiler detection.\n\n There are two different types of compiler detection:\n\n 1. Through the $PATH env variable (front-end detection)\n 2. Through the module system. (back-end detection)\n\n Depending on which operating system is specified, the compiler will be detected\n using one of those methods.\n\n For platforms such as linux and darwin, the operating system is autodetected.\n \"\"\"\n\n def __init__(self, name, version):\n self.name = name.replace(\"-\", \"_\")\n self.version = str(version).replace(\"-\", \"_\")\n\n def __str__(self):\n return \"%s%s\" % (self.name, self.version)\n\n def __repr__(self):\n return self.__str__()\n\n def _cmp_iter(self):\n yield self.name\n yield self.version\n\n def to_dict(self):\n return {\"name\": self.name, \"version\": self.version}\n" }, { "path": "lib/spack/spack/operating_systems/freebsd.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport platform as py_platform\n\nfrom spack.version import Version\n\nfrom ._operating_system import OperatingSystem\n\n\nclass FreeBSDOs(OperatingSystem):\n def __init__(self):\n release = py_platform.release().split(\"-\", 1)[0]\n super().__init__(\"freebsd\", Version(release))\n" }, { "path": "lib/spack/spack/operating_systems/linux_distro.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport platform as py_platform\nimport re\nfrom subprocess import check_output\n\nfrom spack.version import StandardVersion\n\nfrom ._operating_system import OperatingSystem\n\n\ndef kernel_version() -> StandardVersion:\n \"\"\"Return the host's kernel version as a :class:`~spack.version.StandardVersion` object.\"\"\"\n # Strip '+' characters just in case we're running a\n # version built from git/etc\n return StandardVersion.from_string(re.sub(r\"\\+\", r\"\", py_platform.release()))\n\n\nclass LinuxDistro(OperatingSystem):\n \"\"\"This class will represent the autodetected operating system\n for a Linux System. Since there are many different flavors of\n Linux, this class will attempt to encompass them all through\n autodetection using the python module platform and the method\n platform.dist()\n \"\"\"\n\n def __init__(self):\n try:\n # This will throw an error if imported on a non-Linux platform.\n from spack.vendor import distro\n\n distname, version = distro.id(), distro.version()\n except ImportError:\n distname, version = \"unknown\", \"\"\n\n # Grabs major version from tuple on redhat; on other platforms\n # grab the first legal identifier in the version field. On\n # debian you get things like 'wheezy/sid'; sid means unstable.\n # We just record 'wheezy' and don't get quite so detailed.\n version = re.split(r\"[^\\w-]\", version)\n\n if \"ubuntu\" in distname:\n version = \".\".join(version[0:2])\n # openSUSE Tumbleweed is a rolling release which can change\n # more than once in a week, so set version to tumbleweed$GLIBVERS\n elif \"opensuse-tumbleweed\" in distname or \"opensusetumbleweed\" in distname:\n distname = \"opensuse\"\n output = check_output([\"ldd\", \"--version\"]).decode()\n libcvers = re.findall(r\"ldd \\(GNU libc\\) (.*)\", output)\n if len(libcvers) == 1:\n version = \"tumbleweed\" + libcvers[0]\n else:\n version = \"tumbleweed\" + version[0]\n\n else:\n version = version[0]\n\n super().__init__(distname, version)\n" }, { "path": "lib/spack/spack/operating_systems/mac_os.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport platform as py_platform\nimport re\n\nimport spack.llnl.util.lang\nfrom spack.util.executable import Executable\nfrom spack.version import StandardVersion, Version\n\nfrom ._operating_system import OperatingSystem\n\n\n@spack.llnl.util.lang.memoized\ndef macos_version() -> StandardVersion:\n \"\"\"Get the current macOS version as a version object.\n\n This has three mechanisms for determining the macOS version, which is used\n for spack identification (the ``os`` in the spec's ``arch``) and indirectly\n for setting the value of ``MACOSX_DEPLOYMENT_TARGET``, which affects the\n ``minos`` value of the ``LC_BUILD_VERSION`` macho header. Mixing ``minos``\n values can lead to lots of linker warnings, and using a consistent version\n (pinned to the major OS version) allows distribution across clients that\n might be slightly behind.\n\n The version determination is made with three mechanisms in decreasing\n priority:\n\n 1. The ``MACOSX_DEPLOYMENT_TARGET`` variable overrides the actual operating\n system version, just like the value can be used to build for older macOS\n targets on newer systems. Spack currently will truncate this value when\n building packages, but at least the major version will be the same.\n\n 2. The system ``sw_vers`` command reports the actual operating system\n version.\n\n 3. The Python ``platform.mac_ver`` function is a fallback if the operating\n system identification fails, because some Python versions and/or\n installations report the OS\n on which Python was *built* rather than the one on which it is running.\n \"\"\"\n env_ver = os.environ.get(\"MACOSX_DEPLOYMENT_TARGET\", None)\n if env_ver:\n return StandardVersion.from_string(env_ver)\n\n try:\n output = Executable(\"sw_vers\")(output=str, fail_on_error=False)\n except Exception:\n # FileNotFoundError, or spack.util.executable.ProcessError\n pass\n else:\n match = re.search(r\"ProductVersion:\\s*([0-9.]+)\", output)\n if match:\n return StandardVersion.from_string(match.group(1))\n\n # Fall back to python-reported version, which can be inaccurate around\n # macOS 11 (e.g. showing 10.16 for macOS 12)\n return StandardVersion.from_string(py_platform.mac_ver()[0])\n\n\n@spack.llnl.util.lang.memoized\ndef macos_cltools_version():\n \"\"\"Find the last installed version of the CommandLineTools.\n\n The CLT version might only affect the build if it's selected as the macOS\n SDK path.\n \"\"\"\n pkgutil = Executable(\"pkgutil\")\n output = pkgutil(\n \"--pkg-info=com.apple.pkg.cltools_executables\", output=str, fail_on_error=False\n )\n match = re.search(r\"version:\\s*([0-9.]+)\", output)\n if match:\n return Version(match.group(1))\n\n # No CLTools installed by package manager: try Xcode\n output = pkgutil(\"--pkg-info=com.apple.pkg.Xcode\", output=str, fail_on_error=False)\n match = re.search(r\"version:\\s*([0-9.]+)\", output)\n if match:\n return Version(match.group(1))\n\n return None\n\n\n@spack.llnl.util.lang.memoized\ndef macos_sdk_path():\n \"\"\"Return path to the active macOS SDK.\"\"\"\n xcrun = Executable(\"xcrun\")\n return xcrun(\"--show-sdk-path\", output=str).rstrip()\n\n\ndef macos_sdk_version():\n \"\"\"Return the version of the active macOS SDK.\n\n The SDK version usually corresponds to the installed Xcode version and can\n affect how some packages (especially those that use the GUI) can fail. This\n information should somehow be embedded into the future \"compilers are\n dependencies\" feature.\n\n The macOS deployment target cannot be greater than the SDK version, but\n usually it can be at least a few versions less.\n \"\"\"\n xcrun = Executable(\"xcrun\")\n return Version(xcrun(\"--show-sdk-version\", output=str).rstrip())\n\n\nclass MacOs(OperatingSystem):\n \"\"\"This class represents the macOS operating system. This will be\n auto detected using the python platform.mac_ver. The macOS\n platform will be represented using the major version operating\n system name, i.e el capitan, yosemite...etc.\n \"\"\"\n\n def __init__(self):\n \"\"\"Autodetects the mac version from a dictionary.\n\n If the mac version is too old or too new for Spack to recognize,\n will use a generic \"macos\" version string until Spack is updated.\n \"\"\"\n mac_releases = {\n \"10.0\": \"cheetah\",\n \"10.1\": \"puma\",\n \"10.2\": \"jaguar\",\n \"10.3\": \"panther\",\n \"10.4\": \"tiger\",\n \"10.5\": \"leopard\",\n \"10.6\": \"snowleopard\",\n \"10.7\": \"lion\",\n \"10.8\": \"mountainlion\",\n \"10.9\": \"mavericks\",\n \"10.10\": \"yosemite\",\n \"10.11\": \"elcapitan\",\n \"10.12\": \"sierra\",\n \"10.13\": \"highsierra\",\n \"10.14\": \"mojave\",\n \"10.15\": \"catalina\",\n \"10.16\": \"bigsur\",\n \"11\": \"bigsur\",\n \"12\": \"monterey\",\n \"13\": \"ventura\",\n \"14\": \"sonoma\",\n \"15\": \"sequoia\",\n \"26\": \"tahoe\",\n }\n\n version = macos_version()\n\n # Big Sur versions go 11.0, 11.0.1, 11.1 (vs. prior versions that\n # only used the minor component)\n part = 1 if version >= Version(\"11\") else 2\n\n mac_ver = str(version.up_to(part))\n name = mac_releases.get(mac_ver, \"macos\")\n super().__init__(name, mac_ver)\n\n def __str__(self):\n return self.name\n" }, { "path": "lib/spack/spack/operating_systems/windows_os.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport glob\nimport os\nimport pathlib\nimport platform\nimport subprocess\n\nfrom spack.error import SpackError\nfrom spack.llnl.util import tty\nfrom spack.util import windows_registry as winreg\nfrom spack.version import Version\n\nfrom ._operating_system import OperatingSystem\n\n\ndef windows_version():\n \"\"\"Windows version as a Version object\"\"\"\n # include the build number as this provides important information\n # for low lever packages and components like the SDK and WDK\n # The build number is the version component that would otherwise\n # be the patch version in semantic versioning, i.e. z of x.y.z\n return Version(platform.version())\n\n\nclass WindowsOs(OperatingSystem):\n \"\"\"This class represents the Windows operating system. This will be\n auto detected using the python platform.win32_ver() once we have a\n python setup that runs natively. The Windows platform will be\n represented using the major version operating system number, e.g.\n 10.\n \"\"\"\n\n def __init__(self):\n plat_ver = windows_version()\n if plat_ver < Version(\"10\"):\n raise SpackError(\"Spack is not supported on Windows versions older than 10\")\n super().__init__(\"windows{}\".format(plat_ver), plat_ver)\n\n def __str__(self):\n return self.name\n\n @property\n def vs_install_paths(self):\n vs_install_paths = []\n root = os.environ.get(\"ProgramFiles(x86)\") or os.environ.get(\"ProgramFiles\")\n if root:\n try:\n extra_args = {\"encoding\": \"mbcs\", \"errors\": \"strict\"}\n paths = subprocess.check_output( # type: ignore[call-overload] # novermin\n [\n os.path.join(root, \"Microsoft Visual Studio\", \"Installer\", \"vswhere.exe\"),\n \"-prerelease\",\n \"-requires\",\n \"Microsoft.VisualStudio.Component.VC.Tools.x86.x64\",\n \"-property\",\n \"installationPath\",\n \"-products\",\n \"*\",\n ],\n **extra_args,\n ).strip()\n vs_install_paths = paths.split(\"\\n\")\n except (subprocess.CalledProcessError, OSError, UnicodeDecodeError):\n pass\n return vs_install_paths\n\n @property\n def msvc_paths(self):\n return [os.path.join(path, \"VC\", \"Tools\", \"MSVC\") for path in self.vs_install_paths]\n\n @property\n def oneapi_root(self):\n root = os.environ.get(\"ONEAPI_ROOT\", \"\") or os.path.join(\n os.environ.get(\"ProgramFiles(x86)\", \"\"), \"Intel\", \"oneAPI\"\n )\n if os.path.exists(root):\n return root\n\n @property\n def compiler_search_paths(self):\n # First Strategy: Find MSVC directories using vswhere\n _compiler_search_paths = []\n for p in self.msvc_paths:\n _compiler_search_paths.extend(glob.glob(os.path.join(p, \"*\", \"bin\", \"Hostx64\", \"x64\")))\n oneapi_root = self.oneapi_root\n if oneapi_root:\n _compiler_search_paths.extend(\n glob.glob(os.path.join(oneapi_root, \"compiler\", \"**\", \"bin\"), recursive=True)\n )\n\n # Second strategy: Find MSVC via the registry\n def try_query_registry(retry=False):\n winreg_report_error = lambda e: tty.debug(\n 'Windows registry query on \"SOFTWARE\\\\WOW6432Node\\\\Microsoft\"'\n f\"under HKEY_LOCAL_MACHINE: {str(e)}\"\n )\n try:\n # Registry interactions are subject to race conditions, etc and can generally\n # be flakey, do this in a catch block to prevent reg issues from interfering\n # with compiler detection\n msft = winreg.WindowsRegistryView(\n \"SOFTWARE\\\\WOW6432Node\\\\Microsoft\", winreg.HKEY.HKEY_LOCAL_MACHINE\n )\n return msft.find_subkeys(r\"VisualStudio_.*\", recursive=False)\n except OSError as e:\n # OSErrors propagated into caller by Spack's registry module are expected\n # and indicate a known issue with the registry query\n # i.e. user does not have permissions or the key/value\n # doesn't exist\n winreg_report_error(e)\n return []\n except winreg.InvalidRegistryOperation as e:\n # Other errors raised by the Spack's reg module indicate\n # an unexpected error type, and are handled specifically\n # as the underlying cause is difficult/impossible to determine\n # without manually exploring the registry\n # These errors can also be spurious (race conditions)\n # and may resolve on re-execution of the query\n # or are permanent (specific types of permission issues)\n # but the registry raises the same exception for all types of\n # atypical errors\n if retry:\n winreg_report_error(e)\n return []\n\n vs_entries = try_query_registry()\n if not vs_entries:\n # Occasional spurious race conditions can arise when reading the MS reg\n # typically these race conditions resolve immediately and we can safely\n # retry the reg query without waiting\n # Note: Winreg does not support locking\n vs_entries = try_query_registry(retry=True)\n\n vs_paths = []\n\n def clean_vs_path(path):\n path = path.split(\",\")[0].lstrip(\"@\")\n return str((pathlib.Path(path).parent / \"..\\\\..\").resolve())\n\n for entry in vs_entries:\n try:\n val = entry.get_subkey(\"Capabilities\").get_value(\"ApplicationDescription\").value\n vs_paths.append(clean_vs_path(val))\n except FileNotFoundError as e:\n if hasattr(e, \"winerror\") and e.winerror == 2:\n pass\n else:\n raise\n\n _compiler_search_paths.extend(vs_paths)\n return _compiler_search_paths\n" }, { "path": "lib/spack/spack/package.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport warnings\nfrom os import chdir, environ, getcwd, makedirs, mkdir, remove, removedirs\nfrom shutil import move, rmtree\n\n# import most common types used in packages\nfrom typing import Dict, Iterable, List, Optional, Tuple\n\nfrom spack.vendor.macholib.MachO import LC_ID_DYLIB, MachO\n\nimport spack.builder\nimport spack.llnl.util.tty as _tty\nfrom spack.archspec import microarchitecture_flags, microarchitecture_flags_from_target\nfrom spack.build_environment import (\n MakeExecutable,\n ModuleChangePropagator,\n get_cmake_prefix_path,\n get_effective_jobs,\n shared_library_suffix,\n static_library_suffix,\n)\nfrom spack.builder import (\n BaseBuilder,\n Builder,\n BuilderWithDefaults,\n GenericBuilder,\n Package,\n apply_macos_rpath_fixups,\n execute_install_time_tests,\n register_builder,\n)\nfrom spack.compilers.config import find_compilers\nfrom spack.compilers.libraries import CompilerPropertyDetector, compiler_spec\nfrom spack.config import determine_number_of_jobs\nfrom spack.deptypes import ALL_TYPES as all_deptypes\nfrom spack.directives import (\n build_system,\n can_splice,\n conditional,\n conflicts,\n depends_on,\n extends,\n license,\n maintainers,\n patch,\n provides,\n redistribute,\n requires,\n resource,\n variant,\n version,\n)\nfrom spack.error import (\n CompilerError,\n InstallError,\n NoHeadersError,\n NoLibrariesError,\n SpackAPIWarning,\n SpackError,\n)\nfrom spack.hooks.sbang import filter_shebang, sbang_install_path, sbang_shebang_line\nfrom spack.install_test import (\n SkipTest,\n cache_extra_test_sources,\n check_outputs,\n find_required_file,\n get_escaped_text_output,\n install_test_root,\n test_part,\n)\nfrom spack.llnl.util.filesystem import (\n FileFilter,\n FileList,\n HeaderList,\n LibraryList,\n ancestor,\n can_access,\n change_sed_delimiter,\n copy,\n copy_tree,\n filter_file,\n find,\n find_all_headers,\n find_all_libraries,\n find_first,\n find_headers,\n find_libraries,\n find_system_libraries,\n force_remove,\n force_symlink,\n has_shebang,\n install,\n install_tree,\n is_exe,\n join_path,\n keep_modification_time,\n library_extensions,\n mkdirp,\n path_contains_subdirectory,\n readlink,\n remove_directory_contents,\n remove_linked_tree,\n rename,\n safe_remove,\n set_executable,\n set_install_permissions,\n symlink,\n touch,\n windows_sfn,\n working_dir,\n)\nfrom spack.llnl.util.lang import ClassProperty, classproperty, dedupe, memoized\nfrom spack.llnl.util.link_tree import LinkTree\nfrom spack.mixins import filter_compiler_wrappers\nfrom spack.multimethod import default_args, when\nfrom spack.operating_systems.linux_distro import kernel_version\nfrom spack.operating_systems.mac_os import macos_version\nfrom spack.package_base import PackageBase, make_package_test_rpath, on_package_attributes\nfrom spack.package_completions import (\n bash_completion_path,\n fish_completion_path,\n zsh_completion_path,\n)\nfrom spack.package_test import compare_output, compare_output_file, compile_c_and_execute\nfrom spack.paths import spack_script\nfrom spack.phase_callbacks import run_after, run_before\nfrom spack.platforms import host as host_platform\nfrom spack.spec import Spec\nfrom spack.url import substitute_version as substitute_version_in_url\nfrom spack.user_environment import environment_modifications_for_specs\nfrom spack.util.elf import delete_needed_from_elf, delete_rpath, get_elf_compat, parse_elf\nfrom spack.util.environment import EnvironmentModifications, set_env\nfrom spack.util.environment import filter_system_paths as _filter_system_paths\nfrom spack.util.environment import is_system_path as _is_system_path\nfrom spack.util.executable import Executable, ProcessError, which, which_string\nfrom spack.util.filesystem import fix_darwin_install_name\nfrom spack.util.libc import libc_from_dynamic_linker, parse_dynamic_linker\nfrom spack.util.module_cmd import get_path_args_from_module_line\nfrom spack.util.module_cmd import module as module_command\nfrom spack.util.path import get_user\nfrom spack.util.prefix import Prefix\nfrom spack.util.url import join as join_url\nfrom spack.util.windows_registry import HKEY, WindowsRegistryView\nfrom spack.variant import any_combination_of, auto_or_any_combination_of, disjoint_sets\nfrom spack.version import Version, ver\n\n#: Alias for :data:`os.environ`\nenv = environ\n\n#: Alias for :func:`os.chdir`\ncd = chdir\n\n#: Alias for :func:`os.getcwd`\npwd = getcwd\n\n#: Alias for :func:`os.rename`\nrename = rename\n\n#: Alias for :func:`os.makedirs`\nmakedirs = makedirs\n\n#: Alias for :func:`os.mkdir`\nmkdir = mkdir\n\n#: Alias for :func:`os.remove`\nremove = remove\n\n#: Alias for :func:`os.removedirs`\nremovedirs = removedirs\n\n#: Alias for :func:`shutil.move`\nmove = move\n\n#: Alias for :func:`shutil.rmtree`\nrmtree = rmtree\n\n#: Alias for :func:`os.readlink` (with certain Windows-specific changes)\nreadlink = readlink\n\n#: Alias for :func:`os.rename` (with certain Windows-specific changes)\nrename = rename\n\n#: Alias for :func:`os.symlink` (with certain Windows-specific changes)\nsymlink = symlink\n\n# Not an import alias because black and isort disagree about style\ncreate_builder = spack.builder.create\n\n#: MachO class from the ``macholib`` package (vendored in Spack).\nMachO = MachO\n\n#: Constant for MachO ``LC_ID_DYLIB`` load command, from the ``macholib`` package (vendored in\n#: Spack).\nLC_ID_DYLIB = LC_ID_DYLIB\n\n\nclass tty:\n debug = _tty.debug\n error = _tty.error\n info = _tty.info\n msg = _tty.msg\n warn = _tty.warn\n\n\ndef is_system_path(path: str) -> bool:\n \"\"\"Returns :obj:`True` iff the argument is a system path.\n\n .. deprecated:: v2.0\n \"\"\"\n warnings.warn(\n \"spack.package.is_system_path is deprecated\", category=SpackAPIWarning, stacklevel=2\n )\n return _is_system_path(path)\n\n\ndef filter_system_paths(paths: Iterable[str]) -> List[str]:\n \"\"\"Returns a copy of the input where system paths are filtered out.\n\n .. deprecated:: v2.0\n \"\"\"\n warnings.warn(\n \"spack.package.filter_system_paths is deprecated\", category=SpackAPIWarning, stacklevel=2\n )\n return _filter_system_paths(paths)\n\n\n#: Assigning this to :attr:`spack.package_base.PackageBase.flag_handler` means that compiler flags\n#: are passed to the build system. This can be used in any package that derives from a build system\n#: class that implements :meth:`spack.package_base.PackageBase.flags_to_build_system_args`.\n#:\n#: See also :func:`env_flags` and :func:`inject_flags`.\n#:\n#: Example::\n#:\n#: from spack.package import *\n#:\n#: class MyPackage(CMakePackage):\n#: flag_handler = build_system_flags\nbuild_system_flags = PackageBase.build_system_flags\n\n#: Assigning this to :attr:`spack.package_base.PackageBase.flag_handler` means that compiler flags\n#: are set as canonical environment variables.\n#:\n#: See also :func:`build_system_flags` and :func:`inject_flags`.\n#:\n#: Example::\n#:\n#: from spack.package import *\n#:\n#: class MyPackage(MakefilePackage):\n#: flag_handler = env_flags\nenv_flags = PackageBase.env_flags\n\n\n#: This is the default value of :attr:`spack.package_base.PackageBase.flag_handler`, which tells\n#: Spack to inject compiler flags through the compiler wrappers, which means that the build system\n#: will not see them directly. This is typically a good default, but in rare case you may need to\n#: use :func:`env_flags` or :func:`build_system_flags` instead.\n#:\n#: See also :func:`build_system_flags` and :func:`env_flags`.\n#:\n#: Example::\n#:\n#: from spack.package import *\n#:\n#: class MyPackage(MakefilePackage):\n#: flag_handler = inject_flags\ninject_flags = PackageBase.inject_flags\n\n\napi: Dict[str, Tuple[str, ...]] = {\n \"v2.0\": (\n \"BaseBuilder\",\n \"Builder\",\n \"Dict\",\n \"EnvironmentModifications\",\n \"Executable\",\n \"FileFilter\",\n \"FileList\",\n \"HeaderList\",\n \"InstallError\",\n \"LibraryList\",\n \"List\",\n \"MakeExecutable\",\n \"NoHeadersError\",\n \"NoLibrariesError\",\n \"Optional\",\n \"PackageBase\",\n \"Prefix\",\n \"ProcessError\",\n \"SkipTest\",\n \"Spec\",\n \"Version\",\n \"all_deptypes\",\n \"ancestor\",\n \"any_combination_of\",\n \"auto_or_any_combination_of\",\n \"bash_completion_path\",\n \"build_system_flags\",\n \"build_system\",\n \"cache_extra_test_sources\",\n \"can_access\",\n \"can_splice\",\n \"cd\",\n \"change_sed_delimiter\",\n \"check_outputs\",\n \"conditional\",\n \"conflicts\",\n \"copy_tree\",\n \"copy\",\n \"default_args\",\n \"depends_on\",\n \"determine_number_of_jobs\",\n \"disjoint_sets\",\n \"env_flags\",\n \"env\",\n \"extends\",\n \"filter_compiler_wrappers\",\n \"filter_file\",\n \"find_all_headers\",\n \"find_first\",\n \"find_headers\",\n \"find_libraries\",\n \"find_required_file\",\n \"find_system_libraries\",\n \"find\",\n \"fish_completion_path\",\n \"fix_darwin_install_name\",\n \"force_remove\",\n \"force_symlink\",\n \"get_escaped_text_output\",\n \"inject_flags\",\n \"install_test_root\",\n \"install_tree\",\n \"install\",\n \"is_exe\",\n \"join_path\",\n \"keep_modification_time\",\n \"library_extensions\",\n \"license\",\n \"maintainers\",\n \"makedirs\",\n \"mkdir\",\n \"mkdirp\",\n \"move\",\n \"on_package_attributes\",\n \"patch\",\n \"provides\",\n \"pwd\",\n \"redistribute\",\n \"register_builder\",\n \"remove_directory_contents\",\n \"remove_linked_tree\",\n \"remove\",\n \"removedirs\",\n \"rename\",\n \"requires\",\n \"resource\",\n \"rmtree\",\n \"run_after\",\n \"run_before\",\n \"set_executable\",\n \"set_install_permissions\",\n \"symlink\",\n \"test_part\",\n \"touch\",\n \"tty\",\n \"variant\",\n \"ver\",\n \"version\",\n \"when\",\n \"which_string\",\n \"which\",\n \"working_dir\",\n \"zsh_completion_path\",\n ),\n \"v2.1\": (\"CompilerError\", \"SpackError\"),\n \"v2.2\": (\n \"BuilderWithDefaults\",\n \"ClassProperty\",\n \"CompilerPropertyDetector\",\n \"GenericBuilder\",\n \"HKEY\",\n \"LC_ID_DYLIB\",\n \"LinkTree\",\n \"MachO\",\n \"ModuleChangePropagator\",\n \"Package\",\n \"WindowsRegistryView\",\n \"apply_macos_rpath_fixups\",\n \"classproperty\",\n \"compare_output_file\",\n \"compare_output\",\n \"compile_c_and_execute\",\n \"compiler_spec\",\n \"create_builder\",\n \"dedupe\",\n \"delete_needed_from_elf\",\n \"delete_rpath\",\n \"environment_modifications_for_specs\",\n \"execute_install_time_tests\",\n \"filter_shebang\",\n \"filter_system_paths\",\n \"find_all_libraries\",\n \"find_compilers\",\n \"get_cmake_prefix_path\",\n \"get_effective_jobs\",\n \"get_elf_compat\",\n \"get_path_args_from_module_line\",\n \"get_user\",\n \"has_shebang\",\n \"host_platform\",\n \"is_system_path\",\n \"join_url\",\n \"kernel_version\",\n \"libc_from_dynamic_linker\",\n \"macos_version\",\n \"make_package_test_rpath\",\n \"memoized\",\n \"microarchitecture_flags_from_target\",\n \"microarchitecture_flags\",\n \"module_command\",\n \"parse_dynamic_linker\",\n \"parse_elf\",\n \"path_contains_subdirectory\",\n \"readlink\",\n \"safe_remove\",\n \"sbang_install_path\",\n \"sbang_shebang_line\",\n \"set_env\",\n \"shared_library_suffix\",\n \"spack_script\",\n \"static_library_suffix\",\n \"substitute_version_in_url\",\n \"windows_sfn\",\n ),\n}\n\n# Splatting does not work for static analysis tools.\n__all__ = [\n # v2.0\n \"BaseBuilder\",\n \"Builder\",\n \"Dict\",\n \"EnvironmentModifications\",\n \"Executable\",\n \"FileFilter\",\n \"FileList\",\n \"HeaderList\",\n \"InstallError\",\n \"LibraryList\",\n \"List\",\n \"MakeExecutable\",\n \"NoHeadersError\",\n \"NoLibrariesError\",\n \"Optional\",\n \"PackageBase\",\n \"Prefix\",\n \"ProcessError\",\n \"SkipTest\",\n \"Spec\",\n \"Version\",\n \"all_deptypes\",\n \"ancestor\",\n \"any_combination_of\",\n \"auto_or_any_combination_of\",\n \"bash_completion_path\",\n \"build_system_flags\",\n \"build_system\",\n \"cache_extra_test_sources\",\n \"can_access\",\n \"can_splice\",\n \"cd\",\n \"change_sed_delimiter\",\n \"check_outputs\",\n \"conditional\",\n \"conflicts\",\n \"copy_tree\",\n \"copy\",\n \"default_args\",\n \"depends_on\",\n \"determine_number_of_jobs\",\n \"disjoint_sets\",\n \"env_flags\",\n \"env\",\n \"extends\",\n \"filter_compiler_wrappers\",\n \"filter_file\",\n \"find_all_headers\",\n \"find_first\",\n \"find_headers\",\n \"find_libraries\",\n \"find_required_file\",\n \"find_system_libraries\",\n \"find\",\n \"fish_completion_path\",\n \"fix_darwin_install_name\",\n \"force_remove\",\n \"force_symlink\",\n \"get_escaped_text_output\",\n \"inject_flags\",\n \"install_test_root\",\n \"install_tree\",\n \"install\",\n \"is_exe\",\n \"join_path\",\n \"keep_modification_time\",\n \"library_extensions\",\n \"license\",\n \"maintainers\",\n \"makedirs\",\n \"mkdir\",\n \"mkdirp\",\n \"move\",\n \"on_package_attributes\",\n \"patch\",\n \"provides\",\n \"pwd\",\n \"redistribute\",\n \"register_builder\",\n \"remove_directory_contents\",\n \"remove_linked_tree\",\n \"remove\",\n \"removedirs\",\n \"rename\",\n \"requires\",\n \"resource\",\n \"rmtree\",\n \"run_after\",\n \"run_before\",\n \"set_executable\",\n \"set_install_permissions\",\n \"symlink\",\n \"test_part\",\n \"touch\",\n \"tty\",\n \"variant\",\n \"ver\",\n \"version\",\n \"when\",\n \"which_string\",\n \"which\",\n \"working_dir\",\n \"zsh_completion_path\",\n # v2.1\n \"CompilerError\",\n \"SpackError\",\n # v2.2\n \"BuilderWithDefaults\",\n \"ClassProperty\",\n \"CompilerPropertyDetector\",\n \"GenericBuilder\",\n \"HKEY\",\n \"LC_ID_DYLIB\",\n \"LinkTree\",\n \"MachO\",\n \"ModuleChangePropagator\",\n \"Package\",\n \"WindowsRegistryView\",\n \"apply_macos_rpath_fixups\",\n \"classproperty\",\n \"compare_output_file\",\n \"compare_output\",\n \"compile_c_and_execute\",\n \"compiler_spec\",\n \"create_builder\",\n \"dedupe\",\n \"delete_needed_from_elf\",\n \"delete_rpath\",\n \"environment_modifications_for_specs\",\n \"execute_install_time_tests\",\n \"filter_shebang\",\n \"filter_system_paths\",\n \"find_all_libraries\",\n \"find_compilers\",\n \"get_cmake_prefix_path\",\n \"get_effective_jobs\",\n \"get_elf_compat\",\n \"get_path_args_from_module_line\",\n \"get_user\",\n \"has_shebang\",\n \"host_platform\",\n \"is_system_path\",\n \"join_url\",\n \"kernel_version\",\n \"libc_from_dynamic_linker\",\n \"macos_version\",\n \"make_package_test_rpath\",\n \"memoized\",\n \"microarchitecture_flags_from_target\",\n \"microarchitecture_flags\",\n \"module_command\",\n \"parse_dynamic_linker\",\n \"parse_elf\",\n \"path_contains_subdirectory\",\n \"readlink\",\n \"safe_remove\",\n \"sbang_install_path\",\n \"sbang_shebang_line\",\n \"set_env\",\n \"shared_library_suffix\",\n \"spack_script\",\n \"static_library_suffix\",\n \"substitute_version_in_url\",\n \"windows_sfn\",\n]\n\n# These are just here for editor support; they may be set when the build env is set up.\nconfigure: Executable\nmake_jobs: int\nmake: MakeExecutable\nnmake: Executable\nninja: MakeExecutable\npython_include: str\npython_platlib: str\npython_purelib: str\npython: Executable\nspack_cc: str\nspack_cxx: str\nspack_f77: str\nspack_fc: str\nprefix: Prefix\ndso_suffix: str\n" }, { "path": "lib/spack/spack/package_base.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Base class for all Spack packages.\"\"\"\n\nimport base64\nimport collections\nimport copy\nimport errno\nimport functools\nimport glob\nimport hashlib\nimport io\nimport itertools\nimport os\nimport pathlib\nimport re\nimport sys\nimport textwrap\nimport time\nimport traceback\nfrom typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, Type, TypeVar, Union\n\nfrom spack.vendor.typing_extensions import Literal\n\nimport spack.config\nimport spack.dependency\nimport spack.deptypes as dt\nimport spack.directives_meta\nimport spack.error\nimport spack.fetch_strategy as fs\nimport spack.hooks\nimport spack.llnl.util.filesystem as fsys\nimport spack.llnl.util.tty as tty\nimport spack.mirrors.layout\nimport spack.mirrors.mirror\nimport spack.multimethod\nimport spack.patch\nimport spack.phase_callbacks\nimport spack.repo\nimport spack.spec\nimport spack.stage as stg\nimport spack.store\nimport spack.url\nimport spack.util.archive\nimport spack.util.environment\nimport spack.util.executable\nimport spack.util.git\nimport spack.util.naming\nimport spack.util.path\nimport spack.util.web\nimport spack.variant\nfrom spack.compilers.adaptor import DeprecatedCompiler\nfrom spack.error import InstallError, NoURLError, PackageError\nfrom spack.filesystem_view import YamlFilesystemView\nfrom spack.llnl.util.filesystem import (\n AlreadyExistsError,\n find_all_shared_libraries,\n islink,\n symlink,\n)\nfrom spack.llnl.util.lang import ClassProperty, classproperty, dedupe, memoized\nfrom spack.resource import Resource\nfrom spack.util.package_hash import package_hash\nfrom spack.util.typing import SupportsRichComparison\nfrom spack.version import GitVersion, StandardVersion, VersionError, is_git_version\n\nFLAG_HANDLER_RETURN_TYPE = Tuple[\n Optional[Iterable[str]], Optional[Iterable[str]], Optional[Iterable[str]]\n]\nFLAG_HANDLER_TYPE = Callable[[str, Iterable[str]], FLAG_HANDLER_RETURN_TYPE]\n\n#: Filename for the Spack build/install log.\n_spack_build_logfile = \"spack-build-out.txt\"\n\n#: Filename for the Spack build/install environment file.\n_spack_build_envfile = \"spack-build-env.txt\"\n\n#: Filename for the Spack build/install environment modifications file.\n_spack_build_envmodsfile = \"spack-build-env-mods.txt\"\n\n#: Filename for the Spack configure args file.\n_spack_configure_argsfile = \"spack-configure-args.txt\"\n\n#: Filename of json with total build and phase times (seconds)\nspack_times_log = \"install_times.json\"\n\nNO_DEFAULT = object()\n\n\nclass WindowsRPath:\n \"\"\"Collection of functionality surrounding Windows RPATH specific features\n\n This is essentially meaningless for all other platforms\n due to their use of RPATH. All methods within this class are no-ops on\n non Windows. Packages can customize and manipulate this class as\n they would a genuine RPATH, i.e. adding directories that contain\n runtime library dependencies\"\"\"\n\n def win_add_library_dependent(self):\n \"\"\"Return extra set of directories that require linking for package\n\n This method should be overridden by packages that produce\n binaries/libraries/python extension modules/etc that are installed into\n directories outside a package's ``bin``, ``lib``, and ``lib64`` directories,\n but still require linking against one of the packages dependencies, or\n other components of the package itself. No-op otherwise.\n\n Returns:\n List of additional directories that require linking\n \"\"\"\n return []\n\n def win_add_rpath(self):\n \"\"\"Return extra set of rpaths for package\n\n This method should be overridden by packages needing to\n include additional paths to be searched by rpath. No-op otherwise\n\n Returns:\n List of additional rpaths\n \"\"\"\n return []\n\n def windows_establish_runtime_linkage(self):\n \"\"\"Establish RPATH on Windows\n\n Performs symlinking to incorporate rpath dependencies to Windows runtime search paths\n \"\"\"\n # If spec is an external, we should not be modifying its bin directory, as we would\n # be doing in this method\n # Spack should in general not modify things it has not installed\n # we can reasonably expect externals to have their link interface properly established\n if sys.platform == \"win32\" and not self.spec.external:\n win_rpath = WindowsSimulatedRPath(self)\n win_rpath.add_library_dependent(*self.win_add_library_dependent())\n win_rpath.add_rpath(*self.win_add_rpath())\n win_rpath.establish_link()\n\n\n#: Registers which are the detectable packages, by repo and package name\n#: Need a pass of package repositories to be filled.\ndetectable_packages = collections.defaultdict(list)\n\n\nclass DetectablePackageMeta(type):\n \"\"\"Check if a package is detectable and add default implementations\n for the detection function.\n \"\"\"\n\n TAG = \"detectable\"\n\n def __init__(cls, name, bases, attr_dict):\n if hasattr(cls, \"executables\") and hasattr(cls, \"libraries\"):\n msg = \"a package can have either an 'executables' or 'libraries' attribute\"\n raise spack.error.SpackError(f\"{msg} [package '{name}' defines both]\")\n\n # On windows, extend the list of regular expressions to look for\n # filenames ending with \".exe\"\n # (in some cases these regular expressions include \"$\" to avoid\n # pulling in filenames with unexpected suffixes, but this allows\n # for example detecting \"foo.exe\" when the package writer specified\n # that \"foo\" was a possible executable.\n\n # If a package has the executables or libraries attribute then it's\n # assumed to be detectable. Add a tag, so finding them is faster\n if hasattr(cls, \"executables\") or hasattr(cls, \"libraries\"):\n # To add the tag, we need to copy the tags attribute, and attach it to\n # the current class. We don't use append, since it might modify base classes,\n # if \"tags\" is retrieved following the MRO.\n cls.tags = getattr(cls, \"tags\", []) + [DetectablePackageMeta.TAG]\n\n @classmethod\n def platform_executables(cls):\n def to_windows_exe(exe):\n if exe.endswith(\"$\"):\n exe = exe.replace(\"$\", \"%s$\" % spack.util.path.win_exe_ext())\n else:\n exe += spack.util.path.win_exe_ext()\n return exe\n\n plat_exe = []\n if hasattr(cls, \"executables\"):\n for exe in cls.executables:\n if sys.platform == \"win32\":\n exe = to_windows_exe(exe)\n plat_exe.append(exe)\n return plat_exe\n\n @classmethod\n def determine_spec_details(cls, prefix, objs_in_prefix):\n \"\"\"Allow ``spack external find ...`` to locate installations.\n\n Args:\n prefix (str): the directory containing the executables\n or libraries\n objs_in_prefix (set): the executables or libraries that\n match the regex\n\n Returns:\n The list of detected specs for this package\n \"\"\"\n objs_by_version = collections.defaultdict(list)\n # The default filter function is the identity function for the\n # list of executables\n filter_fn = getattr(cls, \"filter_detected_exes\", lambda x, exes: exes)\n objs_in_prefix = filter_fn(prefix, objs_in_prefix)\n for obj in objs_in_prefix:\n try:\n version_str = cls.determine_version(obj)\n if version_str:\n objs_by_version[version_str].append(obj)\n except Exception as e:\n tty.debug(f\"Cannot detect the version of '{obj}' [{str(e)}]\")\n\n specs = []\n for version_str, objs in objs_by_version.items():\n variants = cls.determine_variants(objs, version_str)\n # Normalize output to list\n if not isinstance(variants, list):\n variants = [variants]\n\n for variant in variants:\n if isinstance(variant, str):\n variant = (variant, {})\n variant_str, extra_attributes = variant\n spec_str = f\"{cls.name}@{version_str} {variant_str}\"\n\n # Pop a few reserved keys from extra attributes, since\n # they have a different semantics\n external_path = extra_attributes.pop(\"prefix\", None)\n external_modules = extra_attributes.pop(\"modules\", None)\n try:\n spec = spack.spec.Spec.from_detection(\n spec_str,\n external_path=external_path,\n external_modules=external_modules,\n extra_attributes=extra_attributes,\n )\n except Exception as e:\n tty.debug(f'Parsing failed [spec_str=\"{spec_str}\", error={str(e)}]')\n else:\n specs.append(spec)\n\n return sorted(specs)\n\n @classmethod\n def determine_variants(cls, objs, version_str):\n return \"\"\n\n # Register the class as a detectable package\n detectable_packages[cls.namespace].append(cls.name)\n\n # Attach function implementations to the detectable class\n default = False\n if not hasattr(cls, \"determine_spec_details\"):\n default = True\n cls.determine_spec_details = determine_spec_details\n\n if default and not hasattr(cls, \"determine_version\"):\n msg = (\n 'the package \"{0}\" in the \"{1}\" repo needs to define'\n ' the \"determine_version\" method to be detectable'\n )\n NotImplementedError(msg.format(cls.name, cls.namespace))\n\n if default and not hasattr(cls, \"determine_variants\"):\n cls.determine_variants = determine_variants\n\n # This function should not be overridden by subclasses,\n # as it is not designed for bespoke pkg detection but rather\n # on a per-platform basis\n if \"platform_executables\" in cls.__dict__.keys():\n raise PackageError(\"Packages should not override platform_executables\")\n cls.platform_executables = platform_executables\n\n super(DetectablePackageMeta, cls).__init__(name, bases, attr_dict)\n\n\nclass PackageMeta(\n spack.phase_callbacks.PhaseCallbacksMeta,\n DetectablePackageMeta,\n spack.directives_meta.DirectiveMeta,\n spack.multimethod.MultiMethodMeta,\n):\n \"\"\"\n Package metaclass for supporting directives (e.g., depends_on) and phases\n \"\"\"\n\n def __new__(cls, name, bases, attr_dict):\n \"\"\"\n FIXME: REWRITE\n Instance creation is preceded by phase attribute transformations.\n\n Conveniently transforms attributes to permit extensible phases by\n iterating over the attribute 'phases' and creating / updating private\n InstallPhase attributes in the class that will be initialized in\n __init__.\n \"\"\"\n attr_dict[\"_name\"] = None\n\n return super(PackageMeta, cls).__new__(cls, name, bases, attr_dict)\n\n\ndef on_package_attributes(**attr_dict):\n \"\"\"Decorator: executes instance function only if object has attr values.\n\n Executes the decorated method only if at the moment of calling the\n instance has attributes that are equal to certain values.\n\n Args:\n attr_dict (dict): dictionary mapping attribute names to their\n required values\n \"\"\"\n\n def _execute_under_condition(func):\n @functools.wraps(func)\n def _wrapper(instance, *args, **kwargs):\n # If all the attributes have the value we require, then execute\n has_all_attributes = all([hasattr(instance, key) for key in attr_dict])\n if has_all_attributes:\n has_the_right_values = all(\n [getattr(instance, key) == value for key, value in attr_dict.items()] # NOQA: ignore=E501\n )\n if has_the_right_values:\n func(instance, *args, **kwargs)\n\n return _wrapper\n\n return _execute_under_condition\n\n\nclass PackageViewMixin:\n \"\"\"This collects all functionality related to adding installed Spack\n package to views. Packages can customize how they are added to views by\n overriding these functions.\n \"\"\"\n\n spec: spack.spec.Spec\n\n def view_source(self):\n \"\"\"The source root directory that will be added to the view: files are\n added such that their path relative to the view destination matches\n their path relative to the view source.\n \"\"\"\n return self.spec.prefix\n\n def view_destination(self, view):\n \"\"\"The target root directory: each file is added relative to this\n directory.\n \"\"\"\n return view.get_projection_for_spec(self.spec)\n\n def view_file_conflicts(self, view, merge_map):\n \"\"\"Report any files which prevent adding this package to the view. The\n default implementation looks for any files which already exist.\n Alternative implementations may allow some of the files to exist in\n the view (in this case they would be omitted from the results).\n \"\"\"\n return set(dst for dst in merge_map.values() if os.path.lexists(dst))\n\n def add_files_to_view(self, view, merge_map, skip_if_exists=True):\n \"\"\"Given a map of package files to destination paths in the view, add\n the files to the view. By default this adds all files. Alternative\n implementations may skip some files, for example if other packages\n linked into the view already include the file.\n\n Args:\n view (spack.filesystem_view.FilesystemView): the view that's updated\n merge_map (dict): maps absolute source paths to absolute dest paths for\n all files in from this package.\n skip_if_exists (bool): when True, don't link files in view when they\n already exist. When False, always link files, without checking\n if they already exist.\n \"\"\"\n if skip_if_exists:\n for src, dst in merge_map.items():\n if not os.path.lexists(dst):\n view.link(src, dst, spec=self.spec)\n else:\n for src, dst in merge_map.items():\n view.link(src, dst, spec=self.spec)\n\n def remove_files_from_view(self, view, merge_map):\n \"\"\"Given a map of package files to files currently linked in the view,\n remove the files from the view. The default implementation removes all\n files. Alternative implementations may not remove all files. For\n example if two packages include the same file, it should only be\n removed when both packages are removed.\n \"\"\"\n view.remove_files(merge_map.values())\n\n\nPb = TypeVar(\"Pb\", bound=\"PackageBase\")\n\n# Some typedefs for dealing with when-indexed dictionaries\n#\n# Many of the dictionaries on PackageBase are of the form:\n# { Spec: { K: V } }\n#\n# K might be a variant name, a version, etc. V is a definition of some Spack object.\n# The methods below transform these types of dictionaries.\nK = TypeVar(\"K\", bound=SupportsRichComparison)\nV = TypeVar(\"V\")\n\n\ndef _by_subkey(\n when_indexed_dictionary: Dict[spack.spec.Spec, Dict[K, V]], when: bool = False\n) -> Dict[K, Union[List[V], Dict[spack.spec.Spec, List[V]]]]:\n \"\"\"Convert a dict of dicts keyed by when/subkey into a dict of lists keyed by subkey.\n\n Optional Arguments:\n when: if ``True``, don't discard the ``when`` specs; return a 2-level dictionary\n keyed by subkey and when spec.\n \"\"\"\n # very hard to define this type to be conditional on `when`\n all_by_subkey: Dict[K, Any] = {}\n\n for when_spec, by_key in when_indexed_dictionary.items():\n for key, value in by_key.items():\n if when:\n when_dict = all_by_subkey.setdefault(key, {})\n when_dict.setdefault(when_spec, []).append(value)\n else:\n all_by_subkey.setdefault(key, []).append(value)\n\n # this needs to preserve the insertion order of whens\n return dict(sorted(all_by_subkey.items()))\n\n\ndef _subkeys(when_indexed_dictionary: Dict[spack.spec.Spec, Dict[K, V]]) -> List[K]:\n \"\"\"Get sorted names from dicts keyed by when/name.\"\"\"\n all_keys = set()\n for when, by_key in when_indexed_dictionary.items():\n for key in by_key:\n all_keys.add(key)\n\n return sorted(all_keys)\n\n\ndef _has_subkey(when_indexed_dictionary: Dict[spack.spec.Spec, Dict[K, V]], key: K) -> bool:\n return any(key in dictionary for dictionary in when_indexed_dictionary.values())\n\n\ndef _num_definitions(when_indexed_dictionary: Dict[spack.spec.Spec, Dict[K, V]]) -> int:\n return sum(len(dictionary) for dictionary in when_indexed_dictionary.values())\n\n\ndef _remove_overridden_defs(defs: List[Tuple[spack.spec.Spec, Any]]) -> None:\n \"\"\"Remove definitions from the list if their when specs are satisfied by later ones.\n\n Any such definitions are *always* overridden by their successor, as they will\n match everything the predecessor matches, and the solver will prefer them because of\n their higher precedence.\n\n We can just remove these defs and avoid putting them in the solver. This is also\n useful for, e.g., `spack info`, where we don't want to show a variant from a\n superclass if it is always overridden by a variant defined in a subclass.\n\n Example::\n\n class ROCmPackage:\n variant(\"amdgpu_target\", ..., when=\"+rocm\")\n\n class Hipblas:\n variant(\"amdgpu_target\", ...)\n\n The subclass definition *always* overrides the superclass definition here, but they\n have different when specs and the subclass def won't just replace the one in the\n superclass. In this situation, the subclass should *probably* also have\n ``when=\"+rocm\"``, but we can't guarantee that will always happen when a vdef is\n overridden. So we use this method to remove any overrides we can know statically.\n\n \"\"\"\n i = 0\n while i < len(defs):\n when, _ = defs[i]\n if any(when.satisfies(successor) for successor, _ in defs[i + 1 :]):\n del defs[i]\n else:\n i += 1\n\n\ndef _definitions(\n when_indexed_dictionary: Dict[spack.spec.Spec, Dict[K, V]], key: K\n) -> List[Tuple[spack.spec.Spec, V]]:\n \"\"\"Iterator over (when_spec, Value) for all values with a particular Key.\"\"\"\n # construct a list of defs sorted by precedence\n defs: List[Tuple[spack.spec.Spec, V]] = []\n for when, values_by_key in when_indexed_dictionary.items():\n value_def = values_by_key.get(key)\n if value_def:\n defs.append((when, value_def))\n\n # With multiple definitions, ensure precedence order and simplify overrides\n if len(defs) > 1:\n defs.sort(key=lambda v: getattr(v[1], \"precedence\", 0))\n _remove_overridden_defs(defs)\n\n return defs\n\n\n#: Store whether a given Spec source/binary should not be redistributed.\nclass DisableRedistribute:\n def __init__(self, source, binary):\n self.source = source\n self.binary = binary\n\n\nclass PackageBase(WindowsRPath, PackageViewMixin, metaclass=PackageMeta):\n \"\"\"This is the universal base class for all Spack packages.\n\n At its core, a package consists of a set of software to be installed. A package may focus on a\n piece of software and its associated software dependencies or it may simply be a set, or\n bundle, of software. The former requires defining how to fetch, verify (via, e.g., ``sha256``),\n build, and install that software and the packages it depends on, so that dependencies can be\n installed along with the package itself. The latter, sometimes referred to as a \"no-source\"\n package, requires only defining the packages to be built.\n\n There are two main parts of a Spack package:\n\n 1. **The package class**. Classes contain *directives*, which are functions such as\n :py:func:`spack.package.version`, :py:func:`spack.package.patch`, and\n :py:func:`spack.package.depends_on`, that store metadata on the package class. Directives\n provide the constraints that are used as input to the concretizer.\n\n 2. **Package instances**. Once instantiated with a concrete spec, a package can be passed to\n the :py:class:`spack.installer.PackageInstaller`. It calls methods like :meth:`do_stage` on\n the package instance, and it uses those to drive user-implemented methods like ``def patch``\n and install phases like ``def configure`` and ``def install``.\n\n Packages are imported from package repositories (see :py:mod:`spack.repo`).\n\n For most use cases, package creators typically just add attributes like ``homepage`` and, for\n a code-based package, ``url``, or installation phases such as ``install()``.\n There are many custom ``PackageBase`` subclasses in the ``spack_repo.builtin.build_systems``\n package that make things even easier for specific build systems.\n\n .. note::\n\n Many methods and attributes that appear to be public interface are not meant to be\n overridden by packagers. They are \"final\", but we currently have not adopted the ``@final``\n decorator in the Spack codebase. For example, the ``do_*`` functions are intended only to be\n called internally by Spack commands. These aren't for package writers to override, and\n doing so may break the functionality of the ``PackageBase`` class.\"\"\"\n\n compiler = DeprecatedCompiler()\n\n #: Class level dictionary populated by :func:`~spack.directives.version` directives\n versions: Dict[StandardVersion, Dict[str, Any]]\n #: Class level dictionary populated by :func:`~spack.directives.resource` directives\n resources: Dict[spack.spec.Spec, List[Resource]]\n #: Class level dictionary populated by :func:`~spack.directives.depends_on` and\n #: :func:`~spack.directives.extends` directives\n dependencies: Dict[spack.spec.Spec, Dict[str, spack.dependency.Dependency]]\n #: Class level dictionary populated by :func:`~spack.directives.extends` directives\n extendees: Dict[str, Tuple[spack.spec.Spec, spack.spec.Spec]]\n #: Class level dictionary populated by :func:`~spack.directives.conflicts` directives\n conflicts: Dict[spack.spec.Spec, List[Tuple[spack.spec.Spec, Optional[str]]]]\n #: Class level dictionary populated by :func:`~spack.directives.requires` directives\n requirements: Dict[\n spack.spec.Spec, List[Tuple[Tuple[spack.spec.Spec, ...], str, Optional[str]]]\n ]\n #: Class level dictionary populated by :func:`~spack.directives.provides` directives\n provided: Dict[spack.spec.Spec, Set[spack.spec.Spec]]\n #: Class level dictionary populated by :func:`~spack.directives.provides` directives\n provided_together: Dict[spack.spec.Spec, List[Set[str]]]\n #: Class level dictionary populated by :func:`~spack.directives.patch` directives\n patches: Dict[spack.spec.Spec, List[spack.patch.Patch]]\n #: Class level dictionary populated by :func:`~spack.directives.variant` directives\n variants: Dict[spack.spec.Spec, Dict[str, spack.variant.Variant]]\n #: Class level dictionary populated by :func:`~spack.directives.license` directives\n licenses: Dict[spack.spec.Spec, str]\n #: Class level dictionary populated by :func:`~spack.directives.can_splice` directives\n splice_specs: Dict[spack.spec.Spec, Tuple[spack.spec.Spec, Union[None, str, List[str]]]]\n #: Class level dictionary populated by :func:`~spack.directives.redistribute` directives\n disable_redistribute: Dict[spack.spec.Spec, DisableRedistribute]\n\n #: Must be defined as a fallback for old specs that don't have the ``build_system`` variant\n default_buildsystem: str\n\n #: Use :attr:`~spack.package_base.PackageBase.default_buildsystem` instead of this attribute,\n #: which is deprecated\n legacy_buildsystem: str\n\n #: Used when reporting the build system to users\n build_system_class: str = \"PackageBase\"\n\n #: By default, packages are not virtual\n #: Virtual packages override this attribute\n virtual: bool = False\n\n #: Most Spack packages are used to install source or binary code while\n #: those that do not can be used to install a set of other Spack packages.\n has_code: bool = True\n\n #: By default we build in parallel. Subclasses can override this.\n parallel: bool = True\n\n #: By default do not run tests within package's install()\n run_tests: bool = False\n\n #: Most packages are NOT extendable. Set to True if you want extensions.\n extendable: bool = False\n\n #: When True, add RPATHs for the entire DAG. When False, add RPATHs only\n #: for immediate dependencies.\n transitive_rpaths: bool = True\n\n #: List of shared objects that should be replaced with a different library at\n #: runtime. Typically includes stub libraries like ``libcuda.so``. When linking\n #: against a library listed here, the dependent will only record its soname\n #: or filename, not its absolute path, so that the dynamic linker will search\n #: for it. Note: accepts both file names and directory names, for example\n #: ``[\"libcuda.so\", \"stubs\"]`` will ensure ``libcuda.so`` and all libraries in the\n #: ``stubs`` directory are not bound by path.\n non_bindable_shared_objects: List[str] = []\n\n #: List of fnmatch patterns of library file names (specifically DT_NEEDED entries) that are not\n #: expected to be locatable in RPATHs. Generally this is a problem, and Spack install with\n #: config:shared_linking:strict will cause install failures if such libraries are found.\n #: However, in certain cases it can be hard if not impossible to avoid accidental linking\n #: against system libraries; until that is resolved, this attribute can be used to suppress\n #: errors.\n unresolved_libraries: List[str] = []\n\n #: List of prefix-relative file paths (or a single path). If these do\n #: not exist after install, or if they exist but are not files,\n #: sanity checks fail.\n sanity_check_is_file: List[str] = []\n\n #: List of prefix-relative directory paths (or a single path). If\n #: these do not exist after install, or if they exist but are not\n #: directories, sanity checks will fail.\n sanity_check_is_dir: List[str] = []\n\n #: Boolean. Set to ``True`` for packages that require a manual download.\n #: This is currently used by package sanity tests and generation of a\n #: more meaningful fetch failure error.\n manual_download: bool = False\n\n #: Set of additional options used when fetching package versions.\n fetch_options: Dict[str, Any] = {}\n\n #\n # Set default licensing information\n #\n #: If set to ``True``, this software requires a license.\n #: If set to ``False``, all of the ``license_*`` attributes will\n #: be ignored. Defaults to ``False``.\n license_required: bool = False\n\n #: Contains the symbol used by the license manager to denote\n #: a comment. Defaults to ``#``.\n license_comment: str = \"#\"\n\n #: These are files that the software searches for when\n #: looking for a license. All file paths must be relative to the\n #: installation directory. More complex packages like Intel may require\n #: multiple licenses for individual components. Defaults to the empty list.\n license_files: List[str] = []\n\n #: Environment variables that can be set to tell the\n #: software where to look for a license if it is not in the usual location.\n #: Defaults to the empty list.\n license_vars: List[str] = []\n\n #: A URL pointing to license setup instructions for the software.\n #: Defaults to the empty string.\n license_url: str = \"\"\n\n #: Verbosity level, preserved across installs.\n _verbose = None\n\n #: Package homepage where users can find more information about the package\n homepage: ClassProperty[Optional[str]] = None\n\n #: Default list URL (place to find available versions)\n list_url: ClassProperty[Optional[str]] = None\n\n #: Link depth to which list_url should be searched for new versions\n list_depth: int = 0\n\n #: List of GitHub usernames of package maintainers.\n #: Do not include @ here in order not to unnecessarily ping the users.\n maintainers: List[str] = []\n\n #: Set to ``True`` to indicate the stand-alone test requires a compiler.\n #: It is used to ensure a compiler and build dependencies like ``cmake``\n #: are available to build a custom test code.\n test_requires_compiler: bool = False\n\n #: TestSuite instance used to manage stand-alone tests for 1+ specs.\n test_suite: Optional[Any] = None\n\n def __init__(self, spec: spack.spec.Spec) -> None:\n # this determines how the package should be built.\n self.spec = spec\n\n # Allow custom staging paths for packages\n self.path = None\n\n # Keep track of whether or not this package was installed from\n # a binary cache.\n self.installed_from_binary_cache = False\n\n # Ensure that only one of these two attributes are present\n if getattr(self, \"url\", None) and getattr(self, \"urls\", None):\n msg = \"a package can have either a 'url' or a 'urls' attribute\"\n msg += \" [package '{0.name}' defines both]\"\n raise ValueError(msg.format(self))\n\n # init internal variables\n self._stage: Optional[stg.StageComposite] = None\n # need to track patch stages separately, in order to apply them\n self._patch_stages: List[stg.Stage] = []\n self._fetcher = None\n self._tester: Optional[Any] = None\n\n # Set up timing variables\n self._fetch_time = 0.0\n\n super().__init__()\n\n def __getitem__(self, key: str) -> \"PackageBase\":\n return self.spec[key].package\n\n @classmethod\n def dependency_names(cls):\n return _subkeys(cls.dependencies)\n\n @classmethod\n def dependencies_by_name(cls, when: bool = False):\n return _by_subkey(cls.dependencies, when=when)\n\n # Accessors for variants\n # External code working with Variants should go through the methods below\n\n @classmethod\n def variant_names(cls) -> List[str]:\n return _subkeys(cls.variants)\n\n @classmethod\n def has_variant(cls, name) -> bool:\n return _has_subkey(cls.variants, name)\n\n @classmethod\n def num_variant_definitions(cls) -> int:\n \"\"\"Total number of variant definitions in this class so far.\"\"\"\n return _num_definitions(cls.variants)\n\n @classmethod\n def variant_definitions(cls, name: str) -> List[Tuple[spack.spec.Spec, spack.variant.Variant]]:\n \"\"\"Iterator over (when_spec, Variant) for all variant definitions for a particular name.\"\"\"\n return _definitions(cls.variants, name)\n\n @classmethod\n def variant_items(cls) -> Iterable[Tuple[spack.spec.Spec, Dict[str, spack.variant.Variant]]]:\n \"\"\"Iterate over ``cls.variants.items()`` with overridden definitions removed.\"\"\"\n # Note: This is quadratic in the average number of variant definitions per name.\n # That is likely close to linear in practice, as there are few variants with\n # multiple definitions (but it matters when they are there).\n exclude = {\n name: [id(vdef) for _, vdef in cls.variant_definitions(name)]\n for name in cls.variant_names()\n }\n\n for when, variants_by_name in cls.variants.items():\n filtered_variants_by_name = {\n name: vdef for name, vdef in variants_by_name.items() if id(vdef) in exclude[name]\n }\n\n if filtered_variants_by_name:\n yield when, filtered_variants_by_name\n\n def get_variant(self, name: str) -> spack.variant.Variant:\n \"\"\"Get the highest precedence variant definition matching this package's spec.\n\n Arguments:\n name: name of the variant definition to get\n \"\"\"\n try:\n highest_to_lowest = reversed(self.variant_definitions(name))\n return next(vdef for when, vdef in highest_to_lowest if self.spec.satisfies(when))\n except StopIteration:\n raise ValueError(f\"No variant '{name}' on spec: {self.spec}\")\n\n @classmethod\n def validate_variant_names(self, spec: spack.spec.Spec):\n \"\"\"Check that all variant names on Spec exist in this package.\n\n Raises ``UnknownVariantError`` if invalid variants are on the spec.\n \"\"\"\n names = self.variant_names()\n for v in spec.variants:\n if v not in names:\n raise spack.variant.UnknownVariantError(\n f\"No such variant '{v}' in package {self.name}\", [v]\n )\n\n @classproperty\n def package_dir(cls):\n \"\"\"Directory where the package.py file lives.\"\"\"\n return os.path.abspath(os.path.dirname(cls.module.__file__))\n\n @classproperty\n def module(cls):\n \"\"\"Module instance that this package class is defined in.\n\n We use this to add variables to package modules. This makes\n install() methods easier to write (e.g., can call configure())\n \"\"\"\n return sys.modules[cls.__module__]\n\n @classproperty\n def namespace(cls):\n \"\"\"Spack namespace for the package, which identifies its repo.\"\"\"\n return spack.repo.namespace_from_fullname(cls.__module__)\n\n @classproperty\n def fullname(cls):\n \"\"\"Name of this package, including the namespace\"\"\"\n return \"%s.%s\" % (cls.namespace, cls.name)\n\n @classproperty\n def fullnames(cls):\n \"\"\"Fullnames for this package and any packages from which it inherits.\"\"\"\n fullnames = []\n for base in cls.__mro__:\n if not spack.repo.is_package_module(base.__module__):\n break\n fullnames.append(base.fullname)\n return fullnames\n\n @classproperty\n def name(cls):\n \"\"\"The name of this package.\"\"\"\n if cls._name is None:\n # We cannot know the exact package API version, but we can distinguish between v1\n # v2 based on the module. We don't want to figure out the exact package API version\n # since it requires parsing the repo.yaml.\n module = cls.__module__\n\n if module.startswith(spack.repo.PKG_MODULE_PREFIX_V1):\n version = (1, 0)\n elif module.startswith(spack.repo.PKG_MODULE_PREFIX_V2):\n version = (2, 0)\n else:\n raise ValueError(f\"Package {cls.__qualname__} is not a known Spack package\")\n\n if version < (2, 0):\n # spack.pkg.builtin.package_name.\n _, _, pkg_module = module.rpartition(\".\")\n else:\n # spack_repo.builtin.packages.package_name.package\n pkg_module = module.rsplit(\".\", 2)[-2]\n\n cls._name = spack.util.naming.pkg_dir_to_pkg_name(pkg_module, version)\n return cls._name\n\n @classproperty\n def global_license_dir(cls):\n \"\"\"Returns the directory where license files for all packages are stored.\"\"\"\n return spack.util.path.canonicalize_path(spack.config.get(\"config:license_dir\"))\n\n @property\n def global_license_file(self):\n \"\"\"Returns the path where a global license file for this\n particular package should be stored.\"\"\"\n if not self.license_files:\n return\n return os.path.join(\n self.global_license_dir, self.name, os.path.basename(self.license_files[0])\n )\n\n # Source redistribution must be determined before concretization (because source mirrors work\n # with abstract specs).\n @classmethod\n def redistribute_source(cls, spec):\n \"\"\"Whether it should be possible to add the source of this\n package to a Spack mirror.\"\"\"\n for when_spec, disable_redistribute in cls.disable_redistribute.items():\n if disable_redistribute.source and spec.satisfies(when_spec):\n return False\n return True\n\n @property\n def redistribute_binary(self):\n \"\"\"Whether it should be possible to create a binary out of an installed instance of this\n package.\"\"\"\n for when_spec, disable_redistribute in self.disable_redistribute.items():\n if disable_redistribute.binary and self.spec.satisfies(when_spec):\n return False\n return True\n\n @property\n def keep_werror(self) -> Optional[Literal[\"all\", \"specific\", \"none\"]]:\n \"\"\"Keep ``-Werror`` flags, matches ``config:flags:keep_werror`` to override config.\n\n Valid return values are:\n\n * ``\"all\"``: keep all ``-Werror`` flags.\n * ``\"specific\"``: keep only ``-Werror=specific-warning`` flags.\n * ``\"none\"``: filter out all ``-Werror*`` flags.\n * :data:`None`: respect the user's configuration (``\"none\"`` by default).\n \"\"\"\n if self.spec.satisfies(\"%nvhpc@:23.3\"):\n # Filtering works by replacing -Werror with -Wno-error, but older nvhpc and\n # PGI do not understand -Wno-error, so we disable filtering.\n return \"all\"\n\n elif self.spec.satisfies(\"%nvhpc@23.4:\"):\n # newer nvhpc supports -Wno-error but can't disable specific warnings with\n # -Wno-error=warning. Skip -Werror=warning, but still filter -Werror.\n return \"specific\"\n\n else:\n # use -Werror disablement by default for other compilers\n return None\n\n @property\n def version(self):\n if not self.spec.versions.concrete:\n raise ValueError(\n \"Version requested for a package that does not have a concrete version.\"\n )\n return self.spec.versions[0]\n\n @classmethod\n @memoized\n def version_urls(cls) -> Dict[StandardVersion, str]:\n \"\"\"Dict of explicitly defined URLs for versions of this package.\n\n Return:\n An dict mapping version to url, ordered by version.\n\n A version's URL only appears in the result if it has an an explicitly defined ``url``\n argument. So, this list may be empty if a package only defines ``url`` at the top level.\n \"\"\"\n return {v: args[\"url\"] for v, args in sorted(cls.versions.items()) if \"url\" in args}\n\n def nearest_url(self, version):\n \"\"\"Finds the URL with the \"closest\" version to ``version``.\n\n This uses the following precedence order:\n\n 1. Find the next lowest or equal version with a URL.\n 2. If no lower URL, return the next *higher* URL.\n 3. If no higher URL, return None.\n \"\"\"\n version_urls = self.version_urls()\n\n if version in version_urls:\n return version_urls[version]\n\n last_url = None\n for v, u in self.version_urls().items():\n if v > version:\n if last_url:\n return last_url\n last_url = u\n\n return last_url\n\n def url_for_version(self, version: Union[str, StandardVersion]) -> str:\n \"\"\"Returns a URL from which the specified version of this package may be downloaded.\n\n Arguments:\n version: The version for which a URL is sought.\"\"\"\n return self._implement_all_urls_for_version(version)[0]\n\n def _update_external_dependencies(\n self, extendee_spec: Optional[spack.spec.Spec] = None\n ) -> None:\n \"\"\"\n Method to override in package classes to handle external dependencies\n \"\"\"\n pass\n\n def detect_dev_src_change(self) -> bool:\n \"\"\"\n Method for checking for source code changes to trigger rebuild/reinstall\n \"\"\"\n dev_path_var = self.spec.variants.get(\"dev_path\", None)\n _, record = spack.store.STORE.db.query_by_spec_hash(self.spec.dag_hash())\n assert dev_path_var and record, \"dev_path variant and record must be present\"\n return fsys.recursive_mtime_greater_than(dev_path_var.value, record.installation_time)\n\n @classmethod\n def version_or_package_attr(cls, attr, version, default=NO_DEFAULT):\n \"\"\"\n Get an attribute that could be on the version or package with preference to the version\n \"\"\"\n version_attrs = cls.versions.get(version)\n if version_attrs and attr in version_attrs:\n return version_attrs.get(attr)\n if default is NO_DEFAULT and not hasattr(cls, attr):\n raise PackageError(f\"{attr} attribute not defined on {cls.name}\")\n return getattr(cls, attr, default)\n\n @classmethod\n def needs_commit(cls, version) -> bool:\n \"\"\"\n Method for checking if the package instance needs a commit sha to be found\n \"\"\"\n if isinstance(version, GitVersion):\n return True\n\n ver_attrs = cls.versions.get(version)\n if ver_attrs:\n return bool(ver_attrs.get(\"commit\") or ver_attrs.get(\"tag\") or ver_attrs.get(\"branch\"))\n\n return False\n\n @classmethod\n def _resolve_git_provenance(cls, spec) -> None:\n # early return cases, don't overwrite user intention\n # commit pre-assigned or develop specs don't need commits changed\n # since this would create un-necessary churn\n if \"commit\" in spec.variants or spec.is_develop:\n return\n\n if is_git_version(str(spec.version)):\n ref = spec.version.ref\n else:\n v_attrs = cls.versions.get(spec.version, {})\n if \"commit\" in v_attrs:\n spec.variants[\"commit\"] = spack.variant.SingleValuedVariant(\n \"commit\", v_attrs[\"commit\"]\n )\n return\n ref = v_attrs.get(\"tag\") or v_attrs.get(\"branch\")\n\n if not ref:\n raise VersionError(\n f\"{spec.name}'s version {str(spec.version)} \"\n \"is missing a git ref (commit, tag or branch)\"\n )\n\n # Look for commits in the following places:\n # 1) mirror archive file, (cheapish, local, staticish)\n # 2) URL (cheap, remote, dynamic)\n #\n # If users pre-stage (_LOCAL_CACHE), or use a mirror they can expect\n # consistent commit resolution\n sha = None\n\n # construct a package instance to get fetch/staging together\n pkg_instance = cls(spec.copy())\n\n try:\n pkg_instance.do_fetch(mirror_only=True)\n except spack.error.FetchError:\n pass\n if pkg_instance.stage.archive_file:\n sha = spack.util.archive.retrieve_commit_from_archive(\n pkg_instance.stage.archive_file, ref\n )\n if not sha:\n url = cls.version_or_package_attr(\"git\", spec.version)\n sha = spack.util.git.get_commit_sha(url, ref)\n\n if sha:\n spec.variants[\"commit\"] = spack.variant.SingleValuedVariant(\"commit\", sha)\n\n def resolve_binary_provenance(self):\n \"\"\"\n Method to ensure concrete spec has binary provenance.\n Base implementation will look up git commits when appropriate.\n Packages may override this implementation for custom implementations\n \"\"\"\n self._resolve_git_provenance(self.spec)\n\n def all_urls_for_version(self, version: StandardVersion) -> List[str]:\n \"\"\"Return all URLs derived from version_urls(), url, urls, and\n list_url (if it contains a version) in a package in that order.\n\n Args:\n version: the version for which a URL is sought\n \"\"\"\n uf = None\n if type(self).url_for_version != PackageBase.url_for_version:\n uf = self.url_for_version\n return self._implement_all_urls_for_version(version, uf)\n\n def _implement_all_urls_for_version(\n self,\n version: Union[str, StandardVersion],\n custom_url_for_version: Optional[Callable[[StandardVersion], Optional[str]]] = None,\n ) -> List[str]:\n version = StandardVersion.from_string(version) if isinstance(version, str) else version\n\n urls: List[str] = []\n\n # If we have a specific URL for this version, don't extrapolate.\n url = self.version_urls().get(version)\n if url:\n urls.append(url)\n\n # if there is a custom url_for_version, use it\n if custom_url_for_version is not None:\n u = custom_url_for_version(version)\n if u is not None and u not in urls:\n urls.append(u)\n\n def sub_and_add(u: Optional[str]) -> None:\n if u is None:\n return\n # skip the url if there is no version to replace\n try:\n spack.url.parse_version(u)\n except spack.url.UndetectableVersionError:\n return\n urls.append(spack.url.substitute_version(u, self.url_version(version)))\n\n # If no specific URL, use the default, class-level URL\n sub_and_add(getattr(self, \"url\", None))\n for u in getattr(self, \"urls\", []):\n sub_and_add(u)\n\n sub_and_add(getattr(self, \"list_url\", None))\n\n # if no version-bearing URLs can be found, try them raw\n if not urls:\n default_url = getattr(self, \"url\", getattr(self, \"urls\", [None])[0])\n\n # if no exact match AND no class-level default, use the nearest URL\n if not default_url:\n default_url = self.nearest_url(version)\n\n # if there are NO URLs to go by, then we can't do anything\n if not default_url:\n raise NoURLError(self.__class__)\n urls.append(spack.url.substitute_version(default_url, self.url_version(version)))\n\n return urls\n\n def find_valid_url_for_version(self, version: StandardVersion) -> Optional[str]:\n \"\"\"Returns a URL from which the specified version of this package may be downloaded after\n testing whether the url is valid. Will try ``url``, ``urls``, and :attr:`list_url`\n before failing.\n\n Arguments:\n version: The version for which a URL is sought.\n \"\"\"\n urls = self.all_urls_for_version(version)\n\n for u in urls:\n if spack.util.web.url_exists(u):\n return u\n\n return None\n\n def _make_resource_stage(self, root_stage, resource):\n pretty_resource_name = fsys.polite_filename(f\"{resource.name}-{self.version}\")\n return stg.ResourceStage(\n resource.fetcher,\n root=root_stage,\n resource=resource,\n name=self._resource_stage(resource),\n mirror_paths=spack.mirrors.layout.default_mirror_layout(\n resource.fetcher, os.path.join(self.name, pretty_resource_name)\n ),\n mirrors=spack.mirrors.mirror.MirrorCollection(source=True).values(),\n path=self.path,\n )\n\n def _download_search(self):\n dynamic_fetcher = fs.from_list_url(self)\n return [dynamic_fetcher] if dynamic_fetcher else []\n\n def _make_root_stage(self, fetcher):\n # Construct a mirror path (TODO: get this out of package.py)\n format_string = \"{name}-{version}\"\n pretty_name = self.spec.format_path(format_string)\n mirror_paths = spack.mirrors.layout.default_mirror_layout(\n fetcher, os.path.join(self.name, pretty_name), self.spec\n )\n # Construct a path where the stage should build..\n s = self.spec\n stage_name = stg.compute_stage_name(s)\n stage = stg.Stage(\n fetcher,\n mirror_paths=mirror_paths,\n mirrors=spack.mirrors.mirror.MirrorCollection(source=True).values(),\n name=stage_name,\n path=self.path,\n search_fn=self._download_search,\n )\n return stage\n\n def _make_stages(self) -> Tuple[stg.StageComposite, List[stg.Stage]]:\n \"\"\"Create stages for this package, its resources, and any patches to be applied.\n\n Returns:\n A StageComposite containing all stages created, as well as a list of patch stages for\n any patches that need to be fetched remotely.\n\n The StageComposite is used to manage (create destroy, etc.) the stages.\n\n The list of patch stages will be in the same order that patches are to be applied\n to the package's staged source code. This is needed in order to apply the patches later.\n\n \"\"\"\n # If it's a dev package (not transitively), use a DIY stage object\n dev_path_var = self.spec.variants.get(\"dev_path\", None)\n if dev_path_var:\n dev_path = dev_path_var.value\n link_format = spack.config.get(\"config:develop_stage_link\")\n if not link_format:\n link_format = \"build-{arch}-{hash:7}\"\n if link_format == \"None\":\n stage_link = None\n else:\n stage_link = self.spec.format_path(link_format)\n source_stage = stg.DevelopStage(\n stg.compute_stage_name(self.spec), dev_path, stage_link\n )\n else:\n source_stage = self._make_root_stage(self.fetcher)\n\n # all_stages is source + resources + patches\n all_stages = stg.StageComposite()\n all_stages.append(source_stage)\n all_stages.extend(\n self._make_resource_stage(source_stage, r) for r in self._get_needed_resources()\n )\n\n def make_patch_stage(patch: spack.patch.UrlPatch, uniqe_part: str):\n # UrlPatches can make their own fetchers\n fetcher = patch.fetcher()\n\n # The same package can have multiple patches with the same name but\n # with different contents, therefore apply a subset of the hash.\n fetch_digest = patch.archive_sha256 or patch.sha256\n\n name = f\"{os.path.basename(patch.url)}-{fetch_digest[:7]}\"\n per_package_ref = os.path.join(patch.owner.split(\".\")[-1], name)\n mirror_ref = spack.mirrors.layout.default_mirror_layout(fetcher, per_package_ref)\n\n return stg.Stage(\n fetcher,\n name=f\"{stg.stage_prefix}-{uniqe_part}-patch-{fetch_digest}\",\n mirror_paths=mirror_ref,\n mirrors=spack.mirrors.mirror.MirrorCollection(source=True).values(),\n )\n\n if self.spec.concrete:\n patches = self.spec.patches\n uniqe_part = self.spec.dag_hash(7)\n else:\n # The only code path that gets here is `spack mirror create --all`,\n # which needs all matching patches.\n patch_lists = [\n plist for when, plist in self.patches.items() if self.spec.intersects(when)\n ]\n patches = sum(patch_lists, [])\n uniqe_part = self.name\n\n patch_stages = [\n make_patch_stage(p, uniqe_part) for p in patches if isinstance(p, spack.patch.UrlPatch)\n ]\n all_stages.extend(patch_stages)\n\n return all_stages, patch_stages\n\n @property\n def stage(self):\n \"\"\"Get the build staging area for this package.\n\n This automatically instantiates a ``Stage`` object if the package\n doesn't have one yet, but it does not create the Stage directory\n on the filesystem.\n \"\"\"\n if not self.spec.versions.concrete:\n raise ValueError(\"Cannot retrieve stage for package without concrete version.\")\n if self._stage is None:\n self._stage, self._patch_stages = self._make_stages()\n return self._stage\n\n @stage.setter\n def stage(self, stage: stg.StageComposite):\n \"\"\"Allow a stage object to be set to override the default.\"\"\"\n self._stage = stage\n\n @property\n def env_path(self):\n \"\"\"Return the build environment file path associated with staging.\"\"\"\n return os.path.join(self.stage.path, _spack_build_envfile)\n\n @property\n def env_mods_path(self):\n \"\"\"\n Return the build environment modifications file path associated with\n staging.\n \"\"\"\n return os.path.join(self.stage.path, _spack_build_envmodsfile)\n\n @property\n def metadata_dir(self):\n \"\"\"Return the install metadata directory.\"\"\"\n return spack.store.STORE.layout.metadata_path(self.spec)\n\n @property\n def install_env_path(self):\n \"\"\"\n Return the build environment file path on successful installation.\n \"\"\"\n # Backward compatibility: Return the name of an existing log path;\n # otherwise, return the current install env path name.\n old_filename = os.path.join(self.metadata_dir, \"build.env\")\n if os.path.exists(old_filename):\n return old_filename\n else:\n return os.path.join(self.metadata_dir, _spack_build_envfile)\n\n @property\n def log_path(self):\n \"\"\"Return the build log file path associated with staging.\"\"\"\n return os.path.join(self.stage.path, _spack_build_logfile)\n\n @property\n def phase_log_files(self):\n \"\"\"Find sorted phase log files written to the staging directory\"\"\"\n logs_dir = os.path.join(self.stage.path, \"spack-build-*-out.txt\")\n log_files = glob.glob(logs_dir)\n log_files.sort()\n return log_files\n\n @property\n def install_log_path(self):\n \"\"\"Return the (compressed) build log file path on successful installation\"\"\"\n # Backward compatibility: Return the name of an existing install log.\n for filename in [_spack_build_logfile, \"build.out\", \"build.txt\"]:\n old_log = os.path.join(self.metadata_dir, filename)\n if os.path.exists(old_log):\n return old_log\n\n # Otherwise, return the current install log path name.\n return os.path.join(self.metadata_dir, _spack_build_logfile + \".gz\")\n\n @property\n def configure_args_path(self):\n \"\"\"Return the configure args file path associated with staging.\"\"\"\n return os.path.join(self.stage.path, _spack_configure_argsfile)\n\n @property\n def times_log_path(self):\n \"\"\"Return the times log json file.\"\"\"\n return os.path.join(self.metadata_dir, spack_times_log)\n\n @property\n def install_configure_args_path(self):\n \"\"\"Return the configure args file path on successful installation.\"\"\"\n return os.path.join(self.metadata_dir, _spack_configure_argsfile)\n\n def archive_install_test_log(self):\n \"\"\"Archive the install-phase test log, if present.\"\"\"\n if getattr(self, \"tester\", None):\n self.tester.archive_install_test_log(self.metadata_dir)\n\n @property\n def tester(self):\n import spack.install_test\n\n if not self.spec.versions.concrete:\n raise ValueError(\"Cannot retrieve tester for package without concrete version.\")\n\n if not self._tester:\n self._tester = spack.install_test.PackageTest(self)\n return self._tester\n\n @property\n def fetcher(self):\n if not self.spec.versions.concrete:\n raise ValueError(\"Cannot retrieve fetcher for package without concrete version.\")\n if not self._fetcher:\n # assign private member with the public setter api for error checking\n self.fetcher = fs.for_package_version(self)\n return self._fetcher\n\n @fetcher.setter\n def fetcher(self, f):\n self._fetcher = f\n self._fetcher.set_package(self)\n\n @classmethod\n def dependencies_of_type(cls, deptypes: dt.DepFlag):\n \"\"\"Get names of dependencies that can possibly have these deptypes.\n\n This analyzes the package and determines which dependencies *can*\n be a certain kind of dependency. Note that they may not *always*\n be this kind of dependency, since dependencies can be optional,\n so something may be a build dependency in one configuration and a\n run dependency in another.\n \"\"\"\n return {\n name\n for name, dependencies in cls.dependencies_by_name().items()\n if any(deptypes & dep.depflag for dep in dependencies)\n }\n\n # TODO: allow more than one active extendee.\n @property\n def extendee_spec(self) -> Optional[spack.spec.Spec]:\n \"\"\"Spec of the extendee of this package, or None if it is not an extension.\"\"\"\n if not self.extendees:\n return None\n\n # If the extendee is in the spec's deps already, return that.\n deps = [\n dep\n for dep in self.spec.dependencies(deptype=(\"link\", \"run\"))\n for d, when in self.extendees.values()\n if dep.satisfies(d) and self.spec.satisfies(when)\n ]\n\n if deps:\n assert len(deps) == 1\n return deps[0]\n\n # if the spec is concrete already, then it extends something\n # that is an *optional* dependency, and the dep isn't there.\n if self.spec._concrete:\n return None\n else:\n # If it's not concrete, then return the spec from the\n # extends() directive since that is all we know so far.\n spec_str = next(iter(self.extendees))\n return spack.spec.Spec(spec_str)\n\n @property\n def is_extension(self):\n # if it is concrete, it's only an extension if it actually\n # dependes on the extendee.\n if self.spec._concrete:\n return self.extendee_spec is not None\n else:\n # If not, then it's an extension if it *could* be an extension\n return bool(self.extendees)\n\n def extends(self, spec: spack.spec.Spec) -> bool:\n \"\"\"\n Returns True if this package extends the given spec.\n\n If ``self.spec`` is concrete, this returns whether this package extends\n the given spec.\n\n If ``self.spec`` is not concrete, this returns whether this package may\n extend the given spec.\n \"\"\"\n if spec.name not in self.extendees:\n return False\n s = self.extendee_spec\n return s is not None and spec.satisfies(s)\n\n def provides(self, vpkg_name: str) -> bool:\n \"\"\"\n True if this package provides a virtual package with the specified name\n \"\"\"\n return any(\n any(spec.name == vpkg_name for spec in provided)\n for when_spec, provided in self.provided.items()\n if self.spec.intersects(when_spec)\n )\n\n def intersects(self, spec: spack.spec.Spec) -> bool:\n \"\"\"Context-ful intersection that takes into account package information.\n\n By design, ``Spec.intersects()`` does not know anything about package metadata.\n This avoids unnecessary package lookups and keeps things efficient where extra\n information is not needed, and it decouples ``Spec`` from ``PackageBase``.\n\n In many cases, though, we can rule more cases out in ``intersects()`` if we\n know, for example, that certain variants are always single-valued, or that\n certain variants are conditional on other variants. This adds logic for such\n cases when they are knowable.\n\n Note that because ``intersects()`` is conservative, it can only give false\n positives (\"i.e., the two specs *may* overlap\"), not false negatives. This\n method can fix false positives (i.e. it may return ``False`` when\n ``Spec.intersects()`` would return ``True``, but it will never return ``True``\n when ``Spec.intersects()`` returns ``False``.\n\n \"\"\"\n # Spec.intersects() is right when False\n if not self.spec.intersects(spec):\n return False\n\n def sv_variant_conflicts(spec, variant):\n name = variant.name\n return (\n variant.name in spec.variants\n and all(not d[name].multi for when, d in self.variants.items() if name in d)\n and spec.variants[name].value != variant.value\n )\n\n # Specs don't know if a variant is single- or multi-valued (concretization handles this)\n # But, we know if the spec has a value for a single-valued variant, it *has* to equal the\n # value in self.spec, if there is one.\n for v, variant in spec.variants.items():\n if sv_variant_conflicts(self.spec, variant):\n return False\n\n # if there is no intersecting condition for a conditional variant, it can't exist. e.g.:\n # - cuda_arch= can't be satisfied when ~cuda.\n # - generator= can't be satisfied when build_system=autotools\n def mutually_exclusive(spec, variant_name):\n return all(\n not spec.intersects(when)\n or any(sv_variant_conflicts(spec, wv) for wv in when.variants.values())\n for when, d in self.variants.items()\n if variant_name in d\n )\n\n names = self.variant_names()\n for v in set(itertools.chain(spec.variants, self.spec.variants)):\n if v not in names: # treat unknown variants as intersecting\n continue\n\n if mutually_exclusive(self.spec, v) or mutually_exclusive(spec, v):\n return False\n\n return True\n\n @property\n def virtuals_provided(self):\n \"\"\"\n virtual packages provided by this package with its spec\n \"\"\"\n return [\n vspec\n for when_spec, provided in self.provided.items()\n for vspec in sorted(provided)\n if self.spec.satisfies(when_spec)\n ]\n\n @classmethod\n def provided_virtual_names(cls):\n \"\"\"Return sorted list of names of virtuals that can be provided by this package.\"\"\"\n return sorted(\n set(vpkg.name for virtuals in cls.provided.values() for vpkg in sorted(virtuals))\n )\n\n @property\n def prefix(self):\n \"\"\"Get the prefix into which this package should be installed.\"\"\"\n return self.spec.prefix\n\n @property\n def home(self):\n return self.prefix\n\n @property\n def command(self) -> spack.util.executable.Executable:\n \"\"\"Returns the main executable for this package.\"\"\"\n path = os.path.join(self.home.bin, self.spec.name)\n if fsys.is_exe(path):\n return spack.util.executable.Executable(path)\n raise RuntimeError(f\"Unable to locate {self.spec.name} command in {self.home.bin}\")\n\n def url_version(self, version):\n \"\"\"\n Given a version, this returns a string that should be substituted\n into the package's URL to download that version.\n\n By default, this just returns the version string. Subclasses may need\n to override this, e.g. for boost versions where you need to ensure that\n there are _'s in the download URL.\n \"\"\"\n return str(version)\n\n def remove_prefix(self):\n \"\"\"\n Removes the prefix for a package along with any empty parent\n directories\n \"\"\"\n spack.store.STORE.layout.remove_install_directory(self.spec)\n\n @property\n def download_instr(self) -> str:\n \"\"\"\n Defines the default manual download instructions. Packages can\n override the property to provide more information.\n\n Returns:\n default manual download instructions\n \"\"\"\n required = (\n f\"Manual download is required for {self.spec.name}. \" if self.manual_download else \"\"\n )\n return f\"{required}Refer to {self.homepage} for download instructions.\"\n\n def do_fetch(self, mirror_only=False):\n \"\"\"\n Creates a stage directory and downloads the tarball for this package.\n Working directory will be set to the stage directory.\n \"\"\"\n if not self.has_code or self.spec.external:\n tty.debug(\"No fetch required for {0}\".format(self.name))\n return\n\n checksum = spack.config.get(\"config:checksum\")\n if (\n checksum\n and (self.version not in self.versions)\n and (not isinstance(self.version, GitVersion))\n and (\"dev_path\" not in self.spec.variants)\n ):\n tty.warn(\n \"There is no checksum on file to fetch %s safely.\"\n % self.spec.cformat(\"{name}{@version}\")\n )\n\n # Ask the user whether to skip the checksum if we're\n # interactive, but just fail if non-interactive.\n ck_msg = \"Add a checksum or use --no-checksum to skip this check.\"\n ignore_checksum = False\n if sys.stdout.isatty():\n ignore_checksum = tty.get_yes_or_no(\" Fetch anyway?\", default=False)\n if ignore_checksum:\n tty.debug(\"Fetching with no checksum. {0}\".format(ck_msg))\n\n if not ignore_checksum:\n raise spack.error.FetchError(\n \"Will not fetch %s\" % self.spec.format(\"{name}{@version}\"), ck_msg\n )\n\n deprecated = spack.config.get(\"config:deprecated\")\n if not deprecated and self.versions.get(self.version, {}).get(\"deprecated\", False):\n tty.warn(\n \"{0} is deprecated and may be removed in a future Spack release.\".format(\n self.spec.format(\"{name}{@version}\")\n )\n )\n\n # Ask the user whether to install deprecated version if we're\n # interactive, but just fail if non-interactive.\n dp_msg = (\n \"If you are willing to be a maintainer for this version \"\n \"of the package, submit a PR to remove `deprecated=False\"\n \"`, or use `--deprecated` to skip this check.\"\n )\n ignore_deprecation = False\n if sys.stdout.isatty():\n ignore_deprecation = tty.get_yes_or_no(\" Fetch anyway?\", default=False)\n\n if ignore_deprecation:\n tty.debug(\"Fetching deprecated version. {0}\".format(dp_msg))\n\n if not ignore_deprecation:\n raise spack.error.FetchError(\n \"Will not fetch {0}\".format(self.spec.format(\"{name}{@version}\")), dp_msg\n )\n\n self.stage.create()\n err_msg = None if not self.manual_download else self.download_instr\n start_time = time.time()\n self.stage.fetch(mirror_only, err_msg=err_msg)\n self._fetch_time = time.time() - start_time\n\n if checksum and self.version in self.versions:\n self.stage.check()\n\n self.stage.cache_local()\n\n def do_stage(self, mirror_only=False):\n \"\"\"Unpacks and expands the fetched tarball.\"\"\"\n # Always create the stage directory at this point. Why? A no-code\n # package may want to use the installation process to install metadata.\n self.stage.create()\n\n # Fetch/expand any associated code.\n user_dev_path = spack.config.get(f\"develop:{self.name}:path\", None)\n skip = user_dev_path and os.path.exists(user_dev_path)\n if skip:\n tty.debug(\"Skipping staging because develop path exists\")\n if self.has_code and not self.spec.external and not skip:\n self.do_fetch(mirror_only)\n self.stage.expand_archive()\n else:\n # Support for post-install hooks requires a stage.source_path\n fsys.mkdirp(self.stage.source_path)\n\n def do_patch(self):\n \"\"\"Applies patches if they haven't been applied already.\"\"\"\n if not self.spec.concrete:\n raise ValueError(\"Can only patch concrete packages.\")\n\n # Kick off the stage first. This creates the stage.\n self.do_stage()\n\n # Package can add its own patch function.\n has_patch_fun = hasattr(self, \"patch\") and callable(self.patch)\n\n # Get the patches from the spec (this is a shortcut for the MV-variant)\n patches = self.spec.patches\n\n # If there are no patches, note it.\n if not patches and not has_patch_fun:\n tty.msg(\"No patches needed for {0}\".format(self.name))\n return\n\n # Construct paths to special files in the archive dir used to\n # keep track of whether patches were successfully applied.\n archive_dir = self.stage.source_path\n good_file = os.path.join(archive_dir, \".spack_patched\")\n no_patches_file = os.path.join(archive_dir, \".spack_no_patches\")\n bad_file = os.path.join(archive_dir, \".spack_patch_failed\")\n\n # If we encounter an archive that failed to patch, restage it\n # so that we can apply all the patches again.\n if os.path.isfile(bad_file):\n if self.stage.requires_patch_success:\n tty.debug(\"Patching failed last time. Restaging.\")\n self.stage.restage()\n else:\n # develop specs may have patch failures but should never be restaged\n tty.warn(\n f\"A patch failure was detected in {self.name}.\"\n \" Build errors may occur due to this.\"\n )\n return\n\n # If this file exists, then we already applied all the patches.\n if os.path.isfile(good_file):\n tty.msg(\"Already patched {0}\".format(self.name))\n return\n elif os.path.isfile(no_patches_file):\n tty.msg(\"No patches needed for {0}\".format(self.name))\n return\n\n errors = []\n # Apply all the patches for specs that match this one\n patched = False\n patch_stages = iter(self._patch_stages)\n\n for patch in patches:\n try:\n with fsys.working_dir(self.stage.source_path):\n # get the path either from the stage where it was fetched, or from the Patch\n if isinstance(patch, spack.patch.UrlPatch):\n patch_stage = next(patch_stages)\n patch_path = patch_stage.single_file\n else:\n patch_path = patch.path\n\n spack.patch.apply_patch(\n self.stage.source_path,\n patch_path,\n patch.level,\n patch.working_dir,\n patch.reverse,\n )\n\n tty.msg(f\"Applied patch {patch.path_or_url}\")\n patched = True\n\n except spack.error.SpackError as e:\n # Touch bad file if anything goes wrong.\n fsys.touch(bad_file)\n error_msg = f\"Patch {patch.path_or_url} failed.\"\n if self.stage.requires_patch_success:\n tty.msg(error_msg)\n raise\n else:\n tty.debug(error_msg)\n tty.debug(e)\n errors.append(e)\n\n if has_patch_fun:\n try:\n with fsys.working_dir(self.stage.source_path):\n self.patch()\n tty.msg(\"Ran patch() for {0}\".format(self.name))\n patched = True\n except spack.multimethod.NoSuchMethodError:\n # We are running a multimethod without a default case.\n # If there's no default it means we don't need to patch.\n if not patched:\n # if we didn't apply a patch from a patch()\n # directive, AND the patch function didn't apply, say\n # no patches are needed. Otherwise, we already\n # printed a message for each patch.\n tty.msg(\"No patches needed for {0}\".format(self.name))\n except spack.error.SpackError as e:\n # Touch bad file if anything goes wrong.\n fsys.touch(bad_file)\n error_msg = f\"patch() function failed for {self.name}\"\n if self.stage.requires_patch_success:\n tty.msg(error_msg)\n raise\n else:\n tty.debug(error_msg)\n tty.debug(e)\n errors.append(e)\n\n if not errors:\n # Get rid of any old failed file -- patches have either succeeded\n # or are not needed. This is mostly defensive -- it's needed\n # if we didn't restage\n if os.path.isfile(bad_file):\n os.remove(bad_file)\n\n # touch good or no patches file so that we skip next time.\n if patched:\n fsys.touch(good_file)\n else:\n fsys.touch(no_patches_file)\n\n @classmethod\n def all_patches(cls):\n \"\"\"Retrieve all patches associated with the package.\n\n Retrieves patches on the package itself as well as patches on the\n dependencies of the package.\"\"\"\n patches = []\n for _, patch_list in cls.patches.items():\n for patch in patch_list:\n patches.append(patch)\n\n pkg_deps = cls.dependencies\n for dep_name in pkg_deps:\n for _, dependency in pkg_deps[dep_name].items():\n for _, patch_list in dependency.patches.items():\n for patch in patch_list:\n patches.append(patch)\n\n return patches\n\n def content_hash(self, content: Optional[bytes] = None) -> str:\n \"\"\"Create a hash based on the artifacts and patches used to build this package.\n\n This includes:\n\n * source artifacts (tarballs, repositories) used to build;\n * content hashes (``sha256``'s) of all patches applied by Spack; and\n * canonicalized contents the ``package.py`` recipe used to build.\n\n This hash is only included in Spack's DAG hash for concrete specs, but if it\n happens to be called on a package with an abstract spec, only applicable (i.e.,\n determinable) portions of the hash will be included.\n\n \"\"\"\n # list of components to make up the hash\n hash_content = []\n\n # source artifacts/repositories\n # TODO: resources\n if self.spec.versions.concrete:\n try:\n source_id = fs.for_package_version(self).source_id()\n except (fs.ExtrapolationError, fs.InvalidArgsError, spack.error.NoURLError):\n # ExtrapolationError happens if the package has no fetchers defined.\n # InvalidArgsError happens when there are version directives with args,\n # but none of them identifies an actual fetcher.\n # NoURLError happens if the package is external-only with no url\n source_id = None\n\n if not source_id:\n # TODO? in cases where a digest or source_id isn't available,\n # should this attempt to download the source and set one? This\n # probably only happens for source repositories which are\n # referenced by branch name rather than tag or commit ID.\n from_local_sources = \"dev_path\" in self.spec.variants\n if self.has_code and not self.spec.external and not from_local_sources:\n message = \"Missing a source id for {s.name}@{s.version}\"\n tty.debug(message.format(s=self))\n hash_content.append(\"\".encode(\"utf-8\"))\n else:\n hash_content.append(source_id.encode(\"utf-8\"))\n\n # patch sha256's\n # Only include these if they've been assigned by the concretizer.\n # We check spec._patches_assigned instead of spec.concrete because\n # we have to call package_hash *before* marking specs concrete\n if self.spec._patches_assigned():\n hash_content.extend(\n \":\".join((p.sha256, str(p.level))).encode(\"utf-8\") for p in self.spec.patches\n )\n\n # package.py contents\n hash_content.append(package_hash(self.spec, source=content).encode(\"utf-8\"))\n\n # put it all together and encode as base32\n b32_hash = base64.b32encode(\n hashlib.sha256(bytes().join(sorted(hash_content))).digest()\n ).lower()\n b32_hash = b32_hash.decode(\"utf-8\")\n\n return b32_hash\n\n @property\n def cmake_prefix_paths(self) -> List[str]:\n \"\"\"Return a list of paths to be used in CMake's ``CMAKE_PREFIX_PATH``.\"\"\"\n return [self.prefix]\n\n def _has_make_target(self, target):\n \"\"\"Checks to see if 'target' is a valid target in a Makefile.\n\n Parameters:\n target (str): the target to check for\n\n Returns:\n bool: True if 'target' is found, else False\n \"\"\"\n # Check if we have a Makefile\n for makefile in [\"GNUmakefile\", \"Makefile\", \"makefile\"]:\n if os.path.exists(makefile):\n break\n else:\n tty.debug(\"No Makefile found in the build directory\")\n return False\n\n # Prevent altering LC_ALL for 'make' outside this function\n make = copy.deepcopy(self.module.make)\n\n # Use English locale for missing target message comparison\n make.add_default_env(\"LC_ALL\", \"C\")\n\n # Check if 'target' is a valid target.\n #\n # `make -n target` performs a \"dry run\". It prints the commands that\n # would be run but doesn't actually run them. If the target does not\n # exist, you will see one of the following error messages:\n #\n # GNU Make:\n # make: *** No rule to make target `test'. Stop.\n # *** No rule to make target 'test'. Stop.\n #\n # BSD Make:\n # make: don't know how to make test. Stop\n #\n # Note: \"Stop.\" is not printed when running a Make jobserver (spack env depfile) that runs\n # with `make -k/--keep-going`\n missing_target_msgs = [\n \"No rule to make target `{0}'.\",\n \"No rule to make target '{0}'.\",\n \"don't know how to make {0}.\",\n ]\n\n kwargs = {\n \"fail_on_error\": False,\n \"output\": os.devnull,\n \"error\": str,\n # Remove MAKEFLAGS to avoid inherited flags from Make jobserver (spack env depfile)\n \"extra_env\": {\"MAKEFLAGS\": \"\"},\n }\n\n stderr = make(\"-n\", target, **kwargs)\n\n for missing_target_msg in missing_target_msgs:\n if missing_target_msg.format(target) in stderr:\n tty.debug(\"Target '{0}' not found in {1}\".format(target, makefile))\n return False\n\n return True\n\n def _if_make_target_execute(self, target, *args, **kwargs):\n \"\"\"Runs ``make target`` if 'target' is a valid target in the Makefile.\n\n Parameters:\n target (str): the target to potentially execute\n \"\"\"\n if self._has_make_target(target):\n # Execute target\n self.module.make(target, *args, **kwargs)\n\n def _has_ninja_target(self, target):\n \"\"\"Checks to see if 'target' is a valid target in a Ninja build script.\n\n Parameters:\n target (str): the target to check for\n\n Returns:\n bool: True if 'target' is found, else False\n \"\"\"\n ninja = self.module.ninja\n\n # Check if we have a Ninja build script\n if not os.path.exists(\"build.ninja\"):\n tty.debug(\"No Ninja build script found in the build directory\")\n return False\n\n # Get a list of all targets in the Ninja build script\n # https://ninja-build.org/manual.html#_extra_tools\n all_targets = ninja(\"-t\", \"targets\", \"all\", output=str).split(\"\\n\")\n\n # Check if 'target' is a valid target\n matches = [line for line in all_targets if line.startswith(target + \":\")]\n\n if not matches:\n tty.debug(\"Target '{0}' not found in build.ninja\".format(target))\n return False\n\n return True\n\n def _if_ninja_target_execute(self, target, *args, **kwargs):\n \"\"\"Runs ``ninja target`` if 'target' is a valid target in the Ninja\n build script.\n\n Parameters:\n target (str): the target to potentially execute\n \"\"\"\n if self._has_ninja_target(target):\n # Execute target\n self.module.ninja(target, *args, **kwargs)\n\n def _get_needed_resources(self):\n # We use intersects here cause it would also work if self.spec is abstract\n resources = [\n resource\n for when_spec, resource_list in self.resources.items()\n if self.spec.intersects(when_spec)\n for resource in resource_list\n ]\n # Sorts the resources by the length of the string representing their destination. Since any\n # nested resource must contain another resource's path, that should work\n return sorted(resources, key=lambda res: len(res.destination))\n\n def _resource_stage(self, resource):\n pieces = [\"resource\", resource.name, self.spec.dag_hash()]\n resource_stage_folder = \"-\".join(pieces)\n return resource_stage_folder\n\n def do_test(self, *, dirty=False, externals=False, timeout: Optional[int] = None):\n if self.test_requires_compiler and not any(\n lang in self.spec for lang in (\"c\", \"cxx\", \"fortran\")\n ):\n tty.error(\n f\"Skipping tests for package {self.spec}, since a compiler is required, \"\n f\"but not available\"\n )\n return\n\n kwargs = {\n \"dirty\": dirty,\n \"fake\": False,\n \"context\": \"test\",\n \"externals\": externals,\n \"verbose\": tty.is_verbose(),\n }\n\n self.tester.stand_alone_tests(kwargs, timeout=timeout)\n\n def _unit_test_check(self) -> bool:\n \"\"\"Hook for Spack's own unit tests to assert things about package internals.\n\n Unit tests can override this function to perform checks after\n ``Package.install`` and all post-install hooks run, but before\n the database is updated.\n\n The overridden function may indicate that the install procedure\n should terminate early (before updating the database) by\n returning :data:`False` (or any value such that ``bool(result)`` is\n :data:`False`).\n\n Return:\n :data:`True` to continue, :data:`False` to skip ``install()``\n \"\"\"\n return True\n\n @classmethod\n def inject_flags(cls: Type[Pb], name: str, flags: Iterable[str]) -> FLAG_HANDLER_RETURN_TYPE:\n \"\"\"See :func:`spack.package.inject_flags`.\"\"\"\n return flags, None, None\n\n @classmethod\n def env_flags(cls: Type[Pb], name: str, flags: Iterable[str]) -> FLAG_HANDLER_RETURN_TYPE:\n \"\"\"See :func:`spack.package.env_flags`.\"\"\"\n return None, flags, None\n\n @classmethod\n def build_system_flags(\n cls: Type[Pb], name: str, flags: Iterable[str]\n ) -> FLAG_HANDLER_RETURN_TYPE:\n \"\"\"See :func:`spack.package.build_system_flags`.\"\"\"\n return None, None, flags\n\n def setup_run_environment(self, env: spack.util.environment.EnvironmentModifications) -> None:\n \"\"\"Sets up the run environment for a package.\n\n Args:\n env: environment modifications to be applied when the package is run. Package authors\n can call methods on it to alter the run environment.\n \"\"\"\n pass\n\n def setup_dependent_run_environment(\n self, env: spack.util.environment.EnvironmentModifications, dependent_spec: spack.spec.Spec\n ) -> None:\n \"\"\"Sets up the run environment of packages that depend on this one.\n\n This is similar to ``setup_run_environment``, but it is used to modify the run environment\n of a package that *depends* on this one.\n\n This gives packages like Python and others that follow the extension model a way to\n implement common environment or run-time settings for dependencies.\n\n Args:\n env: environment modifications to be applied when the dependent package is run. Package\n authors can call methods on it to alter the build environment.\n\n dependent_spec: The spec of the dependent package about to be run. This allows the\n extendee (self) to query the dependent's state. Note that *this* package's spec is\n available as ``self.spec``\n \"\"\"\n pass\n\n def setup_dependent_package(self, module, dependent_spec: spack.spec.Spec) -> None:\n \"\"\"Set up module-scope global variables for dependent packages.\n\n This function is called when setting up the build and run environments of a DAG.\n\n Examples:\n\n 1. Extensions often need to invoke the ``python`` interpreter from the Python installation\n being extended. This routine can put a ``python`` Executable as a global in the module\n scope for the extension package to simplify extension installs.\n\n 2. MPI compilers could set some variables in the dependent's scope that point to ``mpicc``,\n ``mpicxx``, etc., allowing them to be called by common name regardless of which MPI is\n used.\n\n Args:\n module: The Python ``module`` object of the dependent package. Packages can use this to\n set module-scope variables for the dependent to use.\n\n dependent_spec: The spec of the dependent package about to be built. This allows the\n extendee (self) to query the dependent's state. Note that *this* package's spec is\n available as ``self.spec``.\n \"\"\"\n pass\n\n _flag_handler: Optional[FLAG_HANDLER_TYPE] = None\n\n @property\n def flag_handler(self) -> FLAG_HANDLER_TYPE:\n if self._flag_handler is None:\n self._flag_handler = PackageBase.inject_flags\n return self._flag_handler\n\n @flag_handler.setter\n def flag_handler(self, var: FLAG_HANDLER_TYPE) -> None:\n self._flag_handler = var\n\n # The flag handler method is called for each of the allowed compiler flags.\n # It returns a triple of inject_flags, env_flags, build_system_flags.\n # The flags returned as inject_flags are injected through the spack\n # compiler wrappers.\n # The flags returned as env_flags are passed to the build system through\n # the environment variables of the same name.\n # The flags returned as build_system_flags are passed to the build system\n # package subclass to be turned into the appropriate part of the standard\n # arguments. This is implemented for build system classes where\n # appropriate and will otherwise raise a NotImplementedError.\n\n def flags_to_build_system_args(self, flags: Dict[str, List[str]]) -> None:\n # Takes flags as a dict name: list of values\n if any(v for v in flags.values()):\n msg = \"The {0} build system\".format(self.__class__.__name__)\n msg += \" cannot take command line arguments for compiler flags\"\n raise NotImplementedError(msg)\n\n @staticmethod\n def uninstall_by_spec(spec, force=False, deprecator=None):\n if not os.path.isdir(spec.prefix):\n # prefix may not exist, but DB may be inconsistent. Try to fix by\n # removing, but omit hooks.\n specs = spack.store.STORE.db.query(spec, installed=True)\n if specs:\n if deprecator:\n spack.store.STORE.db.deprecate(specs[0], deprecator)\n tty.debug(\"Deprecating stale DB entry for {0}\".format(spec.short_spec))\n else:\n spack.store.STORE.db.remove(specs[0])\n tty.debug(\"Removed stale DB entry for {0}\".format(spec.short_spec))\n return\n else:\n raise InstallError(str(spec) + \" is not installed.\")\n\n if not force:\n dependents = spack.store.STORE.db.installed_relatives(\n spec, direction=\"parents\", transitive=True, deptype=(\"link\", \"run\")\n )\n if dependents:\n raise PackageStillNeededError(spec, dependents)\n\n # Try to get the package for the spec\n try:\n pkg = spec.package\n except spack.repo.UnknownEntityError:\n pkg = None\n\n # Pre-uninstall hook runs first.\n with spack.store.STORE.prefix_locker.write_lock(spec):\n if pkg is not None:\n try:\n spack.hooks.pre_uninstall(spec)\n except Exception as error:\n if force:\n error_msg = (\n \"One or more pre_uninstall hooks have failed\"\n \" for {0}, but Spack is continuing with the\"\n \" uninstall\".format(str(spec))\n )\n if isinstance(error, spack.error.SpackError):\n error_msg += \"\\n\\nError message: {0}\".format(str(error))\n tty.warn(error_msg)\n # Note that if the uninstall succeeds then we won't be\n # seeing this error again and won't have another chance\n # to run the hook.\n else:\n raise\n\n # Uninstalling in Spack only requires removing the prefix.\n if not spec.external:\n msg = \"Deleting package prefix [{0}]\"\n tty.debug(msg.format(spec.short_spec))\n # test if spec is already deprecated, not whether we want to\n # deprecate it now\n deprecated = bool(spack.store.STORE.db.deprecator(spec))\n spack.store.STORE.layout.remove_install_directory(spec, deprecated)\n # Delete DB entry\n if deprecator:\n msg = \"deprecating DB entry [{0}] in favor of [{1}]\"\n tty.debug(msg.format(spec.short_spec, deprecator.short_spec))\n spack.store.STORE.db.deprecate(spec, deprecator)\n else:\n msg = \"Deleting DB entry [{0}]\"\n tty.debug(msg.format(spec.short_spec))\n spack.store.STORE.db.remove(spec)\n\n if pkg is not None:\n try:\n spack.hooks.post_uninstall(spec)\n except Exception:\n # If there is a failure here, this is our only chance to do\n # something about it: at this point the Spec has been removed\n # from the DB and prefix, so the post-uninstallation hooks\n # will not have another chance to run.\n error_msg = (\n \"One or more post-uninstallation hooks failed for\"\n \" {0}, but the prefix has been removed (if it is not\"\n \" external).\".format(str(spec))\n )\n tb_msg = traceback.format_exc()\n error_msg += \"\\n\\nThe error:\\n\\n{0}\".format(tb_msg)\n tty.warn(error_msg)\n\n tty.msg(\"Successfully uninstalled {0}\".format(spec.short_spec))\n\n def do_uninstall(self, force=False):\n \"\"\"Uninstall this package by spec.\"\"\"\n # delegate to instance-less method.\n PackageBase.uninstall_by_spec(self.spec, force)\n\n def view(self):\n \"\"\"Create a view with the prefix of this package as the root.\n Extensions added to this view will modify the installation prefix of\n this package.\n \"\"\"\n return YamlFilesystemView(self.prefix, spack.store.STORE.layout)\n\n def do_restage(self):\n \"\"\"Reverts expanded/checked out source to a pristine state.\"\"\"\n self.stage.restage()\n\n def do_clean(self):\n \"\"\"Removes the package's build stage and source tarball.\"\"\"\n self.stage.destroy()\n\n @classmethod\n def format_doc(cls, **kwargs):\n \"\"\"Wrap doc string at 72 characters and format nicely\"\"\"\n indent = kwargs.get(\"indent\", 0)\n\n if not cls.__doc__:\n return \"\"\n\n doc = re.sub(r\"\\s+\", \" \", cls.__doc__)\n lines = textwrap.wrap(doc, 72)\n results = io.StringIO()\n for line in lines:\n results.write((\" \" * indent) + line + \"\\n\")\n return results.getvalue()\n\n @property\n def all_urls(self) -> List[str]:\n \"\"\"A list of all URLs in a package.\n\n Check both class-level and version-specific URLs.\n\n Returns a list of URLs\n \"\"\"\n urls: List[str] = []\n if hasattr(self, \"url\") and self.url:\n urls.append(self.url)\n\n # fetch from first entry in urls to save time\n if hasattr(self, \"urls\") and self.urls:\n urls.append(self.urls[0])\n\n for args in self.versions.values():\n if \"url\" in args:\n urls.append(args[\"url\"])\n return urls\n\n def fetch_remote_versions(\n self, concurrency: Optional[int] = None\n ) -> Dict[StandardVersion, str]:\n \"\"\"Find remote versions of this package.\n\n Uses :attr:`list_url` and any other URLs listed in the package file.\n\n Returns:\n a dictionary mapping versions to URLs\n \"\"\"\n if not self.all_urls:\n return {}\n\n try:\n return spack.url.find_versions_of_archive(\n self.all_urls, self.list_url, self.list_depth, concurrency, reference_package=self\n )\n except spack.util.web.NoNetworkConnectionError as e:\n tty.die(\"Package.fetch_versions couldn't connect to:\", e.url, e.message)\n\n @property\n def rpath(self):\n \"\"\"Get the rpath this package links with, as a list of paths.\"\"\"\n deps = self.spec.dependencies(deptype=\"link\")\n\n # on Windows, libraries of runtime interest are typically\n # stored in the bin directory\n # Do not include Windows system libraries in the rpath interface\n # these libraries are handled automatically by VS/VCVARS and adding\n # Spack derived system libs into the link path or address space of a program\n # can result in conflicting versions, which makes Spack packages less usable\n if sys.platform == \"win32\":\n rpaths = [self.prefix.bin]\n rpaths.extend(\n d.prefix.bin\n for d in deps\n if os.path.isdir(d.prefix.bin)\n and \"windows-system\" not in getattr(d.package, \"tags\", [])\n )\n else:\n rpaths = [self.prefix.lib, self.prefix.lib64]\n rpaths.extend(d.prefix.lib for d in deps if os.path.isdir(d.prefix.lib))\n rpaths.extend(d.prefix.lib64 for d in deps if os.path.isdir(d.prefix.lib64))\n return rpaths\n\n @property\n def rpath_args(self):\n \"\"\"\n Get the rpath args as a string, with -Wl,-rpath, for each element\n \"\"\"\n return \" \".join(\"-Wl,-rpath,%s\" % p for p in self.rpath)\n\n\nclass WindowsSimulatedRPath:\n \"\"\"Class representing Windows filesystem rpath analog\n\n One instance of this class is associated with a package (only on Windows)\n For each lib/binary directory in an associated package, this class introduces\n a symlink to any/all dependent libraries/binaries. This includes the packages\n own bin/lib directories, meaning the libraries are linked to the binary directory\n and vis versa.\n \"\"\"\n\n def __init__(\n self,\n package: PackageBase,\n base_modification_prefix: Optional[Union[str, pathlib.Path]] = None,\n link_install_prefix: bool = True,\n ):\n \"\"\"\n Args:\n package: Package requiring links\n base_modification_prefix: Path representation indicating\n the root directory in which to establish the simulated rpath, ie where the\n symlinks that comprise the \"rpath\" behavior will be installed.\n\n Note: This is a mutually exclusive option with `link_install_prefix` using\n both is an error.\n\n Default: None\n link_install_prefix: Link against package's own install or stage root.\n Packages that run their own executables during build and require rpaths to\n the build directory during build time require this option.\n\n Default: install\n root\n\n Note: This is a mutually exclusive option with `base_modification_prefix`, using\n both is an error.\n \"\"\"\n self.pkg = package\n self._addl_rpaths: set[str] = set()\n if link_install_prefix and base_modification_prefix:\n raise RuntimeError(\n \"Invalid combination of arguments given to WindowsSimulated RPath.\\n\"\n \"Select either `link_install_prefix` to create an install prefix rpath\"\n \" or specify a `base_modification_prefix` for any other link type. \"\n \"Specifying both arguments is invalid.\"\n )\n if not (link_install_prefix or base_modification_prefix):\n raise RuntimeError(\n \"Insufficient arguments given to WindowsSimulatedRpath.\\n\"\n \"WindowsSimulatedRPath requires one of link_install_prefix\"\n \" or base_modification_prefix to be specified.\"\n \" Neither was provided.\"\n )\n\n self.link_install_prefix = link_install_prefix\n if base_modification_prefix:\n self.base_modification_prefix = pathlib.Path(base_modification_prefix)\n else:\n self.base_modification_prefix = pathlib.Path(self.pkg.prefix)\n self._additional_library_dependents: set[pathlib.Path] = set()\n if not self.link_install_prefix:\n tty.debug(f\"Generating rpath for non install context: {base_modification_prefix}\")\n\n @property\n def library_dependents(self):\n \"\"\"\n Set of directories where package binaries/libraries are located.\n \"\"\"\n base_pths = set()\n if self.link_install_prefix:\n base_pths.add(pathlib.Path(self.pkg.prefix.bin))\n base_pths |= self._additional_library_dependents\n return base_pths\n\n def add_library_dependent(self, *dest: Union[str, pathlib.Path]):\n \"\"\"\n Add paths to directories or libraries/binaries to set of\n common paths that need to link against other libraries\n\n Specified paths should fall outside of a package's common\n link paths, i.e. the bin\n directories.\n \"\"\"\n for pth in dest:\n if os.path.isfile(pth):\n new_pth = pathlib.Path(pth).parent\n else:\n new_pth = pathlib.Path(pth)\n path_is_in_prefix = new_pth.is_relative_to(self.base_modification_prefix)\n if not path_is_in_prefix:\n raise RuntimeError(\n f\"Attempting to generate rpath symlink out of rpath context:\\\n{str(self.base_modification_prefix)}\"\n )\n self._additional_library_dependents.add(new_pth)\n\n @property\n def rpaths(self):\n \"\"\"\n Set of libraries this package needs to link against during runtime\n These packages will each be symlinked into the packages lib and binary dir\n \"\"\"\n dependent_libs = []\n for path in self.pkg.rpath:\n dependent_libs.extend(list(find_all_shared_libraries(path, recursive=True)))\n for extra_path in self._addl_rpaths:\n dependent_libs.extend(list(find_all_shared_libraries(extra_path, recursive=True)))\n return set([pathlib.Path(x) for x in dependent_libs])\n\n def add_rpath(self, *paths: str):\n \"\"\"\n Add libraries found at the root of provided paths to runtime linking\n\n These are libraries found outside of the typical scope of rpath linking\n that require manual inclusion in a runtime linking scheme.\n These links are unidirectional, and are only\n intended to bring outside dependencies into this package\n\n Args:\n *paths : arbitrary number of paths to be added to runtime linking\n \"\"\"\n self._addl_rpaths = self._addl_rpaths | set(paths)\n\n def _link(self, path: pathlib.Path, dest_dir: pathlib.Path):\n \"\"\"Perform link step of simulated rpathing, installing\n simlinks of file in path to the dest_dir\n location. This method deliberately prevents\n the case where a path points to a file inside the dest_dir.\n This is because it is both meaningless from an rpath\n perspective, and will cause an error when Developer\n mode is not enabled\"\"\"\n\n def report_already_linked():\n # We have either already symlinked or we are encountering a naming clash\n # either way, we don't want to overwrite existing libraries\n already_linked = islink(str(dest_file))\n tty.debug(\n \"Linking library %s to %s failed, \" % (str(path), str(dest_file))\n + \"already linked.\"\n if already_linked\n else \"library with name %s already exists at location %s.\"\n % (str(file_name), str(dest_dir))\n )\n\n file_name = path.name\n dest_file = dest_dir / file_name\n if not dest_file.exists() and dest_dir.exists() and not dest_file == path:\n try:\n symlink(str(path), str(dest_file))\n # For py2 compatibility, we have to catch the specific Windows error code\n # associate with trying to create a file that already exists (winerror 183)\n # Catch OSErrors missed by the SymlinkError checks\n except OSError as e:\n if sys.platform == \"win32\" and e.errno == errno.EEXIST:\n report_already_linked()\n else:\n raise e\n # catch errors we raise ourselves from Spack\n except AlreadyExistsError:\n report_already_linked()\n\n def establish_link(self):\n \"\"\"\n (sym)link packages to runtime dependencies based on RPath configuration for\n Windows heuristics\n \"\"\"\n # from build_environment.py:463\n # The top-level package is always RPATHed. It hasn't been installed yet\n # so the RPATHs are added unconditionally\n\n # for each binary install dir in self.pkg (i.e. pkg.prefix.bin, pkg.prefix.lib)\n # install a symlink to each dependent library\n\n # do not rpath for system libraries included in the dag\n # we should not be modifying libraries managed by the Windows system\n # as this will negatively impact linker behavior and can result in permission\n # errors if those system libs are not modifiable by Spack\n if \"windows-system\" not in getattr(self.pkg, \"tags\", []):\n for library, lib_dir in itertools.product(self.rpaths, self.library_dependents):\n self._link(library, lib_dir)\n\n\ndef make_package_test_rpath(pkg: PackageBase, test_dir: Union[str, pathlib.Path]) -> None:\n \"\"\"Establishes a temp Windows simulated rpath for the pkg in the testing directory so an\n executable can test the libraries/executables with proper access to dependent dlls.\n\n Note: this is a no-op on all other platforms besides Windows\n\n Args:\n pkg: the package for which the rpath should be computed\n test_dir: the testing directory in which we should construct an rpath\n \"\"\"\n # link_install_prefix as false ensures we're not linking into the install prefix\n mini_rpath = WindowsSimulatedRPath(pkg, link_install_prefix=False)\n # add the testing directory as a location to install rpath symlinks\n mini_rpath.add_library_dependent(test_dir)\n\n # check for whether build_directory is available, if not\n # assume the stage root is the build dir\n build_dir_attr = getattr(pkg, \"build_directory\", None)\n build_directory = build_dir_attr if build_dir_attr else pkg.stage.path\n # add the build dir & build dir bin\n mini_rpath.add_rpath(os.path.join(build_directory, \"bin\"))\n mini_rpath.add_rpath(os.path.join(build_directory))\n # construct rpath\n mini_rpath.establish_link()\n\n\ndef deprecated_version(pkg: PackageBase, version: Union[str, StandardVersion]) -> bool:\n \"\"\"Return True iff the version is deprecated.\n\n Arguments:\n pkg: The package whose version is to be checked.\n version: The version being checked\n \"\"\"\n if not isinstance(version, StandardVersion):\n version = StandardVersion.from_string(version)\n\n details = pkg.versions.get(version)\n return details is not None and details.get(\"deprecated\", False)\n\n\ndef preferred_version(\n pkg: Union[PackageBase, Type[PackageBase]],\n) -> Union[StandardVersion, GitVersion]:\n \"\"\"Returns the preferred versions of the package according to package.py.\n\n Accounts for version deprecation in the package recipe. Doesn't account for\n any user configuration in packages.yaml.\n\n Arguments:\n pkg: The package whose versions are to be assessed.\n \"\"\"\n\n def _version_order(version_info):\n version, info = version_info\n deprecated_key = not info.get(\"deprecated\", False)\n return (deprecated_key, *concretization_version_order(version_info))\n\n version, _ = max(pkg.versions.items(), key=_version_order)\n return version\n\n\ndef non_preferred_version(node: spack.spec.Spec) -> bool:\n \"\"\"Returns True if the spec version is not the preferred one, according to the package.py\"\"\"\n if not node.versions.concrete:\n return False\n\n try:\n return node.version != preferred_version(node.package)\n except ValueError:\n return False\n\n\ndef non_default_variant(node: spack.spec.Spec, variant_name: str) -> bool:\n \"\"\"Returns True if the variant in the spec has a non-default value.\"\"\"\n try:\n default_variant = node.package.get_variant(variant_name).make_default()\n return not node.satisfies(str(default_variant))\n except ValueError:\n # This is the case for special variants like \"patches\" etc.\n return False\n\n\ndef sort_by_pkg_preference(\n versions: Iterable[Union[GitVersion, StandardVersion]],\n *,\n pkg: Union[PackageBase, Type[PackageBase]],\n) -> List[Union[GitVersion, StandardVersion]]:\n \"\"\"Sorts the list of versions passed in input according to the preferences in the package. The\n return value does not contain duplicate versions. Most preferred versions first.\n \"\"\"\n s = [(v, pkg.versions.get(v, {})) for v in dedupe(versions)]\n return [v for v, _ in sorted(s, reverse=True, key=concretization_version_order)]\n\n\ndef concretization_version_order(\n version_info: Tuple[Union[GitVersion, StandardVersion], dict],\n) -> Tuple[bool, bool, bool, bool, Union[GitVersion, StandardVersion]]:\n \"\"\"Version order key for concretization, where preferred > not preferred,\n finite > any infinite component; only if all are the same, do we use default version\n ordering.\n\n Version deprecation needs to be accounted for separately.\n \"\"\"\n version, info = version_info\n return (\n info.get(\"preferred\", False),\n not isinstance(version, GitVersion),\n not version.isdevelop(),\n not version.is_prerelease(),\n version,\n )\n\n\nclass PackageStillNeededError(InstallError):\n \"\"\"Raised when package is still needed by another on uninstall.\"\"\"\n\n def __init__(self, spec, dependents):\n spec_fmt = spack.spec.DEFAULT_FORMAT + \" /{hash:7}\"\n dep_fmt = \"{name}{@versions} /{hash:7}\"\n super().__init__(\n f\"Cannot uninstall {spec.format(spec_fmt)}, \"\n f\"needed by {[dep.format(dep_fmt) for dep in dependents]}\"\n )\n self.spec = spec\n self.dependents = dependents\n\n\nclass InvalidPackageOpError(PackageError):\n \"\"\"Raised when someone tries perform an invalid operation on a package.\"\"\"\n\n\nclass ExtensionError(PackageError):\n \"\"\"Superclass for all errors having to do with extension packages.\"\"\"\n\n\nclass ActivationError(ExtensionError):\n \"\"\"Raised when there are problems activating an extension.\"\"\"\n\n def __init__(self, msg, long_msg=None):\n super().__init__(msg, long_msg)\n\n\nclass DependencyConflictError(spack.error.SpackError):\n \"\"\"Raised when the dependencies cannot be flattened as asked for.\"\"\"\n\n def __init__(self, conflict):\n super().__init__(\"%s conflicts with another file in the flattened directory.\" % (conflict))\n\n\nclass ManualDownloadRequiredError(InvalidPackageOpError):\n \"\"\"Raised when attempting an invalid operation on a package that requires a manual download.\"\"\"\n" }, { "path": "lib/spack/spack/package_completions.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nfrom pathlib import Path\nfrom typing import Union\n\n\"\"\"Functions relating to shell autocompletion scripts for packages.\"\"\"\n\n\ndef bash_completion_path(root: Union[str, Path]) -> Path:\n \"\"\"\n Return standard path for bash completion scripts under root.\n\n Args:\n root: The prefix root under which to generate the path.\n\n Returns:\n Standard path for bash completion scripts under root.\n \"\"\"\n return Path(root) / \"share\" / \"bash-completion\" / \"completions\"\n\n\ndef zsh_completion_path(root: Union[str, Path]) -> Path:\n \"\"\"\n Return standard path for zsh completion scripts under root.\n\n Args:\n root: The prefix root under which to generate the path.\n\n Returns:\n Standard path for zsh completion scripts under root.\n \"\"\"\n return Path(root) / \"share\" / \"zsh\" / \"site-functions\"\n\n\ndef fish_completion_path(root: Union[str, Path]) -> Path:\n \"\"\"\n Return standard path for fish completion scripts under root.\n\n Args:\n root: The prefix root under which to generate the path.\n\n Returns:\n Standard path for fish completion scripts under root.\n \"\"\"\n return Path(root) / \"share\" / \"fish\" / \"vendor_completions.d\"\n" }, { "path": "lib/spack/spack/package_prefs.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport stat\nimport warnings\n\nimport spack.config\nimport spack.error\nimport spack.repo\nimport spack.spec\nfrom spack.error import ConfigError\nfrom spack.version import Version\n\n_lesser_spec_types = {\"compiler\": spack.spec.CompilerSpec, \"version\": Version}\n\n\ndef _spec_type(component):\n \"\"\"Map from component name to spec type for package prefs.\"\"\"\n return _lesser_spec_types.get(component, spack.spec.Spec)\n\n\nclass PackagePrefs:\n \"\"\"Defines the sort order for a set of specs.\n\n Spack's package preference implementation uses PackagePrefs to\n define sort order. The PackagePrefs class looks at Spack's\n packages.yaml configuration and, when called on a spec, returns a key\n that can be used to sort that spec in order of the user's\n preferences.\n\n You can use it like this::\n\n # key function sorts CompilerSpecs for `mpich` in order of preference\n kf = PackagePrefs(\"mpich\", \"compiler\")\n compiler_list.sort(key=kf)\n\n Or like this::\n\n # key function to sort VersionLists for OpenMPI in order of preference.\n kf = PackagePrefs(\"openmpi\", \"version\")\n version_list.sort(key=kf)\n\n Optionally, you can sort in order of preferred virtual dependency\n providers. To do that, provide ``\"providers\"`` and a third argument\n denoting the virtual package (e.g., ``mpi``)::\n\n kf = PackagePrefs(\"trilinos\", \"providers\", \"mpi\")\n provider_spec_list.sort(key=kf)\n\n \"\"\"\n\n def __init__(self, pkgname, component, vpkg=None, all=True):\n self.pkgname = pkgname\n self.component = component\n self.vpkg = vpkg\n self.all = all\n\n self._spec_order = None\n\n def __call__(self, spec):\n \"\"\"Return a key object (an index) that can be used to sort spec.\n\n Sort is done in package order. We don't cache the result of\n this function as Python's sort functions already ensure that the\n key function is called at most once per sorted element.\n \"\"\"\n if self._spec_order is None:\n self._spec_order = self._specs_for_pkg(\n self.pkgname, self.component, self.vpkg, self.all\n )\n spec_order = self._spec_order\n\n # integer is the index of the first spec in order that satisfies\n # spec, or it's a number larger than any position in the order.\n match_index = next(\n (i for i, s in enumerate(spec_order) if spec.intersects(s)), len(spec_order)\n )\n if match_index < len(spec_order) and spec_order[match_index] == spec:\n # If this is called with multiple specs that all satisfy the same\n # minimum index in spec_order, the one which matches that element\n # of spec_order exactly is considered slightly better. Note\n # that because this decreases the value by less than 1, it is not\n # better than a match which occurs at an earlier index.\n match_index -= 0.5\n return match_index\n\n @classmethod\n def order_for_package(cls, pkgname, component, vpkg=None, all=True):\n \"\"\"Given a package name, sort component (e.g, version, compiler, ...),\n and an optional vpkg, return the list from the packages config.\n \"\"\"\n pkglist = [pkgname]\n if all:\n pkglist.append(\"all\")\n\n packages = spack.config.CONFIG.get_config(\"packages\")\n\n for pkg in pkglist:\n pkg_entry = packages.get(pkg)\n if not pkg_entry:\n continue\n\n order = pkg_entry.get(component)\n if not order:\n continue\n\n # vpkg is one more level\n if vpkg is not None:\n order = order.get(vpkg)\n\n if order:\n ret = [str(s).strip() for s in order]\n if component == \"target\":\n ret = [\"target=%s\" % tname for tname in ret]\n return ret\n\n return []\n\n @classmethod\n def _specs_for_pkg(cls, pkgname, component, vpkg=None, all=True):\n \"\"\"Given a sort order specified by the pkgname/component/second_key,\n return a list of CompilerSpecs, VersionLists, or Specs for\n that sorting list.\n \"\"\"\n pkglist = cls.order_for_package(pkgname, component, vpkg, all)\n spec_type = _spec_type(component)\n return [spec_type(s) for s in pkglist]\n\n @classmethod\n def has_preferred_providers(cls, pkgname, vpkg):\n \"\"\"Whether specific package has a preferred vpkg providers.\"\"\"\n return bool(cls.order_for_package(pkgname, \"providers\", vpkg, False))\n\n @classmethod\n def has_preferred_targets(cls, pkg_name):\n \"\"\"Whether specific package has a preferred vpkg providers.\"\"\"\n return bool(cls.order_for_package(pkg_name, \"target\"))\n\n @classmethod\n def preferred_variants(cls, pkg_name):\n \"\"\"Return a VariantMap of preferred variants/values for a spec.\"\"\"\n packages = spack.config.CONFIG.get_config(\"packages\")\n for pkg_cls in (pkg_name, \"all\"):\n variants = packages.get(pkg_cls, {}).get(\"variants\", \"\")\n if variants:\n break\n\n # allow variants to be list or string\n if not isinstance(variants, str):\n variants = \" \".join(variants)\n\n # Only return variants that are actually supported by the package\n pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)\n spec = spack.spec.Spec(f\"{pkg_name} {variants}\")\n return {\n name: variant\n for name, variant in spec.variants.items()\n if name in pkg_cls.variant_names()\n }\n\n\ndef is_spec_buildable(spec):\n \"\"\"Return true if the spec is configured as buildable\"\"\"\n allpkgs = spack.config.get(\"packages\")\n all_buildable = allpkgs.get(\"all\", {}).get(\"buildable\", True)\n so_far = all_buildable # the default \"so far\"\n\n def _package(s):\n pkg_cls = spack.repo.PATH.get_pkg_class(s.name)\n return pkg_cls(s)\n\n # check whether any providers for this package override the default\n if any(\n _package(spec).provides(name) and entry.get(\"buildable\", so_far) != so_far\n for name, entry in allpkgs.items()\n ):\n so_far = not so_far\n\n spec_buildable = allpkgs.get(spec.name, {}).get(\"buildable\", so_far)\n return spec_buildable\n\n\ndef get_package_dir_permissions(spec):\n \"\"\"Return the permissions configured for the spec.\n\n Include the GID bit if group permissions are on. This makes the group\n attribute sticky for the directory. Package-specific settings take\n precedent over settings for ``all``\"\"\"\n perms = get_package_permissions(spec)\n if perms & stat.S_IRWXG and spack.config.get(\"config:allow_sgid\", True):\n perms |= stat.S_ISGID\n if spec.concrete and \"/afs/\" in spec.prefix:\n warnings.warn(\n \"Directory {0} seems to be located on AFS. If you\"\n \" encounter errors, try disabling the allow_sgid option\"\n \" using: spack config add 'config:allow_sgid:false'\".format(spec.prefix)\n )\n return perms\n\n\ndef get_package_permissions(spec):\n \"\"\"Return the permissions configured for the spec.\n\n Package-specific settings take precedence over settings for ``all``\"\"\"\n\n # Get read permissions level\n for name in (spec.name, \"all\"):\n try:\n readable = spack.config.get(\"packages:%s:permissions:read\" % name, \"\")\n if readable:\n break\n except AttributeError:\n readable = \"world\"\n\n # Get write permissions level\n for name in (spec.name, \"all\"):\n try:\n writable = spack.config.get(\"packages:%s:permissions:write\" % name, \"\")\n if writable:\n break\n except AttributeError:\n writable = \"user\"\n\n perms = stat.S_IRWXU\n if readable in (\"world\", \"group\"): # world includes group\n perms |= stat.S_IRGRP | stat.S_IXGRP\n if readable == \"world\":\n perms |= stat.S_IROTH | stat.S_IXOTH\n\n if writable in (\"world\", \"group\"):\n if readable == \"user\":\n raise ConfigError(\n \"Writable permissions may not be more\"\n + \" permissive than readable permissions.\\n\"\n + \" Violating package is %s\" % spec.name\n )\n perms |= stat.S_IWGRP\n if writable == \"world\":\n if readable != \"world\":\n raise ConfigError(\n \"Writable permissions may not be more\"\n + \" permissive than readable permissions.\\n\"\n + \" Violating package is %s\" % spec.name\n )\n perms |= stat.S_IWOTH\n\n return perms\n\n\ndef get_package_group(spec):\n \"\"\"Return the unix group associated with the spec.\n\n Package-specific settings take precedence over settings for ``all``\"\"\"\n for name in (spec.name, \"all\"):\n try:\n group = spack.config.get(\"packages:%s:permissions:group\" % name, \"\")\n if group:\n break\n except AttributeError:\n group = \"\"\n return group\n\n\nclass VirtualInPackagesYAMLError(spack.error.SpackError):\n \"\"\"Raised when a disallowed virtual is found in packages.yaml\"\"\"\n" }, { "path": "lib/spack/spack/package_test.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nfrom typing import List\n\nfrom spack.util.executable import Executable, which\n\n\ndef compile_c_and_execute(\n source_file: str, include_flags: List[str], link_flags: List[str]\n) -> str:\n \"\"\"Compile a C source file with the given include and link flags, execute the resulting binary,\n and return its output as a string. Used in package tests.\"\"\"\n cc = which(\"cc\", required=True)\n flags = include_flags\n flags.extend([source_file])\n cc(\"-c\", *flags)\n name = os.path.splitext(os.path.basename(source_file))[0]\n cc(\"-o\", \"check\", \"%s.o\" % name, *link_flags)\n\n check = Executable(\"./check\")\n return check(output=str)\n\n\ndef compare_output(current_output: str, blessed_output: str) -> None:\n \"\"\"Compare blessed and current output of executables. Used in package tests.\"\"\"\n if not (current_output == blessed_output):\n print(\"Produced output does not match expected output.\")\n print(\"Expected output:\")\n print(\"-\" * 80)\n print(blessed_output)\n print(\"-\" * 80)\n print(\"Produced output:\")\n print(\"-\" * 80)\n print(current_output)\n print(\"-\" * 80)\n raise RuntimeError(\"Output check failed.\", \"See spack_output.log for details\")\n\n\ndef compare_output_file(current_output: str, blessed_output_file: str) -> None:\n \"\"\"Same as above, but when the blessed output is given as a file. Used in package tests.\"\"\"\n with open(blessed_output_file, \"r\", encoding=\"utf-8\") as f:\n blessed_output = f.read()\n\n compare_output(current_output, blessed_output)\n" }, { "path": "lib/spack/spack/patch.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport hashlib\nimport os\nimport pathlib\nimport sys\nfrom typing import TYPE_CHECKING, Any, Dict, Optional, Set, Tuple, Type, Union\n\nimport spack\nimport spack.error\nimport spack.fetch_strategy\nimport spack.llnl.util.filesystem\nimport spack.util.spack_json as sjson\nfrom spack.llnl.url import allowed_archive\nfrom spack.util.crypto import Checker, checksum\nfrom spack.util.executable import which, which_string\n\nif TYPE_CHECKING:\n import spack.package_base\n import spack.repo\n\n\ndef apply_patch(\n source_path: str,\n patch_path: str,\n level: int = 1,\n working_dir: str = \".\",\n reverse: bool = False,\n) -> None:\n \"\"\"Apply the patch at patch_path to code in the stage.\n\n Args:\n stage: stage with code that will be patched\n patch_path: filesystem location for the patch to apply\n level: patch level\n working_dir: relative path *within* the stage to change to\n reverse: reverse the patch\n \"\"\"\n if not patch_path or not os.path.isfile(patch_path):\n raise spack.error.NoSuchPatchError(f\"No such patch: {patch_path}\")\n\n git_utils_path = os.environ.get(\"PATH\", \"\")\n if sys.platform == \"win32\":\n git = which_string(\"git\")\n if git:\n git = pathlib.Path(git)\n git_root = git.parent.parent\n git_root = git_root / \"usr\" / \"bin\"\n git_utils_path = os.pathsep.join([str(git_root), git_utils_path])\n\n args = [\"-s\", \"-p\", str(level), \"-i\", patch_path, \"-d\", working_dir]\n if reverse:\n args.append(\"-R\")\n\n # TODO: Decouple Spack's patch support on Windows from Git\n # for Windows, and instead have Spack directly fetch, install, and\n # utilize that patch.\n # Note for future developers: The GNU port of patch to windows\n # has issues handling CRLF line endings unless the --binary\n # flag is passed.\n patch = which(\"patch\", required=True, path=git_utils_path)\n with spack.llnl.util.filesystem.working_dir(source_path):\n patch(*args)\n\n\nPatchPackageType = Union[\"spack.package_base.PackageBase\", Type[\"spack.package_base.PackageBase\"]]\n\n\nclass Patch:\n \"\"\"Base class for patches.\n\n The owning package is not necessarily the package to apply the patch\n to -- in the case where a dependent package patches its dependency,\n it is the dependent's fullname.\n \"\"\"\n\n sha256: str\n\n def __init__(\n self,\n pkg: PatchPackageType,\n path_or_url: str,\n level: int,\n working_dir: str,\n reverse: bool = False,\n ordering_key: Optional[Tuple[str, int]] = None,\n ) -> None:\n \"\"\"Initialize a new Patch instance.\n\n Args:\n pkg: the package that owns the patch\n path_or_url: the relative path or URL to a patch file\n level: patch level\n working_dir: relative path *within* the stage to change to\n reverse: reverse the patch\n ordering_key: key used to ensure patches are applied in a consistent order\n \"\"\"\n # validate level (must be an integer >= 0)\n if not isinstance(level, int) or not level >= 0:\n raise ValueError(\"Patch level needs to be a non-negative integer.\")\n\n # Attributes shared by all patch subclasses\n self.owner = pkg.fullname\n self.path_or_url = path_or_url # needed for debug output\n self.path: Optional[str] = None # must be set before apply()\n self.level = level\n self.working_dir = working_dir\n self.reverse = reverse\n\n # The ordering key is passed when executing package.py directives, and is only relevant\n # after a solve to build concrete specs with consistently ordered patches. For concrete\n # specs read from a file, we add patches in the order of its patches variants and the\n # ordering_key is irrelevant. In that case, use a default value so we don't need to branch\n # on whether ordering_key is None where it's used, just to make static analysis happy.\n self.ordering_key: Tuple[str, int] = ordering_key or (\"\", 0)\n\n # TODO: Use TypedDict once Spack supports Python 3.8+ only\n def to_dict(self) -> Dict[str, Any]:\n \"\"\"Dictionary representation of the patch.\n\n Returns:\n A dictionary representation.\n \"\"\"\n return {\n \"owner\": self.owner,\n \"sha256\": self.sha256,\n \"level\": self.level,\n \"working_dir\": self.working_dir,\n \"reverse\": self.reverse,\n }\n\n def __eq__(self, other: object) -> bool:\n \"\"\"Equality check.\n\n Args:\n other: another patch\n\n Returns:\n True if both patches have the same checksum, else False\n \"\"\"\n if not isinstance(other, Patch):\n return NotImplemented\n return self.sha256 == other.sha256\n\n def __hash__(self) -> int:\n \"\"\"Unique hash.\n\n Returns:\n A unique hash based on the sha256.\n \"\"\"\n return hash(self.sha256)\n\n\nclass FilePatch(Patch):\n \"\"\"Describes a patch that is retrieved from a file in the repository.\"\"\"\n\n _sha256: Optional[str] = None\n\n def __init__(\n self,\n pkg: PatchPackageType,\n relative_path: str,\n level: int,\n working_dir: str,\n reverse: bool = False,\n ordering_key: Optional[Tuple[str, int]] = None,\n ) -> None:\n \"\"\"Initialize a new FilePatch instance.\n\n Args:\n pkg: the class object for the package that owns the patch\n relative_path: path to patch, relative to the repository directory for a package.\n level: level to pass to patch command\n working_dir: path within the source directory where patch should be applied\n reverse: reverse the patch\n ordering_key: key used to ensure patches are applied in a consistent order\n \"\"\"\n self.relative_path = relative_path\n\n # patches may be defined by relative paths to parent classes\n # search mro to look for the file\n abs_path: Optional[str] = None\n # At different times we call FilePatch on instances and classes\n pkg_cls = pkg if isinstance(pkg, type) else pkg.__class__\n for cls in pkg_cls.__mro__: # type: ignore\n if not hasattr(cls, \"module\"):\n # We've gone too far up the MRO\n break\n\n # Cannot use pkg.package_dir because it's a property and we have\n # classes, not instances.\n pkg_dir = os.path.abspath(os.path.dirname(cls.module.__file__))\n path = os.path.join(pkg_dir, self.relative_path)\n if os.path.exists(path):\n abs_path = path\n break\n\n if abs_path is None:\n msg = \"FilePatch: Patch file %s for \" % relative_path\n msg += \"package %s.%s does not exist.\" % (pkg.namespace, pkg.name)\n raise ValueError(msg)\n\n super().__init__(pkg, abs_path, level, working_dir, reverse, ordering_key)\n self.path = abs_path\n\n @property\n def sha256(self) -> str:\n \"\"\"Get the patch checksum.\n\n Returns:\n The sha256 of the patch file.\n \"\"\"\n if self._sha256 is None and self.path is not None:\n self._sha256 = checksum(hashlib.sha256, self.path)\n assert isinstance(self._sha256, str)\n return self._sha256\n\n @sha256.setter\n def sha256(self, value: str) -> None:\n \"\"\"Set the patch checksum.\n\n Args:\n value: the sha256\n \"\"\"\n self._sha256 = value\n\n def to_dict(self) -> Dict[str, Any]:\n \"\"\"Dictionary representation of the patch.\n\n Returns:\n A dictionary representation.\n \"\"\"\n data = super().to_dict()\n data[\"relative_path\"] = self.relative_path\n return data\n\n\nclass UrlPatch(Patch):\n \"\"\"Describes a patch that is retrieved from a URL.\"\"\"\n\n def __init__(\n self,\n pkg: PatchPackageType,\n url: str,\n level: int = 1,\n *,\n working_dir: str = \".\",\n reverse: bool = False,\n sha256: str, # This is required for UrlPatch\n ordering_key: Optional[Tuple[str, int]] = None,\n archive_sha256: Optional[str] = None,\n ) -> None:\n \"\"\"Initialize a new UrlPatch instance.\n\n Arguments:\n pkg: the package that owns the patch\n url: URL where the patch can be fetched\n level: level to pass to patch command\n working_dir: path within the source directory where patch should be applied\n reverse: reverse the patch\n ordering_key: key used to ensure patches are applied in a consistent order\n sha256: sha256 sum of the patch, used to verify the patch\n archive_sha256: sha256 sum of the *archive*, if the patch is compressed\n (only required for compressed URL patches)\n \"\"\"\n super().__init__(pkg, url, level, working_dir, reverse, ordering_key)\n\n self.url = url\n\n if allowed_archive(self.url) and not archive_sha256:\n raise spack.error.PatchDirectiveError(\n \"Compressed patches require 'archive_sha256' \"\n \"and patch 'sha256' attributes: %s\" % self.url\n )\n self.archive_sha256 = archive_sha256\n\n if not sha256:\n raise spack.error.PatchDirectiveError(\"URL patches require a sha256 checksum\")\n self.sha256 = sha256\n\n def fetcher(self) -> spack.fetch_strategy.FetchStrategy:\n \"\"\"Construct a fetcher that can download (and unpack) this patch.\"\"\"\n # Two checksums, one for compressed file, one for its contents\n if self.archive_sha256 and self.sha256:\n return spack.fetch_strategy.FetchAndVerifyExpandedFile(\n self.url, archive_sha256=self.archive_sha256, expanded_sha256=self.sha256\n )\n else:\n return spack.fetch_strategy.URLFetchStrategy(\n url=self.url, sha256=self.sha256, expand=False\n )\n\n def to_dict(self) -> Dict[str, Any]:\n \"\"\"Dictionary representation of the patch.\n\n Returns:\n A dictionary representation.\n \"\"\"\n data = super().to_dict()\n data[\"url\"] = self.url\n if self.archive_sha256:\n data[\"archive_sha256\"] = self.archive_sha256\n return data\n\n\ndef from_dict(dictionary: Dict[str, Any], repository: \"spack.repo.RepoPath\") -> Patch:\n \"\"\"Create a patch from json dictionary.\n\n Args:\n dictionary: dictionary representation of a patch\n repository: repository containing package\n\n Returns:\n A patch object.\n\n Raises:\n ValueError: If *owner* or *url*/*relative_path* are missing in the dictionary.\n \"\"\"\n owner = dictionary.get(\"owner\")\n if owner is None:\n raise ValueError(f\"Invalid patch dictionary: {dictionary}\")\n assert isinstance(owner, str)\n pkg_cls = repository.get_pkg_class(owner)\n\n if \"url\" in dictionary:\n return UrlPatch(\n pkg_cls,\n dictionary[\"url\"],\n dictionary[\"level\"],\n working_dir=dictionary[\"working_dir\"],\n # Added in v0.22, fallback required for backwards compatibility\n reverse=dictionary.get(\"reverse\", False),\n sha256=dictionary[\"sha256\"],\n archive_sha256=dictionary.get(\"archive_sha256\"),\n )\n\n elif \"relative_path\" in dictionary:\n patch = FilePatch(\n pkg_cls,\n dictionary[\"relative_path\"],\n dictionary[\"level\"],\n dictionary[\"working_dir\"],\n # Added in v0.22, fallback required for backwards compatibility\n dictionary.get(\"reverse\", False),\n )\n\n # If the patch in the repo changes, we cannot get it back, so we\n # just check it and fail here.\n # TODO: handle this more gracefully.\n sha256 = dictionary[\"sha256\"]\n checker = Checker(sha256)\n if patch.path and not checker.check(patch.path):\n raise spack.fetch_strategy.ChecksumError(\n \"sha256 checksum failed for %s\" % patch.path,\n \"Expected %s but got %s \" % (sha256, checker.sum)\n + \"Patch may have changed since concretization.\",\n )\n return patch\n else:\n raise ValueError(\"Invalid patch dictionary: %s\" % dictionary)\n\n\nclass PatchCache:\n \"\"\"Index of patches used in a repository, by sha256 hash.\n\n This allows us to look up patches without loading all packages. It's\n also needed to properly implement dependency patching, as need a way\n to look up patches that come from packages not in the Spec sub-DAG.\n\n The patch index is structured like this in a file (this is YAML, but\n we write JSON)::\n\n patches:\n sha256:\n namespace1.package1:\n \n namespace2.package2:\n \n ... etc. ...\n \"\"\"\n\n def __init__(\n self, repository: \"spack.repo.RepoPath\", data: Optional[Dict[str, Any]] = None\n ) -> None:\n \"\"\"Initialize a new PatchCache instance.\n\n Args:\n repository: repository containing package\n data: nested dictionary of patches\n \"\"\"\n if data is None:\n self.index = {}\n else:\n if \"patches\" not in data:\n raise IndexError(\"invalid patch index; try `spack clean -m`\")\n self.index = data[\"patches\"]\n\n self.repository = repository\n\n @classmethod\n def from_json(cls, stream: Any, repository: \"spack.repo.RepoPath\") -> \"PatchCache\":\n \"\"\"Initialize a new PatchCache instance from JSON.\n\n Args:\n stream: stream of data\n repository: repository containing package\n\n Returns:\n A new PatchCache instance.\n \"\"\"\n return PatchCache(repository=repository, data=sjson.load(stream))\n\n def to_json(self, stream: Any) -> None:\n \"\"\"Dump a JSON representation to a stream.\n\n Args:\n stream: stream of data\n \"\"\"\n sjson.dump({\"patches\": self.index}, stream)\n\n def patch_for_package(\n self, sha256: str, pkg: Type[\"spack.package_base.PackageBase\"], *, validate: bool = False\n ) -> Patch:\n \"\"\"Look up a patch in the index and build a patch object for it.\n\n We build patch objects lazily because building them requires that\n we have information about the package's location in its repo.\n\n Args:\n sha256: sha256 hash to look up\n pkg: Package class to get patch for.\n validate: if True, validate the cached entry against the owner's current package\n class and raise ``PatchLookupError`` if the entry is missing or stale.\n\n Returns:\n The patch object.\n \"\"\"\n sha_index = self.index.get(sha256)\n if not sha_index:\n raise spack.error.PatchLookupError(\n f\"Couldn't find patch for package {pkg.fullname} with sha256: {sha256}\"\n )\n\n # Find patches for this class or any class it inherits from\n for fullname in pkg.fullnames:\n patch_dict = sha_index.get(fullname)\n if patch_dict:\n break\n else:\n raise spack.error.PatchLookupError(\n f\"Couldn't find patch for package {pkg.fullname} with sha256: {sha256}\"\n )\n\n if validate:\n # Validate the cached entry against the owner's current package class\n owner = patch_dict.get(\"owner\")\n if not owner:\n raise spack.error.PatchLookupError(\n f\"Patch for {pkg.fullname} with sha256 {sha256} has no owner in cache\"\n )\n try:\n owner_pkg_cls = self.repository.get_pkg_class(owner)\n current_index = PatchCache._index_patches(owner_pkg_cls, self.repository)\n except Exception as e:\n raise spack.error.PatchLookupError(\n f\"Could not validate patch cache for {pkg.fullname}: {e}\"\n ) from e\n current_sha_index = current_index.get(sha256)\n if not current_sha_index or current_sha_index.get(fullname) != patch_dict:\n raise spack.error.PatchLookupError(\n f\"Stale patch cache entry for {pkg.fullname} with sha256: {sha256}\"\n )\n\n # add the sha256 back (we take it out on write to save space,\n # because it's the index key)\n patch_dict = dict(patch_dict)\n patch_dict[\"sha256\"] = sha256\n return from_dict(patch_dict, repository=self.repository)\n\n def update_packages(self, pkgs_fullname: Set[str]) -> None:\n \"\"\"Update the patch cache.\n\n Args:\n pkg_fullname: package to update.\n \"\"\"\n # remove this package from any patch entries that reference it.\n if self.index:\n empty = []\n for sha256, package_to_patch in self.index.items():\n remove = []\n for fullname, patch_dict in package_to_patch.items():\n if patch_dict[\"owner\"] in pkgs_fullname:\n remove.append(fullname)\n\n for fullname in remove:\n package_to_patch.pop(fullname)\n\n if not package_to_patch:\n empty.append(sha256)\n\n # remove any entries that are now empty\n for sha256 in empty:\n del self.index[sha256]\n\n # update the index with per-package patch indexes\n for pkg_fullname in pkgs_fullname:\n pkg_cls = self.repository.get_pkg_class(pkg_fullname)\n partial_index = self._index_patches(pkg_cls, self.repository)\n for sha256, package_to_patch in partial_index.items():\n p2p = self.index.setdefault(sha256, {})\n p2p.update(package_to_patch)\n\n def update(self, other: \"PatchCache\") -> None:\n \"\"\"Update this cache with the contents of another.\n\n Args:\n other: another patch cache to merge\n \"\"\"\n for sha256, package_to_patch in other.index.items():\n p2p = self.index.setdefault(sha256, {})\n p2p.update(package_to_patch)\n\n @staticmethod\n def _index_patches(\n pkg_class: Type[\"spack.package_base.PackageBase\"], repository: \"spack.repo.RepoPath\"\n ) -> Dict[Any, Any]:\n \"\"\"Patch index for a specific patch.\n\n Args:\n pkg_class: package object to get patches for\n repository: repository containing the package\n\n Returns:\n The patch index for that package.\n \"\"\"\n index = {}\n\n # Add patches from the class\n for cond, patch_list in pkg_class.patches.items():\n for patch in patch_list:\n patch_dict = patch.to_dict()\n patch_dict.pop(\"sha256\") # save some space\n index[patch.sha256] = {pkg_class.fullname: patch_dict}\n\n if not pkg_class._patches_dependencies:\n return index\n\n for deps_by_name in pkg_class.dependencies.values():\n for dependency in deps_by_name.values():\n for patch_list in dependency.patches.values():\n for patch in patch_list:\n dspec_cls = repository.get_pkg_class(dependency.spec.name)\n patch_dict = patch.to_dict()\n patch_dict.pop(\"sha256\") # save some space\n index[patch.sha256] = {dspec_cls.fullname: patch_dict}\n\n return index\n" }, { "path": "lib/spack/spack/paths.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Defines paths that are part of Spack's directory structure.\n\nDo not import other ``spack`` modules here. This module is used\nthroughout Spack and should bring in a minimal number of external\ndependencies.\n\"\"\"\n\nimport os\nfrom pathlib import PurePath\n\nimport spack.llnl.util.filesystem\nimport spack.util.hash as hash\n\n#: This file lives in $prefix/lib/spack/spack/__file__\nprefix = str(PurePath(spack.llnl.util.filesystem.ancestor(__file__, 4)))\n\n#: synonym for prefix\nspack_root = prefix\n\n#: bin directory in the spack prefix\nbin_path = os.path.join(prefix, \"bin\")\n\n#: The spack script itself\nspack_script = os.path.join(bin_path, \"spack\")\n\n#: The sbang script in the spack installation\nsbang_script = os.path.join(bin_path, \"sbang\")\n\n# spack directory hierarchy\nlib_path = os.path.join(prefix, \"lib\", \"spack\")\nmodule_path = os.path.join(lib_path, \"spack\")\nvendor_path = os.path.join(module_path, \"vendor\")\ncommand_path = os.path.join(module_path, \"cmd\")\nanalyzers_path = os.path.join(module_path, \"analyzers\")\nplatform_path = os.path.join(module_path, \"platforms\")\ncompilers_path = os.path.join(module_path, \"compilers\")\noperating_system_path = os.path.join(module_path, \"operating_systems\")\ntest_path = os.path.join(module_path, \"test\")\nhooks_path = os.path.join(module_path, \"hooks\")\nopt_path = os.path.join(prefix, \"opt\")\nshare_path = os.path.join(prefix, \"share\", \"spack\")\netc_path = os.path.join(prefix, \"etc\", \"spack\")\n\n#\n# Things in $spack/etc/spack\n#\ndefault_license_dir = os.path.join(etc_path, \"licenses\")\n\n#\n# Things in $spack/var/spack\n#\nvar_path = os.path.join(prefix, \"var\", \"spack\")\n\n# read-only things in $spack/var/spack\nrepos_path = os.path.join(var_path, \"repos\")\ntest_repos_path = os.path.join(var_path, \"test_repos\")\nmock_packages_path = os.path.join(test_repos_path, \"spack_repo\", \"builtin_mock\")\n\n#\n# Writable things in $spack/var/spack\n# TODO: Deprecate these, as we want a read-only spack prefix by default.\n# TODO: These should probably move to user cache, or some other location.\n#\n# fetch cache for downloaded files\ndefault_fetch_cache_path = os.path.join(var_path, \"cache\")\n\n# GPG paths.\ngpg_keys_path = os.path.join(var_path, \"gpg\")\nmock_gpg_data_path = os.path.join(var_path, \"gpg.mock\", \"data\")\nmock_gpg_keys_path = os.path.join(var_path, \"gpg.mock\", \"keys\")\ngpg_path = os.path.join(opt_path, \"spack\", \"gpg\")\n\n\n#: Not a location itself, but used for when Spack instances\n#: share the same cache base directory for caches that should\n#: not be shared between those instances.\nspack_instance_id = hash.b32_hash(spack_root)[:7]\n\n\n# Below paths are where Spack can write information for the user.\n# Some are caches, some are not exactly caches.\n#\n# The options that start with `default_` below are overridable in\n# `config.yaml`, but they default to use `user_cache_path/`.\n#\n# You can override the top-level directory (the user cache path) by\n# setting `SPACK_USER_CACHE_PATH`. Otherwise it defaults to ~/.spack.\n#\ndef _get_user_cache_path():\n return os.path.expanduser(os.getenv(\"SPACK_USER_CACHE_PATH\") or \"~%s.spack\" % os.sep)\n\n\nuser_cache_path = str(PurePath(_get_user_cache_path()))\n\n#: junit, cdash, etc. reports about builds\nreports_path = os.path.join(user_cache_path, \"reports\")\n\n#: installation test (spack test) output\ndefault_test_path = os.path.join(user_cache_path, \"test\")\n\n#: spack monitor analysis directories\ndefault_monitor_path = os.path.join(reports_path, \"monitor\")\n\n#: git repositories fetched to compare commits to versions\nuser_repos_cache_path = os.path.join(user_cache_path, \"git_repos\")\n\n#: default location where remote package repositories are cloned\npackage_repos_path = os.path.join(user_cache_path, \"package_repos\")\n\n#: bootstrap store for bootstrapping clingo and other tools\ndefault_user_bootstrap_path = os.path.join(user_cache_path, \"bootstrap\")\n\n#: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.)\ndefault_misc_cache_path = os.path.join(user_cache_path, spack_instance_id, \"cache\")\n\n# Below paths pull configuration from the host environment.\n#\n# There are three environment variables you can use to isolate spack from\n# the host environment:\n# - `SPACK_USER_CONFIG_PATH`: override `~/.spack` location (for config and caches)\n# - `SPACK_SYSTEM_CONFIG_PATH`: override `/etc/spack` configuration scope.\n# - `SPACK_DISABLE_LOCAL_CONFIG`: disable both of these locations.\n\n\n# User configuration and caches in $HOME/.spack\ndef _get_user_config_path():\n return os.path.expanduser(os.getenv(\"SPACK_USER_CONFIG_PATH\") or \"~%s.spack\" % os.sep)\n\n\n# Configuration in /etc/spack on the system\ndef _get_system_config_path():\n return os.path.expanduser(\n os.getenv(\"SPACK_SYSTEM_CONFIG_PATH\") or os.sep + os.path.join(\"etc\", \"spack\")\n )\n\n\n#: User configuration location\nuser_config_path = _get_user_config_path()\n\n#: System configuration location\nsystem_config_path = _get_system_config_path()\n\n#: Recorded directory where spack command was originally invoked\nspack_working_dir = None\n\n\ndef set_working_dir():\n \"\"\"Change the working directory to getcwd, or spack prefix if no cwd.\"\"\"\n global spack_working_dir\n try:\n spack_working_dir = os.getcwd()\n except OSError:\n os.chdir(prefix)\n spack_working_dir = prefix\n" }, { "path": "lib/spack/spack/phase_callbacks.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport collections\nfrom typing import Optional\n\nimport spack.llnl.util.lang as lang\n\n#: An object of this kind is a shared global state used to collect callbacks during\n#: class definition time, and is flushed when the class object is created at the end\n#: of the class definition\n#:\n#: Args:\n#: attribute_name (str): name of the attribute that will be attached to the builder\n#: callbacks (list): container used to temporarily aggregate the callbacks\nCallbackTemporaryStage = collections.namedtuple(\n \"CallbackTemporaryStage\", [\"attribute_name\", \"callbacks\"]\n)\n\n#: Shared global state to aggregate \"@run_before\" callbacks\n_RUN_BEFORE = CallbackTemporaryStage(attribute_name=\"_run_before_callbacks\", callbacks=[])\n#: Shared global state to aggregate \"@run_after\" callbacks\n_RUN_AFTER = CallbackTemporaryStage(attribute_name=\"_run_after_callbacks\", callbacks=[])\n\n\nclass PhaseCallbacksMeta(type):\n \"\"\"Permit to register arbitrary functions during class definition and run them\n later, before or after a given install phase.\n\n Each method decorated with ``run_before`` or ``run_after`` gets temporarily\n stored in a global shared state when a class being defined is parsed by the Python\n interpreter. At class definition time that temporary storage gets flushed and a list\n of callbacks is attached to the class being defined.\n \"\"\"\n\n def __new__(mcs, name, bases, attr_dict):\n for temporary_stage in (_RUN_BEFORE, _RUN_AFTER):\n staged_callbacks = temporary_stage.callbacks\n\n # Here we have an adapter from an old-style package. This means there is no\n # hierarchy of builders, and every callback that had to be combined between\n # *Package and *Builder has been combined already by _PackageAdapterMeta\n if name == \"Adapter\":\n continue\n\n # If we are here we have callbacks. To get a complete list, we accumulate all the\n # callbacks from base classes, we deduplicate them, then prepend what we have\n # registered here.\n #\n # The order should be:\n # 1. Callbacks are registered in order within the same class\n # 2. Callbacks defined in derived classes precede those defined in base\n # classes\n callbacks_from_base = []\n for base in bases:\n current_callbacks = getattr(base, temporary_stage.attribute_name, None)\n if not current_callbacks:\n continue\n callbacks_from_base.extend(current_callbacks)\n callbacks_from_base = list(lang.dedupe(callbacks_from_base))\n # Set the callbacks in this class and flush the temporary stage\n attr_dict[temporary_stage.attribute_name] = staged_callbacks[:] + callbacks_from_base\n del temporary_stage.callbacks[:]\n\n return super(PhaseCallbacksMeta, mcs).__new__(mcs, name, bases, attr_dict)\n\n @staticmethod\n def run_after(phase: str, when: Optional[str] = None):\n \"\"\"Decorator to register a function for running after a given phase.\n\n Example::\n\n @run_after(\"install\", when=\"@2:\")\n def install_missing_files(self):\n # Do something after the install phase for versions 2.x and above\n pass\n\n Args:\n phase: phase after which the function must run.\n when: condition under which the function is run (if :obj:`None`, it is always run).\n \"\"\"\n\n def _decorator(fn):\n key = (phase, when)\n item = (key, fn)\n _RUN_AFTER.callbacks.append(item)\n return fn\n\n return _decorator\n\n @staticmethod\n def run_before(phase: str, when: Optional[str] = None):\n \"\"\"Decorator to register a function for running before a given phase.\n\n Example::\n\n @run_before(\"build\", when=\"@2:\")\n def patch_generated_source_file(pkg):\n # Do something before the build phase for versions 2.x and above\n pass\n\n Args:\n phase: phase before which the function must run.\n when: condition under which the function is run (if :obj:`None`, it is always run).\n \"\"\"\n\n def _decorator(fn):\n key = (phase, when)\n item = (key, fn)\n _RUN_BEFORE.callbacks.append(item)\n return fn\n\n return _decorator\n\n\n# Export these names as standalone to be used in packages\nrun_after = PhaseCallbacksMeta.run_after\nrun_before = PhaseCallbacksMeta.run_before\n" }, { "path": "lib/spack/spack/platforms/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport contextlib\nfrom typing import Callable\n\nfrom ._functions import _host, by_name, platforms, reset\nfrom ._platform import Platform\nfrom .darwin import Darwin\nfrom .freebsd import FreeBSD\nfrom .linux import Linux\nfrom .test import Test\nfrom .windows import Windows\n\n__all__ = [\n \"Platform\",\n \"Darwin\",\n \"Linux\",\n \"FreeBSD\",\n \"Test\",\n \"Windows\",\n \"platforms\",\n \"host\",\n \"by_name\",\n \"reset\",\n]\n\n#: The \"real\" platform of the host running Spack. This should not be changed\n#: by any method and is here as a convenient way to refer to the host platform.\nreal_host = _host\n\n#: The current platform used by Spack. May be swapped by the use_platform\n#: context manager.\nhost: Callable[[], Platform] = _host\n\n\nclass _PickleableCallable:\n \"\"\"Class used to pickle a callable that may substitute either\n _platform or _all_platforms. Lambda or nested functions are\n not pickleable.\n \"\"\"\n\n def __init__(self, return_value):\n self.return_value = return_value\n\n def __call__(self):\n return self.return_value\n\n\n@contextlib.contextmanager\ndef use_platform(new_platform):\n global host\n\n import spack.config\n\n assert isinstance(new_platform, Platform), f'\"{new_platform}\" must be an instance of Platform'\n\n original_host_fn = host\n\n try:\n host = _PickleableCallable(new_platform)\n spack.config.CONFIG.clear_caches()\n yield new_platform\n\n finally:\n host = original_host_fn\n spack.config.CONFIG.clear_caches()\n" }, { "path": "lib/spack/spack/platforms/_functions.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport spack.llnl.util.lang\n\nfrom .darwin import Darwin\nfrom .freebsd import FreeBSD\nfrom .linux import Linux\nfrom .test import Test\nfrom .windows import Windows\n\n#: List of all the platform classes known to Spack\nplatforms = [Darwin, Linux, Windows, FreeBSD, Test]\n\n\n@spack.llnl.util.lang.memoized\ndef _host():\n \"\"\"Detect and return the platform for this machine or None if detection fails.\"\"\"\n for platform_cls in sorted(platforms, key=lambda plt: plt.priority):\n if platform_cls.detect():\n return platform_cls()\n assert False, \"No platform detected. Spack cannot run without a platform.\"\n\n\ndef reset():\n \"\"\"The result of the host search is memoized. In case it needs to be recomputed\n we must clear the cache, which is what this function does.\n \"\"\"\n _host.cache_clear()\n\n\n@spack.llnl.util.lang.memoized\ndef cls_by_name(name):\n \"\"\"Return a platform class that corresponds to the given name or None\n if there is no match.\n\n Args:\n name (str): name of the platform\n \"\"\"\n for platform_cls in sorted(platforms, key=lambda plt: plt.priority):\n if name.replace(\"_\", \"\").lower() == platform_cls.__name__.lower():\n return platform_cls\n return None\n\n\ndef by_name(name):\n \"\"\"Return a platform object that corresponds to the given name or None\n if there is no match.\n\n Args:\n name (str): name of the platform\n \"\"\"\n platform_cls = cls_by_name(name)\n return platform_cls() if platform_cls else None\n" }, { "path": "lib/spack/spack/platforms/_platform.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport warnings\nfrom typing import Optional\n\nimport spack.vendor.archspec.cpu\n\nimport spack.llnl.util.lang\n\n\n@spack.llnl.util.lang.lazy_lexicographic_ordering\nclass Platform:\n \"\"\"Platform is an abstract class extended by subclasses.\n\n Platform also contain a priority class attribute. A lower number signifies higher\n priority. These numbers are arbitrarily set and can be changed though often there\n isn't much need unless a new platform is added and the user wants that to be\n detected first.\n \"\"\"\n\n # Subclass sets number. Controls detection order\n priority: Optional[int] = None\n\n #: binary formats used on this platform; used by relocation logic\n binary_formats = [\"elf\"]\n\n default: str\n default_os: str\n\n reserved_targets = [\"default_target\", \"frontend\", \"fe\", \"backend\", \"be\"]\n reserved_oss = [\"default_os\", \"frontend\", \"fe\", \"backend\", \"be\"]\n deprecated_names = [\"frontend\", \"fe\", \"backend\", \"be\"]\n\n def __init__(self, name):\n self.targets = {}\n self.operating_sys = {}\n self.name = name\n self._init_targets()\n\n def add_target(self, name: str, target: spack.vendor.archspec.cpu.Microarchitecture) -> None:\n if name in Platform.reserved_targets:\n msg = f\"{name} is a spack reserved alias and cannot be the name of a target\"\n raise ValueError(msg)\n self.targets[name] = target\n\n def _init_targets(self):\n self.default = spack.vendor.archspec.cpu.host().name\n for name, microarchitecture in spack.vendor.archspec.cpu.TARGETS.items():\n self.add_target(name, microarchitecture)\n\n def target(self, name):\n name = str(name)\n if name in Platform.deprecated_names:\n warnings.warn(f\"target={name} is deprecated, use target={self.default} instead\")\n\n if name in Platform.reserved_targets:\n name = self.default\n\n return self.targets.get(name, None)\n\n def add_operating_system(self, name, os_class):\n if name in Platform.reserved_oss + Platform.deprecated_names:\n msg = f\"{name} is a spack reserved alias and cannot be the name of an OS\"\n raise ValueError(msg)\n self.operating_sys[name] = os_class\n\n def default_target(self):\n return self.target(self.default)\n\n def default_operating_system(self):\n return self.operating_system(self.default_os)\n\n def operating_system(self, name):\n if name in Platform.deprecated_names:\n warnings.warn(f\"os={name} is deprecated, use os={self.default_os} instead\")\n\n if name in Platform.reserved_oss:\n name = self.default_os\n\n return self.operating_sys.get(name, None)\n\n def setup_platform_environment(self, pkg, env):\n \"\"\"Platform-specific build environment modifications.\n\n This method is meant to be overridden by subclasses, when needed.\n \"\"\"\n pass\n\n @classmethod\n def detect(cls):\n \"\"\"Returns True if the host platform is detected to be the current Platform class,\n False otherwise.\n\n Derived classes are responsible for implementing this method.\n \"\"\"\n raise NotImplementedError()\n\n def __repr__(self):\n return self.__str__()\n\n def __str__(self):\n return self.name\n\n def _cmp_iter(self):\n yield self.name\n yield self.default\n yield self.default_os\n\n def targets():\n for t in sorted(self.targets.values()):\n yield t._cmp_iter\n\n yield targets\n\n def oses():\n for o in sorted(self.operating_sys.values()):\n yield o._cmp_iter\n\n yield oses\n" }, { "path": "lib/spack/spack/platforms/cray.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport os\n\n\ndef slingshot_network():\n return os.path.exists(\"/opt/cray/pe\") and (\n os.path.exists(\"/lib64/libcxi.so\") or os.path.exists(\"/usr/lib64/libcxi.so\")\n )\n" }, { "path": "lib/spack/spack/platforms/darwin.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport platform as py_platform\n\nfrom spack.operating_systems.mac_os import MacOs\nfrom spack.version import Version\n\nfrom ._platform import Platform\n\n\nclass Darwin(Platform):\n priority = 89\n\n binary_formats = [\"macho\"]\n\n def __init__(self):\n super().__init__(\"darwin\")\n mac_os = MacOs()\n self.default_os = str(mac_os)\n self.add_operating_system(str(mac_os), mac_os)\n\n @classmethod\n def detect(cls):\n return \"darwin\" in py_platform.system().lower()\n\n def setup_platform_environment(self, pkg, env):\n \"\"\"Specify deployment target based on target OS version.\n\n The ``MACOSX_DEPLOYMENT_TARGET`` environment variable provides a\n default ``-mmacosx-version-min`` argument for GCC and Clang compilers,\n as well as the default value of ``CMAKE_OSX_DEPLOYMENT_TARGET`` for\n CMake-based build systems. The default value for the deployment target\n is usually the major version (11, 10.16, ...) for CMake and Clang, but\n some versions of GCC specify a minor component as well (11.3), leading\n to numerous link warnings about inconsistent or incompatible target\n versions. Setting the environment variable ensures consistent versions\n for an install toolchain target, even when the host macOS version\n changes.\n\n TODO: it may be necessary to add SYSTEM_VERSION_COMPAT for older\n versions of the macosx developer tools; see\n https://github.com/spack/spack/pull/26290 for discussion.\n \"\"\"\n\n os = self.operating_sys[pkg.spec.os]\n version = Version(os.version)\n if len(version) == 1:\n # Version has only one component: add a minor version to prevent\n # potential errors with `ld`,\n # which fails with `-macosx_version_min 11`\n # but succeeds with `-macosx_version_min 11.0`.\n # Most compilers seem to perform this translation automatically,\n # but older GCC does not.\n version = str(version) + \".0\"\n env.set(\"MACOSX_DEPLOYMENT_TARGET\", str(version))\n" }, { "path": "lib/spack/spack/platforms/freebsd.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport platform\n\nfrom spack.operating_systems.freebsd import FreeBSDOs\n\nfrom ._platform import Platform\n\n\nclass FreeBSD(Platform):\n priority = 102\n\n def __init__(self):\n super().__init__(\"freebsd\")\n os = FreeBSDOs()\n self.default_os = str(os)\n self.add_operating_system(str(os), os)\n\n @classmethod\n def detect(cls):\n return platform.system().lower() == \"freebsd\"\n" }, { "path": "lib/spack/spack/platforms/linux.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport platform\n\nfrom spack.operating_systems.linux_distro import LinuxDistro\n\nfrom ._platform import Platform\n\n\nclass Linux(Platform):\n priority = 90\n\n def __init__(self):\n super().__init__(\"linux\")\n linux_dist = LinuxDistro()\n self.default_os = str(linux_dist)\n self.add_operating_system(str(linux_dist), linux_dist)\n\n @classmethod\n def detect(cls):\n return \"linux\" in platform.system().lower()\n" }, { "path": "lib/spack/spack/platforms/test.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport platform\n\nimport spack.vendor.archspec.cpu\n\nimport spack.operating_systems\n\nfrom ._platform import Platform\n\n\nclass Test(Platform):\n priority = 1000000\n\n if platform.system().lower() == \"darwin\":\n binary_formats = [\"macho\"]\n\n default_os = \"debian6\"\n default = \"m1\" if platform.machine() == \"arm64\" else \"core2\"\n\n def __init__(self, name=None):\n name = name or \"test\"\n super().__init__(name)\n self.add_operating_system(\"debian6\", spack.operating_systems.OperatingSystem(\"debian\", 6))\n self.add_operating_system(\"redhat6\", spack.operating_systems.OperatingSystem(\"redhat\", 6))\n\n def _init_targets(self):\n targets = (\"aarch64\", \"m1\") if platform.machine() == \"arm64\" else (\"x86_64\", \"core2\")\n for t in targets:\n self.add_target(t, spack.vendor.archspec.cpu.TARGETS[t])\n\n @classmethod\n def detect(cls):\n return True\n" }, { "path": "lib/spack/spack/platforms/windows.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport platform\n\nfrom spack.operating_systems.windows_os import WindowsOs\n\nfrom ._platform import Platform\n\n\nclass Windows(Platform):\n priority = 101\n\n def __init__(self):\n super().__init__(\"windows\")\n windows_os = WindowsOs()\n self.default_os = str(windows_os)\n self.add_operating_system(str(windows_os), windows_os)\n\n @classmethod\n def detect(cls):\n plat = platform.system().lower()\n return \"cygwin\" in plat or \"win32\" in plat or \"windows\" in plat\n" }, { "path": "lib/spack/spack/projections.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport spack.util.path\n\n\ndef get_projection(projections, spec):\n \"\"\"\n Get the projection for a spec from a projections dict.\n \"\"\"\n all_projection = None\n for spec_like, projection in projections.items():\n if spec.satisfies(spec_like):\n return spack.util.path.substitute_path_variables(projection)\n elif spec_like == \"all\":\n all_projection = spack.util.path.substitute_path_variables(projection)\n return all_projection\n" }, { "path": "lib/spack/spack/provider_index.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Classes and functions to manage providers of virtual dependencies\"\"\"\n\nfrom typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Set, Union\n\nimport spack.error\nimport spack.util.spack_json as sjson\n\nif TYPE_CHECKING:\n import spack.repo\n import spack.spec\n\n\nclass ProviderIndex:\n #: This is a dict of dicts used for finding providers of particular\n #: virtual dependencies. The dict of dicts looks like:\n #:\n #: { vpkg name :\n #: { full vpkg spec : set(packages providing spec) } }\n #:\n #: Callers can use this to first find which packages provide a vpkg,\n #: then find a matching full spec. e.g., in this scenario:\n #:\n #: { 'mpi' :\n #: { mpi@:1.1 : set([mpich]),\n #: mpi@:2.3 : set([mpich2@1.9:]) } }\n #:\n #: Calling providers_for(spec) will find specs that provide a\n #: matching implementation of MPI. Derived class need to construct\n #: this attribute according to the semantics above.\n providers: Dict[str, Dict[\"spack.spec.Spec\", Set[\"spack.spec.Spec\"]]]\n\n def __init__(\n self,\n repository: \"spack.repo.RepoType\",\n specs: Optional[Iterable[\"spack.spec.Spec\"]] = None,\n restrict: bool = False,\n ):\n \"\"\"Provider index based on a single mapping of providers.\n\n Args:\n specs: if provided, will call update on each\n single spec to initialize this provider index.\n\n restrict: \"restricts\" values to the verbatim input specs; do not\n pre-apply package's constraints.\n\n TODO: rename this. It is intended to keep things as broad\n TODO: as possible without overly restricting results, so it is\n TODO: not the best name.\n \"\"\"\n self.repository = repository\n self.restrict = restrict\n self.providers = {}\n if specs:\n self.update_packages(specs)\n\n def providers_for(self, virtual: Union[str, \"spack.spec.Spec\"]) -> List[\"spack.spec.Spec\"]:\n \"\"\"Return a list of specs of all packages that provide virtual packages with the supplied\n spec.\n\n Args:\n virtual: either a Spec or a string name of a virtual package\n \"\"\"\n result: Set[\"spack.spec.Spec\"] = set()\n\n if isinstance(virtual, str):\n # In the common case where just a package name is passed, we can avoid running the\n # spec parser and intersects, since intersects is always true.\n if virtual.isalnum():\n if virtual in self.providers:\n for p_spec, spec_set in self.providers[virtual].items():\n result.update(spec_set)\n return list(result)\n\n from spack.spec import Spec\n\n virtual = Spec(virtual)\n\n # Add all the providers that satisfy the vpkg spec.\n if virtual.name in self.providers:\n for p_spec, spec_set in self.providers[virtual.name].items():\n if p_spec.intersects(virtual, deps=False):\n result.update(spec_set)\n\n return list(result)\n\n def __contains__(self, name):\n return name in self.providers\n\n def __eq__(self, other):\n return self.providers == other.providers\n\n def _transform(self, transform_fun, out_mapping_type=dict):\n \"\"\"Transform this provider index dictionary and return it.\n\n Args:\n transform_fun: transform_fun takes a (vpkg, pset) mapping and runs\n it on each pair in nested dicts.\n out_mapping_type: type to be used internally on the\n transformed (vpkg, pset)\n\n Returns:\n Transformed mapping\n \"\"\"\n return _transform(self.providers, transform_fun, out_mapping_type)\n\n def __str__(self):\n return str(self.providers)\n\n def __repr__(self):\n return repr(self.providers)\n\n def update_packages(self, specs: Iterable[Union[str, \"spack.spec.Spec\"]]):\n \"\"\"Update the provider index with additional virtual specs.\n\n Args:\n spec: spec potentially providing additional virtual specs\n \"\"\"\n from spack.spec import Spec\n\n for spec in specs:\n if not isinstance(spec, Spec):\n spec = Spec(spec)\n\n if not spec.name or self.repository.is_virtual_safe(spec.name):\n # Only non-virtual packages with name can provide virtual specs.\n continue\n\n pkg_cls = self.repository.get_pkg_class(spec.name)\n for provider_spec_readonly, provided_specs in pkg_cls.provided.items():\n for provided_spec in provided_specs:\n # TODO: fix this comment.\n # We want satisfaction other than flags\n provider_spec = provider_spec_readonly.copy()\n provider_spec.compiler_flags = spec.compiler_flags.copy()\n\n if spec.intersects(provider_spec, deps=False):\n provided_name = provided_spec.name\n\n provider_map = self.providers.setdefault(provided_name, {})\n if provided_spec not in provider_map:\n provider_map[provided_spec] = set()\n\n if self.restrict:\n provider_set = provider_map[provided_spec]\n\n # If this package existed in the index before,\n # need to take the old versions out, as they're\n # now more constrained.\n old = {s for s in provider_set if s.name == spec.name}\n provider_set.difference_update(old)\n\n # Now add the new version.\n provider_set.add(spec)\n\n else:\n # Before putting the spec in the map, constrain\n # it so that it provides what was asked for.\n constrained = spec.copy()\n constrained.constrain(provider_spec)\n provider_map[provided_spec].add(constrained)\n\n def to_json(self, stream=None):\n \"\"\"Dump a JSON representation of this object.\n\n Args:\n stream: stream where to dump\n \"\"\"\n provider_list = self._transform(\n lambda vpkg, pset: [vpkg.to_node_dict(), [p.to_node_dict() for p in pset]], list\n )\n\n sjson.dump({\"provider_index\": {\"providers\": provider_list}}, stream)\n\n def merge(self, other):\n \"\"\"Merge another provider index into this one.\n\n Args:\n other (ProviderIndex): provider index to be merged\n \"\"\"\n other = other.copy() # defensive copy.\n\n for pkg in other.providers:\n if pkg not in self.providers:\n self.providers[pkg] = other.providers[pkg]\n continue\n\n spdict, opdict = self.providers[pkg], other.providers[pkg]\n for provided_spec in opdict:\n if provided_spec not in spdict:\n spdict[provided_spec] = opdict[provided_spec]\n continue\n\n spdict[provided_spec] = spdict[provided_spec].union(opdict[provided_spec])\n\n def remove_providers(self, pkg_names: Set[str]):\n \"\"\"Remove a provider from the ProviderIndex.\"\"\"\n empty_pkg_dict = []\n for pkg, pkg_dict in self.providers.items():\n empty_pset = []\n for provided, pset in pkg_dict.items():\n to_remove = {spec for spec in pset if spec.name in pkg_names}\n pset.difference_update(to_remove)\n\n if not pset:\n empty_pset.append(provided)\n\n for provided in empty_pset:\n del pkg_dict[provided]\n\n if not pkg_dict:\n empty_pkg_dict.append(pkg)\n\n for pkg in empty_pkg_dict:\n del self.providers[pkg]\n\n def copy(self):\n \"\"\"Return a deep copy of this index.\"\"\"\n clone = ProviderIndex(repository=self.repository)\n clone.providers = self._transform(lambda vpkg, pset: (vpkg, set((p.copy() for p in pset))))\n return clone\n\n @staticmethod\n def from_json(stream, repository):\n \"\"\"Construct a provider index from its JSON representation.\n\n Args:\n stream: stream where to read from the JSON data\n \"\"\"\n data = sjson.load(stream)\n\n if not isinstance(data, dict):\n raise ProviderIndexError(\"JSON ProviderIndex data was not a dict.\")\n\n if \"provider_index\" not in data:\n raise ProviderIndexError(\"YAML ProviderIndex does not start with 'provider_index'\")\n\n index = ProviderIndex(repository=repository)\n providers = data[\"provider_index\"][\"providers\"]\n from spack.spec import SpecfileLatest\n\n index.providers = _transform(\n providers,\n lambda vpkg, plist: (\n SpecfileLatest.from_node_dict(vpkg),\n set(SpecfileLatest.from_node_dict(p) for p in plist),\n ),\n )\n return index\n\n\ndef _transform(providers, transform_fun, out_mapping_type=dict):\n \"\"\"Syntactic sugar for transforming a providers dict.\n\n Args:\n providers: provider dictionary\n transform_fun: transform_fun takes a (vpkg, pset) mapping and runs\n it on each pair in nested dicts.\n out_mapping_type: type to be used internally on the\n transformed (vpkg, pset)\n\n Returns:\n Transformed mapping\n \"\"\"\n\n def mapiter(mappings):\n if isinstance(mappings, dict):\n return mappings.items()\n else:\n return iter(mappings)\n\n return dict(\n (name, out_mapping_type([transform_fun(vpkg, pset) for vpkg, pset in mapiter(mappings)]))\n for name, mappings in providers.items()\n )\n\n\nclass ProviderIndexError(spack.error.SpackError):\n \"\"\"Raised when there is a problem with a ProviderIndex.\"\"\"\n" }, { "path": "lib/spack/spack/relocate.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport collections\nimport itertools\nimport os\nimport re\nimport sys\nfrom typing import Dict, Iterable, List, Optional\n\nimport spack.vendor.macholib.mach_o\nimport spack.vendor.macholib.MachO\n\nimport spack.llnl.util.filesystem as fs\nimport spack.llnl.util.lang\nimport spack.llnl.util.tty as tty\nimport spack.store\nimport spack.util.elf as elf\nimport spack.util.executable as executable\nfrom spack.llnl.util.filesystem import readlink, symlink\nfrom spack.llnl.util.lang import memoized\n\nfrom .relocate_text import BinaryFilePrefixReplacer, PrefixToPrefix, TextFilePrefixReplacer\n\n\n@memoized\ndef _patchelf() -> Optional[executable.Executable]:\n \"\"\"Return the full path to the patchelf binary, if available, else None.\"\"\"\n import spack.bootstrap\n\n if sys.platform == \"darwin\":\n return None\n\n with spack.bootstrap.ensure_bootstrap_configuration():\n return spack.bootstrap.ensure_patchelf_in_path_or_raise()\n\n\ndef _decode_macho_data(bytestring):\n return bytestring.rstrip(b\"\\x00\").decode(\"ascii\")\n\n\ndef _macho_find_paths(orig_rpaths, deps, idpath, prefix_to_prefix):\n \"\"\"\n Inputs\n original rpaths from mach-o binaries\n dependency libraries for mach-o binaries\n id path of mach-o libraries\n old install directory layout root\n prefix_to_prefix dictionary which maps prefixes in the old directory layout\n to directories in the new directory layout\n Output\n paths_to_paths dictionary which maps all of the old paths to new paths\n \"\"\"\n paths_to_paths = dict()\n # Sort from longest path to shortest, to ensure we try /foo/bar/baz before /foo/bar\n prefix_iteration_order = sorted(prefix_to_prefix, key=len, reverse=True)\n for orig_rpath in orig_rpaths:\n for old_prefix in prefix_iteration_order:\n new_prefix = prefix_to_prefix[old_prefix]\n if orig_rpath.startswith(old_prefix):\n new_rpath = re.sub(re.escape(old_prefix), new_prefix, orig_rpath)\n paths_to_paths[orig_rpath] = new_rpath\n break\n else:\n paths_to_paths[orig_rpath] = orig_rpath\n\n if idpath:\n for old_prefix in prefix_iteration_order:\n new_prefix = prefix_to_prefix[old_prefix]\n if idpath.startswith(old_prefix):\n paths_to_paths[idpath] = re.sub(re.escape(old_prefix), new_prefix, idpath)\n break\n\n for dep in deps:\n for old_prefix in prefix_iteration_order:\n new_prefix = prefix_to_prefix[old_prefix]\n if dep.startswith(old_prefix):\n paths_to_paths[dep] = re.sub(re.escape(old_prefix), new_prefix, dep)\n break\n\n if dep.startswith(\"@\"):\n paths_to_paths[dep] = dep\n\n return paths_to_paths\n\n\ndef _modify_macho_object(cur_path, rpaths, deps, idpath, paths_to_paths):\n \"\"\"\n This function is used to make machO buildcaches on macOS by\n replacing old paths with new paths using install_name_tool\n Inputs:\n mach-o binary to be modified\n original rpaths\n original dependency paths\n original id path if a mach-o library\n dictionary mapping paths in old install layout to new install layout\n \"\"\"\n # avoid error message for libgcc_s\n if \"libgcc_\" in cur_path:\n return\n args = []\n\n if idpath:\n new_idpath = paths_to_paths.get(idpath, None)\n if new_idpath and not idpath == new_idpath:\n args += [(\"-id\", new_idpath)]\n\n for dep in deps:\n new_dep = paths_to_paths.get(dep)\n if new_dep and dep != new_dep:\n args += [(\"-change\", dep, new_dep)]\n\n new_rpaths = []\n for orig_rpath in rpaths:\n new_rpath = paths_to_paths.get(orig_rpath)\n if new_rpath and not orig_rpath == new_rpath:\n args_to_add = (\"-rpath\", orig_rpath, new_rpath)\n if args_to_add not in args and new_rpath not in new_rpaths:\n args += [args_to_add]\n new_rpaths.append(new_rpath)\n\n # Deduplicate and flatten\n args = list(itertools.chain.from_iterable(spack.llnl.util.lang.dedupe(args)))\n install_name_tool = executable.Executable(\"install_name_tool\")\n if args:\n with fs.edit_in_place_through_temporary_file(cur_path) as temp_path:\n install_name_tool(*args, temp_path)\n\n\ndef _macholib_get_paths(cur_path):\n \"\"\"Get rpaths, dependent libraries, and library id of mach-o objects.\"\"\"\n headers = []\n try:\n headers = spack.vendor.macholib.MachO.MachO(cur_path).headers\n except ValueError:\n pass\n if not headers:\n tty.warn(\"Failed to read Mach-O headers: {0}\".format(cur_path))\n commands = []\n else:\n if len(headers) > 1:\n # Reproduce original behavior of only returning the last mach-O\n # header section\n tty.warn(\"Encountered fat binary: {0}\".format(cur_path))\n if headers[-1].filetype == \"dylib_stub\":\n tty.warn(\"File is a stub, not a full library: {0}\".format(cur_path))\n commands = headers[-1].commands\n\n LC_ID_DYLIB = spack.vendor.macholib.mach_o.LC_ID_DYLIB\n LC_LOAD_DYLIB = spack.vendor.macholib.mach_o.LC_LOAD_DYLIB\n LC_RPATH = spack.vendor.macholib.mach_o.LC_RPATH\n\n ident = None\n rpaths = []\n deps = []\n for load_command, dylib_command, data in commands:\n cmd = load_command.cmd\n if cmd == LC_RPATH:\n rpaths.append(_decode_macho_data(data))\n elif cmd == LC_LOAD_DYLIB:\n deps.append(_decode_macho_data(data))\n elif cmd == LC_ID_DYLIB:\n ident = _decode_macho_data(data)\n\n return (rpaths, deps, ident)\n\n\ndef _set_elf_rpaths_and_interpreter(\n target: str, rpaths: List[str], interpreter: Optional[str] = None\n) -> Optional[str]:\n \"\"\"Replace the original RPATH of the target with the paths passed as arguments.\n\n Args:\n target: target executable. Must be an ELF object.\n rpaths: paths to be set in the RPATH\n interpreter: optionally set the interpreter\n\n Returns:\n A string concatenating the stdout and stderr of the call to ``patchelf`` if it was invoked\n \"\"\"\n # Join the paths using ':' as a separator\n rpaths_str = \":\".join(rpaths)\n\n try:\n # TODO: error handling is not great here?\n # TODO: revisit the use of --force-rpath as it might be conditional\n # TODO: if we want to support setting RUNPATH from binary packages\n args = [\"--force-rpath\", \"--set-rpath\", rpaths_str]\n if interpreter:\n args.extend([\"--set-interpreter\", interpreter])\n args.append(target)\n return _patchelf()(*args, output=str, error=str)\n except executable.ProcessError as e:\n tty.warn(str(e))\n return None\n\n\ndef relocate_macho_binaries(path_names, prefix_to_prefix):\n \"\"\"\n Use macholib python package to get the rpaths, dependent libraries\n and library identity for libraries from the MachO object. Modify them\n with the replacement paths queried from the dictionary mapping old layout\n prefixes to hashes and the dictionary mapping hashes to the new layout\n prefixes.\n \"\"\"\n\n for path_name in path_names:\n # Corner case where macho object file ended up in the path name list\n if path_name.endswith(\".o\"):\n continue\n # get the paths in the old prefix\n rpaths, deps, idpath = _macholib_get_paths(path_name)\n # get the mapping of paths in the old prerix to the new prefix\n paths_to_paths = _macho_find_paths(rpaths, deps, idpath, prefix_to_prefix)\n # replace the old paths with new paths\n _modify_macho_object(path_name, rpaths, deps, idpath, paths_to_paths)\n\n\ndef relocate_elf_binaries(binaries: Iterable[str], prefix_to_prefix: Dict[str, str]) -> None:\n \"\"\"Take a list of binaries, and an ordered prefix to prefix mapping, and update the rpaths\n accordingly.\"\"\"\n\n # Transform to binary string\n prefix_to_prefix_bin = {\n k.encode(\"utf-8\"): v.encode(\"utf-8\") for k, v in prefix_to_prefix.items()\n }\n\n for path in binaries:\n try:\n elf.substitute_rpath_and_pt_interp_in_place_or_raise(path, prefix_to_prefix_bin)\n except elf.ElfCStringUpdatesFailed as e:\n # Fall back to `patchelf --set-rpath ... --set-interpreter ...`\n rpaths = e.rpath.new_value.decode(\"utf-8\").split(\":\") if e.rpath else []\n interpreter = e.pt_interp.new_value.decode(\"utf-8\") if e.pt_interp else None\n _set_elf_rpaths_and_interpreter(path, rpaths=rpaths, interpreter=interpreter)\n\n\ndef relocate_links(links: Iterable[str], prefix_to_prefix: Dict[str, str]) -> None:\n \"\"\"Relocate links to a new install prefix.\"\"\"\n regex = re.compile(\"|\".join(re.escape(p) for p in prefix_to_prefix.keys()))\n for link in links:\n old_target = readlink(link)\n if not os.path.isabs(old_target):\n continue\n match = regex.match(old_target)\n if match is None:\n continue\n\n new_target = prefix_to_prefix[match.group()] + old_target[match.end() :]\n os.unlink(link)\n symlink(new_target, link)\n\n\ndef relocate_text(files: Iterable[str], prefix_to_prefix: PrefixToPrefix) -> None:\n \"\"\"Relocate text file from the original installation prefix to the\n new prefix.\n\n Relocation also affects the the path in Spack's sbang script.\n\n Args:\n files: Text files to be relocated\n prefix_to_prefix: ordered prefix to prefix mapping\n \"\"\"\n TextFilePrefixReplacer.from_strings_or_bytes(prefix_to_prefix).apply(files)\n\n\ndef relocate_text_bin(binaries: Iterable[str], prefix_to_prefix: PrefixToPrefix) -> List[str]:\n \"\"\"Replace null terminated path strings hard-coded into binaries.\n\n The new install prefix must be shorter than the original one.\n\n Args:\n binaries: paths to binaries to be relocated\n prefix_to_prefix: ordered prefix to prefix mapping\n\n Raises:\n spack.relocate_text.BinaryTextReplaceError: when the new path is longer than the old path\n \"\"\"\n return BinaryFilePrefixReplacer.from_strings_or_bytes(prefix_to_prefix).apply(binaries)\n\n\ndef is_macho_magic(magic: bytes) -> bool:\n return (\n # In order of popularity: 64-bit mach-o le/be, 32-bit mach-o le/be.\n magic.startswith(b\"\\xcf\\xfa\\xed\\xfe\")\n or magic.startswith(b\"\\xfe\\xed\\xfa\\xcf\")\n or magic.startswith(b\"\\xce\\xfa\\xed\\xfe\")\n or magic.startswith(b\"\\xfe\\xed\\xfa\\xce\")\n # universal binaries: 0xcafebabe be (most common?) or 0xbebafeca le (not sure if exists).\n # Here we need to disambiguate mach-o and JVM class files. In mach-o the next 4 bytes are\n # the number of binaries; in JVM class files it's the java version number. We assume there\n # are less than 10 binaries in a universal binary.\n or (magic.startswith(b\"\\xca\\xfe\\xba\\xbe\") and int.from_bytes(magic[4:8], \"big\") < 10)\n or (magic.startswith(b\"\\xbe\\xba\\xfe\\xca\") and int.from_bytes(magic[4:8], \"little\") < 10)\n )\n\n\ndef is_elf_magic(magic: bytes) -> bool:\n return magic.startswith(b\"\\x7fELF\")\n\n\ndef is_binary(filename: str) -> bool:\n \"\"\"Returns true iff a file is likely binary\"\"\"\n with open(filename, \"rb\") as f:\n magic = f.read(8)\n\n return is_macho_magic(magic) or is_elf_magic(magic)\n\n\n# Memoize this due to repeated calls to libraries in the same directory.\n@spack.llnl.util.lang.memoized\ndef _exists_dir(dirname):\n return os.path.isdir(dirname)\n\n\ndef is_macho_binary(path: str) -> bool:\n try:\n with open(path, \"rb\") as f:\n return is_macho_magic(f.read(4))\n except OSError:\n return False\n\n\ndef fixup_macos_rpath(root, filename):\n \"\"\"Apply rpath fixups to the given file.\n\n Args:\n root: absolute path to the parent directory\n filename: relative path to the library or binary\n\n Returns:\n True if fixups were applied, else False\n \"\"\"\n abspath = os.path.join(root, filename)\n\n if not is_macho_binary(abspath):\n return False\n\n # Get Mach-O header commands\n (rpath_list, deps, id_dylib) = _macholib_get_paths(abspath)\n\n # Convert rpaths list to (name -> number of occurrences)\n add_rpaths = set()\n del_rpaths = set()\n rpaths = collections.defaultdict(int)\n for rpath in rpath_list:\n rpaths[rpath] += 1\n\n args = []\n\n # Check dependencies for non-rpath entries\n spack_root = spack.store.STORE.layout.root\n for name in deps:\n if name.startswith(spack_root):\n tty.debug(\"Spack-installed dependency for {0}: {1}\".format(abspath, name))\n (dirname, basename) = os.path.split(name)\n if dirname != root or dirname in rpaths:\n # Only change the rpath if it's a dependency *or* if the root\n # rpath was already added to the library (this is to prevent\n # GCC or similar getting rpaths when they weren't at all\n # configured)\n args += [\"-change\", name, \"@rpath/\" + basename]\n add_rpaths.add(dirname.rstrip(\"/\"))\n\n # Check for nonexistent rpaths (often added by spack linker overzealousness\n # with both lib/ and lib64/) and duplicate rpaths\n for rpath, count in rpaths.items():\n if rpath.startswith(\"@loader_path\") or rpath.startswith(\"@executable_path\"):\n # Allowable relative paths\n pass\n elif not _exists_dir(rpath):\n tty.debug(\"Nonexistent rpath in {0}: {1}\".format(abspath, rpath))\n del_rpaths.add(rpath)\n elif count > 1:\n # Rpath should only be there once, but it can sometimes be\n # duplicated between Spack's compiler and libtool. If there are\n # more copies of the same one, something is very odd....\n tty_debug = tty.debug if count == 2 else tty.warn\n tty_debug(\"Rpath appears {0} times in {1}: {2}\".format(count, abspath, rpath))\n del_rpaths.add(rpath)\n\n # Delete bad rpaths\n for rpath in del_rpaths:\n args += [\"-delete_rpath\", rpath]\n\n # Add missing rpaths that are not set for deletion\n for rpath in add_rpaths - del_rpaths - set(rpaths):\n args += [\"-add_rpath\", rpath]\n\n if not args:\n # No fixes needed\n return False\n\n with fs.edit_in_place_through_temporary_file(abspath) as temp_path:\n executable.Executable(\"install_name_tool\")(*args, temp_path)\n return True\n\n\ndef fixup_macos_rpaths(spec):\n \"\"\"Remove duplicate and nonexistent rpaths.\n\n Some autotools packages write their own ``-rpath`` entries in addition to\n those implicitly added by the Spack compiler wrappers. On Linux these\n duplicate rpaths are eliminated, but on macOS they result in multiple\n entries which makes it harder to adjust with ``install_name_tool\n -delete_rpath``.\n \"\"\"\n if spec.external or not spec.concrete:\n tty.warn(\"external/abstract spec cannot be fixed up: {0!s}\".format(spec))\n return False\n\n if \"platform=darwin\" not in spec:\n raise NotImplementedError(\"fixup_macos_rpaths requires macOS\")\n\n applied = 0\n\n libs = frozenset([\"lib\", \"lib64\", \"libexec\", \"plugins\", \"Library\", \"Frameworks\"])\n prefix = spec.prefix\n\n if not os.path.exists(prefix):\n raise RuntimeError(\n \"Could not fix up install prefix spec {0} because it does not exist: {1!s}\".format(\n prefix, spec.name\n )\n )\n\n # Explore the installation prefix of the spec\n for root, dirs, files in os.walk(prefix, topdown=True):\n dirs[:] = set(dirs) & libs\n for name in files:\n try:\n needed_fix = fixup_macos_rpath(root, name)\n except Exception as e:\n tty.warn(\"Failed to apply library fixups to: {0}/{1}: {2!s}\".format(root, name, e))\n needed_fix = False\n if needed_fix:\n applied += 1\n\n specname = spec.format(\"{name}{/hash:7}\")\n if applied:\n tty.info(\n \"Fixed rpaths for {0:d} {1} installed to {2}\".format(\n applied, \"binary\" if applied == 1 else \"binaries\", specname\n )\n )\n else:\n tty.debug(\"No rpath fixup needed for \" + specname)\n" }, { "path": "lib/spack/spack/relocate_text.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"This module contains pure-Python classes and functions for replacing\npaths inside text files and binaries.\"\"\"\n\nimport re\nfrom typing import IO, Dict, Iterable, List, Union\n\nimport spack.error\nfrom spack.llnl.util.lang import PatternBytes\n\nPrefix = Union[str, bytes]\nPrefixToPrefix = Union[Dict[str, str], Dict[bytes, bytes]]\n\n\ndef encode_path(p: Prefix) -> bytes:\n return p if isinstance(p, bytes) else p.encode(\"utf-8\")\n\n\ndef _prefix_to_prefix_as_bytes(prefix_to_prefix: PrefixToPrefix) -> Dict[bytes, bytes]:\n return {encode_path(k): encode_path(v) for (k, v) in prefix_to_prefix.items()}\n\n\ndef utf8_path_to_binary_regex(prefix: str) -> PatternBytes:\n \"\"\"Create a binary regex that matches the input path in utf8\"\"\"\n prefix_bytes = re.escape(prefix).encode(\"utf-8\")\n return re.compile(b\"(? PatternBytes:\n all_prefixes = b\"|\".join(re.escape(p) for p in prefixes)\n return re.compile(b\"(? PatternBytes:\n \"\"\"Create a (binary) regex that matches any input path in utf8\"\"\"\n return _byte_strings_to_single_binary_regex(p.encode(\"utf-8\") for p in prefixes)\n\n\ndef filter_identity_mappings(prefix_to_prefix: Dict[bytes, bytes]) -> Dict[bytes, bytes]:\n \"\"\"Drop mappings that are not changed.\"\"\"\n # NOTE: we don't guard against the following case:\n # [/abc/def -> /abc/def, /abc -> /x] *will* be simplified to\n # [/abc -> /x], meaning that after this simplification /abc/def will be\n # mapped to /x/def instead of /abc/def. This should not be a problem.\n return {k: v for k, v in prefix_to_prefix.items() if k != v}\n\n\nclass PrefixReplacer:\n \"\"\"Base class for applying a prefix to prefix map to a list of binaries or text files. Derived\n classes implement _apply_to_file to do the actual work, which is different when it comes to\n binaries and text files.\"\"\"\n\n def __init__(self, prefix_to_prefix: Dict[bytes, bytes]) -> None:\n \"\"\"\n Arguments:\n prefix_to_prefix: An ordered mapping from prefix to prefix. The order is relevant to\n support substring fallbacks, for example\n ``[(\"/first/sub\", \"/x\"), (\"/first\", \"/y\")]`` will ensure /first/sub is matched and\n replaced before /first.\n \"\"\"\n self.prefix_to_prefix = filter_identity_mappings(prefix_to_prefix)\n\n @property\n def is_noop(self) -> bool:\n \"\"\"Returns true when the prefix to prefix map\n is mapping everything to the same location (identity)\n or there are no prefixes to replace.\"\"\"\n return not self.prefix_to_prefix\n\n def apply(self, filenames: Iterable[str]) -> List[str]:\n \"\"\"Returns a list of files that were modified\"\"\"\n changed_files = []\n if self.is_noop:\n return []\n for filename in filenames:\n if self.apply_to_filename(filename):\n changed_files.append(filename)\n return changed_files\n\n def apply_to_filename(self, filename: str) -> bool:\n if self.is_noop:\n return False\n with open(filename, \"rb+\") as f:\n return self.apply_to_file(f)\n\n def apply_to_file(self, f: IO[bytes]) -> bool:\n if self.is_noop:\n return False\n return self._apply_to_file(f)\n\n def _apply_to_file(self, f: IO) -> bool:\n raise NotImplementedError(\"Derived classes must implement this method\")\n\n\nclass TextFilePrefixReplacer(PrefixReplacer):\n \"\"\"This class applies prefix to prefix mappings for relocation\n on text files.\n\n Note that UTF-8 encoding is assumed.\"\"\"\n\n def __init__(self, prefix_to_prefix: Dict[bytes, bytes]):\n \"\"\"\n prefix_to_prefix (OrderedDict): OrderedDictionary where the keys are\n bytes representing the old prefixes and the values are the new.\n \"\"\"\n super().__init__(prefix_to_prefix)\n # Single regex for all paths.\n self.regex = _byte_strings_to_single_binary_regex(self.prefix_to_prefix.keys())\n\n @classmethod\n def from_strings_or_bytes(cls, prefix_to_prefix: PrefixToPrefix) -> \"TextFilePrefixReplacer\":\n \"\"\"Create a TextFilePrefixReplacer from an ordered prefix to prefix map.\"\"\"\n return cls(_prefix_to_prefix_as_bytes(prefix_to_prefix))\n\n def _apply_to_file(self, f: IO) -> bool:\n \"\"\"Text replacement implementation simply reads the entire file\n in memory and applies the combined regex.\"\"\"\n replacement = lambda m: m.group(1) + self.prefix_to_prefix[m.group(2)] + m.group(3)\n data = f.read()\n new_data = re.sub(self.regex, replacement, data)\n if id(data) == id(new_data):\n return False\n f.seek(0)\n f.write(new_data)\n f.truncate()\n return True\n\n\nclass BinaryFilePrefixReplacer(PrefixReplacer):\n def __init__(self, prefix_to_prefix: Dict[bytes, bytes], suffix_safety_size: int = 7) -> None:\n \"\"\"\n prefix_to_prefix: Ordered dictionary where the keys are bytes representing the old prefixes\n and the values are the new\n suffix_safety_size: in case of null terminated strings, what size of the suffix should\n remain to avoid aliasing issues?\n \"\"\"\n assert suffix_safety_size >= 0\n super().__init__(prefix_to_prefix)\n self.suffix_safety_size = suffix_safety_size\n self.regex = self.binary_text_regex(self.prefix_to_prefix.keys(), suffix_safety_size)\n\n @classmethod\n def binary_text_regex(\n cls, binary_prefixes: Iterable[bytes], suffix_safety_size: int = 7\n ) -> PatternBytes:\n \"\"\"Create a regex that looks for exact matches of prefixes, and also tries to match a\n C-string type null terminator in a small lookahead window.\n\n Arguments:\n binary_prefixes: Iterable of byte strings of prefixes to match\n suffix_safety_size: Sizeof the lookahed for null-terminated string.\n \"\"\"\n # Note: it's important not to use capture groups for the prefix, since it destroys\n # performance due to common prefix optimization.\n return re.compile(\n b\"(\"\n + b\"|\".join(re.escape(p) for p in binary_prefixes)\n + b\")([^\\0]{0,%d}\\0)?\" % suffix_safety_size\n )\n\n @classmethod\n def from_strings_or_bytes(\n cls, prefix_to_prefix: PrefixToPrefix, suffix_safety_size: int = 7\n ) -> \"BinaryFilePrefixReplacer\":\n \"\"\"Create a BinaryFilePrefixReplacer from an ordered prefix to prefix map.\n\n Arguments:\n prefix_to_prefix: Ordered mapping of prefix to prefix.\n suffix_safety_size: Number of bytes to retain at the end of a C-string to avoid binary\n string-aliasing issues.\n \"\"\"\n return cls(_prefix_to_prefix_as_bytes(prefix_to_prefix), suffix_safety_size)\n\n def _apply_to_file(self, f: IO[bytes]) -> bool:\n \"\"\"\n Given a file opened in rb+ mode, apply the string replacements as specified by an ordered\n dictionary of prefix to prefix mappings. This method takes special care of null-terminated\n C-strings. C-string constants are problematic because compilers and linkers optimize\n readonly strings for space by aliasing those that share a common suffix (only suffix since\n all of them are null terminated). See https://github.com/spack/spack/pull/31739 and\n https://github.com/spack/spack/pull/32253 for details. Our logic matches the original\n prefix with a ``suffix_safety_size + 1`` lookahead for null bytes. If no null terminator\n is found, we simply pad with leading /, assuming that it's a long C-string; the full\n C-string after replacement has a large suffix in common with its original value. If there\n *is* a null terminator we can do the same as long as the replacement has a sufficiently\n long common suffix with the original prefix. As a last resort when the replacement does\n not have a long enough common suffix, we can try to shorten the string, but this only\n works if the new length is sufficiently short (typically the case when going from large\n padding -> normal path) If the replacement string is longer, or all of the above fails,\n we error out.\n\n Arguments:\n f: file opened in rb+ mode\n\n Returns:\n bool: True if file was modified\n \"\"\"\n assert f.tell() == 0\n\n # We *could* read binary data in chunks to avoid loading all in memory, but it's nasty to\n # deal with matches across boundaries, so let's stick to something simple.\n\n modified = False\n\n for match in self.regex.finditer(f.read()):\n # The matching prefix (old) and its replacement (new)\n old = match.group(1)\n new = self.prefix_to_prefix[old]\n\n # Did we find a trailing null within a N + 1 bytes window after the prefix?\n null_terminated = match.end(0) > match.end(1)\n\n # Suffix string length, excluding the null byte. Only makes sense if null_terminated\n suffix_strlen = match.end(0) - match.end(1) - 1\n\n # How many bytes are we shrinking our string?\n bytes_shorter = len(old) - len(new)\n\n # We can't make strings larger.\n if bytes_shorter < 0:\n raise CannotGrowString(old, new)\n\n # If we don't know whether this is a null terminated C-string (we're looking only N + 1\n # bytes ahead), or if it is and we have a common suffix, we can simply pad with leading\n # dir separators.\n elif (\n not null_terminated\n or suffix_strlen >= self.suffix_safety_size # == is enough, but let's be defensive\n or old[-self.suffix_safety_size + suffix_strlen :]\n == new[-self.suffix_safety_size + suffix_strlen :]\n ):\n replacement = b\"/\" * bytes_shorter + new\n\n # If it *was* null terminated, all that matters is that we can leave N bytes of old\n # suffix in place. Note that > is required since we also insert an additional null\n # terminator.\n elif bytes_shorter > self.suffix_safety_size:\n replacement = new + match.group(2) # includes the trailing null\n\n # Otherwise... we can't :(\n else:\n raise CannotShrinkCString(old, new, match.group()[:-1])\n\n f.seek(match.start())\n f.write(replacement)\n modified = True\n\n return modified\n\n\nclass BinaryTextReplaceError(spack.error.SpackError):\n def __init__(self, msg):\n msg += (\n \" To fix this, compile with more padding \"\n \"(config:install_tree:padded_length), or install to a shorter prefix.\"\n )\n super().__init__(msg)\n\n\nclass CannotGrowString(BinaryTextReplaceError):\n def __init__(self, old, new):\n return super().__init__(\n f\"Cannot replace {old!r} with {new!r} because the new prefix is longer.\"\n )\n\n\nclass CannotShrinkCString(BinaryTextReplaceError):\n def __init__(self, old, new, full_old_string):\n # Just interpolate binary string to not risk issues with invalid unicode, which would be\n # really bad user experience: error in error. We have no clue if we actually deal with a\n # real C-string nor what encoding it has.\n super().__init__(\n f\"Cannot replace {old!r} with {new!r} in the C-string {full_old_string!r}.\"\n )\n" }, { "path": "lib/spack/spack/repo.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport abc\nimport contextlib\nimport difflib\nimport errno\nimport functools\nimport importlib\nimport importlib.machinery\nimport importlib.util\nimport itertools\nimport math\nimport os\nimport re\nimport shutil\nimport stat\nimport sys\nimport traceback\nimport types\nimport uuid\nimport warnings\nfrom typing import (\n TYPE_CHECKING,\n Any,\n Callable,\n Dict,\n Generator,\n Iterator,\n List,\n Mapping,\n Optional,\n Set,\n Tuple,\n Type,\n Union,\n cast,\n)\n\nimport spack\nimport spack.caches\nimport spack.config\nimport spack.error\nimport spack.llnl.path\nimport spack.llnl.util.filesystem as fs\nimport spack.llnl.util.tty as tty\nimport spack.patch\nimport spack.paths\nimport spack.provider_index\nimport spack.tag\nimport spack.util.executable\nimport spack.util.file_cache\nimport spack.util.git\nimport spack.util.hash\nimport spack.util.lock\nimport spack.util.naming as nm\nimport spack.util.path\nimport spack.util.spack_yaml as syaml\nfrom spack.llnl.util.filesystem import working_dir\nfrom spack.llnl.util.lang import Singleton, memoized\n\nif TYPE_CHECKING:\n import spack.package_base\n import spack.spec\n\nPKG_MODULE_PREFIX_V1 = \"spack.pkg.\"\nPKG_MODULE_PREFIX_V2 = \"spack_repo.\"\n\n_API_REGEX = re.compile(r\"^v(\\d+)\\.(\\d+)$\")\n\nSPACK_REPO_INDEX_FILE_NAME = \"spack-repo-index.yaml\"\n\n\ndef package_repository_lock() -> spack.util.lock.Lock:\n \"\"\"Lock for process safety when cloning remote package repositories\"\"\"\n return spack.util.lock.Lock(\n os.path.join(spack.paths.user_cache_path, \"package-repository.lock\")\n )\n\n\ndef is_package_module(fullname: str) -> bool:\n \"\"\"Check if the given module is a package module.\"\"\"\n return fullname.startswith(PKG_MODULE_PREFIX_V1) or (\n fullname.startswith(PKG_MODULE_PREFIX_V2) and fullname.endswith(\".package\")\n )\n\n\ndef namespace_from_fullname(fullname: str) -> str:\n \"\"\"Return the repository namespace only for the full module name.\n\n For instance::\n\n namespace_from_fullname(\"spack.pkg.builtin.hdf5\") == \"builtin\"\n namespace_from_fullname(\"spack_repo.x.y.z.packages.pkg_name.package\") == \"x.y.z\"\n\n Args:\n fullname: full name for the Python module\n \"\"\"\n if fullname.startswith(PKG_MODULE_PREFIX_V1):\n namespace, _, _ = fullname.rpartition(\".\")\n return namespace[len(PKG_MODULE_PREFIX_V1) :]\n elif fullname.startswith(PKG_MODULE_PREFIX_V2) and fullname.endswith(\".package\"):\n return \".\".join(fullname.split(\".\")[1:-3])\n return fullname\n\n\ndef name_from_fullname(fullname: str) -> str:\n \"\"\"Return the package name for the full module name.\n\n For instance::\n\n name_from_fullname(\"spack.pkg.builtin.hdf5\") == \"hdf5\"\n name_from_fullname(\"spack_repo.x.y.z.packages.pkg_name.package\") == \"pkg_name\"\n\n Args:\n fullname: full name for the Python module\n \"\"\"\n if fullname.startswith(PKG_MODULE_PREFIX_V1):\n _, _, pkg_module = fullname.rpartition(\".\")\n return pkg_module\n elif fullname.startswith(PKG_MODULE_PREFIX_V2) and fullname.endswith(\".package\"):\n return fullname.rsplit(\".\", 2)[-2]\n return fullname\n\n\nclass _PrependFileLoader(importlib.machinery.SourceFileLoader):\n def __init__(self, fullname: str, repo: \"Repo\", package_name: str) -> None:\n self.repo = repo\n self.package_name = package_name\n path = repo.filename_for_package_name(package_name)\n self.fullname = fullname\n self.prepend = b\"from spack_repo.builtin.build_systems._package_api_v1 import *\\n\"\n super().__init__(self.fullname, path)\n\n def path_stats(self, path):\n stats = dict(super().path_stats(path))\n stats[\"size\"] += len(self.prepend)\n return stats\n\n def get_data(self, path):\n data = super().get_data(path)\n return self.prepend + data if path == self.path else data\n\n\nclass SpackNamespaceLoader:\n def create_module(self, spec):\n return SpackNamespace(spec.name)\n\n def exec_module(self, module):\n module.__loader__ = self\n\n\nclass ReposFinder:\n \"\"\"MetaPathFinder class that loads a Python module corresponding to an API v1 Spack package.\n\n Returns a loader based on the inspection of the current repository list.\n \"\"\"\n\n #: The current list of repositories.\n repo_path: \"RepoPath\"\n\n def find_spec(self, fullname, python_path, target=None):\n # \"target\" is not None only when calling importlib.reload()\n if target is not None:\n raise RuntimeError('cannot reload module \"{0}\"'.format(fullname))\n\n # Preferred API from https://peps.python.org/pep-0451/\n if not fullname.startswith(PKG_MODULE_PREFIX_V1) and fullname != \"spack.pkg\":\n return None\n\n loader = self.compute_loader(fullname)\n if loader is None:\n return None\n return importlib.util.spec_from_loader(fullname, loader)\n\n def compute_loader(self, fullname: str):\n # namespaces are added to repo, and package modules are leaves.\n namespace, dot, module_name = fullname.rpartition(\".\")\n\n # If it's a module in some repo, or if it is the repo's namespace, let the repo handle it.\n\n if not hasattr(self, \"repo_path\"):\n return None\n\n for repo in self.repo_path.repos:\n # We are using the namespace of the repo and the repo contains the package\n if namespace == repo.full_namespace:\n # With 2 nested conditionals we can call \"repo.real_name\" only once\n package_name = repo.real_name(module_name)\n if package_name:\n return _PrependFileLoader(fullname, repo, package_name)\n\n # We are importing a full namespace like 'spack.pkg.builtin'\n if fullname == repo.full_namespace:\n return SpackNamespaceLoader()\n\n # No repo provides the namespace, but it is a valid prefix of\n # something in the RepoPath.\n if self.repo_path.by_namespace.is_prefix(fullname[len(PKG_MODULE_PREFIX_V1) :]):\n return SpackNamespaceLoader()\n\n return None\n\n\n#\n# These names describe how repos should be laid out in the filesystem.\n#\nrepo_config_name = \"repo.yaml\" # Top-level filename for repo config.\nrepo_index_name = \"index.yaml\" # Top-level filename for repository index.\npackages_dir_name = \"packages\" # Top-level repo directory containing pkgs.\npackage_file_name = \"package.py\" # Filename for packages in a repository.\n\n#: Guaranteed unused default value for some functions.\nNOT_PROVIDED = object()\n\n\ndef builtin_repo() -> \"Repo\":\n \"\"\"Get the test repo if it is active, otherwise the builtin repo.\"\"\"\n try:\n return PATH.get_repo(\"builtin_mock\")\n except UnknownNamespaceError:\n return PATH.get_repo(\"builtin\")\n\n\nclass GitExe:\n # Wrapper around Executable for git to set working directory for all\n # invocations.\n #\n # Not using -C as that is not supported for git < 1.8.5.\n def __init__(self, packages_path: str):\n self._git_cmd = spack.util.git.git(required=True)\n self.packages_dir = packages_path\n\n def __call__(self, *args, **kwargs) -> str:\n with working_dir(self.packages_dir):\n return self._git_cmd(*args, **kwargs, output=str)\n\n\ndef list_packages(rev: str, repo: \"Repo\") -> List[str]:\n \"\"\"List all packages associated with the given revision\"\"\"\n git = GitExe(repo.packages_path)\n\n # git ls-tree does not support ... merge-base syntax, so do it manually\n if rev.endswith(\"...\"):\n ref = rev.replace(\"...\", \"\")\n rev = git(\"merge-base\", ref, \"HEAD\").strip()\n\n output = git(\"ls-tree\", \"-r\", \"--name-only\", rev)\n\n # recursively list the packages directory\n package_paths = [\n line.split(os.sep) for line in output.split(\"\\n\") if line.endswith(\"package.py\")\n ]\n\n # take the directory names with one-level-deep package files\n package_names = [\n nm.pkg_dir_to_pkg_name(line[0], repo.package_api)\n for line in package_paths\n if len(line) == 2\n ]\n\n return sorted(set(package_names))\n\n\ndef diff_packages(rev1: str, rev2: str, repo: \"Repo\") -> Tuple[Set[str], Set[str]]:\n \"\"\"Compute packages lists for the two revisions and return a tuple\n containing all the packages in rev1 but not in rev2 and all the\n packages in rev2 but not in rev1.\"\"\"\n p1 = set(list_packages(rev1, repo))\n p2 = set(list_packages(rev2, repo))\n return p1.difference(p2), p2.difference(p1)\n\n\ndef get_all_package_diffs(type: str, repo: \"Repo\", rev1=\"HEAD^1\", rev2=\"HEAD\") -> Set[str]:\n \"\"\"Get packages changed, added, or removed (or any combination of those) since a commit.\n\n Arguments:\n\n type: String containing one or more of ``A``, ``R``, ``C``.\n rev1: Revision to compare against, default is ``\"HEAD^\"``\n rev2: Revision to compare to rev1, default is ``\"HEAD\"``\n \"\"\"\n lower_type = type.lower()\n if not re.match(\"^[arc]*$\", lower_type):\n tty.die(\n f\"Invalid change type: '{type}'. \"\n \"Can contain only A (added), R (removed), or C (changed)\"\n )\n\n removed, added = diff_packages(rev1, rev2, repo)\n\n git = GitExe(repo.packages_path)\n out = git(\"diff\", \"--relative\", \"--name-only\", rev1, rev2).strip()\n\n lines = [] if not out else re.split(r\"\\s+\", out)\n changed: Set[str] = set()\n for path in lines:\n dir_name, _, _ = path.partition(\"/\")\n if not nm.valid_module_name(dir_name, repo.package_api):\n continue\n pkg_name = nm.pkg_dir_to_pkg_name(dir_name, repo.package_api)\n if pkg_name not in added and pkg_name not in removed:\n changed.add(pkg_name)\n\n packages: Set[str] = set()\n if \"a\" in lower_type:\n packages |= added\n if \"r\" in lower_type:\n packages |= removed\n if \"c\" in lower_type:\n packages |= changed\n\n return packages\n\n\ndef add_package_to_git_stage(packages: List[str], repo: \"Repo\") -> None:\n \"\"\"add a package to the git stage with ``git add``\"\"\"\n git = GitExe(repo.packages_path)\n\n for pkg_name in packages:\n filename = PATH.filename_for_package_name(pkg_name)\n if not os.path.isfile(filename):\n tty.die(f\"No such package: {pkg_name}. Path does not exist:\", filename)\n\n git(\"add\", filename)\n\n\ndef autospec(function):\n \"\"\"Decorator that automatically converts the first argument of a\n function to a Spec.\n \"\"\"\n\n @functools.wraps(function)\n def converter(self, spec_like, *args, **kwargs):\n from spack.spec import Spec\n\n if not isinstance(spec_like, Spec):\n spec_like = Spec(spec_like)\n return function(self, spec_like, *args, **kwargs)\n\n return converter\n\n\nclass SpackNamespace(types.ModuleType):\n \"\"\"Allow lazy loading of modules.\"\"\"\n\n def __init__(self, namespace):\n super().__init__(namespace)\n self.__file__ = \"(spack namespace)\"\n self.__path__ = []\n self.__name__ = namespace\n self.__package__ = namespace\n self.__modules = {}\n\n def __getattr__(self, name):\n \"\"\"Getattr lazily loads modules if they're not already loaded.\"\"\"\n submodule = f\"{self.__package__}.{name}\"\n try:\n setattr(self, name, importlib.import_module(submodule))\n except ImportError:\n msg = \"'{0}' object has no attribute {1}\"\n raise AttributeError(msg.format(type(self), name))\n return getattr(self, name)\n\n\n@contextlib.contextmanager\ndef _directory_fd(path: str) -> Generator[Optional[int], None, None]:\n if sys.platform == \"win32\":\n yield None\n return\n\n fd = os.open(path, os.O_RDONLY)\n try:\n yield fd\n finally:\n os.close(fd)\n\n\nclass FastPackageChecker(Mapping[str, float]):\n \"\"\"Cache that maps package names to the modification times of their ``package.py`` files.\n\n For each repository a cache is maintained at class level, and shared among all instances\n referring to it. Update of the global cache is done lazily during instance initialization.\"\"\"\n\n #: Global cache, reused by every instance\n _paths_cache: Dict[str, Dict[str, float]] = {}\n\n def __init__(self, packages_path: str, package_api: Tuple[int, int]) -> None:\n # The path of the repository managed by this instance\n self.packages_path = packages_path\n self.package_api = package_api\n\n # If the cache we need is not there yet, then build it appropriately\n if packages_path not in self._paths_cache:\n self._paths_cache[packages_path] = self._create_new_cache()\n\n #: Reference to the appropriate entry in the global cache\n self._packages_to_mtime = self._paths_cache[packages_path]\n\n def invalidate(self) -> None:\n \"\"\"Regenerate cache for this checker.\"\"\"\n self._paths_cache[self.packages_path] = self._create_new_cache()\n self._packages_to_mtime = self._paths_cache[self.packages_path]\n\n def _create_new_cache(self) -> Dict[str, float]:\n \"\"\"Create a new cache for packages in a repo.\n\n The implementation here should try to minimize filesystem calls. At the moment, it makes\n one stat call per package. This is reasonably fast, and avoids actually importing packages\n in Spack, which is slow.\"\"\"\n # Create a dictionary that will store the mapping between a\n # package name and its mtime\n cache: Dict[str, float] = {}\n # Don't use os.path.join in the loop cause it's slow and redundant.\n package_py_suffix = f\"{os.path.sep}{package_file_name}\"\n\n # Use a file descriptor for the packages directory to avoid repeated path resolution.\n with _directory_fd(self.packages_path) as fd, os.scandir(self.packages_path) as entries:\n for entry in entries:\n # Construct the file name from the directory\n if sys.platform == \"win32\":\n pkg_file = f\"{entry.path}{package_py_suffix}\"\n else:\n pkg_file = f\"{entry.name}{package_py_suffix}\"\n\n try:\n sinfo = os.stat(pkg_file, dir_fd=fd)\n except OSError as e:\n if e.errno in (errno.ENOENT, errno.ENOTDIR):\n # No package.py file here.\n continue\n elif e.errno == errno.EACCES:\n pkg_file = os.path.join(self.packages_path, entry.name, package_file_name)\n tty.warn(f\"Can't read package file {pkg_file}.\")\n continue\n raise\n\n # If it's not a file, skip it.\n if not stat.S_ISREG(sinfo.st_mode):\n continue\n\n # Only consider package.py files in directories that are valid module names under\n # the current package API\n if not nm.valid_module_name(entry.name, self.package_api):\n x, y = self.package_api\n pkg_file = os.path.join(self.packages_path, entry.name, package_file_name)\n tty.warn(\n f\"Package {pkg_file} cannot be used because `{entry.name}` is not a valid \"\n f\"Spack package module name for Package API v{x}.{y}.\"\n )\n continue\n\n # Store the mtime by package name.\n cache[nm.pkg_dir_to_pkg_name(entry.name, self.package_api)] = sinfo.st_mtime\n\n return cache\n\n def last_mtime(self) -> float:\n return max(self._packages_to_mtime.values())\n\n def modified_since(self, since: float) -> List[str]:\n return [name for name, mtime in self._packages_to_mtime.items() if mtime > since]\n\n def __getitem__(self, item: str) -> float:\n return self._packages_to_mtime[item]\n\n def __iter__(self) -> Iterator[str]:\n return iter(self._packages_to_mtime)\n\n def __len__(self) -> int:\n return len(self._packages_to_mtime)\n\n\nclass Indexer(metaclass=abc.ABCMeta):\n \"\"\"Adaptor for indexes that need to be generated when repos are updated.\"\"\"\n\n def __init__(self, repository):\n self.repository = repository\n self.index = None\n\n def create(self):\n self.index = self._create()\n\n @abc.abstractmethod\n def _create(self):\n \"\"\"Create an empty index and return it.\"\"\"\n\n def needs_update(self, pkg) -> bool:\n \"\"\"Whether an update is needed when the package file hasn't changed.\n\n Returns:\n ``True`` iff this package needs its index updated.\n\n We already automatically update indexes when package files\n change, but other files (like patches) may change underneath the\n package file. This method can be used to check additional\n package-specific files whenever they're loaded, to tell the\n RepoIndex to update the index *just* for that package.\n\n \"\"\"\n return False\n\n @abc.abstractmethod\n def read(self, stream):\n \"\"\"Read this index from a provided file object.\"\"\"\n\n @abc.abstractmethod\n def update(self, pkgs_fullname: Set[str]):\n \"\"\"Update the index in memory with information about a package.\"\"\"\n\n @abc.abstractmethod\n def write(self, stream):\n \"\"\"Write the index to a file object.\"\"\"\n\n\nclass TagIndexer(Indexer):\n \"\"\"Lifecycle methods for a TagIndex on a Repo.\"\"\"\n\n def _create(self) -> spack.tag.TagIndex:\n return spack.tag.TagIndex()\n\n def read(self, stream):\n self.index = spack.tag.TagIndex.from_json(stream)\n\n def update(self, pkgs_fullname: Set[str]):\n self.index.update_packages({p.split(\".\")[-1] for p in pkgs_fullname}, self.repository)\n\n def write(self, stream):\n self.index.to_json(stream)\n\n\nclass ProviderIndexer(Indexer):\n \"\"\"Lifecycle methods for virtual package providers.\"\"\"\n\n def _create(self) -> \"spack.provider_index.ProviderIndex\":\n return spack.provider_index.ProviderIndex(repository=self.repository)\n\n def read(self, stream):\n self.index = spack.provider_index.ProviderIndex.from_json(stream, self.repository)\n\n def update(self, pkgs_fullname: Set[str]):\n is_virtual = lambda name: (\n not self.repository.exists(name) or self.repository.get_pkg_class(name).virtual\n )\n non_virtual_pkgs_fullname = {p for p in pkgs_fullname if not is_virtual(p.split(\".\")[-1])}\n non_virtual_pkgs_names = {p.split(\".\")[-1] for p in non_virtual_pkgs_fullname}\n self.index.remove_providers(non_virtual_pkgs_names)\n self.index.update_packages(non_virtual_pkgs_fullname)\n\n def write(self, stream):\n self.index.to_json(stream)\n\n\nclass PatchIndexer(Indexer):\n \"\"\"Lifecycle methods for patch cache.\"\"\"\n\n def _create(self) -> spack.patch.PatchCache:\n return spack.patch.PatchCache(repository=self.repository)\n\n def needs_update(self):\n # TODO: patches can change under a package and we should handle\n # TODO: it, but we currently punt. This should be refactored to\n # TODO: check whether patches changed each time a package loads,\n # TODO: tell the RepoIndex to reindex them.\n return False\n\n def read(self, stream):\n self.index = spack.patch.PatchCache.from_json(stream, repository=self.repository)\n\n def write(self, stream):\n self.index.to_json(stream)\n\n def update(self, pkgs_fullname: Set[str]):\n self.index.update_packages(pkgs_fullname)\n\n\nclass RepoIndex:\n \"\"\"Container class that manages a set of Indexers for a Repo.\n\n This class is responsible for checking packages in a repository for\n updates (using ``FastPackageChecker``) and for regenerating indexes\n when they're needed.\n\n ``Indexers`` should be added to the ``RepoIndex`` using\n ``add_indexer(name, indexer)``, and they should support the interface\n defined by ``Indexer``, so that the ``RepoIndex`` can read, generate,\n and update stored indices.\n\n Generated indexes are accessed by name via ``__getitem__()``.\"\"\"\n\n def __init__(\n self,\n packages_path: str,\n package_checker: \"Callable[[], FastPackageChecker]\",\n namespace: str,\n cache: spack.util.file_cache.FileCache,\n ):\n self._get_checker = package_checker\n self._checker: Optional[FastPackageChecker] = None\n self.packages_path = packages_path\n if sys.platform == \"win32\":\n self.packages_path = spack.llnl.path.convert_to_posix_path(self.packages_path)\n self.namespace = namespace\n\n self.indexers: Dict[str, Indexer] = {}\n self.indexes: Dict[str, Any] = {}\n self.cache = cache\n\n #: Whether the indexes are up to date with the package repository.\n self.is_fresh = False\n\n @property\n def checker(self) -> FastPackageChecker:\n if self._checker is None:\n self._checker = self._get_checker()\n return self._checker\n\n def add_indexer(self, name: str, indexer: Indexer):\n \"\"\"Add an indexer to the repo index.\n\n Arguments:\n name: name of this indexer\n indexer: object implementing the ``Indexer`` interface\"\"\"\n self.indexers[name] = indexer\n\n def __getitem__(self, name):\n \"\"\"Get an up-to-date index with the specified name.\"\"\"\n return self.get_index(name, allow_stale=False)\n\n def get_index(self, name, allow_stale: bool = False):\n \"\"\"Get the index with the specified name. The index will be updated if it is stale, unless\n allow_stale is True, in which case its contents are not validated against the package\n repository. When no cache is available, the index will be updated regardless of the value\n of allow_stale.\"\"\"\n indexer = self.indexers.get(name)\n if not indexer:\n raise KeyError(\"no such index: %s\" % name)\n if name not in self.indexes or (not allow_stale and not self.is_fresh):\n self._build_all_indexes(allow_stale=allow_stale)\n return self.indexes[name]\n\n def _build_all_indexes(self, allow_stale: bool = False) -> None:\n \"\"\"Build all the indexes at once.\n\n We regenerate *all* indexes whenever *any* index needs an update,\n because the main bottleneck here is loading all the packages. It\n can take tens of seconds to regenerate sequentially, and we'd\n rather only pay that cost once rather than on several\n invocations.\"\"\"\n is_fresh = True\n for name, indexer in self.indexers.items():\n is_fresh &= self._update_index(name, indexer, allow_stale=allow_stale)\n self.is_fresh = is_fresh\n\n def _update_index(self, name: str, indexer: Indexer, allow_stale: bool = False) -> bool:\n \"\"\"Determine which packages need an update, and update indexes. Returns true if the\n index is fresh.\"\"\"\n\n # Filename of the provider index cache (we assume they're all json)\n from spack.spec import SPECFILE_FORMAT_VERSION\n\n cache_filename = f\"{name}/{self.namespace}-specfile_v{SPECFILE_FORMAT_VERSION}-index.json\"\n\n with self.cache.read_transaction(cache_filename) as f:\n # Get the mtime of the cache if it exists, of -inf.\n index_mtime = os.fstat(f.fileno()).st_mtime if f is not None else -math.inf\n\n if f is not None and allow_stale:\n # Cache exists and caller accepts stale data: skip the expensive modified_since.\n indexer.read(f)\n self.indexes[name] = indexer.index\n return False\n\n needs_update = self.checker.modified_since(index_mtime)\n\n if f is not None and not needs_update:\n # Cache exists and is up to date.\n indexer.read(f)\n self.indexes[name] = indexer.index\n return True\n\n # Cache is missing or stale: acquire write lock and rebuild.\n with self.cache.write_transaction(cache_filename) as (old, new):\n old_mtime = os.fstat(old.fileno()).st_mtime if old is not None else -math.inf\n # Re-check in case another writer updated the index while we waited for the lock.\n if old_mtime != index_mtime:\n needs_update = self.checker.modified_since(old_mtime)\n indexer.read(old) if old is not None else indexer.create()\n indexer.update({f\"{self.namespace}.{pkg_name}\" for pkg_name in needs_update})\n indexer.write(new)\n\n self.indexes[name] = indexer.index\n return True\n\n\nclass RepoPath:\n \"\"\"A RepoPath is a list of Repo instances that function as one.\n\n It functions exactly like a Repo, but it operates on the combined\n results of the Repos in its list instead of on a single package\n repository.\n \"\"\"\n\n def __init__(self, *repos: \"Repo\") -> None:\n self.repos: List[Repo] = []\n self.by_namespace = nm.NamespaceTrie()\n self._provider_index: Optional[spack.provider_index.ProviderIndex] = None\n self._patch_index: Optional[spack.patch.PatchCache] = None\n self._index_is_fresh: bool = False\n self._tag_index: Optional[spack.tag.TagIndex] = None\n\n for repo in repos:\n self.put_last(repo)\n\n @staticmethod\n def from_descriptors(\n descriptors: \"RepoDescriptors\",\n cache: spack.util.file_cache.FileCache,\n overrides: Optional[Dict[str, Any]] = None,\n ) -> \"RepoPath\":\n repo_path, errors = descriptors.construct(cache=cache, fetch=True, overrides=overrides)\n\n # Merely warn if package repositories from config could not be constructed.\n if errors:\n for path, error in errors.items():\n tty.warn(f\"Error constructing repository '{path}': {error}\")\n\n return repo_path\n\n @staticmethod\n def from_config(config: spack.config.Configuration) -> \"RepoPath\":\n \"\"\"Create a RepoPath from a configuration object.\"\"\"\n overrides = {\n pkg_name: data[\"package_attributes\"]\n for pkg_name, data in config.get_config(\"packages\").items()\n if pkg_name != \"all\" and \"package_attributes\" in data\n }\n\n return RepoPath.from_descriptors(\n descriptors=RepoDescriptors.from_config(lock=package_repository_lock(), config=config),\n cache=spack.caches.MISC_CACHE,\n overrides=overrides,\n )\n\n def enable(self) -> None:\n \"\"\"Set the relevant search paths for package module loading\"\"\"\n REPOS_FINDER.repo_path = self\n for p in reversed(self.python_paths()):\n if p not in sys.path:\n sys.path.insert(0, p)\n\n def disable(self) -> None:\n \"\"\"Disable the search paths for package module loading\"\"\"\n if hasattr(REPOS_FINDER, \"repo_path\"):\n del REPOS_FINDER.repo_path\n for p in self.python_paths():\n if p in sys.path:\n sys.path.remove(p)\n\n def ensure_unwrapped(self) -> \"RepoPath\":\n \"\"\"Ensure we unwrap this object from any dynamic wrapper (like Singleton)\"\"\"\n return self\n\n def put_first(self, repo: Union[\"Repo\", \"RepoPath\"]) -> None:\n \"\"\"Add repo first in the search path.\"\"\"\n if isinstance(repo, RepoPath):\n for r in reversed(repo.repos):\n self.put_first(r)\n return\n\n self.repos.insert(0, repo)\n self.by_namespace[repo.namespace] = repo\n\n def put_last(self, repo):\n \"\"\"Add repo last in the search path.\"\"\"\n if isinstance(repo, RepoPath):\n for r in repo.repos:\n self.put_last(r)\n return\n\n self.repos.append(repo)\n\n # don't mask any higher-precedence repos with same namespace\n if repo.namespace not in self.by_namespace:\n self.by_namespace[repo.namespace] = repo\n\n def remove(self, repo):\n \"\"\"Remove a repo from the search path.\"\"\"\n if repo in self.repos:\n self.repos.remove(repo)\n\n def get_repo(self, namespace: str) -> \"Repo\":\n \"\"\"Get a repository by namespace.\"\"\"\n if namespace not in self.by_namespace:\n raise UnknownNamespaceError(namespace)\n return self.by_namespace[namespace]\n\n def first_repo(self) -> Optional[\"Repo\"]:\n \"\"\"Get the first repo in precedence order.\"\"\"\n return self.repos[0] if self.repos else None\n\n @memoized\n def _all_package_names_set(self, include_virtuals) -> Set[str]:\n return {name for repo in self.repos for name in repo.all_package_names(include_virtuals)}\n\n @memoized\n def _all_package_names(self, include_virtuals: bool) -> List[str]:\n \"\"\"Return all unique package names in all repositories.\"\"\"\n return sorted(self._all_package_names_set(include_virtuals), key=lambda n: n.lower())\n\n def all_package_names(self, include_virtuals: bool = False) -> List[str]:\n return self._all_package_names(include_virtuals)\n\n def package_path(self, name: str) -> str:\n \"\"\"Get path to package.py file for this repo.\"\"\"\n return self.repo_for_pkg(name).package_path(name)\n\n def all_package_paths(self) -> Generator[str, None, None]:\n for name in self.all_package_names():\n yield self.package_path(name)\n\n def packages_with_tags(self, *tags: str, full: bool = False) -> Set[str]:\n \"\"\"Returns a set of packages matching any of the tags in input.\n\n Args:\n full: if True the package names in the output are fully-qualified\n \"\"\"\n return {\n f\"{repo.namespace}.{pkg}\" if full else pkg\n for repo in self.repos\n for pkg in repo.packages_with_tags(*tags)\n }\n\n def all_package_classes(self) -> Generator[Type[\"spack.package_base.PackageBase\"], None, None]:\n for name in self.all_package_names():\n yield self.get_pkg_class(name)\n\n @property\n def provider_index(self) -> spack.provider_index.ProviderIndex:\n \"\"\"Merged ProviderIndex from all Repos in the RepoPath.\"\"\"\n if self._provider_index is None:\n self._provider_index = spack.provider_index.ProviderIndex(repository=self)\n for repo in reversed(self.repos):\n self._provider_index.merge(repo.provider_index)\n return self._provider_index\n\n @property\n def tag_index(self) -> spack.tag.TagIndex:\n \"\"\"Merged TagIndex from all Repos in the RepoPath.\"\"\"\n if self._tag_index is None:\n self._tag_index = spack.tag.TagIndex()\n for repo in reversed(self.repos):\n self._tag_index.merge(repo.tag_index)\n return self._tag_index\n\n def get_patch_index(self, allow_stale: bool = False) -> spack.patch.PatchCache:\n \"\"\"Return the merged patch index for all repos in this path.\n\n Args:\n allow_stale: if True, return a possibly out-of-date index from cache files,\n avoiding filesystem calls to check whether the index is up to date.\n \"\"\"\n if self._patch_index is not None and (self._index_is_fresh or allow_stale):\n return self._patch_index\n\n index = spack.patch.PatchCache(repository=self)\n for repo in reversed(self.repos):\n index.update(repo.get_patch_index(allow_stale=allow_stale))\n self._patch_index = index\n self._index_is_fresh = not allow_stale\n return self._patch_index\n\n def get_patches_for_package(\n self, sha256s: List[str], pkg_cls: Type[\"spack.package_base.PackageBase\"]\n ) -> List[\"spack.patch.Patch\"]:\n \"\"\"Look up patches by sha256, trying stale cache first to avoid stat calls.\n\n Args:\n sha256s: ordered list of patch sha256 hashes\n pkg_cls: package class the patches belong to\n\n Returns:\n List of Patch objects in the same order as sha256s.\n\n Raises:\n spack.error.PatchLookupError: if a sha256 cannot be found even after a full rebuild.\n \"\"\"\n stale_index = self.get_patch_index(allow_stale=True)\n try:\n return [\n stale_index.patch_for_package(sha256, pkg_cls, validate=True) for sha256 in sha256s\n ]\n except spack.error.PatchLookupError:\n pass\n\n current_index = self.get_patch_index(allow_stale=False)\n return [current_index.patch_for_package(sha256, pkg_cls) for sha256 in sha256s]\n\n def providers_for(self, virtual: Union[str, \"spack.spec.Spec\"]) -> List[\"spack.spec.Spec\"]:\n all_packages = self._all_package_names_set(include_virtuals=False)\n providers = [\n spec\n for spec in self.provider_index.providers_for(virtual)\n if spec.name in all_packages\n ]\n if not providers:\n raise UnknownPackageError(virtual if isinstance(virtual, str) else virtual.fullname)\n return providers\n\n @autospec\n def extensions_for(\n self, extendee_spec: \"spack.spec.Spec\"\n ) -> List[\"spack.package_base.PackageBase\"]:\n from spack.spec import Spec\n\n return [\n pkg_cls(Spec(pkg_cls.name))\n for pkg_cls in self.all_package_classes()\n if pkg_cls(Spec(pkg_cls.name)).extends(extendee_spec)\n ]\n\n def last_mtime(self):\n \"\"\"Time a package file in this repo was last updated.\"\"\"\n return max(repo.last_mtime() for repo in self.repos)\n\n def repo_for_pkg(self, spec: Union[str, \"spack.spec.Spec\"]) -> \"Repo\":\n \"\"\"Given a spec, get the repository for its package.\"\"\"\n # We don't @_autospec this function b/c it's called very frequently\n # and we want to avoid parsing str's into Specs unnecessarily.\n from spack.spec import Spec\n\n if isinstance(spec, Spec):\n namespace = spec.namespace\n name = spec.name\n else:\n # handle strings directly for speed instead of @_autospec'ing\n namespace, _, name = spec.rpartition(\".\")\n\n # If the spec already has a namespace, then return the\n # corresponding repo if we know about it.\n if namespace:\n if namespace not in self.by_namespace:\n raise UnknownNamespaceError(namespace, name=name)\n return self.by_namespace[namespace]\n\n # If there's no namespace, search in the RepoPath.\n for repo in self.repos:\n if name in repo:\n return repo\n\n # If the package isn't in any repo, return the one with\n # highest precedence. This is for commands like `spack edit`\n # that can operate on packages that don't exist yet.\n selected = self.first_repo()\n if selected is None:\n raise UnknownPackageError(name)\n return selected\n\n def get(self, spec: \"spack.spec.Spec\") -> \"spack.package_base.PackageBase\":\n \"\"\"Returns the package associated with the supplied spec.\"\"\"\n from spack.spec import Spec\n\n msg = \"RepoPath.get can only be called on concrete specs\"\n assert isinstance(spec, Spec) and spec.concrete, msg\n return self.repo_for_pkg(spec).get(spec)\n\n def python_paths(self) -> List[str]:\n \"\"\"Return a list of all the Python paths in the repos.\"\"\"\n return [repo.python_path for repo in self.repos if repo.python_path]\n\n def get_pkg_class(self, pkg_name: str) -> Type[\"spack.package_base.PackageBase\"]:\n \"\"\"Find a class for the spec's package and return the class object.\"\"\"\n return self.repo_for_pkg(pkg_name).get_pkg_class(pkg_name)\n\n @autospec\n def dump_provenance(self, spec, path):\n \"\"\"Dump provenance information for a spec to a particular path.\n\n This dumps the package file and any associated patch files.\n Raises UnknownPackageError if not found.\n \"\"\"\n return self.repo_for_pkg(spec).dump_provenance(spec, path)\n\n def dirname_for_package_name(self, pkg_name: str) -> str:\n return self.repo_for_pkg(pkg_name).dirname_for_package_name(pkg_name)\n\n def filename_for_package_name(self, pkg_name: str) -> str:\n return self.repo_for_pkg(pkg_name).filename_for_package_name(pkg_name)\n\n def exists(self, pkg_name: str) -> bool:\n \"\"\"Whether package with the give name exists in the path's repos.\n\n Note that virtual packages do not \"exist\".\n \"\"\"\n return any(repo.exists(pkg_name) for repo in self.repos)\n\n def is_virtual(self, pkg_name: str) -> bool:\n \"\"\"Return True if the package with this name is virtual, False otherwise.\n\n This function use the provider index. If calling from a code block that\n is used to construct the provider index use the ``is_virtual_safe`` function.\n\n Args:\n pkg_name: name of the package we want to check\n \"\"\"\n have_name = bool(pkg_name)\n return have_name and pkg_name in self.provider_index\n\n def is_virtual_safe(self, pkg_name: str) -> bool:\n \"\"\"Return True if the package with this name is virtual, False otherwise.\n\n This function doesn't use the provider index.\n\n Args:\n pkg_name: name of the package we want to check\n \"\"\"\n have_name = bool(pkg_name)\n return have_name and (not self.exists(pkg_name) or self.get_pkg_class(pkg_name).virtual)\n\n def __contains__(self, pkg_name):\n return self.exists(pkg_name)\n\n def marshal(self):\n return (self.repos,)\n\n @staticmethod\n def unmarshal(repos):\n return RepoPath(*repos)\n\n def __reduce__(self):\n return RepoPath.unmarshal, self.marshal()\n\n\ndef _parse_package_api_version(\n config: Dict[str, Any],\n min_api: Tuple[int, int] = spack.min_package_api_version,\n max_api: Tuple[int, int] = spack.package_api_version,\n) -> Tuple[int, int]:\n api = config.get(\"api\")\n if api is None:\n package_api = (1, 0)\n else:\n if not isinstance(api, str):\n raise BadRepoError(f\"Invalid Package API version '{api}'. Must be of the form vX.Y\")\n api_match = _API_REGEX.match(api)\n if api_match is None:\n raise BadRepoError(f\"Invalid Package API version '{api}'. Must be of the form vX.Y\")\n package_api = (int(api_match.group(1)), int(api_match.group(2)))\n\n if min_api <= package_api <= max_api:\n return package_api\n\n min_str = \".\".join(str(i) for i in min_api)\n max_str = \".\".join(str(i) for i in max_api)\n curr_str = \".\".join(str(i) for i in package_api)\n raise BadRepoVersionError(\n api,\n f\"Package API v{curr_str} is not supported by this version of Spack (\"\n f\"must be between v{min_str} and v{max_str})\",\n )\n\n\ndef _validate_and_normalize_subdir(subdir: Any, root: str, package_api: Tuple[int, int]) -> str:\n if not isinstance(subdir, str):\n raise BadRepoError(f\"Invalid subdirectory '{subdir}' in '{root}'. Must be a string\")\n\n if package_api < (2, 0):\n return subdir # In v1.x we did not validate subdir names\n\n if subdir in (\".\", \"\"):\n raise BadRepoError(\n f\"Invalid subdirectory '{subdir}' in '{root}'. Use a symlink packages -> . instead\"\n )\n\n # Otherwise we expect a directory name (not path) that can be used as a Python module.\n if os.sep in subdir:\n raise BadRepoError(\n f\"Invalid subdirectory '{subdir}' in '{root}'. Expected a directory name, not a path\"\n )\n if not nm.valid_module_name(subdir, package_api):\n raise BadRepoError(\n f\"Invalid subdirectory '{subdir}' in '{root}'. Must be a valid Python module name\"\n )\n return subdir\n\n\nclass Repo:\n \"\"\"Class representing a package repository in the filesystem.\n\n Each package repository must have a top-level configuration file called ``repo.yaml``.\n\n It contains the following keys:\n\n ``namespace``\n A Python namespace where the repository's packages should live.\n\n ``subdirectory``\n An optional subdirectory name where packages are placed\n\n ``api``\n A string of the form vX.Y that indicates the Package API version. The default is ``v1.0``.\n For the repo to be compatible with the current version of Spack, the version must be\n greater than or equal to :py:data:`spack.min_package_api_version` and less than or equal to\n :py:data:`spack.package_api_version`.\n \"\"\"\n\n namespace: str\n\n def __init__(\n self,\n root: str,\n *,\n cache: spack.util.file_cache.FileCache,\n overrides: Optional[Dict[str, Any]] = None,\n ) -> None:\n \"\"\"Instantiate a package repository from a filesystem path.\n\n Args:\n root: the root directory of the repository\n cache: file cache associated with this repository\n overrides: dict mapping package name to class attribute overrides for that package\n \"\"\"\n # Root directory, containing _repo.yaml and package dirs\n # Allow roots to by spack-relative by starting with '$spack'\n self.root = spack.util.path.canonicalize_path(root)\n\n # check and raise BadRepoError on fail.\n def check(condition, msg):\n if not condition:\n raise BadRepoError(msg)\n\n # Validate repository layout.\n self.config_file = os.path.join(self.root, repo_config_name)\n check(os.path.isfile(self.config_file), f\"No {repo_config_name} found in '{root}'\")\n\n # Read configuration and validate namespace\n config = self._read_config()\n\n self.package_api = _parse_package_api_version(config)\n self.subdirectory = _validate_and_normalize_subdir(\n config.get(\"subdirectory\", packages_dir_name), root, self.package_api\n )\n self.packages_path = os.path.join(self.root, self.subdirectory)\n self.build_systems_path = os.path.join(self.root, \"build_systems\")\n\n check(\n os.path.isdir(self.packages_path),\n f\"No directory '{self.subdirectory}' found in '{root}'\",\n )\n\n # The parent dir of spack_repo/ which should be added to sys.path for api v2.x\n self.python_path: Optional[str] = None\n\n if self.package_api < (2, 0):\n check(\n \"namespace\" in config,\n f\"{os.path.join(root, repo_config_name)} must define a namespace.\",\n )\n self.namespace = config[\"namespace\"]\n # Note: for Package API v1.x the namespace validation always had bugs, which won't be\n # fixed for compatibility reasons. The regex is missing \"$\" at the end, and it claims\n # to test for valid identifiers, but fails to split on `.` first.\n check(\n isinstance(self.namespace, str)\n and re.match(r\"[a-zA-Z][a-zA-Z0-9_.]+\", self.namespace),\n f\"Invalid namespace '{self.namespace}' in repo '{self.root}'. \"\n \"Namespaces must be valid python identifiers separated by '.'\",\n )\n else:\n # From Package API v2.0 the namespace follows from the directory structure.\n check(\n f\"{os.sep}spack_repo{os.sep}\" in self.root,\n f\"Invalid repository path '{self.root}'. Path must contain 'spack_repo{os.sep}'\",\n )\n derived_namespace = self.root.rpartition(f\"spack_repo{os.sep}\")[2].replace(os.sep, \".\")\n if \"namespace\" in config:\n self.namespace = config[\"namespace\"]\n\n check(\n isinstance(self.namespace, str) and self.namespace == derived_namespace,\n f\"Namespace '{self.namespace}' should be {derived_namespace} or omitted in \"\n f\"{os.path.join(root, repo_config_name)}\",\n )\n else:\n self.namespace = derived_namespace\n\n # strip the namespace directories from the root path to get the python path\n # e.g. /my/pythonpath/spack_repo/x/y/z -> /my/pythonpath\n python_path = self.root\n for _ in self.namespace.split(\".\"):\n python_path = os.path.dirname(python_path)\n self.python_path = os.path.dirname(python_path)\n\n # check that all subdirectories are valid module names\n check(\n all(nm.valid_module_name(x, self.package_api) for x in self.namespace.split(\".\")),\n f\"Invalid namespace '{self.namespace}' in repo '{self.root}'\",\n )\n\n # Set up 'full_namespace' to include the super-namespace\n if self.package_api < (2, 0):\n self.full_namespace = f\"{PKG_MODULE_PREFIX_V1}{self.namespace}\"\n elif self.subdirectory == \".\":\n self.full_namespace = f\"{PKG_MODULE_PREFIX_V2}{self.namespace}\"\n else:\n self.full_namespace = f\"{PKG_MODULE_PREFIX_V2}{self.namespace}.{self.subdirectory}\"\n\n # Keep name components around for checking prefixes.\n self._names = self.full_namespace.split(\".\")\n\n # Class attribute overrides by package name\n self.overrides = overrides or {}\n\n # Maps that goes from package name to corresponding file stat\n self._fast_package_checker: Optional[FastPackageChecker] = None\n\n # Indexes for this repository, computed lazily\n self._repo_index: Optional[RepoIndex] = None\n self._cache = cache\n\n @property\n def package_api_str(self) -> str:\n return f\"v{self.package_api[0]}.{self.package_api[1]}\"\n\n def real_name(self, import_name: str) -> Optional[str]:\n \"\"\"Allow users to import Spack packages using Python identifiers.\n\n In Package API v1.x, there was no canonical module name for a package, and package's dir\n was not necessarily a valid Python module name. For that case we have to guess the actual\n package directory. From Package API v2.0 there is a one-to-one mapping between Spack\n package names and Python module names, so there is no guessing.\n\n For Package API v1.x we support the following one-to-many mappings:\n\n * ``num3proxy`` -> ``3proxy``\n * ``foo_bar`` -> ``foo_bar``, ``foo-bar``\n * ``foo_bar_baz`` -> ``foo_bar_baz``, ``foo-bar-baz``, ``foo_bar-baz``, ``foo-bar_baz``\n \"\"\"\n if self.package_api >= (2, 0):\n if nm.pkg_dir_to_pkg_name(import_name, package_api=self.package_api) in self:\n return import_name\n return None\n\n if import_name in self:\n return import_name\n\n # For v1 generate the possible package names from a module name, and return the first\n # package name that exists in this repo.\n options = nm.possible_spack_module_names(import_name)\n try:\n options.remove(import_name)\n except ValueError:\n pass\n for name in options:\n if name in self:\n return name\n return None\n\n def is_prefix(self, fullname: str) -> bool:\n \"\"\"True if fullname is a prefix of this Repo's namespace.\"\"\"\n parts = fullname.split(\".\")\n return self._names[: len(parts)] == parts\n\n def _read_config(self) -> Dict[str, Any]:\n \"\"\"Check for a YAML config file in this db's root directory.\"\"\"\n try:\n with open(self.config_file, encoding=\"utf-8\") as reponame_file:\n yaml_data = syaml.load(reponame_file)\n\n if (\n not yaml_data\n or \"repo\" not in yaml_data\n or not isinstance(yaml_data[\"repo\"], dict)\n ):\n tty.die(f\"Invalid {repo_config_name} in repository {self.root}\")\n\n return yaml_data[\"repo\"]\n\n except OSError:\n tty.die(f\"Error reading {self.config_file} when opening {self.root}\")\n\n def get(self, spec: \"spack.spec.Spec\") -> \"spack.package_base.PackageBase\":\n \"\"\"Returns the package associated with the supplied spec.\"\"\"\n from spack.spec import Spec\n\n msg = \"Repo.get can only be called on concrete specs\"\n assert isinstance(spec, Spec) and spec.concrete, msg\n # NOTE: we only check whether the package is None here, not whether it\n # actually exists, because we have to load it anyway, and that ends up\n # checking for existence. We avoid constructing FastPackageChecker,\n # which will stat all packages.\n if not spec.name:\n raise UnknownPackageError(None, self)\n\n if spec.namespace and spec.namespace != self.namespace:\n raise UnknownPackageError(spec.name, self.namespace)\n\n package_class = self.get_pkg_class(spec.name)\n try:\n return package_class(spec)\n except spack.error.SpackError:\n # pass these through as their error messages will be fine.\n raise\n except Exception as e:\n # Make sure other errors in constructors hit the error\n # handler by wrapping them\n tty.debug(e)\n raise FailedConstructorError(spec.fullname, *sys.exc_info()) from e\n\n @autospec\n def dump_provenance(self, spec: \"spack.spec.Spec\", path: str) -> None:\n \"\"\"Dump provenance information for a spec to a particular path.\n\n This dumps the package file and any associated patch files.\n Raises UnknownPackageError if not found.\n \"\"\"\n if spec.namespace and spec.namespace != self.namespace:\n raise UnknownPackageError(\n f\"Repository {self.namespace} does not contain package {spec.fullname}.\"\n )\n\n package_path = self.filename_for_package_name(spec.name)\n if not os.path.exists(package_path):\n # Spec has no files (e.g., package, patches) to copy\n tty.debug(f\"{spec.name} does not have a package to dump\")\n return\n\n # Install patch files needed by the (concrete) package.\n fs.mkdirp(path)\n if spec.concrete:\n for patch in itertools.chain.from_iterable(spec.package.patches.values()):\n if patch.path:\n if os.path.exists(patch.path):\n fs.install(patch.path, path)\n else:\n warnings.warn(f\"Patch file did not exist: {patch.path}\")\n\n # Install the package.py file itself.\n fs.install(self.filename_for_package_name(spec.name), path)\n\n @property\n def index(self) -> RepoIndex:\n \"\"\"Construct the index for this repo lazily.\"\"\"\n if self._repo_index is None:\n self._repo_index = RepoIndex(\n self.packages_path, lambda: self._pkg_checker, self.namespace, cache=self._cache\n )\n self._repo_index.add_indexer(\"providers\", ProviderIndexer(self))\n self._repo_index.add_indexer(\"tags\", TagIndexer(self))\n self._repo_index.add_indexer(\"patches\", PatchIndexer(self))\n return self._repo_index\n\n @property\n def provider_index(self) -> spack.provider_index.ProviderIndex:\n \"\"\"A fresh provider index with names *specific* to this repo.\"\"\"\n return self.index[\"providers\"]\n\n @property\n def tag_index(self) -> spack.tag.TagIndex:\n \"\"\"Fresh index of tags and which packages they're defined on.\"\"\"\n return self.index[\"tags\"]\n\n def get_patch_index(self, allow_stale: bool = False) -> spack.patch.PatchCache:\n \"\"\"Index of patches and packages they're defined on. Set allow_stale is True to bypass\n cache validation and return a potentially stale index.\"\"\"\n return self.index.get_index(\"patches\", allow_stale=allow_stale)\n\n def providers_for(self, virtual: Union[str, \"spack.spec.Spec\"]) -> List[\"spack.spec.Spec\"]:\n providers = self.provider_index.providers_for(virtual)\n if not providers:\n raise UnknownPackageError(virtual if isinstance(virtual, str) else virtual.fullname)\n return providers\n\n @autospec\n def extensions_for(\n self, extendee_spec: \"spack.spec.Spec\"\n ) -> List[\"spack.package_base.PackageBase\"]:\n from spack.spec import Spec\n\n result = [pkg_cls(Spec(pkg_cls.name)) for pkg_cls in self.all_package_classes()]\n return [x for x in result if x.extends(extendee_spec)]\n\n def dirname_for_package_name(self, pkg_name: str) -> str:\n \"\"\"Given a package name, get the directory containing its package.py file.\"\"\"\n _, unqualified_name = self.partition_package_name(pkg_name)\n return os.path.join(\n self.packages_path, nm.pkg_name_to_pkg_dir(unqualified_name, self.package_api)\n )\n\n def filename_for_package_name(self, pkg_name: str) -> str:\n \"\"\"Get the filename for the module we should load for a particular\n package. Packages for a Repo live in\n ``$root//package.py``\n\n This will return a proper package.py path even if the\n package doesn't exist yet, so callers will need to ensure\n the package exists before importing.\n \"\"\"\n pkg_dir = self.dirname_for_package_name(pkg_name)\n return os.path.join(pkg_dir, package_file_name)\n\n @property\n def _pkg_checker(self) -> FastPackageChecker:\n if self._fast_package_checker is None:\n self._fast_package_checker = FastPackageChecker(self.packages_path, self.package_api)\n return self._fast_package_checker\n\n def all_package_names(self, include_virtuals: bool = False) -> List[str]:\n \"\"\"Returns a sorted list of all package names in the Repo.\"\"\"\n names = sorted(self._pkg_checker.keys())\n if include_virtuals:\n return names\n return [x for x in names if not self.is_virtual(x)]\n\n def package_path(self, name: str) -> str:\n \"\"\"Get path to package.py file for this repo.\"\"\"\n return os.path.join(\n self.packages_path, nm.pkg_name_to_pkg_dir(name, self.package_api), package_file_name\n )\n\n def all_package_paths(self) -> Generator[str, None, None]:\n for name in self.all_package_names():\n yield self.package_path(name)\n\n def packages_with_tags(self, *tags: str) -> Set[str]:\n v = set(self.all_package_names())\n for tag in tags:\n v.intersection_update(self.tag_index.get_packages(tag.lower()))\n return v\n\n def all_package_classes(self) -> Generator[Type[\"spack.package_base.PackageBase\"], None, None]:\n \"\"\"Iterator over all package *classes* in the repository.\n\n Use this with care, because loading packages is slow.\n \"\"\"\n for name in self.all_package_names():\n yield self.get_pkg_class(name)\n\n def exists(self, pkg_name: str) -> bool:\n \"\"\"Whether a package with the supplied name exists.\"\"\"\n if pkg_name is None:\n return False\n\n # if the FastPackageChecker is already constructed, use it\n if self._fast_package_checker:\n return pkg_name in self._pkg_checker\n\n # if not, check for the package.py file\n path = self.filename_for_package_name(pkg_name)\n return os.path.exists(path)\n\n def last_mtime(self):\n \"\"\"Time a package file in this repo was last updated.\"\"\"\n return self._pkg_checker.last_mtime()\n\n def is_virtual(self, pkg_name: str) -> bool:\n \"\"\"Return True if the package with this name is virtual, False otherwise.\n\n This function use the provider index. If calling from a code block that\n is used to construct the provider index use the ``is_virtual_safe`` function.\n \"\"\"\n return pkg_name in self.provider_index\n\n def is_virtual_safe(self, pkg_name: str) -> bool:\n \"\"\"Return True if the package with this name is virtual, False otherwise.\n\n This function doesn't use the provider index.\n \"\"\"\n return not self.exists(pkg_name) or self.get_pkg_class(pkg_name).virtual\n\n def get_pkg_class(self, pkg_name: str) -> Type[\"spack.package_base.PackageBase\"]:\n \"\"\"Get the class for the package out of its module.\n\n First loads (or fetches from cache) a module for the\n package. Then extracts the package class from the module\n according to Spack's naming convention.\n \"\"\"\n _, pkg_name = self.partition_package_name(pkg_name)\n fullname = f\"{self.full_namespace}.{nm.pkg_name_to_pkg_dir(pkg_name, self.package_api)}\"\n if self.package_api >= (2, 0):\n fullname += \".package\"\n\n class_name = nm.pkg_name_to_class_name(pkg_name)\n\n if not self.exists(pkg_name):\n raise UnknownPackageError(fullname, self)\n\n try:\n if self.python_path:\n sys.path.insert(0, self.python_path)\n module = importlib.import_module(fullname)\n except Exception as e:\n msg = f\"cannot load package '{pkg_name}' from the '{self.namespace}' repository: {e}\"\n raise RepoError(msg) from e\n finally:\n if self.python_path:\n sys.path.remove(self.python_path)\n\n cls = getattr(module, class_name)\n if not isinstance(cls, type):\n tty.die(f\"{pkg_name}.{class_name} is not a class\")\n\n # Early exit if no overrides to apply or undo\n if (\n not self.overrides.get(pkg_name)\n and not hasattr(cls, \"overridden_attrs\")\n and not hasattr(cls, \"attrs_exclusively_from_config\")\n ):\n return cls\n\n def defining_class(myclass, name):\n return next((c for c in myclass.__mro__ if name in c.__dict__), None)\n\n # Clear any prior changes to class attributes in case the class was loaded from the\n # same repo, but with different overrides\n overridden_attrs = getattr(cls, \"overridden_attrs\", {})\n attrs_exclusively_from_config = getattr(cls, \"attrs_exclusively_from_config\", [])\n defclass_attrs = defining_class(cls, \"overridden_attrs\")\n defclass_exclusively_from_config = defining_class(cls, \"attrs_exclusively_from_config\")\n for key, val in overridden_attrs.items():\n setattr(defclass_attrs, key, val)\n for key in attrs_exclusively_from_config:\n delattr(defclass_exclusively_from_config, key)\n\n # Keep track of every class attribute that is overridden: if different overrides\n # dictionaries are used on the same physical repo, we make sure to restore the original\n # config values\n new_overridden_attrs = {}\n new_attrs_exclusively_from_config = set()\n for key, val in self.overrides.get(pkg_name, {}).items():\n if hasattr(cls, key):\n new_overridden_attrs[key] = getattr(cls, key)\n else:\n new_attrs_exclusively_from_config.add(key)\n\n setattr(cls, key, val)\n if new_overridden_attrs:\n setattr(cls, \"overridden_attrs\", dict(new_overridden_attrs))\n elif hasattr(cls, \"overridden_attrs\"):\n delattr(defclass_attrs, \"overridden_attrs\")\n if new_attrs_exclusively_from_config:\n setattr(cls, \"attrs_exclusively_from_config\", new_attrs_exclusively_from_config)\n elif hasattr(cls, \"attrs_exclusively_from_config\"):\n delattr(defclass_exclusively_from_config, \"attrs_exclusively_from_config\")\n\n return cls\n\n def partition_package_name(self, pkg_name: str) -> Tuple[str, str]:\n namespace, pkg_name = partition_package_name(pkg_name)\n if namespace and (namespace != self.namespace):\n raise InvalidNamespaceError(\n f\"Invalid namespace for the '{self.namespace}' repo: {namespace}\"\n )\n\n return namespace, pkg_name\n\n def __str__(self) -> str:\n return f\"Repo '{self.namespace}' at {self.root}\"\n\n def __repr__(self) -> str:\n return self.__str__()\n\n def __contains__(self, pkg_name: str) -> bool:\n return self.exists(pkg_name)\n\n @staticmethod\n def unmarshal(root, cache, overrides):\n \"\"\"Helper method to unmarshal keyword arguments\"\"\"\n return Repo(root, cache=cache, overrides=overrides)\n\n def marshal(self):\n cache = self._cache\n if isinstance(cache, Singleton):\n cache = cache.instance\n return self.root, cache, self.overrides\n\n def __reduce__(self):\n return Repo.unmarshal, self.marshal()\n\n\nRepoType = Union[Repo, RepoPath]\n\n\ndef partition_package_name(pkg_name: str) -> Tuple[str, str]:\n \"\"\"Given a package name that might be fully-qualified, returns the namespace part,\n if present and the unqualified package name.\n\n If the package name is unqualified, the namespace is an empty string.\n\n Args:\n pkg_name: a package name, either unqualified like ``llvm``, or\n fully-qualified, like ``builtin.llvm``\n \"\"\"\n namespace, _, pkg_name = pkg_name.rpartition(\".\")\n return namespace, pkg_name\n\n\ndef get_repo_yaml_dir(\n root: str, namespace: Optional[str], package_api: Tuple[int, int]\n) -> Tuple[str, str]:\n \"\"\"Returns the directory where repo.yaml is located and the effective namespace.\"\"\"\n if package_api < (2, 0):\n namespace = namespace or os.path.basename(root)\n # This ad-hoc regex is left for historical reasons, and should not have a breaking change.\n if not re.match(r\"\\w[\\.\\w-]*\", namespace):\n raise InvalidNamespaceError(f\"'{namespace}' is not a valid namespace.\")\n return root, namespace\n\n # Package API v2 has /spack_repo// structure and requires a namespace\n if namespace is None:\n raise InvalidNamespaceError(\"Namespace must be provided.\")\n\n # if namespace has dots those translate to subdirs of further namespace packages.\n namespace_components = namespace.split(\".\")\n\n if not all(nm.valid_module_name(n, package_api=package_api) for n in namespace_components):\n raise InvalidNamespaceError(f\"'{namespace}' is not a valid namespace.\" % namespace)\n\n return os.path.join(root, \"spack_repo\", *namespace_components), namespace\n\n\ndef create_repo(\n root,\n namespace: Optional[str] = None,\n subdir: str = packages_dir_name,\n package_api: Tuple[int, int] = spack.package_api_version,\n) -> Tuple[str, str]:\n \"\"\"Create a new repository in root with the specified namespace.\n\n If the namespace is not provided, use basename of root.\n Return the canonicalized path and namespace of the created repository.\n \"\"\"\n root = spack.util.path.canonicalize_path(root)\n repo_yaml_dir, namespace = get_repo_yaml_dir(os.path.abspath(root), namespace, package_api)\n\n existed = True\n try:\n dir_entry = next(os.scandir(repo_yaml_dir), None)\n except OSError as e:\n if e.errno == errno.ENOENT:\n existed = False\n dir_entry = None\n else:\n raise BadRepoError(f\"Cannot create new repo in {root}: {e}\")\n\n if dir_entry is not None:\n raise BadRepoError(f\"Cannot create new repo in {root}: directory is not empty.\")\n\n config_path = os.path.join(repo_yaml_dir, repo_config_name)\n\n subdir = _validate_and_normalize_subdir(subdir, root, package_api)\n\n packages_path = os.path.join(repo_yaml_dir, subdir)\n\n try:\n fs.mkdirp(packages_path)\n with open(config_path, \"w\", encoding=\"utf-8\") as config:\n config.write(\"repo:\\n\")\n config.write(f\" namespace: '{namespace}'\\n\")\n if subdir != packages_dir_name:\n config.write(f\" subdirectory: '{subdir}'\\n\")\n x, y = package_api\n config.write(f\" api: v{x}.{y}\\n\")\n\n except OSError as e:\n # try to clean up.\n if existed:\n shutil.rmtree(config_path, ignore_errors=True)\n shutil.rmtree(packages_path, ignore_errors=True)\n else:\n shutil.rmtree(root, ignore_errors=True)\n\n raise BadRepoError(\n \"Failed to create new repository in %s.\" % root, \"Caused by %s: %s\" % (type(e), e)\n ) from e\n\n return repo_yaml_dir, namespace\n\n\ndef from_path(path: str) -> Repo:\n \"\"\"Constructs a Repo using global misc cache.\"\"\"\n return Repo(path, cache=spack.caches.MISC_CACHE)\n\n\nMaybeExecutable = Optional[spack.util.executable.Executable]\n\n\nclass RepoDescriptor:\n \"\"\"Abstract base class for repository data.\"\"\"\n\n def __init__(self, name: Optional[str]) -> None:\n self.name = name\n\n @property\n def _maybe_name(self) -> str:\n \"\"\"Return the name if it exists, otherwise an empty string.\"\"\"\n return f\"{self.name}: \" if self.name else \"\"\n\n def initialize(self, fetch: bool = True, git: MaybeExecutable = None) -> None:\n return None\n\n def update(self, git: MaybeExecutable = None, remote: str = \"origin\") -> None:\n return None\n\n def construct(\n self, cache: spack.util.file_cache.FileCache, overrides: Optional[Dict[str, Any]] = None\n ) -> Dict[str, Union[Repo, Exception]]:\n \"\"\"Construct Repo instances from the descriptor.\"\"\"\n raise RuntimeError(\"construct() must be implemented in subclasses\")\n\n\nclass LocalRepoDescriptor(RepoDescriptor):\n def __init__(self, name: Optional[str], path: str) -> None:\n super().__init__(name)\n self.path = path\n\n def __repr__(self) -> str:\n return f\"{self.__class__.__name__}(name={self.name!r}, path={self.path!r})\"\n\n def construct(\n self, cache: spack.util.file_cache.FileCache, overrides: Optional[Dict[str, Any]] = None\n ) -> Dict[str, Union[Repo, Exception]]:\n try:\n return {self.path: Repo(self.path, cache=cache, overrides=overrides)}\n except RepoError as e:\n return {self.path: e}\n\n\nclass RemoteRepoDescriptor(RepoDescriptor):\n def __init__(\n self,\n *,\n name: Optional[str],\n repository: str,\n branch: Optional[str],\n commit: Optional[str],\n tag: Optional[str],\n destination: str,\n relative_paths: Optional[List[str]],\n lock: spack.util.lock.Lock,\n ) -> None:\n super().__init__(name)\n self.repository = repository\n self.branch = branch\n self.commit = commit\n self.tag = tag\n self.destination = destination\n self.relative_paths = relative_paths\n self.error: Optional[str] = None\n self.write_transaction = spack.util.lock.WriteTransaction(lock)\n self.read_transaction = spack.util.lock.ReadTransaction(lock)\n\n def _fetched(self) -> bool:\n \"\"\"Check if the repository has been fetched by looking for the .git\n directory or file (when a submodule).\"\"\"\n return os.path.exists(os.path.join(self.destination, \".git\"))\n\n def fetched(self) -> bool:\n with self.read_transaction:\n return self._fetched()\n return False\n\n def get_commit(self, git: MaybeExecutable = None):\n git = git or spack.util.git.git(required=True)\n with self.read_transaction:\n if not self._fetched():\n return None\n\n with fs.working_dir(self.destination):\n return git(\"rev-parse\", \"HEAD\", output=str).strip()\n\n def _clone_or_pull(\n self,\n git: spack.util.executable.Executable,\n update: bool = False,\n remote: str = \"origin\",\n depth: Optional[int] = None,\n ) -> None:\n with self.write_transaction:\n try:\n with fs.working_dir(self.destination, create=True):\n # do not fetch if the package repository was fetched by another\n # process while we were waiting for the lock\n fetched = self._fetched()\n if fetched and not update:\n self.read_index_file()\n return\n\n # If depth is not provided, default to:\n # 1. The first time the repo is loaded, download a partial clone.\n # This speeds up CI/CD and other cases where the user never\n # updates the repository.\n # 2. When *updating* an already cloned copy of the repository,\n # perform a full fetch (unshallowing the repo if necessary) to\n # optimize for full history.\n if depth is None and not fetched:\n depth = 2\n\n # setup the repository if it does not exist\n if not fetched:\n spack.util.git.init_git_repo(self.repository, remote=remote, git_exe=git)\n\n # determine the default branch from ls-remote\n # (if no branch, tag, or commit is specified)\n if not (self.commit or self.tag or self.branch):\n # Get HEAD and all branches. On more recent versions of git, this can\n # be done with a single call to `git ls-remote --symref remote HEAD`.\n refs = git(\"ls-remote\", remote, \"HEAD\", \"refs/heads/*\", output=str)\n head_match = re.search(r\"^([0-9a-f]+)\\s+HEAD$\", refs, re.MULTILINE)\n if not head_match:\n self.error = f\"Unable to locate HEAD for {self.repository}\"\n return\n\n head_sha = head_match.group(1)\n\n # Find the first branch that matches this SHA\n branch_match = re.search(\n rf\"^{re.escape(head_sha)}\\s+refs/heads/(\\S+)$\", refs, re.MULTILINE\n )\n if not branch_match:\n self.error = (\n f\"Unable to locate a default branch for {self.repository}\"\n )\n return\n self.branch = branch_match.group(1)\n\n # determine the branch and remote if no config values exist\n elif not (self.commit or self.tag or self.branch):\n self.branch = git(\"rev-parse\", \"--abbrev-ref\", \"HEAD\", output=str).strip()\n remote = git(\"config\", f\"branch.{self.branch}.remote\", output=str).strip()\n\n if self.commit:\n spack.util.git.pull_checkout_commit(\n self.commit, remote=remote, depth=depth, git_exe=git\n )\n\n elif self.tag:\n spack.util.git.pull_checkout_tag(\n self.tag, remote=remote, depth=depth, git_exe=git\n )\n\n elif self.branch:\n # if the branch already exists we should use the\n # previously configured remote\n try:\n output = git(\"config\", f\"branch.{self.branch}.remote\", output=str)\n remote = output.strip()\n except spack.util.executable.ProcessError:\n pass\n spack.util.git.pull_checkout_branch(\n self.branch, remote=remote, depth=depth, git_exe=git\n )\n\n except spack.util.executable.ProcessError:\n self.error = f\"Failed to {'update' if update else 'clone'} repository {self.name}\"\n return\n\n self.read_index_file()\n\n def update(self, git: MaybeExecutable = None, remote: str = \"origin\") -> None:\n if git is None:\n raise RepoError(\"Git executable not found\")\n\n self._clone_or_pull(git, update=True, remote=remote)\n\n if self.error:\n raise RepoError(self.error)\n\n def initialize(self, fetch: bool = True, git: MaybeExecutable = None) -> None:\n \"\"\"Clone the remote repository if it has not been fetched yet and read the index file\n if necessary.\"\"\"\n if self.fetched():\n self.read_index_file()\n return\n\n if not fetch:\n return\n\n if not git:\n self.error = \"Git executable not found\"\n return\n\n self._clone_or_pull(git)\n\n def read_index_file(self) -> None:\n if self.relative_paths is not None:\n return\n\n repo_index_file = os.path.join(self.destination, SPACK_REPO_INDEX_FILE_NAME)\n try:\n with open(repo_index_file, encoding=\"utf-8\") as f:\n index_data = syaml.load(f)\n assert \"repo_index\" in index_data, \"missing 'repo_index' key\"\n repo_index = index_data[\"repo_index\"]\n assert isinstance(repo_index, dict), \"'repo_index' must be a dictionary\"\n assert \"paths\" in repo_index, \"missing 'paths' key in 'repo_index'\"\n sub_paths = repo_index[\"paths\"]\n assert isinstance(sub_paths, list), \"'paths' under 'repo_index' must be a list\"\n except (OSError, syaml.SpackYAMLError, AssertionError) as e:\n self.error = f\"failed to read {repo_index_file}: {e}\"\n return\n\n # validate that this is a list of relative paths.\n if not isinstance(sub_paths, list) or not all(isinstance(p, str) for p in sub_paths):\n self.error = \"invalid repo index file format: expected a list of relative paths.\"\n return\n\n self.relative_paths = sub_paths\n\n def __repr__(self):\n return (\n f\"RemoteRepoDescriptor(name={self.name!r}, \"\n f\"repository={self.repository!r}, \"\n f\"destination={self.destination!r}, \"\n f\"relative_paths={self.relative_paths!r})\"\n )\n\n def construct(\n self, cache: spack.util.file_cache.FileCache, overrides: Optional[Dict[str, Any]] = None\n ) -> Dict[str, Union[Repo, Exception]]:\n if self.error:\n return {self.destination: Exception(self.error)}\n\n repos: Dict[str, Union[Repo, Exception]] = {}\n for subpath in self.relative_paths or []:\n if os.path.isabs(subpath):\n repos[self.destination] = Exception(\n f\"Repository subpath '{subpath}' must be relative\"\n )\n continue\n path = os.path.join(self.destination, subpath)\n try:\n repos[path] = Repo(path, cache=cache, overrides=overrides)\n except RepoError as e:\n repos[path] = e\n return repos\n\n\nclass BrokenRepoDescriptor(RepoDescriptor):\n \"\"\"A descriptor for a broken repository, used to indicate errors in the configuration that\n aren't fatal until the repository is used.\"\"\"\n\n def __init__(self, name: Optional[str], error: str) -> None:\n super().__init__(name)\n self.error = error\n\n def initialize(\n self, fetch: bool = True, git: Optional[spack.util.executable.Executable] = None\n ) -> None:\n pass\n\n def construct(\n self, cache: spack.util.file_cache.FileCache, overrides: Optional[Dict[str, Any]] = None\n ) -> Dict[str, Union[Repo, Exception]]:\n return {self.name or \"\": Exception(self.error)}\n\n\nclass RepoDescriptors(Mapping[str, RepoDescriptor]):\n \"\"\"A collection of repository descriptors.\"\"\"\n\n def __init__(self, descriptors: Dict[str, RepoDescriptor]) -> None:\n self.descriptors = descriptors\n\n def __getitem__(self, name: str) -> RepoDescriptor:\n return self.descriptors[name]\n\n def __iter__(self):\n return iter(self.descriptors.keys())\n\n def __len__(self):\n return len(self.descriptors)\n\n def __contains__(self, name) -> bool:\n return name in self.descriptors\n\n def __repr__(self):\n return f\"RepoDescriptors({self.descriptors!r})\"\n\n @staticmethod\n def from_config(\n lock: spack.util.lock.Lock, config: spack.config.Configuration, scope=None\n ) -> \"RepoDescriptors\":\n return RepoDescriptors(\n {\n name: parse_config_descriptor(name, cfg, lock)\n for name, cfg in config.get_config(\"repos\", scope=scope).items()\n }\n )\n\n def construct(\n self,\n cache: spack.util.file_cache.FileCache,\n fetch: bool = True,\n find_git: Callable[[], MaybeExecutable] = lambda: spack.util.git.git(required=True),\n overrides: Optional[Dict[str, Any]] = None,\n ) -> Tuple[RepoPath, Dict[str, Exception]]:\n \"\"\"Construct a RepoPath from the descriptors.\n\n If init is True, initialize all remote repositories that have not been fetched yet.\n\n Returns:\n A tuple containing a RepoPath instance with all constructed Repos and a dictionary\n mapping paths to exceptions that occurred during construction.\n \"\"\"\n repos: List[Repo] = []\n errors: Dict[str, Exception] = {}\n git: MaybeExecutable = None\n\n for descriptor in self.descriptors.values():\n if fetch and isinstance(descriptor, RemoteRepoDescriptor):\n git = git or find_git()\n descriptor.initialize(fetch=True, git=git)\n else:\n descriptor.initialize(fetch=False)\n\n for path, result in descriptor.construct(cache=cache, overrides=overrides).items():\n if isinstance(result, Repo):\n repos.append(result)\n else:\n errors[path] = result\n\n return RepoPath(*repos), errors\n\n\ndef parse_config_descriptor(\n name: Optional[str], descriptor: Any, lock: spack.util.lock.Lock\n) -> RepoDescriptor:\n \"\"\"Parse a repository descriptor from validated configuration. This does not instantiate Repo\n objects, but merely turns the config into a more useful RepoDescriptor instance.\n\n Args:\n name: the name of the repository, used for error messages\n descriptor: the configuration for the repository, which can be a string (local path),\n or a dictionary with ``git`` key containing git URL and other options.\n\n Returns:\n A RepoDescriptor instance, either LocalRepoDescriptor or RemoteRepoDescriptor.\n\n Raises:\n BadRepoError: if the descriptor is invalid or cannot be parsed.\n RuntimeError: if the descriptor is of an unexpected type.\n\n \"\"\"\n if isinstance(descriptor, str):\n return LocalRepoDescriptor(name, spack.util.path.canonicalize_path(descriptor))\n\n # Should be the case due to config validation.\n assert isinstance(descriptor, dict), \"Repository descriptor must be a string or a dictionary\"\n\n # Configuration validation works per scope, and we want to allow overriding e.g. destination\n # in user config without the user having to repeat the `git` key and value again. This is a\n # hard error, since config validation is a hard error.\n if \"git\" not in descriptor:\n raise RuntimeError(\n f\"Invalid configuration for repository '{name}': {descriptor!r}. A `git` attribute is \"\n \"required for remote repositories.\"\n )\n\n repository = descriptor[\"git\"]\n assert isinstance(repository, str), \"Package repository git URL must be a string\"\n\n destination = descriptor.get(\"destination\", None)\n\n if destination is None: # use a default destination\n dir_name = spack.util.hash.b32_hash(repository)[-7:]\n destination = os.path.join(spack.paths.package_repos_path, dir_name)\n else:\n destination = spack.util.path.canonicalize_path(destination)\n\n return RemoteRepoDescriptor(\n name=name,\n repository=repository,\n branch=descriptor.get(\"branch\"),\n commit=descriptor.get(\"commit\"),\n tag=descriptor.get(\"tag\"),\n destination=destination,\n relative_paths=descriptor.get(\"paths\"),\n lock=lock,\n )\n\n\ndef create_or_construct(\n root: str,\n namespace: Optional[str] = None,\n package_api: Tuple[int, int] = spack.package_api_version,\n) -> Repo:\n \"\"\"Create a repository, or just return a Repo if it already exists.\"\"\"\n repo_yaml_dir, _ = get_repo_yaml_dir(root, namespace, package_api)\n if not os.path.exists(repo_yaml_dir):\n fs.mkdirp(root)\n create_repo(root, namespace=namespace, package_api=package_api)\n return from_path(repo_yaml_dir)\n\n\ndef create_and_enable(config: spack.config.Configuration) -> RepoPath:\n \"\"\"Immediately call enable() on the created RepoPath instance.\"\"\"\n repo_path = RepoPath.from_config(config)\n repo_path.enable()\n return repo_path\n\n\n#: Global package repository instance.\nPATH = cast(RepoPath, Singleton(lambda: create_and_enable(spack.config.CONFIG)))\n\n\n# Add the finder to sys.meta_path\nREPOS_FINDER = ReposFinder()\nsys.meta_path.append(REPOS_FINDER)\n\n\ndef all_package_names(include_virtuals=False):\n \"\"\"Convenience wrapper around ``spack.repo.all_package_names()``.\"\"\"\n return PATH.all_package_names(include_virtuals)\n\n\n@contextlib.contextmanager\ndef use_repositories(\n *paths_and_repos: Union[str, Repo], override: bool = True\n) -> Generator[RepoPath, None, None]:\n \"\"\"Use the repositories passed as arguments within the context manager.\n\n Args:\n *paths_and_repos: paths to the repositories to be used, or\n already constructed Repo objects\n override: if True use only the repositories passed as input,\n if False add them to the top of the list of current repositories.\n Returns:\n Corresponding RepoPath object\n \"\"\"\n paths = {getattr(x, \"root\", x): getattr(x, \"root\", x) for x in paths_and_repos}\n scope_name = f\"use-repo-{uuid.uuid4()}\"\n repos_key = \"repos:\" if override else \"repos\"\n spack.config.CONFIG.push_scope(\n spack.config.InternalConfigScope(name=scope_name, data={repos_key: paths})\n )\n old_repo, new_repo = PATH, RepoPath.from_config(spack.config.CONFIG)\n old_repo.disable()\n enable_repo(new_repo)\n try:\n yield new_repo\n finally:\n spack.config.CONFIG.remove_scope(scope_name=scope_name)\n new_repo.disable()\n enable_repo(old_repo)\n\n\ndef enable_repo(repo_path: RepoPath) -> None:\n \"\"\"Set the global package repository and make them available in module search paths.\"\"\"\n global PATH\n PATH = repo_path\n PATH.enable()\n\n\nclass RepoError(spack.error.SpackError):\n \"\"\"Superclass for repository-related errors.\"\"\"\n\n\nclass NoRepoConfiguredError(RepoError):\n \"\"\"Raised when there are no repositories configured.\"\"\"\n\n\nclass InvalidNamespaceError(RepoError):\n \"\"\"Raised when an invalid namespace is encountered.\"\"\"\n\n\nclass BadRepoError(RepoError):\n \"\"\"Raised when repo layout is invalid.\"\"\"\n\n\nclass BadRepoVersionError(BadRepoError):\n \"\"\"Raised when repo API version is too high or too low for Spack.\"\"\"\n\n def __init__(self, api, *args, **kwargs):\n self.api = api\n super().__init__(*args, **kwargs)\n\n\nclass UnknownEntityError(RepoError):\n \"\"\"Raised when we encounter a package spack doesn't have.\"\"\"\n\n\nclass UnknownPackageError(UnknownEntityError):\n \"\"\"Raised when we encounter a package spack doesn't have.\"\"\"\n\n def __init__(\n self,\n name,\n repo: Optional[Union[Repo, RepoPath, str]] = None,\n *,\n get_close_matches=difflib.get_close_matches,\n ):\n msg = \"Attempting to retrieve anonymous package.\"\n long_msg = None\n if name:\n msg = f\"Package '{name}' not found\"\n if repo:\n if isinstance(repo, Repo):\n msg += f\" in repository '{repo.root}'\"\n elif isinstance(repo, str):\n msg += f\" in repository '{repo}'\"\n\n # Special handling for specs that may have been intended as\n # filenames: prompt the user to ask whether they intended to write\n # './'.\n if name.endswith(\".yaml\"):\n long_msg = \"Did you mean to specify a filename with './{0}'?\"\n long_msg = long_msg.format(name)\n else:\n long_msg = \"Use 'spack create' to create a new package.\"\n\n if not repo:\n repo = PATH.ensure_unwrapped()\n\n # We need to compare the base package name\n pkg_name = name_from_fullname(name)\n similar = []\n if isinstance(repo, (Repo, RepoPath)):\n try:\n similar = get_close_matches(pkg_name, repo.all_package_names())\n except Exception:\n pass\n\n if 1 <= len(similar) <= 5:\n long_msg += \"\\n\\nDid you mean one of the following packages?\\n \"\n long_msg += \"\\n \".join(similar)\n\n super().__init__(msg, long_msg)\n self.name = name\n\n\nclass UnknownNamespaceError(UnknownEntityError):\n \"\"\"Raised when we encounter an unknown namespace\"\"\"\n\n def __init__(self, namespace, name=None):\n msg, long_msg = f\"Unknown namespace: {namespace}\", None\n if name == \"yaml\":\n long_msg = f\"Did you mean to specify a filename with './{namespace}.{name}'?\"\n super().__init__(msg, long_msg)\n\n\nclass FailedConstructorError(RepoError):\n \"\"\"Raised when a package's class constructor fails.\"\"\"\n\n def __init__(self, name, exc_type, exc_obj, exc_tb):\n super().__init__(\n \"Class constructor failed for package '%s'.\" % name,\n \"\\nCaused by:\\n\"\n + (\"%s: %s\\n\" % (exc_type.__name__, exc_obj))\n + \"\".join(traceback.format_tb(exc_tb)),\n )\n self.name = name\n" }, { "path": "lib/spack/spack/repo_migrate.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport ast\nimport difflib\nimport os\nimport re\nimport shutil\nimport sys\nfrom typing import IO, Dict, List, Optional, Set, Tuple\n\nimport spack.repo\nimport spack.util.naming\nimport spack.util.spack_yaml\n\n\ndef _same_contents(f: str, g: str) -> bool:\n \"\"\"Return True if the files have the same contents.\"\"\"\n try:\n with open(f, \"rb\") as f1, open(g, \"rb\") as f2:\n while True:\n b1 = f1.read(4096)\n b2 = f2.read(4096)\n if b1 != b2:\n return False\n if not b1 and not b2:\n break\n return True\n except OSError:\n return False\n\n\ndef migrate_v1_to_v2(\n repo: spack.repo.Repo, *, patch_file: Optional[IO[bytes]], err: IO[str] = sys.stderr\n) -> Tuple[bool, Optional[spack.repo.Repo]]:\n \"\"\"To upgrade a repo from Package API v1 to v2 we need to:\n 1. ensure ``spack_repo/`` parent dirs to the ``repo.yaml`` file.\n 2. rename /package.py to /package.py.\n 3. bump the version in ``repo.yaml``.\n \"\"\"\n if not (1, 0) <= repo.package_api < (2, 0):\n raise RuntimeError(f\"Cannot upgrade from {repo.package_api_str} to v2.0\")\n\n with open(os.path.join(repo.root, \"repo.yaml\"), encoding=\"utf-8\") as f:\n updated_config = spack.util.spack_yaml.load(f)\n updated_config[\"repo\"][\"api\"] = \"v2.0\"\n\n namespace = repo.namespace.split(\".\")\n\n if not all(\n spack.util.naming.valid_module_name(part, package_api=(2, 0)) for part in namespace\n ):\n print(\n f\"Cannot upgrade from v1 to v2, because the namespace '{repo.namespace}' is not a \"\n \"valid Python module\",\n file=err,\n )\n return False, None\n\n try:\n subdirectory = spack.repo._validate_and_normalize_subdir(\n repo.subdirectory, repo.root, package_api=(2, 0)\n )\n except spack.repo.BadRepoError:\n print(\n f\"Cannot upgrade from v1 to v2, because the subdirectory '{repo.subdirectory}' is not \"\n \"a valid Python module\",\n file=err,\n )\n return False, None\n\n new_root = os.path.join(repo.root, \"spack_repo\", *namespace)\n\n ino_to_relpath: Dict[int, str] = {}\n symlink_to_ino: Dict[str, int] = {}\n\n prefix_len = len(repo.root) + len(os.sep)\n\n rename: Dict[str, str] = {}\n dirs_to_create: List[str] = []\n files_to_copy: List[str] = []\n\n errors = False\n\n stack: List[Tuple[str, int]] = [(repo.packages_path, 0)]\n\n while stack:\n path, depth = stack.pop()\n\n try:\n entries = os.scandir(path)\n except OSError:\n continue\n\n for entry in entries:\n rel_path = entry.path[prefix_len:]\n\n if depth == 0 and entry.name in (\"spack_repo\", \"repo.yaml\"):\n continue\n\n ino_to_relpath[entry.inode()] = entry.path[prefix_len:]\n\n if entry.is_symlink():\n try:\n symlink_to_ino[rel_path] = entry.stat(follow_symlinks=True).st_ino\n except OSError:\n symlink_to_ino[rel_path] = -1 # dangling or no access\n\n continue\n\n elif entry.is_dir(follow_symlinks=False):\n if entry.name == \"__pycache__\":\n continue\n\n # check if this is a package\n if depth == 0 and os.path.exists(os.path.join(entry.path, \"package.py\")):\n if \"_\" in entry.name:\n print(\n f\"Invalid package name '{entry.name}': underscores are not allowed in \"\n \"package names, rename the package with hyphens as separators\",\n file=err,\n )\n errors = True\n continue\n pkg_dir = spack.util.naming.pkg_name_to_pkg_dir(entry.name, package_api=(2, 0))\n if pkg_dir != entry.name:\n rename[f\"{subdirectory}{os.sep}{entry.name}\"] = (\n f\"{subdirectory}{os.sep}{pkg_dir}\"\n )\n\n dirs_to_create.append(rel_path)\n\n stack.append((entry.path, depth + 1))\n continue\n\n files_to_copy.append(rel_path)\n\n if errors:\n return False, None\n\n rename_regex = re.compile(\"^(\" + \"|\".join(re.escape(k) for k in rename.keys()) + \")\")\n\n if not patch_file:\n os.makedirs(os.path.join(new_root, repo.subdirectory), exist_ok=True)\n\n def _relocate(rel_path: str) -> Tuple[str, str]:\n old = os.path.join(repo.root, rel_path)\n if rename:\n new_rel = rename_regex.sub(lambda m: rename[m.group(0)], rel_path)\n else:\n new_rel = rel_path\n new = os.path.join(new_root, new_rel)\n return old, new\n\n if patch_file:\n patch_file.write(b\"The following directories, files and symlinks will be created:\\n\")\n\n for rel_path in dirs_to_create:\n _, new_path = _relocate(rel_path)\n if not patch_file:\n try:\n os.mkdir(new_path)\n except FileExistsError: # not an error if the directory already exists\n continue\n else:\n patch_file.write(b\"create directory \")\n patch_file.write(new_path.encode(\"utf-8\"))\n patch_file.write(b\"\\n\")\n\n for rel_path in files_to_copy:\n old_path, new_path = _relocate(rel_path)\n if os.path.lexists(new_path):\n # if we already copied this file, don't error.\n if not _same_contents(old_path, new_path):\n print(\n f\"Cannot upgrade from v1 to v2, because the file '{new_path}' already exists\",\n file=err,\n )\n return False, None\n continue\n if not patch_file:\n shutil.copy2(old_path, new_path)\n else:\n patch_file.write(b\"copy \")\n patch_file.write(old_path.encode(\"utf-8\"))\n patch_file.write(b\" -> \")\n patch_file.write(new_path.encode(\"utf-8\"))\n patch_file.write(b\"\\n\")\n\n for rel_path, ino in symlink_to_ino.items():\n old_path, new_path = _relocate(rel_path)\n if ino in ino_to_relpath:\n # link by path relative to the new root\n _, new_target = _relocate(ino_to_relpath[ino])\n tgt = os.path.relpath(new_target, new_path)\n else:\n tgt = os.path.realpath(old_path)\n\n # no-op if the same, error if different\n if os.path.lexists(new_path):\n if not os.path.islink(new_path) or os.readlink(new_path) != tgt:\n print(\n f\"Cannot upgrade from v1 to v2, because the file '{new_path}' already exists\",\n file=err,\n )\n return False, None\n continue\n\n if not patch_file:\n os.symlink(tgt, new_path)\n else:\n patch_file.write(b\"create symlink \")\n patch_file.write(new_path.encode(\"utf-8\"))\n patch_file.write(b\" -> \")\n patch_file.write(tgt.encode(\"utf-8\"))\n patch_file.write(b\"\\n\")\n\n if not patch_file:\n with open(os.path.join(new_root, \"repo.yaml\"), \"w\", encoding=\"utf-8\") as f:\n spack.util.spack_yaml.dump(updated_config, f)\n updated_repo = spack.repo.from_path(new_root)\n else:\n patch_file.write(b\"\\n\")\n updated_repo = repo # compute the import diff on the v1 repo since v2 doesn't exist yet\n\n result = migrate_v2_imports(\n updated_repo.packages_path, updated_repo.root, patch_file=patch_file, err=err\n )\n\n return result, (updated_repo if patch_file else None)\n\n\ndef _pkg_module_update(pkg_module: str) -> str:\n return re.sub(r\"^num(\\d)\", r\"_\\1\", pkg_module) # num7zip -> _7zip.\n\n\ndef _spack_pkg_to_spack_repo(modulename: str) -> str:\n # rewrite spack.pkg.builtin.foo -> spack_repo.builtin.packages.foo.package\n parts = modulename.split(\".\")\n assert parts[:2] == [\"spack\", \"pkg\"]\n parts[0:2] = [\"spack_repo\"]\n parts.insert(2, \"packages\")\n parts[3] = _pkg_module_update(parts[3])\n parts.append(\"package\")\n return \".\".join(parts)\n\n\ndef migrate_v2_imports(\n packages_dir: str, root: str, patch_file: Optional[IO[bytes]], err: IO[str] = sys.stderr\n) -> bool:\n \"\"\"In Package API v2.0, packages need to explicitly import package classes and a few other\n symbols from the build_systems module. This function automatically adds the missing imports\n to each package.py file in the repository.\"\"\"\n\n symbol_to_module = {\n \"AspellDictPackage\": \"spack_repo.builtin.build_systems.aspell_dict\",\n \"AutotoolsPackage\": \"spack_repo.builtin.build_systems.autotools\",\n \"BundlePackage\": \"spack_repo.builtin.build_systems.bundle\",\n \"CachedCMakePackage\": \"spack_repo.builtin.build_systems.cached_cmake\",\n \"cmake_cache_filepath\": \"spack_repo.builtin.build_systems.cached_cmake\",\n \"cmake_cache_option\": \"spack_repo.builtin.build_systems.cached_cmake\",\n \"cmake_cache_path\": \"spack_repo.builtin.build_systems.cached_cmake\",\n \"cmake_cache_string\": \"spack_repo.builtin.build_systems.cached_cmake\",\n \"CargoPackage\": \"spack_repo.builtin.build_systems.cargo\",\n \"CMakePackage\": \"spack_repo.builtin.build_systems.cmake\",\n \"generator\": \"spack_repo.builtin.build_systems.cmake\",\n \"CompilerPackage\": \"spack_repo.builtin.build_systems.compiler\",\n \"CudaPackage\": \"spack_repo.builtin.build_systems.cuda\",\n \"Package\": \"spack_repo.builtin.build_systems.generic\",\n \"GNUMirrorPackage\": \"spack_repo.builtin.build_systems.gnu\",\n \"GoPackage\": \"spack_repo.builtin.build_systems.go\",\n \"LuaPackage\": \"spack_repo.builtin.build_systems.lua\",\n \"MakefilePackage\": \"spack_repo.builtin.build_systems.makefile\",\n \"MavenPackage\": \"spack_repo.builtin.build_systems.maven\",\n \"MesonPackage\": \"spack_repo.builtin.build_systems.meson\",\n \"MSBuildPackage\": \"spack_repo.builtin.build_systems.msbuild\",\n \"NMakePackage\": \"spack_repo.builtin.build_systems.nmake\",\n \"OctavePackage\": \"spack_repo.builtin.build_systems.octave\",\n \"INTEL_MATH_LIBRARIES\": \"spack_repo.builtin.build_systems.oneapi\",\n \"IntelOneApiLibraryPackage\": \"spack_repo.builtin.build_systems.oneapi\",\n \"IntelOneApiLibraryPackageWithSdk\": \"spack_repo.builtin.build_systems.oneapi\",\n \"IntelOneApiPackage\": \"spack_repo.builtin.build_systems.oneapi\",\n \"IntelOneApiStaticLibraryList\": \"spack_repo.builtin.build_systems.oneapi\",\n \"PerlPackage\": \"spack_repo.builtin.build_systems.perl\",\n \"PythonExtension\": \"spack_repo.builtin.build_systems.python\",\n \"PythonPackage\": \"spack_repo.builtin.build_systems.python\",\n \"QMakePackage\": \"spack_repo.builtin.build_systems.qmake\",\n \"RPackage\": \"spack_repo.builtin.build_systems.r\",\n \"RacketPackage\": \"spack_repo.builtin.build_systems.racket\",\n \"ROCmPackage\": \"spack_repo.builtin.build_systems.rocm\",\n \"RubyPackage\": \"spack_repo.builtin.build_systems.ruby\",\n \"SConsPackage\": \"spack_repo.builtin.build_systems.scons\",\n \"SIPPackage\": \"spack_repo.builtin.build_systems.sip\",\n \"SourceforgePackage\": \"spack_repo.builtin.build_systems.sourceforge\",\n \"SourcewarePackage\": \"spack_repo.builtin.build_systems.sourceware\",\n \"WafPackage\": \"spack_repo.builtin.build_systems.waf\",\n \"XorgPackage\": \"spack_repo.builtin.build_systems.xorg\",\n }\n\n success = True\n\n for f in os.scandir(packages_dir):\n pkg_path = os.path.join(f.path, \"package.py\")\n if (\n f.name in (\"__init__.py\", \"__pycache__\")\n or not f.is_dir(follow_symlinks=False)\n or os.path.islink(pkg_path)\n ):\n print(f\"Skipping {f.path}\", file=err)\n continue\n try:\n with open(pkg_path, \"rb\") as file:\n tree = ast.parse(file.read())\n except FileNotFoundError:\n continue\n except (OSError, SyntaxError) as e:\n print(f\"Skipping {pkg_path}: {e}\", file=err)\n continue\n\n #: Symbols that are referenced in the package and may need to be imported.\n referenced_symbols: Set[str] = set()\n\n #: Set of symbols of interest that are already defined through imports, assignments, or\n #: function definitions.\n defined_symbols: Set[str] = set()\n best_line: Optional[int] = None\n seen_import = False\n module_replacements: Dict[str, str] = {}\n parent: Dict[int, ast.AST] = {}\n\n #: List of (line, col start, old, new) tuples of strings to be replaced inline.\n inline_updates: List[Tuple[int, int, str, str]] = []\n\n #: List of (line from, line to, new lines) tuples of line replacements\n multiline_updates: List[Tuple[int, int, List[str]]] = []\n\n try:\n with open(pkg_path, \"r\", encoding=\"utf-8\", newline=\"\") as file:\n original_lines = file.readlines()\n except (OSError, UnicodeDecodeError) as e:\n success = False\n print(f\"Skipping {pkg_path}: {e}\", file=err)\n continue\n\n if len(original_lines) < 2: # assume package.py files have at least 2 lines...\n continue\n\n if original_lines[0].endswith(\"\\r\\n\"):\n newline = \"\\r\\n\"\n elif original_lines[0].endswith(\"\\n\"):\n newline = \"\\n\"\n elif original_lines[0].endswith(\"\\r\"):\n newline = \"\\r\"\n else:\n success = False\n print(f\"{pkg_path}: unknown line ending, cannot fix\", file=err)\n continue\n\n updated_lines = original_lines.copy()\n\n for node in ast.walk(tree):\n for child in ast.iter_child_nodes(node):\n if isinstance(child, ast.Attribute):\n parent[id(child)] = node\n\n # Get the last import statement from the first block of top-level imports\n if isinstance(node, ast.Module):\n for child in ast.iter_child_nodes(node):\n # if we never encounter an import statement, the best line to add is right\n # before the first node under the module\n if best_line is None and isinstance(child, ast.stmt):\n best_line = child.lineno\n\n # prefer adding right before `from spack.package import ...`\n if isinstance(child, ast.ImportFrom) and child.module == \"spack.package\":\n seen_import = True\n best_line = child.lineno # add it right before spack.package\n break\n\n is_import = isinstance(child, (ast.Import, ast.ImportFrom))\n\n if is_import:\n if isinstance(child, (ast.stmt, ast.expr)):\n end_lineno = getattr(child, \"end_lineno\", None)\n if end_lineno is not None:\n # put it right after the last import statement\n best_line = end_lineno + 1\n else: # old versions of python don't have end_lineno; put it before.\n best_line = child.lineno\n\n if not seen_import and is_import:\n seen_import = True\n elif seen_import and not is_import:\n break\n\n # Function definitions or assignments to variables whose name is a symbol of interest\n # are considered as redefinitions, so we skip them.\n elif isinstance(node, ast.FunctionDef):\n if node.name in symbol_to_module:\n print(\n f\"{pkg_path}:{node.lineno}: redefinition of `{node.name}` skipped\",\n file=err,\n )\n defined_symbols.add(node.name)\n elif isinstance(node, ast.Assign):\n for target in node.targets:\n if isinstance(target, ast.Name) and target.id in symbol_to_module:\n print(\n f\"{pkg_path}:{target.lineno}: redefinition of `{target.id}` skipped\",\n file=err,\n )\n defined_symbols.add(target.id)\n\n # Register symbols that are not imported.\n elif isinstance(node, ast.Name) and node.id in symbol_to_module:\n referenced_symbols.add(node.id)\n\n # Find lines where spack.pkg is used.\n elif (\n isinstance(node, ast.Attribute)\n and isinstance(node.value, ast.Name)\n and node.value.id == \"spack\"\n and node.attr == \"pkg\"\n ):\n # go as many attrs up until we reach a known module name to be replaced\n known_module = \"spack.pkg\"\n ancestor = node\n while True:\n next_parent = parent.get(id(ancestor))\n if next_parent is None or not isinstance(next_parent, ast.Attribute):\n break\n ancestor = next_parent\n known_module = f\"{known_module}.{ancestor.attr}\"\n if known_module in module_replacements:\n break\n\n inline_updates.append(\n (\n ancestor.lineno,\n ancestor.col_offset,\n known_module,\n module_replacements[known_module],\n )\n )\n\n elif isinstance(node, ast.ImportFrom):\n # Keep track of old style spack.pkg imports, to be replaced.\n if node.module and node.module.startswith(\"spack.pkg.\") and node.level == 0:\n depth = node.module.count(\".\")\n\n # not all python versions have end_lineno for ImportFrom\n end_lineno = getattr(node, \"end_lineno\", None)\n\n # simple case of find and replace\n # from spack.pkg.builtin.my_pkg import MyPkg\n # -> from spack_repo.builtin.packages.my_pkg.package import MyPkg\n if depth == 3:\n module_replacements[node.module] = _spack_pkg_to_spack_repo(node.module)\n inline_updates.append(\n (\n node.lineno,\n node.col_offset,\n node.module,\n module_replacements[node.module],\n )\n )\n\n # non-trivial possible multiline case\n # from spack.pkg.builtin import (boost, cmake as foo)\n # -> import spack_repo.builtin.packages.boost.package as boost\n # -> import spack_repo.builtin.packages.cmake.package as foo\n elif depth == 2:\n if end_lineno is None:\n success = False\n print(\n f\"{pkg_path}:{node.lineno}: cannot rewrite {node.module} \"\n \"import statement, since this Python version does not \"\n \"provide end_lineno. Best to update to Python 3.8+\",\n file=err,\n )\n continue\n\n _, _, namespace = node.module.rpartition(\".\")\n indent = original_lines[node.lineno - 1][: node.col_offset]\n new_lines = []\n for alias in node.names:\n pkg_module = _pkg_module_update(alias.name)\n new_lines.append(\n f\"{indent}import spack_repo.{namespace}.packages.\"\n f\"{pkg_module}.package as {alias.asname or pkg_module}\"\n f\"{newline}\"\n )\n multiline_updates.append((node.lineno, end_lineno + 1, new_lines))\n\n else:\n success = False\n print(\n f\"{pkg_path}:{node.lineno}: don't know how to rewrite `{node.module}`\",\n file=err,\n )\n continue\n\n elif node.module is not None and node.level == 1 and \".\" not in node.module:\n # rewrite `from .blt import ...` -> `from ..blt.package import ...`\n pkg_module = _pkg_module_update(node.module)\n inline_updates.append(\n (\n node.lineno,\n node.col_offset,\n f\".{node.module}\",\n f\"..{pkg_module}.package\",\n )\n )\n\n # Subtract the symbols that are imported so we don't repeatedly add imports.\n for alias in node.names:\n if alias.name in symbol_to_module:\n if alias.asname is None:\n defined_symbols.add(alias.name)\n\n # error when symbols are explicitly imported that are no longer available\n if node.module == \"spack.package\" and node.level == 0:\n success = False\n print(\n f\"{pkg_path}:{node.lineno}: `{alias.name}` is imported from \"\n \"`spack.package`, which no longer provides this symbol\",\n file=err,\n )\n\n if alias.asname and alias.asname in symbol_to_module:\n defined_symbols.add(alias.asname)\n\n elif isinstance(node, ast.Import):\n # normal imports are easy find and replace since they are single lines.\n for alias in node.names:\n if alias.asname and alias.asname in symbol_to_module:\n defined_symbols.add(alias.name)\n elif alias.asname is None and alias.name.startswith(\"spack.pkg.\"):\n module_replacements[alias.name] = _spack_pkg_to_spack_repo(alias.name)\n inline_updates.append(\n (\n node.lineno,\n node.col_offset,\n alias.name,\n module_replacements[alias.name],\n )\n )\n\n # Remove imported symbols from the referenced symbols\n referenced_symbols.difference_update(defined_symbols)\n\n # Sort from last to first so we can modify without messing up the line / col offsets\n inline_updates.sort(reverse=True)\n\n # Nothing to change here.\n if not inline_updates and not referenced_symbols:\n continue\n\n # First do module replacements of spack.pkg imports\n for line, col, old, new in inline_updates:\n updated_lines[line - 1] = updated_lines[line - 1][:col] + updated_lines[line - 1][\n col:\n ].replace(old, new, 1)\n\n # Then insert new imports for symbols referenced in the package\n if referenced_symbols:\n if best_line is None:\n print(f\"{pkg_path}: failed to update imports\", file=err)\n success = False\n continue\n\n # Group missing symbols by their module\n missing_imports_by_module: Dict[str, list] = {}\n for symbol in referenced_symbols:\n module = symbol_to_module[symbol]\n if module not in missing_imports_by_module:\n missing_imports_by_module[module] = []\n missing_imports_by_module[module].append(symbol)\n\n new_lines = [\n f\"from {module} import {', '.join(sorted(symbols))}{newline}\"\n for module, symbols in sorted(missing_imports_by_module.items())\n ]\n\n if not seen_import:\n new_lines.extend((newline, newline))\n\n multiline_updates.append((best_line, best_line, new_lines))\n\n multiline_updates.sort(reverse=True)\n for start, end, new_lines in multiline_updates:\n updated_lines[start - 1 : end - 1] = new_lines\n\n if patch_file:\n rel_pkg_path = os.path.relpath(pkg_path, start=root).replace(os.sep, \"/\")\n diff = difflib.unified_diff(\n original_lines,\n updated_lines,\n n=3,\n fromfile=f\"a/{rel_pkg_path}\",\n tofile=f\"b/{rel_pkg_path}\",\n lineterm=\"\\n\",\n )\n for _line in diff:\n patch_file.write(_line.encode(\"utf-8\"))\n continue\n\n tmp_file = pkg_path + \".tmp\"\n\n # binary mode to avoid newline conversion issues; utf-8 was already required upon read.\n with open(tmp_file, \"wb\") as file:\n for _line in updated_lines:\n file.write(_line.encode(\"utf-8\"))\n\n os.replace(tmp_file, pkg_path)\n\n return success\n" }, { "path": "lib/spack/spack/report.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Tools to produce reports of spec installations or tests\"\"\"\n\nimport collections\nimport gzip\nimport os\nimport time\nimport traceback\n\nimport spack.error\n\nreporter = None\nreport_file = None\n\nProperty = collections.namedtuple(\"Property\", [\"name\", \"value\"])\n\n\nclass Record(dict):\n \"\"\"Data class that provides attr-style access to a dictionary\n\n Attributes beginning with ``_`` are reserved for the Record class itself.\"\"\"\n\n def __getattr__(self, name):\n # only called if no attribute exists\n if name in self:\n return self[name]\n raise AttributeError(f\"Record for {self.name} has no attribute {name}\")\n\n def __setattr__(self, name, value):\n if name.startswith(\"_\"):\n super().__setattr__(name, value)\n else:\n self[name] = value\n\n\nclass RequestRecord(Record):\n \"\"\"Data class for recording outcomes for an entire DAG\n\n Each BuildRequest in the installer and each root spec in a TestSuite generates a\n RequestRecord. The ``packages`` list of the RequestRecord is a list of SpecRecord\n objects recording individual data for each node in the Spec represented by the\n RequestRecord.\n\n These data classes are collated by the reporters in lib/spack/spack/reporters\n \"\"\"\n\n def __init__(self, spec):\n super().__init__()\n self._spec = spec\n self.name = spec.name\n self.nerrors = None\n self.nfailures = None\n self.npackages = None\n self.time = None\n self.timestamp = time.strftime(\"%a, %d %b %Y %H:%M:%S\", time.gmtime())\n self.properties = [\n Property(\"architecture\", spec.architecture)\n # Property(\"compiler\", spec.compiler),\n ]\n self.packages = []\n\n def skip_installed(self):\n \"\"\"Insert records for all nodes in the DAG that are no-ops for this request\"\"\"\n for dep in filter(lambda x: x.installed or x.external, self._spec.traverse()):\n record = InstallRecord(dep)\n record.skip(msg=\"Spec external or already installed\")\n self.packages.append(record)\n\n def append_record(self, record):\n self.packages.append(record)\n\n def summarize(self):\n \"\"\"Construct request-level summaries of the individual records\"\"\"\n self.npackages = len(self.packages)\n self.nfailures = len([r for r in self.packages if r.result == \"failure\"])\n self.nerrors = len([r for r in self.packages if r.result == \"error\"])\n self.time = sum(float(r.elapsed_time or 0.0) for r in self.packages)\n\n\nclass SpecRecord(Record):\n \"\"\"Individual record for a single spec within a request\"\"\"\n\n def __init__(self, spec):\n super().__init__()\n self._spec = spec\n self._package = spec.package\n self._start_time = None\n self.name = spec.name\n self.id = spec.dag_hash()\n self.elapsed_time = None\n\n def start(self):\n self._start_time = time.time()\n\n def skip(self, msg):\n self.result = \"skipped\"\n self.elapsed_time = 0.0\n self.message = msg\n\n def fail(self, exc):\n \"\"\"Record failure based on exception type\n\n Errors wrapped by spack.error.InstallError are \"failures\"\n Other exceptions are \"errors\".\n \"\"\"\n if isinstance(exc, spack.error.InstallError):\n self.result = \"failure\"\n self.message = exc.message or \"Installation failure\"\n self.exception = exc.traceback\n else:\n self.result = \"error\"\n self.message = str(exc) or \"Unknown error\"\n self.exception = traceback.format_exc()\n self.stdout = self.fetch_log() + self.message\n assert self._start_time, \"Start time is None\"\n self.elapsed_time = time.time() - self._start_time\n\n def succeed(self):\n \"\"\"Record success for this spec\"\"\"\n self.result = \"success\"\n self.stdout = self.fetch_log()\n assert self._start_time, \"Start time is None\"\n self.elapsed_time = time.time() - self._start_time\n\n\nclass InstallRecord(SpecRecord):\n \"\"\"Record class with specialization for install logs.\"\"\"\n\n def __init__(self, spec):\n super().__init__(spec)\n self.installed_from_binary_cache = None\n\n def fetch_log(self):\n \"\"\"Install log comes from install prefix on success, or stage dir on failure.\"\"\"\n try:\n if os.path.exists(self._package.install_log_path):\n stream = gzip.open(\n self._package.install_log_path, \"rt\", encoding=\"utf-8\", errors=\"replace\"\n )\n else:\n stream = open(self._package.log_path, encoding=\"utf-8\", errors=\"replace\")\n with stream as f:\n return f.read()\n except OSError:\n return f\"Cannot open log for {self._spec.cshort_spec}\"\n\n def succeed(self):\n super().succeed()\n self.installed_from_binary_cache = self._package.installed_from_binary_cache\n\n\nclass NullInstallRecord(InstallRecord):\n \"\"\"No-op drop-in for InstallRecord when no reporter is configured.\n\n Avoids reading log files from disk on every completed build.\"\"\"\n\n def start(self) -> None:\n pass\n\n def succeed(self) -> None:\n pass\n\n def fail(self, exc) -> None:\n pass\n\n def skip(self, msg: str = \"\") -> None:\n pass\n\n\nclass NullRequestRecord(RequestRecord):\n \"\"\"No-op drop-in for RequestRecord when no reporter is configured.\n\n Avoids traversing the DAG and accumulating data that will not be reported.\"\"\"\n\n def __init__(self) -> None:\n dict.__init__(self)\n\n def skip_installed(self) -> None:\n pass\n\n def append_record(self, record) -> None:\n pass\n\n def summarize(self) -> None:\n pass\n\n\nclass TestRecord(SpecRecord):\n \"\"\"Record class with specialization for test logs.\"\"\"\n\n def __init__(self, spec, directory):\n super().__init__(spec)\n self.directory = directory\n\n def fetch_log(self):\n \"\"\"Get output from test log\"\"\"\n log_file = os.path.join(self.directory, self._package.test_suite.test_log_name(self._spec))\n try:\n with open(log_file, \"r\", encoding=\"utf-8\", errors=\"replace\") as stream:\n return \"\".join(stream.readlines())\n except Exception:\n return f\"Cannot open log for {self._spec.cshort_spec}\"\n\n def succeed(self, externals):\n \"\"\"Test reports skip externals by default.\"\"\"\n if self._spec.external and not externals:\n return self.skip(msg=\"Skipping test of external package\")\n\n super().succeed()\n" }, { "path": "lib/spack/spack/reporters/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nfrom .base import Reporter\nfrom .cdash import CDash, CDashConfiguration\nfrom .junit import JUnit\n\n__all__ = [\"JUnit\", \"CDash\", \"CDashConfiguration\", \"Reporter\"]\n" }, { "path": "lib/spack/spack/reporters/base.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nfrom typing import Any, Dict, List\n\n\nclass Reporter:\n \"\"\"Base class for report writers.\"\"\"\n\n def build_report(self, filename: str, specs: List[Dict[str, Any]]):\n raise NotImplementedError(\"must be implemented by derived classes\")\n\n def test_report(self, filename: str, specs: List[Dict[str, Any]]):\n raise NotImplementedError(\"must be implemented by derived classes\")\n\n def concretization_report(self, filename: str, msg: str):\n raise NotImplementedError(\"must be implemented by derived classes\")\n" }, { "path": "lib/spack/spack/reporters/cdash.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport collections\nimport hashlib\nimport io\nimport os\nimport platform\nimport posixpath\nimport re\nimport socket\nimport time\nimport warnings\nimport xml.sax.saxutils\nfrom typing import Dict, Optional\nfrom urllib.parse import urlencode\nfrom urllib.request import Request\n\nimport spack\nimport spack.llnl.util.tty as tty\nimport spack.paths\nimport spack.platforms\nimport spack.spec\nimport spack.tengine\nimport spack.util.git\nimport spack.util.web as web_util\nfrom spack.error import SpackError\nfrom spack.llnl.util.filesystem import working_dir\nfrom spack.util.crypto import checksum\nfrom spack.util.log_parse import parse_log_events\n\nfrom .base import Reporter\nfrom .extract import extract_test_parts\n\n# Mapping Spack phases to the corresponding CTest/CDash phase.\n# TODO: Some of the phases being lumped into configure in the CDash tables\n# TODO: really belong in a separate column, such as \"Setup\".\n# TODO: Would also be nice to have `stage` as a separate phase that could\n# TODO: be lumped into that new column instead of configure, for example.\nMAP_PHASES_TO_CDASH = {\n \"autoreconf\": \"configure\", # AutotoolsBuilder\n \"bootstrap\": \"configure\", # CMakeBuilder\n \"build\": \"build\",\n \"build_processes\": \"build\", # Openloops\n \"cmake\": \"configure\", # CMakeBuilder\n \"configure\": \"configure\",\n \"edit\": \"configure\", # MakefileBuilder\n \"generate_luarocks_config\": \"configure\", # LuaBuilder\n \"hostconfig\": \"configure\", # Lvarray\n \"initconfig\": \"configure\", # CachedCMakeBuilder\n \"install\": \"build\",\n \"meson\": \"configure\", # MesonBuilder\n \"preprocess\": \"configure\", # LuaBuilder\n \"qmake\": \"configure\", # QMakeBuilder\n \"unpack\": \"configure\", # LuaBuilder\n}\n\n# Initialize data structures common to each phase's report.\nCDASH_PHASES = set(MAP_PHASES_TO_CDASH.values())\nCDASH_PHASES.add(\"update\")\n# CDash request timeout in seconds\nSPACK_CDASH_TIMEOUT = 45\n\nCDashConfiguration = collections.namedtuple(\n \"CDashConfiguration\", [\"upload_url\", \"packages\", \"build\", \"site\", \"buildstamp\", \"track\"]\n)\n\n\ndef build_stamp(track, timestamp):\n buildstamp_format = \"%Y%m%d-%H%M-{0}\".format(track)\n return time.strftime(buildstamp_format, time.localtime(timestamp))\n\n\nclass CDash(Reporter):\n \"\"\"Generate reports of spec installations for CDash.\n\n To use this reporter, pass the ``--cdash-upload-url`` argument to\n ``spack install``::\n\n spack install --cdash-upload-url=\\\\\n https://example.com/cdash/submit.php?project=Spack \n\n In this example, results will be uploaded to the *Spack* project on the\n CDash instance hosted at ``https://example.com/cdash``.\n \"\"\"\n\n def __init__(self, configuration: CDashConfiguration):\n #: Set to False if any error occurs when building the CDash report\n self.success = True\n\n # Jinja2 expects `/` path separators\n self.template_dir = \"reports/cdash\"\n self.cdash_upload_url = configuration.upload_url\n\n if self.cdash_upload_url:\n self.buildid_regexp = re.compile(\"([0-9]+)\")\n self.phase_regexp = re.compile(r\"Executing phase: '(.*)'\")\n\n self.authtoken = None\n if \"SPACK_CDASH_AUTH_TOKEN\" in os.environ:\n tty.verbose(\"Using CDash auth token from environment\")\n self.authtoken = os.environ.get(\"SPACK_CDASH_AUTH_TOKEN\")\n\n self.install_command = \" \".join(configuration.packages)\n self.base_buildname = configuration.build or self.install_command\n self.site = configuration.site or socket.gethostname()\n self.osname = platform.system()\n self.osrelease = platform.release()\n self.target = spack.platforms.host().default_target()\n self.starttime = int(time.time())\n self.endtime = self.starttime\n self.buildstamp = (\n configuration.buildstamp\n if configuration.buildstamp\n else build_stamp(configuration.track, self.starttime)\n )\n self.buildIds: Dict[str, str] = {}\n self.revision = \"\"\n git = spack.util.git.git(required=True)\n with working_dir(spack.paths.spack_root):\n self.revision = git(\"rev-parse\", \"HEAD\", output=str).strip()\n self.generator = \"spack-{0}\".format(spack.get_version())\n self.multiple_packages = False\n\n def report_build_name(self, pkg_name):\n buildname = (\n \"{0} - {1}\".format(self.base_buildname, pkg_name)\n if self.multiple_packages\n else self.base_buildname\n )\n if len(buildname) > 190:\n warnings.warn(\"Build name exceeds CDash 190 character maximum and will be truncated.\")\n buildname = buildname[:190]\n return buildname\n\n def build_report_for_package(self, report_dir, package, duration):\n if \"stdout\" not in package:\n # Skip reporting on packages that do not generate output.\n return\n\n self.current_package_name = package[\"name\"]\n self.buildname = self.report_build_name(self.current_package_name)\n report_data = self.initialize_report(report_dir)\n for phase in CDASH_PHASES:\n report_data[phase] = {}\n report_data[phase][\"loglines\"] = []\n report_data[phase][\"status\"] = 0\n report_data[phase][\"starttime\"] = self.starttime\n\n # Track the phases we perform so we know what reports to create.\n # We always report the update step because this is how we tell CDash\n # what revision of Spack we are using.\n phases_encountered = [\"update\"]\n\n # Generate a report for this package.\n current_phase = \"\"\n cdash_phase = \"\"\n for line in package[\"stdout\"].splitlines():\n match = None\n if line.find(\"Executing phase: '\") != -1:\n match = self.phase_regexp.search(line)\n if match:\n current_phase = match.group(1)\n if current_phase not in MAP_PHASES_TO_CDASH:\n current_phase = \"\"\n continue\n cdash_phase = MAP_PHASES_TO_CDASH[current_phase]\n if cdash_phase not in phases_encountered:\n phases_encountered.append(cdash_phase)\n report_data[cdash_phase][\"loglines\"].append(\n str(\"{0} output for {1}:\".format(cdash_phase, package[\"name\"]))\n )\n elif cdash_phase:\n report_data[cdash_phase][\"loglines\"].append(xml.sax.saxutils.escape(line))\n\n # something went wrong pre-cdash \"configure\" phase b/c we have an exception and only\n # \"update\" was encountered.\n # dump the report in the configure line so teams can see what the issue is\n if len(phases_encountered) == 1 and package.get(\"exception\"):\n # TODO this mapping is not ideal since these are pre-configure errors\n # we need to determine if a more appropriate cdash phase can be utilized\n # for now we will add a message to the log explaining this\n cdash_phase = \"configure\"\n phases_encountered.append(cdash_phase)\n\n log_message = (\n \"Pre-configure errors occurred in Spack's process that terminated the \"\n \"build process prematurely.\\nSpack output::\\n{0}\".format(\n xml.sax.saxutils.escape(package[\"exception\"])\n )\n )\n\n report_data[cdash_phase][\"loglines\"].append(log_message)\n\n # Move the build phase to the front of the list if it occurred.\n # This supports older versions of CDash that expect this phase\n # to be reported before all others.\n if \"build\" in phases_encountered:\n build_pos = phases_encountered.index(\"build\")\n phases_encountered.insert(0, phases_encountered.pop(build_pos))\n\n self.endtime = self.starttime + duration\n for phase in phases_encountered:\n report_data[phase][\"endtime\"] = self.endtime\n report_data[phase][\"log\"] = \"\\n\".join(report_data[phase][\"loglines\"])\n errors, warnings = parse_log_events(report_data[phase][\"loglines\"])\n\n # Convert errors to warnings if the package reported success.\n if package[\"result\"] == \"success\":\n warnings = errors + warnings\n errors = []\n\n # Cap the number of errors and warnings at 50 each.\n errors = errors[:50]\n warnings = warnings[:50]\n nerrors = len(errors)\n\n if nerrors > 0:\n self.success = False\n if phase == \"configure\":\n report_data[phase][\"status\"] = 1\n\n if phase == \"build\":\n # Convert log output from ASCII to Unicode and escape for XML.\n def clean_log_event(event):\n event = vars(event)\n event[\"text\"] = xml.sax.saxutils.escape(event[\"text\"])\n event[\"pre_context\"] = xml.sax.saxutils.escape(\"\\n\".join(event[\"pre_context\"]))\n event[\"post_context\"] = xml.sax.saxutils.escape(\n \"\\n\".join(event[\"post_context\"])\n )\n # source_file and source_line_no are either strings or\n # the tuple (None,). Distinguish between these two cases.\n if event[\"source_file\"][0] is None:\n event[\"source_file\"] = \"\"\n event[\"source_line_no\"] = \"\"\n else:\n event[\"source_file\"] = xml.sax.saxutils.escape(event[\"source_file\"])\n return event\n\n report_data[phase][\"errors\"] = []\n report_data[phase][\"warnings\"] = []\n for error in errors:\n report_data[phase][\"errors\"].append(clean_log_event(error))\n for warning in warnings:\n report_data[phase][\"warnings\"].append(clean_log_event(warning))\n\n if phase == \"update\":\n report_data[phase][\"revision\"] = self.revision\n\n # Write the report.\n report_name = phase.capitalize() + \".xml\"\n if self.multiple_packages:\n report_file_name = package[\"name\"] + \"_\" + report_name\n else:\n report_file_name = report_name\n phase_report = os.path.join(report_dir, report_file_name)\n\n with open(phase_report, \"w\", encoding=\"utf-8\") as f:\n env = spack.tengine.make_environment()\n if phase != \"update\":\n # Update.xml stores site information differently\n # than the rest of the CTest XML files.\n site_template = posixpath.join(self.template_dir, \"Site.xml\")\n t = env.get_template(site_template)\n f.write(t.render(report_data))\n\n phase_template = posixpath.join(self.template_dir, report_name)\n t = env.get_template(phase_template)\n f.write(t.render(report_data))\n self.upload(phase_report)\n\n def build_report(self, report_dir, specs):\n # Do an initial scan to determine if we are generating reports for more\n # than one package. When we're only reporting on a single package we\n # do not explicitly include the package's name in the CDash build name.\n self.multiple_packages = False\n num_packages = 0\n for spec in specs:\n spec.summarize()\n\n # Do not generate reports for packages that were installed\n # from the binary cache.\n spec[\"packages\"] = [\n x\n for x in spec[\"packages\"]\n if \"installed_from_binary_cache\" not in x or not x[\"installed_from_binary_cache\"]\n ]\n for package in spec[\"packages\"]:\n if \"stdout\" in package:\n num_packages += 1\n if num_packages > 1:\n self.multiple_packages = True\n break\n if self.multiple_packages:\n break\n\n # Generate reports for each package in each spec.\n for spec in specs:\n duration = 0\n if \"time\" in spec:\n duration = int(spec[\"time\"])\n for package in spec[\"packages\"]:\n self.build_report_for_package(report_dir, package, duration)\n self.finalize_report()\n\n def extract_standalone_test_data(self, package, phases, report_data):\n \"\"\"Extract stand-alone test outputs for the package.\"\"\"\n\n testing = {}\n report_data[\"testing\"] = testing\n testing[\"starttime\"] = self.starttime\n testing[\"endtime\"] = self.starttime\n testing[\"generator\"] = self.generator\n testing[\"parts\"] = extract_test_parts(package[\"name\"], package[\"stdout\"].splitlines())\n\n def report_test_data(self, report_dir, package, phases, report_data):\n \"\"\"Generate and upload the test report(s) for the package.\"\"\"\n for phase in phases:\n # Write the report.\n report_name = phase.capitalize() + \".xml\"\n report_file_name = \"_\".join([package[\"name\"], package[\"id\"], report_name])\n phase_report = os.path.join(report_dir, report_file_name)\n\n with open(phase_report, \"w\", encoding=\"utf-8\") as f:\n env = spack.tengine.make_environment()\n if phase not in [\"update\", \"testing\"]:\n # Update.xml stores site information differently\n # than the rest of the CTest XML files.\n site_template = posixpath.join(self.template_dir, \"Site.xml\")\n t = env.get_template(site_template)\n f.write(t.render(report_data))\n\n phase_template = posixpath.join(self.template_dir, report_name)\n t = env.get_template(phase_template)\n f.write(t.render(report_data))\n\n tty.debug(\"Preparing to upload {0}\".format(phase_report))\n self.upload(phase_report)\n\n def test_report_for_package(self, report_dir, package, duration):\n if \"stdout\" not in package:\n # Skip reporting on packages that did not generate any output.\n tty.debug(\"Skipping report for {0}: No generated output\".format(package[\"name\"]))\n return\n\n self.current_package_name = package[\"name\"]\n if self.base_buildname == self.install_command:\n # The package list is NOT all that helpful in this case\n self.buildname = \"{0}-{1}\".format(self.current_package_name, package[\"id\"])\n else:\n self.buildname = self.report_build_name(self.current_package_name)\n self.endtime = self.starttime + duration\n\n report_data = self.initialize_report(report_dir)\n report_data[\"hostname\"] = socket.gethostname()\n phases = [\"testing\"]\n self.extract_standalone_test_data(package, phases, report_data)\n\n self.report_test_data(report_dir, package, phases, report_data)\n\n def test_report(self, report_dir, specs):\n \"\"\"Generate reports for each package in each spec.\"\"\"\n tty.debug(\"Processing test report\")\n for spec in specs:\n spec.summarize()\n\n duration = 0\n if \"time\" in spec:\n duration = int(spec[\"time\"])\n for package in spec[\"packages\"]:\n self.test_report_for_package(report_dir, package, duration)\n\n self.finalize_report()\n\n def test_skipped_report(\n self, report_dir: str, spec: spack.spec.Spec, reason: Optional[str] = None\n ):\n \"\"\"Explicitly report spec as being skipped (e.g., CI).\n\n Examples are the installation failed or the package is known to have\n broken tests.\n\n Args:\n report_dir: directory where the report is to be written\n spec: spec being tested\n reason: optional reason the test is being skipped\n \"\"\"\n output = \"Skipped {0} package\".format(spec.name)\n if reason:\n output += \"\\n{0}\".format(reason)\n\n package = {\"name\": spec.name, \"id\": spec.dag_hash(), \"result\": \"skipped\", \"stdout\": output}\n self.test_report_for_package(report_dir, package, duration=0.0)\n\n def concretization_report(self, report_dir, msg):\n self.buildname = self.base_buildname\n report_data = self.initialize_report(report_dir)\n report_data[\"update\"] = {}\n report_data[\"update\"][\"starttime\"] = self.starttime\n report_data[\"update\"][\"endtime\"] = self.endtime\n report_data[\"update\"][\"revision\"] = self.revision\n report_data[\"update\"][\"log\"] = msg\n\n env = spack.tengine.make_environment()\n update_template = posixpath.join(self.template_dir, \"Update.xml\")\n t = env.get_template(update_template)\n output_filename = os.path.join(report_dir, \"Update.xml\")\n with open(output_filename, \"w\", encoding=\"utf-8\") as f:\n f.write(t.render(report_data))\n # We don't have a current package when reporting on concretization\n # errors so refer to this report with the base buildname instead.\n self.current_package_name = self.base_buildname\n self.upload(output_filename)\n self.success = False\n self.finalize_report()\n\n def initialize_report(self, report_dir):\n if not os.path.exists(report_dir):\n os.mkdir(report_dir)\n report_data = {}\n report_data[\"buildname\"] = self.buildname\n report_data[\"buildstamp\"] = self.buildstamp\n report_data[\"install_command\"] = self.install_command\n report_data[\"generator\"] = self.generator\n report_data[\"osname\"] = self.osname\n report_data[\"osrelease\"] = self.osrelease\n report_data[\"site\"] = self.site\n report_data[\"target\"] = self.target\n return report_data\n\n def upload(self, filename):\n if not self.cdash_upload_url:\n print(\"Cannot upload {0} due to missing upload url\".format(filename))\n return\n\n # Compute md5 checksum for the contents of this file.\n md5sum = checksum(hashlib.md5, filename, block_size=8192)\n\n with open(filename, \"rb\") as f:\n params_dict = {\n \"build\": self.buildname,\n \"site\": self.site,\n \"stamp\": self.buildstamp,\n \"MD5\": md5sum,\n }\n encoded_params = urlencode(params_dict)\n url = \"{0}&{1}\".format(self.cdash_upload_url, encoded_params)\n request = Request(url, data=f, method=\"PUT\")\n request.add_header(\"Content-Type\", \"text/xml\")\n request.add_header(\"Content-Length\", os.path.getsize(filename))\n if self.authtoken:\n request.add_header(\"Authorization\", \"Bearer {0}\".format(self.authtoken))\n try:\n with web_util.urlopen(request, timeout=SPACK_CDASH_TIMEOUT) as response:\n if self.current_package_name not in self.buildIds:\n resp_value = io.TextIOWrapper(response, encoding=\"utf-8\").read()\n match = self.buildid_regexp.search(resp_value)\n if match:\n buildid = match.group(1)\n self.buildIds[self.current_package_name] = buildid\n except Exception as e:\n print(f\"Upload to CDash failed: {e}\")\n\n def finalize_report(self):\n if self.buildIds:\n tty.msg(\"View your build results here:\")\n for package_name, buildid in self.buildIds.items():\n # Construct and display a helpful link if CDash responded with\n # a buildId.\n build_url = self.cdash_upload_url\n build_url = build_url[0 : build_url.find(\"submit.php\")]\n build_url += \"buildSummary.php?buildid={0}\".format(buildid)\n tty.msg(\"{0}: {1}\".format(package_name, build_url))\n if not self.success:\n raise SpackError(\"Errors encountered, see above for more details\")\n" }, { "path": "lib/spack/spack/reporters/extract.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport re\nimport xml.sax.saxutils\nfrom datetime import datetime\n\nimport spack.llnl.util.tty as tty\nfrom spack.install_test import TestStatus\n\n# The keys here represent the only recognized (ctest/cdash) status values\ncompleted = {\n \"failed\": \"Completed\",\n \"passed\": \"Completed\",\n \"skipped\": \"Completed\",\n \"notrun\": \"No tests to run\",\n}\n\nlog_regexp = re.compile(r\"^==> \\[([0-9:.\\-]*)(?:, [0-9]*)?\\] (.*)\")\nreturns_regexp = re.compile(r\"\\[([0-9 ,]*)\\]\")\n\nskip_msgs = [\"Testing package\", \"Results for\", \"Detected the following\", \"Warning:\"]\nskip_regexps = [re.compile(r\"{0}\".format(msg)) for msg in skip_msgs]\n\nstatus_regexps = [re.compile(r\"^({0})\".format(str(stat))) for stat in TestStatus]\n\n\ndef add_part_output(part, line):\n if part:\n part[\"loglines\"].append(xml.sax.saxutils.escape(line))\n\n\ndef elapsed(current, previous):\n if not (current and previous):\n return 0\n\n diff = current - previous\n tty.debug(\"elapsed = %s - %s = %s\" % (current, previous, diff))\n return diff.total_seconds()\n\n\ndef new_part():\n return {\n \"command\": None,\n \"completed\": \"Unknown\",\n \"desc\": None,\n \"elapsed\": None,\n \"name\": None,\n \"loglines\": [],\n \"output\": None,\n \"status\": None,\n }\n\n\ndef process_part_end(part, curr_time, last_time):\n if part:\n if not part[\"elapsed\"]:\n part[\"elapsed\"] = elapsed(curr_time, last_time)\n\n stat = part[\"status\"]\n if stat in completed:\n if part[\"completed\"] == \"Unknown\":\n part[\"completed\"] = completed[stat]\n elif stat is None or stat == \"unknown\":\n part[\"status\"] = \"passed\"\n part[\"output\"] = \"\\n\".join(part[\"loglines\"])\n\n\ndef timestamp(time_string):\n return datetime.strptime(time_string, \"%Y-%m-%d-%H:%M:%S.%f\")\n\n\ndef skip(line):\n for regex in skip_regexps:\n match = regex.search(line)\n if match:\n return match\n\n\ndef status(line):\n for regex in status_regexps:\n match = regex.search(line)\n if match:\n stat = match.group(0)\n stat = \"notrun\" if stat == \"NO_TESTS\" else stat\n return stat.lower()\n\n\ndef extract_test_parts(default_name, outputs):\n parts = []\n part = {}\n last_time = None\n curr_time = None\n\n for line in outputs:\n line = line.strip()\n if not line:\n add_part_output(part, line)\n continue\n\n if skip(line):\n continue\n\n # The spec was explicitly reported as skipped (e.g., installation\n # failed, package known to have failing tests, won't test external\n # package).\n if line.startswith(\"Skipped\") and line.endswith(\"package\"):\n stat = \"skipped\"\n part = new_part()\n part[\"command\"] = \"Not Applicable\"\n part[\"completed\"] = completed[stat]\n part[\"elapsed\"] = 0.0\n part[\"loglines\"].append(line)\n part[\"name\"] = default_name\n part[\"status\"] = \"notrun\"\n parts.append(part)\n continue\n\n # Process Spack log messages\n if line.find(\"==>\") != -1:\n match = log_regexp.search(line)\n if match:\n curr_time = timestamp(match.group(1))\n msg = match.group(2)\n\n # Skip logged message for caching build-time data\n if msg.startswith(\"Installing\"):\n continue\n\n # Terminate without further parsing if no more test messages\n if \"Completed testing\" in msg:\n # Process last lingering part IF it didn't generate status\n process_part_end(part, curr_time, last_time)\n return parts\n\n # New test parts start \"test: : \".\n if msg.startswith(\"test: \"):\n # Update the last part processed\n process_part_end(part, curr_time, last_time)\n\n part = new_part()\n desc = msg.split(\":\")\n part[\"name\"] = desc[1].strip()\n part[\"desc\"] = \":\".join(desc[2:]).strip()\n parts.append(part)\n\n # There is no guarantee of a 1-to-1 mapping of a test part and\n # a (single) command (or executable) since the introduction of\n # PR 34236.\n #\n # Note that tests where the package does not save the output\n # (e.g., output=str.split, error=str.split) will not have\n # a command printed to the test log.\n elif msg.startswith(\"'\") and msg.endswith(\"'\"):\n if part:\n if part[\"command\"]:\n part[\"command\"] += \"; \" + msg.replace(\"'\", \"\")\n else:\n part[\"command\"] = msg.replace(\"'\", \"\")\n else:\n part = new_part()\n part[\"command\"] = msg.replace(\"'\", \"\")\n\n else:\n # Update the last part processed since a new log message\n # means a non-test action\n process_part_end(part, curr_time, last_time)\n\n else:\n tty.debug(\"Did not recognize test output '{0}'\".format(line))\n\n # Each log message potentially represents a new test part so\n # save off the last timestamp\n last_time = curr_time\n continue\n\n # Check for status values\n stat = status(line)\n if stat:\n if part:\n part[\"status\"] = stat\n add_part_output(part, line)\n else:\n tty.warn(\"No part to add status from '{0}'\".format(line))\n continue\n\n add_part_output(part, line)\n\n # Process the last lingering part IF it didn't generate status\n process_part_end(part, curr_time, last_time)\n\n # If no parts, create a skeleton to flag that the tests are not run\n if not parts:\n part = new_part()\n stat = \"failed\" if outputs[0].startswith(\"Cannot open log\") else \"notrun\"\n\n part[\"command\"] = \"unknown\"\n part[\"completed\"] = completed[stat]\n part[\"elapsed\"] = 0.0\n part[\"name\"] = default_name\n part[\"status\"] = stat\n part[\"output\"] = \"\\n\".join(outputs)\n parts.append(part)\n\n return parts\n" }, { "path": "lib/spack/spack/reporters/junit.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport os\n\nimport spack.tengine\n\nfrom .base import Reporter\n\n\nclass JUnit(Reporter):\n \"\"\"Generate reports of spec installations for JUnit.\"\"\"\n\n _jinja_template = \"reports/junit.xml\"\n\n def concretization_report(self, filename, msg):\n pass\n\n def build_report(self, filename, specs):\n for spec in specs:\n spec.summarize()\n\n if not (os.path.splitext(filename))[1]:\n # Ensure the report name will end with the proper extension;\n # otherwise, it currently defaults to the \"directory\" name.\n filename = filename + \".xml\"\n\n report_data = {\"specs\": specs}\n\n with open(filename, \"w\", encoding=\"utf-8\") as f:\n env = spack.tengine.make_environment()\n t = env.get_template(self._jinja_template)\n f.write(t.render(report_data))\n\n def test_report(self, filename, specs):\n self.build_report(filename, specs)\n" }, { "path": "lib/spack/spack/resource.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Describes an optional resource needed for a build.\n\nTypically a bunch of sources that can be built in-tree within another\npackage to enable optional features.\n\n\"\"\"\n\n\nclass Resource:\n \"\"\"Represents any resource to be fetched by a package.\n\n This includes the main tarball or source archive, as well as extra archives defined\n by the resource() directive.\n\n Aggregates a name, a fetcher, a destination and a placement.\n \"\"\"\n\n def __init__(self, name, fetcher, destination, placement):\n self.name = name\n self.fetcher = fetcher\n self.destination = destination\n self.placement = placement\n" }, { "path": "lib/spack/spack/rewiring.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport tempfile\n\nimport spack.binary_distribution\nimport spack.error\nimport spack.hooks\nimport spack.store\n\n\ndef rewire(spliced_spec):\n \"\"\"Given a spliced spec, this function conducts all the rewiring on all\n nodes in the DAG of that spec.\"\"\"\n assert spliced_spec.spliced\n for spec in spliced_spec.traverse(order=\"post\", root=True):\n if not spec.build_spec.installed:\n # TODO: May want to change this at least for the root spec...\n # TODO: Also remember to import PackageInstaller\n # PackageInstaller([spec.build_spec.package]).install()\n raise PackageNotInstalledError(spliced_spec, spec.build_spec, spec)\n if spec.build_spec is not spec and not spec.installed:\n explicit = spec is spliced_spec\n rewire_node(spec, explicit)\n\n\ndef rewire_node(spec, explicit):\n \"\"\"This function rewires a single node, worrying only about references to\n its subgraph. Binaries, text, and links are all changed in accordance with\n the splice. The resulting package is then 'installed.'\"\"\"\n tempdir = tempfile.mkdtemp()\n\n # Copy spec.build_spec.prefix to spec.prefix through a temporary tarball\n tarball = os.path.join(tempdir, f\"{spec.dag_hash()}.tar.gz\")\n spack.binary_distribution.create_tarball(spec.build_spec, tarball)\n\n spack.hooks.pre_install(spec)\n spack.binary_distribution.extract_buildcache_tarball(tarball, destination=spec.prefix)\n spack.binary_distribution.relocate_package(spec)\n\n # run post install hooks and add to db\n spack.hooks.post_install(spec, explicit)\n spack.store.STORE.db.add(spec, explicit=explicit)\n\n\nclass RewireError(spack.error.SpackError):\n \"\"\"Raised when something goes wrong with rewiring.\"\"\"\n\n def __init__(self, message, long_msg=None):\n super().__init__(message, long_msg)\n\n\nclass PackageNotInstalledError(RewireError):\n \"\"\"Raised when the build_spec for a splice was not installed.\"\"\"\n\n def __init__(self, spliced_spec, build_spec, dep):\n super().__init__(\n \"\"\"Rewire of {0}\n failed due to missing install of build spec {1}\n for spec {2}\"\"\".format(spliced_spec, build_spec, dep)\n )\n" }, { "path": "lib/spack/spack/schema/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"This module contains jsonschema files for all of Spack's YAML formats.\"\"\"\n\nimport copy\nimport typing\nimport warnings\n\nfrom spack.vendor import jsonschema\nfrom spack.vendor.jsonschema import validators\n\nfrom spack.error import SpecSyntaxError\n\n\nclass DeprecationMessage(typing.NamedTuple):\n message: str\n error: bool\n\n\ndef _validate_spec(validator, is_spec, instance, schema):\n \"\"\"Check if all additional keys are valid specs.\"\"\"\n import spack.spec_parser\n\n if not validator.is_type(instance, \"object\"):\n return\n\n properties = schema.get(\"properties\") or {}\n\n for spec_str in instance:\n if spec_str in properties:\n continue\n try:\n spack.spec_parser.parse(spec_str)\n except SpecSyntaxError:\n yield jsonschema.ValidationError(f\"the key '{spec_str}' is not a valid spec\")\n\n\ndef _deprecated_properties(validator, deprecated, instance, schema):\n if not (validator.is_type(instance, \"object\") or validator.is_type(instance, \"array\")):\n return\n\n if not deprecated:\n return\n\n deprecations = {\n name: DeprecationMessage(message=x[\"message\"], error=x[\"error\"])\n for x in deprecated\n for name in x[\"names\"]\n }\n\n # Get a list of the deprecated properties, return if there is none\n issues = [entry for entry in instance if entry in deprecations]\n if not issues:\n return\n\n # Process issues\n errors = []\n for name in issues:\n msg = deprecations[name].message.format(name=name)\n if deprecations[name].error:\n errors.append(msg)\n else:\n warnings.warn(msg)\n\n if errors:\n yield jsonschema.ValidationError(\"\\n\".join(errors))\n\n\nValidator = validators.extend(\n jsonschema.Draft7Validator,\n {\"additionalKeysAreSpecs\": _validate_spec, \"deprecatedProperties\": _deprecated_properties},\n)\n\n\ndef _append(string: str) -> bool:\n \"\"\"Test if a spack YAML string is an append.\n\n See ``spack_yaml`` for details. Keys in Spack YAML can end in ``+:``,\n and if they do, their values append lower-precedence\n configs.\n\n str, str : concatenate strings.\n [obj], [obj] : append lists.\n\n \"\"\"\n return getattr(string, \"append\", False)\n\n\ndef _prepend(string: str) -> bool:\n \"\"\"Test if a spack YAML string is an prepend.\n\n See ``spack_yaml`` for details. Keys in Spack YAML can end in ``+:``,\n and if they do, their values prepend lower-precedence\n configs.\n\n str, str : concatenate strings.\n [obj], [obj] : prepend lists. (default behavior)\n \"\"\"\n return getattr(string, \"prepend\", False)\n\n\ndef override(string: str) -> bool:\n \"\"\"Test if a spack YAML string is an override.\n\n See ``spack_yaml`` for details. Keys in Spack YAML can end in ``::``,\n and if they do, their values completely replace lower-precedence\n configs instead of merging into them.\n\n \"\"\"\n return hasattr(string, \"override\") and string.override\n\n\ndef merge_yaml(dest, source, prepend=False, append=False):\n \"\"\"Merges source into dest; entries in source take precedence over dest.\n\n This routine may modify dest and should be assigned to dest, in\n case dest was None to begin with, e.g.::\n\n dest = merge_yaml(dest, source)\n\n In the result, elements from lists from ``source`` will appear before\n elements of lists from ``dest``. Likewise, when iterating over keys\n or items in merged ``OrderedDict`` objects, keys from ``source`` will\n appear before keys from ``dest``.\n\n Config file authors can optionally end any attribute in a dict\n with ``::`` instead of ``:``, and the key will override that of the\n parent instead of merging.\n\n ``+:`` will extend the default prepend merge strategy to include string concatenation\n ``-:`` will change the merge strategy to append, it also includes string concatenation\n \"\"\"\n\n def they_are(t):\n return isinstance(dest, t) and isinstance(source, t)\n\n # If source is None, overwrite with source.\n if source is None:\n return None\n\n # Source list is prepended (for precedence)\n if they_are(list):\n if append:\n # Make sure to copy ruamel comments\n dest[:] = [x for x in dest if x not in source] + source\n else:\n # Make sure to copy ruamel comments\n dest[:] = source + [x for x in dest if x not in source]\n return dest\n\n # Source dict is merged into dest.\n elif they_are(dict):\n # save dest keys to reinsert later -- this ensures that source items\n # come *before* dest in OrderdDicts\n dest_keys = [dk for dk in dest.keys() if dk not in source]\n\n for sk, sv in source.items():\n # always remove the dest items. Python dicts do not overwrite\n # keys on insert, so this ensures that source keys are copied\n # into dest along with mark provenance (i.e., file/line info).\n merge = sk in dest\n old_dest_value = dest.pop(sk, None)\n\n if merge and not override(sk):\n dest[sk] = merge_yaml(old_dest_value, sv, _prepend(sk), _append(sk))\n else:\n # if sk ended with ::, or if it's new, completely override\n dest[sk] = copy.deepcopy(sv)\n\n # reinsert dest keys so they are last in the result\n for dk in dest_keys:\n dest[dk] = dest.pop(dk)\n\n return dest\n\n elif they_are(str):\n # Concatenate strings in prepend mode\n if prepend:\n return source + dest\n elif append:\n return dest + source\n\n # If we reach here source and dest are either different types or are\n # not both lists or dicts: replace with source.\n return copy.copy(source)\n" }, { "path": "lib/spack/spack/schema/bootstrap.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Schema for bootstrap.yaml configuration file.\"\"\"\n\nfrom typing import Any, Dict\n\n#: Schema of a single source\n_source_schema: Dict[str, Any] = {\n \"type\": \"object\",\n \"description\": \"Bootstrap source configuration\",\n \"properties\": {\n \"name\": {\n \"type\": \"string\",\n \"description\": \"Name of the bootstrap source (e.g., 'github-actions-v0.6', \"\n \"'spack-install')\",\n },\n \"metadata\": {\n \"type\": \"string\",\n \"description\": \"Path to metadata directory containing bootstrap source configuration\",\n },\n },\n \"additionalProperties\": False,\n \"required\": [\"name\", \"metadata\"],\n}\n\n#: schema for dev bootstrap configuration\n_dev_schema: Dict[str, Any] = {\n \"type\": \"object\",\n \"description\": \"Dev Bootstrap configuration\",\n \"properties\": {\n \"enable_source\": {\n \"type\": \"boolean\",\n \"description\": \"Enable bootstrapping dev dependencies from source\",\n }\n },\n}\n\nproperties: Dict[str, Any] = {\n \"bootstrap\": {\n \"type\": \"object\",\n \"description\": \"Configure how Spack bootstraps its own dependencies when needed\",\n \"properties\": {\n \"enable\": {\n \"type\": \"boolean\",\n \"description\": \"Enable or disable bootstrapping entirely\",\n },\n \"root\": {\n \"type\": \"string\",\n \"description\": \"Where to install bootstrapped dependencies\",\n },\n \"sources\": {\n \"type\": \"array\",\n \"items\": _source_schema,\n \"description\": \"List of bootstrap sources tried in order. Each method may \"\n \"bootstrap different software depending on its type (e.g., pre-built binaries, \"\n \"source builds)\",\n },\n \"trusted\": {\n \"type\": \"object\",\n \"additionalProperties\": {\"type\": \"boolean\"},\n \"description\": \"Controls which sources are enabled for automatic bootstrapping\",\n },\n \"dev\": _dev_schema,\n },\n }\n}\n\n#: Full schema with metadata\nschema = {\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Spack bootstrap configuration file schema\",\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": properties,\n}\n" }, { "path": "lib/spack/spack/schema/buildcache_spec.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Schema for a buildcache spec.yaml file\n\n.. literalinclude:: _spack_root/lib/spack/spack/schema/buildcache_spec.py\n :lines: 15-\n\"\"\"\n\nfrom typing import Any, Dict\n\nimport spack.schema.spec\n\nproperties: Dict[str, Any] = {\n # `buildinfo` is no longer needed as of Spack 0.21\n \"buildinfo\": {\"type\": \"object\"},\n \"spec\": {**spack.schema.spec.spec_node, \"additionalProperties\": True},\n \"buildcache_layout_version\": {\"type\": \"number\"},\n}\n\nschema = {\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Spack buildcache specfile schema\",\n \"type\": \"object\",\n \"additionalProperties\": True,\n \"properties\": properties,\n}\n" }, { "path": "lib/spack/spack/schema/cdash.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Schema for cdash.yaml configuration file.\n\n.. literalinclude:: ../spack/schema/cdash.py\n :lines: 13-\n\"\"\"\n\nfrom typing import Any, Dict\n\n#: Properties for inclusion in other schemas\nproperties: Dict[str, Any] = {\n \"cdash\": {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"required\": [\"build-group\"],\n \"description\": \"Configuration for uploading build results to CDash\",\n \"properties\": {\n \"build-group\": {\n \"type\": \"string\",\n \"description\": \"Unique build group name for this stack\",\n },\n \"url\": {\"type\": \"string\", \"description\": \"CDash server URL\"},\n \"project\": {\"type\": \"string\", \"description\": \"CDash project name\"},\n \"site\": {\"type\": \"string\", \"description\": \"Site identifier for CDash reporting\"},\n },\n }\n}\n\n\n#: Full schema with metadata\nschema = {\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Spack cdash configuration file schema\",\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": properties,\n}\n" }, { "path": "lib/spack/spack/schema/ci.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Schema for gitlab-ci.yaml configuration file.\n\n.. literalinclude:: ../spack/schema/ci.py\n :lines: 16-\n\"\"\"\n\nfrom typing import Any, Dict\n\n# Schema for script fields\n# List of lists and/or strings\n# This is similar to what is allowed in\n# the gitlab schema\nscript_schema = {\n \"type\": \"array\",\n \"items\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"array\", \"items\": {\"type\": \"string\"}}]},\n}\n\n# Schema for CI image\nimage_schema = {\n \"oneOf\": [\n {\"type\": \"string\"},\n {\n \"type\": \"object\",\n \"properties\": {\n \"name\": {\"type\": \"string\"},\n \"entrypoint\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}},\n },\n },\n ]\n}\n\n# Additional attributes are allowed and will be forwarded directly to the CI target YAML for each\n# job.\nci_job_attributes = {\n \"type\": \"object\",\n \"additionalProperties\": True,\n \"properties\": {\n \"image\": image_schema,\n \"tags\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}},\n \"variables\": {\n \"type\": \"object\",\n \"patternProperties\": {r\"^[\\w\\-\\.]+$\": {\"type\": [\"string\", \"number\"]}},\n },\n \"before_script\": script_schema,\n \"script\": script_schema,\n \"after_script\": script_schema,\n },\n}\n\nref_ci_job_attributes = {\"$ref\": \"#/definitions/ci_job_attributes\"}\n\nsubmapping_schema = {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"required\": [\"submapping\"],\n \"properties\": {\n \"match_behavior\": {\"type\": \"string\", \"enum\": [\"first\", \"merge\"], \"default\": \"first\"},\n \"submapping\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"required\": [\"match\"],\n \"properties\": {\n \"match\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}},\n \"build-job\": ref_ci_job_attributes,\n \"build-job-remove\": ref_ci_job_attributes,\n },\n },\n },\n },\n}\n\ndynamic_mapping_schema = {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"required\": [\"dynamic-mapping\"],\n \"properties\": {\n \"dynamic-mapping\": {\n \"type\": \"object\",\n \"required\": [\"endpoint\"],\n \"properties\": {\n \"name\": {\"type\": \"string\"},\n # \"endpoint\" cannot have http patternProperties constraint since it is required\n # Constrain is applied in code\n \"endpoint\": {\"type\": \"string\"},\n \"timeout\": {\"type\": \"integer\", \"minimum\": 0},\n \"verify_ssl\": {\"type\": \"boolean\", \"default\": False},\n \"header\": {\"type\": \"object\", \"additionalProperties\": {\"type\": \"string\"}},\n \"allow\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}},\n \"require\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}},\n \"ignore\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}},\n },\n }\n },\n}\n\n\ndef job_schema(name: str):\n return {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": {\n f\"{name}-job\": ref_ci_job_attributes,\n f\"{name}-job-remove\": ref_ci_job_attributes,\n },\n }\n\n\npipeline_gen_schema = {\n \"type\": \"array\",\n \"items\": {\n \"oneOf\": [\n submapping_schema,\n dynamic_mapping_schema,\n job_schema(\"any\"),\n job_schema(\"build\"),\n job_schema(\"cleanup\"),\n job_schema(\"copy\"),\n job_schema(\"noop\"),\n job_schema(\"reindex\"),\n job_schema(\"signing\"),\n ]\n },\n}\n\n#: Properties for inclusion in other schemas\nproperties: Dict[str, Any] = {\n \"ci\": {\n \"type\": \"object\",\n \"properties\": {\n \"pipeline-gen\": pipeline_gen_schema,\n \"rebuild-index\": {\"type\": \"boolean\"},\n \"broken-specs-url\": {\"type\": \"string\"},\n \"broken-tests-packages\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}},\n \"target\": {\"type\": \"string\", \"default\": \"gitlab\"},\n },\n }\n}\n\n#: Full schema with metadata\nschema = {\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Spack CI configuration file schema\",\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"definitions\": {\"ci_job_attributes\": ci_job_attributes},\n \"properties\": properties,\n}\n" }, { "path": "lib/spack/spack/schema/compilers.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Schema for compilers.yaml configuration file.\n\n.. literalinclude:: _spack_root/lib/spack/spack/schema/compilers.py\n :lines: 15-\n\"\"\"\n\nfrom typing import Any, Dict\n\nimport spack.schema.environment\n\nflags: Dict[str, Any] = {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"description\": \"Flags to pass to the compiler during compilation and linking\",\n \"properties\": {\n \"cflags\": {\n \"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}],\n \"description\": \"Flags for C compiler, e.g. -std=c11\",\n },\n \"cxxflags\": {\n \"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}],\n \"description\": \"Flags for C++ compiler, e.g. -std=c++14\",\n },\n \"fflags\": {\n \"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}],\n \"description\": \"Flags for Fortran 77 compiler, e.g. -ffixed-line-length-none\",\n },\n \"cppflags\": {\n \"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}],\n \"description\": \"Flags for C preprocessor, e.g. -DFOO=1\",\n },\n \"ldflags\": {\n \"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}],\n \"description\": \"Flags passed to the compiler driver during linking, e.g. \"\n \"-Wl,--gc-sections\",\n },\n \"ldlibs\": {\n \"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}],\n \"description\": \"Flags for linker libraries, e.g. -lpthread\",\n },\n },\n}\n\n\nextra_rpaths: Dict[str, Any] = {\n \"type\": \"array\",\n \"default\": [],\n \"items\": {\"type\": \"string\"},\n \"description\": \"List of extra rpaths to inject by Spack's compiler wrappers\",\n}\n\nimplicit_rpaths: Dict[str, Any] = {\n \"anyOf\": [{\"type\": \"array\", \"items\": {\"type\": \"string\"}}, {\"type\": \"boolean\"}],\n \"description\": \"List of non-default link directories to register at runtime as rpaths\",\n}\n\n#: Properties for inclusion in other schemas\nproperties: Dict[str, Any] = {\n \"compilers\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": {\n \"compiler\": {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"required\": [\"paths\", \"spec\", \"modules\", \"operating_system\"],\n \"properties\": {\n \"paths\": {\n \"type\": \"object\",\n \"required\": [\"cc\", \"cxx\", \"f77\", \"fc\"],\n \"additionalProperties\": False,\n \"properties\": {\n \"cc\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}]},\n \"cxx\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}]},\n \"f77\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}]},\n \"fc\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}]},\n },\n },\n \"flags\": flags,\n \"spec\": {\"type\": \"string\"},\n \"operating_system\": {\"type\": \"string\"},\n \"target\": {\"type\": \"string\"},\n \"alias\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}]},\n \"modules\": {\n \"anyOf\": [\n {\"type\": \"null\"},\n {\"type\": \"array\", \"items\": {\"type\": \"string\"}},\n ]\n },\n \"implicit_rpaths\": implicit_rpaths,\n \"environment\": spack.schema.environment.ref_env_modifications,\n \"extra_rpaths\": extra_rpaths,\n },\n }\n },\n },\n }\n}\n\n\n#: Full schema with metadata\nschema = {\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Spack compiler configuration file schema\",\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": properties,\n \"definitions\": {\"env_modifications\": spack.schema.environment.env_modifications},\n}\n" }, { "path": "lib/spack/spack/schema/concretizer.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Schema for concretizer.yaml configuration file.\n\n.. literalinclude:: _spack_root/lib/spack/spack/schema/concretizer.py\n :lines: 12-\n\"\"\"\n\nfrom typing import Any, Dict\n\nLIST_OF_SPECS = {\"type\": \"array\", \"items\": {\"type\": \"string\"}}\n\nproperties: Dict[str, Any] = {\n \"concretizer\": {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"description\": \"Concretizer configuration that controls dependency selection, package \"\n \"reuse, and solver behavior\",\n \"properties\": {\n \"force\": {\n \"type\": \"boolean\",\n \"default\": False,\n \"description\": \"Force re-concretization when concretizing environments\",\n },\n \"reuse\": {\n \"description\": \"Controls how aggressively Spack reuses installed packages and \"\n \"build caches during concretization\",\n \"oneOf\": [\n {\n \"type\": \"boolean\",\n \"description\": \"If true, reuse installed packages and build caches for \"\n \"all specs; if false, always perform fresh concretization\",\n },\n {\n \"type\": \"string\",\n \"enum\": [\"dependencies\"],\n \"description\": \"Reuse installed packages and build caches only for \"\n \"dependencies, not root specs\",\n },\n {\n \"type\": \"object\",\n \"description\": \"Advanced reuse configuration with fine-grained control \"\n \"over which specs are reused\",\n \"properties\": {\n \"roots\": {\n \"type\": \"boolean\",\n \"description\": \"If true, root specs are reused; if false, only \"\n \"dependencies of root specs are reused\",\n },\n \"include\": {\n **LIST_OF_SPECS,\n \"description\": \"List of spec constraints. Reusable specs must \"\n \"match at least one constraint\",\n },\n \"exclude\": {\n **LIST_OF_SPECS,\n \"description\": \"List of spec constraints. Reusable specs must \"\n \"not match any constraint\",\n },\n \"from\": {\n \"type\": \"array\",\n \"description\": \"List of sources from which reused specs are taken\",\n \"items\": {\n \"type\": \"object\",\n \"description\": \"Source configuration for reusable specs\",\n \"properties\": {\n \"type\": {\n \"type\": \"string\",\n \"enum\": [\n \"local\",\n \"buildcache\",\n \"external\",\n \"environment\",\n ],\n \"description\": \"Type of source: 'local' (installed \"\n \"packages), 'buildcache' (remote binaries), \"\n \"'external' (system packages), or 'environment' \"\n \"(from specific environment)\",\n },\n \"path\": {\n \"type\": \"string\",\n \"description\": \"Path to the source (for environment \"\n \"type sources)\",\n },\n \"include\": {\n **LIST_OF_SPECS,\n \"description\": \"Spec constraints that must be \"\n \"matched for this source (overrides global include)\",\n },\n \"exclude\": {\n **LIST_OF_SPECS,\n \"description\": \"Spec constraints that must not be \"\n \"matched for this source (overrides global exclude)\",\n },\n },\n },\n },\n },\n },\n ],\n },\n \"targets\": {\n \"type\": \"object\",\n \"description\": \"Controls which target microarchitectures are considered \"\n \"during concretization\",\n \"properties\": {\n \"host_compatible\": {\n \"type\": \"boolean\",\n \"description\": \"If true, only allow targets compatible with the \"\n \"current host; if false, allow any target (e.g., concretize for icelake \"\n \"while running on haswell)\",\n },\n \"granularity\": {\n \"type\": \"string\",\n \"enum\": [\"generic\", \"microarchitectures\"],\n \"description\": \"Target selection granularity: 'microarchitectures' \"\n \"(e.g., haswell, skylake) or 'generic' (e.g., x86_64_v3, aarch64)\",\n },\n },\n },\n \"unify\": {\n \"description\": \"Controls whether environment specs are concretized together \"\n \"or separately\",\n \"oneOf\": [\n {\n \"type\": \"boolean\",\n \"description\": \"If true, concretize environment root specs together \"\n \"for unified dependencies; if false, concretize each spec independently\",\n },\n {\n \"type\": \"string\",\n \"enum\": [\"when_possible\"],\n \"description\": \"Maximizes reuse, while allowing multiple instances of the \"\n \"same package\",\n },\n ],\n },\n \"compiler_mixing\": {\n \"oneOf\": [{\"type\": \"boolean\"}, {\"type\": \"array\"}],\n \"description\": \"Whether to allow compiler mixing between link/run dependencies\",\n },\n \"splice\": {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"description\": \"Configuration for spec splicing: replacing dependencies \"\n \"with ABI-compatible alternatives to improve package reuse\",\n \"properties\": {\n \"explicit\": {\n \"type\": \"array\",\n \"default\": [],\n \"description\": \"List of explicit splice configurations to replace \"\n \"specific dependencies\",\n \"items\": {\n \"type\": \"object\",\n \"required\": [\"target\", \"replacement\"],\n \"additionalProperties\": False,\n \"description\": \"Explicit splice configuration\",\n \"properties\": {\n \"target\": {\n \"type\": \"string\",\n \"description\": \"Abstract spec to be replaced (e.g., 'mpi' \"\n \"or specific package)\",\n },\n \"replacement\": {\n \"type\": \"string\",\n \"description\": \"Concrete spec with hash to use as \"\n \"replacement (e.g., 'mpich/abcdef')\",\n },\n \"transitive\": {\n \"type\": \"boolean\",\n \"default\": False,\n \"description\": \"If true, use transitive splice (conflicts \"\n \"resolved using replacement dependencies); if false, use \"\n \"intransitive splice (conflicts resolved using original \"\n \"dependencies)\",\n },\n },\n },\n },\n \"automatic\": {\n \"type\": \"boolean\",\n \"description\": \"Enable automatic splicing for ABI-compatible packages \"\n \"(experimental feature)\",\n },\n },\n },\n \"duplicates\": {\n \"type\": \"object\",\n \"description\": \"Controls whether the dependency graph can contain multiple \"\n \"configurations of the same package\",\n \"properties\": {\n \"strategy\": {\n \"type\": \"string\",\n \"enum\": [\"none\", \"minimal\", \"full\"],\n \"description\": \"Duplication strategy: 'none' (single config per \"\n \"package), 'minimal' (allow build-tools duplicates), 'full' \"\n \"(experimental: allow full build-tool stack separation)\",\n },\n \"max_dupes\": {\n \"type\": \"object\",\n \"description\": \"Maximum number of duplicates allowed per package when \"\n \"using strategies that permit duplicates\",\n \"additionalProperties\": {\n \"type\": \"integer\",\n \"minimum\": 1,\n \"description\": \"Maximum number of duplicate instances for this \"\n \"package\",\n },\n },\n },\n },\n \"static_analysis\": {\n \"type\": \"boolean\",\n \"description\": \"Enable static analysis to reduce concretization time by \"\n \"generating smaller ASP problems\",\n },\n \"timeout\": {\n \"type\": \"integer\",\n \"minimum\": 0,\n \"description\": \"Maximum time in seconds for the solve phase (0 means no \"\n \"time limit)\",\n },\n \"error_on_timeout\": {\n \"type\": \"boolean\",\n \"description\": \"If true, timeout always results in error; if false, use best \"\n \"suboptimal solution found before timeout (yields unreproducible results)\",\n },\n \"os_compatible\": {\n \"type\": \"object\",\n \"additionalProperties\": {\"type\": \"array\"},\n \"description\": \"Compatibility mapping between operating systems for reuse of \"\n \"compilers and packages (key: target OS, value: list of compatible source OSes)\",\n },\n \"concretization_cache\": {\n \"type\": \"object\",\n \"description\": \"Configuration for caching solver outputs from successful \"\n \"concretization runs\",\n \"properties\": {\n \"enable\": {\n \"type\": \"boolean\",\n \"description\": \"Whether to utilize a cache of solver outputs from \"\n \"successful concretization runs\",\n },\n \"url\": {\n \"type\": \"string\",\n \"description\": \"Path to the location where Spack will root the \"\n \"concretization cache\",\n },\n \"entry_limit\": {\n \"type\": \"integer\",\n \"minimum\": 0,\n \"description\": \"Limit on the number of concretization results that \"\n \"Spack will cache (0 disables pruning)\",\n },\n },\n },\n \"externals\": {\n \"type\": \"object\",\n \"description\": \"Configuration for how Spack handles external packages during \"\n \"concretization\",\n \"properties\": {\n \"completion\": {\n \"type\": \"string\",\n \"enum\": [\"architecture_only\", \"default_variants\"],\n \"description\": \"Controls how missing information (variants, etc.) is \"\n \"completed for external packages: 'architecture_only' completes only \"\n \"mandatory architectural information; 'default_variants' also completes \"\n \"missing variants using their default values\",\n }\n },\n },\n },\n }\n}\n\n\n#: Full schema with metadata\nschema = {\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Spack concretizer configuration file schema\",\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": properties,\n}\n" }, { "path": "lib/spack/spack/schema/config.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Schema for config.yaml configuration file.\n\n.. literalinclude:: _spack_root/lib/spack/spack/schema/config.py\n :lines: 17-\n\"\"\"\n\nfrom typing import Any, Dict\n\nimport spack.schema\nimport spack.schema.projections\n\n#: Properties for inclusion in other schemas\nproperties: Dict[str, Any] = {\n \"config\": {\n \"type\": \"object\",\n \"default\": {},\n \"description\": \"Spack's basic configuration options\",\n \"properties\": {\n \"flags\": {\n \"type\": \"object\",\n \"description\": \"Build flag configuration options\",\n \"properties\": {\n \"keep_werror\": {\n \"type\": \"string\",\n \"enum\": [\"all\", \"specific\", \"none\"],\n \"description\": \"Whether to keep -Werror flags active in package builds\",\n }\n },\n },\n \"shared_linking\": {\n \"description\": \"Control how shared libraries are located at runtime on Linux\",\n \"anyOf\": [\n {\"type\": \"string\", \"enum\": [\"rpath\", \"runpath\"]},\n {\n \"type\": \"object\",\n \"properties\": {\n \"type\": {\n \"type\": \"string\",\n \"enum\": [\"rpath\", \"runpath\"],\n \"description\": \"Whether to use RPATH or RUNPATH for runtime \"\n \"library search paths\",\n },\n \"bind\": {\n \"type\": \"boolean\",\n \"description\": \"Embed absolute paths of dependent libraries \"\n \"directly in ELF binaries (experimental)\",\n },\n \"missing_library_policy\": {\n \"enum\": [\"error\", \"warn\", \"ignore\"],\n \"description\": \"How to handle missing dynamic libraries after \"\n \"installation\",\n },\n },\n },\n ],\n },\n \"install_tree\": {\n \"type\": \"object\",\n \"description\": \"Installation tree configuration\",\n \"properties\": {\n \"root\": {\n \"type\": \"string\",\n \"description\": \"The location where Spack will install packages and \"\n \"their dependencies\",\n },\n \"padded_length\": {\n \"oneOf\": [{\"type\": \"integer\", \"minimum\": 0}, {\"type\": \"boolean\"}],\n \"description\": \"Length to pad installation paths to allow better \"\n \"relocation of binaries (true for max length, integer for specific \"\n \"length)\",\n },\n **spack.schema.projections.ref_properties,\n },\n },\n \"install_hash_length\": {\n \"type\": \"integer\",\n \"minimum\": 1,\n \"description\": \"Length of hash used in installation directory names\",\n },\n \"build_stage\": {\n \"oneOf\": [{\"type\": \"string\"}, {\"type\": \"array\", \"items\": {\"type\": \"string\"}}],\n \"description\": \"Temporary locations Spack can try to use for builds\",\n },\n \"stage_name\": {\n \"type\": \"string\",\n \"description\": \"Name format for build stage directories\",\n },\n \"develop_stage_link\": {\n \"type\": \"string\",\n \"description\": \"Name for development spec build stage directories. Setting to \"\n \"None will disable develop stage links.\",\n },\n \"test_stage\": {\n \"type\": \"string\",\n \"description\": \"Directory in which to run tests and store test results\",\n },\n \"extensions\": {\n \"type\": \"array\",\n \"items\": {\"type\": \"string\"},\n \"description\": \"List of Spack extensions to load\",\n },\n \"template_dirs\": {\n \"type\": \"array\",\n \"items\": {\"type\": \"string\"},\n \"description\": \"Locations where templates should be found\",\n },\n \"license_dir\": {\n \"type\": \"string\",\n \"description\": \"Directory where licenses should be located\",\n },\n \"source_cache\": {\n \"type\": \"string\",\n \"description\": \"Location to cache downloaded tarballs and repositories\",\n },\n \"misc_cache\": {\n \"type\": \"string\",\n \"description\": \"Temporary directory to store long-lived cache files, such as \"\n \"indices of packages\",\n },\n \"environments_root\": {\n \"type\": \"string\",\n \"description\": \"Directory where Spack managed environments are created and stored\",\n },\n \"connect_timeout\": {\n \"type\": \"integer\",\n \"minimum\": 0,\n \"description\": \"Abort downloads after this many seconds if no data is received \"\n \"(0 disables timeout)\",\n },\n \"verify_ssl\": {\n \"type\": \"boolean\",\n \"description\": \"When true, Spack will verify certificates of remote hosts when \"\n \"making SSL connections\",\n },\n \"ssl_certs\": {\n \"type\": \"string\",\n \"description\": \"Path to custom certificates for SSL verification\",\n },\n \"suppress_gpg_warnings\": {\n \"type\": \"boolean\",\n \"description\": \"Suppress GPG warnings from binary package verification\",\n },\n \"debug\": {\n \"type\": \"boolean\",\n \"description\": \"Enable debug mode for additional logging\",\n },\n \"checksum\": {\n \"type\": \"boolean\",\n \"description\": \"When true, Spack verifies downloaded source code using checksums\",\n },\n \"deprecated\": {\n \"type\": \"boolean\",\n \"description\": \"If true, Spack will fetch deprecated versions without warning\",\n },\n \"locks\": {\n \"type\": \"boolean\",\n \"description\": \"When true, concurrent instances of Spack will use locks to avoid \"\n \"conflicts (strongly recommended)\",\n },\n \"dirty\": {\n \"type\": \"boolean\",\n \"description\": \"When true, builds will NOT clean potentially harmful variables \"\n \"from the environment\",\n },\n \"build_language\": {\n \"type\": \"string\",\n \"description\": \"The language the build environment will use (C for English, \"\n \"empty string for user's environment)\",\n },\n \"build_jobs\": {\n \"type\": \"integer\",\n \"minimum\": 1,\n \"description\": \"The maximum number of jobs to use for the build system (e.g. \"\n \"make -j), defaults to 16\",\n },\n \"concurrent_packages\": {\n \"type\": \"integer\",\n \"minimum\": 0,\n \"description\": \"The maximum number of concurrent package builds a single Spack \"\n \"instance will run\",\n },\n \"ccache\": {\n \"type\": \"boolean\",\n \"description\": \"When true, Spack's compiler wrapper will use ccache when \"\n \"compiling C and C++\",\n },\n \"db_lock_timeout\": {\n \"type\": \"integer\",\n \"minimum\": 1,\n \"description\": \"How long to wait to lock the Spack installation database\",\n },\n \"package_lock_timeout\": {\n \"anyOf\": [{\"type\": \"integer\", \"minimum\": 1}, {\"type\": \"null\"}],\n \"description\": \"How long to wait when attempting to modify a package (null for \"\n \"never timeout)\",\n },\n \"allow_sgid\": {\n \"type\": \"boolean\",\n \"description\": \"Allow installation on filesystems that don't allow setgid bit \"\n \"manipulation\",\n },\n \"install_status\": {\n \"type\": \"boolean\",\n \"description\": \"Whether to show status information in the terminal title during \"\n \"the build\",\n },\n \"url_fetch_method\": {\n \"anyOf\": [{\"enum\": [\"urllib\", \"curl\"]}, {\"type\": \"string\", \"pattern\": r\"^curl \"}],\n \"description\": \"The default URL fetch method to use (urllib or curl)\",\n },\n \"additional_external_search_paths\": {\n \"type\": \"array\",\n \"items\": {\"type\": \"string\"},\n \"description\": \"Additional paths to search for external packages\",\n },\n \"binary_index_ttl\": {\n \"type\": \"integer\",\n \"minimum\": 0,\n \"description\": \"Number of seconds a buildcache's index.json is cached locally \"\n \"before probing for updates\",\n },\n \"aliases\": {\n \"type\": \"object\",\n \"additionalProperties\": {\"type\": \"string\"},\n \"description\": \"A mapping of aliases that can be used to define new \"\n \"Spack commands\",\n },\n \"installer\": {\n \"type\": \"string\",\n \"enum\": [\"old\", \"new\"],\n \"description\": \"Which installer to use. The new installer is experimental.\",\n },\n },\n }\n}\n\n\n#: Full schema with metadata\nschema = {\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Spack core configuration file schema\",\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": properties,\n \"definitions\": {\"projections\": spack.schema.projections.projections},\n}\n\n\ndef update(data: dict) -> bool:\n \"\"\"Update the data in place to remove deprecated properties.\n\n Args:\n data: dictionary to be updated\n\n Returns: True if data was changed, False otherwise\n \"\"\"\n changed = False\n data = data[\"config\"]\n shared_linking = data.get(\"shared_linking\", None)\n if isinstance(shared_linking, str):\n # deprecated short-form shared_linking: rpath/runpath\n # add value as `type` in updated shared_linking\n data[\"shared_linking\"] = {\"type\": shared_linking, \"bind\": False}\n changed = True\n return changed\n" }, { "path": "lib/spack/spack/schema/container.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Schema for the ``container`` subsection of Spack environments.\"\"\"\n\nfrom typing import Any, Dict\n\n_stages_from_dockerhub = {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": {\n \"os\": {\"type\": \"string\"},\n \"spack\": {\n \"anyOf\": [\n {\"type\": \"string\"},\n {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": {\n \"url\": {\"type\": \"string\"},\n \"ref\": {\"type\": \"string\"},\n \"resolve_sha\": {\"type\": \"boolean\", \"default\": False},\n \"verify\": {\"type\": \"boolean\", \"default\": False},\n },\n },\n ]\n },\n },\n \"required\": [\"os\", \"spack\"],\n}\n\n_custom_stages = {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": {\"build\": {\"type\": \"string\"}, \"final\": {\"type\": \"string\"}},\n \"required\": [\"build\", \"final\"],\n}\n\n#: List of packages for the schema below\n_list_of_packages = {\"type\": \"array\", \"items\": {\"type\": \"string\"}}\n\n#: Schema for the container attribute included in Spack environments\ncontainer_schema = {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": {\n # The recipe formats that are currently supported by the command\n \"format\": {\"type\": \"string\", \"enum\": [\"docker\", \"singularity\"]},\n # Describes the base image to start from and the version\n # of Spack to be used\n \"images\": {\"anyOf\": [_stages_from_dockerhub, _custom_stages]},\n # Whether or not to strip installed binaries\n \"strip\": {\"type\": \"boolean\", \"default\": True},\n # Additional system packages that are needed at runtime\n \"os_packages\": {\n \"type\": \"object\",\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"enum\": [\"apt\", \"yum\", \"zypper\", \"apk\", \"yum_amazon\"],\n },\n \"update\": {\"type\": \"boolean\"},\n \"build\": _list_of_packages,\n \"final\": _list_of_packages,\n },\n \"additionalProperties\": False,\n },\n # Add labels to the image\n \"labels\": {\"type\": \"object\"},\n # Use a custom template to render the recipe\n \"template\": {\"type\": \"string\", \"default\": None},\n # Reserved for properties that are specific to each format\n \"singularity\": {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"default\": {},\n \"properties\": {\n \"runscript\": {\"type\": \"string\"},\n \"startscript\": {\"type\": \"string\"},\n \"test\": {\"type\": \"string\"},\n \"help\": {\"type\": \"string\"},\n },\n },\n \"docker\": {\"type\": \"object\", \"additionalProperties\": False, \"default\": {}},\n \"depfile\": {\"type\": \"boolean\", \"default\": False},\n },\n}\n\nproperties: Dict[str, Any] = {\"container\": container_schema}\n" }, { "path": "lib/spack/spack/schema/cray_manifest.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Schema for Cray descriptive manifest: this describes a set of\ninstalled packages on the system and also specifies dependency\nrelationships between them (so this provides more information than\nexternal entries in packages configuration).\n\nThis does not specify a configuration - it is an input format\nthat is consumed and transformed into Spack DB records.\n\"\"\"\n\nfrom typing import Any, Dict\n\nproperties: Dict[str, Any] = {\n \"_meta\": {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": {\n \"file-type\": {\"type\": \"string\", \"minLength\": 1},\n \"cpe-version\": {\"type\": \"string\", \"minLength\": 1},\n \"system-type\": {\"type\": \"string\", \"minLength\": 1},\n \"schema-version\": {\"type\": \"string\", \"minLength\": 1},\n # Older schemas use did not have \"cpe-version\", just the\n # schema version; in that case it was just called \"version\"\n \"version\": {\"type\": \"string\", \"minLength\": 1},\n },\n },\n \"compilers\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": {\n \"name\": {\"type\": \"string\", \"minLength\": 1},\n \"version\": {\"type\": \"string\", \"minLength\": 1},\n \"prefix\": {\"type\": \"string\", \"minLength\": 1},\n \"executables\": {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": {\n \"cc\": {\"type\": \"string\", \"minLength\": 1},\n \"cxx\": {\"type\": \"string\", \"minLength\": 1},\n \"fc\": {\"type\": \"string\", \"minLength\": 1},\n },\n },\n \"arch\": {\n \"type\": \"object\",\n \"required\": [\"os\", \"target\"],\n \"additionalProperties\": False,\n \"properties\": {\n \"os\": {\"type\": \"string\", \"minLength\": 1},\n \"target\": {\"type\": \"string\", \"minLength\": 1},\n },\n },\n },\n },\n },\n \"specs\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"required\": [\"name\", \"version\", \"arch\", \"compiler\", \"prefix\", \"hash\"],\n \"additionalProperties\": False,\n \"properties\": {\n \"name\": {\"type\": \"string\", \"minLength\": 1},\n \"version\": {\"type\": \"string\", \"minLength\": 1},\n \"arch\": {\n \"type\": \"object\",\n \"required\": [\"platform\", \"platform_os\", \"target\"],\n \"additionalProperties\": False,\n \"properties\": {\n \"platform\": {\"type\": \"string\", \"minLength\": 1},\n \"platform_os\": {\"type\": \"string\", \"minLength\": 1},\n \"target\": {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"required\": [\"name\"],\n \"properties\": {\"name\": {\"type\": \"string\", \"minLength\": 1}},\n },\n },\n },\n \"compiler\": {\n \"type\": \"object\",\n \"required\": [\"name\", \"version\"],\n \"additionalProperties\": False,\n \"properties\": {\n \"name\": {\"type\": \"string\", \"minLength\": 1},\n \"version\": {\"type\": \"string\", \"minLength\": 1},\n },\n },\n \"dependencies\": {\n \"type\": \"object\",\n \"additionalProperties\": {\n \"type\": \"object\",\n \"required\": [\"hash\"],\n \"additionalProperties\": False,\n \"properties\": {\n \"hash\": {\"type\": \"string\", \"minLength\": 1},\n \"type\": {\"type\": \"array\", \"items\": {\"type\": \"string\", \"minLength\": 1}},\n },\n },\n },\n \"prefix\": {\"type\": \"string\", \"minLength\": 1},\n \"rpm\": {\"type\": \"string\", \"minLength\": 1},\n \"hash\": {\"type\": \"string\", \"minLength\": 1},\n \"parameters\": {\"type\": \"object\"},\n },\n },\n },\n}\n\nschema = {\n \"$schema\": \"http://json-schema.org/schema#\",\n \"title\": \"CPE manifest schema\",\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": properties,\n}\n" }, { "path": "lib/spack/spack/schema/database_index.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Schema for database index.json file\n\n.. literalinclude:: _spack_root/lib/spack/spack/schema/database_index.py\n :lines: 17-\n\"\"\"\n\nfrom typing import Any, Dict\n\nimport spack.schema.spec\n\nproperties: Dict[str, Any] = {\n \"database\": {\n \"type\": \"object\",\n \"required\": [\"installs\", \"version\"],\n \"additionalProperties\": False,\n \"properties\": {\n \"installs\": {\n \"type\": \"object\",\n \"patternProperties\": {\n r\"^[a-z0-9]{32}$\": {\n \"type\": \"object\",\n \"properties\": {\n \"spec\": spack.schema.spec.spec_node,\n \"path\": {\"oneOf\": [{\"type\": \"string\"}, {\"type\": \"null\"}]},\n \"installed\": {\"type\": \"boolean\"},\n \"ref_count\": {\"type\": \"integer\", \"minimum\": 0},\n \"explicit\": {\"type\": \"boolean\"},\n \"installation_time\": {\"type\": \"number\"},\n },\n }\n },\n },\n \"version\": {\"type\": \"string\"},\n },\n }\n}\n\n#: Full schema with metadata\nschema = {\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Spack spec schema\",\n \"type\": \"object\",\n \"required\": [\"database\"],\n \"additionalProperties\": False,\n \"properties\": properties,\n}\n" }, { "path": "lib/spack/spack/schema/definitions.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Schema for definitions\n\n.. literalinclude:: _spack_root/lib/spack/spack/schema/definitions.py\n :lines: 16-\n\"\"\"\n\nfrom typing import Any, Dict\n\nfrom .spec_list import spec_list_schema\n\n#: Properties for inclusion in other schemas\nproperties: Dict[str, Any] = {\n \"definitions\": {\n \"type\": \"array\",\n \"default\": [],\n \"description\": \"Named spec lists to be referred to with $name in the specs section of \"\n \"environments\",\n \"items\": {\n \"type\": \"object\",\n \"description\": \"Named definition entry containing a named spec list and optional \"\n \"conditional 'when' clause\",\n \"properties\": {\n \"when\": {\n \"type\": \"string\",\n \"description\": \"Python code condition evaluated as boolean. Specs are \"\n \"appended to the named list only if the condition is True. Available \"\n \"variables: platform, os, target, arch, arch_str, re, env, hostname\",\n }\n },\n \"additionalProperties\": spec_list_schema,\n },\n }\n}\n\n#: Full schema with metadata\nschema = {\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Spack definitions configuration file schema\",\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": properties,\n}\n" }, { "path": "lib/spack/spack/schema/develop.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nfrom typing import Any, Dict\n\nproperties: Dict[str, Any] = {\n \"develop\": {\n \"type\": \"object\",\n \"default\": {},\n \"description\": \"Configuration for local development of Spack packages\",\n \"additionalProperties\": {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"description\": \"Name of a package to develop, with its spec and optional source path\",\n \"required\": [\"spec\"],\n \"properties\": {\n \"spec\": {\n \"type\": \"string\",\n \"description\": \"Spec of the package to develop, e.g. hdf5@1.12.0\",\n },\n \"path\": {\n \"type\": \"string\",\n \"description\": \"Path to the source code for this package, can be \"\n \"absolute or relative to the environment directory\",\n },\n },\n },\n }\n}\n\n\n#: Full schema with metadata\nschema = {\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Spack repository configuration file schema\",\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": properties,\n}\n" }, { "path": "lib/spack/spack/schema/env.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Schema for env.yaml configuration file.\n\n.. literalinclude:: _spack_root/lib/spack/spack/schema/env.py\n :lines: 19-\n\"\"\"\n\nimport os\nfrom typing import Any, Dict\n\nimport spack.schema.merged\n\nfrom .spec_list import spec_list_properties, spec_list_schema\n\n#: Top level key in a manifest file\nTOP_LEVEL_KEY = \"spack\"\n\n# (DEPRECATED) include concrete entries to be merged under the include key\ninclude_concrete = {\n \"type\": \"array\",\n \"default\": [],\n \"description\": \"List of paths to other environments. Includes concrete specs \"\n \"from their spack.lock files without modifying the source environments. Useful \"\n \"for phased deployments where you want to build on existing concrete specs.\",\n \"items\": {\"type\": \"string\"},\n}\n\ngroup_name_and_deps = {\n \"group\": {\"type\": \"string\", \"description\": \"Name for this group of specs\"},\n \"explicit\": {\n \"type\": \"boolean\",\n \"default\": True,\n \"description\": \"When false, specs in this group are installed as implicit \"\n \"dependencies and are eligible for garbage collection.\",\n },\n \"needs\": {\n \"type\": \"array\",\n \"description\": \"Groups of specs that are needed by this group\",\n \"items\": {\"type\": \"string\"},\n },\n \"override\": {\n \"type\": \"object\",\n \"description\": \"Top-most configuration scope for this group of specs\",\n \"additionalProperties\": False,\n \"properties\": {**spack.schema.merged.ref_sections},\n },\n}\n\n\nproperties: Dict[str, Any] = {\n \"spack\": {\n \"type\": \"object\",\n \"default\": {},\n \"description\": \"Spack environment configuration, including specs, view, and any other \"\n \"config section (config, packages, concretizer, mirrors, etc.)\",\n \"additionalProperties\": False,\n \"properties\": {\n # merged configuration scope schemas\n **spack.schema.merged.ref_sections,\n # extra environment schema properties\n \"specs\": {\n \"type\": \"array\",\n \"description\": \"List of specs to include in the environment, \"\n \"supporting both simple specs and matrix configurations\",\n \"default\": [],\n \"items\": {\n \"anyOf\": [\n {\n \"type\": \"object\",\n \"description\": \"Matrix configuration for generating multiple specs\"\n \" from combinations of constraints\",\n \"additionalProperties\": False,\n \"properties\": {**spec_list_properties},\n },\n {\"type\": \"string\", \"description\": \"Simple spec string\"},\n {\"type\": \"null\"},\n {\n \"type\": \"object\",\n \"description\": \"User spec group with a single matrix\",\n \"additionalProperties\": False,\n \"properties\": {**spec_list_properties, **group_name_and_deps},\n },\n {\n \"type\": \"object\",\n \"description\": \"User spec group with multiple matrices\",\n \"additionalProperties\": False,\n \"properties\": {**group_name_and_deps, \"specs\": spec_list_schema},\n },\n ]\n },\n },\n # (DEPRECATED) include concrete to be merged under the include key\n \"include_concrete\": include_concrete,\n },\n }\n}\n\nschema = {\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Spack environment file schema\",\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": properties,\n \"definitions\": spack.schema.merged.defs,\n}\n\n\ndef update(data: Dict[str, Any]) -> bool:\n \"\"\"Update the spack.yaml data to the new format.\n\n Args:\n data: dictionary to be updated\n\n Returns:\n ``True`` if data was changed, ``False`` otherwise\n \"\"\"\n if not isinstance(data, dict):\n return False\n\n if \"include_concrete\" not in data:\n return False\n\n # Move the old 'include_concrete' paths to reside under the 'include',\n # ensuring that the lock file name is appended.\n includes = []\n for path in data[\"include_concrete\"]:\n if os.path.basename(path) != \"spack.lock\":\n path = os.path.join(path, \"spack.lock\")\n includes.append(path)\n\n # Now add back the includes the environment file already has.\n if \"include\" in data:\n for path in data[\"include\"]:\n includes.append(path)\n\n data[\"include\"] = includes\n del data[\"include_concrete\"]\n\n return True\n" }, { "path": "lib/spack/spack/schema/env_vars.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Schema for env_vars.yaml configuration file.\n\n.. literalinclude:: _spack_root/lib/spack/spack/schema/env_vars.py\n :lines: 15-\n\"\"\"\n\nfrom typing import Any, Dict\n\nimport spack.schema.environment\n\nproperties: Dict[str, Any] = {\"env_vars\": spack.schema.environment.ref_env_modifications}\n\n#: Full schema with metadata\nschema = {\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Spack env_vars configuration file schema\",\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": properties,\n \"definitions\": {\"env_modifications\": spack.schema.environment.env_modifications},\n}\n" }, { "path": "lib/spack/spack/schema/environment.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Schema for environment modifications. Meant for inclusion in other\nschemas.\n\"\"\"\n\nimport collections.abc\nfrom typing import Any, Dict\n\ndictionary_of_strings_or_num = {\n \"type\": \"object\",\n \"additionalProperties\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"number\"}]},\n}\n\nenv_modifications: Dict[str, Any] = {\n \"type\": \"object\",\n \"description\": \"Environment variable modifications to apply at runtime\",\n \"default\": {},\n \"additionalProperties\": False,\n \"properties\": {\n \"set\": {\n \"description\": \"Environment variables to set to specific values\",\n **dictionary_of_strings_or_num,\n },\n \"unset\": {\n \"description\": \"Environment variables to remove/unset\",\n \"default\": [],\n \"type\": \"array\",\n \"items\": {\"type\": \"string\"},\n },\n \"prepend_path\": {\n \"description\": \"Environment variables to prepend values to (typically PATH-like \"\n \"variables)\",\n **dictionary_of_strings_or_num,\n },\n \"append_path\": {\n \"description\": \"Environment variables to append values to (typically PATH-like \"\n \"variables)\",\n **dictionary_of_strings_or_num,\n },\n \"remove_path\": {\n \"description\": \"Values to remove from PATH-like environment variables\",\n **dictionary_of_strings_or_num,\n },\n },\n}\n\n#: $ref pointer for use in merged schema\nref_env_modifications = {\"$ref\": \"#/definitions/env_modifications\"}\n\n\ndef parse(config_obj):\n \"\"\"Returns an EnvironmentModifications object containing the modifications\n parsed from input.\n\n Args:\n config_obj: a configuration dictionary conforming to the\n schema definition for environment modifications\n \"\"\"\n import spack.util.environment as ev\n\n env = ev.EnvironmentModifications()\n for command, variable in config_obj.items():\n # Distinguish between commands that take only a name as argument\n # (e.g. unset) and commands that take a name and a value.\n if isinstance(variable, collections.abc.Sequence):\n for name in variable:\n getattr(env, command)(name)\n else:\n for name, value in variable.items():\n getattr(env, command)(name, value)\n\n return env\n" }, { "path": "lib/spack/spack/schema/include.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Schema for include.yaml configuration file.\n\n.. literalinclude:: _spack_root/lib/spack/spack/schema/include.py\n :lines: 12-\n\"\"\"\n\nfrom typing import Any, Dict\n\n#: Properties for inclusion in other schemas\nproperties: Dict[str, Any] = {\n \"include\": {\n \"type\": \"array\",\n \"default\": [],\n \"additionalProperties\": False,\n \"description\": \"Include external configuration files to pull in configuration from \"\n \"other files/URLs for modular and reusable configurations\",\n \"items\": {\n \"anyOf\": [\n # local, required path\n {\n \"type\": \"string\",\n \"description\": \"Simple include entry specifying path to required \"\n \"configuration file/directory\",\n },\n # local or remote paths that may be optional or conditional\n {\n \"type\": \"object\",\n \"description\": \"Advanced include entry with optional conditions and \"\n \"remote file support\",\n \"properties\": {\n \"when\": {\n \"type\": \"string\",\n \"description\": \"Include this config only when the condition (as \"\n \"Python code) evaluates to true\",\n },\n \"name\": {\"type\": \"string\"},\n \"path_override_env_var\": {\"type\": \"string\"},\n \"path\": {\n \"type\": \"string\",\n \"description\": \"Path to configuration file/directory (absolute, \"\n \"relative, or URL). URLs must be raw file content (GitHub/GitLab \"\n \"raw form). Supports file, ftp, http, https schemes and \"\n \"Spack/environment variables\",\n },\n \"sha256\": {\n \"type\": \"string\",\n \"description\": \"Required SHA256 hash for remote URLs to verify \"\n \"file integrity\",\n },\n \"optional\": {\n \"type\": \"boolean\",\n \"description\": \"If true, include only if path exists; if false \"\n \"(default), path is required and missing files cause errors\",\n },\n \"prefer_modify\": {\"type\": \"boolean\"},\n },\n \"required\": [\"path\"],\n \"additionalProperties\": False,\n },\n # remote git paths that may be optional or conditional\n {\n \"type\": \"object\",\n \"description\": \"Include configuration files from a git repository with \"\n \"conditional and optional support\",\n \"properties\": {\n \"git\": {\n \"type\": \"string\",\n \"description\": \"URL of the git repository to clone (e.g., \"\n \"https://github.com/spack/spack-configs)\",\n },\n \"branch\": {\n \"type\": \"string\",\n \"description\": \"Branch to check out from the repository\",\n },\n \"commit\": {\n \"type\": \"string\",\n \"description\": \"Specific commit SHA to check out from the repository\",\n },\n \"tag\": {\n \"type\": \"string\",\n \"description\": \"Tag to check out from the repository\",\n },\n \"paths\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\",\n \"description\": \"Relative path within the repository to a \"\n \"configuration file to include\",\n },\n \"description\": \"List of relative paths within the repository where \"\n \"configuration files are located\",\n },\n \"name\": {\"type\": \"string\"},\n \"when\": {\n \"type\": \"string\",\n \"description\": \"Include this config only when the condition (as \"\n \"Python code) evaluates to true\",\n },\n \"optional\": {\n \"type\": \"boolean\",\n \"description\": \"If true, include only if repository is accessible; \"\n \"if false (default), inaccessible repository causes errors\",\n },\n },\n \"required\": [\"git\", \"paths\"],\n \"additionalProperties\": False,\n },\n ]\n },\n }\n}\n\n#: Full schema with metadata\nschema = {\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Spack include configuration file schema\",\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": properties,\n}\n" }, { "path": "lib/spack/spack/schema/merged.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Schema for configuration merged into one file.\n\n.. literalinclude:: _spack_root/lib/spack/spack/schema/merged.py\n :lines: 32-\n\"\"\"\n\nfrom typing import Any, Dict\n\nimport spack.schema.bootstrap\nimport spack.schema.cdash\nimport spack.schema.ci\nimport spack.schema.compilers\nimport spack.schema.concretizer\nimport spack.schema.config\nimport spack.schema.container\nimport spack.schema.definitions\nimport spack.schema.develop\nimport spack.schema.env_vars\nimport spack.schema.environment\nimport spack.schema.include\nimport spack.schema.mirrors\nimport spack.schema.modules\nimport spack.schema.packages\nimport spack.schema.projections\nimport spack.schema.repos\nimport spack.schema.toolchains\nimport spack.schema.upstreams\nimport spack.schema.view\n\n#: Properties for inclusion in other schemas\nsections: Dict[str, Any] = {\n **spack.schema.bootstrap.properties,\n **spack.schema.cdash.properties,\n **spack.schema.compilers.properties,\n **spack.schema.concretizer.properties,\n **spack.schema.config.properties,\n **spack.schema.container.properties,\n **spack.schema.ci.properties,\n **spack.schema.definitions.properties,\n **spack.schema.develop.properties,\n **spack.schema.env_vars.properties,\n **spack.schema.include.properties,\n **spack.schema.mirrors.properties,\n **spack.schema.modules.properties,\n **spack.schema.packages.properties,\n **spack.schema.repos.properties,\n **spack.schema.toolchains.properties,\n **spack.schema.upstreams.properties,\n **spack.schema.view.properties,\n}\n\n#: Canonical definitions for JSON Schema $ref\ndefs: Dict[str, Any] = {\n # Section schemas, prefixed to avoid collisions with sub-schema definitions\n **{f\"section_{name}\": schema for name, schema in sections.items()},\n # Sub-schema definitions hoisted for $ref resolution in env.py\n \"ci_job_attributes\": spack.schema.ci.ci_job_attributes,\n \"env_modifications\": spack.schema.environment.env_modifications,\n \"module_file_configuration\": spack.schema.modules.module_file_configuration,\n \"projections\": spack.schema.projections.projections,\n}\n\n#: Properties using $ref pointers into $defs\nref_sections: Dict[str, Any] = {\n name: {\"$ref\": f\"#/definitions/section_{name}\"} for name in sections\n}\n\n#: Full schema with metadata\nschema = {\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Spack merged configuration file schema\",\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": ref_sections,\n \"definitions\": defs,\n}\n" }, { "path": "lib/spack/spack/schema/mirrors.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Schema for mirrors.yaml configuration file.\n\n.. literalinclude:: _spack_root/lib/spack/spack/schema/mirrors.py\n :lines: 13-\n\"\"\"\n\nfrom typing import Any, Dict\n\n#: Common properties for connection specification\nconnection = {\n \"url\": {\n \"type\": \"string\",\n \"description\": \"URL pointing to the mirror directory, can be local filesystem \"\n \"(file://) or remote server (http://, https://, s3://, oci://)\",\n },\n \"view\": {\"type\": \"string\"},\n \"access_pair\": {\n \"type\": \"object\",\n \"description\": \"Authentication credentials for accessing private mirrors with ID and \"\n \"secret pairs\",\n \"required\": [\"secret_variable\"],\n # Only allow id or id_variable to be set, not both\n \"oneOf\": [{\"required\": [\"id\"]}, {\"required\": [\"id_variable\"]}],\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"description\": \"Static access ID or username for authentication\",\n },\n \"id_variable\": {\n \"type\": \"string\",\n \"description\": \"Environment variable name containing the access ID or username\",\n },\n \"secret_variable\": {\n \"type\": \"string\",\n \"description\": \"Environment variable name containing the secret key, password, \"\n \"or access token\",\n },\n },\n },\n \"profile\": {\n \"type\": [\"string\", \"null\"],\n \"description\": \"AWS profile name to use for S3 mirror authentication\",\n },\n \"endpoint_url\": {\n \"type\": [\"string\", \"null\"],\n \"description\": \"Custom endpoint URL for S3-compatible storage services\",\n },\n \"access_token_variable\": {\n \"type\": [\"string\", \"null\"],\n \"description\": \"Environment variable containing an access token for OCI registry \"\n \"authentication\",\n },\n}\n\n\n#: Mirror connection inside pull/push keys\nfetch_and_push = {\n \"description\": \"Mirror connection configuration for fetching or pushing packages, can be a\"\n \"simple URL string or detailed connection object\",\n \"anyOf\": [\n {\n \"type\": \"string\",\n \"description\": \"Simple URL string for basic mirror connections without authentication\",\n },\n {\n \"type\": \"object\",\n \"description\": \"Detailed connection configuration with authentication and custom \"\n \"settings\",\n \"additionalProperties\": False,\n \"properties\": {**connection},\n },\n ],\n}\n\n#: Mirror connection when no pull/push keys are set\nmirror_entry = {\n \"type\": \"object\",\n \"description\": \"Mirror configuration entry supporting both source package archives and \"\n \"binary build caches with optional authentication\",\n \"additionalProperties\": False,\n \"anyOf\": [{\"required\": [\"url\"]}, {\"required\": [\"fetch\"]}, {\"required\": [\"pull\"]}],\n \"properties\": {\n \"source\": {\n \"type\": \"boolean\",\n \"description\": \"Whether this mirror provides source package archives (tarballs) for \"\n \"building from source\",\n },\n \"binary\": {\n \"type\": \"boolean\",\n \"description\": \"Whether this mirror provides binary build caches for installing \"\n \"precompiled packages\",\n },\n \"signed\": {\n \"type\": \"boolean\",\n \"description\": \"Whether to require GPG signature verification for packages from \"\n \"this mirror\",\n },\n \"fetch\": {\n **fetch_and_push,\n \"description\": \"Configuration for fetching/downloading packages from this mirror\",\n },\n \"push\": {\n **fetch_and_push,\n \"description\": \"Configuration for pushing/uploading packages to this mirror for \"\n \"build cache creation\",\n },\n \"autopush\": {\n \"type\": \"boolean\",\n \"description\": \"Automatically push packages to this build cache immediately after \"\n \"they are installed locally\",\n },\n **connection,\n },\n}\n\n#: Properties for inclusion in other schemas\nproperties: Dict[str, Any] = {\n \"mirrors\": {\n \"type\": \"object\",\n \"default\": {},\n \"description\": \"Configure local and remote mirrors that provide repositories of source \"\n \"tarballs and binary build caches for faster package installation\",\n \"additionalProperties\": {\n \"description\": \"Named mirror configuration that can be a simple URL string or \"\n \"detailed mirror entry with authentication and build cache settings\",\n \"anyOf\": [\n {\n \"type\": \"string\",\n \"description\": \"Simple mirror URL for basic source package or build \"\n \"cache access\",\n },\n mirror_entry,\n ],\n },\n }\n}\n\n\n#: Full schema with metadata\nschema = {\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Spack mirror configuration file schema\",\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": properties,\n}\n" }, { "path": "lib/spack/spack/schema/modules.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Schema for modules.yaml configuration file.\n\n.. literalinclude:: _spack_root/lib/spack/spack/schema/modules.py\n :lines: 16-\n\"\"\"\n\nfrom typing import Any, Dict\n\nimport spack.schema.environment\nimport spack.schema.projections\n\n#: Definitions for parts of module schema\narray_of_strings = {\"type\": \"array\", \"default\": [], \"items\": {\"type\": \"string\"}}\n\ndependency_selection = {\"type\": \"string\", \"enum\": [\"none\", \"run\", \"direct\", \"all\"]}\n\nmodule_file_configuration = {\n \"type\": \"object\",\n \"default\": {},\n \"description\": \"Configuration for individual module file behavior and content customization\",\n \"additionalProperties\": False,\n \"properties\": {\n \"filter\": {\n \"type\": \"object\",\n \"default\": {},\n \"description\": \"Filter out specific environment variable modifications from \"\n \"module files\",\n \"additionalProperties\": False,\n \"properties\": {\n \"exclude_env_vars\": {\n \"type\": \"array\",\n \"default\": [],\n \"items\": {\"type\": \"string\"},\n \"description\": \"List of environment variable names to exclude from module \"\n \"file modifications\",\n }\n },\n },\n \"template\": {\n \"type\": \"string\",\n \"description\": \"Path to custom template file for generating module files\",\n },\n \"autoload\": {\n **dependency_selection,\n \"description\": \"Automatically load dependency modules when this module is loaded\",\n },\n \"prerequisites\": {\n **dependency_selection,\n \"description\": \"Mark dependency modules as prerequisites instead of autoloading them\",\n },\n \"conflict\": {\n **array_of_strings,\n \"description\": \"List of modules that conflict with this one and should not be loaded \"\n \"simultaneously\",\n },\n \"load\": {\n **array_of_strings,\n \"description\": \"List of additional modules to load when this module is loaded\",\n },\n \"suffixes\": {\n \"type\": \"object\",\n \"description\": \"Add custom suffixes to module names based on spec matching for better \"\n \"readability\",\n \"additionalKeysAreSpecs\": True,\n \"additionalProperties\": {\"type\": \"string\"}, # key\n },\n \"environment\": spack.schema.environment.ref_env_modifications,\n },\n}\n\nref_module_file_configuration = {\"$ref\": \"#/definitions/module_file_configuration\"}\n\nprojections_scheme = {\"$ref\": \"#/definitions/projections\"}\n\ncommon_props = {\n \"verbose\": {\n \"type\": \"boolean\",\n \"default\": False,\n \"description\": \"Enable verbose output during module file generation\",\n },\n \"hash_length\": {\n \"type\": \"integer\",\n \"minimum\": 0,\n \"default\": 7,\n \"description\": \"Length of package hash to include in module file names (0-32, shorter \"\n \"hashes may cause naming conflicts)\",\n },\n \"include\": {\n **array_of_strings,\n \"description\": \"List of specs to explicitly include for module file generation, even if \"\n \"they would normally be excluded\",\n },\n \"exclude\": {\n **array_of_strings,\n \"description\": \"List of specs to exclude from module file generation\",\n },\n \"exclude_implicits\": {\n \"type\": \"boolean\",\n \"default\": False,\n \"description\": \"Exclude implicit dependencies from module file generation while still \"\n \"allowing autoloading\",\n },\n \"defaults\": {\n **array_of_strings,\n \"description\": \"List of specs for which to create default module symlinks when multiple \"\n \"versions exist\",\n },\n \"hide_implicits\": {\n \"type\": \"boolean\",\n \"default\": False,\n \"description\": \"Hide implicit dependency modules from 'module avail' but still allow \"\n \"autoloading (requires module system support)\",\n },\n \"naming_scheme\": {\n \"type\": \"string\",\n \"description\": \"Custom naming scheme for module files using format strings\",\n },\n \"projections\": {\n **projections_scheme,\n \"description\": \"Custom directory structure and naming convention for module files using \"\n \"projection format\",\n },\n \"all\": ref_module_file_configuration,\n}\n\ntcl_configuration = {\n \"type\": \"object\",\n \"default\": {},\n \"description\": \"Configuration for TCL module files compatible with Environment Modules and \"\n \"Lmod\",\n \"additionalKeysAreSpecs\": True,\n \"properties\": {**common_props},\n \"additionalProperties\": ref_module_file_configuration,\n}\n\nlmod_configuration = {\n \"type\": \"object\",\n \"default\": {},\n \"description\": \"Configuration for Lua module files compatible with Lmod hierarchical module \"\n \"system\",\n \"additionalKeysAreSpecs\": True,\n \"properties\": {\n **common_props,\n \"core_compilers\": {\n **array_of_strings,\n \"description\": \"List of core compilers that are always available at the top level of \"\n \"the Lmod hierarchy\",\n },\n \"hierarchy\": {\n **array_of_strings,\n \"description\": \"List of packages to use for building the Lmod module hierarchy \"\n \"(typically compilers and MPI implementations)\",\n },\n \"core_specs\": {\n **array_of_strings,\n \"description\": \"List of specs that should be placed in the core level of the Lmod \"\n \"hierarchy regardless of dependencies\",\n },\n \"filter_hierarchy_specs\": {\n \"type\": \"object\",\n \"description\": \"Filter which specs are included at different levels of the Lmod \"\n \"hierarchy based on spec matching\",\n \"additionalKeysAreSpecs\": True,\n \"additionalProperties\": array_of_strings,\n },\n },\n \"additionalProperties\": ref_module_file_configuration,\n}\n\nmodule_config_properties = {\n \"use_view\": {\n \"anyOf\": [{\"type\": \"string\"}, {\"type\": \"boolean\"}],\n \"description\": \"Generate modules relative to an environment view instead of install \"\n \"tree (True for default view, string for named view, False to disable)\",\n },\n \"arch_folder\": {\n \"type\": \"boolean\",\n \"description\": \"Whether to include architecture-specific subdirectories in module file \"\n \"paths\",\n },\n \"roots\": {\n \"type\": \"object\",\n \"description\": \"Custom root directories for different module file types\",\n \"properties\": {\n \"tcl\": {\"type\": \"string\", \"description\": \"Root directory for TCL module files\"},\n \"lmod\": {\"type\": \"string\", \"description\": \"Root directory for Lmod module files\"},\n },\n },\n \"enable\": {\n \"type\": \"array\",\n \"default\": [],\n \"description\": \"List of module types to automatically generate during package \"\n \"installation\",\n \"items\": {\"type\": \"string\", \"enum\": [\"tcl\", \"lmod\"]},\n },\n \"lmod\": {\n **lmod_configuration,\n \"description\": \"Configuration for Lmod hierarchical module system\",\n },\n \"tcl\": {\n **tcl_configuration,\n \"description\": \"Configuration for TCL module files compatible with Environment Modules\",\n },\n \"prefix_inspections\": {\n \"type\": \"object\",\n \"description\": \"Control which package subdirectories are added to environment variables \"\n \"(e.g., bin to PATH, lib to LIBRARY_PATH)\",\n \"additionalProperties\": {\n # prefix-relative path to be inspected for existence\n **array_of_strings,\n \"description\": \"List of environment variables to update with this prefix-relative \"\n \"path if it exists\",\n },\n },\n}\n\n\n# Properties for inclusion into other schemas (requires definitions)\nproperties: Dict[str, Any] = {\n \"modules\": {\n \"type\": \"object\",\n \"description\": \"Configure automatic generation of module files for Environment Modules \"\n \"and Lmod to manage user environments at HPC centers\",\n \"properties\": {\n \"prefix_inspections\": {\n \"type\": \"object\",\n \"description\": \"Global prefix inspection settings that apply to all module sets, \"\n \"controlling which subdirectories are added to environment variables\",\n \"additionalProperties\": {\n # prefix-relative path to be inspected for existence\n **array_of_strings,\n \"description\": \"List of environment variables to update with this \"\n \"prefix-relative path if it exists\",\n },\n }\n },\n \"additionalProperties\": {\n \"type\": \"object\",\n \"default\": {},\n \"description\": \"Named module set configuration (e.g., 'default') defining how module \"\n \"files are generated for a specific set of packages\",\n \"additionalProperties\": False,\n \"properties\": module_config_properties,\n },\n }\n}\n\n#: Full schema with metadata\nschema = {\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Spack module file configuration file schema\",\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"definitions\": {\n \"module_file_configuration\": module_file_configuration,\n \"projections\": spack.schema.projections.projections,\n \"env_modifications\": spack.schema.environment.env_modifications,\n },\n \"properties\": properties,\n}\n" }, { "path": "lib/spack/spack/schema/packages.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Schema for packages.yaml configuration files.\n\n.. literalinclude:: _spack_root/lib/spack/spack/schema/packages.py\n :lines: 14-\n\"\"\"\n\nfrom typing import Any, Dict\n\nimport spack.schema.environment\n\nfrom .compilers import extra_rpaths, flags, implicit_rpaths\n\npermissions = {\n \"type\": \"object\",\n \"description\": \"File permissions settings for package installations\",\n \"additionalProperties\": False,\n \"properties\": {\n \"read\": {\n \"type\": \"string\",\n \"enum\": [\"user\", \"group\", \"world\"],\n \"description\": \"Who can read the files installed by a package\",\n },\n \"write\": {\n \"type\": \"string\",\n \"enum\": [\"user\", \"group\", \"world\"],\n \"description\": \"Who can write to the files installed by a package\",\n },\n \"group\": {\n \"type\": \"string\",\n \"description\": \"The group that owns the files installed by a package\",\n },\n },\n}\n\nvariants = {\n \"oneOf\": [{\"type\": \"string\"}, {\"type\": \"array\", \"items\": {\"type\": \"string\"}}],\n \"description\": \"Soft variant preferences as a single spec string or list of variant \"\n \"specifications (ignored if the concretizer can reuse existing installations)\",\n}\n\nrequirements = {\n \"description\": \"Package requirements that must be satisfied during concretization\",\n \"oneOf\": [\n # 'require' can be a list of requirement_groups. each requirement group is a list of one or\n # more specs. Either at least one or exactly one spec in the group must be satisfied\n # (depending on whether you use \"any_of\" or \"one_of\", respectively)\n {\n \"type\": \"array\",\n \"items\": {\n \"oneOf\": [\n {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": {\n \"one_of\": {\n \"type\": \"array\",\n \"items\": {\"type\": \"string\"},\n \"description\": \"List of specs where exactly one must be satisfied\",\n },\n \"any_of\": {\n \"type\": \"array\",\n \"items\": {\"type\": \"string\"},\n \"description\": \"List of specs where at least one must be \"\n \"satisfied\",\n },\n \"spec\": {\n \"type\": \"string\",\n \"description\": \"Single spec requirement that must be satisfied\",\n },\n \"message\": {\n \"type\": \"string\",\n \"description\": \"Custom error message when requirement is not \"\n \"satisfiable\",\n },\n \"when\": {\n \"type\": \"string\",\n \"description\": \"Conditional spec that triggers this requirement\",\n },\n },\n },\n {\"type\": \"string\"},\n ]\n },\n },\n # Shorthand for a single requirement group with one member\n {\"type\": \"string\"},\n ],\n}\n\nprefer_and_conflict = {\n \"type\": \"array\",\n \"items\": {\n \"oneOf\": [\n {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": {\n \"spec\": {\"type\": \"string\", \"description\": \"Spec constraint to apply\"},\n \"message\": {\n \"type\": \"string\",\n \"description\": \"Custom message explaining the constraint\",\n },\n \"when\": {\n \"type\": \"string\",\n \"description\": \"Conditional spec that triggers this constraint\",\n },\n },\n },\n {\"type\": \"string\"},\n ]\n },\n}\n\npackage_attributes = {\n \"type\": \"object\",\n \"description\": \"Class-level attributes to assign to package instances \"\n \"(accessible in package.py methods)\",\n \"additionalProperties\": False,\n \"patternProperties\": {r\"^[a-zA-Z_]\\w*$\": {}},\n}\n\nREQUIREMENT_URL = \"https://spack.readthedocs.io/en/latest/packages_yaml.html#package-requirements\"\n\n#: Properties for inclusion in other schemas\nproperties: Dict[str, Any] = {\n \"packages\": {\n \"type\": \"object\",\n \"description\": \"Package-specific build settings and external package configurations\",\n \"default\": {},\n \"properties\": {\n \"all\": {\n \"type\": \"object\",\n \"description\": \"Default settings that apply to all packages (can be overridden \"\n \"by package-specific settings)\",\n \"default\": {},\n \"additionalProperties\": False,\n \"properties\": {\n \"require\": requirements,\n \"prefer\": {\n \"description\": \"Strong package preferences that influence concretization \"\n \"without imposing hard constraints\",\n **prefer_and_conflict,\n },\n \"conflict\": {\n \"description\": \"Package conflicts that prevent certain spec combinations\",\n **prefer_and_conflict,\n },\n # target names\n \"target\": {\n \"type\": \"array\",\n \"description\": \"Ordered list of soft preferences for target \"\n \"architectures for all packages (ignored if the concretizer can reuse \"\n \"existing installations)\",\n \"default\": [],\n \"items\": {\"type\": \"string\"},\n },\n # compiler specs\n \"compiler\": {\n \"type\": \"array\",\n \"description\": \"Soft preferences for compiler specs for all packages \"\n \"(deprecated)\",\n \"default\": [],\n \"items\": {\"type\": \"string\"},\n },\n \"buildable\": {\n \"type\": \"boolean\",\n \"description\": \"Whether packages should be built from source (false \"\n \"prevents building)\",\n \"default\": True,\n },\n \"permissions\": permissions,\n # If 'get_full_repo' is promoted to a Package-level\n # attribute, it could be useful to set it here\n \"package_attributes\": package_attributes,\n \"providers\": {\n \"type\": \"object\",\n \"description\": \"Soft preferences for providers of virtual packages \"\n \"(ignored if the concretizer can reuse existing installations)\",\n \"default\": {},\n \"additionalProperties\": {\n \"type\": \"array\",\n \"description\": \"Ordered list of preferred providers for this virtual \"\n \"package\",\n \"default\": [],\n \"items\": {\"type\": \"string\"},\n },\n },\n \"variants\": variants,\n },\n \"deprecatedProperties\": [\n {\n \"names\": [\"compiler\"],\n \"message\": \"The packages:all:compiler preference has been deprecated in \"\n \"Spack v1.0, and is currently ignored. It will be removed from config in \"\n \"Spack v1.2.\",\n \"error\": False,\n }\n ],\n }\n },\n # package names\n \"additionalProperties\": {\n \"type\": \"object\",\n \"description\": \"Package-specific settings that override defaults from 'all'\",\n \"default\": {},\n \"additionalProperties\": False,\n \"properties\": {\n \"require\": requirements,\n \"prefer\": {\n \"description\": \"Strong package preferences that influence concretization \"\n \"without imposing hard constraints\",\n **prefer_and_conflict,\n },\n \"conflict\": {\n \"description\": \"Package conflicts that prevent certain spec combinations\",\n **prefer_and_conflict,\n },\n \"version\": {\n \"type\": \"array\",\n \"description\": \"Ordered list of soft preferences for versions for this \"\n \"package (ignored if the concretizer can reuse existing installations)\",\n \"default\": [],\n # version strings\n \"items\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"number\"}]},\n },\n \"buildable\": {\n \"type\": \"boolean\",\n \"description\": \"Whether this package should be built from source (false \"\n \"prevents building)\",\n \"default\": True,\n },\n \"permissions\": permissions,\n # If 'get_full_repo' is promoted to a Package-level\n # attribute, it could be useful to set it here\n \"package_attributes\": package_attributes,\n \"variants\": variants,\n \"externals\": {\n \"type\": \"array\",\n \"description\": \"List of external, system-installed instances of this package\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"spec\": {\n \"type\": \"string\",\n \"description\": \"Spec string describing this external package \"\n \"instance. Typically name@version and relevant variants\",\n },\n \"prefix\": {\n \"type\": \"string\",\n \"description\": \"Installation prefix path for this external \"\n \"package (typically /usr, *excluding* bin/, lib/, etc.)\",\n },\n \"modules\": {\n \"type\": \"array\",\n \"description\": \"Environment modules to load for this external \"\n \"package\",\n \"items\": {\"type\": \"string\"},\n },\n \"id\": {\"type\": \"string\"},\n \"extra_attributes\": {\n \"type\": \"object\",\n \"description\": \"Additional information needed by the package \"\n \"to use this external\",\n \"additionalProperties\": {\"type\": \"string\"},\n \"properties\": {\n \"compilers\": {\n \"type\": \"object\",\n \"description\": \"Compiler executable paths for external \"\n \"compiler packages\",\n \"properties\": {\n \"c\": {\n \"type\": \"string\",\n \"description\": \"Path to the C compiler \"\n \"executable (e.g. /usr/bin/gcc)\",\n },\n \"cxx\": {\n \"type\": \"string\",\n \"description\": \"Path to the C++ compiler \"\n \"executable (e.g. /usr/bin/g++)\",\n },\n \"fortran\": {\n \"type\": \"string\",\n \"description\": \"Path to the Fortran compiler \"\n \"executable (e.g. /usr/bin/gfortran)\",\n },\n },\n \"patternProperties\": {r\"^\\w\": {\"type\": \"string\"}},\n \"additionalProperties\": False,\n },\n \"environment\": spack.schema.environment.ref_env_modifications,\n \"extra_rpaths\": extra_rpaths,\n \"implicit_rpaths\": implicit_rpaths,\n \"flags\": flags,\n },\n },\n \"dependencies\": {\n \"type\": \"array\",\n \"description\": \"List of dependencies for this external package, \"\n \"specifying dependency relationships explicitly\",\n \"items\": {\n \"type\": \"object\",\n \"description\": \"Dependency specification for an external \"\n \"package\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"description\": \"Explicit reference ID to another \"\n \"external package (provides unambiguous reference)\",\n },\n \"spec\": {\n \"type\": \"string\",\n \"description\": \"Spec string that matches an \"\n \"available external package\",\n },\n \"deptypes\": {\n \"oneOf\": [\n {\n \"type\": \"string\",\n \"description\": \"Single dependency type \"\n \"(e.g., 'build', 'link', 'run', 'test')\",\n },\n {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\",\n \"description\": \"Dependency type (e.g., \"\n \"'build', 'link', 'run', 'test')\",\n },\n \"description\": \"List of dependency types \"\n \"(e.g., ['build', 'link'])\",\n },\n ],\n \"description\": \"Dependency types; if not specified, \"\n \"inferred from package recipe\",\n },\n \"virtuals\": {\n \"type\": \"string\",\n \"description\": \"Virtual package name this dependency \"\n \"provides (e.g., 'mpi')\",\n },\n },\n },\n },\n },\n \"additionalProperties\": False,\n \"required\": [\"spec\"],\n },\n },\n },\n },\n }\n}\n\n#: Full schema with metadata\nschema = {\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Spack package configuration file schema\",\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": properties,\n \"definitions\": {\"env_modifications\": spack.schema.environment.env_modifications},\n}\n\n\ndef update(data):\n data = data[\"packages\"]\n changed = False\n for key in data:\n version = data[key].get(\"version\")\n if not version or all(isinstance(v, str) for v in version):\n continue\n\n data[key][\"version\"] = [str(v) for v in version]\n changed = True\n\n return changed\n" }, { "path": "lib/spack/spack/schema/projections.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Schema for projections.yaml configuration file.\n\n.. literalinclude:: _spack_root/lib/spack/spack/schema/projections.py\n :lines: 14-\n\"\"\"\n\nfrom typing import Any, Dict\n\n#: Properties for inclusion in other schemas\nprojections: Dict[str, Any] = {\n \"type\": \"object\",\n \"description\": \"Customize directory structure and naming schemes by mapping specs to \"\n \"format strings.\",\n \"properties\": {\n \"all\": {\n \"type\": \"string\",\n \"description\": \"Default projection format string used as fallback for all specs \"\n \"that do not match other entries. Uses spec format syntax like \"\n '\"{name}/{version}/{hash:16}\".',\n }\n },\n \"additionalKeysAreSpecs\": True,\n \"additionalProperties\": {\n \"type\": \"string\",\n \"description\": \"Projection format string for specs matching this key. Uses spec \"\n \"format syntax supporting tokens like {name}, {version}, {compiler.name}, \"\n \"{^dependency.name}, etc.\",\n },\n}\n\n\n#: $ref pointer for use in merged schema\nref_properties: Dict[str, Any] = {\"projections\": {\"$ref\": \"#/definitions/projections\"}}\n\n#: Full schema with metadata\nschema = {\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Spack view projection configuration file schema\",\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": ref_properties,\n \"definitions\": {\"projections\": projections},\n}\n" }, { "path": "lib/spack/spack/schema/repos.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Schema for repos.yaml configuration file.\n\n.. literalinclude:: _spack_root/lib/spack/spack/schema/repos.py\n :lines: 18-\n\"\"\"\n\nfrom typing import Any, Dict\n\n#: Properties for inclusion in other schemas\nproperties: Dict[str, Any] = {\n \"repos\": {\n \"description\": \"Configuration for package repositories that Spack searches for packages\",\n \"oneOf\": [\n {\n # old format: array of strings\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\",\n \"description\": \"Path to a Spack package repository directory\",\n },\n \"description\": \"Legacy format: list of local paths to package repository \"\n \"directories\",\n },\n {\n # new format: object with named repositories\n \"type\": \"object\",\n \"description\": \"Named repositories mapping configuration names to repository \"\n \"definitions\",\n \"additionalProperties\": {\n \"oneOf\": [\n {\n # local path\n \"type\": \"string\",\n \"description\": \"Path to a local Spack package repository directory \"\n \"containing repo.yaml and packages/\",\n },\n {\n # remote git repository\n \"type\": \"object\",\n \"properties\": {\n \"git\": {\n \"type\": \"string\",\n \"description\": \"Git repository URL for remote package \"\n \"repository\",\n },\n \"branch\": {\n \"type\": \"string\",\n \"description\": \"Git branch name to checkout (default branch \"\n \"if not specified)\",\n },\n \"commit\": {\n \"type\": \"string\",\n \"description\": \"Specific git commit hash to pin the \"\n \"repository to\",\n },\n \"tag\": {\n \"type\": \"string\",\n \"description\": \"Git tag name to pin the repository to\",\n },\n \"destination\": {\n \"type\": \"string\",\n \"description\": \"Custom local directory path where the Git \"\n \"repository should be cloned\",\n },\n \"paths\": {\n \"type\": \"array\",\n \"items\": {\"type\": \"string\"},\n \"description\": \"List of relative paths (from the Git \"\n \"repository root) that contain Spack package repositories \"\n \"(overrides spack-repo-index.yaml)\",\n },\n },\n \"additionalProperties\": False,\n },\n ]\n },\n },\n ],\n \"default\": {},\n }\n}\n\n#: Full schema with metadata\nschema = {\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Spack repository configuration file schema\",\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": properties,\n}\n\n\ndef update(data: Dict[str, Any]) -> bool:\n \"\"\"Update the repos.yaml configuration data to the new format.\"\"\"\n if not isinstance(data[\"repos\"], list):\n return False\n\n from spack.llnl.util import tty\n from spack.repo import from_path\n\n # Convert old format [paths...] to new format {namespace: path, ...}\n repos = {}\n for path in data[\"repos\"]:\n try:\n repo = from_path(path)\n except Exception as e:\n tty.warn(f\"package repository {path} is disabled due to: {e}\")\n continue\n if repo.namespace is not None:\n repos[repo.namespace] = path\n\n data[\"repos\"] = repos\n return True\n" }, { "path": "lib/spack/spack/schema/spec.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Schema for a spec found in spec descriptor or database index.json files\n\n.. literalinclude:: _spack_root/lib/spack/spack/schema/spec.py\n :lines: 15-\n\"\"\"\n\nfrom typing import Any, Dict\n\ntarget = {\n \"description\": \"Target architecture (string for abstract specs, object for concrete specs)\",\n \"oneOf\": [\n {\n \"type\": \"string\",\n \"description\": 'Target as a string (e.g. \"zen2\" or \"haswell:broadwell\") used in '\n \"abstract specs\",\n },\n {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"required\": [\"name\", \"vendor\", \"features\", \"generation\", \"parents\"],\n \"properties\": {\n \"name\": {\"type\": \"string\"},\n \"vendor\": {\"type\": \"string\"},\n \"features\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}},\n \"generation\": {\"type\": \"integer\"},\n \"parents\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}},\n \"cpupart\": {\"type\": \"string\"},\n },\n \"description\": \"Target as an object with detailed fields, used in concrete specs\",\n },\n ],\n}\n\narch = {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": {\n \"platform\": {\n \"type\": [\"string\", \"null\"],\n \"description\": 'Target platform (e.g. \"linux\" or \"darwin\"). May be null for abstract '\n \"specs\",\n },\n \"platform_os\": {\n \"type\": [\"string\", \"null\"],\n \"description\": 'Target operating system (e.g. \"ubuntu24.04\"). May be '\n \"null for abstract specs\",\n },\n \"target\": target,\n },\n}\n\n#: Corresponds to specfile format v1\ndependencies_v1 = {\n \"type\": \"object\",\n \"description\": \"Specfile v1 style dependencies specification (package name to dependency \"\n \"info)\",\n \"additionalProperties\": {\n \"type\": \"object\",\n \"properties\": {\n \"hash\": {\"type\": \"string\", \"description\": \"Unique identifier of the dependency\"},\n \"type\": {\n \"type\": \"array\",\n \"items\": {\"enum\": [\"build\", \"link\", \"run\", \"test\"]},\n \"description\": \"Dependency types\",\n },\n },\n },\n}\n\n#: Corresponds to specfile format v2-v3\ndependencies_v2_v3 = {\n \"type\": \"array\",\n \"description\": \"Specfile v2-v3 style dependencies specification (array of dependencies)\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"required\": [\"name\", \"hash\", \"type\"],\n \"properties\": {\n \"name\": {\"type\": \"string\", \"description\": \"Name of the dependency package\"},\n \"hash\": {\"type\": \"string\", \"description\": \"Unique identifier of the dependency\"},\n \"type\": {\n \"type\": \"array\",\n \"items\": {\"enum\": [\"build\", \"link\", \"run\", \"test\"]},\n \"description\": \"Dependency types\",\n },\n },\n },\n}\n\n#: Corresponds to specfile format v4+\ndependencies_v4_plus = {\n \"type\": \"array\",\n \"description\": \"Specfile v4+ style dependencies specification (array of dependencies)\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"required\": [\"name\", \"hash\", \"parameters\"],\n \"properties\": {\n \"name\": {\"type\": \"string\", \"description\": \"Name of the dependency package\"},\n \"hash\": {\"type\": \"string\", \"description\": \"Unique identifier of the dependency\"},\n \"parameters\": {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"required\": [\"deptypes\", \"virtuals\"],\n \"properties\": {\n \"deptypes\": {\n \"type\": \"array\",\n \"items\": {\"enum\": [\"build\", \"link\", \"run\", \"test\"]},\n \"description\": \"Dependency types\",\n },\n \"virtuals\": {\n \"type\": \"array\",\n \"items\": {\"type\": \"string\"},\n \"description\": \"Virtual dependencies used by the parent\",\n },\n \"direct\": {\n \"type\": \"boolean\",\n \"description\": \"Whether the dependency is direct (only on abstract specs)\",\n },\n },\n },\n },\n },\n}\n\ndependencies = {\"oneOf\": [dependencies_v1, dependencies_v2_v3, dependencies_v4_plus]}\n\nbuild_spec = {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"required\": [\"name\", \"hash\"],\n \"properties\": {\"name\": {\"type\": \"string\"}, \"hash\": {\"type\": \"string\"}},\n \"description\": \"Records the origin spec as it was built (used in splicing)\",\n}\n\n#: Schema for a single spec node (used in both spec files and database entries)\nspec_node = {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"required\": [\"name\"],\n \"properties\": {\n \"name\": {\n \"type\": [\"string\", \"null\"],\n \"description\": \"Name is a string for concrete specs, but may be null for abstract \"\n \"specs\",\n },\n \"hash\": {\"type\": \"string\", \"description\": \"The DAG hash, which identifies the spec\"},\n \"package_hash\": {\"type\": \"string\", \"description\": \"The package hash (concrete specs)\"},\n \"full_hash\": {\n \"type\": \"string\",\n \"description\": \"This hash was used on some specs prior to 0.18\",\n },\n \"build_hash\": {\n \"type\": \"string\",\n \"description\": \"This hash was used on some specs prior to 0.18\",\n },\n \"version\": {\"type\": \"string\", \"description\": \"A single, concrete version (e.g. @=1.2)\"},\n \"versions\": {\n \"type\": \"array\",\n \"items\": {\"type\": \"string\"},\n \"description\": \"Abstract version (e.g. @1.2)\",\n },\n \"propagate\": {\n \"type\": \"array\",\n \"items\": {\"type\": \"string\"},\n \"description\": \"List of variants to propagate (for abstract specs)\",\n },\n \"abstract\": {\n \"type\": \"array\",\n \"items\": {\"type\": \"string\"},\n \"description\": \"List of multi-valued variants that are abstract, i.e. foo=bar,baz \"\n \"instead of foo:=bar,baz (for abstract specs)\",\n },\n \"concrete\": {\n \"type\": \"boolean\",\n \"description\": \"Whether the spec is concrete or not, when omitted defaults to true\",\n },\n \"arch\": arch,\n \"compiler\": {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": {\"name\": {\"type\": \"string\"}, \"version\": {\"type\": \"string\"}},\n \"description\": \"Compiler name and version (in spec file v5 listed as normal \"\n \"dependencies)\",\n },\n \"namespace\": {\"type\": \"string\", \"description\": \"Package repository namespace\"},\n \"parameters\": {\n \"type\": \"object\",\n \"additionalProperties\": True,\n \"description\": \"Variants and other parameters\",\n \"properties\": {\n \"patches\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}},\n \"cflags\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}},\n \"cppflags\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}},\n \"cxxflags\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}},\n \"fflags\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}},\n \"ldflags\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}},\n \"ldlibs\": {\"type\": \"array\", \"items\": {\"type\": \"string\"}},\n },\n },\n \"patches\": {\n \"type\": \"array\",\n \"items\": {\"type\": \"string\"},\n \"description\": \"List of patches, similar to the patches variant under parameters\",\n },\n \"dependencies\": dependencies,\n \"build_spec\": build_spec,\n \"external\": {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"description\": \"If path or module (or both) are set, the spec is an external \"\n \"system-installed package\",\n \"properties\": {\n \"path\": {\n \"type\": [\"string\", \"null\"],\n \"description\": \"Install prefix on the system, e.g. /usr\",\n },\n \"module\": {\n \"anyOf\": [{\"type\": \"array\", \"items\": {\"type\": \"string\"}}, {\"type\": \"null\"}],\n \"description\": 'List of module names, e.g. [\"pkg/1.2\"]',\n },\n \"extra_attributes\": {\n \"type\": \"object\",\n \"description\": \"Package.py specific attributes to use the external package, \"\n \"such as paths to compiler executables\",\n },\n },\n },\n \"annotations\": {\n \"type\": \"object\",\n \"properties\": {\n \"original_specfile_version\": {\"type\": \"number\"},\n \"compiler\": {\"type\": \"string\"},\n },\n \"required\": [\"original_specfile_version\"],\n \"additionalProperties\": False,\n \"description\": \"Currently used to preserve compiler information of old specs when \"\n \"upgrading to a newer spec format\",\n },\n },\n}\n\n#: Properties for inclusion in other schemas\nproperties: Dict[str, Any] = {\n \"spec\": {\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"required\": [\"_meta\", \"nodes\"],\n \"properties\": {\n \"_meta\": {\n \"type\": \"object\",\n \"properties\": {\"version\": {\"type\": \"number\"}},\n \"description\": \"Spec schema version metadata, used for parsing spec files\",\n },\n \"nodes\": {\n \"type\": \"array\",\n \"items\": spec_node,\n \"description\": \"List of spec nodes which, combined with dependencies, induce a \"\n \"DAG\",\n },\n },\n }\n}\n\n#: Full schema with metadata\nschema = {\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Spack spec schema\",\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"required\": [\"spec\"],\n \"properties\": properties,\n}\n" }, { "path": "lib/spack/spack/schema/spec_list.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nmatrix_schema = {\n \"type\": \"array\",\n \"description\": \"List of spec constraint lists whose cross product generates multiple specs\",\n \"items\": {\n \"type\": \"array\",\n \"description\": \"List of spec constraints for this matrix dimension\",\n \"items\": {\"type\": \"string\"},\n },\n}\n\nspec_list_properties = {\n \"matrix\": matrix_schema,\n \"exclude\": {\n \"type\": \"array\",\n \"description\": \"List of specific spec combinations to exclude from the matrix\",\n \"items\": {\"type\": \"string\"},\n },\n}\n\nspec_list_schema = {\n \"type\": \"array\",\n \"description\": \"List of specs to include in the environment, supporting both simple specs and \"\n \"matrix configurations\",\n \"default\": [],\n \"items\": {\n \"anyOf\": [\n {\n \"type\": \"object\",\n \"description\": \"Matrix configuration for generating multiple specs from \"\n \"combinations of constraints\",\n \"additionalProperties\": False,\n \"properties\": {**spec_list_properties},\n },\n {\"type\": \"string\", \"description\": \"Simple spec string\"},\n {\"type\": \"null\"},\n ]\n },\n}\n" }, { "path": "lib/spack/spack/schema/toolchains.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Schema for toolchains.yaml configuration file.\n\n.. literalinclude:: _spack_root/lib/spack/spack/schema/toolchains.py\n :lines: 13-\n\"\"\"\n\nfrom typing import Any, Dict\n\n#: Properties for inclusion in other schemas\nproperties: Dict[str, Any] = {\n \"toolchains\": {\n \"type\": \"object\",\n \"default\": {},\n \"description\": \"Define named compiler sets (toolchains) that group compiler constraints \"\n \"under a single user-defined name for easy reference with specs like %my_toolchain\",\n \"additionalProperties\": {\n \"description\": \"Named toolchain definition that can be referenced in specs to apply \"\n \"a complex set of compiler choices for C, C++, and Fortran\",\n \"oneOf\": [\n {\n \"type\": \"string\",\n \"description\": \"Simple toolchain alias containing a spec string directly\",\n },\n {\n \"type\": \"array\",\n \"description\": \"List of conditional compiler constraints and specifications \"\n \"that define the toolchain behavior\",\n \"items\": {\n \"type\": \"object\",\n \"description\": \"Individual toolchain entry with a spec constraint and \"\n \"optional condition for when it applies\",\n \"properties\": {\n \"spec\": {\n \"type\": \"string\",\n \"description\": \"Spec constraint to apply such as compiler \"\n \"selection (%c=llvm), flags (cflags=-O3), or other virtual \"\n \"dependencies (%mpi=openmpi)\",\n },\n \"when\": {\n \"type\": \"string\",\n \"description\": \"Condition that determines when this spec \"\n \"constraint is applied, typically checking for language \"\n \"dependencies like %c, %cxx, %fortran, or other virtual packages \"\n \"like %mpi\",\n },\n },\n },\n },\n ],\n \"default\": [],\n },\n }\n}\n\n\n#: Full schema with metadata\nschema = {\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Spack toolchain configuration file schema\",\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": properties,\n}\n" }, { "path": "lib/spack/spack/schema/upstreams.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nfrom typing import Any, Dict\n\n#: Properties for inclusion in other schemas\nproperties: Dict[str, Any] = {\n \"upstreams\": {\n \"type\": \"object\",\n \"default\": {},\n \"description\": \"Configuration for chaining Spack installations. Point this Spack \"\n \"instance to other Spack installations to use their installed packages\",\n \"additionalProperties\": {\n \"type\": \"object\",\n \"default\": {},\n \"additionalProperties\": False,\n \"description\": \"Named upstream Spack instance configuration\",\n \"properties\": {\n \"install_tree\": {\n \"type\": \"string\",\n \"description\": \"Path to the opt/spack directory of the upstream Spack \"\n \"installation (or the install_tree root from its config.yaml)\",\n },\n \"modules\": {\n \"type\": \"object\",\n \"description\": \"Configuration to use modules generated by the upstream \"\n \"Spack instance\",\n \"properties\": {\n \"tcl\": {\n \"type\": \"string\",\n \"description\": \"Path to TCL modules directory of the upstream \"\n \"instance\",\n },\n \"lmod\": {\n \"type\": \"string\",\n \"description\": \"Path to Lmod modules directory of the upstream \"\n \"instance\",\n },\n },\n },\n },\n },\n }\n}\n\n#: Full schema with metadata\nschema = {\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Spack core configuration file schema\",\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": properties,\n}\n" }, { "path": "lib/spack/spack/schema/url_buildcache_manifest.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Schema for buildcache entry manifest file\n\n.. literalinclude:: _spack_root/lib/spack/spack/schema/url_buildcache_manifest.py\n :lines: 11-\n\"\"\"\n\nfrom typing import Any, Dict\n\nproperties: Dict[str, Any] = {\n \"version\": {\"type\": \"integer\"},\n \"data\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"required\": [\n \"contentLength\",\n \"mediaType\",\n \"compression\",\n \"checksumAlgorithm\",\n \"checksum\",\n ],\n \"properties\": {\n \"contentLength\": {\"type\": \"integer\"},\n \"mediaType\": {\"type\": \"string\"},\n \"compression\": {\"type\": \"string\"},\n \"checksumAlgorithm\": {\"type\": \"string\"},\n \"checksum\": {\"type\": \"string\"},\n },\n \"additionalProperties\": True,\n },\n },\n}\n\n#: Full schema with metadata\nschema = {\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Buildcache manifest schema\",\n \"type\": \"object\",\n \"required\": [\"version\", \"data\"],\n \"additionalProperties\": True,\n \"properties\": properties,\n}\n" }, { "path": "lib/spack/spack/schema/view.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Schema for view\n\n.. literalinclude:: _spack_root/lib/spack/spack/schema/view.py\n :lines: 15-\n\"\"\"\n\nfrom typing import Any, Dict\n\nimport spack.schema.projections\n\n#: Properties for inclusion in other schemas\nproperties: Dict[str, Any] = {\n \"view\": {\n \"description\": \"Environment filesystem view configuration for creating a directory with \"\n \"traditional structure where all files of installed packages are linked\",\n \"anyOf\": [\n {\n \"type\": \"boolean\",\n \"description\": \"Enable or disable default views. If 'true', the view is \"\n \"generated under .spack-env/view\",\n },\n {\"type\": \"string\", \"description\": \"Path where the default view should be created\"},\n {\n \"type\": \"object\",\n \"description\": \"Advanced view configuration with one or more named view \"\n \"descriptors\",\n \"additionalProperties\": {\n \"description\": \"Named view descriptor (use 'default' for the view activated \"\n \"with environment)\",\n \"required\": [\"root\"],\n \"additionalProperties\": False,\n \"properties\": {\n \"root\": {\n \"type\": \"string\",\n \"description\": \"Root directory path where the view will be created\",\n },\n \"group\": {\n \"oneOf\": [\n {\n \"type\": \"array\",\n \"items\": {\"type\": \"string\"},\n \"description\": \"Groups of specs to include in the view\",\n },\n {\n \"type\": \"string\",\n \"description\": \"Groups of specs to include in the view\",\n },\n ]\n },\n \"link\": {\n \"enum\": [\"roots\", \"all\", \"run\"],\n \"description\": \"Which specs to include: 'all' (environment roots \"\n \"with transitive run+link deps), 'run' (environment roots with \"\n \"transitive run deps), 'roots' (environment roots only)\",\n },\n \"link_type\": {\n \"type\": \"string\",\n \"enum\": [\"symlink\", \"hardlink\", \"copy\"],\n \"description\": \"How files are linked in the view: 'symlink' \"\n \"(default), 'hardlink', or 'copy'\",\n },\n \"select\": {\n \"type\": \"array\",\n \"items\": {\"type\": \"string\"},\n \"description\": \"List of specs to include in the view \"\n \"(default: select everything)\",\n },\n \"exclude\": {\n \"type\": \"array\",\n \"items\": {\"type\": \"string\"},\n \"description\": \"List of specs to exclude from the view \"\n \"(default: exclude nothing)\",\n },\n **spack.schema.projections.ref_properties,\n },\n },\n },\n ],\n }\n}\n\n#: Full schema with metadata\nschema = {\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Spack view configuration file schema\",\n \"type\": \"object\",\n \"additionalProperties\": False,\n \"properties\": properties,\n \"definitions\": {\"projections\": spack.schema.projections.projections},\n}\n" }, { "path": "lib/spack/spack/solver/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n" }, { "path": "lib/spack/spack/solver/asp.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport collections\nimport collections.abc\nimport enum\nimport functools\nimport gzip\nimport io\nimport itertools\nimport json\nimport os\nimport pathlib\nimport pprint\nimport random\nimport re\nimport sys\nimport time\nimport warnings\nfrom typing import (\n Any,\n Callable,\n Dict,\n Generator,\n Iterable,\n Iterator,\n List,\n NamedTuple,\n Optional,\n Sequence,\n Set,\n Tuple,\n Type,\n Union,\n)\n\nimport spack.vendor.archspec.cpu\n\nimport spack\nimport spack.caches\nimport spack.compilers.config\nimport spack.compilers.flags\nimport spack.concretize\nimport spack.config\nimport spack.deptypes as dt\nimport spack.error\nimport spack.llnl.util.lang\nimport spack.llnl.util.tty as tty\nimport spack.package_base\nimport spack.package_prefs\nimport spack.platforms\nimport spack.repo\nimport spack.solver.splicing\nimport spack.spec\nimport spack.store\nimport spack.util.crypto\nimport spack.util.hash\nimport spack.util.lock as lk\nimport spack.util.module_cmd as md\nimport spack.util.path\nimport spack.util.timer\nimport spack.variant as vt\nimport spack.version as vn\nimport spack.version.git_ref_lookup\nfrom spack import traverse\nfrom spack.compilers.libraries import CompilerPropertyDetector\nfrom spack.llnl.util.lang import elide_list\nfrom spack.spec import EMPTY_SPEC\nfrom spack.util.compression import GZipFileType\n\nfrom .core import (\n AspFunction,\n AspVar,\n NodeId,\n SourceContext,\n clingo,\n extract_args,\n fn,\n using_libc_compatibility,\n)\nfrom .input_analysis import create_counter, create_graph_analyzer\nfrom .requirements import RequirementKind, RequirementOrigin, RequirementParser, RequirementRule\nfrom .reuse import ReusableSpecsSelector, SpecFiltersFactory, create_external_parser\nfrom .runtimes import RuntimePropertyRecorder, all_libcs, external_config_with_implicit_externals\nfrom .versions import Provenance\n\nGitOrStandardVersion = Union[vn.GitVersion, vn.StandardVersion]\n\nTransformFunction = Callable[[str, spack.spec.Spec, List[AspFunction]], List[AspFunction]]\n\n\nclass OutputConfiguration(NamedTuple):\n \"\"\"Data class that contains configuration on what a clingo solve should output.\"\"\"\n\n #: Print out coarse timers for different solve phases\n timers: bool\n #: Whether to output Clingo's internal solver statistics\n stats: bool\n #: Optional output stream for the generated ASP program\n out: Optional[io.IOBase]\n #: If True, stop after setup and don't solve\n setup_only: bool\n\n\n#: Default output configuration for a solve\nDEFAULT_OUTPUT_CONFIGURATION = OutputConfiguration(\n timers=False, stats=False, out=None, setup_only=False\n)\n\n\ndef default_clingo_control():\n \"\"\"Return a control object with the default settings used in Spack\"\"\"\n control = clingo().Control()\n control.configuration.configuration = \"tweety\"\n control.configuration.solver.heuristic = \"Domain\"\n control.configuration.solver.opt_strategy = \"usc\"\n return control\n\n\n# Below numbers are used to map names of criteria to the order\n# they appear in the solution. See concretize.lp\n\n# The space of possible priorities for optimization targets\n# is partitioned in the following ranges:\n#\n# [0-100) Optimization criteria for software being reused\n# [100-200) Fixed criteria that are higher priority than reuse, but lower than build\n# [200-300) Optimization criteria for software being built\n# [300-1000) High-priority fixed criteria\n# [1000-inf) Error conditions\n#\n# Each optimization target is a minimization with optimal value 0.\n\n#: High fixed priority offset for criteria that supersede all build criteria\nhigh_fixed_priority_offset = 300\n\n#: Priority offset for \"build\" criteria (regular criterio shifted to\n#: higher priority for specs we have to build)\nbuild_priority_offset = 200\n\n#: Priority offset of \"fixed\" criteria (those w/o build criteria)\nfixed_priority_offset = 100\n\n\nclass OptimizationKind:\n \"\"\"Enum for the optimization KIND of a criteria.\n\n It's not using enum.Enum since it must be serializable.\n \"\"\"\n\n BUILD = 0\n CONCRETE = 1\n OTHER = 2\n\n\nclass OptimizationCriteria(NamedTuple):\n \"\"\"A named tuple describing an optimization criteria.\"\"\"\n\n priority: int\n value: int\n name: str\n kind: OptimizationKind\n\n\ndef build_criteria_names(costs, arg_tuples):\n \"\"\"Construct an ordered mapping from criteria names to costs.\"\"\"\n # pull optimization criteria names out of the solution\n priorities_names = []\n\n for args in arg_tuples:\n priority, name = args[:2]\n priority = int(priority)\n\n # Add the priority of this opt criterion and its name\n if priority < fixed_priority_offset:\n # if the priority is less than fixed_priority_offset, then it\n # has an associated build priority -- the same criterion but for\n # nodes that we have to build.\n priorities_names.append((priority, name, OptimizationKind.CONCRETE))\n build_priority = priority + build_priority_offset\n priorities_names.append((build_priority, name, OptimizationKind.BUILD))\n else:\n priorities_names.append((priority, name, OptimizationKind.OTHER))\n\n # sort the criteria by priority\n priorities_names = sorted(priorities_names, reverse=True)\n\n # We only have opt-criterion values for non-error types\n # error type criteria are excluded (they come first)\n error_criteria = len(costs) - len(priorities_names)\n costs = costs[error_criteria:]\n\n return [\n OptimizationCriteria(priority, value, name, status)\n for (priority, name, status), value in zip(priorities_names, costs)\n ]\n\n\ndef specify(spec):\n if isinstance(spec, spack.spec.Spec):\n return spec\n return spack.spec.Spec(spec)\n\n\n# Caching because the returned function id is used as a cache key\n@functools.lru_cache(maxsize=None)\ndef remove_facts(*to_be_removed: str) -> TransformFunction:\n \"\"\"Returns a transformation function that removes facts from the input list of facts.\"\"\"\n\n def _remove(name: str, spec: spack.spec.Spec, facts: List[AspFunction]) -> List[AspFunction]:\n return [x for x in facts if x.args[0] not in to_be_removed]\n\n return _remove\n\n\ndef identity_for_facts(\n name: str, spec: spack.spec.Spec, facts: List[AspFunction]\n) -> List[AspFunction]:\n return facts\n\n\n# Caching because the returned function id is used as a cache key\n@functools.lru_cache(maxsize=None)\ndef dependency_holds(\n *, dependency_flags: dt.DepFlag, pkg_cls: Type[spack.package_base.PackageBase]\n) -> TransformFunction:\n def _transform_fn(\n name: str, input_spec: spack.spec.Spec, requirements: List[AspFunction]\n ) -> List[AspFunction]:\n result = remove_facts(\"node\", \"virtual_node\")(name, input_spec, requirements) + [\n fn.attr(\"dependency_holds\", pkg_cls.name, name, dt.flag_to_string(t))\n for t in dt.ALL_FLAGS\n if t & dependency_flags\n ]\n if name not in pkg_cls.extendees:\n return result\n return result + [fn.attr(\"extends\", pkg_cls.name, name)]\n\n return _transform_fn\n\n\ndef dag_closure_by_deptype(\n name: str, spec: spack.spec.Spec, facts: List[AspFunction]\n) -> List[AspFunction]:\n edges = spec.edges_to_dependencies()\n # Compute the \"link\" transitive closure with `when: root ^[deptypes=link] `\n if len(edges) == 1:\n edge = edges[0]\n if not edge.direct and edge.depflag == dt.LINK | dt.RUN:\n root, leaf = edge.parent.name, edge.spec.name\n return [fn.attr(\"closure\", root, leaf, \"linkrun\")]\n return facts\n\n\ndef libc_is_compatible(lhs: spack.spec.Spec, rhs: spack.spec.Spec) -> bool:\n return (\n lhs.name == rhs.name\n and lhs.external_path == rhs.external_path\n and lhs.version >= rhs.version\n )\n\n\ndef c_compiler_runs(compiler) -> bool:\n return CompilerPropertyDetector(compiler).compiler_verbose_output() is not None\n\n\ndef extend_flag_list(flag_list, new_flags):\n \"\"\"Extend a list of flags, preserving order and precedence.\n\n Add new_flags at the end of flag_list. If any flags in new_flags are\n already in flag_list, they are moved to the end so that they take\n higher precedence on the compile line.\n\n \"\"\"\n for flag in new_flags:\n if flag in flag_list:\n flag_list.remove(flag)\n flag_list.append(flag)\n\n\ndef _reorder_flags(flag_list: List[spack.spec.CompilerFlag]) -> List[spack.spec.CompilerFlag]:\n \"\"\"Reorder a list of flags to ensure that the order matches that of the flag group.\"\"\"\n if not flag_list:\n return []\n\n if len({x.flag_group for x in flag_list}) != 1 or len({x.source for x in flag_list}) != 1:\n raise InternalConcretizerError(\n \"internal solver error: cannot reorder compiler flags for concretized specs. \"\n \"Please report a bug at https://github.com/spack/spack/issues\"\n )\n\n flag_group = flag_list[0].flag_group\n flag_source = flag_list[0].source\n flag_propagate = flag_list[0].propagate\n # Once we have the flag_group, no need to iterate over the flag_list because the\n # group represents all of them\n return [\n spack.spec.CompilerFlag(\n flag, propagate=flag_propagate, flag_group=flag_group, source=flag_source\n )\n for flag, propagate in spack.compilers.flags.tokenize_flags(\n flag_group, propagate=flag_propagate\n )\n ]\n\n\ndef check_packages_exist(specs):\n \"\"\"Ensure all packages mentioned in specs exist.\"\"\"\n repo = spack.repo.PATH\n for spec in specs:\n for s in spec.traverse():\n try:\n check_passed = repo.repo_for_pkg(s).exists(s.name) or repo.is_virtual(s.name)\n except Exception as e:\n msg = \"Cannot find package: {0}\".format(str(e))\n check_passed = False\n tty.debug(msg)\n\n if not check_passed:\n raise spack.repo.UnknownPackageError(str(s.fullname))\n\n\nclass Result:\n \"\"\"Result of an ASP solve.\"\"\"\n\n def __init__(self, specs):\n self.satisfiable = None\n self.optimal = None\n self.warnings = None\n self.nmodels = 0\n\n # specs ordered by optimization level\n self.answers = []\n\n # names of optimization criteria\n self.criteria = []\n\n # Abstract user requests\n self.abstract_specs = specs\n\n # possible dependencies\n self.possible_dependencies = None\n\n # Concrete specs\n self._concrete_specs_by_input = None\n self._concrete_specs = None\n self._unsolved_specs = None\n\n def raise_if_unsat(self):\n \"\"\"Raise a generic internal error if the result is unsatisfiable.\"\"\"\n if self.satisfiable:\n return\n\n constraints = self.abstract_specs\n if len(constraints) == 1:\n constraints = constraints[0]\n\n raise SolverError(constraints)\n\n @property\n def specs(self):\n \"\"\"List of concretized specs satisfying the initial\n abstract request.\n \"\"\"\n if self._concrete_specs is None:\n self._compute_specs_from_answer_set()\n return self._concrete_specs\n\n @property\n def unsolved_specs(self):\n \"\"\"List of tuples pairing abstract input specs that were not\n solved with their associated candidate spec from the solver\n (if the solve completed).\n \"\"\"\n if self._unsolved_specs is None:\n self._compute_specs_from_answer_set()\n return self._unsolved_specs\n\n @property\n def specs_by_input(self) -> Dict[spack.spec.Spec, spack.spec.Spec]:\n if self._concrete_specs_by_input is None:\n self._compute_specs_from_answer_set()\n return self._concrete_specs_by_input # type: ignore\n\n def _compute_specs_from_answer_set(self):\n if not self.satisfiable:\n self._concrete_specs = []\n self._unsolved_specs = list((x, None) for x in self.abstract_specs)\n self._concrete_specs_by_input = {}\n return\n\n self._concrete_specs, self._unsolved_specs = [], []\n self._concrete_specs_by_input = {}\n best = min(self.answers)\n opt, _, answer = best\n for input_spec in self.abstract_specs:\n # The specs must be unified to get here, so it is safe to associate any satisfying spec\n # with the input. Multiple inputs may be matched to the same concrete spec\n node = SpecBuilder.make_node(pkg=input_spec.name)\n if spack.repo.PATH.is_virtual(input_spec.name):\n providers = [\n spec.name for spec in answer.values() if spec.package.provides(input_spec.name)\n ]\n node = SpecBuilder.make_node(pkg=providers[0])\n candidate = answer.get(node)\n\n if candidate and candidate.satisfies(input_spec):\n self._concrete_specs.append(answer[node])\n self._concrete_specs_by_input[input_spec] = answer[node]\n elif candidate and candidate.build_spec.satisfies(input_spec):\n tty.warn(\n \"explicit splice configuration has caused the concretized spec\"\n f\" {candidate} not to satisfy the input spec {input_spec}\"\n )\n self._concrete_specs.append(answer[node])\n self._concrete_specs_by_input[input_spec] = answer[node]\n else:\n self._unsolved_specs.append((input_spec, candidate))\n\n @staticmethod\n def format_unsolved(unsolved_specs):\n \"\"\"Create a message providing info on unsolved user specs and for\n each one show the associated candidate spec from the solver (if\n there is one).\n \"\"\"\n msg = \"Unsatisfied input specs:\"\n for input_spec, candidate in unsolved_specs:\n msg += f\"\\n\\tInput spec: {str(input_spec)}\"\n if candidate:\n msg += f\"\\n\\tCandidate spec: {candidate.long_spec}\"\n else:\n msg += \"\\n\\t(No candidate specs from solver)\"\n return msg\n\n def to_dict(self) -> dict:\n \"\"\"Produces dict representation of Result object\n\n Does not include anything related to unsatisfiability as we\n are only interested in storing satisfiable results\n \"\"\"\n serial_node_arg = lambda node_dict: (\n f\"\"\"{{\"id\": \"{node_dict.id}\", \"pkg\": \"{node_dict.pkg}\"}}\"\"\"\n )\n ret = dict()\n ret[\"criteria\"] = self.criteria\n ret[\"optimal\"] = self.optimal\n ret[\"warnings\"] = self.warnings\n ret[\"nmodels\"] = self.nmodels\n ret[\"abstract_specs\"] = [str(x) for x in self.abstract_specs]\n ret[\"satisfiable\"] = self.satisfiable\n serial_answers = []\n for answer in self.answers:\n serial_answer = answer[:2]\n serial_answer_dict = {}\n for node, spec in answer[2].items():\n serial_answer_dict[serial_node_arg(node)] = spec.to_dict()\n serial_answer = serial_answer + (serial_answer_dict,)\n serial_answers.append(serial_answer)\n ret[\"answers\"] = serial_answers\n ret[\"specs_by_input\"] = {}\n input_specs = {} if not self.specs_by_input else self.specs_by_input\n for input, spec in input_specs.items():\n ret[\"specs_by_input\"][str(input)] = spec.to_dict()\n return ret\n\n @staticmethod\n def from_dict(obj: dict):\n \"\"\"Returns Result object from compatible dictionary\"\"\"\n\n def _dict_to_node_argument(dict):\n id = dict[\"id\"]\n pkg = dict[\"pkg\"]\n return NodeId(id=id, pkg=pkg)\n\n def _str_to_spec(spec_str):\n return spack.spec.Spec(spec_str)\n\n def _dict_to_spec(spec_dict):\n loaded_spec = spack.spec.Spec.from_dict(spec_dict)\n _ensure_external_path_if_external(loaded_spec)\n spack.spec.Spec.ensure_no_deprecated(loaded_spec)\n return loaded_spec\n\n spec_list = obj.get(\"abstract_specs\")\n if not spec_list:\n raise RuntimeError(\"Invalid json for concretization Result object\")\n if spec_list:\n spec_list = [_str_to_spec(x) for x in spec_list]\n result = Result(spec_list)\n\n criteria = obj.get(\"criteria\")\n result.criteria = (\n None if criteria is None else [OptimizationCriteria(*t) for t in criteria]\n )\n result.optimal = obj.get(\"optimal\")\n result.warnings = obj.get(\"warnings\")\n result.nmodels = obj.get(\"nmodels\")\n result.satisfiable = obj.get(\"satisfiable\")\n result._unsolved_specs = []\n answers = []\n for answer in obj.get(\"answers\", []):\n loaded_answer = answer[:2]\n answer_node_dict = {}\n for node, spec in answer[2].items():\n answer_node_dict[_dict_to_node_argument(json.loads(node))] = _dict_to_spec(spec)\n loaded_answer.append(answer_node_dict)\n answers.append(tuple(loaded_answer))\n result.answers = answers\n result._concrete_specs_by_input = {}\n result._concrete_specs = []\n for input, spec in obj.get(\"specs_by_input\", {}).items():\n result._concrete_specs_by_input[_str_to_spec(input)] = _dict_to_spec(spec)\n result._concrete_specs.append(_dict_to_spec(spec))\n return result\n\n def __eq__(self, other):\n eq = (\n self.satisfiable == other.satisfiable,\n self.optimal == other.optimal,\n self.warnings == other.warnings,\n self.nmodels == other.nmodels,\n self.criteria == other.criteria,\n self.answers == other.answers,\n self.abstract_specs == other.abstract_specs,\n self._concrete_specs_by_input == other._concrete_specs_by_input,\n self._concrete_specs == other._concrete_specs,\n self._unsolved_specs == other._unsolved_specs,\n # Not considered for equality\n # self.control\n # self.possible_dependencies\n # self.possible_dependencies\n )\n return all(eq)\n\n\nclass ConcretizationCache:\n \"\"\"Store for Spack concretization results and statistics\n\n Serializes solver result objects and statistics to json and stores\n at a given endpoint in a cache associated by the sha256 of the\n asp problem and the involved control files.\n \"\"\"\n\n def __init__(self, root: Union[str, None] = None):\n root = root or spack.config.get(\"concretizer:concretization_cache:url\", None)\n if root is None:\n root = os.path.join(spack.caches.misc_cache_location(), \"concretization\")\n self.root = pathlib.Path(spack.util.path.canonicalize_path(root))\n self.root.mkdir(parents=True, exist_ok=True)\n self._lockfile = self.root / \".cc_lock\"\n\n def cleanup(self):\n \"\"\"Prunes the concretization cache according to configured entry\n count limits. Cleanup is done in LRU ordering.\"\"\"\n entry_limit = spack.config.get(\"concretizer:concretization_cache:entry_limit\", 1000)\n\n # determine if we even need to clean up\n entries = list(self.cache_entries())\n if len(entries) <= entry_limit:\n return\n\n # collect stat info for mod time about all entries\n removal_queue = []\n for entry in entries:\n try:\n entry_stat_info = entry.stat()\n # mtime will always be time of last use as we update it after\n # each read and obviously after each write\n mod_time = entry_stat_info.st_mtime\n removal_queue.append((mod_time, entry))\n except FileNotFoundError:\n # don't need to cleanup the file, it's not there!\n pass\n\n removal_queue.sort() # sort items for removal, ascending, so oldest first\n\n # Try to remove the oldest half of the cache.\n for _, entry_to_rm in removal_queue[: entry_limit // 2]:\n # cache bucket was removed by another process -- that's fine; move on\n if not entry_to_rm.exists():\n continue\n\n try:\n with self.write_transaction(entry_to_rm, timeout=1e-6):\n self._safe_remove(entry_to_rm)\n except lk.LockTimeoutError:\n # if we can't get a lock, it's either\n # 1) being read, so it's been used recently, i.e. not a good candidate for LRU,\n # 2) it's already being removed by another process, so we don't care, or\n # 3) system is busy, but we don't really need to wait just for cache cleanup.\n pass # so skip it\n\n def cache_entries(self):\n \"\"\"Generator producing cache entries within a bucket\"\"\"\n for cache_entry in self.root.iterdir():\n # Lockfile starts with \".\"\n # old style concretization cache entries are in directories\n if not cache_entry.name.startswith(\".\") and cache_entry.is_file():\n yield cache_entry\n\n def _results_from_cache(self, cache_entry_file: str) -> Union[Result, None]:\n \"\"\"Returns a Results object from the concretizer cache\n\n Reads the cache hit and uses `Result`'s own deserializer\n to produce a new Result object\n \"\"\"\n\n cache_entry = json.loads(cache_entry_file)\n result_json = cache_entry[\"results\"]\n return Result.from_dict(result_json)\n\n def _stats_from_cache(self, cache_entry_file: str) -> Union[Dict, None]:\n \"\"\"Returns concretization statistic from the\n concretization associated with the cache.\n\n Deserializes the the json representation of the\n statistics covering the cached concretization run\n and returns the Python data structures\n \"\"\"\n return json.loads(cache_entry_file)[\"statistics\"]\n\n def _prefix_digest(self, problem: str) -> str:\n \"\"\"Return the first two characters of, and the full, sha256 of the given asp problem\"\"\"\n return spack.util.hash.b32_hash(problem)\n\n def _cache_path_from_problem(self, problem: str) -> pathlib.Path:\n \"\"\"Returns a Path object representing the path to the cache\n entry for the given problem where the problem is the sha256 of the given asp problem\"\"\"\n prefix = self._prefix_digest(problem)\n return self.root / prefix\n\n def _safe_remove(self, cache_dir: pathlib.Path) -> bool:\n \"\"\"Removes cache entries with handling for the case where the entry has been\n removed already or there are multiple cache entries in a directory\"\"\"\n try:\n cache_dir.unlink()\n return True\n except FileNotFoundError:\n # That's fine, removal is idempotent\n pass\n except OSError as e:\n # Catch other timing/access related issues\n tty.debug(\n f\"Exception occurred while attempting to remove Concretization Cache entry, {e}\"\n )\n pass\n return False\n\n def _lock(self, path: pathlib.Path) -> lk.Lock:\n \"\"\"Returns a lock over the byte range corresponding to the hash of the asp problem.\n\n ``path`` is a path to a file in the cache, and its basename is the hash of the problem.\n\n Args:\n path: absolute or relative path to concretization cache entry to be locked\n \"\"\"\n return lk.Lock(\n str(self._lockfile),\n start=spack.util.hash.base32_prefix_bits(\n path.name, spack.util.crypto.bit_length(sys.maxsize)\n ),\n length=1,\n desc=f\"Concretization cache lock for {path}\",\n )\n\n def read_transaction(\n self, path: pathlib.Path, timeout: Optional[float] = None\n ) -> lk.ReadTransaction:\n \"\"\"Read transactions for concretization cache entries.\n\n Args:\n path: absolute or relative path to the concretization cache entry to be locked\n timeout: give up after this many seconds\n \"\"\"\n return lk.ReadTransaction(self._lock(path), timeout=timeout)\n\n def write_transaction(\n self, path: pathlib.Path, timeout: Optional[float] = None\n ) -> lk.WriteTransaction:\n \"\"\"Write transactions for concretization cache entries\n\n Args:\n path: absolute or relative path to the concretization cache entry to be locked\n timeout: give up after this many seconds\n \"\"\"\n return lk.WriteTransaction(self._lock(path), timeout=timeout)\n\n def store(self, problem: str, result: Result, statistics: List) -> None:\n \"\"\"Creates entry in concretization cache for problem if none exists,\n storing the concretization Result object and statistics in the cache\n as serialized json joined as a single file.\n\n Hash membership is computed based on the sha256 of the provided asp\n problem.\n \"\"\"\n cache_path = self._cache_path_from_problem(problem)\n with self.write_transaction(cache_path, timeout=30):\n if cache_path.exists():\n # if cache path file exists, we already have a cache entry, likely created\n # by another process. Exit early.\n return\n\n with gzip.open(cache_path, \"xb\", compresslevel=6) as cache_entry:\n cache_dict = {\"results\": result.to_dict(), \"statistics\": statistics}\n cache_entry.write(json.dumps(cache_dict).encode())\n\n def fetch(self, problem: str) -> Union[Tuple[Result, Dict], Tuple[None, None]]:\n \"\"\"Returns the concretization cache result for a lookup based on the given problem.\n\n Checks the concretization cache for the given problem, and either returns the\n Python objects cached on disk representing the concretization results and statistics\n or returns none if no cache entry was found.\n \"\"\"\n cache_path = self._cache_path_from_problem(problem)\n if not cache_path.exists():\n return None, None # if exists is false, then there's no chance of a hit\n\n cache_content = None\n try:\n with self.read_transaction(cache_path, timeout=2):\n try:\n with gzip.open(cache_path, \"rb\", compresslevel=6) as f:\n f.peek(1) # Try to read at least one byte\n f.seek(0)\n cache_content = f.read().decode(\"utf-8\")\n\n except OSError:\n # Cache may have been created pre compression check if gzip, and if not,\n # read from plaintext otherwise re raise\n with open(cache_path, \"rb\") as f:\n # raise if this is a gzip file we failed to open\n if GZipFileType().matches_magic(f):\n raise\n cache_content = f.read().decode()\n\n except FileNotFoundError:\n pass # cache miss, already cleaned up\n\n except lk.LockTimeoutError:\n pass # if the lock times, out skip the cache\n\n if not cache_content:\n return None, None\n\n # update mod/access time for use w/ LRU cleanup\n os.utime(cache_path)\n return (self._results_from_cache(cache_content), self._stats_from_cache(cache_content)) # type: ignore\n\n\ndef _is_checksummed_git_version(v):\n return isinstance(v, vn.GitVersion) and v.is_commit\n\n\ndef _is_checksummed_version(version_info: Tuple[GitOrStandardVersion, dict]):\n \"\"\"Returns true iff the version is not a moving target\"\"\"\n version, info = version_info\n if isinstance(version, vn.StandardVersion):\n if any(h in info for h in spack.util.crypto.hashes.keys()) or \"checksum\" in info:\n return True\n return \"commit\" in info and len(info[\"commit\"]) == 40\n return _is_checksummed_git_version(version)\n\n\ndef _spec_with_default_name(spec_str, name):\n \"\"\"Return a spec with a default name if none is provided, used for requirement specs\"\"\"\n spec = spack.spec.Spec(spec_str)\n if not spec.name:\n spec.name = name\n return spec\n\n\nclass ErrorHandler:\n def __init__(self, model, input_specs: List[spack.spec.Spec]):\n self.model = model\n self.input_specs = input_specs\n self.full_model = None\n\n def multiple_values_error(self, attribute, pkg):\n return f'Cannot select a single \"{attribute}\" for package \"{pkg}\"'\n\n def no_value_error(self, attribute, pkg):\n return f'Cannot select a single \"{attribute}\" for package \"{pkg}\"'\n\n def _get_cause_tree(\n self,\n cause: Tuple[str, str],\n conditions: Dict[str, str],\n condition_causes: List[Tuple[Tuple[str, str], Tuple[str, str]]],\n seen: Set,\n indent: str = \" \",\n ) -> List[str]:\n \"\"\"\n Implementation of recursion for self.get_cause_tree. Much of this operates on tuples\n (condition_id, set_id) in which the latter idea means that the condition represented by\n the former held in the condition set represented by the latter.\n \"\"\"\n seen.add(cause)\n parents = [c for e, c in condition_causes if e == cause and c not in seen]\n local = f\"required because {conditions[cause[0]]} \"\n\n return [indent + local] + [\n c\n for parent in parents\n for c in self._get_cause_tree(\n parent, conditions, condition_causes, seen, indent=indent + \" \"\n )\n ]\n\n def get_cause_tree(self, cause: Tuple[str, str]) -> List[str]:\n \"\"\"\n Get the cause tree associated with the given cause.\n\n Arguments:\n cause: The root cause of the tree (final condition)\n\n Returns:\n A list of strings describing the causes, formatted to display tree structure.\n \"\"\"\n conditions: Dict[str, str] = dict(extract_args(self.full_model, \"condition_reason\"))\n condition_causes: List[Tuple[Tuple[str, str], Tuple[str, str]]] = list(\n ((Effect, EID), (Cause, CID))\n for Effect, EID, Cause, CID in extract_args(self.full_model, \"condition_cause\")\n )\n return self._get_cause_tree(cause, conditions, condition_causes, set())\n\n def handle_error(self, msg, *args):\n \"\"\"Handle an error state derived by the solver.\"\"\"\n if msg == \"multiple_values_error\":\n return self.multiple_values_error(*args)\n\n if msg == \"no_value_error\":\n return self.no_value_error(*args)\n\n try:\n idx = args.index(\"startcauses\")\n except ValueError:\n msg_args = args\n causes = []\n else:\n msg_args = args[:idx]\n cause_args = args[idx + 1 :]\n cause_args_conditions = cause_args[::2]\n cause_args_ids = cause_args[1::2]\n causes = list(zip(cause_args_conditions, cause_args_ids))\n\n msg = msg.format(*msg_args)\n\n # For variant formatting, we sometimes have to construct specs\n # to format values properly. Find/replace all occurrences of\n # Spec(...) with the string representation of the spec mentioned\n specs_to_construct = re.findall(r\"Spec\\(([^)]*)\\)\", msg)\n for spec_str in specs_to_construct:\n msg = msg.replace(f\"Spec({spec_str})\", str(spack.spec.Spec(spec_str)))\n\n for cause in set(causes):\n for c in self.get_cause_tree(cause):\n msg += f\"\\n{c}\"\n\n return msg\n\n def message(self, errors) -> str:\n input_specs = \", \".join(elide_list([f\"`{s}`\" for s in self.input_specs], 5))\n header = f\"failed to concretize {input_specs} for the following reasons:\"\n messages = (\n f\" {idx + 1:2}. {self.handle_error(msg, *args)}\"\n for idx, (_, msg, args) in enumerate(errors)\n )\n return \"\\n\".join((header, *messages))\n\n def raise_if_errors(self):\n initial_error_args = extract_args(self.model, \"error\")\n if not initial_error_args:\n return\n\n error_causation = clingo().Control()\n\n parent_dir = pathlib.Path(__file__).parent\n errors_lp = parent_dir / \"error_messages.lp\"\n\n def on_model(model):\n self.full_model = model.symbols(shown=True, terms=True)\n\n with error_causation.backend() as backend:\n for atom in self.model:\n atom_id = backend.add_atom(atom)\n backend.add_rule([atom_id], [], choice=False)\n\n error_causation.load(str(errors_lp))\n error_causation.ground([(\"base\", []), (\"error_messages\", [])])\n _ = error_causation.solve(on_model=on_model)\n\n # No choices so there will be only one model\n error_args = extract_args(self.full_model, \"error\")\n errors = sorted(\n [(int(priority), msg, args) for priority, msg, *args in error_args], reverse=True\n )\n try:\n msg = self.message(errors)\n except Exception as e:\n msg = (\n f\"unexpected error during concretization [{str(e)}]. \"\n f\"Please report a bug at https://github.com/spack/spack/issues\"\n )\n raise spack.error.SpackError(msg) from e\n raise UnsatisfiableSpecError(msg)\n\n\nclass PyclingoDriver:\n def __init__(self, conc_cache: Optional[ConcretizationCache] = None) -> None:\n \"\"\"Driver for the Python clingo interface.\n\n Args:\n conc_cache: concretization cache\n \"\"\"\n # This attribute will be reset at each call to solve\n self.control: Any = None # TODO: fix typing of dynamic clingo import\n self._conc_cache = conc_cache\n\n def _control_file_paths(self, control_files: List[str]) -> List[str]:\n \"\"\"Get absolute paths based on relative paths of control files.\n\n Right now the control files just live next to this file in the Spack tree.\n \"\"\"\n parent_dir = os.path.dirname(__file__)\n return [os.path.join(parent_dir, rel_path) for rel_path in control_files]\n\n def _make_cache_key(self, asp_problem: List[str], control_file_paths: List[str]) -> str:\n \"\"\"Make a key for fetching a solve from the concretization cache.\n\n A key comprises the entire input to clingo, i.e., the problem instance plus the\n control files. The problem instance is assumed to already be sorted and stripped of\n comments and empty lines.\n\n The control files are stripped but not sorted, so changes to the control files will cause\n cache misses if they modify any code.\n\n Arguments:\n asp_problem: list of statements in the ASP program\n control_file_paths: list of paths to control files we'll send to clingo\n \"\"\"\n lines = list(asp_problem)\n for path in control_file_paths:\n with open(path, \"r\", encoding=\"utf-8\") as f:\n lines.extend(strip_asp_problem(f.readlines()))\n\n return \"\\n\".join(lines)\n\n def _run_clingo(\n self,\n specs: List[spack.spec.Spec],\n setup: \"SpackSolverSetup\",\n problem_str: str,\n control_file_paths: List[str],\n timer: spack.util.timer.Timer,\n ) -> Result:\n \"\"\"Actually run clingo and generate a result.\n\n This is the core solve logic once the setup is done and once we know we can't\n fetch a result from cache. See ``solve()`` for caching and setup logic.\n \"\"\"\n # We could just take the cache_key and add it to clingo (since it is the\n # full problem representation), but we load control files separately as it\n # makes clingo give us better, file-aware error messages.\n with timer.measure(\"load\"):\n # Add the problem instance\n self.control.add(\"base\", [], problem_str)\n # Load additinoal files\n for path in control_file_paths:\n self.control.load(path)\n\n # Grounding is the first step in the solve -- it turns our facts\n # and first-order logic rules into propositional logic.\n with timer.measure(\"ground\"):\n self.control.ground([(\"base\", [])])\n\n # With a grounded program, we can run the solve.\n models = [] # stable models if things go well\n\n def on_model(model):\n models.append((model.cost, model.symbols(shown=True, terms=True)))\n\n timer.start(\"solve\")\n # A timeout of 0 means no timeout\n time_limit = spack.config.CONFIG.get(\"concretizer:timeout\", 0)\n timeout_end = time.monotonic() + time_limit if time_limit > 0 else float(\"inf\")\n error_on_timeout = spack.config.CONFIG.get(\"concretizer:error_on_timeout\", True)\n with self.control.solve(on_model=on_model, async_=True) as handle:\n # Allow handling of interrupts every second.\n #\n # pyclingo's `SolveHandle` blocks the calling thread for the duration of each\n # `.wait()` call. Python also requires that signal handlers must be handled in\n # the main thread, so any `KeyboardInterrupt` is postponed until after the\n # `.wait()` call exits the control of pyclingo.\n finished = False\n while not finished and time.monotonic() < timeout_end:\n finished = handle.wait(1.0)\n\n if not finished:\n specs_str = \", \".join(spack.llnl.util.lang.elide_list([str(s) for s in specs], 4))\n header = f\"Spack is taking more than {time_limit} seconds to solve for {specs_str}\"\n if error_on_timeout:\n raise UnsatisfiableSpecError(f\"{header}, stopping concretization\")\n warnings.warn(f\"{header}, using the best configuration found so far\")\n handle.cancel()\n\n solve_result = handle.get()\n timer.stop(\"solve\")\n\n # once done, construct the solve result\n result = Result(specs)\n result.satisfiable = solve_result.satisfiable\n\n if result.satisfiable:\n timer.start(\"construct_specs\")\n # get the best model\n builder = SpecBuilder(specs, hash_lookup=setup.reusable_and_possible)\n min_cost, best_model = min(models)\n\n # first check for errors\n error_handler = ErrorHandler(best_model, specs)\n error_handler.raise_if_errors()\n\n # build specs from spec attributes in the model\n spec_attrs = [(name, tuple(rest)) for name, *rest in extract_args(best_model, \"attr\")]\n answers = builder.build_specs(spec_attrs)\n\n # add best spec to the results\n result.answers.append((list(min_cost), 0, answers))\n\n # get optimization criteria\n criteria_args = extract_args(best_model, \"opt_criterion\")\n result.criteria = build_criteria_names(min_cost, criteria_args)\n\n # record the number of models the solver considered\n result.nmodels = len(models)\n\n # record the possible dependencies in the solve\n result.possible_dependencies = setup.pkgs\n timer.stop(\"construct_specs\")\n timer.stop()\n\n result.raise_if_unsat()\n\n if result.satisfiable and result.unsolved_specs and setup.concretize_everything:\n raise OutputDoesNotSatisfyInputError(result.unsolved_specs)\n\n return result\n\n def solve(\n self,\n setup: \"SpackSolverSetup\",\n specs: List[spack.spec.Spec],\n reuse: Optional[List[spack.spec.Spec]] = None,\n packages_with_externals=None,\n output: Optional[OutputConfiguration] = None,\n control: Optional[Any] = None, # TODO: figure out how to annotate clingo.Control\n allow_deprecated: bool = False,\n ) -> Tuple[Result, Optional[spack.util.timer.Timer], Optional[Dict]]:\n \"\"\"Set up the input and solve for dependencies of ``specs``.\n\n Arguments:\n setup: An object to set up the ASP problem.\n specs: List of ``Spec`` objects to solve for.\n reuse: list of concrete specs that can be reused\n output: configuration object to set the output of this solve.\n control: configuration for the solver. If None, default values will be used\n allow_deprecated: if True, allow deprecated versions in the solve\n\n Return:\n A tuple of the solve result, the timer for the different phases of the\n solve, and the internal statistics from clingo.\n \"\"\"\n from spack.bootstrap import ensure_winsdk_external_or_raise\n\n output = output or DEFAULT_OUTPUT_CONFIGURATION\n timer = spack.util.timer.Timer()\n\n # Initialize the control object for the solver\n self.control = control or default_clingo_control()\n\n # ensure core deps are present on Windows\n # needs to modify active config scope, so cannot be run within\n # bootstrap config scope\n if sys.platform == \"win32\":\n ensure_winsdk_external_or_raise()\n\n # assemble a list of the control files needed for this problem. Some are conditionally\n # included depending on what features we're using in the solve.\n control_files = [\"concretize.lp\", \"heuristic.lp\", \"display.lp\", \"direct_dependency.lp\"]\n if not setup.concretize_everything:\n control_files.append(\"when_possible.lp\")\n if using_libc_compatibility():\n control_files.append(\"libc_compatibility.lp\")\n else:\n control_files.append(\"os_compatibility.lp\")\n if setup.enable_splicing:\n control_files.append(\"splices.lp\")\n\n timer.start(\"setup\")\n problem_builder = setup.setup(\n specs,\n reuse=reuse,\n packages_with_externals=packages_with_externals,\n allow_deprecated=allow_deprecated,\n )\n timer.stop(\"setup\")\n\n timer.start(\"ordering\")\n # print the output with comments, etc. if the user asked\n problem = problem_builder.asp_problem\n if output.out is not None:\n output.out.write(\"\\n\".join(problem))\n\n if output.setup_only:\n return Result(specs), None, None\n\n # strip the problem of comments and empty lines\n problem = strip_asp_problem(problem)\n randomize = \"SPACK_SOLVER_RANDOMIZATION\" in os.environ\n if randomize:\n # create a shuffled copy -- useful for understanding performance variation\n problem = random.sample(problem, len(problem))\n else:\n problem.sort() # sort for deterministic output\n\n timer.stop(\"ordering\")\n\n timer.start(\"cache-check\")\n # load control files to add to the input representation\n control_file_paths = self._control_file_paths(control_files)\n cache_key = self._make_cache_key(problem, control_file_paths)\n\n result, concretization_stats = None, None\n conc_cache_enabled = spack.config.get(\"concretizer:concretization_cache:enable\", False)\n if conc_cache_enabled and self._conc_cache:\n result, concretization_stats = self._conc_cache.fetch(cache_key)\n timer.stop(\"cache-check\")\n\n tty.debug(\"Starting concretizer\")\n\n # run the solver and store the result, if it wasn't cached already\n if not result:\n problem_repr = \"\\n\".join(problem)\n result = self._run_clingo(specs, setup, problem_repr, control_file_paths, timer)\n if conc_cache_enabled and self._conc_cache:\n self._conc_cache.store(cache_key, result, self.control.statistics)\n\n if output.timers:\n timer.write_tty()\n print()\n\n concretization_stats = concretization_stats or self.control.statistics\n if output.stats:\n print(\"Statistics:\")\n pprint.pprint(concretization_stats)\n\n return result, timer, concretization_stats\n\n\nclass ConcreteSpecsByHash(collections.abc.Mapping):\n \"\"\"Mapping containing concrete specs keyed by DAG hash.\n\n The mapping is ensured to be consistent, i.e. if a spec in the mapping has a dependency with\n hash X, it is ensured to be the same object in memory as the spec keyed by X.\n \"\"\"\n\n def __init__(self) -> None:\n self.data: Dict[str, spack.spec.Spec] = {}\n self.explicit: Set[str] = set()\n\n def __getitem__(self, dag_hash: str) -> spack.spec.Spec:\n return self.data[dag_hash]\n\n def explicit_items(self) -> Iterator[Tuple[str, spack.spec.Spec]]:\n \"\"\"Iterate on items that have been added explicitly, and not just as a dependency\n of other nodes.\n \"\"\"\n for h, s in self.items():\n # We need to make an exception for gcc-runtime, until we can splice it.\n if h in self.explicit or s.name == \"gcc-runtime\":\n yield h, s\n\n def add(self, spec: spack.spec.Spec) -> bool:\n \"\"\"Adds a new concrete spec to the mapping. Returns True if the spec was just added,\n False if the spec was already in the mapping.\n\n Calling this function marks the spec as added explicitly.\n\n Args:\n spec: spec to be added\n\n Raises:\n ValueError: if the spec is not concrete\n \"\"\"\n if not spec.concrete:\n msg = (\n f\"trying to store the non-concrete spec '{spec}' in a container \"\n f\"that only accepts concrete\"\n )\n raise ValueError(msg)\n\n dag_hash = spec.dag_hash()\n self.explicit.add(dag_hash)\n if dag_hash in self.data:\n return False\n\n # Here we need to iterate on the input and rewire the copy.\n self.data[spec.dag_hash()] = spec.copy(deps=False)\n nodes_to_reconstruct = [spec]\n\n while nodes_to_reconstruct:\n input_parent = nodes_to_reconstruct.pop()\n container_parent = self.data[input_parent.dag_hash()]\n\n for edge in input_parent.edges_to_dependencies():\n input_child = edge.spec\n container_child = self.data.get(input_child.dag_hash())\n # Copy children that don't exist yet\n if container_child is None:\n container_child = input_child.copy(deps=False)\n self.data[input_child.dag_hash()] = container_child\n nodes_to_reconstruct.append(input_child)\n\n # Rewire edges\n container_parent.add_dependency_edge(\n dependency_spec=container_child, depflag=edge.depflag, virtuals=edge.virtuals\n )\n return True\n\n def __len__(self) -> int:\n return len(self.data)\n\n def __iter__(self):\n return iter(self.data)\n\n\n# types for condition caching in solver setup\nConditionSpecKey = Tuple[str, Optional[TransformFunction]]\nConditionIdFunctionPair = Tuple[int, List[AspFunction]]\nConditionSpecCache = Dict[str, Dict[ConditionSpecKey, ConditionIdFunctionPair]]\n\n\nclass ConstraintOrigin(enum.Enum):\n \"\"\"Generates identifiers that can be passed into the solver attached\n to constraints, and then later retrieved to determine the origin of\n those constraints when ``SpecBuilder`` creates Specs from the solve\n result.\n \"\"\"\n\n CONDITIONAL_SPEC = 0\n DEPENDS_ON = 1\n REQUIRE = 2\n\n @staticmethod\n def _SUFFIXES() -> Dict[\"ConstraintOrigin\", str]:\n return {\n ConstraintOrigin.CONDITIONAL_SPEC: \"_cond\",\n ConstraintOrigin.DEPENDS_ON: \"_dep\",\n ConstraintOrigin.REQUIRE: \"_req\",\n }\n\n @staticmethod\n def append_type_suffix(pkg_id: str, kind: \"ConstraintOrigin\") -> str:\n \"\"\"Given a package identifier and a constraint kind, generate a string ID.\"\"\"\n suffix = ConstraintOrigin._SUFFIXES()[kind]\n return f\"{pkg_id}{suffix}\"\n\n @staticmethod\n def strip_type_suffix(source: str) -> Tuple[int, Optional[str]]:\n \"\"\"Take a combined package/type ID generated by\n ``append_type_suffix``, and extract the package ID and\n an associated weight.\n \"\"\"\n if not source:\n return -1, None\n for kind, suffix in ConstraintOrigin._SUFFIXES().items():\n if source.endswith(suffix):\n return kind.value, source[: -len(suffix)]\n return -1, source\n\n\nclass ConditionIdContext(SourceContext):\n \"\"\"Derived from a ``ConditionContext``: for clause-sets generated by\n imposed/required specs, stores an associated transform.\n\n This is primarily used for tracking whether we are generating clauses\n in the context of a required spec, or for an imposed spec.\n\n Is not a subclass of ``ConditionContext`` because it exists in a\n lower-level context with less information.\n \"\"\"\n\n def __init__(self):\n super().__init__()\n self.transform: Optional[TransformFunction] = None\n\n\nclass ConditionContext(SourceContext):\n \"\"\"Tracks context in which a condition (i.e. ``SpackSolverSetup.condition``)\n is generated (e.g. for a ``depends_on``).\n\n This may modify the required/imposed specs generated as relevant\n for the context.\n \"\"\"\n\n def __init__(self):\n super().__init__()\n # transformation applied to facts from the required spec. Defaults\n # to leave facts as they are.\n self.transform_required: Optional[TransformFunction] = None\n # transformation applied to facts from the imposed spec. Defaults\n # to removing \"node\" and \"virtual_node\" facts.\n self.transform_imposed: Optional[TransformFunction] = None\n # Whether to wrap direct dependency facts as node requirements,\n # imposed by the parent. If None, the default is used, which is:\n # - wrap head of rules\n # - do not wrap body of rules\n self.wrap_node_requirement: Optional[bool] = None\n\n def requirement_context(self) -> ConditionIdContext:\n ctxt = ConditionIdContext()\n ctxt.source = self.source\n ctxt.transform = self.transform_required\n ctxt.wrap_node_requirement = self.wrap_node_requirement\n return ctxt\n\n def impose_context(self) -> ConditionIdContext:\n ctxt = ConditionIdContext()\n ctxt.source = self.source\n ctxt.transform = self.transform_imposed\n ctxt.wrap_node_requirement = self.wrap_node_requirement\n return ctxt\n\n\nclass SpackSolverSetup:\n \"\"\"Class to set up and run a Spack concretization solve.\"\"\"\n\n gen: \"ProblemInstanceBuilder\"\n possible_versions: Dict[str, Dict[GitOrStandardVersion, List[Provenance]]]\n\n def __init__(self, tests: spack.concretize.TestsType = False):\n self.possible_graph = create_graph_analyzer()\n\n # these are all initialized in setup()\n self.requirement_parser = RequirementParser(spack.config.CONFIG)\n self.possible_virtuals: Set[str] = set()\n\n # pkg_name -> version -> list of possible origins (package.py, installed, etc.)\n self.possible_versions = collections.defaultdict(lambda: collections.defaultdict(list))\n self.versions_from_yaml: Dict[str, List[GitOrStandardVersion]] = {}\n self.git_commit_versions: Dict[str, Dict[GitOrStandardVersion, str]] = (\n collections.defaultdict(dict)\n )\n self.deprecated_versions: Dict[str, Set[GitOrStandardVersion]] = collections.defaultdict(\n set\n )\n\n self.possible_compilers: List[spack.spec.Spec] = []\n self.rejected_compilers: Set[spack.spec.Spec] = set()\n self.possible_oses: Set = set()\n self.variant_values_from_specs: Set = set()\n self.version_constraints: Dict[str, Set] = collections.defaultdict(set)\n self.target_constraints: Set = set()\n self.default_targets: List = []\n self.variant_ids_by_def_id: Dict[int, int] = {}\n\n self.reusable_and_possible: ConcreteSpecsByHash = ConcreteSpecsByHash()\n\n self._id_counter: Iterator[int] = itertools.count()\n self._trigger_cache: ConditionSpecCache = collections.defaultdict(dict)\n self._effect_cache: ConditionSpecCache = collections.defaultdict(dict)\n\n # Caches to optimize the setup phase of the solver\n self.target_specs_cache = None\n\n # whether to add installed/binary hashes to the solve\n self.tests = tests\n\n # If False allows for input specs that are not solved\n self.concretize_everything = True\n\n # Set during the call to setup\n self.pkgs: Set[str] = set()\n self.explicitly_required_namespaces: Dict[str, str] = {}\n\n # list of unique libc specs targeted by compilers (or an educated guess if no compiler)\n self.libcs: List[spack.spec.Spec] = []\n\n # If true, we have to load the code for synthesizing splices\n self.enable_splicing: bool = spack.config.CONFIG.get(\"concretizer:splice:automatic\")\n\n def pkg_version_rules(self, pkg: Type[spack.package_base.PackageBase]) -> None:\n \"\"\"Declares known versions, their origins, and their weights.\"\"\"\n version_provenance = self.possible_versions[pkg.name]\n ordered_versions = spack.package_base.sort_by_pkg_preference(\n self.possible_versions[pkg.name], pkg=pkg\n )\n # Account for preferences in packages.yaml, if any\n if pkg.name in self.versions_from_yaml:\n ordered_versions = list(\n spack.llnl.util.lang.dedupe(self.versions_from_yaml[pkg.name] + ordered_versions)\n )\n\n # Set the deprecation penalty, according to the package. This should be enough to move the\n # first version last if deprecated.\n if ordered_versions:\n self.gen.fact(\n fn.pkg_fact(pkg.name, fn.version_deprecation_penalty(len(ordered_versions)))\n )\n\n for weight, declared_version in enumerate(ordered_versions):\n self.gen.fact(fn.pkg_fact(pkg.name, fn.version_declared(declared_version, weight)))\n for origin in version_provenance[declared_version]:\n self.gen.fact(\n fn.pkg_fact(pkg.name, fn.version_origin(declared_version, str(origin)))\n )\n\n for v in self.possible_versions[pkg.name]:\n if pkg.needs_commit(v):\n commit = pkg.version_or_package_attr(\"commit\", v, \"\")\n self.git_commit_versions[pkg.name][v] = commit\n\n # Declare deprecated versions for this package, if any\n deprecated = self.deprecated_versions[pkg.name]\n for v in sorted(deprecated):\n self.gen.fact(fn.pkg_fact(pkg.name, fn.deprecated_version(v)))\n\n def spec_versions(\n self, spec: spack.spec.Spec, *, name: Optional[str] = None\n ) -> List[AspFunction]:\n \"\"\"Return list of clauses expressing spec's version constraints.\"\"\"\n name = spec.name or name\n assert name, \"Internal Error: spec with no name occurred. Please file an issue.\"\n\n if spec.concrete:\n return [fn.attr(\"version\", name, spec.version)]\n\n if spec.versions == vn.any_version:\n return []\n\n # record all version constraints for later\n self.version_constraints[name].add(spec.versions)\n return [fn.attr(\"node_version_satisfies\", name, spec.versions)]\n\n def target_ranges(\n self, spec: spack.spec.Spec, single_target_fn, *, name: Optional[str] = None\n ) -> List[AspFunction]:\n name = spec.name or name\n assert name, \"Internal Error: spec with no name occurred. Please file an issue.\"\n target = spec.architecture.target\n\n # Check if the target is a concrete target\n if str(target) in spack.vendor.archspec.cpu.TARGETS:\n return [single_target_fn(name, target)]\n\n self.target_constraints.add(target)\n return [fn.attr(\"node_target_satisfies\", name, target)]\n\n def conflict_rules(self, pkg):\n for when_spec, conflict_specs in pkg.conflicts.items():\n when_spec_msg = f\"conflict constraint {when_spec}\"\n when_spec_id = self.condition(when_spec, required_name=pkg.name, msg=when_spec_msg)\n when_spec_str = str(when_spec)\n\n for conflict_spec, conflict_msg in conflict_specs:\n conflict_spec_str = str(conflict_spec)\n if conflict_msg is None:\n conflict_msg = f\"{pkg.name}: \"\n if not when_spec_str:\n conflict_msg += f\"conflicts with '{conflict_spec_str}'\"\n else:\n conflict_msg += f\"'{conflict_spec_str}' conflicts with '{when_spec_str}'\"\n\n if not conflict_spec_str:\n conflict_spec_msg = f\"conflict is triggered when {pkg.name}\"\n else:\n conflict_spec_msg = f\"conflict is triggered when {conflict_spec_str}\"\n\n conflict_spec_id = self.condition(\n conflict_spec,\n required_name=conflict_spec.name or pkg.name,\n msg=conflict_spec_msg,\n )\n self.gen.fact(\n fn.pkg_fact(\n pkg.name, fn.conflict(conflict_spec_id, when_spec_id, conflict_msg)\n )\n )\n self.gen.newline()\n\n def config_compatible_os(self):\n \"\"\"Facts about compatible os's specified in configs\"\"\"\n self.gen.h2(\"Compatible OS from concretizer config file\")\n os_data = spack.config.get(\"concretizer:os_compatible\", {})\n for recent, reusable in os_data.items():\n for old in reusable:\n self.gen.fact(fn.os_compatible(recent, old))\n self.gen.newline()\n\n def package_requirement_rules(self, pkg):\n self.emit_facts_from_requirement_rules(self.requirement_parser.rules(pkg))\n\n def pkg_rules(self, pkg, tests):\n pkg = self.pkg_class(pkg)\n\n # Namespace of the package\n self.gen.fact(fn.pkg_fact(pkg.name, fn.namespace(pkg.namespace)))\n\n # versions\n self.pkg_version_rules(pkg)\n self.gen.newline()\n\n # variants\n self.variant_rules(pkg)\n\n # conflicts\n self.conflict_rules(pkg)\n\n # virtuals\n self.package_provider_rules(pkg)\n\n # dependencies\n self.package_dependencies_rules(pkg)\n\n # splices\n if self.enable_splicing:\n self.package_splice_rules(pkg)\n\n self.package_requirement_rules(pkg)\n\n def trigger_rules(self):\n \"\"\"Flushes all the trigger rules collected so far, and clears the cache.\"\"\"\n if not self._trigger_cache:\n return\n\n self.gen.h2(\"Trigger conditions\")\n for name in self._trigger_cache:\n cache = self._trigger_cache[name]\n for (spec_str, _), (trigger_id, requirements) in cache.items():\n self.gen.fact(fn.pkg_fact(name, fn.trigger_id(trigger_id)))\n self.gen.fact(fn.pkg_fact(name, fn.trigger_msg(spec_str)))\n for predicate in requirements:\n self.gen.fact(fn.condition_requirement(trigger_id, *predicate.args))\n self.gen.newline()\n self._trigger_cache.clear()\n\n def effect_rules(self):\n \"\"\"Flushes all the effect rules collected so far, and clears the cache.\"\"\"\n if not self._effect_cache:\n return\n\n self.gen.h2(\"Imposed requirements\")\n for name in sorted(self._effect_cache):\n cache = self._effect_cache[name]\n for (spec_str, _), (effect_id, requirements) in cache.items():\n self.gen.fact(fn.pkg_fact(name, fn.effect_id(effect_id)))\n self.gen.fact(fn.pkg_fact(name, fn.effect_msg(spec_str)))\n for predicate in requirements:\n self.gen.fact(fn.imposed_constraint(effect_id, *predicate.args))\n self.gen.newline()\n self._effect_cache.clear()\n\n def define_variant(\n self,\n pkg: Type[spack.package_base.PackageBase],\n name: str,\n when: spack.spec.Spec,\n variant_def: vt.Variant,\n ):\n pkg_fact = lambda f: self.gen.fact(fn.pkg_fact(pkg.name, f))\n\n # Every variant id has a unique definition (conditional or unconditional), and\n # higher variant id definitions take precedence when variants intersect.\n vid = next(self._id_counter)\n\n # used to find a variant id from its variant definition (for variant values on specs)\n self.variant_ids_by_def_id[id(variant_def)] = vid\n\n if when == EMPTY_SPEC:\n # unconditional variant\n pkg_fact(fn.variant_definition(name, vid))\n else:\n # conditional variant\n msg = f\"Package {pkg.name} has variant '{name}' when {when}\"\n cond_id = self.condition(when, required_name=pkg.name, msg=msg)\n pkg_fact(fn.variant_condition(name, vid, cond_id))\n\n # record type so we can construct the variant when we read it back in\n self.gen.fact(fn.variant_type(vid, variant_def.variant_type.string))\n\n if variant_def.sticky:\n pkg_fact(fn.variant_sticky(vid))\n\n # Get the default values for this variant definition as a tuple\n default_values: Tuple[Union[bool, str], ...] = (variant_def.default,)\n if variant_def.multi:\n default_values = variant_def.make_default().values\n\n for val in default_values:\n pkg_fact(fn.variant_default_value_from_package_py(vid, val))\n\n # Deal with variants that use validator functions\n if variant_def.values_defined_by_validator():\n for penalty, value in enumerate(default_values, 1):\n pkg_fact(fn.variant_possible_value(vid, value))\n pkg_fact(fn.variant_penalty(vid, value, penalty))\n self.gen.newline()\n return\n\n values = variant_def.values or default_values\n\n # If we deal with disjoint sets of values, define the sets\n if isinstance(values, vt.DisjointSetsOfValues):\n for sid, s in enumerate(values.sets):\n for value in s:\n pkg_fact(fn.variant_value_from_disjoint_sets(vid, value, sid))\n\n # Define penalties. Put default values first, otherwise keep the order\n penalty = 1\n for v in default_values:\n pkg_fact(fn.variant_penalty(vid, v, penalty))\n penalty += 1\n\n for v in values:\n if v not in default_values:\n pkg_fact(fn.variant_penalty(vid, v, penalty))\n penalty += 1\n\n # Deal with conditional values\n for value in values:\n if not isinstance(value, vt.ConditionalValue):\n continue\n\n # make a spec indicating whether the variant has this conditional value\n variant_has_value = spack.spec.Spec()\n variant_has_value.variants[name] = vt.VariantValue(\n vt.VariantType.MULTI, name, (value.value,)\n )\n\n if value.when:\n # the conditional value is always \"possible\", but it imposes its when condition as\n # a constraint if the conditional value is taken. This may seem backwards, but it\n # ensures that the conditional can only occur when its condition holds.\n self.condition(\n required_spec=variant_has_value,\n imposed_spec=value.when,\n required_name=pkg.name,\n imposed_name=pkg.name,\n msg=f\"{pkg.name} variant {name} has value '{value.value}' when {value.when}\",\n )\n else:\n vstring = f\"{name}='{value.value}'\"\n\n # We know the value is never allowed statically (when was None), but we can't just\n # ignore it b/c it could come in as a possible value and we need a good error msg.\n # So, it's a conflict -- if the value is somehow used, it'll trigger an error.\n trigger_id = self.condition(\n variant_has_value,\n required_name=pkg.name,\n msg=f\"invalid variant value: {vstring}\",\n )\n constraint_id = self.condition(\n EMPTY_SPEC, required_name=pkg.name, msg=\"empty (total) conflict constraint\"\n )\n msg = f\"variant value {vstring} is conditionally disabled\"\n pkg_fact(fn.conflict(trigger_id, constraint_id, msg))\n\n self.gen.newline()\n\n def define_auto_variant(self, name: str, multi: bool):\n self.gen.h3(f\"Special variant: {name}\")\n vid = next(self._id_counter)\n self.gen.fact(fn.auto_variant(name, vid))\n self.gen.fact(\n fn.variant_type(\n vid, vt.VariantType.MULTI.value if multi else vt.VariantType.SINGLE.value\n )\n )\n\n def variant_rules(self, pkg: Type[spack.package_base.PackageBase]):\n for name in pkg.variant_names():\n self.gen.h3(f\"Variant {name} in package {pkg.name}\")\n for when, variant_def in pkg.variant_definitions(name):\n self.define_variant(pkg, name, when, variant_def)\n\n def _get_condition_id(\n self,\n name: str,\n cond: spack.spec.Spec,\n cache: ConditionSpecCache,\n body: bool,\n context: ConditionIdContext,\n ) -> int:\n \"\"\"Get the id for one half of a condition (either a trigger or an imposed constraint).\n\n Construct a key from the condition spec and any associated transformation, and\n cache the ASP functions that they imply. The saved functions will be output\n later in ``trigger_rules()`` and ``effect_rules()``.\n\n Returns:\n The id of the cached trigger or effect.\n\n \"\"\"\n pkg_cache = cache[name]\n cond_str = str(cond) if cond.name else f\"{name} {cond}\"\n named_cond_key = (cond_str, context.transform)\n\n result = pkg_cache.get(named_cond_key)\n if result:\n return result[0]\n\n cond_id = next(self._id_counter)\n requirements = self.spec_clauses(cond, name=name, body=body, context=context)\n if context.transform:\n requirements = context.transform(name, cond, requirements)\n pkg_cache[named_cond_key] = (cond_id, requirements)\n\n return cond_id\n\n def _condition_clauses(\n self,\n required_spec: spack.spec.Spec,\n imposed_spec: Optional[spack.spec.Spec] = None,\n *,\n required_name: Optional[str] = None,\n imposed_name: Optional[str] = None,\n msg: Optional[str] = None,\n context: Optional[ConditionContext] = None,\n ) -> Tuple[List[AspFunction], int]:\n clauses = []\n required_name = required_spec.name or required_name\n if not required_name:\n raise ValueError(f\"Must provide a name for anonymous condition: '{required_spec}'\")\n\n if not context:\n context = ConditionContext()\n context.transform_imposed = remove_facts(\"node\", \"virtual_node\")\n\n # Check if we can emit the requirements before updating the condition ID counter.\n # In this way, if a condition can't be emitted but the exception is handled in the\n # caller, we won't emit partial facts.\n condition_id = next(self._id_counter)\n requirement_context = context.requirement_context()\n trigger_id = self._get_condition_id(\n required_name,\n required_spec,\n cache=self._trigger_cache,\n body=True,\n context=requirement_context,\n )\n clauses.append(fn.pkg_fact(required_name, fn.condition(condition_id)))\n clauses.append(fn.condition_reason(condition_id, msg))\n clauses.append(fn.pkg_fact(required_name, fn.condition_trigger(condition_id, trigger_id)))\n if not imposed_spec:\n return clauses, condition_id\n\n imposed_name = imposed_spec.name or imposed_name\n if not imposed_name:\n raise ValueError(f\"Must provide a name for imposed constraint: '{imposed_spec}'\")\n\n impose_context = context.impose_context()\n effect_id = self._get_condition_id(\n imposed_name,\n imposed_spec,\n cache=self._effect_cache,\n body=False,\n context=impose_context,\n )\n clauses.append(fn.pkg_fact(required_name, fn.condition_effect(condition_id, effect_id)))\n\n return clauses, condition_id\n\n def condition(\n self,\n required_spec: spack.spec.Spec,\n imposed_spec: Optional[spack.spec.Spec] = None,\n *,\n required_name: Optional[str] = None,\n imposed_name: Optional[str] = None,\n msg: Optional[str] = None,\n context: Optional[ConditionContext] = None,\n ) -> int:\n \"\"\"Generate facts for a dependency or virtual provider condition.\n\n Arguments:\n required_spec: the constraints that triggers this condition\n imposed_spec: the constraints that are imposed when this condition is triggered\n required_name: name for ``required_spec``\n (required if required_spec is anonymous, ignored if not)\n imposed_name: name for ``imposed_spec``\n (required if imposed_spec is anonymous, ignored if not)\n msg: description of the condition\n context: if provided, indicates how to modify the clause-sets for the required/imposed\n specs based on the type of constraint they are generated for (e.g. ``depends_on``)\n Returns:\n int: id of the condition created by this function\n \"\"\"\n clauses, condition_id = self._condition_clauses(\n required_spec=required_spec,\n imposed_spec=imposed_spec,\n required_name=required_name,\n imposed_name=imposed_name,\n msg=msg,\n context=context,\n )\n for clause in clauses:\n self.gen.fact(clause)\n\n return condition_id\n\n def package_provider_rules(self, pkg: Type[spack.package_base.PackageBase]) -> None:\n for vpkg_name in pkg.provided_virtual_names():\n if vpkg_name not in self.possible_virtuals:\n continue\n self.gen.fact(fn.pkg_fact(pkg.name, fn.possible_provider(vpkg_name)))\n\n for when, provided in pkg.provided.items():\n for vpkg in sorted(provided): # type: ignore[type-var]\n if vpkg.name not in self.possible_virtuals:\n continue\n\n msg = f\"{pkg.name} provides {vpkg}{'' if when == EMPTY_SPEC else f' when {when}'}\"\n condition_id = self.condition(when, vpkg, required_name=pkg.name, msg=msg)\n self.gen.fact(\n fn.pkg_fact(pkg.name, fn.provider_condition(condition_id, vpkg.name))\n )\n self.gen.newline()\n\n for when, sets_of_virtuals in pkg.provided_together.items():\n condition_id = self.condition(\n when, required_name=pkg.name, msg=\"Virtuals are provided together\"\n )\n for set_id, virtuals_together in enumerate(sorted(sets_of_virtuals)):\n for name in sorted(virtuals_together):\n self.gen.fact(\n fn.pkg_fact(pkg.name, fn.provided_together(condition_id, set_id, name))\n )\n self.gen.newline()\n\n def package_dependencies_rules(self, pkg):\n \"\"\"Translate ``depends_on`` directives into ASP logic.\"\"\"\n for cond, deps_by_name in pkg.dependencies.items():\n cond_str = str(cond)\n cond_str_suffix = f\" when {cond_str}\" if cond_str else \"\"\n for _, dep in deps_by_name.items():\n depflag = dep.depflag\n # Skip test dependencies if they're not requested\n if not self.tests:\n depflag &= ~dt.TEST\n\n # ... or if they are requested only for certain packages\n elif not isinstance(self.tests, bool) and pkg.name not in self.tests:\n depflag &= ~dt.TEST\n\n # if there are no dependency types to be considered\n # anymore, don't generate the dependency\n if not depflag:\n continue\n\n msg = f\"{pkg.name} depends on {dep.spec}{cond_str_suffix}\"\n context = ConditionContext()\n context.source = ConstraintOrigin.append_type_suffix(\n pkg.name, ConstraintOrigin.DEPENDS_ON\n )\n context.transform_imposed = dependency_holds(dependency_flags=depflag, pkg_cls=pkg)\n self.condition(cond, dep.spec, required_name=pkg.name, msg=msg, context=context)\n self.gen.newline()\n\n def _gen_match_variant_splice_constraints(\n self,\n pkg,\n cond_spec: spack.spec.Spec,\n splice_spec: spack.spec.Spec,\n hash_asp_var: \"AspVar\",\n splice_node,\n match_variants: List[str],\n ):\n # If there are no variants to match, no constraints are needed\n variant_constraints = []\n for i, variant_name in enumerate(match_variants):\n vari_defs = pkg.variant_definitions(variant_name)\n # the spliceable config of the package always includes the variant\n if vari_defs != [] and any(cond_spec.satisfies(s) for (s, _) in vari_defs):\n variant = vari_defs[0][1]\n if variant.multi:\n continue # cannot automatically match multi-valued variants\n value_var = AspVar(f\"VariValue{i}\")\n attr_constraint = fn.attr(\"variant_value\", splice_node, variant_name, value_var)\n hash_attr_constraint = fn.hash_attr(\n hash_asp_var, \"variant_value\", splice_spec.name, variant_name, value_var\n )\n variant_constraints.append(attr_constraint)\n variant_constraints.append(hash_attr_constraint)\n return variant_constraints\n\n def package_splice_rules(self, pkg):\n self.gen.h2(\"Splice rules\")\n for i, (cond, (spec_to_splice, match_variants)) in enumerate(\n sorted(pkg.splice_specs.items())\n ):\n self.version_constraints[pkg.name].add(cond.versions)\n self.version_constraints[spec_to_splice.name].add(spec_to_splice.versions)\n hash_var = AspVar(\"Hash\")\n splice_node = fn.node(AspVar(\"NID\"), pkg.name)\n when_spec_attrs = [\n fn.attr(c.args[0], splice_node, *(c.args[2:]))\n for c in self.spec_clauses(cond, name=pkg.name, body=True, required_from=None)\n if c.args[0] != \"node\"\n ]\n splice_spec_hash_attrs = [\n fn.hash_attr(hash_var, *(c.args))\n for c in self.spec_clauses(spec_to_splice, body=True, required_from=None)\n if c.args[0] != \"node\"\n ]\n if match_variants is None:\n variant_constraints = []\n elif match_variants == \"*\":\n filt_match_variants = set()\n for map in pkg.variants.values():\n for k in map:\n filt_match_variants.add(k)\n filt_match_variants = sorted(filt_match_variants)\n variant_constraints = self._gen_match_variant_splice_constraints(\n pkg, cond, spec_to_splice, hash_var, splice_node, filt_match_variants\n )\n else:\n if any(v in cond.variants or v in spec_to_splice.variants for v in match_variants):\n raise spack.error.PackageError(\n \"Overlap between match_variants and explicitly set variants\"\n )\n variant_constraints = self._gen_match_variant_splice_constraints(\n pkg, cond, spec_to_splice, hash_var, splice_node, match_variants\n )\n\n rule_head = fn.abi_splice_conditions_hold(\n i, splice_node, spec_to_splice.name, hash_var\n )\n rule_body_components = (\n [\n # splice_set_fact,\n fn.attr(\"node\", splice_node),\n fn.installed_hash(spec_to_splice.name, hash_var),\n ]\n + when_spec_attrs\n + splice_spec_hash_attrs\n + variant_constraints\n )\n rule_body = \",\\n \".join(str(r) for r in rule_body_components)\n rule = f\"{rule_head} :-\\n {rule_body}.\"\n self.gen.append(rule)\n\n self.gen.newline()\n\n def virtual_requirements_and_weights(self):\n virtual_preferences = spack.config.CONFIG.get(\"packages:all:providers\", {})\n\n self.gen.h1(\"Virtual requirements and weights\")\n for virtual_str in sorted(self.possible_virtuals):\n self.gen.newline()\n self.gen.h2(f\"Virtual: {virtual_str}\")\n self.gen.fact(fn.virtual(virtual_str))\n\n rules = self.requirement_parser.rules_from_virtual(virtual_str)\n if not rules and virtual_str not in virtual_preferences:\n continue\n\n required, preferred, removed = [], [], set()\n for rule in rules:\n # We don't deal with conditional requirements\n if rule.condition != EMPTY_SPEC:\n continue\n\n if rule.origin == RequirementOrigin.PREFER_YAML:\n preferred.extend(x.name for x in rule.requirements if x.name)\n elif rule.origin == RequirementOrigin.REQUIRE_YAML:\n required.extend(x.name for x in rule.requirements if x.name)\n elif rule.origin == RequirementOrigin.CONFLICT_YAML:\n conflict_spec = rule.requirements[0]\n # For conflicts, we take action only if just a name is used\n if spack.spec.Spec(conflict_spec.name).satisfies(conflict_spec):\n removed.add(conflict_spec.name)\n\n current_preferences = required + preferred + virtual_preferences.get(virtual_str, [])\n current_preferences = [x for x in current_preferences if x not in removed]\n for i, provider in enumerate(spack.llnl.util.lang.dedupe(current_preferences)):\n provider_name = spack.spec.Spec(provider).name\n self.gen.fact(fn.provider_weight_from_config(virtual_str, provider_name, i))\n self.gen.newline()\n\n if rules:\n self.emit_facts_from_requirement_rules(rules)\n self.trigger_rules()\n self.effect_rules()\n\n def emit_facts_from_requirement_rules(self, rules: List[RequirementRule]):\n \"\"\"Generate facts to enforce requirements.\n\n Args:\n rules: rules for which we want facts to be emitted\n \"\"\"\n for requirement_grp_id, rule in enumerate(rules):\n virtual = rule.kind == RequirementKind.VIRTUAL\n\n pkg_name, policy, requirement_grp = rule.pkg_name, rule.policy, rule.requirements\n requirement_weight = 0\n # Propagated preferences have a higher penalty that normal preferences\n weight_multiplier = 2 if rule.origin == RequirementOrigin.INPUT_SPECS else 1\n # Write explicitly if a requirement is conditional or not\n if rule.condition != EMPTY_SPEC:\n msg = f\"activate requirement {requirement_grp_id} if {rule.condition} holds\"\n context = ConditionContext()\n context.transform_required = dag_closure_by_deptype\n try:\n main_condition_id = self.condition(\n rule.condition, required_name=pkg_name, msg=msg, context=context\n )\n except Exception as e:\n if rule.kind != RequirementKind.DEFAULT:\n raise RuntimeError(\n \"cannot emit requirements for the solver: \" + str(e)\n ) from e\n continue\n\n self.gen.fact(\n fn.requirement_conditional(pkg_name, requirement_grp_id, main_condition_id)\n )\n\n self.gen.fact(fn.requirement_group(pkg_name, requirement_grp_id))\n self.gen.fact(fn.requirement_policy(pkg_name, requirement_grp_id, policy))\n if rule.message:\n self.gen.fact(fn.requirement_message(pkg_name, requirement_grp_id, rule.message))\n self.gen.newline()\n\n for input_spec in requirement_grp:\n spec = spack.spec.Spec(input_spec)\n spec.replace_hash()\n if not spec.name:\n spec.name = pkg_name\n spec.attach_git_version_lookup()\n\n when_spec = spec\n if virtual and spec.name != pkg_name:\n when_spec = spack.spec.Spec(f\"^[virtuals={pkg_name}] {spec}\")\n\n try:\n context = ConditionContext()\n context.source = ConstraintOrigin.append_type_suffix(\n pkg_name, ConstraintOrigin.REQUIRE\n )\n context.wrap_node_requirement = True\n if not virtual:\n context.transform_required = remove_facts(\"depends_on\")\n context.transform_imposed = remove_facts(\n \"node\", \"virtual_node\", \"depends_on\"\n )\n # else: for virtuals we want to emit \"node\" and\n # \"virtual_node\" in imposed specs\n\n info_msg = f\"{input_spec} is a requirement for package {pkg_name}\"\n if rule.condition != EMPTY_SPEC:\n info_msg += f\" when {rule.condition}\"\n if rule.message:\n info_msg += f\" ({rule.message})\"\n member_id = self.condition(\n required_spec=when_spec,\n imposed_spec=spec,\n required_name=pkg_name,\n msg=info_msg,\n context=context,\n )\n\n # Conditions don't handle conditional dependencies directly\n # Those are handled separately here\n self.generate_conditional_dep_conditions(spec, member_id)\n except Exception as e:\n # Do not raise if the rule comes from the 'all' subsection, since usability\n # would be impaired. If a rule does not apply for a specific package, just\n # discard it.\n if rule.kind != RequirementKind.DEFAULT:\n raise RuntimeError(\n \"cannot emit requirements for the solver: \" + str(e)\n ) from e\n continue\n\n self.gen.fact(fn.requirement_group_member(member_id, pkg_name, requirement_grp_id))\n self.gen.fact(\n fn.requirement_has_weight(member_id, requirement_weight * weight_multiplier)\n )\n self.gen.newline()\n requirement_weight += 1\n\n def external_packages(self, packages_with_externals):\n \"\"\"Facts on external packages, from packages.yaml and implicit externals.\"\"\"\n self.gen.h1(\"External packages\")\n for pkg_name, data in packages_with_externals.items():\n if pkg_name == \"all\":\n continue\n\n # This package is not among possible dependencies\n if pkg_name not in self.pkgs:\n continue\n\n if not data.get(\"buildable\", True):\n self.gen.h2(f\"External package: {pkg_name}\")\n self.gen.fact(fn.buildable_false(pkg_name))\n\n def preferred_variants(self, pkg_name):\n \"\"\"Facts on concretization preferences, as read from packages.yaml\"\"\"\n preferences = spack.package_prefs.PackagePrefs\n preferred_variants = preferences.preferred_variants(pkg_name)\n if not preferred_variants:\n return\n\n self.gen.h2(f\"Package preferences: {pkg_name}\")\n\n for variant_name in sorted(preferred_variants):\n variant = preferred_variants[variant_name]\n\n # perform validation of the variant and values\n try:\n variant_defs = vt.prevalidate_variant_value(self.pkg_class(pkg_name), variant)\n except (vt.InvalidVariantValueError, KeyError, ValueError) as e:\n tty.debug(\n f\"[SETUP]: rejected {str(variant)} as a preference for {pkg_name}: {str(e)}\"\n )\n continue\n\n for value in variant.values:\n for variant_def in variant_defs:\n self.variant_values_from_specs.add((pkg_name, id(variant_def), value))\n self.gen.fact(\n fn.variant_default_value_from_packages_yaml(pkg_name, variant.name, value)\n )\n\n def target_preferences(self):\n key_fn = spack.package_prefs.PackagePrefs(\"all\", \"target\")\n\n if not self.target_specs_cache:\n self.target_specs_cache = [\n spack.spec.Spec(\"target={0}\".format(target_name))\n for _, target_name in self.default_targets\n ]\n\n package_targets = self.target_specs_cache[:]\n package_targets.sort(key=key_fn)\n for i, preferred in enumerate(package_targets):\n self.gen.fact(fn.target_weight(str(preferred.architecture.target), i))\n\n def spec_clauses(\n self,\n spec: spack.spec.Spec,\n *,\n name: Optional[str] = None,\n body: bool = False,\n transitive: bool = True,\n expand_hashes: bool = False,\n concrete_build_deps=False,\n include_runtimes=False,\n required_from: Optional[str] = None,\n context: Optional[SourceContext] = None,\n ) -> List[AspFunction]:\n \"\"\"Wrap a call to ``_spec_clauses()`` into a try/except block with better error handling.\n\n Arguments are as for ``_spec_clauses()`` except ``required_from``.\n\n Arguments:\n required_from: name of package that caused this call.\n \"\"\"\n try:\n clauses = self._spec_clauses(\n spec,\n name=spec.name or name,\n body=body,\n transitive=transitive,\n expand_hashes=expand_hashes,\n concrete_build_deps=concrete_build_deps,\n include_runtimes=include_runtimes,\n context=context,\n )\n except RuntimeError as exc:\n msg = str(exc)\n if required_from:\n msg += f\" [required from package '{required_from}']\"\n raise RuntimeError(msg)\n return clauses\n\n def _spec_clauses(\n self,\n spec: spack.spec.Spec,\n *,\n name: Optional[str] = None,\n body: bool = False,\n transitive: bool = True,\n expand_hashes: bool = False,\n concrete_build_deps: bool = False,\n include_runtimes: bool = False,\n context: Optional[SourceContext] = None,\n seen: Optional[Set[int]] = None,\n ) -> List[AspFunction]:\n \"\"\"Return a list of clauses for a spec mandates are true.\n\n Arguments:\n spec: the spec to analyze\n name: optional fallback of spec.name (used for anonymous roots)\n body: if True, generate clauses to be used in rule bodies (final values) instead\n of rule heads (setters).\n transitive: if False, don't generate clauses from dependencies (default True)\n expand_hashes: if True, descend into hashes of concrete specs (default False)\n concrete_build_deps: if False, do not include pure build deps of concrete specs\n (as they have no effect on runtime constraints)\n include_runtimes: generate full dependency clauses from runtime libraries that\n are omitted from the solve.\n context: tracks what constraint this clause set is generated for (e.g. a\n ``depends_on`` constraint in a package.py file)\n seen: set of ids of specs that have already been processed (for internal use only)\n\n Normally, if called with ``transitive=True``, ``spec_clauses()`` just generates\n hashes for the dependency requirements of concrete specs. If ``expand_hashes``\n is ``True``, we'll *also* output all the facts implied by transitive hashes,\n which are redundant during a solve but useful outside of one (e.g.,\n for spec ``diff``).\n \"\"\"\n clauses = []\n seen = seen if seen is not None else set()\n name = spec.name or name or \"\"\n seen.add(id(spec))\n\n f: Union[Type[_Head], Type[_Body]] = _Body if body else _Head\n\n if name:\n clauses.append(\n f.node(name) if not spack.repo.PATH.is_virtual(name) else f.virtual_node(name)\n )\n if spec.namespace:\n clauses.append(f.namespace(name, spec.namespace))\n\n clauses.extend(self.spec_versions(spec, name=name))\n\n # seed architecture at the root (we'll propagate later)\n # TODO: use better semantics.\n arch = spec.architecture\n if arch:\n if arch.platform:\n clauses.append(f.node_platform(name, arch.platform))\n if arch.os:\n clauses.append(f.node_os(name, arch.os))\n if arch.target:\n clauses.extend(self.target_ranges(spec, f.node_target, name=name))\n\n # variants\n for vname, variant in sorted(spec.variants.items()):\n # TODO: variant=\"*\" means 'variant is defined to something', which used to\n # be meaningless in concretization, as all variants had to be defined. But\n # now that variants can be conditional, it should force a variant to exist.\n if not variant.values:\n continue\n\n for value in variant.values:\n # ensure that the value *can* be valid for the spec\n if name and not spec.concrete and not spack.repo.PATH.is_virtual(name):\n variant_defs = vt.prevalidate_variant_value(\n self.pkg_class(name), variant, spec\n )\n\n # Record that that this is a valid possible value. Accounts for\n # int/str/etc., where valid values can't be listed in the package\n for variant_def in variant_defs:\n self.variant_values_from_specs.add((name, id(variant_def), value))\n\n if variant.propagate:\n clauses.append(f.propagate(name, fn.variant_value(vname, value)))\n if self.pkg_class(name).has_variant(vname):\n clauses.append(f.variant_value(name, vname, value))\n else:\n variant_clause = f.variant_value(name, vname, value)\n if (\n variant.concrete\n and variant.type == vt.VariantType.MULTI\n and not spec.concrete\n ):\n if body is False:\n variant_clause.args = (\n f\"concrete_{variant_clause.args[0]}\",\n *variant_clause.args[1:],\n )\n else:\n clauses.append(fn.attr(\"concrete_variant_request\", name, vname, value))\n clauses.append(variant_clause)\n\n # compiler flags\n source = context.source if context else \"none\"\n for flag_type, flags in spec.compiler_flags.items():\n flag_group = \" \".join(flags)\n for flag in flags:\n clauses.append(\n f.node_flag(name, fn.node_flag(flag_type, flag, flag_group, source))\n )\n if not spec.concrete and flag.propagate is True:\n clauses.append(\n f.propagate(\n name,\n fn.node_flag(flag_type, flag, flag_group, source),\n fn.edge_types(\"link\", \"run\"),\n )\n )\n\n # Hash for concrete specs\n if spec.concrete:\n # older specs do not have package hashes, so we have to do this carefully\n package_hash = getattr(spec, \"_package_hash\", None)\n if package_hash:\n clauses.append(fn.attr(\"package_hash\", name, package_hash))\n clauses.append(fn.attr(\"hash\", name, spec.dag_hash()))\n if spec.external:\n clauses.append(fn.attr(\"external\", name))\n\n # TODO: a loop over `edges_to_dependencies` is preferred over `edges_from_dependents`\n # since dependents can point to specs out of scope for the solver.\n edges = spec.edges_from_dependents()\n if not body and not spec.concrete:\n virtuals = sorted(set(itertools.chain.from_iterable(edge.virtuals for edge in edges)))\n for virtual in virtuals:\n clauses.append(fn.attr(\"provider_set\", name, virtual))\n clauses.append(fn.attr(\"virtual_node\", virtual))\n else:\n # direct dependencies are handled under `edges_to_dependencies()`\n virtual_iter = (edge.virtuals for edge in edges if not edge.direct)\n virtuals = sorted(set(itertools.chain.from_iterable(virtual_iter)))\n for virtual in virtuals:\n clauses.append(fn.attr(\"virtual_on_incoming_edges\", name, virtual))\n\n # If the spec is external and concrete, we allow all the libcs on the system\n if spec.external and spec.concrete and using_libc_compatibility():\n clauses.append(fn.attr(\"needs_libc\", name))\n for libc in self.libcs:\n clauses.append(fn.attr(\"compatible_libc\", name, libc.name, libc.version))\n\n if not transitive:\n return clauses\n\n # Dependencies\n edge_clauses = []\n for dspec in spec.edges_to_dependencies():\n # Ignore conditional dependencies, they are handled by caller\n if dspec.when != EMPTY_SPEC:\n continue\n\n dep = dspec.spec\n\n if spec.concrete:\n # GCC runtime is solved again by clingo, even on concrete specs, to give\n # the possibility to reuse specs built against a different runtime.\n if dep.name == \"gcc-runtime\":\n edge_clauses.append(\n fn.attr(\"compatible_runtime\", name, dep.name, f\"{dep.version}:\")\n )\n constraint_spec = spack.spec.Spec(f\"{dep.name}@{dep.version}\")\n self.spec_versions(constraint_spec)\n if not include_runtimes:\n continue\n\n # libc is also solved again by clingo, but in this case the compatibility\n # is not encoded in the parent node - so we need to emit explicit facts\n if \"libc\" in dspec.virtuals:\n edge_clauses.append(fn.attr(\"needs_libc\", name))\n for libc in self.libcs:\n if libc_is_compatible(libc, dep):\n edge_clauses.append(\n fn.attr(\"compatible_libc\", name, libc.name, libc.version)\n )\n if not include_runtimes:\n continue\n\n # We know dependencies are real for concrete specs. For abstract\n # specs they just mean the dep is somehow in the DAG.\n for dtype in dt.ALL_FLAGS:\n if not dspec.depflag & dtype:\n continue\n # skip build dependencies of already-installed specs\n if concrete_build_deps or dtype != dt.BUILD:\n edge_clauses.append(\n fn.attr(\"depends_on\", name, dep.name, dt.flag_to_string(dtype))\n )\n for virtual_name in dspec.virtuals:\n edge_clauses.append(\n fn.attr(\"virtual_on_edge\", name, dep.name, virtual_name)\n )\n edge_clauses.append(fn.attr(\"virtual_node\", virtual_name))\n\n # imposing hash constraints for all but pure build deps of\n # already-installed concrete specs.\n if concrete_build_deps or dspec.depflag != dt.BUILD:\n edge_clauses.append(fn.attr(\"hash\", dep.name, dep.dag_hash()))\n elif not concrete_build_deps and dspec.depflag:\n edge_clauses.append(\n fn.attr(\"concrete_build_dependency\", name, dep.name, dep.dag_hash())\n )\n for virtual_name in dspec.virtuals:\n edge_clauses.append(\n fn.attr(\"virtual_on_build_edge\", name, dep.name, virtual_name)\n )\n\n # if the spec is abstract, descend into dependencies.\n # if it's concrete, then the hashes above take care of dependency\n # constraints, but expand the hashes if asked for.\n if (not spec.concrete or expand_hashes) and id(dep) not in seen:\n dependency_clauses = self._spec_clauses(\n dep,\n body=body,\n expand_hashes=expand_hashes,\n concrete_build_deps=concrete_build_deps,\n context=context,\n seen=seen,\n )\n ###\n # Dependency expressed with \"^\"\n ###\n if not dspec.direct:\n edge_clauses.extend(dependency_clauses)\n continue\n\n ###\n # Direct dependencies expressed with \"%\"\n ###\n for dependency_type in dt.flag_to_tuple(dspec.depflag):\n edge_clauses.append(fn.attr(\"depends_on\", name, dep.name, dependency_type))\n\n for virtual in dspec.virtuals:\n dependency_clauses.append(fn.attr(\"virtual_on_edge\", name, dep.name, virtual))\n\n # By default, wrap head of rules, unless the context says otherwise\n wrap_node_requirement = body is False\n if context and context.wrap_node_requirement is not None:\n wrap_node_requirement = context.wrap_node_requirement\n\n if not wrap_node_requirement:\n edge_clauses.extend(dependency_clauses)\n continue\n\n for clause in dependency_clauses:\n clause.name = \"node_requirement\"\n edge_clauses.append(fn.attr(\"direct_dependency\", name, clause))\n\n clauses.extend(edge_clauses)\n return clauses\n\n def define_package_versions_and_validate_preferences(\n self, possible_pkgs: Set[str], *, require_checksum: bool, allow_deprecated: bool\n ):\n \"\"\"Declare any versions in specs not declared in packages.\"\"\"\n packages_yaml = spack.config.CONFIG.get_config(\"packages\")\n for pkg_name in sorted(possible_pkgs):\n pkg_cls = self.pkg_class(pkg_name)\n\n # All the versions from the corresponding package.py file. Since concepts\n # like being a \"develop\" version or being preferred exist only at a\n # package.py level, sort them in this partial list here\n from_package_py = list(pkg_cls.versions.items())\n\n if require_checksum and pkg_cls.has_code:\n from_package_py = [x for x in from_package_py if _is_checksummed_version(x)]\n\n for v, version_info in from_package_py:\n if version_info.get(\"deprecated\", False):\n self.deprecated_versions[pkg_name].add(v)\n if not allow_deprecated:\n continue\n\n self.possible_versions[pkg_name][v].append(Provenance.PACKAGE_PY)\n\n if pkg_name not in packages_yaml or \"version\" not in packages_yaml[pkg_name]:\n continue\n\n # TODO(psakiev) Need facts about versions\n # - requires_commit (associated with tag or branch)\n from_packages_yaml: List[GitOrStandardVersion] = []\n\n for vstr in packages_yaml[pkg_name][\"version\"]:\n cfg_ver = vn.ver(vstr)\n\n if isinstance(cfg_ver, vn.GitVersion):\n if not require_checksum or cfg_ver.is_commit:\n from_packages_yaml.append(cfg_ver)\n else:\n matches = [x for x in self.possible_versions[pkg_name] if x.satisfies(cfg_ver)]\n matches.sort(reverse=True)\n if not matches:\n raise spack.error.ConfigError(\n f\"Preference for version {cfg_ver} does not match any known \"\n f\"version of {pkg_name}\"\n )\n from_packages_yaml.extend(matches)\n\n from_packages_yaml = list(spack.llnl.util.lang.dedupe(from_packages_yaml))\n for v in from_packages_yaml:\n provenance = Provenance.PACKAGES_YAML\n if isinstance(v, vn.GitVersion):\n provenance = Provenance.PACKAGES_YAML_GIT_VERSION\n self.possible_versions[pkg_name][v].append(provenance)\n self.versions_from_yaml[pkg_name] = from_packages_yaml\n\n def define_ad_hoc_versions_from_specs(\n self, specs, origin, *, allow_deprecated: bool, require_checksum: bool\n ):\n \"\"\"Add concrete versions to possible versions from lists of CLI/dev specs.\"\"\"\n for s in traverse.traverse_nodes(specs):\n # If there is a concrete version on the CLI *that we know nothing\n # about*, add it to the known versions.\n version = s.versions.concrete\n\n if version is None or (any((v == version) for v in self.possible_versions[s.name])):\n continue\n\n if require_checksum and not _is_checksummed_git_version(version):\n raise UnsatisfiableSpecError(\n s.format(\"No matching version for constraint {name}{@versions}\")\n )\n\n if not allow_deprecated and version in self.deprecated_versions[s.name]:\n continue\n\n self.possible_versions[s.name][version].append(origin)\n\n def _supported_targets(self, compiler_name, compiler_version, targets):\n \"\"\"Get a list of which targets are supported by the compiler.\n\n Results are ordered most to least recent.\n \"\"\"\n supported, unsupported = [], []\n\n for target in targets:\n try:\n with warnings.catch_warnings():\n warnings.simplefilter(\"ignore\")\n target.optimization_flags(\n compiler_name, compiler_version.dotted_numeric_string\n )\n supported.append(target)\n except spack.vendor.archspec.cpu.UnsupportedMicroarchitecture:\n unsupported.append(target)\n except ValueError:\n unsupported.append(target)\n\n return supported, unsupported\n\n def platform_defaults(self):\n self.gen.h2(\"Default platform\")\n platform = spack.platforms.host()\n self.gen.fact(fn.node_platform_default(platform))\n self.gen.fact(fn.allowed_platform(platform))\n\n def os_defaults(self, specs):\n self.gen.h2(\"Possible operating systems\")\n platform = spack.platforms.host()\n\n # create set of OS's to consider\n buildable = set(platform.operating_sys.keys())\n\n # Consider any OS's mentioned on the command line. We need this to\n # cross-concretize in CI, and for some tests.\n # TODO: OS should really be more than just a label -- rework this.\n for spec in specs:\n if spec.architecture and spec.architecture.os:\n buildable.add(spec.architecture.os)\n\n # make directives for buildable OS's\n for build_os in sorted(buildable):\n self.gen.fact(fn.buildable_os(build_os))\n\n def keyfun(os):\n return (\n os == platform.default_os, # prefer default\n os not in buildable, # then prefer buildables\n os, # then sort by name\n )\n\n all_oses = buildable.union(self.possible_oses)\n ordered_oses = sorted(all_oses, key=keyfun, reverse=True)\n\n # output the preference order of OS's for the concretizer to choose\n for i, os_name in enumerate(ordered_oses):\n self.gen.fact(fn.os(os_name, i))\n\n def target_defaults(self, specs):\n \"\"\"Add facts about targets and target compatibility.\"\"\"\n self.gen.h2(\"Target compatibility\")\n\n # Add targets explicitly requested from specs\n candidate_targets = []\n for x in self.possible_graph.candidate_targets():\n if all(\n self.possible_graph.unreachable(pkg_name=pkg_name, when_spec=f\"target={x}\")\n for pkg_name in self.pkgs\n ):\n tty.debug(f\"[{__name__}] excluding target={x}, cause no package can use it\")\n continue\n candidate_targets.append(x)\n\n host_compatible = spack.config.CONFIG.get(\"concretizer:targets:host_compatible\")\n for spec in specs:\n if not spec.architecture or not spec.architecture.target:\n continue\n\n target_name = spec.target.name\n target = spack.vendor.archspec.cpu.TARGETS.get(target_name)\n if not target:\n if spec.architecture.target_concrete:\n raise spack.error.SpecError(\n f\"the target '{target_name}' in '{spec} is not a known target. \"\n f\"Run 'spack arch --known-targets' to see valid targets.\"\n )\n # range/list constraint (contains ':' or ','): keep existing path\n self.target_ranges(spec, None)\n continue\n\n if target not in candidate_targets and not host_compatible:\n candidate_targets.append(target)\n for ancestor in target.ancestors:\n if ancestor not in candidate_targets:\n candidate_targets.append(ancestor)\n\n platform = spack.platforms.host()\n uarch = spack.vendor.archspec.cpu.TARGETS.get(platform.default)\n best_targets = {uarch.family.name}\n for compiler in self.possible_compilers:\n supported, unsupported = self._supported_targets(\n compiler.name, compiler.version, candidate_targets\n )\n\n for target in supported:\n best_targets.add(target.name)\n self.gen.fact(fn.target_supported(compiler.name, compiler.version, target.name))\n\n if supported:\n self.gen.fact(\n fn.target_supported(compiler.name, compiler.version, uarch.family.name)\n )\n\n for target in unsupported:\n self.gen.fact(\n fn.target_not_supported(compiler.name, compiler.version, target.name)\n )\n\n self.gen.newline()\n\n i = 0\n for target in candidate_targets:\n self.gen.fact(fn.target(target.name))\n self.gen.fact(fn.target_family(target.name, target.family.name))\n self.gen.fact(fn.target_compatible(target.name, target.name))\n # Code for ancestor can run on target\n for ancestor in target.ancestors:\n self.gen.fact(fn.target_compatible(target.name, ancestor.name))\n\n # prefer best possible targets; weight others poorly so\n # they're not used unless set explicitly\n # these are stored to be generated as facts later offset by the\n # number of preferred targets\n if target.name in best_targets:\n self.default_targets.append((i, target.name))\n i += 1\n else:\n self.default_targets.append((100, target.name))\n self.gen.newline()\n\n self.default_targets = list(sorted(set(self.default_targets)))\n self.target_preferences()\n\n def define_version_constraints(self):\n \"\"\"Define what version_satisfies(...) means in ASP logic.\"\"\"\n\n sorted_versions = {}\n for pkg_name in self.possible_versions:\n possible_versions = list(self.possible_versions[pkg_name])\n possible_versions.sort()\n sorted_versions[pkg_name] = possible_versions\n for idx, v in enumerate(possible_versions):\n self.gen.fact(fn.pkg_fact(pkg_name, fn.version_order(v, idx)))\n if v in self.git_commit_versions[pkg_name]:\n sha = self.git_commit_versions[pkg_name].get(v)\n if sha:\n self.gen.fact(fn.pkg_fact(pkg_name, fn.version_has_commit(v, sha)))\n else:\n self.gen.fact(fn.pkg_fact(pkg_name, fn.version_needs_commit(v)))\n self.gen.newline()\n self.gen.newline()\n\n for pkg_name, set_of_versions in sorted(self.version_constraints.items()):\n possible_versions = sorted_versions.get(pkg_name)\n if possible_versions is None:\n continue\n for versions in sorted(set_of_versions):\n # Look for contiguous ranges of versions that satisfy the constraint\n start_idx = None\n for current_idx, v in enumerate(possible_versions):\n if v.satisfies(versions):\n if start_idx is None:\n start_idx = current_idx\n elif start_idx is not None:\n # End of a contiguous satisfying range found\n version_range = fn.version_range(versions, start_idx, current_idx - 1)\n self.gen.fact(fn.pkg_fact(pkg_name, version_range))\n start_idx = None\n if start_idx is not None:\n version_range = fn.version_range(\n versions, start_idx, len(possible_versions) - 1\n )\n self.gen.fact(fn.pkg_fact(pkg_name, version_range))\n self.gen.newline()\n\n def collect_virtual_constraints(self):\n \"\"\"Define versions for constraints on virtuals.\n\n Must be called before define_version_constraints().\n \"\"\"\n\n # extract all the real versions mentioned in version ranges\n def versions_for(v):\n if isinstance(v, vn.StandardVersion):\n yield v\n elif isinstance(v, vn.ClosedOpenRange):\n yield v.lo\n yield vn._prev_version(v.hi)\n elif isinstance(v, vn.VersionList):\n for e in v:\n yield from versions_for(e)\n else:\n raise TypeError(f\"expected version type, found: {type(v)}\")\n\n # Define a set of synthetic possible versions for virtuals that don't define versions in a\n # package.py file. This ensures that `version_satisfies(Package, Constraint, Version)` has\n # the same semantics for virtuals as for regular packages.\n for pkg_name, versions in self.version_constraints.items():\n # Not a virtual package\n if pkg_name not in self.possible_virtuals:\n continue\n\n possible_versions = {pv for v in versions for pv in versions_for(v)}\n for version in possible_versions:\n self.possible_versions[pkg_name][version].append(Provenance.VIRTUAL_CONSTRAINT)\n\n def define_target_constraints(self):\n def _all_targets_satisfiying(single_constraint):\n allowed_targets = []\n\n if \":\" not in single_constraint:\n return [single_constraint]\n\n t_min, _, t_max = single_constraint.partition(\":\")\n for test_target in spack.vendor.archspec.cpu.TARGETS.values():\n # Check lower bound\n if t_min and not t_min <= test_target:\n continue\n\n # Check upper bound\n if t_max and not t_max >= test_target:\n continue\n\n allowed_targets.append(test_target)\n return allowed_targets\n\n cache = {}\n for target_constraint in sorted(self.target_constraints, key=lambda x: x.name):\n # Construct the list of allowed targets for this constraint\n allowed_targets = []\n for single_constraint in str(target_constraint).split(\",\"):\n if single_constraint not in cache:\n cache[single_constraint] = _all_targets_satisfiying(single_constraint)\n allowed_targets.extend(cache[single_constraint])\n\n for target in allowed_targets:\n self.gen.fact(fn.target_satisfies(target_constraint, target))\n self.gen.newline()\n\n def define_variant_values(self):\n \"\"\"Validate variant values from the command line.\n\n Add valid variant values from the command line to the possible values for\n variant definitions.\n\n \"\"\"\n # for determinism, sort by variant ids, not variant def ids (which are object ids)\n def_info = []\n for pkg_name, variant_def_id, value in sorted(self.variant_values_from_specs):\n try:\n vid = self.variant_ids_by_def_id[variant_def_id]\n except KeyError:\n tty.debug(\n f\"[{__name__}] cannot retrieve id of the {value} variant from {pkg_name}\"\n )\n continue\n def_info.append((pkg_name, vid, value))\n\n # Tell the concretizer about possible values from specs seen in spec_clauses().\n for pkg_name, vid, value in sorted(def_info):\n self.gen.fact(fn.pkg_fact(pkg_name, fn.variant_possible_value(vid, value)))\n\n def register_concrete_spec(self, spec, possible: set):\n # tell the solver about any installed packages that could\n # be dependencies (don't tell it about the others)\n if spec.name not in possible:\n return\n\n try:\n # Only consider installed packages for repo we know\n spack.repo.PATH.get(spec)\n except (spack.repo.UnknownNamespaceError, spack.repo.UnknownPackageError) as e:\n tty.debug(f\"[REUSE] Issues when trying to reuse {spec.short_spec}: {str(e)}\")\n return\n\n self.reusable_and_possible.add(spec)\n\n def concrete_specs(self):\n \"\"\"Emit facts for reusable specs\"\"\"\n for h, spec in self.reusable_and_possible.explicit_items():\n # this indicates that there is a spec like this installed\n self.gen.fact(fn.installed_hash(spec.name, h))\n # indirection layer between hash constraints and imposition to allow for splicing\n for pred in self.spec_clauses(spec, body=True, required_from=None):\n self.gen.fact(fn.hash_attr(h, *pred.args))\n self.gen.newline()\n # Declare as possible parts of specs that are not in package.py\n # - Add versions to possible versions\n # - Add OS to possible OS's\n\n for dep in spec.traverse():\n provenance = Provenance.INSTALLED\n if isinstance(dep.version, vn.GitVersion):\n provenance = Provenance.INSTALLED_GIT_VERSION\n\n self.possible_versions[dep.name][dep.version].append(provenance)\n self.possible_oses.add(dep.os)\n\n def define_concrete_input_specs(self, specs: tuple, possible: set):\n # any concrete specs in the input spec list\n for input_spec in specs:\n for spec in input_spec.traverse():\n if spec.concrete:\n self.register_concrete_spec(spec, possible)\n\n def impossible_dependencies_check(self, specs) -> None:\n for edge in traverse.traverse_edges(specs):\n possible_deps = self.pkgs\n if spack.repo.PATH.is_virtual(edge.spec.name):\n possible_deps = self.possible_virtuals\n if edge.spec.name not in possible_deps and not str(edge.when):\n raise InvalidDependencyError(\n f\"'{edge.spec.name}' is not a possible dependency of any root spec\"\n )\n\n def input_spec_version_check(self, specs, allow_deprecated: bool) -> None:\n \"\"\"Raise an error early if no versions available in the solve can satisfy the inputs.\"\"\"\n only_deprecated = []\n impossible = []\n\n for spec in traverse.traverse_nodes(specs):\n if spack.repo.PATH.is_virtual(spec.name):\n continue\n if spec.name not in self.pkgs:\n continue # conditional dependency that won't be satisfied\n\n deprecated = self.deprecated_versions.get(spec.name, set())\n sat_deprecated = [v for v in deprecated if deprecated and v.satisfies(spec.versions)]\n\n possible: Iterable = self.possible_versions.get(spec.name, set())\n sat_possible = [v for v in possible if possible and v.satisfies(spec.versions)]\n\n if sat_deprecated and not sat_possible:\n only_deprecated.append(spec)\n\n if not sat_deprecated and not sat_possible:\n impossible.append(spec)\n\n if not allow_deprecated and only_deprecated:\n raise DeprecatedVersionError(\n \"The following input specs can only be satisfied by deprecated versions:\",\n \" \"\n + \", \".join(str(spec) for spec in only_deprecated)\n + \"\\n\"\n + \"Run with --deprecated to allow Spack to use these versions.\",\n )\n\n if impossible:\n raise InvalidVersionError(\n \"No version exists that satisfies these input specs:\",\n \" \" + \", \".join(str(spec) for spec in impossible),\n )\n\n def setup(\n self,\n specs: Sequence[spack.spec.Spec],\n *,\n reuse: Optional[List[spack.spec.Spec]] = None,\n packages_with_externals=None,\n allow_deprecated: bool = False,\n ) -> \"ProblemInstanceBuilder\":\n \"\"\"Generate an ASP program with relevant constraints for specs.\n\n This calls methods on the solve driver to set up the problem with\n facts and rules from all possible dependencies of the input\n specs, as well as constraints from the specs themselves.\n\n Arguments:\n specs: list of Specs to solve\n reuse: list of concrete specs that can be reused\n packages_with_externals: precomputed packages config with implicit externals\n allow_deprecated: if True adds deprecated versions into the solve\n\n Return:\n A ProblemInstanceBuilder populated with facts and rules for an ASP solve.\n \"\"\"\n # TODO: remove this local import and get rid of dependency on globals\n import spack.environment as ev\n\n reuse = reuse or []\n if packages_with_externals is None:\n packages_with_externals = external_config_with_implicit_externals(spack.config.CONFIG)\n check_packages_exist(specs)\n self.gen = ProblemInstanceBuilder()\n\n # Get compilers from buildcaches only if injected through \"reuse\" specs\n supported_compilers = spack.compilers.config.supported_compilers()\n compilers_from_reuse = {\n x for x in reuse if x.name in supported_compilers and not x.external\n }\n candidate_compilers, self.rejected_compilers = possible_compilers(\n configuration=spack.config.CONFIG\n )\n reuse_from_compilers = traverse.traverse_nodes(\n [x for x in candidate_compilers if not x.external], deptype=(\"link\", \"run\")\n )\n reused_set = set(reuse)\n reuse += [x for x in reuse_from_compilers if x not in reused_set]\n\n candidate_compilers.update(compilers_from_reuse)\n self.possible_compilers = list(candidate_compilers)\n\n # TODO: warning is because mypy doesn't know Spec supports rich comparison via decorator\n self.possible_compilers.sort() # type: ignore[call-arg,call-overload]\n\n self.compiler_mixing()\n\n self.gen.h1(\"Runtimes\")\n injected_dependencies = self.define_runtime_constraints()\n\n node_counter = create_counter(\n list(specs) + injected_dependencies,\n tests=self.tests,\n possible_graph=self.possible_graph,\n )\n self.possible_virtuals = node_counter.possible_virtuals()\n self.pkgs = node_counter.possible_dependencies()\n self.libcs = sorted(all_libcs()) # type: ignore[type-var]\n\n for node in traverse.traverse_nodes(specs):\n if node.namespace is not None:\n self.explicitly_required_namespaces[node.name] = node.namespace\n\n self.requirement_parser.parse_rules_from_input_specs(specs)\n self.gen.h1(\"Generic information\")\n if using_libc_compatibility():\n for libc in self.libcs:\n self.gen.fact(fn.host_libc(libc.name, libc.version))\n\n if not allow_deprecated:\n self.gen.fact(fn.deprecated_versions_not_allowed())\n\n self.gen.newline()\n for pkg_name in spack.compilers.config.supported_compilers():\n self.gen.fact(fn.compiler_package(pkg_name))\n\n # Calculate develop specs\n # they will be used in addition to command line specs\n # in determining known versions/targets/os\n dev_specs: Tuple[spack.spec.Spec, ...] = ()\n env = ev.active_environment()\n if env:\n dev_specs = tuple(\n spack.spec.Spec(info[\"spec\"]).constrained(\n 'dev_path=\"%s\"'\n % spack.util.path.canonicalize_path(info[\"path\"], default_wd=env.path)\n )\n for name, info in env.dev_specs.items()\n )\n\n specs = tuple(specs) # ensure compatible types to add\n\n self.gen.h1(\"Reusable concrete specs\")\n self.define_concrete_input_specs(specs, self.pkgs)\n if reuse:\n self.gen.fact(fn.optimize_for_reuse())\n for reusable_spec in reuse:\n self.register_concrete_spec(reusable_spec, self.pkgs)\n self.concrete_specs()\n\n self.gen.h1(\"Generic statements on possible packages\")\n node_counter.possible_packages_facts(self.gen, fn)\n\n self.gen.h1(\"Possible flags on nodes\")\n for flag in spack.spec.FlagMap.valid_compiler_flags():\n self.gen.fact(fn.flag_type(flag))\n self.gen.newline()\n\n self.gen.h1(\"General Constraints\")\n self.config_compatible_os()\n\n # architecture defaults\n self.platform_defaults()\n self.os_defaults(specs + dev_specs)\n self.target_defaults(specs + dev_specs)\n\n self.virtual_requirements_and_weights()\n self.external_packages(packages_with_externals)\n\n # TODO: make a config option for this undocumented feature\n checksummed = \"SPACK_CONCRETIZER_REQUIRE_CHECKSUM\" in os.environ\n self.define_package_versions_and_validate_preferences(\n self.pkgs, allow_deprecated=allow_deprecated, require_checksum=checksummed\n )\n self.define_ad_hoc_versions_from_specs(\n specs, Provenance.SPEC, allow_deprecated=allow_deprecated, require_checksum=checksummed\n )\n self.define_ad_hoc_versions_from_specs(\n dev_specs,\n Provenance.DEV_SPEC,\n allow_deprecated=allow_deprecated,\n require_checksum=checksummed,\n )\n self.validate_and_define_versions_from_requirements(\n allow_deprecated=allow_deprecated, require_checksum=checksummed\n )\n\n self.gen.h1(\"Package Constraints\")\n for pkg in sorted(self.pkgs):\n self.gen.h2(f\"Package rules: {pkg}\")\n self.pkg_rules(pkg, tests=self.tests)\n self.preferred_variants(pkg)\n\n self.gen.h1(\"Condition Triggers and Imposed Effects\")\n self.trigger_rules()\n self.effect_rules()\n\n self.gen.h1(\"Special variants\")\n self.define_auto_variant(\"dev_path\", multi=False)\n self.define_auto_variant(\"commit\", multi=False)\n self.define_auto_variant(\"patches\", multi=True)\n\n self.gen.h1(\"Develop specs\")\n # Inject dev_path from environment\n for ds in dev_specs:\n self.condition(spack.spec.Spec(ds.name), ds, msg=f\"{ds.name} is a develop spec\")\n self.trigger_rules()\n self.effect_rules()\n\n self.gen.h1(\"Spec Constraints\")\n self.literal_specs(specs)\n\n self.gen.h1(\"Variant Values defined in specs\")\n self.define_variant_values()\n\n self.gen.h1(\"Version Constraints\")\n self.collect_virtual_constraints()\n self.define_version_constraints()\n\n self.gen.h1(\"Target Constraints\")\n self.define_target_constraints()\n\n # once we've done a full traversal and know possible versions, check that the\n # requested solve is at least consistent.\n # do not check dependency and version availability for already concrete specs\n # as they come from reusable specs\n abstract_specs = [s for s in specs if not s.concrete]\n self.impossible_dependencies_check(abstract_specs)\n self.input_spec_version_check(abstract_specs, allow_deprecated)\n\n return self.gen\n\n def compiler_mixing(self):\n should_mix = spack.config.get(\"concretizer:compiler_mixing\", True)\n if should_mix is True:\n return\n # anything besides should_mix: true\n for lang in [\"c\", \"cxx\", \"fortran\"]:\n self.gen.fact(fn.no_compiler_mixing(lang))\n # user specified an allow-list\n if isinstance(should_mix, list):\n for pkg_name in should_mix:\n self.gen.fact(fn.allow_mixing(pkg_name))\n\n def define_runtime_constraints(self) -> List[spack.spec.Spec]:\n \"\"\"Define the constraints to be imposed on the runtimes, and returns a list of\n injected packages.\n \"\"\"\n recorder = RuntimePropertyRecorder(self)\n\n for compiler in self.possible_compilers:\n try:\n compiler_cls = spack.repo.PATH.get_pkg_class(compiler.name)\n except spack.repo.UnknownPackageError:\n pass\n else:\n if hasattr(compiler_cls, \"runtime_constraints\"):\n compiler_cls.runtime_constraints(spec=compiler, pkg=recorder)\n # Inject default flags for compilers\n recorder(\"*\").default_flags(compiler)\n\n # Add a dependency on the compiler wrapper\n compiler_str = f\"{compiler.name} /{compiler.dag_hash()}\"\n for language in (\"c\", \"cxx\", \"fortran\"):\n # Using compiler.name causes a bit of duplication, but that is taken care of by\n # clingo during grounding.\n recorder(\"*\").depends_on(\n \"compiler-wrapper\",\n when=f\"%[deptypes=build virtuals={language}] {compiler.name}\",\n type=\"build\",\n description=f\"Add compiler wrapper when using {compiler.name} for {language}\",\n )\n\n if not using_libc_compatibility():\n continue\n\n current_libc = None\n if compiler.external or compiler.installed:\n current_libc = CompilerPropertyDetector(compiler).default_libc()\n else:\n try:\n current_libc = compiler[\"libc\"]\n except (KeyError, RuntimeError) as e:\n tty.debug(f\"{compiler} cannot determine libc because: {e}\")\n\n if current_libc:\n recorder(\"*\").depends_on(\n \"libc\",\n when=f\"%[deptypes=build] {compiler.name}\",\n type=\"link\",\n description=f\"Add libc when using {compiler.name}\",\n )\n recorder(\"*\").depends_on(\n f\"{current_libc.name}@={current_libc.version}\",\n when=f\"%[deptypes=build] {compiler_str}\",\n type=\"link\",\n description=f\"Libc is {current_libc} when using {compiler}\",\n )\n\n recorder.consume_facts()\n return sorted(recorder.injected_dependencies)\n\n def literal_specs(self, specs):\n for spec in sorted(specs):\n self.gen.h2(f\"Spec: {str(spec)}\")\n condition_id = next(self._id_counter)\n trigger_id = next(self._id_counter)\n\n # Special condition triggered by \"literal_solved\"\n self.gen.fact(fn.literal(trigger_id))\n self.gen.fact(fn.pkg_fact(spec.name, fn.condition_trigger(condition_id, trigger_id)))\n self.gen.fact(fn.condition_reason(condition_id, f\"{spec} requested explicitly\"))\n\n imposed_spec_key = str(spec), None\n cache = self._effect_cache[spec.name]\n if imposed_spec_key in cache:\n effect_id, requirements = cache[imposed_spec_key]\n else:\n effect_id = next(self._id_counter)\n context = SourceContext()\n context.source = \"literal\"\n requirements = self.spec_clauses(spec, context=context)\n root_name = spec.name\n for clause in requirements:\n clause_name = clause.args[0]\n if clause_name == \"variant_set\":\n requirements.append(\n fn.attr(\"variant_default_value_from_cli\", *clause.args[1:])\n )\n elif clause_name in (\"node\", \"virtual_node\", \"hash\"):\n # These facts are needed to compute the \"condition_set\" of the root\n pkg_name = clause.args[1]\n self.gen.fact(fn.mentioned_in_literal(trigger_id, root_name, pkg_name))\n\n requirements.append(\n fn.attr(\n \"virtual_root\" if spack.repo.PATH.is_virtual(spec.name) else \"root\", spec.name\n )\n )\n requirements = [x for x in requirements if x.args[0] != \"depends_on\"]\n cache[imposed_spec_key] = (effect_id, requirements)\n self.gen.fact(fn.pkg_fact(spec.name, fn.condition_effect(condition_id, effect_id)))\n\n # Create subcondition with any conditional dependencies\n # self.spec_clauses does not do anything with conditional\n # dependencies\n self.generate_conditional_dep_conditions(spec, condition_id)\n\n if self.concretize_everything:\n self.gen.fact(fn.solve_literal(trigger_id))\n\n # Trigger rules are needed to allow conditional specs\n self.trigger_rules()\n self.effect_rules()\n\n def generate_conditional_dep_conditions(self, spec: spack.spec.Spec, condition_id: int):\n \"\"\"Generate a subcondition in the trigger for any conditional dependencies.\n\n Dependencies are always modeled by a condition. For conditional dependencies,\n the when-spec is added as a subcondition of the trigger to ensure the dependency\n is only activated when the subcondition holds.\n \"\"\"\n for dspec in spec.traverse_edges():\n # Ignore unconditional deps\n if dspec.when == EMPTY_SPEC:\n continue\n\n # Cannot use \"virtual_node\" attr as key for condition\n # because reused specs do not track virtual nodes.\n # Instead, track whether the parent uses the virtual\n def virtual_handler(\n name: str, input_spec: spack.spec.Spec, requirements: List[AspFunction]\n ) -> List[AspFunction]:\n ret = remove_facts(\"virtual_node\")(name, input_spec, requirements)\n for edge in input_spec.traverse_edges(root=False, cover=\"edges\"):\n if spack.repo.PATH.is_virtual(edge.spec.name):\n parent_name = name if edge.parent is input_spec else edge.parent.name\n ret.append(fn.attr(\"uses_virtual\", parent_name, edge.spec.name))\n return ret\n\n context = ConditionContext()\n context.source = ConstraintOrigin.append_type_suffix(\n dspec.parent.name, ConstraintOrigin.CONDITIONAL_SPEC\n )\n # Default is to remove node-like attrs, override here\n context.transform_required = virtual_handler\n context.transform_imposed = identity_for_facts\n\n try:\n subcondition_id = self.condition(\n dspec.when,\n spack.spec.Spec(dspec.format(unconditional=True)),\n required_name=dspec.parent.name,\n context=context,\n msg=f\"Conditional dependency in ^[when={dspec.when}]{dspec.spec}\",\n )\n self.gen.fact(fn.subcondition(subcondition_id, condition_id))\n except vt.UnknownVariantError as e:\n # A variant in the 'when=' condition can't apply to the parent of the edge\n tty.debug(f\"[{__name__}] cannot emit subcondition for {dspec.format()}: {e}\")\n\n def validate_and_define_versions_from_requirements(\n self, *, allow_deprecated: bool, require_checksum: bool\n ):\n \"\"\"If package requirements mention concrete versions that are not mentioned\n elsewhere, then we need to collect those to mark them as possible\n versions. If they are abstract and statically have no match, then we\n need to throw an error. This function assumes all possible versions are already\n registered in self.possible_versions.\"\"\"\n for pkg_name, d in spack.config.CONFIG.get_config(\"packages\").items():\n if pkg_name == \"all\" or \"require\" not in d:\n continue\n\n for s in traverse.traverse_nodes(self._specs_from_requires(pkg_name, d[\"require\"])):\n name, versions = s.name, s.versions\n\n if name not in self.pkgs or versions == vn.any_version:\n continue\n\n s.attach_git_version_lookup()\n v = versions.concrete\n\n if not v:\n # If the version is not concrete, check it's statically concretizable. If\n # not throw an error, which is just so that users know they need to change\n # their config, instead of getting a hard to decipher concretization error.\n if not any(x for x in self.possible_versions[name] if x.satisfies(versions)):\n raise spack.error.ConfigError(\n f\"Version requirement {versions} on {pkg_name} for {name} \"\n f\"cannot match any known version from package.py or externals\"\n )\n continue\n\n if v in self.possible_versions[name]:\n continue\n\n if not allow_deprecated and v in self.deprecated_versions[name]:\n continue\n\n # If concrete an not yet defined, conditionally define it, like we do for specs\n # from the command line.\n if not require_checksum or _is_checksummed_git_version(v):\n self.possible_versions[name][v].append(Provenance.PACKAGE_REQUIREMENT)\n\n def _specs_from_requires(self, pkg_name, section):\n \"\"\"Collect specs from a requirement rule\"\"\"\n if isinstance(section, str):\n yield _spec_with_default_name(section, pkg_name)\n return\n\n for spec_group in section:\n if isinstance(spec_group, str):\n yield _spec_with_default_name(spec_group, pkg_name)\n continue\n\n # Otherwise it is an object. The object can contain a single\n # \"spec\" constraint, or a list of them with \"any_of\" or\n # \"one_of\" policy.\n if \"spec\" in spec_group:\n yield _spec_with_default_name(spec_group[\"spec\"], pkg_name)\n continue\n\n key = \"one_of\" if \"one_of\" in spec_group else \"any_of\"\n for s in spec_group[key]:\n yield _spec_with_default_name(s, pkg_name)\n\n def pkg_class(self, pkg_name: str) -> Type[spack.package_base.PackageBase]:\n request = pkg_name\n if pkg_name in self.explicitly_required_namespaces:\n namespace = self.explicitly_required_namespaces[pkg_name]\n request = f\"{namespace}.{pkg_name}\"\n return spack.repo.PATH.get_pkg_class(request)\n\n\nclass _Head:\n \"\"\"ASP functions used to express spec clauses in the HEAD of a rule\"\"\"\n\n node = fn.attr(\"node\")\n namespace = fn.attr(\"namespace_set\")\n virtual_node = fn.attr(\"virtual_node\")\n node_platform = fn.attr(\"node_platform_set\")\n node_os = fn.attr(\"node_os_set\")\n node_target = fn.attr(\"node_target_set\")\n variant_value = fn.attr(\"variant_set\")\n node_flag = fn.attr(\"node_flag_set\")\n propagate = fn.attr(\"propagate\")\n\n\nclass _Body:\n \"\"\"ASP functions used to express spec clauses in the BODY of a rule\"\"\"\n\n node = fn.attr(\"node\")\n namespace = fn.attr(\"namespace\")\n virtual_node = fn.attr(\"virtual_node\")\n node_platform = fn.attr(\"node_platform\")\n node_os = fn.attr(\"node_os\")\n node_target = fn.attr(\"node_target\")\n variant_value = fn.attr(\"variant_value\")\n node_flag = fn.attr(\"node_flag\")\n propagate = fn.attr(\"propagate\")\n\n\ndef strip_asp_problem(asp_problem: Iterable[str]) -> List[str]:\n \"\"\"Remove comments and empty lines from an ASP program.\"\"\"\n\n def strip_statement(stmt: str) -> str:\n lines = [line for line in stmt.split(\"\\n\") if not line.startswith(\"%\")]\n return \"\".join(line.strip() for line in lines if line)\n\n value = [strip_statement(stmt) for stmt in asp_problem]\n value = [s for s in value if s]\n\n return value\n\n\nclass ProblemInstanceBuilder:\n \"\"\"Provides an interface to construct a problem instance.\n\n Once all the facts and rules have been added, the problem instance can be retrieved with:\n\n >>> builder = ProblemInstanceBuilder()\n >>> ...\n >>> problem_instance = builder.value()\n\n The problem instance can be added directly to the \"control\" structure of clingo.\n\n \"\"\"\n\n def __init__(self) -> None:\n self.asp_problem: List[str] = []\n\n def fact(self, atom: AspFunction) -> None:\n self.asp_problem.append(f\"{atom}.\")\n\n def append(self, rule: str) -> None:\n self.asp_problem.append(rule)\n\n def title(self, header: str, char: str) -> None:\n sep = char * 76\n self.newline()\n self.asp_problem.append(f\"%{sep}\")\n self.asp_problem.append(f\"% {header}\")\n self.asp_problem.append(f\"%{sep}\")\n\n def h1(self, header: str) -> None:\n self.title(header, \"=\")\n\n def h2(self, header: str) -> None:\n self.title(header, \"-\")\n\n def h3(self, header: str):\n self.asp_problem.append(f\"% {header}\")\n\n def newline(self):\n self.asp_problem.append(\"\")\n\n\ndef possible_compilers(*, configuration) -> Tuple[Set[\"spack.spec.Spec\"], Set[\"spack.spec.Spec\"]]:\n result, rejected = set(), set()\n\n # Compilers defined in configuration\n for c in spack.compilers.config.all_compilers_from(configuration):\n if using_libc_compatibility() and not c_compiler_runs(c):\n rejected.add(c)\n try:\n compiler = c.extra_attributes[\"compilers\"][\"c\"]\n tty.debug(\n f\"the C compiler {compiler} does not exist, or does not run correctly.\"\n f\" The compiler {c} will not be used during concretization.\"\n )\n except KeyError:\n tty.debug(f\"the spec {c} does not provide a C compiler.\")\n\n continue\n\n if using_libc_compatibility() and not CompilerPropertyDetector(c).default_libc():\n rejected.add(c)\n warnings.warn(\n f\"cannot detect libc from {c}. The compiler will not be used \"\n f\"during concretization.\"\n )\n continue\n\n if c in result:\n tty.debug(f\"[{__name__}] duplicate {c.long_spec} compiler found\")\n continue\n\n result.add(c)\n\n # Compilers from the local store\n supported_compilers = spack.compilers.config.supported_compilers()\n for pkg_name in supported_compilers:\n result.update(spack.store.STORE.db.query(pkg_name))\n\n return result, rejected\n\n\nFunctionTupleT = Tuple[str, Tuple[Union[str, NodeId], ...]]\n\n\nclass SpecBuilder:\n \"\"\"Class with actions to rebuild a spec from ASP results.\"\"\"\n\n #: Regex for attributes that don't need actions b/c they aren't used to construct specs.\n ignored_attributes = re.compile(\n \"|\".join(\n [\n r\"^.*_propagate$\",\n r\"^.*_satisfies$\",\n r\"^.*_set$\",\n r\"^compatible_libc$\",\n r\"^dependency_holds$\",\n r\"^package_hash$\",\n r\"^root$\",\n r\"^uses_virtual$\",\n r\"^variant_default_value_from_cli$\",\n r\"^virtual_node$\",\n r\"^virtual_on_incoming_edges$\",\n r\"^virtual_root$\",\n ]\n )\n )\n\n @staticmethod\n def make_node(*, pkg: str) -> NodeId:\n \"\"\"Given a package name, returns the string representation of the \"min_dupe_id\" node in\n the ASP encoding.\n\n Args:\n pkg: name of a package\n \"\"\"\n return NodeId(id=\"0\", pkg=pkg)\n\n def __init__(self, specs, hash_lookup=None):\n self._specs: Dict[NodeId, spack.spec.Spec] = {}\n\n # Matches parent nodes to splice node\n self._splices: Dict[spack.spec.Spec, List[spack.solver.splicing.Splice]] = {}\n self._result = None\n self._command_line_specs = specs\n self._flag_sources: Dict[Tuple[NodeId, str], Set[str]] = collections.defaultdict(\n lambda: set()\n )\n\n # Pass in as arguments reusable specs and plug them in\n # from this dictionary during reconstruction\n self._hash_lookup = hash_lookup or ConcreteSpecsByHash()\n\n def hash(self, node, h):\n if node not in self._specs:\n self._specs[node] = self._hash_lookup[h]\n\n def node(self, node):\n if node not in self._specs:\n self._specs[node] = spack.spec.Spec(node.pkg)\n for flag_type in spack.spec.FlagMap.valid_compiler_flags():\n self._specs[node].compiler_flags[flag_type] = []\n\n def _arch(self, node):\n arch = self._specs[node].architecture\n if not arch:\n arch = spack.spec.ArchSpec()\n self._specs[node].architecture = arch\n return arch\n\n def namespace(self, node, namespace):\n self._specs[node].namespace = namespace\n\n def node_platform(self, node, platform):\n self._arch(node).platform = platform\n\n def node_os(self, node, os):\n self._arch(node).os = os\n\n def node_target(self, node, target):\n self._arch(node).target = target\n\n def variant_selected(self, node, name: str, value: str, variant_type: str, variant_id):\n spec = self._specs[node]\n variant = spec.variants.get(name)\n if not variant:\n spec.variants[name] = vt.VariantValue.from_concretizer(name, value, variant_type)\n else:\n assert variant_type == \"multi\", (\n f\"Can't have multiple values for single-valued variant: \"\n f\"{node}, {name}, {value}, {variant_type}, {variant_id}\"\n )\n variant.append(value)\n\n def version(self, node, version):\n self._specs[node].versions = vn.VersionList([vn.Version(version)])\n\n def node_flag(self, node, node_flag):\n self._specs[node].compiler_flags.add_flag(\n node_flag.flag_type, node_flag.flag, False, node_flag.flag_group, node_flag.source\n )\n\n def depends_on(self, parent_node, dependency_node, type):\n dependency_spec = self._specs[dependency_node]\n depflag = dt.flag_from_string(type)\n self._specs[parent_node].add_dependency_edge(dependency_spec, depflag=depflag, virtuals=())\n\n def virtual_on_edge(self, parent_node, provider_node, virtual):\n dependencies = self._specs[parent_node].edges_to_dependencies(name=(provider_node.pkg))\n provider_spec = self._specs[provider_node]\n dependencies = [x for x in dependencies if id(x.spec) == id(provider_spec)]\n assert len(dependencies) == 1, f\"{virtual}: {provider_node.pkg}\"\n dependencies[0].update_virtuals(virtual)\n\n def reorder_flags(self):\n \"\"\"For each spec, determine the order of compiler flags applied to it.\n\n The solver determines which flags are on nodes; this routine\n imposes order afterwards. The order is:\n\n 1. Flags applied in compiler definitions should come first\n 2. Flags applied by dependents are ordered topologically (with a\n dependency on ``traverse`` to resolve the partial order into a\n stable total order)\n 3. Flags from requirements are then applied (requirements always\n come from the package and never a parent)\n 4. Command-line flags should come last\n\n Additionally, for each source (requirements, compiler, command line, and\n dependents), flags from that source should retain their order and grouping:\n e.g. for ``y cflags=\"-z -a\"`` ``-z`` and ``-a`` should never have any intervening\n flags inserted, and should always appear in that order.\n \"\"\"\n for node, spec in self._specs.items():\n # if bootstrapping, compiler is not in config and has no flags\n flagmap_from_compiler = {\n flag_type: [x for x in values if x.source == \"compiler\"]\n for flag_type, values in spec.compiler_flags.items()\n }\n\n flagmap_from_cli = {}\n for flag_type, values in spec.compiler_flags.items():\n if not values:\n continue\n\n flags = [x for x in values if x.source == \"literal\"]\n if not flags:\n continue\n\n # For compiler flags from literal specs, reorder any flags to\n # the input order from flag.flag_group\n flagmap_from_cli[flag_type] = _reorder_flags(flags)\n\n for flag_type in spec.compiler_flags.valid_compiler_flags():\n ordered_flags = []\n\n # 1. Put compiler flags first\n from_compiler = tuple(flagmap_from_compiler.get(flag_type, []))\n extend_flag_list(ordered_flags, from_compiler)\n\n # 2. Add all sources (the compiler is one of them, so skip any\n # flag group that matches it exactly)\n flag_groups = set()\n for flag in self._specs[node].compiler_flags.get(flag_type, []):\n flag_groups.add(\n spack.spec.CompilerFlag(\n flag.flag_group,\n propagate=flag.propagate,\n flag_group=flag.flag_group,\n source=flag.source,\n )\n )\n\n # For flags that are applied by dependents, put flags from parents\n # before children; we depend on the stability of traverse() to\n # achieve a stable flag order for flags introduced in this manner.\n topo_order = list(s.name for s in spec.traverse(order=\"post\", direction=\"parents\"))\n lex_order = list(sorted(flag_groups))\n\n def _order_index(flag_group):\n source = flag_group.source\n # Note: if 'require: ^dependency cflags=...' is ever possible,\n # this will topologically sort for require as well\n type_index, pkg_source = ConstraintOrigin.strip_type_suffix(source)\n if pkg_source in topo_order:\n major_index = topo_order.index(pkg_source)\n # If for x->y, x has multiple depends_on declarations that\n # are activated, and each adds cflags to y, we fall back on\n # alphabetical ordering to maintain a total order\n minor_index = lex_order.index(flag_group)\n else:\n major_index = len(topo_order) + lex_order.index(flag_group)\n minor_index = 0\n return (type_index, major_index, minor_index)\n\n prioritized_groups = sorted(flag_groups, key=lambda x: _order_index(x))\n\n for grp in prioritized_groups:\n grp_flags = tuple(\n x for (x, y) in spack.compilers.flags.tokenize_flags(grp.flag_group)\n )\n if grp_flags == from_compiler:\n continue\n as_compiler_flags = list(\n spack.spec.CompilerFlag(\n x,\n propagate=grp.propagate,\n flag_group=grp.flag_group,\n source=grp.source,\n )\n for x in grp_flags\n )\n extend_flag_list(ordered_flags, as_compiler_flags)\n\n # 3. Now put cmd-line flags last\n if flag_type in flagmap_from_cli:\n extend_flag_list(ordered_flags, flagmap_from_cli[flag_type])\n\n compiler_flags = spec.compiler_flags.get(flag_type, [])\n msg = f\"{set(compiler_flags)} does not equal {set(ordered_flags)}\"\n assert set(compiler_flags) == set(ordered_flags), msg\n\n spec.compiler_flags.update({flag_type: ordered_flags})\n\n def deprecated(self, node: NodeId, version: str) -> None:\n tty.warn(f'using \"{node.pkg}@{version}\" which is a deprecated version')\n\n def splice_at_hash(\n self, parent_node: NodeId, splice_node: NodeId, child_name: str, child_hash: str\n ):\n parent_spec = self._specs[parent_node]\n splice_spec = self._specs[splice_node]\n splice = spack.solver.splicing.Splice(\n splice_spec, child_name=child_name, child_hash=child_hash\n )\n self._splices.setdefault(parent_spec, []).append(splice)\n\n def build_specs(self, function_tuples: List[FunctionTupleT]) -> List[spack.spec.Spec]:\n # TODO: remove this local import and get rid of dependency on globals\n import spack.environment as ev\n\n attr_key = {\n # hash attributes are handled first, since they imply entire concrete specs\n \"hash\": -5,\n # node attributes are handled next, since they instantiate nodes\n \"node\": -4,\n # evaluated last, so all nodes are fully constructed\n \"virtual_on_edge\": 2,\n }\n\n # Sort functions so that directives building objects are called in the right order\n function_tuples.sort(key=lambda x: attr_key.get(x[0], 0))\n self._specs = {}\n for name, args in function_tuples:\n if SpecBuilder.ignored_attributes.match(name):\n continue\n\n action = getattr(self, name, None)\n\n # print out unknown actions so we can display them for debugging\n if not action:\n msg = f'UNKNOWN SYMBOL: attr(\"{name}\", {\", \".join(str(a) for a in args)})'\n tty.debug(msg)\n continue\n\n msg = (\n \"Internal Error: Uncallable action found in asp.py. Please report to the spack\"\n \" maintainers.\"\n )\n assert action and callable(action), msg\n\n # ignore predicates on virtual packages, as they're used for\n # solving but don't construct anything. Do not ignore error\n # predicates on virtual packages.\n if name != \"error\":\n node = args[0]\n assert isinstance(node, NodeId), (\n f\"internal solver error: expected a node, but got a {type(args[0])}. \"\n \"Please report a bug at https://github.com/spack/spack/issues\"\n )\n\n pkg = node.pkg\n if spack.repo.PATH.is_virtual(pkg):\n continue\n\n # if we've already gotten a concrete spec for this pkg, we're done, unless\n # we're splicing. `splice_at_hash()` is the only action we call on concrete specs.\n spec = self._specs.get(node)\n if spec and spec.concrete and name != \"splice_at_hash\":\n continue\n\n action(*args)\n\n # fix flags after all specs are constructed\n self.reorder_flags()\n\n # inject patches -- note that we' can't use set() to unique the\n # roots here, because the specs aren't complete, and the hash\n # function will loop forever.\n roots = [spec.root for spec in self._specs.values()]\n roots = dict((id(r), r) for r in roots)\n for root in roots.values():\n spack.spec._inject_patches_variant(root)\n\n # Add external paths to specs with just external modules\n for s in self._specs.values():\n _ensure_external_path_if_external(s)\n\n for s in self._specs.values():\n _develop_specs_from_env(s, ev.active_environment())\n\n # check for commits must happen after all version adaptations are complete\n for s in self._specs.values():\n _specs_with_commits(s)\n\n # mark concrete and assign hashes to all specs in the solve\n for root in roots.values():\n root._finalize_concretization()\n\n # Unify hashes (this is to avoid duplicates of runtimes and compilers)\n unifier = ConcreteSpecsByHash()\n keys = list(self._specs)\n for key in keys:\n current_spec = self._specs[key]\n unifier.add(current_spec)\n self._specs[key] = unifier[current_spec.dag_hash()]\n\n # Only attempt to resolve automatic splices if the solver produced any\n if self._splices:\n resolved_splices = spack.solver.splicing._resolve_collected_splices(\n list(self._specs.values()), self._splices\n )\n new_specs = {}\n for node, spec in self._specs.items():\n new_specs[node] = resolved_splices.get(spec, spec)\n self._specs = new_specs\n\n for s in self._specs.values():\n spack.spec.Spec.ensure_no_deprecated(s)\n\n # Add git version lookup info to concrete Specs (this is generated for\n # abstract specs as well but the Versions may be replaced during the\n # concretization process)\n for root in self._specs.values():\n for spec in root.traverse():\n if isinstance(spec.version, vn.GitVersion):\n spec.version.attach_lookup(\n spack.version.git_ref_lookup.GitRefLookup(spec.fullname)\n )\n\n specs = self.execute_explicit_splices()\n return specs\n\n def execute_explicit_splices(self):\n splice_config = spack.config.CONFIG.get(\"concretizer:splice:explicit\", [])\n splice_triples = []\n for splice_set in splice_config:\n target = splice_set[\"target\"]\n replacement = spack.spec.Spec(splice_set[\"replacement\"])\n\n if not replacement.abstract_hash:\n location = getattr(\n splice_set[\"replacement\"], \"_start_mark\", \" at unknown line number\"\n )\n msg = f\"Explicit splice replacement '{replacement}' does not include a hash.\\n\"\n msg += f\"{location}\\n\\n\"\n msg += \" Splice replacements must be specified by hash\"\n raise InvalidSpliceError(msg)\n\n transitive = splice_set.get(\"transitive\", False)\n splice_triples.append((target, replacement, transitive))\n\n specs = {}\n for key, spec in self._specs.items():\n current_spec = spec\n for target, replacement, transitive in splice_triples:\n if target in current_spec:\n # matches root or non-root\n # e.g. mvapich2%gcc\n\n # The first iteration, we need to replace the abstract hash\n if not replacement.concrete:\n replacement.replace_hash()\n current_spec = current_spec.splice(replacement, transitive)\n new_key = NodeId(id=key.id, pkg=current_spec.name)\n specs[new_key] = current_spec\n\n return specs\n\n\ndef _specs_with_commits(spec):\n pkg_class = spack.repo.PATH.get_pkg_class(spec.fullname)\n if not pkg_class.needs_commit(spec.version):\n return\n\n if isinstance(spec.version, vn.GitVersion):\n if \"commit\" not in spec.variants and spec.version.commit_sha:\n spec.variants[\"commit\"] = vt.SingleValuedVariant(\"commit\", spec.version.commit_sha)\n\n pkg_class._resolve_git_provenance(spec)\n\n if \"commit\" not in spec.variants:\n if not spec.is_develop:\n tty.warn(\n f\"Unable to resolve the git commit for {spec.name}. \"\n \"An installation of this binary won't have complete binary provenance.\"\n )\n return\n\n # check integrity of user specified commit shas\n invalid_commit_msg = (\n f\"Internal Error: {spec.name}'s assigned commit {spec.variants['commit'].value}\"\n \" does not meet commit syntax requirements.\"\n )\n assert vn.is_git_commit_sha(spec.variants[\"commit\"].value), invalid_commit_msg\n\n\ndef _ensure_external_path_if_external(spec: spack.spec.Spec) -> None:\n if not spec.external_modules or spec.external_path:\n return\n\n # Get the path from the module the package can override the default\n # (this is mostly needed for Cray)\n pkg_cls = spack.repo.PATH.get_pkg_class(spec.name)\n package = pkg_cls(spec)\n spec.external_path = getattr(package, \"external_prefix\", None) or md.path_from_modules(\n spec.external_modules\n )\n\n\ndef _develop_specs_from_env(spec, env):\n dev_info = env.dev_specs.get(spec.name, {}) if env else {}\n if not dev_info:\n return\n\n path = spack.util.path.canonicalize_path(dev_info[\"path\"], default_wd=env.path)\n\n if \"dev_path\" in spec.variants:\n error_msg = (\n \"Internal Error: The dev_path for spec {name} is not connected to a valid environment\"\n \"path. Please note that develop specs can only be used inside an environment\"\n \"These paths should be the same:\\n\\tdev_path:{dev_path}\\n\\tenv_based_path:{env_path}\"\n ).format(name=spec.name, dev_path=spec.variants[\"dev_path\"], env_path=path)\n\n assert spec.variants[\"dev_path\"].value == path, error_msg\n else:\n spec.variants.setdefault(\"dev_path\", vt.SingleValuedVariant(\"dev_path\", path))\n\n assert spec.satisfies(dev_info[\"spec\"])\n\n\nclass Solver:\n \"\"\"This is the main external interface class for solving.\n\n It manages solver configuration and preferences in one place. It sets up the solve\n and passes the setup method to the driver, as well.\n \"\"\"\n\n def __init__(self, *, specs_factory: Optional[SpecFiltersFactory] = None):\n # Compute possible compilers first, so we see them as externals\n _ = spack.compilers.config.all_compilers(init_config=True)\n\n self._conc_cache = ConcretizationCache()\n self.driver = PyclingoDriver(conc_cache=self._conc_cache)\n\n # Compute packages configuration with implicit externals once and reuse it\n self.packages_with_externals = external_config_with_implicit_externals(spack.config.CONFIG)\n completion_mode = spack.config.CONFIG.get(\"concretizer:externals:completion\")\n self.selector = ReusableSpecsSelector(\n configuration=spack.config.CONFIG,\n external_parser=create_external_parser(self.packages_with_externals, completion_mode),\n factory=specs_factory,\n packages_with_externals=self.packages_with_externals,\n )\n\n @staticmethod\n def _check_input_and_extract_concrete_specs(\n specs: Sequence[spack.spec.Spec],\n ) -> List[spack.spec.Spec]:\n reusable: List[spack.spec.Spec] = []\n analyzer = create_graph_analyzer()\n for root in specs:\n for s in root.traverse():\n if s.concrete:\n reusable.append(s)\n else:\n if spack.repo.PATH.is_virtual(s.name):\n continue\n # Error if direct dependencies cannot be satisfied\n deps = {\n edge.spec.name\n for edge in s.edges_to_dependencies()\n if edge.direct and edge.when == EMPTY_SPEC\n }\n if deps:\n graph = analyzer.possible_dependencies(\n s, allowed_deps=dt.ALL, transitive=False\n )\n deps.difference_update(graph.real_pkgs, graph.virtuals)\n if deps:\n start_str = f\"'{root}'\" if s == root else f\"'{s}' in '{root}'\"\n raise UnsatisfiableSpecError(\n f\"{start_str} cannot depend on {', '.join(deps)}\"\n )\n\n try:\n spack.repo.PATH.get_pkg_class(s.fullname)\n except spack.repo.UnknownPackageError:\n raise UnsatisfiableSpecError(\n f\"cannot concretize '{root}', since '{s.name}' does not exist\"\n )\n\n spack.spec.Spec.ensure_valid_variants(s)\n return reusable\n\n def solve_with_stats(\n self,\n specs: Sequence[spack.spec.Spec],\n out: Optional[io.IOBase] = None,\n timers: bool = False,\n stats: bool = False,\n tests: spack.concretize.TestsType = False,\n setup_only: bool = False,\n allow_deprecated: bool = False,\n ) -> Tuple[Result, Optional[spack.util.timer.Timer], Optional[Dict]]:\n \"\"\"\n Concretize a set of specs and track the timing and statistics for the solve\n\n Arguments:\n specs: List of ``Spec`` objects to solve for.\n out: Optionally write the generate ASP program to a file-like object.\n timers: Print out coarse timers for different solve phases.\n stats: Print out detailed stats from clingo.\n tests: If True, concretize test dependencies for all packages.\n If a tuple of package names, concretize test dependencies for named\n packages (defaults to False: do not concretize test dependencies).\n setup_only: if True, stop after setup and don't solve (default False).\n allow_deprecated: allow deprecated version in the solve\n \"\"\"\n specs = [s.lookup_hash() for s in specs]\n reusable_specs = self._check_input_and_extract_concrete_specs(specs)\n reusable_specs.extend(self.selector.reusable_specs(specs))\n setup = SpackSolverSetup(tests=tests)\n output = OutputConfiguration(timers=timers, stats=stats, out=out, setup_only=setup_only)\n\n result = self.driver.solve(\n setup,\n specs,\n reuse=reusable_specs,\n packages_with_externals=self.packages_with_externals,\n output=output,\n allow_deprecated=allow_deprecated,\n )\n self._conc_cache.cleanup()\n return result\n\n def solve(self, specs: Sequence[spack.spec.Spec], **kwargs) -> Result:\n \"\"\"\n Convenience function for concretizing a set of specs and ignoring timing\n and statistics. Uses the same kwargs as solve_with_stats.\n \"\"\"\n # Check upfront that the variants are admissible\n result, _, _ = self.solve_with_stats(specs, **kwargs)\n return result\n\n def solve_in_rounds(\n self,\n specs: Sequence[spack.spec.Spec],\n out: Optional[io.IOBase] = None,\n timers: bool = False,\n stats: bool = False,\n tests: spack.concretize.TestsType = False,\n allow_deprecated: bool = False,\n ) -> Generator[Result, None, None]:\n \"\"\"Solve for a stable model of specs in multiple rounds.\n\n This relaxes the assumption of solve that everything must be consistent and\n solvable in a single round. Each round tries to maximize the reuse of specs\n from previous rounds.\n\n The function is a generator that yields the result of each round.\n\n Arguments:\n specs (list): list of Specs to solve.\n out: Optionally write the generate ASP program to a file-like object.\n timers (bool): print timing if set to True\n stats (bool): print internal statistics if set to True\n tests (bool): add test dependencies to the solve\n allow_deprecated (bool): allow deprecated version in the solve\n \"\"\"\n specs = [s.lookup_hash() for s in specs]\n reusable_specs = self._check_input_and_extract_concrete_specs(specs)\n reusable_specs.extend(self.selector.reusable_specs(specs))\n setup = SpackSolverSetup(tests=tests)\n\n # Tell clingo that we don't have to solve all the inputs at once\n setup.concretize_everything = False\n\n input_specs = specs\n output = OutputConfiguration(timers=timers, stats=stats, out=out, setup_only=False)\n while True:\n result, _, _ = self.driver.solve(\n setup,\n input_specs,\n reuse=reusable_specs,\n packages_with_externals=self.packages_with_externals,\n output=output,\n allow_deprecated=allow_deprecated,\n )\n yield result\n\n # If we don't have unsolved specs we are done\n if not result.unsolved_specs:\n break\n\n if not result.specs:\n # This is also a problem: no specs were solved for, which means we would be in a\n # loop if we tried again\n raise OutputDoesNotSatisfyInputError(result.unsolved_specs)\n\n input_specs = list(x for (x, y) in result.unsolved_specs)\n for spec in result.specs:\n reusable_specs.extend(spec.traverse())\n\n self._conc_cache.cleanup()\n\n\nclass UnsatisfiableSpecError(spack.error.UnsatisfiableSpecError):\n \"\"\"There was an issue with the spec that was requested (i.e. a user error).\"\"\"\n\n def __init__(self, msg):\n super(spack.error.UnsatisfiableSpecError, self).__init__(msg)\n self.provided = None\n self.required = None\n self.constraint_type = None\n\n\nclass InternalConcretizerError(spack.error.UnsatisfiableSpecError):\n \"\"\"Errors that indicate a bug in Spack.\"\"\"\n\n def __init__(self, msg):\n super(spack.error.UnsatisfiableSpecError, self).__init__(msg)\n self.provided = None\n self.required = None\n self.constraint_type = None\n\n\nclass OutputDoesNotSatisfyInputError(InternalConcretizerError):\n def __init__(\n self, input_to_output: List[Tuple[spack.spec.Spec, Optional[spack.spec.Spec]]]\n ) -> None:\n self.input_to_output = input_to_output\n super().__init__(\n \"internal solver error: the solver completed but produced specs\"\n \" that do not satisfy the request. Please report a bug at \"\n f\"https://github.com/spack/spack/issues\\n\\t{Result.format_unsolved(input_to_output)}\"\n )\n\n\nclass SolverError(InternalConcretizerError):\n \"\"\"For cases where the solver is unable to produce a solution.\n\n Such cases are unexpected because we allow for solutions with errors,\n so for example user specs that are over-constrained should still\n get a solution.\n \"\"\"\n\n def __init__(self, provided):\n msg = (\n \"Spack concretizer internal error. Please submit a bug report at \"\n \"https://github.com/spack/spack and include the command and environment \"\n \"if applicable.\"\n f\"\\n {provided} is unsatisfiable\"\n )\n\n super().__init__(msg)\n\n # Add attribute expected of the superclass interface\n self.required = None\n self.constraint_type = None\n self.provided = provided\n\n\nclass InvalidSpliceError(spack.error.SpackError):\n \"\"\"For cases in which the splice configuration is invalid.\"\"\"\n\n\nclass NoCompilerFoundError(spack.error.SpackError):\n \"\"\"Raised when there is no possible compiler\"\"\"\n\n\nclass InvalidExternalError(spack.error.SpackError):\n \"\"\"Raised when there is no possible compiler\"\"\"\n\n\nclass DeprecatedVersionError(spack.error.SpackError):\n \"\"\"Raised when user directly requests a deprecated version.\"\"\"\n\n\nclass InvalidVersionError(spack.error.SpackError):\n \"\"\"Raised when a version can't be satisfied by any possible versions.\"\"\"\n\n\nclass InvalidDependencyError(spack.error.SpackError):\n \"\"\"Raised when an explicit dependency is not a possible dependency.\"\"\"\n" }, { "path": "lib/spack/spack/solver/concretize.lp", "content": "% Copyright Spack Project Developers. See COPYRIGHT file for details.\n%\n% SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n%=============================================================================\n% This logic program implements Spack's concretizer\n%=============================================================================\n\n% ID of the nodes in the \"root\" link-run sub-DAG\n#const min_dupe_id = 0.\n\n#const direct_link_run = 0.\n#const direct_build = 1.\n\n% Allow clingo to create nodes\n{ attr(\"node\", node(0..X-1, Package)) } :- max_dupes(Package, X), not virtual(Package).\n{ attr(\"virtual_node\", node(0..X-1, Package)) } :- max_dupes(Package, X), virtual(Package).\n\n%%%%\n% Rules to prevent symmetry on duplicates\n%%%%\n\nduplicate_penalty(node(ID1, Package), 1, \"version\")\n :- attr(\"node\", node(ID1, Package)), attr(\"node\", node(ID2, Package)),\n version_weight(node(ID1, Package), Weight1), version_weight(node(ID2, Package), Weight2),\n multiple_unification_sets(Package),\n ID2 = ID1 + 1, Weight2 < Weight1,\n max_dupes(Package, X), X > 1, ID2 < X.\n\n% We can enforce this, and hope the grounder generates fewer rules\n:- provider_weight(ProviderNode1, node(ID1, Virtual), Weight1),\n provider_weight(ProviderNode2, node(ID2, Virtual), Weight2),\n ID2 = ID1 + 1, Weight2 < Weight1,\n max_dupes(Virtual, X), X > 1, ID2 < X.\n\n% Integrity constraints on DAG nodes\n:- attr(\"root\", PackageNode), not attr(\"node\", PackageNode).\n:- attr(\"version\", PackageNode, _), not attr(\"node\", PackageNode), not attr(\"virtual_node\", PackageNode).\n:- attr(\"node_version_satisfies\", PackageNode, _), not attr(\"node\", PackageNode), not attr(\"virtual_node\", PackageNode).\n:- attr(\"hash\", PackageNode, _), not attr(\"node\", PackageNode).\n:- attr(\"node_platform\", PackageNode, _), not attr(\"node\", PackageNode).\n:- attr(\"node_os\", PackageNode, _), not attr(\"node\", PackageNode).\n:- attr(\"node_target\", PackageNode, _), not attr(\"node\", PackageNode).\n:- attr(\"variant_value\", PackageNode, _, _), not attr(\"node\", PackageNode).\n:- attr(\"node_flag\", PackageNode, _), not attr(\"node\", PackageNode).\n:- attr(\"depends_on\", ParentNode, _, _), not attr(\"node\", ParentNode).\n:- attr(\"depends_on\", _, ChildNode, _), not attr(\"node\", ChildNode).\n:- attr(\"virtual_node\", VirtualNode), not provider(_, VirtualNode).\n:- provider(_, VirtualNode), not attr(\"virtual_node\", VirtualNode).\n:- provider(PackageNode, _), not attr(\"node\", PackageNode).\n:- node_has_variant(PackageNode, _, _), not attr(\"node\", PackageNode).\n:- attr(\"variant_set\", PackageNode, _, _), not attr(\"node\", PackageNode).\n:- variant_is_propagated(PackageNode, _), not attr(\"node\", PackageNode).\n\n:- attr(\"root\", node(ID, PackageNode)), ID > min_dupe_id.\n\n% Nodes in the \"root\" unification set cannot depend on non-root nodes if the dependency is \"link\" or \"run\"\n:- attr(\"depends_on\", node(min_dupe_id, Package), node(ID, _), \"link\"), ID != min_dupe_id, unification_set(\"root\", node(min_dupe_id, Package)).\n:- attr(\"depends_on\", node(min_dupe_id, Package), node(ID, _), \"run\"), ID != min_dupe_id, unification_set(\"root\", node(min_dupe_id, Package)).\n\n% Namespaces are statically assigned by a package fact if not otherwise set\nerror(100, \"{0} does not have a namespace\", Package) :- attr(\"node\", node(ID, Package)),\n not attr(\"namespace\", node(ID, Package), _).\n\nerror(100, \"{0} cannot come from both {1} and {2} namespaces\", Package, NS1, NS2) :- attr(\"node\", node(ID, Package)),\n attr(\"namespace\", node(ID, Package), NS1),\n attr(\"namespace\", node(ID, Package), NS2),\n NS1 < NS2.\n\nattr(\"namespace\", node(ID, Package), Namespace) :- attr(\"namespace_set\", node(ID, Package), Namespace).\nattr(\"namespace\", node(ID, Package), Namespace)\n :- attr(\"node\", node(ID, Package)),\n not attr(\"namespace_set\", node(ID, Package), _),\n pkg_fact(Package, namespace(Namespace)).\n\n% Rules on \"unification sets\", i.e. on sets of nodes allowing a single configuration of any given package\nunify(SetID, PackageName) :- unification_set(SetID, node(_, PackageName)).\n\nerror(100000, \"Cannot have multiple nodes for {0} in the same unification set {1}\", PackageName, SetID)\n :- 2 { unification_set(SetID, node(_, PackageName)) }, unify(SetID, PackageName).\n\nunification_set(\"root\", PackageNode) :- attr(\"root\", PackageNode).\nunification_set(\"root\", PackageNode) :- attr(\"virtual_root\", PackageNode).\n\n% A package with a dependency that *is not* build belongs to the same unification set as the parent\nunification_set(SetID, ChildNode) :- attr(\"depends_on\", ParentNode, ChildNode, Type), Type != \"build\", unification_set(SetID, ParentNode).\n\n% A separate unification set can be created if any of the build dependencies can be duplicated\nneeds_build_unification_set(ParentNode) :- attr(\"depends_on\", ParentNode, _, \"build\").\n\n% If any of the build dependencies can be duplicated, they can go into any (\"build\", ID) set\nunification_set((\"build\", ID), node(X, Child))\n :- attr(\"depends_on\", ParentNode, node(X, Child), \"build\"),\n build_set_id(ParentNode, ID),\n needs_build_unification_set(ParentNode).\n\n% A virtual that is on an edge of non-build type belongs to the same unification set as the parent of the provider\nunification_set(SetID, VirtualNode) :-\n virtual_on_edge(ParentNode, ProviderNode, VirtualNode, Type), Type != \"build\",\n unification_set(SetID, ParentNode).\n\n% A virtual that is on a build edge, goes in the build set id of the parent of the provider\nunification_set((\"build\", ID), VirtualNode) :-\n virtual_on_edge(ParentNode, ProviderNode, VirtualNode, \"build\"),\n build_set_id(ParentNode, ID),\n needs_build_unification_set(ParentNode).\n\n% Needed for reused dependencies. A reused dependency has its build edges trimmed, so we\n% only care about the non-build edges.\nunification_set(SetID, node(VirtualID, Virtual)) :-\n concrete(ParentNode),\n attr(\"virtual_on_edge\", ParentNode, ProviderNode, Virtual),\n provider(ProviderNode, node(VirtualID, Virtual)),\n unification_set(SetID, ParentNode).\n\n% Limit the number of unification sets to a reasonable number to avoid combinatorial explosion\n#const max_build_unification_sets = 4.\n1 { build_set_id(ParentNode, 0..max_build_unification_sets-1) } 1 :- needs_build_unification_set(ParentNode).\n\n\n% Compute sub-sets of the nodes, if requested. These can be either the nodes connected\n% to another node by \"link\" edges, or the nodes connected to another node by \"link and\n% \"run\" edges.\n\ncompute_closure(node(ID, Package), \"linkrun\") :-\n condition_requirement(_, \"closure\", Package, _, \"linkrun\"),\n attr(\"node\", node(ID, Package)).\n\nattr(\"closure\", PackageNode, DependencyNode, \"linkrun\") :-\n attr(\"depends_on\", PackageNode, DependencyNode, \"link\"),\n not provider(DependencyNode, node(_, Language)) : language(Language);\n compute_closure(PackageNode, \"linkrun\").\n\nattr(\"closure\", PackageNode, DependencyNode, \"linkrun\") :-\n attr(\"depends_on\", PackageNode, DependencyNode, \"run\"),\n not provider(DependencyNode, node(_, Language)) : language(Language);\n compute_closure(PackageNode, \"linkrun\").\n\nattr(\"closure\", PackageNode, DependencyNode, \"linkrun\") :-\n attr(\"depends_on\", ParentNode, DependencyNode, \"link\"),\n not provider(DependencyNode, node(_, Language)) : language(Language);\n attr(\"closure\", PackageNode, ParentNode, \"linkrun\"),\n compute_closure(PackageNode, \"linkrun\").\n\nattr(\"closure\", PackageNode, DependencyNode, \"linkrun\") :-\n attr(\"depends_on\", ParentNode, DependencyNode, \"run\"),\n not provider(DependencyNode, node(_, Language)) : language(Language);\n attr(\"closure\", PackageNode, ParentNode, \"linkrun\"),\n compute_closure(PackageNode, \"linkrun\").\n\nrelated(NodeA, NodeB) :- attr(\"closure\", NodeA, NodeB, \"linkrun\").\n\n% Do not allow split dependencies, for now. This ensures that we don't construct graphs where e.g.\n% a python extension depends on setuptools@63.4 as a run dependency, but uses e.g. setuptools@68\n% as a build dependency.\n%\n% We'll need to relax the rule before we get to actual cross-compilation\n:- depends_on(ParentNode, node(X, Dependency)), depends_on(ParentNode, node(Y, Dependency)), X < Y.\n\n#defined multiple_unification_sets/1.\n#defined runtime/1.\n\n%----\n% Rules to break symmetry and speed-up searches\n%----\n\n% In the \"root\" unification set only ID = 0 are allowed\n:- unification_set(\"root\", node(ID, _)), ID != 0.\n\n% In the \"root\" unification set we allow only packages from the link-run possible subDAG\n:- unification_set(\"root\", node(_, Package)), not possible_in_link_run(Package), not virtual(Package).\n\n% Each node must belong to at least one unification set\n:- attr(\"node\", PackageNode), not unification_set(_, PackageNode).\n\n% Cannot have a node with an ID, if lower ID of the same package are not used\n:- attr(\"node\", node(ID1, Package)),\n not attr(\"node\", node(ID2, Package)),\n max_dupes(Package, X), ID1=0..X-1, ID2=0..X-1, ID2 < ID1.\n\n:- attr(\"virtual_node\", node(ID1, Package)),\n not attr(\"virtual_node\", node(ID2, Package)),\n max_dupes(Package, X), ID1=0..X-1, ID2=0..X-1, ID2 < ID1.\n\n% Prefer to assign lower ID to virtuals associated with a lower penalty provider\n:- not unification_set(\"root\", node(X, Virtual)),\n not unification_set(\"root\", node(Y, Virtual)),\n X < Y,\n provider_weight(_, node(X, Virtual), WeightX),\n provider_weight(_, node(Y, Virtual), WeightY),\n WeightY < WeightX.\n\n\n%-----------------------------------------------------------------------------\n% Map literal input specs to facts that drive the solve\n%-----------------------------------------------------------------------------\n\n% TODO: literals, at the moment, can only influence the \"root\" unification set. This needs to be extended later.\n\n% Node attributes that need custom rules in ASP, e.g. because they involve multiple nodes\nnode_attributes_with_custom_rules(\"depends_on\").\nnode_attributes_with_custom_rules(\"virtual_on_edge\").\nnode_attributes_with_custom_rules(\"provider_set\").\nnode_attributes_with_custom_rules(\"concrete_variant_set\").\nnode_attributes_with_custom_rules(\"concrete_variant_request\").\nnode_attributes_with_custom_rules(\"closure\").\n\ntrigger_condition_holds(TriggerID, node(min_dupe_id, Package)) :-\n solve_literal(TriggerID),\n pkg_fact(Package, condition_trigger(_, TriggerID)),\n literal(TriggerID).\n\ntrigger_node(TriggerID, Node, Node) :-\n trigger_condition_holds(TriggerID, Node),\n literal(TriggerID).\n\n% Since we trigger the existence of literal nodes from a condition, we need to construct the condition_set/2\nmentioned_in_literal(Root, Mentioned) :- mentioned_in_literal(TriggerID, Root, Mentioned), solve_literal(TriggerID).\nliteral_node(Root, node(min_dupe_id, Root)) :- mentioned_in_literal(Root, Root).\n\n1 { literal_node(Root, node(0..Y-1, Mentioned)) : max_dupes(Mentioned, Y) } 1 :-\n mentioned_in_literal(Root, Mentioned), Mentioned != Root.\n\nbuild_dependency_of_literal_node(LiteralNode, node(X, BuildDependency)) :-\n literal_node(Root, LiteralNode),\n build(LiteralNode),\n build_requirement(LiteralNode, node(X, BuildDependency)),\n attr(\"direct_dependency\", LiteralNode, node_requirement(\"node\", BuildDependency)).\n\ncondition_set(node(min_dupe_id, Root), LiteralNode) :- literal_node(Root, LiteralNode).\ncondition_set(LiteralNode, BuildNode) :- build_dependency_of_literal_node(LiteralNode, BuildNode).\n\n% Generate a comprehensible error for cases like 'foo ^bar' where 'bar' is a build dependency\n% of a transitive dependency of 'foo'\nerror(1, \"{0} is not a direct 'build' or 'test' dependency, or transitive 'link' or 'run' dependency of any root\", Literal)\n :- literal_node(RootPackage, node(X, Literal)),\n not depends_on(node(min_dupe_id, RootPackage), node(X, Literal)),\n not unification_set(\"root\", node(X, Literal)).\n\n:- build_dependency_of_literal_node(LiteralNode, BuildNode),\n not attr(\"depends_on\", LiteralNode, BuildNode, \"build\").\n\n% Discriminate between \"roots\" that have been explicitly requested, and roots that are deduced from \"virtual roots\"\nexplicitly_requested_root(node(min_dupe_id, Package)) :-\n solve_literal(TriggerID),\n trigger_and_effect(Package, TriggerID, EffectID),\n imposed_constraint(EffectID, \"root\", Package).\n\n\n% Keep track of which nodes are associated with which root DAG\nassociated_with_root(RootNode, RootNode) :- attr(\"root\", RootNode).\n\nassociated_with_root(RootNode, ChildNode) :-\n depends_on(ParentNode, ChildNode),\n associated_with_root(RootNode, ParentNode).\n\n% We cannot have a node in the root condition set, that is not associated with that root\n:- attr(\"root\", RootNode),\n condition_set(RootNode, node(X, Package)),\n not virtual(Package),\n not associated_with_root(RootNode, node(X, Package)).\n\n#defined concretize_everything/0.\n#defined literal/1.\n\n% Attributes for node packages which must have a single value\nattr_single_value(\"version\").\nattr_single_value(\"node_platform\").\nattr_single_value(\"node_os\").\nattr_single_value(\"node_target\").\n\n% Error when no attribute is selected\nerror(10000, no_value_error, Attribute, Package)\n :- attr(\"node\", node(ID, Package)),\n attr_single_value(Attribute),\n not attr(Attribute, node(ID, Package), _).\n\n% Error when multiple attr need to be selected\nerror(100, multiple_values_error, Attribute, Package)\n :- attr(\"node\", node(ID, Package)),\n attr_single_value(Attribute),\n 2 { attr(Attribute, node(ID, Package), Value) }.\n\n%-----------------------------------------------------------------------------\n% Version semantics\n%-----------------------------------------------------------------------------\n\nversion_declared(Package, Version) :- pkg_fact(Package, version_order(Version, _)).\n\n% If something is a package, it has only one version and that must be a\n% declared version.\n% We allow clingo to choose any version(s), and infer an error if there\n% is not precisely one version chosen. Error facts are heavily optimized\n% against to ensure they cannot be inferred when a non-error solution is\n% possible\n\nversion_constraint_satisfied(node(ID,Package), Constraint) :-\n attr(\"version\", node(ID,Package), Version),\n pkg_fact(Package, version_order(Version, VersionIdx)),\n pkg_fact(Package, version_range(Constraint, MinIdx, MaxIdx)),\n VersionIdx >= MinIdx, VersionIdx <= MaxIdx.\n\n% Pick a single version among the possible ones\n1 { choose_version(node(ID, Package), Version) : version_declared(Package, Version) } 1 :- attr(\"node\", node(ID, Package)).\n\n% To choose the \"fake\" version of virtual packages, we need a separate rule.\n% Note that a virtual node may or may not have a version, but cannot have more than one.\n{ choose_version(node(ID, Package), Version) : version_declared(Package, Version) } 1\n :- attr(\"virtual_node\", node(ID, Package)),\n virtual(Package).\n\n#defined compiler_package/1.\n\nattr(\"version\", node(ID, Package), Version) :- choose_version(node(ID, Package), Version).\n\n% If we select a deprecated version, mark the package as deprecated\nattr(\"deprecated\", node(ID, Package), Version) :-\n attr(\"version\", node(ID, Package), Version),\n not external(node(ID, Package)),\n pkg_fact(Package, deprecated_version(Version)).\n\nerror(100, \"Package '{0}' needs the deprecated version '{1}', and this is not allowed\", Package, Version)\n :- deprecated_versions_not_allowed(),\n attr(\"version\", node(ID, Package), Version),\n build(node(ID, Package)),\n pkg_fact(Package, deprecated_version(Version)).\n\n% we can't use a weight from an installed spec if we are building it\n% and vice-versa\n\n1 { allowed_origins(Origin): pkg_fact(Package, version_origin(Version, Origin)),\n Origin != \"installed\", Origin != \"packages_yaml\"}\n :- attr(\"version\", node(ID, Package), Version),\n build(node(ID, Package)).\n\n% We cannot use a version declared for an installed package if we end up building it\n:- not pkg_fact(Package, version_origin(Version, \"installed\")),\n not pkg_fact(Package, version_origin(Version, \"installed_git_version\")),\n attr(\"version\", node(ID, Package), Version),\n concrete(node(ID, Package)).\n\nversion_weight(node(ID, Package), Weight)\n :- attr(\"version\", node(ID, Package), Version),\n attr(\"node\", node(ID, Package)),\n pkg_fact(Package, version_declared(Version, Weight)).\n\nversion_deprecation_penalty(node(ID, Package), Penalty)\n :- pkg_fact(Package, deprecated_version(Version)),\n pkg_fact(Package, version_deprecation_penalty(Penalty)),\n attr(\"node\", node(ID, Package)),\n attr(\"version\", node(ID, Package), Version),\n not external(node(ID, Package)).\n\n% More specific error message if the version cannot satisfy some constraint\n% Otherwise covered by `no_version_error` and `versions_conflict_error`.\nerror(10000, \"Cannot satisfy '{0}@{1}'\", Package, Constraint)\n :- attr(\"node_version_satisfies\", node(ID, Package), Constraint),\n not version_constraint_satisfied(node(ID,Package), Constraint).\n\nattr(\"node_version_satisfies\", node(ID, Package), Constraint)\n :- attr(\"version\", node(ID, Package), Version),\n pkg_fact(Package, version_order(Version, VersionIdx)),\n pkg_fact(Package, version_range(Constraint, MinIdx, MaxIdx)),\n VersionIdx >= MinIdx, VersionIdx <= MaxIdx.\n\n\n% if a version needs a commit or has one it can use the commit variant\ncan_accept_commit(Package, Version) :- pkg_fact(Package, version_needs_commit(Version)).\ncan_accept_commit(Package, Version) :- pkg_fact(Package, version_has_commit(Version, _)).\n\n% Specs with a commit variant can't use versions that don't need commits\nerror(10, \"Cannot use commit variant with '{0}@={1}'\", Package, Version)\n :- attr(\"version\", node(ID, Package), Version),\n not can_accept_commit(Package, Version),\n attr(\"variant_value\", node(ID, Package), \"commit\", _).\n\nerror(10, \"Commit '{0}' must match package.py value '{1}' for '{2}@={3}'\", Vsha, Psha, Package, Version)\n :- attr(\"version\", node(ID, Package), Version),\n attr(\"variant_value\", node(ID, Package), \"commit\", Vsha),\n pkg_fact(Package, version_has_commit(Version, Psha)),\n Vsha != Psha.\n\n#defined deprecated_versions_not_allowed/0.\n\n%-----------------------------------------------------------------------------\n% Spec conditions and imposed constraints\n%\n% Given Spack directives like these:\n% depends_on(\"foo@1.0+bar\", when=\"@2.0+variant\")\n% provides(\"mpi@2:\", when=\"@1.9:\")\n%\n% The conditions are `@2.0+variant` and `@1.9:`, and the imposed constraints\n% are `@1.0+bar` on `foo` and `@2:` on `mpi`.\n%-----------------------------------------------------------------------------\n% conditions are specified with `condition_requirement` and hold when\n% corresponding spec attributes hold.\n\n% A \"condition_set(PackageNode, _)\" is the set of nodes on which PackageNode can require / impose conditions\ncondition_set(PackageNode, PackageNode) :- attr(\"node\", PackageNode).\ncondition_set(PackageNode, VirtualNode) :- provider(PackageNode, VirtualNode).\ncondition_set(PackageNode, DependencyNode) :- condition_set(PackageNode, PackageNode), depends_on(PackageNode, DependencyNode).\ncondition_set(ID, VirtualNode) :- condition_set(ID, PackageNode), provider(PackageNode, VirtualNode).\ncondition_set(VirtualNode, X) :- provider(PackageNode, VirtualNode), condition_set(PackageNode, X).\n\ncondition_packages(ID, A1) :- condition_requirement(ID, _, A1).\ncondition_packages(ID, A1) :- condition_requirement(ID, _, A1, _).\ncondition_packages(ID, A1) :- condition_requirement(ID, _, A1, _, _).\ncondition_packages(ID, A1) :- condition_requirement(ID, _, A1, _, _, _).\n\ntrigger_node(ID, node(PackageID, Package), node(PackageID, Package)) :- pkg_fact(Package, trigger_id(ID)), attr(\"node\", node(PackageID, Package)).\ntrigger_node(ID, node(PackageID, Package), node(VirtualID, Virtual)) :- pkg_fact(Virtual, trigger_id(ID)), provider(node(PackageID, Package), node(VirtualID, Virtual)).\n\n% This is the \"real node\" that triggers the request, e.g. if the request started from \"mpi\" this is the mpi provider\ntrigger_real_node(ID, PackageNode) :- trigger_node(ID, PackageNode, _).\n\n% This is the requestor node, which may be a \"real\" or a \"virtual\" node\ntrigger_requestor_node(ID, RequestorNode) :- trigger_node(ID, _, RequestorNode).\n\ntrigger_package_requirement(TriggerNode, A1) :-\n trigger_real_node(ID, TriggerNode),\n condition_packages(ID, A1).\n\ncondition_nodes(PackageNode, node(X, A1))\n :- trigger_package_requirement(PackageNode, A1),\n condition_set(PackageNode, node(X, A1)),\n not self_build_requirement(PackageNode, node(X, A1)).\n\ncannot_hold(TriggerID, PackageNode)\n :- condition_packages(TriggerID, A1),\n not condition_set(PackageNode, node(_, A1)),\n trigger_real_node(TriggerID, PackageNode),\n attr(\"node\", PackageNode).\n\n% Aggregates generic condition requirements with TriggerID, to see if a condition holds\ntrigger_condition_holds(ID, RequestorNode) :-\n trigger_node(ID, PackageNode, RequestorNode);\n satisfied(trigger(PackageNode), condition_requirement(Name, A1)) : condition_requirement(ID, Name, A1);\n satisfied(trigger(PackageNode), condition_requirement(Name, A1, A2)) : condition_requirement(ID, Name, A1, A2);\n satisfied(trigger(PackageNode), condition_requirement(Name, A1, A2, A3)) : condition_requirement(ID, Name, A1, A2, A3);\n satisfied(trigger(PackageNode), condition_requirement(Name, A1, A2, A3, A4)) : condition_requirement(ID, Name, A1, A2, A3, A4);\n not cannot_hold(ID, PackageNode).\n\n\n%%%%\n% Conditions verified on actual nodes in the DAG\n%%%%\n\n% Here we project out the trigger ID from condition requirements, so that we can reduce the space\n% of satisfied facts below. All we care about below is if a condition is met (e.g. a node exists\n% in the DAG), we don't care instead about *which* rule is requesting that.\ngeneric_condition_requirement(Name, A1) :- condition_requirement(ID, Name, A1).\ngeneric_condition_requirement(Name, A1, A2) :- condition_requirement(ID, Name, A1, A2).\ngeneric_condition_requirement(Name, A1, A2, A3) :- condition_requirement(ID, Name, A1, A2, A3).\ngeneric_condition_requirement(Name, A1, A2, A3, A4) :- condition_requirement(ID, Name, A1, A2, A3, A4).\n\nsatisfied(trigger(PackageNode), condition_requirement(Name, A1)) :-\n attr(Name, node(X, A1)),\n generic_condition_requirement(Name, A1),\n condition_nodes(PackageNode, node(X, A1)).\n\nsatisfied(trigger(PackageNode), condition_requirement(Name, A1, A2)) :-\n attr(Name, node(X, A1), A2),\n generic_condition_requirement(Name, A1, A2),\n condition_nodes(PackageNode, node(X, A1)).\n\nsatisfied(trigger(PackageNode), condition_requirement(Name, A1, A2, A3)) :-\n attr(Name, node(X, A1), A2, A3),\n generic_condition_requirement(Name, A1, A2, A3),\n condition_nodes(PackageNode, node(X, A1)),\n not node_attributes_with_custom_rules(Name).\n\nsatisfied(trigger(PackageNode), condition_requirement(Name, A1, A2, A3, A4)) :-\n attr(Name, node(X, A1), A2, A3, A4),\n generic_condition_requirement(Name, A1, A2, A3, A4),\n condition_nodes(PackageNode, node(X, A1)).\n\nsatisfied(trigger(PackageNode), condition_requirement(\"depends_on\", A1, A2, A3)) :-\n attr(\"depends_on\", node(X, A1), node(Y, A2), A3),\n generic_condition_requirement(\"depends_on\", A1, A2, A3),\n condition_nodes(PackageNode, node(X, A1)),\n condition_nodes(PackageNode, node(Y, A2)).\n\nsatisfied(trigger(PackageNode), condition_requirement(\"concrete_variant_request\", A1, A2, A3)) :-\n generic_condition_requirement(\"concrete_variant_request\", A1, A2, A3),\n condition_nodes(PackageNode, node(X, A1)).\n\nsatisfied(trigger(PackageNode), condition_requirement(\"closure\", A1, A2, A3)) :-\n attr(\"closure\", node(X, A1), node(_, A2), A3),\n generic_condition_requirement(\"closure\", A1, A2, A3),\n condition_nodes(PackageNode, node(X, A1)).\n\nsatisfied(trigger(PackageNode), condition_requirement(\"virtual_on_edge\", A1, A2, A3)) :-\n attr(\"virtual_on_edge\", node(X, A1), node(Y, A2), A3),\n generic_condition_requirement(\"virtual_on_edge\", A1, A2, A3),\n condition_nodes(PackageNode, node(X, A1)),\n condition_nodes(PackageNode, node(Y, A2)).\n\n%%%%\n% Conditions verified on pure build deps of reused nodes\n%%%%\n\nsatisfied(trigger(PackageNode), condition_requirement(\"node\", A1)) :-\n trigger_real_node(ID, PackageNode),\n reused_provider(PackageNode, _),\n condition_requirement(ID, \"node\", A1).\n\nsatisfied(trigger(PackageNode), condition_requirement(\"virtual_node\", A1)) :-\n trigger_real_node(ID, PackageNode),\n reused_provider(PackageNode, node(_, A1)),\n condition_requirement(ID, \"virtual_node\", A1).\n\nsatisfied(trigger(PackageNode), condition_requirement(\"virtual_on_incoming_edges\", A1, A2)) :-\n trigger_real_node(ID, PackageNode),\n reused_provider(node(Hash, A1), node(Hash, A2)),\n condition_requirement(ID, \"virtual_on_incoming_edges\", A1, A2).\n\nsatisfied(trigger(node(Hash, Package)), condition_requirement(Name, Package, A1)) :-\n trigger_real_node(ID, node(Hash, Package)),\n reused_provider(node(Hash, Package), node(Hash, Language)),\n hash_attr(Hash, Name, Package, A1),\n condition_requirement(ID, Name, Package, A1).\n\nsatisfied(trigger(node(Hash, Package)), condition_requirement(\"node_version_satisfies\", Package, VersionConstraint)) :-\n trigger_real_node(ID, node(Hash, Package)),\n reused_provider(node(Hash, Package), node(Hash, Language)),\n hash_attr(Hash, \"version\", Package, Version),\n condition_requirement(ID, \"node_version_satisfies\", Package, VersionConstraint),\n pkg_fact(Package, version_order(Version, VersionIdx)),\n pkg_fact(Package, version_range(VersionConstraint, MinIdx, MaxIdx)),\n VersionIdx >= MinIdx, VersionIdx <= MaxIdx.\n\nsatisfied(trigger(node(Hash, Package)), condition_requirement(Name, Package, A1, A2)) :-\n trigger_real_node(ID, node(Hash, Package)),\n reused_provider(node(Hash, Package), node(Hash, Language)),\n hash_attr(Hash, Name, Package, A1, A2),\n condition_requirement(ID, Name, Package, A1, A2).\n\n\ncondition_with_concrete_variant(ID, Package, Variant) :- condition_requirement(ID, \"concrete_variant_request\", Package, Variant, _).\n\ncannot_hold(ID, PackageNode) :-\n not attr(\"variant_value\", node(X, A1), Variant, Value),\n condition_with_concrete_variant(ID, A1, Variant),\n condition_requirement(ID, \"concrete_variant_request\", A1, Variant, Value),\n condition_nodes(PackageNode, node(X, A1)).\n\ncannot_hold(ID, PackageNode) :-\n attr(\"variant_value\", node(X, A1), Variant, Value),\n condition_with_concrete_variant(ID, A1, Variant),\n not condition_requirement(ID, \"concrete_variant_request\", A1, Variant, Value),\n condition_nodes(PackageNode, node(X, A1)).\n\ncondition_holds(ConditionID, node(X, Package))\n :- pkg_fact(Package, condition_trigger(ConditionID, TriggerID)),\n trigger_condition_holds(TriggerID, node(X, Package)).\n\ntrigger_and_effect(Package, ID, TriggerID, EffectID)\n :- pkg_fact(Package, condition_trigger(ID, TriggerID)),\n pkg_fact(Package, condition_effect(ID, EffectID)).\n\ntrigger_and_effect(Package, TriggerID, EffectID) :- trigger_and_effect(Package, ID, TriggerID, EffectID).\n\n% condition_holds(ID, node(ID, Package)) implies all imposed_constraints, unless do_not_impose(ID, node(ID, Package))\n% is derived. This allows imposed constraints to be canceled in special cases.\n\n% Effects of direct conditions hold if the trigger holds\nimpose(EffectID, node(X, Package))\n :- not subcondition(ConditionID, _),\n trigger_and_effect(Package, ConditionID, TriggerID, EffectID),\n trigger_requestor_node(TriggerID, node(X, Package)),\n trigger_condition_holds(TriggerID, node(X, Package)),\n not do_not_impose(EffectID, node(X, Package)).\n\n% Effects of subconditions hold if the trigger holds and the\n% primary condition holds\nimpose(EffectID, node(X, Package))\n :- subcondition(SubconditionID, ConditionID),\n condition_holds(ConditionID, ConditionNode),\n condition_set(ConditionNode, node(X, Package)),\n trigger_and_effect(Package, SubconditionID, TriggerID, EffectID),\n trigger_requestor_node(TriggerID, node(X, Package)),\n trigger_condition_holds(TriggerID, node(X, Package)),\n not do_not_impose(EffectID, node(X, Package)).\n\nimposed_packages(ID, A1) :- imposed_constraint(ID, _, A1).\nimposed_packages(ID, A1) :- imposed_constraint(ID, _, A1, _).\nimposed_packages(ID, A1) :- imposed_constraint(ID, _, A1, _, _).\nimposed_packages(ID, A1) :- imposed_constraint(ID, _, A1, _, _, _).\nimposed_packages(ID, A1) :- imposed_constraint(ID, \"depends_on\", _, A1, _).\n\nimposed_nodes(node(ID, Package), node(X, A1))\n :- condition_set(node(ID, Package), node(X, A1)),\n % We don't want to add build requirements to imposed nodes, to avoid\n % unsat problems when we deal with self-dependencies: gcc@14 %gcc@10\n not self_build_requirement(node(ID, Package), node(X, A1)).\n\nself_build_requirement(node(X, Package), node(Y, Package)) :- build_requirement(node(X, Package), node(Y, Package)).\n\nimposed_nodes(PackageNode, node(X, A1))\n :- imposed_packages(ConditionID, A1),\n condition_set(PackageNode, node(X, A1)),\n attr(\"hash\", PackageNode, ConditionID).\n\n:- imposed_packages(ID, A1), impose(ID, PackageNode), not condition_set(PackageNode, node(_, A1)).\n:- imposed_packages(ID, A1), impose(ID, PackageNode), not imposed_nodes(PackageNode, node(_, A1)).\n\n% Conditions that hold impose may impose constraints on other specs\nattr(Name, node(X, A1)) :- impose(ID, PackageNode), imposed_constraint(ID, Name, A1), imposed_nodes(PackageNode, node(X, A1)).\nattr(Name, node(X, A1), A2) :- impose(ID, PackageNode), imposed_constraint(ID, Name, A1, A2), imposed_nodes(PackageNode, node(X, A1)), not node_attributes_with_custom_rules(Name).\nattr(Name, node(X, A1), A2, A3) :- impose(ID, PackageNode), imposed_constraint(ID, Name, A1, A2, A3), imposed_nodes(PackageNode, node(X, A1)), not node_attributes_with_custom_rules(Name).\nattr(Name, node(X, A1), A2, A3, A4) :- impose(ID, PackageNode), imposed_constraint(ID, Name, A1, A2, A3, A4), imposed_nodes(PackageNode, node(X, A1)).\n\n% Provider set is relevant only for literals, since it's the only place where `^[virtuals=foo] bar`\n% might appear in the HEAD of a rule\n1 {\n attr(\"provider_set\", node(0..MaxProvider-1, Provider), node(0..MaxVirtual-1, Virtual)):\n max_dupes(Provider, MaxProvider), max_dupes(Virtual, MaxVirtual)\n} 1\n :- solve_literal(TriggerID),\n trigger_and_effect(_, TriggerID, EffectID),\n impose(EffectID, _),\n imposed_constraint(EffectID, \"provider_set\", Provider, Virtual).\n\nprovider(ProviderNode, VirtualNode) :- attr(\"provider_set\", ProviderNode, VirtualNode).\n\n% Here we can't use the condition set because it's a recursive definition, that doesn't define the\n% node index, and leads to unsatisfiability. Hence we say that one and only one node index must\n% satisfy the dependency.\n1 { attr(\"depends_on\", node(X, A1), node(0..Y-1, A2), A3) : max_dupes(A2, Y) } 1\n :- impose(ID, node(X, A1)),\n imposed_constraint(ID, \"depends_on\", A1, A2, A3).\n\n% For := we must keep track of the origin of the fact, since we need to check\n% each condition separately, i.e. foo:=a,b in one place and foo:=c in another\n% should not make foo:=a,b,c possible\nattr(\"concrete_variant_set\", node(X, A1), Variant, Value, ID)\n :- impose(ID, PackageNode),\n imposed_nodes(PackageNode, node(X, A1)),\n imposed_constraint(ID, \"concrete_variant_set\", A1, Variant, Value).\n\n\n% The rule below accounts for expressions like:\n%\n% root ^dep %compiler\n%\n% where \"compiler\" is a dependency of \"dep\", but is enforced by a condition imposed by \"root\"\n1 { attr(\"depends_on\", node(A1_DUPE_ID, A1), node(0..Y-1, A2), A3) : max_dupes(A2, Y) } 1\n :- impose(ID, RootNode),\n unification_set(\"root\", RootNode),\n condition_set(RootNode, node(A1_DUPE_ID, A1)),\n not self_build_requirement(RootNode, node(A1_DUPE_ID, A1)),\n imposed_constraint(ID, \"depends_on\", A1, A2, A3).\n\n:- attr(\"direct_dependency\", ParentNode, node_requirement(\"virtual_on_incoming_edges\", ChildPkg, Virtual)),\n not attr(\"virtual_on_edge\", ParentNode, node(_, ChildPkg), Virtual).\n\n% If the parent is built, then we have a build_requirement on another node. For concrete nodes,\n% we don't since we are trimming their build dependencies.\n\n% Concrete nodes\n:- attr(\"direct_dependency\", ParentNode, node_requirement(\"node\", BuildDependency)),\n concrete_build_requirement(ParentNode, BuildDependency),\n concrete(ParentNode),\n not attr(\"concrete_build_dependency\", ParentNode, BuildDependency, _).\n\n:- attr(\"direct_dependency\", ParentNode, node_requirement(\"node_version_satisfies\", BuildDependency, Constraint)),\n concrete_build_requirement(ParentNode, BuildDependency),\n attr(\"concrete_build_dependency\", ParentNode, BuildDependency, BuildDependencyHash),\n not hash_attr(BuildDependencyHash, \"node_version_satisfies\", BuildDependency, Constraint).\n\n:- attr(\"direct_dependency\", ParentNode, node_requirement(\"provider_set\", BuildDependency, Virtual)),\n concrete_build_requirement(ParentNode, BuildDependency),\n attr(\"concrete_build_dependency\", ParentNode, BuildDependency, BuildDependencyHash),\n not attr(\"virtual_on_build_edge\", ParentNode, BuildDependency, Virtual).\n\n% Give a penalty if reuse introduces a node compiled with a compiler that is not used otherwise.\n% The only exception is if the current node is a compiler itself.\ncompiler_from_reuse(Hash, DependencyPackage) :-\n attr(\"concrete_build_dependency\", ParentNode, DependencyPackage, Hash),\n attr(\"virtual_on_build_edge\", ParentNode, DependencyPackage, Virtual),\n not node_compiler(_, ParentNode),\n language(Virtual).\n\ncompiler_penalty_from_reuse(Hash) :-\n compiler_from_reuse(Hash, DependencyPackage),\n not node_compiler(_, node(_, DependencyPackage)),\n % We don't want to give penalties if we're just installing binaries\n will_build_packages().\n\ncompiler_penalty_from_reuse(Hash) :-\n compiler_from_reuse(Hash, DependencyPackage),\n not 1 { attr(\"hash\", node(X, DependencyPackage), Hash) : node_compiler(_, node(X, DependencyPackage)) },\n % We don't want to give penalties if we're just installing binaries\n will_build_packages().\n\n\nerror(100, \"Cannot satisfy the request on {0} to have {1}={2}\", BuildDependency, Variant, Value)\n:- attr(\"direct_dependency\", ParentNode, node_requirement(\"variant_set\", BuildDependency, Variant, Value)),\n concrete_build_requirement(ParentNode, BuildDependency),\n attr(\"concrete_build_dependency\", ParentNode, BuildDependency, BuildDependencyHash),\n not hash_attr(BuildDependencyHash, \"variant_value\", BuildDependency, Variant, Value).\n\nerror(100, \"Cannot satisfy the request on {0} to have the target set to {1}\", BuildDependency, Target)\n:- attr(\"direct_dependency\", ParentNode, node_requirement(\"node_target_set\", BuildDependency, Target)),\n concrete_build_requirement(ParentNode, BuildDependency),\n attr(\"concrete_build_dependency\", ParentNode, BuildDependency, BuildDependencyHash),\n not hash_attr(BuildDependencyHash, \"node_target\", BuildDependency, Target).\n\nerror(100, \"Cannot satisfy the request on {0} to have the os set to {1}\", BuildDependency, NodeOS)\n:- attr(\"direct_dependency\", ParentNode, node_requirement(\"node_os_set\", BuildDependency, NodeOS)),\n concrete_build_requirement(ParentNode, BuildDependency),\n attr(\"concrete_build_dependency\", ParentNode, BuildDependency, BuildDependencyHash),\n not hash_attr(BuildDependencyHash, \"node_os\", BuildDependency, NodeOS).\n\nerror(100, \"Cannot satisfy the request on {0} to have the platform set to {1}\", BuildDependency, Platform)\n:- attr(\"direct_dependency\", ParentNode, node_requirement(\"node_platform_set\", BuildDependency, Platform)),\n concrete_build_requirement(ParentNode, BuildDependency),\n attr(\"concrete_build_dependency\", ParentNode, BuildDependency, BuildDependencyHash),\n not hash_attr(BuildDependencyHash, \"node_platform\", BuildDependency, Platform).\n\nerror(100, \"Cannot satisfy the request on {0} to have the following hash {1}\", BuildDependency, BuildHash)\n:- attr(\"direct_dependency\", ParentNode, node_requirement(\"node_target_set\", BuildDependency, Target)),\n concrete_build_requirement(ParentNode, BuildDependency),\n attr(\"concrete_build_dependency\", ParentNode, BuildDependency, BuildDependencyHash),\n attr(\"direct_dependency\", ParentNode, node_requirement(\"hash\", BuildDependency, BuildHash)),\n BuildHash != BuildDependencyHash.\n\n% Asking for gcc@10 %gcc@9 shouldn't give us back an external gcc@10, just because of the hack\n% we have on externals\n:- attr(\"direct_dependency\", node(X, Parent), node_requirement(\"node\", BuildDependency)),\n Parent == BuildDependency,\n external(node(X, Parent)).\n\n% If a virtual is needed only to build a package, its provider can be considered a build requirement\n1 { virtual_build_requirement(ParentNode, node(0..Y-1, Virtual)) : max_dupes(Virtual, Y) } 1\n :- attr(\"dependency_holds\", ParentNode, Virtual, \"build\"),\n not attr(\"dependency_holds\", ParentNode, Virtual,\"link\"),\n not attr(\"dependency_holds\", ParentNode, Virtual,\"run\"),\n virtual(Virtual).\n\n% The virtual build dependency must be on the correct duplicate\n:- virtual_build_requirement(ParentNode, node(X, Virtual)),\n provider(ProviderNode, node(X, Virtual)),\n not depends_on(ParentNode, ProviderNode).\n\nattr(\"virtual_node\", VirtualNode) :- virtual_build_requirement(ParentNode, VirtualNode).\nbuild_requirement(ParentNode, ProviderNode) :-\n virtual_build_requirement(ParentNode, VirtualNode),\n provider(ProviderNode, VirtualNode),\n not attr(\"depends_on\", ParentNode, ProviderNode, \"link\"),\n not attr(\"depends_on\", ParentNode, ProviderNode, \"run\"),\n attr(\"depends_on\", ParentNode, ProviderNode, \"build\").\n\n% From cli we can have literal expressions like:\n%\n% root %gcc@12.0 ^dep %gcc@11.2\n%\n% Adding a \"direct_dependency\" is a way to discriminate between the incompatible\n% version constraints on \"gcc\" in the \"imposed_constraint\".\nattr(\"node_version_satisfies\", node(X, BuildDependency), Constraint) :-\n attr(\"direct_dependency\", ParentNode, node_requirement(\"node_version_satisfies\", BuildDependency, Constraint)),\n build_requirement(ParentNode, node(X, BuildDependency)).\n\n% Account for properties on the build requirements\n%\n% root %gcc@12.0 ^dep\n%\nattr(\"variant_set\", node(X, BuildDependency), Variant, Value) :-\n attr(\"direct_dependency\", ParentNode, node_requirement(\"variant_set\", BuildDependency, Variant, Value)),\n build_requirement(ParentNode, node(X, BuildDependency)).\n\nattr(\"depends_on\", node(X, Parent), node(Y, BuildDependency), \"build\") :- build_requirement(node(X, Parent), node(Y, BuildDependency)).\n\nattr(\"node_target_set\", node(X, BuildDependency), Target) :-\n attr(\"direct_dependency\", ParentNode, node_requirement(\"node_target_set\", BuildDependency, Target)),\n build_requirement(ParentNode, node(X, BuildDependency)).\n\nattr(\"node_os_set\", node(X, BuildDependency), NodeOS) :-\n attr(\"direct_dependency\", ParentNode, node_requirement(\"node_os_set\", BuildDependency, NodeOS)),\n build_requirement(ParentNode, node(X, BuildDependency)).\n\nattr(\"node_platform_set\", node(X, BuildDependency), NodePlatform) :-\n attr(\"direct_dependency\", ParentNode, node_requirement(\"node_platform_set\", BuildDependency, NodePlatform)),\n build_requirement(ParentNode, node(X, BuildDependency)).\n\nattr(\"node_flag_set\", node(X, BuildDependency), NodeFlag) :-\n attr(\"direct_dependency\", ParentNode, node_requirement(\"node_flag_set\", BuildDependency, NodeFlag)),\n build_requirement(ParentNode, node(X, BuildDependency)).\n\nattr(\"hash\", node(X, BuildDependency), BuildHash) :-\n attr(\"direct_dependency\", ParentNode, node_requirement(\"hash\", BuildDependency, BuildHash)),\n build_requirement(ParentNode, node(X, BuildDependency)).\n\n% For a spec like `hdf5 %cxx=gcc` we need to ensure that\n% 1. gcc is a provider for cxx\n% 2. hdf5 depends on that provider for cxx\n1 { attr(\"provider_set\", node(X, BuildDependency), node(0..Y-1, Virtual)) : max_dupes(Virtual, Y) } 1 :-\n attr(\"direct_dependency\", ParentNode, node_requirement(\"provider_set\", BuildDependency, Virtual)),\n direct_dependency(ParentNode, node(X, BuildDependency)).\n\nerror(10, \"{0} cannot have a dependency on {1}\", Package, Virtual)\n :- attr(\"direct_dependency\", node(ID, Package), node_requirement(\"provider_set\", BuildDependency, Virtual)),\n direct_dependency(node(ID, Package), node(X, BuildDependency)),\n not attr(\"virtual_on_edge\", node(ID, Package), node(X, BuildDependency), Virtual).\n\n% For a spec like `hdf5 %cxx` we need to ensure that the virtual is needed on a direct edge\nerror(10, \"{0} cannot have a dependency on {1}\", Package, Virtual)\n :- attr(\"direct_dependency\", node(ID, Package), node_requirement(\"virtual_node\", Virtual)),\n not attr(\"virtual_on_edge\", node(ID, Package), _, Virtual).\n\n% Reconstruct virtual dependencies for reused specs\nattr(\"virtual_on_edge\", node(X, A1), node(Y, A2), Virtual)\n :- impose(ID, node(X, A1)),\n depends_on(node(X, A1), node(Y, A2)),\n imposed_constraint(ID, \"virtual_on_edge\", A1, A2, Virtual),\n concrete(node(X, A1)).\n\nvirtual_condition_holds(node(Y, A2), Virtual)\n :- impose(ID, node(X, A1)),\n attr(\"virtual_on_edge\", node(X, A1), node(Y, A2), Virtual),\n concrete(node(X, A1)).\n\n% Simplified virtual information for conditionl requirements in\n% conditional dependencies\n% Most specs track virtuals on edges\nattr(\"uses_virtual\", PackageNode, Virtual) :-\n attr(\"virtual_on_edge\", PackageNode, _, Virtual).\n\n% Reused specs don't track a real edge to build-only deps\nattr(\"uses_virtual\", PackageNode, Virtual) :-\n attr(\"virtual_on_build_edge\", PackageNode, _, Virtual).\n\n% we cannot have additional variant values when we are working with concrete specs\n:- attr(\"node\", node(ID, Package)),\n attr(\"hash\", node(ID, Package), Hash),\n attr(\"variant_value\", node(ID, Package), Variant, Value),\n not imposed_constraint(Hash, \"variant_value\", Package, Variant, Value).\n\n% we cannot have additional flag values when we are working with concrete specs\n:- attr(\"node\", node(ID, Package)),\n attr(\"hash\", node(ID, Package), Hash),\n attr(\"node_flag\", node(ID, Package), node_flag(FlagType, Flag, _, _)),\n not imposed_constraint(Hash, \"node_flag\", Package, node_flag(FlagType, Flag, _, _)).\n\n% we cannot have two nodes with the same hash\n:- attr(\"hash\", PackageNode1, Hash), attr(\"hash\", PackageNode2, Hash), PackageNode1 < PackageNode2.\n\n#defined condition/1.\n#defined subcondition/2.\n#defined condition_requirement/3.\n#defined condition_requirement/4.\n#defined condition_requirement/5.\n#defined condition_requirement/6.\n#defined imposed_constraint/3.\n#defined imposed_constraint/4.\n#defined imposed_constraint/5.\n#defined imposed_constraint/6.\n\n%-----------------------------------------------------------------------------\n% Concrete specs\n%-----------------------------------------------------------------------------\n\n% if a package is assigned a hash, it's concrete.\nconcrete(PackageNode) :- attr(\"hash\", PackageNode, _), attr(\"node\", PackageNode).\n\n:- concrete(PackageNode),\n depends_on(PackageNode, DependencyNode),\n not concrete(DependencyNode),\n not abi_splice_conditions_hold(_, DependencyNode, _, _).\n\n%-----------------------------------------------------------------------------\n% Dependency semantics\n%-----------------------------------------------------------------------------\n% Dependencies of any type imply that one package \"depends on\" another\ndepends_on(PackageNode, DependencyNode) :- attr(\"depends_on\", PackageNode, DependencyNode, _).\n\n% Dependencies of concrete specs don't need to be resolved -- they arise from the concrete specs themselves.\ndo_not_impose(EffectID, node(X, Package)) :-\n trigger_and_effect(Package, _, TriggerID, EffectID),\n trigger_condition_holds(TriggerID, node(X, Package)),\n imposed_constraint(EffectID, \"dependency_holds\", Package, _, _),\n concrete(node(X, Package)).\n\n% If a dependency holds on a package node, there must be one and only one dependency node satisfying it\n1 { attr(\"depends_on\", PackageNode, node(0..Y-1, Dependency), Type) : max_dupes(Dependency, Y) } 1\n :- attr(\"dependency_holds\", PackageNode, Dependency, Type),\n not virtual(Dependency).\n\n% all nodes in the graph must be reachable from some root\n% this ensures a user can't say `zlib ^libiconv` (neither of which have any\n% dependencies) and get a two-node unconnected graph\nneeded(PackageNode) :- attr(\"root\", PackageNode).\nneeded(ChildNode) :- edge_needed(ParentNode, ChildNode).\n\nedge_needed(ParentNode, node(X, Child)) :- depends_on(ParentNode, node(X, Child)), runtime(Child).\nedge_needed(ParentNode, ChildNode) :- depends_on(ParentNode, ChildNode) , concrete(ParentNode).\n\nedge_needed(ParentNode, node(X, Child)) :-\n depends_on(ParentNode, node(X, Child)),\n build(ParentNode),\n attr(\"dependency_holds\", ParentNode, Child, _).\n\nvirtual_edge_needed(ParentNode, ChildNode, node(X, Virtual)) :-\n build(ParentNode),\n attr(\"virtual_on_edge\", ParentNode, ChildNode, Virtual),\n provider(ChildNode, node(X, Virtual)).\n\nvirtual_edge_needed(ParentNode, ChildNode, node(X, Virtual)) :-\n concrete(ParentNode),\n concrete(ChildNode),\n provider(ChildNode, node(X, Virtual)),\n attr(\"virtual_on_edge\", ParentNode, ChildNode, Virtual).\n\nvirtual_edge_needed(ParentNode, ChildNode, node(X, Virtual)) :-\n concrete(ParentNode),\n abi_splice_conditions_hold(_, ChildNode, _, _),\n provider(ChildNode, node(X, Virtual)),\n attr(\"virtual_on_edge\", ParentNode, ChildNode, Virtual).\n\n\nedge_needed(ParentNode, ChildNode) :- virtual_edge_needed(ParentNode, ChildNode, _).\nprovider_needed(ChildNode, VirtualNode) :- virtual_edge_needed(_, ChildNode, VirtualNode).\nprovider_needed(ChildNode, VirtualNode) :- attr(\"virtual_root\", VirtualNode), provider(ChildNode, VirtualNode).\n\nerror(10, \"'{0}' is not a valid dependency for any package in the DAG\", Package)\n :- attr(\"node\", node(ID, Package)),\n not needed(node(ID, Package)).\n\n:- depends_on(ParentNode, ChildNode),\n not edge_needed(ParentNode, ChildNode),\n build(ParentNode).\n\n:- provider(PackageNode, VirtualNode),\n not provider_needed(PackageNode, VirtualNode),\n not attr(\"virtual_root\", VirtualNode).\n\n\n% Extensions depending on each other must all extend the same node (e.g. all Python packages\n% depending on each other must depend on the same Python interpreter)\nerror(100, \"{0} and {1} must depend on the same {2}\", ExtensionParent, ExtensionChild, ExtendeePackage)\n :- depends_on(ExtensionParent, ExtensionChild),\n attr(\"extends\", ExtensionParent, ExtendeePackage),\n depends_on(ExtensionParent, node(X, ExtendeePackage)),\n depends_on(ExtensionChild, node(Y, ExtendeePackage)),\n X != Y.\n\n\n%-----------------------------------------------------------------------------\n% Conflicts\n%-----------------------------------------------------------------------------\n\n% Most conflicts are internal to the same package\nconflict_is_cross_package(Package, TriggerID) :-\n pkg_fact(Package, conflict(TriggerID, _, _)),\n pkg_fact(TriggerPackage, condition_trigger(TriggerID, _)),\n TriggerPackage != Package.\n\nconflict_internal(Package, TriggerID, ConstraintID, Msg) :-\n pkg_fact(Package, conflict(TriggerID, ConstraintID, Msg)),\n not conflict_is_cross_package(Package, TriggerID).\n\n% Case 1: conflict is within the same package\nerror(1, Msg) :-\n conflict_internal(Package, TriggerID, ConstraintID, Msg),\n condition_holds(TriggerID, node(ID, Package)),\n condition_holds(ConstraintID, node(ID, Package)),\n build(node(ID, Package)). % ignore conflicts for installed packages\n\n% Case 2: Cross-package conflicts (Rare case - slower)\nerror(1, Msg) :-\n build(node(ID, Package)),\n conflict_is_cross_package(Package, TriggerID),\n pkg_fact(Package, conflict(TriggerID, ConstraintID, Msg)),\n condition_holds(TriggerID, node(ID1, TriggerPackage)),\n condition_holds(ConstraintID, node(ID, Package)),\n unification_set(X, node(ID, Package)),\n unification_set(X, node(ID1, TriggerPackage)).\n\n%-----------------------------------------------------------------------------\n% Virtual dependencies\n%-----------------------------------------------------------------------------\n\n% Package provides to this client at least one virtual from those that need to be provided together\nnode_uses_provider_with_constraints(ClientNode, node(X, Package), ID, SetID) :-\n condition_holds(ID, node(X, Package)),\n pkg_fact(Package, provided_together(ID, SetID, V)),\n attr(\"virtual_on_edge\", ClientNode, node(X, Package), V).\n\n% This error is triggered if the package provides some but not all required virtuals from\n% the set that needs to be provided together\nerror(100, \"Package '{0}' needs to also provide '{1}' (provided_together constraint)\", Package, Virtual)\n:- node_uses_provider_with_constraints(ClientNode, node(X, Package), ID, SetID),\n pkg_fact(Package, provided_together(ID, SetID, Virtual)),\n node_depends_on_virtual(ClientNode, Virtual),\n not attr(\"virtual_on_edge\", ClientNode, node(X, Package), Virtual).\n\n% if a package depends on a virtual, it's not external and we have a\n% provider for that virtual then it depends on the provider\nnode_depends_on_virtual(PackageNode, Virtual, Type)\n :- attr(\"dependency_holds\", PackageNode, Virtual, Type),\n virtual(Virtual).\n\nnode_depends_on_virtual(PackageNode, Virtual) :- node_depends_on_virtual(PackageNode, Virtual, Type).\nvirtual_is_needed(Virtual) :- node_depends_on_virtual(PackageNode, Virtual).\n\n1 { virtual_on_edge(PackageNode, ProviderNode, node(VirtualID, Virtual), Type) : provider(ProviderNode, node(VirtualID, Virtual)) } 1\n :- node_depends_on_virtual(PackageNode, Virtual, Type).\n\n% A package that depends on a virtual with type (\"build\", \"link\") cannot have two providers\n% (one for the \"build\" and one for the \"link\")\n:- node_depends_on_virtual(PackageNode, Virtual), M = #count { VirtualID : virtual_on_edge(PackageNode, ProviderNode, node(VirtualID, Virtual), _) }, M > 1.\n\n\nattr(\"virtual_on_edge\", PackageNode, ProviderNode, Virtual) :- virtual_on_edge(PackageNode, ProviderNode, node(_, Virtual), _).\nattr(\"depends_on\", PackageNode, ProviderNode, Type) :- virtual_on_edge(PackageNode, ProviderNode, _, Type).\n\n% If a virtual node is in the answer set, it must be either a virtual root,\n% or used somewhere\n:- attr(\"virtual_node\", node(_, Virtual)),\n not attr(\"virtual_on_incoming_edges\", _, Virtual),\n not attr(\"virtual_root\", node(_, Virtual)).\n\nattr(\"virtual_on_incoming_edges\", ProviderNode, Virtual)\n :- attr(\"virtual_on_edge\", _, ProviderNode, Virtual).\n\n% This is needed to allow requirement on virtuals,\n% when a virtual root is requested\nattr(\"virtual_on_incoming_edges\", ProviderNode, Virtual)\n :- attr(\"virtual_root\", node(min_dupe_id, Virtual)),\n attr(\"root\", ProviderNode),\n provider(ProviderNode, node(min_dupe_id, Virtual)).\n\n% If a virtual is needed on an edge, at least one virtual node must exist\n:- virtual_is_needed(Virtual), not 1 { attr(\"virtual_node\", node(0..X-1, Virtual)) : max_dupes(Virtual, X) }.\n\n% If there's a virtual node, we must select one and only one provider.\n% The provider must be selected among the possible providers.\n\nerror(100, \"'{0}' cannot be a provider for the '{1}' virtual\", Package, Virtual)\n :- attr(\"provider_set\", node(X, Package), node(Y, Virtual)),\n not virtual_condition_holds( node(X, Package), Virtual).\n\nerror(100, \"Cannot find valid provider for virtual {0}\", Virtual)\n :- attr(\"virtual_node\", node(X, Virtual)),\n not provider(_, node(X, Virtual)).\n\nerror(100, \"Cannot select a single provider for virtual '{0}'\", Virtual)\n :- attr(\"virtual_node\", node(X, Virtual)),\n 2 { provider(P, node(X, Virtual)) }.\n\n% virtual roots imply virtual nodes, and that one provider is a root\nattr(\"virtual_node\", VirtualNode) :- attr(\"virtual_root\", VirtualNode).\n\n% If we asked for a virtual root and we have a provider for that,\n% then the provider is the root package.\nattr(\"root\", PackageNode) :- attr(\"virtual_root\", VirtualNode), provider(PackageNode, VirtualNode).\n\n% The provider is selected among the nodes for which the virtual condition holds\n1 { provider(PackageNode, node(X, Virtual)) :\n attr(\"node\", PackageNode), virtual_condition_holds(PackageNode, Virtual) } 1\n :- attr(\"virtual_node\", node(X, Virtual)).\n\n% The provider provides the virtual if some provider condition holds.\nvirtual_condition_holds(node(ProviderID, Provider), Virtual) :- virtual_condition_holds(ID, node(ProviderID, Provider), Virtual).\nvirtual_condition_holds(ID, node(ProviderID, Provider), Virtual) :-\n pkg_fact(Provider, provider_condition(ID, Virtual)),\n condition_holds(ID, node(ProviderID, Provider)),\n virtual(Virtual).\n\n% If a \"provider\" condition holds, but this package is not a provider, do not impose the \"provider\" condition\ndo_not_impose(EffectID, node(X, Package))\n :- virtual_condition_holds(ID, node(X, Package), Virtual),\n pkg_fact(Package, condition_effect(ID, EffectID)),\n not provider(node(X, Package), node(_, Virtual)).\n\n% Choose the provider among root specs, if possible\n:- provider(ProviderNode, node(min_dupe_id, Virtual)),\n virtual_condition_holds(_, PossibleProvider, Virtual),\n PossibleProvider != ProviderNode,\n explicitly_requested_root(PossibleProvider),\n not self_build_requirement(PossibleProvider, ProviderNode),\n not explicitly_requested_root(ProviderNode),\n not language(Virtual).\n\n% A package cannot be the actual provider for a virtual if it does not\n% fulfill the conditions to provide that virtual\n:- provider(PackageNode, node(VirtualID, Virtual)),\n not virtual_condition_holds(PackageNode, Virtual).\n\n#defined provided_together/3.\n\n%-----------------------------------------------------------------------------\n% Virtual dependency weights\n%-----------------------------------------------------------------------------\n\n% Any configured provider has a weight based on index in the preference list\nprovider_weight(node(ProviderID, Provider), node(VirtualID, Virtual), Weight)\n :- provider(node(ProviderID, Provider), node(VirtualID, Virtual)),\n provider_weight_from_config(Virtual, Provider, Weight).\n\n% Any non-configured provider has a default weight of 100\nprovider_weight(node(ProviderID, Provider), node(VirtualID, Virtual), 100)\n :- provider(node(ProviderID, Provider), node(VirtualID, Virtual)),\n not provider_weight_from_config(Virtual, Provider, _).\n\n% do not warn if generated program contains none of these.\n#defined virtual/1.\n#defined buildable_false/1.\n#defined provider_weight_from_config/3.\n\n%-----------------------------------------------------------------------------\n% External semantics\n%-----------------------------------------------------------------------------\n\nexternal(PackageNode) :- attr(\"external\", PackageNode).\n\n% if a package is not buildable, only concrete specs are allowed\nerror(1000, \"Cannot build {0}, since it is configured `buildable:false` and no externals satisfy the request\", Package)\n :- buildable_false(Package),\n attr(\"node\", node(ID, Package)),\n build(node(ID, Package)).\n\n% Account for compiler annotation on externals\n:- not attr(\"root\", ExternalNode),\n attr(\"external_build_requirement\", ExternalNode, node_requirement(\"node\", Compiler)),\n not node_compiler(_, node(_, Compiler)).\n\n1 { attr(\"node_version_satisfies\", node(X, Compiler), Constraint) : node_compiler(_, node(X, Compiler)) }\n :- not attr(\"root\", ExternalNode),\n attr(\"external_build_requirement\", ExternalNode, node_requirement(\"node\", Compiler)),\n attr(\"external_build_requirement\", ExternalNode, node_requirement(\"node_version_satisfies\", Compiler, Constraint)).\n\n%-----------------------------------------------------------------------------\n% Config required semantics\n%-----------------------------------------------------------------------------\n\npackage_in_dag(Node) :- attr(\"node\", Node).\npackage_in_dag(Node) :- attr(\"virtual_node\", Node).\npackage_in_dag(Node) :- attr(\"reused_virtual_node\", Node).\n\nreused_provider(node(CompilerHash, CompilerName), node(CompilerHash, Virtual)) :-\n language(Virtual),\n attr(\"hash\", PackageNode, Hash),\n attr(\"concrete_build_dependency\", PackageNode, CompilerName, CompilerHash),\n attr(\"virtual_on_build_edge\", PackageNode, CompilerName, Virtual).\n\nattr(\"reused_virtual_node\", VirtualNode) :- reused_provider(_, VirtualNode).\n\ntrigger_node(ID, node(PackageID, Package), node(VirtualID, Virtual)) :- pkg_fact(Virtual, trigger_id(ID)), reused_provider(node(PackageID, Package), node(VirtualID, Virtual)).\n\n\nactivate_requirement(node(ID, Package), X) :-\n package_in_dag(node(ID, Package)),\n requirement_group(Package, X),\n not requirement_conditional(Package, X, _).\n\nactivate_requirement(node(ID, Package), X) :-\n package_in_dag(node(ID, Package)),\n requirement_group(Package, X),\n condition_holds(Y, node(ID, Package)),\n requirement_conditional(Package, X, Y).\n\nactivate_requirement(node(ID, Package), X) :-\n package_in_dag(node(ID, Package)),\n package_in_dag(node(CID, ConditionPackage)),\n requirement_group(Package, X),\n pkg_fact(ConditionPackage, condition(Y)),\n related(node(CID, ConditionPackage), node(ID, Package)),\n condition_holds(Y, node(CID, ConditionPackage)),\n requirement_conditional(Package, X, Y),\n ConditionPackage != Package.\n\nrequirement_group_satisfied(node(ID, Package), GroupID) :-\n 1 { requirement_is_met(GroupID, ConditionID, node(ID, Package)) } 1,\n requirement_policy(Package, GroupID, \"one_of\"),\n activate_requirement(node(ID, Package), GroupID),\n requirement_group(Package, GroupID).\n\nrequirement_weight(node(ID, Package), Group, W) :-\n condition_holds(Y, node(ID, Package)),\n requirement_has_weight(Y, W),\n requirement_group_member(Y, Package, Group),\n requirement_policy(Package, Group, \"one_of\"),\n requirement_group_satisfied(node(ID, Package), Group).\n\n { attr(\"direct_dependency\", node(ID, Package), BuildRequirement) : condition_requirement(TriggerID, \"direct_dependency\", Package, BuildRequirement) } :-\n pkg_fact(Package, condition_trigger(ConditionID, TriggerID)),\n requirement_group_member(ConditionID, Package, Group),\n activate_requirement(node(ID, Package), Group),\n requirement_group(Package, Group).\n\nrequirement_group_satisfied(node(ID, Package), GroupID) :-\n 1 { requirement_is_met(GroupID, ConditionID, node(ID, Package)) } ,\n requirement_policy(Package, GroupID, \"any_of\"),\n activate_requirement(node(ID, Package), GroupID),\n requirement_group(Package, GroupID).\n\n% Derive a fact to represent that the entire requirement, including possible subconditions, is met\nrequirement_is_met(GroupID, ConditionID, node(X, Package)) :-\n requirement_group_member(ConditionID, Package, GroupID),\n condition_holds(ConditionID, node(X, Package)),\n not subcondition(_, ConditionID).\n\nrequirement_is_met(GroupID, ConditionID, node(X, Package)) :-\n requirement_group_member(ConditionID, Package, GroupID),\n condition_holds(ConditionID, node(X, Package)),\n impose(EffectID, node(X, Package)) : subcondition(SubconditionID, ConditionID), pkg_fact(Package, condition_effect(SubconditionID, EffectID)), condition_holds(SubconditionID, node(X, Package));\n subcondition(_, ConditionID).\n\n{ do_not_impose(EffectID, node(X, Package)) } :-\n requirement_group_member(ConditionID, Package, GroupID),\n condition_holds(ConditionID, node(X, Package)),\n subcondition(SubconditionID, ConditionID),\n pkg_fact(Package, condition_effect(SubconditionID, EffectID)).\n\n% clingo decided not to impose a condition for a subcondition that holds\nexclude_requirement_weight(ConditionID, Package, GroupID) :-\n requirement_group_member(ConditionID, Package, GroupID),\n condition_holds(ConditionID, node(X, Package)),\n subcondition(SubconditionID, ConditionID),\n condition_holds(SubconditionID, node(X, Package)),\n do_not_impose(EffectID, node(X, Package)),\n pkg_fact(Package, condition_effect(SubconditionID, EffectID)).\n\n% Do not impose requirements, if the conditional requirement is not active\ndo_not_impose(EffectID, node(ID, Package)) :-\n trigger_condition_holds(TriggerID, node(ID, Package)),\n pkg_fact(Package, condition_trigger(ConditionID, TriggerID)),\n pkg_fact(Package, condition_effect(ConditionID, EffectID)),\n requirement_group_member(ConditionID , Package, RequirementID),\n not activate_requirement(node(ID, Package), RequirementID).\n\ndo_not_impose(EffectID, node(Hash, Virtual)) :-\n trigger_condition_holds(TriggerID, node(Hash, Virtual)),\n pkg_fact(Virtual, condition_trigger(ConditionID, TriggerID)),\n pkg_fact(Virtual, condition_effect(ConditionID, EffectID)),\n requirement_group_member(ConditionID , Virtual, RequirementID),\n reused_provider(_, node(Hash, Virtual)),\n activate_requirement(node(Hash, Virtual), RequirementID).\n\n% When we have a required provider, we need to ensure that the provider/2 facts respect\n% the requirement. This is particularly important for packages that could provide multiple\n% virtuals independently\nrequired_provider(Provider, Virtual)\n :- requirement_group_member(ConditionID, Virtual, RequirementID),\n condition_holds(ConditionID, _),\n virtual(Virtual),\n pkg_fact(Virtual, condition_effect(ConditionID, EffectID)),\n imposed_constraint(EffectID, \"node\", Provider).\n\nerror(1, \"Cannot use {1} for the {0} virtual, but that is required\", Virtual, Provider) :-\n required_provider(Provider, Virtual),\n not reused_provider(node(_, Provider), node(_, Virtual)),\n not provider(node(_, Provider), node(_, Virtual)).\n\n% TODO: the following choice rule allows the solver to add compiler\n% flags if their only source is from a requirement. This is overly-specific\n% and should use a more-generic approach like in https://github.com/spack/spack/pull/37180\n\n{ attr(\"node_flag\", node(ID, Package), NodeFlag) } :-\n requirement_group_member(ConditionID, Package, RequirementID),\n activate_requirement(node(ID, Package), RequirementID),\n pkg_fact(Package, condition_effect(ConditionID, EffectID)),\n imposed_constraint(EffectID, \"node_flag_set\", Package, NodeFlag).\n\n{ attr(\"node_flag\", node(ID, Package), NodeFlag) } :-\n requirement_group_member(ConditionID, Virtual, RequirementID),\n activate_requirement(node(VirtualID, Virtual), RequirementID),\n provider(node(ID, Package), VirtualNode),\n pkg_fact(Virtual, condition_effect(ConditionID, EffectID)),\n imposed_constraint(EffectID, \"node_flag_set\", Package, NodeFlag).\n\nrequirement_weight(node(ID, Package), Group, W) :-\n W = #min {\n Z : requirement_has_weight(Y, Z),\n condition_holds(Y, node(ID, Package)),\n requirement_group_member(Y, Package, Group),\n not exclude_requirement_weight(Y, Package, Group);\n % We need this to avoid an annoying warning during the solve\n % concretize.lp:1151:5-11: info: tuple ignored:\n % #sup@73\n 10000\n },\n requirement_policy(Package, Group, \"any_of\"),\n requirement_group_satisfied(node(ID, Package), Group).\n\nrequirement_penalty(node(ID, Package), Group, W) :-\n requirement_weight(node(ID, Package), Group, W),\n not language(Package).\n\nrequirement_penalty(PackageNode, Language, Group, W) :-\n requirement_weight(node(ID, Language), Group, W),\n language(Language),\n provider(ProviderNode, node(ID, Language)),\n attr(\"virtual_on_edge\", PackageNode, ProviderNode, Language).\n\nrequirement_penalty(PackageNode, Language, Group, W) :-\n requirement_weight(node(CompilerHash, Language), Group, W),\n language(Language),\n reused_provider(node(CompilerHash, CompilerName), node(CompilerHash, Language)),\n attr(\"concrete_build_dependency\", PackageNode, CompilerName, CompilerHash),\n attr(\"virtual_on_build_edge\", PackageNode, CompilerName, Language).\n\nerror(60000, \"cannot satisfy a requirement for package '{0}'.\", Package) :-\n activate_requirement(node(ID, Package), X),\n requirement_group(Package, X),\n not requirement_message(Package, X, _),\n not requirement_group_satisfied(node(ID, Package), X).\n\n\nerror(50000, Message) :-\n activate_requirement(node(ID, Package), X),\n requirement_group(Package, X),\n requirement_message(Package, X, Message),\n not requirement_group_satisfied(node(ID, Package), X).\n\n\n#defined requirement_group/2.\n#defined requirement_conditional/3.\n#defined requirement_message/3.\n#defined requirement_group_member/3.\n#defined requirement_has_weight/2.\n#defined requirement_policy/3.\n\n%-----------------------------------------------------------------------------\n% Variant semantics\n%-----------------------------------------------------------------------------\n% Packages define potentially several definitions for each variant, and depending\n% on their attributes, duplicate nodes for the same package may use different\n% definitions. So the variant logic has several jobs:\n% A. Associate a variant definition with a node, by VariantID\n% B. Associate defaults and attributes (sticky, etc.) for the selected variant ID with the node.\n% C. Once these rules are established for a node, select variant value(s) based on them.\n\n% A: Selecting a variant definition\n\n% Variant definitions come from package facts in two ways:\n% 1. unconditional variants are always defined on all nodes for a given package\nvariant_definition(node(ID, Package), Name, VariantID) :-\n pkg_fact(Package, variant_definition(Name, VariantID)),\n attr(\"node\", node(ID, Package)).\n\n% 2. conditional variants are only defined if the conditions hold for the node\nvariant_definition(node(ID, Package), Name, VariantID) :-\n pkg_fact(Package, variant_condition(Name, VariantID, ConditionID)),\n condition_holds(ConditionID, node(ID, Package)).\n\n% If there are any definitions for a variant on a node, the variant is \"defined\".\nvariant_defined(PackageNode, Name) :- variant_definition(PackageNode, Name, _).\n\n% Solver must pick the variant definition with the highest id. When conditions hold\n% for two or more variant definitions, this prefers the last one defined.\nnode_has_variant(PackageNode, Name, SelectedVariantID) :-\n SelectedVariantID = #max { VariantID : variant_definition(PackageNode, Name, VariantID) },\n variant_defined(PackageNode, Name).\n\n% B: Associating applicable package rules with nodes\n\n% The default value for a variant in a package is what is prescribed:\n% 1. On the command line\n% 2. In packages.yaml (if there's no command line settings)\n% 3. In the package.py file (if there are no settings in packages.yaml and the command line)\n\n% -- Associate the definition's default values with the node\n% note that the package.py variant defaults are associated with a particular definition, but\n% packages.yaml and CLI are associated with just the variant name.\n% Also, settings specified on the CLI apply to all duplicates, but always have\n% `min_dupe_id` as their node id.\nvariant_default_value(node(ID, Package), VariantName, Value) :-\n node_has_variant(node(ID, Package), VariantName, VariantID),\n pkg_fact(Package, variant_default_value_from_package_py(VariantID, Value)),\n not variant_default_value_from_packages_yaml(Package, VariantName, _),\n not attr(\"variant_default_value_from_cli\", node(min_dupe_id, Package), VariantName, _).\n\nvariant_default_value(node(ID, Package), VariantName, Value) :-\n node_has_variant(node(ID, Package), VariantName, _),\n variant_default_value_from_packages_yaml(Package, VariantName, Value),\n not attr(\"variant_default_value_from_cli\", node(min_dupe_id, Package), VariantName, _).\n\nvariant_default_value(node(ID, Package), VariantName, Value) :-\n node_has_variant(node(ID, Package), VariantName, _),\n attr(\"variant_default_value_from_cli\", node(min_dupe_id, Package), VariantName, Value).\n\n% Penalty from the variant definition\npossible_variant_penalty(VariantID, Value, Penalty) :- pkg_fact(Package, variant_penalty(VariantID, Value, Penalty)).\n\n% Use a very high penalty for variant values that are not defined in the package,\n% for instance those defined implicitly by a validator.\npossible_variant_penalty(VariantID, Value, 100) :-\n pkg_fact(Package, variant_possible_value(VariantID, Value)),\n not pkg_fact(Package, variant_penalty(VariantID, Value, _)).\n\nvariant_penalty(node(NodeID, Package), Variant, Value, Penalty) :-\n node_has_variant(node(NodeID, Package), Variant, VariantID),\n attr(\"variant_value\", node(NodeID, Package), Variant, Value),\n possible_variant_penalty(VariantID, Value, Penalty),\n not variant_default_value(node(NodeID, Package), Variant, Value),\n % variants set explicitly from a directive don't count as non-default\n not attr(\"variant_set\", node(NodeID, Package), Variant, Value),\n % variant values forced by propagation don't count as non-default\n not propagate(node(NodeID, Package), variant_value(Variant, Value, _)).\n\n% -- Associate the definition's possible values with the node\nvariant_possible_value(node(ID, Package), VariantName, Value) :-\n node_has_variant(node(ID, Package), VariantName, VariantID),\n pkg_fact(Package, variant_possible_value(VariantID, Value)).\n\nvariant_possible_value(node(ID, Package), VariantName, Value) :-\n node_has_variant(node(ID, Package), VariantName, VariantID),\n pkg_fact(Package, variant_penalty(VariantID, Value, _)).\n\nvariant_value_from_disjoint_sets(node(ID, Package), VariantName, Value1, Set1) :-\n node_has_variant(node(ID, Package), VariantName, VariantID),\n pkg_fact(Package, variant_value_from_disjoint_sets(VariantID, Value1, Set1)).\n\n% -- Associate definition's arity with the node\nvariant_single_value(node(ID, Package), VariantName) :-\n node_has_variant(node(ID, Package), VariantName, VariantID),\n variant_type(VariantID, VariantType),\n VariantType != \"multi\".\n\n% C: Determining variant values on each node\n\n% if a variant is sticky, but not set, its value is the default value\nattr(\"variant_value\", node(ID, Package), Variant, Value) :-\n node_has_variant(node(ID, Package), Variant, VariantID),\n variant_default_value(node(ID, Package), Variant, Value),\n pkg_fact(Package, variant_sticky(VariantID)),\n not attr(\"variant_set\", node(ID, Package), Variant),\n build(node(ID, Package)).\n\n% we can choose variant values from all the possible values for the node\n1 {\n attr(\"variant_value\", PackageNode, Variant, Value) : variant_possible_value(PackageNode, Variant, Value)\n} :-\n node_has_variant(PackageNode, Variant, _),\n build(PackageNode).\n\n% variant_selected is only needed for reconstruction on the python side\nattr(\"variant_selected\", PackageNode, Variant, Value, VariantType, VariantID) :-\n attr(\"variant_value\", PackageNode, Variant, Value),\n node_has_variant(PackageNode, Variant, VariantID),\n variant_type(VariantID, VariantType).\n\n% a variant cannot be set if it is not a variant on the package\nerror(100, \"Cannot set variant '{0}' for package '{1}' because the variant condition cannot be satisfied for the given spec\", Variant, Package)\n :- attr(\"variant_set\", node(ID, Package), Variant),\n not node_has_variant(node(ID, Package), Variant, _),\n build(node(ID, Package)).\n\n% a variant cannot take on a value if it is not a variant of the package\nerror(100, \"Cannot set variant '{0}' for package '{1}' because the variant condition cannot be satisfied for the given spec\", Variant, Package)\n :- attr(\"variant_value\", node(ID, Package), Variant, _),\n not node_has_variant(node(ID, Package), Variant, _),\n build(node(ID, Package)).\n\n% at most one variant value for single-valued variants.\nerror(1000, \"'{0}' requires conflicting variant values 'Spec({1}={2})' and 'Spec({1}={3})'\", Package, Variant, Value1, Value2)\n :- attr(\"node\", node(ID, Package)),\n node_has_variant(node(ID, Package), Variant, _),\n variant_single_value(node(ID, Package), Variant),\n attr(\"variant_value\", node(ID, Package), Variant, Value1),\n attr(\"variant_value\", node(ID, Package), Variant, Value2),\n Value1 < Value2,\n build(node(ID, Package)).\n\nerror(100, \"No valid value for variant '{1}' of package '{0}'\", Package, Variant)\n :- attr(\"node\", node(ID, Package)),\n node_has_variant(node(ID, Package), Variant, _),\n build(node(ID, Package)),\n not attr(\"variant_value\", node(ID, Package), Variant, _).\n\n% if a variant is set to anything, it is considered 'set'.\nattr(\"variant_set\", PackageNode, Variant) :- attr(\"variant_set\", PackageNode, Variant, _).\n\n% Setting a concrete variant implies setting a variant\nconcrete_variant_value(PackageNode, Variant, Value, Origin) :- attr(\"concrete_variant_set\", PackageNode, Variant, Value, Origin).\n\nattr(\"variant_set\", PackageNode, Variant, Value) :- attr(\"concrete_variant_set\", PackageNode, Variant, Value, _).\n\n% Concrete variant values must be in the answer set\n:- concrete_variant_value(PackageNode, Variant, Value, _), not attr(\"variant_value\", PackageNode, Variant, Value).\n\n% Extra variant values are not allowed, if the variant is concrete\nvariant_is_concrete(PackageNode, Variant, Origin) :- concrete_variant_value(PackageNode, Variant, _, Origin).\n\nerror(100, \"The variant {0} in package {1} specified as := has the extra value {2}\", Variant, PackageNode, Value)\n:- variant_is_concrete(PackageNode, Variant, Origin),\n attr(\"variant_value\", PackageNode, Variant, Value),\n not concrete_variant_value(PackageNode, Variant, Value, Origin).\n\n% A variant cannot have a value that is not also a possible value\n% This only applies to packages we need to build -- concrete packages may\n% have been built w/different variants from older/different package versions.\nerror(10, \"'Spec({1}={2})' is not a valid value for '{0}' variant '{1}'\", Package, Variant, Value)\n :- attr(\"variant_value\", node(ID, Package), Variant, Value),\n not variant_possible_value(node(ID, Package), Variant, Value),\n build(node(ID, Package)).\n\n% Some multi valued variants accept multiple values from disjoint sets. Ensure that we\n% respect that constraint and we don't pick values from more than one set at once\nerror(100, \"{0} variant '{1}' cannot have values '{2}' and '{3}' as they come from disjoint value sets\", Package, Variant, Value1, Value2)\n :- attr(\"variant_value\", node(ID, Package), Variant, Value1),\n attr(\"variant_value\", node(ID, Package), Variant, Value2),\n variant_value_from_disjoint_sets(node(ID, Package), Variant, Value1, Set1),\n variant_value_from_disjoint_sets(node(ID, Package), Variant, Value2, Set2),\n Set1 < Set2, % see[1]\n build(node(ID, Package)).\n\n:- attr(\"variant_set\", node(ID, Package), Variant, Value),\n not attr(\"variant_value\", node(ID, Package), Variant, Value).\n\n% In a case with `variant(\"foo\", when=\"+bar\")` and a user request for +foo or ~foo,\n% force +bar to be set too. This gives no penalty if `+bar` is not the default value, and\n% optimizes a long chain of deductions that may cause clingo to hang.\nattr(\"variant_set\", node(ID, Package), AnotherVariant, AnotherValue)\n :- attr(\"variant_set\", node(ID, Package), Variant, Value),\n attr(\"variant_selected\", node(ID, Package), Variant, _, _, VariantID),\n pkg_fact(Package, variant_condition(Variant, VariantID, ConditionID)),\n pkg_fact(Package, condition_trigger(ConditionID, TriggerID)),\n condition_requirement(TriggerID,\"variant_value\",Package, AnotherVariant, AnotherValue),\n build(node(ID, Package)).\n\n% A default variant value that is not used, makes sense only for multi valued variants\nvariant_default_not_used(node(ID, Package), Variant, Value)\n :- variant_default_value(node(ID, Package), Variant, Value),\n node_has_variant(node(ID, Package), Variant, VariantID),\n variant_type(VariantID, VariantType), VariantType == \"multi\",\n not attr(\"variant_value\", node(ID, Package), Variant, Value),\n not propagate(node(ID, Package), variant_value(Variant, Value, _)),\n % variant set explicitly don't count for this metric\n not attr(\"variant_set\", node(ID, Package), Variant, Value),\n attr(\"node\", node(ID, Package)).\n\n% Treat 'none' in a special way - it cannot be combined with other\n% values even if the variant is multi-valued\nerror(100, \"{0} variant '{1}' cannot have values '{2}' and 'none'\", Package, Variant, Value)\n :- attr(\"variant_value\", node(X, Package), Variant, Value),\n attr(\"variant_value\", node(X, Package), Variant, \"none\"),\n Value != \"none\",\n build(node(X, Package)).\n\n% -- Auto variants\n% These don't have to be declared in the package. We allow them to spring into\n% existence when assigned a value.\nvariant_possible_value(PackageNode, Variant, Value)\n :- attr(\"variant_set\", PackageNode, Variant, Value), auto_variant(Variant, _).\n\nnode_has_variant(PackageNode, Variant, VariantID)\n :- attr(\"variant_set\", PackageNode, Variant, _), auto_variant(Variant, VariantID).\n\nvariant_single_value(PackageNode, Variant)\n :- node_has_variant(PackageNode, Variant, VariantID),\n auto_variant(Variant, VariantID),\n variant_type(VariantID, VariantType),\n VariantType != \"multi\".\n\n% to respect requirements/preferences we need to define that an auto_variant is set\n{ attr(\"variant_set\", node(ID, Package), Variant, VariantValue)}\n :- auto_variant(Variant, _ ),\n condition_requirement(TriggerID, \"variant_value\", Package, Variant, VariantValue),\n pkg_fact(Package, condition_trigger(ConditionID, TriggerID)),\n requirement_group_member(ConditionID , Package, RequirementID),\n activate_requirement(node(ID, Package), RequirementID).\n\n% suppress warnings about this atom being unset. It's only set if some\n% spec or some package sets it, and without this, clingo will give\n% warnings like 'info: atom does not occur in any rule head'.\n#defined variant_default_value_from_packages_yaml/3.\n#defined variant_default_value_from_package_py/2.\n\n%-----------------------------------------------------------------------------\n% Propagation semantics\n%-----------------------------------------------------------------------------\n\nnon_default_propagation(variant_value(Name, Value)) :- attr(\"propagate\", RootNode, variant_value(Name, Value)).\n\n% Propagation roots have a corresponding attr(\"propagate\", ...)\npropagate(RootNode, PropagatedAttribute) :- attr(\"propagate\", RootNode, PropagatedAttribute), not non_default_propagation(PropagatedAttribute).\npropagate(RootNode, PropagatedAttribute, EdgeTypes) :- attr(\"propagate\", RootNode, PropagatedAttribute, EdgeTypes).\n\n% Special case variants, to inject the source node in the propagated attribute\npropagate(RootNode, variant_value(Name, Value, RootNode)) :- attr(\"propagate\", RootNode, variant_value(Name, Value)).\n\n% Propagate an attribute along edges to child nodes\npropagate(ChildNode, PropagatedAttribute) :-\n propagate(ParentNode, PropagatedAttribute),\n depends_on(ParentNode, ChildNode).\n\npropagate(ChildNode, PropagatedAttribute, edge_types(DepType1, DepType2)) :-\n propagate(ParentNode, PropagatedAttribute, edge_types(DepType1, DepType2)),\n depends_on(ParentNode, ChildNode),\n 1 { attr(\"depends_on\", ParentNode, ChildNode, DepType1); attr(\"depends_on\", ParentNode, ChildNode, DepType2) }.\n\n%-----------------------------------------------------------------------------\n% Activation of propagated values\n%-----------------------------------------------------------------------------\n\n%----\n% Variants\n%----\n\n% If a variant is propagated, and can be accepted, set its value\nattr(\"variant_value\", PackageNode, Variant, Value) :-\n propagate(PackageNode, variant_value(Variant, Value, _)),\n node_has_variant(PackageNode, Variant, _),\n variant_possible_value(PackageNode, Variant, Value).\n\n% If a variant is propagated, we cannot have extraneous values\nvariant_is_propagated(PackageNode, Variant) :-\n attr(\"variant_value\", PackageNode, Variant, Value),\n propagate(PackageNode, variant_value(Variant, Value, _)),\n not attr(\"variant_set\", PackageNode, Variant).\n\n:- variant_is_propagated(PackageNode, Variant),\n attr(\"variant_value\", PackageNode, Variant, Value),\n not propagate(PackageNode, variant_value(Variant, Value, _)).\n\nerror(100, \"{0} and {1} cannot both propagate variant '{2}' to the shared dependency: {3}\",\n Package1, Package2, Variant, Dependency) :-\n % The variant is a singlevalued variant\n variant_single_value(node(X, Package1), Variant),\n % Dependency is trying to propagate Variant with different values and is not the source package\n propagate(node(Z, Dependency), variant_value(Variant, Value1, node(X, Package1))),\n propagate(node(Z, Dependency), variant_value(Variant, Value2, node(Y, Package2))),\n % Package1 and Package2 and their values are different\n Package1 > Package2, Value1 != Value2,\n not propagate(node(Z, Dependency), variant_value(Variant, _, node(Z, Dependency))).\n\n% Cannot propagate the same variant from two different packages if one is a dependency of the other\nerror(100, \"{0} and {1} cannot both propagate variant '{2}'\", Package1, Package2, Variant) :-\n % The variant is a single-valued variant\n variant_single_value(node(X, Package1), Variant),\n % Package1 and Package2 and their values are different\n Package1 != Package2, Value1 != Value2,\n % Package2 is set to propagate the value from Package1\n propagate(node(Y, Package2), variant_value(Variant, Value2, node(X, Package2))),\n propagate(node(Y, Package2), variant_value(Variant, Value1, node(X, Package1))),\n variant_is_propagated(node(Y, Package2), Variant).\n\n% Cannot propagate a variant if a different value was set for it in a dependency\nerror(100, \"Cannot propagate the variant '{0}' from the package: {1} because package: {2} is set to exclude it\", Variant, Source, Package) :-\n % Package has a Variant and Source is propagating Variant\n attr(\"variant_set\", node(X, Package), Variant, Value1),\n % The packages and values are different\n Source != Package, Value1 != Value2,\n % The variant is a single-valued variant\n variant_single_value(node(X, Package1), Variant),\n % A different value is being propagated from somewhere else\n propagate(node(X, Package), variant_value(Variant, Value2, node(Y, Source))).\n\n%----\n% Flags\n%----\n\n% A propagated flag implies:\n% 1. The same flag type is not set on this node\n% 2. This node has the same compilers as the propagation source\n\nnode_compiler(node(X, Package), node(Y, Compiler)) :- node_compiler(node(X, Package), node(Y, Compiler), Language).\n\nnode_compiler(node(X, Package), node(Y, Compiler), Language) :-\n attr(\"virtual_on_edge\", node(X, Package), node(Y, Compiler), Language),\n compiler(Compiler), language(Language).\n\npropagated_flag(node(PackageID, Package), node_flag(FlagType, Flag, FlagGroup, Source), SourceNode) :-\n propagate(node(PackageID, Package), node_flag(FlagType, Flag, FlagGroup, Source), _),\n not attr(\"node_flag_set\", node(PackageID, Package), node_flag(FlagType, _, _, \"literal\")),\n % Same compilers as propagation source\n node_compiler(node(PackageID, Package), CompilerNode, Language) : node_compiler(SourceNode, CompilerNode, Language);\n attr(\"propagate\", SourceNode, node_flag(FlagType, Flag, FlagGroup, Source), _),\n node(PackageID, Package) != SourceNode,\n not runtime(Package).\n\nattr(\"node_flag\", PackageNode, NodeFlag) :- propagated_flag(PackageNode, NodeFlag, _).\n\n% Cannot propagate the same flag from two distinct sources\nerror(100, \"{0} and {1} cannot both propagate compiler flags '{2}' to {3}\", Source1, Source2, FlagType, Package) :-\n propagated_flag(node(ID, Package), node_flag(FlagType, _, _, _), node(_, Source1)),\n propagated_flag(node(ID, Package), node_flag(FlagType, _, _, _), node(_, Source2)),\n Source1 < Source2.\n\n%----\n% Compiler constraints\n%----\n\n% If a node is built, impose constraints on the compiler coming from dependents\nattr(\"node_version_satisfies\", node(Y, Compiler), VersionRange) :-\n propagate(node(X, Package), node_version_satisfies(Compiler, VersionRange)),\n attr(\"depends_on\", node(X, Package), node(Y, Compiler), \"build\"),\n not runtime(Package).\n\nattr(\"node_version_satisfies\", node(X, Runtime), VersionRange) :-\n attr(\"node\", node(X, Runtime)),\n attr(\"compatible_runtime\", PackageNode, Runtime, VersionRange),\n concrete(PackageNode).\n\n% If a compiler package is depended on with type link, it's used as a library\ncompiler_used_as_a_library(node(X, Child), Hash) :-\n concrete(node(X, Child)),\n attr(\"hash\", node(X, Child), Hash),\n compiler_package(Child), % Used to restrict grounding for this rule\n attr(\"depends_on\", _, node(X, Child), \"link\").\n\n% If a compiler is used for C on a package, it must provide C++ too, if need be, and vice-versa\n:- attr(\"virtual_on_edge\", PackageNode, CompilerNode1, \"c\"),\n attr(\"virtual_on_edge\", PackageNode, CompilerNode2, \"cxx\"),\n CompilerNode1 != CompilerNode2.\n\n% Compiler-unmixing: 1st rule\nunification_set_compiler(\"root\", CompilerNode, Language) :-\n node_compiler(node(ID, Package), CompilerNode, Language),\n no_compiler_mixing(Language),\n not allow_mixing(Package),\n unification_set(\"root\", node(ID, Package)).\n\n% Compiler for a reused node\n% This differs from compiler_from_reuse in that this is about x->y\n% where y is a compiler and x is reused (compiler_from_reuse is\n% is concerned with reuse of the compiler itself)\nreused_node_compiler(PackageNode, node(CompilerHash, Compiler), Language) :-\n concrete(PackageNode),\n attr(\"concrete_build_dependency\", PackageNode, Compiler, CompilerHash),\n attr(\"virtual_on_build_edge\", PackageNode, Compiler, Language),\n language(Language).\n\n% Compiler-unmixing: 2nd rule\n% The compiler appears on a reused node as well as a built node. In\n% that case there will be a generated node() with an ID.\n% While easier to understand than rule 3, in fact this rule addresses\n% a small set of use cases beyond rules 1 and 3: generally speaking\n% rule 1 ensures that all non-reused nodes get a consistent compiler.\n% Rule 3 generates compiler IDs that almost always fail the count\n% rule, but does not \"activate\" when in combination with rule 1 when\n% it is possible to propagate a compiler to another built node in the\n% unification set. This in fact is only really used when the reused\n% node compiler has a node(), but associated with a different\n% unification set.\nunification_set_compiler(\"root\", CompilerNode, Language) :-\n reused_node_compiler(node(ID, Package), node(CompilerHash, Compiler), Language),\n attr(\"hash\", CompilerNode, CompilerHash),\n no_compiler_mixing(Language),\n not allow_mixing(Package),\n unification_set(\"root\", node(ID, Package)).\n\n% Compiler-unmixing: 3rd rule\n% If the compiler only appears in reused nodes, then there is no node()\n% for it; this will always generate an error unless all nodes in the\n% root unification set are reused.\nunification_set_compiler(\"root\", node(CompilerHash, Compiler), Language) :-\n reused_node_compiler(node(ID, Package), node(CompilerHash, Compiler), Language),\n not attr(\"hash\", _, CompilerHash),\n no_compiler_mixing(Language),\n not allow_mixing(Package),\n unification_set(\"root\", node(ID, Package)).\n\n#defined no_compiler_mixing/1.\n#defined allow_mixing/1.\n\n% You can't have >1 compiler for a given language if mixing is disabled\nerror(100, \"Compiler mixing is disabled for the {0} language\", Language) :-\n language(Language),\n\t #count { CompilerNode : unification_set_compiler(\"root\", CompilerNode, Language) } > 1.\n\n%-----------------------------------------------------------------------------\n% Runtimes\n%-----------------------------------------------------------------------------\n\n% Check whether the DAG has any built package\nwill_build_packages() :- build(X).\n\n% \"gcc-runtime\" is always built\n:- concrete(node(X, \"gcc-runtime\")), will_build_packages().\n\n% The \"gcc\" linked to \"gcc-runtime\" must be used by at least another package\n:- attr(\"depends_on\", node(X, \"gcc-runtime\"), node(Y, \"gcc\"), \"build\"),\n node_compiler(_, node(_, \"gcc\")),\n not 2 { attr(\"depends_on\", PackageNode, node(Y, \"gcc\"), \"build\") : attr(\"node\", PackageNode) }.\n\n\n%-----------------------------------------------------------------------------\n% Platform semantics\n%-----------------------------------------------------------------------------\n\n% NOTE: Currently we have a single allowed platform per DAG, therefore there is no\n% need to have additional optimization criteria. If we ever add cross-platform dags,\n% this needs to be changed.\n:- 2 { allowed_platform(Platform) }.\n\n1 { attr(\"node_platform\", PackageNode, Platform) : allowed_platform(Platform) } 1\n :- attr(\"node\", PackageNode).\n\n% setting platform on a node is a hard constraint\nattr(\"node_platform\", PackageNode, Platform)\n :- attr(\"node\", PackageNode), attr(\"node_platform_set\", PackageNode, Platform).\n\n% platform is set if set to anything\nattr(\"node_platform_set\", PackageNode) :- attr(\"node_platform_set\", PackageNode, _).\n\n%-----------------------------------------------------------------------------\n% OS semantics\n%-----------------------------------------------------------------------------\n% convert weighted OS declarations to simple one\nos(OS) :- os(OS, _).\n\n% one os per node\n{ attr(\"node_os\", PackageNode, OS) : os(OS) } :- attr(\"node\", PackageNode).\n\n% can't have a non-buildable OS on a node we need to build\nerror(100, \"Cannot select '{0} os={1}' (operating system '{1}' is not buildable)\", Package, OS)\n :- build(node(X, Package)),\n attr(\"node_os\", node(X, Package), OS),\n not buildable_os(OS).\n\n% give OS choice weights according to os declarations\nnode_os_weight(PackageNode, Weight)\n :- attr(\"node\", PackageNode),\n attr(\"node_os\", PackageNode, OS),\n os(OS, Weight).\n\n% every OS is compatible with itself. We can use `os_compatible` to declare\nos_compatible(OS, OS) :- os(OS).\n\n% Transitive compatibility among operating systems\nos_compatible(OS1, OS3) :- os_compatible(OS1, OS2), os_compatible(OS2, OS3).\n\n% If an OS is set explicitly respect the value\nattr(\"node_os\", PackageNode, OS) :- attr(\"node_os_set\", PackageNode, OS), attr(\"node\", PackageNode).\n\n#defined os_compatible/2.\n\n%-----------------------------------------------------------------------------\n% Target semantics\n%-----------------------------------------------------------------------------\n\n% Each node has only one target chosen among the known targets\n1 { attr(\"node_target\", PackageNode, Target) : target(Target) } 1 :- attr(\"node\", PackageNode).\n\n% If a node must satisfy a target constraint, enforce it\nerror(10, \"'{0} target={1}' cannot satisfy constraint 'target={2}'\", Package, Target, Constraint)\n :- attr(\"node_target\", node(X, Package), Target),\n attr(\"node_target_satisfies\", node(X, Package), Constraint),\n not target_satisfies(Constraint, Target).\n\n% If a node has a target and the target satisfies a constraint, then the target\n% associated with the node satisfies the same constraint\nattr(\"node_target_satisfies\", PackageNode, Constraint)\n :- attr(\"node_target\", PackageNode, Target), target_satisfies(Constraint, Target).\n\n% If a node has a target, all of its dependencies must be compatible with that target\nerror(100, \"Cannot find compatible targets for {0} and {1}\", Package, Dependency)\n :- attr(\"depends_on\", node(X, Package), node(Y, Dependency), Type), Type != \"build\",\n attr(\"node_target\", node(X, Package), Target),\n not node_target_compatible(node(Y, Dependency), Target).\n\n% Intermediate step for performance reasons\n% When the integrity constraint above was formulated including this logic\n% we suffered a substantial performance penalty\nnode_target_compatible(ChildNode, ParentTarget)\n :- attr(\"node_target\", ChildNode, ChildTarget),\n target_compatible(ParentTarget, ChildTarget).\n\n#defined target_satisfies/2.\ncompiler(Compiler) :- target_supported(Compiler, _, _).\n\n% Can't use targets on node if the compiler for the node doesn't support them\nlanguage(\"c\").\nlanguage(\"cuda-lang\").\nlanguage(\"cxx\").\nlanguage(\"fortran\").\nlanguage(\"hip-lang\").\nlanguage_runtime(\"fortran-rt\").\n\nerror(10, \"Only external, or concrete, compilers are allowed for the {0} language\", Language)\n :- provider(ProviderNode, node(_, Language)),\n language(Language),\n build(ProviderNode).\n\nerror(10, \"{0} compiler '{2}@{3}' incompatible with 'target={1}'\", Package, Target, Compiler, Version)\n :- attr(\"node_target\", node(X, Package), Target),\n node_compiler(node(X, Package), node(Y, Compiler)),\n attr(\"version\", node(Y, Compiler), Version),\n target_not_supported(Compiler, Version, Target),\n build(node(X, Package)).\n\n#defined target_supported/3.\n#defined target_not_supported/3.\n\n% if a target is set explicitly, respect it\nattr(\"node_target\", PackageNode, Target)\n :- attr(\"node\", PackageNode), attr(\"node_target_set\", PackageNode, Target).\n\nnode_target_weight(PackageNode, MinWeight)\n :- attr(\"node\", PackageNode),\n attr(\"node_target\", PackageNode, Target),\n target(Target),\n MinWeight = #min { Weight : target_weight(Target, Weight) }.\n\n:- attr(\"node_target\", PackageNode, Target), not node_target_weight(PackageNode, _).\n\n% compatibility rules for targets among nodes\nnode_target_match(ParentNode, DependencyNode)\n :- attr(\"depends_on\", ParentNode, DependencyNode, Type), Type != \"build\",\n attr(\"node_target\", ParentNode, Target),\n attr(\"node_target\", DependencyNode, Target).\n\nnode_target_mismatch(ParentNode, DependencyNode)\n :- attr(\"depends_on\", ParentNode, DependencyNode, Type), Type != \"build\",\n not node_target_match(ParentNode, DependencyNode).\n\n% disallow reusing concrete specs that don't have a compatible target\nerror(100, \"'{0} target={1}' is not compatible with this machine\", Package, Target)\n :- attr(\"node\", node(X, Package)),\n attr(\"node_target\", node(X, Package), Target),\n not target(Target).\n\n%-----------------------------------------------------------------------------\n% Compiler flags\n%-----------------------------------------------------------------------------\n\nattr(\"node_flag\", PackageNode, NodeFlag) :- attr(\"node_flag_set\", PackageNode, NodeFlag).\n\n% If we set \"foo %bar cflags=A ^fee %bar cflags=B\" we want two nodes for \"bar\"\nerror(100, \"Cannot set multiple {0} values for {1} from cli\", FlagType, Package)\n:- attr(\"node_flag_set\", node(X, Package), node_flag(FlagType, _, FlagGroup1, \"literal\")),\n attr(\"node_flag_set\", node(X, Package), node_flag(FlagType, _, FlagGroup2, \"literal\")),\n FlagGroup1 < FlagGroup2.\n\n%-----------------------------------------------------------------------------\n% Installed Packages\n%-----------------------------------------------------------------------------\n\n#defined installed_hash/2.\n#defined abi_splice_conditions_hold/4.\n\n% These are the previously concretized attributes of the installed package as\n% a hash. It has the general form:\n% hash_attr(Hash, Attribute, PackageName, Args*)\n#defined hash_attr/3.\n#defined hash_attr/4.\n#defined hash_attr/5.\n#defined hash_attr/6.\n#defined hash_attr/7.\n\n{ attr(\"hash\", node(ID, PackageName), Hash): installed_hash(PackageName, Hash) } 1 :-\n attr(\"node\", node(ID, PackageName)).\n% you can't choose an installed hash for a dev spec\n:- attr(\"hash\", PackageNode, Hash), attr(\"variant_value\", PackageNode, \"dev_path\", _).\n% You can't install a hash, if it is not installed\n:- attr(\"hash\", node(ID, Package), Hash), not installed_hash(Package, Hash).\n\n% hash_attrs are versions, but can_splice_attr are usually node_version_satisfies\nhash_attr(Hash, \"node_version_satisfies\", PackageName, Constraint) :-\n hash_attr(Hash, \"version\", PackageName, Version),\n pkg_fact(PackageName, version_order(Version, VersionIdx)),\n pkg_fact(PackageName, version_range(Constraint, MinIdx, MaxIdx)),\n VersionIdx >= MinIdx, VersionIdx <= MaxIdx.\n\n% This recovers the exact semantics for hash reuse hash and depends_on are where\n% splices are decided, and virtual_on_edge can result in name-changes, which is\n% why they are all treated separately.\n\nimposed_constraint(Hash, Attr, PackageName) :- hash_attr(Hash, Attr, PackageName), Attr != \"virtual_node\".\n\nimposed_constraint(Hash, Attr, PackageName, A1) :- hash_attr(Hash, Attr, PackageName, A1), Attr != \"hash\".\n\nimposed_constraint(Hash, Attr, PackageName, A1, A2) :-\n hash_attr(Hash, Attr, PackageName, A1, A2),\n Attr != \"depends_on\",\n Attr != \"virtual_on_edge\".\n\nimposed_constraint(Hash, Attr, PackageName, A1, A2, A3) :- hash_attr(Hash, Attr, PackageName, A1, A2, A3).\nimposed_constraint(Hash, \"hash\", PackageName, Hash) :- installed_hash(PackageName, Hash).\n\n% If a compiler is not used as a library, we just enforce \"run\" dependency, so we\n% can get by with a much smaller search space.\navoid_link_dependency(Hash, DepName) :-\n hash_attr(Hash, \"depends_on\", PackageName, DepName, \"link\"),\n not hash_attr(Hash, \"depends_on\", PackageName, DepName, \"run\"),\n hash_attr(Hash, \"hash\", DepName, DepHash),\n compiler_package(PackageName),\n not compiler_used_as_a_library(node(_, PackageName), Hash).\n\n% When a compiler is not used as a library, its transitive run-type dependencies are unified in the\n% build environment (they appear in PATH etc.), but their pure link-type dependencies are NOT. This\n% is different from unification sets that include all link/run deps: the goal is to avoid imposing\n% transitive link constraints from the toolchain, which would add to max_dupes and require us to\n% increase the max_dupes threshold for many packages.\n\n% Base case: direct run dep of a non-library compiler\ncompiler_non_lib_run_dep(DepHash, DepName) :-\n compiler_package(CompilerName),\n not compiler_used_as_a_library(node(_, CompilerName), CompilerHash),\n hash_attr(CompilerHash, \"depends_on\", CompilerName, DepName, \"run\"),\n hash_attr(CompilerHash, \"hash\", DepName, DepHash).\n\n% Recursive case: run deps of run deps (full transitive closure)\ncompiler_non_lib_run_dep(TransDepHash, TransDepName) :-\n compiler_non_lib_run_dep(DepHash, DepName),\n hash_attr(DepHash, \"depends_on\", DepName, TransDepName, \"run\"),\n hash_attr(DepHash, \"hash\", TransDepName, TransDepHash).\n\n% Pure link deps (link but not run) of any node in that closure are avoided\navoid_link_dependency(DepHash, LinkDepName) :-\n compiler_non_lib_run_dep(DepHash, DepName),\n hash_attr(DepHash, \"depends_on\", DepName, LinkDepName, \"link\"),\n not hash_attr(DepHash, \"depends_on\", DepName, LinkDepName, \"run\").\n\n% Without splicing, we simply recover the exact semantics\nimposed_constraint(ParentHash, \"hash\", ChildName, ChildHash) :-\n hash_attr(ParentHash, \"hash\", ChildName, ChildHash),\n ChildHash != ParentHash,\n not avoid_link_dependency(ParentHash, ChildName),\n not abi_splice_conditions_hold(_, _, ChildName, ChildHash).\n\nimposed_constraint(Hash, \"depends_on\", PackageName, DepName, Type) :-\n hash_attr(Hash, \"depends_on\", PackageName, DepName, Type),\n hash_attr(Hash, \"hash\", DepName, DepHash),\n not avoid_link_dependency(Hash, DepName),\n not attr(\"splice_at_hash\", _, _, DepName, DepHash).\n\nimposed_constraint(Hash, \"virtual_on_edge\", PackageName, DepName, VirtName) :-\n hash_attr(Hash, \"virtual_on_edge\", PackageName, DepName, VirtName),\n not avoid_link_dependency(Hash, DepName),\n not attr(\"splice_at_hash\", _, _, DepName,_).\n\nimposed_constraint(Hash, \"virtual_node\", VirtName) :-\n hash_attr(Hash, \"virtual_on_edge\", PackageName, DepName, VirtName),\n hash_attr(Hash, \"virtual_node\", VirtName),\n not avoid_link_dependency(Hash, DepName),\n not attr(\"splice_at_hash\", _, _, DepName,_).\n\n\n% Rules pertaining to attr(\"splice_at_hash\") and abi_splice_conditions_hold will\n% be conditionally loaded from splices.lp\n\nimpose(Hash, PackageNode) :- attr(\"hash\", PackageNode, Hash), attr(\"node\", PackageNode).\n\n% If there is not a hash for a package, we build it.\nbuild(PackageNode) :- attr(\"node\", PackageNode), not concrete(PackageNode).\n\n% Minimizing builds is tricky. We want a minimizing criterion\n\n% because we want to reuse what is available, but\n% we also want things that are built to stick to *default preferences* from\n% the package and from the user. We therefore treat built specs differently and apply\n% a different set of optimization criteria to them. Spack's *first* priority is to\n% reuse what it *can*, but if it builds something, the built specs will respect\n% defaults and preferences. This is implemented by bumping the priority of optimization\n% criteria for built specs -- so that they take precedence over the otherwise\n% topmost-priority criterion to reuse what is installed.\n%\n\n% The priority ranges are:\n% 1000+ Optimizations for concretization errors\n% 300 - 1000 Highest priority optimizations for valid solutions\n% 200 - 299 Shifted priorities for build nodes; correspond to priorities 0 - 99.\n% 100 - 199 Unshifted priorities. Currently only includes minimizing #builds and minimizing dupes.\n% 0 - 99 Priorities for non-built nodes.\n\ntreat_node_as_concrete(node(X, Package)) :- attr(\"node\", node(X, Package)), runtime(Package).\n\nbuild_priority(PackageNode, 200) :- build(PackageNode), attr(\"node\", PackageNode), not treat_node_as_concrete(PackageNode).\nbuild_priority(PackageNode, 0) :- build(PackageNode), attr(\"node\", PackageNode), treat_node_as_concrete(PackageNode).\nbuild_priority(PackageNode, 0) :- concrete(PackageNode), attr(\"node\", PackageNode).\n\n% This statement, which is a hidden feature of clingo, let us avoid cycles in the DAG\n#edge (A, B) : depends_on(A, B).\n\n%-----------------------------------------------------------------\n% Optimization to avoid errors\n%-----------------------------------------------------------------\n% Some errors are handled as rules instead of constraints because\n% it allows us to explain why something failed.\n#minimize{ 0@1000: #true}.\n\n#minimize{ Weight@1000,Msg: error(Weight, Msg) }.\n#minimize{ Weight@1000,Msg,Arg1: error(Weight, Msg, Arg1) }.\n#minimize{ Weight@1000,Msg,Arg1,Arg2: error(Weight, Msg, Arg1, Arg2) }.\n#minimize{ Weight@1000,Msg,Arg1,Arg2,Arg3: error(Weight, Msg, Arg1, Arg2, Arg3) }.\n#minimize{ Weight@1000,Msg,Arg1,Arg2,Arg3,Arg4: error(Weight, Msg, Arg1, Arg2, Arg3, Arg4) }.\n\n%-----------------------------------------------------------------------------\n% How to optimize the spec (high to low priority)\n%-----------------------------------------------------------------------------\n% Each criterion below has:\n% 1. an opt_criterion(ID, Name) fact that describes the criterion, and\n% 2. a `#minimize{ 0@2 : #true }.` statement that ensures the criterion\n% is displayed (clingo doesn't display sums over empty sets by default)\n\n% A condition group specifies one or more specs that must be satisfied.\n% Specs declared first are preferred, so we assign increasing weights and\n% minimize the weights.\nopt_criterion(310, \"requirement weight\").\n#minimize{ 0@310: #true }.\n#minimize {\n Weight@310,PackageNode,Group\n : requirement_penalty(PackageNode, Group, Weight)\n}.\n#minimize {\n Weight@310,PackageNode,Language,Group\n : requirement_penalty(PackageNode, Language, Group, Weight)\n}.\n\n% Try hard to reuse installed packages (i.e., minimize the number built)\nopt_criterion(120, \"number of packages to build (vs. reuse)\").\n#minimize { 0@120: #true }.\n#minimize { 1@120,PackageNode : build(PackageNode), not treat_node_as_concrete(PackageNode) }.\n\nopt_criterion(110, \"number of nodes from the same package\").\n#minimize { 0@110: #true }.\n#minimize { ID@110,Package : attr(\"node\", node(ID, Package)), not self_build_requirement(_, node(ID, Package)) }.\n#minimize { ID@110,Package : attr(\"virtual_node\", node(ID, Package)) }.\n#defined optimize_for_reuse/0.\n\n% Minimize the unification set ID used for build dependencies. This reduces the number of optimal\n% solutions that differ only by which node belongs to which unification set.\nopt_criterion(100, \"build unification sets\").\n#minimize{ 0@100: #true }.\n#minimize{ ID@100,ParentNode : build_set_id(ParentNode, ID) }.\n\n% Minimize the number of deprecated versions being used\nopt_criterion(73, \"deprecated versions used\").\n#minimize{ 0@273: #true }.\n#minimize{ 0@73: #true }.\n#minimize{\n 1@73+Priority,PackageNode\n : attr(\"deprecated\", PackageNode, _),\n not external(PackageNode),\n build_priority(PackageNode, Priority)\n}.\n\n% Minimize the:\n% 1. Version weight\n% 2. Number of variants with a non default value, if not set\n% for the root package.\nopt_criterion(70, \"version badness (roots)\").\n#minimize{ 0@270: #true }.\n#minimize{ 0@70: #true }.\n#minimize {\n Weight@70+Priority,PackageNode\n : attr(\"root\", PackageNode),\n version_weight(PackageNode, Weight),\n build_priority(PackageNode, Priority)\n}.\n#minimize {\n Penalty@70+Priority,PackageNode\n : attr(\"root\", PackageNode),\n version_deprecation_penalty(PackageNode, Penalty),\n build_priority(PackageNode, Priority)\n}.\n\nopt_criterion(65, \"variant penalty (roots)\").\n#minimize{ 0@265: #true }.\n#minimize{ 0@65: #true }.\n#minimize {\n Penalty@65+Priority,PackageNode,Variant,Value\n : variant_penalty(PackageNode, Variant, Value, Penalty),\n attr(\"root\", PackageNode),\n build_priority(PackageNode, Priority)\n}.\n\nopt_criterion(55, \"default values of variants not being used (roots)\").\n#minimize{ 0@255: #true }.\n#minimize{ 0@55: #true }.\n#minimize{\n 1@55+Priority,PackageNode,Variant,Value\n : variant_default_not_used(PackageNode, Variant, Value),\n attr(\"root\", PackageNode),\n build_priority(PackageNode, Priority)\n}.\n\n% Choose the preferred compiler before penalizing variants, to avoid that a variant penalty\n% on e.g. gcc causes clingo to pick another compiler e.g. llvm\nopt_criterion(48, \"preferred compilers\").\n#minimize{ 0@248: #true }.\n#minimize{ 0@48: #true }.\n#minimize{\n Weight@48+Priority,ProviderNode,X,Virtual\n : provider_weight(ProviderNode, node(X, Virtual), Weight),\n language(Virtual),\n build_priority(ProviderNode, Priority)\n}.\n\nopt_criterion(41, \"compiler penalty from reuse\").\n#minimize{ 0@241: #true }.\n#minimize{ 0@41: #true }.\n#minimize{1@41,Hash : compiler_penalty_from_reuse(Hash)}.\n\n% Try to use default variants or variants that have been set\nopt_criterion(40, \"variant penalty (non-roots)\").\n#minimize{ 0@240: #true }.\n#minimize{ 0@40: #true }.\n#minimize {\n Penalty@40+Priority,PackageNode,Variant,Value\n : variant_penalty(PackageNode, Variant, Value, Penalty),\n not attr(\"root\", PackageNode),\n build_priority(PackageNode, Priority)\n}.\n\n% Minimize the weights of all the other providers (mpi, lapack, etc.)\nopt_criterion(38, \"preferred providers (excluded compilers and language runtimes)\").\n#minimize{ 0@238: #true }.\n#minimize{ 0@38: #true }.\n#minimize{\n Weight@38+Priority,ProviderNode,X,Virtual\n : provider_weight(ProviderNode, node(X, Virtual), Weight),\n not language(Virtual), not language_runtime(Virtual),\n build_priority(ProviderNode, Priority)\n}.\n\n% Minimize the number of compilers used on nodes\ncompiler_penalty(PackageNode, C-1) :-\n C = #count { CompilerNode : node_compiler(PackageNode, CompilerNode) },\n node_compiler(PackageNode, _), C > 0.\n\nopt_criterion(36, \"number of compilers used on the same node\").\n#minimize{ 0@236: #true }.\n#minimize{ 0@36: #true }.\n#minimize{\n Penalty@36+Priority,PackageNode\n : compiler_penalty(PackageNode, Penalty), build_priority(PackageNode, Priority)\n}.\n\n\nopt_criterion(30, \"non-preferred OS's\").\n#minimize{ 0@230: #true }.\n#minimize{ 0@30: #true }.\n#minimize{\n Weight@30+Priority,PackageNode\n : node_os_weight(PackageNode, Weight),\n build_priority(PackageNode, Priority)\n}.\n\n% Choose more recent versions for nodes\nopt_criterion(25, \"version badness (non roots)\").\n#minimize{ 0@225: #true }.\n#minimize{ 0@25: #true }.\n#minimize{\n Weight@25+Priority,node(X, Package)\n : version_weight(node(X, Package), Weight),\n build_priority(node(X, Package), Priority),\n not attr(\"root\", node(X, Package)),\n not runtime(Package)\n}.\n#minimize {\n Penalty@25+Priority,node(X, Package)\n : version_deprecation_penalty(node(X, Package), Penalty),\n build_priority(node(X, Package), Priority),\n not attr(\"root\", node(X, Package)),\n not runtime(Package)\n}.\n\n\n% Try to use all the default values of variants\nopt_criterion(20, \"default values of variants not being used (non-roots)\").\n#minimize{ 0@220: #true }.\n#minimize{ 0@20: #true }.\n#minimize{\n 1@20+Priority,PackageNode,Variant,Value\n : variant_default_not_used(PackageNode, Variant, Value),\n not attr(\"root\", PackageNode),\n build_priority(PackageNode, Priority)\n}.\n\n% Minimize the number of mismatches for targets in the DAG, try\n% to select the preferred target.\nopt_criterion(10, \"target mismatches\").\n#minimize{ 0@210: #true }.\n#minimize{ 0@10: #true }.\n#minimize{\n 1@10+Priority,PackageNode,node(ID, Dependency)\n : node_target_mismatch(PackageNode, node(ID, Dependency)),\n build_priority(node(ID, Dependency), Priority),\n not runtime(Dependency)\n}.\n\nopt_criterion(7, \"non-preferred targets\").\n#minimize{ 0@207: #true }.\n#minimize{ 0@7: #true }.\n#minimize{\n Weight@7+Priority,node(X, Package)\n : node_target_weight(node(X, Package), Weight),\n build_priority(node(X, Package), Priority),\n not runtime(Package)\n}.\n\nopt_criterion(5, \"preferred providers (language runtimes)\").\n#minimize{ 0@205: #true }.\n#minimize{ 0@5: #true }.\n#minimize{\n Weight@5+Priority,ProviderNode,X,Virtual\n : provider_weight(ProviderNode, node(X, Virtual), Weight),\n language_runtime(Virtual),\n build_priority(ProviderNode, Priority)\n}.\n\n% Choose more recent versions for runtimes\nopt_criterion(4, \"version badness (runtimes)\").\n#minimize{ 0@204: #true }.\n#minimize{ 0@4: #true }.\n#minimize{\n Weight@4,node(X, Package)\n : version_weight(node(X, Package), Weight),\n runtime(Package)\n}.\n\n% Choose best target for runtimes\nopt_criterion(3, \"non-preferred targets (runtimes)\").\n#minimize{ 0@203: #true }.\n#minimize{ 0@3: #true }.\n#minimize{\n Weight@3,node(X, Package)\n : node_target_weight(node(X, Package), Weight),\n runtime(Package)\n}.\n\n% Try to use the most optimal providers as much as possible\nopt_criterion(2, \"providers on edges\").\n#minimize{ 0@202: #true }.\n#minimize{ 0@2: #true }.\n#minimize{\n Weight@2,ParentNode,ProviderNode,node(X, Virtual)\n : provider_weight(ProviderNode, node(X, Virtual), Weight),\n max_dupes(Virtual, MaxDupes), MaxDupes > 1,\n not attr(\"root\", ProviderNode),\n language(Virtual),\n depends_on(ParentNode, ProviderNode)\n}.\n\n% Try to use latest versions of nodes as much as possible\nopt_criterion(1, \"version badness on edges\").\n#minimize{ 0@201: #true }.\n#minimize{ 0@1: #true }.\n#minimize{\n Weight@1,ParentNode,node(X, Package)\n : version_weight(node(X, Package), Weight),\n multiple_unification_sets(Package),\n not attr(\"root\", node(X, Package)),\n depends_on(ParentNode, node(X, Package))\n}.\n\n% Reduce symmetry on duplicates\nopt_criterion(0, \"penalty on symmetric duplicates\").\n#minimize{ 0@200: #true }.\n#minimize{ 0@0: #true }.\n#minimize{\n Weight@1,PackageNode,Reason : duplicate_penalty(PackageNode, Weight, Reason)\n}.\n\n\n%-----------\n% Notes\n%-----------\n\n% [1] Clingo ensures a total ordering among all atoms. We rely on that total ordering\n% to reduce symmetry in the solution by checking `<` instead of `!=` in symmetric\n% cases. These choices are made without loss of generality.\n" }, { "path": "lib/spack/spack/solver/core.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Low-level wrappers around clingo API and other basic functionality related to ASP\"\"\"\n\nimport importlib\nimport pathlib\nfrom types import ModuleType\nfrom typing import Any, Callable, NamedTuple, Optional, Tuple\n\nimport spack.platforms\nfrom spack.llnl.util import lang\n\n\ndef _ast_getter(*names: str) -> Callable[[Any], Any]:\n \"\"\"Helper to retrieve AST attributes from different versions of the clingo API\"\"\"\n\n def getter(node):\n for name in names:\n result = getattr(node, name, None)\n if result:\n return result\n raise KeyError(f\"node has no such keys: {names}\")\n\n return getter\n\n\nast_type = _ast_getter(\"ast_type\", \"type\")\nast_sym = _ast_getter(\"symbol\", \"term\")\n\n\nclass AspVar:\n \"\"\"Represents a variable in an ASP rule, allows for conditionally generating\n rules\"\"\"\n\n __slots__ = (\"name\",)\n\n def __init__(self, name: str) -> None:\n self.name = name\n\n def __str__(self) -> str:\n return str(self.name)\n\n\n@lang.key_ordering\nclass AspFunction:\n \"\"\"A term in the ASP logic program\"\"\"\n\n __slots__ = (\"name\", \"args\")\n\n def __init__(self, name: str, args: Tuple[Any, ...] = ()) -> None:\n self.name = name\n self.args = args\n\n def _cmp_key(self) -> Tuple[str, Tuple[Any, ...]]:\n return self.name, self.args\n\n def __call__(self, *args: Any) -> \"AspFunction\":\n \"\"\"Return a new instance of this function with added arguments.\n\n Note that calls are additive, so you can do things like::\n\n >>> attr = AspFunction(\"attr\")\n attr()\n\n >>> attr(\"version\")\n attr(\"version\")\n\n >>> attr(\"version\")(\"foo\")\n attr(\"version\", \"foo\")\n\n >>> v = AspFunction(\"attr\", \"version\")\n attr(\"version\")\n\n >>> v(\"foo\", \"bar\")\n attr(\"version\", \"foo\", \"bar\")\n\n \"\"\"\n return AspFunction(self.name, self.args + args)\n\n def __str__(self) -> str:\n parts = []\n for arg in self.args:\n if type(arg) is str:\n arg = arg.replace(\"\\\\\", r\"\\\\\").replace(\"\\n\", r\"\\n\").replace('\"', r\"\\\"\")\n parts.append(f'\"{arg}\"')\n elif type(arg) is AspFunction or type(arg) is int or type(arg) is AspVar:\n parts.append(str(arg))\n else:\n parts.append(f'\"{arg}\"')\n return f\"{self.name}({','.join(parts)})\"\n\n def __repr__(self) -> str:\n return str(self)\n\n\nclass _AspFunctionBuilder:\n def __getattr__(self, name: str) -> AspFunction:\n return AspFunction(name)\n\n\n#: Global AspFunction builder\nfn = _AspFunctionBuilder()\n\n_CLINGO_MODULE: Optional[ModuleType] = None\n\n\ndef clingo() -> ModuleType:\n \"\"\"Lazy imports the Python module for clingo, and returns it.\"\"\"\n if _CLINGO_MODULE is not None:\n return _CLINGO_MODULE\n\n try:\n clingo_mod = importlib.import_module(\"clingo\")\n # Make sure we didn't import an empty module\n _ensure_clingo_or_raise(clingo_mod)\n except ImportError:\n clingo_mod = None\n\n if clingo_mod is not None:\n return _set_clingo_module_cache(clingo_mod)\n\n clingo_mod = _bootstrap_clingo()\n return _set_clingo_module_cache(clingo_mod)\n\n\ndef _set_clingo_module_cache(clingo_mod: ModuleType) -> ModuleType:\n \"\"\"Sets the global cache to the lazy imported clingo module\"\"\"\n global _CLINGO_MODULE\n importlib.import_module(\"clingo.ast\")\n _CLINGO_MODULE = clingo_mod\n return clingo_mod\n\n\ndef _ensure_clingo_or_raise(clingo_mod: ModuleType) -> None:\n \"\"\"Ensures the clingo module can access expected attributes, otherwise raises an error.\"\"\"\n # These are imports that may be problematic at top level (circular imports). They are used\n # only to provide exhaustive details when erroring due to a broken clingo module.\n import spack.config\n import spack.paths as sp\n import spack.util.path as sup\n\n try:\n clingo_mod.Symbol\n except AttributeError:\n assert clingo_mod.__file__ is not None, \"clingo installation is incomplete or invalid\"\n # Reaching this point indicates a broken clingo installation\n # If Spack derived clingo, suggest user re-run bootstrap\n # if non-spack, suggest user investigate installation\n # assume Spack is not responsible for broken clingo\n msg = (\n f\"Clingo installation at {clingo_mod.__file__} is incomplete or invalid.\"\n \"Please repair installation or re-install. \"\n \"Alternatively, consider installing clingo via Spack.\"\n )\n # check whether Spack is responsible\n if (\n pathlib.Path(\n sup.canonicalize_path(\n spack.config.CONFIG.get(\"bootstrap:root\", sp.default_user_bootstrap_path)\n )\n )\n in pathlib.Path(clingo_mod.__file__).parents\n ):\n # Spack is responsible for the broken clingo\n msg = (\n \"Spack bootstrapped copy of Clingo is broken, \"\n \"please re-run the bootstrapping process via command `spack bootstrap now`.\"\n \" If this issue persists, please file a bug at: github.com/spack/spack\"\n )\n raise RuntimeError(\n \"Clingo installation may be broken or incomplete, \"\n \"please verify clingo has been installed correctly\"\n \"\\n\\nClingo does not provide symbol clingo.Symbol\"\n f\"{msg}\"\n )\n\n\ndef clingo_cffi() -> bool:\n \"\"\"Returns True if clingo uses the CFFI interface\"\"\"\n return hasattr(clingo().Symbol, \"_rep\")\n\n\ndef _bootstrap_clingo() -> ModuleType:\n \"\"\"Bootstraps the clingo module and returns it\"\"\"\n import spack.bootstrap\n\n with spack.bootstrap.ensure_bootstrap_configuration():\n spack.bootstrap.ensure_clingo_importable_or_raise()\n clingo_mod = importlib.import_module(\"clingo\")\n\n return clingo_mod\n\n\ndef parse_files(*args, **kwargs):\n \"\"\"Wrapper around clingo parse_files, that dispatches the function according\n to clingo API version.\n \"\"\"\n clingo()\n try:\n return importlib.import_module(\"clingo.ast\").parse_files(*args, **kwargs)\n except (ImportError, AttributeError):\n return clingo().parse_files(*args, **kwargs)\n\n\ndef parse_term(*args, **kwargs):\n \"\"\"Wrapper around clingo parse_term, that dispatches the function according\n to clingo API version.\n \"\"\"\n clingo()\n try:\n return importlib.import_module(\"clingo.symbol\").parse_term(*args, **kwargs)\n except (ImportError, AttributeError):\n return clingo().parse_term(*args, **kwargs)\n\n\nclass NodeId(NamedTuple):\n \"\"\"Represents a node in the DAG\"\"\"\n\n id: str\n pkg: str\n\n\nclass NodeFlag(NamedTuple):\n flag_type: str\n flag: str\n flag_group: str\n source: str\n\n\ndef intermediate_repr(sym):\n \"\"\"Returns an intermediate representation of clingo models for Spack's spec builder.\n\n Currently, transforms symbols from clingo models either to strings or to NodeId objects.\n\n Returns:\n This will turn a ``clingo.Symbol`` into a string or NodeId, or a sequence of\n ``clingo.Symbol`` objects into a tuple of those objects.\n \"\"\"\n # TODO: simplify this when we no longer have to support older clingo versions.\n if isinstance(sym, (list, tuple)):\n return tuple(intermediate_repr(a) for a in sym)\n\n try:\n if sym.name == \"node\":\n return NodeId(\n id=intermediate_repr(sym.arguments[0]), pkg=intermediate_repr(sym.arguments[1])\n )\n elif sym.name == \"node_flag\":\n return NodeFlag(\n flag_type=intermediate_repr(sym.arguments[0]),\n flag=intermediate_repr(sym.arguments[1]),\n flag_group=intermediate_repr(sym.arguments[2]),\n source=intermediate_repr(sym.arguments[3]),\n )\n except RuntimeError:\n # This happens when using clingo w/ CFFI and trying to access \".name\" for symbols\n # that are not functions\n pass\n\n if clingo_cffi():\n # Clingo w/ CFFI will throw an exception on failure\n try:\n return sym.string\n except RuntimeError:\n return str(sym)\n else:\n return sym.string or str(sym)\n\n\ndef extract_args(model, predicate_name):\n \"\"\"Extract the arguments to predicates with the provided name from a model.\n\n Pull out all the predicates with name ``predicate_name`` from the model, and\n return their intermediate representation.\n \"\"\"\n return [intermediate_repr(sym.arguments) for sym in model if sym.name == predicate_name]\n\n\nclass SourceContext:\n \"\"\"Tracks context in which a Spec's clause-set is generated (i.e.\n with ``SpackSolverSetup.spec_clauses``).\n\n Facts generated for the spec may include this context.\n \"\"\"\n\n def __init__(self, *, source: Optional[str] = None):\n # This can be \"literal\" for constraints that come from a user\n # spec (e.g. from the command line); it can be the output of\n # `ConstraintOrigin.append_type_suffix`; the default is \"none\"\n # (which means it isn't important to keep track of the source\n # in that case).\n self.source = \"none\" if source is None else source\n self.wrap_node_requirement: Optional[bool] = None\n\n\ndef using_libc_compatibility() -> bool:\n \"\"\"Returns True if we are currently using libc compatibility\"\"\"\n return spack.platforms.host().name == \"linux\"\n" }, { "path": "lib/spack/spack/solver/direct_dependency.lp", "content": "% Copyright Spack Project Developers. See COPYRIGHT file for details.\n%\n% SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n% A direct dependency is either a runtime requirement, or a build requirement\n1 {\n build_requirement(PackageNode, node(0..X-1, DirectDependency)) : max_dupes(DirectDependency, X);\n runtime_requirement(PackageNode, node(0..X-1, DirectDependency)) : max_dupes(DirectDependency, X)\n} 1 :- attr(\"direct_dependency\", PackageNode, node_requirement(\"node\", DirectDependency)),\n build(PackageNode).\n\n1 {\n concrete_build_requirement(PackageNode, DirectDependency);\n runtime_requirement(PackageNode, node(0..X-1, DirectDependency)) : max_dupes(DirectDependency, X)\n} 1 :- attr(\"direct_dependency\", PackageNode, node_requirement(\"node\", DirectDependency)),\n concrete(PackageNode).\n\ndirect_dependency(ParentNode, ChildNode) :- build_requirement(ParentNode, ChildNode).\ndirect_dependency(ParentNode, ChildNode) :- runtime_requirement(ParentNode, ChildNode).\n\n%%%%\n% Build requirement\n%%%%\n\n% A build requirement that is not concrete has a \"build\" only dependency\nattr(\"depends_on\", node(X, Parent), node(Y, BuildDependency), \"build\") :-\n build_requirement(node(X, Parent), node(Y, BuildDependency)),\n build(node(X, Parent)).\n\n% Any other dependency type is forbidden\n:- build_requirement(ParentNode, ChildNode),\n build(ParentNode),\n attr(\"depends_on\", ParentNode, ChildNode, Type), Type != \"build\".\n\n:- concrete_build_requirement(ParentNode, ChildPackage),\n concrete(ParentNode),\n attr(\"depends_on\", ParentNode, node(_, ChildPackage), _).\n\n%%%%\n% Runtime requirement\n%%%%\n\n:- runtime_requirement(ParentNode, ChildNode),\n not 1 { attr(\"depends_on\", ParentNode, ChildNode, \"link\"); attr(\"depends_on\", ParentNode, ChildNode, \"run\") }.\n\nattr(AttributeName, node(X, ChildPackage), A1)\n :- runtime_requirement(ParentNode, node(X, ChildPackage)),\n attr(\"direct_dependency\", ParentNode, node_requirement(AttributeName, ChildPackage, A1)),\n AttributeName != \"provider_set\".\n\nattr(AttributeName, node(X, ChildPackage), A1, A2)\n :- runtime_requirement(ParentNode, node(X, ChildPackage)),\n attr(\"direct_dependency\", ParentNode, node_requirement(AttributeName, ChildPackage, A1, A2)).\n\nattr(AttributeName, node(X, ChildPackage), A1, A2, A3)\n :- runtime_requirement(ParentNode, node(X, ChildPackage)),\n attr(\"direct_dependency\", ParentNode, node_requirement(AttributeName, ChildPackage, A1, A2, A3)).\n\nattr(AttributeName, node(X, ChildPackage), A1, A2, A3, A4)\n :- runtime_requirement(ParentNode, node(X, ChildPackage)),\n attr(\"direct_dependency\", ParentNode, node_requirement(AttributeName, ChildPackage, A1, A2, A3, A4)).\n" }, { "path": "lib/spack/spack/solver/display.lp", "content": "% Copyright Spack Project Developers. See COPYRIGHT file for details.\n%\n% SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n%=============================================================================-\n% Display Results\n%\n% This section determines what parts of the model are printed at the end\n%==============================================================================\n\n% Spec attributes\n#show attr/2.\n#show attr/3.\n#show attr/4.\n#show attr/5.\n#show attr/6.\n% names of optimization criteria\n#show opt_criterion/2.\n\n% error types\n#show error/2.\n#show error/3.\n#show error/4.\n#show error/5.\n#show error/6.\n\n% for error causation\n#show condition_reason/2.\n\n% For error messages to use later\n#show pkg_fact/2.\n#show condition_holds/2.\n#show imposed_constraint/3.\n#show imposed_constraint/4.\n#show imposed_constraint/5.\n#show imposed_constraint/6.\n#show condition_requirement/3.\n#show condition_requirement/4.\n#show condition_requirement/5.\n#show condition_requirement/6.\n#show node_has_variant/3.\n#show build/1.\n#show external/1.\n#show trigger_and_effect/3.\n#show unification_set/2.\n#show provider/2.\n#show condition_nodes/2.\n#show trigger_node/3.\n#show imposed_nodes/2.\n#show variant_single_value/2.\n\n% debug\n" }, { "path": "lib/spack/spack/solver/error_messages.lp", "content": "% Copyright Spack Project Developers. See COPYRIGHT file for details.\n%\n% SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n%=============================================================================\n% This logic program adds detailed error messages to Spack's concretizer\n%\n% Note that functions used in rule bodies here need to have a corresponding\n% #show line in display.lp, otherwise they won't be passed through to the\n% error solve.\n%=============================================================================\n\n#program error_messages.\n\n% The following \"condition_cause\" rules create a causal tree between trigger\n% conditions by locating the effect conditions that are triggers for another\n% condition. In all these rules, Condition2 is caused by Condition1\n\n% For condition_cause rules, it is not necessary to confirm that the attr is present\n% on the node because `condition_holds` and `impose_constraint` collectively\n% guarantee it.\n% We omit those facts to reduce the burden on the grounder/solver.\n\ncondition_cause(Condition2, ID2, Condition1, ID1) :-\n condition_holds(Condition2, node(ID2, Package2)),\n pkg_fact(Package2, condition_trigger(Condition2, Trigger)),\n condition_requirement(Trigger, Name, Package),\n condition_nodes(TriggerNode, node(ID, Package)),\n trigger_node(Trigger, TriggerNode, node(ID2, Package2)),\n condition_holds(Condition1, node(ID1, Package1)),\n pkg_fact(Package1, condition_effect(Condition1, Effect)),\n imposed_constraint(Effect, Name, Package),\n imposed_nodes(node(ID1, Package1), node(ID, Package)).\n\ncondition_cause(Condition2, ID2, Condition1, ID1) :-\n condition_holds(Condition2, node(ID2, Package2)),\n pkg_fact(Package2, condition_trigger(Condition2, Trigger)),\n condition_requirement(Trigger, Name, Package, A1),\n condition_nodes(TriggerNode, node(ID, Package)),\n trigger_node(Trigger, TriggerNode, node(ID2, Package2)),\n condition_holds(Condition1, node(ID1, Package1)),\n pkg_fact(Package1, condition_effect(Condition1, Effect)),\n imposed_constraint(Effect, Name, Package, A1),\n imposed_nodes(node(ID1, Package1), node(ID, Package)).\n\ncondition_cause(Condition2, ID2, Condition1, ID1) :-\n condition_holds(Condition2, node(ID2, Package2)),\n pkg_fact(Package2, condition_trigger(Condition2, Trigger)),\n condition_requirement(Trigger, Name, Package, A1, A2),\n condition_nodes(TriggerNode, node(ID, Package)),\n trigger_node(Trigger, TriggerNode, node(ID2, Package2)),\n condition_holds(Condition1, node(ID1, Package1)),\n pkg_fact(Package1, condition_effect(Condition1, Effect)),\n imposed_constraint(Effect, Name, Package, A1, A2),\n imposed_nodes(node(ID1, Package1), node(ID, Package)).\n\ncondition_cause(Condition2, ID2, Condition1, ID1) :-\n condition_holds(Condition2, node(ID2, Package2)),\n pkg_fact(Package2, condition_trigger(Condition2, Trigger)),\n condition_requirement(Trigger, Name, Package, A1, A2, A3),\n condition_nodes(TriggerNode, node(ID, Package)),\n trigger_node(Trigger, TriggerNode, node(ID2, Package2)),\n condition_holds(Condition1, node(ID1, Package1)),\n pkg_fact(Package1, condition_effect(Condition1, Effect)),\n imposed_constraint(Effect, Name, Package, A1, A2, A3),\n imposed_nodes(node(ID1, Package1), node(ID, Package)).\n\n% special condition cause for dependency conditions\n% we can't simply impose the existence of the node for dependency conditions\n% because we need to allow for the choice of which dupe ID the node gets\ncondition_cause(Condition2, ID2, Condition1, ID1) :-\n condition_holds(Condition2, node(ID2, Package2)),\n pkg_fact(Package2, condition_trigger(Condition2, Trigger)),\n condition_requirement(Trigger, \"node\", Package),\n condition_nodes(TriggerNode, node(ID, Package)),\n trigger_node(Trigger, TriggerNode, node(ID2, Package2)),\n condition_holds(Condition1, node(ID1, Package1)),\n pkg_fact(Package1, condition_effect(Condition1, Effect)),\n imposed_constraint(Effect, \"dependency_holds\", Parent, Package, Type),\n imposed_nodes(node(ID1, Package1), node(ID, Package)),\n attr(\"depends_on\", node(X, Parent), node(ID, Package), Type).\n\n% The literal startcauses is used to separate the variables that are part of the error from the\n% ones describing the causal tree of the error. After startcauses, each successive pair must be\n% a condition and a condition_set id for which it holds.\n\n#defined choose_version/2.\n\n% More specific error message if the version cannot satisfy some constraint\n% Otherwise covered by `no_version_error` and `versions_conflict_error`.\nerror(10000, \"Cannot satisfy '{0}@{1}' (selected version {2} does not match)\", Package, Constraint, Version, startcauses, ConstraintCause, CauseID)\n :- attr(\"node_version_satisfies\", node(ID, Package), Constraint),\n pkg_fact(TriggerPkg, condition_effect(ConstraintCause, EffectID)),\n imposed_constraint(EffectID, \"node_version_satisfies\", Package, Constraint),\n condition_holds(ConstraintCause, node(CauseID, TriggerPkg)),\n attr(\"version\", node(ID, Package), Version),\n not pkg_fact(Package, version_satisfies(Constraint, Version)),\n choose_version(node(ID, Package), Version).\n\nerror(100, \"Cannot satisfy '{0}@{1}' (version {2} does not match)\", Package, Constraint, Version, startcauses, ConstraintCause, CauseID)\n :- attr(\"node_version_satisfies\", node(ID, Package), Constraint),\n pkg_fact(TriggerPkg, condition_effect(ConstraintCause, EffectID)),\n imposed_constraint(EffectID, \"node_version_satisfies\", Package, Constraint),\n condition_holds(ConstraintCause, node(CauseID, TriggerPkg)),\n attr(\"version\", node(ID, Package), Version),\n not pkg_fact(Package, version_satisfies(Constraint, Version)),\n not choose_version(node(ID, Package), Version).\n\nerror(0, \"Cannot satisfy '{0}@{1}' and '{0}@{2}'\", Package, Constraint1, Constraint2, startcauses, Cause1, C1ID, Cause2, C2ID)\n :- attr(\"node_version_satisfies\", node(ID, Package), Constraint1),\n pkg_fact(TriggerPkg1, condition_effect(Cause1, EffectID1)),\n imposed_constraint(EffectID1, \"node_version_satisfies\", Package, Constraint1),\n condition_holds(Cause1, node(C1ID, TriggerPkg1)),\n % two constraints\n attr(\"node_version_satisfies\", node(ID, Package), Constraint2),\n pkg_fact(TriggerPkg2, condition_effect(Cause2, EffectID2)),\n imposed_constraint(EffectID2, \"node_version_satisfies\", Package, Constraint2),\n condition_holds(Cause2, node(C2ID, TriggerPkg2)),\n % version chosen\n attr(\"version\", node(ID, Package), Version),\n % version satisfies one but not the other\n pkg_fact(Package, version_satisfies(Constraint1, Version)),\n not pkg_fact(Package, version_satisfies(Constraint2, Version)),\n Cause1 < Cause2.\n\n% causation tracking error for no or multiple virtual providers\nerror(0, \"Cannot find a valid provider for virtual {0}\", Virtual, startcauses, Cause, CID)\n :- attr(\"virtual_node\", node(X, Virtual)),\n not provider(_, node(X, Virtual)),\n imposed_constraint(EID, \"dependency_holds\", Parent, Virtual, Type),\n pkg_fact(TriggerPkg, condition_effect(Cause, EID)),\n condition_holds(Cause, node(CID, TriggerPkg)).\n\n% At most one variant value for single-valued variants\nerror(0, \"'{0}' requires conflicting variant values 'Spec({1}={2})' and 'Spec({1}={3})'\", Package, Variant, Value1, Value2, startcauses, Cause1, X, Cause2, X)\n :- attr(\"node\", node(X, Package)),\n node_has_variant(node(X, Package), Variant, VariantID),\n variant_single_value(node(X, Package), Variant),\n build(node(X, Package)),\n attr(\"variant_value\", node(X, Package), Variant, Value1),\n imposed_constraint(EID1, \"variant_set\", Package, Variant, Value1),\n pkg_fact(TriggerPkg1, condition_effect(Cause1, EID1)),\n condition_holds(Cause1, node(X, TriggerPkg1)),\n attr(\"variant_value\", node(X, Package), Variant, Value2),\n imposed_constraint(EID2, \"variant_set\", Package, Variant, Value2),\n pkg_fact(TriggerPkg2, condition_effect(Cause2, EID2)),\n condition_holds(Cause2, node(X, TriggerPkg2)),\n Value1 < Value2. % see[1] in concretize.lp\n\n% error message with causes for conflicts\nerror(0, Msg, startcauses, TriggerID, ID1, ConstraintID, ID2)\n :- attr(\"node\", node(ID, Package)),\n pkg_fact(Package, conflict(TriggerID, ConstraintID, Msg)),\n % node(ID1, TriggerPackage) is node(ID2, Package) in most, but not all, cases\n condition_holds(TriggerID, node(ID1, TriggerPackage)),\n condition_holds(ConstraintID, node(ID2, Package)),\n unification_set(X, node(ID2, Package)),\n unification_set(X, node(ID1, TriggerPackage)),\n build(node(ID, Package)). % ignore conflicts for concrete packages\n\npkg_fact(Package, version_satisfies(Constraint, Version)) :-\n pkg_fact(Package, version_order(Version, VersionIdx)),\n pkg_fact(Package, version_range(Constraint, MinIdx, MaxIdx)),\n VersionIdx >= MinIdx, VersionIdx <= MaxIdx.\n\n% variables to show\n#show error/2.\n#show error/3.\n#show error/4.\n#show error/5.\n#show error/6.\n#show error/7.\n#show error/8.\n#show error/9.\n#show error/10.\n#show error/11.\n\n#show condition_cause/4.\n#show condition_reason/2.\n\n% Define all variables used to avoid warnings at runtime when the model doesn't happen to have one\n#defined error/2.\n#defined error/3.\n#defined error/4.\n#defined error/5.\n#defined error/6.\n#defined error/7.\n#defined error/8.\n#defined error/9.\n#defined error/10.\n#defined error/11.\n#defined attr/2.\n#defined attr/3.\n#defined attr/4.\n#defined attr/5.\n#defined pkg_fact/2.\n#defined imposed_constraint/3.\n#defined imposed_constraint/4.\n#defined imposed_constraint/5.\n#defined imposed_constraint/6.\n#defined condition_cause/4.\n#defined condition_nodes/2.\n#defined condition_requirement/3.\n#defined condition_requirement/4.\n#defined condition_requirement/5.\n#defined condition_requirement/6.\n#defined condition_holds/2.\n#defined unification_set/2.\n#defined external/1.\n#defined trigger_and_effect/3.\n#defined build/1.\n#defined node_has_variant/3.\n#defined provider/2.\n#defined variant_single_value/2.\n" }, { "path": "lib/spack/spack/solver/heuristic.lp", "content": "% Copyright Spack Project Developers. See COPYRIGHT file for details.\n%\n% SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n%=============================================================================\n% Heuristic to speed-up solves\n%=============================================================================\n\n% Decide about DAG atoms, before trying to guess facts used only in\n% the internal representation\n#heuristic attr(\"virtual_node\", node(X, Virtual)). [80, level]\n#heuristic attr(\"node\", PackageNode). [80, level]\n#heuristic attr(\"version\", node(PackageID, Package), Version). [80, level]\n#heuristic attr(\"variant_value\", PackageNode, Variant, Value). [80, level]\n#heuristic attr(\"node_target\", node(PackageID, Package), Target). [80, level]\n#heuristic virtual_on_edge(PackageNode, ProviderNode, Virtual, Type). [80, level]\n\n#heuristic attr(\"virtual_node\", node(X, Virtual)). [600, init]\n#heuristic attr(\"virtual_node\", node(X, Virtual)). [-1, sign]\n#heuristic attr(\"virtual_node\", node(0, Virtual)) : node_depends_on_virtual(PackageNode, Virtual). [1@2, sign]\n#heuristic attr(\"virtual_node\", node(0, \"c\")). [1@3, sign]\n#heuristic attr(\"virtual_node\", node(0, \"cxx\")). [1@3, sign]\n#heuristic attr(\"virtual_node\", node(0, \"libc\")). [1@3, sign]\n\n#heuristic attr(\"node\", PackageNode). [300, init]\n#heuristic attr(\"node\", PackageNode). [ 4, factor]\n#heuristic attr(\"node\", PackageNode). [ -1, sign]\n#heuristic attr(\"node\", node(0, Dependency)) : attr(\"dependency_holds\", ParentNode, Dependency, Type), not virtual(Dependency). [1@2, sign]\n\n#heuristic attr(\"version\", node(PackageID, Package), Version). [30, init]\n#heuristic attr(\"version\", node(PackageID, Package), Version). [-1, sign]\n#heuristic attr(\"version\", node(PackageID, Package), Version) : pkg_fact(Package, version_declared(Version, 0)), attr(\"node\", node(PackageID, Package)). [ 1@2, sign]\n\n% Use default targets\n#heuristic attr(\"node_target\", node(PackageID, Package), Target). [-1, sign]\n#heuristic attr(\"node_target\", node(PackageID, Package), Target) : target_weight(Target, 0), attr(\"node\", node(PackageID, Package)). [1@2, sign]\n\n% Use default variants\n#heuristic attr(\"variant_value\", PackageNode, Variant, Value). [30, init]\n#heuristic attr(\"variant_value\", PackageNode, Variant, Value). [-1, sign]\n#heuristic attr(\"variant_value\", PackageNode, Variant, Value) : variant_default_value(PackageNode, Variant, Value), attr(\"node\", PackageNode). [1@2, sign]\n" }, { "path": "lib/spack/spack/solver/input_analysis.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Classes to analyze the input of a solve, and provide information to set up the ASP problem\"\"\"\n\nimport collections\nfrom typing import Dict, List, NamedTuple, Set, Tuple, Union\n\nimport spack.vendor.archspec.cpu\n\nimport spack.binary_distribution\nimport spack.concretize\nimport spack.config\nimport spack.deptypes as dt\nimport spack.platforms\nimport spack.repo\nimport spack.spec\nimport spack.store\nfrom spack.error import SpackError\nfrom spack.llnl.util import lang, tty\nfrom spack.spec import EMPTY_SPEC\n\n\nclass PossibleGraph(NamedTuple):\n real_pkgs: Set[str]\n virtuals: Set[str]\n edges: Dict[str, Set[str]]\n\n\nclass PossibleDependencyGraph:\n \"\"\"Returns information needed to set up an ASP problem\"\"\"\n\n def unreachable(self, *, pkg_name: str, when_spec: spack.spec.Spec) -> bool:\n \"\"\"Returns true if the context can determine that the condition cannot ever\n be met on pkg_name.\n \"\"\"\n raise NotImplementedError\n\n def candidate_targets(self) -> List[spack.vendor.archspec.cpu.Microarchitecture]:\n \"\"\"Returns a list of targets that are candidate for concretization\"\"\"\n raise NotImplementedError\n\n def possible_dependencies(\n self,\n *specs: Union[spack.spec.Spec, str],\n allowed_deps: dt.DepFlag,\n transitive: bool = True,\n strict_depflag: bool = False,\n expand_virtuals: bool = True,\n ) -> PossibleGraph:\n \"\"\"Returns the set of possible dependencies, and the set of possible virtuals.\n\n Runtime packages, which may be injected by compilers, needs to be added to specs if\n the dependency is not explicit in the package.py recipe.\n\n Args:\n transitive: return transitive dependencies if True, only direct dependencies if False\n allowed_deps: dependency types to consider\n strict_depflag: if True, only the specific dep type is considered, if False any\n deptype that intersects with allowed deptype is considered\n expand_virtuals: expand virtual dependencies into all possible implementations\n \"\"\"\n raise NotImplementedError\n\n\nclass NoStaticAnalysis(PossibleDependencyGraph):\n \"\"\"Implementation that tries to minimize the setup time (i.e. defaults to give fast\n answers), rather than trying to reduce the ASP problem size with more complex analysis.\n \"\"\"\n\n def __init__(self, *, configuration: spack.config.Configuration, repo: spack.repo.RepoPath):\n self.configuration = configuration\n self.repo = repo\n self._platform_condition = spack.spec.Spec(\n f\"platform={spack.platforms.host()} target={spack.vendor.archspec.cpu.host().family}:\"\n )\n\n try:\n self.libc_pkgs = [x.name for x in self.providers_for(\"libc\")]\n except spack.repo.UnknownPackageError:\n self.libc_pkgs = []\n\n def is_virtual(self, name: str) -> bool:\n return self.repo.is_virtual(name)\n\n @lang.memoized\n def is_allowed_on_this_platform(self, *, pkg_name: str) -> bool:\n \"\"\"Returns true if a package is allowed on the current host\"\"\"\n pkg_cls = self.repo.get_pkg_class(pkg_name)\n for when_spec, conditions in pkg_cls.requirements.items():\n # Restrict analysis to unconditional requirements\n if when_spec != EMPTY_SPEC:\n continue\n for requirements, _, _ in conditions:\n if not any(x.intersects(self._platform_condition) for x in requirements):\n tty.debug(f\"[{__name__}] {pkg_name} is not for this platform\")\n return False\n return True\n\n def providers_for(self, virtual_str: str) -> List[spack.spec.Spec]:\n \"\"\"Returns a list of possible providers for the virtual string in input.\"\"\"\n return self.repo.providers_for(virtual_str)\n\n def can_be_installed(self, *, pkg_name) -> bool:\n \"\"\"Returns True if a package can be installed, False otherwise.\"\"\"\n return True\n\n def unreachable(self, *, pkg_name: str, when_spec: spack.spec.Spec) -> bool:\n \"\"\"Returns true if the context can determine that the condition cannot ever\n be met on pkg_name.\n \"\"\"\n return False\n\n def candidate_targets(self) -> List[spack.vendor.archspec.cpu.Microarchitecture]:\n \"\"\"Returns a list of targets that are candidate for concretization\"\"\"\n platform = spack.platforms.host()\n default_target = spack.vendor.archspec.cpu.TARGETS[platform.default]\n\n # Construct the list of targets which are compatible with the host\n candidate_targets = [default_target] + default_target.ancestors\n granularity = self.configuration.get(\"concretizer:targets:granularity\")\n host_compatible = self.configuration.get(\"concretizer:targets:host_compatible\")\n\n # Add targets which are not compatible with the current host\n if not host_compatible:\n additional_targets_in_family = sorted(\n [\n t\n for t in spack.vendor.archspec.cpu.TARGETS.values()\n if (t.family.name == default_target.family.name and t not in candidate_targets)\n ],\n key=lambda x: len(x.ancestors),\n reverse=True,\n )\n candidate_targets += additional_targets_in_family\n\n # Check if we want only generic architecture\n if granularity == \"generic\":\n candidate_targets = [t for t in candidate_targets if t.vendor == \"generic\"]\n\n return candidate_targets\n\n def possible_dependencies(\n self,\n *specs: Union[spack.spec.Spec, str],\n allowed_deps: dt.DepFlag,\n transitive: bool = True,\n strict_depflag: bool = False,\n expand_virtuals: bool = True,\n ) -> PossibleGraph:\n stack = [x for x in self._package_list(specs)]\n virtuals: Set[str] = set()\n edges: Dict[str, Set[str]] = {}\n\n while stack:\n pkg_name = stack.pop()\n\n if pkg_name in edges:\n continue\n\n edges[pkg_name] = set()\n\n # Since libc is not buildable, there is no need to extend the\n # search space with libc dependencies.\n if pkg_name in self.libc_pkgs:\n continue\n\n pkg_cls = self.repo.get_pkg_class(pkg_name=pkg_name)\n for when_spec, dependencies in pkg_cls.dependencies.items():\n # Check if we need to process this condition at all. We can skip the unreachable\n # check if all dependencies in this condition are already accounted for.\n new_dependencies: List[str] = []\n for name, dep in dependencies.items():\n if strict_depflag:\n if dep.depflag != allowed_deps:\n continue\n elif not (dep.depflag & allowed_deps):\n continue\n\n if name in edges[pkg_name] or name in virtuals:\n continue\n\n new_dependencies.append(name)\n\n if not new_dependencies:\n continue\n\n if self.unreachable(pkg_name=pkg_name, when_spec=when_spec):\n tty.debug(\n f\"[{__name__}] Skipping {', '.join(new_dependencies)} dependencies of \"\n f\"{pkg_name}, because {when_spec} is not met\"\n )\n continue\n\n for name in new_dependencies:\n dep_names: Set[str] = set()\n if self.is_virtual(name):\n virtuals.add(name)\n if expand_virtuals:\n providers = self.providers_for(name)\n dep_names = {spec.name for spec in providers}\n else:\n dep_names = {name}\n\n edges[pkg_name].update(dep_names)\n\n if not transitive:\n continue\n\n for dep_name in dep_names:\n if dep_name in edges:\n continue\n\n if not self._is_possible(pkg_name=dep_name):\n continue\n\n stack.append(dep_name)\n\n real_packages = set(edges)\n if not transitive:\n # We exit early, so add children from the edges information\n for root, children in edges.items():\n real_packages.update(x for x in children if self._is_possible(pkg_name=x))\n\n return PossibleGraph(real_pkgs=real_packages, virtuals=virtuals, edges=edges)\n\n def _package_list(self, specs: Tuple[Union[spack.spec.Spec, str], ...]) -> List[str]:\n stack = []\n for current_spec in specs:\n if isinstance(current_spec, str):\n current_spec = spack.spec.Spec(current_spec)\n\n if self.repo.is_virtual(current_spec.name):\n stack.extend([p.name for p in self.providers_for(current_spec.name)])\n continue\n\n stack.append(current_spec.name)\n return sorted(set(stack))\n\n def _has_deptypes(self, dependencies, *, allowed_deps: dt.DepFlag, strict: bool) -> bool:\n if strict is True:\n return any(\n dep.depflag == allowed_deps for deplist in dependencies.values() for dep in deplist\n )\n return any(\n dep.depflag & allowed_deps for deplist in dependencies.values() for dep in deplist\n )\n\n def _is_possible(self, *, pkg_name):\n try:\n return self.is_allowed_on_this_platform(pkg_name=pkg_name) and self.can_be_installed(\n pkg_name=pkg_name\n )\n except spack.repo.UnknownPackageError:\n return False\n\n\nclass StaticAnalysis(NoStaticAnalysis):\n \"\"\"Performs some static analysis of the configuration, store, etc. to provide more precise\n answers on whether some packages can be installed, or used as a provider.\n\n It increases the setup time, but might decrease the grounding and solve time considerably,\n especially when requirements restrict the possible choices for providers.\n \"\"\"\n\n def __init__(\n self,\n *,\n configuration: spack.config.Configuration,\n repo: spack.repo.RepoPath,\n store: spack.store.Store,\n binary_index: spack.binary_distribution.BinaryCacheIndex,\n ):\n self.store = store\n self.binary_index = binary_index\n super().__init__(configuration=configuration, repo=repo)\n\n @lang.memoized\n def providers_for(self, virtual_str: str) -> List[spack.spec.Spec]:\n candidates = super().providers_for(virtual_str)\n result = []\n for spec in candidates:\n if not self._is_provider_candidate(pkg_name=spec.name, virtual=virtual_str):\n continue\n result.append(spec)\n return result\n\n @lang.memoized\n def buildcache_specs(self) -> List[spack.spec.Spec]:\n self.binary_index.update()\n return self.binary_index.get_all_built_specs()\n\n @lang.memoized\n def can_be_installed(self, *, pkg_name) -> bool:\n if self.configuration.get(f\"packages:{pkg_name}:buildable\", True):\n return True\n\n if self.configuration.get(f\"packages:{pkg_name}:externals\", []):\n return True\n\n reuse = self.configuration.get(\"concretizer:reuse\")\n if reuse is not False and self.store.db.query(pkg_name):\n return True\n\n if reuse is not False and any(x.name == pkg_name for x in self.buildcache_specs()):\n return True\n\n tty.debug(f\"[{__name__}] {pkg_name} cannot be installed\")\n return False\n\n @lang.memoized\n def _is_provider_candidate(self, *, pkg_name: str, virtual: str) -> bool:\n if not self.is_allowed_on_this_platform(pkg_name=pkg_name):\n return False\n\n if not self.can_be_installed(pkg_name=pkg_name):\n return False\n\n virtual_spec = spack.spec.Spec(virtual)\n if self.unreachable(pkg_name=virtual_spec.name, when_spec=pkg_name):\n tty.debug(f\"[{__name__}] {pkg_name} cannot be a provider for {virtual}\")\n return False\n\n return True\n\n @lang.memoized\n def unreachable(self, *, pkg_name: str, when_spec: spack.spec.Spec) -> bool:\n \"\"\"Returns true if the context can determine that the condition cannot ever\n be met on pkg_name.\n \"\"\"\n candidates = self.configuration.get(f\"packages:{pkg_name}:require\", [])\n if not candidates and pkg_name != \"all\":\n return self.unreachable(pkg_name=\"all\", when_spec=when_spec)\n\n if not candidates:\n return False\n\n if isinstance(candidates, str):\n candidates = [candidates]\n\n union_requirement = spack.spec.Spec()\n for c in candidates:\n if not isinstance(c, str):\n continue\n try:\n union_requirement.constrain(c)\n except SpackError:\n # Less optimized, but shouldn't fail\n pass\n\n if not union_requirement.intersects(when_spec):\n return True\n\n return False\n\n\ndef create_graph_analyzer() -> PossibleDependencyGraph:\n static_analysis = spack.config.CONFIG.get(\"concretizer:static_analysis\", False)\n if static_analysis:\n return StaticAnalysis(\n configuration=spack.config.CONFIG,\n repo=spack.repo.PATH,\n store=spack.store.STORE,\n binary_index=spack.binary_distribution.BINARY_INDEX,\n )\n return NoStaticAnalysis(configuration=spack.config.CONFIG, repo=spack.repo.PATH)\n\n\nclass Counter:\n \"\"\"Computes the possible packages and the maximum number of duplicates\n allowed for each of them.\n\n Args:\n specs: abstract specs to concretize\n tests: if True, add test dependencies to the list of possible packages\n \"\"\"\n\n def __init__(\n self,\n specs: List[spack.spec.Spec],\n tests: spack.concretize.TestsType,\n possible_graph: PossibleDependencyGraph,\n ) -> None:\n self.possible_graph = possible_graph\n self.specs = specs\n self.link_run_types: dt.DepFlag = dt.LINK | dt.RUN | dt.TEST\n self.all_types: dt.DepFlag = dt.ALL\n if not tests:\n self.link_run_types = dt.LINK | dt.RUN\n self.all_types = dt.LINK | dt.RUN | dt.BUILD\n\n self._possible_dependencies: Set[str] = set()\n self._possible_virtuals: Set[str] = {\n x.name for x in specs if spack.repo.PATH.is_virtual(x.name)\n }\n\n def possible_dependencies(self) -> Set[str]:\n \"\"\"Returns the list of possible dependencies\"\"\"\n self.ensure_cache_values()\n return self._possible_dependencies\n\n def possible_virtuals(self) -> Set[str]:\n \"\"\"Returns the list of possible virtuals\"\"\"\n self.ensure_cache_values()\n return self._possible_virtuals\n\n def ensure_cache_values(self) -> None:\n \"\"\"Ensure the cache values have been computed\"\"\"\n if self._possible_dependencies:\n return\n self._compute_cache_values()\n\n def possible_packages_facts(self, gen: \"spack.solver.asp.ProblemInstanceBuilder\", fn) -> None:\n \"\"\"Emit facts associated with the possible packages\"\"\"\n raise NotImplementedError(\"must be implemented by derived classes\")\n\n def _compute_cache_values(self) -> None:\n raise NotImplementedError(\"must be implemented by derived classes\")\n\n\nclass NoDuplicatesCounter(Counter):\n def _compute_cache_values(self) -> None:\n self._possible_dependencies, virtuals, _ = self.possible_graph.possible_dependencies(\n *self.specs, allowed_deps=self.all_types\n )\n self._possible_virtuals.update(virtuals)\n\n def possible_packages_facts(self, gen: \"spack.solver.asp.ProblemInstanceBuilder\", fn) -> None:\n gen.h2(\"Maximum number of nodes (packages)\")\n for package_name in sorted(self.possible_dependencies()):\n gen.fact(fn.max_dupes(package_name, 1))\n gen.newline()\n gen.h2(\"Maximum number of nodes (virtual packages)\")\n for package_name in sorted(self.possible_virtuals()):\n gen.fact(fn.max_dupes(package_name, 1))\n gen.newline()\n gen.h2(\"Possible package in link-run subDAG\")\n for name in sorted(self.possible_dependencies()):\n gen.fact(fn.possible_in_link_run(name))\n gen.newline()\n\n\nclass MinimalDuplicatesCounter(NoDuplicatesCounter):\n def __init__(\n self,\n specs: List[spack.spec.Spec],\n tests: spack.concretize.TestsType,\n possible_graph: PossibleDependencyGraph,\n ) -> None:\n super().__init__(specs, tests, possible_graph)\n self._link_run: Set[str] = set()\n self._direct_build: Set[str] = set()\n self._total_build: Set[str] = set()\n self._link_run_virtuals: Set[str] = set()\n\n def _compute_cache_values(self) -> None:\n self._link_run, virtuals, _ = self.possible_graph.possible_dependencies(\n *self.specs, allowed_deps=self.link_run_types\n )\n self._possible_virtuals.update(virtuals)\n self._link_run_virtuals.update(virtuals)\n for x in self._link_run:\n reals, virtuals, _ = self.possible_graph.possible_dependencies(\n x, allowed_deps=dt.BUILD, transitive=False, strict_depflag=True\n )\n self._possible_virtuals.update(virtuals)\n self._direct_build.update(reals)\n\n self._total_build, virtuals, _ = self.possible_graph.possible_dependencies(\n *self._direct_build, allowed_deps=self.all_types\n )\n self._possible_virtuals.update(virtuals)\n self._possible_dependencies = set(self._link_run) | set(self._total_build)\n\n def possible_packages_facts(self, gen, fn):\n build_tools = set()\n for current_tag in (\"build-tools\", \"compiler\"):\n build_tools.update(spack.repo.PATH.packages_with_tags(current_tag))\n\n gen.h2(\"Packages with at most a single node\")\n for package_name in sorted(self.possible_dependencies() - build_tools):\n gen.fact(fn.max_dupes(package_name, 1))\n gen.newline()\n\n gen.h2(\"Packages with multiple possible nodes (build-tools)\")\n default = spack.config.CONFIG.get(\"concretizer:duplicates:max_dupes:default\", 1)\n duplicates = spack.config.CONFIG.get(\"concretizer:duplicates:max_dupes\", {})\n for package_name in sorted(self.possible_dependencies() & build_tools):\n max_dupes = duplicates.get(package_name, default)\n gen.fact(fn.max_dupes(package_name, max_dupes))\n if max_dupes > 1:\n gen.fact(fn.multiple_unification_sets(package_name))\n gen.newline()\n\n gen.h2(\"Maximum number of nodes (virtuals)\")\n for package_name in sorted(self.possible_virtuals()):\n max_dupes = duplicates.get(package_name, default)\n gen.fact(fn.max_dupes(package_name, max_dupes))\n gen.newline()\n\n gen.h2(\"Possible package in link-run subDAG\")\n for name in sorted(self._link_run):\n gen.fact(fn.possible_in_link_run(name))\n gen.newline()\n\n\nclass FullDuplicatesCounter(MinimalDuplicatesCounter):\n def possible_packages_facts(self, gen, fn):\n counter = collections.Counter(\n list(self._link_run) + list(self._total_build) + list(self._direct_build)\n )\n gen.h2(\"Maximum number of nodes\")\n for pkg, count in sorted(counter.items(), key=lambda x: (x[1], x[0])):\n count = min(count, 2)\n gen.fact(fn.max_dupes(pkg, count))\n gen.newline()\n\n gen.h2(\"Build unification sets \")\n build_tools = set()\n for current_tag in (\"build-tools\", \"compiler\"):\n build_tools.update(spack.repo.PATH.packages_with_tags(current_tag))\n\n for name in sorted(self.possible_dependencies() & build_tools):\n gen.fact(fn.multiple_unification_sets(name))\n gen.newline()\n\n gen.h2(\"Possible package in link-run subDAG\")\n for name in sorted(self._link_run):\n gen.fact(fn.possible_in_link_run(name))\n gen.newline()\n\n counter = collections.Counter(\n list(self._link_run_virtuals) + list(self._possible_virtuals)\n )\n gen.h2(\"Maximum number of virtual nodes\")\n for pkg, count in sorted(counter.items(), key=lambda x: (x[1], x[0])):\n gen.fact(fn.max_dupes(pkg, count))\n gen.newline()\n\n\ndef create_counter(\n specs: List[spack.spec.Spec],\n tests: spack.concretize.TestsType,\n possible_graph: PossibleDependencyGraph,\n) -> Counter:\n strategy = spack.config.CONFIG.get(\"concretizer:duplicates:strategy\", \"none\")\n if strategy == \"full\":\n return FullDuplicatesCounter(specs, tests=tests, possible_graph=possible_graph)\n if strategy == \"minimal\":\n return MinimalDuplicatesCounter(specs, tests=tests, possible_graph=possible_graph)\n return NoDuplicatesCounter(specs, tests=tests, possible_graph=possible_graph)\n" }, { "path": "lib/spack/spack/solver/libc_compatibility.lp", "content": "% Copyright Spack Project Developers. See COPYRIGHT file for details.\n%\n% SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n%=============================================================================\n% Libc compatibility rules for reusing solves.\n%\n% These rules are used on Linux\n%=============================================================================\n\n% Non-libc reused specs must be host libc compatible. In case we build packages, we get a\n% host compatible libc provider from other rules. If nothing is built, there is no libc provider,\n% since it's pruned from reusable specs, meaning we have to explicitly impose reused specs are host\n% compatible.\n\n% A package cannot be reused if it needs a libc that is not compatible with the current one\nerror(100, \"Cannot reuse {0} since we cannot determine libc compatibility\", ReusedPackage)\n :- provider(node(X, LibcPackage), node(0, \"libc\")),\n attr(\"version\", node(X, LibcPackage), LibcVersion),\n concrete(node(R, ReusedPackage)),\n attr(\"needs_libc\", node(R, ReusedPackage)),\n not attr(\"compatible_libc\", node(R, ReusedPackage), LibcPackage, LibcVersion).\n\n% In case we don't need a provider for libc, ensure there's at least one compatible libc on the host\nerror(100, \"Cannot reuse {0} since we cannot determine libc compatibility\", ReusedPackage)\n :- not provider(_, node(0, \"libc\")),\n concrete(node(R, ReusedPackage)),\n attr(\"needs_libc\", node(R, ReusedPackage)),\n not attr(\"compatible_libc\", node(R, ReusedPackage), _, _).\n\n% The libc provider must be one that a compiler can target\n:- will_build_packages(),\n provider(node(X, LibcPackage), node(0, \"libc\")),\n attr(\"node\", node(X, LibcPackage)),\n attr(\"version\", node(X, LibcPackage), LibcVersion),\n not host_libc(LibcPackage, LibcVersion).\n" }, { "path": "lib/spack/spack/solver/os_compatibility.lp", "content": "% Copyright Spack Project Developers. See COPYRIGHT file for details.\n%\n% SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n%=============================================================================\n% OS compatibility rules for reusing solves.\n% os_compatible(RecentOS, OlderOS)\n% OlderOS binaries can be used on RecentOS\n%\n% These rules are used on every platform, but Linux\n%=============================================================================\n\n% macOS\nos_compatible(\"tahoe\", \"sequoia\").\nos_compatible(\"sequoia\", \"sonoma\").\nos_compatible(\"sonoma\", \"ventura\").\nos_compatible(\"ventura\", \"monterey\").\nos_compatible(\"monterey\", \"bigsur\").\nos_compatible(\"bigsur\", \"catalina\").\n\n% can't have dependencies on incompatible OS's\nerror(100, \"{0} and dependency {1} have incompatible operating systems 'os={2}' and 'os={3}'\", Package, Dependency, PackageNodeOS, DependencyOS)\n :- depends_on(node(X, Package), node(Y, Dependency)),\n attr(\"node_os\", node(X, Package), PackageNodeOS),\n attr(\"node_os\", node(Y, Dependency), DependencyOS),\n not os_compatible(PackageNodeOS, DependencyOS),\n build(node(X, Package)).\n\n% We can select only operating systems compatible with the ones\n% for which we can build software. We need a cardinality constraint\n% since we might have more than one \"buildable_os(OS)\" fact.\n:- not 1 { os_compatible(CurrentOS, ReusedOS) : buildable_os(CurrentOS) },\n attr(\"node_os\", Package, ReusedOS).\n" }, { "path": "lib/spack/spack/solver/requirements.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport enum\nimport warnings\nfrom typing import List, NamedTuple, Optional, Sequence, Tuple, Union\n\nimport spack.vendor.archspec.cpu\n\nimport spack.config\nimport spack.error\nimport spack.package_base\nimport spack.repo\nimport spack.spec\nimport spack.spec_parser\nimport spack.traverse\nimport spack.util.spack_yaml\nfrom spack.enums import PropagationPolicy\nfrom spack.llnl.util import tty\nfrom spack.util.spack_yaml import get_mark_from_yaml_data\n\n\ndef _mark_str(raw) -> str:\n \"\"\"Return a 'file:line: ' prefix from the YAML mark on *raw*, or empty string.\"\"\"\n mark = get_mark_from_yaml_data(raw)\n return f\"{mark.name}:{mark.line + 1}: \" if mark else \"\"\n\n\ndef _check_unknown_targets(\n raw_strs: List[str], specs: List[\"spack.spec.Spec\"], *, always_warn: bool = False\n) -> None:\n \"\"\"Either warns or raises for unknown concrete target names in a set of specs.\n\n UserWarnings are emitted if *always_warn* is True or if there is at least one spec without\n unknown targets. If all the specs have unknown targets raises an error.\n \"\"\"\n specs_with_unknown_targets = [\n (raw, spec)\n for raw, spec in zip(raw_strs, specs)\n if spec.architecture\n and spec.architecture.target_concrete\n and spec.target.name not in spack.vendor.archspec.cpu.TARGETS\n ]\n if not specs_with_unknown_targets:\n return\n\n errors = [\n f\"{_mark_str(raw)}'{spec}' contains unknown targets\"\n for raw, spec in specs_with_unknown_targets\n ]\n if len(errors) == 1:\n msg = f\"{errors[0]}. Run 'spack arch --known-targets' to see valid targets.\"\n else:\n details = \"\\n\".join([f\"{idx}. {part}\" for idx, part in enumerate(errors, 1)])\n msg = (\n f\"unknown targets have been detected in requirements\\n{details}\\n\"\n f\"Run 'spack arch --known-targets' to see valid targets.\"\n )\n if not always_warn and len(specs_with_unknown_targets) == len(specs):\n raise spack.error.SpecError(msg)\n warnings.warn(msg)\n\n\nclass RequirementKind(enum.Enum):\n \"\"\"Purpose / provenance of a requirement\"\"\"\n\n #: Default requirement expressed under the 'all' attribute of packages.yaml\n DEFAULT = enum.auto()\n #: Requirement expressed on a virtual package\n VIRTUAL = enum.auto()\n #: Requirement expressed on a specific package\n PACKAGE = enum.auto()\n\n\nclass RequirementOrigin(enum.Enum):\n \"\"\"Origin of a requirement\"\"\"\n\n REQUIRE_YAML = enum.auto()\n PREFER_YAML = enum.auto()\n CONFLICT_YAML = enum.auto()\n DIRECTIVE = enum.auto()\n INPUT_SPECS = enum.auto()\n\n\nclass RequirementRule(NamedTuple):\n \"\"\"Data class to collect information on a requirement\"\"\"\n\n pkg_name: str\n policy: str\n origin: RequirementOrigin\n requirements: Sequence[spack.spec.Spec]\n condition: spack.spec.Spec\n kind: RequirementKind\n message: Optional[str]\n\n\ndef preference(\n pkg_name: str,\n constraint: spack.spec.Spec,\n condition: spack.spec.Spec = spack.spec.Spec(),\n origin: RequirementOrigin = RequirementOrigin.PREFER_YAML,\n kind: RequirementKind = RequirementKind.PACKAGE,\n message: Optional[str] = None,\n) -> RequirementRule:\n \"\"\"Returns a preference rule\"\"\"\n # A strong preference is defined as:\n #\n # require:\n # - any_of: [spec_str, \"@:\"]\n return RequirementRule(\n pkg_name=pkg_name,\n policy=\"any_of\",\n requirements=[constraint, spack.spec.Spec(\"@:\")],\n kind=kind,\n condition=condition,\n origin=origin,\n message=message,\n )\n\n\ndef conflict(\n pkg_name: str,\n constraint: spack.spec.Spec,\n condition: spack.spec.Spec = spack.spec.Spec(),\n origin: RequirementOrigin = RequirementOrigin.CONFLICT_YAML,\n kind: RequirementKind = RequirementKind.PACKAGE,\n message: Optional[str] = None,\n) -> RequirementRule:\n \"\"\"Returns a conflict rule\"\"\"\n # A conflict is defined as:\n #\n # require:\n # - one_of: [spec_str, \"@:\"]\n return RequirementRule(\n pkg_name=pkg_name,\n policy=\"one_of\",\n requirements=[constraint, spack.spec.Spec(\"@:\")],\n kind=kind,\n condition=condition,\n origin=origin,\n message=message,\n )\n\n\nclass RequirementParser:\n \"\"\"Parses requirements from package.py files and configuration, and returns rules.\"\"\"\n\n def __init__(self, configuration: spack.config.Configuration):\n self.config = configuration\n self.runtime_pkgs = spack.repo.PATH.packages_with_tags(\"runtime\")\n self.compiler_pkgs = spack.repo.PATH.packages_with_tags(\"compiler\")\n self.preferences_from_input: List[Tuple[spack.spec.Spec, str]] = []\n self.toolchains = configuration.get_config(\"toolchains\")\n self._warned_compiler_all: set = set()\n\n def _parse_and_expand(self, string: str, *, named: bool = False) -> spack.spec.Spec:\n result = parse_spec_from_yaml_string(string, named=named)\n if self.toolchains:\n spack.spec_parser.expand_toolchains(result, self.toolchains)\n return result\n\n def rules(self, pkg: spack.package_base.PackageBase) -> List[RequirementRule]:\n result = []\n result.extend(self.rules_from_input_specs(pkg))\n result.extend(self.rules_from_package_py(pkg))\n result.extend(self.rules_from_require(pkg))\n result.extend(self.rules_from_prefer(pkg))\n result.extend(self.rules_from_conflict(pkg))\n return result\n\n def parse_rules_from_input_specs(self, specs: Sequence[spack.spec.Spec]):\n self.preferences_from_input.clear()\n for edge in spack.traverse.traverse_edges(specs, root=False):\n if edge.propagation == PropagationPolicy.PREFERENCE:\n for constraint in _split_edge_on_virtuals(edge):\n root_name = edge.parent.name\n self.preferences_from_input.append((constraint, root_name))\n\n def rules_from_input_specs(self, pkg: spack.package_base.PackageBase) -> List[RequirementRule]:\n return [\n preference(\n pkg.name,\n constraint=s,\n condition=spack.spec.Spec(f\"{root_name} ^[deptypes=link,run]{pkg.name}\"),\n origin=RequirementOrigin.INPUT_SPECS,\n )\n for s, root_name in self.preferences_from_input\n ]\n\n def rules_from_package_py(self, pkg: spack.package_base.PackageBase) -> List[RequirementRule]:\n rules = []\n for when_spec, requirement_list in pkg.requirements.items():\n for requirements, policy, message in requirement_list:\n rules.append(\n RequirementRule(\n pkg_name=pkg.name,\n policy=policy,\n requirements=requirements,\n kind=RequirementKind.PACKAGE,\n condition=when_spec,\n message=message,\n origin=RequirementOrigin.DIRECTIVE,\n )\n )\n return rules\n\n def rules_from_virtual(self, virtual_str: str) -> List[RequirementRule]:\n kind, requests = self._raw_yaml_data(virtual_str, section=\"require\", virtual=True)\n result = self._rules_from_requirements(virtual_str, requests, kind=kind)\n\n kind, requests = self._raw_yaml_data(virtual_str, section=\"prefer\", virtual=True)\n result.extend(self._rules_from_preferences(virtual_str, preferences=requests, kind=kind))\n\n kind, requests = self._raw_yaml_data(virtual_str, section=\"conflict\", virtual=True)\n result.extend(self._rules_from_conflicts(virtual_str, conflicts=requests, kind=kind))\n\n return result\n\n def rules_from_require(self, pkg: spack.package_base.PackageBase) -> List[RequirementRule]:\n kind, requirements = self._raw_yaml_data(pkg.name, section=\"require\")\n return self._rules_from_requirements(pkg.name, requirements, kind=kind)\n\n def rules_from_prefer(self, pkg: spack.package_base.PackageBase) -> List[RequirementRule]:\n kind, preferences = self._raw_yaml_data(pkg.name, section=\"prefer\")\n return self._rules_from_preferences(pkg.name, preferences=preferences, kind=kind)\n\n def _rules_from_preferences(\n self, pkg_name: str, *, preferences, kind: RequirementKind\n ) -> List[RequirementRule]:\n result = []\n for item in preferences:\n if kind == RequirementKind.DEFAULT:\n # Warn about %gcc type of preferences under `all`.\n self._maybe_warn_compiler_in_all(item, \"prefer\")\n spec, condition, msg = self._parse_prefer_conflict_item(item)\n result.append(\n preference(pkg_name, constraint=spec, condition=condition, kind=kind, message=msg)\n )\n return result\n\n def rules_from_conflict(self, pkg: spack.package_base.PackageBase) -> List[RequirementRule]:\n kind, conflicts = self._raw_yaml_data(pkg.name, section=\"conflict\")\n return self._rules_from_conflicts(pkg.name, conflicts=conflicts, kind=kind)\n\n def _rules_from_conflicts(\n self, pkg_name: str, *, conflicts, kind: RequirementKind\n ) -> List[RequirementRule]:\n result = []\n for item in conflicts:\n spec, condition, msg = self._parse_prefer_conflict_item(item)\n result.append(\n conflict(pkg_name, constraint=spec, condition=condition, kind=kind, message=msg)\n )\n return result\n\n def _parse_prefer_conflict_item(self, item):\n # The item is either a string or an object with at least a \"spec\" attribute\n if isinstance(item, str):\n spec = self._parse_and_expand(item)\n condition = spack.spec.Spec()\n message = None\n else:\n spec = self._parse_and_expand(item[\"spec\"])\n condition = spack.spec.Spec(item.get(\"when\"))\n message = item.get(\"message\")\n raw_key = item if isinstance(item, str) else item.get(\"spec\", item)\n _check_unknown_targets([raw_key], [spec], always_warn=True)\n return spec, condition, message\n\n def _raw_yaml_data(self, pkg_name: str, *, section: str, virtual: bool = False):\n config = self.config.get_config(\"packages\")\n data = config.get(pkg_name, {}).get(section, [])\n kind = RequirementKind.PACKAGE\n\n if virtual:\n return RequirementKind.VIRTUAL, data\n\n if not data:\n data = config.get(\"all\", {}).get(section, [])\n kind = RequirementKind.DEFAULT\n return kind, data\n\n def _rules_from_requirements(\n self, pkg_name: str, requirements, *, kind: RequirementKind\n ) -> List[RequirementRule]:\n \"\"\"Manipulate requirements from packages.yaml, and return a list of tuples\n with a uniform structure (name, policy, requirements).\n \"\"\"\n if isinstance(requirements, str):\n requirements = [requirements]\n\n rules = []\n for requirement in requirements:\n # A string is equivalent to a one_of group with a single element\n if isinstance(requirement, str):\n requirement = {\"one_of\": [requirement]}\n\n for policy in (\"spec\", \"one_of\", \"any_of\"):\n if policy not in requirement:\n continue\n\n constraints = requirement[policy]\n # \"spec\" is for specifying a single spec\n if policy == \"spec\":\n constraints = [constraints]\n policy = \"one_of\"\n\n if kind == RequirementKind.DEFAULT:\n # Warn about %gcc type of requirements under `all`.\n self._maybe_warn_compiler_in_all(constraints, \"require\")\n\n # validate specs from YAML first, and fail with line numbers if parsing fails.\n raw_strs = list(constraints)\n constraints = [\n self._parse_and_expand(constraint, named=kind == RequirementKind.VIRTUAL)\n for constraint in raw_strs\n ]\n _check_unknown_targets(raw_strs, constraints)\n when_str = requirement.get(\"when\")\n when = self._parse_and_expand(when_str) if when_str else spack.spec.Spec()\n\n constraints = [\n x\n for x in constraints\n if not self.reject_requirement_constraint(pkg_name, constraint=x, kind=kind)\n ]\n if not constraints:\n continue\n\n rules.append(\n RequirementRule(\n pkg_name=pkg_name,\n policy=policy,\n requirements=constraints,\n kind=kind,\n message=requirement.get(\"message\"),\n condition=when,\n origin=RequirementOrigin.REQUIRE_YAML,\n )\n )\n return rules\n\n def reject_requirement_constraint(\n self, pkg_name: str, *, constraint: spack.spec.Spec, kind: RequirementKind\n ) -> bool:\n \"\"\"Returns True if a requirement constraint should be rejected\"\"\"\n # If it's a specific package requirement, it's never rejected\n if kind != RequirementKind.DEFAULT:\n return False\n\n # Reject requirements with dependencies for runtimes and compilers\n # These are usually requests on compilers, in the form of %\n involves_dependencies = bool(constraint.dependencies())\n if involves_dependencies and (\n pkg_name in self.runtime_pkgs or pkg_name in self.compiler_pkgs\n ):\n tty.debug(f\"[{__name__}] Rejecting '{constraint}' for compiler package {pkg_name}\")\n return True\n\n # Requirements under all: are applied only if they are satisfiable considering only\n # package rules, so e.g. variants must exist etc. Otherwise, they are rejected.\n try:\n s = spack.spec.Spec(pkg_name)\n s.constrain(constraint)\n s.validate_or_raise()\n except spack.error.SpackError as e:\n tty.debug(\n f\"[{__name__}] Rejecting the default '{constraint}' requirement \"\n f\"on '{pkg_name}': {str(e)}\"\n )\n return True\n return False\n\n def _maybe_warn_compiler_in_all(self, items: Union[str, list, dict], section: str) -> None:\n \"\"\"Warn once if a packages:all: prefer/require entry has compiler dependencies.\"\"\"\n # Stick to single items, not complex one_of / any_of groups to keep things simple.\n if isinstance(items, str):\n spec_str = items\n elif isinstance(items, dict) and \"spec\" in items and isinstance(items[\"spec\"], str):\n spec_str = items[\"spec\"]\n elif isinstance(items, list) and len(items) == 1 and isinstance(items[0], str):\n spec_str = items[0]\n else:\n return\n if spec_str in self._warned_compiler_all:\n return\n self._warned_compiler_all.add(spec_str)\n suggestions = []\n for edge in self._parse_and_expand(spec_str).edges_to_dependencies():\n if edge.when != spack.spec.EMPTY_SPEC:\n # Conditional dependencies are fine (includes toolchains after expansion).\n continue\n elif edge.virtuals:\n # The case `%c,cxx=gcc` or similar.\n keys = edge.virtuals\n comment = \"\"\n elif edge.spec.name in self.compiler_pkgs:\n # Just a package `%gcc`.\n keys = (\"c\",)\n comment = \"# For each language virtual (c, cxx, fortran, ...):\\n\"\n else:\n # Maybe %mpich or so? Just give a generic suggestion.\n keys = (\"\",)\n comment = \"# For each virtual:\\n\"\n data = {\"packages\": {k: {section: [str(edge.spec)]} for k in keys}}\n suggestion = spack.util.spack_yaml.dump(data).rstrip()\n suggestions.append(f\"{comment}{suggestion}\")\n if suggestions:\n mark = get_mark_from_yaml_data(spec_str)\n location = f\"{mark.name}:{mark.line + 1}: \" if mark else \"\"\n prefix = (\n f\"{location}'packages: all: {section}: [\\\"{spec_str}\\\"]' applies a dependency \"\n f\"constraint to all packages\"\n )\n suffix = \"Consider instead:\\n\" + \"\\n\".join(suggestions)\n if section == \"prefer\":\n warnings.warn(\n f\"{prefix}. This can lead to unexpected concretizations. This was likely \"\n f\"intended as a preference for a provider of a (language) virtual. {suffix}\"\n )\n else:\n warnings.warn(\n f\"{prefix}. This often leads to concretization errors. This was likely \"\n f\"intended as a requirement for a provider of a (language) virtual. {suffix}\"\n )\n\n\ndef _split_edge_on_virtuals(edge: spack.spec.DependencySpec) -> List[spack.spec.Spec]:\n \"\"\"Split the edge on virtuals and removes the parent.\"\"\"\n if not edge.virtuals:\n return [spack.spec.Spec(str(edge.copy(keep_parent=False)))]\n\n result = []\n # We split on virtuals so that \"%%c,cxx=gcc\" enforces \"%%c=gcc\" and \"%%cxx=gcc\" separately\n for v in edge.virtuals:\n t = edge.copy(keep_parent=False, keep_virtuals=False)\n t.update_virtuals(v)\n t.when = spack.spec.Spec(f\"%{v}\")\n result.append(spack.spec.Spec(str(t)))\n\n return result\n\n\ndef parse_spec_from_yaml_string(string: str, *, named: bool = False) -> spack.spec.Spec:\n \"\"\"Parse a spec from YAML and add file/line info to errors, if it's available.\n\n Parse a ``Spec`` from the supplied string, but also intercept any syntax errors and\n add file/line information for debugging using file/line annotations from the string.\n\n Args:\n string: a string representing a ``Spec`` from config YAML.\n named: if True, the spec must have a name\n \"\"\"\n try:\n result = spack.spec.Spec(string)\n except spack.error.SpecSyntaxError as e:\n mark = get_mark_from_yaml_data(string)\n if mark:\n msg = f\"{mark.name}:{mark.line + 1}: {str(e)}\"\n raise spack.error.SpecSyntaxError(msg) from e\n raise e\n\n if named is True and not result.name:\n msg = f\"expected a named spec, but got '{string}' instead\"\n mark = get_mark_from_yaml_data(string)\n\n # Add a hint in case it's dependencies\n deps = result.dependencies()\n if len(deps) == 1:\n msg = f\"{msg}. Did you mean '{deps[0]}'?\"\n\n if mark:\n msg = f\"{mark.name}:{mark.line + 1}: {msg}\"\n\n raise spack.error.SpackError(msg)\n\n return result\n" }, { "path": "lib/spack/spack/solver/reuse.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport enum\nimport functools\nimport typing\nfrom typing import Any, Callable, List, Mapping, Optional\n\nimport spack.binary_distribution\nimport spack.config\nimport spack.llnl.path\nimport spack.repo\nimport spack.spec\nimport spack.store\nimport spack.traverse\nfrom spack.externals import (\n ExternalSpecsParser,\n complete_architecture,\n complete_variants_and_architecture,\n extract_dicts_from_configuration,\n)\nfrom spack.spec_filter import SpecFilter\n\nfrom .runtimes import all_libcs\n\nif typing.TYPE_CHECKING:\n import spack.environment\n\n\ndef spec_filter_from_store(\n configuration, *, packages_with_externals, include, exclude\n) -> SpecFilter:\n \"\"\"Constructs a filter that takes the specs from the current store.\"\"\"\n is_reusable = functools.partial(\n _is_reusable, packages_with_externals=packages_with_externals, local=True\n )\n factory = functools.partial(_specs_from_store, configuration=configuration)\n return SpecFilter(factory=factory, is_usable=is_reusable, include=include, exclude=exclude)\n\n\ndef spec_filter_from_buildcache(*, packages_with_externals, include, exclude) -> SpecFilter:\n \"\"\"Constructs a filter that takes the specs from the configured buildcaches.\"\"\"\n is_reusable = functools.partial(\n _is_reusable, packages_with_externals=packages_with_externals, local=False\n )\n return SpecFilter(\n factory=_specs_from_mirror, is_usable=is_reusable, include=include, exclude=exclude\n )\n\n\ndef spec_filter_from_environment(*, packages_with_externals, include, exclude, env) -> SpecFilter:\n is_reusable = functools.partial(\n _is_reusable, packages_with_externals=packages_with_externals, local=True\n )\n factory = functools.partial(_specs_from_environment, env=env)\n return SpecFilter(factory=factory, is_usable=is_reusable, include=include, exclude=exclude)\n\n\ndef spec_filter_from_packages_yaml(\n *, external_parser: ExternalSpecsParser, packages_with_externals, include, exclude\n) -> SpecFilter:\n is_reusable = functools.partial(\n _is_reusable, packages_with_externals=packages_with_externals, local=True\n )\n return SpecFilter(\n external_parser.all_specs, is_usable=is_reusable, include=include, exclude=exclude\n )\n\n\ndef _has_runtime_dependencies(spec: spack.spec.Spec) -> bool:\n # Spack v1.0 specs and later\n return spec.original_spec_format() >= 5\n\n\ndef _is_reusable(spec: spack.spec.Spec, packages_with_externals, local: bool) -> bool:\n \"\"\"A spec is reusable if it's not a dev spec, it's imported from the cray manifest, it's not\n external, or it's external with matching packages.yaml entry. The latter prevents two issues:\n\n 1. Externals in build caches: avoid installing an external on the build machine not\n available on the target machine\n 2. Local externals: avoid reusing an external if the local config changes. This helps in\n particular when a user removes an external from packages.yaml, and expects that that\n takes effect immediately.\n\n Arguments:\n spec: the spec to check\n packages_with_externals: the pre-processed packages configuration\n \"\"\"\n if \"dev_path\" in spec.variants:\n return False\n\n if spec.name == \"compiler-wrapper\":\n return False\n\n if not spec.external:\n return _has_runtime_dependencies(spec)\n\n # Cray external manifest externals are always reusable\n if local:\n _, record = spack.store.STORE.db.query_by_spec_hash(spec.dag_hash())\n if record and record.origin == \"external-db\":\n return True\n\n try:\n provided = spack.repo.PATH.get(spec).provided_virtual_names()\n except spack.repo.RepoError:\n provided = []\n\n for name in {spec.name, *provided}:\n for entry in packages_with_externals.get(name, {}).get(\"externals\", []):\n expected_prefix = entry.get(\"prefix\")\n if expected_prefix is not None:\n expected_prefix = spack.llnl.path.path_to_os_path(expected_prefix)[0]\n if (\n spec.satisfies(entry[\"spec\"])\n and spec.external_path == expected_prefix\n and spec.external_modules == entry.get(\"modules\")\n ):\n return True\n\n return False\n\n\ndef _specs_from_store(configuration):\n store = spack.store.create(configuration)\n with store.db.read_transaction():\n return store.db.query(installed=True)\n\n\ndef _specs_from_mirror():\n try:\n return spack.binary_distribution.update_cache_and_get_specs()\n except (spack.binary_distribution.FetchCacheError, IndexError):\n # this is raised when no mirrors had indices.\n # TODO: update mirror configuration so it can indicate that the\n # TODO: source cache (or any mirror really) doesn't have binaries.\n return []\n\n\ndef _specs_from_environment(env):\n \"\"\"Return all concrete specs from the environment. This includes all included concrete\"\"\"\n if env:\n return list(spack.traverse.traverse_nodes([s for _, s in env.concretized_specs()]))\n else:\n return []\n\n\nclass ReuseStrategy(enum.Enum):\n ROOTS = enum.auto()\n DEPENDENCIES = enum.auto()\n NONE = enum.auto()\n\n\ndef create_external_parser(\n packages_with_externals: Any, completion_mode: str\n) -> ExternalSpecsParser:\n \"\"\"Get externals from a pre-processed packages.yaml (with implicit externals).\"\"\"\n external_dicts = extract_dicts_from_configuration(packages_with_externals)\n if completion_mode == \"default_variants\":\n complete_fn = complete_variants_and_architecture\n elif completion_mode == \"architecture_only\":\n complete_fn = complete_architecture\n else:\n raise ValueError(\n f\"Unknown value for concretizer:externals:completion: {completion_mode!r}\"\n )\n return ExternalSpecsParser(external_dicts, complete_node=complete_fn)\n\n\nSpecFiltersFactory = Callable[\n [Callable[[spack.spec.Spec], bool], spack.config.Configuration], List[SpecFilter]\n]\n\n\nclass ReusableSpecsSelector:\n \"\"\"Selects specs that can be reused during concretization.\"\"\"\n\n def __init__(\n self,\n *,\n configuration: spack.config.Configuration,\n external_parser: ExternalSpecsParser,\n packages_with_externals: Any,\n factory: Optional[SpecFiltersFactory] = None,\n ) -> None:\n # Local import to break circular dependencies\n import spack.environment\n\n self.configuration = configuration\n self.store = spack.store.create(configuration)\n self.reuse_strategy = ReuseStrategy.ROOTS\n reuse_yaml = self.configuration.get(\"concretizer:reuse\", False)\n\n self.reuse_sources = []\n if factory is not None:\n is_reusable = functools.partial(\n _is_reusable, packages_with_externals=packages_with_externals, local=True\n )\n self.reuse_sources.extend(factory(is_reusable, configuration))\n\n if not isinstance(reuse_yaml, Mapping):\n self.reuse_sources.append(\n spec_filter_from_packages_yaml(\n external_parser=external_parser,\n packages_with_externals=packages_with_externals,\n include=[],\n exclude=[],\n )\n )\n if reuse_yaml is False:\n self.reuse_strategy = ReuseStrategy.NONE\n return\n\n if reuse_yaml == \"dependencies\":\n self.reuse_strategy = ReuseStrategy.DEPENDENCIES\n self.reuse_sources.extend(\n [\n spec_filter_from_store(\n configuration=self.configuration,\n packages_with_externals=packages_with_externals,\n include=[],\n exclude=[],\n ),\n spec_filter_from_buildcache(\n packages_with_externals=packages_with_externals, include=[], exclude=[]\n ),\n ]\n )\n else:\n has_external_source = False\n roots = reuse_yaml.get(\"roots\", True)\n if roots is True:\n self.reuse_strategy = ReuseStrategy.ROOTS\n else:\n self.reuse_strategy = ReuseStrategy.DEPENDENCIES\n default_include = reuse_yaml.get(\"include\", [])\n default_exclude = reuse_yaml.get(\"exclude\", [])\n default_sources = [{\"type\": \"local\"}, {\"type\": \"buildcache\"}]\n for source in reuse_yaml.get(\"from\", default_sources):\n include = source.get(\"include\", default_include)\n exclude = source.get(\"exclude\", default_exclude)\n if source[\"type\"] == \"environment\" and \"path\" in source:\n env_dir = spack.environment.as_env_dir(source[\"path\"])\n active_env = spack.environment.active_environment()\n if not active_env or env_dir not in active_env.included_concrete_env_root_dirs:\n # If the environment is not included as a concrete environment, use the\n # current specs from its lockfile.\n self.reuse_sources.append(\n spec_filter_from_environment(\n packages_with_externals=packages_with_externals,\n include=include,\n exclude=exclude,\n env=spack.environment.environment_from_name_or_dir(env_dir),\n )\n )\n elif source[\"type\"] == \"local\":\n self.reuse_sources.append(\n spec_filter_from_store(\n self.configuration,\n packages_with_externals=packages_with_externals,\n include=include,\n exclude=exclude,\n )\n )\n elif source[\"type\"] == \"buildcache\":\n self.reuse_sources.append(\n spec_filter_from_buildcache(\n packages_with_externals=packages_with_externals,\n include=include,\n exclude=exclude,\n )\n )\n elif source[\"type\"] == \"external\":\n has_external_source = True\n if include:\n # Since libcs are implicit externals, we need to implicitly include them\n include = include + sorted(all_libcs()) # type: ignore[type-var]\n self.reuse_sources.append(\n spec_filter_from_packages_yaml(\n external_parser=external_parser,\n packages_with_externals=packages_with_externals,\n include=include,\n exclude=exclude,\n )\n )\n\n # If \"external\" is not specified, we assume that all externals have to be included\n if not has_external_source:\n self.reuse_sources.append(\n spec_filter_from_packages_yaml(\n external_parser=external_parser,\n packages_with_externals=packages_with_externals,\n include=[],\n exclude=[],\n )\n )\n\n def reusable_specs(self, specs: List[spack.spec.Spec]) -> List[spack.spec.Spec]:\n result = []\n for reuse_source in self.reuse_sources:\n result.extend(reuse_source.selected_specs())\n # If we only want to reuse dependencies, remove the root specs\n if self.reuse_strategy == ReuseStrategy.DEPENDENCIES:\n result = [spec for spec in result if not any(root in spec for root in specs)]\n\n return result\n" }, { "path": "lib/spack/spack/solver/runtimes.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport itertools\nfrom typing import Any, Dict, Set, Tuple\n\nimport spack.compilers.config\nimport spack.compilers.libraries\nimport spack.config\nimport spack.repo\nimport spack.spec\nimport spack.util.libc\nimport spack.version\n\nfrom .core import SourceContext, fn, using_libc_compatibility\nfrom .versions import Provenance\n\n\nclass RuntimePropertyRecorder:\n \"\"\"An object of this class is injected in callbacks to compilers, to let them declare\n properties of the runtimes they support and of the runtimes they provide, and to add\n runtime dependencies to the nodes using said compiler.\n\n The usage of the object is the following. First, a runtime package name or the wildcard\n \"*\" are passed as an argument to __call__, to set which kind of package we are referring to.\n Then we can call one method with a directive-like API.\n\n Examples:\n >>> pkg = RuntimePropertyRecorder(setup)\n >>> # Every package compiled with %gcc has a link dependency on 'gcc-runtime'\n >>> pkg(\"*\").depends_on(\n ... \"gcc-runtime\",\n ... when=\"%gcc\",\n ... type=\"link\",\n ... description=\"If any package uses %gcc, it depends on gcc-runtime\"\n ... )\n >>> # The version of gcc-runtime is the same as the %gcc used to \"compile\" it\n >>> pkg(\"gcc-runtime\").requires(\"@=9.4.0\", when=\"%gcc@=9.4.0\")\n \"\"\"\n\n def __init__(self, setup):\n self._setup = setup\n self.rules = []\n self.runtime_conditions = set()\n self.injected_dependencies = set()\n # State of this object set in the __call__ method, and reset after\n # each directive-like method\n self.current_package = None\n\n def __call__(self, package_name: str) -> \"RuntimePropertyRecorder\":\n \"\"\"Sets a package name for the next directive-like method call\"\"\"\n assert self.current_package is None, f\"state was already set to '{self.current_package}'\"\n self.current_package = package_name\n return self\n\n def reset(self):\n \"\"\"Resets the current state.\"\"\"\n self.current_package = None\n\n def depends_on(self, dependency_str: str, *, when: str, type: str, description: str) -> None:\n \"\"\"Injects conditional dependencies on packages.\n\n Conditional dependencies can be either \"real\" packages or virtual dependencies.\n\n Args:\n dependency_str: the dependency spec to inject\n when: anonymous condition to be met on a package to have the dependency\n type: dependency type\n description: human-readable description of the rule for adding the dependency\n \"\"\"\n # TODO: The API for this function is not final, and is still subject to change. At\n # TODO: the moment, we implemented only the features strictly needed for the\n # TODO: functionality currently provided by Spack, and we assert nothing else is required.\n msg = \"the 'depends_on' method can be called only with pkg('*')\"\n assert self.current_package == \"*\", msg\n\n when_spec = spack.spec.Spec(when)\n assert not when_spec.name, \"only anonymous when specs are accepted\"\n\n dependency_spec = spack.spec.Spec(dependency_str)\n if dependency_spec.versions != spack.version.any_version:\n self._setup.version_constraints[dependency_spec.name].add(dependency_spec.versions)\n\n self.injected_dependencies.add(dependency_spec)\n body_str, node_variable = self.rule_body_from(when_spec)\n\n head_clauses = self._setup.spec_clauses(dependency_spec, body=False)\n runtime_pkg = dependency_spec.name\n is_virtual = head_clauses[0].args[0] == \"virtual_node\"\n main_rule = (\n f\"% {description}\\n\"\n f'1 {{ attr(\"depends_on\", {node_variable}, node(0..X-1, \"{runtime_pkg}\"), \"{type}\") :'\n f' max_dupes(\"{runtime_pkg}\", X)}} 1:-\\n'\n f\"{body_str}.\"\n )\n if is_virtual:\n main_rule = (\n f\"% {description}\\n\"\n f'attr(\"dependency_holds\", {node_variable}, \"{runtime_pkg}\", \"{type}\") :-\\n'\n f\"{body_str}.\"\n )\n\n self.rules.append(main_rule)\n for clause in head_clauses:\n if clause.args[0] == \"node\":\n continue\n runtime_node = f'node(RuntimeID, \"{runtime_pkg}\")'\n head_str = str(clause).replace(f'\"{runtime_pkg}\"', runtime_node)\n depends_on_constraint = (\n f' attr(\"depends_on\", {node_variable}, {runtime_node}, \"{type}\"),\\n'\n )\n if is_virtual:\n depends_on_constraint = (\n f' attr(\"depends_on\", {node_variable}, ProviderNode, \"{type}\"),\\n'\n f\" provider(ProviderNode, {runtime_node}),\\n\"\n )\n\n rule = f\"{head_str} :-\\n{depends_on_constraint}{body_str}.\"\n self.rules.append(rule)\n\n self.reset()\n\n @staticmethod\n def node_for(name: str) -> str:\n return f'node(ID{name.replace(\"-\", \"_\")}, \"{name}\")'\n\n def rule_body_from(self, when_spec: \"spack.spec.Spec\") -> Tuple[str, str]:\n \"\"\"Computes the rule body from a \"when\" spec, and returns it, along with the\n node variable.\n \"\"\"\n\n node_placeholder = \"XXX\"\n node_variable = \"node(ID, Package)\"\n when_substitutions = {}\n for s in when_spec.traverse(root=False):\n when_substitutions[f'\"{s.name}\"'] = self.node_for(s.name)\n body_clauses = self._setup.spec_clauses(when_spec, name=node_placeholder, body=True)\n for clause in body_clauses:\n if clause.args[0] == \"virtual_on_incoming_edges\":\n # Substitute: attr(\"virtual_on_incoming_edges\", ProviderNode, Virtual)\n # with: attr(\"virtual_on_edge\", ParentNode, ProviderNode, Virtual)\n # (avoid adding virtuals everywhere, if a single edge needs it)\n _, provider, virtual = clause.args\n clause.args = \"virtual_on_edge\", node_placeholder, provider, virtual\n\n # Check for abstract hashes in the body\n for s in when_spec.traverse(root=False):\n if s.abstract_hash:\n body_clauses.append(fn.attr(\"hash\", s.name, s.abstract_hash))\n\n body_str = \",\\n\".join(f\" {x}\" for x in body_clauses)\n body_str = body_str.replace(f'\"{node_placeholder}\"', f\"{node_variable}\")\n for old, replacement in when_substitutions.items():\n body_str = body_str.replace(old, replacement)\n return body_str, node_variable\n\n def requires(self, impose: str, *, when: str):\n \"\"\"Injects conditional requirements on a given package.\n\n Args:\n impose: constraint to be imposed\n when: condition triggering the constraint\n \"\"\"\n msg = \"the 'requires' method cannot be called with pkg('*') or without setting the package\"\n assert self.current_package is not None and self.current_package != \"*\", msg\n\n imposed_spec = spack.spec.Spec(f\"{self.current_package}{impose}\")\n when_spec = spack.spec.Spec(f\"{self.current_package}{when}\")\n\n assert imposed_spec.versions.concrete, f\"{impose} must have a concrete version\"\n\n # Add versions to possible versions\n for s in (imposed_spec, when_spec):\n if not s.versions.concrete:\n continue\n self._setup.possible_versions[s.name][s.version].append(Provenance.RUNTIME)\n\n self.runtime_conditions.add((imposed_spec, when_spec))\n self.reset()\n\n def propagate(self, constraint_str: str, *, when: str):\n msg = \"the 'propagate' method can be called only with pkg('*')\"\n assert self.current_package == \"*\", msg\n\n when_spec = spack.spec.Spec(when)\n assert not when_spec.name, \"only anonymous when specs are accepted\"\n\n when_substitutions = {}\n for s in when_spec.traverse(root=False):\n when_substitutions[f'\"{s.name}\"'] = self.node_for(s.name)\n\n body_str, node_variable = self.rule_body_from(when_spec)\n constraint_spec = spack.spec.Spec(constraint_str)\n\n constraint_clauses = self._setup.spec_clauses(constraint_spec, body=False)\n for clause in constraint_clauses:\n if clause.args[0] == \"node_version_satisfies\":\n self._setup.version_constraints[constraint_spec.name].add(constraint_spec.versions)\n args = f'\"{constraint_spec.name}\", \"{constraint_spec.versions}\"'\n head_str = f\"propagate({node_variable}, node_version_satisfies({args}))\"\n rule = f\"{head_str} :-\\n{body_str}.\"\n self.rules.append(rule)\n\n self.reset()\n\n def default_flags(self, spec: \"spack.spec.Spec\"):\n if not spec.external or \"flags\" not in spec.extra_attributes:\n self.reset()\n return\n\n when_spec = spack.spec.Spec(f\"%[deptypes=build] {spec}\")\n body_str, node_variable = self.rule_body_from(when_spec)\n\n node_placeholder = \"XXX\"\n flags = spec.extra_attributes[\"flags\"]\n root_spec_str = f\"{node_placeholder}\"\n for flag_type, default_values in flags.items():\n root_spec_str = f\"{root_spec_str} {flag_type}='{default_values}'\"\n root_spec = spack.spec.Spec(root_spec_str)\n head_clauses = self._setup.spec_clauses(\n root_spec, body=False, context=SourceContext(source=\"compiler\")\n )\n self.rules.append(f\"% Default compiler flags for {spec}\\n\")\n for clause in head_clauses:\n if clause.args[0] == \"node\":\n continue\n head_str = str(clause).replace(f'\"{node_placeholder}\"', f\"{node_variable}\")\n rule = f\"{head_str} :-\\n{body_str}.\"\n self.rules.append(rule)\n\n self.reset()\n\n def consume_facts(self):\n \"\"\"Consume the facts collected by this object, and emits rules and\n facts for the runtimes.\n \"\"\"\n self._setup.gen.h2(\"Runtimes: declarations\")\n runtime_pkgs = sorted(\n {x.name for x in self.injected_dependencies if not spack.repo.PATH.is_virtual(x.name)}\n )\n for runtime_pkg in runtime_pkgs:\n self._setup.gen.fact(fn.runtime(runtime_pkg))\n self._setup.gen.newline()\n\n self._setup.gen.h2(\"Runtimes: rules\")\n self._setup.gen.newline()\n for rule in self.rules:\n self._setup.gen.append(rule)\n self._setup.gen.newline()\n\n self._setup.gen.h2(\"Runtimes: requirements\")\n for imposed_spec, when_spec in sorted(self.runtime_conditions):\n msg = f\"{when_spec} requires {imposed_spec} at runtime\"\n _ = self._setup.condition(when_spec, imposed_spec=imposed_spec, msg=msg)\n\n self._setup.trigger_rules()\n self._setup.effect_rules()\n\n\ndef _normalize_packages_yaml(packages_yaml: Dict[str, Any]) -> None:\n for pkg_name in list(packages_yaml.keys()):\n is_virtual = spack.repo.PATH.is_virtual(pkg_name)\n if pkg_name == \"all\" or not is_virtual:\n continue\n\n # Remove the virtual entry from the normalized configuration\n data = packages_yaml.pop(pkg_name)\n is_buildable = data.get(\"buildable\", True)\n if not is_buildable:\n for provider in spack.repo.PATH.providers_for(pkg_name):\n entry = packages_yaml.setdefault(provider.name, {})\n entry[\"buildable\"] = False\n\n externals = data.get(\"externals\", [])\n\n def keyfn(x):\n return spack.spec.Spec(x[\"spec\"]).name\n\n for provider, specs in itertools.groupby(externals, key=keyfn):\n entry = packages_yaml.setdefault(provider, {})\n entry.setdefault(\"externals\", []).extend(specs)\n\n\ndef external_config_with_implicit_externals(\n configuration: spack.config.Configuration,\n) -> Dict[str, Any]:\n # Read packages.yaml and normalize it so that it will not contain entries referring to\n # virtual packages.\n packages_yaml = configuration.deepcopy_as_builtin(\"packages\", line_info=True)\n _normalize_packages_yaml(packages_yaml)\n\n # Add externals for libc from compilers on Linux\n if not using_libc_compatibility():\n return packages_yaml\n\n seen = set()\n for compiler in spack.compilers.config.all_compilers_from(configuration):\n libc = spack.compilers.libraries.CompilerPropertyDetector(compiler).default_libc()\n if libc and libc not in seen:\n seen.add(libc)\n entry = {\"spec\": f\"{libc}\", \"prefix\": libc.external_path}\n packages_yaml.setdefault(libc.name, {}).setdefault(\"externals\", []).append(entry)\n return packages_yaml\n\n\ndef all_libcs() -> Set[spack.spec.Spec]:\n \"\"\"Return a set of all libc specs targeted by any configured compiler. If none, fall back to\n libc determined from the current Python process if dynamically linked.\"\"\"\n libcs = set()\n for c in spack.compilers.config.all_compilers_from(spack.config.CONFIG):\n candidate = spack.compilers.libraries.CompilerPropertyDetector(c).default_libc()\n if candidate is not None:\n libcs.add(candidate)\n\n if libcs:\n return libcs\n\n libc = spack.util.libc.libc_from_current_python_process()\n return {libc} if libc else set()\n" }, { "path": "lib/spack/spack/solver/splices.lp", "content": "% Copyright Spack Project Developers. See COPYRIGHT file for details.\n%\n% SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n%=============================================================================\n% These rules are conditionally loaded to handle the synthesis of spliced\n% packages.\n% =============================================================================\n% Consider the concrete spec:\n% foo@2.72%gcc@11.4 arch=linux-ubuntu22.04-icelake build_system=autotools ^bar ...\n% It will emit the following facts for reuse (below is a subset)\n% installed_hash(\"foo\", \"xxxyyy\")\n% hash_attr(\"xxxyyy\", \"hash\", \"foo\", \"xxxyyy\")\n% hash_attr(\"xxxyyy\", \"version\", \"foo\", \"2.72\")\n% hash_attr(\"xxxyyy\", \"node_os\", \"ubuntu22.04\")\n% hash_attr(\"xxxyyy\", \"hash\", \"bar\", \"zzzqqq\")\n% hash_attr(\"xxxyyy\", \"depends_on\", \"foo\", \"bar\", \"link\")\n% Rules that derive abi_splice_conditions_hold will be generated from \n% use of the `can_splice` directive. The will have the following form:\n% can_splice(\"foo@1.0.0+a\", when=\"@1.0.1+a\", match_variants=[\"b\"]) --->\n% abi_splice_conditions_hold(0, node(SID, \"foo\"), \"foo\", BashHash) :-\n% installed_hash(\"foo\", BaseHash),\n% attr(\"node\", node(SID, SpliceName)),\n% attr(\"node_version_satisfies\", node(SID, \"foo\"), \"1.0.1\"),\n% hash_attr(\"hash\", \"node_version_satisfies\", \"foo\", \"1.0.1\"),\n% attr(\"variant_value\", node(SID, \"foo\"), \"a\", \"True\"),\n% hash_attr(\"hash\", \"variant_value\", \"foo\", \"a\", \"True\"),\n% attr(\"variant_value\", node(SID, \"foo\"), \"b\", VariVar0),\n% hash_attr(\"hash\", \"variant_value\", \"foo\", \"b\", VariVar0),\n\n% If the splice is valid (i.e. abi_splice_conditions_hold is derived) in the\n% dependency of a concrete spec the solver free to choose whether to continue\n% with the exact hash semantics by simply imposing the child hash, or introducing\n% a spliced node as the dependency instead \n{ imposed_constraint(ParentHash, \"hash\", ChildName, ChildHash) } :-\n hash_attr(ParentHash, \"hash\", ChildName, ChildHash),\n abi_splice_conditions_hold(_, node(SID, SpliceName), ChildName, ChildHash).\n\nattr(\"splice_at_hash\", ParentNode, node(SID, SpliceName), ChildName, ChildHash) :-\n attr(\"hash\", ParentNode, ParentHash),\n hash_attr(ParentHash, \"hash\", ChildName, ChildHash),\n abi_splice_conditions_hold(_, node(SID, SpliceName), ChildName, ChildHash),\n ParentHash != ChildHash,\n not imposed_constraint(ParentHash, \"hash\", ChildName, ChildHash).\n \n% Names and virtual providers may change when a dependency is spliced in\nimposed_constraint(Hash, \"dependency_holds\", ParentName, SpliceName, Type) :-\n hash_attr(Hash, \"depends_on\", ParentName, DepName, Type),\n hash_attr(Hash, \"hash\", DepName, DepHash),\n attr(\"splice_at_hash\", node(ID, ParentName), node(SID, SpliceName), DepName, DepHash).\n \nimposed_constraint(Hash, \"virtual_on_edge\", ParentName, SpliceName, VirtName) :-\n hash_attr(Hash, \"virtual_on_edge\", ParentName, DepName, VirtName),\n attr(\"splice_at_hash\", node(ID, ParentName), node(SID, SpliceName), DepName, DepHash).\n\n" }, { "path": "lib/spack/spack/solver/splicing.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nfrom functools import cmp_to_key\nfrom typing import Dict, List, NamedTuple\n\nimport spack.deptypes as dt\nfrom spack.spec import Spec\nfrom spack.traverse import by_dag_hash, traverse_nodes\n\n\nclass Splice(NamedTuple):\n #: The spec being spliced into a parent\n splice_spec: Spec\n #: The name of the child that splice spec is replacing\n child_name: str\n #: The hash of the child that ``splice_spec`` is replacing\n child_hash: str\n\n\ndef _resolve_collected_splices(\n specs: List[Spec], splices: Dict[Spec, List[Splice]]\n) -> Dict[Spec, Spec]:\n \"\"\"After all of the specs have been concretized, apply all immediate splices.\n Returns a dict mapping original specs to their resolved counterparts\n \"\"\"\n\n def splice_cmp(s1: Spec, s2: Spec):\n \"\"\"This function can be used to sort a list of specs such that that any\n spec which will be spliced into a parent comes after the parent it will\n be spliced into. This order ensures that transitive splices will be\n executed in the correct order.\n \"\"\"\n\n s1_splices = splices.get(s1, [])\n s2_splices = splices.get(s2, [])\n if any([s2.dag_hash() == splice.splice_spec.dag_hash() for splice in s1_splices]):\n return -1\n elif any([s1.dag_hash() == splice.splice_spec.dag_hash() for splice in s2_splices]):\n return 1\n else:\n return 0\n\n splice_order = sorted(specs, key=cmp_to_key(splice_cmp))\n reverse_topo_order = reversed(\n [x for x in traverse_nodes(splice_order, order=\"topo\", key=by_dag_hash) if x in specs]\n )\n\n already_resolved: Dict[Spec, Spec] = {}\n for spec in reverse_topo_order:\n immediate = splices.get(spec, [])\n if not immediate and not any(\n edge.spec in already_resolved for edge in spec.edges_to_dependencies()\n ):\n continue\n new_spec = spec.copy(deps=False)\n new_spec.clear_caches(ignore=(\"package_hash\",))\n new_spec.build_spec = spec\n for edge in spec.edges_to_dependencies():\n depflag = edge.depflag & ~dt.BUILD\n if any(edge.spec.dag_hash() == splice.child_hash for splice in immediate):\n splice = [s for s in immediate if s.child_hash == edge.spec.dag_hash()][0]\n # If the spec being splice in is also spliced\n splice_spec = already_resolved.get(splice.splice_spec, splice.splice_spec)\n new_spec.add_dependency_edge(splice_spec, depflag=depflag, virtuals=edge.virtuals)\n elif edge.spec in already_resolved:\n new_spec.add_dependency_edge(\n already_resolved[edge.spec], depflag=depflag, virtuals=edge.virtuals\n )\n else:\n new_spec.add_dependency_edge(edge.spec, depflag=depflag, virtuals=edge.virtuals)\n already_resolved[spec] = new_spec\n return already_resolved\n" }, { "path": "lib/spack/spack/solver/versions.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport enum\n\n\nclass Provenance(enum.IntEnum):\n \"\"\"Enumeration of the possible provenances of a version.\"\"\"\n\n # A spec literal\n SPEC = enum.auto()\n # A dev spec literal\n DEV_SPEC = enum.auto()\n # The 'packages' section of the configuration\n PACKAGES_YAML = enum.auto()\n # A git version in the 'packages' section of the configuration\n PACKAGES_YAML_GIT_VERSION = enum.auto()\n # A package requirement\n PACKAGE_REQUIREMENT = enum.auto()\n # A 'package.py' file\n PACKAGE_PY = enum.auto()\n # An installed spec\n INSTALLED = enum.auto()\n # lower provenance for installed git refs so concretizer prefers StandardVersion installs\n INSTALLED_GIT_VERSION = enum.auto()\n # Synthetic versions for virtual packages\n VIRTUAL_CONSTRAINT = enum.auto()\n # A runtime injected from another package (e.g. a compiler)\n RUNTIME = enum.auto()\n\n def __str__(self):\n return f\"{self._name_.lower()}\"\n" }, { "path": "lib/spack/spack/solver/when_possible.lp", "content": "% Copyright Spack Project Developers. See COPYRIGHT file for details.\n%\n% SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n%=============================================================================\n% Minimize the number of literals that are not solved\n%\n% This minimization is used for the \"when_possible\" concretization mode,\n% otherwise we assume that all literals must be solved.\n%=============================================================================\n\n% Give clingo the choice to solve an input spec or not\n{ solve_literal(ID) } :- literal(ID).\nliteral_not_solved(ID) :- not solve_literal(ID), literal(ID).\n\n% Make a problem with \"zero literals solved\" unsat. This is to trigger\n% looking for solutions to the ASP problem with \"errors\", which results\n% in better reporting for users. See #30669 for details.\n1 { solve_literal(ID) : literal(ID) }.\n\n\n% If a literal is not solved, and has subconditions, then the subconditions\n% should not be imposed even if their trigger condition holds\ndo_not_impose(EffectID, node(X, Package)) :-\n literal_not_solved(LiteralID),\n pkg_fact(Package, condition_trigger(ParentConditionID, LiteralID)),\n subcondition(SubconditionID, ParentConditionID),\n pkg_fact(Package, condition_effect(SubconditionID, EffectID)),\n trigger_and_effect(_, TriggerID, EffectID),\n trigger_requestor_node(TriggerID, node(X, Package)).\n\nopt_criterion(320, \"number of input specs not concretized\").\n#minimize{ 0@320: #true }.\n#minimize { 1@320,ID : literal_not_solved(ID) }.\n\n#heuristic solve_literal(ID) : literal(ID). [1, sign]\n#heuristic solve_literal(ID) : literal(ID). [1000, init]\n" }, { "path": "lib/spack/spack/spec.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"\nSpack allows very fine-grained control over how packages are installed and\nover how they are built and configured. To make this easy, it has its own\nsyntax for declaring a dependence. We call a descriptor of a particular\npackage configuration a \"spec\".\n\nThe syntax looks like this:\n\n.. code-block:: sh\n\n $ spack install mpileaks ^openmpi @1.2:1.4 +debug %intel @12.1 target=zen\n 0 1 2 3 4 5 6\n\nThe first part of this is the command, ``spack install``. The rest of the\nline is a spec for a particular installation of the mpileaks package.\n\n0. The package to install\n\n1. A dependency of the package, prefixed by ``^``\n\n2. A version descriptor for the package. This can either be a specific\n version, like ``1.2``, or it can be a range of versions, e.g. ``1.2:1.4``.\n If multiple specific versions or multiple ranges are acceptable, they\n can be separated by commas, e.g. if a package will only build with\n versions 1.0, 1.2-1.4, and 1.6-1.8 of mvapich, you could say::\n\n depends_on(\"mvapich@1.0,1.2:1.4,1.6:1.8\")\n\n3. A compile-time variant of the package. If you need openmpi to be\n built in debug mode for your package to work, you can require it by\n adding ``+debug`` to the openmpi spec when you depend on it. If you do\n NOT want the debug option to be enabled, then replace this with ``-debug``.\n If you would like for the variant to be propagated through all your\n package's dependencies use ``++`` for enabling and ``--`` or ``~~`` for disabling.\n\n4. The name of the compiler to build with.\n\n5. The versions of the compiler to build with. Note that the identifier\n for a compiler version is the same ``@`` that is used for a package version.\n A version list denoted by ``@`` is associated with the compiler only if\n if it comes immediately after the compiler name. Otherwise it will be\n associated with the current package spec.\n\n6. The architecture to build with.\n\"\"\"\n\nimport collections\nimport collections.abc\nimport enum\nimport io\nimport itertools\nimport json\nimport os\nimport pathlib\nimport platform\nimport re\nimport socket\nimport warnings\nfrom typing import (\n Any,\n Callable,\n Dict,\n Iterable,\n List,\n Match,\n Optional,\n Sequence,\n Set,\n Tuple,\n Union,\n overload,\n)\n\nimport spack.vendor.archspec.cpu\nfrom spack.vendor.typing_extensions import Literal\n\nimport spack\nimport spack.aliases\nimport spack.compilers.flags\nimport spack.deptypes as dt\nimport spack.error\nimport spack.hash_types as ht\nimport spack.llnl.path\nimport spack.llnl.string\nimport spack.llnl.util.filesystem as fs\nimport spack.llnl.util.lang as lang\nimport spack.llnl.util.tty as tty\nimport spack.llnl.util.tty.color as clr\nimport spack.patch\nimport spack.paths\nimport spack.platforms\nimport spack.provider_index\nimport spack.repo\nimport spack.spec_parser\nimport spack.traverse\nimport spack.util.hash\nimport spack.util.prefix\nimport spack.util.spack_json as sjson\nimport spack.util.spack_yaml as syaml\nimport spack.variant as vt\nimport spack.version\nimport spack.version as vn\nimport spack.version.git_ref_lookup\n\nfrom .enums import InstallRecordStatus, PropagationPolicy\n\n__all__ = [\n \"CompilerSpec\",\n \"Spec\",\n \"UnsupportedPropagationError\",\n \"DuplicateDependencyError\",\n \"UnsupportedCompilerError\",\n \"DuplicateArchitectureError\",\n \"InvalidDependencyError\",\n \"UnsatisfiableSpecNameError\",\n \"UnsatisfiableVersionSpecError\",\n \"UnsatisfiableArchitectureSpecError\",\n \"UnsatisfiableDependencySpecError\",\n \"AmbiguousHashError\",\n \"InvalidHashError\",\n \"SpecDeprecatedError\",\n]\n\nSPEC_FORMAT_RE = re.compile(\n r\"(?:\" # this is one big or, with matches ordered by priority\n # OPTION 1: escaped character (needs to be first to catch opening \\{)\n # Note that an unterminated \\ at the end of a string is left untouched\n r\"(?:\\\\(.))\"\n r\"|\" # or\n # OPTION 2: an actual format string\n r\"{\" # non-escaped open brace {\n r\"( ?[%@/]|[\\w ][\\w -]*=)?\" # optional sigil (or identifier or space) to print sigil in color\n r\"(?:\\^([^}\\.]+)\\.)?\" # optional ^depname. (to get attr from dependency)\n # after the sigil or depname, we can have a hash expression or another attribute\n r\"(?:\" # one of\n r\"(hash\\b)(?:\\:(\\d+))?\" # hash followed by :\n r\"|\" # or\n r\"([^}]*)\" # another attribute to format\n r\")\" # end one of\n r\"(})?\" # finish format string with non-escaped close brace }, or missing if not present\n r\"|\"\n # OPTION 3: mismatched close brace (option 2 would consume a matched open brace)\n r\"(})\" # brace\n r\")\",\n re.IGNORECASE,\n)\n\n#: Valid pattern for an identifier in Spack\n\nIDENTIFIER_RE = r\"\\w[\\w-]*\"\n\n# Coloring of specs when using color output. Fields are printed with\n# different colors to enhance readability.\n# See spack.llnl.util.tty.color for descriptions of the color codes.\nCOMPILER_COLOR = \"@g\" #: color for highlighting compilers\nVERSION_COLOR = \"@c\" #: color for highlighting versions\nARCHITECTURE_COLOR = \"@m\" #: color for highlighting architectures\nVARIANT_COLOR = \"@B\" #: color for highlighting variants\nHASH_COLOR = \"@K\" #: color for highlighting package hashes\nHIGHLIGHT_COLOR = \"@_R\" #: color for highlighting spec parts on demand\n\n#: Default format for Spec.format(). This format can be round-tripped, so that:\n#: Spec(Spec(\"string\").format()) == Spec(\"string)\"\nDEFAULT_FORMAT = (\n \"{name}{@versions}{compiler_flags}\"\n \"{variants}{ namespace=namespace_if_anonymous}\"\n \"{ platform=architecture.platform}{ os=architecture.os}{ target=architecture.target}\"\n \"{/abstract_hash}\"\n)\n\n#: Display format, which eliminates extra `@=` in the output, for readability.\nDISPLAY_FORMAT = (\n \"{name}{@version}{compiler_flags}\"\n \"{variants}{ namespace=namespace_if_anonymous}\"\n \"{ platform=architecture.platform}{ os=architecture.os}{ target=architecture.target}\"\n \"{/abstract_hash}\"\n \"{compilers}\"\n)\n\n#: Regular expression to pull spec contents out of clearsigned signature\n#: file.\nCLEARSIGN_FILE_REGEX = re.compile(\n (\n r\"^-----BEGIN PGP SIGNED MESSAGE-----\"\n r\"\\s+Hash:\\s+[^\\s]+\\s+(.+)-----BEGIN PGP SIGNATURE-----\"\n ),\n re.MULTILINE | re.DOTALL,\n)\n\n#: specfile format version. Must increase monotonically\nSPECFILE_FORMAT_VERSION = 5\n\n\nclass InstallStatus(enum.Enum):\n \"\"\"Maps install statuses to symbols for display.\n\n Options are artificially disjoint for display purposes\n \"\"\"\n\n installed = \"@g{[+]} \"\n upstream = \"@g{[^]} \"\n external = \"@M{[e]} \"\n absent = \"@K{ - } \"\n missing = \"@r{[-]} \"\n\n\n# regexes used in spec formatting\nOLD_STYLE_FMT_RE = re.compile(r\"\\${[A-Z]+}\")\n\n\ndef ensure_modern_format_string(fmt: str) -> None:\n \"\"\"Ensure that the format string does not contain old ${...} syntax.\"\"\"\n result = OLD_STYLE_FMT_RE.search(fmt)\n if result:\n raise SpecFormatStringError(\n f\"Format string `{fmt}` contains old syntax `{result.group(0)}`. \"\n \"This is no longer supported.\"\n )\n\n\ndef _make_microarchitecture(name: str) -> spack.vendor.archspec.cpu.Microarchitecture:\n if isinstance(name, spack.vendor.archspec.cpu.Microarchitecture):\n return name\n return spack.vendor.archspec.cpu.TARGETS.get(\n name, spack.vendor.archspec.cpu.generic_microarchitecture(name)\n )\n\n\n@lang.lazy_lexicographic_ordering\nclass ArchSpec:\n \"\"\"Aggregate the target platform, the operating system and the target microarchitecture.\"\"\"\n\n ANY_TARGET = _make_microarchitecture(\"*\")\n\n @staticmethod\n def default_arch():\n \"\"\"Return the default architecture\"\"\"\n platform = spack.platforms.host()\n default_os = platform.default_operating_system()\n default_target = platform.default_target()\n arch_tuple = str(platform), str(default_os), str(default_target)\n return ArchSpec(arch_tuple)\n\n __slots__ = \"_platform\", \"_os\", \"_target\"\n\n def __init__(self, spec_or_platform_tuple=(None, None, None)):\n \"\"\"Architecture specification a package should be built with.\n\n Each ArchSpec is comprised of three elements: a platform (e.g. Linux),\n an OS (e.g. RHEL6), and a target (e.g. x86_64).\n\n Args:\n spec_or_platform_tuple (ArchSpec or str or tuple): if an ArchSpec\n is passed it will be duplicated into the new instance.\n Otherwise information on platform, OS and target should be\n passed in either as a spec string or as a tuple.\n \"\"\"\n\n # If the argument to __init__ is a spec string, parse it\n # and construct an ArchSpec\n def _string_or_none(s):\n if s and s != \"None\":\n return str(s)\n return None\n\n # If another instance of ArchSpec was passed, duplicate it\n if isinstance(spec_or_platform_tuple, ArchSpec):\n other = spec_or_platform_tuple\n platform_tuple = other.platform, other.os, other.target\n\n elif isinstance(spec_or_platform_tuple, (str, tuple)):\n spec_fields = spec_or_platform_tuple\n\n # Normalize the string to a tuple\n if isinstance(spec_or_platform_tuple, str):\n spec_fields = spec_or_platform_tuple.split(\"-\")\n if len(spec_fields) != 3:\n msg = \"cannot construct an ArchSpec from {0!s}\"\n raise ValueError(msg.format(spec_or_platform_tuple))\n\n platform, operating_system, target = spec_fields\n platform_tuple = (_string_or_none(platform), _string_or_none(operating_system), target)\n\n self.platform, self.os, self.target = platform_tuple\n\n @staticmethod\n def override(init_spec, change_spec):\n if init_spec:\n new_spec = init_spec.copy()\n else:\n new_spec = ArchSpec()\n if change_spec.platform:\n new_spec.platform = change_spec.platform\n # TODO: if the platform is changed to something that is incompatible\n # with the current os, we should implicitly remove it\n if change_spec.os:\n new_spec.os = change_spec.os\n if change_spec.target:\n new_spec.target = change_spec.target\n return new_spec\n\n def _autospec(self, spec_like):\n if isinstance(spec_like, ArchSpec):\n return spec_like\n return ArchSpec(spec_like)\n\n def _cmp_iter(self):\n yield self.platform\n yield self.os\n if self.target is None:\n yield self.target\n else:\n yield self.target.name\n\n @property\n def platform(self):\n \"\"\"The platform of the architecture.\"\"\"\n return self._platform\n\n @platform.setter\n def platform(self, value):\n # The platform of the architecture spec will be verified as a\n # supported Spack platform before it's set to ensure all specs\n # refer to valid platforms.\n value = str(value) if value is not None else None\n self._platform = value\n\n @property\n def os(self):\n \"\"\"The OS of this ArchSpec.\"\"\"\n return self._os\n\n @os.setter\n def os(self, value):\n # The OS of the architecture spec will update the platform field\n # if the OS is set to one of the reserved OS types so that the\n # default OS type can be resolved. Since the reserved OS\n # information is only available for the host machine, the platform\n # will assumed to be the host machine's platform.\n value = str(value) if value is not None else None\n\n if value in spack.platforms.Platform.reserved_oss:\n curr_platform = str(spack.platforms.host())\n self.platform = self.platform or curr_platform\n\n if self.platform != curr_platform:\n raise ValueError(\n \"Can't set arch spec OS to reserved value '%s' when the \"\n \"arch platform (%s) isn't the current platform (%s)\"\n % (value, self.platform, curr_platform)\n )\n\n spec_platform = spack.platforms.by_name(self.platform)\n value = str(spec_platform.operating_system(value))\n\n self._os = value\n\n @property\n def target(self):\n \"\"\"The target of the architecture.\"\"\"\n return self._target\n\n @target.setter\n def target(self, value):\n # The target of the architecture spec will update the platform field\n # if the target is set to one of the reserved target types so that\n # the default target type can be resolved. Since the reserved target\n # information is only available for the host machine, the platform\n # will assumed to be the host machine's platform.\n\n def target_or_none(t):\n if isinstance(t, spack.vendor.archspec.cpu.Microarchitecture):\n return t\n if t and t != \"None\":\n return _make_microarchitecture(t)\n return None\n\n value = target_or_none(value)\n\n if str(value) in spack.platforms.Platform.reserved_targets:\n curr_platform = str(spack.platforms.host())\n self.platform = self.platform or curr_platform\n\n if self.platform != curr_platform:\n raise ValueError(\n \"Can't set arch spec target to reserved value '%s' when \"\n \"the arch platform (%s) isn't the current platform (%s)\"\n % (value, self.platform, curr_platform)\n )\n\n spec_platform = spack.platforms.by_name(self.platform)\n value = spec_platform.target(value)\n\n self._target = value\n\n def satisfies(self, other: \"ArchSpec\") -> bool:\n \"\"\"Return True if all concrete specs matching self also match other, otherwise False.\n\n Args:\n other: spec to be satisfied\n \"\"\"\n other = self._autospec(other)\n\n # Check platform and os\n for attribute in (\"platform\", \"os\"):\n other_attribute = getattr(other, attribute)\n self_attribute = getattr(self, attribute)\n\n # platform=* or os=*\n if self_attribute and other_attribute == \"*\":\n return True\n\n if other_attribute and self_attribute != other_attribute:\n return False\n\n return self._target_satisfies(other, strict=True)\n\n def intersects(self, other: \"ArchSpec\") -> bool:\n \"\"\"Return True if there exists at least one concrete spec that matches both\n self and other, otherwise False.\n\n This operation is commutative, and if two specs intersect it means that one\n can constrain the other.\n\n Args:\n other: spec to be checked for compatibility\n \"\"\"\n other = self._autospec(other)\n\n # Check platform and os\n for attribute in (\"platform\", \"os\"):\n other_attribute = getattr(other, attribute)\n self_attribute = getattr(self, attribute)\n if other_attribute and self_attribute and self_attribute != other_attribute:\n return False\n\n return self._target_satisfies(other, strict=False)\n\n def _target_satisfies(self, other: \"ArchSpec\", strict: bool) -> bool:\n if strict is True:\n need_to_check = bool(other.target)\n else:\n need_to_check = bool(other.target and self.target)\n\n if not need_to_check:\n return True\n\n # other_target is there and strict=True\n if self.target is None:\n return False\n\n # self.target is not None, and other is target=*\n if other.target == ArchSpec.ANY_TARGET:\n return True\n\n return bool(self._target_intersection(other))\n\n def _target_constrain(self, other: \"ArchSpec\") -> bool:\n if self.target is None and other.target is None:\n return False\n\n if not other._target_satisfies(self, strict=False):\n raise UnsatisfiableArchitectureSpecError(self, other)\n\n if self.target_concrete:\n return False\n\n elif other.target_concrete:\n self.target = other.target\n return True\n\n # Compute the intersection of every combination of ranges in the lists\n results = self._target_intersection(other)\n attribute_str = \",\".join(results)\n\n intersection_target = _make_microarchitecture(attribute_str)\n if self.target == intersection_target:\n return False\n\n self.target = intersection_target\n return True\n\n def _target_intersection(self, other):\n results = []\n\n if not self.target or not other.target:\n return results\n\n for s_target_range in str(self.target).split(\",\"):\n s_min, s_sep, s_max = s_target_range.partition(\":\")\n for o_target_range in str(other.target).split(\",\"):\n o_min, o_sep, o_max = o_target_range.partition(\":\")\n\n if not s_sep:\n # s_target_range is a concrete target\n # get a microarchitecture reference for at least one side\n # of each comparison so we can use archspec comparators\n s_comp = _make_microarchitecture(s_min)\n if not o_sep:\n if s_min == o_min:\n results.append(s_min)\n elif (not o_min or s_comp >= o_min) and (not o_max or s_comp <= o_max):\n results.append(s_min)\n elif not o_sep:\n # \"cast\" to microarchitecture\n o_comp = _make_microarchitecture(o_min)\n if (not s_min or o_comp >= s_min) and (not s_max or o_comp <= s_max):\n results.append(o_min)\n else:\n # Take the \"min\" of the two max, if there is a partial ordering.\n n_max = \"\"\n if s_max and o_max:\n _s_max = _make_microarchitecture(s_max)\n _o_max = _make_microarchitecture(o_max)\n if _s_max.family != _o_max.family:\n continue\n if _s_max <= _o_max:\n n_max = s_max\n elif _o_max < _s_max:\n n_max = o_max\n else:\n continue\n elif s_max:\n n_max = s_max\n elif o_max:\n n_max = o_max\n\n # Take the \"max\" of the two min.\n n_min = \"\"\n if s_min and o_min:\n _s_min = _make_microarchitecture(s_min)\n _o_min = _make_microarchitecture(o_min)\n if _s_min.family != _o_min.family:\n continue\n if _s_min >= _o_min:\n n_min = s_min\n elif _o_min > _s_min:\n n_min = o_min\n else:\n continue\n elif s_min:\n n_min = s_min\n elif o_min:\n n_min = o_min\n\n if n_min and n_max:\n _n_min = _make_microarchitecture(n_min)\n _n_max = _make_microarchitecture(n_max)\n if _n_min.family != _n_max.family or not _n_min <= _n_max:\n continue\n if n_min == n_max:\n results.append(n_min)\n else:\n results.append(f\"{n_min}:{n_max}\")\n elif n_min:\n results.append(f\"{n_min}:\")\n elif n_max:\n results.append(f\":{n_max}\")\n\n return results\n\n def constrain(self, other: \"ArchSpec\") -> bool:\n \"\"\"Projects all architecture fields that are specified in the given\n spec onto the instance spec if they're missing from the instance\n spec.\n\n This will only work if the two specs are compatible.\n\n Args:\n other (ArchSpec or str): constraints to be added\n\n Returns:\n True if the current instance was constrained, False otherwise.\n \"\"\"\n other = self._autospec(other)\n\n if not other.intersects(self):\n raise UnsatisfiableArchitectureSpecError(other, self)\n\n constrained = False\n for attr in (\"platform\", \"os\"):\n svalue, ovalue = getattr(self, attr), getattr(other, attr)\n if svalue is None and ovalue is not None:\n setattr(self, attr, ovalue)\n constrained = True\n\n constrained |= self._target_constrain(other)\n\n return constrained\n\n def copy(self):\n \"\"\"Copy the current instance and returns the clone.\"\"\"\n return ArchSpec(self)\n\n @property\n def concrete(self):\n \"\"\"True if the spec is concrete, False otherwise\"\"\"\n return self.platform and self.os and self.target and self.target_concrete\n\n @property\n def target_concrete(self):\n \"\"\"True if the target is not a range or list.\"\"\"\n return (\n self.target is not None and \":\" not in str(self.target) and \",\" not in str(self.target)\n )\n\n def to_dict(self):\n # Generic targets represent either an architecture family (like x86_64)\n # or a custom micro-architecture\n if self.target.vendor == \"generic\":\n target_data = str(self.target)\n else:\n # Get rid of compiler flag information before turning the uarch into a dict\n target_data = self.target.to_dict()\n target_data.pop(\"compilers\", None)\n return {\"arch\": {\"platform\": self.platform, \"platform_os\": self.os, \"target\": target_data}}\n\n @staticmethod\n def from_dict(d):\n \"\"\"Import an ArchSpec from raw YAML/JSON data\"\"\"\n arch = d[\"arch\"]\n target_name = arch[\"target\"]\n if not isinstance(target_name, str):\n target_name = target_name[\"name\"]\n target = _make_microarchitecture(target_name)\n return ArchSpec((arch[\"platform\"], arch[\"platform_os\"], target))\n\n def __str__(self):\n return \"%s-%s-%s\" % (self.platform, self.os, self.target)\n\n def __repr__(self):\n fmt = \"ArchSpec(({0.platform!r}, {0.os!r}, {1!r}))\"\n return fmt.format(self, str(self.target))\n\n def __contains__(self, string):\n return string in str(self) or string in self.target\n\n def complete_with_defaults(self) -> None:\n default_architecture = ArchSpec.default_arch()\n if not self.platform:\n self.platform = default_architecture.platform\n\n if not self.os:\n self.os = default_architecture.os\n\n if not self.target:\n self.target = default_architecture.target\n\n\nclass CompilerSpec:\n \"\"\"Adaptor to the old compiler spec interface. Exposes just a few attributes\"\"\"\n\n def __init__(self, spec):\n self.spec = spec\n\n @property\n def name(self):\n return self.spec.name\n\n @property\n def version(self):\n return self.spec.version\n\n @property\n def versions(self):\n return self.spec.versions\n\n @property\n def display_str(self):\n \"\"\"Equivalent to ``{compiler.name}{@compiler.version}`` for Specs, without extra\n ``@=`` for readability.\"\"\"\n if self.versions != vn.any_version:\n return self.spec.format(\"{name}{@version}\")\n return self.spec.format(\"{name}\")\n\n def __lt__(self, other):\n if not isinstance(other, CompilerSpec):\n return self.spec < other\n return self.spec < other.spec\n\n def __eq__(self, other):\n if not isinstance(other, CompilerSpec):\n return self.spec == other\n return self.spec == other.spec\n\n def __hash__(self):\n return hash(self.spec)\n\n def __str__(self):\n return str(self.spec)\n\n def _cmp_iter(self):\n return self.spec._cmp_iter()\n\n def __bool__(self):\n if self.spec == Spec():\n return False\n return bool(self.spec)\n\n\nclass DeprecatedCompilerSpec(lang.DeprecatedProperty):\n def __init__(self):\n super().__init__(name=\"compiler\")\n\n def factory(self, instance, owner):\n if instance.original_spec_format() < 5:\n compiler = instance.annotations.compiler_node_attribute\n assert compiler is not None, \"a compiler spec is expected\"\n return CompilerSpec(compiler)\n\n for language in (\"c\", \"cxx\", \"fortran\"):\n deps = instance.dependencies(virtuals=language)\n if deps:\n return CompilerSpec(deps[0])\n\n raise AttributeError(f\"{instance} has no C, C++, or Fortran compiler\")\n\n\n@lang.lazy_lexicographic_ordering\nclass DependencySpec:\n \"\"\"DependencySpecs represent an edge in the DAG, and contain dependency types\n and information on the virtuals being provided.\n\n Dependencies can be one (or more) of several types:\n\n - build: needs to be in the PATH at build time.\n - link: is linked to and added to compiler flags.\n - run: needs to be in the PATH for the package to run.\n\n Args:\n parent: starting node of the edge\n spec: ending node of the edge.\n depflag: represents dependency relationships.\n virtuals: virtual packages provided from child to parent node.\n \"\"\"\n\n __slots__ = \"parent\", \"spec\", \"depflag\", \"virtuals\", \"direct\", \"when\", \"propagation\"\n\n def __init__(\n self,\n parent: \"Spec\",\n spec: \"Spec\",\n *,\n depflag: dt.DepFlag,\n virtuals: Tuple[str, ...],\n direct: bool = False,\n propagation: PropagationPolicy = PropagationPolicy.NONE,\n when: Optional[\"Spec\"] = None,\n ):\n if direct is False and propagation != PropagationPolicy.NONE:\n raise InvalidEdgeError(\"only direct dependencies can be propagated\")\n\n self.parent = parent\n self.spec = spec\n self.depflag = depflag\n self.virtuals = tuple(sorted(set(virtuals)))\n self.direct = direct\n self.propagation = propagation\n self.when = when or EMPTY_SPEC\n\n def update_deptypes(self, depflag: dt.DepFlag) -> bool:\n \"\"\"Update the current dependency types\"\"\"\n old = self.depflag\n new = depflag | old\n if new == old:\n return False\n self.depflag = new\n return True\n\n def update_virtuals(self, virtuals: Union[str, Iterable[str]]) -> bool:\n \"\"\"Update the list of provided virtuals\"\"\"\n old = self.virtuals\n if isinstance(virtuals, str):\n union = {virtuals, *self.virtuals}\n else:\n union = {*virtuals, *self.virtuals}\n if len(union) == len(old):\n return False\n self.virtuals = tuple(sorted(union))\n return True\n\n def copy(self, *, keep_virtuals: bool = True, keep_parent: bool = True) -> \"DependencySpec\":\n \"\"\"Return a copy of this edge\"\"\"\n parent = self.parent if keep_parent else Spec()\n virtuals = self.virtuals if keep_virtuals else ()\n return DependencySpec(\n parent,\n self.spec,\n depflag=self.depflag,\n virtuals=virtuals,\n propagation=self.propagation,\n direct=self.direct,\n when=self.when,\n )\n\n def _constrain(self, other: \"DependencySpec\") -> bool:\n \"\"\"Constrain this edge with another edge. Precondition: parent and child of self and other\n are compatible, and both edges have the same when condition. Used as an internal helper\n function in Spec.constrain.\n\n Args:\n other: edge to use as constraint\n\n Returns:\n True if the current edge was changed, False otherwise.\n \"\"\"\n changed = False\n changed |= self.spec.constrain(other.spec)\n changed |= self.update_deptypes(other.depflag)\n changed |= self.update_virtuals(other.virtuals)\n if not self.direct and other.direct:\n changed = True\n self.direct = True\n return changed\n\n def _cmp_iter(self):\n yield self.parent.name if self.parent else None\n yield self.spec.name if self.spec else None\n yield self.depflag\n yield self.virtuals\n yield self.direct\n yield self.propagation\n yield self.when\n\n def __str__(self) -> str:\n return self.format()\n\n def __repr__(self) -> str:\n keywords = [f\"depflag={self.depflag}\", f\"virtuals={self.virtuals}\"]\n if self.direct:\n keywords.append(f\"direct={self.direct}\")\n\n if self.when != Spec():\n keywords.append(f\"when={self.when}\")\n\n if self.propagation != PropagationPolicy.NONE:\n keywords.append(f\"propagation={self.propagation}\")\n\n keywords_str = \", \".join(keywords)\n return f\"DependencySpec({self.parent.format()!r}, {self.spec.format()!r}, {keywords_str})\"\n\n def format(self, *, unconditional: bool = False) -> str:\n \"\"\"Returns a string, using the spec syntax, representing this edge\n\n Args:\n unconditional: if True, removes any condition statement from the representation\n \"\"\"\n\n parent_str, child_str = self.parent.format(), self.spec.format()\n virtuals_str = f\"virtuals={','.join(self.virtuals)}\" if self.virtuals else \"\"\n\n when_str = \"\"\n if not unconditional and self.when != Spec():\n when_str = f\"when='{self.when}'\"\n\n dep_sigil = \"%\" if self.direct else \"^\"\n if self.propagation == PropagationPolicy.PREFERENCE:\n dep_sigil = \"%%\"\n\n edge_attrs = [x for x in (virtuals_str, when_str) if x]\n\n if edge_attrs:\n return f\"{parent_str} {dep_sigil}[{' '.join(edge_attrs)}] {child_str}\"\n return f\"{parent_str} {dep_sigil}{child_str}\"\n\n def flip(self) -> \"DependencySpec\":\n \"\"\"Flips the dependency and keeps its type. Drops all other information.\"\"\"\n return DependencySpec(\n parent=self.spec, spec=self.parent, depflag=self.depflag, virtuals=()\n )\n\n\nclass CompilerFlag(str):\n \"\"\"Will store a flag value and it's propagation value\n\n Args:\n value (str): the flag's value\n propagate (bool): if ``True`` the flag value will\n be passed to the package's dependencies. If\n ``False`` it will not\n flag_group (str): if this flag was introduced along\n with several flags via a single source, then\n this will store all such flags\n source (str): identifies the type of constraint that\n introduced this flag (e.g. if a package has\n ``depends_on(... cflags=-g)``, then the ``source``\n for \"-g\" would indicate ``depends_on``.\n \"\"\"\n\n propagate: bool\n flag_group: str\n source: str\n\n def __new__(cls, value, **kwargs):\n obj = str.__new__(cls, value)\n obj.propagate = kwargs.pop(\"propagate\", False)\n obj.flag_group = kwargs.pop(\"flag_group\", value)\n obj.source = kwargs.pop(\"source\", None)\n return obj\n\n\n_valid_compiler_flags = [\"cflags\", \"cxxflags\", \"fflags\", \"ldflags\", \"ldlibs\", \"cppflags\"]\n\n\ndef _shared_subset_pair_iterate(container1, container2):\n \"\"\"\n [0, a, c, d, f]\n [a, d, e, f]\n\n yields [(a, a), (d, d), (f, f)]\n\n no repeated elements\n \"\"\"\n a_idx, b_idx = 0, 0\n max_a, max_b = len(container1), len(container2)\n while a_idx < max_a and b_idx < max_b:\n if container1[a_idx] == container2[b_idx]:\n yield (container1[a_idx], container2[b_idx])\n a_idx += 1\n b_idx += 1\n else:\n while container1[a_idx] < container2[b_idx]:\n a_idx += 1\n while container1[a_idx] > container2[b_idx]:\n b_idx += 1\n\n\nclass FlagMap(lang.HashableMap[str, List[CompilerFlag]]):\n def satisfies(self, other):\n return all(f in self and set(self[f]) >= set(other[f]) for f in other)\n\n def intersects(self, other):\n return True\n\n def constrain(self, other):\n \"\"\"Add all flags in other that aren't in self to self.\n\n Return whether the spec changed.\n \"\"\"\n changed = False\n for flag_type in other:\n if flag_type not in self:\n self[flag_type] = other[flag_type]\n changed = True\n else:\n extra_other = set(other[flag_type]) - set(self[flag_type])\n if extra_other:\n self[flag_type] = list(self[flag_type]) + list(\n x for x in other[flag_type] if x in extra_other\n )\n changed = True\n\n # Next, if any flags in other propagate, we force them to propagate in our case\n shared = list(sorted(set(other[flag_type]) - extra_other))\n for x, y in _shared_subset_pair_iterate(shared, sorted(self[flag_type])):\n if y.propagate is True and x.propagate is False:\n changed = True\n y.propagate = False\n\n # TODO: what happens if flag groups with a partial (but not complete)\n # intersection specify different behaviors for flag propagation?\n\n return changed\n\n @staticmethod\n def valid_compiler_flags():\n return _valid_compiler_flags\n\n def copy(self):\n clone = FlagMap()\n for name, compiler_flag in self.items():\n clone[name] = compiler_flag\n return clone\n\n def add_flag(self, flag_type, value, propagation, flag_group=None, source=None):\n \"\"\"Stores the flag's value in CompilerFlag and adds it\n to the FlagMap\n\n Args:\n flag_type (str): the type of flag\n value (str): the flag's value that will be added to the flag_type's\n corresponding list\n propagation (bool): if ``True`` the flag value will be passed to\n the packages' dependencies. If``False`` it will not be passed\n \"\"\"\n flag_group = flag_group or value\n flag = CompilerFlag(value, propagate=propagation, flag_group=flag_group, source=source)\n\n if flag_type not in self:\n self[flag_type] = [flag]\n else:\n self[flag_type].append(flag)\n\n def yaml_entry(self, flag_type):\n \"\"\"Returns the flag type and a list of the flag values since the\n propagation values aren't needed when writing to yaml\n\n Args:\n flag_type (str): the type of flag to get values from\n\n Returns the flag_type and a list of the corresponding flags in\n string format\n \"\"\"\n return flag_type, [str(flag) for flag in self[flag_type]]\n\n def _cmp_iter(self):\n for k, v in sorted(self.dict.items()):\n yield k\n\n def flags():\n for flag in v:\n yield flag\n yield flag.propagate\n\n yield flags\n\n def __str__(self):\n if not self:\n return \"\"\n\n sorted_items = sorted((k, v) for k, v in self.items() if v)\n\n result = \"\"\n for flag_type, flags in sorted_items:\n normal = [f for f in flags if not f.propagate]\n if normal:\n value = spack.spec_parser.quote_if_needed(\" \".join(normal))\n result += f\" {flag_type}={value}\"\n\n propagated = [f for f in flags if f.propagate]\n if propagated:\n value = spack.spec_parser.quote_if_needed(\" \".join(propagated))\n result += f\" {flag_type}=={value}\"\n\n # TODO: somehow add this space only if something follows in Spec.format()\n if sorted_items:\n result += \" \"\n\n return result\n\n\ndef _sort_by_dep_types(dspec: DependencySpec):\n return dspec.depflag\n\n\nEdgeMap = Dict[str, List[DependencySpec]]\n\n\ndef _add_edge_to_map(edge_map: EdgeMap, key: str, edge: DependencySpec) -> None:\n if key in edge_map:\n lst = edge_map[key]\n lst.append(edge)\n lst.sort(key=_sort_by_dep_types)\n else:\n edge_map[key] = [edge]\n\n\ndef _select_edges(\n edge_map: EdgeMap,\n *,\n parent: Optional[str] = None,\n child: Optional[str] = None,\n depflag: dt.DepFlag = dt.ALL,\n virtuals: Optional[Union[str, Sequence[str]]] = None,\n) -> List[DependencySpec]:\n \"\"\"Selects a list of edges and returns them.\n\n If an edge:\n\n - Has *any* of the dependency types passed as argument,\n - Matches the parent and/or child name\n - Provides *any* of the virtuals passed as argument\n\n then it is selected.\n\n Args:\n edge_map: map of edges to select from\n parent: name of the parent package\n child: name of the child package\n depflag: allowed dependency types in flag form\n virtuals: list of virtuals or specific virtual on the edge\n \"\"\"\n if not depflag:\n return []\n\n # Start from all the edges we store\n selected: Iterable[DependencySpec] = itertools.chain.from_iterable(edge_map.values())\n\n # Filter by parent name\n if parent:\n selected = (d for d in selected if d.parent.name == parent)\n\n # Filter by child name\n if child:\n selected = (d for d in selected if d.spec.name == child)\n\n # Filter by allowed dependency types\n if depflag != dt.ALL:\n selected = (dep for dep in selected if not dep.depflag or (depflag & dep.depflag))\n\n # Filter by virtuals\n if virtuals is not None:\n if isinstance(virtuals, str):\n selected = (dep for dep in selected if virtuals in dep.virtuals)\n else:\n selected = (dep for dep in selected if any(v in dep.virtuals for v in virtuals))\n\n return list(selected)\n\n\ndef _headers_default_handler(spec: \"Spec\"):\n \"\"\"Default handler when looking for the 'headers' attribute.\n\n Tries to search for ``*.h`` files recursively starting from\n ``spec.package.home.include``.\n\n Parameters:\n spec: spec that is being queried\n\n Returns:\n HeaderList: The headers in ``prefix.include``\n\n Raises:\n NoHeadersError: If no headers are found\n \"\"\"\n home = getattr(spec.package, \"home\")\n headers = fs.find_headers(\"*\", root=home.include, recursive=True)\n\n if headers:\n return headers\n raise spack.error.NoHeadersError(f\"Unable to locate {spec.name} headers in {home}\")\n\n\ndef _libs_default_handler(spec: \"Spec\"):\n \"\"\"Default handler when looking for the 'libs' attribute.\n\n Tries to search for ``lib{spec.name}`` recursively starting from\n ``spec.package.home``. If ``spec.name`` starts with ``lib``, searches for\n ``{spec.name}`` instead.\n\n Parameters:\n spec: spec that is being queried\n\n Returns:\n LibraryList: The libraries found\n\n Raises:\n NoLibrariesError: If no libraries are found\n \"\"\"\n\n # Variable 'name' is passed to function 'find_libraries', which supports\n # glob characters. For example, we have a package with a name 'abc-abc'.\n # Now, we don't know if the original name of the package is 'abc_abc'\n # (and it generates a library 'libabc_abc.so') or 'abc-abc' (and it\n # generates a library 'libabc-abc.so'). So, we tell the function\n # 'find_libraries' to give us anything that matches 'libabc?abc' and it\n # gives us either 'libabc-abc.so' or 'libabc_abc.so' (or an error)\n # depending on which one exists (there is a possibility, of course, to\n # get something like 'libabcXabc.so, but for now we consider this\n # unlikely).\n name = spec.name.replace(\"-\", \"?\")\n home = getattr(spec.package, \"home\")\n\n # Avoid double 'lib' for packages whose names already start with lib\n if not name.startswith(\"lib\") and not spec.satisfies(\"platform=windows\"):\n name = \"lib\" + name\n\n # If '+shared' search only for shared library; if '~shared' search only for\n # static library; otherwise, first search for shared and then for static.\n search_shared = (\n [True] if (\"+shared\" in spec) else ([False] if (\"~shared\" in spec) else [True, False])\n )\n\n for shared in search_shared:\n # Since we are searching for link libraries, on Windows search only for\n # \".Lib\" extensions by default as those represent import libraries for implicit links.\n libs = fs.find_libraries(name, home, shared=shared, recursive=True, runtime=False)\n if libs:\n return libs\n\n raise spack.error.NoLibrariesError(\n f\"Unable to recursively locate {spec.name} libraries in {home}\"\n )\n\n\nclass ForwardQueryToPackage:\n \"\"\"Descriptor used to forward queries from Spec to Package\"\"\"\n\n def __init__(\n self,\n attribute_name: str,\n default_handler: Optional[Callable[[\"Spec\"], Any]] = None,\n _indirect: bool = False,\n ) -> None:\n \"\"\"Create a new descriptor.\n\n Parameters:\n attribute_name: name of the attribute to be searched for in the Package instance\n default_handler: default function to be called if the attribute was not found in the\n Package instance\n _indirect: temporarily added to redirect a query to another package.\n \"\"\"\n self.attribute_name = attribute_name\n self.default = default_handler\n self.indirect = _indirect\n\n def __get__(self, instance: \"SpecBuildInterface\", cls):\n \"\"\"Retrieves the property from Package using a well defined chain\n of responsibility.\n\n The order of call is:\n\n 1. if the query was through the name of a virtual package try to\n search for the attribute ``{virtual_name}_{attribute_name}``\n in Package\n\n 2. try to search for attribute ``{attribute_name}`` in Package\n\n 3. try to call the default handler\n\n The first call that produces a value will stop the chain.\n\n If no call can handle the request then AttributeError is raised with a\n message indicating that no relevant attribute exists.\n If a call returns None, an AttributeError is raised with a message\n indicating a query failure, e.g. that library files were not found in a\n 'libs' query.\n \"\"\"\n # TODO: this indirection exist solely for `spec[\"python\"].command` to actually return\n # spec[\"python-venv\"].command. It should be removed when `python` is a virtual.\n if self.indirect and instance.indirect_spec:\n pkg = instance.indirect_spec.package\n else:\n pkg = instance.wrapped_obj.package\n try:\n query = instance.last_query\n except AttributeError:\n # There has been no query yet: this means\n # a spec is trying to access its own attributes\n _ = instance.wrapped_obj[instance.wrapped_obj.name] # NOQA: ignore=F841\n query = instance.last_query\n\n callbacks_chain = []\n # First in the chain : specialized attribute for virtual packages\n if query.isvirtual:\n specialized_name = \"{0}_{1}\".format(query.name, self.attribute_name)\n callbacks_chain.append(lambda: getattr(pkg, specialized_name))\n # Try to get the generic method from Package\n callbacks_chain.append(lambda: getattr(pkg, self.attribute_name))\n # Final resort : default callback\n if self.default is not None:\n _default = self.default # make mypy happy\n callbacks_chain.append(lambda: _default(instance.wrapped_obj))\n\n # Trigger the callbacks in order, the first one producing a\n # value wins\n value = None\n message = None\n for f in callbacks_chain:\n try:\n value = f()\n # A callback can return None to trigger an error indicating\n # that the query failed.\n if value is None:\n msg = \"Query of package '{name}' for '{attrib}' failed\\n\"\n msg += \"\\tprefix : {spec.prefix}\\n\"\n msg += \"\\tspec : {spec}\\n\"\n msg += \"\\tqueried as : {query.name}\\n\"\n msg += \"\\textra parameters : {query.extra_parameters}\"\n message = msg.format(\n name=pkg.name,\n attrib=self.attribute_name,\n spec=instance,\n query=instance.last_query,\n )\n else:\n return value\n break\n except AttributeError:\n pass\n # value is 'None'\n if message is not None:\n # Here we can use another type of exception. If we do that, the\n # unit test 'test_getitem_exceptional_paths' in the file\n # lib/spack/spack/test/spec_dag.py will need to be updated to match\n # the type.\n raise AttributeError(message)\n # 'None' value at this point means that there are no appropriate\n # properties defined and no default handler, or that all callbacks\n # raised AttributeError. In this case, we raise AttributeError with an\n # appropriate message.\n fmt = \"'{name}' package has no relevant attribute '{query}'\\n\"\n fmt += \"\\tspec : '{spec}'\\n\"\n fmt += \"\\tqueried as : '{spec.last_query.name}'\\n\"\n fmt += \"\\textra parameters : '{spec.last_query.extra_parameters}'\\n\"\n message = fmt.format(name=pkg.name, query=self.attribute_name, spec=instance)\n raise AttributeError(message)\n\n def __set__(self, instance, value):\n cls_name = type(instance).__name__\n msg = \"'{0}' object attribute '{1}' is read-only\"\n raise AttributeError(msg.format(cls_name, self.attribute_name))\n\n\n# Represents a query state in a BuildInterface object\nQueryState = collections.namedtuple(\"QueryState\", [\"name\", \"extra_parameters\", \"isvirtual\"])\n\n\ndef tree(\n specs: List[\"Spec\"],\n *,\n color: Optional[bool] = None,\n depth: bool = False,\n hashes: bool = False,\n hashlen: Optional[int] = None,\n cover: spack.traverse.CoverType = \"nodes\",\n indent: int = 0,\n format: str = DEFAULT_FORMAT,\n deptypes: Union[dt.DepFlag, dt.DepTypes] = dt.ALL,\n show_types: bool = False,\n depth_first: bool = False,\n recurse_dependencies: bool = True,\n status_fn: Optional[Callable[[\"Spec\"], InstallStatus]] = None,\n prefix: Optional[Callable[[\"Spec\"], str]] = None,\n key: Callable[[\"Spec\"], Any] = id,\n highlight_version_fn: Optional[Callable[[\"Spec\"], bool]] = None,\n highlight_variant_fn: Optional[Callable[[\"Spec\", str], bool]] = None,\n) -> str:\n \"\"\"Prints out specs and their dependencies, tree-formatted with indentation.\n\n Status function may either output a boolean or an InstallStatus\n\n Args:\n color: if True, always colorize the tree. If False, don't colorize the tree. If None,\n use the default from spack.llnl.tty.color\n depth: print the depth from the root\n hashes: if True, print the hash of each node\n hashlen: length of the hash to be printed\n cover: either ``\"nodes\"`` or ``\"edges\"``\n indent: extra indentation for the tree being printed\n format: format to be used to print each node\n deptypes: dependency types to be represented in the tree\n show_types: if True, show the (merged) dependency type of a node\n depth_first: if True, traverse the DAG depth first when representing it as a tree\n recurse_dependencies: if True, recurse on dependencies\n status_fn: optional callable that takes a node as an argument and return its\n installation status\n prefix: optional callable that takes a node as an argument and return its\n installation prefix\n highlight_version_fn: optional callable that returns true on nodes where the version\n needs to be highlighted\n highlight_variant_fn: optional callable that returns true on variants that need\n to be highlighted\n \"\"\"\n out = \"\"\n\n if color is None:\n color = clr.get_color_when()\n\n # reduce deptypes over all in-edges when covering nodes\n if show_types and cover == \"nodes\":\n deptype_lookup: Dict[str, dt.DepFlag] = collections.defaultdict(dt.DepFlag)\n for edge in spack.traverse.traverse_edges(\n specs, cover=\"edges\", deptype=deptypes, root=False\n ):\n deptype_lookup[edge.spec.dag_hash()] |= edge.depflag\n\n # SupportsRichComparisonT issue with List[Spec]\n sorted_specs: List[\"Spec\"] = sorted(specs) # type: ignore[type-var]\n\n for d, dep_spec in spack.traverse.traverse_tree(\n sorted_specs, cover=cover, deptype=deptypes, depth_first=depth_first, key=key\n ):\n node = dep_spec.spec\n\n if prefix is not None:\n out += prefix(node)\n out += \" \" * indent\n\n if depth:\n out += \"%-4d\" % d\n\n if status_fn:\n status = status_fn(node)\n if status in list(InstallStatus):\n out += clr.colorize(status.value, color=color)\n elif status:\n out += clr.colorize(\"@g{[+]} \", color=color)\n else:\n out += clr.colorize(\"@r{[-]} \", color=color)\n\n if hashes:\n out += clr.colorize(\"@K{%s} \", color=color) % node.dag_hash(hashlen)\n\n if show_types:\n if cover == \"nodes\":\n depflag = deptype_lookup[dep_spec.spec.dag_hash()]\n else:\n # when covering edges or paths, we show dependency\n # types only for the edge through which we visited\n depflag = dep_spec.depflag\n\n type_chars = dt.flag_to_chars(depflag)\n out += \"[%s] \" % type_chars\n\n out += \" \" * d\n if d > 0:\n out += \"^\"\n out += (\n node.format(\n format,\n color=color,\n highlight_version_fn=highlight_version_fn,\n highlight_variant_fn=highlight_variant_fn,\n )\n + \"\\n\"\n )\n\n # Check if we wanted just the first line\n if not recurse_dependencies:\n break\n\n return out\n\n\nclass SpecAnnotations:\n def __init__(self) -> None:\n self.original_spec_format = SPECFILE_FORMAT_VERSION\n self.compiler_node_attribute: Optional[\"Spec\"] = None\n\n def with_spec_format(self, spec_format: int) -> \"SpecAnnotations\":\n self.original_spec_format = spec_format\n return self\n\n def with_compiler(self, compiler: \"Spec\") -> \"SpecAnnotations\":\n self.compiler_node_attribute = compiler\n return self\n\n def __repr__(self) -> str:\n result = f\"SpecAnnotations().with_spec_format({self.original_spec_format})\"\n if self.compiler_node_attribute:\n result += f\".with_compiler({str(self.compiler_node_attribute)})\"\n return result\n\n\ndef _anonymous_star(dep: DependencySpec, dep_format: str) -> str:\n \"\"\"Determine if a spec needs a star to disambiguate it from an anonymous spec w/variants.\n\n Returns:\n \"*\" if a star is needed, \"\" otherwise\n \"\"\"\n # named spec never needs star\n if dep.spec.name:\n return \"\"\n\n # virtuals without a name always need *: %c=* @4.0 foo=bar\n if dep.virtuals:\n return \"*\"\n\n # versions are first so checking for @ is faster than != VersionList(':')\n if dep_format.startswith(\"@\"):\n return \"\"\n\n # compiler flags are key-value pairs and can be ambiguous with virtual assignment\n if dep.spec.compiler_flags:\n return \"*\"\n\n # booleans come first, and they don't need a star. key-value pairs do. If there are\n # no key value pairs, we're left with either an empty spec, which needs * as in\n # '^*', or we're left with arch, which is a key value pair, and needs a star.\n if not any(v.type == vt.VariantType.BOOL for v in dep.spec.variants.values()):\n return \"*\"\n\n return \"*\" if dep.spec.architecture else \"\"\n\n\ndef _get_satisfying_edge(\n lhs_node: \"Spec\", rhs_edge: DependencySpec, *, resolve_virtuals: bool\n) -> Optional[DependencySpec]:\n \"\"\"Search for an edge in ``lhs_node`` that satisfies ``rhs_edge``.\"\"\"\n # First check direct deps of all types.\n for lhs_edge in lhs_node.edges_to_dependencies():\n if _satisfies_edge(lhs_edge, rhs_edge, resolve_virtuals):\n return lhs_edge\n\n # Include the historical compiler node if available as an ad-hoc edge.\n compiler_spec = lhs_node.annotations.compiler_node_attribute\n if compiler_spec is not None:\n compiler_edge = DependencySpec(\n lhs_node,\n compiler_spec,\n depflag=dt.BUILD,\n virtuals=(\"c\", \"cxx\", \"fortran\"),\n direct=True,\n )\n if _satisfies_edge(compiler_edge, rhs_edge, resolve_virtuals):\n return compiler_edge\n\n if rhs_edge.direct:\n return None\n\n # BFS through link/run transitive deps (skip depth 1, already checked).\n depflag = dt.LINK | dt.RUN\n queue = collections.deque(lhs_node.edges_to_dependencies(depflag=depflag))\n seen = {id(lhs_edge.spec) for lhs_edge in queue}\n while queue:\n lhs_edge = queue.popleft()\n\n if _satisfies_edge(lhs_edge, rhs_edge, resolve_virtuals):\n return lhs_edge\n\n for lhs_edge in lhs_edge.spec.edges_to_dependencies(depflag=depflag):\n if id(lhs_edge.spec) not in seen:\n seen.add(id(lhs_edge.spec))\n queue.append(lhs_edge)\n\n return None\n\n\ndef _satisfies_edge(lhs: \"DependencySpec\", rhs: \"DependencySpec\", resolve_virtuals: bool) -> bool:\n \"\"\"Helper function for satisfaction tests, which checks edge attributes and the target node.\n It skips verification of the parent node.\"\"\"\n name_mismatch = rhs.spec.name and lhs.spec.name != rhs.spec.name\n if name_mismatch and rhs.spec.name not in lhs.virtuals:\n return False\n\n if not rhs.when._satisfies(lhs.when, resolve_virtuals=resolve_virtuals):\n return False\n\n # Subset semantics for virtuals\n for v in rhs.virtuals:\n if v not in lhs.virtuals:\n return False\n\n # Subset semantics for dependency types\n if (lhs.depflag & rhs.depflag) != rhs.depflag:\n return False\n\n if not name_mismatch:\n return lhs.spec._satisfies_node(rhs.spec, resolve_virtuals=resolve_virtuals)\n\n # Right-hand side is virtual provided by left-hand side. The only node attribute supported is\n # the version of the virtual. Avoid expensive lookups for provider metadata if there's no\n # version constraint to check.\n if rhs.spec.versions == spack.version.any_version:\n return True\n\n if not resolve_virtuals:\n return False\n\n return lhs.spec._provides_virtual(rhs.spec)\n\n\n@lang.lazy_lexicographic_ordering(set_hash=False)\nclass Spec:\n compiler = DeprecatedCompilerSpec()\n\n @staticmethod\n def default_arch():\n \"\"\"Return an anonymous spec for the default architecture\"\"\"\n s = Spec()\n s.architecture = ArchSpec.default_arch()\n return s\n\n def __init__(self, spec_like=None, *, external_path=None, external_modules=None):\n \"\"\"Create a new Spec.\n\n Arguments:\n spec_like: if not provided, we initialize an anonymous Spec that matches any Spec;\n if provided we parse this as a Spec string, or we copy the provided Spec.\n\n Keyword arguments:\n external_path: prefix, if this is a spec for an external package\n external_modules: list of external modules, for an external package using modules\n \"\"\"\n # Copy if spec_like is a Spec.\n if isinstance(spec_like, Spec):\n self._dup(spec_like)\n return\n\n # init an empty spec that matches anything.\n self.name: str = \"\"\n self.versions = vn.VersionList.any()\n self.variants = VariantMap()\n self.architecture = None\n self.compiler_flags = FlagMap()\n self._dependents = {}\n self._dependencies = {}\n self.namespace = None\n self.abstract_hash = None\n\n # initial values for all spec hash types\n for h in ht.HASHES:\n setattr(self, h.attr, None)\n\n # cache for spec's prefix, computed lazily by prefix property\n self._prefix = None\n\n # Python __hash__ is handled separately from the cached spec hashes\n self._dunder_hash = None\n\n # cache of package for this spec\n self._package = None\n\n # whether the spec is concrete or not; set at the end of concretization\n self._concrete = False\n\n # External detection details that can be set by internal Spack calls\n # in the constructor.\n self._external_path = external_path\n self.external_modules = Spec._format_module_list(external_modules)\n\n # This attribute is used to store custom information for external specs.\n self.extra_attributes: Dict[str, Any] = {}\n\n # This attribute holds the original build copy of the spec if it is\n # deployed differently than it was built. None signals that the spec\n # is deployed \"as built.\"\n # Build spec should be the actual build spec unless marked dirty.\n self._build_spec = None\n self.annotations = SpecAnnotations()\n\n if isinstance(spec_like, str):\n spack.spec_parser.parse_one_or_raise(spec_like, self)\n\n elif spec_like is not None:\n raise TypeError(f\"Can't make spec out of {type(spec_like)}\")\n\n @staticmethod\n def _format_module_list(modules):\n \"\"\"Return a module list that is suitable for YAML serialization\n and hash computation.\n\n Given a module list, possibly read from a configuration file,\n return an object that serializes to a consistent YAML string\n before/after round-trip serialization to/from a Spec dictionary\n (stored in JSON format): when read in, the module list may\n contain YAML formatting that is discarded (non-essential)\n when stored as a Spec dictionary; we take care in this function\n to discard such formatting such that the Spec hash does not\n change before/after storage in JSON.\n \"\"\"\n if modules:\n modules = list(modules)\n return modules\n\n @property\n def external_path(self):\n return spack.llnl.path.path_to_os_path(self._external_path)[0]\n\n @external_path.setter\n def external_path(self, ext_path):\n self._external_path = ext_path\n\n @property\n def external(self):\n return bool(self.external_path) or bool(self.external_modules)\n\n @property\n def is_develop(self):\n \"\"\"Return whether the Spec represents a user-developed package\n in a Spack Environment (i.e. using ``spack develop``).\n \"\"\"\n return bool(self.variants.get(\"dev_path\", False))\n\n def clear_dependencies(self):\n \"\"\"Trim the dependencies of this spec.\"\"\"\n self._dependencies.clear()\n\n def clear_edges(self):\n \"\"\"Trim the dependencies and dependents of this spec.\"\"\"\n self._dependencies.clear()\n self._dependents.clear()\n\n def detach(self, deptype=\"all\"):\n \"\"\"Remove any reference that dependencies have of this node.\n\n Args:\n deptype (str or tuple): dependency types tracked by the\n current spec\n \"\"\"\n key = self.dag_hash()\n # Go through the dependencies\n for dep in self.dependencies(deptype=deptype):\n # Remove the spec from dependents\n if self.name in dep._dependents:\n dependents_copy = dep._dependents[self.name]\n del dep._dependents[self.name]\n for edge in dependents_copy:\n if edge.parent.dag_hash() == key:\n continue\n _add_edge_to_map(dep._dependents, edge.parent.name, edge)\n\n def _get_dependency(self, name):\n # WARNING: This function is an implementation detail of the\n # WARNING: original concretizer. Since with that greedy\n # WARNING: algorithm we don't allow multiple nodes from\n # WARNING: the same package in a DAG, here we hard-code\n # WARNING: using index 0 i.e. we assume that we have only\n # WARNING: one edge from package \"name\"\n deps = self.edges_to_dependencies(name=name)\n if len(deps) != 1:\n err_msg = 'expected only 1 \"{0}\" dependency, but got {1}'\n raise spack.error.SpecError(err_msg.format(name, len(deps)))\n return deps[0]\n\n def edges_from_dependents(\n self,\n name: Optional[str] = None,\n depflag: dt.DepFlag = dt.ALL,\n *,\n virtuals: Optional[Union[str, Sequence[str]]] = None,\n ) -> List[DependencySpec]:\n \"\"\"Return a list of edges connecting this node in the DAG\n to parents.\n\n Args:\n name: filter dependents by package name\n depflag: allowed dependency types\n virtuals: allowed virtuals\n \"\"\"\n return _select_edges(self._dependents, parent=name, depflag=depflag, virtuals=virtuals)\n\n def edges_to_dependencies(\n self,\n name: Optional[str] = None,\n depflag: dt.DepFlag = dt.ALL,\n *,\n virtuals: Optional[Union[str, Sequence[str]]] = None,\n ) -> List[DependencySpec]:\n \"\"\"Returns a list of edges connecting this node in the DAG to children.\n\n Args:\n name: filter dependencies by package name\n depflag: allowed dependency types\n virtuals: allowed virtuals\n \"\"\"\n return _select_edges(self._dependencies, child=name, depflag=depflag, virtuals=virtuals)\n\n @property\n def edge_attributes(self) -> str:\n \"\"\"Helper method to print edge attributes in spec strings.\"\"\"\n edges = self.edges_from_dependents()\n if not edges:\n return \"\"\n\n union = DependencySpec(parent=Spec(), spec=self, depflag=0, virtuals=())\n all_direct_edges = all(x.direct for x in edges)\n dep_conditions = set()\n\n for edge in edges:\n union.update_deptypes(edge.depflag)\n union.update_virtuals(edge.virtuals)\n dep_conditions.add(edge.when)\n\n deptypes_str = \"\"\n if not all_direct_edges and union.depflag:\n deptypes_str = f\"deptypes={','.join(dt.flag_to_tuple(union.depflag))}\"\n\n virtuals_str = f\"virtuals={','.join(union.virtuals)}\" if union.virtuals else \"\"\n\n conditions = [str(c) for c in dep_conditions if c != Spec()]\n when_str = f\"when='{','.join(conditions)}'\" if conditions else \"\"\n\n result = \" \".join(filter(lambda x: bool(x), (when_str, deptypes_str, virtuals_str)))\n if result:\n result = f\"[{result}]\"\n return result\n\n def dependencies(\n self,\n name: Optional[str] = None,\n deptype: Union[dt.DepTypes, dt.DepFlag] = dt.ALL,\n *,\n virtuals: Optional[Union[str, Sequence[str]]] = None,\n ) -> List[\"Spec\"]:\n \"\"\"Returns a list of direct dependencies (nodes in the DAG)\n\n Args:\n name: filter dependencies by package name\n deptype: allowed dependency types\n virtuals: allowed virtuals\n \"\"\"\n if not isinstance(deptype, dt.DepFlag):\n deptype = dt.canonicalize(deptype)\n return [\n d.spec for d in self.edges_to_dependencies(name, depflag=deptype, virtuals=virtuals)\n ]\n\n def dependents(\n self, name: Optional[str] = None, deptype: Union[dt.DepTypes, dt.DepFlag] = dt.ALL\n ) -> List[\"Spec\"]:\n \"\"\"Return a list of direct dependents (nodes in the DAG).\n\n Args:\n name: filter dependents by package name\n deptype: allowed dependency types\n \"\"\"\n if not isinstance(deptype, dt.DepFlag):\n deptype = dt.canonicalize(deptype)\n return [d.parent for d in self.edges_from_dependents(name, depflag=deptype)]\n\n def _dependencies_dict(self, depflag: dt.DepFlag = dt.ALL):\n \"\"\"Return a dictionary, keyed by package name, of the direct\n dependencies.\n\n Each value in the dictionary is a list of edges.\n\n Args:\n deptype: allowed dependency types\n \"\"\"\n _sort_fn = lambda x: (x.spec.name, _sort_by_dep_types(x))\n _group_fn = lambda x: x.spec.name\n selected_edges = _select_edges(self._dependencies, depflag=depflag)\n result = {}\n for key, group in itertools.groupby(sorted(selected_edges, key=_sort_fn), key=_group_fn):\n result[key] = list(group)\n return result\n\n def _add_flag(\n self, name: str, value: Union[str, bool], propagate: bool, concrete: bool\n ) -> None:\n \"\"\"Called by the parser to add a known flag\"\"\"\n\n if propagate and name in vt.RESERVED_NAMES:\n raise UnsupportedPropagationError(\n f\"Propagation with '==' is not supported for '{name}'.\"\n )\n\n valid_flags = FlagMap.valid_compiler_flags()\n if name == \"arch\" or name == \"architecture\":\n assert type(value) is str, \"architecture have a string value\"\n parts = tuple(value.split(\"-\"))\n plat, os, tgt = parts if len(parts) == 3 else (None, None, value)\n self._set_architecture(platform=plat, os=os, target=tgt)\n elif name == \"platform\":\n self._set_architecture(platform=value)\n elif name == \"os\" or name == \"operating_system\":\n self._set_architecture(os=value)\n elif name == \"target\":\n self._set_architecture(target=value)\n elif name == \"namespace\":\n self.namespace = value\n elif name in valid_flags:\n assert self.compiler_flags is not None\n assert type(value) is str, f\"{name} must have a string value\"\n flags_and_propagation = spack.compilers.flags.tokenize_flags(value, propagate)\n flag_group = \" \".join(x for (x, y) in flags_and_propagation)\n for flag, propagation in flags_and_propagation:\n self.compiler_flags.add_flag(name, flag, propagation, flag_group)\n else:\n self.variants[name] = vt.VariantValue.from_string_or_bool(\n name, value, propagate=propagate, concrete=concrete\n )\n\n def _set_architecture(self, **kwargs):\n \"\"\"Called by the parser to set the architecture.\"\"\"\n arch_attrs = [\"platform\", \"os\", \"target\"]\n if self.architecture and self.architecture.concrete:\n raise DuplicateArchitectureError(\"Spec cannot have two architectures.\")\n\n if not self.architecture:\n new_vals = tuple(kwargs.get(arg, None) for arg in arch_attrs)\n self.architecture = ArchSpec(new_vals)\n else:\n new_attrvals = [(a, v) for a, v in kwargs.items() if a in arch_attrs]\n for new_attr, new_value in new_attrvals:\n if getattr(self.architecture, new_attr):\n raise DuplicateArchitectureError(f\"Cannot specify '{new_attr}' twice\")\n else:\n setattr(self.architecture, new_attr, new_value)\n\n def _add_dependency(\n self,\n spec: \"Spec\",\n *,\n depflag: dt.DepFlag,\n virtuals: Tuple[str, ...],\n direct: bool = False,\n propagation: PropagationPolicy = PropagationPolicy.NONE,\n when: Optional[\"Spec\"] = None,\n ):\n \"\"\"Called by the parser to add another spec as a dependency.\n\n Args:\n depflag: dependency type for this edge\n virtuals: virtuals on this edge\n direct: if True denotes a direct dependency (associated with the % sigil)\n propagation: propagation policy for this edge\n when: optional condition under which dependency holds\n \"\"\"\n if when is None:\n when = EMPTY_SPEC\n\n if spec.name not in self._dependencies or not spec.name:\n self.add_dependency_edge(\n spec,\n depflag=depflag,\n virtuals=virtuals,\n direct=direct,\n when=when,\n propagation=propagation,\n )\n return\n\n # Keep the intersection of constraints when a dependency is added multiple times with\n # the same deptype. Add a new dependency if it is added with a compatible deptype\n # (for example, a build-only dependency is compatible with a link-only dependency).\n # The only restrictions, currently, are that we cannot add edges with overlapping\n # dependency types and we cannot add multiple edges that have link/run dependency types.\n # See ``spack.deptypes.compatible``.\n orig = self._dependencies[spec.name]\n try:\n dspec = next(\n dspec for dspec in orig if depflag == dspec.depflag and when == dspec.when\n )\n except StopIteration:\n # Error if we have overlapping or incompatible deptypes\n if any(not dt.compatible(dspec.depflag, depflag) for dspec in orig) and all(\n dspec.when == when for dspec in orig\n ):\n edge_attrs = f\"deptypes={dt.flag_to_chars(depflag).strip()}\"\n required_dep_str = f\"^[{edge_attrs}] {str(spec)}\"\n\n raise DuplicateDependencyError(\n f\"{spec.name} is a duplicate dependency, with conflicting dependency types\\n\"\n f\"\\t'{str(self)}' cannot depend on '{required_dep_str}'\"\n )\n\n self.add_dependency_edge(\n spec, depflag=depflag, virtuals=virtuals, direct=direct, when=when\n )\n return\n\n try:\n dspec.spec.constrain(spec)\n dspec.update_virtuals(virtuals=virtuals)\n except spack.error.UnsatisfiableSpecError:\n raise DuplicateDependencyError(\n f\"Cannot depend on incompatible specs '{dspec.spec}' and '{spec}'\"\n )\n\n def add_dependency_edge(\n self,\n dependency_spec: \"Spec\",\n *,\n depflag: dt.DepFlag,\n virtuals: Tuple[str, ...],\n direct: bool = False,\n propagation: PropagationPolicy = PropagationPolicy.NONE,\n when: Optional[\"Spec\"] = None,\n ):\n \"\"\"Add a dependency edge to this spec.\n\n Args:\n dependency_spec: spec of the dependency\n depflag: dependency type for this edge\n virtuals: virtuals provided by this edge\n direct: if True denotes a direct dependency\n propagation: propagation policy for this edge\n when: if non-None, condition under which dependency holds\n \"\"\"\n if when is None:\n when = EMPTY_SPEC\n\n # Check if we need to update edges that are already present\n selected = self._dependencies.get(dependency_spec.name, [])\n for edge in selected:\n has_errors, details = False, []\n msg = f\"cannot update the edge from {edge.parent.name} to {edge.spec.name}\"\n\n if edge.when != when:\n continue\n\n # If the dependency is to an existing spec, we can update dependency\n # types. If it is to a new object, check deptype compatibility.\n if id(edge.spec) != id(dependency_spec) and not dt.compatible(edge.depflag, depflag):\n has_errors = True\n details.append(\n (\n f\"{edge.parent.name} has already an edge matching any\"\n f\" of these types {depflag}\"\n )\n )\n\n if any(v in edge.virtuals for v in virtuals):\n details.append(\n (\n f\"{edge.parent.name} has already an edge matching any\"\n f\" of these virtuals {virtuals}\"\n )\n )\n\n if has_errors:\n raise spack.error.SpecError(msg, \"\\n\".join(details))\n\n for edge in selected:\n if id(dependency_spec) == id(edge.spec) and edge.when == when:\n # If we are here, it means the edge object was previously added to\n # both the parent and the child. When we update this object they'll\n # both see the deptype modification.\n edge.update_deptypes(depflag=depflag)\n edge.update_virtuals(virtuals=virtuals)\n return\n\n edge = DependencySpec(\n self,\n dependency_spec,\n depflag=depflag,\n virtuals=virtuals,\n direct=direct,\n propagation=propagation,\n when=when,\n )\n _add_edge_to_map(self._dependencies, edge.spec.name, edge)\n _add_edge_to_map(dependency_spec._dependents, edge.parent.name, edge)\n\n #\n # Public interface\n #\n @property\n def fullname(self):\n return (\n f\"{self.namespace}.{self.name}\" if self.namespace else (self.name if self.name else \"\")\n )\n\n @property\n def anonymous(self):\n return not self.name and not self.abstract_hash\n\n @property\n def root(self):\n \"\"\"Follow dependent links and find the root of this spec's DAG.\n\n Spack specs have a single root (the package being installed).\n \"\"\"\n # FIXME: In the case of multiple parents this property does not\n # FIXME: make sense. Should we revisit the semantics?\n if not self._dependents:\n return self\n edges_by_package = next(iter(self._dependents.values()))\n return edges_by_package[0].parent.root\n\n @property\n def package(self):\n assert self.concrete, \"{0}: Spec.package can only be called on concrete specs\".format(\n self.name\n )\n if not self._package:\n self._package = spack.repo.PATH.get(self)\n return self._package\n\n @property\n def concrete(self):\n \"\"\"A spec is concrete if it describes a single build of a package.\n\n More formally, a spec is concrete if concretize() has been called\n on it and it has been marked ``_concrete``.\n\n Concrete specs either can be or have been built. All constraints\n have been resolved, optional dependencies have been added or\n removed, a compiler has been chosen, and all variants have\n values.\n \"\"\"\n return self._concrete\n\n @property\n def spliced(self):\n \"\"\"Returns whether or not this Spec is being deployed as built i.e.\n whether or not this Spec has ever been spliced.\n \"\"\"\n return any(s.build_spec is not s for s in self.traverse(root=True))\n\n @property\n def installed(self):\n \"\"\"Installation status of a package.\n\n Returns:\n True if the package has been installed, False otherwise.\n \"\"\"\n if not self.concrete:\n return False\n\n try:\n # If the spec is in the DB, check the installed\n # attribute of the record\n from spack.store import STORE\n\n return STORE.db.get_record(self).installed\n except KeyError:\n # If the spec is not in the DB, the method\n # above raises a Key error\n return False\n\n @property\n def installed_upstream(self):\n \"\"\"Whether the spec is installed in an upstream repository.\n\n Returns:\n True if the package is installed in an upstream, False otherwise.\n \"\"\"\n if not self.concrete:\n return False\n\n from spack.store import STORE\n\n upstream, record = STORE.db.query_by_spec_hash(self.dag_hash())\n return upstream and record and record.installed\n\n @overload\n def traverse(\n self,\n *,\n root: bool = ...,\n order: spack.traverse.OrderType = ...,\n cover: spack.traverse.CoverType = ...,\n direction: spack.traverse.DirectionType = ...,\n deptype: Union[dt.DepFlag, dt.DepTypes] = ...,\n depth: Literal[False] = False,\n key: Callable[[\"Spec\"], Any] = ...,\n visited: Optional[Set[Any]] = ...,\n ) -> Iterable[\"Spec\"]: ...\n\n @overload\n def traverse(\n self,\n *,\n root: bool = ...,\n order: spack.traverse.OrderType = ...,\n cover: spack.traverse.CoverType = ...,\n direction: spack.traverse.DirectionType = ...,\n deptype: Union[dt.DepFlag, dt.DepTypes] = ...,\n depth: Literal[True],\n key: Callable[[\"Spec\"], Any] = ...,\n visited: Optional[Set[Any]] = ...,\n ) -> Iterable[Tuple[int, \"Spec\"]]: ...\n\n def traverse(\n self,\n *,\n root: bool = True,\n order: spack.traverse.OrderType = \"pre\",\n cover: spack.traverse.CoverType = \"nodes\",\n direction: spack.traverse.DirectionType = \"children\",\n deptype: Union[dt.DepFlag, dt.DepTypes] = \"all\",\n depth: bool = False,\n key: Callable[[\"Spec\"], Any] = id,\n visited: Optional[Set[Any]] = None,\n ) -> Iterable[Union[\"Spec\", Tuple[int, \"Spec\"]]]:\n \"\"\"Shorthand for :meth:`~spack.traverse.traverse_nodes`\"\"\"\n return spack.traverse.traverse_nodes(\n [self],\n root=root,\n order=order,\n cover=cover,\n direction=direction,\n deptype=deptype,\n depth=depth,\n key=key,\n visited=visited,\n )\n\n @overload\n def traverse_edges(\n self,\n *,\n root: bool = ...,\n order: spack.traverse.OrderType = ...,\n cover: spack.traverse.CoverType = ...,\n direction: spack.traverse.DirectionType = ...,\n deptype: Union[dt.DepFlag, dt.DepTypes] = ...,\n depth: Literal[False] = False,\n key: Callable[[\"Spec\"], Any] = ...,\n visited: Optional[Set[Any]] = ...,\n ) -> Iterable[DependencySpec]: ...\n\n @overload\n def traverse_edges(\n self,\n *,\n root: bool = ...,\n order: spack.traverse.OrderType = ...,\n cover: spack.traverse.CoverType = ...,\n direction: spack.traverse.DirectionType = ...,\n deptype: Union[dt.DepFlag, dt.DepTypes] = ...,\n depth: Literal[True],\n key: Callable[[\"Spec\"], Any] = ...,\n visited: Optional[Set[Any]] = ...,\n ) -> Iterable[Tuple[int, DependencySpec]]: ...\n\n def traverse_edges(\n self,\n *,\n root: bool = True,\n order: spack.traverse.OrderType = \"pre\",\n cover: spack.traverse.CoverType = \"nodes\",\n direction: spack.traverse.DirectionType = \"children\",\n deptype: Union[dt.DepFlag, dt.DepTypes] = \"all\",\n depth: bool = False,\n key: Callable[[\"Spec\"], Any] = id,\n visited: Optional[Set[Any]] = None,\n ) -> Iterable[Union[DependencySpec, Tuple[int, DependencySpec]]]:\n \"\"\"Shorthand for :meth:`~spack.traverse.traverse_edges`\"\"\"\n return spack.traverse.traverse_edges(\n [self],\n root=root,\n order=order,\n cover=cover,\n direction=direction,\n deptype=deptype,\n depth=depth,\n key=key,\n visited=visited,\n )\n\n @property\n def prefix(self) -> spack.util.prefix.Prefix:\n if not self._concrete:\n raise spack.error.SpecError(f\"Spec is not concrete: {self}\")\n\n if self._prefix is None:\n from spack.store import STORE\n\n _, record = STORE.db.query_by_spec_hash(self.dag_hash())\n if record and record.path:\n self.set_prefix(record.path)\n else:\n self.set_prefix(STORE.layout.path_for_spec(self))\n assert self._prefix is not None\n return self._prefix\n\n def set_prefix(self, value: str) -> None:\n self._prefix = spack.util.prefix.Prefix(spack.llnl.path.convert_to_platform_path(value))\n\n def spec_hash(self, hash: ht.SpecHashDescriptor) -> str:\n \"\"\"Utility method for computing different types of Spec hashes.\n\n Arguments:\n hash: type of hash to generate.\n \"\"\"\n # TODO: currently we strip build dependencies by default. Rethink\n # this when we move to using package hashing on all specs.\n if hash.override is not None:\n return hash.override(self)\n node_dict = self.to_node_dict(hash=hash)\n json_text = json.dumps(\n node_dict, ensure_ascii=True, indent=None, separators=(\",\", \":\"), sort_keys=False\n )\n # This implements \"frankenhashes\", preserving the last 7 characters of the\n # original hash when splicing so that we can avoid relocation issues\n out = spack.util.hash.b32_hash(json_text)\n if self.build_spec is not self:\n return out[:-7] + self.build_spec.spec_hash(hash)[-7:]\n return out\n\n def _cached_hash(\n self, hash: ht.SpecHashDescriptor, length: Optional[int] = None, force: bool = False\n ) -> str:\n \"\"\"Helper function for storing a cached hash on the spec.\n\n This will run spec_hash() with the deptype and package_hash\n parameters, and if this spec is concrete, it will store the value\n in the supplied attribute on this spec.\n\n Arguments:\n hash: type of hash to generate.\n length: length of hash prefix to return (default is full hash string)\n force: cache the hash even if spec is not concrete (default False)\n \"\"\"\n hash_string = getattr(self, hash.attr, None)\n if hash_string:\n return hash_string[:length]\n\n hash_string = self.spec_hash(hash)\n if force or self.concrete:\n setattr(self, hash.attr, hash_string)\n\n return hash_string[:length]\n\n def package_hash(self):\n \"\"\"Compute the hash of the contents of the package for this node\"\"\"\n # Concrete specs with the old DAG hash did not have the package hash, so we do\n # not know what the package looked like at concretization time\n if self.concrete and not self._package_hash:\n raise ValueError(\n \"Cannot call package_hash() on concrete specs with the old dag_hash()\"\n )\n\n return self._cached_hash(ht.package_hash)\n\n def dag_hash(self, length=None):\n \"\"\"This is Spack's default hash, used to identify installations.\n\n NOTE: Versions of Spack prior to 0.18 only included link and run deps.\n NOTE: Versions of Spack prior to 1.0 only did not include test deps.\n\n \"\"\"\n return self._cached_hash(ht.dag_hash, length)\n\n def dag_hash_bit_prefix(self, bits):\n \"\"\"Get the first bits of the DAG hash as an integer type.\"\"\"\n return spack.util.hash.base32_prefix_bits(self.dag_hash(), bits)\n\n def _lookup_hash(self):\n \"\"\"Lookup just one spec with an abstract hash, returning a spec from the the environment,\n store, or finally, binary caches.\"\"\"\n from spack.binary_distribution import BinaryCacheQuery\n from spack.environment import active_environment\n from spack.store import STORE\n\n active_env = active_environment()\n\n # First env, then store, then binary cache\n matches = (\n (active_env.all_matching_specs(self) if active_env else [])\n or STORE.db.query(self, installed=InstallRecordStatus.ANY)\n or BinaryCacheQuery(True)(self)\n )\n\n if not matches:\n raise InvalidHashError(self, self.abstract_hash)\n\n if len(matches) != 1:\n raise AmbiguousHashError(\n f\"Multiple packages specify hash beginning '{self.abstract_hash}'.\", *matches\n )\n\n return matches[0]\n\n def lookup_hash(self):\n \"\"\"Given a spec with an abstract hash, return a copy of the spec with all properties and\n dependencies by looking up the hash in the environment, store, or finally, binary caches.\n This is non-destructive.\"\"\"\n if self.concrete or not any(node.abstract_hash for node in self.traverse()):\n return self\n\n spec = self.copy(deps=False)\n # root spec is replaced\n if spec.abstract_hash:\n spec._dup(self._lookup_hash())\n return spec\n\n # Map the dependencies that need to be replaced\n node_lookup = {\n id(node): node._lookup_hash()\n for node in self.traverse(root=False)\n if node.abstract_hash\n }\n\n # Reconstruct dependencies\n for edge in self.traverse_edges(root=False):\n key = edge.parent.name\n current_node = spec if key == spec.name else spec[key]\n child_node = node_lookup.get(id(edge.spec), edge.spec.copy())\n current_node._add_dependency(\n child_node, depflag=edge.depflag, virtuals=edge.virtuals, direct=edge.direct\n )\n\n return spec\n\n def replace_hash(self):\n \"\"\"Given a spec with an abstract hash, attempt to populate all properties and dependencies\n by looking up the hash in the environment, store, or finally, binary caches.\n This is destructive.\"\"\"\n\n if not any(node for node in self.traverse(order=\"post\") if node.abstract_hash):\n return\n\n self._dup(self.lookup_hash())\n\n def to_node_dict(self, hash: ht.SpecHashDescriptor = ht.dag_hash) -> Dict[str, Any]:\n \"\"\"Create a dictionary representing the state of this Spec.\n\n This method creates the content that is eventually hashed by Spack to create identifiers\n like the DAG hash (see :meth:`dag_hash()`). Example result of this function for the\n ``sqlite`` package::\n\n {\n \"name\": \"sqlite\",\n \"version\": \"3.46.0\",\n \"arch\": {\"platform\": \"linux\", \"platform_os\": \"ubuntu24.04\", \"target\": \"x86_64_v3\"},\n \"namespace\": \"builtin\",\n \"parameters\": {\n \"build_system\": \"autotools\",\n \"column_metadata\": True,\n \"dynamic_extensions\": True,\n \"fts\": True,\n \"functions\": False,\n \"rtree\": True,\n \"cflags\": [],\n \"cppflags\": [],\n \"cxxflags\": [],\n \"fflags\": [],\n \"ldflags\": [],\n \"ldlibs\": [],\n },\n \"package_hash\": \"umcghjlve5347o3q2odo7vfcso2zhxdzmfdba23nkdhe5jntlhia====\",\n \"dependencies\": [\n {\n \"name\": \"compiler-wrapper\",\n \"hash\": \"c5bxlim3zge4snwrwtd6rzuvq2unek6s\",\n \"parameters\": {\"deptypes\": (\"build\",), \"virtuals\": ()},\n },\n {\n \"name\": \"gcc\",\n \"hash\": \"6dzveld2rtt2dkhklxfnery5wbtb5uus\",\n \"parameters\": {\"deptypes\": (\"build\",), \"virtuals\": (\"c\",)},\n },\n ...\n ],\n \"annotations\": {\"original_specfile_version\": 5},\n }\n\n\n Note that the dictionary returned does *not* include the hash of the *root* of the spec,\n though it does include hashes for each dependency and its own package hash.\n\n See :meth:`to_dict()` for a \"complete\" spec hash, with hashes for each node and nodes for\n each dependency (instead of just their hashes).\n\n Arguments:\n hash: type of hash to generate.\n \"\"\"\n d: Dict[str, Any] = {\"name\": self.name}\n\n if self.versions:\n d.update(self.versions.to_dict())\n\n if self.architecture:\n d.update(self.architecture.to_dict())\n\n if self.namespace:\n d[\"namespace\"] = self.namespace\n\n params: Dict[str, Any] = dict(sorted(v.yaml_entry() for v in self.variants.values()))\n\n # Only need the string compiler flag for yaml file\n params.update(\n sorted(\n self.compiler_flags.yaml_entry(flag_type)\n for flag_type in self.compiler_flags.keys()\n )\n )\n\n if params:\n d[\"parameters\"] = params\n\n if params and not self.concrete:\n flag_names = [\n name\n for name, flags in self.compiler_flags.items()\n if any(x.propagate for x in flags)\n ]\n d[\"propagate\"] = sorted(\n itertools.chain(\n [v.name for v in self.variants.values() if v.propagate], flag_names\n )\n )\n d[\"abstract\"] = sorted(v.name for v in self.variants.values() if not v.concrete)\n\n if self.external:\n d[\"external\"] = {\n \"path\": self.external_path,\n \"module\": self.external_modules or None,\n \"extra_attributes\": syaml.sorted_dict(self.extra_attributes),\n }\n\n if not self._concrete:\n d[\"concrete\"] = False\n\n if \"patches\" in self.variants:\n variant = self.variants[\"patches\"]\n if hasattr(variant, \"_patches_in_order_of_appearance\"):\n d[\"patches\"] = variant._patches_in_order_of_appearance\n\n if (\n self._concrete\n and hash.package_hash\n and hasattr(self, \"_package_hash\")\n and self._package_hash\n ):\n # We use the attribute here instead of `self.package_hash()` because this\n # should *always* be assignhed at concretization time. We don't want to try\n # to compute a package hash for concrete spec where a) the package might not\n # exist, or b) the `dag_hash` didn't include the package hash when the spec\n # was concretized.\n package_hash = self._package_hash\n\n # Full hashes are in bytes\n if not isinstance(package_hash, str) and isinstance(package_hash, bytes):\n package_hash = package_hash.decode(\"utf-8\")\n d[\"package_hash\"] = package_hash\n\n # Note: Relies on sorting dict by keys later in algorithm.\n deps = self._dependencies_dict(depflag=hash.depflag)\n if deps:\n dependencies = []\n for name, edges_for_name in sorted(deps.items()):\n for dspec in edges_for_name:\n dep_attrs = {\n \"name\": name,\n hash.name: dspec.spec._cached_hash(hash),\n \"parameters\": {\n \"deptypes\": dt.flag_to_tuple(dspec.depflag),\n \"virtuals\": dspec.virtuals,\n },\n }\n if dspec.direct:\n dep_attrs[\"parameters\"][\"direct\"] = True\n dependencies.append(dep_attrs)\n\n d[\"dependencies\"] = dependencies\n\n # Name is included in case this is replacing a virtual.\n if self._build_spec:\n d[\"build_spec\"] = {\n \"name\": self.build_spec.name,\n hash.name: self.build_spec._cached_hash(hash),\n }\n\n # Annotations\n d[\"annotations\"] = {\"original_specfile_version\": self.annotations.original_spec_format}\n if self.annotations.original_spec_format < 5:\n d[\"annotations\"][\"compiler\"] = str(self.annotations.compiler_node_attribute)\n\n return d\n\n def to_dict(self, hash: ht.SpecHashDescriptor = ht.dag_hash) -> Dict[str, Any]:\n \"\"\"Create a dictionary suitable for writing this spec to YAML or JSON.\n\n This dictionary is like the one that is ultimately written to a ``spec.json`` file in each\n Spack installation directory. For example, for sqlite::\n\n {\n \"spec\": {\n \"_meta\": {\"version\": 5},\n \"nodes\": [\n {\n \"name\": \"sqlite\",\n \"version\": \"3.46.0\",\n \"arch\": {\n \"platform\": \"linux\",\n \"platform_os\": \"ubuntu24.04\",\n \"target\": \"x86_64_v3\"\n },\n \"namespace\": \"builtin\",\n \"parameters\": {\n \"build_system\": \"autotools\",\n \"column_metadata\": True,\n \"dynamic_extensions\": True,\n \"fts\": True,\n \"functions\": False,\n \"rtree\": True,\n \"cflags\": [],\n \"cppflags\": [],\n \"cxxflags\": [],\n \"fflags\": [],\n \"ldflags\": [],\n \"ldlibs\": [],\n },\n \"package_hash\": \"umcghjlve5347o...xdzmfdba23nkdhe5jntlhia====\",\n \"dependencies\": [\n {\n \"name\": \"compiler-wrapper\",\n \"hash\": \"c5bxlim3zge4snwrwtd6rzuvq2unek6s\",\n \"parameters\": {\"deptypes\": (\"build\",), \"virtuals\": ()},\n },\n {\n \"name\": \"gcc\",\n \"hash\": \"6dzveld2rtt2dkhklxfnery5wbtb5uus\",\n \"parameters\": {\"deptypes\": (\"build\",), \"virtuals\": (\"c\",)},\n },\n ...\n ],\n \"annotations\": {\"original_specfile_version\": 5},\n \"hash\": \"a2ubvvqnula6zdppckwqrjf3zmsdzpoh\",\n },\n ...\n ],\n }\n }\n\n Note that this dictionary starts with the ``spec`` key, and what follows is a list starting\n with the root spec, followed by its dependencies in preorder.\n\n The method :meth:`from_dict()` can be used to read back in a spec that has been converted\n to a dictionary, serialized, and read back in.\n \"\"\"\n node_list = [] # Using a list to preserve preorder traversal for hash.\n hash_set = set()\n for s in self.traverse(order=\"pre\", deptype=hash.depflag):\n spec_hash = s._cached_hash(hash)\n\n if spec_hash not in hash_set:\n node_list.append(s.node_dict_with_hashes(hash))\n hash_set.add(spec_hash)\n\n if s.build_spec is not s:\n build_spec_list = s.build_spec.to_dict(hash)[\"spec\"][\"nodes\"]\n for node in build_spec_list:\n node_hash = node[hash.name]\n if node_hash not in hash_set:\n node_list.append(node)\n hash_set.add(node_hash)\n\n return {\"spec\": {\"_meta\": {\"version\": SPECFILE_FORMAT_VERSION}, \"nodes\": node_list}}\n\n def node_dict_with_hashes(self, hash: ht.SpecHashDescriptor = ht.dag_hash) -> Dict[str, Any]:\n \"\"\"Returns a node dict of this spec with the dag hash, and the provided hash (if not\n the dag hash).\"\"\"\n node = self.to_node_dict(hash)\n # All specs have at least a DAG hash\n node[ht.dag_hash.name] = self.dag_hash()\n\n if not self.concrete:\n node[\"concrete\"] = False\n\n # we can also give them other hash types if we want\n if hash.name != ht.dag_hash.name:\n node[hash.name] = self._cached_hash(hash)\n\n return node\n\n def to_yaml(self, stream=None, hash=ht.dag_hash):\n return syaml.dump(self.to_dict(hash), stream=stream, default_flow_style=False)\n\n def to_json(self, stream=None, *, hash=ht.dag_hash, pretty=False):\n return sjson.dump(self.to_dict(hash), stream=stream, pretty=pretty)\n\n @staticmethod\n def from_specfile(path):\n \"\"\"Construct a spec from a JSON or YAML spec file path\"\"\"\n with open(path, \"r\", encoding=\"utf-8\") as fd:\n file_content = fd.read()\n if path.endswith(\".json\"):\n return Spec.from_json(file_content)\n return Spec.from_yaml(file_content)\n\n @staticmethod\n def override(init_spec, change_spec):\n # TODO: this doesn't account for the case where the changed spec\n # (and the user spec) have dependencies\n new_spec = init_spec.copy()\n package_cls = spack.repo.PATH.get_pkg_class(new_spec.name)\n if change_spec.versions and not change_spec.versions == vn.any_version:\n new_spec.versions = change_spec.versions\n\n for vname, value in change_spec.variants.items():\n if vname in package_cls.variant_names():\n if vname in new_spec.variants:\n new_spec.variants.substitute(value)\n else:\n new_spec.variants[vname] = value\n else:\n raise ValueError(\"{0} is not a variant of {1}\".format(vname, new_spec.name))\n\n if change_spec.compiler_flags:\n for flagname, flagvals in change_spec.compiler_flags.items():\n new_spec.compiler_flags[flagname] = flagvals\n if change_spec.architecture:\n new_spec.architecture = ArchSpec.override(\n new_spec.architecture, change_spec.architecture\n )\n return new_spec\n\n @staticmethod\n def from_literal(spec_dict: dict, normal: bool = True) -> \"Spec\":\n \"\"\"Builds a Spec from a dictionary containing the spec literal.\n\n The dictionary must have a single top level key, representing the root,\n and as many secondary level keys as needed in the spec.\n\n The keys can be either a string or a Spec or a tuple containing the\n Spec and the dependency types.\n\n Args:\n spec_dict: the dictionary containing the spec literal\n normal: if :data:`True` the same key appearing at different levels\n of the ``spec_dict`` will map to the same object in memory.\n\n Examples:\n A simple spec ``foo`` with no dependencies::\n\n {\"foo\": None}\n\n A spec ``foo`` with a ``(build, link)`` dependency ``bar``::\n\n {\"foo\":\n {\"bar:build,link\": None}\n }\n\n A spec with a diamond dependency and various build types::\n\n {\"dt-diamond\": {\n \"dt-diamond-left:build,link\": {\n \"dt-diamond-bottom:build\": None\n },\n \"dt-diamond-right:build,link\": {\n \"dt-diamond-bottom:build,link,run\": None\n }\n }}\n\n The same spec with a double copy of ``dt-diamond-bottom`` and\n no diamond structure::\n\n Spec.from_literal({\"dt-diamond\": {\n \"dt-diamond-left:build,link\": {\n \"dt-diamond-bottom:build\": None\n },\n \"dt-diamond-right:build,link\": {\n \"dt-diamond-bottom:build,link,run\": None\n }\n }, normal=False}\n\n Constructing a spec using a Spec object as key::\n\n mpich = Spec(\"mpich\")\n libelf = Spec(\"libelf@1.8.11\")\n expected_normalized = Spec.from_literal({\n \"mpileaks\": {\n \"callpath\": {\n \"dyninst\": {\n \"libdwarf\": {libelf: None},\n libelf: None\n },\n mpich: None\n },\n mpich: None\n },\n })\n\n \"\"\"\n\n # Maps a literal to a Spec, to be sure we are reusing the same object\n spec_cache = LazySpecCache()\n\n def spec_builder(d):\n # The invariant is that the top level dictionary must have\n # only one key\n assert len(d) == 1\n\n # Construct the top-level spec\n spec_like, dep_like = next(iter(d.items()))\n\n # If the requirements was for unique nodes (default)\n # then reuse keys from the local cache. Otherwise build\n # a new node every time.\n if not isinstance(spec_like, Spec):\n spec = spec_cache[spec_like] if normal else Spec(spec_like)\n else:\n spec = spec_like\n\n if dep_like is None:\n return spec\n\n def name_and_dependency_types(s: str) -> Tuple[str, dt.DepFlag]:\n \"\"\"Given a key in the dictionary containing the literal,\n extracts the name of the spec and its dependency types.\n\n Args:\n s: key in the dictionary containing the literal\n \"\"\"\n t = s.split(\":\")\n\n if len(t) > 2:\n msg = 'more than one \":\" separator in key \"{0}\"'\n raise KeyError(msg.format(s))\n\n name = t[0]\n if len(t) == 2:\n depflag = dt.flag_from_strings(dep_str.strip() for dep_str in t[1].split(\",\"))\n else:\n depflag = 0\n return name, depflag\n\n def spec_and_dependency_types(\n s: Union[Spec, Tuple[Spec, str]],\n ) -> Tuple[Spec, dt.DepFlag]:\n \"\"\"Given a non-string key in the literal, extracts the spec\n and its dependency types.\n\n Args:\n s: either a Spec object, or a tuple of Spec and string of dependency types\n \"\"\"\n if isinstance(s, Spec):\n return s, 0\n\n spec_obj, dtypes = s\n return spec_obj, dt.flag_from_strings(dt.strip() for dt in dtypes.split(\",\"))\n\n # Recurse on dependencies\n for s, s_dependencies in dep_like.items():\n if isinstance(s, str):\n dag_node, dep_flag = name_and_dependency_types(s)\n else:\n dag_node, dep_flag = spec_and_dependency_types(s)\n\n dependency_spec = spec_builder({dag_node: s_dependencies})\n spec._add_dependency(dependency_spec, depflag=dep_flag, virtuals=())\n\n return spec\n\n return spec_builder(spec_dict)\n\n @staticmethod\n def from_dict(data) -> \"Spec\":\n \"\"\"Construct a spec from JSON/YAML.\n\n Args:\n data: a nested dict/list data structure read from YAML or JSON.\n \"\"\"\n # Legacy specfile format\n if isinstance(data[\"spec\"], list):\n spec = SpecfileV1.load(data)\n elif int(data[\"spec\"][\"_meta\"][\"version\"]) == 2:\n spec = SpecfileV2.load(data)\n elif int(data[\"spec\"][\"_meta\"][\"version\"]) == 3:\n spec = SpecfileV3.load(data)\n elif int(data[\"spec\"][\"_meta\"][\"version\"]) == 4:\n spec = SpecfileV4.load(data)\n else:\n spec = SpecfileV5.load(data)\n\n # Any git version should\n for s in spec.traverse():\n s.attach_git_version_lookup()\n\n return spec\n\n @staticmethod\n def from_yaml(stream) -> \"Spec\":\n \"\"\"Construct a spec from YAML.\n\n Args:\n stream: string or file object to read from.\n \"\"\"\n data = syaml.load(stream)\n return Spec.from_dict(data)\n\n @staticmethod\n def from_json(stream) -> \"Spec\":\n \"\"\"Construct a spec from JSON.\n\n Args:\n stream: string or file object to read from.\n \"\"\"\n try:\n data = sjson.load(stream)\n return Spec.from_dict(data)\n except Exception as e:\n raise sjson.SpackJSONError(\"error parsing JSON spec:\", e) from e\n\n @staticmethod\n def extract_json_from_clearsig(data):\n m = CLEARSIGN_FILE_REGEX.search(data)\n if m:\n return sjson.load(m.group(1))\n return sjson.load(data)\n\n @staticmethod\n def from_signed_json(stream):\n \"\"\"Construct a spec from clearsigned json spec file.\n\n Args:\n stream: string or file object to read from.\n \"\"\"\n data = stream\n if hasattr(stream, \"read\"):\n data = stream.read()\n\n extracted_json = Spec.extract_json_from_clearsig(data)\n return Spec.from_dict(extracted_json)\n\n @staticmethod\n def from_detection(\n spec_str: str,\n *,\n external_path: str,\n external_modules: Optional[List[str]] = None,\n extra_attributes: Optional[Dict] = None,\n ) -> \"Spec\":\n \"\"\"Construct a spec from a spec string determined during external\n detection and attach extra attributes to it.\n\n Args:\n spec_str: spec string\n external_path: prefix of the external spec\n external_modules: optional module files to be loaded when the external spec is used\n extra_attributes: dictionary containing extra attributes\n \"\"\"\n s = Spec(spec_str, external_path=external_path, external_modules=external_modules)\n extra_attributes = syaml.sorted_dict(extra_attributes or {})\n # This is needed to be able to validate multi-valued variants,\n # otherwise they'll still be abstract in the context of detection.\n substitute_abstract_variants(s)\n s.extra_attributes = extra_attributes\n return s\n\n def _patches_assigned(self):\n \"\"\"Whether patches have been assigned to this spec by the concretizer.\"\"\"\n # FIXME: _patches_in_order_of_appearance is attached after concretization\n # FIXME: to store the order of patches.\n # FIXME: Probably needs to be refactored in a cleaner way.\n if \"patches\" not in self.variants:\n return False\n\n # ensure that patch state is consistent\n patch_variant = self.variants[\"patches\"]\n assert hasattr(patch_variant, \"_patches_in_order_of_appearance\"), (\n \"patches should always be assigned with a patch variant.\"\n )\n\n return True\n\n @staticmethod\n def ensure_no_deprecated(root: \"Spec\") -> None:\n \"\"\"Raise if a deprecated spec is in the dag of the given root spec.\n\n Raises:\n spack.spec.SpecDeprecatedError: if any deprecated spec is found\n \"\"\"\n deprecated = []\n from spack.store import STORE\n\n with STORE.db.read_transaction():\n for x in root.traverse():\n _, rec = STORE.db.query_by_spec_hash(x.dag_hash())\n if rec and rec.deprecated_for:\n deprecated.append(rec)\n if deprecated:\n msg = \"\\n The following specs have been deprecated\"\n msg += \" in favor of specs with the hashes shown:\\n\"\n for rec in deprecated:\n msg += \" %s --> %s\\n\" % (rec.spec, rec.deprecated_for)\n msg += \"\\n\"\n msg += \" For each package listed, choose another spec\\n\"\n raise SpecDeprecatedError(msg)\n\n def _mark_root_concrete(self, value=True):\n \"\"\"Mark just this spec (not dependencies) concrete.\"\"\"\n if (not value) and self.concrete and self.installed:\n return\n self._concrete = value\n self._validate_version()\n for variant in self.variants.values():\n variant.concrete = True\n\n def _validate_version(self):\n # Specs that were concretized with just a git sha as version, without associated\n # Spack version, get their Spack version mapped to develop. This should only apply\n # when reading specs concretized with Spack 0.19 or earlier. Currently Spack always\n # ensures that GitVersion specs have an associated Spack version.\n v = self.versions.concrete\n if not isinstance(v, vn.GitVersion):\n return\n\n try:\n v.ref_version\n except vn.VersionLookupError:\n before = self.cformat(\"{name}{@version}{/hash:7}\")\n v.std_version = vn.StandardVersion.from_string(\"develop\")\n tty.debug(\n f\"the git sha of {before} could not be resolved to spack version; \"\n f\"it has been replaced by {self.cformat('{name}{@version}{/hash:7}')}.\"\n )\n\n def _mark_concrete(self, value=True):\n \"\"\"Mark this spec and its dependencies as concrete.\n\n Only for internal use -- client code should use \"concretize\"\n unless there is a need to force a spec to be concrete.\n \"\"\"\n # if set to false, clear out all hashes (set to None or remove attr)\n # may need to change references to respect None\n for s in self.traverse():\n if (not value) and s.concrete and s.installed:\n continue\n elif not value:\n s.clear_caches()\n s._mark_root_concrete(value)\n\n def _finalize_concretization(self):\n \"\"\"Assign hashes to this spec, and mark it concrete.\n\n There are special semantics to consider for ``package_hash``, because we can't\n call it on *already* concrete specs, but we need to assign it *at concretization\n time* to just-concretized specs. So, the concretizer must assign the package\n hash *before* marking their specs concrete (so that we know which specs were\n already concrete before this latest concretization).\n\n ``dag_hash`` is also tricky, since it cannot compute ``package_hash()`` lazily.\n Because ``package_hash`` needs to be assigned *at concretization time*,\n ``to_node_dict()`` can't just assume that it can compute ``package_hash`` itself\n -- it needs to either see or not see a ``_package_hash`` attribute.\n\n Rules of thumb for ``package_hash``:\n 1. Old-style concrete specs from *before* ``dag_hash`` included ``package_hash``\n will not have a ``_package_hash`` attribute at all.\n 2. New-style concrete specs will have a ``_package_hash`` assigned at\n concretization time.\n 3. Abstract specs will not have a ``_package_hash`` attribute at all.\n\n \"\"\"\n for spec in self.traverse():\n # Already concrete specs either already have a package hash (new dag_hash())\n # or they never will b/c we can't know it (old dag_hash()). Skip them.\n #\n # We only assign package hash to not-yet-concrete specs, for which we know\n # we can compute the hash.\n if not spec.concrete:\n # we need force=True here because package hash assignment has to happen\n # before we mark concrete, so that we know what was *already* concrete.\n spec._cached_hash(ht.package_hash, force=True)\n\n # keep this check here to ensure package hash is saved\n assert getattr(spec, ht.package_hash.attr)\n\n # Mark everything in the spec as concrete\n self._mark_concrete()\n\n # Assign dag_hash (this *could* be done lazily, but it's assigned anyway in\n # ensure_no_deprecated, and it's clearer to see explicitly where it happens).\n # Any specs that were concrete before finalization will already have a cached\n # DAG hash.\n for spec in self.traverse():\n spec._cached_hash(ht.dag_hash)\n\n def index(self, deptype=\"all\"):\n \"\"\"Return a dictionary that points to all the dependencies in this\n spec.\n \"\"\"\n dm = collections.defaultdict(list)\n for spec in self.traverse(deptype=deptype):\n dm[spec.name].append(spec)\n return dm\n\n def validate_or_raise(self):\n \"\"\"Checks that names and values in this spec are real. If they're not,\n it will raise an appropriate exception.\n \"\"\"\n # FIXME: this function should be lazy, and collect all the errors\n # FIXME: before raising the exceptions, instead of being greedy and\n # FIXME: raise just the first one encountered\n for spec in self.traverse():\n # raise an UnknownPackageError if the spec's package isn't real.\n if spec.name and not spack.repo.PATH.is_virtual(spec.name):\n spack.repo.PATH.get_pkg_class(spec.fullname)\n\n # FIXME: atm allow '%' on abstract specs only if they depend on C, C++, or Fortran\n if spec.dependencies(deptype=\"build\"):\n pkg_cls = spack.repo.PATH.get_pkg_class(spec.fullname)\n pkg_dependencies = pkg_cls.dependency_names()\n if not any(x in pkg_dependencies for x in (\"c\", \"cxx\", \"fortran\")):\n raise UnsupportedCompilerError(\n f\"{spec.fullname} does not depend on 'c', 'cxx, or 'fortran'\"\n )\n\n # Ensure correctness of variants (if the spec is not virtual)\n if not spack.repo.PATH.is_virtual(spec.name):\n Spec.ensure_valid_variants(spec)\n substitute_abstract_variants(spec)\n\n @staticmethod\n def ensure_valid_variants(spec: \"Spec\") -> None:\n \"\"\"Ensures that the variant attached to the given spec are valid.\n\n Raises:\n spack.variant.UnknownVariantError: on the first unknown variant found\n \"\"\"\n # concrete variants are always valid\n if spec.concrete:\n return\n\n pkg_cls = spack.repo.PATH.get_pkg_class(spec.fullname)\n pkg_variants = pkg_cls.variant_names()\n # reserved names are variants that may be set on any package\n # but are not necessarily recorded by the package's class\n propagate_variants = [name for name, variant in spec.variants.items() if variant.propagate]\n\n not_existing = set(spec.variants)\n not_existing.difference_update(pkg_variants, vt.RESERVED_NAMES, propagate_variants)\n\n if not_existing:\n raise vt.UnknownVariantError(\n f\"No such variant {not_existing} for spec: '{spec}'\", list(not_existing)\n )\n\n def constrain(self, other, deps=True) -> bool:\n \"\"\"Constrains self with other, and returns True if self changed, False otherwise.\n\n Args:\n other: constraint to be added to self\n deps: if False, constrain only the root node, otherwise constrain dependencies as well\n\n Raises:\n spack.error.UnsatisfiableSpecError: when self cannot be constrained\n \"\"\"\n return self._constrain(other, deps=deps, resolve_virtuals=True)\n\n def _constrain_symbolically(self, other, deps=True) -> bool:\n \"\"\"Constrains self with other, and returns True if self changed, False otherwise.\n\n This function has no notion of virtuals, so it does not need a repository.\n\n Args:\n other: constraint to be added to self\n deps: if False, constrain only the root node, otherwise constrain dependencies as well\n\n Raises:\n spack.error.UnsatisfiableSpecError: when self cannot be constrained\n\n Examples:\n >>> from spack.spec import Spec, UnsatisfiableDependencySpecError\n >>> s = Spec(\"hdf5 ^mpi@4\")\n >>> t = Spec(\"hdf5 ^mpi=openmpi\")\n >>> try:\n ... s.constrain(t)\n ... except UnsatisfiableDependencySpecError as e:\n ... print(e)\n ...\n hdf5 ^mpi=openmpi does not satisfy hdf5 ^mpi@4\n >>> s._constrain_symbolically(t)\n True\n >>> s\n hdf5 ^mpi@4 ^mpi=openmpi\n \"\"\"\n return self._constrain(other, deps=deps, resolve_virtuals=False)\n\n def _constrain(self, other, deps=True, *, resolve_virtuals: bool):\n # If we are trying to constrain a concrete spec, either the spec\n # already satisfies the constraint (and the method returns False)\n # or it raises an exception\n if self.concrete:\n if self._satisfies(other, resolve_virtuals=resolve_virtuals):\n return False\n else:\n raise spack.error.UnsatisfiableSpecError(self, other, \"constrain a concrete spec\")\n\n other = self._autospec(other)\n if other.concrete and other._satisfies(self, resolve_virtuals=resolve_virtuals):\n self._dup(other)\n return True\n\n if other.abstract_hash:\n if not self.abstract_hash or other.abstract_hash.startswith(self.abstract_hash):\n self.abstract_hash = other.abstract_hash\n elif not self.abstract_hash.startswith(other.abstract_hash):\n raise InvalidHashError(self, other.abstract_hash)\n\n if not (self.name == other.name or (not self.name) or (not other.name)):\n raise UnsatisfiableSpecNameError(self.name, other.name)\n\n if (\n other.namespace is not None\n and self.namespace is not None\n and other.namespace != self.namespace\n ):\n raise UnsatisfiableSpecNameError(self.fullname, other.fullname)\n\n if not self.versions.overlaps(other.versions):\n raise UnsatisfiableVersionSpecError(self.versions, other.versions)\n\n for v in [x for x in other.variants if x in self.variants]:\n if not self.variants[v].intersects(other.variants[v]):\n raise vt.UnsatisfiableVariantSpecError(self.variants[v], other.variants[v])\n\n sarch, oarch = self.architecture, other.architecture\n if (\n sarch is not None\n and oarch is not None\n and not self.architecture.intersects(other.architecture)\n ):\n raise UnsatisfiableArchitectureSpecError(sarch, oarch)\n\n changed = False\n\n if not self.name and other.name:\n self.name = other.name\n changed = True\n\n if not self.namespace and other.namespace:\n self.namespace = other.namespace\n changed = True\n\n changed |= self.versions.intersect(other.versions)\n changed |= self._constrain_variants(other)\n\n changed |= self.compiler_flags.constrain(other.compiler_flags)\n\n sarch, oarch = self.architecture, other.architecture\n if sarch is not None and oarch is not None:\n changed |= self.architecture.constrain(other.architecture)\n elif oarch is not None:\n self.architecture = oarch\n changed = True\n\n if deps:\n changed |= self._constrain_dependencies(other, resolve_virtuals=resolve_virtuals)\n\n if other.concrete and not self.concrete and other.satisfies(self):\n self._finalize_concretization()\n\n return changed\n\n def _constrain_dependencies(self, other: \"Spec\", resolve_virtuals: bool = True) -> bool:\n \"\"\"Apply constraints of other spec's dependencies to this spec.\"\"\"\n if not other._dependencies:\n return False\n\n # TODO: might want more detail than this, e.g. specific deps\n # in violation. if this becomes a priority get rid of this\n # check and be more specific about what's wrong.\n if not other._intersects_dependencies(self, resolve_virtuals=resolve_virtuals):\n raise UnsatisfiableDependencySpecError(other, self)\n\n for d in other.traverse(root=False):\n if not d.name:\n raise UnconstrainableDependencySpecError(other)\n changed = False\n for other_edge in other.edges_to_dependencies():\n # Find the first edge in self that matches other_edge by name and when clause.\n for self_edge in self.edges_to_dependencies(other_edge.spec.name):\n if self_edge.when == other_edge.when:\n changed |= self_edge._constrain(other_edge)\n break\n else:\n # Otherwise, a copy of the edge is added as a constraint to self.\n changed = True\n self.add_dependency_edge(\n other_edge.spec.copy(deps=True),\n depflag=other_edge.depflag,\n virtuals=other_edge.virtuals,\n direct=other_edge.direct,\n propagation=other_edge.propagation,\n when=other_edge.when, # no need to copy; when conditions are immutable\n )\n return changed\n\n def constrained(self, other, deps=True):\n \"\"\"Return a constrained copy without modifying this spec.\"\"\"\n clone = self.copy(deps=deps)\n clone.constrain(other, deps)\n return clone\n\n def _autospec(self, spec_like):\n \"\"\"\n Used to convert arguments to specs. If spec_like is a spec, returns\n it. If it's a string, tries to parse a string. If that fails, tries\n to parse a local spec from it (i.e. name is assumed to be self's name).\n \"\"\"\n if isinstance(spec_like, Spec):\n return spec_like\n return Spec(spec_like)\n\n def intersects(self, other: Union[str, \"Spec\"], deps: bool = True) -> bool:\n \"\"\"Return True if there exists at least one concrete spec that matches both\n self and other, otherwise False.\n\n This operation is commutative, and if two specs intersect it means that one\n can constrain the other.\n\n Args:\n other: spec to be checked for compatibility\n deps: if True check compatibility of dependency nodes too, if False only check root\n \"\"\"\n return self._intersects(other=other, deps=deps, resolve_virtuals=True)\n\n def _intersects(\n self, other: Union[str, \"Spec\"], deps: bool = True, resolve_virtuals: bool = True\n ) -> bool:\n if other is EMPTY_SPEC:\n return True\n other = self._autospec(other)\n\n if other.concrete and self.concrete:\n return self.dag_hash() == other.dag_hash()\n\n elif self.concrete:\n return self._satisfies(other, resolve_virtuals=resolve_virtuals)\n\n elif other.concrete:\n return other._satisfies(self, resolve_virtuals=resolve_virtuals)\n\n # From here we know both self and other are not concrete\n self_hash = self.abstract_hash\n other_hash = other.abstract_hash\n\n if (\n self_hash\n and other_hash\n and not (self_hash.startswith(other_hash) or other_hash.startswith(self_hash))\n ):\n return False\n\n # If the names are different, we need to consider virtuals\n if self.name != other.name and self.name and other.name:\n if not resolve_virtuals:\n return False\n\n self_virtual = spack.repo.PATH.is_virtual(self.name)\n other_virtual = spack.repo.PATH.is_virtual(other.name)\n if self_virtual and other_virtual:\n # Two virtual specs intersect only if there are providers for both\n lhs = spack.repo.PATH.providers_for(str(self))\n rhs = spack.repo.PATH.providers_for(str(other))\n intersection = [s for s in lhs if any(s.intersects(z) for z in rhs)]\n return bool(intersection)\n\n # A provider can satisfy a virtual dependency.\n elif self_virtual or other_virtual:\n virtual_spec, non_virtual_spec = (self, other) if self_virtual else (other, self)\n try:\n # Here we might get an abstract spec\n pkg_cls = spack.repo.PATH.get_pkg_class(non_virtual_spec.fullname)\n pkg = pkg_cls(non_virtual_spec)\n except spack.repo.UnknownEntityError:\n # If we can't get package info on this spec, don't treat\n # it as a provider of this vdep.\n return False\n\n if pkg.provides(virtual_spec.name):\n for when_spec, provided in pkg.provided.items():\n if non_virtual_spec.intersects(when_spec, deps=False):\n if any(vpkg.intersects(virtual_spec) for vpkg in provided):\n return True\n return False\n\n # namespaces either match, or other doesn't require one.\n if (\n other.namespace is not None\n and self.namespace is not None\n and self.namespace != other.namespace\n ):\n return False\n\n if self.versions and other.versions:\n if not self.versions.intersects(other.versions):\n return False\n\n if not self._intersects_variants(other):\n return False\n\n if self.architecture and other.architecture:\n if not self.architecture.intersects(other.architecture):\n return False\n\n if not self.compiler_flags.intersects(other.compiler_flags):\n return False\n\n # If we need to descend into dependencies, do it, otherwise we're done.\n if deps:\n return self._intersects_dependencies(other, resolve_virtuals=resolve_virtuals)\n\n return True\n\n def _intersects_dependencies(self, other, resolve_virtuals: bool = True):\n if not other._dependencies or not self._dependencies:\n # one spec *could* eventually satisfy the other\n return True\n\n # Handle first-order constraints directly\n common_dependencies = {x.name for x in self.dependencies()}\n common_dependencies &= {x.name for x in other.dependencies()}\n for name in common_dependencies:\n if not self[name]._intersects(\n other[name], deps=True, resolve_virtuals=resolve_virtuals\n ):\n return False\n\n if not resolve_virtuals:\n return True\n\n # For virtual dependencies, we need to dig a little deeper.\n self_index = spack.provider_index.ProviderIndex(\n repository=spack.repo.PATH, specs=self.traverse(), restrict=True\n )\n other_index = spack.provider_index.ProviderIndex(\n repository=spack.repo.PATH, specs=other.traverse(), restrict=True\n )\n\n # These two loops handle cases where there is an overly restrictive\n # vpkg in one spec for a provider in the other (e.g., mpi@3: is not\n # compatible with mpich2)\n for spec in self.traverse():\n if (\n spack.repo.PATH.is_virtual(spec.name)\n and spec.name in other_index\n and not other_index.providers_for(spec)\n ):\n return False\n\n for spec in other.traverse():\n if (\n spack.repo.PATH.is_virtual(spec.name)\n and spec.name in self_index\n and not self_index.providers_for(spec)\n ):\n return False\n\n return True\n\n def satisfies(self, other: Union[str, \"Spec\"], deps: bool = True) -> bool:\n \"\"\"Return True if all concrete specs matching self also match other, otherwise False.\n\n Args:\n other: spec to be satisfied\n deps: if True, descend to dependencies, otherwise only check root node\n \"\"\"\n return self._satisfies(other=other, deps=deps, resolve_virtuals=True)\n\n def _provides_virtual(self, virtual_spec: \"Spec\") -> bool:\n \"\"\"Return True if this spec provides the given virtual spec.\n\n Args:\n virtual_spec: abstract virtual spec (e.g. ``\"mpi\"`` or ``\"mpi@3:\"``)\n \"\"\"\n if not virtual_spec.name:\n return False\n\n # Get the package instance\n if self.concrete:\n try:\n pkg = self.package\n except spack.repo.UnknownPackageError:\n return False\n else:\n try:\n pkg_cls = spack.repo.PATH.get_pkg_class(self.fullname)\n pkg = pkg_cls(self)\n except spack.repo.UnknownEntityError:\n # If we can't get package info on this spec, don't treat\n # it as a provider of this vdep.\n return False\n\n for when_spec, provided in pkg.provided.items():\n # Don't use satisfies for virtuals, because an abstract vs. abstract spec may use the\n # repo index\n if self.satisfies(when_spec, deps=False) and any(\n provided_virtual.name == virtual_spec.name\n and provided_virtual.versions.intersects(virtual_spec.versions)\n for provided_virtual in provided\n ):\n return True\n\n return False\n\n def _satisfies(\n self, other: Union[str, \"Spec\"], deps: bool = True, resolve_virtuals: bool = True\n ) -> bool:\n \"\"\"Return True if all concrete specs matching self also match other, otherwise False.\n\n Args:\n other: spec to be satisfied\n deps: if True, descend to dependencies, otherwise only check root node\n resolve_virtuals: if True, resolve virtuals in self and other. This requires a\n repository to be available.\n \"\"\"\n if other is EMPTY_SPEC:\n return True\n\n other = self._autospec(other)\n\n if not self._satisfies_node(other, resolve_virtuals=resolve_virtuals):\n return False\n\n # If there are no dependencies on the rhs, or we don't recurse, they are satisfied.\n if not deps or not other._dependencies:\n return True\n\n stack = [(self, other)]\n\n while stack:\n lhs, rhs = stack.pop()\n\n for rhs_edge in rhs.edges_to_dependencies():\n # Skip rhs edges whose when condition doesn't apply to the lhs node.\n if rhs_edge.when is not EMPTY_SPEC and not lhs._intersects(\n rhs_edge.when, resolve_virtuals=resolve_virtuals\n ):\n continue\n\n lhs_edge = _get_satisfying_edge(lhs, rhs_edge, resolve_virtuals=resolve_virtuals)\n\n if not lhs_edge:\n return False\n\n # Recursive case: `^zlib %gcc`\n if not rhs_edge.spec.concrete and rhs_edge.spec._dependencies:\n stack.append((lhs_edge.spec, rhs_edge.spec))\n\n return True\n\n def _satisfies_node(self, other: \"Spec\", resolve_virtuals: bool) -> bool:\n \"\"\"Compares self and other without looking at dependencies\"\"\"\n if other.concrete:\n # The left-hand side must be the same singleton with identical hash. Notice that\n # package hashes can be different for otherwise indistinguishable concrete Spec\n # objects.\n return self.concrete and self.dag_hash() == other.dag_hash()\n\n if self.name != other.name and self.name and other.name:\n # Name mismatch can still be satisfiable if lhs provides the virtual mentioned by rhs.\n if not resolve_virtuals:\n return False\n return self._provides_virtual(other)\n\n # If the right-hand side has an abstract hash, make sure it's a prefix of the\n # left-hand side's (abstract) hash.\n if other.abstract_hash:\n compare_hash = self.dag_hash() if self.concrete else self.abstract_hash\n if not compare_hash or not compare_hash.startswith(other.abstract_hash):\n return False\n\n # namespaces either match, or other doesn't require one.\n if (\n other.namespace is not None\n and self.namespace is not None\n and self.namespace != other.namespace\n ):\n return False\n\n if not self.versions.satisfies(other.versions):\n return False\n\n if not self._satisfies_variants(other):\n return False\n\n if self.architecture and other.architecture:\n if not self.architecture.satisfies(other.architecture):\n return False\n elif other.architecture and not self.architecture:\n return False\n\n if not self.compiler_flags.satisfies(other.compiler_flags):\n return False\n\n return True\n\n def _satisfies_variants(self, other: \"Spec\") -> bool:\n if self.concrete:\n return self._satisfies_variants_when_self_concrete(other)\n return self._satisfies_variants_when_self_abstract(other)\n\n def _satisfies_variants_when_self_concrete(self, other: \"Spec\") -> bool:\n non_propagating, propagating = other.variants.partition_variants()\n result = all(\n name in self.variants and self.variants[name].satisfies(other.variants[name])\n for name in non_propagating\n )\n if not propagating:\n return result\n\n for node in self.traverse():\n if not all(\n node.variants[name].satisfies(other.variants[name])\n for name in propagating\n if name in node.variants\n ):\n return False\n return result\n\n def _satisfies_variants_when_self_abstract(self, other: \"Spec\") -> bool:\n other_non_propagating, other_propagating = other.variants.partition_variants()\n self_non_propagating, self_propagating = self.variants.partition_variants()\n\n # First check variants without propagation set\n result = all(\n name in self_non_propagating\n and (\n self.variants[name].propagate\n or self.variants[name].satisfies(other.variants[name])\n )\n for name in other_non_propagating\n )\n if result is False or (not other_propagating and not self_propagating):\n return result\n\n # Check that self doesn't contradict variants propagated by other\n if other_propagating:\n for node in self.traverse():\n if not all(\n node.variants[name].satisfies(other.variants[name])\n for name in other_propagating\n if name in node.variants\n ):\n return False\n\n # Check that other doesn't contradict variants propagated by self\n if self_propagating:\n for node in other.traverse():\n if not all(\n node.variants[name].satisfies(self.variants[name])\n for name in self_propagating\n if name in node.variants\n ):\n return False\n\n return result\n\n def _intersects_variants(self, other: \"Spec\") -> bool:\n self_dict = self.variants.dict\n other_dict = other.variants.dict\n return all(self_dict[k].intersects(other_dict[k]) for k in other_dict if k in self_dict)\n\n def _constrain_variants(self, other: \"Spec\") -> bool:\n \"\"\"Add all variants in other that aren't in self to self. Also constrain all multi-valued\n variants that are already present. Return True iff self changed\"\"\"\n if other is not None and other._concrete:\n for k in self.variants:\n if k not in other.variants:\n raise vt.UnsatisfiableVariantSpecError(self.variants[k], \"\")\n\n changed = False\n for k in other.variants:\n if k in self.variants:\n if not self.variants[k].intersects(other.variants[k]):\n raise vt.UnsatisfiableVariantSpecError(self.variants[k], other.variants[k])\n # If they are compatible merge them\n changed |= self.variants[k].constrain(other.variants[k])\n else:\n # If it is not present copy it straight away\n self.variants[k] = other.variants[k].copy()\n changed = True\n\n return changed\n\n @property # type: ignore[misc] # decorated prop not supported in mypy\n def patches(self):\n \"\"\"Return patch objects for any patch sha256 sums on this Spec.\n\n This is for use after concretization to iterate over any patches\n associated with this spec.\n\n TODO: this only checks in the package; it doesn't resurrect old\n patches from install directories, but it probably should.\n \"\"\"\n if not hasattr(self, \"_patches\"):\n self._patches = []\n\n # translate patch sha256sums to patch objects by consulting the index\n if self._patches_assigned():\n sha256s = list(self.variants[\"patches\"]._patches_in_order_of_appearance)\n pkg_cls = spack.repo.PATH.get_pkg_class(self.name)\n try:\n self._patches = spack.repo.PATH.get_patches_for_package(sha256s, pkg_cls)\n except spack.error.PatchLookupError as e:\n raise spack.error.SpecError(\n f\"{e}. This usually means the patch was modified or removed. \"\n \"To fix this, either reconcretize or use the original package \"\n \"repository\"\n ) from e\n\n return self._patches\n\n def _dup(\n self,\n other: \"Spec\",\n deps: Union[bool, dt.DepTypes, dt.DepFlag] = True,\n *,\n propagation: Optional[PropagationPolicy] = None,\n ) -> bool:\n \"\"\"Copies \"other\" into self, by overwriting all attributes.\n\n Args:\n other: spec to be copied onto ``self``\n deps: if True copies all the dependencies. If False copies None.\n If deptype, or depflag, copy matching types.\n\n Returns:\n True if ``self`` changed because of the copy operation, False otherwise.\n \"\"\"\n # We don't count dependencies as changes here\n changed = True\n if hasattr(self, \"name\"):\n changed = (\n self.name != other.name\n and self.versions != other.versions\n and self.architecture != other.architecture\n and self.variants != other.variants\n and self.concrete != other.concrete\n and self.external_path != other.external_path\n and self.external_modules != other.external_modules\n and self.compiler_flags != other.compiler_flags\n and self.abstract_hash != other.abstract_hash\n )\n\n self._package = None\n\n # Local node attributes get copied first.\n self.name = other.name\n self.versions = other.versions.copy()\n self.architecture = other.architecture.copy() if other.architecture else None\n self.compiler_flags = other.compiler_flags.copy()\n self.compiler_flags.spec = self\n self.variants = other.variants.copy()\n self._build_spec = other._build_spec\n\n # Clear dependencies\n self._dependents = {}\n self._dependencies = {}\n\n # FIXME: we manage _patches_in_order_of_appearance specially here\n # to keep it from leaking out of spec.py, but we should figure\n # out how to handle it more elegantly in the Variant classes.\n for k, v in other.variants.items():\n patches = getattr(v, \"_patches_in_order_of_appearance\", None)\n if patches:\n self.variants[k]._patches_in_order_of_appearance = patches\n\n self.variants.spec = self\n self.external_path = other.external_path\n self.external_modules = other.external_modules\n self.extra_attributes = other.extra_attributes\n self.namespace = other.namespace\n self.annotations = other.annotations\n\n # If we copy dependencies, preserve DAG structure in the new spec\n if deps:\n # If caller restricted deptypes to be copied, adjust that here.\n # By default, just copy all deptypes\n depflag = dt.ALL\n if isinstance(deps, (tuple, list, str)):\n depflag = dt.canonicalize(deps)\n self._dup_deps(other, depflag, propagation=propagation)\n\n self._prefix = other._prefix\n self._concrete = other._concrete\n\n self.abstract_hash = other.abstract_hash\n\n if self._concrete:\n self._dunder_hash = other._dunder_hash\n for h in ht.HASHES:\n setattr(self, h.attr, getattr(other, h.attr, None))\n else:\n self._dunder_hash = None\n for h in ht.HASHES:\n setattr(self, h.attr, None)\n\n return changed\n\n def _dup_deps(\n self, other, depflag: dt.DepFlag, propagation: Optional[PropagationPolicy] = None\n ):\n def spid(spec):\n return id(spec)\n\n new_specs = {spid(other): self}\n for edge in other.traverse_edges(cover=\"edges\", root=False):\n if edge.depflag and not depflag & edge.depflag:\n continue\n\n if spid(edge.parent) not in new_specs:\n new_specs[spid(edge.parent)] = edge.parent.copy(deps=False)\n\n if spid(edge.spec) not in new_specs:\n new_specs[spid(edge.spec)] = edge.spec.copy(deps=False)\n\n edge_propagation = edge.propagation if propagation is None else propagation\n new_specs[spid(edge.parent)].add_dependency_edge(\n new_specs[spid(edge.spec)],\n depflag=edge.depflag,\n virtuals=edge.virtuals,\n propagation=edge_propagation,\n direct=edge.direct,\n when=edge.when,\n )\n\n def copy(self, deps: Union[bool, dt.DepTypes, dt.DepFlag] = True, **kwargs):\n \"\"\"Make a copy of this spec.\n\n Args:\n deps: Defaults to :data:`True`. If boolean, controls\n whether dependencies are copied (copied if :data:`True`). If a\n DepTypes or DepFlag is provided, *only* matching dependencies are copied.\n kwargs: additional arguments for internal use (passed to ``_dup``).\n\n Returns:\n A copy of this spec.\n\n Examples:\n Deep copy with dependencies::\n\n spec.copy()\n spec.copy(deps=True)\n\n Shallow copy (no dependencies)::\n\n spec.copy(deps=False)\n\n Only build and run dependencies::\n\n deps=(\"build\", \"run\"):\n\n \"\"\"\n clone = Spec.__new__(Spec)\n clone._dup(self, deps=deps, **kwargs)\n return clone\n\n @property\n def version(self):\n if not self.versions.concrete:\n raise spack.error.SpecError(\"Spec version is not concrete: \" + str(self))\n return self.versions[0]\n\n def __getitem__(self, name: str):\n \"\"\"Get a dependency from the spec by its name. This call implicitly\n sets a query state in the package being retrieved. The behavior of\n packages may be influenced by additional query parameters that are\n passed after a colon symbol.\n\n Note that if a virtual package is queried a copy of the Spec is\n returned while for non-virtual a reference is returned.\n \"\"\"\n query_parameters: List[str] = name.split(\":\")\n if len(query_parameters) > 2:\n raise KeyError(\"key has more than one ':' symbol. At most one is admitted.\")\n\n name, query_parameters = query_parameters[0], query_parameters[1:]\n if query_parameters:\n # We have extra query parameters, which are comma separated\n # values\n csv = query_parameters.pop().strip()\n query_parameters = re.split(r\"\\s*,\\s*\", csv)\n\n # Consider all direct dependencies and transitive runtime dependencies\n order = itertools.chain(\n self.edges_to_dependencies(depflag=dt.BUILD | dt.TEST),\n self.traverse_edges(deptype=dt.LINK | dt.RUN, order=\"breadth\", cover=\"edges\"),\n )\n\n try:\n edge = next((e for e in order if e.spec.name == name or name in e.virtuals))\n except StopIteration as e:\n raise KeyError(f\"No spec with name {name} in {self}\") from e\n\n if self._concrete:\n return SpecBuildInterface(\n edge.spec, name, query_parameters, _parent=self, is_virtual=name in edge.virtuals\n )\n\n return edge.spec\n\n def __contains__(self, spec):\n \"\"\"True if this spec or some dependency satisfies the spec.\n\n Note: If ``spec`` is anonymous, we ONLY check whether the root\n satisfies it, NOT dependencies. This is because most anonymous\n specs (e.g., ``@1.2``) don't make sense when applied across an\n entire DAG -- we limit them to the root.\n\n \"\"\"\n spec = self._autospec(spec)\n\n # if anonymous or same name, we only have to look at the root\n if not spec.name or spec.name == self.name:\n return self.satisfies(spec)\n try:\n dep = self[spec.name]\n except KeyError:\n return False\n return dep.satisfies(spec)\n\n def eq_dag(self, other, deptypes=True, vs=None, vo=None):\n \"\"\"True if the full dependency DAGs of specs are equal.\"\"\"\n if vs is None:\n vs = set()\n if vo is None:\n vo = set()\n\n vs.add(id(self))\n vo.add(id(other))\n\n if not self.eq_node(other):\n return False\n\n if len(self._dependencies) != len(other._dependencies):\n return False\n\n ssorted = [self._dependencies[name] for name in sorted(self._dependencies)]\n osorted = [other._dependencies[name] for name in sorted(other._dependencies)]\n for s_dspec, o_dspec in zip(\n itertools.chain.from_iterable(ssorted), itertools.chain.from_iterable(osorted)\n ):\n if deptypes and s_dspec.depflag != o_dspec.depflag:\n return False\n\n s, o = s_dspec.spec, o_dspec.spec\n visited_s = id(s) in vs\n visited_o = id(o) in vo\n\n # Check for duplicate or non-equal dependencies\n if visited_s != visited_o:\n return False\n\n # Skip visited nodes\n if visited_s or visited_o:\n continue\n\n # Recursive check for equality\n if not s.eq_dag(o, deptypes, vs, vo):\n return False\n\n return True\n\n def _cmp_node(self):\n \"\"\"Yield comparable elements of just *this node* and not its deps.\"\"\"\n yield self.name\n yield self.namespace\n yield self.versions\n yield self.variants\n yield self.compiler_flags\n yield self.architecture\n yield self.abstract_hash\n\n # this is not present on older specs\n yield getattr(self, \"_package_hash\", None)\n\n def eq_node(self, other):\n \"\"\"Equality with another spec, not including dependencies.\"\"\"\n return (other is not None) and lang.lazy_eq(self._cmp_node, other._cmp_node)\n\n def _cmp_fast_eq(self, other) -> Optional[bool]:\n \"\"\"Short-circuit compare with other for equality, for lazy_lexicographic_ordering.\"\"\"\n # If there is ever a breaking change to hash computation, whether accidental or purposeful,\n # two specs can be identical modulo DAG hash, depending on what time they were concretized\n # From the perspective of many operation in Spack (database, build cache, etc) a different\n # DAG hash means a different spec. Here we ensure that two otherwise identical specs, one\n # serialized before the hash change and one after, are considered different.\n if self is other:\n return True\n\n if self.concrete and other and other.concrete:\n return self.dag_hash() == other.dag_hash()\n\n return None\n\n def _cmp_iter(self):\n \"\"\"Lazily yield components of self for comparison.\"\"\"\n\n # Spec comparison in Spack needs to be fast, so there are several cases here for\n # performance. The main places we care about this are:\n #\n # * Abstract specs: there are lots of abstract specs in package.py files,\n # which are put into metadata dictionaries and sorted during concretization\n # setup. We want comparing abstract specs to be fast.\n #\n # * Concrete specs: concrete specs are bigger and have lots of nodes and\n # edges. Because of the graph complexity, we need a full, linear time\n # traversal to compare them -- that's pretty much is unavoidable. But they\n # also have precoputed cryptographic hashes (dag_hash()), which we can use\n # to do fast equality comparison. See _cmp_fast_eq() above for the\n # short-circuit logic for hashes.\n #\n # A full traversal involves constructing data structures, visitor objects, etc.,\n # and it can be expensive if we have to do it to compare a bunch of tiny\n # abstract specs. Therefore, there are 3 cases below, which avoid calling\n # `spack.traverse.traverse_edges()` unless necessary.\n #\n # WARNING: the cases below need to be consistent, so don't mess with this code\n # unless you really know what you're doing. Be sure to keep all three consistent.\n #\n # All cases lazily yield:\n #\n # 1. A generator over nodes\n # 2. A generator over canonical edges\n #\n # Canonical edges have consistent ids defined by breadth-first traversal order. That is,\n # the root is always 0, dependencies of the root are 1, 2, 3, etc., and so on.\n #\n # The three cases are:\n #\n # 1. Spec has no dependencies\n # * We can avoid any traversal logic and just yield this node's _cmp_node generator.\n #\n # 2. Spec has dependencies, but dependencies have no dependencies.\n # * We need to sort edges, but we don't need to track visited nodes, which\n # can save us the cost of setting up all the tracking data structures\n # `spack.traverse` uses.\n #\n # 3. Spec has dependencies that have dependencies.\n # * In this case, the spec is *probably* concrete. Equality comparisons\n # will be short-circuited by dag_hash(), but other comparisons will need\n # to lazily enumerate components of the spec. The traversal logic is\n # unavoidable.\n #\n # TODO: consider reworking `spack.traverse` to construct fewer data structures\n # and objects, as this would make all traversals faster and could eliminate the\n # need for the complexity here. It was not clear at the time of writing that how\n # much optimization was possible in `spack.traverse`.\n\n sorted_l1_edges = None\n edge_list = None\n node_ids = None\n\n def nodes():\n nonlocal sorted_l1_edges\n nonlocal edge_list\n nonlocal node_ids\n\n # Level 0: root node\n yield self._cmp_node # always yield the root (this node)\n if not self._dependencies: # done if there are no dependencies\n return\n\n # Level 1: direct dependencies\n # we can yield these in sorted order without tracking visited nodes\n deps_have_deps = False\n sorted_l1_edges = self.edges_to_dependencies(depflag=dt.ALL)\n if len(sorted_l1_edges) > 1:\n sorted_l1_edges = spack.traverse.sort_edges(sorted_l1_edges)\n\n for edge in sorted_l1_edges:\n yield edge.spec._cmp_node\n if edge.spec._dependencies:\n deps_have_deps = True\n\n if not deps_have_deps: # done if level 1 specs have no dependencies\n return\n\n # Level 2: dependencies of direct dependencies\n # now it's general; we need full traverse() to track visited nodes\n l1_specs = [edge.spec for edge in sorted_l1_edges]\n\n # the node_ids dict generates consistent ids based on BFS traversal order\n # these are used to identify edges later\n node_ids = collections.defaultdict(lambda: len(node_ids))\n node_ids[id(self)] # self is 0\n for spec in l1_specs:\n node_ids[id(spec)] # l1 starts at 1\n\n edge_list = []\n for edge in spack.traverse.traverse_edges(\n l1_specs, order=\"breadth\", cover=\"edges\", root=False, visited=set([0])\n ):\n # yield each node only once, and generate a consistent id for it the\n # first time it's encountered.\n if id(edge.spec) not in node_ids:\n yield edge.spec._cmp_node\n node_ids[id(edge.spec)]\n\n if edge.parent is None: # skip fake edge to root\n continue\n\n edge_list.append(\n (\n node_ids[id(edge.parent)],\n node_ids[id(edge.spec)],\n edge.depflag,\n edge.virtuals,\n edge.direct,\n edge.when,\n )\n )\n\n def edges():\n # no edges in single-node graph\n if not self._dependencies:\n return\n\n # level 1 edges all start with zero\n for i, edge in enumerate(sorted_l1_edges, start=1):\n yield (0, i, edge.depflag, edge.virtuals, edge.direct, edge.when)\n\n # yield remaining edges in the order they were encountered during traversal\n if edge_list:\n yield from edge_list\n\n yield nodes\n yield edges\n\n @property\n def namespace_if_anonymous(self):\n return self.namespace if not self.name else None\n\n @property\n def spack_root(self):\n \"\"\"Special field for using ``{spack_root}`` in :meth:`format`.\"\"\"\n return spack.paths.spack_root\n\n @property\n def spack_install(self):\n \"\"\"Special field for using ``{spack_install}`` in :meth:`format`.\"\"\"\n from spack.store import STORE\n\n return STORE.layout.root\n\n def _format_default(self) -> str:\n \"\"\"Fast path for formatting with DEFAULT_FORMAT and no color.\n\n This method manually concatenates the string representation of spec attributes,\n avoiding the regex parsing overhead of the general format() method.\n \"\"\"\n parts = []\n\n if self.name:\n parts.append(self.name)\n\n if self.versions:\n version_str = str(self.versions)\n if version_str and version_str != \":\": # only include if not full range\n parts.append(f\"@{version_str}\")\n\n compiler_flags_str = str(self.compiler_flags)\n if compiler_flags_str:\n parts.append(compiler_flags_str)\n\n variants_str = str(self.variants)\n if variants_str:\n parts.append(variants_str)\n\n if not self.name and self.namespace:\n parts.append(f\" namespace={self.namespace}\")\n\n if self.architecture:\n if self.architecture.platform:\n parts.append(f\" platform={self.architecture.platform}\")\n if self.architecture.os:\n parts.append(f\" os={self.architecture.os}\")\n if self.architecture.target:\n parts.append(f\" target={self.architecture.target}\")\n\n if self.abstract_hash:\n parts.append(f\"/{self.abstract_hash}\")\n\n return \"\".join(parts).strip()\n\n def format(\n self,\n format_string: str = DEFAULT_FORMAT,\n color: Optional[bool] = False,\n *,\n highlight_version_fn: Optional[Callable[[\"Spec\"], bool]] = None,\n highlight_variant_fn: Optional[Callable[[\"Spec\", str], bool]] = None,\n ) -> str:\n r\"\"\"Prints out attributes of a spec according to a format string.\n\n Using an ``{attribute}`` format specifier, any field of the spec can be\n selected. Those attributes can be recursive. For example,\n ``s.format({compiler.version})`` will print the version of the compiler.\n\n If the attribute in a format specifier evaluates to ``None``, then the format\n specifier will evaluate to the empty string, ``\"\"``.\n\n Commonly used attributes of the Spec for format strings include:\n\n .. code-block:: text\n\n name\n version\n compiler_flags\n compilers\n variants\n architecture\n architecture.platform\n architecture.os\n architecture.target\n prefix\n namespace\n\n Some additional special-case properties can be added:\n\n .. code-block:: text\n\n hash[:len] The DAG hash with optional length argument\n spack_root The spack root directory\n spack_install The spack install directory\n\n The ``^`` sigil can be used to access dependencies by name.\n ``s.format({^mpi.name})`` will print the name of the MPI implementation in the\n spec.\n\n The ``@``, ``%``, and ``/`` sigils can be used to include the sigil with the\n printed string. These sigils may only be used with the appropriate attributes,\n listed below:\n\n * ``@``: ``{@version}``, ``{@compiler.version}``\n * ``%``: ``{%compiler}``, ``{%compiler.name}``\n * ``/``: ``{/hash}``, ``{/hash:7}``, etc\n\n The ``@`` sigil may also be used for any other property named ``version``.\n Sigils printed with the attribute string are only printed if the attribute\n string is non-empty, and are colored according to the color of the attribute.\n\n Variants listed by name naturally print with their sigil. For example,\n ``spec.format(\"{variants.debug}\")`` prints either ``+debug`` or ``~debug``\n depending on the name of the variant. Non-boolean variants print as\n ``name=value``. To print variant names or values independently, use\n ``spec.format(\"{variants..name}\")`` or\n ``spec.format(\"{variants..value}\")``.\n\n There are a few attributes on specs that can be specified as key-value pairs\n that are *not* variants, e.g.: ``os``, ``arch``, ``architecture``, ``target``,\n ``namespace``, etc. You can format these with an optional ``key=`` prefix, e.g.\n ``{namespace=namespace}`` or ``{arch=architecture}``, etc. The ``key=`` prefix\n will be colorized along with the value.\n\n When formatting specs, key-value pairs are separated from preceding parts of the\n spec by whitespace. To avoid printing extra whitespace when the formatted\n attribute is not set, you can add whitespace to the key *inside* the braces of\n the format string, e.g.:\n\n .. code-block:: text\n\n { namespace=namespace}\n\n This evaluates to ``\" namespace=builtin\"`` if ``namespace`` is set to ``builtin``,\n and to ``\"\"`` if ``namespace`` is ``None``.\n\n Spec format strings use ``\\`` as the escape character. Use ``\\{`` and ``\\}`` for\n literal braces, and ``\\\\`` for the literal ``\\`` character.\n\n Args:\n format_string: string containing the format to be expanded\n color: True for colorized result; False for no color; None for auto color.\n highlight_version_fn: optional callable that returns true on nodes where the version\n needs to be highlighted\n highlight_variant_fn: optional callable that returns true on variants that need\n to be highlighted\n \"\"\"\n # Fast path for the common case: default format with no color\n if format_string == DEFAULT_FORMAT and color is False:\n return self._format_default()\n\n ensure_modern_format_string(format_string)\n\n def safe_color(sigil: str, string: str, color_fmt: Optional[str]) -> str:\n # avoid colorizing if there is no color or the string is empty\n if (color is False) or not color_fmt or not string:\n return sigil + string\n # escape and add the sigil here to avoid multiple concatenations\n if sigil == \"@\":\n sigil = \"@@\"\n return clr.colorize(f\"{color_fmt}{sigil}{clr.cescape(string)}@.\", color=color)\n\n def format_attribute(match_object: Match) -> str:\n (esc, sig, dep, hash, hash_len, attribute, close_brace, unmatched_close_brace) = (\n match_object.groups()\n )\n if esc:\n return esc\n elif unmatched_close_brace:\n raise SpecFormatStringError(f\"Unmatched close brace: '{format_string}'\")\n elif not close_brace:\n raise SpecFormatStringError(f\"Missing close brace: '{format_string}'\")\n\n current_node = self if dep is None else self[dep]\n current = current_node\n\n # Hash attributes can return early.\n # NOTE: we currently treat abstract_hash like an attribute and ignore\n # any length associated with it. We may want to change that.\n if hash:\n if sig and sig != \"/\":\n raise SpecFormatSigilError(sig, \"DAG hashes\", hash)\n try:\n length = int(hash_len) if hash_len else None\n except ValueError:\n raise SpecFormatStringError(f\"Invalid hash length: '{hash_len}'\")\n return safe_color(sig or \"\", current.dag_hash(length), HASH_COLOR)\n\n if attribute == \"\":\n raise SpecFormatStringError(\"Format string attributes must be non-empty\")\n\n attribute = attribute.lower()\n parts = attribute.split(\".\")\n assert parts\n\n # check that the sigil is valid for the attribute.\n if not sig:\n sig = \"\"\n elif sig == \"@\" and parts[-1] not in (\"versions\", \"version\"):\n raise SpecFormatSigilError(sig, \"versions\", attribute)\n elif sig == \"%\" and attribute not in (\"compiler\", \"compiler.name\"):\n raise SpecFormatSigilError(sig, \"compilers\", attribute)\n elif sig == \"/\" and attribute != \"abstract_hash\":\n raise SpecFormatSigilError(sig, \"DAG hashes\", attribute)\n\n # Iterate over components using getattr to get next element\n for idx, part in enumerate(parts):\n if not part:\n raise SpecFormatStringError(\"Format string attributes must be non-empty\")\n elif part.startswith(\"_\"):\n raise SpecFormatStringError(\"Attempted to format private attribute\")\n elif isinstance(current, VariantMap):\n # subscript instead of getattr for variant names\n try:\n current = current[part]\n except KeyError:\n raise SpecFormatStringError(f\"Variant '{part}' does not exist\")\n else:\n # aliases\n if part == \"arch\":\n part = \"architecture\"\n elif part == \"version\" and not current.versions.concrete:\n # version (singular) requires a concrete versions list. Avoid\n # pedantic errors by using versions (plural) when not concrete.\n # These two are not entirely equivalent for pkg@=1.2.3:\n # - version prints '1.2.3'\n # - versions prints '=1.2.3'\n part = \"versions\"\n try:\n current = getattr(current, part)\n except AttributeError:\n if part == \"compiler\":\n return \"none\"\n elif part == \"specfile_version\":\n return f\"v{current.original_spec_format()}\"\n\n raise SpecFormatStringError(\n f\"Attempted to format attribute {attribute}. \"\n f\"Spec {'.'.join(parts[:idx])} has no attribute {part}\"\n )\n if isinstance(current, vn.VersionList) and current == vn.any_version:\n # don't print empty version lists\n return \"\"\n\n if callable(current):\n raise SpecFormatStringError(\"Attempted to format callable object\")\n\n if current is None:\n # not printing anything\n return \"\"\n\n # Set color codes for various attributes\n color = None\n if \"architecture\" in parts:\n color = ARCHITECTURE_COLOR\n elif \"variants\" in parts or sig.endswith(\"=\"):\n color = VARIANT_COLOR\n elif any(c in parts for c in (\"compiler\", \"compilers\", \"compiler_flags\")):\n color = COMPILER_COLOR\n elif \"version\" in parts or \"versions\" in parts:\n color = VERSION_COLOR\n if highlight_version_fn and highlight_version_fn(current_node):\n color = HIGHLIGHT_COLOR\n\n # return empty string if the value of the attribute is None.\n if current is None:\n return \"\"\n\n # Override the color for single variants, if need be\n if color and highlight_variant_fn and isinstance(current, VariantMap):\n bool_keys, kv_keys = current.partition_keys()\n result = \"\"\n\n for key in bool_keys:\n current_color = color\n if highlight_variant_fn(current_node, key):\n current_color = HIGHLIGHT_COLOR\n\n result += safe_color(sig, str(current[key]), current_color)\n\n for key in kv_keys:\n current_color = color\n if highlight_variant_fn(current_node, key):\n current_color = HIGHLIGHT_COLOR\n\n # Don't highlight the space before the key/value pair\n result += \" \" + safe_color(sig, f\"{current[key]}\", current_color)\n\n return result\n\n # return colored output\n return safe_color(sig, str(current), color)\n\n return SPEC_FORMAT_RE.sub(format_attribute, format_string).strip()\n\n def cformat(self, format_string: str = DEFAULT_FORMAT) -> str:\n \"\"\"Same as :meth:`format`, but color defaults to auto instead of False.\"\"\"\n return self.format(format_string, color=None)\n\n def format_path(\n # self, format_string: str, _path_ctor: Optional[pathlib.PurePath] = None\n self,\n format_string: str,\n _path_ctor: Optional[Callable[[Any], pathlib.PurePath]] = None,\n ) -> str:\n \"\"\"Given a ``format_string`` that is intended as a path, generate a string like from\n :meth:`format`, but eliminate extra path separators introduced by formatting of Spec\n properties.\n\n Path separators explicitly added to the string are preserved, so for example\n ``{name}/{version}`` would generate a directory based on the Spec's name, and a\n subdirectory based on its version; this function guarantees though that the resulting\n string would only have two directories (i.e. that if under normal circumstances that\n ``str(self.version)`` would contain a path separator, it would not in this case).\n \"\"\"\n format_component_with_sep = r\"\\{[^}]*[/\\\\][^}]*}\"\n if re.search(format_component_with_sep, format_string):\n raise SpecFormatPathError(\n f\"Invalid path format string: cannot contain {{/...}}\\n\\t{format_string}\"\n )\n\n path_ctor = _path_ctor or pathlib.PurePath\n format_string_as_path = path_ctor(format_string)\n if format_string_as_path.is_absolute() or (\n # Paths that begin with a single \"\\\" on windows are relative, but we still\n # want to preserve the initial \"\\\\\" to be consistent with PureWindowsPath.\n # Ensure that this '\\' is not passed to polite_filename() so it's not converted to '_'\n (os.name == \"nt\" or path_ctor == pathlib.PureWindowsPath)\n and format_string_as_path.parts[0] == \"\\\\\"\n ):\n output_path_components = [format_string_as_path.parts[0]]\n input_path_components = list(format_string_as_path.parts[1:])\n else:\n output_path_components = []\n input_path_components = list(format_string_as_path.parts)\n\n output_path_components += [\n fs.polite_filename(self.format(part)) for part in input_path_components\n ]\n return str(path_ctor(*output_path_components))\n\n def _format_edge_attributes(self, dep: DependencySpec, deptypes=True, virtuals=True):\n deptypes_str = (\n f\"deptypes={','.join(dt.flag_to_tuple(dep.depflag))}\"\n if deptypes and dep.depflag\n else \"\"\n )\n when_str = f\"when='{(dep.when)}'\" if dep.when != EMPTY_SPEC else \"\"\n virtuals_str = f\"virtuals={','.join(dep.virtuals)}\" if virtuals and dep.virtuals else \"\"\n\n attrs = \" \".join(s for s in (when_str, deptypes_str, virtuals_str) if s)\n if attrs:\n attrs = f\"[{attrs}] \"\n\n return attrs\n\n def _format_dependencies(\n self,\n format_string: str = DEFAULT_FORMAT,\n include: Optional[Callable[[DependencySpec], bool]] = None,\n deptypes: bool = True,\n color: Optional[bool] = False,\n _force_direct: bool = False,\n ):\n \"\"\"Helper for formatting dependencies on specs.\n\n Arguments:\n format_string: format string to use for each dependency\n include: predicate to select which dependencies to include\n deptypes: whether to format deptypes\n color: colorize if True, don't colorize if False, auto-colorize if None\n _force_direct: if True, print all dependencies as direct dependencies\n (to be removed when we have this metadata on concrete edges)\n \"\"\"\n include = include or (lambda dep: True)\n parts = []\n if self.concrete:\n direct = self.edges_to_dependencies()\n transitive: List[DependencySpec] = []\n else:\n direct, transitive = lang.stable_partition(\n self.edges_to_dependencies(), predicate_fn=lambda x: x.direct\n )\n\n # helper for direct and transitive loops below\n def format_edge(edge: DependencySpec, sigil: str, dep_spec: Optional[Spec] = None) -> str:\n dep_spec = dep_spec or edge.spec\n dep_format = dep_spec.format(format_string, color=color)\n\n edge_attributes = (\n self._format_edge_attributes(edge, deptypes=deptypes, virtuals=False)\n if edge.depflag or edge.when != EMPTY_SPEC\n else \"\"\n )\n virtuals = f\"{','.join(edge.virtuals)}=\" if edge.virtuals else \"\"\n star = _anonymous_star(edge, dep_format)\n\n return f\"{sigil}{edge_attributes}{star}{virtuals}{dep_format}\"\n\n # direct dependencies\n for edge in sorted(direct, key=lambda x: x.spec.name):\n if not include(edge):\n continue\n\n # replace legacy compiler names\n old_name = edge.spec.name\n new_name = spack.aliases.BUILTIN_TO_LEGACY_COMPILER.get(old_name)\n try:\n # this is ugly but copies can be expensive\n sigil = \"%\"\n if new_name:\n edge.spec.name = new_name\n\n if edge.propagation == PropagationPolicy.PREFERENCE:\n sigil = \"%%\"\n\n parts.append(format_edge(edge, sigil=sigil, dep_spec=edge.spec))\n finally:\n edge.spec.name = old_name\n\n if self.concrete:\n # Concrete specs should go no further, as the complexity\n # below is O(paths)\n return \" \".join(parts).strip()\n\n # transitive dependencies (with any direct dependencies)\n for edge in sorted(transitive, key=lambda x: x.spec.name):\n if not include(edge):\n continue\n sigil = \"%\" if _force_direct else \"^\" # hack til direct deps represented better\n parts.append(format_edge(edge, sigil, edge.spec))\n\n # also recursively add any direct dependencies of transitive dependencies\n if edge.spec._dependencies:\n parts.append(\n edge.spec._format_dependencies(\n format_string=format_string,\n include=include,\n deptypes=deptypes,\n _force_direct=_force_direct,\n )\n )\n\n return \" \".join(parts).strip()\n\n def _long_spec(self, color: Optional[bool] = False) -> str:\n \"\"\"Helper for :attr:`long_spec` and :attr:`clong_spec`.\"\"\"\n if self.concrete:\n return self.tree(format=DISPLAY_FORMAT, color=color)\n return f\"{self.format(color=color)} {self._format_dependencies(color=color)}\".strip()\n\n def _short_spec(self, color: Optional[bool] = False) -> str:\n \"\"\"Helper for :attr:`short_spec` and :attr:`cshort_spec`.\"\"\"\n return self.format(\n \"{name}{@version}{variants}\"\n \"{ platform=architecture.platform}{ os=architecture.os}{ target=architecture.target}\"\n \"{/hash:7}\",\n color=color,\n )\n\n @property\n def compilers(self):\n if self.original_spec_format() < 5:\n # These specs don't have compilers as dependencies, return the compiler node attribute\n return f\" %{self.compiler}\"\n\n # TODO: get rid of the space here and make formatting smarter\n return \" \" + self._format_dependencies(\n \"{name}{@version}\",\n include=lambda dep: any(lang in dep.virtuals for lang in (\"c\", \"cxx\", \"fortran\")),\n deptypes=False,\n _force_direct=True,\n )\n\n @property\n def long_spec(self):\n \"\"\"Long string of the spec, including dependencies.\"\"\"\n return self._long_spec(color=False)\n\n @property\n def clong_spec(self):\n \"\"\"Returns an auto-colorized version of :attr:`long_spec`.\"\"\"\n return self._long_spec(color=None)\n\n @property\n def short_spec(self):\n \"\"\"Short string of the spec, with hash and without dependencies.\"\"\"\n return self._short_spec(color=False)\n\n @property\n def cshort_spec(self):\n \"\"\"Returns an auto-colorized version of :attr:`short_spec`.\"\"\"\n return self._short_spec(color=None)\n\n @property\n def colored_str(self) -> str:\n \"\"\"Auto-colorized string representation of this spec.\"\"\"\n return self._str(color=None)\n\n def _str(self, color: Optional[bool] = False) -> str:\n \"\"\"String representation of this spec.\n Args:\n color: colorize if True, don't colorize if False, auto-colorize if None\n \"\"\"\n if self._concrete:\n return self.format(\"{name}{@version}{/hash}\", color=color)\n\n if not self._dependencies:\n return self.format(color=color)\n\n return self._long_spec(color=color)\n\n def __str__(self) -> str:\n \"\"\"String representation of this spec.\"\"\"\n return self._str(color=False)\n\n def install_status(self) -> InstallStatus:\n \"\"\"Helper for tree to print DB install status.\"\"\"\n if not self.concrete:\n return InstallStatus.absent\n\n if self.external:\n return InstallStatus.external\n\n from spack.store import STORE\n\n upstream, record = STORE.db.query_by_spec_hash(self.dag_hash())\n if not record:\n return InstallStatus.absent\n elif upstream and record.installed:\n return InstallStatus.upstream\n elif record.installed:\n return InstallStatus.installed\n else:\n return InstallStatus.missing\n\n def _installed_explicitly(self):\n \"\"\"Helper for tree to print DB install status.\"\"\"\n if not self.concrete:\n return None\n try:\n from spack.store import STORE\n\n record = STORE.db.get_record(self)\n return record.explicit\n except KeyError:\n return None\n\n def tree(\n self,\n *,\n color: Optional[bool] = None,\n depth: bool = False,\n hashes: bool = False,\n hashlen: Optional[int] = None,\n cover: spack.traverse.CoverType = \"nodes\",\n indent: int = 0,\n format: str = DEFAULT_FORMAT,\n deptypes: Union[dt.DepTypes, dt.DepFlag] = dt.ALL,\n show_types: bool = False,\n depth_first: bool = False,\n recurse_dependencies: bool = True,\n status_fn: Optional[Callable[[\"Spec\"], InstallStatus]] = None,\n prefix: Optional[Callable[[\"Spec\"], str]] = None,\n key=id,\n highlight_version_fn: Optional[Callable[[\"Spec\"], bool]] = None,\n highlight_variant_fn: Optional[Callable[[\"Spec\", str], bool]] = None,\n ) -> str:\n \"\"\"Prints out this spec and its dependencies, tree-formatted with indentation.\n\n See multi-spec ``spack.spec.tree()`` function for details.\n\n Args:\n specs: List of specs to format.\n color: if True, always colorize the tree. If False, don't colorize the tree. If None,\n use the default from spack.llnl.tty.color\n depth: print the depth from the root\n hashes: if True, print the hash of each node\n hashlen: length of the hash to be printed\n cover: either ``\"nodes\"`` or ``\"edges\"``\n indent: extra indentation for the tree being printed\n format: format to be used to print each node\n deptypes: dependency types to be represented in the tree\n show_types: if True, show the (merged) dependency type of a node\n depth_first: if True, traverse the DAG depth first when representing it as a tree\n recurse_dependencies: if True, recurse on dependencies\n status_fn: optional callable that takes a node as an argument and return its\n installation status\n prefix: optional callable that takes a node as an argument and return its\n installation prefix\n highlight_version_fn: optional callable that returns true on nodes where the version\n needs to be highlighted\n highlight_variant_fn: optional callable that returns true on variants that need\n to be highlighted\n \"\"\"\n return tree(\n [self],\n color=color,\n depth=depth,\n hashes=hashes,\n hashlen=hashlen,\n cover=cover,\n indent=indent,\n format=format,\n deptypes=deptypes,\n show_types=show_types,\n depth_first=depth_first,\n recurse_dependencies=recurse_dependencies,\n status_fn=status_fn,\n prefix=prefix,\n key=key,\n highlight_version_fn=highlight_version_fn,\n highlight_variant_fn=highlight_variant_fn,\n )\n\n def __repr__(self):\n return str(self)\n\n @property\n def platform(self):\n return self.architecture.platform\n\n @property\n def os(self):\n return self.architecture.os\n\n @property\n def target(self):\n return self.architecture.target\n\n @property\n def build_spec(self):\n return self._build_spec or self\n\n @build_spec.setter\n def build_spec(self, value):\n self._build_spec = value\n\n def trim(self, dep_name):\n \"\"\"\n Remove any package that is or provides ``dep_name`` transitively\n from this tree. This can also remove other dependencies if\n they are only present because of ``dep_name``.\n \"\"\"\n for spec in list(self.traverse()):\n new_dependencies = {}\n for pkg_name, edge_list in spec._dependencies.items():\n for edge in edge_list:\n if (dep_name not in edge.virtuals) and (not dep_name == edge.spec.name):\n _add_edge_to_map(new_dependencies, edge.spec.name, edge)\n spec._dependencies = new_dependencies\n\n def _virtuals_provided(self, root):\n \"\"\"Return set of virtuals provided by self in the context of root\"\"\"\n if root is self:\n # Could be using any virtual the package can provide\n return set(v.name for v in self.package.virtuals_provided)\n\n hashes = [s.dag_hash() for s in root.traverse()]\n in_edges = set(\n [edge for edge in self.edges_from_dependents() if edge.parent.dag_hash() in hashes]\n )\n return set().union(*[edge.virtuals for edge in in_edges])\n\n def _splice_match(self, other, self_root, other_root):\n \"\"\"Return True if other is a match for self in a splice of other_root into self_root\n\n Other is a splice match for self if it shares a name, or if self is a virtual provider\n and other provides a superset of the virtuals provided by self. Virtuals provided are\n evaluated in the context of a root spec (self_root for self, other_root for other).\n\n This is a slight oversimplification. Other could be a match for self in the context of\n one edge in self_root and not in the context of another edge. This method could be\n expanded in the future to account for these cases.\n \"\"\"\n if other.name == self.name:\n return True\n\n return bool(\n bool(self._virtuals_provided(self_root))\n and self._virtuals_provided(self_root) <= other._virtuals_provided(other_root)\n )\n\n def _splice_detach_and_add_dependents(self, replacement, context):\n \"\"\"Helper method for Spec._splice_helper.\n\n replacement is a node to splice in, context is the scope of dependents to consider relevant\n to this splice.\"\"\"\n # Update build_spec attributes for all transitive dependents\n # before we start changing their dependencies\n ancestors_in_context = [\n a\n for a in self.traverse(root=False, direction=\"parents\")\n if a in context.traverse(deptype=dt.LINK | dt.RUN)\n ]\n for ancestor in ancestors_in_context:\n # Only set it if it hasn't been spliced before\n ancestor._build_spec = ancestor._build_spec or ancestor.copy()\n ancestor.clear_caches(ignore=(ht.package_hash.attr,))\n for edge in ancestor.edges_to_dependencies(depflag=dt.BUILD):\n if edge.depflag & ~dt.BUILD:\n edge.depflag &= ~dt.BUILD\n else:\n ancestor._dependencies[edge.spec.name].remove(edge)\n edge.spec._dependents[ancestor.name].remove(edge)\n\n # For each direct dependent in the link/run graph, replace the dependency on\n # node with one on replacement\n for edge in self.edges_from_dependents():\n if edge.parent not in ancestors_in_context:\n continue\n\n edge.parent._dependencies[self.name].remove(edge)\n self._dependents[edge.parent.name].remove(edge)\n edge.parent._add_dependency(replacement, depflag=edge.depflag, virtuals=edge.virtuals)\n\n def _splice_helper(self, replacement):\n \"\"\"Main loop of a transitive splice.\n\n The while loop around a traversal of self ensures that changes to self from previous\n iterations are reflected in the traversal. This avoids evaluating irrelevant nodes\n using topological traversal (all incoming edges traversed before any outgoing edge).\n If any node will not be in the end result, its parent will be spliced and it will not\n ever be considered.\n For each node in self, find any analogous node in replacement and swap it in.\n We assume all build deps are handled outside of this method\n\n Arguments:\n replacement: The node that will replace any equivalent node in self\n self_root: The root of the spec that self comes from. This provides the context for\n evaluating whether ``replacement`` is a match for each node of ``self``. See\n ``Spec._splice_match`` and ``Spec._virtuals_provided`` for details.\n other_root: The root of the spec that replacement comes from. This provides the context\n for evaluating whether ``replacement`` is a match for each node of ``self``. See\n ``Spec._splice_match`` and ``Spec._virtuals_provided`` for details.\n \"\"\"\n ids = set(id(s) for s in replacement.traverse())\n\n # Sort all possible replacements by name and virtual for easy access later\n replacements_by_name = collections.defaultdict(list)\n for node in replacement.traverse():\n replacements_by_name[node.name].append(node)\n virtuals = node._virtuals_provided(root=replacement)\n for virtual in virtuals:\n replacements_by_name[virtual].append(node)\n\n changed = True\n while changed:\n changed = False\n\n # Intentionally allowing traversal to change on each iteration\n # using breadth-first traversal to ensure we only reach nodes that will\n # be in final result\n for node in self.traverse(root=False, order=\"topo\", deptype=dt.ALL & ~dt.BUILD):\n # If this node has already been swapped in, don't consider it again\n if id(node) in ids:\n continue\n\n analogs = replacements_by_name[node.name]\n if not analogs:\n # If we have to check for matching virtuals, then we need to check that it\n # matches all virtuals. Use `_splice_match` to validate possible matches\n for virtual in node._virtuals_provided(root=self):\n analogs += [\n r\n for r in replacements_by_name[virtual]\n if node._splice_match(r, self_root=self, other_root=replacement)\n ]\n\n # No match, keep iterating over self\n if not analogs:\n continue\n\n # If there are multiple analogs, this package must satisfy the constraint\n # that a newer version can always replace a lesser version.\n analog = max(analogs, key=lambda s: s.version)\n\n # No splice needed here, keep checking\n if analog == node:\n continue\n\n node._splice_detach_and_add_dependents(analog, context=self)\n changed = True\n break\n\n def splice(self, other: \"Spec\", transitive: bool = True) -> \"Spec\":\n \"\"\"Returns a new, spliced concrete :class:`Spec` with the ``other`` dependency and,\n optionally, its dependencies.\n\n Args:\n other: alternate dependency\n transitive: include other's dependencies\n\n Returns: a concrete, spliced version of the current :class:`Spec`\n\n When transitive is :data:`True`, use the dependencies from ``other`` to reconcile\n conflicting dependencies. When transitive is :data:`False`, use dependencies from self.\n\n For example, suppose we have the following dependency graph:\n\n .. code-block:: text\n\n T\n | \\\\\n Z<-H\n\n Spec ``T`` depends on ``H`` and ``Z``, and ``H`` also depends on ``Z``. Now we want to use\n a different ``H``, called ``H'``. This function can be used to splice in ``H'`` to\n create a new spec, called ``T*``. If ``H'`` was built with ``Z'``, then ``transitive=True``\n will ensure ``H'`` and ``T*`` both depend on ``Z'``:\n\n .. code-block:: text\n\n T*\n | \\\\\n Z'<-H'\n\n If ``transitive=False``, then ``H'`` and ``T*`` will both depend on\n the original ``Z``, resulting in a new ``H'*``:\n\n .. code-block:: text\n\n T*\n | \\\\\n Z<-H'*\n\n Provenance of the build is tracked through the :attr:`build_spec` property\n of the spliced spec and any correspondingly modified dependency specs.\n The build specs are set to that of the original spec, so the original\n spec's provenance is preserved unchanged.\"\"\"\n assert self.concrete\n assert other.concrete\n\n if self._splice_match(other, self_root=self, other_root=other):\n return other.copy()\n\n if not any(\n node._splice_match(other, self_root=self, other_root=other)\n for node in self.traverse(root=False, deptype=dt.LINK | dt.RUN)\n ):\n other_str = other.format(\"{name}/{hash:7}\")\n self_str = self.format(\"{name}/{hash:7}\")\n msg = f\"Cannot splice {other_str} into {self_str}.\"\n msg += f\" Either {self_str} cannot depend on {other_str},\"\n msg += f\" or {other_str} fails to provide a virtual used in {self_str}\"\n raise SpliceError(msg)\n\n # Copies of all non-build deps, build deps will get added at the end\n spec = self.copy(deps=dt.ALL & ~dt.BUILD)\n replacement = other.copy(deps=dt.ALL & ~dt.BUILD)\n\n def make_node_pairs(orig_spec, copied_spec):\n return list(\n zip(\n orig_spec.traverse(deptype=dt.ALL & ~dt.BUILD),\n copied_spec.traverse(deptype=dt.ALL & ~dt.BUILD),\n )\n )\n\n def mask_build_deps(in_spec):\n for edge in in_spec.traverse_edges(cover=\"edges\"):\n edge.depflag &= ~dt.BUILD\n\n if transitive:\n # These pairs will allow us to reattach all direct build deps\n # We need the list of pairs while the two specs still match\n node_pairs = make_node_pairs(self, spec)\n\n # Ignore build deps in the modified spec while doing the splice\n # They will be added back in at the end\n mask_build_deps(spec)\n\n # Transitively splice any relevant nodes from new into base\n # This handles all shared dependencies between self and other\n spec._splice_helper(replacement)\n else:\n # Do the same thing as the transitive splice, but reversed\n node_pairs = make_node_pairs(other, replacement)\n mask_build_deps(replacement)\n replacement._splice_helper(spec)\n\n # Intransitively splice replacement into spec\n # This is very simple now that all shared dependencies have been handled\n for node in spec.traverse(order=\"topo\", deptype=dt.LINK | dt.RUN):\n if node._splice_match(other, self_root=spec, other_root=other):\n node._splice_detach_and_add_dependents(replacement, context=spec)\n\n # For nodes that were spliced, modify the build spec to ensure build deps are preserved\n # For nodes that were not spliced, replace the build deps on the spec itself\n for orig, copy in node_pairs:\n if copy._build_spec:\n copy._build_spec = orig.build_spec.copy()\n else:\n for edge in orig.edges_to_dependencies(depflag=dt.BUILD):\n copy._add_dependency(edge.spec, depflag=dt.BUILD, virtuals=edge.virtuals)\n\n return spec\n\n def mutate(self, mutator, rehash=True) -> bool:\n \"\"\"Mutate concrete spec to match constraints represented by mutator.\n\n Mutation can modify the spec version, variants, compiler flags, and architecture.\n Mutation cannot change the spec name, namespace, dependencies, or abstract_hash.\n Any attribute which is unset will not be touched.\n Variant values can be replaced with the literal ``None`` to remove the variant.\n ``None`` as a variant value is represented by ``VariantValue(..., (None,))``.\n\n If ``rehash``, concrete spec and its dependents have hashes updated.\n\n Returns whether the spec was modified by the mutation\"\"\"\n assert self.concrete\n\n if mutator.name and mutator.name != self.name:\n raise SpecMutationError(f\"Cannot mutate spec name: spec {self} mutator {mutator}\")\n\n if mutator.namespace and mutator.namespace != self.namespace:\n raise SpecMutationError(f\"Cannot mutate spec namespace: spec {self} mutator {mutator}\")\n\n if len(mutator.dependencies()) > 0:\n raise SpecMutationError(f\"Cannot mutate dependencies: spec {self} mutator {mutator}\")\n\n if (\n mutator.versions != vn.VersionList(\":\")\n and not mutator.versions.concrete_range_as_version\n ):\n raise SpecMutationError(\n f\"Cannot mutate abstract version: spec {self} mutator {mutator}\"\n )\n\n if mutator.abstract_hash and mutator.abstract_hash != self.abstract_hash:\n raise SpecMutationError(f\"Cannot mutate abstract_hash: spec {self} mutator {mutator}\")\n\n changed = False\n\n if mutator.versions != vn.VersionList(\":\") and self.versions != mutator.versions:\n self.versions = mutator.versions\n changed = True\n\n for name, variant in mutator.variants.items():\n if variant == self.variants.get(name, None):\n continue\n\n old_variant = self.variants.pop(name, None)\n if not isinstance(variant, vt.VariantValueRemoval): # sigil type for removing variant\n if old_variant:\n variant.type = old_variant.type # coerce variant type to match\n self.variants[name] = variant\n changed = True\n\n for name, flags in mutator.compiler_flags.items():\n if not flags or flags == self.compiler_flags[name]:\n continue\n self.compiler_flags[name] = flags\n changed = True\n\n if mutator.architecture:\n if mutator.platform and mutator.platform != self.architecture.platform:\n self.architecture.platform = mutator.platform\n changed = True\n if mutator.os and mutator.os != self.architecture.os:\n self.architecture.os = mutator.os\n changed = True\n if mutator.target and mutator.target != self.architecture.target:\n self.architecture.target = mutator.target\n changed = True\n\n if changed and rehash:\n roots = []\n for parent in spack.traverse.traverse_nodes([self], direction=\"parents\"):\n if not parent.dependents():\n roots.append(parent)\n # invalidate hashes\n parent._mark_root_concrete(False)\n parent.clear_caches()\n\n for root in roots:\n # compute new hashes on full DAGs\n root._finalize_concretization()\n\n return changed\n\n def clear_caches(self, ignore: Tuple[str, ...] = ()) -> None:\n \"\"\"\n Clears all cached hashes in a Spec, while preserving other properties.\n \"\"\"\n for h in ht.HASHES:\n if h.attr not in ignore:\n if hasattr(self, h.attr):\n setattr(self, h.attr, None)\n for attr in (\"_dunder_hash\", \"_prefix\"):\n if attr not in ignore:\n setattr(self, attr, None)\n\n def __hash__(self):\n # If the spec is concrete, we leverage the dag hash and just use a 64-bit prefix of it.\n # The dag hash has the advantage that it's computed once per concrete spec, and it's saved\n # -- so if we read concrete specs we don't need to recompute the whole hash.\n if self.concrete:\n if not self._dunder_hash:\n self._dunder_hash = self.dag_hash_bit_prefix(64)\n return self._dunder_hash\n\n if not self._dependencies:\n return hash(\n (\n self.name,\n self.namespace,\n self.versions,\n (self.variants if self.variants.dict else None),\n self.architecture,\n self.abstract_hash,\n )\n )\n\n return hash(lang.tuplify(self._cmp_iter))\n\n def __getstate__(self):\n state = self.__dict__.copy()\n # The package is lazily loaded upon demand.\n state.pop(\"_package\", None)\n # As with to_dict, do not include dependents. This avoids serializing more than intended.\n state.pop(\"_dependents\", None)\n\n # Do not pickle attributes dynamically set by SpecBuildInterface\n state.pop(\"wrapped_obj\", None)\n state.pop(\"token\", None)\n state.pop(\"last_query\", None)\n state.pop(\"indirect_spec\", None)\n\n # Optimize variants and compiler_flags serialization\n variants = state.pop(\"variants\", None)\n if variants:\n state[\"_variants_data\"] = variants.dict\n flags = state.pop(\"compiler_flags\", None)\n if flags:\n state[\"_compiler_flags_data\"] = flags.dict\n\n return state\n\n def __setstate__(self, state):\n variants_data = state.pop(\"_variants_data\", None)\n compiler_flags_data = state.pop(\"_compiler_flags_data\", None)\n self.__dict__.update(state)\n self._package = None\n\n # Reconstruct variants and compiler_flags\n self.variants = VariantMap()\n self.compiler_flags = FlagMap()\n if variants_data is not None:\n self.variants.dict = variants_data\n if compiler_flags_data is not None:\n self.compiler_flags.dict = compiler_flags_data\n\n # Reconstruct dependents map\n if not hasattr(self, \"_dependents\"):\n self._dependents = {}\n\n for edges in self._dependencies.values():\n for edge in edges:\n if not hasattr(edge.spec, \"_dependents\"):\n edge.spec._dependents = {}\n _add_edge_to_map(edge.spec._dependents, edge.parent.name, edge)\n\n def attach_git_version_lookup(self):\n # Add a git lookup method for GitVersions\n if not self.name:\n return\n for v in self.versions:\n if isinstance(v, vn.GitVersion) and v.std_version is None:\n v.attach_lookup(spack.version.git_ref_lookup.GitRefLookup(self.fullname))\n\n def original_spec_format(self) -> int:\n \"\"\"Returns the spec format originally used for this spec.\"\"\"\n return self.annotations.original_spec_format\n\n def has_virtual_dependency(self, virtual: str) -> bool:\n return bool(self.dependencies(virtuals=(virtual,)))\n\n\nclass VariantMap(lang.HashableMap[str, vt.VariantValue]):\n \"\"\"Map containing variant instances. New values can be added only\n if the key is not already present.\"\"\"\n\n def __setitem__(self, name, vspec):\n # Raise a TypeError if vspec is not of the right type\n if not isinstance(vspec, vt.VariantValue):\n raise TypeError(\n \"VariantMap accepts only values of variant types \"\n f\"[got {type(vspec).__name__} instead]\"\n )\n\n # Raise an error if the variant was already in this map\n if name in self.dict:\n msg = 'Cannot specify variant \"{0}\" twice'.format(name)\n raise vt.DuplicateVariantError(msg)\n\n # Raise an error if name and vspec.name don't match\n if name != vspec.name:\n raise KeyError(\n f'Inconsistent key \"{name}\", must be \"{vspec.name}\" to match VariantSpec'\n )\n\n # Set the item\n super().__setitem__(name, vspec)\n\n def substitute(self, vspec):\n \"\"\"Substitutes the entry under ``vspec.name`` with ``vspec``.\n\n Args:\n vspec: variant spec to be substituted\n \"\"\"\n if vspec.name not in self:\n raise KeyError(f\"cannot substitute a key that does not exist [{vspec.name}]\")\n\n # Set the item\n super().__setitem__(vspec.name, vspec)\n\n def partition_variants(self):\n non_prop, prop = lang.stable_partition(self.values(), lambda x: not x.propagate)\n # Just return the names\n non_prop = [x.name for x in non_prop]\n prop = [x.name for x in prop]\n return non_prop, prop\n\n def copy(self) -> \"VariantMap\":\n clone = VariantMap()\n for name, variant in self.items():\n clone[name] = variant.copy()\n return clone\n\n def __str__(self):\n if not self:\n return \"\"\n\n # Separate boolean variants from key-value pairs as they print\n # differently. All booleans go first to avoid ' ~foo' strings that\n # break spec reuse in zsh.\n bool_keys, kv_keys = self.partition_keys()\n\n # add spaces before and after key/value variants.\n string = io.StringIO()\n\n for key in bool_keys:\n string.write(str(self[key]))\n\n for key in kv_keys:\n string.write(\" \")\n string.write(str(self[key]))\n\n return string.getvalue()\n\n def partition_keys(self) -> Tuple[List[str], List[str]]:\n \"\"\"Partition the keys of the map into two lists: booleans and key-value pairs.\"\"\"\n bool_keys, kv_keys = lang.stable_partition(\n sorted(self.keys()), lambda x: self[x].type == vt.VariantType.BOOL\n )\n return bool_keys, kv_keys\n\n\nclass SpecBuildInterface(lang.ObjectWrapper, Spec):\n # home is available in the base Package so no default is needed\n home = ForwardQueryToPackage(\"home\", default_handler=None)\n headers = ForwardQueryToPackage(\"headers\", default_handler=_headers_default_handler)\n libs = ForwardQueryToPackage(\"libs\", default_handler=_libs_default_handler)\n command = ForwardQueryToPackage(\"command\", default_handler=None, _indirect=True)\n\n def __init__(\n self,\n spec: \"Spec\",\n name: str,\n query_parameters: List[str],\n _parent: \"Spec\",\n is_virtual: bool,\n ):\n lang.ObjectWrapper.__init__(self, spec)\n # Adding new attributes goes after ObjectWrapper.__init__ call since the ObjectWrapper\n # resets __dict__ to behave like the passed object\n original_spec = getattr(spec, \"wrapped_obj\", spec)\n self.wrapped_obj = original_spec\n self.token = original_spec, name, query_parameters, _parent, is_virtual\n self.last_query = QueryState(\n name=name, extra_parameters=query_parameters, isvirtual=is_virtual\n )\n\n # TODO: this ad-hoc logic makes `spec[\"python\"].command` return\n # `spec[\"python-venv\"].command` and should be removed when `python` is a virtual.\n self.indirect_spec = None\n if spec.name == \"python\":\n python_venvs = _parent.dependencies(\"python-venv\")\n if not python_venvs:\n return\n self.indirect_spec = python_venvs[0]\n\n def __reduce__(self):\n return SpecBuildInterface, self.token\n\n def copy(self, *args, **kwargs):\n return self.wrapped_obj.copy(*args, **kwargs)\n\n\ndef substitute_abstract_variants(spec: Spec):\n \"\"\"Uses the information in ``spec.package`` to turn any variant that needs\n it into a SingleValuedVariant or BoolValuedVariant.\n\n This method is best effort. All variants that can be substituted will be\n substituted before any error is raised.\n\n Args:\n spec: spec on which to operate the substitution\n \"\"\"\n # This method needs to be best effort so that it works in matrix exclusion\n # in $spack/lib/spack/spack/spec_list.py\n unknown = []\n for name, v in spec.variants.items():\n if v.concrete and v.type == vt.VariantType.MULTI:\n continue\n\n if name in (\"dev_path\", \"commit\"):\n v.type = vt.VariantType.SINGLE\n v.concrete = True\n continue\n elif name in vt.RESERVED_NAMES:\n continue\n\n variant_defs = spack.repo.PATH.get_pkg_class(spec.fullname).variant_definitions(name)\n valid_defs = []\n for when, vdef in variant_defs:\n if when.intersects(spec):\n valid_defs.append(vdef)\n\n if not valid_defs:\n if name not in spack.repo.PATH.get_pkg_class(spec.fullname).variant_names():\n unknown.append(name)\n else:\n whens = [str(when) for when, _ in variant_defs]\n raise InvalidVariantForSpecError(v.name, f\"({', '.join(whens)})\", spec)\n continue\n\n pkg_variant, *rest = valid_defs\n if rest:\n continue\n\n new_variant = pkg_variant.make_variant(*v.values)\n pkg_variant.validate_or_raise(new_variant, spec.name)\n spec.variants.substitute(new_variant)\n\n if unknown:\n variants = spack.llnl.string.plural(len(unknown), \"variant\")\n raise vt.UnknownVariantError(\n f\"Tried to set {variants} {spack.llnl.string.comma_and(unknown)}. \"\n f\"{spec.name} has no such {variants}\",\n unknown_variants=unknown,\n )\n\n\ndef parse_with_version_concrete(spec_like: Union[str, Spec]):\n \"\"\"Same as Spec(string), but interprets @x as @=x\"\"\"\n s = Spec(spec_like)\n interpreted_version = s.versions.concrete_range_as_version\n if interpreted_version:\n s.versions = vn.VersionList([interpreted_version])\n return s\n\n\ndef reconstruct_virtuals_on_edges(spec: Spec) -> None:\n \"\"\"Reconstruct virtuals on edges. Used to read from old DB and reindex.\"\"\"\n virtuals_needed: Dict[str, Set[str]] = {}\n virtuals_provided: Dict[str, Set[str]] = {}\n for edge in spec.traverse_edges(cover=\"edges\", root=False):\n parent_key = edge.parent.dag_hash()\n if parent_key not in virtuals_needed:\n # Construct which virtuals are needed by parent\n virtuals_needed[parent_key] = set()\n try:\n parent_pkg = edge.parent.package\n except Exception as e:\n warnings.warn(\n f\"cannot reconstruct virtual dependencies on {edge.parent.name}: {e}\"\n )\n continue\n\n virtuals_needed[parent_key].update(\n name\n for name, when_deps in parent_pkg.dependencies_by_name(when=True).items()\n if spack.repo.PATH.is_virtual(name)\n and any(edge.parent.satisfies(x) for x in when_deps)\n )\n\n if not virtuals_needed[parent_key]:\n continue\n\n child_key = edge.spec.dag_hash()\n if child_key not in virtuals_provided:\n virtuals_provided[child_key] = set()\n try:\n child_pkg = edge.spec.package\n except Exception as e:\n warnings.warn(\n f\"cannot reconstruct virtual dependencies on {edge.parent.name}: {e}\"\n )\n continue\n virtuals_provided[child_key].update(x.name for x in child_pkg.virtuals_provided)\n\n if not virtuals_provided[child_key]:\n continue\n\n virtuals_to_add = virtuals_needed[parent_key] & virtuals_provided[child_key]\n if virtuals_to_add:\n edge.update_virtuals(virtuals_to_add)\n\n\nclass SpecfileReaderBase:\n SPEC_VERSION: int\n\n @classmethod\n def from_node_dict(cls, node):\n spec = Spec()\n\n name, node = cls.name_and_data(node)\n for h in ht.HASHES:\n setattr(spec, h.attr, node.get(h.name, None))\n\n # old anonymous spec files had name=None, we use name=\"\" now\n spec.name = name if isinstance(name, str) else \"\"\n spec.namespace = node.get(\"namespace\", None)\n\n if \"version\" in node or \"versions\" in node:\n spec.versions = vn.VersionList.from_dict(node)\n spec.attach_git_version_lookup()\n\n if \"arch\" in node:\n spec.architecture = ArchSpec.from_dict(node)\n\n propagated_names = node.get(\"propagate\", [])\n abstract_variants = set(node.get(\"abstract\", ()))\n for name, values in node.get(\"parameters\", {}).items():\n propagate = name in propagated_names\n if name in _valid_compiler_flags:\n spec.compiler_flags[name] = []\n for val in values:\n spec.compiler_flags.add_flag(name, val, propagate)\n else:\n spec.variants[name] = vt.VariantValue.from_node_dict(\n name, values, propagate=propagate, abstract=name in abstract_variants\n )\n\n spec.external_path = None\n spec.external_modules = None\n if \"external\" in node:\n # This conditional is needed because sometimes this function is\n # called with a node already constructed that contains a 'versions'\n # and 'external' field. Related to virtual packages provider\n # indexes.\n if node[\"external\"]:\n spec.external_path = node[\"external\"][\"path\"]\n spec.external_modules = node[\"external\"][\"module\"]\n if spec.external_modules is False:\n spec.external_modules = None\n spec.extra_attributes = node[\"external\"].get(\"extra_attributes\") or {}\n\n # specs read in are concrete unless marked abstract\n if node.get(\"concrete\", True):\n spec._mark_root_concrete()\n\n if \"patches\" in node:\n patches = node[\"patches\"]\n if len(patches) > 0:\n mvar = spec.variants.setdefault(\"patches\", vt.MultiValuedVariant(\"patches\", ()))\n mvar.set(*patches)\n # FIXME: Monkey patches mvar to store patches order\n mvar._patches_in_order_of_appearance = patches\n\n # Annotate the compiler spec, might be used later\n if \"annotations\" not in node:\n # Specfile v4 and earlier\n spec.annotations.with_spec_format(cls.SPEC_VERSION)\n if \"compiler\" in node:\n spec.annotations.with_compiler(cls.legacy_compiler(node))\n else:\n spec.annotations.with_spec_format(node[\"annotations\"][\"original_specfile_version\"])\n if \"compiler\" in node[\"annotations\"]:\n spec.annotations.with_compiler(Spec(f\"{node['annotations']['compiler']}\"))\n\n # Don't read dependencies here; from_dict() is used by\n # from_yaml() and from_json() to read the root *and* each dependency\n # spec.\n\n return spec\n\n @classmethod\n def legacy_compiler(cls, node):\n d = node[\"compiler\"]\n return Spec(f\"{d['name']}@{vn.VersionList.from_dict(d)}\")\n\n @classmethod\n def _load(cls, data):\n \"\"\"Construct a spec from JSON/YAML using the format version 2.\n\n This format is used in Spack v0.17, was introduced in\n https://github.com/spack/spack/pull/22845\n\n Args:\n data: a nested dict/list data structure read from YAML or JSON.\n \"\"\"\n # Current specfile format\n nodes = data[\"spec\"][\"nodes\"]\n hash_type = None\n any_deps = False\n\n # Pass 0: Determine hash type\n for node in nodes:\n for _, _, _, dhash_type, _, _ in cls.dependencies_from_node_dict(node):\n any_deps = True\n if dhash_type:\n hash_type = dhash_type\n break\n\n if not any_deps: # If we never see a dependency...\n hash_type = ht.dag_hash.name\n elif not hash_type: # Seen a dependency, still don't know hash_type\n raise spack.error.SpecError(\n \"Spec dictionary contains malformed dependencies. Old format?\"\n )\n\n hash_dict = {}\n root_spec_hash = None\n\n # Pass 1: Create a single lookup dictionary by hash\n for i, node in enumerate(nodes):\n node_hash = node[hash_type]\n node_spec = cls.from_node_dict(node)\n hash_dict[node_hash] = node\n hash_dict[node_hash][\"node_spec\"] = node_spec\n if i == 0:\n root_spec_hash = node_hash\n\n if not root_spec_hash:\n raise spack.error.SpecError(\"Spec dictionary contains no nodes.\")\n\n # Pass 2: Finish construction of all DAG edges (including build specs)\n for node_hash, node in hash_dict.items():\n node_spec = node[\"node_spec\"]\n for _, dhash, dtype, _, virtuals, direct in cls.dependencies_from_node_dict(node):\n node_spec._add_dependency(\n hash_dict[dhash][\"node_spec\"],\n depflag=dt.canonicalize(dtype),\n virtuals=virtuals,\n direct=direct,\n )\n if \"build_spec\" in node.keys():\n _, bhash, _ = cls.extract_build_spec_info_from_node_dict(node, hash_type=hash_type)\n node_spec._build_spec = hash_dict[bhash][\"node_spec\"]\n\n return hash_dict[root_spec_hash][\"node_spec\"]\n\n @classmethod\n def extract_build_spec_info_from_node_dict(cls, node, hash_type=ht.dag_hash.name):\n raise NotImplementedError(\"Subclasses must implement this method.\")\n\n @classmethod\n def read_specfile_dep_specs(cls, deps, hash_type=ht.dag_hash.name):\n raise NotImplementedError(\"Subclasses must implement this method.\")\n\n\nclass SpecfileV1(SpecfileReaderBase):\n SPEC_VERSION = 1\n\n @classmethod\n def load(cls, data):\n \"\"\"Construct a spec from JSON/YAML using the format version 1.\n\n Note: Version 1 format has no notion of a build_spec, and names are\n guaranteed to be unique. This function is guaranteed to read specs as\n old as v0.10 - while it was not checked for older formats.\n\n Args:\n data: a nested dict/list data structure read from YAML or JSON.\n \"\"\"\n nodes = data[\"spec\"]\n\n # Read nodes out of list. Root spec is the first element;\n # dependencies are the following elements.\n dep_list = [cls.from_node_dict(node) for node in nodes]\n if not dep_list:\n raise spack.error.SpecError(\"specfile contains no nodes.\")\n\n deps = {spec.name: spec for spec in dep_list}\n result = dep_list[0]\n\n for node in nodes:\n # get dependency dict from the node.\n name, data = cls.name_and_data(node)\n for dname, _, dtypes, _, virtuals, direct in cls.dependencies_from_node_dict(data):\n deps[name]._add_dependency(\n deps[dname], depflag=dt.canonicalize(dtypes), virtuals=virtuals, direct=direct\n )\n\n reconstruct_virtuals_on_edges(result)\n return result\n\n @classmethod\n def name_and_data(cls, node):\n name = next(iter(node))\n node = node[name]\n return name, node\n\n @classmethod\n def dependencies_from_node_dict(cls, node):\n if \"dependencies\" not in node:\n return []\n\n for t in cls.read_specfile_dep_specs(node[\"dependencies\"]):\n yield t\n\n @classmethod\n def read_specfile_dep_specs(cls, deps, hash_type=ht.dag_hash.name):\n \"\"\"Read the DependencySpec portion of a YAML-formatted Spec.\n This needs to be backward-compatible with older spack spec\n formats so that reindex will work on old specs/databases.\n \"\"\"\n for dep_name, elt in deps.items():\n if isinstance(elt, dict):\n for h in ht.HASHES:\n if h.name in elt:\n dep_hash, deptypes = elt[h.name], elt[\"type\"]\n hash_type = h.name\n virtuals = []\n break\n else: # We never determined a hash type...\n raise spack.error.SpecError(\"Couldn't parse dependency spec.\")\n else:\n raise spack.error.SpecError(\"Couldn't parse dependency types in spec.\")\n yield dep_name, dep_hash, list(deptypes), hash_type, list(virtuals), True\n\n\nclass SpecfileV2(SpecfileReaderBase):\n SPEC_VERSION = 2\n\n @classmethod\n def load(cls, data):\n result = cls._load(data)\n reconstruct_virtuals_on_edges(result)\n return result\n\n @classmethod\n def name_and_data(cls, node):\n return node[\"name\"], node\n\n @classmethod\n def dependencies_from_node_dict(cls, node):\n return cls.read_specfile_dep_specs(node.get(\"dependencies\", []))\n\n @classmethod\n def read_specfile_dep_specs(cls, deps, hash_type=ht.dag_hash.name):\n \"\"\"Read the DependencySpec portion of a YAML-formatted Spec.\n This needs to be backward-compatible with older spack spec\n formats so that reindex will work on old specs/databases.\n \"\"\"\n if not isinstance(deps, list):\n raise spack.error.SpecError(\"Spec dictionary contains malformed dependencies\")\n\n result = []\n for dep in deps:\n elt = dep\n dep_name = dep[\"name\"]\n if isinstance(elt, dict):\n # new format: elements of dependency spec are keyed.\n for h in ht.HASHES:\n if h.name in elt:\n dep_hash, deptypes, hash_type, virtuals, direct = (\n cls.extract_info_from_dep(elt, h)\n )\n break\n else: # We never determined a hash type...\n raise spack.error.SpecError(\"Couldn't parse dependency spec.\")\n else:\n raise spack.error.SpecError(\"Couldn't parse dependency types in spec.\")\n result.append((dep_name, dep_hash, list(deptypes), hash_type, list(virtuals), direct))\n return result\n\n @classmethod\n def extract_info_from_dep(cls, elt, hash):\n dep_hash, deptypes = elt[hash.name], elt[\"type\"]\n hash_type = hash.name\n virtuals = []\n direct = True\n return dep_hash, deptypes, hash_type, virtuals, direct\n\n @classmethod\n def extract_build_spec_info_from_node_dict(cls, node, hash_type=ht.dag_hash.name):\n build_spec_dict = node[\"build_spec\"]\n return build_spec_dict[\"name\"], build_spec_dict[hash_type], hash_type\n\n\nclass SpecfileV3(SpecfileV2):\n SPEC_VERSION = 3\n\n\nclass SpecfileV4(SpecfileV2):\n SPEC_VERSION = 4\n\n @classmethod\n def extract_info_from_dep(cls, elt, hash):\n dep_hash = elt[hash.name]\n deptypes = elt[\"parameters\"][\"deptypes\"]\n hash_type = hash.name\n virtuals = elt[\"parameters\"][\"virtuals\"]\n direct = True\n return dep_hash, deptypes, hash_type, virtuals, direct\n\n @classmethod\n def load(cls, data):\n return cls._load(data)\n\n\nclass SpecfileV5(SpecfileV4):\n SPEC_VERSION = 5\n\n @classmethod\n def legacy_compiler(cls, node):\n raise RuntimeError(\"The 'compiler' option is unexpected in specfiles at v5 or greater\")\n\n @classmethod\n def extract_info_from_dep(cls, elt, hash):\n dep_hash = elt[hash.name]\n deptypes = elt[\"parameters\"][\"deptypes\"]\n hash_type = hash.name\n virtuals = elt[\"parameters\"][\"virtuals\"]\n direct = elt[\"parameters\"].get(\"direct\", False)\n return dep_hash, deptypes, hash_type, virtuals, direct\n\n\n#: Alias to the latest version of specfiles\nSpecfileLatest = SpecfileV5\n\n\nclass LazySpecCache(collections.defaultdict):\n \"\"\"Cache for Specs that uses a spec_like as key, and computes lazily\n the corresponding value ``Spec(spec_like``.\n \"\"\"\n\n def __init__(self):\n super().__init__(Spec)\n\n def __missing__(self, key):\n value = self.default_factory(key)\n self[key] = value\n return value\n\n\ndef save_dependency_specfiles(root: Spec, output_directory: str, dependencies: List[Spec]):\n \"\"\"Given a root spec (represented as a yaml object), index it with a subset\n of its dependencies, and write each dependency to a separate yaml file\n in the output directory. By default, all dependencies will be written\n out. To choose a smaller subset of dependencies to be written, pass a\n list of package names in the dependencies parameter. If the format of the\n incoming spec is not json, that can be specified with the spec_format\n parameter. This can be used to convert from yaml specfiles to the\n json format.\"\"\"\n\n for spec in root.traverse():\n if not any(spec.satisfies(dep) for dep in dependencies):\n continue\n\n json_path = os.path.join(output_directory, f\"{spec.name}.json\")\n\n with open(json_path, \"w\", encoding=\"utf-8\") as fd:\n fd.write(spec.to_json(hash=ht.dag_hash))\n\n\ndef get_host_environment_metadata() -> Dict[str, str]:\n \"\"\"Get the host environment, reduce to a subset that we can store in\n the install directory, and add the spack version.\n \"\"\"\n\n environ = get_host_environment()\n return {\n \"host_os\": environ[\"os\"],\n \"platform\": environ[\"platform\"],\n \"host_target\": environ[\"target\"],\n \"hostname\": environ[\"hostname\"],\n \"spack_version\": spack.get_version(),\n \"kernel_version\": platform.version(),\n }\n\n\ndef get_host_environment() -> Dict[str, Any]:\n \"\"\"Returns a dictionary with host information (not including the os.environ).\"\"\"\n host_platform = spack.platforms.host()\n host_target = host_platform.default_target()\n host_os = host_platform.default_operating_system()\n arch_fmt = \"platform={0} os={1} target={2}\"\n arch_spec = Spec(arch_fmt.format(host_platform, host_os, host_target))\n return {\n \"target\": str(host_target),\n \"os\": str(host_os),\n \"platform\": str(host_platform),\n \"arch\": arch_spec,\n \"architecture\": arch_spec,\n \"arch_str\": str(arch_spec),\n \"hostname\": socket.gethostname(),\n }\n\n\ndef eval_conditional(string):\n \"\"\"Evaluate conditional definitions using restricted variable scope.\"\"\"\n valid_variables = get_host_environment()\n valid_variables.update({\"re\": re, \"env\": os.environ})\n return eval(string, valid_variables)\n\n\ndef _inject_patches_variant(root: Spec) -> None:\n # This dictionary will store object IDs rather than Specs as keys\n # since the Spec __hash__ will change as patches are added to them\n spec_to_patches: Dict[int, Set[spack.patch.Patch]] = {}\n for s in root.traverse():\n assert s.namespace is not None, (\n f\"internal error: {s.name} has no namespace after concretization. \"\n f\"Please report a bug at https://github.com/spack/spack/issues\"\n )\n\n if s.concrete:\n continue\n\n # Add any patches from the package to the spec.\n node_patches = {\n patch\n for cond, patch_list in spack.repo.PATH.get_pkg_class(s.fullname).patches.items()\n if s.satisfies(cond)\n for patch in patch_list\n }\n if node_patches:\n spec_to_patches[id(s)] = node_patches\n\n # Also record all patches required on dependencies by depends_on(..., patch=...)\n for dspec in root.traverse_edges(deptype=dt.ALL, cover=\"edges\", root=False):\n if dspec.spec.concrete:\n continue\n\n pkg_deps = spack.repo.PATH.get_pkg_class(dspec.parent.fullname).dependencies\n\n edge_patches: List[spack.patch.Patch] = []\n for cond, deps_by_name in pkg_deps.items():\n dependency = deps_by_name.get(dspec.spec.name)\n if not dependency:\n continue\n\n if not dspec.parent.satisfies(cond):\n continue\n\n for pcond, patch_list in dependency.patches.items():\n if dspec.spec.satisfies(pcond):\n edge_patches.extend(patch_list)\n\n if edge_patches:\n spec_to_patches.setdefault(id(dspec.spec), set()).update(edge_patches)\n\n for spec in root.traverse():\n if id(spec) not in spec_to_patches:\n continue\n\n patches = list(spec_to_patches[id(spec)])\n variant: vt.VariantValue = spec.variants.setdefault(\n \"patches\", vt.MultiValuedVariant(\"patches\", ())\n )\n variant.set(*(p.sha256 for p in patches))\n # FIXME: Monkey patches variant to store patches order\n ordered_hashes = [(*p.ordering_key, p.sha256) for p in patches if p.ordering_key]\n ordered_hashes.sort()\n tty.debug(\n f\"Ordered hashes [{spec.name}]: \"\n + \", \".join(\"/\".join(str(e) for e in t) for t in ordered_hashes)\n )\n setattr(\n variant, \"_patches_in_order_of_appearance\", [sha256 for _, _, sha256 in ordered_hashes]\n )\n\n\nclass InvalidVariantForSpecError(spack.error.SpecError):\n \"\"\"Raised when an invalid conditional variant is specified.\"\"\"\n\n def __init__(self, variant, when, spec):\n msg = f\"Invalid variant {variant} for spec {spec}.\\n\"\n msg += f\"{variant} is only available for {spec.name} when satisfying one of {when}.\"\n super().__init__(msg)\n\n\nclass UnsupportedPropagationError(spack.error.SpecError):\n \"\"\"Raised when propagation (==) is used with reserved variant names.\"\"\"\n\n\nclass DuplicateDependencyError(spack.error.SpecError):\n \"\"\"Raised when the same dependency occurs in a spec twice.\"\"\"\n\n\nclass UnsupportedCompilerError(spack.error.SpecError):\n \"\"\"Raised when the user asks for a compiler spack doesn't know about.\"\"\"\n\n\nclass DuplicateArchitectureError(spack.error.SpecError):\n \"\"\"Raised when the same architecture occurs in a spec twice.\"\"\"\n\n\nclass InvalidDependencyError(spack.error.SpecError):\n \"\"\"Raised when a dependency in a spec is not actually a dependency\n of the package.\"\"\"\n\n def __init__(self, pkg, deps):\n self.invalid_deps = deps\n super().__init__(\n \"Package {0} does not depend on {1}\".format(pkg, spack.llnl.string.comma_or(deps))\n )\n\n\nclass UnsatisfiableSpecNameError(spack.error.UnsatisfiableSpecError):\n \"\"\"Raised when two specs aren't even for the same package.\"\"\"\n\n def __init__(self, provided, required):\n super().__init__(provided, required, \"name\")\n\n\nclass UnsatisfiableVersionSpecError(spack.error.UnsatisfiableSpecError):\n \"\"\"Raised when a spec version conflicts with package constraints.\"\"\"\n\n def __init__(self, provided, required):\n super().__init__(provided, required, \"version\")\n\n\nclass UnsatisfiableArchitectureSpecError(spack.error.UnsatisfiableSpecError):\n \"\"\"Raised when a spec architecture conflicts with package constraints.\"\"\"\n\n def __init__(self, provided, required):\n super().__init__(provided, required, \"architecture\")\n\n\n# TODO: get rid of this and be more specific about particular incompatible\n# dep constraints\nclass UnsatisfiableDependencySpecError(spack.error.UnsatisfiableSpecError):\n \"\"\"Raised when some dependency of constrained specs are incompatible\"\"\"\n\n def __init__(self, provided, required):\n super().__init__(provided, required, \"dependency\")\n\n\nclass UnconstrainableDependencySpecError(spack.error.SpecError):\n \"\"\"Raised when attempting to constrain by an anonymous dependency spec\"\"\"\n\n def __init__(self, spec):\n msg = \"Cannot constrain by spec '%s'. Cannot constrain by a\" % spec\n msg += \" spec containing anonymous dependencies\"\n super().__init__(msg)\n\n\nclass AmbiguousHashError(spack.error.SpecError):\n def __init__(self, msg, *specs):\n spec_fmt = (\n \"{namespace}.{name}{@version}{variants}\"\n \"{ platform=architecture.platform}{ os=architecture.os}{ target=architecture.target}\"\n \"{/hash:7}\"\n )\n specs_str = \"\\n \" + \"\\n \".join(spec.format(spec_fmt) for spec in specs)\n super().__init__(msg + specs_str)\n\n\nclass InvalidHashError(spack.error.SpecError):\n def __init__(self, spec, hash):\n msg = f\"No spec with hash {hash} could be found to match {spec}.\"\n msg += \" Either the hash does not exist, or it does not match other spec constraints.\"\n super().__init__(msg)\n\n\nclass SpecFormatStringError(spack.error.SpecError):\n \"\"\"Called for errors in Spec format strings.\"\"\"\n\n\nclass SpecFormatPathError(spack.error.SpecError):\n \"\"\"Called for errors in Spec path-format strings.\"\"\"\n\n\nclass SpecFormatSigilError(SpecFormatStringError):\n \"\"\"Called for mismatched sigils and attributes in format strings\"\"\"\n\n def __init__(self, sigil, requirement, used):\n msg = \"The sigil %s may only be used for %s.\" % (sigil, requirement)\n msg += \" It was used with the attribute %s.\" % used\n super().__init__(msg)\n\n\nclass ConflictsInSpecError(spack.error.SpecError, RuntimeError):\n def __init__(self, spec, matches):\n message = 'Conflicts in concretized spec \"{0}\"\\n'.format(spec.short_spec)\n\n visited = set()\n\n long_message = \"\"\n\n match_fmt_default = '{0}. \"{1}\" conflicts with \"{2}\"\\n'\n match_fmt_custom = '{0}. \"{1}\" conflicts with \"{2}\" [{3}]\\n'\n\n for idx, (s, c, w, msg) in enumerate(matches):\n if s not in visited:\n visited.add(s)\n long_message += \"List of matching conflicts for spec:\\n\\n\"\n long_message += s.tree(indent=4) + \"\\n\"\n\n if msg is None:\n long_message += match_fmt_default.format(idx + 1, c, w)\n else:\n long_message += match_fmt_custom.format(idx + 1, c, w, msg)\n\n super().__init__(message, long_message)\n\n\nclass SpecDeprecatedError(spack.error.SpecError):\n \"\"\"Raised when a spec concretizes to a deprecated spec or dependency.\"\"\"\n\n\nclass InvalidSpecDetected(spack.error.SpecError):\n \"\"\"Raised when a detected spec doesn't pass validation checks.\"\"\"\n\n\nclass SpliceError(spack.error.SpecError):\n \"\"\"Raised when a splice is not possible due to dependency or provider\n satisfaction mismatch. The resulting splice would be unusable.\"\"\"\n\n\nclass InvalidEdgeError(spack.error.SpecError):\n \"\"\"Raised when an edge doesn't pass validation checks.\"\"\"\n\n\nclass SpecMutationError(spack.error.SpecError):\n \"\"\"Raised when a mutation is attempted with invalid attributes.\"\"\"\n\n\nclass _ImmutableSpec(Spec):\n \"\"\"An immutable Spec that prevents a class of accidental mutations.\"\"\"\n\n _mutable: bool\n\n def __init__(self, spec_like: Optional[str] = None) -> None:\n object.__setattr__(self, \"_mutable\", True)\n super().__init__(spec_like)\n object.__delattr__(self, \"_mutable\")\n\n def __setstate__(self, state) -> None:\n object.__setattr__(self, \"_mutable\", True)\n super().__setstate__(state)\n object.__delattr__(self, \"_mutable\")\n\n def constrain(self, *args, **kwargs) -> bool:\n assert self._mutable\n return super().constrain(*args, **kwargs)\n\n def add_dependency_edge(self, *args, **kwargs):\n assert self._mutable\n return super().add_dependency_edge(*args, **kwargs)\n\n def __setattr__(self, name, value) -> None:\n assert self._mutable\n super().__setattr__(name, value)\n\n def __delattr__(self, name) -> None:\n assert self._mutable\n object.__delattr__(self, name)\n\n\n#: Immutable empty spec, for fast comparisons and reduced memory usage.\nEMPTY_SPEC = _ImmutableSpec()\n" }, { "path": "lib/spack/spack/spec_filter.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nfrom typing import Callable, List\n\nimport spack.spec\n\n\nclass SpecFilter:\n \"\"\"Given a method to produce a list of specs, this class can filter them according to\n different criteria.\n \"\"\"\n\n def __init__(\n self,\n factory: Callable[[], List[spack.spec.Spec]],\n is_usable: Callable[[spack.spec.Spec], bool],\n include: List[str],\n exclude: List[str],\n ) -> None:\n \"\"\"\n Args:\n factory: factory to produce a list of specs\n is_usable: predicate that takes a spec in input and returns False if the spec\n should not be considered for this filter, True otherwise.\n include: if present, a spec must match at least one entry in the list,\n to be in the output\n exclude: if present, a spec must not match any entry in the list to be in the output\n \"\"\"\n self.factory = factory\n self.is_usable = is_usable\n self.include = include\n self.exclude = exclude\n\n def is_selected(self, s: spack.spec.Spec) -> bool:\n if not self.is_usable(s):\n return False\n\n if self.include and not any(s.satisfies(c) for c in self.include):\n return False\n\n if self.exclude and any(s.satisfies(c) for c in self.exclude):\n return False\n\n return True\n\n def selected_specs(self) -> List[spack.spec.Spec]:\n return [s for s in self.factory() if self.is_selected(s)]\n" }, { "path": "lib/spack/spack/spec_parser.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Parser for spec literals\n\nHere is the EBNF grammar for a spec::\n\n spec = [name] [node_options] { ^[edge_properties] node } |\n [name] [node_options] hash |\n filename\n\n node = name [node_options] |\n [name] [node_options] hash |\n filename\n\n node_options = [@(version_list|version_pair)] [%compiler] { variant }\n edge_properties = [ { bool_variant | key_value } ]\n\n hash = / id\n filename = (.|/|[a-zA-Z0-9-_]*/)([a-zA-Z0-9-_./]*)(.json|.yaml)\n\n name = id | namespace id\n namespace = { id . }\n\n variant = bool_variant | key_value | propagated_bv | propagated_kv\n bool_variant = +id | ~id | -id\n propagated_bv = ++id | ~~id | --id\n key_value = id=id | id=quoted_id\n propagated_kv = id==id | id==quoted_id\n\n compiler = id [@version_list]\n\n version_pair = git_version=vid\n version_list = (version|version_range) [ { , (version|version_range)} ]\n version_range = vid:vid | vid: | :vid | :\n version = vid\n\n git_version = git.(vid) | git_hash\n git_hash = [A-Fa-f0-9]{40}\n\n quoted_id = \" id_with_ws \" | ' id_with_ws '\n id_with_ws = [a-zA-Z0-9_][a-zA-Z_0-9-.\\\\s]*\n vid = [a-zA-Z0-9_][a-zA-Z_0-9-.]*\n id = [a-zA-Z0-9_][a-zA-Z_0-9-]*\n\nIdentifiers using the ``=`` command, such as architectures and\ncompiler flags, require a space before the name.\n\nThere is one context-sensitive part: ids in versions may contain ``.``, while\nother ids may not.\n\nThere is one ambiguity: since ``-`` is allowed in an id, you need to put\nwhitespace space before ``-variant`` for it to be tokenized properly. You can\neither use whitespace, or you can just use ``~variant`` since it means the same\nthing. Spack uses ``~variant`` in directory names and in the canonical form of\nspecs to avoid ambiguity. Both are provided because ``~`` can cause shell\nexpansion when it is the first character in an id typed on the command line.\n\"\"\"\n\nimport json\nimport pathlib\nimport re\nimport sys\nfrom typing import TYPE_CHECKING, Dict, Iterator, List, Optional, Tuple, Union\n\nimport spack.deptypes\nimport spack.error\nimport spack.version\nfrom spack.aliases import LEGACY_COMPILER_TO_BUILTIN\nfrom spack.enums import PropagationPolicy\nfrom spack.llnl.util.tty import color\nfrom spack.tokenize import Token, TokenBase, Tokenizer\n\nif TYPE_CHECKING:\n import spack.spec\n\n#: Valid name for specs and variants. Here we are not using\n#: the previous ``w[\\w.-]*`` since that would match most\n#: characters that can be part of a word in any language\nIDENTIFIER = r\"(?:[a-zA-Z_0-9][a-zA-Z_0-9\\-]*)\"\nDOTTED_IDENTIFIER = rf\"(?:{IDENTIFIER}(?:\\.{IDENTIFIER})+)\"\nGIT_HASH = r\"(?:[A-Fa-f0-9]{40})\"\n#: Git refs include branch names, and can contain ``.`` and ``/``\nGIT_REF = r\"(?:[a-zA-Z_0-9][a-zA-Z_0-9./\\-]*)\"\nGIT_VERSION_PATTERN = rf\"(?:(?:git\\.(?:{GIT_REF}))|(?:{GIT_HASH}))\"\n\n#: Substitute a package for a virtual, e.g., c,cxx=gcc.\n#: NOTE: Overlaps w/KVP; this should be first if matched in sequence.\nVIRTUAL_ASSIGNMENT = (\n r\"(?:\"\n rf\"(?P{IDENTIFIER}(?:,{IDENTIFIER})*)\" # comma-separated virtuals\n rf\"=(?P{DOTTED_IDENTIFIER}|{IDENTIFIER})\" # package to substitute\n r\")\"\n)\n\nSTAR = r\"\\*\"\n\nNAME = r\"[a-zA-Z_0-9][a-zA-Z_0-9\\-.]*\"\n\nHASH = r\"[a-zA-Z_0-9]+\"\n\n#: These are legal values that *can* be parsed bare, without quotes on the command line.\nVALUE = r\"(?:[a-zA-Z_0-9\\-+\\*.,:=%^\\~\\/\\\\]+)\"\n\n#: Quoted values can be *anything* in between quotes, including escaped quotes.\nQUOTED_VALUE = r\"(?:'(?:[^']|(?<=\\\\)')*'|\\\"(?:[^\\\"]|(?<=\\\\)\\\")*\\\")\"\n\nVERSION = r\"=?(?:[a-zA-Z0-9_][a-zA-Z_0-9\\-\\.]*\\b)\"\nVERSION_RANGE = rf\"(?:(?:{VERSION})?:(?:{VERSION}(?!\\s*=))?)\"\nVERSION_LIST = rf\"(?:{VERSION_RANGE}|{VERSION})(?:\\s*,\\s*(?:{VERSION_RANGE}|{VERSION}))*\"\n\nSPLIT_KVP = re.compile(rf\"^({NAME})(:?==?)(.*)$\")\n\n#: A filename starts either with a ``.`` or a ``/`` or a ``{name}/``, or on Windows, a drive letter\n#: followed by a colon and ``\\`` or ``.`` or ``{name}\\``\nWINDOWS_FILENAME = r\"(?:\\.|[a-zA-Z0-9-_]*\\\\|[a-zA-Z]:\\\\)(?:[a-zA-Z0-9-_\\.\\\\]*)(?:\\.json|\\.yaml)\"\nUNIX_FILENAME = r\"(?:\\.|\\/|[a-zA-Z0-9-_]*\\/)(?:[a-zA-Z0-9-_\\.\\/]*)(?:\\.json|\\.yaml)\"\nFILENAME = WINDOWS_FILENAME if sys.platform == \"win32\" else UNIX_FILENAME\n\n#: Regex to strip quotes. Group 2 will be the unquoted string.\nSTRIP_QUOTES = re.compile(r\"^(['\\\"])(.*)\\1$\")\n\n#: Values that match this (e.g., variants, flags) can be left unquoted in Spack output\nNO_QUOTES_NEEDED = re.compile(r\"^[a-zA-Z0-9,/_.\\-\\[\\]]+$\")\n\n\nclass SpecTokens(TokenBase):\n \"\"\"Enumeration of the different token kinds of tokens in the spec grammar.\n\n Order of declaration is extremely important, since text containing specs is parsed with a\n single regex obtained by ``\"|\".join(...)`` of all the regex in the order of declaration.\n \"\"\"\n\n # Dependency, with optional virtual assignment specifier\n START_EDGE_PROPERTIES = r\"(?:(?:\\^|\\%\\%|\\%)\\[)\"\n END_EDGE_PROPERTIES = rf\"(?:\\](?:\\s*{VIRTUAL_ASSIGNMENT})?)\"\n DEPENDENCY = rf\"(?:(?:\\^|\\%\\%|\\%)(?:\\s*{VIRTUAL_ASSIGNMENT})?)\"\n\n # Version\n VERSION_HASH_PAIR = rf\"(?:@(?:{GIT_VERSION_PATTERN})=(?:{VERSION}))\"\n GIT_VERSION = rf\"@(?:{GIT_VERSION_PATTERN})\"\n VERSION = rf\"(?:@\\s*(?:{VERSION_LIST}))\"\n\n # Variants\n PROPAGATED_BOOL_VARIANT = rf\"(?:(?:\\+\\+|~~|--)\\s*{NAME})\"\n BOOL_VARIANT = rf\"(?:[~+-]\\s*{NAME})\"\n PROPAGATED_KEY_VALUE_PAIR = rf\"(?:{NAME}:?==(?:{VALUE}|{QUOTED_VALUE}))\"\n KEY_VALUE_PAIR = rf\"(?:{NAME}:?=(?:{VALUE}|{QUOTED_VALUE}))\"\n\n # FILENAME\n FILENAME = rf\"(?:{FILENAME})\"\n\n # Package name\n FULLY_QUALIFIED_PACKAGE_NAME = rf\"(?:{DOTTED_IDENTIFIER})\"\n UNQUALIFIED_PACKAGE_NAME = rf\"(?:{IDENTIFIER}|{STAR})\"\n\n # DAG hash\n DAG_HASH = rf\"(?:/(?:{HASH}))\"\n\n # White spaces\n WS = r\"(?:\\s+)\"\n\n # Unexpected character(s)\n UNEXPECTED = r\"(?:.[\\s]*)\"\n\n\n#: Tokenizer that includes all the regexes in the SpecTokens enum\nSPEC_TOKENIZER = Tokenizer(SpecTokens)\n\n\ndef tokenize(text: str) -> Iterator[Token]:\n \"\"\"Return a token generator from the text passed as input.\n\n Raises:\n SpecTokenizationError: when unexpected characters are found in the text\n \"\"\"\n for token in SPEC_TOKENIZER.tokenize(text):\n if token.kind == SpecTokens.UNEXPECTED:\n raise SpecTokenizationError(list(SPEC_TOKENIZER.tokenize(text)), text)\n yield token\n\n\ndef parseable_tokens(text: str) -> Iterator[Token]:\n \"\"\"Return non-whitespace tokens from the text passed as input\n\n Raises:\n SpecTokenizationError: when unexpected characters are found in the text\n \"\"\"\n return filter(lambda x: x.kind != SpecTokens.WS, tokenize(text))\n\n\nclass TokenContext:\n \"\"\"Token context passed around by parsers\"\"\"\n\n __slots__ = \"token_stream\", \"current_token\", \"next_token\", \"pushed_tokens\"\n\n def __init__(self, token_stream: Iterator[Token]):\n self.token_stream = token_stream\n self.current_token = None\n self.next_token = None # the next token to be read\n\n # if not empty, back of list is front of stream, and we pop from here instead.\n self.pushed_tokens: List[Token] = []\n\n self.advance()\n\n def advance(self):\n \"\"\"Advance one token\"\"\"\n self.current_token = self.next_token\n if self.pushed_tokens:\n self.next_token = self.pushed_tokens.pop()\n else:\n self.next_token = next(self.token_stream, None)\n\n def accept(self, kind: SpecTokens):\n \"\"\"If the next token is of the specified kind, advance the stream and return True.\n Otherwise return False.\n \"\"\"\n if self.next_token and self.next_token.kind == kind:\n self.advance()\n return True\n return False\n\n def push_front(self, token=Token):\n \"\"\"Push a token onto the front of the stream. Enables a bit of lookahead.\"\"\"\n self.pushed_tokens.append(self.next_token) # back of list is front of stream\n self.next_token = token\n\n def expect(self, *kinds: SpecTokens):\n return self.next_token and self.next_token.kind in kinds\n\n\nclass SpecTokenizationError(spack.error.SpecSyntaxError):\n \"\"\"Syntax error in a spec string\"\"\"\n\n def __init__(self, tokens: List[Token], text: str):\n message = f\"unexpected characters in the spec string\\n{text}\\n\"\n\n underline = \"\"\n for token in tokens:\n is_error = token.kind == SpecTokens.UNEXPECTED\n underline += (\"^\" if is_error else \" \") * (token.end - token.start)\n\n message += color.colorize(f\"@*r{{{underline}}}\")\n super().__init__(message)\n\n\ndef parse_virtual_assignment(context: TokenContext) -> Tuple[str]:\n \"\"\"Look at subvalues and, if present, extract virtual and a push a substitute token.\n\n This handles things like:\n\n * ``^c=gcc``\n * ``^c,cxx=gcc``\n * ``%[when=+bar] c=gcc``\n * ``%[when=+bar] c,cxx=gcc``\n\n Virtual assignment can happen anywhere a dependency node can appear. It is\n shorthand for ``%[virtuals=c,cxx] gcc``.\n\n The ``virtuals=substitute`` key value pair appears in the subvalues of\n :attr:`~spack.spec_parser.SpecTokens.DEPENDENCY` and\n :attr:`~spack.spec_parser.SpecTokens.END_EDGE_PROPERTIES` tokens. We extract the virtuals and\n create a token from the substitute, which is then pushed back on the parser stream so that the\n head of the stream can be parsed like a regular node.\n\n Returns:\n the virtuals assigned, or None if there aren't any\n\n \"\"\"\n assert context.current_token is not None\n\n subvalues = context.current_token.subvalues\n if not subvalues:\n return ()\n\n # build a token for the substitute that we can put back on the stream\n pkg = subvalues[\"substitute\"]\n token_type = SpecTokens.UNQUALIFIED_PACKAGE_NAME\n if \".\" in pkg:\n token_type = SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME\n start = context.current_token.value.index(pkg)\n\n token = Token(token_type, pkg, start, start + len(pkg))\n context.push_front(token)\n\n return tuple(subvalues[\"virtuals\"].split(\",\"))\n\n\nclass SpecParser:\n \"\"\"Parse text into specs\"\"\"\n\n __slots__ = \"literal_str\", \"ctx\"\n\n def __init__(self, literal_str: str):\n self.literal_str = literal_str\n self.ctx = TokenContext(parseable_tokens(literal_str))\n\n def tokens(self) -> List[Token]:\n \"\"\"Return the entire list of token from the initial text. White spaces are\n filtered out.\n \"\"\"\n return list(filter(lambda x: x.kind != SpecTokens.WS, tokenize(self.literal_str)))\n\n def next_spec(\n self, initial_spec: Optional[\"spack.spec.Spec\"] = None\n ) -> Optional[\"spack.spec.Spec\"]:\n \"\"\"Return the next spec parsed from text.\n\n Args:\n initial_spec: object where to parse the spec. If None a new one\n will be created.\n\n Return:\n The spec that was parsed\n \"\"\"\n if not self.ctx.next_token:\n return initial_spec\n\n def add_dependency(dep, **edge_properties):\n \"\"\"wrapper around root_spec._add_dependency\"\"\"\n try:\n target_spec._add_dependency(dep, **edge_properties)\n except spack.error.SpecError as e:\n raise SpecParsingError(str(e), self.ctx.current_token, self.literal_str) from e\n\n if not initial_spec:\n from spack.spec import Spec\n\n initial_spec = Spec()\n root_spec = SpecNodeParser(self.ctx, self.literal_str).parse(initial_spec)\n current_spec = root_spec\n while True:\n if self.ctx.accept(SpecTokens.START_EDGE_PROPERTIES):\n has_edge_attrs = True\n elif self.ctx.accept(SpecTokens.DEPENDENCY):\n has_edge_attrs = False\n else:\n break\n\n is_direct = self.ctx.current_token.value[0] == \"%\"\n propagation = PropagationPolicy.NONE\n if is_direct and self.ctx.current_token.value.startswith(\"%%\"):\n propagation = PropagationPolicy.PREFERENCE\n\n if has_edge_attrs:\n edge_properties = EdgeAttributeParser(self.ctx, self.literal_str).parse()\n edge_properties.setdefault(\"virtuals\", ())\n edge_properties.setdefault(\"depflag\", 0)\n else:\n virtuals = parse_virtual_assignment(self.ctx)\n edge_properties = {\"virtuals\": virtuals, \"depflag\": 0}\n\n edge_properties[\"direct\"] = is_direct\n edge_properties[\"propagation\"] = propagation\n\n dependency = self._parse_node(root_spec)\n\n if is_direct:\n target_spec = current_spec\n if dependency.name in LEGACY_COMPILER_TO_BUILTIN:\n dependency.name = LEGACY_COMPILER_TO_BUILTIN[dependency.name]\n else:\n current_spec = dependency\n target_spec = root_spec\n\n add_dependency(dependency, **edge_properties)\n\n return root_spec\n\n def _parse_node(self, root_spec: \"spack.spec.Spec\", root: bool = True):\n dependency = SpecNodeParser(self.ctx, self.literal_str).parse(root=root)\n if dependency is None:\n msg = (\n \"the dependency sigil and any optional edge attributes must be followed by a \"\n \"package name or a node attribute (version, variant, etc.)\"\n )\n raise SpecParsingError(msg, self.ctx.current_token, self.literal_str)\n if root_spec.concrete:\n raise spack.error.SpecError(str(root_spec), \"^\" + str(dependency))\n return dependency\n\n def all_specs(self) -> List[\"spack.spec.Spec\"]:\n \"\"\"Return all the specs that remain to be parsed\"\"\"\n return list(iter(self.next_spec, None))\n\n\nclass SpecNodeParser:\n \"\"\"Parse a single spec node from a stream of tokens\"\"\"\n\n __slots__ = \"ctx\", \"has_version\", \"literal_str\"\n\n def __init__(self, ctx, literal_str):\n self.ctx = ctx\n self.literal_str = literal_str\n self.has_version = False\n\n def parse(\n self, initial_spec: Optional[\"spack.spec.Spec\"] = None, root: bool = True\n ) -> \"spack.spec.Spec\":\n \"\"\"Parse a single spec node from a stream of tokens\n\n Args:\n initial_spec: object to be constructed\n root: True if we're parsing a root, False if dependency after ^ or %\n\n Return:\n The object passed as argument\n \"\"\"\n if initial_spec is None:\n from spack.spec import Spec\n\n initial_spec = Spec()\n\n if not self.ctx.next_token or self.ctx.expect(SpecTokens.DEPENDENCY):\n return initial_spec\n\n # If we start with a package name we have a named spec, we cannot\n # accept another package name afterwards in a node\n if self.ctx.accept(SpecTokens.UNQUALIFIED_PACKAGE_NAME):\n # if name is '*', this is an anonymous spec\n if self.ctx.current_token.value != \"*\":\n initial_spec.name = self.ctx.current_token.value\n\n elif self.ctx.accept(SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME):\n parts = self.ctx.current_token.value.split(\".\")\n name = parts[-1]\n namespace = \".\".join(parts[:-1])\n initial_spec.name = name\n initial_spec.namespace = namespace\n\n elif self.ctx.accept(SpecTokens.FILENAME):\n return FileParser(self.ctx).parse(initial_spec)\n\n def raise_parsing_error(string: str, cause: Optional[Exception] = None):\n \"\"\"Raise a spec parsing error with token context.\"\"\"\n raise SpecParsingError(string, self.ctx.current_token, self.literal_str) from cause\n\n def add_flag(name: str, value: Union[str, bool], propagate: bool, concrete: bool):\n \"\"\"Wrapper around ``Spec._add_flag()`` that adds parser context to errors raised.\"\"\"\n try:\n initial_spec._add_flag(name, value, propagate, concrete)\n except Exception as e:\n raise_parsing_error(str(e), e)\n\n while True:\n if (\n self.ctx.accept(SpecTokens.VERSION_HASH_PAIR)\n or self.ctx.accept(SpecTokens.GIT_VERSION)\n or self.ctx.accept(SpecTokens.VERSION)\n ):\n if self.has_version:\n raise_parsing_error(\"Spec cannot have multiple versions\")\n\n initial_spec.versions = spack.version.VersionList(\n [spack.version.from_string(self.ctx.current_token.value[1:])]\n )\n initial_spec.attach_git_version_lookup()\n self.has_version = True\n\n elif self.ctx.accept(SpecTokens.BOOL_VARIANT):\n name = self.ctx.current_token.value[1:].strip()\n variant_value = self.ctx.current_token.value[0] == \"+\"\n add_flag(name, variant_value, propagate=False, concrete=True)\n\n elif self.ctx.accept(SpecTokens.PROPAGATED_BOOL_VARIANT):\n name = self.ctx.current_token.value[2:].strip()\n variant_value = self.ctx.current_token.value[0:2] == \"++\"\n add_flag(name, variant_value, propagate=True, concrete=True)\n\n elif self.ctx.accept(SpecTokens.KEY_VALUE_PAIR):\n name, value = self.ctx.current_token.value.split(\"=\", maxsplit=1)\n concrete = name.endswith(\":\")\n if concrete:\n name = name[:-1]\n\n add_flag(\n name, strip_quotes_and_unescape(value), propagate=False, concrete=concrete\n )\n\n elif self.ctx.accept(SpecTokens.PROPAGATED_KEY_VALUE_PAIR):\n name, value = self.ctx.current_token.value.split(\"==\", maxsplit=1)\n concrete = name.endswith(\":\")\n if concrete:\n name = name[:-1]\n add_flag(name, strip_quotes_and_unescape(value), propagate=True, concrete=concrete)\n\n elif self.ctx.expect(SpecTokens.DAG_HASH):\n if initial_spec.abstract_hash:\n break\n self.ctx.accept(SpecTokens.DAG_HASH)\n initial_spec.abstract_hash = self.ctx.current_token.value[1:]\n\n else:\n break\n\n return initial_spec\n\n\nclass FileParser:\n \"\"\"Parse a single spec from a JSON or YAML file\"\"\"\n\n __slots__ = (\"ctx\",)\n\n def __init__(self, ctx):\n self.ctx = ctx\n\n def parse(self, initial_spec: \"spack.spec.Spec\") -> \"spack.spec.Spec\":\n \"\"\"Parse a spec tree from a specfile.\n\n Args:\n initial_spec: object where to parse the spec\n\n Return:\n The initial_spec passed as argument, once constructed\n \"\"\"\n file = pathlib.Path(self.ctx.current_token.value)\n\n if not file.exists():\n raise spack.error.NoSuchSpecFileError(f\"No such spec file: '{file}'\")\n\n from spack.spec import Spec\n\n with file.open(\"r\", encoding=\"utf-8\") as stream:\n if str(file).endswith(\".json\"):\n spec_from_file = Spec.from_json(stream)\n else:\n spec_from_file = Spec.from_yaml(stream)\n initial_spec._dup(spec_from_file)\n return initial_spec\n\n\nclass EdgeAttributeParser:\n __slots__ = \"ctx\", \"literal_str\"\n\n def __init__(self, ctx, literal_str):\n self.ctx = ctx\n self.literal_str = literal_str\n\n def parse(self):\n attributes = {}\n while True:\n if self.ctx.accept(SpecTokens.KEY_VALUE_PAIR):\n name, value = self.ctx.current_token.value.split(\"=\", maxsplit=1)\n if name.endswith(\":\"):\n name = name[:-1]\n value = value.strip(\"'\\\" \").split(\",\")\n attributes[name] = value\n if name not in (\"deptypes\", \"virtuals\", \"when\"):\n msg = (\n \"the only edge attributes that are currently accepted \"\n 'are \"deptypes\", \"virtuals\", and \"when\"'\n )\n raise SpecParsingError(msg, self.ctx.current_token, self.literal_str)\n # TODO: Add code to accept bool variants here as soon as use variants are implemented\n elif self.ctx.accept(SpecTokens.END_EDGE_PROPERTIES):\n virtuals = attributes.get(\"virtuals\", ())\n virtuals += parse_virtual_assignment(self.ctx)\n attributes[\"virtuals\"] = virtuals\n break\n else:\n msg = \"unexpected token in edge attributes\"\n raise SpecParsingError(msg, self.ctx.next_token, self.literal_str)\n\n # Turn deptypes=... to depflag representation\n if \"deptypes\" in attributes:\n deptype_string = attributes.pop(\"deptypes\")\n attributes[\"depflag\"] = spack.deptypes.canonicalize(deptype_string)\n\n # Turn \"when\" into a spec\n if \"when\" in attributes:\n attributes[\"when\"] = parse_one_or_raise(attributes[\"when\"][0])\n\n return attributes\n\n\ndef parse(text: str, *, toolchains: Optional[Dict] = None) -> List[\"spack.spec.Spec\"]:\n \"\"\"Parse text into a list of specs\n\n Args:\n text: text to be parsed\n toolchains: optional toolchain definitions to expand after parsing\n\n Return:\n List of specs\n \"\"\"\n specs = SpecParser(text).all_specs()\n if toolchains:\n cache: Dict[str, \"spack.spec.Spec\"] = {}\n for spec in specs:\n expand_toolchains(spec, toolchains, _cache=cache)\n return specs\n\n\ndef parse_one_or_raise(\n text: str,\n initial_spec: Optional[\"spack.spec.Spec\"] = None,\n *,\n toolchains: Optional[Dict] = None,\n) -> \"spack.spec.Spec\":\n \"\"\"Parse exactly one spec from text and return it, or raise\n\n Args:\n text: text to be parsed\n initial_spec: buffer where to parse the spec. If None a new one will be created.\n toolchains: optional toolchain definitions to expand after parsing\n \"\"\"\n parser = SpecParser(text)\n result = parser.next_spec(initial_spec)\n next_token = parser.ctx.next_token\n\n if next_token:\n message = f\"expected a single spec, but got more:\\n{text}\"\n underline = f\"\\n{' ' * next_token.start}{'^' * len(next_token.value)}\"\n message += color.colorize(f\"@*r{{{underline}}}\")\n raise ValueError(message)\n\n if result is None:\n raise ValueError(\"expected a single spec, but got none\")\n\n if toolchains:\n expand_toolchains(result, toolchains)\n\n return result\n\n\ndef _parse_toolchain_config(toolchain_config: Union[str, List[Dict]]) -> \"spack.spec.Spec\":\n \"\"\"Parse a toolchain config entry (string or list) into a Spec.\"\"\"\n if isinstance(toolchain_config, str):\n toolchain = parse_one_or_raise(toolchain_config)\n _ensure_all_direct_edges(toolchain)\n else:\n from spack.spec import EMPTY_SPEC, Spec\n\n toolchain = Spec()\n for entry in toolchain_config:\n toolchain_part = parse_one_or_raise(entry[\"spec\"])\n when = entry.get(\"when\", \"\")\n _ensure_all_direct_edges(toolchain_part)\n\n if when:\n when_spec = Spec(when)\n for edge in toolchain_part.traverse_edges():\n if edge.when is EMPTY_SPEC:\n edge.when = when_spec.copy()\n else:\n edge.when.constrain(when_spec)\n toolchain.constrain(toolchain_part)\n return toolchain\n\n\ndef _ensure_all_direct_edges(constraint: \"spack.spec.Spec\") -> None:\n \"\"\"Validate that a toolchain spec only has direct (%) edges.\"\"\"\n for edge in constraint.traverse_edges(root=False):\n if not edge.direct:\n raise spack.error.SpecError(\n f\"cannot use '^' in toolchain definitions, and the current \"\n f\"toolchain contains '{edge.format()}'\"\n )\n\n\ndef expand_toolchains(\n spec: \"spack.spec.Spec\",\n toolchains: Dict,\n *,\n _cache: Optional[Dict[str, \"spack.spec.Spec\"]] = None,\n) -> None:\n \"\"\"Replace toolchain placeholder deps with expanded toolchain constraints.\n\n Walks every node in the spec DAG. For each node, finds direct dependency\n edges whose child name is a key in ``toolchains``. Removes the placeholder\n edge, parses the toolchain config, copies with the edge's propagation\n policy, and constrains the node.\n \"\"\"\n if _cache is None:\n _cache = {}\n\n for node in list(spec.traverse()):\n for edge in list(node.edges_to_dependencies()):\n if not edge.direct:\n continue\n name = edge.spec.name\n if name not in toolchains:\n continue\n\n # Remove the placeholder edge (both directions)\n node._dependencies[name].remove(edge)\n if not node._dependencies[name]:\n del node._dependencies[name]\n edge.spec._dependents[node.name].remove(edge)\n if not edge.spec._dependents[node.name]:\n del edge.spec._dependents[node.name]\n\n # Parse and cache toolchain\n if name not in _cache:\n _cache[name] = _parse_toolchain_config(toolchains[name])\n\n propagation = edge.propagation\n propagation_arg = None if propagation != PropagationPolicy.PREFERENCE else propagation\n # Copy so each usage gets a distinct object (solver depends on this)\n toolchain = _cache[name].copy(propagation=propagation_arg)\n node.constrain(toolchain)\n\n\nclass SpecParsingError(spack.error.SpecSyntaxError):\n \"\"\"Error when parsing tokens\"\"\"\n\n def __init__(self, message, token, text):\n message += f\"\\n{text}\"\n if token:\n underline = f\"\\n{' ' * token.start}{'^' * (token.end - token.start)}\"\n message += color.colorize(f\"@*r{{{underline}}}\")\n super().__init__(message)\n\n\ndef strip_quotes_and_unescape(string: str) -> str:\n \"\"\"Remove surrounding single or double quotes from string, if present.\"\"\"\n match = STRIP_QUOTES.match(string)\n if not match:\n return string\n\n # replace any escaped quotes with bare quotes\n quote, result = match.groups()\n return result.replace(rf\"\\{quote}\", quote)\n\n\ndef quote_if_needed(value: str) -> str:\n \"\"\"Add quotes around the value if it requires quotes.\n\n This will add quotes around the value unless it matches :data:`NO_QUOTES_NEEDED`.\n\n This adds:\n\n * single quotes by default\n * double quotes around any value that contains single quotes\n\n If double quotes are used, we json-escape the string. That is, we escape ``\\\\``,\n ``\"``, and control codes.\n\n \"\"\"\n if NO_QUOTES_NEEDED.match(value):\n return value\n\n return json.dumps(value) if \"'\" in value else f\"'{value}'\"\n" }, { "path": "lib/spack/spack/stage.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport abc\nimport errno\nimport glob\nimport hashlib\nimport io\nimport os\nimport shutil\nimport stat\nimport sys\nimport tempfile\nfrom typing import TYPE_CHECKING, Callable, Dict, Generator, Iterable, List, Optional, Set, Union\n\nimport spack.caches\nimport spack.config\nimport spack.error\nimport spack.llnl.string\nimport spack.llnl.util.lang\nimport spack.llnl.util.tty as tty\nimport spack.oci.image\nimport spack.resource\nimport spack.spec\nimport spack.util.crypto\nimport spack.util.lock\nimport spack.util.parallel\nimport spack.util.path as sup\nimport spack.util.url as url_util\nfrom spack import fetch_strategy as fs # breaks a cycle\nfrom spack.llnl.util.filesystem import (\n AlreadyExistsError,\n can_access,\n get_owner_uid,\n getuid,\n install,\n install_tree,\n mkdirp,\n partition_path,\n remove_linked_tree,\n symlink,\n)\nfrom spack.llnl.util.tty.colify import colify\nfrom spack.llnl.util.tty.color import colorize\nfrom spack.util.crypto import bit_length, prefix_bits\nfrom spack.util.editor import editor, executable\nfrom spack.version import StandardVersion, VersionList\n\nif TYPE_CHECKING:\n import spack.mirrors.layout\n import spack.mirrors.mirror\n import spack.mirrors.utils\n\n\n# The well-known stage source subdirectory name.\n_source_path_subdir = \"spack-src\"\n\n# The temporary stage name prefix.\nstage_prefix = \"spack-stage-\"\n\n\ndef compute_stage_name(spec):\n \"\"\"Determine stage name given a spec\"\"\"\n spec_stage_structure = stage_prefix\n if spec.concrete:\n spec_stage_structure += \"{name}-{version}-{hash}\"\n else:\n spec_stage_structure += \"{name}-{version}\"\n # TODO (psakiev, scheibelp) Technically a user could still reintroduce a hash via\n # config:stage_name. This is a fix for how to handle staging an abstract spec (see #51305)\n stage_name_structure = spack.config.get(\"config:stage_name\", default=spec_stage_structure)\n return spec.format_path(format_string=stage_name_structure)\n\n\ndef create_stage_root(path: str) -> None:\n \"\"\"Create the stage root directory and ensure appropriate access perms.\"\"\"\n assert os.path.isabs(path) and len(path.strip()) > 1\n\n err_msg = \"Cannot create stage root {0}: Access to {1} is denied\"\n\n user_uid = getuid()\n\n # Obtain lists of ancestor and descendant paths of the $user node, if any.\n group_paths, user_node, user_paths = partition_path(path, sup.get_user())\n\n for p in group_paths:\n if not os.path.exists(p):\n # Ensure access controls of subdirs created above `$user` inherit\n # from the parent and share the group.\n par_stat = os.stat(os.path.dirname(p))\n mkdirp(p, group=par_stat.st_gid, mode=par_stat.st_mode)\n\n p_stat = os.stat(p)\n if par_stat.st_gid != p_stat.st_gid:\n tty.warn(\n \"Expected {0} to have group {1}, but it is {2}\".format(\n p, par_stat.st_gid, p_stat.st_gid\n )\n )\n\n if par_stat.st_mode & p_stat.st_mode != par_stat.st_mode:\n tty.warn(\n \"Expected {0} to support mode {1}, but it is {2}\".format(\n p, par_stat.st_mode, p_stat.st_mode\n )\n )\n\n if not can_access(p):\n raise OSError(errno.EACCES, err_msg.format(path, p))\n\n # Add the path ending with the $user node to the user paths to ensure paths\n # from $user (on down) meet the ownership and permission requirements.\n if user_node:\n user_paths.insert(0, user_node)\n\n for p in user_paths:\n # Ensure access controls of subdirs from `$user` on down are\n # restricted to the user.\n owner_uid = get_owner_uid(p)\n if user_uid != owner_uid:\n tty.warn(\n \"Expected user {0} to own {1}, but it is owned by {2}\".format(\n user_uid, p, owner_uid\n )\n )\n\n spack_src_subdir = os.path.join(path, _source_path_subdir)\n # When staging into a user-specified directory with `spack stage -p `, we need\n # to ensure the `spack-src` subdirectory exists, as we can't rely on it being\n # created automatically by spack. It's not clear why this is the case for `spack\n # stage -p`, but since `mkdirp()` is idempotent, this should not change the behavior\n # for any other code paths.\n if not os.path.isdir(spack_src_subdir):\n mkdirp(spack_src_subdir, mode=stat.S_IRWXU)\n\n\ndef _first_accessible_path(paths):\n \"\"\"Find the first path that is accessible, creating it if necessary.\"\"\"\n for path in paths:\n try:\n # Ensure the user has access, creating the directory if necessary.\n if os.path.exists(path):\n if can_access(path):\n return path\n else:\n # Now create the stage root with the proper group/perms.\n create_stage_root(path)\n return path\n\n except OSError as e:\n tty.debug(\"OSError while checking stage path %s: %s\" % (path, str(e)))\n\n return None\n\n\ndef _resolve_paths(candidates):\n \"\"\"\n Resolve candidate paths and make user-related adjustments.\n\n Adjustments involve removing extra $user from $tempdir if $tempdir includes\n $user and appending $user if it is not present in the path.\n \"\"\"\n temp_path = sup.canonicalize_path(\"$tempdir\")\n user = sup.get_user()\n tmp_has_usr = user in temp_path.split(os.path.sep)\n\n paths = []\n for path in candidates:\n # Remove the extra `$user` node from a `$tempdir/$user` entry for\n # hosts that automatically append `$user` to `$tempdir`.\n if path.startswith(os.path.join(\"$tempdir\", \"$user\")) and tmp_has_usr:\n path = path.replace(\"/$user\", \"\", 1)\n\n # Ensure the path is unique per user.\n can_path = sup.canonicalize_path(path)\n # When multiple users share a stage root, we can avoid conflicts between\n # them by adding a per-user subdirectory.\n # Avoid doing this on Windows to keep stage absolute path as short as possible.\n if user not in can_path and not sys.platform == \"win32\":\n can_path = os.path.join(can_path, user)\n\n paths.append(can_path)\n\n return paths\n\n\n# Cached stage path root\n_stage_root = None\n\n\ndef get_stage_root():\n global _stage_root\n\n if _stage_root is None:\n candidates = spack.config.get(\"config:build_stage\")\n if isinstance(candidates, str):\n candidates = [candidates]\n\n resolved_candidates = _resolve_paths(candidates)\n path = _first_accessible_path(resolved_candidates)\n if not path:\n raise StageError(\"No accessible stage paths in:\", \" \".join(resolved_candidates))\n\n _stage_root = path\n\n return _stage_root\n\n\ndef _mirror_roots():\n mirrors = spack.config.get(\"mirrors\")\n return [\n (\n sup.substitute_path_variables(root)\n if root.endswith(os.sep)\n else sup.substitute_path_variables(root) + os.sep\n )\n for root in mirrors.values()\n ]\n\n\nclass AbstractStage(abc.ABC):\n \"\"\"Abstract base class for all stage types.\n\n A stage is a directory whose lifetime can be managed with a context\n manager (but persists if the user requests it). Instances can have\n a specified name and if they do, then for all instances that have\n the same name, only one can enter the context manager at a time.\n\n This class defines the interface that all stage types must implement.\n \"\"\"\n\n #: Set to True to error out if patches fail\n requires_patch_success = True\n\n def __init__(self, name, path, keep, lock):\n # TODO: This uses a protected member of tempfile, but seemed the only\n # TODO: way to get a temporary name. It won't be the same as the\n # TODO: temporary stage area in _stage_root.\n self.name = name\n if name is None:\n self.name = stage_prefix + next(tempfile._get_candidate_names())\n\n # Use the provided path or construct an optionally named stage path.\n if path is not None:\n self.path = path\n else:\n self.path = os.path.join(get_stage_root(), self.name)\n\n # Flag to decide whether to delete the stage folder on exit or not\n self.keep = keep\n\n # File lock for the stage directory. We use one file for all\n # stage locks. See spack.database.Database.prefix_locker.lock for\n # details on this approach.\n self._lock = None\n self._use_locks = lock\n\n # When stages are reused, we need to know whether to re-create\n # it. This marks whether it has been created/destroyed.\n self.created = False\n\n def _get_lock(self):\n if not self._lock:\n sha1 = hashlib.sha1(self.name.encode(\"utf-8\")).digest()\n lock_id = prefix_bits(sha1, bit_length(sys.maxsize))\n stage_lock_path = os.path.join(get_stage_root(), \".lock\")\n self._lock = spack.util.lock.Lock(\n stage_lock_path, start=lock_id, length=1, desc=self.name\n )\n return self._lock\n\n def __enter__(self):\n \"\"\"\n Entering a stage context will create the stage directory\n\n Returns:\n self\n \"\"\"\n if self._use_locks:\n self._get_lock().acquire_write(timeout=60)\n self.create()\n return self\n\n def __exit__(self, exc_type, exc_val, exc_tb):\n \"\"\"\n Exiting from a stage context will delete the stage directory unless:\n - it was explicitly requested not to do so\n - an exception has been raised\n\n Args:\n exc_type: exception type\n exc_val: exception value\n exc_tb: exception traceback\n\n Returns:\n Boolean\n \"\"\"\n # Delete when there are no exceptions, unless asked to keep.\n if exc_type is None and not self.keep:\n self.destroy()\n\n if self._use_locks:\n self._get_lock().release_write()\n\n def create(self):\n \"\"\"\n Ensures the top-level (config:build_stage) directory exists.\n \"\"\"\n # User has full permissions and group has only read permissions\n if not os.path.exists(self.path):\n mkdirp(self.path, mode=stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP)\n elif not os.path.isdir(self.path):\n os.remove(self.path)\n mkdirp(self.path, mode=stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP)\n\n # Make sure we can actually do something with the stage we made.\n ensure_access(self.path)\n self.created = True\n\n @abc.abstractmethod\n def destroy(self):\n \"\"\"Remove the stage directory and its contents.\"\"\"\n ...\n\n @abc.abstractmethod\n def fetch(self, mirror_only: bool = False, err_msg: Optional[str] = None) -> None:\n \"\"\"Fetch the source code or resources for this stage.\"\"\"\n ...\n\n @abc.abstractmethod\n def check(self):\n \"\"\"Check the integrity of the fetched resources.\"\"\"\n ...\n\n @abc.abstractmethod\n def expand_archive(self):\n \"\"\"Expand any downloaded archives.\"\"\"\n ...\n\n @abc.abstractmethod\n def restage(self):\n \"\"\"Remove the expanded source and re-expand it.\"\"\"\n ...\n\n @abc.abstractmethod\n def cache_local(self):\n \"\"\"Cache the resources locally.\"\"\"\n ...\n\n @property\n @abc.abstractmethod\n def source_path(self) -> str:\n \"\"\"Return the path to the expanded source code.\"\"\"\n ...\n\n @property\n @abc.abstractmethod\n def expanded(self) -> bool:\n \"\"\"Return True if the source has been expanded.\"\"\"\n ...\n\n @property\n @abc.abstractmethod\n def archive_file(self) -> Optional[str]:\n \"\"\"Return the path to the archive file, or None.\"\"\"\n ...\n\n def cache_mirror(\n self,\n mirror: \"spack.caches.MirrorCache\",\n stats: \"spack.mirrors.utils.MirrorStatsForOneSpec\",\n ) -> None:\n \"\"\"Cache the resources to a mirror (can be no-op).\"\"\"\n pass\n\n def steal_source(self, dest: str) -> None:\n \"\"\"Copy source to another location (can be no-op).\"\"\"\n pass\n\n\nclass Stage(AbstractStage):\n \"\"\"Manages a temporary stage directory for building.\n\n A Stage object is a context manager that handles a directory where\n some source code is downloaded and built before being installed.\n It handles fetching the source code, either as an archive to be\n expanded or by checking it out of a repository. A stage's\n lifecycle looks like this::\n\n with Stage() as stage: # Context manager creates and destroys the\n # stage directory\n stage.fetch() # Fetch a source archive into the stage.\n stage.expand_archive() # Expand the archive into source_path.\n # Build and install the archive.\n # (handled by user of Stage)\n\n When used as a context manager, the stage is automatically\n destroyed if no exception is raised by the context. If an\n exception is raised, the stage is left in the filesystem and NOT\n destroyed, for potential reuse later.\n\n You can also use the stage's create/destroy functions manually,\n like this::\n\n stage = Stage()\n try:\n stage.create() # Explicitly create the stage directory.\n stage.fetch() # Fetch a source archive into the stage.\n stage.expand_archive() # Expand the archive into source_path.\n # Build and install the archive.\n # (handled by user of Stage)\n finally:\n stage.destroy() # Explicitly destroy the stage directory.\n\n There are two kinds of stages: named and unnamed. Named stages\n can persist between runs of spack, e.g. if you fetched a tarball\n but didn't finish building it, you won't have to fetch it again.\n\n Unnamed stages are created using standard mkdtemp mechanisms or\n similar, and are intended to persist for only one run of spack.\n \"\"\"\n\n requires_patch_success = True\n\n def __init__(\n self,\n url_or_fetch_strategy,\n *,\n name=None,\n mirror_paths: Optional[\"spack.mirrors.layout.MirrorLayout\"] = None,\n mirrors: Optional[Iterable[\"spack.mirrors.mirror.Mirror\"]] = None,\n keep=False,\n path=None,\n lock=True,\n search_fn=None,\n ):\n \"\"\"Create a stage object.\n Parameters:\n url_or_fetch_strategy\n URL of the archive to be downloaded into this stage, OR a valid FetchStrategy.\n\n name\n If a name is provided, then this stage is a named stage and will persist between runs\n (or if you construct another stage object later). If name is not provided, then this\n stage will be given a unique name automatically.\n\n mirror_paths\n If provided, Stage will search Spack's mirrors for this archive at each of the\n provided relative mirror paths before using the default fetch strategy.\n\n keep\n By default, when used as a context manager, the Stage is deleted on exit when no\n exceptions are raised. Pass True to keep the stage intact even if no exceptions are\n raised.\n\n path\n If provided, the stage path to use for associated builds.\n\n lock\n True if the stage directory file lock is to be used, False otherwise.\n\n search_fn\n The search function that provides the fetch strategy instance.\n \"\"\"\n super().__init__(name, path, keep, lock)\n\n # TODO: fetch/stage coupling needs to be reworked -- the logic\n # TODO: here is convoluted and not modular enough.\n if isinstance(url_or_fetch_strategy, str):\n self.fetcher = fs.from_url_scheme(url_or_fetch_strategy)\n elif isinstance(url_or_fetch_strategy, fs.FetchStrategy):\n self.fetcher = url_or_fetch_strategy\n else:\n raise ValueError(\"Can't construct Stage without url or fetch strategy\")\n self.fetcher.stage = self\n # self.fetcher can change with mirrors.\n self.default_fetcher = self.fetcher\n self.search_fn = search_fn\n # If we fetch from a mirror, but the original data is from say git, we can currently not\n # prove that they are equal (we don't even have a tree hash in package.py). This bool is\n # used to skip checksum verification and instead warn the user.\n if isinstance(self.default_fetcher, fs.URLFetchStrategy):\n self.skip_checksum_for_mirror = not bool(self.default_fetcher.digest)\n else:\n self.skip_checksum_for_mirror = True\n\n self.srcdir = None\n\n self.mirror_layout = mirror_paths\n self.mirrors = list(mirrors) if mirrors else []\n # Allow users the disable both mirrors and download cache\n self.default_fetcher_only = False\n\n @property\n def expected_archive_files(self) -> List[str]:\n \"\"\"Possible archive file paths.\"\"\"\n fnames: List[str] = []\n expanded = True\n if isinstance(self.default_fetcher, fs.URLFetchStrategy):\n expanded = self.default_fetcher.expand_archive\n fnames.append(url_util.default_download_filename(self.default_fetcher.url))\n\n if self.mirror_layout:\n fnames.append(os.path.basename(self.mirror_layout.path))\n\n paths = [os.path.join(self.path, f) for f in fnames]\n if not expanded:\n # If the download file is not compressed, the \"archive\" is a single file placed in\n # Stage.source_path\n paths.extend(os.path.join(self.source_path, f) for f in fnames)\n\n return paths\n\n @property\n def save_filename(self):\n possible_filenames = self.expected_archive_files\n if possible_filenames:\n # This prefers using the URL associated with the default fetcher if\n # available, so that the fetched resource name matches the remote\n # name\n return possible_filenames[0]\n\n @property\n def archive_file(self) -> Optional[str]:\n \"\"\"Path to the source archive within this stage directory.\"\"\"\n for path in self.expected_archive_files:\n if os.path.exists(path):\n return path\n else:\n return None\n\n @property\n def expanded(self):\n \"\"\"Returns True if source path expanded; else False.\"\"\"\n return os.path.exists(self.source_path)\n\n @property\n def source_path(self):\n \"\"\"Returns the well-known source directory path.\"\"\"\n return os.path.join(self.path, _source_path_subdir)\n\n @property\n def single_file(self):\n assert self.expanded, \"Must expand stage before calling single_file\"\n files = os.listdir(self.source_path)\n assert len(files) == 1, f\"Expected one file in stage, found {files}\"\n return os.path.join(self.source_path, files[0])\n\n def _generate_fetchers(self, mirror_only=False) -> Generator[\"fs.FetchStrategy\", None, None]:\n fetchers: List[fs.FetchStrategy] = []\n if not mirror_only:\n fetchers.append(self.default_fetcher)\n\n # If this archive is normally fetched from a URL, then use the same digest.\n if isinstance(self.default_fetcher, fs.URLFetchStrategy):\n digest = self.default_fetcher.digest\n expand = self.default_fetcher.expand_archive\n extension = self.default_fetcher.extension\n else:\n digest = None\n expand = True\n extension = None\n\n # TODO: move mirror logic out of here and clean it up!\n # TODO: Or @alalazo may have some ideas about how to use a\n # TODO: CompositeFetchStrategy here.\n if not self.default_fetcher_only and self.mirror_layout and self.mirrors:\n # Add URL strategies for all the mirrors with the digest\n # Insert fetchers in the order that the URLs are provided.\n fetchers[:0] = (\n fs.from_url_scheme(\n url_util.join(mirror.fetch_url, *self.mirror_layout.path.split(os.sep)),\n checksum=digest,\n expand=expand,\n extension=extension,\n )\n for mirror in self.mirrors\n if not spack.oci.image.is_oci_url(mirror.fetch_url) # no support for mirrors yet\n )\n\n if not self.default_fetcher_only and self.mirror_layout and self.default_fetcher.cachable:\n fetchers.insert(\n 0,\n spack.caches.FETCH_CACHE.fetcher(\n self.mirror_layout.path, digest, expand=expand, extension=extension\n ),\n )\n\n yield from fetchers\n\n # The search function may be expensive, so wait until now to call it so the user can stop\n # if a prior fetcher succeeded\n if self.search_fn and not mirror_only:\n yield from self.search_fn()\n\n def fetch(self, mirror_only: bool = False, err_msg: Optional[str] = None) -> None:\n \"\"\"Retrieves the code or archive\n\n Args:\n mirror_only: only fetch from a mirror\n err_msg: the error message to display if all fetchers fail or ``None`` for the default\n fetch failure message\n \"\"\"\n errors: List[str] = []\n for fetcher in self._generate_fetchers(mirror_only):\n try:\n fetcher.stage = self\n self.fetcher = fetcher\n self.fetcher.fetch()\n break\n except fs.NoCacheError:\n # Don't bother reporting when something is not cached.\n continue\n except fs.FailedDownloadError as f:\n errors.extend(f\"{fetcher}: {e.__class__.__name__}: {e}\" for e in f.exceptions)\n continue\n except spack.error.SpackError as e:\n errors.append(f\"{fetcher}: {e.__class__.__name__}: {e}\")\n continue\n else:\n self.fetcher = self.default_fetcher\n if err_msg:\n raise spack.error.FetchError(err_msg)\n raise spack.error.FetchError(\n f\"All fetchers failed for {self.name}\", \"\\n\".join(f\" {e}\" for e in errors)\n )\n\n def steal_source(self, dest):\n \"\"\"Copy the source_path directory in its entirety to directory dest\n\n This operation creates/fetches/expands the stage if it is not already,\n and destroys the stage when it is done.\"\"\"\n if not self.created:\n self.create()\n if not self.expanded and not self.archive_file:\n self.fetch()\n if not self.expanded:\n self.expand_archive()\n\n if not os.path.isdir(dest):\n mkdirp(dest)\n\n # glob all files and directories in the source path\n hidden_entries = glob.glob(os.path.join(self.source_path, \".*\"))\n entries = glob.glob(os.path.join(self.source_path, \"*\"))\n\n # Move all files from stage to destination directory\n # Include hidden files for VCS repo history\n for entry in hidden_entries + entries:\n if os.path.isdir(entry):\n d = os.path.join(dest, os.path.basename(entry))\n shutil.copytree(entry, d, symlinks=True)\n else:\n shutil.copy2(entry, dest)\n\n # copy archive file if we downloaded from url -- replaces for vcs\n if self.archive_file and os.path.exists(self.archive_file):\n shutil.copy2(self.archive_file, dest)\n\n # remove leftover stage\n self.destroy()\n\n def check(self):\n \"\"\"Check the downloaded archive against a checksum digest.\"\"\"\n if self.fetcher is not self.default_fetcher and self.skip_checksum_for_mirror:\n cache = isinstance(self.fetcher, fs.CacheURLFetchStrategy)\n if cache:\n secure_msg = \"your download cache is in a secure location\"\n else:\n secure_msg = \"you trust this mirror and have a secure connection\"\n tty.warn(\n f\"Using {'download cache' if cache else 'a mirror'} instead of version control\",\n \"The required sources are normally checked out from a version control system, \"\n f\"but have been archived {'in download cache' if cache else 'on a mirror'}: \"\n f\"{self.fetcher}. Spack lacks a tree hash to verify the integrity of this \"\n f\"archive. Make sure {secure_msg}.\",\n )\n elif spack.config.get(\"config:checksum\"):\n self.fetcher.check()\n\n def cache_local(self):\n spack.caches.FETCH_CACHE.store(self.fetcher, self.mirror_layout.path)\n\n def cache_mirror(\n self,\n mirror: \"spack.caches.MirrorCache\",\n stats: \"spack.mirrors.utils.MirrorStatsForOneSpec\",\n ) -> None:\n \"\"\"Perform a fetch if the resource is not already cached\n\n Arguments:\n mirror: the mirror to cache this Stage's resource in\n stats: this is updated depending on whether the caching operation succeeded or failed\n \"\"\"\n if isinstance(self.default_fetcher, fs.BundleFetchStrategy):\n # BundleFetchStrategy has no source to fetch. The associated fetcher does nothing but\n # the associated stage may still exist. There is currently no method available on the\n # fetcher to distinguish this ('cachable' refers to whether the fetcher refers to a\n # resource with a fixed ID, which is not the same concept as whether there is anything\n # to fetch at all) so we must examine the type of the fetcher.\n return\n\n elif mirror.skip_unstable_versions and not fs.stable_target(self.default_fetcher):\n return\n\n elif not self.mirror_layout:\n return\n\n absolute_storage_path = os.path.join(mirror.root, self.mirror_layout.path)\n\n if os.path.exists(absolute_storage_path):\n stats.already_existed(absolute_storage_path)\n else:\n self.fetch()\n self.check()\n mirror.store(self.fetcher, self.mirror_layout.path)\n stats.added(absolute_storage_path)\n\n self.mirror_layout.make_alias(mirror.root)\n\n def expand_archive(self):\n \"\"\"Changes to the stage directory and attempt to expand the downloaded\n archive. Fail if the stage is not set up or if the archive is not yet\n downloaded.\"\"\"\n if not self.expanded:\n self.fetcher.expand()\n tty.debug(f\"Created stage in {self.path}\")\n else:\n tty.debug(f\"Already staged {self.name} in {self.path}\")\n\n def restage(self):\n \"\"\"Removes the expanded archive path if it exists, then re-expands\n the archive.\n \"\"\"\n self.fetcher.reset()\n\n def destroy(self):\n \"\"\"Removes this stage directory.\"\"\"\n remove_linked_tree(self.path)\n\n # Make sure we don't end up in a removed directory\n try:\n os.getcwd()\n except OSError as e:\n tty.debug(e)\n os.chdir(os.path.dirname(self.path))\n\n # mark as destroyed\n self.created = False\n\n\nclass ResourceStage(Stage):\n def __init__(\n self,\n fetch_strategy: \"fs.FetchStrategy\",\n root: Stage,\n resource: spack.resource.Resource,\n *,\n name=None,\n mirror_paths: Optional[\"spack.mirrors.layout.MirrorLayout\"] = None,\n mirrors: Optional[Iterable[\"spack.mirrors.mirror.Mirror\"]] = None,\n keep=False,\n path=None,\n lock=True,\n search_fn=None,\n ):\n super().__init__(\n fetch_strategy,\n name=name,\n mirror_paths=mirror_paths,\n mirrors=mirrors,\n keep=keep,\n path=path,\n lock=lock,\n search_fn=search_fn,\n )\n self.root_stage = root\n self.resource = resource\n\n def restage(self):\n super().restage()\n self._add_to_root_stage()\n\n def expand_archive(self):\n super().expand_archive()\n self._add_to_root_stage()\n\n def _add_to_root_stage(self):\n \"\"\"\n Move the extracted resource to the root stage (according to placement).\n \"\"\"\n root_stage = self.root_stage\n resource = self.resource\n\n if resource.placement:\n placement = resource.placement\n elif self.srcdir:\n placement = self.srcdir\n else:\n placement = self.source_path\n\n if not isinstance(placement, dict):\n placement = {\"\": placement}\n\n target_path = os.path.join(root_stage.source_path, resource.destination)\n\n try:\n os.makedirs(target_path)\n except OSError as err:\n tty.debug(err)\n if err.errno == errno.EEXIST and os.path.isdir(target_path):\n pass\n else:\n raise\n\n for key, value in placement.items():\n destination_path = os.path.join(target_path, value)\n source_path = os.path.join(self.source_path, key)\n\n if not os.path.exists(destination_path):\n tty.info(\n \"Moving resource stage\\n\\tsource: \"\n \"{stage}\\n\\tdestination: {destination}\".format(\n stage=source_path, destination=destination_path\n )\n )\n\n src = os.path.realpath(source_path)\n\n if os.path.isdir(src):\n install_tree(src, destination_path)\n else:\n install(src, destination_path)\n\n\nclass StageComposite:\n \"\"\"Composite for Stage type objects. The first item in this composite is\n considered to be the root package, and operations that return a value are\n forwarded to it.\"\"\"\n\n def __init__(self):\n self._stages: List[AbstractStage] = []\n\n @classmethod\n def from_iterable(cls, iterable: Iterable[AbstractStage]) -> \"StageComposite\":\n \"\"\"Create a new composite from an iterable of stages.\"\"\"\n composite = cls()\n composite.extend(iterable)\n return composite\n\n def append(self, stage: AbstractStage) -> None:\n \"\"\"Add a stage to the composite.\"\"\"\n self._stages.append(stage)\n\n def extend(self, stages: Iterable[AbstractStage]) -> None:\n \"\"\"Add multiple stages to the composite.\"\"\"\n self._stages.extend(stages)\n\n def __iter__(self):\n \"\"\"Iterate over stages.\"\"\"\n return iter(self._stages)\n\n def __len__(self):\n \"\"\"Return the number of stages.\"\"\"\n return len(self._stages)\n\n def __getitem__(self, index):\n \"\"\"Get a stage by index.\"\"\"\n return self._stages[index]\n\n # Context manager methods - delegate to all stages\n def __enter__(self):\n for stage in self._stages:\n stage.__enter__()\n return self\n\n def __exit__(self, exc_type, exc_val, exc_tb):\n for stage in reversed(self._stages):\n stage.__exit__(exc_type, exc_val, exc_tb)\n\n # Methods that delegate to all stages\n def fetch(self, mirror_only: bool = False, err_msg: Optional[str] = None) -> None:\n \"\"\"Fetch all stages.\"\"\"\n for stage in self._stages:\n stage.fetch(mirror_only, err_msg)\n\n def create(self) -> None:\n \"\"\"Create all stages.\"\"\"\n for stage in self._stages:\n stage.create()\n\n def check(self) -> None:\n \"\"\"Check all stages.\"\"\"\n for stage in self._stages:\n stage.check()\n\n def expand_archive(self) -> None:\n \"\"\"Expand archives for all stages.\"\"\"\n for stage in self._stages:\n stage.expand_archive()\n\n def restage(self) -> None:\n \"\"\"Restage all stages.\"\"\"\n for stage in self._stages:\n stage.restage()\n\n def destroy(self) -> None:\n \"\"\"Destroy all stages.\"\"\"\n for stage in self._stages:\n stage.destroy()\n\n def cache_local(self) -> None:\n \"\"\"Cache all stages locally.\"\"\"\n for stage in self._stages:\n stage.cache_local()\n\n def cache_mirror(\n self,\n mirror: \"spack.caches.MirrorCache\",\n stats: \"spack.mirrors.utils.MirrorStatsForOneSpec\",\n ) -> None:\n \"\"\"Cache all stages to mirror.\"\"\"\n for stage in self._stages:\n stage.cache_mirror(mirror, stats)\n\n def steal_source(self, dest: str) -> None:\n \"\"\"Steal source from all stages.\"\"\"\n for stage in self._stages:\n stage.steal_source(dest)\n\n def disable_mirrors(self) -> None:\n \"\"\"Disable mirrors for all stages that support it.\"\"\"\n for stage in self._stages:\n if isinstance(stage, Stage):\n stage.default_fetcher_only = True\n\n # Properties that act only on the *first* stage in the composite\n @property\n def source_path(self):\n return self._stages[0].source_path\n\n @property\n def expanded(self):\n return self._stages[0].expanded\n\n @property\n def path(self):\n return self._stages[0].path\n\n @property\n def archive_file(self):\n return self._stages[0].archive_file\n\n @property\n def requires_patch_success(self):\n return self._stages[0].requires_patch_success\n\n @property\n def keep(self):\n return self._stages[0].keep\n\n @keep.setter\n def keep(self, value):\n for stage in self._stages:\n stage.keep = value\n\n\nclass DevelopStage(AbstractStage):\n requires_patch_success = False\n\n def __init__(self, name, dev_path, reference_link):\n super().__init__(name=name, path=None, keep=False, lock=True)\n self.dev_path = dev_path\n self._source_path = dev_path\n\n # The path of a link that will point to this stage\n if reference_link:\n if os.path.isabs(reference_link):\n link_path = reference_link\n else:\n link_path = os.path.join(self._source_path, reference_link)\n if not os.path.isdir(os.path.dirname(link_path)):\n raise StageError(f\"The directory containing {link_path} must exist\")\n self.reference_link = link_path\n else:\n self.reference_link = None\n\n @property\n def source_path(self):\n \"\"\"Returns the development source path.\"\"\"\n return self._source_path\n\n @property\n def archive_file(self):\n return None\n\n def fetch(self, mirror_only: bool = False, err_msg: Optional[str] = None) -> None:\n tty.debug(\"No fetching needed for develop stage.\")\n\n def check(self):\n tty.debug(\"No checksum needed for develop stage.\")\n\n def expand_archive(self):\n tty.debug(\"No expansion needed for develop stage.\")\n\n @property\n def expanded(self):\n \"\"\"Returns True since the source_path must exist.\"\"\"\n return True\n\n def create(self):\n super().create()\n if self.reference_link:\n try:\n symlink(self.path, self.reference_link)\n except (AlreadyExistsError, FileExistsError):\n pass\n\n def destroy(self):\n # Destroy all files, but do not follow symlinks\n try:\n shutil.rmtree(self.path)\n except FileNotFoundError:\n pass\n if self.reference_link:\n try:\n os.remove(self.reference_link)\n except FileNotFoundError:\n pass\n self.created = False\n\n def restage(self):\n self.destroy()\n self.create()\n\n def cache_local(self):\n tty.debug(\"Sources for Develop stages are not cached\")\n\n\ndef ensure_access(file):\n \"\"\"Ensure we can access a directory and die with an error if we can't.\"\"\"\n if not can_access(file):\n tty.die(\"Insufficient permissions for %s\" % file)\n\n\ndef purge():\n \"\"\"Remove all build directories in the top-level stage path.\"\"\"\n root = get_stage_root()\n if os.path.isdir(root):\n for stage_dir in os.listdir(root):\n if stage_dir.startswith(stage_prefix) or stage_dir == \".lock\":\n stage_path = os.path.join(root, stage_dir)\n if os.path.isdir(stage_path):\n remove_linked_tree(stage_path)\n else:\n os.remove(stage_path)\n\n\ndef interactive_version_filter(\n url_dict: Dict[StandardVersion, str],\n known_versions: Iterable[StandardVersion] = (),\n *,\n initial_verion_filter: Optional[VersionList] = None,\n url_changes: Set[StandardVersion] = set(),\n input: Callable[..., str] = input,\n) -> Optional[Dict[StandardVersion, str]]:\n \"\"\"Interactively filter the list of spidered versions.\n\n Args:\n url_dict: Dictionary of versions to URLs\n known_versions: Versions that can be skipped because they are already known\n\n Returns:\n Filtered dictionary of versions to URLs or None if the user wants to quit\n \"\"\"\n # Find length of longest string in the list for padding\n version_filter = initial_verion_filter or VersionList([\":\"])\n max_len = max(len(str(v)) for v in url_dict) if url_dict else 0\n sorted_and_filtered = [v for v in url_dict if v.satisfies(version_filter)]\n sorted_and_filtered.sort(reverse=True)\n orig_url_dict = url_dict # only copy when using editor to modify\n print_header = True\n VERSION_COLOR = spack.spec.VERSION_COLOR\n while True:\n if print_header:\n has_filter = version_filter != VersionList([\":\"])\n header = []\n if len(orig_url_dict) > 0 and len(sorted_and_filtered) == len(orig_url_dict):\n header.append(\n f\"Selected {spack.llnl.string.plural(len(sorted_and_filtered), 'version')}\"\n )\n else:\n header.append(\n f\"Selected {len(sorted_and_filtered)} of \"\n f\"{spack.llnl.string.plural(len(orig_url_dict), 'version')}\"\n )\n if sorted_and_filtered and known_versions:\n num_new = sum(1 for v in sorted_and_filtered if v not in known_versions)\n header.append(f\"{spack.llnl.string.plural(num_new, 'new version')}\")\n if has_filter:\n header.append(colorize(f\"Filtered by {VERSION_COLOR}@@{version_filter}@.\"))\n\n version_with_url = [\n colorize(\n f\"{VERSION_COLOR}{str(v):{max_len}}@. {url_dict[v]}\"\n f\"{' @K{# NOTE: change of URL}' if v in url_changes else ''}\"\n )\n for v in sorted_and_filtered\n ]\n tty.msg(\". \".join(header), *spack.llnl.util.lang.elide_list(version_with_url))\n print()\n\n print_header = True\n\n tty.info(colorize(\"Enter @*{number} of versions to take, or use a @*{command}:\"))\n commands = (\n \"@*b{[c]}hecksum\",\n \"@*b{[e]}dit\",\n \"@*b{[f]}ilter\",\n \"@*b{[a]}sk each\",\n \"@*b{[n]}ew only\",\n \"@*b{[r]}estart\",\n \"@*b{[q]}uit\",\n )\n colify(list(map(colorize, commands)), indent=4)\n\n try:\n command = input(colorize(\"@*g{action>} \")).strip().lower()\n except EOFError:\n print()\n command = \"q\"\n\n if command == \"c\":\n break\n elif command == \"e\":\n # Create a temporary file in the stage dir with lines of the form\n # \n # which the user can modify. Once the editor is closed, the file is\n # read back in and the versions to url dict is updated.\n\n # Create a temporary file by hashing its contents.\n buffer = io.StringIO()\n buffer.write(\"# Edit this file to change the versions and urls to fetch\\n\")\n for v in sorted_and_filtered:\n buffer.write(f\"{str(v):{max_len}} {url_dict[v]}\\n\")\n data = buffer.getvalue().encode(\"utf-8\")\n\n short_hash = hashlib.sha1(data).hexdigest()[:7]\n filename = f\"{stage_prefix}versions-{short_hash}.txt\"\n filepath = os.path.join(get_stage_root(), filename)\n\n # Write contents\n with open(filepath, \"wb\") as f:\n f.write(data)\n\n # Open editor\n editor(filepath, exec_fn=executable)\n\n # Read back in\n with open(filepath, \"r\", encoding=\"utf-8\") as f:\n orig_url_dict, url_dict = url_dict, {}\n for line in f:\n line = line.strip()\n # Skip empty lines and comments\n if not line or line.startswith(\"#\"):\n continue\n try:\n version, url = line.split(None, 1)\n except ValueError:\n tty.warn(f\"Couldn't parse: {line}\")\n continue\n try:\n url_dict[StandardVersion.from_string(version)] = url\n except ValueError:\n tty.warn(f\"Invalid version: {version}\")\n continue\n sorted_and_filtered = sorted(url_dict.keys(), reverse=True)\n\n os.unlink(filepath)\n elif command == \"f\":\n tty.msg(\n colorize(\n f\"Examples filters: {VERSION_COLOR}1.2@. \"\n f\"or {VERSION_COLOR}1.1:1.3@. \"\n f\"or {VERSION_COLOR}=1.2, 1.2.2:@.\"\n )\n )\n try:\n # Allow a leading @ version specifier\n filter_spec = input(colorize(\"@*g{filter>} \")).strip().lstrip(\"@\")\n except EOFError:\n print()\n continue\n try:\n version_filter.intersect(VersionList([filter_spec]))\n except ValueError:\n tty.warn(f\"Invalid version specifier: {filter_spec}\")\n continue\n # Apply filter\n sorted_and_filtered = [v for v in sorted_and_filtered if v.satisfies(version_filter)]\n elif command == \"a\":\n i = 0\n while i < len(sorted_and_filtered):\n v = sorted_and_filtered[i]\n try:\n answer = input(f\" {str(v):{max_len}} {url_dict[v]} [Y/n]? \").strip().lower()\n except EOFError:\n # If ^D, don't fully exit, but go back to the command prompt, now with possibly\n # fewer versions\n print()\n break\n if answer in (\"n\", \"no\"):\n del sorted_and_filtered[i]\n elif answer in (\"y\", \"yes\", \"\"):\n i += 1\n else:\n # Went over each version, so go to checksumming\n break\n elif command == \"n\":\n sorted_and_filtered = [v for v in sorted_and_filtered if v not in known_versions]\n elif command == \"r\":\n url_dict = orig_url_dict\n sorted_and_filtered = sorted(url_dict.keys(), reverse=True)\n version_filter = VersionList([\":\"])\n elif command == \"q\":\n try:\n if input(\"Really quit [y/N]? \").strip().lower() in (\"y\", \"yes\"):\n return None\n except EOFError:\n print()\n return None\n else:\n # Last restort: filter the top N versions\n try:\n n = int(command)\n invalid_command = n < 1\n except ValueError:\n invalid_command = True\n\n if invalid_command:\n tty.warn(f\"Ignoring invalid command: {command}\")\n print_header = False\n continue\n\n sorted_and_filtered = sorted_and_filtered[:n]\n\n return {v: url_dict[v] for v in sorted_and_filtered}\n\n\ndef get_checksums_for_versions(\n url_by_version: Dict[StandardVersion, str],\n package_name: str,\n *,\n first_stage_function: Optional[Callable[[str, str], None]] = None,\n keep_stage: bool = False,\n concurrency: Optional[int] = None,\n fetch_options: Optional[Dict[str, str]] = None,\n) -> Dict[StandardVersion, str]:\n \"\"\"Computes the checksums for each version passed in input, and returns the results.\n\n Archives are fetched according to the usl dictionary passed as input.\n\n The ``first_stage_function`` argument allows the caller to inspect the first downloaded\n archive, e.g., to determine the build system.\n\n Args:\n url_by_version: URL keyed by version\n package_name: name of the package\n first_stage_function: function that takes an archive file and a URL; this is run on the\n stage of the first URL downloaded\n keep_stage: whether to keep staging area when command completes\n batch: whether to ask user how many versions to fetch (false) or fetch all versions (true)\n fetch_options: options used for the fetcher (such as timeout or cookies)\n concurrency: maximum number of workers to use for retrieving archives\n\n Returns:\n A dictionary mapping each version to the corresponding checksum\n \"\"\"\n versions = sorted(url_by_version.keys(), reverse=True)\n search_arguments = [(url_by_version[v], v) for v in versions]\n\n version_hashes: Dict[StandardVersion, str] = {}\n errors: List[str] = []\n\n # Don't spawn 16 processes when we need to fetch 2 urls\n if concurrency is not None:\n concurrency = min(concurrency, len(search_arguments))\n else:\n concurrency = min(os.cpu_count() or 1, len(search_arguments))\n\n # The function might have side effects in memory, that would not be reflected in the\n # parent process, if run in a child process. If this pattern happens frequently, we\n # can move this function call *after* having distributed the work to executors.\n if first_stage_function is not None:\n (url, version), search_arguments = search_arguments[0], search_arguments[1:]\n result = _fetch_and_checksum(url, fetch_options, keep_stage, first_stage_function)\n if isinstance(result, Exception):\n errors.append(str(result))\n else:\n version_hashes[version] = result\n\n with spack.util.parallel.make_concurrent_executor(concurrency, require_fork=False) as executor:\n results = [\n (version, executor.submit(_fetch_and_checksum, url, fetch_options, keep_stage))\n for url, version in search_arguments\n ]\n\n for version, future in results:\n result = future.result()\n if isinstance(result, Exception):\n errors.append(str(result))\n else:\n version_hashes[version] = result\n\n for msg in errors:\n tty.debug(msg)\n\n if not version_hashes:\n tty.die(f\"Could not fetch any versions for {package_name}\")\n\n num_hash = len(version_hashes)\n tty.debug(f\"Checksummed {num_hash} version{'' if num_hash == 1 else 's'} of {package_name}:\")\n\n return version_hashes\n\n\ndef _fetch_and_checksum(\n url: str,\n options: Optional[dict],\n keep_stage: bool,\n action_fn: Optional[Callable[[str, str], None]] = None,\n) -> Union[str, Exception]:\n try:\n with Stage(fs.URLFetchStrategy(url=url, fetch_options=options), keep=keep_stage) as stage:\n # Fetch the archive\n stage.fetch()\n archive = stage.archive_file\n assert archive is not None, f\"Archive not found for {url}\"\n if action_fn is not None and archive:\n # Only run first_stage_function the first time,\n # no need to run it every time\n action_fn(archive, url)\n\n # Checksum the archive and add it to the list\n checksum = spack.util.crypto.checksum(hashlib.sha256, archive)\n return checksum\n except Exception as e:\n return Exception(f\"[WORKER] Failed to fetch {url}: {e}\")\n\n\nclass StageError(spack.error.SpackError):\n \"\"\"Superclass for all errors encountered during staging.\"\"\"\n\n\nclass StagePathError(StageError):\n \"\"\"Error encountered with stage path.\"\"\"\n\n\nclass RestageError(StageError):\n \"\"\"Error encountered during restaging.\"\"\"\n\n\nclass VersionFetchError(StageError):\n \"\"\"Raised when we can't determine a URL to fetch a package.\"\"\"\n" }, { "path": "lib/spack/spack/store.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Components that manage Spack's installation tree.\n\nAn install tree, or \"build store\" consists of two parts:\n\n1. A package database that tracks what is installed.\n2. A directory layout that determines how the installations are laid out.\n\nThe store contains all the install prefixes for packages installed by\nSpack. The simplest store could just contain prefixes named by DAG hash,\nbut we use a fancier directory layout to make browsing the store and\ndebugging easier.\n\n\"\"\"\n\nimport contextlib\nimport os\nimport pathlib\nimport re\nimport uuid\nfrom typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Union, cast\n\nimport spack.config\nimport spack.database\nimport spack.directory_layout\nimport spack.error\nimport spack.llnl.util.lang\nimport spack.paths\nimport spack.spec\nimport spack.util.path\nfrom spack.llnl.util import tty\n\n#: default installation root, relative to the Spack install path\nDEFAULT_INSTALL_TREE_ROOT = os.path.join(spack.paths.opt_path, \"spack\")\n\n\ndef parse_install_tree(config_dict: dict) -> Tuple[str, str, Dict[str, str]]:\n \"\"\"Parse config settings and return values relevant to the store object.\n\n Arguments:\n config_dict: dictionary of config values, as returned from ``spack.config.get(\"config\")``\n\n Returns:\n triple of the install tree root, the unpadded install tree\n root (before padding was applied), and the projections for the\n install tree\n\n Encapsulate backwards compatibility capabilities for install_tree\n and deprecated values that are now parsed as part of install_tree.\n \"\"\"\n # The following two configs are equivalent, the first being the old format\n # and the second the new format. The new format is also more flexible.\n\n # config:\n # install_tree: /path/to/root$padding:128\n # install_path_scheme: '{name}-{version}'\n\n # config:\n # install_tree:\n # root: /path/to/root\n # padding: 128\n # projections:\n # all: '{name}-{version}'\n\n install_tree = config_dict.get(\"install_tree\", {})\n\n padded_length: Union[bool, int] = False\n if isinstance(install_tree, str):\n tty.warn(\"Using deprecated format for configuring install_tree\")\n unpadded_root = install_tree\n unpadded_root = spack.util.path.canonicalize_path(unpadded_root)\n # construct projection from previous values for backwards compatibility\n all_projection = config_dict.get(\n \"install_path_scheme\", spack.directory_layout.default_projections[\"all\"]\n )\n\n projections = {\"all\": all_projection}\n else:\n unpadded_root = install_tree.get(\"root\", DEFAULT_INSTALL_TREE_ROOT)\n unpadded_root = spack.util.path.canonicalize_path(unpadded_root)\n\n padded_length = install_tree.get(\"padded_length\", False)\n if padded_length is True:\n padded_length = spack.util.path.get_system_path_max()\n padded_length -= spack.util.path.SPACK_MAX_INSTALL_PATH_LENGTH\n\n projections = install_tree.get(\"projections\", spack.directory_layout.default_projections)\n\n path_scheme = config_dict.get(\"install_path_scheme\", None)\n if path_scheme:\n tty.warn(\n \"Deprecated config value 'install_path_scheme' ignored\"\n \" when using new install_tree syntax\"\n )\n\n # Handle backwards compatibility for padding\n old_pad = re.search(r\"\\$padding(:\\d+)?|\\${padding(:\\d+)?}\", unpadded_root)\n if old_pad:\n if padded_length:\n msg = \"Ignoring deprecated padding option in install_tree root \"\n msg += \"because new syntax padding is present.\"\n tty.warn(msg)\n else:\n unpadded_root = unpadded_root.replace(old_pad.group(0), \"\")\n if old_pad.group(1) or old_pad.group(2):\n length_group = 2 if \"{\" in old_pad.group(0) else 1\n padded_length = int(old_pad.group(length_group)[1:])\n else:\n padded_length = spack.util.path.get_system_path_max()\n padded_length -= spack.util.path.SPACK_MAX_INSTALL_PATH_LENGTH\n\n unpadded_root = unpadded_root.rstrip(os.path.sep)\n\n if padded_length:\n root = spack.util.path.add_padding(unpadded_root, padded_length)\n if len(root) != padded_length:\n msg = \"Cannot pad %s to %s characters.\" % (root, padded_length)\n msg += \" It is already %s characters long\" % len(root)\n tty.warn(msg)\n else:\n root = unpadded_root\n\n return root, unpadded_root, projections\n\n\nclass Store:\n \"\"\"A store is a path full of installed Spack packages.\n\n Stores consist of packages installed according to a ``DirectoryLayout``, along with a database\n of their contents.\n\n The directory layout controls what paths look like and how Spack ensures that each unique spec\n gets its own unique directory (or not, though we don't recommend that).\n\n The database is a single file that caches metadata for the entire Spack installation. It\n prevents us from having to spider the install tree to figure out what's there.\n\n The store is also able to lock installation prefixes, and to mark installation failures.\n\n Args:\n root: path to the root of the install tree\n unpadded_root: path to the root of the install tree without padding. The sbang script has\n to be installed here to work with padded roots\n projections: expression according to guidelines that describes how to construct a path to\n a package prefix in this store\n hash_length: length of the hashes used in the directory layout. Spec hash suffixes will be\n truncated to this length\n upstreams: optional list of upstream databases\n lock_cfg: lock configuration for the database\n \"\"\"\n\n def __init__(\n self,\n root: str,\n unpadded_root: Optional[str] = None,\n projections: Optional[Dict[str, str]] = None,\n hash_length: Optional[int] = None,\n upstreams: Optional[List[spack.database.Database]] = None,\n lock_cfg: spack.database.LockConfiguration = spack.database.NO_LOCK,\n ) -> None:\n self.root = root\n self.unpadded_root = unpadded_root or root\n self.projections = projections\n self.hash_length = hash_length\n self.upstreams = upstreams\n self.lock_cfg = lock_cfg\n self.layout = spack.directory_layout.DirectoryLayout(\n root, projections=projections, hash_length=hash_length\n )\n self.db = spack.database.Database(\n root, upstream_dbs=upstreams, lock_cfg=lock_cfg, layout=self.layout\n )\n\n timeout_format_str = (\n f\"{str(lock_cfg.package_timeout)}s\" if lock_cfg.package_timeout else \"No timeout\"\n )\n tty.debug(\"PACKAGE LOCK TIMEOUT: {0}\".format(str(timeout_format_str)))\n\n self.prefix_locker = spack.database.SpecLocker(\n spack.database.prefix_lock_path(root), default_timeout=lock_cfg.package_timeout\n )\n self.failure_tracker = spack.database.FailureTracker(\n self.root, default_timeout=lock_cfg.package_timeout\n )\n\n def has_padding(self) -> bool:\n \"\"\"Returns True if the store layout includes path padding.\"\"\"\n return self.root != self.unpadded_root\n\n def reindex(self) -> None:\n \"\"\"Convenience function to reindex the store DB with its own layout.\"\"\"\n return self.db.reindex()\n\n def __reduce__(self):\n return Store, (\n self.root,\n self.unpadded_root,\n self.projections,\n self.hash_length,\n self.upstreams,\n self.lock_cfg,\n )\n\n\ndef create(configuration: spack.config.Configuration) -> Store:\n \"\"\"Create a store from the configuration passed as input.\n\n Args:\n configuration: configuration to create a store.\n \"\"\"\n configuration = configuration or spack.config.CONFIG\n config_dict = configuration.get_config(\"config\")\n root, unpadded_root, projections = parse_install_tree(config_dict)\n hash_length = config_dict.get(\"install_hash_length\")\n\n install_roots = [\n install_properties[\"install_tree\"]\n for install_properties in configuration.get_config(\"upstreams\").values()\n ]\n upstreams = _construct_upstream_dbs_from_install_roots(install_roots)\n\n return Store(\n root=root,\n unpadded_root=unpadded_root,\n projections=projections,\n hash_length=hash_length,\n upstreams=upstreams,\n lock_cfg=spack.database.lock_configuration(configuration),\n )\n\n\ndef _create_global() -> Store:\n result = create(configuration=spack.config.CONFIG)\n return result\n\n\n#: Singleton store instance\nSTORE = cast(Store, spack.llnl.util.lang.Singleton(_create_global))\n\n\ndef reinitialize():\n \"\"\"Restore globals to the same state they would have at start-up. Return a token\n containing the state of the store before reinitialization.\n \"\"\"\n global STORE\n\n token = STORE\n STORE = cast(Store, spack.llnl.util.lang.Singleton(_create_global))\n\n return token\n\n\ndef restore(token):\n \"\"\"Restore the environment from a token returned by reinitialize\"\"\"\n global STORE\n STORE = token\n\n\ndef _construct_upstream_dbs_from_install_roots(\n install_roots: List[str],\n) -> List[spack.database.Database]:\n accumulated_upstream_dbs: List[spack.database.Database] = []\n for install_root in reversed(install_roots):\n upstream_dbs = list(accumulated_upstream_dbs)\n next_db = spack.database.Database(\n spack.util.path.canonicalize_path(install_root),\n is_upstream=True,\n upstream_dbs=upstream_dbs,\n )\n next_db._read()\n accumulated_upstream_dbs.insert(0, next_db)\n\n return accumulated_upstream_dbs\n\n\ndef find(\n constraints: Union[str, List[str], List[\"spack.spec.Spec\"]],\n multiple: bool = False,\n query_fn: Optional[Callable[[Any], List[\"spack.spec.Spec\"]]] = None,\n **kwargs,\n) -> List[\"spack.spec.Spec\"]:\n \"\"\"Returns a list of specs matching the constraints passed as inputs.\n\n At least one spec per constraint must match, otherwise the function\n will error with an appropriate message.\n\n By default, this function queries the current store, but a custom query\n function can be passed to hit any other source of concretized specs\n (e.g. a binary cache).\n\n The query function must accept a spec as its first argument.\n\n Args:\n constraints: spec(s) to be matched against installed packages\n multiple: if True multiple matches per constraint are admitted\n query_fn (Callable): query function to get matching specs. By default,\n ``spack.store.STORE.db.query``\n **kwargs: keyword arguments forwarded to the query function\n \"\"\"\n if isinstance(constraints, str):\n constraints = [spack.spec.Spec(constraints)]\n\n matching_specs: List[spack.spec.Spec] = []\n errors = []\n query_fn = query_fn or STORE.db.query\n for spec in constraints:\n current_matches = query_fn(spec, **kwargs)\n\n # For each spec provided, make sure it refers to only one package.\n if not multiple and len(current_matches) > 1:\n msg_fmt = '\"{0}\" matches multiple packages: [{1}]'\n errors.append(msg_fmt.format(spec, \", \".join([m.format() for m in current_matches])))\n\n # No installed package matches the query\n if len(current_matches) == 0 and spec is not any:\n msg_fmt = '\"{0}\" does not match any installed packages'\n errors.append(msg_fmt.format(spec))\n\n matching_specs.extend(current_matches)\n\n if errors:\n raise MatchError(\n message=\"errors occurred when looking for specs in the store\",\n long_message=\"\\n\".join(errors),\n )\n\n return matching_specs\n\n\ndef specfile_matches(filename: str, **kwargs) -> List[\"spack.spec.Spec\"]:\n \"\"\"Same as find but reads the query from a spec file.\n\n Args:\n filename: YAML or JSON file from which to read the query.\n **kwargs: keyword arguments forwarded to :func:`find`\n \"\"\"\n query = [spack.spec.Spec.from_specfile(filename)]\n return find(query, **kwargs)\n\n\ndef ensure_singleton_created() -> None:\n \"\"\"Ensures the lazily evaluated singleton is created\"\"\"\n _ = STORE.db\n\n\n@contextlib.contextmanager\ndef use_store(\n path: Union[str, pathlib.Path], extra_data: Optional[Dict[str, Any]] = None\n) -> Generator[Store, None, None]:\n \"\"\"Use the store passed as argument within the context manager.\n\n Args:\n path: path to the store.\n extra_data: extra configuration under ``config:install_tree`` to be\n taken into account.\n\n Yields:\n Store object associated with the context manager's store\n \"\"\"\n global STORE\n\n assert not isinstance(path, Store), \"cannot pass a store anymore\"\n scope_name = \"use-store-{}\".format(uuid.uuid4())\n data = {\"root\": str(path)}\n if extra_data:\n data.update(extra_data)\n\n # Swap the store with the one just constructed and return it\n spack.config.CONFIG.push_scope(\n spack.config.InternalConfigScope(name=scope_name, data={\"config\": {\"install_tree\": data}})\n )\n temporary_store = create(configuration=spack.config.CONFIG)\n original_store, STORE = STORE, temporary_store\n\n try:\n yield temporary_store\n finally:\n # Restore the original store\n STORE = original_store\n spack.config.CONFIG.remove_scope(scope_name=scope_name)\n\n\nclass MatchError(spack.error.SpackError):\n \"\"\"Error occurring when trying to match specs in store against a constraint\"\"\"\n" }, { "path": "lib/spack/spack/subprocess_context.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"\nThis module handles transmission of Spack state to child processes started\nusing the ``\"spawn\"`` start method. Notably, installations are performed in a\nsubprocess and require transmitting the Package object (in such a way\nthat the repository is available for importing when it is deserialized);\ninstallations performed in Spack unit tests may include additional\nmodifications to global state in memory that must be replicated in the\nchild process.\n\"\"\"\n\nimport importlib\nimport io\nimport multiprocessing\nimport multiprocessing.context\nimport pickle\nfrom types import ModuleType\nfrom typing import TYPE_CHECKING, Optional\n\nimport spack.config\nimport spack.paths\nimport spack.platforms\nimport spack.repo\nimport spack.store\n\nif TYPE_CHECKING:\n import spack.package_base\n\n#: Used in tests to track monkeypatches that need to be restored in child processes\nMONKEYPATCHES: list = []\n\n\ndef serialize(pkg: \"spack.package_base.PackageBase\") -> io.BytesIO:\n serialized_pkg = io.BytesIO()\n pickle.dump(pkg, serialized_pkg)\n serialized_pkg.seek(0)\n return serialized_pkg\n\n\ndef deserialize(serialized_pkg: io.BytesIO) -> \"spack.package_base.PackageBase\":\n pkg = pickle.load(serialized_pkg)\n pkg.spec._package = pkg\n # ensure overwritten package class attributes get applied\n spack.repo.PATH.get_pkg_class(pkg.spec.name)\n return pkg\n\n\nclass SpackTestProcess:\n def __init__(self, fn):\n self.fn = fn\n\n def _restore_and_run(self, fn, test_state):\n test_state.restore()\n fn()\n\n def create(self):\n test_state = GlobalStateMarshaler()\n return multiprocessing.Process(target=self._restore_and_run, args=(self.fn, test_state))\n\n\nclass PackageInstallContext:\n \"\"\"Captures the in-memory process state of a package installation that needs to be transmitted\n to a child process.\"\"\"\n\n def __init__(\n self,\n pkg: \"spack.package_base.PackageBase\",\n *,\n ctx: Optional[multiprocessing.context.BaseContext] = None,\n ):\n ctx = ctx or multiprocessing.get_context()\n self.global_state = GlobalStateMarshaler(ctx=ctx)\n self.pkg = pkg if ctx.get_start_method() == \"fork\" else serialize(pkg)\n self.spack_working_dir = spack.paths.spack_working_dir\n\n def restore(self) -> \"spack.package_base.PackageBase\":\n spack.paths.spack_working_dir = self.spack_working_dir\n self.global_state.restore()\n return deserialize(self.pkg) if isinstance(self.pkg, io.BytesIO) else self.pkg\n\n\nclass GlobalStateMarshaler:\n \"\"\"Class to serialize and restore global state for child processes if needed.\n\n Spack may modify state that is normally read from disk or command line in memory;\n this object is responsible for properly serializing that state to be applied to a subprocess.\n \"\"\"\n\n def __init__(\n self, *, ctx: Optional[Optional[multiprocessing.context.BaseContext]] = None\n ) -> None:\n ctx = ctx or multiprocessing.get_context()\n self.is_forked = ctx.get_start_method() == \"fork\"\n if self.is_forked:\n return\n\n from spack.environment import active_environment\n\n self.config = spack.config.CONFIG.ensure_unwrapped()\n self.platform = spack.platforms.host\n self.store = spack.store.STORE\n self.test_patches = TestPatches.create()\n self.env = active_environment()\n\n def restore(self):\n if self.is_forked:\n return\n spack.config.CONFIG = self.config\n spack.repo.enable_repo(spack.repo.RepoPath.from_config(self.config))\n spack.platforms.host = self.platform\n spack.store.STORE = self.store\n self.test_patches.restore()\n if self.env:\n from spack.environment import activate\n\n activate(self.env)\n\n\nclass TestPatches:\n def __init__(self, module_patches, class_patches):\n self.module_patches = [(x, y, serialize(z)) for (x, y, z) in module_patches]\n self.class_patches = [(x, y, serialize(z)) for (x, y, z) in class_patches]\n\n def restore(self):\n if not self.module_patches and not self.class_patches:\n return\n # this code path is only followed in tests, so use inline imports\n from pydoc import locate\n\n for module_name, attr_name, value in self.module_patches:\n value = pickle.load(value)\n module = importlib.import_module(module_name)\n setattr(module, attr_name, value)\n for class_fqn, attr_name, value in self.class_patches:\n value = pickle.load(value)\n cls = locate(class_fqn)\n setattr(cls, attr_name, value)\n\n @staticmethod\n def create():\n module_patches = []\n class_patches = []\n for target, name in MONKEYPATCHES:\n if isinstance(target, ModuleType):\n new_val = getattr(target, name)\n module_name = target.__name__\n module_patches.append((module_name, name, new_val))\n elif isinstance(target, type):\n new_val = getattr(target, name)\n class_fqn = \".\".join([target.__module__, target.__name__])\n class_patches.append((class_fqn, name, new_val))\n\n return TestPatches(module_patches, class_patches)\n" }, { "path": "lib/spack/spack/tag.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Classes and functions to manage package tags\"\"\"\n\nfrom typing import TYPE_CHECKING, Dict, List, Set\n\nimport spack.error\nimport spack.util.spack_json as sjson\n\nif TYPE_CHECKING:\n import spack.repo\n\n\nclass TagIndex:\n \"\"\"Maps tags to list of package names.\"\"\"\n\n def __init__(self) -> None:\n self.tags: Dict[str, List[str]] = {}\n\n def to_json(self, stream) -> None:\n sjson.dump({\"tags\": self.tags}, stream)\n\n @staticmethod\n def from_json(stream) -> \"TagIndex\":\n d = sjson.load(stream)\n\n if not isinstance(d, dict):\n raise TagIndexError(\"TagIndex data was not a dict.\")\n\n if \"tags\" not in d:\n raise TagIndexError(\"TagIndex data does not start with 'tags'\")\n\n r = TagIndex()\n for tag, packages in d[\"tags\"].items():\n r.tags[tag] = packages\n return r\n\n def get_packages(self, tag: str) -> List[str]:\n \"\"\"Returns all packages associated with the tag.\"\"\"\n return self.tags.get(tag, [])\n\n def merge(self, other: \"TagIndex\") -> None:\n \"\"\"Merge another tag index into this one.\n\n Args:\n other: tag index to be merged\n \"\"\"\n for tag, pkgs in other.tags.items():\n if tag not in self.tags:\n self.tags[tag] = pkgs.copy()\n else:\n self.tags[tag] = sorted({*self.tags[tag], *pkgs})\n\n def update_packages(self, pkg_names: Set[str], repo: \"spack.repo.Repo\") -> None:\n \"\"\"Updates packages in the tag index.\n\n Args:\n pkg_names: names of the packages to be updated\n repo: the repository to get package classes from\n \"\"\"\n # Remove the packages from the list of packages, if present\n for pkg_list in self.tags.values():\n if pkg_names.isdisjoint(pkg_list):\n continue\n pkg_list[:] = [pkg for pkg in pkg_list if pkg not in pkg_names]\n\n # Add them again under the appropriate tags\n for pkg_name in pkg_names:\n pkg_cls = repo.get_pkg_class(pkg_name)\n for tag in getattr(pkg_cls, \"tags\", []):\n tag = tag.lower()\n if tag not in self.tags:\n self.tags[tag] = [pkg_cls.name]\n else:\n self.tags[tag].append(pkg_cls.name)\n\n\nclass TagIndexError(spack.error.SpackError):\n \"\"\"Raised when there is a problem with a TagIndex.\"\"\"\n" }, { "path": "lib/spack/spack/tengine.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport itertools\nimport textwrap\nfrom typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple\n\nimport spack.config\nimport spack.extensions\nimport spack.llnl.util.lang\nfrom spack.util.path import canonicalize_path\n\nif TYPE_CHECKING:\n import spack.vendor.jinja2\n\n\nclass ContextMeta(type):\n \"\"\"Metaclass for Context. It helps reduce the boilerplate in client code.\"\"\"\n\n #: Keeps track of the context properties that have been added\n #: by the class that is being defined\n _new_context_properties: List[str] = []\n\n def __new__(cls, name, bases, attr_dict):\n # Merge all the context properties that are coming from base classes\n # into a list without duplicates.\n context_properties = list(cls._new_context_properties)\n for x in bases:\n try:\n context_properties.extend(x.context_properties)\n except AttributeError:\n pass\n context_properties = list(spack.llnl.util.lang.dedupe(context_properties))\n\n # Flush the list\n cls._new_context_properties = []\n\n # Attach the list to the class being created\n attr_dict[\"context_properties\"] = context_properties\n\n return super(ContextMeta, cls).__new__(cls, name, bases, attr_dict)\n\n @classmethod\n def context_property(cls, func):\n \"\"\"Decorator that adds a function name to the list of new context\n properties, and then returns a property.\n \"\"\"\n name = func.__name__\n cls._new_context_properties.append(name)\n return property(func)\n\n\n#: A saner way to use the decorator\ncontext_property = ContextMeta.context_property\n\n\nclass Context(metaclass=ContextMeta):\n \"\"\"Base class for context classes that are used with the template engine.\"\"\"\n\n context_properties: List[str]\n\n def to_dict(self) -> Dict[str, Any]:\n \"\"\"Returns a dictionary containing all the context properties.\"\"\"\n return {name: getattr(self, name) for name in self.context_properties}\n\n\ndef make_environment(dirs: Optional[Tuple[str, ...]] = None) -> \"spack.vendor.jinja2.Environment\":\n \"\"\"Returns a configured environment for template rendering.\"\"\"\n if dirs is None:\n # Default directories where to search for templates\n dirs = default_template_dirs(spack.config.CONFIG)\n\n return make_environment_from_dirs(dirs)\n\n\n@spack.llnl.util.lang.memoized\ndef make_environment_from_dirs(dirs: Tuple[str, ...]) -> \"spack.vendor.jinja2.Environment\":\n # Import at this scope to avoid slowing Spack startup down\n import spack.vendor.jinja2\n\n # Loader for the templates\n loader = spack.vendor.jinja2.FileSystemLoader(dirs)\n # Environment of the template engine\n env = spack.vendor.jinja2.Environment(loader=loader, trim_blocks=True, lstrip_blocks=True)\n # Custom filters\n _set_filters(env)\n return env\n\n\ndef default_template_dirs(configuration: spack.config.Configuration) -> Tuple[str, ...]:\n config_yaml = configuration.get_config(\"config\")\n builtins = config_yaml.get(\"template_dirs\", [\"$spack/share/spack/templates\"])\n extensions = spack.extensions.get_template_dirs()\n return tuple(canonicalize_path(d) for d in itertools.chain(builtins, extensions))\n\n\n# Extra filters for the template engine environment\n\n\ndef prepend_to_line(text, token):\n \"\"\"Prepends a token to each line in text\"\"\"\n return [token + line for line in text]\n\n\ndef quote(text):\n \"\"\"Quotes each line in text\"\"\"\n return ['\"{0}\"'.format(line) for line in text]\n\n\ndef curly_quote(text):\n \"\"\"Encloses each line of text in curly braces\"\"\"\n return [\"{{{0}}}\".format(line) for line in text]\n\n\ndef _set_filters(env):\n \"\"\"Sets custom filters to the template engine environment\"\"\"\n env.filters[\"textwrap\"] = textwrap.wrap\n env.filters[\"prepend_to_line\"] = prepend_to_line\n env.filters[\"join\"] = \"\\n\".join\n env.filters[\"quote\"] = quote\n env.filters[\"curly_quote\"] = curly_quote\n" }, { "path": "lib/spack/spack/test/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n" }, { "path": "lib/spack/spack/test/architecture.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport platform\n\nimport pytest\n\nimport spack.vendor.archspec.cpu\n\nimport spack.concretize\nimport spack.operating_systems\nimport spack.platforms\nfrom spack.spec import ArchSpec, Spec\n\n\n@pytest.fixture(scope=\"module\")\ndef current_host_platform():\n \"\"\"Return the platform of the current host as detected by the\n 'platform' stdlib package.\n \"\"\"\n current_platform = None\n if \"Linux\" in platform.system():\n current_platform = spack.platforms.Linux()\n elif \"Darwin\" in platform.system():\n current_platform = spack.platforms.Darwin()\n elif \"Windows\" in platform.system():\n current_platform = spack.platforms.Windows()\n elif \"FreeBSD\" in platform.system():\n current_platform = spack.platforms.FreeBSD()\n return current_platform\n\n\n# Valid keywords for os=xxx or target=xxx\nVALID_KEYWORDS = [\"fe\", \"be\", \"frontend\", \"backend\"]\n\nTEST_PLATFORM = spack.platforms.Test()\n\n\n@pytest.fixture(params=([str(x) for x in TEST_PLATFORM.targets] + VALID_KEYWORDS), scope=\"module\")\ndef target_str(request):\n \"\"\"All the possible strings that can be used for targets\"\"\"\n return request.param\n\n\n@pytest.fixture(\n params=([str(x) for x in TEST_PLATFORM.operating_sys] + VALID_KEYWORDS), scope=\"module\"\n)\ndef os_str(request):\n \"\"\"All the possible strings that can be used for operating systems\"\"\"\n return request.param\n\n\ndef test_platform(current_host_platform):\n \"\"\"Check that current host detection return the correct platform\"\"\"\n detected_platform = spack.platforms.real_host()\n assert str(detected_platform) == str(current_host_platform)\n\n\ndef test_user_input_combination(config, target_str, os_str):\n \"\"\"Test for all the valid user input combinations that both the target and\n the operating system match.\n \"\"\"\n spec = Spec(f\"libelf os={os_str} target={target_str}\")\n assert spec.architecture.os == str(TEST_PLATFORM.operating_system(os_str))\n assert spec.architecture.target == TEST_PLATFORM.target(target_str)\n\n\ndef test_default_os_and_target(default_mock_concretization):\n \"\"\"Test that is we don't specify `os=` or `target=` we get the default values\n after concretization.\n \"\"\"\n spec = default_mock_concretization(\"libelf\")\n assert spec.architecture.os == str(TEST_PLATFORM.default_operating_system())\n assert spec.architecture.target == TEST_PLATFORM.default_target()\n\n\ndef test_operating_system_conversion_to_dict():\n operating_system = spack.operating_systems.OperatingSystem(\"os\", \"1.0\")\n assert operating_system.to_dict() == {\"name\": \"os\", \"version\": \"1.0\"}\n\n\n@pytest.mark.parametrize(\n \"item,architecture_str\",\n [\n # We can search the architecture string representation\n (\"linux\", \"linux-ubuntu18.04-haswell\"),\n (\"ubuntu\", \"linux-ubuntu18.04-haswell\"),\n (\"haswell\", \"linux-ubuntu18.04-haswell\"),\n # We can also search flags of the target,\n (\"avx512\", \"linux-ubuntu18.04-icelake\"),\n ],\n)\ndef test_arch_spec_container_semantic(item, architecture_str):\n architecture = ArchSpec(architecture_str)\n assert item in architecture\n\n\n@pytest.mark.regression(\"15306\")\n@pytest.mark.parametrize(\n \"architecture_tuple,constraint_tuple\",\n [\n ((\"linux\", \"ubuntu18.04\", None), (\"linux\", None, \"x86_64\")),\n ((\"linux\", \"ubuntu18.04\", None), (\"linux\", None, \"x86_64:\")),\n ],\n)\ndef test_satisfy_strict_constraint_when_not_concrete(architecture_tuple, constraint_tuple):\n architecture = ArchSpec(architecture_tuple)\n constraint = ArchSpec(constraint_tuple)\n assert not architecture.satisfies(constraint)\n\n\n@pytest.mark.parametrize(\n \"root_target_range,dep_target_range,result\",\n [\n (\"x86_64:nocona\", \"x86_64:core2\", \"nocona\"), # pref not in intersection\n (\"x86_64:core2\", \"x86_64:nocona\", \"nocona\"),\n (\"x86_64:haswell\", \"x86_64:mic_knl\", \"core2\"), # pref in intersection\n (\"ivybridge\", \"nocona:skylake\", \"ivybridge\"), # one side concrete\n (\"haswell:icelake\", \"broadwell\", \"broadwell\"),\n # multiple ranges in lists with multiple overlaps\n (\"x86_64:nocona,haswell:broadwell\", \"nocona:haswell,skylake:\", \"nocona\"),\n # lists with concrete targets, lists compared to ranges\n (\"x86_64,haswell\", \"core2:broadwell\", \"haswell\"),\n ],\n)\n@pytest.mark.usefixtures(\"mock_packages\", \"config\")\n@pytest.mark.skipif(\n str(spack.vendor.archspec.cpu.host().family) != \"x86_64\",\n reason=\"tests are for x86_64 uarch ranges\",\n)\ndef test_concretize_target_ranges(root_target_range, dep_target_range, result, monkeypatch):\n spec = spack.concretize.concretize_one(\n f\"pkg-a foobar=bar target={root_target_range} %gcc@10 ^pkg-b target={dep_target_range}\"\n )\n assert spec.target == spec[\"pkg-b\"].target == result\n" }, { "path": "lib/spack/spack/test/audit.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport pytest\n\nimport spack.audit\nimport spack.config\n\n\n@pytest.mark.parametrize(\n # PKG-PROPERTIES are ubiquitous in mock packages, since they don't use sha256\n # and they don't change the example.com URL very often.\n \"packages,expected_errors\",\n [\n # A non existing variant is used in a conflict directive\n ([\"wrong-variant-in-conflicts\"], [\"PKG-DIRECTIVES\", \"PKG-PROPERTIES\"]),\n # The package declares a non-existing dependency\n ([\"missing-dependency\"], [\"PKG-DIRECTIVES\", \"PKG-PROPERTIES\"]),\n # The package use a non existing variant in a depends_on directive\n ([\"wrong-variant-in-depends-on\"], [\"PKG-DIRECTIVES\", \"PKG-PROPERTIES\"]),\n # This package has a GitHub pull request commit patch URL\n ([\"invalid-github-pull-commits-patch-url\"], [\"PKG-DIRECTIVES\", \"PKG-PROPERTIES\"]),\n # This package has a GitHub patch URL without full_index=1\n ([\"invalid-github-patch-url\"], [\"PKG-DIRECTIVES\", \"PKG-PROPERTIES\"]),\n # This package has invalid GitLab patch URLs\n ([\"invalid-gitlab-patch-url\"], [\"PKG-DIRECTIVES\", \"PKG-PROPERTIES\"]),\n # This package has invalid GitLab patch URLs\n ([\"invalid-selfhosted-gitlab-patch-url\"], [\"PKG-DIRECTIVES\", \"PKG-PROPERTIES\"]),\n # This package has a stand-alone test method in build-time callbacks\n ([\"fail-test-audit\"], [\"PKG-PROPERTIES\"]),\n # This package has stand-alone test methods without non-trivial docstrings\n ([\"fail-test-audit-docstring\"], [\"PKG-PROPERTIES\"]),\n # This package has a stand-alone test method without an implementation\n ([\"fail-test-audit-impl\"], [\"PKG-PROPERTIES\"]),\n # This package has no issues\n ([\"mpileaks\"], None),\n # This package has a conflict with a trigger which cannot constrain the constraint\n # Should not raise an error\n ([\"unconstrainable-conflict\"], None),\n ],\n)\ndef test_package_audits(packages, expected_errors, mock_packages):\n reports = spack.audit.run_group(\"packages\", pkgs=packages)\n\n # Check that errors were reported only for the expected failure\n actual_errors = [check for check, errors in reports if errors]\n msg = \"\\n\".join([str(e) for _, errors in reports for e in errors])\n if expected_errors:\n assert expected_errors == actual_errors, msg\n else:\n assert not actual_errors, msg\n\n\n# Data used in the test below to audit the double definition of a compiler\n_double_compiler_definition = [\n {\n \"compiler\": {\n \"spec\": \"gcc@9.0.1\",\n \"paths\": {\n \"cc\": \"/usr/bin/gcc-9\",\n \"cxx\": \"/usr/bin/g++-9\",\n \"f77\": \"/usr/bin/gfortran-9\",\n \"fc\": \"/usr/bin/gfortran-9\",\n },\n \"flags\": {},\n \"operating_system\": \"ubuntu18.04\",\n \"target\": \"x86_64\",\n \"modules\": [],\n \"environment\": {},\n \"extra_rpaths\": [],\n }\n },\n {\n \"compiler\": {\n \"spec\": \"gcc@9.0.1\",\n \"paths\": {\n \"cc\": \"/usr/bin/gcc-9\",\n \"cxx\": \"/usr/bin/g++-9\",\n \"f77\": \"/usr/bin/gfortran-9\",\n \"fc\": \"/usr/bin/gfortran-9\",\n },\n \"flags\": {\"cflags\": \"-O3\"},\n \"operating_system\": \"ubuntu18.04\",\n \"target\": \"x86_64\",\n \"modules\": [],\n \"environment\": {},\n \"extra_rpaths\": [],\n }\n },\n]\n\n\n# TODO/RepoSplit: Should this not rely on mock packages post split?\n@pytest.mark.parametrize(\n \"config_section,data,failing_check\",\n [\n # Double compiler definitions in compilers.yaml\n (\"compilers\", _double_compiler_definition, \"CFG-COMPILER\"),\n # Multiple definitions of the same external spec in packages.yaml\n (\n \"packages\",\n {\n \"mpileaks\": {\n \"externals\": [\n {\"spec\": \"mpileaks@1.0.0\", \"prefix\": \"/\"},\n {\"spec\": \"mpileaks@1.0.0\", \"prefix\": \"/usr\"},\n ]\n }\n },\n \"CFG-PACKAGES\",\n ),\n ],\n)\ndef test_config_audits(config_section, data, failing_check, mock_packages):\n with spack.config.override(config_section, data):\n reports = spack.audit.run_group(\"configs\")\n assert any((check == failing_check) and errors for check, errors in reports)\n" }, { "path": "lib/spack/spack/test/binary_distribution.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport filecmp\nimport glob\nimport gzip\nimport io\nimport json\nimport os\nimport pathlib\nimport re\nimport tarfile\nimport urllib.error\nimport urllib.request\nimport urllib.response\nfrom pathlib import Path, PurePath\nfrom typing import Any, Callable, Dict, NamedTuple, Optional\n\nimport pytest\n\nimport spack.binary_distribution\nimport spack.concretize\nimport spack.config\nimport spack.environment as ev\nimport spack.hooks.sbang as sbang\nimport spack.main\nimport spack.mirrors.mirror\nimport spack.oci.image\nimport spack.spec\nimport spack.stage\nimport spack.store\nimport spack.url_buildcache\nimport spack.util.gpg\nimport spack.util.spack_yaml as syaml\nimport spack.util.url as url_util\nimport spack.util.web as web_util\nfrom spack.binary_distribution import CannotListKeys, GenerateIndexError\nfrom spack.database import INDEX_JSON_FILE\nfrom spack.installer import PackageInstaller\nfrom spack.llnl.util.filesystem import join_path, readlink, working_dir\nfrom spack.spec import Spec\nfrom spack.url_buildcache import (\n INDEX_MANIFEST_FILE,\n BuildcacheComponent,\n BuildcacheEntryError,\n URLBuildcacheEntry,\n URLBuildcacheEntryV2,\n compression_writer,\n get_entries_from_cache,\n get_url_buildcache_class,\n get_valid_spec_file,\n)\n\npytestmark = pytest.mark.not_on_windows(\"does not run on windows\")\n\nmirror_cmd = spack.main.SpackCommand(\"mirror\")\ninstall_cmd = spack.main.SpackCommand(\"install\")\nuninstall_cmd = spack.main.SpackCommand(\"uninstall\")\nbuildcache_cmd = spack.main.SpackCommand(\"buildcache\")\n\n\n@pytest.fixture\ndef dummy_prefix(tmp_path: pathlib.Path):\n \"\"\"Dummy prefix used for testing tarball creation, validation, extraction\"\"\"\n p = tmp_path / \"prefix\"\n p.mkdir()\n assert os.path.isabs(str(p))\n\n (p / \"bin\").mkdir()\n (p / \"share\").mkdir()\n (p / \".spack\").mkdir()\n\n app = p / \"bin\" / \"app\"\n relative_app_link = p / \"bin\" / \"relative_app_link\"\n absolute_app_link = p / \"bin\" / \"absolute_app_link\"\n data = p / \"share\" / \"file\"\n\n with open(app, \"w\", encoding=\"utf-8\") as f:\n f.write(\"hello world\")\n\n with open(data, \"w\", encoding=\"utf-8\") as f:\n f.write(\"hello world\")\n\n with open(p / \".spack\" / \"binary_distribution\", \"w\", encoding=\"utf-8\") as f:\n f.write(\"{}\")\n\n os.symlink(\"app\", str(relative_app_link))\n os.symlink(str(app), str(absolute_app_link))\n\n return str(p)\n\n\n@pytest.mark.maybeslow\ndef test_buildcache_cmd_smoke_test(tmp_path: pathlib.Path, install_mockery):\n \"\"\"\n Test the creation and installation of buildcaches with default rpaths\n into the default directory layout scheme.\n \"\"\"\n mirror_cmd(\"add\", \"--type\", \"binary\", \"--unsigned\", \"test-mirror\", str(tmp_path))\n\n # Install 'corge' without using a cache\n install_cmd(\"--fake\", \"--no-cache\", \"corge\")\n install_cmd(\"--fake\", \"--no-cache\", \"symly\")\n\n # Create a buildache\n buildcache_cmd(\"push\", \"-u\", str(tmp_path), \"corge\", \"symly\")\n # Test force overwrite create buildcache (-f option)\n buildcache_cmd(\"push\", \"-uf\", str(tmp_path), \"corge\")\n\n # Create mirror index\n buildcache_cmd(\"update-index\", str(tmp_path))\n\n # List the buildcaches in the mirror\n buildcache_cmd(\"list\", \"-alv\")\n\n # Uninstall the package and deps\n uninstall_cmd(\"-y\", \"--dependents\", \"garply\")\n\n # Test installing from build caches\n buildcache_cmd(\"install\", \"-uo\", \"corge\", \"symly\")\n\n # This gives warning that spec is already installed\n buildcache_cmd(\"install\", \"-uo\", \"corge\")\n\n # Test overwrite install\n buildcache_cmd(\"install\", \"-ufo\", \"corge\")\n\n buildcache_cmd(\"keys\", \"-f\")\n buildcache_cmd(\"list\")\n\n buildcache_cmd(\"list\", \"-a\")\n buildcache_cmd(\"list\", \"-l\", \"-v\")\n\n\ndef test_push_and_fetch_keys(mock_gnupghome, tmp_path: pathlib.Path):\n testpath = str(mock_gnupghome)\n\n mirror = os.path.join(testpath, \"mirror\")\n mirrors = {\"test-mirror\": url_util.path_to_file_url(mirror)}\n mirrors = spack.mirrors.mirror.MirrorCollection(mirrors)\n mirror = spack.mirrors.mirror.Mirror(url_util.path_to_file_url(mirror))\n\n gpg_dir1 = os.path.join(testpath, \"gpg1\")\n gpg_dir2 = os.path.join(testpath, \"gpg2\")\n\n # dir 1: create a new key, record its fingerprint, and push it to a new\n # mirror\n with spack.util.gpg.gnupghome_override(gpg_dir1):\n spack.util.gpg.create(name=\"test-key\", email=\"fake@test.key\", expires=\"0\", comment=None)\n\n keys = spack.util.gpg.public_keys()\n assert len(keys) == 1\n fpr = keys[0]\n\n spack.binary_distribution._url_push_keys(\n mirror, keys=[fpr], tmpdir=str(tmp_path), update_index=True\n )\n\n # dir 2: import the key from the mirror, and confirm that its fingerprint\n # matches the one created above\n with spack.util.gpg.gnupghome_override(gpg_dir2):\n assert len(spack.util.gpg.public_keys()) == 0\n\n spack.binary_distribution.get_keys(mirrors=mirrors, install=True, trust=True, force=True)\n\n new_keys = spack.util.gpg.public_keys()\n assert len(new_keys) == 1\n assert new_keys[0] == fpr\n\n\n@pytest.mark.maybeslow\ndef test_built_spec_cache(install_mockery, tmp_path: pathlib.Path):\n \"\"\"Because the buildcache list command fetches the buildcache index\n and uses it to populate the binary_distribution built spec cache, when\n this test calls get_mirrors_for_spec, it is testing the popluation of\n that cache from a buildcache index.\"\"\"\n\n install_cmd(\"--fake\", \"--no-cache\", \"corge\")\n buildcache_cmd(\"push\", \"--unsigned\", \"--update-index\", str(tmp_path), \"corge\")\n mirror_cmd(\"add\", \"--type\", \"binary\", \"--unsigned\", \"test-mirror\", str(tmp_path))\n buildcache_cmd(\"list\", \"-a\", \"-l\")\n\n gspec = spack.concretize.concretize_one(\"garply\")\n cspec = spack.concretize.concretize_one(\"corge\")\n\n for s in [gspec, cspec]:\n results = spack.binary_distribution.get_mirrors_for_spec(s)\n assert len(results) == 1\n assert results[0].url == url_util.path_to_file_url(str(tmp_path))\n\n\ndef fake_dag_hash(spec, length=None):\n # Generate an arbitrary hash that is intended to be different than\n # whatever a Spec reported before (to test actions that trigger when\n # the hash changes)\n return \"tal4c7h4z0gqmixb1eqa92mjoybxn5l6\"[:length]\n\n\n@pytest.mark.usefixtures(\"install_mockery\", \"mock_packages\", \"mock_fetch\", \"temporary_mirror\")\ndef test_spec_needs_rebuild(monkeypatch, tmp_path: pathlib.Path):\n \"\"\"Make sure needs_rebuild properly compares remote hash\n against locally computed one, avoiding unnecessary rebuilds\"\"\"\n\n # Create a temp mirror directory for buildcache usage\n mirror_dir = tmp_path / \"mirror_dir\"\n mirror_url = url_util.path_to_file_url(str(mirror_dir))\n\n s = spack.concretize.concretize_one(\"libdwarf\")\n\n # Install a package\n install_cmd(\"--fake\", s.name)\n\n # Put installed package in the buildcache\n buildcache_cmd(\"push\", \"-u\", str(mirror_dir), s.name)\n\n rebuild = spack.binary_distribution.needs_rebuild(s, mirror_url)\n\n assert not rebuild\n\n # Now monkey patch Spec to change the hash on the package\n monkeypatch.setattr(spack.spec.Spec, \"dag_hash\", fake_dag_hash)\n\n rebuild = spack.binary_distribution.needs_rebuild(s, mirror_url)\n\n assert rebuild\n\n\n@pytest.mark.usefixtures(\"install_mockery\", \"mock_packages\", \"mock_fetch\")\ndef test_generate_index_missing(monkeypatch, tmp_path: pathlib.Path, mutable_config):\n \"\"\"Ensure spack buildcache index only reports available packages\"\"\"\n\n # Create a temp mirror directory for buildcache usage\n mirror_dir = tmp_path / \"mirror_dir\"\n mirror_url = url_util.path_to_file_url(str(mirror_dir))\n spack.config.set(\"mirrors\", {\"test\": mirror_url})\n\n s = spack.concretize.concretize_one(\"libdwarf\")\n\n # Install a package\n install_cmd(\"--fake\", \"--no-cache\", s.name)\n\n # Create a buildcache and update index\n buildcache_cmd(\"push\", \"-u\", str(mirror_dir), s.name)\n buildcache_cmd(\"update-index\", str(mirror_dir))\n\n # Check package and dependency in buildcache\n cache_list = buildcache_cmd(\"list\", \"--allarch\")\n assert \"libdwarf\" in cache_list\n assert \"libelf\" in cache_list\n\n # Remove dependency from cache\n libelf_files = glob.glob(\n os.path.join(\n str(mirror_dir / spack.binary_distribution.buildcache_relative_specs_path()),\n \"libelf\",\n \"*libelf*\",\n )\n )\n os.remove(*libelf_files)\n\n # Update index\n buildcache_cmd(\"update-index\", str(mirror_dir))\n\n with spack.config.override(\"config:binary_index_ttl\", 0):\n # Check dependency not in buildcache\n cache_list = buildcache_cmd(\"list\", \"--allarch\")\n assert \"libdwarf\" in cache_list\n assert \"libelf\" not in cache_list\n\n\n@pytest.mark.usefixtures(\"install_mockery\", \"mock_packages\", \"mock_fetch\")\ndef test_use_bin_index(monkeypatch, tmp_path: pathlib.Path, mutable_config):\n \"\"\"Check use of binary cache index: perform an operation that\n instantiates it, and a second operation that reconstructs it.\n \"\"\"\n index_cache_root = str(tmp_path / \"index_cache\")\n monkeypatch.setattr(\n spack.binary_distribution,\n \"BINARY_INDEX\",\n spack.binary_distribution.BinaryCacheIndex(index_cache_root),\n )\n\n # Create a mirror, configure us to point at it, install a spec, and\n # put it in the mirror\n mirror_dir = tmp_path / \"mirror_dir\"\n mirror_url = url_util.path_to_file_url(str(mirror_dir))\n spack.config.set(\"mirrors\", {\"test\": mirror_url})\n s = spack.concretize.concretize_one(\"libdwarf\")\n install_cmd(\"--fake\", \"--no-cache\", s.name)\n buildcache_cmd(\"push\", \"-u\", str(mirror_dir), s.name)\n buildcache_cmd(\"update-index\", str(mirror_dir))\n\n # Now the test\n buildcache_cmd(\"list\", \"-al\")\n spack.binary_distribution.BINARY_INDEX = spack.binary_distribution.BinaryCacheIndex(\n index_cache_root\n )\n cache_list = buildcache_cmd(\"list\", \"-al\")\n assert \"libdwarf\" in cache_list\n\n\n@pytest.mark.usefixtures(\"install_mockery\", \"mock_packages\", \"mock_fetch\")\ndef test_use_bin_index_active_env_with_view(\n monkeypatch, tmp_path: pathlib.Path, mutable_config, mutable_mock_env_path\n):\n \"\"\"Check use of binary cache index: perform an operation that\n instantiates it, and a second operation that reconstructs it.\n \"\"\"\n index_cache_root = str(tmp_path / \"index_cache\")\n monkeypatch.setattr(\n spack.binary_distribution,\n \"BINARY_INDEX\",\n spack.binary_distribution.BinaryCacheIndex(index_cache_root),\n )\n\n # Create a mirror, configure us to point at it, install a spec, and\n # put it in the mirror\n mirror_dir = tmp_path / \"mirror_dir\"\n mirror_url = url_util.path_to_file_url(str(mirror_dir))\n spack.config.set(\"mirrors\", {\"test\": {\"url\": mirror_url, \"view\": \"test\"}})\n s = spack.concretize.concretize_one(\"libdwarf\")\n\n # Create an environment and install specs for the view\n ev.create(\"testenv\")\n with ev.read(\"testenv\"):\n install_cmd(\"--add\", \"--fake\", \"--no-cache\", s.name)\n buildcache_cmd(\"push\", \"-u\", \"test\", s.name)\n buildcache_cmd(\"update-index\", \"test\")\n\n # Now the test\n buildcache_cmd(\"list\", \"-al\")\n spack.binary_distribution.BINARY_INDEX = spack.binary_distribution.BinaryCacheIndex(\n index_cache_root\n )\n cache_list = buildcache_cmd(\"list\", \"-al\")\n assert \"libdwarf\" in cache_list\n\n\n@pytest.mark.usefixtures(\"install_mockery\", \"mock_packages\", \"mock_fetch\")\ndef test_use_bin_index_with_view(\n monkeypatch, tmp_path: pathlib.Path, mutable_config, mutable_mock_env_path\n):\n \"\"\"Check use of binary cache index: perform an operation that\n instantiates it, and a second operation that reconstructs it.\n \"\"\"\n index_cache_root = str(tmp_path / \"index_cache\")\n monkeypatch.setattr(\n spack.binary_distribution,\n \"BINARY_INDEX\",\n spack.binary_distribution.BinaryCacheIndex(index_cache_root),\n )\n\n # Create a mirror, configure us to point at it, install a spec, and\n # put it in the mirror\n mirror_dir = tmp_path / \"mirror_dir\"\n mirror_url = url_util.path_to_file_url(str(mirror_dir))\n spack.config.set(\"mirrors\", {\"test\": {\"url\": mirror_url, \"view\": \"test\"}})\n s = spack.concretize.concretize_one(\"libdwarf\")\n\n # Create an environment and install specs for the view\n ev.create(\"testenv\")\n with ev.read(\"testenv\"):\n install_cmd(\"--add\", \"--fake\", \"--no-cache\", s.name)\n buildcache_cmd(\"push\", \"-u\", \"test\", s.name)\n\n buildcache_cmd(\"update-index\", \"test\", \"testenv\")\n\n # Now the test\n buildcache_cmd(\"list\", \"-al\")\n spack.binary_distribution.BINARY_INDEX = spack.binary_distribution.BinaryCacheIndex(\n index_cache_root\n )\n cache_list = buildcache_cmd(\"list\", \"-al\")\n assert \"libdwarf\" in cache_list\n\n\ndef test_generate_key_index_failure(monkeypatch, tmp_path: pathlib.Path):\n def list_url(url, recursive=False):\n if \"fails-listing\" in url:\n raise Exception(\"Couldn't list the directory\")\n return [\"first.pub\", \"second.pub\"]\n\n def push_to_url(*args, **kwargs):\n raise Exception(\"Couldn't upload the file\")\n\n monkeypatch.setattr(web_util, \"list_url\", list_url)\n monkeypatch.setattr(web_util, \"push_to_url\", push_to_url)\n\n with pytest.raises(CannotListKeys, match=\"Encountered problem listing keys\"):\n spack.binary_distribution.generate_key_index(\n \"s3://non-existent/fails-listing\", str(tmp_path)\n )\n\n with pytest.raises(GenerateIndexError, match=\"problem pushing .* Couldn't upload\"):\n spack.binary_distribution.generate_key_index(\n \"s3://non-existent/fails-uploading\", str(tmp_path)\n )\n\n\ndef test_generate_package_index_failure(monkeypatch, tmp_path: pathlib.Path, capfd):\n def mock_list_url(url, recursive=False):\n raise Exception(\"Some HTTP error\")\n\n monkeypatch.setattr(web_util, \"list_url\", mock_list_url)\n\n test_url = \"file:///fake/keys/dir\"\n\n with pytest.raises(GenerateIndexError, match=\"Unable to generate package index\"):\n spack.binary_distribution._url_generate_package_index(test_url, str(tmp_path))\n\n assert (\n \"Warning: Encountered problem listing packages at \"\n f\"{test_url}: Some HTTP error\" in capfd.readouterr().err\n )\n\n\ndef test_generate_indices_exception(monkeypatch, tmp_path: pathlib.Path, capfd):\n def mock_list_url(url, recursive=False):\n raise Exception(\"Test Exception handling\")\n\n monkeypatch.setattr(web_util, \"list_url\", mock_list_url)\n\n url = \"file:///fake/keys/dir\"\n\n with pytest.raises(GenerateIndexError, match=f\"Encountered problem listing keys at {url}\"):\n spack.binary_distribution.generate_key_index(url, str(tmp_path))\n\n with pytest.raises(GenerateIndexError, match=\"Unable to generate package index\"):\n spack.binary_distribution._url_generate_package_index(url, str(tmp_path))\n\n assert f\"Encountered problem listing packages at {url}\" in capfd.readouterr().err\n\n\ndef test_update_sbang(tmp_path: pathlib.Path, temporary_mirror, mock_fetch, install_mockery):\n \"\"\"Test relocation of the sbang shebang line in a package script\"\"\"\n s = spack.concretize.concretize_one(\"old-sbang\")\n PackageInstaller([s.package]).install()\n old_prefix, old_sbang_shebang = s.prefix, sbang.sbang_shebang_line()\n old_contents = f\"\"\"\\\n{old_sbang_shebang}\n#!/usr/bin/env python3\n\n{s.prefix.bin}\n\"\"\"\n with open(os.path.join(s.prefix.bin, \"script.sh\"), encoding=\"utf-8\") as f:\n assert f.read() == old_contents\n\n # Create a buildcache with the installed spec.\n buildcache_cmd(\"push\", \"--update-index\", \"--unsigned\", temporary_mirror, f\"/{s.dag_hash()}\")\n\n # Switch the store to the new install tree locations\n with spack.store.use_store(str(tmp_path)):\n s._prefix = None # clear the cached old prefix\n new_prefix, new_sbang_shebang = s.prefix, sbang.sbang_shebang_line()\n assert old_prefix != new_prefix\n assert old_sbang_shebang != new_sbang_shebang\n PackageInstaller(\n [s.package], root_policy=\"cache_only\", dependencies_policy=\"cache_only\", unsigned=True\n ).install()\n\n # Check that the sbang line refers to the new install tree\n new_contents = f\"\"\"\\\n{sbang.sbang_shebang_line()}\n#!/usr/bin/env python3\n\n{s.prefix.bin}\n\"\"\"\n with open(os.path.join(s.prefix.bin, \"script.sh\"), encoding=\"utf-8\") as f:\n assert f.read() == new_contents\n\n\ndef test_FetchCacheError_only_accepts_lists_of_errors():\n with pytest.raises(TypeError, match=\"list\"):\n spack.binary_distribution.FetchCacheError(\"error\")\n\n\ndef test_FetchCacheError_pretty_printing_multiple():\n e = spack.binary_distribution.FetchCacheError([RuntimeError(\"Oops!\"), TypeError(\"Trouble!\")])\n str_e = str(e)\n assert \"Multiple errors\" in str_e\n assert \"Error 1: RuntimeError: Oops!\" in str_e\n assert \"Error 2: TypeError: Trouble!\" in str_e\n assert str_e.rstrip() == str_e\n\n\ndef test_FetchCacheError_pretty_printing_single():\n e = spack.binary_distribution.FetchCacheError([RuntimeError(\"Oops!\")])\n str_e = str(e)\n assert \"Multiple errors\" not in str_e\n assert \"RuntimeError: Oops!\" in str_e\n assert str_e.rstrip() == str_e\n\n\ndef test_text_relocate_if_needed(\n install_mockery, temporary_store, mock_fetch, tmp_path: pathlib.Path\n):\n install_cmd(\"needs-text-relocation\")\n spec = temporary_store.db.query_one(\"needs-text-relocation\")\n tgz_path = tmp_path / \"relocatable.tar.gz\"\n spack.binary_distribution.create_tarball(spec, str(tgz_path))\n\n # extract the .spack/binary_distribution file\n with tarfile.open(tgz_path) as tar:\n entry_name = next(x for x in tar.getnames() if x.endswith(\".spack/binary_distribution\"))\n bd_file = tar.extractfile(entry_name)\n manifest = syaml.load(bd_file)\n\n assert join_path(\"bin\", \"exe\") in manifest[\"relocate_textfiles\"]\n assert join_path(\"bin\", \"otherexe\") not in manifest[\"relocate_textfiles\"]\n assert join_path(\"bin\", \"secretexe\") not in manifest[\"relocate_textfiles\"]\n\n\ndef test_compression_writer(tmp_path: pathlib.Path):\n text = \"This is some text. We might or might not like to compress it as we write.\"\n checksum_algo = \"sha256\"\n\n # Write the data using gzip compression\n compressed_output_path = str(tmp_path / \"compressed_text\")\n with compression_writer(compressed_output_path, \"gzip\", checksum_algo) as (\n compressor,\n checker,\n ):\n compressor.write(text.encode(\"utf-8\"))\n\n compressed_size = checker.length\n compressed_checksum = checker.hexdigest()\n\n with open(compressed_output_path, \"rb\") as f:\n binary_content = f.read()\n\n assert spack.binary_distribution.compute_hash(binary_content) == compressed_checksum\n assert os.stat(compressed_output_path).st_size == compressed_size\n assert binary_content[:2] == b\"\\x1f\\x8b\"\n decompressed_content = gzip.decompress(binary_content).decode(\"utf-8\")\n\n assert decompressed_content == text\n\n # Write the data without compression\n uncompressed_output_path = str(tmp_path / \"uncompressed_text\")\n with compression_writer(uncompressed_output_path, \"none\", checksum_algo) as (\n compressor,\n checker,\n ):\n compressor.write(text.encode(\"utf-8\"))\n\n uncompressed_size = checker.length\n uncompressed_checksum = checker.hexdigest()\n\n with open(uncompressed_output_path, \"r\", encoding=\"utf-8\") as f:\n content = f.read()\n\n assert spack.binary_distribution.compute_hash(content) == uncompressed_checksum\n assert os.stat(uncompressed_output_path).st_size == uncompressed_size\n assert content == text\n\n # Make sure we raise if requesting unknown compression type\n nocare_output_path = str(tmp_path / \"wontwrite\")\n with pytest.raises(BuildcacheEntryError, match=\"Unknown compression type\"):\n with compression_writer(nocare_output_path, \"gsip\", checksum_algo) as (\n compressor,\n checker,\n ):\n compressor.write(text)\n\n\ndef test_v2_etag_fetching_304():\n # Test conditional fetch with etags. If the remote hasn't modified the file\n # it returns 304, which is an HTTPError in urllib-land. That should be\n # handled as success, since it means the local cache is up-to-date.\n def response_304(request: urllib.request.Request):\n url = request.get_full_url()\n if url == f\"https://www.example.com/build_cache/{INDEX_JSON_FILE}\":\n assert request.get_header(\"If-none-match\") == '\"112a8bbc1b3f7f185621c1ee335f0502\"'\n raise urllib.error.HTTPError(\n url,\n 304,\n \"Not Modified\",\n hdrs={}, # type: ignore[arg-type]\n fp=None, # type: ignore[arg-type]\n )\n assert False, \"Should not fetch {}\".format(url)\n\n fetcher = spack.binary_distribution.EtagIndexFetcherV2(\n url=\"https://www.example.com\",\n etag=\"112a8bbc1b3f7f185621c1ee335f0502\",\n urlopen=response_304,\n )\n\n result = fetcher.conditional_fetch()\n assert isinstance(result, spack.binary_distribution.FetchIndexResult)\n assert result.fresh\n\n\ndef test_v2_etag_fetching_200():\n # Test conditional fetch with etags. The remote has modified the file.\n def response_200(request: urllib.request.Request):\n url = request.get_full_url()\n if url == f\"https://www.example.com/build_cache/{INDEX_JSON_FILE}\":\n assert request.get_header(\"If-none-match\") == '\"112a8bbc1b3f7f185621c1ee335f0502\"'\n return urllib.response.addinfourl(\n io.BytesIO(b\"Result\"),\n headers={\"Etag\": '\"59bcc3ad6775562f845953cf01624225\"'}, # type: ignore[arg-type]\n url=url,\n code=200,\n )\n assert False, \"Should not fetch {}\".format(url)\n\n fetcher = spack.binary_distribution.EtagIndexFetcherV2(\n url=\"https://www.example.com\",\n etag=\"112a8bbc1b3f7f185621c1ee335f0502\",\n urlopen=response_200,\n )\n\n result = fetcher.conditional_fetch()\n assert isinstance(result, spack.binary_distribution.FetchIndexResult)\n assert not result.fresh\n assert result.etag == \"59bcc3ad6775562f845953cf01624225\"\n assert result.data == \"Result\" # decoded utf-8.\n assert result.hash == spack.binary_distribution.compute_hash(\"Result\")\n\n\ndef test_v2_etag_fetching_404():\n # Test conditional fetch with etags. The remote has modified the file.\n def response_404(request: urllib.request.Request):\n raise urllib.error.HTTPError(\n request.get_full_url(),\n 404,\n \"Not found\",\n hdrs={\"Etag\": '\"59bcc3ad6775562f845953cf01624225\"'}, # type: ignore[arg-type]\n fp=None,\n )\n\n fetcher = spack.binary_distribution.EtagIndexFetcherV2(\n url=\"https://www.example.com\",\n etag=\"112a8bbc1b3f7f185621c1ee335f0502\",\n urlopen=response_404,\n )\n\n with pytest.raises(spack.binary_distribution.FetchIndexError):\n fetcher.conditional_fetch()\n\n\ndef test_v2_default_index_fetch_200():\n index_json = '{\"Hello\": \"World\"}'\n index_json_hash = spack.binary_distribution.compute_hash(index_json)\n\n def urlopen(request: urllib.request.Request):\n url = request.get_full_url()\n if url.endswith(\"index.json.hash\"):\n return urllib.response.addinfourl( # type: ignore[arg-type]\n io.BytesIO(index_json_hash.encode()),\n headers={}, # type: ignore[arg-type]\n url=url,\n code=200,\n )\n\n elif url.endswith(INDEX_JSON_FILE):\n return urllib.response.addinfourl(\n io.BytesIO(index_json.encode()),\n headers={\"Etag\": '\"59bcc3ad6775562f845953cf01624225\"'}, # type: ignore[arg-type]\n url=url,\n code=200,\n )\n\n assert False, \"Unexpected request {}\".format(url)\n\n fetcher = spack.binary_distribution.DefaultIndexFetcherV2(\n url=\"https://www.example.com\", local_hash=\"outdated\", urlopen=urlopen\n )\n\n result = fetcher.conditional_fetch()\n\n assert isinstance(result, spack.binary_distribution.FetchIndexResult)\n assert not result.fresh\n assert result.etag == \"59bcc3ad6775562f845953cf01624225\"\n assert result.data == index_json\n assert result.hash == index_json_hash\n\n\ndef test_v2_default_index_dont_fetch_index_json_hash_if_no_local_hash():\n # When we don't have local hash, we should not be fetching the\n # remote index.json.hash file, but only index.json.\n index_json = '{\"Hello\": \"World\"}'\n index_json_hash = spack.binary_distribution.compute_hash(index_json)\n\n def urlopen(request: urllib.request.Request):\n url = request.get_full_url()\n if url.endswith(INDEX_JSON_FILE):\n return urllib.response.addinfourl(\n io.BytesIO(index_json.encode()),\n headers={\"Etag\": '\"59bcc3ad6775562f845953cf01624225\"'}, # type: ignore[arg-type]\n url=url,\n code=200,\n )\n\n assert False, \"Unexpected request {}\".format(url)\n\n fetcher = spack.binary_distribution.DefaultIndexFetcherV2(\n url=\"https://www.example.com\", local_hash=None, urlopen=urlopen\n )\n\n result = fetcher.conditional_fetch()\n\n assert isinstance(result, spack.binary_distribution.FetchIndexResult)\n assert result.data == index_json\n assert result.hash == index_json_hash\n assert result.etag == \"59bcc3ad6775562f845953cf01624225\"\n assert not result.fresh\n\n\ndef test_v2_default_index_not_modified():\n index_json = '{\"Hello\": \"World\"}'\n index_json_hash = spack.binary_distribution.compute_hash(index_json)\n\n def urlopen(request: urllib.request.Request):\n url = request.get_full_url()\n if url.endswith(\"index.json.hash\"):\n return urllib.response.addinfourl(\n io.BytesIO(index_json_hash.encode()),\n headers={}, # type: ignore[arg-type]\n url=url,\n code=200,\n )\n\n # No request to index.json should be made.\n assert False, \"Unexpected request {}\".format(url)\n\n fetcher = spack.binary_distribution.DefaultIndexFetcherV2(\n url=\"https://www.example.com\", local_hash=index_json_hash, urlopen=urlopen\n )\n\n assert fetcher.conditional_fetch().fresh\n\n\n@pytest.mark.parametrize(\"index_json\", [b\"\\xa9\", b\"!#%^\"])\ndef test_v2_default_index_invalid_hash_file(index_json):\n # Test invalid unicode / invalid hash type\n index_json_hash = spack.binary_distribution.compute_hash(index_json)\n\n def urlopen(request: urllib.request.Request):\n return urllib.response.addinfourl(\n io.BytesIO(),\n headers={}, # type: ignore[arg-type]\n url=request.get_full_url(),\n code=200,\n )\n\n fetcher = spack.binary_distribution.DefaultIndexFetcherV2(\n url=\"https://www.example.com\", local_hash=index_json_hash, urlopen=urlopen\n )\n\n assert fetcher.get_remote_hash() is None\n\n\ndef test_v2_default_index_json_404():\n # Test invalid unicode / invalid hash type\n index_json = '{\"Hello\": \"World\"}'\n index_json_hash = spack.binary_distribution.compute_hash(index_json)\n\n def urlopen(request: urllib.request.Request):\n url = request.get_full_url()\n if url.endswith(\"index.json.hash\"):\n return urllib.response.addinfourl(\n io.BytesIO(index_json_hash.encode()),\n headers={}, # type: ignore[arg-type]\n url=url,\n code=200,\n )\n\n elif url.endswith(INDEX_JSON_FILE):\n raise urllib.error.HTTPError(\n url,\n code=404,\n msg=\"Not Found\",\n hdrs={\"Etag\": '\"59bcc3ad6775562f845953cf01624225\"'}, # type: ignore[arg-type]\n fp=None,\n )\n\n assert False, \"Unexpected fetch {}\".format(url)\n\n fetcher = spack.binary_distribution.DefaultIndexFetcherV2(\n url=\"https://www.example.com\", local_hash=\"invalid\", urlopen=urlopen\n )\n\n with pytest.raises(spack.binary_distribution.FetchIndexError, match=\"Could not fetch index\"):\n fetcher.conditional_fetch()\n\n\ndef _all_parents(prefix):\n parts = [p for p in prefix.split(\"/\")]\n return [\"/\".join(parts[: i + 1]) for i in range(len(parts))]\n\n\ndef test_tarball_doesnt_include_buildinfo_twice(tmp_path: Path):\n \"\"\"When tarballing a package that was installed from a buildcache, make\n sure that the buildinfo file is not included twice in the tarball.\"\"\"\n p = tmp_path / \"prefix\"\n p.joinpath(\".spack\").mkdir(parents=True)\n\n # Create a binary_distribution file in the .spack folder\n with open(p / \".spack\" / \"binary_distribution\", \"w\", encoding=\"utf-8\") as f:\n f.write(syaml.dump({\"metadata\", \"old\"}))\n\n # Now create a tarball, which should include a new binary_distribution file\n tarball = str(tmp_path / \"prefix.tar.gz\")\n\n spack.binary_distribution._do_create_tarball(\n tarfile_path=tarball, prefix=str(p), buildinfo={\"metadata\": \"new\"}, prefixes_to_relocate=[]\n )\n\n expected_prefix = str(p).lstrip(\"/\")\n\n # Verify we don't have a repeated binary_distribution file,\n # and that the tarball contains the new one, not the old one.\n with tarfile.open(tarball) as tar:\n assert syaml.load(tar.extractfile(f\"{expected_prefix}/.spack/binary_distribution\")) == {\n \"metadata\": \"new\",\n \"relocate_binaries\": [],\n \"relocate_textfiles\": [],\n \"relocate_links\": [],\n }\n assert tar.getnames() == [\n *_all_parents(expected_prefix),\n f\"{expected_prefix}/.spack\",\n f\"{expected_prefix}/.spack/binary_distribution\",\n ]\n\n\ndef test_reproducible_tarball_is_reproducible(tmp_path: Path):\n p = tmp_path / \"prefix\"\n p.joinpath(\"bin\").mkdir(parents=True)\n p.joinpath(\".spack\").mkdir(parents=True)\n app = p / \"bin\" / \"app\"\n\n tarball_1 = str(tmp_path / \"prefix-1.tar.gz\")\n tarball_2 = str(tmp_path / \"prefix-2.tar.gz\")\n\n with open(app, \"w\", encoding=\"utf-8\") as f:\n f.write(\"hello world\")\n\n buildinfo = {\"metadata\": \"yes please\"}\n\n # Create a tarball with a certain mtime of bin/app\n os.utime(app, times=(0, 0))\n spack.binary_distribution._do_create_tarball(\n tarball_1, prefix=str(p), buildinfo=buildinfo, prefixes_to_relocate=[]\n )\n\n # Do it another time with different mtime of bin/app\n os.utime(app, times=(10, 10))\n spack.binary_distribution._do_create_tarball(\n tarball_2, prefix=str(p), buildinfo=buildinfo, prefixes_to_relocate=[]\n )\n\n # They should be bitwise identical:\n assert filecmp.cmp(tarball_1, tarball_2, shallow=False)\n\n expected_prefix = str(p).lstrip(\"/\")\n\n # Sanity check for contents:\n with tarfile.open(tarball_1, mode=\"r\") as f:\n for m in f.getmembers():\n assert m.uid == m.gid == m.mtime == 0\n assert m.uname == m.gname == \"\"\n\n assert set(f.getnames()) == {\n *_all_parents(expected_prefix),\n f\"{expected_prefix}/bin\",\n f\"{expected_prefix}/bin/app\",\n f\"{expected_prefix}/.spack\",\n f\"{expected_prefix}/.spack/binary_distribution\",\n }\n\n\ndef test_tarball_normalized_permissions(tmp_path: pathlib.Path):\n p = tmp_path / \"prefix\"\n p.mkdir()\n (p / \"bin\").mkdir()\n (p / \"share\").mkdir()\n (p / \".spack\").mkdir()\n\n app = p / \"bin\" / \"app\"\n data = p / \"share\" / \"file\"\n tarball = str(tmp_path / \"prefix.tar.gz\")\n\n # Everyone can write & execute. This should turn into 0o755 when the tarball is\n # extracted (on a different system).\n with open(\n app, \"w\", opener=lambda path, flags: os.open(path, flags, 0o777), encoding=\"utf-8\"\n ) as f:\n f.write(\"hello world\")\n\n # User doesn't have execute permissions, but group/world have; this should also\n # turn into 0o644 (user read/write, group&world only read).\n with open(\n data, \"w\", opener=lambda path, flags: os.open(path, flags, 0o477), encoding=\"utf-8\"\n ) as f:\n f.write(\"hello world\")\n\n spack.binary_distribution._do_create_tarball(\n tarball, prefix=str(p), buildinfo={}, prefixes_to_relocate=[]\n )\n\n expected_prefix = str(p).lstrip(\"/\")\n\n with tarfile.open(tarball) as tar:\n path_to_member = {member.name: member for member in tar.getmembers()}\n\n # directories should have 0o755\n assert path_to_member[f\"{expected_prefix}\"].mode == 0o755\n assert path_to_member[f\"{expected_prefix}/bin\"].mode == 0o755\n assert path_to_member[f\"{expected_prefix}/.spack\"].mode == 0o755\n\n # executable-by-user files should be 0o755\n assert path_to_member[f\"{expected_prefix}/bin/app\"].mode == 0o755\n\n # not-executable-by-user files should be 0o644\n assert path_to_member[f\"{expected_prefix}/share/file\"].mode == 0o644\n\n\ndef test_tarball_common_prefix(dummy_prefix, tmp_path: pathlib.Path):\n \"\"\"Tests whether Spack can figure out the package directory from the tarball contents, and\n strip them when extracting. This test creates a CURRENT_BUILD_CACHE_LAYOUT_VERSION=1 type\n tarball where the parent directories of the package prefix are missing. Spack should be able\n to figure out the common prefix and extract the files into the correct location.\"\"\"\n\n # When creating a tarball, Python (and tar) use relative paths,\n # Absolute paths become relative to `/`, so drop the leading `/`.\n assert os.path.isabs(dummy_prefix)\n expected_prefix = PurePath(dummy_prefix).as_posix().lstrip(\"/\")\n\n with working_dir(str(tmp_path)):\n # Create a tarball (using absolute path for prefix dir)\n with tarfile.open(\"example.tar\", mode=\"w\") as tar:\n tar.add(name=dummy_prefix)\n\n # Open, verify common prefix, and extract it.\n with tarfile.open(\"example.tar\", mode=\"r\") as tar:\n common_prefix = spack.binary_distribution._ensure_common_prefix(tar)\n assert common_prefix == expected_prefix\n\n # For consistent behavior across all supported Python versions\n tar.extraction_filter = lambda member, path: member\n\n # Extract into prefix2\n tar.extractall(\n path=\"prefix2\",\n members=spack.binary_distribution._tar_strip_component(tar, common_prefix),\n )\n\n # Verify files are all there at the correct level.\n assert set(os.listdir(\"prefix2\")) == {\"bin\", \"share\", \".spack\"}\n assert set(os.listdir(os.path.join(\"prefix2\", \"bin\"))) == {\n \"app\",\n \"relative_app_link\",\n \"absolute_app_link\",\n }\n assert set(os.listdir(os.path.join(\"prefix2\", \"share\"))) == {\"file\"}\n\n # Relative symlink should still be correct\n assert readlink(os.path.join(\"prefix2\", \"bin\", \"relative_app_link\")) == \"app\"\n\n # Absolute symlink should remain absolute -- this is for relocation to fix up.\n assert readlink(os.path.join(\"prefix2\", \"bin\", \"absolute_app_link\")) == os.path.join(\n dummy_prefix, \"bin\", \"app\"\n )\n\n\ndef test_tarfile_missing_binary_distribution_file(tmp_path: pathlib.Path):\n \"\"\"A tarfile that does not contain a .spack/binary_distribution file cannot be\n used to install.\"\"\"\n with working_dir(str(tmp_path)):\n # An empty .spack dir.\n with tarfile.open(\"empty.tar\", mode=\"w\") as tar:\n tarinfo = tarfile.TarInfo(name=\"example/.spack\")\n tarinfo.type = tarfile.DIRTYPE\n tar.addfile(tarinfo)\n\n with pytest.raises(ValueError, match=\"missing binary_distribution file\"):\n spack.binary_distribution._ensure_common_prefix(tarfile.open(\"empty.tar\", mode=\"r\"))\n\n\ndef test_tarfile_without_common_directory_prefix_fails(tmp_path: pathlib.Path):\n \"\"\"A tarfile that only contains files without a common package directory\n should fail to extract, as we won't know where to put the files.\"\"\"\n with working_dir(str(tmp_path)):\n # Create a broken tarball with just a file, no directories.\n with tarfile.open(\"empty.tar\", mode=\"w\") as tar:\n tar.addfile(\n tarfile.TarInfo(name=\"example/.spack/binary_distribution\"),\n fileobj=io.BytesIO(b\"hello\"),\n )\n\n with pytest.raises(ValueError, match=\"Tarball does not contain a common prefix\"):\n spack.binary_distribution._ensure_common_prefix(tarfile.open(\"empty.tar\", mode=\"r\"))\n\n\ndef test_tarfile_with_files_outside_common_prefix(tmp_path: pathlib.Path, dummy_prefix):\n \"\"\"If a file is outside of the common prefix, we should fail.\"\"\"\n with working_dir(str(tmp_path)):\n with tarfile.open(\"broken.tar\", mode=\"w\") as tar:\n tar.add(name=dummy_prefix)\n tar.addfile(tarfile.TarInfo(name=\"/etc/config_file\"), fileobj=io.BytesIO(b\"hello\"))\n\n with pytest.raises(\n ValueError, match=\"Tarball contains file /etc/config_file outside of prefix\"\n ):\n spack.binary_distribution._ensure_common_prefix(tarfile.open(\"broken.tar\", mode=\"r\"))\n\n\ndef test_tarfile_of_spec_prefix(tmp_path: pathlib.Path):\n \"\"\"Tests whether hardlinks, symlinks, files and dirs are added correctly,\n and that the order of entries is correct.\"\"\"\n prefix = tmp_path / \"prefix\"\n prefix.mkdir()\n\n (prefix / \"a_directory\").mkdir()\n (prefix / \"a_directory\" / \"file\").write_text(\"hello\")\n\n (prefix / \"c_directory\").mkdir()\n (prefix / \"c_directory\" / \"file\").write_text(\"hello\")\n\n (prefix / \"b_directory\").mkdir()\n (prefix / \"b_directory\" / \"file\").write_text(\"hello\")\n\n (prefix / \"file\").write_text(\"hello\")\n os.symlink(str(prefix / \"file\"), str(prefix / \"symlink\"))\n os.link(str(prefix / \"file\"), str(prefix / \"hardlink\"))\n\n file = tmp_path / \"example.tar\"\n\n with tarfile.open(str(file), mode=\"w\") as tar:\n spack.binary_distribution.tarfile_of_spec_prefix(tar, str(prefix), prefixes_to_relocate=[])\n\n expected_prefix = str(prefix).lstrip(\"/\")\n\n with tarfile.open(str(file), mode=\"r\") as tar:\n # Verify that entries are added in depth-first pre-order, files preceding dirs,\n # entries ordered alphabetically\n assert tar.getnames() == [\n *_all_parents(expected_prefix),\n f\"{expected_prefix}/file\",\n f\"{expected_prefix}/hardlink\",\n f\"{expected_prefix}/symlink\",\n f\"{expected_prefix}/a_directory\",\n f\"{expected_prefix}/a_directory/file\",\n f\"{expected_prefix}/b_directory\",\n f\"{expected_prefix}/b_directory/file\",\n f\"{expected_prefix}/c_directory\",\n f\"{expected_prefix}/c_directory/file\",\n ]\n\n # Check that the types are all correct\n assert tar.getmember(f\"{expected_prefix}\").isdir()\n assert tar.getmember(f\"{expected_prefix}/file\").isreg()\n assert tar.getmember(f\"{expected_prefix}/hardlink\").islnk()\n assert tar.getmember(f\"{expected_prefix}/symlink\").issym()\n assert tar.getmember(f\"{expected_prefix}/a_directory\").isdir()\n assert tar.getmember(f\"{expected_prefix}/a_directory/file\").isreg()\n assert tar.getmember(f\"{expected_prefix}/b_directory\").isdir()\n assert tar.getmember(f\"{expected_prefix}/b_directory/file\").isreg()\n assert tar.getmember(f\"{expected_prefix}/c_directory\").isdir()\n assert tar.getmember(f\"{expected_prefix}/c_directory/file\").isreg()\n\n\n@pytest.mark.parametrize(\"layout,expect_success\", [(None, True), (1, True), (2, False)])\ndef test_get_valid_spec_file(tmp_path: pathlib.Path, layout, expect_success):\n # Test reading a spec.json file that does not specify a layout version.\n spec_dict = Spec(\"example\").to_dict()\n path = tmp_path / \"spec.json\"\n effective_layout = layout or 0 # If not specified it should be 0\n\n # Add a layout version\n if layout is not None:\n spec_dict[\"buildcache_layout_version\"] = layout\n\n # Save to file\n with open(path, \"w\", encoding=\"utf-8\") as f:\n json.dump(spec_dict, f)\n\n try:\n spec_dict_disk, layout_disk = get_valid_spec_file(str(path), max_supported_layout=1)\n assert expect_success\n assert spec_dict_disk == spec_dict\n assert layout_disk == effective_layout\n except spack.binary_distribution.InvalidMetadataFile:\n assert not expect_success\n\n\ndef test_get_valid_spec_file_doesnt_exist(tmp_path: pathlib.Path):\n with pytest.raises(spack.binary_distribution.InvalidMetadataFile, match=\"No such file\"):\n get_valid_spec_file(str(tmp_path / \"no-such-file\"), max_supported_layout=1)\n\n\n@pytest.mark.parametrize(\"filename\", [\"spec.json\", \"spec.json.sig\"])\ndef test_get_valid_spec_file_no_json(tmp_path: pathlib.Path, filename):\n tmp_path.joinpath(filename).write_text(\"not json\")\n with pytest.raises(spack.binary_distribution.InvalidMetadataFile):\n get_valid_spec_file(str(tmp_path / filename), max_supported_layout=1)\n\n\n@pytest.mark.usefixtures(\"install_mockery\", \"mock_packages\", \"mock_fetch\", \"temporary_mirror\")\ndef test_url_buildcache_entry_v3(monkeypatch, tmp_path: pathlib.Path):\n \"\"\"Make sure URLBuildcacheEntry behaves as expected\"\"\"\n\n # Create a temp mirror directory for buildcache usage\n mirror_dir = tmp_path / \"mirror_dir\"\n mirror_url = url_util.path_to_file_url(str(mirror_dir))\n\n s = spack.concretize.concretize_one(\"libdwarf\")\n\n # Install libdwarf\n install_cmd(\"--fake\", s.name)\n\n # Push libdwarf to buildcache\n buildcache_cmd(\"push\", \"-u\", str(mirror_dir), s.name)\n\n cache_class = get_url_buildcache_class(\n spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION\n )\n build_cache = cache_class(mirror_url, s, allow_unsigned=True)\n\n manifest = build_cache.read_manifest()\n spec_dict = build_cache.fetch_metadata()\n local_tarball_path = build_cache.fetch_archive()\n\n assert \"spec\" in spec_dict\n\n for blob_record in manifest.data:\n blob_path = build_cache.get_staged_blob_path(blob_record)\n assert os.path.exists(blob_path)\n actual_blob_size = os.stat(blob_path).st_size\n assert blob_record.content_length == actual_blob_size\n\n build_cache.destroy()\n\n assert not os.path.exists(local_tarball_path)\n\n\ndef test_relative_path_components():\n blobs_v3 = URLBuildcacheEntry.get_relative_path_components(BuildcacheComponent.BLOB)\n assert len(blobs_v3) == 1\n assert \"blobs\" in blobs_v3\n\n blobs_v2 = URLBuildcacheEntryV2.get_relative_path_components(BuildcacheComponent.BLOB)\n assert len(blobs_v2) == 1\n assert \"build_cache\" in blobs_v2\n\n v2_spec_url = \"file:///home/me/mymirror/build_cache/linux-ubuntu22.04-sapphirerapids-gcc-12.3.0-gmake-4.4.1-5pddli3htvfe6svs7nbrqmwi5735agi3.spec.json.sig\"\n assert URLBuildcacheEntryV2.get_base_url(v2_spec_url) == \"file:///home/me/mymirror\"\n\n v3_manifest_url = \"file:///home/me/mymirror/v3/manifests/gmake-4.4.1-5pddli3htvfe6svs7nbrqmwi5735agi3.spec.manifest.json\"\n assert URLBuildcacheEntry.get_base_url(v3_manifest_url) == \"file:///home/me/mymirror\"\n\n\n@pytest.mark.parametrize(\n \"spec\",\n [\n # Standard case\n \"short-name@=1.2.3\",\n # Unsupported characters in git version\n f\"git-version@{1:040x}=develop\",\n # Too long of a name\n f\"{'too-long':x<256}@=1.2.3\",\n ],\n)\ndef test_default_tag(spec: str):\n \"\"\"Make sure that computed image tags are valid.\"\"\"\n assert re.fullmatch(\n spack.oci.image.tag, spack.binary_distribution._oci_default_tag(spack.spec.Spec(spec))\n )\n\n\nclass IndexInformation(NamedTuple):\n manifest_contents: Dict[str, Any]\n index_contents: str\n index_hash: str\n manifest_path: str\n index_path: str\n manifest_etag: str\n fetched_blob: Callable[[], bool]\n\n\n@pytest.fixture\ndef mock_index(tmp_path: pathlib.Path, monkeypatch) -> IndexInformation:\n mirror_root = tmp_path / \"mymirror\"\n index_json = '{\"Hello\": \"World\"}'\n index_json_hash = spack.binary_distribution.compute_hash(index_json)\n fetched = False\n\n cache_class = get_url_buildcache_class(\n layout_version=spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION\n )\n\n index_blob_path = os.path.join(\n str(mirror_root),\n *cache_class.get_relative_path_components(BuildcacheComponent.BLOB),\n \"sha256\",\n index_json_hash[:2],\n index_json_hash,\n )\n\n os.makedirs(os.path.dirname(index_blob_path))\n with open(index_blob_path, \"w\", encoding=\"utf-8\") as fd:\n fd.write(index_json)\n\n index_blob_record = spack.binary_distribution.BlobRecord(\n os.stat(index_blob_path).st_size,\n cache_class.BUILDCACHE_INDEX_MEDIATYPE,\n \"none\",\n \"sha256\",\n index_json_hash,\n )\n\n index_manifest = {\n \"version\": cache_class.get_layout_version(),\n \"data\": [index_blob_record.to_dict()],\n }\n\n manifest_json_path = cache_class.get_index_url(str(mirror_root))\n\n os.makedirs(os.path.dirname(manifest_json_path))\n\n with open(manifest_json_path, \"w\", encoding=\"utf-8\") as f:\n json.dump(index_manifest, f)\n\n def fetch_patch(stage, mirror_only: bool = False, err_msg: Optional[str] = None):\n nonlocal fetched\n fetched = True\n\n @property # type: ignore\n def save_filename_patch(stage):\n return str(index_blob_path)\n\n monkeypatch.setattr(spack.stage.Stage, \"fetch\", fetch_patch)\n monkeypatch.setattr(spack.stage.Stage, \"save_filename\", save_filename_patch)\n\n def get_did_fetch():\n # nonlocal fetched\n return fetched\n\n return IndexInformation(\n index_manifest,\n index_json,\n index_json_hash,\n manifest_json_path,\n index_blob_path,\n \"59bcc3ad6775562f845953cf01624225\",\n get_did_fetch,\n )\n\n\ndef test_etag_fetching_304():\n # Test conditional fetch with etags. If the remote hasn't modified the file\n # it returns 304, which is an HTTPError in urllib-land. That should be\n # handled as success, since it means the local cache is up-to-date.\n def response_304(request: urllib.request.Request):\n url = request.get_full_url()\n if url.endswith(INDEX_MANIFEST_FILE):\n assert request.get_header(\"If-none-match\") == '\"112a8bbc1b3f7f185621c1ee335f0502\"'\n raise urllib.error.HTTPError(\n url,\n 304,\n \"Not Modified\",\n hdrs={}, # type: ignore[arg-type]\n fp=None, # type: ignore[arg-type]\n )\n assert False, \"Unexpected request {}\".format(url)\n\n fetcher = spack.binary_distribution.EtagIndexFetcher(\n spack.binary_distribution.MirrorMetadata(\n \"https://www.example.com\", spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION\n ),\n etag=\"112a8bbc1b3f7f185621c1ee335f0502\",\n urlopen=response_304,\n )\n\n result = fetcher.conditional_fetch()\n assert isinstance(result, spack.binary_distribution.FetchIndexResult)\n assert result.fresh\n\n\ndef test_etag_fetching_200(mock_index):\n # Test conditional fetch with etags. The remote has modified the file.\n def response_200(request: urllib.request.Request):\n url = request.get_full_url()\n if url.endswith(INDEX_MANIFEST_FILE):\n assert request.get_header(\"If-none-match\") == '\"112a8bbc1b3f7f185621c1ee335f0502\"'\n return urllib.response.addinfourl(\n io.BytesIO(json.dumps(mock_index.manifest_contents).encode()),\n headers={\"Etag\": f'\"{mock_index.manifest_etag}\"'}, # type: ignore[arg-type]\n url=url,\n code=200,\n )\n assert False, \"Unexpected request {}\".format(url)\n\n fetcher = spack.binary_distribution.EtagIndexFetcher(\n spack.binary_distribution.MirrorMetadata(\n \"https://www.example.com\", spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION\n ),\n etag=\"112a8bbc1b3f7f185621c1ee335f0502\",\n urlopen=response_200,\n )\n\n result = fetcher.conditional_fetch()\n assert isinstance(result, spack.binary_distribution.FetchIndexResult)\n assert not result.fresh\n assert mock_index.fetched_blob()\n assert result.etag == mock_index.manifest_etag\n assert result.data == mock_index.index_contents\n assert result.hash == mock_index.index_hash\n\n\ndef test_etag_fetching_404():\n # Test conditional fetch with etags. The remote has modified the file.\n def response_404(request: urllib.request.Request):\n raise urllib.error.HTTPError(\n request.get_full_url(),\n 404,\n \"Not found\",\n hdrs={\"Etag\": '\"59bcc3ad6775562f845953cf01624225\"'}, # type: ignore[arg-type]\n fp=None,\n )\n\n fetcher = spack.binary_distribution.EtagIndexFetcher(\n spack.binary_distribution.MirrorMetadata(\n \"https://www.example.com\", spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION\n ),\n etag=\"112a8bbc1b3f7f185621c1ee335f0502\",\n urlopen=response_404,\n )\n\n with pytest.raises(spack.binary_distribution.FetchIndexError):\n fetcher.conditional_fetch()\n\n\ndef test_default_index_fetch_200(mock_index):\n # We fetch the manifest and then the index blob if the hash is outdated\n def urlopen(request: urllib.request.Request):\n url = request.get_full_url()\n if url.endswith(INDEX_MANIFEST_FILE):\n return urllib.response.addinfourl( # type: ignore[arg-type]\n io.BytesIO(json.dumps(mock_index.manifest_contents).encode()),\n headers={\"Etag\": f'\"{mock_index.manifest_etag}\"'}, # type: ignore[arg-type]\n url=url,\n code=200,\n )\n\n assert False, \"Unexpected request {}\".format(url)\n\n fetcher = spack.binary_distribution.DefaultIndexFetcher(\n spack.binary_distribution.MirrorMetadata(\n \"https://www.example.com\", spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION\n ),\n local_hash=\"outdated\",\n urlopen=urlopen,\n )\n\n result = fetcher.conditional_fetch()\n\n assert isinstance(result, spack.binary_distribution.FetchIndexResult)\n assert not result.fresh\n assert mock_index.fetched_blob()\n assert result.etag == mock_index.manifest_etag\n assert result.data == mock_index.index_contents\n assert result.hash == mock_index.index_hash\n\n\ndef test_default_index_404():\n # We get a fetch error if the index can't be fetched\n def urlopen(request: urllib.request.Request):\n raise urllib.error.HTTPError(\n request.get_full_url(),\n 404,\n \"Not found\",\n hdrs={\"Etag\": '\"59bcc3ad6775562f845953cf01624225\"'}, # type: ignore[arg-type]\n fp=None,\n )\n\n fetcher = spack.binary_distribution.DefaultIndexFetcher(\n spack.binary_distribution.MirrorMetadata(\n \"https://www.example.com\", spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION\n ),\n local_hash=None,\n urlopen=urlopen,\n )\n\n with pytest.raises(spack.binary_distribution.FetchIndexError):\n fetcher.conditional_fetch()\n\n\ndef test_default_index_not_modified(mock_index):\n # We don't fetch the index blob if hash didn't change\n def urlopen(request: urllib.request.Request):\n url = request.get_full_url()\n if url.endswith(INDEX_MANIFEST_FILE):\n return urllib.response.addinfourl(\n io.BytesIO(json.dumps(mock_index.manifest_contents).encode()),\n headers={}, # type: ignore[arg-type]\n url=url,\n code=200,\n )\n\n # No other request should be made.\n assert False, \"Unexpected request {}\".format(url)\n\n fetcher = spack.binary_distribution.DefaultIndexFetcher(\n spack.binary_distribution.MirrorMetadata(\n \"https://www.example.com\", spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION\n ),\n local_hash=mock_index.index_hash,\n urlopen=urlopen,\n )\n\n assert fetcher.conditional_fetch().fresh\n assert not mock_index.fetched_blob()\n\n\n@pytest.mark.usefixtures(\"install_mockery\", \"mock_packages\")\ndef test_get_entries_from_cache_nested_mirrors(monkeypatch, tmp_path: pathlib.Path):\n \"\"\"Make sure URLBuildcacheEntry behaves as expected\"\"\"\n\n # Create a temp mirror directory for buildcache usage\n mirror_dir = tmp_path / \"mirror_dir\"\n mirror_url = url_util.path_to_file_url(str(mirror_dir))\n\n # Install and push libdwarf to the root mirror\n s = spack.concretize.concretize_one(\"libdwarf\")\n install_cmd(\"--fake\", s.name)\n buildcache_cmd(\"push\", \"-u\", str(mirror_dir), s.name)\n\n # Install and push libzlib to the nested mirror\n s = spack.concretize.concretize_one(\"zlib\")\n install_cmd(\"--fake\", s.name)\n buildcache_cmd(\"push\", \"-u\", str(mirror_dir / \"nested\"), s.name)\n\n spec_manifests, _ = get_entries_from_cache(\n str(mirror_url), str(tmp_path / \"stage\"), BuildcacheComponent.SPEC\n )\n\n nested_mirror_url = url_util.path_to_file_url(str(mirror_dir / \"nested\"))\n spec_manifests_nested, _ = get_entries_from_cache(\n str(nested_mirror_url), str(tmp_path / \"stage\"), BuildcacheComponent.SPEC\n )\n\n # Expected specs in root mirror\n # - gcc-runtime\n # - compiler-wrapper\n # - libelf\n # - libdwarf\n assert len(spec_manifests) == 4\n # Expected specs in nested mirror\n # - zlib\n assert len(spec_manifests_nested) == 1\n\n\ndef test_mirror_metadata():\n mirror_metadata = spack.binary_distribution.MirrorMetadata(\"https://dummy.io/__v3\", 3)\n as_str = str(mirror_metadata)\n from_str = spack.binary_distribution.MirrorMetadata.from_string(as_str)\n\n # Verify values\n assert mirror_metadata.url == \"https://dummy.io/__v3\"\n assert mirror_metadata.version == 3\n assert mirror_metadata.view is None\n\n # Verify round trip\n assert mirror_metadata == from_str\n assert as_str == str(from_str)\n\n with pytest.raises(spack.url_buildcache.MirrorMetadataError, match=\"Malformed string\"):\n spack.binary_distribution.MirrorMetadata.from_string(\"https://dummy.io/__v3@@4\")\n\n\ndef test_mirror_metadata_with_view():\n mirror_metadata = spack.binary_distribution.MirrorMetadata(\n \"https://dummy.io/__v3__@aview\", 3, \"aview\"\n )\n as_str = str(mirror_metadata)\n from_str = spack.binary_distribution.MirrorMetadata.from_string(as_str)\n\n # Verify round trip\n assert mirror_metadata.url == \"https://dummy.io/__v3__@aview\"\n assert mirror_metadata.version == 3\n assert mirror_metadata.view == \"aview\"\n assert mirror_metadata == from_str\n assert as_str == str(from_str)\n\n with pytest.raises(spack.url_buildcache.MirrorMetadataError, match=\"Malformed string\"):\n spack.binary_distribution.MirrorMetadata.from_string(\"https://dummy.io/__v3%asdf__@aview\")\n" }, { "path": "lib/spack/spack/test/bootstrap.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport pathlib\n\nimport pytest\n\nimport spack.bootstrap\nimport spack.bootstrap.clingo\nimport spack.bootstrap.config\nimport spack.bootstrap.core\nimport spack.bootstrap.status\nimport spack.compilers.config\nimport spack.config\nimport spack.environment\nimport spack.store\nimport spack.util.executable\nimport spack.util.path\n\nfrom .conftest import _true\n\n\n@pytest.fixture\ndef active_mock_environment(mutable_config, mutable_mock_env_path):\n with spack.environment.create(\"bootstrap-test\") as env:\n yield env\n\n\n@pytest.mark.regression(\"22294\")\ndef test_store_is_restored_correctly_after_bootstrap(mutable_config, tmp_path: pathlib.Path):\n \"\"\"Tests that the store is correctly swapped during bootstrapping, and restored afterward.\"\"\"\n user_path = str(tmp_path / \"store\")\n with spack.store.use_store(user_path):\n assert spack.store.STORE.root == user_path\n assert spack.config.CONFIG.get(\"config:install_tree:root\") == user_path\n with spack.bootstrap.ensure_bootstrap_configuration():\n assert spack.store.STORE.root == spack.bootstrap.config.store_path()\n assert spack.store.STORE.root == user_path\n assert spack.config.CONFIG.get(\"config:install_tree:root\") == user_path\n\n\n@pytest.mark.regression(\"38963\")\ndef test_store_padding_length_is_zero_during_bootstrapping(mutable_config, tmp_path: pathlib.Path):\n \"\"\"Tests that, even though padded length is set in user config, the bootstrap store maintains\n a padded length of zero.\n \"\"\"\n user_path = str(tmp_path / \"store\")\n with spack.store.use_store(user_path, extra_data={\"padded_length\": 512}):\n assert spack.config.CONFIG.get(\"config:install_tree:padded_length\") == 512\n with spack.bootstrap.ensure_bootstrap_configuration():\n assert spack.store.STORE.root == spack.bootstrap.config.store_path()\n assert spack.config.CONFIG.get(\"config:install_tree:padded_length\") == 0\n assert spack.config.CONFIG.get(\"config:install_tree:padded_length\") == 512\n\n\n@pytest.mark.regression(\"38963\")\ndef test_install_tree_customization_is_respected(mutable_config, tmp_path: pathlib.Path):\n \"\"\"Tests that a custom user store is respected when we exit the bootstrapping\n environment.\n \"\"\"\n spack.store.reinitialize()\n store_dir = tmp_path / \"store\"\n spack.config.CONFIG.set(\"config:install_tree:root\", str(store_dir))\n with spack.bootstrap.ensure_bootstrap_configuration():\n assert spack.store.STORE.root == spack.bootstrap.config.store_path()\n assert (\n spack.config.CONFIG.get(\"config:install_tree:root\")\n == spack.bootstrap.config.store_path()\n )\n assert spack.config.CONFIG.get(\"config:install_tree:padded_length\") == 0\n assert spack.config.CONFIG.get(\"config:install_tree:root\") == str(store_dir)\n assert spack.store.STORE.root == str(store_dir)\n\n\n@pytest.mark.parametrize(\n \"config_value,expected\",\n [\n # Absolute path without expansion\n (\"/opt/spack/bootstrap\", \"/opt/spack/bootstrap/store\"),\n # Path with placeholder\n (\"$spack/opt/bootstrap\", \"$spack/opt/bootstrap/store\"),\n ],\n)\ndef test_store_path_customization(config_value, expected, mutable_config):\n # Set the current configuration to a specific value\n spack.config.set(\"bootstrap:root\", config_value)\n\n # Check the store path\n current = spack.bootstrap.config.store_path()\n assert current == spack.util.path.canonicalize_path(expected)\n\n\ndef test_raising_exception_if_bootstrap_disabled(mutable_config):\n # Disable bootstrapping in config.yaml\n spack.config.set(\"bootstrap:enable\", False)\n\n # Check the correct exception is raised\n with pytest.raises(RuntimeError, match=\"bootstrapping is currently disabled\"):\n spack.bootstrap.config.store_path()\n\n\ndef test_raising_exception_module_importable(config, monkeypatch):\n monkeypatch.setattr(spack.bootstrap.core, \"source_is_enabled\", _true)\n with pytest.raises(ImportError, match='cannot bootstrap the \"asdf\" Python module'):\n spack.bootstrap.core.ensure_module_importable_or_raise(\"asdf\")\n\n\ndef test_raising_exception_executables_in_path(config, monkeypatch):\n monkeypatch.setattr(spack.bootstrap.core, \"source_is_enabled\", _true)\n with pytest.raises(RuntimeError, match=\"cannot bootstrap any of the asdf, fdsa executables\"):\n spack.bootstrap.core.ensure_executables_in_path_or_raise([\"asdf\", \"fdsa\"], \"python\")\n\n\n@pytest.mark.regression(\"25603\")\ndef test_bootstrap_deactivates_environments(active_mock_environment):\n assert spack.environment.active_environment() == active_mock_environment\n with spack.bootstrap.ensure_bootstrap_configuration():\n assert spack.environment.active_environment() is None\n assert spack.environment.active_environment() == active_mock_environment\n\n\n@pytest.mark.regression(\"25805\")\ndef test_bootstrap_disables_modulefile_generation(mutable_config):\n # Be sure to enable both lmod and tcl in modules.yaml\n spack.config.set(\"modules:default:enable\", [\"tcl\", \"lmod\"])\n\n assert \"tcl\" in spack.config.get(\"modules:default:enable\")\n assert \"lmod\" in spack.config.get(\"modules:default:enable\")\n with spack.bootstrap.ensure_bootstrap_configuration():\n assert \"tcl\" not in spack.config.get(\"modules:default:enable\")\n assert \"lmod\" not in spack.config.get(\"modules:default:enable\")\n assert \"tcl\" in spack.config.get(\"modules:default:enable\")\n assert \"lmod\" in spack.config.get(\"modules:default:enable\")\n\n\n@pytest.mark.regression(\"25992\")\n@pytest.mark.requires_executables(\"gcc\")\ndef test_bootstrap_search_for_compilers_with_no_environment(no_packages_yaml, mock_packages):\n assert not spack.compilers.config.all_compilers(init_config=False)\n with spack.bootstrap.ensure_bootstrap_configuration():\n spack.bootstrap.clingo._add_compilers_if_missing()\n assert spack.compilers.config.all_compilers(init_config=False)\n assert not spack.compilers.config.all_compilers(init_config=False)\n\n\n@pytest.mark.regression(\"25992\")\n@pytest.mark.requires_executables(\"gcc\")\ndef test_bootstrap_search_for_compilers_with_environment_active(\n no_packages_yaml, active_mock_environment, mock_packages\n):\n assert not spack.compilers.config.all_compilers(init_config=False)\n with spack.bootstrap.ensure_bootstrap_configuration():\n spack.bootstrap.clingo._add_compilers_if_missing()\n assert spack.compilers.config.all_compilers(init_config=False)\n assert not spack.compilers.config.all_compilers(init_config=False)\n\n\n@pytest.mark.regression(\"26189\")\ndef test_config_yaml_is_preserved_during_bootstrap(mutable_config):\n expected_dir = \"/tmp/test\"\n spack.config.set(\"config:test_stage\", expected_dir, scope=\"command_line\")\n\n assert spack.config.get(\"config:test_stage\") == expected_dir\n with spack.bootstrap.ensure_bootstrap_configuration():\n assert spack.config.get(\"config:test_stage\") == expected_dir\n assert spack.config.get(\"config:test_stage\") == expected_dir\n\n\n@pytest.mark.regression(\"26548\")\ndef test_bootstrap_custom_store_in_environment(mutable_config, tmp_path: pathlib.Path):\n # Test that the custom store in an environment is taken into account\n # during bootstrapping\n spack_yaml = tmp_path / \"spack.yaml\"\n install_root = tmp_path / \"store\"\n spack_yaml.write_text(\n \"\"\"\nspack:\n specs:\n - libelf\n config:\n install_tree:\n root: {0}\n\"\"\".format(install_root)\n )\n with spack.environment.Environment(str(tmp_path)):\n assert spack.environment.active_environment()\n assert spack.config.get(\"config:install_tree:root\") == str(install_root)\n # Don't trigger evaluation here\n with spack.bootstrap.ensure_bootstrap_configuration():\n pass\n assert str(spack.store.STORE.root) == str(install_root)\n\n\ndef test_nested_use_of_context_manager(mutable_config):\n \"\"\"Test nested use of the context manager\"\"\"\n user_config = spack.config.CONFIG\n with spack.bootstrap.ensure_bootstrap_configuration():\n assert spack.config.CONFIG != user_config\n with spack.bootstrap.ensure_bootstrap_configuration():\n assert spack.config.CONFIG != user_config\n assert spack.config.CONFIG == user_config\n\n\n@pytest.mark.parametrize(\"expected_missing\", [False, True])\ndef test_status_function_find_files(\n mutable_config, mock_executable, tmp_path: pathlib.Path, monkeypatch, expected_missing\n):\n if not expected_missing:\n mock_executable(\"foo\", \"echo Hello WWorld!\")\n\n monkeypatch.setattr(\n spack.bootstrap.status,\n \"_optional_requirements\",\n lambda: [spack.bootstrap.status._required_system_executable(\"foo\", \"NOT FOUND\")],\n )\n monkeypatch.setenv(\"PATH\", str(tmp_path / \"bin\"))\n\n _, missing = spack.bootstrap.status_message(\"optional\")\n assert missing is expected_missing\n\n\n@pytest.mark.parametrize(\n \"gpg_in_path,gpg_in_store,expected_missing\",\n [\n (True, False, False), # gpg exists in PATH\n (False, True, False), # gpg exists in bootstrap store\n (False, False, True), # gpg is missing\n ],\n)\ndef test_gpg_status_check(\n mutable_config,\n mock_executable,\n tmp_path: pathlib.Path,\n monkeypatch,\n gpg_in_path,\n gpg_in_store,\n expected_missing,\n):\n \"\"\"Test that gpg/gpg2 status is detected whether it's in PATH or in the bootstrap store.\"\"\"\n # Set up mock PATH with or without gpg\n path_dir = tmp_path / \"bin\"\n path_dir.mkdir(exist_ok=True)\n monkeypatch.setenv(\"PATH\", str(path_dir))\n\n if gpg_in_path:\n mock_executable(\"gpg2\", \"echo GPG 2.3.4\")\n\n # Mock the bootstrap store function\n def mock_executables_in_store(exes, query_spec, query_info=None):\n if not gpg_in_store:\n return False\n\n # Simulate found gpg in bootstrap store\n if query_info is not None:\n query_info[\"spec\"] = \"gnupg@2.5.12\"\n query_info[\"command\"] = spack.util.executable.Executable(\"gpg\")\n return True\n\n monkeypatch.setattr(spack.bootstrap.status, \"_executables_in_store\", mock_executables_in_store)\n\n # Call only the buildcache requirements function directly to isolate the test\n requirements = spack.bootstrap.status._buildcache_requirements()\n\n # Find the gpg entry by examining the calls made to set up requirements\n # We know the first entry in requirements is the gpg entry because of how\n # _buildcache_requirements is structured:\n # Make sure we're not out of bounds\n assert len(requirements) >= 1, \"No gpg requirement found\"\n\n # Check that the gpg requirement matches our expectations\n gpg_req = requirements[0]\n assert gpg_req[0] is not expected_missing\n\n\n@pytest.mark.regression(\"31042\")\ndef test_source_is_disabled(mutable_config):\n # Get the configuration dictionary of the current bootstrapping source\n conf = next(iter(spack.bootstrap.core.bootstrapping_sources()))\n\n # The source is not explicitly enabled or disabled, so the following should return False\n assert not spack.bootstrap.core.source_is_enabled(conf)\n\n # Try to explicitly disable the source and verify that the behavior is the same as above\n spack.config.add(\"bootstrap:trusted:{0}:{1}\".format(conf[\"name\"], False))\n assert not spack.bootstrap.core.source_is_enabled(conf)\n\n\n@pytest.mark.regression(\"45247\")\ndef test_use_store_does_not_try_writing_outside_root(\n tmp_path: pathlib.Path, monkeypatch, mutable_config\n):\n \"\"\"Tests that when we use the 'use_store' context manager, there is no attempt at creating\n a Store outside the given root.\n \"\"\"\n initial_store = mutable_config.get(\"config:install_tree:root\")\n user_store = tmp_path / \"store\"\n\n fn = spack.store.Store.__init__\n\n def _checked_init(self, root, *args, **kwargs):\n fn(self, root, *args, **kwargs)\n assert self.root == str(user_store)\n\n monkeypatch.setattr(spack.store.Store, \"__init__\", _checked_init)\n\n spack.store.reinitialize()\n with spack.store.use_store(user_store):\n assert spack.config.CONFIG.get(\"config:install_tree:root\") == str(user_store)\n assert spack.config.CONFIG.get(\"config:install_tree:root\") == initial_store\n" }, { "path": "lib/spack/spack/test/build_distribution.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport pathlib\nimport shutil\n\nimport pytest\n\nimport spack.binary_distribution as bd\nimport spack.concretize\nimport spack.mirrors.mirror\nfrom spack.installer import PackageInstaller\n\npytestmark = pytest.mark.not_on_windows(\"does not run on windows\")\n\n\ndef test_build_tarball_overwrite(install_mockery, mock_fetch, monkeypatch, tmp_path: pathlib.Path):\n spec = spack.concretize.concretize_one(\"trivial-install-test-package\")\n PackageInstaller([spec.package], fake=True).install()\n\n specs = [spec]\n\n # populate cache, everything is new\n mirror = spack.mirrors.mirror.Mirror.from_local_path(str(tmp_path))\n with bd.make_uploader(mirror) as uploader:\n skipped = uploader.push_or_raise(specs)\n assert not skipped\n\n # should skip all\n with bd.make_uploader(mirror) as uploader:\n skipped = uploader.push_or_raise(specs)\n assert skipped == specs\n\n # with force=True none should be skipped\n with bd.make_uploader(mirror, force=True) as uploader:\n skipped = uploader.push_or_raise(specs)\n assert not skipped\n\n # Remove the tarball, which should cause push to push.\n shutil.rmtree(tmp_path / bd.buildcache_relative_blobs_path())\n\n with bd.make_uploader(mirror) as uploader:\n skipped = uploader.push_or_raise(specs)\n assert not skipped\n" }, { "path": "lib/spack/spack/test/build_environment.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport collections\nimport multiprocessing\nimport os\nimport pathlib\nimport posixpath\nimport sys\nfrom typing import Dict, Optional, Tuple\n\nimport pytest\n\nimport spack.vendor.archspec.cpu\n\nimport spack.build_environment\nimport spack.concretize\nimport spack.config\nimport spack.deptypes as dt\nimport spack.package_base\nimport spack.spec\nimport spack.util.environment\nimport spack.util.module_cmd\nimport spack.util.spack_yaml as syaml\nfrom spack.build_environment import UseMode, _static_to_shared_library, dso_suffix\nfrom spack.context import Context\nfrom spack.installer import PackageInstaller\nfrom spack.llnl.path import Path, convert_to_platform_path\nfrom spack.llnl.util.filesystem import HeaderList, LibraryList\nfrom spack.util.environment import EnvironmentModifications\nfrom spack.util.executable import Executable\n\n\ndef os_pathsep_join(path, *pths):\n out_pth = path\n for pth in pths:\n out_pth = os.pathsep.join([out_pth, pth])\n return out_pth\n\n\ndef prep_and_join(path, *pths):\n return os.path.sep + os.path.join(path, *pths)\n\n\n@pytest.fixture\ndef build_environment(monkeypatch, wrapper_dir, tmp_path: pathlib.Path):\n realcc = \"/bin/mycc\"\n prefix = str(tmp_path)\n\n monkeypatch.setenv(\"SPACK_CC\", realcc)\n monkeypatch.setenv(\"SPACK_CXX\", realcc)\n monkeypatch.setenv(\"SPACK_FC\", realcc)\n\n monkeypatch.setenv(\"SPACK_PREFIX\", prefix)\n monkeypatch.setenv(\"SPACK_COMPILER_WRAPPER_PATH\", \"test\")\n monkeypatch.setenv(\"SPACK_DEBUG_LOG_DIR\", \".\")\n monkeypatch.setenv(\"SPACK_DEBUG_LOG_ID\", \"foo-hashabc\")\n monkeypatch.setenv(\"SPACK_SHORT_SPEC\", \"foo@1.2 arch=linux-rhel6-x86_64 /hashabc\")\n\n monkeypatch.setenv(\"SPACK_CC_RPATH_ARG\", \"-Wl,-rpath,\")\n monkeypatch.setenv(\"SPACK_CXX_RPATH_ARG\", \"-Wl,-rpath,\")\n monkeypatch.setenv(\"SPACK_F77_RPATH_ARG\", \"-Wl,-rpath,\")\n monkeypatch.setenv(\"SPACK_FC_RPATH_ARG\", \"-Wl,-rpath,\")\n monkeypatch.setenv(\"SPACK_CC_LINKER_ARG\", \"-Wl,\")\n monkeypatch.setenv(\"SPACK_CXX_LINKER_ARG\", \"-Wl,\")\n monkeypatch.setenv(\"SPACK_FC_LINKER_ARG\", \"-Wl,\")\n monkeypatch.setenv(\"SPACK_F77_LINKER_ARG\", \"-Wl,\")\n monkeypatch.setenv(\"SPACK_DTAGS_TO_ADD\", \"--disable-new-dtags\")\n monkeypatch.setenv(\"SPACK_DTAGS_TO_STRIP\", \"--enable-new-dtags\")\n monkeypatch.setenv(\"SPACK_SYSTEM_DIRS\", \"/usr/include|/usr/lib\")\n monkeypatch.setenv(\"SPACK_MANAGED_DIRS\", f\"{prefix}/opt/spack\")\n monkeypatch.setenv(\"SPACK_TARGET_ARGS\", \"\")\n\n monkeypatch.delenv(\"SPACK_DEPENDENCIES\", raising=False)\n\n cc = Executable(str(wrapper_dir / \"cc\"))\n cxx = Executable(str(wrapper_dir / \"c++\"))\n fc = Executable(str(wrapper_dir / \"fc\"))\n\n return {\"cc\": cc, \"cxx\": cxx, \"fc\": fc}\n\n\n@pytest.fixture\ndef ensure_env_variables(mutable_config, mock_packages, monkeypatch, working_env):\n \"\"\"Returns a function that takes a dictionary and updates os.environ\n for the test lifetime accordingly. Plugs-in mock config and repo.\n \"\"\"\n\n def _ensure(env_mods):\n for name, value in env_mods.items():\n monkeypatch.setenv(name, value)\n\n return _ensure\n\n\n@pytest.fixture\ndef mock_module_cmd(monkeypatch):\n class Logger:\n def __init__(self, fn=None):\n self.fn = fn\n self.calls = []\n\n def __call__(self, *args, **kwargs):\n self.calls.append((args, kwargs))\n if self.fn:\n return self.fn(*args, **kwargs)\n\n mock_module_cmd = Logger()\n monkeypatch.setattr(spack.build_environment, \"module\", mock_module_cmd)\n monkeypatch.setattr(spack.build_environment, \"_on_cray\", lambda: (True, None))\n return mock_module_cmd\n\n\n@pytest.mark.not_on_windows(\"Static to Shared not supported on Win (yet)\")\ndef test_static_to_shared_library(build_environment):\n os.environ[\"SPACK_TEST_COMMAND\"] = \"dump-args\"\n\n expected = {\n \"linux\": (\n \"/bin/mycc -shared\"\n \" -Wl,--disable-new-dtags\"\n \" -Wl,-soname -Wl,{2} -Wl,--whole-archive {0}\"\n \" -Wl,--no-whole-archive -o {1}\"\n ),\n \"darwin\": (\n \"/bin/mycc -dynamiclib\"\n \" -Wl,--disable-new-dtags\"\n \" -install_name {1} -Wl,-force_load -Wl,{0} -o {1}\"\n ),\n }\n\n static_lib = \"/spack/libfoo.a\"\n\n for arch in (\"linux\", \"darwin\"):\n for shared_lib in (None, \"/spack/libbar.so\"):\n output = _static_to_shared_library(\n arch, build_environment[\"cc\"], static_lib, shared_lib, compiler_output=str\n ).strip()\n\n if not shared_lib:\n shared_lib = \"{0}.{1}\".format(os.path.splitext(static_lib)[0], dso_suffix)\n\n assert set(output.split()) == set(\n expected[arch].format(static_lib, shared_lib, os.path.basename(shared_lib)).split()\n )\n\n\n@pytest.mark.regression(\"8345\")\n@pytest.mark.usefixtures(\"mock_packages\")\n@pytest.mark.not_on_windows(\"Module files are not supported on Windows\")\ndef test_cc_not_changed_by_modules(monkeypatch, mutable_config, working_env, compiler_factory):\n \"\"\"Tests that external module files that are loaded cannot change the\n CC environment variable.\n \"\"\"\n gcc_entry = compiler_factory(spec=\"gcc@14.0.1 languages=c,c++\")\n gcc_entry[\"modules\"] = [\"some_module\"]\n mutable_config.set(\"packages\", {\"gcc\": {\"externals\": [gcc_entry]}})\n\n def _set_wrong_cc(x):\n os.environ[\"CC\"] = \"NOT_THIS_PLEASE\"\n os.environ[\"ANOTHER_VAR\"] = \"THIS_IS_SET\"\n\n monkeypatch.setattr(spack.util.module_cmd, \"load_module\", _set_wrong_cc)\n\n s = spack.concretize.concretize_one(\"cmake %gcc@14\")\n spack.build_environment.setup_package(s.package, dirty=False)\n\n assert os.environ[\"CC\"] != \"NOT_THIS_PLEASE\"\n assert os.environ[\"ANOTHER_VAR\"] == \"THIS_IS_SET\"\n\n\ndef test_setup_dependent_package_inherited_modules(\n working_env, mock_packages, install_mockery, mock_fetch\n):\n # This will raise on regression\n s = spack.concretize.concretize_one(\"cmake-client-inheritor\")\n PackageInstaller([s.package], fake=True).install()\n\n\n@pytest.mark.parametrize(\n \"initial,modifications,expected\",\n [\n # Set and unset variables\n (\n {\"SOME_VAR_STR\": \"\", \"SOME_VAR_NUM\": \"0\"},\n {\"set\": {\"SOME_VAR_STR\": \"SOME_STR\", \"SOME_VAR_NUM\": 1}},\n {\"SOME_VAR_STR\": \"SOME_STR\", \"SOME_VAR_NUM\": \"1\"},\n ),\n ({\"SOME_VAR_STR\": \"\"}, {\"unset\": [\"SOME_VAR_STR\"]}, {\"SOME_VAR_STR\": None}),\n (\n {}, # Set a variable that was not defined already\n {\"set\": {\"SOME_VAR_STR\": \"SOME_STR\"}},\n {\"SOME_VAR_STR\": \"SOME_STR\"},\n ),\n # Append and prepend to the same variable\n (\n {\"EMPTY_PATH_LIST\": prep_and_join(\"path\", \"middle\")},\n {\n \"prepend_path\": {\"EMPTY_PATH_LIST\": prep_and_join(\"path\", \"first\")},\n \"append_path\": {\"EMPTY_PATH_LIST\": prep_and_join(\"path\", \"last\")},\n },\n {\n \"EMPTY_PATH_LIST\": os_pathsep_join(\n prep_and_join(\"path\", \"first\"),\n prep_and_join(\"path\", \"middle\"),\n prep_and_join(\"path\", \"last\"),\n )\n },\n ),\n # Append and prepend from empty variables\n (\n {\"EMPTY_PATH_LIST\": \"\", \"SOME_VAR_STR\": \"\"},\n {\n \"prepend_path\": {\"EMPTY_PATH_LIST\": prep_and_join(\"path\", \"first\")},\n \"append_path\": {\"SOME_VAR_STR\": prep_and_join(\"path\", \"last\")},\n },\n {\n \"EMPTY_PATH_LIST\": prep_and_join(\"path\", \"first\"),\n \"SOME_VAR_STR\": prep_and_join(\"path\", \"last\"),\n },\n ),\n (\n {}, # Same as before but on variables that were not defined\n {\n \"prepend_path\": {\"EMPTY_PATH_LIST\": prep_and_join(\"path\", \"first\")},\n \"append_path\": {\"SOME_VAR_STR\": prep_and_join(\"path\", \"last\")},\n },\n {\n \"EMPTY_PATH_LIST\": prep_and_join(\"path\", \"first\"),\n \"SOME_VAR_STR\": prep_and_join(\"path\", \"last\"),\n },\n ),\n # Remove a path from a list\n (\n {\n \"EMPTY_PATH_LIST\": os_pathsep_join(\n prep_and_join(\"path\", \"first\"),\n prep_and_join(\"path\", \"middle\"),\n prep_and_join(\"path\", \"last\"),\n )\n },\n {\"remove_path\": {\"EMPTY_PATH_LIST\": prep_and_join(\"path\", \"middle\")}},\n {\n \"EMPTY_PATH_LIST\": os_pathsep_join(\n prep_and_join(\"path\", \"first\"), prep_and_join(\"path\", \"last\")\n )\n },\n ),\n (\n {\"EMPTY_PATH_LIST\": prep_and_join(\"only\", \"path\")},\n {\"remove_path\": {\"EMPTY_PATH_LIST\": prep_and_join(\"only\", \"path\")}},\n {\"EMPTY_PATH_LIST\": \"\"},\n ),\n ],\n)\ndef test_compiler_config_modifications(\n initial,\n modifications,\n expected,\n ensure_env_variables,\n compiler_factory,\n mutable_config,\n monkeypatch,\n):\n # Set the environment as per prerequisites\n ensure_env_variables(initial)\n\n gcc_entry = compiler_factory(spec=\"gcc@14.0.1 languages=c,c++\")\n gcc_entry[\"extra_attributes\"][\"environment\"] = modifications\n mutable_config.set(\"packages\", {\"gcc\": {\"externals\": [gcc_entry]}})\n\n def platform_pathsep(pathlist):\n if Path.platform_path == Path.windows:\n pathlist = pathlist.replace(\":\", \";\")\n\n return convert_to_platform_path(pathlist)\n\n pkg = spack.concretize.concretize_one(\"cmake %gcc@14\").package\n # Trigger the modifications\n spack.build_environment.setup_package(pkg, dirty=False)\n\n # Check they were applied\n for name, value in expected.items():\n if value is not None:\n value = platform_pathsep(value)\n assert os.environ[name] == value\n continue\n assert name not in os.environ\n\n\n@pytest.mark.not_on_windows(\"Module files are not supported on Windows\")\ndef test_load_external_modules_error(working_env, monkeypatch):\n \"\"\"Test that load_external_modules raises an exception when a module cannot be loaded\"\"\"\n\n # Create a mock spec object with the minimum attributes needed for the test\n class MockSpec:\n def __init__(self):\n self.external_modules = [\"non_existent_module\"]\n\n def __str__(self):\n return \"mock-external-spec\"\n\n mock_spec = MockSpec()\n\n # Create a simplified SetupContext-like class that only contains what we need\n class MockSetupContext:\n def __init__(self, spec):\n self.external = [(spec, None)]\n\n context = MockSetupContext(mock_spec)\n\n # Mock the load_module function to raise an exception\n def mock_load_module(module_name):\n # Simulate module load failure\n raise spack.util.module_cmd.ModuleLoadError(module_name)\n\n monkeypatch.setattr(spack.util.module_cmd, \"load_module\", mock_load_module)\n\n # Test that load_external_modules raises ModuleLoadError\n with pytest.raises(spack.util.module_cmd.ModuleLoadError):\n spack.build_environment.load_external_modules(context)\n\n\ndef test_external_config_env(mock_packages, mutable_config, working_env):\n cmake_config = {\n \"externals\": [\n {\n \"spec\": \"cmake@1.0\",\n \"prefix\": \"/fake/path\",\n \"extra_attributes\": {\"environment\": {\"set\": {\"TEST_ENV_VAR_SET\": \"yes it's set\"}}},\n }\n ]\n }\n spack.config.set(\"packages:cmake\", cmake_config)\n\n cmake_client = spack.concretize.concretize_one(\"cmake-client\")\n spack.build_environment.setup_package(cmake_client.package, False)\n\n assert os.environ[\"TEST_ENV_VAR_SET\"] == \"yes it's set\"\n\n\n@pytest.mark.regression(\"9107\")\n@pytest.mark.not_on_windows(\"Windows does not support module files\")\ndef test_spack_paths_before_module_paths(\n mutable_config, mock_packages, compiler_factory, monkeypatch, working_env, wrapper_dir\n):\n gcc_entry = compiler_factory(spec=\"gcc@14.0.1 languages=c,c++\")\n gcc_entry[\"modules\"] = [\"some_module\"]\n mutable_config.set(\"packages\", {\"gcc\": {\"externals\": [gcc_entry]}})\n\n module_path = os.path.join(\"path\", \"to\", \"module\")\n monkeypatch.setenv(\"SPACK_COMPILER_WRAPPER_PATH\", wrapper_dir)\n\n def _set_wrong_cc(x):\n os.environ[\"PATH\"] = module_path + os.pathsep + os.environ[\"PATH\"]\n\n monkeypatch.setattr(spack.util.module_cmd, \"load_module\", _set_wrong_cc)\n\n s = spack.concretize.concretize_one(\"cmake\")\n\n spack.build_environment.setup_package(s.package, dirty=False)\n\n paths = os.environ[\"PATH\"].split(os.pathsep)\n assert paths.index(str(wrapper_dir)) < paths.index(module_path)\n\n\ndef test_package_inheritance_module_setup(config, mock_packages, working_env):\n s = spack.concretize.concretize_one(\"multimodule-inheritance\")\n pkg = s.package\n\n spack.build_environment.setup_package(pkg, False)\n\n os.environ[\"TEST_MODULE_VAR\"] = \"failed\"\n\n assert pkg.use_module_variable() == \"test_module_variable\"\n assert os.environ[\"TEST_MODULE_VAR\"] == \"test_module_variable\"\n\n\ndef test_wrapper_variables(\n config, mock_packages, working_env, monkeypatch, installation_dir_with_headers\n):\n \"\"\"Check that build_environment supplies the needed library/include\n directories via the SPACK_LINK_DIRS and SPACK_INCLUDE_DIRS environment\n variables.\n \"\"\"\n\n # https://github.com/spack/spack/issues/13969\n cuda_headers = HeaderList(\n [\n \"prefix/include/cuda_runtime.h\",\n \"prefix/include/cuda/atomic\",\n \"prefix/include/cuda/std/detail/libcxx/include/ctype.h\",\n ]\n )\n cuda_include_dirs = cuda_headers.directories\n assert posixpath.join(\"prefix\", \"include\") in cuda_include_dirs\n assert (\n posixpath.join(\"prefix\", \"include\", \"cuda\", \"std\", \"detail\", \"libcxx\", \"include\")\n not in cuda_include_dirs\n )\n\n root = spack.concretize.concretize_one(\"dt-diamond\")\n\n for s in root.traverse():\n s.set_prefix(f\"/{s.name}-prefix/\")\n\n dep_pkg = root[\"dt-diamond-left\"].package\n dep_lib_paths = [\"/test/path/to/ex1.so\", \"/test/path/to/subdir/ex2.so\"]\n dep_lib_dirs = [\"/test/path/to\", \"/test/path/to/subdir\"]\n dep_libs = LibraryList(dep_lib_paths)\n\n dep2_pkg = root[\"dt-diamond-right\"].package\n dep2_pkg.spec.set_prefix(str(installation_dir_with_headers))\n\n setattr(dep_pkg, \"libs\", dep_libs)\n try:\n pkg = root.package\n env_mods = EnvironmentModifications()\n spack.build_environment.set_wrapper_variables(pkg, env_mods)\n\n env_mods.apply_modifications()\n\n def normpaths(paths):\n return list(os.path.normpath(p) for p in paths)\n\n link_dir_var = os.environ[\"SPACK_LINK_DIRS\"]\n assert normpaths(link_dir_var.split(\":\")) == normpaths(dep_lib_dirs)\n\n root_libdirs = [\"/dt-diamond-prefix/lib\", \"/dt-diamond-prefix/lib64\"]\n rpath_dir_var = os.environ[\"SPACK_RPATH_DIRS\"]\n # The 'lib' and 'lib64' subdirectories of the root package prefix\n # should always be rpathed and should be the first rpaths\n assert normpaths(rpath_dir_var.split(\":\")) == normpaths(root_libdirs + dep_lib_dirs)\n\n header_dir_var = os.environ[\"SPACK_INCLUDE_DIRS\"]\n\n # The default implementation looks for header files only\n # in /include and subdirectories\n prefix = str(installation_dir_with_headers)\n include_dirs = normpaths(header_dir_var.split(os.pathsep))\n\n assert os.path.join(prefix, \"include\") in include_dirs\n assert os.path.join(prefix, \"include\", \"boost\") not in include_dirs\n assert os.path.join(prefix, \"path\", \"to\") not in include_dirs\n assert os.path.join(prefix, \"path\", \"to\", \"subdir\") not in include_dirs\n\n finally:\n delattr(dep_pkg, \"libs\")\n\n\ndef test_external_prefixes_last(mutable_config, mock_packages, working_env, monkeypatch):\n # Sanity check: under normal circumstances paths associated with\n # dt-diamond-left would appear first. We'll mark it as external in\n # the test to check if the associated paths are placed last.\n assert \"dt-diamond-left\" < \"dt-diamond-right\"\n\n cfg_data = syaml.load_config(\n \"\"\"\\\ndt-diamond-left:\n externals:\n - spec: dt-diamond-left@1.0\n prefix: /fake/path1\n buildable: false\n\"\"\"\n )\n spack.config.set(\"packages\", cfg_data)\n top = spack.concretize.concretize_one(\"dt-diamond\")\n\n def _trust_me_its_a_dir(path):\n return True\n\n monkeypatch.setattr(os.path, \"isdir\", _trust_me_its_a_dir)\n\n env_mods = EnvironmentModifications()\n spack.build_environment.set_wrapper_variables(top.package, env_mods)\n\n env_mods.apply_modifications()\n link_dir_var = os.environ[\"SPACK_LINK_DIRS\"]\n link_dirs = link_dir_var.split(\":\")\n external_lib_paths = set(\n [os.path.normpath(\"/fake/path1/lib\"), os.path.normpath(\"/fake/path1/lib64\")]\n )\n # The external lib paths should be the last two entries of the list and\n # should not appear anywhere before the last two entries\n assert set(os.path.normpath(x) for x in link_dirs[-2:]) == external_lib_paths\n assert not (set(os.path.normpath(x) for x in link_dirs[:-2]) & external_lib_paths)\n\n\ndef test_parallel_false_is_not_propagating(default_mock_concretization):\n \"\"\"Test that parallel=False is not propagating to dependencies\"\"\"\n # a foobar=bar (parallel = False)\n # |\n # b (parallel =True)\n s = default_mock_concretization(\"pkg-a foobar=bar\")\n\n spack.build_environment.set_package_py_globals(s.package, context=Context.BUILD)\n assert s[\"pkg-a\"].package.module.make_jobs == 1\n\n spack.build_environment.set_package_py_globals(s[\"pkg-b\"].package, context=Context.BUILD)\n assert s[\"pkg-b\"].package.module.make_jobs == spack.config.determine_number_of_jobs(\n parallel=s[\"pkg-b\"].package.parallel\n )\n\n\n@pytest.mark.parametrize(\n \"config_setting,expected_flag\",\n [(\"runpath\", \"--enable-new-dtags\"), (\"rpath\", \"--disable-new-dtags\")],\n)\n@pytest.mark.skipif(sys.platform != \"linux\", reason=\"dtags make sense only on linux\")\ndef test_setting_dtags_based_on_config(\n config_setting, expected_flag, config, mock_packages, working_env\n):\n # Pick a random package to be able to set compiler's variables\n s = spack.concretize.concretize_one(\"cmake\")\n with spack.config.override(\"config:shared_linking\", {\"type\": config_setting, \"bind\": False}):\n env = spack.build_environment.setup_package(s.package, dirty=False)\n modifications = env.group_by_name()\n assert \"SPACK_DTAGS_TO_STRIP\" in modifications\n assert \"SPACK_DTAGS_TO_ADD\" in modifications\n assert len(modifications[\"SPACK_DTAGS_TO_ADD\"]) == 1\n assert len(modifications[\"SPACK_DTAGS_TO_STRIP\"]) == 1\n\n dtags_to_add = modifications[\"SPACK_DTAGS_TO_ADD\"][0]\n assert dtags_to_add.value == expected_flag\n\n\ndef test_module_globals_available_at_setup_dependent_time(\n monkeypatch, mutable_config, mock_packages, working_env\n):\n \"\"\"Spack built package externaltest depends on an external package\n externaltool. Externaltool's setup_dependent_package needs to be able to\n access globals on the dependent\"\"\"\n\n def setup_dependent_package(module, dependent_spec):\n # Make sure set_package_py_globals was already called on\n # dependents\n # ninja is always set by the setup context and is not None\n dependent_module = dependent_spec.package.module\n assert hasattr(dependent_module, \"ninja\")\n assert dependent_module.ninja is not None\n dependent_spec.package.test_attr = True\n\n externaltool = spack.concretize.concretize_one(\"externaltest\")\n monkeypatch.setattr(\n externaltool[\"externaltool\"].package, \"setup_dependent_package\", setup_dependent_package\n )\n spack.build_environment.setup_package(externaltool.package, False)\n assert externaltool.package.test_attr\n\n\ndef test_build_jobs_sequential_is_sequential():\n assert (\n spack.config.determine_number_of_jobs(\n parallel=False,\n max_cpus=8,\n config=spack.config.create_from(\n spack.config.InternalConfigScope(\"command_line\", {\"config\": {\"build_jobs\": 8}}),\n spack.config.InternalConfigScope(\"defaults\", {\"config\": {\"build_jobs\": 8}}),\n ),\n )\n == 1\n )\n\n\ndef test_build_jobs_command_line_overrides():\n assert (\n spack.config.determine_number_of_jobs(\n parallel=True,\n max_cpus=1,\n config=spack.config.create_from(\n spack.config.InternalConfigScope(\"command_line\", {\"config\": {\"build_jobs\": 10}}),\n spack.config.InternalConfigScope(\"defaults\", {\"config\": {\"build_jobs\": 1}}),\n ),\n )\n == 10\n )\n assert (\n spack.config.determine_number_of_jobs(\n parallel=True,\n max_cpus=100,\n config=spack.config.create_from(\n spack.config.InternalConfigScope(\"command_line\", {\"config\": {\"build_jobs\": 10}}),\n spack.config.InternalConfigScope(\"defaults\", {\"config\": {\"build_jobs\": 100}}),\n ),\n )\n == 10\n )\n\n\ndef test_build_jobs_defaults():\n assert (\n spack.config.determine_number_of_jobs(\n parallel=True,\n max_cpus=10,\n config=spack.config.create_from(\n spack.config.InternalConfigScope(\"defaults\", {\"config\": {\"build_jobs\": 1}})\n ),\n )\n == 1\n )\n assert (\n spack.config.determine_number_of_jobs(\n parallel=True,\n max_cpus=10,\n config=spack.config.create_from(\n spack.config.InternalConfigScope(\"defaults\", {\"config\": {\"build_jobs\": 100}})\n ),\n )\n == 10\n )\n\n\nclass TestModuleMonkeyPatcher:\n def test_getting_attributes(self, default_mock_concretization):\n s = default_mock_concretization(\"libelf\")\n module_wrapper = spack.build_environment.ModuleChangePropagator(s.package)\n assert module_wrapper.Libelf == s.package.module.Libelf\n\n def test_setting_attributes(self, default_mock_concretization):\n s = default_mock_concretization(\"libelf\")\n module = s.package.module\n module_wrapper = spack.build_environment.ModuleChangePropagator(s.package)\n\n # Setting an attribute has an immediate effect\n module_wrapper.SOME_ATTRIBUTE = 1\n assert module.SOME_ATTRIBUTE == 1\n\n # We can also propagate the settings to classes in the MRO\n module_wrapper.propagate_changes_to_mro()\n for cls in s.package.__class__.__mro__:\n current_module = cls.module\n if current_module == spack.package_base:\n break\n assert current_module.SOME_ATTRIBUTE == 1\n\n\ndef test_effective_deptype_build_environment(default_mock_concretization):\n s = default_mock_concretization(\"dttop\")\n\n # [ ] dttop@1.0 #\n # [b ] ^dtbuild1@1.0 # <- direct build dep\n # [b ] ^dtbuild2@1.0 # <- indirect build-only dep is dropped\n # [bl ] ^dtlink2@1.0 # <- linkable, and runtime dep of build dep\n # [ r ] ^dtrun2@1.0 # <- non-linkable, executable runtime dep of build dep\n # [bl ] ^dtlink1@1.0 # <- direct build dep\n # [bl ] ^dtlink3@1.0 # <- linkable, and runtime dep of build dep\n # [b ] ^dtbuild2@1.0 # <- indirect build-only dep is dropped\n # [bl ] ^dtlink4@1.0 # <- linkable, and runtime dep of build dep\n # [ r ] ^dtrun1@1.0 # <- run-only dep is pruned (should it be in PATH?)\n # [bl ] ^dtlink5@1.0 # <- children too\n # [ r ] ^dtrun3@1.0 # <- children too\n # [b ] ^dtbuild3@1.0 # <- children too\n\n expected_flags = {\n \"dttop\": UseMode.ROOT,\n \"dtbuild1\": UseMode.BUILDTIME_DIRECT,\n \"dtlink1\": UseMode.BUILDTIME_DIRECT | UseMode.BUILDTIME,\n \"dtlink3\": UseMode.BUILDTIME | UseMode.RUNTIME,\n \"dtlink4\": UseMode.BUILDTIME | UseMode.RUNTIME,\n \"dtrun2\": UseMode.RUNTIME | UseMode.RUNTIME_EXECUTABLE,\n \"dtlink2\": UseMode.RUNTIME,\n }\n\n for spec, effective_type in spack.build_environment.effective_deptypes(\n s, context=Context.BUILD\n ):\n assert effective_type & expected_flags.pop(spec.name) == effective_type\n assert not expected_flags, f\"Missing {expected_flags.keys()} from effective_deptypes\"\n\n\ndef test_effective_deptype_run_environment(default_mock_concretization):\n s = default_mock_concretization(\"dttop\")\n\n # [ ] dttop@1.0 #\n # [b ] ^dtbuild1@1.0 # <- direct build-only dep is pruned\n # [b ] ^dtbuild2@1.0 # <- children too\n # [bl ] ^dtlink2@1.0 # <- children too\n # [ r ] ^dtrun2@1.0 # <- children too\n # [bl ] ^dtlink1@1.0 # <- runtime, not executable\n # [bl ] ^dtlink3@1.0 # <- runtime, not executable\n # [b ] ^dtbuild2@1.0 # <- indirect build only dep is pruned\n # [bl ] ^dtlink4@1.0 # <- runtime, not executable\n # [ r ] ^dtrun1@1.0 # <- runtime and executable\n # [bl ] ^dtlink5@1.0 # <- runtime, not executable\n # [ r ] ^dtrun3@1.0 # <- runtime and executable\n # [b ] ^dtbuild3@1.0 # <- indirect build-only dep is pruned\n\n expected_flags = {\n \"dttop\": UseMode.ROOT,\n \"dtlink1\": UseMode.RUNTIME,\n \"dtlink3\": UseMode.BUILDTIME | UseMode.RUNTIME,\n \"dtlink4\": UseMode.BUILDTIME | UseMode.RUNTIME,\n \"dtrun1\": UseMode.RUNTIME | UseMode.RUNTIME_EXECUTABLE,\n \"dtlink5\": UseMode.RUNTIME,\n \"dtrun3\": UseMode.RUNTIME | UseMode.RUNTIME_EXECUTABLE,\n }\n\n for spec, effective_type in spack.build_environment.effective_deptypes(s, context=Context.RUN):\n assert effective_type & expected_flags.pop(spec.name) == effective_type\n assert not expected_flags, f\"Missing {expected_flags.keys()} from effective_deptypes\"\n\n\ndef test_monkey_patching_works_across_virtual(default_mock_concretization):\n \"\"\"Assert that a monkeypatched attribute is found regardless we access through the\n real name or the virtual name.\n \"\"\"\n s = default_mock_concretization(\"mpileaks ^mpich\")\n s[\"mpich\"].foo = \"foo\"\n assert s[\"mpich\"].foo == \"foo\"\n assert s[\"mpi\"].foo == \"foo\"\n\n\ndef test_clear_compiler_related_runtime_variables_of_build_deps(default_mock_concretization):\n \"\"\"Verify that Spack drops CC, CXX, FC and F77 from the dependencies related build environment\n variable changes if they are set in setup_run_environment. Spack manages those variables\n elsewhere.\"\"\"\n s = default_mock_concretization(\"build-env-compiler-var-a\")\n ctx = spack.build_environment.SetupContext(s, context=Context.BUILD)\n result = {}\n ctx.get_env_modifications().apply_modifications(result)\n assert \"CC\" not in result\n assert \"CXX\" not in result\n assert \"FC\" not in result\n assert \"F77\" not in result\n assert result[\"ANOTHER_VAR\"] == \"this-should-be-present\"\n\n\ndef test_rpath_with_duplicate_link_deps():\n \"\"\"If we have two instances of one package in the same link sub-dag, only the newest version is\n rpath'ed. This is for runtime support without splicing.\"\"\"\n runtime_1 = spack.spec.Spec(\"runtime@=1.0\")\n runtime_2 = spack.spec.Spec(\"runtime@=2.0\")\n child = spack.spec.Spec(\"child@=1.0\")\n root = spack.spec.Spec(\"root@=1.0\")\n\n root.add_dependency_edge(child, depflag=dt.LINK, virtuals=())\n root.add_dependency_edge(runtime_2, depflag=dt.LINK, virtuals=())\n child.add_dependency_edge(runtime_1, depflag=dt.LINK, virtuals=())\n\n rpath_deps = spack.build_environment._get_rpath_deps_from_spec(root, transitive_rpaths=True)\n assert child in rpath_deps\n assert runtime_2 in rpath_deps\n assert runtime_1 not in rpath_deps\n\n\n@pytest.mark.parametrize(\n \"compiler_spec,target_name,expected_flags\",\n [\n # Semver versions\n (\"gcc@4.7.2\", \"ivybridge\", \"-march=core-avx-i -mtune=core-avx-i\"),\n (\"clang@3.5\", \"x86_64\", \"-march=x86-64 -mtune=generic\"),\n (\"apple-clang@9.1.0\", \"x86_64\", \"-march=x86-64\"),\n (\"gcc@=9.2.0\", \"haswell\", \"-march=haswell -mtune=haswell\"),\n # Check that custom string versions are accepted\n (\"gcc@=9.2.0-foo\", \"icelake\", \"-march=icelake-client -mtune=icelake-client\"),\n # Check that the special case for Apple's clang is treated correctly\n # i.e. it won't try to detect the version again\n (\"apple-clang@=9.1.0\", \"x86_64\", \"-march=x86-64\"),\n ],\n)\n@pytest.mark.filterwarnings(\"ignore:microarchitecture specific\")\n@pytest.mark.not_on_windows(\"Windows doesn't support the compiler wrapper\")\ndef test_optimization_flags(compiler_spec, target_name, expected_flags, compiler_factory):\n target = spack.vendor.archspec.cpu.TARGETS[target_name]\n compiler = spack.spec.parse_with_version_concrete(compiler_spec)\n opt_flags = spack.build_environment.optimization_flags(compiler, target)\n assert opt_flags == expected_flags\n\n\n@pytest.mark.skipif(\n str(spack.vendor.archspec.cpu.host().family) != \"x86_64\",\n reason=\"tests check specific x86_64 uarch flags\",\n)\n@pytest.mark.not_on_windows(\"Windows doesn't support the compiler wrapper\")\ndef test_optimization_flags_are_using_node_target(default_mock_concretization, monkeypatch):\n \"\"\"Tests that we are using the target on the node to be compiled to retrieve the uarch\n specific flags, and not the target of the compiler.\n \"\"\"\n compiler_wrapper_pkg = default_mock_concretization(\"compiler-wrapper target=core2\").package\n mpileaks = default_mock_concretization(\"mpileaks target=x86_64\")\n\n env = EnvironmentModifications()\n compiler_wrapper_pkg.setup_dependent_build_environment(env, mpileaks)\n actions = env.group_by_name()[\"SPACK_TARGET_ARGS_CC\"]\n\n assert len(actions) == 1 and isinstance(actions[0], spack.util.environment.SetEnv)\n assert actions[0].value == \"-march=x86-64 -mtune=generic\"\n\n\n@pytest.mark.regression(\"49827\")\n@pytest.mark.parametrize(\n \"gcc_config,expected_rpaths\",\n [\n (\n \"\"\"\\\ngcc:\n externals:\n - spec: gcc@14.2.0 languages:=c,c++,fortran\n prefix: /fake/path1\n extra_attributes:\n compilers:\n c: /fake/path1\n cxx: /fake/path1\n fortran: /fake/path1\n extra_rpaths:\n - /extra/rpaths1\n - /extra/rpaths2\n\"\"\",\n \"/extra/rpaths1:/extra/rpaths2\",\n ),\n (\n \"\"\"\\\ngcc:\n externals:\n - spec: gcc@14.2.0 languages=c,c++,fortran\n prefix: /fake/path1\n extra_attributes:\n compilers:\n c: /fake/path1\n cxx: /fake/path1\n fortran: /fake/path1\n\"\"\",\n None,\n ),\n ],\n)\n@pytest.mark.not_on_windows(\"Windows doesn't use the compiler-wrapper\")\ndef test_extra_rpaths_is_set(\n working_env, mutable_config, mock_packages, gcc_config, expected_rpaths\n):\n \"\"\"Tests that using a compiler with an 'extra_rpaths' section will set the corresponding\n SPACK_COMPILER_EXTRA_RPATHS variable for the wrapper.\n \"\"\"\n cfg_data = syaml.load_config(gcc_config)\n spack.config.set(\"packages\", cfg_data)\n mpich = spack.concretize.concretize_one(\"mpich %gcc@14\")\n spack.build_environment.setup_package(mpich.package, dirty=False)\n\n if expected_rpaths is not None:\n assert os.environ[\"SPACK_COMPILER_EXTRA_RPATHS\"] == expected_rpaths\n else:\n assert \"SPACK_COMPILER_EXTRA_RPATHS\" not in os.environ\n\n\nclass _TestProcess:\n calls: Dict[str, int] = collections.defaultdict(int)\n terminated = False\n runtime = 0\n\n def __init__(self, *, target, args, pkg, read_pipe, timeout):\n self.alive = None\n self.exitcode = 0\n self._reset()\n self.read_pipe = read_pipe\n self.timeout = timeout\n\n def start(self):\n self.calls[\"start\"] += 1\n self.alive = True\n\n def poll(self):\n return True\n\n def complete(self):\n return None\n\n def is_alive(self):\n self.calls[\"is_alive\"] += 1\n return self.alive\n\n def join(self, timeout: Optional[int] = None):\n self.calls[\"join\"] += 1\n if timeout is not None and timeout > self.runtime:\n self.alive = False\n\n def terminate(self):\n self.calls[\"terminate\"] += 1\n self._set_terminated()\n self.alive = False\n # Do not set exit code. A non-zero exit code will trigger an error\n # instead of gracefully inspecting values for test\n\n @classmethod\n def _set_terminated(cls):\n cls.terminated = True\n\n @classmethod\n def _reset(cls):\n cls.calls.clear()\n cls.terminated = False\n\n\nclass _TestPipe:\n def close(self):\n pass\n\n def recv(self):\n if _TestProcess.terminated is True:\n return 1\n return 0\n\n\ndef _pipe_fn(*, duplex: bool = False) -> Tuple[_TestPipe, _TestPipe]:\n return _TestPipe(), _TestPipe()\n\n\n@pytest.fixture()\ndef mock_build_process(monkeypatch):\n monkeypatch.setattr(spack.build_environment, \"BuildProcess\", _TestProcess)\n monkeypatch.setattr(multiprocessing, \"Pipe\", _pipe_fn)\n\n def _factory(*, runtime: int):\n _TestProcess.runtime = runtime\n\n return _factory\n\n\n@pytest.mark.parametrize(\n \"runtime,timeout,expected_calls\",\n [\n # execution time < timeout\n (2, 5, {\"start\": 1, \"join\": 1, \"is_alive\": 1}),\n # execution time > timeout\n (5, 2, {\"start\": 1, \"join\": 1, \"is_alive\": 1, \"terminate\": 1}),\n ],\n)\ndef test_build_process_timeout(mock_build_process, runtime, timeout, expected_calls):\n \"\"\"Tests that we make the correct function calls in different timeout scenarios.\"\"\"\n mock_build_process(runtime=runtime)\n process = spack.build_environment.start_build_process(\n pkg=None, function=None, kwargs={}, timeout=timeout\n )\n _ = spack.build_environment.complete_build_process(process)\n\n assert _TestProcess.calls == expected_calls\n" }, { "path": "lib/spack/spack/test/build_system_guess.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport pathlib\n\nimport pytest\n\nimport spack.cmd.create\nimport spack.stage\nimport spack.util.executable\nimport spack.util.url as url_util\n\n\n@pytest.fixture(\n scope=\"function\",\n params=[\n (\"configure\", \"autotools\"),\n (\"CMakeLists.txt\", \"cmake\"),\n (\"project.pro\", \"qmake\"),\n (\"pom.xml\", \"maven\"),\n (\"SConstruct\", \"scons\"),\n (\"waf\", \"waf\"),\n (\"argbah.rockspec\", \"lua\"),\n (\"setup.py\", \"python\"),\n (\"NAMESPACE\", \"r\"),\n (\"WORKSPACE\", \"bazel\"),\n (\"Makefile.PL\", \"perlmake\"),\n (\"Build.PL\", \"perlbuild\"),\n (\"foo.gemspec\", \"ruby\"),\n (\"Rakefile\", \"ruby\"),\n (\"setup.rb\", \"ruby\"),\n (\"GNUmakefile\", \"makefile\"),\n (\"makefile\", \"makefile\"),\n (\"Makefile\", \"makefile\"),\n (\"meson.build\", \"meson\"),\n (\"configure.py\", \"sip\"),\n (\"foobar\", \"generic\"),\n ],\n)\ndef url_and_build_system(request, tmp_path: pathlib.Path):\n \"\"\"Sets up the resources to be pulled by the stage with\n the appropriate file name and returns their url along with\n the correct build-system guess\n \"\"\"\n tar = spack.util.executable.which(\"tar\", required=True)\n import spack.llnl.util.filesystem as fs\n\n with fs.working_dir(str(tmp_path)):\n filename, system = request.param\n archive_dir = tmp_path / \"archive\"\n archive_dir.mkdir()\n (archive_dir / filename).touch()\n tar(\"czf\", \"archive.tar.gz\", \"archive\")\n url = url_util.path_to_file_url(str(tmp_path / \"archive.tar.gz\"))\n yield url, system\n\n\ndef test_build_systems(url_and_build_system):\n url, build_system = url_and_build_system\n with spack.stage.Stage(url) as stage:\n stage.fetch()\n guesser = spack.cmd.create.BuildSystemAndLanguageGuesser()\n guesser(stage.archive_file, url)\n assert build_system == guesser.build_system\n" }, { "path": "lib/spack/spack/test/builder.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport os\nimport pathlib\n\nimport pytest\n\nimport spack.builder\nimport spack.concretize\nimport spack.paths\nimport spack.repo\nfrom spack.llnl.util.filesystem import touch\n\n\n@pytest.fixture()\ndef builder_test_repository(config):\n builder_test_path = os.path.join(spack.paths.test_repos_path, \"spack_repo\", \"builder_test\")\n with spack.repo.use_repositories(builder_test_path) as mock_repo:\n yield mock_repo\n\n\n@pytest.mark.parametrize(\n \"spec_str,expected_values\",\n [\n (\n \"callbacks@2.0\",\n [\n (\"BEFORE_INSTALL_1_CALLED\", \"1\"),\n (\"BEFORE_INSTALL_2_CALLED\", \"1\"),\n (\"CALLBACKS_INSTALL_CALLED\", \"1\"),\n (\"AFTER_INSTALL_1_CALLED\", \"1\"),\n (\"TEST_VALUE\", \"3\"),\n (\"INSTALL_VALUE\", \"CALLBACKS\"),\n ],\n ),\n # The last callback is conditional on \"@1.0\", check it's being executed\n (\n \"callbacks@1.0\",\n [\n (\"BEFORE_INSTALL_1_CALLED\", \"1\"),\n (\"BEFORE_INSTALL_2_CALLED\", \"1\"),\n (\"CALLBACKS_INSTALL_CALLED\", \"1\"),\n (\"AFTER_INSTALL_1_CALLED\", \"1\"),\n (\"AFTER_INSTALL_2_CALLED\", \"1\"),\n (\"TEST_VALUE\", \"4\"),\n (\"INSTALL_VALUE\", \"CALLBACKS\"),\n ],\n ),\n # The package below adds to \"callbacks\" using inheritance, test that using super()\n # works with builder hierarchies\n (\n \"inheritance@1.0\",\n [\n (\"DERIVED_BEFORE_INSTALL_CALLED\", \"1\"),\n (\"BEFORE_INSTALL_1_CALLED\", \"1\"),\n (\"BEFORE_INSTALL_2_CALLED\", \"1\"),\n (\"CALLBACKS_INSTALL_CALLED\", \"1\"),\n (\"INHERITANCE_INSTALL_CALLED\", \"1\"),\n (\"AFTER_INSTALL_1_CALLED\", \"1\"),\n (\"AFTER_INSTALL_2_CALLED\", \"1\"),\n (\"TEST_VALUE\", \"4\"),\n (\"INSTALL_VALUE\", \"INHERITANCE\"),\n ],\n ),\n # Generate custom phases using a GenericBuilder\n (\n \"custom-phases\",\n [(\"CONFIGURE_CALLED\", \"1\"), (\"INSTALL_CALLED\", \"1\"), (\"LAST_PHASE\", \"INSTALL\")],\n ),\n # Old-style package, with phase defined in base builder\n (\"old-style-autotools@1.0\", [(\"AFTER_AUTORECONF_1_CALLED\", \"1\")]),\n (\"old-style-autotools@2.0\", [(\"AFTER_AUTORECONF_2_CALLED\", \"1\")]),\n (\"old-style-custom-phases\", [(\"AFTER_CONFIGURE_CALLED\", \"1\"), (\"TEST_VALUE\", \"0\")]),\n ],\n)\n@pytest.mark.usefixtures(\"builder_test_repository\", \"config\")\n@pytest.mark.disable_clean_stage_check\ndef test_callbacks_and_installation_procedure(spec_str, expected_values, working_env):\n \"\"\"Test the correct execution of callbacks and installation procedures for packages.\"\"\"\n s = spack.concretize.concretize_one(spec_str)\n builder = spack.builder.create(s.package)\n for phase_fn in builder:\n phase_fn.execute()\n\n # Check calls have produced the expected side effects\n for var_name, expected in expected_values:\n assert os.environ[var_name] == expected, os.environ\n\n\n@pytest.mark.usefixtures(\"builder_test_repository\", \"config\")\n@pytest.mark.parametrize(\n \"spec_str,method_name,expected\",\n [\n # Call a function defined on the package, which calls the same function defined\n # on the super(builder)\n (\"old-style-autotools\", \"configure_args\", [\"--with-foo\"]),\n # Call a function defined on the package, which calls the same function defined on the\n # super(pkg), which calls the same function defined in the super(builder)\n (\"old-style-derived\", \"configure_args\", [\"--with-bar\", \"--with-foo\"]),\n ],\n)\ndef test_old_style_compatibility_with_super(spec_str, method_name, expected):\n s = spack.concretize.concretize_one(spec_str)\n builder = spack.builder.create(s.package)\n value = getattr(builder, method_name)()\n assert value == expected\n\n\n@pytest.mark.not_on_windows(\"log_ouput cannot currently be used outside of subprocess on Windows\")\n@pytest.mark.regression(\"33928\")\n@pytest.mark.usefixtures(\"builder_test_repository\", \"config\", \"working_env\")\n@pytest.mark.disable_clean_stage_check\ndef test_build_time_tests_are_executed_from_default_builder():\n s = spack.concretize.concretize_one(\"old-style-autotools\")\n builder = spack.builder.create(s.package)\n builder.pkg.run_tests = True\n for phase_fn in builder:\n phase_fn.execute()\n\n assert os.environ.get(\"CHECK_CALLED\") == \"1\", \"Build time tests not executed\"\n assert os.environ.get(\"INSTALLCHECK_CALLED\") == \"1\", \"Install time tests not executed\"\n\n\n@pytest.mark.regression(\"34518\")\n@pytest.mark.usefixtures(\"builder_test_repository\", \"config\", \"working_env\")\ndef test_monkey_patching_wrapped_pkg():\n \"\"\"Confirm 'run_tests' is accessible through wrappers.\"\"\"\n s = spack.concretize.concretize_one(\"old-style-autotools\")\n builder = spack.builder.create(s.package)\n assert s.package.run_tests is False\n assert builder.pkg.run_tests is False\n assert builder.pkg_with_dispatcher.run_tests is False\n\n s.package.run_tests = True\n assert builder.pkg.run_tests is True\n assert builder.pkg_with_dispatcher.run_tests is True\n\n\n@pytest.mark.regression(\"34440\")\n@pytest.mark.usefixtures(\"builder_test_repository\", \"config\", \"working_env\")\ndef test_monkey_patching_test_log_file():\n \"\"\"Confirm 'test_log_file' is accessible through wrappers.\"\"\"\n s = spack.concretize.concretize_one(\"old-style-autotools\")\n builder = spack.builder.create(s.package)\n\n s.package.tester.test_log_file = \"/some/file\"\n assert builder.pkg.tester.test_log_file == \"/some/file\"\n assert builder.pkg_with_dispatcher.tester.test_log_file == \"/some/file\"\n\n\n# Windows context manager's __exit__ fails with ValueError (\"I/O operation\n# on closed file\").\n@pytest.mark.not_on_windows(\"Does not run on windows\")\ndef test_install_time_test_callback(tmp_path: pathlib.Path, config, mock_packages, mock_stage):\n \"\"\"Confirm able to run stand-alone test as a post-install callback.\"\"\"\n s = spack.concretize.concretize_one(\"py-test-callback\")\n builder = spack.builder.create(s.package)\n builder.pkg.run_tests = True\n s.package.tester.test_log_file = str(tmp_path / \"install_test.log\")\n touch(s.package.tester.test_log_file)\n\n for phase_fn in builder:\n phase_fn.execute()\n\n with open(s.package.tester.test_log_file, \"r\", encoding=\"utf-8\") as f:\n results = f.read().replace(\"\\n\", \" \")\n assert \"PyTestCallback test\" in results\n\n\n@pytest.mark.regression(\"43097\")\n@pytest.mark.usefixtures(\"builder_test_repository\", \"config\")\ndef test_mixins_with_builders(working_env):\n \"\"\"Tests that run_after and run_before callbacks are accumulated correctly,\n when mixins are used with builders.\n \"\"\"\n s = spack.concretize.concretize_one(\"builder-and-mixins\")\n builder = spack.builder.create(s.package)\n\n # Check that callbacks added by the mixin are in the list\n assert any(fn.__name__ == \"before_install\" for _, fn in builder._run_before_callbacks)\n assert any(fn.__name__ == \"after_install\" for _, fn in builder._run_after_callbacks)\n\n # Check that callback from the GenericBuilder are in the list too\n assert any(fn.__name__ == \"sanity_check_prefix\" for _, fn in builder._run_after_callbacks)\n\n\ndef test_reading_api_v20_attributes():\n \"\"\"Tests that we can read attributes from API v2.0 builders.\"\"\"\n\n class TestBuilder(spack.builder.Builder):\n legacy_methods = (\"configure\", \"install\")\n legacy_attributes = (\"foo\", \"bar\")\n legacy_long_methods = (\"baz\", \"fee\")\n\n methods = spack.builder.package_methods(TestBuilder)\n assert methods == (\"configure\", \"install\")\n attributes = spack.builder.package_attributes(TestBuilder)\n assert attributes == (\"foo\", \"bar\")\n long_methods = spack.builder.package_long_methods(TestBuilder)\n assert long_methods == (\"baz\", \"fee\")\n\n\ndef test_reading_api_v22_attributes():\n \"\"\"Tests that we can read attributes from API v2.2 builders.\"\"\"\n\n class TestBuilder(spack.builder.Builder):\n package_methods = (\"configure\", \"install\")\n package_attributes = (\"foo\", \"bar\")\n package_long_methods = (\"baz\", \"fee\")\n\n methods = spack.builder.package_methods(TestBuilder)\n assert methods == (\"configure\", \"install\")\n attributes = spack.builder.package_attributes(TestBuilder)\n assert attributes == (\"foo\", \"bar\")\n long_methods = spack.builder.package_long_methods(TestBuilder)\n assert long_methods == (\"baz\", \"fee\")\n\n\n@pytest.mark.regression(\"51917\")\n@pytest.mark.usefixtures(\"builder_test_repository\", \"config\")\ndef test_builder_when_inheriting_just_package(working_env):\n \"\"\"Tests that if we inherit a package from another package that has a builder defined,\n but we don't need to modify the builder ourselves, we'll get the builder of the base\n package class.\n \"\"\"\n base_spec = spack.concretize.concretize_one(\"callbacks\")\n derived_spec = spack.concretize.concretize_one(\"inheritance-only-package\")\n\n base_builder = spack.builder.create(base_spec.package)\n derived_builder = spack.builder.create(derived_spec.package)\n\n # The derived class doesn't redefine a builder, so we should\n # get the builder of the base class.\n assert type(base_builder) is type(derived_builder)\n" }, { "path": "lib/spack/spack/test/buildrequest.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport pytest\n\nimport spack.concretize\nimport spack.deptypes as dt\nimport spack.installer as inst\nimport spack.repo\nimport spack.spec\n\n\ndef test_build_request_errors(install_mockery):\n with pytest.raises(ValueError, match=\"must be a package\"):\n inst.BuildRequest(\"abc\", {})\n\n spec = spack.spec.Spec(\"trivial-install-test-package\")\n pkg_cls = spack.repo.PATH.get_pkg_class(spec.name)\n with pytest.raises(ValueError, match=\"must have a concrete spec\"):\n inst.BuildRequest(pkg_cls(spec), {})\n\n\ndef test_build_request_basics(install_mockery):\n spec = spack.concretize.concretize_one(\"dependent-install\")\n assert spec.concrete\n\n # Ensure key properties match expectations\n request = inst.BuildRequest(spec.package, {})\n assert not request.pkg.stop_before_phase\n assert not request.pkg.last_phase\n assert request.spec == spec.package.spec\n\n # Ensure key default install arguments are set\n assert \"install_package\" in request.install_args\n assert \"install_deps\" in request.install_args\n\n\ndef test_build_request_strings(install_mockery):\n \"\"\"Tests of BuildRequest repr and str for coverage purposes.\"\"\"\n # Using a package with one dependency\n spec = spack.concretize.concretize_one(\"dependent-install\")\n assert spec.concrete\n\n # Ensure key properties match expectations\n request = inst.BuildRequest(spec.package, {})\n\n # Cover __repr__\n irep = request.__repr__()\n assert irep.startswith(request.__class__.__name__)\n\n # Cover __str__\n istr = str(request)\n assert \"package=dependent-install\" in istr\n assert \"install_args=\" in istr\n\n\n@pytest.mark.parametrize(\n \"root_policy,dependencies_policy,package_deptypes,dependencies_deptypes\",\n [\n (\"auto\", \"auto\", dt.BUILD | dt.LINK | dt.RUN, dt.BUILD | dt.LINK | dt.RUN),\n (\"cache_only\", \"auto\", dt.LINK | dt.RUN, dt.BUILD | dt.LINK | dt.RUN),\n (\"auto\", \"cache_only\", dt.BUILD | dt.LINK | dt.RUN, dt.LINK | dt.RUN),\n (\"cache_only\", \"cache_only\", dt.LINK | dt.RUN, dt.LINK | dt.RUN),\n ],\n)\ndef test_build_request_deptypes(\n install_mockery, root_policy, dependencies_policy, package_deptypes, dependencies_deptypes\n):\n s = spack.concretize.concretize_one(\"dependent-install\")\n\n build_request = inst.BuildRequest(\n s.package, {\"root_policy\": root_policy, \"dependencies_policy\": dependencies_policy}\n )\n\n actual_package_deptypes = build_request.get_depflags(s.package)\n actual_dependency_deptypes = build_request.get_depflags(s[\"dependency-install\"].package)\n\n assert actual_package_deptypes == package_deptypes\n assert actual_dependency_deptypes == dependencies_deptypes\n" }, { "path": "lib/spack/spack/test/buildtask.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport pytest\n\nimport spack.concretize\nimport spack.error\nimport spack.installer as inst\nimport spack.repo\nimport spack.spec\n\n\ndef test_build_task_errors(install_mockery):\n \"\"\"Check expected errors when instantiating a BuildTask.\"\"\"\n spec = spack.spec.Spec(\"trivial-install-test-package\")\n pkg_cls = spack.repo.PATH.get_pkg_class(spec.name)\n\n # The value of the request argument is expected to not be checked.\n for pkg in [None, \"abc\"]:\n with pytest.raises(TypeError, match=\"must be a package\"):\n inst.BuildTask(pkg, None)\n\n with pytest.raises(ValueError, match=\"must have a concrete spec\"):\n inst.BuildTask(pkg_cls(spec), None)\n\n # Using a concretized package now means the request argument is checked.\n spec = spack.concretize.concretize_one(spec)\n assert spec.concrete\n\n with pytest.raises(TypeError, match=\"is not a valid build request\"):\n inst.BuildTask(spec.package, None)\n\n # Using a valid package and spec, the next check is the status argument.\n request = inst.BuildRequest(spec.package, {})\n\n with pytest.raises(TypeError, match=\"is not a valid build status\"):\n inst.BuildTask(spec.package, request, status=\"queued\")\n\n # Now we can check that build tasks cannot be create when the status\n # indicates the task is/should've been removed.\n with pytest.raises(spack.error.InstallError, match=\"Cannot create a task\"):\n inst.BuildTask(spec.package, request, status=inst.BuildStatus.REMOVED)\n\n # Also make sure to not accept an incompatible installed argument value.\n with pytest.raises(TypeError, match=\"'installed' be a 'set', not 'str'\"):\n inst.BuildTask(spec.package, request, installed=\"mpileaks\")\n\n\ndef test_build_task_basics(install_mockery):\n spec = spack.concretize.concretize_one(\"dependent-install\")\n assert spec.concrete\n\n # Ensure key properties match expectations\n request = inst.BuildRequest(spec.package, {})\n task = inst.BuildTask(spec.package, request=request, status=inst.BuildStatus.QUEUED)\n assert not task.explicit\n assert task.priority == len(task.uninstalled_deps)\n assert task.key == (task.priority, task.sequence)\n\n # Ensure flagging installed works as expected\n assert len(task.uninstalled_deps) > 0\n assert task.dependencies == task.uninstalled_deps\n task.flag_installed(task.dependencies)\n assert len(task.uninstalled_deps) == 0\n assert task.priority == 0\n\n\ndef test_build_task_strings(install_mockery):\n \"\"\"Tests of build_task repr and str for coverage purposes.\"\"\"\n # Using a package with one dependency\n spec = spack.concretize.concretize_one(\"dependent-install\")\n assert spec.concrete\n\n # Ensure key properties match expectations\n request = inst.BuildRequest(spec.package, {})\n task = inst.BuildTask(spec.package, request=request, status=inst.BuildStatus.QUEUED)\n\n # Cover __repr__\n irep = task.__repr__()\n assert irep.startswith(task.__class__.__name__)\n assert \"BuildStatus.QUEUED\" in irep\n assert \"sequence=\" in irep\n\n # Cover __str__\n istr = str(task)\n assert \"status=queued\" in istr # == BuildStatus.QUEUED\n assert \"#dependencies=1\" in istr\n assert \"priority=\" in istr\n" }, { "path": "lib/spack/spack/test/cache_fetch.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport pathlib\n\nimport pytest\n\nimport spack.config\nimport spack.util.url as url_util\nfrom spack.fetch_strategy import CacheURLFetchStrategy, NoCacheError\nfrom spack.llnl.util.filesystem import mkdirp\nfrom spack.stage import Stage\n\n\n@pytest.mark.parametrize(\"_fetch_method\", [\"curl\", \"urllib\"])\ndef test_fetch_missing_cache(tmp_path: pathlib.Path, _fetch_method):\n \"\"\"Ensure raise a missing cache file.\"\"\"\n testpath = str(tmp_path)\n non_existing = os.path.join(testpath, \"non-existing\")\n with spack.config.override(\"config:url_fetch_method\", _fetch_method):\n url = url_util.path_to_file_url(non_existing)\n fetcher = CacheURLFetchStrategy(url=url)\n with Stage(fetcher, path=testpath):\n with pytest.raises(NoCacheError, match=r\"No cache\"):\n fetcher.fetch()\n\n\n@pytest.mark.parametrize(\"_fetch_method\", [\"curl\", \"urllib\"])\ndef test_fetch(tmp_path: pathlib.Path, _fetch_method):\n \"\"\"Ensure a fetch after expanding is effectively a no-op.\"\"\"\n cache_dir = tmp_path / \"cache\"\n stage_dir = tmp_path / \"stage\"\n cache_dir.mkdir()\n stage_dir.mkdir()\n cache = cache_dir / \"cache.tar.gz\"\n cache.touch()\n url = url_util.path_to_file_url(str(cache))\n with spack.config.override(\"config:url_fetch_method\", _fetch_method):\n fetcher = CacheURLFetchStrategy(url=url)\n with Stage(fetcher, path=str(stage_dir)) as stage:\n source_path = stage.source_path\n mkdirp(source_path)\n fetcher.fetch()\n" }, { "path": "lib/spack/spack/test/cc.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"\nThis test checks that the Spack cc compiler wrapper is parsing\narguments correctly.\n\"\"\"\n\nimport os\n\nimport pytest\n\nimport spack.build_environment\nimport spack.config\nfrom spack.util.environment import SYSTEM_DIR_CASE_ENTRY, set_env\nfrom spack.util.executable import Executable, ProcessError\n\n#\n# Complicated compiler test command\n#\ntest_args = [\n \"-I/test/include\",\n \"-L/test/lib\",\n \"-L/with space/lib\",\n \"-I/other/include\",\n \"arg1\",\n \"-Wl,--start-group\",\n \"arg2\",\n \"-Wl,-rpath,/first/rpath\",\n \"arg3\",\n \"-Wl,-rpath\",\n \"-Wl,/second/rpath\",\n \"-llib1\",\n \"-llib2\",\n \"arg4\",\n \"-Wl,--end-group\",\n \"-Xlinker\",\n \"-rpath\",\n \"-Xlinker\",\n \"/third/rpath\",\n \"-Xlinker\",\n \"-rpath\",\n \"-Xlinker\",\n \"/fourth/rpath\",\n \"-Wl,--rpath,/fifth/rpath\",\n \"-Wl,--rpath\",\n \"-Wl,/sixth/rpath\",\n \"-llib3\",\n \"-llib4\",\n \"arg5\",\n \"arg6\",\n \"-DGCC_ARG_WITH_PERENS=(A B C)\",\n '\"-DDOUBLE_QUOTED_ARG\"',\n \"'-DSINGLE_QUOTED_ARG'\",\n]\n\n#\n# Pieces of the test command above, as they should be parsed out.\n#\n# `_wl_rpaths` are for the compiler (with -Wl,), and `_rpaths` are raw\n# -rpath arguments for the linker.\n#\ntest_include_paths = [\"-I/test/include\", \"-I/other/include\"]\n\ntest_library_paths = [\"-L/test/lib\", \"-L/with space/lib\"]\n\ntest_wl_rpaths = [\n \"-Wl,-rpath,/first/rpath\",\n \"-Wl,-rpath,/second/rpath\",\n \"-Wl,-rpath,/third/rpath\",\n \"-Wl,-rpath,/fourth/rpath\",\n \"-Wl,-rpath,/fifth/rpath\",\n \"-Wl,-rpath,/sixth/rpath\",\n]\n\ntest_rpaths = [\n \"-rpath\",\n \"/first/rpath\",\n \"-rpath\",\n \"/second/rpath\",\n \"-rpath\",\n \"/third/rpath\",\n \"-rpath\",\n \"/fourth/rpath\",\n \"-rpath\",\n \"/fifth/rpath\",\n \"-rpath\",\n \"/sixth/rpath\",\n]\n\ntest_args_without_paths = [\n \"arg1\",\n \"-Wl,--start-group\",\n \"arg2\",\n \"arg3\",\n \"-llib1\",\n \"-llib2\",\n \"arg4\",\n \"-Wl,--end-group\",\n \"-llib3\",\n \"-llib4\",\n \"arg5\",\n \"arg6\",\n \"-DGCC_ARG_WITH_PERENS=(A B C)\",\n '\"-DDOUBLE_QUOTED_ARG\"',\n \"'-DSINGLE_QUOTED_ARG'\",\n]\n\n#: The prefix of the package being mock installed\npkg_prefix = \"/spack-test-prefix\"\n\n\n#: the \"real\" compiler the wrapper is expected to invoke\nreal_cc = \"/bin/mycc\"\n\n# mock flags to use in the wrapper environment\nspack_cppflags = [\"-g\", \"-O1\", \"-DVAR=VALUE\"]\nspack_cflags = [\"-Wall\"]\nspack_cxxflags = [\"-Werror\"]\nspack_fflags = [\"-w\"]\nspack_ldflags = [\"-Wl,--gc-sections\", \"-L\", \"foo\"]\nspack_ldlibs = [\"-lfoo\"]\n\nlheaderpad = [\"-Wl,-headerpad_max_install_names\"]\nheaderpad = [\"-headerpad_max_install_names\"]\n\ntarget_args = [\"-march=znver2\", \"-mtune=znver2\"]\ntarget_args_fc = [\"-march=znver4\", \"-mtune=znver4\"]\n\n# common compile arguments: includes, libs, -Wl linker args, other args\ncommon_compile_args = (\n test_include_paths\n + test_library_paths\n + [\"-Wl,--disable-new-dtags\"]\n + test_wl_rpaths\n + test_args_without_paths\n)\n\npytestmark = pytest.mark.not_on_windows(\"does not run on windows\")\n\n\n@pytest.fixture(scope=\"function\")\ndef wrapper_environment(working_env):\n with set_env(\n SPACK_CC=real_cc,\n SPACK_CXX=real_cc,\n SPACK_FC=real_cc,\n SPACK_PREFIX=pkg_prefix,\n SPACK_COMPILER_WRAPPER_PATH=\"test\",\n SPACK_DEBUG_LOG_DIR=\".\",\n SPACK_DEBUG_LOG_ID=\"foo-hashabc\",\n SPACK_SHORT_SPEC=\"foo@1.2 arch=linux-rhel6-x86_64 /hashabc\",\n SPACK_SYSTEM_DIRS=SYSTEM_DIR_CASE_ENTRY,\n SPACK_MANAGED_DIRS=\"/path/to/spack-1/opt/spack/*|/path/to/spack-2/opt/spack/*\",\n SPACK_CC_RPATH_ARG=\"-Wl,-rpath,\",\n SPACK_CXX_RPATH_ARG=\"-Wl,-rpath,\",\n SPACK_F77_RPATH_ARG=\"-Wl,-rpath,\",\n SPACK_FC_RPATH_ARG=\"-Wl,-rpath,\",\n SPACK_LINK_DIRS=None,\n SPACK_INCLUDE_DIRS=None,\n SPACK_RPATH_DIRS=None,\n SPACK_TARGET_ARGS_CC=\"-march=znver2 -mtune=znver2\",\n SPACK_TARGET_ARGS_CXX=\"-march=znver2 -mtune=znver2\",\n SPACK_TARGET_ARGS_FORTRAN=\"-march=znver4 -mtune=znver4\",\n SPACK_CC_LINKER_ARG=\"-Wl,\",\n SPACK_CXX_LINKER_ARG=\"-Wl,\",\n SPACK_FC_LINKER_ARG=\"-Wl,\",\n SPACK_F77_LINKER_ARG=\"-Wl,\",\n SPACK_DTAGS_TO_ADD=\"--disable-new-dtags\",\n SPACK_DTAGS_TO_STRIP=\"--enable-new-dtags\",\n SPACK_COMPILER_FLAGS_KEEP=\"\",\n SPACK_COMPILER_FLAGS_REPLACE=\"-Werror*|\",\n ):\n yield\n\n\n@pytest.fixture()\ndef wrapper_flags():\n with set_env(\n SPACK_CPPFLAGS=\" \".join(spack_cppflags),\n SPACK_CFLAGS=\" \".join(spack_cflags),\n SPACK_CXXFLAGS=\" \".join(spack_cxxflags),\n SPACK_FFLAGS=\" \".join(spack_fflags),\n SPACK_LDFLAGS=\" \".join(spack_ldflags),\n SPACK_LDLIBS=\" \".join(spack_ldlibs),\n ):\n yield\n\n\ndef check_args(cc, args, expected):\n \"\"\"Check output arguments that cc produces when called with args.\n\n This assumes that cc will print debug command output with one element\n per line, so that we see whether arguments that should (or shouldn't)\n contain spaces are parsed correctly.\n \"\"\"\n cc = Executable(str(cc))\n with set_env(SPACK_TEST_COMMAND=\"dump-args\"):\n cc_modified_args = cc(*args, output=str).strip().split(\"\\n\")\n assert cc_modified_args == expected\n\n\ndef check_args_contents(cc, args, must_contain, must_not_contain):\n \"\"\"Check output arguments that cc produces when called with args.\n\n This assumes that cc will print debug command output with one element\n per line, so that we see whether arguments that should (or shouldn't)\n contain spaces are parsed correctly.\n \"\"\"\n cc = Executable(str(cc))\n with set_env(SPACK_TEST_COMMAND=\"dump-args\"):\n cc_modified_args = cc(*args, output=str).strip().split(\"\\n\")\n for a in must_contain:\n assert a in cc_modified_args\n for a in must_not_contain:\n assert a not in cc_modified_args\n\n\ndef check_env_var(executable, var, expected):\n \"\"\"Check environment variables updated by the passed compiler wrapper\n\n This assumes that cc will print debug output when it's environment\n contains SPACK_TEST_COMMAND=dump-env-\n \"\"\"\n executable = Executable(str(executable))\n with set_env(SPACK_TEST_COMMAND=\"dump-env-\" + var):\n output = executable(*test_args, output=str).strip()\n assert executable.path + \": \" + var + \": \" + expected == output\n\n\ndef dump_mode(cc, args):\n \"\"\"Make cc dump the mode it detects, and return it.\"\"\"\n cc = Executable(str(cc))\n with set_env(SPACK_TEST_COMMAND=\"dump-mode\"):\n return cc(*args, output=str).strip()\n\n\ndef test_no_wrapper_environment(wrapper_dir):\n cc = Executable(str(wrapper_dir / \"cc\"))\n with pytest.raises(ProcessError):\n output = cc(output=str)\n assert \"Spack compiler must be run from Spack\" in output\n\n\ndef test_modes(wrapper_environment, wrapper_dir):\n cc = wrapper_dir / \"cc\"\n cxx = wrapper_dir / \"c++\"\n cpp = wrapper_dir / \"cpp\"\n ld = wrapper_dir / \"ld\"\n\n # vcheck\n assert dump_mode(cc, [\"-I/include\", \"--version\"]) == \"vcheck\"\n assert dump_mode(cc, [\"-I/include\", \"-V\"]) == \"vcheck\"\n assert dump_mode(cc, [\"-I/include\", \"-v\"]) == \"vcheck\"\n assert dump_mode(cc, [\"-I/include\", \"-dumpversion\"]) == \"vcheck\"\n assert dump_mode(cc, [\"-I/include\", \"--version\", \"-c\"]) == \"vcheck\"\n assert dump_mode(cc, [\"-I/include\", \"-V\", \"-o\", \"output\"]) == \"vcheck\"\n\n # cpp\n assert dump_mode(cc, [\"-E\"]) == \"cpp\"\n assert dump_mode(cxx, [\"-E\"]) == \"cpp\"\n assert dump_mode(cpp, []) == \"cpp\"\n\n # as\n assert dump_mode(cc, [\"-S\"]) == \"as\"\n\n # ccld\n assert dump_mode(cc, []) == \"ccld\"\n assert dump_mode(cc, [\"foo.c\", \"-o\", \"foo\"]) == \"ccld\"\n assert dump_mode(cc, [\"foo.c\", \"-o\", \"foo\", \"-Wl,-rpath,foo\"]) == \"ccld\"\n assert dump_mode(cc, [\"foo.o\", \"bar.o\", \"baz.o\", \"-o\", \"foo\", \"-Wl,-rpath,foo\"]) == \"ccld\"\n\n # ld\n assert dump_mode(ld, []) == \"ld\"\n assert dump_mode(ld, [\"foo.o\", \"bar.o\", \"baz.o\", \"-o\", \"foo\", \"-Wl,-rpath,foo\"]) == \"ld\"\n\n\n@pytest.mark.regression(\"37179\")\ndef test_expected_args(wrapper_environment, wrapper_dir):\n cc = wrapper_dir / \"cc\"\n fc = wrapper_dir / \"fc\"\n ld = wrapper_dir / \"ld\"\n\n # ld_unterminated_rpath\n check_args(\n ld,\n [\"foo.o\", \"bar.o\", \"baz.o\", \"-o\", \"foo\", \"-rpath\"],\n [\"ld\", \"--disable-new-dtags\", \"foo.o\", \"bar.o\", \"baz.o\", \"-o\", \"foo\", \"-rpath\"],\n )\n\n # xlinker_unterminated_rpath\n check_args(\n cc,\n [\"foo.o\", \"bar.o\", \"baz.o\", \"-o\", \"foo\", \"-Xlinker\", \"-rpath\"],\n [real_cc]\n + target_args\n + [\n \"-Wl,--disable-new-dtags\",\n \"foo.o\",\n \"bar.o\",\n \"baz.o\",\n \"-o\",\n \"foo\",\n \"-Xlinker\",\n \"-rpath\",\n ],\n )\n\n # wl_unterminated_rpath\n check_args(\n cc,\n [\"foo.o\", \"bar.o\", \"baz.o\", \"-o\", \"foo\", \"-Wl,-rpath\"],\n [real_cc]\n + target_args\n + [\"-Wl,--disable-new-dtags\", \"foo.o\", \"bar.o\", \"baz.o\", \"-o\", \"foo\", \"-Wl,-rpath\"],\n )\n\n # Wl_parsing\n check_args(\n cc,\n [\"-Wl,-rpath,/a,--enable-new-dtags,-rpath=/b,--rpath\", \"-Wl,/c\"],\n [real_cc]\n + target_args\n + [\"-Wl,--disable-new-dtags\", \"-Wl,-rpath,/a\", \"-Wl,-rpath,/b\", \"-Wl,-rpath,/c\"],\n )\n\n # Wl_parsing_with_missing_value\n check_args(\n cc,\n [\"-Wl,-rpath=/a,-rpath=\", \"-Wl,--rpath=\"],\n [real_cc] + target_args + [\"-Wl,--disable-new-dtags\", \"-Wl,-rpath,/a\"],\n )\n\n # Wl_parsing_NAG_is_ignored\n check_args(\n fc,\n [\"-Wl,-Wl,,x,,y,,z\"],\n [real_cc] + target_args_fc + [\"-Wl,--disable-new-dtags\", \"-Wl,-Wl,,x,,y,,z\"],\n )\n\n # Xlinker_parsing\n #\n # -Xlinker ... -Xlinker may have compiler flags in between, like -O3 in this\n # example. Also check that a trailing -Xlinker (which is a compiler error) is not\n # dropped or given an empty argument.\n check_args(\n cc,\n [\n \"-Xlinker\",\n \"-rpath\",\n \"-O3\",\n \"-Xlinker\",\n \"/a\",\n \"-Xlinker\",\n \"--flag\",\n \"-Xlinker\",\n \"-rpath=/b\",\n \"-Xlinker\",\n ],\n [real_cc]\n + target_args\n + [\n \"-Wl,--disable-new-dtags\",\n \"-Wl,-rpath,/a\",\n \"-Wl,-rpath,/b\",\n \"-O3\",\n \"-Xlinker\",\n \"--flag\",\n \"-Xlinker\",\n ],\n )\n\n # rpath_without_value\n #\n # cc -Wl,-rpath without a value shouldn't drop -Wl,-rpath;\n # same for -Xlinker\n check_args(\n cc,\n [\"-Wl,-rpath\", \"-O3\", \"-g\"],\n [real_cc] + target_args + [\"-Wl,--disable-new-dtags\", \"-O3\", \"-g\", \"-Wl,-rpath\"],\n )\n check_args(\n cc,\n [\"-Xlinker\", \"-rpath\", \"-O3\", \"-g\"],\n [real_cc] + target_args + [\"-Wl,--disable-new-dtags\", \"-O3\", \"-g\", \"-Xlinker\", \"-rpath\"],\n )\n\n # dep_rapth\n check_args(cc, test_args, [real_cc] + target_args + common_compile_args)\n\n # dep_include\n with set_env(SPACK_INCLUDE_DIRS=\"x\"):\n check_args(\n cc,\n test_args,\n [real_cc]\n + target_args\n + test_include_paths\n + [\"-Ix\"]\n + test_library_paths\n + [\"-Wl,--disable-new-dtags\"]\n + test_wl_rpaths\n + test_args_without_paths,\n )\n\n # dep_lib\n #\n # Ensure a single dependency RPATH is added\n with set_env(SPACK_LINK_DIRS=\"x\", SPACK_RPATH_DIRS=\"x\"):\n check_args(\n cc,\n test_args,\n [real_cc]\n + target_args\n + test_include_paths\n + test_library_paths\n + [\"-Lx\"]\n + [\"-Wl,--disable-new-dtags\"]\n + test_wl_rpaths\n + [\"-Wl,-rpath,x\"]\n + test_args_without_paths,\n )\n\n # dep_lib_no_rpath\n #\n # Ensure a single dependency link flag is added with no dep RPATH\n with set_env(SPACK_LINK_DIRS=\"x\"):\n check_args(\n cc,\n test_args,\n [real_cc]\n + target_args\n + test_include_paths\n + test_library_paths\n + [\"-Lx\"]\n + [\"-Wl,--disable-new-dtags\"]\n + test_wl_rpaths\n + test_args_without_paths,\n )\n\n # dep_lib_no_lib\n # Ensure a single dependency RPATH is added with no -L\n with set_env(SPACK_RPATH_DIRS=\"x\"):\n check_args(\n cc,\n test_args,\n [real_cc]\n + target_args\n + test_include_paths\n + test_library_paths\n + [\"-Wl,--disable-new-dtags\"]\n + test_wl_rpaths\n + [\"-Wl,-rpath,x\"]\n + test_args_without_paths,\n )\n\n # ccld_deps\n # Ensure all flags are added in ccld mode\n with set_env(\n SPACK_INCLUDE_DIRS=\"xinc:yinc:zinc\",\n SPACK_RPATH_DIRS=\"xlib:ylib:zlib\",\n SPACK_LINK_DIRS=\"xlib:ylib:zlib\",\n ):\n check_args(\n cc,\n test_args,\n [real_cc]\n + target_args\n + test_include_paths\n + [\"-Ixinc\", \"-Iyinc\", \"-Izinc\"]\n + test_library_paths\n + [\"-Lxlib\", \"-Lylib\", \"-Lzlib\"]\n + [\"-Wl,--disable-new-dtags\"]\n + test_wl_rpaths\n + [\"-Wl,-rpath,xlib\", \"-Wl,-rpath,ylib\", \"-Wl,-rpath,zlib\"]\n + test_args_without_paths,\n )\n\n # ccld_deps_isystem\n #\n # Ensure all flags are added in ccld mode.\n # When a build uses -isystem, Spack should inject it's\n # include paths using -isystem. Spack will insert these\n # after any provided -isystem includes, but before any\n # system directories included using -isystem\n with set_env(\n SPACK_INCLUDE_DIRS=\"xinc:yinc:zinc\",\n SPACK_RPATH_DIRS=\"xlib:ylib:zlib\",\n SPACK_LINK_DIRS=\"xlib:ylib:zlib\",\n ):\n mytest_args = test_args + [\"-isystem\", \"fooinc\"]\n check_args(\n cc,\n mytest_args,\n [real_cc]\n + target_args\n + test_include_paths\n + [\"-isystem\", \"fooinc\", \"-isystem\", \"xinc\", \"-isystem\", \"yinc\", \"-isystem\", \"zinc\"]\n + test_library_paths\n + [\"-Lxlib\", \"-Lylib\", \"-Lzlib\"]\n + [\"-Wl,--disable-new-dtags\"]\n + test_wl_rpaths\n + [\"-Wl,-rpath,xlib\", \"-Wl,-rpath,ylib\", \"-Wl,-rpath,zlib\"]\n + test_args_without_paths,\n )\n\n # cc_deps\n # Ensure -L and RPATHs are not added in cc mode\n with set_env(\n SPACK_INCLUDE_DIRS=\"xinc:yinc:zinc\",\n SPACK_RPATH_DIRS=\"xlib:ylib:zlib\",\n SPACK_LINK_DIRS=\"xlib:ylib:zlib\",\n ):\n check_args(\n cc,\n [\"-c\"] + test_args,\n [real_cc]\n + target_args\n + test_include_paths\n + [\"-Ixinc\", \"-Iyinc\", \"-Izinc\"]\n + test_library_paths\n + [\"-c\"]\n + test_args_without_paths,\n )\n\n # ccld_with_system_dirs\n # Ensure all flags are added in ccld mode\n with set_env(\n SPACK_INCLUDE_DIRS=\"xinc:yinc:zinc\",\n SPACK_RPATH_DIRS=\"xlib:ylib:zlib\",\n SPACK_LINK_DIRS=\"xlib:ylib:zlib\",\n ):\n sys_path_args = [\n \"-I/usr/include\",\n \"-L/usr/local/lib\",\n \"-Wl,-rpath,/usr/lib64\",\n \"-I/usr/local/include\",\n \"-L/lib64/\",\n ]\n check_args(\n cc,\n sys_path_args + test_args,\n [real_cc]\n + target_args\n + test_include_paths\n + [\"-Ixinc\", \"-Iyinc\", \"-Izinc\"]\n + [\"-I/usr/include\", \"-I/usr/local/include\"]\n + test_library_paths\n + [\"-Lxlib\", \"-Lylib\", \"-Lzlib\"]\n + [\"-L/usr/local/lib\", \"-L/lib64/\"]\n + [\"-Wl,--disable-new-dtags\"]\n + test_wl_rpaths\n + [\"-Wl,-rpath,xlib\", \"-Wl,-rpath,ylib\", \"-Wl,-rpath,zlib\"]\n + [\"-Wl,-rpath,/usr/lib64\"]\n + test_args_without_paths,\n )\n\n # ccld_with_system_dirs_isystem\n # Ensure all flags are added in ccld mode.\n # Ensure that includes are in the proper\n # place when a build uses -isystem, and uses\n # system directories in the include paths\n with set_env(\n SPACK_INCLUDE_DIRS=\"xinc:yinc:zinc\",\n SPACK_RPATH_DIRS=\"xlib:ylib:zlib\",\n SPACK_LINK_DIRS=\"xlib:ylib:zlib\",\n ):\n sys_path_args = [\n \"-isystem\",\n \"/usr/include\",\n \"-L/usr/local/lib\",\n \"-Wl,-rpath,/usr/lib64\",\n \"-isystem\",\n \"/usr/local/include\",\n \"-L/lib64/\",\n ]\n check_args(\n cc,\n sys_path_args + test_args,\n [real_cc]\n + target_args\n + test_include_paths\n + [\"-isystem\", \"xinc\", \"-isystem\", \"yinc\", \"-isystem\", \"zinc\"]\n + [\"-isystem\", \"/usr/include\", \"-isystem\", \"/usr/local/include\"]\n + test_library_paths\n + [\"-Lxlib\", \"-Lylib\", \"-Lzlib\"]\n + [\"-L/usr/local/lib\", \"-L/lib64/\"]\n + [\"-Wl,--disable-new-dtags\"]\n + test_wl_rpaths\n + [\"-Wl,-rpath,xlib\", \"-Wl,-rpath,ylib\", \"-Wl,-rpath,zlib\"]\n + [\"-Wl,-rpath,/usr/lib64\"]\n + test_args_without_paths,\n )\n\n # ld_deps\n # Ensure no (extra) -I args or -Wl, are passed in ld mode\n with set_env(\n SPACK_INCLUDE_DIRS=\"xinc:yinc:zinc\",\n SPACK_RPATH_DIRS=\"xlib:ylib:zlib\",\n SPACK_LINK_DIRS=\"xlib:ylib:zlib\",\n ):\n check_args(\n ld,\n test_args,\n [\"ld\"]\n + test_include_paths\n + test_library_paths\n + [\"-Lxlib\", \"-Lylib\", \"-Lzlib\"]\n + [\"--disable-new-dtags\"]\n + test_rpaths\n + [\"-rpath\", \"xlib\", \"-rpath\", \"ylib\", \"-rpath\", \"zlib\"]\n + test_args_without_paths,\n )\n\n # ld_deps_no_rpath\n # Ensure SPACK_LINK_DEPS controls -L for ld\n with set_env(SPACK_INCLUDE_DIRS=\"xinc:yinc:zinc\", SPACK_LINK_DIRS=\"xlib:ylib:zlib\"):\n check_args(\n ld,\n test_args,\n [\"ld\"]\n + test_include_paths\n + test_library_paths\n + [\"-Lxlib\", \"-Lylib\", \"-Lzlib\"]\n + [\"--disable-new-dtags\"]\n + test_rpaths\n + test_args_without_paths,\n )\n\n # ld_deps_no_link\n # Ensure SPACK_RPATH_DEPS controls -rpath for ld\n with set_env(SPACK_INCLUDE_DIRS=\"xinc:yinc:zinc\", SPACK_RPATH_DIRS=\"xlib:ylib:zlib\"):\n check_args(\n ld,\n test_args,\n [\"ld\"]\n + test_include_paths\n + test_library_paths\n + [\"--disable-new-dtags\"]\n + test_rpaths\n + [\"-rpath\", \"xlib\", \"-rpath\", \"ylib\", \"-rpath\", \"zlib\"]\n + test_args_without_paths,\n )\n\n\ndef test_expected_args_with_flags(wrapper_environment, wrapper_flags, wrapper_dir):\n cc = wrapper_dir / \"cc\"\n cxx = wrapper_dir / \"c++\"\n cpp = wrapper_dir / \"cpp\"\n fc = wrapper_dir / \"fc\"\n ld = wrapper_dir / \"ld\"\n\n # ld_flags\n check_args(\n ld,\n test_args,\n [\"ld\"]\n + test_include_paths\n + test_library_paths\n + [\"--disable-new-dtags\"]\n + test_rpaths\n + test_args_without_paths\n + spack_ldlibs,\n )\n\n # cpp_flags\n check_args(\n cpp,\n test_args,\n [\"cpp\"]\n + test_include_paths\n + test_library_paths\n + test_args_without_paths\n + spack_cppflags,\n )\n\n # cc_flags\n check_args(\n cc,\n test_args,\n [real_cc]\n + target_args\n + test_include_paths\n + [\"-Lfoo\"]\n + test_library_paths\n + [\"-Wl,--disable-new-dtags\"]\n + test_wl_rpaths\n + test_args_without_paths\n + spack_cppflags\n + spack_cflags\n + [\"-Wl,--gc-sections\"]\n + spack_ldlibs,\n )\n\n # cxx_flags\n check_args(\n cxx,\n test_args,\n [real_cc]\n + target_args\n + test_include_paths\n + [\"-Lfoo\"]\n + test_library_paths\n + [\"-Wl,--disable-new-dtags\"]\n + test_wl_rpaths\n + test_args_without_paths\n + spack_cppflags\n + [\"-Wl,--gc-sections\"]\n + spack_ldlibs,\n )\n\n # fc_flags\n check_args(\n fc,\n test_args,\n [real_cc]\n + target_args_fc\n + test_include_paths\n + [\"-Lfoo\"]\n + test_library_paths\n + [\"-Wl,--disable-new-dtags\"]\n + test_wl_rpaths\n + test_args_without_paths\n + spack_fflags\n + spack_cppflags\n + [\"-Wl,--gc-sections\"]\n + spack_ldlibs,\n )\n\n # always_cflags\n with set_env(SPACK_ALWAYS_CFLAGS=\"-always1 -always2\"):\n check_args(\n cc,\n [\"-v\", \"--cmd-line-v-opt\"],\n [real_cc] + [\"-always1\", \"-always2\"] + [\"-v\", \"--cmd-line-v-opt\"],\n )\n\n\ndef test_system_path_cleanup(wrapper_environment, wrapper_dir):\n \"\"\"Ensure SPACK_COMPILER_WRAPPER_PATH is removed from PATH, even with trailing /\n\n The compiler wrapper has to ensure that it is not called nested\n like it would happen when gcc's collect2 looks in PATH for ld.\n\n To prevent nested calls, the compiler wrapper removes the elements\n of SPACK_COMPILER_WRAPPER_PATH from PATH. Autotest's generated testsuite appends\n a / to each element of PATH when adding AUTOTEST_PATH.\n Thus, ensure that PATH cleanup works even with trailing /.\n \"\"\"\n cc = wrapper_dir / \"cc\"\n system_path = \"/bin:/usr/bin:/usr/local/bin\"\n with set_env(SPACK_COMPILER_WRAPPER_PATH=str(wrapper_dir), SPACK_CC=\"true\"):\n with set_env(PATH=str(wrapper_dir) + \":\" + system_path):\n check_env_var(cc, \"PATH\", system_path)\n with set_env(PATH=str(wrapper_dir) + \"/:\" + system_path):\n check_env_var(cc, \"PATH\", system_path)\n\n\ndef test_ld_deps_partial(wrapper_environment, wrapper_dir):\n \"\"\"Make sure ld -r (partial link) is handled correctly on OS's where it\n doesn't accept rpaths.\n \"\"\"\n ld = wrapper_dir / \"ld\"\n with set_env(SPACK_INCLUDE_DIRS=\"xinc\", SPACK_RPATH_DIRS=\"xlib\", SPACK_LINK_DIRS=\"xlib\"):\n # TODO: do we need to add RPATHs on other platforms like Linux?\n # TODO: Can't we treat them the same?\n os.environ[\"SPACK_SHORT_SPEC\"] = \"foo@1.2=linux-x86_64\"\n check_args(\n ld,\n [\"-r\"] + test_args,\n [\"ld\"]\n + test_include_paths\n + test_library_paths\n + [\"-Lxlib\"]\n + [\"--disable-new-dtags\"]\n + test_rpaths\n + [\"-rpath\", \"xlib\"]\n + [\"-r\"]\n + test_args_without_paths,\n )\n\n # rpaths from the underlying command will still appear\n # Spack will not add its own rpaths.\n os.environ[\"SPACK_SHORT_SPEC\"] = \"foo@1.2=darwin-x86_64\"\n check_args(\n ld,\n [\"-r\"] + test_args,\n [\"ld\"]\n + headerpad\n + test_include_paths\n + test_library_paths\n + [\"-Lxlib\"]\n + [\"--disable-new-dtags\"]\n + test_rpaths\n + [\"-r\"]\n + test_args_without_paths,\n )\n\n\ndef test_ccache_prepend_for_cc(wrapper_environment, wrapper_dir):\n cc = wrapper_dir / \"cc\"\n with set_env(SPACK_CCACHE_BINARY=\"ccache\"):\n os.environ[\"SPACK_SHORT_SPEC\"] = \"foo@1.2=linux-x86_64\"\n check_args(\n cc,\n test_args,\n [\"ccache\"]\n + [real_cc] # ccache prepended in cc mode\n + target_args\n + common_compile_args,\n )\n os.environ[\"SPACK_SHORT_SPEC\"] = \"foo@1.2=darwin-x86_64\"\n check_args(\n cc,\n test_args,\n [\"ccache\"]\n + [real_cc] # ccache prepended in cc mode\n + target_args\n + lheaderpad\n + common_compile_args,\n )\n\n\ndef test_no_ccache_prepend_for_fc(wrapper_environment, wrapper_dir):\n fc = wrapper_dir / \"fc\"\n os.environ[\"SPACK_SHORT_SPEC\"] = \"foo@1.2=linux-x86_64\"\n check_args(\n fc,\n test_args,\n # no ccache for Fortran\n [real_cc] + target_args_fc + common_compile_args,\n )\n os.environ[\"SPACK_SHORT_SPEC\"] = \"foo@1.2=darwin-x86_64\"\n check_args(\n fc,\n test_args,\n # no ccache for Fortran\n [real_cc] + target_args_fc + lheaderpad + common_compile_args,\n )\n\n\ndef test_keep_and_replace(wrapper_environment, wrapper_dir):\n cc = wrapper_dir / \"cc\"\n werror_specific = [\"-Werror=meh\"]\n werror = [\"-Werror\"]\n werror_all = werror_specific + werror\n with set_env(SPACK_COMPILER_FLAGS_KEEP=\"\", SPACK_COMPILER_FLAGS_REPLACE=\"-Werror*|\"):\n check_args_contents(cc, test_args + werror_all, [\"-Wl,--end-group\"], werror_all)\n with set_env(SPACK_COMPILER_FLAGS_KEEP=\"-Werror=*\", SPACK_COMPILER_FLAGS_REPLACE=\"-Werror*|\"):\n check_args_contents(cc, test_args + werror_all, werror_specific, werror)\n with set_env(\n SPACK_COMPILER_FLAGS_KEEP=\"-Werror=*\",\n SPACK_COMPILER_FLAGS_REPLACE=\"-Werror*| -llib1| -Wl*|\",\n ):\n check_args_contents(\n cc, test_args + werror_all, werror_specific, werror + [\"-llib1\", \"-Wl,--rpath\"]\n )\n\n\n@pytest.mark.parametrize(\n \"cfg_override,initial,expected,must_be_gone\",\n [\n # Set and unset variables\n (\n \"config:flags:keep_werror:all\",\n [\"-Werror\", \"-Werror=specific\", \"-bah\"],\n [\"-Werror\", \"-Werror=specific\", \"-bah\"],\n [],\n ),\n (\n \"config:flags:keep_werror:specific\",\n [\"-Werror\", \"-Werror=specific\", \"-Werror-specific2\", \"-bah\"],\n [\"-Wno-error\", \"-Werror=specific\", \"-Werror-specific2\", \"-bah\"],\n [\"-Werror\"],\n ),\n (\n \"config:flags:keep_werror:none\",\n [\"-Werror\", \"-Werror=specific\", \"-bah\"],\n [\"-Wno-error\", \"-Wno-error=specific\", \"-bah\"],\n [\"-Werror\", \"-Werror=specific\"],\n ),\n # check non-standard -Werror opts like -Werror-implicit-function-declaration\n (\n \"config:flags:keep_werror:all\",\n [\"-Werror\", \"-Werror-implicit-function-declaration\", \"-bah\"],\n [\"-Werror\", \"-Werror-implicit-function-declaration\", \"-bah\"],\n [],\n ),\n (\n \"config:flags:keep_werror:specific\",\n [\"-Werror\", \"-Werror-implicit-function-declaration\", \"-bah\"],\n [\"-Wno-error\", \"-Werror-implicit-function-declaration\", \"-bah\"],\n [\"-Werror\"],\n ),\n (\n \"config:flags:keep_werror:none\",\n [\"-Werror\", \"-Werror-implicit-function-declaration\", \"-bah\"],\n [\"-Wno-error\", \"-bah\", \"-Wno-error=implicit-function-declaration\"],\n [\"-Werror\", \"-Werror-implicit-function-declaration\"],\n ),\n ],\n)\n@pytest.mark.usefixtures(\"wrapper_environment\", \"mutable_config\")\ndef test_flag_modification(cfg_override, initial, expected, must_be_gone, wrapper_dir):\n cc = wrapper_dir / \"cc\"\n spack.config.add(cfg_override)\n env = spack.build_environment.clean_environment()\n\n keep_werror = spack.config.get(\"config:flags:keep_werror\")\n spack.build_environment._add_werror_handling(keep_werror, env)\n env.apply_modifications()\n check_args_contents(cc, test_args[:3] + initial, expected, must_be_gone)\n\n\n@pytest.mark.regression(\"9160\")\ndef test_disable_new_dtags(wrapper_environment, wrapper_flags, wrapper_dir):\n cc = Executable(str(wrapper_dir / \"cc\"))\n ld = Executable(str(wrapper_dir / \"ld\"))\n with set_env(SPACK_TEST_COMMAND=\"dump-args\"):\n result = ld(*test_args, output=str).strip().split(\"\\n\")\n assert \"--disable-new-dtags\" in result\n result = cc(*test_args, output=str).strip().split(\"\\n\")\n assert \"-Wl,--disable-new-dtags\" in result\n\n\n@pytest.mark.regression(\"9160\")\ndef test_filter_enable_new_dtags(wrapper_environment, wrapper_flags, wrapper_dir):\n cc = Executable(str(wrapper_dir / \"cc\"))\n ld = Executable(str(wrapper_dir / \"ld\"))\n with set_env(SPACK_TEST_COMMAND=\"dump-args\"):\n result = ld(*(test_args + [\"--enable-new-dtags\"]), output=str)\n result = result.strip().split(\"\\n\")\n assert \"--enable-new-dtags\" not in result\n\n result = cc(*(test_args + [\"-Wl,--enable-new-dtags\"]), output=str)\n result = result.strip().split(\"\\n\")\n assert \"-Wl,--enable-new-dtags\" not in result\n\n\n@pytest.mark.regression(\"22643\")\ndef test_linker_strips_loopopt(wrapper_environment, wrapper_flags, wrapper_dir):\n cc = Executable(str(wrapper_dir / \"cc\"))\n ld = Executable(str(wrapper_dir / \"ld\"))\n with set_env(SPACK_TEST_COMMAND=\"dump-args\"):\n # ensure that -loopopt=0 is not present in ld mode\n result = ld(*(test_args + [\"-loopopt=0\"]), output=str)\n result = result.strip().split(\"\\n\")\n assert \"-loopopt=0\" not in result\n\n # ensure that -loopopt=0 is not present in ccld mode\n result = cc(*(test_args + [\"-loopopt=0\"]), output=str)\n result = result.strip().split(\"\\n\")\n assert \"-loopopt=0\" not in result\n\n # ensure that -loopopt=0 *is* present in cc mode\n # The \"-c\" argument is needed for cc to be detected\n # as compile only (cc) mode.\n result = cc(*(test_args + [\"-loopopt=0\", \"-c\", \"x.c\"]), output=str)\n result = result.strip().split(\"\\n\")\n assert \"-loopopt=0\" in result\n\n\ndef test_spack_managed_dirs_are_prioritized(wrapper_environment, wrapper_dir):\n cc = Executable(str(wrapper_dir / \"cc\"))\n\n # We have two different stores with 5 packages divided over them\n pkg1 = \"/path/to/spack-1/opt/spack/linux-ubuntu22.04-zen2/gcc-13.2.0/pkg-1.0-abcdef\"\n pkg2 = \"/path/to/spack-1/opt/spack/linux-ubuntu22.04-zen2/gcc-13.2.0/pkg-2.0-abcdef\"\n pkg3 = \"/path/to/spack-2/opt/spack/linux-ubuntu22.04-zen2/gcc-13.2.0/pkg-3.0-abcdef\"\n pkg4 = \"/path/to/spack-2/opt/spack/linux-ubuntu22.04-zen2/gcc-13.2.0/pkg-4.0-abcdef\"\n pkg5 = \"/path/to/spack-2/opt/spack/linux-ubuntu22.04-zen2/gcc-13.2.0/pkg-5.0-abcdef\"\n\n variables = {\n # cppflags, ldflags from the command line, config or package.py take highest priority\n \"SPACK_CPPFLAGS\": f\"-I/usr/local/include -I/external-1/include -I{pkg1}/include\",\n \"SPACK_LDFLAGS\": f\"-L/usr/local/lib -L/external-1/lib -L{pkg1}/lib \"\n f\"-Wl,-rpath,/usr/local/lib -Wl,-rpath,/external-1/lib -Wl,-rpath,{pkg1}/lib\",\n # automatic -L, -Wl,-rpath, -I flags from dependencies -- on the spack side they are\n # already partitioned into \"spack owned prefixes\" and \"non-spack owned prefixes\"\n \"SPACK_STORE_LINK_DIRS\": f\"{pkg4}/lib:{pkg5}/lib\",\n \"SPACK_STORE_RPATH_DIRS\": f\"{pkg4}/lib:{pkg5}/lib\",\n \"SPACK_STORE_INCLUDE_DIRS\": f\"{pkg4}/include:{pkg5}/include\",\n \"SPACK_LINK_DIRS\": \"/external-3/lib:/external-4/lib\",\n \"SPACK_RPATH_DIRS\": \"/external-3/lib:/external-4/lib\",\n \"SPACK_INCLUDE_DIRS\": \"/external-3/include:/external-4/include\",\n }\n\n with set_env(SPACK_TEST_COMMAND=\"dump-args\", **variables):\n effective_call = (\n cc(\n # system paths\n \"-I/usr/include\",\n \"-L/usr/lib\",\n \"-Wl,-rpath,/usr/lib\",\n # some other externals\n \"-I/external-2/include\",\n \"-L/external-2/lib\",\n \"-Wl,-rpath,/external-2/lib\",\n # relative paths are considered \"spack managed\" since they are in the stage dir\n \"-I..\",\n \"-L..\",\n \"-Wl,-rpath,..\", # pathological but simpler for the test.\n # spack store paths\n f\"-I{pkg2}/include\",\n f\"-I{pkg3}/include\",\n f\"-L{pkg2}/lib\",\n f\"-L{pkg3}/lib\",\n f\"-Wl,-rpath,{pkg2}/lib\",\n f\"-Wl,-rpath,{pkg3}/lib\",\n \"hello.c\",\n \"-o\",\n \"hello\",\n output=str,\n )\n .strip()\n .split(\"\\n\")\n )\n\n dash_I = [flag[2:] for flag in effective_call if flag.startswith(\"-I\")]\n dash_L = [flag[2:] for flag in effective_call if flag.startswith(\"-L\")]\n dash_Wl_rpath = [flag[11:] for flag in effective_call if flag.startswith(\"-Wl,-rpath\")]\n\n assert dash_I == [\n # spack owned dirs from SPACK_*FLAGS\n f\"{pkg1}/include\",\n # spack owned dirs from command line & automatic flags for deps (in that order)]\n \"..\",\n f\"{pkg2}/include\", # from command line\n f\"{pkg3}/include\", # from command line\n f\"{pkg4}/include\", # from SPACK_STORE_INCLUDE_DIRS\n f\"{pkg5}/include\", # from SPACK_STORE_INCLUDE_DIRS\n # non-system dirs from SPACK_*FLAGS\n \"/external-1/include\",\n # non-system dirs from command line & automatic flags for deps (in that order)\n \"/external-2/include\", # from command line\n \"/external-3/include\", # from SPACK_INCLUDE_DIRS\n \"/external-4/include\", # from SPACK_INCLUDE_DIRS\n # system dirs from SPACK_*FLAGS\n \"/usr/local/include\",\n # system dirs from command line\n \"/usr/include\",\n ]\n\n assert (\n dash_L\n == dash_Wl_rpath\n == [\n # spack owned dirs from SPACK_*FLAGS\n f\"{pkg1}/lib\",\n # spack owned dirs from command line & automatic flags for deps (in that order)\n \"..\",\n f\"{pkg2}/lib\", # from command line\n f\"{pkg3}/lib\", # from command line\n f\"{pkg4}/lib\", # from SPACK_STORE_LINK_DIRS\n f\"{pkg5}/lib\", # from SPACK_STORE_LINK_DIRS\n # non-system dirs from SPACK_*FLAGS\n \"/external-1/lib\",\n # non-system dirs from command line & automatic flags for deps (in that order)\n \"/external-2/lib\", # from command line\n \"/external-3/lib\", # from SPACK_LINK_DIRS\n \"/external-4/lib\", # from SPACK_LINK_DIRS\n # system dirs from SPACK_*FLAGS\n \"/usr/local/lib\",\n # system dirs from command line\n \"/usr/lib\",\n ]\n )\n" }, { "path": "lib/spack/spack/test/ci.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport io\nimport os\nimport pathlib\nimport subprocess\nfrom urllib.error import HTTPError\n\nimport pytest\n\nimport spack.ci as ci\nimport spack.concretize\nimport spack.environment as ev\nimport spack.error\nimport spack.llnl.util.filesystem as fs\nimport spack.paths\nimport spack.repo as repo\nimport spack.util.git\nfrom spack.spec import Spec\nfrom spack.test.conftest import MockHTTPResponse, RepoBuilder\nfrom spack.version import Version\n\npytestmark = [pytest.mark.usefixtures(\"mock_packages\")]\n\n\n@pytest.fixture\ndef repro_dir(tmp_path: pathlib.Path):\n result = tmp_path / \"repro_dir\"\n result.mkdir()\n with fs.working_dir(str(tmp_path)):\n yield result\n\n\ndef test_filter_added_checksums_new_checksum(mock_git_package_changes):\n repo, filename, commits = mock_git_package_changes\n\n checksum_versions = {\n \"3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04\": Version(\"2.1.5\"),\n \"a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a\": Version(\"2.1.4\"),\n \"6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200\": Version(\"2.0.7\"),\n \"86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8\": Version(\"2.0.0\"),\n }\n\n with fs.working_dir(repo.packages_path):\n assert ci.filter_added_checksums(\n checksum_versions.keys(), filename, from_ref=commits[-1], to_ref=commits[-2]\n ) == [\"3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04\"]\n\n\ndef test_filter_added_checksums_new_commit(mock_git_package_changes):\n repo, filename, commits = mock_git_package_changes\n\n checksum_versions = {\n \"74253725f884e2424a0dd8ae3f69896d5377f325\": Version(\"2.1.6\"),\n \"3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04\": Version(\"2.1.5\"),\n \"a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a\": Version(\"2.1.4\"),\n \"6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200\": Version(\"2.0.7\"),\n \"86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8\": Version(\"2.0.0\"),\n }\n\n with fs.working_dir(repo.packages_path):\n assert ci.filter_added_checksums(\n checksum_versions, filename, from_ref=commits[-2], to_ref=commits[-3]\n ) == [\"74253725f884e2424a0dd8ae3f69896d5377f325\"]\n\n\ndef test_pipeline_dag(config, repo_builder: RepoBuilder):\n r\"\"\"Test creation, pruning, and traversal of PipelineDAG using the\n following package dependency graph:\n\n a a\n /| /|\n c b c b\n |\\ prune 'd' /|\\\n e d =====> e | g\n | |\\ | |\n h | g h |\n \\| \\|\n f f\n\n \"\"\"\n repo_builder.add_package(\"pkg-h\", dependencies=[(\"pkg-f\", None, None)])\n repo_builder.add_package(\"pkg-g\")\n repo_builder.add_package(\"pkg-f\")\n repo_builder.add_package(\"pkg-e\", dependencies=[(\"pkg-h\", None, None)])\n repo_builder.add_package(\"pkg-d\", dependencies=[(\"pkg-f\", None, None), (\"pkg-g\", None, None)])\n repo_builder.add_package(\"pkg-c\")\n repo_builder.add_package(\"pkg-b\", dependencies=[(\"pkg-d\", None, None), (\"pkg-e\", None, None)])\n repo_builder.add_package(\"pkg-a\", dependencies=[(\"pkg-b\", None, None), (\"pkg-c\", None, None)])\n\n with repo.use_repositories(repo_builder.root):\n spec_a = spack.concretize.concretize_one(\"pkg-a\")\n\n key_a = ci.common.PipelineDag.key(spec_a)\n key_b = ci.common.PipelineDag.key(spec_a[\"pkg-b\"])\n key_c = ci.common.PipelineDag.key(spec_a[\"pkg-c\"])\n key_d = ci.common.PipelineDag.key(spec_a[\"pkg-d\"])\n key_e = ci.common.PipelineDag.key(spec_a[\"pkg-e\"])\n key_f = ci.common.PipelineDag.key(spec_a[\"pkg-f\"])\n key_g = ci.common.PipelineDag.key(spec_a[\"pkg-g\"])\n key_h = ci.common.PipelineDag.key(spec_a[\"pkg-h\"])\n\n pipeline = ci.common.PipelineDag([spec_a])\n\n expected_bottom_up_traversal = {\n key_a: 4,\n key_b: 3,\n key_c: 0,\n key_d: 1,\n key_e: 2,\n key_f: 0,\n key_g: 0,\n key_h: 1,\n }\n\n visited = []\n for stage, node in pipeline.traverse_nodes(direction=\"parents\"):\n assert expected_bottom_up_traversal[node.key] == stage\n visited.append(node.key)\n\n assert len(visited) == len(expected_bottom_up_traversal)\n assert all(k in visited for k in expected_bottom_up_traversal.keys())\n\n expected_top_down_traversal = {\n key_a: 0,\n key_b: 1,\n key_c: 1,\n key_d: 2,\n key_e: 2,\n key_f: 4,\n key_g: 3,\n key_h: 3,\n }\n\n visited = []\n for stage, node in pipeline.traverse_nodes(direction=\"children\"):\n assert expected_top_down_traversal[node.key] == stage\n visited.append(node.key)\n\n assert len(visited) == len(expected_top_down_traversal)\n assert all(k in visited for k in expected_top_down_traversal.keys())\n\n pipeline.prune(key_d)\n b_children = pipeline.nodes[key_b].children\n assert len(b_children) == 3\n assert all([k in b_children for k in [key_e, key_f, key_g]])\n\n # check another bottom-up traversal after pruning pkg-d\n expected_bottom_up_traversal = {\n key_a: 4,\n key_b: 3,\n key_c: 0,\n key_e: 2,\n key_f: 0,\n key_g: 0,\n key_h: 1,\n }\n\n visited = []\n for stage, node in pipeline.traverse_nodes(direction=\"parents\"):\n assert expected_bottom_up_traversal[node.key] == stage\n visited.append(node.key)\n\n assert len(visited) == len(expected_bottom_up_traversal)\n assert all(k in visited for k in expected_bottom_up_traversal.keys())\n\n # check top-down traversal after pruning pkg-d\n expected_top_down_traversal = {\n key_a: 0,\n key_b: 1,\n key_c: 1,\n key_e: 2,\n key_f: 4,\n key_g: 2,\n key_h: 3,\n }\n\n visited = []\n for stage, node in pipeline.traverse_nodes(direction=\"children\"):\n assert expected_top_down_traversal[node.key] == stage\n visited.append(node.key)\n\n assert len(visited) == len(expected_top_down_traversal)\n assert all(k in visited for k in expected_top_down_traversal.keys())\n\n a_deps_direct = [n.spec for n in pipeline.get_dependencies(pipeline.nodes[key_a])]\n assert all([s in a_deps_direct for s in [spec_a[\"pkg-b\"], spec_a[\"pkg-c\"]]])\n\n\n@pytest.mark.not_on_windows(\"Not supported on Windows (yet)\")\ndef test_import_signing_key(mock_gnupghome):\n signing_key_dir = spack.paths.mock_gpg_keys_path\n signing_key_path = os.path.join(signing_key_dir, \"package-signing-key\")\n with open(signing_key_path, encoding=\"utf-8\") as fd:\n signing_key = fd.read()\n\n # Just make sure this does not raise any exceptions\n ci.import_signing_key(signing_key)\n\n\ndef test_download_and_extract_artifacts(tmp_path: pathlib.Path, monkeypatch):\n monkeypatch.setenv(\"GITLAB_PRIVATE_TOKEN\", \"faketoken\")\n\n url = \"https://www.nosuchurlexists.itsfake/artifacts.zip\"\n working_dir = tmp_path / \"repro\"\n test_artifacts_path = os.path.join(\n spack.paths.test_path, \"data\", \"ci\", \"gitlab\", \"artifacts.zip\"\n )\n\n def _urlopen_OK(*args, **kwargs):\n with open(test_artifacts_path, \"rb\") as f:\n return MockHTTPResponse(\n \"200\", \"OK\", {\"Content-Type\": \"application/zip\"}, io.BytesIO(f.read())\n )\n\n monkeypatch.setattr(ci, \"urlopen\", _urlopen_OK)\n\n ci.download_and_extract_artifacts(url, str(working_dir))\n\n found_zip = fs.find(working_dir, \"artifacts.zip\")\n assert len(found_zip) == 0\n\n found_install = fs.find(working_dir, \"install.sh\")\n assert len(found_install) == 1\n\n def _urlopen_500(*args, **kwargs):\n raise HTTPError(url, 500, \"Internal Server Error\", {}, None)\n\n monkeypatch.setattr(ci, \"urlopen\", _urlopen_500)\n\n with pytest.raises(spack.error.SpackError):\n ci.download_and_extract_artifacts(url, str(working_dir))\n\n\ndef test_ci_copy_stage_logs_to_artifacts_fail(\n tmp_path: pathlib.Path, default_mock_concretization, capfd\n):\n \"\"\"The copy will fail because the spec is not concrete so does not have\n a package.\"\"\"\n log_dir = tmp_path / \"log_dir\"\n concrete_spec = default_mock_concretization(\"printing-package\")\n ci.copy_stage_logs_to_artifacts(concrete_spec, str(log_dir))\n _, err = capfd.readouterr()\n assert \"Unable to copy files\" in err\n assert \"No such file or directory\" in err\n\n\ndef test_ci_copy_test_logs_to_artifacts_fail(tmp_path: pathlib.Path, capfd):\n log_dir = tmp_path / \"log_dir\"\n\n ci.copy_test_logs_to_artifacts(\"no-such-dir\", str(log_dir))\n _, err = capfd.readouterr()\n assert \"Cannot copy test logs\" in err\n\n stage_dir = tmp_path / \"stage_dir\"\n stage_dir.mkdir()\n ci.copy_test_logs_to_artifacts(str(stage_dir), str(log_dir))\n _, err = capfd.readouterr()\n assert \"Unable to copy files\" in err\n assert \"No such file or directory\" in err\n\n\ndef test_setup_spack_repro_version(\n tmp_path: pathlib.Path, capfd, last_two_git_commits, monkeypatch\n):\n c1, c2 = last_two_git_commits\n repro_dir = tmp_path / \"repro\"\n spack_dir = repro_dir / \"spack\"\n spack_dir.mkdir(parents=True)\n\n prefix_save = spack.paths.prefix\n monkeypatch.setattr(spack.paths, \"prefix\", \"/garbage\")\n\n ret = ci.setup_spack_repro_version(str(repro_dir), c2, c1)\n _, err = capfd.readouterr()\n\n assert not ret\n assert \"Unable to find the path\" in err\n\n monkeypatch.setattr(spack.paths, \"prefix\", prefix_save)\n monkeypatch.setattr(spack.util.git, \"git\", lambda: None)\n\n ret = ci.setup_spack_repro_version(str(repro_dir), c2, c1)\n out, err = capfd.readouterr()\n\n assert not ret\n assert \"requires git\" in err\n\n class mock_git_cmd:\n def __init__(self, *args, **kwargs):\n self.returncode = 0\n self.check = None\n\n def __call__(self, *args, **kwargs):\n if self.check:\n self.returncode = self.check(*args, **kwargs)\n else:\n self.returncode = 0\n\n git_cmd = mock_git_cmd()\n\n monkeypatch.setattr(spack.util.git, \"git\", lambda: git_cmd)\n\n git_cmd.check = lambda *a, **k: 1 if len(a) > 2 and a[2] == c2 else 0\n ret = ci.setup_spack_repro_version(str(repro_dir), c2, c1)\n _, err = capfd.readouterr()\n\n assert not ret\n assert \"Missing commit: {0}\".format(c2) in err\n\n git_cmd.check = lambda *a, **k: 1 if len(a) > 2 and a[2] == c1 else 0\n ret = ci.setup_spack_repro_version(str(repro_dir), c2, c1)\n _, err = capfd.readouterr()\n\n assert not ret\n assert \"Missing commit: {0}\".format(c1) in err\n\n git_cmd.check = lambda *a, **k: 1 if a[0] == \"clone\" else 0\n ret = ci.setup_spack_repro_version(str(repro_dir), c2, c1)\n _, err = capfd.readouterr()\n\n assert not ret\n assert \"Unable to clone\" in err\n\n git_cmd.check = lambda *a, **k: 1 if a[0] == \"checkout\" else 0\n ret = ci.setup_spack_repro_version(str(repro_dir), c2, c1)\n _, err = capfd.readouterr()\n\n assert not ret\n assert \"Unable to checkout\" in err\n\n git_cmd.check = lambda *a, **k: 1 if \"merge\" in a else 0\n ret = ci.setup_spack_repro_version(str(repro_dir), c2, c1)\n _, err = capfd.readouterr()\n\n assert not ret\n assert \"Unable to merge {0}\".format(c1) in err\n\n\ndef test_get_spec_filter_list(mutable_mock_env_path, mutable_mock_repo):\n \"\"\"Tests that, given an active environment and list of touched pkgs, we get the right\n list of possibly-changed env specs.\n\n The test concretizes the following environment:\n\n [ ] hypre@=0.2.15+shared build_system=generic\n [bl ] ^openblas-with-lapack@=0.2.15 build_system=generic\n [ ] mpileaks@=2.3~debug~opt+shared+static build_system=generic\n [bl ] ^callpath@=1.0 build_system=generic\n [bl ] ^dyninst@=8.2 build_system=generic\n [bl ] ^libdwarf@=20130729 build_system=generic\n [bl ] ^libelf@=0.8.13 build_system=generic\n [b ] ^gcc@=10.2.1 build_system=generic languages='c,c++,fortran'\n [ l ] ^gcc-runtime@=10.2.1 build_system=generic\n [bl ] ^mpich@=3.0.4~debug build_system=generic\n\n and simulates a change in libdwarf.\n \"\"\"\n e1 = ev.create(\"test\")\n e1.add(\"mpileaks\")\n e1.add(\"hypre\")\n e1.concretize()\n\n touched = {\"libdwarf\"}\n\n # Make sure we return the correct set of possibly affected specs,\n # given a dependent traversal depth and the fact that the touched\n # package is libdwarf. Passing traversal depth of None or something\n # equal to or larger than the greatest depth in the graph are\n # equivalent and result in traversal of all specs from the touched\n # package to the root. Passing negative traversal depth results in\n # no spec traversals. Passing any other number yields differing\n # numbers of possibly affected specs.\n\n full_set = {\n \"mpileaks\",\n \"mpich\",\n \"callpath\",\n \"dyninst\",\n \"libdwarf\",\n \"libelf\",\n \"gcc\",\n \"gcc-runtime\",\n \"compiler-wrapper\",\n }\n depth_2_set = {\n \"mpich\",\n \"callpath\",\n \"dyninst\",\n \"libdwarf\",\n \"libelf\",\n \"gcc\",\n \"gcc-runtime\",\n \"compiler-wrapper\",\n }\n depth_1_set = {\"dyninst\", \"libdwarf\", \"libelf\", \"gcc\", \"gcc-runtime\", \"compiler-wrapper\"}\n depth_0_set = {\"libdwarf\", \"libelf\", \"gcc\", \"gcc-runtime\", \"compiler-wrapper\"}\n\n expectations = {\n None: full_set,\n 3: full_set,\n 100: full_set,\n -1: set(),\n 0: depth_0_set,\n 1: depth_1_set,\n 2: depth_2_set,\n }\n\n for key, val in expectations.items():\n affected_specs = ci.get_spec_filter_list(e1, touched, dependent_traverse_depth=key)\n affected_pkg_names = {s.name for s in affected_specs}\n assert affected_pkg_names == val\n\n\n@pytest.mark.regression(\"29947\")\ndef test_affected_specs_on_first_concretization(mutable_mock_env_path):\n e = ev.create(\"first_concretization\")\n e.add(\"mpileaks~shared\")\n e.add(\"mpileaks+shared\")\n e.concretize()\n\n affected_specs = ci.get_spec_filter_list(e, {\"callpath\"})\n mpileaks_specs = [s for s in affected_specs if s.name == \"mpileaks\"]\n assert len(mpileaks_specs) == 2, e.all_specs()\n\n\n@pytest.mark.not_on_windows(\"Reliance on bash script not supported on Windows\")\ndef test_ci_process_command(repro_dir):\n result = ci.process_command(\"help\", commands=[], repro_dir=str(repro_dir))\n help_sh = repro_dir / \"help.sh\"\n assert help_sh.exists() and not result\n\n\n@pytest.mark.not_on_windows(\"Reliance on bash script not supported on Windows\")\ndef test_ci_process_command_fail(repro_dir, monkeypatch):\n msg = \"subprocess wait exception\"\n\n def _fail(self, args):\n raise RuntimeError(msg)\n\n monkeypatch.setattr(subprocess.Popen, \"__init__\", _fail)\n with pytest.raises(RuntimeError, match=msg):\n ci.process_command(\"help\", [], str(repro_dir))\n\n\ndef test_ci_create_buildcache(working_env, config, monkeypatch):\n \"\"\"Test that create_buildcache returns a list of objects with the correct\n keys and types.\"\"\"\n monkeypatch.setattr(ci, \"push_to_build_cache\", lambda a, b, c: True)\n\n results = ci.create_buildcache(\n Spec(), destination_mirror_urls=[\"file:///fake-url-one\", \"file:///fake-url-two\"]\n )\n\n assert len(results) == 2\n result1, result2 = results\n assert result1.success\n assert result1.url == \"file:///fake-url-one\"\n assert result2.success\n assert result2.url == \"file:///fake-url-two\"\n\n results = ci.create_buildcache(Spec(), destination_mirror_urls=[\"file:///fake-url-one\"])\n\n assert len(results) == 1\n assert results[0].success\n assert results[0].url == \"file:///fake-url-one\"\n\n\ndef test_ci_run_standalone_tests_missing_requirements(\n working_env, default_mock_concretization, capfd\n):\n \"\"\"This test case checks for failing prerequisite checks.\"\"\"\n ci.run_standalone_tests()\n err = capfd.readouterr()[1]\n assert \"Job spec is required\" in err\n\n args = {\"job_spec\": default_mock_concretization(\"printing-package\")}\n ci.run_standalone_tests(**args)\n err = capfd.readouterr()[1]\n assert \"Reproduction directory is required\" in err\n\n\n@pytest.mark.not_on_windows(\"Reliance on bash script not supported on Windows\")\ndef test_ci_run_standalone_tests_not_installed_junit(\n tmp_path: pathlib.Path, repro_dir, working_env, mock_test_stage, capfd\n):\n log_file = tmp_path / \"junit.xml\"\n\n ci.run_standalone_tests(\n log_file=str(log_file),\n job_spec=spack.concretize.concretize_one(\"printing-package\"),\n repro_dir=str(repro_dir),\n fail_fast=True,\n )\n err = capfd.readouterr()[1]\n assert \"No installed packages\" in err\n assert os.path.getsize(log_file) > 0\n\n\n@pytest.mark.not_on_windows(\"Reliance on bash script not supported on Windows\")\ndef test_ci_run_standalone_tests_not_installed_cdash(\n tmp_path: pathlib.Path, repro_dir, working_env, mock_test_stage, capfd\n):\n \"\"\"Test run_standalone_tests with cdash and related options.\"\"\"\n log_file = tmp_path / \"junit.xml\"\n\n # Cover when CDash handler provided (with the log file as well)\n ci_cdash = {\n \"url\": \"file://fake\",\n \"build-group\": \"fake-group\",\n \"project\": \"ci-unit-testing\",\n \"site\": \"fake-site\",\n }\n os.environ[\"SPACK_CDASH_BUILD_NAME\"] = \"ci-test-build\"\n os.environ[\"SPACK_CDASH_BUILD_STAMP\"] = \"ci-test-build-stamp\"\n os.environ[\"CI_RUNNER_DESCRIPTION\"] = \"test-runner\"\n handler = ci.CDashHandler(ci_cdash)\n ci.run_standalone_tests(\n log_file=str(log_file),\n job_spec=spack.concretize.concretize_one(\"printing-package\"),\n repro_dir=str(repro_dir),\n cdash=handler,\n )\n out = capfd.readouterr()[0]\n # CDash *and* log file output means log file ignored\n assert \"xml option is ignored with CDash\" in out\n\n # copy test results (though none)\n artifacts_dir = tmp_path / \"artifacts\"\n artifacts_dir.mkdir()\n handler.copy_test_results(str(tmp_path), str(artifacts_dir))\n err = capfd.readouterr()[1]\n assert \"Unable to copy files\" in err\n assert \"No such file or directory\" in err\n\n\ndef test_ci_skipped_report(tmp_path: pathlib.Path, config):\n \"\"\"Test explicit skipping of report as well as CI's 'package' arg.\"\"\"\n pkg = \"trivial-smoke-test\"\n spec = spack.concretize.concretize_one(pkg)\n ci_cdash = {\n \"url\": \"file://fake\",\n \"build-group\": \"fake-group\",\n \"project\": \"ci-unit-testing\",\n \"site\": \"fake-site\",\n }\n os.environ[\"SPACK_CDASH_BUILD_NAME\"] = \"fake-test-build\"\n os.environ[\"SPACK_CDASH_BUILD_STAMP\"] = \"ci-test-build-stamp\"\n os.environ[\"CI_RUNNER_DESCRIPTION\"] = \"test-runner\"\n handler = ci.CDashHandler(ci_cdash)\n reason = \"Testing skip\"\n handler.report_skipped(spec, str(tmp_path), reason=reason)\n\n reports = [name for name in tmp_path.iterdir() if str(name).endswith(\"Testing.xml\")]\n assert len(reports) == 1\n expected = f\"Skipped {pkg} package\"\n with open(reports[0], \"r\", encoding=\"utf-8\") as f:\n have = [0, 0]\n for line in f:\n if expected in line:\n have[0] += 1\n elif reason in line:\n have[1] += 1\n assert all(count == 1 for count in have)\n" }, { "path": "lib/spack/spack/test/cmd/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n" }, { "path": "lib/spack/spack/test/cmd/arch.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nfrom spack.main import SpackCommand\n\narch = SpackCommand(\"arch\")\n\n\ndef test_arch():\n \"\"\"Sanity check ``spack arch`` to make sure it works.\"\"\"\n\n arch()\n arch(\"-f\")\n arch(\"--frontend\")\n arch(\"-b\")\n arch(\"--backend\")\n\n\ndef test_arch_platform():\n \"\"\"Sanity check ``spack arch --platform`` to make sure it works.\"\"\"\n\n arch(\"-p\")\n arch(\"--platform\")\n arch(\"-f\", \"-p\")\n arch(\"-b\", \"-p\")\n\n\ndef test_arch_operating_system():\n \"\"\"Sanity check ``spack arch --operating-system`` to make sure it works.\"\"\"\n\n arch(\"-o\")\n arch(\"--operating-system\")\n arch(\"-f\", \"-o\")\n arch(\"-b\", \"-o\")\n\n\ndef test_arch_target():\n \"\"\"Sanity check ``spack arch --target`` to make sure it works.\"\"\"\n\n arch(\"-t\")\n arch(\"--target\")\n arch(\"-f\", \"-t\")\n arch(\"-b\", \"-t\")\n\n\ndef test_display_targets():\n arch(\"--known-targets\")\n" }, { "path": "lib/spack/spack/test/cmd/audit.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport pytest\n\nimport spack.audit\nfrom spack.main import SpackCommand\nfrom spack.test.conftest import MockHTTPResponse\n\naudit = SpackCommand(\"audit\")\n\n\n@pytest.mark.parametrize(\n \"pkgs,expected_returncode\",\n [\n # A single package with issues, should exit 1\n ([\"wrong-variant-in-conflicts\"], 1),\n # A \"sane\" package should exit 0\n ([\"mpileaks\"], 0),\n # A package with issues and a package without should exit 1\n ([\"wrong-variant-in-conflicts\", \"mpileaks\"], 1),\n ([\"mpileaks\", \"wrong-variant-in-conflicts\"], 1),\n ],\n)\ndef test_audit_packages(pkgs, expected_returncode, mutable_config, mock_packages):\n \"\"\"Sanity check ``spack audit packages`` to make sure it works.\"\"\"\n audit(\"packages\", *pkgs, fail_on_error=False)\n assert audit.returncode == expected_returncode\n\n\ndef test_audit_configs(mutable_config, mock_packages):\n \"\"\"Sanity check ``spack audit packages`` to make sure it works.\"\"\"\n audit(\"configs\", fail_on_error=False)\n # The mock configuration has duplicate definitions of some compilers\n assert audit.returncode == 1\n\n\ndef test_audit_packages_https(mutable_config, mock_packages, monkeypatch):\n \"\"\"Test audit packages-https with mocked network calls.\"\"\"\n monkeypatch.setattr(spack.audit, \"urlopen\", lambda url: MockHTTPResponse(200, \"OK\"))\n\n # Without providing --all should fail\n audit(\"packages-https\", fail_on_error=False)\n # The mock configuration has duplicate definitions of some compilers\n assert audit.returncode == 1\n\n # This uses http and should fail\n audit(\"packages-https\", \"test-dependency\", fail_on_error=False)\n assert audit.returncode == 1\n\n # providing one or more package names with https should work\n audit(\"packages-https\", \"cmake\", fail_on_error=True)\n assert audit.returncode == 0\n\n # providing one or more package names with https should work\n audit(\"packages-https\", \"cmake\", \"conflict\", fail_on_error=True)\n assert audit.returncode == 0\n" }, { "path": "lib/spack/spack/test/cmd/blame.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport collections\nimport os\nfrom pathlib import Path\n\nimport pytest\n\nimport spack.cmd.blame\nimport spack.paths\nimport spack.util.spack_json as sjson\nfrom spack.cmd.blame import ensure_full_history, git_prefix, package_repo_root\nfrom spack.llnl.util.filesystem import mkdirp, working_dir\nfrom spack.main import SpackCommand, SpackCommandError\nfrom spack.repo import RepoDescriptors\nfrom spack.util.executable import ProcessError\n\npytestmark = pytest.mark.usefixtures(\"git\")\n\nblame = SpackCommand(\"blame\")\n\n\ndef test_blame_by_modtime(mock_packages):\n \"\"\"Sanity check the blame command to make sure it works.\"\"\"\n out = blame(\"--time\", \"mpich\")\n assert \"LAST_COMMIT\" in out\n assert \"AUTHOR\" in out\n assert \"EMAIL\" in out\n\n\ndef test_blame_by_percent(mock_packages):\n \"\"\"Sanity check the blame command to make sure it works.\"\"\"\n out = blame(\"--percent\", \"mpich\")\n assert \"LAST_COMMIT\" in out\n assert \"AUTHOR\" in out\n assert \"EMAIL\" in out\n\n\ndef test_blame_file():\n \"\"\"Sanity check the blame command to make sure it works.\"\"\"\n with working_dir(spack.paths.prefix):\n out = blame(os.path.join(\"bin\", \"spack\"))\n assert \"LAST_COMMIT\" in out\n assert \"AUTHOR\" in out\n assert \"EMAIL\" in out\n\n\ndef test_blame_file_missing():\n \"\"\"Ensure attempt to get blame for missing file fails.\"\"\"\n with pytest.raises(SpackCommandError):\n out = blame(os.path.join(\"missing\", \"file.txt\"))\n assert \"does not exist\" in out\n\n\ndef test_blame_directory():\n \"\"\"Ensure attempt to get blame for path that is a directory fails.\"\"\"\n with pytest.raises(SpackCommandError):\n out = blame(\".\")\n assert \"not tracked\" in out\n\n\ndef test_blame_file_outside_spack_repo(tmp_path: Path):\n \"\"\"Ensure attempts to get blame outside a package repository are flagged.\"\"\"\n test_file = tmp_path / \"test\"\n test_file.write_text(\"This is a test\")\n with pytest.raises(SpackCommandError):\n out = blame(str(test_file))\n assert \"not within a spack repo\" in out\n\n\ndef test_blame_spack_not_git_clone(monkeypatch):\n \"\"\"Ensure attempt to get blame when spack not a git clone fails.\"\"\"\n non_git_dir = os.path.join(spack.paths.prefix, \"..\")\n monkeypatch.setattr(spack.paths, \"prefix\", non_git_dir)\n\n with pytest.raises(SpackCommandError):\n out = blame(\".\")\n assert \"not in a git clone\" in out\n\n\ndef test_blame_json(mock_packages):\n \"\"\"Ensure that we can output json as a blame.\"\"\"\n with working_dir(spack.paths.prefix):\n out = blame(\"--json\", \"mpich\")\n\n # Test loading the json, and top level keys\n loaded = sjson.load(out)\n assert \"authors\" in out\n assert \"totals\" in out\n\n # Authors should be a list\n assert len(loaded[\"authors\"]) > 0\n\n # Each of authors and totals has these shared keys\n keys = [\"last_commit\", \"lines\", \"percentage\"]\n for key in keys:\n assert key in loaded[\"totals\"]\n\n # But authors is a list of multiple\n for key in keys + [\"author\", \"email\"]:\n assert key in loaded[\"authors\"][0]\n\n\n@pytest.mark.not_on_windows(\"git hangs\")\ndef test_blame_by_git(mock_packages):\n \"\"\"Sanity check the blame command to make sure it works.\"\"\"\n out = blame(\"--git\", \"mpich\")\n assert \"class Mpich\" in out\n assert ' homepage = \"http://www.mpich.org\"' in out\n\n\ndef test_repo_root_local_descriptor(mock_git_version_info, monkeypatch):\n \"\"\"Sanity check blame's package repository root using a local repo descriptor.\"\"\"\n\n # create a mock descriptor for the mock local repository\n MockLocalDescriptor = collections.namedtuple(\"MockLocalDescriptor\", [\"path\"])\n repo_path, filename, _ = mock_git_version_info\n git_repo_path = Path(repo_path)\n spack_repo_path = git_repo_path / \"spack_repo\"\n spack_repo_path.mkdir()\n\n repo_descriptor = MockLocalDescriptor(spack_repo_path)\n\n def _from_config(*args, **kwargs):\n return {\"mock\": repo_descriptor}\n\n monkeypatch.setattr(RepoDescriptors, \"from_config\", _from_config)\n\n # The parent of the git repository is outside the package repo root\n path = (git_repo_path / \"..\").resolve()\n prefix = package_repo_root((path / \"..\").resolve())\n assert prefix is None\n\n # The base repository directory is the git root of the package repo\n prefix = package_repo_root(git_repo_path)\n assert prefix == git_repo_path\n\n # The file under the base repository directory also has the package git root\n prefix = package_repo_root(git_repo_path / filename)\n assert prefix == git_repo_path\n\n\ndef test_repo_root_remote_descriptor(mock_git_version_info, monkeypatch):\n \"\"\"Sanity check blame's package repository root using a remote repo descriptor.\"\"\"\n\n # create a mock descriptor for the mock local repository\n MockRemoteDescriptor = collections.namedtuple(\"MockRemoteDescriptor\", [\"destination\"])\n repo_path, filename, _ = mock_git_version_info\n git_repo_path = Path(repo_path)\n\n repo_descriptor = MockRemoteDescriptor(git_repo_path)\n\n def _from_config(*args, **kwargs):\n return {\"mock\": repo_descriptor}\n\n monkeypatch.setattr(RepoDescriptors, \"from_config\", _from_config)\n\n # The parent of the git repository is outside the package repo root\n path = (git_repo_path / \"..\").resolve()\n prefix = package_repo_root((path / \"..\").resolve())\n assert prefix is None\n\n # The base repository directory is the git root of the package repo\n prefix = package_repo_root(git_repo_path)\n assert prefix == git_repo_path\n\n\ndef test_git_prefix_bad(tmp_path: Path):\n \"\"\"Exercise git_prefix paths with arguments that will not return success.\"\"\"\n assert git_prefix(\"no/such/file.txt\") is None\n\n with pytest.raises(SystemExit):\n git_prefix(tmp_path)\n\n\ndef test_ensure_full_history_shallow_works(mock_git_version_info, monkeypatch):\n \"\"\"Ensure a git that \"supports\" '--unshallow' \"completes\" without incident.\"\"\"\n\n def _git(*args, **kwargs):\n if \"--help\" in args:\n return \"--unshallow\"\n else:\n return \"\"\n\n repo_path, filename, _ = mock_git_version_info\n shallow_dir = os.path.join(repo_path, \".git\", \"shallow\")\n mkdirp(shallow_dir)\n\n # Need to patch the blame command's\n monkeypatch.setattr(spack.cmd.blame, \"git\", _git)\n ensure_full_history(repo_path, filename)\n\n\ndef test_ensure_full_history_shallow_fails(mock_git_version_info, monkeypatch, capfd):\n \"\"\"Ensure a git that supports '--unshallow' but fails generates useful error.\"\"\"\n error_msg = \"Mock git cannot fetch.\"\n\n def _git(*args, **kwargs):\n if \"--help\" in args:\n return \"--unshallow\"\n else:\n raise ProcessError(error_msg)\n\n repo_path, filename, _ = mock_git_version_info\n shallow_dir = os.path.join(repo_path, \".git\", \"shallow\")\n mkdirp(shallow_dir)\n\n # Need to patch the blame command's since 'git' already used by\n # mock_git_versioninfo\n monkeypatch.setattr(spack.cmd.blame, \"git\", _git)\n with pytest.raises(SystemExit):\n ensure_full_history(repo_path, filename)\n\n out = capfd.readouterr()\n assert error_msg in out[1]\n\n\ndef test_ensure_full_history_shallow_old_git(mock_git_version_info, monkeypatch, capfd):\n \"\"\"Ensure a git that doesn't support '--unshallow' fails.\"\"\"\n\n def _git(*args, **kwargs):\n return \"\"\n\n repo_path, filename, _ = mock_git_version_info\n shallow_dir = os.path.join(repo_path, \".git\", \"shallow\")\n mkdirp(shallow_dir)\n\n # Need to patch the blame command's since 'git' already used by\n # mock_git_versioninfo\n monkeypatch.setattr(spack.cmd.blame, \"git\", _git)\n with pytest.raises(SystemExit):\n ensure_full_history(repo_path, filename)\n\n out = capfd.readouterr()\n assert \"Use a newer\" in out[1]\n" }, { "path": "lib/spack/spack/test/cmd/bootstrap.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport os\nimport pathlib\n\nimport pytest\n\nimport spack.bootstrap\nimport spack.bootstrap.core\nimport spack.cmd.mirror\nimport spack.concretize\nimport spack.config\nimport spack.environment as ev\nimport spack.main\nimport spack.spec\n\n_bootstrap = spack.main.SpackCommand(\"bootstrap\")\n\n\n@pytest.mark.parametrize(\"scope\", [None, \"site\", \"system\", \"user\"])\ndef test_enable_and_disable(mutable_config, scope):\n scope_args = []\n if scope:\n scope_args = [\"--scope={0}\".format(scope)]\n\n _bootstrap(\"enable\", *scope_args)\n assert spack.config.get(\"bootstrap:enable\", scope=scope) is True\n\n _bootstrap(\"disable\", *scope_args)\n assert spack.config.get(\"bootstrap:enable\", scope=scope) is False\n\n\n@pytest.mark.parametrize(\"scope\", [None, \"site\", \"system\", \"user\"])\ndef test_root_get_and_set(mutable_config, tmp_path, scope):\n scope_args, path = [], str(tmp_path)\n if scope:\n scope_args = [\"--scope={0}\".format(scope)]\n\n _bootstrap(\"root\", path, *scope_args)\n out = _bootstrap(\"root\", *scope_args)\n assert out.strip() == path\n\n\n@pytest.mark.parametrize(\"scopes\", [(\"site\",), (\"system\", \"user\")])\ndef test_reset_in_file_scopes(mutable_config, scopes):\n # Assert files are created in the right scopes\n bootstrap_yaml_files = []\n for s in scopes:\n _bootstrap(\"disable\", \"--scope={0}\".format(s))\n scope_path = spack.config.CONFIG.scopes[s].path\n bootstrap_yaml = os.path.join(scope_path, \"bootstrap.yaml\")\n assert os.path.exists(bootstrap_yaml)\n bootstrap_yaml_files.append(bootstrap_yaml)\n\n _bootstrap(\"reset\", \"-y\")\n for bootstrap_yaml in bootstrap_yaml_files:\n assert not os.path.exists(bootstrap_yaml)\n\n\ndef test_reset_in_environment(mutable_mock_env_path, mutable_config):\n env = spack.main.SpackCommand(\"env\")\n env(\"create\", \"bootstrap-test\")\n current_environment = ev.read(\"bootstrap-test\")\n\n with current_environment:\n _bootstrap(\"disable\")\n assert spack.config.get(\"bootstrap:enable\") is False\n _bootstrap(\"reset\", \"-y\")\n # We have no default settings in tests\n assert spack.config.get(\"bootstrap:enable\") is None\n\n # Check that reset didn't delete the entire file\n spack_yaml = os.path.join(current_environment.path, \"spack.yaml\")\n assert os.path.exists(spack_yaml)\n\n\ndef test_reset_in_file_scopes_overwrites_backup_files(mutable_config):\n # Create a bootstrap.yaml with some config\n _bootstrap(\"disable\", \"--scope=site\")\n scope_path = spack.config.CONFIG.scopes[\"site\"].path\n bootstrap_yaml = os.path.join(scope_path, \"bootstrap.yaml\")\n assert os.path.exists(bootstrap_yaml)\n\n # Reset the bootstrap configuration\n _bootstrap(\"reset\", \"-y\")\n backup_file = bootstrap_yaml + \".bkp\"\n assert not os.path.exists(bootstrap_yaml)\n assert os.path.exists(backup_file)\n\n # Iterate another time\n _bootstrap(\"disable\", \"--scope=site\")\n assert os.path.exists(bootstrap_yaml)\n assert os.path.exists(backup_file)\n _bootstrap(\"reset\", \"-y\")\n assert not os.path.exists(bootstrap_yaml)\n assert os.path.exists(backup_file)\n\n\ndef test_list_sources(config):\n # Get the merged list and ensure we get our defaults\n output = _bootstrap(\"list\")\n assert \"github-actions\" in output\n\n # Ask for a specific scope and check that the list of sources is empty\n output = _bootstrap(\"list\", \"--scope\", \"user\")\n assert \"No method available\" in output\n\n\n@pytest.mark.parametrize(\"command,value\", [(\"enable\", True), (\"disable\", False)])\ndef test_enable_or_disable_sources(mutable_config, command, value):\n key = \"bootstrap:trusted:github-actions\"\n trusted = spack.config.get(key, default=None)\n assert trusted is None\n\n _bootstrap(command, \"github-actions\")\n trusted = spack.config.get(key, default=None)\n assert trusted is value\n\n\ndef test_enable_or_disable_fails_with_no_method(mutable_config):\n with pytest.raises(RuntimeError, match=\"no bootstrapping method\"):\n _bootstrap(\"enable\", \"foo\")\n\n\ndef test_enable_or_disable_fails_with_more_than_one_method(mutable_config):\n wrong_config = {\n \"sources\": [\n {\"name\": \"github-actions\", \"metadata\": \"$spack/share/spack/bootstrap/github-actions\"},\n {\"name\": \"github-actions\", \"metadata\": \"$spack/share/spack/bootstrap/github-actions\"},\n ],\n \"trusted\": {},\n }\n with spack.config.override(\"bootstrap\", wrong_config):\n with pytest.raises(RuntimeError, match=\"more than one\"):\n _bootstrap(\"enable\", \"github-actions\")\n\n\n@pytest.mark.parametrize(\"use_existing_dir\", [True, False])\ndef test_add_failures_for_non_existing_files(\n mutable_config, tmp_path: pathlib.Path, use_existing_dir\n):\n metadata_dir = str(tmp_path) if use_existing_dir else \"/foo/doesnotexist\"\n with pytest.raises(RuntimeError, match=\"does not exist\"):\n _bootstrap(\"add\", \"mock-mirror\", metadata_dir)\n\n\ndef test_add_failures_for_already_existing_name(mutable_config):\n with pytest.raises(RuntimeError, match=\"already exist\"):\n _bootstrap(\"add\", \"github-actions\", \"some-place\")\n\n\ndef test_remove_failure_for_non_existing_names(mutable_config):\n with pytest.raises(RuntimeError, match=\"cannot find\"):\n _bootstrap(\"remove\", \"mock-mirror\")\n\n\ndef test_remove_and_add_a_source(mutable_config):\n # Check we start with a single bootstrapping source\n sources = spack.bootstrap.core.bootstrapping_sources()\n assert len(sources) == 1\n\n # Remove it and check the result\n _bootstrap(\"remove\", \"github-actions\")\n sources = spack.bootstrap.core.bootstrapping_sources()\n assert not sources\n\n # Add it back and check we restored the initial state\n _bootstrap(\"add\", \"github-actions\", \"$spack/share/spack/bootstrap/github-actions-v2\")\n sources = spack.bootstrap.core.bootstrapping_sources()\n assert len(sources) == 1\n\n\n@pytest.mark.maybeslow\n@pytest.mark.not_on_windows(\"Not supported on Windows (yet)\")\ndef test_bootstrap_mirror_metadata(mutable_config, linux_os, monkeypatch, tmp_path: pathlib.Path):\n \"\"\"Test that `spack bootstrap mirror` creates a folder that can be ingested by\n `spack bootstrap add`. Here we don't download data, since that would be an\n expensive operation for a unit test.\n \"\"\"\n old_create = spack.cmd.mirror.create\n monkeypatch.setattr(spack.cmd.mirror, \"create\", lambda p, s: old_create(p, []))\n monkeypatch.setattr(spack.concretize, \"concretize_one\", lambda p: spack.spec.Spec(p))\n\n # Create the mirror in a temporary folder\n compilers = [\n {\n \"compiler\": {\n \"spec\": \"gcc@12.0.1\",\n \"operating_system\": \"{0.name}{0.version}\".format(linux_os),\n \"modules\": [],\n \"paths\": {\n \"cc\": \"/usr/bin\",\n \"cxx\": \"/usr/bin\",\n \"fc\": \"/usr/bin\",\n \"f77\": \"/usr/bin\",\n },\n }\n }\n ]\n with spack.config.override(\"compilers\", compilers):\n _bootstrap(\"mirror\", str(tmp_path))\n\n # Register the mirror\n metadata_dir = tmp_path / \"metadata\" / \"sources\"\n _bootstrap(\"add\", \"--trust\", \"test-mirror\", str(metadata_dir))\n\n assert _bootstrap.returncode == 0\n assert any(m[\"name\"] == \"test-mirror\" for m in spack.bootstrap.core.bootstrapping_sources())\n" }, { "path": "lib/spack/spack/test/cmd/build_env.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport pathlib\nimport pickle\nimport sys\n\nimport pytest\n\nimport spack.error\nfrom spack.llnl.util.filesystem import working_dir\nfrom spack.main import SpackCommand\n\nbuild_env = SpackCommand(\"build-env\")\n\n\n@pytest.mark.parametrize(\"pkg\", [(\"pkg-c\",), (\"pkg-c\", \"--\")])\n@pytest.mark.usefixtures(\"config\", \"mock_packages\", \"working_env\")\ndef test_it_just_runs(pkg):\n build_env(*pkg)\n\n\n@pytest.mark.usefixtures(\"config\", \"mock_packages\", \"working_env\")\ndef test_error_when_multiple_specs_are_given():\n output = build_env(\"libelf libdwarf\", fail_on_error=False)\n assert \"only takes one spec\" in output\n\n\n@pytest.mark.parametrize(\"args\", [(\"--\", \"/bin/sh\", \"-c\", \"echo test\"), (\"--\",), ()])\n@pytest.mark.usefixtures(\"config\", \"mock_packages\", \"working_env\")\ndef test_build_env_requires_a_spec(args):\n output = build_env(*args, fail_on_error=False)\n assert \"requires a spec\" in output\n\n\n_out_file = \"env.out\"\n\n\n@pytest.mark.parametrize(\"shell\", [\"pwsh\", \"bat\"] if sys.platform == \"win32\" else [\"sh\"])\n@pytest.mark.usefixtures(\"config\", \"mock_packages\", \"working_env\")\ndef test_dump(shell_as, shell, tmp_path: pathlib.Path):\n with working_dir(str(tmp_path)):\n build_env(\"--dump\", _out_file, \"pkg-c\")\n with open(_out_file, encoding=\"utf-8\") as f:\n if shell == \"pwsh\":\n assert any(line.startswith(\"$Env:PATH\") for line in f.readlines())\n elif shell == \"bat\":\n assert any(line.startswith('set \"PATH=') for line in f.readlines())\n else:\n assert any(line.startswith(\"PATH=\") for line in f.readlines())\n\n\n@pytest.mark.usefixtures(\"config\", \"mock_packages\", \"working_env\")\ndef test_pickle(tmp_path: pathlib.Path):\n with working_dir(str(tmp_path)):\n build_env(\"--pickle\", _out_file, \"pkg-c\")\n environment = pickle.load(open(_out_file, \"rb\"))\n assert isinstance(environment, dict)\n assert \"PATH\" in environment\n\n\ndef test_failure_when_uninstalled_deps(config, mock_packages):\n with pytest.raises(\n spack.error.SpackError, match=\"Not all dependencies of dttop are installed\"\n ):\n build_env(\"dttop\")\n" }, { "path": "lib/spack/spack/test/cmd/buildcache.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport errno\nimport json\nimport os\nimport pathlib\nimport shutil\nimport urllib.parse\nfrom datetime import datetime, timedelta\nfrom typing import Dict, List\n\nimport pytest\n\nimport spack.binary_distribution\nimport spack.buildcache_migrate as migrate\nimport spack.buildcache_prune\nimport spack.cmd.buildcache\nimport spack.concretize\nimport spack.environment as ev\nimport spack.error\nimport spack.main\nimport spack.mirrors.mirror\nimport spack.spec\nimport spack.util.url as url_util\nimport spack.util.web as web_util\nfrom spack.installer import PackageInstaller\nfrom spack.llnl.util.filesystem import copy_tree, find, getuid\nfrom spack.llnl.util.lang import nullcontext\nfrom spack.paths import test_path\nfrom spack.url_buildcache import (\n BuildcacheComponent,\n URLBuildcacheEntry,\n URLBuildcacheEntryV2,\n check_mirror_for_layout,\n get_url_buildcache_class,\n)\n\nbuildcache = spack.main.SpackCommand(\"buildcache\")\ninstall = spack.main.SpackCommand(\"install\")\nenv = spack.main.SpackCommand(\"env\")\nadd = spack.main.SpackCommand(\"add\")\ngpg = spack.main.SpackCommand(\"gpg\")\nmirror = spack.main.SpackCommand(\"mirror\")\nuninstall = spack.main.SpackCommand(\"uninstall\")\n\npytestmark = pytest.mark.not_on_windows(\"does not run on windows\")\n\n\n@pytest.fixture()\ndef mock_get_specs(database, monkeypatch):\n specs = database.query_local()\n monkeypatch.setattr(spack.binary_distribution, \"update_cache_and_get_specs\", lambda: specs)\n\n\n@pytest.fixture()\ndef mock_get_specs_multiarch(database, monkeypatch):\n specs = [spec.copy() for spec in database.query_local()]\n\n # make one spec that is NOT the test architecture\n for spec in specs:\n if spec.name == \"mpileaks\":\n spec.architecture = spack.spec.ArchSpec(\"linux-rhel7-x86_64\")\n break\n\n monkeypatch.setattr(spack.binary_distribution, \"update_cache_and_get_specs\", lambda: specs)\n\n\n@pytest.mark.db\n@pytest.mark.regression(\"13757\")\ndef test_buildcache_list_duplicates(mock_get_specs):\n output = buildcache(\"list\", \"mpileaks\", \"@2.3\")\n\n assert output.count(\"mpileaks\") == 3\n\n\n@pytest.mark.db\n@pytest.mark.regression(\"17827\")\ndef test_buildcache_list_allarch(database, mock_get_specs_multiarch):\n output = buildcache(\"list\", \"--allarch\")\n assert output.count(\"mpileaks\") == 3\n\n output = buildcache(\"list\")\n assert output.count(\"mpileaks\") == 2\n\n\ndef tests_buildcache_create_env(\n install_mockery, mock_fetch, tmp_path: pathlib.Path, mutable_mock_env_path\n):\n \"\"\" \"Ensure that buildcache create creates output files from env\"\"\"\n pkg = \"trivial-install-test-package\"\n\n env(\"create\", \"--without-view\", \"test\")\n with ev.read(\"test\"):\n add(pkg)\n install()\n\n buildcache(\"push\", \"--unsigned\", str(tmp_path))\n\n spec = spack.concretize.concretize_one(pkg)\n\n mirror_url = tmp_path.as_uri()\n\n cache_class = get_url_buildcache_class(\n layout_version=spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION\n )\n cache_entry = cache_class(mirror_url, spec, allow_unsigned=True)\n assert cache_entry.exists([BuildcacheComponent.SPEC, BuildcacheComponent.TARBALL])\n cache_entry.destroy()\n\n\ndef test_buildcache_create_fails_on_noargs(tmp_path: pathlib.Path):\n \"\"\"Ensure that buildcache create fails when given no args or\n environment.\"\"\"\n with pytest.raises(spack.main.SpackCommandError):\n buildcache(\"push\", \"--unsigned\", str(tmp_path))\n\n\n@pytest.mark.skipif(getuid() == 0, reason=\"user is root\")\ndef test_buildcache_create_fail_on_perm_denied(\n install_mockery, mock_fetch, tmp_path: pathlib.Path\n):\n \"\"\"Ensure that buildcache create fails on permission denied error.\"\"\"\n install(\"trivial-install-test-package\")\n\n tmp_path.chmod(0)\n with pytest.raises(OSError) as error:\n buildcache(\"push\", \"--unsigned\", str(tmp_path), \"trivial-install-test-package\")\n assert error.value.errno == errno.EACCES\n tmp_path.chmod(0o700)\n\n\ndef test_update_key_index(\n tmp_path: pathlib.Path,\n mutable_mock_env_path,\n install_mockery,\n mock_packages,\n mock_fetch,\n mock_stage,\n mock_gnupghome,\n):\n \"\"\"Test the update-index command with the --keys option\"\"\"\n working_dir = tmp_path / \"working_dir\"\n working_dir.mkdir()\n\n mirror_dir = working_dir / \"mirror\"\n mirror_url = mirror_dir.as_uri()\n\n mirror(\"add\", \"test-mirror\", mirror_url)\n\n gpg(\"create\", \"Test Signing Key\", \"nobody@nowhere.com\")\n\n s = spack.concretize.concretize_one(\"libdwarf\")\n\n # Install a package\n install(\"--fake\", s.name)\n\n # Put installed package in the buildcache, which, because we're signing\n # it, should result in the public key getting pushed to the buildcache\n # as well.\n buildcache(\"push\", str(mirror_dir), s.name)\n\n # Now make sure that when we pass the \"--keys\" argument to update-index\n # it causes the index to get update.\n buildcache(\"update-index\", \"--keys\", str(mirror_dir))\n\n key_dir_list = os.listdir(\n os.path.join(str(mirror_dir), spack.binary_distribution.buildcache_relative_keys_path())\n )\n\n uninstall(\"-y\", s.name)\n mirror(\"rm\", \"test-mirror\")\n\n assert \"keys.manifest.json\" in key_dir_list\n\n\ndef test_buildcache_autopush(tmp_path: pathlib.Path, install_mockery, mock_fetch):\n \"\"\"Test buildcache with autopush\"\"\"\n mirror_dir = tmp_path / \"mirror\"\n mirror_autopush_dir = tmp_path / \"mirror_autopush\"\n\n mirror(\"add\", \"--unsigned\", \"mirror\", mirror_dir.as_uri())\n mirror(\"add\", \"--autopush\", \"--unsigned\", \"mirror-autopush\", mirror_autopush_dir.as_uri())\n\n s = spack.concretize.concretize_one(\"libdwarf\")\n\n # Install and generate build cache index\n PackageInstaller([s.package], fake=True, explicit=True).install()\n\n manifest_file = URLBuildcacheEntry.get_manifest_filename(s)\n specs_dirs = os.path.join(\n *URLBuildcacheEntry.get_relative_path_components(BuildcacheComponent.SPEC), s.name\n )\n\n assert not (mirror_dir / specs_dirs / manifest_file).exists()\n assert (mirror_autopush_dir / specs_dirs / manifest_file).exists()\n\n\ndef test_buildcache_sync(\n mutable_mock_env_path,\n install_mockery,\n mock_packages,\n mock_fetch,\n mock_stage,\n tmp_path: pathlib.Path,\n):\n \"\"\"\n Make sure buildcache sync works in an environment-aware manner, ignoring\n any specs that may be in the mirror but not in the environment.\n \"\"\"\n working_dir = tmp_path / \"working_dir\"\n working_dir.mkdir()\n\n src_mirror_dir = working_dir / \"src_mirror\"\n src_mirror_url = src_mirror_dir.as_uri()\n\n dest_mirror_dir = working_dir / \"dest_mirror\"\n dest_mirror_url = dest_mirror_dir.as_uri()\n\n in_env_pkg = \"trivial-install-test-package\"\n out_env_pkg = \"libdwarf\"\n\n def verify_mirror_contents():\n dest_list = os.listdir(\n os.path.join(\n str(dest_mirror_dir), spack.binary_distribution.buildcache_relative_specs_path()\n )\n )\n\n found_pkg = False\n\n for p in dest_list:\n assert out_env_pkg not in p\n if in_env_pkg in p:\n found_pkg = True\n\n assert found_pkg, f\"Expected to find {in_env_pkg} in {dest_mirror_dir}\"\n\n # Install a package and put it in the buildcache\n s = spack.concretize.concretize_one(out_env_pkg)\n install(\"--fake\", s.name)\n buildcache(\"push\", \"-u\", \"-f\", src_mirror_url, s.name)\n\n env(\"create\", \"--without-view\", \"test\")\n with ev.read(\"test\"):\n add(in_env_pkg)\n install()\n buildcache(\"push\", \"-u\", \"-f\", src_mirror_url, in_env_pkg)\n\n # Now run the spack buildcache sync command with all the various options\n # for specifying mirrors\n\n # Use urls to specify mirrors\n buildcache(\"sync\", src_mirror_url, dest_mirror_url)\n\n verify_mirror_contents()\n shutil.rmtree(str(dest_mirror_dir))\n\n # Use local directory paths to specify fs locations\n buildcache(\"sync\", str(src_mirror_dir), str(dest_mirror_dir))\n\n verify_mirror_contents()\n shutil.rmtree(str(dest_mirror_dir))\n\n # Use mirror names to specify mirrors\n mirror(\"add\", \"src\", src_mirror_url)\n mirror(\"add\", \"dest\", dest_mirror_url)\n mirror(\"add\", \"ignored\", \"file:///dummy/io\")\n\n buildcache(\"sync\", \"src\", \"dest\")\n\n verify_mirror_contents()\n shutil.rmtree(str(dest_mirror_dir))\n\n cache_class = get_url_buildcache_class(\n layout_version=spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION\n )\n\n def manifest_insert(manifest, spec, dest_url):\n manifest[spec.dag_hash()] = {\n \"src\": cache_class.get_manifest_url(spec, src_mirror_url),\n \"dest\": cache_class.get_manifest_url(spec, dest_url),\n }\n\n manifest_file = str(tmp_path / \"manifest_dest.json\")\n with open(manifest_file, \"w\", encoding=\"utf-8\") as fd:\n test_env = ev.active_environment()\n assert test_env is not None\n\n manifest: Dict[str, Dict[str, str]] = {}\n for spec in test_env.specs_by_hash.values():\n manifest_insert(manifest, spec, dest_mirror_url)\n json.dump(manifest, fd)\n\n buildcache(\"sync\", \"--manifest-glob\", manifest_file)\n\n verify_mirror_contents()\n shutil.rmtree(str(dest_mirror_dir))\n\n manifest_file = str(tmp_path / \"manifest_bad_dest.json\")\n with open(manifest_file, \"w\", encoding=\"utf-8\") as fd:\n manifest_2: Dict[str, Dict[str, str]] = {}\n for spec in test_env.specs_by_hash.values():\n manifest_insert(manifest_2, spec, url_util.join(dest_mirror_url, \"invalid_path\"))\n json.dump(manifest_2, fd)\n\n # Trigger the warning\n output = buildcache(\"sync\", \"--manifest-glob\", manifest_file, \"dest\", \"ignored\")\n\n assert \"Ignoring unused argument: ignored\" in output\n\n verify_mirror_contents()\n shutil.rmtree(str(dest_mirror_dir))\n\n\ndef test_buildcache_create_install(\n mutable_mock_env_path,\n install_mockery,\n mock_packages,\n mock_fetch,\n mock_stage,\n tmp_path: pathlib.Path,\n):\n \"\"\" \"Ensure that buildcache create creates output files\"\"\"\n pkg = \"trivial-install-test-package\"\n install(pkg)\n\n buildcache(\"push\", \"--unsigned\", str(tmp_path), pkg)\n\n mirror_url = tmp_path.as_uri()\n\n spec = spack.concretize.concretize_one(pkg)\n cache_class = get_url_buildcache_class(\n layout_version=spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION\n )\n cache_entry = cache_class(mirror_url, spec, allow_unsigned=True)\n manifest_path = os.path.join(\n str(tmp_path),\n *cache_class.get_relative_path_components(BuildcacheComponent.SPEC),\n spec.name,\n cache_class.get_manifest_filename(spec),\n )\n\n assert os.path.exists(manifest_path)\n cache_entry.read_manifest()\n spec_blob_record = cache_entry.get_blob_record(BuildcacheComponent.SPEC)\n tarball_blob_record = cache_entry.get_blob_record(BuildcacheComponent.TARBALL)\n\n spec_blob_path = os.path.join(\n str(tmp_path), *cache_class.get_blob_path_components(spec_blob_record)\n )\n assert os.path.exists(spec_blob_path)\n\n tarball_blob_path = os.path.join(\n str(tmp_path), *cache_class.get_blob_path_components(tarball_blob_record)\n )\n assert os.path.exists(tarball_blob_path)\n\n cache_entry.destroy()\n\n\ndef _mock_uploader(tmp_path: pathlib.Path):\n class DontUpload(spack.binary_distribution.Uploader):\n def __init__(self):\n super().__init__(\n spack.mirrors.mirror.Mirror.from_local_path(str(tmp_path)), False, False\n )\n self.pushed = []\n\n def push(self, specs: List[spack.spec.Spec]):\n self.pushed.extend(s.name for s in specs)\n return [], []\n\n return DontUpload()\n\n\n@pytest.mark.parametrize(\n \"things_to_install,expected\",\n [\n (\n \"\",\n [\n \"dttop\",\n \"dtbuild1\",\n \"dtbuild2\",\n \"dtlink2\",\n \"dtrun2\",\n \"dtlink1\",\n \"dtlink3\",\n \"dtlink4\",\n \"dtrun1\",\n \"dtlink5\",\n \"dtrun3\",\n \"dtbuild3\",\n ],\n ),\n (\n \"dependencies\",\n [\n \"dtbuild1\",\n \"dtbuild2\",\n \"dtlink2\",\n \"dtrun2\",\n \"dtlink1\",\n \"dtlink3\",\n \"dtlink4\",\n \"dtrun1\",\n \"dtlink5\",\n \"dtrun3\",\n \"dtbuild3\",\n ],\n ),\n (\"package\", [\"dttop\"]),\n ],\n)\ndef test_correct_specs_are_pushed(\n things_to_install,\n expected,\n tmp_path: pathlib.Path,\n monkeypatch,\n default_mock_concretization,\n temporary_store,\n):\n spec = default_mock_concretization(\"dttop\")\n PackageInstaller([spec.package], explicit=True, fake=True).install()\n slash_hash = f\"/{spec.dag_hash()}\"\n\n uploader = _mock_uploader(tmp_path)\n\n monkeypatch.setattr(\n spack.binary_distribution, \"make_uploader\", lambda *args, **kwargs: uploader\n )\n\n buildcache_create_args = [\"create\", \"--unsigned\"]\n\n if things_to_install != \"\":\n buildcache_create_args.extend([\"--only\", things_to_install])\n\n buildcache_create_args.extend([str(tmp_path), slash_hash])\n\n buildcache(*buildcache_create_args)\n\n # Order is not guaranteed, so we can't just compare lists\n assert set(uploader.pushed) == set(expected)\n\n # Ensure no duplicates\n assert len(set(uploader.pushed)) == len(uploader.pushed)\n\n\n@pytest.mark.parametrize(\"signed\", [True, False])\ndef test_push_and_install_with_mirror_marked_unsigned_does_not_require_extra_flags(\n tmp_path: pathlib.Path, mutable_database, mock_gnupghome, signed\n):\n \"\"\"Tests whether marking a mirror as unsigned makes it possible to push and install to/from\n it without requiring extra flags on the command line (and no signing keys configured).\"\"\"\n\n # Create a named mirror with signed set to True or False\n add_flag = \"--signed\" if signed else \"--unsigned\"\n mirror(\"add\", add_flag, \"my-mirror\", str(tmp_path))\n spec = mutable_database.query_local(\"libelf\", installed=True)[0]\n\n # Push\n if signed:\n # Need to pass \"--unsigned\" to override the mirror's default\n args = [\"push\", \"--update-index\", \"--unsigned\", \"my-mirror\", f\"/{spec.dag_hash()}\"]\n else:\n # No need to pass \"--unsigned\" if the mirror is unsigned\n args = [\"push\", \"--update-index\", \"my-mirror\", f\"/{spec.dag_hash()}\"]\n\n buildcache(*args)\n\n spec.package.do_uninstall(force=True)\n PackageInstaller(\n [spec.package],\n explicit=True,\n root_policy=\"cache_only\",\n dependencies_policy=\"cache_only\",\n unsigned=True if signed else None,\n ).install()\n\n\ndef test_skip_no_redistribute(mock_packages, config):\n specs = list(spack.concretize.concretize_one(\"no-redistribute-dependent\").traverse())\n filtered = spack.cmd.buildcache._skip_no_redistribute_for_public(specs)\n assert not any(s.name == \"no-redistribute\" for s in filtered)\n assert any(s.name == \"no-redistribute-dependent\" for s in filtered)\n\n\ndef test_best_effort_vs_fail_fast_when_dep_not_installed(tmp_path: pathlib.Path, mutable_database):\n \"\"\"When --fail-fast is passed, the push command should fail if it immediately finds an\n uninstalled dependency. Otherwise, failure to push one dependency shouldn't prevent the\n others from being pushed.\"\"\"\n\n mirror(\"add\", \"--unsigned\", \"my-mirror\", str(tmp_path))\n\n # Uninstall mpich so that its dependent mpileaks can't be pushed\n for s in mutable_database.query_local(\"mpich\"):\n s.package.do_uninstall(force=True)\n\n with pytest.raises(spack.cmd.buildcache.PackagesAreNotInstalledError, match=\"mpich\"):\n buildcache(\"push\", \"--update-index\", \"--fail-fast\", \"my-mirror\", \"mpileaks^mpich\")\n\n # nothing should be pushed due to --fail-fast.\n assert not os.listdir(tmp_path)\n assert not spack.binary_distribution.update_cache_and_get_specs()\n\n with pytest.raises(spack.cmd.buildcache.PackageNotInstalledError):\n buildcache(\"push\", \"--update-index\", \"my-mirror\", \"mpileaks^mpich\")\n\n specs = spack.binary_distribution.update_cache_and_get_specs()\n\n # everything but mpich should be pushed\n mpileaks = mutable_database.query_local(\"mpileaks^mpich\")[0]\n assert set(specs) == {s for s in mpileaks.traverse() if s.name != \"mpich\"}\n\n\ndef test_push_without_build_deps(\n tmp_path: pathlib.Path, temporary_store, mock_packages, mutable_config\n):\n \"\"\"Spack should not error when build deps are uninstalled and --without-build-dependenies is\n passed.\"\"\"\n\n mirror(\"add\", \"--unsigned\", \"my-mirror\", str(tmp_path))\n\n s = spack.concretize.concretize_one(\"dtrun3\")\n PackageInstaller([s.package], explicit=True, fake=True).install()\n s[\"dtbuild3\"].package.do_uninstall()\n\n # fails when build deps are required\n with pytest.raises(spack.error.SpackError, match=\"package not installed\"):\n buildcache(\n \"push\", \"--update-index\", \"--with-build-dependencies\", \"my-mirror\", f\"/{s.dag_hash()}\"\n )\n\n # succeeds when build deps are not required\n buildcache(\n \"push\", \"--update-index\", \"--without-build-dependencies\", \"my-mirror\", f\"/{s.dag_hash()}\"\n )\n assert spack.binary_distribution.update_cache_and_get_specs() == [s]\n\n\n@pytest.fixture(scope=\"function\")\ndef v2_buildcache_layout(tmp_path: pathlib.Path):\n def _layout(signedness: str = \"signed\"):\n source_path = str(pathlib.Path(test_path) / \"data\" / \"mirrors\" / \"v2_layout\" / signedness)\n test_mirror_path = tmp_path / \"mirror\"\n copy_tree(source_path, test_mirror_path)\n return test_mirror_path\n\n return _layout\n\n\ndef test_check_mirror_for_layout(v2_buildcache_layout, mutable_config, capfd):\n \"\"\"Check printed warning in the presence of v2 layout binary mirrors\"\"\"\n test_mirror_path = v2_buildcache_layout(\"unsigned\")\n\n check_mirror_for_layout(spack.mirrors.mirror.Mirror.from_local_path(str(test_mirror_path)))\n err = str(capfd.readouterr()[1])\n assert all([word in err for word in [\"Warning\", \"missing\", \"layout\"]])\n\n\ndef test_url_buildcache_entry_v2_exists(\n v2_buildcache_layout, mock_packages, mutable_config, do_not_check_runtimes_on_reuse\n):\n \"\"\"Test existence check for v2 buildcache entries\"\"\"\n test_mirror_path = v2_buildcache_layout(\"unsigned\")\n mirror_url = pathlib.Path(test_mirror_path).as_uri()\n mirror(\"add\", \"v2mirror\", mirror_url)\n\n output = buildcache(\"list\", \"-a\", \"-l\")\n\n assert \"Fetching an index from a v2 binary mirror layout\" in output\n assert \"deprecated\" in output\n\n v2_cache_class = URLBuildcacheEntryV2\n\n # If you don't give it a spec, it returns False\n build_cache = v2_cache_class(mirror_url)\n assert not build_cache.exists([BuildcacheComponent.SPEC, BuildcacheComponent.TARBALL])\n\n spec = spack.concretize.concretize_one(\"libdwarf\")\n\n # In v2 we have to ask for both, because we need to have the spec to have the tarball\n build_cache = v2_cache_class(mirror_url, spec, allow_unsigned=True)\n assert not build_cache.exists([BuildcacheComponent.TARBALL])\n assert not build_cache.exists([BuildcacheComponent.SPEC])\n # But if we do ask for both, they should be there in this case\n assert build_cache.exists([BuildcacheComponent.SPEC, BuildcacheComponent.TARBALL])\n\n spec_path = build_cache._get_spec_url(spec, mirror_url, ext=\".spec.json\")[7:]\n tarball_path = build_cache._get_tarball_url(spec, mirror_url)[7:]\n\n os.remove(tarball_path)\n build_cache = v2_cache_class(mirror_url, spec, allow_unsigned=True)\n assert not build_cache.exists([BuildcacheComponent.SPEC, BuildcacheComponent.TARBALL])\n\n os.remove(spec_path)\n build_cache = v2_cache_class(mirror_url, spec, allow_unsigned=True)\n assert not build_cache.exists([BuildcacheComponent.SPEC, BuildcacheComponent.TARBALL])\n\n\n@pytest.mark.parametrize(\"signing\", [\"unsigned\", \"signed\"])\ndef test_install_v2_layout(\n signing,\n v2_buildcache_layout,\n mock_packages,\n mutable_config,\n mutable_mock_env_path,\n install_mockery,\n mock_gnupghome,\n do_not_check_runtimes_on_reuse,\n):\n \"\"\"Ensure we can still install from signed and unsigned v2 buildcache\"\"\"\n test_mirror_path = v2_buildcache_layout(signing)\n mirror(\"add\", \"my-mirror\", str(test_mirror_path))\n\n # Trust original signing key (no-op if this is the unsigned pass)\n buildcache(\"keys\", \"--install\", \"--trust\")\n\n output = install(\"--fake\", \"--no-check-signature\", \"libdwarf\")\n\n assert \"Extracting libelf\" in output\n assert \"libelf: Successfully installed\" in output\n assert \"Extracting libdwarf\" in output\n assert \"libdwarf: Successfully installed\" in output\n assert \"Installing a spec from a v2 binary mirror layout\" in output\n assert \"Fetching an index from a v2 binary mirror layout\" in output\n assert \"deprecated\" in output\n\n\ndef test_basic_migrate_unsigned(v2_buildcache_layout, mutable_config):\n \"\"\"Make sure first unsigned migration results in usable buildcache,\n leaving the previous layout in place. Also test that a subsequent one\n doesn't need to migrate anything, and that using --delete-existing\n removes the previous layout\"\"\"\n\n test_mirror_path = v2_buildcache_layout(\"unsigned\")\n mirror(\"add\", \"my-mirror\", str(test_mirror_path))\n\n output = buildcache(\"migrate\", \"--unsigned\", \"my-mirror\")\n\n # The output indicates both specs were migrated\n assert output.count(\"Successfully migrated\") == 6\n\n build_cache_path = str(test_mirror_path / \"build_cache\")\n\n # Without \"--delete-existing\" and \"--yes-to-all\", migration leaves the\n # previous layout in place\n assert os.path.exists(build_cache_path)\n assert os.path.isdir(build_cache_path)\n\n # Now list the specs available under the new layout\n output = buildcache(\"list\", \"--allarch\")\n\n assert \"libdwarf\" in output and \"libelf\" in output\n\n output = buildcache(\"migrate\", \"--unsigned\", \"--delete-existing\", \"--yes-to-all\", \"my-mirror\")\n\n # A second migration of the same mirror indicates neither spec\n # needs to be migrated\n assert output.count(\"No need to migrate\") == 6\n\n # When we provide \"--delete-existing\" and \"--yes-to-all\", migration\n # removes the old layout\n assert not os.path.exists(build_cache_path)\n\n\ndef test_basic_migrate_signed(v2_buildcache_layout, mock_gnupghome, mutable_config):\n \"\"\"Test a signed migration requires a signing key, requires the public\n key originally used to sign the pkgs, fails and prints reasonable messages\n if those requirements are unmet, and eventually succeeds when they are met.\"\"\"\n test_mirror_path = v2_buildcache_layout(\"signed\")\n mirror(\"add\", \"my-mirror\", str(test_mirror_path))\n\n with pytest.raises(migrate.MigrationException) as error:\n buildcache(\"migrate\", \"my-mirror\")\n\n # Without a signing key spack fails and explains why\n assert error.value.message == \"Signed migration requires exactly one secret key in keychain\"\n\n # Create a signing key and trust the key used to sign the pkgs originally\n gpg(\"create\", \"New Test Signing Key\", \"noone@nowhere.org\")\n\n output = buildcache(\"migrate\", \"my-mirror\")\n\n # Without trusting the original signing key, spack fails with an explanation\n assert \"Failed to verify signature of libelf\" in output\n assert \"Failed to verify signature of libdwarf\" in output\n assert \"did you mean to perform an unsigned migration\" in output\n\n # Trust original signing key (since it's in the original layout location,\n # this is where the monkeypatched attribute is used)\n output = buildcache(\"keys\", \"--install\", \"--trust\")\n\n output = buildcache(\"migrate\", \"my-mirror\")\n\n # Once we have the proper keys, migration should succeed\n assert \"Successfully migrated libelf\" in output\n assert \"Successfully migrated libelf\" in output\n\n # Now list the specs available under the new layout\n output = buildcache(\"list\", \"--allarch\")\n\n assert \"libdwarf\" in output and \"libelf\" in output\n\n\ndef test_unsigned_migrate_of_signed_mirror(v2_buildcache_layout, mutable_config):\n \"\"\"Test spack can do an unsigned migration of a signed buildcache by\n ignoring signatures and skipping re-signing.\"\"\"\n\n test_mirror_path = v2_buildcache_layout(\"signed\")\n mirror(\"add\", \"my-mirror\", str(test_mirror_path))\n\n output = buildcache(\"migrate\", \"--unsigned\", \"--delete-existing\", \"--yes-to-all\", \"my-mirror\")\n\n # Now list the specs available under the new layout\n output = buildcache(\"list\", \"--allarch\")\n\n assert \"libdwarf\" in output and \"libelf\" in output\n\n # We should find two spec manifest files, one for each spec\n file_list = find(test_mirror_path, \"*.spec.manifest.json\")\n assert len(file_list) == 6\n assert any([\"libdwarf\" in file for file in file_list])\n assert any([\"libelf\" in file for file in file_list])\n\n # The two spec manifest files should be unsigned\n for file_path in file_list:\n with open(file_path, \"r\", encoding=\"utf-8\") as fd:\n assert json.load(fd)\n\n\ndef test_migrate_requires_index(v2_buildcache_layout, mutable_config):\n \"\"\"Test spack fails with a reasonable error message when mirror does\n not have an index\"\"\"\n\n test_mirror_path = v2_buildcache_layout(\"unsigned\")\n v2_index_path = test_mirror_path / \"build_cache\" / \"index.json\"\n v2_index_hash_path = test_mirror_path / \"build_cache\" / \"index.json.hash\"\n os.remove(str(v2_index_path))\n os.remove(str(v2_index_hash_path))\n\n mirror(\"add\", \"my-mirror\", str(test_mirror_path))\n\n with pytest.raises(migrate.MigrationException) as error:\n buildcache(\"migrate\", \"--unsigned\", \"my-mirror\")\n\n # If the buildcache has no index, spack fails and explains why\n assert error.value.message == \"Buildcache migration requires a buildcache index\"\n\n\n@pytest.mark.parametrize(\"dry_run\", [False, True])\ndef test_buildcache_prune_no_orphans(tmp_path, mutable_database, mock_gnupghome, dry_run):\n mirror(\"add\", \"--unsigned\", \"my-mirror\", str(tmp_path))\n spec = mutable_database.query_local(\"libelf\", installed=True)[0]\n buildcache(\"push\", \"--update-index\", \"my-mirror\", f\"/{spec.dag_hash()}\")\n\n cmd_args = [\"prune\", \"my-mirror\"]\n if dry_run:\n cmd_args.append(\"--dry-run\")\n output = buildcache(*cmd_args)\n\n assert \"0 orphaned objects from mirror:\" in output\n\n\n@pytest.mark.parametrize(\"dry_run\", [False, True])\ndef test_buildcache_prune_orphaned_blobs(tmp_path, mutable_database, mock_gnupghome, dry_run):\n # Create a mirror and push a package to it\n mirror_directory = str(tmp_path)\n\n mirror(\"add\", \"--unsigned\", \"my-mirror\", mirror_directory)\n spec = mutable_database.query_local(\"libelf\", installed=True)[0]\n buildcache(\"push\", \"--update-index\", \"my-mirror\", f\"/{spec.dag_hash()}\")\n\n cache_entry = URLBuildcacheEntry(\n mirror_url=f\"file://{mirror_directory}\", spec=spec, allow_unsigned=True\n )\n\n blob_urls = [\n URLBuildcacheEntry.get_blob_url(mirror_url=f\"file://{mirror_directory}\", record=blob)\n for blob in cache_entry.read_manifest().data\n ]\n\n # Remove the manifest from the cache, orphaning the blobs\n manifest_url = URLBuildcacheEntry.get_manifest_url(\n spec, mirror_url=f\"file://{mirror_directory}\"\n )\n web_util.remove_url(manifest_url)\n\n # Ensure the blobs are still there before pruning\n assert all(web_util.url_exists(blob_url) for blob_url in blob_urls)\n\n cmd_args = [\"prune\", \"my-mirror\"]\n if dry_run:\n cmd_args.append(\"--dry-run\")\n output = buildcache(*cmd_args)\n\n # Ensure the blobs are gone after pruning (or not if dry_run is True)\n assert all(web_util.url_exists(blob_url) == dry_run for blob_url in blob_urls)\n\n assert \"Found 2 blob(s) with no manifest\" in output\n\n cache_entry.destroy()\n\n\n@pytest.mark.parametrize(\"dry_run\", [False, True])\ndef test_buildcache_prune_orphaned_manifest(tmp_path, mutable_database, mock_gnupghome, dry_run):\n # Create a mirror and push a package to it\n mirror_directory = str(tmp_path)\n\n mirror(\"add\", \"--unsigned\", \"my-mirror\", mirror_directory)\n spec = mutable_database.query_local(\"libelf\", installed=True)[0]\n buildcache(\"push\", \"--update-index\", \"my-mirror\", f\"/{spec.dag_hash()}\")\n\n # Create a cache entry and read the manifest, which should succeed\n # as we haven't pruned anything yet\n cache_entry = URLBuildcacheEntry(\n mirror_url=f\"file://{mirror_directory}\", spec=spec, allow_unsigned=True\n )\n manifest = cache_entry.read_manifest()\n\n manifest_url = f\"file://{cache_entry.get_manifest_url(spec=spec, mirror_url=mirror_directory)}\"\n\n # Remove the blobs from the cache, orphaning the manifest\n for blob_file in manifest.data:\n blob_url = cache_entry.get_blob_url(mirror_url=mirror_directory, record=blob_file)\n web_util.remove_url(url=f\"file://{blob_url}\")\n\n cmd_args = [\"prune\", \"my-mirror\"]\n if dry_run:\n cmd_args.append(\"--dry-run\")\n output = buildcache(*cmd_args)\n\n # Ensure the manifest is gone after pruning (or not if dry_run is True)\n assert web_util.url_exists(manifest_url) == dry_run\n\n assert \"Found 1 manifest(s) that are missing blobs\" in output\n\n cache_entry.destroy()\n\n\n@pytest.mark.parametrize(\"dry_run\", [False, True])\ndef test_buildcache_prune_direct_with_keeplist(\n tmp_path: pathlib.Path, mutable_database, mock_gnupghome, dry_run\n):\n \"\"\"Test direct pruning functionality with a keeplist file\"\"\"\n mirror_directory = str(tmp_path)\n mirror(\"add\", \"--unsigned\", \"my-mirror\", mirror_directory)\n\n # Install and push multiple packages\n specs = mutable_database.query_local(\"libelf\", installed=True)\n spec1 = specs[0]\n\n cache_entry = URLBuildcacheEntry(\n mirror_url=f\"file://{mirror_directory}\", spec=spec1, allow_unsigned=True\n )\n manifest_url = cache_entry.get_manifest_url(spec1, f\"file://{mirror_directory}\")\n\n # Push the first spec (package only, no dependencies)\n buildcache(\"push\", \"--only\", \"package\", \"--update-index\", \"my-mirror\", f\"/{spec1.dag_hash()}\")\n\n # Create a keeplist file that includes only spec1\n keeplist_file = tmp_path / \"keeplist.txt\"\n keeplist_file.write_text(f\"{spec1.dag_hash()}\\n\")\n\n # Run direct pruning\n cmd_args = [\"prune\", \"my-mirror\", \"--keeplist\", str(keeplist_file)]\n if dry_run:\n cmd_args.append(\"--dry-run\")\n output = buildcache(*cmd_args)\n\n # Since all packages are in the keeplist, nothing should be pruned\n assert web_util.url_exists(manifest_url)\n assert \"No specs to prune - all specs are in the keeplist\" in output\n\n\n@pytest.mark.parametrize(\"dry_run\", [False, True])\ndef test_buildcache_prune_direct_removes_unlisted(\n tmp_path: pathlib.Path, mutable_database, mock_gnupghome, dry_run\n):\n \"\"\"Test that direct pruning removes specs not in the keeplist\"\"\"\n mirror_directory = str(tmp_path)\n mirror(\"add\", \"--unsigned\", \"my-mirror\", mirror_directory)\n\n # Install and push a package (package only, no dependencies)\n specs = mutable_database.query_local(\"libelf\", installed=True)\n spec1 = specs[0]\n buildcache(\"push\", \"--only\", \"package\", \"--update-index\", \"my-mirror\", f\"/{spec1.dag_hash()}\")\n\n # Create a keeplist file that excludes the pushed package\n keeplist_file = tmp_path / \"keeplist.txt\"\n keeplist_file.write_text(\"0\" * 32)\n\n cache_entry = URLBuildcacheEntry(\n mirror_url=f\"file://{mirror_directory}\", spec=spec1, allow_unsigned=True\n )\n manifest_url = cache_entry.get_manifest_url(spec1, f\"file://{mirror_directory}\")\n\n assert web_util.url_exists(manifest_url)\n\n # Run direct pruning\n cmd_args = [\"prune\", \"my-mirror\", \"--keeplist\", str(keeplist_file)]\n if dry_run:\n cmd_args.append(\"--dry-run\")\n buildcache(*cmd_args)\n\n assert web_util.url_exists(manifest_url) == dry_run\n\n\ndef test_buildcache_prune_direct_empty_keeplist_fails(\n tmp_path: pathlib.Path, mutable_database, mock_gnupghome\n):\n \"\"\"Test that direct pruning fails with an empty keeplist file\"\"\"\n mirror_directory = str(tmp_path)\n mirror(\"add\", \"--unsigned\", \"my-mirror\", mirror_directory)\n\n # Create empty keeplist file\n keeplist_file = tmp_path / \"empty_keeplist.txt\"\n keeplist_file.write_text(\"\")\n\n # Should fail with empty keeplist\n with pytest.raises(spack.buildcache_prune.BuildcachePruningException):\n buildcache(\"prune\", \"my-mirror\", \"--keeplist\", str(keeplist_file))\n\n\n@pytest.mark.parametrize(\"dry_run\", [False, True])\ndef test_buildcache_prune_with_invalid_keep_hash(\n tmp_path: pathlib.Path, mutable_database, mock_gnupghome, dry_run: bool\n):\n mirror_directory = str(tmp_path)\n mirror(\"add\", \"--unsigned\", \"my-mirror\", mirror_directory)\n\n # Create a keeplist file that includes an invalid hash\n keeplist_file = tmp_path / \"keeplist.txt\"\n keeplist_file.write_text(\"this_is_an_invalid_hash\")\n\n cmd_args = [\"prune\", \"my-mirror\", \"--keeplist\", str(keeplist_file)]\n if dry_run:\n cmd_args.append(\"--dry-run\")\n\n with pytest.raises(spack.buildcache_prune.BuildcachePruningException):\n buildcache(*cmd_args)\n\n\ndef test_buildcache_prune_new_specs_race_condition(\n tmp_path: pathlib.Path, mutable_database, mock_gnupghome, monkeypatch: pytest.MonkeyPatch\n):\n \"\"\"Test that specs uploaded after pruning begins are protected\"\"\"\n mirror_directory = str(tmp_path)\n mirror(\"add\", \"--unsigned\", \"my-mirror\", mirror_directory)\n\n spec = mutable_database.query_local(\"libelf\", installed=True)[0]\n\n buildcache(\"push\", \"--only\", \"package\", \"--update-index\", \"my-mirror\", f\"/{spec.dag_hash()}\")\n\n cache_entry = URLBuildcacheEntry(\n mirror_url=f\"file://{mirror_directory}\", spec=spec, allow_unsigned=True\n )\n manifest_url = cache_entry.get_manifest_url(spec, f\"file://{mirror_directory}\")\n\n def mock_stat_url(url: str):\n \"\"\"\n Mock the stat_url function for testing.\n\n For the specific spec created in this test, fake its mtime so that it appears to\n have been created after the pruning started.\n \"\"\"\n if url == manifest_url:\n return 1, datetime.now().timestamp() + timedelta(minutes=10).total_seconds()\n parsed_url = urllib.parse.urlparse(url)\n stat_result = pathlib.Path(parsed_url.path).stat()\n return stat_result.st_size, stat_result.st_mtime\n\n monkeypatch.setattr(web_util, \"stat_url\", mock_stat_url)\n\n keeplist_file = tmp_path / \"keeplist.txt\"\n keeplist_file.write_text(\"0\" * 32)\n\n # Run end-to-end buildcache prune - this should not delete `libelf`, despite it\n # not being in the keeplist, because its mtime is after the pruning started\n assert web_util.url_exists(manifest_url)\n buildcache(\"prune\", \"my-mirror\", \"--keeplist\", str(keeplist_file))\n assert web_util.url_exists(manifest_url)\n\n\ndef create_env_from_concrete_spec(spec: spack.spec.Spec):\n \"\"\"Build cache index view source is current active environment\"\"\"\n # Create a unique environment for this spec only\n env_name = f\"specenv-{spec.dag_hash()}\"\n if not ev.exists(env_name):\n env(\"create\", \"--without-view\", env_name)\n\n e = ev.environment_from_name_or_dir(env_name)\n with e:\n add(f\"{spec.name}/{spec.dag_hash()}\")\n # This should handle updating the environment to mark all packages as installed\n install()\n return e\n\n\ndef args_for_active_env(spec: spack.spec.Spec):\n \"\"\"Build cache index view source is an active environment\"\"\"\n env = create_env_from_concrete_spec(spec)\n return [env, []]\n\n\ndef args_for_env_by_path(spec: spack.spec.Spec):\n \"\"\"Build cache index view source is an environment path\"\"\"\n env = create_env_from_concrete_spec(spec)\n return [nullcontext(), [env.path]]\n\n\ndef args_for_env_by_name(spec: spack.spec.Spec):\n \"\"\"Build cache index view source is a managed environment name\"\"\"\n env = create_env_from_concrete_spec(spec)\n return [nullcontext(), [env.name]]\n\n\ndef read_specs_in_index(mirror_directory, view):\n \"\"\"Read specs hashes from a build cache index (ie. a database file)\n\n This assumes the layout of the database holds the spec hashes under:\n database -> installs -> hashes...\n \"\"\"\n mirror_metadata = spack.binary_distribution.MirrorMetadata(\n f\"file://{mirror_directory}\", spack.mirrors.mirror.SUPPORTED_LAYOUT_VERSIONS[0], view\n )\n fetcher = spack.binary_distribution.DefaultIndexFetcher(mirror_metadata, None)\n result = fetcher.conditional_fetch()\n db_dict = json.loads(result.data)\n return set([h for h in db_dict[\"database\"][\"installs\"]])\n\n\ndef test_buildcache_create_view_failure(tmp_path, mutable_config, mutable_mock_env_path):\n mirror_directory = str(tmp_path)\n mirror(\"add\", \"--unsigned\", \"my-mirror\", mirror_directory)\n\n # No spec sources should raise an exception\n with pytest.raises(spack.main.SpackCommandError):\n command_args = [\"update-index\", \"--name\", \"test_view\", \"my-mirror\"]\n buildcache(*command_args)\n\n # spec sources should raise an exception\n expect = \"no such environment 'DEADBEEF'\"\n with pytest.raises(spack.error.SpackError, match=expect):\n command_args = [\"update-index\", \"--name\", \"test_view\", \"my-mirror\", \"DEADBEEF\"]\n buildcache(*command_args)\n\n\ndef test_buildcache_create_view_empty(\n tmp_path, mutable_config, mutable_database, mutable_mock_env_path\n):\n mirror_directory = str(tmp_path)\n mirror(\"add\", \"--unsigned\", \"my-mirror\", mirror_directory)\n\n # Push a spec to the cache\n mpileaks_specs = mutable_database.query_local(\"mpileaks\")\n buildcache(\"push\", \"my-mirror\", mpileaks_specs[0].format(\"{/hash}\"))\n\n # Make sure the view doesn't exist yet\n with pytest.raises(spack.binary_distribution.FetchIndexError):\n hashes_in_view = read_specs_in_index(mirror_directory, \"test_view\")\n\n # Write a minimal lockfile (this is not validated/read by an environment)\n empty_manifest = tmp_path / \"emptylock\" / \"spack.yaml\"\n empty_manifest.parent.mkdir(exist_ok=False)\n empty_manifest.write_text(\"spack: {}\", encoding=\"utf-8\")\n empty_lockfile = tmp_path / \"emptylock\" / \"spack.lock\"\n empty_lockfile.write_text(\n '{\"_meta\": {\"lockfile-version\": 1}, \"roots\": [], \"concrete_specs\": {}}', encoding=\"utf-8\"\n )\n # Create a view with no specs\n command_args = [\n \"update-index\",\n \"--force\",\n \"--name\",\n \"test_view\",\n \"my-mirror\",\n str(empty_lockfile.parent),\n ]\n out = buildcache(*command_args)\n assert \"No specs found for view, creating an empty index\" in out\n hashes_in_view = read_specs_in_index(mirror_directory, \"test_view\")\n # Assert there are no hashes in the view\n assert not hashes_in_view\n\n\n@pytest.mark.parametrize(\n \"source_args\", (args_for_active_env, args_for_env_by_path, args_for_env_by_name)\n)\ndef test_buildcache_create_view(\n tmp_path, mutable_config, mutable_database, mutable_mock_env_path, source_args\n):\n mirror_directory = str(tmp_path)\n mirror(\"add\", \"--unsigned\", \"my-mirror\", mirror_directory)\n\n # Push all of the mpileaks packages to the cache\n mpileaks_specs = mutable_database.query_local(\"mpileaks\")\n for s in mpileaks_specs:\n buildcache(\"push\", \"my-mirror\", s.format(\"{/hash}\"))\n\n # Grab all of the hashes for each mpileaks spec\n mpileaks_0_hashes = set([s.dag_hash() for s in mpileaks_specs[0].traverse()])\n\n # Make sure the view doesn't exist yet\n with pytest.raises(spack.binary_distribution.FetchIndexError):\n hashes_in_view = read_specs_in_index(mirror_directory, \"test_view\")\n\n # Create a view using a parametrized source that contains one of the mpileaks\n context, extra_args = source_args(mpileaks_specs[0])\n with context:\n command_args = [\"update-index\", \"--name\", \"test_view\", \"my-mirror\"] + extra_args\n buildcache(*command_args)\n\n hashes_in_view = read_specs_in_index(mirror_directory, \"test_view\")\n # Assert all of the hashes for mpileaks_0_hashes exist in the view, and no other hashes\n assert hashes_in_view == mpileaks_0_hashes\n\n # Test fail to overwrite without force\n expect = \"Index already exists. To overwrite or update pass --force or --append respectively\"\n with pytest.raises(spack.error.SpackError, match=expect):\n command_args = [\n \"update-index\",\n \"--name\",\n \"test_view\",\n \"my-mirror\",\n mpileaks_specs[2].format(\"{/hash}\"),\n ]\n buildcache(*command_args)\n\n hashes_in_view = read_specs_in_index(mirror_directory, \"test_view\")\n # Assert all of the hashes for mpileaks_0_hashes exist in the view, and no other hashes\n assert hashes_in_view == mpileaks_0_hashes\n\n\n@pytest.mark.parametrize(\n \"source_args\", (args_for_active_env, args_for_env_by_path, args_for_env_by_name)\n)\ndef test_buildcache_create_view_append(\n tmp_path, mutable_config, mutable_database, mutable_mock_env_path, source_args\n):\n mirror_directory = str(tmp_path)\n mirror(\"add\", \"--unsigned\", \"my-mirror\", mirror_directory)\n\n # Push all of the mpileaks packages to the cache\n mpileaks_specs = mutable_database.query_local(\"mpileaks\")\n for s in mpileaks_specs:\n buildcache(\"push\", \"my-mirror\", s.format(\"{/hash}\"))\n\n # Grab all of the hashes for each mpileaks spec\n mpileaks_0_hashes = set([s.dag_hash() for s in mpileaks_specs[0].traverse()])\n mpileaks_1_hashes = set([s.dag_hash() for s in mpileaks_specs[1].traverse()])\n\n # Make sure the view doesn't exist yet\n with pytest.raises(spack.binary_distribution.FetchIndexError):\n hashes_in_view = read_specs_in_index(mirror_directory, \"test_view\")\n\n # Test append to empty index view\n context, extra_args = source_args(mpileaks_specs[0])\n with context:\n command_args = [\n \"update-index\",\n \"-y\",\n \"--append\",\n \"--name\",\n \"test_view\",\n \"my-mirror\",\n ] + extra_args\n buildcache(*command_args)\n\n hashes_in_view = read_specs_in_index(mirror_directory, \"test_view\")\n # Assert all of the hashes for mpileaks_0_hashes exist in the view, and no other hashes\n assert hashes_in_view == mpileaks_0_hashes\n\n # Test append to existing index view\n context, extra_args = source_args(mpileaks_specs[1])\n with context:\n command_args = [\n \"update-index\",\n \"-y\",\n \"--append\",\n \"--name\",\n \"test_view\",\n \"my-mirror\",\n ] + extra_args\n buildcache(*command_args)\n\n hashes_in_view = read_specs_in_index(mirror_directory, \"test_view\")\n # Assert all of the hashes for mpileaks_0_hashes and mpileaks_1_hashes exist in the view,\n # and no other hashes\n assert hashes_in_view == (mpileaks_0_hashes | mpileaks_1_hashes)\n\n\n@pytest.mark.parametrize(\n \"source_args\", (args_for_active_env, args_for_env_by_path, args_for_env_by_name)\n)\ndef test_buildcache_create_view_overwrite(\n tmp_path, mutable_config, mutable_database, mutable_mock_env_path, source_args\n):\n mirror_directory = str(tmp_path)\n mirror(\"add\", \"--unsigned\", \"my-mirror\", mirror_directory)\n\n # Push all of the mpileaks packages to the cache\n mpileaks_specs = mutable_database.query_local(\"mpileaks\")\n for s in mpileaks_specs:\n buildcache(\"push\", \"my-mirror\", s.format(\"{/hash}\"))\n\n # Grab all of the hashes for each mpileaks spec\n mpileaks_0_hashes = set([s.dag_hash() for s in mpileaks_specs[0].traverse()])\n mpileaks_1_hashes = set([s.dag_hash() for s in mpileaks_specs[1].traverse()])\n\n # Make sure the view doesn't exist yet\n with pytest.raises(spack.binary_distribution.FetchIndexError):\n hashes_in_view = read_specs_in_index(mirror_directory, \"test_view\")\n\n # Create a view using a parametrized source that contains one of the mpileaks\n context, extra_args = source_args(mpileaks_specs[0])\n with context:\n command_args = [\"update-index\", \"--name\", \"test_view\", \"my-mirror\"] + extra_args\n buildcache(*command_args)\n\n hashes_in_view = read_specs_in_index(mirror_directory, \"test_view\")\n # Assert all of the hashes for mpileaks_0_hashes exist in the view, and no other hashes\n assert hashes_in_view == mpileaks_0_hashes\n\n # Override a view with force\n context, extra_args = source_args(mpileaks_specs[1])\n with context:\n command_args = [\"update-index\", \"--force\", \"--name\", \"test_view\", \"my-mirror\"] + extra_args\n buildcache(*command_args)\n\n hashes_in_view = read_specs_in_index(mirror_directory, \"test_view\")\n # Assert all of the hashes for mpileaks_1_hashes exist in the view, and no other hashes\n assert hashes_in_view == mpileaks_1_hashes\n\n\ndef test_buildcache_create_view_non_active_env(\n tmp_path, mutable_config, mutable_database, mutable_mock_env_path\n):\n mirror_directory = str(tmp_path)\n mirror(\"add\", \"--unsigned\", \"my-mirror\", mirror_directory)\n\n # Push all of the mpileaks packages to the cache\n mpileaks_specs = mutable_database.query_local(\"mpileaks\")\n for s in mpileaks_specs:\n buildcache(\"push\", \"my-mirror\", s.format(\"{/hash}\"))\n\n # Grab all of the hashes for each mpileaks spec\n mpileaks_0_hashes = set([s.dag_hash() for s in mpileaks_specs[0].traverse()])\n\n # Make sure the view doesn't exist yet\n with pytest.raises(spack.binary_distribution.FetchIndexError):\n hashes_in_view = read_specs_in_index(mirror_directory, \"test_view\")\n\n # Create a view from an environment name that is different from the current active environment\n _, extra_args = args_for_env_by_name(\n mpileaks_specs[0]\n ) # Get args for env by name using mpileaks[0]\n context, _ = args_for_active_env(\n mpileaks_specs[1]\n ) # Get the context for an active env using mpileaks[1]\n with context:\n command_args = [\"update-index\", \"--name\", \"test_view\", \"my-mirror\"] + extra_args\n buildcache(*command_args)\n\n hashes_in_view = read_specs_in_index(mirror_directory, \"test_view\")\n # Assert all of the hashes for mpileaks_0_hashes exist in the view, and no other hashes\n assert hashes_in_view == mpileaks_0_hashes\n\n\n@pytest.mark.parametrize(\"view\", (None, \"test_view\"))\n@pytest.mark.disable_clean_stage_check\ndef test_buildcache_check_index_full(\n tmp_path, mutable_config, mutable_database, mutable_mock_env_path, view\n):\n view_args = [\"--name\", view] if view is not None else []\n mirror_directory = str(tmp_path)\n mirror(\"add\", \"--unsigned\", *view_args, \"my-mirror\", mirror_directory)\n\n # Push all of the mpileaks packages to the cache\n mpileaks_specs = mutable_database.query_local(\"mpileaks\")\n for s in mpileaks_specs:\n buildcache(\"push\", \"my-mirror\", s.format(\"{/hash}\"))\n\n # Update index using and active environment containing mpileaks[0]\n context, extra_args = args_for_active_env(mpileaks_specs[0])\n with context:\n buildcache(\"update-index\", \"my-mirror\", *extra_args)\n\n out = buildcache(\"check-index\", \"--verify\", \"exists\", \"manifests\", \"blobs\", \"--\", \"my-mirror\")\n # Everything thing be good here\n assert \"Index exists in mirror: my-mirror\"\n assert \"Missing specs: 0\"\n assert \"Missing blobs: 0\"\n if view:\n assert \"Unindexed specs: n/a\" in out\n else:\n assert \"Unindexed specs: 0\" in out\n\n # Remove the index blob\n # This creates index not for both view and non-view indices\n # For non-view indices this is also a missing blob\n index_name = \"index.manifest.json\"\n if view:\n index_name = os.path.join(view, index_name)\n blob_path = tmp_path / \"blobs\" / \"sha256\"\n with open(tmp_path / \"v3\" / \"manifests\" / \"index\" / index_name, \"r\", encoding=\"utf-8\") as fd:\n manifest = json.load(fd)\n print(manifest)\n digest = manifest[\"data\"][0][\"checksum\"]\n blob_path = blob_path / digest[:2] / digest\n\n # Delete the index manifest\n os.remove(blob_path)\n\n out = buildcache(\"check-index\", \"--verify\", \"exists\", \"manifests\", \"blobs\", \"--\", \"my-mirror\")\n # Everything thing be good here\n assert \"Index does not exist in mirror: my-mirror\"\n assert \"Missing specs: 0\"\n if view:\n assert \"Unindexed specs: n/a\" in out\n assert \"Missing blobs: 0\"\n else:\n assert \"The index blob is missing\" in out\n assert \"Unindexed specs: 15\" in out\n assert \"Missing blobs: 1\"\n\n\ndef test_buildcache_push_with_group(\n tmp_path: pathlib.Path, monkeypatch, install_mockery, mock_fetch, mutable_mock_env_path\n):\n \"\"\"Tests that --group pushes only specs from the requested group.\"\"\"\n env_dir = tmp_path / \"myenv\"\n env_dir.mkdir()\n (env_dir / \"spack.yaml\").write_text(\n \"\"\"\\\nspack:\n specs:\n - libelf\n - group: extra\n specs:\n - libdwarf\n view: false\n\"\"\"\n )\n\n mirror_dir = tmp_path / \"mirror\"\n mirror_dir.mkdir()\n\n uploader = _mock_uploader(mirror_dir)\n monkeypatch.setattr(\n spack.binary_distribution, \"make_uploader\", lambda *args, **kwargs: uploader\n )\n\n with ev.Environment(env_dir) as e:\n e.concretize()\n e.write()\n for _, root in e.concretized_specs():\n PackageInstaller([root.package], explicit=True, fake=True).install()\n\n buildcache(\"push\", \"--unsigned\", \"--only\", \"package\", \"--group\", \"extra\", str(mirror_dir))\n\n assert uploader.pushed == [\"libdwarf\"]\n\n\ndef test_buildcache_push_with_multiple_groups(\n tmp_path: pathlib.Path, monkeypatch, install_mockery, mock_fetch, mutable_mock_env_path\n):\n \"\"\"Tests that --group can be repeated to push specs from multiple groups.\"\"\"\n env_dir = tmp_path / \"myenv\"\n env_dir.mkdir()\n (env_dir / \"spack.yaml\").write_text(\n \"\"\"\\\nspack:\n specs:\n - libelf\n - group: extra\n specs:\n - libdwarf\n view: false\n\"\"\"\n )\n\n mirror_dir = tmp_path / \"mirror\"\n mirror_dir.mkdir()\n\n uploader = _mock_uploader(mirror_dir)\n monkeypatch.setattr(\n spack.binary_distribution, \"make_uploader\", lambda *args, **kwargs: uploader\n )\n\n with ev.Environment(env_dir) as e:\n e.concretize()\n e.write()\n for _, root in e.concretized_specs():\n PackageInstaller([root.package], explicit=True, fake=True).install()\n\n buildcache(\n \"push\",\n \"--unsigned\",\n \"--only\",\n \"package\",\n \"--group\",\n \"default\",\n \"--group\",\n \"extra\",\n str(mirror_dir),\n )\n\n assert set(uploader.pushed) == {\"libelf\", \"libdwarf\"}\n assert len(uploader.pushed) == len(set(uploader.pushed))\n\n\ndef test_buildcache_push_group_nonexistent_errors(tmp_path: pathlib.Path, mutable_mock_env_path):\n \"\"\"Tests that --group with a nonexistent group name raises an error.\"\"\"\n env_dir = tmp_path / \"myenv\"\n env_dir.mkdir()\n (env_dir / \"spack.yaml\").write_text(\n \"\"\"\\\nspack:\n specs:\n - libelf\n view: false\n\"\"\"\n )\n\n mirror_dir = tmp_path / \"mirror\"\n mirror_dir.mkdir()\n\n with ev.Environment(env_dir):\n with pytest.raises(spack.main.SpackCommandError):\n buildcache(\n \"push\", \"--unsigned\", \"--group\", \"nonexistent\", str(mirror_dir), fail_on_error=True\n )\n\n\ndef test_buildcache_push_group_and_specs_mutually_exclusive(\n tmp_path: pathlib.Path, mutable_mock_env_path\n):\n \"\"\"Tests that --group and explicit specs on the command line are mutually exclusive.\"\"\"\n env_dir = tmp_path / \"myenv\"\n env_dir.mkdir()\n (env_dir / \"spack.yaml\").write_text(\n \"\"\"\\\nspack:\n specs:\n - libelf\n view: false\n\"\"\"\n )\n\n mirror_dir = tmp_path / \"mirror\"\n mirror_dir.mkdir()\n\n with ev.Environment(env_dir):\n with pytest.raises(spack.main.SpackCommandError):\n buildcache(\n \"push\",\n \"--unsigned\",\n \"--group\",\n \"default\",\n str(mirror_dir),\n \"libelf\",\n fail_on_error=True,\n )\n\n\ndef test_buildcache_push_group_requires_active_env(tmp_path: pathlib.Path):\n \"\"\"Tests that ck--group without an active environment produces an error.\"\"\"\n mirror_dir = tmp_path / \"mirror\"\n mirror_dir.mkdir()\n\n with pytest.raises(spack.main.SpackCommandError):\n buildcache(\"push\", \"--unsigned\", \"--group\", \"default\", str(mirror_dir), fail_on_error=True)\n" }, { "path": "lib/spack/spack/test/cmd/cd.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nfrom spack.main import SpackCommand\n\ncd = SpackCommand(\"cd\")\n\n\ndef test_cd():\n \"\"\"Sanity check the cd command to make sure it works.\"\"\"\n\n out = cd()\n\n assert \"To set up shell support\" in out\n" }, { "path": "lib/spack/spack/test/cmd/checksum.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport pathlib\n\nimport pytest\n\nimport spack.cmd.checksum\nimport spack.concretize\nimport spack.error\nimport spack.package_base\nimport spack.repo\nimport spack.stage\nimport spack.util.web\nfrom spack.main import SpackCommand\nfrom spack.package_base import ManualDownloadRequiredError\nfrom spack.stage import interactive_version_filter\nfrom spack.version import Version\n\nspack_checksum = SpackCommand(\"checksum\")\n\n\n@pytest.fixture\ndef no_add(monkeypatch):\n def add_versions_to_pkg(pkg, version_lines, open_in_editor):\n raise AssertionError(\"Should not be called\")\n\n monkeypatch.setattr(spack.cmd.checksum, \"add_versions_to_pkg\", add_versions_to_pkg)\n\n\n@pytest.fixture\ndef can_fetch_versions(monkeypatch, no_add):\n \"\"\"Fake successful version detection.\"\"\"\n\n def fetch_remote_versions(pkg, concurrency):\n return {Version(\"1.0\"): \"https://www.example.com/pkg-1.0.tar.gz\"}\n\n def get_checksums_for_versions(url_by_version, package_name, **kwargs):\n return {\n v: \"abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890\"\n for v in url_by_version\n }\n\n def url_exists(url, curl=None):\n return True\n\n monkeypatch.setattr(\n spack.package_base.PackageBase, \"fetch_remote_versions\", fetch_remote_versions\n )\n monkeypatch.setattr(spack.stage, \"get_checksums_for_versions\", get_checksums_for_versions)\n monkeypatch.setattr(spack.util.web, \"url_exists\", url_exists)\n\n\n@pytest.fixture\ndef cannot_fetch_versions(monkeypatch, no_add):\n \"\"\"Fake unsuccessful version detection.\"\"\"\n\n def fetch_remote_versions(pkg, concurrency):\n return {}\n\n def get_checksums_for_versions(url_by_version, package_name, **kwargs):\n return {}\n\n def url_exists(url, curl=None):\n return False\n\n monkeypatch.setattr(\n spack.package_base.PackageBase, \"fetch_remote_versions\", fetch_remote_versions\n )\n monkeypatch.setattr(spack.stage, \"get_checksums_for_versions\", get_checksums_for_versions)\n monkeypatch.setattr(spack.util.web, \"url_exists\", url_exists)\n\n\n@pytest.mark.parametrize(\n \"arguments,expected\",\n [\n ([\"--batch\", \"patch\"], (True, False, False, False, False)),\n ([\"--latest\", \"patch\"], (False, True, False, False, False)),\n ([\"--preferred\", \"patch\"], (False, False, True, False, False)),\n ([\"--add-to-package\", \"patch\"], (False, False, False, True, False)),\n ([\"--verify\", \"patch\"], (False, False, False, False, True)),\n ],\n)\ndef test_checksum_args(arguments, expected):\n parser = argparse.ArgumentParser()\n spack.cmd.checksum.setup_parser(parser)\n args = parser.parse_args(arguments)\n check = args.batch, args.latest, args.preferred, args.add_to_package, args.verify\n assert check == expected\n\n\n@pytest.mark.parametrize(\n \"arguments,expected\",\n [\n ([\"--batch\", \"preferred-test\"], \"version of preferred-test\"),\n ([\"--latest\", \"preferred-test\"], \"Found 1 version\"),\n ([\"--preferred\", \"preferred-test\"], \"Found 1 version\"),\n ([\"--verify\", \"preferred-test\"], \"Verified 1 of 1\"),\n ([\"--verify\", \"zlib\", \"1.2.13\"], \"1.2.13 [-] No previous checksum\"),\n ],\n)\ndef test_checksum(arguments, expected, mock_packages, can_fetch_versions):\n output = spack_checksum(*arguments)\n assert expected in output\n\n # --verify doesn't print versions strings like other flags\n if \"--verify\" not in arguments:\n assert \"version(\" in output\n\n\ndef input_from_commands(*commands):\n \"\"\"Create a function that returns the next command from a list of inputs for interactive spack\n checksum. If None is encountered, this is equivalent to EOF / ^D.\"\"\"\n commands = iter(commands)\n\n def _input(prompt):\n cmd = next(commands)\n if cmd is None:\n raise EOFError\n assert isinstance(cmd, str)\n return cmd\n\n return _input\n\n\ndef test_checksum_interactive_filter():\n # Filter effectively by 1:1.0, then checksum.\n input = input_from_commands(\"f\", \"@1:\", \"f\", \"@:1.0\", \"c\")\n assert interactive_version_filter(\n {\n Version(\"1.1\"): \"https://www.example.com/pkg-1.1.tar.gz\",\n Version(\"1.0.1\"): \"https://www.example.com/pkg-1.0.1.tar.gz\",\n Version(\"1.0\"): \"https://www.example.com/pkg-1.0.tar.gz\",\n Version(\"0.9\"): \"https://www.example.com/pkg-0.9.tar.gz\",\n },\n input=input,\n ) == {\n Version(\"1.0.1\"): \"https://www.example.com/pkg-1.0.1.tar.gz\",\n Version(\"1.0\"): \"https://www.example.com/pkg-1.0.tar.gz\",\n }\n\n\ndef test_checksum_interactive_return_from_filter_prompt():\n # Enter and then exit filter subcommand.\n input = input_from_commands(\"f\", None, \"c\")\n assert interactive_version_filter(\n {\n Version(\"1.1\"): \"https://www.example.com/pkg-1.1.tar.gz\",\n Version(\"1.0.1\"): \"https://www.example.com/pkg-1.0.1.tar.gz\",\n Version(\"1.0\"): \"https://www.example.com/pkg-1.0.tar.gz\",\n Version(\"0.9\"): \"https://www.example.com/pkg-0.9.tar.gz\",\n },\n input=input,\n ) == {\n Version(\"1.1\"): \"https://www.example.com/pkg-1.1.tar.gz\",\n Version(\"1.0.1\"): \"https://www.example.com/pkg-1.0.1.tar.gz\",\n Version(\"1.0\"): \"https://www.example.com/pkg-1.0.tar.gz\",\n Version(\"0.9\"): \"https://www.example.com/pkg-0.9.tar.gz\",\n }\n\n\ndef test_checksum_interactive_quit_returns_none():\n # Quit after filtering something out (y to confirm quit)\n input = input_from_commands(\"f\", \"@1:\", \"q\", \"y\")\n assert (\n interactive_version_filter(\n {\n Version(\"1.1\"): \"https://www.example.com/pkg-1.1.tar.gz\",\n Version(\"1.0\"): \"https://www.example.com/pkg-1.0.tar.gz\",\n Version(\"0.9\"): \"https://www.example.com/pkg-0.9.tar.gz\",\n },\n input=input,\n )\n is None\n )\n\n\ndef test_checksum_interactive_reset_resets():\n # Filter 1:, then reset, then filter :0, should just given 0.9 (it was filtered out\n # before reset)\n input = input_from_commands(\"f\", \"@1:\", \"r\", \"f\", \":0\", \"c\")\n assert interactive_version_filter(\n {\n Version(\"1.1\"): \"https://www.example.com/pkg-1.1.tar.gz\",\n Version(\"1.0\"): \"https://www.example.com/pkg-1.0.tar.gz\",\n Version(\"0.9\"): \"https://www.example.com/pkg-0.9.tar.gz\",\n },\n input=input,\n ) == {Version(\"0.9\"): \"https://www.example.com/pkg-0.9.tar.gz\"}\n\n\ndef test_checksum_interactive_ask_each():\n # Ask each should run on the filtered list. First select 1.x, then select only the second\n # entry, which is 1.0.1.\n input = input_from_commands(\"f\", \"@1:\", \"a\", \"n\", \"y\", \"n\")\n assert interactive_version_filter(\n {\n Version(\"1.1\"): \"https://www.example.com/pkg-1.1.tar.gz\",\n Version(\"1.0.1\"): \"https://www.example.com/pkg-1.0.1.tar.gz\",\n Version(\"1.0\"): \"https://www.example.com/pkg-1.0.tar.gz\",\n Version(\"0.9\"): \"https://www.example.com/pkg-0.9.tar.gz\",\n },\n input=input,\n ) == {Version(\"1.0.1\"): \"https://www.example.com/pkg-1.0.1.tar.gz\"}\n\n\ndef test_checksum_interactive_quit_from_ask_each():\n # Enter ask each mode, select the second item, then quit from submenu, then checksum, which\n # should still include the last item at which ask each stopped.\n input = input_from_commands(\"a\", \"n\", \"y\", None, \"c\")\n assert interactive_version_filter(\n {\n Version(\"1.1\"): \"https://www.example.com/pkg-1.1.tar.gz\",\n Version(\"1.0\"): \"https://www.example.com/pkg-1.0.tar.gz\",\n Version(\"0.9\"): \"https://www.example.com/pkg-0.9.tar.gz\",\n },\n input=input,\n ) == {\n Version(\"1.0\"): \"https://www.example.com/pkg-1.0.tar.gz\",\n Version(\"0.9\"): \"https://www.example.com/pkg-0.9.tar.gz\",\n }\n\n\ndef test_checksum_interactive_nothing_left():\n \"\"\"If nothing is left after interactive filtering, return an empty dict.\"\"\"\n input = input_from_commands(\"f\", \"@2\", \"c\")\n assert (\n interactive_version_filter(\n {\n Version(\"1.1\"): \"https://www.example.com/pkg-1.1.tar.gz\",\n Version(\"1.0\"): \"https://www.example.com/pkg-1.0.tar.gz\",\n Version(\"0.9\"): \"https://www.example.com/pkg-0.9.tar.gz\",\n },\n input=input,\n )\n == {}\n )\n\n\ndef test_checksum_interactive_new_only():\n # The 1.0 version is known already, and should be dropped on `n`.\n input = input_from_commands(\"n\", \"c\")\n assert interactive_version_filter(\n {\n Version(\"1.1\"): \"https://www.example.com/pkg-1.1.tar.gz\",\n Version(\"1.0\"): \"https://www.example.com/pkg-1.0.tar.gz\",\n Version(\"0.9\"): \"https://www.example.com/pkg-0.9.tar.gz\",\n },\n known_versions=[Version(\"1.0\")],\n input=input,\n ) == {\n Version(\"1.1\"): \"https://www.example.com/pkg-1.1.tar.gz\",\n Version(\"0.9\"): \"https://www.example.com/pkg-0.9.tar.gz\",\n }\n\n\ndef test_checksum_interactive_top_n():\n \"\"\"Test integers select top n versions\"\"\"\n input = input_from_commands(\"2\", \"c\")\n assert interactive_version_filter(\n {\n Version(\"1.1\"): \"https://www.example.com/pkg-1.1.tar.gz\",\n Version(\"1.0\"): \"https://www.example.com/pkg-1.0.tar.gz\",\n Version(\"0.9\"): \"https://www.example.com/pkg-0.9.tar.gz\",\n },\n input=input,\n ) == {\n Version(\"1.1\"): \"https://www.example.com/pkg-1.1.tar.gz\",\n Version(\"1.0\"): \"https://www.example.com/pkg-1.0.tar.gz\",\n }\n\n\ndef test_checksum_interactive_unrecognized_command():\n \"\"\"Unrecognized commands should be ignored\"\"\"\n input = input_from_commands(\"-1\", \"0\", \"hello\", \"c\")\n v = {Version(\"1.1\"): \"https://www.example.com/pkg-1.1.tar.gz\"}\n assert interactive_version_filter(v.copy(), input=input) == v\n\n\ndef test_checksum_versions(mock_packages, can_fetch_versions, monkeypatch):\n pkg_cls = spack.repo.PATH.get_pkg_class(\"zlib\")\n versions = [str(v) for v in pkg_cls.versions]\n output = spack_checksum(\"zlib\", *versions)\n assert \"Found 3 versions\" in output\n assert \"version(\" in output\n\n\ndef test_checksum_missing_version(mock_packages, cannot_fetch_versions):\n output = spack_checksum(\"preferred-test\", \"99.99.99\", fail_on_error=False)\n assert \"Could not find any remote versions\" in output\n output = spack_checksum(\"--add-to-package\", \"preferred-test\", \"99.99.99\", fail_on_error=False)\n assert \"Could not find any remote versions\" in output\n\n\ndef test_checksum_deprecated_version(mock_packages, can_fetch_versions):\n output = spack_checksum(\"deprecated-versions\", \"1.1.0\", fail_on_error=False)\n assert \"Version 1.1.0 is deprecated\" in output\n output = spack_checksum(\n \"--add-to-package\", \"deprecated-versions\", \"1.1.0\", fail_on_error=False\n )\n assert \"Version 1.1.0 is deprecated\" in output\n\n\ndef test_checksum_url(mock_packages, config):\n pkg_cls = spack.repo.PATH.get_pkg_class(\"zlib\")\n with pytest.raises(spack.error.SpecSyntaxError):\n spack_checksum(f\"{pkg_cls.url}\")\n\n\ndef test_checksum_verification_fails(default_mock_concretization, capfd, can_fetch_versions):\n spec = spack.concretize.concretize_one(\"zlib\")\n pkg = spec.package\n versions = list(pkg.versions.keys())\n version_hashes = {versions[0]: \"abadhash\", Version(\"0.1\"): \"123456789\"}\n with pytest.raises(SystemExit):\n spack.cmd.checksum.print_checksum_status(pkg, version_hashes)\n out = str(capfd.readouterr())\n assert out.count(\"Correct\") == 0\n assert \"No previous checksum\" in out\n assert \"Invalid checksum\" in out\n\n\ndef test_checksum_manual_download_fails(mock_packages, monkeypatch):\n \"\"\"Confirm that checksumming a manually downloadable package fails.\"\"\"\n name = \"zlib\"\n pkg_cls = spack.repo.PATH.get_pkg_class(name)\n versions = [str(v) for v in pkg_cls.versions]\n monkeypatch.setattr(spack.package_base.PackageBase, \"manual_download\", True)\n\n # First check that the exception is raised with the default download\n # instructions.\n with pytest.raises(ManualDownloadRequiredError, match=f\"required for {name}\"):\n spack_checksum(name, *versions)\n\n # Now check that the exception is raised with custom download instructions.\n error = \"Cannot calculate the checksum for a manually downloaded package.\"\n monkeypatch.setattr(spack.package_base.PackageBase, \"download_instr\", error)\n with pytest.raises(ManualDownloadRequiredError, match=error):\n spack_checksum(name, *versions)\n\n\ndef test_upate_package_contents(tmp_path: pathlib.Path):\n \"\"\"Test that the package.py file is updated with the new versions.\"\"\"\n pkg_path = tmp_path / \"package.py\"\n pkg_path.write_text(\n \"\"\"\\\nfrom spack.package import *\n\nclass Zlib(Package):\n homepage = \"http://zlib.net\"\n url = \"http://zlib.net/fossils/zlib-1.2.11.tar.gz\"\n\n version(\"1.2.11\", sha256=\"c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1\")\n version(\"1.2.8\", sha256=\"36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d\")\n version(\"1.2.3\", sha256=\"1795c7d067a43174113fdf03447532f373e1c6c57c08d61d9e4e9be5e244b05e\")\n variant(\"pic\", default=True, description=\"test\")\n\n def install(self, spec, prefix):\n make(\"install\")\n\"\"\"\n )\n version_lines = \"\"\"\\\n version(\"1.2.13\", sha256=\"abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890\")\n version(\"1.2.5\", sha256=\"abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890\")\n version(\"1.2.3\", sha256=\"1795c7d067a43174113fdf03447532f373e1c6c57c08d61d9e4e9be5e244b05e\")\n\"\"\"\n # ruff: disable[E501]\n # two new versions are added\n assert spack.cmd.checksum.add_versions_to_pkg(str(pkg_path), version_lines) == 2\n assert (\n pkg_path.read_text()\n == \"\"\"\\\nfrom spack.package import *\n\nclass Zlib(Package):\n homepage = \"http://zlib.net\"\n url = \"http://zlib.net/fossils/zlib-1.2.11.tar.gz\"\n\n version(\"1.2.13\", sha256=\"abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890\") # FIXME\n version(\"1.2.11\", sha256=\"c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1\")\n version(\"1.2.8\", sha256=\"36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d\")\n version(\"1.2.5\", sha256=\"abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890\") # FIXME\n version(\"1.2.3\", sha256=\"1795c7d067a43174113fdf03447532f373e1c6c57c08d61d9e4e9be5e244b05e\")\n variant(\"pic\", default=True, description=\"test\")\n\n def install(self, spec, prefix):\n make(\"install\")\n\"\"\"\n )\n\n\n# ruff: enable[E501]\n" }, { "path": "lib/spack/spack/test/cmd/ci.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport json\nimport os\nimport pathlib\nimport shutil\nfrom typing import NamedTuple\n\nimport pytest\n\nimport spack.vendor.jsonschema\n\nimport spack.binary_distribution\nimport spack.ci as ci\nimport spack.cmd\nimport spack.cmd.ci\nimport spack.concretize\nimport spack.environment as ev\nimport spack.hash_types as ht\nimport spack.main\nimport spack.paths\nimport spack.repo\nimport spack.spec\nimport spack.stage\nimport spack.util.spack_yaml as syaml\nimport spack.util.web\nimport spack.version\nfrom spack.ci import gitlab as gitlab_generator\nfrom spack.ci.common import PipelineDag, PipelineOptions, SpackCIConfig\nfrom spack.ci.generator_registry import generator\nfrom spack.cmd.ci import FAILED_CREATE_BUILDCACHE_CODE\nfrom spack.error import SpackError\nfrom spack.llnl.util.filesystem import mkdirp, working_dir\nfrom spack.schema.database_index import schema as db_idx_schema\nfrom spack.test.conftest import MockHTTPResponse, RepoBuilder\n\nconfig_cmd = spack.main.SpackCommand(\"config\")\nci_cmd = spack.main.SpackCommand(\"ci\")\nenv_cmd = spack.main.SpackCommand(\"env\")\nmirror_cmd = spack.main.SpackCommand(\"mirror\")\ngpg_cmd = spack.main.SpackCommand(\"gpg\")\ninstall_cmd = spack.main.SpackCommand(\"install\")\nuninstall_cmd = spack.main.SpackCommand(\"uninstall\")\nbuildcache_cmd = spack.main.SpackCommand(\"buildcache\")\n\npytestmark = [\n pytest.mark.usefixtures(\"mock_packages\"),\n pytest.mark.not_on_windows(\"does not run on windows\"),\n pytest.mark.maybeslow,\n]\n\n\n@pytest.fixture()\ndef ci_base_environment(working_env, tmp_path: pathlib.Path):\n os.environ[\"CI_PROJECT_DIR\"] = str(tmp_path)\n os.environ[\"CI_PIPELINE_ID\"] = \"7192\"\n os.environ[\"CI_JOB_NAME\"] = \"mock\"\n\n\n@pytest.fixture(scope=\"function\")\ndef mock_git_repo(git, tmp_path: pathlib.Path):\n \"\"\"Create a mock git repo with two commits, the last one creating\n a .gitlab-ci.yml\"\"\"\n\n repo_path = str(tmp_path / \"mockspackrepo\")\n mkdirp(repo_path)\n\n with working_dir(repo_path):\n git(\"init\")\n\n git(\"config\", \"--local\", \"user.email\", \"testing@spack.io\")\n git(\"config\", \"--local\", \"user.name\", \"Spack Testing\")\n\n # This path is used to satisfy git root detection and detection of environment changed\n path_to_env = os.path.sep.join((\"no\", \"such\", \"env\", \"path\", \"spack.yaml\"))\n os.makedirs(os.path.dirname(path_to_env))\n with open(path_to_env, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\nspack:\n specs:\n - a\n\"\"\"\n )\n\n git(\"add\", path_to_env)\n\n with open(\"README.md\", \"w\", encoding=\"utf-8\") as f:\n f.write(\"# Introduction\")\n\n # initial commit with README\n git(\"add\", \"README.md\")\n git(\"-c\", \"commit.gpgsign=false\", \"commit\", \"-m\", \"initial commit\")\n\n with open(\".gitlab-ci.yml\", \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\ntestjob:\n script:\n - echo \"success\"\n \"\"\"\n )\n\n # second commit, adding a .gitlab-ci.yml\n git(\"add\", \".gitlab-ci.yml\")\n git(\"-c\", \"commit.gpgsign=false\", \"commit\", \"-m\", \"add a .gitlab-ci.yml\")\n\n yield repo_path\n\n\n@pytest.fixture()\ndef ci_generate_test(\n tmp_path: pathlib.Path, mutable_mock_env_path, install_mockery, ci_base_environment\n):\n \"\"\"Returns a function that creates a new test environment, and runs 'spack generate'\n on it, given the content of the spack.yaml file.\n\n Additional positional arguments will be added to the 'spack generate' call.\n \"\"\"\n\n def _func(spack_yaml_content, *args, fail_on_error=True):\n spack_yaml = tmp_path / \"spack.yaml\"\n spack_yaml.write_text(spack_yaml_content)\n ev.create(\"test\", init_file=spack_yaml, with_view=False)\n outputfile = tmp_path / \".gitlab-ci.yml\"\n with ev.read(\"test\"):\n output = ci_cmd(\n \"generate\", \"--output-file\", str(outputfile), *args, fail_on_error=fail_on_error\n )\n\n return spack_yaml, outputfile, output\n\n return _func\n\n\ndef test_ci_generate_with_env(ci_generate_test, tmp_path: pathlib.Path, mock_binary_index):\n \"\"\"Make sure we can get a .gitlab-ci.yml from an environment file\n which has the gitlab-ci, cdash, and mirrors sections.\n \"\"\"\n mirror_url = tmp_path / \"ci-mirror\"\n spack_yaml, outputfile, _ = ci_generate_test(\n f\"\"\"\\\nspack:\n definitions:\n - old-gcc-pkgs:\n - archive-files\n - callpath\n # specify ^openblas-with-lapack to ensure that builtin.mock repo flake8\n # package (which can also provide lapack) is not chosen, as it violates\n # a package-level check which requires exactly one fetch strategy (this\n # is apparently not an issue for other tests that use it).\n - hypre@0.2.15 ^openblas-with-lapack\n specs:\n - matrix:\n - [$old-gcc-pkgs]\n mirrors:\n buildcache-destination: {mirror_url}\n ci:\n pipeline-gen:\n - submapping:\n - match:\n - arch=test-debian6-core2\n build-job:\n tags:\n - donotcare\n image: donotcare\n - match:\n - arch=test-debian6-m1\n build-job:\n tags:\n - donotcare\n image: donotcare\n - cleanup-job:\n image: donotcare\n tags: [donotcare]\n - reindex-job:\n script:: [hello, world]\n custom_attribute: custom!\n cdash:\n build-group: Not important\n url: https://my.fake.cdash\n project: Not used\n site: Nothing\n\"\"\",\n \"--artifacts-root\",\n str(tmp_path / \"my_artifacts_root\"),\n )\n yaml_contents = syaml.load(outputfile.read_text())\n\n assert \"workflow\" in yaml_contents\n assert \"rules\" in yaml_contents[\"workflow\"]\n assert yaml_contents[\"workflow\"][\"rules\"] == [{\"when\": \"always\"}]\n\n assert \"stages\" in yaml_contents\n assert len(yaml_contents[\"stages\"]) == 6\n assert yaml_contents[\"stages\"][0] == \"stage-0\"\n assert yaml_contents[\"stages\"][5] == \"stage-rebuild-index\"\n\n assert \"rebuild-index\" in yaml_contents\n rebuild_job = yaml_contents[\"rebuild-index\"]\n assert (\n rebuild_job[\"script\"][0]\n == f\"spack -v buildcache update-index --keys {mirror_url.as_uri()}\"\n )\n assert rebuild_job[\"custom_attribute\"] == \"custom!\"\n\n assert \"variables\" in yaml_contents\n assert \"SPACK_ARTIFACTS_ROOT\" in yaml_contents[\"variables\"]\n assert yaml_contents[\"variables\"][\"SPACK_ARTIFACTS_ROOT\"] == \"my_artifacts_root\"\n\n\ndef test_ci_generate_with_env_missing_section(\n ci_generate_test, tmp_path: pathlib.Path, mock_binary_index\n):\n \"\"\"Make sure we get a reasonable message if we omit gitlab-ci section\"\"\"\n env_yaml = f\"\"\"\\\nspack:\n specs:\n - archive-files\n mirrors:\n buildcache-destination: {tmp_path / \"ci-mirror\"}\n\"\"\"\n expect = \"Environment does not have a `ci` configuration\"\n with pytest.raises(ci.SpackCIError, match=expect):\n ci_generate_test(env_yaml)\n\n\ndef test_ci_generate_with_cdash_token(\n ci_generate_test, tmp_path: pathlib.Path, mock_binary_index, monkeypatch\n):\n \"\"\"Make sure we it doesn't break if we configure cdash\"\"\"\n monkeypatch.setenv(\"SPACK_CDASH_AUTH_TOKEN\", \"notreallyatokenbutshouldnotmatter\")\n spack_yaml_content = f\"\"\"\\\nspack:\n specs:\n - archive-files\n mirrors:\n buildcache-destination: {tmp_path / \"ci-mirror\"}\n ci:\n pipeline-gen:\n - submapping:\n - match:\n - archive-files\n build-job:\n tags:\n - donotcare\n image: donotcare\n cdash:\n build-group: Not important\n url: {(tmp_path / \"cdash\").as_uri()}\n project: Not used\n site: Nothing\n\"\"\"\n\n def _urlopen(*args, **kwargs):\n return MockHTTPResponse.with_json(200, \"OK\", headers={}, body={})\n\n monkeypatch.setattr(ci.common, \"_urlopen\", _urlopen)\n\n spack_yaml, original_file, output = ci_generate_test(spack_yaml_content)\n yaml_contents = syaml.load(original_file.read_text())\n\n # That fake token should have resulted in being unable to\n # register build group with cdash, but the workload should\n # still have been generated.\n assert \"Failed to create or retrieve buildgroup\" in output\n expected_keys = [\"rebuild-index\", \"stages\", \"variables\", \"workflow\"]\n assert all([key in yaml_contents.keys() for key in expected_keys])\n\n\ndef test_ci_generate_with_custom_settings(\n ci_generate_test, tmp_path: pathlib.Path, mock_binary_index, monkeypatch\n):\n \"\"\"Test use of user-provided scripts and attributes\"\"\"\n monkeypatch.setattr(spack, \"get_version\", lambda: \"0.15.3\")\n monkeypatch.setattr(spack, \"get_spack_commit\", lambda: \"big ol commit sha\")\n spack_yaml, outputfile, _ = ci_generate_test(\n f\"\"\"\\\nspack:\n specs:\n - archive-files\n mirrors:\n buildcache-destination: {tmp_path / \"ci-mirror\"}\n ci:\n pipeline-gen:\n - submapping:\n - match:\n - archive-files\n build-job:\n tags:\n - donotcare\n variables:\n ONE: plain-string-value\n TWO: ${{INTERP_ON_BUILD}}\n before_script:\n - mkdir /some/path\n - pushd /some/path\n - git clone ${{SPACK_REPO}}\n - cd spack\n - git checkout ${{SPACK_REF}}\n - popd\n script:\n - spack -d ci rebuild\n after_script:\n - rm -rf /some/path/spack\n custom_attribute: custom!\n artifacts:\n paths:\n - some/custom/artifact\n\"\"\"\n )\n yaml_contents = syaml.load(outputfile.read_text())\n\n assert yaml_contents[\"variables\"][\"SPACK_VERSION\"] == \"0.15.3\"\n assert yaml_contents[\"variables\"][\"SPACK_CHECKOUT_VERSION\"] == \"big ol commit sha\"\n\n assert any(\"archive-files\" in key for key in yaml_contents)\n for ci_key, ci_obj in yaml_contents.items():\n if \"archive-files\" not in ci_key:\n continue\n\n # Ensure we have variables, possibly interpolated\n assert ci_obj[\"variables\"][\"ONE\"] == \"plain-string-value\"\n assert ci_obj[\"variables\"][\"TWO\"] == \"${INTERP_ON_BUILD}\"\n\n # Ensure we have scripts verbatim\n assert ci_obj[\"before_script\"] == [\n \"mkdir /some/path\",\n \"pushd /some/path\",\n \"git clone ${SPACK_REPO}\",\n \"cd spack\",\n \"git checkout ${SPACK_REF}\",\n \"popd\",\n ]\n assert ci_obj[\"script\"][1].startswith(\"cd \")\n ci_obj[\"script\"][1] = \"cd ENV\"\n assert ci_obj[\"script\"] == [\n \"spack -d ci rebuild\",\n \"cd ENV\",\n \"spack env activate --without-view .\",\n \"spack spec /$SPACK_JOB_SPEC_DAG_HASH\",\n \"spack ci rebuild\",\n ]\n assert ci_obj[\"after_script\"] == [\"rm -rf /some/path/spack\"]\n\n # Ensure we have the custom attributes\n assert \"some/custom/artifact\" in ci_obj[\"artifacts\"][\"paths\"]\n assert ci_obj[\"custom_attribute\"] == \"custom!\"\n\n\ndef test_ci_generate_pkg_with_deps(ci_generate_test, tmp_path: pathlib.Path, ci_base_environment):\n \"\"\"Test pipeline generation for a package w/ dependencies\"\"\"\n spack_yaml, outputfile, _ = ci_generate_test(\n f\"\"\"\\\nspack:\n specs:\n - dependent-install\n mirrors:\n buildcache-destination: {tmp_path / \"ci-mirror\"}\n ci:\n pipeline-gen:\n - submapping:\n - match:\n - dependent-install\n build-job:\n tags:\n - donotcare\n - match:\n - dependency-install\n build-job:\n tags:\n - donotcare\n\"\"\"\n )\n yaml_contents = syaml.load(outputfile.read_text())\n\n found = []\n for ci_key, ci_obj in yaml_contents.items():\n if \"dependency-install\" in ci_key:\n assert \"stage\" in ci_obj\n assert ci_obj[\"stage\"] == \"stage-0\"\n found.append(\"dependency-install\")\n if \"dependent-install\" in ci_key:\n assert \"stage\" in ci_obj\n assert ci_obj[\"stage\"] == \"stage-1\"\n found.append(\"dependent-install\")\n\n assert \"dependent-install\" in found\n assert \"dependency-install\" in found\n\n\ndef test_ci_generate_for_pr_pipeline(ci_generate_test, tmp_path: pathlib.Path, monkeypatch):\n \"\"\"Test generation of a PR pipeline with disabled rebuild-index\"\"\"\n monkeypatch.setenv(\"SPACK_PIPELINE_TYPE\", \"spack_pull_request\")\n\n spack_yaml, outputfile, _ = ci_generate_test(\n f\"\"\"\\\nspack:\n specs:\n - dependent-install\n mirrors:\n buildcache-destination: {tmp_path / \"ci-mirror\"}\n ci:\n pipeline-gen:\n - submapping:\n - match:\n - dependent-install\n build-job:\n tags:\n - donotcare\n - match:\n - dependency-install\n build-job:\n tags:\n - donotcare\n - cleanup-job:\n image: donotcare\n tags: [donotcare]\n rebuild-index: False\n\"\"\"\n )\n yaml_contents = syaml.load(outputfile.read_text())\n\n assert \"rebuild-index\" not in yaml_contents\n assert \"variables\" in yaml_contents\n assert \"SPACK_PIPELINE_TYPE\" in yaml_contents[\"variables\"]\n assert (\n ci.common.PipelineType[yaml_contents[\"variables\"][\"SPACK_PIPELINE_TYPE\"]]\n == ci.common.PipelineType.PULL_REQUEST\n )\n\n\ndef test_ci_generate_with_external_pkg(ci_generate_test, tmp_path: pathlib.Path, monkeypatch):\n \"\"\"Make sure we do not generate jobs for external pkgs\"\"\"\n spack_yaml, outputfile, _ = ci_generate_test(\n f\"\"\"\\\nspack:\n specs:\n - archive-files\n - externaltest\n mirrors:\n buildcache-destination: {tmp_path / \"ci-mirror\"}\n ci:\n pipeline-gen:\n - submapping:\n - match:\n - archive-files\n - externaltest\n build-job:\n tags:\n - donotcare\n image: donotcare\n\"\"\"\n )\n yaml_contents = syaml.load(outputfile.read_text())\n # Check that the \"externaltool\" package was not erroneously staged\n assert all(\"externaltool\" not in key for key in yaml_contents)\n\n\ndef test_ci_rebuild_missing_config(tmp_path: pathlib.Path, working_env, mutable_mock_env_path):\n spack_yaml = tmp_path / \"spack.yaml\"\n spack_yaml.write_text(\n \"\"\"\n spack:\n specs:\n - archive-files\n \"\"\"\n )\n\n env_cmd(\"create\", \"test\", str(spack_yaml))\n env_cmd(\"activate\", \"--without-view\", \"--sh\", \"test\")\n out = ci_cmd(\"rebuild\", fail_on_error=False)\n assert \"env containing ci\" in out\n env_cmd(\"deactivate\")\n\n\ndef _signing_key():\n signing_key_path = pathlib.Path(spack.paths.mock_gpg_keys_path) / \"package-signing-key\"\n return signing_key_path.read_text()\n\n\nclass RebuildEnv(NamedTuple):\n broken_spec_file: pathlib.Path\n ci_job_url: str\n ci_pipeline_url: str\n env_dir: pathlib.Path\n log_dir: pathlib.Path\n mirror_dir: pathlib.Path\n mirror_url: str\n repro_dir: pathlib.Path\n root_spec_dag_hash: str\n test_dir: pathlib.Path\n working_dir: pathlib.Path\n\n\ndef create_rebuild_env(\n tmp_path: pathlib.Path, pkg_name: str, broken_tests: bool = False\n) -> RebuildEnv:\n scratch = tmp_path / \"working_dir\"\n log_dir = scratch / \"logs\"\n repro_dir = scratch / \"repro\"\n test_dir = scratch / \"test\"\n env_dir = scratch / \"concrete_env\"\n mirror_dir = scratch / \"mirror\"\n broken_specs_path = scratch / \"naughty-list\"\n\n mirror_url = mirror_dir.as_uri()\n\n ci_job_url = \"https://some.domain/group/project/-/jobs/42\"\n ci_pipeline_url = \"https://some.domain/group/project/-/pipelines/7\"\n\n env_dir.mkdir(parents=True)\n with open(env_dir / \"spack.yaml\", \"w\", encoding=\"utf-8\") as f:\n f.write(\n f\"\"\"\nspack:\n definitions:\n - packages: [{pkg_name}]\n specs:\n - $packages\n mirrors:\n buildcache-destination: {mirror_dir}\n ci:\n broken-specs-url: {broken_specs_path.as_uri()}\n broken-tests-packages: {json.dumps([pkg_name] if broken_tests else [])}\n pipeline-gen:\n - submapping:\n - match:\n - {pkg_name}\n build-job:\n tags:\n - donotcare\n image: donotcare\n cdash:\n build-group: Not important\n url: https://my.fake.cdash\n project: Not used\n site: Nothing\n\"\"\"\n )\n\n with ev.Environment(env_dir) as env:\n env.concretize()\n env.write()\n\n shutil.copy(env_dir / \"spack.yaml\", tmp_path / \"spack.yaml\")\n\n root_spec_dag_hash = env.concrete_roots()[0].dag_hash()\n\n return RebuildEnv(\n broken_spec_file=broken_specs_path / root_spec_dag_hash,\n ci_job_url=ci_job_url,\n ci_pipeline_url=ci_pipeline_url,\n env_dir=env_dir,\n log_dir=log_dir,\n mirror_dir=mirror_dir,\n mirror_url=mirror_url,\n repro_dir=repro_dir,\n root_spec_dag_hash=root_spec_dag_hash,\n test_dir=test_dir,\n working_dir=scratch,\n )\n\n\ndef activate_rebuild_env(tmp_path: pathlib.Path, pkg_name: str, rebuild_env: RebuildEnv):\n env_cmd(\"activate\", \"--without-view\", \"--sh\", \"-d\", \".\")\n\n # Create environment variables as gitlab would do it\n os.environ.update(\n {\n \"SPACK_ARTIFACTS_ROOT\": str(rebuild_env.working_dir),\n \"SPACK_JOB_LOG_DIR\": str(rebuild_env.log_dir),\n \"SPACK_JOB_REPRO_DIR\": str(rebuild_env.repro_dir),\n \"SPACK_JOB_TEST_DIR\": str(rebuild_env.test_dir),\n \"SPACK_LOCAL_MIRROR_DIR\": str(rebuild_env.mirror_dir),\n \"SPACK_CONCRETE_ENV_DIR\": str(rebuild_env.env_dir),\n \"CI_PIPELINE_ID\": \"7192\",\n \"SPACK_SIGNING_KEY\": _signing_key(),\n \"SPACK_JOB_SPEC_DAG_HASH\": rebuild_env.root_spec_dag_hash,\n \"SPACK_JOB_SPEC_PKG_NAME\": pkg_name,\n \"SPACK_COMPILER_ACTION\": \"NONE\",\n \"SPACK_CDASH_BUILD_NAME\": pkg_name,\n \"SPACK_REMOTE_MIRROR_URL\": rebuild_env.mirror_url,\n \"SPACK_PIPELINE_TYPE\": \"spack_protected_branch\",\n \"CI_JOB_URL\": rebuild_env.ci_job_url,\n \"CI_PIPELINE_URL\": rebuild_env.ci_pipeline_url,\n \"CI_PROJECT_DIR\": str(tmp_path / \"ci-project\"),\n }\n )\n\n\n@pytest.mark.parametrize(\"broken_tests\", [True, False])\ndef test_ci_rebuild_mock_success(\n tmp_path: pathlib.Path,\n working_env,\n mutable_mock_env_path,\n install_mockery,\n mock_gnupghome,\n mock_fetch,\n mock_binary_index,\n monkeypatch,\n broken_tests,\n):\n pkg_name = \"archive-files\"\n rebuild_env = create_rebuild_env(tmp_path, pkg_name, broken_tests)\n\n monkeypatch.setattr(spack.cmd.ci, \"SPACK_COMMAND\", \"echo\")\n\n with working_dir(rebuild_env.env_dir):\n activate_rebuild_env(tmp_path, pkg_name, rebuild_env)\n\n out = ci_cmd(\"rebuild\", \"--tests\", fail_on_error=False)\n\n # We didn\"t really run the build so build output file(s) are missing\n assert \"Unable to copy files\" in out\n assert \"No such file or directory\" in out\n\n if broken_tests:\n # We generate a skipped tests report in this case\n assert \"Unable to run stand-alone tests\" in out\n else:\n # No installation means no package to test and no test log to copy\n assert \"Cannot copy test logs\" in out\n\n\ndef test_ci_rebuild_mock_failure_to_push(\n tmp_path: pathlib.Path,\n working_env,\n mutable_mock_env_path,\n install_mockery,\n mock_gnupghome,\n mock_fetch,\n mock_binary_index,\n ci_base_environment,\n monkeypatch,\n):\n pkg_name = \"trivial-install-test-package\"\n rebuild_env = create_rebuild_env(tmp_path, pkg_name)\n\n # Mock the install script succuess\n def mock_success(*args, **kwargs):\n return 0\n\n monkeypatch.setattr(ci, \"process_command\", mock_success)\n\n # Mock failure to push to the build cache\n def mock_push_or_raise(*args, **kwargs):\n raise spack.binary_distribution.PushToBuildCacheError(\n \"Encountered problem pushing binary : \"\n )\n\n monkeypatch.setattr(spack.binary_distribution.Uploader, \"push_or_raise\", mock_push_or_raise)\n\n with working_dir(rebuild_env.env_dir):\n activate_rebuild_env(tmp_path, pkg_name, rebuild_env)\n\n with pytest.raises(spack.main.SpackCommandError) as e:\n ci_cmd(\"rebuild\")\n assert e.value.code == FAILED_CREATE_BUILDCACHE_CODE\n\n\ndef test_ci_require_signing(\n tmp_path: pathlib.Path,\n working_env,\n mutable_mock_env_path,\n mock_gnupghome,\n ci_base_environment,\n monkeypatch,\n):\n spack_yaml = tmp_path / \"spack.yaml\"\n spack_yaml.write_text(\n f\"\"\"\nspack:\n specs:\n - archive-files\n mirrors:\n buildcache-destination: {tmp_path / \"ci-mirror\"}\n ci:\n pipeline-gen:\n - submapping:\n - match:\n - archive-files\n build-job:\n tags:\n - donotcare\n image: donotcare\n\"\"\"\n )\n env_cmd(\"activate\", \"--without-view\", \"--sh\", \"-d\", str(spack_yaml.parent))\n\n # Run without the variable to make sure we don't accidentally require signing\n output = ci_cmd(\"rebuild\", fail_on_error=False)\n assert \"spack must have exactly one signing key\" not in output\n\n # Now run with the variable to make sure it works\n monkeypatch.setenv(\"SPACK_REQUIRE_SIGNING\", \"True\")\n output = ci_cmd(\"rebuild\", fail_on_error=False)\n assert \"spack must have exactly one signing key\" in output\n env_cmd(\"deactivate\")\n\n\ndef test_ci_nothing_to_rebuild(\n tmp_path: pathlib.Path,\n working_env,\n mutable_mock_env_path,\n install_mockery,\n monkeypatch,\n mock_fetch,\n ci_base_environment,\n mock_binary_index,\n):\n scratch = tmp_path / \"working_dir\"\n mirror_dir = scratch / \"mirror\"\n mirror_url = mirror_dir.as_uri()\n\n with open(tmp_path / \"spack.yaml\", \"w\", encoding=\"utf-8\") as f:\n f.write(\n f\"\"\"\nspack:\n definitions:\n - packages: [archive-files]\n specs:\n - $packages\n mirrors:\n buildcache-destination: {mirror_url}\n ci:\n pipeline-gen:\n - submapping:\n - match:\n - archive-files\n build-job:\n tags:\n - donotcare\n image: donotcare\n\"\"\"\n )\n\n install_cmd(\"archive-files\")\n buildcache_cmd(\"push\", \"-f\", \"-u\", \"--update-index\", mirror_url, \"archive-files\")\n\n with working_dir(tmp_path):\n env_cmd(\"create\", \"test\", \"./spack.yaml\")\n with ev.read(\"test\") as env:\n env.concretize()\n\n # Create environment variables as gitlab would do it\n os.environ.update(\n {\n \"SPACK_ARTIFACTS_ROOT\": str(scratch),\n \"SPACK_JOB_LOG_DIR\": \"log_dir\",\n \"SPACK_JOB_REPRO_DIR\": \"repro_dir\",\n \"SPACK_JOB_TEST_DIR\": \"test_dir\",\n \"SPACK_CONCRETE_ENV_DIR\": str(tmp_path),\n \"SPACK_JOB_SPEC_DAG_HASH\": env.concrete_roots()[0].dag_hash(),\n \"SPACK_JOB_SPEC_PKG_NAME\": \"archive-files\",\n \"SPACK_COMPILER_ACTION\": \"NONE\",\n }\n )\n\n ci_out = ci_cmd(\"rebuild\")\n\n assert \"No need to rebuild archive-files\" in ci_out\n\n env_cmd(\"deactivate\")\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_push_to_build_cache(\n tmp_path: pathlib.Path,\n mutable_mock_env_path,\n install_mockery,\n mock_fetch,\n mock_gnupghome,\n ci_base_environment,\n mock_binary_index,\n):\n scratch = tmp_path / \"working_dir\"\n mirror_dir = scratch / \"mirror\"\n mirror_url = mirror_dir.as_uri()\n\n ci.import_signing_key(_signing_key())\n\n with working_dir(tmp_path):\n with open(\"spack.yaml\", \"w\", encoding=\"utf-8\") as f:\n f.write(\n f\"\"\"\\\nspack:\n definitions:\n - packages: [patchelf]\n specs:\n - $packages\n mirrors:\n buildcache-destination: {mirror_url}\n ci:\n pipeline-gen:\n - submapping:\n - match:\n - patchelf\n build-job:\n tags:\n - donotcare\n image: donotcare\n - cleanup-job:\n tags:\n - nonbuildtag\n image: basicimage\n - any-job:\n tags:\n - nonbuildtag\n image: basicimage\n custom_attribute: custom!\n\"\"\"\n )\n env_cmd(\"create\", \"test\", \"./spack.yaml\")\n with ev.read(\"test\") as current_env:\n current_env.concretize()\n install_cmd(\"--keep-stage\")\n\n concrete_spec = list(current_env.roots())[0]\n spec_json = concrete_spec.to_json(hash=ht.dag_hash)\n json_path = str(tmp_path / \"spec.json\")\n with open(json_path, \"w\", encoding=\"utf-8\") as ypfd:\n ypfd.write(spec_json)\n\n for s in concrete_spec.traverse():\n ci.push_to_build_cache(s, mirror_url, True)\n\n # Now test the --prune-dag (default) option of spack ci generate\n mirror_cmd(\"add\", \"test-ci\", mirror_url)\n\n outputfile_pruned = str(tmp_path / \"pruned_pipeline.yml\")\n ci_cmd(\"generate\", \"--output-file\", outputfile_pruned)\n\n with open(outputfile_pruned, encoding=\"utf-8\") as f:\n contents = f.read()\n yaml_contents = syaml.load(contents)\n # Make sure there are no other spec jobs or rebuild-index\n assert set(yaml_contents.keys()) == {\"no-specs-to-rebuild\", \"workflow\"}\n\n the_elt = yaml_contents[\"no-specs-to-rebuild\"]\n assert \"tags\" in the_elt\n assert \"nonbuildtag\" in the_elt[\"tags\"]\n assert \"image\" in the_elt\n assert the_elt[\"image\"] == \"basicimage\"\n assert the_elt[\"custom_attribute\"] == \"custom!\"\n\n assert \"rules\" in yaml_contents[\"workflow\"]\n assert yaml_contents[\"workflow\"][\"rules\"] == [{\"when\": \"always\"}]\n\n outputfile_not_pruned = str(tmp_path / \"unpruned_pipeline.yml\")\n ci_cmd(\"generate\", \"--no-prune-dag\", \"--output-file\", outputfile_not_pruned)\n\n # Test the --no-prune-dag option of spack ci generate\n with open(outputfile_not_pruned, encoding=\"utf-8\") as f:\n contents = f.read()\n yaml_contents = syaml.load(contents)\n\n found_spec_job = False\n\n for ci_key in yaml_contents.keys():\n if \"patchelf\" in ci_key:\n the_elt = yaml_contents[ci_key]\n assert \"variables\" in the_elt\n job_vars = the_elt[\"variables\"]\n assert \"SPACK_SPEC_NEEDS_REBUILD\" in job_vars\n assert job_vars[\"SPACK_SPEC_NEEDS_REBUILD\"] == \"False\"\n assert the_elt[\"custom_attribute\"] == \"custom!\"\n found_spec_job = True\n\n assert found_spec_job\n\n mirror_cmd(\"rm\", \"test-ci\")\n\n # Test generating buildcache index while we have bin mirror\n buildcache_cmd(\"update-index\", mirror_url)\n\n # Validate resulting buildcache (database) index\n layout_version = spack.binary_distribution.CURRENT_BUILD_CACHE_LAYOUT_VERSION\n mirror_metadata = spack.binary_distribution.MirrorMetadata(mirror_url, layout_version)\n index_fetcher = spack.binary_distribution.DefaultIndexFetcher(mirror_metadata, None)\n result = index_fetcher.conditional_fetch()\n spack.vendor.jsonschema.validate(json.loads(result.data), db_idx_schema)\n\n # Now that index is regenerated, validate \"buildcache list\" output\n assert \"patchelf\" in buildcache_cmd(\"list\")\n\n logs_dir = scratch / \"logs_dir\"\n logs_dir.mkdir()\n ci.copy_stage_logs_to_artifacts(concrete_spec, str(logs_dir))\n assert \"spack-build-out.txt.gz\" in os.listdir(logs_dir)\n\n\ndef test_push_to_build_cache_exceptions(monkeypatch, tmp_path: pathlib.Path, capfd):\n def push_or_raise(*args, **kwargs):\n raise spack.binary_distribution.PushToBuildCacheError(\"Error: Access Denied\")\n\n monkeypatch.setattr(spack.binary_distribution.Uploader, \"push_or_raise\", push_or_raise)\n\n # Input doesn't matter, as we are faking exceptional output\n url = tmp_path.as_uri()\n ci.push_to_build_cache(spack.spec.Spec(), url, False)\n assert f\"Problem writing to {url}: Error: Access Denied\" in capfd.readouterr().err\n\n\n@pytest.mark.parametrize(\"match_behavior\", [\"first\", \"merge\"])\n@pytest.mark.parametrize(\"git_version\", [\"big ol commit sha\", None])\ndef test_ci_generate_override_runner_attrs(\n ci_generate_test, tmp_path: pathlib.Path, monkeypatch, match_behavior, git_version\n):\n \"\"\"Test that we get the behavior we want with respect to the provision\n of runner attributes like tags, variables, and scripts, both when we\n inherit them from the top level, as well as when we override one or\n more at the runner level\"\"\"\n monkeypatch.setattr(spack, \"spack_version\", \"0.20.0.test0\")\n monkeypatch.setattr(spack, \"get_version\", lambda: \"0.20.0.test0 (blah)\")\n monkeypatch.setattr(spack, \"get_spack_commit\", lambda: git_version)\n spack_yaml, outputfile, _ = ci_generate_test(\n f\"\"\"\\\nspack:\n specs:\n - dependent-install\n - pkg-a\n mirrors:\n buildcache-destination: {tmp_path / \"ci-mirror\"}\n ci:\n pipeline-gen:\n - match_behavior: {match_behavior}\n submapping:\n - match:\n - dependent-install\n build-job:\n tags:\n - specific-one\n variables:\n THREE: specificvarthree\n - match:\n - dependency-install\n - match:\n - pkg-a\n build-job:\n tags:\n - specific-a-2\n - match:\n - pkg-a\n build-job-remove:\n tags:\n - toplevel2\n build-job:\n tags:\n - specific-a\n variables:\n ONE: specificvarone\n TWO: specificvartwo\n before_script::\n - - custom pre step one\n script::\n - - custom main step\n after_script::\n - custom post step one\n - build-job:\n tags:\n - toplevel\n - toplevel2\n variables:\n ONE: toplevelvarone\n TWO: toplevelvartwo\n before_script:\n - - pre step one\n - pre step two\n script::\n - - main step\n after_script:\n - - post step one\n - cleanup-job:\n image: donotcare\n tags: [donotcare]\n\"\"\"\n )\n\n yaml_contents = syaml.load(outputfile.read_text())\n\n assert \"variables\" in yaml_contents\n global_vars = yaml_contents[\"variables\"]\n assert \"SPACK_VERSION\" in global_vars\n assert global_vars[\"SPACK_VERSION\"] == \"0.20.0.test0 (blah)\"\n assert \"SPACK_CHECKOUT_VERSION\" in global_vars\n assert global_vars[\"SPACK_CHECKOUT_VERSION\"] == git_version or \"v0.20.0.test0\"\n\n for ci_key in yaml_contents.keys():\n if ci_key.startswith(\"pkg-a\"):\n # Make sure pkg-a's attributes override variables, and all the\n # scripts. Also, make sure the 'toplevel' tag doesn't\n # appear twice, but that a's specific extra tag does appear\n the_elt = yaml_contents[ci_key]\n assert the_elt[\"variables\"][\"ONE\"] == \"specificvarone\"\n assert the_elt[\"variables\"][\"TWO\"] == \"specificvartwo\"\n assert \"THREE\" not in the_elt[\"variables\"]\n assert len(the_elt[\"tags\"]) == (2 if match_behavior == \"first\" else 3)\n assert \"specific-a\" in the_elt[\"tags\"]\n if match_behavior == \"merge\":\n assert \"specific-a-2\" in the_elt[\"tags\"]\n assert \"toplevel\" in the_elt[\"tags\"]\n assert \"toplevel2\" not in the_elt[\"tags\"]\n assert len(the_elt[\"before_script\"]) == 1\n assert the_elt[\"before_script\"][0] == \"custom pre step one\"\n assert len(the_elt[\"script\"]) == 1\n assert the_elt[\"script\"][0] == \"custom main step\"\n assert len(the_elt[\"after_script\"]) == 1\n assert the_elt[\"after_script\"][0] == \"custom post step one\"\n if \"dependency-install\" in ci_key:\n # Since the dependency-install match omits any\n # runner-attributes, make sure it inherited all the\n # top-level attributes.\n the_elt = yaml_contents[ci_key]\n assert the_elt[\"variables\"][\"ONE\"] == \"toplevelvarone\"\n assert the_elt[\"variables\"][\"TWO\"] == \"toplevelvartwo\"\n assert \"THREE\" not in the_elt[\"variables\"]\n assert len(the_elt[\"tags\"]) == 2\n assert \"toplevel\" in the_elt[\"tags\"]\n assert \"toplevel2\" in the_elt[\"tags\"]\n assert len(the_elt[\"before_script\"]) == 2\n assert the_elt[\"before_script\"][0] == \"pre step one\"\n assert the_elt[\"before_script\"][1] == \"pre step two\"\n assert len(the_elt[\"script\"]) == 1\n assert the_elt[\"script\"][0] == \"main step\"\n assert len(the_elt[\"after_script\"]) == 1\n assert the_elt[\"after_script\"][0] == \"post step one\"\n if \"dependent-install\" in ci_key:\n # The dependent-install match specifies that we keep the two\n # top level variables, but add a third specific one. It\n # also adds a custom tag which should be combined with\n # the top-level tag.\n the_elt = yaml_contents[ci_key]\n assert the_elt[\"variables\"][\"ONE\"] == \"toplevelvarone\"\n assert the_elt[\"variables\"][\"TWO\"] == \"toplevelvartwo\"\n assert the_elt[\"variables\"][\"THREE\"] == \"specificvarthree\"\n assert len(the_elt[\"tags\"]) == 3\n assert \"specific-one\" in the_elt[\"tags\"]\n assert \"toplevel\" in the_elt[\"tags\"]\n assert \"toplevel2\" in the_elt[\"tags\"]\n assert len(the_elt[\"before_script\"]) == 2\n assert the_elt[\"before_script\"][0] == \"pre step one\"\n assert the_elt[\"before_script\"][1] == \"pre step two\"\n assert len(the_elt[\"script\"]) == 1\n assert the_elt[\"script\"][0] == \"main step\"\n assert len(the_elt[\"after_script\"]) == 1\n assert the_elt[\"after_script\"][0] == \"post step one\"\n\n\ndef test_ci_rebuild_index(\n tmp_path: pathlib.Path,\n working_env,\n mutable_mock_env_path,\n install_mockery,\n mock_fetch,\n mock_binary_index,\n):\n scratch = tmp_path / \"working_dir\"\n mirror_dir = scratch / \"mirror\"\n mirror_url = mirror_dir.as_uri()\n\n with open(tmp_path / \"spack.yaml\", \"w\", encoding=\"utf-8\") as f:\n f.write(\n f\"\"\"\nspack:\n specs:\n - callpath\n mirrors:\n buildcache-destination: {mirror_url}\n ci:\n pipeline-gen:\n - submapping:\n - match:\n - patchelf\n build-job:\n tags:\n - donotcare\n image: donotcare\n\"\"\"\n )\n\n with working_dir(tmp_path):\n env_cmd(\"create\", \"test\", \"./spack.yaml\")\n with ev.read(\"test\"):\n concrete_spec = spack.concretize.concretize_one(\"callpath\")\n with open(tmp_path / \"spec.json\", \"w\", encoding=\"utf-8\") as f:\n f.write(concrete_spec.to_json(hash=ht.dag_hash))\n\n install_cmd(\"--fake\", str(tmp_path / \"spec.json\"))\n buildcache_cmd(\"push\", \"-u\", \"-f\", mirror_url, \"callpath\")\n ci_cmd(\"rebuild-index\")\n\n output = buildcache_cmd(\"list\", \"-L\", \"--allarch\")\n assert concrete_spec.dag_hash() + \" callpath\" in output\n\n\ndef test_ci_get_stack_changed(mock_git_repo, monkeypatch):\n \"\"\"Test that we can detect the change to .gitlab-ci.yml in a\n mock spack git repo.\"\"\"\n monkeypatch.setattr(spack.paths, \"prefix\", mock_git_repo)\n fake_env_path = os.path.join(\n spack.paths.prefix, os.path.sep.join((\"no\", \"such\", \"env\", \"path\"))\n )\n assert ci.stack_changed(fake_env_path) is True\n\n\ndef test_ci_generate_prune_untouched(\n ci_generate_test, monkeypatch, tmp_path: pathlib.Path, repo_builder: RepoBuilder\n):\n \"\"\"Test pipeline generation with pruning works to eliminate\n specs that were not affected by a change\"\"\"\n monkeypatch.setenv(\"SPACK_PRUNE_UNTOUCHED\", \"TRUE\") # enables pruning of untouched specs\n\n def fake_compute_affected(repo, rev1=None, rev2=None):\n if \"mock\" in os.path.basename(repo.root):\n return [\"libdwarf\"]\n else:\n return [\"pkg-c\"]\n\n def fake_stack_changed(env_path):\n return False\n\n def fake_change_revisions(env_path):\n return \"HEAD^\", \"HEAD\"\n\n repo_builder.add_package(\"pkg-a\", dependencies=[(\"pkg-b\", None, None)])\n repo_builder.add_package(\"pkg-b\", dependencies=[(\"pkg-c\", None, None)])\n repo_builder.add_package(\"pkg-c\")\n repo_builder.add_package(\"pkg-d\")\n\n monkeypatch.setattr(ci, \"compute_affected_packages\", fake_compute_affected)\n monkeypatch.setattr(ci, \"stack_changed\", fake_stack_changed)\n monkeypatch.setattr(ci, \"get_change_revisions\", fake_change_revisions)\n\n with spack.repo.use_repositories(repo_builder.root, override=False):\n spack_yaml, outputfile, _ = ci_generate_test(\n f\"\"\"\\\nspack:\n specs:\n - archive-files\n - callpath\n - pkg-a\n - pkg-d\n mirrors:\n buildcache-destination: {tmp_path / \"ci-mirror\"}\n ci:\n pipeline-gen:\n - build-job:\n tags:\n - donotcare\n image: donotcare\n\"\"\"\n )\n\n # Dependency graph rooted at callpath\n # callpath -> dyninst -> libelf\n # -> libdwarf -> libelf\n # -> mpich\n env_hashes = {}\n with ev.read(\"test\") as active_env:\n active_env.concretize()\n for s in active_env.all_specs():\n env_hashes[s.name] = s.dag_hash()\n\n yaml_contents = syaml.load(outputfile.read_text())\n\n generated_hashes = []\n for ci_key in yaml_contents.keys():\n if \"variables\" in yaml_contents[ci_key]:\n generated_hashes.append(yaml_contents[ci_key][\"variables\"][\"SPACK_JOB_SPEC_DAG_HASH\"])\n\n assert env_hashes[\"archive-files\"] not in generated_hashes\n assert env_hashes[\"pkg-d\"] not in generated_hashes\n for spec_name in [\n \"callpath\",\n \"dyninst\",\n \"mpich\",\n \"libdwarf\",\n \"libelf\",\n \"pkg-a\",\n \"pkg-b\",\n \"pkg-c\",\n ]:\n assert env_hashes[spec_name] in generated_hashes\n\n\ndef test_ci_subcommands_without_mirror(\n tmp_path: pathlib.Path,\n mutable_mock_env_path,\n install_mockery,\n ci_base_environment,\n mock_binary_index,\n):\n \"\"\"Make sure we catch if there is not a mirror and report an error\"\"\"\n with open(tmp_path / \"spack.yaml\", \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n specs:\n - archive-files\n ci:\n pipeline-gen:\n - submapping:\n - match:\n - archive-files\n build-job:\n tags:\n - donotcare\n image: donotcare\n\"\"\"\n )\n\n with working_dir(tmp_path):\n env_cmd(\"create\", \"test\", \"./spack.yaml\")\n\n with ev.read(\"test\"):\n # Check the 'generate' subcommand\n expect = \"spack ci generate requires a mirror named 'buildcache-destination'\"\n with pytest.raises(ci.SpackCIError, match=expect):\n ci_cmd(\"generate\", \"--output-file\", str(tmp_path / \".gitlab-ci.yml\"))\n\n # Also check the 'rebuild-index' subcommand\n output = ci_cmd(\"rebuild-index\", fail_on_error=False)\n assert \"spack ci rebuild-index requires an env containing a mirror\" in output\n\n\ndef test_ci_generate_read_broken_specs_url(\n tmp_path: pathlib.Path,\n mutable_mock_env_path,\n install_mockery,\n mock_packages,\n monkeypatch,\n ci_base_environment,\n):\n \"\"\"Verify that `broken-specs-url` works as intended\"\"\"\n spec_a = spack.concretize.concretize_one(\"pkg-a\")\n a_dag_hash = spec_a.dag_hash()\n\n spec_flattendeps = spack.concretize.concretize_one(\"dependent-install\")\n flattendeps_dag_hash = spec_flattendeps.dag_hash()\n\n broken_specs_url = tmp_path.as_uri()\n\n # Mark 'a' as broken (but not 'dependent-install')\n broken_spec_a_url = \"{0}/{1}\".format(broken_specs_url, a_dag_hash)\n job_stack = \"job_stack\"\n a_job_url = \"a_job_url\"\n ci.write_broken_spec(\n broken_spec_a_url, spec_a.name, job_stack, a_job_url, \"pipeline_url\", spec_a.to_dict()\n )\n\n # Test that `spack ci generate` notices this broken spec and fails.\n with open(tmp_path / \"spack.yaml\", \"w\", encoding=\"utf-8\") as f:\n f.write(\n f\"\"\"\\\nspack:\n specs:\n - dependent-install\n - pkg-a\n mirrors:\n buildcache-destination: {(tmp_path / \"ci-mirror\").as_uri()}\n ci:\n broken-specs-url: \"{broken_specs_url}\"\n pipeline-gen:\n - submapping:\n - match:\n - pkg-a\n - dependent-install\n - pkg-b\n - dependency-install\n build-job:\n tags:\n - donotcare\n image: donotcare\n\"\"\"\n )\n\n with working_dir(tmp_path):\n env_cmd(\"create\", \"test\", \"./spack.yaml\")\n with ev.read(\"test\"):\n # Check output of the 'generate' subcommand\n output = ci_cmd(\"generate\", fail_on_error=False)\n assert \"known to be broken\" in output\n\n expected = (\n f\"{spec_a.name}/{a_dag_hash[:7]} (in stack {job_stack}) was \"\n f\"reported broken here: {a_job_url}\"\n )\n assert expected in output\n\n not_expected = f\"dependent-install/{flattendeps_dag_hash[:7]} (in stack\"\n assert not_expected not in output\n\n\ndef test_ci_generate_external_signing_job(\n install_mockery, ci_generate_test, tmp_path: pathlib.Path, monkeypatch\n):\n \"\"\"Verify that in external signing mode: 1) each rebuild jobs includes\n the location where the binary hash information is written and 2) we\n properly generate a final signing job in the pipeline.\"\"\"\n monkeypatch.setenv(\"SPACK_PIPELINE_TYPE\", \"spack_protected_branch\")\n _, outputfile, _ = ci_generate_test(\n f\"\"\"\\\nspack:\n specs:\n - archive-files\n mirrors:\n buildcache-destination: {(tmp_path / \"ci-mirror\").as_uri()}\n ci:\n pipeline-gen:\n - submapping:\n - match:\n - archive-files\n build-job:\n tags:\n - donotcare\n image: donotcare\n - signing-job:\n tags:\n - nonbuildtag\n - secretrunner\n image:\n name: customdockerimage\n entrypoint: []\n variables:\n IMPORTANT_INFO: avalue\n script::\n - echo hello\n custom_attribute: custom!\n\"\"\"\n )\n yaml_contents = syaml.load(outputfile.read_text())\n\n assert \"sign-pkgs\" in yaml_contents\n signing_job = yaml_contents[\"sign-pkgs\"]\n assert \"tags\" in signing_job\n signing_job_tags = signing_job[\"tags\"]\n for expected_tag in [\"notary\", \"protected\", \"aws\"]:\n assert expected_tag in signing_job_tags\n assert signing_job[\"custom_attribute\"] == \"custom!\"\n\n\ndef test_ci_reproduce(\n tmp_path: pathlib.Path,\n mutable_mock_env_path,\n install_mockery,\n monkeypatch,\n last_two_git_commits,\n ci_base_environment,\n mock_binary_index,\n):\n repro_dir = tmp_path / \"repro_dir\"\n image_name = \"org/image:tag\"\n\n with open(tmp_path / \"spack.yaml\", \"w\", encoding=\"utf-8\") as f:\n f.write(\n f\"\"\"\nspack:\n definitions:\n - packages: [archive-files]\n specs:\n - $packages\n mirrors:\n buildcache-destination: {tmp_path / \"ci-mirror\"}\n ci:\n pipeline-gen:\n - submapping:\n - match:\n - archive-files\n build-job:\n tags:\n - donotcare\n image: {image_name}\n\"\"\"\n )\n\n with working_dir(tmp_path), ev.Environment(\".\") as env:\n env.concretize()\n env.write()\n\n def fake_download_and_extract_artifacts(url, work_dir, merge_commit_test=True):\n with working_dir(tmp_path), ev.Environment(\".\") as env:\n if not os.path.exists(repro_dir):\n repro_dir.mkdir()\n\n job_spec = env.concrete_roots()[0]\n with open(repro_dir / \"archivefiles.json\", \"w\", encoding=\"utf-8\") as f:\n f.write(job_spec.to_json(hash=ht.dag_hash))\n artifacts_root = repro_dir / \"jobs_scratch_dir\"\n pipeline_path = artifacts_root / \"pipeline.yml\"\n\n ci_cmd(\n \"generate\",\n \"--output-file\",\n str(pipeline_path),\n \"--artifacts-root\",\n str(artifacts_root),\n )\n\n job_name = gitlab_generator.get_job_name(job_spec)\n\n with open(repro_dir / \"repro.json\", \"w\", encoding=\"utf-8\") as f:\n f.write(\n json.dumps(\n {\n \"job_name\": job_name,\n \"job_spec_json\": \"archivefiles.json\",\n \"ci_project_dir\": str(repro_dir),\n }\n )\n )\n\n with open(repro_dir / \"install.sh\", \"w\", encoding=\"utf-8\") as f:\n f.write(\"#!/bin/sh\\n\\n#fake install\\nspack install blah\\n\")\n\n with open(repro_dir / \"spack_info.txt\", \"w\", encoding=\"utf-8\") as f:\n if merge_commit_test:\n f.write(\n f\"\\nMerge {last_two_git_commits[1]} into {last_two_git_commits[0]}\\n\\n\"\n )\n else:\n f.write(f\"\\ncommit {last_two_git_commits[1]}\\n\\n\")\n\n return \"jobs_scratch_dir\"\n\n monkeypatch.setattr(ci, \"download_and_extract_artifacts\", fake_download_and_extract_artifacts)\n rep_out = ci_cmd(\n \"reproduce-build\",\n \"https://example.com/api/v1/projects/1/jobs/2/artifacts\",\n \"--working-dir\",\n str(repro_dir),\n )\n # Make sure the script was generated\n assert (repro_dir / \"start.sh\").exists()\n\n # Make sure we tell the user where it is when not in interactive mode\n assert f\"$ {repro_dir}/start.sh\" in rep_out\n\n # Ensure the correct commits are used\n assert f\"checkout_commit: {last_two_git_commits[0]}\" in rep_out\n assert f\"merge_commit: {last_two_git_commits[1]}\" in rep_out\n\n # Test re-running in dirty working dir\n with pytest.raises(SpackError, match=f\"{repro_dir}\"):\n rep_out = ci_cmd(\n \"reproduce-build\",\n \"https://example.com/api/v1/projects/1/jobs/2/artifacts\",\n \"--working-dir\",\n str(repro_dir),\n )\n\n # Cleanup between tests\n shutil.rmtree(repro_dir)\n\n # Test --use-local-head\n rep_out = ci_cmd(\n \"reproduce-build\",\n \"https://example.com/api/v1/projects/1/jobs/2/artifacts\",\n \"--use-local-head\",\n \"--working-dir\",\n str(repro_dir),\n )\n\n # Make sure we are checkout out the HEAD commit without a merge commit\n assert \"checkout_commit: HEAD\" in rep_out\n assert \"merge_commit: None\" in rep_out\n\n # Test the case where the spack_info.txt is not a merge commit\n monkeypatch.setattr(\n ci,\n \"download_and_extract_artifacts\",\n lambda url, wd: fake_download_and_extract_artifacts(url, wd, False),\n )\n\n # Cleanup between tests\n shutil.rmtree(repro_dir)\n\n rep_out = ci_cmd(\n \"reproduce-build\",\n \"https://example.com/api/v1/projects/1/jobs/2/artifacts\",\n \"--working-dir\",\n str(repro_dir),\n )\n # Make sure the script was generated\n assert (repro_dir / \"start.sh\").exists()\n\n # Make sure we tell the user where it is when not in interactive mode\n assert f\"$ {repro_dir}/start.sh\" in rep_out\n\n # Ensure the correct commit is used (different than HEAD)\n assert f\"checkout_commit: {last_two_git_commits[1]}\" in rep_out\n assert \"merge_commit: None\" in rep_out\n\n\n@pytest.mark.parametrize(\n \"url_in,url_out\",\n [\n (\n \"https://example.com/api/v4/projects/1/jobs/2/artifacts\",\n \"https://example.com/api/v4/projects/1/jobs/2/artifacts\",\n ),\n (\n \"https://example.com/spack/spack/-/jobs/123456/artifacts/download\",\n \"https://example.com/spack/spack/-/jobs/123456/artifacts/download\",\n ),\n (\n \"https://example.com/spack/spack/-/jobs/123456\",\n \"https://example.com/spack/spack/-/jobs/123456/artifacts/download\",\n ),\n (\n \"https://example.com/spack/spack/-/jobs/////123456////?x=y#z\",\n \"https://example.com/spack/spack/-/jobs/123456/artifacts/download\",\n ),\n ],\n)\ndef test_reproduce_build_url_validation(url_in, url_out):\n assert spack.cmd.ci._gitlab_artifacts_url(url_in) == url_out\n\n\ndef test_reproduce_build_url_validation_fails():\n \"\"\"Wrong URLs should cause an exception\"\"\"\n with pytest.raises(spack.main.SpackCommandError):\n ci_cmd(\"reproduce-build\", \"example.com/spack/spack/-/jobs/123456/artifacts/download\")\n\n with pytest.raises(spack.main.SpackCommandError):\n ci_cmd(\"reproduce-build\", \"https://example.com/spack/spack/-/issues\")\n\n with pytest.raises(spack.main.SpackCommandError):\n ci_cmd(\"reproduce-build\", \"https://example.com/spack/spack/-\")\n\n\n@pytest.mark.parametrize(\n \"subcmd\", [(\"\"), (\"generate\"), (\"rebuild-index\"), (\"rebuild\"), (\"reproduce-build\")]\n)\ndef test_ci_help(subcmd):\n \"\"\"Make sure `spack ci` --help describes the (sub)command help.\"\"\"\n out = spack.main.SpackCommand(\"ci\")(subcmd, \"--help\", fail_on_error=False)\n\n usage = \" ci {0}{1}[\".format(subcmd, \" \" if subcmd else \"\")\n assert usage in out\n\n\ndef test_docstring_utils():\n def example_function():\n \"\"\"\\\n this is the first line\n\n this is not the first line\n \"\"\"\n pass\n\n assert spack.cmd.doc_first_line(example_function) == \"this is the first line\"\n assert spack.cmd.doc_dedented(example_function) == (\n \"this is the first line\\n\\nthis is not the first line\\n\"\n )\n\n\ndef test_gitlab_config_scopes(install_mockery, ci_generate_test, tmp_path: pathlib.Path):\n \"\"\"Test pipeline generation with included configs\"\"\"\n # Create an included config scope\n configs_path = tmp_path / \"gitlab\" / \"configs\"\n configs_path.mkdir(parents=True, exist_ok=True)\n with open(configs_path / \"ci.yaml\", \"w\", encoding=\"utf-8\") as fd:\n fd.write(\n \"\"\"\nci:\n pipeline-gen:\n - reindex-job:\n variables:\n CI_JOB_SIZE: small\n KUBERNETES_CPU_REQUEST: 10\n KUBERNETES_MEMORY_REQUEST: 100\n tags: [\"spack\", \"service\"]\n\"\"\"\n )\n\n rel_configs_path = configs_path.relative_to(tmp_path)\n manifest, outputfile, _ = ci_generate_test(\n f\"\"\"\\\nspack:\n config:\n install_tree:\n root: {tmp_path / \"opt\"}\n include:\n - {rel_configs_path}\n - path: {rel_configs_path}\n - {configs_path}\n - when: 'False'\n path: https://dummy.io\n view: false\n specs:\n - dependent-install\n mirrors:\n buildcache-destination: {tmp_path / \"ci-mirror\"}\n ci:\n pipeline-gen:\n - build-job:\n image: \"ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01\"\n tags: [\"some_tag\"]\n\"\"\"\n )\n\n yaml_contents = syaml.load(outputfile.read_text())\n\n assert \"rebuild-index\" in yaml_contents\n\n rebuild_job = yaml_contents[\"rebuild-index\"]\n assert \"tags\" in rebuild_job\n assert \"variables\" in rebuild_job\n\n rebuild_tags = rebuild_job[\"tags\"]\n rebuild_vars = rebuild_job[\"variables\"]\n assert all([t in rebuild_tags for t in [\"spack\", \"service\"]])\n expected_vars = [\"CI_JOB_SIZE\", \"KUBERNETES_CPU_REQUEST\", \"KUBERNETES_MEMORY_REQUEST\"]\n assert all([v in rebuild_vars for v in expected_vars])\n\n # Read the concrete environment and ensure the relative path was updated\n conc_env_path = tmp_path / \"jobs_scratch_dir\" / \"concrete_environment\"\n conc_env_manifest = conc_env_path / \"spack.yaml\"\n\n env_manifest = syaml.load(conc_env_manifest.read_text())\n assert \"include\" in env_manifest[\"spack\"]\n\n # Ensure relative path include correctly updated\n # Ensure the relocated concrete env includes point to the same location\n rel_conc_path = env_manifest[\"spack\"][\"include\"][0]\n abs_conc_path = (conc_env_path / rel_conc_path).absolute().resolve()\n assert str(abs_conc_path) == os.path.join(ev.as_env_dir(\"test\"), \"gitlab\", \"configs\")\n\n # Ensure relative path include with \"path\" correctly updated\n # Ensure the relocated concrete env includes point to the same location\n rel_conc_path = env_manifest[\"spack\"][\"include\"][1][\"path\"]\n abs_conc_path = (conc_env_path / rel_conc_path).absolute().resolve()\n assert str(abs_conc_path) == os.path.join(ev.as_env_dir(\"test\"), \"gitlab\", \"configs\")\n\n # Ensure absolute path is unchanged\n # Ensure the relocated concrete env includes point to the same location\n abs_config_path = env_manifest[\"spack\"][\"include\"][2]\n assert str(abs_config_path) == str(configs_path)\n\n # Ensure URL path is unchanged\n url_config_path = env_manifest[\"spack\"][\"include\"][3][\"path\"]\n assert str(url_config_path) == \"https://dummy.io\"\n\n\ndef test_ci_generate_mirror_config(\n tmp_path: pathlib.Path,\n mutable_mock_env_path,\n install_mockery,\n monkeypatch,\n ci_base_environment,\n mock_binary_index,\n):\n \"\"\"Make sure the correct mirror gets used as the buildcache destination\"\"\"\n fst, snd = (tmp_path / \"first\").as_uri(), (tmp_path / \"second\").as_uri()\n with open(tmp_path / \"spack.yaml\", \"w\", encoding=\"utf-8\") as f:\n f.write(\n f\"\"\"\\\nspack:\n specs:\n - archive-files\n mirrors:\n some-mirror: {fst}\n buildcache-destination: {snd}\n ci:\n pipeline-gen:\n - submapping:\n - match:\n - archive-files\n build-job:\n tags:\n - donotcare\n image: donotcare\n\"\"\"\n )\n\n with ev.Environment(tmp_path):\n ci_cmd(\"generate\", \"--output-file\", str(tmp_path / \".gitlab-ci.yml\"))\n\n with open(tmp_path / \".gitlab-ci.yml\", encoding=\"utf-8\") as f:\n pipeline_doc = syaml.load(f)\n assert fst not in pipeline_doc[\"rebuild-index\"][\"script\"][0]\n assert snd in pipeline_doc[\"rebuild-index\"][\"script\"][0]\n\n\ndef dynamic_mapping_setup(tmp_path: pathlib.Path):\n filename = str(tmp_path / \"spack.yaml\")\n with open(filename, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n specs:\n - pkg-a\n mirrors:\n buildcache-destination: https://my.fake.mirror\n ci:\n pipeline-gen:\n - dynamic-mapping:\n endpoint: https://fake.spack.io/mapper\n require: [\"variables\"]\n ignore: [\"ignored_field\"]\n allow: [\"variables\", \"retry\"]\n\"\"\"\n )\n\n spec_a = spack.concretize.concretize_one(\"pkg-a\")\n\n return gitlab_generator.get_job_name(spec_a)\n\n\ndef test_ci_dynamic_mapping_empty(\n tmp_path: pathlib.Path,\n working_env,\n mutable_mock_env_path,\n install_mockery,\n mock_packages,\n monkeypatch,\n ci_base_environment,\n):\n # The test will always return an empty dictionary\n def _urlopen(*args, **kwargs):\n return MockHTTPResponse.with_json(200, \"OK\", headers={}, body={})\n\n monkeypatch.setattr(ci.common, \"_urlopen\", _urlopen)\n\n _ = dynamic_mapping_setup(tmp_path)\n with working_dir(str(tmp_path)):\n env_cmd(\"create\", \"test\", \"./spack.yaml\")\n outputfile = str(tmp_path / \".gitlab-ci.yml\")\n\n with ev.read(\"test\"):\n output = ci_cmd(\"generate\", \"--output-file\", outputfile)\n assert \"Response missing required keys: ['variables']\" in output\n\n\ndef test_ci_dynamic_mapping_full(\n tmp_path: pathlib.Path,\n working_env,\n mutable_mock_env_path,\n install_mockery,\n mock_packages,\n monkeypatch,\n ci_base_environment,\n):\n def _urlopen(*args, **kwargs):\n return MockHTTPResponse.with_json(\n 200,\n \"OK\",\n headers={},\n body={\"variables\": {\"MY_VAR\": \"hello\"}, \"ignored_field\": 0, \"unallowed_field\": 0},\n )\n\n monkeypatch.setattr(ci.common, \"_urlopen\", _urlopen)\n\n label = dynamic_mapping_setup(tmp_path)\n with working_dir(str(tmp_path)):\n env_cmd(\"create\", \"test\", \"./spack.yaml\")\n outputfile = str(tmp_path / \".gitlab-ci.yml\")\n\n with ev.read(\"test\"):\n ci_cmd(\"generate\", \"--output-file\", outputfile)\n\n with open(outputfile, encoding=\"utf-8\") as of:\n pipeline_doc = syaml.load(of.read())\n assert label in pipeline_doc\n job = pipeline_doc[label]\n\n assert job.get(\"variables\", {}).get(\"MY_VAR\") == \"hello\"\n assert \"ignored_field\" not in job\n assert \"unallowed_field\" not in job\n\n\ndef test_ci_generate_unknown_generator(\n ci_generate_test,\n tmp_path: pathlib.Path,\n mutable_mock_env_path,\n install_mockery,\n mock_packages,\n ci_base_environment,\n):\n \"\"\"Ensure unrecognized ci targets are detected and the user\n sees an intelligible and actionable message\"\"\"\n src_mirror_url = tmp_path / \"ci-src-mirror\"\n bin_mirror_url = tmp_path / \"ci-bin-mirror\"\n spack_yaml_contents = f\"\"\"\nspack:\n specs:\n - archive-files\n mirrors:\n some-mirror: {src_mirror_url}\n buildcache-destination: {bin_mirror_url}\n ci:\n target: unknown\n pipeline-gen:\n - submapping:\n - match:\n - archive-files\n build-job:\n tags:\n - donotcare\n image: donotcare\n\"\"\"\n expect = \"Spack CI module cannot generate a pipeline for format unknown\"\n with pytest.raises(ci.SpackCIError, match=expect):\n ci_generate_test(spack_yaml_contents)\n\n\ndef test_ci_generate_copy_only(\n ci_generate_test,\n tmp_path: pathlib.Path,\n monkeypatch,\n mutable_mock_env_path,\n install_mockery,\n mock_packages,\n ci_base_environment,\n):\n \"\"\"Ensure the correct jobs are generated for a copy-only pipeline,\n and verify that pipeline manifest is produced containing the right\n number of entries.\"\"\"\n src_mirror_url = tmp_path / \"ci-src-mirror\"\n bin_mirror_url = tmp_path / \"ci-bin-mirror\"\n copy_mirror_url = tmp_path / \"ci-copy-mirror\"\n\n monkeypatch.setenv(\"SPACK_PIPELINE_TYPE\", \"spack_copy_only\")\n monkeypatch.setenv(\"SPACK_COPY_BUILDCACHE\", copy_mirror_url)\n\n spack_yaml_contents = f\"\"\"\nspack:\n specs:\n - archive-files\n mirrors:\n buildcache-source:\n fetch: {src_mirror_url}\n push: {src_mirror_url}\n source: False\n binary: True\n buildcache-destination:\n fetch: {bin_mirror_url}\n push: {bin_mirror_url}\n source: False\n binary: True\n ci:\n target: gitlab\n pipeline-gen:\n - submapping:\n - match:\n - archive-files\n build-job:\n tags:\n - donotcare\n image: donotcare\n\"\"\"\n _, output_file, _ = ci_generate_test(spack_yaml_contents)\n\n with open(output_file, encoding=\"utf-8\") as of:\n pipeline_doc = syaml.load(of.read())\n\n expected_keys = [\"copy\", \"rebuild-index\", \"stages\", \"variables\", \"workflow\"]\n assert all([k in pipeline_doc for k in expected_keys])\n\n # Make sure there are only two jobs and two stages\n stages = pipeline_doc[\"stages\"]\n copy_stage = \"copy\"\n rebuild_index_stage = \"stage-rebuild-index\"\n\n assert len(stages) == 2\n assert stages[0] == copy_stage\n assert stages[1] == rebuild_index_stage\n\n rebuild_index_job = pipeline_doc[\"rebuild-index\"]\n assert rebuild_index_job[\"stage\"] == rebuild_index_stage\n\n copy_job = pipeline_doc[\"copy\"]\n assert copy_job[\"stage\"] == copy_stage\n\n # Make sure a pipeline manifest was generated\n output_directory = os.path.dirname(output_file)\n assert \"SPACK_ARTIFACTS_ROOT\" in pipeline_doc[\"variables\"]\n artifacts_root = pipeline_doc[\"variables\"][\"SPACK_ARTIFACTS_ROOT\"]\n pipeline_manifest_path = os.path.join(\n output_directory, artifacts_root, \"specs_to_copy\", \"copy_rebuilt_specs.json\"\n )\n\n assert os.path.exists(pipeline_manifest_path)\n assert os.path.isfile(pipeline_manifest_path)\n\n with open(pipeline_manifest_path, encoding=\"utf-8\") as fd:\n manifest_data = json.load(fd)\n\n with ev.read(\"test\") as active_env:\n active_env.concretize()\n for s in active_env.all_specs():\n assert s.dag_hash() in manifest_data\n\n\n@generator(\"unittestgenerator\")\ndef generate_unittest_pipeline(\n pipeline: PipelineDag, spack_ci: SpackCIConfig, options: PipelineOptions\n):\n \"\"\"Define a custom pipeline generator for the target 'unittestgenerator'.\"\"\"\n output_file = options.output_file\n assert output_file is not None\n with open(output_file, \"w\", encoding=\"utf-8\") as fd:\n fd.write(\"unittestpipeline\\n\")\n for _, node in pipeline.traverse_nodes(direction=\"children\"):\n release_spec = node.spec\n fd.write(f\" {release_spec.name}\\n\")\n\n\ndef test_ci_generate_alternate_target(\n ci_generate_test,\n tmp_path: pathlib.Path,\n mutable_mock_env_path,\n install_mockery,\n mock_packages,\n ci_base_environment,\n):\n \"\"\"Ensure the above pipeline generator was correctly registered and\n is used to generate a pipeline for the stack/config defined here.\"\"\"\n bin_mirror_url = tmp_path / \"ci-bin-mirror\"\n\n spack_yaml_contents = f\"\"\"\nspack:\n specs:\n - archive-files\n - externaltest\n mirrors:\n buildcache-destination: {bin_mirror_url}\n ci:\n target: unittestgenerator\n pipeline-gen:\n - submapping:\n - match:\n - archive-files\n build-job:\n tags:\n - donotcare\n image: donotcare\n\"\"\"\n _, output_file, _ = ci_generate_test(spack_yaml_contents, \"--no-prune-externals\")\n\n with open(output_file, encoding=\"utf-8\") as of:\n pipeline_doc = of.read()\n\n assert pipeline_doc.startswith(\"unittestpipeline\")\n assert \"externaltest\" in pipeline_doc\n\n\ndef test_ci_generate_forward_variables(\n ci_generate_test,\n tmp_path: pathlib.Path,\n mutable_mock_env_path,\n install_mockery,\n mock_packages,\n ci_base_environment,\n):\n \"\"\"Ensure the above pipeline generator was correctly registered and\n is used to generate a pipeline for the stack/config defined here.\"\"\"\n bin_mirror_url = tmp_path / \"ci-bin-mirror\"\n\n spack_yaml_contents = f\"\"\"\nspack:\n specs:\n - archive-files\n - externaltest\n mirrors:\n buildcache-destination: {bin_mirror_url}\n ci:\n target: gitlab\n pipeline-gen:\n - submapping:\n - match:\n - archive-files\n build-job:\n tags:\n - donotcare\n image: donotcare\n\"\"\"\n noforward_vars = [\"NO_FORWARD_VAR\"]\n forward_vars = [\"TEST_VAR\", \"ANOTHER_TEST_VAR\"]\n for v in forward_vars + noforward_vars:\n os.environ[v] = f\"{v}_BEEF\"\n\n fwd_arg = \" --forward-variable \"\n _, output_file, _ = ci_generate_test(\n spack_yaml_contents, fwd_arg.strip(), *fwd_arg.join(forward_vars).split()\n )\n\n with open(output_file, encoding=\"utf-8\") as fd:\n pipeline_yaml = syaml.load(fd.read())\n\n for v in forward_vars:\n assert v in pipeline_yaml[\"variables\"]\n assert pipeline_yaml[\"variables\"][v] == f\"{v}_BEEF\"\n\n for v in noforward_vars:\n assert v not in pipeline_yaml[\"variables\"]\n\n\n@pytest.fixture\ndef fetch_versions_match(monkeypatch):\n \"\"\"Fake successful checksums returned from downloaded tarballs.\"\"\"\n\n def get_checksums_for_versions(url_by_version, package_name, **kwargs):\n pkg_cls = spack.repo.PATH.get_pkg_class(package_name)\n return {v: pkg_cls.versions[v][\"sha256\"] for v in url_by_version}\n\n monkeypatch.setattr(spack.stage, \"get_checksums_for_versions\", get_checksums_for_versions)\n monkeypatch.setattr(spack.util.web, \"url_exists\", lambda url: True)\n\n\n@pytest.fixture\ndef fetch_versions_invalid(monkeypatch):\n \"\"\"Fake successful checksums returned from downloaded tarballs.\"\"\"\n\n def get_checksums_for_versions(url_by_version, package_name, **kwargs):\n return {\n v: \"abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890\"\n for v in url_by_version\n }\n\n monkeypatch.setattr(spack.stage, \"get_checksums_for_versions\", get_checksums_for_versions)\n monkeypatch.setattr(spack.util.web, \"url_exists\", lambda url: True)\n\n\n@pytest.mark.parametrize(\"versions\", [[\"2.1.4\"], [\"2.1.4\", \"2.1.5\"]])\ndef test_ci_validate_standard_versions_valid(capfd, mock_packages, fetch_versions_match, versions):\n spec = spack.spec.Spec(\"diff-test\")\n pkg = spack.repo.PATH.get_pkg_class(spec.name)(spec)\n version_list = [spack.version.Version(v) for v in versions]\n\n assert spack.cmd.ci.validate_standard_versions(pkg, version_list)\n\n out, err = capfd.readouterr()\n for version in versions:\n assert f\"Validated diff-test@{version}\" in out\n\n\n@pytest.mark.parametrize(\"versions\", [[\"2.1.4\"], [\"2.1.4\", \"2.1.5\"]])\ndef test_ci_validate_standard_versions_invalid(\n capfd, mock_packages, fetch_versions_invalid, versions\n):\n spec = spack.spec.Spec(\"diff-test\")\n pkg = spack.repo.PATH.get_pkg_class(spec.name)(spec)\n version_list = [spack.version.Version(v) for v in versions]\n\n assert spack.cmd.ci.validate_standard_versions(pkg, version_list) is False\n\n out, err = capfd.readouterr()\n for version in versions:\n assert f\"Invalid checksum found diff-test@{version}\" in err\n\n\n@pytest.mark.parametrize(\"versions\", [[(\"1.0\", -2)], [(\"1.1\", -4), (\"2.0\", -6)]])\ndef test_ci_validate_git_versions_valid(\n capfd, monkeypatch, mock_packages, mock_git_version_info, versions\n):\n spec = spack.spec.Spec(\"diff-test\")\n pkg_class = spack.repo.PATH.get_pkg_class(spec.name)\n pkg = pkg_class(spec)\n version_list = [spack.version.Version(v) for v, _ in versions]\n\n repo_path, filename, commits = mock_git_version_info\n version_commit_dict = {\n spack.version.Version(v): {\"tag\": f\"v{v}\", \"commit\": commits[c]} for v, c in versions\n }\n\n monkeypatch.setattr(pkg_class, \"git\", repo_path)\n monkeypatch.setattr(pkg_class, \"versions\", version_commit_dict)\n\n assert spack.cmd.ci.validate_git_versions(pkg, version_list)\n\n out, err = capfd.readouterr()\n for version in version_list:\n assert f\"Validated diff-test@{version}\" in out\n\n\n@pytest.mark.parametrize(\"versions\", [[(\"1.0\", -3)], [(\"1.1\", -5), (\"2.0\", -5)]])\ndef test_ci_validate_git_versions_bad_tag(\n capfd, monkeypatch, mock_packages, mock_git_version_info, versions\n):\n spec = spack.spec.Spec(\"diff-test\")\n pkg_class = spack.repo.PATH.get_pkg_class(spec.name)\n pkg = pkg_class(spec)\n version_list = [spack.version.Version(v) for v, _ in versions]\n\n repo_path, filename, commits = mock_git_version_info\n version_commit_dict = {\n spack.version.Version(v): {\"tag\": f\"v{v}\", \"commit\": commits[c]} for v, c in versions\n }\n\n monkeypatch.setattr(pkg_class, \"git\", repo_path)\n monkeypatch.setattr(pkg_class, \"versions\", version_commit_dict)\n\n assert spack.cmd.ci.validate_git_versions(pkg, version_list) is False\n\n out, err = capfd.readouterr()\n for version in version_list:\n assert f\"Mismatched tag <-> commit found for diff-test@{version}\" in err\n\n\n@pytest.mark.parametrize(\"versions\", [[(\"1.0\", -2)], [(\"1.1\", -4), (\"2.0\", -6), (\"3.0\", -6)]])\ndef test_ci_validate_git_versions_invalid(\n capfd, monkeypatch, mock_packages, mock_git_version_info, versions\n):\n spec = spack.spec.Spec(\"diff-test\")\n pkg_class = spack.repo.PATH.get_pkg_class(spec.name)\n pkg = pkg_class(spec)\n version_list = [spack.version.Version(v) for v, _ in versions]\n\n repo_path, filename, commits = mock_git_version_info\n version_commit_dict = {\n spack.version.Version(v): {\n \"tag\": f\"v{v}\",\n \"commit\": \"abcdefabcdefabcdefabcdefabcdefabcdefabc\",\n }\n for v, c in versions\n }\n\n monkeypatch.setattr(pkg_class, \"git\", repo_path)\n monkeypatch.setattr(pkg_class, \"versions\", version_commit_dict)\n\n assert spack.cmd.ci.validate_git_versions(pkg, version_list) is False\n\n out, err = capfd.readouterr()\n for version in version_list:\n assert f\"Invalid commit for diff-test@{version}\" in err\n\n\ndef mock_packages_path(path):\n def packages_path():\n return path\n\n return packages_path\n\n\n@pytest.fixture\ndef verify_standard_versions_valid(monkeypatch):\n def validate_standard_versions(pkg, versions):\n for version in versions:\n print(f\"Validated {pkg.name}@{version}\")\n return True\n\n monkeypatch.setattr(spack.cmd.ci, \"validate_standard_versions\", validate_standard_versions)\n\n\n@pytest.fixture\ndef verify_git_versions_valid(monkeypatch):\n def validate_git_versions(pkg, versions):\n for version in versions:\n print(f\"Validated {pkg.name}@{version}\")\n return True\n\n monkeypatch.setattr(spack.cmd.ci, \"validate_git_versions\", validate_git_versions)\n\n\n@pytest.fixture\ndef verify_standard_versions_invalid(monkeypatch):\n def validate_standard_versions(pkg, versions):\n for version in versions:\n print(f\"Invalid checksum found {pkg.name}@{version}\")\n return False\n\n monkeypatch.setattr(spack.cmd.ci, \"validate_standard_versions\", validate_standard_versions)\n\n\n@pytest.fixture\ndef verify_standard_versions_invalid_duplicates(monkeypatch):\n def validate_standard_versions(pkg, versions):\n for version in versions:\n if str(version) == \"2.1.7\":\n print(f\"Validated {pkg.name}@{version}\")\n else:\n print(f\"Invalid checksum found {pkg.name}@{version}\")\n return False\n\n monkeypatch.setattr(spack.cmd.ci, \"validate_standard_versions\", validate_standard_versions)\n\n\n@pytest.fixture\ndef verify_git_versions_invalid(monkeypatch):\n def validate_git_versions(pkg, versions):\n for version in versions:\n print(f\"Invalid commit for {pkg.name}@{version}\")\n return False\n\n monkeypatch.setattr(spack.cmd.ci, \"validate_git_versions\", validate_git_versions)\n\n\ndef test_ci_verify_versions_valid(\n monkeypatch,\n mock_packages,\n mock_git_package_changes,\n verify_standard_versions_valid,\n verify_git_versions_valid,\n):\n repo, _, commits = mock_git_package_changes\n with spack.repo.use_repositories(repo):\n monkeypatch.setattr(spack.repo, \"builtin_repo\", lambda: repo)\n\n out = ci_cmd(\"verify-versions\", commits[-1], commits[-3])\n assert \"Validated diff-test@2.1.5\" in out\n assert \"Validated diff-test@2.1.6\" in out\n\n\ndef test_ci_verify_versions_invalid(\n monkeypatch,\n mock_packages,\n mock_git_package_changes,\n verify_standard_versions_invalid,\n verify_git_versions_invalid,\n):\n repo, _, commits = mock_git_package_changes\n with spack.repo.use_repositories(repo):\n monkeypatch.setattr(spack.repo, \"builtin_repo\", lambda: repo)\n\n out = ci_cmd(\"verify-versions\", commits[-1], commits[-3], fail_on_error=False)\n assert \"Invalid checksum found diff-test@2.1.5\" in out\n assert \"Invalid commit for diff-test@2.1.6\" in out\n\n\ndef test_ci_verify_versions_standard_duplicates(\n monkeypatch,\n mock_packages,\n mock_git_package_changes,\n verify_standard_versions_invalid_duplicates,\n):\n repo, _, commits = mock_git_package_changes\n with spack.repo.use_repositories(repo):\n monkeypatch.setattr(spack.repo, \"builtin_repo\", lambda: repo)\n\n out = ci_cmd(\"verify-versions\", commits[-3], commits[-4], fail_on_error=False)\n print(f\"'{out}'\")\n assert \"Validated diff-test@2.1.7\" in out\n assert \"Invalid checksum found diff-test@2.1.8\" in out\n\n\ndef test_ci_verify_versions_manual_package(monkeypatch, mock_packages, mock_git_package_changes):\n repo, _, commits = mock_git_package_changes\n with spack.repo.use_repositories(repo):\n monkeypatch.setattr(spack.repo, \"builtin_repo\", lambda: repo)\n\n pkg_class = spack.repo.PATH.get_pkg_class(\"diff-test\")\n monkeypatch.setattr(pkg_class, \"manual_download\", True)\n\n out = ci_cmd(\"verify-versions\", commits[-1], commits[-2])\n assert \"Skipping manual download package: diff-test\" in out\n" }, { "path": "lib/spack/spack/test/cmd/clean.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport pathlib\n\nimport pytest\n\nimport spack.caches\nimport spack.cmd.clean\nimport spack.llnl.util.filesystem as fs\nimport spack.main\nimport spack.package_base\nimport spack.stage\nimport spack.store\n\nclean = spack.main.SpackCommand(\"clean\")\n\n\n@pytest.fixture()\ndef mock_calls_for_clean(monkeypatch):\n counts = {}\n\n class Counter:\n def __init__(self, name):\n self.name = name\n counts[name] = 0\n\n def __call__(self, *args, **kwargs):\n counts[self.name] += 1\n\n monkeypatch.setattr(spack.package_base.PackageBase, \"do_clean\", Counter(\"package\"))\n monkeypatch.setattr(spack.stage, \"purge\", Counter(\"stages\"))\n monkeypatch.setattr(spack.caches.FETCH_CACHE, \"destroy\", Counter(\"downloads\"), raising=False)\n monkeypatch.setattr(spack.caches.MISC_CACHE, \"destroy\", Counter(\"caches\"))\n monkeypatch.setattr(spack.store.STORE.failure_tracker, \"clear_all\", Counter(\"failures\"))\n monkeypatch.setattr(spack.cmd.clean, \"remove_python_cache\", Counter(\"python_cache\"))\n monkeypatch.setattr(spack.cmd.clean, \"remove_python_cache\", Counter(\"python_cache\"))\n monkeypatch.setattr(fs, \"remove_directory_contents\", Counter(\"bootstrap\"))\n\n yield counts\n\n\nall_effects = [\"stages\", \"downloads\", \"caches\", \"failures\", \"python_cache\", \"bootstrap\"]\n\n\n@pytest.mark.usefixtures(\"mock_packages\")\n@pytest.mark.parametrize(\n \"command_line,effects\",\n [\n (\"mpileaks\", [\"package\"]),\n (\"-s\", [\"stages\"]),\n (\"-sd\", [\"stages\", \"downloads\"]),\n (\"-m\", [\"caches\"]),\n (\"-f\", [\"failures\"]),\n (\"-p\", [\"python_cache\"]),\n (\"-a\", all_effects),\n (\"\", []),\n ],\n)\ndef test_function_calls(command_line, effects, mock_calls_for_clean, mutable_config):\n mutable_config.set(\"bootstrap\", {\"root\": \"fake\"})\n\n # Call the command with the supplied command line\n clean(command_line)\n\n # Assert that we called the expected functions the correct\n # number of times\n for name in [\"package\"] + all_effects:\n assert mock_calls_for_clean[name] == (1 if name in effects else 0)\n\n\ndef test_remove_python_cache(tmp_path: pathlib.Path, monkeypatch):\n cache_files = [\"file1.pyo\", \"file2.pyc\"]\n source_file = \"file1.py\"\n\n def _setup_files(directory):\n # Create a python cache and source file.\n cache_dir = fs.join_path(directory, \"__pycache__\")\n fs.mkdirp(cache_dir)\n fs.touch(fs.join_path(directory, source_file))\n fs.touch(fs.join_path(directory, cache_files[0]))\n for filename in cache_files:\n # Ensure byte code files in python cache directory\n fs.touch(fs.join_path(cache_dir, filename))\n\n def _check_files(directory):\n # Ensure the python cache created by _setup_files is removed\n # and the source file is not.\n assert os.path.exists(fs.join_path(directory, source_file))\n assert not os.path.exists(fs.join_path(directory, cache_files[0]))\n assert not os.path.exists(fs.join_path(directory, \"__pycache__\"))\n\n source_dir = fs.join_path(str(tmp_path), \"lib\", \"spack\", \"spack\")\n var_dir = fs.join_path(str(tmp_path), \"var\", \"spack\", \"stuff\")\n\n for d in [source_dir, var_dir]:\n _setup_files(d)\n\n # Patching the path variables from-import'd by spack.cmd.clean is needed\n # to ensure the paths used by the command for this test reflect the\n # temporary directory locations and not those from spack.paths when\n # the clean command's module was imported.\n monkeypatch.setattr(spack.cmd.clean, \"lib_path\", source_dir)\n monkeypatch.setattr(spack.cmd.clean, \"var_path\", var_dir)\n\n spack.cmd.clean.remove_python_cache()\n\n for d in [source_dir, var_dir]:\n _check_files(d)\n" }, { "path": "lib/spack/spack/test/cmd/commands.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport filecmp\nimport os\nimport pathlib\nimport shutil\nimport sys\nimport textwrap\n\nimport pytest\n\nimport spack.cmd\nimport spack.cmd.commands\nimport spack.config\nimport spack.main\nimport spack.paths\nfrom spack.cmd.commands import _dest_to_fish_complete, _positional_to_subroutine\nfrom spack.util.executable import Executable\n\n\ndef commands(*args: str) -> str:\n \"\"\"Run `spack commands args...` and return output as a string. It's a separate process so that\n we run through the main Spack command logic and avoid caching issues.\"\"\"\n python = Executable(sys.executable)\n return python(spack.paths.spack_script, \"commands\", *args, output=str)\n\n\ndef test_names():\n \"\"\"Test default output of spack commands.\"\"\"\n out1 = commands().strip().splitlines()\n assert out1 == spack.cmd.all_commands()\n assert \"rm\" not in out1\n\n out2 = commands(\"--aliases\").strip().splitlines()\n assert out1 != out2\n assert \"rm\" in out2\n\n out3 = commands(\"--format=names\").strip().splitlines()\n assert out1 == out3\n\n\ndef test_subcommands():\n \"\"\"Test subcommand traversal.\"\"\"\n out1 = commands(\"--format=subcommands\")\n assert \"spack mirror create\" in out1\n assert \"spack buildcache list\" in out1\n assert \"spack repo add\" in out1\n assert \"spack pkg diff\" in out1\n assert \"spack url parse\" in out1\n assert \"spack view symlink\" in out1\n assert \"spack rm\" not in out1\n assert \"spack compiler add\" not in out1\n\n out2 = commands(\"--aliases\", \"--format=subcommands\")\n assert \"spack mirror create\" in out2\n assert \"spack buildcache list\" in out2\n assert \"spack repo add\" in out2\n assert \"spack pkg diff\" in out2\n assert \"spack url parse\" in out2\n assert \"spack view symlink\" in out2\n assert \"spack rm\" in out2\n assert \"spack compiler add\" in out2\n\n\ndef test_alias_overrides_builtin(mutable_config: spack.config.Configuration, capfd):\n \"\"\"Test that spack commands cannot be overridden by aliases.\"\"\"\n mutable_config.set(\"config:aliases\", {\"install\": \"find\"})\n cmd, args = spack.main.resolve_alias(\"install\", [\"install\", \"-v\"])\n assert cmd == \"install\" and args == [\"install\", \"-v\"]\n out = capfd.readouterr().err\n assert \"Alias 'install' (mapping to 'find') attempts to override built-in command\" in out\n\n\ndef test_alias_with_space(mutable_config: spack.config.Configuration, capfd):\n \"\"\"Test that spack aliases with spaces are rejected.\"\"\"\n mutable_config.set(\"config:aliases\", {\"foo bar\": \"find\"})\n cmd, args = spack.main.resolve_alias(\"install\", [\"install\", \"-v\"])\n assert cmd == \"install\" and args == [\"install\", \"-v\"]\n out = capfd.readouterr().err\n assert \"Alias 'foo bar' (mapping to 'find') contains a space, which is not supported\" in out\n\n\ndef test_alias_resolves_properly(mutable_config: spack.config.Configuration):\n \"\"\"Test that spack aliases resolve properly.\"\"\"\n mutable_config.set(\"config:aliases\", {\"my_find\": \"find\"})\n cmd, args = spack.main.resolve_alias(\"my_find\", [\"my_find\", \"-v\"])\n assert cmd == \"find\" and args == [\"find\", \"-v\"]\n\n\ndef test_rst():\n \"\"\"Do some simple sanity checks of the rst writer.\"\"\"\n out1 = commands(\"--format=rst\")\n assert \"spack mirror create\" in out1\n assert \"spack buildcache list\" in out1\n assert \"spack repo add\" in out1\n assert \"spack pkg diff\" in out1\n assert \"spack url parse\" in out1\n assert \"spack view symlink\" in out1\n assert \"spack rm\" not in out1\n assert \"spack compiler add\" not in out1\n\n out2 = commands(\"--aliases\", \"--format=rst\")\n assert \"spack mirror create\" in out2\n assert \"spack buildcache list\" in out2\n assert \"spack repo add\" in out2\n assert \"spack pkg diff\" in out2\n assert \"spack url parse\" in out2\n assert \"spack view symlink\" in out2\n assert \"spack rm\" in out2\n assert \"spack compiler add\" in out2\n\n\ndef test_rst_with_input_files(tmp_path: pathlib.Path):\n filename = tmp_path / \"file.rst\"\n with filename.open(\"w\") as f:\n f.write(\n \"\"\"\n.. _cmd-spack-fetch:\ncmd-spack-list:\n.. _cmd-spack-stage:\n_cmd-spack-install:\n.. _cmd-spack-patch:\n\"\"\"\n )\n\n out = commands(\"--format=rst\", str(filename))\n for name in [\"fetch\", \"stage\", \"patch\"]:\n assert (\":ref:`More documentation `\" % name) in out\n\n for name in [\"list\", \"install\"]:\n assert (\":ref:`More documentation `\" % name) not in out\n\n\ndef test_rst_with_header(tmp_path: pathlib.Path):\n local_commands = spack.main.SpackCommand(\"commands\")\n fake_header = \"this is a header!\\n\\n\"\n\n filename = tmp_path / \"header.txt\"\n with filename.open(\"w\") as f:\n f.write(fake_header)\n\n out = local_commands(\"--format=rst\", \"--header\", str(filename))\n assert out.startswith(fake_header)\n\n with pytest.raises(spack.main.SpackCommandError):\n local_commands(\"--format=rst\", \"--header\", \"asdfjhkf\")\n\n\ndef test_rst_update(tmp_path: pathlib.Path):\n update_file = tmp_path / \"output\"\n\n commands(\"--update\", str(update_file))\n assert update_file.exists()\n\n\ndef test_update_with_header(tmp_path: pathlib.Path):\n update_file = tmp_path / \"output\"\n\n commands(\"--update\", str(update_file))\n assert update_file.exists()\n fake_header = \"this is a header!\\n\\n\"\n\n filename = tmp_path / \"header.txt\"\n with filename.open(\"w\") as f:\n f.write(fake_header)\n\n commands(\"--update\", str(update_file), \"--header\", str(filename))\n\n\ndef test_bash_completion():\n \"\"\"Test the bash completion writer.\"\"\"\n out1 = commands(\"--format=bash\")\n\n # Make sure header not included\n assert \"_bash_completion_spack() {\" not in out1\n assert \"_all_packages() {\" not in out1\n\n # Make sure subcommands appear\n assert \"_spack_remove() {\" in out1\n assert \"_spack_compiler_find() {\" in out1\n\n # Make sure aliases don't appear\n assert \"_spack_rm() {\" not in out1\n assert \"_spack_compiler_add() {\" not in out1\n\n # Make sure options appear\n assert \"-h --help\" in out1\n\n # Make sure subcommands are called\n for function in _positional_to_subroutine.values():\n assert function in out1\n\n out2 = commands(\"--aliases\", \"--format=bash\")\n\n # Make sure aliases appear\n assert \"_spack_rm() {\" in out2\n assert \"_spack_compiler_add() {\" in out2\n\n\ndef test_bash_completion_choices():\n \"\"\"Test that bash completion includes choices for positional arguments.\"\"\"\n out = commands(\"--format=bash\")\n\n # `spack env view` has a positional `action` with choices\n assert 'SPACK_COMPREPLY=\"disable enable regenerate\"' in out\n\n\ndef test_fish_completion():\n \"\"\"Test the fish completion writer.\"\"\"\n out1 = commands(\"--format=fish\")\n\n # Make sure header not included\n assert \"function __fish_spack_argparse\" not in out1\n assert \"complete -c spack --erase\" not in out1\n\n # Make sure subcommands appear\n assert \"__fish_spack_using_command remove\" in out1\n assert \"__fish_spack_using_command compiler find\" in out1\n\n # Make sure aliases don't appear\n assert \"__fish_spack_using_command rm\" not in out1\n assert \"__fish_spack_using_command compiler add\" not in out1\n\n # Make sure options appear\n assert \"-s h -l help\" in out1\n\n # Make sure subcommands are called\n for complete_cmd in _dest_to_fish_complete.values():\n assert complete_cmd in out1\n\n out2 = commands(\"--aliases\", \"--format=fish\")\n\n # Make sure aliases appear\n assert \"__fish_spack_using_command rm\" in out2\n assert \"__fish_spack_using_command compiler add\" in out2\n\n\n@pytest.mark.parametrize(\"shell\", [\"bash\", \"fish\"])\ndef test_update_completion_arg(shell, tmp_path: pathlib.Path, monkeypatch):\n \"\"\"Test the update completion flag.\"\"\"\n\n (tmp_path / shell).mkdir()\n mock_infile = tmp_path / shell / f\"spack-completion.{shell}\"\n mock_outfile = tmp_path / f\"spack-completion.{shell}\"\n\n mock_args = {\n shell: {\n \"aliases\": True,\n \"format\": shell,\n \"header\": str(mock_infile),\n \"update\": str(mock_outfile),\n }\n }\n\n # make a mock completion file missing the --update-completion argument\n real_args = spack.cmd.commands.update_completion_args\n shutil.copy(real_args[shell][\"header\"], mock_args[shell][\"header\"])\n with open(real_args[shell][\"update\"], encoding=\"utf-8\") as old:\n old_file = old.read()\n with open(mock_args[shell][\"update\"], \"w\", encoding=\"utf-8\") as mock:\n mock.write(old_file.replace(\"update-completion\", \"\"))\n\n monkeypatch.setattr(spack.cmd.commands, \"update_completion_args\", mock_args)\n\n local_commands = spack.main.SpackCommand(\"commands\")\n # ensure things fail if --update-completion isn't specified alone\n with pytest.raises(spack.main.SpackCommandError):\n local_commands(\"--update-completion\", \"-a\")\n\n # ensure arg is restored\n assert \"update-completion\" not in mock_outfile.read_text()\n local_commands(\"--update-completion\")\n assert \"update-completion\" in mock_outfile.read_text()\n\n\n# Note: this test is never expected to be supported on Windows\n@pytest.mark.not_on_windows(\"Shell completion script generator fails on windows\")\n@pytest.mark.parametrize(\"shell\", [\"bash\", \"fish\"])\ndef test_updated_completion_scripts(shell, tmp_path: pathlib.Path):\n \"\"\"Make sure our shell tab completion scripts remain up-to-date.\"\"\"\n\n width = 72\n lines = textwrap.wrap(\n \"It looks like Spack's command-line interface has been modified. \"\n \"If differences are more than your global 'include:' scopes, please \"\n \"update Spack's shell tab completion scripts by running:\",\n width,\n )\n lines.append(\"\\n spack commands --update-completion\\n\")\n lines.extend(\n textwrap.wrap(\n \"and adding the changed files (minus your global 'include:' scopes) \"\n \"to your pull request.\",\n width,\n )\n )\n msg = \"\\n\".join(lines)\n\n header = os.path.join(spack.paths.share_path, shell, f\"spack-completion.{shell}\")\n script = f\"spack-completion.{shell}\"\n old_script = os.path.join(spack.paths.share_path, script)\n new_script = str(tmp_path / script)\n\n commands(\"--aliases\", \"--format\", shell, \"--header\", header, \"--update\", new_script)\n\n if not filecmp.cmp(old_script, new_script):\n # If there is a diff, something is wrong: in that case output what the diff is.\n import difflib\n\n with open(old_script, \"r\", encoding=\"utf-8\") as f1, open(\n new_script, \"r\", encoding=\"utf-8\"\n ) as f2:\n l1 = f1.readlines()\n l2 = f2.readlines()\n diff = difflib.unified_diff(l1, l2, fromfile=old_script, tofile=new_script)\n msg += \"\\nDiff failure:\\n\\n\" + \"\".join(diff)\n raise AssertionError(msg)\n" }, { "path": "lib/spack/spack/test/cmd/common/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n" }, { "path": "lib/spack/spack/test/cmd/common/arguments.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\n\nimport pytest\n\nimport spack.cmd\nimport spack.cmd.common.arguments as arguments\nimport spack.config\nimport spack.environment as ev\nimport spack.main\n\n\n@pytest.fixture()\ndef job_parser():\n # --jobs needs to write to a command_line config scope, so this is the only\n # scope we create.\n p = argparse.ArgumentParser()\n arguments.add_common_arguments(p, [\"jobs\"])\n scopes = [spack.config.InternalConfigScope(\"command_line\", {\"config\": {}})]\n\n with spack.config.use_configuration(*scopes):\n yield p\n\n\ndef test_setting_jobs_flag(job_parser):\n namespace = job_parser.parse_args([\"-j\", \"24\"])\n assert namespace.jobs == 24\n assert spack.config.get(\"config:build_jobs\", scope=\"command_line\") == 24\n\n\ndef test_omitted_job_flag(job_parser):\n namespace = job_parser.parse_args([])\n assert namespace.jobs is None\n assert spack.config.get(\"config:build_jobs\") is None\n\n\ndef test_negative_integers_not_allowed_for_parallel_jobs(job_parser):\n with pytest.raises(ValueError) as exc_info:\n job_parser.parse_args([\"-j\", \"-2\"])\n\n assert \"expected a positive integer\" in str(exc_info.value)\n\n\n@pytest.mark.parametrize(\n \"specs,cflags,propagation,negated_variants\",\n [\n (['coreutils cflags=\"-O3 -g\"'], [\"-O3\", \"-g\"], [False, False], []),\n (['coreutils cflags==\"-O3 -g\"'], [\"-O3\", \"-g\"], [True, True], []),\n ([\"coreutils\", \"cflags=-O3 -g\"], [\"-O3\", \"-g\"], [False, False], []),\n ([\"coreutils\", \"cflags==-O3 -g\"], [\"-O3\", \"-g\"], [True, True], []),\n ([\"coreutils\", \"cflags=-O3\", \"-g\"], [\"-O3\"], [False], [\"g\"]),\n ],\n)\n@pytest.mark.regression(\"12951\")\ndef test_parse_spec_flags_with_spaces(specs, cflags, propagation, negated_variants):\n spec_list = spack.cmd.parse_specs(specs)\n assert len(spec_list) == 1\n\n s = spec_list.pop()\n\n compiler_flags = [flag for flag in s.compiler_flags[\"cflags\"]]\n flag_propagation = [flag.propagate for flag in s.compiler_flags[\"cflags\"]]\n\n assert compiler_flags == cflags\n assert flag_propagation == propagation\n assert list(s.variants.keys()) == negated_variants\n for v in negated_variants:\n assert \"~{0}\".format(v) in s\n\n\ndef test_match_spec_env(mock_packages, mutable_mock_env_path):\n \"\"\"\n Concretize a spec with non-default options in an environment. Make\n sure that when we ask for a matching spec when the environment is\n active that we get the instance concretized in the environment.\n \"\"\"\n # Initial sanity check: we are planning on choosing a non-default\n # value, so make sure that is in fact not the default.\n check_defaults = spack.cmd.parse_specs([\"pkg-a\"], concretize=True)[0]\n assert not check_defaults.satisfies(\"foobar=baz\")\n\n e = ev.create(\"test\")\n e.add(\"pkg-a foobar=baz\")\n e.concretize()\n with e:\n env_spec = spack.cmd.matching_spec_from_env(spack.cmd.parse_specs([\"pkg-a\"])[0])\n assert env_spec.satisfies(\"foobar=baz\")\n assert env_spec.concrete\n\n\ndef test_multiple_env_match_raises_error(mock_packages, mutable_mock_env_path):\n e = ev.create(\"test\")\n e.add(\"pkg-a foobar=baz\")\n e.add(\"pkg-a foobar=fee\")\n e.concretize()\n with e:\n with pytest.raises(ev.SpackEnvironmentError) as exc_info:\n spack.cmd.matching_spec_from_env(spack.cmd.parse_specs([\"pkg-a\"])[0])\n\n assert \"matches multiple specs\" in exc_info.value.message\n\n\ndef test_root_and_dep_match_returns_root(mock_packages, mutable_mock_env_path):\n e = ev.create(\"test\")\n e.add(\"pkg-b@0.9\")\n e.add(\"pkg-a foobar=bar\") # Depends on b, should choose b@1.0\n e.concretize()\n with e:\n # This query matches the root b and b as a dependency of a. In that\n # case the root instance should be preferred.\n env_spec1 = spack.cmd.matching_spec_from_env(spack.cmd.parse_specs([\"pkg-b\"])[0])\n assert env_spec1.satisfies(\"@0.9\")\n\n env_spec2 = spack.cmd.matching_spec_from_env(spack.cmd.parse_specs([\"pkg-b@1.0\"])[0])\n assert env_spec2\n\n\n@pytest.mark.parametrize(\n \"arg,conf\",\n [\n (\"--reuse\", True),\n (\"--fresh\", False),\n (\"--reuse-deps\", \"dependencies\"),\n (\"--fresh-roots\", \"dependencies\"),\n ],\n)\ndef test_concretizer_arguments(mutable_config, mock_packages, arg, conf):\n \"\"\"Ensure that ConfigSetAction is doing the right thing.\"\"\"\n spec = spack.main.SpackCommand(\"spec\")\n\n assert spack.config.get(\"concretizer:reuse\", None, scope=\"command_line\") is None\n\n spec(arg, \"zlib\")\n\n assert spack.config.get(\"concretizer:reuse\", None) == conf\n assert spack.config.get(\"concretizer:reuse\", None, scope=\"command_line\") == conf\n\n\ndef test_use_buildcache_type():\n assert arguments.use_buildcache(\"only\") == (\"only\", \"only\")\n assert arguments.use_buildcache(\"never\") == (\"never\", \"never\")\n assert arguments.use_buildcache(\"auto\") == (\"auto\", \"auto\")\n assert arguments.use_buildcache(\"package:never,dependencies:only\") == (\"never\", \"only\")\n assert arguments.use_buildcache(\"only,package:never\") == (\"never\", \"only\")\n assert arguments.use_buildcache(\"package:only,package:never\") == (\"never\", \"auto\")\n assert arguments.use_buildcache(\"auto , package: only\") == (\"only\", \"auto\")\n\n with pytest.raises(argparse.ArgumentTypeError):\n assert arguments.use_buildcache(\"pkg:only,deps:never\")\n\n with pytest.raises(argparse.ArgumentTypeError):\n assert arguments.use_buildcache(\"sometimes\")\n\n\ndef test_missing_config_scopes_are_valid_scope_arguments(mock_missing_dir_include_scopes):\n \"\"\"Test that if an included scope does not have a directory or file,\n we can still specify it as a scope as an argument\"\"\"\n a = argparse.ArgumentParser()\n a.add_argument(\n \"--scope\",\n action=arguments.ConfigScope,\n default=lambda: spack.config.default_modify_scope(),\n help=\"configuration scope to modify\",\n )\n namespace = a.parse_args([\"--scope\", \"sub_base\"])\n assert namespace.scope == \"sub_base\"\n\n\ndef test_missing_config_scopes_not_valid_read_scope(mock_missing_dir_include_scopes):\n \"\"\"Ensures that if a missing include scope is the subject of a read\n operation, we fail at the argparse level\"\"\"\n a = argparse.ArgumentParser()\n a.add_argument(\n \"--scope\",\n action=arguments.ConfigScope,\n type=arguments.config_scope_readable_validator,\n default=lambda: spack.config.default_modify_scope(),\n help=\"configuration scope to modify\",\n )\n with pytest.raises(SystemExit):\n a.parse_args([\"--scope\", \"sub_base\"])\n" }, { "path": "lib/spack/spack/test/cmd/common/spec_strings.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport pathlib\n\nimport spack.cmd.common.spec_strings\n\n\ndef test_spec_strings(tmp_path: pathlib.Path):\n (tmp_path / \"example.py\").write_text(\n \"\"\"\\\ndef func(x):\n print(\"dont fix %s me\" % x, 3)\n return x.satisfies(\"+foo %gcc +bar\") and x.satisfies(\"%gcc +baz\")\n\"\"\"\n )\n (tmp_path / \"example.json\").write_text(\n \"\"\"\\\n{\n \"spec\": [\n \"+foo %gcc +bar~nope ^dep %clang +yup @3.2 target=x86_64 /abcdef ^another %gcc \",\n \"%gcc +baz\"\n ],\n \"%gcc x=y\": 2\n}\n\"\"\"\n )\n (tmp_path / \"example.yaml\").write_text(\n \"\"\"\\\nspec:\n - \"+foo %gcc +bar\"\n - \"%gcc +baz\"\n - \"this is fine %clang\"\n\"%gcc x=y\": 2\n\"\"\"\n )\n\n issues = set()\n\n def collect_issues(path: str, line: int, col: int, old: str, new: str):\n issues.add((path, line, col, old, new))\n\n # check for issues with custom handler\n spack.cmd.common.spec_strings._check_spec_strings(\n [\n str(tmp_path / \"nonexistent.py\"),\n str(tmp_path / \"example.py\"),\n str(tmp_path / \"example.json\"),\n str(tmp_path / \"example.yaml\"),\n ],\n handler=collect_issues,\n )\n\n assert issues == {\n (\n str(tmp_path / \"example.json\"),\n 3,\n 9,\n \"+foo %gcc +bar~nope ^dep %clang +yup @3.2 target=x86_64 /abcdef ^another %gcc \",\n \"+foo +bar~nope %gcc ^dep +yup @3.2 target=x86_64 /abcdef %clang ^another %gcc \",\n ),\n (str(tmp_path / \"example.json\"), 4, 9, \"%gcc +baz\", \"+baz %gcc\"),\n (str(tmp_path / \"example.json\"), 6, 5, \"%gcc x=y\", \"x=y %gcc\"),\n (str(tmp_path / \"example.py\"), 3, 23, \"+foo %gcc +bar\", \"+foo +bar %gcc\"),\n (str(tmp_path / \"example.py\"), 3, 57, \"%gcc +baz\", \"+baz %gcc\"),\n (str(tmp_path / \"example.yaml\"), 2, 5, \"+foo %gcc +bar\", \"+foo +bar %gcc\"),\n (str(tmp_path / \"example.yaml\"), 3, 5, \"%gcc +baz\", \"+baz %gcc\"),\n (str(tmp_path / \"example.yaml\"), 5, 1, \"%gcc x=y\", \"x=y %gcc\"),\n }\n\n # fix the issues in the files\n spack.cmd.common.spec_strings._check_spec_strings(\n [\n str(tmp_path / \"nonexistent.py\"),\n str(tmp_path / \"example.py\"),\n str(tmp_path / \"example.json\"),\n str(tmp_path / \"example.yaml\"),\n ],\n handler=spack.cmd.common.spec_strings._spec_str_fix_handler,\n )\n\n assert (\n (tmp_path / \"example.json\").read_text()\n == \"\"\"\\\n{\n \"spec\": [\n \"+foo +bar~nope %gcc ^dep +yup @3.2 target=x86_64 /abcdef %clang ^another %gcc \",\n \"+baz %gcc\"\n ],\n \"x=y %gcc\": 2\n}\n\"\"\"\n )\n assert (\n (tmp_path / \"example.py\").read_text()\n == \"\"\"\\\ndef func(x):\n print(\"dont fix %s me\" % x, 3)\n return x.satisfies(\"+foo +bar %gcc\") and x.satisfies(\"+baz %gcc\")\n\"\"\"\n )\n assert (\n (tmp_path / \"example.yaml\").read_text()\n == \"\"\"\\\nspec:\n - \"+foo +bar %gcc\"\n - \"+baz %gcc\"\n - \"this is fine %clang\"\n\"x=y %gcc\": 2\n\"\"\"\n )\n" }, { "path": "lib/spack/spack/test/cmd/compiler.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport os\nimport shutil\n\nimport pytest\n\nimport spack.cmd.compiler\nimport spack.compilers.config\nimport spack.config\nimport spack.main\nimport spack.util.pattern\nimport spack.version\n\ncompiler = spack.main.SpackCommand(\"compiler\")\n\npytestmark = [pytest.mark.usefixtures(\"mock_packages\")]\n\n\n@pytest.fixture\ndef compilers_dir(mock_executable):\n \"\"\"Create a directory with some mock compiler scripts in it.\n\n Scripts are:\n - clang\n - clang++\n - gcc\n - g++\n - gfortran-8\n\n \"\"\"\n clang_path = mock_executable(\n \"clang\",\n output=\"\"\"\nif [ \"$1\" = \"--version\" ]; then\n echo \"clang version 11.0.0 (clang-1100.0.33.16)\"\n echo \"Target: x86_64-apple-darwin18.7.0\"\n echo \"Thread model: posix\"\n echo \"InstalledDir: /dummy\"\nelse\n echo \"clang: error: no input files\"\n exit 1\nfi\n\"\"\",\n )\n shutil.copy(clang_path, clang_path.parent / \"clang++\")\n\n gcc_script = \"\"\"\nif [ \"$1\" = \"-dumpversion\" ]; then\n echo \"8\"\nelif [ \"$1\" = \"-dumpfullversion\" ]; then\n echo \"8.4.0\"\nelif [ \"$1\" = \"--version\" ]; then\n echo \"{0} (GCC) 8.4.0 20120313 (Red Hat 8.4.0-1)\"\n echo \"Copyright (C) 2010 Free Software Foundation, Inc.\"\nelse\n echo \"{1}: fatal error: no input files\"\n echo \"compilation terminated.\"\n exit 1\nfi\n\"\"\"\n mock_executable(\"gcc-8\", output=gcc_script.format(\"gcc\", \"gcc-8\"))\n mock_executable(\"g++-8\", output=gcc_script.format(\"g++\", \"g++-8\"))\n mock_executable(\"gfortran-8\", output=gcc_script.format(\"GNU Fortran\", \"gfortran-8\"))\n\n return clang_path.parent\n\n\n@pytest.mark.not_on_windows(\"Cannot execute bash script on Windows\")\n@pytest.mark.regression(\"11678,13138\")\ndef test_compiler_find_without_paths(no_packages_yaml, working_env, mock_executable):\n \"\"\"Tests that 'spack compiler find' looks into PATH by default, if no specific path\n is given.\n \"\"\"\n gcc_path = mock_executable(\"gcc\", output='echo \"0.0.0\"')\n\n os.environ[\"PATH\"] = str(gcc_path.parent)\n output = compiler(\"find\", \"--scope=site\")\n\n assert \"gcc\" in output\n\n\n@pytest.mark.regression(\"37996\")\ndef test_compiler_remove(mutable_config):\n \"\"\"Tests that we can remove a compiler from configuration.\"\"\"\n assert any(\n compiler.satisfies(\"gcc@=9.4.0\") for compiler in spack.compilers.config.all_compilers()\n )\n args = spack.util.pattern.Bunch(all=True, compiler_spec=\"gcc@9.4.0\", add_paths=[], scope=None)\n spack.cmd.compiler.compiler_remove(args)\n assert not any(\n compiler.satisfies(\"gcc@=9.4.0\") for compiler in spack.compilers.config.all_compilers()\n )\n\n\n@pytest.mark.regression(\"37996\")\ndef test_removing_compilers_from_multiple_scopes(mutable_config):\n # Duplicate \"site\" scope into \"user\" scope\n site_config = spack.config.get(\"packages\", scope=\"site\")\n spack.config.set(\"packages\", site_config, scope=\"user\")\n\n assert any(\n compiler.satisfies(\"gcc@=9.4.0\") for compiler in spack.compilers.config.all_compilers()\n )\n args = spack.util.pattern.Bunch(all=True, compiler_spec=\"gcc@9.4.0\", add_paths=[], scope=None)\n spack.cmd.compiler.compiler_remove(args)\n assert not any(\n compiler.satisfies(\"gcc@=9.4.0\") for compiler in spack.compilers.config.all_compilers()\n )\n\n\n@pytest.mark.not_on_windows(\"Cannot execute bash script on Windows\")\ndef test_compiler_add(mutable_config, mock_executable):\n \"\"\"Tests that we can add a compiler to configuration.\"\"\"\n expected_version = \"4.5.3\"\n gcc_path = mock_executable(\n \"gcc\",\n output=f\"\"\"\\\nfor arg in \"$@\"; do\n if [ \"$arg\" = -dumpversion ]; then\n echo '{expected_version}'\n fi\ndone\n\"\"\",\n )\n bin_dir = gcc_path.parent\n root_dir = bin_dir.parent\n\n compilers_before_find = set(spack.compilers.config.all_compilers())\n args = spack.util.pattern.Bunch(\n all=None,\n compiler_spec=None,\n add_paths=[str(root_dir)],\n scope=None,\n mixed_toolchain=False,\n jobs=1,\n )\n spack.cmd.compiler.compiler_find(args)\n compilers_after_find = set(spack.compilers.config.all_compilers())\n\n compilers_added_by_find = compilers_after_find - compilers_before_find\n assert len(compilers_added_by_find) == 1\n new_compiler = compilers_added_by_find.pop()\n assert new_compiler.version == spack.version.Version(expected_version)\n\n\n@pytest.mark.not_on_windows(\"Cannot execute bash script on Windows\")\n@pytest.mark.regression(\"17590\")\ndef test_compiler_find_prefer_no_suffix(no_packages_yaml, working_env, compilers_dir):\n \"\"\"Ensure that we'll pick 'clang' over 'clang-gpu' when there is a choice.\"\"\"\n clang_path = compilers_dir / \"clang\"\n shutil.copy(clang_path, clang_path.parent / \"clang-gpu\")\n shutil.copy(clang_path, clang_path.parent / \"clang++-gpu\")\n\n os.environ[\"PATH\"] = str(compilers_dir)\n output = compiler(\"find\", \"--scope=site\")\n\n assert \"llvm@11.0.0\" in output\n assert \"gcc@8.4.0\" in output\n\n compilers = spack.compilers.config.all_compilers_from(no_packages_yaml, scope=\"site\")\n clang = [x for x in compilers if x.satisfies(\"llvm@11\")]\n\n assert len(clang) == 1\n assert clang[0].extra_attributes[\"compilers\"][\"c\"] == str(compilers_dir / \"clang\")\n assert clang[0].extra_attributes[\"compilers\"][\"cxx\"] == str(compilers_dir / \"clang++\")\n\n\n@pytest.mark.not_on_windows(\"Cannot execute bash script on Windows\")\ndef test_compiler_find_path_order(no_packages_yaml, working_env, compilers_dir):\n \"\"\"When the same compiler version is found in two PATH directories, only the first\n entry in PATH is kept and a warning is emitted for the duplicate.\n \"\"\"\n new_dir = compilers_dir / \"first_in_path\"\n new_dir.mkdir()\n for name in (\"gcc-8\", \"g++-8\", \"gfortran-8\"):\n shutil.copy(compilers_dir / name, new_dir / name)\n # Set PATH to have the new folder searched first\n os.environ[\"PATH\"] = f\"{str(new_dir)}:{str(compilers_dir)}\"\n\n with pytest.warns(UserWarning, match=\"gcc@\"):\n compiler(\"find\", \"--scope=site\")\n\n compilers = spack.compilers.config.all_compilers(scope=\"site\")\n gcc = [x for x in compilers if x.satisfies(\"gcc@8.4\")]\n\n # Duplicate is dropped. Only the first entry in PATH is kept\n assert len(gcc) == 1\n assert gcc[0].extra_attributes[\"compilers\"] == {\n \"c\": str(new_dir / \"gcc-8\"),\n \"cxx\": str(new_dir / \"g++-8\"),\n \"fortran\": str(new_dir / \"gfortran-8\"),\n }\n\n\ndef test_compiler_list_empty(no_packages_yaml, compilers_dir, monkeypatch):\n \"\"\"Spack should not automatically search for compilers when listing them and none are\n available. And when stdout is not a tty like in tests, there should be no output and\n no error exit code.\n \"\"\"\n monkeypatch.setenv(\"PATH\", str(compilers_dir), prepend=\":\")\n out = compiler(\"list\")\n assert not out\n assert compiler.returncode == 0\n\n\n@pytest.mark.parametrize(\n \"external,expected\",\n [\n (\n {\n \"spec\": \"gcc@=7.7.7 languages=c,cxx,fortran os=foobar target=x86_64\",\n \"prefix\": \"/path/to/fake\",\n \"modules\": [\"gcc/7.7.7\", \"foobar\"],\n \"extra_attributes\": {\n \"compilers\": {\n \"c\": \"/path/to/fake/gcc\",\n \"cxx\": \"/path/to/fake/g++\",\n \"fortran\": \"/path/to/fake/gfortran\",\n },\n \"flags\": {\"fflags\": \"-ffree-form\"},\n },\n },\n \"\"\"gcc@7.7.7 languages=c,cxx,fortran os=foobar target=x86_64:\n paths:\n cc = /path/to/fake/gcc\n cxx = /path/to/fake/g++\n\\t\\tf77 = /path/to/fake/gfortran\n\\t\\tfc = /path/to/fake/gfortran\n\\tflags:\n\\t\\tfflags = ['-ffree-form']\n\\tmodules = ['gcc/7.7.7', 'foobar']\n\\toperating system = foobar\n\"\"\",\n )\n ],\n)\ndef test_compilers_shows_packages_yaml(\n external, expected, no_packages_yaml, working_env, compilers_dir\n):\n \"\"\"Spack should see a single compiler defined from packages.yaml\"\"\"\n external[\"prefix\"] = external[\"prefix\"].format(prefix=os.path.dirname(compilers_dir))\n gcc_entry = {\"externals\": [external]}\n\n packages = spack.config.get(\"packages\")\n packages[\"gcc\"] = gcc_entry\n spack.config.set(\"packages\", packages)\n\n out = compiler(\"list\", fail_on_error=True)\n assert out.count(\"gcc@7.7.7\") == 1\n" }, { "path": "lib/spack/spack/test/cmd/concretize.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\nimport pytest\n\nimport spack.environment as ev\nfrom spack import spack_version\nfrom spack.main import SpackCommand\n\npytestmark = pytest.mark.usefixtures(\"mutable_config\", \"mutable_mock_repo\")\n\nenv = SpackCommand(\"env\")\nadd = SpackCommand(\"add\")\nconcretize = SpackCommand(\"concretize\")\n\n\nunification_strategies = [False, True, \"when_possible\"]\n\n\n@pytest.mark.parametrize(\"unify\", unification_strategies)\ndef test_concretize_all_test_dependencies(unify, mutable_config, mutable_mock_env_path):\n \"\"\"Check all test dependencies are concretized.\"\"\"\n env(\"create\", \"test\")\n\n with ev.read(\"test\") as e:\n mutable_config.set(\"concretizer:unify\", unify)\n add(\"depb\")\n concretize(\"--test\", \"all\")\n assert e.matching_spec(\"test-dependency\")\n\n\n@pytest.mark.parametrize(\"unify\", unification_strategies)\ndef test_concretize_root_test_dependencies_not_recursive(\n unify, mutable_config, mutable_mock_env_path\n):\n \"\"\"Check that test dependencies are not concretized recursively.\"\"\"\n env(\"create\", \"test\")\n\n with ev.read(\"test\") as e:\n mutable_config.set(\"concretizer:unify\", unify)\n add(\"depb\")\n concretize(\"--test\", \"root\")\n assert e.matching_spec(\"test-dependency\") is None\n\n\n@pytest.mark.parametrize(\"unify\", unification_strategies)\ndef test_concretize_root_test_dependencies_are_concretized(\n unify, mutable_config, mutable_mock_env_path\n):\n \"\"\"Check that root test dependencies are concretized.\"\"\"\n env(\"create\", \"test\")\n\n with ev.read(\"test\") as e:\n mutable_config.set(\"concretizer:unify\", unify)\n add(\"pkg-a\")\n add(\"pkg-b\")\n concretize(\"--test\", \"root\")\n assert e.matching_spec(\"test-dependency\")\n\n data = e._to_lockfile_dict()\n assert data[\"spack\"][\"version\"] == spack_version\n" }, { "path": "lib/spack/spack/test/cmd/config.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport functools\nimport json\nimport os\nimport pathlib\nimport re\n\nimport pytest\n\nimport spack.concretize\nimport spack.config\nimport spack.database\nimport spack.environment as ev\nimport spack.llnl.util.filesystem as fs\nimport spack.main\nimport spack.schema.config\nimport spack.store\nimport spack.util.spack_yaml as syaml\n\nconfig = spack.main.SpackCommand(\"config\")\nenv = spack.main.SpackCommand(\"env\")\n\npytestmark = pytest.mark.usefixtures(\"mock_packages\")\n\n\ndef _create_config(scope=None, data={}, section=\"packages\"):\n scope = scope or spack.config.default_modify_scope()\n cfg_file = spack.config.CONFIG.get_config_filename(scope, section)\n with open(cfg_file, \"w\", encoding=\"utf-8\") as f:\n syaml.dump(data, stream=f)\n return cfg_file\n\n\n@pytest.fixture()\ndef config_yaml_v015(mutable_config):\n \"\"\"Create a packages.yaml in the old format\"\"\"\n old_data = {\n \"config\": {\"install_tree\": \"/fake/path\", \"install_path_scheme\": \"{name}-{version}\"}\n }\n return functools.partial(_create_config, data=old_data, section=\"config\")\n\n\nscope_path_re = r\"\\(([^\\)]+)\\)\"\n\n\n@pytest.mark.parametrize(\n \"path,types\",\n [\n (False, []),\n (True, []),\n (False, [\"path\"]),\n (False, [\"env\"]),\n (False, [\"internal\", \"include\"]),\n ],\n)\ndef test_config_scopes(path, types, mutable_mock_env_path):\n ev.create(\"test\")\n scopes_cmd = [\"scopes\"]\n if path:\n scopes_cmd.append(\"-p\")\n if types:\n scopes_cmd.extend([\"-t\", *types])\n output = config(*scopes_cmd).split()\n if not types or any(i in (\"all\", \"internal\") for i in types):\n assert \"command_line\" in output\n assert \"_builtin\" in output\n if types:\n if not any(i in (\"all\", \"path\", \"include\") for i in types):\n assert \"site\" not in output\n if not any(i in (\"all\", \"env\", \"include\", \"path\") for i in types):\n assert not output or all(\":\" not in x for x in output)\n if not any(i in (\"all\", \"env\", \"path\") for i in types):\n assert not output or all(not x.startswith(\"env:\") for x in output)\n if not any(i in (\"all\", \"internal\") for i in types):\n assert \"command_line\" not in output\n assert \"_builtin\" not in output\n if path:\n paths = (x[1] for x in (re.fullmatch(scope_path_re, s) for s in output) if x)\n assert all(os.sep in x for x in paths)\n\n\n@pytest.mark.parametrize(\"type\", [\"path\", \"include\", \"internal\", \"env\"])\ndef test_config_scopes_include(type):\n \"\"\"Ensure that `spack config scopes -vt TYPE outputs only scopes of that type.\"\"\"\n scopes_cmd = [\"scopes\", \"-vt\", type]\n output = config(*scopes_cmd).strip()\n lines = output.split(\"\\n\")\n assert not output or all([type in line for line in lines[1:]])\n\n\ndef test_config_scopes_section(mutable_config):\n scopes_cmd = [\"scopes\", \"-v\", \"packages\"]\n output = config(*scopes_cmd).strip()\n lines = output.split(\"\\n\")\n\n lines_by_scope_name = {line.split()[0]: line for line in lines}\n assert \"absent\" in lines_by_scope_name[\"command_line\"]\n assert \"absent\" in lines_by_scope_name[\"_builtin\"]\n assert \"active\" in lines_by_scope_name[\"site\"]\n\n\ndef test_include_overrides(mutable_config):\n output = config(\"scopes\").strip()\n lines = output.split(\"\\n\")\n assert \"user\" in lines\n assert \"system\" in lines\n assert \"site\" in lines\n assert \"_builtin\" in lines\n\n mutable_config.push_scope(spack.config.InternalConfigScope(\"override\", {\"include:\": []}))\n\n # overridden scopes are not shown without `-v`\n output = config(\"scopes\").strip()\n lines = output.split(\"\\n\")\n assert \"user\" not in lines\n assert \"system\" not in lines\n assert \"site\" not in lines\n\n # scopes with ConfigScopePriority.DEFAULTS remain\n assert \"_builtin\" in lines\n\n # overridden scopes are shown with `-v` and marked 'override'\n output = config(\"scopes\", \"-v\").strip()\n lines = output.split(\"\\n\")\n assert \"override\" in next(line for line in lines if line.startswith(\"user\"))\n assert \"override\" in next(line for line in lines if line.startswith(\"system\"))\n assert \"override\" in next(line for line in lines if line.startswith(\"site\"))\n\n\ndef test_blame_override(mutable_config):\n # includes are present when section is specified\n output = config(\"blame\", \"include\").strip()\n include_path = re.escape(os.path.join(mutable_config.scopes[\"site\"].path, \"include.yaml\"))\n assert re.search(rf\"{include_path}:\\d+\\s+\\- path: base\", output)\n\n # includes are also present when section is NOT specified\n output = config(\"blame\").strip()\n assert re.search(rf\"{include_path}:\\d+\\s+\\- path: base\", output)\n\n mutable_config.push_scope(spack.config.InternalConfigScope(\"override\", {\"include:\": []}))\n\n # site includes are not present when overridden\n output = config(\"blame\", \"include\").strip()\n assert not re.search(rf\"{include_path}:\\d+\\s+\\- path: base\", output)\n assert \"include: []\" in output\n\n output = config(\"blame\").strip()\n assert not re.search(rf\"{include_path}:\\d+\\s+\\- path: base\", output)\n assert \"include: []\" in output\n\n\ndef test_config_scopes_path(mutable_config):\n scopes_cmd = [\"scopes\", \"-p\"]\n output = config(*scopes_cmd).strip()\n lines = output.split(\"\\n\")\n\n lines_by_scope_name = {line.split()[0]: line for line in lines}\n assert f\"{os.sep}user{os.sep}\" in lines_by_scope_name[\"user\"]\n assert f\"{os.sep}system{os.sep}\" in lines_by_scope_name[\"system\"]\n assert f\"{os.sep}site{os.sep}\" in lines_by_scope_name[\"site\"]\n\n\ndef test_get_config_scope(mock_low_high_config):\n assert config(\"get\", \"compilers\").strip() == \"compilers: {}\"\n\n\ndef test_get_config_roundtrip(mutable_config):\n \"\"\"Test that ``spack config get [--json]
`` roundtrips correctly.\"\"\"\n json_roundtrip = json.loads(config(\"get\", \"--json\", \"config\"))\n yaml_roundtrip = syaml.load(config(\"get\", \"config\"))\n assert json_roundtrip[\"config\"] == yaml_roundtrip[\"config\"] == mutable_config.get(\"config\")\n\n\ndef test_get_all_config_roundtrip(mutable_config):\n \"\"\"Test that ``spack config get [--json]`` roundtrips correctly.\"\"\"\n json_roundtrip = json.loads(config(\"get\", \"--json\"))\n yaml_roundtrip = syaml.load(config(\"get\"))\n assert json_roundtrip == yaml_roundtrip\n for section in spack.config.SECTION_SCHEMAS:\n assert json_roundtrip[\"spack\"][section] == mutable_config.get(section)\n\n\ndef test_get_config_scope_merged(mock_low_high_config):\n low_path = mock_low_high_config.scopes[\"low\"].path\n high_path = mock_low_high_config.scopes[\"high\"].path\n\n fs.mkdirp(low_path)\n fs.mkdirp(high_path)\n\n with open(os.path.join(low_path, \"repos.yaml\"), \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nrepos:\n repo3: repo3\n\"\"\"\n )\n\n with open(os.path.join(high_path, \"repos.yaml\"), \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nrepos:\n repo1: repo1\n repo2: repo2\n\"\"\"\n )\n\n assert (\n config(\"get\", \"repos\").strip()\n == \"\"\"repos:\n repo1: repo1\n repo2: repo2\n repo3: repo3\"\"\"\n )\n\n\ndef test_config_edit(mutable_config, working_env):\n \"\"\"Ensure `spack config edit` edits the right paths.\"\"\"\n\n dms = spack.config.default_modify_scope(\"compilers\")\n dms_path = spack.config.CONFIG.scopes[dms].path\n user_path = spack.config.CONFIG.scopes[\"user\"].path\n\n comp_path = os.path.join(dms_path, \"compilers.yaml\")\n repos_path = os.path.join(user_path, \"repos.yaml\")\n\n assert config(\"edit\", \"--print-file\", \"compilers\").strip() == comp_path\n assert config(\"edit\", \"--print-file\", \"repos\").strip() == repos_path\n\n\ndef test_config_get_gets_spack_yaml(mutable_mock_env_path):\n with ev.create(\"test\") as env:\n assert \"mpileaks\" not in config(\"get\")\n env.add(\"mpileaks\")\n env.write()\n assert \"mpileaks\" in config(\"get\")\n\n\ndef test_config_edit_edits_spack_yaml(mutable_mock_env_path):\n env = ev.create(\"test\")\n with env:\n assert config(\"edit\", \"--print-file\").strip() == env.manifest_path\n\n\ndef test_config_add_with_scope_adds_to_scope(mutable_config, mutable_mock_env_path):\n \"\"\"Test adding to non-env config scope with an active environment\"\"\"\n env = ev.create(\"test\")\n with env:\n config(\"--scope=user\", \"add\", \"config:install_tree:root:/usr\")\n assert spack.config.get(\"config:install_tree:root\", scope=\"user\") == \"/usr\"\n\n\ndef test_config_edit_fails_correctly_with_no_env(mutable_mock_env_path):\n output = config(\"edit\", \"--print-file\", fail_on_error=False)\n assert \"requires a section argument or an active environment\" in output\n\n\ndef test_config_list():\n output = config(\"list\")\n assert \"compilers\" in output\n assert \"packages\" in output\n\n\ndef test_config_add(mutable_empty_config):\n config(\"add\", \"config:dirty:true\")\n output = config(\"get\", \"config\")\n\n assert (\n output\n == \"\"\"config:\n dirty: true\n\"\"\"\n )\n\n\ndef test_config_add_list(mutable_empty_config):\n config(\"add\", \"config:template_dirs:test1\")\n config(\"add\", \"config:template_dirs:[test2]\")\n config(\"add\", \"config:template_dirs:test3\")\n output = config(\"get\", \"config\")\n\n assert (\n output\n == \"\"\"config:\n template_dirs:\n - test3\n - test2\n - test1\n\"\"\"\n )\n\n\ndef test_config_add_override(mutable_empty_config):\n config(\"--scope\", \"site\", \"add\", \"config:template_dirs:test1\")\n config(\"add\", \"config:template_dirs:[test2]\")\n output = config(\"get\", \"config\")\n\n assert (\n output\n == \"\"\"config:\n template_dirs:\n - test2\n - test1\n\"\"\"\n )\n\n config(\"add\", \"config::template_dirs:[test2]\")\n output = config(\"get\", \"config\")\n\n assert (\n output\n == \"\"\"config:\n template_dirs:\n - test2\n\"\"\"\n )\n\n\ndef test_config_add_override_leaf(mutable_empty_config):\n config(\"--scope\", \"site\", \"add\", \"config:template_dirs:test1\")\n config(\"add\", \"config:template_dirs:[test2]\")\n output = config(\"get\", \"config\")\n\n assert (\n output\n == \"\"\"config:\n template_dirs:\n - test2\n - test1\n\"\"\"\n )\n\n config(\"add\", \"config:template_dirs::[test2]\")\n output = config(\"get\", \"config\")\n\n assert (\n output\n == \"\"\"config:\n 'template_dirs:':\n - test2\n\"\"\"\n )\n\n\ndef test_config_add_update_dict(mutable_empty_config):\n config(\"add\", \"packages:hdf5:version:[1.0.0]\")\n output = config(\"get\", \"packages\")\n\n expected = \"packages:\\n hdf5:\\n version: [1.0.0]\\n\"\n assert output == expected\n\n\ndef test_config_with_c_argument(mutable_empty_config):\n # I don't know how to add a spack argument to a Spack Command, so we test this way\n config_file = \"config:install_tree:root:/path/to/config.yaml\"\n parser = spack.main.make_argument_parser()\n args = parser.parse_args([\"-c\", config_file])\n assert config_file in args.config_vars\n\n # Add the path to the config\n config(\"add\", args.config_vars[0])\n output = config(\"get\", \"config\")\n assert \"config:\\n install_tree:\\n root: /path/to/config.yaml\" in output\n\n\ndef test_config_add_ordered_dict(mutable_empty_config):\n config(\"add\", \"mirrors:first:/path/to/first\")\n config(\"add\", \"mirrors:second:/path/to/second\")\n output = config(\"get\", \"mirrors\")\n\n assert (\n output\n == \"\"\"mirrors:\n first: /path/to/first\n second: /path/to/second\n\"\"\"\n )\n\n\ndef test_config_add_interpret_oneof(mutable_empty_config):\n # Regression test for a bug that would raise a validation error\n config(\"add\", \"packages:all:target:[x86_64]\")\n config(\"add\", \"packages:all:variants:~shared\")\n\n\ndef test_config_add_invalid_fails(mutable_empty_config):\n config(\"add\", \"packages:all:variants:+debug\")\n with pytest.raises((spack.config.ConfigFormatError, AttributeError)):\n config(\"add\", \"packages:all:True\")\n\n\ndef test_config_add_from_file(mutable_empty_config, tmp_path: pathlib.Path):\n contents = \"\"\"spack:\n config:\n dirty: true\n\"\"\"\n\n file = str(tmp_path / \"spack.yaml\")\n with open(file, \"w\", encoding=\"utf-8\") as f:\n f.write(contents)\n config(\"add\", \"-f\", file)\n output = config(\"get\", \"config\")\n\n assert (\n output\n == \"\"\"config:\n dirty: true\n\"\"\"\n )\n\n\ndef test_config_add_from_file_multiple(mutable_empty_config, tmp_path: pathlib.Path):\n contents = \"\"\"spack:\n config:\n dirty: true\n template_dirs: [test1]\n\"\"\"\n\n file = str(tmp_path / \"spack.yaml\")\n with open(file, \"w\", encoding=\"utf-8\") as f:\n f.write(contents)\n config(\"add\", \"-f\", file)\n output = config(\"get\", \"config\")\n\n assert (\n output\n == \"\"\"config:\n dirty: true\n template_dirs: [test1]\n\"\"\"\n )\n\n\ndef test_config_add_override_from_file(mutable_empty_config, tmp_path: pathlib.Path):\n config(\"--scope\", \"site\", \"add\", \"config:template_dirs:test1\")\n contents = \"\"\"spack:\n config::\n template_dirs: [test2]\n\"\"\"\n\n file = str(tmp_path / \"spack.yaml\")\n with open(file, \"w\", encoding=\"utf-8\") as f:\n f.write(contents)\n config(\"add\", \"-f\", file)\n output = config(\"get\", \"config\")\n\n assert (\n output\n == \"\"\"config:\n template_dirs: [test2]\n\"\"\"\n )\n\n\ndef test_config_add_override_leaf_from_file(mutable_empty_config, tmp_path: pathlib.Path):\n config(\"--scope\", \"site\", \"add\", \"config:template_dirs:test1\")\n contents = \"\"\"spack:\n config:\n template_dirs:: [test2]\n\"\"\"\n\n file = str(tmp_path / \"spack.yaml\")\n with open(file, \"w\", encoding=\"utf-8\") as f:\n f.write(contents)\n config(\"add\", \"-f\", file)\n output = config(\"get\", \"config\")\n\n assert (\n output\n == \"\"\"config:\n 'template_dirs:': [test2]\n\"\"\"\n )\n\n\ndef test_config_add_update_dict_from_file(mutable_empty_config, tmp_path: pathlib.Path):\n config(\"add\", \"packages:all:require:['%gcc']\")\n\n # contents to add to file\n contents = \"\"\"spack:\n packages:\n all:\n target: [x86_64]\n\"\"\"\n\n # create temp file and add it to config\n file = str(tmp_path / \"spack.yaml\")\n with open(file, \"w\", encoding=\"utf-8\") as f:\n f.write(contents)\n config(\"add\", \"-f\", file)\n\n # get results\n output = config(\"get\", \"packages\")\n\n # added config comes before prior config\n expected = \"\"\"packages:\n all:\n target: [x86_64]\n require: ['%gcc']\n\"\"\"\n\n assert expected == output\n\n\ndef test_config_add_invalid_file_fails(tmp_path: pathlib.Path):\n # contents to add to file\n # invalid because version requires a list\n contents = \"\"\"spack:\n packages:\n hdf5:\n version: 1.0.0\n\"\"\"\n\n # create temp file and add it to config\n file = str(tmp_path / \"spack.yaml\")\n with open(file, \"w\", encoding=\"utf-8\") as f:\n f.write(contents)\n\n with pytest.raises((spack.config.ConfigFormatError)):\n config(\"add\", \"-f\", file)\n\n\ndef test_config_remove_value(mutable_empty_config):\n config(\"add\", \"config:dirty:true\")\n config(\"remove\", \"config:dirty:true\")\n output = config(\"get\", \"config\")\n\n assert (\n output\n == \"\"\"config: {}\n\"\"\"\n )\n\n\ndef test_config_remove_alias_rm(mutable_empty_config):\n config(\"add\", \"config:dirty:true\")\n config(\"rm\", \"config:dirty:true\")\n output = config(\"get\", \"config\")\n\n assert (\n output\n == \"\"\"config: {}\n\"\"\"\n )\n\n\ndef test_config_remove_dict(mutable_empty_config):\n config(\"add\", \"config:dirty:true\")\n config(\"rm\", \"config:dirty\")\n output = config(\"get\", \"config\")\n\n assert (\n output\n == \"\"\"config: {}\n\"\"\"\n )\n\n\ndef test_remove_from_list(mutable_empty_config):\n config(\"add\", \"config:template_dirs:test1\")\n config(\"add\", \"config:template_dirs:[test2]\")\n config(\"add\", \"config:template_dirs:test3\")\n config(\"remove\", \"config:template_dirs:test2\")\n output = config(\"get\", \"config\")\n\n assert (\n output\n == \"\"\"config:\n template_dirs:\n - test3\n - test1\n\"\"\"\n )\n\n\ndef test_remove_list(mutable_empty_config):\n config(\"add\", \"config:template_dirs:test1\")\n config(\"add\", \"config:template_dirs:[test2]\")\n config(\"add\", \"config:template_dirs:test3\")\n config(\"remove\", \"config:template_dirs:[test2]\")\n output = config(\"get\", \"config\")\n\n assert (\n output\n == \"\"\"config:\n template_dirs:\n - test3\n - test1\n\"\"\"\n )\n\n\ndef test_config_add_to_env(mutable_empty_config, mutable_mock_env_path):\n env(\"create\", \"test\")\n with ev.read(\"test\"):\n config(\"add\", \"config:dirty:true\")\n output = config(\"get\")\n\n expected = \"\"\" config:\n dirty: true\n\"\"\"\n assert expected in output\n\n\ndef test_config_add_to_env_preserve_comments(\n mutable_empty_config, mutable_mock_env_path, tmp_path: pathlib.Path\n):\n filepath = str(tmp_path / \"spack.yaml\")\n manifest = \"\"\"# comment\nspack: # comment\n # comment\n specs: # comment\n - foo # comment\n # comment\n view: true # comment\n packages: # comment\n # comment\n all: # comment\n # comment\n compiler: [gcc] # comment\n\"\"\"\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n f.write(manifest)\n env = ev.Environment(str(tmp_path))\n with env:\n config(\"add\", \"config:dirty:true\")\n output = config(\"get\")\n\n assert \"# comment\" in output\n assert \"dirty: true\" in output\n\n\ndef test_config_remove_from_env(mutable_empty_config, mutable_mock_env_path):\n env(\"create\", \"test\")\n with ev.read(\"test\"):\n config(\"add\", \"config:dirty:true\")\n output = config(\"get\")\n assert \"dirty: true\" in output\n\n with ev.read(\"test\"):\n config(\"rm\", \"config:dirty\")\n output = config(\"get\")\n assert \"dirty: true\" not in output\n\n\ndef test_config_update_not_needed(mutable_config):\n data_before = spack.config.get(\"repos\")\n config(\"update\", \"-y\", \"repos\")\n data_after = spack.config.get(\"repos\")\n assert data_before == data_after\n\n\ndef test_config_update_shared_linking(mutable_config):\n # Old syntax: config:shared_linking:rpath/runpath\n # New syntax: config:shared_linking:{type:rpath/runpath,bind:True/False}\n with spack.config.override(\"config:shared_linking\", \"runpath\"):\n assert spack.config.get(\"config:shared_linking:type\") == \"runpath\"\n assert not spack.config.get(\"config:shared_linking:bind\")\n\n\ndef test_config_prefer_upstream(\n tmp_path_factory: pytest.TempPathFactory,\n install_mockery,\n mock_fetch,\n mutable_config,\n gen_mock_layout,\n monkeypatch,\n):\n \"\"\"Check that when a dependency package is recorded as installed in\n an upstream database that it is not reinstalled.\n \"\"\"\n\n mock_db_root = str(tmp_path_factory.mktemp(\"mock_db_root\"))\n prepared_db = spack.database.Database(mock_db_root, layout=gen_mock_layout(\"a\"))\n\n for spec in [\"hdf5 +mpi\", \"hdf5 ~mpi\", \"boost+debug~icu+graph\", \"dependency-install\", \"patch\"]:\n dep = spack.concretize.concretize_one(spec)\n prepared_db.add(dep)\n\n downstream_db_root = str(tmp_path_factory.mktemp(\"mock_downstream_db_root\"))\n db_for_test = spack.database.Database(downstream_db_root, upstream_dbs=[prepared_db])\n monkeypatch.setattr(spack.store.STORE, \"db\", db_for_test)\n\n output = config(\"prefer-upstream\")\n scope = spack.config.default_modify_scope(\"packages\")\n cfg_file = spack.config.CONFIG.get_config_filename(scope, \"packages\")\n packages = syaml.load(open(cfg_file, encoding=\"utf-8\"))[\"packages\"]\n\n # Make sure only the non-default variants are set.\n assert packages[\"boost\"] == {\"variants\": \"+debug +graph\", \"version\": [\"1.63.0\"]}\n assert packages[\"dependency-install\"] == {\"version\": [\"2.0\"]}\n # Ensure that neither variant gets listed for hdf5, since they conflict\n assert packages[\"hdf5\"] == {\"version\": [\"2.3\"]}\n\n # Make sure a message about the conflicting hdf5's was given.\n assert \"- hdf5\" in output\n\n\ndef test_environment_config_update(tmp_path: pathlib.Path, mutable_config, monkeypatch):\n with open(tmp_path / \"spack.yaml\", \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n config:\n ccache: true\n\"\"\"\n )\n\n def update_config(data):\n data[\"config\"][\"ccache\"] = False\n return True\n\n monkeypatch.setattr(spack.schema.config, \"update\", update_config)\n\n with ev.Environment(str(tmp_path)):\n config(\"update\", \"-y\", \"config\")\n\n with ev.Environment(str(tmp_path)) as e:\n assert not e.manifest.yaml_content[\"spack\"][\"config\"][\"ccache\"]\n\n\n_GROUP_OVERRIDE_SPACK_YAML = \"\"\"\\\nspack:\n specs:\n - group: mygroup\n specs:\n - zlib\n override:\n packages:\n zlib:\n version: ['1.2.13']\n\"\"\"\n\n\n@pytest.mark.parametrize(\"cmd_str\", [\"get\", \"blame\"])\ndef test_config_with_group_shows_override_packages(cmd_str, tmp_path, mutable_config):\n \"\"\"Tests that packages should show that group's override packages config,\n when the option is given.\n \"\"\"\n (tmp_path / \"spack.yaml\").write_text(_GROUP_OVERRIDE_SPACK_YAML)\n\n with ev.Environment(str(tmp_path)):\n output = config(cmd_str, \"packages\")\n assert \"1.2.13\" not in output\n if cmd_str == \"blame\":\n assert \"env:groups:mygroup\" not in output\n output = config(cmd_str, \"--group=mygroup\", \"packages\")\n assert \"1.2.13\" in output\n if cmd_str == \"blame\":\n assert \"env:groups:mygroup\" in output\n\n\n@pytest.mark.parametrize(\"cmd_str\", [\"get\", \"blame\"])\ndef test_config_with_group_requires_active_environment(cmd_str, mutable_config):\n \"\"\"Tests that using groups outside an environment should give a clear error.\"\"\"\n output = config(cmd_str, \"--group=mygroup\", \"packages\", fail_on_error=False)\n assert config.returncode != 0\n assert \"--group requires an active environment\" in output\n\n\n@pytest.mark.parametrize(\"cmd_str\", [\"get\", \"blame\"])\ndef test_config_with_unknown_group_gives_clear_error(cmd_str, tmp_path, mutable_config):\n \"\"\"Tests that using a non-existing group gives a clear error.\"\"\"\n (tmp_path / \"spack.yaml\").write_text(\"spack:\\n specs:\\n - zlib\\n\")\n with ev.Environment(str(tmp_path)):\n output = config(cmd_str, \"--group=nonexistent\", \"packages\", fail_on_error=False)\n assert config.returncode != 0\n assert \"'nonexistent' not found in\" in output\n" }, { "path": "lib/spack/spack/test/cmd/create.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport pathlib\nimport tarfile\n\nimport pytest\n\nimport spack.cmd.create\nimport spack.url\nfrom spack.main import SpackCommand\nfrom spack.url import UndetectableNameError\nfrom spack.util.executable import which\n\ncreate = SpackCommand(\"create\")\n\n\n@pytest.mark.parametrize(\n \"args,name,expected\",\n [\n # Basic package cases\n ([\"/test-package\"], \"test-package\", [r\"TestPackage(Package)\", r\"def install(self\"]),\n (\n [\"-n\", \"test-named-package\", \"file://example.tar.gz\"],\n \"test-named-package\",\n [r\"TestNamedPackage(Package)\", r\"def install(self\"],\n ),\n ([\"file://example.tar.gz\"], \"example\", [r\"Example(Package)\", r\"def install(self\"]),\n (\n [\"-n\", \"test-license\"],\n \"test-license\",\n [r'license(\"UNKNOWN\", checked_by=\"github_user1\")'],\n ),\n # Template-specific cases\n (\n [\"-t\", \"autoreconf\", \"/test-autoreconf\"],\n \"test-autoreconf\",\n [\n r\"TestAutoreconf(AutotoolsPackage)\",\n r'depends_on(\"autoconf',\n r\"def autoreconf(self\",\n r\"def configure_args(self\",\n ],\n ),\n (\n [\"-t\", \"autotools\", \"/test-autotools\"],\n \"test-autotools\",\n [r\"TestAutotools(AutotoolsPackage)\", r\"def configure_args(self\"],\n ),\n (\n [\"-t\", \"bazel\", \"/test-bazel\"],\n \"test-bazel\",\n [r\"TestBazel(Package)\", r'depends_on(\"bazel', r\"bazel()\"],\n ),\n ([\"-t\", \"bundle\", \"/test-bundle\"], \"test-bundle\", [r\"TestBundle(BundlePackage)\"]),\n (\n [\"-t\", \"cmake\", \"/test-cmake\"],\n \"test-cmake\",\n [r\"TestCmake(CMakePackage)\", r\"def cmake_args(self\"],\n ),\n (\n [\"-t\", \"intel\", \"/test-intel\"],\n \"test-intel\",\n [r\"TestIntel(IntelOneApiPackage)\", r\"setup_environment\"],\n ),\n (\n [\"-t\", \"makefile\", \"/test-makefile\"],\n \"test-makefile\",\n [r\"TestMakefile(MakefilePackage)\", r\"def edit(self\", r\"makefile\"],\n ),\n (\n [\"-t\", \"meson\", \"/test-meson\"],\n \"test-meson\",\n [r\"TestMeson(MesonPackage)\", r\"def meson_args(self\"],\n ),\n (\n [\"-t\", \"octave\", \"/test-octave\"],\n \"octave-test-octave\",\n [r\"OctaveTestOctave(OctavePackage)\", r'extends(\"octave', r'depends_on(\"octave'],\n ),\n (\n [\"-t\", \"perlbuild\", \"/test-perlbuild\"],\n \"perl-test-perlbuild\",\n [\n r\"PerlTestPerlbuild(PerlPackage)\",\n r'depends_on(\"perl-module-build',\n r\"def configure_args(self\",\n ],\n ),\n (\n [\"-t\", \"perlmake\", \"/test-perlmake\"],\n \"perl-test-perlmake\",\n [r\"PerlTestPerlmake(PerlPackage)\", r'depends_on(\"perl-', r\"def configure_args(self\"],\n ),\n (\n [\"-t\", \"python\", \"/test-python\"],\n \"py-test-python\",\n [r\"PyTestPython(PythonPackage)\", r'depends_on(\"py-', r\"def config_settings(self\"],\n ),\n (\n [\"-t\", \"qmake\", \"/test-qmake\"],\n \"test-qmake\",\n [r\"TestQmake(QMakePackage)\", r\"def qmake_args(self\"],\n ),\n (\n [\"-t\", \"r\", \"/test-r\"],\n \"r-test-r\",\n [r\"RTestR(RPackage)\", r'depends_on(\"r-', r\"def configure_args(self\"],\n ),\n (\n [\"-t\", \"scons\", \"/test-scons\"],\n \"test-scons\",\n [r\"TestScons(SConsPackage)\", r\"def build_args(self\"],\n ),\n (\n [\"-t\", \"sip\", \"/test-sip\"],\n \"py-test-sip\",\n [r\"PyTestSip(SIPPackage)\", r\"def configure_args(self\"],\n ),\n ([\"-t\", \"waf\", \"/test-waf\"], \"test-waf\", [r\"TestWaf(WafPackage)\", r\"configure_args()\"]),\n ],\n)\ndef test_create_template(mock_test_repo, args, name, expected):\n \"\"\"Test template creation.\"\"\"\n repo, repodir = mock_test_repo\n\n create(\"--skip-editor\", *args)\n\n filename = repo.filename_for_package_name(name)\n assert os.path.exists(filename)\n\n with open(filename, \"r\", encoding=\"utf-8\") as package_file:\n content = package_file.read()\n for entry in expected:\n assert entry in content\n\n black = which(\"black\", required=False)\n if not black:\n pytest.skip(\"checking blackness of `spack create` output requires black\")\n\n black(\"--check\", \"--diff\", filename)\n\n\n@pytest.mark.parametrize(\n \"name,expected\", [(\" \", \"name must be provided\"), (\"bad#name\", \"name can only contain\")]\n)\ndef test_create_template_bad_name(mock_test_repo, name, expected):\n \"\"\"Test template creation with bad name options.\"\"\"\n output = create(\"--skip-editor\", \"-n\", name, fail_on_error=False)\n assert expected in output\n assert create.returncode != 0\n\n\ndef test_build_system_guesser_no_stage():\n \"\"\"Test build system guesser when stage not provided.\"\"\"\n guesser = spack.cmd.create.BuildSystemAndLanguageGuesser()\n\n # Ensure get the expected build system\n with pytest.raises(AttributeError, match=\"'NoneType' object has no attribute\"):\n guesser(None, \"/the/url/does/not/matter\")\n\n\ndef test_build_system_guesser_octave(tmp_path: pathlib.Path):\n \"\"\"\n Test build system guesser for the special case, where the same base URL\n identifies the build system rather than guessing the build system from\n files contained in the archive.\n \"\"\"\n url, expected = \"downloads.sourceforge.net/octave/\", \"octave\"\n guesser = spack.cmd.create.BuildSystemAndLanguageGuesser()\n\n # Ensure get the expected build system\n guesser(str(tmp_path / \"archive.tar.gz\"), url)\n assert guesser.build_system == expected\n\n # Also ensure get the correct template\n bs = spack.cmd.create.get_build_system(None, url, guesser)\n assert bs == expected\n\n\n@pytest.mark.parametrize(\n \"url,expected\", [(\"testname\", \"testname\"), (\"file://example.com/archive.tar.gz\", \"archive\")]\n)\ndef test_get_name_urls(url, expected):\n \"\"\"Test get_name with different URLs.\"\"\"\n name = spack.cmd.create.get_name(None, url)\n assert name == expected\n\n\ndef test_get_name_error(monkeypatch, capfd):\n \"\"\"Test get_name UndetectableNameError exception path.\"\"\"\n\n def _parse_name_offset(path, v):\n raise UndetectableNameError(path)\n\n monkeypatch.setattr(spack.url, \"parse_name_offset\", _parse_name_offset)\n\n url = \"downloads.sourceforge.net/noapp/\"\n\n with pytest.raises(SystemExit):\n spack.cmd.create.get_name(None, url)\n captured = capfd.readouterr()\n assert \"Couldn't guess a name\" in str(captured)\n\n\ndef test_no_url():\n \"\"\"Test creation of package without a URL.\"\"\"\n create(\"--skip-editor\", \"-n\", \"create-new-package\")\n\n\n@pytest.mark.parametrize(\n \"source_files,languages\",\n [\n ([\"fst.c\", \"snd.C\"], [\"c\", \"cxx\"]),\n ([\"fst.c\", \"snd.cxx\"], [\"c\", \"cxx\"]),\n ([\"fst.F\", \"snd.cc\"], [\"cxx\", \"fortran\"]),\n ([\"fst.f\", \"snd.c\"], [\"c\", \"fortran\"]),\n ([\"fst.jl\", \"snd.py\"], []),\n ],\n)\ndef test_language_and_build_system_detection(tmp_path: pathlib.Path, source_files, languages):\n \"\"\"Test that languages are detected from tarball, and the build system is guessed from the\n most top-level build system file.\"\"\"\n\n def add(tar: tarfile.TarFile, name: str, type):\n tarinfo = tarfile.TarInfo(name)\n tarinfo.type = type\n tar.addfile(tarinfo)\n\n tarball = str(tmp_path / \"example.tar.gz\")\n\n with tarfile.open(tarball, \"w:gz\") as tar:\n add(tar, \"./third-party/\", tarfile.DIRTYPE)\n add(tar, \"./third-party/example/\", tarfile.DIRTYPE)\n add(tar, \"./third-party/example/CMakeLists.txt\", tarfile.REGTYPE) # false positive\n add(tar, \"./configure\", tarfile.REGTYPE) # actual build system\n add(tar, \"./src/\", tarfile.DIRTYPE)\n for file in source_files:\n add(tar, f\"src/{file}\", tarfile.REGTYPE)\n\n guesser = spack.cmd.create.BuildSystemAndLanguageGuesser()\n guesser(str(tarball), \"https://example.com\")\n\n assert guesser.build_system == \"autotools\"\n assert guesser.languages == languages\n" }, { "path": "lib/spack/spack/test/cmd/debug.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport platform\n\nimport spack\nimport spack.cmd.debug\nimport spack.platforms\nimport spack.repo\nimport spack.spec\nfrom spack.main import SpackCommand\nfrom spack.test.conftest import _return_none\n\ndebug = SpackCommand(\"debug\")\n\n\ndef test_report():\n out = debug(\"report\")\n host_platform = spack.platforms.host()\n host_os = host_platform.default_operating_system()\n host_target = host_platform.default_target()\n architecture = spack.spec.ArchSpec((str(host_platform), str(host_os), str(host_target)))\n\n assert spack.spack_version in out\n assert spack.get_spack_commit() in out\n assert platform.python_version() in out\n assert str(architecture) in out\n\n\ndef test_get_builtin_repo_info_local_repo(mock_git_version_info, monkeypatch):\n \"\"\"Confirm local git repo descriptor returns expected path.\"\"\"\n path = mock_git_version_info[0]\n\n def _from_config(*args, **kwargs):\n return {\"builtin\": spack.repo.LocalRepoDescriptor(\"builtin\", path)}\n\n monkeypatch.setattr(spack.repo.RepoDescriptors, \"from_config\", _from_config)\n assert path in spack.cmd.debug._get_builtin_repo_info()\n\n\ndef test_get_builtin_repo_info_unsupported_type(mock_git_version_info, monkeypatch):\n \"\"\"Confirm None is return if the 'builtin' repo descriptor's type is unsupported.\"\"\"\n\n def _from_config(*args, **kwargs):\n path = mock_git_version_info[0]\n return {\"builtin\": path}\n\n monkeypatch.setattr(spack.repo.RepoDescriptors, \"from_config\", _from_config)\n assert spack.cmd.debug._get_builtin_repo_info() is None\n\n\ndef test_get_builtin_repo_info_no_builtin(monkeypatch):\n \"\"\"Confirm None is return if there is no 'builtin' repo descriptor.\"\"\"\n\n def _from_config(*args, **kwargs):\n return {\"local\": \"/assumes/no/descriptor/needed\"}\n\n monkeypatch.setattr(spack.repo.RepoDescriptors, \"from_config\", _from_config)\n assert spack.cmd.debug._get_builtin_repo_info() is None\n\n\ndef test_get_builtin_repo_info_bad_destination(mock_git_version_info, monkeypatch):\n \"\"\"Confirm git failure of a repository returns None.\"\"\"\n\n def _from_config(*args, **kwargs):\n path = mock_git_version_info[0]\n return {\"builtin\": spack.repo.LocalRepoDescriptor(\"builtin\", f\"{path}/missing\")}\n\n monkeypatch.setattr(spack.repo.RepoDescriptors, \"from_config\", _from_config)\n assert spack.cmd.debug._get_builtin_repo_info() is None\n\n\ndef test_get_spack_repo_info_no_commit(monkeypatch):\n \"\"\"Confirm the version is returned if there is no spack commit.\"\"\"\n\n monkeypatch.setattr(spack, \"get_spack_commit\", _return_none)\n assert spack.cmd.debug._get_spack_repo_info() == spack.spack_version\n" }, { "path": "lib/spack/spack/test/cmd/deconcretize.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport pytest\n\nimport spack.environment as ev\nfrom spack.main import SpackCommand, SpackCommandError\n\ndeconcretize = SpackCommand(\"deconcretize\")\n\n\n@pytest.fixture(scope=\"function\")\ndef test_env(mutable_mock_env_path, mock_packages):\n ev.create(\"test\")\n with ev.read(\"test\") as e:\n e.add(\"pkg-a@2.0 foobar=bar ^pkg-b@1.0\")\n e.add(\"pkg-a@1.0 foobar=bar ^pkg-b@0.9\")\n e.concretize()\n e.write()\n\n\ndef test_deconcretize_dep(test_env):\n with ev.read(\"test\") as e:\n deconcretize(\"-y\", \"pkg-b@1.0\")\n specs = [s for s, _ in e.concretized_specs()]\n\n assert len(specs) == 1\n assert specs[0].satisfies(\"pkg-a@1.0\")\n\n\ndef test_deconcretize_all_dep(test_env):\n with ev.read(\"test\") as e:\n with pytest.raises(SpackCommandError):\n deconcretize(\"-y\", \"pkg-b\")\n deconcretize(\"-y\", \"--all\", \"pkg-b\")\n specs = [s for s, _ in e.concretized_specs()]\n\n assert len(specs) == 0\n\n\ndef test_deconcretize_root(test_env):\n with ev.read(\"test\") as e:\n output = deconcretize(\"-y\", \"--root\", \"pkg-b@1.0\")\n assert \"No matching specs to deconcretize\" in output\n assert len(e.concretized_roots) == 2\n\n deconcretize(\"-y\", \"--root\", \"pkg-a@2.0\")\n specs = [s for s, _ in e.concretized_specs()]\n\n assert len(specs) == 1\n assert specs[0].satisfies(\"pkg-a@1.0\")\n\n\ndef test_deconcretize_all_root(test_env):\n with ev.read(\"test\") as e:\n with pytest.raises(SpackCommandError):\n deconcretize(\"-y\", \"--root\", \"pkg-a\")\n\n output = deconcretize(\"-y\", \"--root\", \"--all\", \"pkg-b\")\n assert \"No matching specs to deconcretize\" in output\n assert len(e.concretized_roots) == 2\n\n deconcretize(\"-y\", \"--root\", \"--all\", \"pkg-a\")\n specs = [s for s, _ in e.concretized_specs()]\n\n assert len(specs) == 0\n\n\ndef test_deconcretize_all(test_env):\n with ev.read(\"test\") as e:\n with pytest.raises(SpackCommandError):\n deconcretize()\n deconcretize(\"-y\", \"--all\")\n specs = [s for s, _ in e.concretized_specs()]\n\n assert len(specs) == 0\n" }, { "path": "lib/spack/spack/test/cmd/dependencies.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport re\n\nimport pytest\n\nimport spack.store\nfrom spack.llnl.util.tty.color import color_when\nfrom spack.main import SpackCommand\n\ndependencies = SpackCommand(\"dependencies\")\n\nMPIS = [\n \"intel-parallel-studio\",\n \"low-priority-provider\",\n \"mpich\",\n \"mpich2\",\n \"multi-provider-mpi\",\n \"zmpi\",\n]\nCOMPILERS = [\"gcc\", \"llvm\", \"compiler-with-deps\"]\nMPI_DEPS = [\"fake\"]\nCOMPILER_DEPS = [\"binutils-for-test\", \"zlib\"]\n\n\n@pytest.mark.parametrize(\n \"cli_args,expected\",\n [\n ([\"mpileaks\"], set([\"callpath\"] + MPIS + COMPILERS)),\n (\n [\"--transitive\", \"mpileaks\"],\n set(\n [\"callpath\", \"dyninst\", \"libdwarf\", \"libelf\"]\n + MPIS\n + MPI_DEPS\n + COMPILERS\n + COMPILER_DEPS\n ),\n ),\n ([\"--transitive\", \"--deptype=link,run\", \"dtbuild1\"], {\"dtlink2\", \"dtrun2\"}),\n ([\"--transitive\", \"--deptype=build\", \"dtbuild1\"], {\"dtbuild2\", \"dtlink2\"}),\n ([\"--transitive\", \"--deptype=link\", \"dtbuild1\"], {\"dtlink2\"}),\n ],\n)\ndef test_direct_dependencies(cli_args, expected, mock_packages):\n out = dependencies(*cli_args)\n result = set(re.split(r\"\\s+\", out.strip()))\n assert expected == result\n\n\n@pytest.mark.db\ndef test_direct_installed_dependencies(mock_packages, database):\n with color_when(False):\n out = dependencies(\"--installed\", \"mpileaks^mpich\")\n\n root = spack.store.STORE.db.query_one(\"mpileaks ^mpich\")\n\n lines = [line for line in out.strip().split(\"\\n\") if line and not line.startswith(\"--\")]\n hashes = {re.split(r\"\\s+\", line)[0] for line in lines}\n expected = {s.dag_hash(7) for s in root.dependencies()}\n\n assert expected == hashes\n\n\n@pytest.mark.db\ndef test_transitive_installed_dependencies(mock_packages, database):\n with color_when(False):\n out = dependencies(\"--installed\", \"--transitive\", \"mpileaks^zmpi\")\n\n root = spack.store.STORE.db.query_one(\"mpileaks ^zmpi\")\n\n lines = [line for line in out.strip().split(\"\\n\") if line and not line.startswith(\"--\")]\n hashes = {re.split(r\"\\s+\", line)[0] for line in lines}\n expected = {s.dag_hash(7) for s in root.traverse(root=False)}\n\n assert expected == hashes\n" }, { "path": "lib/spack/spack/test/cmd/dependents.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport re\n\nimport pytest\n\nimport spack.store\nfrom spack.llnl.util.tty.color import color_when\nfrom spack.main import SpackCommand\n\ndependents = SpackCommand(\"dependents\")\n\n\ndef test_immediate_dependents(mock_packages):\n out = dependents(\"libelf\")\n actual = set(re.split(r\"\\s+\", out.strip()))\n assert actual == set(\n [\n \"dyninst\",\n \"libdwarf\",\n \"patch-a-dependency\",\n \"patch-several-dependencies\",\n \"quantum-espresso\",\n \"conditionally-patch-dependency\",\n ]\n )\n\n\ndef test_transitive_dependents(mock_packages):\n out = dependents(\"--transitive\", \"libelf\")\n actual = set(re.split(r\"\\s+\", out.strip()))\n assert actual == {\n \"callpath\",\n \"dyninst\",\n \"libdwarf\",\n \"mixing-parent\",\n \"mpileaks\",\n \"multivalue-variant\",\n \"singlevalue-variant-dependent\",\n \"trilinos\",\n \"patch-a-dependency\",\n \"patch-several-dependencies\",\n \"quantum-espresso\",\n \"conditionally-patch-dependency\",\n }\n\n\n@pytest.mark.db\ndef test_immediate_installed_dependents(mock_packages, database):\n with color_when(False):\n out = dependents(\"--installed\", \"libelf\")\n\n lines = [li for li in out.strip().split(\"\\n\") if not li.startswith(\"--\")]\n hashes = set([re.split(r\"\\s+\", li)[0] for li in lines if li])\n\n expected = set(\n [spack.store.STORE.db.query_one(s).dag_hash(7) for s in [\"dyninst\", \"libdwarf\"]]\n )\n\n libelf = spack.store.STORE.db.query_one(\"libelf\")\n expected = set([d.dag_hash(7) for d in libelf.dependents()])\n\n assert expected == hashes\n\n\n@pytest.mark.db\ndef test_transitive_installed_dependents(mock_packages, database):\n with color_when(False):\n out = dependents(\"--installed\", \"--transitive\", \"fake\")\n\n lines = [li for li in out.strip().split(\"\\n\") if li and not li.startswith(\"--\")]\n hashes = set([re.split(r\"\\s+\", li)[0] for li in lines])\n\n expected = set(\n [\n spack.store.STORE.db.query_one(s).dag_hash(7)\n for s in [\"zmpi\", \"callpath^zmpi\", \"mpileaks^zmpi\"]\n ]\n )\n\n assert expected == hashes\n" }, { "path": "lib/spack/spack/test/cmd/deprecate.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport pathlib\n\nimport pytest\n\nimport spack.concretize\nimport spack.spec\nimport spack.store\nfrom spack.enums import InstallRecordStatus\nfrom spack.main import SpackCommand\n\ninstall = SpackCommand(\"install\")\nuninstall = SpackCommand(\"uninstall\")\ndeprecate = SpackCommand(\"deprecate\")\nfind = SpackCommand(\"find\")\n\n# Unit tests should not be affected by the user's managed environments\npytestmark = pytest.mark.usefixtures(\"mutable_mock_env_path\")\n\n\ndef test_deprecate(mock_packages, mock_archive, mock_fetch, install_mockery):\n install(\"--fake\", \"libelf@0.8.13\")\n install(\"--fake\", \"libelf@0.8.10\")\n\n all_installed = spack.store.STORE.db.query(\"libelf\")\n assert len(all_installed) == 2\n\n deprecate(\"-y\", \"libelf@0.8.10\", \"libelf@0.8.13\")\n\n non_deprecated = spack.store.STORE.db.query(\"libelf\")\n all_available = spack.store.STORE.db.query(\"libelf\", installed=InstallRecordStatus.ANY)\n assert all_available == all_installed\n assert non_deprecated == spack.store.STORE.db.query(\"libelf@0.8.13\")\n\n\ndef test_deprecate_fails_no_such_package(mock_packages, mock_archive, mock_fetch, install_mockery):\n \"\"\"Tests that deprecating a spec that is not installed fails.\n\n Tests that deprecating without the ``-i`` option in favor of a spec that\n is not installed fails.\"\"\"\n output = deprecate(\"-y\", \"libelf@0.8.10\", \"libelf@0.8.13\", fail_on_error=False)\n assert \"Spec 'libelf@0.8.10' matches no installed packages\" in output\n\n install(\"--fake\", \"libelf@0.8.10\")\n\n output = deprecate(\"-y\", \"libelf@0.8.10\", \"libelf@0.8.13\", fail_on_error=False)\n assert \"Spec 'libelf@0.8.13' matches no installed packages\" in output\n\n\ndef test_deprecate_install(mock_packages, mock_archive, mock_fetch, install_mockery, monkeypatch):\n \"\"\"Tests that the -i option allows us to deprecate in favor of a spec\n that is not yet installed.\n \"\"\"\n install(\"--fake\", \"libelf@0.8.10\")\n to_deprecate = spack.store.STORE.db.query(\"libelf\")\n assert len(to_deprecate) == 1\n\n deprecate(\"-y\", \"-i\", \"libelf@0.8.10\", \"libelf@0.8.13\")\n\n non_deprecated = spack.store.STORE.db.query(\"libelf\")\n deprecated = spack.store.STORE.db.query(\"libelf\", installed=InstallRecordStatus.DEPRECATED)\n assert deprecated == to_deprecate\n assert len(non_deprecated) == 1\n assert non_deprecated[0].satisfies(\"libelf@0.8.13\")\n\n\ndef test_deprecate_deps(mock_packages, mock_archive, mock_fetch, install_mockery):\n \"\"\"Test that the deprecate command deprecates all dependencies properly.\"\"\"\n install(\"--fake\", \"libdwarf@20130729 ^libelf@0.8.13\")\n install(\"--fake\", \"libdwarf@20130207 ^libelf@0.8.10\")\n\n new_spec = spack.concretize.concretize_one(\"libdwarf@20130729^libelf@0.8.13\")\n old_spec = spack.concretize.concretize_one(\"libdwarf@20130207^libelf@0.8.10\")\n\n all_installed = spack.store.STORE.db.query()\n\n deprecate(\"-y\", \"-d\", \"libdwarf@20130207\", \"libdwarf@20130729\")\n\n non_deprecated = spack.store.STORE.db.query()\n all_available = spack.store.STORE.db.query(installed=InstallRecordStatus.ANY)\n deprecated = spack.store.STORE.db.query(installed=InstallRecordStatus.DEPRECATED)\n\n assert all_available == all_installed\n assert sorted(all_available) == sorted(deprecated + non_deprecated)\n\n assert sorted(non_deprecated) == sorted(new_spec.traverse())\n assert sorted(deprecated) == sorted([old_spec, old_spec[\"libelf\"]])\n\n\ndef test_uninstall_deprecated(mock_packages, mock_archive, mock_fetch, install_mockery):\n \"\"\"Tests that we can still uninstall deprecated packages.\"\"\"\n install(\"--fake\", \"libelf@0.8.13\")\n install(\"--fake\", \"libelf@0.8.10\")\n\n deprecate(\"-y\", \"libelf@0.8.10\", \"libelf@0.8.13\")\n\n non_deprecated = spack.store.STORE.db.query()\n\n uninstall(\"-y\", \"libelf@0.8.10\")\n\n assert spack.store.STORE.db.query() == spack.store.STORE.db.query(\n installed=InstallRecordStatus.ANY\n )\n assert spack.store.STORE.db.query() == non_deprecated\n\n\ndef test_deprecate_already_deprecated(mock_packages, mock_archive, mock_fetch, install_mockery):\n \"\"\"Tests that we can re-deprecate a spec to change its deprecator.\"\"\"\n install(\"--fake\", \"libelf@0.8.13\")\n install(\"--fake\", \"libelf@0.8.12\")\n install(\"--fake\", \"libelf@0.8.10\")\n\n deprecated_spec = spack.concretize.concretize_one(\"libelf@0.8.10\")\n\n deprecate(\"-y\", \"libelf@0.8.10\", \"libelf@0.8.12\")\n\n deprecator = spack.store.STORE.db.deprecator(deprecated_spec)\n assert deprecator == spack.concretize.concretize_one(\"libelf@0.8.12\")\n\n deprecate(\"-y\", \"libelf@0.8.10\", \"libelf@0.8.13\")\n\n non_deprecated = spack.store.STORE.db.query(\"libelf\")\n all_available = spack.store.STORE.db.query(\"libelf\", installed=InstallRecordStatus.ANY)\n assert len(non_deprecated) == 2\n assert len(all_available) == 3\n\n deprecator = spack.store.STORE.db.deprecator(deprecated_spec)\n assert deprecator == spack.concretize.concretize_one(\"libelf@0.8.13\")\n\n\ndef test_deprecate_deprecator(mock_packages, mock_archive, mock_fetch, install_mockery):\n \"\"\"Tests that when a deprecator spec is deprecated, its deprecatee specs\n are updated to point to the new deprecator.\"\"\"\n install(\"--fake\", \"libelf@0.8.13\")\n install(\"--fake\", \"libelf@0.8.12\")\n install(\"--fake\", \"libelf@0.8.10\")\n\n first_deprecated_spec = spack.concretize.concretize_one(\"libelf@0.8.10\")\n second_deprecated_spec = spack.concretize.concretize_one(\"libelf@0.8.12\")\n final_deprecator = spack.concretize.concretize_one(\"libelf@0.8.13\")\n\n deprecate(\"-y\", \"libelf@0.8.10\", \"libelf@0.8.12\")\n\n deprecator = spack.store.STORE.db.deprecator(first_deprecated_spec)\n assert deprecator == second_deprecated_spec\n\n deprecate(\"-y\", \"libelf@0.8.12\", \"libelf@0.8.13\")\n\n non_deprecated = spack.store.STORE.db.query(\"libelf\")\n all_available = spack.store.STORE.db.query(\"libelf\", installed=InstallRecordStatus.ANY)\n assert len(non_deprecated) == 1\n assert len(all_available) == 3\n\n first_deprecator = spack.store.STORE.db.deprecator(first_deprecated_spec)\n assert first_deprecator == final_deprecator\n second_deprecator = spack.store.STORE.db.deprecator(second_deprecated_spec)\n assert second_deprecator == final_deprecator\n\n\ndef test_concretize_deprecated(mock_packages, mock_archive, mock_fetch, install_mockery):\n \"\"\"Tests that the concretizer throws an error if we concretize to a\n deprecated spec\"\"\"\n install(\"--fake\", \"libelf@0.8.13\")\n install(\"--fake\", \"libelf@0.8.10\")\n\n deprecate(\"-y\", \"libelf@0.8.10\", \"libelf@0.8.13\")\n\n spec = spack.spec.Spec(\"libelf@0.8.10\")\n with pytest.raises(spack.spec.SpecDeprecatedError):\n spack.concretize.concretize_one(spec)\n\n\n@pytest.mark.usefixtures(\"mock_packages\", \"mock_archive\", \"mock_fetch\", \"install_mockery\")\n@pytest.mark.regression(\"46915\")\ndef test_deprecate_spec_with_external_dependency(\n mutable_config, temporary_store, tmp_path: pathlib.Path\n):\n \"\"\"Tests that we can deprecate a spec that has an external dependency\"\"\"\n packages_yaml = {\n \"libelf\": {\n \"buildable\": False,\n \"externals\": [{\"spec\": \"libelf@0.8.13\", \"prefix\": str(tmp_path / \"libelf\")}],\n }\n }\n mutable_config.set(\"packages\", packages_yaml)\n\n install(\"--fake\", \"dyninst ^libdwarf@=20111030\")\n install(\"--fake\", \"libdwarf@=20130729\")\n\n # Ensure we are using the external libelf\n db = temporary_store.db\n libelf = db.query_one(\"libelf\")\n assert libelf.external\n\n deprecated_spec = db.query_one(\"libdwarf@=20111030\")\n new_libdwarf = db.query_one(\"libdwarf@=20130729\")\n deprecate(\"-y\", \"libdwarf@=20111030\", \"libdwarf@=20130729\")\n\n assert db.deprecator(deprecated_spec) == new_libdwarf\n" }, { "path": "lib/spack/spack/test/cmd/dev_build.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport pathlib\n\nimport pytest\n\nimport spack.concretize\nimport spack.environment as ev\nimport spack.error\nimport spack.llnl.util.filesystem as fs\nimport spack.main\nimport spack.repo\nimport spack.spec\nimport spack.store\nfrom spack.main import SpackCommand\n\ndev_build = SpackCommand(\"dev-build\")\ninstall = SpackCommand(\"install\")\nenv = SpackCommand(\"env\")\n\npytestmark = [pytest.mark.disable_clean_stage_check]\n\n\ndef test_dev_build_basics(tmp_path: pathlib.Path, install_mockery):\n spec = spack.concretize.concretize_one(\n spack.spec.Spec(f\"dev-build-test-install@0.0.0 dev_path={tmp_path}\")\n )\n\n assert \"dev_path\" in spec.variants\n\n with fs.working_dir(str(tmp_path)):\n with open(spec.package.filename, \"w\", encoding=\"utf-8\") as f: # type: ignore\n f.write(spec.package.original_string) # type: ignore\n\n dev_build(\"dev-build-test-install@0.0.0\")\n\n assert spec.package.filename in os.listdir(spec.prefix)\n with open(os.path.join(spec.prefix, spec.package.filename), \"r\", encoding=\"utf-8\") as f:\n assert f.read() == spec.package.replacement_string\n\n assert os.path.exists(str(tmp_path))\n\n\ndef test_dev_build_before(tmp_path: pathlib.Path, install_mockery, installer_variant):\n spec = spack.concretize.concretize_one(\n spack.spec.Spec(f\"dev-build-test-install@0.0.0 dev_path={tmp_path}\")\n )\n\n with fs.working_dir(str(tmp_path)):\n with open(spec.package.filename, \"w\", encoding=\"utf-8\") as f: # type: ignore\n f.write(spec.package.original_string) # type: ignore\n\n dev_build(\"-b\", \"edit\", \"dev-build-test-install@0.0.0\")\n\n assert spec.package.filename in os.listdir(os.getcwd()) # type: ignore\n with open(spec.package.filename, \"r\", encoding=\"utf-8\") as f: # type: ignore\n assert f.read() == spec.package.original_string # type: ignore\n\n assert not os.path.exists(spec.prefix)\n\n\n@pytest.mark.parametrize(\"last_phase\", [\"edit\", \"install\"])\ndef test_dev_build_until(tmp_path: pathlib.Path, install_mockery, last_phase, installer_variant):\n spec = spack.concretize.concretize_one(\n spack.spec.Spec(f\"dev-build-test-install@0.0.0 dev_path={tmp_path}\")\n )\n\n with fs.working_dir(str(tmp_path)):\n with open(spec.package.filename, \"w\", encoding=\"utf-8\") as f: # type: ignore\n f.write(spec.package.original_string) # type: ignore\n\n dev_build(\"--until\", last_phase, \"dev-build-test-install@0.0.0\")\n\n assert spec.package.filename in os.listdir(os.getcwd()) # type: ignore\n with open(spec.package.filename, \"r\", encoding=\"utf-8\") as f: # type: ignore\n assert f.read() == spec.package.replacement_string # type: ignore\n\n assert not os.path.exists(spec.prefix)\n assert not spack.store.STORE.db.query(spec, installed=True)\n\n\ndef test_dev_build_before_until(tmp_path: pathlib.Path, install_mockery, installer_variant):\n spec = spack.concretize.concretize_one(\n spack.spec.Spec(f\"dev-build-test-install@0.0.0 dev_path={tmp_path}\")\n )\n\n with fs.working_dir(str(tmp_path)):\n with open(spec.package.filename, \"w\", encoding=\"utf-8\") as f:\n f.write(spec.package.original_string)\n\n with pytest.raises(spack.main.SpackCommandError):\n dev_build(\"-u\", \"edit\", \"-b\", \"edit\", \"dev-build-test-install@0.0.0\")\n\n bad_phase = \"phase_that_does_not_exist\"\n not_allowed = \"is not a valid phase\"\n not_installed = \"was not installed\"\n out = dev_build(\"-u\", bad_phase, \"dev-build-test-install@0.0.0\", fail_on_error=False)\n assert bad_phase in out\n assert not_allowed in out\n if installer_variant == \"old\":\n assert not_installed in out\n\n out = dev_build(\"-b\", bad_phase, \"dev-build-test-install@0.0.0\", fail_on_error=False)\n assert bad_phase in out\n assert not_allowed in out\n if installer_variant == \"old\":\n assert not_installed in out\n\n\ndef _print_spack_short_spec(*args):\n print(f\"SPACK_SHORT_SPEC={os.environ['SPACK_SHORT_SPEC']}\")\n\n\ndef test_dev_build_drop_in(\n tmp_path: pathlib.Path, mock_packages, monkeypatch, install_mockery, working_env\n):\n monkeypatch.setattr(os, \"execvp\", _print_spack_short_spec)\n with fs.working_dir(str(tmp_path)):\n output = dev_build(\"-b\", \"edit\", \"--drop-in\", \"sh\", \"dev-build-test-install@0.0.0\")\n assert \"SPACK_SHORT_SPEC=dev-build-test-install@0.0.0\" in output\n\n\ndef test_dev_build_fails_already_installed(tmp_path: pathlib.Path, install_mockery):\n spec = spack.concretize.concretize_one(\n spack.spec.Spec(\"dev-build-test-install@0.0.0 dev_path=%s\" % str(tmp_path))\n )\n\n with fs.working_dir(str(tmp_path)):\n with open(spec.package.filename, \"w\", encoding=\"utf-8\") as f:\n f.write(spec.package.original_string)\n\n dev_build(\"dev-build-test-install@0.0.0\")\n output = dev_build(\"dev-build-test-install@0.0.0\", fail_on_error=False)\n assert \"Already installed in %s\" % spec.prefix in output\n\n\ndef test_dev_build_fails_no_spec():\n output = dev_build(fail_on_error=False)\n assert \"requires a package spec argument\" in output\n\n\ndef test_dev_build_fails_multiple_specs(mock_packages):\n output = dev_build(\"libelf\", \"libdwarf\", fail_on_error=False)\n assert \"only takes one spec\" in output\n\n\ndef test_dev_build_fails_nonexistent_package_name(mock_packages):\n output = \"\"\n\n try:\n dev_build(\"no_such_package\")\n assert False, \"no exception was raised!\"\n except spack.repo.UnknownPackageError as e:\n output = e.message\n\n assert \"Package 'no_such_package' not found\" in output\n\n\ndef test_dev_build_fails_no_version(mock_packages):\n output = dev_build(\"dev-build-test-install\", fail_on_error=False)\n assert \"dev-build spec must have a single, concrete version\" in output\n\n\ndef test_dev_build_can_parse_path_with_at_symbol(tmp_path: pathlib.Path, install_mockery):\n special_char_dir = tmp_path / \"tmp@place\"\n special_char_dir.mkdir()\n spec = spack.concretize.concretize_one(\n f'dev-build-test-install@0.0.0 dev_path=\"{special_char_dir}\"'\n )\n\n with fs.working_dir(str(special_char_dir)):\n with open(spec.package.filename, \"w\", encoding=\"utf-8\") as f:\n f.write(spec.package.original_string)\n dev_build(\"dev-build-test-install@0.0.0\")\n assert spec.package.filename in os.listdir(spec.prefix)\n\n\ndef test_dev_build_env(\n tmp_path: pathlib.Path, install_mockery, mutable_mock_env_path, installer_variant\n):\n \"\"\"Test Spack does dev builds for packages in develop section of env.\"\"\"\n # setup dev-build-test-install package for dev build\n build_dir = tmp_path / \"build\"\n build_dir.mkdir()\n spec = spack.concretize.concretize_one(\n spack.spec.Spec(\"dev-build-test-install@0.0.0 dev_path=%s\" % str(build_dir))\n )\n\n with fs.working_dir(str(build_dir)):\n with open(spec.package.filename, \"w\", encoding=\"utf-8\") as f:\n f.write(spec.package.original_string)\n\n # setup environment\n envdir = tmp_path / \"env\"\n envdir.mkdir()\n with fs.working_dir(str(envdir)):\n with open(\"spack.yaml\", \"w\", encoding=\"utf-8\") as f:\n f.write(\n f\"\"\"\\\nspack:\n specs:\n - dev-build-test-install@0.0.0\n\n develop:\n dev-build-test-install:\n spec: dev-build-test-install@0.0.0\n path: {os.path.relpath(str(build_dir), start=str(envdir))}\n\"\"\"\n )\n env(\"create\", \"test\", \"./spack.yaml\")\n with ev.read(\"test\"):\n install()\n\n assert spec.package.filename in os.listdir(spec.prefix)\n with open(os.path.join(spec.prefix, spec.package.filename), \"r\", encoding=\"utf-8\") as f:\n assert f.read() == spec.package.replacement_string\n\n\ndef test_dev_build_env_with_vars(\n tmp_path: pathlib.Path, install_mockery, mutable_mock_env_path, monkeypatch, installer_variant\n):\n \"\"\"Test Spack does dev builds for packages in develop section of env (path with variables).\"\"\"\n # setup dev-build-test-install package for dev build\n build_dir = tmp_path / \"build\"\n build_dir.mkdir()\n spec = spack.concretize.concretize_one(\n spack.spec.Spec(f\"dev-build-test-install@0.0.0 dev_path={str(build_dir)}\")\n )\n\n # store the build path in an environment variable that will be used in the environment\n monkeypatch.setenv(\"CUSTOM_BUILD_PATH\", str(build_dir))\n\n with fs.working_dir(str(build_dir)), open(spec.package.filename, \"w\", encoding=\"utf-8\") as f:\n f.write(spec.package.original_string)\n\n # setup environment\n envdir = tmp_path / \"env\"\n envdir.mkdir()\n with fs.working_dir(str(envdir)):\n with open(\"spack.yaml\", \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n specs:\n - dev-build-test-install@0.0.0\n\n develop:\n dev-build-test-install:\n spec: dev-build-test-install@0.0.0\n path: $CUSTOM_BUILD_PATH\n\"\"\"\n )\n env(\"create\", \"test\", \"./spack.yaml\")\n with ev.read(\"test\"):\n install()\n\n assert spec.package.filename in os.listdir(spec.prefix)\n with open(os.path.join(spec.prefix, spec.package.filename), \"r\", encoding=\"utf-8\") as f:\n assert f.read() == spec.package.replacement_string\n\n\ndef test_dev_build_env_version_mismatch(\n tmp_path: pathlib.Path, install_mockery, mutable_mock_env_path, installer_variant\n):\n \"\"\"Test Spack constraints concretization by develop specs.\"\"\"\n # setup dev-build-test-install package for dev build\n build_dir = tmp_path / \"build\"\n build_dir.mkdir()\n spec = spack.concretize.concretize_one(\n spack.spec.Spec(\"dev-build-test-install@0.0.0 dev_path=%s\" % str(tmp_path))\n )\n\n with fs.working_dir(str(build_dir)):\n with open(spec.package.filename, \"w\", encoding=\"utf-8\") as f:\n f.write(spec.package.original_string)\n\n # setup environment\n envdir = tmp_path / \"env\"\n envdir.mkdir()\n with fs.working_dir(str(envdir)):\n with open(\"spack.yaml\", \"w\", encoding=\"utf-8\") as f:\n f.write(\n f\"\"\"\\\nspack:\n specs:\n - dev-build-test-install@0.0.0\n\n develop:\n dev-build-test-install:\n spec: dev-build-test-install@1.1.1\n path: {build_dir}\n\"\"\"\n )\n\n env(\"create\", \"test\", \"./spack.yaml\")\n with ev.read(\"test\"):\n with pytest.raises((RuntimeError, spack.error.UnsatisfiableSpecError)):\n install()\n\n\ndef test_dev_build_multiple(\n tmp_path: pathlib.Path, install_mockery, mutable_mock_env_path, mock_fetch\n):\n \"\"\"Test spack install with multiple developer builds\n\n Test that only the root needs to be specified in the environment\n Test that versions known only from the dev specs are included in the solve,\n even if they come from a non-root\n \"\"\"\n # setup dev-build-test-install package for dev build\n # Wait to concretize inside the environment to set dev_path on the specs;\n # without the environment, the user would need to set dev_path for both the\n # root and dependency if they wanted a dev build for both.\n leaf_dir = tmp_path / \"leaf\"\n leaf_dir.mkdir()\n leaf_spec = spack.spec.Spec(\"dev-build-test-install@=1.0.0\") # non-existing version\n leaf_pkg_cls = spack.repo.PATH.get_pkg_class(leaf_spec.name)\n with fs.working_dir(str(leaf_dir)):\n with open(leaf_pkg_cls.filename, \"w\", encoding=\"utf-8\") as f: # type: ignore\n f.write(leaf_pkg_cls.original_string) # type: ignore\n\n # setup dev-build-test-dependent package for dev build\n # don't concretize outside environment -- dev info will be wrong\n root_dir = tmp_path / \"root\"\n root_dir.mkdir()\n root_spec = spack.spec.Spec(\"dev-build-test-dependent@0.0.0\")\n root_pkg_cls = spack.repo.PATH.get_pkg_class(root_spec.name)\n with fs.working_dir(str(root_dir)):\n with open(root_pkg_cls.filename, \"w\", encoding=\"utf-8\") as f: # type: ignore\n f.write(root_pkg_cls.original_string) # type: ignore\n\n # setup environment\n envdir = tmp_path / \"env\"\n envdir.mkdir()\n with fs.working_dir(str(envdir)):\n with open(\"spack.yaml\", \"w\", encoding=\"utf-8\") as f:\n f.write(\n f\"\"\"\\\nspack:\n specs:\n - dev-build-test-dependent@0.0.0\n\n develop:\n dev-build-test-install:\n path: {leaf_dir}\n spec: dev-build-test-install@=1.0.0\n dev-build-test-dependent:\n spec: dev-build-test-dependent@0.0.0\n path: {root_dir}\n\"\"\"\n )\n\n env(\"create\", \"test\", \"./spack.yaml\")\n with ev.read(\"test\"):\n # Do concretization inside environment for dev info\n # These specs are the source of truth to compare against the installs\n leaf_spec = spack.concretize.concretize_one(leaf_spec)\n root_spec = spack.concretize.concretize_one(root_spec)\n\n # Do install\n install()\n\n for spec in (leaf_spec, root_spec):\n filename = spec.package.filename # type: ignore\n assert filename in os.listdir(spec.prefix)\n with open(os.path.join(spec.prefix, filename), \"r\", encoding=\"utf-8\") as f:\n assert f.read() == spec.package.replacement_string\n\n\ndef test_dev_build_env_dependency(\n tmp_path: pathlib.Path, install_mockery, mock_fetch, mutable_mock_env_path\n):\n \"\"\"\n Test non-root specs in an environment are properly marked for dev builds.\n \"\"\"\n # setup dev-build-test-install package for dev build\n build_dir = tmp_path / \"build\"\n build_dir.mkdir()\n spec = spack.spec.Spec(\"dependent-of-dev-build@0.0.0\")\n dep_spec = spack.spec.Spec(\"dev-build-test-install\")\n\n with fs.working_dir(str(build_dir)):\n dep_pkg_cls = spack.repo.PATH.get_pkg_class(dep_spec.name)\n with open(dep_pkg_cls.filename, \"w\", encoding=\"utf-8\") as f: # type: ignore\n f.write(dep_pkg_cls.original_string) # type: ignore\n\n # setup environment\n envdir = tmp_path / \"env\"\n envdir.mkdir()\n with fs.working_dir(str(envdir)):\n with open(\"spack.yaml\", \"w\", encoding=\"utf-8\") as f:\n f.write(\n f\"\"\"\\\nspack:\n specs:\n - dependent-of-dev-build@0.0.0\n\n develop:\n dev-build-test-install:\n spec: dev-build-test-install@0.0.0\n path: {os.path.relpath(str(build_dir), start=str(envdir))}\n\"\"\"\n )\n env(\"create\", \"test\", \"./spack.yaml\")\n with ev.read(\"test\"):\n # concretize in the environment to get the dev build info\n # equivalent to setting dev_build and dev_path variants\n # on all specs above\n spec = spack.concretize.concretize_one(spec)\n dep_spec = spack.concretize.concretize_one(dep_spec)\n install()\n\n # Ensure that both specs installed properly\n assert dep_spec.package.filename in os.listdir(dep_spec.prefix)\n assert os.path.exists(spec.prefix)\n\n # Ensure variants set properly; ensure build_dir is absolute and normalized\n for dep in (dep_spec, spec[\"dev-build-test-install\"]):\n assert dep.satisfies(\"dev_path=%s\" % str(build_dir))\n assert spec.satisfies(\"^dev_path=*\")\n\n\n@pytest.mark.parametrize(\"test_spec\", [\"dev-build-test-install\", \"dependent-of-dev-build\"])\ndef test_dev_build_rebuild_on_source_changes(\n test_spec, tmp_path: pathlib.Path, install_mockery, mutable_mock_env_path, mock_fetch\n):\n \"\"\"Test dev builds rebuild on changes to source code.\n\n ``test_spec = dev-build-test-install`` tests rebuild for changes to package\n ``test_spec = dependent-of-dev-build`` tests rebuild for changes to dep\n \"\"\"\n # setup dev-build-test-install package for dev build\n build_dir = tmp_path / \"build\"\n build_dir.mkdir()\n spec = spack.concretize.concretize_one(\n spack.spec.Spec(\"dev-build-test-install@0.0.0 dev_path=%s\" % str(build_dir))\n )\n\n def reset_string():\n with fs.working_dir(str(build_dir)):\n with open(spec.package.filename, \"w\", encoding=\"utf-8\") as f: # type: ignore\n f.write(spec.package.original_string) # type: ignore\n\n reset_string()\n\n # setup environment\n envdir = tmp_path / \"env\"\n envdir.mkdir()\n with fs.working_dir(str(envdir)):\n with open(\"spack.yaml\", \"w\", encoding=\"utf-8\") as f:\n f.write(\n f\"\"\"\\\nspack:\n specs:\n - {test_spec}@0.0.0\n\n develop:\n dev-build-test-install:\n spec: dev-build-test-install@0.0.0\n path: {build_dir}\n\"\"\"\n )\n\n env(\"create\", \"test\", \"./spack.yaml\")\n with ev.read(\"test\"):\n install()\n\n reset_string() # so the package will accept rebuilds\n\n fs.touch(os.path.join(str(build_dir), \"test\"))\n output = install()\n\n assert f\"Installing {test_spec}\" in output\n" }, { "path": "lib/spack/spack/test/cmd/develop.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport os\nimport pathlib\nimport shutil\n\nimport pytest\n\nimport spack.concretize\nimport spack.config\nimport spack.environment as ev\nimport spack.llnl.util.filesystem as fs\nimport spack.package_base\nimport spack.spec\nimport spack.stage\nimport spack.util.git\nimport spack.util.path\nfrom spack.error import SpackError\nfrom spack.fetch_strategy import URLFetchStrategy\nfrom spack.main import SpackCommand\n\nadd = SpackCommand(\"add\")\ndevelop = SpackCommand(\"develop\")\nenv = SpackCommand(\"env\")\n\n\n@pytest.mark.usefixtures(\"mutable_mock_env_path\", \"mock_packages\", \"mock_fetch\", \"mutable_config\")\nclass TestDevelop:\n def check_develop(self, env, spec, path=None, build_dir=None):\n path = path or spec.name\n\n # check in memory representation\n assert spec.name in env.dev_specs\n dev_specs_entry = env.dev_specs[spec.name]\n assert dev_specs_entry[\"path\"] == path\n assert dev_specs_entry[\"spec\"] == str(spec)\n\n # check yaml representation\n dev_config = spack.config.get(\"develop\", {})\n assert spec.name in dev_config\n yaml_entry = dev_config[spec.name]\n assert yaml_entry[\"spec\"] == str(spec)\n if path == spec.name:\n # default paths aren't written out\n assert \"path\" not in yaml_entry\n else:\n assert yaml_entry[\"path\"] == path\n\n if build_dir is not None:\n scope = env.scope_name\n assert build_dir == spack.config.get(\n \"packages:{}:package_attributes:build_directory\".format(spec.name), scope\n )\n\n def test_develop_no_path_no_clone(self):\n env(\"create\", \"test\")\n with ev.read(\"test\") as e:\n # develop checks that the path exists\n fs.mkdirp(os.path.join(e.path, \"mpich\"))\n develop(\"--no-clone\", \"mpich@1.0\")\n self.check_develop(e, spack.spec.Spec(\"mpich@=1.0\"))\n\n def test_develop_no_clone(self, tmp_path: pathlib.Path):\n env(\"create\", \"test\")\n with ev.read(\"test\") as e:\n develop(\"--no-clone\", \"-p\", str(tmp_path), \"mpich@1.0\")\n self.check_develop(e, spack.spec.Spec(\"mpich@=1.0\"), str(tmp_path))\n\n def test_develop_no_version(self, tmp_path: pathlib.Path):\n env(\"create\", \"test\")\n with ev.read(\"test\") as e:\n develop(\"--no-clone\", \"-p\", str(tmp_path), \"mpich\")\n self.check_develop(e, spack.spec.Spec(\"mpich@=main\"), str(tmp_path))\n\n def test_develop(self):\n env(\"create\", \"test\")\n with ev.read(\"test\") as e:\n develop(\"mpich@1.0\")\n self.check_develop(e, spack.spec.Spec(\"mpich@=1.0\"))\n\n def test_develop_no_args(self):\n env(\"create\", \"test\")\n with ev.read(\"test\") as e:\n # develop and remove it\n develop(\"mpich@1.0\")\n shutil.rmtree(os.path.join(e.path, \"mpich\"))\n\n # test develop with no args\n develop()\n self.check_develop(e, spack.spec.Spec(\"mpich@=1.0\"))\n\n def test_develop_build_directory(self):\n env(\"create\", \"test\")\n with ev.read(\"test\") as e:\n develop(\"-b\", \"test_build_dir\", \"mpich@1.0\")\n self.check_develop(e, spack.spec.Spec(\"mpich@=1.0\"), None, \"test_build_dir\")\n\n def test_develop_twice(self):\n env(\"create\", \"test\")\n with ev.read(\"test\") as e:\n develop(\"mpich@1.0\")\n self.check_develop(e, spack.spec.Spec(\"mpich@=1.0\"))\n\n develop(\"mpich@1.0\")\n # disk representation isn't updated unless we write\n # second develop command doesn't change it, so we don't write\n # but we check disk representation\n e.write()\n self.check_develop(e, spack.spec.Spec(\"mpich@=1.0\"))\n assert len(e.dev_specs) == 1\n\n def test_develop_update_path(self, tmp_path: pathlib.Path):\n env(\"create\", \"test\")\n with ev.read(\"test\") as e:\n develop(\"mpich@1.0\")\n develop(\"-p\", str(tmp_path), \"mpich@1.0\")\n self.check_develop(e, spack.spec.Spec(\"mpich@=1.0\"), str(tmp_path))\n assert len(e.dev_specs) == 1\n\n def test_develop_update_spec(self):\n env(\"create\", \"test\")\n with ev.read(\"test\") as e:\n develop(\"mpich@1.0\")\n develop(\"mpich@2.0\")\n self.check_develop(e, spack.spec.Spec(\"mpich@=2.0\"))\n assert len(e.dev_specs) == 1\n\n def test_develop_applies_changes(self, monkeypatch):\n env(\"create\", \"test\")\n with ev.read(\"test\") as e:\n e.add(\"mpich@1.0\")\n e.concretize()\n e.write()\n\n monkeypatch.setattr(spack.stage.Stage, \"steal_source\", lambda x, y: None)\n develop(\"mpich@1.0\")\n\n # Check modifications actually worked\n spec = next(e.roots())\n assert spec.satisfies(\"dev_path=*\")\n\n def test_develop_applies_changes_parents(self, monkeypatch):\n env(\"create\", \"test\")\n with ev.read(\"test\") as e:\n e.add(\"hdf5^mpich@1.0\")\n e.concretize()\n e.write()\n\n orig_hash = next(e.roots()).dag_hash()\n\n monkeypatch.setattr(spack.stage.Stage, \"steal_source\", lambda x, y: None)\n develop(\"mpich@1.0\")\n\n # Check modifications actually worked\n new_hdf5 = next(e.roots())\n assert new_hdf5.dag_hash() != orig_hash\n assert new_hdf5[\"mpi\"].satisfies(\"dev_path=*\")\n\n def test_develop_applies_changes_spec_conflict(self, monkeypatch):\n env(\"create\", \"test\")\n with ev.read(\"test\") as e:\n e.add(\"mpich@1.0\")\n e.concretize()\n e.write()\n\n monkeypatch.setattr(spack.stage.Stage, \"steal_source\", lambda x, y: None)\n with pytest.raises(ev.SpackEnvironmentDevelopError, match=\"conflicts with concrete\"):\n develop(\"mpich@1.1\")\n\n def test_develop_applies_changes_path(self, monkeypatch):\n env(\"create\", \"test\")\n with ev.read(\"test\") as e:\n e.add(\"mpich@1.0\")\n e.concretize()\n e.write()\n\n # canonicalize paths relative to env\n testpath1 = spack.util.path.canonicalize_path(\"test/path1\", e.path)\n testpath2 = spack.util.path.canonicalize_path(\"test/path2\", e.path)\n\n monkeypatch.setattr(spack.stage.Stage, \"steal_source\", lambda x, y: None)\n # Testing that second call to develop successfully changes both config and specs\n for path in (testpath1, testpath2):\n develop(\"--path\", path, \"mpich@1.0\")\n\n # Check modifications actually worked\n spec = next(e.roots())\n assert spec.satisfies(f\"dev_path={path}\")\n assert spack.config.get(\"develop:mpich:path\") == path\n\n def test_develop_no_modify(self, monkeypatch):\n env(\"create\", \"test\")\n with ev.read(\"test\") as e:\n e.add(\"mpich@1.0\")\n e.concretize()\n e.write()\n\n monkeypatch.setattr(spack.stage.Stage, \"steal_source\", lambda x, y: None)\n develop(\"--no-modify-concrete-specs\", \"mpich@1.0\")\n\n # Check modifications were not applied\n spec = next(e.roots())\n assert not spec.satisfies(\"dev_path=*\")\n\n def test_develop_canonicalize_path(self, monkeypatch):\n env(\"create\", \"test\")\n with ev.read(\"test\") as e:\n e.add(\"mpich@1.0\")\n e.concretize()\n e.write()\n\n path = \"../$user\"\n abspath = spack.util.path.canonicalize_path(path, e.path)\n\n def check_path(stage, dest):\n assert dest == abspath\n\n monkeypatch.setattr(spack.stage.Stage, \"steal_source\", check_path)\n\n develop(\"-p\", path, \"mpich@1.0\")\n self.check_develop(e, spack.spec.Spec(\"mpich@=1.0\"), path)\n\n # Check modifications actually worked\n spec = next(e.roots())\n assert spec.satisfies(\"dev_path=%s\" % abspath)\n\n def test_develop_canonicalize_path_no_args(self, monkeypatch):\n env(\"create\", \"test\")\n with ev.read(\"test\") as e:\n e.add(\"mpich@1.0\")\n e.concretize()\n e.write()\n\n path = \"$user\"\n abspath = spack.util.path.canonicalize_path(path, e.path)\n\n def check_path(stage, dest):\n assert dest == abspath\n\n monkeypatch.setattr(spack.stage.Stage, \"steal_source\", check_path)\n\n # Defensive check to ensure canonicalization failures don't pollute FS\n assert abspath.startswith(e.path)\n\n # Create path to allow develop to modify env\n fs.mkdirp(abspath)\n develop(\"--no-clone\", \"-p\", path, \"mpich@1.0\")\n self.check_develop(e, spack.spec.Spec(\"mpich@=1.0\"), path)\n\n # Remove path to ensure develop with no args runs staging code\n os.rmdir(abspath)\n\n develop()\n self.check_develop(e, spack.spec.Spec(\"mpich@=1.0\"), path)\n\n # Check modifications actually worked\n spec = next(e.roots())\n assert spec.satisfies(\"dev_path=%s\" % abspath)\n\n\ndef _git_commit_list(git_repo_dir):\n git = spack.util.git.git()\n with fs.working_dir(git_repo_dir):\n output = git(\"log\", \"--pretty=format:%h\", \"-n\", \"20\", output=str)\n return output.strip().split()\n\n\ndef test_develop_full_git_repo(\n mutable_mock_env_path,\n mock_git_version_info,\n install_mockery,\n mock_packages,\n monkeypatch,\n mutable_config,\n request,\n):\n repo_path, filename, commits = mock_git_version_info\n monkeypatch.setattr(\n spack.package_base.PackageBase, \"git\", \"file://%s\" % repo_path, raising=False\n )\n\n spec = spack.concretize.concretize_one(\"git-test-commit@1.2\")\n try:\n spec.package.do_stage()\n commits = _git_commit_list(spec.package.stage[0].source_path)\n # Outside of \"spack develop\" Spack will only pull exactly the commit it\n # needs, with no additional history\n assert len(commits) == 1\n finally:\n spec.package.do_clean()\n\n # Now use \"spack develop\": look at the resulting dev_path and make\n # sure the git repo pulled includes the full branch history (or rather,\n # more than just one commit).\n env(\"create\", \"test\")\n with ev.read(\"test\") as e:\n add(\"git-test-commit@1.2\")\n e.concretize()\n e.write()\n\n develop(\"git-test-commit@1.2\")\n e.write()\n\n spec = e.all_specs()[0]\n develop_dir = spec.variants[\"dev_path\"].value\n commits = _git_commit_list(develop_dir)\n assert len(commits) > 1\n\n\ndef test_recursive(mutable_mock_env_path, install_mockery, mock_fetch):\n env(\"create\", \"test\")\n\n with ev.read(\"test\") as e:\n add(\"indirect-mpich@1.0\")\n e.concretize()\n e.write()\n specs = e.all_specs()\n\n assert len(specs) > 1\n develop(\"--recursive\", \"mpich\")\n\n expected_dev_specs = [\"mpich\", \"direct-mpich\", \"indirect-mpich\"]\n for spec in expected_dev_specs:\n assert spec in e.dev_specs\n\n spec = next(e.roots())\n for dep in spec.traverse():\n assert dep.satisfies(\"dev_path=*\") == (dep.name in expected_dev_specs)\n\n\ndef test_develop_fails_with_multiple_concrete_versions(\n mutable_mock_env_path, install_mockery, mock_fetch, mutable_config\n):\n env(\"create\", \"test\")\n\n with ev.read(\"test\") as e:\n add(\"indirect-mpich@1.0\")\n add(\"indirect-mpich@0.9\")\n mutable_config.set(\"concretizer:unify\", False)\n e.concretize()\n\n with pytest.raises(SpackError) as develop_error:\n develop(\"indirect-mpich\", fail_on_error=True)\n\n error_str = \"has multiple concrete instances in the graph\"\n assert error_str in str(develop_error.value)\n\n\ndef test_concretize_dev_path_with_at_symbol_in_env(\n mutable_mock_env_path, tmp_path: pathlib.Path, mock_packages\n):\n spec_like = \"develop-test@develop\"\n\n develop_dir = tmp_path / \"build@location\"\n develop_dir.mkdir()\n env(\"create\", \"test_at_sym\")\n\n with ev.read(\"test_at_sym\") as e:\n add(spec_like)\n e.concretize()\n e.write()\n develop(f\"--path={develop_dir}\", spec_like)\n result = e.concrete_roots()\n\n assert len(result) == 1\n cspec = result[0]\n assert cspec.satisfies(spec_like), cspec\n assert cspec.is_develop, cspec\n assert str(develop_dir) in cspec.variants[\"dev_path\"], cspec\n\n\ndef _failing_fn(*args, **kwargs):\n # This stands in for a function that should never be called as\n # part of a test.\n assert False\n\n\n@pytest.mark.parametrize(\"_devpath_should_exist\", [True, False])\n@pytest.mark.disable_clean_stage_check\ndef test_develop_with_devpath_staging(\n monkeypatch,\n mutable_mock_env_path,\n mock_packages,\n tmp_path: pathlib.Path,\n mock_archive,\n install_mockery,\n mock_fetch,\n mock_resource_fetch,\n mock_stage,\n _devpath_should_exist,\n):\n # If the specified develop path exists, a resource should not be\n # downloaded at all at install time. Otherwise, it should be.\n\n env(\"create\", \"test\")\n\n develop_dir = tmp_path / \"build@location\"\n if _devpath_should_exist:\n develop_dir.mkdir()\n monkeypatch.setattr(URLFetchStrategy, \"fetch\", _failing_fn)\n\n spec_like = \"simple-resource@1.0\"\n\n with ev.read(\"test\") as e:\n e.add(spec_like)\n e.concretize()\n e.write()\n develop(f\"--path={develop_dir}\", spec_like)\n\n e.install_all()\n\n expected_resource_path = develop_dir / \"resource.tgz\"\n if _devpath_should_exist:\n # If we made it here, we didn't try to download anything.\n pass\n else:\n assert os.path.exists(expected_resource_path)\n" }, { "path": "lib/spack/spack/test/cmd/diff.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\n\nimport pytest\n\nimport spack.cmd.diff\nimport spack.concretize\nimport spack.main\nimport spack.paths\nimport spack.repo\nimport spack.util.spack_json as sjson\nimport spack.version\n\ninstall_cmd = spack.main.SpackCommand(\"install\")\ndiff_cmd = spack.main.SpackCommand(\"diff\")\nfind_cmd = spack.main.SpackCommand(\"find\")\n\n# Note that the hash of p1 will differ depending on the variant chosen\n# we probably always want to omit that from diffs\n# p1____\n# | \\\n# p2 v1\n# | ____/ |\n# p3 p4\n\n# i1 and i2 provide v1 (and both have the same dependencies)\n\n# All packages have an associated variant\n\n\n@pytest.fixture\ndef test_repo(config):\n builder_test_path = os.path.join(spack.paths.test_repos_path, \"spack_repo\", \"diff\")\n with spack.repo.use_repositories(builder_test_path) as mock_repo:\n yield mock_repo\n\n\ndef test_diff_ignore(test_repo):\n specA = spack.concretize.concretize_one(\"p1+usev1\")\n specB = spack.concretize.concretize_one(\"p1~usev1\")\n\n c1 = spack.cmd.diff.compare_specs(specA, specB, to_string=False)\n\n def match(function, name, args):\n limit = len(args)\n return function.name == name and list(args[:limit]) == list(function.args[:limit])\n\n def find(function_list, name, args):\n return any(match(f, name, args) for f in function_list)\n\n assert find(c1[\"a_not_b\"], \"node_os\", [\"p4\"])\n\n c2 = spack.cmd.diff.compare_specs(specA, specB, ignore_packages=[\"v1\"], to_string=False)\n\n assert not find(c2[\"a_not_b\"], \"node_os\", [\"p4\"])\n assert find(c2[\"intersect\"], \"node_os\", [\"p3\"])\n\n # Check ignoring changes on multiple packages\n\n specA = spack.concretize.concretize_one(\"p1+usev1 ^p3+p3var\")\n specA = spack.concretize.concretize_one(\"p1~usev1 ^p3~p3var\")\n\n c3 = spack.cmd.diff.compare_specs(specA, specB, to_string=False)\n assert find(c3[\"a_not_b\"], \"variant_value\", [\"p3\", \"p3var\"])\n\n c4 = spack.cmd.diff.compare_specs(specA, specB, ignore_packages=[\"v1\", \"p3\"], to_string=False)\n assert not find(c4[\"a_not_b\"], \"node_os\", [\"p4\"])\n assert not find(c4[\"a_not_b\"], \"variant_value\", [\"p3\"])\n\n\ndef test_diff_cmd(install_mockery, mock_fetch, mock_archive, mock_packages):\n \"\"\"Test that we can install two packages and diff them\"\"\"\n\n specA = spack.concretize.concretize_one(\"mpileaks\")\n specB = spack.concretize.concretize_one(\"mpileaks+debug\")\n\n # Specs should be the same as themselves\n c = spack.cmd.diff.compare_specs(specA, specA, to_string=True)\n assert len(c[\"a_not_b\"]) == 0\n assert len(c[\"b_not_a\"]) == 0\n\n # Calculate the comparison (c)\n c = spack.cmd.diff.compare_specs(specA, specB, to_string=True)\n\n # these particular diffs should have the same length b/c there aren't\n # any node differences -- just value differences.\n assert len(c[\"a_not_b\"]) == len(c[\"b_not_a\"])\n\n # ensure that variant diffs are in here the result\n assert [\"variant_value\", \"mpileaks debug False\"] in c[\"a_not_b\"]\n assert [\"variant_value\", \"mpileaks debug True\"] in c[\"b_not_a\"]\n\n # ensure that hash diffs are in here the result\n assert [\"hash\", \"mpileaks %s\" % specA.dag_hash()] in c[\"a_not_b\"]\n assert [\"hash\", \"mpileaks %s\" % specB.dag_hash()] in c[\"b_not_a\"]\n\n\ndef test_diff_runtimes(install_mockery, mock_fetch, mock_archive, mock_packages):\n \"\"\"Test that we can install two packages and diff them\"\"\"\n\n specA = spack.concretize.concretize_one(\"mpileaks\")\n specB = specA.copy()\n specB[\"gcc-runtime\"].versions = spack.version.VersionList([spack.version.Version(\"0.0.0\")])\n\n # Specs should be the same as themselves\n c = spack.cmd.diff.compare_specs(specA, specB, to_string=True)\n assert [\"version\", \"gcc-runtime 0.0.0\"] in c[\"b_not_a\"]\n\n\ndef test_load_first(install_mockery, mock_fetch, mock_archive, mock_packages):\n \"\"\"Test with and without the --first option\"\"\"\n install_cmd(\"--fake\", \"mpileaks\")\n\n # Only one version of mpileaks will work\n diff_cmd(\"mpileaks\", \"mpileaks\")\n\n # 2 specs are required for a diff\n with pytest.raises(spack.main.SpackCommandError):\n diff_cmd(\"mpileaks\")\n with pytest.raises(spack.main.SpackCommandError):\n diff_cmd(\"mpileaks\", \"mpileaks\", \"mpileaks\")\n\n # Ensure they are the same\n assert \"No differences\" in diff_cmd(\"mpileaks\", \"mpileaks\")\n output = diff_cmd(\"--json\", \"mpileaks\", \"mpileaks\")\n result = sjson.load(output)\n\n assert not result[\"a_not_b\"]\n assert not result[\"b_not_a\"]\n\n assert \"mpileaks\" in result[\"a_name\"]\n assert \"mpileaks\" in result[\"b_name\"]\n\n # spot check attributes in the intersection to ensure they describe the spec\n assert \"intersect\" in result\n assert all(\n [\"node\", dep] in result[\"intersect\"]\n for dep in (\"mpileaks\", \"callpath\", \"dyninst\", \"libelf\", \"libdwarf\", \"mpich\")\n )\n\n assert all(\n len([diff for diff in result[\"intersect\"] if diff[0] == attr]) == 9\n for attr in (\n \"version\",\n \"node_target\",\n \"node_platform\",\n \"node_os\",\n \"node\",\n \"package_hash\",\n \"hash\",\n )\n )\n\n # After we install another version, it should ask us to disambiguate\n install_cmd(\"mpileaks+debug\")\n\n # There are two versions of mpileaks\n with pytest.raises(spack.main.SpackCommandError):\n diff_cmd(\"mpileaks\", \"mpileaks+debug\")\n\n # But if we tell it to use the first, it won't try to disambiguate\n assert \"variant\" in diff_cmd(\"--first\", \"mpileaks\", \"mpileaks+debug\")\n\n # This matches them exactly\n debug_hash = find_cmd(\"--format\", \"{hash}\", \"mpileaks+debug\").strip()\n no_debug_hashes = find_cmd(\"--format\", \"{hash}\", \"mpileaks~debug\")\n no_debug_hash = no_debug_hashes.split()[0]\n output = diff_cmd(\n \"--json\", \"mpileaks/{0}\".format(debug_hash), \"mpileaks/{0}\".format(no_debug_hash)\n )\n result = sjson.load(output)\n\n assert [\"hash\", \"mpileaks %s\" % debug_hash] in result[\"a_not_b\"]\n assert [\"variant_value\", \"mpileaks debug True\"] in result[\"a_not_b\"]\n\n assert [\"hash\", \"mpileaks %s\" % no_debug_hash] in result[\"b_not_a\"]\n assert [\"variant_value\", \"mpileaks debug False\"] in result[\"b_not_a\"]\n" }, { "path": "lib/spack/spack/test/cmd/edit.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport pathlib\n\nimport spack.paths\nimport spack.repo\nimport spack.util.editor\nfrom spack.main import SpackCommand\n\nedit = SpackCommand(\"edit\")\n\n\ndef test_edit_packages(monkeypatch, mock_packages: spack.repo.RepoPath):\n \"\"\"Test spack edit pkg-a pkg-b\"\"\"\n path_a = mock_packages.filename_for_package_name(\"pkg-a\")\n path_b = mock_packages.filename_for_package_name(\"pkg-b\")\n called = False\n\n def editor(*args: str, **kwargs):\n nonlocal called\n called = True\n assert args[0] == path_a\n assert args[1] == path_b\n\n monkeypatch.setattr(spack.util.editor, \"editor\", editor)\n edit(\"pkg-a\", \"pkg-b\")\n assert called\n\n\ndef test_edit_files(monkeypatch, mock_packages):\n \"\"\"Test spack edit --build-system autotools cmake\"\"\"\n called = False\n\n def editor(*args: str, **kwargs):\n nonlocal called\n called = True\n from spack_repo.builtin_mock.build_systems import autotools, cmake # type: ignore\n\n assert os.path.samefile(args[0], autotools.__file__)\n assert os.path.samefile(args[1], cmake.__file__)\n\n monkeypatch.setattr(spack.util.editor, \"editor\", editor)\n edit(\"--build-system\", \"autotools\", \"cmake\")\n assert called\n\n\ndef test_edit_non_default_build_system(monkeypatch, mock_packages, mutable_config):\n called = False\n\n def editor(*args: str, **kwargs):\n nonlocal called\n called = True\n from spack_repo.builtin_mock.build_systems import autotools, cmake # type: ignore\n\n assert os.path.samefile(args[0], autotools.__file__)\n assert os.path.samefile(args[1], cmake.__file__)\n\n monkeypatch.setattr(spack.util.editor, \"editor\", editor)\n\n # set up an additional repo\n extra_repo_dir = pathlib.Path(spack.paths.test_repos_path) / \"spack_repo\" / \"requirements_test\"\n with spack.repo.use_repositories(str(extra_repo_dir), override=False):\n edit(\"--build-system\", \"builtin_mock.autotools\", \"builtin_mock.cmake\")\n assert called\n" }, { "path": "lib/spack/spack/test/cmd/env.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport contextlib\nimport filecmp\nimport glob\nimport io\nimport os\nimport pathlib\nimport shutil\nimport sys\nfrom argparse import Namespace\nfrom typing import Any, Dict, Optional\n\nimport pytest\n\nimport spack.cmd.env\nimport spack.concretize\nimport spack.config\nimport spack.environment as ev\nimport spack.environment.depfile as depfile\nimport spack.error\nimport spack.llnl.util.filesystem as fs\nimport spack.llnl.util.link_tree\nimport spack.llnl.util.tty as tty\nimport spack.main\nimport spack.modules\nimport spack.modules.tcl\nimport spack.package_base\nimport spack.paths\nimport spack.repo\nimport spack.schema.env\nimport spack.solver.asp\nimport spack.stage\nimport spack.store\nimport spack.util.environment\nimport spack.util.spack_json as sjson\nimport spack.util.spack_yaml\nfrom spack.cmd.env import _env_create\nfrom spack.installer import PackageInstaller\nfrom spack.llnl.util.filesystem import readlink\nfrom spack.llnl.util.lang import dedupe\nfrom spack.main import SpackCommand, SpackCommandError\nfrom spack.spec import Spec\nfrom spack.stage import stage_prefix\nfrom spack.test.conftest import RepoBuilder\nfrom spack.traverse import traverse_nodes\nfrom spack.util.executable import Executable\nfrom spack.util.path import substitute_path_variables\nfrom spack.version import Version\n\n# TODO-27021\n# everything here uses the mock_env_path\npytestmark = [\n pytest.mark.usefixtures(\"mutable_config\", \"mutable_mock_env_path\", \"mutable_mock_repo\"),\n pytest.mark.maybeslow,\n pytest.mark.not_on_windows(\"Envs unsupported on Windows\"),\n]\n\nenv = SpackCommand(\"env\")\ninstall = SpackCommand(\"install\")\nadd = SpackCommand(\"add\")\nchange = SpackCommand(\"change\")\nconfig = SpackCommand(\"config\")\nremove = SpackCommand(\"remove\")\nconcretize = SpackCommand(\"concretize\")\nstage = SpackCommand(\"stage\")\nuninstall = SpackCommand(\"uninstall\")\nfind = SpackCommand(\"find\")\ndevelop = SpackCommand(\"develop\")\nmodule = SpackCommand(\"module\")\n\nsep = os.sep\n\n\ndef setup_combined_multiple_env():\n env(\"create\", \"test1\")\n test1 = ev.read(\"test1\")\n with test1:\n add(\"mpich@1.0\")\n test1.concretize()\n test1.write()\n\n env(\"create\", \"test2\")\n test2 = ev.read(\"test2\")\n with test2:\n add(\"libelf\")\n test2.concretize()\n test2.write()\n\n env(\"create\", \"--include-concrete\", \"test1\", \"--include-concrete\", \"test2\", \"combined_env\")\n combined = ev.read(\"combined_env\")\n return test1, test2, combined\n\n\n@pytest.fixture()\ndef environment_from_manifest(tmp_path: pathlib.Path):\n \"\"\"Returns a new environment named 'test' from the content of a manifest file.\"\"\"\n\n def _create(content):\n spack_yaml = tmp_path / ev.manifest_name\n spack_yaml.write_text(content)\n return _env_create(\"test\", init_file=str(spack_yaml))\n\n return _create\n\n\ndef check_mpileaks_and_deps_in_view(viewdir: pathlib.Path):\n \"\"\"Check that the expected install directories exist.\"\"\"\n assert (viewdir / \".spack\" / \"mpileaks\").exists()\n assert (viewdir / \".spack\" / \"libdwarf\").exists()\n\n\ndef check_viewdir_removal(viewdir: pathlib.Path):\n \"\"\"Check that the uninstall/removal worked.\"\"\"\n assert not (viewdir / \".spack\").exists() or list((viewdir / \".spack\").iterdir()) == [\n viewdir / \"projections.yaml\"\n ]\n\n\ndef test_env_track_nonexistent_path_fails():\n with pytest.raises(spack.main.SpackCommandError):\n env(\"track\", \"path/does/not/exist\")\n\n assert \"doesn't contain an environment\" in env.output\n\n\ndef test_env_track_existing_env_fails():\n env(\"create\", \"track_test\")\n\n with pytest.raises(spack.main.SpackCommandError):\n env(\"track\", \"--name\", \"track_test\", ev.environment_dir_from_name(\"track_test\"))\n\n assert \"environment named track_test already exists\" in env.output\n\n\ndef test_env_track_valid(tmp_path: pathlib.Path):\n with fs.working_dir(str(tmp_path)):\n # create an independent environment\n env(\"create\", \"-d\", \".\")\n\n # test tracking an environment in known store\n env(\"track\", \"--name\", \"test1\", \".\")\n\n # test removing environment to ensure independent isn't deleted\n env(\"rm\", \"-y\", \"test1\")\n\n assert os.path.isfile(\"spack.yaml\")\n\n\ndef test_env_untrack_valid(tmp_path: pathlib.Path):\n with fs.working_dir(str(tmp_path)):\n # create an independent environment\n env(\"create\", \"-d\", \".\")\n\n # test tracking an environment in known store\n env(\"track\", \"--name\", \"test_untrack\", \".\")\n env(\"untrack\", \"--yes-to-all\", \"test_untrack\")\n\n # check that environment was successfully untracked\n out = env(\"ls\")\n assert \"test_untrack\" not in out\n\n\ndef test_env_untrack_invalid_name():\n # test untracking an environment that doesn't exist\n env_name = \"invalid_environment_untrack\"\n\n out = env(\"untrack\", env_name)\n\n assert f\"Environment '{env_name}' does not exist\" in out\n\n\ndef test_env_untrack_when_active(tmp_path: pathlib.Path):\n env_name = \"test_untrack_active\"\n\n with fs.working_dir(str(tmp_path)):\n # create an independent environment\n env(\"create\", \"-d\", \".\")\n\n # test tracking an environment in known store\n env(\"track\", \"--name\", env_name, \".\")\n\n active_env = ev.read(env_name)\n with active_env:\n output = env(\"untrack\", \"--yes-to-all\", env_name, fail_on_error=False)\n assert env.error is not None\n\n # check that environment could not be untracked while active\n assert f\"'{env_name}' can't be untracked while activated\" in output\n\n env(\"untrack\", \"-f\", env_name)\n out = env(\"ls\")\n assert env_name not in out\n\n\ndef test_env_untrack_managed():\n env_name = \"test_untrack_managed\"\n\n # create an managed environment\n env(\"create\", env_name)\n\n with pytest.raises(spack.main.SpackCommandError):\n env(\"untrack\", env_name)\n\n # check that environment could not be untracked while active\n assert f\"'{env_name}' is not a tracked env\" in env.output\n\n\n@pytest.fixture()\ndef installed_environment(\n tmp_path: pathlib.Path, mock_fetch, mock_packages, mock_archive, install_mockery\n):\n spack_yaml = tmp_path / \"spack.yaml\"\n\n @contextlib.contextmanager\n def _installed_environment(content):\n spack_yaml.write_text(content)\n with fs.working_dir(tmp_path):\n env(\"create\", \"test\", \"./spack.yaml\")\n with ev.read(\"test\") as current_environment:\n current_environment.concretize()\n current_environment.install_all(fake=True)\n current_environment.write(regenerate=True)\n\n with ev.read(\"test\") as current_environment:\n yield current_environment\n\n return _installed_environment\n\n\n@pytest.fixture()\ndef template_combinatorial_env(tmp_path: pathlib.Path):\n \"\"\"Returns a template base environment for tests. Since the environment configuration is\n extended using str.format, we need double '{' escaping for the projections.\n \"\"\"\n view_dir = tmp_path / \"view\"\n return f\"\"\"\\\n spack:\n definitions:\n - packages: [mpileaks, callpath]\n - targets: ['target=x86_64', 'target=core2']\n specs:\n - matrix:\n - [$packages]\n - [$targets]\n\n view:\n combinatorial:\n root: {view_dir}\n {{view_config}}\n projections:\n 'all': '{{{{architecture.target}}}}/{{{{name}}}}-{{{{version}}}}'\n \"\"\"\n\n\ndef test_add():\n e = ev.create(\"test\")\n e.add(\"mpileaks\")\n assert Spec(\"mpileaks\") in e.user_specs\n\n\ndef test_change_match_spec():\n env(\"create\", \"test\")\n\n e = ev.read(\"test\")\n with e:\n add(\"mpileaks@2.1\")\n add(\"mpileaks@2.2\")\n\n change(\"--match-spec\", \"mpileaks@2.2\", \"mpileaks@2.3\")\n\n assert not any(x.intersects(\"mpileaks@2.2\") for x in e.user_specs)\n assert any(x.intersects(\"mpileaks@2.3\") for x in e.user_specs)\n\n\ndef test_change_multiple_matches():\n env(\"create\", \"test\")\n\n e = ev.read(\"test\")\n with e:\n add(\"mpileaks@2.1\")\n add(\"mpileaks@2.2\")\n add(\"libelf@0.8.12%clang\")\n\n change(\"--match-spec\", \"mpileaks\", \"-a\", \"mpileaks%gcc\")\n\n assert all(x.intersects(\"%gcc\") for x in e.user_specs if x.name == \"mpileaks\")\n assert any(x.intersects(\"%clang\") for x in e.user_specs if x.name == \"libelf\")\n\n\ndef test_env_add_virtual():\n env(\"create\", \"test\")\n e = ev.read(\"test\")\n e.add(\"mpi\")\n e.concretize()\n\n assert len(e.concretized_roots) == 1\n spec = e.specs_by_hash[e.concretized_roots[0].hash]\n assert spec.intersects(\"mpi\")\n\n\ndef test_env_add_nonexistent_fails():\n env(\"create\", \"test\")\n\n e = ev.read(\"test\")\n with pytest.raises(ev.SpackEnvironmentError, match=r\"no such package\"):\n e.add(\"thispackagedoesnotexist\")\n\n\ndef test_env_list(mutable_mock_env_path):\n env(\"create\", \"foo\")\n env(\"create\", \"bar\")\n env(\"create\", \"baz\")\n\n out = env(\"list\")\n\n assert \"foo\" in out\n assert \"bar\" in out\n assert \"baz\" in out\n\n # make sure `spack env list` skips invalid things in var/spack/env\n (mutable_mock_env_path / \".DS_Store\").touch()\n out = env(\"list\")\n\n assert \"foo\" in out\n assert \"bar\" in out\n assert \"baz\" in out\n assert \".DS_Store\" not in out\n\n\ndef test_env_remove():\n env(\"create\", \"foo\")\n env(\"create\", \"bar\")\n\n out = env(\"list\")\n assert \"foo\" in out\n assert \"bar\" in out\n\n foo = ev.read(\"foo\")\n with foo:\n with pytest.raises(SpackCommandError):\n env(\"remove\", \"-y\", \"foo\")\n assert \"foo\" in env(\"list\")\n\n env(\"remove\", \"-y\", \"foo\")\n out = env(\"list\")\n assert \"foo\" not in out\n assert \"bar\" in out\n\n env(\"remove\", \"-y\", \"bar\")\n out = env(\"list\")\n assert \"foo\" not in out\n assert \"bar\" not in out\n\n\ndef test_env_rename_managed():\n # Need real environment\n with pytest.raises(spack.main.SpackCommandError):\n env(\"rename\", \"foo\", \"bar\")\n assert \"The specified name does not correspond to a managed spack environment\" in env.output\n\n env(\"create\", \"foo\")\n\n out = env(\"list\")\n assert \"foo\" in out\n\n out = env(\"rename\", \"foo\", \"bar\")\n assert \"Successfully renamed environment foo to bar\" in out\n\n out = env(\"list\")\n assert \"foo\" not in out\n assert \"bar\" in out\n\n bar = ev.read(\"bar\")\n with bar:\n # Cannot rename active environment\n with pytest.raises(spack.main.SpackCommandError):\n env(\"rename\", \"bar\", \"baz\")\n assert \"Cannot rename active environment\" in env.output\n\n env(\"create\", \"qux\")\n\n # Cannot rename to an active environment (even with force flag)\n with pytest.raises(spack.main.SpackCommandError):\n env(\"rename\", \"-f\", \"qux\", \"bar\")\n assert \"bar is an active environment\" in env.output\n\n # Can rename inactive environment when another's active\n out = env(\"rename\", \"qux\", \"quux\")\n assert \"Successfully renamed environment qux to quux\" in out\n\n out = env(\"list\")\n assert \"bar\" in out\n assert \"baz\" not in out\n\n env(\"create\", \"baz\")\n\n # Cannot rename to existing environment without --force\n with pytest.raises(spack.main.SpackCommandError):\n env(\"rename\", \"bar\", \"baz\")\n errmsg = (\n \"The new name corresponds to an existing environment;\"\n \" specify the --force flag to overwrite it.\"\n )\n assert errmsg in env.output\n\n env(\"rename\", \"-f\", \"bar\", \"baz\")\n out = env(\"list\")\n assert \"bar\" not in out\n assert \"baz\" in out\n\n\ndef test_env_rename_independent(tmp_path: pathlib.Path):\n # Need real environment\n with pytest.raises(spack.main.SpackCommandError):\n env(\"rename\", \"-d\", \"./non-existing\", \"./also-non-existing\")\n assert \"The specified path does not correspond to a valid spack environment\" in env.output\n\n anon_foo = str(tmp_path / \"foo\")\n env(\"create\", \"-d\", anon_foo)\n\n anon_bar = str(tmp_path / \"bar\")\n out = env(\"rename\", \"-d\", anon_foo, anon_bar)\n assert f\"Successfully renamed environment {anon_foo} to {anon_bar}\" in out\n assert not ev.is_env_dir(anon_foo)\n assert ev.is_env_dir(anon_bar)\n\n # Cannot rename active environment\n anon_baz = str(tmp_path / \"baz\")\n env(\"activate\", \"--sh\", \"-d\", anon_bar)\n with pytest.raises(spack.main.SpackCommandError):\n env(\"rename\", \"-d\", anon_bar, anon_baz)\n assert \"Cannot rename active environment\" in env.output\n env(\"deactivate\", \"--sh\")\n\n assert ev.is_env_dir(anon_bar)\n assert not ev.is_env_dir(anon_baz)\n\n # Cannot rename to existing environment without --force\n env(\"create\", \"-d\", anon_baz)\n with pytest.raises(spack.main.SpackCommandError):\n env(\"rename\", \"-d\", anon_bar, anon_baz)\n errmsg = (\n \"The new path corresponds to an existing environment;\"\n \" specify the --force flag to overwrite it.\"\n )\n assert errmsg in env.output\n assert ev.is_env_dir(anon_bar)\n assert ev.is_env_dir(anon_baz)\n\n env(\"rename\", \"-f\", \"-d\", anon_bar, anon_baz)\n assert not ev.is_env_dir(anon_bar)\n assert ev.is_env_dir(anon_baz)\n\n # Cannot rename to existing (non-environment) path without --force\n qux = tmp_path / \"qux\"\n qux.mkdir()\n anon_qux = str(qux)\n assert not ev.is_env_dir(anon_qux)\n\n with pytest.raises(spack.main.SpackCommandError):\n env(\"rename\", \"-d\", anon_baz, anon_qux)\n errmsg = \"The new path already exists; specify the --force flag to overwrite it.\"\n assert errmsg in env.output\n\n env(\"rename\", \"-f\", \"-d\", anon_baz, anon_qux)\n assert not ev.is_env_dir(anon_baz)\n assert ev.is_env_dir(anon_qux)\n\n\ndef test_concretize():\n e = ev.create(\"test\")\n e.add(\"mpileaks\")\n e.concretize()\n\n assert len(e.concretized_roots) == 1\n assert e.concretized_roots[0].root == Spec(\"mpileaks\")\n\n\ndef test_env_specs_partition(install_mockery, mock_fetch):\n e = ev.create(\"test\")\n e.add(\"cmake-client\")\n e.concretize()\n\n # Single not installed root spec.\n roots_already_installed, roots_to_install = e._partition_roots_by_install_status()\n assert len(roots_already_installed) == 0\n assert len(roots_to_install) == 1\n assert roots_to_install[0].name == \"cmake-client\"\n\n # Single installed root.\n e.install_all(fake=True)\n roots_already_installed, roots_to_install = e._partition_roots_by_install_status()\n assert len(roots_already_installed) == 1\n assert roots_already_installed[0].name == \"cmake-client\"\n assert len(roots_to_install) == 0\n\n # One installed root, one not installed root.\n e.add(\"mpileaks\")\n e.concretize()\n roots_already_installed, roots_to_install = e._partition_roots_by_install_status()\n assert len(roots_already_installed) == 1\n assert len(roots_to_install) == 1\n assert roots_already_installed[0].name == \"cmake-client\"\n assert roots_to_install[0].name == \"mpileaks\"\n\n\ndef test_env_install_all(install_mockery, mock_fetch):\n e = ev.create(\"test\")\n e.add(\"cmake-client\")\n e.concretize()\n e.install_all(fake=True)\n spec = next(x for x in e.all_specs_generator() if x.name == \"cmake-client\")\n assert spec.installed\n\n\ndef test_env_install_single_spec(install_mockery, mock_fetch, installer_variant):\n env(\"create\", \"test\")\n install = SpackCommand(\"install\")\n\n e = ev.read(\"test\")\n with e:\n install(\"--fake\", \"--add\", \"cmake-client\")\n\n e = ev.read(\"test\")\n assert len(e.concretized_roots) == 1\n\n item = e.concretized_roots[0]\n assert list(e.user_specs) == [Spec(\"cmake-client\")]\n assert item.root == Spec(\"cmake-client\")\n assert e.specs_by_hash[item.hash].name == \"cmake-client\"\n\n\n@pytest.mark.parametrize(\"unify\", [True, False, \"when_possible\"])\n@pytest.mark.parametrize(\"reuse\", [True, False])\ndef test_env_install_include_concrete_env(\n unify, reuse, install_mockery, mock_fetch, mutable_config\n):\n test1, test2, combined = setup_combined_multiple_env()\n\n if unify is False:\n combined.manifest.set_default_view(False)\n\n with combined:\n mutable_config.set(\"concretizer:unify\", unify)\n mutable_config.set(\"concretizer:reuse\", reuse)\n combined.add(\"mpileaks\")\n combined.concretize()\n combined.write()\n install(\"--fake\")\n\n test1_user_spec_hashes = [x.hash for x in test1.concretized_roots]\n test2_user_spec_hashes = [x.hash for x in test2.concretized_roots]\n\n for spec in combined.all_specs():\n assert spec.installed\n\n assert test1_user_spec_hashes == [\n x.hash for x in combined.included_concretized_roots[test1.path]\n ]\n assert test2_user_spec_hashes == [\n x.hash for x in combined.included_concretized_roots[test2.path]\n ]\n\n mpileaks_hash = combined.concretized_roots[0].hash\n mpileaks = combined.specs_by_hash[mpileaks_hash]\n if unify is False and reuse is False:\n # check that unification is not by accident\n assert mpileaks[\"mpi\"].dag_hash() not in test1_user_spec_hashes\n else:\n assert mpileaks[\"mpi\"].dag_hash() in test1_user_spec_hashes\n assert mpileaks[\"libelf\"].dag_hash() in test2_user_spec_hashes\n\n\ndef test_env_roots_marked_explicit(install_mockery, mock_fetch, installer_variant):\n install = SpackCommand(\"install\")\n install(\"--fake\", \"dependent-install\")\n\n # Check one explicit, one implicit install\n dependent = spack.store.STORE.db.query(explicit=True)\n dependency = spack.store.STORE.db.query(explicit=False)\n assert len(dependent) == 1\n assert len(dependency) == 1\n\n env(\"create\", \"test\")\n with ev.read(\"test\") as e:\n # make implicit install a root of the env\n e.add(dependency[0].name)\n e.concretize()\n e.install_all()\n\n explicit = spack.store.STORE.db.query(explicit=True)\n assert len(explicit) == 2\n\n\ndef test_env_modifications_error_on_activate(install_mockery, mock_fetch, monkeypatch, capfd):\n env(\"create\", \"test\")\n install = SpackCommand(\"install\")\n\n e = ev.read(\"test\")\n with e:\n install(\"--fake\", \"--add\", \"cmake-client\")\n\n def setup_error(pkg, env):\n raise RuntimeError(\"cmake-client had issues!\")\n\n pkg = spack.repo.PATH.get_pkg_class(\"cmake-client\")\n monkeypatch.setattr(pkg, \"setup_run_environment\", setup_error)\n\n ev.shell.activate(e)\n\n _, err = capfd.readouterr()\n assert \"cmake-client had issues!\" in err\n assert \"Warning: could not load runtime environment\" in err\n\n\ndef test_activate_adds_transitive_run_deps_to_path(install_mockery, mock_fetch, monkeypatch):\n env(\"create\", \"test\")\n install = SpackCommand(\"install\")\n\n e = ev.read(\"test\")\n with e:\n install(\"--add\", \"--fake\", \"depends-on-run-env\")\n\n env_variables = {}\n ev.shell.activate(e).apply_modifications(env_variables)\n assert env_variables[\"DEPENDENCY_ENV_VAR\"] == \"1\"\n\n\ndef test_env_definition_symlink(install_mockery, mock_fetch, tmp_path: pathlib.Path):\n filepath = str(tmp_path / \"spack.yaml\")\n filepath_mid = str(tmp_path / \"spack_mid.yaml\")\n\n env(\"create\", \"test\")\n e = ev.read(\"test\")\n e.add(\"mpileaks\")\n\n os.rename(e.manifest_path, filepath)\n os.symlink(filepath, filepath_mid)\n os.symlink(filepath_mid, e.manifest_path)\n\n e.concretize()\n e.write()\n\n assert os.path.islink(e.manifest_path)\n assert os.path.islink(filepath_mid)\n\n\ndef test_env_install_two_specs_same_dep(\n install_mockery, mock_fetch, tmp_path: pathlib.Path, monkeypatch\n):\n \"\"\"Test installation of two packages that share a dependency with no\n connection and the second specifying the dependency as a 'build'\n dependency.\n \"\"\"\n path = tmp_path / \"spack.yaml\"\n\n with fs.working_dir(str(tmp_path)):\n with open(str(path), \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n specs:\n - pkg-a\n - depb\n\"\"\"\n )\n\n env(\"create\", \"test\", \"spack.yaml\")\n\n with ev.read(\"test\"):\n out = install(\"--fake\")\n\n # Ensure both packages reach install phase processing and are installed\n out = str(out)\n assert \"depb: Successfully installed\" in out\n assert \"pkg-a: Successfully installed\" in out\n\n depb = spack.store.STORE.db.query_one(\"depb\", installed=True)\n assert depb, \"Expected depb to be installed\"\n\n a = spack.store.STORE.db.query_one(\"pkg-a\", installed=True)\n assert a, \"Expected pkg-a to be installed\"\n\n\ndef test_remove_after_concretize():\n e = ev.create(\"test\")\n\n e.add(\"mpileaks\")\n e.concretize()\n\n e.add(\"python\")\n e.concretize()\n\n e.remove(\"mpileaks\")\n assert Spec(\"mpileaks\") not in e.user_specs\n assert any(s.name == \"mpileaks\" for s in e.all_specs_generator())\n\n e.add(\"mpileaks\")\n assert any(s.name == \"mpileaks\" for s in e.user_specs)\n\n e.remove(\"mpileaks\", force=True)\n assert Spec(\"mpileaks\") not in e.user_specs\n assert not any(s.name == \"mpileaks\" for s in e.all_specs_generator())\n\n\ndef test_remove_before_concretize(mutable_config):\n \"\"\"Tests the effect of concretization after adding and removing specs\"\"\"\n with ev.create(\"test\") as e:\n mutable_config.set(\"concretizer:unify\", True)\n e.add(\"mpileaks\")\n e.concretize()\n assert len(e.concretized_roots) == 1\n assert e.concrete_roots()[0].satisfies(\"mpileaks\")\n\n e.remove(\"mpileaks\")\n e.concretize()\n assert not e.concretized_roots\n\n\ndef test_remove_command():\n env(\"create\", \"test\")\n assert \"test\" in env(\"list\")\n\n with ev.read(\"test\"):\n add(\"mpileaks\")\n\n with ev.read(\"test\"):\n assert \"mpileaks\" in find()\n assert \"mpileaks@\" not in find()\n assert \"mpileaks@\" not in find(\"--show-concretized\")\n\n with ev.read(\"test\"):\n remove(\"mpileaks\")\n\n with ev.read(\"test\"):\n assert \"mpileaks\" not in find()\n assert \"mpileaks@\" not in find()\n assert \"mpileaks@\" not in find(\"--show-concretized\")\n\n with ev.read(\"test\"):\n add(\"mpileaks\")\n\n with ev.read(\"test\"):\n assert \"mpileaks\" in find()\n assert \"mpileaks@\" not in find()\n assert \"mpileaks@\" not in find(\"--show-concretized\")\n\n with ev.read(\"test\"):\n concretize()\n\n with ev.read(\"test\"):\n assert \"mpileaks\" in find()\n assert \"mpileaks@\" not in find()\n assert \"mpileaks@\" in find(\"--show-concretized\")\n\n with ev.read(\"test\"):\n remove(\"mpileaks\")\n\n with ev.read(\"test\"):\n assert \"mpileaks\" not in find()\n # removed but still in last concretized specs\n assert \"mpileaks@\" in find(\"--show-concretized\")\n\n with ev.read(\"test\"):\n concretize()\n\n with ev.read(\"test\"):\n assert \"mpileaks\" not in find()\n assert \"mpileaks@\" not in find()\n # now the lockfile is regenerated and it's gone.\n assert \"mpileaks@\" not in find(\"--show-concretized\")\n\n\ndef test_remove_command_all():\n # Need separate ev.read calls for each command to ensure we test round-trip to disk\n env(\"create\", \"test\")\n test_pkgs = (\"mpileaks\", \"zlib\")\n\n with ev.read(\"test\"):\n for name in test_pkgs:\n add(name)\n\n with ev.read(\"test\"):\n for name in test_pkgs:\n assert name in find()\n assert f\"{name}@\" not in find()\n\n with ev.read(\"test\"):\n remove(\"-a\")\n\n with ev.read(\"test\"):\n for name in test_pkgs:\n assert name not in find()\n\n\ndef test_bad_remove_included_env():\n env(\"create\", \"test\")\n test = ev.read(\"test\")\n\n with test:\n add(\"mpileaks\")\n\n test.concretize()\n test.write()\n\n env(\"create\", \"--include-concrete\", \"test\", \"combined_env\")\n\n with pytest.raises(SpackCommandError):\n env(\"remove\", \"test\")\n\n\ndef test_force_remove_included_env():\n env(\"create\", \"test\")\n test = ev.read(\"test\")\n\n with test:\n add(\"mpileaks\")\n\n test.concretize()\n test.write()\n\n env(\"create\", \"--include-concrete\", \"test\", \"combined_env\")\n\n rm_output = env(\"remove\", \"-f\", \"-y\", \"test\")\n list_output = env(\"list\")\n\n assert \"'test' is used by environment 'combined_env'\" in rm_output\n assert \"test\" not in list_output\n\n\ndef test_environment_status(tmp_path: pathlib.Path, monkeypatch):\n with fs.working_dir(str(tmp_path)):\n assert \"No active environment\" in env(\"status\")\n\n with ev.create(\"test\"):\n assert \"In environment test\" in env(\"status\")\n\n with ev.create_in_dir(\"local_dir\"):\n assert os.path.join(os.getcwd(), \"local_dir\") in env(\"status\")\n\n e = ev.create_in_dir(\"myproject\")\n e.write()\n with fs.working_dir(str(tmp_path / \"myproject\")):\n with e:\n assert \"in current directory\" in env(\"status\")\n\n\ndef test_env_status_broken_view(\n mutable_mock_env_path,\n mock_archive,\n mock_fetch,\n mock_custom_repository,\n install_mockery,\n tmp_path: pathlib.Path,\n):\n with ev.create_in_dir(tmp_path):\n install(\"--add\", \"--fake\", \"trivial-install-test-package\")\n\n # switch to a new repo that doesn't include the installed package\n # test that Spack detects the missing package and warns the user\n with spack.repo.use_repositories(mock_custom_repository):\n with ev.Environment(tmp_path):\n output = env(\"status\")\n assert \"includes out of date packages or repos\" in output\n\n # Test that the warning goes away when it's fixed\n with ev.Environment(tmp_path):\n output = env(\"status\")\n assert \"includes out of date packages or repos\" not in output\n\n\ndef test_env_activate_broken_view(\n mutable_mock_env_path, mock_archive, mock_fetch, mock_custom_repository, install_mockery\n):\n with ev.create(\"test\"):\n install(\"--add\", \"--fake\", \"trivial-install-test-package\")\n\n # switch to a new repo that doesn't include the installed package\n # test that Spack detects the missing package and fails gracefully\n with spack.repo.use_repositories(mock_custom_repository):\n wrong_repo = env(\"activate\", \"--sh\", \"test\")\n assert \"Warning: could not load runtime environment\" in wrong_repo\n assert \"Unknown namespace: builtin_mock\" in wrong_repo\n\n # test replacing repo fixes it\n normal_repo = env(\"activate\", \"--sh\", \"test\")\n assert \"Warning: could not load runtime environment\" not in normal_repo\n assert \"Unknown namespace: builtin_mock\" not in normal_repo\n\n\ndef test_to_lockfile_dict():\n e = ev.create(\"test\")\n e.add(\"mpileaks\")\n e.concretize()\n context_dict = e._to_lockfile_dict()\n\n e_copy = ev.create(\"test_copy\")\n\n e_copy._read_lockfile_dict(context_dict)\n assert e.specs_by_hash == e_copy.specs_by_hash\n\n\ndef test_env_repo():\n e = ev.create(\"test\")\n e.add(\"mpileaks\")\n e.write()\n\n with ev.read(\"test\"):\n concretize()\n\n pkg_cls = e.repo.get_pkg_class(\"mpileaks\")\n assert pkg_cls.name == \"mpileaks\"\n assert pkg_cls.namespace == \"builtin_mock\"\n\n\ndef test_user_removed_spec(environment_from_manifest):\n \"\"\"Ensure a user can remove from any position in the spack.yaml file.\"\"\"\n before = environment_from_manifest(\n \"\"\"\\\nspack:\n specs:\n - mpileaks\n - hypre\n - libelf\n\"\"\"\n )\n before.concretize()\n before.write()\n\n # user modifies yaml externally to spack and removes hypre\n with open(before.manifest_path, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n specs:\n - mpileaks\n - libelf\n\"\"\"\n )\n\n after = ev.read(\"test\")\n after.concretize()\n after.write()\n\n read = ev.read(\"test\")\n assert not any(x.name == \"hypre\" for x in read.all_specs_generator())\n\n\ndef test_lockfile_spliced_specs(environment_from_manifest, install_mockery):\n \"\"\"Test that an environment can round-trip a spliced spec.\"\"\"\n # Create a local install for zmpi to splice in\n # Default concretization is not using zmpi\n zmpi = spack.concretize.concretize_one(\"zmpi\")\n PackageInstaller([zmpi.package], fake=True).install()\n\n e1 = environment_from_manifest(\n f\"\"\"\nspack:\n specs:\n - mpileaks\n concretizer:\n splice:\n explicit:\n - target: mpi\n replacement: zmpi/{zmpi.dag_hash()}\n\"\"\"\n )\n with e1:\n e1.concretize()\n e1.write()\n\n # By reading into a second environment, we force a round trip to json\n e2 = _env_create(\"test2\", init_file=e1.lock_path)\n\n # The one spec is mpileaks\n for _, spec in e2.concretized_specs():\n assert spec.spliced\n assert spec[\"mpi\"].satisfies(f\"zmpi@{zmpi.version}\")\n assert spec[\"mpi\"].build_spec.satisfies(zmpi)\n\n\ndef test_init_from_lockfile(environment_from_manifest):\n \"\"\"Test that an environment can be instantiated from a lockfile.\"\"\"\n e1 = environment_from_manifest(\n \"\"\"\nspack:\n specs:\n - mpileaks\n - hypre\n - libelf\n\"\"\"\n )\n e1.concretize()\n e1.write()\n\n e2 = _env_create(\"test2\", init_file=e1.lock_path)\n\n for s1, s2 in zip(e1.user_specs, e2.user_specs):\n assert s1 == s2\n\n for r1, r2 in zip(e1.concretized_roots, e2.concretized_roots):\n assert r1 == r2\n\n assert e1.specs_by_hash == e2.specs_by_hash\n\n\ndef test_init_from_yaml(environment_from_manifest):\n \"\"\"Test that an environment can be instantiated from a lockfile.\"\"\"\n e1 = environment_from_manifest(\n \"\"\"\nspack:\n specs:\n - mpileaks\n - hypre\n - libelf\n\"\"\"\n )\n e1.concretize()\n e1.write()\n\n e2 = _env_create(\"test2\", init_file=e1.manifest_path)\n\n for s1, s2 in zip(e1.user_specs, e2.user_specs):\n assert s1 == s2\n\n assert not e2.concretized_roots\n assert not e2.specs_by_hash\n\n\n@pytest.mark.parametrize(\"use_name\", (True, False))\ndef test_init_from_env(use_name, environment_from_manifest):\n \"\"\"Test that an environment can be instantiated from an environment dir\"\"\"\n e1 = environment_from_manifest(\n \"\"\"\nspack:\n specs:\n - mpileaks\n - hypre\n - libelf\n\"\"\"\n )\n\n with e1:\n # Test that relative paths in the env are not rewritten\n # Test that relative paths outside the env are\n dev_config = {\n \"libelf\": {\"spec\": \"libelf\", \"path\": \"./libelf\"},\n \"mpileaks\": {\"spec\": \"mpileaks\", \"path\": \"../mpileaks\"},\n }\n spack.config.set(\"develop\", dev_config)\n fs.touch(os.path.join(e1.path, \"libelf\"))\n\n e1.concretize()\n e1.write()\n\n e2 = _env_create(\"test2\", init_file=\"test\" if use_name else e1.path)\n\n for s1, s2 in zip(e1.user_specs, e2.user_specs):\n assert s1 == s2\n\n assert e2.concretized_roots == e1.concretized_roots\n assert e2.specs_by_hash == e1.specs_by_hash\n\n assert os.path.exists(os.path.join(e2.path, \"libelf\"))\n with e2:\n assert e2.dev_specs[\"libelf\"][\"path\"] == \"./libelf\"\n assert e2.dev_specs[\"mpileaks\"][\"path\"] == os.path.join(\n os.path.dirname(e1.path), \"mpileaks\"\n )\n\n\ndef test_init_from_env_no_spackfile(tmp_path):\n with pytest.raises(ev.SpackEnvironmentError, match=\"not a valid environment\"):\n _env_create(\"test\", init_file=str(tmp_path))\n\n\ndef test_init_from_yaml_relative_includes(tmp_path: pathlib.Path):\n files = [\n \"relative_copied/packages.yaml\",\n \"./relative_copied/compilers.yaml\",\n \"repos.yaml\",\n \"./config.yaml\",\n ]\n\n manifest = f\"\"\"\nspack:\n specs: []\n include: {files}\n\"\"\"\n\n e1_path = tmp_path / \"e1\"\n e1_manifest = e1_path / \"spack.yaml\"\n e1_path.mkdir(parents=True, exist_ok=True)\n with open(e1_manifest, \"w\", encoding=\"utf-8\") as f:\n f.write(manifest)\n\n for f in files:\n (e1_path / f).parent.mkdir(parents=True, exist_ok=True)\n (e1_path / f).touch()\n\n e2 = _env_create(\"test2\", init_file=str(e1_manifest))\n\n for f in files:\n assert os.path.exists(os.path.join(e2.path, f))\n\n\n# TODO: Should we be supporting relative path rewrites when creating new env from existing?\n# TODO: If so, then this should confirm that the absolute include paths in the new env exist.\ndef test_init_from_yaml_relative_includes_outside_env(tmp_path: pathlib.Path):\n \"\"\"Ensure relative includes to files outside the environment fail.\"\"\"\n files = [\"../outside_env/repos.yaml\"]\n\n manifest = f\"\"\"\nspack:\n specs: []\n include:\n - path: {files[0]}\n\"\"\"\n\n # subdir to ensure parent of environment dir is not shared\n e1_path = tmp_path / \"e1_subdir\" / \"e1\"\n e1_manifest = e1_path / \"spack.yaml\"\n e1_path.mkdir(parents=True, exist_ok=True)\n with open(e1_manifest, \"w\", encoding=\"utf-8\") as f:\n f.write(manifest)\n\n for f in files:\n file_path = e1_path / f\n file_path.parent.mkdir(parents=True, exist_ok=True)\n file_path.touch()\n\n with pytest.raises(ValueError, match=\"does not exist\"):\n _ = _env_create(\"test2\", init_file=str(e1_manifest))\n\n\ndef test_env_view_external_prefix(tmp_path: pathlib.Path, mutable_database, mock_packages):\n fake_prefix = tmp_path / \"a-prefix\"\n fake_bin = fake_prefix / \"bin\"\n fake_bin.mkdir(parents=True, exist_ok=False)\n\n manifest_dir = tmp_path / \"environment\"\n manifest_dir.mkdir(parents=True, exist_ok=False)\n manifest_file = manifest_dir / ev.manifest_name\n manifest_file.write_text(\n \"\"\"\\\nspack:\n specs:\n - pkg-a\n view: true\n\"\"\"\n )\n\n external_config = io.StringIO(\n \"\"\"\\\npackages:\n pkg-a:\n externals:\n - spec: pkg-a@2.0\n prefix: {a_prefix}\n buildable: false\n\"\"\".format(a_prefix=str(fake_prefix))\n )\n external_config_dict = spack.util.spack_yaml.load_config(external_config)\n\n test_scope = spack.config.InternalConfigScope(\"env-external-test\", data=external_config_dict)\n with spack.config.override(test_scope):\n e = ev.create(\"test\", manifest_file)\n e.concretize()\n # Note: normally installing specs in a test environment requires doing\n # a fake install, but not for external specs since no actions are\n # taken to install them. The installation commands also include\n # post-installation functions like DB-registration, so are important\n # to do (otherwise the package is not considered installed).\n e.install_all()\n e.write()\n\n env_mod = spack.util.environment.EnvironmentModifications()\n e.add_view_to_env(env_mod, \"default\")\n env_variables: Dict[str, str] = {}\n env_mod.apply_modifications(env_variables)\n assert str(fake_bin) in env_variables[\"PATH\"]\n\n\ndef test_init_with_file_and_remove(tmp_path: pathlib.Path, monkeypatch):\n \"\"\"Ensure a user can remove from any position in the spack.yaml file.\"\"\"\n path = tmp_path / \"spack.yaml\"\n\n with fs.working_dir(str(tmp_path)):\n with open(str(path), \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n specs:\n - mpileaks\n\"\"\"\n )\n\n env(\"create\", \"test\", \"spack.yaml\")\n\n out = env(\"list\")\n assert \"test\" in out\n\n with ev.read(\"test\"):\n assert \"mpileaks\" in find()\n\n env(\"remove\", \"-y\", \"test\")\n\n out = env(\"list\")\n assert \"test\" not in out\n\n\ndef test_env_with_config(environment_from_manifest):\n e = environment_from_manifest(\n \"\"\"\nspack:\n specs:\n - mpileaks\n packages:\n mpileaks:\n version: [\"2.2\"]\n\"\"\"\n )\n with e:\n e.concretize()\n\n mpileaks_hash = next(x.hash for x in e.concretized_roots if x.root == Spec(\"mpileaks\"))\n mpileaks = e.specs_by_hash[mpileaks_hash]\n assert mpileaks.satisfies(\"mpileaks@2.2\")\n\n\ndef test_with_config_bad_include_create(environment_from_manifest):\n \"\"\"Confirm missing required include raises expected exception.\"\"\"\n err = \"does not exist\"\n with pytest.raises(ValueError, match=err):\n environment_from_manifest(\n \"\"\"\nspack:\n include:\n - /no/such/directory\n\"\"\"\n )\n\n\ndef test_with_config_bad_include_activate(environment_from_manifest, tmp_path: pathlib.Path):\n env_root = tmp_path / \"env-root\"\n env_root.mkdir()\n include1 = env_root / \"include1.yaml\"\n include1.touch()\n\n spack_yaml = env_root / ev.manifest_name\n spack_yaml.write_text(\n \"\"\"\nspack:\n include:\n - ./include1.yaml\n\"\"\"\n )\n\n with ev.Environment(env_root) as e:\n e.concretize()\n\n # We've created an environment with included config file (which does\n # exist). Now we remove it and check that we get a sensible error.\n\n os.remove(include1)\n with pytest.raises(ValueError, match=\"does not exist\"):\n ev.activate(ev.Environment(env_root))\n\n assert ev.active_environment() is None\n\n\ndef test_env_with_include_config_files_same_basename(\n tmp_path: pathlib.Path, environment_from_manifest\n):\n file1 = tmp_path / \"path\" / \"to\" / \"included-config.yaml\"\n file1.parent.mkdir(parents=True, exist_ok=True)\n with open(file1, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\n packages:\n libelf:\n version: [\"0.8.10\"]\n \"\"\"\n )\n\n file2 = tmp_path / \"second\" / \"path\" / \"included-config.yaml\"\n file2.parent.mkdir(parents=True, exist_ok=True)\n with open(file2, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\n packages:\n mpileaks:\n version: [\"2.2\"]\n \"\"\"\n )\n\n e = environment_from_manifest(\n f\"\"\"\nspack:\n include:\n - {file1}\n - {file2}\n specs:\n - libelf\n - mpileaks\n\"\"\"\n )\n\n with e:\n e.concretize()\n\n mpileaks_hash = next(x.hash for x in e.concretized_roots if x.root == Spec(\"mpileaks\"))\n mpileaks = e.specs_by_hash[mpileaks_hash]\n assert mpileaks.satisfies(\"mpileaks@2.2\")\n\n libelf_hash = next(x.hash for x in e.concretized_roots if x.root == Spec(\"libelf\"))\n libelf = e.specs_by_hash[libelf_hash]\n assert libelf.satisfies(\"libelf@0.8.10\")\n\n\n@pytest.fixture(scope=\"function\")\ndef packages_file(tmp_path: pathlib.Path):\n \"\"\"Return the path to the packages configuration file.\"\"\"\n raw_yaml = \"\"\"\npackages:\n mpileaks:\n version: [\"2.2\"]\n\"\"\"\n config_dir = tmp_path / \"testconfig\"\n config_dir.mkdir()\n filename = config_dir / \"packages.yaml\"\n filename.write_text(raw_yaml)\n yield filename\n\n\ndef mpileaks_env_config(include_path):\n \"\"\"Return the contents of an environment that includes the provided\n path and lists mpileaks as the sole spec.\"\"\"\n return \"\"\"\\\nspack:\n include:\n - {0}\n specs:\n - mpileaks\n\"\"\".format(include_path)\n\n\ndef test_env_with_included_config_file(mutable_mock_env_path, packages_file):\n \"\"\"Test inclusion of a relative packages configuration file added to an\n existing environment.\n \"\"\"\n env_root = mutable_mock_env_path\n env_root.mkdir(parents=True, exist_ok=True)\n include_filename = \"included-config.yaml\"\n included_path = env_root / include_filename\n shutil.move(str(packages_file), included_path)\n\n spack_yaml = env_root / ev.manifest_name\n spack_yaml.write_text(\n f\"\"\"\\\nspack:\n include:\n - {os.path.join(\".\", include_filename)}\n specs:\n - mpileaks\n\"\"\"\n )\n\n e = ev.Environment(env_root)\n with e:\n e.concretize()\n\n mpileaks_hash = next(x.hash for x in e.concretized_roots if x.root == Spec(\"mpileaks\"))\n mpileaks = e.specs_by_hash[mpileaks_hash]\n assert mpileaks.satisfies(\"mpileaks@2.2\")\n\n\ndef test_config_change_existing(\n mutable_mock_env_path, tmp_path: pathlib.Path, mock_packages, mutable_config\n):\n \"\"\"Test ``config change`` with config in the ``spack.yaml`` as well as an\n included file scope.\n \"\"\"\n\n env_path = tmp_path / \"test_config\"\n env_path.mkdir(parents=True, exist_ok=True)\n included_file = \"included-packages.yaml\"\n included_path = env_path / included_file\n with open(included_path, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\npackages:\n mpich:\n require:\n - spec: \"@3.0.2\"\n libelf:\n require: \"@0.8.10\"\n bowtie:\n require:\n - one_of: [\"@1.3.0\", \"@1.2.0\"]\n\"\"\"\n )\n\n spack_yaml = env_path / ev.manifest_name\n spack_yaml.write_text(\n f\"\"\"\\\nspack:\n packages:\n mpich:\n require:\n - spec: \"+debug\"\n include:\n - {os.path.join(\".\", included_file)}\n specs: []\n\"\"\"\n )\n\n mutable_config.set(\"config:misc_cache\", str(tmp_path / \"cache\"))\n e = ev.Environment(env_path)\n with e:\n # List of requirements, flip a variant\n config(\"change\", \"packages:mpich:require:~debug\")\n test_spec = spack.concretize.concretize_one(\"mpich\")\n assert test_spec.satisfies(\"@3.0.2~debug\")\n\n # List of requirements, change the version (in a different scope)\n config(\"change\", \"packages:mpich:require:@3.0.3\")\n test_spec = spack.concretize.concretize_one(\"mpich\")\n assert test_spec.satisfies(\"@3.0.3\")\n\n # \"require:\" as a single string, also try specifying\n # a spec string that requires enclosing in quotes as\n # part of the config path\n config(\"change\", 'packages:libelf:require:\"@0.8.12:\"')\n spack.concretize.concretize_one(\"libelf@0.8.12\")\n # No need for assert, if there wasn't a failure, we\n # changed the requirement successfully.\n\n # Use change to add a requirement for a package that\n # has no requirements defined\n config(\"change\", \"packages:fftw:require:+mpi\")\n test_spec = spack.concretize.concretize_one(\"fftw\")\n assert test_spec.satisfies(\"+mpi\")\n config(\"change\", \"packages:fftw:require:~mpi\")\n test_spec = spack.concretize.concretize_one(\"fftw\")\n assert test_spec.satisfies(\"~mpi\")\n config(\"change\", \"packages:fftw:require:@1.0\")\n test_spec = spack.concretize.concretize_one(\"fftw\")\n assert test_spec.satisfies(\"@1.0~mpi\")\n\n # Use \"--match-spec\" to change one spec in a \"one_of\"\n # list\n config(\"change\", \"packages:bowtie:require:@1.2.2\", \"--match-spec\", \"@1.2.0\")\n # confirm that we can concretize to either value\n spack.concretize.concretize_one(\"bowtie@1.3.0\")\n spack.concretize.concretize_one(\"bowtie@1.2.2\")\n # confirm that we cannot concretize to the old value\n with pytest.raises(spack.solver.asp.UnsatisfiableSpecError):\n spack.concretize.concretize_one(\"bowtie@1.2.0\")\n\n\ndef test_config_change_new(\n mutable_mock_env_path, tmp_path: pathlib.Path, mock_packages, mutable_config\n):\n spack_yaml = tmp_path / ev.manifest_name\n spack_yaml.write_text(\n \"\"\"\\\nspack:\n specs: []\n\"\"\"\n )\n\n with ev.Environment(tmp_path):\n config(\"change\", \"packages:mpich:require:~debug\")\n with pytest.raises(spack.solver.asp.UnsatisfiableSpecError):\n spack.concretize.concretize_one(\"mpich+debug\")\n spack.concretize.concretize_one(\"mpich~debug\")\n\n # Now check that we raise an error if we need to add a require: constraint\n # when preexisting config manually specified it as a singular spec\n spack_yaml.write_text(\n \"\"\"\\\nspack:\n specs: []\n packages:\n mpich:\n require: \"@3.0.3\"\n\"\"\"\n )\n with ev.Environment(tmp_path):\n assert spack.concretize.concretize_one(\"mpich\").satisfies(\"@3.0.3\")\n with pytest.raises(spack.error.ConfigError, match=\"not a list\"):\n config(\"change\", \"packages:mpich:require:~debug\")\n\n\ndef test_env_with_included_config_file_url(\n tmp_path: pathlib.Path, mutable_empty_config, packages_file\n):\n \"\"\"Test configuration inclusion of a file whose path is a URL before\n the environment is concretized.\"\"\"\n\n spack_yaml = tmp_path / \"spack.yaml\"\n with spack_yaml.open(\"w\") as f:\n f.write(\"spack:\\n include:\\n - {0}\\n\".format(packages_file.as_uri()))\n\n env = ev.Environment(str(tmp_path))\n ev.activate(env)\n\n cfg = spack.config.get(\"packages\")\n assert cfg[\"mpileaks\"][\"version\"] == [\"2.2\"]\n\n\ndef test_env_with_included_config_scope(mutable_mock_env_path, packages_file):\n \"\"\"Test inclusion of a package file from the environment's configuration\n stage directory. This test is intended to represent a case where a remote\n file has already been staged.\"\"\"\n env_root = mutable_mock_env_path\n config_scope_path = env_root / \"config\"\n\n # Copy the packages.yaml file to the environment configuration\n # directory, so it is picked up during concretization. (Using\n # copy instead of rename in case the fixture scope changes.)\n config_scope_path.mkdir(parents=True, exist_ok=True)\n include_filename = packages_file.name\n included_path = config_scope_path / include_filename\n fs.copy(str(packages_file), included_path)\n\n # Configure the environment to include file(s) from the environment's\n # remote configuration stage directory.\n spack_yaml = env_root / ev.manifest_name\n spack_yaml.write_text(mpileaks_env_config(config_scope_path))\n\n # Ensure the concretized environment reflects contents of the\n # packages.yaml file.\n e = ev.Environment(env_root)\n with e:\n e.concretize()\n\n mpileaks_hash = next(x.hash for x in e.concretized_roots if x.root == Spec(\"mpileaks\"))\n mpileaks = e.specs_by_hash[mpileaks_hash]\n assert mpileaks.satisfies(\"mpileaks@2.2\")\n\n\ndef test_env_with_included_config_var_path(tmp_path: pathlib.Path, packages_file):\n \"\"\"Test inclusion of a package configuration file with path variables\n \"staged\" in the environment's configuration stage directory.\"\"\"\n included_file = str(packages_file)\n env_path = tmp_path\n config_var_path = os.path.join(\"$tempdir\", \"included-packages.yaml\")\n\n spack_yaml = env_path / ev.manifest_name\n spack_yaml.write_text(mpileaks_env_config(config_var_path))\n\n config_real_path = substitute_path_variables(config_var_path)\n shutil.move(included_file, config_real_path)\n assert os.path.exists(config_real_path)\n\n e = ev.Environment(env_path)\n with e:\n e.concretize()\n\n mpileaks_hash = next(x.hash for x in e.concretized_roots if x.root == Spec(\"mpileaks\"))\n mpileaks = e.specs_by_hash[mpileaks_hash]\n assert mpileaks.satisfies(\"mpileaks@2.2\")\n\n\ndef test_env_with_included_config_precedence(tmp_path: pathlib.Path):\n \"\"\"Test included scope and manifest precedence when including a package\n configuration file.\"\"\"\n\n included_file = \"included-packages.yaml\"\n included_path = tmp_path / included_file\n with open(included_path, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\npackages:\n mpileaks:\n version: [\"2.2\"]\n libelf:\n version: [\"0.8.10\"]\n\"\"\"\n )\n\n spack_yaml = tmp_path / ev.manifest_name\n spack_yaml.write_text(\n f\"\"\"\\\nspack:\n packages:\n libelf:\n version: [\"0.8.12\"]\n include:\n - {os.path.join(\".\", included_file)}\n specs:\n - mpileaks\n\"\"\"\n )\n\n e = ev.Environment(tmp_path)\n with e:\n e.concretize()\n\n mpileaks_hash = next(x.hash for x in e.concretized_roots if x.root == Spec(\"mpileaks\"))\n mpileaks = e.specs_by_hash[mpileaks_hash]\n\n # ensure included scope took effect\n assert mpileaks.satisfies(\"mpileaks@2.2\")\n\n # ensure env file takes precedence\n assert mpileaks[\"libelf\"].satisfies(\"libelf@0.8.12\")\n\n\ndef test_env_with_included_configs_precedence(tmp_path: pathlib.Path):\n \"\"\"Test precedence of multiple included configuration files.\"\"\"\n file1 = \"high-config.yaml\"\n file2 = \"low-config.yaml\"\n\n spack_yaml = tmp_path / ev.manifest_name\n spack_yaml.write_text(\n f\"\"\"\\\nspack:\n include:\n - {os.path.join(\".\", file1)} # this one should take precedence\n - {os.path.join(\".\", file2)}\n specs:\n - mpileaks\n\"\"\"\n )\n\n with open(tmp_path / file1, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\npackages:\n libelf:\n version: [\"0.8.10\"] # this should override libelf version below\n\"\"\"\n )\n\n with open(tmp_path / file2, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\npackages:\n mpileaks:\n version: [\"2.2\"]\n libelf:\n version: [\"0.8.12\"]\n\"\"\"\n )\n\n e = ev.Environment(tmp_path)\n with e:\n e.concretize()\n\n mpileaks_hash = next(x.hash for x in e.concretized_roots if x.root == Spec(\"mpileaks\"))\n mpileaks = e.specs_by_hash[mpileaks_hash]\n\n # ensure the included package spec took precedence over manifest spec\n assert mpileaks.satisfies(\"mpileaks@2.2\")\n\n # ensure the first included package spec took precedence over one from second\n assert mpileaks[\"libelf\"].satisfies(\"libelf@0.8.10\")\n\n\n@pytest.mark.regression(\"39248\")\ndef test_bad_env_yaml_format_remove(mutable_mock_env_path):\n badenv = \"badenv\"\n env(\"create\", badenv)\n filename = mutable_mock_env_path / \"spack.yaml\"\n with open(filename, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\n - mpileaks\n\"\"\"\n )\n\n assert badenv in env(\"list\")\n env(\"remove\", \"-y\", badenv)\n assert badenv not in env(\"list\")\n\n\n@pytest.mark.regression(\"39248\")\n@pytest.mark.parametrize(\n \"error,message,contents\",\n [\n (\n spack.config.ConfigFormatError,\n \"not of type\",\n \"\"\"\\\nspack:\n specs: mpi@2.0\n\"\"\",\n ),\n (\n ev.SpackEnvironmentConfigError,\n \"duplicate key\",\n \"\"\"\\\nspack:\n packages:\n all:\n providers:\n mpi: [mvapich2]\n mpi: [mpich]\n\"\"\",\n ),\n (\n spack.config.ConfigFormatError,\n \"'specks' was unexpected\",\n \"\"\"\\\nspack:\n specks:\n - libdwarf\n\"\"\",\n ),\n ],\n)\ndef test_bad_env_yaml_create_fails(\n tmp_path: pathlib.Path, mutable_mock_env_path, error, message, contents\n):\n \"\"\"Ensure creation with invalid yaml does NOT create or leave the environment.\"\"\"\n filename = tmp_path / ev.manifest_name\n filename.write_text(contents)\n env_name = \"bad_env\"\n with pytest.raises(error, match=message):\n env(\"create\", env_name, str(filename))\n\n assert env_name not in env(\"list\")\n manifest = mutable_mock_env_path / env_name / ev.manifest_name\n assert not os.path.exists(str(manifest))\n\n\n@pytest.mark.regression(\"39248\")\n@pytest.mark.parametrize(\"answer\", [\"-y\", \"\"])\ndef test_multi_env_remove(mutable_mock_env_path, monkeypatch, answer):\n \"\"\"Test removal (or not) of a valid and invalid environment\"\"\"\n remove_environment = answer == \"-y\"\n monkeypatch.setattr(tty, \"get_yes_or_no\", lambda prompt, default: remove_environment)\n\n environments = [\"goodenv\", \"badenv\"]\n for e in environments:\n env(\"create\", e)\n\n # Ensure the bad environment contains invalid yaml\n filename = mutable_mock_env_path / environments[1] / ev.manifest_name\n filename.write_text(\n \"\"\"\\\n - libdwarf\n\"\"\"\n )\n\n assert all(e in env(\"list\") for e in environments)\n\n args = [answer] if answer else []\n args.extend(environments)\n output = env(\"remove\", *args, fail_on_error=False)\n\n if remove_environment is True:\n # Successfully removed (and reported removal) of *both* environments\n assert not all(e in env(\"list\") for e in environments)\n assert output.count(\"Successfully removed\") == len(environments)\n else:\n # Not removing any of the environments\n assert all(e in env(\"list\") for e in environments)\n\n\ndef test_env_loads(install_mockery, mock_fetch, mock_modules_root):\n env(\"create\", \"test\")\n\n with ev.read(\"test\"):\n add(\"mpileaks\")\n concretize()\n install(\"--fake\")\n module(\"tcl\", \"refresh\", \"-y\")\n\n with ev.read(\"test\"):\n env(\"loads\")\n\n e = ev.read(\"test\")\n\n loads_file = os.path.join(e.path, \"loads\")\n assert os.path.exists(loads_file)\n\n with open(loads_file, encoding=\"utf-8\") as f:\n contents = f.read()\n assert \"module load mpileaks\" in contents\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_stage(mock_stage, mock_fetch, install_mockery):\n env(\"create\", \"test\")\n with ev.read(\"test\"):\n add(\"mpileaks\")\n add(\"zmpi\")\n concretize()\n stage()\n\n root = str(mock_stage)\n\n def check_stage(spec):\n spec = spack.concretize.concretize_one(spec)\n for dep in spec.traverse():\n stage_name = f\"{stage_prefix}{dep.name}-{dep.version}-{dep.dag_hash()}\"\n if dep.external:\n assert not os.path.exists(os.path.join(root, stage_name))\n else:\n assert os.path.isdir(os.path.join(root, stage_name))\n\n check_stage(\"mpileaks\")\n check_stage(\"zmpi\")\n\n\ndef test_env_commands_die_with_no_env_arg():\n # these fail in argparse when given no arg\n with pytest.raises(SpackCommandError):\n env(\"create\")\n with pytest.raises(SpackCommandError):\n env(\"remove\")\n\n # these have an optional env arg and raise errors via tty.die\n with pytest.raises(SpackCommandError):\n env(\"loads\")\n\n # This should NOT raise an error with no environment\n # it just tells the user there isn't an environment\n env(\"status\")\n\n\ndef test_env_blocks_uninstall(mock_stage, mock_fetch, install_mockery):\n env(\"create\", \"test\")\n with ev.read(\"test\"):\n add(\"mpileaks\")\n install(\"--fake\")\n\n out = uninstall(\"-y\", \"mpileaks\", fail_on_error=False)\n assert uninstall.returncode == 1\n assert \"The following environments still reference these specs\" in out\n\n\ndef test_roots_display_with_variants():\n env(\"create\", \"test\")\n with ev.read(\"test\"):\n add(\"boost+shared\")\n\n with ev.read(\"test\"):\n out = find()\n\n assert \"boost+shared\" in out\n\n\ndef test_uninstall_keeps_in_env(mock_stage, mock_fetch, install_mockery):\n # 'spack uninstall' without --remove should not change the environment\n # spack.yaml file, just uninstall specs\n env(\"create\", \"test\")\n with ev.read(\"test\"):\n add(\"mpileaks\")\n add(\"libelf\")\n install(\"--fake\")\n\n test = ev.read(\"test\")\n # Save this spec to check later if it is still in the env\n (mpileaks_hash,) = list(x for x, y in test.specs_by_hash.items() if y.name == \"mpileaks\")\n user_specs_before = test.user_specs\n user_spec_hashes_before = {x.hash for x in test.concretized_roots}\n\n with ev.read(\"test\"):\n uninstall(\"-ya\")\n\n test = ev.read(\"test\")\n assert {x.hash for x in test.concretized_roots} == user_spec_hashes_before\n assert test.user_specs.specs == user_specs_before.specs\n assert mpileaks_hash in test.specs_by_hash\n assert not test.specs_by_hash[mpileaks_hash].installed\n\n\ndef test_uninstall_removes_from_env(mock_stage, mock_fetch, install_mockery):\n # 'spack uninstall --remove' should update the environment\n env(\"create\", \"test\")\n with ev.read(\"test\"):\n add(\"mpileaks\")\n add(\"libelf\")\n install(\"--fake\")\n\n with ev.read(\"test\"):\n uninstall(\"-y\", \"-a\", \"--remove\")\n\n test = ev.read(\"test\")\n assert not test.specs_by_hash\n assert not test.concretized_roots\n assert not test.user_specs\n\n\ndef test_indirect_build_dep(repo_builder: RepoBuilder):\n \"\"\"Simple case of X->Y->Z where Y is a build/link dep and Z is a\n build-only dep. Make sure this concrete DAG is preserved when writing the\n environment out and reading it back.\n \"\"\"\n repo_builder.add_package(\"z\")\n repo_builder.add_package(\"y\", dependencies=[(\"z\", \"build\", None)])\n repo_builder.add_package(\"x\", dependencies=[(\"y\", None, None)])\n\n with spack.repo.use_repositories(repo_builder.root):\n x_spec = Spec(\"x\")\n x_concretized = spack.concretize.concretize_one(x_spec)\n\n _env_create(\"test\", with_view=False)\n e = ev.read(\"test\")\n e.add(x_spec)\n e.concretize()\n e.write()\n\n e_read = ev.read(\"test\")\n assert len(e_read.concretized_roots) == 1\n x_env_hash = e_read.concretized_roots[0].hash\n x_env_spec = e_read.specs_by_hash[x_env_hash]\n assert x_env_spec == x_concretized\n\n\ndef test_store_different_build_deps(repo_builder: RepoBuilder):\n r\"\"\"Ensure that an environment can store two instances of a build-only\n dependency::\n\n x y\n /| (l) | (b)\n (b) | y z2\n \\| (b)\n z1\n\n \"\"\"\n repo_builder.add_package(\"z\")\n repo_builder.add_package(\"y\", dependencies=[(\"z\", \"build\", None)])\n repo_builder.add_package(\"x\", dependencies=[(\"y\", None, None), (\"z\", \"build\", None)])\n\n with spack.repo.use_repositories(repo_builder.root):\n y_spec = Spec(\"y ^z@3\")\n y_concretized = spack.concretize.concretize_one(y_spec)\n\n x_spec = Spec(\"x ^z@2\")\n x_concretized = spack.concretize.concretize_one(x_spec)\n\n # Even though x chose a different 'z', the y it chooses should be identical\n # *aside* from the dependency on 'z'. The dag_hash() will show the difference\n # in build dependencies.\n assert x_concretized[\"y\"].eq_node(y_concretized)\n assert x_concretized[\"y\"].dag_hash() != y_concretized.dag_hash()\n\n _env_create(\"test\", with_view=False)\n e = ev.read(\"test\")\n e.add(y_spec)\n e.add(x_spec)\n e.concretize()\n e.write()\n\n e_read = ev.read(\"test\")\n y_env_hash, x_env_hash = [x.hash for x in e_read.concretized_roots]\n\n y_read = e_read.specs_by_hash[y_env_hash]\n x_read = e_read.specs_by_hash[x_env_hash]\n\n # make sure the DAG hashes and build deps are preserved after\n # a round trip to/from the lockfile\n assert x_read[\"z\"] != y_read[\"z\"]\n assert x_read[\"z\"].dag_hash() != y_read[\"z\"].dag_hash()\n\n assert x_read[\"y\"].eq_node(y_read)\n assert x_read[\"y\"].dag_hash() != y_read.dag_hash()\n\n\ndef test_env_updates_view_install(tmp_path: pathlib.Path, mock_stage, mock_fetch, install_mockery):\n view_dir = tmp_path / \"view\"\n env(\"create\", \"--with-view=%s\" % view_dir, \"test\")\n with ev.read(\"test\"):\n add(\"mpileaks\")\n install(\"--fake\")\n\n check_mpileaks_and_deps_in_view(view_dir)\n\n\ndef test_env_view_fails(\n tmp_path: pathlib.Path, mock_packages, mock_stage, mock_fetch, install_mockery\n):\n # We currently ignore file-file conflicts for the prefix merge,\n # so in principle there will be no errors in this test. But\n # the .spack metadata dir is handled separately and is more strict.\n # It also throws on file-file conflicts. That's what we're checking here\n # by adding the same package twice to a view.\n view_dir = tmp_path / \"view\"\n env(\"create\", \"--with-view=%s\" % view_dir, \"test\")\n with ev.read(\"test\"):\n add(\"libelf\")\n add(\"libelf cflags=-g\")\n with pytest.raises(\n ev.SpackEnvironmentViewError, match=\"two specs project to the same prefix\"\n ):\n install(\"--fake\")\n\n\ndef test_env_view_fails_dir_file(\n tmp_path: pathlib.Path, mock_packages, mock_stage, mock_fetch, install_mockery\n):\n # This environment view fails to be created because a file\n # and a dir are in the same path. Test that it mentions the problematic path.\n view_dir = tmp_path / \"view\"\n env(\"create\", \"--with-view=%s\" % view_dir, \"test\")\n with ev.read(\"test\"):\n add(\"view-file\")\n add(\"view-dir\")\n with pytest.raises(\n spack.llnl.util.link_tree.MergeConflictSummary, match=os.path.join(\"bin\", \"x\")\n ):\n install()\n\n\ndef test_env_view_succeeds_symlinked_dir_file(\n tmp_path: pathlib.Path, mock_packages, mock_stage, mock_fetch, install_mockery\n):\n # A symlinked dir and an ordinary dir merge happily\n view_dir = tmp_path / \"view\"\n env(\"create\", \"--with-view=%s\" % view_dir, \"test\")\n with ev.read(\"test\"):\n add(\"view-symlinked-dir\")\n add(\"view-dir\")\n install()\n x_dir = os.path.join(str(view_dir), \"bin\", \"x\")\n assert os.path.exists(os.path.join(x_dir, \"file_in_dir\"))\n assert os.path.exists(os.path.join(x_dir, \"file_in_symlinked_dir\"))\n\n\ndef test_env_without_view_install(tmp_path: pathlib.Path, mock_stage, mock_fetch, install_mockery):\n # Test enabling a view after installing specs\n env(\"create\", \"--without-view\", \"test\")\n\n test_env = ev.read(\"test\")\n with pytest.raises(ev.SpackEnvironmentError):\n test_env.default_view\n\n view_dir = tmp_path / \"view\"\n\n with ev.read(\"test\"):\n add(\"mpileaks\")\n install(\"--fake\")\n\n env(\"view\", \"enable\", str(view_dir))\n\n # After enabling the view, the specs should be linked into the environment\n # view dir\n check_mpileaks_and_deps_in_view(view_dir)\n\n\n@pytest.mark.parametrize(\"env_name\", [True, False])\ndef test_env_include_concrete_env_yaml(env_name):\n env(\"create\", \"test\")\n test = ev.read(\"test\")\n\n with test:\n add(\"mpileaks\")\n test.concretize()\n test.write()\n\n environ = \"test\" if env_name else test.path\n\n env(\"create\", \"--include-concrete\", environ, \"combined_env\")\n\n combined = ev.read(\"combined_env\")\n combined_yaml = combined.manifest[\"spack\"]\n\n assert ev.lockfile_include_key in combined_yaml\n assert test.path in combined_yaml[ev.lockfile_include_key]\n\n\n@pytest.mark.regression(\"45766\")\n@pytest.mark.parametrize(\"format\", [\"v1\", \"v2\", \"v3\"])\ndef test_env_include_concrete_old_env(format):\n lockfile = os.path.join(spack.paths.test_path, \"data\", \"legacy_env\", f\"{format}.lock\")\n # create an env from old .lock file -- this does not update the format\n env(\"create\", \"old-env\", lockfile)\n env(\"create\", \"--include-concrete\", \"old-env\", \"test\")\n\n assert ev.read(\"old-env\").all_specs() == ev.read(\"test\").all_specs()\n\n\ndef test_env_bad_include_concrete_env():\n with pytest.raises(ev.SpackEnvironmentError):\n env(\"create\", \"--include-concrete\", \"nonexistent_env\", \"combined_env\")\n\n\ndef test_env_not_concrete_include_concrete_env():\n env(\"create\", \"test\")\n test = ev.read(\"test\")\n\n with test:\n add(\"mpileaks\")\n\n with pytest.raises(ev.SpackEnvironmentError):\n env(\"create\", \"--include-concrete\", \"test\", \"combined_env\")\n\n\ndef test_env_multiple_include_concrete_envs():\n test1, test2, combined = setup_combined_multiple_env()\n\n combined_yaml = combined.manifest[\"spack\"]\n\n assert test1.path in combined_yaml[ev.lockfile_include_key][0]\n assert test2.path in combined_yaml[ev.lockfile_include_key][1]\n\n # No local specs in the combined env\n assert not combined_yaml[\"specs\"]\n\n\ndef test_env_include_concrete_envs_lockfile():\n test1, test2, combined = setup_combined_multiple_env()\n\n combined_yaml = combined.manifest[\"spack\"]\n\n assert ev.lockfile_include_key in combined_yaml\n assert test1.path in combined_yaml[ev.lockfile_include_key]\n\n with open(combined.lock_path, encoding=\"utf-8\") as f:\n lockfile_as_dict = combined._read_lockfile(f)\n\n assert set(\n entry[\"hash\"] for entry in lockfile_as_dict[ev.lockfile_include_key][test1.path][\"roots\"]\n ) == set(test1.specs_by_hash)\n assert set(\n entry[\"hash\"] for entry in lockfile_as_dict[ev.lockfile_include_key][test2.path][\"roots\"]\n ) == set(test2.specs_by_hash)\n\n\ndef test_env_include_concrete_add_env():\n test1, test2, combined = setup_combined_multiple_env()\n\n # create new env & concretize\n env(\"create\", \"new\")\n new_env = ev.read(\"new\")\n with new_env:\n add(\"mpileaks\")\n\n new_env.concretize()\n new_env.write()\n\n # add new env to combined\n combined.included_concrete_env_root_dirs.append(new_env.path)\n\n # assert thing haven't changed yet\n with open(combined.lock_path, encoding=\"utf-8\") as f:\n lockfile_as_dict = combined._read_lockfile(f)\n\n assert new_env.path not in lockfile_as_dict[ev.lockfile_include_key].keys()\n\n # concretize combined env with new env\n combined.concretize()\n combined.write()\n\n # assert changes\n with open(combined.lock_path, encoding=\"utf-8\") as f:\n lockfile_as_dict = combined._read_lockfile(f)\n\n assert new_env.path in lockfile_as_dict[ev.lockfile_include_key].keys()\n\n\ndef test_env_include_concrete_remove_env():\n test1, test2, combined = setup_combined_multiple_env()\n\n # remove test2 from combined\n combined.included_concrete_env_root_dirs = [test1.path]\n\n # assert test2 is still in combined's lockfile\n with open(combined.lock_path, encoding=\"utf-8\") as f:\n lockfile_as_dict = combined._read_lockfile(f)\n\n assert test2.path in lockfile_as_dict[ev.lockfile_include_key].keys()\n\n # reconcretize combined\n combined.concretize()\n combined.write()\n\n # assert test2 is not in combined's lockfile\n with open(combined.lock_path, encoding=\"utf-8\") as f:\n lockfile_as_dict = combined._read_lockfile(f)\n\n assert test2.path not in lockfile_as_dict[ev.lockfile_include_key].keys()\n\n\ndef configure_reuse(reuse_mode, combined_env) -> Optional[ev.Environment]:\n override_env = None\n _config: Dict[Any, Any] = {}\n if reuse_mode == \"true\":\n _config = {\"concretizer\": {\"reuse\": True}}\n elif reuse_mode == \"from_environment\":\n _config = {\"concretizer\": {\"reuse\": {\"from\": [{\"type\": \"environment\"}]}}}\n elif reuse_mode == \"from_environment_test1\":\n _config = {\"concretizer\": {\"reuse\": {\"from\": [{\"type\": \"environment\", \"path\": \"test1\"}]}}}\n elif reuse_mode == \"from_environment_external_test\":\n # Create a new environment called external_test that enables the \"debug\"\n # The default is \"~debug\"\n env(\"create\", \"external_test\")\n override_env = ev.read(\"external_test\")\n with override_env:\n add(\"mpich@1.0 +debug\")\n override_env.concretize()\n override_env.write()\n\n # Reuse from the environment that is not included.\n # Specify the requirement for the debug variant. By default this would concretize to use\n # mpich@3.0 but with include concrete the mpich@1.0 +debug version from the\n # \"external_test\" environment will be used.\n _config = {\n \"concretizer\": {\"reuse\": {\"from\": [{\"type\": \"environment\", \"path\": \"external_test\"}]}},\n \"packages\": {\"mpich\": {\"require\": [\"+debug\"]}},\n }\n elif reuse_mode == \"from_environment_raise\":\n _config = {\n \"concretizer\": {\"reuse\": {\"from\": [{\"type\": \"environment\", \"path\": \"not-a-real-env\"}]}}\n }\n # Disable unification in these tests to avoid confusing reuse due to unification using an\n # include concrete spec vs reuse due to the reuse configuration\n _config[\"concretizer\"].update({\"unify\": False})\n\n combined_env.manifest.configuration.update(_config)\n combined_env.manifest.changed = True\n combined_env.write()\n\n return override_env\n\n\n@pytest.mark.parametrize(\n \"reuse_mode\",\n [\n \"true\",\n \"from_environment\",\n \"from_environment_test1\",\n \"from_environment_external_test\",\n \"from_environment_raise\",\n ],\n)\ndef test_env_include_concrete_reuse(reuse_mode):\n # The default mpi version is 3.x provided by mpich in the mock repo.\n # This test verifies that concretizing with an included concrete\n # environment with \"concretizer:reuse:true\" the included\n # concrete spec overrides the default with mpi@1.0.\n test1, _, combined = setup_combined_multiple_env()\n\n # Set the reuse mode for the environment\n override_env = configure_reuse(reuse_mode, combined)\n if override_env:\n # If there is an override environment (ie. testing reuse with\n # an external environment) update it here.\n test1 = override_env\n\n # Capture the test1 specs included by combined\n test1_specs_by_hash = test1.specs_by_hash\n\n try:\n # Add mpileaks to the combined environment\n with combined:\n add(\"mpileaks\")\n combined.concretize()\n comb_specs_by_hash = combined.specs_by_hash\n\n # create reference env with mpileaks that does not use reuse\n # This should concretize to the default version of mpich (3.0)\n env(\"create\", \"new\")\n ref_env = ev.read(\"new\")\n with ref_env:\n add(\"mpileaks\")\n ref_env.concretize()\n ref_specs_by_hash = ref_env.specs_by_hash\n\n # Ensure that the mpich used by the mpileaks is the mpich from the reused test environment\n comb_mpileaks_spec = [s for s in comb_specs_by_hash.values() if s.name == \"mpileaks\"]\n test1_mpich_spec = [s for s in test1_specs_by_hash.values() if s.name == \"mpich\"]\n assert len(comb_mpileaks_spec) == 1\n assert len(test1_mpich_spec) == 1\n assert comb_mpileaks_spec[0][\"mpich\"].dag_hash() == test1_mpich_spec[0].dag_hash()\n\n # None of the references specs (using mpich@3) reuse specs from test1.\n # This tests that the reuse is not happening coincidently\n assert not any([s in test1_specs_by_hash for s in ref_specs_by_hash])\n\n # Make sure the raise tests raises\n assert \"raise\" not in reuse_mode\n except ev.SpackEnvironmentError:\n assert \"raise\" in reuse_mode\n\n\n@pytest.mark.parametrize(\"unify\", [True, False, \"when_possible\"])\ndef test_env_include_concrete_env_reconcretized(mutable_config, unify):\n \"\"\"Double check to make sure that concrete_specs for the local specs is empty\n after reconcretizing.\n \"\"\"\n _, _, combined = setup_combined_multiple_env()\n\n with open(combined.lock_path, encoding=\"utf-8\") as f:\n lockfile_as_dict = combined._read_lockfile(f)\n\n assert not lockfile_as_dict[\"roots\"]\n assert not lockfile_as_dict[\"concrete_specs\"]\n\n with combined:\n mutable_config.set(\"concretizer:unify\", unify)\n combined.concretize()\n combined.write()\n\n with open(combined.lock_path, encoding=\"utf-8\") as f:\n lockfile_as_dict = combined._read_lockfile(f)\n\n assert not lockfile_as_dict[\"roots\"]\n assert not lockfile_as_dict[\"concrete_specs\"]\n\n\ndef test_concretize_include_concrete_env():\n \"\"\"Tests that if we update an included environment, and later we re-concretize the environment\n that includes it, we use the latest version of the concrete specs.\n \"\"\"\n test1, _, combined = setup_combined_multiple_env()\n\n # Update test1 environment\n with test1:\n add(\"mpileaks\")\n test1.concretize()\n test1.write()\n\n # Check the test1 environment includes mpileaks, while the combined environment does not\n assert Spec(\"mpileaks\") in {x.root for x in test1.concretized_roots}\n assert Spec(\"mpileaks\") not in {\n x.root for x in combined.included_concretized_roots[test1.path]\n }\n\n # If we update the combined environment, it will include mpileaks too\n combined.concretize()\n combined.write()\n assert Spec(\"mpileaks\") in {x.root for x in combined.included_concretized_roots[test1.path]}\n\n\ndef test_concretize_nested_include_concrete_envs():\n env(\"create\", \"test1\")\n test1 = ev.read(\"test1\")\n with test1:\n add(\"zlib\")\n test1.concretize()\n test1.write()\n\n env(\"create\", \"--include-concrete\", \"test1\", \"test2\")\n test2 = ev.read(\"test2\")\n with test2:\n add(\"libelf\")\n test2.concretize()\n test2.write()\n\n env(\"create\", \"--include-concrete\", \"test2\", \"test3\")\n test3 = ev.read(\"test3\")\n\n with open(test3.lock_path, encoding=\"utf-8\") as f:\n lockfile_as_dict = test3._read_lockfile(f)\n\n assert test2.path in lockfile_as_dict[ev.lockfile_include_key]\n assert (\n test1.path\n in lockfile_as_dict[ev.lockfile_include_key][test2.path][ev.lockfile_include_key]\n )\n\n assert Spec(\"zlib\") in {x.root for x in test3.included_concretized_roots[test1.path]}\n\n\ndef test_concretize_nested_included_concrete():\n \"\"\"Confirm that nested included environments use specs concretized at\n environment creation time and change with reconcretization.\"\"\"\n env(\"create\", \"test1\")\n test1 = ev.read(\"test1\")\n with test1:\n add(\"zlib\")\n test1.concretize()\n test1.write()\n\n # test2 should include test1 with zlib\n env(\"create\", \"--include-concrete\", \"test1\", \"test2\")\n test2 = ev.read(\"test2\")\n with test2:\n add(\"libelf\")\n test2.concretize()\n test2.write()\n\n assert Spec(\"zlib\") in {x.root for x in test2.included_concretized_roots[test1.path]}\n\n # Modify/re-concretize test1 to replace zlib with mpileaks\n with test1:\n remove(\"zlib\")\n add(\"mpileaks\")\n test1.concretize()\n test1.write()\n\n # test3 should include the latest concretization of test1\n env(\"create\", \"--include-concrete\", \"test1\", \"test3\")\n test3 = ev.read(\"test3\")\n with test3:\n add(\"callpath\")\n test3.concretize()\n test3.write()\n\n included_roots = test3.included_concretized_roots[test1.path]\n assert len(included_roots) == 1\n assert Spec(\"mpileaks\") in {x.root for x in included_roots}\n\n # The last concretization of test4's included environments should have test2\n # with the original concretized test1 spec and test3 with the re-concretized\n # test1 spec.\n env(\"create\", \"--include-concrete\", \"test2\", \"--include-concrete\", \"test3\", \"test4\")\n test4 = ev.read(\"test4\")\n\n def included_included_spec(path1, path2):\n included_path1 = test4.included_concrete_spec_data[path1]\n included_path2 = included_path1[ev.lockfile_include_key][path2]\n return included_path2[\"roots\"][0][\"spec\"]\n\n included_test2_test1 = included_included_spec(test2.path, test1.path)\n assert \"zlib\" in included_test2_test1\n\n included_test3_test1 = included_included_spec(test3.path, test1.path)\n assert \"mpileaks\" in included_test3_test1\n\n # test4's concretized specs should reflect the original concretization.\n concrete_specs = [s for s, _ in test4.concretized_specs()]\n expected = [Spec(s) for s in [\"libelf\", \"zlib\", \"mpileaks\", \"callpath\"]]\n assert all(s in concrete_specs for s in expected)\n\n # Re-concretize test2 to reflect the new concretization of included test1\n # to remove zlib and write it out so it can be picked up by test4.\n # Re-concretize test4 to reflect the re-concretization of included test2\n # and ensure that its included specs are up-to-date\n test2.concretize()\n test2.write()\n test4.concretize()\n\n concrete_specs = [s for s, _ in test4.concretized_specs()]\n assert Spec(\"zlib\") not in concrete_specs\n\n # Expecting mpileaks to appear only once\n expected = [Spec(s) for s in [\"libelf\", \"mpileaks\", \"callpath\"]]\n assert len(concrete_specs) == 3 and all(s in concrete_specs for s in expected)\n\n\ndef test_env_config_view_default(\n environment_from_manifest, mock_stage, mock_fetch, install_mockery\n):\n # This config doesn't mention whether a view is enabled\n environment_from_manifest(\n \"\"\"\nspack:\n specs:\n - mpileaks\n\"\"\"\n )\n\n with ev.read(\"test\"):\n install(\"--fake\")\n\n e = ev.read(\"test\")\n\n # Check that metadata folder for this spec exists\n assert os.path.isdir(os.path.join(e.default_view.view()._root, \".spack\", \"mpileaks\"))\n\n\ndef test_env_updates_view_install_package(\n tmp_path: pathlib.Path, mock_stage, mock_fetch, install_mockery\n):\n view_dir = tmp_path / \"view\"\n env(\"create\", \"--with-view=%s\" % view_dir, \"test\")\n with ev.read(\"test\"):\n install(\"--fake\", \"--add\", \"mpileaks\")\n\n assert os.path.exists(str(view_dir / \".spack/mpileaks\"))\n\n\ndef test_env_updates_view_add_concretize(\n tmp_path: pathlib.Path, mock_stage, mock_fetch, install_mockery\n):\n view_dir = tmp_path / \"view\"\n env(\"create\", \"--with-view=%s\" % view_dir, \"test\")\n install(\"--fake\", \"mpileaks\")\n with ev.read(\"test\"):\n add(\"mpileaks\")\n concretize()\n\n check_mpileaks_and_deps_in_view(view_dir)\n\n\ndef test_env_updates_view_uninstall(\n tmp_path: pathlib.Path, mock_stage, mock_fetch, install_mockery\n):\n view_dir = tmp_path / \"view\"\n env(\"create\", \"--with-view=%s\" % view_dir, \"test\")\n with ev.read(\"test\"):\n install(\"--fake\", \"--add\", \"mpileaks\")\n\n check_mpileaks_and_deps_in_view(view_dir)\n\n with ev.read(\"test\"):\n uninstall(\"-ay\")\n\n check_viewdir_removal(view_dir)\n\n\ndef test_env_updates_view_uninstall_referenced_elsewhere(\n tmp_path: pathlib.Path, mock_stage, mock_fetch, install_mockery\n):\n view_dir = tmp_path / \"view\"\n env(\"create\", \"--with-view=%s\" % view_dir, \"test\")\n install(\"--fake\", \"mpileaks\")\n with ev.read(\"test\"):\n add(\"mpileaks\")\n concretize()\n\n check_mpileaks_and_deps_in_view(view_dir)\n\n with ev.read(\"test\"):\n uninstall(\"-ay\")\n\n check_viewdir_removal(view_dir)\n\n\ndef test_env_updates_view_remove_concretize(\n tmp_path: pathlib.Path, mock_stage, mock_fetch, install_mockery\n):\n view_dir = tmp_path / \"view\"\n env(\"create\", \"--with-view=%s\" % view_dir, \"test\")\n install(\"--fake\", \"mpileaks\")\n with ev.read(\"test\"):\n add(\"mpileaks\")\n concretize()\n\n check_mpileaks_and_deps_in_view(view_dir)\n\n with ev.read(\"test\"):\n remove(\"mpileaks\")\n concretize()\n\n check_viewdir_removal(view_dir)\n\n\ndef test_env_updates_view_force_remove(\n tmp_path: pathlib.Path, mock_stage, mock_fetch, install_mockery\n):\n view_dir = tmp_path / \"view\"\n env(\"create\", \"--with-view=%s\" % view_dir, \"test\")\n with ev.read(\"test\"):\n install(\"--add\", \"--fake\", \"mpileaks\")\n\n check_mpileaks_and_deps_in_view(view_dir)\n\n with ev.read(\"test\"):\n remove(\"-f\", \"mpileaks\")\n\n check_viewdir_removal(view_dir)\n\n\ndef test_env_activate_view_fails(mock_stage, mock_fetch, install_mockery):\n \"\"\"Sanity check on env activate to make sure it requires shell support\"\"\"\n out = env(\"activate\", \"test\")\n assert \"To set up shell support\" in out\n\n\ndef test_stack_yaml_definitions(tmp_path: pathlib.Path):\n filename = str(tmp_path / \"spack.yaml\")\n with open(filename, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n definitions:\n - packages: [mpileaks, callpath]\n specs:\n - $packages\n\"\"\"\n )\n with fs.working_dir(str(tmp_path)):\n env(\"create\", \"test\", \"./spack.yaml\")\n test = ev.read(\"test\")\n\n assert Spec(\"mpileaks\") in test.user_specs\n assert Spec(\"callpath\") in test.user_specs\n\n\ndef test_stack_yaml_definitions_as_constraints(tmp_path: pathlib.Path):\n filename = str(tmp_path / \"spack.yaml\")\n with open(filename, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n definitions:\n - packages: [mpileaks, callpath]\n - mpis: [mpich, openmpi]\n specs:\n - matrix:\n - [$packages]\n - [$^mpis]\n\"\"\"\n )\n with fs.working_dir(str(tmp_path)):\n env(\"create\", \"test\", \"./spack.yaml\")\n test = ev.read(\"test\")\n\n assert Spec(\"mpileaks^mpich\") in test.user_specs\n assert Spec(\"callpath^mpich\") in test.user_specs\n assert Spec(\"mpileaks^openmpi\") in test.user_specs\n assert Spec(\"callpath^openmpi\") in test.user_specs\n\n\ndef test_stack_yaml_definitions_as_constraints_on_matrix(tmp_path: pathlib.Path):\n filename = str(tmp_path / \"spack.yaml\")\n with open(filename, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n definitions:\n - packages: [mpileaks, callpath]\n - mpis:\n - matrix:\n - [mpich]\n - ['@3.0.4', '@3.0.3']\n specs:\n - matrix:\n - [$packages]\n - [$^mpis]\n\"\"\"\n )\n with fs.working_dir(str(tmp_path)):\n env(\"create\", \"test\", \"./spack.yaml\")\n test = ev.read(\"test\")\n\n assert Spec(\"mpileaks^mpich@3.0.4\") in test.user_specs\n assert Spec(\"callpath^mpich@3.0.4\") in test.user_specs\n assert Spec(\"mpileaks^mpich@3.0.3\") in test.user_specs\n assert Spec(\"callpath^mpich@3.0.3\") in test.user_specs\n\n\n@pytest.mark.regression(\"12095\")\ndef test_stack_yaml_definitions_write_reference(tmp_path: pathlib.Path):\n filename = str(tmp_path / \"spack.yaml\")\n with open(filename, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n definitions:\n - packages: [mpileaks, callpath]\n - indirect: [$packages]\n specs:\n - $packages\n\"\"\"\n )\n with fs.working_dir(str(tmp_path)):\n env(\"create\", \"test\", \"./spack.yaml\")\n\n with ev.read(\"test\"):\n concretize()\n test = ev.read(\"test\")\n\n assert Spec(\"mpileaks\") in test.user_specs\n assert Spec(\"callpath\") in test.user_specs\n\n\ndef test_stack_yaml_add_to_list(tmp_path: pathlib.Path):\n filename = str(tmp_path / \"spack.yaml\")\n with open(filename, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n definitions:\n - packages: [mpileaks, callpath]\n specs:\n - $packages\n\"\"\"\n )\n with fs.working_dir(str(tmp_path)):\n env(\"create\", \"test\", \"./spack.yaml\")\n with ev.read(\"test\"):\n add(\"-l\", \"packages\", \"libelf\")\n\n test = ev.read(\"test\")\n\n assert Spec(\"libelf\") in test.user_specs\n assert Spec(\"mpileaks\") in test.user_specs\n assert Spec(\"callpath\") in test.user_specs\n\n\ndef test_stack_yaml_remove_from_list(tmp_path: pathlib.Path):\n filename = str(tmp_path / \"spack.yaml\")\n with open(filename, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n definitions:\n - packages: [mpileaks, callpath]\n specs:\n - $packages\n\"\"\"\n )\n with fs.working_dir(str(tmp_path)):\n env(\"create\", \"test\", \"./spack.yaml\")\n with ev.read(\"test\"):\n remove(\"-l\", \"packages\", \"mpileaks\")\n\n test = ev.read(\"test\")\n\n assert Spec(\"mpileaks\") not in test.user_specs\n assert Spec(\"callpath\") in test.user_specs\n\n\ndef test_stack_yaml_remove_from_list_force(tmp_path: pathlib.Path):\n spack_yaml = tmp_path / ev.manifest_name\n spack_yaml.write_text(\n \"\"\"\\\nspack:\n definitions:\n - packages: [mpileaks, callpath]\n specs:\n - matrix:\n - [$packages]\n - [^mpich, ^zmpi]\n\"\"\"\n )\n\n env(\"create\", \"test\", str(spack_yaml))\n with ev.read(\"test\"):\n concretize()\n remove(\"-f\", \"-l\", \"packages\", \"mpileaks\")\n find_output = find(\"-c\")\n\n assert \"mpileaks\" not in find_output\n\n test = ev.read(\"test\")\n assert len(test.user_specs) == 2\n assert Spec(\"callpath ^zmpi\") in test.user_specs\n assert Spec(\"callpath ^mpich\") in test.user_specs\n\n\ndef test_stack_yaml_remove_from_matrix_no_effect(tmp_path: pathlib.Path):\n filename = str(tmp_path / \"spack.yaml\")\n with open(filename, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n definitions:\n - packages:\n - matrix:\n - [mpileaks, callpath]\n - [target=default_target]\n specs:\n - $packages\n\"\"\"\n )\n with fs.working_dir(str(tmp_path)):\n env(\"create\", \"test\", \"./spack.yaml\")\n with ev.read(\"test\") as e:\n before = e.user_specs.specs\n remove(\"-l\", \"packages\", \"mpileaks\")\n after = e.user_specs.specs\n\n assert before == after\n\n\ndef test_stack_yaml_force_remove_from_matrix(tmp_path: pathlib.Path):\n filename = str(tmp_path / \"spack.yaml\")\n with open(filename, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n definitions:\n - packages:\n - matrix:\n - [mpileaks, callpath]\n - [target=default_target]\n specs:\n - $packages\n\"\"\"\n )\n with fs.working_dir(str(tmp_path)):\n env(\"create\", \"test\", \"./spack.yaml\")\n with ev.read(\"test\") as e:\n e.concretize()\n\n before_user = e.user_specs.specs\n concretized_roots_before = e.concretized_roots\n\n remove(\"-f\", \"-l\", \"packages\", \"mpileaks\")\n\n after_user = e.user_specs.specs\n concretized_roots_after = e.concretized_roots\n\n assert before_user == after_user\n\n mpileaks_spec = Spec(\"mpileaks target=default_target\")\n assert mpileaks_spec in {x.root for x in concretized_roots_before}\n assert mpileaks_spec not in {x.root for x in concretized_roots_after}\n\n\ndef test_stack_definition_extension(tmp_path: pathlib.Path):\n filename = str(tmp_path / \"spack.yaml\")\n with open(filename, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n definitions:\n - packages: [libelf, mpileaks]\n - packages: [callpath]\n specs:\n - $packages\n\"\"\"\n )\n with fs.working_dir(str(tmp_path)):\n env(\"create\", \"test\", \"./spack.yaml\")\n\n test = ev.read(\"test\")\n\n assert Spec(\"libelf\") in test.user_specs\n assert Spec(\"mpileaks\") in test.user_specs\n assert Spec(\"callpath\") in test.user_specs\n\n\ndef test_stack_definition_conditional_false(tmp_path: pathlib.Path):\n filename = str(tmp_path / \"spack.yaml\")\n with open(filename, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n definitions:\n - packages: [libelf, mpileaks]\n - packages: [callpath]\n when: 'False'\n specs:\n - $packages\n\"\"\"\n )\n with fs.working_dir(str(tmp_path)):\n env(\"create\", \"test\", \"./spack.yaml\")\n\n test = ev.read(\"test\")\n\n assert Spec(\"libelf\") in test.user_specs\n assert Spec(\"mpileaks\") in test.user_specs\n assert Spec(\"callpath\") not in test.user_specs\n\n\ndef test_stack_definition_conditional_true(tmp_path: pathlib.Path):\n filename = str(tmp_path / \"spack.yaml\")\n with open(filename, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n definitions:\n - packages: [libelf, mpileaks]\n - packages: [callpath]\n when: 'True'\n specs:\n - $packages\n\"\"\"\n )\n with fs.working_dir(str(tmp_path)):\n env(\"create\", \"test\", \"./spack.yaml\")\n\n test = ev.read(\"test\")\n\n assert Spec(\"libelf\") in test.user_specs\n assert Spec(\"mpileaks\") in test.user_specs\n assert Spec(\"callpath\") in test.user_specs\n\n\ndef test_stack_definition_conditional_with_variable(tmp_path: pathlib.Path):\n filename = str(tmp_path / \"spack.yaml\")\n with open(filename, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n definitions:\n - packages: [libelf, mpileaks]\n - packages: [callpath]\n when: platform == 'test'\n specs:\n - $packages\n\"\"\"\n )\n with fs.working_dir(str(tmp_path)):\n env(\"create\", \"test\", \"./spack.yaml\")\n\n test = ev.read(\"test\")\n\n assert Spec(\"libelf\") in test.user_specs\n assert Spec(\"mpileaks\") in test.user_specs\n assert Spec(\"callpath\") in test.user_specs\n\n\ndef test_stack_definition_conditional_with_satisfaction(tmp_path: pathlib.Path):\n filename = str(tmp_path / \"spack.yaml\")\n with open(filename, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n definitions:\n - packages: [libelf, mpileaks]\n when: arch.satisfies('platform=foo') # will be \"test\" when testing\n - packages: [callpath]\n when: arch.satisfies('platform=test')\n specs:\n - $packages\n\"\"\"\n )\n with fs.working_dir(str(tmp_path)):\n env(\"create\", \"test\", \"./spack.yaml\")\n\n test = ev.read(\"test\")\n\n assert Spec(\"libelf\") not in test.user_specs\n assert Spec(\"mpileaks\") not in test.user_specs\n assert Spec(\"callpath\") in test.user_specs\n\n\ndef test_stack_definition_complex_conditional(tmp_path: pathlib.Path):\n filename = str(tmp_path / \"spack.yaml\")\n with open(filename, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n definitions:\n - packages: [libelf, mpileaks]\n - packages: [callpath]\n when: re.search(r'foo', hostname) and env['test'] == 'THISSHOULDBEFALSE'\n specs:\n - $packages\n\"\"\"\n )\n with fs.working_dir(str(tmp_path)):\n env(\"create\", \"test\", \"./spack.yaml\")\n\n test = ev.read(\"test\")\n\n assert Spec(\"libelf\") in test.user_specs\n assert Spec(\"mpileaks\") in test.user_specs\n assert Spec(\"callpath\") not in test.user_specs\n\n\ndef test_stack_definition_conditional_invalid_variable(tmp_path: pathlib.Path):\n filename = str(tmp_path / \"spack.yaml\")\n with open(filename, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n definitions:\n - packages: [libelf, mpileaks]\n - packages: [callpath]\n when: bad_variable == 'test'\n specs:\n - $packages\n\"\"\"\n )\n with fs.working_dir(str(tmp_path)):\n with pytest.raises(NameError):\n env(\"create\", \"test\", \"./spack.yaml\")\n\n\ndef test_stack_definition_conditional_add_write(tmp_path: pathlib.Path):\n filename = str(tmp_path / \"spack.yaml\")\n with open(filename, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n definitions:\n - packages: [libelf, mpileaks]\n - packages: [callpath]\n when: platform == 'test'\n specs:\n - $packages\n\"\"\"\n )\n with fs.working_dir(str(tmp_path)):\n env(\"create\", \"test\", \"./spack.yaml\")\n with ev.read(\"test\"):\n add(\"-l\", \"packages\", \"zmpi\")\n\n test = ev.read(\"test\")\n\n packages_lists = list(\n filter(lambda x: \"packages\" in x, test.manifest[\"spack\"][\"definitions\"])\n )\n\n assert len(packages_lists) == 2\n assert \"callpath\" not in packages_lists[0][\"packages\"]\n assert \"callpath\" in packages_lists[1][\"packages\"]\n assert \"zmpi\" in packages_lists[0][\"packages\"]\n assert \"zmpi\" not in packages_lists[1][\"packages\"]\n\n\ndef test_stack_combinatorial_view(\n installed_environment, template_combinatorial_env, tmp_path: pathlib.Path\n):\n \"\"\"Tests creating a default view for a combinatorial stack.\"\"\"\n view_dir = tmp_path / \"view\"\n with installed_environment(template_combinatorial_env.format(view_config=\"\")) as test:\n for spec in traverse_nodes(test.concrete_roots(), deptype=(\"link\", \"run\")):\n if spec.name == \"gcc-runtime\":\n continue\n current_dir = view_dir / f\"{spec.architecture.target}\" / f\"{spec.name}-{spec.version}\"\n assert current_dir.exists() and current_dir.is_dir()\n\n\ndef test_stack_view_select(\n installed_environment, template_combinatorial_env, tmp_path: pathlib.Path\n):\n view_dir = tmp_path / \"view\"\n content = template_combinatorial_env.format(view_config=\"select: ['target=x86_64']\\n\")\n with installed_environment(content) as test:\n for spec in traverse_nodes(test.concrete_roots(), deptype=(\"link\", \"run\")):\n if spec.name == \"gcc-runtime\":\n continue\n current_dir = view_dir / f\"{spec.architecture.target}\" / f\"{spec.name}-{spec.version}\"\n assert current_dir.exists() is spec.satisfies(\"target=x86_64\")\n\n\ndef test_stack_view_exclude(\n installed_environment, template_combinatorial_env, tmp_path: pathlib.Path\n):\n view_dir = tmp_path / \"view\"\n content = template_combinatorial_env.format(view_config=\"exclude: [callpath]\\n\")\n with installed_environment(content) as test:\n for spec in traverse_nodes(test.concrete_roots(), deptype=(\"link\", \"run\")):\n if spec.name == \"gcc-runtime\":\n continue\n current_dir = view_dir / f\"{spec.architecture.target}\" / f\"{spec.name}-{spec.version}\"\n assert current_dir.exists() is not spec.satisfies(\"callpath\")\n\n\ndef test_stack_view_select_and_exclude(\n installed_environment, template_combinatorial_env, tmp_path: pathlib.Path\n):\n view_dir = tmp_path / \"view\"\n content = template_combinatorial_env.format(\n view_config=\"\"\"select: ['target=x86_64']\n exclude: [callpath]\n\"\"\"\n )\n with installed_environment(content) as test:\n for spec in traverse_nodes(test.concrete_roots(), deptype=(\"link\", \"run\")):\n if spec.name == \"gcc-runtime\":\n continue\n current_dir = view_dir / f\"{spec.architecture.target}\" / f\"{spec.name}-{spec.version}\"\n assert current_dir.exists() is (\n spec.satisfies(\"target=x86_64\") and not spec.satisfies(\"callpath\")\n )\n\n\ndef test_view_link_roots(\n installed_environment, template_combinatorial_env, tmp_path: pathlib.Path\n):\n view_dir = tmp_path / \"view\"\n content = template_combinatorial_env.format(\n view_config=\"\"\"select: ['target=x86_64']\n exclude: [callpath]\n link: 'roots'\n \"\"\"\n )\n with installed_environment(content) as test:\n for spec in traverse_nodes(test.concrete_roots(), deptype=(\"link\", \"run\")):\n if spec.name == \"gcc-runtime\":\n continue\n current_dir = view_dir / f\"{spec.architecture.target}\" / f\"{spec.name}-{spec.version}\"\n expected_exists = spec in test.roots() and (\n spec.satisfies(\"target=x86_64\") and not spec.satisfies(\"callpath\")\n )\n assert current_dir.exists() == expected_exists\n\n\ndef test_view_link_run(\n tmp_path: pathlib.Path, mock_fetch, mock_packages, mock_archive, install_mockery\n):\n yaml = str(tmp_path / \"spack.yaml\")\n viewdir = str(tmp_path / \"view\")\n envdir = str(tmp_path)\n with open(yaml, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\nspack:\n specs:\n - dttop\n\n view:\n combinatorial:\n root: %s\n link: run\n projections:\n all: '{name}'\"\"\"\n % viewdir\n )\n\n with ev.Environment(envdir):\n install(\"--fake\")\n\n # make sure transitive run type deps are in the view\n for pkg in (\"dtrun1\", \"dtrun3\"):\n assert os.path.exists(os.path.join(viewdir, pkg))\n\n # and non-run-type deps are not.\n for pkg in (\n \"dtlink1\",\n \"dtlink2\",\n \"dtlink3\",\n \"dtlink4\",\n \"dtlink5dtbuild1\",\n \"dtbuild2\",\n \"dtbuild3\",\n ):\n assert not os.path.exists(os.path.join(viewdir, pkg))\n\n\n@pytest.mark.parametrize(\"link_type\", [\"hardlink\", \"copy\", \"symlink\"])\ndef test_view_link_type(link_type, installed_environment, tmp_path: pathlib.Path):\n view_dir = tmp_path / \"view\"\n with installed_environment(\n f\"\"\"\\\nspack:\n specs:\n - mpileaks\n view:\n default:\n root: {view_dir}\n link_type: {link_type}\"\"\"\n ) as test:\n for spec in test.roots():\n # Assertions are based on the behavior of the \"--fake\" install\n bin_file = pathlib.Path(test.default_view.view()._root) / \"bin\" / spec.name\n assert bin_file.exists()\n assert bin_file.is_symlink() == (link_type == \"symlink\")\n\n\ndef test_view_link_all(installed_environment, template_combinatorial_env, tmp_path: pathlib.Path):\n view_dir = tmp_path / \"view\"\n content = template_combinatorial_env.format(\n view_config=\"\"\"select: ['target=x86_64']\n exclude: [callpath]\n link: 'all'\n \"\"\"\n )\n\n with installed_environment(content) as test:\n for spec in traverse_nodes(test.concrete_roots(), deptype=(\"link\", \"run\")):\n if spec.name == \"gcc-runtime\":\n continue\n current_dir = view_dir / f\"{spec.architecture.target}\" / f\"{spec.name}-{spec.version}\"\n assert current_dir.exists() == (\n spec.satisfies(\"target=x86_64\") and not spec.satisfies(\"callpath\")\n )\n\n\ndef test_stack_view_activate_from_default(\n installed_environment, template_combinatorial_env, tmp_path: pathlib.Path\n):\n view_dir = tmp_path / \"view\"\n content = template_combinatorial_env.format(view_config=\"select: ['target=x86_64']\")\n # Replace the name of the view\n content = content.replace(\"combinatorial:\", \"default:\")\n with installed_environment(content):\n shell = env(\"activate\", \"--sh\", \"test\")\n assert \"PATH\" in shell, shell\n assert str(view_dir / \"bin\") in shell\n assert \"FOOBAR=mpileaks\" in shell\n\n\ndef test_envvar_set_in_activate(tmp_path: pathlib.Path, mock_packages, install_mockery):\n spack_yaml = tmp_path / \"spack.yaml\"\n env_vars_yaml = tmp_path / \"env_vars.yaml\"\n\n env_vars_yaml.write_text(\n \"\"\"\nenv_vars:\n set:\n CONFIG_ENVAR_SET_IN_ENV_LOAD: \"True\"\n\"\"\"\n )\n\n spack_yaml.write_text(\n \"\"\"\nspack:\n include:\n - env_vars.yaml\n specs:\n - cmake%gcc\n env_vars:\n set:\n SPACK_ENVAR_SET_IN_ENV_LOAD: \"True\"\n\"\"\"\n )\n\n env(\"create\", \"test\", str(spack_yaml))\n with ev.read(\"test\"):\n install(\"--fake\")\n\n test_env = ev.read(\"test\")\n output = env(\"activate\", \"--sh\", \"test\")\n\n assert \"SPACK_ENVAR_SET_IN_ENV_LOAD=True\" in output\n assert \"CONFIG_ENVAR_SET_IN_ENV_LOAD=True\" in output\n\n with test_env:\n with spack.util.environment.set_env(\n SPACK_ENVAR_SET_IN_ENV_LOAD=\"True\", CONFIG_ENVAR_SET_IN_ENV_LOAD=\"True\"\n ):\n output = env(\"deactivate\", \"--sh\")\n assert \"unset SPACK_ENVAR_SET_IN_ENV_LOAD\" in output\n assert \"unset CONFIG_ENVAR_SET_IN_ENV_LOAD\" in output\n\n\ndef test_stack_view_no_activate_without_default(\n installed_environment, template_combinatorial_env, tmp_path: pathlib.Path\n):\n view_dir = tmp_path / \"view\"\n content = template_combinatorial_env.format(view_config=\"select: ['target=x86_64']\")\n with installed_environment(content):\n shell = env(\"activate\", \"--sh\", \"test\")\n assert \"PATH\" not in shell\n assert str(view_dir) not in shell\n\n\n@pytest.mark.parametrize(\"include_views\", [True, False, \"split\"])\ndef test_stack_view_multiple_views(installed_environment, tmp_path: pathlib.Path, include_views):\n \"\"\"Test multiple views as both included views (True), as both environment\n views (False), or as one included and the other in the environment.\n \"\"\"\n # Write the view configuration and or manifest file\n view_filename = tmp_path / \"view.yaml\"\n base_content = \"\"\"\\\n definitions:\n - packages: [mpileaks, cmake]\n - targets: ['target=x86_64', 'target=core2']\n specs:\n - matrix:\n - [$packages]\n - [$targets]\n\"\"\"\n\n include_content = f\" include:\\n - {view_filename}\\n\"\n view_line = \" view:\\n\"\n\n comb_dir = tmp_path / \"combinatorial-view\"\n comb_view = \"\"\"\\\n{0}combinatorial:\n{0} root: {1}\n{0} exclude: [target=core2]\n{0} projections:\n\"\"\"\n\n projection = \" 'all': '{architecture.target}/{name}-{version}'\"\n\n default_dir = tmp_path / \"default-view\"\n default_view = \"\"\"\\\n{0}default:\n{0} root: {1}\n{0} select: ['target=x86_64']\n\"\"\"\n\n content = \"spack:\\n\"\n indent = \" \"\n if include_views is True:\n # Include both the gcc and combinatorial views\n view = \"view:\\n\" + default_view.format(indent, str(default_dir))\n view += comb_view.format(indent, str(comb_dir)) + indent + projection\n view_filename.write_text(view)\n content += include_content + base_content\n elif include_views == \"split\":\n # Include the gcc view and inline the combinatorial view\n view = \"view:\\n\" + default_view.format(indent, str(default_dir))\n view_filename.write_text(view)\n content += include_content + base_content + view_line\n indent += \" \"\n content += comb_view.format(indent, str(comb_dir)) + indent + projection\n else:\n # Inline both the gcc and combinatorial views in the environment.\n indent += \" \"\n content += base_content + view_line\n content += default_view.format(indent, str(default_dir))\n content += comb_view.format(indent, str(comb_dir)) + indent + projection\n\n with installed_environment(content) as e:\n assert os.path.exists(str(default_dir / \"bin\"))\n for spec in traverse_nodes(e.concrete_roots(), deptype=(\"link\", \"run\")):\n if spec.name == \"gcc-runtime\":\n continue\n current_dir = comb_dir / f\"{spec.architecture.target}\" / f\"{spec.name}-{spec.version}\"\n assert current_dir.exists() is not spec.satisfies(\"target=core2\")\n\n\ndef test_env_activate_sh_prints_shell_output(mock_stage, mock_fetch, install_mockery):\n \"\"\"Check the shell commands output by ``spack env activate --sh``.\n\n This is a cursory check; ``share/spack/qa/setup-env-test.sh`` checks\n for correctness.\n \"\"\"\n env(\"create\", \"test\")\n\n out = env(\"activate\", \"--sh\", \"test\")\n assert \"export SPACK_ENV=\" in out\n assert \"export PS1=\" not in out\n assert \"alias despacktivate=\" in out\n\n out = env(\"activate\", \"--sh\", \"--prompt\", \"test\")\n assert \"export SPACK_ENV=\" in out\n assert \"export PS1=\" in out\n assert \"alias despacktivate=\" in out\n\n\ndef test_env_activate_csh_prints_shell_output(mock_stage, mock_fetch, install_mockery):\n \"\"\"Check the shell commands output by ``spack env activate --csh``.\"\"\"\n env(\"create\", \"test\")\n\n out = env(\"activate\", \"--csh\", \"test\")\n assert \"setenv SPACK_ENV\" in out\n assert \"setenv set prompt\" not in out\n assert \"alias despacktivate\" in out\n\n out = env(\"activate\", \"--csh\", \"--prompt\", \"test\")\n assert \"setenv SPACK_ENV\" in out\n assert \"set prompt=\" in out\n assert \"alias despacktivate\" in out\n\n\n@pytest.mark.regression(\"12719\")\ndef test_env_activate_default_view_root_unconditional(mutable_mock_env_path):\n \"\"\"Check that the root of the default view in the environment is added\n to the shell unconditionally.\"\"\"\n env(\"create\", \"test\")\n\n with ev.read(\"test\") as e:\n viewdir = e.default_view.root\n\n out = env(\"activate\", \"--sh\", \"test\")\n viewdir_bin = os.path.join(viewdir, \"bin\")\n\n assert (\n \"export PATH={0}\".format(viewdir_bin) in out\n or \"export PATH='{0}\".format(viewdir_bin) in out\n or 'export PATH=\"{0}'.format(viewdir_bin) in out\n )\n\n\ndef test_env_activate_custom_view(tmp_path: pathlib.Path, mock_packages):\n \"\"\"Check that an environment can be activated with a non-default view.\"\"\"\n env_template = tmp_path / \"spack.yaml\"\n default_dir = tmp_path / \"defaultdir\"\n nondefaultdir = tmp_path / \"nondefaultdir\"\n with open(env_template, \"w\", encoding=\"utf-8\") as f:\n f.write(\n f\"\"\"\\\nspack:\n specs: [a]\n view:\n default:\n root: {default_dir}\n nondefault:\n root: {nondefaultdir}\"\"\"\n )\n env(\"create\", \"test\", str(env_template))\n shell = env(\"activate\", \"--sh\", \"--with-view\", \"nondefault\", \"test\")\n assert os.path.join(nondefaultdir, \"bin\") in shell\n\n\ndef test_concretize_user_specs_together(mutable_config):\n with ev.create(\"coconcretization\") as e:\n mutable_config.set(\"concretizer:unify\", True)\n\n # Concretize a first time using 'mpich' as the MPI provider\n e.add(\"mpileaks\")\n e.add(\"mpich\")\n e.concretize()\n\n assert all(\"mpich\" in spec for _, spec in e.concretized_specs())\n assert all(\"mpich2\" not in spec for _, spec in e.concretized_specs())\n\n # Concretize a second time using 'mpich2' as the MPI provider\n e.remove(\"mpich\")\n e.add(\"mpich2\")\n\n exc_cls = spack.error.UnsatisfiableSpecError\n\n # Concretizing without invalidating the concrete spec for mpileaks fails\n with pytest.raises(exc_cls):\n e.concretize()\n e.concretize(force=True)\n\n assert all(\"mpich2\" in spec for _, spec in e.concretized_specs())\n assert all(\"mpich\" not in spec for _, spec in e.concretized_specs())\n\n # Concretize again without changing anything, check everything\n # stays the same\n e.concretize()\n\n assert all(\"mpich2\" in spec for _, spec in e.concretized_specs())\n assert all(\"mpich\" not in spec for _, spec in e.concretized_specs())\n\n\ndef test_duplicate_packages_raise_when_concretizing_together(mutable_config):\n with ev.create(\"coconcretization\") as e:\n mutable_config.set(\"concretizer:unify\", True)\n e.add(\"mpileaks+opt\")\n e.add(\"mpileaks~opt\")\n e.add(\"mpich\")\n\n exc_cls = spack.error.UnsatisfiableSpecError\n match = r\"You could consider setting `concretizer:unify`\"\n\n with pytest.raises(exc_cls, match=match):\n e.concretize()\n\n\ndef test_env_write_only_non_default():\n env(\"create\", \"test\")\n\n e = ev.read(\"test\")\n with open(e.manifest_path, \"r\", encoding=\"utf-8\") as f:\n yaml = f.read()\n\n assert yaml == ev.default_manifest_yaml()\n\n\n@pytest.mark.regression(\"20526\")\ndef test_env_write_only_non_default_nested(tmp_path: pathlib.Path):\n # setup an environment file\n # the environment includes configuration because nested configs proved the\n # most difficult to avoid writing.\n filename = \"spack.yaml\"\n filepath = str(tmp_path / filename)\n contents = \"\"\"\\\nspack:\n specs:\n - matrix:\n - [mpileaks]\n packages:\n all:\n compiler: [gcc]\n view: true\n\"\"\"\n\n # create environment with some structure\n with open(filepath, \"w\", encoding=\"utf-8\") as f:\n f.write(contents)\n env(\"create\", \"test\", filepath)\n\n # concretize\n with ev.read(\"test\") as e:\n concretize()\n e.write()\n\n with open(e.manifest_path, \"r\", encoding=\"utf-8\") as f:\n manifest = f.read()\n\n assert manifest == contents\n\n\n@pytest.mark.regression(\"18147\")\ndef test_can_update_attributes_with_override(tmp_path: pathlib.Path):\n spack_yaml = \"\"\"\nspack:\n mirrors::\n test: /foo/bar\n packages:\n cmake:\n paths:\n cmake@3.18.1: /usr\n specs:\n - hdf5\n\"\"\"\n abspath = tmp_path / \"spack.yaml\"\n abspath.write_text(spack_yaml)\n\n # Check that an update does not raise\n env(\"update\", \"-y\", str(tmp_path))\n\n\n@pytest.mark.regression(\"18338\")\ndef test_newline_in_commented_sequence_is_not_an_issue(tmp_path: pathlib.Path):\n spack_yaml = \"\"\"\nspack:\n specs:\n - dyninst\n packages:\n libelf:\n externals:\n - spec: libelf@0.8.13\n modules:\n - libelf/3.18.1\n\n concretizer:\n unify: false\n\"\"\"\n abspath = tmp_path / \"spack.yaml\"\n abspath.write_text(spack_yaml)\n\n def extract_dag_hash(environment):\n _, dyninst = next(iter(environment.specs_by_hash.items()))\n return dyninst[\"libelf\"].dag_hash()\n\n # Concretize a first time and create a lockfile\n with ev.Environment(str(tmp_path)) as e:\n concretize()\n libelf_first_hash = extract_dag_hash(e)\n\n # Check that a second run won't error\n with ev.Environment(str(tmp_path)) as e:\n concretize()\n libelf_second_hash = extract_dag_hash(e)\n\n assert libelf_first_hash == libelf_second_hash\n\n\n@pytest.mark.regression(\"18441\")\ndef test_lockfile_not_deleted_on_write_error(tmp_path: pathlib.Path, monkeypatch):\n raw_yaml = \"\"\"\nspack:\n specs:\n - dyninst\n packages:\n libelf:\n externals:\n - spec: libelf@0.8.13\n prefix: /usr\n\"\"\"\n spack_yaml = tmp_path / \"spack.yaml\"\n spack_yaml.write_text(raw_yaml)\n spack_lock = tmp_path / \"spack.lock\"\n\n # Concretize a first time and create a lockfile\n with ev.Environment(str(tmp_path)):\n concretize()\n assert os.path.exists(str(spack_lock))\n\n # If I run concretize again and there's an error during write,\n # the spack.lock file shouldn't disappear from disk\n def _write_helper_raise(self):\n raise RuntimeError(\"some error\")\n\n monkeypatch.setattr(ev.environment.EnvironmentManifestFile, \"flush\", _write_helper_raise)\n with ev.Environment(str(tmp_path)) as e:\n e.concretize(force=True)\n with pytest.raises(RuntimeError):\n e.clear()\n e.write()\n assert os.path.exists(str(spack_lock))\n\n\ndef _setup_develop_packages(tmp_path: pathlib.Path):\n \"\"\"Sets up a structure ./init_env/spack.yaml, ./build_folder, ./dest_env\n where spack.yaml has a relative develop path to build_folder\"\"\"\n init_env = tmp_path / \"init_env\"\n build_folder = tmp_path / \"build_folder\"\n dest_env = tmp_path / \"dest_env\"\n\n init_env.mkdir(parents=True, exist_ok=True)\n build_folder.mkdir(parents=True, exist_ok=True)\n dest_env.mkdir(parents=True, exist_ok=True)\n\n raw_yaml = \"\"\"\nspack:\n specs: ['mypkg1', 'mypkg2']\n develop:\n mypkg1:\n path: ../build_folder\n spec: mypkg@main\n mypkg2:\n path: /some/other/path\n spec: mypkg@main\n\"\"\"\n spack_yaml = init_env / \"spack.yaml\"\n spack_yaml.write_text(raw_yaml)\n\n return init_env, build_folder, dest_env, spack_yaml\n\n\ndef test_rewrite_rel_dev_path_new_dir(tmp_path: pathlib.Path):\n \"\"\"Relative develop paths should be rewritten for new environments in\n a different directory from the original manifest file\"\"\"\n _, build_folder, dest_env, spack_yaml = _setup_develop_packages(tmp_path)\n\n env(\"create\", \"-d\", str(dest_env), str(spack_yaml))\n with ev.Environment(str(dest_env)) as e:\n assert e.dev_specs[\"mypkg1\"][\"path\"] == str(build_folder)\n assert e.dev_specs[\"mypkg2\"][\"path\"] == sep + os.path.join(\"some\", \"other\", \"path\")\n\n\ndef test_rewrite_rel_dev_path_named_env(tmp_path: pathlib.Path):\n \"\"\"Relative develop paths should by default be rewritten for new named\n environment\"\"\"\n _, build_folder, _, spack_yaml = _setup_develop_packages(tmp_path)\n env(\"create\", \"named_env\", str(spack_yaml))\n with ev.read(\"named_env\") as e:\n assert e.dev_specs[\"mypkg1\"][\"path\"] == str(build_folder)\n assert e.dev_specs[\"mypkg2\"][\"path\"] == sep + os.path.join(\"some\", \"other\", \"path\")\n\n\ndef test_does_not_rewrite_rel_dev_path_when_keep_relative_is_set(tmp_path: pathlib.Path):\n \"\"\"Relative develop paths should not be rewritten when --keep-relative is\n passed to create\"\"\"\n _, _, _, spack_yaml = _setup_develop_packages(tmp_path)\n env(\"create\", \"--keep-relative\", \"named_env\", str(spack_yaml))\n with ev.read(\"named_env\") as e:\n assert e.dev_specs[\"mypkg1\"][\"path\"] == \"../build_folder\"\n assert e.dev_specs[\"mypkg2\"][\"path\"] == \"/some/other/path\"\n\n\n@pytest.mark.regression(\"23440\")\ndef test_custom_version_concretize_together(mutable_config):\n # Custom versions should be permitted in specs when\n # concretizing together\n with ev.create(\"custom_version\") as e:\n mutable_config.set(\"concretizer:unify\", True)\n # Concretize a first time using 'mpich' as the MPI provider\n e.add(\"hdf5@=myversion\")\n e.add(\"mpich\")\n e.concretize()\n assert any(spec.satisfies(\"hdf5@myversion\") for _, spec in e.concretized_specs())\n\n\ndef test_modules_relative_to_views(environment_from_manifest, install_mockery, mock_fetch):\n environment_from_manifest(\n \"\"\"\nspack:\n specs:\n - trivial-install-test-package\n modules:\n default:\n enable:: [tcl]\n use_view: true\n roots:\n tcl: modules\n\"\"\"\n )\n\n with ev.read(\"test\") as e:\n install(\"--fake\")\n user_spec_hash = e.concretized_roots[0].hash\n spec = e.specs_by_hash[user_spec_hash]\n view_prefix = e.default_view.get_projection_for_spec(spec)\n modules_glob = \"%s/modules/**/*/*\" % e.path\n modules = glob.glob(modules_glob)\n assert len(modules) == 1\n module = modules[0]\n\n with open(module, \"r\", encoding=\"utf-8\") as f:\n contents = f.read()\n\n assert view_prefix in contents\n assert spec.prefix not in contents\n\n\ndef test_modules_exist_after_env_install(installed_environment, monkeypatch):\n # Some caching issue\n monkeypatch.setattr(spack.modules.tcl, \"configuration_registry\", {})\n with installed_environment(\n \"\"\"\nspack:\n specs:\n - mpileaks\n modules:\n default:\n enable:: [tcl]\n use_view: true\n roots:\n tcl: uses_view\n full:\n enable:: [tcl]\n roots:\n tcl: without_view\n\"\"\"\n ) as e:\n specs = e.all_specs()\n for module_set in (\"uses_view\", \"without_view\"):\n modules = glob.glob(f\"{e.path}/{module_set}/**/*/*\")\n assert len(modules) == len(specs), \"Not all modules were generated\"\n for spec in specs:\n if spec.external:\n continue\n\n module = next((m for m in modules if os.path.dirname(m).endswith(spec.name)), None)\n assert module, f\"Module for {spec} not found\"\n\n # Now verify that modules have paths pointing into the view instead of the package\n # prefix if and only if they set use_view to true.\n with open(module, \"r\", encoding=\"utf-8\") as f:\n contents = f.read()\n\n if module_set == \"uses_view\":\n assert e.default_view.get_projection_for_spec(spec) in contents\n assert spec.prefix not in contents\n else:\n assert e.default_view.get_projection_for_spec(spec) not in contents\n assert spec.prefix in contents\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_install_develop_keep_stage(\n environment_from_manifest, install_mockery, mock_fetch, monkeypatch, tmp_path: pathlib.Path\n):\n \"\"\"Develop a dependency of a package and make sure that the associated\n stage for the package is retained after a successful install.\n \"\"\"\n environment_from_manifest(\n \"\"\"\nspack:\n specs:\n - mpileaks\n\"\"\"\n )\n\n monkeypatch.setattr(spack.stage.DevelopStage, \"destroy\", _always_fail)\n\n with ev.read(\"test\") as e:\n libelf_dev_path = tmp_path / \"libelf-test-dev-path\"\n libelf_dev_path.mkdir(parents=True)\n develop(f\"--path={libelf_dev_path}\", \"libelf@0.8.13\")\n concretize()\n (libelf_spec,) = e.all_matching_specs(\"libelf\")\n (mpileaks_spec,) = e.all_matching_specs(\"mpileaks\")\n assert not os.path.exists(libelf_spec.package.stage.path)\n assert not os.path.exists(mpileaks_spec.package.stage.path)\n install(\"--fake\")\n assert os.path.exists(libelf_spec.package.stage.path)\n assert not os.path.exists(mpileaks_spec.package.stage.path)\n\n\n# Helper method for test_install_develop_keep_stage\ndef _always_fail(cls, *args, **kwargs):\n raise Exception(\"Restage or destruction of dev stage detected during install\")\n\n\n@pytest.mark.regression(\"24148\")\ndef test_virtual_spec_concretize_together(mutable_config):\n # An environment should permit to concretize \"mpi\"\n with ev.create(\"virtual_spec\") as e:\n mutable_config.set(\"concretizer:unify\", True)\n e.add(\"mpi\")\n e.concretize()\n assert any(s.package.provides(\"mpi\") for _, s in e.concretized_specs())\n\n\ndef test_query_develop_specs(tmp_path: pathlib.Path):\n \"\"\"Test whether a spec is develop'ed or not\"\"\"\n srcdir = tmp_path / \"here\"\n srcdir.mkdir()\n\n env(\"create\", \"test\")\n with ev.read(\"test\") as e:\n e.add(\"mpich\")\n e.add(\"mpileaks\")\n develop(\"--no-clone\", \"-p\", str(srcdir), \"mpich@=1\")\n\n assert e.is_develop(Spec(\"mpich\"))\n assert not e.is_develop(Spec(\"mpileaks\"))\n\n\n@pytest.mark.parametrize(\"method\", [spack.cmd.env.env_activate, spack.cmd.env.env_deactivate])\n@pytest.mark.parametrize(\n \"env,no_env,env_dir\", [(\"b\", False, None), (None, True, None), (None, False, \"path/\")]\n)\ndef test_activation_and_deactivation_ambiguities(method, env, no_env, env_dir, capfd):\n \"\"\"spack [-e x | -E | -D x/] env [activate | deactivate] y are ambiguous\"\"\"\n args = Namespace(\n shell=\"sh\", env_name=\"a\", env=env, no_env=no_env, env_dir=env_dir, keep_relative=False\n )\n with pytest.raises(SystemExit):\n method(args)\n _, err = capfd.readouterr()\n assert \"is ambiguous\" in err\n\n\n@pytest.mark.regression(\"26548\")\ndef test_custom_store_in_environment(mutable_config, tmp_path: pathlib.Path):\n spack_yaml = tmp_path / \"spack.yaml\"\n install_root = tmp_path / \"store\"\n spack_yaml.write_text(\n \"\"\"\nspack:\n specs:\n - libelf\n config:\n install_tree:\n root: {0}\n\"\"\".format(install_root)\n )\n current_store_root = str(spack.store.STORE.root)\n assert str(current_store_root) != str(install_root)\n with ev.Environment(str(tmp_path)):\n assert str(spack.store.STORE.root) == str(install_root)\n assert str(spack.store.STORE.root) == current_store_root\n\n\ndef test_activate_temp(monkeypatch, tmp_path: pathlib.Path):\n \"\"\"Tests whether `spack env activate --temp` creates an environment in a\n temporary directory\"\"\"\n env_dir = lambda: str(tmp_path)\n monkeypatch.setattr(spack.cmd.env, \"create_temp_env_directory\", env_dir)\n shell = env(\"activate\", \"--temp\", \"--sh\")\n active_env_var = next(line for line in shell.splitlines() if ev.spack_env_var in line)\n assert str(tmp_path) in active_env_var\n assert ev.is_env_dir(str(tmp_path))\n\n\n@pytest.mark.parametrize(\n \"conflict_arg\", [[\"--dir\"], [\"--keep-relative\"], [\"--with-view\", \"foo\"], [\"env\"]]\n)\ndef test_activate_parser_conflicts_with_temp(conflict_arg):\n with pytest.raises(SpackCommandError):\n env(\"activate\", \"--sh\", \"--temp\", *conflict_arg)\n\n\ndef test_create_and_activate_managed(tmp_path: pathlib.Path):\n with fs.working_dir(str(tmp_path)):\n shell = env(\"activate\", \"--without-view\", \"--create\", \"--sh\", \"foo\")\n active_env_var = next(line for line in shell.splitlines() if ev.spack_env_var in line)\n assert str(tmp_path) in active_env_var\n active_ev = ev.active_environment()\n assert active_ev and \"foo\" == active_ev.name\n env(\"deactivate\")\n\n\ndef test_create_and_activate_independent(tmp_path: pathlib.Path):\n with fs.working_dir(str(tmp_path)):\n env_dir = os.path.join(str(tmp_path), \"foo\")\n shell = env(\"activate\", \"--without-view\", \"--create\", \"--sh\", env_dir)\n active_env_var = next(line for line in shell.splitlines() if ev.spack_env_var in line)\n assert str(env_dir) in active_env_var\n assert ev.is_env_dir(env_dir)\n env(\"deactivate\")\n\n\ndef test_activate_default(monkeypatch):\n \"\"\"Tests whether `spack env activate` creates / activates the default\n environment\"\"\"\n assert not ev.exists(\"default\")\n\n # Activating it the first time should create it\n env(\"activate\", \"--sh\")\n env(\"deactivate\", \"--sh\")\n assert ev.exists(\"default\")\n\n # Activating it while it already exists should work\n env(\"activate\", \"--sh\")\n env(\"deactivate\", \"--sh\")\n assert ev.exists(\"default\")\n\n env(\"remove\", \"-y\", \"default\")\n assert not ev.exists(\"default\")\n\n\ndef test_env_view_fail_if_symlink_points_elsewhere(\n tmp_path: pathlib.Path, install_mockery, mock_fetch\n):\n view = str(tmp_path / \"view\")\n # Put a symlink to an actual directory in view\n non_view_dir = str(tmp_path / \"dont-delete-me\")\n os.mkdir(non_view_dir)\n os.symlink(non_view_dir, view)\n with ev.create(\"env\", with_view=view):\n add(\"libelf\")\n install(\"--fake\")\n assert os.path.isdir(non_view_dir)\n\n\ndef test_failed_view_cleanup(tmp_path: pathlib.Path, mock_stage, mock_fetch, install_mockery):\n \"\"\"Tests whether Spack cleans up after itself when a view fails to create\"\"\"\n view_dir = tmp_path / \"view\"\n with ev.create(\"env\", with_view=str(view_dir)):\n add(\"libelf\")\n install(\"--fake\")\n\n # Save the current view directory.\n resolved_view = view_dir.resolve(strict=True)\n all_views = resolved_view.parent\n views_before = list(all_views.iterdir())\n\n # Add a spec that results in view clash when creating a view\n with ev.read(\"env\"):\n add(\"libelf cflags=-O3\")\n with pytest.raises(ev.SpackEnvironmentViewError):\n install(\"--fake\")\n\n # Make sure there is no broken view in the views directory, and the current\n # view is the original view from before the failed regenerate attempt.\n views_after = list(all_views.iterdir())\n assert views_before == views_after\n assert view_dir.samefile(resolved_view), view_dir\n\n\ndef test_environment_view_target_already_exists(\n tmp_path: pathlib.Path, mock_stage, mock_fetch, install_mockery\n):\n \"\"\"When creating a new view, Spack should check whether\n the new view dir already exists. If so, it should not be\n removed or modified.\"\"\"\n\n # Create a new environment\n view = str(tmp_path / \"view\")\n env(\"create\", \"--with-view={0}\".format(view), \"test\")\n with ev.read(\"test\"):\n add(\"libelf\")\n install(\"--fake\")\n\n # Empty the underlying view\n real_view = os.path.realpath(view)\n assert os.listdir(real_view) # make sure it had *some* contents\n shutil.rmtree(real_view)\n\n # Replace it with something new.\n os.mkdir(real_view)\n fs.touch(os.path.join(real_view, \"file\"))\n\n # Remove the symlink so Spack can't know about the \"previous root\"\n os.unlink(view)\n\n # Regenerate the view, which should realize it can't write into the same dir.\n msg = \"Failed to generate environment view\"\n with ev.read(\"test\"):\n with pytest.raises(ev.SpackEnvironmentViewError, match=msg):\n env(\"view\", \"regenerate\")\n\n # Make sure the dir was left untouched.\n assert not os.path.lexists(view)\n assert os.listdir(real_view) == [\"file\"]\n\n\ndef test_environment_query_spec_by_hash(mock_stage, mock_fetch, install_mockery):\n env(\"create\", \"test\")\n with ev.read(\"test\"):\n add(\"libdwarf\")\n concretize()\n with ev.read(\"test\") as e:\n spec = e.matching_spec(\"libelf\")\n install(\"--fake\", f\"/{spec.dag_hash()}\")\n with ev.read(\"test\") as e:\n assert not e.matching_spec(\"libdwarf\").installed\n assert e.matching_spec(\"libelf\").installed\n\n\n@pytest.mark.parametrize(\"lockfile\", [\"v1\", \"v2\", \"v3\"])\ndef test_read_old_lock_and_write_new(tmp_path: pathlib.Path, lockfile):\n # v1 lockfiles stored by a coarse DAG hash that did not include build deps.\n # They could not represent multiple build deps with different build hashes.\n #\n # v2 and v3 lockfiles are keyed by a \"build hash\", so they can represent specs\n # with different build deps but the same DAG hash. However, those two specs\n # could never have been built together, because they cannot coexist in a\n # Spack DB, which is keyed by DAG hash. The second one would just be a no-op\n # no-op because its DAG hash was already in the DB.\n #\n # Newer Spack uses a fine-grained DAG hash that includes build deps, package hash,\n # and more. But, we still have to identify old specs by their original DAG hash.\n # Essentially, the name (hash) we give something in Spack at concretization time is\n # its name forever (otherwise we'd need to relocate prefixes and disrupt existing\n # installations). So, we just discard the second conflicting dtbuild1 version when\n # reading v2 and v3 lockfiles. This is what old Spack would've done when installing\n # the environment, anyway.\n #\n # This test ensures the behavior described above.\n lockfile_path = os.path.join(spack.paths.test_path, \"data\", \"legacy_env\", \"%s.lock\" % lockfile)\n\n # read in the JSON from a legacy lockfile\n with open(lockfile_path, encoding=\"utf-8\") as f:\n old_dict = sjson.load(f)\n\n # read all DAG hashes from the legacy lockfile and record its shadowed DAG hash.\n old_hashes = set()\n shadowed_hash = None\n for key, spec_dict in old_dict[\"concrete_specs\"].items():\n if \"hash\" not in spec_dict:\n # v1 and v2 key specs by their name in concrete_specs\n name, spec_dict = next(iter(spec_dict.items()))\n else:\n # v3 lockfiles have a `name` field and key by hash\n name = spec_dict[\"name\"]\n\n # v1 lockfiles do not have a \"hash\" field -- they use the key.\n dag_hash = key if lockfile == \"v1\" else spec_dict[\"hash\"]\n old_hashes.add(dag_hash)\n\n # v1 lockfiles can't store duplicate build dependencies, so they\n # will not have a shadowed hash.\n if lockfile != \"v1\":\n # v2 and v3 lockfiles store specs by build hash, so they can have multiple\n # keys for the same DAG hash. We discard the second one (dtbuild@1.0).\n if name == \"dtbuild1\" and spec_dict[\"version\"] == \"1.0\":\n shadowed_hash = dag_hash\n\n # make an env out of the old lockfile -- env should be able to read v1/v2/v3\n test_lockfile_path = str(tmp_path / \"spack.lock\")\n shutil.copy(lockfile_path, test_lockfile_path)\n _env_create(\"test\", init_file=test_lockfile_path, with_view=False)\n\n # re-read the old env as a new lockfile\n e = ev.read(\"test\")\n hashes = set(e._to_lockfile_dict()[\"concrete_specs\"])\n\n # v1 doesn't have duplicate build deps.\n # in v2 and v3, the shadowed hash will be gone.\n if shadowed_hash:\n old_hashes -= set([shadowed_hash])\n\n # make sure we see the same hashes in old and new lockfiles\n assert old_hashes == hashes\n\n\ndef test_read_v1_lock_creates_backup(tmp_path: pathlib.Path):\n \"\"\"When reading a version-1 lockfile, make sure that a backup of that file\n is created.\n \"\"\"\n v1_lockfile_path = pathlib.Path(spack.paths.test_path) / \"data\" / \"legacy_env\" / \"v1.lock\"\n test_lockfile_path = tmp_path / \"init\" / ev.lockfile_name\n test_lockfile_path.parent.mkdir(parents=True, exist_ok=False)\n shutil.copy(v1_lockfile_path, test_lockfile_path)\n\n e = ev.create_in_dir(tmp_path, init_file=test_lockfile_path)\n assert os.path.exists(e._lock_backup_v1_path)\n assert filecmp.cmp(e._lock_backup_v1_path, v1_lockfile_path)\n\n\n@pytest.mark.parametrize(\"lockfile\", [\"v1\", \"v2\", \"v3\"])\ndef test_read_legacy_lockfile_and_reconcretize(\n mock_stage, mock_fetch, install_mockery, lockfile, tmp_path: pathlib.Path\n):\n # In legacy lockfiles v2 and v3 (keyed by build hash), there may be multiple\n # versions of the same spec with different build dependencies, which means\n # they will have different build hashes but the same DAG hash.\n # In the case of DAG hash conflicts, we always keep the spec associated with\n # whichever root spec came first in the \"roots\" list.\n #\n # After reconcretization with the *new*, finer-grained DAG hash, there should no\n # longer be conflicts, and the previously conflicting specs can coexist in the\n # same environment.\n test_path = pathlib.Path(spack.paths.test_path)\n lockfile_content = test_path / \"data\" / \"legacy_env\" / f\"{lockfile}.lock\"\n legacy_lockfile_path = tmp_path / ev.lockfile_name\n shutil.copy(lockfile_content, legacy_lockfile_path)\n\n # The order of the root specs in this environment is:\n # [\n # wci7a3a -> dttop ^dtbuild1@0.5,\n # 5zg6wxw -> dttop ^dtbuild1@1.0\n # ]\n # So in v2 and v3 lockfiles we have two versions of dttop with the same DAG\n # hash but different build hashes.\n\n env(\"create\", \"test\", str(legacy_lockfile_path))\n test = ev.read(\"test\")\n assert len(test.specs_by_hash) == 1\n\n single_root = next(iter(test.specs_by_hash.values()))\n\n # v1 only has version 1.0, because v1 was keyed by DAG hash, and v1.0 overwrote\n # v0.5 on lockfile creation. v2 only has v0.5, because we specifically prefer\n # the one that would be installed when we read old lockfiles.\n if lockfile == \"v1\":\n assert single_root[\"dtbuild1\"].version == Version(\"1.0\")\n else:\n assert single_root[\"dtbuild1\"].version == Version(\"0.5\")\n\n # Now forcefully reconcretize\n with ev.read(\"test\"):\n concretize(\"-f\")\n\n # After reconcretizing, we should again see two roots, one depending on each\n # of the dtbuild1 versions specified in the roots of the original lockfile.\n test = ev.read(\"test\")\n assert len(test.specs_by_hash) == 2\n\n expected_versions = set([Version(\"0.5\"), Version(\"1.0\")])\n current_versions = set(s[\"dtbuild1\"].version for s in test.specs_by_hash.values())\n assert current_versions == expected_versions\n\n\ndef _parse_dry_run_package_installs(make_output):\n \"\"\"Parse `spack install ... # ` output from a make dry run.\"\"\"\n return [\n Spec(line.split(\"# \")[1]).name\n for line in make_output.splitlines()\n if line.startswith(\"spack\")\n ]\n\n\n@pytest.mark.parametrize(\n \"depfile_flags,expected_installs\",\n [\n # This installs the full environment\n (\n [\"--use-buildcache=never\"],\n [\n \"dtbuild1\",\n \"dtbuild2\",\n \"dtbuild3\",\n \"dtlink1\",\n \"dtlink2\",\n \"dtlink3\",\n \"dtlink4\",\n \"dtlink5\",\n \"dtrun1\",\n \"dtrun2\",\n \"dtrun3\",\n \"dttop\",\n ],\n ),\n # This prunes build deps at depth > 0\n (\n [\"--use-buildcache=package:never,dependencies:only\"],\n [\n \"dtbuild1\",\n \"dtlink1\",\n \"dtlink2\",\n \"dtlink3\",\n \"dtlink4\",\n \"dtlink5\",\n \"dtrun1\",\n \"dtrun2\",\n \"dtrun3\",\n \"dttop\",\n ],\n ),\n # This prunes all build deps\n (\n [\"--use-buildcache=only\"],\n [\"dtlink1\", \"dtlink3\", \"dtlink4\", \"dtlink5\", \"dtrun1\", \"dtrun3\", \"dttop\"],\n ),\n # Test whether pruning of build deps is correct if we explicitly include one\n # that is also a dependency of a root.\n (\n [\"--use-buildcache=only\", \"dttop\", \"dtbuild1\"],\n [\n \"dtbuild1\",\n \"dtlink1\",\n \"dtlink2\",\n \"dtlink3\",\n \"dtlink4\",\n \"dtlink5\",\n \"dtrun1\",\n \"dtrun2\",\n \"dtrun3\",\n \"dttop\",\n ],\n ),\n ],\n)\ndef test_environment_depfile_makefile(\n depfile_flags, expected_installs, tmp_path: pathlib.Path, mock_packages\n):\n env(\"create\", \"test\")\n make = Executable(\"make\")\n makefile = str(tmp_path / \"Makefile\")\n with ev.read(\"test\"):\n add(\"dttop\")\n concretize()\n\n # Disable jobserver so we can do a dry run.\n with ev.read(\"test\"):\n env(\n \"depfile\",\n \"-o\",\n makefile,\n \"--make-disable-jobserver\",\n \"--make-prefix=prefix\",\n *depfile_flags,\n )\n\n # Do make dry run.\n out = make(\"-n\", \"-f\", makefile, \"SPACK=spack\", output=str)\n\n specs_that_make_would_install = _parse_dry_run_package_installs(out)\n\n # Check that all specs are there (without duplicates)\n assert set(specs_that_make_would_install) == set(expected_installs)\n assert len(specs_that_make_would_install) == len(set(specs_that_make_would_install))\n\n\ndef test_depfile_safe_format():\n \"\"\"Test that depfile.MakefileSpec.safe_format() escapes target names.\"\"\"\n\n class SpecLike:\n def format(self, _):\n return \"abc@def=ghi\"\n\n spec = depfile.MakefileSpec(SpecLike())\n assert spec.safe_format(\"{name}\") == \"abc_def_ghi\"\n assert spec.unsafe_format(\"{name}\") == \"abc@def=ghi\"\n\n\ndef test_depfile_works_with_gitversions(tmp_path: pathlib.Path, mock_packages, monkeypatch):\n \"\"\"Git versions may contain = chars, which should be escaped in targets,\n otherwise they're interpreted as makefile variable assignments.\"\"\"\n monkeypatch.setattr(spack.package_base.PackageBase, \"git\", \"repo.git\", raising=False)\n env(\"create\", \"test\")\n\n make = Executable(\"make\")\n makefile = str(tmp_path / \"Makefile\")\n\n # Create an environment with dttop and dtlink1 both at a git version,\n # and generate a depfile\n with ev.read(\"test\"):\n add(f\"dttop@{'a' * 40}=1.0 ^dtlink1@{'b' * 40}=1.0\")\n concretize()\n env(\"depfile\", \"-o\", makefile, \"--make-disable-jobserver\", \"--make-prefix=prefix\")\n\n # Do a dry run on the generated depfile\n out = make(\"-n\", \"-f\", makefile, \"SPACK=spack\", output=str)\n\n # Check that all specs are there (without duplicates)\n specs_that_make_would_install = _parse_dry_run_package_installs(out)\n expected_installs = [\n \"dtbuild1\",\n \"dtbuild2\",\n \"dtbuild3\",\n \"dtlink1\",\n \"dtlink2\",\n \"dtlink3\",\n \"dtlink4\",\n \"dtlink5\",\n \"dtrun1\",\n \"dtrun2\",\n \"dtrun3\",\n \"dttop\",\n ]\n assert set(specs_that_make_would_install) == set(expected_installs)\n assert len(specs_that_make_would_install) == len(set(specs_that_make_would_install))\n\n\n@pytest.mark.parametrize(\n \"picked_package,expected_installs\",\n [\n (\n \"dttop\",\n [\n \"dtbuild2\",\n \"dtlink2\",\n \"dtrun2\",\n \"dtbuild1\",\n \"dtlink4\",\n \"dtlink3\",\n \"dtlink1\",\n \"dtlink5\",\n \"dtbuild3\",\n \"dtrun3\",\n \"dtrun1\",\n \"dttop\",\n ],\n ),\n (\"dtrun1\", [\"dtlink5\", \"dtbuild3\", \"dtrun3\", \"dtrun1\"]),\n ],\n)\ndef test_depfile_phony_convenience_targets(\n picked_package, expected_installs: set, tmp_path: pathlib.Path, mock_packages\n):\n \"\"\"Check whether convenience targets \"install/%\" and \"install-deps/%\" are created for\n each package if \"--make-prefix\" is absent.\"\"\"\n make = Executable(\"make\")\n with fs.working_dir(str(tmp_path)):\n with ev.create_in_dir(\".\"):\n add(\"dttop\")\n concretize()\n\n with ev.Environment(\".\") as e:\n picked_spec = e.matching_spec(picked_package)\n env(\"depfile\", \"-o\", \"Makefile\", \"--make-disable-jobserver\")\n\n # Phony install/* target should install picked package and all its deps\n specs_that_make_would_install = _parse_dry_run_package_installs(\n make(\n \"-n\",\n picked_spec.format(\"install/{name}-{version}-{hash}\"),\n \"SPACK=spack\",\n output=str,\n )\n )\n\n assert set(specs_that_make_would_install) == set(expected_installs)\n assert len(specs_that_make_would_install) == len(set(specs_that_make_would_install))\n\n # Phony install-deps/* target shouldn't install picked package\n specs_that_make_would_install = _parse_dry_run_package_installs(\n make(\n \"-n\",\n picked_spec.format(\"install-deps/{name}-{version}-{hash}\"),\n \"SPACK=spack\",\n output=str,\n )\n )\n\n assert set(specs_that_make_would_install) == set(expected_installs) - {picked_package}\n assert len(specs_that_make_would_install) == len(set(specs_that_make_would_install))\n\n\ndef test_environment_depfile_out(tmp_path: pathlib.Path, mock_packages):\n env(\"create\", \"test\")\n makefile_path = str(tmp_path / \"Makefile\")\n with ev.read(\"test\"):\n add(\"libdwarf\")\n concretize()\n with ev.read(\"test\"):\n env(\"depfile\", \"-G\", \"make\", \"-o\", makefile_path)\n stdout = env(\"depfile\", \"-G\", \"make\")\n with open(makefile_path, \"r\", encoding=\"utf-8\") as f:\n assert stdout == f.read()\n\n\ndef test_spack_package_ids_variable(tmp_path: pathlib.Path, mock_packages):\n # Integration test for post-install hooks through prefix/SPACK_PACKAGE_IDS\n # variable\n env(\"create\", \"test\")\n makefile_path = str(tmp_path / \"Makefile\")\n include_path = str(tmp_path / \"include.mk\")\n\n # Create env and generate depfile in include.mk with prefix example/\n with ev.read(\"test\"):\n add(\"libdwarf\")\n concretize()\n\n with ev.read(\"test\"):\n env(\n \"depfile\",\n \"-G\",\n \"make\",\n \"--make-disable-jobserver\",\n \"--make-prefix=example\",\n \"-o\",\n include_path,\n )\n\n # Include in Makefile and create target that depend on SPACK_PACKAGE_IDS\n with open(makefile_path, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\nall: post-install\n\ninclude include.mk\n\nexample/post-install/%: example/install/%\n\\t$(info post-install: $(HASH)) # noqa: W191,E101\n\npost-install: $(addprefix example/post-install/,$(example/SPACK_PACKAGE_IDS))\n\"\"\"\n )\n make = Executable(\"make\")\n\n # Do dry run.\n out = make(\"-n\", \"-C\", str(tmp_path), \"SPACK=spack\", output=str)\n\n # post-install: should've been executed\n with ev.read(\"test\") as test:\n for s in test.all_specs():\n assert \"post-install: {}\".format(s.dag_hash()) in out\n\n\ndef test_depfile_empty_does_not_error(tmp_path: pathlib.Path):\n # For empty environments Spack should create a depfile that does nothing\n make = Executable(\"make\")\n makefile = str(tmp_path / \"Makefile\")\n\n env(\"create\", \"test\")\n with ev.read(\"test\"):\n env(\"depfile\", \"-o\", makefile)\n\n make(\"-f\", makefile)\n\n assert make.returncode == 0\n\n\ndef test_unify_when_possible_works_around_conflicts(mutable_config):\n with ev.create(\"coconcretization\") as e:\n mutable_config.set(\"concretizer:unify\", \"when_possible\")\n e.add(\"mpileaks+opt\")\n e.add(\"mpileaks~opt\")\n e.add(\"mpich\")\n e.concretize()\n\n assert len([x for x in e.all_specs() if x.satisfies(\"mpileaks\")]) == 2\n assert len([x for x in e.all_specs() if x.satisfies(\"mpileaks+opt\")]) == 1\n assert len([x for x in e.all_specs() if x.satisfies(\"mpileaks~opt\")]) == 1\n assert len([x for x in e.all_specs() if x.satisfies(\"mpich\")]) == 1\n\n\ndef test_env_include_packages_url(\n tmp_path: pathlib.Path, mutable_empty_config, mock_fetch_url_text, mock_curl_configs\n):\n \"\"\"Test inclusion of a (GitHub) URL.\"\"\"\n develop_url = \"https://github.com/fake/fake/blob/develop/\"\n default_packages = develop_url + \"etc/fake/defaults/packages.yaml\"\n sha256 = \"8d428c600b215e3b4a207a08236659dfc2c9ae2782c35943a00ee4204a135702\"\n spack_yaml = tmp_path / \"spack.yaml\"\n with open(spack_yaml, \"w\", encoding=\"utf-8\") as f:\n f.write(\n f\"\"\"\\\nspack:\n include:\n - path: {default_packages}\n sha256: {sha256}\n\"\"\"\n )\n\n with spack.config.override(\"config:url_fetch_method\", \"curl\"):\n env = ev.Environment(str(tmp_path))\n ev.activate(env)\n\n # Make sure a setting from test/data/config/packages.yaml is present\n cfg = spack.config.get(\"packages\")\n assert \"mpich\" in cfg[\"all\"][\"providers\"][\"mpi\"]\n\n\ndef test_relative_view_path_on_command_line_is_made_absolute(tmp_path: pathlib.Path):\n with fs.working_dir(str(tmp_path)):\n env(\"create\", \"--with-view\", \"view\", \"--dir\", \"env\")\n environment = ev.Environment(os.path.join(\".\", \"env\"))\n environment.regenerate_views()\n assert os.path.samefile(\"view\", environment.default_view.root)\n\n\ndef test_environment_created_in_users_location(mutable_mock_env_path, tmp_path: pathlib.Path):\n \"\"\"Test that an environment is created in a location based on the config\"\"\"\n env_dir = str(mutable_mock_env_path)\n\n assert str(tmp_path) in env_dir\n assert not os.path.isdir(env_dir)\n\n dir_name = \"user_env\"\n env(\"create\", dir_name)\n out = env(\"list\")\n\n assert dir_name in out\n assert env_dir in ev.root(dir_name)\n assert os.path.isdir(os.path.join(env_dir, dir_name))\n\n\ndef test_environment_created_from_lockfile_has_view(\n mock_packages, temporary_store, tmp_path: pathlib.Path\n):\n \"\"\"When an env is created from a lockfile, a view should be generated for it\"\"\"\n env_a = str(tmp_path / \"a\")\n env_b = str(tmp_path / \"b\")\n\n # Create an environment and install a package in it\n env(\"create\", \"-d\", env_a)\n with ev.Environment(env_a):\n add(\"libelf\")\n install(\"--fake\")\n\n # Create another environment from the lockfile of the first environment\n env(\"create\", \"-d\", env_b, os.path.join(env_a, \"spack.lock\"))\n\n # Make sure the view was created\n with ev.Environment(env_b) as e:\n assert os.path.isdir(e.view_path_default)\n\n\ndef test_env_view_disabled(tmp_path: pathlib.Path, mutable_mock_env_path):\n \"\"\"Ensure an inlined view being disabled means not even the default view\n is created (since the case doesn't appear to be covered in this module).\"\"\"\n spack_yaml = tmp_path / ev.manifest_name\n spack_yaml.write_text(\n \"\"\"\\\nspack:\n specs:\n - mpileaks\n view: false\n\"\"\"\n )\n env(\"create\", \"disabled\", str(spack_yaml))\n with ev.read(\"disabled\") as e:\n e.concretize()\n\n assert len(e.views) == 0\n assert not os.path.exists(e.view_path_default)\n\n\n@pytest.mark.parametrize(\"first\", [\"false\", \"true\", \"custom\"])\ndef test_env_include_mixed_views(\n tmp_path: pathlib.Path, mutable_config, mutable_mock_env_path, first\n):\n \"\"\"Ensure including path and boolean views in different combinations result\n in the creation of only the first view if it is not disabled.\"\"\"\n false_yaml = tmp_path / \"false-view.yaml\"\n false_yaml.write_text(\"view: false\\n\")\n\n true_yaml = tmp_path / \"true-view.yaml\"\n true_yaml.write_text(\"view: true\\n\")\n\n custom_name = \"my-test-view\"\n custom_view = tmp_path / custom_name\n custom_yaml = tmp_path / \"custom-view.yaml\"\n custom_yaml.write_text(\n f\"\"\"\nview:\n {custom_name}:\n root: {custom_view}\n\"\"\"\n )\n\n if first == \"false\":\n order = [false_yaml, true_yaml, custom_yaml]\n elif first == \"true\":\n order = [true_yaml, custom_yaml, false_yaml]\n else:\n order = [custom_yaml, false_yaml, true_yaml]\n includes = [f\" - {yaml}\\n\" for yaml in order]\n\n spack_yaml = tmp_path / ev.manifest_name\n spack_yaml.write_text(\n f\"\"\"\\\nspack:\n include:\n{\"\".join(includes)}\n specs:\n - mpileaks\n\"\"\"\n )\n\n env(\"create\", \"test\", str(spack_yaml))\n with ev.read(\"test\") as e:\n concretize()\n\n # Only the first included view should be created if view not disabled by it\n assert len(e.views) == 0 if first == \"false\" else 1\n if first == \"true\":\n assert os.path.exists(e.view_path_default)\n else:\n assert not os.path.exists(e.view_path_default)\n\n if first == \"custom\":\n assert os.path.exists(custom_view)\n else:\n assert not os.path.exists(custom_view)\n\n\ndef test_stack_view_multiple_views_same_name(\n installed_environment, template_combinatorial_env, tmp_path: pathlib.Path\n):\n \"\"\"Test multiple views with the same name combine settings with precedence\n given to the options in spack.yaml.\"\"\"\n # Write the view configuration and or manifest file\n\n view_filename = tmp_path / \"view.yaml\"\n default_dir = tmp_path / \"default-view\"\n default_view = f\"\"\"\\\nview:\n default:\n root: {default_dir}\n select: ['target=x86_64']\n projections:\n all: '{{architecture.target}}/{{name}}-{{version}}-from-view'\n\"\"\"\n view_filename.write_text(default_view)\n\n view_dir = tmp_path / \"view\"\n with installed_environment(\n f\"\"\"\\\nspack:\n include:\n - {view_filename}\n definitions:\n - packages: [mpileaks, cmake]\n - targets: ['target=x86_64', 'target=core2']\n\n specs:\n - matrix:\n - [$packages]\n - [$targets]\n\n view:\n default:\n root: {view_dir}\n exclude: ['cmake']\n projections:\n all: '{{architecture.target}}/{{name}}-{{version}}'\n\"\"\"\n ) as e:\n # the view root in the included view should NOT exist\n assert not os.path.exists(str(default_dir))\n\n for spec in traverse_nodes(e.concrete_roots(), deptype=(\"link\", \"run\")):\n # no specs will exist in the included view projection\n base_dir = view_dir / f\"{spec.architecture.target}\"\n included_dir = base_dir / f\"{spec.name}-{spec.version}-from-view\"\n assert not included_dir.exists()\n\n # only target=x86_64 specs (selected in the included view) that\n # are also not cmake (excluded in the environment view) should exist\n if spec.name == \"gcc-runtime\":\n continue\n current_dir = view_dir / f\"{spec.architecture.target}\" / f\"{spec.name}-{spec.version}\"\n assert current_dir.exists() is not (\n spec.satisfies(\"cmake\") or spec.satisfies(\"target=core2\")\n )\n\n\ndef test_env_view_resolves_identical_file_conflicts(\n tmp_path: pathlib.Path, install_mockery, mock_fetch\n):\n \"\"\"When files clash in a view, but refer to the same file on disk (for example, the dependent\n symlinks to a file in the dependency at the same relative path), Spack links the first regular\n file instead of symlinks. This is important for copy type views where we need the underlying\n file to be copied instead of the symlink (when a symlink would be copied, it would become a\n self-referencing symlink after relocation). The test uses a symlink type view though, since\n that keeps track of the original file path.\"\"\"\n with ev.create(\"env\", with_view=tmp_path / \"view\") as e:\n add(\"view-resolve-conflict-top\")\n install()\n top = e.matching_spec(\"view-resolve-conflict-top\").prefix\n bottom = e.matching_spec(\"view-file\").prefix\n\n # In this example we have `./bin/x` in 3 prefixes, two links, one regular file. We expect the\n # regular file to be linked into the view. There are also 2 links at `./bin/y`, but no regular\n # file, so we expect standard behavior: first entry is linked into the view.\n\n # view-resolve-conflict-top/bin/\n # x -> view-file/bin/x\n # y -> view-resolve-conflict-middle/bin/y # expect this y to be linked\n # view-resolve-conflict-middle/bin/\n # x -> view-file/bin/x\n # y -> view-file/bin/x\n # view-file/bin/\n # x # expect this x to be linked\n\n assert readlink(tmp_path / \"view\" / \"bin\" / \"x\") == bottom.bin.x\n assert readlink(tmp_path / \"view\" / \"bin\" / \"y\") == top.bin.y\n\n\ndef test_env_view_ignores_different_file_conflicts(\n tmp_path: pathlib.Path, install_mockery, mock_fetch\n):\n \"\"\"Test that file-file conflicts for two unique files in environment views are ignored, and\n that the dependent's file is linked into the view, not the dependency's file.\"\"\"\n with ev.create(\"env\", with_view=tmp_path / \"view\") as e:\n add(\"view-ignore-conflict\")\n install()\n prefix_dependent = e.matching_spec(\"view-ignore-conflict\").prefix\n # The dependent's file is linked into the view\n assert readlink(tmp_path / \"view\" / \"bin\" / \"x\") == prefix_dependent.bin.x\n\n\n@pytest.mark.regression(\"51054\")\ndef test_non_str_repos(installed_environment):\n with installed_environment(\n \"\"\"\\\nspack:\n repos:\n builtin:\n branch: develop\"\"\"\n ):\n pass\n\n\ndef test_concretized_specs_and_include_concrete(mutable_config):\n \"\"\"Tests the consistency of concretized specs when there are either\n duplicate input specs or duplicate hashes.\n \"\"\"\n # Create a structure like this one\n #\n # Local specs:\n # - mpileaks -> hash1\n # - libelf@0.8.12 -> hash2\n # - pkg-a -> hash3\n #\n # Included specs:\n # - mpileaks -> hash4\n # - libelf -> hash2\n # - pkg-a -> hash3\n env(\"create\", \"included-env\")\n with ev.read(\"included-env\") as e:\n e.add(\"mpileaks\")\n e.add(\"libelf\")\n e.add(\"pkg-a\")\n mutable_config.set(\n \"packages\", {\"mpileaks\": {\"require\": [\"@2.2\"]}, \"libelf\": {\"require\": [\"@0.8.12\"]}}\n )\n included_pairs = e.concretize()\n e.write()\n\n env(\"create\", \"--include-concrete\", \"included-env\", \"main-env\")\n with ev.read(\"main-env\") as e:\n e.add(\"mpileaks\")\n e.add(\"libelf@0.8.12\")\n e.add(\"pkg-a\")\n mutable_config.set(\"packages\", {\"mpileaks\": {\"require\": [\"@2.3\"]}})\n spec_pairs = e.concretize()\n concretized_specs = list(e.concretized_specs())\n assert list(dedupe(spec_pairs + included_pairs)) == concretized_specs\n assert len(concretized_specs) == 5\n\n\ndef test_view_can_select_group_of_specs(installed_environment, tmp_path: pathlib.Path):\n \"\"\"Tests that we can select groups of specs in a view and exclude other groups\"\"\"\n view_dir = tmp_path / \"view\"\n with installed_environment(\n f\"\"\"\\\nspack:\n specs:\n - group: apps1\n specs:\n - mpileaks\n - group: apps2\n specs:\n - cmake\n - group: apps3\n specs:\n - pkg-a\n view:\n default:\n root: {view_dir}\n group: [apps1, apps2]\n\"\"\"\n ) as test:\n for item in test.concretized_roots:\n # Assertions are based on the behavior of the \"--fake\" install\n bin_file = pathlib.Path(test.default_view.view()._root) / \"bin\" / item.root.name\n assert not bin_file.exists() if item.group == \"apps3\" else bin_file.exists()\n\n\ndef test_view_can_select_group_of_specs_using_string(\n installed_environment, tmp_path: pathlib.Path\n):\n \"\"\"Tests that we can select groups of specs in a view and exclude other groups\"\"\"\n view_dir = tmp_path / \"view\"\n with installed_environment(\n f\"\"\"\\\nspack:\n specs:\n - group: apps1\n specs:\n - mpileaks\n - group: apps2\n specs:\n - cmake\n view:\n default:\n root: {view_dir}\n group: apps1\n\"\"\"\n ) as test:\n for item in test.concretized_roots:\n # Assertions are based on the behavior of the \"--fake\" install\n bin_file = pathlib.Path(test.default_view.view()._root) / \"bin\" / item.root.name\n assert not bin_file.exists() if item.group == \"apps2\" else bin_file.exists()\n\n\ndef test_env_include_concrete_only(tmp_path, mock_packages, mutable_config):\n \"\"\"Confirm that an environment that only includes a concrete environment actually loads it.\"\"\"\n specs = [\"libdwarf\", \"libelf\"]\n\n include_dir = tmp_path / \"includes\"\n include_dir.mkdir()\n include_manifest = include_dir / ev.manifest_name\n include_manifest.write_text(\n f\"\"\"\\\nspack:\n specs:\n - {specs[0]}\n - {specs[1]}\n\"\"\"\n )\n include_env = ev.create(\"test_include\", include_manifest)\n include_env.concretize()\n include_env.write()\n\n include_lockfile = include_env.lock_path\n assert os.path.exists(include_lockfile)\n\n manifest_file = tmp_path / ev.manifest_name\n manifest_file.write_text(\n f\"\"\"\\\nspack:\n include:\n - {str(include_lockfile)}\n\"\"\"\n )\n e = ev.create(\"test\", manifest_file)\n\n # Confirm the only specs the environment has are those loaded from the\n # lockfile.\n assert len(e.user_specs) == 0\n all_concrete = [s for s, _ in e.concretized_specs()]\n for spec in specs:\n assert Spec(spec) in all_concrete\n\n\n@pytest.mark.parametrize(\n \"concrete,includes\",\n [\n ([\"$HOME/path/to/other/environment\"], []),\n ([\"$HOME/path/to/another/environment\"], [\"a/b\", \"$HOME/includes\"]),\n ],\n)\ndef test_env_update_include_concrete(tmp_path: pathlib.Path, concrete, includes):\n \"\"\"Confirm update of include_concrete converts it to include.\"\"\"\n\n config = {\"include_concrete\": concrete}\n if includes:\n config[\"include\"] = includes\n new_concrete = [os.path.join(p, ev.lockfile_name) for p in concrete]\n assert spack.schema.env.update(config)\n assert \"include_concrete\" not in config\n assert config[\"include\"] == new_concrete + includes\n\n\ndef test_include_concrete_deprecation_warning(\n tmp_path: pathlib.Path, environment_from_manifest, capfd\n):\n try:\n environment_from_manifest(\n \"\"\"\\\nspack:\n include_concrete:\n - /path/to/some/environment\n\"\"\"\n )\n except ev.SpackEnvironmentError:\n pass\n\n _, err = capfd.readouterr()\n assert \"should be 'include'\" in err\n\n\ndef test_env_include_concrete_relative_path(tmp_path, mock_packages, mutable_config):\n \"\"\"Tests that a relative path in 'include' for a spack.lock is resolved relative to the\n manifest file, not the current working directory.\n \"\"\"\n # Create and concretize the included environment.\n include_dir = tmp_path / \"include_env\"\n include_dir.mkdir()\n (include_dir / ev.manifest_name).write_text(\n \"\"\"\\\nspack:\n specs:\n - libdwarf\n\"\"\"\n )\n with ev.Environment(str(include_dir)) as e:\n e.concretize()\n e.write()\n assert os.path.exists(e.lock_path)\n\n # Create the main environment in a sibling directory, using a *relative* path\n main_dir = tmp_path / \"main_env\"\n main_dir.mkdir()\n relative_lockfile = f\"../include_env/{ev.lockfile_name}\"\n (main_dir / ev.manifest_name).write_text(\n f\"\"\"\\\nspack:\n include:\n - {relative_lockfile}\n\"\"\"\n )\n with ev.Environment(str(main_dir)) as e:\n e.concretize()\n e.write()\n assert len(e.user_specs) == 0\n assert [s for s, _ in e.concretized_specs()] == [Spec(\"libdwarf\")]\n\n\ndef test_env_include_concrete_git_lockfile(tmp_path, mock_packages, mutable_config, monkeypatch):\n \"\"\"Tests that a spack.lock listed inside a git-based include is resolved using the\n clone destination as the base, not the manifest directory.\n \"\"\"\n # Create and concretize the included environment.\n include_dir = tmp_path / \"include_env\"\n include_dir.mkdir()\n (include_dir / ev.manifest_name).write_text(\n \"\"\"\\\nspack:\n specs:\n - libdwarf\n\"\"\"\n )\n with ev.Environment(str(include_dir)) as e:\n e.concretize()\n e.write()\n assert os.path.exists(e.lock_path)\n\n # Simulate a cloned git repo: the spack.lock lives at a subpath within the clone.\n clone_dest = tmp_path / \"git_clone\"\n lock_subpath = \"envs/staging/spack.lock\"\n lock_in_clone = clone_dest / \"envs\" / \"staging\" / ev.lockfile_name\n lock_in_clone.parent.mkdir(parents=True)\n shutil.copy(e.lock_path, lock_in_clone)\n # is_env_dir() requires spack.yaml alongside spack.lock\n shutil.copy(os.path.join(e.path, ev.manifest_name), lock_in_clone.parent)\n\n # Prevent actual git operations; return the pre-built clone destination.\n monkeypatch.setattr(\n spack.config.GitIncludePaths, \"_clone\", lambda self, parent_scope: str(clone_dest)\n )\n\n main_dir = tmp_path / \"main_env\"\n main_dir.mkdir()\n (main_dir / ev.manifest_name).write_text(\n f\"\"\"\\\nspack:\n include:\n - git: https://example.com/configs.git\n branch: main\n paths:\n - {lock_subpath}\n\"\"\"\n )\n with ev.Environment(str(main_dir)) as e:\n e.concretize()\n e.write()\n assert len(e.user_specs) == 0\n assert [s for s, _ in e.concretized_specs()] == [Spec(\"libdwarf\")]\n\n\n@pytest.mark.skipif(sys.platform != \"linux\", reason=\"Target is linux-specific\")\ndef test_compiler_target_env(mock_packages, environment_from_manifest):\n \"\"\"Tests that Spack doesn't drop flag definitions on compilers\n when a target is required in config.\n \"\"\"\n\n cflags = \"-Wall\"\n env = environment_from_manifest(\n f\"\"\"\\\nspack:\n specs:\n - libdwarf %c=gcc@12.100.100\n packages:\n all:\n require:\n - \"target=x86_64_v3\"\n gcc:\n externals:\n - spec: gcc@12.100.100 languages:=c,c++\n prefix: /fake\n extra_attributes:\n compilers:\n c: /fake/bin/gcc\n cxx: /fake/bin/g++\n flags:\n cflags: {cflags}\n require: \"gcc\"\n\"\"\"\n )\n\n with env:\n env.concretize()\n libdwarf = env.concrete_roots()[0]\n assert libdwarf.satisfies(\"cflags=-Wall\")\n # Sanity check: make sure the target we expect was applied to the\n # compiler entry\n assert libdwarf[\"c\"].satisfies(\"gcc@12.100.100 languages:=c,c++ target=x86_64_v3\")\n\n\n@pytest.mark.regression(\"52247\")\ndef test_create_with_orphaned_directory(mutable_mock_env_path: pathlib.Path):\n \"\"\"Tests that an orphaned environment directory (directory exists, no spack.yaml) must not\n prevent 'spack env create' from creating a new environment with that name.\n \"\"\"\n orphaned = mutable_mock_env_path / \"test1\"\n orphaned_subdir = orphaned / \".spack-env\"\n orphaned_subdir.mkdir(parents=True)\n\n # The orphaned directory must not be seen as an existing environment\n assert not ev.exists(\"test1\")\n\n # Creating an environment over an orphaned directory must succeed\n env(\"create\", \"test1\")\n\n assert ev.exists(\"test1\")\n assert \"test1\" in env(\"list\")\n\n\n@pytest.mark.parametrize(\n \"setup\",\n [\n # valid environment: spack.yaml is a regular file\n pytest.param(\"valid\", id=\"valid\"),\n # orphaned directory: no spack.yaml at all\n pytest.param(\"orphaned\", id=\"orphaned\"),\n # broken manifest symlink: spack.yaml points to a non-existent target\n pytest.param(\"broken_symlink\", id=\"broken_symlink\"),\n ],\n)\n@pytest.mark.regression(\"52247\")\ndef test_exists_consistent_with_all_environment_names(\n mutable_mock_env_path: pathlib.Path, setup: str\n):\n \"\"\"Tests that exists() and all_environment_names() agree on whether an environment exists.\"\"\"\n env_dir = mutable_mock_env_path / \"myenv\"\n env_dir.mkdir(parents=True)\n manifest = env_dir / ev.manifest_name\n\n if setup == \"valid\":\n manifest.write_text(ev.default_manifest_yaml())\n elif setup == \"orphaned\":\n pass # no manifest\n elif setup == \"broken_symlink\":\n manifest.symlink_to(\"/nonexistent/spack.yaml\")\n\n listed = \"myenv\" in ev.all_environment_names()\n assert ev.exists(\"myenv\") == listed\n" }, { "path": "lib/spack/spack/test/cmd/extensions.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\nimport pytest\n\nimport spack.concretize\nfrom spack.installer import PackageInstaller\nfrom spack.main import SpackCommand, SpackCommandError\n\nextensions = SpackCommand(\"extensions\")\n\n\n@pytest.fixture\ndef python_database(mock_packages, mutable_database):\n specs = [\n spack.concretize.concretize_one(s) for s in [\"python\", \"py-extension1\", \"py-extension2\"]\n ]\n PackageInstaller([s.package for s in specs], explicit=True, fake=True).install()\n yield\n\n\n@pytest.mark.not_on_windows(\"All Fetchers Failed\")\n@pytest.mark.db\ndef test_extensions(mock_packages, python_database):\n ext2 = spack.concretize.concretize_one(\"py-extension2\")\n\n def check_output(ni):\n output = extensions(\"python\")\n packages = extensions(\"-s\", \"packages\", \"python\")\n installed = extensions(\"-s\", \"installed\", \"python\")\n assert \"==> python@2.7.11\" in output\n assert \"==> 4 extensions\" in output\n assert \"py-extension1\" in output\n assert \"py-extension2\" in output\n assert \"python-venv\" in output\n\n assert \"==> 4 extensions\" in packages\n assert \"py-extension1\" in packages\n assert \"py-extension2\" in packages\n assert \"python-venv\" in packages\n assert \"installed\" not in packages\n\n assert f\"{ni if ni else 'None'} installed\" in output\n assert f\"{ni if ni else 'None'} installed\" in installed\n\n check_output(3)\n ext2.package.do_uninstall(force=True)\n check_output(2)\n\n\ndef test_extensions_no_arguments(mock_packages):\n out = extensions()\n assert \"python\" in out\n\n\ndef test_extensions_raises_if_not_extendable(mock_packages):\n with pytest.raises(SpackCommandError):\n extensions(\"flake8\")\n\n\ndef test_extensions_raises_if_multiple_specs(mock_packages):\n with pytest.raises(SpackCommandError):\n extensions(\"python\", \"flake8\")\n" }, { "path": "lib/spack/spack/test/cmd/external.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport os\nimport pathlib\nimport sys\n\nimport pytest\n\nimport spack.cmd.external\nimport spack.config\nimport spack.cray_manifest\nimport spack.detection\nimport spack.detection.path\nimport spack.repo\nfrom spack.llnl.util.filesystem import getuid, touch\nfrom spack.main import SpackCommand\nfrom spack.spec import Spec\n\npytestmark = [pytest.mark.usefixtures(\"mock_packages\")]\n\n\n@pytest.fixture\ndef executables_found(monkeypatch):\n def _factory(result):\n def _mock_search(path_hints=None):\n return result\n\n monkeypatch.setattr(spack.detection.path, \"executables_in_path\", _mock_search)\n\n return _factory\n\n\ndef define_plat_exe(exe):\n if sys.platform == \"win32\":\n exe += \".bat\"\n return exe\n\n\ndef test_find_external_update_config(mutable_config):\n entries = [\n Spec.from_detection(\"cmake@1.foo\", external_path=\"/x/y1\"),\n Spec.from_detection(\"cmake@3.17.2\", external_path=\"/x/y2\"),\n ]\n pkg_to_entries = {\"cmake\": entries}\n\n scope = spack.config.default_modify_scope(\"packages\")\n spack.detection.update_configuration(pkg_to_entries, scope=scope, buildable=True)\n\n pkgs_cfg = spack.config.get(\"packages\")\n cmake_cfg = pkgs_cfg[\"cmake\"]\n cmake_externals = cmake_cfg[\"externals\"]\n\n assert {\"spec\": \"cmake@1.foo\", \"prefix\": \"/x/y1\"} in cmake_externals\n assert {\"spec\": \"cmake@3.17.2\", \"prefix\": \"/x/y2\"} in cmake_externals\n\n\ndef test_get_executables(working_env, mock_executable):\n cmake_path1 = mock_executable(\"cmake\", output=\"echo cmake version 1.foo\")\n path_to_exe = spack.detection.executables_in_path([os.path.dirname(cmake_path1)])\n cmake_exe = define_plat_exe(\"cmake\")\n assert path_to_exe[str(cmake_path1)] == cmake_exe\n\n\nexternal = SpackCommand(\"external\")\n\n\n# TODO: this test should be made to work, but in the meantime it is\n# causing intermittent (spurious) CI failures on all PRs\n@pytest.mark.not_on_windows(\"Test fails intermittently on Windows\")\ndef test_find_external_cmd_not_buildable(\n mutable_config, working_env, mock_executable, monkeypatch\n):\n \"\"\"When the user invokes 'spack external find --not-buildable', the config\n for any package where Spack finds an external version should be marked as\n not buildable.\n \"\"\"\n version = \"1.foo\"\n\n @classmethod\n def _determine_version(cls, exe):\n return version\n\n cmake_cls = spack.repo.PATH.get_pkg_class(\"cmake\")\n monkeypatch.setattr(cmake_cls, \"determine_version\", _determine_version)\n\n cmake_path = mock_executable(\"cmake\", output=f\"echo cmake version {version}\")\n os.environ[\"PATH\"] = str(cmake_path.parent)\n external(\"find\", \"--not-buildable\", \"cmake\")\n pkgs_cfg = spack.config.get(\"packages\")\n assert \"cmake\" in pkgs_cfg\n assert not pkgs_cfg[\"cmake\"][\"buildable\"]\n\n\n@pytest.mark.parametrize(\n \"names,tags,exclude,expected\",\n [\n # find -all\n (\n None,\n [\"detectable\"],\n [],\n [\n \"builtin_mock.cmake\",\n \"builtin_mock.find-externals1\",\n \"builtin_mock.gcc\",\n \"builtin_mock.intel-oneapi-compilers\",\n \"builtin_mock.llvm\",\n \"builtin_mock.mpich\",\n ],\n ),\n # find --all --exclude find-externals1\n (\n None,\n [\"detectable\"],\n [\"builtin_mock.find-externals1\"],\n [\n \"builtin_mock.cmake\",\n \"builtin_mock.gcc\",\n \"builtin_mock.intel-oneapi-compilers\",\n \"builtin_mock.llvm\",\n \"builtin_mock.mpich\",\n ],\n ),\n (\n None,\n [\"detectable\"],\n [\"find-externals1\"],\n [\n \"builtin_mock.cmake\",\n \"builtin_mock.gcc\",\n \"builtin_mock.intel-oneapi-compilers\",\n \"builtin_mock.llvm\",\n \"builtin_mock.mpich\",\n ],\n ),\n # find hwloc (and mock hwloc is not detectable)\n ([\"hwloc\"], [\"detectable\"], [], []),\n ],\n)\ndef test_package_selection(names, tags, exclude, expected):\n \"\"\"Tests various cases of selecting packages\"\"\"\n # In the mock repo we only have 'find-externals1' that is detectable\n result = spack.cmd.external.packages_to_search_for(names=names, tags=tags, exclude=exclude)\n assert set(result) == set(expected)\n\n\ndef test_find_external_no_manifest(mutable_config, working_env, monkeypatch):\n \"\"\"The user runs 'spack external find'; the default path for storing\n manifest files does not exist. Ensure that the command does not\n fail.\n \"\"\"\n monkeypatch.setenv(\"PATH\", \"\")\n monkeypatch.setattr(\n spack.cray_manifest, \"default_path\", os.path.join(\"a\", \"path\", \"that\", \"doesnt\", \"exist\")\n )\n external(\"find\")\n\n\ndef test_find_external_empty_default_manifest_dir(\n mutable_config, working_env, tmp_path: pathlib.Path, monkeypatch\n):\n \"\"\"The user runs 'spack external find'; the default path for storing\n manifest files exists but is empty. Ensure that the command does not\n fail.\n \"\"\"\n empty_manifest_dir = str(tmp_path / \"manifest_dir\")\n (tmp_path / \"manifest_dir\").mkdir()\n monkeypatch.setenv(\"PATH\", \"\")\n monkeypatch.setattr(spack.cray_manifest, \"default_path\", empty_manifest_dir)\n external(\"find\")\n\n\n@pytest.mark.not_on_windows(\"Can't chmod on Windows\")\n@pytest.mark.skipif(getuid() == 0, reason=\"user is root\")\ndef test_find_external_manifest_with_bad_permissions(\n mutable_config, working_env, tmp_path: pathlib.Path, monkeypatch\n):\n \"\"\"The user runs 'spack external find'; the default path for storing\n manifest files exists but with insufficient permissions. Check that\n the command does not fail.\n \"\"\"\n test_manifest_dir = str(tmp_path / \"manifest_dir\")\n (tmp_path / \"manifest_dir\").mkdir()\n test_manifest_file_path = os.path.join(test_manifest_dir, \"badperms.json\")\n touch(test_manifest_file_path)\n monkeypatch.setenv(\"PATH\", \"\")\n monkeypatch.setattr(spack.cray_manifest, \"default_path\", test_manifest_dir)\n try:\n os.chmod(test_manifest_file_path, 0)\n output = external(\"find\")\n assert \"insufficient permissions\" in output\n assert \"Skipping manifest and continuing\" in output\n finally:\n os.chmod(test_manifest_file_path, 0o700)\n\n\ndef test_find_external_manifest_failure(mutable_config, tmp_path: pathlib.Path, monkeypatch):\n \"\"\"The user runs 'spack external find'; the manifest parsing fails with\n some exception. Ensure that the command still succeeds (i.e. moves on\n to other external detection mechanisms).\n \"\"\"\n # First, create an empty manifest file (without a file to read, the\n # manifest parsing is skipped)\n test_manifest_dir = str(tmp_path / \"manifest_dir\")\n (tmp_path / \"manifest_dir\").mkdir()\n test_manifest_file_path = os.path.join(test_manifest_dir, \"test.json\")\n touch(test_manifest_file_path)\n\n def fail():\n raise Exception()\n\n monkeypatch.setattr(spack.cmd.external, \"_collect_and_consume_cray_manifest_files\", fail)\n monkeypatch.setenv(\"PATH\", \"\")\n output = external(\"find\")\n assert \"Skipping manifest and continuing\" in output\n\n\ndef test_find_external_merge(mutable_config):\n \"\"\"Checks that 'spack find external' doesn't overwrite an existing spec in packages.yaml.\"\"\"\n pkgs_cfg_init = {\n \"find-externals1\": {\n \"externals\": [{\"spec\": \"find-externals1@1.1\", \"prefix\": \"/preexisting-prefix\"}],\n \"buildable\": False,\n }\n }\n\n mutable_config.update_config(\"packages\", pkgs_cfg_init)\n entries = [\n Spec.from_detection(\"find-externals1@1.1\", external_path=\"/x/y1\"),\n Spec.from_detection(\"find-externals1@1.2\", external_path=\"/x/y2\"),\n ]\n pkg_to_entries = {\"find-externals1\": entries}\n scope = spack.config.default_modify_scope(\"packages\")\n spack.detection.update_configuration(pkg_to_entries, scope=scope, buildable=True)\n\n pkgs_cfg = spack.config.get(\"packages\")\n pkg_cfg = pkgs_cfg[\"find-externals1\"]\n pkg_externals = pkg_cfg[\"externals\"]\n\n assert {\"spec\": \"find-externals1@1.1\", \"prefix\": \"/preexisting-prefix\"} in pkg_externals\n assert {\"spec\": \"find-externals1@1.2\", \"prefix\": \"/x/y2\"} in pkg_externals\n\n\ndef test_list_detectable_packages(mutable_config):\n external(\"list\")\n assert external.returncode == 0\n\n\ndef test_overriding_prefix(mock_executable, mutable_config, monkeypatch):\n gcc_exe = mock_executable(\"gcc\", output=\"echo 4.2.1\")\n search_dir = gcc_exe.parent\n\n @classmethod\n def _determine_variants(cls, exes, version_str):\n return \"languages=c\", {\"prefix\": \"/opt/gcc/bin\", \"compilers\": {\"c\": exes[0]}}\n\n gcc_cls = spack.repo.PATH.get_pkg_class(\"gcc\")\n monkeypatch.setattr(gcc_cls, \"determine_variants\", _determine_variants)\n\n finder = spack.detection.path.ExecutablesFinder()\n detected_specs = finder.find(\n pkg_name=\"gcc\", initial_guess=[str(search_dir)], repository=spack.repo.PATH\n )\n\n assert len(detected_specs) == 1\n\n gcc = detected_specs[0]\n assert gcc.name == \"gcc\"\n assert gcc.external_path == os.path.sep + os.path.join(\"opt\", \"gcc\", \"bin\")\n\n\n@pytest.mark.not_on_windows(\"Fails spuriously on Windows\")\ndef test_new_entries_are_reported_correctly(mock_executable, mutable_config, monkeypatch):\n # Prepare an environment to detect a fake gcc\n gcc_exe = mock_executable(\"gcc\", output=\"echo 4.2.1\")\n prefix = os.path.dirname(gcc_exe)\n monkeypatch.setenv(\"PATH\", prefix)\n\n # The first run will find and add the external gcc\n output = external(\"find\", \"gcc\")\n assert \"The following specs have been\" in output\n\n # The second run should report that no new external\n # has been found\n output = external(\"find\", \"gcc\")\n assert \"No new external packages detected\" in output\n\n\n@pytest.mark.parametrize(\"command_args\", [(\"-t\", \"build-tools\"), (\"-t\", \"build-tools\", \"cmake\")])\n@pytest.mark.not_on_windows(\"the test uses bash scripts\")\ndef test_use_tags_for_detection(command_args, mock_executable, mutable_config, monkeypatch):\n versions = {\"cmake\": \"3.19.1\", \"openssl\": \"2.8.3\"}\n\n @classmethod\n def _determine_version(cls, exe):\n return versions[os.path.basename(exe)]\n\n cmake_cls = spack.repo.PATH.get_pkg_class(\"cmake\")\n monkeypatch.setattr(cmake_cls, \"determine_version\", _determine_version)\n\n # Prepare an environment to detect a fake cmake\n cmake_exe = mock_executable(\"cmake\", output=f\"echo cmake version {versions['cmake']}\")\n prefix = os.path.dirname(cmake_exe)\n monkeypatch.setenv(\"PATH\", prefix)\n\n openssl_exe = mock_executable(\"openssl\", output=f\"OpenSSL {versions['openssl']}\")\n prefix = os.path.dirname(openssl_exe)\n monkeypatch.setenv(\"PATH\", prefix)\n\n # Test that we detect specs\n output = external(\"find\", *command_args)\n assert \"The following specs have been\" in output\n assert \"cmake\" in output\n assert \"openssl\" not in output\n\n\n@pytest.mark.regression(\"38733\")\n@pytest.mark.not_on_windows(\"the test uses bash scripts\")\ndef test_failures_in_scanning_do_not_result_in_an_error(\n mock_executable, monkeypatch, mutable_config\n):\n \"\"\"Tests that scanning paths with wrong permissions, won't cause `external find` to error.\"\"\"\n cmake_exe1 = mock_executable(\n \"cmake\", output=\"echo cmake version 3.19.1\", subdir=(\"first\", \"bin\")\n )\n cmake_exe2 = mock_executable(\n \"cmake\", output=\"echo cmake version 3.23.3\", subdir=(\"second\", \"bin\")\n )\n\n @classmethod\n def _determine_version(cls, exe):\n name = pathlib.Path(exe).parent.parent.name\n if name == \"first\":\n return \"3.19.1\"\n elif name == \"second\":\n return \"3.23.3\"\n assert False, f\"Unexpected exe path {exe}\"\n\n cmake_cls = spack.repo.PATH.get_pkg_class(\"cmake\")\n monkeypatch.setattr(cmake_cls, \"determine_version\", _determine_version)\n monkeypatch.setenv(\"PATH\", f\"{cmake_exe1.parent}{os.pathsep}{cmake_exe2.parent}\")\n\n try:\n # Remove access from the first directory executable\n cmake_exe1.parent.chmod(0o600)\n output = external(\"find\", \"cmake\")\n finally:\n cmake_exe1.parent.chmod(0o700)\n\n assert external.returncode == 0\n assert \"The following specs have been\" in output\n assert \"cmake\" in output\n assert \"3.19.1\" in output\n assert \"3.23.3\" in output\n\n\ndef test_detect_virtuals(mock_executable, mutable_config, monkeypatch):\n \"\"\"Test whether external find --not-buildable sets virtuals as non-buildable (unless user\n config sets them to buildable)\"\"\"\n version = \"4.0.2\"\n\n @classmethod\n def _determine_version(cls, exe):\n return version\n\n cmake_cls = spack.repo.PATH.get_pkg_class(\"mpich\")\n monkeypatch.setattr(cmake_cls, \"determine_version\", _determine_version)\n\n mpich = mock_executable(\"mpichversion\", output=f\"echo MPICH Version: {version}\")\n prefix = os.path.dirname(mpich)\n external(\"find\", \"--path\", prefix, \"--not-buildable\", \"mpich\")\n\n # Check that mpich was correctly detected\n mpich = mutable_config.get(\"packages:mpich\")\n assert mpich[\"buildable\"] is False\n assert Spec(mpich[\"externals\"][0][\"spec\"]).satisfies(f\"mpich@{version}\")\n\n # Check that the virtual package mpi was marked as non-buildable\n assert mutable_config.get(\"packages:mpi:buildable\") is False\n\n # Delete the mpich entry, and set mpi explicitly to buildable\n mutable_config.set(\"packages:mpich\", {})\n mutable_config.set(\"packages:mpi:buildable\", True)\n\n # Run the detection again\n external(\"find\", \"--path\", prefix, \"--not-buildable\", \"mpich\")\n\n # Check that the mpi:buildable entry was not overwritten\n assert mutable_config.get(\"packages:mpi:buildable\") is True\n" }, { "path": "lib/spack/spack/test/cmd/fetch.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport pytest\n\nimport spack.environment as ev\nfrom spack.main import SpackCommand, SpackCommandError\n\n# everything here uses the mock_env_path\npytestmark = pytest.mark.usefixtures(\n \"mutable_mock_env_path\", \"mutable_config\", \"mutable_mock_repo\"\n)\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_fetch_in_env(mock_archive, mock_stage, mock_fetch, install_mockery):\n SpackCommand(\"env\")(\"create\", \"test\")\n with ev.read(\"test\"):\n SpackCommand(\"add\")(\"python\")\n with pytest.raises(SpackCommandError):\n SpackCommand(\"fetch\")()\n SpackCommand(\"concretize\")()\n SpackCommand(\"fetch\")()\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_fetch_single_spec(mock_archive, mock_stage, mock_fetch, install_mockery):\n SpackCommand(\"fetch\")(\"mpileaks\")\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_fetch_multiple_specs(mock_archive, mock_stage, mock_fetch, install_mockery):\n SpackCommand(\"fetch\")(\"mpileaks\", \"gcc@3.0\", \"python\")\n\n\ndef test_fetch_no_argument():\n with pytest.raises(SpackCommandError):\n SpackCommand(\"fetch\")()\n" }, { "path": "lib/spack/spack/test/cmd/find.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport io\nimport json\nimport os\nimport pathlib\n\nimport pytest\n\nimport spack.cmd\nimport spack.cmd.find\nimport spack.concretize\nimport spack.environment as ev\nimport spack.package_base\nimport spack.paths\nimport spack.repo\nimport spack.spec\nimport spack.store\nimport spack.user_environment as uenv\nfrom spack.enums import InstallRecordStatus\nfrom spack.llnl.util.filesystem import working_dir\nfrom spack.main import SpackCommand\nfrom spack.test.utilities import SpackCommandArgs\nfrom spack.util.pattern import Bunch\n\nfind = SpackCommand(\"find\")\nenv = SpackCommand(\"env\")\ninstall = SpackCommand(\"install\")\n\nbase32_alphabet = \"abcdefghijklmnopqrstuvwxyz234567\"\n\n\n@pytest.fixture(scope=\"module\")\ndef parser():\n \"\"\"Returns the parser for the module command\"\"\"\n prs = argparse.ArgumentParser()\n spack.cmd.find.setup_parser(prs)\n return prs\n\n\n@pytest.fixture()\ndef specs():\n s = []\n return s\n\n\n@pytest.fixture()\ndef mock_display(monkeypatch, specs):\n \"\"\"Monkeypatches the display function to return its first argument\"\"\"\n\n def display(x, *args, **kwargs):\n specs.extend(x)\n\n monkeypatch.setattr(spack.cmd, \"display_specs\", display)\n\n\ndef test_query_arguments():\n query_arguments = spack.cmd.find.query_arguments\n\n # Default arguments\n args = Bunch(\n only_missing=False,\n missing=False,\n only_deprecated=False,\n deprecated=False,\n unknown=False,\n explicit=False,\n implicit=False,\n start_date=\"2018-02-23\",\n end_date=None,\n install_tree=\"all\",\n )\n\n q_args = query_arguments(args)\n assert \"installed\" in q_args\n assert \"predicate_fn\" in q_args\n assert \"explicit\" in q_args\n assert q_args[\"installed\"] == InstallRecordStatus.INSTALLED\n assert q_args[\"predicate_fn\"] is None\n assert q_args[\"explicit\"] is None\n assert \"start_date\" in q_args\n assert \"end_date\" not in q_args\n assert q_args[\"install_tree\"] == \"all\"\n\n # Check that explicit works correctly\n args.explicit = True\n q_args = query_arguments(args)\n assert q_args[\"explicit\"] is True\n\n args.explicit = False\n args.implicit = True\n q_args = query_arguments(args)\n assert q_args[\"explicit\"] is False\n\n\n@pytest.mark.db\n@pytest.mark.usefixtures(\"database\", \"mock_display\")\ndef test_tag1(parser, specs):\n args = parser.parse_args([\"--tag\", \"tag1\"])\n spack.cmd.find.find(parser, args)\n\n assert len(specs) == 2\n assert \"mpich\" in [x.name for x in specs]\n assert \"mpich2\" in [x.name for x in specs]\n\n\n@pytest.mark.db\n@pytest.mark.usefixtures(\"database\", \"mock_display\")\ndef test_tag2(parser, specs):\n args = parser.parse_args([\"--tag\", \"tag2\"])\n spack.cmd.find.find(parser, args)\n\n assert len(specs) == 1\n assert \"mpich\" in [x.name for x in specs]\n\n\n@pytest.mark.db\n@pytest.mark.usefixtures(\"database\", \"mock_display\")\ndef test_tag2_tag3(parser, specs):\n args = parser.parse_args([\"--tag\", \"tag2\", \"--tag\", \"tag3\"])\n spack.cmd.find.find(parser, args)\n\n assert len(specs) == 0\n\n\n@pytest.mark.parametrize(\n \"args,with_namespace\", [([], False), ([\"--namespace\"], True), ([\"--namespaces\"], True)]\n)\n@pytest.mark.db\ndef test_namespaces_shown_correctly(args, with_namespace, database):\n \"\"\"Test that --namespace(s) works. Old syntax is --namespace\"\"\"\n assert (\"builtin_mock.zmpi\" in find(*args)) == with_namespace\n\n\n@pytest.mark.db\ndef test_find_cli_output_format(database, mock_tty_stdout):\n assert find(\"zmpi\").endswith(\n \"\"\"\\\nzmpi@1.0\n==> 1 installed package\n\"\"\"\n )\n\n\ndef _check_json_output(spec_list):\n assert len(spec_list) == 3\n assert all(spec[\"name\"] == \"mpileaks\" for spec in spec_list)\n assert all(spec[\"hash\"] for spec in spec_list)\n\n deps = [spec[\"dependencies\"] for spec in spec_list]\n assert sum([\"zmpi\" in [node[\"name\"] for d in deps for node in d]]) == 1\n assert sum([\"mpich\" in [node[\"name\"] for d in deps for node in d]]) == 1\n assert sum([\"mpich2\" in [node[\"name\"] for d in deps for node in d]]) == 1\n\n\ndef _check_json_output_deps(spec_list):\n assert len(spec_list) == 16\n\n names = [spec[\"name\"] for spec in spec_list]\n assert names.count(\"mpileaks\") == 3\n assert names.count(\"callpath\") == 3\n assert names.count(\"zmpi\") == 1\n assert names.count(\"mpich\") == 1\n assert names.count(\"mpich2\") == 1\n assert names.count(\"fake\") == 1\n assert names.count(\"dyninst\") == 1\n assert names.count(\"libdwarf\") == 1\n assert names.count(\"libelf\") == 1\n\n\n@pytest.mark.db\ndef test_find_json(database):\n output = find(\"--json\", \"mpileaks\")\n spec_list = json.loads(output)\n _check_json_output(spec_list)\n\n\n@pytest.mark.db\ndef test_find_json_deps(database):\n output = find(\"-d\", \"--json\", \"mpileaks\")\n spec_list = json.loads(output)\n _check_json_output_deps(spec_list)\n\n\n@pytest.mark.db\ndef test_display_json(database, capfd):\n specs = [\n spack.concretize.concretize_one(s)\n for s in [\"mpileaks ^zmpi\", \"mpileaks ^mpich\", \"mpileaks ^mpich2\"]\n ]\n\n spack.cmd.display_specs_as_json(specs)\n spec_list = json.loads(capfd.readouterr()[0])\n _check_json_output(spec_list)\n\n spack.cmd.display_specs_as_json(specs + specs + specs)\n spec_list = json.loads(capfd.readouterr()[0])\n _check_json_output(spec_list)\n\n\n@pytest.mark.db\ndef test_display_json_deps(database, capfd):\n specs = [\n spack.concretize.concretize_one(s)\n for s in [\"mpileaks ^zmpi\", \"mpileaks ^mpich\", \"mpileaks ^mpich2\"]\n ]\n\n spack.cmd.display_specs_as_json(specs, deps=True)\n spec_list = json.loads(capfd.readouterr()[0])\n _check_json_output_deps(spec_list)\n\n spack.cmd.display_specs_as_json(specs + specs + specs, deps=True)\n spec_list = json.loads(capfd.readouterr()[0])\n _check_json_output_deps(spec_list)\n\n\n@pytest.mark.regression(\"52219\")\ndef test_display_abstract_hash():\n spec = spack.spec.Spec(\"/foobar\")\n out = io.StringIO()\n\n spack.cmd.display_specs([spec], output=out) # errors on failure\n assert \"/foobar\" in out.getvalue()\n\n\n@pytest.mark.db\ndef test_find_format(database, config):\n output = find(\"--format\", \"{name}-{^mpi.name}\", \"mpileaks\")\n assert set(output.strip().split(\"\\n\")) == {\n \"mpileaks-zmpi\",\n \"mpileaks-mpich\",\n \"mpileaks-mpich2\",\n }\n\n output = find(\"--format\", \"{name}-{version}-{compiler.name}-{^mpi.name}\", \"mpileaks\")\n assert \"installed package\" not in output\n assert set(output.strip().split(\"\\n\")) == {\n \"mpileaks-2.3-gcc-zmpi\",\n \"mpileaks-2.3-gcc-mpich\",\n \"mpileaks-2.3-gcc-mpich2\",\n }\n\n output = find(\"--format\", \"{name}-{^mpi.name}-{hash:7}\", \"mpileaks\")\n elements = output.strip().split(\"\\n\")\n assert set(e[:-7] for e in elements) == {\n \"mpileaks-zmpi-\",\n \"mpileaks-mpich-\",\n \"mpileaks-mpich2-\",\n }\n\n # hashes are in base32\n for e in elements:\n for c in e[-7:]:\n assert c in base32_alphabet\n\n\n@pytest.mark.db\ndef test_find_format_deps(database, config):\n output = find(\"-d\", \"--format\", \"{name}-{version}\", \"mpileaks\", \"^zmpi\")\n assert (\n output\n == \"\"\"\\\nmpileaks-2.3\n callpath-1.0\n dyninst-8.2\n libdwarf-20130729\n libelf-0.8.13\n compiler-wrapper-1.0\n gcc-10.2.1\n gcc-runtime-10.2.1\n zmpi-1.0\n fake-1.0\n\n\"\"\"\n )\n\n\n@pytest.mark.db\ndef test_find_format_deps_paths(database, config):\n output = find(\"-dp\", \"--format\", \"{name}-{version}\", \"mpileaks\", \"^zmpi\")\n mpileaks = spack.concretize.concretize_one(\"mpileaks ^zmpi\")\n assert (\n output\n == f\"\"\"\\\nmpileaks-2.3 {mpileaks.prefix}\n callpath-1.0 {mpileaks[\"callpath\"].prefix}\n dyninst-8.2 {mpileaks[\"dyninst\"].prefix}\n libdwarf-20130729 {mpileaks[\"libdwarf\"].prefix}\n libelf-0.8.13 {mpileaks[\"libelf\"].prefix}\n compiler-wrapper-1.0 {mpileaks[\"compiler-wrapper\"].prefix}\n gcc-10.2.1 {mpileaks[\"gcc\"].prefix}\n gcc-runtime-10.2.1 {mpileaks[\"gcc-runtime\"].prefix}\n zmpi-1.0 {mpileaks[\"zmpi\"].prefix}\n fake-1.0 {mpileaks[\"fake\"].prefix}\n\n\"\"\"\n )\n\n\n@pytest.mark.db\ndef test_find_very_long(database, config):\n output = find(\"-L\", \"--no-groups\", \"mpileaks\")\n\n specs = [\n spack.concretize.concretize_one(s)\n for s in [\"mpileaks ^zmpi\", \"mpileaks ^mpich\", \"mpileaks ^mpich2\"]\n ]\n\n assert set(output.strip().split(\"\\n\")) == set(\n [(\"%s mpileaks@2.3\" % s.dag_hash()) for s in specs]\n )\n\n\n@pytest.mark.db\ndef test_find_not_found(database, config):\n output = find(\"foobarbaz\", fail_on_error=False)\n assert \"No package matches the query: foobarbaz\" in output\n assert find.returncode == 1\n\n\n@pytest.mark.db\ndef test_find_no_sections(database, config):\n output = find()\n assert \"-----------\" in output\n\n output = find(\"--no-groups\")\n assert \"-----------\" not in output\n assert \"==>\" not in output\n\n\n@pytest.mark.db\ndef test_find_command_basic_usage(database):\n output = find()\n assert \"mpileaks\" in output\n\n\n@pytest.mark.regression(\"9875\")\ndef test_find_prefix_in_env(\n mutable_mock_env_path, install_mockery, mock_fetch, mock_packages, mock_archive\n):\n \"\"\"Test `find` formats requiring concrete specs work in environments.\"\"\"\n env(\"create\", \"test\")\n with ev.read(\"test\"):\n install(\"--fake\", \"--add\", \"mpileaks\")\n find(\"-p\")\n find(\"-l\")\n find(\"-L\")\n # Would throw error on regression\n\n\ndef test_find_specs_include_concrete_env(\n mutable_mock_env_path, mutable_mock_repo, tmp_path: pathlib.Path\n):\n path = tmp_path / \"spack.yaml\"\n\n with working_dir(str(tmp_path)):\n with open(str(path), \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n specs:\n - mpileaks\n\"\"\"\n )\n env(\"create\", \"test1\", \"spack.yaml\")\n\n test1 = ev.read(\"test1\")\n test1.concretize()\n test1.write()\n\n with working_dir(str(tmp_path)):\n with open(str(path), \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n specs:\n - libelf\n\"\"\"\n )\n env(\"create\", \"test2\", \"spack.yaml\")\n\n test2 = ev.read(\"test2\")\n test2.concretize()\n test2.write()\n\n env(\"create\", \"--include-concrete\", \"test1\", \"--include-concrete\", \"test2\", \"combined_env\")\n\n with ev.read(\"combined_env\"):\n output = find()\n\n assert \"no root specs\" in output\n assert \"Included specs\" in output\n assert \"mpileaks\" in output\n assert \"libelf\" in output\n\n\ndef test_find_specs_nested_include_concrete_env(\n mutable_mock_env_path, mutable_mock_repo, tmp_path: pathlib.Path\n):\n path = tmp_path / \"spack.yaml\"\n\n with working_dir(str(tmp_path)):\n with open(str(path), \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n specs:\n - mpileaks\n\"\"\"\n )\n env(\"create\", \"test1\", \"spack.yaml\")\n\n test1 = ev.read(\"test1\")\n test1.concretize()\n test1.write()\n\n env(\"create\", \"--include-concrete\", \"test1\", \"test2\")\n test2 = ev.read(\"test2\")\n test2.add(\"libelf\")\n test2.concretize()\n test2.write()\n\n env(\"create\", \"--include-concrete\", \"test2\", \"test3\")\n\n with ev.read(\"test3\"):\n output = find()\n\n assert \"no root specs\" in output\n assert \"Included specs\" in output\n assert \"mpileaks\" in output\n assert \"libelf\" in output\n\n\ndef test_find_loaded(database, working_env):\n output = find(\"--loaded\", \"--group\")\n assert output == \"\"\n\n os.environ[uenv.spack_loaded_hashes_var] = os.pathsep.join(\n [x.dag_hash() for x in spack.store.STORE.db.query()]\n )\n output = find(\"--loaded\")\n expected = find()\n assert output == expected\n\n\n@pytest.mark.regression(\"37712\")\ndef test_environment_with_version_range_in_compiler_doesnt_fail(\n tmp_path: pathlib.Path, mock_packages\n):\n \"\"\"Tests that having an active environment with a root spec containing a compiler constrained\n by a version range (i.e. @X.Y rather the single version than @=X.Y) doesn't result in an error\n when invoking \"spack find\".\n \"\"\"\n test_environment = ev.create_in_dir(tmp_path)\n test_environment.add(\"zlib %gcc@12.1.0\")\n test_environment.write()\n\n with test_environment:\n output = find()\n assert \"zlib\" in output\n\n\n# a0 d0\n# / \\ / \\\n# b0 c0 e0\n\n\n@pytest.fixture\ndef test_repo(mock_stage):\n with spack.repo.use_repositories(\n os.path.join(spack.paths.test_repos_path, \"spack_repo\", \"find\")\n ) as mock_packages_repo:\n yield mock_packages_repo\n\n\ndef test_find_concretized_not_installed(\n mutable_mock_env_path, install_mockery, mock_fetch, test_repo, mock_archive\n):\n \"\"\"Test queries against installs of specs against fake repo.\n\n Given A, B, C, D, E, create an environment and install A.\n Add and concretize (but do not install) D.\n Test a few queries after force uninstalling a dependency of A (but not\n A itself).\n \"\"\"\n add = SpackCommand(\"add\")\n concretize = SpackCommand(\"concretize\")\n uninstall = SpackCommand(\"uninstall\")\n\n def _query(_e, *args):\n return spack.cmd.find._find_query(SpackCommandArgs(\"find\")(*args), _e)\n\n def _nresults(_qresult):\n return len(_qresult[0]), len(_qresult[1])\n\n env(\"create\", \"test\")\n with ev.read(\"test\") as e:\n install(\"--fake\", \"--add\", \"a0\")\n\n assert _nresults(_query(e)) == (3, 0)\n assert _nresults(_query(e, \"--explicit\")) == (1, 0)\n\n add(\"d0\")\n concretize(\"--reuse\")\n\n # At this point d0 should use existing c0, but d/e\n # are not installed in the env\n\n # --explicit, --deprecated, --start-date, etc. are all\n # filters on records, and therefore don't apply to\n # concretized-but-not-installed results\n assert _nresults(_query(e, \"--explicit\")) == (1, 2)\n\n assert _nresults(_query(e)) == (3, 2)\n assert _nresults(_query(e, \"-c\", \"d0\")) == (0, 1)\n\n uninstall(\"-f\", \"-y\", \"b0\")\n\n # b0 is now missing (it is not installed, but has an\n # installed parent)\n\n assert _nresults(_query(e)) == (2, 3)\n # b0 is \"double-counted\" here: it meets the --missing\n # criteria, and also now qualifies as a\n # concretized-but-not-installed spec\n assert _nresults(_query(e, \"--missing\")) == (3, 3)\n assert _nresults(_query(e, \"--only-missing\")) == (1, 3)\n\n # Tags are not attached to install records, so they\n # can modify the concretized-but-not-installed results\n\n assert _nresults(_query(e, \"--tag=tag0\")) == (1, 0)\n assert _nresults(_query(e, \"--tag=tag1\")) == (1, 1)\n assert _nresults(_query(e, \"--tag=tag2\")) == (0, 1)\n\n\n@pytest.mark.usefixtures(\"install_mockery\", \"mock_fetch\")\ndef test_find_based_on_commit_sha(mock_git_version_info, monkeypatch):\n repo_path, filename, commits = mock_git_version_info\n file_url = pathlib.Path(repo_path).as_uri()\n\n monkeypatch.setattr(spack.package_base.PackageBase, \"git\", file_url, raising=False)\n\n install(\"--fake\", f\"git-test-commit commit={commits[0]}\")\n output = find(f\"commit={commits[0]}\")\n assert \"git-test-commit\" in output\n\n\n@pytest.mark.usefixtures(\"mock_packages\")\n@pytest.mark.parametrize(\n \"spack_yaml,expected,not_expected\",\n [\n (\n \"\"\"\nspack:\n specs:\n - mpileaks\n - group: extras\n specs:\n - libelf\n\"\"\",\n [\n \"2 root specs\",\n # Group names\n \"extras\",\n \"default\",\n # root specs\n \"mpileaks\",\n \"libelf\",\n ],\n [],\n ),\n (\n \"\"\"\nspack:\n specs:\n - group: tools\n specs:\n - libelf\n\"\"\",\n [\"1 root spec\", \"tools\", \"libelf\"],\n [\"1 root specs\", \"default\"],\n ),\n ],\n)\ndef test_find_env_with_groups(spack_yaml, expected, not_expected, tmp_path: pathlib.Path):\n \"\"\"Tests that the output of spack find contains expected matches when using an\n environment with groups.\n \"\"\"\n (tmp_path / \"spack.yaml\").write_text(spack_yaml)\n with ev.Environment(tmp_path):\n output = find()\n\n assert all(x in output for x in expected)\n assert all(x not in output for x in not_expected)\n" }, { "path": "lib/spack/spack/test/cmd/gc.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport pathlib\n\nimport pytest\n\nimport spack.concretize\nimport spack.deptypes as dt\nimport spack.environment as ev\nimport spack.main\nimport spack.spec\nimport spack.traverse\nfrom spack.installer import PackageInstaller\n\ngc = spack.main.SpackCommand(\"gc\")\nadd = spack.main.SpackCommand(\"add\")\ninstall = spack.main.SpackCommand(\"install\")\n\n\n@pytest.mark.db\ndef test_gc_without_build_dependency(mutable_database):\n assert \"There are no unused specs.\" in gc(\"-yb\")\n # 'gcc' is a pure build dependency in the DB\n assert \"There are no unused specs.\" not in gc(\"-y\")\n\n\n@pytest.mark.db\ndef test_gc_with_build_dependency(mutable_database):\n s = spack.concretize.concretize_one(\"simple-inheritance\")\n PackageInstaller([s.package], explicit=True, fake=True).install()\n\n assert \"There are no unused specs.\" in gc(\"-yb\")\n assert \"Successfully uninstalled cmake\" in gc(\"-y\")\n assert \"There are no unused specs.\" in gc(\"-y\")\n\n\n@pytest.mark.db\ndef test_gc_with_constraints(mutable_database):\n s_cmake1 = spack.concretize.concretize_one(\"simple-inheritance ^cmake@3.4.3\")\n s_cmake2 = spack.concretize.concretize_one(\"simple-inheritance ^cmake@3.23.1\")\n PackageInstaller([s_cmake1.package], explicit=True, fake=True).install()\n PackageInstaller([s_cmake2.package], explicit=True, fake=True).install()\n\n assert \"There are no unused specs.\" in gc(\"python\")\n\n assert \"Successfully uninstalled cmake@3.4.3\" in gc(\"-y\", \"cmake@3.4.3\")\n assert \"There are no unused specs.\" in gc(\"-y\", \"cmake@3.4.3\")\n\n assert \"Successfully uninstalled cmake\" in gc(\"-y\", \"cmake@3.23.1\")\n assert \"There are no unused specs.\" in gc(\"-y\", \"cmake\")\n\n\n@pytest.mark.db\ndef test_gc_with_environment(mutable_database, mutable_mock_env_path):\n s = spack.concretize.concretize_one(\"simple-inheritance\")\n PackageInstaller([s.package], explicit=True, fake=True).install()\n\n e = ev.create(\"test_gc\")\n with e:\n add(\"cmake\")\n install()\n assert mutable_database.query_local(\"cmake\")\n output = gc(\"-by\")\n assert \"Restricting garbage collection\" in output\n assert \"There are no unused specs\" in output\n\n\n@pytest.mark.db\ndef test_gc_with_build_dependency_in_environment(mutable_database, mutable_mock_env_path):\n s = spack.concretize.concretize_one(\"simple-inheritance\")\n PackageInstaller([s.package], explicit=True, fake=True).install()\n\n e = ev.create(\"test_gc\")\n with e:\n add(\"simple-inheritance\")\n install()\n assert mutable_database.query_local(\"simple-inheritance\")\n output = gc(\"-yb\")\n assert \"Restricting garbage collection\" in output\n assert \"There are no unused specs\" in output\n\n with e:\n assert mutable_database.query_local(\"simple-inheritance\")\n fst = gc(\"-y\")\n assert \"Restricting garbage collection\" in fst\n assert \"Successfully uninstalled cmake\" in fst\n snd = gc(\"-y\")\n assert \"Restricting garbage collection\" in snd\n assert \"There are no unused specs\" in snd\n\n\n@pytest.mark.db\ndef test_gc_except_any_environments(mutable_database, mutable_mock_env_path):\n \"\"\"Tests whether the garbage collector can remove all specs except those still needed in some\n environment (needed in the sense of roots + link/run deps).\"\"\"\n assert mutable_database.query_local(\"zmpi\")\n\n e = ev.create(\"test_gc\")\n e.add(\"simple-inheritance\")\n e.concretize()\n e.install_all(fake=True)\n e.write()\n\n assert mutable_database.query_local(\"simple-inheritance\")\n assert not e.all_matching_specs(spack.spec.Spec(\"zmpi\"))\n\n output = gc(\"-yE\")\n assert \"Restricting garbage collection\" not in output\n assert \"Successfully uninstalled zmpi\" in output\n assert not mutable_database.query_local(\"zmpi\")\n\n # All runtime specs in this env should still be installed.\n assert all(\n s.installed\n for s in spack.traverse.traverse_nodes(e.concrete_roots(), deptype=dt.LINK | dt.RUN)\n )\n\n\n@pytest.mark.db\ndef test_gc_except_specific_environments(mutable_database, mutable_mock_env_path):\n s = spack.concretize.concretize_one(\"simple-inheritance\")\n PackageInstaller([s.package], explicit=True, fake=True).install()\n\n assert mutable_database.query_local(\"zmpi\")\n\n e = ev.create(\"test_gc\")\n with e:\n add(\"simple-inheritance\")\n install()\n assert mutable_database.query_local(\"simple-inheritance\")\n\n output = gc(\"-ye\", \"test_gc\")\n assert \"Restricting garbage collection\" not in output\n assert \"Successfully uninstalled zmpi\" in output\n assert not mutable_database.query_local(\"zmpi\")\n\n\n@pytest.mark.db\ndef test_gc_except_nonexisting_dir_env(\n mutable_database, mutable_mock_env_path, tmp_path: pathlib.Path\n):\n output = gc(\"-ye\", str(tmp_path), fail_on_error=False)\n assert \"No such environment\" in output\n assert gc.returncode == 1\n\n\n@pytest.mark.db\ndef test_gc_except_specific_dir_env(\n mutable_database, mutable_mock_env_path, tmp_path: pathlib.Path\n):\n s = spack.concretize.concretize_one(\"simple-inheritance\")\n PackageInstaller([s.package], explicit=True, fake=True).install()\n\n assert mutable_database.query_local(\"zmpi\")\n\n e = ev.create_in_dir(str(tmp_path))\n with e:\n add(\"simple-inheritance\")\n install()\n assert mutable_database.query_local(\"simple-inheritance\")\n\n output = gc(\"-ye\", str(tmp_path))\n assert \"Restricting garbage collection\" not in output\n assert \"Successfully uninstalled zmpi\" in output\n assert not mutable_database.query_local(\"zmpi\")\n\n\n@pytest.fixture\ndef mock_installed_environment(mutable_database, mutable_mock_env_path):\n\n def _create_environment(name, spack_yaml):\n tmp_env = ev.create(name)\n spack_yaml_path = pathlib.Path(tmp_env.path) / \"spack.yaml\"\n spack_yaml_path.write_text(spack_yaml)\n e = ev.read(name)\n with ev.read(name):\n e.concretize()\n e.install_all(fake=True)\n e.write()\n return e\n\n return _create_environment\n\n\n@pytest.mark.db\n@pytest.mark.parametrize(\n \"explicit,expected_explicit,expected_implicit\",\n [\n (True, [\"gcc@14.0.1\", \"openblas\", \"dyninst\"], []),\n (False, [\"dyninst\"], [\"gcc@14.0.1\", \"openblas\"]),\n ],\n)\ndef test_gc_with_explicit_groups(\n explicit, expected_explicit, expected_implicit, mutable_database, mock_installed_environment\n):\n \"\"\"Tests the semantics of the \"explicit\" attribute of environment groups\"\"\"\n e = mock_installed_environment(\n \"test_gc_explicit\",\n f\"\"\"\nspack:\n config:\n installer: new\n specs:\n - group: base\n explicit: {explicit}\n specs:\n - gcc@14.0.1\n - openblas\n - group: apps\n needs: [base]\n specs:\n - dyninst %c=gcc@14.0.1\n\"\"\",\n )\n\n # Test DB status\n for query in expected_explicit:\n assert mutable_database.query_local(query, explicit=True)\n\n for query in expected_implicit:\n assert mutable_database.query_local(query, explicit=False)\n\n with e:\n output = gc(\"-y\")\n\n # Test gc behavior\n for query in expected_implicit:\n assert f\"Successfully uninstalled {query}\" in output\n\n for query in expected_explicit:\n assert f\"Successfully uninstalled {query}\" not in output\n" }, { "path": "lib/spack/spack/test/cmd/gpg.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport pathlib\n\nimport pytest\n\nimport spack.binary_distribution\nimport spack.llnl.util.filesystem as fs\nimport spack.util.gpg\nfrom spack.main import SpackCommand\nfrom spack.paths import mock_gpg_data_path, mock_gpg_keys_path\nfrom spack.util.executable import ProcessError\n\n#: spack command used by tests below\ngpg = SpackCommand(\"gpg\")\nbootstrap = SpackCommand(\"bootstrap\")\nmirror = SpackCommand(\"mirror\")\n\npytestmark = pytest.mark.not_on_windows(\"does not run on windows\")\n\n\n# test gpg command detection\n@pytest.mark.parametrize(\n \"cmd_name,version\",\n [\n (\"gpg\", \"undetectable\"), # undetectable version\n (\"gpg\", \"gpg (GnuPG) 1.3.4\"), # insufficient version\n (\"gpg\", \"gpg (GnuPG) 2.2.19\"), # sufficient version\n (\"gpg2\", \"gpg (GnuPG) 2.2.19\"), # gpg2 command\n ],\n)\ndef test_find_gpg(cmd_name, version, tmp_path: pathlib.Path, mock_gnupghome, monkeypatch):\n TEMPLATE = '#!/bin/sh\\necho \"{version}\"\\n'\n\n with fs.working_dir(str(tmp_path)):\n for fname in (cmd_name, \"gpgconf\"):\n with open(fname, \"w\", encoding=\"utf-8\") as f:\n f.write(TEMPLATE.format(version=version))\n fs.set_executable(fname)\n\n monkeypatch.setenv(\"PATH\", str(tmp_path))\n if version == \"undetectable\" or version.endswith(\"1.3.4\"):\n with pytest.raises(spack.util.gpg.SpackGPGError):\n spack.util.gpg.init(force=True)\n else:\n spack.util.gpg.init(force=True)\n assert spack.util.gpg.GPG is not None\n assert spack.util.gpg.GPGCONF is not None\n\n\ndef test_no_gpg_in_path(tmp_path: pathlib.Path, mock_gnupghome, monkeypatch, mutable_config):\n monkeypatch.setenv(\"PATH\", str(tmp_path))\n bootstrap(\"disable\")\n with pytest.raises(RuntimeError):\n spack.util.gpg.init(force=True)\n\n\n@pytest.mark.maybeslow\ndef test_gpg(tmp_path: pathlib.Path, mutable_config, mock_gnupghome):\n # Verify a file with an empty keyring.\n with pytest.raises(ProcessError):\n gpg(\"verify\", os.path.join(mock_gpg_data_path, \"content.txt\"))\n\n # Import the default key.\n gpg(\"init\", \"--from\", mock_gpg_keys_path)\n\n # List the keys.\n # TODO: Test the output here.\n gpg(\"list\", \"--trusted\")\n gpg(\"list\", \"--signing\")\n\n # Verify the file now that the key has been trusted.\n gpg(\"verify\", os.path.join(mock_gpg_data_path, \"content.txt\"))\n\n # Untrust the default key.\n gpg(\"untrust\", \"Spack testing\")\n\n # Now that the key is untrusted, verification should fail.\n with pytest.raises(ProcessError):\n gpg(\"verify\", os.path.join(mock_gpg_data_path, \"content.txt\"))\n\n # Create a file to test signing.\n test_path = tmp_path / \"to-sign.txt\"\n with open(str(test_path), \"w+\", encoding=\"utf-8\") as fout:\n fout.write(\"Test content for signing.\\n\")\n\n # Signing without a private key should fail.\n with pytest.raises(RuntimeError) as exc_info:\n gpg(\"sign\", str(test_path))\n assert exc_info.value.args[0] == \"no signing keys are available\"\n\n # Create a key for use in the tests.\n keypath = tmp_path / \"testing-1.key\"\n gpg(\n \"create\",\n \"--comment\",\n \"Spack testing key\",\n \"--export\",\n str(keypath),\n \"Spack testing 1\",\n \"spack@googlegroups.com\",\n )\n keyfp = spack.util.gpg.signing_keys()[0]\n\n # List the keys.\n # TODO: Test the output here.\n gpg(\"list\")\n gpg(\"list\", \"--trusted\")\n gpg(\"list\", \"--signing\")\n\n # Signing with the default (only) key.\n gpg(\"sign\", str(test_path))\n\n # Verify the file we just verified.\n gpg(\"verify\", str(test_path))\n\n # Export the key for future use.\n export_path = tmp_path / \"export.testing.key\"\n gpg(\"export\", str(export_path))\n\n # Test exporting the private key\n private_export_path = tmp_path / \"export-secret.testing.key\"\n gpg(\"export\", \"--secret\", str(private_export_path))\n\n # Ensure we exported the right content!\n with open(str(private_export_path), \"r\", encoding=\"utf-8\") as fd:\n content = fd.read()\n assert \"BEGIN PGP PRIVATE KEY BLOCK\" in content\n\n # and for the public key\n with open(str(export_path), \"r\", encoding=\"utf-8\") as fd:\n content = fd.read()\n assert \"BEGIN PGP PUBLIC KEY BLOCK\" in content\n\n # Create a second key for use in the tests.\n gpg(\"create\", \"--comment\", \"Spack testing key\", \"Spack testing 2\", \"spack@googlegroups.com\")\n\n # List the keys.\n # TODO: Test the output here.\n gpg(\"list\", \"--trusted\")\n gpg(\"list\", \"--signing\")\n\n test_path = tmp_path / \"to-sign-2.txt\"\n with open(str(test_path), \"w+\", encoding=\"utf-8\") as fout:\n fout.write(\"Test content for signing.\\n\")\n\n # Signing with multiple signing keys is ambiguous.\n with pytest.raises(RuntimeError) as exc_info:\n gpg(\"sign\", str(test_path))\n assert exc_info.value.args[0] == \"multiple signing keys are available; please choose one\"\n\n # Signing with a specified key.\n gpg(\"sign\", \"--key\", keyfp, str(test_path))\n\n # Untrusting signing keys needs a flag.\n with pytest.raises(ProcessError):\n gpg(\"untrust\", \"Spack testing 1\")\n\n # Untrust the key we created.\n gpg(\"untrust\", \"--signing\", keyfp)\n\n # Verification should now fail.\n with pytest.raises(ProcessError):\n gpg(\"verify\", str(test_path))\n\n # Trust the exported key.\n gpg(\"trust\", str(export_path))\n\n # Verification should now succeed again.\n gpg(\"verify\", str(test_path))\n\n relative_keys_path = spack.binary_distribution.buildcache_relative_keys_path()\n\n # Publish the keys using a directory path\n test_path = tmp_path / \"dir_cache\"\n os.makedirs(f\"{test_path}\")\n gpg(\"publish\", \"--rebuild-index\", \"-d\", str(test_path))\n assert os.path.exists(f\"{test_path}/{relative_keys_path}/keys.manifest.json\")\n\n # Publish the keys using a mirror url\n test_path = tmp_path / \"url_cache\"\n os.makedirs(f\"{test_path}\")\n test_url = test_path.as_uri()\n gpg(\"publish\", \"--rebuild-index\", \"--mirror-url\", test_url)\n assert os.path.exists(f\"{test_path}/{relative_keys_path}/keys.manifest.json\")\n\n # Publish the keys using a mirror name\n test_path = tmp_path / \"named_cache\"\n os.makedirs(f\"{test_path}\")\n mirror_url = test_path.as_uri()\n mirror(\"add\", \"gpg\", mirror_url)\n gpg(\"publish\", \"--rebuild-index\", \"-m\", \"gpg\")\n assert os.path.exists(f\"{test_path}/{relative_keys_path}/keys.manifest.json\")\n" }, { "path": "lib/spack/spack/test/cmd/graph.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport pytest\n\nfrom spack.main import SpackCommand, SpackCommandError\n\ngraph = SpackCommand(\"graph\")\n\n\n@pytest.mark.db\n@pytest.mark.usefixtures(\"mock_packages\", \"database\")\ndef test_graph_ascii():\n \"\"\"Tests spack graph --ascii\"\"\"\n graph(\"--ascii\", \"dt-diamond\")\n\n\n@pytest.mark.db\n@pytest.mark.usefixtures(\"mock_packages\", \"database\")\ndef test_graph_dot():\n \"\"\"Tests spack graph --dot\"\"\"\n graph(\"--dot\", \"dt-diamond\")\n\n\n@pytest.mark.db\n@pytest.mark.usefixtures(\"mock_packages\", \"database\")\ndef test_graph_static():\n \"\"\"Tests spack graph --static\"\"\"\n graph(\"--static\", \"dt-diamond\")\n\n\n@pytest.mark.db\n@pytest.mark.usefixtures(\"mock_packages\", \"database\")\ndef test_graph_installed():\n \"\"\"Tests spack graph --installed\"\"\"\n\n graph(\"--installed\")\n\n with pytest.raises(SpackCommandError):\n graph(\"--installed\", \"dt-diamond\")\n\n\n@pytest.mark.db\n@pytest.mark.usefixtures(\"mock_packages\", \"database\")\ndef test_graph_deptype():\n \"\"\"Tests spack graph --deptype\"\"\"\n graph(\"--deptype\", \"all\", \"dt-diamond\")\n\n\ndef test_graph_no_specs():\n \"\"\"Tests spack graph with no arguments\"\"\"\n\n with pytest.raises(SpackCommandError):\n graph()\n" }, { "path": "lib/spack/spack/test/cmd/help.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nfrom spack.main import SpackCommand\n\n\ndef test_reuse_after_help():\n \"\"\"Test `spack help` can be called twice with the same SpackCommand.\"\"\"\n help_cmd = SpackCommand(\"help\")\n help_cmd()\n help_cmd()\n\n\ndef test_help():\n \"\"\"Sanity check the help command to make sure it works.\"\"\"\n help_cmd = SpackCommand(\"help\")\n out = help_cmd()\n assert \"Common spack commands:\" in out\n assert \"Options:\" in out\n\n\ndef test_help_all():\n \"\"\"Test the spack help --all flag\"\"\"\n help_cmd = SpackCommand(\"help\")\n out = help_cmd(\"--all\")\n assert \"Commands:\" in out\n assert \"Options:\" in out\n\n\ndef test_help_spec():\n \"\"\"Test the spack help --spec flag\"\"\"\n help_cmd = SpackCommand(\"help\")\n out = help_cmd(\"--spec\")\n assert \"spec expression syntax:\" in out\n\n\ndef test_help_subcommand():\n \"\"\"Test the spack help subcommand argument\"\"\"\n help_cmd = SpackCommand(\"help\")\n out = help_cmd(\"help\")\n assert \"get help on spack and its commands\" in out\n" }, { "path": "lib/spack/spack/test/cmd/info.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport re\n\nimport pytest\n\nfrom spack.main import SpackCommand, SpackCommandError\nfrom spack.repo import UnknownPackageError\n\npytestmark = [pytest.mark.usefixtures(\"mock_packages\")]\n\ninfo = SpackCommand(\"info\")\n\n\ndef test_package_suggestion():\n with pytest.raises(UnknownPackageError) as exc_info:\n info(\"vtk\")\n assert \"Did you mean one of the following packages?\" in str(exc_info.value)\n\n\ndef test_deprecated_option_warns():\n info(\"--variants-by-name\", \"vtk-m\")\n assert \"--variants-by-name is deprecated\" in info.output\n\n\n# no specs, more than one spec\n@pytest.mark.parametrize(\"args\", [[], [\"vtk-m\", \"zmpi\"]])\ndef test_info_failures(args):\n with pytest.raises(SpackCommandError):\n info(*args)\n\n\ndef test_info_noversion():\n \"\"\"Check that a mock package with no versions outputs None.\"\"\"\n output = info(\"noversion\")\n\n assert \"Preferred\\n None\" not in output\n assert \"Safe\\n None\" not in output\n assert \"Deprecated\\n None\" not in output\n\n\n@pytest.mark.parametrize(\n \"pkg_query,expected\", [(\"zlib\", \"False\"), (\"find-externals1\", \"True (version)\")]\n)\ndef test_is_externally_detectable(pkg_query, expected):\n output = info(\"--detectable\", pkg_query)\n assert f\"Externally Detectable:\\n {expected}\" in output\n\n\n@pytest.mark.parametrize(\n \"pkg_query\",\n [\"vtk-m\", \"gcc\"], # This should ensure --test's c_names processing loop covered\n)\n@pytest.mark.parametrize(\"extra_args\", [[], [\"--by-name\"]])\ndef test_info_fields(pkg_query, extra_args):\n expected_fields = (\n \"Description:\",\n \"Homepage:\",\n \"Externally Detectable:\",\n \"Safe versions:\",\n \"Variants:\",\n \"Installation Phases:\",\n \"Virtual Packages:\",\n \"Tags:\",\n \"Licenses:\",\n )\n\n output = info(\"--all\", *extra_args, pkg_query)\n assert all(field in output for field in expected_fields)\n\n\n@pytest.mark.parametrize(\n \"args,in_output,not_in_output\",\n [\n # no variants\n ([\"package-base-extendee\"], [r\"Variants:\\n\\s*None\"], []),\n # test that long lines wrap around\n (\n [\"long-boost-dependency+longdep\"],\n [\n r\"boost\\+atomic\\+chrono\\+date_time\\+filesystem\\+graph\\+iostreams\\+locale\\n\"\n r\"\\s*build, link\"\n ],\n [],\n ),\n (\n [\"long-boost-dependency~longdep\"],\n [],\n [\n r\"boost\\+atomic\\+chrono\\+date_time\\+filesystem\\+graph\\+iostreams\\+locale\\n\"\n r\"\\s*build, link\"\n ],\n ),\n # conditional licenses change output\n ([\"licenses-1 +foo\"], [\"MIT\"], [\"Apache-2.0\"]),\n ([\"licenses-1 ~foo\"], [\"Apache-2.0\"], [\"MIT\"]),\n # filtering bowtie versions\n ([\"bowtie\"], [\"1.4.0\", \"1.3.0\", \"1.2.2\", \"1.2.0\"], []),\n ([\"bowtie@1.2:\"], [\"1.4.0\", \"1.3.0\", \"1.2.2\", \"1.2.0\"], []),\n ([\"bowtie@1.3:\"], [\"1.4.0\", \"1.3.0\"], [\"1.2.2\", \"1.2.0\"]),\n ([\"bowtie@1.2\"], [\"1.2.2\", \"1.2.0\"], [\"1.3.0\"]), # 1.4.0 still shown as preferred\n # many dependencies with suggestion to filter\n (\n [\"many-conditional-deps\"],\n [\"consider this for a simpler view:\\n spack info many-conditional-deps~cuda~rocm\"],\n [],\n ),\n (\n [\"many-conditional-deps ~cuda\"],\n [\"consider this for a simpler view:\\n spack info many-conditional-deps~cuda~rocm\"],\n [],\n ),\n (\n [\"many-conditional-deps ~rocm\"],\n [\"consider this for a simpler view:\\n spack info many-conditional-deps~cuda~rocm\"],\n [],\n ),\n ([\"many-conditional-deps ~cuda ~rocm\"], [], [\"consider this for a simpler view:\"]),\n # Ensure spack info knows that build_system is a single value variant\n (\n [\"dual-cmake-autotools\"],\n [r\"when\\s*build_system=cmake\", r\"when\\s*build_system=autotools\"],\n [],\n ),\n (\n [\"dual-cmake-autotools build_system=cmake\"],\n [r\"when\\s*build_system=cmake\"],\n [r\"when\\s*build_system=autotools\"],\n ),\n # Ensure that gemerator=make implies build_system=cmake and therefore no autotools\n (\n [\"dual-cmake-autotools generator=make\"],\n [r\"when\\s*build_system=cmake\"],\n [r\"when\\s*build_system=autotools\"],\n ),\n (\n [\"optional-dep-test\"],\n [\n r\"when \\^pkg-g\",\n r\"when \\%intel\",\n r\"when \\%intel\\@64\\.1\",\n r\"when \\%clang@34\\:40\",\n r\"when \\^pkg\\-f\",\n ],\n [],\n ),\n ],\n)\n@pytest.mark.parametrize(\"by_name\", [True, False])\ndef test_info_output(by_name, args, in_output, not_in_output, monkeypatch):\n monkeypatch.setenv(\"COLUMNS\", \"80\")\n by_name_arg = [\"--by-name\"] if by_name else [\"--by-when\"]\n output = info(*(by_name_arg + args))\n\n for io in in_output:\n assert re.search(io, output)\n for nio in not_in_output:\n assert not re.search(nio, output)\n" }, { "path": "lib/spack/spack/test/cmd/init_py_functions.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport pathlib\n\nimport pytest\n\nimport spack.config\nimport spack.environment as ev\nimport spack.error\nimport spack.solver.asp as asp\nimport spack.store\nfrom spack.cmd import (\n CommandNameError,\n PythonNameError,\n cmd_name,\n matching_specs_from_env,\n parse_specs,\n python_name,\n require_cmd_name,\n require_python_name,\n)\n\n\ndef test_require_python_name():\n \"\"\"Python module names should not contain dashes---ensure that\n require_python_name() raises the appropriate exception if one is\n detected.\n \"\"\"\n require_python_name(\"okey_dokey\")\n with pytest.raises(PythonNameError):\n require_python_name(\"okey-dokey\")\n require_python_name(python_name(\"okey-dokey\"))\n\n\ndef test_require_cmd_name():\n \"\"\"By convention, Spack command names should contain dashes rather than\n underscores---ensure that require_cmd_name() raises the appropriate\n exception if underscores are detected.\n \"\"\"\n require_cmd_name(\"okey-dokey\")\n with pytest.raises(CommandNameError):\n require_cmd_name(\"okey_dokey\")\n require_cmd_name(cmd_name(\"okey_dokey\"))\n\n\n@pytest.mark.parametrize(\n \"unify,spec_strs,error\",\n [\n # single spec\n (True, [\"zmpi\"], None),\n (False, [\"mpileaks\"], None),\n # multiple specs, some from hash some from file\n (True, [\"zmpi\", \"mpileaks^zmpi\", \"libelf\"], None),\n (True, [\"mpileaks^zmpi\", \"mpileaks^mpich\", \"libelf\"], spack.error.SpecError),\n (False, [\"mpileaks^zmpi\", \"mpileaks^mpich\", \"libelf\"], None),\n ],\n)\ndef test_special_cases_concretization_parse_specs(\n unify, spec_strs, error, monkeypatch, mutable_config, mutable_database, tmp_path: pathlib.Path\n):\n \"\"\"Test that special cases in parse_specs(concretize=True) bypass solver\"\"\"\n\n # monkeypatch to ensure we do not call the actual concretizer\n def _fail(*args, **kwargs):\n assert False\n\n monkeypatch.setattr(asp.SpackSolverSetup, \"setup\", _fail)\n\n spack.config.set(\"concretizer:unify\", unify)\n\n args = [f\"/{spack.store.STORE.db.query(s)[0].dag_hash()}\" for s in spec_strs]\n if len(args) > 1:\n # We convert the last one to a specfile input\n filename = tmp_path / \"spec.json\"\n spec = parse_specs(args[-1], concretize=True)[0]\n with open(filename, \"w\", encoding=\"utf-8\") as f:\n spec.to_json(f)\n args[-1] = str(filename)\n\n if error:\n with pytest.raises(error):\n parse_specs(args, concretize=True)\n else:\n # assertion error from monkeypatch above if test fails\n parse_specs(args, concretize=True)\n\n\n@pytest.mark.parametrize(\n \"unify,spec_strs,error\",\n [\n # single spec\n (True, [\"zmpi\"], None),\n (False, [\"mpileaks\"], None),\n # multiple specs, some from hash some from file\n (True, [\"zmpi\", \"mpileaks^zmpi\", \"libelf\"], None),\n (True, [\"mpileaks^zmpi\", \"mpileaks^mpich\", \"libelf\"], spack.error.SpecError),\n (False, [\"mpileaks^zmpi\", \"mpileaks^mpich\", \"libelf\"], None),\n ],\n)\ndef test_special_cases_concretization_matching_specs_from_env(\n unify,\n spec_strs,\n error,\n monkeypatch,\n mutable_config,\n mutable_database,\n tmp_path: pathlib.Path,\n mutable_mock_env_path,\n):\n \"\"\"Test that special cases in parse_specs(concretize=True) bypass solver\"\"\"\n\n # monkeypatch to ensure we do not call the actual concretizer\n def _fail(*args, **kwargs):\n assert False\n\n monkeypatch.setattr(asp.SpackSolverSetup, \"setup\", _fail)\n\n spack.config.set(\"concretizer:unify\", unify)\n\n ev.create(\"test\")\n env = ev.read(\"test\")\n\n args = [f\"/{spack.store.STORE.db.query(s)[0].dag_hash()}\" for s in spec_strs]\n if len(args) > 1:\n # We convert the last one to a specfile input\n filename = tmp_path / \"spec.json\"\n spec = parse_specs(args[-1], concretize=True)[0]\n with open(filename, \"w\", encoding=\"utf-8\") as f:\n spec.to_json(f)\n args[-1] = str(filename)\n\n with env:\n specs = parse_specs(args, concretize=False)\n if error:\n with pytest.raises(error):\n matching_specs_from_env(specs)\n else:\n # assertion error from monkeypatch above if test fails\n matching_specs_from_env(specs)\n" }, { "path": "lib/spack/spack/test/cmd/install.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport argparse\nimport builtins\nimport filecmp\nimport gzip\nimport itertools\nimport os\nimport pathlib\nimport re\nimport sys\nimport time\n\nimport pytest\n\nimport spack.build_environment\nimport spack.cmd.common.arguments\nimport spack.cmd.install\nimport spack.concretize\nimport spack.config\nimport spack.environment as ev\nimport spack.error\nimport spack.hash_types as ht\nimport spack.installer\nimport spack.llnl.util.filesystem as fs\nimport spack.llnl.util.tty as tty\nimport spack.package_base\nimport spack.store\nfrom spack.error import SpackError, SpecSyntaxError\nfrom spack.installer import PackageInstaller\nfrom spack.main import SpackCommand\nfrom spack.spec import Spec\n\ninstall = SpackCommand(\"install\")\nenv = SpackCommand(\"env\")\nadd = SpackCommand(\"add\")\nmirror = SpackCommand(\"mirror\")\nuninstall = SpackCommand(\"uninstall\")\nbuildcache = SpackCommand(\"buildcache\")\nfind = SpackCommand(\"find\")\n\n\n@pytest.fixture()\ndef noop_install(monkeypatch):\n def noop(*args, **kwargs):\n pass\n\n monkeypatch.setattr(spack.installer.PackageInstaller, \"install\", noop)\n\n\ndef test_install_package_and_dependency(\n tmp_path: pathlib.Path,\n mock_packages,\n mock_archive,\n mock_fetch,\n install_mockery,\n installer_variant,\n):\n log = \"test\"\n with fs.working_dir(str(tmp_path)):\n install(\"--fake\", \"--log-format=junit\", f\"--log-file={log}\", \"libdwarf\")\n\n files = list(tmp_path.iterdir())\n filename = tmp_path / f\"{log}.xml\"\n assert filename in files\n\n content = filename.read_text()\n\n assert 'tests=\"5\"' in content\n assert 'failures=\"0\"' in content\n assert 'errors=\"0\"' in content\n\n\ndef _check_runtests_none(pkg):\n assert not pkg.run_tests\n\n\ndef _check_runtests_dttop(pkg):\n assert pkg.run_tests == (pkg.name == \"dttop\")\n\n\ndef _check_runtests_all(pkg):\n assert pkg.run_tests\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_install_runtests_notests(monkeypatch, mock_packages, install_mockery):\n monkeypatch.setattr(spack.package_base.PackageBase, \"_unit_test_check\", _check_runtests_none)\n install(\"-v\", \"dttop\")\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_install_runtests_root(monkeypatch, mock_packages, install_mockery):\n monkeypatch.setattr(spack.package_base.PackageBase, \"_unit_test_check\", _check_runtests_dttop)\n install(\"--test=root\", \"dttop\")\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_install_runtests_all(monkeypatch, mock_packages, install_mockery):\n monkeypatch.setattr(spack.package_base.PackageBase, \"_unit_test_check\", _check_runtests_all)\n install(\"--test=all\", \"pkg-a\")\n\n\ndef test_install_package_already_installed(\n tmp_path: pathlib.Path,\n mock_packages,\n mock_archive,\n mock_fetch,\n install_mockery,\n installer_variant,\n):\n with fs.working_dir(str(tmp_path)):\n install(\"--fake\", \"libdwarf\")\n install(\"--fake\", \"--log-format=junit\", \"--log-file=test.xml\", \"libdwarf\")\n\n files = list(tmp_path.iterdir())\n filename = tmp_path / \"test.xml\"\n assert filename in files\n\n content = filename.read_text()\n assert 'tests=\"5\"' in content\n assert 'failures=\"0\"' in content\n assert 'errors=\"0\"' in content\n\n skipped = [line for line in content.split(\"\\n\") if \"skipped\" in line]\n assert len(skipped) == 5\n\n\n@pytest.mark.parametrize(\n \"arguments,expected\",\n [\n ([], spack.config.get(\"config:dirty\")), # default from config file\n ([\"--clean\"], False),\n ([\"--dirty\"], True),\n ],\n)\ndef test_install_dirty_flag(arguments, expected):\n parser = argparse.ArgumentParser()\n spack.cmd.install.setup_parser(parser)\n args = parser.parse_args(arguments)\n assert args.dirty == expected\n\n\ndef test_package_output(install_mockery, mock_fetch):\n \"\"\"\n Ensure output printed from pkgs is captured by output redirection.\n \"\"\"\n # we can't use output capture here because it interferes with Spack's\n # logging. TODO: see whether we can get multiple log_outputs to work\n # when nested AND in pytest\n spec = spack.concretize.concretize_one(\"printing-package\")\n pkg = spec.package\n PackageInstaller([pkg], explicit=True, verbose=True, tests=sys.platform != \"win32\").install()\n\n with gzip.open(pkg.install_log_path, \"rt\") as f:\n out = f.read()\n\n # make sure that output from the actual package file appears in the\n # right place in the build log.\n assert \"BEFORE INSTALL\" in out\n assert \"AFTER INSTALL\" in out\n\n if not sys.platform == \"win32\":\n # Check that install-time test log contains check and installcheck output\n log_path = pkg.tester.archived_install_test_log\n assert os.path.exists(log_path), f\"Missing install-time test log at {log_path}\"\n\n with open(log_path, \"r\", encoding=\"utf-8\") as f:\n test_log_contents = f.read()\n\n assert \"PRINTING PACKAGE CHECK\" in test_log_contents\n assert \"PRINTING PACKAGE INSTALLCHECK\" in test_log_contents\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_install_output_on_build_error(mock_packages, mock_archive, mock_fetch, install_mockery):\n \"\"\"\n This test used to assume receiving full output, but since we've updated\n spack to generate logs on the level of phases, it will only return the\n last phase, install.\n \"\"\"\n # capfd interferes with Spack's capturing\n out = install(\"-v\", \"build-error\", fail_on_error=False)\n assert \"Installing build-error\" in out\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_install_output_on_python_error(mock_packages, mock_archive, mock_fetch, install_mockery):\n out = install(\"failing-build\", fail_on_error=False)\n assert isinstance(install.error, spack.build_environment.ChildError)\n assert install.error.name == \"InstallError\"\n assert 'raise InstallError(\"Expected failure.\")' in out\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_install_with_source(\n mock_packages, mock_archive, mock_fetch, install_mockery, installer_variant\n):\n \"\"\"Verify that source has been copied into place.\"\"\"\n install(\"--source\", \"--keep-stage\", \"trivial-install-test-package\")\n spec = spack.concretize.concretize_one(\"trivial-install-test-package\")\n src = os.path.join(spec.prefix.share, \"trivial-install-test-package\", \"src\")\n assert filecmp.cmp(\n os.path.join(mock_archive.path, \"configure\"), os.path.join(src, \"configure\")\n )\n\n\ndef test_install_env_variables(\n mock_packages, mock_archive, mock_fetch, install_mockery, installer_variant\n):\n spec = spack.concretize.concretize_one(\"pkg-c\")\n install(\"pkg-c\")\n assert os.path.isfile(spec.package.install_env_path)\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_show_log_on_error(mock_packages, mock_archive, mock_fetch, install_mockery):\n \"\"\"\n Make sure --show-log-on-error works.\n \"\"\"\n out = install(\"--show-log-on-error\", \"build-error\", fail_on_error=False)\n assert isinstance(install.error, spack.build_environment.ChildError)\n assert install.error.pkg.name == \"build-error\"\n\n assert \"Installing build-error\" in out\n assert \"See build log for details:\" in out\n\n\ndef test_install_overwrite(\n mock_packages, mock_archive, mock_fetch, install_mockery, installer_variant\n):\n \"\"\"Tests installing a spec, and then re-installing it in the same prefix.\"\"\"\n spec = spack.concretize.concretize_one(\"pkg-c\")\n install(\"pkg-c\")\n\n # Ignore manifest and install times\n manifest = os.path.join(\n spec.prefix,\n spack.store.STORE.layout.metadata_dir,\n spack.store.STORE.layout.manifest_file_name,\n )\n ignores = [manifest, spec.package.times_log_path]\n\n assert os.path.exists(spec.prefix)\n expected_md5 = fs.hash_directory(spec.prefix, ignore=ignores)\n\n # Modify the first installation to be sure the content is not the same\n # as the one after we reinstalled\n with open(os.path.join(spec.prefix, \"only_in_old\"), \"w\", encoding=\"utf-8\") as f:\n f.write(\"This content is here to differentiate installations.\")\n\n bad_md5 = fs.hash_directory(spec.prefix, ignore=ignores)\n\n assert bad_md5 != expected_md5\n\n install(\"--overwrite\", \"-y\", \"pkg-c\")\n\n assert os.path.exists(spec.prefix)\n assert fs.hash_directory(spec.prefix, ignore=ignores) == expected_md5\n assert fs.hash_directory(spec.prefix, ignore=ignores) != bad_md5\n\n\ndef test_install_overwrite_not_installed(\n mock_packages, mock_archive, mock_fetch, install_mockery, installer_variant\n):\n \"\"\"Tests that overwrite doesn't fail if the package is not installed\"\"\"\n spec = spack.concretize.concretize_one(\"pkg-c\")\n assert not os.path.exists(spec.prefix)\n install(\"--overwrite\", \"-y\", \"pkg-c\")\n assert os.path.exists(spec.prefix)\n\n\ndef test_install_commit(mock_git_version_info, install_mockery, mock_packages, monkeypatch):\n \"\"\"Test installing a git package from a commit.\n\n This ensures Spack associates commit versions with their packages in time to do\n version lookups. Details of version lookup tested elsewhere.\n\n \"\"\"\n repo_path, filename, commits = mock_git_version_info\n file_url = pathlib.Path(repo_path).as_uri()\n\n monkeypatch.setattr(spack.package_base.PackageBase, \"git\", file_url, raising=False)\n\n # Use the earliest commit in the repository\n spec = spack.concretize.concretize_one(f\"git-test-commit@{commits[-1]}\")\n PackageInstaller([spec.package], explicit=True).install()\n\n # Ensure first commit file contents were written\n installed = os.listdir(spec.prefix.bin)\n assert filename in installed\n with open(spec.prefix.bin.join(filename), \"r\", encoding=\"utf-8\") as f:\n content = f.read().strip()\n assert content == \"[0]\" # contents are weird for another test\n\n\ndef test_install_overwrite_multiple(\n mock_packages, mock_archive, mock_fetch, install_mockery, installer_variant\n):\n # Try to install a spec and then to reinstall it.\n libdwarf = spack.concretize.concretize_one(\"libdwarf\")\n cmake = spack.concretize.concretize_one(\"cmake\")\n\n install(\"--fake\", \"libdwarf\")\n install(\"--fake\", \"cmake\")\n\n ld_manifest = os.path.join(\n libdwarf.prefix,\n spack.store.STORE.layout.metadata_dir,\n spack.store.STORE.layout.manifest_file_name,\n )\n\n ld_ignores = [ld_manifest, libdwarf.package.times_log_path]\n\n assert os.path.exists(libdwarf.prefix)\n expected_libdwarf_md5 = fs.hash_directory(libdwarf.prefix, ignore=ld_ignores)\n\n cm_manifest = os.path.join(\n cmake.prefix,\n spack.store.STORE.layout.metadata_dir,\n spack.store.STORE.layout.manifest_file_name,\n )\n\n cm_ignores = [cm_manifest, cmake.package.times_log_path]\n assert os.path.exists(cmake.prefix)\n expected_cmake_md5 = fs.hash_directory(cmake.prefix, ignore=cm_ignores)\n\n # Modify the first installation to be sure the content is not the same\n # as the one after we reinstalled\n with open(os.path.join(libdwarf.prefix, \"only_in_old\"), \"w\", encoding=\"utf-8\") as f:\n f.write(\"This content is here to differentiate installations.\")\n with open(os.path.join(cmake.prefix, \"only_in_old\"), \"w\", encoding=\"utf-8\") as f:\n f.write(\"This content is here to differentiate installations.\")\n\n bad_libdwarf_md5 = fs.hash_directory(libdwarf.prefix, ignore=ld_ignores)\n bad_cmake_md5 = fs.hash_directory(cmake.prefix, ignore=cm_ignores)\n\n assert bad_libdwarf_md5 != expected_libdwarf_md5\n assert bad_cmake_md5 != expected_cmake_md5\n\n install(\"--fake\", \"--overwrite\", \"-y\", \"libdwarf\", \"cmake\")\n assert os.path.exists(libdwarf.prefix)\n assert os.path.exists(cmake.prefix)\n\n ld_hash = fs.hash_directory(libdwarf.prefix, ignore=ld_ignores)\n cm_hash = fs.hash_directory(cmake.prefix, ignore=cm_ignores)\n assert ld_hash == expected_libdwarf_md5\n assert cm_hash == expected_cmake_md5\n assert ld_hash != bad_libdwarf_md5\n assert cm_hash != bad_cmake_md5\n\n\n@pytest.mark.usefixtures(\"mock_packages\", \"mock_archive\", \"mock_fetch\", \"install_mockery\")\ndef test_install_conflicts(conflict_spec):\n # Make sure that spec with conflicts raises a SpackError\n with pytest.raises(SpackError):\n install(conflict_spec)\n\n\n@pytest.mark.usefixtures(\"mock_packages\", \"mock_archive\", \"mock_fetch\", \"install_mockery\")\ndef test_install_invalid_spec():\n # Make sure that invalid specs raise a SpackError\n with pytest.raises(SpecSyntaxError, match=\"unexpected characters\"):\n install(\"conflict%~\")\n\n\n@pytest.mark.disable_clean_stage_check\n@pytest.mark.usefixtures(\"mock_packages\", \"mock_archive\", \"mock_fetch\", \"install_mockery\")\n@pytest.mark.parametrize(\n \"exc_typename,msg\",\n [(\"RuntimeError\", \"something weird happened\"), (\"ValueError\", \"spec is not concrete\")],\n)\ndef test_junit_output_with_failures(tmp_path: pathlib.Path, exc_typename, msg, installer_variant):\n with fs.working_dir(str(tmp_path)):\n install(\n \"--verbose\",\n \"--log-format=junit\",\n \"--log-file=test.xml\",\n \"raiser\",\n \"exc_type={0}\".format(exc_typename),\n 'msg=\"{0}\"'.format(msg),\n fail_on_error=False,\n )\n\n # New installer considers Python exceptions ordinary build failures.\n if installer_variant == \"old\":\n assert isinstance(install.error, spack.build_environment.ChildError)\n assert install.error.name == exc_typename\n assert install.error.pkg.name == \"raiser\"\n\n files = list(tmp_path.iterdir())\n filename = tmp_path / \"test.xml\"\n assert filename in files\n\n content = filename.read_text()\n\n # Count failures and errors correctly\n assert 'tests=\"1\"' in content\n assert 'failures=\"1\"' in content\n assert 'errors=\"0\"' in content\n\n # Nothing should have succeeded\n assert 'tests=\"0\"' not in content\n assert 'failures=\"0\"' not in content\n\n # We want to have both stdout and stderr\n assert \"\" in content\n assert msg in content\n\n\ndef _throw(task, exc_typename, exc_type, msg):\n # Self is a spack.installer.Task\n exc_type = getattr(builtins, exc_typename)\n exc = exc_type(msg)\n task.fail(exc)\n\n\ndef _runtime_error(task, *args, **kwargs):\n _throw(task, \"RuntimeError\", spack.error.InstallError, \"something weird happened\")\n\n\ndef _keyboard_error(task, *args, **kwargs):\n _throw(task, \"KeyboardInterrupt\", KeyboardInterrupt, \"Ctrl-C strikes again\")\n\n\n@pytest.mark.disable_clean_stage_check\n@pytest.mark.parametrize(\n \"exc_typename,expected_exc,msg\",\n [\n (\"RuntimeError\", spack.error.InstallError, \"something weird happened\"),\n (\"KeyboardInterrupt\", KeyboardInterrupt, \"Ctrl-C strikes again\"),\n ],\n)\ndef test_junit_output_with_errors(\n exc_typename,\n expected_exc,\n msg,\n mock_packages,\n mock_archive,\n mock_fetch,\n install_mockery,\n tmp_path: pathlib.Path,\n monkeypatch,\n):\n throw = _keyboard_error if expected_exc is KeyboardInterrupt else _runtime_error\n monkeypatch.setattr(spack.installer.BuildTask, \"complete\", throw)\n\n with fs.working_dir(str(tmp_path)):\n install(\n \"--verbose\",\n \"--log-format=junit\",\n \"--log-file=test.xml\",\n \"trivial-install-test-dependent\",\n fail_on_error=False,\n )\n\n assert isinstance(install.error, expected_exc)\n\n files = list(tmp_path.iterdir())\n filename = tmp_path / \"test.xml\"\n assert filename in files\n\n content = filename.read_text()\n\n # Only original error is reported, dependent\n # install is skipped and it is not an error.\n assert 'tests=\"0\"' not in content\n assert 'failures=\"0\"' in content\n assert 'errors=\"0\"' not in content\n\n # Nothing should have succeeded\n assert 'errors=\"0\"' not in content\n\n # We want to have both stdout and stderr\n assert \"\" in content\n assert f'error message=\"{msg}\"' in content\n\n\n@pytest.fixture(params=[\"yaml\", \"json\"])\ndef spec_format(request):\n return request.param\n\n\n@pytest.mark.usefixtures(\"noop_install\", \"mock_packages\", \"config\")\n@pytest.mark.parametrize(\n \"clispecs,filespecs\",\n [\n [[], [\"mpi\"]],\n [[], [\"mpi\", \"boost\"]],\n [[\"cmake\"], [\"mpi\"]],\n [[\"cmake\", \"libelf\"], []],\n [[\"cmake\", \"libelf\"], [\"mpi\", \"boost\"]],\n ],\n)\ndef test_install_mix_cli_and_files(spec_format, clispecs, filespecs, tmp_path: pathlib.Path):\n args = clispecs\n\n for spec in filespecs:\n filepath = tmp_path / (spec + f\".{spec_format}\")\n args = [str(filepath)] + args\n s = spack.concretize.concretize_one(spec)\n with filepath.open(\"w\") as f:\n s.to_yaml(f) if spec_format == \"yaml\" else s.to_json(f)\n\n install(*args, fail_on_error=False)\n assert install.returncode == 0\n\n\ndef test_extra_files_are_archived(\n mock_packages, mock_archive, mock_fetch, install_mockery, installer_variant\n):\n s = spack.concretize.concretize_one(\"archive-files\")\n\n install(\"archive-files\")\n\n archive_dir = os.path.join(spack.store.STORE.layout.metadata_path(s), \"archived-files\")\n config_log = os.path.join(archive_dir, mock_archive.expanded_archive_basedir, \"config.log\")\n assert os.path.exists(config_log)\n\n errors_txt = os.path.join(archive_dir, \"errors.txt\")\n assert os.path.exists(errors_txt)\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_cdash_report_concretization_error(\n tmp_path: pathlib.Path, mock_fetch, install_mockery, conflict_spec, installer_variant\n):\n with fs.working_dir(str(tmp_path)):\n with pytest.raises(SpackError):\n install(\"--log-format=cdash\", \"--log-file=cdash_reports\", conflict_spec)\n report_dir = tmp_path / \"cdash_reports\"\n assert report_dir in list(tmp_path.iterdir())\n report_file = report_dir / \"Update.xml\"\n assert report_file in list(report_dir.iterdir())\n content = report_file.read_text()\n assert \"\" in content\n # The message is different based on using the\n # new or the old concretizer\n expected_messages = (\"Conflicts in concretized spec\", \"conflicts with\")\n assert any(x in content for x in expected_messages)\n\n\n@pytest.mark.not_on_windows(\"Windows log_output logs phase header out of order\")\n@pytest.mark.disable_clean_stage_check\ndef test_cdash_upload_build_error(\n capfd, tmp_path: pathlib.Path, mock_fetch, install_mockery, installer_variant\n):\n with fs.working_dir(str(tmp_path)):\n with pytest.raises(SpackError):\n install(\n \"--log-format=cdash\",\n \"--log-file=cdash_reports\",\n \"--cdash-upload-url=http://localhost/fakeurl/submit.php?project=Spack\",\n \"build-error\",\n )\n report_dir = tmp_path / \"cdash_reports\"\n assert report_dir in list(tmp_path.iterdir())\n report_file = report_dir / \"Build.xml\"\n assert report_file in list(report_dir.iterdir())\n content = report_file.read_text()\n assert \"configure: error: in /path/to/some/file:\" in content\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_cdash_upload_clean_build(\n tmp_path: pathlib.Path, mock_fetch, install_mockery, installer_variant\n):\n with fs.working_dir(str(tmp_path)):\n install(\"--log-file=cdash_reports\", \"--log-format=cdash\", \"pkg-c\")\n report_dir = tmp_path / \"cdash_reports\"\n assert report_dir in list(tmp_path.iterdir())\n report_file = report_dir / \"Build.xml\"\n assert report_file in list(report_dir.iterdir())\n content = report_file.read_text()\n assert \"\" in content\n assert \"\" not in content\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_cdash_upload_extra_params(\n tmp_path: pathlib.Path, mock_fetch, install_mockery, installer_variant\n):\n with fs.working_dir(str(tmp_path)):\n install(\n \"--log-file=cdash_reports\",\n \"--log-format=cdash\",\n \"--cdash-build=my_custom_build\",\n \"--cdash-site=my_custom_site\",\n \"--cdash-track=my_custom_track\",\n \"pkg-c\",\n )\n report_dir = tmp_path / \"cdash_reports\"\n assert report_dir in list(tmp_path.iterdir())\n report_file = report_dir / \"Build.xml\"\n assert report_file in list(report_dir.iterdir())\n content = report_file.read_text()\n assert 'Site BuildName=\"my_custom_build\"' in content\n assert 'Name=\"my_custom_site\"' in content\n assert \"-my_custom_track\" in content\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_cdash_buildstamp_param(\n tmp_path: pathlib.Path, mock_fetch, install_mockery, installer_variant\n):\n with fs.working_dir(str(tmp_path)):\n cdash_track = \"some_mocked_track\"\n buildstamp_format = f\"%Y%m%d-%H%M-{cdash_track}\"\n buildstamp = time.strftime(buildstamp_format, time.localtime(int(time.time())))\n install(\n \"--log-file=cdash_reports\",\n \"--log-format=cdash\",\n f\"--cdash-buildstamp={buildstamp}\",\n \"pkg-c\",\n )\n report_dir = tmp_path / \"cdash_reports\"\n assert report_dir in list(tmp_path.iterdir())\n report_file = report_dir / \"Build.xml\"\n assert report_file in list(report_dir.iterdir())\n content = report_file.read_text()\n assert buildstamp in content\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_cdash_install_from_spec_json(\n tmp_path: pathlib.Path,\n mock_fetch,\n install_mockery,\n mock_packages,\n mock_archive,\n installer_variant,\n):\n with fs.working_dir(str(tmp_path)):\n spec_json_path = str(tmp_path / \"spec.json\")\n\n pkg_spec = spack.concretize.concretize_one(\"pkg-c\")\n with open(spec_json_path, \"w\", encoding=\"utf-8\") as fd:\n fd.write(pkg_spec.to_json(hash=ht.dag_hash))\n\n install(\n \"--log-format=cdash\",\n \"--log-file=cdash_reports\",\n \"--cdash-build=my_custom_build\",\n \"--cdash-site=my_custom_site\",\n \"--cdash-track=my_custom_track\",\n spec_json_path,\n )\n\n report_dir = tmp_path / \"cdash_reports\"\n assert report_dir in list(tmp_path.iterdir())\n report_file = report_dir / \"Configure.xml\"\n assert report_file in list(report_dir.iterdir())\n content = report_file.read_text()\n install_command_regex = re.compile(\n r\"(.+)\", re.MULTILINE | re.DOTALL\n )\n m = install_command_regex.search(content)\n assert m\n install_command = m.group(1)\n assert \"pkg-c@\" in install_command\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_build_error_output(capfd, mock_fetch, install_mockery):\n with pytest.raises(spack.build_environment.ChildError) as e:\n install(\"build-error\")\n assert \"configure: error: in /path/to/some/file:\" in install.output\n assert \"configure: error: in /path/to/some/file:\" in e.value.long_message\n assert \"configure: error: cannot run C compiled programs.\" in install.output\n assert \"configure: error: cannot run C compiled programs.\" in e.value.long_message\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_build_warning_output(mock_fetch, install_mockery):\n with pytest.raises(spack.build_environment.ChildError) as e:\n install(\"build-warnings\")\n assert \"WARNING: ALL CAPITAL WARNING!\" in install.output\n assert \"WARNING: ALL CAPITAL WARNING!\" in e.value.long_message\n assert \"foo.c:89: warning: some weird warning!\" in install.output\n assert \"foo.c:89: warning: some weird warning!\" in e.value.long_message\n\n\ndef test_cache_only_fails(mock_fetch, install_mockery):\n # libelf from cache fails to install, which automatically removes the\n # the libdwarf build task\n out = install(\"--cache-only\", \"libdwarf\", fail_on_error=False)\n\n assert \"Failed to install gcc-runtime\" in out\n assert \"Skipping build of libdwarf\" in out\n assert \"was not installed\" in out\n\n # Check that failure prefix locks are still cached\n failed_packages = [\n pkg_name for dag_hash, pkg_name in spack.store.STORE.failure_tracker.locker.locks.keys()\n ]\n assert \"libelf\" in failed_packages\n assert \"libdwarf\" in failed_packages\n\n\ndef test_install_only_dependencies(mock_fetch, install_mockery, installer_variant):\n dep = spack.concretize.concretize_one(\"dependency-install\")\n root = spack.concretize.concretize_one(\"dependent-install\")\n\n install(\"--only\", \"dependencies\", \"dependent-install\")\n\n assert os.path.exists(dep.prefix)\n assert not os.path.exists(root.prefix)\n\n\ndef test_install_only_package(mock_fetch, install_mockery):\n msg = \"\"\n try:\n install(\"--only\", \"package\", \"dependent-install\")\n except spack.error.InstallError as e:\n msg = str(e)\n\n assert \"Cannot proceed with dependent-install\" in msg\n assert \"1 uninstalled dependency\" in msg\n\n\ndef test_install_deps_then_package(mock_fetch, install_mockery, installer_variant):\n dep = spack.concretize.concretize_one(\"dependency-install\")\n root = spack.concretize.concretize_one(\"dependent-install\")\n\n install(\"--only\", \"dependencies\", \"dependent-install\")\n assert os.path.exists(dep.prefix)\n assert not os.path.exists(root.prefix)\n\n install(\"--only\", \"package\", \"dependent-install\")\n assert os.path.exists(root.prefix)\n\n\n# Unit tests should not be affected by the user's managed environments\n@pytest.mark.not_on_windows(\"Environment views not supported on windows. Revisit after #34701\")\n@pytest.mark.regression(\"12002\")\ndef test_install_only_dependencies_in_env(\n mutable_mock_env_path, mock_fetch, install_mockery, installer_variant\n):\n env(\"create\", \"test\")\n\n with ev.read(\"test\"):\n dep = spack.concretize.concretize_one(\"dependency-install\")\n root = spack.concretize.concretize_one(\"dependent-install\")\n\n install(\"-v\", \"--only\", \"dependencies\", \"--add\", \"dependent-install\")\n\n assert os.path.exists(dep.prefix)\n assert not os.path.exists(root.prefix)\n\n\n# Unit tests should not be affected by the user's managed environments\n@pytest.mark.regression(\"12002\")\ndef test_install_only_dependencies_of_all_in_env(\n mutable_mock_env_path, mock_fetch, install_mockery, installer_variant\n):\n env(\"create\", \"--without-view\", \"test\")\n\n with ev.read(\"test\"):\n roots = [\n spack.concretize.concretize_one(\"dependent-install@1.0\"),\n spack.concretize.concretize_one(\"dependent-install@2.0\"),\n ]\n\n add(\"dependent-install@1.0\")\n add(\"dependent-install@2.0\")\n install(\"--only\", \"dependencies\")\n\n for root in roots:\n assert not os.path.exists(root.prefix)\n for dep in root.traverse(root=False):\n assert os.path.exists(dep.prefix)\n\n\n# Unit tests should not be affected by the user's managed environments\ndef test_install_no_add_in_env(\n tmp_path: pathlib.Path, mutable_mock_env_path, mock_fetch, install_mockery, installer_variant\n):\n # To test behavior of --add option, we create the following environment:\n #\n # mpileaks\n # ^callpath\n # ^dyninst\n # ^libelf@0.8.13 # or latest, really\n # ^libdwarf\n # ^mpich\n # libelf@0.8.10\n # pkg-a~bvv\n # ^pkg-b\n # pkg-a\n # ^pkg-b\n e = ev.create(\"test\", with_view=False)\n e.add(\"mpileaks\")\n e.add(\"libelf@0.8.10\") # so env has both root and dep libelf specs\n e.add(\"pkg-a\")\n e.add(\"pkg-a ~bvv\")\n e.concretize()\n e.write()\n env_specs = e.all_specs()\n\n a_spec = None\n b_spec = None\n mpi_spec = None\n\n # First find and remember some target concrete specs in the environment\n for e_spec in env_specs:\n if e_spec.satisfies(Spec(\"pkg-a ~bvv\")):\n a_spec = e_spec\n elif e_spec.name == \"pkg-b\":\n b_spec = e_spec\n elif e_spec.satisfies(Spec(\"mpi\")):\n mpi_spec = e_spec\n\n assert a_spec\n assert a_spec.concrete\n\n assert b_spec\n assert b_spec.concrete\n assert b_spec not in e.roots()\n\n assert mpi_spec\n assert mpi_spec.concrete\n\n # Activate the environment\n with e:\n # Assert using --no-add with a spec not in the env fails\n inst_out = install(\"--fake\", \"--no-add\", \"boost\", fail_on_error=False)\n\n assert \"Specs can be added to the environment with 'spack add \" in inst_out\n\n # Without --add, ensure that two packages \"a\" get installed\n inst_out = install(\"--fake\", \"pkg-a\")\n assert len([x for x in e.all_specs() if x.installed and x.name == \"pkg-a\"]) == 2\n\n # Install an unambiguous dependency spec (that already exists as a dep\n # in the environment) and make sure it gets installed (w/ deps),\n # but is not added to the environment.\n install(\"dyninst\")\n\n find_output = find(\"-l\")\n assert \"dyninst\" in find_output\n assert \"libdwarf\" in find_output\n assert \"libelf\" in find_output\n assert \"callpath\" not in find_output\n\n post_install_specs = e.all_specs()\n assert all([s in env_specs for s in post_install_specs])\n\n # Make sure we can install a concrete dependency spec from a spec.json\n # file on disk, and the spec is installed but not added as a root\n mpi_spec_json_path = tmp_path / f\"{mpi_spec.name}.json\"\n with open(mpi_spec_json_path, \"w\", encoding=\"utf-8\") as fd:\n fd.write(mpi_spec.to_json(hash=ht.dag_hash))\n\n install(str(mpi_spec_json_path))\n assert mpi_spec not in e.roots()\n\n find_output = find(\"-l\")\n assert mpi_spec.name in find_output\n\n # Install an unambiguous depependency spec (that already exists as a\n # dep in the environment) with --add and make sure it is added as a\n # root of the environment as well as installed.\n assert b_spec not in e.roots()\n\n install(\"--fake\", \"--add\", \"pkg-b\")\n\n assert b_spec in e.roots()\n assert b_spec not in e.uninstalled_specs()\n\n # Install a novel spec with --add and make sure it is added as a root\n # and installed.\n install(\"--fake\", \"--add\", \"bowtie\")\n\n assert any([s.name == \"bowtie\" for s in e.roots()])\n assert not any([s.name == \"bowtie\" for s in e.uninstalled_specs()])\n\n\ndef test_install_help_does_not_show_cdash_options():\n \"\"\"\n Make sure `spack install --help` does not describe CDash arguments\n \"\"\"\n assert \"CDash URL\" not in install(\"--help\")\n\n\ndef test_install_help_cdash():\n \"\"\"Make sure `spack install --help-cdash` describes CDash arguments\"\"\"\n install_cmd = SpackCommand(\"install\")\n out = install_cmd(\"--help-cdash\")\n assert \"CDash URL\" in out\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_cdash_auth_token(\n tmp_path: pathlib.Path, mock_fetch, install_mockery, monkeypatch, installer_variant\n):\n with fs.working_dir(str(tmp_path)):\n monkeypatch.setenv(\"SPACK_CDASH_AUTH_TOKEN\", \"asdf\")\n out = install(\"--fake\", \"-v\", \"--log-file=cdash_reports\", \"--log-format=cdash\", \"pkg-a\")\n assert \"Using CDash auth token from environment\" in out\n\n\n@pytest.mark.not_on_windows(\"Windows log_output logs phase header out of order\")\n@pytest.mark.disable_clean_stage_check\ndef test_cdash_configure_warning(\n tmp_path: pathlib.Path, mock_fetch, install_mockery, installer_variant\n):\n with fs.working_dir(str(tmp_path)):\n # Test would fail if install raised an error.\n\n # Ensure that even on non-x86_64 architectures, there are no\n # dependencies installed\n spec = spack.concretize.concretize_one(\"configure-warning\")\n spec.clear_dependencies()\n specfile = \"./spec.json\"\n with open(specfile, \"w\", encoding=\"utf-8\") as f:\n f.write(spec.to_json())\n install(\"--log-file=cdash_reports\", \"--log-format=cdash\", specfile)\n # Verify Configure.xml exists with expected contents.\n report_dir = tmp_path / \"cdash_reports\"\n assert report_dir.exists()\n report_file = report_dir / \"Configure.xml\"\n assert report_file.exists()\n content = report_file.read_text()\n assert \"foo: No such file or directory\" in content\n\n\ndef test_install_fails_no_args(tmp_path: pathlib.Path):\n # ensure no spack.yaml in directory\n with fs.working_dir(str(tmp_path)):\n output = install(fail_on_error=False)\n\n # check we got the short version of the error message with no spack.yaml\n assert \"requires a package argument or active environment\" in output\n assert \"spack env activate .\" not in output\n assert \"using the `spack.yaml` in this directory\" not in output\n\n\ndef test_install_fails_no_args_suggests_env_activation(tmp_path: pathlib.Path):\n # ensure spack.yaml in directory\n (tmp_path / \"spack.yaml\").touch()\n\n with fs.working_dir(str(tmp_path)):\n output = install(fail_on_error=False)\n\n # check we got the long version of the error message with spack.yaml\n assert \"requires a package argument or active environment\" in output\n assert \"spack env activate .\" in output\n assert \"using the `spack.yaml` in this directory\" in output\n\n\n# Unit tests should not be affected by the user's managed environments\n@pytest.mark.not_on_windows(\"Environment views not supported on windows. Revisit after #34701\")\ndef test_install_env_with_tests_all(\n mutable_mock_env_path, mock_packages, mock_fetch, install_mockery, installer_variant\n):\n env(\"create\", \"test\")\n with ev.read(\"test\"):\n test_dep = spack.concretize.concretize_one(\"test-dependency\")\n add(\"depb\")\n install(\"--fake\", \"--test\", \"all\")\n assert os.path.exists(test_dep.prefix)\n\n\n# Unit tests should not be affected by the user's managed environments\n@pytest.mark.not_on_windows(\"Environment views not supported on windows. Revisit after #34701\")\ndef test_install_env_with_tests_root(\n mutable_mock_env_path, mock_packages, mock_fetch, install_mockery, installer_variant\n):\n env(\"create\", \"test\")\n with ev.read(\"test\"):\n test_dep = spack.concretize.concretize_one(\"test-dependency\")\n add(\"depb\")\n install(\"--fake\", \"--test\", \"root\")\n assert not os.path.exists(test_dep.prefix)\n\n\n# Unit tests should not be affected by the user's managed environments\n@pytest.mark.not_on_windows(\"Environment views not supported on windows. Revisit after #34701\")\ndef test_install_empty_env(\n mutable_mock_env_path, mock_packages, mock_fetch, install_mockery, installer_variant\n):\n env_name = \"empty\"\n env(\"create\", env_name)\n with ev.read(env_name):\n out = install(fail_on_error=False)\n\n assert env_name in out\n assert \"environment\" in out\n assert \"no specs to install\" in out\n\n\n@pytest.mark.not_on_windows(\"Windows logger I/O operation on closed file when install fails\")\n@pytest.mark.disable_clean_stage_check\n@pytest.mark.parametrize(\n \"name,method\",\n [\n (\"test-build-callbacks\", \"undefined-build-test\"),\n (\"test-install-callbacks\", \"undefined-install-test\"),\n ],\n)\ndef test_installation_fail_tests(install_mockery, mock_fetch, name, method):\n \"\"\"Confirm build-time tests with unknown methods fail.\"\"\"\n output = install(\"--test=root\", \"--no-cache\", name, fail_on_error=False)\n\n # Check that there is a single test failure reported\n assert output.count(\"TestFailure: 1 test failed\") == 1\n\n # Check that the method appears twice: no attribute error and in message\n assert output.count(method) == 2\n assert output.count(\"method not implemented\") == 1\n\n # Check that the path to the test log file is also output\n assert \"See test log for details\" in output\n\n\n# Unit tests should not be affected by the user's managed environments\n@pytest.mark.not_on_windows(\"Buildcache not supported on windows\")\ndef test_install_use_buildcache(\n mutable_mock_env_path,\n mock_packages,\n mock_fetch,\n mock_archive,\n mock_binary_index,\n tmp_path: pathlib.Path,\n install_mockery,\n):\n \"\"\"\n Make sure installing with use-buildcache behaves correctly.\n \"\"\"\n\n package_name = \"dependent-install\"\n dependency_name = \"dependency-install\"\n\n def validate(mode, out, pkg):\n def assert_auto(pkg, out):\n assert \"==> Extracting {0}\".format(pkg) in out\n\n def assert_only(pkg, out):\n assert \"==> Extracting {0}\".format(pkg) in out\n\n def assert_never(pkg, out):\n assert \"==> {0}: Executing phase: 'install'\".format(pkg) in out\n\n if mode == \"auto\":\n assert_auto(pkg, out)\n elif mode == \"only\":\n assert_only(pkg, out)\n else:\n assert_never(pkg, out)\n\n def install_use_buildcache(opt):\n out = install(\n \"--no-check-signature\", \"--use-buildcache\", opt, package_name, fail_on_error=True\n )\n\n pkg_opt, dep_opt = spack.cmd.common.arguments.use_buildcache(opt)\n validate(dep_opt, out, dependency_name)\n validate(pkg_opt, out, package_name)\n\n # Clean up installed packages\n uninstall(\"-y\", \"-a\")\n\n # Setup the mirror\n # Create a temp mirror directory for buildcache usage\n mirror_dir = tmp_path / \"mirror_dir\"\n mirror_url = mirror_dir.as_uri()\n\n # Populate the buildcache\n install(package_name)\n buildcache(\"push\", \"-u\", \"-f\", str(mirror_dir), package_name, dependency_name)\n\n # Uninstall the all of the packages for clean slate\n uninstall(\"-y\", \"-a\")\n\n # Configure the mirror where we put that buildcache w/ the compiler\n mirror(\"add\", \"test-mirror\", mirror_url)\n\n # Install using the matrix of possible combinations with --use-buildcache\n for pkg, deps in itertools.product([\"auto\", \"only\", \"never\"], repeat=2):\n tty.debug(\n \"Testing `spack install --use-buildcache package:{0},dependencies:{1}`\".format(\n pkg, deps\n )\n )\n install_use_buildcache(\"package:{0},dependencies:{1}\".format(pkg, deps))\n install_use_buildcache(\"dependencies:{0},package:{1}\".format(deps, pkg))\n\n # Install using a default override option\n # Alternative to --cache-only (always) or --no-cache (never)\n for opt in [\"auto\", \"only\", \"never\"]:\n install_use_buildcache(opt)\n\n\n@pytest.mark.not_on_windows(\"Windows logger I/O operation on closed file when install fails\")\n@pytest.mark.regression(\"34006\")\n@pytest.mark.disable_clean_stage_check\ndef test_padded_install_runtests_root(install_mockery, mock_fetch):\n spack.config.set(\"config:install_tree:padded_length\", 255)\n output = install(\n \"--verbose\", \"--test=root\", \"--no-cache\", \"test-build-callbacks\", fail_on_error=False\n )\n assert \"method not implemented [undefined-build-test]\" in output\n\n\n@pytest.mark.regression(\"35337\")\ndef test_report_filename_for_cdash(install_mockery, mock_fetch):\n \"\"\"Test that the temporary file used to write the XML for CDash is not the upload URL\"\"\"\n parser = argparse.ArgumentParser()\n spack.cmd.install.setup_parser(parser)\n args = parser.parse_args(\n [\"--cdash-upload-url\", \"https://blahblah/submit.php?project=debugging\", \"pkg-a\"]\n )\n specs = spack.cmd.install.concrete_specs_from_cli(args, {})\n filename = spack.cmd.install.report_filename(args, specs)\n assert filename != \"https://blahblah/submit.php?project=debugging\"\n\n\ndef test_setting_concurrent_packages_flag(mutable_config):\n \"\"\"Ensure that the number of concurrent packages is properly set from the command-line flag\"\"\"\n install = SpackCommand(\"install\")\n install(\"--concurrent-packages\", \"8\", fail_on_error=False)\n assert spack.config.get(\"config:concurrent_packages\", scope=\"command_line\") == 8\n\n\ndef test_invalid_concurrent_packages_flag(mutable_config):\n \"\"\"Test that an invalid value for --concurrent-packages CLI flag raises a ValueError\"\"\"\n install = SpackCommand(\"install\")\n with pytest.raises(ValueError, match=\"expected a positive integer\"):\n install(\"--concurrent-packages\", \"-2\")\n\n\n@pytest.mark.skipif(sys.platform == \"win32\", reason=\"Feature disabled on windows due to locking\")\ndef test_concurrent_packages_set_in_config(mutable_config, mock_packages):\n \"\"\"Ensure that the number of concurrent packages is properly set from adding to config\"\"\"\n spack.config.set(\"config:concurrent_packages\", 3)\n spec = spack.concretize.concretize_one(\"pkg-a\")\n installer = spack.installer.PackageInstaller([spec.package])\n assert installer.concurrent_packages == 3\n" }, { "path": "lib/spack/spack/test/cmd/is_git_repo.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport pathlib\nimport sys\n\nimport pytest\n\nimport spack.cmd\nimport spack.fetch_strategy\nfrom spack.llnl.util.filesystem import mkdirp, working_dir\nfrom spack.version import ver\n\n\n@pytest.fixture(scope=\"function\")\ndef git_tmp_worktree(git, tmp_path: pathlib.Path, mock_git_version_info):\n \"\"\"Create new worktree in a temporary folder and monkeypatch\n spack.paths.prefix to point to it.\n \"\"\"\n\n # We need `git worktree remove` for this fixture, which was added in 2.17.0.\n # See https://github.com/git/git/commit/cc73385cf6c5c229458775bc92e7dbbe24d11611\n git_version = spack.fetch_strategy.GitFetchStrategy.version_from_git(git)\n if git_version < ver(\"2.17.0\"):\n pytest.skip(\"git_tmp_worktree requires git v2.17.0\")\n\n with working_dir(mock_git_version_info[0]):\n # TODO: This is fragile and should be high priority for\n # follow up fixes. 27021\n # Path length is occasionally too long on Windows\n # the following reduces the path length to acceptable levels\n if sys.platform == \"win32\":\n long_pth = str(tmp_path).split(os.path.sep)\n tmp_worktree = os.path.sep.join(long_pth[:-1])\n else:\n tmp_worktree = str(tmp_path)\n worktree_root = os.path.sep.join([tmp_worktree, \"wrktree\"])\n\n mkdirp(worktree_root)\n\n git(\"worktree\", \"add\", \"--detach\", worktree_root, \"HEAD\")\n\n yield worktree_root\n\n git(\"worktree\", \"remove\", \"--force\", worktree_root)\n\n\ndef test_is_git_repo_in_worktree(git_tmp_worktree):\n \"\"\"Verify that spack.cmd.spack_is_git_repo() can identify a git repository\n in a worktree.\n \"\"\"\n assert spack.cmd.is_git_repo(git_tmp_worktree)\n\n\ndef test_spack_is_git_repo_nongit(tmp_path: pathlib.Path, monkeypatch):\n \"\"\"Verify that spack.cmd.spack_is_git_repo() correctly returns False if we\n are in a non-git directory.\n \"\"\"\n assert not spack.cmd.is_git_repo(str(tmp_path))\n" }, { "path": "lib/spack/spack/test/cmd/license.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport pathlib\nimport re\n\nimport pytest\n\nimport spack.paths\nfrom spack.llnl.util.filesystem import mkdirp, touch\nfrom spack.main import SpackCommand\n\nlicense = SpackCommand(\"license\")\n\npytestmark = pytest.mark.not_on_windows(\"does not run on windows\")\n\n\ndef test_list_files():\n files = license(\"list-files\").strip().split(\"\\n\")\n assert all(f.startswith(spack.paths.prefix) for f in files)\n assert os.path.join(spack.paths.bin_path, \"spack\") in files\n assert os.path.abspath(__file__) in files\n\n\ndef test_verify(tmp_path: pathlib.Path):\n source_dir = tmp_path / \"lib\" / \"spack\" / \"spack\"\n mkdirp(str(source_dir))\n\n no_header = source_dir / \"no_header.py\"\n touch(str(no_header))\n\n lgpl_header = source_dir / \"lgpl_header.py\"\n with lgpl_header.open(\"w\") as f:\n f.write(\n \"\"\"\\\n# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: LGPL-2.1-only\n\"\"\"\n )\n\n not_in_first_n_lines = source_dir / \"not_in_first_n_lines.py\"\n with not_in_first_n_lines.open(\"w\") as f:\n f.write(\n \"\"\"\\\n#\n#\n#\n#\n# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"\n )\n\n correct_header = source_dir / \"correct_header.py\"\n with correct_header.open(\"w\") as f:\n f.write(\n \"\"\"\\\n# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"\n )\n\n out = license(\"--root\", str(tmp_path), \"verify\", fail_on_error=False)\n\n assert str(no_header) in out\n assert str(lgpl_header) in out\n assert str(not_in_first_n_lines) in out\n assert str(correct_header) not in out\n assert \"3 improperly licensed files\" in out\n assert re.search(r\"files not containing expected license:\\s*1\", out)\n assert re.search(r\"files with wrong SPDX-License-Identifier:\\s*1\", out)\n assert re.search(r\"files without license in first 6 lines:\\s*1\", out)\n\n assert license.returncode == 1\n" }, { "path": "lib/spack/spack/test/cmd/list.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport pathlib\n\nimport pytest\n\nimport spack.cmd.list\nimport spack.paths\nimport spack.repo\nimport spack.util.git\nfrom spack.main import SpackCommand\nfrom spack.test.conftest import RepoBuilder\n\npytestmark = [pytest.mark.usefixtures(\"mock_packages\")]\n\nlist = SpackCommand(\"list\")\n\n\ndef test_list():\n output = list()\n assert \"bzip2\" in output\n assert \"hdf5\" in output\n\n\ndef test_list_cli_output_format(mock_tty_stdout):\n assert (\n list(\"mpileaks\")\n == \"\"\"\\\nmpileaks\n==> 1 packages\n\"\"\"\n )\n\n\ndef test_list_filter():\n output = list(\"py-*\")\n assert \"py-extension1\" in output\n assert \"py-extension2\" in output\n assert \"py-extension3\" in output\n assert \"python\" not in output\n assert \"mpich\" not in output\n\n output = list(\"py\")\n assert \"py-extension1\" in output\n assert \"py-extension2\" in output\n assert \"py-extension3\" in output\n assert \"python\" in output\n assert \"mpich\" not in output\n\n\ndef test_list_search_description():\n output = list(\"--search-description\", \"one build dependency\")\n assert \"depb\" in output\n\n\ndef test_list_format_name_only():\n output = list(\"--format\", \"name_only\")\n assert \"zmpi\" in output\n assert \"hdf5\" in output\n\n\ndef test_list_format_version_json():\n output = list(\"--format\", \"version_json\")\n assert '{\"name\": \"zmpi\",' in output\n assert '{\"name\": \"dyninst\",' in output\n assert \"packages/zmpi/package.py\" in output\n\n import json\n\n json.loads(output)\n\n\ndef test_list_format_html():\n output = list(\"--format\", \"html\")\n assert '
' in output\n assert \"

zmpi\" in output\n\n assert '
' in output\n assert \"

hdf5\" in output\n assert \"packages/hdf5/package.py\" in output\n\n\n@pytest.mark.parametrize(\n \"url\",\n [\n \"git@github.com:username/spack-packages.git\",\n \"https://github.com/username/spack-packages.git\",\n \"git@github.com:username/spack.git\",\n \"https://github.com/username/spack.git\",\n ],\n)\ndef test_list_url_schemes(mock_util_executable, url):\n \"\"\"Confirm the command handles supported repository URLs.\"\"\"\n pkg_name = \"hdf5\"\n\n _, _, registered_responses = mock_util_executable\n registered_responses[\"config\"] = url\n registered_responses[\"rev-parse\"] = f\"path/to/builtin/packages/{pkg_name}/\"\n\n output = list(\"--format\", \"version_json\", pkg_name)\n assert f\"{registered_responses['rev-parse']}package.py\" in output\n assert os.path.basename(url).replace(\".git\", \"\") in output\n\n\ndef test_list_format_local_repo(tmp_path: pathlib.Path):\n \"\"\"Confirm a file path is returned for local repository.\"\"\"\n pkg_name = \"mypkg\"\n repo_root = tmp_path / \"repos\" / \"spack_repo\" / \"builtin\"\n repo_root.mkdir(parents=True)\n (repo_root / \"repo.yaml\").write_text(\"repo:\\n namespace: builtin\\n api: v2.2\\n\")\n package_root = repo_root / \"packages\" / pkg_name\n package_root.mkdir(parents=True)\n (package_root / \"package.py\").write_text(\n \"\"\"\\\nfrom spack.package import *\n\nclass Mypkg(Package):\n pass\n\"\"\"\n )\n\n test_repo = spack.repo.from_path(str(repo_root))\n with spack.repo.use_repositories(test_repo):\n # Confirm a path is returned when fail to retrieve the remote origin URL\n output = list(\"--format\", \"version_json\", pkg_name)\n assert \"github.com\" not in output\n assert f\"packages/{pkg_name}/package.py\" in output\n\n\ndef test_list_format_non_github_repo(tmp_path: pathlib.Path, mock_util_executable):\n \"\"\"Confirm a file path is returned for a non-github repository.\"\"\"\n pkg_name = \"mypkg\"\n repo_root = tmp_path / \"my\" / \"project\" / \"spack_repo\" / \"builtin\"\n repo_root.mkdir(parents=True)\n (repo_root / \"repo.yaml\").write_text(\"repo:\\n namespace: builtin\\n api: v2.2\\n\")\n package_root = repo_root / \"packages\" / pkg_name\n package_root.mkdir(parents=True)\n package_path = package_root / \"package.py\"\n package_path.write_text(\n \"\"\"\\\nfrom spack.package import *\n\nclass Mypkg(Package):\n pass\n\"\"\"\n )\n\n test_repo = spack.repo.from_path(str(repo_root))\n with spack.repo.use_repositories(test_repo):\n # Confirm a path is returned for a non-standard spack repository\n _, _, registered_responses = mock_util_executable\n registered_responses[\"config\"] = \"https://gitlab.com/username/my-packages.git\"\n registered_responses[\"rev-parse\"] = str(package_root) + os.sep\n\n output = list(\"--format\", \"version_json\", pkg_name)\n assert package_path.as_uri() in output\n\n\ndef test_list_update(tmp_path: pathlib.Path):\n update_file = tmp_path / \"output\"\n\n # not yet created when list is run\n list(\"--update\", str(update_file))\n assert update_file.exists()\n with update_file.open() as f:\n assert f.read()\n\n # created but older than any package\n with update_file.open(\"w\") as f:\n f.write(\"empty\\n\")\n os.utime(str(update_file), (0, 0)) # Set mtime to 0\n list(\"--update\", str(update_file))\n assert update_file.exists()\n with update_file.open() as f:\n assert f.read() != \"empty\\n\"\n\n # newer than any packages\n with update_file.open(\"w\") as f:\n f.write(\"empty\\n\")\n list(\"--update\", str(update_file))\n assert update_file.exists()\n with update_file.open() as f:\n assert f.read() == \"empty\\n\"\n\n\ndef test_list_tags():\n output = list(\"--tag\", \"tag1\")\n assert \"mpich\" in output\n assert \"mpich2\" in output\n\n output = list(\"--tag\", \"tag2\")\n assert \"mpich\\n\" in output\n assert \"mpich2\" not in output\n\n output = list(\"--tag\", \"tag3\")\n assert \"mpich\\n\" not in output\n assert \"mpich2\" in output\n\n\ndef test_list_count():\n output = list(\"--count\")\n assert int(output.strip()) == len(spack.repo.all_package_names())\n\n output = list(\"--count\", \"py-\")\n assert int(output.strip()) == len(\n [name for name in spack.repo.all_package_names() if \"py-\" in name]\n )\n\n\ndef test_list_repos():\n with spack.repo.use_repositories(\n os.path.join(spack.paths.test_repos_path, \"spack_repo\", \"builtin_mock\"),\n os.path.join(spack.paths.test_repos_path, \"spack_repo\", \"builder_test\"),\n ):\n total_pkgs = len(list().strip().split())\n mock_pkgs = len(list(\"-r\", \"builtin_mock\").strip().split())\n builder_pkgs = len(list(\"-r\", \"builder_test\").strip().split())\n both_repos = len(list(\"-r\", \"builtin_mock\", \"-r\", \"builder_test\").strip().split())\n\n assert total_pkgs > mock_pkgs > builder_pkgs\n assert both_repos == total_pkgs\n\n\n@pytest.mark.usefixtures(\"config\")\ndef test_list_github_url_fails(repo_builder: RepoBuilder, monkeypatch):\n with spack.repo.use_repositories(repo_builder.root):\n repo_builder.add_package(\"pkg-a\")\n repo = spack.repo.PATH.repos[0]\n pkg = repo.get_pkg_class(\"pkg-a\")\n\n old_path = repo.python_path\n try:\n # Check that a repository with no python path has no URL\n monkeypatch.setattr(repo, \"python_path\", None)\n assert spack.cmd.list.github_url(pkg) is None, (\n \"Expected no python path means unable to determine the repo URL\"\n )\n\n # Check that a repository path that doesn't exist has no URL\n monkeypatch.setattr(repo, \"python_path\", \"/repo/root/does/not/exists\")\n assert spack.cmd.list.github_url(pkg) is None, (\n \"Expected bad repo path means unable to determine the repo URL\"\n )\n finally:\n monkeypatch.setattr(repo, \"python_path\", old_path)\n\n # Check that missing git results in the file path\n monkeypatch.setattr(spack.util.git, \"git\", lambda: None)\n filepath = spack.cmd.list.github_url(pkg)\n assert filepath and filepath.startswith(\"file://\"), (\n \"Expected missing 'git' results in a file URI\"\n )\n" }, { "path": "lib/spack/spack/test/cmd/load.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport os\nimport re\nimport sys\n\nimport pytest\n\nimport spack.concretize\nimport spack.user_environment as uenv\nfrom spack.main import SpackCommand\n\nload = SpackCommand(\"load\")\nunload = SpackCommand(\"unload\")\ninstall = SpackCommand(\"install\")\nlocation = SpackCommand(\"location\")\n\n\ndef test_manpath_trailing_colon(\n install_mockery, mock_fetch, mock_archive, mock_packages, working_env\n):\n (shell, set_command, commandsep) = (\n (\"--bat\", 'set \"%s=%s\"', \"\\n\")\n if sys.platform == \"win32\"\n else (\"--sh\", \"export %s=%s\", \";\")\n )\n\n # Test that the commands generated by load add the MANPATH prefix\n # inspections. Also test that Spack correctly preserves the default/existing\n # manpath search path via a trailing colon\n install(\"--fake\", \"mpileaks\")\n\n sh_out = load(shell, \"mpileaks\")\n lines = [line.strip(\"\\n\") for line in sh_out.split(commandsep)]\n assert any(re.match(set_command % (\"MANPATH\", \".*\" + os.pathsep), ln) for ln in lines)\n os.environ[\"MANPATH\"] = \"/tmp/man\" + os.pathsep\n\n sh_out = load(shell, \"mpileaks\")\n lines = [line.strip(\"\\n\") for line in sh_out.split(commandsep)]\n assert any(\n re.match(set_command % (\"MANPATH\", \".*\" + os.pathsep + \"/tmp/man\" + os.pathsep), ln)\n for ln in lines\n )\n\n\ndef test_load_recursive(install_mockery, mock_fetch, mock_archive, mock_packages, working_env):\n def test_load_shell(shell, set_command):\n \"\"\"Test that `spack load` applies prefix inspections of its required runtime deps in\n topo-order\"\"\"\n install(\"--fake\", \"mpileaks\")\n mpileaks_spec = spack.concretize.concretize_one(\"mpileaks\")\n\n # Ensure our reference variable is clean.\n os.environ[\"CMAKE_PREFIX_PATH\"] = \"/hello\" + os.pathsep + \"/world\"\n\n shell_out = load(shell, \"mpileaks\")\n\n def extract_value(output, variable):\n match = re.search(set_command % variable, output, flags=re.MULTILINE)\n value = match.group(1)\n return value.split(os.pathsep)\n\n # Map a prefix found in CMAKE_PREFIX_PATH back to a package name in mpileaks' DAG.\n prefix_to_pkg = lambda prefix: next(\n s.name for s in mpileaks_spec.traverse() if s.prefix == prefix\n )\n\n paths_shell = extract_value(shell_out, \"CMAKE_PREFIX_PATH\")\n\n # We should've prepended new paths, and keep old ones.\n assert paths_shell[-2:] == [\"/hello\", \"/world\"]\n\n # All but the last two paths are added by spack load; lookup what packages they're from.\n pkgs = [prefix_to_pkg(p) for p in paths_shell[:-2]]\n\n # Do we have all the runtime packages?\n assert set(pkgs) == set(\n s.name for s in mpileaks_spec.traverse(deptype=(\"link\", \"run\"), root=True)\n )\n\n # Finally, do we list them in topo order?\n for i, pkg in enumerate(pkgs):\n assert {s.name for s in mpileaks_spec[pkg].traverse(direction=\"parents\")}.issubset(\n pkgs[: i + 1]\n )\n\n # Lastly, do we keep track that mpileaks was loaded?\n assert (\n extract_value(shell_out, uenv.spack_loaded_hashes_var)[0] == mpileaks_spec.dag_hash()\n )\n return paths_shell\n\n if sys.platform == \"win32\":\n shell, set_command = (\"--bat\", r'set \"%s=(.*)\"')\n test_load_shell(shell, set_command)\n else:\n params = [(\"--sh\", r\"export %s=([^;]*)\"), (\"--csh\", r\"setenv %s ([^;]*)\")]\n shell, set_command = params[0]\n paths_sh = test_load_shell(shell, set_command)\n shell, set_command = params[1]\n paths_csh = test_load_shell(shell, set_command)\n assert paths_sh == paths_csh\n\n\n@pytest.mark.parametrize(\n \"shell,set_command\",\n (\n [(\"--bat\", 'set \"%s=%s\"')]\n if sys.platform == \"win32\"\n else [(\"--sh\", \"export %s=%s\"), (\"--csh\", \"setenv %s %s\")]\n ),\n)\ndef test_load_includes_run_env(\n shell, set_command, install_mockery, mock_fetch, mock_archive, mock_packages\n):\n \"\"\"Tests that environment changes from the package's\n `setup_run_environment` method are added to the user environment in\n addition to the prefix inspections\"\"\"\n install(\"--fake\", \"mpileaks\")\n\n shell_out = load(shell, \"mpileaks\")\n\n assert set_command % (\"FOOBAR\", \"mpileaks\") in shell_out\n\n\ndef test_load_first(install_mockery, mock_fetch, mock_archive, mock_packages):\n \"\"\"Test with and without the --first option\"\"\"\n shell = \"--bat\" if sys.platform == \"win32\" else \"--sh\"\n install(\"--fake\", \"libelf@0.8.12\")\n install(\"--fake\", \"libelf@0.8.13\")\n\n # Now there are two versions of libelf, which should cause an error\n out = load(shell, \"libelf\", fail_on_error=False)\n assert \"matches multiple packages\" in out\n assert \"Use a more specific spec\" in out\n\n # Using --first should avoid the error condition\n load(shell, \"--first\", \"libelf\")\n\n\ndef test_load_fails_no_shell(install_mockery, mock_fetch, mock_archive, mock_packages):\n \"\"\"Test that spack load prints an error message without a shell.\"\"\"\n install(\"--fake\", \"mpileaks\")\n\n out = load(\"mpileaks\", fail_on_error=False)\n assert \"To set up shell support\" in out\n\n\n@pytest.mark.parametrize(\n \"shell,set_command,unset_command\",\n (\n [(\"--bat\", 'set \"%s=%s\"', 'set \"%s=\"')]\n if sys.platform == \"win32\"\n else [(\"--sh\", \"export %s=%s\", \"unset %s\"), (\"--csh\", \"setenv %s %s\", \"unsetenv %s\")]\n ),\n)\ndef test_unload(\n shell,\n set_command,\n unset_command,\n install_mockery,\n mock_fetch,\n mock_archive,\n mock_packages,\n working_env,\n):\n \"\"\"Tests that any variables set in the user environment are undone by the\n unload command\"\"\"\n install(\"--fake\", \"mpileaks\")\n mpileaks_spec = spack.concretize.concretize_one(\"mpileaks\")\n\n # Set so unload has something to do\n os.environ[\"FOOBAR\"] = \"mpileaks\"\n os.environ[uenv.spack_loaded_hashes_var] = (\"%s\" + os.pathsep + \"%s\") % (\n mpileaks_spec.dag_hash(),\n \"garbage\",\n )\n\n shell_out = unload(shell, \"mpileaks\")\n\n assert (unset_command % \"FOOBAR\") in shell_out\n\n assert set_command % (uenv.spack_loaded_hashes_var, \"garbage\") in shell_out\n\n\ndef test_unload_fails_no_shell(\n install_mockery, mock_fetch, mock_archive, mock_packages, working_env\n):\n \"\"\"Test that spack unload prints an error message without a shell.\"\"\"\n install(\"--fake\", \"mpileaks\")\n mpileaks_spec = spack.concretize.concretize_one(\"mpileaks\")\n os.environ[uenv.spack_loaded_hashes_var] = mpileaks_spec.dag_hash()\n\n out = unload(\"mpileaks\", fail_on_error=False)\n assert \"To set up shell support\" in out\n" }, { "path": "lib/spack/spack/test/cmd/location.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport shutil\n\nimport pytest\n\nimport spack.concretize\nimport spack.environment as ev\nimport spack.main\nimport spack.paths\nimport spack.repo\nimport spack.stage\nfrom spack.llnl.util.filesystem import mkdirp\nfrom spack.main import SpackCommand\n\n# Everything here uses (or can use) the mock config and database.\npytestmark = [pytest.mark.usefixtures(\"mutable_config\", \"mutable_database\")]\n\n# location prints out \"locations of packages and spack directories\"\nlocation = SpackCommand(\"location\")\nenv = SpackCommand(\"env\")\n\n\n@pytest.fixture\ndef mock_spec():\n # Make it look like the source was actually expanded.\n s = spack.concretize.concretize_one(\"externaltest\")\n source_path = s.package.stage.source_path\n mkdirp(source_path)\n yield s, s.package\n # Remove the spec from the mock stage area.\n shutil.rmtree(s.package.stage.path)\n\n\ndef test_location_first(install_mockery, mock_fetch, mock_archive, mock_packages):\n \"\"\"Test with and without the --first option\"\"\"\n install = SpackCommand(\"install\")\n install(\"--fake\", \"libelf@0.8.12\")\n install(\"--fake\", \"libelf@0.8.13\")\n # This would normally return an error without --first\n assert location(\"--first\", \"--install-dir\", \"libelf\")\n\n\ndef test_location_build_dir(mock_spec):\n \"\"\"Tests spack location --build-dir.\"\"\"\n spec, pkg = mock_spec\n assert location(\"--build-dir\", spec.name).strip() == pkg.stage.source_path\n\n\n@pytest.mark.regression(\"22738\")\ndef test_location_source_dir(mock_spec):\n \"\"\"Tests spack location --source-dir.\"\"\"\n spec, pkg = mock_spec\n assert location(\"--source-dir\", spec.name).strip() == pkg.stage.source_path\n assert location(spec.name).strip() == pkg.stage.source_path\n\n\ndef test_location_source_dir_missing():\n \"\"\"Tests spack location --source-dir with a missing source directory.\"\"\"\n spec = \"mpileaks\"\n prefix = \"==> Error: \"\n expected = (\n \"%sSource directory does not exist yet. Run this to create it:\"\n \"%s spack stage %s\" % (prefix, \"\\n\", spec)\n )\n out = location(\"--source-dir\", spec, fail_on_error=False).strip()\n assert out == expected\n\n\n@pytest.mark.parametrize(\n \"options\",\n [([]), ([\"--source-dir\", \"mpileaks\"]), ([\"--env\", \"missing-env\"]), ([\"spec1\", \"spec2\"])],\n)\ndef test_location_cmd_error(options):\n \"\"\"Ensure the proper error is raised with problematic location options.\"\"\"\n with pytest.raises(spack.main.SpackCommandError) as e:\n location(*options)\n assert e.value.code == 1\n\n\ndef test_location_env_exists(mutable_mock_env_path):\n \"\"\"Tests spack location --env for an existing environment.\"\"\"\n e = ev.create(\"example\")\n e.write()\n assert location(\"--env\", \"example\").strip() == e.path\n\n\ndef test_location_with_active_env(mutable_mock_env_path):\n \"\"\"Tests spack location --env with active env\"\"\"\n e = ev.create(\"example\")\n e.write()\n with e:\n assert location(\"--env\").strip() == e.path\n\n\ndef test_location_env_missing():\n \"\"\"Tests spack location --env.\"\"\"\n missing_env_name = \"missing-env\"\n error = \"==> Error: no such environment: '%s'\" % missing_env_name\n out = location(\"--env\", missing_env_name, fail_on_error=False).strip()\n assert out == error\n\n\n@pytest.mark.db\n@pytest.mark.not_on_windows(\"Broken on Windows\")\ndef test_location_install_dir(mock_spec):\n \"\"\"Tests spack location --install-dir.\"\"\"\n spec, _ = mock_spec\n assert location(\"--install-dir\", spec.name).strip() == spec.prefix\n\n\n@pytest.mark.db\ndef test_location_package_dir(mock_spec):\n \"\"\"Tests spack location --package-dir.\"\"\"\n spec, pkg = mock_spec\n assert location(\"--package-dir\", spec.name).strip() == pkg.package_dir\n\n\n@pytest.mark.db\n@pytest.mark.parametrize(\n \"option,expected\",\n [\n (\"--module-dir\", spack.paths.module_path),\n (\"--packages\", spack.paths.mock_packages_path),\n (\"--spack-root\", spack.paths.prefix),\n ],\n)\ndef test_location_paths_options(option, expected):\n \"\"\"Tests basic spack.paths location command options.\"\"\"\n assert location(option).strip() == expected\n\n\n@pytest.mark.parametrize(\n \"specs,expected\",\n [([], \"You must supply a spec.\"), ([\"spec1\", \"spec2\"], \"Too many specs. Supply only one.\")],\n)\ndef test_location_spec_errors(specs, expected):\n \"\"\"Tests spack location with bad spec options.\"\"\"\n error = \"==> Error: %s\" % expected\n assert location(*specs, fail_on_error=False).strip() == error\n\n\n@pytest.mark.db\ndef test_location_stage_dir(mock_spec):\n \"\"\"Tests spack location --stage-dir.\"\"\"\n spec, pkg = mock_spec\n assert location(\"--stage-dir\", spec.name).strip() == pkg.stage.path\n\n\n@pytest.mark.db\ndef test_location_stages(mock_spec):\n \"\"\"Tests spack location --stages.\"\"\"\n assert location(\"--stages\").strip() == spack.stage.get_stage_root()\n\n\ndef test_location_specified_repo():\n \"\"\"Tests spack location --repo .\"\"\"\n with spack.repo.use_repositories(\n os.path.join(spack.paths.test_repos_path, \"spack_repo\", \"builtin_mock\"),\n os.path.join(spack.paths.test_repos_path, \"spack_repo\", \"builder_test\"),\n ):\n assert location(\"--repo\").strip() == spack.repo.PATH.get_repo(\"builtin_mock\").root\n assert (\n location(\"--repo\", \"builtin_mock\").strip()\n == spack.repo.PATH.get_repo(\"builtin_mock\").root\n )\n assert (\n location(\"--packages\", \"builder_test\").strip()\n == spack.repo.PATH.get_repo(\"builder_test\").root\n )\n assert (\n location(\"--repo\", \"nonexistent\", fail_on_error=False).strip()\n == \"==> Error: no such repository: 'nonexistent'\"\n )\n" }, { "path": "lib/spack/spack/test/cmd/logs.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport gzip\nimport os\nimport sys\nimport tempfile\nfrom contextlib import contextmanager\nfrom io import BytesIO, TextIOWrapper\n\nimport pytest\n\nimport spack.cmd.logs\nimport spack.concretize\nimport spack.error\nimport spack.spec\nfrom spack.main import SpackCommand\n\nlogs = SpackCommand(\"logs\")\ninstall = SpackCommand(\"install\")\n\n\n@contextmanager\ndef stdout_as_buffered_text_stream():\n \"\"\"Attempt to simulate \"typical\" interface for stdout when user is\n running Spack/Python from terminal. \"spack log\" should not be run\n for all possible cases of what stdout might look like, in\n particular some programmatic redirections of stdout like StringIO\n are not meant to be supported by this command; more-generally,\n mechanisms that depend on decoding binary output prior to write\n are not supported for \"spack log\".\n \"\"\"\n original_stdout = sys.stdout\n\n with tempfile.TemporaryFile(mode=\"w+b\") as tf:\n sys.stdout = TextIOWrapper(tf, encoding=\"utf-8\")\n try:\n yield tf\n finally:\n sys.stdout = original_stdout\n\n\ndef _rewind_collect_and_decode(rw_stream):\n rw_stream.seek(0)\n return rw_stream.read().decode(\"utf-8\")\n\n\ndef test_logs_cmd_errors(install_mockery, mock_fetch, mock_archive, mock_packages):\n spec = spack.concretize.concretize_one(\"pkg-c\")\n assert not spec.installed\n\n with pytest.raises(spack.error.SpackError, match=\"is not installed or staged\"):\n logs(\"pkg-c\")\n\n with pytest.raises(spack.error.SpackError, match=\"Too many specs\"):\n logs(\"pkg-c mpi\")\n\n install(\"pkg-c\")\n os.remove(spec.package.install_log_path)\n with pytest.raises(spack.error.SpackError, match=\"No logs are available\"):\n logs(\"pkg-c\")\n\n\ndef _write_string_to_path(string, path):\n \"\"\"Write a string to a file, preserving newline format in the string.\"\"\"\n with open(path, \"wb\") as f:\n f.write(string.encode(\"utf-8\"))\n\n\ndef test_dump_logs(install_mockery, mock_fetch, mock_archive, mock_packages):\n \"\"\"Test that ``spack log`` can find (and print) the logs for partial\n builds and completed installs.\n\n Also make sure that for compressed logs, that we automatically\n decompress them.\n \"\"\"\n cmdline_spec = spack.spec.Spec(\"libelf\")\n concrete_spec = spack.concretize.concretize_one(cmdline_spec)\n\n # Sanity check, make sure this test is checking what we want: to\n # start with\n assert not concrete_spec.installed\n\n stage_log_content = \"test_log stage output\\nanother line\"\n installed_log_content = \"test_log install output\\nhere to test multiple lines\"\n\n with concrete_spec.package.stage:\n _write_string_to_path(stage_log_content, concrete_spec.package.log_path)\n with stdout_as_buffered_text_stream() as redirected_stdout:\n spack.cmd.logs._logs(cmdline_spec, concrete_spec)\n assert _rewind_collect_and_decode(redirected_stdout) == stage_log_content\n\n install(\"--fake\", \"libelf\")\n\n # Sanity check: make sure a path is recorded, regardless of whether\n # it exists (if it does exist, we will overwrite it with content\n # in this test)\n assert concrete_spec.package.install_log_path\n\n with gzip.open(concrete_spec.package.install_log_path, \"wb\") as compressed_file:\n bstream = BytesIO(installed_log_content.encode(\"utf-8\"))\n compressed_file.writelines(bstream)\n\n with stdout_as_buffered_text_stream() as redirected_stdout:\n spack.cmd.logs._logs(cmdline_spec, concrete_spec)\n assert _rewind_collect_and_decode(redirected_stdout) == installed_log_content\n\n with concrete_spec.package.stage:\n _write_string_to_path(stage_log_content, concrete_spec.package.log_path)\n # We re-create the stage, but \"spack log\" should ignore that\n # if the package is installed\n with stdout_as_buffered_text_stream() as redirected_stdout:\n spack.cmd.logs._logs(cmdline_spec, concrete_spec)\n assert _rewind_collect_and_decode(redirected_stdout) == installed_log_content\n" }, { "path": "lib/spack/spack/test/cmd/maintainers.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport re\n\nimport pytest\n\nimport spack.main\nimport spack.repo\n\npytestmark = [pytest.mark.usefixtures(\"mock_packages\")]\n\nmaintainers = spack.main.SpackCommand(\"maintainers\")\n\nMAINTAINED_PACKAGES = [\n \"gcc-runtime\",\n \"maintainers-1\",\n \"maintainers-2\",\n \"maintainers-3\",\n \"py-extension1\",\n]\n\n\ndef split(output):\n \"\"\"Split command line output into an array.\"\"\"\n output = output.strip()\n return re.split(r\"\\s+\", output) if output else []\n\n\ndef test_maintained():\n out = split(maintainers(\"--maintained\"))\n assert out == MAINTAINED_PACKAGES\n\n\ndef test_unmaintained():\n out = split(maintainers(\"--unmaintained\"))\n assert out == sorted(set(spack.repo.all_package_names()) - set(MAINTAINED_PACKAGES))\n\n\ndef test_all():\n out = split(maintainers(\"--all\"))\n assert out == [\n \"gcc-runtime:\",\n \"haampie\",\n \"maintainers-1:\",\n \"user1,\",\n \"user2\",\n \"maintainers-2:\",\n \"user2,\",\n \"user3\",\n \"maintainers-3:\",\n \"user0,\",\n \"user1,\",\n \"user2,\",\n \"user3\",\n \"py-extension1:\",\n \"user1,\",\n \"user2\",\n ]\n\n out = split(maintainers(\"--all\", \"maintainers-1\"))\n assert out == [\"maintainers-1:\", \"user1,\", \"user2\"]\n\n\ndef test_all_by_user():\n out = split(maintainers(\"--all\", \"--by-user\"))\n assert out == [\n \"haampie:\",\n \"gcc-runtime\",\n \"user0:\",\n \"maintainers-3\",\n \"user1:\",\n \"maintainers-1,\",\n \"maintainers-3,\",\n \"py-extension1\",\n \"user2:\",\n \"maintainers-1,\",\n \"maintainers-2,\",\n \"maintainers-3,\",\n \"py-extension1\",\n \"user3:\",\n \"maintainers-2,\",\n \"maintainers-3\",\n ]\n\n out = split(maintainers(\"--all\", \"--by-user\", \"user1\", \"user2\"))\n assert out == [\n \"user1:\",\n \"maintainers-1,\",\n \"maintainers-3,\",\n \"py-extension1\",\n \"user2:\",\n \"maintainers-1,\",\n \"maintainers-2,\",\n \"maintainers-3,\",\n \"py-extension1\",\n ]\n\n\ndef test_no_args():\n with pytest.raises(spack.main.SpackCommandError):\n maintainers()\n\n\ndef test_no_args_by_user():\n with pytest.raises(spack.main.SpackCommandError):\n maintainers(\"--by-user\")\n\n\ndef test_mutex_args_fail():\n with pytest.raises(spack.main.SpackCommandError):\n maintainers(\"--maintained\", \"--unmaintained\")\n\n\ndef test_maintainers_list_packages():\n out = split(maintainers(\"maintainers-1\"))\n assert out == [\"user1\", \"user2\"]\n\n out = split(maintainers(\"maintainers-1\", \"maintainers-2\"))\n assert out == [\"user1\", \"user2\", \"user3\"]\n\n out = split(maintainers(\"maintainers-2\"))\n assert out == [\"user2\", \"user3\"]\n\n\ndef test_maintainers_list_fails():\n out = maintainers(\"pkg-a\", fail_on_error=False)\n assert not out\n assert maintainers.returncode == 1\n\n\ndef test_maintainers_list_by_user():\n out = split(maintainers(\"--by-user\", \"user1\"))\n assert out == [\"maintainers-1\", \"maintainers-3\", \"py-extension1\"]\n\n out = split(maintainers(\"--by-user\", \"user1\", \"user2\"))\n assert out == [\"maintainers-1\", \"maintainers-2\", \"maintainers-3\", \"py-extension1\"]\n\n out = split(maintainers(\"--by-user\", \"user2\"))\n assert out == [\"maintainers-1\", \"maintainers-2\", \"maintainers-3\", \"py-extension1\"]\n\n out = split(maintainers(\"--by-user\", \"user3\"))\n assert out == [\"maintainers-2\", \"maintainers-3\"]\n" }, { "path": "lib/spack/spack/test/cmd/mark.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport pytest\n\nimport spack.store\nfrom spack.main import SpackCommand, SpackCommandError\n\ngc = SpackCommand(\"gc\")\nmark = SpackCommand(\"mark\")\ninstall = SpackCommand(\"install\")\nuninstall = SpackCommand(\"uninstall\")\n\n# Unit tests should not be affected by the user's managed environments\npytestmark = pytest.mark.usefixtures(\"mutable_mock_env_path\")\n\n\n@pytest.mark.db\ndef test_mark_mode_required(mutable_database):\n with pytest.raises(SpackCommandError):\n mark(\"-a\")\n\n\n@pytest.mark.db\ndef test_mark_spec_required(mutable_database):\n with pytest.raises(SpackCommandError):\n mark(\"-i\")\n\n\n@pytest.mark.db\ndef test_mark_all_explicit(mutable_database):\n mark(\"-e\", \"-a\")\n gc(\"-y\")\n all_specs = spack.store.STORE.layout.all_specs()\n assert len(all_specs) == 17\n\n\n@pytest.mark.db\ndef test_mark_all_implicit(mutable_database):\n mark(\"-i\", \"-a\")\n gc(\"-y\")\n all_specs = spack.store.STORE.layout.all_specs()\n assert len(all_specs) == 0\n\n\n@pytest.mark.db\ndef test_mark_one_explicit(mutable_database):\n mark(\"-e\", \"libelf\")\n uninstall(\"-y\", \"-a\", \"mpileaks\")\n gc(\"-y\")\n all_specs = spack.store.STORE.layout.all_specs()\n assert len(all_specs) == 4\n\n\n@pytest.mark.db\ndef test_mark_one_implicit(mutable_database):\n mark(\"-i\", \"externaltest\")\n gc(\"-y\")\n all_specs = spack.store.STORE.layout.all_specs()\n assert len(all_specs) == 15\n\n\n@pytest.mark.db\ndef test_mark_all_implicit_then_explicit(mutable_database):\n mark(\"-i\", \"-a\")\n mark(\"-e\", \"-a\")\n gc(\"-y\")\n all_specs = spack.store.STORE.layout.all_specs()\n assert len(all_specs) == 17\n" }, { "path": "lib/spack/spack/test/cmd/mirror.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport pathlib\n\nimport pytest\n\nimport spack.binary_distribution\nimport spack.cmd.mirror\nimport spack.concretize\nimport spack.config\nimport spack.environment as ev\nimport spack.error\nimport spack.mirrors.utils\nimport spack.package_base\nimport spack.spec\nimport spack.util.git\nimport spack.util.url as url_util\nimport spack.version\nfrom spack.main import SpackCommand, SpackCommandError\nfrom spack.mirrors.utils import MirrorStatsForAllSpecs, MirrorStatsForOneSpec\n\nconfig = SpackCommand(\"config\")\nmirror = SpackCommand(\"mirror\")\nenv = SpackCommand(\"env\")\nadd = SpackCommand(\"add\")\nconcretize = SpackCommand(\"concretize\")\ninstall = SpackCommand(\"install\")\nbuildcache = SpackCommand(\"buildcache\")\nuninstall = SpackCommand(\"uninstall\")\n\npytestmark = pytest.mark.not_on_windows(\"does not run on windows\")\n\n\n@pytest.mark.disable_clean_stage_check\n@pytest.mark.regression(\"8083\")\ndef test_regression_8083(tmp_path: pathlib.Path, mock_packages, mock_fetch, config):\n output = mirror(\"create\", \"-d\", str(tmp_path), \"externaltool\")\n assert \"Skipping\" in output\n assert \"as it is an external spec\" in output\n\n\n# Unit tests should not be affected by the user's managed environments\n@pytest.mark.regression(\"12345\")\ndef test_mirror_from_env(mutable_mock_env_path, tmp_path: pathlib.Path, mock_packages, mock_fetch):\n mirror_dir = str(tmp_path / \"mirror\")\n env_name = \"test\"\n\n env(\"create\", env_name)\n with ev.read(env_name):\n add(\"trivial-install-test-package\")\n add(\"git-test\")\n concretize()\n with spack.config.override(\"config:checksum\", False):\n mirror(\"create\", \"-d\", mirror_dir, \"--all\")\n\n e = ev.read(env_name)\n assert set(os.listdir(mirror_dir)) == set([s.name for s in e.user_specs])\n for spec in e.specs_by_hash.values():\n mirror_res = os.listdir(os.path.join(mirror_dir, spec.name))\n expected = [\"%s.tar.gz\" % spec.format(\"{name}-{version}\")]\n assert mirror_res == expected\n\n\ndef test_mirror_cli_parallel_args(\n tmp_path, mock_packages, mock_fetch, mutable_mock_env_path, monkeypatch\n):\n \"\"\"Test the CLI parallel args\"\"\"\n mirror_dir = str(tmp_path / \"mirror\")\n env_name = \"test-parallel\"\n\n def mock_create_mirror_for_all_specs(mirror_specs, path, skip_unstable_versions, workers):\n assert path == mirror_dir\n assert workers == 2\n\n monkeypatch.setattr(\n spack.cmd.mirror, \"create_mirror_for_all_specs\", mock_create_mirror_for_all_specs\n )\n\n env(\"create\", env_name)\n with ev.read(env_name):\n add(\"trivial-install-test-package\")\n add(\"git-test\")\n concretize()\n with spack.config.override(\"config:checksum\", False):\n mirror(\"create\", \"-d\", mirror_dir, \"--all\", \"-j\", \"2\")\n\n\ndef test_mirror_from_env_parallel(tmp_path, mock_packages, mock_fetch, mutable_mock_env_path):\n \"\"\"Directly test create_mirror_for_all_specs with parallel option\"\"\"\n mirror_dir = str(tmp_path / \"mirror\")\n env_name = \"test-parallel\"\n\n env(\"create\", env_name)\n with ev.read(env_name):\n add(\"trivial-install-test-package\")\n add(\"git-test\")\n concretize()\n\n e = ev.read(env_name)\n specs = list(e.specs_by_hash.values())\n\n with spack.config.override(\"config:checksum\", False):\n mirror_stats = spack.cmd.mirror.create_mirror_for_all_specs(\n specs, mirror_dir, False, workers=2\n )\n\n assert len(mirror_stats.errors) == 0\n assert set(os.listdir(mirror_dir)) == set([s.name for s in e.user_specs])\n for spec in e.specs_by_hash.values():\n mirror_res = os.listdir(os.path.join(mirror_dir, spec.name))\n expected = [\"%s.tar.gz\" % spec.format(\"{name}-{version}\")]\n assert mirror_res == expected\n\n\ndef test_mirror_stats_merge():\n \"\"\"Test MirrorStats merge functionality\"\"\"\n spec1 = \"package@1.0\"\n spec2 = \"package@2.0\"\n spec3 = \"package@3.0\"\n\n s1 = MirrorStatsForOneSpec(spec1)\n s1.added(\"/test/path/1\")\n s1.added(\"/test/path/2\")\n s1.finalize()\n\n s2 = MirrorStatsForOneSpec(spec2)\n s2.already_existed(\"/test/path/3\")\n s2.finalize()\n\n all_stats = MirrorStatsForAllSpecs()\n\n # Check before merge, should be empty\n present, mirrored, errors = all_stats.stats()\n assert len(present) == 0\n assert len(mirrored) == 0\n assert len(errors) == 0\n\n # Merge package 1 and 2\n all_stats.merge(s1)\n all_stats.merge(s2)\n\n # Check after merge\n present, mirrored, errors = all_stats.stats()\n assert present.count(spec2) == 1\n assert mirrored.count(spec1) == 1\n assert len(present) == 1\n assert len(mirrored) == 1\n assert len(errors) == 0\n\n # Merge package 3\n s3 = MirrorStatsForOneSpec(spec3)\n s3.already_existed(\"/test/path/4\")\n s3.added(\"/test/path/5\")\n s3.finalize()\n all_stats.merge(s3)\n\n present, mirrored, errors = all_stats.stats()\n assert present.count(spec3) == 1\n assert mirrored.count(spec3) == 1\n assert len(present) == 2\n assert len(mirrored) == 2\n assert len(errors) == 0\n\n\n# Test for command line-specified spec in concretized environment\ndef test_mirror_spec_from_env(\n mutable_mock_env_path, tmp_path: pathlib.Path, mock_packages, mock_fetch\n):\n mirror_dir = str(tmp_path / \"mirror-B\")\n env_name = \"test\"\n\n env(\"create\", env_name)\n with ev.read(env_name):\n add(\"simple-standalone-test@0.9\")\n concretize()\n with spack.config.override(\"config:checksum\", False):\n mirror(\"create\", \"-d\", mirror_dir, \"simple-standalone-test\")\n\n e = ev.read(env_name)\n assert set(os.listdir(mirror_dir)) == set([s.name for s in e.user_specs])\n spec = e.concrete_roots()[0]\n mirror_res = os.listdir(os.path.join(mirror_dir, spec.name))\n expected = [\"%s.tar.gz\" % spec.format(\"{name}-{version}\")]\n assert mirror_res == expected\n\n\n@pytest.fixture\ndef source_for_pkg_with_hash(mock_packages, tmp_path: pathlib.Path):\n s = spack.concretize.concretize_one(\"trivial-pkg-with-valid-hash\")\n local_url_basename = os.path.basename(s.package.url)\n local_path = tmp_path / local_url_basename\n local_path.write_text(s.package.hashed_content, encoding=\"utf-8\")\n local_url = url_util.path_to_file_url(str(local_path))\n s.package.versions[spack.version.Version(\"1.0\")][\"url\"] = local_url\n\n\ndef test_mirror_skip_unstable(\n tmp_path_factory: pytest.TempPathFactory, mock_packages, config, source_for_pkg_with_hash\n):\n mirror_dir = str(tmp_path_factory.mktemp(\"mirror-dir\"))\n\n specs = [\n spack.concretize.concretize_one(x) for x in [\"git-test\", \"trivial-pkg-with-valid-hash\"]\n ]\n spack.cmd.mirror.create(mirror_dir, specs, skip_unstable_versions=True)\n\n assert set(os.listdir(mirror_dir)) - set([\"_source-cache\"]) == set(\n [\"trivial-pkg-with-valid-hash\"]\n )\n\n\nclass MockMirrorArgs:\n def __init__(\n self,\n specs=None,\n all=False,\n file=None,\n versions_per_spec=None,\n dependencies=False,\n exclude_file=None,\n exclude_specs=None,\n directory=None,\n private=False,\n ):\n self.specs = specs or []\n self.all = all\n self.file = file\n self.versions_per_spec = versions_per_spec\n self.dependencies = dependencies\n self.exclude_file = exclude_file\n self.exclude_specs = exclude_specs\n self.private = private\n self.directory = directory\n\n\ndef test_exclude_specs(mock_packages, config):\n args = MockMirrorArgs(\n specs=[\"mpich\"], versions_per_spec=\"all\", exclude_specs=\"mpich@3.0.1:3.0.2 mpich@1.0\"\n )\n\n mirror_specs = spack.cmd.mirror._specs_to_mirror(args)\n expected_include = set(\n spack.concretize.concretize_one(x) for x in [\"mpich@3.0.3\", \"mpich@3.0.4\", \"mpich@3.0\"]\n )\n expected_exclude = set(spack.spec.Spec(x) for x in [\"mpich@3.0.1\", \"mpich@3.0.2\", \"mpich@1.0\"])\n assert expected_include <= set(mirror_specs)\n assert not any(spec.satisfies(y) for spec in mirror_specs for y in expected_exclude)\n\n\ndef test_exclude_specs_public_mirror(mock_packages, config):\n args = MockMirrorArgs(\n specs=[\"no-redistribute-dependent\"],\n versions_per_spec=\"all\",\n dependencies=True,\n private=False,\n )\n\n mirror_specs = spack.cmd.mirror._specs_to_mirror(args)\n assert not any(s.name == \"no-redistribute\" for s in mirror_specs)\n assert any(s.name == \"no-redistribute-dependent\" for s in mirror_specs)\n\n\ndef test_exclude_file(mock_packages, tmp_path: pathlib.Path, config):\n exclude_path = tmp_path / \"test-exclude.txt\"\n exclude_path.write_text(\n \"\"\"\\\nmpich@3.0.1:3.0.2\nmpich@1.0\n\"\"\",\n encoding=\"utf-8\",\n )\n\n args = MockMirrorArgs(specs=[\"mpich\"], versions_per_spec=\"all\", exclude_file=str(exclude_path))\n\n mirror_specs = spack.cmd.mirror._specs_to_mirror(args)\n expected_include = set(\n spack.concretize.concretize_one(x) for x in [\"mpich@3.0.3\", \"mpich@3.0.4\", \"mpich@3.0\"]\n )\n expected_exclude = set(spack.spec.Spec(x) for x in [\"mpich@3.0.1\", \"mpich@3.0.2\", \"mpich@1.0\"])\n assert expected_include <= set(mirror_specs)\n assert not any(spec.satisfies(y) for spec in mirror_specs for y in expected_exclude)\n\n\ndef test_mirror_remove_by_scope(mutable_config, tmp_path: pathlib.Path):\n # add a new mirror to two scopes\n mirror(\"add\", \"--scope=site\", \"mock\", str(tmp_path / \"mock_mirror\"))\n mirror(\"add\", \"--scope=system\", \"mock\", str(tmp_path / \"mock_mirror\"))\n\n # Confirm that it is not removed when the scope is incorrect\n with pytest.raises(SpackCommandError):\n mirror(\"remove\", \"--scope=user\", \"mock\")\n output = mirror(\"list\")\n assert \"mock\" in output\n\n # Confirm that when the scope is specified, it is only removed from that scope\n mirror(\"remove\", \"--scope=site\", \"mock\")\n site_output = mirror(\"list\", \"--scope=site\")\n system_output = mirror(\"list\", \"--scope=system\")\n assert \"mock\" not in site_output\n assert \"mock\" in system_output\n\n # Confirm that when the scope is not specified, it is removed from top scope\n mirror(\"add\", \"--scope=site\", \"mock\", str(tmp_path / \"mock_mirror\"))\n mirror(\"remove\", \"mock\")\n site_output = mirror(\"list\", \"--scope=site\")\n system_output = mirror(\"list\", \"--scope=system\")\n assert \"mock\" not in site_output\n assert \"mock\" in system_output\n\n # Check that the `--all-scopes` option works\n mirror(\"add\", \"--scope=site\", \"mock\", str(tmp_path / \"mockrepo\"))\n mirror(\"remove\", \"--all-scopes\", \"mock\")\n output = mirror(\"list\")\n assert \"mock\" not in output\n\n\ndef test_mirror_crud(mutable_config):\n mirror(\"add\", \"mirror\", \"http://spack.io\")\n\n output = mirror(\"remove\", \"mirror\")\n assert \"Removed mirror\" in output\n\n mirror(\"add\", \"mirror\", \"http://spack.io\")\n\n # no-op\n output = mirror(\"set-url\", \"mirror\", \"http://spack.io\")\n assert \"No changes made\" in output\n\n output = mirror(\"set-url\", \"--push\", \"mirror\", \"s3://spack-public\")\n assert not output\n\n # no-op\n output = mirror(\"set-url\", \"--push\", \"mirror\", \"s3://spack-public\")\n assert \"No changes made\" in output\n\n output = mirror(\"remove\", \"mirror\")\n assert \"Removed mirror\" in output\n\n # Test S3 connection info token as variable\n mirror(\"add\", \"--s3-access-token-variable\", \"aaaaaazzzzz\", \"mirror\", \"s3://spack-public\")\n\n output = mirror(\"remove\", \"mirror\")\n assert \"Removed mirror\" in output\n\n def do_add_set_seturl_access_pair(\n id_arg, secret_arg, mirror_name=\"mirror\", mirror_url=\"s3://spack-public\"\n ):\n # Test connection info id/key\n output = mirror(\"add\", id_arg, \"foo\", secret_arg, \"bar\", mirror_name, mirror_url)\n\n output = config(\"blame\", \"mirrors\")\n assert all([x in output for x in (\"foo\", \"bar\", mirror_name, mirror_url)])\n\n output = mirror(\"set\", id_arg, \"foo_set\", secret_arg, \"bar_set\", mirror_name)\n output = config(\"blame\", \"mirrors\")\n assert all([x in output for x in (\"foo_set\", \"bar_set\", mirror_name, mirror_url)])\n if \"variable\" not in secret_arg:\n output = mirror(\n \"set\", id_arg, \"foo_set\", secret_arg + \"-variable\", \"bar_set_var\", mirror_name\n )\n assert \"support for plain text secrets\" not in output\n output = config(\"blame\", \"mirrors\")\n assert all([x in output for x in (\"foo_set\", \"bar_set_var\", mirror_name, mirror_url)])\n\n output = mirror(\n \"set-url\",\n id_arg,\n \"foo_set_url\",\n secret_arg,\n \"bar_set_url\",\n \"--push\",\n mirror_name,\n mirror_url + \"-push\",\n )\n output = config(\"blame\", \"mirrors\")\n assert all(\n [\n x in output\n for x in (\"foo_set_url\", \"bar_set_url\", mirror_name, mirror_url + \"-push\")\n ]\n )\n\n output = mirror(\"set\", id_arg, \"a\", mirror_name)\n assert \"No changes made to mirror\" not in output\n\n output = mirror(\"set\", secret_arg, \"b\", mirror_name)\n assert \"No changes made to mirror\" not in output\n\n output = mirror(\"set-url\", id_arg, \"c\", mirror_name, mirror_url)\n assert \"No changes made to mirror\" not in output\n\n output = mirror(\"set-url\", secret_arg, \"d\", mirror_name, mirror_url)\n assert \"No changes made to mirror\" not in output\n\n output = mirror(\"remove\", mirror_name)\n assert \"Removed mirror\" in output\n\n output = mirror(\"add\", id_arg, \"foo\", mirror_name, mirror_url)\n assert \"Expected both parts of the access pair to be specified. \" in output\n\n output = mirror(\"set-url\", id_arg, \"bar\", mirror_name, mirror_url)\n assert \"Expected both parts of the access pair to be specified. \" in output\n\n output = mirror(\"set\", id_arg, \"bar\", mirror_name)\n assert \"Expected both parts of the access pair to be specified. \" in output\n\n output = mirror(\"remove\", mirror_name)\n assert \"Removed mirror\" in output\n\n output = mirror(\"add\", secret_arg, \"bar\", mirror_name, mirror_url)\n assert \"Expected both parts of the access pair to be specified. \" in output\n\n output = mirror(\"set-url\", secret_arg, \"bar\", mirror_name, mirror_url)\n assert \"Expected both parts of the access pair to be specified. \" in output\n\n output = mirror(\"set\", secret_arg, \"bar\", mirror_name)\n assert \"Expected both parts of the access pair to be specified. \" in output\n\n output = mirror(\"remove\", mirror_name)\n assert \"Removed mirror\" in output\n\n output = mirror(\"list\")\n assert \"No mirrors configured\" in output\n\n do_add_set_seturl_access_pair(\"--s3-access-key-id\", \"--s3-access-key-secret-variable\")\n do_add_set_seturl_access_pair(\"--s3-access-key-id-variable\", \"--s3-access-key-secret-variable\")\n\n # Test OCI connection info user/password\n do_add_set_seturl_access_pair(\"--oci-username\", \"--oci-password-variable\")\n do_add_set_seturl_access_pair(\"--oci-username-variable\", \"--oci-password-variable\")\n\n # Test S3 connection info with endpoint URL\n mirror(\n \"add\",\n \"--s3-access-token-variable\",\n \"aaaaaazzzzz\",\n \"--s3-endpoint-url\",\n \"http://localhost/\",\n \"mirror\",\n \"s3://spack-public\",\n )\n\n output = mirror(\"remove\", \"mirror\")\n assert \"Removed mirror\" in output\n\n output = mirror(\"list\")\n assert \"No mirrors configured\" in output\n\n # Test GCS Mirror\n mirror(\"add\", \"mirror\", \"gs://spack-test\")\n\n output = mirror(\"remove\", \"mirror\")\n assert \"Removed mirror\" in output\n\n output = mirror(\"list\")\n assert \"No mirrors configured\" in output\n\n\ndef test_mirror_nonexisting(mutable_config):\n with pytest.raises(SpackCommandError):\n mirror(\"remove\", \"not-a-mirror\")\n\n with pytest.raises(SpackCommandError):\n mirror(\"set-url\", \"not-a-mirror\", \"http://spack.io\")\n\n\ndef test_mirror_name_collision(mutable_config):\n mirror(\"add\", \"first\", \"1\")\n\n with pytest.raises(SpackCommandError):\n mirror(\"add\", \"first\", \"1\")\n\n\n# Unit tests should not be affected by the user's managed environments\ndef test_mirror_destroy(\n mutable_mock_env_path,\n install_mockery,\n mock_packages,\n mock_fetch,\n mock_archive,\n mutable_config,\n monkeypatch,\n tmp_path: pathlib.Path,\n):\n # Create a temp mirror directory for buildcache usage\n mirror_dir = tmp_path / \"mirror_dir\"\n mirror_url = mirror_dir.as_uri()\n mirror(\"add\", \"atest\", mirror_url)\n\n spec_name = \"libdwarf\"\n\n # Put a binary package in a buildcache\n install(\"--fake\", \"--no-cache\", spec_name)\n buildcache(\"push\", \"-u\", \"-f\", str(mirror_dir), spec_name)\n\n blobs_path = spack.binary_distribution.buildcache_relative_blobs_path()\n\n contents = os.listdir(str(mirror_dir))\n assert blobs_path in contents\n\n # Destroy mirror by name\n mirror(\"destroy\", \"-m\", \"atest\")\n\n assert not os.path.exists(str(mirror_dir))\n\n uninstall(\"-y\", spec_name)\n mirror(\"remove\", \"atest\")\n\n\n@pytest.mark.usefixtures(\"mock_packages\")\nclass TestMirrorCreate:\n @pytest.mark.regression(\"31736\", \"31985\")\n def test_all_specs_with_all_versions_dont_concretize(self):\n args = MockMirrorArgs(all=True, exclude_file=None, exclude_specs=None)\n mirror_specs = spack.cmd.mirror._specs_to_mirror(args)\n assert all(not s.concrete for s in mirror_specs)\n\n @pytest.mark.parametrize(\n \"cli_args,error_str\",\n [\n # Passed more than one among -f --all\n (\n {\"specs\": None, \"file\": \"input.txt\", \"all\": True},\n \"cannot specify specs with a file if\",\n ),\n (\n {\"specs\": \"hdf5\", \"file\": \"input.txt\", \"all\": False},\n \"cannot specify specs with a file AND\",\n ),\n ({\"specs\": None, \"file\": None, \"all\": False}, \"no packages were specified\"),\n # Passed -n along with --all\n (\n {\"specs\": None, \"file\": None, \"all\": True, \"versions_per_spec\": 2},\n \"cannot specify '--versions_per-spec'\",\n ),\n ],\n )\n def test_error_conditions(self, cli_args, error_str):\n args = MockMirrorArgs(**cli_args)\n with pytest.raises(spack.error.SpackError, match=error_str):\n spack.cmd.mirror.mirror_create(args)\n\n @pytest.mark.parametrize(\n \"cli_args,not_expected\",\n [\n (\n {\n \"specs\": \"boost bowtie callpath\",\n \"exclude_specs\": \"bowtie\",\n \"dependencies\": False,\n },\n [\"bowtie\"],\n ),\n (\n {\n \"specs\": \"boost bowtie callpath\",\n \"exclude_specs\": \"bowtie callpath\",\n \"dependencies\": False,\n },\n [\"bowtie\", \"callpath\"],\n ),\n (\n {\n \"specs\": \"boost bowtie callpath\",\n \"exclude_specs\": \"bowtie\",\n \"dependencies\": True,\n },\n [\"bowtie\"],\n ),\n ],\n )\n def test_exclude_specs_from_user(self, cli_args, not_expected, config):\n mirror_specs = spack.cmd.mirror._specs_to_mirror(MockMirrorArgs(**cli_args))\n assert not any(s.satisfies(y) for s in mirror_specs for y in not_expected)\n\n @pytest.mark.parametrize(\"abstract_specs\", [(\"bowtie\", \"callpath\")])\n def test_specs_from_cli_are_the_same_as_from_file(\n self, abstract_specs, config, tmp_path: pathlib.Path\n ):\n args = MockMirrorArgs(specs=\" \".join(abstract_specs))\n specs_from_cli = spack.cmd.mirror.concrete_specs_from_user(args)\n\n input_file = tmp_path / \"input.txt\"\n input_file.write_text(\"\\n\".join(abstract_specs))\n args = MockMirrorArgs(file=str(input_file))\n specs_from_file = spack.cmd.mirror.concrete_specs_from_user(args)\n\n assert specs_from_cli == specs_from_file\n\n @pytest.mark.parametrize(\n \"input_specs,nversions\",\n [(\"callpath\", 1), (\"mpich\", 4), (\"callpath mpich\", 3), (\"callpath mpich\", \"all\")],\n )\n def test_versions_per_spec_produces_concrete_specs(self, input_specs, nversions, config):\n args = MockMirrorArgs(specs=input_specs, versions_per_spec=nversions)\n specs = spack.cmd.mirror.concrete_specs_from_user(args)\n assert all(s.concrete for s in specs)\n\n\ndef test_mirror_type(mutable_config):\n \"\"\"Test the mirror set command\"\"\"\n mirror(\"add\", \"example\", \"--type\", \"binary\", \"http://example.com\")\n assert spack.config.get(\"mirrors:example\") == {\n \"url\": \"http://example.com\",\n \"source\": False,\n \"binary\": True,\n }\n\n mirror(\"set\", \"example\", \"--type\", \"source\")\n assert spack.config.get(\"mirrors:example\") == {\n \"url\": \"http://example.com\",\n \"source\": True,\n \"binary\": False,\n }\n\n mirror(\"set\", \"example\", \"--type\", \"binary\")\n assert spack.config.get(\"mirrors:example\") == {\n \"url\": \"http://example.com\",\n \"source\": False,\n \"binary\": True,\n }\n mirror(\"set\", \"example\", \"--type\", \"binary\", \"--type\", \"source\")\n assert spack.config.get(\"mirrors:example\") == {\n \"url\": \"http://example.com\",\n \"source\": True,\n \"binary\": True,\n }\n\n\ndef test_mirror_set_2(mutable_config):\n \"\"\"Test the mirror set command\"\"\"\n mirror(\"add\", \"example\", \"http://example.com\")\n mirror(\n \"set\",\n \"example\",\n \"--push\",\n \"--url\",\n \"http://example2.com\",\n \"--s3-access-key-id\",\n \"username\",\n \"--s3-access-key-secret-variable\",\n \"password\",\n )\n\n assert spack.config.get(\"mirrors:example\") == {\n \"url\": \"http://example.com\",\n \"push\": {\n \"url\": \"http://example2.com\",\n \"access_pair\": {\"id\": \"username\", \"secret_variable\": \"password\"},\n },\n }\n\n\ndef test_mirror_add_set_signed(mutable_config):\n mirror(\"add\", \"--signed\", \"example\", \"http://example.com\")\n assert spack.config.get(\"mirrors:example\") == {\"url\": \"http://example.com\", \"signed\": True}\n mirror(\"set\", \"--unsigned\", \"example\")\n assert spack.config.get(\"mirrors:example\") == {\"url\": \"http://example.com\", \"signed\": False}\n mirror(\"set\", \"--signed\", \"example\")\n assert spack.config.get(\"mirrors:example\") == {\"url\": \"http://example.com\", \"signed\": True}\n\n\ndef test_mirror_add_set_autopush(mutable_config):\n # Add mirror without autopush\n mirror(\"add\", \"example\", \"http://example.com\")\n assert spack.config.get(\"mirrors:example\") == \"http://example.com\"\n mirror(\"set\", \"--no-autopush\", \"example\")\n assert spack.config.get(\"mirrors:example\") == {\"url\": \"http://example.com\", \"autopush\": False}\n mirror(\"set\", \"--autopush\", \"example\")\n assert spack.config.get(\"mirrors:example\") == {\"url\": \"http://example.com\", \"autopush\": True}\n mirror(\"set\", \"--no-autopush\", \"example\")\n assert spack.config.get(\"mirrors:example\") == {\"url\": \"http://example.com\", \"autopush\": False}\n mirror(\"remove\", \"example\")\n\n # Add mirror with autopush\n mirror(\"add\", \"--autopush\", \"example\", \"http://example.com\")\n assert spack.config.get(\"mirrors:example\") == {\"url\": \"http://example.com\", \"autopush\": True}\n mirror(\"set\", \"--autopush\", \"example\")\n assert spack.config.get(\"mirrors:example\") == {\"url\": \"http://example.com\", \"autopush\": True}\n mirror(\"set\", \"--no-autopush\", \"example\")\n assert spack.config.get(\"mirrors:example\") == {\"url\": \"http://example.com\", \"autopush\": False}\n mirror(\"set\", \"--autopush\", \"example\")\n assert spack.config.get(\"mirrors:example\") == {\"url\": \"http://example.com\", \"autopush\": True}\n mirror(\"remove\", \"example\")\n\n\n@pytest.mark.require_provenance\n@pytest.mark.disable_clean_stage_check\n@pytest.mark.parametrize(\"mirror_knows_commit\", (True, False))\ndef test_git_provenance_url_fails_mirror_resolves_commit(\n git,\n mock_git_repository,\n mock_packages,\n monkeypatch,\n tmp_path: pathlib.Path,\n mutable_config,\n mirror_knows_commit,\n):\n \"\"\"Extract git commit from a source mirror since other methods failed\"\"\"\n repo_path = mock_git_repository.path\n monkeypatch.setattr(\n spack.package_base.PackageBase, \"git\", f\"file://{repo_path}\", raising=False\n )\n monkeypatch.setattr(spack.util.git, \"get_commit_sha\", lambda x, y: None, raising=False)\n\n gold_commit = git(\"-C\", repo_path, \"rev-parse\", \"main\", output=str).strip()\n # create a fake mirror\n mirror_path = str(tmp_path / \"test-mirror\")\n if mirror_knows_commit:\n mirror(\"create\", \"-d\", mirror_path, f\"git-test-commit@main commit={gold_commit}\")\n else:\n mirror(\"create\", \"-d\", mirror_path, \"git-test-commit@main\")\n mirror(\"add\", \"--type\", \"source\", \"test-mirror\", mirror_path)\n\n spec = spack.concretize.concretize_one(\"git-test-commit@main\")\n\n assert spec.package.fetcher.source_id() == gold_commit\n assert \"commit\" in spec.variants\n assert spec.variants[\"commit\"].value == gold_commit\n\n\n@pytest.mark.require_provenance\n@pytest.mark.disable_clean_stage_check\ndef test_git_provenance_relative_to_mirror(\n git, mock_git_version_info, mock_packages, monkeypatch, tmp_path: pathlib.Path, mutable_config\n):\n \"\"\"Integration test to evaluate how commit resolution should behave with a mirror\n\n We want to confirm that the mirror doesn't break users ability to get a more recent commit\n Use `mock_git_version_info` repo because it has function scope and we can mess with the git\n history.\n \"\"\"\n repo_path, _, _ = mock_git_version_info\n monkeypatch.setattr(\n spack.package_base.PackageBase, \"git\", f\"file://{repo_path}\", raising=False\n )\n\n # create a fake mirror\n mirror_path = str(tmp_path / \"test-mirror\")\n mirror(\"create\", \"-d\", mirror_path, \"git-test-commit@main\")\n mirror(\"add\", \"--type\", \"source\", \"test-mirror\", mirror_path)\n mirror_commit = git(\"-C\", repo_path, \"rev-parse\", \"main\", output=str).strip()\n\n # push the commit past mirror\n git(\"-C\", repo_path, \"checkout\", \"main\", output=str)\n git(\"-C\", repo_path, \"commit\", \"--no-gpg-sign\", \"--allow-empty\", \"-m\", \"bump sha\")\n head_commit = git(\"-C\", repo_path, \"rev-parse\", \"main\", output=str).strip()\n\n spec_mirror = spack.concretize.concretize_one(\"git-test-commit@main\")\n assert spec_mirror.variants[\"commit\"].value == mirror_commit\n\n spec_head = spack.concretize.concretize_one(f\"git-test-commit@main commit={head_commit}\")\n assert spec_head.variants[\"commit\"].value == head_commit\n\n\n@pytest.mark.usefixtures(\"mock_packages\")\ndef test_mirror_skip_placeholder_pkg(tmp_path: pathlib.Path):\n \"\"\"Test a placeholder package which should skip during mirror all\"\"\"\n from spack.repo import PATH\n\n spec = spack.spec.Spec(\"placeholder@1.5\")\n pkg_cls = PATH.get_pkg_class(spec.name)\n pkg_obj = pkg_cls(spec)\n mirror_cache = spack.mirrors.utils.get_mirror_cache(str(tmp_path))\n mirror_stats = spack.mirrors.utils.MirrorStatsForOneSpec(spec)\n result = spack.mirrors.utils.create_mirror_from_package_object(\n pkg_obj, mirror_cache, mirror_stats\n )\n assert result is False\n assert not mirror_stats.errors\n" }, { "path": "lib/spack/spack/test/cmd/module.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport re\n\nimport pytest\n\nimport spack.concretize\nimport spack.config\nimport spack.main\nimport spack.modules\nimport spack.modules.lmod\nimport spack.repo\nimport spack.store\nfrom spack.installer import PackageInstaller\n\nmodule = spack.main.SpackCommand(\"module\")\n\npytestmark = pytest.mark.not_on_windows(\"does not run on windows\")\n\n\n#: make sure module files are generated for all the tests here\n@pytest.fixture(scope=\"module\", autouse=True)\ndef ensure_module_files_are_there(mock_packages_repo, mock_store, mock_configuration_scopes):\n \"\"\"Generate module files for module tests.\"\"\"\n module = spack.main.SpackCommand(\"module\")\n with spack.store.use_store(str(mock_store)):\n with spack.config.use_configuration(*mock_configuration_scopes):\n with spack.repo.use_repositories(mock_packages_repo):\n module(\"tcl\", \"refresh\", \"-y\")\n\n\ndef _module_files(module_type, *specs):\n specs = [spack.concretize.concretize_one(x) for x in specs]\n writer_cls = spack.modules.module_types[module_type]\n return [writer_cls(spec, \"default\").layout.filename for spec in specs]\n\n\n@pytest.fixture(\n params=[\n [\"rm\", \"doesnotexist\"], # Try to remove a non existing module\n [\"find\", \"mpileaks\"], # Try to find a module with multiple matches\n [\"find\", \"doesnotexist\"], # Try to find a module with no matches\n [\"find\", \"--unknown_args\"], # Try to give an unknown argument\n ]\n)\ndef failure_args(request):\n \"\"\"A list of arguments that will cause a failure\"\"\"\n return request.param\n\n\n@pytest.fixture(params=[\"tcl\", \"lmod\"])\ndef module_type(request):\n return request.param\n\n\n# TODO : test the --delete-tree option\n# TODO : this requires having a separate directory for test modules\n# TODO : add tests for loads and find to check the prompt format\n\n\n@pytest.mark.db\ndef test_exit_with_failure(database, module_type, failure_args):\n with pytest.raises(spack.main.SpackCommandError):\n module(module_type, *failure_args)\n\n\n@pytest.mark.db\ndef test_remove_and_add(database, module_type):\n \"\"\"Tests adding and removing a tcl module file.\"\"\"\n\n if module_type == \"lmod\":\n # TODO: Testing this with lmod requires mocking\n # TODO: the core compilers\n return\n\n rm_cli_args = [\"rm\", \"-y\", \"mpileaks\"]\n module_files = _module_files(module_type, \"mpileaks\")\n for item in module_files:\n assert os.path.exists(item)\n\n module(module_type, *rm_cli_args)\n for item in module_files:\n assert not os.path.exists(item)\n\n module(module_type, \"refresh\", \"-y\", \"mpileaks\")\n for item in module_files:\n assert os.path.exists(item)\n\n\n@pytest.mark.db\n@pytest.mark.parametrize(\"cli_args\", [[\"libelf\"], [\"--full-path\", \"libelf\"]])\ndef test_find(database, cli_args, module_type):\n if module_type == \"lmod\":\n # TODO: Testing this with lmod requires mocking\n # TODO: the core compilers\n return\n\n module(module_type, *([\"find\"] + cli_args))\n\n\n@pytest.mark.db\n@pytest.mark.usefixtures(\"database\")\n@pytest.mark.regression(\"2215\")\ndef test_find_fails_on_multiple_matches():\n # As we installed multiple versions of mpileaks, the command will\n # fail because of multiple matches\n out = module(\"tcl\", \"find\", \"mpileaks\", fail_on_error=False)\n assert module.returncode == 1\n assert \"matches multiple packages\" in out\n\n # Passing multiple packages from the command line also results in the\n # same failure\n out = module(\"tcl\", \"find\", \"mpileaks ^mpich\", \"libelf\", fail_on_error=False)\n assert module.returncode == 1\n assert \"matches multiple packages\" in out\n\n\n@pytest.mark.db\n@pytest.mark.usefixtures(\"database\")\n@pytest.mark.regression(\"2570\")\ndef test_find_fails_on_non_existing_packages():\n # Another way the command might fail is if the package does not exist\n out = module(\"tcl\", \"find\", \"doesnotexist\", fail_on_error=False)\n assert module.returncode == 1\n assert \"matches no package\" in out\n\n\n@pytest.mark.db\n@pytest.mark.usefixtures(\"database\")\ndef test_find_recursive():\n # If we call find without options it should return only one module\n out = module(\"tcl\", \"find\", \"mpileaks ^zmpi\")\n assert len(out.split()) == 1\n\n # If instead we call it with the recursive option the length should\n # be greater\n out = module(\"tcl\", \"find\", \"-r\", \"mpileaks ^zmpi\")\n assert len(out.split()) > 1\n\n\n@pytest.mark.db\ndef test_find_recursive_excluded(mutable_database, module_configuration):\n module_configuration(\"exclude\")\n\n module(\"lmod\", \"refresh\", \"-y\", \"--delete-tree\")\n module(\"lmod\", \"find\", \"-r\", \"mpileaks ^mpich\")\n\n\n@pytest.mark.db\ndef test_loads_recursive_excluded(mutable_database, module_configuration):\n module_configuration(\"exclude\")\n\n module(\"lmod\", \"refresh\", \"-y\", \"--delete-tree\")\n output = module(\"lmod\", \"loads\", \"-r\", \"mpileaks ^mpich\")\n lines = output.split(\"\\n\")\n\n assert any(re.match(r\"[^#]*module load.*mpileaks\", ln) for ln in lines)\n assert not any(re.match(r\"[^#]module load.*callpath\", ln) for ln in lines)\n assert any(re.match(r\"## excluded or missing.*callpath\", ln) for ln in lines)\n\n # TODO: currently there is no way to separate stdout and stderr when\n # invoking a SpackCommand. Supporting this requires refactoring\n # SpackCommand, or log_output, or both.\n # start_of_warning = spack.cmd.modules._missing_modules_warning[:10]\n # assert start_of_warning not in output\n\n\n# Needed to make the 'module_configuration' fixture below work\nwriter_cls = spack.modules.lmod.LmodModulefileWriter\n\n\n@pytest.mark.db\ndef test_setdefault_command(mutable_database, mutable_config):\n data = {\n \"default\": {\n \"enable\": [\"lmod\"],\n \"lmod\": {\"core_compilers\": [\"clang@3.3\"], \"hierarchy\": [\"mpi\"]},\n }\n }\n spack.config.set(\"modules\", data)\n # Install two different versions of pkg-a\n other_spec, preferred = \"pkg-a@1.0\", \"pkg-a@2.0\"\n\n specs = [\n spack.concretize.concretize_one(other_spec),\n spack.concretize.concretize_one(preferred),\n ]\n PackageInstaller([s.package for s in specs], explicit=True, fake=True).install()\n\n writers = {\n preferred: writer_cls(specs[1], \"default\"),\n other_spec: writer_cls(specs[0], \"default\"),\n }\n\n # Create two module files for the same software\n module(\"lmod\", \"refresh\", \"-y\", \"--delete-tree\", preferred, other_spec)\n\n # Assert initial directory state: no link and all module files present\n link_name = os.path.join(os.path.dirname(writers[preferred].layout.filename), \"default\")\n for k in preferred, other_spec:\n assert os.path.exists(writers[k].layout.filename)\n assert not os.path.exists(link_name)\n\n # Set the default to be the other spec\n module(\"lmod\", \"setdefault\", other_spec)\n\n # Check that a link named 'default' exists, and points to the right file\n for k in preferred, other_spec:\n assert os.path.exists(writers[k].layout.filename)\n assert os.path.exists(link_name) and os.path.islink(link_name)\n assert os.path.realpath(link_name) == os.path.realpath(writers[other_spec].layout.filename)\n\n # Reset the default to be the preferred spec\n module(\"lmod\", \"setdefault\", preferred)\n\n # Check that a link named 'default' exists, and points to the right file\n for k in preferred, other_spec:\n assert os.path.exists(writers[k].layout.filename)\n assert os.path.exists(link_name) and os.path.islink(link_name)\n assert os.path.realpath(link_name) == os.path.realpath(writers[preferred].layout.filename)\n" }, { "path": "lib/spack/spack/test/cmd/pkg.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport pathlib\nimport shutil\n\nimport pytest\n\nimport spack.cmd\nimport spack.cmd.pkg\nimport spack.main\nimport spack.paths\nimport spack.repo\nimport spack.util.executable\nimport spack.util.file_cache\nfrom spack.llnl.util.filesystem import mkdirp, working_dir\n\npkg = spack.main.SpackCommand(\"pkg\")\n\n#: new fake package template\npkg_template = \"\"\"\\\nfrom spack.package import *\n\nclass {name}(Package):\n homepage = \"http://www.example.com\"\n url = \"http://www.example.com/test-1.0.tar.gz\"\n\n version(\"1.0\", md5=\"0123456789abcdef0123456789abcdef\")\n\n def install(self, spec, prefix):\n pass\n\"\"\"\n\n\n# Force all tests to use a git repository *in* the mock packages repo.\n@pytest.fixture(scope=\"module\")\ndef _builtin_mock_copy(\n git: spack.util.executable.Executable, tmp_path_factory: pytest.TempPathFactory\n):\n \"\"\"Copy the builtin_mock repo and make a mutable git repo inside it.\"\"\"\n root_dir: pathlib.Path = tmp_path_factory.mktemp(\"builtin_mock_copy\")\n # create spack_repo subdir\n (root_dir / \"spack_repo\").mkdir()\n repo_dir = root_dir / \"spack_repo\" / \"builtin_mock\"\n shutil.copytree(spack.paths.mock_packages_path, str(repo_dir))\n\n repo_cache = spack.util.file_cache.FileCache(root_dir / \"cache\")\n mock_repo = spack.repo.Repo(str(repo_dir), cache=repo_cache)\n\n with working_dir(mock_repo.packages_path):\n git(\"init\")\n\n # initial commit with mock packages\n # the -f is necessary in case people ignore build-* in their ignores\n git(\"add\", \"-f\", \".\")\n git(\"config\", \"user.email\", \"testing@spack.io\")\n git(\"config\", \"user.name\", \"Spack Testing\")\n git(\"-c\", \"commit.gpgsign=false\", \"commit\", \"-m\", \"initial mock repo commit\")\n\n # add commit with mockpkg-a, mockpkg-b, mockpkg-c packages\n mkdirp(\"mockpkg_a\", \"mockpkg_b\", \"mockpkg_c\")\n with open(\"mockpkg_a/package.py\", \"w\", encoding=\"utf-8\") as f:\n f.write(pkg_template.format(name=\"PkgA\"))\n with open(\"mockpkg_b/package.py\", \"w\", encoding=\"utf-8\") as f:\n f.write(pkg_template.format(name=\"PkgB\"))\n with open(\"mockpkg_c/package.py\", \"w\", encoding=\"utf-8\") as f:\n f.write(pkg_template.format(name=\"PkgC\"))\n git(\"add\", \"mockpkg_a\", \"mockpkg_b\", \"mockpkg_c\")\n git(\"-c\", \"commit.gpgsign=false\", \"commit\", \"-m\", \"add mockpkg-a, mockpkg-b, mockpkg-c\")\n\n # remove mockpkg-c, add mockpkg-d\n with open(\"mockpkg_b/package.py\", \"a\", encoding=\"utf-8\") as f:\n f.write(\"\\n# change mockpkg-b\")\n git(\"add\", \"mockpkg_b\")\n mkdirp(\"mockpkg_d\")\n with open(\"mockpkg_d/package.py\", \"w\", encoding=\"utf-8\") as f:\n f.write(pkg_template.format(name=\"PkgD\"))\n git(\"add\", \"mockpkg_d\")\n git(\"rm\", \"-rf\", \"mockpkg_c\")\n git(\n \"-c\",\n \"commit.gpgsign=false\",\n \"commit\",\n \"-m\",\n \"change mockpkg-b, remove mockpkg-c, add mockpkg-d\",\n )\n\n yield mock_repo\n\n\n@pytest.fixture\ndef builtin_mock_copy(_builtin_mock_copy: spack.repo.Repo):\n \"\"\"Fixture that enables a copy of the builtin_mock repo.\"\"\"\n with spack.repo.use_repositories(_builtin_mock_copy):\n yield _builtin_mock_copy\n\n\ndef test_pkg_add(git, builtin_mock_copy: spack.repo.Repo):\n with working_dir(builtin_mock_copy.packages_path):\n mkdirp(\"mockpkg_e\")\n with open(\"mockpkg_e/package.py\", \"w\", encoding=\"utf-8\") as f:\n f.write(pkg_template.format(name=\"PkgE\"))\n\n pkg(\"add\", \"mockpkg-e\")\n\n with working_dir(builtin_mock_copy.packages_path):\n try:\n assert \"A mockpkg_e/package.py\" in git(\"status\", \"--short\", output=str)\n finally:\n shutil.rmtree(\"mockpkg_e\")\n # Removing a package mid-run disrupts Spack's caching\n if spack.repo.PATH.repos[0]._fast_package_checker:\n spack.repo.PATH.repos[0]._fast_package_checker.invalidate()\n\n with pytest.raises(spack.main.SpackCommandError):\n pkg(\"add\", \"does-not-exist\")\n\n\n@pytest.mark.not_on_windows(\"stdout format conflict\")\ndef test_pkg_list(builtin_mock_copy: spack.repo.Repo):\n # Be sure to include virtual packages since packages with stand-alone\n # tests may inherit additional tests from the virtuals they provide,\n # such as packages that implement `mpi`.\n mock_pkg_names = {\n name\n for name in builtin_mock_copy.all_package_names(include_virtuals=True)\n if not name.startswith(\"mockpkg-\")\n }\n\n out = pkg(\"list\", \"HEAD^^\").split()\n assert sorted(mock_pkg_names) == sorted(out)\n\n out = pkg(\"list\", \"HEAD^\").split()\n assert sorted(mock_pkg_names.union([\"mockpkg-a\", \"mockpkg-b\", \"mockpkg-c\"])) == sorted(out)\n\n out = pkg(\"list\", \"HEAD\").split()\n assert sorted(mock_pkg_names.union([\"mockpkg-a\", \"mockpkg-b\", \"mockpkg-d\"])) == sorted(out)\n\n # test with three dots to make sure pkg calls `git merge-base`\n out = pkg(\"list\", \"HEAD^^...\").split()\n assert sorted(mock_pkg_names) == sorted(out)\n\n\n@pytest.mark.not_on_windows(\"stdout format conflict\")\ndef test_pkg_diff(builtin_mock_copy: spack.repo.Repo):\n out = pkg(\"diff\", \"HEAD^^\", \"HEAD^\").split()\n assert out == [\"HEAD^:\", \"mockpkg-a\", \"mockpkg-b\", \"mockpkg-c\"]\n\n out = pkg(\"diff\", \"HEAD^^\", \"HEAD\").split()\n assert out == [\"HEAD:\", \"mockpkg-a\", \"mockpkg-b\", \"mockpkg-d\"]\n\n out = pkg(\"diff\", \"HEAD^\", \"HEAD\").split()\n assert out == [\"HEAD^:\", \"mockpkg-c\", \"HEAD:\", \"mockpkg-d\"]\n\n\n@pytest.mark.not_on_windows(\"stdout format conflict\")\ndef test_pkg_added(builtin_mock_copy: spack.repo.Repo):\n out = pkg(\"added\", \"HEAD^^\", \"HEAD^\").split()\n assert [\"mockpkg-a\", \"mockpkg-b\", \"mockpkg-c\"] == out\n\n out = pkg(\"added\", \"HEAD^^\", \"HEAD\").split()\n assert [\"mockpkg-a\", \"mockpkg-b\", \"mockpkg-d\"] == out\n\n out = pkg(\"added\", \"HEAD^\", \"HEAD\").split()\n assert [\"mockpkg-d\"] == out\n\n out = pkg(\"added\", \"HEAD\", \"HEAD\").split()\n assert out == []\n\n\n@pytest.mark.not_on_windows(\"stdout format conflict\")\ndef test_pkg_removed(builtin_mock_copy: spack.repo.Repo):\n out = pkg(\"removed\", \"HEAD^^\", \"HEAD^\").split()\n assert out == []\n\n out = pkg(\"removed\", \"HEAD^^\", \"HEAD\").split()\n assert out == []\n\n out = pkg(\"removed\", \"HEAD^\", \"HEAD\").split()\n assert out == [\"mockpkg-c\"]\n\n\n@pytest.mark.not_on_windows(\"stdout format conflict\")\ndef test_pkg_changed(builtin_mock_copy: spack.repo.Repo):\n out = pkg(\"changed\", \"HEAD^^\", \"HEAD^\").split()\n assert out == []\n\n out = pkg(\"changed\", \"--type\", \"c\", \"HEAD^^\", \"HEAD^\").split()\n assert out == []\n\n out = pkg(\"changed\", \"--type\", \"a\", \"HEAD^^\", \"HEAD^\").split()\n assert out == [\"mockpkg-a\", \"mockpkg-b\", \"mockpkg-c\"]\n\n out = pkg(\"changed\", \"--type\", \"r\", \"HEAD^^\", \"HEAD^\").split()\n assert out == []\n\n out = pkg(\"changed\", \"--type\", \"ar\", \"HEAD^^\", \"HEAD^\").split()\n assert out == [\"mockpkg-a\", \"mockpkg-b\", \"mockpkg-c\"]\n\n out = pkg(\"changed\", \"--type\", \"arc\", \"HEAD^^\", \"HEAD^\").split()\n assert out == [\"mockpkg-a\", \"mockpkg-b\", \"mockpkg-c\"]\n\n out = pkg(\"changed\", \"HEAD^\", \"HEAD\").split()\n assert out == [\"mockpkg-b\"]\n\n out = pkg(\"changed\", \"--type\", \"c\", \"HEAD^\", \"HEAD\").split()\n assert out == [\"mockpkg-b\"]\n\n out = pkg(\"changed\", \"--type\", \"a\", \"HEAD^\", \"HEAD\").split()\n assert out == [\"mockpkg-d\"]\n\n out = pkg(\"changed\", \"--type\", \"r\", \"HEAD^\", \"HEAD\").split()\n assert out == [\"mockpkg-c\"]\n\n out = pkg(\"changed\", \"--type\", \"ar\", \"HEAD^\", \"HEAD\").split()\n assert out == [\"mockpkg-c\", \"mockpkg-d\"]\n\n out = pkg(\"changed\", \"--type\", \"arc\", \"HEAD^\", \"HEAD\").split()\n assert out == [\"mockpkg-b\", \"mockpkg-c\", \"mockpkg-d\"]\n\n # invalid type argument\n with pytest.raises(spack.main.SpackCommandError):\n pkg(\"changed\", \"--type\", \"foo\")\n\n\ndef test_pkg_fails_when_not_git_repo(monkeypatch):\n monkeypatch.setattr(spack.cmd, \"spack_is_git_repo\", lambda: False)\n with pytest.raises(spack.main.SpackCommandError):\n pkg(\"added\")\n\n\ndef test_pkg_source_requires_one_arg(mock_packages):\n with pytest.raises(spack.main.SpackCommandError):\n pkg(\"source\", \"a\", \"b\")\n\n with pytest.raises(spack.main.SpackCommandError):\n pkg(\"source\", \"--canonical\", \"a\", \"b\")\n\n\ndef test_pkg_source(mock_packages):\n fake_source = pkg(\"source\", \"fake\")\n\n fake_file = spack.repo.PATH.filename_for_package_name(\"fake\")\n with open(fake_file, encoding=\"utf-8\") as f:\n contents = f.read()\n assert fake_source == contents\n\n\ndef test_pkg_canonical_source(mock_packages):\n source = pkg(\"source\", \"multimethod\")\n assert '@when(\"@2.0\")' in source\n assert \"Check that multimethods work with boolean values\" in source\n\n canonical_1 = pkg(\"source\", \"--canonical\", \"multimethod@1.0\")\n assert \"@when\" not in canonical_1\n assert \"should_not_be_reached by diamond inheritance test\" not in canonical_1\n assert \"return 'base@1.0'\" in canonical_1\n assert \"return 'base@2.0'\" not in canonical_1\n assert \"return 'first_parent'\" not in canonical_1\n assert \"'should_not_be_reached by diamond inheritance test'\" not in canonical_1\n\n canonical_2 = pkg(\"source\", \"--canonical\", \"multimethod@2.0\")\n assert \"@when\" not in canonical_2\n assert \"return 'base@1.0'\" not in canonical_2\n assert \"return 'base@2.0'\" in canonical_2\n assert \"return 'first_parent'\" in canonical_2\n assert \"'should_not_be_reached by diamond inheritance test'\" not in canonical_2\n\n canonical_3 = pkg(\"source\", \"--canonical\", \"multimethod@3.0\")\n assert \"@when\" not in canonical_3\n assert \"return 'base@1.0'\" not in canonical_3\n assert \"return 'base@2.0'\" not in canonical_3\n assert \"return 'first_parent'\" not in canonical_3\n assert \"'should_not_be_reached by diamond inheritance test'\" not in canonical_3\n\n canonical_4 = pkg(\"source\", \"--canonical\", \"multimethod@4.0\")\n assert \"@when\" not in canonical_4\n assert \"return 'base@1.0'\" not in canonical_4\n assert \"return 'base@2.0'\" not in canonical_4\n assert \"return 'first_parent'\" not in canonical_4\n assert \"'should_not_be_reached by diamond inheritance test'\" in canonical_4\n\n\ndef test_pkg_hash(mock_packages):\n output = pkg(\"hash\", \"pkg-a\", \"pkg-b\").split()\n assert len(output) == 2 and all(len(elt) == 32 for elt in output)\n\n output = pkg(\"hash\", \"multimethod\").split()\n assert len(output) == 1 and all(len(elt) == 32 for elt in output)\n\n\ngroup_args = [\n \"/path/one.py\", # 12\n \"/path/two.py\", # 12\n \"/path/three.py\", # 14\n \"/path/four.py\", # 13\n \"/path/five.py\", # 13\n \"/path/six.py\", # 12\n \"/path/seven.py\", # 14\n \"/path/eight.py\", # 14\n \"/path/nine.py\", # 13\n \"/path/ten.py\", # 12\n]\n\n\n@pytest.mark.parametrize(\n [\"args\", \"max_group_size\", \"prefix_length\", \"max_group_length\", \"lengths\", \"error\"],\n [\n (group_args, 3, 0, 1, None, ValueError), # element too long\n (group_args, 3, 0, 13, None, ValueError), # element too long\n (group_args, 3, 12, 25, None, ValueError), # prefix and words too long\n (group_args, 3, 0, 25, [2, 1, 1, 1, 1, 1, 1, 1, 1], None),\n (group_args, 3, 0, 26, [2, 1, 1, 2, 1, 1, 2], None),\n (group_args, 3, 0, 40, [3, 3, 2, 2], None),\n (group_args, 3, 0, 43, [3, 3, 3, 1], None),\n (group_args, 4, 0, 54, [4, 3, 3], None),\n (group_args, 4, 0, 56, [4, 4, 2], None),\n ([], 500, 0, None, [], None),\n ],\n)\ndef test_group_arguments(\n mock_packages, args, max_group_size, prefix_length, max_group_length, lengths, error\n):\n generator = spack.cmd.group_arguments(\n args,\n max_group_size=max_group_size,\n prefix_length=prefix_length,\n max_group_length=max_group_length,\n )\n\n # just check that error cases raise\n if error:\n with pytest.raises(ValueError):\n list(generator)\n return\n\n groups = list(generator)\n assert sum(groups, []) == args\n assert [len(group) for group in groups] == lengths\n assert all(\n sum(len(elt) for elt in group) + (len(group) - 1) <= max_group_length for group in groups\n )\n\n\n@pytest.mark.skipif(not spack.cmd.pkg.get_grep(), reason=\"grep is not installed\")\ndef test_pkg_grep(mock_packages):\n # only splice-* mock packages have the string \"splice\" in them\n pkg(\"grep\", \"-l\", \"splice\")\n assert pkg.output.strip() == \"\\n\".join(\n spack.repo.PATH.get_pkg_class(name).module.__file__\n for name in [\n \"depends-on-manyvariants\",\n \"manyvariants\",\n \"splice-a\",\n \"splice-depends-on-t\",\n \"splice-h\",\n \"splice-t\",\n \"splice-vh\",\n \"splice-vt\",\n \"splice-z\",\n \"virtual-abi-1\",\n \"virtual-abi-2\",\n \"virtual-abi-multi\",\n ]\n )\n\n # ensure that this string isn't found\n with pytest.raises(spack.main.SpackCommandError):\n pkg(\"grep\", \"abcdefghijklmnopqrstuvwxyz\")\n assert pkg.returncode == 1\n assert pkg.output.strip() == \"\"\n\n # ensure that we return > 1 for an error\n with pytest.raises(spack.main.SpackCommandError):\n pkg(\"grep\", \"--foobarbaz-not-an-option\")\n assert pkg.returncode == 2\n" }, { "path": "lib/spack/spack/test/cmd/print_shell_vars.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nfrom spack.main import print_setup_info\n\n\ndef test_print_shell_vars_sh(capfd):\n print_setup_info(\"sh\")\n out, _ = capfd.readouterr()\n\n assert \"_sp_sys_type=\" in out\n assert \"_sp_tcl_roots=\" in out\n assert \"_sp_lmod_roots=\" in out\n assert \"_sp_module_prefix\" not in out\n\n\ndef test_print_shell_vars_csh(capfd):\n print_setup_info(\"csh\")\n out, _ = capfd.readouterr()\n\n assert \"set _sp_sys_type = \" in out\n assert \"set _sp_tcl_roots = \" in out\n assert \"set _sp_lmod_roots = \" in out\n assert \"set _sp_module_prefix = \" not in out\n" }, { "path": "lib/spack/spack/test/cmd/providers.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\nimport pytest\n\nfrom spack.main import SpackCommand\n\npytestmark = [pytest.mark.usefixtures(\"mock_packages\")]\n\nproviders = SpackCommand(\"providers\")\n\n\n@pytest.mark.parametrize(\n \"pkg\",\n [(\"mpi\",), (\"mpi@2\",), (\"mpi\", \"lapack\"), (\"\",)], # Lists all the available virtual packages\n)\ndef test_it_just_runs(pkg):\n providers(*pkg)\n\n\n@pytest.mark.parametrize(\n \"vpkg,provider_list\",\n [\n (\n (\"mpi\",),\n [\n \"intel-parallel-studio\",\n \"low-priority-provider\",\n \"mpich@3:\",\n \"mpich2\",\n \"multi-provider-mpi@1.10.0\",\n \"multi-provider-mpi@2.0.0\",\n \"zmpi\",\n ],\n ),\n (\n (\"lapack\", \"something\"),\n [\n \"intel-parallel-studio\",\n \"low-priority-provider\",\n \"netlib-lapack\",\n \"openblas-with-lapack\",\n \"simple-inheritance\",\n \"splice-a\",\n \"splice-h\",\n \"splice-vh\",\n ],\n ), # Call 2 virtual packages at once\n ],\n)\ndef test_provider_lists(vpkg, provider_list):\n output = providers(*vpkg)\n for item in provider_list:\n assert item in output\n\n\n@pytest.mark.parametrize(\n \"pkg,error_cls\",\n [\n (\"zlib\", ValueError),\n (\"foo\", ValueError), # Trying to call with a package that does not exist\n ],\n)\ndef test_it_just_fails(pkg, error_cls):\n with pytest.raises(error_cls):\n providers(pkg)\n" }, { "path": "lib/spack/spack/test/cmd/python.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport platform\nimport sys\n\nimport pytest\n\nimport spack\nfrom spack.main import SpackCommand\n\npython = SpackCommand(\"python\")\n\n\ndef test_python():\n out = python(\"-c\", \"import spack; print(spack.spack_version)\")\n assert out.strip() == spack.spack_version\n\n\ndef test_python_interpreter_path():\n out = python(\"--path\")\n assert out.strip() == sys.executable\n\n\ndef test_python_version():\n out = python(\"-V\")\n assert platform.python_version() in out\n\n\ndef test_python_with_module():\n # pytest rewrites a lot of modules, which interferes with runpy, so\n # it's hard to test this. Trying to import a module like sys, that\n # has no code associated with it, raises an error reliably in python\n # 2 and 3, which indicates we successfully ran runpy.run_module.\n with pytest.raises(ImportError, match=\"No code object\"):\n python(\"-m\", \"sys\")\n\n\ndef test_python_raises():\n out = python(\"--foobar\", fail_on_error=False)\n assert \"Error: Unknown arguments\" in out\n" }, { "path": "lib/spack/spack/test/cmd/reindex.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport pathlib\nimport shutil\n\nimport spack.store\nfrom spack.database import Database\nfrom spack.enums import InstallRecordStatus\nfrom spack.main import SpackCommand\n\ninstall = SpackCommand(\"install\")\ndeprecate = SpackCommand(\"deprecate\")\nreindex = SpackCommand(\"reindex\")\n\n\ndef test_reindex_basic(mock_packages, mock_archive, mock_fetch, install_mockery):\n install(\"--fake\", \"libelf@0.8.13\")\n install(\"--fake\", \"libelf@0.8.12\")\n all_installed = spack.store.STORE.db.query()\n reindex()\n assert spack.store.STORE.db.query() == all_installed\n\n\ndef _clear_db(tmp_path: pathlib.Path):\n empty_db = Database(str(tmp_path))\n with empty_db.write_transaction():\n pass\n shutil.rmtree(spack.store.STORE.db.database_directory)\n shutil.copytree(empty_db.database_directory, spack.store.STORE.db.database_directory)\n # force a re-read of the database\n assert len(spack.store.STORE.db.query()) == 0\n\n\ndef test_reindex_db_deleted(\n mock_packages, mock_archive, mock_fetch, install_mockery, tmp_path: pathlib.Path\n):\n install(\"--fake\", \"libelf@0.8.13\")\n install(\"--fake\", \"libelf@0.8.12\")\n all_installed = spack.store.STORE.db.query()\n _clear_db(tmp_path)\n reindex()\n assert spack.store.STORE.db.query() == all_installed\n\n\ndef test_reindex_with_deprecated_packages(\n mock_packages, mock_archive, mock_fetch, install_mockery, tmp_path: pathlib.Path\n):\n install(\"--fake\", \"libelf@0.8.13\")\n install(\"--fake\", \"libelf@0.8.12\")\n\n deprecate(\"-y\", \"libelf@0.8.12\", \"libelf@0.8.13\")\n\n db = spack.store.STORE.db\n\n all_installed = db.query(installed=InstallRecordStatus.ANY)\n non_deprecated = db.query(installed=True)\n\n _clear_db(tmp_path)\n\n reindex()\n\n assert db.query(installed=InstallRecordStatus.ANY) == all_installed\n assert db.query(installed=True) == non_deprecated\n\n old_libelf = db.query_local_by_spec_hash(\n db.query_local(\"libelf@0.8.12\", installed=InstallRecordStatus.ANY)[0].dag_hash()\n )\n new_libelf = db.query_local_by_spec_hash(\n db.query_local(\"libelf@0.8.13\", installed=True)[0].dag_hash()\n )\n assert old_libelf is not None and new_libelf is not None\n assert old_libelf.deprecated_for == new_libelf.spec.dag_hash()\n assert new_libelf.deprecated_for is None\n assert new_libelf.ref_count == 1\n" }, { "path": "lib/spack/spack/test/cmd/repo.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport io\nimport os\nimport pathlib\nimport sys\nfrom typing import Dict, Optional, Union\n\nimport pytest\n\nimport spack.cmd.repo\nimport spack.config\nimport spack.environment as ev\nimport spack.main\nimport spack.repo\nimport spack.repo_migrate\nfrom spack.error import SpackError\nfrom spack.llnl.util.filesystem import working_dir\nfrom spack.util.executable import Executable\n\nrepo = spack.main.SpackCommand(\"repo\")\nenv = spack.main.SpackCommand(\"env\")\n\n\ndef test_help_option():\n # Test 'spack repo --help' to check basic import works\n # and the command exits successfully\n repo(\"--help\")\n assert repo.returncode in (None, 0)\n\n\ndef test_create_add_list_remove(mutable_config, tmp_path: pathlib.Path):\n # Create a new repository and check that the expected\n # files are there\n repo(\"create\", str(tmp_path), \"mockrepo\")\n assert (tmp_path / \"spack_repo\" / \"mockrepo\" / \"repo.yaml\").exists()\n\n # Add the new repository and check it appears in the list output\n repo(\"add\", \"--scope=site\", str(tmp_path / \"spack_repo\" / \"mockrepo\"))\n output = repo(\"list\", \"--scope=site\")\n assert \"mockrepo\" in output\n\n # Then remove it and check it's not there\n repo(\"remove\", \"--scope=site\", str(tmp_path / \"spack_repo\" / \"mockrepo\"))\n output = repo(\"list\", \"--scope=site\")\n assert \"mockrepo\" not in output\n\n\ndef test_repo_remove_by_scope(mutable_config, tmp_path: pathlib.Path):\n # Create and add a new repo\n repo(\"create\", str(tmp_path), \"mockrepo\")\n repo(\"add\", \"--scope=site\", str(tmp_path / \"spack_repo\" / \"mockrepo\"))\n repo(\"add\", \"--scope=system\", str(tmp_path / \"spack_repo\" / \"mockrepo\"))\n\n # Confirm that it is not removed when the scope is incorrect\n with pytest.raises(spack.main.SpackCommandError):\n repo(\"remove\", \"--scope=user\", \"mockrepo\")\n output = repo(\"list\")\n assert \"mockrepo\" in output\n\n # Confirm that when the scope is specified, it is only removed from that scope\n repo(\"remove\", \"--scope=site\", \"mockrepo\")\n site_output = repo(\"list\", \"--scope=site\")\n system_output = repo(\"list\", \"--scope=system\")\n assert \"mockrepo\" not in site_output\n assert \"mockrepo\" in system_output\n\n # Confirm that when the scope is not specified, it is removed from top scope with it present\n repo(\"add\", \"--scope=site\", str(tmp_path / \"spack_repo\" / \"mockrepo\"))\n repo(\"remove\", \"mockrepo\")\n site_output = repo(\"list\", \"--scope=site\")\n system_output = repo(\"list\", \"--scope=system\")\n assert \"mockrepo\" not in site_output\n assert \"mockrepo\" in system_output\n\n # Check that the `--all-scopes` option removes from all scopes\n repo(\"add\", \"--scope=site\", str(tmp_path / \"spack_repo\" / \"mockrepo\"))\n repo(\"remove\", \"--all-scopes\", \"mockrepo\")\n output = repo(\"list\")\n assert \"mockrepo\" not in output\n\n\ndef test_env_repo_path_vars_substitution(\n tmp_path: pathlib.Path, install_mockery, mutable_mock_env_path, monkeypatch\n):\n \"\"\"Test Spack correctly substitutes repo paths with environment variables when creating an\n environment from a manifest file.\"\"\"\n\n monkeypatch.setenv(\"CUSTOM_REPO_PATH\", \".\")\n\n # setup environment from spack.yaml\n envdir = tmp_path / \"env\"\n envdir.mkdir()\n with working_dir(str(envdir)):\n with open(\"spack.yaml\", \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n specs: []\n\n repos:\n current_dir: $CUSTOM_REPO_PATH\n\"\"\"\n )\n # creating env from manifest file\n env(\"create\", \"test\", \"./spack.yaml\")\n # check that repo path was correctly substituted with the environment variable\n current_dir = os.getcwd()\n with ev.read(\"test\") as newenv:\n repos_specs = spack.config.get(\"repos\", default={}, scope=newenv.scope_name)\n assert current_dir in repos_specs.values()\n\n\nOLD_7ZIP = b\"\"\"\\\n# some comment\n\nfrom spack.package import *\nfrom .blt import linker_helpers\n\nclass _7zip(Package):\n pass\n\"\"\"\n\nNEW_7ZIP = b\"\"\"\\\n# some comment\n\nfrom spack_repo.builtin.build_systems.generic import Package\nfrom spack.package import *\nfrom ..blt.package import linker_helpers\n\nclass _7zip(Package):\n pass\n\"\"\"\n\n# this is written like this to be explicit about line endings and indentation\nOLD_NUMPY = (\n b\"# some comment\\r\\n\"\n b\"\\r\\n\"\n b\"import spack.pkg.builtin.foo, spack.pkg.builtin.bar\\r\\n\"\n b\"from spack.package import *\\r\\n\"\n b\"from something.unrelated import AutotoolsPackage\\r\\n\"\n b\"from spack.pkg.builtin.num7zip import _7zip\\r\\n\"\n b\"\\r\\n\"\n b\"\\r\\n\"\n b\"class PyNumpy(CMakePackage, AutotoolsPackage):\\r\\n\"\n b\"\\tgenerator('ninja')\\r\\n\"\n b\"\\r\\n\"\n b\"\\tdef example(self):\\r\\n\"\n b\"\\t\\t# unchanged comment: spack.pkg.builtin.foo.something\\r\\n\"\n b\"\\t\\treturn spack.pkg.builtin.foo.example(), foo, baz\\r\\n\"\n)\n\nNEW_NUMPY = (\n b\"# some comment\\r\\n\"\n b\"\\r\\n\"\n b\"import spack_repo.builtin.packages.foo.package, spack_repo.builtin.packages.bar.package\\r\\n\"\n b\"from spack_repo.builtin.build_systems.cmake import CMakePackage, generator\\r\\n\"\n b\"from spack.package import *\\r\\n\"\n b\"from something.unrelated import AutotoolsPackage\\r\\n\"\n b\"from spack_repo.builtin.packages._7zip.package import _7zip\\r\\n\"\n b\"\\r\\n\"\n b\"\\r\\n\"\n b\"class PyNumpy(CMakePackage, AutotoolsPackage):\\r\\n\"\n b\"\\tgenerator('ninja')\\r\\n\"\n b\"\\r\\n\"\n b\"\\tdef example(self):\\r\\n\"\n b\"\\t\\t# unchanged comment: spack.pkg.builtin.foo.something\\r\\n\"\n b\"\\t\\treturn spack_repo.builtin.packages.foo.package.example(), foo, baz\\r\\n\"\n)\n\nOLD_NONTRIVIAL_IMPORT = b\"\"\"\\\nif True:\n from spack.pkg.builtin import (\n foo,\n bar as baz,\n num7zip as my7zip\n )\n\nclass NonTrivialImport(Package):\n pass\n\"\"\"\n\n\nNEW_NONTRIVIAL_IMPORT = b\"\"\"\\\nfrom spack_repo.builtin.build_systems.generic import Package\n\n\nif True:\n import spack_repo.builtin.packages.foo.package as foo\n import spack_repo.builtin.packages.bar.package as baz\n import spack_repo.builtin.packages._7zip.package as my7zip\n\nclass NonTrivialImport(Package):\n pass\n\"\"\"\n\n\ndef test_repo_migrate(tmp_path: pathlib.Path, config):\n old_root, _ = spack.repo.create_repo(str(tmp_path), \"org.repo\", package_api=(1, 0))\n pkgs_path = pathlib.Path(spack.repo.from_path(old_root).packages_path)\n new_root = pathlib.Path(old_root) / \"spack_repo\" / \"org\" / \"repo\"\n\n pkg_7zip_old = pkgs_path / \"7zip\" / \"package.py\"\n pkg_numpy_old = pkgs_path / \"py-numpy\" / \"package.py\"\n pkg_py_7zip_new = new_root / \"packages\" / \"_7zip\" / \"package.py\"\n pkg_py_numpy_new = new_root / \"packages\" / \"py_numpy\" / \"package.py\"\n\n pkg_7zip_old.parent.mkdir(parents=True)\n pkg_numpy_old.parent.mkdir(parents=True)\n\n pkg_7zip_old.write_bytes(OLD_7ZIP)\n pkg_numpy_old.write_bytes(OLD_NUMPY)\n\n repo(\"migrate\", \"--fix\", old_root)\n\n # old files are not touched since they are moved\n assert pkg_7zip_old.read_bytes() == OLD_7ZIP\n assert pkg_numpy_old.read_bytes() == OLD_NUMPY\n\n # new files are created and have updated contents\n assert pkg_py_7zip_new.read_bytes() == NEW_7ZIP\n assert pkg_py_numpy_new.read_bytes() == NEW_NUMPY\n\n\ndef test_migrate_diff(git: Executable, tmp_path: pathlib.Path):\n root, _ = spack.repo.create_repo(str(tmp_path), \"foo\", package_api=(2, 0))\n r = pathlib.Path(root)\n pkg_7zip = r / \"packages\" / \"_7zip\" / \"package.py\"\n pkg_py_numpy = r / \"packages\" / \"py_numpy\" / \"package.py\"\n pkg_broken = r / \"packages\" / \"broken\" / \"package.py\"\n pkg_nontrivial_import = r / \"packages\" / \"non_trivial_import\" / \"package.py\"\n\n pkg_7zip.parent.mkdir(parents=True)\n pkg_py_numpy.parent.mkdir(parents=True)\n pkg_broken.parent.mkdir(parents=True)\n pkg_nontrivial_import.parent.mkdir(parents=True)\n\n pkg_7zip.write_bytes(OLD_7ZIP)\n pkg_py_numpy.write_bytes(OLD_NUMPY)\n pkg_broken.write_bytes(b\"syntax(error\")\n pkg_nontrivial_import.write_bytes(OLD_NONTRIVIAL_IMPORT)\n\n stderr = io.StringIO()\n\n with open(tmp_path / \"imports.patch\", \"wb\") as patch_file:\n spack.repo_migrate.migrate_v2_imports(\n str(r / \"packages\"), str(r), patch_file=patch_file, err=stderr\n )\n\n err_output = stderr.getvalue()\n\n assert f\"Skipping {pkg_broken}\" in err_output\n\n # apply the patch and verify the changes\n with working_dir(str(r)):\n git(\"apply\", str(tmp_path / \"imports.patch\"))\n\n # Git may change line endings upon applying the patch, so let Python normalize in TextIOWrapper\n # and compare strings instead of bytes.\n assert (\n pkg_7zip.read_text(encoding=\"utf-8\")\n == io.TextIOWrapper(io.BytesIO(NEW_7ZIP), encoding=\"utf-8\").read()\n )\n assert (\n pkg_py_numpy.read_text(encoding=\"utf-8\")\n == io.TextIOWrapper(io.BytesIO(NEW_NUMPY), encoding=\"utf-8\").read()\n )\n\n # the multi-line non-trivial import rewrite cannot be done in Python < 3.8 because it doesn't\n # support end_lineno in ast.ImportFrom. So here we check that it's either warned about or\n # modified correctly.\n if sys.version_info >= (3, 8):\n assert (\n pkg_nontrivial_import.read_text(encoding=\"utf-8\")\n == io.TextIOWrapper(io.BytesIO(NEW_NONTRIVIAL_IMPORT), encoding=\"utf-8\").read()\n )\n else:\n assert (\n f\"{pkg_nontrivial_import}:2: cannot rewrite spack.pkg.builtin import statement\"\n in err_output\n )\n\n\nclass MockRepo(spack.repo.Repo):\n def __init__(self, namespace: str):\n self.namespace = namespace\n\n\nclass MockDescriptor(spack.repo.RepoDescriptor):\n def __init__(self, to_construct: Dict[str, Union[spack.repo.Repo, Exception]]):\n self.to_construct = to_construct\n self.initialized = False\n\n def initialize(self, fetch=True, git=None) -> None:\n self.initialized = True\n\n def get_commit(self, git: Optional[Executable] = None):\n pass\n\n def update(self, git: Optional[Executable] = None, remote: Optional[str] = \"origin\") -> None:\n pass\n\n def construct(self, cache, overrides=None):\n assert self.initialized, \"MockDescriptor must be initialized before construction\"\n return self.to_construct\n\n\ndef make_repo_config(repo_config: Optional[dict] = None) -> spack.config.Configuration:\n \"\"\"Create a Configuration instance with writable scope and optional repo configuration.\"\"\"\n scope = spack.config.InternalConfigScope(\"test\", {\"repos\": repo_config or {}})\n scope.writable = True\n config = spack.config.Configuration()\n config.push_scope(scope)\n return config\n\n\ndef test_add_repo_name_already_exists(tmp_path: pathlib.Path):\n \"\"\"Test _add_repo raises error when name already exists in config.\"\"\"\n # Set up existing config with the same name\n config = make_repo_config({\"test_name\": \"/some/path\"})\n\n # Should raise error when name already exists\n with pytest.raises(SpackError, match=\"A repository with the name 'test_name' already exists\"):\n spack.cmd.repo._add_repo(\n str(tmp_path), name=\"test_name\", scope=None, paths=[], destination=None, config=config\n )\n\n\ndef test_add_repo_destination_with_local_path(tmp_path: pathlib.Path):\n \"\"\"Test _add_repo raises error when args are added that do not apply to local paths.\"\"\"\n # Should raise error when destination is provided with local path\n with pytest.raises(\n SpackError, match=\"The 'destination' argument is only valid for git repositories\"\n ):\n spack.cmd.repo._add_repo(\n str(tmp_path),\n name=\"test_name\",\n scope=None,\n paths=[],\n destination=\"/some/destination\",\n config=make_repo_config(),\n )\n with pytest.raises(SpackError, match=\"The --paths flag is only valid for git repositories\"):\n spack.cmd.repo._add_repo(\n str(tmp_path),\n name=\"test_name\",\n scope=None,\n paths=[\"path1\", \"path2\"],\n destination=None,\n config=make_repo_config(),\n )\n\n\ndef test_add_repo_computed_key_already_exists(tmp_path: pathlib.Path, monkeypatch):\n \"\"\"Test _add_repo raises error when computed key already exists in config.\"\"\"\n\n def mock_parse_config_descriptor(name, entry, lock):\n return MockDescriptor({str(tmp_path): MockRepo(\"test_repo\")})\n\n monkeypatch.setattr(spack.repo, \"parse_config_descriptor\", mock_parse_config_descriptor)\n\n # Should raise error when computed key already exists\n with pytest.raises(SpackError, match=\"A repository with the name 'test_repo' already exists\"):\n spack.cmd.repo._add_repo(\n str(tmp_path),\n name=None, # Will use namespace as key\n scope=None,\n paths=[],\n destination=None,\n config=make_repo_config({\"test_repo\": \"/some/path\"}),\n )\n\n\ndef test_add_repo_git_url_with_paths(monkeypatch):\n \"\"\"Test _add_repo correctly handles git URL with multiple paths.\"\"\"\n config = make_repo_config({\"test_repo\": \"/some/path\"})\n\n def mock_parse_config_descriptor(name, entry, lock):\n # Verify the entry has the expected git structure\n assert \"git\" in entry\n assert entry[\"git\"] == \"https://example.com/repo.git\"\n assert entry[\"paths\"] == [\"path1\", \"path2\"]\n return MockDescriptor({\"/some/path\": MockRepo(\"git_repo\")})\n\n monkeypatch.setattr(spack.repo, \"parse_config_descriptor\", mock_parse_config_descriptor)\n\n # Should succeed with git URL and multiple paths\n key = spack.cmd.repo._add_repo(\n \"https://example.com/repo.git\",\n name=\"git_test\",\n scope=None,\n paths=[\"path1\", \"path2\"],\n destination=None,\n config=config,\n )\n\n assert key == \"git_test\"\n repos = config.get(\"repos\", scope=None)\n assert \"git_test\" in repos\n assert repos[\"git_test\"][\"git\"] == \"https://example.com/repo.git\"\n assert repos[\"git_test\"][\"paths\"] == [\"path1\", \"path2\"]\n\n\ndef test_add_repo_git_url_with_destination(monkeypatch):\n \"\"\"Test _add_repo correctly handles git URL with destination.\"\"\"\n config = make_repo_config({\"test_repo\": \"/some/path\"})\n\n def mock_parse_config_descriptor(name, entry, lock):\n # Verify the entry has the expected git structure\n assert \"git\" in entry\n assert entry[\"git\"] == \"https://example.com/repo.git\"\n assert entry[\"destination\"] == \"/custom/destination\"\n return MockDescriptor({\"/some/path\": MockRepo(\"git_repo\")})\n\n monkeypatch.setattr(spack.repo, \"parse_config_descriptor\", mock_parse_config_descriptor)\n\n # Should succeed with git URL and destination\n key = spack.cmd.repo._add_repo(\n \"https://example.com/repo.git\",\n name=\"git_test\",\n scope=None,\n paths=[],\n destination=\"/custom/destination\",\n config=config,\n )\n\n assert key == \"git_test\"\n repos = config.get(\"repos\", scope=None)\n assert \"git_test\" in repos\n assert repos[\"git_test\"][\"git\"] == \"https://example.com/repo.git\"\n assert repos[\"git_test\"][\"destination\"] == \"/custom/destination\"\n\n\ndef test_add_repo_ssh_git_url_detection(monkeypatch):\n \"\"\"Test _add_repo correctly detects SSH git URLs.\"\"\"\n config = make_repo_config({\"test_repo\": \"/some/path\"})\n\n def mock_parse_config_descriptor(name, entry, lock):\n # Verify the entry has the expected git structure\n assert \"git\" in entry\n assert entry[\"git\"] == \"git@github.com:user/repo.git\"\n return MockDescriptor({\"/some/path\": MockRepo(\"git_repo\")})\n\n monkeypatch.setattr(spack.repo, \"parse_config_descriptor\", mock_parse_config_descriptor)\n\n # Should detect SSH URL as git URL (colon not preceded by forward slash)\n key = spack.cmd.repo._add_repo(\n \"git@github.com:user/repo.git\",\n name=\"ssh_git_test\",\n scope=None,\n paths=[],\n destination=None,\n config=config,\n )\n\n assert key == \"ssh_git_test\"\n repos = config.get(\"repos\", scope=None)\n assert \"ssh_git_test\" in repos\n assert repos[\"ssh_git_test\"][\"git\"] == \"git@github.com:user/repo.git\"\n\n\ndef test_add_repo_no_usable_repositories_error(monkeypatch):\n \"\"\"Test that _add_repo raises SpackError when no usable repositories can be constructed.\"\"\"\n config = make_repo_config()\n\n def mock_parse_config_descriptor(name, entry, lock):\n return MockDescriptor(\n {\"/path1\": Exception(\"Invalid repo\"), \"/path2\": Exception(\"Another error\")}\n )\n\n monkeypatch.setattr(spack.repo, \"parse_config_descriptor\", mock_parse_config_descriptor)\n\n with pytest.raises(\n SpackError, match=\"No package repository could be constructed from /invalid/path\"\n ):\n spack.cmd.repo._add_repo(\n \"/invalid/path\",\n name=\"test_repo\",\n scope=None,\n paths=[],\n destination=None,\n config=config,\n )\n\n\ndef test_add_repo_multiple_repos_no_name_error(monkeypatch):\n \"\"\"Test that _add_repo raises SpackError when multiple repositories found without\n specifying --name.\"\"\"\n\n def mock_parse_config_descriptor(name, entry, lock):\n return MockDescriptor({\"/path1\": MockRepo(\"repo1\"), \"/path2\": MockRepo(\"repo2\")})\n\n monkeypatch.setattr(spack.repo, \"parse_config_descriptor\", mock_parse_config_descriptor)\n\n with pytest.raises(\n SpackError, match=\"Multiple package repositories found, please specify a name\"\n ):\n spack.cmd.repo._add_repo(\n \"/path/with/multiple/repos\",\n name=None, # No name specified\n scope=None,\n paths=[],\n destination=None,\n config=make_repo_config(),\n )\n\n\ndef test_add_repo_git_url_basic_success(monkeypatch):\n \"\"\"Test successful addition of a git repository.\"\"\"\n config = make_repo_config()\n\n def mock_parse_config_descriptor(name, entry, lock):\n # Verify git entry structure\n assert isinstance(entry, dict)\n assert entry[\"git\"] == \"https://github.com/example/repo.git\"\n return MockDescriptor({\"/git/path\": MockRepo(\"git_repo\")})\n\n monkeypatch.setattr(spack.repo, \"parse_config_descriptor\", mock_parse_config_descriptor)\n\n key = spack.cmd.repo._add_repo(\n \"https://github.com/example/repo.git\",\n name=\"test_git_repo\",\n scope=None,\n paths=[],\n destination=None,\n config=config,\n )\n\n assert key == \"test_git_repo\"\n repos_config = config.get(\"repos\", scope=None)\n assert \"test_git_repo\" in repos_config\n assert \"git\" in repos_config[\"test_git_repo\"]\n\n\ndef test_add_repo_git_url_with_custom_destination(monkeypatch):\n \"\"\"Test successful addition of a git repository with destination.\"\"\"\n config = make_repo_config()\n\n def mock_parse_config_descriptor(name, entry, lock):\n # Verify git entry structure with destination\n assert isinstance(entry, dict)\n assert \"git\" in entry\n assert \"destination\" in entry\n assert entry[\"destination\"] == \"/custom/destination\"\n return MockDescriptor({\"/git/path\": MockRepo(\"git_repo\")})\n\n monkeypatch.setattr(spack.repo, \"parse_config_descriptor\", mock_parse_config_descriptor)\n\n key = spack.cmd.repo._add_repo(\n \"git@github.com:example/repo.git\",\n name=\"test_git_repo\",\n scope=None,\n paths=[],\n destination=\"/custom/destination\",\n config=config,\n )\n\n assert key == \"test_git_repo\"\n\n\ndef test_add_repo_git_url_with_single_repo_path_new(monkeypatch):\n \"\"\"Test successful addition of a git repository with repo_path.\"\"\"\n config = make_repo_config()\n\n def mock_parse_config_descriptor(name, entry, lock):\n # Verify git entry structure with repo_path\n assert isinstance(entry, dict)\n assert \"git\" in entry\n assert \"paths\" in entry\n assert entry[\"paths\"] == [\"subdirectory/repo\"]\n return MockDescriptor({\"/git/path\": MockRepo(\"git_repo\")})\n\n monkeypatch.setattr(spack.repo, \"parse_config_descriptor\", mock_parse_config_descriptor)\n\n key = spack.cmd.repo._add_repo(\n \"https://github.com/example/repo.git\",\n name=\"test_git_repo\",\n scope=None,\n paths=[\"subdirectory/repo\"],\n destination=None,\n config=config,\n )\n\n assert key == \"test_git_repo\"\n\n\ndef test_add_repo_local_path_success(monkeypatch, tmp_path: pathlib.Path):\n \"\"\"Test successful addition of a local repository.\"\"\"\n config = make_repo_config()\n\n def mock_parse_config_descriptor(name, entry, lock):\n # Verify local path entry\n assert isinstance(entry, str)\n return MockDescriptor({str(tmp_path): MockRepo(\"test_repo\")})\n\n monkeypatch.setattr(spack.repo, \"parse_config_descriptor\", mock_parse_config_descriptor)\n\n key = spack.cmd.repo._add_repo(\n str(tmp_path),\n name=\"test_local_repo\",\n scope=None,\n paths=[],\n destination=None,\n config=config,\n )\n\n assert key == \"test_local_repo\"\n # Verify the local path was added\n repos_config = config.get(\"repos\")\n assert \"test_local_repo\" in repos_config\n assert repos_config[\"test_local_repo\"] == str(tmp_path)\n\n\ndef test_add_repo_auto_name_from_namespace(monkeypatch, tmp_path: pathlib.Path):\n \"\"\"Test successful addition of a repository with auto-generated name from namespace.\"\"\"\n config = make_repo_config()\n\n def mock_parse_config_descriptor(name, entry, lock):\n return MockDescriptor({str(tmp_path): MockRepo(\"auto_name_repo\")})\n\n monkeypatch.setattr(spack.repo, \"parse_config_descriptor\", mock_parse_config_descriptor)\n\n key = spack.cmd.repo._add_repo(\n str(tmp_path),\n name=None, # No name specified, should use namespace\n scope=None,\n paths=[],\n destination=None,\n config=config,\n )\n\n assert key == \"auto_name_repo\"\n # Verify the repo was added with the namespace as key\n repos_config = config.get(\"repos\", scope=None)\n assert \"auto_name_repo\" in repos_config\n assert repos_config[\"auto_name_repo\"] == str(tmp_path)\n\n\ndef test_add_repo_partial_repo_construction_warning(monkeypatch, capfd):\n \"\"\"Test that _add_repo issues warnings for repos that can't be constructed but\n succeeds if at least one can be.\"\"\"\n\n def mock_parse_config_descriptor(name, entry, lock):\n return MockDescriptor(\n {\n \"/good/path\": MockRepo(\"good_repo\"),\n \"/bad/path\": Exception(\"Failed to construct repo\"),\n }\n )\n\n monkeypatch.setattr(spack.repo, \"parse_config_descriptor\", mock_parse_config_descriptor)\n\n key = spack.cmd.repo._add_repo(\n \"/mixed/path\",\n name=\"test_mixed_repo\",\n scope=None,\n paths=[],\n destination=None,\n config=make_repo_config(),\n )\n\n assert key == \"test_mixed_repo\"\n\n # Check that a warning was issued for the failed repo\n captured = capfd.readouterr()\n assert \"Skipping package repository\" in captured.err\n\n\n@pytest.mark.parametrize(\n \"test_url,expected_type\",\n [\n (\"ssh://git@github.com/user/repo.git\", \"git\"), # ssh URL\n (\"git://github.com/user/repo.git\", \"git\"), # git protocol\n (\"user@host:repo.git\", \"git\"), # SSH short form\n (\"file:///local/path\", \"git\"), # file URL\n (\"/local/path\", \"local\"), # local path\n (\"./relative/path\", \"local\"), # relative path\n (\"C:\\\\Windows\\\\Path\", \"local\"), # Windows path\n ],\n)\ndef test_add_repo_git_url_detection_edge_cases(monkeypatch, test_url, expected_type):\n \"\"\"Test edge cases for git URL detection.\"\"\"\n config = make_repo_config()\n\n def mock_parse_config_descriptor(name, entry, lock):\n return MockDescriptor({\"/path\": MockRepo(\"test_repo\")})\n\n monkeypatch.setattr(spack.repo, \"parse_config_descriptor\", mock_parse_config_descriptor)\n\n spack.cmd.repo._add_repo(\n test_url, name=None, scope=None, paths=[], destination=None, config=config\n )\n\n entry = config.get(\"repos\").get(\"test_repo\")\n\n if expected_type == \"git\":\n assert entry == {\"git\": test_url}\n else:\n assert isinstance(entry, str)\n\n\ndef test_repo_set_git_config(mutable_config):\n \"\"\"Test that 'spack repo set' properly modifies git repository configurations.\"\"\"\n # Set up initial git repository config in defaults scope\n git_url = \"https://github.com/example/test-repo.git\"\n initial_config = {\"repos\": {\"test-repo\": {\"git\": git_url}}}\n spack.config.set(\"repos\", initial_config[\"repos\"], scope=\"site\")\n\n # Test setting destination and paths\n repo(\"set\", \"--scope=user\", \"--destination\", \"/custom/path\", \"test-repo\")\n repo(\"set\", \"--scope=user\", \"--path\", \"subdir1\", \"--path\", \"subdir2\", \"test-repo\")\n\n # Check that the user config has the updated entry\n user_repos = spack.config.get(\"repos\", scope=\"user\")\n assert user_repos[\"test-repo\"][\"paths\"] == [\"subdir1\", \"subdir2\"]\n assert user_repos[\"test-repo\"][\"destination\"] == \"/custom/path\"\n\n # Check that site scope is unchanged\n site_repos = spack.config.get(\"repos\", scope=\"site\")\n assert \"destination\" not in site_repos[\"test-repo\"]\n\n\ndef test_repo_set_nonexistent_repo(mutable_config):\n with pytest.raises(SpackError, match=\"No repository with namespace 'nonexistent'\"):\n repo(\"set\", \"--destination\", \"/some/path\", \"nonexistent\")\n\n\ndef test_repo_set_does_not_work_on_local_path(mutable_config):\n spack.config.set(\"repos\", {\"local-repo\": \"/local/path\"}, scope=\"site\")\n with pytest.raises(SpackError, match=\"is not a git repository\"):\n repo(\"set\", \"--destination\", \"/some/path\", \"local-repo\")\n\n\ndef test_add_repo_prepends_instead_of_appends(monkeypatch, tmp_path: pathlib.Path):\n \"\"\"Test that newly added repositories are prepended to the configuration,\n giving them higher priority than existing repositories.\"\"\"\n existing_path = str(tmp_path / \"existing_repo\")\n new_path = str(tmp_path / \"new_repo\")\n\n config = make_repo_config({\"existing_repo\": existing_path})\n\n def mock_parse_config_descriptor(name, entry, lock):\n return MockDescriptor({new_path: MockRepo(\"new_repo\")})\n\n monkeypatch.setattr(spack.repo, \"parse_config_descriptor\", mock_parse_config_descriptor)\n\n # Add a new repository\n key = spack.cmd.repo._add_repo(\n path_or_repo=new_path,\n name=\"new_repo\",\n scope=None,\n paths=[],\n destination=None,\n config=config,\n )\n\n assert key == \"new_repo\"\n\n # Check that the new repository is first in the configuration\n repos_config = config.get(\"repos\", scope=None)\n repo_names = list(repos_config.keys())\n\n # The new repository should be first (highest priority)\n assert repo_names == [\"new_repo\", \"existing_repo\"]\n assert repos_config[\"new_repo\"] == new_path\n assert repos_config[\"existing_repo\"] == existing_path\n\n\ndef test_repo_list_format_flags(\n mutable_config: spack.config.Configuration, tmp_path: pathlib.Path\n):\n \"\"\"Test the --config-names and --namespaces flags for repo list command\"\"\"\n # Fake a git monorepo with two package repositories\n (tmp_path / \"monorepo\" / \".git\").mkdir(parents=True)\n repo(\"create\", str(tmp_path / \"monorepo\"), \"repo_one\")\n repo(\"create\", str(tmp_path / \"monorepo\"), \"repo_two\")\n\n mutable_config.set(\n \"repos\",\n {\n # git repo that provides two package repositories\n \"monorepo\": {\n \"git\": \"https://example.com/monorepo.git\",\n \"destination\": str(tmp_path / \"monorepo\"),\n \"paths\": [\"spack_repo/repo_one\", \"spack_repo/repo_two\"],\n },\n # git repo that is not yet cloned\n \"uninitialized\": {\n \"git\": \"https://example.com/uninitialized.git\",\n \"destination\": str(tmp_path / \"uninitialized\"),\n },\n # invalid local repository\n \"misconfigured\": str(tmp_path / \"misconfigured\"),\n },\n scope=\"site\",\n )\n\n # Test default table format, which shows one line per package repository\n table_output = repo(\"list\")\n assert \"[+] repo_one\" in table_output\n assert \"[+] repo_two\" in table_output\n assert \" - uninitialized\" in table_output\n assert \"[-] misconfigured\" in table_output\n\n # Test --namespaces flag\n namespaces_output = repo(\"list\", \"--namespaces\")\n assert namespaces_output.strip().split(\"\\n\") == [\"repo_one\", \"repo_two\"]\n\n # Test --names flag\n config_names_output = repo(\"list\", \"--names\")\n config_names_lines = config_names_output.strip().split(\"\\n\")\n assert config_names_lines == [\"monorepo\", \"uninitialized\", \"misconfigured\"]\n\n\ndef test_repo_list_json_output(mutable_config: spack.config.Configuration, tmp_path: pathlib.Path):\n \"\"\"Test the --json flag for repo list command.\n\n This test verifies that:\n 1. The --json flag produces valid JSON output\n 2. The output contains the expected repository information\n 3. Different repository types (installed, uninitialized, error)\n are correctly represented\n \"\"\"\n import json\n\n # Fake a git monorepo with two package repositories\n monorepo_path = tmp_path / \"monorepo\"\n (monorepo_path / \".git\").mkdir(parents=True)\n repo(\"create\", str(monorepo_path), \"repo_one\")\n repo(\"create\", str(monorepo_path), \"repo_two\")\n\n # Configure repositories in Spack\n test_repos = {\n # git repo that provides two package repositories\n \"monorepo\": {\n \"git\": \"https://example.com/monorepo.git\",\n \"destination\": str(monorepo_path),\n \"paths\": [\"spack_repo/repo_one\", \"spack_repo/repo_two\"],\n },\n # git repo that is not yet cloned\n \"uninitialized\": {\n \"git\": \"https://example.com/uninitialized.git\",\n \"destination\": str(tmp_path / \"uninitialized\"),\n },\n # invalid local repository\n \"misconfigured\": str(tmp_path / \"misconfigured\"),\n }\n mutable_config.set(\"repos\", test_repos, scope=\"site\")\n\n # Get and parse JSON output\n json_output = repo(\"list\", \"--json\")\n repo_data = json.loads(json_output)\n\n # Verify we got a list of repositories\n assert isinstance(repo_data, list), \"Expected JSON output to be a list\"\n\n # Index repositories by namespace for easier validation\n repos_by_namespace = {}\n for item in repo_data:\n # Check all required fields are present\n required_fields = [\"name\", \"namespace\", \"path\", \"api_version\", \"status\", \"error\"]\n for field in required_fields:\n assert field in item, f\"Repository missing required field: {field}\"\n\n # Store by namespace for later validation\n repos_by_namespace[item[\"namespace\"]] = item\n\n # Verify installed repositories (repo_one and repo_two)\n for namespace in [\"repo_one\", \"repo_two\"]:\n assert namespace in repos_by_namespace, f\"Missing repository: {namespace}\"\n repo_info = repos_by_namespace[namespace]\n assert repo_info[\"name\"] == \"monorepo\", f\"Incorrect name for {namespace}\"\n assert repo_info[\"status\"] == \"installed\", f\"Incorrect status for {namespace}\"\n assert repo_info[\"error\"] is None, f\"Unexpected error for {namespace}\"\n assert repo_info[\"api_version\"], f\"Missing API version for {namespace}\"\n\n # Verify uninitialized repository\n assert \"uninitialized\" in repos_by_namespace, \"Missing uninitialized repository\"\n uninit_repo = repos_by_namespace[\"uninitialized\"]\n assert uninit_repo[\"name\"] == \"uninitialized\", \"Incorrect name for uninitialized repo\"\n assert uninit_repo[\"status\"] == \"uninitialized\", \"Incorrect status for uninitialized repo\"\n\n # Verify misconfigured repository\n assert \"misconfigured\" in repos_by_namespace, \"Missing misconfigured repository\"\n misc_repo = repos_by_namespace[\"misconfigured\"]\n assert misc_repo[\"name\"] == \"misconfigured\", \"Incorrect name for misconfigured repo\"\n assert misc_repo[\"status\"] == \"error\", \"Incorrect status for misconfigured repo\"\n assert misc_repo[\"error\"] is not None, \"Missing error message for misconfigured repo\"\n\n\n@pytest.mark.parametrize(\n \"repo_name,flags\",\n [\n (\"new_repo\", []),\n (\"new_repo\", [\"--branch\", \"develop\"]),\n (\"new_repo\", [\"--branch\", \"develop\", \"--remote\", \"upstream\"]),\n (\"new_repo\", [\"--tag\", \"v1.0\"]),\n (\"new_repo\", [\"--commit\", \"abc123\"]),\n ],\n)\ndef test_repo_update_successful_flags(monkeypatch, mutable_config, repo_name, flags):\n \"\"\"Test repo update with flags.\"\"\"\n\n def mock_parse_config_descriptor(name, entry, lock):\n return MockDescriptor({\"/path\": MockRepo(\"new_repo\")})\n\n monkeypatch.setattr(spack.repo, \"parse_config_descriptor\", mock_parse_config_descriptor)\n monkeypatch.setattr(spack.repo, \"RemoteRepoDescriptor\", MockDescriptor)\n\n repos_config = spack.config.get(\"repos\")\n repos_config[repo_name] = {\"git\": \"https://github.com/example/repo.git\"}\n spack.config.set(\"repos\", repos_config)\n\n repo(\"update\", repo_name, *flags)\n\n # check that the branch,tag,commit was updated in the configuration\n repos_config = spack.config.get(\"repos\")\n\n if \"--branch\" in flags:\n assert repos_config[repo_name][\"branch\"] == \"develop\"\n\n if \"--tag\" in flags:\n assert repos_config[repo_name][\"tag\"] == \"v1.0\"\n\n if \"--commit\" in flags:\n assert repos_config[repo_name][\"commit\"] == \"abc123\"\n\n\n@pytest.mark.parametrize(\n \"flags\",\n [\n [\"--branch\", \"develop\"],\n [\"--branch\", \"develop\", \"new_repo_1\", \"new_repo_2\"],\n [\"--branch\", \"develop\", \"unknown_repo\"],\n ],\n)\ndef test_repo_update_invalid_flags(monkeypatch, mutable_config, flags):\n \"\"\"Test repo update with invalid flags.\"\"\"\n\n with pytest.raises(SpackError):\n repo(\"update\", *flags)\n\n\ndef test_repo_show_version_updates_no_changes(mock_git_package_changes):\n \"\"\"Test that show-version-updates handles empty results gracefully\"\"\"\n test_repo, _, commits = mock_git_package_changes\n\n with spack.repo.use_repositories(test_repo):\n # Use the same commit for both refs - no changes\n output = repo(\"show-version-updates\", test_repo.root, commits[-1], commits[-1])\n\n # Should have warning message\n assert \"No packages were added or changed\" in output\n\n # Should not have any specs\n assert \"diff-test@\" not in output\n\n\ndef test_repo_show_version_updates_success(mock_git_package_changes):\n \"\"\"Test that show-version-updates successfully outputs the correct specs\"\"\"\n test_repo, _, commits = mock_git_package_changes\n\n with spack.repo.use_repositories(test_repo):\n # commits are ordered from newest to oldest after reversal\n # commits[-2] = add v2.1.5, commits[-4] = add v2.1.7 and v2.1.8\n # Find versions added between these commits\n # Includes v2.1.6 (git version), v2.1.7, and v2.1.8 (sha256 versions)\n output = repo(\"show-version-updates\", test_repo.root, commits[-2], commits[-4])\n\n # Verify all three versions are included\n assert \"diff-test@\" in output\n assert \"2.1.6\" in output\n assert \"2.1.7\" in output\n assert \"2.1.8\" in output\n\n # Should have three specs\n lines = [\n line.strip()\n for line in output.strip().split(\"\\n\")\n if line.strip() and \"Warning\" not in line\n ]\n assert len(lines) == 3\n\n\ndef test_repo_show_version_updates_excludes_manual_packages(monkeypatch, mock_git_package_changes):\n \"\"\"Test --no-manual-packages flag excludes packages with manual_download=True\"\"\"\n test_repo, _, commits = mock_git_package_changes\n\n with spack.repo.use_repositories(test_repo):\n # Set manual_download=True on the package\n pkg_class = spack.repo.PATH.get_pkg_class(\"diff-test\")\n monkeypatch.setattr(pkg_class, \"manual_download\", True)\n\n # Run show-version-updates with --no-manual-packages flag\n output = repo(\n \"show-version-updates\",\n \"--no-manual-packages\",\n test_repo.root,\n commits[-2],\n commits[-4],\n )\n\n # Package should be excluded\n assert \"diff-test@\" not in output\n assert \"No packages were added or changed\" in output\n\n\ndef test_repo_show_version_updates_excludes_non_redistributable(\n monkeypatch, mock_git_package_changes\n):\n \"\"\"Test --only-redistributable flag excludes packages if redistribute_source returns False\"\"\"\n test_repo, _, commits = mock_git_package_changes\n\n with spack.repo.use_repositories(test_repo):\n # Set redistribute_source to return False\n pkg_class = spack.repo.PATH.get_pkg_class(\"diff-test\")\n monkeypatch.setattr(pkg_class, \"redistribute_source\", classmethod(lambda cls, spec: False))\n\n # Run show-version-updates with --only-redistributable flag\n output = repo(\n \"show-version-updates\",\n \"--only-redistributable\",\n test_repo.root,\n commits[-2],\n commits[-4],\n )\n\n # Package should be excluded\n assert \"diff-test@\" not in output\n assert \"No new package versions found\" in output\n\n\ndef test_repo_show_version_updates_excludes_git_versions(mock_git_package_changes):\n \"\"\"Test --no-git-versions flag excludes versions from git (tag/commit)\"\"\"\n test_repo, _, commits = mock_git_package_changes\n\n with spack.repo.use_repositories(test_repo):\n # commits[-3] = add v2.1.6 (git version), commits[-4] = add v2.1.7 and v2.1.8 (sha256)\n # Without --no-git-versions, v2.1.6 would be included\n output = repo(\n \"show-version-updates\", \"--no-git-versions\", test_repo.root, commits[-3], commits[-4]\n )\n\n # v2.1.6 (git version) should be excluded\n assert \"2.1.6\" not in output\n\n # v2.1.7 and v2.1.8 (sha256 versions) should be included\n assert \"diff-test@\" in output\n assert \"2.1.7\" in output\n assert \"2.1.8\" in output\n" }, { "path": "lib/spack/spack/test/cmd/resource.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport os\nimport sys\n\nfrom spack.main import SpackCommand\n\nresource = SpackCommand(\"resource\")\n\n#: these are hashes used in mock packages\nmock_hashes = (\n [\n \"abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234\",\n \"1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd\",\n \"b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c\",\n \"c45c1564f70def3fc1a6e22139f62cb21cd190cc3a7dbe6f4120fa59ce33dcb8\",\n \"24eceabef5fe8f575ff4b438313dc3e7b30f6a2d1c78841fbbe3b9293a589277\",\n \"689b8f9b32cb1d2f9271d29ea3fca2e1de5df665e121fca14e1364b711450deb\",\n \"208fcfb50e5a965d5757d151b675ca4af4ce2dfd56401721b6168fae60ab798f\",\n \"bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c\",\n \"7d865e959b2466918c9863afca942d0fb89d7c9ac0c99bafc3749504ded97730\",\n ]\n if sys.platform != \"win32\"\n else [\n \"abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234\",\n \"1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd\",\n \"d0df7988457ec999c148a4a2af25ce831bfaad13954ba18a4446374cb0aef55e\",\n \"aeb16c4dec1087e39f2330542d59d9b456dd26d791338ae6d80b6ffd10c89dfa\",\n \"mid21234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234\",\n \"ff34cb21271d16dbf928374f610bb5dd593d293d311036ddae86c4846ff79070\",\n \"bf874c7dd3a83cf370fdc17e496e341de06cd596b5c66dbf3c9bb7f6c139e3ee\",\n \"3c5b65abcd6a3b2c714dbf7c31ff65fe3748a1adc371f030c283007ca5534f11\",\n ]\n)\n\n\ndef test_resource_list(mock_packages):\n out = resource(\"list\")\n\n for h in mock_hashes:\n assert h in out\n\n assert \"url:\" in out\n assert \"applies to:\" in out\n assert \"patched by:\" in out\n assert \"path:\" in out\n\n assert (\n os.path.join(\n \"spack_repo\", \"builtin_mock\", \"packages\", \"patch_a_dependency\", \"libelf.patch\"\n )\n in out\n )\n assert \"applies to: builtin_mock.libelf\" in out\n assert \"patched by: builtin_mock.patch-a-dependency\" in out\n\n\ndef test_resource_list_only_hashes(mock_packages):\n out = resource(\"list\", \"--only-hashes\")\n\n for h in mock_hashes:\n assert h in out\n\n\ndef test_resource_show(mock_packages):\n test_hash = (\n \"c45c1564f70def3fc1a6e22139f62cb21cd190cc3a7dbe6f4120fa59ce33dcb8\"\n if sys.platform != \"win32\"\n else \"3c5b65abcd6a3b2c714dbf7c31ff65fe3748a1adc371f030c283007ca5534f11\"\n )\n out = resource(\"show\", test_hash)\n\n assert out.startswith(test_hash)\n assert (\n os.path.join(\n \"spack_repo\", \"builtin_mock\", \"packages\", \"patch_a_dependency\", \"libelf.patch\"\n )\n in out\n )\n assert \"applies to: builtin_mock.libelf\" in out\n assert \"patched by: builtin_mock.patch-a-dependency\" in out\n\n assert len(out.strip().split(\"\\n\")) == 4\n" }, { "path": "lib/spack/spack/test/cmd/spec.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport re\n\nimport pytest\n\nimport spack.config\nimport spack.environment as ev\nimport spack.error\nimport spack.spec\nimport spack.store\nfrom spack.main import SpackCommand, SpackCommandError\n\n# Unit tests should not be affected by the user's managed environments\npytestmark = pytest.mark.usefixtures(\n \"mutable_mock_env_path\", \"mutable_config\", \"mutable_mock_repo\"\n)\n\nspec = SpackCommand(\"spec\")\n\n\ndef test_spec():\n output = spec(\"mpileaks\")\n\n assert \"mpileaks@2.3\" in output\n assert \"callpath@1.0\" in output\n assert \"dyninst@8.2\" in output\n assert \"libdwarf@20130729\" in output\n assert \"libelf@0.8.1\" in output\n assert \"mpich@3.0.4\" in output\n\n\ndef test_spec_concretizer_args(mutable_database):\n \"\"\"End-to-end test of CLI concretizer prefs.\n\n It's here to make sure that everything works from CLI\n options to `solver.py`, and that config options are not\n lost along the way.\n \"\"\"\n # remove two non-preferred mpileaks installations\n # so that reuse will pick up the zmpi one\n uninstall = SpackCommand(\"uninstall\")\n uninstall(\"-y\", \"mpileaks^mpich\")\n uninstall(\"-y\", \"mpileaks^mpich2\")\n\n # get the hash of mpileaks^zmpi\n mpileaks_zmpi = spack.store.STORE.db.query_one(\"mpileaks^zmpi\")\n h = mpileaks_zmpi.dag_hash()[:7]\n\n output = spec(\"--fresh\", \"-l\", \"mpileaks\")\n assert h not in output\n\n output = spec(\"--reuse\", \"-l\", \"mpileaks\")\n assert h in output\n\n\ndef test_spec_parse_dependency_variant_value():\n \"\"\"Verify that we can provide multiple key=value variants to multiple separate\n packages within a spec string.\"\"\"\n output = spec(\"multivalue-variant fee=barbaz ^ pkg-a foobar=baz\")\n\n assert \"fee=barbaz\" in output\n assert \"foobar=baz\" in output\n\n\ndef test_spec_parse_cflags_quoting():\n \"\"\"Verify that compiler flags can be provided to a spec from the command line.\"\"\"\n output = spec(\"--yaml\", 'gcc cflags=\"-Os -pipe\" cxxflags=\"-flto -Os\"')\n gh_flagged = spack.spec.Spec.from_yaml(output)\n\n assert [\"-Os\", \"-pipe\"] == gh_flagged.compiler_flags[\"cflags\"]\n assert [\"-flto\", \"-Os\"] == gh_flagged.compiler_flags[\"cxxflags\"]\n\n\ndef test_spec_yaml():\n output = spec(\"--yaml\", \"mpileaks\")\n\n mpileaks = spack.spec.Spec.from_yaml(output)\n assert \"mpileaks\" in mpileaks\n assert \"callpath\" in mpileaks\n assert \"dyninst\" in mpileaks\n assert \"libdwarf\" in mpileaks\n assert \"libelf\" in mpileaks\n assert \"mpich\" in mpileaks\n\n\ndef test_spec_json():\n output = spec(\"--json\", \"mpileaks\")\n\n mpileaks = spack.spec.Spec.from_json(output)\n assert \"mpileaks\" in mpileaks\n assert \"callpath\" in mpileaks\n assert \"dyninst\" in mpileaks\n assert \"libdwarf\" in mpileaks\n assert \"libelf\" in mpileaks\n assert \"mpich\" in mpileaks\n\n\ndef test_spec_format(mutable_database):\n output = spec(\"--format\", \"{name}-{^mpi.name}\", \"mpileaks^mpich\")\n assert output.rstrip(\"\\n\") == \"mpileaks-mpich\"\n\n\ndef _parse_types(string):\n \"\"\"Parse deptypes for specs from `spack spec -t` output.\"\"\"\n lines = string.strip().split(\"\\n\")\n\n result = {}\n for line in lines:\n match = re.match(r\"\\[([^]]*)\\]\\s*\\^?([^@]*)@\", line)\n if match:\n types, name = match.groups()\n result.setdefault(name, []).append(types)\n result[name] = sorted(result[name])\n return result\n\n\ndef test_spec_deptypes_nodes():\n output = spec(\"--types\", \"--cover\", \"nodes\", \"--no-install-status\", \"dt-diamond\")\n types = _parse_types(output)\n\n assert types[\"dt-diamond\"] == [\" \"]\n assert types[\"dt-diamond-left\"] == [\"bl \"]\n assert types[\"dt-diamond-right\"] == [\"bl \"]\n assert types[\"dt-diamond-bottom\"] == [\"blr \"]\n\n\ndef test_spec_deptypes_edges():\n output = spec(\"--types\", \"--cover\", \"edges\", \"--no-install-status\", \"dt-diamond\")\n types = _parse_types(output)\n\n assert types[\"dt-diamond\"] == [\" \"]\n assert types[\"dt-diamond-left\"] == [\"bl \"]\n assert types[\"dt-diamond-right\"] == [\"bl \"]\n assert types[\"dt-diamond-bottom\"] == [\"b \", \"blr \"]\n\n\ndef test_spec_returncode():\n with pytest.raises(SpackCommandError):\n spec()\n assert spec.returncode == 1\n\n\ndef test_spec_parse_error():\n with pytest.raises(spack.error.SpecSyntaxError) as e:\n spec(\"1.15:\")\n\n # make sure the error is formatted properly\n error_msg = \"unexpected characters in the spec string\\n1.15:\\n ^\"\n assert error_msg in str(e.value)\n\n\ndef test_env_aware_spec(mutable_mock_env_path):\n env = ev.create(\"test\")\n env.add(\"mpileaks\")\n\n with env:\n output = spec()\n assert \"mpileaks@2.3\" in output\n assert \"callpath@1.0\" in output\n assert \"dyninst@8.2\" in output\n assert \"libdwarf@20130729\" in output\n assert \"libelf@0.8.1\" in output\n assert \"mpich@3.0.4\" in output\n\n\n@pytest.mark.parametrize(\n \"name, version, error\",\n [\n (\"develop-branch-version\", \"f3c7206350ac8ee364af687deaae5c574dcfca2c=develop\", None),\n (\"develop-branch-version\", \"git.\" + \"a\" * 40 + \"=develop\", None),\n (\"callpath\", \"f3c7206350ac8ee364af687deaae5c574dcfca2c=1.0\", spack.error.PackageError),\n (\"develop-branch-version\", \"git.foo=0.2.15\", None),\n ],\n)\n@pytest.mark.use_package_hash\ndef test_spec_version_assigned_git_ref_as_version(name, version, error):\n if error:\n with pytest.raises(error):\n output = spec(name + \"@\" + version)\n else:\n output = spec(name + \"@\" + version)\n assert version in output\n\n\n@pytest.mark.parametrize(\n \"unify, spec_hash_args, match, error\",\n [\n # success cases with unfiy:true\n (True, [\"mpileaks_mpich\"], \"mpich\", None),\n (True, [\"mpileaks_zmpi\"], \"zmpi\", None),\n (True, [\"mpileaks_mpich\", \"dyninst\"], \"mpich\", None),\n (True, [\"mpileaks_zmpi\", \"dyninst\"], \"zmpi\", None),\n # same success cases with unfiy:false\n (False, [\"mpileaks_mpich\"], \"mpich\", None),\n (False, [\"mpileaks_zmpi\"], \"zmpi\", None),\n (False, [\"mpileaks_mpich\", \"dyninst\"], \"mpich\", None),\n (False, [\"mpileaks_zmpi\", \"dyninst\"], \"zmpi\", None),\n # cases with unfiy:false\n (True, [\"mpileaks_mpich\", \"mpileaks_zmpi\"], \"callpath, mpileaks\", spack.error.SpecError),\n (False, [\"mpileaks_mpich\", \"mpileaks_zmpi\"], \"zmpi\", None),\n ],\n)\ndef test_spec_unification_from_cli(\n install_mockery, mutable_config, mutable_database, unify, spec_hash_args, match, error\n):\n \"\"\"Ensure specs grouped together on the CLI are concretized together when unify:true.\"\"\"\n spack.config.set(\"concretizer:unify\", unify)\n\n db = spack.store.STORE.db\n spec_lookup = {\n \"mpileaks_mpich\": db.query_one(\"mpileaks ^mpich\").dag_hash(),\n \"mpileaks_zmpi\": db.query_one(\"mpileaks ^zmpi\").dag_hash(),\n \"dyninst\": db.query_one(\"dyninst\").dag_hash(),\n }\n\n hashes = [f\"/{spec_lookup[name]}\" for name in spec_hash_args]\n if error:\n with pytest.raises(error, match=match):\n output = spec(*hashes)\n else:\n output = spec(*hashes)\n assert match in output\n" }, { "path": "lib/spack/spack/test/cmd/stage.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport pathlib\n\nimport pytest\n\nimport spack.config\nimport spack.environment as ev\nimport spack.package_base\nimport spack.traverse\nfrom spack.cmd.stage import StageFilter\nfrom spack.main import SpackCommand, SpackCommandError\nfrom spack.spec import Spec\nfrom spack.version import Version\n\nstage = SpackCommand(\"stage\")\nenv = SpackCommand(\"env\")\n\npytestmark = pytest.mark.usefixtures(\"install_mockery\", \"mock_packages\")\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_stage_spec(monkeypatch):\n \"\"\"Verify that staging specs works.\"\"\"\n\n expected = set([\"trivial-install-test-package\", \"mpileaks\"])\n\n def fake_stage(pkg, mirror_only=False):\n expected.remove(pkg.name)\n\n monkeypatch.setattr(spack.package_base.PackageBase, \"do_stage\", fake_stage)\n\n stage(\"trivial-install-test-package\", \"mpileaks\")\n\n assert len(expected) == 0\n\n\n@pytest.fixture(scope=\"function\")\ndef check_stage_path(monkeypatch, tmp_path: pathlib.Path):\n expected_path = str(tmp_path / \"x\")\n\n def fake_stage(pkg, mirror_only=False):\n assert pkg.path == expected_path\n\n monkeypatch.setattr(spack.package_base.PackageBase, \"do_stage\", fake_stage)\n\n return expected_path\n\n\ndef test_stage_path(check_stage_path):\n \"\"\"Verify that --path only works with single specs.\"\"\"\n stage(\"--path={0}\".format(check_stage_path), \"trivial-install-test-package\")\n\n\ndef test_stage_path_errors_multiple_specs(check_stage_path):\n \"\"\"Verify that --path only works with single specs.\"\"\"\n with pytest.raises(SpackCommandError):\n stage(f\"--path={check_stage_path}\", \"trivial-install-test-package\", \"mpileaks\")\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_stage_with_env_outside_env(mutable_mock_env_path, monkeypatch):\n \"\"\"Verify that stage concretizes specs not in environment instead of erroring.\"\"\"\n\n def fake_stage(pkg, mirror_only=False):\n assert pkg.name == \"trivial-install-test-package\"\n assert pkg.path is None\n\n monkeypatch.setattr(spack.package_base.PackageBase, \"do_stage\", fake_stage)\n\n e = ev.create(\"test\")\n e.add(\"mpileaks\")\n e.concretize()\n\n with e:\n stage(\"trivial-install-test-package\")\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_stage_with_env_inside_env(mutable_mock_env_path, monkeypatch):\n \"\"\"Verify that stage filters specs in environment instead of reconcretizing.\"\"\"\n\n def fake_stage(pkg, mirror_only=False):\n assert pkg.name == \"mpileaks\"\n assert pkg.version == Version(\"100.100\")\n\n monkeypatch.setattr(spack.package_base.PackageBase, \"do_stage\", fake_stage)\n\n e = ev.create(\"test\")\n e.add(\"mpileaks@=100.100\")\n e.concretize()\n\n with e:\n stage(\"mpileaks\")\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_stage_full_env(mutable_mock_env_path, monkeypatch):\n \"\"\"Verify that stage filters specs in environment.\"\"\"\n\n e = ev.create(\"test\")\n e.add(\"mpileaks@=100.100\")\n e.concretize()\n\n # list all the package names that should be staged\n expected, externals = set(), set()\n for dep in spack.traverse.traverse_nodes(e.concrete_roots()):\n expected.add(dep.name)\n if dep.external:\n externals.add(dep.name)\n\n # pop the package name from the list instead of actually staging\n def fake_stage(pkg, mirror_only=False):\n expected.remove(pkg.name)\n\n monkeypatch.setattr(spack.package_base.PackageBase, \"do_stage\", fake_stage)\n\n with e:\n stage()\n\n assert expected == externals\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_concretizer_arguments(mock_packages, mock_fetch):\n \"\"\"Make sure stage also has --reuse and --fresh flags.\"\"\"\n stage(\"--reuse\", \"trivial-install-test-package\")\n assert spack.config.get(\"concretizer:reuse\", None) is True\n\n stage(\"--fresh\", \"trivial-install-test-package\")\n assert spack.config.get(\"concretizer:reuse\", None) is False\n\n\n@pytest.mark.maybeslow\n@pytest.mark.parametrize(\"externals\", [[\"libelf\"], []])\n@pytest.mark.parametrize(\n \"installed, skip_installed\", [([\"libdwarf\"], False), ([\"libdwarf\"], True)]\n)\n@pytest.mark.parametrize(\"exclusions\", [[\"mpich\", \"callpath\"], []])\ndef test_stage_spec_filters(\n mutable_mock_env_path,\n mock_packages,\n mock_fetch,\n externals,\n installed,\n skip_installed,\n exclusions,\n monkeypatch,\n):\n e = ev.create(\"test\")\n e.add(\"mpileaks@=100.100\")\n e.concretize()\n all_specs = e.all_specs()\n\n def is_installed(self):\n return self.name in installed\n\n if skip_installed:\n monkeypatch.setattr(Spec, \"installed\", is_installed)\n\n should_be_filtered = []\n for spec in all_specs:\n for ext in externals:\n if spec.satisfies(Spec(ext)):\n spec.external_path = \"/usr\"\n assert spec.external\n should_be_filtered.append(spec)\n for ins in installed:\n if skip_installed and spec.satisfies(Spec(ins)):\n assert spec.installed\n should_be_filtered.append(spec)\n for exc in exclusions:\n if spec.satisfies(Spec(exc)):\n should_be_filtered.append(spec)\n\n filter = StageFilter(exclusions, skip_installed=skip_installed)\n specs_to_stage = [s for s in all_specs if not filter(s)]\n specs_were_filtered = [skip not in specs_to_stage for skip in should_be_filtered]\n\n assert all(specs_were_filtered), (\n f\"Packages associated with bools: {[s.name for s in should_be_filtered]}\"\n )\n" }, { "path": "lib/spack/spack/test/cmd/style.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport filecmp\nimport io\nimport os\nimport pathlib\nimport shutil\nimport sys\n\nimport pytest\n\nimport spack.cmd.style\nimport spack.main\nimport spack.paths\nimport spack.repo\nfrom spack.cmd.style import _run_import_check, changed_files\nfrom spack.llnl.util.filesystem import FileFilter, working_dir\nfrom spack.util.executable import which\n\n#: directory with sample style files\nstyle_data = os.path.join(spack.paths.test_path, \"data\", \"style\")\n\n\nstyle = spack.main.SpackCommand(\"style\")\n\npytestmark = pytest.mark.skipif(\n sys.platform == \"win32\", reason=\"CI uses cross drive paths that raise errors with relpath\"\n)\n\nRUFF = which(\"ruff\")\nMYPY = which(\"mypy\")\n\n\n@pytest.fixture(autouse=True)\ndef has_develop_branch(git):\n \"\"\"spack style requires git and a develop branch to run -- skip if we're missing either.\"\"\"\n git(\"show-ref\", \"--verify\", \"--quiet\", \"refs/heads/develop\", fail_on_error=False)\n if git.returncode != 0:\n pytest.skip(\"requires git and a develop branch\")\n\n\n@pytest.fixture(scope=\"function\")\ndef ruff_package(tmp_path: pathlib.Path):\n \"\"\"Style only checks files that have been modified. This fixture makes a small\n change to the ``ruff`` mock package, yields the filename, then undoes the\n change on cleanup.\n \"\"\"\n repo = spack.repo.from_path(spack.paths.mock_packages_path)\n filename = repo.filename_for_package_name(\"ruff\")\n rel_path = os.path.dirname(os.path.relpath(filename, spack.paths.prefix))\n tmp = tmp_path / rel_path / \"ruff-ci-package.py\"\n tmp.parent.mkdir(parents=True, exist_ok=True)\n tmp.touch()\n tmp = str(tmp)\n\n shutil.copy(filename, tmp)\n package = FileFilter(tmp)\n package.filter(\"state = 'unmodified'\", \"state = 'modified'\", string=True)\n yield tmp\n\n\n@pytest.fixture\ndef ruff_package_with_errors(scope=\"function\"):\n \"\"\"A ruff package with errors.\"\"\"\n repo = spack.repo.from_path(spack.paths.mock_packages_path)\n filename = repo.filename_for_package_name(\"ruff\")\n tmp = filename + \".tmp\"\n\n shutil.copy(filename, tmp)\n package = FileFilter(tmp)\n\n # this is a ruff error (quote style and spacing before/after operator)\n package.filter('state = \"unmodified\"', \"state = 'modified'\", string=True)\n\n # this is two ruff errors (unused import) (orderign)\n package.filter(\n \"from spack.package import *\", \"from spack.package import *\\nimport os\", string=True\n )\n yield tmp\n\n\ndef test_changed_files_from_git_rev_base(git, tmp_path: pathlib.Path):\n \"\"\"Test arbitrary git ref as base.\"\"\"\n with working_dir(str(tmp_path)):\n git(\"init\")\n git(\"checkout\", \"-b\", \"main\")\n git(\"config\", \"user.name\", \"test user\")\n git(\"config\", \"user.email\", \"test@user.com\")\n git(\"commit\", \"--no-gpg-sign\", \"--allow-empty\", \"-m\", \"initial commit\")\n\n (tmp_path / \"bin\").mkdir(parents=True, exist_ok=True)\n (tmp_path / \"bin\" / \"spack\").touch()\n assert changed_files(base=\"HEAD\") == [pathlib.Path(\"bin/spack\")]\n assert changed_files(base=\"main\") == [pathlib.Path(\"bin/spack\")]\n\n git(\"add\", \"bin/spack\")\n git(\"commit\", \"--no-gpg-sign\", \"-m\", \"v1\")\n assert changed_files(base=\"HEAD\") == []\n assert changed_files(base=\"HEAD~\") == [pathlib.Path(\"bin/spack\")]\n\n\ndef test_changed_no_base(git, tmp_path: pathlib.Path, capfd):\n \"\"\"Ensure that we fail gracefully with no base branch.\"\"\"\n (tmp_path / \"bin\").mkdir(parents=True, exist_ok=True)\n (tmp_path / \"bin\" / \"spack\").touch()\n with working_dir(str(tmp_path)):\n git(\"init\")\n git(\"config\", \"user.name\", \"test user\")\n git(\"config\", \"user.email\", \"test@user.com\")\n git(\"add\", \".\")\n git(\"commit\", \"--no-gpg-sign\", \"-m\", \"initial commit\")\n\n with pytest.raises(SystemExit):\n changed_files(base=\"foobar\")\n\n out, err = capfd.readouterr()\n assert \"This repository does not have a 'foobar'\" in err\n\n\ndef test_changed_files_all_files(mock_packages):\n # it's hard to guarantee \"all files\", so do some sanity checks.\n files = {\n os.path.join(spack.paths.prefix, os.path.normpath(path))\n for path in changed_files(all_files=True)\n }\n\n # spack has a lot of files -- check that we're in the right ballpark\n assert len(files) > 500\n\n # a builtin package\n zlib = spack.repo.PATH.get_pkg_class(\"zlib\")\n zlib_file = zlib.module.__file__\n if zlib_file.endswith(\"pyc\"):\n zlib_file = zlib_file[:-1]\n assert zlib_file in files\n\n # a core spack file\n assert os.path.join(spack.paths.module_path, \"spec.py\") in files\n\n # a mock package\n repo = spack.repo.from_path(spack.paths.mock_packages_path)\n filename = repo.filename_for_package_name(\"ruff\")\n assert filename in files\n\n # this test\n assert __file__ in files\n\n # ensure externals are excluded\n assert not any(f.startswith(spack.paths.vendor_path) for f in files)\n\n\ndef test_bad_root(tmp_path: pathlib.Path):\n \"\"\"Ensure that `spack style` doesn't run on non-spack directories.\"\"\"\n output = style(\"--root\", str(tmp_path), fail_on_error=False)\n assert \"This does not look like a valid spack root\" in output\n assert style.returncode != 0\n\n\n@pytest.fixture\ndef external_style_root(git, ruff_package_with_errors, tmp_path: pathlib.Path):\n \"\"\"Create a mock repository for running spack style.\"\"\"\n # create a sort-of spack-looking directory\n script = tmp_path / \"bin\" / \"spack\"\n script.parent.mkdir(parents=True, exist_ok=True)\n script.touch()\n spack_dir = tmp_path / \"lib\" / \"spack\" / \"spack\"\n spack_dir.mkdir(parents=True, exist_ok=True)\n (spack_dir / \"__init__.py\").touch()\n llnl_dir = tmp_path / \"lib\" / \"spack\" / \"llnl\"\n llnl_dir.mkdir(parents=True, exist_ok=True)\n (llnl_dir / \"__init__.py\").touch()\n\n # create a base develop branch\n with working_dir(str(tmp_path)):\n git(\"init\")\n git(\"config\", \"user.name\", \"test user\")\n git(\"config\", \"user.email\", \"test@user.com\")\n git(\"add\", \".\")\n git(\"commit\", \"--no-gpg-sign\", \"-m\", \"initial commit\")\n git(\"branch\", \"-m\", \"develop\")\n git(\"checkout\", \"-b\", \"feature\")\n\n # copy the buggy package in\n py_file = spack_dir / \"dummy.py\"\n py_file.touch()\n shutil.copy(ruff_package_with_errors, str(py_file))\n\n yield tmp_path, py_file\n\n\n@pytest.mark.skipif(not RUFF, reason=\"ruff is not installed.\")\ndef test_fix_style(external_style_root):\n \"\"\"Make sure spack style --fix works.\"\"\"\n tmp_path, py_file = external_style_root\n\n broken_dummy = os.path.join(style_data, \"broken.dummy\")\n broken_py = str(tmp_path / \"lib\" / \"spack\" / \"spack\" / \"broken.py\")\n fixed_py = os.path.join(style_data, \"fixed.py\")\n\n shutil.copy(broken_dummy, broken_py)\n assert not filecmp.cmp(broken_py, fixed_py)\n\n # dummy.py is in the same directory and will raise errors unrelated to this\n # check, don't fail on those errors, just check to make sure\n # we fixed the intended file correctly\n # Note: can't just specify the correct file due to cross drive issues on Windows\n style(\n \"--root\", str(tmp_path), \"--tool\", \"ruff-check,ruff-format\", \"--fix\", fail_on_error=False\n )\n assert filecmp.cmp(broken_py, fixed_py)\n\n\n@pytest.mark.skipif(not RUFF, reason=\"ruff is not installed.\")\n@pytest.mark.skipif(not MYPY, reason=\"mypy is not installed.\")\ndef test_external_root(external_style_root):\n \"\"\"Ensure we can run in a separate root directory w/o configuration files.\"\"\"\n tmp_path, py_file = external_style_root\n\n # make sure tools are finding issues with external root,\n # not the real one.\n output = style(\"--root-relative\", \"--root\", str(tmp_path), fail_on_error=False)\n\n # make sure it failed\n assert style.returncode != 0\n\n # ruff-check error\n assert \"Import block is un-sorted or un-formatted\\n --> lib/spack/spack/dummy.py\" in output\n\n # mypy error\n assert 'lib/spack/spack/dummy.py:45: error: Name \"version\" is not defined' in output\n\n # ruff-format error\n assert \"--- lib/spack/spack/dummy.py\" in output\n assert \"+++ lib/spack/spack/dummy.py\" in output\n\n # ruff-check error\n assert \"`os` imported but unused\\n --> lib/spack/spack/dummy.py\" in output\n\n\n@pytest.mark.skipif(not RUFF, reason=\"ruff is not installed.\")\ndef test_style(ruff_package, tmp_path: pathlib.Path):\n root_relative = os.path.relpath(ruff_package, spack.paths.prefix)\n\n # use a working directory to test cwd-relative paths, as tests run in\n # the spack prefix by default\n with working_dir(str(tmp_path)):\n relative = os.path.relpath(ruff_package)\n\n # one specific arg\n output = style(\"--tool\", \"ruff-check\", ruff_package, fail_on_error=False)\n assert relative in output\n assert \"spack style checks were clean\" in output\n\n # specific file that isn't changed\n output = style(\"--tool\", \"ruff-check\", __file__, fail_on_error=False)\n assert relative not in output\n assert __file__ in output\n assert \"spack style checks were clean\" in output\n\n # root-relative paths\n output = style(\"--tool\", \"ruff-check\", \"--root-relative\", ruff_package)\n assert root_relative in output\n assert \"spack style checks were clean\" in output\n\n\n@pytest.mark.skipif(not RUFF, reason=\"ruff is not installed.\")\ndef test_style_with_errors(ruff_package_with_errors):\n root_relative = os.path.relpath(ruff_package_with_errors, spack.paths.prefix)\n output = style(\n \"--tool\", \"ruff-check\", \"--root-relative\", ruff_package_with_errors, fail_on_error=False\n )\n assert root_relative in output\n assert style.returncode != 0\n assert \"spack style found errors\" in output\n\n\n@pytest.mark.skipif(not RUFF, reason=\"ruff is not installed.\")\ndef test_style_with_ruff_format(ruff_package_with_errors):\n output = style(\"--tool\", \"ruff-format\", ruff_package_with_errors, fail_on_error=False)\n assert \"ruff-format found errors\" in output\n assert style.returncode != 0\n assert \"spack style found errors\" in output\n\n\ndef test_skip_tools():\n output = style(\"--skip\", \"import,ruff-check,ruff-format,mypy\")\n assert \"Nothing to run\" in output\n\n\n@pytest.mark.skipif(sys.version_info < (3, 9), reason=\"requires Python 3.9+\")\ndef test_run_import_check(tmp_path: pathlib.Path):\n file = tmp_path / \"issues.py\"\n contents = '''\nimport spack.cmd\nimport spack.config # do not drop this import because of this comment\nimport spack.repo\nimport spack.repo_utils\n\nfrom spack_repo.builtin_mock.build_systems import autotools\n\n# this comment about spack.error should not be removed\nclass Example(autotools.AutotoolsPackage):\n \"\"\"this is a docstring referencing unused spack.error.SpackError, which is fine\"\"\"\n pass\n\ndef foo(config: \"spack.error.SpackError\"):\n # the type hint is quoted, so it should not be removed\n spack.util.executable.Executable(\"example\")\n print(spack.__version__)\n print(spack.repo_utils.__file__)\n\nimport spack.enums\nfrom spack.enums import ConfigScopePriority\n\nimport spack.util.url as url_util\ndef something(y: spack.util.url.Url): ...\n'''\n file.write_text(contents)\n root = str(tmp_path)\n output_buf = io.StringIO()\n exit_code = _run_import_check(\n [file],\n fix=False,\n out=output_buf,\n root_relative=False,\n root=pathlib.Path(spack.paths.prefix),\n working_dir=pathlib.Path(root),\n )\n output = output_buf.getvalue()\n\n assert \"issues.py: redundant import: spack.cmd\" in output\n assert \"issues.py: redundant import: spack.repo\" in output\n assert \"issues.py: redundant import: spack.config\" not in output # comment prevents removal\n assert \"issues.py: redundant import: spack.enums\" in output # imported via from-import\n assert \"issues.py: missing import: spack\" in output # used by spack.__version__\n assert \"issues.py: missing import: spack.util.executable\" in output\n assert \"issues.py: missing import: spack.util.url\" in output # used in type hint\n assert \"issues.py: missing import: spack.error\" not in output # not directly used\n assert exit_code == 1\n assert file.read_text() == contents # fix=False should not change the file\n\n # run it with --fix, should have the same output.\n output_buf = io.StringIO()\n exit_code = _run_import_check(\n [file],\n fix=True,\n out=output_buf,\n root_relative=False,\n root=pathlib.Path(spack.paths.prefix),\n working_dir=pathlib.Path(root),\n )\n output = output_buf.getvalue()\n assert exit_code == 1\n assert \"issues.py: redundant import: spack.cmd\" in output\n assert \"issues.py: redundant import: spack.enums\" in output\n assert \"issues.py: missing import: spack\" in output\n assert \"issues.py: missing import: spack.util.executable\" in output\n assert \"issues.py: missing import: spack.util.url\" in output\n\n # after fix a second fix is idempotent\n output_buf = io.StringIO()\n exit_code = _run_import_check(\n [file],\n fix=True,\n out=output_buf,\n root_relative=False,\n root=pathlib.Path(spack.paths.prefix),\n working_dir=pathlib.Path(root),\n )\n output = output_buf.getvalue()\n assert exit_code == 0\n assert not output\n\n # check that the file was fixed\n new_contents = file.read_text()\n assert \"import spack.cmd\" not in new_contents\n assert \"import spack.enums\" not in new_contents\n assert \"import spack\\n\" in new_contents\n assert \"import spack.util.executable\\n\" in new_contents\n assert \"import spack.util.url\\n\" in new_contents\n\n\n@pytest.mark.skipif(sys.version_info < (3, 9), reason=\"requires Python 3.9+\")\ndef test_run_import_check_syntax_error_and_missing(tmp_path: pathlib.Path):\n (tmp_path / \"syntax-error.py\").write_text(\"\"\"this 'is n(ot python code\"\"\")\n output_buf = io.StringIO()\n exit_code = _run_import_check(\n [tmp_path / \"syntax-error.py\", tmp_path / \"missing.py\"],\n fix=False,\n out=output_buf,\n root_relative=True,\n root=tmp_path,\n working_dir=tmp_path / \"does-not-matter\",\n )\n output = output_buf.getvalue()\n assert \"syntax-error.py: could not parse\" in output\n assert \"missing.py: could not parse\" in output\n assert exit_code == 1\n\n\ndef test_case_sensitive_imports(tmp_path: pathlib.Path):\n # example.Example is a name, while example.example is a module.\n (tmp_path / \"lib\" / \"spack\" / \"example\").mkdir(parents=True)\n (tmp_path / \"lib\" / \"spack\" / \"example\" / \"__init__.py\").write_text(\"class Example:\\n pass\")\n (tmp_path / \"lib\" / \"spack\" / \"example\" / \"example.py\").write_text(\"foo = 1\")\n assert spack.cmd.style._module_part(tmp_path, \"example.Example\") == \"example\"\n\n\ndef test_pkg_imports():\n assert (\n spack.cmd.style._module_part(pathlib.Path(spack.paths.prefix), \"spack.pkg.builtin.boost\")\n is None\n )\n assert spack.cmd.style._module_part(pathlib.Path(spack.paths.prefix), \"spack.pkg\") is None\n" }, { "path": "lib/spack/spack/test/cmd/tags.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport spack.concretize\nimport spack.main\nimport spack.repo\nfrom spack.installer import PackageInstaller\n\ntags = spack.main.SpackCommand(\"tags\")\n\n\ndef test_tags_bad_options(mock_packages):\n out = tags(\"-a\", \"tag1\", fail_on_error=False)\n assert \"option OR provide\" in out\n\n\ndef test_tags_no_installed(install_mockery, mock_fetch):\n out = tags(\"-i\")\n assert \"No installed\" in out\n\n\ndef test_tags_invalid_tag(mock_packages):\n out = tags(\"nosuchtag\")\n assert \"None\" in out\n\n\ndef test_tags_all_mock_tags(mock_packages):\n out = tags()\n for tag in [\"tag1\", \"tag2\", \"tag3\"]:\n assert tag in out\n\n\ndef test_tags_all_mock_tag_packages(mock_packages):\n out = tags(\"-a\")\n for pkg in [\"mpich\\n\", \"mpich2\\n\"]:\n assert pkg in out\n\n\ndef test_tags_no_tags(repo_builder):\n repo_builder.add_package(\"pkg-a\")\n with spack.repo.use_repositories(repo_builder.root):\n out = tags()\n assert \"No tagged\" in out\n\n\ndef test_tags_installed(install_mockery, mock_fetch):\n s = spack.concretize.concretize_one(\"mpich\")\n PackageInstaller([s.package], explicit=True, fake=True).install()\n\n out = tags(\"-i\")\n for tag in [\"tag1\", \"tag2\"]:\n assert tag in out\n\n out = tags(\"-i\", \"tag1\")\n assert \"mpich\" in out\n\n out = tags(\"-i\", \"tag3\")\n assert \"No installed\" in out\n" }, { "path": "lib/spack/spack/test/cmd/test.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport argparse\nimport os\nimport pathlib\n\nimport pytest\n\nimport spack.cmd.common.arguments\nimport spack.cmd.test\nimport spack.concretize\nimport spack.config\nimport spack.install_test\nimport spack.paths\nfrom spack.install_test import TestStatus\nfrom spack.llnl.util.filesystem import copy_tree, working_dir\nfrom spack.main import SpackCommand\n\ninstall = SpackCommand(\"install\")\nspack_test = SpackCommand(\"test\")\n\npytestmark = pytest.mark.not_on_windows(\"does not run on windows\")\n\n\ndef test_test_package_not_installed(\n mock_packages, mock_archive, mock_fetch, install_mockery, mock_test_stage\n):\n output = spack_test(\"run\", \"libdwarf\")\n\n assert \"No installed packages match spec libdwarf\" in output\n\n\n@pytest.mark.parametrize(\n \"arguments,expected\",\n [\n ([\"run\"], spack.config.get(\"config:dirty\")), # default from config file\n ([\"run\", \"--clean\"], False),\n ([\"run\", \"--dirty\"], True),\n ],\n)\ndef test_test_dirty_flag(arguments, expected):\n parser = argparse.ArgumentParser()\n spack.cmd.test.setup_parser(parser)\n args = parser.parse_args(arguments)\n assert args.dirty == expected\n\n\ndef test_test_dup_alias(mock_test_stage, mock_packages, mock_archive, mock_fetch, install_mockery):\n \"\"\"Ensure re-using an alias fails with suggestion to change.\"\"\"\n install(\"--fake\", \"libdwarf\")\n\n # Run the (no) tests with the alias once\n spack_test(\"run\", \"--alias\", \"libdwarf\", \"libdwarf\")\n\n # Try again with the alias but don't let it fail on the error\n out = spack_test(\"run\", \"--alias\", \"libdwarf\", \"libdwarf\", fail_on_error=False)\n\n assert \"already exists\" in out and \"Try another alias\" in out\n\n\ndef test_test_output(mock_test_stage, mock_packages, mock_archive, mock_fetch, install_mockery):\n \"\"\"Ensure output printed from pkgs is captured by output redirection.\"\"\"\n install(\"printing-package\")\n spack_test(\"run\", \"--alias\", \"printpkg\", \"printing-package\")\n\n stage_files = os.listdir(mock_test_stage)\n assert len(stage_files) == 1\n\n # Grab test stage directory contents\n testdir = os.path.join(mock_test_stage, stage_files[0])\n testdir_files = os.listdir(testdir)\n testlogs = [name for name in testdir_files if str(name).endswith(\"out.txt\")]\n assert len(testlogs) == 1\n\n # Grab the output from the test log to confirm expected result\n outfile = os.path.join(testdir, testlogs[0])\n with open(outfile, \"r\", encoding=\"utf-8\") as f:\n output = f.read()\n assert \"test_print\" in output\n assert \"PASSED\" in output\n\n\n@pytest.mark.parametrize(\n \"pkg_name,failure\", [(\"test-error\", \"exited with status 1\"), (\"test-fail\", \"not callable\")]\n)\ndef test_test_output_fails(\n mock_packages, mock_archive, mock_fetch, install_mockery, mock_test_stage, pkg_name, failure\n):\n \"\"\"Confirm stand-alone test failure with expected outputs.\"\"\"\n install(pkg_name)\n out = spack_test(\"run\", pkg_name, fail_on_error=False)\n\n # Confirm package-specific failure is in the output\n assert failure in out\n\n # Confirm standard failure tagging AND test log reference also output\n assert \"TestFailure\" in out\n assert \"See test log for details\" in out\n\n\n@pytest.mark.usefixtures(\"mock_packages\", \"mock_archive\", \"mock_fetch\", \"install_mockery\")\n@pytest.mark.parametrize(\n \"pkg_name,msgs\",\n [\n (\"test-error\", [\"exited with status 1\", \"TestFailure\"]),\n (\"test-fail\", [\"not callable\", \"TestFailure\"]),\n ],\n)\ndef test_junit_output_with_failures(tmp_path: pathlib.Path, mock_test_stage, pkg_name, msgs):\n \"\"\"Confirm stand-alone test failure expected outputs in JUnit reporting.\"\"\"\n install(pkg_name)\n with working_dir(str(tmp_path)):\n spack_test(\n \"run\", \"--log-format=junit\", \"--log-file=test.xml\", pkg_name, fail_on_error=False\n )\n\n files = list(tmp_path.iterdir())\n filename = tmp_path / \"test.xml\"\n assert filename in files\n\n content = filename.read_text()\n\n # Count failures and errors correctly\n assert 'tests=\"1\"' in content\n assert 'failures=\"1\"' in content\n assert 'errors=\"0\"' in content\n\n # We want to have both stdout and stderr\n assert \"\" in content\n for msg in msgs:\n assert msg in content\n\n\ndef test_cdash_output_test_error(\n tmp_path: pathlib.Path,\n mock_fetch,\n install_mockery,\n mock_packages,\n mock_archive,\n mock_test_stage,\n):\n \"\"\"Confirm stand-alone test error expected outputs in CDash reporting.\"\"\"\n install(\"test-error\")\n with working_dir(str(tmp_path)):\n spack_test(\n \"run\",\n \"--log-format=cdash\",\n \"--log-file=cdash_reports\",\n \"test-error\",\n fail_on_error=False,\n )\n report_dir = tmp_path / \"cdash_reports\"\n reports = [name for name in report_dir.iterdir() if str(name).endswith(\"Testing.xml\")]\n assert len(reports) == 1\n content = reports[0].read_text()\n assert \"Command exited with status 1\" in content\n\n\ndef test_cdash_upload_clean_test(\n tmp_path: pathlib.Path,\n mock_fetch,\n install_mockery,\n mock_packages,\n mock_archive,\n mock_test_stage,\n):\n install(\"printing-package\")\n with working_dir(str(tmp_path)):\n spack_test(\"run\", \"--log-file=cdash_reports\", \"--log-format=cdash\", \"printing-package\")\n report_dir = tmp_path / \"cdash_reports\"\n reports = [name for name in report_dir.iterdir() if str(name).endswith(\"Testing.xml\")]\n assert len(reports) == 1\n content = reports[0].read_text()\n assert \"passed\" in content\n assert \"Running test_print\" in content, \"Expected first command output\"\n assert \"second command\" in content, \"Expected second command output\"\n assert \"\" in content\n assert \"\" not in content\n\n\ndef test_test_help_does_not_show_cdash_options(mock_test_stage):\n \"\"\"Make sure `spack test --help` does not describe CDash arguments\"\"\"\n spack_test(\"run\", \"--help\")\n assert \"CDash URL\" not in spack_test.output\n\n\ndef test_test_help_cdash(mock_test_stage):\n \"\"\"Make sure `spack test --help-cdash` describes CDash arguments\"\"\"\n out = spack_test(\"run\", \"--help-cdash\")\n assert \"CDash URL\" in out\n\n\ndef test_test_list_all(mock_packages):\n \"\"\"Confirm `spack test list --all` returns all packages with test methods\"\"\"\n pkgs = spack_test(\"list\", \"--all\").strip().split()\n assert set(pkgs) == {\n \"py-numpy\",\n \"fail-test-audit\",\n \"fail-test-audit-docstring\",\n \"fail-test-audit-impl\",\n \"mpich\",\n \"perl-extension\",\n \"printing-package\",\n \"py-extension1\",\n \"py-extension2\",\n \"py-test-callback\",\n \"simple-standalone-test\",\n \"test-error\",\n \"test-fail\",\n }\n\n\ndef test_test_list(mock_packages, mock_archive, mock_fetch, install_mockery):\n pkg_with_tests = \"printing-package\"\n install(pkg_with_tests)\n output = spack_test(\"list\")\n assert pkg_with_tests in output\n\n\ndef test_read_old_results(mock_packages, mock_test_stage):\n \"\"\"Take test data generated before the switch to full hash everywhere\n and make sure we can still read it in\"\"\"\n # Test data was generated with:\n # spack install printing-package\n # spack test run --alias printpkg printing-package\n\n test_data_src = os.path.join(spack.paths.test_path, \"data\", \"test\", \"test_stage\")\n\n # Copy the old test data into the mock stage directory\n copy_tree(test_data_src, mock_test_stage)\n\n # The find command should print info about the old test, under\n # the alias used at test generation time\n find_output = spack_test(\"find\")\n assert \"printpkg\" in find_output\n\n # The results command should still print the old test results\n results_output = spack_test(\"results\")\n assert str(TestStatus.PASSED) in results_output\n\n\ndef test_test_results_none(mock_packages, mock_test_stage):\n name = \"trivial\"\n spec = spack.concretize.concretize_one(\"trivial-smoke-test\")\n suite = spack.install_test.TestSuite([spec], name)\n suite.ensure_stage()\n spack.install_test.write_test_suite_file(suite)\n results = spack_test(\"results\", name)\n assert \"has no results\" in results\n assert \"if it is running\" in results\n\n\n@pytest.mark.parametrize(\n \"status\", [TestStatus.FAILED, TestStatus.NO_TESTS, TestStatus.SKIPPED, TestStatus.PASSED]\n)\ndef test_test_results_status(mock_packages, mock_test_stage, status):\n \"\"\"Confirm 'spack test results' returns expected status.\"\"\"\n name = \"trivial\"\n spec = spack.concretize.concretize_one(\"trivial-smoke-test\")\n suite = spack.install_test.TestSuite([spec], name)\n suite.ensure_stage()\n spack.install_test.write_test_suite_file(suite)\n suite.write_test_result(spec, status)\n\n for opt in [\"\", \"--failed\", \"--log\"]:\n args = [\"results\", name]\n if opt:\n args.insert(1, opt)\n\n results = spack_test(*args)\n if opt == \"--failed\" and status != TestStatus.FAILED:\n assert str(status) not in results\n else:\n assert str(status) in results\n assert \"1 {0}\".format(status.lower()) in results\n\n\n@pytest.mark.regression(\"35337\")\ndef test_report_filename_for_cdash(install_mockery, mock_fetch):\n \"\"\"Test that the temporary file used to write Testing.xml for CDash is not the upload URL\"\"\"\n name = \"trivial\"\n spec = spack.concretize.concretize_one(\"trivial-smoke-test\")\n suite = spack.install_test.TestSuite([spec], name)\n suite.ensure_stage()\n\n parser = argparse.ArgumentParser()\n spack.cmd.test.setup_parser(parser)\n args = parser.parse_args(\n [\n \"run\",\n \"--cdash-upload-url=https://blahblah/submit.php?project=debugging\",\n \"trivial-smoke-test\",\n ]\n )\n\n spack.cmd.common.arguments.sanitize_reporter_options(args)\n filename = spack.cmd.test.report_filename(args, suite)\n assert filename != \"https://blahblah/submit.php?project=debugging\"\n\n\ndef test_test_output_multiple_specs(\n mock_test_stage, mock_packages, mock_archive, mock_fetch, install_mockery\n):\n \"\"\"Ensure proper reporting for suite with skipped, failing, and passed tests.\"\"\"\n install(\"test-error\", \"simple-standalone-test@0.9\", \"simple-standalone-test@1.0\")\n out = spack_test(\"run\", \"test-error\", \"simple-standalone-test\", fail_on_error=False)\n\n # Note that a spec with passing *and* skipped tests is still considered\n # to have passed at this level. If you want to see the spec-specific\n # part result summaries, you'll have to look at the \"test-out.txt\" files\n # for each spec.\n assert \"1 failed, 2 passed of 3 specs\" in out\n" }, { "path": "lib/spack/spack/test/cmd/undevelop.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport pathlib\n\nimport spack.concretize\nimport spack.environment as ev\nfrom spack.llnl.util.filesystem import working_dir\nfrom spack.main import SpackCommand\n\nundevelop = SpackCommand(\"undevelop\")\nenv = SpackCommand(\"env\")\nconcretize = SpackCommand(\"concretize\")\n\n\ndef test_undevelop(tmp_path: pathlib.Path, mutable_config, mock_packages, mutable_mock_env_path):\n # setup environment\n envdir = tmp_path / \"env\"\n envdir.mkdir()\n with working_dir(str(envdir)):\n with open(\"spack.yaml\", \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n specs:\n - mpich\n\n develop:\n mpich:\n spec: mpich@1.0\n path: /fake/path\n\"\"\"\n )\n\n env(\"create\", \"test\", \"./spack.yaml\")\n with ev.read(\"test\"):\n before = spack.concretize.concretize_one(\"mpich\")\n undevelop(\"mpich\")\n after = spack.concretize.concretize_one(\"mpich\")\n\n # Removing dev spec from environment changes concretization\n assert before.satisfies(\"dev_path=*\")\n assert not after.satisfies(\"dev_path=*\")\n\n\ndef test_undevelop_all(\n tmp_path: pathlib.Path, mutable_config, mock_packages, mutable_mock_env_path\n):\n # setup environment\n envdir = tmp_path / \"env\"\n envdir.mkdir()\n with working_dir(str(envdir)):\n with open(\"spack.yaml\", \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n specs:\n - mpich\n\n develop:\n mpich:\n spec: mpich@1.0\n path: /fake/path\n\"\"\"\n )\n\n env(\"create\", \"test\", \"./spack.yaml\")\n with ev.read(\"test\"):\n before = spack.concretize.concretize_one(\"mpich\")\n undevelop(\"--all\")\n after = spack.concretize.concretize_one(\"mpich\")\n\n # Removing dev spec from environment changes concretization\n assert before.satisfies(\"dev_path=*\")\n assert not after.satisfies(\"dev_path=*\")\n\n\ndef test_undevelop_nonexistent(\n tmp_path: pathlib.Path, mutable_config, mock_packages, mutable_mock_env_path\n):\n # setup environment\n envdir = tmp_path / \"env\"\n envdir.mkdir()\n with working_dir(str(envdir)):\n with open(\"spack.yaml\", \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n specs:\n - mpich\n\n develop:\n mpich:\n spec: mpich@1.0\n path: /fake/path\n\"\"\"\n )\n\n env(\"create\", \"test\", \"./spack.yaml\")\n with ev.read(\"test\") as e:\n concretize()\n before = e.specs_by_hash\n undevelop(\"package-not-in-develop\") # does nothing\n concretize(\"-f\")\n after = e.specs_by_hash\n\n # nothing should have changed\n assert before == after\n" }, { "path": "lib/spack/spack/test/cmd/uninstall.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\nimport pytest\n\nimport spack.cmd.uninstall\nimport spack.environment\nimport spack.llnl.util.tty as tty\nimport spack.store\nfrom spack.enums import InstallRecordStatus\nfrom spack.main import SpackCommand, SpackCommandError\n\nuninstall = SpackCommand(\"uninstall\")\ninstall = SpackCommand(\"install\")\n\n# Unit tests should not be affected by the user's managed environments\npytestmark = pytest.mark.usefixtures(\"mutable_mock_env_path\")\n\n\nclass MockArgs:\n def __init__(self, packages, all=False, force=False, dependents=False):\n self.packages = packages\n self.all = all\n self.force = force\n self.dependents = dependents\n self.yes_to_all = True\n\n\n@pytest.mark.db\ndef test_multiple_matches(mutable_database):\n \"\"\"Test unable to uninstall when multiple matches.\"\"\"\n with pytest.raises(SpackCommandError):\n uninstall(\"-y\", \"mpileaks\")\n\n\n@pytest.mark.db\ndef test_installed_dependents(mutable_database):\n \"\"\"Test can't uninstall when there are installed dependents.\"\"\"\n with pytest.raises(SpackCommandError):\n uninstall(\"-y\", \"libelf\")\n\n\n@pytest.mark.db\ndef test_correct_installed_dependents(mutable_database):\n # Test whether we return the right dependents.\n\n # Take callpath from the database\n callpath = spack.store.STORE.db.query_local(\"callpath\")[0]\n\n # Ensure it still has dependents and dependencies\n dependents = callpath.dependents(deptype=(\"run\", \"link\"))\n dependencies = callpath.dependencies(deptype=(\"run\", \"link\"))\n assert dependents and dependencies\n\n # Uninstall it, so it's missing.\n callpath.package.do_uninstall(force=True)\n\n # Retrieve all dependent hashes\n dependents = spack.cmd.uninstall.installed_dependents(dependencies)\n assert dependents\n\n dependent_hashes = [s.dag_hash() for s in dependents]\n set_dependent_hashes = set(dependent_hashes)\n\n # Assert uniqueness\n assert len(dependent_hashes) == len(set_dependent_hashes)\n\n # Ensure parents of callpath are listed\n assert all(s.dag_hash() in set_dependent_hashes for s in dependents)\n\n # Ensure callpath itself is not, since it was missing.\n assert callpath.dag_hash() not in set_dependent_hashes\n\n\n@pytest.mark.db\ndef test_recursive_uninstall(mutable_database):\n \"\"\"Test recursive uninstall.\"\"\"\n uninstall(\"-y\", \"-a\", \"--dependents\", \"callpath\")\n\n # query specs with multiple configurations\n all_specs = spack.store.STORE.layout.all_specs()\n mpileaks_specs = [s for s in all_specs if s.satisfies(\"mpileaks\")]\n callpath_specs = [s for s in all_specs if s.satisfies(\"callpath\")]\n mpi_specs = [s for s in all_specs if s.satisfies(\"mpi\")]\n\n assert len(mpileaks_specs) == 0\n assert len(callpath_specs) == 0\n assert len(mpi_specs) == 3\n\n\n@pytest.mark.db\n@pytest.mark.regression(\"3690\")\n@pytest.mark.parametrize(\"constraint,expected_number_of_specs\", [(\"dyninst\", 10), (\"libelf\", 8)])\ndef test_uninstall_spec_with_multiple_roots(\n constraint, expected_number_of_specs, mutable_database\n):\n uninstall(\"-y\", \"-a\", \"--dependents\", constraint)\n all_specs = spack.store.STORE.layout.all_specs()\n assert len(all_specs) == expected_number_of_specs\n\n\n@pytest.mark.db\n@pytest.mark.parametrize(\"constraint,expected_number_of_specs\", [(\"dyninst\", 16), (\"libelf\", 16)])\ndef test_force_uninstall_spec_with_ref_count_not_zero(\n constraint, expected_number_of_specs, mutable_database\n):\n uninstall(\"-f\", \"-y\", constraint)\n all_specs = spack.store.STORE.layout.all_specs()\n assert len(all_specs) == expected_number_of_specs\n\n\n@pytest.mark.db\ndef test_force_uninstall_and_reinstall_by_hash(mutable_database):\n \"\"\"Test forced uninstall and reinstall of old specs.\"\"\"\n # this is the spec to be removed\n callpath_spec = spack.store.STORE.db.query_one(\"callpath ^mpich\")\n dag_hash = callpath_spec.dag_hash()\n\n # ensure can look up by hash and that it's a dependent of mpileaks\n def validate_callpath_spec(installed):\n assert installed is True or installed is False\n\n specs = spack.store.STORE.db.get_by_hash(dag_hash, installed=installed)\n assert len(specs) == 1 and specs[0] == callpath_spec\n\n specs = spack.store.STORE.db.get_by_hash(dag_hash[:7], installed=installed)\n assert len(specs) == 1 and specs[0] == callpath_spec\n\n specs = spack.store.STORE.db.get_by_hash(dag_hash, installed=InstallRecordStatus.ANY)\n assert len(specs) == 1 and specs[0] == callpath_spec\n\n specs = spack.store.STORE.db.get_by_hash(dag_hash[:7], installed=InstallRecordStatus.ANY)\n assert len(specs) == 1 and specs[0] == callpath_spec\n\n specs = spack.store.STORE.db.get_by_hash(dag_hash, installed=not installed)\n assert specs is None\n\n specs = spack.store.STORE.db.get_by_hash(dag_hash[:7], installed=not installed)\n assert specs is None\n\n mpileaks_spec = spack.store.STORE.db.query_one(\"mpileaks ^mpich\")\n assert callpath_spec in mpileaks_spec\n\n spec = spack.store.STORE.db.query_one(\"callpath ^mpich\", installed=installed)\n assert spec == callpath_spec\n\n spec = spack.store.STORE.db.query_one(\"callpath ^mpich\", installed=InstallRecordStatus.ANY)\n assert spec == callpath_spec\n\n spec = spack.store.STORE.db.query_one(\"callpath ^mpich\", installed=not installed)\n assert spec is None\n\n validate_callpath_spec(True)\n\n uninstall(\"-y\", \"-f\", \"callpath ^mpich\")\n\n # ensure that you can still look up by hash and see deps, EVEN though\n # the callpath spec is missing.\n validate_callpath_spec(False)\n\n # BUT, make sure that the removed callpath spec is not in queries\n def db_specs():\n all_specs = spack.store.STORE.layout.all_specs()\n return (\n all_specs,\n [s for s in all_specs if s.satisfies(\"mpileaks\")],\n [s for s in all_specs if s.satisfies(\"callpath\")],\n [s for s in all_specs if s.satisfies(\"mpi\")],\n )\n\n all_specs, mpileaks_specs, callpath_specs, mpi_specs = db_specs()\n total_specs = len(all_specs)\n assert total_specs == 16\n assert len(mpileaks_specs) == 3\n assert len(callpath_specs) == 2\n assert len(mpi_specs) == 3\n\n # Now, REINSTALL the spec and make sure everything still holds\n install(\"--fake\", \"/%s\" % dag_hash[:7])\n\n validate_callpath_spec(True)\n\n all_specs, mpileaks_specs, callpath_specs, mpi_specs = db_specs()\n assert len(all_specs) == total_specs + 1 # back to total_specs+1\n assert len(mpileaks_specs) == 3\n assert len(callpath_specs) == 3 # back to 3\n assert len(mpi_specs) == 3\n\n\n@pytest.mark.db\n@pytest.mark.regression(\"15773\")\ndef test_in_memory_consistency_when_uninstalling(mutable_database, monkeypatch):\n \"\"\"Test that uninstalling doesn't raise warnings\"\"\"\n\n def _warn(*args, **kwargs):\n raise RuntimeError(\"a warning was triggered!\")\n\n monkeypatch.setattr(tty, \"warn\", _warn)\n # Now try to uninstall and check this doesn't trigger warnings\n uninstall(\"-y\", \"-a\")\n\n\n# Note: I want to use https://docs.pytest.org/en/7.1.x/how-to/skipping.html#skip-all-test-functions-of-a-class-or-module\n# the style formatter insists on separating these two lines.\nclass TestUninstallFromEnv:\n \"\"\"Tests an installation with two environments e1 and e2, which each have\n shared package installations:\n\n e1 has diamond-link-left -> diamond-link-bottom\n\n e2 has diamond-link-right -> diamond-link-bottom\n \"\"\"\n\n env = SpackCommand(\"env\")\n add = SpackCommand(\"add\")\n concretize = SpackCommand(\"concretize\")\n find = SpackCommand(\"find\")\n\n @pytest.fixture(scope=\"function\")\n def environment_setup(self, mock_packages, mutable_database, install_mockery):\n TestUninstallFromEnv.env(\"create\", \"e1\")\n e1 = spack.environment.read(\"e1\")\n with e1:\n TestUninstallFromEnv.add(\"diamond-link-left\")\n TestUninstallFromEnv.add(\"diamond-link-bottom\")\n TestUninstallFromEnv.concretize()\n install(\"--fake\")\n\n TestUninstallFromEnv.env(\"create\", \"e2\")\n e2 = spack.environment.read(\"e2\")\n with e2:\n TestUninstallFromEnv.add(\"diamond-link-right\")\n TestUninstallFromEnv.add(\"diamond-link-bottom\")\n TestUninstallFromEnv.concretize()\n install(\"--fake\")\n yield \"environment_setup\"\n TestUninstallFromEnv.env(\"rm\", \"e1\", \"-y\")\n TestUninstallFromEnv.env(\"rm\", \"e2\", \"-y\")\n\n def test_basic_env_sanity(self, environment_setup):\n for env_name in [\"e1\", \"e2\"]:\n e = spack.environment.read(env_name)\n with e:\n for _, concretized_spec in e.concretized_specs():\n assert concretized_spec.installed\n\n def test_uninstall_force_dependency_shared_between_envs(self, environment_setup):\n \"\"\"If you \"spack uninstall -f --dependents diamond-link-bottom\" from\n e1, then all packages should be uninstalled (but not removed) from\n both e1 and e2.\n \"\"\"\n e1 = spack.environment.read(\"e1\")\n with e1:\n uninstall(\"-f\", \"-y\", \"--dependents\", \"diamond-link-bottom\")\n\n # The specs should still be in the environment, since\n # --remove was not specified\n assert set(root.name for (root, _) in e1.concretized_specs()) == set(\n [\"diamond-link-left\", \"diamond-link-bottom\"]\n )\n\n for _, concretized_spec in e1.concretized_specs():\n assert not concretized_spec.installed\n\n # Everything in e2 depended on diamond-link-bottom, so should also\n # have been uninstalled. The roots should be unchanged though.\n e2 = spack.environment.read(\"e2\")\n with e2:\n assert set(root.name for (root, _) in e2.concretized_specs()) == set(\n [\"diamond-link-right\", \"diamond-link-bottom\"]\n )\n for _, concretized_spec in e2.concretized_specs():\n assert not concretized_spec.installed\n\n def test_uninstall_remove_dependency_shared_between_envs(self, environment_setup):\n \"\"\"If you \"spack uninstall --dependents --remove diamond-link-bottom\" from\n e1, then all packages are removed from e1 (it is now empty);\n diamond-link-left is also uninstalled (since only e1 needs it) but\n diamond-link-bottom is not uninstalled (since e2 needs it).\n \"\"\"\n e1 = spack.environment.read(\"e1\")\n with e1:\n dtdiamondleft = next(\n concrete\n for (_, concrete) in e1.concretized_specs()\n if concrete.name == \"diamond-link-left\"\n )\n output = uninstall(\"-y\", \"--dependents\", \"--remove\", \"diamond-link-bottom\")\n assert \"The following specs will be removed but not uninstalled\" in output\n assert not list(e1.roots())\n assert not dtdiamondleft.installed\n\n # Since -f was not specified, all specs in e2 should still be installed\n # (and e2 should be unchanged)\n e2 = spack.environment.read(\"e2\")\n with e2:\n assert set(root.name for (root, _) in e2.concretized_specs()) == set(\n [\"diamond-link-right\", \"diamond-link-bottom\"]\n )\n for _, concretized_spec in e2.concretized_specs():\n assert concretized_spec.installed\n\n def test_uninstall_dependency_shared_between_envs_fail(self, environment_setup):\n \"\"\"If you \"spack uninstall --dependents diamond-link-bottom\" from\n e1 (without --remove or -f), then this should fail (this is needed by\n e2).\n \"\"\"\n e1 = spack.environment.read(\"e1\")\n with e1:\n output = uninstall(\"-y\", \"--dependents\", \"diamond-link-bottom\", fail_on_error=False)\n assert \"There are still dependents.\" in output\n assert \"use `spack env remove`\" in output\n\n # The environment should be unchanged and nothing should have been\n # uninstalled\n assert set(root.name for (root, _) in e1.concretized_specs()) == set(\n [\"diamond-link-left\", \"diamond-link-bottom\"]\n )\n for _, concretized_spec in e1.concretized_specs():\n assert concretized_spec.installed\n\n def test_uninstall_force_and_remove_dependency_shared_between_envs(self, environment_setup):\n \"\"\"If you \"spack uninstall -f --dependents --remove diamond-link-bottom\" from\n e1, then all packages should be uninstalled and removed from e1.\n All packages will also be uninstalled from e2, but the roots will\n remain unchanged.\n \"\"\"\n e1 = spack.environment.read(\"e1\")\n with e1:\n dtdiamondleft = next(\n concrete\n for (_, concrete) in e1.concretized_specs()\n if concrete.name == \"diamond-link-left\"\n )\n uninstall(\"-f\", \"-y\", \"--dependents\", \"--remove\", \"diamond-link-bottom\")\n assert not list(e1.roots())\n assert not dtdiamondleft.installed\n\n e2 = spack.environment.read(\"e2\")\n with e2:\n assert set(root.name for (root, _) in e2.concretized_specs()) == set(\n [\"diamond-link-right\", \"diamond-link-bottom\"]\n )\n for _, concretized_spec in e2.concretized_specs():\n assert not concretized_spec.installed\n\n def test_uninstall_keep_dependents_dependency_shared_between_envs(self, environment_setup):\n \"\"\"If you \"spack uninstall -f --remove diamond-link-bottom\" from\n e1, then diamond-link-bottom should be uninstalled, which leaves\n \"dangling\" references in both environments, since\n diamond-link-left and diamond-link-right both need it.\n \"\"\"\n e1 = spack.environment.read(\"e1\")\n with e1:\n dtdiamondleft = next(\n concrete\n for (_, concrete) in e1.concretized_specs()\n if concrete.name == \"diamond-link-left\"\n )\n uninstall(\"-f\", \"-y\", \"--remove\", \"diamond-link-bottom\")\n # diamond-link-bottom was removed from the list of roots (note that\n # it would still be installed since diamond-link-left depends on it)\n assert set(x.name for x in e1.roots()) == set([\"diamond-link-left\"])\n assert dtdiamondleft.installed\n\n e2 = spack.environment.read(\"e2\")\n with e2:\n assert set(root.name for (root, _) in e2.concretized_specs()) == set(\n [\"diamond-link-right\", \"diamond-link-bottom\"]\n )\n dtdiamondright = next(\n concrete\n for (_, concrete) in e2.concretized_specs()\n if concrete.name == \"diamond-link-right\"\n )\n assert dtdiamondright.installed\n dtdiamondbottom = next(\n concrete\n for (_, concrete) in e2.concretized_specs()\n if concrete.name == \"diamond-link-bottom\"\n )\n assert not dtdiamondbottom.installed\n" }, { "path": "lib/spack/spack/test/cmd/unit_test.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\n\nimport pytest\n\nfrom spack.main import SpackCommand\n\npytest.skip(\"Recursive pytest is brittle\", allow_module_level=True)\n\nspack_test = SpackCommand(\"unit-test\")\ncmd_test_py = os.path.join(\"lib\", \"spack\", \"spack\", \"test\", \"cmd\", \"unit_test.py\")\n\n\ndef test_list():\n output = spack_test(\"--list\")\n assert \"unit_test.py\" in output\n assert \"spec_semantics.py\" in output\n assert \"test_list\" not in output\n\n\ndef test_list_with_pytest_arg():\n output = spack_test(\"--list\", cmd_test_py)\n assert cmd_test_py in output.strip()\n\n\ndef test_list_with_keywords():\n # Here we removed querying with a \"/\" to separate directories\n # since the behavior is inconsistent across different pytest\n # versions, see https://stackoverflow.com/a/48814787/771663\n output = spack_test(\"--list\", \"-k\", \"unit_test.py\")\n assert cmd_test_py in output.strip()\n\n\ndef test_list_long():\n output = spack_test(\"--list-long\")\n assert \"unit_test.py::\\n\" in output\n assert \"test_list\" in output\n assert \"test_list_with_pytest_arg\" in output\n assert \"test_list_with_keywords\" in output\n assert \"test_list_long\" in output\n assert \"test_list_long_with_pytest_arg\" in output\n assert \"test_list_names\" in output\n assert \"test_list_names_with_pytest_arg\" in output\n\n assert \"spec_dag.py::\\n\" in output\n assert \"test_installed_deps\" in output\n assert \"test_test_deptype\" in output\n\n\ndef test_list_long_with_pytest_arg():\n output = spack_test(\"--list-long\", cmd_test_py)\n\n assert \"unit_test.py::\\n\" in output\n assert \"test_list\" in output\n assert \"test_list_with_pytest_arg\" in output\n assert \"test_list_with_keywords\" in output\n assert \"test_list_long\" in output\n assert \"test_list_long_with_pytest_arg\" in output\n assert \"test_list_names\" in output\n assert \"test_list_names_with_pytest_arg\" in output\n\n assert \"spec_dag.py::\\n\" not in output\n assert \"test_installed_deps\" not in output\n assert \"test_test_deptype\" not in output\n\n\ndef test_list_names():\n output = spack_test(\"--list-names\")\n assert \"unit_test.py::test_list\\n\" in output\n assert \"unit_test.py::test_list_with_pytest_arg\\n\" in output\n assert \"unit_test.py::test_list_with_keywords\\n\" in output\n assert \"unit_test.py::test_list_long\\n\" in output\n assert \"unit_test.py::test_list_long_with_pytest_arg\\n\" in output\n assert \"unit_test.py::test_list_names\\n\" in output\n assert \"unit_test.py::test_list_names_with_pytest_arg\\n\" in output\n\n assert \"spec_dag.py::test_installed_deps\\n\" in output\n assert \"spec_dag.py::test_test_deptype\\n\" in output\n\n\ndef test_list_names_with_pytest_arg():\n output = spack_test(\"--list-names\", cmd_test_py)\n assert \"unit_test.py::test_list\\n\" in output\n assert \"unit_test.py::test_list_with_pytest_arg\\n\" in output\n assert \"unit_test.py::test_list_with_keywords\\n\" in output\n assert \"unit_test.py::test_list_long\\n\" in output\n assert \"unit_test.py::test_list_long_with_pytest_arg\\n\" in output\n assert \"unit_test.py::test_list_names\\n\" in output\n assert \"unit_test.py::test_list_names_with_pytest_arg\\n\" in output\n\n assert \"spec_dag.py::test_installed_deps\\n\" not in output\n assert \"spec_dag.py::test_test_deptype\\n\" not in output\n\n\ndef test_pytest_help():\n output = spack_test(\"--pytest-help\")\n assert \"-k EXPRESSION\" in output\n assert \"pytest-warnings:\" in output\n assert \"--collect-only\" in output\n" }, { "path": "lib/spack/spack/test/cmd/url.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport re\n\nimport pytest\n\nimport spack.repo\nfrom spack.cmd.url import name_parsed_correctly, url_summary, version_parsed_correctly\nfrom spack.main import SpackCommand\nfrom spack.url import UndetectableVersionError\n\nurl = SpackCommand(\"url\")\n\n\nclass MyPackage:\n def __init__(self, name, versions):\n self.name = name\n self.versions = versions\n\n\ndef test_name_parsed_correctly():\n # Expected True\n assert name_parsed_correctly(MyPackage(\"netcdf\", []), \"netcdf\")\n assert name_parsed_correctly(MyPackage(\"r-devtools\", []), \"devtools\")\n assert name_parsed_correctly(MyPackage(\"py-numpy\", []), \"numpy\")\n assert name_parsed_correctly(MyPackage(\"octave-splines\", []), \"splines\")\n assert name_parsed_correctly(MyPackage(\"th-data\", []), \"TH.data\")\n assert name_parsed_correctly(MyPackage(\"imagemagick\", []), \"ImageMagick\")\n\n # Expected False\n assert not name_parsed_correctly(MyPackage(\"\", []), \"hdf5\")\n assert not name_parsed_correctly(MyPackage(\"hdf5\", []), \"\")\n assert not name_parsed_correctly(MyPackage(\"yaml-cpp\", []), \"yamlcpp\")\n assert not name_parsed_correctly(MyPackage(\"yamlcpp\", []), \"yaml-cpp\")\n assert not name_parsed_correctly(MyPackage(\"r-py-parser\", []), \"parser\")\n assert not name_parsed_correctly(MyPackage(\"oce\", []), \"oce-0.18.0\")\n\n\ndef test_version_parsed_correctly():\n # Expected True\n assert version_parsed_correctly(MyPackage(\"\", [\"1.2.3\"]), \"1.2.3\")\n assert version_parsed_correctly(MyPackage(\"\", [\"5.4a\", \"5.4b\"]), \"5.4a\")\n assert version_parsed_correctly(MyPackage(\"\", [\"5.4a\", \"5.4b\"]), \"5.4b\")\n assert version_parsed_correctly(MyPackage(\"\", [\"1.63.0\"]), \"1_63_0\")\n assert version_parsed_correctly(MyPackage(\"\", [\"0.94h\"]), \"094h\")\n\n # Expected False\n assert not version_parsed_correctly(MyPackage(\"\", []), \"1.2.3\")\n assert not version_parsed_correctly(MyPackage(\"\", [\"1.2.3\"]), \"\")\n assert not version_parsed_correctly(MyPackage(\"\", [\"1.2.3\"]), \"1.2.4\")\n assert not version_parsed_correctly(MyPackage(\"\", [\"3.4a\"]), \"3.4\")\n assert not version_parsed_correctly(MyPackage(\"\", [\"3.4\"]), \"3.4b\")\n assert not version_parsed_correctly(MyPackage(\"\", [\"0.18.0\"]), \"oce-0.18.0\")\n\n\ndef test_url_parse():\n url(\"parse\", \"http://zlib.net/fossils/zlib-1.2.10.tar.gz\")\n\n\ndef test_url_with_no_version_fails():\n # No version in URL\n with pytest.raises(UndetectableVersionError):\n url(\"parse\", \"http://www.netlib.org/voronoi/triangle.zip\")\n\n\ndef test_url_list(mock_packages):\n out = url(\"list\")\n total_urls = len(out.split(\"\\n\"))\n\n # The following two options should not change the number of URLs printed.\n out = url(\"list\", \"--color\", \"--extrapolation\")\n colored_urls = len(out.split(\"\\n\"))\n assert colored_urls == total_urls\n\n # The following options should print fewer URLs than the default.\n # If they print the same number of URLs, something is horribly broken.\n # If they say we missed 0 URLs, something is probably broken too.\n out = url(\"list\", \"--incorrect-name\")\n incorrect_name_urls = len(out.split(\"\\n\"))\n assert 0 < incorrect_name_urls < total_urls\n\n out = url(\"list\", \"--incorrect-version\")\n incorrect_version_urls = len(out.split(\"\\n\"))\n assert 0 < incorrect_version_urls < total_urls\n\n out = url(\"list\", \"--correct-name\")\n correct_name_urls = len(out.split(\"\\n\"))\n assert 0 < correct_name_urls < total_urls\n\n out = url(\"list\", \"--correct-version\")\n correct_version_urls = len(out.split(\"\\n\"))\n assert 0 < correct_version_urls < total_urls\n\n\ndef test_url_summary(mock_packages):\n \"\"\"Test the URL summary command.\"\"\"\n # test url_summary, the internal function that does the work\n (total_urls, correct_names, correct_versions, name_count_dict, version_count_dict) = (\n url_summary(None)\n )\n\n assert 0 < correct_names <= sum(name_count_dict.values()) <= total_urls\n assert 0 < correct_versions <= sum(version_count_dict.values()) <= total_urls\n\n # make sure it agrees with the actual command.\n out = url(\"summary\")\n out_total_urls = int(re.search(r\"Total URLs found:\\s*(\\d+)\", out).group(1))\n assert out_total_urls == total_urls\n\n out_correct_names = int(re.search(r\"Names correctly parsed:\\s*(\\d+)\", out).group(1))\n assert out_correct_names == correct_names\n\n out_correct_versions = int(re.search(r\"Versions correctly parsed:\\s*(\\d+)\", out).group(1))\n assert out_correct_versions == correct_versions\n\n\ndef test_url_stats(mock_packages):\n output = url(\"stats\")\n npkgs = \"%d packages\" % len(spack.repo.all_package_names())\n assert npkgs in output\n assert \"url\" in output\n assert \"git\" in output\n assert \"schemes\" in output\n assert \"versions\" in output\n assert \"resources\" in output\n\n output = url(\"stats\", \"--show-issues\")\n npkgs = \"%d packages\" % len(spack.repo.all_package_names())\n assert npkgs in output\n assert \"url\" in output\n assert \"git\" in output\n assert \"schemes\" in output\n assert \"versions\" in output\n assert \"resources\" in output\n\n assert \"Package URLs with md5 hashes\" in output\n assert \"needs-relocation\" in output\n assert \"https://cmake.org/files/v3.4/cmake-0.0.0.tar.gz\" in output\n\n assert \"Package URLs with http urls\" in output\n assert \"zmpi\" in output\n assert \"http://www.spack-fake-zmpi.org/downloads/zmpi-1.0.tar.gz\" in output\n" }, { "path": "lib/spack/spack/test/cmd/verify.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Tests for the `spack verify` command\"\"\"\n\nimport os\nimport pathlib\nimport platform\n\nimport pytest\n\nimport spack.cmd.verify\nimport spack.concretize\nimport spack.installer\nimport spack.llnl.util.filesystem as fs\nimport spack.store\nimport spack.util.executable\nimport spack.util.spack_json as sjson\nimport spack.verify\nfrom spack.main import SpackCommand, SpackCommandError\nfrom spack.spec import Spec\n\nverify = SpackCommand(\"verify\")\ninstall = SpackCommand(\"install\")\n\n\ndef skip_unless_linux(f):\n return pytest.mark.skipif(\n str(platform.system()) != \"Linux\", reason=\"only tested on linux for now\"\n )(f)\n\n\ndef test_single_file_verify_cmd(tmp_path: pathlib.Path):\n # Test the verify command interface to verifying a single file.\n filedir = tmp_path / \"a\" / \"b\" / \"c\" / \"d\"\n filepath = filedir / \"file\"\n metadir = tmp_path / spack.store.STORE.layout.metadata_dir\n\n fs.mkdirp(str(filedir))\n fs.mkdirp(str(metadir))\n\n with open(str(filepath), \"w\", encoding=\"utf-8\") as f:\n f.write(\"I'm a file\")\n\n data = spack.verify.create_manifest_entry(str(filepath))\n\n manifest_file = metadir / spack.store.STORE.layout.manifest_file_name\n\n with open(str(manifest_file), \"w\", encoding=\"utf-8\") as f:\n sjson.dump({str(filepath): data}, f)\n\n results = verify(\"manifest\", \"-f\", str(filepath), fail_on_error=False)\n assert not results\n\n os.utime(str(filepath), (0, 0))\n with open(str(filepath), \"w\", encoding=\"utf-8\") as f:\n f.write(\"I changed.\")\n\n results = verify(\"manifest\", \"-f\", str(filepath), fail_on_error=False)\n\n expected = [\"hash\"]\n mtime = os.stat(str(filepath)).st_mtime\n if mtime != data[\"time\"]:\n expected.append(\"mtime\")\n\n assert results\n assert str(filepath) in results\n assert all(x in results for x in expected)\n\n results = verify(\"manifest\", \"-fj\", str(filepath), fail_on_error=False)\n res = sjson.load(results)\n assert len(res) == 1\n errors = res.pop(str(filepath))\n assert sorted(errors) == sorted(expected)\n\n\ndef test_single_spec_verify_cmd(mock_packages, mock_archive, mock_fetch, install_mockery):\n # Test the verify command interface to verify a single spec\n install(\"--fake\", \"libelf\")\n s = spack.concretize.concretize_one(\"libelf\")\n prefix = s.prefix\n hash = s.dag_hash()\n\n results = verify(\"manifest\", \"/%s\" % hash, fail_on_error=False)\n assert not results\n\n new_file = os.path.join(prefix, \"new_file_for_verify_test\")\n with open(new_file, \"w\", encoding=\"utf-8\") as f:\n f.write(\"New file\")\n\n results = verify(\"manifest\", \"/%s\" % hash, fail_on_error=False)\n assert new_file in results\n assert \"added\" in results\n\n results = verify(\"manifest\", \"-j\", \"/%s\" % hash, fail_on_error=False)\n res = sjson.load(results)\n assert len(res) == 1\n assert res[new_file] == [\"added\"]\n\n\n@pytest.mark.requires_executables(\"gcc\")\n@skip_unless_linux\ndef test_libraries(tmp_path: pathlib.Path, install_mockery, mock_fetch):\n gcc = spack.util.executable.which(\"gcc\", required=True)\n s = spack.concretize.concretize_one(\"libelf\")\n spack.installer.PackageInstaller([s.package], fake=True).install()\n\n # There are no ELF files so the verification should pass\n verify(\"libraries\", f\"/{s.dag_hash()}\")\n\n # Now put main_with_rpath linking to libf.so inside the prefix and verify again. This should\n # work because libf.so can be located in the rpath.\n (tmp_path / \"f.c\").write_text(\"void f(void){return;}\")\n (tmp_path / \"main.c\").write_text(\"void f(void); int main(void){f();return 0;}\")\n\n gcc(\"-shared\", \"-fPIC\", \"-o\", str(tmp_path / \"libf.so\"), str(tmp_path / \"f.c\"))\n gcc(\n \"-o\",\n str(s.prefix.bin.main_with_rpath),\n str(tmp_path / \"main.c\"),\n \"-L\",\n str(tmp_path),\n f\"-Wl,-rpath,{tmp_path}\",\n \"-lf\",\n )\n verify(\"libraries\", f\"/{s.dag_hash()}\")\n\n # Now put main_without_rpath linking to libf.so inside the prefix and verify again. This should\n # fail because libf.so cannot be located in the rpath.\n gcc(\n \"-o\",\n str(s.prefix.bin.main_without_rpath),\n str(tmp_path / \"main.c\"),\n \"-L\",\n str(tmp_path),\n \"-lf\",\n )\n\n with pytest.raises(SpackCommandError):\n verify(\"libraries\", f\"/{s.dag_hash()}\")\n\n # Check the error message\n msg = spack.cmd.verify._verify_libraries(s, [])\n assert msg is not None and \"libf.so => not found\" in msg\n\n # And check that we can make it pass by ignoring it.\n assert spack.cmd.verify._verify_libraries(s, [\"libf.so\"]) is None\n\n\ndef test_verify_versions(mock_packages):\n missing = \"thisisnotapackage\"\n unknown = \"deprecated-versions@=thisisnotaversion\"\n deprecated = \"deprecated-versions@=1.1.0\"\n good = \"deprecated-versions@=1.0.0\"\n\n strs = (missing, unknown, deprecated, good)\n\n specs = [Spec(c) for c in strs] + [Spec(f\"deprecated-client@=1.1.0^{c}\") for c in strs]\n for spec in specs:\n spec._mark_concrete()\n\n msg_lines = spack.cmd.verify._verify_version(specs)\n assert \"3 installed packages have unknown/deprecated\" in msg_lines[0]\n assert \"thisisnotapackage\" in msg_lines[1]\n assert \"Cannot load package\" in msg_lines[1]\n assert \"version thisisnotaversion unknown to Spack\" in msg_lines[2]\n assert \"deprecated version 1.1.0\" in msg_lines[3]\n" }, { "path": "lib/spack/spack/test/cmd/versions.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport pytest\n\nimport spack.url\nfrom spack.main import SpackCommand\nfrom spack.version import Version\n\nversions = SpackCommand(\"versions\")\n\n\npytestmark = [pytest.mark.usefixtures(\"mock_packages\")]\n\n\ndef _mock_find_versions_of_archive(*args, **kwargs):\n return {\n Version(\"1.3.1\"): \"https://zlib.net/zlib-1.3.1.tar.gz\",\n Version(\"1.3\"): \"https://zlib.net/zlib-1.3.tar.gz\",\n Version(\"1.2.13\"): \"https://zlib.net/zlib-1.2.13.tar.gz\",\n }\n\n\ndef test_safe_versions():\n \"\"\"Only test the safe versions of a package.\"\"\"\n assert versions(\"--safe\", \"zlib\") == \" 1.2.11\\n 1.2.8\\n 1.2.3\\n\"\n\n\ndef test_remote_versions(monkeypatch):\n \"\"\"Test a package for which remote versions should be available.\"\"\"\n monkeypatch.setattr(spack.url, \"find_versions_of_archive\", _mock_find_versions_of_archive)\n assert versions(\"zlib\") == \" 1.2.11\\n 1.2.8\\n 1.2.3\\n 1.3.1\\n 1.3\\n 1.2.13\\n\"\n\n\ndef test_remote_versions_only(monkeypatch):\n \"\"\"Test a package for which remote versions should be available.\"\"\"\n monkeypatch.setattr(spack.url, \"find_versions_of_archive\", _mock_find_versions_of_archive)\n assert versions(\"--remote\", \"zlib\") == \" 1.3.1\\n 1.3\\n 1.2.13\\n\"\n\n\ndef test_new_versions_only(monkeypatch):\n \"\"\"Test a package for which new versions should be available.\"\"\"\n from spack_repo.builtin_mock.packages.brillig.package import Brillig # type: ignore[import]\n\n def mock_fetch_remote_versions(*args, **kwargs):\n mock_remote_versions = {\n # new version, we expect this to be in output:\n Version(\"99.99.99\"): {},\n # some packages use '3.2' equivalently to '3.2.0'\n # thus '3.2.1' is considered to be a new version\n # and expected in the output also\n Version(\"3.2.1\"): {}, # new version, we expect this to be in output\n Version(\"3.2\"): {},\n Version(\"1.0.0\"): {},\n }\n return mock_remote_versions\n\n mock_versions = {\n # already checksummed versions:\n Version(\"3.2\"): {},\n Version(\"1.0.0\"): {},\n }\n monkeypatch.setattr(Brillig, \"versions\", mock_versions)\n monkeypatch.setattr(Brillig, \"fetch_remote_versions\", mock_fetch_remote_versions)\n v = versions(\"--new\", \"brillig\")\n assert v.strip(\" \\n\\t\") == \"99.99.99\\n 3.2.1\"\n\n\ndef test_no_unchecksummed_versions(monkeypatch):\n \"\"\"Test a package for which no unchecksummed versions are available.\"\"\"\n\n def mock_find_versions_of_archive(*args, **kwargs):\n \"\"\"Mock find_versions_of_archive to avoid network calls.\"\"\"\n # Return some fake versions for bzip2\n return {\n Version(\"1.0.8\"): \"https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz\",\n Version(\"1.0.7\"): \"https://sourceware.org/pub/bzip2/bzip2-1.0.7.tar.gz\",\n }\n\n monkeypatch.setattr(spack.url, \"find_versions_of_archive\", mock_find_versions_of_archive)\n\n versions(\"bzip2\")\n\n\ndef test_versions_no_url():\n \"\"\"Test a package with versions but without a ``url`` attribute.\"\"\"\n assert versions(\"attributes-foo-app\") == \" 1.0\\n\"\n\n\ndef test_no_versions_no_url():\n \"\"\"Test a package without versions or a ``url`` attribute.\"\"\"\n assert versions(\"no-url-or-version\") == \"\"\n" }, { "path": "lib/spack/spack/test/cmd/view.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport pathlib\nimport sys\n\nimport pytest\n\nimport spack.concretize\nimport spack.main\nimport spack.util.spack_yaml as s_yaml\nfrom spack.installer import PackageInstaller\nfrom spack.llnl.util.filesystem import _windows_can_symlink\nfrom spack.main import SpackCommand\n\nextensions = SpackCommand(\"extensions\")\ninstall = SpackCommand(\"install\")\nview = SpackCommand(\"view\")\n\nif sys.platform == \"win32\":\n if not _windows_can_symlink():\n pytest.skip(\n \"Windows must be able to create symlinks to run tests.\", allow_module_level=True\n )\n # TODO: Skipping hardlink command testing on windows until robust checks can be added.\n # See https://github.com/spack/spack/pull/46335#discussion_r1757411915\n commands = [\"symlink\", \"add\", \"copy\", \"relocate\"]\nelse:\n commands = [\"hardlink\", \"symlink\", \"hard\", \"add\", \"copy\", \"relocate\"]\n\n\ndef create_projection_file(tmp_path: pathlib.Path, projection):\n if \"projections\" not in projection:\n projection = {\"projections\": projection}\n projection_file = tmp_path / \"projection\" / \"projection.yaml\"\n projection_file.parent.mkdir(parents=True, exist_ok=True)\n projection_file.write_text(s_yaml.dump(projection))\n return projection_file\n\n\n@pytest.mark.parametrize(\"cmd\", commands)\ndef test_view_link_type(\n tmp_path: pathlib.Path, mock_packages, mock_archive, mock_fetch, install_mockery, cmd\n):\n install(\"--fake\", \"libdwarf\")\n view_dir = tmp_path / f\"view_{cmd}\"\n view(cmd, str(view_dir), \"libdwarf\")\n package_bin = view_dir / \"bin\" / \"libdwarf\"\n assert package_bin.exists()\n\n # Check that we use symlinks for and only for the appropriate subcommands\n is_link_cmd = cmd in (\"symlink\", \"add\")\n assert os.path.islink(str(package_bin)) == is_link_cmd\n\n\n@pytest.mark.parametrize(\"add_cmd\", commands)\ndef test_view_link_type_remove(\n tmp_path: pathlib.Path, mock_packages, mock_archive, mock_fetch, install_mockery, add_cmd\n):\n install(\"needs-relocation\")\n viewpath = str(tmp_path / \"view_{0}\".format(add_cmd))\n (tmp_path / \"view_{0}\".format(add_cmd)).mkdir()\n view(add_cmd, viewpath, \"needs-relocation\")\n bindir = os.path.join(viewpath, \"bin\")\n assert os.path.exists(bindir)\n\n view(\"remove\", viewpath, \"needs-relocation\")\n assert not os.path.exists(bindir)\n\n\n@pytest.mark.parametrize(\"cmd\", commands)\ndef test_view_projections(\n tmp_path: pathlib.Path, mock_packages, mock_archive, mock_fetch, install_mockery, cmd\n):\n install(\"--fake\", \"libdwarf@20130207\")\n view_dir = tmp_path / f\"view_{cmd}\"\n\n view_projection = {\"projections\": {\"all\": \"{name}-{version}\"}}\n projection_file = create_projection_file(tmp_path, view_projection)\n view(cmd, str(view_dir), f\"--projection-file={projection_file}\", \"libdwarf\")\n\n package_bin = view_dir / \"libdwarf-20130207\" / \"bin\" / \"libdwarf\"\n assert package_bin.exists()\n\n # Check that we use symlinks for and only for the appropriate subcommands\n is_symlink_cmd = cmd in (\"symlink\", \"add\")\n assert package_bin.is_symlink() == is_symlink_cmd\n\n\ndef test_view_multiple_projections(\n tmp_path: pathlib.Path, mock_packages, mock_archive, mock_fetch, install_mockery\n):\n install(\"--fake\", \"libdwarf@20130207\")\n install(\"--fake\", \"extendee@1.0\")\n view_dir = tmp_path / \"view\"\n\n view_projection = s_yaml.syaml_dict(\n [(\"extendee\", \"{name}-{architecture.platform}\"), (\"all\", \"{name}-{version}\")]\n )\n\n projection_file = create_projection_file(tmp_path, view_projection)\n view(\"add\", str(view_dir), f\"--projection-file={projection_file}\", \"libdwarf\", \"extendee\")\n\n libdwarf_prefix = view_dir / \"libdwarf-20130207\" / \"bin\"\n extendee_prefix = view_dir / \"extendee-test\" / \"bin\"\n assert libdwarf_prefix.exists()\n assert extendee_prefix.exists()\n\n\ndef test_view_multiple_projections_all_first(\n tmp_path: pathlib.Path, mock_packages, mock_archive, mock_fetch, install_mockery\n):\n install(\"--fake\", \"libdwarf@20130207\")\n install(\"--fake\", \"extendee@1.0\")\n view_dir = tmp_path / \"view\"\n\n view_projection = s_yaml.syaml_dict(\n [(\"all\", \"{name}-{version}\"), (\"extendee\", \"{name}-{architecture.platform}\")]\n )\n\n projection_file = create_projection_file(tmp_path, view_projection)\n view(\"add\", str(view_dir), f\"--projection-file={projection_file}\", \"libdwarf\", \"extendee\")\n\n libdwarf_prefix = view_dir / \"libdwarf-20130207\" / \"bin\"\n extendee_prefix = view_dir / \"extendee-test\" / \"bin\"\n assert libdwarf_prefix.exists()\n assert extendee_prefix.exists()\n\n\ndef test_view_external(\n tmp_path: pathlib.Path, mock_packages, mock_archive, mock_fetch, install_mockery\n):\n install(\"externaltool\")\n viewpath = str(tmp_path / \"view\")\n (tmp_path / \"view\").mkdir()\n output = view(\"symlink\", viewpath, \"externaltool\")\n assert \"Skipping external package: externaltool\" in output\n\n\ndef test_view_extension(\n tmp_path: pathlib.Path, mock_packages, mock_archive, mock_fetch, install_mockery\n):\n install(\"extendee\")\n install(\"extension1@1.0\")\n install(\"extension1@2.0\")\n install(\"extension2@1.0\")\n viewpath = str(tmp_path / \"view\")\n (tmp_path / \"view\").mkdir()\n view(\"symlink\", viewpath, \"extension1@1.0\")\n all_installed = extensions(\"--show\", \"installed\", \"extendee\")\n assert \"extension1@1.0\" in all_installed\n assert \"extension1@2.0\" in all_installed\n assert \"extension2@1.0\" in all_installed\n assert os.path.exists(os.path.join(viewpath, \"bin\", \"extension1\"))\n\n\ndef test_view_extension_remove(\n tmp_path: pathlib.Path, mock_packages, mock_archive, mock_fetch, install_mockery\n):\n install(\"extendee\")\n install(\"extension1@1.0\")\n viewpath = str(tmp_path / \"view\")\n (tmp_path / \"view\").mkdir()\n view(\"symlink\", viewpath, \"extension1@1.0\")\n view(\"remove\", viewpath, \"extension1@1.0\")\n all_installed = extensions(\"--show\", \"installed\", \"extendee\")\n assert \"extension1@1.0\" in all_installed\n assert not os.path.exists(os.path.join(viewpath, \"bin\", \"extension1\"))\n\n\ndef test_view_extension_conflict(\n tmp_path: pathlib.Path, mock_packages, mock_archive, mock_fetch, install_mockery\n):\n install(\"extendee\")\n install(\"extension1@1.0\")\n install(\"extension1@2.0\")\n viewpath = str(tmp_path / \"view\")\n (tmp_path / \"view\").mkdir()\n view(\"symlink\", viewpath, \"extension1@1.0\")\n output = view(\"symlink\", viewpath, \"extension1@2.0\")\n assert \"Package conflict detected\" in output\n\n\ndef test_view_extension_conflict_ignored(\n tmp_path: pathlib.Path, mock_packages, mock_archive, mock_fetch, install_mockery\n):\n install(\"extendee\")\n install(\"extension1@1.0\")\n install(\"extension1@2.0\")\n viewpath = str(tmp_path / \"view\")\n (tmp_path / \"view\").mkdir()\n view(\"symlink\", viewpath, \"extension1@1.0\")\n view(\"symlink\", viewpath, \"-i\", \"extension1@2.0\")\n with open(os.path.join(viewpath, \"bin\", \"extension1\"), \"r\", encoding=\"utf-8\") as fin:\n assert fin.read() == \"1.0\"\n\n\ndef test_view_fails_with_missing_projections_file(tmp_path: pathlib.Path):\n viewpath = str(tmp_path / \"view\")\n (tmp_path / \"view\").mkdir()\n projection_file = str(tmp_path / \"nonexistent\")\n with pytest.raises(spack.main.SpackCommandError):\n view(\"symlink\", \"--projection-file\", projection_file, viewpath, \"foo\")\n\n\n@pytest.mark.parametrize(\"with_projection\", [False, True])\n@pytest.mark.parametrize(\"cmd\", [\"symlink\", \"copy\"])\ndef test_view_files_not_ignored(\n tmp_path: pathlib.Path,\n mock_packages,\n mock_archive,\n mock_fetch,\n install_mockery,\n cmd,\n with_projection,\n):\n spec = spack.concretize.concretize_one(\"view-not-ignored\")\n pkg = spec.package\n PackageInstaller([pkg], explicit=True).install()\n pkg.assert_installed(spec.prefix)\n\n install(\"view-file\") # Arbitrary package to add noise\n\n viewpath = str(tmp_path / \"view_{0}\".format(cmd))\n (tmp_path / \"view_{0}\".format(cmd)).mkdir()\n\n if with_projection:\n proj = str(tmp_path / \"proj.yaml\")\n with open(proj, \"w\", encoding=\"utf-8\") as f:\n f.write('{\"projections\":{\"all\":\"{name}\"}}')\n prefix_in_view = os.path.join(viewpath, \"view-not-ignored\")\n args = [\"--projection-file\", proj]\n else:\n prefix_in_view = viewpath\n args = []\n\n view(cmd, *(args + [viewpath, \"view-not-ignored\", \"view-file\"]))\n pkg.assert_installed(prefix_in_view)\n\n view(\"remove\", viewpath, \"view-not-ignored\")\n pkg.assert_not_installed(prefix_in_view)\n" }, { "path": "lib/spack/spack/test/cmd_extensions.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport contextlib\nimport os\nimport pathlib\nimport sys\n\nimport pytest\n\nimport spack.cmd\nimport spack.config\nimport spack.extensions\nimport spack.main\n\n\nclass Extension:\n \"\"\"Helper class to simplify the creation of simple command extension\n directory structures with a conventional format for testing.\n \"\"\"\n\n def __init__(self, name, root: pathlib.Path):\n \"\"\"Create a command extension.\n\n Args:\n name (str): The name of the command extension.\n root (path object): The temporary root for the command extension\n (e.g. from tmp_path.mkdir()).\n \"\"\"\n self.name = name\n self.pname = spack.cmd.python_name(name)\n self.root = root\n self.main = self.root / self.pname\n self.main.mkdir(parents=True, exist_ok=True)\n self.cmd = self.main / \"cmd\"\n self.cmd.mkdir(parents=True, exist_ok=True)\n\n def add_command(self, command_name, contents):\n \"\"\"Add a command to this command extension.\n\n Args:\n command_name (str): The name of the command.\n contents (str): the desired contents of the new command module\n file.\"\"\"\n spack.cmd.require_cmd_name(command_name)\n python_name = spack.cmd.python_name(command_name)\n cmd = self.cmd / (python_name + \".py\")\n cmd.write_text(contents)\n\n\n@pytest.fixture(scope=\"function\")\ndef extension_creator(tmp_path: pathlib.Path, config):\n \"\"\"Create a basic extension command directory structure\"\"\"\n\n @contextlib.contextmanager\n def _ce(extension_name=\"testcommand\"):\n root = tmp_path / (\"spack-\" + extension_name)\n root.mkdir()\n extension = Extension(extension_name, root)\n with spack.config.override(\"config:extensions\", [str(extension.root)]):\n yield extension\n\n list_of_modules = list(sys.modules.keys())\n try:\n yield _ce\n finally:\n to_be_deleted = [x for x in sys.modules if x not in list_of_modules]\n for module_name in to_be_deleted:\n del sys.modules[module_name]\n\n\n@pytest.fixture(scope=\"function\")\ndef hello_world_extension(extension_creator):\n \"\"\"Create an extension with a hello-world command.\"\"\"\n with extension_creator() as extension:\n extension.add_command(\n \"hello-world\",\n \"\"\"\ndescription = \"hello world extension command\"\nsection = \"test command\"\nlevel = \"long\"\n\ndef setup_parser(subparser):\n pass\n\n\ndef hello_world(parser, args):\n print('Hello world!')\n\"\"\",\n )\n yield extension\n\n\n@pytest.fixture(scope=\"function\")\ndef hello_world_cmd(hello_world_extension):\n \"\"\"Create and return an invocable \"hello-world\" extension command.\"\"\"\n yield spack.main.SpackCommand(\"hello-world\")\n\n\n@pytest.fixture(scope=\"function\")\ndef hello_world_with_module_in_root(extension_creator):\n \"\"\"Create a \"hello-world\" extension command with additional code in the\n root folder.\n \"\"\"\n\n @contextlib.contextmanager\n def _hwwmir(extension_name=None):\n with (\n extension_creator(extension_name) if extension_name else extension_creator()\n ) as extension:\n # Note that the namespace of the extension is derived from the\n # fixture.\n extension.add_command(\n \"hello\",\n \"\"\"\n# Test an absolute import\nfrom spack.extensions.{ext_pname}.implementation import hello_world\n\n# Test a relative import\nfrom ..implementation import hello_folks\n\ndescription = \"hello world extension command\"\nsection = \"test command\"\nlevel = \"long\"\n\n# Test setting a global variable in setup_parser and retrieving\n# it in the command\nglobal_message = 'foo'\n\ndef setup_parser(subparser):\n sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='subcommand')\n global global_message\n sp.add_parser('world', help='Print Hello world!')\n sp.add_parser('folks', help='Print Hello folks!')\n sp.add_parser('global', help='Print Hello folks!')\n global_message = 'bar'\n\ndef hello(parser, args):\n if args.subcommand == 'world':\n hello_world()\n elif args.subcommand == 'folks':\n hello_folks()\n elif args.subcommand == 'global':\n print(global_message)\n\"\"\".format(ext_pname=extension.pname),\n )\n\n init_file = extension.main / \"__init__.py\"\n init_file.touch()\n implementation = extension.main / \"implementation.py\"\n implementation.write_text(\n \"\"\"\ndef hello_world():\n print('Hello world!')\n\ndef hello_folks():\n print('Hello folks!')\n\"\"\"\n )\n yield spack.main.SpackCommand(\"hello\")\n\n yield _hwwmir\n\n\ndef test_simple_command_extension(hello_world_cmd):\n \"\"\"Basic test of a functioning command.\"\"\"\n output = hello_world_cmd()\n assert \"Hello world!\" in output\n\n\ndef test_multi_extension_search(hello_world_extension, extension_creator):\n \"\"\"Ensure we can find an extension command even if it's not in the first\n place we look.\n \"\"\"\n\n with extension_creator(\"testcommand2\"):\n assert (\"Hello world\") in spack.main.SpackCommand(\"hello-world\")()\n\n\ndef test_duplicate_module_load(hello_world_cmd, capfd):\n \"\"\"Ensure duplicate module load attempts are successful.\n\n The command module will already have been loaded once by the\n hello_world_cmd fixture.\n \"\"\"\n parser = spack.main.make_argument_parser()\n args = []\n hw_cmd = spack.cmd.get_command(hello_world_cmd.command_name)\n hw_cmd(parser, args)\n captured = capfd.readouterr()\n assert captured == (\"Hello world!\\n\", \"\")\n\n\n@pytest.mark.parametrize(\n \"extension_name\", [None, \"hyphenated-extension\"], ids=[\"simple\", \"hyphenated_extension_name\"]\n)\ndef test_command_with_import(extension_name, hello_world_with_module_in_root):\n \"\"\"Ensure we can write a functioning command with multiple imported\n subcommands, including where the extension name contains a hyphen.\n \"\"\"\n with hello_world_with_module_in_root(extension_name) as hello_world:\n output = hello_world(\"world\")\n assert \"Hello world!\" in output\n output = hello_world(\"folks\")\n assert \"Hello folks!\" in output\n output = hello_world(\"global\")\n assert \"bar\" in output\n\n\ndef test_missing_command():\n \"\"\"Ensure that we raise the expected exception if the desired command is\n not present.\n \"\"\"\n with pytest.raises(spack.cmd.CommandNotFoundError):\n spack.cmd.get_module(\"no-such-command\")\n\n\n@pytest.mark.parametrize(\n \"extension_path,expected_exception\",\n [\n (\"/my/bad/extension\", spack.extensions.ExtensionNamingError),\n (\"\", spack.extensions.ExtensionNamingError),\n (\"/my/bad/spack--extra-hyphen\", spack.extensions.ExtensionNamingError),\n (\"/my/good/spack-extension\", spack.cmd.CommandNotFoundError),\n (\"/my/still/good/spack-extension/\", spack.cmd.CommandNotFoundError),\n (\"/my/spack-hyphenated-extension\", spack.cmd.CommandNotFoundError),\n ],\n ids=[\"no_stem\", \"vacuous\", \"leading_hyphen\", \"basic_good\", \"trailing_slash\", \"hyphenated\"],\n)\ndef test_extension_naming(tmp_path: pathlib.Path, extension_path, expected_exception, config):\n \"\"\"Ensure that we are correctly validating configured extension paths\n for conformity with the rules: the basename should match\n ``spack-``; may have embedded hyphens but not begin with one.\n \"\"\"\n # NOTE: if the directory is a valid extension directory name the \"vacuous\" test will\n # fail because it resolves to current working directory\n import spack.llnl.util.filesystem as fs\n\n with fs.working_dir(str(tmp_path)):\n with spack.config.override(\"config:extensions\", [extension_path]):\n with pytest.raises(expected_exception):\n spack.cmd.get_module(\"no-such-command\")\n\n\ndef test_missing_command_function(extension_creator, capfd):\n \"\"\"Ensure we die as expected if a command module does not have the\n expected command function defined.\n \"\"\"\n with extension_creator() as extension:\n extension.add_command(\"bad-cmd\", \"\"\"\\ndescription = \"Empty command implementation\"\\n\"\"\")\n with pytest.raises(SystemExit):\n spack.cmd.get_module(\"bad-cmd\")\n capture = capfd.readouterr()\n assert \"must define function 'bad_cmd'.\" in capture[1]\n\n\ndef test_get_command_paths(config):\n \"\"\"Exercise the construction of extension command search paths.\"\"\"\n extensions = (\"extension-1\", \"extension-2\")\n ext_paths = []\n expected_cmd_paths = []\n for ext in extensions:\n ext_path = os.path.join(\"my\", \"path\", \"to\", \"spack-\" + ext)\n ext_paths.append(ext_path)\n path = os.path.join(ext_path, spack.cmd.python_name(ext), \"cmd\")\n path = os.path.abspath(path)\n expected_cmd_paths.append(path)\n\n with spack.config.override(\"config:extensions\", ext_paths):\n assert spack.extensions.get_command_paths() == expected_cmd_paths\n\n\ndef test_variable_in_extension_path(config, working_env):\n \"\"\"Test variables in extension paths.\"\"\"\n os.environ[\"_MY_VAR\"] = os.path.join(\"my\", \"var\")\n ext_paths = [os.path.join(\"~\", \"${_MY_VAR}\", \"spack-extension-1\")]\n # Home env variable is USERPROFILE on Windows\n home_env = \"USERPROFILE\" if sys.platform == \"win32\" else \"HOME\"\n expected_ext_paths = [\n os.path.join(os.environ[home_env], os.environ[\"_MY_VAR\"], \"spack-extension-1\")\n ]\n with spack.config.override(\"config:extensions\", ext_paths):\n assert spack.extensions.get_extension_paths() == expected_ext_paths\n\n\n@pytest.mark.parametrize(\n \"command_name,contents,exception\",\n [\n (\"bad-cmd\", \"from oopsie.daisy import bad\\n\", ImportError),\n (\"bad-cmd\", \"\"\"var = bad_function_call('blech')\\n\"\"\", NameError),\n (\"bad-cmd\", \")\\n\", SyntaxError),\n ],\n ids=[\"ImportError\", \"NameError\", \"SyntaxError\"],\n)\ndef test_failing_command(command_name, contents, exception, extension_creator):\n \"\"\"Ensure that the configured command fails to import with the specified\n error.\n \"\"\"\n with extension_creator() as extension:\n extension.add_command(command_name, contents)\n with pytest.raises(exception):\n spack.extensions.get_module(command_name)\n" }, { "path": "lib/spack/spack/test/compilers/conversion.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Tests conversions from compilers.yaml\"\"\"\n\nimport pathlib\n\nimport pytest\n\nfrom spack.compilers.config import CompilerFactory\n\npytestmark = [pytest.mark.usefixtures(\"config\", \"mock_packages\")]\n\n\n@pytest.fixture()\ndef mock_compiler(mock_executable):\n gcc = mock_executable(\"gcc\", \"echo 13.2.0\")\n gxx = mock_executable(\"g++\", \"echo 13.2.0\")\n gfortran = mock_executable(\"gfortran\", \"echo 13.2.0\")\n return {\n \"spec\": \"gcc@13.2.0\",\n \"paths\": {\"cc\": str(gcc), \"cxx\": str(gxx), \"f77\": str(gfortran), \"fc\": str(gfortran)},\n }\n\n\n# - compiler:\n# spec: clang@=10.0.0\n# paths:\n# cc: /usr/bin/clang\n# cxx: /usr/bin/clang++\n# f77: null\n# fc: null\n# flags: {}\n# operating_system: ubuntu20.04\n# target: x86_64\n# modules: []\n# environment: {}\n# extra_rpaths: []\n\n\ndef test_basic_compiler_conversion(mock_compiler, tmp_path: pathlib.Path):\n \"\"\"Tests the conversion of a compiler using a single toolchain, with default options.\"\"\"\n compilers = CompilerFactory.from_legacy_yaml(mock_compiler)\n compiler_spec = compilers[0]\n assert compiler_spec.satisfies(\"gcc@13.2.0 languages=c,c++,fortran\")\n assert compiler_spec.external\n assert compiler_spec.external_path == str(tmp_path)\n\n for language in (\"c\", \"cxx\", \"fortran\"):\n assert language in compiler_spec.extra_attributes[\"compilers\"]\n\n\ndef test_compiler_conversion_with_flags(mock_compiler):\n \"\"\"Tests that flags are converted appropriately for external compilers\"\"\"\n mock_compiler[\"flags\"] = {\"cflags\": \"-O3\", \"cxxflags\": \"-O2 -g\"}\n compiler_spec = CompilerFactory.from_legacy_yaml(mock_compiler)[0]\n assert compiler_spec.external\n assert \"flags\" in compiler_spec.extra_attributes\n assert compiler_spec.extra_attributes[\"flags\"][\"cflags\"] == \"-O3\"\n assert compiler_spec.extra_attributes[\"flags\"][\"cxxflags\"] == \"-O2 -g\"\n\n\ndef test_compiler_conversion_with_environment(mock_compiler):\n \"\"\"Tests that custom environment modifications are converted appropriately\n for external compilers\n \"\"\"\n mods = {\"set\": {\"FOO\": \"foo\", \"BAR\": \"bar\"}, \"unset\": [\"BAZ\"]}\n mock_compiler[\"environment\"] = mods\n compiler_spec = CompilerFactory.from_legacy_yaml(mock_compiler)[0]\n assert compiler_spec.external\n assert \"environment\" in compiler_spec.extra_attributes\n assert compiler_spec.extra_attributes[\"environment\"] == mods\n\n\ndef test_compiler_conversion_extra_rpaths(mock_compiler):\n \"\"\"Tests that extra rpaths are converted appropriately for external compilers\"\"\"\n mock_compiler[\"extra_rpaths\"] = [\"/foo/bar\"]\n compiler_spec = CompilerFactory.from_legacy_yaml(mock_compiler)[0]\n assert compiler_spec.external\n assert \"extra_rpaths\" in compiler_spec.extra_attributes\n assert compiler_spec.extra_attributes[\"extra_rpaths\"] == [\"/foo/bar\"]\n\n\ndef test_compiler_conversion_modules(mock_compiler):\n \"\"\"Tests that modules are converted appropriately for external compilers\"\"\"\n modules = [\"foo/4.1.2\", \"bar/5.1.4\"]\n mock_compiler[\"modules\"] = modules\n compiler_spec = CompilerFactory.from_legacy_yaml(mock_compiler)[0]\n assert compiler_spec.external\n assert compiler_spec.external_modules == modules\n\n\n@pytest.mark.regression(\"49717\")\ndef test_compiler_conversion_corrupted_paths(mock_compiler):\n \"\"\"Tests that compiler entries with corrupted path do not raise\"\"\"\n mock_compiler[\"paths\"] = {\"cc\": \"gcc\", \"cxx\": \"g++\", \"fc\": \"gfortran\", \"f77\": \"gfortran\"}\n # Test this call doesn't raise\n compiler_spec = CompilerFactory.from_legacy_yaml(mock_compiler)\n assert compiler_spec == []\n" }, { "path": "lib/spack/spack/test/compilers/libraries.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport copy\nimport os\nimport pathlib\n\nimport pytest\n\nimport spack.compilers.config\nimport spack.compilers.libraries\nimport spack.llnl.util.filesystem as fs\nimport spack.util.executable\nimport spack.util.module_cmd\n\nwithout_flag_output = \"ld -L/path/to/first/lib -L/path/to/second/lib64\"\nwith_flag_output = \"ld -L/path/to/first/with/flag/lib -L/path/to/second/lib64\"\n\n\ndef call_compiler(exe, *args, **kwargs):\n # This method can replace Executable.__call__ to emulate a compiler that\n # changes libraries depending on a flag.\n if \"--correct-flag\" in exe.exe:\n return with_flag_output\n return without_flag_output\n\n\n@pytest.fixture()\ndef mock_gcc(config):\n compilers = spack.compilers.config.all_compilers_from(configuration=config)\n assert compilers, \"No compilers available\"\n\n compilers.sort(key=lambda x: (x.name == \"gcc\", x.version))\n # Deepcopy is used to avoid more boilerplate when changing the \"extra_attributes\"\n return copy.deepcopy(compilers[-1])\n\n\n@pytest.mark.usefixtures(\"mock_packages\")\nclass TestCompilerPropertyDetector:\n @pytest.mark.parametrize(\n \"language,flagname\",\n [\n (\"cxx\", \"cxxflags\"),\n (\"cxx\", \"cppflags\"),\n (\"cxx\", \"ldflags\"),\n (\"c\", \"cflags\"),\n (\"c\", \"cppflags\"),\n ],\n )\n @pytest.mark.not_on_windows(\"Not supported on Windows\")\n def test_compile_dummy_c_source(self, mock_gcc, monkeypatch, language, flagname):\n monkeypatch.setattr(spack.util.executable.Executable, \"__call__\", call_compiler)\n for key in list(mock_gcc.extra_attributes[\"compilers\"]):\n if key == language:\n continue\n mock_gcc.extra_attributes[\"compilers\"].pop(key)\n\n detector = spack.compilers.libraries.CompilerPropertyDetector(mock_gcc)\n\n # Test without flags\n assert detector._compile_dummy_c_source() == without_flag_output\n\n # Set flags and test\n if flagname:\n mock_gcc.extra_attributes.setdefault(\"flags\", {})\n monkeypatch.setitem(mock_gcc.extra_attributes[\"flags\"], flagname, \"--correct-flag\")\n assert detector._compile_dummy_c_source() == with_flag_output\n\n def test_compile_dummy_c_source_no_path(self, mock_gcc):\n mock_gcc.extra_attributes[\"compilers\"] = {}\n detector = spack.compilers.libraries.CompilerPropertyDetector(mock_gcc)\n assert detector._compile_dummy_c_source() is None\n\n def test_compile_dummy_c_source_no_verbose_flags(self, mock_gcc, monkeypatch):\n monkeypatch.setattr(mock_gcc.package, \"verbose_flags\", \"\")\n detector = spack.compilers.libraries.CompilerPropertyDetector(mock_gcc)\n assert detector._compile_dummy_c_source() is None\n\n @pytest.mark.not_on_windows(\"Module files are not supported on Windows\")\n def test_compile_dummy_c_source_load_env(self, mock_gcc, monkeypatch, tmp_path: pathlib.Path):\n gcc = tmp_path / \"gcc\"\n gcc.write_text(\n f\"\"\"#!/bin/sh\n if [ \"$ENV_SET\" = \"1\" ] && [ \"$MODULE_LOADED\" = \"1\" ]; then\n printf '{without_flag_output}'\n fi\n \"\"\"\n )\n fs.set_executable(str(gcc))\n\n # Set module load to turn compiler on\n def module(*args):\n if args[0] == \"show\":\n return \"\"\n elif args[0] == \"load\":\n monkeypatch.setenv(\"MODULE_LOADED\", \"1\")\n monkeypatch.setenv(\"LOADEDMODULES\", \"turn_on\")\n\n monkeypatch.setattr(spack.util.module_cmd, \"module\", module)\n\n mock_gcc.extra_attributes[\"compilers\"][\"c\"] = str(gcc)\n mock_gcc.extra_attributes[\"environment\"] = {\"set\": {\"ENV_SET\": \"1\"}}\n mock_gcc.external_modules = [\"turn_on\"]\n\n detector = spack.compilers.libraries.CompilerPropertyDetector(mock_gcc)\n assert detector._compile_dummy_c_source() == without_flag_output\n\n @pytest.mark.not_on_windows(\"Not supported on Windows\")\n def test_implicit_rpaths(self, mock_gcc, dirs_with_libfiles, monkeypatch):\n lib_to_dirs, all_dirs = dirs_with_libfiles\n\n detector = spack.compilers.libraries.CompilerPropertyDetector(mock_gcc)\n monkeypatch.setattr(\n spack.compilers.libraries.CompilerPropertyDetector,\n \"_compile_dummy_c_source\",\n lambda self: \"ld \" + \" \".join(f\"-L{d}\" for d in all_dirs),\n )\n\n retrieved_rpaths = detector.implicit_rpaths()\n assert set(retrieved_rpaths) == set(lib_to_dirs[\"libstdc++\"] + lib_to_dirs[\"libgfortran\"])\n\n def test_compiler_environment(self, working_env, mock_gcc, monkeypatch):\n \"\"\"Test whether environment modifications are applied in compiler_environment\"\"\"\n monkeypatch.delenv(\"TEST\", raising=False)\n mock_gcc.extra_attributes[\"environment\"] = {\"set\": {\"TEST\": \"yes\"}}\n detector = spack.compilers.libraries.CompilerPropertyDetector(mock_gcc)\n with detector.compiler_environment():\n assert os.environ[\"TEST\"] == \"yes\"\n\n @pytest.mark.not_on_windows(\"Module files are not supported on Windows\")\n def test_compiler_invalid_module_raises(self, working_env, mock_gcc, monkeypatch):\n \"\"\"Test if an exception is raised when a module cannot be loaded\"\"\"\n\n def mock_load_module(module_name):\n # Simulate module load failure\n raise spack.util.module_cmd.ModuleLoadError(module_name)\n\n monkeypatch.setattr(spack.util.module_cmd, \"load_module\", mock_load_module)\n\n mock_gcc.external_modules = [\"non_existent\"]\n detector = spack.compilers.libraries.CompilerPropertyDetector(mock_gcc)\n\n with pytest.raises(spack.util.module_cmd.ModuleLoadError):\n with detector.compiler_environment():\n pass\n" }, { "path": "lib/spack/spack/test/concretization/compiler_runtimes.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport pathlib\n\nimport pytest\n\nimport spack.vendor.archspec.cpu\n\nimport spack.concretize\nimport spack.config\nimport spack.paths\nimport spack.repo\nimport spack.solver.asp\nimport spack.spec\nfrom spack.environment.environment import ViewDescriptor\nfrom spack.solver.reuse import create_external_parser, spec_filter_from_packages_yaml\nfrom spack.solver.runtimes import external_config_with_implicit_externals\nfrom spack.version import Version\n\n\ndef _concretize_with_reuse(*, root_str, reused_str, config):\n reused_spec = spack.concretize.concretize_one(reused_str)\n packages_with_externals = external_config_with_implicit_externals(config)\n completion_mode = config.get(\"concretizer:externals:completion\")\n external_specs = spec_filter_from_packages_yaml(\n external_parser=create_external_parser(packages_with_externals, completion_mode),\n packages_with_externals=packages_with_externals,\n include=[],\n exclude=[],\n ).selected_specs()\n setup = spack.solver.asp.SpackSolverSetup(tests=False)\n driver = spack.solver.asp.PyclingoDriver()\n result, _, _ = driver.solve(\n setup, [spack.spec.Spec(f\"{root_str}\")], reuse=[reused_spec] + external_specs\n )\n root = result.specs[0]\n return root, reused_spec\n\n\n@pytest.fixture\ndef runtime_repo(mutable_config):\n repo = os.path.join(spack.paths.test_repos_path, \"spack_repo\", \"compiler_runtime_test\")\n with spack.repo.use_repositories(repo) as mock_repo:\n yield mock_repo\n\n\ndef test_correct_gcc_runtime_is_injected_as_dependency(runtime_repo):\n s = spack.concretize.concretize_one(\"pkg-a%gcc@10.2.1 ^pkg-b%gcc@9.4.0\")\n a, b = s[\"pkg-a\"], s[\"pkg-b\"]\n\n # Both a and b should depend on the same gcc-runtime directly\n assert a.dependencies(\"gcc-runtime\") == b.dependencies(\"gcc-runtime\")\n\n # And the gcc-runtime version should be that of the newest gcc used in the dag.\n assert a[\"gcc-runtime\"].version == Version(\"10.2.1\")\n\n\n@pytest.mark.regression(\"41972\")\ndef test_external_nodes_do_not_have_runtimes(runtime_repo, mutable_config, tmp_path: pathlib.Path):\n \"\"\"Tests that external nodes don't have runtime dependencies.\"\"\"\n\n packages_yaml = {\"pkg-b\": {\"externals\": [{\"spec\": \"pkg-b@1.0\", \"prefix\": f\"{str(tmp_path)}\"}]}}\n spack.config.set(\"packages\", packages_yaml)\n\n s = spack.concretize.concretize_one(\"pkg-a%gcc@10.2.1\")\n\n a, b = s[\"pkg-a\"], s[\"pkg-b\"]\n\n # Since b is an external, it doesn't depend on gcc-runtime\n assert a.dependencies(\"gcc-runtime\")\n assert a.dependencies(\"pkg-b\")\n assert not b.dependencies(\"gcc-runtime\")\n\n\n@pytest.mark.parametrize(\n \"root_str,reused_str,expected,nruntime\",\n [\n # The reused runtime is older than we need, thus we'll add a more recent one for a\n (\n \"pkg-a%gcc@10.2.1\",\n \"pkg-b%gcc@9.4.0\",\n {\"pkg-a\": \"gcc-runtime@10.2.1\", \"pkg-b\": \"gcc-runtime@9.4.0\"},\n 2,\n ),\n # The root is compiled with an older compiler, thus we'll NOT reuse the runtime from b\n (\n \"pkg-a%gcc@9.4.0\",\n \"pkg-b%gcc@10.2.1\",\n {\"pkg-a\": \"gcc-runtime@9.4.0\", \"pkg-b\": \"gcc-runtime@9.4.0\"},\n 1,\n ),\n # Same as before, but tests that we can reuse from a more generic target\n pytest.param(\n \"pkg-a%gcc@9.4.0\",\n \"pkg-b target=x86_64 %gcc@10.2.1\",\n {\"pkg-a\": \"gcc-runtime@9.4.0\", \"pkg-b\": \"gcc-runtime@9.4.0\"},\n 1,\n marks=pytest.mark.skipif(\n str(spack.vendor.archspec.cpu.host().family) != \"x86_64\",\n reason=\"test data is x86_64 specific\",\n ),\n ),\n pytest.param(\n \"pkg-a%gcc@10.2.1\",\n \"pkg-b target=x86_64 %gcc@9.4.0\",\n {\n \"pkg-a\": \"gcc-runtime@10.2.1 target=core2\",\n \"pkg-b\": \"gcc-runtime@9.4.0 target=x86_64\",\n },\n 2,\n marks=pytest.mark.skipif(\n str(spack.vendor.archspec.cpu.host().family) != \"x86_64\",\n reason=\"test data is x86_64 specific\",\n ),\n ),\n ],\n)\n@pytest.mark.regression(\"44444\")\ndef test_reusing_specs_with_gcc_runtime(\n root_str, reused_str, expected, nruntime, runtime_repo, mutable_config\n):\n \"\"\"Tests that we can reuse specs with a \"gcc-runtime\" leaf node. In particular, checks\n that the semantic for gcc-runtimes versions accounts for reused packages too.\n\n Reusable runtime versions should be lower, or equal, to that of parent nodes.\n \"\"\"\n root, reused_spec = _concretize_with_reuse(\n root_str=root_str, reused_str=reused_str, config=mutable_config\n )\n\n runtime_a = root.dependencies(\"gcc-runtime\")[0]\n assert runtime_a.satisfies(expected[\"pkg-a\"]), runtime_a.tree()\n runtime_b = root[\"pkg-b\"].dependencies(\"gcc-runtime\")[0]\n assert runtime_b.satisfies(expected[\"pkg-b\"])\n\n runtimes = [x for x in root.traverse() if x.name == \"gcc-runtime\"]\n assert len(runtimes) == nruntime\n\n\n@pytest.mark.parametrize(\n \"root_str,reused_str,expected,not_expected\",\n [\n # Ensure that, whether we have multiple runtimes in the DAG or not,\n # we always link only the latest version\n (\"pkg-a%gcc@10.2.1\", \"pkg-b%gcc@9.4.0\", [\"gcc-runtime@10.2.1\"], [\"gcc-runtime@9.4.0\"])\n ],\n)\ndef test_views_can_handle_duplicate_runtime_nodes(\n root_str,\n reused_str,\n expected,\n not_expected,\n runtime_repo,\n tmp_path: pathlib.Path,\n monkeypatch,\n mutable_config,\n):\n \"\"\"Tests that an environment is able to select the latest version of a runtime node to be\n linked in a view, in case more than one compatible version is in the DAG.\n \"\"\"\n root, reused_spec = _concretize_with_reuse(\n root_str=root_str, reused_str=reused_str, config=mutable_config\n )\n\n # Mock the installation status to allow selecting nodes for the view\n monkeypatch.setattr(spack.spec.Spec, \"installed\", True)\n nodes = list(root.traverse())\n\n view = ViewDescriptor(str(tmp_path), str(tmp_path))\n candidate_specs = view.specs_for_view(nodes)\n\n for x in expected:\n assert any(node.satisfies(x) for node in candidate_specs)\n\n for x in not_expected:\n assert all(not node.satisfies(x) for node in candidate_specs)\n\n\ndef test_runtimes_can_be_concretized_as_standalone(runtime_repo):\n \"\"\"Tests that we can concretize a runtime as a standalone\"\"\"\n gcc_runtime = spack.concretize.concretize_one(\"gcc-runtime\")\n\n deps = gcc_runtime.dependencies()\n assert len(deps) == 1\n gcc = deps[0]\n assert gcc_runtime.version == gcc.version\n\n\ndef test_runtimes_are_not_reused_if_compiler_not_used(runtime_repo, mutable_config):\n \"\"\"Tests that, if we can reuse specs with a more recent runtime version than the compiler we\n asked for, we will not end-up with a DAG using the recent runtime, and the old compiler.\n \"\"\"\n root, reused = _concretize_with_reuse(\n root_str=\"pkg-a %gcc@9\", reused_str=\"pkg-a %gcc@10\", config=mutable_config\n )\n\n assert \"gcc-runtime\" in root\n gcc_runtime, gcc = root[\"gcc-runtime\"], root[\"gcc\"]\n assert gcc_runtime.satisfies(\"@9\") and not gcc_runtime.satisfies(\"@10\")\n assert gcc.satisfies(\"@9\") and not gcc.satisfies(\"@10\")\n # Same gcc used for both languages\n assert root[\"c\"] == root[\"cxx\"]\n" }, { "path": "lib/spack/spack/test/concretization/conditional_dependencies.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport pytest\n\nimport spack.concretize\nimport spack.spec\n\n\n@pytest.mark.parametrize(\n \"abstract_spec,expected,not_expected\",\n [\n # Set +mpi explicitly\n (\n \"hdf5+mpi ^[when='^mpi' virtuals=mpi] zmpi\",\n [\"%[virtuals=mpi] zmpi\", \"^mpi\", \"%mpi\"],\n [\"%[virtuals=mpi] mpich\"],\n ),\n (\n \"hdf5+mpi %[when='%mpi' virtuals=mpi] zmpi\",\n [\"%[virtuals=mpi] zmpi\", \"^mpi\", \"%mpi\"],\n [\"%[virtuals=mpi] mpich\"],\n ),\n (\n \"hdf5+mpi %[when='+mpi' virtuals=mpi] zmpi\",\n [\"%[virtuals=mpi] zmpi\", \"^mpi\", \"%mpi\"],\n [\"%[virtuals=mpi] mpich\"],\n ),\n (\n \"hdf5+mpi ^[when='^mpi' virtuals=mpi] mpich\",\n [\"%[virtuals=mpi] mpich\", \"^mpi\", \"%mpi\"],\n [\"%[virtuals=mpi] zmpi\"],\n ),\n (\n \"hdf5+mpi %[when='%mpi' virtuals=mpi] mpich\",\n [\"%[virtuals=mpi] mpich\", \"^mpi\", \"%mpi\"],\n [\"%[virtuals=mpi] zmpi\"],\n ),\n # Use the default, which is to have +mpi\n (\n \"hdf5 ^[when='^mpi' virtuals=mpi] zmpi\",\n [\"%[virtuals=mpi] zmpi\", \"^mpi\", \"%mpi\"],\n [\"%[virtuals=mpi] mpich\"],\n ),\n (\n \"hdf5 %[when='%mpi' virtuals=mpi] zmpi\",\n [\"%[virtuals=mpi] zmpi\", \"^mpi\", \"%mpi\"],\n [\"%[virtuals=mpi] mpich\"],\n ),\n (\n \"hdf5 %[when='+mpi' virtuals=mpi] zmpi\",\n [\"%[virtuals=mpi] zmpi\", \"^mpi\", \"%mpi\"],\n [\"%[virtuals=mpi] mpich\"],\n ),\n # Set ~mpi explicitly\n (\"hdf5~mpi ^[when='^mpi' virtuals=mpi] zmpi\", [], [\"%[virtuals=mpi] zmpi\", \"^mpi\"]),\n (\"hdf5~mpi %[when='%mpi' virtuals=mpi] zmpi\", [], [\"%[virtuals=mpi] zmpi\", \"^mpi\"]),\n (\"hdf5~mpi %[when='+mpi' virtuals=mpi] zmpi\", [], [\"%[virtuals=mpi] zmpi\", \"^mpi\"]),\n ],\n)\ndef test_conditional_mpi_dependency(\n abstract_spec, expected, not_expected, default_mock_concretization\n):\n \"\"\"Test concretizing conditional mpi dependencies.\"\"\"\n concrete = default_mock_concretization(abstract_spec)\n\n for x in expected:\n assert concrete.satisfies(x), x\n\n for x in not_expected:\n assert not concrete.satisfies(x), x\n\n assert concrete.satisfies(abstract_spec)\n\n\n@pytest.mark.parametrize(\"c\", [True, False])\n@pytest.mark.parametrize(\"cxx\", [True, False])\n@pytest.mark.parametrize(\"fortran\", [True, False])\ndef test_conditional_compilers(c, cxx, fortran, mutable_config, mock_packages, config_two_gccs):\n \"\"\"Test concretizing with conditional compilers, using every combination of +~c, +~cxx,\n and +~fortran.\n \"\"\"\n # Abstract spec parametrized to depend/not on c/cxx/fortran\n # and with conditional dependencies for each on the less preferred gcc\n abstract = spack.spec.Spec(f\"conditional-languages c={c} cxx={cxx} fortran={fortran}\")\n concrete_unconstrained = spack.concretize.concretize_one(abstract)\n abstract.constrain(\n \"^[when='%c' virtuals=c]gcc@10.3.1 \"\n \"^[when='%cxx' virtuals=cxx]gcc@10.3.1 \"\n \"^[when='%fortran' virtuals=fortran]gcc@10.3.1\"\n )\n concrete = spack.concretize.concretize_one(abstract)\n\n # We should get the dependency we specified for each language we enabled\n assert concrete.satisfies(\"%[virtuals=c]gcc@10.3.1\") == c\n assert concrete.satisfies(\"%[virtuals=cxx]gcc@10.3.1\") == cxx\n assert concrete.satisfies(\"%[virtuals=fortran]gcc@10.3.1\") == fortran\n\n # The only time the two concrete specs are the same is if we don't use gcc at all\n assert (concrete == concrete_unconstrained) == (not any((c, cxx, fortran)))\n" }, { "path": "lib/spack/spack/test/concretization/core.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport difflib\nimport json\nimport os\nimport pathlib\nimport platform\nimport re\nimport sys\nfrom typing import Any, Dict\n\nimport pytest\n\nimport spack.vendor.archspec.cpu\nimport spack.vendor.jinja2\n\nimport spack.archspec\nimport spack.binary_distribution\nimport spack.cmd\nimport spack.compilers.config\nimport spack.compilers.libraries\nimport spack.concretize\nimport spack.config\nimport spack.deptypes as dt\nimport spack.environment as ev\nimport spack.error\nimport spack.hash_types as ht\nimport spack.llnl.util.lang\nimport spack.package_base\nimport spack.paths\nimport spack.platforms\nimport spack.platforms.test\nimport spack.repo\nimport spack.solver.asp\nimport spack.solver.core\nimport spack.solver.input_analysis\nimport spack.solver.reuse\nimport spack.solver.runtimes\nimport spack.spec\nimport spack.spec_filter\nimport spack.util.file_cache\nimport spack.util.hash\nimport spack.util.spack_yaml as syaml\nimport spack.variant as vt\nfrom spack.externals import ExternalDependencyError\nfrom spack.installer import PackageInstaller\nfrom spack.solver.asp import Result\nfrom spack.solver.reuse import create_external_parser, spec_filter_from_packages_yaml\nfrom spack.solver.runtimes import external_config_with_implicit_externals\nfrom spack.spec import Spec\nfrom spack.test.conftest import RepoBuilder\nfrom spack.version import Version, VersionList, ver\n\n\ndef check_spec(abstract, concrete):\n if abstract.versions.concrete:\n assert abstract.versions == concrete.versions\n\n if abstract.variants:\n for name in abstract.variants:\n avariant = abstract.variants[name]\n cvariant = concrete.variants[name]\n assert avariant.value == cvariant.value\n\n if abstract.compiler_flags:\n for flag in abstract.compiler_flags:\n aflag = abstract.compiler_flags[flag]\n cflag = concrete.compiler_flags[flag]\n assert set(aflag) <= set(cflag)\n\n for name in spack.repo.PATH.get_pkg_class(abstract.name).variant_names():\n assert name in concrete.variants\n\n for flag in concrete.compiler_flags.valid_compiler_flags():\n assert flag in concrete.compiler_flags\n\n if abstract.architecture and abstract.architecture.concrete:\n assert abstract.architecture == concrete.architecture\n\n\ndef check_concretize(abstract_spec):\n abstract = Spec(abstract_spec)\n concrete = spack.concretize.concretize_one(abstract)\n assert not abstract.concrete\n assert concrete.concrete\n check_spec(abstract, concrete)\n return concrete\n\n\ndef _true():\n return True\n\n\n@pytest.fixture(scope=\"function\", autouse=True)\ndef binary_compatibility(monkeypatch, request):\n \"\"\"Selects whether we use OS compatibility for binaries, or libc compatibility.\"\"\"\n if spack.platforms.real_host().name != \"linux\":\n return\n\n if \"mock_packages\" not in request.fixturenames:\n # Only builtin_mock has a mock glibc package\n return\n\n if \"database\" in request.fixturenames or \"mutable_database\" in request.fixturenames:\n # Databases have been created without glibc support\n return\n\n monkeypatch.setattr(spack.solver.core, \"using_libc_compatibility\", _true)\n monkeypatch.setattr(spack.solver.runtimes, \"using_libc_compatibility\", _true)\n monkeypatch.setattr(spack.solver.asp, \"using_libc_compatibility\", _true)\n\n\n@pytest.fixture(\n params=[\n # no_deps\n \"libelf\",\n \"libelf@0.8.13\",\n # dag\n \"callpath\",\n \"mpileaks\",\n \"libelf\",\n # variant\n \"mpich+debug\",\n \"mpich~debug\",\n \"mpich debug=True\",\n \"mpich\",\n # compiler flags\n 'mpich cppflags=\"-O3\"',\n 'mpich cppflags==\"-O3\"',\n # with virtual\n \"mpileaks ^mpi\",\n \"mpileaks ^mpi@:1.1\",\n \"mpileaks ^mpi@2:\",\n \"mpileaks ^mpi@2.1\",\n \"mpileaks ^mpi@2.2\",\n \"mpileaks ^mpi@2.2\",\n \"mpileaks ^mpi@:1\",\n \"mpileaks ^mpi@1.2:2\",\n # conflict not triggered\n \"conflict\",\n \"conflict~foo%clang\",\n \"conflict-parent%gcc\",\n # Direct dependency with different deptypes\n \"mpileaks %[deptypes=link] mpich\",\n ]\n)\ndef spec(request):\n \"\"\"Spec to be concretized\"\"\"\n return request.param\n\n\n@pytest.fixture(\n params=[\n # Mocking the host detection\n \"haswell\",\n \"broadwell\",\n \"skylake\",\n \"icelake\",\n # Using preferred targets from packages.yaml\n \"icelake-preference\",\n \"cannonlake-preference\",\n ]\n)\ndef current_host(request, monkeypatch):\n # is_preference is not empty if we want to supply the\n # preferred target via packages.yaml\n cpu, _, is_preference = request.param.partition(\"-\")\n\n monkeypatch.setattr(spack.platforms.Test, \"default\", cpu)\n monkeypatch.setattr(\n spack.archspec, \"HOST_TARGET_FAMILY\", spack.vendor.archspec.cpu.TARGETS[\"x86_64\"]\n )\n if not is_preference:\n target = spack.vendor.archspec.cpu.TARGETS[cpu]\n monkeypatch.setattr(spack.vendor.archspec.cpu, \"host\", lambda: target)\n yield target\n else:\n target = spack.vendor.archspec.cpu.TARGETS[\"sapphirerapids\"]\n monkeypatch.setattr(spack.vendor.archspec.cpu, \"host\", lambda: target)\n with spack.config.override(\"packages:all\", {\"target\": [cpu]}):\n yield target\n\n\n@pytest.fixture(scope=\"function\", params=[True, False])\ndef fuzz_dep_order(request, monkeypatch):\n \"\"\"Meta-function that tweaks the order of iteration over dependencies in a package.\"\"\"\n\n def reverser(pkg_name):\n if request.param:\n pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)\n reversed_dict = dict(reversed(list(pkg_cls.dependencies.items())))\n monkeypatch.setattr(pkg_cls, \"dependencies\", reversed_dict)\n\n return reverser\n\n\n@pytest.fixture()\ndef repo_with_changing_recipe(tmp_path_factory: pytest.TempPathFactory, mutable_mock_repo):\n repos_dir: pathlib.Path = tmp_path_factory.mktemp(\"repos_dir\")\n root, _ = spack.repo.create_repo(str(repos_dir), \"changing\")\n packages_dir = pathlib.Path(root, \"packages\")\n\n root_pkg_str = \"\"\"\nfrom spack_repo.builtin_mock.build_systems.generic import Package\nfrom spack.package import *\n\nclass Root(Package):\n homepage = \"http://www.example.com\"\n url = \"http://www.example.com/root-1.0.tar.gz\"\n\n version(\"1.0\", sha256=\"abcde\")\n depends_on(\"middle\")\n depends_on(\"changing\")\n\n conflicts(\"^changing~foo\")\n\"\"\"\n package_py = packages_dir / \"root\" / \"package.py\"\n package_py.parent.mkdir(parents=True)\n package_py.write_text(root_pkg_str)\n\n middle_pkg_str = \"\"\"\nfrom spack_repo.builtin_mock.build_systems.generic import Package\nfrom spack.package import *\n\nclass Middle(Package):\n homepage = \"http://www.example.com\"\n url = \"http://www.example.com/root-1.0.tar.gz\"\n\n version(\"1.0\", sha256=\"abcde\")\n depends_on(\"changing\")\n\"\"\"\n package_py = packages_dir / \"middle\" / \"package.py\"\n package_py.parent.mkdir(parents=True)\n package_py.write_text(middle_pkg_str)\n\n changing_template = \"\"\"\nfrom spack_repo.builtin_mock.build_systems.generic import Package\nfrom spack.package import *\n\nclass Changing(Package):\n homepage = \"http://www.example.com\"\n url = \"http://www.example.com/changing-1.0.tar.gz\"\n\n\n{% if not delete_version %}\n version(\"1.0\", sha256=\"abcde\")\n{% endif %}\n version(\"0.9\", sha256=\"abcde\")\n\n{% if not delete_variant %}\n variant(\"fee\", default=True, description=\"nope\")\n{% endif %}\n variant(\"foo\", default=True, description=\"nope\")\n{% if add_variant %}\n variant(\"fum\", default=True, description=\"nope\")\n variant(\"fum2\", default=True, description=\"nope\")\n{% endif %}\n\"\"\"\n\n with spack.repo.use_repositories(root, override=False) as repos:\n\n class _ChangingPackage:\n default_context = [\n (\"delete_version\", True),\n (\"delete_variant\", False),\n (\"add_variant\", False),\n ]\n\n def __init__(self):\n cache_dir = tmp_path_factory.mktemp(\"cache\")\n self.repo_cache = spack.util.file_cache.FileCache(str(cache_dir))\n self.repo = spack.repo.Repo(root, cache=self.repo_cache)\n\n def change(self, changes=None):\n changes = changes or {}\n context = dict(self.default_context)\n context.update(changes)\n # Remove the repo object and delete Python modules\n repos.remove(self.repo)\n # TODO: this mocks a change in the recipe that should happen in a\n # TODO: different process space. Leaving this comment as a hint\n # TODO: in case tests using this fixture start failing.\n for module in [x for x in sys.modules if x.startswith(\"spack_repo.changing\")]:\n del sys.modules[module]\n\n # Change the recipe\n t = spack.vendor.jinja2.Template(changing_template)\n changing_pkg_str = t.render(**context)\n package_py = packages_dir / \"changing\" / \"package.py\"\n package_py.parent.mkdir(parents=True, exist_ok=True)\n package_py.write_text(changing_pkg_str)\n\n # Re-add the repository\n self.repo = spack.repo.Repo(root, cache=self.repo_cache)\n repos.put_first(self.repo)\n\n _changing_pkg = _ChangingPackage()\n _changing_pkg.change(\n {\"delete_version\": False, \"delete_variant\": False, \"add_variant\": False}\n )\n yield _changing_pkg\n\n\n@pytest.fixture()\ndef clang12_with_flags(compiler_factory):\n c = compiler_factory(spec=\"llvm@12.2.0+clang os=redhat6\")\n c[\"extra_attributes\"][\"flags\"] = {\"cflags\": \"-O3\", \"cxxflags\": \"-O3\"}\n return c\n\n\n@pytest.fixture()\ndef gcc11_with_flags(compiler_factory):\n c = compiler_factory(spec=\"gcc@11.1.0 languages:=c,c++,fortran os=redhat6\")\n c[\"extra_attributes\"][\"flags\"] = {\"cflags\": \"-O0 -g\", \"cxxflags\": \"-O0 -g\", \"fflags\": \"-O0 -g\"}\n return c\n\n\ndef weights_from_result(result: Result, *, name: str) -> Dict[str, int]:\n weights = {}\n for x in result.criteria:\n if x.name == name and x.kind == spack.solver.asp.OptimizationKind.CONCRETE:\n weights[\"reused\"] = x.value\n elif x.name == name and x.kind == spack.solver.asp.OptimizationKind.BUILD:\n weights[\"built\"] = x.value\n return weights\n\n\n# This must use the mutable_config fixture because the test\n# adjusting_default_target_based_on_compiler uses the current_host fixture,\n# which changes the config.\n@pytest.mark.usefixtures(\"mutable_config\", \"mock_packages\")\nclass TestConcretize:\n def test_concretize(self, spec):\n check_concretize(spec)\n\n def test_concretize_mention_build_dep(self):\n spec = check_concretize(\"cmake-client ^cmake@=3.21.3\")\n\n # Check parent's perspective of child\n to_dependencies = spec.edges_to_dependencies(name=\"cmake\")\n assert len(to_dependencies) == 1\n assert to_dependencies[0].depflag == dt.BUILD\n\n # Check child's perspective of parent\n cmake = spec[\"cmake\"]\n from_dependents = cmake.edges_from_dependents(name=\"cmake-client\")\n assert len(from_dependents) == 1\n assert from_dependents[0].depflag == dt.BUILD\n\n def test_concretize_preferred_version(self):\n spec = check_concretize(\"python\")\n assert spec.version == ver(\"=2.7.11\")\n spec = check_concretize(\"python@3.5.1\")\n assert spec.version == ver(\"=3.5.1\")\n\n def test_concretize_with_restricted_virtual(self):\n check_concretize(\"mpileaks ^mpich2\")\n\n concrete = check_concretize(\"mpileaks ^mpich2@1.1\")\n assert concrete[\"mpich2\"].satisfies(\"mpich2@1.1\")\n\n concrete = check_concretize(\"mpileaks ^mpich2@1.2\")\n assert concrete[\"mpich2\"].satisfies(\"mpich2@1.2\")\n\n concrete = check_concretize(\"mpileaks ^mpich2@:1.5\")\n assert concrete[\"mpich2\"].satisfies(\"mpich2@:1.5\")\n\n concrete = check_concretize(\"mpileaks ^mpich2@:1.3\")\n assert concrete[\"mpich2\"].satisfies(\"mpich2@:1.3\")\n\n concrete = check_concretize(\"mpileaks ^mpich2@:1.2\")\n assert concrete[\"mpich2\"].satisfies(\"mpich2@:1.2\")\n\n concrete = check_concretize(\"mpileaks ^mpich2@:1.1\")\n assert concrete[\"mpich2\"].satisfies(\"mpich2@:1.1\")\n\n concrete = check_concretize(\"mpileaks ^mpich2@1.1:\")\n assert concrete[\"mpich2\"].satisfies(\"mpich2@1.1:\")\n\n concrete = check_concretize(\"mpileaks ^mpich2@1.5:\")\n assert concrete[\"mpich2\"].satisfies(\"mpich2@1.5:\")\n\n concrete = check_concretize(\"mpileaks ^mpich2@1.3.1:1.4\")\n assert concrete[\"mpich2\"].satisfies(\"mpich2@1.3.1:1.4\")\n\n def test_concretize_with_provides_when(self):\n \"\"\"Make sure insufficient versions of MPI are not in providers list when\n we ask for some advanced version.\n \"\"\"\n repo = spack.repo.PATH\n assert not any(s.intersects(\"mpich2@:1.0\") for s in repo.providers_for(\"mpi@2.1\"))\n assert not any(s.intersects(\"mpich2@:1.1\") for s in repo.providers_for(\"mpi@2.2\"))\n assert not any(s.intersects(\"mpich@:1\") for s in repo.providers_for(\"mpi@2\"))\n assert not any(s.intersects(\"mpich@:1\") for s in repo.providers_for(\"mpi@3\"))\n assert not any(s.intersects(\"mpich2\") for s in repo.providers_for(\"mpi@3\"))\n\n def test_provides_handles_multiple_providers_of_same_version(self):\n \"\"\" \"\"\"\n providers = spack.repo.PATH.providers_for(\"mpi@3.0\")\n\n # Note that providers are repo-specific, so we don't misinterpret\n # providers, but vdeps are not namespace-specific, so we can\n # associate vdeps across repos.\n assert Spec(\"builtin_mock.multi-provider-mpi@1.10.3\") in providers\n assert Spec(\"builtin_mock.multi-provider-mpi@1.10.2\") in providers\n assert Spec(\"builtin_mock.multi-provider-mpi@1.10.1\") in providers\n assert Spec(\"builtin_mock.multi-provider-mpi@1.10.0\") in providers\n assert Spec(\"builtin_mock.multi-provider-mpi@1.8.8\") in providers\n\n def test_different_compilers_get_different_flags(\n self, mutable_config, clang12_with_flags, gcc11_with_flags\n ):\n \"\"\"Tests that nodes get the flags of the associated compiler.\"\"\"\n mutable_config.set(\n \"packages\",\n {\n \"llvm\": {\"externals\": [clang12_with_flags]},\n \"gcc\": {\"externals\": [gcc11_with_flags]},\n },\n )\n t = spack.vendor.archspec.cpu.host().family\n client = spack.concretize.concretize_one(\n Spec(\n f\"cmake-client platform=test os=redhat6 target={t} %gcc@11.1.0\"\n f\" ^cmake platform=test os=redhat6 target={t} %clang@12.2.0\"\n )\n )\n cmake = client[\"cmake\"]\n assert set(client.compiler_flags[\"cflags\"]) == {\"-O0\", \"-g\"}\n assert set(cmake.compiler_flags[\"cflags\"]) == {\"-O3\"}\n assert set(client.compiler_flags[\"fflags\"]) == {\"-O0\", \"-g\"}\n assert not set(cmake.compiler_flags[\"fflags\"])\n\n @pytest.mark.regression(\"9908\")\n def test_spec_flags_maintain_order(self, mutable_config, gcc11_with_flags):\n \"\"\"Tests that Spack assembles flags in a consistent way (i.e. with the same ordering),\n for successive concretizations.\n \"\"\"\n mutable_config.set(\"packages\", {\"gcc\": {\"externals\": [gcc11_with_flags]}})\n spec_str = \"libelf os=redhat6 %gcc@11.1.0\"\n for _ in range(3):\n s = spack.concretize.concretize_one(spec_str)\n assert all(\n s.compiler_flags[x] == [\"-O0\", \"-g\"] for x in (\"cflags\", \"cxxflags\", \"fflags\")\n )\n\n @pytest.mark.parametrize(\n \"spec_str,expected,not_expected\",\n [\n # Simple flag propagation from the root\n (\"hypre cflags=='-g' ^openblas\", [\"hypre cflags='-g'\", \"^openblas cflags='-g'\"], []),\n (\n \"hypre cflags='-g' ^openblas\",\n [\"hypre cflags='-g'\", \"^openblas\"],\n [\"^openblas cflags='-g'\"],\n ),\n # Setting a flag overrides propagation\n (\n \"hypre cflags=='-g' ^openblas cflags='-O3'\",\n [\"hypre cflags='-g'\", \"^openblas cflags='-O3'\"],\n [\"^openblas cflags='-g'\"],\n ),\n # Propagation doesn't go across build dependencies\n (\n \"cmake-client cflags=='-O2 -g'\",\n [\"cmake-client cflags=='-O2 -g'\", \"^cmake\"],\n [\"cmake cflags=='-O2 -g'\"],\n ),\n ],\n )\n def test_compiler_flag_propagation(self, spec_str, expected, not_expected):\n root = spack.concretize.concretize_one(spec_str)\n\n for constraint in expected:\n assert root.satisfies(constraint)\n\n for constraint in not_expected:\n assert not root.satisfies(constraint)\n\n def test_mixing_compilers_only_affects_subdag(self):\n \"\"\"Tests that, when we mix compilers, the one with lower penalty is used for nodes\n where the compiler is not forced.\n \"\"\"\n spec = spack.concretize.concretize_one(\"dt-diamond%clang ^dt-diamond-bottom%gcc\")\n # This is intended to traverse the \"root\" unification set, and check compilers\n # on the nodes in the set\n for x in spec.traverse(deptype=(\"link\", \"run\")):\n if \"c\" not in x or not x.name.startswith(\"dt-diamond\"):\n continue\n expected_gcc = x.name != \"dt-diamond\"\n assert bool(x.dependencies(name=\"llvm\", deptype=\"build\")) is not expected_gcc, x.tree()\n assert bool(x.dependencies(name=\"gcc\", deptype=\"build\")) is expected_gcc\n assert x.satisfies(\"%clang\") is not expected_gcc\n assert x.satisfies(\"%gcc\") is expected_gcc\n\n def test_disable_mixing_prevents_mixing(self):\n with spack.config.override(\"concretizer\", {\"compiler_mixing\": False}):\n with pytest.raises(spack.error.UnsatisfiableSpecError):\n spack.concretize.concretize_one(\"dt-diamond%clang ^dt-diamond-bottom%gcc\")\n\n def test_disable_mixing_is_per_language(self):\n with spack.config.override(\"concretizer\", {\"compiler_mixing\": False}):\n spack.concretize.concretize_one(\"openblas %c=llvm %fortran=gcc\")\n\n def test_disable_mixing_override_by_package(self):\n with spack.config.override(\"concretizer\", {\"compiler_mixing\": [\"dt-diamond-bottom\"]}):\n root = spack.concretize.concretize_one(\"dt-diamond%clang ^dt-diamond-bottom%gcc\")\n assert root.satisfies(\"%clang\")\n assert root[\"dt-diamond-bottom\"].satisfies(\"%gcc\")\n assert root[\"dt-diamond-left\"].satisfies(\"%clang\")\n\n with pytest.raises(spack.error.UnsatisfiableSpecError):\n spack.concretize.concretize_one(\"dt-diamond%clang ^dt-diamond-left%gcc\")\n\n def test_disable_mixing_reuse(self, fake_db_install):\n # Install a spec\n left = spack.concretize.concretize_one(\"dt-diamond-left %gcc\")\n fake_db_install(left)\n assert left.satisfies(\"%c=gcc\")\n lefthash = left.dag_hash()[:7]\n\n # Check if mixing works when it's allowed\n spack.concretize.concretize_one(f\"dt-diamond%clang ^/{lefthash}\")\n\n # Now try to use it with compiler mixing disabled\n with spack.config.override(\"concretizer\", {\"compiler_mixing\": False}):\n with pytest.raises(spack.error.UnsatisfiableSpecError):\n spack.concretize.concretize_one(f\"dt-diamond%clang ^/{lefthash}\")\n\n # Should be able to reuse if the compilers match\n spack.concretize.concretize_one(f\"dt-diamond%gcc ^/{lefthash}\")\n\n def test_disable_mixing_reuse_and_built(self, fake_db_install):\n r\"\"\"In this case we have\n\n x\n |\\\n y z\n\n Where y is a link dependency and z is a build dependency.\n We install y with a compiler c1, and we make sure we cannot\n ask for `x%c2 ^z%c1 ^/y\n\n This looks similar to `test_disable_mixing_reuse`. But the\n compiler nodes are handled differently in this case: this\n is the only test that explicitly exercises compiler unmixing\n rule #2.\n \"\"\"\n dep1 = spack.concretize.concretize_one(\"libdwarf %gcc\")\n fake_db_install(dep1)\n assert dep1.satisfies(\"%c=gcc\")\n dep1hash = dep1.dag_hash()[:7]\n\n spack.concretize.concretize_one(f\"mixing-parent%clang ^cmake%gcc ^/{dep1hash}\")\n\n with spack.config.override(\"concretizer\", {\"compiler_mixing\": False}):\n with pytest.raises(spack.error.UnsatisfiableSpecError, match=\"mixing is disabled\"):\n spack.concretize.concretize_one(f\"mixing-parent%clang ^cmake%gcc ^/{dep1hash}\")\n\n def test_disable_mixing_allow_compiler_link(self):\n \"\"\"Check if we can use a compiler when mixing is disabled, and\n still depend on a separate compiler package (in the latter case\n not using it as a compiler but rather for some utility it\n provides).\n \"\"\"\n with spack.config.override(\"concretizer\", {\"compiler_mixing\": False}):\n x = spack.concretize.concretize_one(\"llvm-client%gcc\")\n assert x.satisfies(\"%cxx=gcc\")\n assert x.satisfies(\"%c=gcc\")\n assert \"llvm\" in x\n\n def test_compiler_run_dep_link_dep_not_forced(self, temporary_store):\n \"\"\"When a compiler is used as a pure build dependency, its transitive run-reachable deps\n are unified in the build environment, but their pure link-type dependencies must NOT be\n forced onto the package.\n\n Scenario: compiler-with-deps has a run+link dep on binutils-for-test, which has a pure\n link dep on zlib. A package that depends on zlib and uses compiler-with-deps as its C\n compiler should be free to pick its own zlib (here: zlib@1.2.8) independently of the\n toolchain's zlib (zlib@1.2.11). Without the fix the imposed hash from binutils-for-test\n forces the toolchain version onto the package, causing a conflict.\n \"\"\"\n # Pre-install the compiler with its transitive deps binutils-for-test and zlib@1.2.11\n compiler = spack.concretize.concretize_one(\"compiler-with-deps ^zlib@1.2.11\")\n assert compiler[\"zlib\"].satisfies(\"@1.2.11\")\n PackageInstaller([compiler.package], fake=True, explicit=True).install()\n\n # Concretize a package that depends on a different zlib from its compiler's toolchain.\n pkg = spack.concretize.concretize_one(\n \"pkg-with-zlib-dep %c=compiler-with-deps ^zlib@1.2.8\"\n )\n\n assert pkg[\"zlib\"].satisfies(\"@1.2.8\")\n\n def test_disable_mixing_env(\n self, mutable_mock_env_path, tmp_path: pathlib.Path, mock_packages, mutable_config\n ):\n spack_yaml = tmp_path / ev.manifest_name\n spack_yaml.write_text(\n \"\"\"\\\nspack:\n specs:\n - dt-diamond%gcc\n - dt-diamond%clang\n concretizer:\n compiler_mixing: false\n unify: when_possible\n\"\"\"\n )\n\n with ev.Environment(tmp_path) as e:\n e.concretize()\n for root in e.roots():\n if root.satisfies(\"%gcc\"):\n assert root[\"dt-diamond-left\"].satisfies(\"%gcc\")\n assert root[\"dt-diamond-bottom\"].satisfies(\"%gcc\")\n else:\n assert root[\"dt-diamond-left\"].satisfies(\"%llvm\")\n assert root[\"dt-diamond-bottom\"].satisfies(\"%llvm\")\n\n def test_compiler_inherited_upwards(self):\n spec = spack.concretize.concretize_one(\"dt-diamond ^dt-diamond-bottom%clang\")\n for x in spec.traverse(deptype=(\"link\", \"run\")):\n if \"c\" not in x:\n continue\n assert x.satisfies(\"%clang\")\n\n def test_architecture_deep_inheritance(self, mock_targets, compiler_factory):\n \"\"\"Make sure that indirect dependencies receive architecture\n information from the root even when partial architecture information\n is provided by an intermediate dependency.\n \"\"\"\n cnl_compiler = compiler_factory(\n spec=\"gcc@4.5.0 os=CNL languages:=c,c++,fortran target=nocona\"\n )\n with spack.config.override(\"packages\", {\"gcc\": {\"externals\": [cnl_compiler]}}):\n spec_str = \"mpileaks os=CNL target=nocona %gcc@4.5.0 ^dyninst os=CNL ^callpath os=CNL\"\n spec = spack.concretize.concretize_one(spec_str)\n for s in spec.traverse(root=False, deptype=(\"link\", \"run\")):\n if s.external:\n continue\n assert s.architecture.target == spec.architecture.target\n\n def test_compiler_flags_from_user_are_grouped(self):\n spec = Spec('pkg-a cflags=\"-O -foo-flag foo-val\" platform=test %gcc')\n spec = spack.concretize.concretize_one(spec)\n cflags = spec.compiler_flags[\"cflags\"]\n assert any(x == \"-foo-flag foo-val\" for x in cflags)\n\n def concretize_multi_provider(self):\n s = Spec(\"mpileaks ^multi-provider-mpi@3.0\")\n s = spack.concretize.concretize_one(s)\n assert s[\"mpi\"].version == ver(\"1.10.3\")\n\n def test_concretize_dependent_with_singlevalued_variant_type(self):\n s = Spec(\"singlevalue-variant-dependent-type\")\n s = spack.concretize.concretize_one(s)\n\n @pytest.mark.parametrize(\"spec,version\", [(\"dealii\", \"develop\"), (\"xsdk\", \"0.4.0\")])\n def concretize_difficult_packages(self, a, b):\n \"\"\"Test a couple of large packages that are often broken due\n to current limitations in the concretizer\"\"\"\n s = Spec(a + \"@\" + b)\n s = spack.concretize.concretize_one(s)\n assert s[a].version == ver(b)\n\n def test_concretize_two_virtuals(self):\n \"\"\"Test a package with multiple virtual dependencies.\"\"\"\n spack.concretize.concretize_one(\"hypre\")\n\n def test_concretize_two_virtuals_with_one_bound(self, mutable_mock_repo):\n \"\"\"Test a package with multiple virtual dependencies and one preset.\"\"\"\n spack.concretize.concretize_one(\"hypre ^openblas\")\n\n def test_concretize_two_virtuals_with_two_bound(self):\n \"\"\"Test a package with multiple virtual deps and two of them preset.\"\"\"\n spack.concretize.concretize_one(\"hypre ^netlib-lapack\")\n\n def test_concretize_two_virtuals_with_dual_provider(self):\n \"\"\"Test a package with multiple virtual dependencies and force a provider\n that provides both.\n \"\"\"\n spack.concretize.concretize_one(\"hypre ^openblas-with-lapack\")\n\n @pytest.mark.parametrize(\"max_dupes_default\", [1, 2, 3])\n def test_concretize_two_virtuals_with_dual_provider_and_a_conflict(\n self, max_dupes_default, mutable_config\n ):\n \"\"\"Test a package with multiple virtual dependencies and force a\n provider that provides both, and another conflicting package that\n provides one.\n \"\"\"\n mutable_config.set(\"concretizer:duplicates:max_dupes:default\", max_dupes_default)\n s = Spec(\"hypre ^openblas-with-lapack ^netlib-lapack\")\n with pytest.raises(spack.error.SpackError):\n spack.concretize.concretize_one(s)\n\n @pytest.mark.parametrize(\n \"spec_str,expected_propagation\",\n [\n # Propagates past a node that doesn't have the variant\n (\"hypre~~shared ^openblas\", [(\"hypre\", \"~shared\"), (\"openblas\", \"~shared\")]),\n # Propagates from root node to all nodes\n (\n \"ascent~~shared +adios2\",\n [(\"ascent\", \"~shared\"), (\"adios2\", \"~shared\"), (\"bzip2\", \"~shared\")],\n ),\n # Propagate from a node that is not the root node\n (\n \"ascent +adios2 ^adios2~~shared\",\n [(\"ascent\", \"+shared\"), (\"adios2\", \"~shared\"), (\"bzip2\", \"~shared\")],\n ),\n ],\n )\n def test_concretize_propagate_disabled_variant(self, spec_str, expected_propagation):\n \"\"\"Tests various patterns of boolean variant propagation\"\"\"\n spec = spack.concretize.concretize_one(spec_str)\n for key, expected_satisfies in expected_propagation:\n spec[key].satisfies(expected_satisfies)\n\n def test_concretize_propagate_variant_not_dependencies(self):\n \"\"\"Test that when propagating a variant it is not propagated to dependencies that\n do not have that variant\"\"\"\n spec = Spec(\"quantum-espresso~~invino\")\n spec = spack.concretize.concretize_one(spec)\n\n for dep in spec.traverse(root=False):\n assert \"invino\" not in dep.variants.keys()\n\n def test_concretize_propagate_variant_exclude_dependency_fail(self):\n \"\"\"Tests that a propagating variant cannot be allowed to be excluded by any of\n the source package's dependencies\"\"\"\n spec = Spec(\"hypre ~~shared ^openblas +shared\")\n with pytest.raises(spack.error.UnsatisfiableSpecError):\n spec = spack.concretize.concretize_one(spec)\n\n def test_concretize_propagate_same_variant_from_direct_dep_fail(self):\n \"\"\"Test that when propagating a variant from the source package and a direct\n dependency also propagates the same variant with a different value. Raises error\"\"\"\n spec = Spec(\"ascent +adios2 ++shared ^adios2 ~~shared\")\n with pytest.raises(spack.error.UnsatisfiableSpecError):\n spec = spack.concretize.concretize_one(spec)\n\n def test_concretize_propagate_same_variant_in_dependency_fail(self):\n \"\"\"Test that when propagating a variant from the source package, none of it's\n dependencies can propagate that variant with a different value. Raises error.\"\"\"\n spec = Spec(\"ascent +adios2 ++shared ^bzip2 ~~shared\")\n with pytest.raises(spack.error.UnsatisfiableSpecError):\n spec = spack.concretize.concretize_one(spec)\n\n def test_concretize_propagate_same_variant_virtual_dependency_fail(self):\n \"\"\"Test that when propagating a variant from the source package and a direct\n dependency (that is a virtual pkg) also propagates the same variant with a\n different value. Raises error\"\"\"\n spec = Spec(\"hypre ++shared ^openblas ~~shared\")\n with pytest.raises(spack.error.UnsatisfiableSpecError):\n spec = spack.concretize.concretize_one(spec)\n\n def test_concretize_propagate_same_variant_multiple_sources_diamond_dep_fail(self):\n \"\"\"Test that fails when propagating the same variant with different values from multiple\n sources that share a dependency\"\"\"\n spec = Spec(\"parent-foo-bar ^dependency-foo-bar++bar ^direct-dep-foo-bar~~bar\")\n with pytest.raises(spack.error.UnsatisfiableSpecError):\n spec = spack.concretize.concretize_one(spec)\n\n def test_concretize_propagate_specified_variant(self):\n \"\"\"Test that only the specified variant is propagated to the dependencies\"\"\"\n spec = Spec(\"parent-foo-bar ~~foo\")\n spec = spack.concretize.concretize_one(spec)\n\n assert spec.satisfies(\"^dependency-foo-bar~foo\")\n assert spec.satisfies(\"^second-dependency-foo-bar-fee~foo\")\n assert spec.satisfies(\"^direct-dep-foo-bar~foo\")\n\n assert not spec.satisfies(\"^dependency-foo-bar+bar\")\n assert not spec.satisfies(\"^second-dependency-foo-bar-fee+bar\")\n assert not spec.satisfies(\"^direct-dep-foo-bar+bar\")\n\n def test_concretize_propagate_one_variant(self):\n \"\"\"Test that you can specify to propagate one variant and not all\"\"\"\n spec = Spec(\"parent-foo-bar ++bar ~foo\")\n spec = spack.concretize.concretize_one(spec)\n\n assert spec.satisfies(\"~foo\") and not spec.satisfies(\"^dependency-foo-bar~foo\")\n assert spec.satisfies(\"+bar\") and spec.satisfies(\"^dependency-foo-bar+bar\")\n\n def test_concretize_propagate_through_first_level_deps(self):\n \"\"\"Test that boolean valued variants can be propagated past first level\n dependencies even if the first level dependency does have the variant\"\"\"\n spec = Spec(\"parent-foo-bar-fee ++fee\")\n spec = spack.concretize.concretize_one(spec)\n\n assert spec.satisfies(\"+fee\") and not spec.satisfies(\"dependency-foo-bar+fee\")\n assert spec.satisfies(\"^second-dependency-foo-bar-fee+fee\")\n\n def test_concretize_propagate_multiple_variants(self):\n \"\"\"Test that multiple boolean valued variants can be propagated from\n the same source package\"\"\"\n spec = Spec(\"parent-foo-bar-fee ~~foo ++bar\")\n spec = spack.concretize.concretize_one(spec)\n\n assert spec.satisfies(\"~foo\") and spec.satisfies(\"+bar\")\n assert spec.satisfies(\"^dependency-foo-bar ~foo +bar\")\n assert spec.satisfies(\"^second-dependency-foo-bar-fee ~foo +bar\")\n\n def test_concretize_propagate_multiple_variants_mulitple_sources(self):\n \"\"\"Test the propagates multiple different variants for multiple sources\n in a diamond dependency\"\"\"\n spec = Spec(\"parent-foo-bar ^dependency-foo-bar++bar ^direct-dep-foo-bar~~foo\")\n spec = spack.concretize.concretize_one(spec)\n\n assert spec.satisfies(\"^second-dependency-foo-bar-fee+bar\")\n assert spec.satisfies(\"^second-dependency-foo-bar-fee~foo\")\n assert not spec.satisfies(\"^dependency-foo-bar~foo\")\n assert not spec.satisfies(\"^direct-dep-foo-bar+bar\")\n\n def test_concretize_propagate_single_valued_variant(self):\n \"\"\"Test propagation for single valued variants\"\"\"\n spec = Spec(\"multivalue-variant libs==static\")\n spec = spack.concretize.concretize_one(spec)\n\n assert spec.satisfies(\"libs=static\")\n assert spec.satisfies(\"^pkg-a libs=static\")\n\n def test_concretize_propagate_multivalue_variant(self):\n \"\"\"Test that multivalue variants are propagating the specified value(s)\n to their dependencies. The dependencies should not have the default value\"\"\"\n spec = Spec(\"multivalue-variant foo==baz,fee\")\n spec = spack.concretize.concretize_one(spec)\n\n assert spec.satisfies(\"^pkg-a foo=baz,fee\")\n assert spec.satisfies(\"^pkg-b foo=baz,fee\")\n assert not spec.satisfies(\"^pkg-a foo=bar\")\n assert not spec.satisfies(\"^pkg-b foo=bar\")\n\n def test_concretize_propagate_multiple_multivalue_variant(self):\n \"\"\"Tests propagating the same mulitvalued variant from different sources allows\n the dependents to accept all propagated values\"\"\"\n spec = Spec(\"multivalue-variant foo==bar ^pkg-a foo==baz\")\n spec = spack.concretize.concretize_one(spec)\n\n assert spec.satisfies(\"multivalue-variant foo=bar\")\n assert spec.satisfies(\"^pkg-a foo=bar,baz\")\n assert spec.satisfies(\"^pkg-b foo=bar,baz\")\n\n def test_concretize_propagate_variant_not_in_source(self):\n \"\"\"Test that variant is still propagated even if the source pkg\n doesn't have the variant\"\"\"\n spec = Spec(\"callpath++debug\")\n spec = spack.concretize.concretize_one(spec)\n\n assert spec.satisfies(\"^mpich+debug\")\n assert not spec.satisfies(\"callpath+debug\")\n assert not spec.satisfies(\"^dyninst+debug\")\n\n def test_concretize_propagate_variant_multiple_deps_not_in_source(self):\n \"\"\"Test that a variant can be propagated to multiple dependencies\n when the variant is not in the source package\"\"\"\n spec = Spec(\"netlib-lapack++shared\")\n spec = spack.concretize.concretize_one(spec)\n\n assert spec.satisfies(\"^openblas+shared\")\n assert spec.satisfies(\"^perl+shared\")\n assert not spec.satisfies(\"netlib-lapack+shared\")\n\n def test_concretize_propagate_variant_second_level_dep_not_in_source(self):\n \"\"\"Test that a variant can be propagated past first level dependencies\n when the variant is not in the source package or any of the first level\n dependencies\"\"\"\n spec = Spec(\"parent-foo-bar ++fee\")\n spec = spack.concretize.concretize_one(spec)\n\n assert spec.satisfies(\"^second-dependency-foo-bar-fee +fee\")\n assert not spec.satisfies(\"parent-foo-bar +fee\")\n\n def test_no_matching_compiler_specs(self):\n s = Spec(\"pkg-a %gcc@0.0.0\")\n with pytest.raises(spack.solver.asp.InvalidVersionError):\n spack.concretize.concretize_one(s)\n\n def test_no_compilers_for_arch(self):\n s = Spec(\"pkg-a arch=linux-rhel0-x86_64\")\n with pytest.raises(spack.error.SpackError):\n s = spack.concretize.concretize_one(s)\n\n def test_virtual_is_fully_expanded_for_callpath(self):\n # force dependence on fake \"zmpi\" by asking for MPI 10.0\n spec = Spec(\"callpath ^mpi@10.0\")\n assert len(spec.dependencies(name=\"mpi\")) == 1\n assert \"fake\" not in spec\n\n spec = spack.concretize.concretize_one(spec)\n assert len(spec.dependencies(name=\"zmpi\")) == 1\n assert all(not d.dependencies(name=\"mpi\") for d in spec.traverse())\n assert all(x in spec for x in (\"zmpi\", \"mpi\"))\n\n edges_to_zmpi = spec.edges_to_dependencies(name=\"zmpi\")\n assert len(edges_to_zmpi) == 1\n assert \"fake\" in edges_to_zmpi[0].spec\n\n def test_virtual_is_fully_expanded_for_mpileaks(self):\n spec = Spec(\"mpileaks ^mpi@10.0\")\n assert len(spec.dependencies(name=\"mpi\")) == 1\n assert \"fake\" not in spec\n\n spec = spack.concretize.concretize_one(spec)\n assert len(spec.dependencies(name=\"zmpi\")) == 1\n assert len(spec.dependencies(name=\"callpath\")) == 1\n\n callpath = spec.dependencies(name=\"callpath\")[0]\n assert len(callpath.dependencies(name=\"zmpi\")) == 1\n\n zmpi = callpath.dependencies(name=\"zmpi\")[0]\n assert len(zmpi.dependencies(name=\"fake\")) == 1\n\n assert all(not d.dependencies(name=\"mpi\") for d in spec.traverse())\n assert all(x in spec for x in (\"zmpi\", \"mpi\"))\n\n @pytest.mark.parametrize(\n \"spec_str,expected,not_expected\",\n [\n # clang (llvm~flang) only provides C, and C++ compilers, while gcc has also fortran\n #\n # If we ask mpileaks%clang, then %gcc must be used for fortran, and since\n # %gcc is preferred to clang in config, it will be used for most nodes\n (\n \"mpileaks %clang\",\n {\"mpileaks\": \"%clang\", \"libdwarf\": \"%gcc\", \"libelf\": \"%gcc\"},\n {\"libdwarf\": \"%clang\", \"libelf\": \"%clang\"},\n ),\n (\n \"mpileaks %clang@:15.0.0\",\n {\"mpileaks\": \"%clang\", \"libdwarf\": \"%gcc\", \"libelf\": \"%gcc\"},\n {\"libdwarf\": \"%clang\", \"libelf\": \"%clang\"},\n ),\n (\n \"mpileaks %gcc\",\n {\"mpileaks\": \"%gcc\", \"libdwarf\": \"%gcc\", \"libelf\": \"%gcc\"},\n {\"mpileaks\": \"%clang\", \"libdwarf\": \"%clang\", \"libelf\": \"%clang\"},\n ),\n (\n \"mpileaks %gcc@10.2.1\",\n {\"mpileaks\": \"%gcc\", \"libdwarf\": \"%gcc\", \"libelf\": \"%gcc\"},\n {\"mpileaks\": \"%clang\", \"libdwarf\": \"%clang\", \"libelf\": \"%clang\"},\n ),\n # dyninst doesn't require fortran, so %clang is propagated\n (\n \"dyninst %clang\",\n {\"dyninst\": \"%clang\", \"libdwarf\": \"%clang\", \"libelf\": \"%clang\"},\n {\"libdwarf\": \"%gcc\", \"libelf\": \"%gcc\"},\n ),\n ],\n )\n def test_compiler_inheritance(self, spec_str, expected, not_expected):\n \"\"\"Spack tries to propagate compilers as much as possible, but prefers using a single\n toolchain on a node, rather than mixing them.\n \"\"\"\n spec = spack.concretize.concretize_one(spec_str)\n for name, constraint in expected.items():\n assert spec[name].satisfies(constraint)\n\n for name, constraint in not_expected.items():\n assert not spec[name].satisfies(constraint)\n\n def test_external_package(self):\n \"\"\"Tests that an external is preferred, if present, and that it does not\n have dependencies.\n \"\"\"\n spec = spack.concretize.concretize_one(\"externaltool\")\n assert spec.external_path == os.path.sep + os.path.join(\"path\", \"to\", \"external_tool\")\n assert not spec.dependencies()\n\n def test_nobuild_package(self):\n \"\"\"Test that a non-buildable package raise an error if no specs\n in packages.yaml are compatible with the request.\n \"\"\"\n spec = Spec(\"externaltool%clang\")\n with pytest.raises(spack.error.SpecError):\n spec = spack.concretize.concretize_one(spec)\n\n def test_external_and_virtual(self, mutable_config):\n mutable_config.set(\"packages:stuff\", {\"buildable\": False})\n spec = spack.concretize.concretize_one(\"externaltest\")\n assert spec[\"externaltool\"].external_path == os.path.sep + os.path.join(\n \"path\", \"to\", \"external_tool\"\n )\n # \"stuff\" is a virtual provided by externalvirtual\n assert spec[\"stuff\"].external_path == os.path.sep + os.path.join(\n \"path\", \"to\", \"external_virtual_clang\"\n )\n\n def test_compiler_child(self):\n s = Spec(\"mpileaks target=x86_64 %clang ^dyninst%gcc\")\n s = spack.concretize.concretize_one(s)\n assert s[\"mpileaks\"].satisfies(\"%clang\")\n assert s[\"dyninst\"].satisfies(\"%gcc\")\n\n def test_conflicts_in_spec(self, conflict_spec):\n s = Spec(conflict_spec)\n with pytest.raises(spack.error.SpackError):\n s = spack.concretize.concretize_one(s)\n\n def test_conflicts_show_cores(self, conflict_spec, monkeypatch):\n s = Spec(conflict_spec)\n with pytest.raises(spack.error.SpackError) as e:\n s = spack.concretize.concretize_one(s)\n\n assert \"conflict\" in e.value.message\n\n def test_conflict_in_all_directives_true(self):\n s = Spec(\"when-directives-true\")\n with pytest.raises(spack.error.SpackError):\n s = spack.concretize.concretize_one(s)\n\n @pytest.mark.parametrize(\"spec_str\", [\"unsat-provider@1.0+foo\"])\n def test_no_conflict_in_external_specs(self, spec_str):\n # Modify the configuration to have the spec with conflict\n # registered as an external\n ext = Spec(spec_str)\n data = {\"externals\": [{\"spec\": spec_str, \"prefix\": \"/fake/path\"}]}\n spack.config.set(\"packages::{0}\".format(ext.name), data)\n ext = spack.concretize.concretize_one(ext) # failure raises exception\n\n def test_regression_issue_4492(self):\n # Constructing a spec which has no dependencies, but is otherwise\n # concrete is kind of difficult. What we will do is to concretize\n # a spec, and then modify it to have no dependency and reset the\n # cache values.\n\n s = Spec(\"mpileaks\")\n s = spack.concretize.concretize_one(s)\n\n # Check that now the Spec is concrete, store the hash\n assert s.concrete\n\n # Remove the dependencies and reset caches\n s.clear_dependencies()\n s._concrete = False\n\n assert not s.concrete\n\n @pytest.mark.regression(\"7239\")\n def test_regression_issue_7239(self):\n # Constructing a SpecBuildInterface from another SpecBuildInterface\n # results in an inconsistent MRO\n\n # Normal Spec\n s = Spec(\"mpileaks\")\n s = spack.concretize.concretize_one(s)\n\n assert spack.llnl.util.lang.ObjectWrapper not in s.__class__.__mro__\n\n # Spec wrapped in a build interface\n build_interface = s[\"mpileaks\"]\n assert spack.llnl.util.lang.ObjectWrapper in build_interface.__class__.__mro__\n\n # Mimics asking the build interface from a build interface\n build_interface = s[\"mpileaks\"][\"mpileaks\"]\n assert spack.llnl.util.lang.ObjectWrapper in build_interface.__class__.__mro__\n\n @pytest.mark.regression(\"7705\")\n def test_regression_issue_7705(self):\n # spec.package.provides(name) doesn't account for conditional\n # constraints in the concretized spec\n s = Spec(\"simple-inheritance~openblas\")\n s = spack.concretize.concretize_one(s)\n\n assert not s.package.provides(\"lapack\")\n\n @pytest.mark.regression(\"7941\")\n def test_regression_issue_7941(self):\n # The string representation of a spec containing\n # an explicit multi-valued variant and a dependency\n # might be parsed differently than the originating spec\n s = Spec(\"pkg-a foobar=bar ^pkg-b\")\n t = Spec(str(s))\n\n s = spack.concretize.concretize_one(s)\n t = spack.concretize.concretize_one(t)\n\n assert s.dag_hash() == t.dag_hash()\n\n @pytest.mark.parametrize(\n \"abstract_specs\",\n [\n # Establish a baseline - concretize a single spec\n (\"mpileaks\",),\n # When concretized together with older version of callpath\n # and dyninst it uses those older versions\n (\"mpileaks\", \"callpath@0.9\", \"dyninst@8.1.1\"),\n # Handle recursive syntax within specs\n (\"mpileaks\", \"callpath@0.9 ^dyninst@8.1.1\", \"dyninst\"),\n # Test specs that have overlapping dependencies but are not\n # one a dependency of the other\n (\"mpileaks\", \"direct-mpich\"),\n ],\n )\n def test_simultaneous_concretization_of_specs(self, abstract_specs):\n abstract_specs = [Spec(x) for x in abstract_specs]\n concrete_specs = spack.concretize._concretize_specs_together(abstract_specs)\n\n # Check there's only one configuration of each package in the DAG\n names = set(\n dep.name for spec in concrete_specs for dep in spec.traverse(deptype=(\"link\", \"run\"))\n )\n for name in names:\n name_specs = set(spec[name] for spec in concrete_specs if name in spec)\n assert len(name_specs) == 1\n\n # Check that there's at least one Spec that satisfies the\n # initial abstract request\n for aspec in abstract_specs:\n assert any(cspec.satisfies(aspec) for cspec in concrete_specs)\n\n # Make sure the concrete spec are top-level specs with no dependents\n for spec in concrete_specs:\n assert not spec.dependents()\n\n @pytest.mark.parametrize(\"spec\", [\"noversion\", \"noversion-bundle\"])\n def test_noversion_pkg(self, spec):\n \"\"\"Test concretization failures for no-version packages.\"\"\"\n with pytest.raises(spack.error.SpackError):\n spack.concretize.concretize_one(spec)\n\n @pytest.mark.not_on_windows(\"Not supported on Windows (yet)\")\n @pytest.mark.parametrize(\n \"spec,compiler_spec,best_achievable\",\n [\n (\n \"mpileaks%gcc@=4.4.7 ^dyninst@=10.2.1 target=x86_64:\",\n \"gcc@4.4.7 languages=c,c++,fortran\",\n \"core2\",\n ),\n (\"mpileaks target=x86_64: %gcc@=4.8\", \"gcc@4.8 languages=c,c++,fortran\", \"haswell\"),\n (\n \"mpileaks target=x86_64: %gcc@=5.3.0\",\n \"gcc@5.3.0 languages=c,c++,fortran\",\n \"broadwell\",\n ),\n ],\n )\n @pytest.mark.regression(\"13361\", \"20537\")\n @pytest.mark.usefixtures(\"mock_targets\")\n def test_adjusting_default_target_based_on_compiler(\n self, spec, compiler_spec, best_achievable, current_host, compiler_factory, mutable_config\n ):\n best_achievable = spack.vendor.archspec.cpu.TARGETS[best_achievable]\n expected = best_achievable if best_achievable < current_host else current_host\n mutable_config.set(\n \"packages\", {\"gcc\": {\"externals\": [compiler_factory(spec=f\"{compiler_spec}\")]}}\n )\n s = spack.concretize.concretize_one(spec)\n assert str(s.architecture.target) == str(expected)\n\n @pytest.mark.parametrize(\n \"constraint,expected\", [(\"%gcc@10.2\", \"@=10.2.1\"), (\"%gcc@10.2:\", \"@=10.2.1\")]\n )\n def test_compiler_version_matches_any_entry_in_packages_yaml(self, constraint, expected):\n # The behavior here has changed since #8735 / #14730. Now %gcc@10.2 is an abstract\n # compiler spec, and it should first find a matching compiler gcc@=10.2.1\n s = spack.concretize.concretize_one(f\"mpileaks {constraint}\")\n gcc_deps = s.dependencies(name=\"gcc\", deptype=\"build\")\n assert len(gcc_deps) == 1\n assert gcc_deps[0].satisfies(expected)\n\n def test_concretize_anonymous(self):\n with pytest.raises(spack.error.SpackError):\n s = Spec(\"+variant\")\n s = spack.concretize.concretize_one(s)\n\n @pytest.mark.parametrize(\"spec_str\", [\"mpileaks ^%gcc\", \"mpileaks ^cflags=-g\"])\n def test_concretize_anonymous_dep(self, spec_str):\n with pytest.raises(spack.error.SpackError):\n s = Spec(spec_str)\n s = spack.concretize.concretize_one(s)\n\n @pytest.mark.parametrize(\n \"spec_str,expected_str\",\n [\n # Unconstrained versions select default compiler (gcc@10.2.1)\n (\"bowtie@1.4.0\", \"%gcc@10.2.1\"),\n # Version with conflicts and no valid gcc select another compiler\n (\"bowtie@1.3.0\", \"%clang@15.0.0\"),\n # If a higher gcc is available, with a worse os, still prefer that,\n # assuming the two operating systems are compatible\n (\"bowtie@1.2.2 %gcc\", \"%gcc@11.1.0\"),\n ],\n )\n def test_compiler_conflicts_in_package_py(\n self, spec_str, expected_str, gcc11_with_flags, mutable_config\n ):\n mutable_config.set(\n \"concretizer:os_compatible\", {\"debian6\": [\"redhat6\"], \"redhat6\": [\"debian6\"]}\n )\n with spack.config.override(\"packages\", {\"gcc\": {\"externals\": [gcc11_with_flags]}}):\n s = spack.concretize.concretize_one(spec_str)\n assert s.satisfies(expected_str)\n\n @pytest.mark.parametrize(\n \"spec_str,expected,unexpected\",\n [\n (\"conditional-variant-pkg@1.0\", [\"two_whens\"], [\"version_based\", \"variant_based\"]),\n (\"conditional-variant-pkg@2.0\", [\"version_based\", \"variant_based\"], [\"two_whens\"]),\n (\n \"conditional-variant-pkg@2.0~version_based\",\n [\"version_based\"],\n [\"variant_based\", \"two_whens\"],\n ),\n (\n \"conditional-variant-pkg@2.0+version_based+variant_based\",\n [\"version_based\", \"variant_based\", \"two_whens\"],\n [],\n ),\n ],\n )\n def test_conditional_variants(self, spec_str, expected, unexpected):\n s = spack.concretize.concretize_one(spec_str)\n\n for var in expected:\n assert s.satisfies(\"%s=*\" % var)\n for var in unexpected:\n assert not s.satisfies(\"%s=*\" % var)\n\n @pytest.mark.parametrize(\n \"bad_spec\",\n [\n \"@1.0~version_based\",\n \"@1.0+version_based\",\n \"@2.0~version_based+variant_based\",\n \"@2.0+version_based~variant_based+two_whens\",\n ],\n )\n def test_conditional_variants_fail(self, bad_spec):\n with pytest.raises(\n (spack.error.UnsatisfiableSpecError, spack.spec.InvalidVariantForSpecError)\n ):\n _ = spack.concretize.concretize_one(\"conditional-variant-pkg\" + bad_spec)\n\n @pytest.mark.parametrize(\n \"spec_str,expected,unexpected\",\n [\n (\"py-extension3 ^python@3.5.1\", [], [\"py-extension1\"]),\n (\"py-extension3 ^python@2.7.11\", [\"py-extension1\"], []),\n (\"py-extension3@1.0 ^python@2.7.11\", [\"patchelf@0.9\"], []),\n (\"py-extension3@1.1 ^python@2.7.11\", [\"patchelf@0.9\"], []),\n (\"py-extension3@1.0 ^python@3.5.1\", [\"patchelf@0.10\"], []),\n ],\n )\n def test_conditional_dependencies(self, spec_str, expected, unexpected, fuzz_dep_order):\n \"\"\"Tests that conditional dependencies are correctly attached.\n\n The original concretizer can be sensitive to the iteration order over the dependencies of\n a package, so we use a fuzzer function to test concretization with dependencies iterated\n forwards and backwards.\n \"\"\"\n fuzz_dep_order(\"py-extension3\") # test forwards and backwards\n\n s = spack.concretize.concretize_one(spec_str)\n\n for dep in expected:\n msg = '\"{0}\" is not in \"{1}\" and was expected'\n assert dep in s, msg.format(dep, spec_str)\n\n for dep in unexpected:\n msg = '\"{0}\" is in \"{1}\" but was unexpected'\n assert dep not in s, msg.format(dep, spec_str)\n\n @pytest.mark.parametrize(\n \"spec_str,patched_deps\",\n [\n (\"patch-several-dependencies\", [(\"libelf\", 1), (\"fake\", 2)]),\n (\"patch-several-dependencies@1.0\", [(\"libelf\", 1), (\"fake\", 2), (\"libdwarf\", 1)]),\n (\n \"patch-several-dependencies@1.0 ^libdwarf@20111030\",\n [(\"libelf\", 1), (\"fake\", 2), (\"libdwarf\", 2)],\n ),\n (\"patch-several-dependencies ^libelf@0.8.10\", [(\"libelf\", 2), (\"fake\", 2)]),\n (\"patch-several-dependencies +foo\", [(\"libelf\", 2), (\"fake\", 2)]),\n ],\n )\n def test_patching_dependencies(self, spec_str, patched_deps):\n s = spack.concretize.concretize_one(spec_str)\n\n for dep, num_patches in patched_deps:\n assert s[dep].satisfies(\"patches=*\")\n assert len(s[dep].variants[\"patches\"].value) == num_patches\n\n @pytest.mark.regression(\"267,303,1781,2310,2632,3628\")\n @pytest.mark.parametrize(\n \"spec_str, expected\",\n [\n # Need to understand that this configuration is possible\n # only if we use the +mpi variant, which is not the default\n (\"fftw ^mpich\", [\"+mpi\"]),\n # This spec imposes two orthogonal constraints on a dependency,\n # one of which is conditional. The original concretizer fail since\n # when it applies the first constraint, it sets the unknown variants\n # of the dependency to their default values\n (\"quantum-espresso\", [\"^fftw@1.0+mpi\"]),\n # This triggers a conditional dependency on ^fftw@1.0\n (\"quantum-espresso\", [\"^openblas\"]),\n # This constructs a constraint for a dependency og the type\n # @x.y:x.z where the lower bound is unconditional, the upper bound\n # is conditional to having a variant set\n (\"quantum-espresso\", [\"^libelf@0.8.12\"]),\n (\"quantum-espresso~veritas\", [\"^libelf@0.8.13\"]),\n ],\n )\n def test_working_around_conflicting_defaults(self, spec_str, expected):\n s = spack.concretize.concretize_one(spec_str)\n\n assert s.concrete\n for constraint in expected:\n assert s.satisfies(constraint)\n\n @pytest.mark.regression(\"5651\")\n def test_package_with_constraint_not_met_by_external(self):\n \"\"\"Check that if we have an external package A at version X.Y in\n packages.yaml, but our spec doesn't allow X.Y as a version, then\n a new version of A is built that meets the requirements.\n \"\"\"\n packages_yaml = {\"libelf\": {\"externals\": [{\"spec\": \"libelf@0.8.13\", \"prefix\": \"/usr\"}]}}\n spack.config.set(\"packages\", packages_yaml)\n\n # quantum-espresso+veritas requires libelf@:0.8.12\n s = spack.concretize.concretize_one(\"quantum-espresso+veritas\")\n assert s.satisfies(\"^libelf@0.8.12\")\n assert not s[\"libelf\"].external\n\n @pytest.mark.regression(\"9744\")\n def test_cumulative_version_ranges_with_different_length(self):\n s = spack.concretize.concretize_one(\"cumulative-vrange-root\")\n assert s.concrete\n assert s.satisfies(\"^cumulative-vrange-bottom@2.2\")\n\n @pytest.mark.regression(\"9937\")\n def test_dependency_conditional_on_another_dependency_state(self):\n root_str = \"variant-on-dependency-condition-root\"\n dep_str = \"variant-on-dependency-condition-a\"\n spec_str = \"{0} ^{1}\".format(root_str, dep_str)\n\n s = spack.concretize.concretize_one(spec_str)\n assert s.concrete\n assert s.satisfies(\"^variant-on-dependency-condition-b\")\n\n s = spack.concretize.concretize_one(spec_str + \"+x\")\n assert s.concrete\n assert s.satisfies(\"^variant-on-dependency-condition-b\")\n\n s = spack.concretize.concretize_one(spec_str + \"~x\")\n assert s.concrete\n assert not s.satisfies(\"^variant-on-dependency-condition-b\")\n\n def test_external_that_would_require_a_virtual_dependency(self):\n s = spack.concretize.concretize_one(\"requires-virtual\")\n\n assert s.external\n assert \"stuff\" not in s\n\n def test_transitive_conditional_virtual_dependency(self, mutable_config):\n \"\"\"Test that an external is used as provider if the virtual is non-buildable\"\"\"\n mutable_config.set(\"packages:stuff\", {\"buildable\": False})\n s = spack.concretize.concretize_one(\"transitive-conditional-virtual-dependency\")\n\n # Test that the default +stuff~mpi is maintained, and the right provider is selected\n assert s.satisfies(\"^conditional-virtual-dependency +stuff~mpi\")\n assert s.satisfies(\"^[virtuals=stuff] externalvirtual\")\n\n @pytest.mark.regression(\"20040\")\n def test_conditional_provides_or_depends_on(self):\n # Check that we can concretize correctly a spec that can either\n # provide a virtual or depend on it based on the value of a variant\n s = spack.concretize.concretize_one(\"v1-consumer ^conditional-provider +disable-v1\")\n assert \"v1-provider\" in s\n assert s[\"v1\"].name == \"v1-provider\"\n assert s[\"v2\"].name == \"conditional-provider\"\n\n @pytest.mark.regression(\"20079\")\n @pytest.mark.parametrize(\n \"spec_str,tests_arg,with_dep,without_dep\",\n [\n # Check that True is treated correctly and attaches test deps\n # to all nodes in the DAG\n (\"pkg-a\", True, [\"pkg-a\"], []),\n (\"pkg-a foobar=bar\", True, [\"pkg-a\", \"pkg-b\"], []),\n # Check that a list of names activates the dependency only for\n # packages in that list\n (\"pkg-a foobar=bar\", [\"pkg-a\"], [\"pkg-a\"], [\"pkg-b\"]),\n (\"pkg-a foobar=bar\", [\"pkg-b\"], [\"pkg-b\"], [\"pkg-a\"]),\n # Check that False disregard test dependencies\n (\"pkg-a foobar=bar\", False, [], [\"pkg-a\", \"pkg-b\"]),\n ],\n )\n def test_activating_test_dependencies(self, spec_str, tests_arg, with_dep, without_dep):\n s = spack.concretize.concretize_one(spec_str, tests=tests_arg)\n\n for pkg_name in with_dep:\n msg = \"Cannot find test dependency in package '{0}'\"\n node = s[pkg_name]\n assert node.dependencies(deptype=\"test\"), msg.format(pkg_name)\n\n for pkg_name in without_dep:\n msg = \"Test dependency in package '{0}' is unexpected\"\n node = s[pkg_name]\n assert not node.dependencies(deptype=\"test\"), msg.format(pkg_name)\n\n @pytest.mark.regression(\"19981\")\n def test_target_ranges_in_conflicts(self):\n with pytest.raises(spack.error.SpackError):\n spack.concretize.concretize_one(\"impossible-concretization\")\n\n def test_target_compatibility(self):\n with pytest.raises(spack.error.SpackError):\n spack.concretize.concretize_one(\n Spec(\"libdwarf target=x86_64 ^libelf target=x86_64_v2\")\n )\n\n @pytest.mark.regression(\"20040\")\n def test_variant_not_default(self):\n s = spack.concretize.concretize_one(\"ecp-viz-sdk\")\n\n # Check default variant value for the package\n assert \"+dep\" in s[\"conditional-constrained-dependencies\"]\n\n # Check that non-default variant values are forced on the dependency\n d = s[\"dep-with-variants\"]\n assert \"+foo+bar+baz\" in d\n\n def test_all_patches_applied(self):\n uuidpatch = (\n \"a60a42b73e03f207433c5579de207c6ed61d58e4d12dd3b5142eb525728d89ea\"\n if sys.platform != \"win32\"\n else \"d0df7988457ec999c148a4a2af25ce831bfaad13954ba18a4446374cb0aef55e\"\n )\n localpatch = \"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\"\n spec = Spec(\"conditionally-patch-dependency+jasper\")\n spec = spack.concretize.concretize_one(spec)\n assert (uuidpatch, localpatch) == spec[\"libelf\"].variants[\"patches\"].value\n\n def test_dont_select_version_that_brings_more_variants_in(self):\n s = spack.concretize.concretize_one(\"dep-with-variants-if-develop-root\")\n assert s[\"dep-with-variants-if-develop\"].satisfies(\"@1.0\")\n\n @pytest.mark.regression(\"20244,20736\")\n @pytest.mark.parametrize(\n \"spec_str,is_external,expected\",\n [\n # These are all externals, and 0_8 is a version not in package.py\n (\"externaltool@1.0\", True, \"@1.0\"),\n (\"externaltool@0.9\", True, \"@0.9\"),\n (\"externaltool@0_8\", True, \"@0_8\"),\n # This external package is buildable, has a custom version\n # in packages.yaml that is greater than the ones in package.py\n # and specifies a variant\n (\"external-buildable-with-variant +baz\", True, \"@1.1.special +baz\"),\n (\"external-buildable-with-variant ~baz\", False, \"@1.0 ~baz\"),\n (\"external-buildable-with-variant@1.0: ~baz\", False, \"@1.0 ~baz\"),\n # This uses an external version that meets the condition for\n # having an additional dependency, but the dependency shouldn't\n # appear in the answer set\n (\"external-buildable-with-variant@0.9 +baz\", True, \"@0.9\"),\n # This package has an external version declared that would be\n # the least preferred if Spack had to build it\n (\"old-external\", True, \"@1.0.0\"),\n ],\n )\n def test_external_package_versions(self, spec_str, is_external, expected):\n s = spack.concretize.concretize_one(spec_str)\n assert s.external == is_external\n assert s.satisfies(expected)\n\n @pytest.mark.parametrize(\"dev_first\", [True, False])\n @pytest.mark.parametrize(\n \"spec\", [\"dev-build-test-install\", \"dev-build-test-dependent ^dev-build-test-install\"]\n )\n @pytest.mark.parametrize(\"mock_db\", [True, False])\n def test_reuse_does_not_overwrite_dev_specs(\n self, dev_first, spec, mock_db, tmp_path: pathlib.Path, temporary_store, monkeypatch\n ):\n \"\"\"Test that reuse does not mix dev specs with non-dev specs.\n\n Tests for either order (dev specs are not reused for non-dev, and\n non-dev specs are not reused for dev specs)\n Tests for a spec in which the root is developed and a spec in\n which a dep is developed.\n Tests for both reuse from database and reuse from buildcache\"\"\"\n # dev and non-dev specs that are otherwise identical\n spec = Spec(spec)\n dev_spec = spec.copy()\n dev_spec[\"dev-build-test-install\"].constrain(f\"dev_path={tmp_path}\")\n\n # run the test in both orders\n first_spec = dev_spec if dev_first else spec\n second_spec = spec if dev_first else dev_spec\n\n # concretize and setup spack to reuse in the appropriate manner\n first_spec = spack.concretize.concretize_one(first_spec)\n\n def mock_fn(*args, **kwargs):\n return [first_spec]\n\n if mock_db:\n temporary_store.db.add(first_spec)\n else:\n monkeypatch.setattr(spack.binary_distribution, \"update_cache_and_get_specs\", mock_fn)\n\n # concretize and ensure we did not reuse\n with spack.config.override(\"concretizer:reuse\", True):\n second_spec = spack.concretize.concretize_one(second_spec)\n assert first_spec.dag_hash() != second_spec.dag_hash()\n\n @pytest.mark.regression(\"20292\")\n @pytest.mark.parametrize(\n \"context\",\n [\n {\"add_variant\": True, \"delete_variant\": False},\n {\"add_variant\": False, \"delete_variant\": True},\n {\"add_variant\": True, \"delete_variant\": True},\n ],\n )\n def test_reuse_installed_packages_when_package_def_changes(\n self, context, mutable_database, repo_with_changing_recipe\n ):\n # test applies only with reuse turned off in concretizer\n spack.config.set(\"concretizer:reuse\", False)\n\n # Install a spec\n root = spack.concretize.concretize_one(\"root\")\n dependency = root[\"changing\"].copy()\n PackageInstaller([root.package], fake=True, explicit=True).install()\n\n # Modify package.py\n repo_with_changing_recipe.change(context)\n\n # Try to concretize with the spec installed previously\n new_root_with_reuse = spack.concretize.concretize_one(\n Spec(\"root ^/{0}\".format(dependency.dag_hash()))\n )\n\n new_root_without_reuse = spack.concretize.concretize_one(\"root\")\n\n # validate that the graphs are the same with reuse, but not without\n assert ht.build_hash(root) == ht.build_hash(new_root_with_reuse)\n assert ht.build_hash(root) != ht.build_hash(new_root_without_reuse)\n\n # DAG hash should be the same with reuse since only the dependency changed\n assert root.dag_hash() == new_root_with_reuse.dag_hash()\n\n # Structure and package hash will be different without reuse\n assert root.dag_hash() != new_root_without_reuse.dag_hash()\n\n @pytest.mark.regression(\"43663\")\n def test_no_reuse_when_variant_condition_does_not_hold(self, mutable_database, mock_packages):\n spack.config.set(\"concretizer:reuse\", True)\n\n # Install a spec for which the `version_based` variant condition does not hold\n old = spack.concretize.concretize_one(\"conditional-variant-pkg @1\")\n PackageInstaller([old.package], fake=True, explicit=True).install()\n\n # Then explicitly require a spec with `+version_based`, which shouldn't reuse previous spec\n new1 = spack.concretize.concretize_one(\"conditional-variant-pkg +version_based\")\n assert new1.satisfies(\"@2 +version_based\")\n\n new2 = spack.concretize.concretize_one(\"conditional-variant-pkg +two_whens\")\n assert new2.satisfies(\"@2 +two_whens +version_based\")\n\n def test_reuse_with_flags(self, mutable_database, mutable_config):\n spack.config.set(\"concretizer:reuse\", True)\n spec = spack.concretize.concretize_one(\"pkg-a cflags=-g cxxflags=-g\")\n PackageInstaller([spec.package], fake=True, explicit=True).install()\n testspec = spack.concretize.concretize_one(\"pkg-a cflags=-g\")\n assert testspec == spec, testspec.tree()\n\n @pytest.mark.regression(\"20784\")\n def test_concretization_of_test_dependencies(self):\n # With clingo we emit dependency_conditions regardless of the type\n # of the dependency. We need to ensure that there's at least one\n # dependency type declared to infer that the dependency holds.\n s = spack.concretize.concretize_one(\"test-dep-with-imposed-conditions\")\n assert \"c\" not in s\n\n @pytest.mark.parametrize(\n \"spec_str\", [\"wrong-variant-in-conflicts\", \"wrong-variant-in-depends-on\"]\n )\n def test_error_message_for_inconsistent_variants(self, spec_str):\n s = Spec(spec_str)\n with pytest.raises(vt.UnknownVariantError):\n s = spack.concretize.concretize_one(s)\n\n @pytest.mark.regression(\"22533\")\n @pytest.mark.parametrize(\n \"spec_str,variant_name,expected_values\",\n [\n # Test the default value 'auto'\n (\"mvapich2\", \"file_systems\", (\"auto\",)),\n # Test setting a single value from the disjoint set\n (\"mvapich2 file_systems=lustre\", \"file_systems\", (\"lustre\",)),\n # Test setting multiple values from the disjoint set\n (\"mvapich2 file_systems=lustre,gpfs\", \"file_systems\", (\"lustre\", \"gpfs\")),\n ],\n )\n def test_mv_variants_disjoint_sets_from_spec(self, spec_str, variant_name, expected_values):\n s = spack.concretize.concretize_one(spec_str)\n assert set(expected_values) == set(s.variants[variant_name].value)\n\n @pytest.mark.regression(\"22533\")\n def test_mv_variants_disjoint_sets_from_packages_yaml(self):\n external_mvapich2 = {\n \"mvapich2\": {\n \"buildable\": False,\n \"externals\": [{\"spec\": \"mvapich2@2.3.1 file_systems=nfs,ufs\", \"prefix\": \"/usr\"}],\n }\n }\n spack.config.set(\"packages\", external_mvapich2)\n\n s = spack.concretize.concretize_one(\"mvapich2\")\n assert set(s.variants[\"file_systems\"].value) == set([\"ufs\", \"nfs\"])\n\n @pytest.mark.regression(\"22596\")\n def test_external_with_non_default_variant_as_dependency(self):\n # This package depends on another that is registered as an external\n # with 'buildable: true' and a variant with a non-default value set\n s = spack.concretize.concretize_one(\"trigger-external-non-default-variant\")\n\n assert \"~foo\" in s[\"external-non-default-variant\"]\n assert \"~bar\" in s[\"external-non-default-variant\"]\n assert s[\"external-non-default-variant\"].external\n\n @pytest.mark.regression(\"22718\")\n @pytest.mark.parametrize(\n \"spec_str,expected_compiler\",\n [(\"mpileaks\", \"%gcc@10.2.1\"), (\"mpileaks ^mpich%clang@15.0.0\", \"%clang@15.0.0\")],\n )\n def test_compiler_is_unique(self, spec_str, expected_compiler):\n s = spack.concretize.concretize_one(spec_str)\n\n for node in s.traverse():\n if not node.satisfies(\"^ c\"):\n continue\n assert node.satisfies(expected_compiler)\n\n @pytest.mark.parametrize(\n \"spec_str,expected_dict\",\n [\n # Check the defaults from the package (libs=shared)\n (\"multivalue-variant\", {\"libs=shared\": True, \"libs=static\": False}),\n # Check that libs=static doesn't extend the default\n (\"multivalue-variant libs=static\", {\"libs=shared\": False, \"libs=static\": True}),\n ],\n )\n def test_multivalued_variants_from_cli(self, spec_str, expected_dict):\n s = spack.concretize.concretize_one(spec_str)\n\n for constraint, value in expected_dict.items():\n assert s.satisfies(constraint) == value\n\n @pytest.mark.regression(\"22351\")\n @pytest.mark.parametrize(\n \"spec_str,expected\",\n [\n # Version 1.1.0 is deprecated and should not be selected, unless we\n # explicitly asked for that\n (\"deprecated-versions\", \"deprecated-versions@1.0.0\"),\n (\"deprecated-versions@=1.1.0\", \"deprecated-versions@1.1.0\"),\n ],\n )\n def test_deprecated_versions_not_selected(self, spec_str, expected):\n with spack.config.override(\"config:deprecated\", True):\n s = spack.concretize.concretize_one(spec_str)\n s.satisfies(expected)\n\n @pytest.mark.regression(\"24196\")\n def test_version_badness_more_important_than_default_mv_variants(self):\n # If a dependency had an old version that for some reason pulls in\n # a transitive dependency with a multi-valued variant, that old\n # version was preferred because of the order of our optimization\n # criteria.\n s = spack.concretize.concretize_one(\"root\")\n assert s[\"gmt\"].satisfies(\"@2.0\")\n\n @pytest.mark.regression(\"24205\")\n def test_provider_must_meet_requirements(self):\n # A package can be a provider of a virtual only if the underlying\n # requirements are met.\n s = Spec(\"unsat-virtual-dependency\")\n with pytest.raises((RuntimeError, spack.error.UnsatisfiableSpecError)):\n s = spack.concretize.concretize_one(s)\n\n @pytest.mark.regression(\"23951\")\n def test_newer_dependency_adds_a_transitive_virtual(self):\n # Ensure that a package doesn't concretize any of its transitive\n # dependencies to an old version because newer versions pull in\n # a new virtual dependency. The possible concretizations here are:\n #\n # root@1.0 <- middle@1.0 <- leaf@2.0 <- blas\n # root@1.0 <- middle@1.0 <- leaf@1.0\n #\n # and \"blas\" is pulled in only by newer versions of \"leaf\"\n s = spack.concretize.concretize_one(\"root-adds-virtual\")\n assert s[\"leaf-adds-virtual\"].satisfies(\"@2.0\")\n assert \"blas\" in s\n\n @pytest.mark.regression(\"26718\")\n def test_versions_in_virtual_dependencies(self):\n # Ensure that a package that needs a given version of a virtual\n # package doesn't end up using a later implementation\n s = spack.concretize.concretize_one(\"hpcviewer@2019.02\")\n assert s[\"java\"].satisfies(\"virtual-with-versions@1.8.0\")\n\n @pytest.mark.regression(\"26866\")\n def test_non_default_provider_of_multiple_virtuals(self):\n s = spack.concretize.concretize_one(\"many-virtual-consumer ^low-priority-provider\")\n assert s[\"mpi\"].name == \"low-priority-provider\"\n assert s[\"lapack\"].name == \"low-priority-provider\"\n\n for virtual_pkg in (\"mpi\", \"lapack\"):\n for pkg in spack.repo.PATH.providers_for(virtual_pkg):\n if pkg.name == \"low-priority-provider\":\n continue\n assert pkg not in s\n\n @pytest.mark.regression(\"27237\")\n @pytest.mark.parametrize(\n \"spec_str,expect_installed\",\n [(\"mpich\", True), (\"mpich+debug\", False), (\"mpich~debug\", True)],\n )\n def test_concrete_specs_are_not_modified_on_reuse(\n self, mutable_database, spec_str, expect_installed\n ):\n # Test the internal consistency of solve + DAG reconstruction\n # when reused specs are added to the mix. This prevents things\n # like additional constraints being added to concrete specs in\n # the answer set produced by clingo.\n with spack.config.override(\"concretizer:reuse\", True):\n s = spack.concretize.concretize_one(spec_str)\n assert s.installed is expect_installed\n assert s.satisfies(spec_str)\n\n @pytest.mark.regression(\"26721,19736\")\n def test_sticky_variant_in_package(self):\n # Here we test that a sticky variant cannot be changed from its default value\n # by the ASP solver if not set explicitly. The package used in the test needs\n # to have +allow-gcc set to be concretized with %gcc and clingo is not allowed\n # to change the default ~allow-gcc\n with pytest.raises(spack.error.SpackError):\n spack.concretize.concretize_one(\"sticky-variant %gcc\")\n\n s = spack.concretize.concretize_one(\"sticky-variant+allow-gcc %gcc\")\n assert s.satisfies(\"%gcc\") and s.satisfies(\"+allow-gcc\")\n\n s = spack.concretize.concretize_one(\"sticky-variant %clang\")\n assert s.satisfies(\"%clang\") and s.satisfies(\"~allow-gcc\")\n\n @pytest.mark.regression(\"42172\")\n @pytest.mark.parametrize(\n \"spec,allow_gcc\",\n [\n (\"sticky-variant@1.0+allow-gcc\", True),\n (\"sticky-variant@1.0~allow-gcc\", False),\n # FIXME (externals as concrete) (\"sticky-variant@1.0\", False),\n ],\n )\n def test_sticky_variant_in_external(self, spec, allow_gcc):\n # setup external for sticky-variant+allow-gcc\n config = {\"externals\": [{\"spec\": spec, \"prefix\": \"/fake/path\"}], \"buildable\": False}\n spack.config.set(\"packages:sticky-variant\", config)\n\n maybe = spack.llnl.util.lang.nullcontext if allow_gcc else pytest.raises\n with maybe(spack.error.SpackError):\n s = spack.concretize.concretize_one(\"sticky-variant-dependent%gcc\")\n\n if allow_gcc:\n assert s.satisfies(\"%gcc\")\n assert s[\"sticky-variant\"].satisfies(\"+allow-gcc\")\n assert s[\"sticky-variant\"].external\n\n def test_do_not_invent_new_concrete_versions_unless_necessary(self):\n # ensure we select a known satisfying version rather than creating\n # a new '2.7' version.\n assert ver(\"=2.7.11\") == spack.concretize.concretize_one(\"python@2.7\").version\n\n # Here there is no known satisfying version - use the one on the spec.\n assert ver(\"=2.7.21\") == spack.concretize.concretize_one(\"python@=2.7.21\").version\n\n @pytest.mark.parametrize(\n \"spec_str,valid\",\n [\n (\"conditional-values-in-variant@1.62.0 cxxstd=17\", False),\n (\"conditional-values-in-variant@1.62.0 cxxstd=2a\", False),\n (\"conditional-values-in-variant@1.72.0 cxxstd=2a\", False),\n # Ensure disjoint set of values work too\n (\"conditional-values-in-variant@1.72.0 staging=flexpath\", False),\n # Ensure conditional values set False fail too\n (\"conditional-values-in-variant foo=bar\", False),\n (\"conditional-values-in-variant foo=foo\", True),\n ],\n )\n def test_conditional_values_in_variants(self, spec_str, valid):\n s = Spec(spec_str)\n raises = pytest.raises((RuntimeError, spack.error.UnsatisfiableSpecError))\n with spack.llnl.util.lang.nullcontext() if valid else raises:\n s = spack.concretize.concretize_one(s)\n\n def test_conditional_values_in_conditional_variant(self):\n \"\"\"Test that conditional variants play well with conditional possible values\"\"\"\n s = spack.concretize.concretize_one(\"conditional-values-in-variant@1.50.0\")\n assert \"cxxstd\" not in s.variants\n\n s = spack.concretize.concretize_one(\"conditional-values-in-variant@1.60.0\")\n assert \"cxxstd\" in s.variants\n\n def test_target_granularity(self):\n # The test architecture uses core2 as the default target. Check that when\n # we configure Spack for \"generic\" granularity we concretize for x86_64\n default_target = spack.platforms.test.Test.default\n generic_target = spack.vendor.archspec.cpu.TARGETS[default_target].generic.name\n s = Spec(\"python\")\n assert spack.concretize.concretize_one(s).satisfies(\"target=%s\" % default_target)\n with spack.config.override(\"concretizer:targets\", {\"granularity\": \"generic\"}):\n assert spack.concretize.concretize_one(s).satisfies(\"target=%s\" % generic_target)\n\n def test_host_compatible_concretization(self):\n # Check that after setting \"host_compatible\" to false we cannot concretize.\n # Here we use \"k10\" to set a target non-compatible with the current host\n # to avoid a lot of boilerplate when mocking the test platform. The issue\n # is that the defaults for the test platform are very old, so there's no\n # compiler supporting e.g. icelake etc.\n s = Spec(\"python target=k10\")\n assert spack.concretize.concretize_one(s)\n with spack.config.override(\"concretizer:targets\", {\"host_compatible\": True}):\n with pytest.raises(spack.error.SpackError):\n spack.concretize.concretize_one(s)\n\n def test_add_microarchitectures_on_explicit_request(self):\n # Check that if we consider only \"generic\" targets, we can still solve for\n # specific microarchitectures on explicit requests\n with spack.config.override(\"concretizer:targets\", {\"granularity\": \"generic\"}):\n s = spack.concretize.concretize_one(\"python target=k10\")\n assert s.satisfies(\"target=k10\")\n\n @pytest.mark.regression(\"29201\")\n def test_delete_version_and_reuse(self, mutable_database, repo_with_changing_recipe):\n \"\"\"Test that we can reuse installed specs with versions not\n declared in package.py\n \"\"\"\n root = spack.concretize.concretize_one(\"root\")\n PackageInstaller([root.package], fake=True, explicit=True).install()\n repo_with_changing_recipe.change({\"delete_version\": True})\n\n with spack.config.override(\"concretizer:reuse\", True):\n new_root = spack.concretize.concretize_one(\"root\")\n\n assert root.dag_hash() == new_root.dag_hash()\n\n @pytest.mark.regression(\"29201\")\n def test_installed_version_is_selected_only_for_reuse(\n self, mutable_database, repo_with_changing_recipe\n ):\n \"\"\"Test that a version coming from an installed spec is a possible\n version only for reuse\n \"\"\"\n # Install a dependency that cannot be reused with \"root\"\n # because of a conflict in a variant, then delete its version\n dependency = spack.concretize.concretize_one(\"changing@1.0~foo\")\n PackageInstaller([dependency.package], fake=True, explicit=True).install()\n repo_with_changing_recipe.change({\"delete_version\": True})\n\n with spack.config.override(\"concretizer:reuse\", True):\n new_root = spack.concretize.concretize_one(\"root\")\n\n assert not new_root[\"changing\"].satisfies(\"@1.0\")\n\n @pytest.mark.regression(\"28259\")\n def test_reuse_with_unknown_namespace_dont_raise(\n self, temporary_store, mock_custom_repository\n ):\n with spack.repo.use_repositories(mock_custom_repository, override=False):\n s = spack.concretize.concretize_one(\"pkg-c\")\n assert s.namespace != \"builtin_mock\"\n PackageInstaller([s.package], fake=True, explicit=True).install()\n\n with spack.config.override(\"concretizer:reuse\", True):\n s = spack.concretize.concretize_one(\"pkg-c\")\n assert s.namespace == \"builtin_mock\"\n\n @pytest.mark.regression(\"45538\")\n def test_reuse_from_other_namespace_no_raise(\n self, temporary_store, monkeypatch, repo_builder: RepoBuilder\n ):\n repo_builder.add_package(\"zlib\")\n\n builtin = spack.concretize.concretize_one(\"zlib\")\n PackageInstaller([builtin.package], fake=True, explicit=True).install()\n\n with spack.repo.use_repositories(repo_builder.root, override=False):\n with spack.config.override(\"concretizer:reuse\", True):\n zlib = spack.concretize.concretize_one(f\"{repo_builder.namespace}.zlib\")\n\n assert zlib.namespace == repo_builder.namespace\n\n @pytest.mark.regression(\"28259\")\n def test_reuse_with_unknown_package_dont_raise(\n self, temporary_store, monkeypatch, repo_builder: RepoBuilder\n ):\n repo_builder.add_package(\"pkg-c\")\n with spack.repo.use_repositories(repo_builder.root, override=False):\n s = spack.concretize.concretize_one(\"pkg-c\")\n assert s.namespace == repo_builder.namespace\n PackageInstaller([s.package], fake=True, explicit=True).install()\n del sys.modules[f\"spack_repo.{repo_builder.namespace}.packages.pkg_c\"]\n repo_builder.remove(\"pkg-c\")\n with spack.repo.use_repositories(repo_builder.root, override=False) as repos:\n repos.repos[0]._pkg_checker.invalidate()\n with spack.config.override(\"concretizer:reuse\", True):\n s = spack.concretize.concretize_one(\"pkg-c\")\n assert s.namespace == \"builtin_mock\"\n\n @pytest.mark.parametrize(\n \"specs,checks\",\n [\n ([\"libelf\", \"libelf@0.8.10\"], {\"libelf\": 1}),\n ([\"libdwarf%gcc\", \"libelf%clang\"], {\"libdwarf\": 1, \"libelf\": 1}),\n ([\"libdwarf%gcc\", \"libdwarf%clang\"], {\"libdwarf\": 2, \"libelf\": 1}),\n ([\"libdwarf^libelf@0.8.12\", \"libdwarf^libelf@0.8.13\"], {\"libdwarf\": 2, \"libelf\": 2}),\n ([\"hdf5\", \"zmpi\"], {\"zmpi\": 1, \"fake\": 1}),\n ([\"hdf5\", \"mpich\"], {\"mpich\": 1}),\n ([\"hdf5^zmpi\", \"mpich\"], {\"mpi\": 2, \"mpich\": 1, \"zmpi\": 1, \"fake\": 1}),\n ([\"mpi\", \"zmpi\"], {\"mpi\": 1, \"mpich\": 0, \"zmpi\": 1, \"fake\": 1}),\n ([\"mpi\", \"mpich\"], {\"mpi\": 1, \"mpich\": 1, \"zmpi\": 0}),\n ],\n )\n def test_best_effort_coconcretize(self, specs, checks):\n specs = [Spec(s) for s in specs]\n solver = spack.solver.asp.Solver()\n solver.reuse = False\n concrete_specs = set()\n for result in solver.solve_in_rounds(specs):\n for s in result.specs:\n concrete_specs.update(s.traverse())\n\n for matching_spec, expected_count in checks.items():\n matches = [x for x in concrete_specs if x.satisfies(matching_spec)]\n assert len(matches) == expected_count\n\n @pytest.mark.parametrize(\n \"specs,expected_spec,occurrences\",\n [\n # The algorithm is greedy, and it might decide to solve the \"best\"\n # spec early in which case reuse is suboptimal. In this case the most\n # recent version of libdwarf is selected and concretized to libelf@0.8.13\n (\n [\n \"libdwarf@20111030^libelf@0.8.10\",\n \"libdwarf@20130207^libelf@0.8.12\",\n \"libdwarf@20130729\",\n ],\n \"libelf@0.8.12\",\n 1,\n ),\n # Check we reuse the best libelf in the environment\n (\n [\n \"libdwarf@20130729^libelf@0.8.10\",\n \"libdwarf@20130207^libelf@0.8.12\",\n \"libdwarf@20111030\",\n ],\n \"libelf@0.8.12\",\n 2,\n ),\n ([\"libdwarf@20130729\", \"libdwarf@20130207\", \"libdwarf@20111030\"], \"libelf@0.8.13\", 3),\n # We need to solve in 2 rounds and we expect mpich to be preferred to zmpi\n ([\"hdf5+mpi\", \"zmpi\", \"mpich\"], \"mpich\", 2),\n ],\n )\n def test_best_effort_coconcretize_preferences(self, specs, expected_spec, occurrences):\n \"\"\"Test package preferences during coconcretization.\"\"\"\n specs = [Spec(s) for s in specs]\n solver = spack.solver.asp.Solver()\n solver.reuse = False\n concrete_specs = {}\n for result in solver.solve_in_rounds(specs):\n concrete_specs.update(result.specs_by_input)\n\n counter = 0\n for spec in concrete_specs.values():\n if expected_spec in spec:\n counter += 1\n assert counter == occurrences, concrete_specs\n\n def test_solve_in_rounds_all_unsolved(self, monkeypatch, mock_packages):\n specs = [Spec(x) for x in [\"libdwarf%gcc\", \"libdwarf%clang\"]]\n solver = spack.solver.asp.Solver()\n solver.reuse = False\n\n simulate_unsolved_property = list((x, None) for x in specs)\n monkeypatch.setattr(spack.solver.asp.Result, \"unsolved_specs\", simulate_unsolved_property)\n monkeypatch.setattr(spack.solver.asp.Result, \"specs\", list())\n\n with pytest.raises(spack.solver.asp.OutputDoesNotSatisfyInputError):\n list(solver.solve_in_rounds(specs))\n\n def test_coconcretize_reuse_and_virtuals(self):\n reusable_specs = []\n for s in [\"mpileaks ^mpich\", \"zmpi\"]:\n reusable_specs.extend(spack.concretize.concretize_one(s).traverse(root=True))\n\n root_specs = [Spec(\"mpileaks\"), Spec(\"zmpi\")]\n\n with spack.config.override(\"concretizer:reuse\", True):\n solver = spack.solver.asp.Solver()\n setup = spack.solver.asp.SpackSolverSetup()\n result, _, _ = solver.driver.solve(setup, root_specs, reuse=reusable_specs)\n\n for spec in result.specs:\n assert \"zmpi\" in spec\n\n @pytest.mark.regression(\"30864\")\n def test_misleading_error_message_on_version(self, mutable_database):\n # For this bug to be triggered we need a reusable dependency\n # that is not optimal in terms of optimization scores.\n # We pick an old version of \"b\"\n reusable_specs = [spack.concretize.concretize_one(\"non-existing-conditional-dep@1.0\")]\n root_spec = Spec(\"non-existing-conditional-dep@2.0\")\n\n with spack.config.override(\"concretizer:reuse\", True):\n solver = spack.solver.asp.Solver()\n setup = spack.solver.asp.SpackSolverSetup()\n with pytest.raises(spack.solver.asp.UnsatisfiableSpecError, match=\"Cannot satisfy\"):\n solver.driver.solve(setup, [root_spec], reuse=reusable_specs)\n\n @pytest.mark.regression(\"31148\")\n def test_version_weight_and_provenance(self, mutable_config):\n \"\"\"Test package preferences during concretization.\"\"\"\n reusable_specs = [\n spack.concretize.concretize_one(spec_str) for spec_str in (\"pkg-b@0.9\", \"pkg-b@1.0\")\n ]\n root_spec = Spec(\"pkg-a foobar=bar\")\n\n packages_with_externals = external_config_with_implicit_externals(mutable_config)\n completion_mode = mutable_config.get(\"concretizer:externals:completion\")\n external_specs = spec_filter_from_packages_yaml(\n external_parser=create_external_parser(packages_with_externals, completion_mode),\n packages_with_externals=packages_with_externals,\n include=[],\n exclude=[],\n ).selected_specs()\n with spack.config.override(\"concretizer:reuse\", True):\n solver = spack.solver.asp.Solver()\n setup = spack.solver.asp.SpackSolverSetup()\n result, _, _ = solver.driver.solve(\n setup, [root_spec], reuse=reusable_specs + external_specs\n )\n # Version badness should be > 0 only for reused specs. For instance, for pkg-b\n # the version provenance is:\n #\n # pkg_fact(\"pkg-b\", version_declared(\"1.0\", 0)).\n # pkg_fact(\"pkg-b\", version_origin(\"1.0\", \"installed\")).\n # pkg_fact(\"pkg-b\", version_origin(\"1.0\", \"package_py\")).\n # pkg_fact(\"pkg-b\", version_declared(\"0.9\", 1)).\n # pkg_fact(\"pkg-b\", version_origin(\"0.9\", \"installed\")).\n # pkg_fact(\"pkg-b\", version_origin(\"0.9\", \"package_py\")).\n\n weights = weights_from_result(result, name=\"version badness (non roots)\")\n assert weights[\"reused\"] == 3 and weights[\"built\"] == 0\n\n result_spec = result.specs[0]\n assert result_spec.satisfies(\"^pkg-b@1.0\")\n assert result_spec[\"pkg-b\"].dag_hash() == reusable_specs[1].dag_hash()\n\n @pytest.mark.regression(\"51112\")\n def test_variant_penalty(self, mutable_config):\n \"\"\"Test package preferences during concretization.\"\"\"\n packages_with_externals = external_config_with_implicit_externals(mutable_config)\n completion_mode = mutable_config.get(\"concretizer:externals:completion\")\n external_specs = spec_filter_from_packages_yaml(\n external_parser=create_external_parser(packages_with_externals, completion_mode),\n packages_with_externals=packages_with_externals,\n include=[],\n exclude=[],\n ).selected_specs()\n\n # The variant definition is similar to\n #\n # % Variant cxxstd in package trilinos\n # pkg_fact(\"trilinos\",variant_definition(\"cxxstd\",195)).\n # variant_type(195,\"single\").\n # pkg_fact(\"trilinos\",variant_default_value_from_package_py(195,\"14\")).\n # pkg_fact(\"trilinos\",variant_penalty(195,\"14\",1)).\n # pkg_fact(\"trilinos\",variant_penalty(195,\"17\",2)).\n # pkg_fact(\"trilinos\",variant_penalty(195,\"20\",3)).\n # pkg_fact(\"trilinos\",variant_possible_value(195,\"14\")).\n # pkg_fact(\"trilinos\",variant_possible_value(195,\"17\")).\n # pkg_fact(\"trilinos\",variant_possible_value(195,\"20\")).\n\n solver = spack.solver.asp.Solver()\n setup = spack.solver.asp.SpackSolverSetup()\n\n # Ensure that since the default value of 14 cannot be taken, we select \"17\"\n result, _, _ = solver.driver.solve(setup, [Spec(\"trilinos\")], reuse=external_specs)\n\n weights = weights_from_result(result, name=\"variant penalty (roots)\")\n assert weights[\"reused\"] == 0 and weights[\"built\"] == 2\n\n trilinos = result.specs[0]\n assert trilinos.satisfies(\"cxxstd=17\")\n\n # If we disable \"17\", then \"20\" is next, and the penalty is higher\n result, _, _ = solver.driver.solve(\n setup, [Spec(\"trilinos+disable17\")], reuse=external_specs\n )\n\n weights = weights_from_result(result, name=\"variant penalty (roots)\")\n assert weights[\"reused\"] == 0 and weights[\"built\"] == 3\n\n trilinos = result.specs[0]\n assert trilinos.satisfies(\"cxxstd=20\")\n\n # Test a disjoint set of values to ensure declared package order is respected\n result, _, _ = solver.driver.solve(setup, [Spec(\"mvapich2\")], reuse=external_specs)\n\n weights = weights_from_result(result, name=\"variant penalty (roots)\")\n assert weights[\"reused\"] == 0 and weights[\"built\"] == 0\n mvapich2 = result.specs[0]\n assert mvapich2.satisfies(\"file_systems=auto\")\n\n result, _, _ = solver.driver.solve(setup, [Spec(\"mvapich2+noauto\")], reuse=external_specs)\n\n weights = weights_from_result(result, name=\"variant penalty (roots)\")\n assert weights[\"reused\"] == 0 and weights[\"built\"] == 2\n mvapich2 = result.specs[0]\n assert mvapich2.satisfies(\"file_systems=lustre\")\n\n @pytest.mark.regression(\"51267\")\n @pytest.mark.parametrize(\n \"packages_config,expected\",\n [\n # Two preferences on different virtuals\n (\n \"\"\"\n packages:\n c:\n prefer:\n - clang\n mpi:\n prefer:\n - mpich2\n \"\"\",\n [\n 'provider_weight_from_config(\"mpi\",\"mpich2\",0).',\n 'provider_weight_from_config(\"c\",\"clang\",0).',\n ],\n ),\n # A requirement and a preference on the same virtual\n (\n \"\"\"\n packages:\n c:\n require:\n - gcc\n prefer:\n - clang\n \"\"\",\n [\n 'provider_weight_from_config(\"c\",\"gcc\",0).',\n 'provider_weight_from_config(\"c\",\"clang\",1).',\n ],\n ),\n (\n \"\"\"\n packages:\n c:\n require:\n - clang\n prefer:\n - gcc\n \"\"\",\n [\n 'provider_weight_from_config(\"c\",\"gcc\",1).',\n 'provider_weight_from_config(\"c\",\"clang\",0).',\n ],\n ),\n # Multiple requirements with priorities\n (\n \"\"\"\n packages:\n all:\n providers:\n mpi: [low-priority-mpi]\n mpi:\n require:\n - any_of: [mpich2, zmpi]\n prefer:\n - mpich\n \"\"\",\n [\n 'provider_weight_from_config(\"mpi\",\"mpich2\",0).',\n 'provider_weight_from_config(\"mpi\",\"zmpi\",1).',\n 'provider_weight_from_config(\"mpi\",\"mpich\",2).',\n 'provider_weight_from_config(\"mpi\",\"low-priority-mpi\",3).',\n ],\n ),\n # Configuration with conflicts\n (\n \"\"\"\n packages:\n all:\n providers:\n mpi: [mpich, low-priority-mpi]\n mpi:\n require:\n - mpich2\n conflict:\n - mpich\n \"\"\",\n [\n 'provider_weight_from_config(\"mpi\",\"mpich2\",0).',\n 'provider_weight_from_config(\"mpi\",\"low-priority-mpi\",1).',\n ],\n ),\n (\n \"\"\"\n packages:\n all:\n providers:\n mpi: [mpich, low-priority-mpi]\n mpi:\n require:\n - mpich2\n conflict:\n - mpich@1\n \"\"\",\n [\n 'provider_weight_from_config(\"mpi\",\"mpich2\",0).',\n 'provider_weight_from_config(\"mpi\",\"mpich\",1).',\n 'provider_weight_from_config(\"mpi\",\"low-priority-mpi\",2).',\n ],\n ),\n ],\n )\n def test_requirements_and_weights(self, packages_config, expected, mutable_config):\n \"\"\"Checks that requirements and strong preferences on virtual packages influence the\n weights for providers, even if \"package preferences\" are not set consistently.\n \"\"\"\n packages_yaml = syaml.load_config(packages_config)\n mutable_config.set(\"packages\", packages_yaml[\"packages\"])\n\n setup = spack.solver.asp.SpackSolverSetup()\n asp_problem = setup.setup([Spec(\"mpileaks\")], reuse=[], allow_deprecated=False).asp_problem\n\n assert all(x in asp_problem for x in expected)\n\n def test_reuse_succeeds_with_config_compatible_os(self):\n root_spec = Spec(\"pkg-b\")\n s = spack.concretize.concretize_one(root_spec)\n other_os = s.copy()\n mock_os = \"ubuntu2204\"\n other_os.architecture = spack.spec.ArchSpec(\n \"test-{os}-{target}\".format(os=mock_os, target=str(s.architecture.target))\n )\n reusable_specs = [other_os]\n overrides = {\"concretizer\": {\"reuse\": True, \"os_compatible\": {s.os: [mock_os]}}}\n custom_scope = spack.config.InternalConfigScope(\"concretize_override\", overrides)\n with spack.config.override(custom_scope):\n solver = spack.solver.asp.Solver()\n setup = spack.solver.asp.SpackSolverSetup()\n result, _, _ = solver.driver.solve(setup, [root_spec], reuse=reusable_specs)\n concrete_spec = result.specs[0]\n assert concrete_spec.satisfies(\"os={}\".format(other_os.architecture.os))\n\n def test_git_hash_assigned_version_is_preferred(self):\n hash = \"a\" * 40\n s = Spec(\"develop-branch-version@%s=develop\" % hash)\n c = spack.concretize.concretize_one(s)\n assert hash in str(c)\n\n @pytest.mark.parametrize(\"git_ref\", (\"a\" * 40, \"0.2.15\", \"main\"))\n def test_git_ref_version_is_equivalent_to_specified_version(self, git_ref):\n s = Spec(\"develop-branch-version@git.%s=develop\" % git_ref)\n c = spack.concretize.concretize_one(s)\n assert git_ref in str(c)\n assert s.satisfies(\"@develop\")\n assert s.satisfies(\"@0.1:\")\n\n @pytest.mark.parametrize(\"git_ref\", (\"a\" * 40, \"0.2.15\", \"fbranch\"))\n def test_git_ref_version_succeeds_with_unknown_version(self, git_ref):\n # main is not defined in the package.py for this file\n s = Spec(\"develop-branch-version@git.%s=main\" % git_ref)\n s = spack.concretize.concretize_one(s)\n assert s.satisfies(\"develop-branch-version@main\")\n\n @pytest.mark.regression(\"31484\")\n def test_installed_externals_are_reused(\n self, mutable_database, repo_with_changing_recipe, tmp_path: pathlib.Path\n ):\n \"\"\"Tests that external specs that are in the DB can be reused, if they result in a\n better optimization score.\n \"\"\"\n external_conf = {\n \"changing\": {\n \"buildable\": False,\n \"externals\": [{\"spec\": \"changing@1.0\", \"prefix\": str(tmp_path)}],\n }\n }\n spack.config.set(\"packages\", external_conf)\n\n # Install the external spec\n middle_pkg = spack.concretize.concretize_one(\"middle\")\n PackageInstaller([middle_pkg.package], fake=True, explicit=True).install()\n assert middle_pkg[\"changing\"].external\n changing_external = middle_pkg[\"changing\"]\n\n # Modify the package.py file\n repo_with_changing_recipe.change({\"delete_variant\": True})\n\n # Try to concretize the external without reuse and confirm the hash changed\n with spack.config.override(\"concretizer:reuse\", False):\n root_no_reuse = spack.concretize.concretize_one(\"root\")\n assert root_no_reuse[\"changing\"].dag_hash() != changing_external.dag_hash()\n\n # ... while with reuse we have the same hash\n with spack.config.override(\"concretizer:reuse\", True):\n root_with_reuse = spack.concretize.concretize_one(\"root\")\n assert root_with_reuse[\"changing\"].dag_hash() == changing_external.dag_hash()\n\n @pytest.mark.regression(\"31484\")\n def test_user_can_select_externals_with_require(\n self, mutable_database, tmp_path: pathlib.Path\n ):\n \"\"\"Test that users have means to select an external even in presence of reusable specs.\"\"\"\n external_conf: Dict[str, Any] = {\n \"mpi\": {\"buildable\": False},\n \"multi-provider-mpi\": {\n \"externals\": [{\"spec\": \"multi-provider-mpi@2.0.0\", \"prefix\": str(tmp_path)}]\n },\n }\n spack.config.set(\"packages\", external_conf)\n\n # mpich and others are installed, so check that\n # fresh use the external, reuse does not\n with spack.config.override(\"concretizer:reuse\", False):\n mpi_spec = spack.concretize.concretize_one(\"mpi\")\n assert mpi_spec.name == \"multi-provider-mpi\"\n\n with spack.config.override(\"concretizer:reuse\", True):\n mpi_spec = spack.concretize.concretize_one(\"mpi\")\n assert mpi_spec.name != \"multi-provider-mpi\"\n\n external_conf[\"mpi\"][\"require\"] = \"multi-provider-mpi\"\n spack.config.set(\"packages\", external_conf)\n\n with spack.config.override(\"concretizer:reuse\", True):\n mpi_spec = spack.concretize.concretize_one(\"mpi\")\n assert mpi_spec.name == \"multi-provider-mpi\"\n\n @pytest.mark.regression(\"31484\")\n def test_installed_specs_disregard_conflicts(self, mutable_database, monkeypatch):\n \"\"\"Test that installed specs do not trigger conflicts. This covers for the rare case\n where a conflict is added on a package after a spec matching the conflict was installed.\n \"\"\"\n # Add a conflict to \"mpich\" that match an already installed \"mpich~debug\"\n pkg_cls = spack.repo.PATH.get_pkg_class(\"mpich\")\n monkeypatch.setitem(pkg_cls.conflicts, Spec(), [(Spec(\"~debug\"), None)])\n\n # If we concretize with --fresh the conflict is taken into account\n with spack.config.override(\"concretizer:reuse\", False):\n s = spack.concretize.concretize_one(\"mpich\")\n assert s.satisfies(\"+debug\")\n\n # If we concretize with --reuse it is not, since \"mpich~debug\" was already installed\n with spack.config.override(\"concretizer:reuse\", True):\n s = spack.concretize.concretize_one(\"mpich\")\n assert s.installed\n assert s.satisfies(\"~debug\"), s\n\n @pytest.mark.regression(\"32471\")\n def test_require_targets_are_allowed(self, mutable_config, mutable_database):\n \"\"\"Test that users can set target constraints under the require attribute.\"\"\"\n # Configuration to be added to packages.yaml\n required_target = spack.vendor.archspec.cpu.TARGETS[\n spack.platforms.test.Test.default\n ].family\n external_conf = {\"all\": {\"require\": f\"target={required_target}\"}}\n mutable_config.set(\"packages\", external_conf)\n\n with spack.config.override(\"concretizer:reuse\", False):\n spec = spack.concretize.concretize_one(\"mpich\")\n\n for s in spec.traverse(deptype=(\"link\", \"run\")):\n assert s.satisfies(f\"target={required_target}\")\n\n target = spack.platforms.test.Test.default\n\n def test_external_python_extension_find_dependency_from_config(self, mutable_config, tmp_path):\n \"\"\"Tests that an external Python extension gets a dependency on Python.\"\"\"\n packages_yaml = f\"\"\"\npackages:\n py-extension1:\n buildable: false\n externals:\n - spec: py-extension1@2.0\n prefix: {tmp_path / \"py-extension1\"}\n python:\n externals:\n - spec: python@3.8.13\n prefix: {tmp_path / \"python\"}\n\"\"\"\n configuration = syaml.load_config(packages_yaml)\n mutable_config.set(\"packages\", configuration[\"packages\"])\n py_extension = spack.concretize.concretize_one(\"py-extension1\")\n\n assert py_extension.external\n assert py_extension[\"python\"].external\n assert py_extension[\"python\"].prefix == str(tmp_path / \"python\")\n\n @pytest.mark.regression(\"36190\")\n @pytest.mark.parametrize(\n \"specs\",\n [\n [\"mpileaks^ callpath ^dyninst@8.1.1:8 ^mpich2@1.3:1\"],\n [\"multivalue-variant ^pkg-a@2:2\"],\n [\"v1-consumer ^conditional-provider@1:1 +disable-v1\"],\n ],\n )\n def test_result_specs_is_not_empty(self, mutable_config, specs):\n \"\"\"Check that the implementation of \"result.specs\" is correct in cases where we\n know a concretization exists.\n \"\"\"\n specs = [Spec(s) for s in specs]\n packages_with_externals = external_config_with_implicit_externals(mutable_config)\n completion_mode = mutable_config.get(\"concretizer:externals:completion\")\n external_specs = spec_filter_from_packages_yaml(\n external_parser=create_external_parser(packages_with_externals, completion_mode),\n packages_with_externals=packages_with_externals,\n include=[],\n exclude=[],\n ).selected_specs()\n solver = spack.solver.asp.Solver()\n setup = spack.solver.asp.SpackSolverSetup()\n result, _, _ = solver.driver.solve(setup, specs, reuse=external_specs)\n assert result.specs\n\n @pytest.mark.regression(\"38664\")\n def test_unsolved_specs_raises_error(self, monkeypatch, mock_packages):\n \"\"\"Check that the solver raises an exception when input specs are not\n satisfied.\n \"\"\"\n specs = [Spec(\"zlib\")]\n solver = spack.solver.asp.Solver()\n setup = spack.solver.asp.SpackSolverSetup()\n\n simulate_unsolved_property = list((x, None) for x in specs)\n\n monkeypatch.setattr(spack.solver.asp.Result, \"unsolved_specs\", simulate_unsolved_property)\n\n with pytest.raises(\n spack.solver.asp.InternalConcretizerError,\n match=\"the solver completed but produced specs\",\n ):\n solver.driver.solve(setup, specs, reuse=[])\n\n @pytest.mark.regression(\"43141\")\n @pytest.mark.parametrize(\n \"spec_str,expected_match\",\n [\n # A package does not exist\n (\"pkg-a ^foo\", \"since 'foo' does not exist\"),\n # Request a compiler for a package that doesn't need it\n (\"pkg-c %gcc\", \"cannot depend on gcc\"),\n ],\n )\n def test_errors_on_statically_checked_preconditions(self, spec_str, expected_match):\n \"\"\"Tests that the solver can report a case where the compiler cannot be set\"\"\"\n with pytest.raises(spack.error.UnsatisfiableSpecError, match=expected_match):\n spack.concretize.concretize_one(spec_str)\n\n @pytest.mark.regression(\"36339\")\n @pytest.mark.parametrize(\n \"compiler_str,expected\",\n [\n (\"gcc@:9\", \"@=9.4.0\"),\n (\"gcc@:10\", \"@=10.2.1\"),\n (\"gcc@10\", \"@=10.2.1\"),\n (\"gcc@10:\", \"@=10.2.1\"),\n ],\n )\n def test_compiler_match_constraints_when_selected(self, compiler_str, expected):\n \"\"\"Test that, when multiple compilers with the same name are in the configuration\n we ensure that the selected one matches all the required constraints.\n \"\"\"\n s = spack.concretize.concretize_one(f\"pkg-a %{compiler_str}\")\n assert s[\"gcc\"].satisfies(expected)\n\n @pytest.mark.parametrize(\"spec_str\", [\"mpileaks\", \"mpileaks ^mpich\"])\n def test_virtuals_are_annotated_on_edges(self, spec_str):\n \"\"\"Tests that information on virtuals is annotated on DAG edges\"\"\"\n spec = spack.concretize.concretize_one(spec_str)\n mpi_provider = spec[\"mpi\"].name\n\n edges = spec.edges_to_dependencies(name=mpi_provider)\n assert len(edges) == 1 and edges[0].virtuals == (\"mpi\",)\n edges = spec.edges_to_dependencies(name=\"callpath\")\n assert len(edges) == 1 and edges[0].virtuals == ()\n\n @pytest.mark.parametrize(\"transitive\", [True, False])\n def test_explicit_splices(\n self, mutable_config, database_mutable_config, mock_packages, transitive, capfd\n ):\n mpich_spec = database_mutable_config.query(\"mpich\")[0]\n splice_info = {\n \"target\": \"mpi\",\n \"replacement\": f\"/{mpich_spec.dag_hash()}\",\n \"transitive\": transitive,\n }\n spack.config.CONFIG.set(\"concretizer\", {\"splice\": {\"explicit\": [splice_info]}})\n\n spec = spack.concretize.concretize_one(\"hdf5 ^zmpi\")\n\n assert spec.satisfies(f\"^mpich@{mpich_spec.version}\")\n assert spec.build_spec.dependencies(name=\"zmpi\", deptype=\"link\")\n assert spec[\"mpi\"].build_spec.satisfies(mpich_spec)\n assert not spec.build_spec.satisfies(f\"^mpich/{mpich_spec.dag_hash()}\")\n assert not spec.dependencies(name=\"zmpi\", deptype=\"link\")\n\n captured = capfd.readouterr()\n assert \"Warning: explicit splice configuration has caused\" in captured.err\n assert \"hdf5 ^zmpi\" in captured.err\n assert str(spec) in captured.err\n\n def test_explicit_splice_fails_nonexistent(mutable_config, mock_packages, mock_store):\n splice_info = {\"target\": \"mpi\", \"replacement\": \"mpich/doesnotexist\"}\n spack.config.CONFIG.set(\"concretizer\", {\"splice\": {\"explicit\": [splice_info]}})\n\n with pytest.raises(spack.spec.InvalidHashError):\n _ = spack.concretize.concretize_one(\"hdf5^zmpi\")\n\n def test_explicit_splice_fails_no_hash(mutable_config, mock_packages, mock_store):\n splice_info = {\"target\": \"mpi\", \"replacement\": \"mpich\"}\n spack.config.CONFIG.set(\"concretizer\", {\"splice\": {\"explicit\": [splice_info]}})\n\n with pytest.raises(spack.solver.asp.InvalidSpliceError, match=\"must be specified by hash\"):\n _ = spack.concretize.concretize_one(\"hdf5^zmpi\")\n\n def test_explicit_splice_non_match_nonexistent_succeeds(\n mutable_config, mock_packages, mock_store\n ):\n \"\"\"When we have a nonexistent splice configured but are not using it, don't fail.\"\"\"\n splice_info = {\"target\": \"will_not_match\", \"replacement\": \"nonexistent/doesnotexist\"}\n spack.config.CONFIG.set(\"concretizer\", {\"splice\": {\"explicit\": [splice_info]}})\n spec = spack.concretize.concretize_one(\"zlib\")\n # the main test is that it does not raise\n assert not spec.spliced\n\n @pytest.mark.db\n @pytest.mark.parametrize(\n \"spec_str,mpi_name\",\n [(\"mpileaks\", \"mpich\"), (\"mpileaks ^mpich2\", \"mpich2\"), (\"mpileaks ^zmpi\", \"zmpi\")],\n )\n def test_virtuals_are_reconstructed_on_reuse(self, spec_str, mpi_name, mutable_database):\n \"\"\"Tests that when we reuse a spec, virtual on edges are reconstructed correctly\"\"\"\n with spack.config.override(\"concretizer:reuse\", True):\n spec = spack.concretize.concretize_one(spec_str)\n assert spec.installed\n mpi_edges = spec.edges_to_dependencies(mpi_name)\n assert len(mpi_edges) == 1\n assert \"mpi\" in mpi_edges[0].virtuals\n\n def test_dont_define_new_version_from_input_if_checksum_required(self, working_env):\n os.environ[\"SPACK_CONCRETIZER_REQUIRE_CHECKSUM\"] = \"yes\"\n with pytest.raises(spack.error.UnsatisfiableSpecError):\n # normally spack concretizes to @=3.0 if it's not defined in package.py, except\n # when checksums are required\n spack.concretize.concretize_one(\"pkg-a@=3.0\")\n\n @pytest.mark.regression(\"39570\")\n @pytest.mark.db\n def test_reuse_python_from_cli_and_extension_from_db(self, mutable_database):\n \"\"\"Tests that reusing python with and explicit request on the command line, when the spec\n also reuses a python extension from the DB, doesn't fail.\n \"\"\"\n s = spack.concretize.concretize_one(\"py-extension1\")\n python_hash = s[\"python\"].dag_hash()\n PackageInstaller([s.package], fake=True, explicit=True).install()\n\n with spack.config.override(\"concretizer:reuse\", True):\n with_reuse = spack.concretize.concretize_one(f\"py-extension2 ^/{python_hash}\")\n\n with spack.config.override(\"concretizer:reuse\", False):\n without_reuse = spack.concretize.concretize_one(\"py-extension2\")\n\n assert with_reuse.dag_hash() == without_reuse.dag_hash()\n\n @pytest.mark.regression(\"35536\")\n @pytest.mark.parametrize(\n \"spec_str,expected_namespaces\",\n [\n # Single node with fully qualified namespace\n (\"builtin_mock.gmake\", {\"gmake\": \"builtin_mock\"}),\n # Dependency with fully qualified namespace\n (\"hdf5 ^builtin_mock.gmake\", {\"gmake\": \"builtin_mock\", \"hdf5\": \"duplicates_test\"}),\n (\"hdf5 ^gmake\", {\"gmake\": \"duplicates_test\", \"hdf5\": \"duplicates_test\"}),\n ],\n )\n def test_select_lower_priority_package_from_repository_stack(\n self, spec_str, expected_namespaces\n ):\n \"\"\"Tests that a user can explicitly select a lower priority, fully qualified dependency\n from cli.\n \"\"\"\n # 'builtin_mock\" and \"duplicates_test\" share a 'gmake' package\n additional_repo = os.path.join(\n spack.paths.test_repos_path, \"spack_repo\", \"duplicates_test\"\n )\n with spack.repo.use_repositories(additional_repo, override=False):\n s = spack.concretize.concretize_one(spec_str)\n\n for name, namespace in expected_namespaces.items():\n assert s[name].concrete\n assert s[name].namespace == namespace\n\n def test_reuse_specs_from_non_available_compilers(self, mutable_config, mutable_database):\n \"\"\"Tests that we can reuse specs with compilers that are not configured locally.\"\"\"\n # All the specs in the mutable DB have been compiled with %gcc@10.2.1\n mpileaks = [s for s in mutable_database.query_local() if s.name == \"mpileaks\"]\n\n # Remove gcc@10.2.1\n remover = spack.compilers.config.CompilerRemover(mutable_config)\n remover.mark_compilers(match=\"gcc@=10.2.1\")\n remover.flush()\n mutable_config.set(\"concretizer:reuse\", True)\n\n # mpileaks is in the database, it will be reused with gcc@=10.2.1\n root = spack.concretize.concretize_one(\"mpileaks\")\n assert root.satisfies(\"%gcc@10.2.1\")\n assert any(root.dag_hash() == x.dag_hash() for x in mpileaks)\n\n # fftw is not in the database, therefore it will be compiled with gcc@=9.4.0\n root = spack.concretize.concretize_one(\"fftw~mpi\")\n assert root.satisfies(\"%gcc@9.4.0\")\n\n @pytest.mark.regression(\"43406\")\n def test_externals_with_platform_explicitly_set(self, tmp_path: pathlib.Path):\n \"\"\"Tests that users can specify platform=xxx in an external spec\"\"\"\n external_conf = {\n \"mpich\": {\n \"buildable\": False,\n \"externals\": [{\"spec\": \"mpich@=2.0.0 platform=test\", \"prefix\": str(tmp_path)}],\n }\n }\n spack.config.set(\"packages\", external_conf)\n s = spack.concretize.concretize_one(\"mpich\")\n assert s.external\n\n @pytest.mark.regression(\"43267\")\n def test_spec_with_build_dep_from_json(self, tmp_path: pathlib.Path):\n \"\"\"Tests that we can correctly concretize a spec, when we express its dependency as a\n concrete spec to be read from JSON.\n\n The bug was triggered by missing virtuals on edges that were trimmed from pure build\n dependencies.\n \"\"\"\n build_dep = spack.concretize.concretize_one(\"dttop\")\n json_file = tmp_path / \"build.json\"\n json_file.write_text(build_dep.to_json())\n s = spack.concretize.concretize_one(f\"dtuse ^{str(json_file)}\")\n assert s[\"dttop\"].dag_hash() == build_dep.dag_hash()\n\n @pytest.mark.regression(\"44040\")\n def test_exclude_specs_from_reuse(self, monkeypatch):\n r\"\"\"Tests that we can exclude a spec from reuse when concretizing, and that the spec\n is not added back to the solve as a dependency of another reusable spec.\n\n The expected spec is:\n\n o callpath@1.0\n |\\\n o | mpich@3.0.4\n |\\ \\\n | |\\ \\\n | | | o dyninst@8.2\n | |_|/|\n |/| |/|\n | |/|/|\n | | | |\\\n | | | | o libdwarf@20130729\n | |_|_|/|\n |/| |_|/|\n | |/| |/|\n | | |/|/\n | | | o libelf@0.8.13\n | |_|/|\n |/| |/|\n | |/|/\n | o | gcc-runtime@10.5.0\n |/| |\n | |/\n o | glibc@2.31\n /\n o gcc@10.5.0\n \"\"\"\n # Prepare a mock mirror that returns an old version of dyninst\n request_str = \"callpath ^mpich\"\n reused = spack.concretize.concretize_one(f\"{request_str} ^dyninst@8.1.1\")\n monkeypatch.setattr(spack.solver.reuse, \"_specs_from_mirror\", lambda: [reused])\n\n # Exclude dyninst from reuse, so we expect that the old version is not taken into account\n with spack.config.override(\n \"concretizer:reuse\",\n {\"from\": [{\"type\": \"buildcache\", \"exclude\": [\"dyninst\"]}, {\"type\": \"external\"}]},\n ):\n result = spack.concretize.concretize_one(request_str)\n\n assert result.dag_hash() != reused.dag_hash()\n assert result[\"mpich\"].dag_hash() == reused[\"mpich\"].dag_hash()\n assert result[\"dyninst\"].dag_hash() != reused[\"dyninst\"].dag_hash()\n assert result[\"dyninst\"].satisfies(\"@=8.2\")\n for dep in result[\"dyninst\"].traverse(root=False):\n assert dep.dag_hash() == reused[dep.name].dag_hash()\n\n @pytest.mark.regression(\"44091\")\n @pytest.mark.parametrize(\n \"included_externals\",\n [\n [\"deprecated-versions\"],\n # Try the empty list, to ensure that in that case everything will be included\n # since filtering should happen only when the list is non-empty\n [],\n ],\n )\n def test_include_specs_from_externals_and_libcs(\n self, included_externals, mutable_config, tmp_path: pathlib.Path\n ):\n \"\"\"Tests that when we include specs from externals, we always include libcs.\"\"\"\n mutable_config.set(\n \"packages\",\n {\n \"deprecated-versions\": {\n \"externals\": [{\"spec\": \"deprecated-versions@1.1.0\", \"prefix\": str(tmp_path)}]\n }\n },\n )\n request_str = \"deprecated-client\"\n\n # When using the external the version is selected even if deprecated\n with spack.config.override(\n \"concretizer:reuse\", {\"from\": [{\"type\": \"external\", \"include\": included_externals}]}\n ):\n result = spack.concretize.concretize_one(request_str)\n\n assert result[\"deprecated-versions\"].satisfies(\"@1.1.0\")\n\n # When excluding it, we pick the non-deprecated version\n with spack.config.override(\n \"concretizer:reuse\",\n {\"from\": [{\"type\": \"external\", \"exclude\": [\"deprecated-versions\"]}]},\n ):\n result = spack.concretize.concretize_one(request_str)\n\n assert result[\"deprecated-versions\"].satisfies(\"@1.0.0\")\n\n @pytest.mark.regression(\"44085\")\n def test_can_reuse_concrete_externals_for_dependents(self, mutable_config):\n \"\"\"Test that external specs that are in the DB can be reused. This means they are\n preferred to concretizing another external from packages.yaml\n \"\"\"\n packages_yaml = {\n \"externaltool\": {\"externals\": [{\"spec\": \"externaltool@0.9\", \"prefix\": \"/fake/path\"}]}\n }\n mutable_config.set(\"packages\", packages_yaml)\n # Concretize with v0.9 to get a suboptimal spec, since we have gcc@10 available\n external_spec = spack.concretize.concretize_one(\"externaltool@0.9\")\n assert external_spec.external\n\n root_specs = [Spec(\"sombrero\")]\n with spack.config.override(\"concretizer:reuse\", True):\n solver = spack.solver.asp.Solver()\n setup = spack.solver.asp.SpackSolverSetup()\n result, _, _ = solver.driver.solve(setup, root_specs, reuse=[external_spec])\n\n assert len(result.specs) == 1\n sombrero = result.specs[0]\n assert sombrero[\"externaltool\"].dag_hash() == external_spec.dag_hash()\n\n def test_cannot_reuse_host_incompatible_libc(self):\n \"\"\"Test whether reuse concretization correctly fails to reuse a spec with a host\n incompatible libc.\"\"\"\n if not spack.solver.core.using_libc_compatibility():\n pytest.skip(\"This test requires libc nodes\")\n\n # We install b@1 ^glibc@2.30, and b@0 ^glibc@2.28. The former is not host compatible, the\n # latter is.\n fst = spack.concretize.concretize_one(\"pkg-b@1\")\n fst._mark_concrete(False)\n fst.dependencies(\"glibc\")[0].versions = VersionList([\"=2.30\"])\n fst._mark_concrete(True)\n snd = spack.concretize.concretize_one(\"pkg-b@0\")\n\n # The spec b@1 ^glibc@2.30 is \"more optimal\" than b@0 ^glibc@2.28, but due to glibc\n # incompatibility, it should not be reused.\n solver = spack.solver.asp.Solver()\n setup = spack.solver.asp.SpackSolverSetup()\n result, _, _ = solver.driver.solve(setup, [Spec(\"pkg-b\")], reuse=[fst, snd])\n assert len(result.specs) == 1\n assert result.specs[0] == snd\n\n @pytest.mark.regression(\"45321\")\n @pytest.mark.parametrize(\n \"corrupted_str\",\n [\n \"cmake@3.4.3 foo=bar\", # cmake has no variant \"foo\"\n \"mvdefaults@1.0 foo=a,d\", # variant \"foo\" has no value \"d\"\n \"cmake %gcc\", # spec has no version\n ],\n )\n def test_corrupted_external_does_not_halt_concretization(self, corrupted_str, mutable_config):\n \"\"\"Tests that having a wrong variant in an external spec doesn't stop concretization\"\"\"\n corrupted_spec = Spec(corrupted_str)\n packages_yaml = {\n f\"{corrupted_spec.name}\": {\n \"externals\": [{\"spec\": corrupted_str, \"prefix\": \"/dev/null\"}]\n }\n }\n mutable_config.set(\"packages\", packages_yaml)\n # Assert we don't raise due to the corrupted external entry above\n s = spack.concretize.concretize_one(\"pkg-a\")\n assert s.concrete\n\n @pytest.mark.regression(\"44828\")\n @pytest.mark.not_on_windows(\"Tests use linux paths\")\n def test_correct_external_is_selected_from_packages_yaml(self, mutable_config):\n \"\"\"Tests that when filtering external specs, the correct external is selected to\n reconstruct the prefix, and other external attributes.\n \"\"\"\n packages_yaml = {\n \"mpileaks\": {\n \"externals\": [\n {\"spec\": \"mpileaks@2.3 +opt\", \"prefix\": \"/tmp/prefix1\"},\n {\"spec\": \"mpileaks@2.3 ~opt\", \"prefix\": \"/tmp/prefix2\"},\n ]\n }\n }\n concretizer_yaml = {\n \"reuse\": {\"roots\": True, \"from\": [{\"type\": \"external\", \"exclude\": [\"+opt\"]}]}\n }\n mutable_config.set(\"packages\", packages_yaml)\n mutable_config.set(\"concretizer\", concretizer_yaml)\n\n s = spack.concretize.concretize_one(\"mpileaks\")\n\n # Check that we got the properties from the right external\n assert s.external\n assert s.satisfies(\"~opt\")\n assert s.prefix == \"/tmp/prefix2\"\n\n def test_git_based_version_must_exist_to_use_ref(self):\n # gmake should fail, only has sha256\n with pytest.raises(spack.error.UnsatisfiableSpecError) as e:\n spack.concretize.concretize_one(f\"gmake commit={'a' * 40}\")\n assert \"Cannot use commit variant with\" in e.value.message\n\n\n@pytest.fixture()\ndef duplicates_test_repository():\n repository_path = os.path.join(spack.paths.test_repos_path, \"spack_repo\", \"duplicates_test\")\n with spack.repo.use_repositories(repository_path) as mock_repo:\n yield mock_repo\n\n\n@pytest.mark.usefixtures(\"mutable_config\", \"duplicates_test_repository\")\nclass TestConcretizeSeparately:\n \"\"\"Collects test on separate concretization\"\"\"\n\n @pytest.mark.parametrize(\"strategy\", [\"minimal\", \"full\"])\n def test_two_gmake(self, strategy):\n \"\"\"Tests that we can concretize a spec with nodes using the same build\n dependency pinned at different versions.\n\n o hdf5@1.0\n |\\\n o | pinned-gmake@1.0\n o | gmake@3.0\n /\n o gmake@4.1\n\n \"\"\"\n spack.config.CONFIG.set(\"concretizer:duplicates:strategy\", strategy)\n s = spack.concretize.concretize_one(\"hdf5\")\n\n # Check that hdf5 depends on gmake@=4.1\n hdf5_gmake = s[\"hdf5\"].dependencies(name=\"gmake\", deptype=\"build\")\n assert len(hdf5_gmake) == 1 and hdf5_gmake[0].satisfies(\"@=4.1\")\n\n # Check that pinned-gmake depends on gmake@=3.0\n pinned_gmake = s[\"pinned-gmake\"].dependencies(name=\"gmake\", deptype=\"build\")\n assert len(pinned_gmake) == 1 and pinned_gmake[0].satisfies(\"@=3.0\")\n\n @pytest.mark.parametrize(\"strategy\", [\"minimal\", \"full\"])\n def test_two_setuptools(self, strategy):\n \"\"\"Tests that we can concretize separate build dependencies, when we are dealing\n with extensions.\n\n o py-shapely@1.25.0\n |\\\n | |\\\n | o | py-setuptools@60\n |/ /\n | o py-numpy@1.25.0\n |/|\n | |\\\n | o | py-setuptools@59\n |/ /\n o | python@3.11.2\n o | gmake@3.0\n /\n o gmake@4.1\n\n \"\"\"\n spack.config.CONFIG.set(\"concretizer:duplicates:strategy\", strategy)\n s = spack.concretize.concretize_one(\"py-shapely\")\n # Requirements on py-shapely\n setuptools = s[\"py-shapely\"].dependencies(name=\"py-setuptools\", deptype=\"build\")\n assert len(setuptools) == 1 and setuptools[0].satisfies(\"@=60\")\n\n # Requirements on py-numpy\n setuptools = s[\"py-numpy\"].dependencies(name=\"py-setuptools\", deptype=\"build\")\n assert len(setuptools) == 1 and setuptools[0].satisfies(\"@=59\")\n gmake = s[\"py-numpy\"].dependencies(name=\"gmake\", deptype=\"build\")\n assert len(gmake) == 1 and gmake[0].satisfies(\"@=4.1\")\n\n # Requirements on python\n gmake = s[\"python\"].dependencies(name=\"gmake\", deptype=\"build\")\n assert len(gmake) == 1 and gmake[0].satisfies(\"@=3.0\")\n\n def test_solution_without_cycles(self):\n \"\"\"Tests that when we concretize a spec with cycles, a fallback kicks in to recompute\n a solution without cycles.\n \"\"\"\n s = spack.concretize.concretize_one(\"cycle-a\")\n assert s[\"cycle-a\"].satisfies(\"+cycle\")\n assert s[\"cycle-b\"].satisfies(\"~cycle\")\n\n s = spack.concretize.concretize_one(\"cycle-b\")\n assert s[\"cycle-a\"].satisfies(\"~cycle\")\n assert s[\"cycle-b\"].satisfies(\"+cycle\")\n\n @pytest.mark.parametrize(\"strategy\", [\"minimal\", \"full\"])\n def test_pure_build_virtual_dependency(self, strategy):\n \"\"\"Tests that we can concretize a pure build virtual dependency, and ensures that\n pure build virtual dependencies are accounted in the list of possible virtual\n dependencies.\n\n virtual-build@1.0\n | [type=build, virtual=pkgconfig]\n pkg-config@1.0\n \"\"\"\n spack.config.CONFIG.set(\"concretizer:duplicates:strategy\", strategy)\n\n s = spack.concretize.concretize_one(\"virtual-build\")\n assert s[\"pkgconfig\"].name == \"pkg-config\"\n\n @pytest.mark.regression(\"40595\")\n def test_no_multiple_solutions_with_different_edges_same_nodes(self):\n r\"\"\"Tests that the root node, which has a dependency on py-setuptools without constraint,\n doesn't randomly pick one of the two setuptools (@=59, @=60) needed by its dependency.\n\n o py-floating@1.25.0/3baitsp\n |\\\n | |\\\n | | |\\\n | o | | py-shapely@1.25.0/4hep6my\n |/| | |\n | |\\| |\n | | |/\n | |/|\n | | o py-setuptools@60/cwhbthc\n | |/\n |/|\n | o py-numpy@1.25.0/5q5fx4d\n |/|\n | |\\\n | o | py-setuptools@59/jvsa7sd\n |/ /\n o | python@3.11.2/pdmjekv\n o | gmake@3.0/jv7k2bl\n /\n o gmake@4.1/uo6ot3d\n \"\"\"\n spec_str = \"py-floating\"\n\n root = spack.concretize.concretize_one(spec_str)\n assert root[\"py-shapely\"].satisfies(\"^py-setuptools@=60\")\n assert root[\"py-numpy\"].satisfies(\"^py-setuptools@=59\")\n\n edges = root.edges_to_dependencies(\"py-setuptools\")\n assert len(edges) == 1\n assert edges[0].spec.satisfies(\"@=60\")\n\n def test_build_environment_is_unified(self):\n \"\"\"A pure build dep that is marked build-tool can creates its own unification set. This\n test ensures that its sibling build dependencies are unified with it, together with their\n runtime dependencies. It ensures the same package cannot appear multiple times in a single\n build environment, for example when it's both a direct build dep, as well as pulled in as\n a transitive runtime dep of a sibling build dep.\"\"\"\n spack.config.CONFIG.set(\"concretizer:duplicates\", {\"max_dupes\": {\"unify-build-deps-c\": 2}})\n\n # Fails because unify-build-deps-c version @1 and @2 are needed in the build environment\n with pytest.raises(spack.solver.asp.UnsatisfiableSpecError):\n spack.concretize.concretize_one(\"unify-build-deps-a@1.0\")\n\n # Succeeds because unify-build-deps-c version @2 is not needed in the build environment\n spack.concretize.concretize_one(\"unify-build-deps-a@2.0\")\n\n # Lastly, a sanity check that max_dupes is a requirement for this to work.\n spack.config.CONFIG.set(\"concretizer:duplicates\", {\"max_dupes\": {\"unify-build-deps-c\": 1}})\n with pytest.raises(spack.solver.asp.UnsatisfiableSpecError):\n spack.concretize.concretize_one(\"unify-build-deps-a@2.0\")\n\n @pytest.mark.regression(\"43647\")\n def test_specifying_different_versions_build_deps(self):\n \"\"\"Tests that we can concretize a spec with nodes using the same build\n dependency pinned at different versions, when the constraint is specified\n in the root spec.\n\n o hdf5@1.0\n |\\\n o | pinned-gmake@1.0\n o | gmake@3.0\n /\n o gmake@4.1\n\n \"\"\"\n hdf5_str = \"hdf5@1.0 ^gmake@4.1\"\n pinned_str = \"pinned-gmake@1.0 ^gmake@3.0\"\n input_specs = [Spec(hdf5_str), Spec(pinned_str)]\n solver = spack.solver.asp.Solver()\n result = solver.solve(input_specs)\n\n assert any(x.satisfies(hdf5_str) for x in result.specs)\n assert any(x.satisfies(pinned_str) for x in result.specs)\n\n @pytest.mark.regression(\"44289\")\n def test_all_extensions_depend_on_same_extendee(self):\n \"\"\"Tests that we don't reuse dependencies that bring in a different extendee\"\"\"\n setuptools = spack.concretize.concretize_one(\"py-setuptools ^python@3.10\")\n\n solver = spack.solver.asp.Solver()\n setup = spack.solver.asp.SpackSolverSetup()\n result, _, _ = solver.driver.solve(\n setup, [Spec(\"py-floating ^python@3.11\")], reuse=list(setuptools.traverse())\n )\n assert len(result.specs) == 1\n\n floating = result.specs[0]\n assert all(setuptools.dag_hash() != x.dag_hash() for x in floating.traverse())\n pythons = [x for x in floating.traverse() if x.name == \"python\"]\n assert len(pythons) == 1 and pythons[0].satisfies(\"@3.11\")\n\n\n@pytest.mark.parametrize(\n \"v_str,v_opts,checksummed\",\n [\n (\"1.2.3\", {\"sha256\": f\"{1:064x}\"}, True),\n # it's not about the version being \"infinite\",\n # but whether it has a digest\n (\"develop\", {\"sha256\": f\"{1:064x}\"}, True),\n # other hash types\n (\"1.2.3\", {\"checksum\": f\"{1:064x}\"}, True),\n (\"1.2.3\", {\"md5\": f\"{1:032x}\"}, True),\n (\"1.2.3\", {\"sha1\": f\"{1:040x}\"}, True),\n (\"1.2.3\", {\"sha224\": f\"{1:056x}\"}, True),\n (\"1.2.3\", {\"sha384\": f\"{1:096x}\"}, True),\n (\"1.2.3\", {\"sha512\": f\"{1:0128x}\"}, True),\n # no digest key\n (\"1.2.3\", {\"bogus\": f\"{1:064x}\"}, False),\n # git version with full commit sha\n (\"1.2.3\", {\"commit\": f\"{1:040x}\"}, True),\n (f\"{1:040x}=1.2.3\", {}, True),\n # git version with short commit sha\n (\"1.2.3\", {\"commit\": f\"{1:07x}\"}, False),\n (f\"{1:07x}=1.2.3\", {}, False),\n # git tag is a moving target\n (\"1.2.3\", {\"tag\": \"v1.2.3\"}, False),\n (\"1.2.3\", {\"tag\": \"v1.2.3\", \"commit\": f\"{1:07x}\"}, False),\n # git branch is a moving target\n (\"1.2.3\", {\"branch\": \"releases/1.2\"}, False),\n # git ref is a moving target\n (\"git.branch=1.2.3\", {}, False),\n ],\n)\ndef test_drop_moving_targets(v_str, v_opts, checksummed):\n v = Version(v_str)\n assert spack.solver.asp._is_checksummed_version((v, v_opts)) == checksummed\n\n\nclass TestConcreteSpecsByHash:\n \"\"\"Tests the container of concrete specs\"\"\"\n\n @pytest.mark.parametrize(\n \"input_specs\", [[\"pkg-a\"], [\"pkg-a foobar=bar\", \"pkg-b\"], [\"pkg-a foobar=baz\", \"pkg-b\"]]\n )\n def test_adding_specs(self, input_specs, default_mock_concretization):\n \"\"\"Tests that concrete specs in the container are equivalent, but stored as different\n objects in memory.\n \"\"\"\n container = spack.solver.asp.ConcreteSpecsByHash()\n input_specs = [spack.concretize.concretize_one(s) for s in input_specs]\n for s in input_specs:\n container.add(s)\n\n for root in input_specs:\n for node in root.traverse(root=True):\n assert node == container[node.dag_hash()]\n assert node.dag_hash() in container\n assert node is not container[node.dag_hash()]\n\n\n@pytest.fixture()\ndef edges_test_repository():\n repository_path = os.path.join(spack.paths.test_repos_path, \"spack_repo\", \"edges_test\")\n with spack.repo.use_repositories(repository_path) as mock_repo:\n yield mock_repo\n\n\n@pytest.mark.usefixtures(\"mutable_config\", \"edges_test_repository\")\nclass TestConcretizeEdges:\n \"\"\"Collects tests on edge properties\"\"\"\n\n @pytest.mark.parametrize(\n \"spec_str,expected_satisfies,expected_not_satisfies\",\n [\n (\"conditional-edge\", [\"^zlib@2.0\"], [\"^zlib-api\"]),\n (\"conditional-edge~foo\", [\"^zlib@2.0\"], [\"^zlib-api\"]),\n (\n \"conditional-edge+foo\",\n [\"^zlib@1.0\", \"^zlib-api\", \"^[virtuals=zlib-api] zlib\"],\n [\"^[virtuals=mpi] zlib\"],\n ),\n ],\n )\n def test_condition_triggered_by_edge_property(\n self, spec_str, expected_satisfies, expected_not_satisfies\n ):\n \"\"\"Tests that we can enforce constraints based on edge attributes\"\"\"\n s = spack.concretize.concretize_one(spec_str)\n\n for expected in expected_satisfies:\n assert s.satisfies(expected), str(expected)\n\n for not_expected in expected_not_satisfies:\n assert not s.satisfies(not_expected), str(not_expected)\n\n def test_virtuals_provided_together_but_only_one_required_in_dag(self):\n \"\"\"Tests that we can use a provider that provides more than one virtual together,\n and is providing only one, iff the others are not needed in the DAG.\n\n o blas-only-client\n | [virtual=blas]\n o openblas (provides blas and lapack together)\n\n \"\"\"\n s = spack.concretize.concretize_one(\"blas-only-client ^openblas\")\n assert s.satisfies(\"^[virtuals=blas] openblas\")\n assert not s.satisfies(\"^[virtuals=blas,lapack] openblas\")\n\n\ndef test_reusable_externals_match(mock_packages, tmp_path: pathlib.Path):\n spec = Spec(\"mpich@4.1~debug build_system=generic arch=linux-ubuntu23.04-zen2 %gcc@13.1.0\")\n spec.external_path = str(tmp_path)\n spec.external_modules = [\"mpich/4.1\"]\n spec._mark_concrete()\n assert spack.solver.reuse._is_reusable(\n spec,\n {\n \"mpich\": {\n \"externals\": [\n {\"spec\": \"mpich@4.1\", \"prefix\": str(tmp_path), \"modules\": [\"mpich/4.1\"]}\n ]\n }\n },\n local=False,\n )\n\n\ndef test_reusable_externals_match_virtual(mock_packages, tmp_path: pathlib.Path):\n spec = Spec(\"mpich@4.1~debug build_system=generic arch=linux-ubuntu23.04-zen2 %gcc@13.1.0\")\n spec.external_path = str(tmp_path)\n spec.external_modules = [\"mpich/4.1\"]\n spec._mark_concrete()\n assert spack.solver.reuse._is_reusable(\n spec,\n {\n \"mpi\": {\n \"externals\": [\n {\"spec\": \"mpich@4.1\", \"prefix\": str(tmp_path), \"modules\": [\"mpich/4.1\"]}\n ]\n }\n },\n local=False,\n )\n\n\ndef test_reusable_externals_different_prefix(mock_packages, tmp_path: pathlib.Path):\n spec = Spec(\"mpich@4.1~debug build_system=generic arch=linux-ubuntu23.04-zen2 %gcc@13.1.0\")\n spec.external_path = \"/other/path\"\n spec.external_modules = [\"mpich/4.1\"]\n spec._mark_concrete()\n assert not spack.solver.reuse._is_reusable(\n spec,\n {\n \"mpich\": {\n \"externals\": [\n {\"spec\": \"mpich@4.1\", \"prefix\": str(tmp_path), \"modules\": [\"mpich/4.1\"]}\n ]\n }\n },\n local=False,\n )\n\n\n@pytest.mark.parametrize(\"modules\", [None, [\"mpich/4.1\", \"libfabric/1.19\"]])\ndef test_reusable_externals_different_modules(mock_packages, tmp_path: pathlib.Path, modules):\n spec = Spec(\"mpich@4.1~debug build_system=generic arch=linux-ubuntu23.04-zen2 %gcc@13.1.0\")\n spec.external_path = str(tmp_path)\n spec.external_modules = modules\n spec._mark_concrete()\n assert not spack.solver.reuse._is_reusable(\n spec,\n {\n \"mpich\": {\n \"externals\": [\n {\"spec\": \"mpich@4.1\", \"prefix\": str(tmp_path), \"modules\": [\"mpich/4.1\"]}\n ]\n }\n },\n local=False,\n )\n\n\ndef test_reusable_externals_different_spec(mock_packages, tmp_path: pathlib.Path):\n spec = Spec(\"mpich@4.1~debug build_system=generic arch=linux-ubuntu23.04-zen2 %gcc@13.1.0\")\n spec.external_path = str(tmp_path)\n spec._mark_concrete()\n assert not spack.solver.reuse._is_reusable(\n spec,\n {\"mpich\": {\"externals\": [{\"spec\": \"mpich@4.1 +debug\", \"prefix\": str(tmp_path)}]}},\n local=False,\n )\n\n\ndef test_concretization_version_order():\n versions = [\n (Version(\"develop\"), {}),\n (Version(\"1.0\"), {}),\n (Version(\"2.0\"), {\"deprecated\": True}),\n (Version(\"1.1\"), {}),\n (Version(\"1.1alpha1\"), {}),\n (Version(\"0.9\"), {\"preferred\": True}),\n ]\n result = [\n v\n for v, _ in sorted(\n versions, key=spack.package_base.concretization_version_order, reverse=True\n )\n ]\n assert result == [\n Version(\"0.9\"), # preferred\n Version(\"2.0\"), # deprecation is accounted for separately\n Version(\"1.1\"), # latest non-deprecated final version\n Version(\"1.0\"), # latest non-deprecated final version\n Version(\"1.1alpha1\"), # prereleases\n Version(\"develop\"), # likely development version\n ]\n\n\n@pytest.mark.parametrize(\n \"roots,reuse_yaml,expected,not_expected,expected_length\",\n [\n (\n [\"mpileaks\"],\n {\"roots\": True, \"include\": [\"^mpich\"]},\n [\"^mpich\"],\n [\"^mpich2\", \"^zmpi\"],\n # Reused from store + externals\n 2 + 15,\n ),\n (\n [\"mpileaks\"],\n {\"roots\": True, \"include\": [\"externaltest\"]},\n [\"externaltest\"],\n [\"^mpich\", \"^mpich2\", \"^zmpi\"],\n # Reused from store + externals\n 1 + 15,\n ),\n ],\n)\n@pytest.mark.usefixtures(\"mutable_database\", \"mock_store\")\n@pytest.mark.not_on_windows(\"Expected length is different on Windows\")\ndef test_filtering_reused_specs(\n roots, reuse_yaml, expected, not_expected, expected_length, mutable_config\n):\n \"\"\"Tests that we can select which specs are to be reused, using constraints as filters\"\"\"\n # Assume all specs have a runtime dependency\n mutable_config.set(\"concretizer:reuse\", reuse_yaml)\n packages_with_externals = spack.solver.runtimes.external_config_with_implicit_externals(\n mutable_config\n )\n completion_mode = mutable_config.get(\"concretizer:externals:completion\")\n selector = spack.solver.asp.ReusableSpecsSelector(\n configuration=mutable_config,\n external_parser=create_external_parser(packages_with_externals, completion_mode),\n packages_with_externals=packages_with_externals,\n )\n specs = selector.reusable_specs(roots)\n\n assert len(specs) == expected_length\n\n for constraint in expected:\n assert all(x.satisfies(constraint) for x in specs if not x.external)\n\n for constraint in not_expected:\n assert all(not x.satisfies(constraint) for x in specs if not x.external)\n\n\n@pytest.mark.usefixtures(\"mutable_database\", \"mock_store\")\n@pytest.mark.parametrize(\n \"reuse_yaml,expected_length\",\n [\n (\n {\"from\": [{\"type\": \"local\"}]},\n # Local store + externals\n 19 + 15,\n ),\n (\n {\"from\": [{\"type\": \"buildcache\"}]},\n # Local store + externals\n 0 + 15,\n ),\n ],\n)\n@pytest.mark.not_on_windows(\"Expected length is different on Windows\")\ndef test_selecting_reused_sources(reuse_yaml, expected_length, mutable_config):\n \"\"\"Tests that we can turn on/off sources of reusable specs\"\"\"\n # Assume all specs have a runtime dependency\n mutable_config.set(\"concretizer:reuse\", reuse_yaml)\n packages_with_externals = spack.solver.runtimes.external_config_with_implicit_externals(\n mutable_config\n )\n completion_mode = mutable_config.get(\"concretizer:externals:completion\")\n selector = spack.solver.asp.ReusableSpecsSelector(\n configuration=mutable_config,\n external_parser=create_external_parser(packages_with_externals, completion_mode),\n packages_with_externals=packages_with_externals,\n )\n specs = selector.reusable_specs([\"mpileaks\"])\n assert len(specs) == expected_length\n\n # Compiler wrapper is not reused, as it might have changed from previous installations\n assert not [x for x in specs if x.name == \"compiler-wrapper\"]\n\n\n@pytest.mark.parametrize(\n \"specs,include,exclude,expected\",\n [\n # \"foo\" discarded by include rules (everything compiled with GCC)\n ([\"cmake@3.27.9 %gcc\", \"foo %clang\"], [\"%gcc\"], [], [\"cmake@3.27.9 %gcc\"]),\n # \"cmake\" discarded by exclude rules (everything compiled with GCC but cmake)\n ([\"cmake@3.27.9 %gcc\", \"foo %gcc\"], [\"%gcc\"], [\"cmake\"], [\"foo %gcc\"]),\n ],\n)\ndef test_spec_filters(specs, include, exclude, expected):\n specs = [Spec(x) for x in specs]\n expected = [Spec(x) for x in expected]\n f = spack.spec_filter.SpecFilter(\n factory=lambda: specs, is_usable=lambda x: True, include=include, exclude=exclude\n )\n assert f.selected_specs() == expected\n\n\n@pytest.mark.regression(\"38484\")\ndef test_git_ref_version_can_be_reused(install_mockery):\n first_spec = spack.concretize.concretize_one(\n spack.spec.Spec(\"git-ref-package@git.2.1.5=2.1.5~opt\")\n )\n PackageInstaller([first_spec.package], fake=True, explicit=True).install()\n\n with spack.config.override(\"concretizer:reuse\", True):\n # reproducer of the issue is that spack will solve when there is a change to the base spec\n second_spec = spack.concretize.concretize_one(\n spack.spec.Spec(\"git-ref-package@git.2.1.5=2.1.5+opt\")\n )\n assert second_spec.dag_hash() != first_spec.dag_hash()\n # we also want to confirm that reuse actually works so leave variant off to\n # let solver reuse\n third_spec = spack.spec.Spec(\"git-ref-package@git.2.1.5=2.1.5\")\n assert first_spec.satisfies(third_spec)\n third_spec = spack.concretize.concretize_one(third_spec)\n assert third_spec.dag_hash() == first_spec.dag_hash()\n\n\n@pytest.mark.parametrize(\"standard_version\", [\"2.0.0\", \"2.1.5\", \"2.1.6\"])\ndef test_reuse_prefers_standard_over_git_versions(standard_version, install_mockery):\n \"\"\"\n order matters in this test. typically reuse would pick the highest versioned installed match\n but we want to prefer the standard version over git ref based versions\n so install git ref last and ensure it is not picked up by reuse\n \"\"\"\n standard_spec = spack.concretize.concretize_one(\n spack.spec.Spec(f\"git-ref-package@{standard_version}\")\n )\n PackageInstaller([standard_spec.package], fake=True, explicit=True).install()\n\n git_spec = spack.concretize.concretize_one(\"git-ref-package@git.2.1.5=2.1.5\")\n PackageInstaller([git_spec.package], fake=True, explicit=True).install()\n\n with spack.config.override(\"concretizer:reuse\", True):\n test_spec = spack.concretize.concretize_one(\"git-ref-package@2\")\n assert git_spec.dag_hash() != test_spec.dag_hash()\n assert standard_spec.dag_hash() == test_spec.dag_hash()\n\n\n@pytest.mark.parametrize(\"unify\", [True, \"when_possible\", False])\ndef test_spec_unification(unify, mutable_config, mock_packages):\n spack.config.set(\"concretizer:unify\", unify)\n a = \"pkg-a\"\n a_restricted = \"pkg-a^pkg-b foo=baz\"\n b = \"pkg-b foo=none\"\n\n unrestricted = spack.cmd.parse_specs([a, b], concretize=True)\n a_concrete_unrestricted = [s for s in unrestricted if s.name == \"pkg-a\"][0]\n b_concrete_unrestricted = [s for s in unrestricted if s.name == \"pkg-b\"][0]\n assert (a_concrete_unrestricted[\"pkg-b\"] == b_concrete_unrestricted) == (unify is not False)\n\n maybe_fails = pytest.raises if unify is True else spack.llnl.util.lang.nullcontext\n with maybe_fails(spack.solver.asp.UnsatisfiableSpecError):\n _ = spack.cmd.parse_specs([a_restricted, b], concretize=True)\n\n\n@pytest.mark.not_on_windows(\"parallelism unsupported on Windows\")\n@pytest.mark.enable_parallelism\ndef test_parallel_concretization(mutable_config, mock_packages):\n \"\"\"Test whether parallel unify-false style concretization works.\"\"\"\n specs = [(Spec(\"pkg-a\"), None), (Spec(\"pkg-b\"), None)]\n result = spack.concretize.concretize_separately(specs)\n assert {s.name for s, _ in result} == {\"pkg-a\", \"pkg-b\"}\n\n\n@pytest.mark.usefixtures(\"mutable_config\", \"mock_packages\")\n@pytest.mark.parametrize(\n \"spec_str, error_type\",\n [\n (f\"git-ref-package@main commit={'a' * 40}\", None),\n (f\"git-ref-package@main commit={'a' * 39}\", AssertionError),\n (f\"git-ref-package@2.1.6 commit={'a' * 40}\", spack.error.UnsatisfiableSpecError),\n (f\"git-ref-package@git.2.1.6=2.1.6 commit={'a' * 40}\", None),\n (f\"git-ref-package@git.{'a' * 40}=2.1.6 commit={'a' * 40}\", None),\n ],\n)\ndef test_spec_containing_commit_variant(spec_str, error_type):\n spec = spack.spec.Spec(spec_str)\n if error_type is None:\n spack.concretize.concretize_one(spec)\n else:\n with pytest.raises(error_type):\n spack.concretize.concretize_one(spec)\n\n\n@pytest.mark.usefixtures(\"mutable_config\", \"mock_packages\")\n@pytest.mark.parametrize(\n \"spec_str\",\n [\n f\"git-test-commit@git.main commit={'a' * 40}\",\n f\"git-test-commit@git.v1.0 commit={'a' * 40}\",\n \"git-test-commit@{sha} commit={sha}\",\n \"git-test-commit@{sha} commit=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\n ],\n)\ndef test_spec_with_commit_interacts_with_lookup(mock_git_version_info, monkeypatch, spec_str):\n # This test will be short lived. Technically we could do further checks with a Lookup\n # but skipping impl since we are going to deprecate\n repo_path, filename, commits = mock_git_version_info\n file_url = pathlib.Path(repo_path).as_uri()\n monkeypatch.setattr(spack.package_base.PackageBase, \"git\", file_url, raising=False)\n spec = spack.spec.Spec(spec_str.format(sha=commits[-1]))\n spack.concretize.concretize_one(spec)\n\n\n@pytest.mark.usefixtures(\"mutable_config\", \"mock_packages\")\n@pytest.mark.parametrize(\"version_str\", [f\"git.{'a' * 40}=main\", \"git.2.1.5=main\"])\ndef test_relationship_git_versions_and_commit_variant(version_str):\n \"\"\"\n Confirm that GitVersions auto assign and populates the commit variant correctly\n \"\"\"\n # This should be a short lived test and can be deleted when we remove GitVersions\n spec = spack.spec.Spec(f\"git-ref-package@{version_str}\")\n spec = spack.concretize.concretize_one(spec)\n if spec.version.commit_sha:\n assert spec.version.commit_sha == spec.variants[\"commit\"].value\n else:\n assert \"commit\" not in spec.variants\n\n\n@pytest.mark.usefixtures(\"install_mockery\")\ndef test_abstract_commit_spec_reuse():\n commit = \"abcd\" * 10\n spec_str_1 = f\"git-ref-package@develop commit={commit}\"\n spec_str_2 = f\"git-ref-package commit={commit}\"\n spec1 = spack.concretize.concretize_one(spec_str_1)\n PackageInstaller([spec1.package], fake=True, explicit=True).install()\n\n with spack.config.override(\"concretizer:reuse\", True):\n spec2 = spack.concretize.concretize_one(spec_str_2)\n assert spec2.dag_hash() == spec1.dag_hash()\n\n\n@pytest.mark.usefixtures(\"install_mockery\")\n@pytest.mark.parametrize(\n \"installed_commit, incoming_commit, reusable\",\n [(\"a\" * 40, \"b\" * 40, False), (None, \"b\" * 40, False), (\"a\" * 40, None, True)],\n)\ndef test_commit_variant_can_be_reused(installed_commit, incoming_commit, reusable):\n # install a non-default variant to test if reuse picks it\n if installed_commit:\n spec_str_1 = f\"git-ref-package@develop commit={installed_commit} ~opt\"\n else:\n spec_str_1 = \"git-ref-package@develop ~opt\"\n\n if incoming_commit:\n spec_str_2 = f\"git-ref-package@develop commit={incoming_commit}\"\n else:\n spec_str_2 = \"git-ref-package@develop\"\n\n spec1 = spack.concretize.concretize_one(spack.spec.Spec(spec_str_1))\n PackageInstaller([spec1.package], fake=True, explicit=True).install()\n\n with spack.config.override(\"concretizer:reuse\", True):\n spec2 = spack.spec.Spec(spec_str_2)\n spec2 = spack.concretize.concretize_one(spec2)\n assert (spec1.dag_hash() == spec2.dag_hash()) == reusable\n\n\n@pytest.mark.regression(\"42679\")\n@pytest.mark.parametrize(\"compiler_str\", [\"gcc@=9.4.0\", \"gcc@=9.4.0-foo\"])\ndef test_selecting_compiler_with_suffix(mutable_config, mock_packages, compiler_str):\n \"\"\"Tests that we can select compilers whose versions differ only for a suffix.\"\"\"\n packages_yaml = syaml.load_config(\n \"\"\"\npackages:\n gcc:\n externals:\n - spec: \"gcc@9.4.0-foo languages='c,c++'\"\n prefix: /path\n extra_attributes:\n compilers:\n c: /path/bin/gcc\n cxx: /path/bin/g++\n\"\"\"\n )\n mutable_config.set(\"packages\", packages_yaml[\"packages\"])\n s = spack.concretize.concretize_one(f\"libelf %{compiler_str}\")\n assert s[\"c\"].satisfies(compiler_str)\n\n\ndef test_duplicate_compiler_in_externals(mutable_config, mock_packages):\n \"\"\"Tests that having duplicate compilers in packages.yaml do not raise and error.\"\"\"\n packages_yaml = syaml.load_config(\n \"\"\"\npackages:\n gcc:\n externals:\n - spec: \"gcc@9.4.0 languages='c,c++'\"\n prefix: /path\n extra_attributes:\n compilers:\n c: /path/bin/gcc\n cxx: /path/bin/g++\n - spec: \"gcc@9.4.0 languages='c,c++'\"\n prefix: /path\n extra_attributes:\n compilers:\n c: /path/bin/gcc\n cxx: /path/bin/g++\n\"\"\"\n )\n mutable_config.set(\"packages\", packages_yaml[\"packages\"])\n s = spack.concretize.concretize_one(\"libelf %gcc@9.4\")\n assert s[\"c\"].satisfies(\"gcc@9.4.0\")\n\n\n@pytest.mark.parametrize(\n \"spec_str,expected\",\n [\n (\"gcc@14 %gcc@9.4.0\", [\"gcc@14\", \"%c,cxx=gcc@9.4.0\", \"^gcc-runtime@9.4.0\"]),\n # If we don't specify a compiler, we should get the default compiler which is gcc\n (\"gcc@14\", [\"gcc@14\", \"%c,cxx=gcc@10\", \"^gcc-runtime@10\"]),\n ],\n)\ndef test_compiler_can_depend_on_themselves_to_build(\n spec_str, expected, default_mock_concretization\n):\n \"\"\"Tests that a compiler can depend on \"itself\" to bootstrap.\"\"\"\n s = default_mock_concretization(spec_str)\n assert not s.external\n for c in expected:\n assert s.satisfies(c)\n\n\ndef test_compiler_attribute_is_tolerated_in_externals(\n mutable_config, mock_packages, tmp_path: pathlib.Path\n):\n \"\"\"Tests that we don't error out if an external specifies a compiler in the old way,\n provided that a suitable external compiler exists.\n \"\"\"\n packages_yaml = syaml.load_config(\n f\"\"\"\npackages:\n cmake:\n externals:\n - spec: \"cmake@3.27.4 %gcc@10\"\n prefix: {tmp_path}\n buildable: false\n\"\"\"\n )\n mutable_config.set(\"packages\", packages_yaml[\"packages\"])\n s = spack.concretize.concretize_one(\"cmake\")\n assert s.external and s.external_path == str(tmp_path)\n\n\ndef test_compiler_can_be_built_with_other_compilers(config, mock_packages):\n \"\"\"Tests that a compiler can be built also with another compiler.\"\"\"\n s = spack.concretize.concretize_one(\"llvm@18 +clang %gcc\")\n assert s.satisfies(\"llvm@18\")\n\n c_compiler = s.dependencies(virtuals=(\"c\",))\n assert len(c_compiler) == 1 and c_compiler[0].satisfies(\"gcc@10\")\n\n\n@pytest.mark.parametrize(\n \"spec_str,expected\",\n [\n # Only one compiler is in the DAG, so pick the external associated with it\n (\"dyninst %clang\", \"clang\"),\n (\"dyninst %gcc\", \"gcc\"),\n # Both compilers are in the DAG, so pick the best external according to other criteria\n (\"dyninst %clang ^libdwarf%gcc\", \"clang\"),\n (\"dyninst %gcc ^libdwarf%clang\", \"clang\"),\n ],\n)\ndef test_compiler_match_for_externals_is_taken_into_account(\n spec_str, expected, mutable_config, mock_packages, tmp_path: pathlib.Path\n):\n \"\"\"Tests that compiler annotation for externals are somehow taken into account for a match\"\"\"\n packages_yaml = syaml.load_config(\n f\"\"\"\npackages:\n libelf:\n externals:\n - spec: \"libelf@0.8.12 %gcc@10\"\n prefix: {tmp_path / \"gcc\"}\n - spec: \"libelf@0.8.13 %clang\"\n prefix: {tmp_path / \"clang\"}\n\"\"\"\n )\n mutable_config.set(\"packages\", packages_yaml[\"packages\"])\n s = spack.concretize.concretize_one(spec_str)\n libelf = s[\"libelf\"]\n assert libelf.external and libelf.external_path == str(tmp_path / expected)\n\n\n@pytest.mark.parametrize(\n \"spec_str,expected\",\n [\n # Only one compiler is in the DAG, so pick the external associated with it\n (\"dyninst %gcc@10\", \"libelf-gcc10\"),\n (\"dyninst %gcc@9\", \"libelf-gcc9\"),\n # Both compilers are in the DAG, so pick the best external according to other criteria\n (\"dyninst %gcc@10 ^libdwarf%gcc@9\", \"libelf-gcc9\"),\n ],\n)\ndef test_compiler_match_for_externals_with_versions(\n spec_str, expected, mutable_config, mock_packages, tmp_path: pathlib.Path\n):\n \"\"\"Tests that version constraints are taken into account for compiler annotations\n on externals\n \"\"\"\n packages_yaml = syaml.load_config(\n f\"\"\"\npackages:\n libelf:\n buildable: false\n externals:\n - spec: \"libelf@0.8.12 %gcc@10\"\n prefix: {tmp_path / \"libelf-gcc10\"}\n - spec: \"libelf@0.8.13 %gcc@9.4.0\"\n prefix: {tmp_path / \"libelf-gcc9\"}\n\"\"\"\n )\n mutable_config.set(\"packages\", packages_yaml[\"packages\"])\n s = spack.concretize.concretize_one(spec_str)\n libelf = s[\"libelf\"]\n assert libelf.external and libelf.external_path == str(tmp_path / expected)\n\n\ndef test_specifying_compilers_with_virtuals_syntax(default_mock_concretization):\n \"\"\"Tests that we can pin compilers to nodes using the %[virtuals=...] syntax\"\"\"\n # clang will be used for both C and C++, since they are provided together\n mpich = default_mock_concretization(\"mpich %[virtuals=fortran] gcc %clang\")\n\n assert mpich[\"fortran\"].satisfies(\"gcc\")\n assert mpich[\"c\"].satisfies(\"llvm\")\n assert mpich[\"cxx\"].satisfies(\"llvm\")\n\n # gcc is the default compiler\n mpileaks = default_mock_concretization(\n \"mpileaks ^libdwarf %gcc ^mpich %[virtuals=fortran] gcc %clang\"\n )\n\n assert mpileaks[\"c\"].satisfies(\"gcc\")\n\n libdwarf = mpileaks[\"libdwarf\"]\n assert libdwarf[\"c\"].satisfies(\"gcc\")\n assert libdwarf[\"c\"].satisfies(\"gcc\")\n\n mpich = mpileaks[\"mpi\"]\n assert mpich[\"fortran\"].satisfies(\"gcc\")\n assert mpich[\"c\"].satisfies(\"llvm\")\n assert mpich[\"cxx\"].satisfies(\"llvm\")\n\n\n@pytest.mark.regression(\"49847\")\n@pytest.mark.xfail(sys.platform == \"win32\", reason=\"issues with install mockery\")\ndef test_reuse_when_input_specifies_build_dep(install_mockery):\n \"\"\"Test that we can reuse a spec when specifying build dependencies in the input\"\"\"\n pkgb_old = spack.concretize.concretize_one(spack.spec.Spec(\"pkg-b@0.9 %gcc@9\"))\n PackageInstaller([pkgb_old.package], fake=True, explicit=True).install()\n\n with spack.config.override(\"concretizer:reuse\", True):\n result = spack.concretize.concretize_one(\"pkg-b %gcc\")\n assert pkgb_old.dag_hash() == result.dag_hash()\n\n result = spack.concretize.concretize_one(\"pkg-a ^pkg-b %gcc@9\")\n assert pkgb_old.dag_hash() == result[\"pkg-b\"].dag_hash()\n assert result.satisfies(\"%gcc@9\")\n\n result = spack.concretize.concretize_one(\"pkg-a %gcc@10 ^pkg-b %gcc@9\")\n assert pkgb_old.dag_hash() == result[\"pkg-b\"].dag_hash()\n\n\n@pytest.mark.regression(\"49847\")\ndef test_reuse_when_requiring_build_dep(install_mockery, mutable_config):\n \"\"\"Test that we can reuse a spec when specifying build dependencies in requirements\"\"\"\n mutable_config.set(\"packages:all:require\", \"%gcc\")\n pkgb_old = spack.concretize.concretize_one(spack.spec.Spec(\"pkg-b@0.9\"))\n PackageInstaller([pkgb_old.package], fake=True, explicit=True).install()\n\n with spack.config.override(\"concretizer:reuse\", True):\n result = spack.concretize.concretize_one(\"pkg-b\")\n assert pkgb_old.dag_hash() == result.dag_hash(), result.tree()\n\n\n@pytest.mark.regression(\"50167\")\ndef test_input_analysis_and_conditional_requirements(default_mock_concretization):\n \"\"\"Tests that input analysis doesn't account for conditional requirement\n to discard possible dependencies.\n\n If the requirement is conditional, and impossible to achieve on the current\n platform, the valid search space is still the complement of the condition that\n activates the requirement.\n \"\"\"\n libceed = default_mock_concretization(\"libceed\")\n assert libceed[\"libxsmm\"].satisfies(\"@main\")\n assert libceed[\"libxsmm\"].satisfies(\"platform=test\")\n\n\n@pytest.mark.parametrize(\n \"compiler_str,expected,not_expected\",\n [\n # Compilers are matched to some other external, so the compiler that picked is concrete\n (\"gcc@10\", [\"%gcc\", \"%gcc@10\"], [\"%clang\", \"%gcc@9\"]),\n (\"gcc@9.4.0\", [\"%gcc\", \"%gcc@9\"], [\"%clang\", \"%gcc@10\"]),\n (\"clang\", [\"%clang\", \"%llvm+clang\"], [\"%gcc\", \"%gcc@9\", \"%gcc@10\"]),\n ],\n)\n@pytest.mark.regression(\"49841\")\ndef test_installing_external_with_compilers_directly(\n compiler_str, expected, not_expected, mutable_config, mock_packages, tmp_path: pathlib.Path\n):\n \"\"\"Tests that version constraints are taken into account for compiler annotations\n on externals\n \"\"\"\n spec_str = f\"libelf@0.8.12 %{compiler_str}\"\n packages_yaml = syaml.load_config(\n f\"\"\"\npackages:\n libelf:\n buildable: false\n externals:\n - spec: {spec_str}\n prefix: {tmp_path / \"libelf\"}\n\"\"\"\n )\n mutable_config.set(\"packages\", packages_yaml[\"packages\"])\n s = spack.concretize.concretize_one(spec_str)\n\n assert s.external\n assert all(s.satisfies(c) for c in expected)\n assert all(not s.satisfies(c) for c in not_expected)\n\n\n@pytest.mark.regression(\"49841\")\ndef test_using_externals_with_compilers(mutable_config, mock_packages, tmp_path: pathlib.Path):\n \"\"\"Tests that version constraints are taken into account for compiler annotations\n on externals, even imposed as transitive deps.\n \"\"\"\n packages_yaml = syaml.load_config(\n f\"\"\"\npackages:\n libelf:\n buildable: false\n externals:\n - spec: libelf@0.8.12 %gcc@10\n prefix: {tmp_path / \"libelf\"}\n\"\"\"\n )\n mutable_config.set(\"packages\", packages_yaml[\"packages\"])\n\n with pytest.raises(spack.error.SpackError):\n spack.concretize.concretize_one(\"dyninst%gcc@10.2.1 ^libelf@0.8.12 %gcc@:9\")\n\n s = spack.concretize.concretize_one(\"dyninst%gcc@10.2.1 ^libelf@0.8.12 %gcc@10:\")\n\n libelf = s[\"libelf\"]\n assert libelf.external and libelf.satisfies(\"%gcc\")\n\n\n@pytest.mark.regression(\"50161\")\ndef test_installed_compiler_and_better_external(install_mockery, mutable_config):\n \"\"\"Tests that we always prefer a higher-priority external compiler, when we have a\n lower-priority compiler installed, and we try to concretize a spec without specifying\n the compiler dependency.\n \"\"\"\n pkg_b = spack.concretize.concretize_one(spack.spec.Spec(\"pkg-b %clang\"))\n PackageInstaller([pkg_b.package], fake=True, explicit=True).install()\n\n with spack.config.override(\"concretizer:reuse\", False):\n pkg_a = spack.concretize.concretize_one(\"pkg-a\")\n assert pkg_a[\"c\"].satisfies(\"gcc@10\"), pkg_a.tree()\n assert pkg_a[\"pkg-b\"][\"c\"].satisfies(\"gcc@10\")\n\n with spack.config.override(\"concretizer:reuse\", False):\n mpileaks = spack.concretize.concretize_one(\"mpileaks\")\n assert mpileaks.satisfies(\"%gcc@10\")\n\n\n@pytest.mark.regression(\"50006\")\ndef test_concrete_multi_valued_variants_in_externals(\n mutable_config, mock_packages, tmp_path: pathlib.Path\n):\n \"\"\"Tests that concrete multivalued variants in externals cannot be extended with additional\n values when concretizing.\n \"\"\"\n packages_yaml = syaml.load_config(\n f\"\"\"\npackages:\n gcc:\n buildable: false\n externals:\n - spec: gcc@12.1.0 languages:='c,c++'\n prefix: {tmp_path / \"gcc-12\"}\n extra_attributes:\n compilers:\n c: {tmp_path / \"gcc-12\"}/bin/gcc\n cxx: {tmp_path / \"gcc-12\"}/bin/g++\n\n - spec: gcc@14.1.0 languages:=fortran\n prefix: {tmp_path / \"gcc-14\"}\n extra_attributes:\n compilers:\n fortran: {tmp_path / \"gcc-14\"}/bin/gfortran\n\"\"\"\n )\n mutable_config.set(\"packages\", packages_yaml[\"packages\"])\n\n with pytest.raises(spack.solver.asp.UnsatisfiableSpecError):\n spack.concretize.concretize_one(\"pkg-b %gcc@14\")\n\n s = spack.concretize.concretize_one(\"pkg-b %gcc\")\n assert s[\"c\"].satisfies(\"gcc@12.1.0\"), s.tree()\n assert s[\"c\"].external\n assert s[\"c\"].satisfies(\"languages=c,c++\") and not s[\"c\"].satisfies(\"languages=fortran\")\n\n\ndef test_concrete_multi_valued_in_input_specs(default_mock_concretization):\n \"\"\"Tests that we can use := to specify exactly multivalued variants in input specs.\"\"\"\n s = default_mock_concretization(\"gcc languages:=fortran\")\n assert not s.external and s[\"c\"].external\n assert s.satisfies(\"languages:=fortran\")\n assert not s.satisfies(\"languages=c\") and not s.satisfies(\"languages=c++\")\n\n\ndef test_concrete_multi_valued_variants_in_requirements(mutable_config, mock_packages):\n \"\"\"Tests that concrete multivalued variants can be imposed by requirements.\"\"\"\n packages_yaml = syaml.load_config(\n \"\"\"\npackages:\n pkg-a:\n require:\n - libs:=static\n\"\"\"\n )\n mutable_config.set(\"packages\", packages_yaml[\"packages\"])\n\n with pytest.raises(spack.solver.asp.UnsatisfiableSpecError):\n spack.concretize.concretize_one(\"pkg-a libs=shared\")\n spack.concretize.concretize_one(\"pkg-a libs=shared,static\")\n\n s = spack.concretize.concretize_one(\"pkg-a\")\n assert s.satisfies(\"libs:=static\")\n assert not s.satisfies(\"libs=shared\")\n\n\ndef test_concrete_multi_valued_variants_in_depends_on(default_mock_concretization):\n \"\"\"Tests the use of := in depends_on directives\"\"\"\n with pytest.raises(spack.solver.asp.UnsatisfiableSpecError):\n default_mock_concretization(\"gmt-concrete-mv-dependency ^mvdefaults foo:=c\")\n default_mock_concretization(\"gmt-concrete-mv-dependency ^mvdefaults foo:=a,c\")\n default_mock_concretization(\"gmt-concrete-mv-dependency ^mvdefaults foo:=b,c\")\n\n s = default_mock_concretization(\"gmt-concrete-mv-dependency\")\n assert s.satisfies(\"^mvdefaults foo:=a,b\"), s.tree()\n assert not s.satisfies(\"^mvdefaults foo=c\")\n\n\ndef test_concrete_multi_valued_variants_when_args(default_mock_concretization):\n \"\"\"Tests the use of := in conflicts and when= arguments\"\"\"\n # Check conflicts(\"foo:=a,b\", when=\"@0.9\")\n with pytest.raises(spack.solver.asp.UnsatisfiableSpecError):\n default_mock_concretization(\"mvdefaults@0.9 foo:=a,b\")\n\n for c in (\"foo:=a\", \"foo:=a,b,c\", \"foo:=a,c\", \"foo:=b,c\"):\n s = default_mock_concretization(f\"mvdefaults@0.9 {c}\")\n assert s.satisfies(c)\n\n # Check depends_on(\"pkg-b\", when=\"foo:=b,c\")\n s = default_mock_concretization(\"mvdefaults foo:=b,c\")\n assert s.satisfies(\"^pkg-b\")\n\n for c in (\"foo:=a\", \"foo:=a,b,c\", \"foo:=a,b\", \"foo:=a,c\"):\n s = default_mock_concretization(f\"mvdefaults {c}\")\n assert not s.satisfies(\"^pkg-b\")\n\n\n@pytest.mark.usefixtures(\"mock_packages\")\n@pytest.mark.parametrize(\n \"constraint_in_yaml,unsat_request,sat_request\",\n [\n # Arch parts\n pytest.param(\n \"target=x86_64\",\n \"target=core2\",\n \"target=x86_64\",\n marks=pytest.mark.skipif(\n platform.machine() != \"x86_64\", reason=\"only valid for x86_64\"\n ),\n ),\n pytest.param(\n \"target=core2\",\n \"target=x86_64\",\n \"target=core2\",\n marks=pytest.mark.skipif(\n platform.machine() != \"x86_64\", reason=\"only valid for x86_64\"\n ),\n ),\n (\"os=debian6\", \"os=redhat6\", \"os=debian6\"),\n (\"platform=test\", \"platform=linux\", \"platform=test\"),\n # Variants\n (\"~lld\", \"+lld\", \"~lld\"),\n (\"+lld\", \"~lld\", \"+lld\"),\n ],\n)\ndef test_spec_parts_on_fresh_compilers(\n constraint_in_yaml, unsat_request, sat_request, mutable_config, tmp_path: pathlib.Path\n):\n \"\"\"Tests that spec parts like targets and variants in `% target= `\n are associated with `package` for `%` just as they would be for `^`, when we concretize\n without reusing.\n \"\"\"\n packages_yaml = syaml.load_config(\n f\"\"\"\n packages:\n llvm::\n buildable: false\n externals:\n - spec: \"llvm@20 +clang {constraint_in_yaml}\"\n prefix: {tmp_path / \"llvm-20\"}\n \"\"\"\n )\n mutable_config.set(\"packages\", packages_yaml[\"packages\"])\n\n # Check the abstract spec is formed correctly\n abstract_spec = Spec(f\"pkg-a %llvm@20 +clang {unsat_request}\")\n assert abstract_spec[\"llvm\"].satisfies(f\"@20 +clang {unsat_request}\")\n\n # Check that we can't concretize the spec, since llvm is not buildable\n with pytest.raises(spack.solver.asp.UnsatisfiableSpecError):\n spack.concretize.concretize_one(abstract_spec)\n\n # Check we can instead concretize if we use the correct constraint\n s = spack.concretize.concretize_one(f\"pkg-a %llvm@20 +clang {sat_request}\")\n assert s[\"c\"].external and s[\"c\"].satisfies(f\"@20 +clang {sat_request}\")\n\n\n@pytest.mark.usefixtures(\"mock_packages\", \"mutable_database\")\n@pytest.mark.parametrize(\n \"constraint_in_yaml,unsat_request,sat_request\",\n [\n # Arch parts\n pytest.param(\n \"target=x86_64\",\n \"target=core2\",\n \"target=x86_64\",\n marks=pytest.mark.skipif(\n platform.machine() != \"x86_64\", reason=\"only valid for x86_64\"\n ),\n ),\n pytest.param(\n \"target=core2\",\n \"target=x86_64\",\n \"target=core2\",\n marks=pytest.mark.skipif(\n platform.machine() != \"x86_64\", reason=\"only valid for x86_64\"\n ),\n ),\n (\"os=debian6\", \"os=redhat6\", \"os=debian6\"),\n (\"platform=test\", \"platform=linux\", \"platform=test\"),\n # Variants\n (\"~lld\", \"+lld\", \"~lld\"),\n (\"+lld\", \"~lld\", \"+lld\"),\n ],\n)\ndef test_spec_parts_on_reused_compilers(\n constraint_in_yaml, unsat_request, sat_request, mutable_config, tmp_path: pathlib.Path\n):\n \"\"\"Tests that requests of the form % are considered for reused\n specs, even though build dependency are not part of the ASP problem.\n \"\"\"\n packages_yaml = syaml.load_config(\n f\"\"\"\n packages:\n c:\n require: llvm\n cxx:\n require: llvm\n llvm::\n buildable: false\n externals:\n - spec: \"llvm+clang@20 {constraint_in_yaml}\"\n prefix: {tmp_path / \"llvm-20\"}\n mpileaks:\n buildable: true\n \"\"\"\n )\n mutable_config.set(\"packages\", packages_yaml[\"packages\"])\n\n # Install the spec\n installed_spec = spack.concretize.concretize_one(f\"mpileaks %llvm@20 {sat_request}\")\n PackageInstaller([installed_spec.package], fake=True, explicit=True).install()\n\n # Make mpileaks not buildable\n mutable_config.set(\"packages:mpileaks:buildable\", False)\n\n # Check we can't concretize with the unsat request...\n with pytest.raises(spack.solver.asp.UnsatisfiableSpecError):\n spack.concretize.concretize_one(f\"mpileaks %llvm@20 {unsat_request}\")\n\n # ...but we can with the original constraint\n with spack.config.override(\"concretizer:reuse\", True):\n s = spack.concretize.concretize_one(f\"mpileaks %llvm@20 {sat_request}\")\n\n assert s.dag_hash() == installed_spec.dag_hash()\n\n\ndef test_use_compiler_by_hash(mock_packages, mutable_database, mutable_config):\n \"\"\"Tests that we can reuse an installed compiler specifying its hash\"\"\"\n installed_spec = spack.concretize.concretize_one(\"gcc@14.0\")\n PackageInstaller([installed_spec.package], fake=True, explicit=True).install()\n\n with spack.config.override(\"concretizer:reuse\", True):\n s = spack.concretize.concretize_one(f\"mpileaks %gcc/{installed_spec.dag_hash()}\")\n\n assert s[\"c\"].dag_hash() == installed_spec.dag_hash()\n\n\n@pytest.mark.parametrize(\n \"spec_str,expected,not_expected\",\n [\n # Simple build requirement on gcc, as a provider for c\n (\n \"mpileaks %gcc\",\n [\"%[deptypes=build] gcc\"],\n [\"%[deptypes=link] gcc\", \"%[deptypes=run] gcc\"],\n ),\n # Require mpich as a direct dependency of mpileaks\n (\n \"mpileaks %[deptypes=link] mpich\",\n [\"%[deptypes=build,link] mpich\", \"^callpath%[deptypes=build,link] mpich\"],\n [\"%[deptypes=run] mpich\"],\n ),\n (\n \"mpileaks %[deptypes=link] mpich+debug\", # non-default variant\n [\"%[deptypes=build,link] mpich+debug\"],\n [\"% mpich~debug\"],\n ),\n # Require mpich as a direct dependency of two nodes, with compatible constraints\n (\n \"mpileaks %mpich+debug ^callpath %mpich@3.0.3\", # non-default variant\n [\n \"%[deptypes=build,link] mpich@3.0.3+debug\",\n \"^callpath %[deptypes=build,link] mpich@3.0.3+debug\",\n ],\n [\"%mpich~debug\"],\n ),\n # Package that has a conditional link dependency on a compiler\n (\"emacs +native\", [\"%[virtuals=c deptypes=build,link] gcc\"], []),\n (\"emacs +native %gcc\", [\"%[virtuals=c deptypes=build,link] gcc\"], []),\n (\"emacs +native %[virtuals=c] gcc\", [\"%[virtuals=c deptypes=build,link] gcc\"], []),\n # Package that depends on llvm as a library and also needs C and C++ compilers\n (\n \"llvm-client\",\n [\"%[virtuals=c,cxx deptypes=build] gcc\", \"%[deptypes=build,link] llvm\"],\n [\"%c=llvm\"],\n ),\n (\n \"llvm-client %c,cxx=gcc\",\n [\"%[virtuals=c,cxx deptypes=build] gcc\", \"%[deptypes=build,link] llvm\"],\n [\"%c=llvm\"],\n ),\n (\"llvm-client %c,cxx=llvm\", [\"%[virtuals=c,cxx deptypes=build,link] llvm\"], [\"%gcc\"]),\n ],\n)\ndef test_specifying_direct_dependencies(\n spec_str, expected, not_expected, default_mock_concretization\n):\n \"\"\"Tests solving % in different scenarios, either for runtime or buildtime dependencies.\"\"\"\n concrete_spec = default_mock_concretization(spec_str)\n\n for c in expected:\n assert concrete_spec.satisfies(c)\n\n for c in not_expected:\n assert not concrete_spec.satisfies(c)\n\n\n@pytest.mark.parametrize(\n \"spec_str,conditional_spec,expected\",\n [\n # Abstract spec is False, cause the set of possible solutions in the rhs is smaller\n (\"mpich\", \"%[when=+debug] llvm\", (False, True)),\n # Abstract spec is True, since we know the condition never applies\n (\"mpich~debug\", \"%[when=+debug] llvm\", (True, True)),\n # In this case we know the condition applies\n (\"mpich+debug\", \"%[when=+debug] llvm\", (False, False)),\n (\"mpich+debug %llvm+clang\", \"%[when=+debug] llvm\", (True, True)),\n (\"mpich+debug\", \"%[when=+debug] gcc\", (False, True)),\n # Conditional specs on the lhs\n (\"mpich %[when=+debug] gcc\", \"mpich %gcc\", (False, True)),\n (\"mpich %[when=+debug] gcc\", \"mpich %llvm\", (False, False)),\n (\"mpich %[when=+debug] gcc\", \"mpich %[when=+debug] gcc\", (True, True)),\n (\"mpileaks ^[when=+opt] callpath@0.9\", \"mpileaks ^callpath@1.0\", (False, True)),\n (\"mpileaks ^[when=+opt] callpath@1.0\", \"mpileaks ^callpath@1.0\", (False, True)),\n (\"mpileaks ^[when=+opt] callpath@1.0\", \"mpileaks ^[when=+opt] callpath@1.0\", (True, True)),\n # Conditional specs on both sides\n (\n \"mpileaks ^[when=+opt] callpath@1.0\",\n \"mpileaks ^[when=+opt+debug] callpath@1.0\",\n (True, True),\n ),\n (\n \"mpileaks ^[when=+opt+debug] callpath@1.0\",\n \"mpileaks ^[when=+opt] callpath@1.0\",\n (False, True),\n ),\n (\n \"mpileaks ^[when=+opt] callpath@1.0\",\n \"mpileaks ^[when=~debug] callpath@1.0\",\n (False, True),\n ),\n # Different conditional specs associated with different nodes in the DAG, where one does\n # not apply since the condition is not met\n (\n \"mpileaks %[when='%mpi' virtuals=mpi] zmpi ^libelf %[when='%mpi' virtuals=mpi] mpich\",\n \"mpileaks %[virtuals=mpi] zmpi\",\n (False, True),\n ),\n (\n \"mpileaks %[when='%mpi' virtuals=mpi] mpich ^libelf %[when='%mpi' virtuals=mpi] zmpi\",\n \"mpileaks %[virtuals=mpi] mpich\",\n (False, True),\n ),\n ],\n)\ndef test_satisfies_conditional_spec(\n spec_str, conditional_spec, expected, default_mock_concretization\n):\n \"\"\"Tests satisfies semantic when testing an abstract spec and its concretized counterpart\n with a conditional spec.\n \"\"\"\n abstract_spec = Spec(spec_str)\n concrete_spec = default_mock_concretization(spec_str)\n expected_abstract, expected_concrete = expected\n\n assert abstract_spec.satisfies(conditional_spec) is expected_abstract\n assert concrete_spec.satisfies(conditional_spec) is expected_concrete\n assert concrete_spec.satisfies(abstract_spec)\n\n\n@pytest.mark.not_on_windows(\"Tests use linux paths\")\n@pytest.mark.regression(\"51001\")\ndef test_selecting_externals_with_compilers_as_root(mutable_config, mock_packages):\n \"\"\"Tests that we can select externals that have a compiler in their spec, even when\n they are root.\n \"\"\"\n packages_yaml = syaml.load_config(\n \"\"\"\npackages:\n gcc::\n externals:\n - spec: \"gcc@9.4.0 languages='c,c++'\"\n prefix: /path\n extra_attributes:\n compilers:\n c: /path/bin/gcc\n cxx: /path/bin/g++\n llvm::\n buildable: false\n externals:\n - spec: \"llvm@20 +clang\"\n prefix: /path\n extra_attributes:\n compilers:\n c: /path/bin/gcc\n cxx: /path/bin/g++\n mpich:\n buildable: false\n externals:\n - spec: \"mpich@3.4.3 %gcc\"\n prefix: /path/mpich/gcc\n - spec: \"mpich@3.4.3 %clang\"\n prefix: /path/mpich/clang\n\"\"\"\n )\n mutable_config.set(\"packages\", packages_yaml[\"packages\"])\n\n # Select mpich as the root spec\n s = spack.concretize.concretize_one(\"mpich %clang\")\n assert s.external\n assert s.prefix == \"/path/mpich/clang\"\n\n s = spack.concretize.concretize_one(\"mpich %gcc\")\n assert s.external\n assert s.prefix == \"/path/mpich/gcc\"\n\n # Select mpich as a dependency\n s = spack.concretize.concretize_one(\"mpileaks ^mpi=mpich %clang\")\n assert s[\"mpi\"].external\n assert s[\"mpi\"].prefix == \"/path/mpich/clang\"\n\n s = spack.concretize.concretize_one(\"mpileaks ^mpi=mpich %gcc\")\n assert s[\"mpi\"].external\n assert s[\"mpi\"].prefix == \"/path/mpich/gcc\"\n\n\n@pytest.mark.not_on_windows(\"Tests use linux paths\")\n@pytest.mark.regression(\"51001\")\n@pytest.mark.parametrize(\n \"external_compiler,spec_str\", [(\"gcc@8\", \"mpich %gcc@8.4\"), (\"gcc@8.4.0\", \"mpich %gcc@8\")]\n)\ndef test_selecting_externals_with_compilers_and_versions(\n external_compiler, spec_str, mutable_config, mock_packages\n):\n \"\"\"Tests different scenarios of having a compiler specified with a version constraint, either\n in the input spec or in the external spec.\n \"\"\"\n packages_yaml = syaml.load_config(\n f\"\"\"\npackages:\n gcc:\n externals:\n - spec: \"gcc@8.4.0 languages='c,c++'\"\n prefix: /path\n extra_attributes:\n compilers:\n c: /path/bin/gcc\n cxx: /path/bin/g++\n mpich:\n buildable: false\n externals:\n - spec: \"mpich@3.4.3 %{external_compiler}\"\n prefix: /path/mpich/gcc\n - spec: \"mpich@3.4.3 %clang\"\n prefix: /path/mpich/clang\n\"\"\"\n )\n mutable_config.set(\"packages\", packages_yaml[\"packages\"])\n s = spack.concretize.concretize_one(spec_str)\n assert s.external\n assert s.prefix == \"/path/mpich/gcc\"\n\n\n@pytest.mark.regression(\"51001\")\n@pytest.mark.parametrize(\n \"external_compiler,spec_str,error_match\",\n [\n # Compiler is underspecified\n (\"gcc\", \"mpich %gcc\", \"there are multiple external specs\"),\n (\"gcc@9\", \"mpich %gcc\", \"there are multiple external specs\"),\n # Compiler does not exist\n (\"%oneapi\", \"mpich %gcc@8\", \"there is no\"),\n ],\n)\ndef test_errors_when_specifying_externals_with_compilers(\n external_compiler, spec_str, error_match, mutable_config, mock_packages\n):\n \"\"\"Tests different errors that can occur in an external spec with a compiler specified.\"\"\"\n packages_yaml = syaml.load_config(\n f\"\"\"\npackages:\n mpich:\n buildable: false\n externals:\n - spec: \"mpich@3.4.3 %{external_compiler}\"\n prefix: /path/mpich/gcc\n - spec: \"mpich@3.4.3 %clang\"\n prefix: /path/mpich/clang\n\"\"\"\n )\n mutable_config.set(\"packages\", packages_yaml[\"packages\"])\n with pytest.raises(ExternalDependencyError, match=error_match):\n _ = spack.concretize.concretize_one(spec_str)\n\n\n@pytest.mark.regression(\"51146,51067\")\ndef test_caret_in_input_cannot_set_transitive_build_dependencies(default_mock_concretization):\n \"\"\"Tests that a caret in the input spec does not set transitive build dependencies, and errors\n with an appropriate message.\n \"\"\"\n with pytest.raises(spack.solver.asp.UnsatisfiableSpecError, match=\"transitive 'link' or\"):\n _ = default_mock_concretization(\"multivalue-variant ^gmake\")\n\n\n@pytest.mark.regression(\"51167\")\n@pytest.mark.require_provenance\ndef test_commit_variant_enters_the_hash(mutable_config, mock_packages, monkeypatch):\n \"\"\"Tests that an implicit commit variant, obtained from resolving the commit sha of a branch,\n enters the hash of the spec.\n \"\"\"\n\n first_call = True\n\n def _mock_resolve(spec) -> None:\n if first_call:\n spec.variants[\"commit\"] = vt.SingleValuedVariant(\"commit\", f\"{'b' * 40}\")\n return\n\n spec.variants[\"commit\"] = vt.SingleValuedVariant(\"commit\", f\"{'a' * 40}\")\n\n monkeypatch.setattr(spack.package_base.PackageBase, \"_resolve_git_provenance\", _mock_resolve)\n\n before = spack.concretize.concretize_one(\"git-ref-package@develop\")\n first_call = False\n after = spack.concretize.concretize_one(\"git-ref-package@develop\")\n\n assert before.package.needs_commit(before.version)\n assert before.satisfies(f\"commit={'b' * 40}\")\n assert after.satisfies(f\"commit={'a' * 40}\")\n assert before.dag_hash() != after.dag_hash()\n\n\n@pytest.mark.regression(\"51180\")\ndef test_reuse_with_mixed_compilers(mutable_config, mock_packages):\n \"\"\"Tests that potentially reusing a spec with a mixed compiler set, will not interfere\n with a request on one of the languages for the same package.\n \"\"\"\n packages_yaml = syaml.load_config(\n \"\"\"\npackages:\n gcc:\n externals:\n - spec: \"gcc@15.1 languages='c,c++,fortran'\"\n prefix: /path1\n extra_attributes:\n compilers:\n c: /path1/bin/gcc\n cxx: /path1/bin/g++\n fortran: /path1/bin/gfortran\n llvm:\n externals:\n - spec: \"llvm@20 +flang+clang\"\n prefix: /path2\n extra_attributes:\n compilers:\n c: /path2/bin/clang\n cxx: /path2/bin/clang++\n fortran: /path2/bin/flang\n\"\"\"\n )\n mutable_config.set(\"packages\", packages_yaml[\"packages\"])\n\n s = spack.concretize.concretize_one(\"openblas %c=gcc %fortran=llvm\")\n reusable_specs = list(s.traverse(root=True))\n\n root_specs = [Spec(\"openblas %fortran=gcc\")]\n\n with spack.config.override(\"concretizer:reuse\", True):\n solver = spack.solver.asp.Solver()\n setup = spack.solver.asp.SpackSolverSetup()\n result, _, _ = solver.driver.solve(setup, root_specs, reuse=reusable_specs)\n\n assert len(result.specs) == 1\n r = result.specs[0]\n assert r.satisfies(\"openblas %fortran=gcc\")\n assert r.dag_hash() != s.dag_hash()\n\n\n@pytest.mark.regression(\"51224\")\ndef test_when_possible_above_all(mutable_config, mock_packages):\n \"\"\"Tests that the criterion to solve as many specs as possible is above all other criteria.\"\"\"\n specs = [Spec(\"pkg-a\"), Spec(\"pkg-b\")]\n solver = spack.solver.asp.Solver()\n\n for result in solver.solve_in_rounds(specs):\n criteria = sorted(result.criteria, reverse=True)\n assert criteria[0].name == \"number of input specs not concretized\"\n\n\ndef test_concretization_cache_roundtrip(\n mock_packages, use_concretization_cache, monkeypatch, mutable_config\n):\n \"\"\"Tests whether we can write the results of a clingo solve to the cache\n and load the same spec request from the cache to produce identical specs\"\"\"\n\n assert spack.config.get(\"concretizer:concretization_cache:enable\")\n\n # run one standard concretization to populate the cache and the setup method\n # memoization\n h = spack.concretize.concretize_one(\"hdf5\")\n\n # ASP output should be stable, concretizing the same spec\n # should have the same problem output\n # assert that we're not storing any new cache entries\n def _ensure_no_store(self, problem: str, result, statistics, test=False):\n # always throw, we never want to reach this code path\n assert False, \"Concretization cache hit expected\"\n\n # Assert that we're actually hitting the cache\n cache_fetch = spack.solver.asp.ConcretizationCache.fetch\n\n def _ensure_cache_hits(self, problem: str):\n result, statistics = cache_fetch(self, problem)\n assert result, \"Expected successful concretization cache hit\"\n assert statistics, \"Expected statistics to be non null on cache hit\"\n return result, statistics\n\n monkeypatch.setattr(spack.solver.asp.ConcretizationCache, \"store\", _ensure_no_store)\n monkeypatch.setattr(spack.solver.asp.ConcretizationCache, \"fetch\", _ensure_cache_hits)\n # ensure subsequent concretizations of the same spec produce the same spec\n # object\n for _ in range(5):\n hdf5 = spack.concretize.concretize_one(\"hdf5\")\n\n assert h.to_json(pretty=True) == hdf5.to_json(pretty=True)\n assert h == hdf5\n\n\ndef test_concretization_cache_roundtrip_result(use_concretization_cache):\n \"\"\"Ensure the concretization cache doesn't change Solver Result objects.\"\"\"\n specs = [Spec(\"hdf5\")]\n solver = spack.solver.asp.Solver()\n\n result1 = solver.solve(specs)\n result2 = solver.solve(specs)\n\n assert result1 == result2\n\n\ndef test_concretization_cache_count_cleanup(use_concretization_cache, mutable_config):\n \"\"\"Tests to ensure we are cleaning the cache when we should be respective to the\n number of entries allowed in the cache\"\"\"\n conc_cache_dir = use_concretization_cache\n\n spack.config.set(\"concretizer:concretization_cache:entry_limit\", 1000)\n\n def names():\n return set(\n x.name\n for x in conc_cache_dir.iterdir()\n if (not x.is_dir() and not x.name.startswith(\".\"))\n )\n\n assert len(names()) == 0\n\n for i in range(1000):\n name = spack.util.hash.b32_hash(f\"mock_cache_file_{i}\")\n mock_cache_file = conc_cache_dir / name\n mock_cache_file.touch()\n\n before = names()\n assert len(before) == 1000\n\n # cleanup should be run after the 1,001st execution\n spack.concretize.concretize_one(\"hdf5\")\n\n # ensure that half the elements were removed and that one more was created\n after = names()\n assert len(after) == 501\n assert len(after - before) == 1 # one additional hash added by 1001st concretization\n\n\ndef test_concretization_cache_uncompressed_entry(use_concretization_cache, monkeypatch):\n def _store(self, problem, result, statistics):\n cache_path = self._cache_path_from_problem(problem)\n with self.write_transaction(cache_path) as exists:\n if exists:\n return\n try:\n with open(cache_path, \"x\", encoding=\"utf-8\") as cache_entry:\n cache_dict = {\"results\": result.to_dict(), \"statistics\": statistics}\n cache_entry.write(json.dumps(cache_dict))\n except FileExistsError:\n pass\n\n monkeypatch.setattr(spack.solver.asp.ConcretizationCache, \"store\", _store)\n # Store the results in plaintext\n spack.concretize.concretize_one(\"zlib\")\n # Ensure fetch can handle the plaintext cache entry\n spack.concretize.concretize_one(\"zlib\")\n\n\n@pytest.mark.parametrize(\n \"asp_file\",\n [\n \"concretize.lp\",\n \"heuristic.lp\",\n \"display.lp\",\n \"direct_dependency.lp\",\n \"when_possible.lp\",\n \"libc_compatibility.lp\",\n \"os_compatibility.lp\",\n \"splices.lp\",\n ],\n)\ndef test_concretization_cache_asp_canonicalization(asp_file):\n path = os.path.join(os.path.dirname(spack.solver.asp.__file__), asp_file)\n\n with open(path, \"r\", encoding=\"utf-8\") as f:\n original = [line.strip() for line in f.readlines()]\n stripped = spack.solver.asp.strip_asp_problem(original)\n\n diff = list(difflib.unified_diff(original, stripped))\n\n assert all(\n [\n line == \"-\" or line.startswith(\"-%\")\n for line in diff\n if line.startswith(\"-\") and not line.startswith(\"---\")\n ]\n )\n\n\n@pytest.mark.parametrize(\n \"node_completion,expected,not_expected\",\n [\n (\"architecture_only\", [\"+clang\", \"~flang\", \"platform=test\"], [\"lld=*\"]),\n (\n \"default_variants\",\n [\"+clang\", \"~flang\", \"+lld\", \"platform=test\"],\n [\"~clang\", \"+flang\", \"~lld\"],\n ),\n ],\n)\ndef test_external_node_completion_from_config(\n node_completion, expected, not_expected, mutable_config, mock_packages\n):\n \"\"\"Tests the different options for external node completion in the configuration file.\"\"\"\n mutable_config.set(\"concretizer:externals:completion\", node_completion)\n\n s = spack.concretize.concretize_one(\"llvm\")\n\n assert s.external\n assert all(s.satisfies(c) for c in expected)\n assert all(not s.satisfies(c) for c in not_expected)\n\n\n@pytest.mark.parametrize(\n \"spec_str,packages_yaml,expected\",\n [\n (\n \"mpileaks\",\n \"\"\"\npackages:\n mpileaks:\n externals:\n - spec: \"mpileaks@2.3~debug+opt\"\n prefix: /user/path\n dependencies:\n - id: callpath_id\n deptypes: link\n - id: mpich_id\n deptypes:\n - \"build\"\n - \"link\"\n virtuals: \"mpi\"\n callpath:\n externals:\n - spec: \"callpath@1.0\"\n prefix: /user/path\n id: callpath_id\n dependencies:\n - id: mpich_id\n deptypes:\n - \"build\"\n - \"link\"\n virtuals: \"mpi\"\n mpich:\n externals:\n - spec: \"mpich@3.0.4\"\n prefix: /user/path\n id: mpich_id\n\"\"\",\n [\n \"%mpi=mpich@3.0.4\",\n \"^callpath %mpi=mpich@3.0.4\",\n \"%[deptypes=link] callpath\",\n \"%[deptypes=build,link] mpich\",\n ],\n ),\n # Same, but using `spec:` instead of `id:` for dependencies\n (\n \"mpileaks\",\n \"\"\"\npackages:\n mpileaks:\n externals:\n - spec: \"mpileaks@2.3~debug+opt\"\n prefix: /user/path\n dependencies:\n - spec: callpath\n deptypes: link\n - spec: mpich\n virtuals: \"mpi\"\n callpath:\n externals:\n - spec: \"callpath@1.0\"\n prefix: /user/path\n dependencies:\n - spec: mpich\n virtuals: \"mpi\"\n mpich:\n externals:\n - spec: \"mpich@3.0.4\"\n prefix: /user/path\n\"\"\",\n [\n \"%mpi=mpich@3.0.4\",\n \"^callpath %mpi=mpich@3.0.4\",\n \"%[deptypes=link] callpath\",\n \"%[deptypes=build,link] mpich\",\n ],\n ),\n ],\n)\ndef test_external_specs_with_dependencies(\n spec_str, packages_yaml, expected, mutable_config, mock_packages\n):\n \"\"\"Tests that we can reconstruct external specs with dependencies.\"\"\"\n configuration = syaml.load_config(packages_yaml)\n mutable_config.set(\"packages\", configuration[\"packages\"])\n s = spack.concretize.concretize_one(spec_str)\n assert all(node.external for node in s.traverse())\n assert all(s.satisfies(c) for c in expected)\n\n\n@pytest.mark.parametrize(\n \"default_target,expected\",\n [\n # Specific target requested\n (\"x86_64_v3\", [\"callpath target=x86_64_v3\", \"^mpich target=x86_64_v3\"]),\n # With ranges, be conservative by default\n (\":x86_64_v3\", [\"callpath target=x86_64\", \"^mpich target=x86_64\"]),\n (\"x86_64:x86_64_v3\", [\"callpath target=x86_64\", \"^mpich target=x86_64\"]),\n (\"x86_64:\", [\"callpath target=x86_64\", \"^mpich target=x86_64\"]),\n ],\n)\n@pytest.mark.skipif(\n spack.vendor.archspec.cpu.host().family != \"x86_64\", reason=\"test data for x86_64\"\n)\ndef test_target_requirements(default_target, expected, mutable_config, mock_packages):\n \"\"\"Tests different scenarios where targets might be constrained by configuration and are not\n specified in external specs\n \"\"\"\n configuration = syaml.load_config(\n f\"\"\"\npackages:\n all:\n require:\n - \"target={default_target}\"\n callpath:\n buildable: false\n externals:\n - spec: \"callpath@1.0\"\n prefix: /user/path\n id: callpath_id\n dependencies:\n - id: mpich_id\n deptypes:\n - \"build\"\n - \"link\"\n virtuals: \"mpi\"\n mpich:\n externals:\n - spec: \"mpich@3.0.4\"\n prefix: /user/path\n id: mpich_id\n\"\"\"\n )\n mutable_config.set(\"packages\", configuration[\"packages\"])\n s = spack.concretize.concretize_one(\"callpath\")\n assert s.external\n assert all(s.satisfies(x) for x in expected), s.tree()\n\n\n@pytest.mark.parametrize(\n \"spec_str,inline,yaml\",\n [\n (\n \"cmake-client\",\n \"\"\"\npackages:\n cmake-client:\n externals:\n - spec: cmake-client@1.0 %cmake\n prefix: /mock\n cmake:\n externals:\n - spec: cmake@3.23.0\n prefix: /mock\n\"\"\",\n \"\"\"\npackages:\n cmake-client:\n externals:\n - spec: cmake-client@1.0\n prefix: /mock\n dependencies:\n - spec: cmake\n cmake:\n externals:\n - spec: cmake@3.23.0\n prefix: /mock\n\"\"\",\n ),\n (\n \"mpileaks\",\n \"\"\"\npackages:\n mpileaks:\n externals:\n - spec: \"mpileaks@2.3~debug+opt %mpi=mpich %[deptypes=link] callpath\"\n prefix: /user/path\n callpath:\n externals:\n - spec: \"callpath@1.0 %mpi=mpich\"\n prefix: /user/path\n mpich:\n externals:\n - spec: \"mpich@3.0.4\"\n prefix: /user/path\n\"\"\",\n \"\"\"\npackages:\n mpileaks:\n externals:\n - spec: \"mpileaks@2.3~debug+opt\"\n prefix: /user/path\n dependencies:\n - spec: callpath\n deptypes: link\n - spec: mpich\n virtuals: \"mpi\"\n callpath:\n externals:\n - spec: \"callpath@1.0\"\n prefix: /user/path\n dependencies:\n - spec: mpich\n virtuals: \"mpi\"\n mpich:\n externals:\n - spec: \"mpich@3.0.4\"\n prefix: /user/path\n\"\"\",\n ),\n ],\n)\ndef test_external_inline_equivalent_to_yaml(spec_str, inline, yaml, mutable_config, mock_packages):\n \"\"\"Tests that the inline syntax for external specs is equivalent to the YAML syntax.\"\"\"\n configuration = syaml.load_config(inline)\n mutable_config.set(\"packages\", configuration[\"packages\"])\n inline_spec = spack.concretize.concretize_one(spec_str)\n\n configuration = syaml.load_config(yaml)\n mutable_config.set(\"packages\", configuration[\"packages\"])\n yaml_spec = spack.concretize.concretize_one(spec_str)\n\n assert inline_spec == yaml_spec\n\n\n@pytest.mark.regression(\"51556\")\ndef test_reusing_gcc_same_version_different_libcs(monkeypatch, mutable_config, mock_packages):\n \"\"\"Tests that Spack can solve for specs when it reuses 2 GCCs at the same version,\n but injecting different libcs.\n \"\"\"\n packages_yaml = syaml.load_config(\n \"\"\"\npackages:\n gcc:\n externals:\n - spec: \"gcc@12.3.0 languages='c,c++,fortran' os=debian6\"\n prefix: /path\n extra_attributes:\n compilers:\n c: /path/bin/gcc\n cxx: /path/bin/g++\n fortran: /path/bin/gfortran\n - spec: \"gcc@12.3.0 languages='c,c++,fortran' os=redhat6\"\n prefix: /path\n extra_attributes:\n compilers:\n c: /path/bin/gcc\n cxx: /path/bin/g++\n fortran: /path/bin/gfortran\n\"\"\"\n )\n mutable_config.set(\"packages\", packages_yaml[\"packages\"])\n mutable_config.set(\"concretizer:reuse\", True)\n\n def _mock_libc(self):\n if self.spec.satisfies(\"os=debian6\"):\n return spack.spec.Spec(\"glibc@=2.31\", external_path=\"/rocky9/path\")\n return spack.spec.Spec(\"glibc@=2.28\", external_path=\"/rocky8/path\")\n\n monkeypatch.setattr(\n spack.compilers.libraries.CompilerPropertyDetector, \"default_libc\", _mock_libc\n )\n\n # This should not raise\n mpileaks = spack.concretize.concretize_one(\"mpileaks %c=gcc@12\")\n\n assert mpileaks.satisfies(\"%c=gcc@12\")\n\n\ndef test_concrete_specs_skip_prechecks(mock_packages):\n \"\"\"Test that concrete specs are not checked for unknown versions and dependencies.\"\"\"\n\n specs = [spack.spec.Spec(\"zlib\"), spack.spec.Spec(\"deprecated-versions@=1.1.0\")]\n\n with pytest.raises(spack.solver.asp.DeprecatedVersionError):\n spack.solver.asp.SpackSolverSetup().setup(specs)\n\n with spack.config.override(\"config:deprecated\", True):\n concrete_spec = spack.concretize.concretize_one(specs[1])\n\n # Try again with the same version but a concrete spec\n specs[1] = concrete_spec\n\n spack.solver.asp.SpackSolverSetup().setup(specs)\n\n\n@pytest.mark.regression(\"51683\")\ndef test_activating_variant_for_conditional_language_dependency(default_mock_concretization):\n \"\"\"Tests that a dependency on a conditional language can be concretized, and that the solver\n turn on the correct variant to enable the language dependency\n \"\"\"\n # To trigger the bug, we need at least another node needing fortran, in this case mpich\n s = default_mock_concretization(\"mpileaks %fortran=gcc %mpi=mpich\")\n assert s.satisfies(\"+fortran\")\n\n # Try just asking for fortran, without the provider\n s = default_mock_concretization(\"mpileaks %fortran %mpi=mpich\")\n assert s.satisfies(\"+fortran\")\n\n\ndef test_when_condition_with_direct_dependency_on_virtual_provider(default_mock_concretization):\n \"\"\"If a when condition contains a direct dependency on a provider of a virtual, it should only\n trigger if the provider is used for that current package, and not if the provider happens to be\n a dependency, without its virtual being depended on.\"\"\"\n s = default_mock_concretization(\"direct-dep-virtuals-one\")\n assert s.satisfies(\"%netlib-blas\")\n assert s[\"direct-dep-virtuals-two\"].satisfies(\"%blas=netlib-blas\")\n\n\ndef test_conflict_with_direct_dependency_on_virtual_provider(default_mock_concretization):\n \"\"\"Test that conflicts on virtual providers as direct dependencies work\"\"\"\n s = default_mock_concretization(\"conflict-virtual\")\n assert s.satisfies(\"%blas=netlib-blas\")\n\n with pytest.raises(spack.solver.asp.UnsatisfiableSpecError):\n default_mock_concretization(\"conflict-virtual +conflict_direct\")\n\n with pytest.raises(spack.solver.asp.UnsatisfiableSpecError):\n default_mock_concretization(\"conflict-virtual +conflict_transitive\")\n\n\ndef test_imposed_spec_dependency_duplication(mock_packages: spack.repo.Repo):\n \"\"\"Tests that imposed dependencies triggered by identical conditions are grouped together,\n and that imposed dependencies that differ on a deptype are not grouped together.\"\"\"\n # The trigger-and-effect-deps pkg has 4 conditions, 2 triggers, and 4 effects in total:\n # +x -> depends on pkg-a with deptype link\n # +x -> depends on pkg-b with deptype link\n # +y -> depends on pkg-a with deptype run\n # +y -> depends on pkg-b with deptype run\n pkg = mock_packages.get_pkg_class(\"trigger-and-effect-deps\")\n setup = spack.solver.asp.SpackSolverSetup()\n setup.gen = spack.solver.asp.ProblemInstanceBuilder()\n setup.package_dependencies_rules(pkg)\n setup.trigger_rules()\n setup.effect_rules()\n asp = setup.gen.asp_problem\n\n # There should be 4 conditions total\n assert len([line for line in asp if re.search(r\"condition\\(\\d+\\)\", line)]) == 4\n # There should be 2 triggers total\n assert len([line for line in asp if re.search(r\"trigger_id\\(\\d+\\)\", line)]) == 2\n # There should be 4 effects total\n assert len([line for line in asp if re.search(r\"effect_id\\(\\d+\\)\", line)]) == 4\n\n\n@pytest.mark.regression(\"51842\")\n@pytest.mark.parametrize(\n \"spec_str,expected\",\n [\n (\"variant-function-validator\", \"generator=make %adios2~bzip2\"),\n (\"variant-function-validator generator=make\", \"generator=make %adios2~bzip2\"),\n (\"variant-function-validator generator=ninja\", \"generator=ninja %adios2+bzip2\"),\n (\"variant-function-validator generator=other\", \"generator=other %adios2+bzip2\"),\n ],\n)\ndef test_penalties_for_variant_defined_by_function(\n default_mock_concretization, spec_str, expected\n):\n \"\"\"Tests that we have penalties for variants defined by functions, and that variant values\n are consistent with defaults and optimization rules.\n \"\"\"\n s = default_mock_concretization(spec_str)\n assert s.satisfies(expected)\n\n\ndef test_default_values_used_if_subset_required_by_dependent(mock_packages):\n \"\"\"If a dependent requires *at least* a subset of default values of a multi-valued variant of\n a dependency, that should not influence concretization; the default values should be used.\"\"\"\n # multivalue-variant-multi-defaults-dependent requires myvariant=bar without baz.\n a = spack.concretize.concretize_one(\"multivalue-variant-multi-defaults-dependent\")\n # we still end up using baz, and we don't drop it to avoid an extra dependency.\n assert a.satisfies(\"%multivalue-variant-multi-defaults myvariant=bar,baz\")\n\n\ndef test_virtual_gets_multiple_dupes(mock_packages, config):\n \"\"\"Tests that virtual packages always get multiple dupes, according to what we have in\n the configuration files.\n \"\"\"\n specs = [spack.spec.Spec(\"pkg-with-c-link-dep\")]\n possible_graph = spack.solver.input_analysis.NoStaticAnalysis(\n configuration=spack.config.CONFIG, repo=spack.repo.PATH\n )\n counter = spack.solver.input_analysis.MinimalDuplicatesCounter(\n specs, tests=False, possible_graph=possible_graph\n )\n gen = spack.solver.asp.ProblemInstanceBuilder()\n counter.possible_packages_facts(gen, spack.solver.core.fn)\n\n asp = gen.asp_problem\n # \"c\" is a compiler language virtual and must allow multiple nodes, not be capped at 1\n selected_lines = [line for line in asp if line.startswith('max_dupes(\"c\"')]\n assert len(selected_lines) == 1\n max_dupes_c = selected_lines[0]\n assert 'max_dupes(\"c\",2).' == max_dupes_c, f\"should have max_dupes=2, but got: {max_dupes_c}\"\n\n\ndef test_compiler_selection_when_external_has_variant_penalty(mutable_config, mock_packages):\n \"\"\"Tests that a compiler that should be preferred is not swapped with a less preferred\n compiler because of penalties on variants.\n \"\"\"\n packages_yaml = syaml.load_config(\n \"\"\"\npackages:\n gcc::\n externals:\n - spec: \"gcc@15.2.0 languages='c,c++' ~binutils\"\n prefix: /path\n extra_attributes:\n compilers:\n c: /path/bin/gcc\n cxx: /path/bin/g++\n llvm::\n buildable: false\n externals:\n - spec: \"llvm@20 +clang\"\n prefix: /path\n extra_attributes:\n compilers:\n c: /path/bin/gcc\n cxx: /path/bin/g++\n\"\"\"\n )\n mutable_config.set(\"packages\", packages_yaml[\"packages\"])\n\n concrete = spack.concretize.concretize_one(\"libdwarf\")\n\n # GCC is the preferred provider, but has a penalty on its variants\n assert concrete.satisfies(\"%gcc@15.2.0 ~binutils\"), concrete.tree()\n # LLVM is the second provider choice, with no penalty on variants\n assert not concrete.satisfies(\"%llvm@20 +clang\")\n\n\ndef test_mpi_selection_when_external_has_variant_penalty(mutable_config, mock_packages):\n \"\"\"Tests that conflicting with a default provider doesn't cause a variant values to be\n flipped to avoid the variant dependency.\n \"\"\"\n packages_yaml = syaml.load_config(\n \"\"\"\npackages:\n all:\n variants: +mpi\n mpich:\n buildable: false\n\"\"\"\n )\n mutable_config.set(\"packages\", packages_yaml[\"packages\"])\n\n concrete = spack.concretize.concretize_one(\"transitive-conditional-virtual-dependency\")\n\n # GCC is the preferred provider, but has a penalty on its variants\n assert concrete.satisfies(\"%conditional-virtual-dependency+mpi\"), concrete.tree()\n # LLVM is the second provider choice, with no penalty on variants\n assert concrete.satisfies(\"^mpi=zmpi\")\n\n\ndef test_preferring_different_compilers_for_different_languages(mutable_config, mock_packages):\n \"\"\"Tests that in a case where we prefer different compilers for different languages, steering\n towards using a unique toolchain is lower priority with respect to flipping variants to turn\n off a language, or selecting a non-default provider.\n \"\"\"\n packages_yaml = syaml.load_config(\n \"\"\"\npackages:\n all:\n providers:\n c:: [llvm, gcc]\n cxx:: [llvm, gcc]\n fortran:: [gcc]\n c:\n prefer:\n - llvm\n cxx:\n prefer:\n - llvm\n fortran:\n prefer:\n - gcc\n mpileaks:\n variants: +fortran\n\"\"\"\n )\n mutable_config.set(\"packages\", packages_yaml[\"packages\"])\n\n mpileaks = spack.concretize.concretize_one(\"mpileaks\")\n\n assert mpileaks.satisfies(\"%c,cxx=llvm %fortran=gcc\"), mpileaks.tree()\n assert mpileaks.satisfies(\"%mpi=mpich\")\n assert mpileaks[\"mpich\"].satisfies(\"%c,cxx=llvm %fortran=gcc\")\n" }, { "path": "lib/spack/spack/test/concretization/errors.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Regression tests for concretizer error messages.\n\nEvery test asserts two properties:\n1. The correct exception type is raised.\n2. The message contains every \"actionable part\" -- a string from the user's\n input (spec token, config key, package name) that helps identify what to\n change.\n\"\"\"\n\nimport pathlib\nfrom io import StringIO\nfrom typing import List\n\nimport pytest\n\nimport spack.concretize\nimport spack.config\nimport spack.error\nimport spack.main\nimport spack.solver.asp\nimport spack.spec\n\nversion_error_messages = [\n \"Cannot satisfy\",\n \" required because quantum-espresso depends on fftw@:1.0\",\n \" required because quantum-espresso ^fftw@1.1: requested explicitly\",\n \" required because quantum-espresso ^fftw@1.1: requested explicitly\",\n]\n\nexternal_error_messages = [\n \"Cannot build quantum-espresso, since it is configured `buildable:false` and \"\n \"no externals satisfy the request\"\n]\n\nvariant_error_messages = [\n \"'fftw' requires conflicting variant values '~mpi' and '+mpi'\",\n \" required because quantum-espresso depends on fftw+mpi when +invino\",\n \" required because quantum-espresso+invino ^fftw~mpi requested explicitly\",\n \" required because quantum-espresso+invino ^fftw~mpi requested explicitly\",\n]\n\nexternal_config = {\n \"packages:quantum-espresso\": {\n \"buildable\": False,\n \"externals\": [{\"spec\": \"quantum-espresso@1.0~veritas\", \"prefix\": \"/path/to/qe\"}],\n }\n}\n\n\n@pytest.mark.parametrize(\n \"error_messages,config_set,spec\",\n [\n (version_error_messages, {}, \"quantum-espresso^fftw@1.1:\"),\n (external_error_messages, external_config, \"quantum-espresso+veritas\"),\n (variant_error_messages, {}, \"quantum-espresso+invino^fftw~mpi\"),\n ],\n)\ndef test_error_messages(error_messages, config_set, spec, mock_packages, mutable_config):\n for path, conf in config_set.items():\n spack.config.set(path, conf)\n\n with pytest.raises(spack.solver.asp.UnsatisfiableSpecError) as e:\n _ = spack.concretize.concretize_one(spec)\n\n for em in error_messages:\n assert em in str(e.value), str(e.value)\n\n\n@pytest.mark.parametrize(\n \"spec\", [\"deprecated-versions@1.1.0\", \"deprecated-client ^deprecated-versions@1.1.0\"]\n)\ndef test_deprecated_version_error(spec, mock_packages, mutable_config):\n with pytest.raises(spack.solver.asp.DeprecatedVersionError, match=\"deprecated-versions@1.1.0\"):\n _ = spack.concretize.concretize_one(spec)\n\n spack.config.set(\"config:deprecated\", True)\n spack.concretize.concretize_one(spec)\n\n\n@pytest.mark.parametrize(\n \"spec\", [\"deprecated-versions@99.9\", \"deprecated-client ^deprecated-versions@99.9\"]\n)\ndef test_nonexistent_version_error(spec, mock_packages, mutable_config):\n with pytest.raises(spack.solver.asp.InvalidVersionError, match=\"deprecated-versions@99.9\"):\n _ = spack.concretize.concretize_one(spec)\n\n\ndef test_internal_error_handling_formatting(tmp_path: pathlib.Path):\n log = StringIO()\n input_to_output = [\n (spack.spec.Spec(\"foo+x\"), spack.spec.Spec(\"foo@=1.0~x\")),\n (spack.spec.Spec(\"bar+y\"), spack.spec.Spec(\"x@=1.0~y\")),\n (spack.spec.Spec(\"baz+z\"), None),\n ]\n spack.main._handle_solver_bug(\n spack.solver.asp.OutputDoesNotSatisfyInputError(input_to_output), root=tmp_path, out=log\n )\n\n output = log.getvalue()\n assert \"the following specs were not solved:\\n - baz+z\\n\" in output\n assert (\n \"the following specs were concretized, but do not satisfy the input:\\n\"\n \" - input: foo+x\\n\"\n \" output: foo@=1.0~x\\n\"\n \" - input: bar+y\\n\"\n \" output: x@=1.0~y\"\n ) in output\n\n files = {f.name: str(f) for f in tmp_path.glob(\"spack-asp-*/*.json\")}\n assert {\"input-1.json\", \"input-2.json\", \"output-1.json\", \"output-2.json\"} == set(files.keys())\n\n assert spack.spec.Spec.from_specfile(files[\"input-1.json\"]) == spack.spec.Spec(\"foo+x\")\n assert spack.spec.Spec.from_specfile(files[\"input-2.json\"]) == spack.spec.Spec(\"bar+y\")\n assert spack.spec.Spec.from_specfile(files[\"output-1.json\"]) == spack.spec.Spec(\"foo@=1.0~x\")\n assert spack.spec.Spec.from_specfile(files[\"output-2.json\"]) == spack.spec.Spec(\"x@=1.0~y\")\n\n\ndef assert_actionable_error(exc_info, *required_part: str) -> None:\n \"\"\"Verify that the error message contains every required part, which is usually a string that\n the user can recognize in their own input.\n \"\"\"\n msg = str(exc_info.value)\n missing = [h for h in required_part if h not in msg]\n assert not missing, f\"Error message is missing parts {missing!r}\\nFull message:\\n{msg}\"\n\n\n@pytest.mark.parametrize(\n \"input_spec,expected_parts\",\n [\n # fftw is constrained to ~mpi by the explicit request, but quantum-espresso\n # requires fftw+mpi when +invino. Both values cannot coexist.\n pytest.param(\n \"quantum-espresso+invino^fftw~mpi\", [\"fftw\", \"mpi\"], id=\"variant_value_conflict\"\n ),\n # The user requests a variant that does not exist on the package.\n pytest.param(\n \"quantum-espresso+nonexistent\",\n [\"quantum-espresso\", \"nonexistent\", \"No such variant\"],\n id=\"variant_undefined\",\n ),\n # quantum-espresso has only version 1.0; @:0.1 cannot be satisfied.\n pytest.param(\n \"quantum-espresso@:0.1\",\n [\"quantum-espresso@:0.1\", \"No version exists\"],\n id=\"version_constraint_unsatisfied\",\n ),\n # hypre propagates ~~shared to its deps, but openblas is explicitly +shared.\n pytest.param(\n \"hypre ~~shared ^openblas +shared\",\n [\"shared\", \"hypre\", \"'openblas' requires conflicting variant values\"],\n id=\"propagation_excluded\",\n ),\n # dependency-foo-bar (++bar) and direct-dep-foo-bar (~~bar) both propagate\n # variant \"bar\" with different values to their shared transitive dependency.\n pytest.param(\n \"parent-foo-bar ^dependency-foo-bar++bar ^direct-dep-foo-bar~~bar\",\n [\"cannot both propagate variant 'bar'\"],\n id=\"propagation_conflict_to_dep\",\n ),\n # gmake is a build dependency of a transitive dep, not directly reachable\n # via link/run from multivalue-variant.\n pytest.param(\n \"multivalue-variant ^gmake\",\n [\"gmake is not a direct 'build' or\"],\n id=\"literal_not_in_dag\",\n ),\n # mvapich2 file_systems uses auto_or_any_combination_of, but \"auto\" and \"lustre\"\n # come from disjoint sets and cannot be combined.\n pytest.param(\n \"mvapich2 file_systems=auto,lustre\",\n [\"mvapich2\", \"file_systems\", \"the value 'auto' is mutually exclusive\"],\n id=\"variant_disjoint_sets\",\n ),\n ],\n)\ndef test_input_spec_driven_errors(\n input_spec: str, expected_parts: List[str], mock_packages, mutable_config\n) -> None:\n \"\"\"Tests errors caused by a token in the CLI input spec. The message must name both the\n affected package and the specific token (variant, version, flag, dep) the user supplied.\n \"\"\"\n with pytest.raises(spack.error.SpackError) as exc_info:\n spack.concretize.concretize_one(input_spec)\n assert_actionable_error(exc_info, *expected_parts)\n\n\n@pytest.mark.parametrize(\n \"packages_config,input_spec,expected_parts\",\n [\n # quantum-espresso is set buildable:false; the available external does not\n # satisfy +veritas, so no valid spec can be found.\n pytest.param(\n {\n \"packages:quantum-espresso\": {\n \"buildable\": False,\n \"externals\": [\n {\"spec\": \"quantum-espresso@1.0~veritas\", \"prefix\": \"/path/to/qe\"}\n ],\n }\n },\n \"quantum-espresso+veritas\",\n [\"quantum-espresso\", \"it is configured `buildable:false`\"],\n id=\"buildable_false\",\n ),\n # The user provided a packages.yaml `require:` with a message field. The error must surface\n # the custom message so the user knows the policy and the package name so they can find\n # the config section.\n pytest.param(\n {\n \"packages:libelf\": {\n \"require\": [{\"spec\": \"%clang\", \"message\": \"must be compiled with clang\"}]\n }\n },\n \"libelf%gcc\",\n [\"libelf\", \"must be compiled with clang\"],\n id=\"requirement_unsatisfied_custom_message\",\n ),\n # Generic message must still name the package so the user knows which entry to look at\n pytest.param(\n {\"packages:libelf\": {\"require\": [\"%clang\"]}},\n \"libelf%gcc\",\n [\"libelf\"],\n id=\"requirement_unsatisfied_generic\",\n ),\n ],\n)\ndef test_config_driven_errors(\n packages_config, input_spec: str, expected_parts: List[str], mock_packages, mutable_config\n) -> None:\n \"\"\"Tests errors caused by user configuration, e,g, a setting in packages.yaml. The message must\n identify the package and the config value to fix.\n \"\"\"\n for path, conf in packages_config.items():\n spack.config.set(path, conf)\n\n with pytest.raises(spack.error.SpackError) as exc_info:\n spack.concretize.concretize_one(input_spec)\n assert_actionable_error(exc_info, *expected_parts)\n\n\n@pytest.mark.parametrize(\n \"input_spec,expected_handles\",\n [\n # conflict-parent@0.9 has conflicts(\"^conflict~foo\", when=\"@0.9\"). When the user requests\n # `^conflict~foo` the conflict fires. The auto-generated message includes the package name\n # and the when-spec version, giving the user two places to look.\n pytest.param(\n \"conflict-parent@0.9 ^conflict~foo\",\n [\"conflict-parent\", \"'^conflict~foo' conflicts with '@0.9'\"],\n id=\"conflicts_directive\",\n ),\n # requires-clang has `requires(\"%clang\", msg=\"can only be compiled with Clang\")`. When\n # compiled with %gcc the requirement is unsatisfied and the custom message is shown\n pytest.param(\"requires-clang %gcc\", [\"requires-clang\", \"Clang\"], id=\"requires_directive\"),\n ],\n)\ndef test_package_py_driven_errors(\n input_spec: str, expected_handles: List[str], mock_packages, mutable_config\n) -> None:\n \"\"\"Tests errors involving directives in package.py recipes. The error message must name the\n package whose directive caused the failure.\n \"\"\"\n with pytest.raises(spack.error.SpackError) as exc_info:\n spack.concretize.concretize_one(input_spec)\n assert_actionable_error(exc_info, *expected_handles)\n" }, { "path": "lib/spack/spack/test/concretization/flag_mixing.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"\nThese tests include the following package DAGs:\n\nFirstly, w, x, y where w and x apply cflags to y.\n\nw\n|\\\nx |\n|/\ny\n\nSecondly, v, y which where v does not apply cflags to y - this is for testing\nmixing with compiler flag propagation in the absence of compiler flags applied\nby dependents.\n\nv\n|\ny\n\nFinally, a diamond dag to check that the topological order is resolved into\na total order:\n\nt\n|\\\nu x\n|/\ny\n\"\"\"\n\nimport pathlib\n\nimport pytest\n\nimport spack.concretize\nimport spack.config\nimport spack.environment as ev\nimport spack.paths\nimport spack.repo\nimport spack.spec\nimport spack.util.spack_yaml as syaml\n\n\n@pytest.fixture\ndef test_repo(mutable_config, monkeypatch, mock_stage):\n repo_dir = pathlib.Path(spack.paths.test_repos_path) / \"spack_repo\" / \"flags_test\"\n with spack.repo.use_repositories(str(repo_dir)) as mock_packages_repo:\n yield mock_packages_repo\n\n\ndef update_concretize_scope(conf_str, section):\n conf = syaml.load_config(conf_str)\n spack.config.set(section, conf[section], scope=\"concretize\")\n\n\ndef test_mix_spec_and_requirements(concretize_scope, test_repo):\n conf_str = \"\"\"\\\npackages:\n y:\n require: cflags=\"-c\"\n\"\"\"\n update_concretize_scope(conf_str, \"packages\")\n\n s1 = spack.concretize.concretize_one('y cflags=\"-a\"')\n assert s1.satisfies('cflags=\"-a -c\"')\n\n\ndef test_mix_spec_and_dependent(concretize_scope, test_repo):\n s1 = spack.concretize.concretize_one('x ^y cflags=\"-a\"')\n assert s1[\"y\"].satisfies('cflags=\"-a -d1\"')\n\n\ndef _compiler_cfg_one_entry_with_cflags(cflags):\n return f\"\"\"\\\npackages:\n gcc:\n externals:\n - spec: gcc@12.100.100 languages:=c,c++\n prefix: /fake\n extra_attributes:\n compilers:\n c: /fake/bin/gcc\n cxx: /fake/bin/g++\n flags:\n cflags: {cflags}\n\"\"\"\n\n\ndef test_mix_spec_and_compiler_cfg(concretize_scope, test_repo):\n conf_str = _compiler_cfg_one_entry_with_cflags(\"-Wall\")\n update_concretize_scope(conf_str, \"packages\")\n\n s1 = spack.concretize.concretize_one('y cflags=\"-O2\" %gcc@12.100.100')\n assert s1.satisfies('cflags=\"-Wall -O2\"')\n\n\ndef test_pkg_flags_from_compiler_and_none(concretize_scope, mock_packages):\n packages_yaml = f\"\"\"\n{_compiler_cfg_one_entry_with_cflags(\"-Wall\")}\n llvm:\n externals:\n - spec: llvm+clang@19.1.0\n prefix: /fake\n extra_attributes:\n compilers:\n c: /fake/bin/clang\n cxx: /fake/bin/clang++\n\"\"\"\n update_concretize_scope(packages_yaml, \"packages\")\n\n s1 = spack.spec.Spec(\"cmake%gcc@12.100.100\")\n s2 = spack.spec.Spec(\"cmake-client^cmake%clang@19.1.0\")\n concrete = dict(spack.concretize.concretize_together([(s1, None), (s2, None)]))\n\n assert concrete[s1].compiler_flags[\"cflags\"] == [\"-Wall\"]\n assert concrete[s2][\"cmake\"].compiler_flags[\"cflags\"] == []\n\n\n@pytest.mark.parametrize(\n \"cmd_flags,req_flags,cmp_flags,dflags,expected_order\",\n [\n (\"-a -b\", \"-c\", None, False, \"-c -a -b\"),\n (\"-x7 -x4\", \"-x5 -x6\", None, False, \"-x5 -x6 -x7 -x4\"),\n (\"-x7 -x4\", \"-x5 -x6\", \"-x3 -x8\", False, \"-x3 -x8 -x5 -x6 -x7 -x4\"),\n (\"-x7 -x4\", \"-x5 -x6\", \"-x3 -x8\", True, \"-x3 -x8 -d1 -d2 -x5 -x6 -x7 -x4\"),\n (\"-x7 -x4\", None, \"-x3 -x8\", False, \"-x3 -x8 -x7 -x4\"),\n (\"-x7 -x4\", None, \"-x3 -x8\", True, \"-x3 -x8 -d1 -d2 -x7 -x4\"),\n # The remaining test cover cases of intersection\n (\"-a -b\", \"-a -c\", None, False, \"-c -a -b\"),\n (\"-a -b\", None, \"-a -c\", False, \"-c -a -b\"),\n (\"-a -b\", \"-a -c\", \"-a -d\", False, \"-d -c -a -b\"),\n (\"-a -d2 -d1\", \"-d2 -c\", \"-d1 -b\", True, \"-b -c -a -d2 -d1\"),\n (\"-a\", \"-d0 -d2 -c\", \"-d1 -b\", True, \"-b -d1 -d0 -d2 -c -a\"),\n ],\n)\ndef test_flag_order_and_grouping(\n concretize_scope, test_repo, cmd_flags, req_flags, cmp_flags, dflags, expected_order\n):\n \"\"\"Check consistent flag ordering and grouping on a package \"y\"\n with flags introduced from a variety of sources.\n\n The ordering rules are explained in ``asp.SpecBuilder.reorder_flags``.\n \"\"\"\n conf_str = \"\"\"\npackages:\n\"\"\"\n if cmp_flags:\n conf_str = _compiler_cfg_one_entry_with_cflags(cmp_flags)\n\n if req_flags:\n conf_str = f\"\"\"\\\n{conf_str}\n y:\n require: cflags=\"{req_flags}\"\n\"\"\"\n\n update_concretize_scope(conf_str, \"packages\")\n\n compiler_spec = \"\"\n if cmp_flags:\n compiler_spec = \"%gcc@12.100.100\"\n\n cmd_flags_str = f'cflags=\"{cmd_flags}\"' if cmd_flags else \"\"\n\n if dflags:\n spec_str = f\"x+activatemultiflag {compiler_spec} ^y {cmd_flags_str}\"\n expected_dflags = \"-d1 -d2\"\n else:\n spec_str = f\"y {cmd_flags_str} {compiler_spec}\"\n expected_dflags = None\n\n root_spec = spack.concretize.concretize_one(spec_str)\n spec = root_spec[\"y\"]\n satisfy_flags = \" \".join(x for x in [cmd_flags, req_flags, cmp_flags, expected_dflags] if x)\n assert spec.satisfies(f'cflags=\"{satisfy_flags}\"')\n assert spec.compiler_flags[\"cflags\"] == expected_order.split()\n\n\ndef test_two_dependents_flag_mixing(concretize_scope, test_repo):\n root_spec1 = spack.concretize.concretize_one(\"w~moveflaglater\")\n spec1 = root_spec1[\"y\"]\n assert spec1.compiler_flags[\"cflags\"] == \"-d0 -d1 -d2\".split()\n\n root_spec2 = spack.concretize.concretize_one(\"w+moveflaglater\")\n spec2 = root_spec2[\"y\"]\n assert spec2.compiler_flags[\"cflags\"] == \"-d3 -d1 -d2\".split()\n\n\ndef test_propagate_and_compiler_cfg(concretize_scope, test_repo):\n conf_str = _compiler_cfg_one_entry_with_cflags(\"-f2\")\n update_concretize_scope(conf_str, \"packages\")\n\n root_spec = spack.concretize.concretize_one(\"v cflags=='-f1' %gcc@12.100.100\")\n assert root_spec[\"y\"].satisfies(\"cflags='-f1 -f2'\")\n\n\ndef test_propagate_and_pkg_dep(concretize_scope, test_repo):\n root_spec1 = spack.concretize.concretize_one(\"x ~activatemultiflag cflags=='-f1'\")\n assert root_spec1[\"y\"].satisfies(\"cflags='-f1 -d1'\")\n\n\ndef test_propagate_and_require(concretize_scope, test_repo):\n conf_str = \"\"\"\\\npackages:\n y:\n require: cflags=\"-f2\"\n\"\"\"\n update_concretize_scope(conf_str, \"packages\")\n\n root_spec1 = spack.concretize.concretize_one(\"v cflags=='-f1'\")\n assert root_spec1[\"y\"].satisfies(\"cflags='-f1 -f2'\")\n\n # Next, check that a requirement does not \"undo\" a request for\n # propagation from the command-line spec\n conf_str = \"\"\"\\\npackages:\n v:\n require: cflags=\"-f1\"\n\"\"\"\n update_concretize_scope(conf_str, \"packages\")\n\n root_spec2 = spack.concretize.concretize_one(\"v cflags=='-f1'\")\n assert root_spec2[\"y\"].satisfies(\"cflags='-f1'\")\n\n # Note: requirements cannot enforce propagation: any attempt to do\n # so will generate a concretization error; this likely relates to\n # the note about #37180 in concretize.lp\n\n\ndef test_dev_mix_flags(tmp_path: pathlib.Path, concretize_scope, mutable_mock_env_path, test_repo):\n src_dir = tmp_path / \"x-src\"\n\n env_content = f\"\"\"\\\nspack:\n specs:\n - y cflags=='-fsanitize=address' %gcc@12.100.100\n develop:\n y:\n spec: y cflags=='-fsanitize=address'\n path: {src_dir}\n\"\"\"\n\n conf_str = _compiler_cfg_one_entry_with_cflags(\"-f1\")\n update_concretize_scope(conf_str, \"packages\")\n\n manifest_file = tmp_path / ev.manifest_name\n manifest_file.write_text(env_content)\n e = ev.create(\"test\", manifest_file)\n with e:\n e.concretize()\n e.write()\n\n (result,) = list(j for i, j in e.concretized_specs() if j.name == \"y\")\n\n assert result[\"y\"].satisfies(\"cflags='-fsanitize=address -f1'\")\n\n\ndef test_diamond_dep_flag_mixing(concretize_scope, test_repo):\n \"\"\"A diamond where each dependent applies flags to the bottom\n dependency. The goal is to ensure that the flag ordering is\n (a) topological and (b) repeatable for elements not subject to\n this partial ordering (i.e. the flags for the left and right\n nodes of the diamond always appear in the same order).\n `Spec.traverse` is responsible for handling both of these needs.\n \"\"\"\n root_spec1 = spack.concretize.concretize_one(\"t\")\n spec1 = root_spec1[\"y\"]\n assert spec1.satisfies('cflags=\"-c1 -c2 -d1 -d2 -e1 -e2\"')\n assert spec1.compiler_flags[\"cflags\"] == \"-c1 -c2 -e1 -e2 -d1 -d2\".split()\n\n\ndef test_flag_injection_different_compilers(mock_packages, mutable_config):\n \"\"\"Tests that flag propagation is not activated on nodes with a compiler that is different\n from the propagation source.\n \"\"\"\n s = spack.concretize.concretize_one('mpileaks cflags==\"-O2\" %gcc ^callpath %llvm')\n assert s.satisfies('cflags=\"-O2\"') and s[\"c\"].name == \"gcc\"\n assert not s[\"callpath\"].satisfies('cflags=\"-O2\"') and s[\"callpath\"][\"c\"].name == \"llvm\"\n\n\n@pytest.mark.regression(\"51209\")\n@pytest.mark.parametrize(\n \"spec_str,expected,not_expected\",\n [\n # gcc using flags compiled with another gcc not using flags\n (\"gcc@14 cflags='-O3'\", [\"gcc@14 cflags='-O3'\", \"%gcc@10\"], [\"%gcc cflags='-O3'\"]),\n # Parent and child, imposing different flags on gmake\n (\n \"7zip-dependent %gmake cflags='-O2' ^7zip %gmake cflags='-g'\",\n [\"%gmake cflags='-O2'\", \"^7zip %gmake cflags='-g'\"],\n [\"%gmake cflags='-g'\"],\n ),\n ],\n)\ndef test_flags_and_duplicate_nodes(spec_str, expected, not_expected, default_mock_concretization):\n \"\"\"Tests that we can concretize a spec with flags on a node that is present with duplicates\n in the DAG. For instance, a compiler built with a previous version of itself.\n \"\"\"\n s = default_mock_concretization(spec_str)\n assert all(s.satisfies(x) for x in expected)\n assert all(not s.satisfies(x) for x in not_expected)\n" }, { "path": "lib/spack/spack/test/concretization/preferences.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport stat\n\nimport pytest\n\nimport spack.concretize\nimport spack.config\nimport spack.package_prefs\nimport spack.repo\nimport spack.util.module_cmd\nimport spack.util.spack_yaml as syaml\nfrom spack.error import ConfigError\nfrom spack.spec import Spec\nfrom spack.version import Version\n\n\n@pytest.fixture()\ndef configure_permissions():\n conf = syaml.load_config(\n \"\"\"\\\nall:\n permissions:\n read: group\n write: group\n group: all\nmpich:\n permissions:\n read: user\n write: user\nmpileaks:\n permissions:\n write: user\n group: mpileaks\ncallpath:\n permissions:\n write: world\n\"\"\"\n )\n spack.config.set(\"packages\", conf, scope=\"concretize\")\n\n yield\n\n\ndef concretize(abstract_spec):\n return spack.concretize.concretize_one(abstract_spec)\n\n\ndef update_packages(pkgname, section, value):\n \"\"\"Update config and reread package list\"\"\"\n conf = {pkgname: {section: value}}\n spack.config.set(\"packages\", conf, scope=\"concretize\")\n\n\ndef assert_variant_values(spec, **variants):\n concrete = concretize(spec)\n for variant, value in variants.items():\n assert concrete.variants[variant].value == value\n\n\n@pytest.mark.usefixtures(\"concretize_scope\", \"mock_packages\")\nclass TestConcretizePreferences:\n @pytest.mark.parametrize(\n \"package_name,variant_value,expected_results\",\n [\n (\n \"mpileaks\",\n \"~debug~opt+shared+static\",\n {\"debug\": False, \"opt\": False, \"shared\": True, \"static\": True},\n ),\n # Check that using a list of variants instead of a single string works\n (\n \"mpileaks\",\n [\"~debug\", \"~opt\", \"+shared\", \"+static\"],\n {\"debug\": False, \"opt\": False, \"shared\": True, \"static\": True},\n ),\n # Use different values for the variants and check them again\n (\n \"mpileaks\",\n [\"+debug\", \"+opt\", \"~shared\", \"-static\"],\n {\"debug\": True, \"opt\": True, \"shared\": False, \"static\": False},\n ),\n # Check a multivalued variant with multiple values set\n (\n \"multivalue-variant\",\n [\"foo=bar,baz\", \"fee=bar\"],\n {\"foo\": (\"bar\", \"baz\"), \"fee\": \"bar\"},\n ),\n (\"singlevalue-variant\", [\"fum=why\"], {\"fum\": \"why\"}),\n ],\n )\n def test_preferred_variants(self, package_name, variant_value, expected_results):\n \"\"\"Test preferred variants are applied correctly\"\"\"\n update_packages(package_name, \"variants\", variant_value)\n assert_variant_values(package_name, **expected_results)\n\n @pytest.mark.regression(\"50921\")\n @pytest.mark.parametrize(\"config_type\", (\"require\", \"prefer\"))\n def test_preferred_commit_variant(self, config_type):\n \"\"\"Tests that we can use auto-variants in requirements and preferences.\"\"\"\n commit_value = \"b\" * 40\n name = \"git-ref-package\"\n value = f\"commit={commit_value}\"\n update_packages(name, config_type, [value])\n assert_variant_values(name, **{\"commit\": commit_value})\n\n def test_preferred_variants_from_wildcard(self):\n \"\"\"\n Test that 'foo=*' concretizes to any value\n \"\"\"\n update_packages(\"multivalue-variant\", \"variants\", \"foo=bar\")\n assert_variant_values(\"multivalue-variant foo=*\", foo=(\"bar\",))\n\n def test_preferred_target(self, mutable_mock_repo):\n \"\"\"Test preferred targets are applied correctly\"\"\"\n spec = concretize(\"mpich\")\n default = str(spec.target)\n preferred = str(spec.target.family)\n\n update_packages(\"all\", \"target\", [preferred])\n spec = concretize(\"mpich\")\n assert str(spec.target) == preferred\n\n spec = concretize(\"mpileaks\")\n assert str(spec[\"mpileaks\"].target) == preferred\n assert str(spec[\"mpi\"].target) == preferred\n\n update_packages(\"all\", \"target\", [default])\n spec = concretize(\"mpileaks\")\n assert str(spec[\"mpileaks\"].target) == default\n assert str(spec[\"mpi\"].target) == default\n\n def test_preferred_versions(self):\n \"\"\"Test preferred package versions are applied correctly\"\"\"\n update_packages(\"mpileaks\", \"version\", [\"2.3\"])\n spec = concretize(\"mpileaks\")\n assert spec.version == Version(\"2.3\")\n\n update_packages(\"mpileaks\", \"version\", [\"2.2\"])\n spec = concretize(\"mpileaks\")\n assert spec.version == Version(\"2.2\")\n\n def test_preferred_versions_mixed_version_types(self):\n update_packages(\"mixedversions\", \"version\", [\"=2.0\"])\n spec = concretize(\"mixedversions\")\n assert spec.version == Version(\"2.0\")\n\n def test_preferred_providers(self):\n \"\"\"Test preferred providers of virtual packages are\n applied correctly\n \"\"\"\n update_packages(\"all\", \"providers\", {\"mpi\": [\"mpich\"]})\n spec = concretize(\"mpileaks\")\n assert \"mpich\" in spec\n\n update_packages(\"all\", \"providers\", {\"mpi\": [\"zmpi\"]})\n spec = concretize(\"mpileaks\")\n assert \"zmpi\" in spec\n\n @pytest.mark.parametrize(\n \"update,expected\",\n [\n (\n {\"url\": \"http://www.somewhereelse.com/mpileaks-1.0.tar.gz\"},\n \"http://www.somewhereelse.com/mpileaks-2.3.tar.gz\",\n ),\n ({}, \"http://www.spack.llnl.gov/mpileaks-2.3.tar.gz\"),\n ],\n )\n def test_config_set_pkg_property_url(self, update, expected, mock_packages_repo):\n \"\"\"Test setting an existing attribute in the package class\"\"\"\n update_packages(\"mpileaks\", \"package_attributes\", update)\n with spack.repo.use_repositories(mock_packages_repo):\n spec = concretize(\"mpileaks\")\n assert spec.package.fetcher.url == expected\n\n def test_config_set_pkg_property_new(self, mock_packages_repo):\n \"\"\"Test that you can set arbitrary attributes on the Package class\"\"\"\n conf = syaml.load_config(\n \"\"\"\\\nmpileaks:\n package_attributes:\n v1: 1\n v2: true\n v3: yesterday\n v4: \"true\"\n v5:\n x: 1\n y: 2\n v6:\n - 1\n - 2\n\"\"\"\n )\n spack.config.set(\"packages\", conf, scope=\"concretize\")\n with spack.repo.use_repositories(mock_packages_repo):\n spec = concretize(\"mpileaks\")\n assert spec.package.v1 == 1\n assert spec.package.v2 is True\n assert spec.package.v3 == \"yesterday\"\n assert spec.package.v4 == \"true\"\n assert dict(spec.package.v5) == {\"x\": 1, \"y\": 2}\n assert list(spec.package.v6) == [1, 2]\n\n update_packages(\"mpileaks\", \"package_attributes\", {})\n with spack.repo.use_repositories(mock_packages_repo):\n spec = concretize(\"mpileaks\")\n with pytest.raises(AttributeError):\n spec.package.v1\n\n def test_preferred(self):\n \"\"\" \"Test packages with some version marked as preferred=True\"\"\"\n spec = spack.concretize.concretize_one(\"python\")\n assert spec.version == Version(\"2.7.11\")\n\n # now add packages.yaml with versions other than preferred\n # ensure that once config is in place, non-preferred version is used\n update_packages(\"python\", \"version\", [\"3.5.0\"])\n spec = spack.concretize.concretize_one(\"python\")\n assert spec.version == Version(\"3.5.0\")\n\n def test_preferred_undefined_raises(self):\n \"\"\"Preference should not specify an undefined version\"\"\"\n update_packages(\"python\", \"version\", [\"3.5.0.1\"])\n spec = Spec(\"python\")\n with pytest.raises(ConfigError):\n spack.concretize.concretize_one(spec)\n\n def test_preferred_truncated(self):\n \"\"\"Versions without \"=\" are treated as version ranges: if there is\n a satisfying version defined in the package.py, we should use that\n (don't define a new version).\n \"\"\"\n update_packages(\"python\", \"version\", [\"3.5\"])\n spec = spack.concretize.concretize_one(\"python\")\n assert spec.satisfies(\"@3.5.1\")\n\n def test_develop(self):\n \"\"\"Test concretization with develop-like versions\"\"\"\n spec = spack.concretize.concretize_one(\"develop-test\")\n assert spec.version == Version(\"0.2.15\")\n spec = spack.concretize.concretize_one(\"develop-test2\")\n assert spec.version == Version(\"0.2.15\")\n\n # now add packages.yaml with develop-like versions\n # ensure that once config is in place, develop-like version is used\n update_packages(\"develop-test\", \"version\", [\"develop\"])\n spec = spack.concretize.concretize_one(\"develop-test\")\n assert spec.version == Version(\"develop\")\n\n update_packages(\"develop-test2\", \"version\", [\"0.2.15.develop\"])\n spec = spack.concretize.concretize_one(\"develop-test2\")\n assert spec.version == Version(\"0.2.15.develop\")\n\n def test_external_mpi(self):\n # make sure this doesn't give us an external first.\n spec = spack.concretize.concretize_one(\"mpi\")\n assert not spec.external and spec.package.provides(\"mpi\")\n\n # load config\n conf = syaml.load_config(\n \"\"\"\\\nall:\n providers:\n mpi: [mpich]\nmpich:\n buildable: false\n externals:\n - spec: mpich@3.0.4\n prefix: /dummy/path\n\"\"\"\n )\n spack.config.set(\"packages\", conf, scope=\"concretize\")\n\n # ensure that once config is in place, external is used\n spec = spack.concretize.concretize_one(\"mpi\")\n assert spec[\"mpich\"].external_path == os.path.sep + os.path.join(\"dummy\", \"path\")\n\n def test_external_module(self, monkeypatch):\n \"\"\"Test that packages can find externals specified by module\n\n The specific code for parsing the module is tested elsewhere.\n This just tests that the preference is accounted for\"\"\"\n\n # make sure this doesn't give us an external first.\n def mock_module(cmd, module):\n return \"prepend-path PATH /dummy/path\"\n\n monkeypatch.setattr(spack.util.module_cmd, \"module\", mock_module)\n\n spec = spack.concretize.concretize_one(\"mpi\")\n assert not spec.external and spec.package.provides(\"mpi\")\n\n # load config\n conf = syaml.load_config(\n \"\"\"\\\nall:\n providers:\n mpi: [mpich]\nmpi:\n buildable: false\n externals:\n - spec: mpich@3.0.4\n modules: [dummy]\n\"\"\"\n )\n spack.config.set(\"packages\", conf, scope=\"concretize\")\n\n # ensure that once config is in place, external is used\n spec = spack.concretize.concretize_one(\"mpi\")\n assert spec[\"mpich\"].external_path == os.path.sep + os.path.join(\"dummy\", \"path\")\n\n def test_buildable_false(self):\n conf = syaml.load_config(\n \"\"\"\\\nlibelf:\n buildable: false\n\"\"\"\n )\n spack.config.set(\"packages\", conf, scope=\"concretize\")\n spec = Spec(\"libelf\")\n assert not spack.package_prefs.is_spec_buildable(spec)\n\n spec = Spec(\"mpich\")\n assert spack.package_prefs.is_spec_buildable(spec)\n\n def test_buildable_false_virtual(self):\n conf = syaml.load_config(\n \"\"\"\\\nmpi:\n buildable: false\n\"\"\"\n )\n spack.config.set(\"packages\", conf, scope=\"concretize\")\n spec = Spec(\"libelf\")\n assert spack.package_prefs.is_spec_buildable(spec)\n\n spec = Spec(\"mpich\")\n assert not spack.package_prefs.is_spec_buildable(spec)\n\n def test_buildable_false_all(self):\n conf = syaml.load_config(\n \"\"\"\\\nall:\n buildable: false\n\"\"\"\n )\n spack.config.set(\"packages\", conf, scope=\"concretize\")\n spec = Spec(\"libelf\")\n assert not spack.package_prefs.is_spec_buildable(spec)\n\n spec = Spec(\"mpich\")\n assert not spack.package_prefs.is_spec_buildable(spec)\n\n def test_buildable_false_all_true_package(self):\n conf = syaml.load_config(\n \"\"\"\\\nall:\n buildable: false\nlibelf:\n buildable: true\n\"\"\"\n )\n spack.config.set(\"packages\", conf, scope=\"concretize\")\n spec = Spec(\"libelf\")\n assert spack.package_prefs.is_spec_buildable(spec)\n\n spec = Spec(\"mpich\")\n assert not spack.package_prefs.is_spec_buildable(spec)\n\n def test_buildable_false_all_true_virtual(self):\n conf = syaml.load_config(\n \"\"\"\\\nall:\n buildable: false\nmpi:\n buildable: true\n\"\"\"\n )\n spack.config.set(\"packages\", conf, scope=\"concretize\")\n spec = Spec(\"libelf\")\n assert not spack.package_prefs.is_spec_buildable(spec)\n\n spec = Spec(\"mpich\")\n assert spack.package_prefs.is_spec_buildable(spec)\n\n def test_buildable_false_virtual_true_pacakge(self):\n conf = syaml.load_config(\n \"\"\"\\\nmpi:\n buildable: false\nmpich:\n buildable: true\n\"\"\"\n )\n spack.config.set(\"packages\", conf, scope=\"concretize\")\n\n spec = Spec(\"zmpi\")\n assert not spack.package_prefs.is_spec_buildable(spec)\n\n spec = Spec(\"mpich\")\n assert spack.package_prefs.is_spec_buildable(spec)\n\n def test_config_permissions_from_all(self, configure_permissions):\n # Although these aren't strictly about concretization, they are\n # configured in the same file and therefore convenient to test here.\n # Make sure we can configure readable and writable\n\n # Test inheriting from 'all'\n spec = Spec(\"zmpi\")\n perms = spack.package_prefs.get_package_permissions(spec)\n assert perms == stat.S_IRWXU | stat.S_IRWXG\n\n dir_perms = spack.package_prefs.get_package_dir_permissions(spec)\n assert dir_perms == stat.S_IRWXU | stat.S_IRWXG | stat.S_ISGID\n\n group = spack.package_prefs.get_package_group(spec)\n assert group == \"all\"\n\n def test_config_permissions_from_package(self, configure_permissions):\n # Test overriding 'all'\n spec = Spec(\"mpich\")\n perms = spack.package_prefs.get_package_permissions(spec)\n assert perms == stat.S_IRWXU\n\n dir_perms = spack.package_prefs.get_package_dir_permissions(spec)\n assert dir_perms == stat.S_IRWXU\n\n group = spack.package_prefs.get_package_group(spec)\n assert group == \"all\"\n\n def test_config_permissions_differ_read_write(self, configure_permissions):\n # Test overriding group from 'all' and different readable/writable\n spec = Spec(\"mpileaks\")\n perms = spack.package_prefs.get_package_permissions(spec)\n assert perms == stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP\n\n dir_perms = spack.package_prefs.get_package_dir_permissions(spec)\n expected = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_ISGID\n assert dir_perms == expected\n\n group = spack.package_prefs.get_package_group(spec)\n assert group == \"mpileaks\"\n\n def test_config_perms_fail_write_gt_read(self, configure_permissions):\n # Test failure for writable more permissive than readable\n spec = Spec(\"callpath\")\n with pytest.raises(ConfigError):\n spack.package_prefs.get_package_permissions(spec)\n\n @pytest.mark.regression(\"20040\")\n def test_variant_not_flipped_to_pull_externals(self):\n \"\"\"Test that a package doesn't prefer pulling in an\n external to using the default value of a variant.\n \"\"\"\n s = spack.concretize.concretize_one(\"vdefault-or-external-root\")\n\n assert \"~external\" in s[\"vdefault-or-external\"]\n assert \"externaltool\" not in s\n\n @pytest.mark.regression(\"25585\")\n def test_dependencies_cant_make_version_parent_score_better(self):\n \"\"\"Test that a package can't select a worse version for a\n dependent because doing so it can pull-in a dependency\n that makes the overall version score even or better and maybe\n has a better score in some lower priority criteria.\n \"\"\"\n s = spack.concretize.concretize_one(\"version-test-root\")\n\n assert s.satisfies(\"^version-test-pkg@2.4.6\")\n assert \"version-test-dependency-preferred\" not in s\n\n @pytest.mark.regression(\"26598\")\n def test_multivalued_variants_are_lower_priority_than_providers(self):\n \"\"\"Test that the rule to maximize the number of values for multivalued\n variants is considered at lower priority than selecting the default\n provider for virtual dependencies.\n\n This ensures that we don't e.g. select openmpi over mpich even if we\n specified mpich as the default mpi provider, just because openmpi supports\n more fabrics by default.\n \"\"\"\n with spack.config.override(\n \"packages:all\", {\"providers\": {\"somevirtual\": [\"some-virtual-preferred\"]}}\n ):\n s = spack.concretize.concretize_one(\"somevirtual\")\n assert s.name == \"some-virtual-preferred\"\n\n @pytest.mark.regression(\"26721,19736\")\n def test_sticky_variant_accounts_for_packages_yaml(self):\n with spack.config.override(\"packages:sticky-variant\", {\"variants\": \"+allow-gcc\"}):\n s = spack.concretize.concretize_one(\"sticky-variant %gcc\")\n assert s.satisfies(\"%gcc\") and s.satisfies(\"+allow-gcc\")\n\n @pytest.mark.regression(\"41134\")\n def test_default_preference_variant_different_type_does_not_error(self):\n \"\"\"Tests that a different type for an existing variant in the 'all:' section of\n packages.yaml doesn't fail with an error.\n \"\"\"\n with spack.config.override(\"packages:all\", {\"variants\": \"+foo\"}):\n s = spack.concretize.concretize_one(\"pkg-a\")\n assert s.satisfies(\"foo=bar\")\n\n def test_version_preference_cannot_generate_buildable_versions(self):\n \"\"\"Tests that a version preference not mentioned in package.py cannot be used in\n a built spec.\n \"\"\"\n mpileaks_external = syaml.load_config(\n \"\"\"\n mpileaks:\n # Version 0.9 is not mentioned in package.py\n version: [\"0.9\"]\n buildable: true\n externals:\n - spec: mpileaks@0.9 +debug\n prefix: /path\n \"\"\"\n )\n\n with spack.config.override(\"packages\", mpileaks_external):\n # Asking for mpileaks+debug results in the external being chosen\n mpileaks = spack.concretize.concretize_one(\"mpileaks+debug\")\n assert mpileaks.external and mpileaks.satisfies(\"@0.9 +debug\")\n\n # Asking for ~debug results in the highest known version being chosen\n mpileaks = spack.concretize.concretize_one(\"mpileaks~debug\")\n assert not mpileaks.external and mpileaks.satisfies(\"@2.3 ~debug\")\n" }, { "path": "lib/spack/spack/test/concretization/requirements.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport pathlib\n\nimport pytest\n\nimport spack.concretize\nimport spack.config\nimport spack.error\nimport spack.installer\nimport spack.package_base\nimport spack.paths\nimport spack.platforms\nimport spack.repo\nimport spack.solver.asp\nimport spack.spec\nimport spack.store\nimport spack.util.spack_yaml as syaml\nimport spack.version\nfrom spack.installer import PackageInstaller\nfrom spack.solver.asp import InternalConcretizerError, UnsatisfiableSpecError\nfrom spack.solver.reuse import create_external_parser, spec_filter_from_packages_yaml\nfrom spack.solver.runtimes import external_config_with_implicit_externals\nfrom spack.spec import Spec\nfrom spack.util.url import path_to_file_url\n\n\ndef update_packages_config(conf_str):\n conf = syaml.load_config(conf_str)\n spack.config.set(\"packages\", conf[\"packages\"], scope=\"concretize\")\n\n\n@pytest.fixture\ndef test_repo(mutable_config, monkeypatch, mock_stage):\n repo_dir = pathlib.Path(spack.paths.test_repos_path) / \"spack_repo\" / \"requirements_test\"\n with spack.repo.use_repositories(str(repo_dir)) as mock_packages_repo:\n yield mock_packages_repo\n\n\ndef test_one_package_multiple_reqs(concretize_scope, test_repo):\n conf_str = \"\"\"\\\npackages:\n y:\n require:\n - \"@2.4\"\n - \"~shared\"\n\"\"\"\n update_packages_config(conf_str)\n y_spec = spack.concretize.concretize_one(\"y\")\n assert y_spec.satisfies(\"@2.4~shared\")\n\n\ndef test_requirement_isnt_optional(concretize_scope, test_repo):\n \"\"\"If a user spec requests something that directly conflicts\n with a requirement, make sure we get an error.\n \"\"\"\n conf_str = \"\"\"\\\npackages:\n x:\n require: \"@1.0\"\n\"\"\"\n update_packages_config(conf_str)\n with pytest.raises(UnsatisfiableSpecError):\n spack.concretize.concretize_one(\"x@1.1\")\n\n\ndef test_require_undefined_version(concretize_scope, test_repo):\n \"\"\"If a requirement specifies a numbered version that isn't in\n the associated package.py and isn't part of a Git hash\n equivalence (hash=number), then Spack should raise an error\n (it is assumed this is a typo, and raising the error here\n avoids a likely error when Spack attempts to fetch the version).\n \"\"\"\n conf_str = \"\"\"\\\npackages:\n x:\n require: \"@1.2\"\n\"\"\"\n update_packages_config(conf_str)\n with pytest.raises(spack.error.ConfigError):\n spack.concretize.concretize_one(\"x\")\n\n\ndef test_require_truncated(concretize_scope, test_repo):\n \"\"\"A requirement specifies a version range, with satisfying\n versions defined in the package.py. Make sure we choose one\n of the defined versions (vs. allowing the requirement to\n define a new version).\n \"\"\"\n conf_str = \"\"\"\\\npackages:\n x:\n require: \"@1\"\n\"\"\"\n update_packages_config(conf_str)\n xspec = spack.concretize.concretize_one(\"x\")\n assert xspec.satisfies(\"@1.1\")\n\n\ndef test_git_user_supplied_reference_satisfaction(\n concretize_scope, test_repo, mock_git_version_info, monkeypatch\n):\n repo_path, filename, commits = mock_git_version_info\n\n monkeypatch.setattr(\n spack.package_base.PackageBase, \"git\", path_to_file_url(repo_path), raising=False\n )\n\n hash_eq_ver = Spec(f\"v@{commits[0]}=2.2\")\n hash_eq_ver_copy = Spec(f\"v@{commits[0]}=2.2\")\n just_hash = Spec(f\"v@{commits[0]}\")\n just_ver = Spec(\"v@=2.2\")\n hash_eq_other_ver = Spec(f\"v@{commits[0]}=2.3\")\n\n assert not hash_eq_ver == just_hash\n assert not hash_eq_ver.satisfies(just_hash)\n assert not hash_eq_ver.intersects(just_hash)\n\n # Git versions and literal versions are distinct versions, like\n # pkg@10.1.0 and pkg@10.1.0-suffix are distinct versions.\n assert not hash_eq_ver.satisfies(just_ver)\n assert not just_ver.satisfies(hash_eq_ver)\n assert not hash_eq_ver.intersects(just_ver)\n assert hash_eq_ver != just_ver\n assert just_ver != hash_eq_ver\n assert not hash_eq_ver == just_ver\n assert not just_ver == hash_eq_ver\n\n # When a different version is associated, they're not equal\n assert not hash_eq_ver.satisfies(hash_eq_other_ver)\n assert not hash_eq_other_ver.satisfies(hash_eq_ver)\n assert not hash_eq_ver.intersects(hash_eq_other_ver)\n assert not hash_eq_other_ver.intersects(hash_eq_ver)\n assert hash_eq_ver != hash_eq_other_ver\n assert hash_eq_other_ver != hash_eq_ver\n assert not hash_eq_ver == hash_eq_other_ver\n assert not hash_eq_other_ver == hash_eq_ver\n\n # These should be equal\n assert hash_eq_ver == hash_eq_ver_copy\n assert not hash_eq_ver != hash_eq_ver_copy\n assert hash_eq_ver.satisfies(hash_eq_ver_copy)\n assert hash_eq_ver_copy.satisfies(hash_eq_ver)\n assert hash_eq_ver.intersects(hash_eq_ver_copy)\n assert hash_eq_ver_copy.intersects(hash_eq_ver)\n\n\ndef test_requirement_adds_new_version(\n concretize_scope, test_repo, mock_git_version_info, monkeypatch\n):\n repo_path, filename, commits = mock_git_version_info\n monkeypatch.setattr(\n spack.package_base.PackageBase, \"git\", path_to_file_url(repo_path), raising=False\n )\n\n a_commit_hash = commits[0]\n conf_str = \"\"\"\\\npackages:\n v:\n require: \"@{0}=2.2\"\n\"\"\".format(a_commit_hash)\n update_packages_config(conf_str)\n\n s1 = spack.concretize.concretize_one(\"v\")\n assert s1.satisfies(\"@2.2\")\n # Make sure the git commit info is retained\n assert isinstance(s1.version, spack.version.GitVersion)\n assert s1.version.ref == a_commit_hash\n\n\ndef test_requirement_adds_version_satisfies(\n concretize_scope, test_repo, mock_git_version_info, monkeypatch\n):\n \"\"\"Make sure that new versions added by requirements are factored into\n conditions. In this case create a new version that satisfies a\n depends_on condition and make sure it is triggered (i.e. the\n dependency is added).\n \"\"\"\n repo_path, filename, commits = mock_git_version_info\n monkeypatch.setattr(\n spack.package_base.PackageBase, \"git\", path_to_file_url(repo_path), raising=False\n )\n\n # Sanity check: early version of T does not include U\n s0 = spack.concretize.concretize_one(\"t@2.0\")\n assert \"u\" not in s0\n\n conf_str = \"\"\"\\\npackages:\n t:\n require: \"@{0}=2.2\"\n\"\"\".format(commits[0])\n update_packages_config(conf_str)\n\n s1 = spack.concretize.concretize_one(\"t\")\n assert \"u\" in s1\n assert s1.satisfies(\"@2.2\")\n\n\n@pytest.mark.parametrize(\"require_checksum\", (True, False))\ndef test_requirement_adds_git_hash_version(\n require_checksum, concretize_scope, test_repo, mock_git_version_info, monkeypatch\n):\n # A full commit sha is a checksummed version, so this test should pass in both cases\n if require_checksum:\n monkeypatch.setenv(\"SPACK_CONCRETIZER_REQUIRE_CHECKSUM\", \"yes\")\n\n repo_path, filename, commits = mock_git_version_info\n monkeypatch.setattr(\n spack.package_base.PackageBase, \"git\", path_to_file_url(repo_path), raising=False\n )\n\n a_commit_hash = commits[0]\n conf_str = f\"\"\"\\\npackages:\n v:\n require: \"@{a_commit_hash}\"\n\"\"\"\n update_packages_config(conf_str)\n\n s1 = spack.concretize.concretize_one(\"v\")\n assert isinstance(s1.version, spack.version.GitVersion)\n assert s1.satisfies(f\"v@{a_commit_hash}\")\n\n\ndef test_requirement_adds_multiple_new_versions(\n concretize_scope, test_repo, mock_git_version_info, monkeypatch\n):\n repo_path, filename, commits = mock_git_version_info\n monkeypatch.setattr(\n spack.package_base.PackageBase, \"git\", path_to_file_url(repo_path), raising=False\n )\n\n conf_str = f\"\"\"\\\npackages:\n v:\n require:\n - one_of: [\"@{commits[0]}=2.2\", \"@{commits[1]}=2.3\"]\n\"\"\"\n update_packages_config(conf_str)\n\n assert spack.concretize.concretize_one(\"v\").satisfies(f\"@{commits[0]}=2.2\")\n assert spack.concretize.concretize_one(\"v@2.3\").satisfies(f\"v@{commits[1]}=2.3\")\n\n\n# TODO: this belongs in the concretize_preferences test module but uses\n# fixtures defined only here\ndef test_preference_adds_new_version(\n concretize_scope, test_repo, mock_git_version_info, monkeypatch\n):\n \"\"\"Normally a preference cannot define a new version, but that constraint\n is ignored if the version is a Git hash-based version.\n \"\"\"\n repo_path, filename, commits = mock_git_version_info\n monkeypatch.setattr(\n spack.package_base.PackageBase, \"git\", path_to_file_url(repo_path), raising=False\n )\n\n conf_str = f\"\"\"\\\npackages:\n v:\n version: [\"{commits[0]}=2.2\", \"{commits[1]}=2.3\"]\n\"\"\"\n update_packages_config(conf_str)\n\n assert spack.concretize.concretize_one(\"v\").satisfies(f\"@{commits[0]}=2.2\")\n assert spack.concretize.concretize_one(\"v@2.3\").satisfies(f\"@{commits[1]}=2.3\")\n\n # When installing by hash, a lookup is triggered, so it's not mapped to =2.3.\n s3 = spack.concretize.concretize_one(f\"v@{commits[1]}\")\n assert s3.satisfies(f\"v@{commits[1]}\")\n assert not s3.satisfies(\"@2.3\")\n\n\ndef test_external_adds_new_version_that_is_preferred(concretize_scope, test_repo):\n \"\"\"Test that we can use a version, not declared in package recipe, as the\n preferred version if that version appears in an external spec.\n \"\"\"\n conf_str = \"\"\"\\\npackages:\n y:\n version: [\"2.7\"]\n externals:\n - spec: y@2.7 # Not defined in y\n prefix: /fake/nonexistent/path/\n buildable: false\n\"\"\"\n update_packages_config(conf_str)\n\n spec = spack.concretize.concretize_one(\"x\")\n assert spec[\"y\"].satisfies(\"@2.7\")\n assert spack.version.Version(\"2.7\") not in spec[\"y\"].package.versions\n\n\ndef test_requirement_is_successfully_applied(concretize_scope, test_repo):\n \"\"\"If a simple requirement can be satisfied, make sure the\n concretization succeeds and the requirement spec is applied.\n \"\"\"\n s1 = spack.concretize.concretize_one(\"x\")\n # Without any requirements/preferences, the later version is preferred\n assert s1.satisfies(\"@1.1\")\n\n conf_str = \"\"\"\\\npackages:\n x:\n require: \"@1.0\"\n\"\"\"\n update_packages_config(conf_str)\n s2 = spack.concretize.concretize_one(\"x\")\n # The requirement forces choosing the earlier version\n assert s2.satisfies(\"@1.0\")\n\n\ndef test_require_hash(mock_fetch, install_mockery, concretize_scope, test_repo):\n \"\"\"Apply a requirement to use a specific hash.\n\n Install multiple hashes to ensure non-default concretization\"\"\"\n s1 = spack.concretize.concretize_one(\"x@1.1\")\n s2 = spack.concretize.concretize_one(\"x@1.0\")\n\n builder = spack.installer.PackageInstaller([s1.package, s2.package], fake=True)\n builder.install()\n\n conf_str = f\"\"\"\\\npackages:\n x:\n require: x/{s2.dag_hash()}\n\"\"\"\n update_packages_config(conf_str)\n\n test_spec = spack.concretize.concretize_one(\"x\")\n assert test_spec == s2\n\n\ndef test_multiple_packages_requirements_are_respected(concretize_scope, test_repo):\n \"\"\"Apply requirements to two packages; make sure the concretization\n succeeds and both requirements are respected.\n \"\"\"\n conf_str = \"\"\"\\\npackages:\n x:\n require: \"@1.0\"\n y:\n require: \"@2.4\"\n\"\"\"\n update_packages_config(conf_str)\n spec = spack.concretize.concretize_one(\"x\")\n assert spec[\"x\"].satisfies(\"@1.0\")\n assert spec[\"y\"].satisfies(\"@2.4\")\n\n\ndef test_oneof(concretize_scope, test_repo):\n \"\"\"'one_of' allows forcing the concretizer to satisfy one of\n the specs in the group (but not all have to be satisfied).\n \"\"\"\n conf_str = \"\"\"\\\npackages:\n y:\n require:\n - one_of: [\"@2.4\", \"~shared\"]\n\"\"\"\n update_packages_config(conf_str)\n spec = spack.concretize.concretize_one(\"x\")\n # The concretizer only has to satisfy one of @2.4/~shared, and @2.4\n # comes first so it is prioritized\n assert spec[\"y\"].satisfies(\"@2.4+shared\")\n\n\ndef test_one_package_multiple_oneof_groups(concretize_scope, test_repo):\n \"\"\"One package has two 'one_of' groups; check that both are\n applied.\n \"\"\"\n conf_str = \"\"\"\\\npackages:\n y:\n require:\n - one_of: [\"@2.4%gcc\", \"@2.5%clang\"]\n - one_of: [\"@2.5~shared\", \"@2.4+shared\"]\n\"\"\"\n update_packages_config(conf_str)\n\n s1 = spack.concretize.concretize_one(\"y@2.5\")\n assert s1.satisfies(\"~shared%clang\")\n\n s2 = spack.concretize.concretize_one(\"y@2.4\")\n assert s2.satisfies(\"+shared%gcc\")\n\n\n@pytest.mark.regression(\"34241\")\ndef test_require_cflags(concretize_scope, mock_packages):\n \"\"\"Ensures that flags can be required from configuration.\"\"\"\n conf_str = \"\"\"\\\npackages:\n mpich2:\n require: cflags=\"-g\"\n mpi:\n require: mpich cflags=\"-O1\"\n\"\"\"\n update_packages_config(conf_str)\n\n mpich2 = spack.concretize.concretize_one(\"mpich2\")\n assert mpich2.satisfies(\"cflags=-g\")\n\n mpileaks = spack.concretize.concretize_one(\"mpileaks\")\n assert mpileaks[\"mpi\"].satisfies(\"mpich cflags=-O1\")\n\n mpi = spack.concretize.concretize_one(\"mpi\")\n assert mpi.satisfies(\"mpich cflags=-O1\")\n\n\ndef test_requirements_for_package_that_is_not_needed(concretize_scope, test_repo):\n \"\"\"Specify requirements for specs that are not concretized or\n a dependency of a concretized spec (in other words, none of\n the requirements are used for the requested spec).\n \"\"\"\n # Note that the exact contents aren't important since this isn't\n # intended to be used, but the important thing is that a number of\n # packages have requirements applied\n conf_str = \"\"\"\\\npackages:\n x:\n require: \"@1.0\"\n y:\n require:\n - one_of: [\"@2.4%gcc\", \"@2.5%clang\"]\n - one_of: [\"@2.5~shared\", \"@2.4+shared\"]\n\"\"\"\n update_packages_config(conf_str)\n\n s1 = spack.concretize.concretize_one(\"v\")\n assert s1.satisfies(\"@2.1\")\n\n\ndef test_oneof_ordering(concretize_scope, test_repo):\n \"\"\"Ensure that earlier elements of 'one_of' have higher priority.\n This priority should override default priority (e.g. choosing\n later versions).\n \"\"\"\n conf_str = \"\"\"\\\npackages:\n y:\n require:\n - one_of: [\"@2.4\", \"@2.5\"]\n\"\"\"\n update_packages_config(conf_str)\n\n s1 = spack.concretize.concretize_one(\"y\")\n assert s1.satisfies(\"@2.4\")\n\n s2 = spack.concretize.concretize_one(\"y@2.5\")\n assert s2.satisfies(\"@2.5\")\n\n\ndef test_reuse_oneof(concretize_scope, test_repo, tmp_path: pathlib.Path, mock_fetch):\n conf_str = \"\"\"\\\npackages:\n y:\n require:\n - one_of: [\"@2.5\", \"~shared\"]\n\"\"\"\n\n store_dir = tmp_path / \"store\"\n with spack.store.use_store(str(store_dir)):\n s1 = spack.concretize.concretize_one(\"y@2.5~shared\")\n PackageInstaller([s1.package], fake=True, explicit=True).install()\n\n update_packages_config(conf_str)\n\n with spack.config.override(\"concretizer:reuse\", True):\n s2 = spack.concretize.concretize_one(\"y\")\n assert not s2.satisfies(\"@2.5~shared\")\n\n\n@pytest.mark.parametrize(\n \"allow_deprecated,expected,not_expected\",\n [(True, [\"@=2.3\", \"%gcc\"], []), (False, [\"%gcc\"], [\"@=2.3\"])],\n)\ndef test_requirements_and_deprecated_versions(\n allow_deprecated, expected, not_expected, concretize_scope, test_repo\n):\n \"\"\"Tests the expected behavior of requirements and deprecated versions.\n\n If deprecated versions are not allowed, concretization should just pick\n the other requirement.\n\n If deprecated versions are allowed, both requirements are honored.\n \"\"\"\n # 2.3 is a deprecated versions. Ensure that any_of picks both constraints,\n # since they are possible\n conf_str = \"\"\"\\\npackages:\n y:\n require:\n - any_of: [\"@=2.3\", \"%gcc\"]\n\"\"\"\n update_packages_config(conf_str)\n\n with spack.config.override(\"config:deprecated\", allow_deprecated):\n s1 = spack.concretize.concretize_one(\"y\")\n for constrain in expected:\n assert s1.satisfies(constrain)\n\n for constrain in not_expected:\n assert not s1.satisfies(constrain)\n\n\n@pytest.mark.parametrize(\"spec_str,requirement_str\", [(\"x\", \"%gcc\"), (\"x\", \"%clang\")])\ndef test_default_requirements_with_all(spec_str, requirement_str, concretize_scope, test_repo):\n \"\"\"Test that default requirements are applied to all packages.\"\"\"\n conf_str = f\"\"\"\\\npackages:\n all:\n require: \"{requirement_str}\"\n\"\"\"\n update_packages_config(conf_str)\n\n spec = spack.concretize.concretize_one(spec_str)\n assert \"c\" in spec\n for s in spec.traverse():\n if \"c\" in s and s.name not in (\"gcc\", \"llvm\"):\n assert s.satisfies(requirement_str)\n\n\n@pytest.mark.parametrize(\n \"requirements,expectations\",\n [\n ((\"%gcc\", \"%clang\"), (\"%gcc\", \"%clang\")),\n ((\"~shared%gcc\", \"@1.0\"), (\"~shared%gcc\", \"@1.0+shared\")),\n ],\n)\ndef test_default_and_package_specific_requirements(\n concretize_scope, requirements, expectations, test_repo\n):\n \"\"\"Test that specific package requirements override default package requirements.\"\"\"\n generic_req, specific_req = requirements\n generic_exp, specific_exp = expectations\n conf_str = f\"\"\"\\\npackages:\n all:\n require: \"{generic_req}\"\n x:\n require: \"{specific_req}\"\n\"\"\"\n update_packages_config(conf_str)\n\n spec = spack.concretize.concretize_one(\"x\")\n assert spec.satisfies(specific_exp)\n assert spec[\"y\"].satisfies(generic_exp)\n\n\n@pytest.mark.parametrize(\"mpi_requirement\", [\"mpich\", \"mpich2\", \"zmpi\"])\ndef test_requirements_on_virtual(mpi_requirement, concretize_scope, mock_packages):\n conf_str = f\"\"\"\\\npackages:\n mpi:\n require: \"{mpi_requirement}\"\n\"\"\"\n update_packages_config(conf_str)\n\n spec = spack.concretize.concretize_one(\"callpath\")\n assert \"mpi\" in spec\n assert mpi_requirement in spec\n\n\n@pytest.mark.parametrize(\n \"mpi_requirement,specific_requirement\",\n [(\"mpich\", \"@3.0.3\"), (\"mpich2\", \"%clang\"), (\"zmpi\", \"%gcc\")],\n)\ndef test_requirements_on_virtual_and_on_package(\n mpi_requirement, specific_requirement, concretize_scope, mock_packages\n):\n conf_str = f\"\"\"\\\npackages:\n mpi:\n require: \"{mpi_requirement}\"\n {mpi_requirement}:\n require: \"{specific_requirement}\"\n\"\"\"\n update_packages_config(conf_str)\n\n spec = spack.concretize.concretize_one(\"callpath\")\n assert \"mpi\" in spec\n assert mpi_requirement in spec\n assert spec[\"mpi\"].satisfies(specific_requirement)\n\n\ndef test_incompatible_virtual_requirements_raise(concretize_scope, mock_packages):\n conf_str = \"\"\"\\\n packages:\n mpi:\n require: \"mpich\"\n \"\"\"\n update_packages_config(conf_str)\n\n spec = Spec(\"callpath^zmpi\")\n # TODO (multiple nodes): recover a better error message later\n with pytest.raises((UnsatisfiableSpecError, InternalConcretizerError)):\n spack.concretize.concretize_one(spec)\n\n\ndef test_non_existing_variants_under_all(concretize_scope, mock_packages):\n conf_str = \"\"\"\\\n packages:\n all:\n require:\n - any_of: [\"~foo\", \"@:\"]\n \"\"\"\n update_packages_config(conf_str)\n\n spec = spack.concretize.concretize_one(\"callpath^zmpi\")\n assert \"~foo\" not in spec\n\n\n@pytest.mark.parametrize(\n \"packages_yaml,spec_str,expected_satisfies\",\n [\n # In the tests below we set the compiler preference to \"gcc\" to be explicit on the\n # fact that \"clang\" is not the preferred compiler. That helps making more robust the\n # tests that verify enforcing \"%clang\" as a requirement.\n (\n \"\"\"\\\n packages:\n all:\n compiler: [\"gcc\", \"clang\"]\n\n libelf:\n require:\n - one_of: [\"%clang\"]\n when: \"@0.8.13\"\n\"\"\",\n \"libelf\",\n [(\"@0.8.13%clang\", True), (\"%gcc\", False)],\n ),\n (\n \"\"\"\\\n packages:\n all:\n compiler: [\"gcc\", \"clang\"]\n\n libelf:\n require:\n - one_of: [\"%clang\"]\n when: \"@0.8.13\"\n\"\"\",\n \"libelf@0.8.12\",\n [(\"%clang\", False), (\"%gcc\", True)],\n ),\n (\n \"\"\"\\\n packages:\n all:\n compiler: [\"gcc\", \"clang\"]\n\n libelf:\n require:\n - spec: \"%clang\"\n when: \"@0.8.13\"\n\"\"\",\n \"libelf@0.8.12\",\n [(\"%clang\", False), (\"%gcc\", True)],\n ),\n (\n \"\"\"\\\n packages:\n all:\n compiler: [\"gcc\", \"clang\"]\n\n libelf:\n require:\n - spec: \"@0.8.13\"\n when: \"%clang\"\n\"\"\",\n \"libelf@0.8.13%gcc\",\n [(\"%clang\", False), (\"%gcc\", True), (\"@0.8.13\", True)],\n ),\n ],\n)\ndef test_conditional_requirements_from_packages_yaml(\n packages_yaml, spec_str, expected_satisfies, concretize_scope, mock_packages\n):\n \"\"\"Test that conditional requirements are required when the condition is met,\n and optional when the condition is not met.\n \"\"\"\n update_packages_config(packages_yaml)\n spec = spack.concretize.concretize_one(spec_str)\n for match_str, expected in expected_satisfies:\n assert spec.satisfies(match_str) is expected\n\n\n@pytest.mark.parametrize(\n \"packages_yaml,spec_str,expected_message\",\n [\n (\n \"\"\"\\\n packages:\n mpileaks:\n require:\n - one_of: [\"~debug\"]\n message: \"debug is not allowed\"\n\"\"\",\n \"mpileaks+debug\",\n \"debug is not allowed\",\n ),\n (\n \"\"\"\\\n packages:\n libelf:\n require:\n - one_of: [\"%clang\"]\n message: \"can only be compiled with clang\"\n\"\"\",\n \"libelf%gcc\",\n \"can only be compiled with clang\",\n ),\n (\n \"\"\"\\\n packages:\n libelf:\n require:\n - one_of: [\"%clang\"]\n when: platform=test\n message: \"can only be compiled with clang on the test platform\"\n \"\"\",\n \"libelf%gcc\",\n \"can only be compiled with clang on \",\n ),\n (\n \"\"\"\\\n packages:\n libelf:\n require:\n - spec: \"%clang\"\n when: platform=test\n message: \"can only be compiled with clang on the test platform\"\n \"\"\",\n \"libelf%gcc\",\n \"can only be compiled with clang on \",\n ),\n (\n \"\"\"\\\n packages:\n libelf:\n require:\n - one_of: [\"%clang\", \"%intel\"]\n when: platform=test\n message: \"can only be compiled with clang or intel on the test platform\"\n \"\"\",\n \"libelf%gcc\",\n \"can only be compiled with clang or intel\",\n ),\n ],\n)\ndef test_requirements_fail_with_custom_message(\n packages_yaml, spec_str, expected_message, concretize_scope, mock_packages\n):\n \"\"\"Test that specs failing due to requirements not being satisfiable fail with a\n custom error message.\n \"\"\"\n update_packages_config(packages_yaml)\n with pytest.raises(spack.error.SpackError, match=expected_message):\n spack.concretize.concretize_one(spec_str)\n\n\ndef test_skip_requirement_when_default_requirement_condition_cannot_be_met(\n concretize_scope, mock_packages\n):\n \"\"\"Tests that we can express a requirement condition under 'all' also in cases where\n the corresponding condition spec mentions variants or versions that don't exist in the\n package. For those packages the requirement rule is not emitted, since it can be\n determined to be always false.\n \"\"\"\n packages_yaml = \"\"\"\n packages:\n all:\n require:\n - one_of: [\"%clang\"]\n when: \"+shared\"\n \"\"\"\n update_packages_config(packages_yaml)\n s = spack.concretize.concretize_one(\"mpileaks\")\n\n assert s.satisfies(\"+shared %clang\")\n # Sanity checks that 'callpath' doesn't have the shared variant, but that didn't\n # cause failures during concretization.\n assert \"shared\" not in s[\"callpath\"].variants\n\n\ndef test_requires_directive(mock_packages, config):\n # This package requires either clang or gcc\n s = spack.concretize.concretize_one(\"requires-clang-or-gcc\")\n assert s.satisfies(\"%gcc\")\n s = spack.concretize.concretize_one(\"requires-clang-or-gcc %gcc\")\n assert s.satisfies(\"%gcc\")\n s = spack.concretize.concretize_one(\"requires-clang-or-gcc %clang\")\n # Test both the real package (llvm) and its alias (clang)\n assert s.satisfies(\"%llvm\") and s.satisfies(\"%clang\")\n\n # This package can only be compiled with clang\n s = spack.concretize.concretize_one(\"requires-clang\")\n assert s.satisfies(\"%llvm\")\n s = spack.concretize.concretize_one(\"requires-clang %clang\")\n assert s.satisfies(\"%llvm\")\n with pytest.raises(spack.error.SpackError, match=\"can only be compiled with Clang\"):\n spack.concretize.concretize_one(\"requires-clang %gcc\")\n\n\n@pytest.mark.parametrize(\n \"packages_yaml\",\n [\n # Simple string\n \"\"\"\n packages:\n all:\n require: \"+shared\"\n \"\"\",\n # List of strings\n \"\"\"\n packages:\n all:\n require:\n - \"+shared\"\n \"\"\",\n # Objects with attributes\n \"\"\"\n packages:\n all:\n require:\n - spec: \"+shared\"\n \"\"\",\n \"\"\"\n packages:\n all:\n require:\n - one_of: [\"+shared\"]\n \"\"\",\n ],\n)\ndef test_default_requirements_semantic(packages_yaml, concretize_scope, mock_packages):\n \"\"\"Tests that requirements under 'all:' are by default applied only if the variant/property\n required exists, but are strict otherwise.\n\n For example:\n\n packages:\n all:\n require: \"+shared\"\n\n should enforce the value of \"+shared\" when a Boolean variant named \"shared\" exists. This is\n not overridable from the command line, so with the configuration above:\n\n > spack spec zlib~shared\n\n is unsatisfiable.\n \"\"\"\n update_packages_config(packages_yaml)\n\n # Regular zlib concretize to+shared\n s = spack.concretize.concretize_one(\"zlib\")\n assert s.satisfies(\"+shared\")\n\n # If we specify the variant we can concretize only the one matching the constraint\n s = spack.concretize.concretize_one(\"zlib+shared\")\n assert s.satisfies(\"+shared\")\n with pytest.raises(UnsatisfiableSpecError):\n spack.concretize.concretize_one(\"zlib~shared\")\n\n # A spec without the shared variant still concretize\n s = spack.concretize.concretize_one(\"pkg-a\")\n assert not s.satisfies(\"pkg-a+shared\")\n assert not s.satisfies(\"pkg-a~shared\")\n\n\n@pytest.mark.parametrize(\n \"packages_yaml,spec_str,expected,not_expected\",\n [\n # The package has a 'libs' mv variant defaulting to 'libs=shared'\n (\n \"\"\"\n packages:\n all:\n require: \"+libs\"\n \"\"\",\n \"multivalue-variant\",\n [\"libs=shared\"],\n [\"libs=static\", \"+libs\"],\n ),\n (\n \"\"\"\n packages:\n all:\n require: \"libs=foo\"\n \"\"\",\n \"multivalue-variant\",\n [\"libs=shared\"],\n [\"libs=static\", \"libs=foo\"],\n ),\n (\n # (TODO): revisit this case when we'll have exact value semantic for mv variants\n \"\"\"\n packages:\n all:\n require: \"libs=static\"\n \"\"\",\n \"multivalue-variant\",\n [\"libs=static\", \"libs=shared\"],\n [],\n ),\n (\n # Constraint apply as a whole, so having a non-existing variant\n # invalidate the entire constraint\n \"\"\"\n packages:\n all:\n require: \"libs=static+feefoo\"\n \"\"\",\n \"multivalue-variant\",\n [\"libs=shared\"],\n [\"libs=static\"],\n ),\n ],\n)\ndef test_default_requirements_semantic_with_mv_variants(\n packages_yaml, spec_str, expected, not_expected, concretize_scope, mock_packages\n):\n \"\"\"Tests that requirements under 'all:' are behaving correctly under cases that could stem\n from MV variants.\n \"\"\"\n update_packages_config(packages_yaml)\n s = spack.concretize.concretize_one(spec_str)\n\n for constraint in expected:\n assert s.satisfies(constraint), constraint\n\n for constraint in not_expected:\n assert not s.satisfies(constraint), constraint\n\n\n@pytest.mark.regression(\"42084\")\ndef test_requiring_package_on_multiple_virtuals(concretize_scope, mock_packages):\n update_packages_config(\n \"\"\"\n packages:\n all:\n providers:\n scalapack: [netlib-scalapack]\n blas:\n require: intel-parallel-studio\n lapack:\n require: intel-parallel-studio\n scalapack:\n require: intel-parallel-studio\n \"\"\"\n )\n s = spack.concretize.concretize_one(\"dla-future\")\n\n assert s[\"blas\"].name == \"intel-parallel-studio\"\n assert s[\"lapack\"].name == \"intel-parallel-studio\"\n assert s[\"scalapack\"].name == \"intel-parallel-studio\"\n\n\n@pytest.mark.parametrize(\n \"packages_yaml,spec_str,expected,not_expected\",\n [\n (\n \"\"\"\n packages:\n all:\n prefer:\n - \"%clang\"\n \"\"\",\n \"multivalue-variant\",\n [\"%[virtuals=c] llvm\"],\n [\"%gcc\"],\n ),\n (\n \"\"\"\n packages:\n all:\n prefer:\n - \"%clang\"\n \"\"\",\n \"multivalue-variant %gcc\",\n [\"%[virtuals=c] gcc\"],\n [\"%llvm\"],\n ),\n # Test parsing objects instead of strings\n (\n \"\"\"\n packages:\n all:\n prefer:\n - spec: \"%clang\"\n \"\"\",\n \"multivalue-variant\",\n [\"%[virtuals=c] llvm\"],\n [\"%gcc\"],\n ),\n # Test using preferences on virtuals\n (\n \"\"\"\n packages:\n all:\n providers:\n mpi: [mpich]\n mpi:\n prefer:\n - zmpi\n \"\"\",\n \"mpileaks\",\n [\"^[virtuals=mpi] zmpi\"],\n [\"^[virtuals=mpi] mpich\"],\n ),\n (\n \"\"\"\n packages:\n all:\n providers:\n mpi: [mpich]\n mpi:\n prefer:\n - zmpi\n \"\"\",\n \"mpileaks ^[virtuals=mpi] mpich\",\n [\"^[virtuals=mpi] mpich\"],\n [\"^[virtuals=mpi] zmpi\"],\n ),\n # Tests that strong preferences can be overridden by requirements\n (\n \"\"\"\n packages:\n all:\n providers:\n mpi: [zmpi]\n mpi:\n require:\n - mpich\n prefer:\n - zmpi\n \"\"\",\n \"mpileaks\",\n [\"^[virtuals=mpi] mpich\"],\n [\"^[virtuals=mpi] zmpi\"],\n ),\n ],\n)\ndef test_strong_preferences_packages_yaml(\n packages_yaml, spec_str, expected, not_expected, concretize_scope, mock_packages\n):\n \"\"\"Tests that strong preferences are taken into account for compilers.\"\"\"\n update_packages_config(packages_yaml)\n s = spack.concretize.concretize_one(spec_str)\n\n for constraint in expected:\n assert s.satisfies(constraint)\n\n for constraint in not_expected:\n assert not s.satisfies(constraint)\n\n\n@pytest.mark.parametrize(\n \"packages_yaml,spec_str\",\n [\n (\n \"\"\"\n packages:\n all:\n conflict:\n - \"%clang\"\n \"\"\",\n \"multivalue-variant %clang\",\n ),\n # Use an object instead of a string in configuration\n (\n \"\"\"\n packages:\n all:\n conflict:\n - spec: \"%clang\"\n message: \"cannot use clang\"\n \"\"\",\n \"multivalue-variant %clang\",\n ),\n (\n \"\"\"\n packages:\n multivalue-variant:\n conflict:\n - spec: \"%clang\"\n when: \"@2\"\n message: \"cannot use clang with version 2\"\n \"\"\",\n \"multivalue-variant@=2.3 %clang\",\n ),\n # Test using conflict on virtual\n (\n \"\"\"\n packages:\n mpi:\n conflict:\n - mpich\n \"\"\",\n \"mpileaks ^[virtuals=mpi] mpich\",\n ),\n ],\n)\ndef test_conflict_packages_yaml(packages_yaml, spec_str, concretize_scope, mock_packages):\n \"\"\"Tests conflicts that are specified from configuration files.\"\"\"\n update_packages_config(packages_yaml)\n with pytest.raises(UnsatisfiableSpecError):\n spack.concretize.concretize_one(spec_str)\n\n\n@pytest.mark.parametrize(\n \"spec_str,expected,not_expected\",\n [\n (\n \"forward-multi-value+cuda cuda_arch=10 ^dependency-mv~cuda\",\n [\"cuda_arch=10\", \"^dependency-mv~cuda\"],\n [\"cuda_arch=11\", \"^dependency-mv cuda_arch=10\", \"^dependency-mv cuda_arch=11\"],\n ),\n (\n \"forward-multi-value+cuda cuda_arch=10 ^dependency-mv+cuda\",\n [\"cuda_arch=10\", \"^dependency-mv cuda_arch=10\"],\n [\"cuda_arch=11\", \"^dependency-mv cuda_arch=11\"],\n ),\n (\n \"forward-multi-value+cuda cuda_arch=11 ^dependency-mv+cuda\",\n [\"cuda_arch=11\", \"^dependency-mv cuda_arch=11\"],\n [\"cuda_arch=10\", \"^dependency-mv cuda_arch=10\"],\n ),\n (\n \"forward-multi-value+cuda cuda_arch=10,11 ^dependency-mv+cuda\",\n [\"cuda_arch=10,11\", \"^dependency-mv cuda_arch=10,11\"],\n [],\n ),\n ],\n)\ndef test_forward_multi_valued_variant_using_requires(\n spec_str, expected, not_expected, config, mock_packages\n):\n \"\"\"Tests that a package can forward multivalue variants to dependencies, using\n `requires` directives of the form:\n\n for _val in (\"shared\", \"static\"):\n requires(f\"^some-virtual-mv libs={_val}\", when=f\"libs={_val}^some-virtual-mv\")\n \"\"\"\n s = spack.concretize.concretize_one(spec_str)\n\n for constraint in expected:\n assert s.satisfies(constraint)\n\n for constraint in not_expected:\n assert not s.satisfies(constraint)\n\n\ndef test_strong_preferences_higher_priority_than_reuse(concretize_scope, mock_packages):\n \"\"\"Tests that strong preferences have a higher priority than reusing specs.\"\"\"\n reused_spec = spack.concretize.concretize_one(\"adios2~bzip2\")\n reuse_nodes = list(reused_spec.traverse())\n root_specs = [Spec(\"ascent+adios2\")]\n\n # Check that without further configuration adios2 is reused\n with spack.config.override(\"concretizer:reuse\", True):\n solver = spack.solver.asp.Solver()\n setup = spack.solver.asp.SpackSolverSetup()\n result, _, _ = solver.driver.solve(setup, root_specs, reuse=reuse_nodes)\n ascent = result.specs[0]\n assert ascent[\"adios2\"].dag_hash() == reused_spec.dag_hash(), ascent\n\n # If we stick a preference, adios2 is not reused\n update_packages_config(\n \"\"\"\n packages:\n adios2:\n prefer:\n - \"+bzip2\"\n\"\"\"\n )\n with spack.config.override(\"concretizer:reuse\", True):\n solver = spack.solver.asp.Solver()\n setup = spack.solver.asp.SpackSolverSetup()\n result, _, _ = solver.driver.solve(setup, root_specs, reuse=reuse_nodes)\n ascent = result.specs[0]\n\n assert ascent[\"adios2\"].dag_hash() != reused_spec.dag_hash()\n assert ascent[\"adios2\"].satisfies(\"+bzip2\")\n\n # A preference is still preference, so we can override from input\n with spack.config.override(\"concretizer:reuse\", True):\n solver = spack.solver.asp.Solver()\n setup = spack.solver.asp.SpackSolverSetup()\n result, _, _ = solver.driver.solve(\n setup, [Spec(\"ascent+adios2^adios2~bzip2\")], reuse=reuse_nodes\n )\n ascent = result.specs[0]\n assert ascent[\"adios2\"].dag_hash() == reused_spec.dag_hash(), ascent\n\n\n@pytest.mark.parametrize(\n \"packages_yaml,err_match\",\n [\n (\n \"\"\"\npackages:\n mpi:\n require:\n - \"+bzip2\"\n\"\"\",\n \"expected a named spec\",\n ),\n (\n \"\"\"\npackages:\n mpi:\n require:\n - one_of: [\"+bzip2\", openmpi]\n\"\"\",\n \"expected a named spec\",\n ),\n (\n \"\"\"\npackages:\n mpi:\n require:\n - \"^mpich\"\n\"\"\",\n \"Did you mean\",\n ),\n ],\n)\ndef test_anonymous_spec_cannot_be_used_in_virtual_requirements(\n packages_yaml, err_match, concretize_scope, mock_packages\n):\n \"\"\"Tests that using anonymous specs in requirements for virtual packages raises an\n appropriate error message.\n \"\"\"\n update_packages_config(packages_yaml)\n with pytest.raises(spack.error.SpackError, match=err_match):\n spack.concretize.concretize_one(\"mpileaks\")\n\n\ndef test_virtual_requirement_respects_any_of(concretize_scope, mock_packages):\n \"\"\"Tests that \"any of\" requirements can be used with virtuals\"\"\"\n conf_str = \"\"\"\\\n packages:\n mpi:\n require:\n - any_of: [\"mpich2\", \"mpich\"]\n \"\"\"\n update_packages_config(conf_str)\n\n s = spack.concretize.concretize_one(\"mpileaks\")\n assert s.satisfies(\"^[virtuals=mpi] mpich2\")\n\n s = spack.concretize.concretize_one(\"mpileaks ^mpich2\")\n assert s.satisfies(\"^[virtuals=mpi] mpich2\")\n\n s = spack.concretize.concretize_one(\"mpileaks ^mpich\")\n assert s.satisfies(\"^[virtuals=mpi] mpich\")\n\n with pytest.raises(spack.error.SpackError):\n spack.concretize.concretize_one(\"mpileaks ^[virtuals=mpi] zmpi\")\n\n\n@pytest.mark.parametrize(\n \"packages_yaml,expected_reuse,expected_contraints\",\n [\n (\n \"\"\"\npackages:\n all:\n require:\n - \"%gcc\"\n \"\"\",\n True,\n # To minimize installed specs we reuse pkg-b compiler, since the requirement allows it\n [\"%gcc@9\"],\n ),\n (\n \"\"\"\npackages:\n all:\n require:\n - \"%gcc@10\"\n \"\"\",\n False,\n [\"%gcc@10\"],\n ),\n (\n \"\"\"\npackages:\n all:\n require:\n - \"%gcc@9\"\n \"\"\",\n True,\n [\"%gcc@9\"],\n ),\n ],\n)\n@pytest.mark.regression(\"49847\")\ndef test_requirements_on_compilers_and_reuse(\n concretize_scope,\n mock_packages,\n mutable_config,\n packages_yaml,\n expected_reuse,\n expected_contraints,\n):\n \"\"\"Tests that we can require compilers with `%` in configuration files, and still get reuse\n of specs (even though reused specs have no build dependency in the ASP encoding).\n \"\"\"\n input_spec = \"pkg-a\"\n\n reused_spec = spack.concretize.concretize_one(\"pkg-b@0.9 %gcc@9\")\n reused_nodes = list(reused_spec.traverse())\n update_packages_config(packages_yaml)\n root_specs = [Spec(input_spec)]\n packages_with_externals = external_config_with_implicit_externals(mutable_config)\n completion_mode = mutable_config.get(\"concretizer:externals:completion\")\n external_specs = spec_filter_from_packages_yaml(\n external_parser=create_external_parser(packages_with_externals, completion_mode),\n packages_with_externals=packages_with_externals,\n include=[],\n exclude=[],\n ).selected_specs()\n\n with spack.config.override(\"concretizer:reuse\", True):\n solver = spack.solver.asp.Solver()\n setup = spack.solver.asp.SpackSolverSetup()\n result, _, _ = solver.driver.solve(setup, root_specs, reuse=reused_nodes + external_specs)\n pkga = result.specs[0]\n is_pkgb_reused = pkga[\"pkg-b\"].dag_hash() == reused_spec.dag_hash()\n\n assert is_pkgb_reused == expected_reuse\n for c in expected_contraints:\n assert pkga.satisfies(c)\n\n\n@pytest.mark.parametrize(\n \"abstract,req_is_noop\",\n [\n (\"hdf5+mpi\", False),\n (\"hdf5~mpi\", True),\n (\"conditional-languages+c\", False),\n (\"conditional-languages+cxx\", False),\n (\"conditional-languages+fortran\", False),\n (\"conditional-languages~c~cxx~fortran\", True),\n ],\n)\ndef test_requirements_conditional_deps(\n abstract, req_is_noop, mutable_config, mock_packages, config_two_gccs\n):\n required_spec = (\n \"%[when='^c' virtuals=c]gcc@10.3.1 \"\n \"%[when='^cxx' virtuals=cxx]gcc@10.3.1 \"\n \"%[when='^fortran' virtuals=fortran]gcc@10.3.1 \"\n \"^[when='^mpi' virtuals=mpi]zmpi\"\n )\n abstract = spack.spec.Spec(abstract)\n\n no_requirements = spack.concretize.concretize_one(abstract)\n spack.config.CONFIG.set(f\"packages:{abstract.name}\", {\"require\": required_spec})\n requirements = spack.concretize.concretize_one(abstract)\n\n assert requirements.satisfies(required_spec)\n assert (requirements == no_requirements) == req_is_noop # show the reqs change concretization\n\n\n@pytest.mark.regression(\"50898\")\ndef test_preferring_compilers_can_be_overridden(mutable_config, mock_packages):\n \"\"\"Tests that we can override preferences for languages, without triggering an error.\"\"\"\n mutable_config.set(\"packages:c\", {\"prefer\": [\"llvm\"]})\n\n s = spack.spec.Spec(\"pkg-a %gcc ^pkg-b %llvm\")\n concrete = spack.concretize.concretize_one(s)\n\n assert concrete.satisfies(\"%c=gcc\")\n assert concrete[\"pkg-b\"].satisfies(\"%c=llvm\")\n\n\n@pytest.mark.regression(\"50955\")\ndef test_multiple_externals_and_requirement(\n concretize_scope, mock_packages, tmp_path: pathlib.Path\n):\n \"\"\"Tests that we can concretize a required virtual, when we have multiple externals specs for\n it, differing only by the compiler.\n \"\"\"\n packages_yaml = f\"\"\"\npackages:\n c:\n require: gcc\n mpi:\n require: mpich\n mpich:\n buildable: false\n externals:\n - spec: \"mpich@4.3.0 %gcc@10\"\n prefix: {tmp_path / \"gcc\"}\n - spec: \"mpich@4.3.0 %clang\"\n prefix: {tmp_path / \"clang\"}\n\"\"\"\n update_packages_config(packages_yaml)\n\n s = spack.spec.Spec(\"mpileaks\")\n concrete = spack.concretize.concretize_one(s)\n\n assert concrete.satisfies(\"%gcc\")\n assert concrete[\"mpi\"].satisfies(\"mpich@4.3.0\")\n assert concrete[\"mpi\"].prefix == str(tmp_path / \"gcc\")\n\n\n@pytest.mark.regression(\"51262\")\n@pytest.mark.parametrize(\n \"input_constraint\",\n [\n # Override the compiler preference with a different version of gcc\n \"%c=gcc@10\",\n # Same, but without specifying the virtual\n \"%gcc@10\",\n # Override the mpi preference with a different version of mpich\n \"%mpi=mpich@3 ~debug\",\n # Override the mpi preference with a different provider\n \"%mpi=mpich2\",\n ],\n)\ndef test_overriding_preference_with_provider_details(\n input_constraint, concretize_scope, mock_packages, tmp_path: pathlib.Path\n):\n \"\"\"Tests that if we have a preference with provider details, such as a version range,\n or a variant, we can override it from the command line, while we can't do the same\n when we have a requirement.\n \"\"\"\n # A preference can be overridden\n packages_yaml = \"\"\"\npackages:\n c:\n prefer:\n - gcc@9\n mpi:\n prefer:\n - mpich@3 +debug\n\"\"\"\n update_packages_config(packages_yaml)\n concrete = spack.concretize.concretize_one(f\"mpileaks {input_constraint}\")\n assert concrete.satisfies(input_constraint)\n\n # A requirement cannot\n packages_yaml = \"\"\"\n packages:\n c:\n require:\n - gcc@9\n mpi:\n require:\n - mpich@3 +debug\n \"\"\"\n update_packages_config(packages_yaml)\n with pytest.raises(UnsatisfiableSpecError):\n spack.concretize.concretize_one(f\"mpileaks {input_constraint}\")\n\n\n@pytest.mark.parametrize(\n \"initial_preference,current_preference\",\n [\n # Different provider\n (\"llvm\", \"gcc\"),\n (\"gcc\", \"llvm\"),\n # Different version of the same provider\n (\"gcc@9\", \"gcc@10\"),\n (\"gcc@10\", \"gcc@9\"),\n # Different configuration of the same provider\n (\"llvm+lld\", \"llvm~lld\"),\n (\"llvm~lld\", \"llvm+lld\"),\n ],\n)\n@pytest.mark.parametrize(\"constraint_kind\", [\"require\", \"prefer\"])\ndef test_language_preferences_and_reuse(\n initial_preference,\n current_preference,\n constraint_kind,\n concretize_scope,\n mutable_config,\n mock_packages,\n):\n \"\"\"Tests that language preferences are respected when reusing specs.\"\"\"\n\n # Install mpileaks with a non-default variant to avoid \"accidental\" reuse\n packages_yaml = f\"\"\"\npackages:\n c:\n {constraint_kind}:\n - {initial_preference}\n cxx:\n {constraint_kind}:\n - {initial_preference}\n llvm:\n externals:\n - spec: \"llvm@15.0.0 +clang~flang ~lld\"\n prefix: /path1\n extra_attributes:\n compilers:\n c: /path1/bin/clang\n cxx: /path1/bin/clang++\n\"\"\"\n update_packages_config(packages_yaml)\n initial_mpileaks = spack.concretize.concretize_one(\"mpileaks+debug\")\n reused_nodes = list(initial_mpileaks.traverse())\n packages_with_externals = external_config_with_implicit_externals(mutable_config)\n completion_mode = mutable_config.get(\"concretizer:externals:completion\")\n external_specs = spec_filter_from_packages_yaml(\n external_parser=create_external_parser(packages_with_externals, completion_mode),\n packages_with_externals=packages_with_externals,\n include=[],\n exclude=[],\n ).selected_specs()\n\n # Ask for just \"mpileaks\" and check the spec is reused\n with spack.config.override(\"concretizer:reuse\", True):\n solver = spack.solver.asp.Solver()\n setup = spack.solver.asp.SpackSolverSetup()\n result, _, _ = solver.driver.solve(\n setup, [Spec(\"mpileaks\")], reuse=reused_nodes + external_specs\n )\n reused_mpileaks = result.specs[0]\n\n assert reused_mpileaks.dag_hash() == initial_mpileaks.dag_hash()\n\n # Change the language preferences and verify reuse is not happening\n packages_yaml = f\"\"\"\npackages:\n c:\n {constraint_kind}:\n - {current_preference}\n cxx:\n {constraint_kind}:\n - {current_preference}\n llvm:\n externals:\n - spec: \"llvm@15.0.0 +clang~flang ~lld\"\n prefix: /path1\n extra_attributes:\n compilers:\n c: /path1/bin/clang\n cxx: /path1/bin/clang++\n\"\"\"\n update_packages_config(packages_yaml)\n with spack.config.override(\"concretizer:reuse\", True):\n solver = spack.solver.asp.Solver()\n setup = spack.solver.asp.SpackSolverSetup()\n result, _, _ = solver.driver.solve(\n setup, [Spec(\"mpileaks\")], reuse=reused_nodes + external_specs\n )\n mpileaks = result.specs[0]\n\n assert initial_mpileaks.dag_hash() != mpileaks.dag_hash()\n for node in mpileaks.traverse():\n assert node.satisfies(f\"%[when=%c]c={current_preference}\")\n assert node.satisfies(f\"%[when=%cxx]cxx={current_preference}\")\n\n\ndef test_external_spec_completion_with_targets_required(\n concretize_scope, mock_packages, tmp_path: pathlib.Path\n):\n \"\"\"Tests that we can concretize a spec needing externals, when we require a specific target,\n without extra configuration.\n \"\"\"\n current_platform = spack.platforms.host()\n packages_yaml = f\"\"\"\n packages:\n all:\n require:\n - target={current_platform.default}\n mpich:\n buildable: false\n externals:\n - spec: \"mpich@4.3.0\"\n prefix: {tmp_path / \"mpich\"}\n \"\"\"\n update_packages_config(packages_yaml)\n\n s = spack.spec.Spec(\"mpileaks\")\n concrete = spack.concretize.concretize_one(s)\n\n assert concrete.satisfies(f\"target={current_platform.default}\")\n\n\ndef test_penalties_for_language_preferences(concretize_scope, mock_packages):\n \"\"\"Tests the default behavior when we use more than one compiler package in a DAG,\n under different scenarios.\n \"\"\"\n # This test uses gcc compilers providing c,cxx and fortran, and clang providing only c and cxx\n dependency_names = [\"mpi\", \"callpath\", \"libdwarf\", \"libelf\"]\n\n # If we don't express requirements, Spack tries to use a single compiler package if possible\n s = spack.concretize.concretize_one(\"mpileaks %c=gcc@10\")\n assert s.satisfies(\"%c=gcc@10\")\n assert all(s[name].satisfies(\"%c=gcc@10\") for name in dependency_names)\n\n # Same with clang, if nothing else requires fortran\n s = spack.concretize.concretize_one(\"mpileaks %c=clang ^mpi=mpich2\")\n assert s.satisfies(\"%c=clang\")\n assert all(s[name].satisfies(\"%c=clang\") for name in dependency_names)\n\n # If something brings in fortran that node is compiled entirely with gcc,\n # because currently we prefer to use a single toolchain for any node\n s = spack.concretize.concretize_one(\"mpileaks %c=clang ^mpi=mpich\")\n assert s.satisfies(\"%c=clang\")\n assert s[\"mpich\"].satisfies(\"%c,cxx,fortran=gcc@10\")\n\n # If we prefer compilers in configuration, that has a higher priority\n update_packages_config(\n \"\"\"\n packages:\n c:\n prefer: [gcc]\n cxx:\n prefer: [gcc]\n fortran:\n prefer: [gcc]\n\"\"\"\n )\n\n s = spack.concretize.concretize_one(\"mpileaks %c=clang ^mpi=mpich2\")\n assert s.satisfies(\"%c=clang\")\n assert all(s[name].satisfies(\"%c=gcc@10\") for name in dependency_names)\n\n # Mixed compilers in the preferences\n update_packages_config(\n \"\"\"\n packages:\n c:\n prefer: [llvm]\n cxx:\n prefer: [llvm]\n fortran:\n prefer: [gcc]\n\"\"\"\n )\n\n s = spack.concretize.concretize_one(\"mpileaks %c=gcc ^mpi=mpich\")\n assert s.satisfies(\"%c=gcc@10\")\n assert all(s[name].satisfies(\"%c=clang\") for name in dependency_names)\n assert s[\"mpi\"].satisfies(\"%c,cxx=clang %fortran=gcc@10\")\n" }, { "path": "lib/spack/spack/test/concretization/splicing.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Test ABI-based splicing of dependencies\"\"\"\n\nfrom typing import List\n\nimport pytest\n\nimport spack.concretize\nimport spack.config\nimport spack.deptypes as dt\nfrom spack.installer import PackageInstaller\nfrom spack.solver.asp import SolverError, UnsatisfiableSpecError\n\n\ndef _make_specs_non_buildable(specs: List[str]):\n output_config = {}\n for spec in specs:\n output_config[spec] = {\"buildable\": False}\n return output_config\n\n\n@pytest.fixture\ndef install_specs(mutable_database, mock_packages, mutable_config, install_mockery):\n \"\"\"Returns a function that concretizes and installs a list of abstract specs\"\"\"\n mutable_config.set(\"concretizer:reuse\", True)\n\n def _impl(*specs_str):\n concrete_specs = [spack.concretize.concretize_one(s) for s in specs_str]\n PackageInstaller([s.package for s in concrete_specs], fake=True, explicit=True).install()\n return concrete_specs\n\n return _impl\n\n\ndef _enable_splicing():\n spack.config.set(\"concretizer:splice\", {\"automatic\": True})\n\n\n@pytest.mark.parametrize(\"spec_str\", [\"splice-z\", \"splice-h@1\"])\ndef test_spec_reuse(spec_str, install_specs, mutable_config):\n \"\"\"Tests reuse of splice-z, without splicing, as a root and as a dependency of splice-h\"\"\"\n splice_z = install_specs(\"splice-z@1.0.0+compat\")[0]\n mutable_config.set(\"packages\", _make_specs_non_buildable([\"splice-z\"]))\n concrete = spack.concretize.concretize_one(spec_str)\n assert concrete[\"splice-z\"].satisfies(splice_z)\n\n\n@pytest.mark.regression(\"48578\")\ndef test_splice_installed_hash(install_specs, mutable_config):\n \"\"\"Tests splicing the dependency of an installed spec, for another installed spec\"\"\"\n splice_t, splice_h = install_specs(\n \"splice-t@1 ^splice-h@1.0.0+compat ^splice-z@1.0.0\",\n \"splice-h@1.0.2+compat ^splice-z@1.0.0\",\n )\n packages_config = _make_specs_non_buildable([\"splice-t\", \"splice-h\"])\n mutable_config.set(\"packages\", packages_config)\n\n goal_spec = \"splice-t@1 ^splice-h@1.0.2+compat ^splice-z@1.0.0\"\n with pytest.raises(UnsatisfiableSpecError):\n spack.concretize.concretize_one(goal_spec)\n _enable_splicing()\n concrete = spack.concretize.concretize_one(goal_spec)\n\n # splice-t has a dependency that is changing, thus its hash should be different\n assert concrete.dag_hash() != splice_t.dag_hash()\n assert concrete.build_spec.satisfies(splice_t)\n assert not concrete.satisfies(splice_t)\n\n # splice-h is reused, so the hash should stay the same\n assert concrete[\"splice-h\"].satisfies(splice_h)\n assert concrete[\"splice-h\"].build_spec.satisfies(splice_h)\n assert concrete[\"splice-h\"].dag_hash() == splice_h.dag_hash()\n\n\ndef test_splice_build_splice_node(install_specs, mutable_config):\n \"\"\"Tests splicing the dependency of an installed spec, for a spec that is yet to be built\"\"\"\n splice_t = install_specs(\"splice-t@1 ^splice-h@1.0.0+compat ^splice-z@1.0.0+compat\")[0]\n mutable_config.set(\"packages\", _make_specs_non_buildable([\"splice-t\"]))\n\n goal_spec = \"splice-t@1 ^splice-h@1.0.2+compat ^splice-z@1.0.0+compat\"\n with pytest.raises(UnsatisfiableSpecError):\n spack.concretize.concretize_one(goal_spec)\n\n _enable_splicing()\n concrete = spack.concretize.concretize_one(goal_spec)\n\n # splice-t has a dependency that is changing, thus its hash should be different\n assert concrete.dag_hash() != splice_t.dag_hash()\n assert concrete.build_spec.satisfies(splice_t)\n assert not concrete.satisfies(splice_t)\n\n # splice-h should be different\n assert concrete[\"splice-h\"].dag_hash() != splice_t[\"splice-h\"].dag_hash()\n assert concrete[\"splice-h\"].build_spec.dag_hash() == concrete[\"splice-h\"].dag_hash()\n\n\ndef test_double_splice(install_specs, mutable_config):\n \"\"\"Tests splicing two dependencies of an installed spec, for other installed specs\"\"\"\n splice_t, splice_h, splice_z = install_specs(\n \"splice-t@1 ^splice-h@1.0.0+compat ^splice-z@1.0.0+compat\",\n \"splice-h@1.0.2+compat ^splice-z@1.0.1+compat\",\n \"splice-z@1.0.2+compat\",\n )\n mutable_config.set(\"packages\", _make_specs_non_buildable([\"splice-t\", \"splice-h\", \"splice-z\"]))\n\n goal_spec = \"splice-t@1 ^splice-h@1.0.2+compat ^splice-z@1.0.2+compat\"\n with pytest.raises(UnsatisfiableSpecError):\n spack.concretize.concretize_one(goal_spec)\n\n _enable_splicing()\n concrete = spack.concretize.concretize_one(goal_spec)\n\n # splice-t and splice-h have a dependency that is changing, thus its hash should be different\n assert concrete.dag_hash() != splice_t.dag_hash()\n assert concrete.build_spec.satisfies(splice_t)\n assert not concrete.satisfies(splice_t)\n\n assert concrete[\"splice-h\"].dag_hash() != splice_h.dag_hash()\n assert concrete[\"splice-h\"].build_spec.satisfies(splice_h)\n assert not concrete[\"splice-h\"].satisfies(splice_h)\n\n # splice-z is reused, so the hash should stay the same\n assert concrete[\"splice-z\"].dag_hash() == splice_z.dag_hash()\n\n\n@pytest.mark.parametrize(\n \"original_spec,goal_spec\",\n [\n # `virtual-abi-1` can be spliced for `virtual-abi-multi abi=one` and vice-versa\n (\n \"depends-on-virtual-with-abi ^virtual-abi-1\",\n \"depends-on-virtual-with-abi ^virtual-abi-multi abi=one\",\n ),\n (\n \"depends-on-virtual-with-abi ^virtual-abi-multi abi=one\",\n \"depends-on-virtual-with-abi ^virtual-abi-1\",\n ),\n # `virtual-abi-2` can be spliced for `virtual-abi-multi abi=two` and vice-versa\n (\n \"depends-on-virtual-with-abi ^virtual-abi-2\",\n \"depends-on-virtual-with-abi ^virtual-abi-multi abi=two\",\n ),\n (\n \"depends-on-virtual-with-abi ^virtual-abi-multi abi=two\",\n \"depends-on-virtual-with-abi ^virtual-abi-2\",\n ),\n ],\n)\ndef test_virtual_multi_splices_in(original_spec, goal_spec, install_specs, mutable_config):\n \"\"\"Tests that we can splice a virtual dependency with a different, but compatible, provider.\"\"\"\n original = install_specs(original_spec)[0]\n mutable_config.set(\"packages\", _make_specs_non_buildable([\"depends-on-virtual-with-abi\"]))\n\n with pytest.raises(UnsatisfiableSpecError):\n spack.concretize.concretize_one(goal_spec)\n\n _enable_splicing()\n spliced = spack.concretize.concretize_one(goal_spec)\n\n assert spliced.dag_hash() != original.dag_hash()\n assert spliced.build_spec.dag_hash() == original.dag_hash()\n assert spliced[\"virtual-with-abi\"].name != spliced.build_spec[\"virtual-with-abi\"].name\n\n\n@pytest.mark.parametrize(\n \"original_spec,goal_spec\",\n [\n # can_splice(\"manyvariants@1.0.0\", when=\"@1.0.1\", match_variants=\"*\")\n (\n \"depends-on-manyvariants ^manyvariants@1.0.0+a+b c=v1 d=v2\",\n \"depends-on-manyvariants ^manyvariants@1.0.1+a+b c=v1 d=v2\",\n ),\n (\n \"depends-on-manyvariants ^manyvariants@1.0.0~a~b c=v3 d=v3\",\n \"depends-on-manyvariants ^manyvariants@1.0.1~a~b c=v3 d=v3\",\n ),\n # can_splice(\"manyvariants@2.0.0+a~b\", when=\"@2.0.1~a+b\", match_variants=[\"c\", \"d\"])\n (\n \"depends-on-manyvariants@2.0 ^manyvariants@2.0.0+a~b c=v3 d=v2\",\n \"depends-on-manyvariants@2.0 ^manyvariants@2.0.1~a+b c=v3 d=v2\",\n ),\n # can_splice(\"manyvariants@2.0.0 c=v1 d=v1\", when=\"@2.0.1+a+b\")\n (\n \"depends-on-manyvariants@2.0 ^manyvariants@2.0.0~a~b c=v1 d=v1\",\n \"depends-on-manyvariants@2.0 ^manyvariants@2.0.1+a+b c=v3 d=v3\",\n ),\n ],\n)\ndef test_manyvariant_matching_variant_splice(\n original_spec, goal_spec, install_specs, mutable_config\n):\n \"\"\"Tests splicing with different kind of matching on variants\"\"\"\n original = install_specs(original_spec)[0]\n mutable_config.set(\"packages\", {\"depends-on-manyvariants\": {\"buildable\": False}})\n\n with pytest.raises((UnsatisfiableSpecError, SolverError)):\n spack.concretize.concretize_one(goal_spec)\n\n _enable_splicing()\n spliced = spack.concretize.concretize_one(goal_spec)\n\n assert spliced.dag_hash() != original.dag_hash()\n assert spliced.build_spec.dag_hash() == original.dag_hash()\n\n # The spliced 'manyvariants' is yet to be built\n assert spliced[\"manyvariants\"].dag_hash() != original[\"manyvariants\"].dag_hash()\n assert spliced[\"manyvariants\"].build_spec.dag_hash() == spliced[\"manyvariants\"].dag_hash()\n\n\ndef test_external_splice_same_name(install_specs, mutable_config):\n \"\"\"Tests that externals can be spliced for non-external specs\"\"\"\n original_splice_h, original_splice_t = install_specs(\n \"splice-h@1.0.0 ^splice-z@1.0.0+compat\",\n \"splice-t@1.0 ^splice-h@1.0.1 ^splice-z@1.0.1+compat\",\n )\n mutable_config.set(\"packages\", _make_specs_non_buildable([\"splice-t\", \"splice-h\"]))\n mutable_config.set(\n \"packages\",\n {\n \"splice-z\": {\n \"externals\": [{\"spec\": \"splice-z@1.0.2+compat\", \"prefix\": \"/usr\"}],\n \"buildable\": False,\n }\n },\n )\n\n _enable_splicing()\n concrete_splice_h = spack.concretize.concretize_one(\"splice-h@1.0.0 ^splice-z@1.0.2\")\n concrete_splice_t = spack.concretize.concretize_one(\n \"splice-t@1.0 ^splice-h@1.0.1 ^splice-z@1.0.2\"\n )\n\n assert concrete_splice_h.dag_hash() != original_splice_h.dag_hash()\n assert concrete_splice_h.build_spec.dag_hash() == original_splice_h.dag_hash()\n assert concrete_splice_h[\"splice-z\"].external\n\n assert concrete_splice_t.dag_hash() != original_splice_t.dag_hash()\n assert concrete_splice_t.build_spec.dag_hash() == original_splice_t.dag_hash()\n assert concrete_splice_t[\"splice-z\"].external\n\n assert concrete_splice_t[\"splice-z\"].dag_hash() == concrete_splice_h[\"splice-z\"].dag_hash()\n\n\ndef test_spliced_build_deps_only_in_build_spec(install_specs):\n \"\"\"Tests that build specs are not reported in the spliced spec\"\"\"\n install_specs(\"splice-t@1.0 ^splice-h@1.0.1 ^splice-z@1.0.0\")\n\n _enable_splicing()\n spliced = spack.concretize.concretize_one(\"splice-t@1.0 ^splice-h@1.0.2 ^splice-z@1.0.0\")\n build_spec = spliced.build_spec\n\n # Spec has been spliced\n assert build_spec.dag_hash() != spliced.dag_hash()\n # Build spec has spliced build dependencies\n assert build_spec.dependencies(\"splice-h\", dt.BUILD)\n assert build_spec.dependencies(\"splice-z\", dt.BUILD)\n # Spliced build dependencies are removed\n assert len(spliced.dependencies(None, dt.BUILD)) == 0\n\n\ndef test_spliced_transitive_dependency(install_specs, mutable_config):\n \"\"\"Tests that build specs are not reported, even for spliced transitive dependencies\"\"\"\n install_specs(\"splice-depends-on-t@1.0 ^splice-h@1.0.1\")\n mutable_config.set(\"packages\", _make_specs_non_buildable([\"splice-depends-on-t\"]))\n\n _enable_splicing()\n spliced = spack.concretize.concretize_one(\"splice-depends-on-t^splice-h@1.0.2\")\n\n # Spec has been spliced\n assert spliced.build_spec.dag_hash() != spliced.dag_hash()\n assert spliced[\"splice-t\"].build_spec.dag_hash() != spliced[\"splice-t\"].dag_hash()\n\n # Spliced build dependencies are removed\n assert len(spliced.dependencies(None, dt.BUILD)) == 0\n assert len(spliced[\"splice-t\"].dependencies(None, dt.BUILD)) == 0\n" }, { "path": "lib/spack/spack/test/config.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport collections\nimport io\nimport os\nimport pathlib\nimport sys\nimport tempfile\nimport textwrap\nfrom datetime import date\n\nimport pytest\n\nimport spack\nimport spack.config\nimport spack.directory_layout\nimport spack.environment as ev\nimport spack.error\nimport spack.llnl.util.filesystem as fs\nimport spack.package_base\nimport spack.paths\nimport spack.platforms\nimport spack.repo\nimport spack.schema.compilers\nimport spack.schema.config\nimport spack.schema.env\nimport spack.schema.include\nimport spack.schema.mirrors\nimport spack.schema.repos\nimport spack.spec\nimport spack.store\nimport spack.util.executable\nimport spack.util.git\nimport spack.util.path as spack_path\nimport spack.util.spack_yaml as syaml\nfrom spack.enums import ConfigScopePriority\nfrom spack.llnl.util.filesystem import getuid, join_path, touch\nfrom spack.util.spack_yaml import DictWithLineInfo\n\n# sample config data\nconfig_low = {\n \"config\": {\n \"install_tree\": {\"root\": \"install_tree_path\"},\n \"build_stage\": [\"path1\", \"path2\", \"path3\"],\n }\n}\n\nconfig_override_all = {\"config:\": {\"install_tree:\": {\"root\": \"override_all\"}}}\n\nconfig_override_key = {\"config\": {\"install_tree:\": {\"root\": \"override_key\"}}}\n\nconfig_merge_list = {\"config\": {\"build_stage\": [\"patha\", \"pathb\"]}}\n\nconfig_override_list = {\"config\": {\"build_stage:\": [\"pathd\", \"pathe\"]}}\n\nconfig_merge_dict = {\"config\": {\"aliases\": {\"ls\": \"find\", \"dev\": \"develop\"}}}\n\nconfig_override_dict = {\"config\": {\"aliases:\": {\"be\": \"build-env\", \"deps\": \"dependencies\"}}}\n\n\n@pytest.fixture()\ndef env_yaml(tmp_path: pathlib.Path):\n \"\"\"Return a sample env.yaml for test purposes\"\"\"\n env_yaml = str(tmp_path / \"env.yaml\")\n with open(env_yaml, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n config:\n verify_ssl: False\n dirty: False\n packages:\n all:\n compiler: [ 'gcc@4.5.3' ]\n repos:\n z: /x/y/z\n\"\"\"\n )\n return env_yaml\n\n\ndef check_compiler_config(comps, *compiler_names):\n \"\"\"Check that named compilers in comps match Spack's config.\"\"\"\n config = spack.config.get(\"compilers\")\n compiler_list = [\"cc\", \"cxx\", \"f77\", \"fc\"]\n flag_list = [\"cflags\", \"cxxflags\", \"fflags\", \"cppflags\", \"ldflags\", \"ldlibs\"]\n param_list = [\"modules\", \"paths\", \"spec\", \"operating_system\"]\n for compiler in config:\n conf = compiler[\"compiler\"]\n if conf[\"spec\"] in compiler_names:\n comp = next(\n (c[\"compiler\"] for c in comps if c[\"compiler\"][\"spec\"] == conf[\"spec\"]), None\n )\n if not comp:\n raise ValueError(\"Bad config spec\")\n for p in param_list:\n assert conf[p] == comp[p]\n for f in flag_list:\n expected = comp.get(\"flags\", {}).get(f, None)\n actual = conf.get(\"flags\", {}).get(f, None)\n assert expected == actual\n for c in compiler_list:\n expected = comp[\"paths\"][c]\n actual = conf[\"paths\"][c]\n assert expected == actual\n\n\n#\n# Some sample compiler config data and tests.\n#\na_comps = {\n \"compilers\": [\n {\n \"compiler\": {\n \"paths\": {\"cc\": \"/gcc473\", \"cxx\": \"/g++473\", \"f77\": None, \"fc\": None},\n \"modules\": None,\n \"spec\": \"gcc@4.7.3\",\n \"operating_system\": \"CNL10\",\n }\n },\n {\n \"compiler\": {\n \"paths\": {\"cc\": \"/gcc450\", \"cxx\": \"/g++450\", \"f77\": \"gfortran\", \"fc\": \"gfortran\"},\n \"modules\": None,\n \"spec\": \"gcc@4.5.0\",\n \"operating_system\": \"CNL10\",\n }\n },\n {\n \"compiler\": {\n \"paths\": {\"cc\": \"/gcc422\", \"cxx\": \"/g++422\", \"f77\": \"gfortran\", \"fc\": \"gfortran\"},\n \"flags\": {\"cppflags\": \"-O0 -fpic\", \"fflags\": \"-f77\"},\n \"modules\": None,\n \"spec\": \"gcc@4.2.2\",\n \"operating_system\": \"CNL10\",\n }\n },\n {\n \"compiler\": {\n \"paths\": {\n \"cc\": \"\",\n \"cxx\": \"\",\n \"f77\": \"\",\n \"fc\": \"\",\n },\n \"modules\": None,\n \"spec\": \"clang@3.3\",\n \"operating_system\": \"CNL10\",\n }\n },\n ]\n}\n\nb_comps = {\n \"compilers\": [\n {\n \"compiler\": {\n \"paths\": {\"cc\": \"/icc100\", \"cxx\": \"/icp100\", \"f77\": None, \"fc\": None},\n \"modules\": None,\n \"spec\": \"icc@10.0\",\n \"operating_system\": \"CNL10\",\n }\n },\n {\n \"compiler\": {\n \"paths\": {\"cc\": \"/icc111\", \"cxx\": \"/icp111\", \"f77\": \"ifort\", \"fc\": \"ifort\"},\n \"modules\": None,\n \"spec\": \"icc@11.1\",\n \"operating_system\": \"CNL10\",\n }\n },\n {\n \"compiler\": {\n \"paths\": {\"cc\": \"/icc123\", \"cxx\": \"/icp123\", \"f77\": \"ifort\", \"fc\": \"ifort\"},\n \"flags\": {\"cppflags\": \"-O3\", \"fflags\": \"-f77rtl\"},\n \"modules\": None,\n \"spec\": \"icc@12.3\",\n \"operating_system\": \"CNL10\",\n }\n },\n {\n \"compiler\": {\n \"paths\": {\n \"cc\": \"\",\n \"cxx\": \"\",\n \"f77\": \"\",\n \"fc\": \"\",\n },\n \"modules\": None,\n \"spec\": \"clang@3.3\",\n \"operating_system\": \"CNL10\",\n }\n },\n ]\n}\n\n\n@pytest.fixture()\ndef compiler_specs():\n \"\"\"Returns a couple of compiler specs needed for the tests\"\"\"\n a = [ac[\"compiler\"][\"spec\"] for ac in a_comps[\"compilers\"]]\n b = [bc[\"compiler\"][\"spec\"] for bc in b_comps[\"compilers\"]]\n CompilerSpecs = collections.namedtuple(\"CompilerSpecs\", [\"a\", \"b\"])\n return CompilerSpecs(a=a, b=b)\n\n\ndef test_write_key_in_memory(mock_low_high_config, compiler_specs):\n # Write b_comps \"on top of\" a_comps.\n spack.config.set(\"compilers\", a_comps[\"compilers\"], scope=\"low\")\n spack.config.set(\"compilers\", b_comps[\"compilers\"], scope=\"high\")\n\n # Make sure the config looks how we expect.\n check_compiler_config(a_comps[\"compilers\"], *compiler_specs.a)\n check_compiler_config(b_comps[\"compilers\"], *compiler_specs.b)\n\n\ndef test_write_key_to_disk(mock_low_high_config, compiler_specs):\n # Write b_comps \"on top of\" a_comps.\n spack.config.set(\"compilers\", a_comps[\"compilers\"], scope=\"low\")\n spack.config.set(\"compilers\", b_comps[\"compilers\"], scope=\"high\")\n\n # Clear caches so we're forced to read from disk.\n spack.config.CONFIG.clear_caches()\n\n # Same check again, to ensure consistency.\n check_compiler_config(a_comps[\"compilers\"], *compiler_specs.a)\n check_compiler_config(b_comps[\"compilers\"], *compiler_specs.b)\n\n\ndef test_write_to_same_priority_file(mock_low_high_config, compiler_specs):\n # Write b_comps in the same file as a_comps.\n spack.config.set(\"compilers\", a_comps[\"compilers\"], scope=\"low\")\n spack.config.set(\"compilers\", b_comps[\"compilers\"], scope=\"low\")\n\n # Clear caches so we're forced to read from disk.\n spack.config.CONFIG.clear_caches()\n\n # Same check again, to ensure consistency.\n check_compiler_config(a_comps[\"compilers\"], *compiler_specs.a)\n check_compiler_config(b_comps[\"compilers\"], *compiler_specs.b)\n\n\n#\n# Sample repo data and tests\n#\nrepos_low = {\"repos\": {\"low\": \"/some/path\"}}\nrepos_high = {\"repos\": {\"high\": \"/some/other/path\"}}\n\n# Test setting config values via path in filename\n\n\ndef test_add_config_path(mutable_config):\n # Try setting a new install tree root\n path = \"config:install_tree:root:/path/to/config.yaml\"\n spack.config.add(path)\n set_value = spack.config.get(\"config\")[\"install_tree\"][\"root\"]\n assert set_value == \"/path/to/config.yaml\"\n\n # Now a package:all setting\n path = \"packages:all:target:[x86_64]\"\n spack.config.add(path)\n targets = spack.config.get(\"packages\")[\"all\"][\"target\"]\n assert \"x86_64\" in targets\n\n # Try quotes to escape brackets\n path = (\n \"config:install_tree:projections:cmake:\"\n \"'{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'\"\n )\n spack.config.add(path)\n set_value = spack.config.get(\"config\")[\"install_tree\"][\"projections\"][\"cmake\"]\n assert set_value == \"{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}\"\n\n path = 'modules:default:tcl:all:environment:set:\"{name}_ROOT\":\"{prefix}\"'\n spack.config.add(path)\n set_value = spack.config.get(\"modules\")[\"default\"][\"tcl\"][\"all\"][\"environment\"][\"set\"]\n assert r\"{name}_ROOT\" in set_value\n assert set_value[r\"{name}_ROOT\"] == r\"{prefix}\"\n assert spack.config.get('modules:default:tcl:all:environment:set:\"{name}_ROOT\"') == r\"{prefix}\"\n\n # NOTE:\n # The config path: \"config:install_tree:root:\" is unique in that it can accept multiple\n # schemas (such as a dropped \"root\" component) which is atypical and may lead to passing tests\n # when the behavior is in reality incorrect.\n # the config path below is such that no subkey accepts a string as a valid entry in our schema\n\n # try quotes to escape colons\n path = \"config:build_stage:'C:\\\\path\\\\to\\\\config.yaml'\"\n spack.config.add(path)\n set_value = spack.config.get(\"config\")[\"build_stage\"]\n assert \"C:\\\\path\\\\to\\\\config.yaml\" in set_value\n\n\n@pytest.mark.regression(\"17543,23259\")\ndef test_add_config_path_with_enumerated_type(mutable_config):\n spack.config.add(\"config:flags:keep_werror:all\")\n assert spack.config.get(\"config\")[\"flags\"][\"keep_werror\"] == \"all\"\n\n spack.config.add(\"config:flags:keep_werror:specific\")\n assert spack.config.get(\"config\")[\"flags\"][\"keep_werror\"] == \"specific\"\n\n with pytest.raises(spack.error.ConfigError):\n spack.config.add(\"config:flags:keep_werror:foo\")\n\n\ndef test_add_config_filename(mock_low_high_config, tmp_path: pathlib.Path):\n config_yaml = tmp_path / \"config-filename.yaml\"\n config_yaml.touch()\n with config_yaml.open(\"w\") as f:\n syaml.dump_config(config_low, f)\n\n spack.config.add_from_file(str(config_yaml), scope=\"low\")\n assert \"build_stage\" in spack.config.get(\"config\")\n build_stages = spack.config.get(\"config\")[\"build_stage\"]\n for stage in config_low[\"config\"][\"build_stage\"]:\n assert stage in build_stages\n\n\n# repos\ndef test_write_list_in_memory(mock_low_high_config):\n spack.config.set(\"repos\", repos_low[\"repos\"], scope=\"low\")\n spack.config.set(\"repos\", repos_high[\"repos\"], scope=\"high\")\n\n config = spack.config.get(\"repos\")\n assert config == {**repos_high[\"repos\"], **repos_low[\"repos\"]}\n\n\nclass MockEnv:\n def __init__(self, path):\n self.path = path\n\n\ndef test_substitute_config_variables(mock_low_high_config, monkeypatch, tmp_path: pathlib.Path):\n # Test $spack substitution at the start (valid on all platforms)\n assert os.path.join(spack.paths.prefix, \"foo\", \"bar\", \"baz\") == spack_path.canonicalize_path(\n \"$spack/foo/bar/baz/\"\n )\n\n assert os.path.join(spack.paths.prefix, \"foo\", \"bar\", \"baz\") == spack_path.canonicalize_path(\n \"${spack}/foo/bar/baz/\"\n )\n\n # Test $spack substitution in the middle. This only makes sense when using posix paths.\n if sys.platform != \"win32\":\n prefix = spack.paths.prefix.lstrip(os.sep)\n base = str(tmp_path)\n\n assert os.path.join(base, \"foo\", \"bar\", \"baz\", prefix) == spack_path.canonicalize_path(\n os.path.join(base, \"foo\", \"bar\", \"baz\", \"$spack\")\n )\n\n assert os.path.join(\n base, \"foo\", \"bar\", \"baz\", prefix, \"foo\", \"bar\", \"baz\"\n ) == spack_path.canonicalize_path(\n os.path.join(base, \"foo\", \"bar\", \"baz\", \"$spack\", \"foo\", \"bar\", \"baz\")\n )\n\n assert os.path.join(base, \"foo\", \"bar\", \"baz\", prefix) == spack_path.canonicalize_path(\n os.path.join(base, \"foo\", \"bar\", \"baz\", \"${spack}\")\n )\n\n assert os.path.join(\n base, \"foo\", \"bar\", \"baz\", prefix, \"foo\", \"bar\", \"baz\"\n ) == spack_path.canonicalize_path(\n os.path.join(base, \"foo\", \"bar\", \"baz\", \"${spack}\", \"foo\", \"bar\", \"baz\")\n )\n\n assert os.path.join(\n base, \"foo\", \"bar\", \"baz\", prefix, \"foo\", \"bar\", \"baz\"\n ) != spack_path.canonicalize_path(\n os.path.join(base, \"foo\", \"bar\", \"baz\", \"${spack\", \"foo\", \"bar\", \"baz\")\n )\n\n # $env replacement is a no-op when no environment is active\n assert spack_path.canonicalize_path(\n os.path.join(str(tmp_path), \"foo\", \"bar\", \"baz\", \"$env\")\n ) == os.path.join(str(tmp_path), \"foo\", \"bar\", \"baz\", \"$env\")\n\n # Fake an active environment and $env is replaced properly\n fake_env_path = str(tmp_path / \"quux\" / \"quuux\")\n monkeypatch.setattr(ev, \"active_environment\", lambda: MockEnv(fake_env_path))\n assert spack_path.canonicalize_path(\"$env/foo/bar/baz\") == os.path.join(\n fake_env_path, os.path.join(\"foo\", \"bar\", \"baz\")\n )\n\n # relative paths without source information are relative to cwd\n assert spack_path.canonicalize_path(os.path.join(\"foo\", \"bar\", \"baz\")) == os.path.abspath(\n os.path.join(\"foo\", \"bar\", \"baz\")\n )\n\n # relative paths with source information are relative to the file\n spack.config.set(\n \"modules:default\", {\"roots\": {\"lmod\": os.path.join(\"foo\", \"bar\", \"baz\")}}, scope=\"low\"\n )\n spack.config.CONFIG.clear_caches()\n path = spack.config.get(\"modules:default:roots:lmod\")\n assert spack_path.canonicalize_path(path) == os.path.normpath(\n os.path.join(mock_low_high_config.scopes[\"low\"].path, os.path.join(\"foo\", \"bar\", \"baz\"))\n )\n\n # test architecture information is in replacements\n assert spack_path.canonicalize_path(\n os.path.join(\"foo\", \"$platform\", \"bar\")\n ) == os.path.abspath(os.path.join(\"foo\", \"test\", \"bar\"))\n\n host_target = spack.platforms.host().default_target()\n host_target_family = str(host_target.family)\n assert spack_path.canonicalize_path(\n os.path.join(\"foo\", \"$target_family\", \"bar\")\n ) == os.path.abspath(os.path.join(\"foo\", host_target_family, \"bar\"))\n\n\npackages_merge_low = {\"packages\": {\"foo\": {\"variants\": [\"+v1\"]}, \"bar\": {\"variants\": [\"+v2\"]}}}\n\npackages_merge_high = {\n \"packages\": {\n \"foo\": {\"version\": [\"a\"]},\n \"bar\": {\"version\": [\"b\"], \"variants\": [\"+v3\"]},\n \"baz\": {\"version\": [\"c\"]},\n }\n}\n\n\n@pytest.mark.regression(\"7924\")\ndef test_merge_with_defaults(mock_low_high_config, write_config_file):\n \"\"\"This ensures that specified preferences merge with defaults as\n expected. Originally all defaults were initialized with the\n exact same object, which led to aliasing problems. Therefore\n the test configs used here leave 'version' blank for multiple\n packages in 'packages_merge_low'.\n \"\"\"\n write_config_file(\"packages\", packages_merge_low, \"low\")\n write_config_file(\"packages\", packages_merge_high, \"high\")\n cfg = spack.config.get(\"packages\")\n\n assert cfg[\"foo\"][\"version\"] == [\"a\"]\n assert cfg[\"bar\"][\"version\"] == [\"b\"]\n assert cfg[\"baz\"][\"version\"] == [\"c\"]\n\n\ndef test_substitute_user(mock_low_high_config, tmp_path: pathlib.Path):\n user = spack_path.get_user()\n base = str(tmp_path)\n assert os.path.join(base, \"foo\", \"bar\", user, \"baz\") == spack_path.canonicalize_path(\n os.path.join(base, \"foo\", \"bar\", \"$user\", \"baz\")\n )\n\n\ndef test_substitute_user_cache(mock_low_high_config):\n user_cache_path = spack.paths.user_cache_path\n assert os.path.join(user_cache_path, \"baz\") == spack_path.canonicalize_path(\n os.path.join(\"$user_cache_path\", \"baz\")\n )\n\n\ndef test_substitute_tempdir(mock_low_high_config):\n tempdir = tempfile.gettempdir()\n assert tempdir == spack_path.canonicalize_path(\"$tempdir\")\n assert os.path.join(tempdir, \"foo\", \"bar\", \"baz\") == spack_path.canonicalize_path(\n os.path.join(\"$tempdir\", \"foo\", \"bar\", \"baz\")\n )\n\n\ndef test_substitute_date(mock_low_high_config):\n test_path = os.path.join(\"hello\", \"world\", \"on\", \"$date\")\n new_path = spack_path.canonicalize_path(test_path)\n assert \"$date\" in test_path\n assert date.today().strftime(\"%Y-%m-%d\") in new_path\n\n\ndef test_substitute_spack_version():\n version = spack.spack_version_info\n assert spack_path.canonicalize_path(\n \"spack$spack_short_version/test\"\n ) == spack_path.canonicalize_path(f\"spack{version[0]}.{version[1]}/test\")\n\n\nPAD_STRING = spack_path.SPACK_PATH_PADDING_CHARS\nMAX_PATH_LEN = spack_path.get_system_path_max()\nMAX_PADDED_LEN = MAX_PATH_LEN - spack_path.SPACK_MAX_INSTALL_PATH_LENGTH\nreps = [PAD_STRING for _ in range((MAX_PADDED_LEN // len(PAD_STRING) + 1) + 2)]\nfull_padded_string = os.path.join(os.sep + \"path\", os.sep.join(reps))[:MAX_PADDED_LEN]\n\n\n@pytest.mark.parametrize(\n \"config_settings_fn,expected_fn\",\n [\n (lambda p: [], lambda p: [None, None, None]),\n (\n lambda p: [[\"config:install_tree:root\", os.path.join(str(p), \"path\")]],\n lambda p: [os.path.join(str(p), \"path\"), None, None],\n ),\n (\n lambda p: [[\"config:install_tree:projections\", {\"all\": \"{name}\"}]],\n lambda p: [None, None, {\"all\": \"{name}\"}],\n ),\n ],\n)\ndef test_parse_install_tree(config_settings_fn, expected_fn, mutable_config, tmp_path):\n config_settings = config_settings_fn(tmp_path)\n expected = expected_fn(tmp_path)\n\n expected_root = expected[0] or mutable_config.get(\"config:install_tree:root\")\n expected_unpadded_root = expected[1] or expected_root\n expected_proj = expected[2] or spack.directory_layout.default_projections\n\n # config settings is a list of 2-element lists, [path, value]\n # where path is a config path and value is the value to set at that path\n # these can be \"splatted\" in as the arguments to config.set\n for config_setting in config_settings:\n mutable_config.set(*config_setting)\n\n config_dict = mutable_config.get(\"config\")\n root, unpadded_root, projections = spack.store.parse_install_tree(config_dict)\n assert root == expected_root\n assert unpadded_root == expected_unpadded_root\n assert projections == expected_proj\n\n\ndef test_change_or_add(mutable_config, mock_packages):\n spack.config.add(\"packages:a:version:['1.0']\", scope=\"user\")\n\n spack.config.add(\"packages:b:version:['1.1']\", scope=\"system\")\n\n class ChangeTest:\n def __init__(self, pkg_name, new_version):\n self.pkg_name = pkg_name\n self.new_version = new_version\n\n def find_fn(self, section):\n return self.pkg_name in section\n\n def change_fn(self, section):\n pkg_section = section.get(self.pkg_name, {})\n pkg_section[\"version\"] = self.new_version\n section[self.pkg_name] = pkg_section\n\n change1 = ChangeTest(\"b\", [\"1.2\"])\n spack.config.change_or_add(\"packages\", change1.find_fn, change1.change_fn)\n assert \"b\" not in mutable_config.get(\"packages\", scope=\"user\")\n assert mutable_config.get(\"packages\")[\"b\"][\"version\"] == [\"1.2\"]\n\n change2 = ChangeTest(\"c\", [\"1.0\"])\n spack.config.change_or_add(\"packages\", change2.find_fn, change2.change_fn)\n assert \"c\" in mutable_config.get(\"packages\", scope=\"user\")\n\n\n@pytest.mark.not_on_windows(\"Padding unsupported on Windows\")\n@pytest.mark.parametrize(\n \"config_settings,expected\",\n [\n (\n [\n [\"config:install_tree:root\", os.sep + \"path\"],\n [\"config:install_tree:padded_length\", 11],\n ],\n [os.path.join(os.sep + \"path\", PAD_STRING[:5]), os.sep + \"path\", None],\n ),\n (\n [[\"config:install_tree:root\", \"/path/$padding:11\"]],\n [os.path.join(os.sep + \"path\", PAD_STRING[:5]), os.sep + \"path\", None],\n ),\n ([[\"config:install_tree:padded_length\", False]], [None, None, None]),\n (\n [\n [\"config:install_tree:padded_length\", True],\n [\"config:install_tree:root\", os.sep + \"path\"],\n ],\n [full_padded_string, os.sep + \"path\", None],\n ),\n ],\n)\ndef test_parse_install_tree_padded(config_settings, expected, mutable_config):\n expected_root = expected[0] or mutable_config.get(\"config:install_tree:root\")\n expected_unpadded_root = expected[1] or expected_root\n expected_proj = expected[2] or spack.directory_layout.default_projections\n\n # config settings is a list of 2-element lists, [path, value]\n # where path is a config path and value is the value to set at that path\n # these can be \"splatted\" in as the arguments to config.set\n for config_setting in config_settings:\n mutable_config.set(*config_setting)\n\n config_dict = mutable_config.get(\"config\")\n root, unpadded_root, projections = spack.store.parse_install_tree(config_dict)\n assert root == expected_root\n assert unpadded_root == expected_unpadded_root\n assert projections == expected_proj\n\n\ndef test_read_config(mock_low_high_config, write_config_file):\n write_config_file(\"config\", config_low, \"low\")\n assert spack.config.get(\"config\") == config_low[\"config\"]\n\n\ndef test_read_config_override_all(mock_low_high_config, write_config_file):\n write_config_file(\"config\", config_low, \"low\")\n write_config_file(\"config\", config_override_all, \"high\")\n assert spack.config.get(\"config\") == {\"install_tree\": {\"root\": \"override_all\"}}\n\n\ndef test_read_config_override_key(mock_low_high_config, write_config_file):\n write_config_file(\"config\", config_low, \"low\")\n write_config_file(\"config\", config_override_key, \"high\")\n assert spack.config.get(\"config\") == {\n \"install_tree\": {\"root\": \"override_key\"},\n \"build_stage\": [\"path1\", \"path2\", \"path3\"],\n }\n\n\ndef test_read_config_merge_list(mock_low_high_config, write_config_file):\n write_config_file(\"config\", config_low, \"low\")\n write_config_file(\"config\", config_merge_list, \"high\")\n assert spack.config.get(\"config\") == {\n \"install_tree\": {\"root\": \"install_tree_path\"},\n \"build_stage\": [\"patha\", \"pathb\", \"path1\", \"path2\", \"path3\"],\n }\n\n\ndef test_read_config_override_list(mock_low_high_config, write_config_file):\n write_config_file(\"config\", config_low, \"low\")\n write_config_file(\"config\", config_override_list, \"high\")\n assert spack.config.get(\"config\") == {\n \"install_tree\": {\"root\": \"install_tree_path\"},\n \"build_stage\": config_override_list[\"config\"][\"build_stage:\"],\n }\n\n\ndef test_internal_config_update(mock_low_high_config, write_config_file):\n write_config_file(\"config\", config_low, \"low\")\n\n before = mock_low_high_config.get(\"config\")\n assert before[\"install_tree\"][\"root\"] == \"install_tree_path\"\n\n # add an internal configuration scope\n scope = spack.config.InternalConfigScope(\"command_line\")\n assert \"InternalConfigScope\" in repr(scope)\n\n mock_low_high_config.push_scope(scope)\n\n command_config = mock_low_high_config.get(\"config\", scope=\"command_line\")\n command_config[\"install_tree\"] = {\"root\": \"foo/bar\"}\n\n mock_low_high_config.set(\"config\", command_config, scope=\"command_line\")\n\n after = mock_low_high_config.get(\"config\")\n assert after[\"install_tree\"][\"root\"] == \"foo/bar\"\n\n\ndef test_internal_config_filename(mock_low_high_config, write_config_file):\n write_config_file(\"config\", config_low, \"low\")\n mock_low_high_config.push_scope(spack.config.InternalConfigScope(\"command_line\"))\n\n with pytest.raises(NotImplementedError):\n mock_low_high_config.get_config_filename(\"command_line\", \"config\")\n\n\ndef test_mark_internal():\n data = {\n \"config\": {\n \"bool\": False,\n \"int\": 6,\n \"numbers\": [1, 2, 3],\n \"string\": \"foo\",\n \"dict\": {\"more_numbers\": [1, 2, 3], \"another_string\": \"foo\", \"another_int\": 7},\n }\n }\n\n marked = spack.config._mark_internal(data, \"x\")\n\n # marked version should be equal to the original\n assert data == marked\n\n def assert_marked(obj):\n if type(obj) is bool:\n return # can't subclass bool, so can't mark it\n\n assert hasattr(obj, \"_start_mark\") and obj._start_mark.name == \"x\"\n assert hasattr(obj, \"_end_mark\") and obj._end_mark.name == \"x\"\n\n # everything in the marked version should have marks\n checks = (\n marked.keys(),\n marked.values(),\n marked[\"config\"].keys(),\n marked[\"config\"].values(),\n marked[\"config\"][\"numbers\"],\n marked[\"config\"][\"dict\"].keys(),\n marked[\"config\"][\"dict\"].values(),\n marked[\"config\"][\"dict\"][\"more_numbers\"],\n )\n\n for seq in checks:\n for obj in seq:\n assert_marked(obj)\n\n\ndef test_internal_config_from_data():\n config = spack.config.create_from(\n spack.config.InternalConfigScope(\n \"_builtin\", {\"config\": {\"verify_ssl\": False, \"build_jobs\": 6}}\n )\n )\n\n assert config.get(\"config:verify_ssl\", scope=\"_builtin\") is False\n assert config.get(\"config:build_jobs\", scope=\"_builtin\") == 6\n\n assert config.get(\"config:verify_ssl\") is False\n assert config.get(\"config:build_jobs\") == 6\n\n # push one on top and see what happens.\n config.push_scope(\n spack.config.InternalConfigScope(\n \"higher\", {\"config\": {\"checksum\": True, \"verify_ssl\": True}}\n )\n )\n\n assert config.get(\"config:verify_ssl\", scope=\"_builtin\") is False\n assert config.get(\"config:build_jobs\", scope=\"_builtin\") == 6\n\n assert config.get(\"config:verify_ssl\", scope=\"higher\") is True\n assert config.get(\"config:build_jobs\", scope=\"higher\") is None\n\n assert config.get(\"config:verify_ssl\") is True\n assert config.get(\"config:build_jobs\") == 6\n assert config.get(\"config:checksum\") is True\n\n assert config.get(\"config:checksum\", scope=\"_builtin\") is None\n assert config.get(\"config:checksum\", scope=\"higher\") is True\n\n\ndef test_keys_are_ordered(configuration_dir):\n \"\"\"Test that keys in Spack YAML files retain their order from the file.\"\"\"\n expected_order = (\n \"./bin\",\n \"./man\",\n \"./share/man\",\n \"./share/aclocal\",\n \"./lib/pkgconfig\",\n \"./lib64/pkgconfig\",\n \"./share/pkgconfig\",\n \"./\",\n )\n\n config_scope = spack.config.DirectoryConfigScope(\"modules\", configuration_dir / \"site\")\n\n data = config_scope.get_section(\"modules\")\n\n prefix_inspections = data[\"modules\"][\"prefix_inspections\"]\n\n for actual, expected in zip(prefix_inspections, expected_order):\n assert actual == expected\n\n\ndef test_config_format_error(mutable_config):\n \"\"\"This is raised when we try to write a bad configuration.\"\"\"\n with pytest.raises(spack.config.ConfigFormatError):\n spack.config.set(\"compilers\", {\"bad\": \"data\"}, scope=\"site\")\n\n\ndef get_config_error(filename, schema, yaml_string):\n \"\"\"Parse a YAML string and return the resulting ConfigFormatError.\n\n Fail if there is no ConfigFormatError\n \"\"\"\n with open(filename, \"w\", encoding=\"utf-8\") as f:\n f.write(yaml_string)\n\n # parse and return error, or fail.\n try:\n spack.config.read_config_file(filename, schema)\n except spack.config.ConfigFormatError as e:\n return e\n else:\n pytest.fail(\"ConfigFormatError was not raised!\")\n\n\ndef test_config_parse_dict_in_list(tmp_path: pathlib.Path):\n with fs.working_dir(str(tmp_path)):\n e = get_config_error(\n \"repos.yaml\",\n spack.schema.repos.schema,\n \"\"\"\\\nrepos:\n a: https://foobar.com/foo\n b: https://foobar.com/bar\n c:\n error:\n - abcdef\n d: https://foobar.com/baz\n\"\"\",\n )\n assert \"repos.yaml:2\" in str(e)\n\n\ndef test_config_parse_str_not_bool(tmp_path: pathlib.Path):\n with fs.working_dir(str(tmp_path)):\n e = get_config_error(\n \"config.yaml\",\n spack.schema.config.schema,\n \"\"\"\\\nconfig:\n verify_ssl: False\n checksum: foobar\n dirty: True\n\"\"\",\n )\n assert \"config.yaml:3\" in str(e)\n\n\ndef test_config_parse_list_in_dict(tmp_path: pathlib.Path):\n with fs.working_dir(str(tmp_path)):\n e = get_config_error(\n \"mirrors.yaml\",\n spack.schema.mirrors.schema,\n \"\"\"\\\nmirrors:\n foo: http://foobar.com/baz\n bar: http://barbaz.com/foo\n baz: http://bazfoo.com/bar\n travis: [1, 2, 3]\n\"\"\",\n )\n assert \"mirrors.yaml:5\" in str(e)\n\n\ndef test_bad_config_section(mock_low_high_config):\n \"\"\"Test that getting or setting a bad section gives an error.\"\"\"\n with pytest.raises(spack.config.ConfigSectionError):\n spack.config.set(\"foobar\", \"foobar\")\n\n with pytest.raises(spack.config.ConfigSectionError):\n spack.config.get(\"foobar\")\n\n\ndef test_nested_override():\n \"\"\"Ensure proper scope naming of nested overrides.\"\"\"\n base_name = spack.config._OVERRIDES_BASE_NAME\n\n def _check_scopes(num_expected, debug_values):\n scope_names = [\n s.name for s in spack.config.CONFIG.scopes.values() if s.name.startswith(base_name)\n ]\n\n for i in range(num_expected):\n name = \"{0}{1}\".format(base_name, i)\n assert name in scope_names\n\n data = spack.config.CONFIG.get_config(\"config\", name)\n assert data[\"debug\"] == debug_values[i]\n\n # Check results from single and nested override\n with spack.config.override(\"config:debug\", True):\n with spack.config.override(\"config:debug\", False):\n _check_scopes(2, [True, False])\n\n _check_scopes(1, [True])\n\n\ndef test_alternate_override(monkeypatch):\n \"\"\"Ensure proper scope naming of override when conflict present.\"\"\"\n base_name = spack.config._OVERRIDES_BASE_NAME\n\n def _matching_scopes(regexpr):\n return [spack.config.InternalConfigScope(\"{0}1\".format(base_name))]\n\n # Check that the alternate naming works\n monkeypatch.setattr(spack.config.CONFIG, \"matching_scopes\", _matching_scopes)\n\n with spack.config.override(\"config:debug\", False):\n name = \"{0}2\".format(base_name)\n\n scope_names = [\n s.name for s in spack.config.CONFIG.scopes.values() if s.name.startswith(base_name)\n ]\n assert name in scope_names\n\n data = spack.config.CONFIG.get_config(\"config\", name)\n assert data[\"debug\"] is False\n\n\ndef test_immutable_scope(tmp_path: pathlib.Path):\n config_yaml = str(tmp_path / \"config.yaml\")\n with open(config_yaml, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nconfig:\n install_tree:\n root: dummy_tree_value\n\"\"\"\n )\n scope = spack.config.DirectoryConfigScope(\"test\", str(tmp_path), writable=False)\n\n data = scope.get_section(\"config\")\n assert data is not None\n assert data[\"config\"][\"install_tree\"] == {\"root\": \"dummy_tree_value\"}\n\n with pytest.raises(spack.error.ConfigError):\n scope._write_section(\"config\")\n\n\ndef test_single_file_scope(config, env_yaml):\n scope = spack.config.SingleFileScope(\n \"env\", env_yaml, spack.schema.env.schema, yaml_path=[\"spack\"]\n )\n\n with spack.config.override(scope):\n # from the single-file config\n assert spack.config.get(\"config:verify_ssl\") is False\n assert spack.config.get(\"config:dirty\") is False\n\n # from the lower config scopes\n assert spack.config.get(\"config:checksum\") is True\n assert spack.config.get(\"config:checksum\") is True\n assert spack.config.get(\"packages:externalmodule:buildable\") is False\n assert spack.config.get(\"repos\") == {\n \"z\": \"/x/y/z\",\n \"builtin_mock\": \"$spack/var/spack/test_repos/spack_repo/builtin_mock\",\n }\n\n\ndef test_single_file_scope_section_override(tmp_path: pathlib.Path, config):\n \"\"\"Check that individual config sections can be overridden in an\n environment config. The config here primarily differs in that the\n ``packages`` section is intended to override all other scopes (using the\n \"::\" syntax).\n \"\"\"\n env_yaml = str(tmp_path / \"env.yaml\")\n with open(env_yaml, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n config:\n verify_ssl: False\n packages::\n all:\n target: [ x86_64 ]\n repos:\n z: /x/y/z\n\"\"\"\n )\n\n scope = spack.config.SingleFileScope(\n \"env\", env_yaml, spack.schema.env.schema, yaml_path=[\"spack\"]\n )\n\n with spack.config.override(scope):\n # from the single-file config\n assert spack.config.get(\"config:verify_ssl\") is False\n assert spack.config.get(\"packages:all:target\") == [\"x86_64\"]\n\n # from the lower config scopes\n assert spack.config.get(\"config:checksum\") is True\n assert not spack.config.get(\"packages:externalmodule\")\n assert spack.config.get(\"repos\") == {\n \"z\": \"/x/y/z\",\n \"builtin_mock\": \"$spack/var/spack/test_repos/spack_repo/builtin_mock\",\n }\n\n\ndef test_write_empty_single_file_scope(tmp_path: pathlib.Path):\n env_schema = spack.schema.env.schema\n config_file = tmp_path / \"config.yaml\"\n config_file.touch()\n scope = spack.config.SingleFileScope(\"test\", str(config_file), env_schema, yaml_path=[\"spack\"])\n scope._write_section(\"config\")\n # confirm we can write empty config\n assert not scope.get_section(\"config\")\n\n\ndef check_schema(name, file_contents):\n \"\"\"Check a Spack YAML schema against some data\"\"\"\n f = io.StringIO(file_contents)\n data = syaml.load_config(f)\n spack.config.validate(data, name)\n\n\ndef test_good_env_yaml():\n check_schema(\n spack.schema.env.schema,\n \"\"\"\\\nspack:\n config:\n verify_ssl: False\n dirty: False\n repos:\n - ~/my/repo/location\n mirrors:\n remote: /foo/bar/baz\n compilers:\n - compiler:\n spec: cce@2.1\n operating_system: cnl\n modules: []\n paths:\n cc: /path/to/cc\n cxx: /path/to/cxx\n fc: /path/to/fc\n f77: /path/to/f77\n\"\"\",\n )\n\n\ndef test_bad_env_yaml():\n with pytest.raises(spack.config.ConfigFormatError):\n check_schema(\n spack.schema.env.schema,\n \"\"\"\\\nspack:\n foobar:\n verify_ssl: False\n dirty: False\n\"\"\",\n )\n\n\ndef test_bad_config_yaml():\n with pytest.raises(spack.config.ConfigFormatError):\n check_schema(\n spack.schema.config.schema,\n \"\"\"\\\nconfig:\n verify_ssl: False\n install_tree:\n root:\n extra_level: foo\n\"\"\",\n )\n\n\ndef test_bad_include_yaml():\n with pytest.raises(spack.config.ConfigFormatError, match=\"is not of type\"):\n check_schema(\n spack.schema.include.schema,\n \"\"\"\\\ninclude: $HOME/include.yaml\n\"\"\",\n )\n\n\ndef test_bad_mirrors_yaml():\n with pytest.raises(spack.config.ConfigFormatError):\n check_schema(\n spack.schema.mirrors.schema,\n \"\"\"\\\nmirrors:\n local: True\n\"\"\",\n )\n\n\ndef test_bad_repos_yaml():\n with pytest.raises(spack.config.ConfigFormatError):\n check_schema(\n spack.schema.repos.schema,\n \"\"\"\\\nrepos:\n True\n\"\"\",\n )\n\n\ndef test_bad_compilers_yaml():\n with pytest.raises(spack.config.ConfigFormatError):\n check_schema(\n spack.schema.compilers.schema,\n \"\"\"\\\ncompilers:\n key_instead_of_list: 'value'\n\"\"\",\n )\n\n with pytest.raises(spack.config.ConfigFormatError):\n check_schema(\n spack.schema.compilers.schema,\n \"\"\"\\\ncompilers:\n - shmompiler:\n environment: /bad/value\n\"\"\",\n )\n\n with pytest.raises(spack.config.ConfigFormatError):\n check_schema(\n spack.schema.compilers.schema,\n \"\"\"\\\ncompilers:\n - compiler:\n fenfironfent: /bad/value\n\"\"\",\n )\n\n\ndef test_internal_config_section_override(mock_low_high_config, write_config_file):\n write_config_file(\"config\", config_merge_list, \"low\")\n wanted_list = config_override_list[\"config\"][\"build_stage:\"]\n mock_low_high_config.push_scope(\n spack.config.InternalConfigScope(\"high\", {\"config:\": {\"build_stage\": wanted_list}})\n )\n assert mock_low_high_config.get(\"config:build_stage\") == wanted_list\n\n\ndef test_internal_config_dict_override(mock_low_high_config, write_config_file):\n write_config_file(\"config\", config_merge_dict, \"low\")\n wanted_dict = config_override_dict[\"config\"][\"aliases:\"]\n mock_low_high_config.push_scope(spack.config.InternalConfigScope(\"high\", config_override_dict))\n assert mock_low_high_config.get(\"config:aliases\") == wanted_dict\n\n\ndef test_internal_config_list_override(mock_low_high_config, write_config_file):\n write_config_file(\"config\", config_merge_list, \"low\")\n wanted_list = config_override_list[\"config\"][\"build_stage:\"]\n mock_low_high_config.push_scope(spack.config.InternalConfigScope(\"high\", config_override_list))\n assert mock_low_high_config.get(\"config:build_stage\") == wanted_list\n\n\ndef test_set_section_override(mock_low_high_config, write_config_file):\n write_config_file(\"config\", config_merge_list, \"low\")\n wanted_list = config_override_list[\"config\"][\"build_stage:\"]\n with spack.config.override(\"config::build_stage\", wanted_list):\n assert mock_low_high_config.get(\"config:build_stage\") == wanted_list\n assert config_merge_list[\"config\"][\"build_stage\"] == mock_low_high_config.get(\n \"config:build_stage\"\n )\n\n\ndef test_set_list_override(mock_low_high_config, write_config_file):\n write_config_file(\"config\", config_merge_list, \"low\")\n wanted_list = config_override_list[\"config\"][\"build_stage:\"]\n with spack.config.override(\"config:build_stage:\", wanted_list):\n assert wanted_list == mock_low_high_config.get(\"config:build_stage\")\n assert config_merge_list[\"config\"][\"build_stage\"] == mock_low_high_config.get(\n \"config:build_stage\"\n )\n\n\ndef test_set_dict_override(mock_low_high_config, write_config_file):\n write_config_file(\"config\", config_merge_dict, \"low\")\n wanted_dict = config_override_dict[\"config\"][\"aliases:\"]\n with spack.config.override(\"config:aliases:\", wanted_dict):\n assert wanted_dict == mock_low_high_config.get(\"config:aliases\")\n assert config_merge_dict[\"config\"][\"aliases\"] == mock_low_high_config.get(\"config:aliases\")\n\n\ndef test_set_bad_path(config):\n with pytest.raises(ValueError):\n with spack.config.override(\":bad:path\", \"\"):\n pass\n\n\ndef test_bad_path_double_override(config):\n with pytest.raises(syaml.SpackYAMLError, match=\"Meaningless second override\"):\n with spack.config.override(\"bad::double:override::directive\", \"\"):\n pass\n\n\ndef test_license_dir_config(mutable_config, mock_packages, tmp_path):\n \"\"\"Ensure license directory is customizable\"\"\"\n expected_dir = spack.paths.default_license_dir\n assert spack.config.get(\"config:license_dir\") == expected_dir\n assert spack.package_base.PackageBase.global_license_dir == expected_dir\n assert spack.repo.PATH.get_pkg_class(\"pkg-a\").global_license_dir == expected_dir\n\n abs_path = str(tmp_path / \"foo\" / \"bar\" / \"baz\")\n spack.config.set(\"config:license_dir\", abs_path)\n assert spack.config.get(\"config:license_dir\") == abs_path\n assert spack.package_base.PackageBase.global_license_dir == abs_path\n assert spack.repo.PATH.get_pkg_class(\"pkg-a\").global_license_dir == abs_path\n\n\n@pytest.mark.regression(\"22547\")\ndef test_single_file_scope_cache_clearing(env_yaml):\n scope = spack.config.SingleFileScope(\n \"env\", env_yaml, spack.schema.env.schema, yaml_path=[\"spack\"]\n )\n # Check that we can retrieve data from the single file scope\n before = scope.get_section(\"config\")\n assert before\n # Clear the cache of the Single file scope\n scope.clear()\n # Check that the section can be retrieved again and it's\n # the same as before\n after = scope.get_section(\"config\")\n assert after\n assert before == after\n\n\n@pytest.mark.regression(\"22611\")\ndef test_internal_config_scope_cache_clearing():\n \"\"\"\n An InternalConfigScope object is constructed from data that is already\n in memory, therefore it doesn't have any cache to clear. Here we ensure\n that calling the clear method is consistent with that..\n \"\"\"\n data = {\"config\": {\"build_jobs\": 10}}\n internal_scope = spack.config.InternalConfigScope(\"internal\", data)\n # Ensure that the initial object is properly set\n assert internal_scope.sections[\"config\"] == data\n # Call the clear method\n internal_scope.clear()\n # Check that this didn't affect the scope object\n assert internal_scope.sections[\"config\"] == data\n\n\ndef test_system_config_path_is_overridable(working_env):\n p = \"/some/path\"\n os.environ[\"SPACK_SYSTEM_CONFIG_PATH\"] = p\n assert spack.paths._get_system_config_path() == p\n\n\ndef test_system_config_path_is_default_when_env_var_is_empty(working_env):\n os.environ[\"SPACK_SYSTEM_CONFIG_PATH\"] = \"\"\n assert os.sep + os.path.join(\"etc\", \"spack\") == spack.paths._get_system_config_path()\n\n\ndef test_user_config_path_is_overridable(working_env):\n p = \"/some/path\"\n os.environ[\"SPACK_USER_CONFIG_PATH\"] = p\n assert p == spack.paths._get_user_config_path()\n\n\ndef test_user_config_path_is_default_when_env_var_is_empty(working_env):\n os.environ[\"SPACK_USER_CONFIG_PATH\"] = \"\"\n assert os.path.expanduser(\"~%s.spack\" % os.sep) == spack.paths._get_user_config_path()\n\n\ndef test_default_install_tree(monkeypatch, default_config):\n s = spack.spec.Spec(\"nonexistent@x.y.z arch=foo-bar-baz\")\n monkeypatch.setattr(s, \"dag_hash\", lambda length: \"abc123\")\n _, _, projections = spack.store.parse_install_tree(spack.config.get(\"config\"))\n assert s.format(projections[\"all\"]) == \"foo-baz/nonexistent-x.y.z-abc123\"\n\n\n@pytest.fixture\ndef mock_include_scope(tmp_path):\n for subdir in [\"defaults\", \"test1\", \"test2\", \"test3\"]:\n path = tmp_path / subdir\n path.mkdir()\n\n include = tmp_path / \"include.yaml\"\n with include.open(\"w\", encoding=\"utf-8\") as f:\n f.write(\n textwrap.dedent(\n \"\"\"\\\n include::\n - name: \"test1\"\n path: \"test1\"\n when: '\"SPACK_DISABLE_LOCAL_CONFIG\" not in env'\n\n - name: \"test2\"\n path: \"test2\"\n\n - name: \"test3\"\n path: \"test3\"\n when: '\"SPACK_DISABLE_LOCAL_CONFIG\" not in env'\n \"\"\"\n )\n )\n\n yield tmp_path\n\n\n@pytest.fixture\ndef include_config_factory(mock_include_scope):\n def make_config():\n cfg = spack.config.Configuration()\n cfg.push_scope(\n spack.config.DirectoryConfigScope(\"defaults\", str(mock_include_scope / \"defaults\")),\n priority=ConfigScopePriority.DEFAULTS,\n )\n cfg.push_scope(\n spack.config.DirectoryConfigScope(\"tmp_path\", str(mock_include_scope)),\n priority=ConfigScopePriority.CONFIG_FILES,\n )\n return cfg\n\n yield make_config\n\n\ndef test_modify_scope_precedence(working_env, include_config_factory, tmp_path):\n \"\"\"Test how spack selects the scope to modify when commands write config.\"\"\"\n\n cfg = include_config_factory()\n\n # ensure highest precedence writable scope is selected by default\n assert cfg.highest_precedence_scope().name == \"tmp_path\"\n\n include_yaml = tmp_path / \"include.yaml\"\n subdir = tmp_path / \"subdir\"\n subdir2 = tmp_path / \"subdir2\"\n subdir.mkdir()\n subdir2.mkdir()\n\n with include_yaml.open(\"w\", encoding=\"utf-8\") as f:\n f.write(\n textwrap.dedent(\n \"\"\"\\\n include::\n - name: \"subdir\"\n path: \"subdir\"\n \"\"\"\n )\n )\n\n cfg.push_scope(\n spack.config.DirectoryConfigScope(\"override\", str(tmp_path)),\n priority=ConfigScopePriority.CONFIG_FILES,\n )\n\n # ensure override scope is selected when it is on top\n assert cfg.highest_precedence_scope().name == \"override\"\n\n cfg.remove_scope(\"override\")\n\n with include_yaml.open(\"w\", encoding=\"utf-8\") as f:\n f.write(\n textwrap.dedent(\n \"\"\"\\\n include::\n - name: \"subdir\"\n path: \"subdir\"\n prefer_modify: true\n \"\"\"\n )\n )\n\n cfg.push_scope(\n spack.config.DirectoryConfigScope(\"override\", str(tmp_path)),\n priority=ConfigScopePriority.CONFIG_FILES,\n )\n\n # if the top scope prefers another, ensure it is selected\n assert cfg.highest_precedence_scope().name == \"subdir\"\n\n cfg.remove_scope(\"override\")\n\n with include_yaml.open(\"w\", encoding=\"utf-8\") as f:\n f.write(\n textwrap.dedent(\n \"\"\"\\\n include::\n - name: \"subdir\"\n path: \"subdir\"\n - name: \"subdir2\"\n path: \"subdir2\"\n prefer_modify: true\n \"\"\"\n )\n )\n\n cfg.push_scope(\n spack.config.DirectoryConfigScope(\"override\", str(tmp_path)),\n priority=ConfigScopePriority.CONFIG_FILES,\n )\n\n # if there are multiple scopes and one is preferred, make sure it's that one\n assert cfg.highest_precedence_scope().name == \"subdir2\"\n\n\ndef test_local_config_can_be_disabled(working_env, include_config_factory):\n \"\"\"Ensure that SPACK_DISABLE_LOCAL_CONFIG disables configurations with `when:`.\"\"\"\n os.environ[\"SPACK_DISABLE_LOCAL_CONFIG\"] = \"true\"\n cfg = include_config_factory()\n assert \"defaults\" in cfg.scopes\n assert \"test1\" not in cfg.scopes\n assert \"test2\" in cfg.scopes\n assert \"test3\" not in cfg.scopes\n\n os.environ[\"SPACK_DISABLE_LOCAL_CONFIG\"] = \"\"\n cfg = include_config_factory()\n assert \"defaults\" in cfg.scopes\n assert \"test1\" not in cfg.scopes\n assert \"test2\" in cfg.scopes\n assert \"test3\" not in cfg.scopes\n\n del os.environ[\"SPACK_DISABLE_LOCAL_CONFIG\"]\n cfg = include_config_factory()\n assert \"defaults\" in cfg.scopes\n assert \"test1\" in cfg.scopes\n assert \"test2\" in cfg.scopes\n assert \"test3\" in cfg.scopes\n\n\ndef test_override_included_config(working_env, tmp_path, include_config_factory):\n override_scope = tmp_path / \"override\"\n override_scope.mkdir()\n\n include_yaml = override_scope / \"include.yaml\"\n subdir = override_scope / \"subdir\"\n subdir.mkdir()\n anotherdir = override_scope / \"anotherdir\"\n anotherdir.mkdir()\n\n with include_yaml.open(\"w\", encoding=\"utf-8\") as f:\n f.write(\n textwrap.dedent(\n \"\"\"\\\n include::\n - name: \"subdir\"\n path: \"subdir\"\n \"\"\"\n )\n )\n\n with (subdir / \"include.yaml\").open(\"w\", encoding=\"utf-8\") as f:\n f.write(\n textwrap.dedent(\n \"\"\"\\\n include:\n - name: \"anotherdir\"\n path: \"../anotherdir\"\n \"\"\"\n )\n )\n\n # check the mock config is correct\n cfg = include_config_factory()\n\n assert \"defaults\" in cfg.scopes\n assert \"tmp_path\" in cfg.scopes\n assert \"test1\" in cfg.scopes\n assert \"test2\" in cfg.scopes\n assert \"test3\" in cfg.scopes\n\n active_names = [s.name for s in cfg.active_scopes]\n assert \"defaults\" in active_names\n assert \"tmp_path\" in active_names\n assert \"test1\" in active_names\n assert \"test2\" in active_names\n assert \"test3\" in active_names\n\n includes = str(cfg.get(\"include\"))\n assert \"subdir\" not in includes\n assert \"anotherdir\" not in includes\n assert \"test1\" in includes\n assert \"test2\" in includes\n assert \"test3\" in includes\n\n # push a scope that overrides everything under it but includes a subdir.\n # its included subdir should be active, but scopes *not* included by the overriding\n # scope should not.\n cfg.push_scope(\n spack.config.DirectoryConfigScope(\"override\", str(override_scope)),\n priority=ConfigScopePriority.CONFIG_FILES,\n )\n\n assert \"defaults\" in cfg.scopes\n assert \"tmp_path\" in cfg.scopes\n assert \"test1\" in cfg.scopes\n assert \"test2\" in cfg.scopes\n assert \"test3\" in cfg.scopes\n assert \"override\" in cfg.scopes\n assert \"subdir\" in cfg.scopes\n assert \"anotherdir\" in cfg.scopes\n\n active_names = [s.name for s in cfg.active_scopes]\n assert \"defaults\" in active_names\n assert \"tmp_path\" in active_names\n assert \"test1\" not in active_names\n assert \"test2\" not in active_names\n assert \"test3\" not in active_names\n assert \"override\" in active_names\n assert \"subdir\" in active_names\n assert \"anotherdir\" not in active_names\n\n includes = str(cfg.get(\"include\"))\n assert \"subdir\" in includes\n assert \"anotherdir\" not in includes\n assert \"test1\" not in includes\n assert \"test2\" not in includes\n assert \"test3\" not in includes\n\n # remove the override and ensure everything is back to normal\n cfg.remove_scope(\"override\")\n\n assert \"defaults\" in cfg.scopes\n assert \"tmp_path\" in cfg.scopes\n assert \"test1\" in cfg.scopes\n assert \"test2\" in cfg.scopes\n assert \"test3\" in cfg.scopes\n\n active_names = [s.name for s in cfg.active_scopes]\n assert \"defaults\" in active_names\n assert \"tmp_path\" in active_names\n assert \"test1\" in active_names\n assert \"test2\" in active_names\n assert \"test3\" in active_names\n\n includes = str(cfg.get(\"include\"))\n assert \"subdir\" not in includes\n assert \"anotherdir\" not in includes\n assert \"test1\" in includes\n assert \"test2\" in includes\n assert \"test3\" in includes\n\n\ndef test_user_cache_path_is_overridable(working_env):\n p = \"/some/path\"\n os.environ[\"SPACK_USER_CACHE_PATH\"] = p\n assert spack.paths._get_user_cache_path() == p\n\n\ndef test_user_cache_path_is_default_when_env_var_is_empty(working_env):\n os.environ[\"SPACK_USER_CACHE_PATH\"] = \"\"\n assert os.path.expanduser(\"~%s.spack\" % os.sep) == spack.paths._get_user_cache_path()\n\n\ndef test_config_file_dir_failure(tmp_path: pathlib.Path, mutable_empty_config):\n with pytest.raises(spack.config.ConfigFileError, match=\"not a file\"):\n spack.config.read_config_file(str(tmp_path))\n\n\n@pytest.mark.not_on_windows(\"chmod not supported on Windows\")\n@pytest.mark.skipif(getuid() == 0, reason=\"user is root\")\ndef test_config_file_read_perms_failure(tmp_path: pathlib.Path, mutable_empty_config):\n \"\"\"Test reading a configuration file without permissions to ensure\n ConfigFileError is raised.\"\"\"\n filename = join_path(str(tmp_path), \"test.yaml\")\n touch(filename)\n os.chmod(filename, 0o200)\n\n with pytest.raises(spack.config.ConfigFileError, match=\"not readable\"):\n spack.config.read_config_file(filename)\n\n\ndef test_config_file_read_invalid_yaml(tmp_path: pathlib.Path, mutable_empty_config):\n \"\"\"Test reading a configuration file with invalid (unparsable) YAML\n raises a ConfigFileError.\"\"\"\n filename = join_path(str(tmp_path), \"test.yaml\")\n with open(filename, \"w\", encoding=\"utf-8\") as f:\n f.write(\"spack:\\nview\")\n\n with pytest.raises(spack.config.ConfigFileError, match=\"parsing YAML\"):\n spack.config.read_config_file(filename)\n\n\n@pytest.mark.parametrize(\n \"path,it_should_work,expected_parsed\",\n [\n (\"x:y:z\", True, [\"x:\", \"y:\", \"z\"]),\n (\"x+::y:z\", True, [\"x+::\", \"y:\", \"z\"]),\n ('x:y:\"{z}\"', True, [\"x:\", \"y:\", '\"{z}\"']),\n ('x:\"y\"+:z', True, [\"x:\", '\"y\"+:', \"z\"]),\n ('x:\"y\"trail:z', False, None),\n (\"x:y:[1.0]\", True, [\"x:\", \"y:\", \"[1.0]\"]),\n (\"x:y:['1.0']\", True, [\"x:\", \"y:\", \"['1.0']\"]),\n (\"x:{y}:z\", True, [\"x:\", \"{y}:\", \"z\"]),\n (\"x:'{y}':z\", True, [\"x:\", \"'{y}':\", \"z\"]),\n (\"x:{y}\", True, [\"x:\", \"{y}\"]),\n ],\n)\ndef test_config_path_dsl(path, it_should_work, expected_parsed):\n if it_should_work:\n assert spack.config.ConfigPath._validate(path) == expected_parsed\n else:\n with pytest.raises(ValueError):\n spack.config.ConfigPath._validate(path)\n\n\n@pytest.mark.regression(\"48254\")\ndef test_env_activation_preserves_command_line_scope(mutable_mock_env_path):\n \"\"\"Check that the \"command_line\" scope remains the highest priority scope, when we activate,\n or deactivate, environments.\n \"\"\"\n expected_cl_scope = spack.config.CONFIG.highest()\n assert expected_cl_scope.name == \"command_line\"\n\n # Creating an environment pushes a new scope\n ev.create(\"test\")\n with ev.read(\"test\"):\n assert spack.config.CONFIG.highest() == expected_cl_scope\n\n # No active environment pops the scope\n with ev.no_active_environment():\n assert spack.config.CONFIG.highest() == expected_cl_scope\n assert spack.config.CONFIG.highest() == expected_cl_scope\n\n # Switch the environment to another one\n ev.create(\"test-2\")\n with ev.read(\"test-2\"):\n assert spack.config.CONFIG.highest() == expected_cl_scope\n assert spack.config.CONFIG.highest() == expected_cl_scope\n\n assert spack.config.CONFIG.highest() == expected_cl_scope\n\n\n@pytest.mark.regression(\"48414\")\n@pytest.mark.regression(\"49188\")\ndef test_env_activation_preserves_config_scopes(mutable_mock_env_path):\n \"\"\"Check that the priority of scopes is respected when merging configuration files.\"\"\"\n custom_scope = spack.config.InternalConfigScope(\"custom_scope\")\n spack.config.CONFIG.push_scope(custom_scope, priority=ConfigScopePriority.CUSTOM)\n expected_scopes_without_env = [\"custom_scope\", \"command_line\"]\n expected_scopes_with_first_env = [\"env:test\", \"custom_scope\", \"command_line\"]\n expected_scopes_with_second_env = [\"env:test-2\", \"custom_scope\", \"command_line\"]\n\n def highest_priority_scopes(config, *, nscopes):\n return list(config.scopes)[-nscopes:]\n\n assert highest_priority_scopes(spack.config.CONFIG, nscopes=2) == expected_scopes_without_env\n # Creating an environment pushes a new scope\n ev.create(\"test\")\n with ev.read(\"test\"):\n assert (\n highest_priority_scopes(spack.config.CONFIG, nscopes=3)\n == expected_scopes_with_first_env\n )\n\n # No active environment pops the scope\n with ev.no_active_environment():\n assert (\n highest_priority_scopes(spack.config.CONFIG, nscopes=2)\n == expected_scopes_without_env\n )\n assert (\n highest_priority_scopes(spack.config.CONFIG, nscopes=3)\n == expected_scopes_with_first_env\n )\n\n # Switch the environment to another one\n ev.create(\"test-2\")\n with ev.read(\"test-2\"):\n assert (\n highest_priority_scopes(spack.config.CONFIG, nscopes=3)\n == expected_scopes_with_second_env\n )\n assert (\n highest_priority_scopes(spack.config.CONFIG, nscopes=3)\n == expected_scopes_with_first_env\n )\n\n assert highest_priority_scopes(spack.config.CONFIG, nscopes=2) == expected_scopes_without_env\n\n\n@pytest.mark.regression(\"51059\")\ndef test_config_include_similar_name(tmp_path: pathlib.Path):\n config_a = tmp_path / \"a\" / \"config\"\n config_b = tmp_path / \"b\" / \"config\"\n\n os.makedirs(config_a)\n with open(config_a / \"config.yaml\", \"w\", encoding=\"utf-8\") as fd:\n syaml.dump_config({\"config\": {\"install_tree\": {\"root\": str(tmp_path)}}}, fd)\n\n os.makedirs(config_b)\n with open(config_b / \"config.yaml\", \"w\", encoding=\"utf-8\") as fd:\n syaml.dump_config({\"config\": {\"install_tree\": {\"padded_length\": 64}}}, fd)\n\n with open(tmp_path / \"include.yaml\", \"w\", encoding=\"utf-8\") as fd:\n syaml.dump_config({\"include\": [str(config_a), str(config_b)]}, fd)\n\n config = spack.config.create_from(spack.config.DirectoryConfigScope(\"test\", str(tmp_path)))\n\n # Ensure all of the scopes are found\n assert len(config.matching_scopes(\"^test$\")) == 1\n assert len(config.matching_scopes(\"^test:a/config$\")) == 1\n assert len(config.matching_scopes(\"^test:b/config$\")) == 1\n\n\ndef test_deepcopy_as_builtin(env_yaml):\n cfg = spack.config.create_from(\n spack.config.SingleFileScope(\"env\", env_yaml, spack.schema.env.schema, yaml_path=[\"spack\"])\n )\n config_copy = cfg.deepcopy_as_builtin(\"config\")\n assert config_copy == cfg.get_config(\"config\")\n assert type(config_copy) is DictWithLineInfo\n assert type(config_copy[\"verify_ssl\"]) is bool\n\n packages_copy = cfg.deepcopy_as_builtin(\"packages\")\n assert type(packages_copy) is DictWithLineInfo\n assert type(packages_copy[\"all\"]) is DictWithLineInfo\n assert type(packages_copy[\"all\"][\"compiler\"]) is list\n assert type(packages_copy[\"all\"][\"compiler\"][0]) is str\n\n\ndef test_included_optional_include_scopes():\n with pytest.raises(NotImplementedError):\n spack.config.OptionalInclude({}).scopes(spack.config.ConfigScope(\"fail\"))\n\n\ndef test_included_path_string(\n tmp_path: pathlib.Path, mock_low_high_config, ensure_debug, monkeypatch, capfd\n):\n path = tmp_path / \"local\" / \"config.yaml\"\n path.parent.mkdir()\n include = spack.config.included_path(path)\n assert isinstance(include, spack.config.IncludePath)\n assert include.path == str(path)\n assert not include.optional\n assert include.evaluate_condition()\n\n parent_scope = mock_low_high_config.scopes[\"low\"]\n\n # Trigger failure when required path does not exist\n with pytest.raises(ValueError, match=\"does not exist\"):\n include.scopes(parent_scope)\n\n # First successful pass builds the scope\n path.touch()\n scopes = include.scopes(parent_scope)\n assert scopes and len(scopes) == 1\n assert isinstance(scopes[0], spack.config.SingleFileScope)\n\n # Second pass uses the scopes previously built\n assert include._scopes is not None\n scopes = include.scopes(parent_scope)\n captured = capfd.readouterr()[1]\n assert \"Using existing scopes\" in captured\n\n\ndef test_included_path_string_no_parent_path(\n tmp_path: pathlib.Path, config, ensure_debug, monkeypatch\n):\n \"\"\"Use a relative include path and no parent scope path so destination\n will be rooted in the current working directory (usually SPACK_ROOT).\"\"\"\n entry = {\"path\": \"config.yaml\", \"optional\": True}\n include = spack.config.included_path(entry)\n parent_scope = spack.config.InternalConfigScope(\"parent-scope\")\n included_scopes = include.scopes(parent_scope)\n # ensure scope is returned even if there is no parent path\n assert len(included_scopes) == 1\n # ensure scope for include is singlefile as it ends in .yaml\n assert isinstance(included_scopes[0], spack.config.SingleFileScope)\n destination = include.destination\n curr_dir = os.getcwd()\n assert curr_dir == os.path.commonprefix([curr_dir, destination]) # type: ignore[list-item]\n\n\ndef test_included_path_substitution():\n # check a straight path substitution\n entry = {\"path\": \"$user_cache_path/path/to/config.yaml\"}\n include = spack.config.included_path(entry)\n assert spack.paths.user_cache_path in include.path\n\n # check path through an environment variable\n path = \"/path/to/project/packages.yaml\"\n os.environ[\"SPACK_TEST_PATH_SUB\"] = path\n entry = {\"name\": \"vartest\", \"path\": \"$SPACK_TEST_PATH_SUB\"}\n include = spack.config.included_path(entry)\n assert path in include.path\n\n\ndef test_included_path_conditional_bad_when(\n tmp_path: pathlib.Path, mock_low_high_config, ensure_debug, capfd\n):\n path = tmp_path / \"local\"\n path.mkdir()\n entry = {\"path\": str(path), \"when\": 'platform == \"nosuchplatform\"', \"optional\": True}\n include = spack.config.included_path(entry)\n assert isinstance(include, spack.config.IncludePath)\n assert include.path == entry[\"path\"]\n assert include.when == entry[\"when\"]\n assert include.optional\n assert not include.evaluate_condition()\n\n scopes = include.scopes(mock_low_high_config.scopes[\"low\"])\n captured = capfd.readouterr()[1]\n assert \"condition is not satisfied\" in captured\n assert not scopes\n\n\ndef test_included_path_conditional_success(tmp_path: pathlib.Path, mock_low_high_config):\n path = tmp_path / \"local\"\n path.mkdir()\n entry = {\"path\": str(path), \"when\": 'platform == \"test\"', \"optional\": True}\n include = spack.config.included_path(entry)\n assert isinstance(include, spack.config.IncludePath)\n assert include.path == entry[\"path\"]\n assert include.when == entry[\"when\"]\n assert include.optional\n assert include.evaluate_condition()\n\n scopes = include.scopes(mock_low_high_config.scopes[\"low\"])\n assert scopes and len(scopes) == 1\n assert isinstance(scopes[0], spack.config.DirectoryConfigScope)\n\n\ndef test_included_path_git_missing_args():\n # must have one or more of: branch, tag and commit so fail if missing any\n entry = {\"git\": \"https://example.com/windows/configs.git\", \"paths\": [\"config.yaml\"]}\n with pytest.raises(spack.error.ConfigError, match=\"specify one or more\"):\n spack.config.included_path(entry)\n\n # must have one or more paths\n entry[\"tag\"] = \"v1.0\"\n entry[\"paths\"] = []\n with pytest.raises(spack.error.ConfigError, match=\"must include one or more\"):\n spack.config.included_path(entry)\n\n\ndef test_included_path_git_unsat(\n tmp_path: pathlib.Path, mock_low_high_config, ensure_debug, monkeypatch, capfd\n):\n paths = [\"config.yaml\", \"packages.yaml\"]\n entry = {\n \"git\": \"https://example.com/windows/configs.git\",\n \"tag\": \"v1.0\",\n \"paths\": paths,\n \"when\": 'platform == \"nosuchplatform\"',\n }\n include = spack.config.included_path(entry)\n assert isinstance(include, spack.config.GitIncludePaths)\n assert include.git == entry[\"git\"]\n assert include.tag == entry[\"tag\"]\n assert include.paths == entry[\"paths\"]\n assert include.when == entry[\"when\"]\n assert not include.optional and not include.evaluate_condition()\n\n scopes = include.scopes(mock_low_high_config.scopes[\"low\"])\n captured = capfd.readouterr()[1]\n assert \"condition is not satisfied\" in captured\n assert not scopes\n\n\ndef test_included_path_git_substitutions():\n # check path substitutions for the git url *and* paths\n paths = [\"./$platform/config.yaml\", \"$platform/packages.yaml\"]\n entry = {\n \"git\": \"https://example.com/$platform/configs.git\",\n \"branch\": \"develop\",\n \"name\": \"site\",\n \"paths\": paths,\n \"when\": 'platform == \"test\"',\n }\n include = spack.config.included_path(entry)\n assert isinstance(include, spack.config.GitIncludePaths)\n assert not include.optional and include.evaluate_condition()\n assert \"test\" in include.git, \"Expected the git url to contain the platform\"\n for path in include.paths:\n assert \"test\" in path, \"Expected the included git path to contain the platform\"\n\n # check environment substitution for the git url\n url = \"https://example.com/path/to/configs.git\"\n os.environ[\"SPACK_TEST_URL_SUB\"] = url\n entry[\"git\"] = \"$SPACK_TEST_URL_SUB\"\n include = spack.config.included_path(entry)\n assert include.git == url, \"Expected git url environment var substitution\"\n\n\n@pytest.mark.parametrize(\n \"key,value\", [(\"branch\", \"main\"), (\"commit\", \"abcdef123456\"), (\"tag\", \"v1.0\")]\n)\ndef test_included_path_git(\n tmp_path: pathlib.Path, mock_low_high_config, ensure_debug, monkeypatch, key, value, capfd\n):\n \"\"\"Check git includes for branch, commit, and tag using relative paths.\n\n Note the mock config fixture does NOT create the scope path so a temporary\n directory will be used for caching the files.\n \"\"\"\n\n # Specifying two relative paths, one explicit, one implicit\n paths = [\"./config.yaml\", \"packages.yaml\"]\n entry = {\n \"git\": \"https://example.com/windows/configs.git\",\n key: value,\n \"name\": \"site\",\n \"paths\": paths,\n \"when\": 'platform == \"test\"',\n }\n include = spack.config.included_path(entry)\n assert isinstance(include, spack.config.GitIncludePaths)\n assert not include.optional and include.evaluate_condition()\n\n # set up minimal git and repository operations\n class MockIncludeGit(spack.util.executable.Executable):\n def __init__(self, required: bool):\n pass\n\n def __call__(self, *args, **kwargs) -> str: # type: ignore\n action = args[0]\n\n if action == \"config\":\n return \"origin\"\n\n return \"\"\n\n monkeypatch.setattr(spack.util.git, \"git\", MockIncludeGit)\n\n def _init_repo(*args, **kwargs):\n # Make sure the directory exists, where assuming called from within\n # the working directory.\n fs.mkdirp(fs.join_path(os.getcwd(), \".git\"))\n\n def _checkout(*args, **kwargs):\n # Make sure the files exist at the clone destination, where assuming\n # called from within the working directory.\n with fs.working_dir(os.getcwd()):\n for p in paths:\n fs.touch(p)\n\n monkeypatch.setattr(spack.util.git, \"init_git_repo\", _init_repo)\n monkeypatch.setattr(spack.util.git, f\"pull_checkout_{key}\", _checkout)\n\n # First successful pass builds the scope\n parent_scope = mock_low_high_config.scopes[\"low\"]\n scopes = include.scopes(parent_scope)\n assert len(scopes) == len(paths)\n\n base_paths = [os.path.basename(p) for p in paths]\n for scope in scopes:\n assert isinstance(scope, spack.config.SingleFileScope)\n assert os.path.basename(scope.path) in base_paths # type: ignore[union-attr]\n assert scope.name.split(\":\")[1] in base_paths\n\n # Second pass uses the scopes previously built.\n # Only need to do this for one of the parameters.\n if key == \"branch\":\n assert include._scopes is not None\n scopes = include.scopes(parent_scope)\n captured = capfd.readouterr()[1]\n assert \"Using existing scopes\" in captured\n\n # A direct clone now returns already cloned destination and debug message.\n # Again only need to run this test once.\n if key == \"tag\":\n assert include._clone(parent_scope) == include.destination\n captured = capfd.readouterr()[1]\n assert \"already cloned\" in captured\n\n\n@pytest.mark.parametrize(\"path\", [\"./config.yaml\", \"/path/to/my/special/package.yaml\"])\ndef test_included_path_local_no_dest(path):\n \"\"\"Confirm that local paths have no cache destination.\"\"\"\n entry = {\"path\": path}\n include = spack.config.included_path(entry)\n destination = include.base_directory(entry[\"path\"])\n assert not destination, f\"Expected local include ({include}) to NOT have a cache destination\"\n\n\ndef test_included_path_url_temp_dest(mock_low_high_config):\n \"\"\"Check that remote (raw) path under different scopes end up with temporary\n cache destinations.\"\"\"\n entry = {\n \"path\": \"https://github.com/path/to/raw/config/config.yaml\",\n \"sha256\": \"26e871804a92cd07bb3d611b31b4156ae93d35b6a6d6e0ef3a67871fcb1d258b\",\n }\n include = spack.config.included_path(entry)\n\n parent_scope = mock_low_high_config.scopes[\"low\"]\n parent_scope.path = \"\"\n pre = f\"Expected temporary cache destination for raw include path ({include}) for \"\n\n for scope in [None, parent_scope]:\n rest = \"parent scope with no path\" if scope else \"no parent scope\"\n destination = include.base_directory(entry[\"path\"], parent_scope=scope)\n dest_dir = str(pathlib.Path(destination).parent)\n temp_dir = tempfile.gettempdir()\n assert dest_dir == temp_dir, pre + rest\n\n\ndef test_included_path_git_temp_dest(mock_low_high_config):\n \"\"\"Check a remote (relative) path with different parent scope options that\n result in a temporary cache destination.\"\"\"\n entry = {\n \"git\": \"https://example.com/linux/configs.git\",\n \"branch\": \"develop\",\n \"paths\": [\"config.yaml\"],\n }\n include = spack.config.included_path(entry)\n parent_scope = mock_low_high_config.scopes[\"low\"]\n parent_scope.path = \"\"\n pre = f\"Expected temporary cache destination for git include path ({include}) for \"\n\n for scope in [None, parent_scope]:\n rest = \"parent scope with no path\" if scope else \"no parent scope\"\n destination = include.base_directory(entry[\"git\"], parent_scope=scope)\n dest_dir = str(pathlib.Path(destination).parent)\n temp_dir = tempfile.gettempdir()\n assert dest_dir == temp_dir, pre + rest\n\n\ndef test_included_path_git_errs(tmp_path: pathlib.Path, mock_low_high_config, monkeypatch):\n monkeypatch.setattr(spack.paths, \"user_cache_path\", str(tmp_path))\n\n paths = [\"concretizer.yaml\"]\n entry = {\n \"git\": \"https://example.com/linux/configs.git\",\n \"branch\": \"develop\",\n \"paths\": paths,\n \"when\": 'platform == \"test\"',\n }\n include = spack.config.included_path(entry)\n parent_scope = mock_low_high_config.scopes[\"low\"]\n\n # fail to initialize the repository\n def _failing_init(*args, **kwargs):\n raise spack.util.executable.ProcessError(\"mock init repo failure\")\n\n monkeypatch.setattr(spack.util.git, \"init_git_repo\", _failing_init)\n\n with pytest.raises(spack.error.ConfigError, match=\"Unable to initialize\"):\n include.scopes(parent_scope)\n\n # fail in git config (so use default remote) *and* git checkout\n def _init_repo(*args, **kwargs):\n fs.mkdirp(fs.join_path(include.destination, \".git\"))\n\n class MockIncludeGit(spack.util.executable.Executable):\n def __init__(self, required: bool):\n pass\n\n def __call__(self, *args, **kwargs) -> str: # type: ignore\n raise spack.util.executable.ProcessError(\"mock git failure\")\n\n monkeypatch.setattr(spack.util.git, \"init_git_repo\", _init_repo)\n monkeypatch.setattr(spack.util.git, \"git\", MockIncludeGit)\n\n with pytest.raises(spack.error.ConfigError, match=\"Unable to check out\"):\n include.scopes(parent_scope)\n\n # set up invalid option failure\n include.branch = \"\" # type: ignore[union-attr]\n with pytest.raises(spack.error.ConfigError, match=\"Missing or unsupported options\"):\n include.scopes(parent_scope)\n\n\ndef test_missing_include_scope_list(mock_missing_dir_include_scopes):\n \"\"\"Tests that an included scope with a non existent file/directory\n is still listed as a scope under spack.config.CONFIG.scopes\"\"\"\n assert \"sub_base\" in list(spack.config.CONFIG.scopes), (\n \"Missing Optional Scope Missing from Config Scopes\"\n )\n\n\ndef test_missing_include_scope_writable_list(mock_missing_dir_include_scopes):\n \"\"\"Tests that missing include scopes are included in writeable config lists\"\"\"\n assert [x for x in spack.config.CONFIG.writable_scopes if x.name == \"sub_base\"]\n\n\ndef test_missing_include_scope_not_readable_list(mock_missing_dir_include_scopes):\n \"\"\"Tests that missing include scopes are not included in existing config lists\"\"\"\n existing_scopes = [x for x in spack.config.CONFIG.existing_scopes if x.name != \"sub_base\"]\n assert len(existing_scopes) == 1\n assert existing_scopes[0].name != \"sub_base\"\n\n\ndef test_missing_include_scope_default_created_as_dir_scope(mock_missing_dir_include_scopes):\n \"\"\"Tests that an optional include with no existing file/directory and no yaml extension\n is created as a directoryscope object\"\"\"\n missing_inc_scope = spack.config.CONFIG.scopes[\"sub_base\"]\n assert isinstance(missing_inc_scope, spack.config.DirectoryConfigScope)\n\n\ndef test_missing_include_scope_yaml_ext_is_file_scope(mock_missing_file_include_scopes):\n \"\"\"Tests that an optional include scope with no existing file/directory and a\n yaml extension is created as a file scope\"\"\"\n missing_inc_scope = spack.config.CONFIG.scopes[\"sub_base\"]\n assert isinstance(missing_inc_scope, spack.config.SingleFileScope)\n\n\ndef test_missing_include_scope_writeable_not_readable(mock_missing_dir_include_scopes):\n \"\"\"Tests that an included scope with a non existent file/directory\n can be written to (and created)\"\"\"\n assert spack.config.CONFIG.scopes[\"sub_base\"].writable, (\n \"Missing Optional Scope should be writable\"\n )\n assert not spack.config.CONFIG.scopes[\"sub_base\"].exists, (\n \"Missing Optional Scope should not exist\"\n )\n\n\ndef test_missing_include_scope_empty_read(mock_missing_dir_include_scopes):\n \"\"\"Tests that an included scope with a non existent file/directory\n returns an empty dict on read and has \"exists\" set to false\"\"\"\n assert spack.config.CONFIG.get(\"config\", scope=\"sub_base\") == {}, (\n \"Missing optional include scope does not return an empty value.\"\n )\n assert not spack.config.CONFIG.scopes[\"sub_base\"].exists, (\n \"Missing optional include should not be created on read\"\n )\n\n\ndef test_missing_include_scope_file_empty_read(mock_missing_file_include_scopes):\n \"\"\"Tests that an include scope with a non existent file returns an empty\n dict and has exists set to false\"\"\"\n assert spack.config.CONFIG.get(\"config\", scope=\"sub_base\") == {}, (\n \"Missing optional include scope does not return an empty value.\"\n )\n assert not spack.config.CONFIG.scopes[\"sub_base\"].exists, (\n \"Missing optional include should not be created on read\"\n )\n\n\ndef test_missing_include_scope_write_directory(mock_missing_dir_include_scopes):\n \"\"\"Tests that an include scope with a non existent directory\n creates said directory and the appropriate section file on write\"\"\"\n install_tree = syaml.syaml_dict({\"install_tree\": {\"root\": \"$spack/tmp/spack\"}})\n spack.config.CONFIG.set(\"config\", install_tree, scope=\"sub_base\")\n assert os.path.exists(spack.config.CONFIG.scopes[\"sub_base\"].path)\n install_root = spack.config.CONFIG.get(\"config:install_tree:root\", scope=\"sub_base\")\n assert install_root == \"$spack/tmp/spack\"\n\n\ndef test_missing_include_scope_write_file(mock_missing_file_include_scopes):\n \"\"\"Tests that an include scope with a non existent file creates said file\n with the appropriate section entry\"\"\"\n install_tree = syaml.syaml_dict({\"install_tree\": {\"root\": \"$spack/tmp/spack\"}})\n spack.config.CONFIG.set(\"config\", install_tree, scope=\"sub_base\")\n assert os.path.exists(spack.config.CONFIG.scopes[\"sub_base\"].path)\n install_root = spack.config.CONFIG.get(\"config:install_tree:root\", scope=\"sub_base\")\n assert install_root == \"$spack/tmp/spack\"\n\n\ndef test_config_scope_empty_write(tmp_path: pathlib.Path):\n \"\"\"Confirm skipping attempt to write non-existent scope section.\"\"\"\n config_scope = spack.config.DirectoryConfigScope(\"test\", str(tmp_path))\n\n assert config_scope.get_section(\"include\") is None\n\n\ndef test_include_bad_parent_scope(tmp_path: pathlib.Path):\n \"\"\"Test parent scope validation.\"\"\"\n path = tmp_path / \"config.yaml\"\n path.touch()\n entry = {\"path\": str(path)}\n include = spack.config.included_path(entry)\n\n # Confirm require a ConfigScope parent\n with pytest.raises(AssertionError, match=\"configuration scope\"):\n _ = include.scopes(\"_builtin\") # type: ignore\n\n # Confirm require a named parent scope\n for name in [\"\", \" \"]:\n parent_scope = spack.config.InternalConfigScope(name, spack.config.CONFIG_DEFAULTS)\n with pytest.raises(AssertionError, match=\"must have a name\"):\n _ = include.scopes(parent_scope)\n\n\ndef test_config_invalid_scope(mock_low_high_config):\n err = \"Must be one of \\\\['low', 'high'\\\\]\" # noqa: W605\n with pytest.raises(ValueError, match=err):\n spack.config.CONFIG.get_config_filename(\"noscope\", \"nosection\")\n" }, { "path": "lib/spack/spack/test/config_values.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport pathlib\n\nimport pytest\n\nimport spack.concretize\nimport spack.store\n\n\n@pytest.mark.parametrize(\"hash_length\", [1, 2, 3, 4, 5, 9])\n@pytest.mark.usefixtures(\"mock_packages\")\ndef test_set_install_hash_length(hash_length, mutable_config, tmp_path: pathlib.Path):\n mutable_config.set(\"config:install_hash_length\", hash_length)\n with spack.store.use_store(str(tmp_path)):\n spec = spack.concretize.concretize_one(\"libelf\")\n prefix = spec.prefix\n hash_str = prefix.rsplit(\"-\")[-1]\n assert len(hash_str) == hash_length\n\n\n@pytest.mark.usefixtures(\"mock_packages\")\ndef test_set_install_hash_length_upper_case(mutable_config, tmp_path: pathlib.Path):\n mutable_config.set(\"config:install_hash_length\", 5)\n with spack.store.use_store(\n str(tmp_path), extra_data={\"projections\": {\"all\": \"{name}-{HASH}\"}}\n ):\n spec = spack.concretize.concretize_one(\"libelf\")\n prefix = spec.prefix\n hash_str = prefix.rsplit(\"-\")[-1]\n assert len(hash_str) == 5\n" }, { "path": "lib/spack/spack/test/conftest.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport base64\nimport collections\nimport datetime\nimport email.message\nimport errno\nimport functools\nimport inspect\nimport io\nimport itertools\nimport json\nimport os\nimport re\nimport shutil\nimport stat\nimport sys\nimport tempfile\nimport textwrap\nimport xml.etree.ElementTree\nfrom pathlib import Path\nfrom typing import Callable, List, Optional, Tuple\n\nimport pytest\n\nimport spack.vendor.archspec.cpu\nimport spack.vendor.archspec.cpu.microarchitecture\nimport spack.vendor.archspec.cpu.schema\n\nimport spack.binary_distribution\nimport spack.bootstrap\nimport spack.caches\nimport spack.compilers.config\nimport spack.compilers.libraries\nimport spack.concretize\nimport spack.config\nimport spack.directives_meta\nimport spack.environment as ev\nimport spack.error\nimport spack.extensions\nimport spack.hash_types\nimport spack.llnl.util.lang\nimport spack.llnl.util.lock\nimport spack.llnl.util.tty as tty\nimport spack.llnl.util.tty.color\nimport spack.modules.common\nimport spack.package_base\nimport spack.paths\nimport spack.platforms\nimport spack.repo\nimport spack.solver.asp\nimport spack.solver.reuse\nimport spack.spec\nimport spack.stage\nimport spack.store\nimport spack.subprocess_context\nimport spack.tengine\nimport spack.util.executable\nimport spack.util.file_cache\nimport spack.util.git\nimport spack.util.gpg\nimport spack.util.naming\nimport spack.util.parallel\nimport spack.util.spack_yaml as syaml\nimport spack.util.url as url_util\nimport spack.util.web\nimport spack.version\nfrom spack.enums import ConfigScopePriority\nfrom spack.fetch_strategy import URLFetchStrategy\nfrom spack.installer import PackageInstaller\nfrom spack.llnl.util.filesystem import (\n copy,\n copy_tree,\n join_path,\n mkdirp,\n remove_linked_tree,\n working_dir,\n)\nfrom spack.main import SpackCommand\nfrom spack.util.pattern import Bunch\nfrom spack.util.remote_file_cache import raw_github_gitlab_url\n\nmirror_cmd = SpackCommand(\"mirror\")\n\n\ndef _recursive_chmod(path: Path, mode: int):\n \"\"\"Recursively change permissions of a directory and all its contents.\"\"\"\n path.chmod(mode)\n for root, dirs, files in os.walk(path):\n for file in files:\n os.chmod(os.path.join(root, file), mode)\n for dir in dirs:\n os.chmod(os.path.join(root, dir), mode)\n\n\n@pytest.fixture(autouse=True)\ndef clear_sys_modules():\n \"\"\"Clear package repos from sys.modules before each test.\"\"\"\n for key in list(sys.modules.keys()):\n if key.startswith(\"spack_repo.\") or key == \"spack_repo\":\n del sys.modules[key]\n yield\n\n\n@pytest.fixture(autouse=True)\ndef check_config_fixture(request):\n if \"config\" in request.fixturenames and \"mutable_config\" in request.fixturenames:\n raise RuntimeError(\"'config' and 'mutable_config' are both requested\")\n\n\ndef ensure_configuration_fixture_run_before(request):\n \"\"\"Ensure that fixture mutating the configuration run before the one where\n the function is called.\n \"\"\"\n if \"config\" in request.fixturenames:\n request.getfixturevalue(\"config\")\n if \"mutable_config\" in request.fixturenames:\n request.getfixturevalue(\"mutable_config\")\n\n\n@pytest.fixture(scope=\"session\")\ndef git():\n \"\"\"Fixture for tests that use git.\"\"\"\n try:\n return spack.util.git.git(required=True)\n except spack.util.executable.CommandNotFoundError:\n pytest.skip(\"requires git to be installed\")\n\n\n#\n# Return list of shas for latest two git commits in local spack repo\n#\n@pytest.fixture(scope=\"session\")\ndef last_two_git_commits(git):\n spack_git_path = spack.paths.prefix\n with working_dir(spack_git_path):\n git_log_out = git(\"log\", \"-n\", \"2\", output=str, error=os.devnull)\n\n regex = re.compile(r\"^commit\\s([^\\s]+$)\", re.MULTILINE)\n yield regex.findall(git_log_out)\n\n\ndef write_file(filename, contents):\n with open(filename, \"w\", encoding=\"utf-8\") as f:\n f.write(contents)\n\n\ncommit_counter = 0\n\n\n@pytest.fixture\ndef override_git_repos_cache_path(tmp_path: Path):\n saved = spack.paths.user_repos_cache_path\n tmp_git_path = tmp_path / \"git-repo-cache-path-for-tests\"\n tmp_git_path.mkdir()\n spack.paths.user_repos_cache_path = str(tmp_git_path)\n yield\n spack.paths.user_repos_cache_path = saved\n\n\n@pytest.fixture\ndef mock_git_version_info(git, tmp_path: Path, override_git_repos_cache_path):\n \"\"\"Create a mock git repo with known structure\n\n The structure of commits in this repo is as follows::\n\n | o fourth 1.x commit (1.2)\n | o third 1.x commit\n | |\n o | fourth main commit (v2.0)\n o | third main commit\n | |\n | o second 1.x commit (v1.1)\n | o first 1.x commit\n | /\n |/\n o second commit (v1.0)\n o first commit\n\n The repo consists of a single file, in which the GitVersion.std_version representation\n of each commit is expressed as a string.\n\n Important attributes of the repo for test coverage are: multiple branches,\n version tags on multiple branches, and version order is not equal to time\n order or topological order.\n \"\"\"\n repo_dir = tmp_path / \"git_version_info_repo\"\n repo_dir.mkdir()\n repo_path = str(repo_dir)\n filename = \"file.txt\"\n\n def commit(message):\n global commit_counter\n git(\n \"commit\",\n \"--no-gpg-sign\",\n \"--date\",\n \"2020-01-%02d 12:0:00 +0300\" % commit_counter,\n \"-am\",\n message,\n )\n commit_counter += 1\n\n with working_dir(repo_path):\n git(\"init\")\n\n git(\"config\", \"user.name\", \"Spack\")\n git(\"config\", \"user.email\", \"spack@spack.io\")\n git(\"checkout\", \"-b\", \"main\")\n\n commits = []\n\n def latest_commit():\n return git(\"rev-list\", \"-n1\", \"HEAD\", output=str, error=str).strip()\n\n # Add two commits on main branch\n # A commit without a previous version counts as \"0\"\n write_file(filename, \"[0]\")\n git(\"add\", filename)\n commit(\"first commit\")\n commits.append(latest_commit())\n\n # Tag second commit as v1.0\n write_file(filename, \"[1, 0]\")\n commit(\"second commit\")\n commits.append(latest_commit())\n git(\"tag\", \"v1.0\")\n\n # Add two commits and a tag on 1.x branch\n git(\"checkout\", \"-b\", \"1.x\")\n write_file(filename, \"[1, 0, 'git', 1]\")\n commit(\"first 1.x commit\")\n commits.append(latest_commit())\n\n write_file(filename, \"[1, 1]\")\n commit(\"second 1.x commit\")\n commits.append(latest_commit())\n git(\"tag\", \"v1.1\")\n\n # Add two commits and a tag on main branch\n git(\"checkout\", \"main\")\n write_file(filename, \"[1, 0, 'git', 1]\")\n commit(\"third main commit\")\n commits.append(latest_commit())\n write_file(filename, \"[2, 0]\")\n commit(\"fourth main commit\")\n commits.append(latest_commit())\n git(\"tag\", \"v2.0\")\n\n # Add two more commits on 1.x branch to ensure we aren't cheating by using time\n git(\"checkout\", \"1.x\")\n write_file(filename, \"[1, 1, 'git', 1]\")\n commit(\"third 1.x commit\")\n commits.append(latest_commit())\n write_file(filename, \"[1, 2]\")\n commit(\"fourth 1.x commit\")\n commits.append(latest_commit())\n git(\"tag\", \"1.2\") # test robust parsing to different syntax, no v\n\n # The commits are ordered with the last commit first in the list\n commits = list(reversed(commits))\n\n # Return the git directory to install, the filename used, and the commits\n yield repo_path, filename, commits\n\n\n@pytest.fixture\ndef mock_git_package_changes(git, tmp_path: Path, override_git_repos_cache_path, monkeypatch):\n \"\"\"Create a mock git repo with known structure of package edits\n\n The structure of commits in this repo is as follows::\n\n o diff-test: add v2.1.7 and v2.1.8 (invalid duplicated checksum)\n |\n o diff-test: add v2.1.6 (from a git ref)\n |\n o diff-test: add v2.1.5 (from source tarball)\n |\n o diff-test: new package (testing multiple added versions)\n\n The repo consists of a single package.py file for DiffTest.\n\n Important attributes of the repo for test coverage are: multiple package\n versions are added with some coming from a tarball and some from git refs.\n \"\"\"\n filename = \"diff_test/package.py\"\n\n repo_path, _ = spack.repo.create_repo(str(tmp_path), namespace=\"myrepo\")\n cache_dir = tmp_path / \"cache\"\n cache_dir.mkdir()\n repo_cache = spack.util.file_cache.FileCache(str(cache_dir))\n\n repo = spack.repo.Repo(repo_path, cache=repo_cache)\n\n def commit(message):\n global commit_counter\n git(\n \"commit\",\n \"--no-gpg-sign\",\n \"--date\",\n \"2020-01-%02d 12:0:00 +0300\" % commit_counter,\n \"-am\",\n message,\n )\n commit_counter += 1\n\n with working_dir(repo.packages_path):\n git(\"init\")\n\n git(\"config\", \"user.name\", \"Spack\")\n git(\"config\", \"user.email\", \"spack@spack.io\")\n\n commits = []\n\n def latest_commit():\n return git(\"rev-list\", \"-n1\", \"HEAD\", output=str, error=str).strip()\n\n os.makedirs(os.path.dirname(filename))\n\n # add diff-test as a new package to the repository\n shutil.copy2(f\"{spack.paths.test_path}/data/conftest/diff-test/package-0.txt\", filename)\n git(\"add\", filename)\n commit(\"diff-test: new package\")\n commits.append(latest_commit())\n\n # add v2.1.5 to diff-test\n shutil.copy2(f\"{spack.paths.test_path}/data/conftest/diff-test/package-1.txt\", filename)\n git(\"add\", filename)\n commit(\"diff-test: add v2.1.5\")\n commits.append(latest_commit())\n\n # add v2.1.6 to diff-test\n shutil.copy2(f\"{spack.paths.test_path}/data/conftest/diff-test/package-2.txt\", filename)\n git(\"add\", filename)\n commit(\"diff-test: add v2.1.6\")\n commits.append(latest_commit())\n\n # add v2.1.7 and v2.1.8 to diff-test\n shutil.copy2(f\"{spack.paths.test_path}/data/conftest/diff-test/package-3.txt\", filename)\n git(\"add\", filename)\n commit(\"diff-test: add v2.1.7 and v2.1.8\")\n commits.append(latest_commit())\n\n # The commits are ordered with the last commit first in the list\n commits = list(reversed(commits))\n\n # Return the git directory to install, the filename used, and the commits\n yield repo, filename, commits\n\n\n@pytest.fixture(autouse=True)\ndef clear_recorded_monkeypatches():\n yield\n spack.subprocess_context.MONKEYPATCHES.clear()\n\n\n@pytest.fixture(scope=\"session\", autouse=True)\ndef record_monkeypatch_setattr():\n import _pytest\n\n saved_setattr = _pytest.monkeypatch.MonkeyPatch.setattr\n\n def record_setattr(cls, target, name, value, *args, **kwargs):\n spack.subprocess_context.MONKEYPATCHES.append((target, name))\n saved_setattr(cls, target, name, value, *args, **kwargs)\n\n _pytest.monkeypatch.MonkeyPatch.setattr = record_setattr\n try:\n yield\n finally:\n _pytest.monkeypatch.MonkeyPatch.setattr = saved_setattr\n\n\ndef _can_access(path, perms):\n return False\n\n\n@pytest.fixture\ndef no_path_access(monkeypatch):\n monkeypatch.setattr(os, \"access\", _can_access)\n\n\n#\n# Disable any active Spack environment BEFORE all tests\n#\n@pytest.fixture(scope=\"session\", autouse=True)\ndef clean_user_environment():\n spack_env_value = os.environ.pop(ev.spack_env_var, None)\n with ev.no_active_environment():\n yield\n if spack_env_value:\n os.environ[ev.spack_env_var] = spack_env_value\n\n\n#\n# Make sure global state of active env does not leak between tests.\n#\n@pytest.fixture(scope=\"function\", autouse=True)\ndef clean_test_environment():\n yield\n ev.deactivate()\n\n\ndef _host():\n \"\"\"Mock archspec host so there is no inconsistency on the Windows platform\n This function cannot be local as it needs to be pickleable\"\"\"\n return spack.vendor.archspec.cpu.Microarchitecture(\"x86_64\", [], \"generic\", [], {}, 0)\n\n\n@pytest.fixture(scope=\"function\")\ndef archspec_host_is_spack_test_host(monkeypatch):\n monkeypatch.setattr(spack.vendor.archspec.cpu, \"host\", _host)\n\n\n# Hooks to add command line options or set other custom behaviors.\n# They must be placed here to be found by pytest. See:\n#\n# https://docs.pytest.org/en/latest/writing_plugins.html\n#\ndef pytest_addoption(parser):\n group = parser.getgroup(\"Spack specific command line options\")\n group.addoption(\n \"--fast\",\n action=\"store_true\",\n default=False,\n help='runs only \"fast\" unit tests, instead of the whole suite',\n )\n\n\ndef pytest_collection_modifyitems(config, items):\n if not config.getoption(\"--fast\"):\n # --fast not given, run all the tests\n return\n\n slow_tests = [\"db\", \"network\", \"maybeslow\"]\n skip_as_slow = pytest.mark.skip(reason=\"skipped slow test [--fast command line option given]\")\n for item in items:\n if any(x in item.keywords for x in slow_tests):\n item.add_marker(skip_as_slow)\n\n\n@pytest.fixture(scope=\"function\")\ndef use_concretization_cache(mock_packages, mutable_config, tmp_path: Path):\n \"\"\"Enables the use of the concretization cache\"\"\"\n conc_cache_dir = tmp_path / \"concretization\"\n conc_cache_dir.mkdir()\n\n # ensure we have an isolated concretization cache while using fixture\n with spack.config.override(\n \"concretizer:concretization_cache\", {\"enable\": True, \"url\": str(conc_cache_dir)}\n ):\n yield conc_cache_dir\n\n\n#\n# These fixtures are applied to all tests\n#\n@pytest.fixture(scope=\"function\", autouse=True)\ndef no_chdir():\n \"\"\"Ensure that no test changes Spack's working directory.\n\n This prevents Spack tests (and therefore Spack commands) from\n changing the working directory and causing other tests to fail\n mysteriously. Tests should use ``working_dir`` or ``py.path``'s\n ``.as_cwd()`` instead of ``os.chdir`` to avoid failing this check.\n\n We assert that the working directory hasn't changed, unless the\n original wd somehow ceased to exist.\n\n \"\"\"\n original_wd = os.getcwd()\n yield\n if os.path.isdir(original_wd):\n assert os.getcwd() == original_wd\n\n\ndef onerror(func, path, error_info):\n # Python on Windows is unable to remove paths without\n # write (IWUSR) permissions (such as those generated by Git on Windows)\n # This method changes file permissions to allow removal by Python\n os.chmod(path, stat.S_IWUSR)\n func(path)\n\n\n@pytest.fixture(scope=\"function\", autouse=True)\ndef mock_stage(tmp_path_factory: pytest.TempPathFactory, monkeypatch, request):\n \"\"\"Establish the temporary build_stage for the mock archive.\"\"\"\n # The approach with this autouse fixture is to set the stage root\n # instead of using spack.config.override() to avoid configuration\n # conflicts with dozens of tests that rely on other configuration\n # fixtures, such as config.\n\n if \"nomockstage\" in request.keywords:\n # Tests can opt-out with @pytest.mark.nomockstage\n yield None\n return\n\n # Set the build stage to the requested path\n new_stage = tmp_path_factory.mktemp(\"mock-stage\")\n\n # Ensure the source directory exists within the new stage path\n source_path = new_stage / spack.stage._source_path_subdir\n source_path.mkdir(parents=True, exist_ok=True)\n\n monkeypatch.setattr(spack.stage, \"_stage_root\", str(new_stage))\n\n yield str(new_stage)\n\n # Clean up the test stage directory\n if new_stage.is_dir():\n shutil.rmtree(new_stage, onerror=onerror)\n\n\n@pytest.fixture(scope=\"session\")\ndef mock_stage_for_database(tmp_path_factory: pytest.TempPathFactory, monkeypatch_session):\n \"\"\"A session-scoped analog of mock_stage, so that the mock_store\n fixture uses its own stage vs. the global stage root for spack.\n \"\"\"\n new_stage = tmp_path_factory.mktemp(\"mock-stage\")\n\n source_path = new_stage / spack.stage._source_path_subdir\n source_path.mkdir(parents=True, exist_ok=True)\n\n monkeypatch_session.setattr(spack.stage, \"_stage_root\", str(new_stage))\n\n yield str(new_stage)\n\n # Clean up the test stage directory\n if new_stage.is_dir():\n shutil.rmtree(new_stage, onerror=onerror)\n\n\n@pytest.fixture(scope=\"session\")\ndef ignore_stage_files():\n \"\"\"Session-scoped helper for check_for_leftover_stage_files.\n\n Used to track which leftover files in the stage have been seen.\n \"\"\"\n # to start with, ignore the .lock file at the stage root.\n return set([\".lock\", spack.stage._source_path_subdir, \"build_cache\"])\n\n\ndef remove_whatever_it_is(path):\n \"\"\"Type-agnostic remove.\"\"\"\n if os.path.isfile(path):\n os.remove(path)\n elif os.path.islink(path):\n remove_linked_tree(path)\n else:\n shutil.rmtree(path, onerror=onerror)\n\n\n@pytest.fixture\ndef working_env():\n saved_env = os.environ.copy()\n yield\n # os.environ = saved_env doesn't work\n # it causes module_parsing::test_module_function to fail\n # when it's run after any test using this fixutre\n os.environ.clear()\n os.environ.update(saved_env)\n\n\n@pytest.fixture(scope=\"function\", autouse=True)\ndef check_for_leftover_stage_files(request, mock_stage, ignore_stage_files):\n \"\"\"\n Ensure that each (mock_stage) test leaves a clean stage when done.\n\n Tests that are expected to dirty the stage can disable the check by\n adding::\n\n @pytest.mark.disable_clean_stage_check\n\n and the associated stage files will be removed.\n \"\"\"\n yield\n\n if mock_stage is None:\n # When tests opt out with @pytest.mark.nomockstage, do not check for left-over files\n return\n\n files_in_stage = set()\n try:\n stage_files = os.listdir(mock_stage)\n files_in_stage = set(stage_files) - ignore_stage_files\n except OSError as err:\n if err.errno == errno.ENOENT or err.errno == errno.EINVAL:\n pass\n else:\n raise\n\n if \"disable_clean_stage_check\" in request.keywords:\n # clean up after tests that are expected to be dirty\n for f in files_in_stage:\n path = os.path.join(mock_stage, f)\n remove_whatever_it_is(path)\n else:\n ignore_stage_files |= files_in_stage\n assert not files_in_stage\n\n\nclass MockCache:\n def store(self, copy_cmd, relative_dest):\n pass\n\n def fetcher(self, target_path, digest, **kwargs):\n return MockCacheFetcher()\n\n\nclass MockCacheFetcher:\n def fetch(self):\n raise spack.error.FetchError(\"Mock cache always fails for tests\")\n\n def __str__(self):\n return \"[mock fetch cache]\"\n\n\n@pytest.fixture(autouse=True)\ndef mock_fetch_cache(monkeypatch):\n \"\"\"Substitutes spack.paths.FETCH_CACHE with a mock object that does nothing\n and raises on fetch.\n \"\"\"\n monkeypatch.setattr(spack.caches, \"FETCH_CACHE\", MockCache())\n\n\n@pytest.fixture()\ndef mock_binary_index(monkeypatch, tmp_path_factory: pytest.TempPathFactory):\n \"\"\"Changes the directory for the binary index and creates binary index for\n every test. Clears its own index when it's done.\n \"\"\"\n tmpdir = tmp_path_factory.mktemp(\"mock_binary_index\")\n index_path = tmpdir / \"binary_index\"\n mock_index = spack.binary_distribution.BinaryCacheIndex(str(index_path))\n monkeypatch.setattr(spack.binary_distribution, \"BINARY_INDEX\", mock_index)\n yield\n\n\n@pytest.fixture(autouse=True)\ndef _skip_if_missing_executables(request, monkeypatch):\n \"\"\"Permits to mark tests with 'require_executables' and skip the\n tests if the executables passed as arguments are not found.\n \"\"\"\n marker = request.node.get_closest_marker(\"requires_executables\")\n if marker:\n required_execs = marker.args\n missing_execs = [x for x in required_execs if spack.util.executable.which(x) is None]\n if missing_execs:\n msg = \"could not find executables: {0}\"\n pytest.skip(msg.format(\", \".join(missing_execs)))\n\n # In case we require a compiler, clear the caches used to speed-up detection\n monkeypatch.setattr(spack.compilers.libraries.DefaultDynamicLinkerFilter, \"_CACHE\", {})\n\n\n@pytest.fixture(scope=\"session\")\ndef test_platform():\n return spack.platforms.Test()\n\n\n@pytest.fixture(autouse=True, scope=\"session\")\ndef _use_test_platform(test_platform):\n # This is the only context manager used at session scope (see note\n # below for more insight) since we want to use the test platform as\n # a default during tests.\n with spack.platforms.use_platform(test_platform):\n yield\n\n\n#\n# Note on context managers used by fixtures\n#\n# Because these context managers modify global state, they should really\n# ONLY be used persistently (i.e., around yield statements) in\n# function-scoped fixtures, OR in autouse session- or module-scoped\n# fixtures.\n#\n# If they're used in regular tests or in module-scoped fixtures that are\n# then injected as function arguments, weird things can happen, because\n# the original state won't be restored until *after* the fixture is\n# destroyed. This makes sense for an autouse fixture, where you know\n# everything in the module/session is going to need the modified\n# behavior, but modifying global state for one function in a way that\n# won't be restored until after the module or session is done essentially\n# leaves garbage behind for other tests.\n#\n# In general, we should module- or session-scope the *STATE* required for\n# these global objects, but we shouldn't module- or session-scope their\n# *USE*, or things can get really confusing.\n#\n\n\n#\n# Test-specific fixtures\n#\n@pytest.fixture(scope=\"session\")\ndef mock_packages_repo():\n yield spack.repo.from_path(spack.paths.mock_packages_path)\n\n\ndef _pkg_install_fn(pkg, spec, prefix):\n # sanity_check_prefix requires something in the install directory\n mkdirp(prefix.bin)\n\n\n@pytest.fixture\ndef mock_pkg_install(monkeypatch):\n monkeypatch.setattr(spack.package_base.PackageBase, \"install\", _pkg_install_fn, raising=False)\n\n\n@pytest.fixture(scope=\"function\")\ndef fake_db_install(tmp_path):\n \"\"\"This fakes \"enough\" of the installation process to make Spack\n think of a spec as being installed as far as the concretizer\n and parser are concerned. It does not run any build phase defined\n in the package, simply acting as though the installation had\n completed successfully.\n\n It allows doing things like\n\n ``spack.concretize.concretize_one(f\"x ^/hash-of-y\")``\n\n after doing something like ``fake_db_install(y)``\n \"\"\"\n with spack.store.use_store(str(tmp_path)) as the_store:\n\n def _install(a_spec):\n the_store.db.add(a_spec)\n\n yield _install\n\n\n@pytest.fixture(scope=\"function\")\ndef mock_packages(mock_packages_repo, mock_pkg_install, request):\n \"\"\"Use the 'builtin_mock' repository instead of 'builtin'\"\"\"\n ensure_configuration_fixture_run_before(request)\n with spack.repo.use_repositories(mock_packages_repo) as mock_repo:\n yield mock_repo\n\n\n@pytest.fixture(scope=\"function\")\ndef mutable_mock_repo(mock_packages_repo, request):\n \"\"\"Function-scoped mock packages, for tests that need to modify them.\"\"\"\n ensure_configuration_fixture_run_before(request)\n mock_repo = spack.repo.from_path(spack.paths.mock_packages_path)\n with spack.repo.use_repositories(mock_repo) as mock_packages_repo:\n yield mock_packages_repo\n\n\nclass RepoBuilder:\n \"\"\"Build a mock repository in a directory\"\"\"\n\n _counter = 0\n\n def __init__(self, root_directory: str) -> None:\n RepoBuilder._counter += 1\n namespace = f\"test_namespace_{RepoBuilder._counter}\"\n repo_root = os.path.join(root_directory, namespace)\n os.makedirs(repo_root, exist_ok=True)\n self.template_dirs = (os.path.join(spack.paths.share_path, \"templates\"),)\n self.root, self.namespace = spack.repo.create_repo(repo_root, namespace)\n self.build_system_name = f\"test_build_system_{self.namespace}\"\n self._add_build_system()\n\n def add_package(\n self,\n name: str,\n dependencies: Optional[List[Tuple[str, Optional[str], Optional[str]]]] = None,\n ) -> None:\n \"\"\"Create a mock package in the repository, using a Jinja2 template.\n\n Args:\n name: name of the new package\n dependencies: list of (\"dep_spec\", \"dep_type\", \"condition\") tuples.\n Both \"dep_type\" and \"condition\" can default to ``None`` in which case\n ``spack.dependency.default_deptype`` and ``spack.spec.Spec()`` are used.\n \"\"\"\n dependencies = dependencies or []\n context = {\n \"cls_name\": spack.util.naming.pkg_name_to_class_name(name),\n \"dependencies\": dependencies,\n }\n template = spack.tengine.make_environment_from_dirs(self.template_dirs).get_template(\n \"mock-repository/package.pyt\"\n )\n package_py = self._recipe_filename(name)\n os.makedirs(os.path.dirname(package_py), exist_ok=True)\n with open(package_py, \"w\", encoding=\"utf-8\") as f:\n f.write(template.render(context))\n\n def remove(self, name: str) -> None:\n package_py = self._recipe_filename(name)\n shutil.rmtree(os.path.dirname(package_py))\n\n def _add_build_system(self) -> None:\n \"\"\"Add spack_repo..build_systems.test_build_system with\n build_system=test_build_system_.\"\"\"\n template = spack.tengine.make_environment_from_dirs(self.template_dirs).get_template(\n \"mock-repository/build_system.pyt\"\n )\n text = template.render({\"build_system_name\": self.build_system_name})\n build_system_py = os.path.join(self.root, \"build_systems\", \"test_build_system.py\")\n os.makedirs(os.path.dirname(build_system_py), exist_ok=True)\n with open(build_system_py, \"w\", encoding=\"utf-8\") as f:\n f.write(text)\n\n def _recipe_filename(self, name: str) -> str:\n return os.path.join(\n self.root,\n \"packages\",\n spack.util.naming.pkg_name_to_pkg_dir(name, package_api=(2, 0)),\n \"package.py\",\n )\n\n\n@pytest.fixture\ndef repo_builder(tmp_path: Path):\n return RepoBuilder(str(tmp_path))\n\n\n@pytest.fixture()\ndef mock_custom_repository(tmp_path: Path, mutable_mock_repo):\n \"\"\"Create a custom repository with a single package \"c\" and return its path.\"\"\"\n builder = RepoBuilder(str(tmp_path))\n builder.add_package(\"pkg-c\")\n return builder.root\n\n\n@pytest.fixture(scope=\"session\")\ndef linux_os():\n \"\"\"Returns a named tuple with attributes 'name' and 'version'\n representing the OS.\n \"\"\"\n platform = spack.platforms.host()\n name, version = \"debian\", \"6\"\n if platform.name == \"linux\":\n current_os = platform.default_operating_system()\n name, version = current_os.name, current_os.version\n LinuxOS = collections.namedtuple(\"LinuxOS\", [\"name\", \"version\"])\n return LinuxOS(name=name, version=version)\n\n\n@pytest.fixture\ndef ensure_debug(monkeypatch):\n current_debug_level = tty.debug_level()\n tty.set_debug(1)\n\n yield\n\n tty.set_debug(current_debug_level)\n\n\n@pytest.fixture\ndef default_config():\n \"\"\"Isolates the default configuration from the user configs.\n\n This ensures we can test the real default configuration without having\n tests fail when the user overrides the defaults that we test against.\"\"\"\n defaults_path = os.path.join(spack.paths.etc_path, \"defaults\")\n if sys.platform == \"win32\":\n defaults_path = os.path.join(defaults_path, \"windows\")\n with spack.config.use_configuration(defaults_path) as defaults_config:\n yield defaults_config\n\n\n@pytest.fixture(scope=\"session\")\ndef mock_uarch_json(tmp_path_factory: pytest.TempPathFactory):\n \"\"\"Mock microarchitectures.json with test architecture descriptions.\"\"\"\n tmpdir = tmp_path_factory.mktemp(\"microarchitectures\")\n\n uarch_json_source = (\n Path(spack.paths.test_path) / \"data\" / \"microarchitectures\" / \"microarchitectures.json\"\n )\n uarch_json_dest = tmpdir / \"microarchitectures.json\"\n shutil.copy2(uarch_json_source, uarch_json_dest)\n yield str(uarch_json_dest)\n\n\n@pytest.fixture(scope=\"session\")\ndef mock_uarch_configuration(mock_uarch_json):\n \"\"\"Create mock dictionaries for the spack.vendor.archspec.cpu.\"\"\"\n\n def load_json():\n with open(mock_uarch_json, encoding=\"utf-8\") as f:\n return json.load(f)\n\n targets_json = load_json()\n targets = spack.vendor.archspec.cpu.microarchitecture._known_microarchitectures()\n\n yield targets_json, targets\n\n\n@pytest.fixture(scope=\"function\")\ndef mock_targets(mock_uarch_configuration, monkeypatch):\n \"\"\"Use this fixture to enable mock uarch targets for testing.\"\"\"\n targets_json, targets = mock_uarch_configuration\n monkeypatch.setattr(spack.vendor.archspec.cpu.schema, \"TARGETS_JSON\", targets_json)\n monkeypatch.setattr(spack.vendor.archspec.cpu.microarchitecture, \"TARGETS\", targets)\n\n\n@pytest.fixture(scope=\"session\")\ndef configuration_dir(tmp_path_factory: pytest.TempPathFactory, linux_os):\n \"\"\"Copies mock configuration files in a temporary directory. Returns the\n directory path.\n \"\"\"\n tmp_path = tmp_path_factory.mktemp(\"configurations\")\n install_tree_root = tmp_path_factory.mktemp(\"opt\")\n modules_root = tmp_path_factory.mktemp(\"share\")\n tcl_root = modules_root / \"modules\"\n tcl_root.mkdir()\n lmod_root = modules_root / \"lmod\"\n lmod_root.mkdir()\n\n # /data/config has mock config yaml files in it\n # copy these to the site config.\n test_config = Path(spack.paths.test_path) / \"data\" / \"config\"\n shutil.copytree(test_config, tmp_path / \"site\")\n\n # Create temporary 'defaults', 'site' and 'user' folders\n (tmp_path / \"user\").mkdir()\n\n # Fill out config.yaml, compilers.yaml and modules.yaml templates.\n locks = sys.platform != \"win32\"\n config = tmp_path / \"site\" / \"config.yaml\"\n config_template = test_config / \"config.yaml\"\n config.write_text(config_template.read_text().format(install_tree_root, locks))\n\n target = str(spack.vendor.archspec.cpu.host().family)\n compilers = tmp_path / \"site\" / \"packages.yaml\"\n compilers_template = test_config / \"packages.yaml\"\n compilers.write_text(compilers_template.read_text().format(linux_os=linux_os, target=target))\n\n modules = tmp_path / \"site\" / \"modules.yaml\"\n modules_template = test_config / \"modules.yaml\"\n modules.write_text(modules_template.read_text().format(tcl_root, lmod_root))\n\n for scope in (\"spack\", \"user\", \"site\", \"system\"):\n scope_path = tmp_path / scope\n scope_path.mkdir(exist_ok=True)\n\n include = tmp_path / \"spack\" / \"include.yaml\"\n # Need to use relative include paths here so it works for mutable_config fixture too\n with include.open(\"w\", encoding=\"utf-8\") as f:\n f.write(\n textwrap.dedent(\n \"\"\"\n include:\n # user configuration scope\n - name: \"user\"\n path_override_env_var: SPACK_USER_CONFIG_PATH\n path: ../user\n optional: true\n prefer_modify: true\n when: '\"SPACK_DISABLE_LOCAL_CONFIG\" not in env'\n\n # site configuration scope\n - name: \"site\"\n path: ../site\n optional: true\n\n # system configuration scope\n - name: \"system\"\n path_override_env_var: SPACK_SYSTEM_CONFIG_PATH\n path: ../system\n optional: true\n when: '\"SPACK_DISABLE_LOCAL_CONFIG\" not in env'\n \"\"\"\n )\n )\n yield tmp_path\n\n\ndef _create_mock_configuration_scopes(configuration_dir):\n \"\"\"Create the configuration scopes used in `config` and `mutable_config`.\"\"\"\n return [\n (\n ConfigScopePriority.DEFAULTS,\n spack.config.InternalConfigScope(\"_builtin\", spack.config.CONFIG_DEFAULTS),\n ),\n (\n ConfigScopePriority.CONFIG_FILES,\n spack.config.DirectoryConfigScope(\"spack\", str(configuration_dir / \"spack\")),\n ),\n (ConfigScopePriority.COMMAND_LINE, spack.config.InternalConfigScope(\"command_line\")),\n ]\n\n\n@pytest.fixture(scope=\"session\")\ndef mock_configuration_scopes(configuration_dir):\n \"\"\"Create a persistent Configuration object from the configuration_dir.\"\"\"\n yield _create_mock_configuration_scopes(configuration_dir)\n\n\n@pytest.fixture(scope=\"function\")\ndef config(mock_configuration_scopes):\n \"\"\"This fixture activates/deactivates the mock configuration.\"\"\"\n with spack.config.use_configuration(*mock_configuration_scopes) as config:\n yield config\n\n\n@pytest.fixture(scope=\"function\")\ndef mutable_config(tmp_path_factory: pytest.TempPathFactory, configuration_dir):\n \"\"\"Like config, but tests can modify the configuration.\"\"\"\n mutable_dir = tmp_path_factory.mktemp(\"mutable_config\") / \"tmp\"\n shutil.copytree(configuration_dir, mutable_dir)\n\n scopes = _create_mock_configuration_scopes(mutable_dir)\n with spack.config.use_configuration(*scopes) as cfg:\n yield cfg\n\n\n@pytest.fixture(scope=\"function\")\ndef mutable_empty_config(tmp_path_factory: pytest.TempPathFactory, configuration_dir):\n \"\"\"Empty configuration that can be modified by the tests.\"\"\"\n mutable_dir = tmp_path_factory.mktemp(\"mutable_config\") / \"tmp\"\n scopes = [\n spack.config.DirectoryConfigScope(name, str(mutable_dir / name))\n for name in [\"site\", \"system\", \"user\"]\n ]\n\n with spack.config.use_configuration(*scopes) as cfg:\n yield cfg\n\n\n# From https://github.com/pytest-dev/pytest/issues/363#issuecomment-1335631998\n# Current suggested implementation from issue compatible with pytest >= 6.2\n# this may be subject to change as new versions of Pytest are released\n# and update the suggested solution\n@pytest.fixture(scope=\"session\")\ndef monkeypatch_session():\n with pytest.MonkeyPatch.context() as monkeypatch:\n yield monkeypatch\n\n\n@pytest.fixture(autouse=True)\ndef mock_wsdk_externals(monkeypatch):\n \"\"\"Skip check for required external packages on Windows during testing.\"\"\"\n monkeypatch.setattr(spack.bootstrap, \"ensure_winsdk_external_or_raise\", _return_none)\n\n\n@pytest.fixture(scope=\"function\")\ndef concretize_scope(mutable_config, tmp_path: Path):\n \"\"\"Adds a scope for concretization preferences\"\"\"\n concretize_dir = tmp_path / \"concretize\"\n concretize_dir.mkdir()\n with spack.config.override(\n spack.config.DirectoryConfigScope(\"concretize\", str(concretize_dir))\n ):\n yield str(concretize_dir)\n\n spack.repo.PATH._provider_index = None\n\n\n@pytest.fixture\ndef no_packages_yaml(mutable_config):\n \"\"\"Creates a temporary configuration without compilers.yaml\"\"\"\n for local_config in mutable_config.scopes.values():\n if not isinstance(local_config, spack.config.DirectoryConfigScope):\n continue\n compilers_yaml = local_config.get_section_filename(\"packages\")\n if os.path.exists(compilers_yaml):\n os.remove(compilers_yaml)\n mutable_config.clear_caches()\n return mutable_config\n\n\n@pytest.fixture()\ndef mock_low_high_config(tmp_path: Path):\n \"\"\"Mocks two configuration scopes: 'low' and 'high'.\"\"\"\n scopes = [\n spack.config.DirectoryConfigScope(name, str(tmp_path / name)) for name in [\"low\", \"high\"]\n ]\n\n with spack.config.use_configuration(*scopes) as config:\n yield config\n\n\ndef create_config_scope(path: Path, name: str) -> spack.config.DirectoryConfigScope:\n \"\"\"helper for creating config scopes with included file/directory scopes\n that do not have existing representation on the filesystem\"\"\"\n base_scope_dir = path / \"base\"\n config_data = syaml.syaml_dict(\n {\n \"include\": [\n {\n \"name\": \"sub_base\",\n \"path\": str(path / name),\n \"optional\": True,\n \"prefer_modify\": True,\n }\n ]\n }\n )\n base_scope_dir.mkdir()\n with open(str(base_scope_dir / \"include.yaml\"), \"w+\", encoding=\"utf-8\") as f:\n syaml.dump_config(config_data, stream=f, default_flow_style=False)\n scope = spack.config.DirectoryConfigScope(\"base\", str(base_scope_dir))\n return scope\n\n\n@pytest.fixture()\ndef mock_missing_dir_include_scopes(tmp_path: Path):\n \"\"\"Mocks a config scope containing optional directory scope\n includes that do not have representation on the filesystem\"\"\"\n scope = create_config_scope(tmp_path, \"sub\")\n\n with spack.config.use_configuration(scope) as config:\n yield config\n\n\n@pytest.fixture\ndef mock_missing_file_include_scopes(tmp_path: Path):\n \"\"\"Mocks a config scope containing optional file scope\n includes that do not have representation on the filesystem\"\"\"\n scope = create_config_scope(tmp_path, \"sub.yaml\")\n\n with spack.config.use_configuration(scope) as config:\n yield config\n\n\ndef _populate(mock_db):\n r\"\"\"Populate a mock database with packages.\n\n Here is what the mock DB looks like (explicit roots at top):\n\n o mpileaks o mpileaks' o mpileaks'' o externaltest o trivial-smoke-test\n |\\ |\\ |\\ |\n | o callpath | o callpath' | o callpath'' o externaltool\n |/| |/| |/| |\n o | mpich o | mpich2 o | zmpi o externalvirtual\n | | o | fake\n | | |\n | |______________/\n | .____________/\n |/\n o dyninst\n |\\\n | o libdwarf\n |/\n o libelf\n \"\"\"\n\n def _install(spec):\n s = spack.concretize.concretize_one(spec)\n PackageInstaller([s.package], fake=True, explicit=True).install()\n\n _install(\"mpileaks ^mpich\")\n _install(\"mpileaks ^mpich2\")\n _install(\"mpileaks ^zmpi\")\n _install(\"externaltest ^externalvirtual\")\n _install(\"trivial-smoke-test\")\n\n\n@pytest.fixture(scope=\"session\")\ndef _store_dir_and_cache(tmp_path_factory: pytest.TempPathFactory):\n \"\"\"Returns the directory where to build the mock database and\n where to cache it.\n \"\"\"\n store = tmp_path_factory.mktemp(\"mock_store\")\n cache = tmp_path_factory.mktemp(\"mock_store_cache\")\n return store, cache\n\n\n@pytest.fixture(scope=\"session\")\ndef mock_store(\n tmp_path_factory: pytest.TempPathFactory,\n mock_packages_repo,\n mock_configuration_scopes,\n _store_dir_and_cache: Tuple[Path, Path],\n mock_stage_for_database,\n):\n \"\"\"Creates a read-only mock database with some packages installed note\n that the ref count for dyninst here will be 3, as it's recycled\n across each install.\n\n This does not actually activate the store for use by Spack -- see the\n ``database`` fixture for that.\n\n \"\"\"\n store_path, store_cache = _store_dir_and_cache\n _mock_wsdk_externals = spack.bootstrap.ensure_winsdk_external_or_raise\n\n # Make the DB filesystem read-only to ensure constructors don't modify anything in it.\n # We want Spack to be able to point to a DB on a read-only filesystem easily.\n _recursive_chmod(store_path, 0o555)\n\n # If the cache does not exist populate the store and create it\n if not os.path.exists(str(store_cache / \".spack-db\")):\n with spack.config.use_configuration(*mock_configuration_scopes):\n with spack.store.use_store(str(store_path)) as store:\n with spack.repo.use_repositories(mock_packages_repo):\n # make the DB filesystem writable only while we populate it\n _recursive_chmod(store_path, 0o755)\n try:\n spack.bootstrap.ensure_winsdk_external_or_raise = _return_none\n _populate(store.db)\n finally:\n spack.bootstrap.ensure_winsdk_external_or_raise = _mock_wsdk_externals\n _recursive_chmod(store_path, 0o555)\n\n _recursive_chmod(store_cache, 0o755)\n copy_tree(str(store_path), str(store_cache))\n _recursive_chmod(store_cache, 0o555)\n\n yield store_path\n\n\n@pytest.fixture(scope=\"function\")\ndef database(mock_store, mock_packages, config):\n \"\"\"This activates the mock store, packages, AND config.\"\"\"\n with spack.store.use_store(str(mock_store)) as store:\n yield store.db\n # Force reading the database again between tests\n store.db.last_seen_verifier = \"\"\n\n\n@pytest.fixture(scope=\"function\")\ndef database_mutable_config(mock_store, mock_packages, mutable_config, monkeypatch):\n \"\"\"This activates the mock store, packages, AND config.\"\"\"\n with spack.store.use_store(str(mock_store)) as store:\n yield store.db\n store.db.last_seen_verifier = \"\"\n\n\n@pytest.fixture(scope=\"function\")\ndef mutable_database(database_mutable_config, _store_dir_and_cache: Tuple[Path, Path]):\n \"\"\"Writeable version of the fixture, restored to its initial state\n after each test.\n \"\"\"\n # Make the database writeable, as we are going to modify it\n store_path, store_cache = _store_dir_and_cache\n _recursive_chmod(store_path, 0o755)\n\n yield database_mutable_config\n\n # Restore the initial state by copying the content of the cache back into\n # the store and making the database read-only\n shutil.rmtree(store_path)\n copy_tree(str(store_cache), str(store_path))\n _recursive_chmod(store_path, 0o555)\n\n\n@pytest.fixture()\ndef dirs_with_libfiles(tmp_path_factory: pytest.TempPathFactory):\n lib_to_libfiles = {\n \"libstdc++\": [\"libstdc++.so\", \"libstdc++.tbd\"],\n \"libgfortran\": [\"libgfortran.a\", \"libgfortran.dylib\"],\n \"libirc\": [\"libirc.a\", \"libirc.so\"],\n }\n\n root = tmp_path_factory.mktemp(\"root\")\n lib_to_dirs = {}\n i = 0\n for lib, libfiles in lib_to_libfiles.items():\n dirs = []\n for libfile in libfiles:\n lib_dir = root / str(i)\n lib_dir.mkdir()\n (lib_dir / libfile).touch()\n dirs.append(str(lib_dir))\n i += 1\n lib_to_dirs[lib] = dirs\n\n all_dirs = list(itertools.chain.from_iterable(lib_to_dirs.values()))\n\n yield lib_to_dirs, all_dirs\n\n\ndef _return_none(*args):\n return None\n\n\n@pytest.fixture(autouse=True)\ndef disable_compiler_output_cache(monkeypatch):\n monkeypatch.setattr(\n spack.compilers.libraries, \"COMPILER_CACHE\", spack.compilers.libraries.CompilerCache()\n )\n\n\n@pytest.fixture(scope=\"function\")\ndef install_mockery(temporary_store: spack.store.Store, mutable_config, mock_packages):\n \"\"\"Hooks a fake install directory, DB, and stage directory into Spack.\"\"\"\n # We use a fake package, so temporarily disable checksumming\n with spack.config.override(\"config:checksum\", False):\n yield\n\n # Wipe out any cached prefix failure locks (associated with the session-scoped mock archive)\n temporary_store.failure_tracker.clear_all()\n\n\n@pytest.fixture(scope=\"function\")\ndef temporary_mirror(mutable_config, tmp_path_factory):\n mirror_dir = tmp_path_factory.mktemp(\"mirror\")\n mirror_cmd(\"add\", \"test-mirror-func\", mirror_dir.as_uri())\n yield str(mirror_dir)\n\n\n@pytest.fixture(scope=\"function\")\ndef temporary_store(tmp_path: Path, request):\n \"\"\"Hooks a temporary empty store for the test function.\"\"\"\n ensure_configuration_fixture_run_before(request)\n temporary_store_path = tmp_path / \"opt\"\n with spack.store.use_store(str(temporary_store_path)) as s:\n yield s\n if temporary_store_path.exists():\n shutil.rmtree(temporary_store_path)\n\n\n@pytest.fixture()\ndef mock_fetch(mock_archive, monkeypatch):\n \"\"\"Fake the URL for a package so it downloads from a file.\"\"\"\n monkeypatch.setattr(\n spack.package_base.PackageBase, \"fetcher\", URLFetchStrategy(url=mock_archive.url)\n )\n\n\nclass MockResourceFetcherGenerator:\n def __init__(self, url):\n self.url = url\n\n def _generate_fetchers(self, *args, **kwargs):\n return [URLFetchStrategy(url=self.url)]\n\n\n@pytest.fixture()\ndef mock_resource_fetch(mock_archive, monkeypatch):\n \"\"\"Fake fetcher generator that works with resource stages to redirect to a file.\"\"\"\n mfg = MockResourceFetcherGenerator(mock_archive.url)\n monkeypatch.setattr(spack.stage.ResourceStage, \"_generate_fetchers\", mfg._generate_fetchers)\n\n\nclass MockLayout:\n def __init__(self, root):\n self.root = root\n\n def path_for_spec(self, spec):\n return os.path.sep.join([self.root, spec.name + \"-\" + spec.dag_hash()])\n\n def ensure_installed(self, spec):\n pass\n\n\n@pytest.fixture()\ndef gen_mock_layout(tmp_path: Path):\n # Generate a MockLayout in a temporary directory. In general the prefixes\n # specified by MockLayout should never be written to, but this ensures\n # that even if they are, that it causes no harm\n def create_layout(root):\n subroot = tmp_path / root\n subroot.mkdir(parents=True, exist_ok=True)\n return MockLayout(str(subroot))\n\n yield create_layout\n\n\nclass MockConfig:\n def __init__(self, configuration, writer_key):\n self._configuration = configuration\n self.writer_key = writer_key\n\n def configuration(self, module_set_name):\n return self._configuration\n\n def writer_configuration(self, module_set_name):\n return self.configuration(module_set_name)[self.writer_key]\n\n\nclass ConfigUpdate:\n def __init__(self, root_for_conf, writer_mod, writer_key, monkeypatch):\n self.root_for_conf = root_for_conf\n self.writer_mod = writer_mod\n self.writer_key = writer_key\n self.monkeypatch = monkeypatch\n\n def __call__(self, filename):\n file = os.path.join(self.root_for_conf, filename + \".yaml\")\n with open(file, encoding=\"utf-8\") as f:\n config_settings = syaml.load_config(f)\n spack.config.set(\"modules:default\", config_settings)\n mock_config = MockConfig(config_settings, self.writer_key)\n\n self.monkeypatch.setattr(spack.modules.common, \"configuration\", mock_config.configuration)\n self.monkeypatch.setattr(\n self.writer_mod, \"configuration\", mock_config.writer_configuration\n )\n self.monkeypatch.setattr(self.writer_mod, \"configuration_registry\", {})\n\n\n@pytest.fixture()\ndef module_configuration(monkeypatch, request, mutable_config):\n \"\"\"Reads the module configuration file from the mock ones prepared\n for tests and monkeypatches the right classes to hook it in.\n \"\"\"\n # Class of the module file writer\n writer_cls = getattr(request.module, \"writer_cls\")\n # Module where the module file writer is defined\n writer_mod = inspect.getmodule(writer_cls)\n # Key for specific settings relative to this module type\n writer_key = str(writer_mod.__name__).split(\".\")[-1]\n # Root folder for configuration\n root_for_conf = os.path.join(spack.paths.test_path, \"data\", \"modules\", writer_key)\n\n # ConfigUpdate, when called, will modify configuration, so we need to use\n # the mutable_config fixture\n return ConfigUpdate(root_for_conf, writer_mod, writer_key, monkeypatch)\n\n\n@pytest.fixture()\ndef mock_gnupghome(monkeypatch):\n # GNU PGP can't handle paths longer than 108 characters (wtf!@#$) so we\n # have to make our own tmp_path with a shorter name than pytest's.\n # This comes up because tmp paths on macOS are already long-ish, and\n # pytest makes them longer.\n try:\n spack.util.gpg.init()\n except spack.util.gpg.SpackGPGError:\n if not spack.util.gpg.GPG:\n pytest.skip(\"This test requires gpg\")\n\n short_name_tmpdir = tempfile.mkdtemp()\n with spack.util.gpg.gnupghome_override(short_name_tmpdir):\n yield short_name_tmpdir\n\n # clean up, since we are doing this manually\n # Ignore errors cause we seem to be hitting a bug similar to\n # https://bugs.python.org/issue29699 in CI (FileNotFoundError: [Errno 2] No such\n # file or directory: 'S.gpg-agent.extra').\n shutil.rmtree(short_name_tmpdir, ignore_errors=True)\n\n\n##########\n# Fake archives and repositories\n##########\n\n\n@pytest.fixture(scope=\"session\", params=[(\".tar.gz\", \"z\")])\ndef mock_archive(request, tmp_path_factory: pytest.TempPathFactory):\n \"\"\"Creates a very simple archive directory with a configure script and a\n makefile that installs to a prefix. Tars it up into an archive.\n \"\"\"\n try:\n tar = spack.util.executable.which(\"tar\", required=True)\n except spack.util.executable.CommandNotFoundError:\n pytest.skip(\"requires tar to be installed\")\n\n tmpdir = tmp_path_factory.mktemp(\"mock-archive-dir\")\n source_dir = tmpdir / spack.stage._source_path_subdir\n source_dir.mkdir()\n repodir = source_dir\n\n # Create the configure script\n configure_path = str(source_dir / \"configure\")\n with open(configure_path, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"#!/bin/sh\\n\"\n \"prefix=$(echo $1 | sed 's/--prefix=//')\\n\"\n \"cat > Makefile < bunch. Each bunch includes; all the args\n # that must be specified as part of a version() declaration (used to\n # manufacture a version for the 'git-test' package); the associated\n # revision for the version; a file associated with (and particular to)\n # that revision/branch.\n checks = {\n \"default\": Bunch(revision=default_branch, file=r0_file, args={\"git\": url}),\n \"branch\": Bunch(revision=branch, file=branch_file, args={\"git\": url, \"branch\": branch}),\n \"tag-branch\": Bunch(\n revision=tag_branch, file=tag_file, args={\"git\": url, \"branch\": tag_branch}\n ),\n \"tag\": Bunch(revision=tag, file=tag_file, args={\"git\": url, \"tag\": tag}),\n \"commit\": Bunch(\n revision=r1, file=r1_file, args={\"git\": url, \"branch\": branch, \"commit\": r1}\n ),\n \"annotated-tag\": Bunch(revision=a_tag, file=r2_file, args={\"git\": url, \"tag\": a_tag}),\n # In this case, the version() args do not include a 'git' key:\n # this is the norm for packages, so this tests how the fetching logic\n # would most-commonly assemble a Git fetcher\n \"default-no-per-version-git\": Bunch(\n revision=default_branch, file=r0_file, args={\"branch\": default_branch}\n ),\n \"many-directories\": Bunch(\n revision=multiple_directories_branch,\n file=dir_files[0],\n args={\"git\": url, \"branch\": multiple_directories_branch},\n ),\n }\n\n t = Bunch(\n checks=checks,\n url=url,\n hash=rev_hash,\n path=str(repodir),\n git_exe=git,\n unversioned_commit=r2,\n )\n yield t\n\n\n@pytest.fixture(scope=\"function\")\ndef mock_git_test_package(mock_git_repository, mutable_mock_repo, monkeypatch):\n # install a fake git version in the package class\n pkg_class = spack.repo.PATH.get_pkg_class(\"git-test\")\n monkeypatch.delattr(pkg_class, \"git\")\n monkeypatch.setitem(pkg_class.versions, spack.version.Version(\"git\"), mock_git_repository.url)\n return pkg_class\n\n\n@pytest.fixture(scope=\"session\")\ndef mock_hg_repository(tmp_path_factory: pytest.TempPathFactory):\n \"\"\"Creates a very simple hg repository with two commits.\"\"\"\n try:\n hg = spack.util.executable.which(\"hg\", required=True)\n except spack.util.executable.CommandNotFoundError:\n pytest.skip(\"requires mercurial to be installed\")\n\n tmpdir = tmp_path_factory.mktemp(\"mock-hg-repo-dir\")\n source_dir = tmpdir / spack.stage._source_path_subdir\n source_dir.mkdir()\n repodir = source_dir\n\n get_rev = lambda: hg(\"id\", \"-i\", output=str).strip()\n\n # Initialize the repository\n with working_dir(str(repodir)):\n url = url_util.path_to_file_url(str(repodir))\n hg(\"init\")\n\n # Commit file r0\n r0_file = \"r0_file\"\n (repodir / r0_file).touch()\n hg(\"add\", r0_file)\n hg(\"commit\", \"-m\", \"revision 0\", \"-u\", \"test\")\n r0 = get_rev()\n\n # Commit file r1\n r1_file = \"r1_file\"\n (repodir / r1_file).touch()\n hg(\"add\", r1_file)\n hg(\"commit\", \"-m\", \"revision 1\", \"-u\", \"test\")\n r1 = get_rev()\n\n checks = {\n \"default\": Bunch(revision=r1, file=r1_file, args={\"hg\": str(repodir)}),\n \"rev0\": Bunch(revision=r0, file=r0_file, args={\"hg\": str(repodir), \"revision\": r0}),\n }\n t = Bunch(checks=checks, url=url, hash=get_rev, path=str(repodir))\n yield t\n\n\n@pytest.fixture(scope=\"session\")\ndef mock_svn_repository(tmp_path_factory: pytest.TempPathFactory):\n \"\"\"Creates a very simple svn repository with two commits.\"\"\"\n try:\n svn = spack.util.executable.which(\"svn\", required=True)\n svnadmin = spack.util.executable.which(\"svnadmin\", required=True)\n except spack.util.executable.CommandNotFoundError:\n pytest.skip(\"requires svn to be installed\")\n\n tmpdir = tmp_path_factory.mktemp(\"mock-svn-stage\")\n source_dir = tmpdir / spack.stage._source_path_subdir\n source_dir.mkdir()\n repodir = source_dir\n url = url_util.path_to_file_url(str(repodir))\n\n # Initialize the repository\n with working_dir(str(repodir)):\n # NOTE: Adding --pre-1.5-compatible works for NERSC\n # Unknown if this is also an issue at other sites.\n svnadmin(\"create\", \"--pre-1.5-compatible\", str(repodir))\n\n # Import a structure (first commit)\n r0_file = \"r0_file\"\n tmp_path = tmpdir / \"tmp-path\"\n tmp_path.mkdir()\n (tmp_path / r0_file).touch()\n svn(\"import\", str(tmp_path), url, \"-m\", \"Initial import r0\")\n shutil.rmtree(tmp_path, onerror=onerror)\n\n # Second commit\n r1_file = \"r1_file\"\n svn(\"checkout\", url, str(tmp_path))\n (tmp_path / r1_file).touch()\n\n with working_dir(str(tmp_path)):\n svn(\"add\", str(tmp_path / r1_file))\n svn(\"ci\", \"-m\", \"second revision r1\")\n\n shutil.rmtree(tmp_path, onerror=onerror)\n r0 = \"1\"\n r1 = \"2\"\n\n checks = {\n \"default\": Bunch(revision=r1, file=r1_file, args={\"svn\": url}),\n \"rev0\": Bunch(revision=r0, file=r0_file, args={\"svn\": url, \"revision\": r0}),\n }\n\n def get_rev():\n output = svn(\"info\", \"--xml\", output=str)\n info = xml.etree.ElementTree.fromstring(output)\n return info.find(\"entry/commit\").get(\"revision\")\n\n t = Bunch(checks=checks, url=url, hash=get_rev, path=str(repodir))\n yield t\n\n\n@pytest.fixture(scope=\"function\")\ndef mutable_mock_env_path(tmp_path: Path, mutable_config, monkeypatch):\n \"\"\"Fixture for mocking the internal spack environments directory.\"\"\"\n mock_path = tmp_path / \"mock-env-path\"\n mutable_config.set(\"config:environments_root\", str(mock_path))\n monkeypatch.setattr(ev.environment, \"default_env_path\", str(mock_path))\n return mock_path\n\n\n@pytest.fixture()\ndef installation_dir_with_headers(tmp_path_factory: pytest.TempPathFactory):\n \"\"\"Mock installation tree with a few headers placed in different\n subdirectories. Shouldn't be modified by tests as it is session\n scoped.\n \"\"\"\n root = tmp_path_factory.mktemp(\"prefix\")\n\n # Create a few header files:\n #\n # \n # |-- include\n # | |--boost\n # | | |-- ex3.h\n # | |-- ex3.h\n # |-- path\n # |-- to\n # |-- ex1.h\n # |-- subdir\n # |-- ex2.h\n #\n (root / \"include\" / \"boost\").mkdir(parents=True)\n (root / \"include\" / \"boost\" / \"ex3.h\").touch()\n (root / \"include\" / \"ex3.h\").touch()\n (root / \"path\" / \"to\").mkdir(parents=True)\n (root / \"path\" / \"to\" / \"ex1.h\").touch()\n (root / \"path\" / \"to\" / \"subdir\").mkdir()\n (root / \"path\" / \"to\" / \"subdir\" / \"ex2.h\").touch()\n\n return root\n\n\n##########\n# Specs of various kind\n##########\n\n\n@pytest.fixture(params=[\"conflict+foo%clang\", \"conflict-parent@0.9^conflict~foo\"])\ndef conflict_spec(request):\n \"\"\"Specs which violate constraints specified with the \"conflicts\"\n directive in the \"conflict\" package.\n \"\"\"\n return request.param\n\n\n@pytest.fixture(scope=\"module\")\ndef mock_test_repo(tmp_path_factory: pytest.TempPathFactory):\n \"\"\"Create an empty repository.\"\"\"\n repo_namespace = \"mock_test_repo\"\n repodir = tmp_path_factory.mktemp(repo_namespace)\n packages_dir = repodir / spack.repo.packages_dir_name\n packages_dir.mkdir()\n yaml_path = repodir / \"repo.yaml\"\n yaml_path.write_text(\n \"\"\"\nrepo:\n namespace: mock_test_repo\n\"\"\"\n )\n\n with spack.repo.use_repositories(str(repodir)) as repo:\n yield repo, repodir\n\n shutil.rmtree(str(repodir))\n\n\n@pytest.fixture(scope=\"function\")\ndef mock_clone_repo(tmp_path_factory: pytest.TempPathFactory):\n \"\"\"Create a cloned repository.\"\"\"\n repo_namespace = \"mock_clone_repo\"\n repodir = tmp_path_factory.mktemp(repo_namespace)\n yaml_path = repodir / \"repo.yaml\"\n yaml_path.write_text(\n \"\"\"\nrepo:\n namespace: mock_clone_repo\n\"\"\"\n )\n\n shutil.copytree(\n os.path.join(spack.paths.mock_packages_path, spack.repo.packages_dir_name),\n os.path.join(str(repodir), spack.repo.packages_dir_name),\n )\n\n with spack.repo.use_repositories(str(repodir)) as repo:\n yield repo, repodir\n\n shutil.rmtree(str(repodir))\n\n\n##########\n# Class and fixture to work around problems raising exceptions in directives,\n# which cause tests like test_from_list_url to hang for Python 2.x metaclass\n# processing.\n#\n# At this point only version and patch directive handling has been addressed.\n##########\n\n\nclass MockBundle:\n has_code = False\n name = \"mock-bundle\"\n\n\n@pytest.fixture\ndef mock_directive_bundle():\n \"\"\"Return a mock bundle package for directive tests.\"\"\"\n return MockBundle()\n\n\n@pytest.fixture\ndef clear_directive_functions():\n \"\"\"Clear all overridden directive functions for subsequent tests.\"\"\"\n yield\n\n # Make sure any directive functions overridden by tests are cleared before\n # proceeding with subsequent tests that may depend on the original\n # functions.\n spack.directives_meta.DirectiveMeta._directives_to_be_executed.clear()\n\n\n@pytest.fixture\ndef mock_executable(tmp_path: Path):\n \"\"\"Factory to create a mock executable in a temporary directory that\n output a custom string when run.\n \"\"\"\n shebang = \"#!/bin/sh\\n\" if sys.platform != \"win32\" else \"@ECHO OFF\\n\"\n\n def _factory(name, output, subdir=(\"bin\",)):\n executable_dir = tmp_path.joinpath(*subdir)\n executable_dir.mkdir(parents=True, exist_ok=True)\n executable_path = executable_dir / name\n if sys.platform == \"win32\":\n executable_path = executable_dir / (name + \".bat\")\n executable_path.write_text(f\"{shebang}{output}\\n\")\n executable_path.chmod(0o755)\n return executable_path\n\n return _factory\n\n\n@pytest.fixture()\ndef mock_test_stage(mutable_config, tmp_path: Path):\n # NOTE: This fixture MUST be applied after any fixture that uses\n # the config fixture under the hood\n # No need to unset because we use mutable_config\n tmp_stage = str(tmp_path / \"test_stage\")\n mutable_config.set(\"config:test_stage\", tmp_stage)\n\n yield tmp_stage\n\n\n@pytest.fixture(autouse=True)\ndef inode_cache():\n spack.llnl.util.lock.FILE_TRACKER.purge()\n yield\n # TODO: it is a bug when the file tracker is non-empty after a test,\n # since it means a lock was not released, or the inode was not purged\n # when acquiring the lock failed. So, we could assert that here, but\n # currently there are too many issues to fix, so look for the more\n # serious issue of having a closed file descriptor in the cache.\n assert not any(f.fh.closed for f in spack.llnl.util.lock.FILE_TRACKER._descriptors.values())\n spack.llnl.util.lock.FILE_TRACKER.purge()\n\n\n@pytest.fixture(autouse=True)\ndef brand_new_binary_cache():\n yield\n spack.binary_distribution.BINARY_INDEX = spack.llnl.util.lang.Singleton(\n spack.binary_distribution.BinaryCacheIndex\n )\n\n\ndef _trivial_package_hash(spec: spack.spec.Spec) -> str:\n \"\"\"Return a trivial package hash for tests to avoid expensive AST parsing.\"\"\"\n # Pad package name to consistent length and cap at 32 chars for realistic hash length\n return base64.b32encode(f\"{spec.name:<32}\".encode()[:32]).decode().lower()\n\n\n@pytest.fixture(autouse=True)\ndef mock_package_hash_for_tests(request, monkeypatch):\n \"\"\"Replace expensive package hash computation with trivial one for tests.\n Tests can force the real package hash by using the @pytest.mark.use_package_hash marker.\"\"\"\n if \"use_package_hash\" in request.keywords:\n yield\n return\n pkg_hash = spack.hash_types.package_hash\n idx = spack.hash_types.HASHES.index(pkg_hash)\n mock_pkg_hash = spack.hash_types.SpecHashDescriptor(\n depflag=0, package_hash=True, name=\"package_hash\", override=_trivial_package_hash\n )\n monkeypatch.setattr(spack.hash_types, \"package_hash\", mock_pkg_hash)\n try:\n spack.hash_types.HASHES[idx] = mock_pkg_hash\n yield\n finally:\n spack.hash_types.HASHES[idx] = pkg_hash\n\n\n@pytest.fixture()\ndef noncyclical_dir_structure(tmp_path: Path):\n \"\"\"\n Create some non-trivial directory structure with\n symlinks to dirs and dangling symlinks, but no cycles::\n\n .\n |-- a/\n | |-- d/\n | |-- file_1\n | |-- to_file_1 -> file_1\n | `-- to_c -> ../c\n |-- b -> a\n |-- c/\n | |-- dangling_link -> nowhere\n | `-- file_2\n `-- file_3\n \"\"\"\n d = tmp_path / \"nontrivial-dir\"\n d.mkdir()\n j = os.path.join\n\n with working_dir(str(d)):\n os.mkdir(j(\"a\"))\n os.mkdir(j(\"a\", \"d\"))\n with open(j(\"a\", \"file_1\"), \"wb\"):\n pass\n os.symlink(j(\"file_1\"), j(\"a\", \"to_file_1\"))\n os.symlink(j(\"..\", \"c\"), j(\"a\", \"to_c\"))\n os.symlink(j(\"a\"), j(\"b\"))\n os.mkdir(j(\"c\"))\n os.symlink(j(\"nowhere\"), j(\"c\", \"dangling_link\"))\n with open(j(\"c\", \"file_2\"), \"wb\"):\n pass\n with open(j(\"file_3\"), \"wb\"):\n pass\n yield d\n\n\n@pytest.fixture(scope=\"function\")\ndef mock_config_data():\n config_data_dir = os.path.join(spack.paths.test_path, \"data\", \"config\")\n return config_data_dir, os.listdir(config_data_dir)\n\n\n@pytest.fixture(scope=\"function\")\ndef mock_curl_configs(mock_config_data, monkeypatch):\n \"\"\"\n Mock curl-based retrieval of configuration files from the web by grabbing\n them from the test data configuration directory.\n\n Fetches a single (configuration) file if the name matches one in the test\n data directory.\n \"\"\"\n config_data_dir, config_files = mock_config_data\n\n class MockCurl:\n def __init__(self):\n self.returncode = None\n\n def __call__(self, *args, **kwargs):\n url = [a for a in args if a.startswith(\"http\")][0]\n basename = os.path.basename(url)\n if os.path.splitext(url)[1]:\n if basename in config_files:\n filename = os.path.join(config_data_dir, basename)\n\n with open(filename, \"r\", encoding=\"utf-8\") as f:\n lines = f.readlines()\n write_file(os.path.basename(filename), \"\".join(lines))\n\n self.returncode = 0\n else:\n # This is a \"404\" and is technically only returned if -f\n # flag is provided to curl.\n tty.msg(\"curl: (22) The requested URL returned error: 404\")\n self.returncode = 22\n\n monkeypatch.setattr(spack.util.web, \"require_curl\", MockCurl)\n\n\n@pytest.fixture(scope=\"function\")\ndef mock_fetch_url_text(mock_config_data, monkeypatch):\n \"\"\"Mock spack.util.web.fetch_url_text.\"\"\"\n\n stage_dir, config_files = mock_config_data\n\n def _fetch_text_file(url, dest_dir):\n raw_url = raw_github_gitlab_url(url)\n mkdirp(dest_dir)\n basename = os.path.basename(raw_url)\n src = join_path(stage_dir, basename)\n dest = join_path(dest_dir, basename)\n copy(src, dest)\n return dest\n\n monkeypatch.setattr(spack.util.web, \"fetch_url_text\", _fetch_text_file)\n\n\n@pytest.fixture(scope=\"function\")\ndef mock_tty_stdout(monkeypatch):\n \"\"\"Make sys.stdout.isatty() return True, while forcing no color output.\"\"\"\n monkeypatch.setattr(sys.stdout, \"isatty\", lambda: True)\n with spack.llnl.util.tty.color.color_when(\"never\"):\n yield\n\n\n@pytest.fixture\ndef prefix_like():\n return \"package-0.0.0.a1-hashhashhashhashhashhashhashhash\"\n\n\n@pytest.fixture()\ndef prefix_tmpdir(tmp_path: Path, prefix_like: str):\n prefix_dir = tmp_path / prefix_like\n prefix_dir.mkdir()\n return prefix_dir\n\n\n@pytest.fixture()\ndef binary_with_rpaths(prefix_tmpdir: Path):\n \"\"\"Factory fixture that compiles an ELF binary setting its RPATH. Relative\n paths are encoded with `$ORIGIN` prepended.\n \"\"\"\n\n def _factory(rpaths, message=\"Hello world!\", dynamic_linker=\"/lib64/ld-linux.so.2\"):\n source = prefix_tmpdir / \"main.c\"\n source.write_text(\n \"\"\"\n #include \n int main(){{\n printf(\"{0}\");\n }}\n \"\"\".format(message)\n )\n gcc = spack.util.executable.which(\"gcc\", required=True)\n executable = source.parent / \"main.x\"\n # Encode relative RPATHs using `$ORIGIN` as the root prefix\n rpaths = [x if os.path.isabs(x) else os.path.join(\"$ORIGIN\", x) for x in rpaths]\n opts = [\n \"-Wl,--disable-new-dtags\",\n f\"-Wl,-rpath={':'.join(rpaths)}\",\n f\"-Wl,--dynamic-linker,{dynamic_linker}\",\n str(source),\n \"-o\",\n str(executable),\n ]\n gcc(*opts)\n return executable\n\n return _factory\n\n\n@pytest.fixture(scope=\"session\")\ndef concretized_specs_cache():\n \"\"\"Cache for mock concrete specs\"\"\"\n return {}\n\n\n@pytest.fixture\ndef default_mock_concretization(\n config, mock_packages, concretized_specs_cache\n) -> Callable[[str], spack.spec.Spec]:\n \"\"\"Return the default mock concretization of a spec literal, obtained using the mock\n repository and the mock configuration.\n\n This fixture is unsafe to call in a test when either the default configuration or mock\n repository are not used or have been modified.\n \"\"\"\n\n def _func(spec_str, tests=False):\n key = spec_str, tests\n if key not in concretized_specs_cache:\n concretized_specs_cache[key] = spack.concretize.concretize_one(\n spack.spec.Spec(spec_str), tests=tests\n )\n return concretized_specs_cache[key].copy()\n\n return _func\n\n\n@pytest.fixture\ndef shell_as(shell):\n if sys.platform != \"win32\":\n yield\n return\n if shell not in (\"pwsh\", \"bat\"):\n raise RuntimeError(\"Shell must be one of supported Windows shells (pwsh|bat)\")\n try:\n # fetch and store old shell type\n _shell = os.environ.get(\"SPACK_SHELL\", None)\n os.environ[\"SPACK_SHELL\"] = shell\n yield\n finally:\n # restore old shell if one was set\n if _shell:\n os.environ[\"SPACK_SHELL\"] = _shell\n\n\n@pytest.fixture()\ndef nullify_globals(request, monkeypatch):\n ensure_configuration_fixture_run_before(request)\n monkeypatch.setattr(spack.config, \"CONFIG\", None)\n monkeypatch.setattr(spack.caches, \"MISC_CACHE\", None)\n monkeypatch.setattr(spack.caches, \"FETCH_CACHE\", None)\n monkeypatch.setattr(spack.repo, \"PATH\", None)\n monkeypatch.setattr(spack.store, \"STORE\", None)\n\n\ndef pytest_runtest_setup(item):\n # Skip test marked \"not_on_windows\" if they're run on Windows\n not_on_windows_marker = item.get_closest_marker(name=\"not_on_windows\")\n if not_on_windows_marker and sys.platform == \"win32\":\n pytest.skip(*not_on_windows_marker.args)\n\n # Skip items marked \"only windows\" if they're run anywhere but Windows\n only_windows_marker = item.get_closest_marker(name=\"only_windows\")\n if only_windows_marker and sys.platform != \"win32\":\n pytest.skip(*only_windows_marker.args)\n\n\n@pytest.fixture(autouse=True)\ndef disable_parallelism(monkeypatch, request):\n \"\"\"Disable process pools in tests. Enabled by default to avoid oversubscription when running\n under pytest-xdist. Can be overridden with `@pytest.mark.enable_parallelism`.\"\"\"\n if \"enable_parallelism\" not in request.keywords:\n monkeypatch.setattr(spack.util.parallel, \"ENABLE_PARALLELISM\", False)\n\n\ndef _root_path(x, y, *, path):\n return path\n\n\n@pytest.fixture\ndef mock_modules_root(tmp_path: Path, monkeypatch):\n \"\"\"Sets the modules root to a temporary directory, to avoid polluting configuration scopes.\"\"\"\n fn = functools.partial(_root_path, path=str(tmp_path))\n monkeypatch.setattr(spack.modules.common, \"root_path\", fn)\n\n\n@pytest.fixture()\ndef compiler_factory():\n \"\"\"Factory for a compiler dict, taking a spec and an OS as arguments.\"\"\"\n\n def _factory(*, spec):\n return {\n \"spec\": f\"{spec}\",\n \"prefix\": \"/path\",\n \"extra_attributes\": {\"compilers\": {\"c\": \"/path/bin/cc\", \"cxx\": \"/path/bin/cxx\"}},\n }\n\n return _factory\n\n\n@pytest.fixture()\ndef host_architecture_str():\n \"\"\"Returns the broad architecture family (x86_64, aarch64, etc.)\"\"\"\n return str(spack.vendor.archspec.cpu.host().family)\n\n\ndef _true(x):\n return True\n\n\ndef _libc_from_python(self):\n return spack.spec.Spec(\"glibc@=2.28\", external_path=\"/some/path\")\n\n\n@pytest.fixture()\ndef do_not_check_runtimes_on_reuse(monkeypatch):\n monkeypatch.setattr(spack.solver.reuse, \"_has_runtime_dependencies\", _true)\n\n\n@pytest.fixture(autouse=True, scope=\"session\")\ndef _c_compiler_always_exists():\n fn = spack.solver.asp.c_compiler_runs\n spack.solver.asp.c_compiler_runs = _true\n mthd = spack.compilers.libraries.CompilerPropertyDetector.default_libc\n spack.compilers.libraries.CompilerPropertyDetector.default_libc = _libc_from_python\n yield\n spack.solver.asp.c_compiler_runs = fn\n spack.compilers.libraries.CompilerPropertyDetector.default_libc = mthd\n\n\n@pytest.fixture(scope=\"session\")\ndef mock_test_cache(tmp_path_factory: pytest.TempPathFactory):\n cache_dir = tmp_path_factory.mktemp(\"cache\")\n return spack.util.file_cache.FileCache(cache_dir)\n\n\nclass MockHTTPResponse(io.IOBase):\n \"\"\"This is a mock HTTP response, which implements part of http.client.HTTPResponse\"\"\"\n\n def __init__(self, status, reason, headers=None, body=None):\n self.msg = None\n self.version = 11\n self.url = None\n self.headers = email.message.EmailMessage()\n self.status = status\n self.code = status\n self.reason = reason\n self.debuglevel = 0\n self._body = body\n\n if headers is not None:\n for key, value in headers.items():\n self.headers[key] = value\n\n @classmethod\n def with_json(cls, status, reason, headers=None, body=None):\n \"\"\"Create a mock HTTP response with JSON string as body\"\"\"\n body = io.BytesIO(json.dumps(body).encode(\"utf-8\"))\n return cls(status, reason, headers, body)\n\n def readable(self):\n return True\n\n def read(self, *args, **kwargs):\n return self._body.read(*args, **kwargs)\n\n def getheader(self, name, default=None):\n self.headers.get(name, default)\n\n def getheaders(self):\n return self.headers.items()\n\n def fileno(self):\n return 0\n\n def getcode(self):\n return self.status\n\n def info(self):\n return self.headers\n\n\n@pytest.fixture()\ndef mock_runtimes(config, mock_packages):\n return mock_packages.packages_with_tags(\"runtime\")\n\n\n@pytest.fixture()\ndef write_config_file(tmp_path: Path):\n \"\"\"Returns a function that writes a config file.\"\"\"\n\n def _write(config, data, scope):\n config_dir = tmp_path / scope\n config_dir.mkdir(parents=True, exist_ok=True)\n config_yaml = config_dir / (config + \".yaml\")\n with config_yaml.open(\"w\") as f:\n syaml.dump_config(data, f)\n return config_yaml\n\n return _write\n\n\ndef _include_cache_root():\n return join_path(str(tempfile.mkdtemp()), \"user_cache\", \"includes\")\n\n\n@pytest.fixture()\ndef wrapper_dir(install_mockery):\n \"\"\"Installs the compiler wrapper and returns the prefix where the script is installed.\"\"\"\n wrapper = spack.concretize.concretize_one(\"compiler-wrapper\")\n wrapper_pkg = wrapper.package\n PackageInstaller([wrapper_pkg], explicit=True).install()\n return wrapper_pkg.bin_dir()\n\n\ndef _noop(*args, **kwargs):\n pass\n\n\n@pytest.fixture(autouse=True)\ndef no_compilers_init(monkeypatch):\n \"\"\"Disables automatic compiler initialization\"\"\"\n monkeypatch.setattr(spack.compilers.config, \"_init_packages_yaml\", _noop)\n\n\n@pytest.fixture(autouse=True)\ndef skip_provenance_check(monkeypatch, request):\n \"\"\"Skip binary provenance check for git versions\n\n Binary provenance checks require querying git repositories and mirrors.\n The infrastructure for this is complex and a heavy lift for simple things like spec syntax\n checks. This fixture defaults to skipping this check, but can be overridden with the\n @pytest.mark.require_provenance decorator\n \"\"\"\n if \"require_provenance\" not in request.keywords:\n monkeypatch.setattr(spack.package_base.PackageBase, \"_resolve_git_provenance\", _noop)\n\n\n@pytest.fixture(scope=\"function\")\ndef config_two_gccs(mutable_config):\n # Configure two gcc compilers that could be concretized to\n extra_attributes_block = {\n \"compilers\": {\"c\": \"/path/to/gcc\", \"cxx\": \"/path/to/g++\", \"fortran\": \"/path/to/fortran\"}\n }\n mutable_config.set(\n \"packages:gcc:externals::\",\n [\n {\n \"spec\": \"gcc@12.3.1 languages=c,c++,fortran\",\n \"prefix\": \"/path\",\n \"extra_attributes\": extra_attributes_block,\n },\n {\n \"spec\": \"gcc@10.3.1 languages=c,c++,fortran\",\n \"prefix\": \"/path\",\n \"extra_attributes\": extra_attributes_block,\n },\n ],\n )\n\n\n@pytest.fixture(scope=\"function\")\ndef mock_util_executable(monkeypatch):\n logger = []\n should_fail = []\n registered_reponses = {}\n\n def mock_call(self, *args, **kwargs):\n cmd = self.exe + list(args)\n str_cmd = \" \".join(map(str, cmd))\n logger.append(str_cmd)\n for failure_key in should_fail:\n if failure_key in str_cmd:\n self.returncode = 1\n if kwargs.get(\"fail_on_error\", True):\n raise spack.util.executable.ProcessError(f\"Failed: {str_cmd}\")\n return\n for key, value in registered_reponses.items():\n if key in str_cmd:\n return value\n self.returncode = 0\n\n monkeypatch.setattr(spack.util.executable.Executable, \"__call__\", mock_call)\n yield logger, should_fail, registered_reponses\n\n\n@pytest.fixture()\ndef reset_extension_paths():\n \"\"\"Clears the cache used for entry points, both in setup and tear-down.\n Needed if a test stresses parts related to computing paths for Spack extensions\n \"\"\"\n spack.extensions.extension_paths_from_entry_points.cache_clear()\n yield\n spack.extensions.extension_paths_from_entry_points.cache_clear()\n\n\n@pytest.fixture(params=[\"old\", \"new\"])\ndef installer_variant(request):\n \"\"\"Parametrize a test over the old and new installer.\"\"\"\n if request.param == \"new\" and sys.platform == \"win32\":\n pytest.skip(\"New installer not supported on Windows\")\n with spack.config.override(\"config:installer\", request.param):\n yield request.param\n" }, { "path": "lib/spack/spack/test/container/cli.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport pytest\n\nimport spack.container.images\nimport spack.llnl.util.filesystem as fs\nimport spack.main\n\ncontainerize = spack.main.SpackCommand(\"containerize\")\n\n\ndef test_command(default_config, container_config_dir):\n with fs.working_dir(container_config_dir):\n output = containerize()\n assert \"FROM spack/ubuntu-jammy\" in output\n\n\ndef test_listing_possible_os():\n output = containerize(\"--list-os\")\n\n for expected_os in spack.container.images.all_bootstrap_os():\n assert expected_os in output\n\n\n@pytest.mark.maybeslow\n@pytest.mark.requires_executables(\"git\")\ndef test_bootstrap_phase(minimal_configuration, config_dumper):\n minimal_configuration[\"spack\"][\"container\"][\"images\"] = {\n \"os\": \"amazonlinux:2\",\n \"spack\": {\"resolve_sha\": False},\n }\n spack_yaml_dir = config_dumper(minimal_configuration)\n\n with fs.working_dir(spack_yaml_dir):\n output = containerize()\n\n # Check for the presence of the Git commands\n assert \"git init\" in output\n assert \"git fetch\" in output\n assert \"git checkout\" in output\n" }, { "path": "lib/spack/spack/test/container/conftest.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport pathlib\n\nimport pytest\n\nimport spack.util.spack_yaml as syaml\n\n\n@pytest.fixture()\ndef minimal_configuration():\n return {\n \"spack\": {\n \"specs\": [\"gromacs\", \"mpich\", \"fftw precision=float\"],\n \"container\": {\n \"format\": \"docker\",\n \"images\": {\"os\": \"ubuntu:22.04\", \"spack\": \"develop\"},\n },\n }\n }\n\n\n@pytest.fixture()\ndef config_dumper(tmp_path: pathlib.Path):\n \"\"\"Function that dumps an environment config in a temporary folder.\"\"\"\n\n def dumper(configuration):\n content = syaml.dump(configuration, default_flow_style=False)\n (tmp_path / \"spack.yaml\").write_text(content or \"\", encoding=\"utf-8\")\n return str(tmp_path)\n\n return dumper\n\n\n@pytest.fixture()\ndef container_config_dir(minimal_configuration, config_dumper):\n return config_dumper(minimal_configuration)\n" }, { "path": "lib/spack/spack/test/container/docker.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport re\n\nimport pytest\n\nimport spack.container.writers as writers\n\n\ndef test_manifest(minimal_configuration):\n writer = writers.create(minimal_configuration)\n manifest_str = writer.manifest\n for line in manifest_str.split(\"\\n\"):\n assert \"echo\" in line\n\n\ndef test_build_and_run_images(minimal_configuration):\n writer = writers.create(minimal_configuration)\n\n # Test the output of run property\n run = writer.run\n assert run.image == \"ubuntu:22.04\"\n\n # Test the output of the build property\n build = writer.build\n assert build.image == \"spack/ubuntu-jammy:develop\"\n\n\ndef test_packages(minimal_configuration):\n # In this minimal configuration we don't have packages\n writer = writers.create(minimal_configuration)\n assert writer.os_packages_build is None\n assert writer.os_packages_final is None\n\n # If we add them a list should be returned\n pkgs = [\"libgomp1\"]\n minimal_configuration[\"spack\"][\"container\"][\"os_packages\"] = {\"final\": pkgs}\n writer = writers.create(minimal_configuration)\n p = writer.os_packages_final\n assert p.update\n assert p.install\n assert p.clean\n assert p.list == pkgs\n\n\ndef test_container_os_packages_command(minimal_configuration):\n # In this minimal configuration we don't have packages\n writer = writers.create(minimal_configuration)\n assert writer.os_packages_build is None\n assert writer.os_packages_final is None\n\n # If we add them a list should be returned\n minimal_configuration[\"spack\"][\"container\"][\"images\"] = {\n \"build\": \"custom-build:latest\",\n \"final\": \"custom-final:latest\",\n }\n minimal_configuration[\"spack\"][\"container\"][\"os_packages\"] = {\n \"command\": \"zypper\",\n \"final\": [\"libgomp1\"],\n }\n writer = writers.create(minimal_configuration)\n p = writer.os_packages_final\n assert \"zypper update -y\" in p.update\n assert \"zypper install -y\" in p.install\n assert \"zypper clean -a\" in p.clean\n\n\ndef test_ensure_render_works(minimal_configuration, default_config):\n # Here we just want to ensure that nothing is raised\n writer = writers.create(minimal_configuration)\n writer()\n\n\ndef test_strip_is_set_from_config(minimal_configuration):\n writer = writers.create(minimal_configuration)\n assert writer.strip is True\n\n minimal_configuration[\"spack\"][\"container\"][\"strip\"] = False\n writer = writers.create(minimal_configuration)\n assert writer.strip is False\n\n\ndef test_custom_base_images(minimal_configuration):\n \"\"\"Test setting custom base images from configuration file\"\"\"\n minimal_configuration[\"spack\"][\"container\"][\"images\"] = {\n \"build\": \"custom-build:latest\",\n \"final\": \"custom-final:latest\",\n }\n writer = writers.create(minimal_configuration)\n\n assert writer.bootstrap.image is None\n assert writer.build.image == \"custom-build:latest\"\n assert writer.run.image == \"custom-final:latest\"\n\n\n@pytest.mark.parametrize(\n \"images_cfg,expected\",\n [\n (\n {\"os\": \"amazonlinux:2\", \"spack\": \"develop\"},\n {\n \"bootstrap_image\": \"amazonlinux:2\",\n \"build_image\": \"bootstrap\",\n \"final_image\": \"amazonlinux:2\",\n },\n )\n ],\n)\ndef test_base_images_with_bootstrap(minimal_configuration, images_cfg, expected):\n \"\"\"Check that base images are computed correctly when a\n bootstrap phase is present\n \"\"\"\n minimal_configuration[\"spack\"][\"container\"][\"images\"] = images_cfg\n writer = writers.create(minimal_configuration)\n\n for property_name, value in expected.items():\n assert getattr(writer, property_name) == value\n\n\ndef test_error_message_invalid_os(minimal_configuration):\n minimal_configuration[\"spack\"][\"container\"][\"images\"][\"os\"] = \"invalid:1\"\n with pytest.raises(ValueError, match=\"invalid operating system\"):\n writers.create(minimal_configuration)\n\n\n@pytest.mark.regression(\"34629,18030\")\ndef test_not_stripping_all_symbols(minimal_configuration):\n \"\"\"Tests that we are not stripping all symbols, so that libraries can still be\n used for linking.\n \"\"\"\n minimal_configuration[\"spack\"][\"container\"][\"strip\"] = True\n content = writers.create(minimal_configuration)()\n assert \"xargs strip\" in content\n assert \"xargs strip -s\" not in content\n\n\n@pytest.mark.regression(\"22341\")\ndef test_using_single_quotes_in_dockerfiles(minimal_configuration):\n \"\"\"Tests that Dockerfiles written by Spack use single quotes in manifest, to avoid issues\n with shell substitution. This may happen e.g. when users have \"definitions:\" they want to\n expand in dockerfiles.\n \"\"\"\n manifest_in_docker = writers.create(minimal_configuration).manifest\n assert not re.search(r\"echo\\s*\\\"\", manifest_in_docker, flags=re.MULTILINE)\n assert re.search(r\"echo\\s*'\", manifest_in_docker)\n" }, { "path": "lib/spack/spack/test/container/images.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport os\n\nimport pytest\n\nimport spack.container\nimport spack.container.images\n\n\n@pytest.mark.parametrize(\n \"image,spack_version,expected\",\n [\n (\"ubuntu:22.04\", \"develop\", (\"spack/ubuntu-jammy\", \"develop\")),\n (\"ubuntu:22.04\", \"0.14.0\", (\"spack/ubuntu-jammy\", \"0.14.0\")),\n ],\n)\ndef test_build_info(image, spack_version, expected):\n output = spack.container.images.build_info(image, spack_version)\n assert output == expected\n\n\n@pytest.mark.parametrize(\"image\", [\"ubuntu:22.04\"])\ndef test_package_info(image):\n pkg_manager = spack.container.images.os_package_manager_for(image)\n update, install, clean = spack.container.images.commands_for(pkg_manager)\n assert update\n assert install\n assert clean\n\n\n@pytest.mark.parametrize(\n \"extra_config,expected_msg\",\n [\n ({\"modules\": {\"enable\": [\"tcl\"]}}, 'the subsection \"modules\" in'),\n ({\"concretizer\": {\"unify\": False}}, '\"concretizer:unify\" is not set to \"true\"'),\n (\n {\"config\": {\"install_tree\": {\"root\": \"/some/dir\"}}},\n 'the \"config:install_tree\" attribute has been set',\n ),\n ({\"view\": \"/some/dir\"}, 'the \"view\" attribute has been set'),\n ],\n)\ndef test_validate(extra_config, expected_msg, minimal_configuration, config_dumper):\n minimal_configuration[\"spack\"].update(extra_config)\n spack_yaml_dir = config_dumper(minimal_configuration)\n spack_yaml = os.path.join(spack_yaml_dir, \"spack.yaml\")\n\n with pytest.warns(UserWarning) as w:\n spack.container.validate(spack_yaml)\n\n # Tests are designed to raise only one warning\n assert len(w) == 1\n assert expected_msg in str(w.pop().message)\n" }, { "path": "lib/spack/spack/test/container/singularity.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport pytest\n\nimport spack.container.writers as writers\n\n\n@pytest.fixture\ndef singularity_configuration(minimal_configuration):\n minimal_configuration[\"spack\"][\"container\"][\"format\"] = \"singularity\"\n return minimal_configuration\n\n\ndef test_ensure_render_works(default_config, singularity_configuration):\n container_config = singularity_configuration[\"spack\"][\"container\"]\n assert container_config[\"format\"] == \"singularity\"\n # Here we just want to ensure that nothing is raised\n writer = writers.create(singularity_configuration)\n writer()\n\n\n@pytest.mark.parametrize(\n \"properties,expected\",\n [\n (\n {\"runscript\": \"/opt/view/bin/h5ls\"},\n {\"runscript\": \"/opt/view/bin/h5ls\", \"startscript\": \"\", \"test\": \"\", \"help\": \"\"},\n )\n ],\n)\ndef test_singularity_specific_properties(properties, expected, singularity_configuration):\n # Set the property in the configuration\n container_config = singularity_configuration[\"spack\"][\"container\"]\n for name, value in properties.items():\n container_config.setdefault(\"singularity\", {})[name] = value\n\n # Assert the properties return the expected values\n writer = writers.create(singularity_configuration)\n for name, value in expected.items():\n assert getattr(writer, name) == value\n\n\n@pytest.mark.regression(\"34629,18030\")\ndef test_not_stripping_all_symbols(singularity_configuration):\n \"\"\"Tests that we are not stripping all symbols, so that libraries can still be\n used for linking.\n \"\"\"\n singularity_configuration[\"spack\"][\"container\"][\"strip\"] = True\n content = writers.create(singularity_configuration)()\n assert \"xargs strip\" in content\n assert \"xargs strip -s\" not in content\n" }, { "path": "lib/spack/spack/test/cray_manifest.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"\nNote that where possible, this should produce specs using `entries_to_specs`\nrather than `spec_from_entry`, since the former does additional work to\nestablish dependency relationships (and in general the manifest-parsing\nlogic needs to consume all related specs in a single pass).\n\"\"\"\n\nimport json\nimport pathlib\n\nimport pytest\n\nimport spack.vendor.archspec.cpu\n\nimport spack.cmd\nimport spack.cmd.external\nimport spack.compilers.config\nimport spack.concretize\nimport spack.cray_manifest\nimport spack.platforms\nimport spack.platforms.test\nimport spack.solver.reuse\nimport spack.spec\nimport spack.store\nfrom spack.cray_manifest import compiler_from_entry, entries_to_specs\n\npytestmark = [\n pytest.mark.skipif(\n str(spack.platforms.host()) != \"linux\", reason=\"Cray manifest files are only for linux\"\n ),\n pytest.mark.usefixtures(\"mutable_config\", \"mock_packages\"),\n]\n\n\nclass JsonSpecEntry:\n def __init__(self, name, hash, prefix, version, arch, compiler, dependencies, parameters):\n self.name = name\n self.hash = hash\n self.prefix = prefix\n self.version = version\n self.arch = arch\n self.compiler = compiler\n self.dependencies = dependencies\n self.parameters = parameters\n\n def to_dict(self):\n return {\n \"name\": self.name,\n \"hash\": self.hash,\n \"prefix\": self.prefix,\n \"version\": self.version,\n \"arch\": self.arch,\n \"compiler\": self.compiler,\n \"dependencies\": self.dependencies,\n \"parameters\": self.parameters,\n }\n\n def as_dependency(self, deptypes):\n return (self.name, {\"hash\": self.hash, \"type\": list(deptypes)})\n\n\nclass JsonArchEntry:\n def __init__(self, platform, os, target):\n self.platform = platform\n self.os = os\n self.target = target\n\n def spec_json(self):\n return {\"platform\": self.platform, \"platform_os\": self.os, \"target\": {\"name\": self.target}}\n\n def compiler_json(self):\n return {\"os\": self.os, \"target\": self.target}\n\n\nclass JsonCompilerEntry:\n def __init__(self, *, name, version, arch=None, executables=None, prefix=None):\n self.name = name\n self.version = version\n self.arch = arch or JsonArchEntry(\"anyplatform\", \"anyos\", \"anytarget\")\n self.executables = executables or {\"cc\": \"cc\", \"cxx\": \"cxx\", \"fc\": \"fc\"}\n self.prefix = prefix\n\n def compiler_json(self):\n result = {\n \"name\": self.name,\n \"version\": self.version,\n \"arch\": self.arch.compiler_json(),\n \"executables\": self.executables,\n }\n # See https://github.com/spack/spack/pull/40061\n if self.prefix is not None:\n result[\"prefix\"] = self.prefix\n return result\n\n def spec_json(self):\n \"\"\"The compiler spec only lists the name/version, not\n arch/executables.\n \"\"\"\n return {\"name\": self.name, \"version\": self.version}\n\n\n@pytest.fixture\ndef _common_arch(test_platform):\n generic = spack.vendor.archspec.cpu.TARGETS[test_platform.default].family\n return JsonArchEntry(platform=test_platform.name, os=\"redhat6\", target=generic.name)\n\n\n@pytest.fixture\ndef _common_compiler(_common_arch):\n return JsonCompilerEntry(\n name=\"gcc\",\n version=\"10.2.0.2112\",\n arch=_common_arch,\n executables={\n \"cc\": \"/path/to/compiler/cc\",\n \"cxx\": \"/path/to/compiler/cxx\",\n \"fc\": \"/path/to/compiler/fc\",\n },\n )\n\n\n@pytest.fixture\ndef _other_compiler(_common_arch):\n return JsonCompilerEntry(\n name=\"clang\",\n version=\"3.0.0\",\n arch=_common_arch,\n executables={\n \"cc\": \"/path/to/compiler/clang\",\n \"cxx\": \"/path/to/compiler/clang++\",\n \"fc\": \"/path/to/compiler/flang\",\n },\n )\n\n\n@pytest.fixture\ndef _raw_json_x(_common_arch):\n return {\n \"name\": \"packagex\",\n \"hash\": \"hash-of-x\",\n \"prefix\": \"/path/to/packagex-install/\",\n \"version\": \"1.0\",\n \"arch\": _common_arch.spec_json(),\n \"compiler\": {\"name\": \"gcc\", \"version\": \"10.2.0.2112\"},\n \"dependencies\": {\"packagey\": {\"hash\": \"hash-of-y\", \"type\": [\"link\"]}},\n \"parameters\": {\"precision\": [\"double\", \"float\"]},\n }\n\n\ndef test_manifest_compatibility(_common_arch, _common_compiler, _raw_json_x):\n \"\"\"Make sure that JsonSpecEntry outputs the expected JSON structure\n by comparing it with JSON parsed from an example string. This\n ensures that the testing objects like JsonSpecEntry produce the\n same JSON structure as the expected file format.\n \"\"\"\n y = JsonSpecEntry(\n name=\"packagey\",\n hash=\"hash-of-y\",\n prefix=\"/path/to/packagey-install/\",\n version=\"1.0\",\n arch=_common_arch.spec_json(),\n compiler=_common_compiler.spec_json(),\n dependencies={},\n parameters={},\n )\n\n x = JsonSpecEntry(\n name=\"packagex\",\n hash=\"hash-of-x\",\n prefix=\"/path/to/packagex-install/\",\n version=\"1.0\",\n arch=_common_arch.spec_json(),\n compiler=_common_compiler.spec_json(),\n dependencies=dict([y.as_dependency(deptypes=[\"link\"])]),\n parameters={\"precision\": [\"double\", \"float\"]},\n )\n\n x_from_entry = x.to_dict()\n assert x_from_entry == _raw_json_x\n\n\ndef test_compiler_from_entry(mock_executable):\n \"\"\"Tests that we can detect a compiler from a valid entry in the Cray manifest\"\"\"\n cc = mock_executable(\"gcc\", output=\"echo 7.5.0\")\n cxx = mock_executable(\"g++\", output=\"echo 7.5.0\")\n fc = mock_executable(\"gfortran\", output=\"echo 7.5.0\")\n\n compiler = compiler_from_entry(\n JsonCompilerEntry(\n name=\"gcc\",\n version=\"7.5.0\",\n arch=JsonArchEntry(platform=\"linux\", os=\"centos8\", target=\"x86_64\"),\n prefix=str(cc.parent),\n executables={\"cc\": \"gcc\", \"cxx\": \"g++\", \"fc\": \"gfortran\"},\n ).compiler_json(),\n manifest_path=\"/example/file\",\n )\n\n assert compiler.satisfies(\"gcc@7.5.0 target=x86_64 os=centos8\")\n assert compiler.extra_attributes[\"compilers\"][\"c\"] == str(cc)\n assert compiler.extra_attributes[\"compilers\"][\"cxx\"] == str(cxx)\n assert compiler.extra_attributes[\"compilers\"][\"fortran\"] == str(fc)\n\n\n@pytest.fixture\ndef generate_openmpi_entries(_common_arch, _common_compiler):\n \"\"\"Generate two example JSON entries that refer to an OpenMPI\n installation and a hwloc dependency.\n \"\"\"\n # The hashes need to be padded with 'a' at the end to align with 8-byte\n # boundaries (for base-32 decoding)\n hwloc = JsonSpecEntry(\n name=\"hwloc\",\n hash=\"hwlocfakehashaaa\",\n prefix=\"/path/to/hwloc-install/\",\n version=\"2.0.3\",\n arch=_common_arch.spec_json(),\n compiler=_common_compiler.spec_json(),\n dependencies={},\n parameters={},\n )\n\n # This includes a variant which is guaranteed not to appear in the\n # OpenMPI package: we need to make sure we can use such package\n # descriptions.\n openmpi = JsonSpecEntry(\n name=\"openmpi\",\n hash=\"openmpifakehasha\",\n prefix=\"/path/to/openmpi-install/\",\n version=\"4.1.0\",\n arch=_common_arch.spec_json(),\n compiler=_common_compiler.spec_json(),\n dependencies=dict([hwloc.as_dependency(deptypes=[\"link\"])]),\n parameters={\"internal-hwloc\": False, \"fabrics\": [\"psm\"], \"missing_variant\": True},\n )\n\n return list(x.to_dict() for x in [openmpi, hwloc])\n\n\ndef test_generate_specs_from_manifest(generate_openmpi_entries):\n \"\"\"Given JSON entries, check that we can form a set of Specs\n including dependency references.\n \"\"\"\n specs = entries_to_specs(generate_openmpi_entries)\n (openmpi_spec,) = list(x for x in specs.values() if x.name == \"openmpi\")\n assert openmpi_spec[\"hwloc\"]\n\n\ndef test_translate_cray_platform_to_linux(monkeypatch, _common_compiler):\n \"\"\"Manifests might list specs on newer Cray platforms as being \"cray\",\n but Spack identifies such platforms as \"linux\". Make sure we\n automatically transform these entries.\n \"\"\"\n test_linux_platform = spack.platforms.test.Test(\"linux\")\n\n def the_host_is_linux():\n return test_linux_platform\n\n monkeypatch.setattr(spack.platforms, \"host\", the_host_is_linux)\n\n cray_arch = JsonArchEntry(platform=\"cray\", os=\"rhel8\", target=\"x86_64\")\n spec_json = JsonSpecEntry(\n name=\"mpich\",\n hash=\"craympichfakehashaaa\",\n prefix=\"/path/to/cray-mpich/\",\n version=\"1.0.0\",\n arch=cray_arch.spec_json(),\n compiler=_common_compiler.spec_json(),\n dependencies={},\n parameters={},\n ).to_dict()\n\n (spec,) = entries_to_specs([spec_json]).values()\n assert spec.architecture.platform == \"linux\"\n\n\n@pytest.mark.parametrize(\n \"name_in_manifest,expected_name\",\n [(\"nvidia\", \"nvhpc\"), (\"rocm\", \"llvm-amdgpu\"), (\"clang\", \"llvm\")],\n)\ndef test_translated_compiler_name(name_in_manifest, expected_name):\n assert spack.cray_manifest.translated_compiler_name(name_in_manifest) == expected_name\n\n\ndef test_failed_translate_compiler_name(_common_arch):\n unknown_compiler = JsonCompilerEntry(name=\"unknown\", version=\"1.0\")\n\n with pytest.raises(spack.compilers.config.UnknownCompilerError):\n compiler_from_entry(unknown_compiler.compiler_json(), manifest_path=\"/example/file\")\n\n spec_json = JsonSpecEntry(\n name=\"packagey\",\n hash=\"hash-of-y\",\n prefix=\"/path/to/packagey-install/\",\n version=\"1.0\",\n arch=_common_arch.spec_json(),\n compiler=unknown_compiler.spec_json(),\n dependencies={},\n parameters={},\n ).to_dict()\n\n with pytest.raises(spack.compilers.config.UnknownCompilerError):\n entries_to_specs([spec_json])\n\n\n@pytest.fixture\ndef manifest_content(generate_openmpi_entries, _common_compiler, _other_compiler):\n return {\n \"_meta\": {\n \"file-type\": \"cray-pe-json\",\n \"system-type\": \"EX\",\n \"schema-version\": \"1.3\",\n \"cpe-version\": \"22.06\",\n },\n \"specs\": generate_openmpi_entries,\n \"compilers\": [_common_compiler.compiler_json(), _other_compiler.compiler_json()],\n }\n\n\ndef test_read_cray_manifest(temporary_store, manifest_file):\n \"\"\"Check that (a) we can read the cray manifest and add it to the Spack\n Database and (b) we can concretize specs based on that.\n \"\"\"\n spack.cray_manifest.read(str(manifest_file), True)\n\n query_specs = temporary_store.db.query(\"openmpi\")\n assert any(x.dag_hash() == \"openmpifakehasha\" for x in query_specs)\n\n concretized_spec = spack.concretize.concretize_one(\"depends-on-openmpi ^/openmpifakehasha\")\n assert concretized_spec[\"hwloc\"].dag_hash() == \"hwlocfakehashaaa\"\n\n\ndef test_read_cray_manifest_add_compiler_failure(temporary_store, manifest_file, monkeypatch):\n \"\"\"Tests the Cray manifest can be read even if some compilers cannot be added.\"\"\"\n\n def _mock(entry, *, manifest_path):\n if entry[\"name\"] == \"clang\":\n raise RuntimeError(\"cannot determine the compiler\")\n return spack.spec.Spec(f\"{entry['name']}@{entry['version']}\")\n\n monkeypatch.setattr(spack.cray_manifest, \"compiler_from_entry\", _mock)\n\n spack.cray_manifest.read(str(manifest_file), True)\n query_specs = spack.store.STORE.db.query(\"openmpi\")\n assert any(x.dag_hash() == \"openmpifakehasha\" for x in query_specs)\n\n\ndef test_read_cray_manifest_twice_no_duplicates(\n mutable_config, temporary_store, manifest_file, monkeypatch, tmp_path: pathlib.Path\n):\n def _mock(entry, *, manifest_path):\n return spack.spec.Spec(f\"{entry['name']}@{entry['version']}\", external_path=str(tmp_path))\n\n monkeypatch.setattr(spack.cray_manifest, \"compiler_from_entry\", _mock)\n\n # Read the manifest twice\n spack.cray_manifest.read(str(manifest_file), True)\n spack.cray_manifest.read(str(manifest_file), True)\n\n config_data = mutable_config.get(\"packages\")[\"gcc\"]\n assert \"externals\" in config_data\n\n specs = [spack.spec.Spec(x[\"spec\"]) for x in config_data[\"externals\"]]\n assert len(specs) == len(set(specs))\n assert len([c for c in specs if c.satisfies(\"gcc@10.2.0.2112\")]) == 1\n\n\ndef test_read_old_manifest_v1_2(tmp_path: pathlib.Path, temporary_store):\n \"\"\"Test reading a file using the older format ('version' instead of 'schema-version').\"\"\"\n manifest = tmp_path / \"manifest_dir\" / \"test.json\"\n manifest.parent.mkdir(parents=True)\n manifest.write_text(\n \"\"\"\\\n{\n \"_meta\": {\n \"file-type\": \"cray-pe-json\",\n \"system-type\": \"EX\",\n \"version\": \"1.3\"\n },\n \"specs\": []\n}\n\"\"\"\n )\n spack.cray_manifest.read(str(manifest), True)\n\n\ndef test_convert_validation_error(\n tmp_path: pathlib.Path, mutable_config, mock_packages, temporary_store\n):\n manifest_dir = tmp_path / \"manifest_dir\"\n manifest_dir.mkdir()\n # Does not parse as valid JSON\n invalid_json_path = manifest_dir / \"invalid-json.json\"\n with open(invalid_json_path, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\n{\n\"\"\"\n )\n with pytest.raises(spack.cray_manifest.ManifestValidationError) as e:\n spack.cray_manifest.read(invalid_json_path, True)\n str(e)\n\n # Valid JSON, but does not conform to schema (schema-version is not a string\n # of length > 0)\n invalid_schema_path = manifest_dir / \"invalid-schema.json\"\n with open(invalid_schema_path, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\n{\n \"_meta\": {\n \"file-type\": \"cray-pe-json\",\n \"system-type\": \"EX\",\n \"schema-version\": \"\"\n },\n \"specs\": []\n}\n\"\"\"\n )\n with pytest.raises(spack.cray_manifest.ManifestValidationError) as e:\n spack.cray_manifest.read(invalid_schema_path, True)\n\n\n@pytest.fixture\ndef manifest_file(tmp_path: pathlib.Path, manifest_content):\n \"\"\"Create a manifest file in a directory. Used by 'spack external'.\"\"\"\n filename = tmp_path / \"external-db.json\"\n with open(filename, \"w\", encoding=\"utf-8\") as db_file:\n json.dump(manifest_content, db_file)\n return filename\n\n\ndef test_find_external_nonempty_default_manifest_dir(\n temporary_store, mutable_mock_repo, monkeypatch, manifest_file\n):\n \"\"\"The user runs 'spack external find'; the default manifest directory\n contains a manifest file. Ensure that the specs are read.\n \"\"\"\n monkeypatch.setenv(\"PATH\", \"\")\n monkeypatch.setattr(spack.cray_manifest, \"default_path\", str(manifest_file.parent))\n spack.cmd.external._collect_and_consume_cray_manifest_files(ignore_default_dir=False)\n specs = temporary_store.db.query(\"hwloc\")\n assert any(x.dag_hash() == \"hwlocfakehashaaa\" for x in specs)\n\n\ndef test_reusable_externals_cray_manifest(temporary_store, manifest_file):\n \"\"\"The concretizer should be able to reuse specs imported from a manifest without a\n externals config entry in packages.yaml\"\"\"\n spack.cray_manifest.read(path=str(manifest_file), apply_updates=True)\n\n # Get any imported spec\n spec = temporary_store.db.query_local()[0]\n\n # Reusable if imported locally\n assert spack.solver.reuse._is_reusable(spec, packages_with_externals={}, local=True)\n\n # If cray manifest entries end up in a build cache somehow, they are not reusable\n assert not spack.solver.reuse._is_reusable(spec, packages_with_externals={}, local=False)\n" }, { "path": "lib/spack/spack/test/cvs_fetch.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport pathlib\n\nimport pytest\n\nimport spack.concretize\nfrom spack.fetch_strategy import CvsFetchStrategy\nfrom spack.llnl.util.filesystem import mkdirp, touch, working_dir\nfrom spack.stage import Stage\nfrom spack.util.executable import which\nfrom spack.version import Version\n\npytestmark = pytest.mark.skipif(not which(\"cvs\"), reason=\"requires CVS to be installed\")\n\n\n@pytest.mark.parametrize(\"type_of_test\", [\"default\", \"branch\", \"date\"])\ndef test_fetch(type_of_test, mock_cvs_repository, config, mutable_mock_repo):\n \"\"\"Tries to:\n\n 1. Fetch the repo using a fetch strategy constructed with\n supplied args (they depend on type_of_test).\n 2. Check whether the checkout is on the correct branch or date\n 3. Check if the test_file is in the checked out repository.\n 4. Add and remove some files, then reset the repo, and\n ensure it's all there again.\n\n CVS does not have the notion of a unique branch; branches and revisions\n are managed separately for every file.\n \"\"\"\n # Retrieve the right test parameters\n test = mock_cvs_repository.checks[type_of_test]\n get_branch = mock_cvs_repository.get_branch\n get_date = mock_cvs_repository.get_date\n\n # Construct the package under test\n spec = spack.concretize.concretize_one(\"cvs-test\")\n spec.package.versions[Version(\"cvs\")] = test.args\n\n # Enter the stage directory and check some properties\n with spec.package.stage:\n spec.package.do_stage()\n\n with working_dir(spec.package.stage.source_path):\n # Check branch\n if test.branch is not None:\n assert get_branch() == test.branch\n\n # Check date\n if test.date is not None:\n assert get_date() <= test.date\n\n file_path = os.path.join(spec.package.stage.source_path, test.file)\n assert os.path.isdir(spec.package.stage.source_path)\n assert os.path.isfile(file_path)\n\n os.unlink(file_path)\n assert not os.path.isfile(file_path)\n\n untracked_file = \"foobarbaz\"\n touch(untracked_file)\n assert os.path.isfile(untracked_file)\n spec.package.do_restage()\n assert not os.path.isfile(untracked_file)\n\n assert os.path.isdir(spec.package.stage.source_path)\n assert os.path.isfile(file_path)\n\n\ndef test_cvs_extra_fetch(tmp_path: pathlib.Path):\n \"\"\"Ensure a fetch after downloading is effectively a no-op.\"\"\"\n testpath = str(tmp_path)\n\n fetcher = CvsFetchStrategy(cvs=\":pserver:not-a-real-cvs-repo%module=not-a-real-module\")\n assert fetcher is not None\n\n with Stage(fetcher, path=testpath) as stage:\n assert stage is not None\n\n source_path = stage.source_path\n mkdirp(source_path)\n\n # TODO: This doesn't look as if it was testing what this function's\n # comment says it is testing. However, the other `test_*_extra_fetch`\n # functions (for svn, git, hg) use equivalent code.\n #\n # We're calling `fetcher.fetch` twice as this might be what we want to\n # do, and it can't hurt. See\n # for a discussion on this.\n\n # Fetch once\n fetcher.fetch()\n # Fetch a second time\n fetcher.fetch()\n" }, { "path": "lib/spack/spack/test/data/compiler_verbose_output/cce-8.6.5.txt", "content": "rm foo\n\n/opt/cray/pe/cce/8.6.5/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld /usr/lib64//crt1.o /usr/lib64//crti.o /opt/gcc/6.1.0/snos/lib/gcc/x86_64-suse-linux/6.1.0//crtbeginT.o /opt/gcc/6.1.0/snos/lib/gcc/x86_64-suse-linux/6.1.0//crtfastmath.o /opt/cray/pe/cce/8.6.5/cce/x86_64/lib/no_mmap.o foo.o -Bstatic -rpath=/opt/cray/pe/cce/8.6.5/cce/x86_64/lib -L /opt/gcc/6.1.0/snos/lib64 -rpath=/opt/cray/pe/gcc-libs -L /usr/lib64 -L /lib64 -L /opt/cray/dmapp/default/lib64 -L /opt/cray/pe/mpt/7.7.0/gni/mpich-cray/8.6/lib -L /opt/cray/dmapp/default/lib64 -L /opt/cray/pe/mpt/7.7.0/gni/mpich-cray/8.6/lib -L /opt/cray/pe/libsci/17.12.1/CRAY/8.6/x86_64/lib -L /opt/cray/rca/2.2.16-6.0.5.0_15.34__g5e09e6d.ari/lib64 -L /opt/cray/pe/pmi/5.0.13/lib64 -L /opt/cray/xpmem/2.2.4-6.0.5.0_4.8__g35d5e73.ari/lib64 -L /opt/cray/dmapp/7.1.1-6.0.5.0_49.8__g1125556.ari/lib64 -L /opt/cray/ugni/6.0.14-6.0.5.0_16.9__g19583bb.ari/lib64 -L /opt/cray/udreg/2.3.2-6.0.5.0_13.12__ga14955a.ari/lib64 -L /opt/cray/alps/6.5.28-6.0.5.0_18.6__g13a91b6.ari/lib64 -L /opt/cray/pe/atp/2.1.1/libApp -L /opt/cray/pe/cce/8.6.5/cce/x86_64/lib/pkgconfig/../ -L /opt/cray/wlm_detect/1.3.2-6.0.5.0_3.1__g388ccd5.ari/lib64 --no-as-needed -lAtpSigHandler -lAtpSigHCommData --undefined=_ATP_Data_Globals --undefined=__atpHandlerInstall -lpthread -lmpichcxx_cray -lrt -lpthread -lugni -lpmi -lsci_cray_mpi_mp -lm -lf -lsci_cray_mp -lmpich_cray -lrt -lpthread -lugni -lpmi -lsci_cray_mp -lcraymp -lm -lpthread -lf -lhugetlbfs -lpgas-dmapp -lfi -lu -lrt --undefined=dmapp_get_flag_nbi -ldmapp -lugni -ludreg -lpthread -lm -lcray-c++-rts -lstdc++ -lxpmem -ldmapp -lpthread -lpmi -lpthread -lalpslli -lpthread -lwlm_detect -lugni -lpthread -lalpsutil -lpthread -lrca -ludreg -lquadmath -lm -lomp -lcraymp -lpthread -lrt -ldl -lcray-c++-rts -lstdc++ -lm -lmodules -lm -lfi -lm -lquadmath -lcraymath -lm -lgfortran -lquadmath -lf -lm -lpthread -lu -lrt -ldl -lcray-c++-rts -lstdc++ -lm -lcsup --as-needed -latomic --no-as-needed -lcray-c++-rts -lstdc++ -lsupc++ -lstdc++ -lpthread --start-group -lc -lcsup -lgcc_eh -lm -lgcc --end-group -T/opt/cray/pe/cce/8.6.5/cce/x86_64/lib/2.23.1.cce.ld -L /opt/gcc/6.1.0/snos/lib/gcc/x86_64-suse-linux/6.1.0 -L /opt/cray/pe/cce/8.6.5/binutils/x86_64/x86_64-pc-linux-gnu/..//x86_64-unknown-linux-gnu/lib -EL -o foo --undefined=__pthread_initialize_minimal /opt/gcc/6.1.0/snos/lib/gcc/x86_64-suse-linux/6.1.0//crtend.o /usr/lib64//crtn.o\n\n/opt/cray/pe/cce/8.6.5/binutils/x86_64/x86_64-pc-linux-gnu/bin/objcopy --remove-section=.note.ftn_module_data foo\nrm /tmp/pe_27645//pldir/PL_path\nrm /tmp/pe_27645//pldir/PL_module_list\nrm /tmp/pe_27645//pldir/PL_global_data\nrmdir /tmp/pe_27645//pldir\nrmdir /tmp/pe_27645/\n" }, { "path": "lib/spack/spack/test/data/compiler_verbose_output/clang-4.0.1.txt", "content": "clang version 4.0.1 (tags/RELEASE_401/final)\nTarget: x86_64-unknown-linux-gnu\nThread model: posix\nInstalledDir: /usr/bin\nFound candidate GCC installation: /usr/bin/../lib/gcc/x86_64-redhat-linux/7\nFound candidate GCC installation: /usr/lib/gcc/x86_64-redhat-linux/7\nSelected GCC installation: /usr/bin/../lib/gcc/x86_64-redhat-linux/7\nCandidate multilib: .;@m64\nCandidate multilib: 32;@m32\nSelected multilib: .;@m64\n \"/usr/bin/clang-4.0\" -cc1 -triple x86_64-unknown-linux-gnu -emit-obj -mrelax-all -disable-free -disable-llvm-verifier -discard-value-names -main-file-name main.c -mrelocation-model static -mthread-model posix -mdisable-fp-elim -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -v -dwarf-column-info -debugger-tuning=gdb -resource-dir /usr/bin/../lib64/clang/4.0.1 -internal-isystem /usr/local/include -internal-isystem /usr/bin/../lib64/clang/4.0.1/include -internal-externc-isystem /include -internal-externc-isystem /usr/include -fdebug-compilation-dir /tmp/spack-test -ferror-limit 19 -fmessage-length 0 -fobjc-runtime=gcc -fdiagnostics-show-option -o /tmp/main-bf64f0.o -x c main.c\nclang -cc1 version 4.0.1 based upon LLVM 4.0.1 default target x86_64-unknown-linux-gnu\nignoring nonexistent directory \"/include\"\n#include \"...\" search starts here:\n#include <...> search starts here:\n /usr/local/include\n /usr/bin/../lib64/clang/4.0.1/include\n /usr/include\nEnd of search list.\n \"/usr/bin/ld\" --hash-style=gnu --no-add-needed --build-id --eh-frame-hdr -m elf_x86_64 -dynamic-linker /lib64/ld-linux-x86-64.so.2 -o output /usr/bin/../lib/gcc/x86_64-redhat-linux/7/../../../../lib64/crt1.o /usr/bin/../lib/gcc/x86_64-redhat-linux/7/../../../../lib64/crti.o /usr/bin/../lib/gcc/x86_64-redhat-linux/7/crtbegin.o -L/usr/bin/../lib/gcc/x86_64-redhat-linux/7 -L/usr/bin/../lib/gcc/x86_64-redhat-linux/7/../../../../lib64 -L/usr/bin/../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/bin/../lib/gcc/x86_64-redhat-linux/7/../../.. -L/usr/bin/../lib -L/lib -L/usr/lib /tmp/main-bf64f0.o -lgcc --as-needed -lgcc_s --no-as-needed -lc -lgcc --as-needed -lgcc_s --no-as-needed /usr/bin/../lib/gcc/x86_64-redhat-linux/7/crtend.o /usr/bin/../lib/gcc/x86_64-redhat-linux/7/../../../../lib64/crtn.o\n" }, { "path": "lib/spack/spack/test/data/compiler_verbose_output/clang-9.0.0-apple-ld.txt", "content": "@(#)PROGRAM:ld PROJECT:ld64-305\nconfigured to support archs: armv6 armv7 armv7s arm64 i386 x86_64 x86_64h armv6m armv7k armv7m armv7em (tvOS)\nLibrary search paths:\n\t/usr/local/lib\n\t/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.13.sdk/usr/lib\nFramework search paths:\n\t /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.13.sdk/System/Library/Frameworks/\n" }, { "path": "lib/spack/spack/test/data/compiler_verbose_output/collect2-6.3.0-gnu-ld.txt", "content": "collect2 version 6.5.0\n/usr/bin/ld -plugin /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/liblto_plugin.so -plugin-opt=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/lto-wrapper -plugin-opt=-fresolution=/tmp/ccbFmewQ.res -plugin-opt=-pass-through=-lgcc_s -plugin-opt=-pass-through=-lgcc -plugin-opt=-pass-through=-lc -plugin-opt=-pass-through=-lgcc_s -plugin-opt=-pass-through=-lgcc -rpath /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib64 --eh-frame-hdr -m elf_x86_64 -dynamic-linker /lib64/ld-linux-x86-64.so.2 -o output /usr/lib/x86_64-linux-gnu/crt1.o /usr/lib/x86_64-linux-gnu/crti.o /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/crtbegin.o -L/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0 -L/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/../../../../lib64 -L/lib/x86_64-linux-gnu -L/lib/../lib64 -L/usr/lib/x86_64-linux-gnu -L/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/../../.. -v /tmp/ccxz6i1I.o -lstdc++ -lm -lgcc_s -lgcc -lc -lgcc_s -lgcc /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/crtend.o /usr/lib/x86_64-linux-gnu/crtn.o\nGNU ld (GNU Binutils for Debian) 2.28\n" }, { "path": "lib/spack/spack/test/data/compiler_verbose_output/gcc-7.3.1.txt", "content": "Using built-in specs.\nCOLLECT_GCC=/usr/bin/gcc\nCOLLECT_LTO_WRAPPER=/usr/libexec/gcc/x86_64-redhat-linux/7/lto-wrapper\nOFFLOAD_TARGET_NAMES=nvptx-none\nOFFLOAD_TARGET_DEFAULT=1\nTarget: x86_64-redhat-linux\nConfigured with: ../configure --enable-bootstrap --enable-languages=c,c++,objc,obj-c++,fortran,ada,go,lto --prefix=/usr --mandir=/usr/share/man --infodir=/usr/share/info --with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-shared --enable-threads=posix --enable-checking=release --enable-multilib --with-system-zlib --enable-__cxa_atexit --disable-libunwind-exceptions --enable-gnu-unique-object --enable-linker-build-id --with-gcc-major-version-only --with-linker-hash-style=gnu --enable-plugin --enable-initfini-array --with-isl --enable-libmpx --enable-offload-targets=nvptx-none --without-cuda-driver --enable-gnu-indirect-function --with-tune=generic --with-arch_32=i686 --build=x86_64-redhat-linux\nThread model: posix\ngcc version 7.3.1 20180130 (Red Hat 7.3.1-2) (GCC) \nCOLLECT_GCC_OPTIONS='-v' '-o' 'output' '-mtune=generic' '-march=x86-64'\n /usr/libexec/gcc/x86_64-redhat-linux/7/cc1 -quiet -v main.c -quiet -dumpbase main.c -mtune=generic -march=x86-64 -auxbase main -version -o /tmp/ccM76aqK.s\nGNU C11 (GCC) version 7.3.1 20180130 (Red Hat 7.3.1-2) (x86_64-redhat-linux)\n\tcompiled by GNU C version 7.3.1 20180130 (Red Hat 7.3.1-2), GMP version 6.1.2, MPFR version 3.1.5, MPC version 1.0.2, isl version isl-0.16.1-GMP\n\nGGC heuristics: --param ggc-min-expand=100 --param ggc-min-heapsize=131072\nignoring nonexistent directory \"/usr/lib/gcc/x86_64-redhat-linux/7/include-fixed\"\nignoring nonexistent directory \"/usr/lib/gcc/x86_64-redhat-linux/7/../../../../x86_64-redhat-linux/include\"\n#include \"...\" search starts here:\n#include <...> search starts here:\n /usr/lib/gcc/x86_64-redhat-linux/7/include\n /usr/local/include\n /usr/include\nEnd of search list.\nGNU C11 (GCC) version 7.3.1 20180130 (Red Hat 7.3.1-2) (x86_64-redhat-linux)\n\tcompiled by GNU C version 7.3.1 20180130 (Red Hat 7.3.1-2), GMP version 6.1.2, MPFR version 3.1.5, MPC version 1.0.2, isl version isl-0.16.1-GMP\n\nGGC heuristics: --param ggc-min-expand=100 --param ggc-min-heapsize=131072\nCompiler executable checksum: ad7c3a488cf591743af375264d348c5c\nCOLLECT_GCC_OPTIONS='-v' '-o' 'output' '-mtune=generic' '-march=x86-64'\n as -v --64 -o /tmp/ccYFphwj.o /tmp/ccM76aqK.s\nGNU assembler version 2.27 (x86_64-redhat-linux) using BFD version version 2.27-28.fc26\nCOMPILER_PATH=/usr/libexec/gcc/x86_64-redhat-linux/7/:/usr/libexec/gcc/x86_64-redhat-linux/7/:/usr/libexec/gcc/x86_64-redhat-linux/:/usr/lib/gcc/x86_64-redhat-linux/7/:/usr/lib/gcc/x86_64-redhat-linux/\nLIBRARY_PATH=/usr/lib/gcc/x86_64-redhat-linux/7/:/usr/lib/gcc/x86_64-redhat-linux/7/../../../../lib64/:/lib/../lib64/:/usr/lib/../lib64/:/usr/lib/gcc/x86_64-redhat-linux/7/../../../:/lib/:/usr/lib/\nCOLLECT_GCC_OPTIONS='-v' '-o' 'output' '-mtune=generic' '-march=x86-64'\n /usr/libexec/gcc/x86_64-redhat-linux/7/collect2 -plugin /usr/libexec/gcc/x86_64-redhat-linux/7/liblto_plugin.so -plugin-opt=/usr/libexec/gcc/x86_64-redhat-linux/7/lto-wrapper -plugin-opt=-fresolution=/tmp/ccw0b6CS.res -plugin-opt=-pass-through=-lgcc -plugin-opt=-pass-through=-lgcc_s -plugin-opt=-pass-through=-lc -plugin-opt=-pass-through=-lgcc -plugin-opt=-pass-through=-lgcc_s --build-id --no-add-needed --eh-frame-hdr --hash-style=gnu -m elf_x86_64 -dynamic-linker /lib64/ld-linux-x86-64.so.2 -o output /usr/lib/gcc/x86_64-redhat-linux/7/../../../../lib64/crt1.o /usr/lib/gcc/x86_64-redhat-linux/7/../../../../lib64/crti.o /usr/lib/gcc/x86_64-redhat-linux/7/crtbegin.o -L/usr/lib/gcc/x86_64-redhat-linux/7 -L/usr/lib/gcc/x86_64-redhat-linux/7/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/lib/gcc/x86_64-redhat-linux/7/../../.. /tmp/ccYFphwj.o -lgcc --as-needed -lgcc_s --no-as-needed -lc -lgcc --as-needed -lgcc_s --no-as-needed /usr/lib/gcc/x86_64-redhat-linux/7/crtend.o /usr/lib/gcc/x86_64-redhat-linux/7/../../../../lib64/crtn.o\nCOLLECT_GCC_OPTIONS='-v' '-o' 'output' '-mtune=generic' '-march=x86-64'\n" }, { "path": "lib/spack/spack/test/data/compiler_verbose_output/icc-16.0.3.txt", "content": "icc.orig version 16.0.3 (gcc version 4.9.3 compatibility)\nld /lib/../lib64/crt1.o /lib/../lib64/crti.o /usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3/crtbegin.o --eh-frame-hdr --build-id -dynamic-linker /lib64/ld-linux-x86-64.so.2 -m elf_x86_64 -o blah -L/usr/tce/packages/intel/intel-16.0.3/compilers_and_libraries_2016.3.210/linux/compiler/lib/intel64_lin -L/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3/ -L/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3/../../../../lib64 -L/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3/../../../../lib64/ -L/lib/../lib64 -L/lib/../lib64/ -L/usr/lib/../lib64 -L/usr/lib/../lib64/ -L/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3/../../../ -L/lib64 -L/lib/ -L/usr/lib64 -L/usr/lib -rpath /usr/tce/packages/intel/intel-16.0.3/lib/intel64 -rpath=/usr/tce/packages/gcc/default/lib64 -Bdynamic -Bstatic -limf -lsvml -lirng -Bdynamic -lm -Bstatic -lipgo -ldecimal --as-needed -Bdynamic -lcilkrts -lstdc++ --no-as-needed -lgcc -lgcc_s -Bstatic -lirc -lsvml -Bdynamic -lc -lgcc -lgcc_s -Bstatic -lirc_s -Bdynamic -ldl -lc /usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3/crtend.o /lib/../lib64/crtn.o\n/lib/../lib64/crt1.o: In function `_start':\n(.text+0x20): undefined reference to `main'\n" }, { "path": "lib/spack/spack/test/data/compiler_verbose_output/nag-6.2-gcc-6.5.0.txt", "content": "NAG Fortran Compiler Release 6.2(Chiyoda) Build 6223\nReading specs from /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/specs\nCOLLECT_GCC=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/bin/gcc\nCOLLECT_LTO_WRAPPER=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/lto-wrapper\nTarget: x86_64-pc-linux-gnu\nConfigured with: /tmp/m300488/spack-stage/spack-stage-gcc-6.5.0-4sdjgrsboy3lowtq3t7pmp7rx3ogkqtz/spack-src/configure --prefix=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs --with-pkgversion='Spack GCC' --with-bugurl=https://github.com/spack/spack/issues --disable-multilib --enable-languages=c,c++,fortran --disable-nls --with-mpfr=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/mpfr-3.1.6-w63rspk --with-gmp=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gmp-6.1.2-et64cuj --with-system-zlib --with-mpc=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/mpc-1.1.0-en66k4t --with-isl=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/isl-0.18-62v4uyg\nThread model: posix\ngcc version 6.5.0 (Spack GCC) \nCOMPILER_PATH=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/\nLIBRARY_PATH=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/../../../../lib64/:/lib/x86_64-linux-gnu/:/lib/../lib64/:/usr/lib/x86_64-linux-gnu/:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/../../../:/lib/:/usr/lib/\nCOLLECT_GCC_OPTIONS='-m64' '-o' 'output' '-v' '-mtune=generic' '-march=x86-64'\n /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/collect2 -plugin /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/liblto_plugin.so -plugin-opt=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/lto-wrapper -plugin-opt=-fresolution=/tmp/ccBpU203.res -plugin-opt=-pass-through=-lgcc -plugin-opt=-pass-through=-lgcc_s -plugin-opt=-pass-through=-lc -plugin-opt=-pass-through=-lgcc -plugin-opt=-pass-through=-lgcc_s -rpath /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib64 --eh-frame-hdr -m elf_x86_64 -dynamic-linker /lib64/ld-linux-x86-64.so.2 -o output /usr/lib/x86_64-linux-gnu/crt1.o /usr/lib/x86_64-linux-gnu/crti.o /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/crtbegin.o -L/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0 -L/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/../../../../lib64 -L/lib/x86_64-linux-gnu -L/lib/../lib64 -L/usr/lib/x86_64-linux-gnu -L/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/../../.. /sw/stretch-x64/nag/nag-6.2/lib/NAG_Fortran/f62init.o /sw/stretch-x64/nag/nag-6.2/lib/NAG_Fortran/quickfit.o /tmp/main.000786.o -rpath /sw/stretch-x64/nag/nag-6.2/lib/NAG_Fortran /sw/stretch-x64/nag/nag-6.2/lib/NAG_Fortran/libf62rts.so /sw/stretch-x64/nag/nag-6.2/lib/NAG_Fortran/libf62rts.a -lm -lgcc --as-needed -lgcc_s --no-as-needed -lc -lgcc --as-needed -lgcc_s --no-as-needed /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/crtend.o /usr/lib/x86_64-linux-gnu/crtn.o\nCOLLECT_GCC_OPTIONS='-m64' '-o' 'output' '-v' '-mtune=generic' '-march=x86-64'\n" }, { "path": "lib/spack/spack/test/data/compiler_verbose_output/obscure-parsing-rules.txt", "content": "This is synthetic data to test parsing cases for which I could not find compiler output\nld -LIBPATH:/first/path /LIBPATH:/second/path -libpath:/third/path\ncollect2 version ld -LIBPATH:/skip/path -L/skip/this/too\n" }, { "path": "lib/spack/spack/test/data/compiler_verbose_output/xl-13.1.5.txt", "content": "export XL_CONFIG=/opt/ibm/xlC/13.1.5/etc/xlc.cfg.centos.7.gcc.4.8.5:xlc \n/usr/bin/ld --eh-frame-hdr -Qy -melf64lppc /usr/lib/gcc/ppc64le-redhat-linux/4.8.5/../../../../lib64/crt1.o /usr/lib/gcc/ppc64le-redhat-linux/4.8.5/../../../../lib64/crti.o /usr/lib/gcc/ppc64le-redhat-linux/4.8.5/crtbegin.o -L/opt/ibm/xlsmp/4.1.5/lib -L/opt/ibm/xlmass/8.1.5/lib -L/opt/ibm/xlC/13.1.5/lib -R/opt/ibm/lib -L/usr/lib/gcc/ppc64le-redhat-linux/4.8.5 -L/usr/lib/gcc/ppc64le-redhat-linux/4.8.5/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/lib/gcc/ppc64le-redhat-linux/4.8.5/../../.. --no-toc-optimize -o foo foo.o -dynamic-linker /lib64/ld64.so.2 --enable-new-dtags -lxlopt -lxl --as-needed -ldl --no-as-needed -lgcc_s --as-needed -lpthread --no-as-needed -lgcc -lm -lc -lgcc_s -lgcc /usr/lib/gcc/ppc64le-redhat-linux/4.8.5/crtend.o /usr/lib/gcc/ppc64le-redhat-linux/4.8.5/../../../../lib64/crtn.o \nrm /tmp/xlcW0iQ4uI8\nrm /tmp/xlcW1aPLBFY\nrm /tmp/xlcW2ALFICO\n" }, { "path": "lib/spack/spack/test/data/compression/Foo", "content": "TEST\n" }, { "path": "lib/spack/spack/test/data/compression/Foo.cxx", "content": "" }, { "path": "lib/spack/spack/test/data/config/base/config.yaml", "content": "# This file is here strictly so that the base include directory will work\nconfig:\n dirty: false\n" }, { "path": "lib/spack/spack/test/data/config/bootstrap.yaml", "content": "bootstrap:\n sources:\n - name: 'github-actions'\n metadata: $spack/share/spack/bootstrap/github-actions-v2\n trusted: {}\n" }, { "path": "lib/spack/spack/test/data/config/concretizer.yaml", "content": "concretizer:\n reuse: true\n targets:\n granularity: microarchitectures\n host_compatible: false\n\n duplicates:\n strategy: minimal\n max_dupes:\n default: 1\n # Virtuals\n c: 2\n cxx: 2\n fortran: 1\n # Regular packages\n cmake: 2\n gmake: 2\n python: 2\n python-venv: 2\n py-cython: 2\n py-flit-core: 2\n py-pip: 2\n py-setuptools: 2\n py-versioneer: 2\n py-wheel: 2\n xcb-proto: 2\n # Compilers\n gcc: 2\n llvm: 2\n\n concretization_cache:\n enable: false\n" }, { "path": "lib/spack/spack/test/data/config/config.yaml", "content": "config:\n install_tree:\n root: {0}\n template_dirs:\n - $spack/share/spack/templates\n - $spack/lib/spack/spack/test/data/templates\n - $spack/lib/spack/spack/test/data/templates_again\n build_stage:\n - $tempdir/$user/spack-stage\n source_cache: $user_cache_path/source\n misc_cache: $user_cache_path/cache\n verify_ssl: true\n ssl_certs: $SSL_CERT_FILE\n checksum: true\n installer: old # many tests are based on stdout from old installer\n dirty: false\n locks: {1}\n" }, { "path": "lib/spack/spack/test/data/config/include.yaml", "content": "include:\n - path: base\n" }, { "path": "lib/spack/spack/test/data/config/modules.yaml", "content": "# -------------------------------------------------------------------------\n# This is the default configuration for Spack's module file generation.\n#\n# Settings here are versioned with Spack and are intended to provide\n# sensible defaults out of the box. Spack maintainers should edit this\n# file to keep it current.\n#\n# Users can override these settings by editing the following files.\n#\n# Per-spack-instance settings (overrides defaults):\n# $SPACK_ROOT/etc/spack/modules.yaml\n#\n# Per-user settings (overrides default and site settings):\n# ~/.spack/modules.yaml\n# -------------------------------------------------------------------------\nmodules:\n prefix_inspections:\n ./bin: [PATH]\n ./man: [MANPATH]\n ./share/man: [MANPATH] \n ./share/aclocal: [ACLOCAL_PATH]\n ./lib/pkgconfig: [PKG_CONFIG_PATH]\n ./lib64/pkgconfig: [PKG_CONFIG_PATH]\n ./share/pkgconfig: [PKG_CONFIG_PATH]\n ./: [CMAKE_PREFIX_PATH]\n default:\n roots:\n tcl: {0}\n lmod: {1}\n enable: []\n tcl:\n all:\n autoload: direct\n lmod:\n all:\n autoload: direct\n hierarchy:\n - mpi\n" }, { "path": "lib/spack/spack/test/data/config/packages.yaml", "content": "packages:\n all:\n providers:\n c: [gcc, llvm]\n cxx: [gcc, llvm]\n fortran: [gcc]\n fortran-rt: [gcc-runtime]\n libc: [glibc]\n libgfortran: [gcc-runtime]\n mpi: [mpich, zmpi]\n lapack: [openblas-with-lapack]\n blas: [openblas]\n externaltool:\n buildable: False\n externals:\n - spec: externaltool@1.0\n prefix: /path/to/external_tool\n - spec: externaltool@0.9\n prefix: /usr\n - spec: externaltool@0_8\n prefix: /usr\n externalvirtual:\n buildable: False\n externals:\n - spec: externalvirtual@2.0\n prefix: /path/to/external_virtual_clang\n - spec: externalvirtual@1.0\n prefix: /path/to/external_virtual_gcc\n externalmodule:\n buildable: False\n externals:\n - spec: externalmodule@1.0\n modules:\n - external-module\n 'requires-virtual':\n buildable: False\n externals:\n - spec: requires-virtual@2.0\n prefix: /usr\n 'external-buildable-with-variant':\n buildable: True\n externals:\n - spec: external-buildable-with-variant@1.1.special +baz\n prefix: /usr\n - spec: external-buildable-with-variant@0.9 +baz\n prefix: /usr\n 'old-external':\n buildable: True\n externals:\n - spec: old-external@1.0.0\n prefix: /usr\n 'external-non-default-variant':\n buildable: True\n externals:\n - spec: external-non-default-variant@3.8.7~foo~bar\n prefix: /usr\n version-test-dependency-preferred:\n version: ['5.2.5']\n\n # Compilers\n gcc:\n externals:\n - spec: \"gcc@9.4.0 languages='c,c++' os={linux_os.name}{linux_os.version}\"\n prefix: /path\n extra_attributes:\n compilers:\n c: /path/bin/gcc\n cxx: /path/bin/g++\n - spec: \"gcc@9.4.1 languages='c,c++' os=redhat6\"\n prefix: /path\n extra_attributes:\n compilers:\n c: /path/bin/gcc\n cxx: /path/bin/g++\n - spec: \"gcc@10.2.1 languages='c,c++,fortran' os={linux_os.name}{linux_os.version}\"\n prefix: /path\n extra_attributes:\n compilers:\n c: /path/bin/gcc-10\n cxx: /path/bin/g++-10\n fortran: /path/bin/gfortran-10\n llvm:\n externals:\n - spec: \"llvm@15.0.0 +clang~flang os={linux_os.name}{linux_os.version}\"\n prefix: /path\n extra_attributes:\n compilers:\n c: /path/bin/clang\n cxx: /path/bin/clang++\n glibc:\n buildable: false\n" }, { "path": "lib/spack/spack/test/data/config/repos.yaml", "content": "repos:\n builtin_mock: $spack/var/spack/test_repos/spack_repo/builtin_mock\n" }, { "path": "lib/spack/spack/test/data/conftest/diff-test/package-0.txt", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nfrom spack.package_base import PackageBase\nfrom spack.package import *\n\n\nclass DiffTest(PackageBase):\n \"\"\"zlib replacement with optimizations for next generation systems.\"\"\"\n\n homepage = \"https://github.com/zlib-ng/zlib-ng\"\n url = \"https://github.com/zlib-ng/zlib-ng/archive/2.0.0.tar.gz\"\n git = \"https://github.com/zlib-ng/zlib-ng.git\"\n\n license(\"Zlib\")\n\n version(\"2.1.4\", sha256=\"a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a\")\n version(\"2.0.0\", sha256=\"86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8\")\n version(\"2.0.7\", sha256=\"6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200\")\n" }, { "path": "lib/spack/spack/test/data/conftest/diff-test/package-1.txt", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nfrom spack.package_base import PackageBase\nfrom spack.package import *\n\n\nclass DiffTest(PackageBase):\n \"\"\"zlib replacement with optimizations for next generation systems.\"\"\"\n\n homepage = \"https://github.com/zlib-ng/zlib-ng\"\n url = \"https://github.com/zlib-ng/zlib-ng/archive/2.0.0.tar.gz\"\n git = \"https://github.com/zlib-ng/zlib-ng.git\"\n\n license(\"Zlib\")\n\n version(\"2.1.5\", sha256=\"3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04\")\n version(\"2.1.4\", sha256=\"a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a\")\n version(\"2.0.7\", sha256=\"6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200\")\n version(\"2.0.0\", sha256=\"86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8\")\n" }, { "path": "lib/spack/spack/test/data/conftest/diff-test/package-2.txt", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nfrom spack.package_base import PackageBase\nfrom spack.package import *\n\n\nclass DiffTest(PackageBase):\n \"\"\"zlib replacement with optimizations for next generation systems.\"\"\"\n\n homepage = \"https://github.com/zlib-ng/zlib-ng\"\n url = \"https://github.com/zlib-ng/zlib-ng/archive/2.0.0.tar.gz\"\n git = \"https://github.com/zlib-ng/zlib-ng.git\"\n\n license(\"Zlib\")\n\n version(\"2.1.6\", tag=\"2.1.6\", commit=\"74253725f884e2424a0dd8ae3f69896d5377f325\")\n version(\"2.1.5\", sha256=\"3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04\")\n version(\"2.1.4\", sha256=\"a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a\")\n version(\"2.0.7\", sha256=\"6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200\")\n version(\"2.0.0\", sha256=\"86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8\")\n" }, { "path": "lib/spack/spack/test/data/conftest/diff-test/package-3.txt", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nfrom spack.package_base import PackageBase\nfrom spack.package import *\n\n\nclass DiffTest(PackageBase):\n \"\"\"zlib replacement with optimizations for next generation systems.\"\"\"\n\n homepage = \"https://github.com/zlib-ng/zlib-ng\"\n url = \"https://github.com/zlib-ng/zlib-ng/archive/2.0.0.tar.gz\"\n git = \"https://github.com/zlib-ng/zlib-ng.git\"\n\n license(\"Zlib\")\n\n version(\"2.1.8\", sha256=\"59e68f67cbb16999842daeb517cdd86fc25b177b4affd335cd72b76ddc2a46d8\")\n version(\"2.1.7\", sha256=\"59e68f67cbb16999842daeb517cdd86fc25b177b4affd335cd72b76ddc2a46d8\")\n version(\"2.1.6\", tag=\"2.1.6\", commit=\"74253725f884e2424a0dd8ae3f69896d5377f325\")\n version(\"2.1.5\", sha256=\"3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04\")\n version(\"2.1.4\", sha256=\"a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a\")\n version(\"2.0.7\", sha256=\"6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200\")\n version(\"2.0.0\", sha256=\"86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8\")\n" }, { "path": "lib/spack/spack/test/data/directory_search/README.txt", "content": "This directory tree is made up to test that search functions will return a stable ordered sequence." }, { "path": "lib/spack/spack/test/data/directory_search/a/c.h", "content": "" }, { "path": "lib/spack/spack/test/data/directory_search/a/foobar.txt", "content": "" }, { "path": "lib/spack/spack/test/data/directory_search/a/libc.a", "content": "" }, { "path": "lib/spack/spack/test/data/directory_search/a/libc.lib", "content": "" }, { "path": "lib/spack/spack/test/data/directory_search/b/b.h", "content": "" }, { "path": "lib/spack/spack/test/data/directory_search/b/bar.txp", "content": "" }, { "path": "lib/spack/spack/test/data/directory_search/b/d.h", "content": "" }, { "path": "lib/spack/spack/test/data/directory_search/b/liba.a", "content": "" }, { "path": "lib/spack/spack/test/data/directory_search/b/liba.lib", "content": "" }, { "path": "lib/spack/spack/test/data/directory_search/b/libd.a", "content": "" }, { "path": "lib/spack/spack/test/data/directory_search/b/libd.lib", "content": "" }, { "path": "lib/spack/spack/test/data/directory_search/c/a.h", "content": "" }, { "path": "lib/spack/spack/test/data/directory_search/c/bar.txt", "content": "" }, { "path": "lib/spack/spack/test/data/directory_search/c/libb.a", "content": "" }, { "path": "lib/spack/spack/test/data/directory_search/c/libb.lib", "content": "" }, { "path": "lib/spack/spack/test/data/filter_file/start_stop.txt", "content": "A\nB\nC\nD" }, { "path": "lib/spack/spack/test/data/filter_file/x86_cpuid_info.c", "content": "/****************************/\n/* THIS IS OPEN SOURCE CODE */\n/****************************/\n\n/* \n* File: x86_cpuid_info.c\n* Author: Dan Terpstra\n* terpstra@eecs.utk.edu\n* complete rewrite of linux-memory.c to conform to latest docs\n* and convert Intel to a table driven implementation.\n* Now also supports multiple TLB descriptors\n*/\n\n#include \n#include \n#include \n#include \"papi.h\"\n#include \"papi_internal.h\"\n\nstatic void init_mem_hierarchy( PAPI_mh_info_t * mh_info );\nstatic int init_amd( PAPI_mh_info_t * mh_info, int *levels );\nstatic short int _amd_L2_L3_assoc( unsigned short int pattern );\nstatic int init_intel( PAPI_mh_info_t * mh_info , int *levels);\n\n#if defined( __amd64__ ) || defined (__x86_64__)\nstatic inline void\ncpuid( unsigned int *a, unsigned int *b, unsigned int *c, unsigned int *d )\n{\n\tunsigned int op = *a;\n\t__asm__(\"cpuid;\"\n\t\t: \"=a\" (*a), \"=b\" (*b), \"=c\" (*c), \"=d\" (*d)\n\t\t: \"a\" (op) );\n}\n#else\nstatic inline void\ncpuid( unsigned int *a, unsigned int *b, unsigned int *c, unsigned int *d )\n{\n\tunsigned int op = *a;\n\t// .byte 0x53 == push ebx. it's universal for 32 and 64 bit\n\t// .byte 0x5b == pop ebx.\n\t// Some gcc's (4.1.2 on Core2) object to pairing push/pop and ebx in 64 bit mode.\n\t// Using the opcode directly avoids this problem.\n __asm__ __volatile__( \".byte 0x53\\n\\tcpuid\\n\\tmovl %%ebx, %%esi\\n\\t.byte 0x5b\":\"=a\"( *a ), \"=S\"( *b ), \"=c\"( *c ),\n\t\t\t\t\t\t \"=d\"\n\t\t\t\t\t\t ( *d )\n :\t\t\t\t\t \"a\"( op ) );\n}\n#endif\n\nint\n_x86_cache_info( PAPI_mh_info_t * mh_info )\n{\n\tint retval = 0;\n\tunion\n\t{\n\t\tstruct\n\t\t{\n\t\t\tunsigned int ax, bx, cx, dx;\n\t\t} e;\n\t\tchar vendor[20];\t\t\t /* leave room for terminator bytes */\n\t} reg;\n\n\t/* Don't use cpu_type to determine the processor.\n\t * get the information directly from the chip.\n\t */\n\treg.e.ax = 0;\t\t\t /* function code 0: vendor string */\n\t/* The vendor string is composed of EBX:EDX:ECX.\n\t * by swapping the register addresses in the call below,\n\t * the string is correctly composed in the char array.\n\t */\n\tcpuid( ®.e.ax, ®.e.bx, ®.e.dx, ®.e.cx );\n\treg.vendor[16] = 0;\n\tMEMDBG( \"Vendor: %s\\n\", ®.vendor[4] );\n\n\tinit_mem_hierarchy( mh_info );\n\n\tif ( !strncmp( \"GenuineIntel\", ®.vendor[4], 12 ) ) {\n\t init_intel( mh_info, &mh_info->levels);\n\t} else if ( !strncmp( \"AuthenticAMD\", ®.vendor[4], 12 ) ) {\n\t init_amd( mh_info, &mh_info->levels );\n\t} else {\n\t\tMEMDBG( \"Unsupported cpu type; Not Intel or AMD x86\\n\" );\n\t\treturn PAPI_ENOIMPL;\n\t}\n\n\t/* This works only because an empty cache element is initialized to 0 */\n\tMEMDBG( \"Detected L1: %d L2: %d L3: %d\\n\",\n\t\t\tmh_info->level[0].cache[0].size + mh_info->level[0].cache[1].size,\n\t\t\tmh_info->level[1].cache[0].size + mh_info->level[1].cache[1].size,\n\t\t\tmh_info->level[2].cache[0].size + mh_info->level[2].cache[1].size );\n\treturn retval;\n}\n\nstatic void\ninit_mem_hierarchy( PAPI_mh_info_t * mh_info )\n{\n\tint i, j;\n\tPAPI_mh_level_t *L = mh_info->level;\n\n\t/* initialize entire memory hierarchy structure to benign values */\n\tfor ( i = 0; i < PAPI_MAX_MEM_HIERARCHY_LEVELS; i++ ) {\n\t\tfor ( j = 0; j < PAPI_MH_MAX_LEVELS; j++ ) {\n\t\t\tL[i].tlb[j].type = PAPI_MH_TYPE_EMPTY;\n\t\t\tL[i].tlb[j].num_entries = 0;\n\t\t\tL[i].tlb[j].associativity = 0;\n\t\t\tL[i].cache[j].type = PAPI_MH_TYPE_EMPTY;\n\t\t\tL[i].cache[j].size = 0;\n\t\t\tL[i].cache[j].line_size = 0;\n\t\t\tL[i].cache[j].num_lines = 0;\n\t\t\tL[i].cache[j].associativity = 0;\n\t\t}\n\t}\n}\n\nstatic short int\n_amd_L2_L3_assoc( unsigned short int pattern )\n{\n\t/* From \"CPUID Specification\" #25481 Rev 2.28, April 2008 */\n\tshort int assoc[16] =\n\t\t{ 0, 1, 2, -1, 4, -1, 8, -1, 16, -1, 32, 48, 64, 96, 128, SHRT_MAX };\n\tif ( pattern > 0xF )\n\t\treturn -1;\n\treturn ( assoc[pattern] );\n}\n\n/* Cache configuration for AMD Athlon/Duron */\nstatic int\ninit_amd( PAPI_mh_info_t * mh_info, int *num_levels )\n{\n\tunion\n\t{\n\t\tstruct\n\t\t{\n\t\t\tunsigned int ax, bx, cx, dx;\n\t\t} e;\n\t\tunsigned char byt[16];\n\t} reg;\n\tint i, j, levels = 0;\n\tPAPI_mh_level_t *L = mh_info->level;\n\n\t/*\n\t * Layout of CPU information taken from :\n\t * \"CPUID Specification\" #25481 Rev 2.28, April 2008 for most current info.\n\t */\n\n\tMEMDBG( \"Initializing AMD memory info\\n\" );\n\t/* AMD level 1 cache info */\n\treg.e.ax = 0x80000005;\t /* extended function code 5: L1 Cache and TLB Identifiers */\n\tcpuid( ®.e.ax, ®.e.bx, ®.e.cx, ®.e.dx );\n\n\tMEMDBG( \"e.ax=%#8.8x e.bx=%#8.8x e.cx=%#8.8x e.dx=%#8.8x\\n\",\n\t\t\treg.e.ax, reg.e.bx, reg.e.cx, reg.e.dx );\n\tMEMDBG\n\t\t( \":\\neax: %#x %#x %#x %#x\\nebx: %#x %#x %#x %#x\\necx: %#x %#x %#x %#x\\nedx: %#x %#x %#x %#x\\n\",\n\t\t reg.byt[0], reg.byt[1], reg.byt[2], reg.byt[3], reg.byt[4],\n\t\t reg.byt[5], reg.byt[6], reg.byt[7], reg.byt[8], reg.byt[9],\n\t\t reg.byt[10], reg.byt[11], reg.byt[12], reg.byt[13], reg.byt[14],\n\t\t reg.byt[15] );\n\n\t/* NOTE: We assume L1 cache and TLB always exists */\n\t/* L1 TLB info */\n\n\t/* 4MB memory page information; half the number of entries as 2MB */\n\tL[0].tlb[0].type = PAPI_MH_TYPE_INST;\n\tL[0].tlb[0].num_entries = reg.byt[0] / 2;\n\tL[0].tlb[0].page_size = 4096 << 10;\n\tL[0].tlb[0].associativity = reg.byt[1];\n\n\tL[0].tlb[1].type = PAPI_MH_TYPE_DATA;\n\tL[0].tlb[1].num_entries = reg.byt[2] / 2;\n\tL[0].tlb[1].page_size = 4096 << 10;\n\tL[0].tlb[1].associativity = reg.byt[3];\n\n\t/* 2MB memory page information */\n\tL[0].tlb[2].type = PAPI_MH_TYPE_INST;\n\tL[0].tlb[2].num_entries = reg.byt[0];\n\tL[0].tlb[2].page_size = 2048 << 10;\n\tL[0].tlb[2].associativity = reg.byt[1];\n\n\tL[0].tlb[3].type = PAPI_MH_TYPE_DATA;\n\tL[0].tlb[3].num_entries = reg.byt[2];\n\tL[0].tlb[3].page_size = 2048 << 10;\n\tL[0].tlb[3].associativity = reg.byt[3];\n\n\t/* 4k page information */\n\tL[0].tlb[4].type = PAPI_MH_TYPE_INST;\n\tL[0].tlb[4].num_entries = reg.byt[4];\n\tL[0].tlb[4].page_size = 4 << 10;\n\tL[0].tlb[4].associativity = reg.byt[5];\n\n\tL[0].tlb[5].type = PAPI_MH_TYPE_DATA;\n\tL[0].tlb[5].num_entries = reg.byt[6];\n\tL[0].tlb[5].page_size = 4 << 10;\n\tL[0].tlb[5].associativity = reg.byt[7];\n\n\tfor ( i = 0; i < PAPI_MH_MAX_LEVELS; i++ ) {\n\t\tif ( L[0].tlb[i].associativity == 0xff )\n\t\t\tL[0].tlb[i].associativity = SHRT_MAX;\n\t}\n\n\t/* L1 D-cache info */\n\tL[0].cache[0].type =\n\t\tPAPI_MH_TYPE_DATA | PAPI_MH_TYPE_WB | PAPI_MH_TYPE_PSEUDO_LRU;\n\tL[0].cache[0].size = reg.byt[11] << 10;\n\tL[0].cache[0].associativity = reg.byt[10];\n\tL[0].cache[0].line_size = reg.byt[8];\n\t/* Byt[9] is \"Lines per tag\" */\n\t/* Is that == lines per cache? */\n\t/* L[0].cache[1].num_lines = reg.byt[9]; */\n\tif ( L[0].cache[0].line_size )\n\t\tL[0].cache[0].num_lines = L[0].cache[0].size / L[0].cache[0].line_size;\n\tMEMDBG( \"D-Cache Line Count: %d; Computed: %d\\n\", reg.byt[9],\n\t\t\tL[0].cache[0].num_lines );\n\n\t/* L1 I-cache info */\n\tL[0].cache[1].type = PAPI_MH_TYPE_INST;\n\tL[0].cache[1].size = reg.byt[15] << 10;\n\tL[0].cache[1].associativity = reg.byt[14];\n\tL[0].cache[1].line_size = reg.byt[12];\n\t/* Byt[13] is \"Lines per tag\" */\n\t/* Is that == lines per cache? */\n\t/* L[0].cache[1].num_lines = reg.byt[13]; */\n\tif ( L[0].cache[1].line_size )\n\t\tL[0].cache[1].num_lines = L[0].cache[1].size / L[0].cache[1].line_size;\n\tMEMDBG( \"I-Cache Line Count: %d; Computed: %d\\n\", reg.byt[13],\n\t\t\tL[0].cache[1].num_lines );\n\n\tfor ( i = 0; i < 2; i++ ) {\n\t\tif ( L[0].cache[i].associativity == 0xff )\n\t\t\tL[0].cache[i].associativity = SHRT_MAX;\n\t}\n\n\t/* AMD L2/L3 Cache and L2 TLB info */\n\t/* NOTE: For safety we assume L2 and L3 cache and TLB may not exist */\n\n\treg.e.ax = 0x80000006;\t /* extended function code 6: L2/L3 Cache and L2 TLB Identifiers */\n\tcpuid( ®.e.ax, ®.e.bx, ®.e.cx, ®.e.dx );\n\n\tMEMDBG( \"e.ax=%#8.8x e.bx=%#8.8x e.cx=%#8.8x e.dx=%#8.8x\\n\",\n\t\t\treg.e.ax, reg.e.bx, reg.e.cx, reg.e.dx );\n\tMEMDBG\n\t\t( \":\\neax: %#x %#x %#x %#x\\nebx: %#x %#x %#x %#x\\necx: %#x %#x %#x %#x\\nedx: %#x %#x %#x %#x\\n\",\n\t\t reg.byt[0], reg.byt[1], reg.byt[2], reg.byt[3], reg.byt[4],\n\t\t reg.byt[5], reg.byt[6], reg.byt[7], reg.byt[8], reg.byt[9],\n\t\t reg.byt[10], reg.byt[11], reg.byt[12], reg.byt[13], reg.byt[14],\n\t\t reg.byt[15] );\n\n\t/* L2 TLB info */\n\n\tif ( reg.byt[0] | reg.byt[1] ) {\t/* Level 2 ITLB exists */\n\t\t/* 4MB ITLB page information; half the number of entries as 2MB */\n\t\tL[1].tlb[0].type = PAPI_MH_TYPE_INST;\n\t\tL[1].tlb[0].num_entries =\n\t\t\t( ( ( short ) ( reg.byt[1] & 0xF ) << 8 ) + reg.byt[0] ) / 2;\n\t\tL[1].tlb[0].page_size = 4096 << 10;\n\t\tL[1].tlb[0].associativity =\n\t\t\t_amd_L2_L3_assoc( ( reg.byt[1] & 0xF0 ) >> 4 );\n\n\t\t/* 2MB ITLB page information */\n\t\tL[1].tlb[2].type = PAPI_MH_TYPE_INST;\n\t\tL[1].tlb[2].num_entries = L[1].tlb[0].num_entries * 2;\n\t\tL[1].tlb[2].page_size = 2048 << 10;\n\t\tL[1].tlb[2].associativity = L[1].tlb[0].associativity;\n\t}\n\n\tif ( reg.byt[2] | reg.byt[3] ) {\t/* Level 2 DTLB exists */\n\t\t/* 4MB DTLB page information; half the number of entries as 2MB */\n\t\tL[1].tlb[1].type = PAPI_MH_TYPE_DATA;\n\t\tL[1].tlb[1].num_entries =\n\t\t\t( ( ( short ) ( reg.byt[3] & 0xF ) << 8 ) + reg.byt[2] ) / 2;\n\t\tL[1].tlb[1].page_size = 4096 << 10;\n\t\tL[1].tlb[1].associativity =\n\t\t\t_amd_L2_L3_assoc( ( reg.byt[3] & 0xF0 ) >> 4 );\n\n\t\t/* 2MB DTLB page information */\n\t\tL[1].tlb[3].type = PAPI_MH_TYPE_DATA;\n\t\tL[1].tlb[3].num_entries = L[1].tlb[1].num_entries * 2;\n\t\tL[1].tlb[3].page_size = 2048 << 10;\n\t\tL[1].tlb[3].associativity = L[1].tlb[1].associativity;\n\t}\n\n\t/* 4k page information */\n\tif ( reg.byt[4] | reg.byt[5] ) {\t/* Level 2 ITLB exists */\n\t\tL[1].tlb[4].type = PAPI_MH_TYPE_INST;\n\t\tL[1].tlb[4].num_entries =\n\t\t\t( ( short ) ( reg.byt[5] & 0xF ) << 8 ) + reg.byt[4];\n\t\tL[1].tlb[4].page_size = 4 << 10;\n\t\tL[1].tlb[4].associativity =\n\t\t\t_amd_L2_L3_assoc( ( reg.byt[5] & 0xF0 ) >> 4 );\n\t}\n\tif ( reg.byt[6] | reg.byt[7] ) {\t/* Level 2 DTLB exists */\n\t\tL[1].tlb[5].type = PAPI_MH_TYPE_DATA;\n\t\tL[1].tlb[5].num_entries =\n\t\t\t( ( short ) ( reg.byt[7] & 0xF ) << 8 ) + reg.byt[6];\n\t\tL[1].tlb[5].page_size = 4 << 10;\n\t\tL[1].tlb[5].associativity =\n\t\t\t_amd_L2_L3_assoc( ( reg.byt[7] & 0xF0 ) >> 4 );\n\t}\n\n\t/* AMD Level 2 cache info */\n\tif ( reg.e.cx ) {\n\t\tL[1].cache[0].type =\n\t\t\tPAPI_MH_TYPE_UNIFIED | PAPI_MH_TYPE_WT | PAPI_MH_TYPE_PSEUDO_LRU;\n\t\tL[1].cache[0].size = ( int ) ( ( reg.e.cx & 0xffff0000 ) >> 6 );\t/* right shift by 16; multiply by 2^10 */\n\t\tL[1].cache[0].associativity =\n\t\t\t_amd_L2_L3_assoc( ( reg.byt[9] & 0xF0 ) >> 4 );\n\t\tL[1].cache[0].line_size = reg.byt[8];\n/*\t\tL[1].cache[0].num_lines = reg.byt[9]&0xF; */\n\t\tif ( L[1].cache[0].line_size )\n\t\t\tL[1].cache[0].num_lines =\n\t\t\t\tL[1].cache[0].size / L[1].cache[0].line_size;\n\t\tMEMDBG( \"U-Cache Line Count: %d; Computed: %d\\n\", reg.byt[9] & 0xF,\n\t\t\t\tL[1].cache[0].num_lines );\n\t}\n\n\t/* AMD Level 3 cache info (shared across cores) */\n\tif ( reg.e.dx ) {\n\t\tL[2].cache[0].type =\n\t\t\tPAPI_MH_TYPE_UNIFIED | PAPI_MH_TYPE_WT | PAPI_MH_TYPE_PSEUDO_LRU;\n\t\tL[2].cache[0].size = ( int ) ( reg.e.dx & 0xfffc0000 ) << 1;\t/* in blocks of 512KB (2^19) */\n\t\tL[2].cache[0].associativity =\n\t\t\t_amd_L2_L3_assoc( ( reg.byt[13] & 0xF0 ) >> 4 );\n\t\tL[2].cache[0].line_size = reg.byt[12];\n/*\t\tL[2].cache[0].num_lines = reg.byt[13]&0xF; */\n\t\tif ( L[2].cache[0].line_size )\n\t\t\tL[2].cache[0].num_lines =\n\t\t\t\tL[2].cache[0].size / L[2].cache[0].line_size;\n\t\tMEMDBG( \"U-Cache Line Count: %d; Computed: %d\\n\", reg.byt[13] & 0xF,\n\t\t\t\tL[1].cache[0].num_lines );\n\t}\n\tfor ( i = 0; i < PAPI_MAX_MEM_HIERARCHY_LEVELS; i++ ) {\n\t\tfor ( j = 0; j < PAPI_MH_MAX_LEVELS; j++ ) {\n\t\t\t/* Compute the number of levels of hierarchy actually used */\n\t\t\tif ( L[i].tlb[j].type != PAPI_MH_TYPE_EMPTY ||\n\t\t\t\t L[i].cache[j].type != PAPI_MH_TYPE_EMPTY )\n\t\t\t\tlevels = i + 1;\n\t\t}\n\t}\n\t*num_levels = levels;\n\treturn PAPI_OK;\n}\n\n /*\n * The data from this table now comes from figure 3-17 in\n * the Intel Architectures Software Reference Manual 2A\n * (cpuid instruction section)\n * \n * Pretviously the information was provided by\n * \"Intel Processor Identification and the CPUID Instruction\",\n * Application Note, AP-485, Nov 2008, 241618-033\n * Updated to AP-485, Aug 2009, 241618-036\n *\n * The following data structure and its instantiation trys to\n * capture all the information in Section 2.1.3 of the above\n * document. Not all of it is used by PAPI, but it could be.\n * As the above document is revised, this table should be\n * updated.\n */\n\n#define TLB_SIZES 3\t\t\t /* number of different page sizes for a single TLB descriptor */\nstruct _intel_cache_info\n{\n\tint descriptor;\t\t\t\t\t /* 0x00 - 0xFF: register descriptor code */\n\tint level;\t\t\t\t\t\t /* 1 to PAPI_MH_MAX_LEVELS */\n\tint type;\t\t\t\t\t\t /* Empty, instr, data, vector, unified | TLB */\n\tint size[TLB_SIZES];\t\t\t /* cache or TLB page size(s) in kB */\n\tint associativity;\t\t\t\t /* SHRT_MAX == fully associative */\n\tint sector;\t\t\t\t\t\t /* 1 if cache is sectored; else 0 */\n\tint line_size;\t\t\t\t\t /* for cache */\n\tint entries;\t\t\t\t\t /* for TLB */\n};\n\nstatic struct _intel_cache_info intel_cache[] = {\n// 0x01\n\t{.descriptor = 0x01,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_INST,\n\t .size[0] = 4,\n\t .associativity = 4,\n\t .entries = 32,\n\t },\n// 0x02\n\t{.descriptor = 0x02,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_INST,\n\t .size[0] = 4096,\n\t .associativity = SHRT_MAX,\n\t .entries = 2,\n\t },\n// 0x03\n\t{.descriptor = 0x03,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_DATA,\n\t .size[0] = 4,\n\t .associativity = 4,\n\t .entries = 64,\n\t },\n// 0x04\n\t{.descriptor = 0x04,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_DATA,\n\t .size[0] = 4096,\n\t .associativity = 4,\n\t .entries = 8,\n\t },\n// 0x05\n\t{.descriptor = 0x05,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_DATA,\n\t .size[0] = 4096,\n\t .associativity = 4,\n\t .entries = 32,\n\t },\n// 0x06\n\t{.descriptor = 0x06,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_INST,\n\t .size[0] = 8,\n\t .associativity = 4,\n\t .line_size = 32,\n\t },\n// 0x08\n\t{.descriptor = 0x08,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_INST,\n\t .size[0] = 16,\n\t .associativity = 4,\n\t .line_size = 32,\n\t },\n// 0x09\n\t{.descriptor = 0x09,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_INST,\n\t .size[0] = 32,\n\t .associativity = 4,\n\t .line_size = 64,\n\t },\n// 0x0A\n\t{.descriptor = 0x0A,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_DATA,\n\t .size[0] = 8,\n\t .associativity = 2,\n\t .line_size = 32,\n\t },\n// 0x0B\n\t{.descriptor = 0x0B,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_INST,\n\t .size[0] = 4096,\n\t .associativity = 4,\n\t .entries = 4,\n\t }, \n// 0x0C\n\t{.descriptor = 0x0C,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_DATA,\n\t .size[0] = 16,\n\t .associativity = 4,\n\t .line_size = 32,\n\t },\n// 0x0D\n\t{.descriptor = 0x0D,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_DATA,\n\t .size[0] = 16,\n\t .associativity = 4,\n\t .line_size = 64,\n\t },\n// 0x0E\n\t{.descriptor = 0x0E,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_DATA,\n\t .size[0] = 24,\n\t .associativity = 6,\n\t .line_size = 64,\n\t }, \n// 0x21\n\t{.descriptor = 0x21,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 256,\n\t .associativity = 8,\n\t .line_size = 64,\n\t },\n// 0x22\n\t{.descriptor = 0x22,\n\t .level = 3,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 512,\n\t .associativity = 4,\n\t .sector = 1,\n\t .line_size = 64,\n\t },\n// 0x23\n\t{.descriptor = 0x23,\n\t .level = 3,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 1024,\n\t .associativity = 8,\n\t .sector = 1,\n\t .line_size = 64,\n\t },\n// 0x25\n\t{.descriptor = 0x25,\n\t .level = 3,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 2048,\n\t .associativity = 8,\n\t .sector = 1,\n\t .line_size = 64,\n\t },\n// 0x29\n\t{.descriptor = 0x29,\n\t .level = 3,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 4096,\n\t .associativity = 8,\n\t .sector = 1,\n\t .line_size = 64,\n\t },\n// 0x2C\n\t{.descriptor = 0x2C,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_DATA,\n\t .size[0] = 32,\n\t .associativity = 8,\n\t .line_size = 64,\n\t },\n// 0x30\n\t{.descriptor = 0x30,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_INST,\n\t .size[0] = 32,\n\t .associativity = 8,\n\t .line_size = 64,\n\t },\n// 0x39\n\t{.descriptor = 0x39,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 128,\n\t .associativity = 4,\n\t .sector = 1,\n\t .line_size = 64,\n\t },\n// 0x3A\n\t{.descriptor = 0x3A,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 192,\n\t .associativity = 6,\n\t .sector = 1,\n\t .line_size = 64,\n\t },\n// 0x3B\n\t{.descriptor = 0x3B,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 128,\n\t .associativity = 2,\n\t .sector = 1,\n\t .line_size = 64,\n\t },\n// 0x3C\n\t{.descriptor = 0x3C,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 256,\n\t .associativity = 4,\n\t .sector = 1,\n\t .line_size = 64,\n\t },\n// 0x3D\n\t{.descriptor = 0x3D,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 384,\n\t .associativity = 6,\n\t .sector = 1,\n\t .line_size = 64,\n\t },\n// 0x3E\n\t{.descriptor = 0x3E,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 512,\n\t .associativity = 4,\n\t .sector = 1,\n\t .line_size = 64,\n\t },\n// 0x40: no last level cache (??)\n// 0x41\n\t{.descriptor = 0x41,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 128,\n\t .associativity = 4,\n\t .line_size = 32,\n\t },\n// 0x42\n\t{.descriptor = 0x42,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 256,\n\t .associativity = 4,\n\t .line_size = 32,\n\t },\n// 0x43\n\t{.descriptor = 0x43,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 512,\n\t .associativity = 4,\n\t .line_size = 32,\n\t },\n// 0x44\n\t{.descriptor = 0x44,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 1024,\n\t .associativity = 4,\n\t .line_size = 32,\n\t },\n// 0x45\n\t{.descriptor = 0x45,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 2048,\n\t .associativity = 4,\n\t .line_size = 32,\n\t },\n// 0x46\n\t{.descriptor = 0x46,\n\t .level = 3,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 4096,\n\t .associativity = 4,\n\t .line_size = 64,\n\t },\n// 0x47\n\t{.descriptor = 0x47,\n\t .level = 3,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 8192,\n\t .associativity = 8,\n\t .line_size = 64,\n\t },\n// 0x48\n\t{.descriptor = 0x48,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 3072,\n\t .associativity = 12,\n\t .line_size = 64,\n\t },\n// 0x49 NOTE: for family 0x0F model 0x06 this is level 3\n\t{.descriptor = 0x49,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 4096,\n\t .associativity = 16,\n\t .line_size = 64,\n\t },\n// 0x4A\n\t{.descriptor = 0x4A,\n\t .level = 3,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 6144,\n\t .associativity = 12,\n\t .line_size = 64,\n\t },\n// 0x4B\n\t{.descriptor = 0x4B,\n\t .level = 3,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 8192,\n\t .associativity = 16,\n\t .line_size = 64,\n\t },\n// 0x4C\n\t{.descriptor = 0x4C,\n\t .level = 3,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 12288,\n\t .associativity = 12,\n\t .line_size = 64,\n\t },\n// 0x4D\n\t{.descriptor = 0x4D,\n\t .level = 3,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 16384,\n\t .associativity = 16,\n\t .line_size = 64,\n\t },\n// 0x4E\n\t{.descriptor = 0x4E,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 6144,\n\t .associativity = 24,\n\t .line_size = 64,\n\t },\n// 0x4F\n\t{.descriptor = 0x4F,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_INST,\n\t .size[0] = 4,\n\t .associativity = SHRT_MAX,\n\t .entries = 32,\n\t },\n// 0x50\n\t{.descriptor = 0x50,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_INST,\n\t .size = {4, 2048, 4096},\n\t .associativity = SHRT_MAX,\n\t .entries = 64,\n\t },\n// 0x51\n\t{.descriptor = 0x51,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_INST,\n\t .size = {4, 2048, 4096},\n\t .associativity = SHRT_MAX,\n\t .entries = 128,\n\t },\n// 0x52\n\t{.descriptor = 0x52,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_INST,\n\t .size = {4, 2048, 4096},\n\t .associativity = SHRT_MAX,\n\t .entries = 256,\n\t },\n// 0x55\n\t{.descriptor = 0x55,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_INST,\n\t .size = {2048, 4096, 0},\n\t .associativity = SHRT_MAX,\n\t .entries = 7,\n\t },\n// 0x56\n\t{.descriptor = 0x56,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_DATA,\n\t .size[0] = 4096,\n\t .associativity = 4,\n\t .entries = 16,\n\t },\n// 0x57\n\t{.descriptor = 0x57,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_DATA,\n\t .size[0] = 4,\n\t .associativity = 4,\n\t .entries = 16,\n\t },\n// 0x59\n\t{.descriptor = 0x59,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_DATA,\n\t .size[0] = 4,\n\t .associativity = SHRT_MAX,\n\t .entries = 16,\n\t }, \n// 0x5A\n\t{.descriptor = 0x5A,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_DATA,\n\t .size = {2048, 4096, 0},\n\t .associativity = 4,\n\t .entries = 32,\n\t },\n// 0x5B\n\t{.descriptor = 0x5B,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_DATA,\n\t .size = {4, 4096, 0},\n\t .associativity = SHRT_MAX,\n\t .entries = 64,\n\t },\n// 0x5C\n\t{.descriptor = 0x5C,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_DATA,\n\t .size = {4, 4096, 0},\n\t .associativity = SHRT_MAX,\n\t .entries = 128,\n\t },\n// 0x5D\n\t{.descriptor = 0x5D,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_DATA,\n\t .size = {4, 4096, 0},\n\t .associativity = SHRT_MAX,\n\t .entries = 256,\n\t },\n// 0x60\n\t{.descriptor = 0x60,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_DATA,\n\t .size[0] = 16,\n\t .associativity = 8,\n\t .sector = 1,\n\t .line_size = 64,\n\t },\n// 0x66\n\t{.descriptor = 0x66,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_DATA,\n\t .size[0] = 8,\n\t .associativity = 4,\n\t .sector = 1,\n\t .line_size = 64,\n\t },\n// 0x67\n\t{.descriptor = 0x67,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_DATA,\n\t .size[0] = 16,\n\t .associativity = 4,\n\t .sector = 1,\n\t .line_size = 64,\n\t },\n// 0x68\n\t{.descriptor = 0x68,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_DATA,\n\t .size[0] = 32,\n\t .associativity = 4,\n\t .sector = 1,\n\t .line_size = 64,\n\t },\n// 0x70\n\t{.descriptor = 0x70,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TRACE,\n\t .size[0] = 12,\n\t .associativity = 8,\n\t },\n// 0x71\n\t{.descriptor = 0x71,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TRACE,\n\t .size[0] = 16,\n\t .associativity = 8,\n\t },\n// 0x72\n\t{.descriptor = 0x72,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TRACE,\n\t .size[0] = 32,\n\t .associativity = 8,\n\t },\n// 0x73\n\t{.descriptor = 0x73,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TRACE,\n\t .size[0] = 64,\n\t .associativity = 8,\n\t },\n// 0x78\n\t{.descriptor = 0x78,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 1024,\n\t .associativity = 4,\n\t .line_size = 64,\n\t },\n// 0x79\n\t{.descriptor = 0x79,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 128,\n\t .associativity = 8,\n\t .sector = 1,\n\t .line_size = 64,\n\t },\n// 0x7A\n\t{.descriptor = 0x7A,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 256,\n\t .associativity = 8,\n\t .sector = 1,\n\t .line_size = 64,\n\t },\n// 0x7B\n\t{.descriptor = 0x7B,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 512,\n\t .associativity = 8,\n\t .sector = 1,\n\t .line_size = 64,\n\t },\n// 0x7C\n\t{.descriptor = 0x7C,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 1024,\n\t .associativity = 8,\n\t .sector = 1,\n\t .line_size = 64,\n\t },\n// 0x7D\n\t{.descriptor = 0x7D,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 2048,\n\t .associativity = 8,\n\t .line_size = 64,\n\t },\n// 0x7F\n\t{.descriptor = 0x7F,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 512,\n\t .associativity = 2,\n\t .line_size = 64,\n\t },\n// 0x80\n\t{.descriptor = 0x80,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 512,\n\t .associativity = 8,\n\t .line_size = 64,\n\t }, \n// 0x82\n\t{.descriptor = 0x82,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 256,\n\t .associativity = 8,\n\t .line_size = 32,\n\t },\n// 0x83\n\t{.descriptor = 0x83,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 512,\n\t .associativity = 8,\n\t .line_size = 32,\n\t },\n// 0x84\n\t{.descriptor = 0x84,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 1024,\n\t .associativity = 8,\n\t .line_size = 32,\n\t },\n// 0x85\n\t{.descriptor = 0x85,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 2048,\n\t .associativity = 8,\n\t .line_size = 32,\n\t },\n// 0x86\n\t{.descriptor = 0x86,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 512,\n\t .associativity = 4,\n\t .line_size = 64,\n\t },\n// 0x87\n\t{.descriptor = 0x87,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 1024,\n\t .associativity = 8,\n\t .line_size = 64,\n\t },\n// 0xB0\n\t{.descriptor = 0xB0,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_INST,\n\t .size[0] = 4,\n\t .associativity = 4,\n\t .entries = 128,\n\t },\n// 0xB1 NOTE: This is currently the only instance where .entries\n// is dependent on .size. It's handled as a code exception.\n// If other instances appear in the future, the structure\n// should probably change to accomodate it.\n\t{.descriptor = 0xB1,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_INST,\n\t .size = {2048, 4096, 0},\n\t .associativity = 4,\n\t .entries = 8,\t\t\t /* or 4 if size = 4096 */\n\t },\n// 0xB2\n\t{.descriptor = 0xB2,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_INST,\n\t .size[0] = 4,\n\t .associativity = 4,\n\t .entries = 64,\n\t },\n// 0xB3\n\t{.descriptor = 0xB3,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_DATA,\n\t .size[0] = 4,\n\t .associativity = 4,\n\t .entries = 128,\n\t },\n// 0xB4\n\t{.descriptor = 0xB4,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_DATA,\n\t .size[0] = 4,\n\t .associativity = 4,\n\t .entries = 256,\n\t },\n// 0xBA\n\t{.descriptor = 0xBA,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_DATA,\n\t .size[0] = 4,\n\t .associativity = 4,\n\t .entries = 64,\n\t }, \n// 0xC0\n\t{.descriptor = 0xBA,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_DATA,\n\t .size = {4,4096},\n\t .associativity = 4,\n\t .entries = 8,\n\t }, \n// 0xCA\n\t{.descriptor = 0xCA,\n\t .level = 2,\n\t .type = PAPI_MH_TYPE_TLB | PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 4,\n\t .associativity = 4,\n\t .entries = 512,\n\t },\n// 0xD0\n\t{.descriptor = 0xD0,\n\t .level = 3,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 512,\n\t .associativity = 4,\n\t .line_size = 64,\n\t },\n// 0xD1\n\t{.descriptor = 0xD1,\n\t .level = 3,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 1024,\n\t .associativity = 4,\n\t .line_size = 64,\n\t },\n// 0xD2\n\t{.descriptor = 0xD2,\n\t .level = 3,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 2048,\n\t .associativity = 4,\n\t .line_size = 64,\n\t },\n// 0xD6\n\t{.descriptor = 0xD6,\n\t .level = 3,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 1024,\n\t .associativity = 8,\n\t .line_size = 64,\n\t },\n// 0xD7\n\t{.descriptor = 0xD7,\n\t .level = 3,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 2048,\n\t .associativity = 8,\n\t .line_size = 64,\n\t },\n// 0xD8\n\t{.descriptor = 0xD8,\n\t .level = 3,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 4096,\n\t .associativity = 8,\n\t .line_size = 64,\n\t },\n// 0xDC\n\t{.descriptor = 0xDC,\n\t .level = 3,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 1536,\n\t .associativity = 12,\n\t .line_size = 64,\n\t },\n// 0xDD\n\t{.descriptor = 0xDD,\n\t .level = 3,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 3072,\n\t .associativity = 12,\n\t .line_size = 64,\n\t },\n// 0xDE\n\t{.descriptor = 0xDE,\n\t .level = 3,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 6144,\n\t .associativity = 12,\n\t .line_size = 64,\n\t },\n// 0xE2\n\t{.descriptor = 0xE2,\n\t .level = 3,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 2048,\n\t .associativity = 16,\n\t .line_size = 64,\n\t },\n// 0xE3\n\t{.descriptor = 0xE3,\n\t .level = 3,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 4096,\n\t .associativity = 16,\n\t .line_size = 64,\n\t },\n// 0xE4\n\t{.descriptor = 0xE4,\n\t .level = 3,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 8192,\n\t .associativity = 16,\n\t .line_size = 64,\n\t },\n// 0xEA\n\t{.descriptor = 0xEA,\n\t .level = 3,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 12288,\n\t .associativity = 24,\n\t .line_size = 64,\n\t },\n// 0xEB\n\t{.descriptor = 0xEB,\n\t .level = 3,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 18432,\n\t .associativity = 24,\n\t .line_size = 64,\n\t },\n// 0xEC\n\t{.descriptor = 0xEC,\n\t .level = 3,\n\t .type = PAPI_MH_TYPE_UNIFIED,\n\t .size[0] = 24576,\n\t .associativity = 24,\n\t .line_size = 64,\n\t },\n// 0xF0\n\t{.descriptor = 0xF0,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_PREF,\n\t .size[0] = 64,\n\t },\n// 0xF1\n\t{.descriptor = 0xF1,\n\t .level = 1,\n\t .type = PAPI_MH_TYPE_PREF,\n\t .size[0] = 128,\n\t },\n};\n\n#ifdef DEBUG\nstatic void\nprint_intel_cache_table( )\n{\n\tint i, j, k =\n\t\t( int ) ( sizeof ( intel_cache ) /\n\t\t\t\t sizeof ( struct _intel_cache_info ) );\n\tfor ( i = 0; i < k; i++ ) {\n\t\tprintf( \"%d.\\tDescriptor: %#x\\n\", i, intel_cache[i].descriptor );\n\t\tprintf( \"\\t Level: %d\\n\", intel_cache[i].level );\n\t\tprintf( \"\\t Type: %d\\n\", intel_cache[i].type );\n\t\tprintf( \"\\t Size(s): \" );\n\t\tfor ( j = 0; j < TLB_SIZES; j++ )\n\t\t\tprintf( \"%d, \", intel_cache[i].size[j] );\n\t\tprintf( \"\\n\" );\n\t\tprintf( \"\\t Assoc: %d\\n\", intel_cache[i].associativity );\n\t\tprintf( \"\\t Sector: %d\\n\", intel_cache[i].sector );\n\t\tprintf( \"\\t Line Size: %d\\n\", intel_cache[i].line_size );\n\t\tprintf( \"\\t Entries: %d\\n\", intel_cache[i].entries );\n\t\tprintf( \"\\n\" );\n\t}\n}\n#endif\n\n/* Given a specific cache descriptor, this routine decodes the information from a table\n * of such descriptors and fills out one or more records in a PAPI data structure.\n * Called only by init_intel()\n */\nstatic void\nintel_decode_descriptor( struct _intel_cache_info *d, PAPI_mh_level_t * L )\n{\n\tint i, next;\n\tint level = d->level - 1;\n\tPAPI_mh_tlb_info_t *t;\n\tPAPI_mh_cache_info_t *c;\n\n\tif ( d->descriptor == 0x49 ) {\t/* special case */\n\t\tunsigned int r_eax, r_ebx, r_ecx, r_edx;\n\t\tr_eax = 0x1;\t\t /* function code 1: family & model */\n\t\tcpuid( &r_eax, &r_ebx, &r_ecx, &r_edx );\n\t\t/* override table for Family F, model 6 only */\n\t\tif ( ( r_eax & 0x0FFF3FF0 ) == 0xF60 )\n\t\t\tlevel = 3;\n\t}\n\tif ( d->type & PAPI_MH_TYPE_TLB ) {\n\t\tfor ( next = 0; next < PAPI_MH_MAX_LEVELS - 1; next++ ) {\n\t\t\tif ( L[level].tlb[next].type == PAPI_MH_TYPE_EMPTY )\n\t\t\t\tbreak;\n\t\t}\n\t\t/* expand TLB entries for multiple possible page sizes */\n\t\tfor ( i = 0; i < TLB_SIZES && next < PAPI_MH_MAX_LEVELS && d->size[i];\n\t\t\t i++, next++ ) {\n// printf(\"Level %d Descriptor: %#x TLB type %#x next: %d, i: %d\\n\", level, d->descriptor, d->type, next, i);\n\t\t\tt = &L[level].tlb[next];\n\t\t\tt->type = PAPI_MH_CACHE_TYPE( d->type );\n\t\t\tt->num_entries = d->entries;\n\t\t\tt->page_size = d->size[i] << 10;\t/* minimum page size in KB */\n\t\t\tt->associativity = d->associativity;\n\t\t\t/* another special case */\n\t\t\tif ( d->descriptor == 0xB1 && d->size[i] == 4096 )\n\t\t\t\tt->num_entries = d->entries / 2;\n\t\t}\n\t} else {\n\t\tfor ( next = 0; next < PAPI_MH_MAX_LEVELS - 1; next++ ) {\n\t\t\tif ( L[level].cache[next].type == PAPI_MH_TYPE_EMPTY )\n\t\t\t\tbreak;\n\t\t}\n// printf(\"Level %d Descriptor: %#x Cache type %#x next: %d\\n\", level, d->descriptor, d->type, next);\n\t\tc = &L[level].cache[next];\n\t\tc->type = PAPI_MH_CACHE_TYPE( d->type );\n\t\tc->size = d->size[0] << 10;\t/* convert from KB to bytes */\n\t\tc->associativity = d->associativity;\n\t\tif ( d->line_size ) {\n\t\t\tc->line_size = d->line_size;\n\t\t\tc->num_lines = c->size / c->line_size;\n\t\t}\n\t}\n}\n\n#if defined(__amd64__) || defined(__x86_64__)\nstatic inline void\ncpuid2( unsigned int*eax, unsigned int* ebx,\n\t\tunsigned int*ecx, unsigned int *edx,\n\t\tunsigned int index, unsigned int ecx_in )\n{\n\t__asm__ __volatile__ (\"cpuid;\"\n\t\t: \"=a\" (*eax), \"=b\" (*ebx), \"=c\" (*ecx), \"=d\" (*edx)\n\t\t: \"0\" (index), \"2\"(ecx_in) );\n}\n#else\nstatic inline void\ncpuid2 ( unsigned int* eax, unsigned int* ebx, \n unsigned int* ecx, unsigned int* edx, \n unsigned int index, unsigned int ecx_in )\n{\n unsigned int a,b,c,d;\n __asm__ __volatile__ (\".byte 0x53\\n\\tcpuid\\n\\tmovl %%ebx, %%esi\\n\\t.byte 0x5b\"\n\t\t: \"=a\" (a), \"=S\" (b), \"=c\" (c), \"=d\" (d) \\\n\t\t: \"0\" (index), \"2\"(ecx_in) );\n *eax = a; *ebx = b; *ecx = c; *edx = d;\n}\n#endif\n\n\nstatic int\ninit_intel_leaf4( PAPI_mh_info_t * mh_info, int *num_levels )\n{\n\n unsigned int eax, ebx, ecx, edx;\n unsigned int maxidx, ecx_in;\n int next;\n\n int cache_type,cache_level,cache_selfinit,cache_fullyassoc;\n int cache_linesize,cache_partitions,cache_ways,cache_sets;\n\n PAPI_mh_cache_info_t *c;\n\n *num_levels=0;\n\n cpuid2(&eax,&ebx,&ecx,&edx, 0, 0);\n maxidx = eax;\n \n if (maxidx<4) {\n MEMDBG(\"Warning! CPUID Index 4 not supported!\\n\");\n return PAPI_ENOSUPP;\n }\n\n ecx_in=0;\n while(1) {\n cpuid2(&eax,&ebx,&ecx,&edx, 4, ecx_in);\n\n\n \n /* decoded as per table 3-12 in Intel Software Developer's Manual Volume 2A */\n \n cache_type=eax&0x1f;\n if (cache_type==0) break; \n \n cache_level=(eax>>5)&0x3;\n cache_selfinit=(eax>>8)&0x1;\n cache_fullyassoc=(eax>>9)&0x1;\n\n cache_linesize=(ebx&0xfff)+1;\n cache_partitions=((ebx>>12)&0x3ff)+1;\n cache_ways=((ebx>>22)&0x3ff)+1;\n \n cache_sets=(ecx)+1;\n\n /* should we export this info?\n\n cache_maxshare=((eax>>14)&0xfff)+1;\n cache_maxpackage=((eax>>26)&0x3f)+1;\n \n cache_wb=(edx)&1;\n cache_inclusive=(edx>>1)&1;\n cache_indexing=(edx>>2)&1;\n */\n\n if (cache_level>*num_levels) *num_levels=cache_level;\n\n /* find next slot available to hold cache info */\n for ( next = 0; next < PAPI_MH_MAX_LEVELS - 1; next++ ) {\n if ( mh_info->level[cache_level-1].cache[next].type == PAPI_MH_TYPE_EMPTY ) break;\n }\n\n c=&(mh_info->level[cache_level-1].cache[next]);\n\n switch(cache_type) {\n case 1: MEMDBG(\"L%d Data Cache\\n\",cache_level); \n\tc->type=PAPI_MH_TYPE_DATA;\n\tbreak;\n case 2: MEMDBG(\"L%d Instruction Cache\\n\",cache_level); \n\tc->type=PAPI_MH_TYPE_INST;\n\tbreak;\n case 3: MEMDBG(\"L%d Unified Cache\\n\",cache_level); \n\tc->type=PAPI_MH_TYPE_UNIFIED;\n\tbreak;\n }\n \n if (cache_selfinit) { MEMDBG(\"\\tSelf-init\\n\"); }\n if (cache_fullyassoc) { MEMDBG(\"\\tFully Associtative\\n\"); }\n \n //MEMDBG(\"\\tMax logical processors sharing cache: %d\\n\",cache_maxshare);\n //MEMDBG(\"\\tMax logical processors sharing package: %d\\n\",cache_maxpackage);\n \n MEMDBG(\"\\tCache linesize: %d\\n\",cache_linesize);\n\n MEMDBG(\"\\tCache partitions: %d\\n\",cache_partitions);\n MEMDBG(\"\\tCache associaticity: %d\\n\",cache_ways);\n\n MEMDBG(\"\\tCache sets: %d\\n\",cache_sets);\n MEMDBG(\"\\tCache size = %dkB\\n\",\n\t (cache_ways*cache_partitions*cache_linesize*cache_sets)/1024);\n\n //MEMDBG(\"\\tWBINVD/INVD acts on lower caches: %d\\n\",cache_wb);\n //MEMDBG(\"\\tCache is not inclusive: %d\\n\",cache_inclusive);\n //MEMDBG(\"\\tComplex cache indexing: %d\\n\",cache_indexing);\n\n c->line_size=cache_linesize;\n if (cache_fullyassoc) {\n c->associativity=SHRT_MAX;\n }\n else {\n c->associativity=cache_ways;\n }\n c->size=(cache_ways*cache_partitions*cache_linesize*cache_sets);\n c->num_lines=cache_ways*cache_partitions*cache_sets;\n \n ecx_in++;\n }\n return PAPI_OK;\n}\n\nstatic int\ninit_intel_leaf2( PAPI_mh_info_t * mh_info , int *num_levels)\n{\n\t/* cpuid() returns memory copies of 4 32-bit registers\n\t * this union allows them to be accessed as either registers\n\t * or individual bytes. Remember that Intel is little-endian.\n\t */\n\tunion\n\t{\n\t\tstruct\n\t\t{\n\t\t\tunsigned int ax, bx, cx, dx;\n\t\t} e;\n\t\tunsigned char descrip[16];\n\t} reg;\n\n\tint r;\t\t\t\t\t\t\t /* register boundary index */\n\tint b;\t\t\t\t\t\t\t /* byte index into a register */\n\tint i;\t\t\t\t\t\t\t /* byte index into the descrip array */\n\tint t;\t\t\t\t\t\t\t /* table index into the static descriptor table */\n\tint count;\t\t\t\t\t\t /* how many times to call cpuid; from eax:lsb */\n\tint size;\t\t\t\t\t\t /* size of the descriptor table */\n\tint last_level = 0;\t\t\t\t /* how many levels in the cache hierarchy */\n\n\t/* All of Intel's cache info is in 1 call to cpuid\n\t * however it is a table lookup :(\n\t */\n\tMEMDBG( \"Initializing Intel Cache and TLB descriptors\\n\" );\n\n#ifdef DEBUG\n\tif ( ISLEVEL( DEBUG_MEMORY ) )\n\t\tprint_intel_cache_table( );\n#endif\n\n\treg.e.ax = 0x2;\t\t\t /* function code 2: cache descriptors */\n\tcpuid( ®.e.ax, ®.e.bx, ®.e.cx, ®.e.dx );\n\n\tMEMDBG( \"e.ax=%#8.8x e.bx=%#8.8x e.cx=%#8.8x e.dx=%#8.8x\\n\",\n\t\t\treg.e.ax, reg.e.bx, reg.e.cx, reg.e.dx );\n\tMEMDBG\n\t\t( \":\\nd0: %#x %#x %#x %#x\\nd1: %#x %#x %#x %#x\\nd2: %#x %#x %#x %#x\\nd3: %#x %#x %#x %#x\\n\",\n\t\t reg.descrip[0], reg.descrip[1], reg.descrip[2], reg.descrip[3],\n\t\t reg.descrip[4], reg.descrip[5], reg.descrip[6], reg.descrip[7],\n\t\t reg.descrip[8], reg.descrip[9], reg.descrip[10], reg.descrip[11],\n\t\t reg.descrip[12], reg.descrip[13], reg.descrip[14], reg.descrip[15] );\n\n\tcount = reg.descrip[0];\t /* # times to repeat CPUID call. Not implemented. */\n\n\t/* Knights Corner at least returns 0 here */\n\tif (count==0) goto early_exit;\n\n\tsize = ( sizeof ( intel_cache ) / sizeof ( struct _intel_cache_info ) );\t/* # descriptors */\n\tMEMDBG( \"Repeat cpuid(2,...) %d times. If not 1, code is broken.\\n\",\n\t\t\tcount );\n\tif (count!=1) {\n\t fprintf(stderr,\"Warning: Unhandled cpuid count of %d\\n\",count);\n\t}\n\n\tfor ( r = 0; r < 4; r++ ) {\t/* walk the registers */\n\t\tif ( ( reg.descrip[r * 4 + 3] & 0x80 ) == 0 ) {\t/* only process if high order bit is 0 */\n\t\t\tfor ( b = 3; b >= 0; b-- ) {\t/* walk the descriptor bytes from high to low */\n\t\t\t\ti = r * 4 + b;\t/* calculate an index into the array of descriptors */\n\t\t\t\tif ( i ) {\t /* skip the low order byte in eax [0]; it's the count (see above) */\n\t\t\t\t if ( reg.descrip[i] == 0xff ) {\n\t\t\t\t MEMDBG(\"Warning! PAPI x86_cache: must implement cpuid leaf 4\\n\");\n\t\t\t\t return PAPI_ENOSUPP;\n\t\t\t\t /* we might continue instead */\n\t\t\t\t /* in order to get TLB info */\n\t\t\t\t /* continue; */\n\t\t\t\t }\n\t\t\t\t\tfor ( t = 0; t < size; t++ ) {\t/* walk the descriptor table */\t\t\t\t\t \n\t\t\t\t\t\tif ( reg.descrip[i] == intel_cache[t].descriptor ) {\t/* find match */\n\t\t\t\t\t\t\tif ( intel_cache[t].level > last_level )\n\t\t\t\t\t\t\t\tlast_level = intel_cache[t].level;\n\t\t\t\t\t\t\tintel_decode_descriptor( &intel_cache[t],\n\t\t\t\t\t\t\t\t\t\t\t\t\t mh_info->level );\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\nearly_exit:\n\tMEMDBG( \"# of Levels: %d\\n\", last_level );\n\t*num_levels=last_level;\n\treturn PAPI_OK;\n}\n\n\nstatic int\ninit_intel( PAPI_mh_info_t * mh_info, int *levels )\n{\n\n int result;\n int num_levels;\n\n /* try using the oldest leaf2 method first */\n result=init_intel_leaf2(mh_info, &num_levels);\n \n if (result!=PAPI_OK) {\n /* All Core2 and newer also support leaf4 detection */\n /* Starting with Westmere *only* leaf4 is supported */\n result=init_intel_leaf4(mh_info, &num_levels);\n }\n\n *levels=num_levels;\n return PAPI_OK;\n}\n\n\n/* Returns 1 if hypervisor detected */\n/* Returns 0 if none found. */\nint \n_x86_detect_hypervisor(char *vendor_name)\n{\n unsigned int eax, ebx, ecx, edx;\n char hyper_vendor_id[13];\n\n cpuid2(&eax, &ebx, &ecx, &edx,0x1,0);\n /* This is the hypervisor bit, ecx bit 31 */\n if (ecx&0x80000000) {\n /* There are various values in the 0x4000000X range */\n /* It is questionable how standard they are */\n /* For now we just return the name. */\n cpuid2(&eax, &ebx, &ecx, &edx, 0x40000000,0);\n memcpy(hyper_vendor_id + 0, &ebx, 4);\n memcpy(hyper_vendor_id + 4, &ecx, 4);\n memcpy(hyper_vendor_id + 8, &edx, 4);\n hyper_vendor_id[12] = '\\0';\n strncpy(vendor_name,hyper_vendor_id,PAPI_MAX_STR_LEN);\n return 1;\n }\n else {\n strncpy(vendor_name,\"none\",PAPI_MAX_STR_LEN);\n }\n return 0;\n}\n" }, { "path": "lib/spack/spack/test/data/make/affirmative/capital_makefile/Makefile", "content": "# Tests that Spack checks for Makefile\n\ncheck:\n" }, { "path": "lib/spack/spack/test/data/make/affirmative/check_test/Makefile", "content": "# Tests that Spack detects target when it is the first of two targets\n\ncheck test:\n" }, { "path": "lib/spack/spack/test/data/make/affirmative/expansion/Makefile", "content": "# Tests that Spack can handle variable expansion targets\n\nTARGETS = check\n\n$(TARGETS):\n" }, { "path": "lib/spack/spack/test/data/make/affirmative/gnu_makefile/GNUmakefile", "content": "# Tests that Spack checks for GNUmakefile\n\ncheck:\n" }, { "path": "lib/spack/spack/test/data/make/affirmative/include/Makefile", "content": "# Tests that Spack detects targets in include files\n\ninclude make.mk\n" }, { "path": "lib/spack/spack/test/data/make/affirmative/include/make.mk", "content": "check:\n" }, { "path": "lib/spack/spack/test/data/make/affirmative/lowercase_makefile/makefile", "content": "# Tests that Spack checks for makefile\n\ncheck:\n" }, { "path": "lib/spack/spack/test/data/make/affirmative/prerequisites/Makefile", "content": "# Tests that Spack detects a target even if it is followed by prerequisites\n\ncheck: check-recursive\n\ncheck-recursive:\n" }, { "path": "lib/spack/spack/test/data/make/affirmative/spaces/Makefile", "content": "# Tests that Spack allows spaces following the target name\n\ncheck :\n" }, { "path": "lib/spack/spack/test/data/make/affirmative/test_check/Makefile", "content": "# Tests that Spack detects target when it is the second of two targets\n\ntest check:\n" }, { "path": "lib/spack/spack/test/data/make/affirmative/three_targets/Makefile", "content": "# Tests that Spack detects a target if it is in the middle of a list\n\nfoo check bar:\n" }, { "path": "lib/spack/spack/test/data/make/negative/no_makefile/readme.txt", "content": "# Tests that Spack ignores directories without a Makefile\n\ncheck:\n" }, { "path": "lib/spack/spack/test/data/make/negative/partial_match/Makefile", "content": "# Tests that Spack ignores targets that contain a partial match\n\ncheckinstall:\n\ninstallcheck:\n\nfoo-check-bar:\n\nfoo_check_bar:\n\nfoo/check/bar:\n" }, { "path": "lib/spack/spack/test/data/make/negative/variable/Makefile", "content": "# Tests that Spack ignores variable definitions\n\ncheck = FOO\n\ncheck := BAR\n" }, { "path": "lib/spack/spack/test/data/microarchitectures/microarchitectures.json", "content": "{\n \"microarchitectures\": {\n \"x86\": {\n \"from\": null,\n \"vendor\": \"generic\",\n \"features\": []\n },\n \"i686\": {\n \"from\": \"x86\",\n \"vendor\": \"GenuineIntel\",\n \"features\": []\n },\n \"pentium2\": {\n \"from\": \"i686\",\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\"\n ]\n },\n \"pentium3\": {\n \"from\": \"pentium2\",\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\"\n ]\n },\n \"pentium4\": {\n \"from\": \"pentium3\",\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\"\n ]\n },\n \"prescott\": {\n \"from\": \"pentium4\",\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"sse3\"\n ]\n },\n \"x86_64\": {\n \"from\": null,\n \"vendor\": \"generic\",\n \"features\": [],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"4.2.0:\",\n \"name\": \"x86-64\",\n \"flags\": \"-march={name} -mtune=generic\"\n },\n {\n \"versions\": \":4.1.2\",\n \"name\": \"x86-64\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"0.0.0-apple:\",\n \"name\": \"x86-64\",\n \"flags\": \"-march={name}\"\n },\n {\n \"versions\": \":\",\n \"name\": \"x86-64\",\n \"flags\": \"-march={name} -mtune=generic\"\n }\n ],\n \"intel\": {\n \"versions\": \":\",\n \"name\": \"pentium4\",\n \"flags\": \"-march={name} -mtune=generic\"\n }\n }\n },\n \"nocona\": {\n \"from\": \"x86_64\",\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"sse3\"\n ],\n \"compilers\": {\n \"gcc\": {\n \"versions\": \"4.0.4:\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n \"clang\": {\n \"versions\": \"3.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n \"intel\": {\n \"versions\": \"16.0:\",\n \"name\": \"pentium4\",\n \"flags\": \"-march={name} -mtune=generic\"\n }\n }\n },\n \"core2\": {\n \"from\": \"nocona\",\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"ssse3\"\n ],\n \"compilers\": {\n \"gcc\": {\n \"versions\": \"4.3.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n \"clang\": {\n \"versions\": \"3.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n \"intel\": {\n \"versions\": \"16.0:\",\n \"flags\": \"-march={name} -mtune={name}}\"\n }\n }\n },\n \"nehalem\": {\n \"from\": \"core2\",\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"popcnt\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"4.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n {\n \"versions\": \"4.6:4.8.5\",\n \"name\": \"corei7\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": {\n \"versions\": \"3.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n \"intel\": {\n \"versions\": \"16.0:\",\n \"name\": \"corei7\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n }\n },\n \"westmere\": {\n \"from\": \"nehalem\",\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"popcnt\",\n \"aes\",\n \"pclmulqdq\"\n ],\n \"compilers\": {\n \"gcc\": {\n \"versions\": \"4.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n \"clang\": {\n \"versions\": \"3.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n \"intel\": {\n \"versions\": \"16.0:\",\n \"name\": \"corei7\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n }\n },\n \"sandybridge\": {\n \"from\": \"westmere\",\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"popcnt\",\n \"aes\",\n \"pclmulqdq\",\n \"avx\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"4.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n {\n \"versions\": \"4.6:4.8.5\",\n \"name\": \"corei7-avx\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": {\n \"versions\": \"3.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n \"intel\": [\n {\n \"versions\": \"16.0:17.9.0\",\n \"name\": \"corei7-avx\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n {\n \"versions\": \"18.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ]\n }\n },\n \"ivybridge\": {\n \"from\": \"sandybridge\",\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"popcnt\",\n \"aes\",\n \"pclmulqdq\",\n \"avx\",\n \"rdrand\",\n \"f16c\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"4.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n {\n \"versions\": \"4.6:4.8.5\",\n \"name\": \"core-avx-i\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": {\n \"versions\": \"3.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n \"intel\": [\n {\n \"versions\": \"16.0:17.9.0\",\n \"name\": \"core-avx-i\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n {\n \"versions\": \"18.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ]\n }\n },\n \"haswell\": {\n \"from\": \"ivybridge\",\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"popcnt\",\n \"aes\",\n \"pclmulqdq\",\n \"avx\",\n \"rdrand\",\n \"f16c\",\n \"movbe\",\n \"fma\",\n \"avx2\",\n \"bmi1\",\n \"bmi2\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"4.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n {\n \"versions\": \"4.8:4.8.5\",\n \"name\": \"core-avx2\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": {\n \"versions\": \"3.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n \"intel\": [\n {\n \"versions\": \"16.0:17.9.0\",\n \"name\": \"core-avx2\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n {\n \"versions\": \"18.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ]\n }\n },\n \"broadwell\": {\n \"from\": \"haswell\",\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"popcnt\",\n \"aes\",\n \"pclmulqdq\",\n \"avx\",\n \"rdrand\",\n \"f16c\",\n \"movbe\",\n \"fma\",\n \"avx2\",\n \"bmi1\",\n \"bmi2\",\n \"rdseed\",\n \"adx\"\n ],\n \"compilers\": {\n \"gcc\": {\n \"versions\": \"4.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n \"clang\": {\n \"versions\": \"3.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n \"intel\": {\n \"versions\": \"18.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n }\n },\n \"skylake\": {\n \"from\": \"broadwell\",\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"popcnt\",\n \"aes\",\n \"pclmulqdq\",\n \"avx\",\n \"rdrand\",\n \"f16c\",\n \"movbe\",\n \"fma\",\n \"avx2\",\n \"bmi1\",\n \"bmi2\",\n \"rdseed\",\n \"adx\",\n \"clflushopt\",\n \"xsavec\",\n \"xsaveopt\"\n ],\n \"compilers\": {\n \"gcc\": {\n \"versions\": \"6.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n \"clang\": {\n \"versions\": \"3.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n \"intel\": {\n \"versions\": \"18.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n }\n }\n },\n \"feature_aliases\": {\n \"sse3\": {\n \"reason\": \"ssse3 is a superset of sse3 and might be the only one listed\",\n \"any_of\": [\n \"ssse3\"\n ]\n },\n \"avx512\": {\n \"reason\": \"avx512 indicates generic support for any of the avx512 instruction sets\",\n \"any_of\": [\n \"avx512f\",\n \"avx512vl\",\n \"avx512bw\",\n \"avx512dq\",\n \"avx512cd\"\n ]\n },\n \"fma\": {\n \"reason\": \"FMA has been supported by PowerISA since Power1, but might not be listed in features\",\n \"families\": [\n \"ppc64le\",\n \"ppc64\"\n ]\n },\n \"sse4.1\": {\n \"reason\": \"permits to refer to sse4_1 also as sse4.1\",\n \"any_of\": [\n \"sse4_1\"\n ]\n },\n \"sse4.2\": {\n \"reason\": \"permits to refer to sse4_2 also as sse4.2\",\n \"any_of\": [\n \"sse4_2\"\n ]\n }\n },\n \"conversions\": {\n \"description\": \"Conversions that map some platform specific values to canonical values\",\n \"arm_vendors\": {\n \"0x41\": \"ARM\",\n \"0x42\": \"Broadcom\",\n \"0x43\": \"Cavium\",\n \"0x44\": \"DEC\",\n \"0x46\": \"Fujitsu\",\n \"0x48\": \"HiSilicon\",\n \"0x49\": \"Infineon Technologies AG\",\n \"0x4d\": \"Motorola\",\n \"0x4e\": \"Nvidia\",\n \"0x50\": \"APM\",\n \"0x51\": \"Qualcomm\",\n \"0x53\": \"Samsung\",\n \"0x56\": \"Marvell\",\n \"0x61\": \"Apple\",\n \"0x66\": \"Faraday\",\n \"0x68\": \"HXT\",\n \"0x69\": \"Intel\"\n },\n \"darwin_flags\": {\n \"sse4.1\": \"sse4_1\",\n \"sse4.2\": \"sse4_2\",\n \"avx1.0\": \"avx\",\n \"clfsopt\": \"clflushopt\",\n \"xsave\": \"xsavec xsaveopt\"\n }\n }\n}\n" }, { "path": "lib/spack/spack/test/data/mirrors/legacy_yaml/build_cache/test-debian6-core2-gcc-4.5.0-zlib-1.2.11-t5mczux3tfqpxwmg7egp7axy2jvyulqk.spec.yaml", "content": "spec:\n- zlib:\n version: 1.2.11\n arch:\n platform: test\n platform_os: debian6\n target:\n name: core2\n vendor: GenuineIntel\n features:\n - mmx\n - sse\n - sse2\n - ssse3\n generation: 0\n parents:\n - nocona\n compiler:\n name: gcc\n version: 4.5.0\n namespace: builtin_mock\n parameters:\n optimize: true\n pic: true\n shared: true\n cflags: []\n cppflags: []\n cxxflags: []\n fflags: []\n ldflags: []\n ldlibs: []\n package_hash: eukp6mqxxlfuxslsodbwbqtsznajielhh4avm2vgteo4ifdsjgjq====\n hash: t5mczux3tfqpxwmg7egp7axy2jvyulqk\n full_hash: 6j4as6r3qd4qhf77yu44reyn2u6ggbuq\n build_hash: t5mczux3tfqpxwmg7egp7axy2jvyulqk\nbinary_cache_checksum:\n hash_algorithm: sha256\n hash: a62b50aee38bb5d6d1cbf9cd2b0badaf3eaa282cd6db0472b4468ff968a5e7f2\nbuildinfo:\n relative_prefix: test-debian6-core2/gcc-4.5.0/zlib-1.2.11-t5mczux3tfqpxwmg7egp7axy2jvyulqk\n relative_rpaths: false\n" }, { "path": "lib/spack/spack/test/data/mirrors/signed_json/linux-ubuntu18.04-haswell-gcc-8.4.0-zlib-1.2.12-g7otk5dra3hifqxej36m5qzm7uyghqgb.spec.json.sig", "content": "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA512\n\n{\n \"spec\": {\n \"_meta\": {\n \"version\": 2\n },\n \"nodes\": [\n {\n \"name\": \"zlib\",\n \"version\": \"1.2.12\",\n \"arch\": {\n \"platform\": \"linux\",\n \"platform_os\": \"ubuntu18.04\",\n \"target\": {\n \"name\": \"haswell\",\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"aes\",\n \"avx\",\n \"avx2\",\n \"bmi1\",\n \"bmi2\",\n \"f16c\",\n \"fma\",\n \"mmx\",\n \"movbe\",\n \"pclmulqdq\",\n \"popcnt\",\n \"rdrand\",\n \"sse\",\n \"sse2\",\n \"sse4_1\",\n \"sse4_2\",\n \"ssse3\"\n ],\n \"generation\": 0,\n \"parents\": [\n \"ivybridge\",\n \"x86_64_v3\"\n ]\n }\n },\n \"compiler\": {\n \"name\": \"gcc\",\n \"version\": \"8.4.0\"\n },\n \"namespace\": \"builtin\",\n \"parameters\": {\n \"optimize\": true,\n \"patches\": [\n \"0d38234384870bfd34dfcb738a9083952656f0c766a0f5990b1893076b084b76\"\n ],\n \"pic\": true,\n \"shared\": true,\n \"cflags\": [],\n \"cppflags\": [],\n \"cxxflags\": [],\n \"fflags\": [],\n \"ldflags\": [],\n \"ldlibs\": []\n },\n \"patches\": [\n \"0d38234384870bfd34dfcb738a9083952656f0c766a0f5990b1893076b084b76\"\n ],\n \"package_hash\": \"bm7rut622h3yt5mpm4kvf7pmh7tnmueezgk5yquhr2orbmixwxuq====\",\n \"hash\": \"g7otk5dra3hifqxej36m5qzm7uyghqgb\",\n \"full_hash\": \"fx2fyri7bv3vpz2rhke6g3l3dwxda4t6\",\n \"build_hash\": \"g7otk5dra3hifqxej36m5qzm7uyghqgb\"\n }\n ]\n },\n \"binary_cache_checksum\": {\n \"hash_algorithm\": \"sha256\",\n \"hash\": \"5b9a180f14e0d04b17b1b0c2a26cf3beae448d77d1bda4279283ca4567d0be90\"\n },\n \"buildinfo\": {\n \"relative_prefix\": \"linux-ubuntu18.04-haswell/gcc-8.4.0/zlib-1.2.12-g7otk5dra3hifqxej36m5qzm7uyghqgb\",\n \"relative_rpaths\": false\n }\n}\n-----BEGIN PGP SIGNATURE-----\n\niQEzBAEBCgAdFiEEz8AGj4zHZe4OaI2OQ0Tg92UAr50FAmJ5lD4ACgkQQ0Tg92UA\nr52LEggAl/wXlOlHDnjWvqBlqAn3gaJEZ5PDVPczk6k0w+SNfDGrHfWJnL2c23Oq\nCssbHylSgAFvaPT1frbiLfZj6L4j4Ym1qsxIlGNsVfW7Pbc4yNF0flqYMdWKXbgY\n2sQoPegIKK7EBtpjDf0+VRYfJTMqjsSgjT/o+nTkg9oAnvU23EqXI5uiY84Z5z6l\nCKLBm0GWg7MzI0u8NdiQMVNYVatvvZ8EQpblEUQ7jD4Bo0yoSr33Qdq8uvu4ZdlW\nbvbIgeY3pTPF13g9uNznHLxW4j9BWQnOtHFI5UKQnYRner504Yoz9k+YxhwuDlaY\nTrOxvHe9hG1ox1AP4tqQc+HsNpm5Kg==\n=gi2R\n-----END PGP SIGNATURE-----\n" }, { "path": "lib/spack/spack/test/data/mirrors/v2_layout/signed/build_cache/_pgp/CBAB2C1032C6FF5078049EC0FA61D50C12CAD37E.pub", "content": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmQINBGgnhhYBEAC5LOSkJlxL4rRDBLDatswpzAw7NQnONW37hwOauEf6rlw/wk6J\n2D1l/jjmGwyo1iHOEu1/26fMuXMmG0vAxOQJFrkoKAgxDUD9nL0GqTJyg0+yTCN6\nxsWsrIZi+8oNDXYzLiejICZorc+ri11kcZdA+WE2hWPRStmJH75afpSd7XfNijqb\nMPfDZBcr+pLeARSH11BTfb8Dtm9qN//+X+pNIUqeHL9hLu/W9hb3GCfXqnsCQJA1\nWMFTrbCcPYm0R7EevMnscFvS8xbhocBPDwZ12f4W5CugrL29X4Vx9SaUlIyy/+SC\n2Gwi8Yq78Y4dTN7N5aA8L169/uqy4Tx7/966wMkUYXk7UxmH9E0ol5EZYnY9SCj6\nxLtMNKA+NLwESj0azaWEzxfztyNdTYfG8Eaa/QGFs1YVGhYdmcEp8KDbQg5FBeCA\nI6MUcH0XWOTJaZI/oEtukMYHzBt9jyyq6Gp45TiQvOou0wE+w/zJcd9Td23R81KW\nGfMh5r80NET/bx88vee4NNHkWCphhqs53rIrhWV3y3WKaWp7DfP3WMiTBJ+Yc+PI\n0vMIHKYNy+OqwTjmwgKdN1w1xZhLG7hx0sAdcZGP7q0A6381HtucgS/fucDogMnW\nH3anE8UGx4HBRjyXsuOaOAgNw2K4IwancUSf67WSzji3AiP46sUun5ERNQARAQAB\ntBlTcGFjayA8c3BhY2tAc3BhY2suc3BhY2s+iQJXBBMBCgBBFiEEy6ssEDLG/1B4\nBJ7A+mHVDBLK034FAmgnhhYCGwMFCQWjmoAFCwkIBwICIgIGFQoJCAsCBBYCAwEC\nHgcCF4AACgkQ+mHVDBLK034zWhAAtjm802qaTSCvB9WvY1RM65/B1GUK3ZEv3fw/\nDvt3xd3mh+rzWBTJ8t7+/cPaOq7qOGnfUateHgou+0T6lgCLkrwr4lFa6yZSUATb\nxcnopcA0Dal218UcIRb20PjPtoKu3Tt9JFceXJGCTYoGz5HbkOemwkR8B+4qMRPW\nsn1IhV32eig2HUzrUXVOv6WomMtk2qUpND0WnTlZo3EoInJeTzdlXkOR3lRLADM9\nyPM6Rp8AV/ykM9DztL4SinzyZjqEM7o1H7EFITZSlkjcBPvqDlvowZGN8TVbG9TQ\n8Nfz8BYF3SVaPduwXwhbE9D8jqtNt652IZ1+1KbMii1l4deu0UYx8BSfJjNANTTU\njFDiyNaGnn5OsZXNllsyAHWky6ApyBD9qFxxNr0kiWbVrrN6s2u4ghm5Hgtdx40v\nhA9+kvB2mtV/HklUkwDTJ6Ytgp5veh8GKvBD9eAWIitl6w153Rba5LkZbk2ijK6k\noyN9Ge/YloSMwXpIEnE7/SRE1o5vye294BZjyqnr+U+wzbEYbC7eXJ0peDCbpbZc\n0kxMDDbrhmHeEaHeWF30hm6WBaUT4SUcPj5BiV3mt3BhtRgAwA3SvuSenk2yRzR8\ntBES4b/RBmOczfs4w4m5rAmfVNkNwykry4M2jPCJhVA2qG8q1gLxf+AvaPcAvQ8D\nkmDeNLI=\n=CYuA\n-----END PGP PUBLIC KEY BLOCK-----\n" }, { "path": "lib/spack/spack/test/data/mirrors/v2_layout/signed/build_cache/_pgp/index.json", "content": "{\"keys\":{\"CBAB2C1032C6FF5078049EC0FA61D50C12CAD37E\":{}}}\n" }, { "path": "lib/spack/spack/test/data/mirrors/v2_layout/signed/build_cache/index.json", "content": "{\"database\":{\"version\":\"8\",\"installs\":{\"rqh2vuf6fqwkmipzgi2wjx352mq7y7ez\":{\"spec\":{\"name\":\"libelf\",\"version\":\"0.8.13\",\"arch\":{\"platform\":\"test\",\"platform_os\":\"debian6\",\"target\":{\"name\":\"core2\",\"vendor\":\"GenuineIntel\",\"features\":[\"mmx\",\"sse\",\"sse2\",\"ssse3\"],\"generation\":0,\"parents\":[\"nocona\"],\"cpupart\":\"\"}},\"namespace\":\"builtin_mock\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ejr32l7tkp6uhdrlunqv4adkuxqwyac7vbqcjvg6dh72mll4cpiq====\",\"annotations\":{\"original_specfile_version\":4,\"compiler\":\"gcc@=10.2.1\"},\"hash\":\"rqh2vuf6fqwkmipzgi2wjx352mq7y7ez\"},\"ref_count\":1,\"in_buildcache\":true},\"sk2gqqz4n5njmvktycnd25wq25jxiqkr\":{\"spec\":{\"name\":\"libdwarf\",\"version\":\"20130729\",\"arch\":{\"platform\":\"test\",\"platform_os\":\"debian6\",\"target\":{\"name\":\"core2\",\"vendor\":\"GenuineIntel\",\"features\":[\"mmx\",\"sse\",\"sse2\",\"ssse3\"],\"generation\":0,\"parents\":[\"nocona\"],\"cpupart\":\"\"}},\"namespace\":\"builtin_mock\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"n7axrpelzl5kjuctt4yoaaf33gvgnik6cx7fjudwhc6hvywdrr4q====\",\"dependencies\":[{\"name\":\"libelf\",\"hash\":\"rqh2vuf6fqwkmipzgi2wjx352mq7y7ez\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":4,\"compiler\":\"gcc@=10.2.1\"},\"hash\":\"sk2gqqz4n5njmvktycnd25wq25jxiqkr\"},\"ref_count\":0,\"in_buildcache\":true},\"qeehcxyvluwnihsc2qxstmpomtxo3lrc\":{\"spec\":{\"name\":\"compiler-wrapper\",\"version\":\"1.0\",\"arch\":{\"platform\":\"test\",\"platform_os\":\"debian6\",\"target\":{\"name\":\"m1\",\"vendor\":\"Apple\",\"features\":[\"aes\",\"asimd\",\"asimddp\",\"asimdfhm\",\"asimdhp\",\"asimdrdm\",\"atomics\",\"cpuid\",\"crc32\",\"dcpodp\",\"dcpop\",\"dit\",\"evtstrm\",\"fcma\",\"flagm\",\"flagm2\",\"fp\",\"fphp\",\"frint\",\"ilrcpc\",\"jscvt\",\"lrcpc\",\"paca\",\"pacg\",\"pmull\",\"sb\",\"sha1\",\"sha2\",\"sha3\",\"sha512\",\"ssbs\",\"uscat\"],\"generation\":0,\"parents\":[\"armv8.4a\"],\"cpupart\":\"0x022\"}},\"namespace\":\"builtin_mock\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ss7ybgvqf2fa2lvkf67eavllfxpxthiml2dobtkdq6wn7zkczteq====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"qeehcxyvluwnihsc2qxstmpomtxo3lrc\"},\"ref_count\":2,\"in_buildcache\":true},\"vd7v4ssgnoqdplgxyig3orum67n4vmhq\":{\"spec\":{\"name\":\"gcc\",\"version\":\"10.2.1\",\"arch\":{\"platform\":\"test\",\"platform_os\":\"debian6\",\"target\":\"aarch64\"},\"namespace\":\"builtin_mock\",\"parameters\":{\"build_system\":\"generic\",\"languages\":[\"c\",\"c++\",\"fortran\"],\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"external\":{\"path\":\"/path\",\"module\":null,\"extra_attributes\":{\"compilers\":{\"c\":\"/path/bin/gcc-10\",\"cxx\":\"/path/bin/g++-10\",\"fortran\":\"/path/bin/gfortran-10\"}}},\"package_hash\":\"a7d6wvl2mh4od3uue3yxqonc7r7ihw3n3ldedu4kevqa32oy2ysa====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"vd7v4ssgnoqdplgxyig3orum67n4vmhq\"},\"ref_count\":3,\"in_buildcache\":false},\"izgzpzeljwairalfjm3k6fntbb64nt6n\":{\"spec\":{\"name\":\"gcc-runtime\",\"version\":\"10.2.1\",\"arch\":{\"platform\":\"test\",\"platform_os\":\"debian6\",\"target\":{\"name\":\"m1\",\"vendor\":\"Apple\",\"features\":[\"aes\",\"asimd\",\"asimddp\",\"asimdfhm\",\"asimdhp\",\"asimdrdm\",\"atomics\",\"cpuid\",\"crc32\",\"dcpodp\",\"dcpop\",\"dit\",\"evtstrm\",\"fcma\",\"flagm\",\"flagm2\",\"fp\",\"fphp\",\"frint\",\"ilrcpc\",\"jscvt\",\"lrcpc\",\"paca\",\"pacg\",\"pmull\",\"sb\",\"sha1\",\"sha2\",\"sha3\",\"sha512\",\"ssbs\",\"uscat\"],\"generation\":0,\"parents\":[\"armv8.4a\"],\"cpupart\":\"0x022\"}},\"namespace\":\"builtin_mock\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"up2pdsw5tfvmn5gwgb3opl46la3uxoptkr3udmradd54s7qo72ha====\",\"dependencies\":[{\"name\":\"gcc\",\"hash\":\"vd7v4ssgnoqdplgxyig3orum67n4vmhq\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"izgzpzeljwairalfjm3k6fntbb64nt6n\"},\"ref_count\":2,\"in_buildcache\":true},\"jr3yipyxyjulcdvckwwwjrrumis7glpa\":{\"spec\":{\"name\":\"libelf\",\"version\":\"0.8.13\",\"arch\":{\"platform\":\"test\",\"platform_os\":\"debian6\",\"target\":{\"name\":\"m1\",\"vendor\":\"Apple\",\"features\":[\"aes\",\"asimd\",\"asimddp\",\"asimdfhm\",\"asimdhp\",\"asimdrdm\",\"atomics\",\"cpuid\",\"crc32\",\"dcpodp\",\"dcpop\",\"dit\",\"evtstrm\",\"fcma\",\"flagm\",\"flagm2\",\"fp\",\"fphp\",\"frint\",\"ilrcpc\",\"jscvt\",\"lrcpc\",\"paca\",\"pacg\",\"pmull\",\"sb\",\"sha1\",\"sha2\",\"sha3\",\"sha512\",\"ssbs\",\"uscat\"],\"generation\":0,\"parents\":[\"armv8.4a\"],\"cpupart\":\"0x022\"}},\"namespace\":\"builtin_mock\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ejr32l7tkp6uhdrlunqv4adkuxqwyac7vbqcjvg6dh72mll4cpiq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"qeehcxyvluwnihsc2qxstmpomtxo3lrc\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vd7v4ssgnoqdplgxyig3orum67n4vmhq\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"izgzpzeljwairalfjm3k6fntbb64nt6n\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"jr3yipyxyjulcdvckwwwjrrumis7glpa\"},\"ref_count\":1,\"in_buildcache\":true},\"u5uz3dcch5if4eve4sef67o2rf2lbfgh\":{\"spec\":{\"name\":\"libdwarf\",\"version\":\"20130729\",\"arch\":{\"platform\":\"test\",\"platform_os\":\"debian6\",\"target\":{\"name\":\"m1\",\"vendor\":\"Apple\",\"features\":[\"aes\",\"asimd\",\"asimddp\",\"asimdfhm\",\"asimdhp\",\"asimdrdm\",\"atomics\",\"cpuid\",\"crc32\",\"dcpodp\",\"dcpop\",\"dit\",\"evtstrm\",\"fcma\",\"flagm\",\"flagm2\",\"fp\",\"fphp\",\"frint\",\"ilrcpc\",\"jscvt\",\"lrcpc\",\"paca\",\"pacg\",\"pmull\",\"sb\",\"sha1\",\"sha2\",\"sha3\",\"sha512\",\"ssbs\",\"uscat\"],\"generation\":0,\"parents\":[\"armv8.4a\"],\"cpupart\":\"0x022\"}},\"namespace\":\"builtin_mock\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"n7axrpelzl5kjuctt4yoaaf33gvgnik6cx7fjudwhc6hvywdrr4q====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"qeehcxyvluwnihsc2qxstmpomtxo3lrc\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vd7v4ssgnoqdplgxyig3orum67n4vmhq\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"izgzpzeljwairalfjm3k6fntbb64nt6n\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"libelf\",\"hash\":\"jr3yipyxyjulcdvckwwwjrrumis7glpa\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"u5uz3dcch5if4eve4sef67o2rf2lbfgh\"},\"ref_count\":0,\"in_buildcache\":true}}}}" }, { "path": "lib/spack/spack/test/data/mirrors/v2_layout/signed/build_cache/index.json.hash", "content": "81a5add9d75b27fc4d16a4f72685b54903973366531b98c65e8cf5376758a817\n" }, { "path": "lib/spack/spack/test/data/mirrors/v2_layout/signed/build_cache/test-debian6-core2-gcc-10.2.1-libdwarf-20130729-sk2gqqz4n5njmvktycnd25wq25jxiqkr.spec.json.sig", "content": "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA512\n\n{\n\"spec\":{\n\"_meta\":{\n\"version\":4\n},\n\"nodes\":[\n{\n\"name\":\"libdwarf\",\n\"version\":\"20130729\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":{\n\"name\":\"core2\",\n\"vendor\":\"GenuineIntel\",\n\"features\":[\n\"mmx\",\n\"sse\",\n\"sse2\",\n\"ssse3\"\n],\n\"generation\":0,\n\"parents\":[\n\"nocona\"\n],\n\"cpupart\":\"\"\n}\n},\n\"compiler\":{\n\"name\":\"gcc\",\n\"version\":\"10.2.1\"\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"package_hash\":\"n7axrpelzl5kjuctt4yoaaf33gvgnik6cx7fjudwhc6hvywdrr4q====\",\n\"dependencies\":[\n{\n\"name\":\"libelf\",\n\"hash\":\"rqh2vuf6fqwkmipzgi2wjx352mq7y7ez\",\n\"parameters\":{\n\"deptypes\":[\n\"build\",\n\"link\"\n],\n\"virtuals\":[]\n}\n}\n],\n\"hash\":\"sk2gqqz4n5njmvktycnd25wq25jxiqkr\"\n},\n{\n\"name\":\"libelf\",\n\"version\":\"0.8.13\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":{\n\"name\":\"core2\",\n\"vendor\":\"GenuineIntel\",\n\"features\":[\n\"mmx\",\n\"sse\",\n\"sse2\",\n\"ssse3\"\n],\n\"generation\":0,\n\"parents\":[\n\"nocona\"\n],\n\"cpupart\":\"\"\n}\n},\n\"compiler\":{\n\"name\":\"gcc\",\n\"version\":\"10.2.1\"\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"package_hash\":\"ejr32l7tkp6uhdrlunqv4adkuxqwyac7vbqcjvg6dh72mll4cpiq====\",\n\"hash\":\"rqh2vuf6fqwkmipzgi2wjx352mq7y7ez\"\n}\n]\n},\n\"buildcache_layout_version\":2,\n\"binary_cache_checksum\":{\n\"hash_algorithm\":\"sha256\",\n\"hash\":\"811f500a89ae7d2f61e2c0ef6f56e352dfbac245ae88275809088a1481489d5b\"\n}\n}\n-----BEGIN PGP SIGNATURE-----\n\niQIzBAEBCgAdFiEEy6ssEDLG/1B4BJ7A+mHVDBLK034FAmgnhqIACgkQ+mHVDBLK\n0373kg/+Iy7pfWoAa465XtWUyf87KcjmJ1hE4OmfMc9sA7kdKNYPfmttxfp8jCU5\ngRc8RnQ5K+h4GWGl9nd6bFOT3oZSBH9WnH33gcnStHubwvHzhY05ZmlKjXKKTJmG\nrcQ8+vVv/e8KfMatydPuXQmAzbJ0pr2bGnicT8fs/W35hgcyygDZvDqJo3m+q4H7\nuu4C3LnaixAf7kCZefdxReYvFBNz9Qovws3+LqVFPxWgqo4zYt1PcI24UhCpL2YJ\n6XJySW7e0rR64bwCZR/owy504aUC64wr8kM19MMJAoB0R4zciJ0YyY8xLfRMI3Tr\nJTPetuTN7ncKJ2kZJ5L+KbeYnr4+CA5ZYmjyAM5NSJ3fTXuEu477H+1XovcJtP1s\nIZS10UWX452QEBXE5nWAludmiw4BenyR2Lccg2QfER8jbiZf3U3do43aGoI5U8rg\nqf1kQ/dMcIX6oSrbxMKymdsuf6e8UCSys3KNwb44UdSBiihgYFtiMfGtQ6Ixsvky\nTB+EwweUY6LtBuep1fh+M1tHgo9qCxUH79duor0JRDgQ/VLeO6e1RCptc7EHnQZQ\nmZK7YjVtHYWzyOZ4KsWuLYBSAMvKDhrTxI8cxp816NNGUfj1jmBQR/5vn6d7nMwX\nPmWrQV9O2e899Mv30VVR9XDf6tJoT+BPvS4Kc5hw/LxjaBbAxXo=\n=Zprh\n-----END PGP SIGNATURE-----\n" }, { "path": "lib/spack/spack/test/data/mirrors/v2_layout/signed/build_cache/test-debian6-core2-gcc-10.2.1-libelf-0.8.13-rqh2vuf6fqwkmipzgi2wjx352mq7y7ez.spec.json.sig", "content": "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA512\n\n{\n\"spec\":{\n\"_meta\":{\n\"version\":4\n},\n\"nodes\":[\n{\n\"name\":\"libelf\",\n\"version\":\"0.8.13\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":{\n\"name\":\"core2\",\n\"vendor\":\"GenuineIntel\",\n\"features\":[\n\"mmx\",\n\"sse\",\n\"sse2\",\n\"ssse3\"\n],\n\"generation\":0,\n\"parents\":[\n\"nocona\"\n],\n\"cpupart\":\"\"\n}\n},\n\"compiler\":{\n\"name\":\"gcc\",\n\"version\":\"10.2.1\"\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"package_hash\":\"ejr32l7tkp6uhdrlunqv4adkuxqwyac7vbqcjvg6dh72mll4cpiq====\",\n\"hash\":\"rqh2vuf6fqwkmipzgi2wjx352mq7y7ez\"\n}\n]\n},\n\"buildcache_layout_version\":2,\n\"binary_cache_checksum\":{\n\"hash_algorithm\":\"sha256\",\n\"hash\":\"48c8aa769a62535f9d9f613722e3d3f5a48b91fde3c99a644b22f277a4502d75\"\n}\n}\n-----BEGIN PGP SIGNATURE-----\n\niQIzBAEBCgAdFiEEy6ssEDLG/1B4BJ7A+mHVDBLK034FAmgnhqIACgkQ+mHVDBLK\n036q+Q//XkOoRoZ5g3uyQTXTV3w6YCUezkvGv+WRV4oZfj0CElKf4KoW5bhdtWEM\nEBRC4UuFturk7m1KrgztKsEFq7vx0TxvbWjj5R64swrwczKkD7i5xjMhWZn0nrpk\nkzeKJw8zCr+o+qAHUoqTZAAf1GaMOwCKN8rZ5zrulbkrugPY783UKJtfyJc8+BPT\ndixOerTC5cvzFNHENIKXMTh7Pbww2jdnFCn2eGA1kmyJGkRFhKKQ9kerlUcfOdQB\nw51jMfgZRoG/hvSnrlrYHJQx1hpUiBV5eyEcLHnlbiJj7cNTvqcrt2nHpy/1Co1H\n5uiQou5I8ETTvTQrtWNgCtUBg1ZqaKZw8tanSY4cHXoeP5s4uQl1yTEGCEDDFB9y\nE/yO9xTfak3Avv1h6FZ2Lw+ipVLnlurtpo/jGmr4UgoKV4MZ1hFSseIEWQVyXJ+4\nkP2gZ/LZF84eYqRKANYGWbKp/fKJQgnn/nhKgySfx4dKHJFRpVNgiGzNYyYwOtOC\nBWrLIqgvETl+MZZPMPwt8T7ZCYIR5fzQ1itGM3ffmsh9DIvRyu32DRWBcqgiDE7o\n866L+C6Kk2RyCS8dB3Ep4LW7kO42k0Rq6cvkO8wV+CjbTF/i8OQEclDMxr+ruoN0\nIKEp2thRZA39iDHGAIPyCsryrZhpEJ+uOfMykWKc0j957CpXLck=\n=Qmpp\n-----END PGP SIGNATURE-----\n" }, { "path": "lib/spack/spack/test/data/mirrors/v2_layout/signed/build_cache/test-debian6-m1-gcc-10.2.1-libdwarf-20130729-u5uz3dcch5if4eve4sef67o2rf2lbfgh.spec.json.sig", "content": "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA512\n\n{\n\"spec\":{\n\"_meta\":{\n\"version\":5\n},\n\"nodes\":[\n{\n\"name\":\"libdwarf\",\n\"version\":\"20130729\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":{\n\"name\":\"m1\",\n\"vendor\":\"Apple\",\n\"features\":[\n\"aes\",\n\"asimd\",\n\"asimddp\",\n\"asimdfhm\",\n\"asimdhp\",\n\"asimdrdm\",\n\"atomics\",\n\"cpuid\",\n\"crc32\",\n\"dcpodp\",\n\"dcpop\",\n\"dit\",\n\"evtstrm\",\n\"fcma\",\n\"flagm\",\n\"flagm2\",\n\"fp\",\n\"fphp\",\n\"frint\",\n\"ilrcpc\",\n\"jscvt\",\n\"lrcpc\",\n\"paca\",\n\"pacg\",\n\"pmull\",\n\"sb\",\n\"sha1\",\n\"sha2\",\n\"sha3\",\n\"sha512\",\n\"ssbs\",\n\"uscat\"\n],\n\"generation\":0,\n\"parents\":[\n\"armv8.4a\"\n],\n\"cpupart\":\"0x022\"\n}\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"package_hash\":\"n7axrpelzl5kjuctt4yoaaf33gvgnik6cx7fjudwhc6hvywdrr4q====\",\n\"dependencies\":[\n{\n\"name\":\"compiler-wrapper\",\n\"hash\":\"qeehcxyvluwnihsc2qxstmpomtxo3lrc\",\n\"parameters\":{\n\"deptypes\":[\n\"build\"\n],\n\"virtuals\":[]\n}\n},\n{\n\"name\":\"gcc\",\n\"hash\":\"vd7v4ssgnoqdplgxyig3orum67n4vmhq\",\n\"parameters\":{\n\"deptypes\":[\n\"build\"\n],\n\"virtuals\":[\n\"c\",\n\"cxx\"\n]\n}\n},\n{\n\"name\":\"gcc-runtime\",\n\"hash\":\"izgzpzeljwairalfjm3k6fntbb64nt6n\",\n\"parameters\":{\n\"deptypes\":[\n\"link\"\n],\n\"virtuals\":[]\n}\n},\n{\n\"name\":\"libelf\",\n\"hash\":\"jr3yipyxyjulcdvckwwwjrrumis7glpa\",\n\"parameters\":{\n\"deptypes\":[\n\"build\",\n\"link\"\n],\n\"virtuals\":[]\n}\n}\n],\n\"annotations\":{\n\"original_specfile_version\":5\n},\n\"hash\":\"u5uz3dcch5if4eve4sef67o2rf2lbfgh\"\n},\n{\n\"name\":\"compiler-wrapper\",\n\"version\":\"1.0\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":{\n\"name\":\"m1\",\n\"vendor\":\"Apple\",\n\"features\":[\n\"aes\",\n\"asimd\",\n\"asimddp\",\n\"asimdfhm\",\n\"asimdhp\",\n\"asimdrdm\",\n\"atomics\",\n\"cpuid\",\n\"crc32\",\n\"dcpodp\",\n\"dcpop\",\n\"dit\",\n\"evtstrm\",\n\"fcma\",\n\"flagm\",\n\"flagm2\",\n\"fp\",\n\"fphp\",\n\"frint\",\n\"ilrcpc\",\n\"jscvt\",\n\"lrcpc\",\n\"paca\",\n\"pacg\",\n\"pmull\",\n\"sb\",\n\"sha1\",\n\"sha2\",\n\"sha3\",\n\"sha512\",\n\"ssbs\",\n\"uscat\"\n],\n\"generation\":0,\n\"parents\":[\n\"armv8.4a\"\n],\n\"cpupart\":\"0x022\"\n}\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"package_hash\":\"ss7ybgvqf2fa2lvkf67eavllfxpxthiml2dobtkdq6wn7zkczteq====\",\n\"annotations\":{\n\"original_specfile_version\":5\n},\n\"hash\":\"qeehcxyvluwnihsc2qxstmpomtxo3lrc\"\n},\n{\n\"name\":\"gcc\",\n\"version\":\"10.2.1\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":\"aarch64\"\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"languages\":[\n\"c\",\n\"c++\",\n\"fortran\"\n],\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"external\":{\n\"path\":\"/path\",\n\"module\":null,\n\"extra_attributes\":{\n\"compilers\":{\n\"c\":\"/path/bin/gcc-10\",\n\"cxx\":\"/path/bin/g++-10\",\n\"fortran\":\"/path/bin/gfortran-10\"\n}\n}\n},\n\"package_hash\":\"a7d6wvl2mh4od3uue3yxqonc7r7ihw3n3ldedu4kevqa32oy2ysa====\",\n\"annotations\":{\n\"original_specfile_version\":5\n},\n\"hash\":\"vd7v4ssgnoqdplgxyig3orum67n4vmhq\"\n},\n{\n\"name\":\"gcc-runtime\",\n\"version\":\"10.2.1\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":{\n\"name\":\"m1\",\n\"vendor\":\"Apple\",\n\"features\":[\n\"aes\",\n\"asimd\",\n\"asimddp\",\n\"asimdfhm\",\n\"asimdhp\",\n\"asimdrdm\",\n\"atomics\",\n\"cpuid\",\n\"crc32\",\n\"dcpodp\",\n\"dcpop\",\n\"dit\",\n\"evtstrm\",\n\"fcma\",\n\"flagm\",\n\"flagm2\",\n\"fp\",\n\"fphp\",\n\"frint\",\n\"ilrcpc\",\n\"jscvt\",\n\"lrcpc\",\n\"paca\",\n\"pacg\",\n\"pmull\",\n\"sb\",\n\"sha1\",\n\"sha2\",\n\"sha3\",\n\"sha512\",\n\"ssbs\",\n\"uscat\"\n],\n\"generation\":0,\n\"parents\":[\n\"armv8.4a\"\n],\n\"cpupart\":\"0x022\"\n}\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"package_hash\":\"up2pdsw5tfvmn5gwgb3opl46la3uxoptkr3udmradd54s7qo72ha====\",\n\"dependencies\":[\n{\n\"name\":\"gcc\",\n\"hash\":\"vd7v4ssgnoqdplgxyig3orum67n4vmhq\",\n\"parameters\":{\n\"deptypes\":[\n\"build\"\n],\n\"virtuals\":[]\n}\n}\n],\n\"annotations\":{\n\"original_specfile_version\":5\n},\n\"hash\":\"izgzpzeljwairalfjm3k6fntbb64nt6n\"\n},\n{\n\"name\":\"libelf\",\n\"version\":\"0.8.13\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":{\n\"name\":\"m1\",\n\"vendor\":\"Apple\",\n\"features\":[\n\"aes\",\n\"asimd\",\n\"asimddp\",\n\"asimdfhm\",\n\"asimdhp\",\n\"asimdrdm\",\n\"atomics\",\n\"cpuid\",\n\"crc32\",\n\"dcpodp\",\n\"dcpop\",\n\"dit\",\n\"evtstrm\",\n\"fcma\",\n\"flagm\",\n\"flagm2\",\n\"fp\",\n\"fphp\",\n\"frint\",\n\"ilrcpc\",\n\"jscvt\",\n\"lrcpc\",\n\"paca\",\n\"pacg\",\n\"pmull\",\n\"sb\",\n\"sha1\",\n\"sha2\",\n\"sha3\",\n\"sha512\",\n\"ssbs\",\n\"uscat\"\n],\n\"generation\":0,\n\"parents\":[\n\"armv8.4a\"\n],\n\"cpupart\":\"0x022\"\n}\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"package_hash\":\"ejr32l7tkp6uhdrlunqv4adkuxqwyac7vbqcjvg6dh72mll4cpiq====\",\n\"dependencies\":[\n{\n\"name\":\"compiler-wrapper\",\n\"hash\":\"qeehcxyvluwnihsc2qxstmpomtxo3lrc\",\n\"parameters\":{\n\"deptypes\":[\n\"build\"\n],\n\"virtuals\":[]\n}\n},\n{\n\"name\":\"gcc\",\n\"hash\":\"vd7v4ssgnoqdplgxyig3orum67n4vmhq\",\n\"parameters\":{\n\"deptypes\":[\n\"build\"\n],\n\"virtuals\":[\n\"c\"\n]\n}\n},\n{\n\"name\":\"gcc-runtime\",\n\"hash\":\"izgzpzeljwairalfjm3k6fntbb64nt6n\",\n\"parameters\":{\n\"deptypes\":[\n\"link\"\n],\n\"virtuals\":[]\n}\n}\n],\n\"annotations\":{\n\"original_specfile_version\":5\n},\n\"hash\":\"jr3yipyxyjulcdvckwwwjrrumis7glpa\"\n}\n]\n},\n\"buildcache_layout_version\":2,\n\"binary_cache_checksum\":{\n\"hash_algorithm\":\"sha256\",\n\"hash\":\"0898457b4cc4b18d71059ea254667fb6690f5933c82e1627f9fed3606488dbca\"\n}\n}\n-----BEGIN PGP SIGNATURE-----\n\niQIzBAEBCgAdFiEEy6ssEDLG/1B4BJ7A+mHVDBLK034FAmgnhqIACgkQ+mHVDBLK\n035oXBAAj12qztxIYhTbNRq0jpk7/ZfCLRDz/XyqzKx2JbS+p3DfZruVZV/OMZ9I\nHlj9GYxQEwLGVsEMXoZDWtUytcte3m6sCG6H8fZGKw6IWQ6eiDR5i7TJWSuPvWGU\nNMH57kvSJlICLP9x6NWjQeyLAI4I3kASk+Ei/WHAGqIiP9CR1O5IXheMusPDAEjd\n2IR7khPvJTwpD6rzMHPou9BWk0Jqefb9qHhaJnc0Ga1D5HCS2VdGltViQ0XCX7/7\nnkWV9ad9NOvbO9oQYIW1jRY8D9Iw9vp2d77Dv5eUzI8or5c5x0VFAHpQL0FUxIR9\nLpHWUohDiAp3M4kmZqLBPl1Qf2jAXFXiSmcrLhKD5eWhdiwn3Bkhs2JiSiJpHt6K\nSa970evIFcGw6sUBGznsuFxmXFfp84LYvzIVjacuzkm9WDvbEE/5pa2b5Pxr7BmH\nd2xDmAYmZVOso6INf3ZEXOyMBPWyGyq9Hy/8Nyg/+7w2d4ICEG/z/N13VsTqRoXc\nrb8I0xDE9iCXCelQJYlJcJ2UMZk9E76zd3Bd2WcgCTrrnHsg0fBjmNeyPJcBN8hA\nam5Lq/Cxqm2Jo2qnjoVmCt8/TBkvT2w8PTpR5uTEbLDl2ghyzxyBkX7a8ldKx55f\naL8/OxN+u0pyISTDs5AoZ1YbhgDMiBiZV8ZDIB8PzU8pE78De3Q=\n=YbRr\n-----END PGP SIGNATURE-----\n" }, { "path": "lib/spack/spack/test/data/mirrors/v2_layout/signed/build_cache/test-debian6-m1-gcc-10.2.1-libelf-0.8.13-jr3yipyxyjulcdvckwwwjrrumis7glpa.spec.json.sig", "content": "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA512\n\n{\n\"spec\":{\n\"_meta\":{\n\"version\":5\n},\n\"nodes\":[\n{\n\"name\":\"libelf\",\n\"version\":\"0.8.13\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":{\n\"name\":\"m1\",\n\"vendor\":\"Apple\",\n\"features\":[\n\"aes\",\n\"asimd\",\n\"asimddp\",\n\"asimdfhm\",\n\"asimdhp\",\n\"asimdrdm\",\n\"atomics\",\n\"cpuid\",\n\"crc32\",\n\"dcpodp\",\n\"dcpop\",\n\"dit\",\n\"evtstrm\",\n\"fcma\",\n\"flagm\",\n\"flagm2\",\n\"fp\",\n\"fphp\",\n\"frint\",\n\"ilrcpc\",\n\"jscvt\",\n\"lrcpc\",\n\"paca\",\n\"pacg\",\n\"pmull\",\n\"sb\",\n\"sha1\",\n\"sha2\",\n\"sha3\",\n\"sha512\",\n\"ssbs\",\n\"uscat\"\n],\n\"generation\":0,\n\"parents\":[\n\"armv8.4a\"\n],\n\"cpupart\":\"0x022\"\n}\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"package_hash\":\"ejr32l7tkp6uhdrlunqv4adkuxqwyac7vbqcjvg6dh72mll4cpiq====\",\n\"dependencies\":[\n{\n\"name\":\"compiler-wrapper\",\n\"hash\":\"qeehcxyvluwnihsc2qxstmpomtxo3lrc\",\n\"parameters\":{\n\"deptypes\":[\n\"build\"\n],\n\"virtuals\":[]\n}\n},\n{\n\"name\":\"gcc\",\n\"hash\":\"vd7v4ssgnoqdplgxyig3orum67n4vmhq\",\n\"parameters\":{\n\"deptypes\":[\n\"build\"\n],\n\"virtuals\":[\n\"c\"\n]\n}\n},\n{\n\"name\":\"gcc-runtime\",\n\"hash\":\"izgzpzeljwairalfjm3k6fntbb64nt6n\",\n\"parameters\":{\n\"deptypes\":[\n\"link\"\n],\n\"virtuals\":[]\n}\n}\n],\n\"annotations\":{\n\"original_specfile_version\":5\n},\n\"hash\":\"jr3yipyxyjulcdvckwwwjrrumis7glpa\"\n},\n{\n\"name\":\"compiler-wrapper\",\n\"version\":\"1.0\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":{\n\"name\":\"m1\",\n\"vendor\":\"Apple\",\n\"features\":[\n\"aes\",\n\"asimd\",\n\"asimddp\",\n\"asimdfhm\",\n\"asimdhp\",\n\"asimdrdm\",\n\"atomics\",\n\"cpuid\",\n\"crc32\",\n\"dcpodp\",\n\"dcpop\",\n\"dit\",\n\"evtstrm\",\n\"fcma\",\n\"flagm\",\n\"flagm2\",\n\"fp\",\n\"fphp\",\n\"frint\",\n\"ilrcpc\",\n\"jscvt\",\n\"lrcpc\",\n\"paca\",\n\"pacg\",\n\"pmull\",\n\"sb\",\n\"sha1\",\n\"sha2\",\n\"sha3\",\n\"sha512\",\n\"ssbs\",\n\"uscat\"\n],\n\"generation\":0,\n\"parents\":[\n\"armv8.4a\"\n],\n\"cpupart\":\"0x022\"\n}\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"package_hash\":\"ss7ybgvqf2fa2lvkf67eavllfxpxthiml2dobtkdq6wn7zkczteq====\",\n\"annotations\":{\n\"original_specfile_version\":5\n},\n\"hash\":\"qeehcxyvluwnihsc2qxstmpomtxo3lrc\"\n},\n{\n\"name\":\"gcc\",\n\"version\":\"10.2.1\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":\"aarch64\"\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"languages\":[\n\"c\",\n\"c++\",\n\"fortran\"\n],\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"external\":{\n\"path\":\"/path\",\n\"module\":null,\n\"extra_attributes\":{\n\"compilers\":{\n\"c\":\"/path/bin/gcc-10\",\n\"cxx\":\"/path/bin/g++-10\",\n\"fortran\":\"/path/bin/gfortran-10\"\n}\n}\n},\n\"package_hash\":\"a7d6wvl2mh4od3uue3yxqonc7r7ihw3n3ldedu4kevqa32oy2ysa====\",\n\"annotations\":{\n\"original_specfile_version\":5\n},\n\"hash\":\"vd7v4ssgnoqdplgxyig3orum67n4vmhq\"\n},\n{\n\"name\":\"gcc-runtime\",\n\"version\":\"10.2.1\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":{\n\"name\":\"m1\",\n\"vendor\":\"Apple\",\n\"features\":[\n\"aes\",\n\"asimd\",\n\"asimddp\",\n\"asimdfhm\",\n\"asimdhp\",\n\"asimdrdm\",\n\"atomics\",\n\"cpuid\",\n\"crc32\",\n\"dcpodp\",\n\"dcpop\",\n\"dit\",\n\"evtstrm\",\n\"fcma\",\n\"flagm\",\n\"flagm2\",\n\"fp\",\n\"fphp\",\n\"frint\",\n\"ilrcpc\",\n\"jscvt\",\n\"lrcpc\",\n\"paca\",\n\"pacg\",\n\"pmull\",\n\"sb\",\n\"sha1\",\n\"sha2\",\n\"sha3\",\n\"sha512\",\n\"ssbs\",\n\"uscat\"\n],\n\"generation\":0,\n\"parents\":[\n\"armv8.4a\"\n],\n\"cpupart\":\"0x022\"\n}\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"package_hash\":\"up2pdsw5tfvmn5gwgb3opl46la3uxoptkr3udmradd54s7qo72ha====\",\n\"dependencies\":[\n{\n\"name\":\"gcc\",\n\"hash\":\"vd7v4ssgnoqdplgxyig3orum67n4vmhq\",\n\"parameters\":{\n\"deptypes\":[\n\"build\"\n],\n\"virtuals\":[]\n}\n}\n],\n\"annotations\":{\n\"original_specfile_version\":5\n},\n\"hash\":\"izgzpzeljwairalfjm3k6fntbb64nt6n\"\n}\n]\n},\n\"buildcache_layout_version\":2,\n\"binary_cache_checksum\":{\n\"hash_algorithm\":\"sha256\",\n\"hash\":\"c068bcd1a27a3081c07ba775d83e90228e340bb6a7f0d55deb18a462760c4bcf\"\n}\n}\n-----BEGIN PGP SIGNATURE-----\n\niQIzBAEBCgAdFiEEy6ssEDLG/1B4BJ7A+mHVDBLK034FAmgnhqIACgkQ+mHVDBLK\n0356nQ//aVMUZU8Ly8/b1H4nvKM8Vyd275aFK64rvO89mERDNiYIOKk1pmYSMldU\n+ltx2iIfVTUCEWYYJb/4UXWmw6SLAXIZ5mtrkALDAeDSih4wqIdevM3yii7pn8Oh\n/OEyDX8N5k05pnxFLYqR/2gA6vvdxHFd9/h4/zy2Z5w6m1hXb5jtS2ECyYN72nYN\n8QnnkXWZYturOhb4GawWY1l/rHIBqAseCQXSGR6UyrHTEGLUgT0+VQZwgxLNM4uG\nxj4xCDTgKiOesa5+3WE8Ug2wDIm48Prvg4qFmNrofguRNiIsNrl5k7wRiJWdfkjc\ngzs9URYddoCTRR2wpN0CaAQ268UlwZUCjPSrxgCNeqRi4Ob9Q4n37TKXNcVw46Ud\nMXRezAf+wyPGkq4vudh7cu11mHUcTeev82GM5bYQa6dSna3WvPpie/rx0TZYRkKE\nhesDW/41ZtFDANfXa7r011ngS5zZwak3zUaoqOdLNhN/xL4TFsZ19uSUdSZHAgSk\n9Sr3xodwV2D5H6gDuOtAo1vRod1Fx+yoi3BubX0sI5QuFgvtJrHVZmVj2bnGMBKI\ngR17q1ZHOmp3yPhVE9ZsiLKn9r3yIsfVhoTB6mXOnvq2q1fBxyrEpIGzIUmWfuTm\nvLn4nXt7PD78msiG/GZt6fShYBAwVfuvG+M1AQrsyGGoW2Bty7M=\n=hLvB\n-----END PGP SIGNATURE-----\n" }, { "path": "lib/spack/spack/test/data/mirrors/v2_layout/signed/build_cache/test-debian6-m1-none-none-compiler-wrapper-1.0-qeehcxyvluwnihsc2qxstmpomtxo3lrc.spec.json.sig", "content": "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA512\n\n{\n\"spec\":{\n\"_meta\":{\n\"version\":5\n},\n\"nodes\":[\n{\n\"name\":\"compiler-wrapper\",\n\"version\":\"1.0\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":{\n\"name\":\"m1\",\n\"vendor\":\"Apple\",\n\"features\":[\n\"aes\",\n\"asimd\",\n\"asimddp\",\n\"asimdfhm\",\n\"asimdhp\",\n\"asimdrdm\",\n\"atomics\",\n\"cpuid\",\n\"crc32\",\n\"dcpodp\",\n\"dcpop\",\n\"dit\",\n\"evtstrm\",\n\"fcma\",\n\"flagm\",\n\"flagm2\",\n\"fp\",\n\"fphp\",\n\"frint\",\n\"ilrcpc\",\n\"jscvt\",\n\"lrcpc\",\n\"paca\",\n\"pacg\",\n\"pmull\",\n\"sb\",\n\"sha1\",\n\"sha2\",\n\"sha3\",\n\"sha512\",\n\"ssbs\",\n\"uscat\"\n],\n\"generation\":0,\n\"parents\":[\n\"armv8.4a\"\n],\n\"cpupart\":\"0x022\"\n}\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"package_hash\":\"ss7ybgvqf2fa2lvkf67eavllfxpxthiml2dobtkdq6wn7zkczteq====\",\n\"annotations\":{\n\"original_specfile_version\":5\n},\n\"hash\":\"qeehcxyvluwnihsc2qxstmpomtxo3lrc\"\n}\n]\n},\n\"buildcache_layout_version\":2,\n\"binary_cache_checksum\":{\n\"hash_algorithm\":\"sha256\",\n\"hash\":\"2c1c5576e30b7063aa02a22111eb24b3f2a93c35ac0f64b4e491c7078706c0ea\"\n}\n}\n-----BEGIN PGP SIGNATURE-----\n\niQIzBAEBCgAdFiEEy6ssEDLG/1B4BJ7A+mHVDBLK034FAmgnhqIACgkQ+mHVDBLK\n037BIQ//U30gx1qTt5cQs+I6fwqQSase8DT7Hi8VdYxMuBTVbEpnPScNpcH03ITC\nKWVbXvEAPBdoWEfAHpuOJr2pm013dYXaWp1k0G6pLSvnR17LEDTJs0ixAurH4vDr\n4VXPerPR57sMi0WYomi1+dJhvA3S85+m6KBPLhXgi9Y28leDrFpjBwxVoIN7yUP2\ntenMI9jAoGh/hts1pIPbALmKbeGUKC2MPu9MF0CtkbbE1VOkeJ6jkZLGki7AAYZ0\nTSWAeWDk6EG90TZ6ls2anUPI1mNc7JdPqq8L0+jWAwLJi3i/JiDAGUM99hpu9cCF\nNvZn+eQFOKrE0WG1KsF4vQilOAuE3P+QLomcfZdf2UNi73XPWIF5j46r50oPmXZE\n+mVUyw7CUbHMZlXvWml0pdugEER1Kyc2nLZdLZYAT92AsPbAcDBQKsm1xf66lOB+\nFPPLc97oybcFFldrjmUJAASJBeAihZG1aDm6dYBxtynMzzRGdq2+R1chHMOQ5Wej\n8ZvyRv+TOPUTtRkAxrUpq6wA+BUoq+OBDltOs9mXUIcV3rpOq5nTjKZ5FLMtGaDw\nNo0E5gwceDDLeshT9nAHaqcmSY1LK+/5+aDxOFRm4yRTI+GLJzg8FZCJbJRLstrD\nTs4zKdcb0kukKdE9raqWw7xuhbjz2ORiEicZzckzvB1Lx38bG2s=\n=T5l5\n-----END PGP SIGNATURE-----\n" }, { "path": "lib/spack/spack/test/data/mirrors/v2_layout/signed/build_cache/test-debian6-m1-none-none-gcc-runtime-10.2.1-izgzpzeljwairalfjm3k6fntbb64nt6n.spec.json.sig", "content": "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA512\n\n{\n\"spec\":{\n\"_meta\":{\n\"version\":5\n},\n\"nodes\":[\n{\n\"name\":\"gcc-runtime\",\n\"version\":\"10.2.1\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":{\n\"name\":\"m1\",\n\"vendor\":\"Apple\",\n\"features\":[\n\"aes\",\n\"asimd\",\n\"asimddp\",\n\"asimdfhm\",\n\"asimdhp\",\n\"asimdrdm\",\n\"atomics\",\n\"cpuid\",\n\"crc32\",\n\"dcpodp\",\n\"dcpop\",\n\"dit\",\n\"evtstrm\",\n\"fcma\",\n\"flagm\",\n\"flagm2\",\n\"fp\",\n\"fphp\",\n\"frint\",\n\"ilrcpc\",\n\"jscvt\",\n\"lrcpc\",\n\"paca\",\n\"pacg\",\n\"pmull\",\n\"sb\",\n\"sha1\",\n\"sha2\",\n\"sha3\",\n\"sha512\",\n\"ssbs\",\n\"uscat\"\n],\n\"generation\":0,\n\"parents\":[\n\"armv8.4a\"\n],\n\"cpupart\":\"0x022\"\n}\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"package_hash\":\"up2pdsw5tfvmn5gwgb3opl46la3uxoptkr3udmradd54s7qo72ha====\",\n\"dependencies\":[\n{\n\"name\":\"gcc\",\n\"hash\":\"vd7v4ssgnoqdplgxyig3orum67n4vmhq\",\n\"parameters\":{\n\"deptypes\":[\n\"build\"\n],\n\"virtuals\":[]\n}\n}\n],\n\"annotations\":{\n\"original_specfile_version\":5\n},\n\"hash\":\"izgzpzeljwairalfjm3k6fntbb64nt6n\"\n},\n{\n\"name\":\"gcc\",\n\"version\":\"10.2.1\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":\"aarch64\"\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"languages\":[\n\"c\",\n\"c++\",\n\"fortran\"\n],\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"external\":{\n\"path\":\"/path\",\n\"module\":null,\n\"extra_attributes\":{\n\"compilers\":{\n\"c\":\"/path/bin/gcc-10\",\n\"cxx\":\"/path/bin/g++-10\",\n\"fortran\":\"/path/bin/gfortran-10\"\n}\n}\n},\n\"package_hash\":\"a7d6wvl2mh4od3uue3yxqonc7r7ihw3n3ldedu4kevqa32oy2ysa====\",\n\"annotations\":{\n\"original_specfile_version\":5\n},\n\"hash\":\"vd7v4ssgnoqdplgxyig3orum67n4vmhq\"\n}\n]\n},\n\"buildcache_layout_version\":2,\n\"binary_cache_checksum\":{\n\"hash_algorithm\":\"sha256\",\n\"hash\":\"f33e7a6798a5fb2db6e538d3a530cc79b298e36d56a1df385d93889a9ba431d0\"\n}\n}\n-----BEGIN PGP SIGNATURE-----\n\niQIzBAEBCgAdFiEEy6ssEDLG/1B4BJ7A+mHVDBLK034FAmgnhqIACgkQ+mHVDBLK\n037zHBAAsqy4wItctMqauuna+JjxT1HM7YJElXzqjOWmxyuAzUzjXlhR2DBd/2TI\nZEN2q3Z3XY9sCjhZ/4c9wDfMNYLUBLMHuenyV3fOqsfIVL8NprrkGc5mOiJ8HbRk\nu00qXWogsYSEmbGrlfDKf4HmZtgPNs82+Li1MD5udDUzyApuVbObJumSRh6/1QHm\nBcQZgMlSCd8xsTxJudXKAnfpemqE41LF0znuU0x5Hj/hU1A3CELynQrLEYnJpzpR\nja2l341cBQKNy86kX1/eHQtBJverjFoD3Nx4per8/qUc+xTH0ejMuseyd9P3RLnd\nWShY8Uk72f1OLGzq5RvayP1M/dBWedajKz5gYOD19pCuFEdQm1LkZhxRWJ35PYMV\nCqzY/uJgs33zyYkNJKO8CKG5j7Y8zOuZ3YFN8DKmoWa+lC4gFIsXm42BttqiQ5+x\nQ65YkX/DdPYO6dcUety1j3NuNr70W6PsLyqKBny1WOzKCx25nmzftS0OA76F6UZA\nhDneqltGrYEQTowU5I7V14f3SMeO8xje3BcqhOAn956/JJObd5VbwqcHwcslwEJA\ntL3361qbpkc7xURnhciV1eL3RYR9Q4xDnvI1i/k8J8E8W373TviK3r2MG/oKZ6N9\nn+ehBZhSIT+QUgqylATekoMQfohNVbDQEsQhj96Ky1CC2Iqo1/c=\n=UIyv\n-----END PGP SIGNATURE-----\n" }, { "path": "lib/spack/spack/test/data/mirrors/v2_layout/unsigned/build_cache/index.json", "content": "{\"database\":{\"version\":\"8\",\"installs\":{\"rqh2vuf6fqwkmipzgi2wjx352mq7y7ez\":{\"spec\":{\"name\":\"libelf\",\"version\":\"0.8.13\",\"arch\":{\"platform\":\"test\",\"platform_os\":\"debian6\",\"target\":{\"name\":\"core2\",\"vendor\":\"GenuineIntel\",\"features\":[\"mmx\",\"sse\",\"sse2\",\"ssse3\"],\"generation\":0,\"parents\":[\"nocona\"],\"cpupart\":\"\"}},\"namespace\":\"builtin_mock\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ejr32l7tkp6uhdrlunqv4adkuxqwyac7vbqcjvg6dh72mll4cpiq====\",\"annotations\":{\"original_specfile_version\":4,\"compiler\":\"gcc@=10.2.1\"},\"hash\":\"rqh2vuf6fqwkmipzgi2wjx352mq7y7ez\"},\"ref_count\":1,\"in_buildcache\":true},\"qeehcxyvluwnihsc2qxstmpomtxo3lrc\":{\"spec\":{\"name\":\"compiler-wrapper\",\"version\":\"1.0\",\"arch\":{\"platform\":\"test\",\"platform_os\":\"debian6\",\"target\":{\"name\":\"m1\",\"vendor\":\"Apple\",\"features\":[\"aes\",\"asimd\",\"asimddp\",\"asimdfhm\",\"asimdhp\",\"asimdrdm\",\"atomics\",\"cpuid\",\"crc32\",\"dcpodp\",\"dcpop\",\"dit\",\"evtstrm\",\"fcma\",\"flagm\",\"flagm2\",\"fp\",\"fphp\",\"frint\",\"ilrcpc\",\"jscvt\",\"lrcpc\",\"paca\",\"pacg\",\"pmull\",\"sb\",\"sha1\",\"sha2\",\"sha3\",\"sha512\",\"ssbs\",\"uscat\"],\"generation\":0,\"parents\":[\"armv8.4a\"],\"cpupart\":\"0x022\"}},\"namespace\":\"builtin_mock\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ss7ybgvqf2fa2lvkf67eavllfxpxthiml2dobtkdq6wn7zkczteq====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"qeehcxyvluwnihsc2qxstmpomtxo3lrc\"},\"ref_count\":2,\"in_buildcache\":true},\"vd7v4ssgnoqdplgxyig3orum67n4vmhq\":{\"spec\":{\"name\":\"gcc\",\"version\":\"10.2.1\",\"arch\":{\"platform\":\"test\",\"platform_os\":\"debian6\",\"target\":\"aarch64\"},\"namespace\":\"builtin_mock\",\"parameters\":{\"build_system\":\"generic\",\"languages\":[\"c\",\"c++\",\"fortran\"],\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"external\":{\"path\":\"/path\",\"module\":null,\"extra_attributes\":{\"compilers\":{\"c\":\"/path/bin/gcc-10\",\"cxx\":\"/path/bin/g++-10\",\"fortran\":\"/path/bin/gfortran-10\"}}},\"package_hash\":\"a7d6wvl2mh4od3uue3yxqonc7r7ihw3n3ldedu4kevqa32oy2ysa====\",\"annotations\":{\"original_specfile_version\":5},\"hash\":\"vd7v4ssgnoqdplgxyig3orum67n4vmhq\"},\"ref_count\":3,\"in_buildcache\":false},\"izgzpzeljwairalfjm3k6fntbb64nt6n\":{\"spec\":{\"name\":\"gcc-runtime\",\"version\":\"10.2.1\",\"arch\":{\"platform\":\"test\",\"platform_os\":\"debian6\",\"target\":{\"name\":\"m1\",\"vendor\":\"Apple\",\"features\":[\"aes\",\"asimd\",\"asimddp\",\"asimdfhm\",\"asimdhp\",\"asimdrdm\",\"atomics\",\"cpuid\",\"crc32\",\"dcpodp\",\"dcpop\",\"dit\",\"evtstrm\",\"fcma\",\"flagm\",\"flagm2\",\"fp\",\"fphp\",\"frint\",\"ilrcpc\",\"jscvt\",\"lrcpc\",\"paca\",\"pacg\",\"pmull\",\"sb\",\"sha1\",\"sha2\",\"sha3\",\"sha512\",\"ssbs\",\"uscat\"],\"generation\":0,\"parents\":[\"armv8.4a\"],\"cpupart\":\"0x022\"}},\"namespace\":\"builtin_mock\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"up2pdsw5tfvmn5gwgb3opl46la3uxoptkr3udmradd54s7qo72ha====\",\"dependencies\":[{\"name\":\"gcc\",\"hash\":\"vd7v4ssgnoqdplgxyig3orum67n4vmhq\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"izgzpzeljwairalfjm3k6fntbb64nt6n\"},\"ref_count\":2,\"in_buildcache\":true},\"jr3yipyxyjulcdvckwwwjrrumis7glpa\":{\"spec\":{\"name\":\"libelf\",\"version\":\"0.8.13\",\"arch\":{\"platform\":\"test\",\"platform_os\":\"debian6\",\"target\":{\"name\":\"m1\",\"vendor\":\"Apple\",\"features\":[\"aes\",\"asimd\",\"asimddp\",\"asimdfhm\",\"asimdhp\",\"asimdrdm\",\"atomics\",\"cpuid\",\"crc32\",\"dcpodp\",\"dcpop\",\"dit\",\"evtstrm\",\"fcma\",\"flagm\",\"flagm2\",\"fp\",\"fphp\",\"frint\",\"ilrcpc\",\"jscvt\",\"lrcpc\",\"paca\",\"pacg\",\"pmull\",\"sb\",\"sha1\",\"sha2\",\"sha3\",\"sha512\",\"ssbs\",\"uscat\"],\"generation\":0,\"parents\":[\"armv8.4a\"],\"cpupart\":\"0x022\"}},\"namespace\":\"builtin_mock\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"ejr32l7tkp6uhdrlunqv4adkuxqwyac7vbqcjvg6dh72mll4cpiq====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"qeehcxyvluwnihsc2qxstmpomtxo3lrc\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vd7v4ssgnoqdplgxyig3orum67n4vmhq\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"izgzpzeljwairalfjm3k6fntbb64nt6n\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"jr3yipyxyjulcdvckwwwjrrumis7glpa\"},\"ref_count\":1,\"in_buildcache\":true},\"u5uz3dcch5if4eve4sef67o2rf2lbfgh\":{\"spec\":{\"name\":\"libdwarf\",\"version\":\"20130729\",\"arch\":{\"platform\":\"test\",\"platform_os\":\"debian6\",\"target\":{\"name\":\"m1\",\"vendor\":\"Apple\",\"features\":[\"aes\",\"asimd\",\"asimddp\",\"asimdfhm\",\"asimdhp\",\"asimdrdm\",\"atomics\",\"cpuid\",\"crc32\",\"dcpodp\",\"dcpop\",\"dit\",\"evtstrm\",\"fcma\",\"flagm\",\"flagm2\",\"fp\",\"fphp\",\"frint\",\"ilrcpc\",\"jscvt\",\"lrcpc\",\"paca\",\"pacg\",\"pmull\",\"sb\",\"sha1\",\"sha2\",\"sha3\",\"sha512\",\"ssbs\",\"uscat\"],\"generation\":0,\"parents\":[\"armv8.4a\"],\"cpupart\":\"0x022\"}},\"namespace\":\"builtin_mock\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"n7axrpelzl5kjuctt4yoaaf33gvgnik6cx7fjudwhc6hvywdrr4q====\",\"dependencies\":[{\"name\":\"compiler-wrapper\",\"hash\":\"qeehcxyvluwnihsc2qxstmpomtxo3lrc\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[]}},{\"name\":\"gcc\",\"hash\":\"vd7v4ssgnoqdplgxyig3orum67n4vmhq\",\"parameters\":{\"deptypes\":[\"build\"],\"virtuals\":[\"c\",\"cxx\"]}},{\"name\":\"gcc-runtime\",\"hash\":\"izgzpzeljwairalfjm3k6fntbb64nt6n\",\"parameters\":{\"deptypes\":[\"link\"],\"virtuals\":[]}},{\"name\":\"libelf\",\"hash\":\"jr3yipyxyjulcdvckwwwjrrumis7glpa\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":5},\"hash\":\"u5uz3dcch5if4eve4sef67o2rf2lbfgh\"},\"ref_count\":0,\"in_buildcache\":true},\"sk2gqqz4n5njmvktycnd25wq25jxiqkr\":{\"spec\":{\"name\":\"libdwarf\",\"version\":\"20130729\",\"arch\":{\"platform\":\"test\",\"platform_os\":\"debian6\",\"target\":{\"name\":\"core2\",\"vendor\":\"GenuineIntel\",\"features\":[\"mmx\",\"sse\",\"sse2\",\"ssse3\"],\"generation\":0,\"parents\":[\"nocona\"],\"cpupart\":\"\"}},\"namespace\":\"builtin_mock\",\"parameters\":{\"build_system\":\"generic\",\"cflags\":[],\"cppflags\":[],\"cxxflags\":[],\"fflags\":[],\"ldflags\":[],\"ldlibs\":[]},\"package_hash\":\"n7axrpelzl5kjuctt4yoaaf33gvgnik6cx7fjudwhc6hvywdrr4q====\",\"dependencies\":[{\"name\":\"libelf\",\"hash\":\"rqh2vuf6fqwkmipzgi2wjx352mq7y7ez\",\"parameters\":{\"deptypes\":[\"build\",\"link\"],\"virtuals\":[]}}],\"annotations\":{\"original_specfile_version\":4,\"compiler\":\"gcc@=10.2.1\"},\"hash\":\"sk2gqqz4n5njmvktycnd25wq25jxiqkr\"},\"ref_count\":0,\"in_buildcache\":true}}}}" }, { "path": "lib/spack/spack/test/data/mirrors/v2_layout/unsigned/build_cache/index.json.hash", "content": "fc129b8fab649ab4c5623c874c73bd998a76fd30d2218b9d99340d045c1ec759\n" }, { "path": "lib/spack/spack/test/data/mirrors/v2_layout/unsigned/build_cache/test-debian6-core2-gcc-10.2.1-libdwarf-20130729-sk2gqqz4n5njmvktycnd25wq25jxiqkr.spec.json", "content": "{\n\"spec\":{\n\"_meta\":{\n\"version\":4\n},\n\"nodes\":[\n{\n\"name\":\"libdwarf\",\n\"version\":\"20130729\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":{\n\"name\":\"core2\",\n\"vendor\":\"GenuineIntel\",\n\"features\":[\n\"mmx\",\n\"sse\",\n\"sse2\",\n\"ssse3\"\n],\n\"generation\":0,\n\"parents\":[\n\"nocona\"\n],\n\"cpupart\":\"\"\n}\n},\n\"compiler\":{\n\"name\":\"gcc\",\n\"version\":\"10.2.1\"\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"package_hash\":\"n7axrpelzl5kjuctt4yoaaf33gvgnik6cx7fjudwhc6hvywdrr4q====\",\n\"dependencies\":[\n{\n\"name\":\"libelf\",\n\"hash\":\"rqh2vuf6fqwkmipzgi2wjx352mq7y7ez\",\n\"parameters\":{\n\"deptypes\":[\n\"build\",\n\"link\"\n],\n\"virtuals\":[]\n}\n}\n],\n\"hash\":\"sk2gqqz4n5njmvktycnd25wq25jxiqkr\"\n},\n{\n\"name\":\"libelf\",\n\"version\":\"0.8.13\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":{\n\"name\":\"core2\",\n\"vendor\":\"GenuineIntel\",\n\"features\":[\n\"mmx\",\n\"sse\",\n\"sse2\",\n\"ssse3\"\n],\n\"generation\":0,\n\"parents\":[\n\"nocona\"\n],\n\"cpupart\":\"\"\n}\n},\n\"compiler\":{\n\"name\":\"gcc\",\n\"version\":\"10.2.1\"\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"package_hash\":\"ejr32l7tkp6uhdrlunqv4adkuxqwyac7vbqcjvg6dh72mll4cpiq====\",\n\"hash\":\"rqh2vuf6fqwkmipzgi2wjx352mq7y7ez\"\n}\n]\n},\n\"buildcache_layout_version\":2,\n\"binary_cache_checksum\":{\n\"hash_algorithm\":\"sha256\",\n\"hash\":\"f31bce1bfdcaa9b11cd02b869dd07a843db9819737399b99ac614ab3552bd4b6\"\n}\n}" }, { "path": "lib/spack/spack/test/data/mirrors/v2_layout/unsigned/build_cache/test-debian6-core2-gcc-10.2.1-libelf-0.8.13-rqh2vuf6fqwkmipzgi2wjx352mq7y7ez.spec.json", "content": "{\n\"spec\":{\n\"_meta\":{\n\"version\":4\n},\n\"nodes\":[\n{\n\"name\":\"libelf\",\n\"version\":\"0.8.13\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":{\n\"name\":\"core2\",\n\"vendor\":\"GenuineIntel\",\n\"features\":[\n\"mmx\",\n\"sse\",\n\"sse2\",\n\"ssse3\"\n],\n\"generation\":0,\n\"parents\":[\n\"nocona\"\n],\n\"cpupart\":\"\"\n}\n},\n\"compiler\":{\n\"name\":\"gcc\",\n\"version\":\"10.2.1\"\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"package_hash\":\"ejr32l7tkp6uhdrlunqv4adkuxqwyac7vbqcjvg6dh72mll4cpiq====\",\n\"hash\":\"rqh2vuf6fqwkmipzgi2wjx352mq7y7ez\"\n}\n]\n},\n\"buildcache_layout_version\":2,\n\"binary_cache_checksum\":{\n\"hash_algorithm\":\"sha256\",\n\"hash\":\"523ebc3691b0687cf93fcd477002972ea7d6da5d3e8d46636b30d4f1052fcbf2\"\n}\n}" }, { "path": "lib/spack/spack/test/data/mirrors/v2_layout/unsigned/build_cache/test-debian6-m1-gcc-10.2.1-libdwarf-20130729-u5uz3dcch5if4eve4sef67o2rf2lbfgh.spec.json", "content": "{\n\"spec\":{\n\"_meta\":{\n\"version\":5\n},\n\"nodes\":[\n{\n\"name\":\"libdwarf\",\n\"version\":\"20130729\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":{\n\"name\":\"m1\",\n\"vendor\":\"Apple\",\n\"features\":[\n\"aes\",\n\"asimd\",\n\"asimddp\",\n\"asimdfhm\",\n\"asimdhp\",\n\"asimdrdm\",\n\"atomics\",\n\"cpuid\",\n\"crc32\",\n\"dcpodp\",\n\"dcpop\",\n\"dit\",\n\"evtstrm\",\n\"fcma\",\n\"flagm\",\n\"flagm2\",\n\"fp\",\n\"fphp\",\n\"frint\",\n\"ilrcpc\",\n\"jscvt\",\n\"lrcpc\",\n\"paca\",\n\"pacg\",\n\"pmull\",\n\"sb\",\n\"sha1\",\n\"sha2\",\n\"sha3\",\n\"sha512\",\n\"ssbs\",\n\"uscat\"\n],\n\"generation\":0,\n\"parents\":[\n\"armv8.4a\"\n],\n\"cpupart\":\"0x022\"\n}\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"package_hash\":\"n7axrpelzl5kjuctt4yoaaf33gvgnik6cx7fjudwhc6hvywdrr4q====\",\n\"dependencies\":[\n{\n\"name\":\"compiler-wrapper\",\n\"hash\":\"qeehcxyvluwnihsc2qxstmpomtxo3lrc\",\n\"parameters\":{\n\"deptypes\":[\n\"build\"\n],\n\"virtuals\":[]\n}\n},\n{\n\"name\":\"gcc\",\n\"hash\":\"vd7v4ssgnoqdplgxyig3orum67n4vmhq\",\n\"parameters\":{\n\"deptypes\":[\n\"build\"\n],\n\"virtuals\":[\n\"c\",\n\"cxx\"\n]\n}\n},\n{\n\"name\":\"gcc-runtime\",\n\"hash\":\"izgzpzeljwairalfjm3k6fntbb64nt6n\",\n\"parameters\":{\n\"deptypes\":[\n\"link\"\n],\n\"virtuals\":[]\n}\n},\n{\n\"name\":\"libelf\",\n\"hash\":\"jr3yipyxyjulcdvckwwwjrrumis7glpa\",\n\"parameters\":{\n\"deptypes\":[\n\"build\",\n\"link\"\n],\n\"virtuals\":[]\n}\n}\n],\n\"annotations\":{\n\"original_specfile_version\":5\n},\n\"hash\":\"u5uz3dcch5if4eve4sef67o2rf2lbfgh\"\n},\n{\n\"name\":\"compiler-wrapper\",\n\"version\":\"1.0\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":{\n\"name\":\"m1\",\n\"vendor\":\"Apple\",\n\"features\":[\n\"aes\",\n\"asimd\",\n\"asimddp\",\n\"asimdfhm\",\n\"asimdhp\",\n\"asimdrdm\",\n\"atomics\",\n\"cpuid\",\n\"crc32\",\n\"dcpodp\",\n\"dcpop\",\n\"dit\",\n\"evtstrm\",\n\"fcma\",\n\"flagm\",\n\"flagm2\",\n\"fp\",\n\"fphp\",\n\"frint\",\n\"ilrcpc\",\n\"jscvt\",\n\"lrcpc\",\n\"paca\",\n\"pacg\",\n\"pmull\",\n\"sb\",\n\"sha1\",\n\"sha2\",\n\"sha3\",\n\"sha512\",\n\"ssbs\",\n\"uscat\"\n],\n\"generation\":0,\n\"parents\":[\n\"armv8.4a\"\n],\n\"cpupart\":\"0x022\"\n}\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"package_hash\":\"ss7ybgvqf2fa2lvkf67eavllfxpxthiml2dobtkdq6wn7zkczteq====\",\n\"annotations\":{\n\"original_specfile_version\":5\n},\n\"hash\":\"qeehcxyvluwnihsc2qxstmpomtxo3lrc\"\n},\n{\n\"name\":\"gcc\",\n\"version\":\"10.2.1\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":\"aarch64\"\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"languages\":[\n\"c\",\n\"c++\",\n\"fortran\"\n],\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"external\":{\n\"path\":\"/path\",\n\"module\":null,\n\"extra_attributes\":{\n\"compilers\":{\n\"c\":\"/path/bin/gcc-10\",\n\"cxx\":\"/path/bin/g++-10\",\n\"fortran\":\"/path/bin/gfortran-10\"\n}\n}\n},\n\"package_hash\":\"a7d6wvl2mh4od3uue3yxqonc7r7ihw3n3ldedu4kevqa32oy2ysa====\",\n\"annotations\":{\n\"original_specfile_version\":5\n},\n\"hash\":\"vd7v4ssgnoqdplgxyig3orum67n4vmhq\"\n},\n{\n\"name\":\"gcc-runtime\",\n\"version\":\"10.2.1\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":{\n\"name\":\"m1\",\n\"vendor\":\"Apple\",\n\"features\":[\n\"aes\",\n\"asimd\",\n\"asimddp\",\n\"asimdfhm\",\n\"asimdhp\",\n\"asimdrdm\",\n\"atomics\",\n\"cpuid\",\n\"crc32\",\n\"dcpodp\",\n\"dcpop\",\n\"dit\",\n\"evtstrm\",\n\"fcma\",\n\"flagm\",\n\"flagm2\",\n\"fp\",\n\"fphp\",\n\"frint\",\n\"ilrcpc\",\n\"jscvt\",\n\"lrcpc\",\n\"paca\",\n\"pacg\",\n\"pmull\",\n\"sb\",\n\"sha1\",\n\"sha2\",\n\"sha3\",\n\"sha512\",\n\"ssbs\",\n\"uscat\"\n],\n\"generation\":0,\n\"parents\":[\n\"armv8.4a\"\n],\n\"cpupart\":\"0x022\"\n}\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"package_hash\":\"up2pdsw5tfvmn5gwgb3opl46la3uxoptkr3udmradd54s7qo72ha====\",\n\"dependencies\":[\n{\n\"name\":\"gcc\",\n\"hash\":\"vd7v4ssgnoqdplgxyig3orum67n4vmhq\",\n\"parameters\":{\n\"deptypes\":[\n\"build\"\n],\n\"virtuals\":[]\n}\n}\n],\n\"annotations\":{\n\"original_specfile_version\":5\n},\n\"hash\":\"izgzpzeljwairalfjm3k6fntbb64nt6n\"\n},\n{\n\"name\":\"libelf\",\n\"version\":\"0.8.13\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":{\n\"name\":\"m1\",\n\"vendor\":\"Apple\",\n\"features\":[\n\"aes\",\n\"asimd\",\n\"asimddp\",\n\"asimdfhm\",\n\"asimdhp\",\n\"asimdrdm\",\n\"atomics\",\n\"cpuid\",\n\"crc32\",\n\"dcpodp\",\n\"dcpop\",\n\"dit\",\n\"evtstrm\",\n\"fcma\",\n\"flagm\",\n\"flagm2\",\n\"fp\",\n\"fphp\",\n\"frint\",\n\"ilrcpc\",\n\"jscvt\",\n\"lrcpc\",\n\"paca\",\n\"pacg\",\n\"pmull\",\n\"sb\",\n\"sha1\",\n\"sha2\",\n\"sha3\",\n\"sha512\",\n\"ssbs\",\n\"uscat\"\n],\n\"generation\":0,\n\"parents\":[\n\"armv8.4a\"\n],\n\"cpupart\":\"0x022\"\n}\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"package_hash\":\"ejr32l7tkp6uhdrlunqv4adkuxqwyac7vbqcjvg6dh72mll4cpiq====\",\n\"dependencies\":[\n{\n\"name\":\"compiler-wrapper\",\n\"hash\":\"qeehcxyvluwnihsc2qxstmpomtxo3lrc\",\n\"parameters\":{\n\"deptypes\":[\n\"build\"\n],\n\"virtuals\":[]\n}\n},\n{\n\"name\":\"gcc\",\n\"hash\":\"vd7v4ssgnoqdplgxyig3orum67n4vmhq\",\n\"parameters\":{\n\"deptypes\":[\n\"build\"\n],\n\"virtuals\":[\n\"c\"\n]\n}\n},\n{\n\"name\":\"gcc-runtime\",\n\"hash\":\"izgzpzeljwairalfjm3k6fntbb64nt6n\",\n\"parameters\":{\n\"deptypes\":[\n\"link\"\n],\n\"virtuals\":[]\n}\n}\n],\n\"annotations\":{\n\"original_specfile_version\":5\n},\n\"hash\":\"jr3yipyxyjulcdvckwwwjrrumis7glpa\"\n}\n]\n},\n\"buildcache_layout_version\":2,\n\"binary_cache_checksum\":{\n\"hash_algorithm\":\"sha256\",\n\"hash\":\"2a6f045996e4998ec37679e1aa8a245795fbbffaf9844692ba2de6eeffcbc722\"\n}\n}" }, { "path": "lib/spack/spack/test/data/mirrors/v2_layout/unsigned/build_cache/test-debian6-m1-gcc-10.2.1-libelf-0.8.13-jr3yipyxyjulcdvckwwwjrrumis7glpa.spec.json", "content": "{\n\"spec\":{\n\"_meta\":{\n\"version\":5\n},\n\"nodes\":[\n{\n\"name\":\"libelf\",\n\"version\":\"0.8.13\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":{\n\"name\":\"m1\",\n\"vendor\":\"Apple\",\n\"features\":[\n\"aes\",\n\"asimd\",\n\"asimddp\",\n\"asimdfhm\",\n\"asimdhp\",\n\"asimdrdm\",\n\"atomics\",\n\"cpuid\",\n\"crc32\",\n\"dcpodp\",\n\"dcpop\",\n\"dit\",\n\"evtstrm\",\n\"fcma\",\n\"flagm\",\n\"flagm2\",\n\"fp\",\n\"fphp\",\n\"frint\",\n\"ilrcpc\",\n\"jscvt\",\n\"lrcpc\",\n\"paca\",\n\"pacg\",\n\"pmull\",\n\"sb\",\n\"sha1\",\n\"sha2\",\n\"sha3\",\n\"sha512\",\n\"ssbs\",\n\"uscat\"\n],\n\"generation\":0,\n\"parents\":[\n\"armv8.4a\"\n],\n\"cpupart\":\"0x022\"\n}\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"package_hash\":\"ejr32l7tkp6uhdrlunqv4adkuxqwyac7vbqcjvg6dh72mll4cpiq====\",\n\"dependencies\":[\n{\n\"name\":\"compiler-wrapper\",\n\"hash\":\"qeehcxyvluwnihsc2qxstmpomtxo3lrc\",\n\"parameters\":{\n\"deptypes\":[\n\"build\"\n],\n\"virtuals\":[]\n}\n},\n{\n\"name\":\"gcc\",\n\"hash\":\"vd7v4ssgnoqdplgxyig3orum67n4vmhq\",\n\"parameters\":{\n\"deptypes\":[\n\"build\"\n],\n\"virtuals\":[\n\"c\"\n]\n}\n},\n{\n\"name\":\"gcc-runtime\",\n\"hash\":\"izgzpzeljwairalfjm3k6fntbb64nt6n\",\n\"parameters\":{\n\"deptypes\":[\n\"link\"\n],\n\"virtuals\":[]\n}\n}\n],\n\"annotations\":{\n\"original_specfile_version\":5\n},\n\"hash\":\"jr3yipyxyjulcdvckwwwjrrumis7glpa\"\n},\n{\n\"name\":\"compiler-wrapper\",\n\"version\":\"1.0\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":{\n\"name\":\"m1\",\n\"vendor\":\"Apple\",\n\"features\":[\n\"aes\",\n\"asimd\",\n\"asimddp\",\n\"asimdfhm\",\n\"asimdhp\",\n\"asimdrdm\",\n\"atomics\",\n\"cpuid\",\n\"crc32\",\n\"dcpodp\",\n\"dcpop\",\n\"dit\",\n\"evtstrm\",\n\"fcma\",\n\"flagm\",\n\"flagm2\",\n\"fp\",\n\"fphp\",\n\"frint\",\n\"ilrcpc\",\n\"jscvt\",\n\"lrcpc\",\n\"paca\",\n\"pacg\",\n\"pmull\",\n\"sb\",\n\"sha1\",\n\"sha2\",\n\"sha3\",\n\"sha512\",\n\"ssbs\",\n\"uscat\"\n],\n\"generation\":0,\n\"parents\":[\n\"armv8.4a\"\n],\n\"cpupart\":\"0x022\"\n}\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"package_hash\":\"ss7ybgvqf2fa2lvkf67eavllfxpxthiml2dobtkdq6wn7zkczteq====\",\n\"annotations\":{\n\"original_specfile_version\":5\n},\n\"hash\":\"qeehcxyvluwnihsc2qxstmpomtxo3lrc\"\n},\n{\n\"name\":\"gcc\",\n\"version\":\"10.2.1\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":\"aarch64\"\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"languages\":[\n\"c\",\n\"c++\",\n\"fortran\"\n],\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"external\":{\n\"path\":\"/path\",\n\"module\":null,\n\"extra_attributes\":{\n\"compilers\":{\n\"c\":\"/path/bin/gcc-10\",\n\"cxx\":\"/path/bin/g++-10\",\n\"fortran\":\"/path/bin/gfortran-10\"\n}\n}\n},\n\"package_hash\":\"a7d6wvl2mh4od3uue3yxqonc7r7ihw3n3ldedu4kevqa32oy2ysa====\",\n\"annotations\":{\n\"original_specfile_version\":5\n},\n\"hash\":\"vd7v4ssgnoqdplgxyig3orum67n4vmhq\"\n},\n{\n\"name\":\"gcc-runtime\",\n\"version\":\"10.2.1\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":{\n\"name\":\"m1\",\n\"vendor\":\"Apple\",\n\"features\":[\n\"aes\",\n\"asimd\",\n\"asimddp\",\n\"asimdfhm\",\n\"asimdhp\",\n\"asimdrdm\",\n\"atomics\",\n\"cpuid\",\n\"crc32\",\n\"dcpodp\",\n\"dcpop\",\n\"dit\",\n\"evtstrm\",\n\"fcma\",\n\"flagm\",\n\"flagm2\",\n\"fp\",\n\"fphp\",\n\"frint\",\n\"ilrcpc\",\n\"jscvt\",\n\"lrcpc\",\n\"paca\",\n\"pacg\",\n\"pmull\",\n\"sb\",\n\"sha1\",\n\"sha2\",\n\"sha3\",\n\"sha512\",\n\"ssbs\",\n\"uscat\"\n],\n\"generation\":0,\n\"parents\":[\n\"armv8.4a\"\n],\n\"cpupart\":\"0x022\"\n}\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"package_hash\":\"up2pdsw5tfvmn5gwgb3opl46la3uxoptkr3udmradd54s7qo72ha====\",\n\"dependencies\":[\n{\n\"name\":\"gcc\",\n\"hash\":\"vd7v4ssgnoqdplgxyig3orum67n4vmhq\",\n\"parameters\":{\n\"deptypes\":[\n\"build\"\n],\n\"virtuals\":[]\n}\n}\n],\n\"annotations\":{\n\"original_specfile_version\":5\n},\n\"hash\":\"izgzpzeljwairalfjm3k6fntbb64nt6n\"\n}\n]\n},\n\"buildcache_layout_version\":2,\n\"binary_cache_checksum\":{\n\"hash_algorithm\":\"sha256\",\n\"hash\":\"59141e49bd05abe40639360cd9422020513781270a3461083fee0eba2af62ca0\"\n}\n}" }, { "path": "lib/spack/spack/test/data/mirrors/v2_layout/unsigned/build_cache/test-debian6-m1-none-none-compiler-wrapper-1.0-qeehcxyvluwnihsc2qxstmpomtxo3lrc.spec.json", "content": "{\n\"spec\":{\n\"_meta\":{\n\"version\":5\n},\n\"nodes\":[\n{\n\"name\":\"compiler-wrapper\",\n\"version\":\"1.0\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":{\n\"name\":\"m1\",\n\"vendor\":\"Apple\",\n\"features\":[\n\"aes\",\n\"asimd\",\n\"asimddp\",\n\"asimdfhm\",\n\"asimdhp\",\n\"asimdrdm\",\n\"atomics\",\n\"cpuid\",\n\"crc32\",\n\"dcpodp\",\n\"dcpop\",\n\"dit\",\n\"evtstrm\",\n\"fcma\",\n\"flagm\",\n\"flagm2\",\n\"fp\",\n\"fphp\",\n\"frint\",\n\"ilrcpc\",\n\"jscvt\",\n\"lrcpc\",\n\"paca\",\n\"pacg\",\n\"pmull\",\n\"sb\",\n\"sha1\",\n\"sha2\",\n\"sha3\",\n\"sha512\",\n\"ssbs\",\n\"uscat\"\n],\n\"generation\":0,\n\"parents\":[\n\"armv8.4a\"\n],\n\"cpupart\":\"0x022\"\n}\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"package_hash\":\"ss7ybgvqf2fa2lvkf67eavllfxpxthiml2dobtkdq6wn7zkczteq====\",\n\"annotations\":{\n\"original_specfile_version\":5\n},\n\"hash\":\"qeehcxyvluwnihsc2qxstmpomtxo3lrc\"\n}\n]\n},\n\"buildcache_layout_version\":2,\n\"binary_cache_checksum\":{\n\"hash_algorithm\":\"sha256\",\n\"hash\":\"f27ddff0ef4268acbe816c51f6f1fc907dc1010d31f2d6556b699c80f026c47d\"\n}\n}" }, { "path": "lib/spack/spack/test/data/mirrors/v2_layout/unsigned/build_cache/test-debian6-m1-none-none-gcc-runtime-10.2.1-izgzpzeljwairalfjm3k6fntbb64nt6n.spec.json", "content": "{\n\"spec\":{\n\"_meta\":{\n\"version\":5\n},\n\"nodes\":[\n{\n\"name\":\"gcc-runtime\",\n\"version\":\"10.2.1\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":{\n\"name\":\"m1\",\n\"vendor\":\"Apple\",\n\"features\":[\n\"aes\",\n\"asimd\",\n\"asimddp\",\n\"asimdfhm\",\n\"asimdhp\",\n\"asimdrdm\",\n\"atomics\",\n\"cpuid\",\n\"crc32\",\n\"dcpodp\",\n\"dcpop\",\n\"dit\",\n\"evtstrm\",\n\"fcma\",\n\"flagm\",\n\"flagm2\",\n\"fp\",\n\"fphp\",\n\"frint\",\n\"ilrcpc\",\n\"jscvt\",\n\"lrcpc\",\n\"paca\",\n\"pacg\",\n\"pmull\",\n\"sb\",\n\"sha1\",\n\"sha2\",\n\"sha3\",\n\"sha512\",\n\"ssbs\",\n\"uscat\"\n],\n\"generation\":0,\n\"parents\":[\n\"armv8.4a\"\n],\n\"cpupart\":\"0x022\"\n}\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"package_hash\":\"up2pdsw5tfvmn5gwgb3opl46la3uxoptkr3udmradd54s7qo72ha====\",\n\"dependencies\":[\n{\n\"name\":\"gcc\",\n\"hash\":\"vd7v4ssgnoqdplgxyig3orum67n4vmhq\",\n\"parameters\":{\n\"deptypes\":[\n\"build\"\n],\n\"virtuals\":[]\n}\n}\n],\n\"annotations\":{\n\"original_specfile_version\":5\n},\n\"hash\":\"izgzpzeljwairalfjm3k6fntbb64nt6n\"\n},\n{\n\"name\":\"gcc\",\n\"version\":\"10.2.1\",\n\"arch\":{\n\"platform\":\"test\",\n\"platform_os\":\"debian6\",\n\"target\":\"aarch64\"\n},\n\"namespace\":\"builtin_mock\",\n\"parameters\":{\n\"build_system\":\"generic\",\n\"languages\":[\n\"c\",\n\"c++\",\n\"fortran\"\n],\n\"cflags\":[],\n\"cppflags\":[],\n\"cxxflags\":[],\n\"fflags\":[],\n\"ldflags\":[],\n\"ldlibs\":[]\n},\n\"external\":{\n\"path\":\"/path\",\n\"module\":null,\n\"extra_attributes\":{\n\"compilers\":{\n\"c\":\"/path/bin/gcc-10\",\n\"cxx\":\"/path/bin/g++-10\",\n\"fortran\":\"/path/bin/gfortran-10\"\n}\n}\n},\n\"package_hash\":\"a7d6wvl2mh4od3uue3yxqonc7r7ihw3n3ldedu4kevqa32oy2ysa====\",\n\"annotations\":{\n\"original_specfile_version\":5\n},\n\"hash\":\"vd7v4ssgnoqdplgxyig3orum67n4vmhq\"\n}\n]\n},\n\"buildcache_layout_version\":2,\n\"binary_cache_checksum\":{\n\"hash_algorithm\":\"sha256\",\n\"hash\":\"348f23717c5641fa6bbb90862e62bf632367511c53e9c6450584f0d000841320\"\n}\n}" }, { "path": "lib/spack/spack/test/data/modules/lmod/alter_environment.yaml", "content": "enable:\n - lmod\nlmod:\n core_compilers:\n - 'clang@3.3'\n\n hierarchy:\n - mpi\n\n all:\n autoload: none\n filter:\n exclude_env_vars:\n - CMAKE_PREFIX_PATH\n environment:\n set:\n '{name}_ROOT': '{prefix}'\n\n 'platform=test target=x86_64':\n environment:\n set:\n FOO: 'foo'\n unset:\n - BAR\n\n 'platform=test target=core2':\n load:\n - 'foo/bar'\n" }, { "path": "lib/spack/spack/test/data/modules/lmod/autoload_all.yaml", "content": "enable:\n - lmod\nlmod:\n core_compilers:\n - 'clang@3.3'\n hierarchy:\n - mpi\n verbose: true\n\n all:\n autoload: all\n" }, { "path": "lib/spack/spack/test/data/modules/lmod/autoload_direct.yaml", "content": "enable:\n - lmod\nlmod:\n core_compilers:\n - 'clang@3.3'\n hierarchy:\n - mpi\n\n all:\n autoload: direct\n" }, { "path": "lib/spack/spack/test/data/modules/lmod/complex_hierarchy.yaml", "content": "enable:\n - lmod\nlmod:\n hash_length: 0\n\n core_compilers:\n - 'clang@15.0.0'\n\n core_specs:\n - 'mpich@3.0.1'\n\n hierarchy:\n - lapack\n - blas\n - mpi\n - python\n\n filter_hierarchy_specs:\n 'mpileaks@:2.1': [mpi]\n\n verbose: false\n\n all:\n autoload: all\n" }, { "path": "lib/spack/spack/test/data/modules/lmod/conflicts.yaml", "content": "enable:\n - lmod\nlmod:\n core_compilers:\n - 'clang@3.3'\n all:\n autoload: none\n conflict:\n - '{name}'\n - 'intel/14.0.1'\n" }, { "path": "lib/spack/spack/test/data/modules/lmod/core_compilers.yaml", "content": "enable:\n - lmod\nlmod:\n core_compilers:\n - 'clang@15.0.0'\n" }, { "path": "lib/spack/spack/test/data/modules/lmod/core_compilers_at_equal.yaml", "content": "enable:\n - lmod\nlmod:\n core_compilers:\n - 'clang@=15.0.0'\n" }, { "path": "lib/spack/spack/test/data/modules/lmod/core_compilers_empty.yaml", "content": "enable:\n - lmod\nlmod:\n all:\n autoload: none\n core_compilers: []\n hierarchy:\n - mpi\n" }, { "path": "lib/spack/spack/test/data/modules/lmod/exclude.yaml", "content": "enable:\n - lmod\nlmod:\n core_compilers:\n - 'clang@3.3'\n hierarchy:\n - mpi\n exclude:\n - callpath\n\n all:\n autoload: direct\n" }, { "path": "lib/spack/spack/test/data/modules/lmod/hide_implicits.yaml", "content": "enable:\n - lmod\nlmod:\n hide_implicits: true\n hash_length: 0\n core_compilers:\n - 'clang@3.3'\n hierarchy:\n - mpi\n\n all:\n autoload: direct\n" }, { "path": "lib/spack/spack/test/data/modules/lmod/missing_core_compilers.yaml", "content": "enable:\n - lmod\nlmod:\n all:\n autoload: none\n hierarchy:\n - mpi\n" }, { "path": "lib/spack/spack/test/data/modules/lmod/module_path_separator.yaml", "content": "enable:\n - lmod\nlmod:\n all:\n autoload: none\n core_compilers:\n - 'clang@3.3'\n" }, { "path": "lib/spack/spack/test/data/modules/lmod/no_arch.yaml", "content": "enable:\n - lmod\narch_folder: false\nlmod:\n all:\n autoload: none\n core_compilers:\n - 'clang@3.3'" }, { "path": "lib/spack/spack/test/data/modules/lmod/no_hash.yaml", "content": "enable:\n - lmod\nlmod:\n all:\n autoload: none\n hash_length: 0\n\n core_compilers:\n - 'clang@3.3'\n\n hierarchy:\n - mpi\n" }, { "path": "lib/spack/spack/test/data/modules/lmod/override_template.yaml", "content": "enable:\n - lmod\nlmod:\n core_compilers:\n - 'clang@3.3'\n hierarchy:\n - mpi\n\n all:\n template: 'override_from_modules.txt'\n autoload: none\n" }, { "path": "lib/spack/spack/test/data/modules/lmod/projections.yaml", "content": "enable:\n - lmod\nlmod:\n all:\n autoload: none\n projections:\n all: '{name}/v{version}'\n mpileaks: '{name}-mpiprojection'\n" }, { "path": "lib/spack/spack/test/data/modules/lmod/with_view.yaml", "content": "enable:\n - lmod\nuse_view: default\nlmod:\n all:\n autoload: none\n core_compilers:\n - 'clang@3.3'\n" }, { "path": "lib/spack/spack/test/data/modules/lmod/wrong_conflicts.yaml", "content": "enable:\n - lmod\nlmod:\n core_compilers:\n - 'clang@3.3'\n projections:\n all: '{name}/{version}-{compiler.name}'\n all:\n autoload: none\n conflict:\n - '{name}/{compiler.name}'\n" }, { "path": "lib/spack/spack/test/data/modules/tcl/alter_environment.yaml", "content": "enable:\n - tcl\ntcl:\n all:\n autoload: none\n filter:\n exclude_env_vars:\n - CMAKE_PREFIX_PATH\n environment:\n set:\n '{name}_ROOT': '{prefix}'\n\n 'platform=test target=x86_64':\n environment:\n set:\n FOO: 'foo'\n OMPI_MCA_mpi_leave_pinned: '1'\n unset:\n - BAR\n\n 'platform=test target=core2':\n load:\n - 'foo/bar'\n" }, { "path": "lib/spack/spack/test/data/modules/tcl/autoload_all.yaml", "content": "enable:\n - tcl\ntcl:\n verbose: true\n all:\n autoload: all\n" }, { "path": "lib/spack/spack/test/data/modules/tcl/autoload_direct.yaml", "content": "enable:\n - tcl\ntcl:\n all:\n autoload: direct\n" }, { "path": "lib/spack/spack/test/data/modules/tcl/autoload_with_constraints.yaml", "content": "enable:\n - tcl\ntcl:\n all:\n autoload: none\n ^mpich2:\n autoload: direct\n ^python:\n autoload: direct\n" }, { "path": "lib/spack/spack/test/data/modules/tcl/conflicts.yaml", "content": "enable:\n - tcl\ntcl:\n projections:\n all: '{name}/{version}-{compiler.name}'\n all:\n autoload: none\n conflict:\n - '{name}'\n - 'intel/14.0.1'\n" }, { "path": "lib/spack/spack/test/data/modules/tcl/exclude.yaml", "content": "enable:\n - tcl\ntcl:\n include:\n - zmpi\n exclude:\n - callpath\n - mpi\n all:\n autoload: direct\n" }, { "path": "lib/spack/spack/test/data/modules/tcl/exclude_implicits.yaml", "content": "# DEPRECATED: remove this in ?\n# See `hide_implicits.yaml` for the new syntax\nenable:\n - tcl\ntcl:\n exclude_implicits: true\n hash_length: 0\n all:\n autoload: direct\n" }, { "path": "lib/spack/spack/test/data/modules/tcl/hide_implicits.yaml", "content": "enable:\n - tcl\ntcl:\n hide_implicits: true\n hash_length: 0\n all:\n autoload: direct\n" }, { "path": "lib/spack/spack/test/data/modules/tcl/invalid_naming_scheme.yaml", "content": "enable:\n - tcl\ntcl:\n all:\n autoload: none\n # {variants} is not allowed in the naming scheme, see #2884\n projections:\n all: '{name}/{version}-{compiler.name}-{variants}'\n" }, { "path": "lib/spack/spack/test/data/modules/tcl/invalid_token_in_env_var_name.yaml", "content": "enable:\n - tcl\ntcl:\n all:\n autoload: none\n filter:\n exclude_env_vars:\n - CMAKE_PREFIX_PATH\n environment:\n set:\n '{name}_ROOT_{prefix}': '{prefix}'\n\n 'platform=test target=x86_64':\n environment:\n set:\n FOO_{variants}: 'foo'\n unset:\n - BAR\n\n 'platform=test target=x86':\n load:\n - 'foo/bar'\n" }, { "path": "lib/spack/spack/test/data/modules/tcl/module_path_separator.yaml", "content": "enable:\n - tcl\ntcl:\n all:\n autoload: none\n" }, { "path": "lib/spack/spack/test/data/modules/tcl/naming_scheme.yaml", "content": "enable:\n - tcl\ntcl:\n all:\n autoload: none\n naming_scheme: '{name}/{version}-{compiler.name}'\n" }, { "path": "lib/spack/spack/test/data/modules/tcl/no_arch.yaml", "content": "enable:\n - tcl\narch_folder: false\ntcl:\n all:\n autoload: none\n projections:\n all: ''" }, { "path": "lib/spack/spack/test/data/modules/tcl/override_config.yaml", "content": "enable:\n - tcl\ntcl:\n all:\n autoload: none\n suffixes:\n '^mpich': mpich\n mpileaks:\n suffixes:\n '+static': static\n mpileaks+opt::\n suffixes:\n '~debug': over\n '^mpich': ridden\n" }, { "path": "lib/spack/spack/test/data/modules/tcl/override_template.yaml", "content": "enable:\n - tcl\ntcl:\n all:\n autoload: none\n template: 'override_from_modules.txt'\n" }, { "path": "lib/spack/spack/test/data/modules/tcl/prerequisites_all.yaml", "content": "enable:\n - tcl\ntcl:\n all:\n autoload: none\n prerequisites: all\n" }, { "path": "lib/spack/spack/test/data/modules/tcl/prerequisites_direct.yaml", "content": "enable:\n - tcl\ntcl:\n all:\n autoload: none\n prerequisites: direct\n" }, { "path": "lib/spack/spack/test/data/modules/tcl/projections.yaml", "content": "enable:\n - tcl\ntcl:\n all:\n autoload: none\n projections:\n all: '{name}/{version}-{compiler.name}'\n mpileaks: '{name}-mpiprojection'\n" }, { "path": "lib/spack/spack/test/data/modules/tcl/suffix-format.yaml", "content": "enable:\n - tcl\ntcl:\n all:\n autoload: none\n mpileaks:\n suffixes:\n mpileaks: 'debug={variants.debug.value}'\n '^mpi': 'mpi={^mpi.name}-v{^mpi.version}'\n" }, { "path": "lib/spack/spack/test/data/modules/tcl/suffix.yaml", "content": "enable:\n - tcl\ntcl:\n all:\n autoload: none\n mpileaks:\n suffixes:\n '+opt': baz\n '+debug': foo\n '^mpich': foo\n '~debug': bar\n" }, { "path": "lib/spack/spack/test/data/modules/tcl/wrong_conflicts.yaml", "content": "enable:\n - tcl\ntcl:\n projections:\n all: '{name}/{version}-{compiler.name}'\n all:\n autoload: none\n conflict:\n - '{name}/{compiler.name}'\n" }, { "path": "lib/spack/spack/test/data/ninja/.gitignore", "content": ".ninja_deps\n.ninja_log\n" }, { "path": "lib/spack/spack/test/data/ninja/affirmative/check_test/build.ninja", "content": "# Tests that Spack detects target when it is the first of two targets\n\nrule cc\n command = true\n\nbuild check test: cc\n" }, { "path": "lib/spack/spack/test/data/ninja/affirmative/include/build.ninja", "content": "# Tests that Spack can handle targets in include files\n\ninclude include.ninja\n" }, { "path": "lib/spack/spack/test/data/ninja/affirmative/include/include.ninja", "content": "rule cc\n command = true\n\nbuild check: cc\n" }, { "path": "lib/spack/spack/test/data/ninja/affirmative/simple/build.ninja", "content": "# Tests that Spack can handle a simple Ninja build script\n\nrule cc\n command = true\n\nbuild check: cc\n" }, { "path": "lib/spack/spack/test/data/ninja/affirmative/spaces/build.ninja", "content": "# Tests that Spack allows spaces following the target name\n\nrule cc\n command = true\n\nbuild check : cc\n" }, { "path": "lib/spack/spack/test/data/ninja/affirmative/subninja/build.ninja", "content": "# Tests that Spack can handle targets in subninja files\n\nsubninja subninja.ninja\n" }, { "path": "lib/spack/spack/test/data/ninja/affirmative/subninja/subninja.ninja", "content": "rule cc\n command = true\n\nbuild check: cc\n" }, { "path": "lib/spack/spack/test/data/ninja/affirmative/test_check/build.ninja", "content": "# Tests that Spack detects target when it is the second of two targets\n\nrule cc\n command = true\n\nbuild test check: cc\n" }, { "path": "lib/spack/spack/test/data/ninja/affirmative/three_targets/build.ninja", "content": "# Tests that Spack detects a target if it is in the middle of a list\n\nrule cc\n command = true\n\nbuild foo check bar: cc\n" }, { "path": "lib/spack/spack/test/data/ninja/negative/no_ninja/readme.txt", "content": "# Tests that Spack ignores directories without a Ninja build script\n\ncflags = -Wall\n\nrule cc\n command = gcc $cflags -c $in -o $out\n\nbuild check: cc foo.c\n" }, { "path": "lib/spack/spack/test/data/ninja/negative/partial_match/build.ninja", "content": "# Tests that Spack ignores targets that contain a partial match\n\ncflags = -Wall\n\nrule cc\n command = gcc $cflags -c $in -o $out\n\nbuild installcheck: cc foo.c\n\nbuild checkinstall: cc foo.c\n\nbuild foo-check-bar: cc foo.c\n\nbuild foo_check_bar: cc foo.c\n\nbuild foo/check/bar: cc foo.c\n" }, { "path": "lib/spack/spack/test/data/ninja/negative/rule/build.ninja", "content": "# Tests that Spack ignores rule names\n\ncflags = -Wall\n\nrule check\n command = gcc $cflags -c $in -o $out\n\nbuild foo: check foo.c\n" }, { "path": "lib/spack/spack/test/data/ninja/negative/variable/build.ninja", "content": "# Tests that Spack ignores variable definitions\n\ncheck = -Wall\n\nrule cc\n command = gcc $check -c $in -o $out\n\nbuild foo: cc foo.c\n" }, { "path": "lib/spack/spack/test/data/patch/foo.patch", "content": "--- a/foo.txt\t2017-09-25 21:24:33.000000000 -0700\n+++ b/foo.txt\t2017-09-25 14:31:17.000000000 -0700\n@@ -1,2 +1,3 @@\n+zeroth line\n first line\n-second line\n+third line\n" }, { "path": "lib/spack/spack/test/data/sourceme_first.bat", "content": "@echo off\r\nrem C:\\lib\\spack\\spack\\test\\data\r\nrem\r\nrem Copyright 2013-2024 Lawrence Livermore National Security, LLC and other\r\nrem Spack Project Developers. See the top-level COPYRIGHT file for details.\r\nrem\r\nrem SPDX-License-Identifier: (Apache-2.0 OR MIT)\r\n\r\nset NEW_VAR=new\r\nset UNSET_ME=overridden\r\n" }, { "path": "lib/spack/spack/test/data/sourceme_first.sh", "content": "#!/usr/bin/env bash\n#\n# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\n\nexport NEW_VAR='new'\nexport UNSET_ME='overridden'\n" }, { "path": "lib/spack/spack/test/data/sourceme_lmod.sh", "content": "#!/usr/bin/env bash\n#\n# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nexport LMOD_VARIABLE=foo\nexport LMOD_ANOTHER_VARIABLE=bar\nexport NEW_VAR=new\n" }, { "path": "lib/spack/spack/test/data/sourceme_modules.bat", "content": "@echo off\r\nsetlocal\r\n\r\nrem C:\\lib\\spack\\spack\\test\\data\r\nrem\r\nrem Copyright 2013-2024 Lawrence Livermore National Security, LLC and other\r\nrem Spack Project Developers. See the top-level COPYRIGHT file for details.\r\nrem\r\nrem SPDX-License-Identifier: (Apache-2.0 OR MIT)\r\n\r\n:_module_raw val_1\r\n\r\nexit /b 0\r\n\r\n:module\r\nexit /b 0\r\n\r\n:ml\r\nexit /b 0\r\n\r\nset \"_module_raw=call :_module_raw\"\r\nset \"mod=call :mod\"\r\nset \"ml=call :ml\"\r\n\r\nset MODULES_AUTO_HANDLING=1\r\nset __MODULES_LMCONFLICT=bar^&foo\r\nset NEW_VAR=new\r\n" }, { "path": "lib/spack/spack/test/data/sourceme_modules.sh", "content": "#!/usr/bin/env bash\n#\n# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n_module_raw() { return 1; };\nmodule() { return 1; };\nml() { return 1; };\nexport -f _module_raw;\nexport -f module;\nexport -f ml;\n\nexport MODULES_AUTO_HANDLING=1\nexport __MODULES_LMCONFLICT=bar&foo\nexport NEW_VAR=new\n" }, { "path": "lib/spack/spack/test/data/sourceme_parameters.bat", "content": "@echo off\r\nrem \"C:\\lib\\spack\\spack\\test\\data\r\nrem\r\nrem Copyright 2013-2024 Lawrence Livermore National Security, LLC and other\r\nrem Spack Project Developers. See the top-level COPYRIGHT file for details.\r\nrem\r\nrem SPDX-License-Identifier: (Apache-2.0 OR MIT)\r\n\r\nif \"%1\" == \"intel64\" (\r\n set FOO=intel64\r\n) else (\r\n set FOO=default\r\n)\r\n" }, { "path": "lib/spack/spack/test/data/sourceme_parameters.sh", "content": "#!/usr/bin/env bash\n#\n# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\n\nif [[ \"$1\" == \"intel64\" ]] ; then\n export FOO='intel64'\nelse\n export FOO='default'\nfi\n" }, { "path": "lib/spack/spack/test/data/sourceme_second.bat", "content": "@echo off\r\nrem \"C:\\lib\\spack\\spack\\test\\data\r\nrem\r\nrem Copyright 2013-2024 Lawrence Livermore National Security, LLC and other\r\nrem Spack Project Developers. See the top-level COPYRIGHT file for details.\r\nrem\r\nrem SPDX-License-Identifier: (Apache-2.0 OR MIT)\r\n\r\nset PATH_LIST=C:\\path\\first;C:\\path\\second;C:\\path\\fourth\r\nset EMPTY_PATH_LIST=\r\n" }, { "path": "lib/spack/spack/test/data/sourceme_second.sh", "content": "#!/usr/bin/env bash\n#\n# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\n\nexport PATH_LIST='/path/first:/path/second:/path/fourth'\nunset EMPTY_PATH_LIST\n" }, { "path": "lib/spack/spack/test/data/sourceme_unicode.bat", "content": "@echo off\r\nrem \"C:\\lib\\spack\\spack\\test\\data\r\nrem\r\nrem Copyright 2013-2024 Lawrence Livermore National Security, LLC and other\r\nrem Spack Project Developers. See the top-level COPYRIGHT file for details.\r\nrem\r\nrem SPDX-License-Identifier: (Apache-2.0 OR MIT)\r\n\r\n\r\n\r\nrem Set an environment variable with some unicode in it to ensure that\r\nrem Spack can decode it.\r\nrem\r\nrem This has caused squashed commits on develop to break, as some\r\nrem committers use unicode in their messages, and Travis sets the\r\nrem current commit message in an environment variable.\r\nchcp 65001 > nul\r\nset UNICODE_VAR=don\\xe2\\x80\\x99t\r\nchcp 437 > nul\r\n" }, { "path": "lib/spack/spack/test/data/sourceme_unicode.sh", "content": "#!/usr/bin/env bash\n#\n# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\n\n# Set an environment variable with some unicode in it to ensure that\n# Spack can decode it.\n#\n# This has caused squashed commits on develop to break, as some\n# committers use unicode in their messages, and Travis sets the\n# current commit message in an environment variable.\nexport UNICODE_VAR='don\\xe2\\x80\\x99t'\n" }, { "path": "lib/spack/spack/test/data/sourceme_unset.bat", "content": "@echo off\r\nrem C:\\lib\\spack\\spack\\test\\data\r\nrem\r\nrem Copyright 2013-2024 Lawrence Livermore National Security, LLC and other\r\nrem Spack Project Developers. See the top-level COPYRIGHT file for details.\r\nrem\r\nrem SPDX-License-Identifier: (Apache-2.0 OR MIT)\r\n\r\nset UNSET_ME=\r\n" }, { "path": "lib/spack/spack/test/data/sourceme_unset.sh", "content": "#!/usr/bin/env bash\n#\n# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nunset UNSET_ME\n" }, { "path": "lib/spack/spack/test/data/style/broken.dummy", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport sys\nimport os\n\ndef this_is_a_function():\n \"\"\"This is a docstring.\"\"\"\n def this_should_be_offset():\n sys.stdout.write(os.name)\n" }, { "path": "lib/spack/spack/test/data/style/fixed.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport sys\n\n\ndef this_is_a_function():\n \"\"\"This is a docstring.\"\"\"\n\n def this_should_be_offset():\n sys.stdout.write(os.name)\n" }, { "path": "lib/spack/spack/test/data/templates/a.txt", "content": "Hello {{ word }}!\n" }, { "path": "lib/spack/spack/test/data/templates/extension.tcl", "content": "{% extends \"modules/modulefile.tcl\" %}\n{% block footer %}\nputs stderr \"{{ sentence }}\"\n{% endblock %}\n" }, { "path": "lib/spack/spack/test/data/templates/override.txt", "content": "Override successful!\n" }, { "path": "lib/spack/spack/test/data/templates_again/b.txt", "content": "Howdy {{ word }}!\n" }, { "path": "lib/spack/spack/test/data/templates_again/override_from_modules.txt", "content": "Override even better!\n" }, { "path": "lib/spack/spack/test/data/test/test_stage/gavrxt67t7yaiwfek7dds7lgokmoaiin/printing-package-1.0-hzgcoow-test-out.txt", "content": "==> Testing package printing-package-1.0-hzgcoow\n==> [2022-12-06-20:21:46.550943] test: test_print: Test python print example.\n==> [2022-12-06-20:21:46.553219] '/usr/tce/bin/python' '-c' 'print(\"Running test_print\")'\nRunning test_print\n==> [2022-12-06-20:21:46.721077] '/usr/tce/bin/python' '-c' 'print(\"Running test_print\")'\nPASSED: test_print\n==> [2022-12-06-20:21:46.822608] Completed testing\n" }, { "path": "lib/spack/spack/test/data/test/test_stage/gavrxt67t7yaiwfek7dds7lgokmoaiin/printing-package-1.0-hzgcoow-tested.txt", "content": "" }, { "path": "lib/spack/spack/test/data/test/test_stage/gavrxt67t7yaiwfek7dds7lgokmoaiin/results.txt", "content": "printing-package-1.0-hzgcoow PASSED\n" }, { "path": "lib/spack/spack/test/data/unparse/README.md", "content": "# Test data for unparser\n\nThese are test packages for testing Spack's unparser. They are used to ensure that the\ncanonical unparser used for Spack's package hash remains consistent across Python\nversions.\n\nAll of these were copied from mainline Spack packages, and they have been renamed with\n`.txt` suffixes so that they're not considered proper source files by the various\ncheckers used in Spack CI.\n\nThese packages were chosen for various reasons, but mainly because:\n\n1. They're some of the more complex packages in Spack, and they exercise more unparser\n features than other packages.\n\n2. Each of these packages has some interesting feature that was hard to unparse\n consistently across Python versions. See docstrings in packages for details.\n" }, { "path": "lib/spack/spack/test/data/unparse/amdfftw.txt", "content": "# -*- python -*-\n# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"This is an unparser test package.\n\n``amdfftw`` was chosen for its complexity and because it uses a negative array\nindex that was not being unparsed consistently from Python 2 to 3.\n\n\"\"\"\n\nimport os\n\nfrom spack import *\nfrom spack.pkg.builtin.fftw import FftwBase\n\n\nclass Amdfftw(FftwBase):\n \"\"\"FFTW (AMD Optimized version) is a comprehensive collection of\n fast C routines for computing the Discrete Fourier Transform (DFT)\n and various special cases thereof.\n\n It is an open-source implementation of the Fast Fourier transform\n algorithm. It can compute transforms of real and complex-values\n arrays of arbitrary size and dimension.\n AMD Optimized FFTW is the optimized FFTW implementation targeted\n for AMD CPUs.\n\n For single precision build, please use precision value as float.\n Example : spack install amdfftw precision=float\n \"\"\"\n\n _name = \"amdfftw\"\n homepage = \"https://developer.amd.com/amd-aocl/fftw/\"\n url = \"https://github.com/amd/amd-fftw/archive/3.0.tar.gz\"\n git = \"https://github.com/amd/amd-fftw.git\"\n\n maintainers(\"amd-toolchain-support\")\n\n version(\"3.1\", sha256=\"3e777f3acef13fa1910db097e818b1d0d03a6a36ef41186247c6ab1ab0afc132\")\n version(\"3.0.1\", sha256=\"87030c6bbb9c710f0a64f4f306ba6aa91dc4b182bb804c9022b35aef274d1a4c\")\n version(\"3.0\", sha256=\"a69deaf45478a59a69f77c4f7e9872967f1cfe996592dd12beb6318f18ea0bcd\")\n version(\"2.2\", sha256=\"de9d777236fb290c335860b458131678f75aa0799c641490c644c843f0e246f8\")\n\n variant(\"shared\", default=True, description=\"Builds a shared version of the library\")\n variant(\"openmp\", default=True, description=\"Enable OpenMP support\")\n variant(\"threads\", default=False, description=\"Enable SMP threads support\")\n variant(\"debug\", default=False, description=\"Builds a debug version of the library\")\n variant(\n \"amd-fast-planner\",\n default=False,\n description=\"Option to reduce the planning time without much\"\n \"tradeoff in the performance. It is supported for\"\n \"Float and double precisions only.\",\n )\n variant(\"amd-top-n-planner\", default=False, description=\"Build with amd-top-n-planner support\")\n variant(\n \"amd-mpi-vader-limit\", default=False, description=\"Build with amd-mpi-vader-limit support\"\n )\n variant(\"static\", default=False, description=\"Build with static suppport\")\n variant(\"amd-trans\", default=False, description=\"Build with amd-trans suppport\")\n variant(\"amd-app-opt\", default=False, description=\"Build with amd-app-opt suppport\")\n\n depends_on(\"texinfo\")\n\n provides(\"fftw-api@3\", when=\"@2:\")\n\n conflicts(\n \"precision=quad\",\n when=\"@2.2 %aocc\",\n msg=\"Quad precision is not supported by AOCC clang version 2.2\",\n )\n conflicts(\n \"+debug\", when=\"@2.2 %aocc\", msg=\"debug mode is not supported by AOCC clang version 2.2\"\n )\n conflicts(\"%gcc@:7.2\", when=\"@2.2:\", msg=\"GCC version above 7.2 is required for AMDFFTW\")\n conflicts(\n \"+amd-fast-planner \", when=\"+mpi\", msg=\"mpi thread is not supported with amd-fast-planner\"\n )\n conflicts(\n \"+amd-fast-planner\", when=\"@2.2\", msg=\"amd-fast-planner is supported from 3.0 onwards\"\n )\n conflicts(\n \"+amd-fast-planner\",\n when=\"precision=quad\",\n msg=\"Quad precision is not supported with amd-fast-planner\",\n )\n conflicts(\n \"+amd-fast-planner\",\n when=\"precision=long_double\",\n msg=\"long_double precision is not supported with amd-fast-planner\",\n )\n conflicts(\n \"+amd-top-n-planner\",\n when=\"@:3.0.0\",\n msg=\"amd-top-n-planner is supported from 3.0.1 onwards\",\n )\n conflicts(\n \"+amd-top-n-planner\",\n when=\"precision=long_double\",\n msg=\"long_double precision is not supported with amd-top-n-planner\",\n )\n conflicts(\n \"+amd-top-n-planner\",\n when=\"precision=quad\",\n msg=\"Quad precision is not supported with amd-top-n-planner\",\n )\n conflicts(\n \"+amd-top-n-planner\",\n when=\"+amd-fast-planner\",\n msg=\"amd-top-n-planner cannot be used with amd-fast-planner\",\n )\n conflicts(\n \"+amd-top-n-planner\", when=\"+threads\", msg=\"amd-top-n-planner works only for single thread\"\n )\n conflicts(\n \"+amd-top-n-planner\", when=\"+mpi\", msg=\"mpi thread is not supported with amd-top-n-planner\"\n )\n conflicts(\n \"+amd-top-n-planner\",\n when=\"+openmp\",\n msg=\"openmp thread is not supported with amd-top-n-planner\",\n )\n conflicts(\n \"+amd-mpi-vader-limit\",\n when=\"@:3.0.0\",\n msg=\"amd-mpi-vader-limit is supported from 3.0.1 onwards\",\n )\n conflicts(\n \"+amd-mpi-vader-limit\",\n when=\"precision=quad\",\n msg=\"Quad precision is not supported with amd-mpi-vader-limit\",\n )\n conflicts(\"+amd-trans\", when=\"+threads\", msg=\"amd-trans works only for single thread\")\n conflicts(\"+amd-trans\", when=\"+mpi\", msg=\"mpi thread is not supported with amd-trans\")\n conflicts(\"+amd-trans\", when=\"+openmp\", msg=\"openmp thread is not supported with amd-trans\")\n conflicts(\n \"+amd-trans\",\n when=\"precision=long_double\",\n msg=\"long_double precision is not supported with amd-trans\",\n )\n conflicts(\n \"+amd-trans\", when=\"precision=quad\", msg=\"Quad precision is not supported with amd-trans\"\n )\n conflicts(\"+amd-app-opt\", when=\"@:3.0.1\", msg=\"amd-app-opt is supported from 3.1 onwards\")\n conflicts(\"+amd-app-opt\", when=\"+mpi\", msg=\"mpi thread is not supported with amd-app-opt\")\n conflicts(\n \"+amd-app-opt\",\n when=\"precision=long_double\",\n msg=\"long_double precision is not supported with amd-app-opt\",\n )\n conflicts(\n \"+amd-app-opt\",\n when=\"precision=quad\",\n msg=\"Quad precision is not supported with amd-app-opt\",\n )\n\n def configure(self, spec, prefix):\n \"\"\"Configure function\"\"\"\n # Base options\n options = [\"--prefix={0}\".format(prefix), \"--enable-amd-opt\"]\n\n # Check if compiler is AOCC\n if \"%aocc\" in spec:\n options.append(\"CC={0}\".format(os.path.basename(spack_cc)))\n options.append(\"FC={0}\".format(os.path.basename(spack_fc)))\n options.append(\"F77={0}\".format(os.path.basename(spack_fc)))\n\n if \"+debug\" in spec:\n options.append(\"--enable-debug\")\n\n if \"+mpi\" in spec:\n options.append(\"--enable-mpi\")\n options.append(\"--enable-amd-mpifft\")\n else:\n options.append(\"--disable-mpi\")\n options.append(\"--disable-amd-mpifft\")\n\n options.extend(self.enable_or_disable(\"shared\"))\n options.extend(self.enable_or_disable(\"openmp\"))\n options.extend(self.enable_or_disable(\"threads\"))\n options.extend(self.enable_or_disable(\"amd-fast-planner\"))\n options.extend(self.enable_or_disable(\"amd-top-n-planner\"))\n options.extend(self.enable_or_disable(\"amd-mpi-vader-limit\"))\n options.extend(self.enable_or_disable(\"static\"))\n options.extend(self.enable_or_disable(\"amd-trans\"))\n options.extend(self.enable_or_disable(\"amd-app-opt\"))\n\n if not self.compiler.f77 or not self.compiler.fc:\n options.append(\"--disable-fortran\")\n\n # Cross compilation is supported in amd-fftw by making use of target\n # variable to set AMD_ARCH configure option.\n # Spack user can not directly use AMD_ARCH for this purpose but should\n # use target variable to set appropriate -march option in AMD_ARCH.\n arch = spec.architecture\n options.append(\n \"AMD_ARCH={0}\".format(arch.target.optimization_flags(spec.compiler).split(\"=\")[-1])\n )\n\n # Specific SIMD support.\n # float and double precisions are supported\n simd_features = [\"sse2\", \"avx\", \"avx2\"]\n\n simd_options = []\n for feature in simd_features:\n msg = \"--enable-{0}\" if feature in spec.target else \"--disable-{0}\"\n simd_options.append(msg.format(feature))\n\n # When enabling configure option \"--enable-amd-opt\", do not use the\n # configure option \"--enable-generic-simd128\" or\n # \"--enable-generic-simd256\"\n\n # Double is the default precision, for all the others we need\n # to enable the corresponding option.\n enable_precision = {\n \"float\": [\"--enable-float\"],\n \"double\": None,\n \"long_double\": [\"--enable-long-double\"],\n \"quad\": [\"--enable-quad-precision\"],\n }\n\n # Different precisions must be configured and compiled one at a time\n configure = Executable(\"../configure\")\n for precision in self.selected_precisions:\n opts = (enable_precision[precision] or []) + options[:]\n\n # SIMD optimizations are available only for float and double\n if precision in (\"float\", \"double\"):\n opts += simd_options\n\n with working_dir(precision, create=True):\n configure(*opts)\n" }, { "path": "lib/spack/spack/test/data/unparse/grads.txt", "content": "# -*- python -*-\n# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"This is an unparser test package.\n\n``grads`` was chosen because it has an embedded comment that looks like a docstring,\nwhich should be removed when doing canonical unparsing.\n\n\"\"\"\n\nfrom spack import *\n\n\nclass Grads(AutotoolsPackage):\n \"\"\"The Grid Analysis and Display System (GrADS) is an interactive\n desktop tool that is used for easy access, manipulation, and visualization\n of earth science data. GrADS has two data models for handling gridded and\n station data. GrADS supports many data file formats, including\n binary (stream or sequential), GRIB (version 1 and 2), NetCDF,\n HDF (version 4 and 5), and BUFR (for station data).\"\"\"\n\n homepage = \"http://cola.gmu.edu/grads/grads.php\"\n url = \"ftp://cola.gmu.edu/grads/2.2/grads-2.2.1-src.tar.gz\"\n\n version('2.2.1', sha256='695e2066d7d131720d598bac0beb61ac3ae5578240a5437401dc0ffbbe516206')\n\n variant('geotiff', default=True, description=\"Enable GeoTIFF support\")\n variant('shapefile', default=True, description=\"Enable Shapefile support\")\n\n \"\"\"\n # FIXME: Fails with undeclared functions (tdefi, tdef, ...) in gauser.c\n variant('hdf5', default=False, description=\"Enable HDF5 support\")\n variant('hdf4', default=False, description=\"Enable HDF4 support\")\n variant('netcdf', default=False, description=\"Enable NetCDF support\")\n depends_on('hdf5', when='+hdf5')\n depends_on('hdf', when='+hdf4')\n depends_on('netcdf-c', when='+netcdf')\n \"\"\"\n\n depends_on('libgeotiff', when='+geotiff')\n depends_on('shapelib', when='+shapefile')\n depends_on('udunits')\n depends_on('libgd')\n depends_on('libxmu')\n depends_on('cairo +X +pdf +fc +ft')\n depends_on('readline')\n depends_on('pkgconfig', type='build')\n\n def setup_build_environment(self, env: EnvironmentModifications) -> None:\n env.set('SUPPLIBS', '/')\n\n def setup_run_environment(self, env: EnvironmentModifications) -> None:\n env.set('GADDIR', self.prefix.data)\n\n @run_after('install')\n def copy_data(self):\n with working_dir(self.build_directory):\n install_tree('data', self.prefix.data)\n with working_dir(self.package_dir):\n install('udpt', self.prefix.data)\n filter_file(\n r'({lib})',\n self.prefix.lib,\n self.prefix.data.udpt\n )\n\n def configure_args(self):\n args = []\n args.extend(self.with_or_without('geotiff'))\n return args\n" }, { "path": "lib/spack/spack/test/data/unparse/legion.txt", "content": "# -*- python -*-\n# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"This is an unparser test package.\n\n``legion`` was chosen because it was very complex and because it was the only package in\nSpack that used a multi-argument print statement, which needs to be handled consistently\nacross python versions *despite* the fact that it produces different ASTs and different\nsemantics for Python 2 and 3.\n\n\"\"\"\n\nimport os\n\nfrom spack import *\n\n\nclass Legion(CMakePackage):\n \"\"\"Legion is a data-centric parallel programming system for writing\n portable high performance programs targeted at distributed heterogeneous\n architectures. Legion presents abstractions which allow programmers to\n describe properties of program data (e.g. independence, locality). By\n making the Legion programming system aware of the structure of program\n data, it can automate many of the tedious tasks programmers currently\n face, including correctly extracting task- and data-level parallelism\n and moving data around complex memory hierarchies. A novel mapping\n interface provides explicit programmer controlled placement of data in\n the memory hierarchy and assignment of tasks to processors in a way\n that is orthogonal to correctness, thereby enabling easy porting and\n tuning of Legion applications to new architectures.\"\"\"\n\n homepage = \"https://legion.stanford.edu/\"\n git = \"https://github.com/StanfordLegion/legion.git\"\n\n maintainers('pmccormick', 'streichler')\n tags = ['e4s']\n version('21.03.0', tag='legion-21.03.0')\n version('stable', branch='stable')\n version('master', branch='master')\n version('cr', branch='control_replication')\n\n depends_on(\"cmake@3.16:\", type='build')\n # TODO: Need to spec version of MPI v3 for use of the low-level MPI transport\n # layer. At present the MPI layer is still experimental and we discourge its\n # use for general (not legion development) use cases.\n depends_on('mpi', when='network=mpi')\n depends_on('mpi', when='network=gasnet') # MPI is required to build gasnet (needs mpicc).\n depends_on('ucx', when='conduit=ucx')\n depends_on('mpi', when='conduit=mpi')\n depends_on('cuda@10.0:11.9', when='+cuda_unsupported_compiler')\n depends_on('cuda@10.0:11.9', when='+cuda')\n depends_on('hdf5', when='+hdf5')\n depends_on('hwloc', when='+hwloc')\n\n # cuda-centric\n # reminder for arch numbers to names: 60=pascal, 70=volta, 75=turing, 80=ampere\n # TODO: we could use a map here to clean up and use naming vs. numbers.\n cuda_arch_list = ('60', '70', '75', '80')\n for nvarch in cuda_arch_list:\n depends_on('kokkos@3.3.01+cuda+cuda_lambda+wrapper cuda_arch={0}'.format(nvarch),\n when='%gcc+kokkos+cuda cuda_arch={0}'.format(nvarch))\n depends_on(\"kokkos@3.3.01+cuda+cuda_lambda~wrapper cuda_arch={0}\".format(nvarch),\n when=\"%clang+kokkos+cuda cuda_arch={0}\".format(nvarch))\n\n depends_on('kokkos@3.3.01~cuda', when='+kokkos~cuda')\n depends_on(\"kokkos@3.3.01~cuda+openmp\", when='+kokkos+openmp')\n\n depends_on('python@3', when='+python')\n depends_on('papi', when='+papi')\n depends_on('zlib', when='+zlib')\n\n # TODO: Need a AMD/HIP variant to match support landing in 21.03.0.\n\n # Network transport layer: the underlying data transport API should be used for\n # distributed data movement. For Legion, gasnet is the currently the most\n # mature. We have many users that default to using no network layer for\n # day-to-day development thus we default to 'none'. MPI support is new and\n # should be considered as a beta release.\n variant('network', default='none',\n values=('gasnet', 'mpi', 'none'),\n description=\"The network communications/transport layer to use.\",\n multi=False)\n\n # Add Gasnet tarball dependency in spack managed manner\n # TODO: Provide less mutable tag instead of branch\n resource(name='stanfordgasnet',\n git='https://github.com/StanfordLegion/gasnet.git',\n destination='stanfordgasnet',\n branch='master',\n when='network=gasnet')\n\n # We default to automatically embedding a gasnet build. To override this\n # point the package a pre-installed version of GASNet-Ex via the gasnet_root\n # variant.\n #\n # make sure we have a valid directory provided for gasnet_root...\n def validate_gasnet_root(value):\n if value == 'none':\n return True\n\n if not os.path.isdir(value):\n print((\"gasnet_root:\", value, \"-- no such directory.\"))\n print(\"gasnet_root:\", value, \"-- no such directory.\")\n return False\n else:\n return True\n\n variant('gasnet_root',\n default='none',\n values=validate_gasnet_root,\n description=\"Path to a pre-installed version of GASNet (prefix directory).\",\n multi=False)\n conflicts('gasnet_root', when=\"network=mpi\")\n\n variant('conduit', default='none',\n values=('aries', 'ibv', 'udp', 'mpi', 'ucx', 'none'),\n description=\"The gasnet conduit(s) to enable.\",\n multi=False)\n\n conflicts('conduit=none', when='network=gasnet',\n msg=\"a conduit must be selected when 'network=gasnet'\")\n\n gasnet_conduits = ('aries', 'ibv', 'udp', 'mpi', 'ucx')\n for c in gasnet_conduits:\n conflict_str = 'conduit=%s' % c\n conflicts(conflict_str, when='network=mpi',\n msg=\"conduit attribute requires 'network=gasnet'.\")\n conflicts(conflict_str, when='network=none',\n msg=\"conduit attribute requires 'network=gasnet'.\")\n\n variant('gasnet_debug', default=False,\n description=\"Build gasnet with debugging enabled.\")\n conflicts('+gasnet_debug', when='network=mpi')\n conflicts('+gasnet_debug', when='network=none')\n\n variant('shared', default=False,\n description=\"Build shared libraries.\")\n\n variant('bounds_checks', default=False,\n description=\"Enable bounds checking in Legion accessors.\")\n\n variant('privilege_checks', default=False,\n description=\"Enable runtime privildge checks in Legion accessors.\")\n\n variant('enable_tls', default=False,\n description=\"Enable thread-local-storage of the Legion context.\")\n\n variant('output_level', default='warning',\n # Note: these values are dependent upon those used in the cmake config.\n values=(\"spew\", \"debug\", \"info\", \"print\", \"warning\", \"error\", \"fatal\",\n \"none\"),\n description=\"Set the compile-time logging level.\",\n multi=False)\n\n variant('spy', default=False,\n description=\"Enable detailed logging for Legion Spy debugging.\")\n\n # note: we will be dependent upon spack's latest-and-greatest cuda version...\n variant('cuda', default=False,\n description=\"Enable CUDA support.\")\n variant('cuda_hijack', default=False,\n description=\"Hijack application calls into the CUDA runtime (+cuda).\")\n variant('cuda_arch', default='70',\n values=cuda_arch_list,\n description=\"GPU/CUDA architecture to build for.\",\n multi=False)\n variant('cuda_unsupported_compiler', default=False,\n description=\"Disable nvcc version check (--allow-unsupported-compiler).\")\n conflicts('+cuda_hijack', when='~cuda')\n\n variant('fortran', default=False,\n description=\"Enable Fortran bindings.\")\n\n variant('hdf5', default=False,\n description=\"Enable support for HDF5.\")\n\n variant('hwloc', default=False,\n description=\"Use hwloc for topology awareness.\")\n\n variant('kokkos', default=False,\n description=\"Enable support for interoperability with Kokkos.\")\n\n variant('bindings', default=False,\n description=\"Build runtime language bindings (excl. Fortran).\")\n\n variant('libdl', default=True,\n description=\"Enable support for dynamic object/library loading.\")\n\n variant('openmp', default=False,\n description=\"Enable support for OpenMP within Legion tasks.\")\n\n variant('papi', default=False,\n description=\"Enable PAPI performance measurements.\")\n\n variant('python', default=False,\n description=\"Enable Python support.\")\n\n variant('zlib', default=True,\n description=\"Enable zlib support.\")\n\n variant('redop_complex', default=False,\n description=\"Use reduction operators for complex types.\")\n\n variant('max_dims', values=int, default=3,\n description=\"Set max number of dimensions for logical regions.\")\n variant('max_fields', values=int, default=512,\n description=\"Maximum number of fields allowed in a logical region.\")\n\n def cmake_args(self):\n spec = self.spec\n cmake_cxx_flags = []\n options = []\n\n if 'network=gasnet' in spec:\n options.append('-DLegion_NETWORKS=gasnetex')\n if spec.variants['gasnet_root'].value != 'none':\n gasnet_dir = spec.variants['gasnet_root'].value\n options.append('-DGASNet_ROOT_DIR=%s' % gasnet_dir)\n else:\n gasnet_dir = join_path(self.stage.source_path,\n \"stanfordgasnet\",\n \"gasnet\")\n options.append('-DLegion_EMBED_GASNet=ON')\n options.append('-DLegion_EMBED_GASNet_LOCALSRC=%s' % gasnet_dir)\n\n gasnet_conduit = spec.variants['conduit'].value\n options.append('-DGASNet_CONDUIT=%s' % gasnet_conduit)\n\n if '+gasnet_debug' in spec:\n options.append('-DLegion_EMBED_GASNet_CONFIGURE_ARGS=--enable-debug')\n elif 'network=mpi' in spec:\n options.append('-DLegion_NETWORKS=mpi')\n if spec.variants['gasnet_root'].value != 'none':\n raise InstallError(\"'gasnet_root' is only valid when 'network=gasnet'.\")\n else:\n if spec.variants['gasnet_root'].value != 'none':\n raise InstallError(\"'gasnet_root' is only valid when 'network=gasnet'.\")\n options.append('-DLegion_EMBED_GASNet=OFF')\n\n if '+shared' in spec:\n options.append('-DBUILD_SHARED_LIBS=ON')\n else:\n options.append('-DBUILD_SHARED_LIBS=OFF')\n\n if '+bounds_checks' in spec:\n # default is off.\n options.append('-DLegion_BOUNDS_CHECKS=ON')\n if '+privilege_checks' in spec:\n # default is off.\n options.append('-DLegion_PRIVILEGE_CHECKS=ON')\n if '+enable_tls' in spec:\n # default is off.\n options.append('-DLegion_ENABLE_TLS=ON')\n if 'output_level' in spec:\n level = str.upper(spec.variants['output_level'].value)\n options.append('-DLegion_OUTPUT_LEVEL=%s' % level)\n if '+spy' in spec:\n # default is off.\n options.append('-DLegion_SPY=ON')\n\n if '+cuda' in spec:\n cuda_arch = spec.variants['cuda_arch'].value\n options.append('-DLegion_USE_CUDA=ON')\n options.append('-DLegion_GPU_REDUCTIONS=ON')\n options.append('-DLegion_CUDA_ARCH=%s' % cuda_arch)\n if '+cuda_hijack' in spec:\n options.append('-DLegion_HIJACK_CUDART=ON')\n else:\n options.append('-DLegion_HIJACK_CUDART=OFF')\n\n if '+cuda_unsupported_compiler' in spec:\n options.append('-DCUDA_NVCC_FLAGS:STRING=--allow-unsupported-compiler')\n\n if '+fortran' in spec:\n # default is off.\n options.append('-DLegion_USE_Fortran=ON')\n\n if '+hdf5' in spec:\n # default is off.\n options.append('-DLegion_USE_HDF5=ON')\n\n if '+hwloc' in spec:\n # default is off.\n options.append('-DLegion_USE_HWLOC=ON')\n\n if '+kokkos' in spec:\n # default is off.\n options.append('-DLegion_USE_Kokkos=ON')\n os.environ['KOKKOS_CXX_COMPILER'] = spec['kokkos'].kokkos_cxx\n\n if '+libdl' in spec:\n # default is on.\n options.append('-DLegion_USE_LIBDL=ON')\n else:\n options.append('-DLegion_USE_LIBDL=OFF')\n\n if '+openmp' in spec:\n # default is off.\n options.append('-DLegion_USE_OpenMP=ON')\n\n if '+papi' in spec:\n # default is off.\n options.append('-DLegion_USE_PAPI=ON')\n\n if '+python' in spec:\n # default is off.\n options.append('-DLegion_USE_Python=ON')\n\n if '+zlib' in spec:\n # default is on.\n options.append('-DLegion_USE_ZLIB=ON')\n else:\n options.append('-DLegion_USE_ZLIB=OFF')\n\n if '+redop_complex' in spec:\n # default is off.\n options.append('-DLegion_REDOP_COMPLEX=ON')\n\n if '+bindings' in spec:\n # default is off.\n options.append('-DLegion_BUILD_BINDINGS=ON')\n options.append('-DLegion_REDOP_COMPLEX=ON') # required for bindings\n options.append('-DLegion_USE_Fortran=ON')\n\n if spec.variants['build_type'].value == 'Debug':\n cmake_cxx_flags.extend([\n '-DDEBUG_REALM',\n '-DDEBUG_LEGION',\n '-ggdb',\n ])\n\n maxdims = int(spec.variants['max_dims'].value)\n # TODO: sanity check if maxdims < 0 || > 9???\n options.append('-DLegion_MAX_DIM=%d' % maxdims)\n\n maxfields = int(spec.variants['max_fields'].value)\n if (maxfields <= 0):\n maxfields = 512\n # make sure maxfields is a power of two. if not,\n # find the next largest power of two and use that...\n if (maxfields & (maxfields - 1) != 0):\n while maxfields & maxfields - 1:\n maxfields = maxfields & maxfields - 1\n maxfields = maxfields << 1\n options.append('-DLegion_MAX_FIELDS=%d' % maxfields)\n\n # This disables Legion's CMake build system's logic for targeting the native\n # CPU architecture in favor of Spack-provided compiler flags\n options.append('-DBUILD_MARCH:STRING=')\n return options\n\n @run_after('install')\n def cache_test_sources(self):\n \"\"\"Copy the example source files after the package is installed to an\n install test subdirectory for use during `spack test run`.\"\"\"\n cache_extra_test_sources(self, [join_path('examples', 'local_function_tasks')])\n\n def test_run_local_function_tasks(self):\n \"\"\"Build and run external application example\"\"\"\n\n test_dir = join_path(\n self.test_suite.current_test_cache_dir, \"examples\", \"local_function_tasks\"\n )\n\n if not os.path.exists(test_dir):\n raise SkipTest(f\"{test_dir} must exist\")\n\n cmake_args = [\n f\"-DCMAKE_C_COMPILER={self.compiler.cc}\",\n f\"-DCMAKE_CXX_COMPILER={self.compiler.cxx}\",\n f\"-DLegion_DIR={join_path(self.prefix, 'share', 'Legion', 'cmake')}\",\n ]\n\n with working_dir(test_dir):\n cmake = self.spec[\"cmake\"].command\n cmake(*cmake_args)\n\n make = which(\"make\")\n make()\n\n exe = which(\"local_function_tasks\")\n exe()\n" }, { "path": "lib/spack/spack/test/data/unparse/llvm.txt", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport os\nimport os.path\nimport re\nimport sys\n\nimport llnl.util.tty as tty\n\nimport spack.build_environment\nimport spack.util.executable\nfrom spack.package import *\n\n\nclass Llvm(CMakePackage, CudaPackage):\n \"\"\"The LLVM Project is a collection of modular and reusable compiler and\n toolchain technologies. Despite its name, LLVM has little to do\n with traditional virtual machines, though it does provide helpful\n libraries that can be used to build them. The name \"LLVM\" itself\n is not an acronym; it is the full name of the project.\n \"\"\"\n\n homepage = \"https://llvm.org/\"\n url = \"https://github.com/llvm/llvm-project/archive/llvmorg-7.1.0.tar.gz\"\n list_url = \"https://releases.llvm.org/download.html\"\n git = \"https://github.com/llvm/llvm-project\"\n maintainers(\"trws\", \"haampie\")\n\n tags = [\"e4s\"]\n\n generator = \"Ninja\"\n\n family = \"compiler\" # Used by lmod\n\n # fmt: off\n version('main', branch='main')\n version('14.0.6', sha256='98f15f842700bdb7220a166c8d2739a03a72e775b67031205078f39dd756a055')\n version('14.0.5', sha256='a4a57f029cb81f04618e05853f05fc2d21b64353c760977d8e7799bf7218a23a')\n version('14.0.4', sha256='1333236f9bee38658762076be4236cb5ebf15ae9b7f2bfce6946b96ae962dc73')\n version('14.0.3', sha256='0e1d049b050127ecf6286107e9a4400b0550f841d5d2288b9d31fd32ed0683d5')\n version('14.0.2', sha256='ca52232b3451c8e017f00eb882277707c13e30fac1271ec97015f6d0eeb383d1')\n version('14.0.1', sha256='c8be00406e872c8a24f8571cf6f5517b73ae707104724b1fd1db2f0af9544019')\n version('14.0.0', sha256='87b1a068b370df5b79a892fdb2935922a8efb1fddec4cc506e30fe57b6a1d9c4')\n version('13.0.1', sha256='09c50d558bd975c41157364421820228df66632802a4a6a7c9c17f86a7340802')\n version('13.0.0', sha256='a1131358f1f9f819df73fa6bff505f2c49d176e9eef0a3aedd1fdbce3b4630e8')\n version('12.0.1', sha256='66b64aa301244975a4aea489f402f205cde2f53dd722dad9e7b77a0459b4c8df')\n version('12.0.0', sha256='8e6c99e482bb16a450165176c2d881804976a2d770e0445af4375e78a1fbf19c')\n version('11.1.0', sha256='53a0719f3f4b0388013cfffd7b10c7d5682eece1929a9553c722348d1f866e79')\n version('11.0.1', sha256='9c7ad8e8ec77c5bde8eb4afa105a318fd1ded7dff3747d14f012758719d7171b')\n version('11.0.0', sha256='8ad4ddbafac4f2c8f2ea523c2c4196f940e8e16f9e635210537582a48622a5d5')\n version('10.0.1', sha256='c7ccb735c37b4ec470f66a6c35fbae4f029c0f88038f6977180b1a8ddc255637')\n version('10.0.0', sha256='b81c96d2f8f40dc61b14a167513d87c0d813aae0251e06e11ae8a4384ca15451')\n version('9.0.1', sha256='be7b034641a5fda51ffca7f5d840b1a768737779f75f7c4fd18fe2d37820289a')\n version('9.0.0', sha256='7807fac25330e24e9955ca46cd855dd34bbc9cc4fdba8322366206654d1036f2')\n version('8.0.1', sha256='5b18f6111c7aee7c0933c355877d4abcfe6cb40c1a64178f28821849c725c841')\n version('8.0.0', sha256='d81238b4a69e93e29f74ce56f8107cbfcf0c7d7b40510b7879e98cc031e25167')\n version('7.1.0', sha256='71c93979f20e01f1a1cc839a247945f556fa5e63abf2084e8468b238080fd839')\n version('7.0.1', sha256='f17a6cd401e8fd8f811fbfbb36dcb4f455f898c9d03af4044807ad005df9f3c0')\n version('6.0.1', sha256='aefadceb231f4c195fe6d6cd3b1a010b269c8a22410f339b5a089c2e902aa177')\n version('6.0.0', sha256='1946ec629c88d30122afa072d3c6a89cc5d5e4e2bb28dc63b2f9ebcc7917ee64')\n version('5.0.2', sha256='fe87aa11558c08856739bfd9bd971263a28657663cb0c3a0af01b94f03b0b795')\n version('5.0.1', sha256='84ca454abf262579814a2a2b846569f6e0cb3e16dc33ca3642b4f1dff6fbafd3')\n version('5.0.0', sha256='1f1843315657a4371d8ca37f01265fa9aae17dbcf46d2d0a95c1fdb3c6a4bab6')\n version('4.0.1', sha256='cd664fb3eec3208c08fb61189c00c9118c290b3be5adb3215a97b24255618be5')\n version('4.0.0', sha256='28ca4b2fc434cb1f558e8865386c233c2a6134437249b8b3765ae745ffa56a34')\n version('3.9.1', sha256='f5b6922a5c65f9232f83d89831191f2c3ccf4f41fdd8c63e6645bbf578c4ab92')\n version('3.9.0', sha256='9c6563a72c8b5b79941c773937d997dd2b1b5b3f640136d02719ec19f35e0333')\n version('3.8.1', sha256='69360f0648fde0dc3d3c4b339624613f3bc2a89c4858933bc3871a250ad02826')\n version('3.8.0', sha256='b5cc5974cc2fd4e9e49e1bbd0700f872501a8678bd9694fa2b36c65c026df1d1')\n version('3.7.1', sha256='d2cb0eb9b8eb21e07605bfe5e7a5c6c5f5f8c2efdac01ec1da6ffacaabe4195a')\n version('3.7.0', sha256='dc00bc230be2006fb87b84f6fe4800ca28bc98e6692811a98195da53c9cb28c6')\n version('3.6.2', sha256='f75d703a388ba01d607f9cf96180863a5e4a106827ade17b221d43e6db20778a')\n version('3.5.1', sha256='5d739684170d5b2b304e4fb521532d5c8281492f71e1a8568187bfa38eb5909d')\n # fmt: on\n\n # NOTE: The debug version of LLVM is an order of magnitude larger than\n # the release version, and may take up 20-30 GB of space. If you want\n # to save space, build with `build_type=Release`.\n\n variant(\n \"clang\", default=True, description=\"Build the LLVM C/C++/Objective-C compiler frontend\"\n )\n variant(\n \"flang\",\n default=False,\n when=\"@11: +clang\",\n description=\"Build the LLVM Fortran compiler frontend \"\n \"(experimental - parser only, needs GCC)\",\n )\n variant(\n \"omp_debug\",\n default=False,\n description=\"Include debugging code in OpenMP runtime libraries\",\n )\n variant(\"lldb\", default=True, when=\"+clang\", description=\"Build the LLVM debugger\")\n variant(\"lld\", default=True, description=\"Build the LLVM linker\")\n variant(\"mlir\", default=False, when=\"@10:\", description=\"Build with MLIR support\")\n variant(\n \"internal_unwind\", default=True, when=\"+clang\", description=\"Build the libcxxabi libunwind\"\n )\n variant(\n \"polly\",\n default=True,\n description=\"Build the LLVM polyhedral optimization plugin, \" \"only builds for 3.7.0+\",\n )\n variant(\n \"libcxx\", default=True, when=\"+clang\", description=\"Build the LLVM C++ standard library\"\n )\n variant(\n \"compiler-rt\",\n when=\"+clang\",\n default=True,\n description=\"Build LLVM compiler runtime, including sanitizers\",\n )\n variant(\n \"gold\",\n default=(sys.platform != \"darwin\"),\n description=\"Add support for LTO with the gold linker plugin\",\n )\n variant(\"split_dwarf\", default=False, description=\"Build with split dwarf information\")\n variant(\n \"llvm_dylib\",\n default=True,\n description=\"Build a combined LLVM shared library with all components\",\n )\n variant(\n \"link_llvm_dylib\",\n default=False,\n when=\"+llvm_dylib\",\n description=\"Link LLVM tools against the LLVM shared library\",\n )\n variant(\n \"targets\",\n default=\"none\",\n description=(\n \"What targets to build. Spack's target family is always added \"\n \"(e.g. X86 is automatically enabled when targeting znver2).\"\n ),\n values=(\n \"all\",\n \"none\",\n \"aarch64\",\n \"amdgpu\",\n \"arm\",\n \"avr\",\n \"bpf\",\n \"cppbackend\",\n \"hexagon\",\n \"lanai\",\n \"mips\",\n \"msp430\",\n \"nvptx\",\n \"powerpc\",\n \"riscv\",\n \"sparc\",\n \"systemz\",\n \"webassembly\",\n \"x86\",\n \"xcore\",\n ),\n multi=True,\n )\n variant(\n \"build_type\",\n default=\"Release\",\n description=\"CMake build type\",\n values=(\"Debug\", \"Release\", \"RelWithDebInfo\", \"MinSizeRel\"),\n )\n variant(\n \"omp_tsan\",\n default=False,\n when=\"@6:\",\n description=\"Build with OpenMP capable thread sanitizer\",\n )\n variant(\n \"omp_as_runtime\",\n default=True,\n when=\"+clang @12:\",\n description=\"Build OpenMP runtime via ENABLE_RUNTIME by just-built Clang\",\n )\n variant(\n \"code_signing\",\n default=False,\n when=\"+lldb platform=darwin\",\n description=\"Enable code-signing on macOS\",\n )\n variant(\"python\", default=False, description=\"Install python bindings\")\n variant(\"version_suffix\", default=\"none\", description=\"Add a symbol suffix\")\n variant(\n \"shlib_symbol_version\",\n default=\"none\",\n description=\"Add shared library symbol version\",\n when=\"@13:\",\n )\n variant(\n \"z3\", default=False, when=\"+clang @8:\", description=\"Use Z3 for the clang static analyzer\"\n )\n\n provides(\"libllvm@14\", when=\"@14.0.0:14\")\n provides(\"libllvm@13\", when=\"@13.0.0:13\")\n provides(\"libllvm@12\", when=\"@12.0.0:12\")\n provides(\"libllvm@11\", when=\"@11.0.0:11\")\n provides(\"libllvm@10\", when=\"@10.0.0:10\")\n provides(\"libllvm@9\", when=\"@9.0.0:9\")\n provides(\"libllvm@8\", when=\"@8.0.0:8\")\n provides(\"libllvm@7\", when=\"@7.0.0:7\")\n provides(\"libllvm@6\", when=\"@6.0.0:6\")\n provides(\"libllvm@5\", when=\"@5.0.0:5\")\n provides(\"libllvm@4\", when=\"@4.0.0:4\")\n provides(\"libllvm@3\", when=\"@3.0.0:3\")\n\n extends(\"python\", when=\"+python\")\n\n # Build dependency\n depends_on(\"cmake@3.4.3:\", type=\"build\")\n depends_on(\"cmake@3.13.4:\", type=\"build\", when=\"@12:\")\n depends_on(\"ninja\", type=\"build\")\n depends_on(\"python@2.7:2.8\", when=\"@:4 ~python\", type=\"build\")\n depends_on(\"python\", when=\"@5: ~python\", type=\"build\")\n depends_on(\"pkgconfig\", type=\"build\")\n\n # Universal dependency\n depends_on(\"python@2.7:2.8\", when=\"@:4+python\")\n depends_on(\"python\", when=\"@5:+python\")\n\n # clang and clang-tools dependencies\n depends_on(\"z3@4.7.1:\", when=\"+z3\")\n\n # openmp dependencies\n depends_on(\"perl-data-dumper\", type=(\"build\"))\n depends_on(\"hwloc\")\n depends_on(\"libelf\", when=\"+cuda\") # libomptarget\n depends_on(\"libffi\", when=\"+cuda\") # libomptarget\n\n # llvm-config --system-libs libraries.\n depends_on(\"zlib-api\")\n\n # lldb dependencies\n depends_on(\"swig\", when=\"+lldb\")\n depends_on(\"libedit\", when=\"+lldb\")\n depends_on(\"ncurses\", when=\"+lldb\")\n depends_on(\"py-six\", when=\"@5.0.0: +lldb +python\")\n\n # gold support, required for some features\n depends_on(\"binutils+gold+ld+plugins\", when=\"+gold\")\n\n # polly plugin\n depends_on(\"gmp\", when=\"@:3.6 +polly\")\n depends_on(\"isl\", when=\"@:3.6 +polly\")\n\n # Older LLVM do not build with newer compilers, and vice versa\n conflicts(\"%gcc@8:\", when=\"@:5\")\n conflicts(\"%gcc@:5.0\", when=\"@8:\")\n # clang/lib: a lambda parameter cannot shadow an explicitly captured entity\n conflicts(\"%clang@8:\", when=\"@:4\")\n # Internal compiler error on gcc 8.4 on aarch64 https://bugzilla.redhat.com/show_bug.cgi?id=1958295\n conflicts(\"%gcc@8.4:8.4.9\", when=\"@12: target=aarch64:\")\n\n # When these versions are concretized, but not explicitly with +libcxx, these\n # conflicts will enable clingo to set ~libcxx, making the build successful:\n\n # libc++ of LLVM13, see https://libcxx.llvm.org/#platform-and-compiler-support\n # @13 does not support %gcc@:10 https://bugs.llvm.org/show_bug.cgi?id=51359#c1\n # GCC 11 - latest stable release per GCC release page\n # Clang: 11, 12 - latest two stable releases per LLVM release page\n # AppleClang 12 - latest stable release per Xcode release page\n conflicts(\"%gcc@:10\", when=\"@13:+libcxx\")\n conflicts(\"%clang@:10\", when=\"@13:+libcxx\")\n conflicts(\"%apple-clang@:11\", when=\"@13:+libcxx\")\n\n # libcxx-4 and compiler-rt-4 fail to build with \"newer\" clang and gcc versions:\n conflicts(\"%gcc@7:\", when=\"@:4+libcxx\")\n conflicts(\"%clang@6:\", when=\"@:4+libcxx\")\n conflicts(\"%apple-clang@6:\", when=\"@:4+libcxx\")\n conflicts(\"%gcc@7:\", when=\"@:4+compiler-rt\")\n conflicts(\"%clang@6:\", when=\"@:4+compiler-rt\")\n conflicts(\"%apple-clang@6:\", when=\"@:4+compiler-rt\")\n\n # cuda_arch value must be specified\n conflicts(\"cuda_arch=none\", when=\"+cuda\", msg=\"A value for cuda_arch must be specified.\")\n\n # LLVM bug https://bugs.llvm.org/show_bug.cgi?id=48234\n # CMake bug: https://gitlab.kitware.com/cmake/cmake/-/issues/21469\n # Fixed in upstream versions of both\n conflicts(\"^cmake@3.19.0\", when=\"@6:11.0.0\")\n\n # Github issue #4986\n patch(\"llvm_gcc7.patch\", when=\"@4.0.0:4.0.1+lldb %gcc@7.0:\")\n\n # sys/ustat.h has been removed in favour of statfs from glibc-2.28. Use fixed sizes:\n patch(\"llvm5-sanitizer-ustat.patch\", when=\"@4:6.0.0+compiler-rt\")\n\n # Fix lld templates: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=230463\n patch(\"llvm4-lld-ELF-Symbols.patch\", when=\"@4+lld%clang@6:\")\n patch(\"llvm5-lld-ELF-Symbols.patch\", when=\"@5+lld%clang@7:\")\n\n # Fix missing std:size_t in 'llvm@4:5' when built with '%clang@7:'\n patch(\"xray_buffer_queue-cstddef.patch\", when=\"@4:5+compiler-rt%clang@7:\")\n\n # https://github.com/llvm/llvm-project/commit/947f9692440836dcb8d88b74b69dd379d85974ce\n patch(\"sanitizer-ipc_perm_mode.patch\", when=\"@5:7+compiler-rt%clang@11:\")\n patch(\"sanitizer-ipc_perm_mode.patch\", when=\"@5:9+compiler-rt%gcc@9:\")\n\n # github.com/spack/spack/issues/24270: MicrosoftDemangle for %gcc@10: and %clang@13:\n patch(\"missing-includes.patch\", when=\"@8\")\n\n # Backport from llvm master + additional fix\n # see https://bugs.llvm.org/show_bug.cgi?id=39696\n # for a bug report about this problem in llvm master.\n patch(\"constexpr_longdouble.patch\", when=\"@6:8+libcxx\")\n patch(\"constexpr_longdouble_9.0.patch\", when=\"@9:10.0.0+libcxx\")\n\n # Backport from llvm master; see\n # https://bugs.llvm.org/show_bug.cgi?id=38233\n # for a bug report about this problem in llvm master.\n patch(\"llvm_py37.patch\", when=\"@4:6 ^python@3.7:\")\n\n # https://bugs.llvm.org/show_bug.cgi?id=39696\n patch(\"thread-p9.patch\", when=\"@:10 +libcxx\")\n\n # https://github.com/spack/spack/issues/19625,\n # merged in llvm-11.0.0_rc2, but not found in 11.0.1\n patch(\"lldb_external_ncurses-10.patch\", when=\"@10.0.0:11.0.1+lldb\")\n\n # https://github.com/spack/spack/issues/19908\n # merged in llvm main prior to 12.0.0\n patch(\"llvm_python_path.patch\", when=\"@:11\")\n\n # Workaround for issue https://github.com/spack/spack/issues/18197\n patch(\"llvm7_intel.patch\", when=\"@7 %intel@18.0.2,19.0.0:19.1.99\")\n\n # Remove cyclades support to build against newer kernel headers\n # https://reviews.llvm.org/D102059\n patch(\"no_cyclades.patch\", when=\"@10:12.0.0\")\n patch(\"no_cyclades9.patch\", when=\"@6:9\")\n\n patch(\"llvm-gcc11.patch\", when=\"@9:11%gcc@11:\")\n\n # add -lpthread to build OpenMP libraries with Fujitsu compiler\n patch(\"llvm12-thread.patch\", when=\"@12 %fj\")\n patch(\"llvm13-thread.patch\", when=\"@13 %fj\")\n\n # avoid build failed with Fujitsu compiler\n patch(\"llvm13-fujitsu.patch\", when=\"@13 %fj\")\n\n # patch for missing hwloc.h include for libompd\n patch(\"llvm14-hwloc-ompd.patch\", when=\"@14\")\n\n # make libflags a list in openmp subproject when ~omp_as_runtime\n patch(\"libomp-libflags-as-list.patch\", when=\"@3.7:\")\n\n # The functions and attributes below implement external package\n # detection for LLVM. See:\n #\n # https://spack.readthedocs.io/en/latest/packaging_guide.html#making-a-package-discoverable-with-spack-external-find\n executables = [\"clang\", \"flang\", \"ld.lld\", \"lldb\"]\n\n @classmethod\n def filter_detected_exes(cls, prefix, exes_in_prefix):\n result = []\n for exe in exes_in_prefix:\n # Executables like lldb-vscode-X are daemon listening\n # on some port and would hang Spack during detection.\n # clang-cl and clang-cpp are dev tools that we don't\n # need to test\n if any(x in exe for x in (\"vscode\", \"cpp\", \"-cl\", \"-gpu\")):\n continue\n result.append(exe)\n return result\n\n @classmethod\n def determine_version(cls, exe):\n version_regex = re.compile(\n # Normal clang compiler versions are left as-is\n r\"clang version ([^ )\\n]+)-svn[~.\\w\\d-]*|\"\n # Don't include hyphenated patch numbers in the version\n # (see https://github.com/spack/spack/pull/14365 for details)\n r\"clang version ([^ )\\n]+?)-[~.\\w\\d-]*|\"\n r\"clang version ([^ )\\n]+)|\"\n # LLDB\n r\"lldb version ([^ )\\n]+)|\"\n # LLD\n r\"LLD ([^ )\\n]+) \\(compatible with GNU linkers\\)\"\n )\n try:\n compiler = Executable(exe)\n output = compiler(\"--version\", output=str, error=str)\n if \"Apple\" in output:\n return None\n match = version_regex.search(output)\n if match:\n return match.group(match.lastindex)\n except spack.util.executable.ProcessError:\n pass\n except Exception as e:\n tty.debug(e)\n\n return None\n\n @classmethod\n def determine_variants(cls, exes, version_str):\n variants, compilers = [\"+clang\"], {}\n lld_found, lldb_found = False, False\n for exe in exes:\n if \"clang++\" in exe:\n compilers[\"cxx\"] = exe\n elif \"clang\" in exe:\n compilers[\"c\"] = exe\n elif \"flang\" in exe:\n variants.append(\"+flang\")\n compilers[\"fc\"] = exe\n compilers[\"f77\"] = exe\n elif \"ld.lld\" in exe:\n lld_found = True\n compilers[\"ld\"] = exe\n elif \"lldb\" in exe:\n lldb_found = True\n compilers[\"lldb\"] = exe\n\n variants.append(\"+lld\" if lld_found else \"~lld\")\n variants.append(\"+lldb\" if lldb_found else \"~lldb\")\n\n return \"\".join(variants), {\"compilers\": compilers}\n\n @classmethod\n def validate_detected_spec(cls, spec, extra_attributes):\n # For LLVM 'compilers' is a mandatory attribute\n msg = 'the extra attribute \"compilers\" must be set for ' 'the detected spec \"{0}\"'.format(\n spec\n )\n assert \"compilers\" in extra_attributes, msg\n compilers = extra_attributes[\"compilers\"]\n for key in (\"c\", \"cxx\"):\n msg = \"{0} compiler not found for {1}\"\n assert key in compilers, msg.format(key, spec)\n\n @property\n def cc(self):\n msg = \"cannot retrieve C compiler [spec is not concrete]\"\n assert self.spec.concrete, msg\n if self.spec.external:\n return self.spec.extra_attributes[\"compilers\"].get(\"c\", None)\n result = None\n if \"+clang\" in self.spec:\n result = os.path.join(self.spec.prefix.bin, \"clang\")\n return result\n\n @property\n def cxx(self):\n msg = \"cannot retrieve C++ compiler [spec is not concrete]\"\n assert self.spec.concrete, msg\n if self.spec.external:\n return self.spec.extra_attributes[\"compilers\"].get(\"cxx\", None)\n result = None\n if \"+clang\" in self.spec:\n result = os.path.join(self.spec.prefix.bin, \"clang++\")\n return result\n\n @property\n def fc(self):\n msg = \"cannot retrieve Fortran compiler [spec is not concrete]\"\n assert self.spec.concrete, msg\n if self.spec.external:\n return self.spec.extra_attributes[\"compilers\"].get(\"fc\", None)\n result = None\n if \"+flang\" in self.spec:\n result = os.path.join(self.spec.prefix.bin, \"flang\")\n return result\n\n @property\n def f77(self):\n msg = \"cannot retrieve Fortran 77 compiler [spec is not concrete]\"\n assert self.spec.concrete, msg\n if self.spec.external:\n return self.spec.extra_attributes[\"compilers\"].get(\"f77\", None)\n result = None\n if \"+flang\" in self.spec:\n result = os.path.join(self.spec.prefix.bin, \"flang\")\n return result\n\n @property\n def libs(self):\n return LibraryList(self.llvm_config(\"--libfiles\", \"all\", result=\"list\"))\n\n @run_before(\"cmake\")\n def codesign_check(self):\n if self.spec.satisfies(\"+code_signing\"):\n codesign = which(\"codesign\")\n mkdir(\"tmp\")\n llvm_check_file = join_path(\"tmp\", \"llvm_check\")\n copy(\"/usr/bin/false\", llvm_check_file)\n try:\n codesign(\"-f\", \"-s\", \"lldb_codesign\", \"--dryrun\", llvm_check_file)\n\n except ProcessError:\n # Newer LLVM versions have a simple script that sets up\n # automatically when run with sudo priviliges\n setup = Executable(\"./lldb/scripts/macos-setup-codesign.sh\")\n try:\n setup()\n except Exception:\n raise RuntimeError(\n \"spack was unable to either find or set up\"\n \"code-signing on your system. Please refer to\"\n \"https://lldb.llvm.org/resources/build.html#\"\n \"code-signing-on-macos for details on how to\"\n \"create this identity.\"\n )\n\n def flag_handler(self, name, flags):\n if name == \"cxxflags\":\n flags.append(self.compiler.cxx11_flag)\n return (None, flags, None)\n elif name == \"ldflags\" and self.spec.satisfies(\"%intel\"):\n flags.append(\"-shared-intel\")\n return (None, flags, None)\n return (flags, None, None)\n\n def setup_build_environment(self, env: EnvironmentModifications) -> None:\n \"\"\"When using %clang, add only its ld.lld-$ver and/or ld.lld to our PATH\"\"\"\n if self.compiler.name in [\"clang\", \"apple-clang\"]:\n for lld in \"ld.lld-{0}\".format(self.compiler.version.version[0]), \"ld.lld\":\n bin = os.path.join(os.path.dirname(self.compiler.cc), lld)\n sym = os.path.join(self.stage.path, \"ld.lld\")\n if os.path.exists(bin) and not os.path.exists(sym):\n mkdirp(self.stage.path)\n os.symlink(bin, sym)\n env.prepend_path(\"PATH\", self.stage.path)\n\n def setup_run_environment(self, env: EnvironmentModifications) -> None:\n if \"+clang\" in self.spec:\n env.set(\"CC\", join_path(self.spec.prefix.bin, \"clang\"))\n env.set(\"CXX\", join_path(self.spec.prefix.bin, \"clang++\"))\n if \"+flang\" in self.spec:\n env.set(\"FC\", join_path(self.spec.prefix.bin, \"flang\"))\n env.set(\"F77\", join_path(self.spec.prefix.bin, \"flang\"))\n\n root_cmakelists_dir = \"llvm\"\n\n def cmake_args(self):\n spec = self.spec\n define = CMakePackage.define\n from_variant = self.define_from_variant\n\n python = spec[\"python\"]\n cmake_args = [\n define(\"LLVM_REQUIRES_RTTI\", True),\n define(\"LLVM_ENABLE_RTTI\", True),\n define(\"LLVM_ENABLE_EH\", True),\n define(\"LLVM_ENABLE_TERMINFO\", False),\n define(\"LLVM_ENABLE_LIBXML2\", False),\n define(\"CLANG_DEFAULT_OPENMP_RUNTIME\", \"libomp\"),\n define(\"PYTHON_EXECUTABLE\", python.command.path),\n define(\"LIBOMP_USE_HWLOC\", True),\n define(\"LIBOMP_HWLOC_INSTALL_DIR\", spec[\"hwloc\"].prefix),\n ]\n\n version_suffix = spec.variants[\"version_suffix\"].value\n if version_suffix != \"none\":\n cmake_args.append(define(\"LLVM_VERSION_SUFFIX\", version_suffix))\n\n shlib_symbol_version = spec.variants.get(\"shlib_symbol_version\", None)\n if shlib_symbol_version is not None and shlib_symbol_version.value != \"none\":\n cmake_args.append(define(\"LLVM_SHLIB_SYMBOL_VERSION\", shlib_symbol_version.value))\n\n if python.version >= Version(\"3\"):\n cmake_args.append(define(\"Python3_EXECUTABLE\", python.command.path))\n else:\n cmake_args.append(define(\"Python2_EXECUTABLE\", python.command.path))\n\n projects = []\n runtimes = []\n\n if \"+cuda\" in spec:\n cmake_args.extend(\n [\n define(\"CUDA_TOOLKIT_ROOT_DIR\", spec[\"cuda\"].prefix),\n define(\n \"LIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES\",\n \",\".join(spec.variants[\"cuda_arch\"].value),\n ),\n define(\n \"CLANG_OPENMP_NVPTX_DEFAULT_ARCH\",\n \"sm_{0}\".format(spec.variants[\"cuda_arch\"].value[-1]),\n ),\n ]\n )\n if \"+omp_as_runtime\" in spec:\n cmake_args.extend(\n [\n define(\"LIBOMPTARGET_NVPTX_ENABLE_BCLIB\", True),\n # work around bad libelf detection in libomptarget\n define(\n \"LIBOMPTARGET_DEP_LIBELF_INCLUDE_DIR\", spec[\"libelf\"].prefix.include\n ),\n ]\n )\n else:\n # still build libomptarget but disable cuda\n cmake_args.extend(\n [\n define(\"CUDA_TOOLKIT_ROOT_DIR\", \"IGNORE\"),\n define(\"CUDA_SDK_ROOT_DIR\", \"IGNORE\"),\n define(\"CUDA_NVCC_EXECUTABLE\", \"IGNORE\"),\n define(\"LIBOMPTARGET_DEP_CUDA_DRIVER_LIBRARIES\", \"IGNORE\"),\n ]\n )\n\n cmake_args.append(from_variant(\"LIBOMPTARGET_ENABLE_DEBUG\", \"omp_debug\"))\n\n if \"+lldb\" in spec:\n projects.append(\"lldb\")\n cmake_args.append(define(\"LLDB_ENABLE_LIBEDIT\", True))\n cmake_args.append(define(\"LLDB_ENABLE_NCURSES\", True))\n cmake_args.append(define(\"LLDB_ENABLE_LIBXML2\", False))\n if spec.version >= Version(\"10\"):\n cmake_args.append(from_variant(\"LLDB_ENABLE_PYTHON\", \"python\"))\n else:\n cmake_args.append(define(\"LLDB_DISABLE_PYTHON\", \"~python\" in spec))\n if spec.satisfies(\"@5.0.0: +python\"):\n cmake_args.append(define(\"LLDB_USE_SYSTEM_SIX\", True))\n\n if \"+gold\" in spec:\n cmake_args.append(define(\"LLVM_BINUTILS_INCDIR\", spec[\"binutils\"].prefix.include))\n\n if \"+clang\" in spec:\n projects.append(\"clang\")\n projects.append(\"clang-tools-extra\")\n if \"+omp_as_runtime\" in spec:\n runtimes.append(\"openmp\")\n else:\n projects.append(\"openmp\")\n\n if \"@8\" in spec:\n cmake_args.append(from_variant(\"CLANG_ANALYZER_ENABLE_Z3_SOLVER\", \"z3\"))\n elif \"@9:\" in spec:\n cmake_args.append(from_variant(\"LLVM_ENABLE_Z3_SOLVER\", \"z3\"))\n\n if \"+flang\" in spec:\n projects.append(\"flang\")\n if \"+lld\" in spec:\n projects.append(\"lld\")\n if \"+compiler-rt\" in spec:\n projects.append(\"compiler-rt\")\n if \"+libcxx\" in spec:\n projects.append(\"libcxx\")\n projects.append(\"libcxxabi\")\n if \"+mlir\" in spec:\n projects.append(\"mlir\")\n if \"+internal_unwind\" in spec:\n projects.append(\"libunwind\")\n if \"+polly\" in spec:\n projects.append(\"polly\")\n cmake_args.append(define(\"LINK_POLLY_INTO_TOOLS\", True))\n\n cmake_args.extend(\n [\n define(\"BUILD_SHARED_LIBS\", False),\n from_variant(\"LLVM_BUILD_LLVM_DYLIB\", \"llvm_dylib\"),\n from_variant(\"LLVM_LINK_LLVM_DYLIB\", \"link_llvm_dylib\"),\n from_variant(\"LLVM_USE_SPLIT_DWARF\", \"split_dwarf\"),\n # By default on Linux, libc++.so is a ldscript. CMake fails to add\n # CMAKE_INSTALL_RPATH to it, which fails. Statically link libc++abi.a\n # into libc++.so, linking with -lc++ or -stdlib=libc++ is enough.\n define(\"LIBCXX_ENABLE_STATIC_ABI_LIBRARY\", True),\n ]\n )\n\n cmake_args.append(define(\"LLVM_TARGETS_TO_BUILD\", get_llvm_targets_to_build(spec)))\n\n cmake_args.append(from_variant(\"LIBOMP_TSAN_SUPPORT\", \"omp_tsan\"))\n\n if self.compiler.name == \"gcc\":\n compiler = Executable(self.compiler.cc)\n gcc_output = compiler(\"-print-search-dirs\", output=str, error=str)\n\n for line in gcc_output.splitlines():\n if line.startswith(\"install:\"):\n # Get path and strip any whitespace\n # (causes oddity with ancestor)\n gcc_prefix = line.split(\":\")[1].strip()\n gcc_prefix = ancestor(gcc_prefix, 4)\n break\n cmake_args.append(define(\"GCC_INSTALL_PREFIX\", gcc_prefix))\n\n if self.spec.satisfies(\"~code_signing platform=darwin\"):\n cmake_args.append(define(\"LLDB_USE_SYSTEM_DEBUGSERVER\", True))\n\n # Semicolon seperated list of projects to enable\n cmake_args.append(define(\"LLVM_ENABLE_PROJECTS\", projects))\n\n # Semicolon seperated list of runtimes to enable\n if runtimes:\n cmake_args.append(define(\"LLVM_ENABLE_RUNTIMES\", runtimes))\n\n return cmake_args\n\n @run_after(\"install\")\n def post_install(self):\n spec = self.spec\n define = CMakePackage.define\n\n # unnecessary if we build openmp via LLVM_ENABLE_RUNTIMES\n if \"+cuda ~omp_as_runtime\" in self.spec:\n ompdir = \"build-bootstrapped-omp\"\n prefix_paths = spack.build_environment.get_cmake_prefix_path(self)\n prefix_paths.append(str(spec.prefix))\n # rebuild libomptarget to get bytecode runtime library files\n with working_dir(ompdir, create=True):\n cmake_args = [\n \"-G\",\n \"Ninja\",\n define(\"CMAKE_BUILD_TYPE\", spec.variants[\"build_type\"].value),\n define(\"CMAKE_C_COMPILER\", spec.prefix.bin + \"/clang\"),\n define(\"CMAKE_CXX_COMPILER\", spec.prefix.bin + \"/clang++\"),\n define(\"CMAKE_INSTALL_PREFIX\", spec.prefix),\n define(\"CMAKE_PREFIX_PATH\", prefix_paths),\n ]\n cmake_args.extend(self.cmake_args())\n cmake_args.extend(\n [\n define(\"LIBOMPTARGET_NVPTX_ENABLE_BCLIB\", True),\n define(\n \"LIBOMPTARGET_DEP_LIBELF_INCLUDE_DIR\", spec[\"libelf\"].prefix.include\n ),\n self.stage.source_path + \"/openmp\",\n ]\n )\n\n cmake(*cmake_args)\n ninja()\n ninja(\"install\")\n if \"+python\" in self.spec:\n install_tree(\"llvm/bindings/python\", python_platlib)\n\n if \"+clang\" in self.spec:\n install_tree(\"clang/bindings/python\", python_platlib)\n\n with working_dir(self.build_directory):\n install_tree(\"bin\", join_path(self.prefix, \"libexec\", \"llvm\"))\n\n def llvm_config(self, *args, **kwargs):\n lc = Executable(self.prefix.bin.join(\"llvm-config\"))\n if not kwargs.get(\"output\"):\n kwargs[\"output\"] = str\n ret = lc(*args, **kwargs)\n if kwargs.get(\"result\") == \"list\":\n return ret.split()\n else:\n return ret\n\n\ndef get_llvm_targets_to_build(spec):\n targets = spec.variants[\"targets\"].value\n\n # Build everything?\n if \"all\" in targets:\n return \"all\"\n\n # Convert targets variant values to CMake LLVM_TARGETS_TO_BUILD array.\n spack_to_cmake = {\n \"aarch64\": \"AArch64\",\n \"amdgpu\": \"AMDGPU\",\n \"arm\": \"ARM\",\n \"avr\": \"AVR\",\n \"bpf\": \"BPF\",\n \"cppbackend\": \"CppBackend\",\n \"hexagon\": \"Hexagon\",\n \"lanai\": \"Lanai\",\n \"mips\": \"Mips\",\n \"msp430\": \"MSP430\",\n \"nvptx\": \"NVPTX\",\n \"powerpc\": \"PowerPC\",\n \"riscv\": \"RISCV\",\n \"sparc\": \"Sparc\",\n \"systemz\": \"SystemZ\",\n \"webassembly\": \"WebAssembly\",\n \"x86\": \"X86\",\n \"xcore\": \"XCore\",\n }\n\n if \"none\" in targets:\n llvm_targets = set()\n else:\n llvm_targets = set(spack_to_cmake[target] for target in targets)\n\n if spec.target.family in (\"x86\", \"x86_64\"):\n llvm_targets.add(\"X86\")\n elif spec.target.family == \"arm\":\n llvm_targets.add(\"ARM\")\n elif spec.target.family == \"aarch64\":\n llvm_targets.add(\"AArch64\")\n elif spec.target.family in (\"sparc\", \"sparc64\"):\n llvm_targets.add(\"Sparc\")\n elif spec.target.family in (\"ppc64\", \"ppc64le\", \"ppc\", \"ppcle\"):\n llvm_targets.add(\"PowerPC\")\n\n return list(llvm_targets)\n" }, { "path": "lib/spack/spack/test/data/unparse/mfem.txt", "content": "# -*- python -*-\n# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"This is an unparser test package.\n\n``mfem`` was chosen because it's one of the most complex packages in Spack, because it\nuses ``@when`` functions, because it has ``configure()`` calls with star-args in\ndifferent locations, and beacuse it has a function with embedded unicode that needs to\nbe unparsed consistently between Python versions.\n\n\"\"\"\n\nimport os\nimport shutil\nimport sys\n\nfrom spack import *\n\n\nclass Mfem(Package, CudaPackage, ROCmPackage):\n \"\"\"Free, lightweight, scalable C++ library for finite element methods.\"\"\"\n\n tags = ['fem', 'finite-elements', 'high-order', 'amr', 'hpc', 'radiuss', 'e4s']\n\n homepage = 'http://www.mfem.org'\n git = 'https://github.com/mfem/mfem.git'\n\n maintainers('v-dobrev', 'tzanio', 'acfisher', 'goxberry', 'markcmiller86')\n\n test_requires_compiler = True\n\n # Recommended mfem builds to test when updating this file: see the shell\n # script 'test_builds.sh' in the same directory as this file.\n\n # mfem is downloaded from a URL shortener at request of upstream\n # author Tzanio Kolev . See here:\n # https://github.com/mfem/mfem/issues/53\n #\n # The following procedure should be used to verify security when a\n # new version is added:\n #\n # 1. Verify that no checksums on old versions have changed.\n #\n # 2. Verify that the shortened URL for the new version is listed at:\n # https://mfem.org/download/\n #\n # 3. Use http://getlinkinfo.com or similar to verify that the\n # underling download link for the latest version comes has the\n # prefix: http://mfem.github.io/releases\n #\n # If this quick verification procedure fails, additional discussion\n # will be required to verify the new version.\n\n # 'develop' is a special version that is always larger (or newer) than any\n # other version.\n version('develop', branch='master')\n\n version('4.3.0',\n sha256='3a495602121b986049286ea0b23512279cdbdfb43c15c42a1511b521051fbe38',\n url='https://bit.ly/mfem-4-3', extension='tar.gz')\n\n version('4.2.0',\n '4352a225b55948d2e73a5ee88cece0e88bdbe7ba6726a23d68b2736d3221a86d',\n url='https://bit.ly/mfem-4-2', extension='tar.gz')\n\n version('4.1.0',\n '4c83fdcf083f8e2f5b37200a755db843cdb858811e25a8486ad36b2cbec0e11d',\n url='https://bit.ly/mfem-4-1', extension='tar.gz')\n\n # Tagged development version used by xSDK\n version('4.0.1-xsdk', commit='c55c80d17b82d80de04b849dd526e17044f8c99a')\n\n version('4.0.0',\n 'df5bdac798ea84a263979f6fbf79de9013e1c55562f95f98644c3edcacfbc727',\n url='https://bit.ly/mfem-4-0', extension='tar.gz')\n\n # Tagged development version used by the laghos package:\n version('3.4.1-laghos-v2.0', tag='laghos-v2.0')\n\n version('3.4.0',\n sha256='4e73e4fe0482636de3c5dc983cd395839a83cb16f6f509bd88b053e8b3858e05',\n url='https://bit.ly/mfem-3-4', extension='tar.gz')\n\n version('3.3.2',\n sha256='b70fa3c5080b9ec514fc05f4a04ff74322b99ac4ecd6d99c229f0ed5188fc0ce',\n url='https://goo.gl/Kd7Jk8', extension='tar.gz')\n\n # Tagged development version used by the laghos package:\n version('3.3.1-laghos-v1.0', tag='laghos-v1.0')\n\n version('3.3',\n sha256='b17bd452593aada93dc0fee748fcfbbf4f04ce3e7d77fdd0341cc9103bcacd0b',\n url='http://goo.gl/Vrpsns', extension='tar.gz')\n\n version('3.2',\n sha256='2938c3deed4ec4f7fd5b5f5cfe656845282e86e2dcd477d292390058b7b94340',\n url='http://goo.gl/Y9T75B', extension='tar.gz')\n\n version('3.1',\n sha256='841ea5cf58de6fae4de0f553b0e01ebaab9cd9c67fa821e8a715666ecf18fc57',\n url='http://goo.gl/xrScXn', extension='tar.gz')\n\n variant('static', default=True,\n description='Build static library')\n variant('shared', default=False,\n description='Build shared library')\n variant('mpi', default=True,\n description='Enable MPI parallelism')\n # Can we make the default value for 'metis' to depend on the 'mpi' value?\n variant('metis', default=True,\n description='Enable METIS support')\n variant('openmp', default=False,\n description='Enable OpenMP parallelism')\n # Note: '+cuda' and 'cuda_arch' variants are added by the CudaPackage\n # Note: '+rocm' and 'amdgpu_target' variants are added by the ROCmPackage\n variant('occa', default=False, description='Enable OCCA backend')\n variant('raja', default=False, description='Enable RAJA backend')\n variant('libceed', default=False, description='Enable libCEED backend')\n variant('umpire', default=False, description='Enable Umpire support')\n variant('amgx', default=False, description='Enable NVIDIA AmgX solver support')\n\n variant('threadsafe', default=False,\n description=('Enable thread safe features.'\n ' Required for OpenMP.'\n ' May cause minor performance issues.'))\n variant('superlu-dist', default=False,\n description='Enable MPI parallel, sparse direct solvers')\n variant('strumpack', default=False,\n description='Enable support for STRUMPACK')\n variant('suite-sparse', default=False,\n description='Enable serial, sparse direct solvers')\n variant('petsc', default=False,\n description='Enable PETSc solvers, preconditioners, etc.')\n variant('slepc', default=False,\n description='Enable SLEPc integration')\n variant('sundials', default=False,\n description='Enable Sundials time integrators')\n variant('pumi', default=False,\n description='Enable functionality based on PUMI')\n variant('gslib', default=False,\n description='Enable functionality based on GSLIB')\n variant('mpfr', default=False,\n description='Enable precise, 1D quadrature rules')\n variant('lapack', default=False,\n description='Use external blas/lapack routines')\n variant('debug', default=False,\n description='Build debug instead of optimized version')\n variant('netcdf', default=False,\n description='Enable Cubit/Genesis reader')\n variant('conduit', default=False,\n description='Enable binary data I/O using Conduit')\n variant('zlib', default=True,\n description='Support zip\\'d streams for I/O')\n variant('gnutls', default=False,\n description='Enable secure sockets using GnuTLS')\n variant('libunwind', default=False,\n description='Enable backtrace on error support using Libunwind')\n # TODO: SIMD, Ginkgo, ADIOS2, HiOp, MKL CPardiso, Axom/Sidre\n variant('timer', default='auto',\n values=('auto', 'std', 'posix', 'mac', 'mpi'),\n description='Timing functions to use in mfem::StopWatch')\n variant('examples', default=False,\n description='Build and install examples')\n variant('miniapps', default=False,\n description='Build and install miniapps')\n\n conflicts('+shared', when='@:3.3.2')\n conflicts('~static~shared')\n conflicts('~threadsafe', when='@:3+openmp')\n\n conflicts('+cuda', when='@:3')\n conflicts('+rocm', when='@:4.1')\n conflicts('+cuda+rocm')\n conflicts('+netcdf', when='@:3.1')\n conflicts('+superlu-dist', when='@:3.1')\n # STRUMPACK support was added in mfem v3.3.2, however, here we allow only\n # strumpack v3+ support for which is available starting with mfem v4.0:\n conflicts('+strumpack', when='@:3')\n conflicts('+gnutls', when='@:3.1')\n conflicts('+zlib', when='@:3.2')\n conflicts('+mpfr', when='@:3.2')\n conflicts('+petsc', when='@:3.2')\n conflicts('+slepc', when='@:4.1')\n conflicts('+sundials', when='@:3.2')\n conflicts('+pumi', when='@:3.3.2')\n conflicts('+gslib', when='@:4.0')\n conflicts('timer=mac', when='@:3.3.0')\n conflicts('timer=mpi', when='@:3.3.0')\n conflicts('~metis+mpi', when='@:3.3.0')\n conflicts('+metis~mpi', when='@:3.3.0')\n conflicts('+conduit', when='@:3.3.2')\n conflicts('+occa', when='mfem@:3')\n conflicts('+raja', when='mfem@:3')\n conflicts('+libceed', when='mfem@:4.0')\n conflicts('+umpire', when='mfem@:4.0')\n conflicts('+amgx', when='mfem@:4.1')\n conflicts('+amgx', when='~cuda')\n conflicts('+mpi~cuda ^hypre+cuda')\n\n conflicts('+superlu-dist', when='~mpi')\n conflicts('+strumpack', when='~mpi')\n conflicts('+petsc', when='~mpi')\n conflicts('+slepc', when='~petsc')\n conflicts('+pumi', when='~mpi')\n conflicts('timer=mpi', when='~mpi')\n\n depends_on('mpi', when='+mpi')\n depends_on('hypre@2.10.0:2.13', when='@:3.3+mpi')\n depends_on('hypre@:2.20.0', when='@3.4:4.2+mpi')\n depends_on('hypre@:2.23.0', when='@4.3.0+mpi')\n depends_on('hypre', when='+mpi')\n\n depends_on('metis', when='+metis')\n depends_on('blas', when='+lapack')\n depends_on('lapack@3.0:', when='+lapack')\n\n depends_on('sundials@2.7.0', when='@:3.3.0+sundials~mpi')\n depends_on('sundials@2.7.0+mpi+hypre', when='@:3.3.0+sundials+mpi')\n depends_on('sundials@2.7.0:', when='@3.3.2:+sundials~mpi')\n depends_on('sundials@2.7.0:+mpi+hypre', when='@3.3.2:+sundials+mpi')\n depends_on('sundials@5.0.0:', when='@4.0.1-xsdk:+sundials~mpi')\n depends_on('sundials@5.0.0:+mpi+hypre', when='@4.0.1-xsdk:+sundials+mpi')\n for sm_ in CudaPackage.cuda_arch_values:\n depends_on('sundials@5.4.0:+cuda cuda_arch={0}'.format(sm_),\n when='@4.2.0:+sundials+cuda cuda_arch={0}'.format(sm_))\n depends_on('pumi@2.2.3:', when='@4.2.0:+pumi')\n depends_on('pumi', when='+pumi~shared')\n depends_on('pumi+shared', when='+pumi+shared')\n depends_on('gslib@1.0.5:+mpi', when='+gslib+mpi')\n depends_on('gslib@1.0.5:~mpi~mpiio', when='+gslib~mpi')\n depends_on('suite-sparse', when='+suite-sparse')\n depends_on('superlu-dist', when='+superlu-dist')\n depends_on('strumpack@3.0.0:', when='+strumpack~shared')\n depends_on('strumpack@3.0.0:+shared', when='+strumpack+shared')\n for sm_ in CudaPackage.cuda_arch_values:\n depends_on('strumpack+cuda cuda_arch={0}'.format(sm_),\n when='+strumpack+cuda cuda_arch={0}'.format(sm_))\n # The PETSc tests in MFEM will fail if PETSc is not configured with\n # SuiteSparse and MUMPS. On the other hand, if we require the variants\n # '+suite-sparse+mumps' of PETSc, the xsdk package concretization fails.\n depends_on('petsc@3.8:+mpi+double+hypre', when='+petsc')\n depends_on('slepc@3.8.0:', when='+slepc')\n # Recommended when building outside of xsdk:\n # depends_on('petsc@3.8:+mpi+double+hypre+suite-sparse+mumps',\n # when='+petsc')\n depends_on('mpfr', when='+mpfr')\n depends_on('netcdf-c@4.1.3:', when='+netcdf')\n depends_on('unwind', when='+libunwind')\n depends_on('zlib', when='+zlib')\n depends_on('gnutls', when='+gnutls')\n depends_on('conduit@0.3.1:,master:', when='+conduit')\n depends_on('conduit+mpi', when='+conduit+mpi')\n\n # The MFEM 4.0.0 SuperLU interface fails when using hypre@2.16.0 and\n # superlu-dist@6.1.1. See https://github.com/mfem/mfem/issues/983.\n # This issue was resolved in v4.1.\n conflicts('+superlu-dist',\n when='mfem@:4.0 ^hypre@2.16.0: ^superlu-dist@6:')\n # The STRUMPACK v3 interface in MFEM seems to be broken as of MFEM v4.1\n # when using hypre version >= 2.16.0.\n # This issue is resolved in v4.2.\n conflicts('+strumpack', when='mfem@4.0.0:4.1 ^hypre@2.16.0:')\n conflicts('+strumpack ^strumpack+cuda', when='~cuda')\n\n depends_on('occa@1.0.8:', when='@:4.1+occa')\n depends_on('occa@1.1.0:', when='@4.2.0:+occa')\n depends_on('occa+cuda', when='+occa+cuda')\n # TODO: propagate '+rocm' variant to occa when it is supported\n\n depends_on('raja@0.10.0:', when='@4.0.1:+raja')\n depends_on('raja@0.7.0:0.9.0', when='@4.0.0+raja')\n for sm_ in CudaPackage.cuda_arch_values:\n depends_on('raja+cuda cuda_arch={0}'.format(sm_),\n when='+raja+cuda cuda_arch={0}'.format(sm_))\n for gfx in ROCmPackage.amdgpu_targets:\n depends_on('raja+rocm amdgpu_target={0}'.format(gfx),\n when='+raja+rocm amdgpu_target={0}'.format(gfx))\n\n depends_on('libceed@0.6:', when='@:4.1+libceed')\n depends_on('libceed@0.7:', when='@4.2.0:+libceed')\n for sm_ in CudaPackage.cuda_arch_values:\n depends_on('libceed+cuda cuda_arch={0}'.format(sm_),\n when='+libceed+cuda cuda_arch={0}'.format(sm_))\n for gfx in ROCmPackage.amdgpu_targets:\n depends_on('libceed+rocm amdgpu_target={0}'.format(gfx),\n when='+libceed+rocm amdgpu_target={0}'.format(gfx))\n\n depends_on('umpire@2.0.0:', when='+umpire')\n for sm_ in CudaPackage.cuda_arch_values:\n depends_on('umpire+cuda cuda_arch={0}'.format(sm_),\n when='+umpire+cuda cuda_arch={0}'.format(sm_))\n for gfx in ROCmPackage.amdgpu_targets:\n depends_on('umpire+rocm amdgpu_target={0}'.format(gfx),\n when='+umpire+rocm amdgpu_target={0}'.format(gfx))\n\n # AmgX: propagate the cuda_arch and mpi settings:\n for sm_ in CudaPackage.cuda_arch_values:\n depends_on('amgx+mpi cuda_arch={0}'.format(sm_),\n when='+amgx+mpi cuda_arch={0}'.format(sm_))\n depends_on('amgx~mpi cuda_arch={0}'.format(sm_),\n when='+amgx~mpi cuda_arch={0}'.format(sm_))\n\n patch('mfem_ppc_build.patch', when='@3.2:3.3.0 arch=ppc64le')\n patch('mfem-3.4.patch', when='@3.4.0')\n patch('mfem-3.3-3.4-petsc-3.9.patch',\n when='@3.3.0:3.4.0 +petsc ^petsc@3.9.0:')\n patch('mfem-4.2-umpire.patch', when='@4.2.0+umpire')\n patch('mfem-4.2-slepc.patch', when='@4.2.0+slepc')\n patch('mfem-4.2-petsc-3.15.0.patch', when='@4.2.0+petsc ^petsc@3.15.0:')\n patch('mfem-4.3-hypre-2.23.0.patch', when='@4.3.0')\n patch('mfem-4.3-cusparse-11.4.patch', when='@4.3.0+cuda')\n\n # Patch to fix MFEM makefile syntax error. See\n # https://github.com/mfem/mfem/issues/1042 for the bug report and\n # https://github.com/mfem/mfem/pull/1043 for the bugfix contributed\n # upstream.\n patch('mfem-4.0.0-makefile-syntax-fix.patch', when='@4.0.0')\n phases = ['configure', 'build', 'install']\n\n def setup_build_environment(self, env: EnvironmentModifications) -> None:\n env.unset('MFEM_DIR')\n env.unset('MFEM_BUILD_DIR')\n\n #\n # Note: Although MFEM does support CMake configuration, MFEM\n # development team indicates that vanilla GNU Make is the\n # preferred mode of configuration of MFEM and the mode most\n # likely to be up to date in supporting *all* of MFEM's\n # configuration options. So, don't use CMake\n #\n def configure(self, spec, prefix):\n\n def yes_no(varstr):\n return 'YES' if varstr in self.spec else 'NO'\n\n # See also find_system_libraries in lib/spack/llnl/util/filesystem.py\n # where the same list of paths is used.\n sys_lib_paths = [\n '/lib64',\n '/lib',\n '/usr/lib64',\n '/usr/lib',\n '/usr/local/lib64',\n '/usr/local/lib']\n\n def is_sys_lib_path(dir):\n return dir in sys_lib_paths\n\n xcompiler = ''\n xlinker = '-Wl,'\n if '+cuda' in spec:\n xcompiler = '-Xcompiler='\n xlinker = '-Xlinker='\n cuda_arch = None if '~cuda' in spec else spec.variants['cuda_arch'].value\n\n # We need to add rpaths explicitly to allow proper export of link flags\n # from within MFEM.\n\n # Similar to spec[pkg].libs.ld_flags but prepends rpath flags too.\n # Also does not add system library paths as defined by 'sys_lib_paths'\n # above -- this is done to avoid issues like this:\n # https://github.com/mfem/mfem/issues/1088.\n def ld_flags_from_library_list(libs_list):\n flags = ['%s-rpath,%s' % (xlinker, dir)\n for dir in libs_list.directories\n if not is_sys_lib_path(dir)]\n flags += ['-L%s' % dir for dir in libs_list.directories\n if not is_sys_lib_path(dir)]\n flags += [libs_list.link_flags]\n return ' '.join(flags)\n\n def ld_flags_from_dirs(pkg_dirs_list, pkg_libs_list):\n flags = ['%s-rpath,%s' % (xlinker, dir) for dir in pkg_dirs_list\n if not is_sys_lib_path(dir)]\n flags += ['-L%s' % dir for dir in pkg_dirs_list\n if not is_sys_lib_path(dir)]\n flags += ['-l%s' % lib for lib in pkg_libs_list]\n return ' '.join(flags)\n\n def find_optional_library(name, prefix):\n for shared in [True, False]:\n for path in ['lib64', 'lib']:\n lib = find_libraries(name, join_path(prefix, path),\n shared=shared, recursive=False)\n if lib:\n return lib\n return LibraryList([])\n\n # Determine how to run MPI tests, e.g. when using '--test=root', when\n # Spack is run inside a batch system job.\n mfem_mpiexec = 'mpirun'\n mfem_mpiexec_np = '-np'\n if 'SLURM_JOBID' in os.environ:\n mfem_mpiexec = 'srun'\n mfem_mpiexec_np = '-n'\n elif 'LSB_JOBID' in os.environ:\n if 'LLNL_COMPUTE_NODES' in os.environ:\n mfem_mpiexec = 'lrun'\n mfem_mpiexec_np = '-n'\n else:\n mfem_mpiexec = 'jsrun'\n mfem_mpiexec_np = '-p'\n\n metis5_str = 'NO'\n if ('+metis' in spec) and spec['metis'].satisfies('@5:'):\n metis5_str = 'YES'\n\n zlib_var = 'MFEM_USE_ZLIB' if (spec.satisfies('@4.1.0:')) else \\\n 'MFEM_USE_GZSTREAM'\n\n options = [\n 'PREFIX=%s' % prefix,\n 'MFEM_USE_MEMALLOC=YES',\n 'MFEM_DEBUG=%s' % yes_no('+debug'),\n # NOTE: env['CXX'] is the spack c++ compiler wrapper. The real\n # compiler is defined by env['SPACK_CXX'].\n 'CXX=%s' % env['CXX'],\n 'MFEM_USE_LIBUNWIND=%s' % yes_no('+libunwind'),\n '%s=%s' % (zlib_var, yes_no('+zlib')),\n 'MFEM_USE_METIS=%s' % yes_no('+metis'),\n 'MFEM_USE_METIS_5=%s' % metis5_str,\n 'MFEM_THREAD_SAFE=%s' % yes_no('+threadsafe'),\n 'MFEM_USE_MPI=%s' % yes_no('+mpi'),\n 'MFEM_USE_LAPACK=%s' % yes_no('+lapack'),\n 'MFEM_USE_SUPERLU=%s' % yes_no('+superlu-dist'),\n 'MFEM_USE_STRUMPACK=%s' % yes_no('+strumpack'),\n 'MFEM_USE_SUITESPARSE=%s' % yes_no('+suite-sparse'),\n 'MFEM_USE_SUNDIALS=%s' % yes_no('+sundials'),\n 'MFEM_USE_PETSC=%s' % yes_no('+petsc'),\n 'MFEM_USE_SLEPC=%s' % yes_no('+slepc'),\n 'MFEM_USE_PUMI=%s' % yes_no('+pumi'),\n 'MFEM_USE_GSLIB=%s' % yes_no('+gslib'),\n 'MFEM_USE_NETCDF=%s' % yes_no('+netcdf'),\n 'MFEM_USE_MPFR=%s' % yes_no('+mpfr'),\n 'MFEM_USE_GNUTLS=%s' % yes_no('+gnutls'),\n 'MFEM_USE_OPENMP=%s' % yes_no('+openmp'),\n 'MFEM_USE_CONDUIT=%s' % yes_no('+conduit'),\n 'MFEM_USE_CUDA=%s' % yes_no('+cuda'),\n 'MFEM_USE_HIP=%s' % yes_no('+rocm'),\n 'MFEM_USE_OCCA=%s' % yes_no('+occa'),\n 'MFEM_USE_RAJA=%s' % yes_no('+raja'),\n 'MFEM_USE_AMGX=%s' % yes_no('+amgx'),\n 'MFEM_USE_CEED=%s' % yes_no('+libceed'),\n 'MFEM_USE_UMPIRE=%s' % yes_no('+umpire'),\n 'MFEM_MPIEXEC=%s' % mfem_mpiexec,\n 'MFEM_MPIEXEC_NP=%s' % mfem_mpiexec_np]\n\n cxxflags = spec.compiler_flags['cxxflags']\n\n if cxxflags:\n # Add opt/debug flags if they are not present in global cxx flags\n opt_flag_found = any(f in self.compiler.opt_flags\n for f in cxxflags)\n debug_flag_found = any(f in self.compiler.debug_flags\n for f in cxxflags)\n\n if '+debug' in spec:\n if not debug_flag_found:\n cxxflags.append('-g')\n if not opt_flag_found:\n cxxflags.append('-O0')\n else:\n if not opt_flag_found:\n cxxflags.append('-O2')\n\n cxxflags = [(xcompiler + flag) for flag in cxxflags]\n if '+cuda' in spec:\n cxxflags += [\n '-x=cu --expt-extended-lambda -arch=sm_%s' % cuda_arch,\n '-ccbin %s' % (spec['mpi'].mpicxx if '+mpi' in spec\n else env['CXX'])]\n if self.spec.satisfies('@4.0.0:'):\n cxxflags.append(self.compiler.cxx11_flag)\n # The cxxflags are set by the spack c++ compiler wrapper. We also\n # set CXXFLAGS explicitly, for clarity, and to properly export the\n # cxxflags in the variable MFEM_CXXFLAGS in config.mk.\n options += ['CXXFLAGS=%s' % ' '.join(cxxflags)]\n\n if '~static' in spec:\n options += ['STATIC=NO']\n if '+shared' in spec:\n options += [\n 'SHARED=YES',\n 'PICFLAG=%s' % (xcompiler + self.compiler.cxx_pic_flag)]\n\n if '+mpi' in spec:\n options += ['MPICXX=%s' % spec['mpi'].mpicxx]\n hypre = spec['hypre']\n # The hypre package always links with 'blas' and 'lapack'.\n all_hypre_libs = hypre.libs + hypre['lapack'].libs + \\\n hypre['blas'].libs\n options += [\n 'HYPRE_OPT=-I%s' % hypre.prefix.include,\n 'HYPRE_LIB=%s' % ld_flags_from_library_list(all_hypre_libs)]\n\n if '+metis' in spec:\n options += [\n 'METIS_OPT=-I%s' % spec['metis'].prefix.include,\n 'METIS_LIB=%s' %\n ld_flags_from_library_list(spec['metis'].libs)]\n\n if '+lapack' in spec:\n lapack_blas = spec['lapack'].libs + spec['blas'].libs\n options += [\n # LAPACK_OPT is not used\n 'LAPACK_LIB=%s' % ld_flags_from_library_list(lapack_blas)]\n\n if '+superlu-dist' in spec:\n lapack_blas = spec['lapack'].libs + spec['blas'].libs\n options += [\n 'SUPERLU_OPT=-I%s -I%s' %\n (spec['superlu-dist'].prefix.include,\n spec['parmetis'].prefix.include),\n 'SUPERLU_LIB=%s %s' %\n (ld_flags_from_dirs([spec['superlu-dist'].prefix.lib,\n spec['parmetis'].prefix.lib],\n ['superlu_dist', 'parmetis']),\n ld_flags_from_library_list(lapack_blas))]\n\n if '+strumpack' in spec:\n strumpack = spec['strumpack']\n sp_opt = ['-I%s' % strumpack.prefix.include]\n sp_lib = [ld_flags_from_library_list(strumpack.libs)]\n # Parts of STRUMPACK use fortran, so we need to link with the\n # fortran library and also the MPI fortran library:\n if '~shared' in strumpack:\n if os.path.basename(env['FC']) == 'gfortran':\n gfortran = Executable(env['FC'])\n libext = 'dylib' if sys.platform == 'darwin' else 'so'\n libfile = os.path.abspath(gfortran(\n '-print-file-name=libgfortran.%s' % libext,\n output=str).strip())\n gfortran_lib = LibraryList(libfile)\n sp_lib += [ld_flags_from_library_list(gfortran_lib)]\n if ('^mpich' in strumpack) or ('^mvapich2' in strumpack):\n sp_lib += ['-lmpifort']\n elif '^openmpi' in strumpack:\n sp_lib += ['-lmpi_mpifh']\n elif '^spectrum-mpi' in strumpack:\n sp_lib += ['-lmpi_ibm_mpifh']\n if '+openmp' in strumpack:\n # The '+openmp' in the spec means strumpack will TRY to find\n # OpenMP; if not found, we should not add any flags -- how do\n # we figure out if strumpack found OpenMP?\n if not self.spec.satisfies('%apple-clang'):\n sp_opt += [xcompiler + self.compiler.openmp_flag]\n if '^parmetis' in strumpack:\n parmetis = strumpack['parmetis']\n sp_opt += [parmetis.headers.cpp_flags]\n sp_lib += [ld_flags_from_library_list(parmetis.libs)]\n if '^netlib-scalapack' in strumpack:\n scalapack = strumpack['scalapack']\n sp_opt += ['-I%s' % scalapack.prefix.include]\n sp_lib += [ld_flags_from_dirs([scalapack.prefix.lib],\n ['scalapack'])]\n elif '^scalapack' in strumpack:\n scalapack = strumpack['scalapack']\n sp_opt += [scalapack.headers.cpp_flags]\n sp_lib += [ld_flags_from_library_list(scalapack.libs)]\n if '+butterflypack' in strumpack:\n bp = strumpack['butterflypack']\n sp_opt += ['-I%s' % bp.prefix.include]\n sp_lib += [ld_flags_from_dirs([bp.prefix.lib],\n ['dbutterflypack',\n 'zbutterflypack'])]\n if '+zfp' in strumpack:\n zfp = strumpack['zfp']\n sp_opt += ['-I%s' % zfp.prefix.include]\n sp_lib += [ld_flags_from_dirs([zfp.prefix.lib], ['zfp'])]\n if '+cuda' in strumpack:\n # assuming also ('+cuda' in spec)\n sp_lib += ['-lcusolver', '-lcublas']\n options += [\n 'STRUMPACK_OPT=%s' % ' '.join(sp_opt),\n 'STRUMPACK_LIB=%s' % ' '.join(sp_lib)]\n\n if '+suite-sparse' in spec:\n ss_spec = 'suite-sparse:' + self.suitesparse_components\n options += [\n 'SUITESPARSE_OPT=-I%s' % spec[ss_spec].prefix.include,\n 'SUITESPARSE_LIB=%s' %\n ld_flags_from_library_list(spec[ss_spec].libs)]\n\n if '+sundials' in spec:\n sun_spec = 'sundials:' + self.sundials_components\n options += [\n 'SUNDIALS_OPT=%s' % spec[sun_spec].headers.cpp_flags,\n 'SUNDIALS_LIB=%s' %\n ld_flags_from_library_list(spec[sun_spec].libs)]\n\n if '+petsc' in spec:\n petsc = spec['petsc']\n if '+shared' in petsc:\n options += [\n 'PETSC_OPT=%s' % petsc.headers.cpp_flags,\n 'PETSC_LIB=%s' % ld_flags_from_library_list(petsc.libs)]\n else:\n options += ['PETSC_DIR=%s' % petsc.prefix]\n\n if '+slepc' in spec:\n slepc = spec['slepc']\n options += [\n 'SLEPC_OPT=%s' % slepc.headers.cpp_flags,\n 'SLEPC_LIB=%s' % ld_flags_from_library_list(slepc.libs)]\n\n if '+pumi' in spec:\n pumi_libs = ['pumi', 'crv', 'ma', 'mds', 'apf', 'pcu', 'gmi',\n 'parma', 'lion', 'mth', 'apf_zoltan', 'spr']\n options += [\n 'PUMI_OPT=-I%s' % spec['pumi'].prefix.include,\n 'PUMI_LIB=%s' %\n ld_flags_from_dirs([spec['pumi'].prefix.lib], pumi_libs)]\n\n if '+gslib' in spec:\n options += [\n 'GSLIB_OPT=-I%s' % spec['gslib'].prefix.include,\n 'GSLIB_LIB=%s' %\n ld_flags_from_dirs([spec['gslib'].prefix.lib], ['gs'])]\n\n if '+netcdf' in spec:\n lib_flags = ld_flags_from_dirs([spec['netcdf-c'].prefix.lib],\n ['netcdf'])\n hdf5 = spec['hdf5:hl']\n if hdf5.satisfies('~shared'):\n hdf5_libs = hdf5.libs\n hdf5_libs += LibraryList(find_system_libraries('libdl'))\n lib_flags += \" \" + ld_flags_from_library_list(hdf5_libs)\n options += [\n 'NETCDF_OPT=-I%s' % spec['netcdf-c'].prefix.include,\n 'NETCDF_LIB=%s' % lib_flags]\n\n if '+zlib' in spec:\n if \"@:3.3.2\" in spec:\n options += ['ZLIB_DIR=%s' % spec['zlib'].prefix]\n else:\n options += [\n 'ZLIB_OPT=-I%s' % spec['zlib'].prefix.include,\n 'ZLIB_LIB=%s' %\n ld_flags_from_library_list(spec['zlib'].libs)]\n\n if '+mpfr' in spec:\n options += [\n 'MPFR_OPT=-I%s' % spec['mpfr'].prefix.include,\n 'MPFR_LIB=%s' %\n ld_flags_from_dirs([spec['mpfr'].prefix.lib], ['mpfr'])]\n\n if '+gnutls' in spec:\n options += [\n 'GNUTLS_OPT=-I%s' % spec['gnutls'].prefix.include,\n 'GNUTLS_LIB=%s' %\n ld_flags_from_dirs([spec['gnutls'].prefix.lib], ['gnutls'])]\n\n if '+libunwind' in spec:\n libunwind = spec['unwind']\n headers = find_headers('libunwind', libunwind.prefix.include)\n headers.add_macro('-g')\n libs = find_optional_library('libunwind', libunwind.prefix)\n # When mfem uses libunwind, it also needs 'libdl'.\n libs += LibraryList(find_system_libraries('libdl'))\n options += [\n 'LIBUNWIND_OPT=%s' % headers.cpp_flags,\n 'LIBUNWIND_LIB=%s' % ld_flags_from_library_list(libs)]\n\n if '+openmp' in spec:\n options += [\n 'OPENMP_OPT=%s' % (xcompiler + self.compiler.openmp_flag)]\n\n if '+cuda' in spec:\n options += [\n 'CUDA_CXX=%s' % join_path(spec['cuda'].prefix, 'bin', 'nvcc'),\n 'CUDA_ARCH=sm_%s' % cuda_arch]\n\n if '+rocm' in spec:\n amdgpu_target = ','.join(spec.variants['amdgpu_target'].value)\n options += [\n 'HIP_CXX=%s' % spec['hip'].hipcc,\n 'HIP_ARCH=%s' % amdgpu_target]\n\n if '+occa' in spec:\n options += ['OCCA_OPT=-I%s' % spec['occa'].prefix.include,\n 'OCCA_LIB=%s' %\n ld_flags_from_dirs([spec['occa'].prefix.lib],\n ['occa'])]\n\n if '+raja' in spec:\n options += ['RAJA_OPT=-I%s' % spec['raja'].prefix.include,\n 'RAJA_LIB=%s' %\n ld_flags_from_dirs([spec['raja'].prefix.lib],\n ['RAJA'])]\n\n if '+amgx' in spec:\n amgx = spec['amgx']\n if '+shared' in amgx:\n options += ['AMGX_OPT=-I%s' % amgx.prefix.include,\n 'AMGX_LIB=%s' %\n ld_flags_from_library_list(amgx.libs)]\n else:\n options += ['AMGX_DIR=%s' % amgx.prefix]\n\n if '+libceed' in spec:\n options += ['CEED_OPT=-I%s' % spec['libceed'].prefix.include,\n 'CEED_LIB=%s' %\n ld_flags_from_dirs([spec['libceed'].prefix.lib],\n ['ceed'])]\n\n if '+umpire' in spec:\n options += ['UMPIRE_OPT=-I%s' % spec['umpire'].prefix.include,\n 'UMPIRE_LIB=%s' %\n ld_flags_from_library_list(spec['umpire'].libs)]\n\n timer_ids = {'std': '0', 'posix': '2', 'mac': '4', 'mpi': '6'}\n timer = spec.variants['timer'].value\n if timer != 'auto':\n options += ['MFEM_TIMER_TYPE=%s' % timer_ids[timer]]\n\n if '+conduit' in spec:\n conduit = spec['conduit']\n headers = HeaderList(find(conduit.prefix.include, 'conduit.hpp',\n recursive=True))\n conduit_libs = ['libconduit', 'libconduit_relay',\n 'libconduit_blueprint']\n libs = find_libraries(conduit_libs, conduit.prefix.lib,\n shared=('+shared' in conduit))\n libs += LibraryList(find_system_libraries('libdl'))\n if '+hdf5' in conduit:\n hdf5 = conduit['hdf5']\n headers += find_headers('hdf5', hdf5.prefix.include)\n libs += hdf5.libs\n\n ##################\n # cyrush note:\n ##################\n # spack's HeaderList is applying too much magic, undermining us:\n #\n # It applies a regex to strip back to the last \"include\" dir\n # in the path. In our case we need to pass the following\n # as part of the CONDUIT_OPT flags:\n #\n # -I/include/conduit\n #\n # I tried several ways to present this path to the HeaderList,\n # but the regex always kills the trailing conduit dir\n # breaking build.\n #\n # To resolve the issue, we simply join our own string with\n # the headers results (which are important b/c they handle\n # hdf5 paths when enabled).\n ##################\n\n # construct proper include path\n conduit_include_path = conduit.prefix.include.conduit\n # add this path to the found flags\n conduit_opt_flags = \"-I{0} {1}\".format(conduit_include_path,\n headers.cpp_flags)\n\n options += [\n 'CONDUIT_OPT=%s' % conduit_opt_flags,\n 'CONDUIT_LIB=%s' % ld_flags_from_library_list(libs)]\n\n make('config', *options, parallel=False)\n make('info', parallel=False)\n\n def build(self, spec, prefix):\n make('lib')\n\n @run_after('build')\n def check_or_test(self):\n # Running 'make check' or 'make test' may fail if MFEM_MPIEXEC or\n # MFEM_MPIEXEC_NP are not set appropriately.\n if not self.run_tests:\n # check we can build ex1 (~mpi) or ex1p (+mpi).\n make('-C', 'examples', 'ex1p' if ('+mpi' in self.spec) else 'ex1',\n parallel=False)\n # make('check', parallel=False)\n else:\n make('all')\n make('test', parallel=False)\n\n def install(self, spec, prefix):\n make('install', parallel=False)\n\n # TODO: The way the examples and miniapps are being installed is not\n # perfect. For example, the makefiles do not work.\n\n install_em = ('+examples' in spec) or ('+miniapps' in spec)\n if install_em and ('+shared' in spec):\n make('examples/clean', 'miniapps/clean')\n # This is a hack to get the examples and miniapps to link with the\n # installed shared mfem library:\n with working_dir('config'):\n os.rename('config.mk', 'config.mk.orig')\n copy(str(self.config_mk), 'config.mk')\n shutil.copystat('config.mk.orig', 'config.mk')\n\n prefix_share = join_path(prefix, 'share', 'mfem')\n\n if '+examples' in spec:\n make('examples')\n install_tree('examples', join_path(prefix_share, 'examples'))\n\n if '+miniapps' in spec:\n make('miniapps')\n install_tree('miniapps', join_path(prefix_share, 'miniapps'))\n\n if install_em:\n install_tree('data', join_path(prefix_share, 'data'))\n\n examples_src_dir = 'examples'\n examples_data_dir = 'data'\n\n @run_after('install')\n def cache_test_sources(self):\n \"\"\"Copy the example source files after the package is installed to an\n install test subdirectory for use during `spack test run`.\"\"\"\n cache_extra_test_sources(self, [self.examples_src_dir, self.examples_data_dir])\n\n def test_ex10(self):\n \"\"\"build and run ex10(p)\"\"\"\n # MFEM has many examples to serve as a suitable smoke check. ex10\n # was chosen arbitrarily among the examples that work both with\n # MPI and without it\n test_dir = join_path(self.test_suite.current_test_cache_dir, self.examples_src_dir)\n\n mesh = join_path(\"..\", self.examples_data_dir, \"beam-quad.mesh\")\n test_exe = \"ex10p\" if (\"+mpi\" in self.spec) else \"ex10\"\n\n with working_dir(test_dir):\n make = which(\"make\")\n make(f\"CONFIG_MK={self.config_mk}\", test_exe, \"parallel=False\")\n\n ex10 = which(test_exe)\n ex10(\"--mesh\", mesh)\n\n # this patch is only needed for mfem 4.1, where a few\n # released files include byte order marks\n @when('@4.1.0')\n def patch(self):\n # Remove the byte order mark since it messes with some compilers\n files_with_bom = [\n 'fem/gslib.hpp', 'fem/gslib.cpp', 'linalg/hiop.hpp',\n 'miniapps/gslib/field-diff.cpp', 'miniapps/gslib/findpts.cpp',\n 'miniapps/gslib/pfindpts.cpp']\n bom = '\\xef\\xbb\\xbf' if sys.version_info < (3,) else u'\\ufeff'\n for f in files_with_bom:\n filter_file(bom, '', f)\n\n @property\n def suitesparse_components(self):\n \"\"\"Return the SuiteSparse components needed by MFEM.\"\"\"\n ss_comps = 'umfpack,cholmod,colamd,amd,camd,ccolamd,suitesparseconfig'\n if self.spec.satisfies('@3.2:'):\n ss_comps = 'klu,btf,' + ss_comps\n return ss_comps\n\n @property\n def sundials_components(self):\n \"\"\"Return the SUNDIALS components needed by MFEM.\"\"\"\n spec = self.spec\n sun_comps = 'arkode,cvodes,nvecserial,kinsol'\n if '+mpi' in spec:\n if spec.satisfies('@4.2:'):\n sun_comps += ',nvecparallel,nvecmpiplusx'\n else:\n sun_comps += ',nvecparhyp,nvecparallel'\n if '+cuda' in spec and '+cuda' in spec['sundials']:\n sun_comps += ',nveccuda'\n return sun_comps\n\n @property\n def headers(self):\n \"\"\"Export the main mfem header, mfem.hpp.\n \"\"\"\n hdrs = HeaderList(find(self.prefix.include, 'mfem.hpp',\n recursive=False))\n return hdrs or None\n\n @property\n def libs(self):\n \"\"\"Export the mfem library file.\n \"\"\"\n libs = find_libraries('libmfem', root=self.prefix.lib,\n shared=('+shared' in self.spec), recursive=False)\n return libs or None\n\n @property\n def config_mk(self):\n \"\"\"Export the location of the config.mk file.\n This property can be accessed using spec['mfem'].package.config_mk\n \"\"\"\n dirs = [self.prefix, self.prefix.share.mfem]\n for d in dirs:\n f = join_path(d, 'config.mk')\n if os.access(f, os.R_OK):\n return FileList(f)\n return FileList(find(self.prefix, 'config.mk', recursive=True))\n\n @property\n def test_mk(self):\n \"\"\"Export the location of the test.mk file.\n This property can be accessed using spec['mfem'].package.test_mk.\n In version 3.3.2 and newer, the location of test.mk is also defined\n inside config.mk, variable MFEM_TEST_MK.\n \"\"\"\n dirs = [self.prefix, self.prefix.share.mfem]\n for d in dirs:\n f = join_path(d, 'test.mk')\n if os.access(f, os.R_OK):\n return FileList(f)\n return FileList(find(self.prefix, 'test.mk', recursive=True))\n" }, { "path": "lib/spack/spack/test/data/unparse/py-torch.txt", "content": "# -*- python -*-\n# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"This is an unparser test package.\n\n``py-torch`` was chosen for its complexity and because it has an ``@when`` function\nthat can be removed statically, as well as several decorated @run_after functions\nthat should be preserved.\n\n\"\"\"\n\nimport os\nimport sys\n\nfrom spack import *\n\n\nclass PyTorch(PythonPackage, CudaPackage):\n \"\"\"Tensors and Dynamic neural networks in Python\n with strong GPU acceleration.\"\"\"\n\n homepage = \"https://pytorch.org/\"\n git = \"https://github.com/pytorch/pytorch.git\"\n\n # Exact set of modules is version- and variant-specific, just attempt to import the\n # core libraries to ensure that the package was successfully installed.\n import_modules = [\"torch\", \"torch.autograd\", \"torch.nn\", \"torch.utils\"]\n\n version(\"master\", branch=\"master\", submodules=True)\n version(\"1.10.1\", tag=\"v1.10.1\", submodules=True)\n version(\"1.10.0\", tag=\"v1.10.0\", submodules=True)\n version(\"1.9.1\", tag=\"v1.9.1\", submodules=True)\n version(\"1.9.0\", tag=\"v1.9.0\", submodules=True)\n version(\"1.8.2\", tag=\"v1.8.2\", submodules=True)\n version(\"1.8.1\", tag=\"v1.8.1\", submodules=True)\n version(\"1.8.0\", tag=\"v1.8.0\", submodules=True)\n version(\"1.7.1\", tag=\"v1.7.1\", submodules=True)\n version(\"1.7.0\", tag=\"v1.7.0\", submodules=True)\n version(\"1.6.0\", tag=\"v1.6.0\", submodules=True)\n version(\"1.5.1\", tag=\"v1.5.1\", submodules=True)\n version(\"1.5.0\", tag=\"v1.5.0\", submodules=True)\n version(\"1.4.1\", tag=\"v1.4.1\", submodules=True)\n version(\n \"1.4.0\",\n tag=\"v1.4.0\",\n submodules=True,\n deprecated=True,\n submodules_delete=[\"third_party/fbgemm\"],\n )\n version(\"1.3.1\", tag=\"v1.3.1\", submodules=True)\n version(\"1.3.0\", tag=\"v1.3.0\", submodules=True)\n version(\"1.2.0\", tag=\"v1.2.0\", submodules=True)\n version(\"1.1.0\", tag=\"v1.1.0\", submodules=True)\n version(\"1.0.1\", tag=\"v1.0.1\", submodules=True)\n version(\"1.0.0\", tag=\"v1.0.0\", submodules=True)\n version(\n \"0.4.1\",\n tag=\"v0.4.1\",\n submodules=True,\n deprecated=True,\n submodules_delete=[\"third_party/nervanagpu\"],\n )\n version(\"0.4.0\", tag=\"v0.4.0\", submodules=True, deprecated=True)\n version(\"0.3.1\", tag=\"v0.3.1\", submodules=True, deprecated=True)\n\n is_darwin = sys.platform == \"darwin\"\n\n # All options are defined in CMakeLists.txt.\n # Some are listed in setup.py, but not all.\n variant(\"caffe2\", default=True, description=\"Build Caffe2\")\n variant(\"test\", default=False, description=\"Build C++ test binaries\")\n variant(\"cuda\", default=not is_darwin, description=\"Use CUDA\")\n variant(\"rocm\", default=False, description=\"Use ROCm\")\n variant(\"cudnn\", default=not is_darwin, description=\"Use cuDNN\")\n variant(\"fbgemm\", default=True, description=\"Use FBGEMM (quantized 8-bit server operators)\")\n variant(\"kineto\", default=True, description=\"Use Kineto profiling library\")\n variant(\"magma\", default=not is_darwin, description=\"Use MAGMA\")\n variant(\"metal\", default=is_darwin, description=\"Use Metal for Caffe2 iOS build\")\n variant(\"nccl\", default=not is_darwin, description=\"Use NCCL\")\n variant(\"nnpack\", default=True, description=\"Use NNPACK\")\n variant(\"numa\", default=not is_darwin, description=\"Use NUMA\")\n variant(\"numpy\", default=True, description=\"Use NumPy\")\n variant(\"openmp\", default=True, description=\"Use OpenMP for parallel code\")\n variant(\"qnnpack\", default=True, description=\"Use QNNPACK (quantized 8-bit operators)\")\n variant(\"valgrind\", default=not is_darwin, description=\"Use Valgrind\")\n variant(\"xnnpack\", default=True, description=\"Use XNNPACK\")\n variant(\"mkldnn\", default=True, description=\"Use MKLDNN\")\n variant(\"distributed\", default=not is_darwin, description=\"Use distributed\")\n variant(\"mpi\", default=not is_darwin, description=\"Use MPI for Caffe2\")\n variant(\"gloo\", default=not is_darwin, description=\"Use Gloo\")\n variant(\"tensorpipe\", default=not is_darwin, description=\"Use TensorPipe\")\n variant(\"onnx_ml\", default=True, description=\"Enable traditional ONNX ML API\")\n variant(\"breakpad\", default=True, description=\"Enable breakpad crash dump library\")\n\n conflicts(\"+cuda\", when=\"+rocm\")\n conflicts(\"+cudnn\", when=\"~cuda\")\n conflicts(\"+magma\", when=\"~cuda\")\n conflicts(\"+nccl\", when=\"~cuda~rocm\")\n conflicts(\"+nccl\", when=\"platform=darwin\")\n conflicts(\"+numa\", when=\"platform=darwin\", msg=\"Only available on Linux\")\n conflicts(\"+valgrind\", when=\"platform=darwin\", msg=\"Only available on Linux\")\n conflicts(\"+mpi\", when=\"~distributed\")\n conflicts(\"+gloo\", when=\"~distributed\")\n conflicts(\"+tensorpipe\", when=\"~distributed\")\n conflicts(\"+kineto\", when=\"@:1.7\")\n conflicts(\"+valgrind\", when=\"@:1.7\")\n conflicts(\"~caffe2\", when=\"@0.4.0:1.6\") # no way to disable caffe2?\n conflicts(\"+caffe2\", when=\"@:0.3.1\") # caffe2 did not yet exist?\n conflicts(\"+tensorpipe\", when=\"@:1.5\")\n conflicts(\"+xnnpack\", when=\"@:1.4\")\n conflicts(\"~onnx_ml\", when=\"@:1.4\") # no way to disable ONNX?\n conflicts(\"+rocm\", when=\"@:0.4\")\n conflicts(\"+cudnn\", when=\"@:0.4\")\n conflicts(\"+fbgemm\", when=\"@:0.4,1.4.0\")\n conflicts(\"+qnnpack\", when=\"@:0.4\")\n conflicts(\"+mkldnn\", when=\"@:0.4\")\n conflicts(\"+breakpad\", when=\"@:1.9\") # Option appeared in 1.10.0\n conflicts(\"+breakpad\", when=\"target=ppc64:\", msg=\"Unsupported\")\n conflicts(\"+breakpad\", when=\"target=ppc64le:\", msg=\"Unsupported\")\n\n conflicts(\n \"cuda_arch=none\",\n when=\"+cuda\",\n msg=\"Must specify CUDA compute capabilities of your GPU, see \"\n \"https://developer.nvidia.com/cuda-gpus\",\n )\n\n # Required dependencies\n depends_on(\"cmake@3.5:\", type=\"build\")\n # Use Ninja generator to speed up build times, automatically used if found\n depends_on(\"ninja@1.5:\", when=\"@1.1.0:\", type=\"build\")\n # See python_min_version in setup.py\n depends_on(\"python@3.6.2:\", when=\"@1.7.1:\", type=(\"build\", \"link\", \"run\"))\n depends_on(\"python@3.6.1:\", when=\"@1.6.0:1.7.0\", type=(\"build\", \"link\", \"run\"))\n depends_on(\"python@3.5:\", when=\"@1.5.0:1.5\", type=(\"build\", \"link\", \"run\"))\n depends_on(\"python@2.7:2.8,3.5:\", when=\"@1.4.0:1.4\", type=(\"build\", \"link\", \"run\"))\n depends_on(\"python@2.7:2.8,3.5:3.7\", when=\"@:1.3\", type=(\"build\", \"link\", \"run\"))\n depends_on(\"py-setuptools\", type=(\"build\", \"run\"))\n depends_on(\"py-future\", when=\"@1.5:\", type=(\"build\", \"run\"))\n depends_on(\"py-future\", when=\"@1.1: ^python@:2\", type=(\"build\", \"run\"))\n depends_on(\"py-pyyaml\", type=(\"build\", \"run\"))\n depends_on(\"py-typing\", when=\"@0.4: ^python@:3.4\", type=(\"build\", \"run\"))\n depends_on(\"py-typing-extensions\", when=\"@1.7:\", type=(\"build\", \"run\"))\n depends_on(\"py-pybind11@2.6.2\", when=\"@1.8.0:\", type=(\"build\", \"link\", \"run\"))\n depends_on(\"py-pybind11@2.3.0\", when=\"@1.1.0:1.7\", type=(\"build\", \"link\", \"run\"))\n depends_on(\"py-pybind11@2.2.4\", when=\"@1.0.0:1.0\", type=(\"build\", \"link\", \"run\"))\n depends_on(\"py-pybind11@2.2.2\", when=\"@0.4.0:0.4\", type=(\"build\", \"link\", \"run\"))\n depends_on(\"py-dataclasses\", when=\"@1.7: ^python@3.6.0:3.6\", type=(\"build\", \"run\"))\n depends_on(\"py-tqdm\", type=\"run\")\n depends_on(\"py-protobuf\", when=\"@0.4:\", type=(\"build\", \"run\"))\n depends_on(\"protobuf\", when=\"@0.4:\")\n depends_on(\"blas\")\n depends_on(\"lapack\")\n depends_on(\"eigen\", when=\"@0.4:\")\n # https://github.com/pytorch/pytorch/issues/60329\n # depends_on('cpuinfo@2020-12-17', when='@1.8.0:')\n # depends_on('cpuinfo@2020-06-11', when='@1.6.0:1.7')\n # https://github.com/shibatch/sleef/issues/427\n # depends_on('sleef@3.5.1_2020-12-22', when='@1.8.0:')\n # https://github.com/pytorch/pytorch/issues/60334\n # depends_on('sleef@3.4.0_2019-07-30', when='@1.6.0:1.7')\n # https://github.com/Maratyszcza/FP16/issues/18\n # depends_on('fp16@2020-05-14', when='@1.6.0:')\n depends_on(\"pthreadpool@2021-04-13\", when=\"@1.9.0:\")\n depends_on(\"pthreadpool@2020-10-05\", when=\"@1.8.0:1.8\")\n depends_on(\"pthreadpool@2020-06-15\", when=\"@1.6.0:1.7\")\n depends_on(\"psimd@2020-05-17\", when=\"@1.6.0:\")\n depends_on(\"fxdiv@2020-04-17\", when=\"@1.6.0:\")\n depends_on(\"benchmark\", when=\"@1.6:+test\")\n\n # Optional dependencies\n depends_on(\"cuda@7.5:\", when=\"+cuda\", type=(\"build\", \"link\", \"run\"))\n depends_on(\"cuda@9:\", when=\"@1.1:+cuda\", type=(\"build\", \"link\", \"run\"))\n depends_on(\"cuda@9.2:\", when=\"@1.6:+cuda\", type=(\"build\", \"link\", \"run\"))\n depends_on(\"cudnn@6.0:7\", when=\"@:1.0+cudnn\")\n depends_on(\"cudnn@7.0:7\", when=\"@1.1.0:1.5+cudnn\")\n depends_on(\"cudnn@7.0:\", when=\"@1.6.0:+cudnn\")\n depends_on(\"magma\", when=\"+magma\")\n depends_on(\"nccl\", when=\"+nccl\")\n depends_on(\"numactl\", when=\"+numa\")\n depends_on(\"py-numpy\", when=\"+numpy\", type=(\"build\", \"run\"))\n depends_on(\"llvm-openmp\", when=\"%apple-clang +openmp\")\n depends_on(\"valgrind\", when=\"+valgrind\")\n # https://github.com/pytorch/pytorch/issues/60332\n # depends_on('xnnpack@2021-02-22', when='@1.8.0:+xnnpack')\n # depends_on('xnnpack@2020-03-23', when='@1.6.0:1.7+xnnpack')\n depends_on(\"mpi\", when=\"+mpi\")\n # https://github.com/pytorch/pytorch/issues/60270\n # depends_on('gloo@2021-05-04', when='@1.9.0:+gloo')\n # depends_on('gloo@2020-09-18', when='@1.7.0:1.8+gloo')\n # depends_on('gloo@2020-03-17', when='@1.6.0:1.6+gloo')\n # https://github.com/pytorch/pytorch/issues/60331\n # depends_on('onnx@1.8.0_2020-11-03', when='@1.8.0:+onnx_ml')\n # depends_on('onnx@1.7.0_2020-05-31', when='@1.6.0:1.7+onnx_ml')\n depends_on(\"mkl\", when=\"+mkldnn\")\n\n # Test dependencies\n depends_on(\"py-hypothesis\", type=\"test\")\n depends_on(\"py-six\", type=\"test\")\n depends_on(\"py-psutil\", type=\"test\")\n\n # Fix BLAS being overridden by MKL\n # https://github.com/pytorch/pytorch/issues/60328\n patch(\n \"https://patch-diff.githubusercontent.com/raw/pytorch/pytorch/pull/59220.patch\",\n sha256=\"e37afffe45cf7594c22050109942370e49983ad772d12ebccf508377dc9dcfc9\",\n when=\"@1.2.0:\",\n )\n\n # Fixes build on older systems with glibc <2.12\n patch(\n \"https://patch-diff.githubusercontent.com/raw/pytorch/pytorch/pull/55063.patch\",\n sha256=\"e17eaa42f5d7c18bf0d7c37d7b0910127a01ad53fdce3e226a92893356a70395\",\n when=\"@1.1.0:1.8.1\",\n )\n\n # Fixes CMake configuration error when XNNPACK is disabled\n # https://github.com/pytorch/pytorch/pull/35607\n # https://github.com/pytorch/pytorch/pull/37865\n patch(\"xnnpack.patch\", when=\"@1.5.0:1.5\")\n\n # Fixes build error when ROCm is enabled for pytorch-1.5 release\n patch(\"rocm.patch\", when=\"@1.5.0:1.5+rocm\")\n\n # Fixes fatal error: sleef.h: No such file or directory\n # https://github.com/pytorch/pytorch/pull/35359\n # https://github.com/pytorch/pytorch/issues/26555\n # patch('sleef.patch', when='@1.0.0:1.5')\n\n # Fixes compilation with Clang 9.0.0 and Apple Clang 11.0.3\n # https://github.com/pytorch/pytorch/pull/37086\n patch(\n \"https://github.com/pytorch/pytorch/commit/e921cd222a8fbeabf5a3e74e83e0d8dfb01aa8b5.patch\",\n sha256=\"17561b16cd2db22f10c0fe1fdcb428aecb0ac3964ba022a41343a6bb8cba7049\",\n when=\"@1.1:1.5\",\n )\n\n # Removes duplicate definition of getCusparseErrorString\n # https://github.com/pytorch/pytorch/issues/32083\n patch(\"cusparseGetErrorString.patch\", when=\"@0.4.1:1.0^cuda@10.1.243:\")\n\n # Fixes 'FindOpenMP.cmake'\n # to detect openmp settings used by Fujitsu compiler.\n patch(\"detect_omp_of_fujitsu_compiler.patch\", when=\"%fj\")\n\n # Fix compilation of +distributed~tensorpipe\n # https://github.com/pytorch/pytorch/issues/68002\n patch(\n \"https://github.com/pytorch/pytorch/commit/c075f0f633fa0136e68f0a455b5b74d7b500865c.patch\",\n sha256=\"e69e41b5c171bfb00d1b5d4ee55dd5e4c8975483230274af4ab461acd37e40b8\",\n when=\"@1.10.0+distributed~tensorpipe\",\n )\n\n # Both build and install run cmake/make/make install\n # Only run once to speed up build times\n phases = [\"install\"]\n\n @property\n def libs(self):\n root = join_path(\n self.prefix, self.spec[\"python\"].package.site_packages_dir, \"torch\", \"lib\"\n )\n return find_libraries(\"libtorch\", root)\n\n @property\n def headers(self):\n root = join_path(\n self.prefix, self.spec[\"python\"].package.site_packages_dir, \"torch\", \"include\"\n )\n headers = find_all_headers(root)\n headers.directories = [root]\n return headers\n\n @when(\"@1.5.0:\")\n def patch(self):\n # https://github.com/pytorch/pytorch/issues/52208\n filter_file(\n \"torch_global_deps PROPERTIES LINKER_LANGUAGE C\",\n \"torch_global_deps PROPERTIES LINKER_LANGUAGE CXX\",\n \"caffe2/CMakeLists.txt\",\n )\n\n def setup_build_environment(self, env: EnvironmentModifications) -> None:\n \"\"\"Set environment variables used to control the build.\n\n PyTorch's ``setup.py`` is a thin wrapper around ``cmake``.\n In ``tools/setup_helpers/cmake.py``, you can see that all\n environment variables that start with ``BUILD_``, ``USE_``,\n or ``CMAKE_``, plus a few more explicitly specified variable\n names, are passed directly to the ``cmake`` call. Therefore,\n most flags defined in ``CMakeLists.txt`` can be specified as\n environment variables.\n \"\"\"\n\n def enable_or_disable(variant, keyword=\"USE\", var=None, newer=False):\n \"\"\"Set environment variable to enable or disable support for a\n particular variant.\n\n Parameters:\n variant (str): the variant to check\n keyword (str): the prefix to use for enabling/disabling\n var (str): CMake variable to set. Defaults to variant.upper()\n newer (bool): newer variants that never used NO_*\n \"\"\"\n if var is None:\n var = variant.upper()\n\n # Version 1.1.0 switched from NO_* to USE_* or BUILD_*\n # But some newer variants have always used USE_* or BUILD_*\n if self.spec.satisfies(\"@1.1:\") or newer:\n if \"+\" + variant in self.spec:\n env.set(keyword + \"_\" + var, \"ON\")\n else:\n env.set(keyword + \"_\" + var, \"OFF\")\n else:\n if \"+\" + variant in self.spec:\n env.unset(\"NO_\" + var)\n else:\n env.set(\"NO_\" + var, \"ON\")\n\n # Build in parallel to speed up build times\n env.set(\"MAX_JOBS\", make_jobs)\n\n # Spack logs have trouble handling colored output\n env.set(\"COLORIZE_OUTPUT\", \"OFF\")\n\n if self.spec.satisfies(\"@0.4:\"):\n enable_or_disable(\"test\", keyword=\"BUILD\")\n\n if self.spec.satisfies(\"@1.7:\"):\n enable_or_disable(\"caffe2\", keyword=\"BUILD\")\n\n enable_or_disable(\"cuda\")\n if \"+cuda\" in self.spec:\n # cmake/public/cuda.cmake\n # cmake/Modules_CUDA_fix/upstream/FindCUDA.cmake\n env.unset(\"CUDA_ROOT\")\n torch_cuda_arch = \";\".join(\n \"{0:.1f}\".format(float(i) / 10.0) for i in self.spec.variants[\"cuda_arch\"].value\n )\n env.set(\"TORCH_CUDA_ARCH_LIST\", torch_cuda_arch)\n\n enable_or_disable(\"rocm\")\n\n enable_or_disable(\"cudnn\")\n if \"+cudnn\" in self.spec:\n # cmake/Modules_CUDA_fix/FindCUDNN.cmake\n env.set(\"CUDNN_INCLUDE_DIR\", self.spec[\"cudnn\"].prefix.include)\n env.set(\"CUDNN_LIBRARY\", self.spec[\"cudnn\"].libs[0])\n\n enable_or_disable(\"fbgemm\")\n if self.spec.satisfies(\"@1.8:\"):\n enable_or_disable(\"kineto\")\n enable_or_disable(\"magma\")\n enable_or_disable(\"metal\")\n if self.spec.satisfies(\"@1.10:\"):\n enable_or_disable(\"breakpad\")\n\n enable_or_disable(\"nccl\")\n if \"+nccl\" in self.spec:\n env.set(\"NCCL_LIB_DIR\", self.spec[\"nccl\"].libs.directories[0])\n env.set(\"NCCL_INCLUDE_DIR\", self.spec[\"nccl\"].prefix.include)\n\n # cmake/External/nnpack.cmake\n enable_or_disable(\"nnpack\")\n\n enable_or_disable(\"numa\")\n if \"+numa\" in self.spec:\n # cmake/Modules/FindNuma.cmake\n env.set(\"NUMA_ROOT_DIR\", self.spec[\"numactl\"].prefix)\n\n # cmake/Modules/FindNumPy.cmake\n enable_or_disable(\"numpy\")\n # cmake/Modules/FindOpenMP.cmake\n enable_or_disable(\"openmp\", newer=True)\n enable_or_disable(\"qnnpack\")\n if self.spec.satisfies(\"@1.3:\"):\n enable_or_disable(\"qnnpack\", var=\"PYTORCH_QNNPACK\")\n if self.spec.satisfies(\"@1.8:\"):\n enable_or_disable(\"valgrind\")\n if self.spec.satisfies(\"@1.5:\"):\n enable_or_disable(\"xnnpack\")\n enable_or_disable(\"mkldnn\")\n enable_or_disable(\"distributed\")\n enable_or_disable(\"mpi\")\n # cmake/Modules/FindGloo.cmake\n enable_or_disable(\"gloo\", newer=True)\n if self.spec.satisfies(\"@1.6:\"):\n enable_or_disable(\"tensorpipe\")\n\n if \"+onnx_ml\" in self.spec:\n env.set(\"ONNX_ML\", \"ON\")\n else:\n env.set(\"ONNX_ML\", \"OFF\")\n\n if not self.spec.satisfies(\"@master\"):\n env.set(\"PYTORCH_BUILD_VERSION\", self.version)\n env.set(\"PYTORCH_BUILD_NUMBER\", 0)\n\n # BLAS to be used by Caffe2\n # Options defined in cmake/Dependencies.cmake and cmake/Modules/FindBLAS.cmake\n if self.spec[\"blas\"].name == \"atlas\":\n env.set(\"BLAS\", \"ATLAS\")\n env.set(\"WITH_BLAS\", \"atlas\")\n elif self.spec[\"blas\"].name in [\"blis\", \"amdblis\"]:\n env.set(\"BLAS\", \"BLIS\")\n env.set(\"WITH_BLAS\", \"blis\")\n elif self.spec[\"blas\"].name == \"eigen\":\n env.set(\"BLAS\", \"Eigen\")\n elif self.spec[\"lapack\"].name in [\"libflame\", \"amdlibflame\"]:\n env.set(\"BLAS\", \"FLAME\")\n env.set(\"WITH_BLAS\", \"FLAME\")\n elif self.spec[\"blas\"].name in [\"intel-mkl\", \"intel-parallel-studio\", \"intel-oneapi-mkl\"]:\n env.set(\"BLAS\", \"MKL\")\n env.set(\"WITH_BLAS\", \"mkl\")\n elif self.spec[\"blas\"].name == \"openblas\":\n env.set(\"BLAS\", \"OpenBLAS\")\n env.set(\"WITH_BLAS\", \"open\")\n elif self.spec[\"blas\"].name == \"veclibfort\":\n env.set(\"BLAS\", \"vecLib\")\n env.set(\"WITH_BLAS\", \"veclib\")\n else:\n env.set(\"BLAS\", \"Generic\")\n env.set(\"WITH_BLAS\", \"generic\")\n\n # Don't use vendored third-party libraries when possible\n env.set(\"BUILD_CUSTOM_PROTOBUF\", \"OFF\")\n env.set(\"USE_SYSTEM_NCCL\", \"ON\")\n env.set(\"USE_SYSTEM_EIGEN_INSTALL\", \"ON\")\n if self.spec.satisfies(\"@0.4:\"):\n env.set(\"pybind11_DIR\", self.spec[\"py-pybind11\"].prefix)\n env.set(\"pybind11_INCLUDE_DIR\", self.spec[\"py-pybind11\"].prefix.include)\n if self.spec.satisfies(\"@1.10:\"):\n env.set(\"USE_SYSTEM_PYBIND11\", \"ON\")\n # https://github.com/pytorch/pytorch/issues/60334\n # if self.spec.satisfies('@1.8:'):\n # env.set('USE_SYSTEM_SLEEF', 'ON')\n if self.spec.satisfies(\"@1.6:\"):\n # env.set('USE_SYSTEM_LIBS', 'ON')\n # https://github.com/pytorch/pytorch/issues/60329\n # env.set('USE_SYSTEM_CPUINFO', 'ON')\n # https://github.com/pytorch/pytorch/issues/60270\n # env.set('USE_SYSTEM_GLOO', 'ON')\n # https://github.com/Maratyszcza/FP16/issues/18\n # env.set('USE_SYSTEM_FP16', 'ON')\n env.set(\"USE_SYSTEM_PTHREADPOOL\", \"ON\")\n env.set(\"USE_SYSTEM_PSIMD\", \"ON\")\n env.set(\"USE_SYSTEM_FXDIV\", \"ON\")\n env.set(\"USE_SYSTEM_BENCHMARK\", \"ON\")\n # https://github.com/pytorch/pytorch/issues/60331\n # env.set('USE_SYSTEM_ONNX', 'ON')\n # https://github.com/pytorch/pytorch/issues/60332\n # env.set('USE_SYSTEM_XNNPACK', 'ON')\n\n @run_before(\"install\")\n def build_amd(self):\n if \"+rocm\" in self.spec:\n python(os.path.join(\"tools\", \"amd_build\", \"build_amd.py\"))\n\n @run_after(\"install\")\n @on_package_attributes(run_tests=True)\n def install_test(self):\n with working_dir(\"test\"):\n python(\"run_test.py\")\n\n # Tests need to be re-added since `phases` was overridden\n run_after(\"install\")(PythonPackage._run_default_install_time_test_callbacks)\n run_after(\"install\")(PythonPackage.sanity_check_prefix)\n" }, { "path": "lib/spack/spack/test/data/unparse/trilinos.txt", "content": "# -*- python -*-\n# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"This is an unparser test package.\n\n``trilinos`` was chosen because it's one of the most complex packages in Spack, because\nit has a lot of nested ``with when():`` blocks, and because it has loops and nested\nlogic at the package level.\n\n\"\"\"\n\nimport os\nimport sys\n\nfrom spack import *\nfrom spack.build_environment import dso_suffix\nfrom spack.error import NoHeadersError\nfrom spack.operating_systems.mac_os import macos_version\nfrom spack.pkg.builtin.kokkos import Kokkos\n\n# Trilinos is complicated to build, as an inspiration a couple of links to\n# other repositories which build it:\n# https://github.com/hpcugent/easybuild-easyblocks/blob/master/easybuild/easyblocks/t/trilinos.py#L111\n# https://github.com/koecher/candi/blob/master/deal.II-toolchain/packages/trilinos.package\n# https://gitlab.com/configurations/cluster-config/blob/master/trilinos.sh\n# https://github.com/Homebrew/homebrew-science/blob/master/trilinos.rb and some\n# relevant documentation/examples:\n# https://github.com/trilinos/Trilinos/issues/175\n\n\nclass Trilinos(CMakePackage, CudaPackage):\n \"\"\"The Trilinos Project is an effort to develop algorithms and enabling\n technologies within an object-oriented software framework for the solution\n of large-scale, complex multi-physics engineering and scientific problems.\n A unique design feature of Trilinos is its focus on packages.\n \"\"\"\n\n homepage = \"https://trilinos.org/\"\n url = \"https://github.com/trilinos/Trilinos/archive/trilinos-release-12-12-1.tar.gz\"\n git = \"https://github.com/trilinos/Trilinos.git\"\n\n maintainers(\"keitat\", \"sethrj\", \"kuberry\")\n\n tags = [\"e4s\"]\n\n # ###################### Versions ##########################\n\n version(\"master\", branch=\"master\")\n version(\"develop\", branch=\"develop\")\n version(\n \"13.2.0\", commit=\"4a5f7906a6420ee2f9450367e9cc95b28c00d744\"\n ) # tag trilinos-release-13-2-0\n version(\n \"13.0.1\", commit=\"4796b92fb0644ba8c531dd9953e7a4878b05c62d\", preferred=True\n ) # tag trilinos-release-13-0-1\n version(\n \"13.0.0\", commit=\"9fec35276d846a667bc668ff4cbdfd8be0dfea08\"\n ) # tag trilinos-release-13-0-0\n version(\n \"12.18.1\", commit=\"55a75997332636a28afc9db1aee4ae46fe8d93e7\"\n ) # tag trilinos-release-12-8-1\n version(\"12.14.1\", sha256=\"52a4406cca2241f5eea8e166c2950471dd9478ad6741cbb2a7fc8225814616f0\")\n version(\"12.12.1\", sha256=\"5474c5329c6309224a7e1726cf6f0d855025b2042959e4e2be2748bd6bb49e18\")\n version(\"12.10.1\", sha256=\"ab81d917196ffbc21c4927d42df079dd94c83c1a08bda43fef2dd34d0c1a5512\")\n version(\"12.8.1\", sha256=\"d20fe60e31e3ba1ef36edecd88226240a518f50a4d6edcc195b88ee9dda5b4a1\")\n version(\"12.6.4\", sha256=\"1c7104ba60ee8cc4ec0458a1c4f6a26130616bae7580a7b15f2771a955818b73\")\n version(\"12.6.3\", sha256=\"4d28298bb4074eef522db6cd1626f1a934e3d80f292caf669b8846c0a458fe81\")\n version(\"12.6.2\", sha256=\"8be7e3e1166cc05aea7f856cc8033182e8114aeb8f87184cb38873bfb2061779\")\n version(\"12.6.1\", sha256=\"4b38ede471bed0036dcb81a116fba8194f7bf1a9330da4e29c3eb507d2db18db\")\n version(\"12.4.2\", sha256=\"fd2c12e87a7cedc058bcb8357107ffa2474997aa7b17b8e37225a1f7c32e6f0e\")\n version(\"12.2.1\", sha256=\"088f303e0dc00fb4072b895c6ecb4e2a3ad9a2687b9c62153de05832cf242098\")\n version(\"12.0.1\", sha256=\"eee7c19ca108538fa1c77a6651b084e06f59d7c3307dae77144136639ab55980\")\n version(\"11.14.3\", sha256=\"e37fa5f69103576c89300e14d43ba77ad75998a54731008b25890d39892e6e60\")\n version(\"11.14.2\", sha256=\"f22b2b0df7b88e28b992e19044ba72b845292b93cbbb3a948488199647381119\")\n version(\"11.14.1\", sha256=\"f10fc0a496bf49427eb6871c80816d6e26822a39177d850cc62cf1484e4eec07\")\n\n # ###################### Variants ##########################\n\n # Build options\n variant(\"complex\", default=False, description=\"Enable complex numbers in Trilinos\")\n variant(\"cuda_rdc\", default=False, description=\"turn on RDC for CUDA build\")\n variant(\"cxxstd\", default=\"14\", values=[\"11\", \"14\", \"17\"], multi=False)\n variant(\"debug\", default=False, description=\"Enable runtime safety and debug checks\")\n variant(\n \"explicit_template_instantiation\",\n default=True,\n description=\"Enable explicit template instantiation (ETI)\",\n )\n variant(\n \"float\", default=False, description=\"Enable single precision (float) numbers in Trilinos\"\n )\n variant(\"fortran\", default=True, description=\"Compile with Fortran support\")\n variant(\n \"gotype\",\n default=\"long_long\",\n values=(\"int\", \"long\", \"long_long\", \"all\"),\n multi=False,\n description=\"global ordinal type for Tpetra\",\n )\n variant(\"openmp\", default=False, description=\"Enable OpenMP\")\n variant(\"python\", default=False, description=\"Build PyTrilinos wrappers\")\n variant(\"shared\", default=True, description=\"Enables the build of shared libraries\")\n variant(\"wrapper\", default=False, description=\"Use nvcc-wrapper for CUDA build\")\n\n # TPLs (alphabet order)\n variant(\"adios2\", default=False, description=\"Enable ADIOS2\")\n variant(\"boost\", default=False, description=\"Compile with Boost\")\n variant(\"hdf5\", default=False, description=\"Compile with HDF5\")\n variant(\"hypre\", default=False, description=\"Compile with Hypre preconditioner\")\n variant(\"mpi\", default=True, description=\"Compile with MPI parallelism\")\n variant(\"mumps\", default=False, description=\"Compile with support for MUMPS solvers\")\n variant(\"suite-sparse\", default=False, description=\"Compile with SuiteSparse solvers\")\n variant(\"superlu-dist\", default=False, description=\"Compile with SuperluDist solvers\")\n variant(\"superlu\", default=False, description=\"Compile with SuperLU solvers\")\n variant(\"strumpack\", default=False, description=\"Compile with STRUMPACK solvers\")\n variant(\"x11\", default=False, description=\"Compile with X11 when +exodus\")\n\n # Package options (alphabet order)\n variant(\"amesos\", default=True, description=\"Compile with Amesos\")\n variant(\"amesos2\", default=True, description=\"Compile with Amesos2\")\n variant(\"anasazi\", default=True, description=\"Compile with Anasazi\")\n variant(\"aztec\", default=True, description=\"Compile with Aztec\")\n variant(\"belos\", default=True, description=\"Compile with Belos\")\n variant(\"chaco\", default=False, description=\"Compile with Chaco from SEACAS\")\n variant(\"epetra\", default=True, description=\"Compile with Epetra\")\n variant(\"epetraext\", default=True, description=\"Compile with EpetraExt\")\n variant(\"exodus\", default=False, description=\"Compile with Exodus from SEACAS\")\n variant(\"ifpack\", default=True, description=\"Compile with Ifpack\")\n variant(\"ifpack2\", default=True, description=\"Compile with Ifpack2\")\n variant(\"intrepid\", default=False, description=\"Enable Intrepid\")\n variant(\"intrepid2\", default=False, description=\"Enable Intrepid2\")\n variant(\"isorropia\", default=False, description=\"Compile with Isorropia\")\n variant(\"gtest\", default=False, description=\"Build vendored Googletest\")\n variant(\"kokkos\", default=True, description=\"Compile with Kokkos\")\n variant(\"ml\", default=True, description=\"Compile with ML\")\n variant(\"minitensor\", default=False, description=\"Compile with MiniTensor\")\n variant(\"muelu\", default=True, description=\"Compile with Muelu\")\n variant(\"nox\", default=False, description=\"Compile with NOX\")\n variant(\"piro\", default=False, description=\"Compile with Piro\")\n variant(\"phalanx\", default=False, description=\"Compile with Phalanx\")\n variant(\"rol\", default=False, description=\"Compile with ROL\")\n variant(\"rythmos\", default=False, description=\"Compile with Rythmos\")\n variant(\"sacado\", default=True, description=\"Compile with Sacado\")\n variant(\"stk\", default=False, description=\"Compile with STK\")\n variant(\"shards\", default=False, description=\"Compile with Shards\")\n variant(\"shylu\", default=False, description=\"Compile with ShyLU\")\n variant(\"stokhos\", default=False, description=\"Compile with Stokhos\")\n variant(\"stratimikos\", default=False, description=\"Compile with Stratimikos\")\n variant(\"teko\", default=False, description=\"Compile with Teko\")\n variant(\"tempus\", default=False, description=\"Compile with Tempus\")\n variant(\"tpetra\", default=True, description=\"Compile with Tpetra\")\n variant(\"trilinoscouplings\", default=False, description=\"Compile with TrilinosCouplings\")\n variant(\"zoltan\", default=False, description=\"Compile with Zoltan\")\n variant(\"zoltan2\", default=False, description=\"Compile with Zoltan2\")\n\n # Internal package options (alphabetical order)\n variant(\"basker\", default=False, description=\"Compile with the Basker solver in Amesos2\")\n variant(\"epetraextbtf\", default=False, description=\"Compile with BTF in EpetraExt\")\n variant(\n \"epetraextexperimental\",\n default=False,\n description=\"Compile with experimental in EpetraExt\",\n )\n variant(\n \"epetraextgraphreorderings\",\n default=False,\n description=\"Compile with graph reorderings in EpetraExt\",\n )\n\n # External package options\n variant(\"dtk\", default=False, description=\"Enable DataTransferKit (deprecated)\")\n variant(\"scorec\", default=False, description=\"Enable SCOREC\")\n variant(\"mesquite\", default=False, description=\"Enable Mesquite (deprecated)\")\n\n resource(\n name=\"dtk\",\n git=\"https://github.com/ornl-cees/DataTransferKit.git\",\n commit=\"4fe4d9d56cfd4f8a61f392b81d8efd0e389ee764\", # branch dtk-3.0\n placement=\"DataTransferKit\",\n when=\"+dtk @12.14.0:12.14\",\n )\n resource(\n name=\"dtk\",\n git=\"https://github.com/ornl-cees/DataTransferKit.git\",\n commit=\"edfa050cd46e2274ab0a0b7558caca0079c2e4ca\", # tag 3.1-rc1\n placement=\"DataTransferKit\",\n submodules=True,\n when=\"+dtk @12.18.0:12.18\",\n )\n resource(\n name=\"scorec\",\n git=\"https://github.com/SCOREC/core.git\",\n commit=\"73c16eae073b179e45ec625a5abe4915bc589af2\", # tag v2.2.5\n placement=\"SCOREC\",\n when=\"+scorec\",\n )\n resource(\n name=\"mesquite\",\n url=\"https://github.com/trilinos/mesquite/archive/trilinos-release-12-12-1.tar.gz\",\n sha256=\"e0d09b0939dbd461822477449dca611417316e8e8d8268fd795debb068edcbb5\",\n placement=\"packages/mesquite\",\n when=\"+mesquite @12.12.1:12.16\",\n )\n resource(\n name=\"mesquite\",\n git=\"https://github.com/trilinos/mesquite.git\",\n commit=\"20a679679b5cdf15bf573d66c5dc2b016e8b9ca1\", # branch trilinos-release-12-12-1\n placement=\"packages/mesquite\",\n when=\"+mesquite @12.18.1:12.18\",\n )\n resource(\n name=\"mesquite\",\n git=\"https://github.com/trilinos/mesquite.git\",\n tag=\"develop\",\n placement=\"packages/mesquite\",\n when=\"+mesquite @master\",\n )\n\n # ###################### Conflicts ##########################\n\n # Epetra packages\n with when(\"~epetra\"):\n conflicts(\"+amesos\")\n conflicts(\"+aztec\")\n conflicts(\"+epetraext\")\n conflicts(\"+ifpack\")\n conflicts(\"+isorropia\")\n conflicts(\"+ml\", when=\"@13.2:\")\n with when(\"~epetraext\"):\n conflicts(\"+isorropia\")\n conflicts(\"+teko\")\n conflicts(\"+epetraextbtf\")\n conflicts(\"+epetraextexperimental\")\n conflicts(\"+epetraextgraphreorderings\")\n\n # Tpetra packages\n with when(\"~kokkos\"):\n conflicts(\"+cuda\")\n conflicts(\"+tpetra\")\n conflicts(\"+intrepid2\")\n conflicts(\"+phalanx\")\n with when(\"~tpetra\"):\n conflicts(\"+amesos2\")\n conflicts(\"+dtk\")\n conflicts(\"+ifpack2\")\n conflicts(\"+muelu\")\n conflicts(\"+teko\")\n conflicts(\"+zoltan2\")\n\n with when(\"+teko\"):\n conflicts(\"~amesos\")\n conflicts(\"~anasazi\")\n conflicts(\"~aztec\")\n conflicts(\"~ifpack\")\n conflicts(\"~ml\")\n conflicts(\"~stratimikos\")\n conflicts(\"@:12 gotype=long\")\n\n # Known requirements from tribits dependencies\n conflicts(\"+aztec\", when=\"~fortran\")\n conflicts(\"+basker\", when=\"~amesos2\")\n conflicts(\"+minitensor\", when=\"~boost\")\n conflicts(\"+ifpack2\", when=\"~belos\")\n conflicts(\"+intrepid\", when=\"~sacado\")\n conflicts(\"+intrepid\", when=\"~shards\")\n conflicts(\"+intrepid2\", when=\"~shards\")\n conflicts(\"+isorropia\", when=\"~zoltan\")\n conflicts(\"+phalanx\", when=\"~sacado\")\n conflicts(\"+scorec\", when=\"~mpi\")\n conflicts(\"+scorec\", when=\"~shards\")\n conflicts(\"+scorec\", when=\"~stk\")\n conflicts(\"+scorec\", when=\"~zoltan\")\n conflicts(\"+tempus\", when=\"~nox\")\n conflicts(\"+zoltan2\", when=\"~zoltan\")\n\n # Only allow DTK with Trilinos 12.14, 12.18\n conflicts(\"+dtk\", when=\"~boost\")\n conflicts(\"+dtk\", when=\"~intrepid2\")\n conflicts(\"+dtk\", when=\"@:12.12,13:\")\n\n # Installed FindTrilinos are broken in SEACAS if Fortran is disabled\n # see https://github.com/trilinos/Trilinos/issues/3346\n conflicts(\"+exodus\", when=\"@:13.0.1 ~fortran\")\n # Only allow Mesquite with Trilinos 12.12 and up, and master\n conflicts(\"+mesquite\", when=\"@:12.10,master\")\n # Strumpack is only available as of mid-2021\n conflicts(\"+strumpack\", when=\"@:13.0\")\n # Can only use one type of SuperLU\n conflicts(\"+superlu-dist\", when=\"+superlu\")\n # For Trilinos v11 we need to force SuperLUDist=OFF, since only the\n # deprecated SuperLUDist v3.3 together with an Amesos patch is working.\n conflicts(\"+superlu-dist\", when=\"@11.4.1:11.14.3\")\n # see https://github.com/trilinos/Trilinos/issues/3566\n conflicts(\n \"+superlu-dist\", when=\"+float+amesos2+explicit_template_instantiation^superlu-dist@5.3.0:\"\n )\n # Amesos, conflicting types of double and complex SLU_D\n # see https://trilinos.org/pipermail/trilinos-users/2015-March/004731.html\n # and https://trilinos.org/pipermail/trilinos-users/2015-March/004802.html\n conflicts(\"+superlu-dist\", when=\"+complex+amesos2\")\n # https://github.com/trilinos/Trilinos/issues/2994\n conflicts(\n \"+shared\",\n when=\"+stk platform=darwin\",\n msg=\"Cannot build Trilinos with STK as a shared library on Darwin.\",\n )\n conflicts(\"+adios2\", when=\"@:12.14.1\")\n conflicts(\"cxxstd=11\", when=\"@master:\")\n conflicts(\"cxxstd=11\", when=\"+wrapper ^cuda@6.5.14\")\n conflicts(\"cxxstd=14\", when=\"+wrapper ^cuda@6.5.14:8.0.61\")\n conflicts(\"cxxstd=17\", when=\"+wrapper ^cuda@6.5.14:10.2.89\")\n\n # Multi-value gotype only applies to trilinos through 12.14\n conflicts(\"gotype=all\", when=\"@12.15:\")\n\n # CUDA without wrapper requires clang\n for _compiler in spack.compilers.supported_compilers():\n if _compiler != \"clang\":\n conflicts(\n \"+cuda\",\n when=\"~wrapper %\" + _compiler,\n msg=\"trilinos~wrapper+cuda can only be built with the \" \"Clang compiler\",\n )\n conflicts(\"+cuda_rdc\", when=\"~cuda\")\n conflicts(\"+wrapper\", when=\"~cuda\")\n conflicts(\"+wrapper\", when=\"%clang\")\n\n # Old trilinos fails with new CUDA (see #27180)\n conflicts(\"@:13.0.1 +cuda\", when=\"^cuda@11:\")\n\n # stokhos fails on xl/xl_r\n conflicts(\"+stokhos\", when=\"%xl\")\n conflicts(\"+stokhos\", when=\"%xl_r\")\n\n # Fortran mangling fails on Apple M1 (see spack/spack#25900)\n conflicts(\"@:13.0.1 +fortran\", when=\"target=m1\")\n\n # ###################### Dependencies ##########################\n\n depends_on(\"adios2\", when=\"+adios2\")\n depends_on(\"blas\")\n depends_on(\"boost\", when=\"+boost\")\n depends_on(\"cgns\", when=\"+exodus\")\n depends_on(\"hdf5+hl\", when=\"+hdf5\")\n depends_on(\"hypre~internal-superlu~int64\", when=\"+hypre\")\n depends_on(\"kokkos-nvcc-wrapper\", when=\"+wrapper\")\n depends_on(\"lapack\")\n # depends_on('perl', type=('build',)) # TriBITS finds but doesn't use...\n depends_on(\"libx11\", when=\"+x11\")\n depends_on(\"matio\", when=\"+exodus\")\n depends_on(\"metis\", when=\"+zoltan\")\n depends_on(\"mpi\", when=\"+mpi\")\n depends_on(\"netcdf-c\", when=\"+exodus\")\n depends_on(\"parallel-netcdf\", when=\"+exodus+mpi\")\n depends_on(\"parmetis\", when=\"+mpi +zoltan\")\n depends_on(\"parmetis\", when=\"+scorec\")\n depends_on(\"py-mpi4py\", when=\"+mpi+python\", type=(\"build\", \"run\"))\n depends_on(\"py-numpy\", when=\"+python\", type=(\"build\", \"run\"))\n depends_on(\"python\", when=\"+python\")\n depends_on(\"python\", when=\"@13.2: +ifpack +hypre\", type=\"build\")\n depends_on(\"python\", when=\"@13.2: +ifpack2 +hypre\", type=\"build\")\n depends_on(\"scalapack\", when=\"+mumps\")\n depends_on(\"scalapack\", when=\"+strumpack+mpi\")\n depends_on(\"strumpack+shared\", when=\"+strumpack\")\n depends_on(\"suite-sparse\", when=\"+suite-sparse\")\n depends_on(\"superlu-dist\", when=\"+superlu-dist\")\n depends_on(\"superlu@4.3 +pic\", when=\"+superlu\")\n depends_on(\"swig\", when=\"+python\")\n depends_on(\"zlib\", when=\"+zoltan\")\n\n # Trilinos' Tribits config system is limited which makes it very tricky to\n # link Amesos with static MUMPS, see\n # https://trilinos.org/docs/dev/packages/amesos2/doc/html/classAmesos2_1_1MUMPS.html\n # One could work it out by getting linking flags from mpif90 --showme:link\n # (or alike) and adding results to -DTrilinos_EXTRA_LINK_FLAGS together\n # with Blas and Lapack and ScaLAPACK and Blacs and -lgfortran and it may\n # work at the end. But let's avoid all this by simply using shared libs\n depends_on(\"mumps@5.0:+shared\", when=\"+mumps\")\n\n for _flag in (\"~mpi\", \"+mpi\"):\n depends_on(\"hdf5\" + _flag, when=\"+hdf5\" + _flag)\n depends_on(\"mumps\" + _flag, when=\"+mumps\" + _flag)\n for _flag in (\"~openmp\", \"+openmp\"):\n depends_on(\"mumps\" + _flag, when=\"+mumps\" + _flag)\n\n depends_on(\"hwloc\", when=\"@13: +kokkos\")\n depends_on(\"hwloc+cuda\", when=\"@13: +kokkos+cuda\")\n depends_on(\"hypre@develop\", when=\"@master: +hypre\")\n depends_on(\"netcdf-c+mpi+parallel-netcdf\", when=\"+exodus+mpi@12.12.1:\")\n depends_on(\"superlu-dist@4.4:5.3\", when=\"@12.6.2:12.12.1+superlu-dist\")\n depends_on(\"superlu-dist@5.4:6.2.0\", when=\"@12.12.2:13.0.0+superlu-dist\")\n depends_on(\"superlu-dist@6.3.0:\", when=\"@13.0.1:99 +superlu-dist\")\n depends_on(\"superlu-dist@:4.3\", when=\"@11.14.1:12.6.1+superlu-dist\")\n depends_on(\"superlu-dist@develop\", when=\"@master: +superlu-dist\")\n\n # ###################### Patches ##########################\n\n patch(\"umfpack_from_suitesparse.patch\", when=\"@11.14.1:12.8.1\")\n for _compiler in [\"xl\", \"xl_r\", \"clang\"]:\n patch(\"xlf_seacas.patch\", when=\"@12.10.1:12.12.1 %\" + _compiler)\n patch(\"xlf_tpetra.patch\", when=\"@12.12.1 %\" + _compiler)\n patch(\"fix_clang_errors_12_18_1.patch\", when=\"@12.18.1%clang\")\n patch(\"cray_secas_12_12_1.patch\", when=\"@12.12.1%cce\")\n patch(\"cray_secas.patch\", when=\"@12.14.1:%cce\")\n\n # workaround an NVCC bug with c++14 (https://github.com/trilinos/Trilinos/issues/6954)\n # avoid calling deprecated functions with CUDA-11\n patch(\"fix_cxx14_cuda11.patch\", when=\"@13.0.0:13.0.1 cxxstd=14 ^cuda@11:\")\n # Allow building with +teko gotype=long\n patch(\n \"https://github.com/trilinos/Trilinos/commit/b17f20a0b91e0b9fc5b1b0af3c8a34e2a4874f3f.patch\",\n sha256=\"dee6c55fe38eb7f6367e1896d6bc7483f6f9ab8fa252503050cc0c68c6340610\",\n when=\"@13.0.0:13.0.1 +teko gotype=long\",\n )\n\n def flag_handler(self, name, flags):\n is_cce = self.spec.satisfies(\"%cce\")\n\n if name == \"cxxflags\":\n spec = self.spec\n if \"+mumps\" in spec:\n # see https://github.com/trilinos/Trilinos/blob/master/packages/amesos/README-MUMPS\n flags.append(\"-DMUMPS_5_0\")\n if \"+stk platform=darwin\" in spec:\n flags.append(\"-DSTK_NO_BOOST_STACKTRACE\")\n if \"+stk%intel\" in spec:\n # Workaround for Intel compiler segfaults with STK and IPO\n flags.append(\"-no-ipo\")\n if \"+wrapper\" in spec:\n flags.append(\"--expt-extended-lambda\")\n elif name == \"ldflags\" and is_cce:\n flags.append(\"-fuse-ld=gold\")\n\n if is_cce:\n return (None, None, flags)\n return (flags, None, None)\n\n def url_for_version(self, version):\n url = \"https://github.com/trilinos/Trilinos/archive/trilinos-release-{0}.tar.gz\"\n return url.format(version.dashed)\n\n def setup_dependent_run_environment(self, env: EnvironmentModifications, dependent_spec: Spec) -> None:\n if \"+cuda\" in self.spec:\n # currently Trilinos doesn't perform the memory fence so\n # it relies on blocking CUDA kernel launch. This is needed\n # in case the dependent app also run a CUDA backend via Trilinos\n env.set(\"CUDA_LAUNCH_BLOCKING\", \"1\")\n\n def setup_dependent_package(self, module, dependent_spec):\n if \"+wrapper\" in self.spec:\n self.spec.kokkos_cxx = self.spec[\"kokkos-nvcc-wrapper\"].kokkos_cxx\n else:\n self.spec.kokkos_cxx = spack_cxx\n\n def setup_build_environment(self, env: EnvironmentModifications) -> None:\n spec = self.spec\n if \"+cuda\" in spec and \"+wrapper\" in spec:\n if \"+mpi\" in spec:\n env.set(\"OMPI_CXX\", spec[\"kokkos-nvcc-wrapper\"].kokkos_cxx)\n env.set(\"MPICH_CXX\", spec[\"kokkos-nvcc-wrapper\"].kokkos_cxx)\n env.set(\"MPICXX_CXX\", spec[\"kokkos-nvcc-wrapper\"].kokkos_cxx)\n else:\n env.set(\"CXX\", spec[\"kokkos-nvcc-wrapper\"].kokkos_cxx)\n\n def cmake_args(self):\n options = []\n\n spec = self.spec\n define = CMakePackage.define\n define_from_variant = self.define_from_variant\n\n def _make_definer(prefix):\n def define_enable(suffix, value=None):\n key = prefix + suffix\n if value is None:\n # Default to lower-case spec\n value = suffix.lower()\n elif isinstance(value, bool):\n # Explicit true/false\n return define(key, value)\n return define_from_variant(key, value)\n\n return define_enable\n\n # Return \"Trilinos_ENABLE_XXX\" for spec \"+xxx\" or boolean value\n define_trilinos_enable = _make_definer(\"Trilinos_ENABLE_\")\n # Same but for TPLs\n define_tpl_enable = _make_definer(\"TPL_ENABLE_\")\n\n # #################### Base Settings #######################\n\n options.extend(\n [\n define(\"Trilinos_VERBOSE_CONFIGURE\", False),\n define_from_variant(\"BUILD_SHARED_LIBS\", \"shared\"),\n define_from_variant(\"CMAKE_CXX_STANDARD\", \"cxxstd\"),\n define_trilinos_enable(\"ALL_OPTIONAL_PACKAGES\", False),\n define_trilinos_enable(\"ALL_PACKAGES\", False),\n define_trilinos_enable(\"CXX11\", True),\n define_trilinos_enable(\"DEBUG\", \"debug\"),\n define_trilinos_enable(\"EXAMPLES\", False),\n define_trilinos_enable(\"SECONDARY_TESTED_CODE\", True),\n define_trilinos_enable(\"TESTS\", False),\n define_trilinos_enable(\"Fortran\"),\n define_trilinos_enable(\"OpenMP\"),\n define_trilinos_enable(\n \"EXPLICIT_INSTANTIATION\", \"explicit_template_instantiation\"\n ),\n ]\n )\n\n # ################## Trilinos Packages #####################\n\n options.extend(\n [\n define_trilinos_enable(\"Amesos\"),\n define_trilinos_enable(\"Amesos2\"),\n define_trilinos_enable(\"Anasazi\"),\n define_trilinos_enable(\"AztecOO\", \"aztec\"),\n define_trilinos_enable(\"Belos\"),\n define_trilinos_enable(\"Epetra\"),\n define_trilinos_enable(\"EpetraExt\"),\n define_trilinos_enable(\"FEI\", False),\n define_trilinos_enable(\"Gtest\"),\n define_trilinos_enable(\"Ifpack\"),\n define_trilinos_enable(\"Ifpack2\"),\n define_trilinos_enable(\"Intrepid\"),\n define_trilinos_enable(\"Intrepid2\"),\n define_trilinos_enable(\"Isorropia\"),\n define_trilinos_enable(\"Kokkos\"),\n define_trilinos_enable(\"MiniTensor\"),\n define_trilinos_enable(\"Mesquite\"),\n define_trilinos_enable(\"ML\"),\n define_trilinos_enable(\"MueLu\"),\n define_trilinos_enable(\"NOX\"),\n define_trilinos_enable(\"Pamgen\", False),\n define_trilinos_enable(\"Panzer\", False),\n define_trilinos_enable(\"Pike\", False),\n define_trilinos_enable(\"Piro\"),\n define_trilinos_enable(\"Phalanx\"),\n define_trilinos_enable(\"PyTrilinos\", \"python\"),\n define_trilinos_enable(\"ROL\"),\n define_trilinos_enable(\"Rythmos\"),\n define_trilinos_enable(\"Sacado\"),\n define_trilinos_enable(\"SCOREC\"),\n define_trilinos_enable(\"Shards\"),\n define_trilinos_enable(\"ShyLU\"),\n define_trilinos_enable(\"STK\"),\n define_trilinos_enable(\"Stokhos\"),\n define_trilinos_enable(\"Stratimikos\"),\n define_trilinos_enable(\"Teko\"),\n define_trilinos_enable(\"Tempus\"),\n define_trilinos_enable(\"Tpetra\"),\n define_trilinos_enable(\"TrilinosCouplings\"),\n define_trilinos_enable(\"Zoltan\"),\n define_trilinos_enable(\"Zoltan2\"),\n define_tpl_enable(\"Cholmod\", False),\n define_from_variant(\"EpetraExt_BUILD_BTF\", \"epetraextbtf\"),\n define_from_variant(\"EpetraExt_BUILD_EXPERIMENTAL\", \"epetraextexperimental\"),\n define_from_variant(\n \"EpetraExt_BUILD_GRAPH_REORDERINGS\", \"epetraextgraphreorderings\"\n ),\n define_from_variant(\"Amesos2_ENABLE_Basker\", \"basker\"),\n ]\n )\n\n if \"+dtk\" in spec:\n options.extend(\n [\n define(\"Trilinos_EXTRA_REPOSITORIES\", \"DataTransferKit\"),\n define_trilinos_enable(\"DataTransferKit\", True),\n ]\n )\n\n if \"+exodus\" in spec:\n options.extend(\n [\n define_trilinos_enable(\"SEACAS\", True),\n define_trilinos_enable(\"SEACASExodus\", True),\n define_trilinos_enable(\"SEACASIoss\", True),\n define_trilinos_enable(\"SEACASEpu\", True),\n define_trilinos_enable(\"SEACASExodiff\", True),\n define_trilinos_enable(\"SEACASNemspread\", True),\n define_trilinos_enable(\"SEACASNemslice\", True),\n ]\n )\n else:\n options.extend(\n [\n define_trilinos_enable(\"SEACASExodus\", False),\n define_trilinos_enable(\"SEACASIoss\", False),\n ]\n )\n\n if \"+chaco\" in spec:\n options.extend(\n [\n define_trilinos_enable(\"SEACAS\", True),\n define_trilinos_enable(\"SEACASChaco\", True),\n ]\n )\n else:\n # don't disable SEACAS, could be needed elsewhere\n options.extend(\n [\n define_trilinos_enable(\"SEACASChaco\", False),\n define_trilinos_enable(\"SEACASNemslice\", False),\n ]\n )\n\n if \"+stratimikos\" in spec:\n # Explicitly enable Thyra (ThyraCore is required). If you don't do\n # this, then you get \"NOT setting ${pkg}_ENABLE_Thyra=ON since\n # Thyra is NOT enabled at this point!\" leading to eventual build\n # errors if using MueLu because `Xpetra_ENABLE_Thyra` is set to\n # off.\n options.append(define_trilinos_enable(\"Thyra\", True))\n\n # Add thyra adapters based on package enables\n options.extend(\n define_trilinos_enable(\"Thyra\" + pkg + \"Adapters\", pkg.lower())\n for pkg in [\"Epetra\", \"EpetraExt\", \"Tpetra\"]\n )\n\n # ######################### TPLs #############################\n\n def define_tpl(trilinos_name, spack_name, have_dep):\n options.append(define(\"TPL_ENABLE_\" + trilinos_name, have_dep))\n if not have_dep:\n return\n depspec = spec[spack_name]\n libs = depspec.libs\n try:\n options.extend(\n [define(trilinos_name + \"_INCLUDE_DIRS\", depspec.headers.directories)]\n )\n except NoHeadersError:\n # Handle case were depspec does not have headers\n pass\n\n options.extend(\n [\n define(trilinos_name + \"_ROOT\", depspec.prefix),\n define(trilinos_name + \"_LIBRARY_NAMES\", libs.names),\n define(trilinos_name + \"_LIBRARY_DIRS\", libs.directories),\n ]\n )\n\n # Enable these TPLs explicitly from variant options.\n # Format is (TPL name, variant name, Spack spec name)\n tpl_variant_map = [\n (\"ADIOS2\", \"adios2\", \"adios2\"),\n (\"Boost\", \"boost\", \"boost\"),\n (\"CUDA\", \"cuda\", \"cuda\"),\n (\"HDF5\", \"hdf5\", \"hdf5\"),\n (\"HYPRE\", \"hypre\", \"hypre\"),\n (\"MUMPS\", \"mumps\", \"mumps\"),\n (\"UMFPACK\", \"suite-sparse\", \"suite-sparse\"),\n (\"SuperLU\", \"superlu\", \"superlu\"),\n (\"SuperLUDist\", \"superlu-dist\", \"superlu-dist\"),\n (\"X11\", \"x11\", \"libx11\"),\n ]\n if spec.satisfies(\"@13.0.2:\"):\n tpl_variant_map.append((\"STRUMPACK\", \"strumpack\", \"strumpack\"))\n\n for tpl_name, var_name, spec_name in tpl_variant_map:\n define_tpl(tpl_name, spec_name, spec.variants[var_name].value)\n\n # Enable these TPLs based on whether they're in our spec; prefer to\n # require this way so that packages/features disable availability\n tpl_dep_map = [\n (\"BLAS\", \"blas\"),\n (\"CGNS\", \"cgns\"),\n (\"LAPACK\", \"lapack\"),\n (\"Matio\", \"matio\"),\n (\"METIS\", \"metis\"),\n (\"Netcdf\", \"netcdf-c\"),\n (\"SCALAPACK\", \"scalapack\"),\n (\"Zlib\", \"zlib\"),\n ]\n if spec.satisfies(\"@12.12.1:\"):\n tpl_dep_map.append((\"Pnetcdf\", \"parallel-netcdf\"))\n if spec.satisfies(\"@13:\"):\n tpl_dep_map.append((\"HWLOC\", \"hwloc\"))\n\n for tpl_name, dep_name in tpl_dep_map:\n define_tpl(tpl_name, dep_name, dep_name in spec)\n\n # MPI settings\n options.append(define_tpl_enable(\"MPI\"))\n if \"+mpi\" in spec:\n # Force Trilinos to use the MPI wrappers instead of raw compilers\n # to propagate library link flags for linkers that require fully\n # resolved symbols in shared libs (such as macOS and some newer\n # Ubuntu)\n options.extend(\n [\n define(\"CMAKE_C_COMPILER\", spec[\"mpi\"].mpicc),\n define(\"CMAKE_CXX_COMPILER\", spec[\"mpi\"].mpicxx),\n define(\"CMAKE_Fortran_COMPILER\", spec[\"mpi\"].mpifc),\n define(\"MPI_BASE_DIR\", spec[\"mpi\"].prefix),\n ]\n )\n\n # ParMETIS dependencies have to be transitive explicitly\n have_parmetis = \"parmetis\" in spec\n options.append(define_tpl_enable(\"ParMETIS\", have_parmetis))\n if have_parmetis:\n options.extend(\n [\n define(\n \"ParMETIS_LIBRARY_DIRS\",\n [spec[\"parmetis\"].prefix.lib, spec[\"metis\"].prefix.lib],\n ),\n define(\"ParMETIS_LIBRARY_NAMES\", [\"parmetis\", \"metis\"]),\n define(\n \"TPL_ParMETIS_INCLUDE_DIRS\",\n spec[\"parmetis\"].headers.directories + spec[\"metis\"].headers.directories,\n ),\n ]\n )\n\n if spec.satisfies(\"^superlu-dist@4.0:\"):\n options.extend([define(\"HAVE_SUPERLUDIST_LUSTRUCTINIT_2ARG\", True)])\n\n if spec.satisfies(\"^parallel-netcdf\"):\n options.extend(\n [\n define(\"TPL_Netcdf_Enables_Netcdf4\", True),\n define(\"TPL_Netcdf_PARALLEL\", True),\n define(\"PNetCDF_ROOT\", spec[\"parallel-netcdf\"].prefix),\n ]\n )\n\n # ################# Explicit template instantiation #################\n\n complex_s = spec.variants[\"complex\"].value\n float_s = spec.variants[\"float\"].value\n\n options.extend(\n [define(\"Teuchos_ENABLE_COMPLEX\", complex_s), define(\"Teuchos_ENABLE_FLOAT\", float_s)]\n )\n\n if \"+tpetra +explicit_template_instantiation\" in spec:\n options.append(define_from_variant(\"Tpetra_INST_OPENMP\", \"openmp\"))\n options.extend(\n [\n define(\"Tpetra_INST_DOUBLE\", True),\n define(\"Tpetra_INST_COMPLEX_DOUBLE\", complex_s),\n define(\"Tpetra_INST_COMPLEX_FLOAT\", float_s and complex_s),\n define(\"Tpetra_INST_FLOAT\", float_s),\n define(\"Tpetra_INST_SERIAL\", True),\n ]\n )\n\n gotype = spec.variants[\"gotype\"].value\n if gotype == \"all\":\n # default in older Trilinos versions to enable multiple GOs\n options.extend(\n [\n define(\"Tpetra_INST_INT_INT\", True),\n define(\"Tpetra_INST_INT_LONG\", True),\n define(\"Tpetra_INST_INT_LONG_LONG\", True),\n ]\n )\n else:\n options.extend(\n [\n define(\"Tpetra_INST_INT_INT\", gotype == \"int\"),\n define(\"Tpetra_INST_INT_LONG\", gotype == \"long\"),\n define(\"Tpetra_INST_INT_LONG_LONG\", gotype == \"long_long\"),\n ]\n )\n\n # ################# Kokkos ######################\n\n if \"+kokkos\" in spec:\n arch = Kokkos.get_microarch(spec.target)\n if arch:\n options.append(define(\"Kokkos_ARCH_\" + arch.upper(), True))\n\n define_kok_enable = _make_definer(\"Kokkos_ENABLE_\")\n options.extend(\n [\n define_kok_enable(\"CUDA\"),\n define_kok_enable(\"OPENMP\" if spec.version >= Version(\"13\") else \"OpenMP\"),\n ]\n )\n if \"+cuda\" in spec:\n options.extend(\n [\n define_kok_enable(\"CUDA_UVM\", True),\n define_kok_enable(\"CUDA_LAMBDA\", True),\n define_kok_enable(\"CUDA_RELOCATABLE_DEVICE_CODE\", \"cuda_rdc\"),\n ]\n )\n arch_map = Kokkos.spack_cuda_arch_map\n options.extend(\n define(\"Kokkos_ARCH_\" + arch_map[arch].upper(), True)\n for arch in spec.variants[\"cuda_arch\"].value\n )\n\n # ################# System-specific ######################\n\n # Fortran lib (assumes clang is built with gfortran!)\n if \"+fortran\" in spec and spec.compiler.name in [\"gcc\", \"clang\", \"apple-clang\"]:\n fc = Executable(spec[\"mpi\"].mpifc) if (\"+mpi\" in spec) else Executable(spack_fc)\n libgfortran = fc(\"--print-file-name\", \"libgfortran.\" + dso_suffix, output=str).strip()\n # if libgfortran is equal to \"libgfortran.\" then\n # print-file-name failed, use static library instead\n if libgfortran == \"libgfortran.\" + dso_suffix:\n libgfortran = fc(\"--print-file-name\", \"libgfortran.a\", output=str).strip()\n # -L -lgfortran required for OSX\n # https://github.com/spack/spack/pull/25823#issuecomment-917231118\n options.append(\n define(\n \"Trilinos_EXTRA_LINK_FLAGS\", \"-L%s/ -lgfortran\" % os.path.dirname(libgfortran)\n )\n )\n\n if sys.platform == \"darwin\" and macos_version() >= Version(\"10.12\"):\n # use @rpath on Sierra due to limit of dynamic loader\n options.append(define(\"CMAKE_MACOSX_RPATH\", True))\n else:\n options.append(define(\"CMAKE_INSTALL_NAME_DIR\", self.prefix.lib))\n\n return options\n\n @run_after(\"install\")\n def filter_python(self):\n # When trilinos is built with Python, libpytrilinos is included\n # through cmake configure files. Namely, Trilinos_LIBRARIES in\n # TrilinosConfig.cmake contains pytrilinos. This leads to a\n # run-time error: Symbol not found: _PyBool_Type and prevents\n # Trilinos to be used in any C++ code, which links executable\n # against the libraries listed in Trilinos_LIBRARIES. See\n # https://github.com/trilinos/Trilinos/issues/569 and\n # https://github.com/trilinos/Trilinos/issues/866\n # A workaround is to remove PyTrilinos from the COMPONENTS_LIST\n # and to remove -lpytrilonos from Makefile.export.Trilinos\n if \"+python\" in self.spec:\n filter_file(\n r\"(SET\\(COMPONENTS_LIST.*)(PyTrilinos;)(.*)\",\n (r\"\\1\\3\"),\n \"%s/cmake/Trilinos/TrilinosConfig.cmake\" % self.prefix.lib,\n )\n filter_file(r\"-lpytrilinos\", \"\", \"%s/Makefile.export.Trilinos\" % self.prefix.include)\n\n def setup_run_environment(self, env: EnvironmentModifications) -> None:\n if \"+exodus\" in self.spec:\n env.prepend_path(\"PYTHONPATH\", self.prefix.lib)\n\n if \"+cuda\" in self.spec:\n # currently Trilinos doesn't perform the memory fence so\n # it relies on blocking CUDA kernel launch.\n env.set(\"CUDA_LAUNCH_BLOCKING\", \"1\")\n" }, { "path": "lib/spack/spack/test/data/web/1.html", "content": "\n \n This is page 1.\n \n \n list_depth=2 follows this.\n\n foo-1.0.0.tar.gz\n \n\n" }, { "path": "lib/spack/spack/test/data/web/2.html", "content": "\n \n This is page 2.\n \n \n list_depth=3 follows this.\n list_depth=3 follows this too.\n\n foo-2.0.0.tar.gz\n foo-2.0.0b2.tar.gz\n \n\n" }, { "path": "lib/spack/spack/test/data/web/3.html", "content": "\n \n This is page 3.\n \n \n This link is already visited.\n\n foo-3.0.tar.gz\n foo-3.0a1.tar.gz\n \n\n" }, { "path": "lib/spack/spack/test/data/web/4.html", "content": "\n \n This is page 4.\n \n \n This page is terminal and has no links to other pages.\n\n foo-4.5.tar.gz.\n foo-4.1-rc5.tar.gz.\n foo-4.5.0.tar.gz.\n \n\n" }, { "path": "lib/spack/spack/test/data/web/fragment.html", "content": "foo-5.0.0.tar.gz\n" }, { "path": "lib/spack/spack/test/data/web/index.html", "content": "\n \n This is the root page.\n \n \n list_depth=1 follows this.\n\n foo-0.0.0.tar.gz\n \n\n" }, { "path": "lib/spack/spack/test/data/web/index_with_fragment.html", "content": "\n \n This is the root page.\n \n \n This is a page with an include-fragment element.\n\n \n \n

Loading...

\n \n \n\n" }, { "path": "lib/spack/spack/test/data/web/index_with_javascript.html", "content": "\n \n This is the root page.\n \n \n This is a page with a Vue javascript drop down with links as used in GitLab.\n\n
\n \n\n" }, { "path": "lib/spack/spack/test/database.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Check the database is functioning properly, both in memory and in its file.\"\"\"\n\nimport contextlib\nimport datetime\nimport functools\nimport gzip\nimport json\nimport os\nimport pathlib\nimport re\nimport shutil\nimport sys\n\nimport pytest\n\nimport spack.config\nimport spack.subprocess_context\nfrom spack.directory_layout import DirectoryLayoutError\n\ntry:\n import uuid\n\n _use_uuid = True\nexcept ImportError:\n _use_uuid = False\n\nimport spack.vendor.jsonschema\n\nimport spack.concretize\nimport spack.database\nimport spack.deptypes as dt\nimport spack.llnl.util.filesystem as fs\nimport spack.llnl.util.lock as lk\nimport spack.package_base\nimport spack.paths\nimport spack.repo\nimport spack.spec\nimport spack.store\nimport spack.util.lock\nimport spack.version as vn\nfrom spack.enums import InstallRecordStatus\nfrom spack.installer import PackageInstaller\nfrom spack.llnl.util.tty.colify import colify\nfrom spack.schema.database_index import schema\nfrom spack.test.conftest import RepoBuilder\nfrom spack.util.executable import Executable\n\npytestmark = pytest.mark.db\n\n\n@contextlib.contextmanager\ndef writable(database):\n \"\"\"Allow a database to be written inside this context manager.\"\"\"\n old_lock, old_is_upstream = database.lock, database.is_upstream\n db_root = pathlib.Path(database.root)\n\n try:\n # this is safe on all platforms during tests (tests get their own tmpdirs)\n database.lock = spack.util.lock.Lock(str(database._lock_path), enable=False)\n database.is_upstream = False\n db_root.chmod(mode=0o755)\n with database.write_transaction():\n yield\n finally:\n db_root.chmod(mode=0o555)\n database.lock = old_lock\n database.is_upstream = old_is_upstream\n\n\n@pytest.fixture()\ndef upstream_and_downstream_db(tmp_path: pathlib.Path, gen_mock_layout):\n \"\"\"Fixture for a pair of stores: upstream and downstream.\n\n Upstream API prohibits writing to an upstream, so we also return a writable version\n of the upstream DB for tests to use.\n\n \"\"\"\n mock_db_root = tmp_path / \"mock_db_root\"\n mock_db_root.mkdir()\n mock_db_root.chmod(0o555)\n\n upstream_db = spack.database.Database(\n str(mock_db_root), is_upstream=True, layout=gen_mock_layout(\"a\")\n )\n with writable(upstream_db):\n upstream_db._write()\n\n downstream_db_root = tmp_path / \"mock_downstream_db_root\"\n downstream_db_root.mkdir()\n downstream_db_root.chmod(0o755)\n\n downstream_db = spack.database.Database(\n str(downstream_db_root), upstream_dbs=[upstream_db], layout=gen_mock_layout(\"b\")\n )\n downstream_db._write()\n\n yield upstream_db, downstream_db\n\n\n@pytest.mark.parametrize(\n \"install_tree,result\",\n [\n (\"all\", [\"pkg-b\", \"pkg-c\", \"gcc-runtime\", \"gcc\", \"compiler-wrapper\"]),\n (\"upstream\", [\"pkg-c\"]),\n (\"local\", [\"pkg-b\", \"gcc-runtime\", \"gcc\", \"compiler-wrapper\"]),\n (\"{u}\", [\"pkg-c\"]),\n (\"{d}\", [\"pkg-b\", \"gcc-runtime\", \"gcc\", \"compiler-wrapper\"]),\n ],\n ids=[\"all\", \"upstream\", \"local\", \"upstream_path\", \"downstream_path\"],\n)\ndef test_query_by_install_tree(\n install_tree, result, upstream_and_downstream_db, mock_packages, monkeypatch, config\n):\n up_db, down_db = upstream_and_downstream_db\n\n # Set the upstream DB to contain \"pkg-c\" and downstream to contain \"pkg-b\")\n b = spack.concretize.concretize_one(\"pkg-b\")\n c = spack.concretize.concretize_one(\"pkg-c\")\n with writable(up_db):\n up_db.add(c)\n up_db._read()\n down_db.add(b)\n\n specs = down_db.query(install_tree=install_tree.format(u=up_db.root, d=down_db.root))\n assert {s.name for s in specs} == set(result)\n\n\ndef test_spec_installed_upstream(\n upstream_and_downstream_db, mock_custom_repository, config, monkeypatch\n):\n \"\"\"Test whether Spec.installed_upstream() works.\"\"\"\n upstream_db, downstream_db = upstream_and_downstream_db\n\n # a known installed spec should say that it's installed\n with spack.repo.use_repositories(mock_custom_repository):\n spec = spack.concretize.concretize_one(\"pkg-c\")\n assert not spec.installed\n assert not spec.installed_upstream\n\n with writable(upstream_db):\n upstream_db.add(spec)\n upstream_db._read()\n\n monkeypatch.setattr(spack.store.STORE, \"db\", downstream_db)\n assert spec.installed\n assert spec.installed_upstream\n assert spec.copy().installed\n\n # an abstract spec should say it's not installed\n spec = spack.spec.Spec(\"not-a-real-package\")\n assert not spec.installed\n assert not spec.installed_upstream\n\n\n@pytest.mark.usefixtures(\"config\")\ndef test_installed_upstream(upstream_and_downstream_db, repo_builder: RepoBuilder):\n upstream_db, downstream_db = upstream_and_downstream_db\n\n repo_builder.add_package(\"x\")\n repo_builder.add_package(\"z\")\n repo_builder.add_package(\"y\", dependencies=[(\"z\", None, None)])\n repo_builder.add_package(\"w\", dependencies=[(\"x\", None, None), (\"y\", None, None)])\n\n with spack.repo.use_repositories(repo_builder.root):\n spec = spack.concretize.concretize_one(\"w\")\n with writable(upstream_db):\n for dep in spec.traverse(root=False):\n upstream_db.add(dep)\n upstream_db._read()\n\n for dep in spec.traverse(root=False):\n record = downstream_db.get_by_hash(dep.dag_hash())\n assert record is not None\n with pytest.raises(spack.database.ForbiddenLockError):\n upstream_db.get_by_hash(dep.dag_hash())\n\n new_spec = spack.concretize.concretize_one(\"w\")\n downstream_db.add(new_spec)\n for dep in new_spec.traverse(root=False):\n upstream, record = downstream_db.query_by_spec_hash(dep.dag_hash())\n assert upstream\n assert record.path == upstream_db.layout.path_for_spec(dep)\n upstream, record = downstream_db.query_by_spec_hash(new_spec.dag_hash())\n assert not upstream\n assert record.installed\n\n upstream_db._check_ref_counts()\n downstream_db._check_ref_counts()\n\n\ndef test_missing_upstream_build_dep(\n upstream_and_downstream_db,\n tmp_path: pathlib.Path,\n monkeypatch,\n config,\n repo_builder: RepoBuilder,\n):\n upstream_db, downstream_db = upstream_and_downstream_db\n\n z_y_prefix = str(tmp_path / \"z-y\")\n\n def fail_for_z(spec):\n if spec.prefix == z_y_prefix:\n raise DirectoryLayoutError(\"Fake layout error for z\")\n\n upstream_db.layout.ensure_installed = fail_for_z\n\n repo_builder.add_package(\"z\")\n repo_builder.add_package(\"y\", dependencies=[(\"z\", \"build\", None)])\n\n monkeypatch.setattr(spack.store.STORE, \"db\", downstream_db)\n\n with spack.repo.use_repositories(repo_builder.root):\n y = spack.concretize.concretize_one(\"y\")\n z_y = y[\"z\"]\n z_y.set_prefix(z_y_prefix)\n\n with writable(upstream_db):\n upstream_db.add(y)\n upstream_db._read()\n\n upstream, record = downstream_db.query_by_spec_hash(z_y.dag_hash())\n assert upstream\n assert not record.installed\n\n assert y.installed\n assert y.installed_upstream\n\n assert not z_y.installed\n assert not z_y.installed_upstream\n\n # Now add z to downstream with non-triggering prefix\n # and make sure z *is* installed\n\n z_new = z_y.copy()\n z_new.set_prefix(str(tmp_path / \"z-new\"))\n downstream_db.add(z_new)\n\n assert z_new.installed\n assert not z_new.installed_upstream\n\n\ndef test_removed_upstream_dep(\n upstream_and_downstream_db, capfd, config, repo_builder: RepoBuilder\n):\n upstream_db, downstream_db = upstream_and_downstream_db\n\n repo_builder.add_package(\"z\")\n repo_builder.add_package(\"y\", dependencies=[(\"z\", None, None)])\n\n with spack.repo.use_repositories(repo_builder.root):\n y = spack.concretize.concretize_one(\"y\")\n z = y[\"z\"]\n\n # add dependency to upstream, dependents to downstream\n with writable(upstream_db):\n upstream_db.add(z)\n upstream_db._read()\n downstream_db.add(y)\n\n # remove the dependency from the upstream DB\n with writable(upstream_db):\n upstream_db.remove(z)\n upstream_db._read()\n\n # then rereading the downstream DB should warn about the missing dep\n downstream_db._read_from_file(downstream_db._index_path)\n assert (\n f\"Missing dependency not in database: y/{y.dag_hash(7)} needs z\"\n in capfd.readouterr().err\n )\n\n\n@pytest.mark.usefixtures(\"config\")\ndef test_add_to_upstream_after_downstream(upstream_and_downstream_db, repo_builder: RepoBuilder):\n \"\"\"An upstream DB can add a package after it is installed in the downstream\n DB. When a package is recorded as installed in both, the results should\n refer to the downstream DB.\n \"\"\"\n upstream_db, downstream_db = upstream_and_downstream_db\n\n repo_builder.add_package(\"x\")\n\n with spack.repo.use_repositories(repo_builder.root):\n spec = spack.concretize.concretize_one(\"x\")\n\n downstream_db.add(spec)\n with writable(upstream_db):\n upstream_db.add(spec)\n upstream_db._read()\n\n upstream, record = downstream_db.query_by_spec_hash(spec.dag_hash())\n # Even though the package is recorded as installed in the upstream DB,\n # we prefer the locally-installed instance\n assert not upstream\n\n qresults = downstream_db.query(\"x\")\n assert len(qresults) == 1\n (queried_spec,) = qresults\n try:\n orig_db = spack.store.STORE.db\n spack.store.STORE.db = downstream_db\n assert queried_spec.prefix == downstream_db.layout.path_for_spec(spec)\n finally:\n spack.store.STORE.db = orig_db\n\n\ndef test_cannot_write_upstream(tmp_path: pathlib.Path, mock_packages, config):\n # Instantiate the database that will be used as the upstream DB and make\n # sure it has an index file\n with spack.database.Database(str(tmp_path)).write_transaction():\n pass\n\n # Create it as an upstream\n db = spack.database.Database(str(tmp_path), is_upstream=True)\n\n with pytest.raises(spack.database.ForbiddenLockError):\n db.add(spack.concretize.concretize_one(\"pkg-a\"))\n\n\n@pytest.mark.usefixtures(\"config\", \"temporary_store\")\ndef test_recursive_upstream_dbs(\n tmp_path: pathlib.Path, gen_mock_layout, repo_builder: RepoBuilder\n):\n roots = [str(tmp_path / x) for x in [\"a\", \"b\", \"c\"]]\n for root in roots:\n pathlib.Path(root).mkdir(parents=True, exist_ok=True)\n layouts = [gen_mock_layout(x) for x in [\"ra\", \"rb\", \"rc\"]]\n\n repo_builder.add_package(\"z\")\n repo_builder.add_package(\"y\", dependencies=[(\"z\", None, None)])\n repo_builder.add_package(\"x\", dependencies=[(\"y\", None, None)])\n\n with spack.repo.use_repositories(repo_builder.root):\n spec = spack.concretize.concretize_one(\"x\")\n db_c = spack.database.Database(roots[2], layout=layouts[2])\n db_c.add(spec[\"z\"])\n\n db_b = spack.database.Database(roots[1], upstream_dbs=[db_c], layout=layouts[1])\n db_b.add(spec[\"y\"])\n\n db_a = spack.database.Database(roots[0], upstream_dbs=[db_b, db_c], layout=layouts[0])\n db_a.add(spec[\"x\"])\n\n upstream_dbs_from_scratch = spack.store._construct_upstream_dbs_from_install_roots(\n [roots[1], roots[2]]\n )\n db_a_from_scratch = spack.database.Database(\n roots[0], upstream_dbs=upstream_dbs_from_scratch\n )\n\n assert db_a_from_scratch.db_for_spec_hash(spec.dag_hash()) == (db_a_from_scratch)\n assert (\n db_a_from_scratch.db_for_spec_hash(spec[\"y\"].dag_hash())\n == (upstream_dbs_from_scratch[0])\n )\n assert (\n db_a_from_scratch.db_for_spec_hash(spec[\"z\"].dag_hash())\n == (upstream_dbs_from_scratch[1])\n )\n\n db_a_from_scratch._check_ref_counts()\n upstream_dbs_from_scratch[0]._check_ref_counts()\n upstream_dbs_from_scratch[1]._check_ref_counts()\n\n assert db_a_from_scratch.installed_relatives(spec) == set(spec.traverse(root=False))\n assert db_a_from_scratch.installed_relatives(spec[\"z\"], direction=\"parents\") == set(\n [spec, spec[\"y\"]]\n )\n\n\n@pytest.fixture()\ndef usr_folder_exists(monkeypatch):\n \"\"\"The ``/usr`` folder is assumed to be existing in some tests. This\n fixture makes it such that its existence is mocked, so we have no\n requirements on the system running tests.\n \"\"\"\n isdir = os.path.isdir\n\n @functools.wraps(os.path.isdir)\n def mock_isdir(path):\n if path == \"/usr\":\n return True\n return isdir(path)\n\n monkeypatch.setattr(os.path, \"isdir\", mock_isdir)\n\n\ndef _print_ref_counts():\n \"\"\"Print out all ref counts for the graph used here, for debugging\"\"\"\n recs = []\n\n def add_rec(spec):\n cspecs = spack.store.STORE.db.query(spec, installed=InstallRecordStatus.ANY)\n\n if not cspecs:\n recs.append(\"[ %-7s ] %-20s-\" % (\"\", spec))\n else:\n key = cspecs[0].dag_hash()\n rec = spack.store.STORE.db.get_record(cspecs[0])\n recs.append(\"[ %-7s ] %-20s%d\" % (key[:7], spec, rec.ref_count))\n\n with spack.store.STORE.db.read_transaction():\n add_rec(\"mpileaks ^mpich\")\n add_rec(\"callpath ^mpich\")\n add_rec(\"mpich\")\n\n add_rec(\"mpileaks ^mpich2\")\n add_rec(\"callpath ^mpich2\")\n add_rec(\"mpich2\")\n\n add_rec(\"mpileaks ^zmpi\")\n add_rec(\"callpath ^zmpi\")\n add_rec(\"zmpi\")\n add_rec(\"fake\")\n\n add_rec(\"dyninst\")\n add_rec(\"libdwarf\")\n add_rec(\"libelf\")\n\n colify(recs, cols=3)\n\n\ndef _check_merkleiness():\n \"\"\"Ensure the spack database is a valid merkle graph.\"\"\"\n all_specs = spack.store.STORE.db.query(installed=InstallRecordStatus.ANY)\n\n seen = {}\n for spec in all_specs:\n for dep in spec.dependencies():\n hash_key = dep.dag_hash()\n if hash_key not in seen:\n seen[hash_key] = id(dep)\n else:\n assert seen[hash_key] == id(dep)\n\n\ndef _check_db_sanity(database):\n \"\"\"Utility function to check db against install layout.\"\"\"\n pkg_in_layout = sorted(spack.store.STORE.layout.all_specs())\n actual = sorted(database.query())\n\n externals = sorted([x for x in actual if x.external])\n nexpected = len(pkg_in_layout) + len(externals)\n\n assert nexpected == len(actual)\n\n non_external_in_db = sorted([x for x in actual if not x.external])\n\n for e, a in zip(pkg_in_layout, non_external_in_db):\n assert e == a\n\n _check_merkleiness()\n\n\ndef _check_remove_and_add_package(database: spack.database.Database, spec):\n \"\"\"Remove a spec from the DB, then add it and make sure everything's\n still ok once it is added. This checks that it was\n removed, that it's back when added again, and that ref\n counts are consistent.\n \"\"\"\n original = database.query()\n database._check_ref_counts()\n\n # Remove spec\n concrete_spec = database.remove(spec)\n database._check_ref_counts()\n remaining = database.query()\n\n # ensure spec we removed is gone\n assert len(original) - 1 == len(remaining)\n assert all(s in original for s in remaining)\n assert concrete_spec not in remaining\n\n # add it back and make sure everything is ok.\n database.add(concrete_spec)\n installed = database.query()\n assert concrete_spec in installed\n assert installed == original\n\n # sanity check against directory layout and check ref counts.\n _check_db_sanity(database)\n database._check_ref_counts()\n\n\ndef _mock_install(spec: str):\n s = spack.concretize.concretize_one(spec)\n PackageInstaller([s.package], fake=True, explicit=True).install()\n\n\ndef _mock_remove(spec):\n specs = spack.store.STORE.db.query(spec)\n assert len(specs) == 1\n spec = specs[0]\n spec.package.do_uninstall(spec)\n\n\ndef test_default_queries(database):\n # Testing a package whose name *doesn't* start with 'lib'\n # to ensure the library has 'lib' prepended to the name\n rec = database.get_record(\"zmpi\")\n\n spec = rec.spec\n\n libraries = spec[\"zmpi\"].libs\n assert len(libraries) == 1\n assert libraries.names[0] == \"zmpi\"\n\n headers = spec[\"zmpi\"].headers\n assert len(headers) == 1\n assert headers.names[0] == \"zmpi\"\n\n command = spec[\"zmpi\"].command\n assert isinstance(command, Executable)\n assert command.name == \"zmpi\"\n assert os.path.exists(command.path)\n\n # Testing a package whose name *does* start with 'lib'\n # to ensure the library doesn't have a double 'lib' prefix\n rec = database.get_record(\"libelf\")\n\n spec = rec.spec\n\n libraries = spec[\"libelf\"].libs\n assert len(libraries) == 1\n assert libraries.names[0] == \"elf\"\n\n headers = spec[\"libelf\"].headers\n assert len(headers) == 1\n assert headers.names[0] == \"libelf\"\n\n command = spec[\"libelf\"].command\n assert isinstance(command, Executable)\n assert command.name == \"libelf\"\n assert os.path.exists(command.path)\n\n\ndef test_005_db_exists(database):\n \"\"\"Make sure db cache file exists after creating.\"\"\"\n index_file = os.path.join(database.root, \".spack-db\", spack.database.INDEX_JSON_FILE)\n lock_file = os.path.join(database.root, \".spack-db\", spack.database._LOCK_FILE)\n assert os.path.exists(str(index_file))\n # Lockfiles not currently supported on Windows\n if sys.platform != \"win32\":\n assert os.path.exists(str(lock_file))\n\n with open(index_file, encoding=\"utf-8\") as fd:\n index_object = json.load(fd)\n spack.vendor.jsonschema.validate(index_object, schema)\n\n\ndef test_010_all_install_sanity(database):\n \"\"\"Ensure that the install layout reflects what we think it does.\"\"\"\n all_specs = spack.store.STORE.layout.all_specs()\n assert len(all_specs) == 17\n\n # Query specs with multiple configurations\n mpileaks_specs = [s for s in all_specs if s.satisfies(\"mpileaks\")]\n callpath_specs = [s for s in all_specs if s.satisfies(\"callpath\")]\n mpi_specs = [s for s in all_specs if s.satisfies(\"mpi\")]\n\n assert len(mpileaks_specs) == 3\n assert len(callpath_specs) == 3\n assert len(mpi_specs) == 3\n\n # Query specs with single configurations\n dyninst_specs = [s for s in all_specs if s.satisfies(\"dyninst\")]\n libdwarf_specs = [s for s in all_specs if s.satisfies(\"libdwarf\")]\n libelf_specs = [s for s in all_specs if s.satisfies(\"libelf\")]\n\n assert len(dyninst_specs) == 1\n assert len(libdwarf_specs) == 1\n assert len(libelf_specs) == 1\n\n # Query by dependency\n assert len([s for s in all_specs if s.satisfies(\"mpileaks ^mpich\")]) == 1\n assert len([s for s in all_specs if s.satisfies(\"mpileaks ^mpich2\")]) == 1\n assert len([s for s in all_specs if s.satisfies(\"mpileaks ^zmpi\")]) == 1\n\n\ndef test_015_write_and_read(mutable_database):\n # write and read DB\n with spack.store.STORE.db.write_transaction():\n specs = spack.store.STORE.db.query()\n recs = [spack.store.STORE.db.get_record(s) for s in specs]\n\n for spec, rec in zip(specs, recs):\n new_rec = spack.store.STORE.db.get_record(spec)\n assert new_rec.ref_count == rec.ref_count\n assert new_rec.spec == rec.spec\n assert new_rec.path == rec.path\n assert new_rec.installed == rec.installed\n\n\ndef test_016_roundtrip_spliced_spec(mutable_database):\n build_spec = spack.concretize.concretize_one(\"splice-t\")\n replacement = spack.concretize.concretize_one(\"splice-h+foo\")\n spec = build_spec.splice(replacement)\n\n spack.store.STORE.db.add(spec)\n spack.store.STORE.db._state_is_inconsistent = True # force re-read\n\n _, spec_record = spack.store.STORE.db.query_by_spec_hash(spec.dag_hash())\n _, buildspec_record = spack.store.STORE.db.query_by_spec_hash(spec.build_spec.dag_hash())\n\n assert spec_record.spec == spec\n assert spec_record.spec.build_spec == spec.build_spec\n assert buildspec_record # buildspec needs to be recorded in db\n\n\ndef test_017_write_and_read_without_uuid(mutable_database, monkeypatch):\n monkeypatch.setattr(spack.database, \"_use_uuid\", False)\n # write and read DB\n with spack.store.STORE.db.write_transaction():\n specs = spack.store.STORE.db.query()\n recs = [spack.store.STORE.db.get_record(s) for s in specs]\n\n for spec, rec in zip(specs, recs):\n new_rec = spack.store.STORE.db.get_record(spec)\n assert new_rec.ref_count == rec.ref_count\n assert new_rec.spec == rec.spec\n assert new_rec.path == rec.path\n assert new_rec.installed == rec.installed\n\n\ndef test_020_db_sanity(database):\n \"\"\"Make sure query() returns what's actually in the db.\"\"\"\n _check_db_sanity(database)\n\n\ndef test_025_reindex(mutable_database):\n \"\"\"Make sure reindex works and ref counts are valid.\"\"\"\n spack.store.STORE.reindex()\n _check_db_sanity(mutable_database)\n\n\ndef test_026_reindex_after_deprecate(mutable_database):\n \"\"\"Make sure reindex works and ref counts are valid after deprecation.\"\"\"\n mpich = mutable_database.query_one(\"mpich\")\n zmpi = mutable_database.query_one(\"zmpi\")\n mutable_database.deprecate(mpich, zmpi)\n\n spack.store.STORE.reindex()\n _check_db_sanity(mutable_database)\n\n\nclass ReadModify:\n \"\"\"Provide a function which can execute in a separate process that removes\n a spec from the database.\n \"\"\"\n\n def __call__(self):\n # check that other process can read DB\n _check_db_sanity(spack.store.STORE.db)\n with spack.store.STORE.db.write_transaction():\n _mock_remove(\"mpileaks ^zmpi\")\n\n\ndef test_030_db_sanity_from_another_process(mutable_database):\n spack_process = spack.subprocess_context.SpackTestProcess(ReadModify())\n p = spack_process.create()\n p.start()\n p.join()\n\n # ensure child process change is visible in parent process\n with mutable_database.read_transaction():\n assert len(mutable_database.query(\"mpileaks ^zmpi\")) == 0\n\n\ndef test_040_ref_counts(database):\n \"\"\"Ensure that we got ref counts right when we read the DB.\"\"\"\n database._check_ref_counts()\n\n\ndef test_041_ref_counts_deprecate(mutable_database):\n \"\"\"Ensure that we have appropriate ref counts after deprecating\"\"\"\n mpich = mutable_database.query_one(\"mpich\")\n zmpi = mutable_database.query_one(\"zmpi\")\n\n mutable_database.deprecate(mpich, zmpi)\n mutable_database._check_ref_counts()\n\n\ndef test_050_basic_query(database):\n \"\"\"Ensure querying database is consistent with what is installed.\"\"\"\n # query everything\n total_specs = len(spack.store.STORE.db.query())\n assert total_specs == 20\n\n # query specs with multiple configurations\n mpileaks_specs = database.query(\"mpileaks\")\n callpath_specs = database.query(\"callpath\")\n mpi_specs = database.query(\"mpi\")\n\n assert len(mpileaks_specs) == 3\n assert len(callpath_specs) == 3\n assert len(mpi_specs) == 3\n\n # query specs with single configurations\n dyninst_specs = database.query(\"dyninst\")\n libdwarf_specs = database.query(\"libdwarf\")\n libelf_specs = database.query(\"libelf\")\n\n assert len(dyninst_specs) == 1\n assert len(libdwarf_specs) == 1\n assert len(libelf_specs) == 1\n\n # Query by dependency\n assert len(database.query(\"mpileaks ^mpich\")) == 1\n assert len(database.query(\"mpileaks ^mpich2\")) == 1\n assert len(database.query(\"mpileaks ^zmpi\")) == 1\n\n # Query by date\n assert len(database.query(start_date=datetime.datetime.min)) == total_specs\n assert len(database.query(start_date=datetime.datetime.max)) == 0\n assert len(database.query(end_date=datetime.datetime.min)) == 0\n assert len(database.query(end_date=datetime.datetime.max)) == total_specs\n\n\ndef test_060_remove_and_add_root_package(mutable_database):\n _check_remove_and_add_package(mutable_database, \"mpileaks ^mpich\")\n\n\ndef test_070_remove_and_add_dependency_package(mutable_database):\n _check_remove_and_add_package(mutable_database, \"dyninst\")\n\n\ndef test_080_root_ref_counts(mutable_database):\n rec = mutable_database.get_record(\"mpileaks ^mpich\")\n\n # Remove a top-level spec from the DB\n mutable_database.remove(\"mpileaks ^mpich\")\n\n # record no longer in DB\n assert mutable_database.query(\"mpileaks ^mpich\", installed=InstallRecordStatus.ANY) == []\n\n # record's deps have updated ref_counts\n assert mutable_database.get_record(\"callpath ^mpich\").ref_count == 0\n assert mutable_database.get_record(\"mpich\").ref_count == 1\n\n # Put the spec back\n mutable_database.add(rec.spec)\n\n # record is present again\n assert len(mutable_database.query(\"mpileaks ^mpich\", installed=InstallRecordStatus.ANY)) == 1\n\n # dependencies have ref counts updated\n assert mutable_database.get_record(\"callpath ^mpich\").ref_count == 1\n assert mutable_database.get_record(\"mpich\").ref_count == 2\n\n\ndef test_090_non_root_ref_counts(mutable_database):\n mutable_database.get_record(\"mpileaks ^mpich\")\n mutable_database.get_record(\"callpath ^mpich\")\n\n # \"force remove\" a non-root spec from the DB\n mutable_database.remove(\"callpath ^mpich\")\n\n # record still in DB but marked uninstalled\n assert mutable_database.query(\"callpath ^mpich\", installed=True) == []\n assert len(mutable_database.query(\"callpath ^mpich\", installed=InstallRecordStatus.ANY)) == 1\n\n # record and its deps have same ref_counts\n assert (\n mutable_database.get_record(\"callpath ^mpich\", installed=InstallRecordStatus.ANY).ref_count\n == 1\n )\n assert mutable_database.get_record(\"mpich\").ref_count == 2\n\n # remove only dependent of uninstalled callpath record\n mutable_database.remove(\"mpileaks ^mpich\")\n\n # record and parent are completely gone.\n assert mutable_database.query(\"mpileaks ^mpich\", installed=InstallRecordStatus.ANY) == []\n assert mutable_database.query(\"callpath ^mpich\", installed=InstallRecordStatus.ANY) == []\n\n # mpich ref count updated properly.\n mpich_rec = mutable_database.get_record(\"mpich\")\n assert mpich_rec.ref_count == 0\n\n\ndef test_100_no_write_with_exception_on_remove(database):\n def fail_while_writing():\n with database.write_transaction():\n _mock_remove(\"mpileaks ^zmpi\")\n raise Exception()\n\n with database.read_transaction():\n assert len(database.query(\"mpileaks ^zmpi\", installed=InstallRecordStatus.ANY)) == 1\n\n with pytest.raises(Exception):\n fail_while_writing()\n\n # reload DB and make sure zmpi is still there.\n with database.read_transaction():\n assert len(database.query(\"mpileaks ^zmpi\", installed=InstallRecordStatus.ANY)) == 1\n\n\ndef test_110_no_write_with_exception_on_install(database):\n def fail_while_writing():\n with database.write_transaction():\n _mock_install(\"cmake\")\n raise Exception()\n\n with database.read_transaction():\n assert database.query(\"cmake\", installed=InstallRecordStatus.ANY) == []\n\n with pytest.raises(Exception):\n fail_while_writing()\n\n # reload DB and make sure cmake was not written.\n with database.read_transaction():\n assert database.query(\"cmake\", installed=InstallRecordStatus.ANY) == []\n\n\ndef test_115_reindex_with_packages_not_in_repo(mutable_database, repo_builder: RepoBuilder):\n # Dont add any package definitions to this repository, the idea is that\n # packages should not have to be defined in the repository once they\n # are installed\n with spack.repo.use_repositories(repo_builder.root):\n spack.store.STORE.reindex()\n _check_db_sanity(mutable_database)\n\n\ndef test_external_entries_in_db(mutable_database):\n rec = mutable_database.get_record(\"mpileaks ^zmpi\")\n assert rec.spec.external_path is None\n assert not rec.spec.external_modules\n\n rec = mutable_database.get_record(\"externaltool\")\n assert rec.spec.external_path == os.path.sep + os.path.join(\"path\", \"to\", \"external_tool\")\n assert not rec.spec.external_modules\n assert rec.explicit is False\n\n PackageInstaller([rec.spec.package], fake=True, explicit=True).install()\n rec = mutable_database.get_record(\"externaltool\")\n assert rec.spec.external_path == os.path.sep + os.path.join(\"path\", \"to\", \"external_tool\")\n assert not rec.spec.external_modules\n assert rec.explicit is True\n\n\n@pytest.mark.regression(\"8036\")\ndef test_regression_issue_8036(mutable_database, usr_folder_exists):\n # The test ensures that the external package prefix is treated as\n # existing. Even when the package prefix exists, the package should\n # not be considered installed until it is added to the database by\n # the installer with install().\n s = spack.concretize.concretize_one(\"externaltool@0.9\")\n assert not s.installed\n\n # Now install the external package and check again the `installed` property\n PackageInstaller([s.package], fake=True, explicit=True).install()\n assert s.installed\n\n\n@pytest.mark.regression(\"11118\")\ndef test_old_external_entries_prefix(mutable_database: spack.database.Database):\n with open(spack.store.STORE.db._index_path, \"r\", encoding=\"utf-8\") as f:\n db_obj = json.loads(f.read())\n\n spack.vendor.jsonschema.validate(db_obj, schema)\n\n s, *_ = mutable_database.query(\"externaltool\")\n\n db_obj[\"database\"][\"installs\"][s.dag_hash()][\"path\"] = \"None\"\n\n with open(spack.store.STORE.db._index_path, \"w\", encoding=\"utf-8\") as f:\n f.write(json.dumps(db_obj))\n if _use_uuid:\n with open(spack.store.STORE.db._verifier_path, \"w\", encoding=\"utf-8\") as f:\n f.write(str(uuid.uuid4()))\n\n record = spack.store.STORE.db.get_record(s)\n assert record is not None\n assert record.path is None\n assert record.spec._prefix is None\n assert record.spec.prefix == record.spec.external_path\n\n\ndef test_uninstall_by_spec(mutable_database):\n with mutable_database.write_transaction():\n for spec in mutable_database.query():\n if spec.installed:\n spack.package_base.PackageBase.uninstall_by_spec(spec, force=True)\n else:\n mutable_database.remove(spec)\n assert len(mutable_database.query()) == 0\n\n\ndef test_query_unused_specs(mutable_database):\n # This spec installs a fake cmake as a build only dependency\n s = spack.concretize.concretize_one(\"simple-inheritance\")\n PackageInstaller([s.package], fake=True, explicit=True).install()\n\n si = s.dag_hash()\n ml_mpich = spack.store.STORE.db.query_one(\"mpileaks ^mpich\").dag_hash()\n ml_mpich2 = spack.store.STORE.db.query_one(\"mpileaks ^mpich2\").dag_hash()\n ml_zmpi = spack.store.STORE.db.query_one(\"mpileaks ^zmpi\").dag_hash()\n externaltest = spack.store.STORE.db.query_one(\"externaltest\").dag_hash()\n trivial_smoke_test = spack.store.STORE.db.query_one(\"trivial-smoke-test\").dag_hash()\n\n def check_unused(roots, deptype, expected):\n unused = spack.store.STORE.db.unused_specs(root_hashes=roots, deptype=deptype)\n assert set(u.name for u in unused) == set(expected)\n\n default_dt = dt.LINK | dt.RUN\n check_unused(None, default_dt, [\"cmake\", \"gcc\", \"compiler-wrapper\"])\n check_unused(\n [si, ml_mpich, ml_mpich2, ml_zmpi, externaltest],\n default_dt,\n [\"trivial-smoke-test\", \"cmake\", \"gcc\", \"compiler-wrapper\"],\n )\n check_unused(\n [si, ml_mpich, ml_mpich2, ml_zmpi, externaltest],\n dt.LINK | dt.RUN | dt.BUILD,\n [\"trivial-smoke-test\"],\n )\n check_unused(\n [si, ml_mpich, ml_mpich2, externaltest, trivial_smoke_test],\n dt.LINK | dt.RUN | dt.BUILD,\n [\"mpileaks\", \"callpath\", \"zmpi\", \"fake\"],\n )\n check_unused(\n [si, ml_mpich, ml_mpich2, ml_zmpi],\n default_dt,\n [\n \"trivial-smoke-test\",\n \"cmake\",\n \"externaltest\",\n \"externaltool\",\n \"externalvirtual\",\n \"gcc\",\n \"compiler-wrapper\",\n ],\n )\n\n\n@pytest.mark.regression(\"10019\")\ndef test_query_spec_with_conditional_dependency(mutable_database):\n # The issue is triggered by having dependencies that are\n # conditional on a Boolean variant\n s = spack.concretize.concretize_one(\"hdf5~mpi\")\n PackageInstaller([s.package], fake=True, explicit=True).install()\n\n results = spack.store.STORE.db.query_local(\"hdf5 ^mpich\")\n assert not results\n\n\n@pytest.mark.regression(\"10019\")\ndef test_query_spec_with_non_conditional_virtual_dependency(database):\n # Ensure the same issue doesn't come up for virtual\n # dependency that are not conditional on variants\n results = spack.store.STORE.db.query_local(\"mpileaks ^mpich\")\n assert len(results) == 1\n\n\ndef test_query_virtual_spec(database):\n \"\"\"Make sure we can query for virtuals in the DB\"\"\"\n results = spack.store.STORE.db.query_local(\"mpi\")\n assert len(results) == 3\n names = [s.name for s in results]\n assert all(name in names for name in [\"mpich\", \"mpich2\", \"zmpi\"])\n\n\ndef test_failed_spec_path_error(mutable_database):\n \"\"\"Ensure spec not concrete check is covered.\"\"\"\n s = spack.spec.Spec(\"pkg-a\")\n with pytest.raises(AssertionError, match=\"concrete spec required\"):\n spack.store.STORE.failure_tracker.mark(s)\n\n\n@pytest.mark.db\ndef test_clear_failure_keep(mutable_database, monkeypatch, capfd):\n \"\"\"Add test coverage for clear_failure operation when to be retained.\"\"\"\n\n def _is(self, spec):\n return True\n\n # Pretend the spec has been failure locked\n monkeypatch.setattr(spack.database.FailureTracker, \"lock_taken\", _is)\n\n s = spack.concretize.concretize_one(\"pkg-a\")\n spack.store.STORE.failure_tracker.clear(s)\n out = capfd.readouterr()[0]\n assert \"Retaining failure marking\" in out\n\n\n@pytest.mark.db\ndef test_clear_failure_forced(mutable_database, monkeypatch, capfd):\n \"\"\"Add test coverage for clear_failure operation when force.\"\"\"\n\n def _is(self, spec):\n return True\n\n # Pretend the spec has been failure locked\n monkeypatch.setattr(spack.database.FailureTracker, \"lock_taken\", _is)\n # Ensure raise OSError when try to remove the non-existent marking\n monkeypatch.setattr(spack.database.FailureTracker, \"persistent_mark\", _is)\n\n s = spack.concretize.concretize_one(\"pkg-a\")\n spack.store.STORE.failure_tracker.clear(s, force=True)\n out = capfd.readouterr()[1]\n assert \"Removing failure marking despite lock\" in out\n assert \"Unable to remove failure marking\" in out\n\n\n@pytest.mark.db\ndef test_mark_failed(mutable_database, monkeypatch, tmp_path: pathlib.Path, capfd):\n \"\"\"Add coverage to mark_failed.\"\"\"\n\n def _raise_exc(lock):\n raise lk.LockTimeoutError(lk.LockType.WRITE, \"/mock-lock\", 1.234, 10)\n\n with fs.working_dir(str(tmp_path)):\n s = spack.concretize.concretize_one(\"pkg-a\")\n\n # Ensure attempt to acquire write lock on the mark raises the exception\n monkeypatch.setattr(lk.Lock, \"acquire_write\", _raise_exc)\n\n spack.store.STORE.failure_tracker.mark(s)\n out = str(capfd.readouterr()[1])\n assert \"Unable to mark pkg-a as failed\" in out\n\n spack.store.STORE.failure_tracker.clear_all()\n\n\n@pytest.mark.db\ndef test_prefix_failed(mutable_database, monkeypatch):\n \"\"\"Add coverage to failed operation.\"\"\"\n\n s = spack.concretize.concretize_one(\"pkg-a\")\n\n # Confirm the spec is not already marked as failed\n assert not spack.store.STORE.failure_tracker.has_failed(s)\n\n # Check that a failure entry is sufficient\n spack.store.STORE.failure_tracker.mark(s)\n assert spack.store.STORE.failure_tracker.has_failed(s)\n\n # Remove the entry and check again\n spack.store.STORE.failure_tracker.clear(s)\n assert not spack.store.STORE.failure_tracker.has_failed(s)\n\n # Now pretend that the prefix failure is locked\n monkeypatch.setattr(spack.database.FailureTracker, \"lock_taken\", lambda self, spec: True)\n assert spack.store.STORE.failure_tracker.has_failed(s)\n\n\ndef test_prefix_write_lock_error(mutable_database, monkeypatch):\n \"\"\"Cover the prefix write lock exception.\"\"\"\n\n def _raise(db, spec):\n raise lk.LockError(\"Mock lock error\")\n\n s = spack.concretize.concretize_one(\"pkg-a\")\n\n # Ensure subsequent lock operations fail\n monkeypatch.setattr(lk.Lock, \"acquire_write\", _raise)\n\n with pytest.raises(Exception):\n with spack.store.STORE.prefix_locker.write_lock(s):\n assert False\n\n\n@pytest.mark.regression(\"26600\")\ndef test_database_works_with_empty_dir(tmp_path: pathlib.Path):\n # Create the lockfile and failures directory otherwise\n # we'll get a permission error on Database creation\n db_dir = tmp_path / \".spack-db\"\n db_dir.mkdir()\n (db_dir / spack.database._LOCK_FILE).touch()\n (db_dir / \"failures\").mkdir()\n tmp_path.chmod(mode=0o555)\n db = spack.database.Database(str(tmp_path))\n with db.read_transaction():\n db.query()\n # Check that reading an empty directory didn't create a new index.json\n assert not os.path.exists(db._index_path)\n\n\n@pytest.mark.parametrize(\n \"query_arg,exc_type,msg_str\",\n [\n ([\"callpath\"], spack.store.MatchError, \"matches multiple packages\"),\n ([\"tensorflow\"], spack.store.MatchError, \"does not match any\"),\n ],\n)\ndef test_store_find_failures(database, query_arg, exc_type, msg_str):\n with pytest.raises(exc_type) as exc_info:\n spack.store.find(query_arg, multiple=False)\n assert msg_str in str(exc_info.value)\n\n\ndef test_store_find_accept_string(database):\n result = spack.store.find(\"callpath\", multiple=True)\n assert len(result) == 3\n\n\ndef test_reindex_removed_prefix_is_not_installed(mutable_database, mock_store, capfd):\n \"\"\"When a prefix of a dependency is removed and the database is reindexed,\n the spec should still be added through the dependent, but should be listed as\n not installed.\"\"\"\n\n # Remove libelf from the filesystem\n prefix = mutable_database.query_one(\"libelf\").prefix\n assert prefix.startswith(str(mock_store))\n shutil.rmtree(prefix)\n\n # Reindex should pick up libelf as a dependency of libdwarf\n spack.store.STORE.reindex()\n\n # Reindexing should warn about libelf not found on the filesystem\n assert re.search(\n \"libelf@0.8.13.+ was marked installed in the database \"\n \"but was not found on the file system\",\n capfd.readouterr().err,\n )\n\n # And we should still have libelf in the database, but not installed.\n assert not mutable_database.query_one(\"libelf\", installed=True)\n assert mutable_database.query_one(\"libelf\", installed=False)\n\n\ndef test_reindex_when_all_prefixes_are_removed(mutable_database, mock_store):\n # Remove all non-external installations from the filesystem\n for spec in spack.store.STORE.db.query_local():\n if not spec.external:\n assert spec.prefix.startswith(str(mock_store))\n shutil.rmtree(spec.prefix)\n\n # Make sure we have some explicitly installed specs\n num = len(mutable_database.query_local(installed=True, explicit=True))\n assert num > 0\n\n # Reindex uses the current index to repopulate itself\n spack.store.STORE.reindex()\n\n # Make sure all explicit specs are still there, but are now uninstalled.\n specs = mutable_database.query_local(installed=False, explicit=True)\n assert len(specs) == num\n\n # And make sure they can be removed from the database (covers the case where\n # `ref_count == 0 and not installed`, which hits some obscure branches.\n for s in specs:\n mutable_database.remove(s)\n\n assert len(mutable_database.query_local(installed=False, explicit=True)) == 0\n\n\n@pytest.mark.parametrize(\n \"spec_str,parent_name,expected_nparents\",\n [(\"dyninst\", \"callpath\", 3), (\"libelf\", \"dyninst\", 1), (\"libelf\", \"libdwarf\", 1)],\n)\n@pytest.mark.regression(\"11983\")\ndef test_check_parents(spec_str, parent_name, expected_nparents, database):\n \"\"\"Check that a spec returns the correct number of parents.\"\"\"\n s = database.query_one(spec_str)\n\n parents = s.dependents(name=parent_name)\n assert len(parents) == expected_nparents\n\n edges = s.edges_from_dependents(name=parent_name)\n assert len(edges) == expected_nparents\n\n\ndef test_db_all_hashes(database):\n # ensure we get the right number of hashes without a read transaction\n hashes = database.all_hashes()\n assert len(hashes) == 20\n\n # and make sure the hashes match\n with database.read_transaction():\n assert set(s.dag_hash() for s in database.query()) == set(hashes)\n\n\ndef test_consistency_of_dependents_upon_remove(mutable_database):\n # Check the initial state\n s = mutable_database.query_one(\"dyninst\")\n parents = s.dependents(name=\"callpath\")\n assert len(parents) == 3\n\n # Remove a dependent (and all its dependents)\n mutable_database.remove(\"mpileaks ^callpath ^mpich2\")\n mutable_database.remove(\"callpath ^mpich2\")\n\n # Check the final state\n s = mutable_database.query_one(\"dyninst\")\n parents = s.dependents(name=\"callpath\")\n assert len(parents) == 2\n\n\n@pytest.mark.regression(\"30187\")\ndef test_query_installed_when_package_unknown(database, repo_builder: RepoBuilder):\n \"\"\"Test that we can query the installation status of a spec\n when we don't know its package.py\n \"\"\"\n with spack.repo.use_repositories(repo_builder.root):\n specs = database.query(\"mpileaks\")\n for s in specs:\n # Assert that we can query the installation methods even though we\n # don't have the package.py available\n assert s.installed\n assert not s.installed_upstream\n with pytest.raises(spack.repo.UnknownNamespaceError):\n s.package\n\n\ndef test_error_message_when_using_too_new_db(database, monkeypatch):\n \"\"\"Sometimes the database format needs to be bumped. When that happens, we have forward\n incompatibilities that need to be reported in a clear way to the user, in case we moved\n back to an older version of Spack. This test ensures that the error message for a too\n new database version stays comprehensible across refactoring of the database code.\n \"\"\"\n monkeypatch.setattr(spack.database, \"_DB_VERSION\", vn.Version(\"0\"))\n with pytest.raises(\n spack.database.InvalidDatabaseVersionError, match=\"you need a newer Spack version\"\n ):\n spack.database.Database(database.root)._read()\n\n\n@pytest.mark.parametrize(\n \"lock_cfg\",\n [spack.database.NO_LOCK, spack.database.NO_TIMEOUT, spack.database.DEFAULT_LOCK_CFG, None],\n)\ndef test_database_construction_doesnt_use_globals(\n tmp_path: pathlib.Path, config, nullify_globals, lock_cfg\n):\n lock_cfg = lock_cfg or spack.database.lock_configuration(config)\n db = spack.database.Database(str(tmp_path), lock_cfg=lock_cfg)\n with db.write_transaction():\n pass # ensure the DB is written\n assert os.path.exists(db.database_directory)\n\n\ndef test_database_read_works_with_trailing_data(\n tmp_path: pathlib.Path, default_mock_concretization\n):\n # Populate a database\n root = str(tmp_path)\n db = spack.database.Database(root, layout=None)\n spec = default_mock_concretization(\"pkg-a\")\n db.add(spec)\n specs_in_db = db.query_local()\n assert spec in specs_in_db\n\n # Append anything to the end of the database file\n with open(db._index_path, \"a\", encoding=\"utf-8\") as f:\n f.write(json.dumps({\"hello\": \"world\"}))\n\n # Read the database and check that it ignores the trailing data\n assert spack.database.Database(root).query_local() == specs_in_db\n\n\ndef test_database_errors_with_just_a_version_key(mutable_database):\n next_version = f\"{spack.database._DB_VERSION}.next\"\n with open(mutable_database._index_path, \"w\", encoding=\"utf-8\") as f:\n f.write(json.dumps({\"database\": {\"version\": next_version}}))\n\n with pytest.raises(spack.database.InvalidDatabaseVersionError):\n spack.database.Database(mutable_database.root).query_local()\n\n\ndef test_reindex_with_upstreams(tmp_path: pathlib.Path, monkeypatch, mock_packages, config):\n # Reindexing should not put install records of upstream entries into the local database. Here\n # we install `mpileaks` locally with dependencies in the upstream. And we even install\n # `mpileaks` with the same hash in the upstream. After reindexing, `mpileaks` should still be\n # in the local db, and `callpath` should not.\n mpileaks = spack.concretize.concretize_one(\"mpileaks\")\n callpath = mpileaks.dependencies(\"callpath\")[0]\n\n upstream_store = spack.store.create(\n spack.config.create_from(\n spack.config.InternalConfigScope(\n \"cfg\", {\"config\": {\"install_tree\": {\"root\": str(tmp_path / \"upstream\")}}}\n )\n )\n )\n\n monkeypatch.setattr(spack.store, \"STORE\", upstream_store)\n PackageInstaller([callpath.package], fake=True, explicit=True).install()\n\n local_store = spack.store.create(\n spack.config.create_from(\n spack.config.InternalConfigScope(\n \"cfg\",\n {\n \"config\": {\"install_tree\": {\"root\": str(tmp_path / \"local\")}},\n \"upstreams\": {\"my-upstream\": {\"install_tree\": str(tmp_path / \"upstream\")}},\n },\n )\n )\n )\n monkeypatch.setattr(spack.store, \"STORE\", local_store)\n PackageInstaller([mpileaks.package], fake=True, explicit=True).install()\n\n # Sanity check that callpath is from upstream.\n assert not local_store.db.query_local(\"callpath\")\n assert local_store.db.query(\"callpath\")\n\n # Install mpileaks also upstream with the same hash to ensure that determining upstreamness\n # checks local installs before upstream databases, even when the local database is being\n # reindexed.\n monkeypatch.setattr(spack.store, \"STORE\", upstream_store)\n PackageInstaller([mpileaks.package], fake=True, explicit=True).install()\n\n # Delete the local database\n shutil.rmtree(local_store.db.database_directory)\n\n # Create a new instance s.t. we don't have cached specs in memory\n reindexed_local_store = spack.store.create(\n spack.config.create_from(\n spack.config.InternalConfigScope(\n \"cfg\",\n {\n \"config\": {\"install_tree\": {\"root\": str(tmp_path / \"local\")}},\n \"upstreams\": {\"my-upstream\": {\"install_tree\": str(tmp_path / \"upstream\")}},\n },\n )\n )\n )\n reindexed_local_store.db.reindex()\n\n assert not reindexed_local_store.db.query_local(\"callpath\")\n assert reindexed_local_store.db.query(\"callpath\") == [callpath]\n assert reindexed_local_store.db.query_local(\"mpileaks\") == [mpileaks]\n\n\n@pytest.mark.regression(\"47101\")\ndef test_query_with_predicate_fn(database):\n all_specs = database.query()\n\n # Name starts with a string\n specs = database.query(predicate_fn=lambda x: x.spec.name.startswith(\"mpil\"))\n assert specs and all(x.name.startswith(\"mpil\") for x in specs)\n assert len(specs) < len(all_specs)\n\n # Recipe is currently known/unknown\n specs = database.query(predicate_fn=lambda x: spack.repo.PATH.exists(x.spec.name))\n assert specs == all_specs\n\n specs = database.query(predicate_fn=lambda x: not spack.repo.PATH.exists(x.spec.name))\n assert not specs\n\n\n@pytest.mark.regression(\"49964\")\ndef test_querying_reindexed_database_specfilev5(tmp_path: pathlib.Path):\n \"\"\"Tests that we can query a reindexed database from before compilers as dependencies,\n and get appropriate results for % and similar selections.\n \"\"\"\n test_path = pathlib.Path(spack.paths.test_path)\n zipfile = test_path / \"data\" / \"database\" / \"index.json.v7_v8.json.gz\"\n with gzip.open(str(zipfile), \"rt\", encoding=\"utf-8\") as f:\n data = json.load(f)\n\n index_json = tmp_path / spack.database._DB_DIRNAME / spack.database.INDEX_JSON_FILE\n index_json.parent.mkdir(parents=True)\n index_json.write_text(json.dumps(data))\n\n db = spack.database.Database(str(tmp_path))\n\n specs = db.query(\"%gcc\")\n\n assert len(specs) == 8\n assert len([x for x in specs if x.external]) == 2\n assert len([x for x in specs if x.original_spec_format() < 5]) == 8\n" }, { "path": "lib/spack/spack/test/detection.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport collections\nimport pathlib\n\nimport pytest\n\nimport spack.config\nimport spack.detection\nimport spack.detection.common\nimport spack.detection.path\nimport spack.repo\nimport spack.spec\n\n\ndef test_detection_update_config(mutable_config):\n # mock detected package\n detected_packages = collections.defaultdict(list)\n detected_packages[\"cmake\"] = [spack.spec.Spec(\"cmake@3.27.5\", external_path=\"/usr/bin\")]\n\n # update config for new package\n spack.detection.common.update_configuration(detected_packages)\n # Check entries in 'packages.yaml'\n packages_yaml = spack.config.get(\"packages\")\n assert \"cmake\" in packages_yaml\n assert \"externals\" in packages_yaml[\"cmake\"]\n externals = packages_yaml[\"cmake\"][\"externals\"]\n assert len(externals) == 1\n external_gcc = externals[0]\n assert external_gcc[\"spec\"] == \"cmake@3.27.5\"\n assert external_gcc[\"prefix\"] == \"/usr/bin\"\n\n\ndef test_dedupe_paths(tmp_path: pathlib.Path):\n \"\"\"Test that ``dedupe_paths`` deals with symlinked directories, retaining the target\"\"\"\n x = tmp_path / \"x\"\n y = tmp_path / \"y\"\n z = tmp_path / \"z\"\n\n x.mkdir()\n y.mkdir()\n z.symlink_to(\"x\", target_is_directory=True)\n\n # dedupe repeated dirs, should preserve order\n assert spack.detection.path.dedupe_paths([str(x), str(y), str(x)]) == [str(x), str(y)]\n assert spack.detection.path.dedupe_paths([str(y), str(x), str(y)]) == [str(y), str(x)]\n\n # dedupe repeated symlinks\n assert spack.detection.path.dedupe_paths([str(z), str(y), str(z)]) == [str(z), str(y)]\n assert spack.detection.path.dedupe_paths([str(y), str(z), str(y)]) == [str(y), str(z)]\n\n # when both symlink and target are present, only target is retained, and it comes at the\n # priority of the first occurrence.\n assert spack.detection.path.dedupe_paths([str(x), str(y), str(z)]) == [str(x), str(y)]\n assert spack.detection.path.dedupe_paths([str(z), str(y), str(x)]) == [str(x), str(y)]\n assert spack.detection.path.dedupe_paths([str(y), str(z), str(x)]) == [str(y), str(x)]\n\n\n@pytest.mark.usefixtures(\"mock_packages\")\ndef test_detect_specs_deduplicates_across_prefixes(tmp_path, monkeypatch):\n \"\"\"Tests that the same spec detected at two different prefixes should yield only one result.\n\n Returning both causes duplicate externals in packages.yaml and non-deterministic hashes\n during concretization.\n \"\"\"\n # Create two independent bin/ directories, each containing the same executable name.\n prefix_a = tmp_path / \"prefix_a\"\n prefix_b = tmp_path / \"prefix_b\"\n (prefix_a / \"bin\").mkdir(parents=True)\n (prefix_b / \"bin\").mkdir(parents=True)\n exe_a = prefix_a / \"bin\" / \"cmake\"\n exe_b = prefix_b / \"bin\" / \"cmake\"\n exe_a.touch()\n exe_b.touch()\n\n cmake_cls = spack.repo.PATH.get_pkg_class(\"cmake\")\n\n # Patch determine_spec_details to always return the same spec, regardless of prefix.\n @classmethod\n def _same_spec(cls, prefix, exes_in_prefix):\n return spack.spec.Spec(\"cmake@3.17.1\")\n\n monkeypatch.setattr(cmake_cls, \"determine_spec_details\", _same_spec)\n\n finder = spack.detection.path.ExecutablesFinder()\n detected = finder.detect_specs(\n pkg=cmake_cls, paths=[str(exe_a), str(exe_b)], repo_path=spack.repo.PATH\n )\n\n # Both prefixes produce cmake@3.17.1; only the first should be kept.\n assert len(detected) == 1\n" }, { "path": "lib/spack/spack/test/directives.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nfrom collections import namedtuple\n\nimport pytest\n\nimport spack.concretize\nimport spack.directives\nimport spack.repo\nimport spack.spec\nimport spack.version\nfrom spack.directives import _make_when_spec, depends_on, extends, patch\nfrom spack.directives_meta import DirectiveDictDescriptor, DirectiveMeta\nfrom spack.spec import Spec\n\n\ndef test_false_directives_do_not_exist(mock_packages):\n \"\"\"Ensure directives that evaluate to False at import time are added to\n dicts on packages.\n \"\"\"\n cls = spack.repo.PATH.get_pkg_class(\"when-directives-false\")\n assert not cls.dependencies\n assert not cls.resources\n assert not cls.patches\n\n\ndef test_true_directives_exist(mock_packages):\n \"\"\"Ensure directives that evaluate to True at import time are added to\n dicts on packages.\n \"\"\"\n cls = spack.repo.PATH.get_pkg_class(\"when-directives-true\")\n\n assert cls.dependencies\n assert \"extendee\" in cls.dependencies[spack.spec.Spec()]\n assert \"pkg-b\" in cls.dependencies[spack.spec.Spec()]\n\n assert cls.resources\n assert spack.spec.Spec() in cls.resources\n\n assert cls.patches\n assert spack.spec.Spec() in cls.patches\n\n\ndef test_constraints_from_context(mock_packages):\n pkg_cls = spack.repo.PATH.get_pkg_class(\"with-constraint-met\")\n\n assert pkg_cls.dependencies\n assert \"pkg-b\" in pkg_cls.dependencies[spack.spec.Spec(\"@1.0\")]\n\n assert pkg_cls.conflicts\n assert (spack.spec.Spec(\"%gcc\"), None) in pkg_cls.conflicts[spack.spec.Spec(\"+foo@1.0\")]\n\n\n@pytest.mark.regression(\"26656\")\ndef test_constraints_from_context_are_merged(mock_packages):\n pkg_cls = spack.repo.PATH.get_pkg_class(\"with-constraint-met\")\n\n assert pkg_cls.dependencies\n assert \"pkg-c\" in pkg_cls.dependencies[spack.spec.Spec(\"@0.14:15 ^pkg-b@3.8:4.0\")]\n\n\n@pytest.mark.regression(\"27754\")\ndef test_extends_spec(config, mock_packages):\n extender = spack.concretize.concretize_one(\"extends-spec\")\n extendee = spack.concretize.concretize_one(\"extendee\")\n\n assert extender.dependencies\n assert extender.package.extends(extendee)\n\n\n@pytest.mark.regression(\"48024\")\ndef test_conditionally_extends_transitive_dep(config, mock_packages):\n spec = spack.concretize.concretize_one(\"conditionally-extends-transitive-dep\")\n\n assert not spec.package.extendee_spec\n\n\n@pytest.mark.regression(\"48025\")\ndef test_conditionally_extends_direct_dep(config, mock_packages):\n spec = spack.concretize.concretize_one(\"conditionally-extends-direct-dep\")\n\n assert not spec.package.extendee_spec\n\n\n@pytest.mark.regression(\"34368\")\ndef test_error_on_anonymous_dependency(config, mock_packages):\n pkg = spack.repo.PATH.get_pkg_class(\"pkg-a\")\n with pytest.raises(spack.directives.DependencyError):\n spack.directives._execute_depends_on(pkg, spack.spec.Spec(\"@4.5\"))\n\n\n@pytest.mark.regression(\"34879\")\n@pytest.mark.parametrize(\n \"package_name,expected_maintainers\",\n [\n (\"maintainers-1\", [\"user1\", \"user2\"]),\n # Extends PythonPackage\n (\"py-extension1\", [\"user1\", \"user2\"]),\n # Extends maintainers-1\n (\"maintainers-3\", [\"user0\", \"user1\", \"user2\", \"user3\"]),\n ],\n)\ndef test_maintainer_directive(config, mock_packages, package_name, expected_maintainers):\n pkg_cls = spack.repo.PATH.get_pkg_class(package_name)\n assert pkg_cls.maintainers == expected_maintainers\n\n\n@pytest.mark.parametrize(\n \"package_name,expected_licenses\", [(\"licenses-1\", [(\"MIT\", \"+foo\"), (\"Apache-2.0\", \"~foo\")])]\n)\ndef test_license_directive(config, mock_packages, package_name, expected_licenses):\n pkg_cls = spack.repo.PATH.get_pkg_class(package_name)\n for license in expected_licenses:\n assert spack.spec.Spec(license[1]) in pkg_cls.licenses\n assert license[0] == pkg_cls.licenses[spack.spec.Spec(license[1])]\n\n\ndef test_duplicate_exact_range_license():\n package = namedtuple(\"package\", [\"licenses\", \"name\"])\n package.licenses = {spack.spec.Spec(\"+foo\"): \"Apache-2.0\"}\n package.name = \"test_package\"\n\n msg = (\n r\"test_package is specified as being licensed as MIT when \\+foo, but it is also \"\n r\"specified as being licensed under Apache-2.0 when \\+foo, which conflict.\"\n )\n\n with pytest.raises(spack.directives.OverlappingLicenseError, match=msg):\n spack.directives._execute_license(package, \"MIT\", \"+foo\")\n\n\ndef test_overlapping_duplicate_licenses():\n package = namedtuple(\"package\", [\"licenses\", \"name\"])\n package.licenses = {spack.spec.Spec(\"+foo\"): \"Apache-2.0\"}\n package.name = \"test_package\"\n\n msg = (\n r\"test_package is specified as being licensed as MIT when \\+bar, but it is also \"\n r\"specified as being licensed under Apache-2.0 when \\+foo, which conflict.\"\n )\n\n with pytest.raises(spack.directives.OverlappingLicenseError, match=msg):\n spack.directives._execute_license(package, \"MIT\", \"+bar\")\n\n\ndef test_version_type_validation():\n # A version should be a string or an int, not a float, because it leads to subtle issues\n # such as 3.10 being interpreted as 3.1.\n\n package = namedtuple(\"package\", [\"name\"])\n\n msg = r\"python: declared version '.+' in package should be a string or int\\.\"\n\n # Pass a float\n with pytest.raises(spack.version.VersionError, match=msg):\n spack.directives._execute_version(package(name=\"python\"), ver=3.10, kwargs={})\n\n # Try passing a bogus type; it's just that we want a nice error message\n with pytest.raises(spack.version.VersionError, match=msg):\n spack.directives._execute_version(package(name=\"python\"), ver={}, kwargs={})\n\n\n@pytest.mark.parametrize(\n \"spec_str,distribute_src,distribute_bin\",\n [\n (\"redistribute-x@1.1~foo\", False, False),\n (\"redistribute-x@1.2+foo\", False, False),\n (\"redistribute-x@1.2~foo\", False, True),\n (\"redistribute-x@1.0~foo\", False, True),\n (\"redistribute-x@1.3+foo\", True, True),\n (\"redistribute-y@2.0\", False, False),\n (\"redistribute-y@2.1+bar\", False, False),\n ],\n)\ndef test_redistribute_directive(mock_packages, spec_str, distribute_src, distribute_bin):\n spec = spack.spec.Spec(spec_str)\n assert spack.repo.PATH.get_pkg_class(spec.fullname).redistribute_source(spec) == distribute_src\n concretized_spec = spack.concretize.concretize_one(spec)\n assert concretized_spec.package.redistribute_binary == distribute_bin\n\n\ndef test_redistribute_override_when():\n \"\"\"Allow a user to call `redistribute` twice to separately disable\n source and binary distribution for the same when spec.\n\n The second call should not undo the effect of the first.\n \"\"\"\n\n class MockPackage:\n name = \"mock\"\n disable_redistribute = {}\n\n cls = MockPackage\n spack.directives._execute_redistribute(cls, source=False, binary=None, when=\"@1.0\")\n spec_key = spack.directives._make_when_spec(\"@1.0\")\n assert not cls.disable_redistribute[spec_key].binary\n assert cls.disable_redistribute[spec_key].source\n spack.directives._execute_redistribute(cls, source=None, binary=False, when=\"@1.0\")\n assert cls.disable_redistribute[spec_key].binary\n assert cls.disable_redistribute[spec_key].source\n\n\n@pytest.mark.regression(\"51248\")\ndef test_direct_dependencies_from_when_context_are_retained(mock_packages):\n \"\"\"Tests that direct dependencies from the \"when\" context manager don't lose the \"direct\"\n attribute when turned into directives on the package class.\n \"\"\"\n pkg_cls = spack.repo.PATH.get_pkg_class(\"with-constraint-met\")\n # Direct dependency in a \"when\" single context manager\n assert spack.spec.Spec(\"%pkg-b\") in pkg_cls.dependencies\n # Direct dependency in a \"when\" nested context manager\n assert spack.spec.Spec(\"@2 %c=gcc %pkg-c %pkg-b@:4.0\") in pkg_cls.dependencies\n # Nested ^foo followed by %foo\n assert spack.spec.Spec(\"%pkg-c\") in pkg_cls.dependencies\n # Nested ^foo followed by ^foo %gcc\n assert spack.spec.Spec(\"^pkg-c %gcc\") in pkg_cls.dependencies\n\n\ndef test_directives_meta_combine_when():\n x, y, z = \"+x ^dep +a\", \"+y ^dep +b\", \"+z\"\n assert _make_when_spec((x, y, z)) == Spec(\"+x +y +z ^dep +a +b\")\n assert _make_when_spec((x, y)) == Spec(\"+x +y ^dep +a +b\")\n assert _make_when_spec((x,)) == Spec(\"+x ^dep +a\")\n\n\ndef test_directive_descriptor_init():\n # when `pkg.variants` is initialized, only the `variant` directive should run\n variants = DirectiveDictDescriptor(\"variants\")\n assert variants.directives_to_run == [\"variant\"]\n assert variants.dicts_to_init == [\"variants\"]\n\n # when `pkg.dependencies` is initialized, `depends_on` and `extends` should run, and also\n # `pkg.extendees` should be initialized\n dependencies = DirectiveDictDescriptor(\"dependencies\")\n assert dependencies.directives_to_run == [\"depends_on\", \"extends\"]\n assert dependencies.dicts_to_init == [\"dependencies\", \"extendees\"]\n\n # when `pkg.provided` is initialized, so should `pkg.provided_together`, and only the\n # provides directive should run\n provided = DirectiveDictDescriptor(\"provided\")\n assert provided.directives_to_run == [\"provides\"]\n assert provided.dicts_to_init == [\"provided\", \"provided_together\"]\n\n # idem for `pkg.provided_together`\n provided_together = DirectiveDictDescriptor(\"provided_together\")\n assert provided_together.directives_to_run == [\"provides\"]\n assert provided_together.dicts_to_init == [\"provided\", \"provided_together\"]\n\n # when specifying patches on dependencies with `depends_on` and `extends`, the `pkg.patches`\n # dict is not affects -- they are stored on a Dependency object.\n patches = DirectiveDictDescriptor(\"patches\")\n assert patches.directives_to_run == [\"patch\"]\n assert patches.dicts_to_init == [\"patches\"]\n\n\ndef test_directive_laziness():\n class ExamplePackage(metaclass=DirectiveMeta):\n name = \"example-package\"\n depends_on(\"foo\")\n extends(\"bar\", when=\"+bar\")\n\n # Initially, no directive dicts are initialized\n assert ExamplePackage._dependencies is None # type: ignore\n assert ExamplePackage._extendees is None # type: ignore\n assert ExamplePackage._variants is None # type: ignore\n\n # Only when we access the dependencies descriptor, the relevant dicts (dependencies, extendees)\n # are initialized, while others remain None\n dependencies = ExamplePackage.dependencies # type: ignore\n assert type(ExamplePackage._dependencies) is dict # type: ignore\n assert type(ExamplePackage._extendees) is dict # type: ignore\n assert ExamplePackage._variants is None # type: ignore\n\n # The dependencies dict is populated with the expected entries\n assert \"foo\" in dependencies[spack.spec.Spec()]\n assert \"bar\" in dependencies[spack.spec.Spec(\"+bar\")]\n\n\ndef test_patched_dependencies_sets_class_attribute():\n sha256 = \"a\" * 64\n\n class PatchesDependencies(metaclass=DirectiveMeta):\n name = \"patches-dependencies\"\n depends_on(\"dependency\", patches=patch(\"https://example.com/diff.patch\", sha256=sha256))\n\n assert PatchesDependencies._patches_dependencies is True\n assert not PatchesDependencies.patches # type: ignore\n\n class DoesNotPatchDependencies(metaclass=DirectiveMeta):\n name = \"does-not-patch-dependencies\"\n fullname = \"does-not-patch-dependencies\"\n patch(\"https://example.com/diff.patch\", sha256=sha256)\n\n assert DoesNotPatchDependencies._patches_dependencies is False\n assert DoesNotPatchDependencies.patches # type: ignore\n" }, { "path": "lib/spack/spack/test/directory_layout.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"\nThis test verifies that the Spack directory layout works properly.\n\"\"\"\n\nimport os\nimport pathlib\nfrom pathlib import Path\n\nimport pytest\n\nimport spack.concretize\nimport spack.hash_types\nimport spack.paths\nimport spack.repo\nimport spack.util.file_cache\nfrom spack.directory_layout import DirectoryLayout, InvalidDirectoryLayoutParametersError\nfrom spack.llnl.path import path_to_os_path\nfrom spack.spec import Spec\n\n# number of packages to test (to reduce test time)\nmax_packages = 10\n\n\ndef test_yaml_directory_layout_parameters(tmp_path: pathlib.Path, default_mock_concretization):\n \"\"\"This tests the various parameters that can be used to configure\n the install location\"\"\"\n spec = default_mock_concretization(\"python\")\n\n # Ensure default layout matches expected spec format\n layout_default = DirectoryLayout(str(tmp_path))\n path_default = layout_default.relative_path_for_spec(spec)\n assert path_default == str(\n Path(spec.format(\"{architecture.platform}-{architecture.target}/{name}-{version}-{hash}\"))\n )\n\n # Test hash_length parameter works correctly\n layout_10 = DirectoryLayout(str(tmp_path), hash_length=10)\n path_10 = layout_10.relative_path_for_spec(spec)\n layout_7 = DirectoryLayout(str(tmp_path), hash_length=7)\n path_7 = layout_7.relative_path_for_spec(spec)\n\n assert len(path_default) - len(path_10) == 22\n assert len(path_default) - len(path_7) == 25\n\n # Test path_scheme\n arch, package7 = path_7.split(os.sep)\n projections_package7 = {\"all\": \"{name}-{version}-{hash:7}\"}\n layout_package7 = DirectoryLayout(str(tmp_path), projections=projections_package7)\n path_package7 = layout_package7.relative_path_for_spec(spec)\n\n assert package7 == path_package7\n\n # Test separation of architecture or namespace\n spec2 = spack.concretize.concretize_one(\"libelf\")\n\n arch_scheme = (\n \"{architecture.platform}/{architecture.target}/{architecture.os}/{name}/{version}/{hash:7}\"\n )\n ns_scheme = \"{architecture}/{namespace}/{name}-{version}-{hash:7}\"\n arch_ns_scheme_projections = {\"all\": arch_scheme, \"python\": ns_scheme}\n layout_arch_ns = DirectoryLayout(str(tmp_path), projections=arch_ns_scheme_projections)\n\n arch_path_spec2 = layout_arch_ns.relative_path_for_spec(spec2)\n assert arch_path_spec2 == str(Path(spec2.format(arch_scheme)))\n\n ns_path_spec = layout_arch_ns.relative_path_for_spec(spec)\n assert ns_path_spec == str(Path(spec.format(ns_scheme)))\n\n # Ensure conflicting parameters caught\n with pytest.raises(InvalidDirectoryLayoutParametersError):\n DirectoryLayout(str(tmp_path), hash_length=20, projections=projections_package7)\n\n\ndef test_read_and_write_spec(temporary_store, config, mock_packages):\n \"\"\"This goes through each package in spack and creates a directory for\n it. It then ensures that the spec for the directory's\n installed package can be read back in consistently, and\n finally that the directory can be removed by the directory\n layout.\n \"\"\"\n layout = temporary_store.layout\n pkg_names = list(spack.repo.PATH.all_package_names())[:max_packages]\n\n for name in pkg_names:\n if name.startswith(\"external\"):\n # External package tests cannot be installed\n continue\n\n # If a spec fails to concretize, just skip it. If it is a\n # real error, it will be caught by concretization tests.\n try:\n spec = spack.concretize.concretize_one(name)\n except Exception:\n continue\n\n layout.create_install_directory(spec)\n\n install_dir = path_to_os_path(layout.path_for_spec(spec))[0]\n spec_path = layout.spec_file_path(spec)\n\n # Ensure directory has been created in right place.\n assert os.path.isdir(install_dir)\n assert install_dir.startswith(temporary_store.root)\n\n # Ensure spec file exists when directory is created\n assert os.path.isfile(spec_path)\n assert spec_path.startswith(install_dir)\n\n # Make sure spec file can be read back in to get the original spec\n spec_from_file = layout.read_spec(spec_path)\n\n stored_deptypes = spack.hash_types.dag_hash\n expected = spec.copy(deps=stored_deptypes)\n expected._mark_concrete()\n\n assert expected.concrete\n assert expected == spec_from_file\n assert expected.eq_dag(spec_from_file)\n assert spec_from_file.concrete\n\n # Ensure that specs that come out \"normal\" are really normal.\n with open(spec_path, encoding=\"utf-8\") as spec_file:\n read_separately = Spec.from_yaml(spec_file.read())\n\n # TODO: revise this when build deps are in dag_hash\n norm = read_separately.copy(deps=stored_deptypes)\n assert norm == spec_from_file\n assert norm.eq_dag(spec_from_file)\n\n # TODO: revise this when build deps are in dag_hash\n conc = spack.concretize.concretize_one(read_separately).copy(deps=stored_deptypes)\n assert conc == spec_from_file\n assert conc.eq_dag(spec_from_file)\n\n assert expected.dag_hash() == spec_from_file.dag_hash()\n\n # Ensure directories are properly removed\n layout.remove_install_directory(spec)\n assert not os.path.isdir(install_dir)\n assert not os.path.exists(install_dir)\n\n\ndef test_handle_unknown_package(temporary_store, config, mock_packages, tmp_path: pathlib.Path):\n \"\"\"This test ensures that spack can at least do *some*\n operations with packages that are installed but that it\n does not know about. This is actually not such an uncommon\n scenario with spack; it can happen when you switch from a\n git branch where you're working on a new package.\n\n This test ensures that the directory layout stores enough\n information about installed packages' specs to uninstall\n or query them again if the package goes away.\n \"\"\"\n layout = temporary_store.layout\n\n repo_cache = spack.util.file_cache.FileCache(tmp_path / \"cache\")\n mock_db = spack.repo.Repo(spack.paths.mock_packages_path, cache=repo_cache)\n\n not_in_mock = set.difference(\n set(spack.repo.all_package_names()), set(mock_db.all_package_names())\n )\n packages = list(not_in_mock)[:max_packages]\n\n # Create all the packages that are not in mock.\n installed_specs = {}\n for pkg_name in packages:\n # If a spec fails to concretize, just skip it. If it is a\n # real error, it will be caught by concretization tests.\n try:\n spec = spack.concretize.concretize_one(pkg_name)\n except Exception:\n continue\n\n layout.create_install_directory(spec)\n installed_specs[spec] = layout.path_for_spec(spec)\n\n with spack.repo.use_repositories(spack.paths.mock_packages_path):\n # Now check that even without the package files, we know\n # enough to read a spec from the spec file.\n for spec, path in installed_specs.items():\n spec_from_file = layout.read_spec(os.path.join(path, \".spack\", \"spec.json\"))\n\n # To satisfy these conditions, directory layouts need to\n # read in concrete specs from their install dirs somehow.\n assert path == layout.path_for_spec(spec_from_file)\n assert spec == spec_from_file\n assert spec.eq_dag(spec_from_file)\n assert spec.dag_hash() == spec_from_file.dag_hash()\n\n\ndef test_find(temporary_store, config, mock_packages):\n \"\"\"Test that finding specs within an install layout works.\"\"\"\n layout = temporary_store.layout\n package_names = list(spack.repo.PATH.all_package_names())[:max_packages]\n\n # Create install prefixes for all packages in the list\n installed_specs = {}\n for name in package_names:\n if name.startswith(\"external\"):\n # External package tests cannot be installed\n continue\n spec = spack.concretize.concretize_one(name)\n installed_specs[spec.name] = spec\n layout.create_install_directory(spec)\n\n # Make sure all the installed specs appear in\n # DirectoryLayout.all_specs()\n found_specs = dict((s.name, s) for s in layout.all_specs())\n for name, spec in found_specs.items():\n assert name in found_specs\n assert found_specs[name].eq_dag(spec)\n\n\ndef test_yaml_directory_layout_build_path(tmp_path: pathlib.Path, default_mock_concretization):\n \"\"\"This tests build path method.\"\"\"\n spec = default_mock_concretization(\"python\")\n layout = DirectoryLayout(str(tmp_path))\n rel_path = os.path.join(layout.metadata_dir, layout.packages_dir)\n assert layout.build_packages_path(spec) == os.path.join(spec.prefix, rel_path)\n" }, { "path": "lib/spack/spack/test/entry_points.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport pathlib\nimport sys\n\nimport pytest\n\nimport spack.config\nimport spack.extensions\nimport spack.llnl.util.lang\n\n\nclass MockConfigEntryPoint:\n def __init__(self, tmp_path: pathlib.Path):\n self.dir = tmp_path\n self.name = \"mypackage_config\"\n\n def load(self):\n etc_path = self.dir.joinpath(\"spack/etc\")\n etc_path.mkdir(exist_ok=True, parents=True)\n f = self.dir / \"spack/etc/config.yaml\"\n with open(f, \"w\", encoding=\"utf-8\") as fh:\n fh.write(\"config:\\n install_tree:\\n root: /spam/opt\\n\")\n\n def ep():\n return self.dir / \"spack/etc\"\n\n return ep\n\n\nclass MockExtensionsEntryPoint:\n def __init__(self, tmp_path: pathlib.Path):\n self.dir = tmp_path\n self.name = \"mypackage_extensions\"\n\n def load(self):\n cmd_path = self.dir.joinpath(\"spack/spack-myext/myext/cmd\")\n cmd_path.mkdir(exist_ok=True, parents=True)\n f = self.dir / \"spack/spack-myext/myext/cmd/spam.py\"\n with open(f, \"w\", encoding=\"utf-8\") as fh:\n fh.write(\"description = 'hello world extension command'\\n\")\n fh.write(\"section = 'test command'\\n\")\n fh.write(\"level = 'long'\\n\")\n fh.write(\"def setup_parser(subparser):\\n pass\\n\")\n fh.write(\"def spam(parser, args):\\n print('spam for all!')\\n\")\n\n def ep():\n return self.dir / \"spack/spack-myext\"\n\n return ep\n\n\ndef entry_points_factory(tmp_path: pathlib.Path):\n def entry_points(group=None):\n if group == \"spack.config\":\n return (MockConfigEntryPoint(tmp_path),)\n elif group == \"spack.extensions\":\n return (MockExtensionsEntryPoint(tmp_path),)\n return ()\n\n return entry_points\n\n\n@pytest.fixture()\ndef mock_get_entry_points(tmp_path: pathlib.Path, reset_extension_paths, monkeypatch):\n entry_points = entry_points_factory(tmp_path)\n monkeypatch.setattr(spack.llnl.util.lang, \"get_entry_points\", entry_points)\n\n\ndef test_spack_entry_point_config(tmp_path: pathlib.Path, mock_get_entry_points):\n \"\"\"Test config scope entry point\"\"\"\n config_paths = dict(spack.config.config_paths_from_entry_points())\n config_path = config_paths.get(\"plugin-mypackage_config\")\n my_config_path = tmp_path / \"spack/etc\"\n if config_path is None:\n raise ValueError(\"Did not find entry point config in %s\" % str(config_paths))\n else:\n assert os.path.samefile(config_path, my_config_path)\n config = spack.config.create()\n assert config.get(\"config:install_tree:root\", scope=\"plugin-mypackage_config\") == \"/spam/opt\"\n\n\ndef test_spack_entry_point_extension(tmp_path: pathlib.Path, mock_get_entry_points):\n \"\"\"Test config scope entry point\"\"\"\n my_ext = tmp_path / \"spack/spack-myext\"\n extensions = spack.extensions.get_extension_paths()\n found = bool([ext for ext in extensions if os.path.samefile(ext, my_ext)])\n if not found:\n raise ValueError(\"Did not find extension in %s\" % \", \".join(extensions))\n extensions = spack.extensions.extension_paths_from_entry_points()\n found = bool([ext for ext in extensions if os.path.samefile(ext, my_ext)])\n if not found:\n raise ValueError(\"Did not find extension in %s\" % \", \".join(extensions))\n root = spack.extensions.load_extension(\"myext\")\n assert os.path.samefile(root, my_ext)\n module = spack.extensions.get_module(\"spam\")\n assert module is not None\n\n\n@pytest.mark.skipif(sys.version_info[:2] < (3, 8), reason=\"Python>=3.8 required\")\ndef test_llnl_util_lang_get_entry_points(tmp_path: pathlib.Path, monkeypatch):\n import importlib.metadata # type: ignore # novermin\n\n monkeypatch.setattr(importlib.metadata, \"entry_points\", entry_points_factory(tmp_path))\n\n entry_points = list(spack.llnl.util.lang.get_entry_points(group=\"spack.config\"))\n assert isinstance(entry_points[0], MockConfigEntryPoint)\n\n entry_points = list(spack.llnl.util.lang.get_entry_points(group=\"spack.extensions\"))\n assert isinstance(entry_points[0], MockExtensionsEntryPoint)\n" }, { "path": "lib/spack/spack/test/env.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Test environment internals without CLI\"\"\"\n\nimport filecmp\nimport json\nimport os\nimport pathlib\nimport pickle\n\nimport pytest\n\nimport spack.config\nimport spack.environment as ev\nimport spack.llnl.util.filesystem as fs\nimport spack.platforms\nimport spack.solver.asp\nimport spack.spec\nimport spack.spec_parser\nfrom spack.enums import ConfigScopePriority\nfrom spack.environment import SpackEnvironmentConfigError\nfrom spack.environment.environment import (\n EnvironmentManifestFile,\n SpackEnvironmentViewError,\n _error_on_nonempty_view_dir,\n)\nfrom spack.environment.list import UndefinedReferenceError\nfrom spack.traverse import traverse_nodes\n\npytestmark = [\n pytest.mark.not_on_windows(\"Envs are not supported on windows\"),\n pytest.mark.usefixtures(\"mock_packages\"),\n]\n\n\nclass TestDirectoryInitialization:\n def test_environment_dir_from_name(self, mutable_mock_env_path):\n \"\"\"Test the function mapping a managed environment name to its folder.\"\"\"\n env = ev.create(\"test\")\n environment_dir = ev.environment_dir_from_name(\"test\")\n assert env.path == environment_dir\n with pytest.raises(ev.SpackEnvironmentError, match=\"environment already exists\"):\n ev.environment_dir_from_name(\"test\", exists_ok=False)\n\n def test_environment_dir_from_nested_name(self, mutable_mock_env_path):\n \"\"\"Test the function mapping a nested managed environment name to its folder.\"\"\"\n env = ev.create(\"group/test\")\n environment_dir = ev.environment_dir_from_name(\"group/test\")\n assert env.path == environment_dir\n with pytest.raises(ev.SpackEnvironmentError, match=\"environment already exists\"):\n ev.environment_dir_from_name(\"group/test\", exists_ok=False)\n\n\ndef test_hash_change_no_rehash_concrete(tmp_path: pathlib.Path, config):\n # create an environment\n env_path = tmp_path / \"env_dir\"\n env_path.mkdir(exist_ok=False)\n env = ev.create_in_dir(env_path)\n env.write()\n\n # add a spec with a rewritten build hash\n spec = spack.spec.Spec(\"mpileaks\")\n env.add(spec)\n env.concretize()\n\n # rewrite the hash\n old_hash, new_hash = env.concretized_roots[0].hash, \"abc\"\n env.specs_by_hash[old_hash]._hash = new_hash # type: ignore[attr-defined]\n env.concretized_roots[0].hash = new_hash\n env.specs_by_hash[new_hash] = env.specs_by_hash[old_hash]\n del env.specs_by_hash[old_hash]\n env.write()\n\n # Read environment\n read_in = ev.Environment(env_path)\n\n # Ensure read hashes are used (rewritten hash seen on read)\n hashes = [x.hash for x in read_in.concretized_roots]\n assert hashes\n assert hashes[0] in read_in.specs_by_hash\n _hash = read_in.specs_by_hash[hashes[0]]._hash # type: ignore[attr-defined]\n assert _hash == new_hash\n\n\ndef test_env_change_spec(tmp_path: pathlib.Path, config):\n env_path = tmp_path / \"env_dir\"\n env_path.mkdir(exist_ok=False)\n env = ev.create_in_dir(env_path)\n env.write()\n\n spec = spack.spec.Spec(\"mpileaks@2.1~shared+debug\")\n env.add(spec)\n env.write()\n\n change_spec = spack.spec.Spec(\"mpileaks@2.2\")\n env.change_existing_spec(change_spec)\n (spec,) = env.added_specs()\n assert spec == spack.spec.Spec(\"mpileaks@2.2~shared+debug\")\n\n change_spec = spack.spec.Spec(\"mpileaks~debug\")\n env.change_existing_spec(change_spec)\n (spec,) = env.added_specs()\n assert spec == spack.spec.Spec(\"mpileaks@2.2~shared~debug\")\n\n\n_test_matrix_yaml = \"\"\"\\\nspack:\n definitions:\n - compilers: [\"%gcc\", \"%clang\"]\n - desired_specs: [\"mpileaks@2.1\"]\n specs:\n - matrix:\n - [$compilers]\n - [$desired_specs]\n\"\"\"\n\n\ndef test_env_change_spec_in_definition(tmp_path: pathlib.Path, mutable_mock_env_path):\n manifest_file = tmp_path / ev.manifest_name\n manifest_file.write_text(_test_matrix_yaml)\n e = ev.create(\"test\", manifest_file)\n e.concretize()\n e.write()\n\n assert any(x.intersects(\"mpileaks@2.1%gcc\") for x in e.user_specs)\n\n with e:\n e.change_existing_spec(spack.spec.Spec(\"mpileaks@2.2\"), list_name=\"desired_specs\")\n e.write()\n\n # Ensure changed specs are in memory\n assert any(x.intersects(\"mpileaks@2.2%gcc\") for x in e.user_specs)\n assert not any(x.intersects(\"mpileaks@2.1%gcc\") for x in e.user_specs)\n\n # Now make sure the changes can be read from the modified config\n e = ev.read(\"test\")\n assert any(x.intersects(\"mpileaks@2.2%gcc\") for x in e.user_specs)\n assert not any(x.intersects(\"mpileaks@2.1%gcc\") for x in e.user_specs)\n\n\ndef test_env_change_spec_in_matrix_raises_error(tmp_path: pathlib.Path, mutable_mock_env_path):\n manifest_file = tmp_path / ev.manifest_name\n manifest_file.write_text(_test_matrix_yaml)\n e = ev.create(\"test\", manifest_file)\n e.concretize()\n e.write()\n\n with pytest.raises(ev.SpackEnvironmentError) as error:\n e.change_existing_spec(spack.spec.Spec(\"mpileaks@2.2\"))\n assert \"Cannot directly change specs in matrices\" in str(error)\n\n\ndef test_activate_should_require_an_env():\n with pytest.raises(TypeError):\n ev.activate(env=\"name\")\n\n with pytest.raises(TypeError):\n ev.activate(env=None)\n\n\ndef test_user_view_path_is_not_canonicalized_in_yaml(tmp_path: pathlib.Path, config):\n # When spack.yaml files are checked into version control, we\n # don't want view: ./relative to get canonicalized on disk.\n\n # We create a view in /env_dir\n env_path = str(tmp_path / \"env_dir\")\n (tmp_path / \"env_dir\").mkdir()\n\n # And use a relative path to specify the view dir\n view = os.path.join(\".\", \"view\")\n\n # Which should always resolve to the following independent of cwd.\n absolute_view = os.path.join(env_path, \"view\")\n\n # Serialize environment with relative view path\n with fs.working_dir(str(tmp_path)):\n fst = ev.create_in_dir(env_path, with_view=view)\n fst.regenerate_views()\n\n # The view link should be created\n assert os.path.isdir(absolute_view)\n\n # Deserialize and check if the view path is still relative in yaml\n # and also check that the getter is pointing to the right dir.\n with fs.working_dir(str(tmp_path)):\n snd = ev.Environment(env_path)\n assert snd.manifest[\"spack\"][\"view\"] == view\n assert os.path.samefile(snd.default_view.root, absolute_view)\n\n\ndef test_environment_cant_modify_environments_root(tmp_path: pathlib.Path):\n filename = str(tmp_path / \"spack.yaml\")\n with open(filename, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\n spack:\n config:\n environments_root: /a/black/hole\n view: false\n specs: []\n \"\"\"\n )\n with fs.working_dir(str(tmp_path)):\n with pytest.raises(ev.SpackEnvironmentError):\n e = ev.Environment(str(tmp_path))\n ev.activate(e)\n\n\n@pytest.mark.regression(\"35420\")\n@pytest.mark.parametrize(\n \"original_content\",\n [\n \"\"\"\\\nspack:\n specs:\n - matrix:\n # test\n - - a\n concretizer:\n unify: false\"\"\"\n ],\n)\ndef test_roundtrip_spack_yaml_with_comments(original_content, config, tmp_path: pathlib.Path):\n \"\"\"Ensure that round-tripping a spack.yaml file doesn't change its content.\"\"\"\n spack_yaml = tmp_path / \"spack.yaml\"\n spack_yaml.write_text(original_content)\n\n e = ev.Environment(tmp_path)\n e.manifest.flush()\n\n content = spack_yaml.read_text()\n assert content == original_content\n\n\ndef test_adding_anonymous_specs_to_env_fails(tmp_path: pathlib.Path):\n \"\"\"Tests that trying to add an anonymous spec to the 'specs' section of an environment\n raises an exception\n \"\"\"\n env = ev.create_in_dir(tmp_path)\n with pytest.raises(ev.SpackEnvironmentError, match=\"cannot add anonymous\"):\n env.add(\"%gcc\")\n\n\ndef test_removing_from_non_existing_list_fails(tmp_path: pathlib.Path):\n \"\"\"Tests that trying to remove a spec from a non-existing definition fails.\"\"\"\n env = ev.create_in_dir(tmp_path)\n with pytest.raises(ev.SpackEnvironmentError, match=\"'bar' does not exist\"):\n env.remove(\"%gcc\", list_name=\"bar\")\n\n\n@pytest.mark.parametrize(\n \"init_view,update_value\",\n [\n (True, False),\n (True, \"./view\"),\n (False, True),\n (\"./view\", True),\n (\"./view\", False),\n (True, True),\n (False, False),\n ],\n)\ndef test_update_default_view(init_view, update_value, tmp_path: pathlib.Path, config):\n \"\"\"Tests updating the default view with different values.\"\"\"\n env = ev.create_in_dir(tmp_path, with_view=init_view)\n env.update_default_view(update_value)\n env.write(regenerate=True)\n if not isinstance(update_value, bool):\n assert env.default_view.raw_root == update_value\n\n expected_value = update_value\n if isinstance(init_view, str) and update_value is True:\n expected_value = init_view\n\n assert env.manifest.yaml_content[\"spack\"][\"view\"] == expected_value\n\n\n@pytest.mark.parametrize(\n \"initial_content,update_value,expected_view\",\n [\n (\n \"\"\"\nspack:\n specs:\n - mpileaks\n view:\n default:\n root: ./view-gcc\n select: ['%gcc']\n link_type: symlink\n\"\"\",\n \"./another-view\",\n {\"root\": \"./another-view\", \"select\": [\"%gcc\"], \"link_type\": \"symlink\"},\n ),\n (\n \"\"\"\nspack:\n specs:\n - mpileaks\n view:\n default:\n root: ./view-gcc\n select: ['%gcc']\n link_type: symlink\n\"\"\",\n True,\n {\"root\": \"./view-gcc\", \"select\": [\"%gcc\"], \"link_type\": \"symlink\"},\n ),\n ],\n)\ndef test_update_default_complex_view(\n initial_content, update_value, expected_view, tmp_path: pathlib.Path, config\n):\n spack_yaml = tmp_path / \"spack.yaml\"\n spack_yaml.write_text(initial_content)\n\n env = ev.Environment(tmp_path)\n env.update_default_view(update_value)\n env.write(regenerate=True)\n\n assert env.default_view.to_dict() == expected_view\n\n\n@pytest.mark.parametrize(\"filename\", [ev.manifest_name, ev.lockfile_name])\ndef test_cannot_initialize_in_dir_with_init_file(tmp_path: pathlib.Path, filename):\n \"\"\"Tests that initializing an environment in a directory with an already existing\n spack.yaml or spack.lock raises an exception.\n \"\"\"\n init_file = tmp_path / filename\n init_file.touch()\n with pytest.raises(ev.SpackEnvironmentError, match=\"cannot initialize\"):\n ev.create_in_dir(tmp_path)\n\n\ndef test_cannot_initiliaze_if_dirname_exists_as_a_file(tmp_path: pathlib.Path):\n \"\"\"Tests that initializing an environment using as a location an existing file raises\n an error.\n \"\"\"\n dir_name = tmp_path / \"dir\"\n dir_name.touch()\n with pytest.raises(ev.SpackEnvironmentError, match=\"cannot initialize\"):\n ev.create_in_dir(dir_name)\n\n\ndef test_cannot_initialize_if_init_file_does_not_exist(tmp_path: pathlib.Path):\n \"\"\"Tests that initializing an environment passing a non-existing init file raises an error.\"\"\"\n init_file = tmp_path / ev.manifest_name\n with pytest.raises(ev.SpackEnvironmentError, match=\"cannot initialize\"):\n ev.create_in_dir(tmp_path, init_file=init_file)\n\n\ndef test_environment_pickle(tmp_path: pathlib.Path):\n env1 = ev.create_in_dir(tmp_path)\n obj = pickle.dumps(env1)\n env2 = pickle.loads(obj)\n assert isinstance(env2, ev.Environment)\n\n\ndef test_error_on_nonempty_view_dir(tmp_path: pathlib.Path):\n \"\"\"Error when the target is not an empty dir\"\"\"\n with fs.working_dir(str(tmp_path)):\n os.mkdir(\"empty_dir\")\n os.mkdir(\"nonempty_dir\")\n with open(os.path.join(\"nonempty_dir\", \"file\"), \"wb\"):\n pass\n os.symlink(\"empty_dir\", \"symlinked_empty_dir\")\n os.symlink(\"does_not_exist\", \"broken_link\")\n os.symlink(\"broken_link\", \"file\")\n\n # This is OK.\n _error_on_nonempty_view_dir(\"empty_dir\")\n\n # This is not OK.\n with pytest.raises(SpackEnvironmentViewError):\n _error_on_nonempty_view_dir(\"nonempty_dir\")\n\n with pytest.raises(SpackEnvironmentViewError):\n _error_on_nonempty_view_dir(\"symlinked_empty_dir\")\n\n with pytest.raises(SpackEnvironmentViewError):\n _error_on_nonempty_view_dir(\"broken_link\")\n\n with pytest.raises(SpackEnvironmentViewError):\n _error_on_nonempty_view_dir(\"file\")\n\n\ndef test_can_add_specs_to_environment_without_specs_attribute(tmp_path: pathlib.Path, config):\n \"\"\"Sometimes users have template manifest files, and save one line in the YAML file by\n removing the empty 'specs: []' attribute. This test ensures that adding a spec to an\n environment without the 'specs' attribute, creates the attribute first instead of returning\n an error.\n \"\"\"\n spack_yaml = tmp_path / \"spack.yaml\"\n spack_yaml.write_text(\n \"\"\"\nspack:\n view: true\n concretizer:\n unify: true\n \"\"\"\n )\n env = ev.Environment(tmp_path)\n env.add(\"pkg-a\")\n\n assert len(env.user_specs) == 1\n assert env.manifest.yaml_content[\"spack\"][\"specs\"] == [\"pkg-a\"]\n\n\n@pytest.mark.parametrize(\n \"original_yaml,new_spec,expected_yaml\",\n [\n (\n \"\"\"spack:\n specs:\n # baz\n - zlib\n\"\"\",\n \"libdwarf\",\n \"\"\"spack:\n specs:\n # baz\n - zlib\n - libdwarf\n\"\"\",\n )\n ],\n)\ndef test_preserving_comments_when_adding_specs(\n original_yaml, new_spec, expected_yaml, config, tmp_path: pathlib.Path\n):\n \"\"\"Ensure that round-tripping a spack.yaml file doesn't change its content.\"\"\"\n spack_yaml = tmp_path / \"spack.yaml\"\n spack_yaml.write_text(original_yaml)\n\n e = ev.Environment(str(tmp_path))\n e.add(new_spec)\n e.write()\n\n content = spack_yaml.read_text()\n assert content == expected_yaml\n\n\n@pytest.mark.parametrize(\"filename\", [ev.lockfile_name, \"as9582g54.lock\", \"m3ia54s.json\"])\n@pytest.mark.regression(\"37410\")\ndef test_initialize_from_lockfile(tmp_path: pathlib.Path, filename):\n \"\"\"Some users have workflows where they store multiple lockfiles in the\n same directory, and pick one of them to create an environment depending\n on external parameters e.g. while running CI jobs. This test ensures that\n Spack can create environments from lockfiles that are not necessarily named\n 'spack.lock' and can thus coexist in the same directory.\n \"\"\"\n\n init_file = tmp_path / filename\n env_dir = tmp_path / \"env_dir\"\n init_file.write_text('{ \"roots\": [] }\\n')\n ev.initialize_environment_dir(env_dir, init_file)\n\n assert os.path.exists(env_dir / ev.lockfile_name)\n assert filecmp.cmp(env_dir / ev.lockfile_name, init_file, shallow=False)\n\n\ndef test_cannot_initialize_from_bad_lockfile(tmp_path: pathlib.Path):\n \"\"\"Test that we fail on an incorrectly constructed lockfile\"\"\"\n\n init_file = tmp_path / ev.lockfile_name\n env_dir = tmp_path / \"env_dir\"\n\n init_file.write_text(\"Not a legal JSON file\\n\")\n\n with pytest.raises(ev.SpackEnvironmentError, match=\"from lockfile\"):\n ev.initialize_environment_dir(env_dir, init_file)\n\n\n@pytest.mark.parametrize(\"filename\", [\"random.txt\", \"random.yaml\", ev.manifest_name])\n@pytest.mark.regression(\"37410\")\ndef test_initialize_from_random_file_as_manifest(tmp_path: pathlib.Path, filename):\n \"\"\"Some users have workflows where they store multiple lockfiles in the\n same directory, and pick one of them to create an environment depending\n on external parameters e.g. while running CI jobs. This test ensures that\n Spack can create environments from manifest that are not necessarily named\n 'spack.yaml' and can thus coexist in the same directory.\n \"\"\"\n\n init_file = tmp_path / filename\n env_dir = tmp_path / \"env_dir\"\n\n init_file.write_text(\n \"\"\"\\\nspack:\n view: true\n concretizer:\n unify: true\n specs: []\n\"\"\"\n )\n\n ev.create_in_dir(env_dir, init_file)\n\n assert not os.path.exists(env_dir / ev.lockfile_name)\n assert os.path.exists(env_dir / ev.manifest_name)\n assert filecmp.cmp(env_dir / ev.manifest_name, init_file, shallow=False)\n\n\ndef test_error_message_when_using_too_new_lockfile(tmp_path: pathlib.Path):\n \"\"\"Sometimes the lockfile format needs to be bumped. When that happens, we have forward\n incompatibilities that need to be reported in a clear way to the user, in case we moved\n back to an older version of Spack. This test ensures that the error message for a too\n new lockfile version stays comprehensible across refactoring of the environment code.\n \"\"\"\n init_file = tmp_path / ev.lockfile_name\n env_dir = tmp_path / \"env_dir\"\n init_file.write_text(\n \"\"\"\n{\n \"_meta\": {\n \"file-type\": \"spack-lockfile\",\n \"lockfile-version\": 100,\n \"specfile-version\": 3\n },\n \"roots\": [],\n \"concrete_specs\": {}\n}\\n\n\"\"\"\n )\n ev.initialize_environment_dir(env_dir, init_file)\n with pytest.raises(ev.SpackEnvironmentError, match=\"You need to use a newer Spack version.\"):\n ev.Environment(env_dir)\n\n\n@pytest.mark.regression(\"38240\")\n@pytest.mark.parametrize(\n \"unify_in_lower_scope,unify_in_spack_yaml\",\n [\n (True, False),\n (True, \"when_possible\"),\n (False, True),\n (False, \"when_possible\"),\n (\"when_possible\", False),\n (\"when_possible\", True),\n ],\n)\ndef test_environment_concretizer_scheme_used(\n tmp_path: pathlib.Path, mutable_config, unify_in_lower_scope, unify_in_spack_yaml\n):\n \"\"\"Tests that \"unify\" settings in spack.yaml always take precedence over settings in lower\n configuration scopes.\n \"\"\"\n manifest = tmp_path / \"spack.yaml\"\n manifest.write_text(\n f\"\"\"\\\nspack:\n specs:\n - mpileaks\n concretizer:\n unify: {str(unify_in_spack_yaml).lower()}\n\"\"\"\n )\n mutable_config.set(\"concretizer:unify\", unify_in_lower_scope)\n assert mutable_config.get(\"concretizer:unify\") == unify_in_lower_scope\n with ev.Environment(manifest.parent):\n assert mutable_config.get(\"concretizer:unify\") == unify_in_spack_yaml\n\n\n@pytest.mark.parametrize(\"unify_in_config\", [True, False, \"when_possible\"])\ndef test_environment_config_scheme_used(tmp_path: pathlib.Path, unify_in_config):\n \"\"\"Tests that \"unify\" settings in lower configuration scopes is taken into account,\n if absent in spack.yaml.\n \"\"\"\n manifest = tmp_path / \"spack.yaml\"\n manifest.write_text(\n \"\"\"\\\nspack:\n specs:\n - mpileaks\n\"\"\"\n )\n\n with spack.config.override(\"concretizer:unify\", unify_in_config):\n with ev.Environment(manifest.parent):\n assert spack.config.CONFIG.get(\"concretizer:unify\") == unify_in_config\n\n\n@pytest.mark.parametrize(\n \"spec_str,expected_raise,expected_spec\",\n [\n # vendorsb vendors \"b\" only when @=1.1\n (\"vendorsb\", False, \"vendorsb@=1.0\"),\n (\"vendorsb@=1.1\", True, None),\n ],\n)\ndef test_conflicts_with_packages_that_are_not_dependencies(\n spec_str, expected_raise, expected_spec, tmp_path: pathlib.Path, config\n):\n \"\"\"Tests that we cannot concretize two specs together, if one conflicts with the other,\n even though they don't have a dependency relation.\n \"\"\"\n manifest = tmp_path / \"spack.yaml\"\n manifest.write_text(\n f\"\"\"\\\nspack:\n specs:\n - {spec_str}\n - pkg-b\n concretizer:\n unify: true\n\"\"\"\n )\n with ev.Environment(manifest.parent) as e:\n if expected_raise:\n with pytest.raises(spack.solver.asp.UnsatisfiableSpecError):\n e.concretize()\n else:\n e.concretize()\n assert any(s.satisfies(expected_spec) for s in e.concrete_roots())\n\n\n@pytest.mark.regression(\"39455\")\n@pytest.mark.parametrize(\n \"possible_mpi_spec,unify\", [(\"mpich\", False), (\"mpich\", True), (\"zmpi\", False), (\"zmpi\", True)]\n)\ndef test_requires_on_virtual_and_potential_providers(\n possible_mpi_spec, unify, tmp_path: pathlib.Path, config\n):\n \"\"\"Tests that in an environment we can add packages explicitly, even though they provide\n a virtual package, and we require the provider of the same virtual to be another package,\n if they are added explicitly by their name.\n \"\"\"\n manifest = tmp_path / \"spack.yaml\"\n manifest.write_text(\n f\"\"\"\\\n spack:\n specs:\n - {possible_mpi_spec}\n - mpich2\n - mpileaks\n packages:\n mpi:\n require: mpich2\n concretizer:\n unify: {unify}\n \"\"\"\n )\n with ev.Environment(manifest.parent) as e:\n e.concretize()\n assert e.matching_spec(possible_mpi_spec)\n assert e.matching_spec(\"mpich2\")\n\n mpileaks = e.matching_spec(\"mpileaks\")\n assert mpileaks.satisfies(\"^mpich2\")\n assert mpileaks[\"mpi\"].satisfies(\"mpich2\")\n assert not mpileaks.satisfies(f\"^{possible_mpi_spec}\")\n\n\n@pytest.mark.regression(\"39387\")\n@pytest.mark.parametrize(\n \"spec_str\", [\"mpileaks +opt\", \"mpileaks +opt ~shared\", \"mpileaks ~shared +opt\"]\n)\ndef test_manifest_file_removal_works_if_spec_is_not_normalized(tmp_path: pathlib.Path, spec_str):\n \"\"\"Tests that we can remove a spec from a manifest file even if its string\n representation is not normalized.\n \"\"\"\n manifest = tmp_path / \"spack.yaml\"\n manifest.write_text(\n f\"\"\"\\\nspack:\n specs:\n - {spec_str}\n\"\"\"\n )\n s = spack.spec.Spec(spec_str)\n spack_yaml = EnvironmentManifestFile(tmp_path)\n # Doing a round trip str -> Spec -> str normalizes the representation\n spack_yaml.remove_user_spec(str(s))\n spack_yaml.flush()\n\n assert spec_str not in manifest.read_text()\n\n\n@pytest.mark.regression(\"39387\")\n@pytest.mark.parametrize(\n \"duplicate_specs,expected_number\",\n [\n # Swap variants, versions, etc. add spaces\n ([\"foo +bar ~baz\", \"foo ~baz +bar\"], 3),\n ([\"foo @1.0 ~baz %gcc\", \"foo ~baz @1.0%gcc\"], 3),\n # Item 1 and 3 are exactly the same\n ([\"zlib +shared\", \"zlib +shared\", \"zlib +shared\"], 4),\n ],\n)\ndef test_removing_spec_from_manifest_with_exact_duplicates(\n duplicate_specs, expected_number, tmp_path: pathlib.Path\n):\n \"\"\"Tests that we can remove exact duplicates from a manifest file.\n\n Note that we can't get in a state with duplicates using only CLI, but this might happen\n on user edited spack.yaml files.\n \"\"\"\n manifest = tmp_path / \"spack.yaml\"\n manifest.write_text(\n f\"\"\"\\\n spack:\n specs: [{\", \".join(duplicate_specs)} , \"zlib\"]\n \"\"\"\n )\n\n with ev.Environment(tmp_path) as env:\n assert len(env.user_specs) == expected_number\n env.remove(duplicate_specs[0])\n env.write()\n\n assert \"+shared\" not in manifest.read_text()\n assert \"zlib\" in manifest.read_text()\n with ev.Environment(tmp_path) as env:\n assert len(env.user_specs) == 1\n\n\n@pytest.mark.regression(\"35298\")\ndef test_variant_propagation_with_unify_false(tmp_path: pathlib.Path, config):\n \"\"\"Spack distributes concretizations to different processes, when unify:false is selected and\n the number of roots is 2 or more. When that happens, the specs to be concretized need to be\n properly reconstructed on the worker process, if variant propagation was requested.\n \"\"\"\n manifest = tmp_path / \"spack.yaml\"\n manifest.write_text(\n \"\"\"\n spack:\n specs:\n - parent-foo ++foo\n - pkg-c\n concretizer:\n unify: false\n \"\"\"\n )\n with ev.Environment(tmp_path) as env:\n env.concretize()\n\n root = env.matching_spec(\"parent-foo\")\n for node in root.traverse():\n assert node.satisfies(\"+foo\")\n\n\ndef test_env_with_include_defs(mutable_mock_env_path):\n \"\"\"Test environment with included definitions file.\"\"\"\n env_path = mutable_mock_env_path\n env_path.mkdir()\n defs_file = env_path / \"definitions.yaml\"\n defs_file.write_text(\n \"\"\"definitions:\n- core_specs: [libdwarf, libelf]\n- compilers: ['%gcc']\n\"\"\"\n )\n\n spack_yaml = env_path / ev.manifest_name\n spack_yaml.write_text(\n f\"\"\"spack:\n include:\n - {defs_file.as_uri()}\n\n definitions:\n - my_packages: [zlib]\n\n specs:\n - matrix:\n - [$core_specs]\n - [$compilers]\n - $my_packages\n\"\"\"\n )\n\n e = ev.Environment(env_path)\n with e:\n e.concretize()\n\n\ndef test_env_with_include_def_missing(mutable_mock_env_path):\n \"\"\"Test environment with included definitions file that is missing a definition.\"\"\"\n env_path = mutable_mock_env_path\n env_path.mkdir()\n filename = \"missing-def.yaml\"\n defs_file = env_path / filename\n defs_file.write_text(\"definitions:\\n- my_compilers: ['%gcc']\\n\")\n\n spack_yaml = env_path / ev.manifest_name\n spack_yaml.write_text(\n f\"\"\"spack:\n include:\n - {defs_file.as_uri()}\n\n specs:\n - matrix:\n - [$core_specs]\n - [$my_compilers]\n\"\"\"\n )\n\n with pytest.raises(UndefinedReferenceError, match=r\"which is not defined\"):\n _ = ev.Environment(env_path)\n\n\n@pytest.mark.regression(\"41292\")\n@pytest.mark.parametrize(\"unify\", [\"true\", \"false\", \"when_possible\"])\ndef test_deconcretize_then_concretize_does_not_error(mutable_mock_env_path, unify):\n \"\"\"Tests that, after having deconcretized a spec, we can reconcretize an environment which\n has 2 or more user specs mapping to the same concrete spec.\n \"\"\"\n mutable_mock_env_path.mkdir()\n spack_yaml = mutable_mock_env_path / ev.manifest_name\n spack_yaml.write_text(\n f\"\"\"spack:\n specs:\n # These two specs concretize to the same hash\n - pkg-c\n - pkg-c@1.0\n # Spec used to trigger the bug\n - pkg-a\n concretizer:\n unify: {unify}\n \"\"\"\n )\n e = ev.Environment(mutable_mock_env_path)\n # Initial state\n assert len(e.user_specs) == 3\n assert len(e.concretized_roots) == 0\n\n with e:\n e.concretize()\n assert len(e.user_specs) == 3\n assert len(e.concretized_roots) == 3\n assert all(x.new for x in e.concretized_roots)\n\n e.deconcretize_by_user_spec(spack.spec.Spec(\"pkg-a\"))\n assert len(e.user_specs) == 3\n assert len(e.concretized_roots) == 2\n assert all(x.new for x in e.concretized_roots)\n\n e.concretize()\n assert len(e.user_specs) == 3\n assert len(e.concretized_roots) == 3\n assert all(x.new for x in e.concretized_roots)\n\n all_root_hashes = {x.dag_hash() for x in e.concrete_roots()}\n assert len(all_root_hashes) == 2\n\n\n@pytest.mark.regression(\"44216\")\ndef test_root_version_weights_for_old_versions(mutable_mock_env_path):\n \"\"\"Tests that, when we select two old versions of root specs that have the same version\n optimization penalty, both are considered.\n \"\"\"\n mutable_mock_env_path.mkdir()\n spack_yaml = mutable_mock_env_path / ev.manifest_name\n spack_yaml.write_text(\n \"\"\"spack:\n specs:\n # allow any version, but the most recent\n - bowtie@:1.3\n # allows only the third most recent, so penalty is 2\n - gcc@1\n concretizer:\n unify: true\n \"\"\"\n )\n e = ev.Environment(mutable_mock_env_path)\n with e:\n e.concretize()\n\n bowtie = [x for x in e.concrete_roots() if x.name == \"bowtie\"][0]\n gcc = [x for x in e.concrete_roots() if x.name == \"gcc\"][0]\n\n assert bowtie.satisfies(\"@=1.3.0\")\n assert gcc.satisfies(\"@=1.0\")\n\n\ndef test_env_view_on_empty_dir_is_fine(tmp_path: pathlib.Path, config, temporary_store):\n \"\"\"Tests that creating a view pointing to an empty dir is not an error.\"\"\"\n view_dir = tmp_path / \"view\"\n view_dir.mkdir()\n env = ev.create_in_dir(tmp_path, with_view=\"view\")\n env.add(\"mpileaks\")\n env.concretize()\n env.install_all(fake=True)\n env.regenerate_views()\n assert view_dir.is_symlink()\n\n\ndef test_env_view_on_non_empty_dir_errors(tmp_path: pathlib.Path, config, temporary_store):\n \"\"\"Tests that creating a view pointing to a non-empty dir errors.\"\"\"\n view_dir = tmp_path / \"view\"\n view_dir.mkdir()\n (view_dir / \"file\").write_text(\"\")\n env = ev.create_in_dir(tmp_path, with_view=\"view\")\n env.add(\"mpileaks\")\n env.concretize()\n env.install_all(fake=True)\n with pytest.raises(ev.SpackEnvironmentError, match=\"because it is a non-empty dir\"):\n env.regenerate_views()\n\n\n@pytest.mark.parametrize(\n \"matrix_line\", [(\"^zmpi\", \"^mpich\"), (\"~shared\", \"+shared\"), (\"shared=False\", \"+shared-libs\")]\n)\n@pytest.mark.regression(\"40791\")\ndef test_stack_enforcement_is_strict(tmp_path: pathlib.Path, matrix_line, config):\n \"\"\"Ensure that constraints in matrices are applied strictly after expansion, to avoid\n inconsistencies between abstract user specs and concrete specs.\n \"\"\"\n manifest = tmp_path / \"spack.yaml\"\n manifest.write_text(\n f\"\"\"\\\nspack:\n definitions:\n - packages: [libelf, mpileaks]\n - install:\n - matrix:\n - [$packages]\n - [{\", \".join(item for item in matrix_line)}]\n specs:\n - $install\n concretizer:\n unify: false\n\"\"\"\n )\n # Here we raise different exceptions depending on whether we solve serially or not\n with pytest.raises(Exception):\n with ev.Environment(tmp_path) as e:\n e.concretize()\n\n\ndef test_only_roots_are_explicitly_installed(tmp_path: pathlib.Path, config, temporary_store):\n \"\"\"When installing specific non-root specs from an environment, we continue to mark them\n as implicitly installed. What makes installs explicit is that they are root of the env.\"\"\"\n env = ev.create_in_dir(tmp_path)\n env.add(\"mpileaks\")\n env.concretize()\n mpileaks = env.concrete_roots()[0]\n callpath = mpileaks[\"callpath\"]\n env.install_specs([callpath], fake=True)\n assert callpath in temporary_store.db.query(explicit=False)\n env.install_specs([mpileaks], fake=True)\n assert temporary_store.db.query(explicit=True) == [mpileaks]\n\n\ndef test_environment_from_name_or_dir(mutable_mock_env_path):\n test_env = ev.create(\"test\")\n\n name_env = ev.environment_from_name_or_dir(test_env.name)\n assert name_env.name == test_env.name\n assert name_env.path == test_env.path\n\n dir_env = ev.environment_from_name_or_dir(test_env.path)\n assert dir_env.name == test_env.name\n assert dir_env.path == test_env.path\n\n nested_test_env = ev.create(\"group/test\")\n\n nested_name_env = ev.environment_from_name_or_dir(nested_test_env.name)\n assert nested_name_env.name == nested_test_env.name\n assert nested_name_env.path == nested_test_env.path\n\n nested_dir_env = ev.environment_from_name_or_dir(nested_test_env.path)\n assert nested_dir_env.name == nested_test_env.name\n assert nested_dir_env.path == nested_test_env.path\n\n with pytest.raises(ev.SpackEnvironmentError, match=\"no such environment\"):\n _ = ev.environment_from_name_or_dir(\"fake-env\")\n\n\ndef test_env_include_configs(mutable_mock_env_path):\n \"\"\"check config and package values using new include schema\"\"\"\n env_path = mutable_mock_env_path\n env_path.mkdir()\n\n this_os = spack.platforms.host().default_os\n config_root = env_path / this_os\n config_root.mkdir()\n config_path = str(config_root / \"config.yaml\")\n with open(config_path, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nconfig:\n verify_ssl: False\n\"\"\"\n )\n\n packages_path = str(env_path / \"packages.yaml\")\n with open(packages_path, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\npackages:\n python:\n require:\n - spec: \"@3.11:\"\n\"\"\"\n )\n\n spack_yaml = env_path / ev.manifest_name\n spack_yaml.write_text(\n f\"\"\"\\\nspack:\n include:\n - path: {config_path}\n optional: true\n - path: {packages_path}\n\"\"\"\n )\n\n e = ev.Environment(env_path)\n with e.manifest.use_config():\n assert not spack.config.get(\"config:verify_ssl\")\n python_reqs = spack.config.get(\"packages\")[\"python\"][\"require\"]\n req_specs = set(x[\"spec\"] for x in python_reqs)\n assert req_specs == set([\"@3.11:\"])\n\n\ndef test_using_multiple_compilers_on_a_node_is_discouraged(tmp_path: pathlib.Path, mutable_config):\n \"\"\"Tests that when we specify % Spack tries to use that compiler for all the\n languages needed by that node.\n \"\"\"\n manifest = tmp_path / \"spack.yaml\"\n manifest.write_text(\n \"\"\"\\\nspack:\n specs:\n - mpileaks%clang ^mpich%gcc\n concretizer:\n unify: true\n\"\"\"\n )\n with ev.Environment(tmp_path) as e:\n e.concretize()\n mpileaks = e.concrete_roots()[0]\n\n assert not mpileaks.satisfies(\"%gcc\") and mpileaks.satisfies(\"%clang\")\n assert len(mpileaks.dependencies(virtuals=(\"c\", \"cxx\"))) == 1\n\n mpich = mpileaks[\"mpich\"]\n assert mpich.satisfies(\"%gcc\") and not mpich.satisfies(\"%clang\")\n assert len(mpich.dependencies(virtuals=(\"c\", \"cxx\"))) == 1\n\n\n@pytest.mark.parametrize(\n [\"spack_yaml\", \"expected\", \"not_expected\"],\n [\n # Define a toolchain in spack.yaml\n (\n \"\"\"\\\nspack:\n specs:\n - mpileaks %llvm-toolchain\n toolchains:\n llvm-toolchain:\n - spec: \"%[virtuals=c] llvm\"\n when: \"%c\"\n - spec: \"%[virtuals=cxx] llvm\"\n when: \"%cxx\"\n concretizer:\n unify: true\n\"\"\",\n [\"%[virtuals=c] llvm\", \"^[virtuals=mpi] mpich\"],\n [\"%[virtuals=c] gcc\"],\n ),\n # Use a toolchain in a default requirement\n (\n \"\"\"\\\n spack:\n specs:\n - mpileaks\n toolchains:\n llvm-toolchain:\n - spec: \"%[virtuals=c] llvm\"\n when: \"%c\"\n - spec: \"%[virtuals=cxx] llvm\"\n when: \"%cxx\"\n - spec: \"%[virtuals=mpi] zmpi\"\n when: \"%mpi\"\n packages:\n all:\n require:\n - \"%llvm-toolchain\"\n concretizer:\n unify: true\n \"\"\",\n [\"%[virtuals=c] llvm\", \"%[virtuals=mpi] zmpi\", \"^callpath %[virtuals=c] llvm\"],\n [\"%[virtuals=c] gcc\"],\n ),\n ],\n)\ndef test_toolchain_definitions_are_allowed(\n spack_yaml, expected, not_expected, tmp_path: pathlib.Path, mutable_config\n):\n \"\"\"Tests that we can use toolchain definitions in spack.yaml files.\"\"\"\n manifest = tmp_path / \"spack.yaml\"\n manifest.write_text(spack_yaml)\n with ev.Environment(tmp_path) as e:\n e.concretize()\n mpileaks = e.concrete_roots()[0]\n\n for c in expected:\n assert mpileaks.satisfies(c)\n\n for c in not_expected:\n assert not mpileaks.satisfies(c)\n\n\nMIXED_TOOLCHAIN = \"\"\"\n - spec: \"%[virtuals=c] llvm\"\n when: \"%c\"\n - spec: \"%[virtuals=cxx] llvm\"\n when: \"%cxx\"\n - spec: \"%[virtuals=fortran] gcc\"\n when: \"%fortran\"\n - spec: \"%[virtuals=mpi] mpich\"\n when: \"%mpi\"\n\"\"\"\n\n\n@pytest.mark.parametrize(\"unify\", [\"true\", \"false\", \"when_possible\"])\ndef test_single_toolchain_and_matrix(unify, tmp_path: pathlib.Path, mutable_config):\n \"\"\"Tests that toolchains can be used with matrices in environments\"\"\"\n spack_yaml = f\"\"\"\nspack:\n specs:\n - matrix:\n - [mpileaks, dt-diamond-right]\n - [\"%mixed-toolchain\"]\n toolchains:\n mixed-toolchain:\n {MIXED_TOOLCHAIN}\n concretizer:\n unify: {unify}\n\"\"\"\n manifest = tmp_path / \"spack.yaml\"\n manifest.write_text(spack_yaml)\n with ev.Environment(tmp_path) as e:\n e.concretize()\n roots = e.concrete_roots()\n\n expected = [\n \"%[when='%c' virtuals=c] llvm\",\n \"%[when='%cxx' virtuals=cxx] llvm\",\n \"%[when='%fortran' virtuals=fortran] gcc\",\n \"%[when='%mpi' virtuals=mpi] mpich\",\n ]\n for c in expected:\n assert all(s.satisfies(c) for s in roots)\n\n not_expected = [\"^zmpi\", \"%[virtuals=c] gcc\"]\n for c in not_expected:\n assert all(not s.satisfies(c) for s in roots)\n\n\nGCC_ZMPI = \"\"\"\n - spec: \"%[virtuals=c] gcc\"\n when: \"%c\"\n - spec: \"%[virtuals=cxx] gcc\"\n when: \"%cxx\"\n - spec: \"%[virtuals=fortran] gcc\"\n when: \"%fortran\"\n - spec: \"%[virtuals=mpi] zmpi\"\n when: \"%mpi\"\n\"\"\"\n\n\n@pytest.mark.parametrize(\"unify\", [\"false\", \"when_possible\"])\ndef test_toolchains_as_matrix_dimension(unify, tmp_path: pathlib.Path, mutable_config):\n \"\"\"Tests expanding a matrix using different toolchains as the last dimension\"\"\"\n spack_yaml = f\"\"\"\nspack:\n specs:\n - matrix:\n - [mpileaks, dt-diamond-right]\n - [\"%mixed-toolchain\", \"%gcc-zmpi\"]\n toolchains:\n mixed-toolchain:\n {MIXED_TOOLCHAIN}\n gcc-zmpi:\n {GCC_ZMPI}\n concretizer:\n unify: {unify}\n\"\"\"\n manifest = tmp_path / \"spack.yaml\"\n manifest.write_text(spack_yaml)\n with ev.Environment(tmp_path) as e:\n e.concretize()\n roots = e.concrete_roots()\n\n mpileaks_gcc = [s for s in roots if s.satisfies(\"mpileaks %[virtuals=c] gcc\")][0]\n mpileaks_clang = [s for s in roots if s.satisfies(\"mpileaks %[virtuals=c] clang\")][0]\n\n # GCC-MPICH toolchain\n assert not mpileaks_gcc.satisfies(\"%[virtuals=mpi] mpich\")\n assert mpileaks_gcc.satisfies(\"%[virtuals=mpi] zmpi\")\n\n # Mixed toolchain\n assert mpileaks_clang.satisfies(\"%[virtuals=mpi] mpich\")\n assert not mpileaks_clang.satisfies(\"%[virtuals=mpi] zmpi\")\n assert mpileaks_clang[\"mpich\"].satisfies(\"%[virtuals=fortran] gcc\")\n\n\n@pytest.mark.parametrize(\"unify\", [\"true\", \"false\", \"when_possible\"])\n@pytest.mark.parametrize(\"requirement_type\", [\"require\", \"prefer\"])\ndef test_using_toolchain_as_requirement(\n unify, requirement_type, tmp_path: pathlib.Path, mutable_config\n):\n \"\"\"Tests using a toolchain as a default requirement in an environment\"\"\"\n spack_yaml = f\"\"\"\nspack:\n specs:\n - mpileaks\n - dt-diamond-right\n toolchains:\n mixed-toolchain:\n {MIXED_TOOLCHAIN}\n packages:\n all:\n {requirement_type}:\n - \"%mixed-toolchain\"\n concretizer:\n unify: {unify}\n\"\"\"\n manifest = tmp_path / \"spack.yaml\"\n manifest.write_text(spack_yaml)\n with ev.Environment(tmp_path) as e:\n e.concretize()\n roots = e.concrete_roots()\n\n mpileaks = [s for s in roots if s.satisfies(\"mpileaks\")][0]\n\n assert mpileaks.satisfies(\"%[virtuals=mpi] mpich\")\n assert mpileaks.satisfies(\"^[virtuals=mpi] mpich\")\n\n mpich = mpileaks[\"mpi\"]\n assert mpich.satisfies(\"%[virtuals=c] llvm\")\n assert mpich.satisfies(\"%[virtuals=cxx] llvm\")\n assert mpich.satisfies(\"%[virtuals=fortran] gcc\")\n\n\n@pytest.mark.parametrize(\"unify\", [\"false\", \"when_possible\"])\ndef test_using_toolchain_as_preferences(unify, tmp_path: pathlib.Path, mutable_config):\n \"\"\"Tests using a toolchain as a strong preference in an environment\"\"\"\n spack_yaml = f\"\"\"\nspack:\n specs:\n - dt-diamond-right %gcc-zmpi\n toolchains:\n mixed-toolchain:\n {MIXED_TOOLCHAIN}\n gcc-zmpi:\n {GCC_ZMPI}\n packages:\n all:\n prefer:\n - \"%mixed-toolchain\"\n concretizer:\n unify: {unify}\n\"\"\"\n manifest = tmp_path / \"spack.yaml\"\n manifest.write_text(spack_yaml)\n with ev.Environment(tmp_path) as e:\n e.concretize()\n roots = e.concrete_roots()\n\n dt = [s for s in roots if s.satisfies(\"dt-diamond-right\")][0]\n assert dt.satisfies(\"%[virtuals=c] gcc\")\n\n\n@pytest.mark.parametrize(\"unify\", [\"true\", \"false\", \"when_possible\"])\ndef test_mixing_toolchains_in_an_input_spec(unify, tmp_path: pathlib.Path, mutable_config):\n \"\"\"Tests using a toolchain as a strong preference in an environment\"\"\"\n spack_yaml = f\"\"\"\nspack:\n specs:\n - mpileaks %mixed-toolchain ^libelf %gcc-zmpi\n toolchains:\n mixed-toolchain:\n {MIXED_TOOLCHAIN}\n gcc-zmpi:\n {GCC_ZMPI}\n concretizer:\n unify: {unify}\n\"\"\"\n manifest = tmp_path / \"spack.yaml\"\n manifest.write_text(spack_yaml)\n with ev.Environment(tmp_path) as e:\n e.concretize()\n roots = e.concrete_roots()\n\n mpileaks = [s for s in roots if s.satisfies(\"mpileaks\")][0]\n assert mpileaks.satisfies(\"%[virtuals=mpi] mpich\")\n assert mpileaks.satisfies(\"^[virtuals=mpi] mpich\")\n assert mpileaks.satisfies(\"%[virtuals=c] llvm\")\n\n libelf = mpileaks[\"libelf\"]\n assert libelf.satisfies(\"%[virtuals=c] gcc\") # libelf only depends on c\n\n\ndef test_reuse_environment_dependencies(tmp_path: pathlib.Path, mutable_config):\n \"\"\"Tests reusing specs from a separate, and concrete, environment.\"\"\"\n base = tmp_path / \"base\"\n base.mkdir()\n\n # Concretize the first environment asking for a non-default spec. In this way we'll know\n # that reuse from the derived environment is not accidental.\n manifest_base = base / \"spack.yaml\"\n manifest_base.write_text(\n \"\"\"\nspack:\n specs:\n - pkg-a@1.0\n packages:\n pkg-b:\n require:\n - \"@0.9\"\n\"\"\"\n )\n with ev.Environment(base) as e:\n e.concretize()\n # We need the spack.lock for reuse in the derived environment\n e.write(regenerate=False)\n base_pkga = e.concrete_roots()[0]\n\n # Create a second environment, reuse from the previous one and check pkg-a is the same\n derived = tmp_path / \"derived\"\n derived.mkdir()\n manifest_derived = derived / \"spack.yaml\"\n manifest_derived.write_text(\n f\"\"\"\nspack:\n specs:\n - pkg-a\n concretizer:\n reuse:\n from:\n - type: environment\n path: {base}\n\"\"\"\n )\n with ev.Environment(derived) as e:\n e.concretize()\n derived_pkga = e.concrete_roots()[0]\n\n assert base_pkga.dag_hash() == derived_pkga.dag_hash()\n\n\n@pytest.mark.parametrize(\n \"spack_yaml\",\n [\n # Use a plain requirement for callpath\n \"\"\"\nspack:\n specs:\n - mpileaks %%c,cxx=gcc\n - mpileaks %%c,cxx=llvm\n packages:\n callpath:\n require:\n - \"%c=gcc\"\n concretizer:\n unify: false\n\"\"\",\n # Propagate a toolchain\n \"\"\"\nspack:\n specs:\n - mpileaks %%c,cxx=gcc\n - mpileaks %%llvm_toolchain\n toolchains:\n llvm_toolchain:\n - spec: \"%c=llvm\"\n when: \"%c\"\n - spec: \"%cxx=llvm\"\n when: \"%cxx\"\n packages:\n callpath:\n require:\n - \"%c=gcc\"\n concretizer:\n unify: false\n\"\"\",\n # Override callpath from input spec\n \"\"\"\nspack:\n specs:\n - mpileaks %%c,cxx=gcc ^callpath %c=gcc\n - mpileaks %%llvm_toolchain ^callpath %c=gcc\n toolchains:\n llvm_toolchain:\n - spec: \"%c=llvm\"\n when: \"%c\"\n - spec: \"%cxx=llvm\"\n when: \"%cxx\"\n concretizer:\n unify: false\n\"\"\",\n ],\n)\ndef test_dependency_propagation_in_environments(spack_yaml, tmp_path, mutable_config):\n \"\"\"Tests that we can enforce compiler preferences using %% in environments.\"\"\"\n manifest = tmp_path / \"spack.yaml\"\n manifest.write_text(spack_yaml)\n with ev.Environment(tmp_path) as e:\n e.concretize()\n roots = e.concrete_roots()\n\n mpileaks_gcc = [s for s in roots if s.satisfies(\"mpileaks %c=gcc\")][0]\n for c in (\"%[when=%c]c=gcc\", \"%[when=%cxx]cxx=gcc\"):\n assert all(x.satisfies(c) for x in mpileaks_gcc.traverse() if x.name != \"callpath\")\n\n mpileaks_llvm = [s for s in roots if s.satisfies(\"mpileaks %c=llvm\")][0]\n for c in (\"%[when=%c]c=llvm\", \"%[when=%cxx]cxx=llvm\"):\n assert all(x.satisfies(c) for x in mpileaks_llvm.traverse() if x.name != \"callpath\")\n\n assert mpileaks_gcc[\"callpath\"].satisfies(\"%c=gcc\")\n assert mpileaks_llvm[\"callpath\"].satisfies(\"%c=gcc\")\n\n\n@pytest.mark.parametrize(\n \"spack_yaml,exception_nodes\",\n [\n # trilinos and its link/run subdag are compiled with clang, all other nodes use gcc\n (\n \"\"\"\nspack:\n specs:\n - trilinos %%c,cxx=clang\n packages:\n c:\n prefer:\n - gcc\n cxx:\n prefer:\n - gcc\n\"\"\",\n set(),\n ),\n # callpath and its link/run subdag are compiled with clang, all other nodes use gcc\n (\n \"\"\"\nspack:\n specs:\n - trilinos ^callpath %%c,cxx=clang\n packages:\n c:\n prefer:\n - gcc\n cxx:\n prefer:\n - gcc\n\"\"\",\n {\"trilinos\", \"mpich\", \"py-numpy\"},\n ),\n # trilinos and its link/run subdag, with the exception of mpich, are compiled with clang.\n # All other nodes use gcc.\n (\n \"\"\"\nspack:\n specs:\n - trilinos %%c,cxx=clang ^mpich %c=gcc\n packages:\n c:\n prefer:\n - gcc\n cxx:\n prefer:\n - gcc\n\"\"\",\n {\"mpich\"},\n ),\n (\n \"\"\"\nspack:\n specs:\n - trilinos %%c,cxx=clang\n packages:\n c:\n prefer:\n - gcc\n cxx:\n prefer:\n - gcc\n mpich:\n require:\n - \"%c=gcc\"\n\"\"\",\n {\"mpich\"},\n ),\n ],\n)\ndef test_double_percent_semantics(spack_yaml, exception_nodes, tmp_path, mutable_config):\n \"\"\"Tests semantics of %% in environments, when combined with other features.\n\n The test assumes clang is the propagated compiler, and gcc is the preferred compiler.\n \"\"\"\n manifest = tmp_path / \"spack.yaml\"\n manifest.write_text(spack_yaml)\n with ev.Environment(tmp_path) as e:\n e.concretize()\n trilinos = e.concrete_roots()[0]\n\n runtime_nodes = [\n x for x in trilinos.traverse(deptype=(\"link\", \"run\")) if x.name not in exception_nodes\n ]\n remaining_nodes = [x for x in trilinos.traverse() if x not in runtime_nodes]\n\n for x in runtime_nodes:\n error_msg = f\"\\n{x.tree()} does not use clang while expected to\"\n assert x.satisfies(\"%[when=%c]c=clang %[when=%cxx]cxx=clang\"), error_msg\n\n for x in remaining_nodes:\n error_msg = f\"\\n{x.tree()} does not use gcc while expected to\"\n assert x.satisfies(\"%[when=%c]c=gcc %[when=%cxx]cxx=gcc\"), error_msg\n\n\ndef test_cannot_use_double_percent_with_require(tmp_path, mutable_config):\n \"\"\"Tests that %% cannot be used with a requirement on languages, since they'll conflict.\"\"\"\n # trilinos wants to use clang, but we require gcc, so Spack will error\n spack_yaml = \"\"\"\nspack:\n specs:\n - trilinos %%c,cxx=clang\n packages:\n c:\n require:\n - gcc\n cxx:\n require:\n - gcc\n\"\"\"\n manifest = tmp_path / \"spack.yaml\"\n manifest.write_text(spack_yaml)\n with ev.Environment(tmp_path) as e:\n with pytest.raises(spack.solver.asp.UnsatisfiableSpecError, match=\"failed to concretize\"):\n e.concretize()\n\n\n@pytest.mark.parametrize(\n \"spack_yaml\",\n [\n # Specs with reuse on\n \"\"\"\nspack:\n specs:\n - trilinos\n - mpileaks\n concretizer:\n reuse: true\n\"\"\",\n # Package with conditional dependency\n \"\"\"\nspack:\n specs:\n - ascent+adios2\n - fftw+mpi\n\"\"\",\n \"\"\"\nspack:\n specs:\n - ascent~adios2\n - fftw~mpi\n\"\"\",\n \"\"\"\nspack:\n specs:\n - ascent+adios2\n - fftw~mpi\n\"\"\",\n ],\n)\ndef test_static_analysis_in_environments(spack_yaml, tmp_path, mutable_config):\n \"\"\"Tests that concretizations with and without static analysis produce the same results.\"\"\"\n manifest = tmp_path / \"spack.yaml\"\n manifest.write_text(spack_yaml)\n with ev.Environment(tmp_path) as e:\n e.concretize()\n no_static_analysis = {x.dag_hash() for x in e.concrete_roots()}\n\n mutable_config.set(\"concretizer:static_analysis\", True)\n with ev.Environment(tmp_path) as e:\n e.concretize()\n static_analysis = {x.dag_hash() for x in e.concrete_roots()}\n\n assert no_static_analysis == static_analysis\n\n\n@pytest.mark.regression(\"51606\")\ndef test_ids_when_using_toolchain_twice_in_a_spec(tmp_path, mutable_config):\n \"\"\"Tests that using the same toolchain twice in a spec constructs different objects\"\"\"\n spack_yaml = \"\"\"\nspack:\n toolchains:\n llvmtc:\n - spec: \"%c=llvm\"\n when: \"%c\"\n - spec: \"%cxx=llvm\"\n when: \"%cxx\"\n gnu:\n - spec: \"%c=gcc@10\"\n when: \"%c\"\n - spec: \"%cxx=gcc@10\"\n when: \"%cxx\"\n # This is missing the conditional when= on purpose\n - spec: \"%fortran=gcc@10\"\n\"\"\"\n manifest = tmp_path / \"spack.yaml\"\n manifest.write_text(spack_yaml)\n with ev.Environment(tmp_path):\n # We rely on this behavior when emitting facts for the solver\n toolchains = spack.config.CONFIG.get(\"toolchains\", {})\n s = spack.spec_parser.parse(\"mpileaks %gnu ^callpath %gnu\", toolchains=toolchains)[0]\n assert id(s[\"gcc\"]) != id(s[\"callpath\"][\"gcc\"])\n\n\ndef test_installed_specs_disregards_deprecation(tmp_path, mutable_config):\n \"\"\"Tests that installed specs disregard deprecation. This is to avoid weird ordering issues,\n where an old version that _is not_ declared in package.py is considered as _not_ deprecated,\n and is preferred to a newer version that is explicitly marked as deprecated.\n \"\"\"\n spack_yaml = \"\"\"\nspack:\n specs:\n - mpileaks\n packages:\n c:\n require:\n - gcc\n cxx:\n require:\n - gcc\n gcc::\n externals:\n - spec: gcc@7.3.1 languages:='c,c++,fortran'\n prefix: /path\n extra_attributes:\n compilers:\n c: /path/bin/gcc\n cxx: /path/bin/g++\n fortran: /path/bin/gfortran\n - spec: gcc@=12.4.0 languages:='c,c++,fortran'\n prefix: /usr\n extra_attributes:\n compilers:\n c: /usr/bin/gcc\n cxx: /usr/bin/g++\n fortran: /usr/bin/gfortran\n\"\"\"\n manifest = tmp_path / \"spack.yaml\"\n manifest.write_text(spack_yaml)\n with ev.Environment(tmp_path) as e:\n e.concretize()\n mpileaks = e.concrete_roots()[0]\n\n for node in mpileaks.traverse():\n if node.satisfies(\"%c\"):\n assert node.satisfies(\"%c=gcc@12\"), node.tree()\n assert not node.satisfies(\"%c=gcc@7\"), node.tree()\n\n\n@pytest.fixture()\ndef create_temporary_manifest(tmp_path):\n manifest_path = tmp_path / \"spack.yaml\"\n\n def _create(spack_yaml: str):\n manifest_path.write_text(spack_yaml)\n return EnvironmentManifestFile(tmp_path)\n\n return _create\n\n\n@pytest.mark.usefixtures(\"mutable_config\")\nclass TestEnvironmentGroups:\n \"\"\"Tests for the environment \"groups\" feature\"\"\"\n\n def test_manifest_and_groups(self, create_temporary_manifest):\n \"\"\"Tests a basic case of reading groups from a manifest file\"\"\"\n manifest = create_temporary_manifest(\n \"\"\"\n spack:\n specs:\n - mpileaks\n - group: compiler\n matrix:\n - [gcc@14]\n - group: apps\n needs: [compiler]\n specs:\n - matrix:\n - [mpileaks]\n - [\"%gcc@14\"]\n - mpich\n - libelf\n \"\"\"\n )\n # Check manifest properties\n assert set(manifest.groups()) == {\"default\", \"compiler\", \"apps\"}\n\n assert manifest.user_specs(group=\"default\") == manifest.user_specs()\n assert manifest.user_specs() == [\"mpileaks\", \"libelf\"]\n assert manifest.user_specs(group=\"compiler\") == [{\"matrix\": [[\"gcc@14\"]]}]\n assert manifest.user_specs(group=\"apps\") == [\n {\"matrix\": [[\"mpileaks\"], [\"%gcc@14\"]]},\n \"mpich\",\n ]\n\n assert manifest.needs(group=\"default\") == ()\n assert manifest.needs(group=\"compiler\") == ()\n assert manifest.needs(group=\"apps\") == (\"compiler\",)\n\n # Check user specs within the environment\n e = ev.Environment(manifest.manifest_dir)\n assert e.user_specs.specs == [spack.spec.Spec(\"mpileaks\"), spack.spec.Spec(\"libelf\")]\n\n compiler_specs = e.user_specs_by(group=\"compiler\")\n assert compiler_specs.name == \"specs:compiler\"\n assert compiler_specs.specs == [spack.spec.Spec(\"gcc@14\")]\n\n apps_specs = e.user_specs_by(group=\"apps\")\n assert apps_specs.name == \"specs:apps\"\n assert apps_specs.specs == [spack.spec.Spec(\"mpileaks %gcc@14\"), spack.spec.Spec(\"mpich\")]\n\n def test_cannot_define_group_twice(self, create_temporary_manifest):\n \"\"\"Tests that defining the same group twice raises an error\"\"\"\n with pytest.raises(SpackEnvironmentConfigError, match=\"defined more than once\"):\n create_temporary_manifest(\n \"\"\"\n spack:\n specs:\n - group: compiler\n matrix:\n - [gcc@14]\n - group: compiler\n matrix:\n - [llvm@20]\n\"\"\"\n )\n\n def test_matrix_can_be_expanded_in_groups(self, create_temporary_manifest):\n \"\"\"Tests that definitions can be expanded also for matrix groups\"\"\"\n manifest = create_temporary_manifest(\n \"\"\"\nspack:\n definitions:\n - compilers: [\"%gcc\", \"%clang\"]\n - desired_specs: [\"mpileaks@2.1\"]\n specs:\n - group: apps\n specs:\n - matrix:\n - [$desired_specs]\n - [$compilers]\n - mpich\n\"\"\"\n )\n e = ev.Environment(manifest.manifest_dir)\n assert e.user_specs.specs == []\n assert e.user_specs_by(group=\"apps\").specs == [\n spack.spec.Spec(\"mpileaks@2.1 %gcc\"),\n spack.spec.Spec(\"mpileaks@2.1 %clang\"),\n spack.spec.Spec(\"mpich\"),\n ]\n\n def test_environment_without_groups_use_lockfile_v6(self, create_temporary_manifest):\n manifest = create_temporary_manifest(\n \"\"\"\nspack:\n specs:\n - mpileaks\n - pkg-a\n\"\"\"\n )\n with ev.Environment(manifest.manifest_dir) as e:\n e.concretize()\n lockfile_data = e._to_lockfile_dict()\n assert lockfile_data[\"_meta\"][\"lockfile-version\"] == 6\n assert all(\"group\" not in x for x in lockfile_data[\"roots\"])\n\n def test_independent_groups_concretization(self, create_temporary_manifest):\n \"\"\"Tests that groups of specs without dependencies among them can be concretized\n correctly\n \"\"\"\n manifest = create_temporary_manifest(\n \"\"\"\n spack:\n specs:\n - mpileaks\n - group: compiler\n matrix:\n - [gcc@14]\n - libelf\n \"\"\"\n )\n\n with ev.Environment(manifest.manifest_dir) as e:\n e.concretize()\n roots = e.concrete_roots()\n assert len(roots) == 3\n\n default_specs = list(e.concretized_specs_by(group=\"default\"))\n assert len(default_specs) == 2\n\n compiler_specs = list(e.concretized_specs_by(group=\"compiler\"))\n assert len(compiler_specs) == 1\n\n def test_independent_group_dont_reuse(self, create_temporary_manifest):\n \"\"\"Tests that there is no cross-groups reuse among groups of specs without dependencies.\"\"\"\n manifest = create_temporary_manifest(\n \"\"\"\n spack:\n specs:\n - mpileaks@2.2\n - group: app\n matrix:\n - [mpileaks]\n \"\"\"\n )\n\n with ev.Environment(manifest.manifest_dir) as e:\n e.concretize()\n\n _, default_mpileaks = list(e.concretized_specs_by(group=\"default\"))[0]\n assert default_mpileaks.satisfies(\"@2.2\")\n\n _, app_mpileaks = list(e.concretized_specs_by(group=\"app\"))[0]\n assert app_mpileaks.satisfies(\"@2.3\")\n\n def test_relying_on_a_dependency_group(self, create_temporary_manifest):\n \"\"\"Tests that a group of specs that would not concretize without a dependency group\n works correctly.\n \"\"\"\n manifest = create_temporary_manifest(\n \"\"\"\n spack:\n specs:\n - group: app\n matrix:\n - [mpileaks]\n - [\"%c,cxx=gcc@14\"]\n \"\"\"\n )\n\n # We have no gcc@14 configured, so this will raise an error\n with ev.Environment(manifest.manifest_dir) as e:\n with pytest.raises(spack.solver.asp.UnsatisfiableSpecError):\n e.concretize()\n\n manifest = create_temporary_manifest(\n \"\"\"\n spack:\n specs:\n - group: compiler\n specs:\n - gcc@14\n - group: mpileaks\n needs: [compiler]\n matrix:\n - [mpileaks]\n - [\"%c,cxx=gcc@14\"]\n \"\"\"\n )\n\n # In this case gcc@14 is taken from the \"needed\" group\n with ev.Environment(manifest.manifest_dir) as e:\n e.concretize()\n\n _, gcc = next(iter(e.concretized_specs_by(group=\"compiler\")))\n assert gcc.satisfies(\"gcc@14\")\n _, mpileaks = next(iter(e.concretized_specs_by(group=\"mpileaks\")))\n assert mpileaks[\"c\"].dag_hash() == gcc.dag_hash()\n\n def test_manifest_can_contain_config_override(self, mutable_config, create_temporary_manifest):\n manifest = create_temporary_manifest(\n \"\"\"\n spack:\n concretizer:\n unify: False\n specs:\n - group: compiler\n override:\n concretizer:\n unify: True\n \"\"\"\n )\n\n with ev.Environment(manifest.manifest_dir) as e:\n assert mutable_config.get_config(\"concretizer\")[\"unify\"] is False\n\n # Assert the internal scope works when used manually\n override = manifest.config_override(group=\"compiler\")\n mutable_config.push_scope(\n override, priority=ConfigScopePriority.ENVIRONMENT_SPEC_GROUPS\n )\n assert mutable_config.get_config(\"concretizer\")[\"unify\"] is True\n mutable_config.remove_scope(override.name)\n assert mutable_config.get_config(\"concretizer\")[\"unify\"] is False\n\n # Assert the context manager works too\n with e.config_override_for_group(group=\"compiler\"):\n assert mutable_config.get_config(\"concretizer\")[\"unify\"] is True\n assert mutable_config.get_config(\"concretizer\")[\"unify\"] is False\n\n def test_overriding_concretization_properties_per_group(self, create_temporary_manifest):\n manifest = create_temporary_manifest(\n \"\"\"\n spack:\n concretizer:\n unify: True\n specs:\n - group: compiler\n specs:\n - gcc@14\n - group: scalapacks\n needs: [compiler]\n matrix:\n - [netlib-scalapack]\n - [\"%mpi=mpich\", \"%mpi=mpich2\"]\n - [\"%lapack=openblas-with-lapack\", \"%lapack=netlib-lapack\"]\n override:\n concretizer:\n unify: False\n packages:\n c:\n prefer: [gcc@14]\n cxx:\n prefer: [gcc@14]\n fortran:\n prefer: [gcc@14]\n \"\"\"\n )\n\n with ev.Environment(manifest.manifest_dir) as e:\n e.concretize()\n\n assert len(list(e.concretized_specs_by(group=\"compiler\"))) == 1\n\n gcc = next(x for _, x in e.concretized_specs_by(group=\"compiler\"))\n assert gcc.satisfies(\"gcc@14\") and not gcc.external\n assert gcc.satisfies(\"%c,cxx=gcc\")\n gcc_hash = gcc.dag_hash()\n\n assert len(list(e.concretized_specs_by(group=\"scalapacks\"))) == 4\n scalapacks = [x for _, x in e.concretized_specs_by(group=\"scalapacks\")]\n for node in traverse_nodes(scalapacks, deptype=(\"link\", \"run\")):\n assert node.satisfies(f\"%[when=c]c=gcc/{gcc_hash}\")\n assert node.satisfies(f\"%[when=cxx]cxx=gcc/{gcc_hash}\")\n assert node.satisfies(f\"%[when=fortran]fortran=gcc/{gcc_hash}\")\n\n def test_missing_needs_group_gives_clear_error(self, create_temporary_manifest):\n \"\"\"Tests that referencing a non-existent group in 'needs' gives a clear error message\n that includes the name of the blocked group and the missing dependency.\n \"\"\"\n manifest = create_temporary_manifest(\n \"\"\"\nspack:\n specs:\n - group: apps\n needs: [nonexistent]\n specs:\n - mpileaks\n\"\"\"\n )\n with ev.Environment(manifest.manifest_dir) as e:\n with pytest.raises(\n ev.SpackEnvironmentConfigError, match=r\"but 'nonexistent' is not a defined group\"\n ):\n e.concretize()\n\n def test_cyclic_group_dependencies_give_clear_error(self, create_temporary_manifest):\n \"\"\"Tests that cyclic group dependencies give a clear error message that mentions\n the groups involved in the cycle.\n \"\"\"\n manifest = create_temporary_manifest(\n \"\"\"\nspack:\n specs:\n - group: alpha\n needs: [beta]\n specs:\n - mpileaks\n - group: beta\n needs: [alpha]\n specs:\n - zlib\n\"\"\"\n )\n with ev.Environment(manifest.manifest_dir) as e:\n with pytest.raises(ev.SpackEnvironmentConfigError, match=r\"among groups: alpha, beta\"):\n e.concretize()\n\n def test_from_lockfile_preserves_groups(self, tmp_path):\n \"\"\"Tests that EnvironmentManifestFile.from_lockfile reconstructs groups correctly\n from a v7 lockfile that contains group information in its roots.\n \"\"\"\n lockfile_data = {\n \"_meta\": {\"file-type\": \"spack-lockfile\", \"lockfile-version\": 7, \"specfile-version\": 5},\n \"roots\": [\n {\"hash\": \"aaa\", \"spec\": \"mpileaks\", \"group\": \"default\"},\n {\"hash\": \"bbb\", \"spec\": \"libelf\", \"group\": \"default\"},\n {\"hash\": \"ccc\", \"spec\": \"gcc@14\", \"group\": \"compilers\"},\n ],\n \"concrete_specs\": {},\n }\n lockfile_path = tmp_path / \"spack.lock\"\n lockfile_path.write_text(json.dumps(lockfile_data))\n\n manifest = EnvironmentManifestFile.from_lockfile(tmp_path)\n\n # The reconstructed manifest must have both groups\n assert set(manifest.groups()) == {\"default\", \"compilers\"}\n assert manifest.user_specs(group=\"default\") == [\"mpileaks\", \"libelf\"]\n assert manifest.user_specs(group=\"compilers\") == [\"gcc@14\"]\n\n def test_from_lockfile_without_groups_stays_default(self, tmp_path):\n \"\"\"Tests that a lockfile without group info (v6 and earlier) reconstructs all specs\n into the default group only.\n \"\"\"\n lockfile_data = {\n \"_meta\": {\"file-type\": \"spack-lockfile\", \"lockfile-version\": 6, \"specfile-version\": 5},\n \"roots\": [{\"hash\": \"aaa\", \"spec\": \"mpileaks\"}, {\"hash\": \"bbb\", \"spec\": \"libelf\"}],\n \"concrete_specs\": {},\n }\n lockfile_path = tmp_path / \"spack.lock\"\n lockfile_path.write_text(json.dumps(lockfile_data))\n\n manifest = EnvironmentManifestFile.from_lockfile(tmp_path)\n\n assert set(manifest.groups()) == {\"default\"}\n assert manifest.user_specs(group=\"default\") == [\"mpileaks\", \"libelf\"]\n\n\n@pytest.mark.regression(\"51995\")\ndef test_mixed_compilers_and_libllvm(tmp_path, config):\n \"\"\"Tests that we divide virtual nodes correctly among unification sets.\n\n This test concretizes a unified environment where one package uses gcc as a C++ compiler\n and depends on llvm as a provider of libllvm, while the other package uses llvm as a C++\n compiler.\n \"\"\"\n spack_yaml = \"\"\"\nspack:\n specs:\n - paraview %cxx=llvm\n - mesa %cxx=gcc %libllvm=llvm\n packages:\n c:\n prefer:\n - gcc\n cxx:\n prefer:\n - gcc\n gcc::\n externals:\n - spec: gcc@13.2.0 languages:='c,c++,fortran'\n prefix: /path\n extra_attributes:\n compilers:\n c: /path/bin/gcc\n cxx: /path/bin/g++\n fortran: /path/bin/gfortran\n llvm::\n externals:\n - spec: llvm@20.1.8+clang+flang+lld+lldb\n prefix: /usr\n extra_attributes:\n compilers:\n c: /usr/bin/gcc\n cxx: /usr/bin/g++\n fortran: /usr/bin/gfortran\n concretizer:\n unify: true\n\"\"\"\n manifest = tmp_path / \"spack.yaml\"\n manifest.write_text(spack_yaml)\n with ev.Environment(tmp_path) as e:\n e.concretize()\n\n for x in e.concrete_roots():\n if x.name == \"mesa\":\n mesa = x\n else:\n paraview = x\n\n assert paraview.satisfies(\"%cxx=llvm@20\")\n assert paraview.satisfies(f\"%{mesa}\")\n assert mesa.satisfies(\"%cxx=gcc %libllvm=llvm\")\n assert paraview[\"cxx\"].dag_hash() == mesa[\"libllvm\"].dag_hash()\n\n\n@pytest.mark.regression(\"51512\")\ndef test_unified_environment_with_mixed_compilers_and_fortran(tmp_path, config):\n \"\"\"Tests that we can concretize a unified environment using two C/C++ compilers for the root\n specs and GCC for Fortran, where both roots depend on Fortran.\n \"\"\"\n spack_yaml = \"\"\"\n spack:\n specs:\n - mpich %c,cxx=llvm\n - openblas %c,fortran=gcc\n packages:\n gcc::\n externals:\n - spec: gcc@13.2.0 languages:='c,c++,fortran'\n prefix: /path\n extra_attributes:\n compilers:\n c: /path/bin/gcc\n cxx: /path/bin/g++\n fortran: /path/bin/gfortran\n llvm::\n externals:\n - spec: llvm@20.1.8+clang~flang\n prefix: /usr\n extra_attributes:\n compilers:\n c: /usr/bin/gcc\n cxx: /usr/bin/g++\n fortran: /usr/bin/gfortran\n concretizer:\n unify: true\n \"\"\"\n manifest = tmp_path / \"spack.yaml\"\n manifest.write_text(spack_yaml)\n with ev.Environment(tmp_path) as e:\n e.concretize()\n\n for x in e.concrete_roots():\n if x.name == \"mpich\":\n mpich = x\n else:\n openblas = x\n\n assert mpich.satisfies(\"%c,cxx=llvm\")\n assert mpich.satisfies(\"%fortran=gcc\")\n assert openblas.satisfies(\"%c,fortran=gcc\")\n assert mpich[\"fortran\"].dag_hash() == openblas[\"fortran\"].dag_hash()\n" }, { "path": "lib/spack/spack/test/environment/mutate.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport platform\n\nimport pytest\n\nimport spack.concretize\nimport spack.config\nimport spack.environment as ev\nimport spack.spec\nfrom spack.main import SpackCommand\n\npytestmark = [\n pytest.mark.usefixtures(\"mutable_config\", \"mutable_mock_env_path\", \"mutable_mock_repo\"),\n pytest.mark.not_on_windows(\"Envs unsupported on Windows\"),\n]\n\n# See lib/spack/spack/platforms/test.py for how targets are defined on the Test platform\ntest_targets = (\"m1\", \"aarch64\") if platform.machine() == \"arm64\" else (\"core2\", \"x86_64\")\n\nchange = SpackCommand(\"change\")\n\n\n@pytest.mark.parametrize(\"dep\", [True, False])\n@pytest.mark.parametrize(\n \"orig_constraint,mutated_constraint\",\n [\n (\"@3.23.1\", \"@3.4.3\"),\n (\"cflags=-O3\", \"cflags='-O0 -g'\"),\n (\"os=debian6\", \"os=redhat6\"),\n (f\"target={test_targets[0]}\", f\"target={test_targets[1]}\"),\n (\"build_system=generic\", \"build_system=foo\"),\n (\n f\"@3.4.3 cflags=-g os=debian6 target={test_targets[1]} build_system=generic\",\n f\"@3.23.1 cflags=-O3 os=redhat6 target={test_targets[0]} build_system=foo\",\n ),\n ],\n)\ndef test_mutate_internals(dep, orig_constraint, mutated_constraint):\n \"\"\"\n Check that Environment.mutate and Spec.mutate work for several different constraint types.\n\n Includes check that environment.mutate rehashing gets the same answer as spec.mutate rehashing.\n \"\"\"\n ev.create(\"test\")\n env = ev.read(\"test\")\n\n spack.config.set(\"packages:cmake\", {\"require\": orig_constraint})\n\n root_name = \"cmake-client\" if dep else \"cmake\"\n env.add(root_name)\n env.concretize()\n\n root_spec = next(env.roots()).copy()\n cmake_spec = root_spec[\"cmake\"] if dep else root_spec\n orig_cmake_spec = cmake_spec.copy()\n orig_hash = root_spec.dag_hash()\n\n for spec in env.all_specs_generator():\n if spec.name == \"cmake\":\n assert spec.satisfies(orig_constraint)\n\n selector = spack.spec.Spec(\"cmake\")\n mutator = spack.spec.Spec(mutated_constraint)\n env.mutate(selector=selector, mutator=mutator)\n cmake_spec.mutate(mutator)\n\n for spec in env.all_specs_generator():\n if spec.name == \"cmake\":\n assert spec.satisfies(mutated_constraint)\n assert cmake_spec.satisfies(mutated_constraint)\n\n # Make sure that we're not changing variant types single/multi\n for name, variant in cmake_spec.variants.items():\n assert variant.type == orig_cmake_spec.variants[name].type\n\n new_hash = next(env.roots()).dag_hash()\n assert new_hash != orig_hash\n assert root_spec.dag_hash() != orig_hash\n assert root_spec.dag_hash() == new_hash\n\n\n@pytest.mark.parametrize(\"constraint\", [\"foo\", \"foo.bar\", \"foo%cmake@1.0\", \"foo@1.1:\", \"foo/abc\"])\ndef test_mutate_spec_invalid(constraint):\n spec = spack.concretize.concretize_one(\"cmake-client\")\n with pytest.raises(spack.spec.SpecMutationError):\n spec.mutate(spack.spec.Spec(constraint))\n\n\ndef _test_mutate_from_cli(args, create=True):\n if create:\n ev.create(\"test\")\n\n env = ev.read(\"test\")\n\n if create:\n env.add(\"cmake-client%cmake@3.4.3\")\n env.add(\"cmake-client%cmake@3.23.1\")\n env.concretize()\n env.write()\n\n with env:\n change(*args)\n\n return list(env.roots())\n\n\ndef test_mutate_from_cli():\n match_spec = \"%cmake@3.4.3\"\n constraint = \"@3.0\"\n args = [\"--concrete\", f\"--match-spec={match_spec}\", constraint]\n roots = _test_mutate_from_cli(args)\n\n assert any(r.satisfies(match_spec) for r in roots)\n for root in roots:\n if root.satisfies(\"match_spec\"):\n assert root.satisfies(constraint)\n\n\ndef test_mutate_from_cli_multiple():\n match_spec = \"%cmake@3.4.3\"\n constraint1 = \"@3.0\"\n constraint2 = \"build_system=foo\"\n args = [\"--concrete\", f\"--match-spec={match_spec}\", constraint1, constraint2]\n roots = _test_mutate_from_cli(args)\n\n assert any(r.satisfies(match_spec) for r in roots)\n for root in roots:\n if root.satisfies(\"match_spec\"):\n assert root.satisfies(constraint1)\n assert root.satisfies(constraint2)\n\n\ndef test_mutate_from_cli_no_abstract():\n match_spec = \"cmake\"\n constraint = \"@3.0\"\n args = [\"--concrete\", f\"--match-spec={match_spec}\", constraint]\n\n with pytest.raises(ValueError, match=\"Cannot change abstract spec\"):\n _ = _test_mutate_from_cli(args)\n\n args = [\"--concrete-only\"] + args[1:]\n roots = _test_mutate_from_cli(args, create=False)\n\n for root in roots:\n assert root[match_spec].satisfies(constraint)\n\n\ndef test_mutate_from_cli_all_no_match_spec():\n constraint = \"cmake-client@3.0\"\n args = [\"--concrete\", \"--all\", constraint]\n roots = _test_mutate_from_cli(args)\n\n for root in roots:\n assert root.satisfies(constraint)\n" }, { "path": "lib/spack/spack/test/environment_modifications.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport pathlib\nimport sys\n\nimport pytest\n\nimport spack.util.environment as environment\nfrom spack.paths import spack_root\nfrom spack.util.environment import (\n AppendPath,\n EnvironmentModifications,\n PrependPath,\n RemovePath,\n SetEnv,\n UnsetEnv,\n filter_system_paths,\n is_system_path,\n)\n\ndatadir = os.path.join(spack_root, \"lib\", \"spack\", \"spack\", \"test\", \"data\")\n\nshell_extension = \".bat\" if sys.platform == \"win32\" else \".sh\"\n\n# Returns a list of paths, including system ones\nmiscellaneous_unix_sys_paths = [\n \"/usr/include\",\n \"/usr/local/lib\",\n \"/usr/local\",\n \"/usr/local/include\",\n \"/usr/local/lib64\",\n \"/usr/local/../bin\",\n \"/lib\",\n \"/\",\n \"/usr\",\n \"/usr/\",\n \"/usr/bin\",\n \"/bin64\",\n \"/lib64\",\n \"/include\",\n \"/include/\",\n]\n\nmiscellaneous_win_sys_paths = [\n \"C:\\\\Users\",\n \"C:\\\\\",\n \"C:\\\\Program Files\",\n \"C:\\\\Program Files (x86)\",\n \"C:\\\\ProgramData\",\n]\n\nmiscellaneous_win_paths = [\"C:\\\\dev\\\\spack_window\"]\n\nmiscellaneous_unix_paths = [\n \"/usr/local/Cellar/gcc/5.3.0/lib\",\n \"/usr/local/opt/some-package/lib\",\n \"/usr/opt/lib\",\n \"/opt/some-package/include\",\n \"/opt/some-package/local/..\",\n]\n\n\ndef test_inspect_path(tmp_path: pathlib.Path):\n inspections = {\n \"bin\": [\"PATH\"],\n \"man\": [\"MANPATH\"],\n \"share/man\": [\"MANPATH\"],\n \"share/aclocal\": [\"ACLOCAL_PATH\"],\n \"lib\": [\"LIBRARY_PATH\", \"LD_LIBRARY_PATH\"],\n \"lib64\": [\"LIBRARY_PATH\", \"LD_LIBRARY_PATH\"],\n \"include\": [\"CPATH\"],\n \"lib/pkgconfig\": [\"PKG_CONFIG_PATH\"],\n \"lib64/pkgconfig\": [\"PKG_CONFIG_PATH\"],\n \"share/pkgconfig\": [\"PKG_CONFIG_PATH\"],\n \"\": [\"CMAKE_PREFIX_PATH\"],\n }\n\n (tmp_path / \"bin\").mkdir()\n (tmp_path / \"lib\").mkdir()\n (tmp_path / \"include\").mkdir()\n\n env = environment.inspect_path(str(tmp_path), inspections)\n names = [item.name for item in env]\n assert \"PATH\" in names\n assert \"LIBRARY_PATH\" in names\n assert \"LD_LIBRARY_PATH\" in names\n assert \"CPATH\" in names\n\n\ndef test_exclude_paths_from_inspection():\n inspections = {\n \"lib\": [\"LIBRARY_PATH\", \"LD_LIBRARY_PATH\"],\n \"lib64\": [\"LIBRARY_PATH\", \"LD_LIBRARY_PATH\"],\n \"include\": [\"CPATH\"],\n }\n\n env = environment.inspect_path(\"/usr\", inspections, exclude=is_system_path)\n\n assert len(env) == 0\n\n\ndef make_path(*path):\n \"\"\"Joins given components a,b,c... to form a valid absolute\n path for the current host OS being tested.\n Created path does not necessarily exist on system.\n \"\"\"\n abs_path_prefix = \"C:\\\\\" if sys.platform == \"win32\" else \"/\"\n return os.path.join(abs_path_prefix, *path)\n\n\ndef make_pathlist(paths):\n \"\"\"Makes a fake list of platform specific paths\"\"\"\n return os.pathsep.join(\n [make_path(*path) if isinstance(path, list) else make_path(path) for path in paths]\n )\n\n\n@pytest.fixture\ndef system_paths_for_os():\n return miscellaneous_win_sys_paths if sys.platform == \"win32\" else miscellaneous_unix_sys_paths\n\n\n@pytest.fixture\ndef non_system_paths_for_os():\n return miscellaneous_win_paths if sys.platform == \"win32\" else miscellaneous_unix_paths\n\n\n@pytest.fixture()\ndef prepare_environment_for_tests(working_env, system_paths_for_os):\n \"\"\"Sets a few dummy variables in the current environment, that will be\n useful for the tests below.\n \"\"\"\n os.environ[\"UNSET_ME\"] = \"foo\"\n os.environ[\"EMPTY_PATH_LIST\"] = \"\"\n os.environ[\"PATH_LIST\"] = make_pathlist([[\"path\", \"second\"], [\"path\", \"third\"]])\n os.environ[\"REMOVE_PATH_LIST\"] = make_pathlist(\n [\n [\"a\", \"b\"],\n [\"duplicate\"],\n [\"a\", \"c\"],\n [\"remove\", \"this\"],\n [\"a\", \"d\"],\n [\"duplicate\"],\n [\"f\", \"g\"],\n ]\n )\n # grab arbitrary system path\n sys_path = system_paths_for_os[0] + os.pathsep\n os.environ[\"PATH_LIST_WITH_SYSTEM_PATHS\"] = sys_path + os.environ[\"REMOVE_PATH_LIST\"]\n os.environ[\"PATH_LIST_WITH_DUPLICATES\"] = os.environ[\"REMOVE_PATH_LIST\"]\n\n\n@pytest.fixture\ndef env(prepare_environment_for_tests):\n \"\"\"Returns an empty EnvironmentModifications object.\"\"\"\n return EnvironmentModifications()\n\n\n@pytest.fixture\ndef files_to_be_sourced():\n \"\"\"Returns a list of files to be sourced\"\"\"\n return [\n os.path.join(datadir, \"sourceme_first\" + shell_extension),\n os.path.join(datadir, \"sourceme_second\" + shell_extension),\n os.path.join(datadir, \"sourceme_parameters\" + shell_extension),\n os.path.join(datadir, \"sourceme_unicode\" + shell_extension),\n ]\n\n\ndef test_set(env):\n \"\"\"Tests setting values in the environment.\"\"\"\n\n # Here we are storing the commands to set a couple of variables\n env.set(\"A\", \"dummy value\")\n env.set(\"B\", 3)\n\n # ...and then we are executing them\n env.apply_modifications()\n\n assert \"dummy value\" == os.environ[\"A\"]\n assert str(3) == os.environ[\"B\"]\n\n\ndef test_append_flags(env):\n \"\"\"Tests appending to a value in the environment.\"\"\"\n\n # Store a couple of commands\n env.append_flags(\"APPEND_TO_ME\", \"flag1\")\n env.append_flags(\"APPEND_TO_ME\", \"flag2\")\n\n # ... execute the commands\n env.apply_modifications()\n\n assert \"flag1 flag2\" == os.environ[\"APPEND_TO_ME\"]\n\n\ndef test_unset(env):\n \"\"\"Tests unsetting values in the environment.\"\"\"\n\n # Assert that the target variable is there and unset it\n assert \"foo\" == os.environ[\"UNSET_ME\"]\n env.unset(\"UNSET_ME\")\n env.apply_modifications()\n\n # Trying to retrieve is after deletion should cause a KeyError\n with pytest.raises(KeyError):\n os.environ[\"UNSET_ME\"]\n\n\ndef test_filter_system_paths(system_paths_for_os, non_system_paths_for_os):\n \"\"\"Tests that the filtering of system paths works as expected.\"\"\"\n filtered = filter_system_paths(system_paths_for_os + non_system_paths_for_os)\n assert filtered == non_system_paths_for_os\n\n\ndef test_set_path(env):\n \"\"\"Tests setting paths in an environment variable.\"\"\"\n name = \"A\"\n elements = [\"foo\", \"bar\", \"baz\"]\n # Check setting paths with a specific separator\n env.set_path(name, elements, separator=os.pathsep)\n env.apply_modifications()\n\n expected = os.pathsep.join(elements)\n assert expected == os.environ[name]\n\n\ndef test_path_manipulation(env):\n \"\"\"Tests manipulating list of paths in the environment.\"\"\"\n env.prepend_path(\"PATH_LIST\", make_path(\"path\", \"first\"))\n env.append_path(\"PATH_LIST\", make_path(\"path\", \"fourth\"))\n env.append_path(\"PATH_LIST\", make_path(\"path\", \"last\"))\n\n env.remove_path(\"REMOVE_PATH_LIST\", make_path(\"remove\", \"this\"))\n env.remove_path(\"REMOVE_PATH_LIST\", make_path(\"duplicate\") + os.sep)\n\n env.prune_duplicate_paths(\"PATH_LIST_WITH_DUPLICATES\")\n\n env.apply_modifications()\n\n assert os.environ[\"PATH_LIST\"] == make_pathlist(\n [\n [\"path\", \"first\"],\n [\"path\", \"second\"],\n [\"path\", \"third\"],\n [\"path\", \"fourth\"],\n [\"path\", \"last\"],\n ]\n )\n assert os.environ[\"REMOVE_PATH_LIST\"] == make_pathlist(\n [[\"a\", \"b\"], [\"a\", \"c\"], [\"a\", \"d\"], [\"f\", \"g\"]]\n )\n\n assert os.environ[\"PATH_LIST_WITH_DUPLICATES\"].count(make_path(\"duplicate\")) == 1\n\n\n@pytest.mark.not_on_windows(\"Skip unix path tests on Windows\")\ndef test_unix_system_path_manipulation(env):\n \"\"\"Tests manipulting paths that have special meaning as system paths on Unix\"\"\"\n env.deprioritize_system_paths(\"PATH_LIST_WITH_SYSTEM_PATHS\")\n env.apply_modifications()\n\n assert not os.environ[\"PATH_LIST_WITH_SYSTEM_PATHS\"].startswith(\n make_pathlist([[\"usr\", \"include\" + os.pathsep]])\n )\n assert os.environ[\"PATH_LIST_WITH_SYSTEM_PATHS\"].endswith(make_pathlist([[\"usr\", \"include\"]]))\n\n\n@pytest.mark.skipif(sys.platform != \"win32\", reason=\"Skip Windows paths on not Windows\")\ndef test_windows_system_path_manipulation(env):\n \"\"\"Tests manipulting paths that have special meaning as system paths on Windows\"\"\"\n env.deprioritize_system_paths(\"PATH_LIST_WITH_SYSTEM_PATHS\")\n env.apply_modifications()\n\n assert not os.environ[\"PATH_LIST_WITH_SYSTEM_PATHS\"].startswith(\n make_pathlist([[\"C:\", \"Users\" + os.pathsep]])\n )\n assert os.environ[\"PATH_LIST_WITH_SYSTEM_PATHS\"].endswith(make_pathlist([[\"C:\", \"Users\"]]))\n\n\ndef test_extend(env):\n \"\"\"Tests that we can construct a list of environment modifications\n starting from another list.\n \"\"\"\n env.set(\"A\", \"dummy value\")\n env.set(\"B\", 3)\n copy_construct = EnvironmentModifications(env)\n\n assert len(copy_construct) == 2\n\n for x, y in zip(env, copy_construct):\n assert x is y\n\n\n@pytest.mark.usefixtures(\"prepare_environment_for_tests\")\ndef test_source_files(files_to_be_sourced):\n \"\"\"Tests the construction of a list of environment modifications that are\n the result of sourcing a file.\n \"\"\"\n env = EnvironmentModifications()\n for filename in files_to_be_sourced:\n if filename.endswith(\"sourceme_parameters\" + shell_extension):\n env.extend(EnvironmentModifications.from_sourcing_file(filename, \"intel64\"))\n else:\n env.extend(EnvironmentModifications.from_sourcing_file(filename))\n\n modifications = env.group_by_name()\n\n # This is sensitive to the user's environment; can include\n # spurious entries for things like PS1\n #\n # TODO: figure out how to make a bit more robust.\n assert len(modifications) >= 5\n\n # Set new variables\n assert len(modifications[\"NEW_VAR\"]) == 1\n assert isinstance(modifications[\"NEW_VAR\"][0], SetEnv)\n assert modifications[\"NEW_VAR\"][0].value == \"new\"\n\n assert len(modifications[\"FOO\"]) == 1\n assert isinstance(modifications[\"FOO\"][0], SetEnv)\n assert modifications[\"FOO\"][0].value == \"intel64\"\n\n # Unset variables\n assert len(modifications[\"EMPTY_PATH_LIST\"]) == 1\n assert isinstance(modifications[\"EMPTY_PATH_LIST\"][0], UnsetEnv)\n\n # Modified variables\n assert len(modifications[\"UNSET_ME\"]) == 1\n assert isinstance(modifications[\"UNSET_ME\"][0], SetEnv)\n assert modifications[\"UNSET_ME\"][0].value == \"overridden\"\n\n assert len(modifications[\"PATH_LIST\"]) == 3\n assert isinstance(modifications[\"PATH_LIST\"][0], RemovePath)\n assert modifications[\"PATH_LIST\"][0].value == make_path(\"path\", \"third\")\n assert isinstance(modifications[\"PATH_LIST\"][1], AppendPath)\n assert modifications[\"PATH_LIST\"][1].value == make_path(\"path\", \"fourth\")\n assert isinstance(modifications[\"PATH_LIST\"][2], PrependPath)\n assert modifications[\"PATH_LIST\"][2].value == make_path(\"path\", \"first\")\n\n\n@pytest.mark.regression(\"8345\")\ndef test_preserve_environment(prepare_environment_for_tests):\n # UNSET_ME is defined, and will be unset in the context manager,\n # NOT_SET is not in the environment and will be set within the\n # context manager, PATH_LIST is set and will be changed.\n with environment.preserve_environment(\"UNSET_ME\", \"NOT_SET\", \"PATH_LIST\"):\n os.environ[\"NOT_SET\"] = \"a\"\n assert os.environ[\"NOT_SET\"] == \"a\"\n\n del os.environ[\"UNSET_ME\"]\n assert \"UNSET_ME\" not in os.environ\n\n os.environ[\"PATH_LIST\"] = \"changed\"\n\n assert \"NOT_SET\" not in os.environ\n assert os.environ[\"UNSET_ME\"] == \"foo\"\n assert os.environ[\"PATH_LIST\"] == make_pathlist([[\"path\", \"second\"], [\"path\", \"third\"]])\n\n\n@pytest.mark.parametrize(\n \"files,expected,deleted\",\n [\n # Sets two variables\n (\n (os.path.join(datadir, \"sourceme_first\" + shell_extension),),\n {\"NEW_VAR\": \"new\", \"UNSET_ME\": \"overridden\"},\n [],\n ),\n # Check if we can set a variable to different values depending\n # on command line parameters\n (\n (os.path.join(datadir, \"sourceme_parameters\" + shell_extension),),\n {\"FOO\": \"default\"},\n [],\n ),\n (\n ([os.path.join(datadir, \"sourceme_parameters\" + shell_extension), \"intel64\"],),\n {\"FOO\": \"intel64\"},\n [],\n ),\n # Check unsetting variables\n (\n (os.path.join(datadir, \"sourceme_second\" + shell_extension),),\n {\n \"PATH_LIST\": make_pathlist(\n [[\"path\", \"first\"], [\"path\", \"second\"], [\"path\", \"fourth\"]]\n )\n },\n [\"EMPTY_PATH_LIST\"],\n ),\n # Check that order of sourcing matters\n (\n (\n os.path.join(datadir, \"sourceme_unset\" + shell_extension),\n os.path.join(datadir, \"sourceme_first\" + shell_extension),\n ),\n {\"NEW_VAR\": \"new\", \"UNSET_ME\": \"overridden\"},\n [],\n ),\n (\n (\n os.path.join(datadir, \"sourceme_first\" + shell_extension),\n os.path.join(datadir, \"sourceme_unset\" + shell_extension),\n ),\n {\"NEW_VAR\": \"new\"},\n [\"UNSET_ME\"],\n ),\n ],\n)\n@pytest.mark.usefixtures(\"prepare_environment_for_tests\")\ndef test_environment_from_sourcing_files(files, expected, deleted):\n env = environment.environment_after_sourcing_files(*files)\n\n # Test that variables that have been modified are still there and contain\n # the expected output\n for name, value in expected.items():\n assert name in env\n assert value in env[name]\n\n # Test that variables that have been unset are not there\n for name in deleted:\n assert name not in env\n\n\ndef test_clear(env):\n env.set(\"A\", \"dummy value\")\n assert len(env) > 0\n env.clear()\n assert len(env) == 0\n\n\n@pytest.mark.parametrize(\n \"env,exclude,include\",\n [\n # Check we can exclude a literal\n ({\"SHLVL\": \"1\"}, [\"SHLVL\"], []),\n # Check include takes precedence\n ({\"SHLVL\": \"1\"}, [\"SHLVL\"], [\"SHLVL\"]),\n ],\n)\ndef test_sanitize_literals(env, exclude, include):\n after = environment.sanitize(env, exclude, include)\n\n # Check that all the included variables are there\n assert all(x in after for x in include)\n\n # Check that the excluded variables that are not\n # included are there\n exclude = list(set(exclude) - set(include))\n assert all(x not in after for x in exclude)\n\n\n@pytest.mark.parametrize(\n \"env,exclude,include,expected,deleted\",\n [\n # Check we can exclude using a regex\n ({\"SHLVL\": \"1\"}, [\"SH.*\"], [], [], [\"SHLVL\"]),\n # Check we can include using a regex\n ({\"SHLVL\": \"1\"}, [\"SH.*\"], [\"SH.*\"], [\"SHLVL\"], []),\n # Check regex to exclude Environment Modules related vars\n (\n {\"MODULES_LMALTNAME\": \"1\", \"MODULES_LMCONFLICT\": \"2\"},\n [\"MODULES_(.*)\"],\n [],\n [],\n [\"MODULES_LMALTNAME\", \"MODULES_LMCONFLICT\"],\n ),\n (\n {\"A_modquar\": \"1\", \"b_modquar\": \"2\", \"C_modshare\": \"3\"},\n [r\"(\\w*)_mod(quar|share)\"],\n [],\n [],\n [\"A_modquar\", \"b_modquar\", \"C_modshare\"],\n ),\n (\n {\"__MODULES_LMTAG\": \"1\", \"__MODULES_LMPREREQ\": \"2\"},\n [\"__MODULES_(.*)\"],\n [],\n [],\n [\"__MODULES_LMTAG\", \"__MODULES_LMPREREQ\"],\n ),\n ],\n)\ndef test_sanitize_regex(env, exclude, include, expected, deleted):\n after = environment.sanitize(env, exclude, include)\n\n assert all(x in after for x in expected)\n assert all(x not in after for x in deleted)\n\n\n@pytest.mark.regression(\"12085\")\n@pytest.mark.parametrize(\n \"before,after,search_list\",\n [\n # Set environment variables\n ({}, {\"FOO\": \"foo\"}, [environment.SetEnv(\"FOO\", \"foo\")]),\n # Unset environment variables\n ({\"FOO\": \"foo\"}, {}, [environment.UnsetEnv(\"FOO\")]),\n # Append paths to an environment variable\n (\n {\"FOO_PATH\": make_pathlist([[\"a\", \"path\"]])},\n {\"FOO_PATH\": make_pathlist([[\"a\", \"path\"], [\"b\", \"path\"]])},\n [environment.AppendPath(\"FOO_PATH\", make_pathlist([[\"b\", \"path\"]]))],\n ),\n (\n {},\n {\"FOO_PATH\": make_pathlist([[\"a\", \"path\"], [\"b\", \"path\"]])},\n [environment.AppendPath(\"FOO_PATH\", make_pathlist([[\"a\", \"path\"], [\"b\", \"path\"]]))],\n ),\n (\n {\"FOO_PATH\": make_pathlist([[\"a\", \"path\"], [\"b\", \"path\"]])},\n {\"FOO_PATH\": make_pathlist([[\"b\", \"path\"]])},\n [environment.RemovePath(\"FOO_PATH\", make_pathlist([[\"a\", \"path\"]]))],\n ),\n (\n {\"FOO_PATH\": make_pathlist([[\"a\", \"path\"], [\"b\", \"path\"]])},\n {\"FOO_PATH\": make_pathlist([[\"a\", \"path\"], [\"c\", \"path\"]])},\n [\n environment.RemovePath(\"FOO_PATH\", make_pathlist([[\"b\", \"path\"]])),\n environment.AppendPath(\"FOO_PATH\", make_pathlist([[\"c\", \"path\"]])),\n ],\n ),\n (\n {\"FOO_PATH\": make_pathlist([[\"a\", \"path\"], [\"b\", \"path\"]])},\n {\"FOO_PATH\": make_pathlist([[\"c\", \"path\"], [\"a\", \"path\"]])},\n [\n environment.RemovePath(\"FOO_PATH\", make_pathlist([[\"b\", \"path\"]])),\n environment.PrependPath(\"FOO_PATH\", make_pathlist([[\"c\", \"path\"]])),\n ],\n ),\n # Modify two variables in the same environment\n (\n {\"FOO\": \"foo\", \"BAR\": \"bar\"},\n {\"FOO\": \"baz\", \"BAR\": \"baz\"},\n [environment.SetEnv(\"FOO\", \"baz\"), environment.SetEnv(\"BAR\", \"baz\")],\n ),\n ],\n)\ndef test_from_environment_diff(before, after, search_list):\n mod = environment.EnvironmentModifications.from_environment_diff(before, after)\n\n for item in search_list:\n assert item in mod\n\n\n@pytest.mark.not_on_windows(\"Lmod not supported on Windows\")\n@pytest.mark.regression(\"15775\")\ndef test_exclude_lmod_variables():\n # Construct the list of environment modifications\n file = os.path.join(datadir, \"sourceme_lmod.sh\")\n env = EnvironmentModifications.from_sourcing_file(file)\n\n # Check that variables related to lmod are not in there\n modifications = env.group_by_name()\n assert not any(x.startswith(\"LMOD_\") for x in modifications)\n\n\n@pytest.mark.regression(\"13504\")\ndef test_exclude_modules_variables():\n # Construct the list of environment modifications\n file = os.path.join(datadir, \"sourceme_modules\" + shell_extension)\n env = EnvironmentModifications.from_sourcing_file(file)\n\n # Check that variables related to modules are not in there\n modifications = env.group_by_name()\n assert not any(x.startswith(\"MODULES_\") for x in modifications)\n assert not any(x.startswith(\"__MODULES_\") for x in modifications)\n assert not any(x.startswith(\"BASH_FUNC_ml\") for x in modifications)\n assert not any(x.startswith(\"BASH_FUNC_module\") for x in modifications)\n assert not any(x.startswith(\"BASH_FUNC__module_raw\") for x in modifications)\n" }, { "path": "lib/spack/spack/test/error_messages.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport os.path\nimport re\nfrom contextlib import contextmanager\nfrom typing import Iterable, Optional\n\nimport pytest\n\nimport spack.vendor.archspec.cpu\n\nimport spack.config\nimport spack.error\nimport spack.repo\nimport spack.util.file_cache\nimport spack.util.spack_yaml as syaml\nfrom spack.concretize import concretize_one\nfrom spack.main import SpackCommand\n\nsolve = SpackCommand(\"solve\")\n\n\ndef update_packages_config(conf_str):\n conf = syaml.load_config(conf_str)\n spack.config.set(\"packages\", conf[\"packages\"], scope=\"concretize\")\n\n\n_pkgx1 = (\n \"x1\",\n \"\"\"\\\nclass X1(Package):\n version(\"1.2\")\n version(\"1.1\")\n\n depends_on(\"x2\")\n depends_on(\"x3\")\n\"\"\",\n)\n\n\n_pkgx2 = (\n \"x2\",\n \"\"\"\\\nclass X2(Package):\n version(\"2.1\")\n version(\"2.0\")\n\n depends_on(\"x4@4.1\")\n\"\"\",\n)\n\n\n_pkgx3 = (\n \"x3\",\n \"\"\"\\\nclass X3(Package):\n version(\"3.5\")\n version(\"3.4\")\n\n depends_on(\"x4@4.0\")\n\"\"\",\n)\n\n\n_pkgx4 = (\n \"x4\",\n \"\"\"\\\nclass X4(Package):\n version(\"4.1\")\n version(\"4.0\")\n\"\"\",\n)\n\n\n_pkgy1 = (\n \"y1\",\n \"\"\"\\\nclass Y1(Package):\n version(\"1.2\")\n version(\"1.1\")\n\n depends_on(\"y2+v1\")\n depends_on(\"y3\")\n\"\"\",\n)\n\n\n_pkgy2 = (\n \"y2\",\n \"\"\"\\\nclass Y2(Package):\n version(\"2.1\")\n version(\"2.0\")\n\n variant(\"v1\", default=True)\n\n depends_on(\"y4@4.1\", when=\"+v1\")\n depends_on(\"y4\")\n\"\"\",\n)\n\n\n_pkgy3 = (\n \"y3\",\n \"\"\"\\\nclass Y3(Package):\n version(\"3.5\")\n version(\"3.4\")\n\n depends_on(\"y4@4.0\")\n\"\"\",\n)\n\n\n_pkgy4 = (\n \"y4\",\n \"\"\"\\\nclass Y4(Package):\n version(\"4.1\")\n version(\"4.0\")\n\"\"\",\n)\n\n\n_pkgz1 = (\n \"z1\",\n \"\"\"\\\nclass Z1(Package):\n version(\"1.2\")\n version(\"1.1\")\n\n variant(\"v1\", default=True)\n\n depends_on(\"z2\")\n\n depends_on(\"z3\")\n depends_on(\"z3+v2\", when=\"~v1\")\n\n conflicts(\"+v1\", when=\"@:1.1\")\n\"\"\",\n)\n\n\n_pkgz2 = (\n \"z2\",\n \"\"\"\\\nclass Z2(Package):\n version(\"3.1\")\n version(\"3.0\")\n\n depends_on(\"z3@:2.0\")\n\"\"\",\n)\n\n\n_pkgz3 = (\n \"z3\",\n \"\"\"\\\nclass Z3(Package):\n version(\"2.1\")\n version(\"2.0\")\n\n variant(\"v2\", default=True, when=\"@2.1:\")\n\"\"\",\n)\n\n\n# Cluster of packages that includes requirements - goal is to \"chain\"\n# the requirements like other constraints.\n_pkgw4 = (\n \"w4\",\n \"\"\"\\\nclass W4(Package):\n version(\"2.1\")\n version(\"2.0\")\n\n variant(\"v1\", default=True)\n\n depends_on(\"w2\")\n depends_on(\"w2@:2.0\", when=\"@:2.0\")\n\n depends_on(\"w3\")\n depends_on(\"w3+v1\", when=\"@2.0\")\n\"\"\",\n)\n\n\n_pkgw3 = (\n \"w3\",\n \"\"\"\\\nclass W3(Package):\n version(\"2.1\")\n version(\"2.0\")\n\n variant(\"v1\", default=True)\n\n requires(\"~v1\", when=\"@2.1\")\n\n depends_on(\"w1\")\n\"\"\",\n)\n\n\n_pkgw2 = (\n \"w2\",\n \"\"\"\\\nclass W2(Package):\n version(\"2.1\")\n version(\"2.0\")\n\n variant(\"v1\", default=True)\n\n depends_on(\"w1\")\n\"\"\",\n)\n\n\n_pkgw1 = (\n \"w1\",\n \"\"\"\\\nclass W1(Package):\n version(\"2.1\")\n version(\"2.0\")\n\n variant(\"v1\", default=True)\n\"\"\",\n)\n\n\n# Like the W* packages, but encodes the config requirements constraints\n# into the packages to see if that improves the error from\n# test_errmsg_requirements_2\n_pkgt4 = (\n \"t4\",\n \"\"\"\\\nclass T4(Package):\n version(\"2.1\")\n version(\"2.0\")\n\n variant(\"v1\", default=True)\n\n depends_on(\"t2\")\n depends_on(\"t2@:2.0\", when=\"@:2.0\")\n\n depends_on(\"t3\")\n depends_on(\"t3~v1\", when=\"@2.0\")\n\"\"\",\n)\n\n\n_pkgt3 = (\n \"t3\",\n \"\"\"\\\nclass T3(Package):\n version(\"2.1\")\n version(\"2.0\")\n\n variant(\"v1\", default=True)\n\n requires(\"+v1\", when=\"@2.1\")\n\n depends_on(\"t1\")\n\"\"\",\n)\n\n\n_pkgt2 = (\n \"t2\",\n \"\"\"\\\nclass T2(Package):\n version(\"2.1\")\n version(\"2.0\")\n\n variant(\"v1\", default=True)\n\n requires(\"~v1\", when=\"@:2.0\")\n\n depends_on(\"t1\")\n\"\"\",\n)\n\n\n_pkgt1 = (\n \"t1\",\n \"\"\"\\\nclass T1(Package):\n version(\"2.1\")\n version(\"2.0\")\n\n variant(\"v1\", default=True)\n\"\"\",\n)\n\n\nall_pkgs = [\n _pkgx1,\n _pkgx2,\n _pkgx3,\n _pkgx4,\n _pkgy1,\n _pkgy2,\n _pkgy3,\n _pkgy4,\n _pkgz1,\n _pkgz2,\n _pkgz3,\n _pkgw1,\n _pkgw2,\n _pkgw3,\n _pkgw4,\n _pkgt1,\n _pkgt2,\n _pkgt3,\n _pkgt4,\n]\n\n\ndef _add_import(pkg_def):\n return (\n \"\"\"\\\nfrom spack.package import *\nfrom spack.package import Package\n\"\"\"\n + pkg_def\n )\n\n\nall_pkgs = list((x, _add_import(y)) for (x, y) in all_pkgs)\n\n\n_repo_name_id = 0\n\n\ndef create_test_repo(tmp_path, pkg_name_content_tuples):\n global _repo_name_id\n\n repo_name = f\"testrepo{str(_repo_name_id)}\"\n repo_path = tmp_path / \"spack_repo\" / repo_name\n os.makedirs(repo_path)\n with open(repo_path / \"__init__.py\", \"w\", encoding=\"utf-8\"):\n pass\n repo_yaml = os.path.join(repo_path, \"repo.yaml\")\n with open(str(repo_yaml), \"w\", encoding=\"utf-8\") as f:\n f.write(\n f\"\"\"\\\nrepo:\n namespace: {repo_name}\n api: v2.1\n\"\"\"\n )\n\n _repo_name_id += 1\n\n packages_dir = repo_path / \"packages\"\n os.mkdir(packages_dir)\n with open(packages_dir / \"__init__.py\", \"w\", encoding=\"utf-8\"):\n pass\n for pkg_name, pkg_str in pkg_name_content_tuples:\n pkg_dir = packages_dir / pkg_name\n os.mkdir(pkg_dir)\n pkg_file = pkg_dir / \"package.py\"\n with open(str(pkg_file), \"w\", encoding=\"utf-8\") as f:\n f.write(pkg_str)\n\n repo_cache = spack.util.file_cache.FileCache(str(tmp_path / \"cache\"))\n return spack.repo.Repo(str(repo_path), cache=repo_cache)\n\n\n@pytest.fixture\ndef _create_test_repo(tmp_path, mutable_config):\n yield create_test_repo(tmp_path, all_pkgs)\n\n\n@pytest.fixture\ndef test_repo(_create_test_repo, monkeypatch, mock_stage):\n with spack.repo.use_repositories(_create_test_repo) as mock_repo_path:\n yield mock_repo_path\n\n\n@contextmanager\ndef expect_failure_and_print(should_mention=None):\n got_an_error_as_expected = False\n err_msg = None\n try:\n yield\n except spack.error.UnsatisfiableSpecError as e:\n got_an_error_as_expected = True\n err_msg = str(e)\n if not got_an_error_as_expected:\n raise ValueError(\"A failure was supposed to occur in this context manager\")\n elif not err_msg:\n raise ValueError(\"No error message for failed concretization\")\n print(err_msg)\n check_error(err_msg, should_mention)\n\n\ndef check_error(msg, should_mention: Optional[Iterable] = None):\n excludes = [\n \"failed to concretize .* for the following reasons:\",\n \"Cannot satisfy .*\",\n \"required because .* requested explicitly\",\n \"cannot satisfy a requirement for package .*\",\n ]\n lines = msg.split(\"\\n\")\n should_mention = set(should_mention) if should_mention else set()\n should_mention_hits = set()\n remaining = []\n for line in lines:\n for p in should_mention:\n if re.search(p, line):\n should_mention_hits.add(p)\n if any(re.search(p, line) for p in excludes):\n continue\n remaining.append(line)\n if not remaining:\n raise ValueError(\"The error message contains only generic statements\")\n should_mention_misses = should_mention - should_mention_hits\n if should_mention_misses:\n raise ValueError(f\"The error message did not contain: {sorted(should_mention_misses)}\")\n\n\ndef test_diamond_with_pkg_conflict1(concretize_scope, test_repo):\n concretize_one(\"x2\")\n concretize_one(\"x3\")\n concretize_one(\"x4\")\n\n important_points = [\"x2 depends on x4@4.1\", \"x3 depends on x4@4.0\"]\n\n with expect_failure_and_print(should_mention=important_points):\n concretize_one(\"x1\")\n\n\ndef test_diamond_with_pkg_conflict2(concretize_scope, test_repo):\n important_points = [\n r\"y2 depends on y4@4.1 when \\+v1\",\n r\"y1 depends on y2\\+v1\",\n r\"y3 depends on y4@4.0\",\n ]\n\n with expect_failure_and_print(should_mention=important_points):\n concretize_one(\"y1\")\n\n\n@pytest.mark.xfail(reason=\"Not addressed yet\")\ndef test_version_range_null(concretize_scope, test_repo):\n with expect_failure_and_print():\n concretize_one(\"x2@3:4\")\n\n\n# This error message is hard to follow: neither z2 or z3\n# are mentioned, so if this hierarchy had 10 other \"OK\"\n# packages, a user would be conducting a tedious manual\n# search\n@pytest.mark.xfail(reason=\"Not addressed yet\")\ndef test_null_variant_for_requested_version(concretize_scope, test_repo):\n r\"\"\"\n Z1_ (@:1.1 -> !v1)\n | \\\n Z2 |\n \\ |\n \\|\n Z3 (z1~v1 -> z3+v2)\n (z2 ^z3:2.0)\n (v2 only exists for @2.1:)\n \"\"\"\n concretize_one(\"z1\")\n\n with expect_failure_and_print(should_mention=[\"z2\"]):\n concretize_one(\"z1@1.1\")\n\n\ndef test_errmsg_requirements_1(concretize_scope, test_repo):\n # w4 has: depends_on(\"w3+v1\", when=\"@2.0\")\n # w3 has: requires(\"~v1\", when=\"@2.1\")\n\n important_points = [\n r\"w4 depends on w3\\+v1 when @2.0\",\n r\"w4@:2.0 \\^w3@2.1 requested explicitly\",\n r\"~v1 is a requirement for package w3 when @2.1\",\n ]\n\n with expect_failure_and_print(should_mention=important_points):\n concretize_one(\"w4@:2.0 ^w3@2.1\")\n\n\ndef test_errmsg_requirements_cfg(concretize_scope, test_repo):\n conf_str = \"\"\"\\\npackages:\n w2:\n require:\n - one_of: [\"~v1\"]\n when: \"@2.0\"\n\"\"\"\n update_packages_config(conf_str)\n\n important_points = [\n r\"~v1 is a requirement for package w2 when @2.0\",\n r\"w4 depends on w2@:2.0 when @:2.0\",\n r\"w4@2.0 \\^w2\\+v1 requested explicitly\",\n ]\n\n # w4 has: depends_on(\"w2@:2.0\", when=\"@:2.0\")\n with expect_failure_and_print(should_mention=important_points):\n concretize_one(\"w4@2.0 ^w2+v1\")\n\n\n# This reencodes prior test test_errmsg_requirements_cfg\n# in terms of package `requires`,\ndef test_errmsg_requirements_directives(concretize_scope, test_repo):\n # t4 has: depends_on(\"t2@:2.0\", when=\"@:2.0\")\n # t2 has: requires(\"~v1\", when=\"@:2.0\")\n\n important_points = [\n r\"~v1 is a requirement for package t2 when @:2.0\",\n r\"t4 depends on t2@:2.0 when @:2.0\",\n r\"t4@:2.0 \\^t2\\+v1 requested explicitly\",\n ]\n\n with expect_failure_and_print(should_mention=important_points):\n concretize_one(\"t4@:2.0 ^t2+v1\")\n\n\n# Simulates a user error: package is specified as external with a version,\n# but a different version was required in config.\ndef test_errmsg_requirements_external_mismatch(concretize_scope, test_repo):\n conf_str = \"\"\"\\\npackages:\n t1:\n buildable: false\n externals:\n - spec: \"t1@2.1\"\n prefix: /a/path/that/doesnt/need/to/exist/\n require:\n - spec: \"t1@2.0\"\n\"\"\"\n update_packages_config(conf_str)\n\n important_points = [\"no externals satisfy the request\"]\n\n with expect_failure_and_print(should_mention=important_points):\n concretize_one(\"t1\")\n\n\n@pytest.mark.parametrize(\"section\", [\"prefer\", \"require\"])\ndef test_warns_on_compiler_constraint_in_all(concretize_scope, mock_packages, section):\n \"\"\"Compiler constraints under packages:all: are a footgun and should warn.\"\"\"\n update_packages_config(f\"packages:\\n all:\\n {section}:\\n - '%c=gcc'\\n\")\n with pytest.warns(UserWarning, match=\"packages: all:\"):\n concretize_one(\"gmake\")\n\n\n@pytest.mark.regression(\"52209\")\ndef test_unknown_concrete_target_in_input_spec(concretize_scope, test_repo):\n \"\"\"Tests that an input spec with an unknown concrete target raises a clear error naming\n the bad target, rather than a confusing 'cannot satisfy constraint' solver error.\n \"\"\"\n spec_str = \"x4 target=not-a-real-uarch\"\n with pytest.raises(spack.error.SpackError) as exc_info:\n concretize_one(spec_str)\n check_error(str(exc_info.value), should_mention=[spec_str, \"not a known target\"])\n\n\n@pytest.mark.regression(\"52209\")\ndef test_require_single_unknown_target_errors(concretize_scope, test_repo):\n \"\"\"Tests that a single-option require with an unknown target raises a clear error.\"\"\"\n target_str = \"target=not-a-real-uarch\"\n update_packages_config(\n f\"\"\"\\\npackages:\n x4:\n require: {target_str}\n\"\"\"\n )\n with pytest.raises(spack.error.SpackError) as exc_info:\n concretize_one(\"x4\")\n check_error(str(exc_info.value), should_mention=[target_str, \"unknown target\"])\n\n\n@pytest.mark.regression(\"52209\")\ndef test_require_all_unknown_targets_errors(concretize_scope, test_repo):\n \"\"\"Tests that a group where every option has an unknown target also raises a clear error.\"\"\"\n update_packages_config(\n \"\"\"\\\npackages:\n x4:\n require:\n - any_of: [\"target=not-a-real-uarch\", \"target=also-fake\"]\n\"\"\"\n )\n with pytest.raises(spack.error.SpackError) as exc_info:\n concretize_one(\"x4\")\n check_error(\n str(exc_info.value),\n should_mention=[\"target=not-a-real-uarch\", \"target=also-fake\", \"unknown target\"],\n )\n\n\n@pytest.mark.regression(\"52209\")\n@pytest.mark.skipif(\n str(spack.vendor.archspec.cpu.host().family) != \"x86_64\", reason=\"test assumes x86_64 uarchs\"\n)\ndef test_require_mixed_unknown_and_valid_target_warns(concretize_scope, test_repo):\n \"\"\"Tests that a \"require\" group with at least one valid option just warns.\"\"\"\n update_packages_config(\n \"\"\"\\\npackages:\n x4:\n require:\n - one_of: [\"target=not-a-real-uarch\", \"target=x86_64\"]\n\"\"\"\n )\n with pytest.warns(UserWarning, match=\"not-a-real-uarch\"):\n concretize_one(\"x4\")\n\n\n@pytest.mark.regression(\"52209\")\ndef test_prefer_unknown_target_warns(concretize_scope, test_repo):\n \"\"\"A preference with an unknown target has the @: fallback, so it only warns.\"\"\"\n update_packages_config(\n \"\"\"\\\npackages:\n x4:\n prefer: [\"target=not-a-real-uarch\"]\n\"\"\"\n )\n with pytest.warns(UserWarning, match=\"not-a-real-uarch\"):\n concretize_one(\"x4\")\n" }, { "path": "lib/spack/spack/test/externals.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nfrom typing import List\n\nimport pytest\n\nfrom spack.vendor.archspec.cpu import TARGETS\n\nimport spack.archspec\nimport spack.traverse\nfrom spack.externals import (\n DuplicateExternalError,\n ExternalDict,\n ExternalSpecError,\n ExternalSpecsParser,\n complete_architecture,\n complete_variants_and_architecture,\n)\n\npytestmark = pytest.mark.usefixtures(\"config\", \"mock_packages\")\n\n\n@pytest.mark.parametrize(\n \"externals_dict,expected_length,expected_queries\",\n [\n # Empty dictionary case\n ([], 0, {\"gmake\": 0}),\n # Single spec case\n (\n [{\"spec\": \"gmake@1.0\", \"prefix\": \"/path/to/gmake\"}],\n 1,\n {\"gmake\": 1, \"gmake@1.0\": 1, \"gmake@2.0\": 0},\n ),\n # Multiple specs case\n (\n [\n {\"spec\": \"gmake@1.0\", \"prefix\": \"/path/to/gmake1\"},\n {\"spec\": \"gmake@2.0\", \"prefix\": \"/path/to/gmake2\"},\n {\"spec\": \"gcc@1.0\", \"prefix\": \"/path/to/gcc\"},\n ],\n 3,\n {\"gmake\": 2, \"gmake@2\": 1, \"gcc\": 1, \"baz\": 0},\n ),\n # Case with modules and extra attributes\n (\n [\n {\n \"spec\": \"gmake@1.0\",\n \"prefix\": \"/path/to/gmake\",\n \"modules\": [\"module1\", \"module2\"],\n \"extra_attributes\": {\"attr1\": \"value1\"},\n }\n ],\n 1,\n {\"gmake\": 1},\n ),\n ],\n)\ndef test_basic_parsing(externals_dict, expected_length, expected_queries):\n \"\"\"Tests parsing external specs, in some basic cases\"\"\"\n parser = ExternalSpecsParser(externals_dict)\n\n assert len(parser.all_specs()) == expected_length\n assert len(parser.specs_by_external_id) == expected_length\n for node in parser.all_specs():\n assert node.concrete\n\n for query, expected in expected_queries.items():\n assert len(parser.query(query)) == expected\n\n\n@pytest.mark.parametrize(\n \"externals_dict,expected_triplet\",\n [\n ([{\"spec\": \"gmake@1.0\", \"prefix\": \"/path/to/gmake1\"}], (\"test\", \"debian6\", \"aarch64\")),\n (\n [{\"spec\": \"gmake@1.0 target=icelake\", \"prefix\": \"/path/to/gmake1\"}],\n (\"test\", \"debian6\", \"icelake\"),\n ),\n (\n [{\"spec\": \"gmake@1.0 platform=linux target=icelake\", \"prefix\": \"/path/to/gmake1\"}],\n (\"linux\", \"debian6\", \"icelake\"),\n ),\n (\n [{\"spec\": \"gmake@1.0 os=rhel8\", \"prefix\": \"/path/to/gmake1\"}],\n (\"test\", \"rhel8\", \"aarch64\"),\n ),\n ],\n)\ndef test_external_specs_architecture_completion(\n externals_dict: List[ExternalDict], expected_triplet, monkeypatch\n):\n \"\"\"Tests the completion of external specs architectures when using the default behavior\"\"\"\n monkeypatch.setattr(spack.archspec, \"HOST_TARGET_FAMILY\", TARGETS[\"aarch64\"])\n parser = ExternalSpecsParser(externals_dict)\n\n expected_platform, expected_os, expected_target = expected_triplet\n\n for node in parser.all_specs():\n assert node.architecture is not None\n assert node.architecture.platform == expected_platform\n assert node.architecture.os == expected_os\n assert node.target == expected_target\n\n\ndef test_external_specs_parser_with_missing_packages():\n \"\"\"Tests the parsing of external specs when some packages are missing\"\"\"\n externals_dict: List[ExternalDict] = [\n {\"spec\": \"gmake@1.0\", \"prefix\": \"/path/to/gmake1\"},\n {\"spec\": \"gmake@2.0\", \"prefix\": \"/path/to/gmake2\"},\n {\"spec\": \"gcc@1.0\", \"prefix\": \"/path/to/gcc\"},\n # This package does not exist in the builtin_mock repository\n {\"spec\": \"baz@1.0\", \"prefix\": \"/path/to/baz\"},\n ]\n\n external_specs = ExternalSpecsParser(externals_dict, allow_nonexisting=True).all_specs()\n assert len(external_specs) == 3\n assert len([x for x in external_specs if x.satisfies(\"gmake\")]) == 2\n assert len([x for x in external_specs if x.satisfies(\"gcc\")]) == 1\n\n with pytest.raises(ExternalSpecError, match=\"Package 'baz' does not exist\"):\n ExternalSpecsParser(externals_dict, allow_nonexisting=False)\n\n\ndef test_externals_with_duplicate_id():\n \"\"\"Tests the parsing of external specs when some specs have the same id\"\"\"\n externals_dict: List[ExternalDict] = [\n {\"spec\": \"gmake@1.0\", \"prefix\": \"/path/to/gmake1\", \"id\": \"gmake\"},\n {\"spec\": \"gmake@2.0\", \"prefix\": \"/path/to/gmake2\", \"id\": \"gmake\"},\n {\"spec\": \"gcc@1.0\", \"prefix\": \"/path/to/gcc\", \"id\": \"gcc\"},\n ]\n\n with pytest.raises(DuplicateExternalError, match=\"cannot have the same external id\"):\n ExternalSpecsParser(externals_dict)\n\n\n@pytest.mark.parametrize(\n \"externals_dicts,expected,not_expected\",\n [\n # o ascent@0.9.2\n # o adios2@2.7.1\n # o bzip2@1.0.8\n (\n [\n {\n \"spec\": \"ascent@0.9.2+adios2+shared\",\n \"prefix\": \"/user/path\",\n \"id\": \"ascent\",\n \"dependencies\": [{\"id\": \"adios2\", \"deptypes\": [\"build\", \"link\"]}],\n },\n {\n \"spec\": \"adios2@2.7.1+shared\",\n \"prefix\": \"/user/path\",\n \"id\": \"adios2\",\n \"dependencies\": [{\"id\": \"bzip2\", \"deptypes\": [\"build\", \"link\"]}],\n },\n {\"spec\": \"bzip2@1.0.8+shared\", \"prefix\": \"/user/path\", \"id\": \"bzip2\"},\n ],\n {\n \"ascent\": [\"%[deptypes=build,link] adios2@2.7.1\"],\n \"adios2\": [\"%[deptypes=build,link] bzip2@1.0.8\"],\n },\n {},\n ),\n # o ascent@0.9.2\n # |\\\n # | o adios2@2.7.1\n # |/\n # o bzip2@1.0.8\n (\n [\n {\n \"spec\": \"ascent@0.9.2+adios2+shared\",\n \"prefix\": \"/user/path\",\n \"id\": \"ascent\",\n \"dependencies\": [\n {\"id\": \"adios2\", \"deptypes\": \"link\"},\n {\"id\": \"bzip2\", \"deptypes\": \"run\"},\n ],\n },\n {\n \"spec\": \"adios2@2.7.1+shared\",\n \"prefix\": \"/user/path\",\n \"id\": \"adios2\",\n \"dependencies\": [{\"id\": \"bzip2\", \"deptypes\": [\"build\", \"link\"]}],\n },\n {\"spec\": \"bzip2@1.0.8+shared\", \"prefix\": \"/user/path\", \"id\": \"bzip2\"},\n ],\n {\n \"ascent\": [\"%[deptypes=link] adios2@2.7.1\", \"%[deptypes=run] bzip2@1.0.8\"],\n \"adios2\": [\"%[deptypes=build,link] bzip2@1.0.8\"],\n },\n {\n \"ascent\": [\n \"%[deptypes=build] adios2@2.7.1\",\n \"%[deptypes=run] adios2@2.7.1\",\n \"%[deptypes=build] bzip2@1.0.8\",\n \"%[deptypes=link] bzip2@1.0.8\",\n ]\n },\n ),\n # Same, but specifying dependencies by spec: instead of id:\n (\n [\n {\n \"spec\": \"ascent@0.9.2+adios2+shared\",\n \"prefix\": \"/user/path\",\n \"dependencies\": [\n {\"spec\": \"adios2\", \"deptypes\": \"link\"},\n {\"spec\": \"bzip2\", \"deptypes\": \"run\"},\n ],\n },\n {\n \"spec\": \"adios2@2.7.1+shared\",\n \"prefix\": \"/user/path\",\n \"dependencies\": [{\"spec\": \"bzip2\", \"deptypes\": [\"build\", \"link\"]}],\n },\n {\"spec\": \"bzip2@1.0.8+shared\", \"prefix\": \"/user/path\"},\n ],\n {\n \"ascent\": [\"%[deptypes=link] adios2@2.7.1\", \"%[deptypes=run] bzip2@1.0.8\"],\n \"adios2\": [\"%[deptypes=build,link] bzip2@1.0.8\"],\n },\n {\n \"ascent\": [\n \"%[deptypes=build] adios2@2.7.1\",\n \"%[deptypes=run] adios2@2.7.1\",\n \"%[deptypes=build] bzip2@1.0.8\",\n \"%[deptypes=link] bzip2@1.0.8\",\n ]\n },\n ),\n # Inline specification for\n # o mpileaks@2.2\n # | \\\n # | o callpath@1.0\n # | /\n # o gcc@15.0.1\n (\n [\n [\n {\"spec\": \"mpileaks@2.2 %gcc %callpath\", \"prefix\": \"/user/path\"},\n {\"spec\": \"callpath@1.0\", \"prefix\": \"/user/path\"},\n {\"spec\": \"gcc@15.0.1 languages=c,c++\", \"prefix\": \"/user/path\"},\n ],\n {\"mpileaks\": [\"%[deptypes=build] gcc@15\", \"%[deptypes=build,link] callpath@1.0\"]},\n {\"mpileaks\": [\"%[deptypes=link] gcc@15\"]},\n ]\n ),\n # CMake dependency should be inferred of `deptypes=build`\n # o cmake-client\n # |\n # o cmake@3.23.1\n (\n [\n [\n {\"spec\": \"cmake-client@1.0 %cmake\", \"prefix\": \"/user/path\"},\n {\"spec\": \"cmake@3.23.1\", \"prefix\": \"/user/path\"},\n ],\n {\"cmake-client\": [\"%[deptypes=build] cmake\"]},\n {\"cmake-client\": [\"%[deptypes=link] cmake\", \"%[deptypes=run] cmake\"]},\n ]\n ),\n ],\n)\ndef test_externals_with_dependencies(externals_dicts: List[ExternalDict], expected, not_expected):\n \"\"\"Tests constructing externals with dependencies\"\"\"\n parser = ExternalSpecsParser(externals_dicts)\n\n for query_spec, expected_list in expected.items():\n result = parser.query(query_spec)\n assert len(result) == 1\n assert all(result[0].satisfies(c) for c in expected_list)\n\n for query_spec, not_expected_list in not_expected.items():\n result = parser.query(query_spec)\n assert len(result) == 1\n assert all(not result[0].satisfies(c) for c in not_expected_list)\n\n # Assert all nodes have the namespace set\n for node in spack.traverse.traverse_nodes(parser.all_specs()):\n assert node.namespace is not None\n\n\n@pytest.mark.parametrize(\n \"externals_dicts,expected_length,not_expected\",\n [\n ([{\"spec\": \"mpileaks\", \"prefix\": \"/user/path\", \"id\": \"mpileaks\"}], 0, [\"mpileaks\"]),\n ([{\"spec\": \"mpileaks@2:\", \"prefix\": \"/user/path\", \"id\": \"mpileaks\"}], 0, [\"mpileaks\"]),\n ],\n)\ndef test_externals_without_concrete_version(\n externals_dicts: List[ExternalDict], expected_length, not_expected\n):\n \"\"\"Tests parsing externals, when some dicts are malformed and don't have a concrete version\"\"\"\n parser = ExternalSpecsParser(externals_dicts)\n result = parser.all_specs()\n\n assert len(result) == expected_length\n for c in not_expected:\n assert all(not s.satisfies(c) for s in result)\n\n\n@pytest.mark.parametrize(\n \"externals_dict,completion_fn,expected,not_expected\",\n [\n (\n [{\"spec\": \"mpileaks@2.3\", \"prefix\": \"/user/path\"}],\n complete_architecture,\n {\"mpileaks\": [\"platform=test\"]},\n {\"mpileaks\": [\"debug=*\", \"opt=*\", \"shared=*\", \"static=*\"]},\n ),\n (\n [{\"spec\": \"mpileaks@2.3\", \"prefix\": \"/user/path\"}],\n complete_variants_and_architecture,\n {\"mpileaks\": [\"platform=test\", \"~debug\", \"~opt\", \"+shared\", \"+static\"]},\n {\"mpileaks\": [\"+debug\", \"+opt\", \"~shared\", \"~static\"]},\n ),\n ],\n)\ndef test_external_node_completion(\n externals_dict: List[ExternalDict], completion_fn, expected, not_expected\n):\n \"\"\"Tests the completion of external specs with different node completion\"\"\"\n parser = ExternalSpecsParser(externals_dict, complete_node=completion_fn)\n\n for query_spec, expected_list in expected.items():\n result = parser.query(query_spec)\n assert len(result) == 1\n for expected in expected_list:\n assert result[0].satisfies(expected)\n\n for query_spec, expected_list in not_expected.items():\n result = parser.query(query_spec)\n assert len(result) == 1\n for expected in expected_list:\n assert not result[0].satisfies(expected)\n\n # Assert all nodes have the namespace set\n for node in spack.traverse.traverse_nodes(parser.all_specs()):\n assert node.namespace is not None\n\n\n@pytest.mark.regression(\"52179\")\ndef test_external_spec_single_valued_variant_type_is_corrected():\n \"\"\"Tests that an external spec string including a single-valued variant is parsed correctly.\"\"\"\n externals_dict = [\n {\"spec\": \"dual-cmake-autotools@1.0 build_system=autotools\", \"prefix\": \"/usr/dual\"}\n ]\n parser = ExternalSpecsParser(externals_dict, complete_node=complete_variants_and_architecture)\n specs = parser.all_specs()\n assert len(specs) == 1\n spec = specs[0]\n\n # Single-valued variants return the value, not a tuple of values\n build_system_value = spec.variants[\"build_system\"].value\n assert build_system_value == \"autotools\", (\n f\"Expected 'autotools' but got {build_system_value!r} \"\n f\"(type: {type(build_system_value).__name__})\"\n )\n\n\n@pytest.mark.regression(\"52179\")\ndef test_external_spec_multi_valued_variant_is_not_changed():\n \"\"\"Tests that multi-valued variants in external specs are preserved as they are, even if the\n definition in package.py says otherwise.\n \"\"\"\n # Package.py prescribes a single-valued variant in this case\n externals_dict = [{\"spec\": \"variant-values@1.0 v=foo,bar\", \"prefix\": \"/usr/variant-values\"}]\n parser = ExternalSpecsParser(externals_dict, complete_node=complete_variants_and_architecture)\n specs = parser.all_specs()\n assert len(specs) == 1\n assert specs[0].variants[\"v\"].value == (\"bar\", \"foo\")\n" }, { "path": "lib/spack/spack/test/fetch_strategy.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nfrom io import StringIO\n\nimport pytest\n\nfrom spack import fetch_strategy\n\n\ndef test_fetchstrategy_bad_url_scheme():\n \"\"\"Ensure that trying to make a fetch strategy from a URL with an\n unsupported scheme fails as expected.\"\"\"\n\n with pytest.raises(ValueError):\n fetcher = fetch_strategy.from_url_scheme(\"bogus-scheme://example.com/a/b/c\") # noqa: F841\n\n\n@pytest.mark.parametrize(\n \"expected,total_bytes\",\n [\n (\" 0.00 B\", 0),\n (\" 999.00 B\", 999),\n (\" 1.00 KB\", 1000),\n (\" 2.05 KB\", 2048),\n (\" 1.00 MB\", 1e6),\n (\" 12.30 MB\", 1.23e7),\n (\" 1.23 GB\", 1.23e9),\n (\" 999.99 GB\", 9.9999e11),\n (\"5000.00 GB\", 5e12),\n ],\n)\ndef test_format_bytes(expected, total_bytes):\n assert fetch_strategy._format_bytes(total_bytes) == expected\n\n\n@pytest.mark.parametrize(\n \"expected,total_bytes,elapsed\",\n [\n (\" 0.0 B/s\", 0, 0), # no time passed -- defaults to 1s.\n (\" 0.0 B/s\", 0, 1),\n (\" 999.0 B/s\", 999, 1),\n (\" 1.0 KB/s\", 1000, 1),\n (\" 500.0 B/s\", 1000, 2),\n (\" 2.0 KB/s\", 2048, 1),\n (\" 1.0 MB/s\", 1e6, 1),\n (\" 500.0 KB/s\", 1e6, 2),\n (\" 12.3 MB/s\", 1.23e7, 1),\n (\" 1.2 GB/s\", 1.23e9, 1),\n (\" 999.9 GB/s\", 9.999e11, 1),\n (\"5000.0 GB/s\", 5e12, 1),\n ],\n)\ndef test_format_speed(expected, total_bytes, elapsed):\n assert fetch_strategy._format_speed(total_bytes, elapsed) == expected\n\n\ndef test_fetch_progress_unknown_size():\n # time stamps in seconds, with 0.1s delta except 1.5 -> 1.55.\n time_stamps = iter([1.0, 1.5, 1.55, 2.0, 3.0, 5.0, 5.5, 5.5])\n progress = fetch_strategy.FetchProgress(total_bytes=None, get_time=lambda: next(time_stamps))\n assert progress.start_time == 1.0\n out = StringIO()\n\n progress.advance(1000, out)\n assert progress.last_printed == 1.5\n progress.advance(50, out)\n assert progress.last_printed == 1.5 # does not print, too early after last print\n progress.advance(2000, out)\n assert progress.last_printed == 2.0\n progress.advance(3000, out)\n assert progress.last_printed == 3.0\n progress.advance(4000, out)\n assert progress.last_printed == 5.0\n progress.advance(4000, out)\n assert progress.last_printed == 5.5\n progress.print(final=True, out=out) # finalize download\n\n outputs = [\n \"\\r [ | ] 1.00 KB @ 2.0 KB/s\",\n \"\\r [ / ] 3.05 KB @ 3.0 KB/s\",\n \"\\r [ - ] 6.05 KB @ 3.0 KB/s\",\n \"\\r [ \\\\ ] 10.05 KB @ 2.5 KB/s\", # have to escape \\ here but is aligned in output\n \"\\r [ | ] 14.05 KB @ 3.1 KB/s\",\n \"\\r [100%] 14.05 KB @ 3.1 KB/s\\n\", # final print: no spinner; newline\n ]\n\n assert out.getvalue() == \"\".join(outputs)\n\n\ndef test_fetch_progress_known_size():\n time_stamps = iter([1.0, 1.5, 3.0, 4.0, 4.0])\n progress = fetch_strategy.FetchProgress(total_bytes=6000, get_time=lambda: next(time_stamps))\n out = StringIO()\n progress.advance(1000, out) # time 1.5\n progress.advance(2000, out) # time 3.0\n progress.advance(3000, out) # time 4.0\n progress.print(final=True, out=out)\n\n outputs = [\n \"\\r [ 17%] 1.00 KB @ 2.0 KB/s\",\n \"\\r [ 50%] 3.00 KB @ 1.5 KB/s\",\n \"\\r [100%] 6.00 KB @ 2.0 KB/s\",\n \"\\r [100%] 6.00 KB @ 2.0 KB/s\\n\", # final print has newline\n ]\n\n assert out.getvalue() == \"\".join(outputs)\n\n\ndef test_fetch_progress_disabled():\n \"\"\"When disabled, FetchProgress shouldn't print anything when advanced\"\"\"\n\n def get_time():\n raise RuntimeError(\"Should not be called\")\n\n progress = fetch_strategy.FetchProgress(enabled=False, get_time=get_time)\n out = StringIO()\n progress.advance(1000, out)\n progress.advance(2000, out)\n progress.print(final=True, out=out)\n assert progress.last_printed == 0\n assert not out.getvalue()\n\n\n@pytest.mark.parametrize(\n \"header,value,total_bytes\",\n [\n (\"Content-Length\", \"1234\", 1234),\n (\"Content-Length\", \"0\", 0),\n (\"Content-Length\", \"-10\", 0),\n (\"Content-Length\", \"not a number\", 0),\n (\"Not-Content-Length\", \"1234\", 0),\n ],\n)\ndef test_fetch_progress_from_headers(header, value, total_bytes):\n time_stamps = iter([1.0, 1.5, 3.0, 4.0, 4.0])\n progress = fetch_strategy.FetchProgress.from_headers(\n {header: value}, get_time=lambda: next(time_stamps), enabled=True\n )\n assert progress.total_bytes == total_bytes\n assert progress.enabled\n assert progress.start_time == 1.0\n\n\ndef test_fetch_progress_from_headers_disabled():\n progress = fetch_strategy.FetchProgress.from_headers(\n {\"Content-Length\": \"1234\"}, get_time=lambda: 1.0, enabled=False\n )\n assert not progress.enabled\n" }, { "path": "lib/spack/spack/test/flag_handlers.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\n\nimport pytest\n\nimport spack.build_environment\nimport spack.concretize\nfrom spack.package import build_system_flags, env_flags, inject_flags\n\n\n@pytest.fixture()\ndef temp_env():\n old_env = os.environ.copy()\n yield\n os.environ.clear()\n os.environ.update(old_env)\n\n\ndef add_o3_to_build_system_cflags(pkg, name, flags):\n build_system_flags = []\n if name == \"cflags\":\n build_system_flags.append(\"-O3\")\n return flags, None, build_system_flags\n\n\n@pytest.mark.usefixtures(\"config\", \"mock_packages\")\nclass TestFlagHandlers:\n def test_no_build_system_flags(self, temp_env):\n # Test that both autotools and cmake work getting no build_system flags\n s1 = spack.concretize.concretize_one(\"cmake-client\")\n spack.build_environment.setup_package(s1.package, False)\n\n s2 = spack.concretize.concretize_one(\"patchelf\")\n spack.build_environment.setup_package(s2.package, False)\n\n # Use cppflags as a canary\n assert \"SPACK_CPPFLAGS\" not in os.environ\n assert \"CPPFLAGS\" not in os.environ\n\n def test_unbound_method(self, temp_env):\n # Other tests test flag_handlers set as bound methods and functions.\n # This tests an unbound method in python2 (no change in python3).\n s = spack.concretize.concretize_one(\"mpileaks cppflags=-g\")\n s.package.flag_handler = s.package.__class__.inject_flags\n spack.build_environment.setup_package(s.package, False)\n assert os.environ[\"SPACK_CPPFLAGS\"] == \"-g\"\n assert \"CPPFLAGS\" not in os.environ\n\n def test_inject_flags(self, temp_env):\n s = spack.concretize.concretize_one(\"mpileaks cppflags=-g\")\n s.package.flag_handler = inject_flags\n spack.build_environment.setup_package(s.package, False)\n assert os.environ[\"SPACK_CPPFLAGS\"] == \"-g\"\n assert \"CPPFLAGS\" not in os.environ\n\n def test_env_flags(self, temp_env):\n s = spack.concretize.concretize_one(\"mpileaks cppflags=-g\")\n s.package.flag_handler = env_flags\n spack.build_environment.setup_package(s.package, False)\n assert os.environ[\"CPPFLAGS\"] == \"-g\"\n assert \"SPACK_CPPFLAGS\" not in os.environ\n\n def test_build_system_flags_cmake(self, temp_env):\n s = spack.concretize.concretize_one(\"cmake-client cppflags=-g\")\n s.package.flag_handler = build_system_flags\n spack.build_environment.setup_package(s.package, False)\n assert \"SPACK_CPPFLAGS\" not in os.environ\n assert \"CPPFLAGS\" not in os.environ\n assert set(s.package.cmake_flag_args) == {\n \"-DCMAKE_C_FLAGS=-g\",\n \"-DCMAKE_CXX_FLAGS=-g\",\n \"-DCMAKE_Fortran_FLAGS=-g\",\n }\n\n def test_build_system_flags_autotools(self, temp_env):\n s = spack.concretize.concretize_one(\"patchelf cppflags=-g\")\n s.package.flag_handler = build_system_flags\n spack.build_environment.setup_package(s.package, False)\n assert \"SPACK_CPPFLAGS\" not in os.environ\n assert \"CPPFLAGS\" not in os.environ\n assert \"CPPFLAGS=-g\" in s.package.configure_flag_args\n\n def test_build_system_flags_not_implemented(self, temp_env):\n \"\"\"Test the command line flags method raises a NotImplementedError\"\"\"\n s = spack.concretize.concretize_one(\"mpileaks cppflags=-g\")\n s.package.flag_handler = build_system_flags\n try:\n spack.build_environment.setup_package(s.package, False)\n assert False\n except NotImplementedError:\n assert True\n\n def test_add_build_system_flags_autotools(self, temp_env):\n s = spack.concretize.concretize_one(\"patchelf cppflags=-g\")\n s.package.flag_handler = add_o3_to_build_system_cflags\n spack.build_environment.setup_package(s.package, False)\n assert \"-g\" in os.environ[\"SPACK_CPPFLAGS\"]\n assert \"CPPFLAGS\" not in os.environ\n assert s.package.configure_flag_args == [\"CFLAGS=-O3\"]\n\n def test_add_build_system_flags_cmake(self, temp_env):\n s = spack.concretize.concretize_one(\"cmake-client cppflags=-g\")\n s.package.flag_handler = add_o3_to_build_system_cflags\n spack.build_environment.setup_package(s.package, False)\n assert \"-g\" in os.environ[\"SPACK_CPPFLAGS\"]\n assert \"CPPFLAGS\" not in os.environ\n assert s.package.cmake_flag_args == [\"-DCMAKE_C_FLAGS=-O3\"]\n\n def test_ld_flags_cmake(self, temp_env):\n s = spack.concretize.concretize_one(\"cmake-client ldflags=-mthreads\")\n s.package.flag_handler = build_system_flags\n spack.build_environment.setup_package(s.package, False)\n assert \"SPACK_LDFLAGS\" not in os.environ\n assert \"LDFLAGS\" not in os.environ\n assert set(s.package.cmake_flag_args) == {\n \"-DCMAKE_EXE_LINKER_FLAGS=-mthreads\",\n \"-DCMAKE_MODULE_LINKER_FLAGS=-mthreads\",\n \"-DCMAKE_SHARED_LINKER_FLAGS=-mthreads\",\n }\n\n def test_ld_libs_cmake(self, temp_env):\n s = spack.concretize.concretize_one(\"cmake-client ldlibs=-lfoo\")\n s.package.flag_handler = build_system_flags\n spack.build_environment.setup_package(s.package, False)\n assert \"SPACK_LDLIBS\" not in os.environ\n assert \"LDLIBS\" not in os.environ\n assert set(s.package.cmake_flag_args) == {\n \"-DCMAKE_C_STANDARD_LIBRARIES=-lfoo\",\n \"-DCMAKE_CXX_STANDARD_LIBRARIES=-lfoo\",\n \"-DCMAKE_Fortran_STANDARD_LIBRARIES=-lfoo\",\n }\n\n def test_flag_handler_no_modify_specs(self, temp_env):\n def test_flag_handler(self, name, flags):\n flags.append(\"-foo\")\n return (flags, None, None)\n\n s = spack.concretize.concretize_one(\"cmake-client\")\n s.package.flag_handler = test_flag_handler\n spack.build_environment.setup_package(s.package, False)\n\n assert not s.compiler_flags[\"cflags\"]\n assert os.environ[\"SPACK_CFLAGS\"] == \"-foo\"\n" }, { "path": "lib/spack/spack/test/gcs_fetch.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport pathlib\n\nimport spack.fetch_strategy\nimport spack.stage\n\n\ndef test_gcsfetchstrategy_downloaded(tmp_path: pathlib.Path):\n \"\"\"Ensure fetch with archive file already downloaded is a noop.\"\"\"\n archive = tmp_path / \"gcs.tar.gz\"\n\n class Archived_GCSFS(spack.fetch_strategy.GCSFetchStrategy):\n @property\n def archive_file(self):\n return str(archive)\n\n fetcher = Archived_GCSFS(url=\"gs://example/gcs.tar.gz\")\n with spack.stage.Stage(fetcher, path=str(tmp_path)):\n fetcher.fetch()\n" }, { "path": "lib/spack/spack/test/git_fetch.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport copy\nimport os\nimport pathlib\nimport shutil\n\nimport pytest\n\nimport spack.concretize\nimport spack.config\nimport spack.error\nimport spack.fetch_strategy\nimport spack.package_base\nimport spack.platforms\nimport spack.repo\nimport spack.util.git\nfrom spack.fetch_strategy import GitFetchStrategy\nfrom spack.llnl.util.filesystem import mkdirp, touch, working_dir\nfrom spack.package_base import PackageBase\nfrom spack.spec import Spec\nfrom spack.stage import Stage\nfrom spack.variant import SingleValuedVariant\nfrom spack.version import Version\n\n_mock_transport_error = \"Mock HTTP transport error\"\nmin_opt_string = \".\".join(map(str, spack.util.git.MIN_OPT_VERSION))\nmin_direct_commit = \".\".join(map(str, spack.util.git.MIN_DIRECT_COMMIT_FETCH))\n\n\n@pytest.fixture(params=[None, \"1.8.5.2\", \"1.8.5.1\", \"1.7.10\", \"1.7.1\", \"1.7.0\"])\ndef git_version(git, request, monkeypatch):\n \"\"\"Tests GitFetchStrategy behavior for different git versions.\n\n GitFetchStrategy tries to optimize using features of newer git\n versions, but needs to work with older git versions. To ensure code\n paths for old versions still work, we fake it out here and make it\n use the backward-compatibility code paths with newer git versions.\n \"\"\"\n real_git_version = Version(spack.util.git.extract_git_version_str(git))\n\n if request.param is None:\n # Don't patch; run with the real git_version method.\n yield real_git_version\n else:\n test_git_version = Version(request.param)\n if test_git_version > real_git_version:\n pytest.skip(\"Can't test clone logic for newer version of git.\")\n\n # Patch the fetch strategy to think it's using a lower git version.\n # we use this to test what we'd need to do with older git versions\n # using a newer git installation.\n monkeypatch.setattr(spack.util.git, \"extract_git_version_str\", lambda _: request.param)\n yield test_git_version\n\n\n@pytest.fixture\ndef mock_bad_git(mock_util_executable):\n \"\"\"\n Test GitFetchStrategy behavior with a bad git command for git >= 1.7.1\n to trigger a SpackError.\n \"\"\"\n\n _, should_fail, registered_respones = mock_util_executable\n should_fail.extend([\"clone\", \"fetch\"])\n registered_respones[\"--version\"] = \"1.7.1\"\n\n\ndef test_bad_git(tmp_path: pathlib.Path, mock_bad_git):\n \"\"\"Trigger a SpackError when attempt a fetch with a bad git.\"\"\"\n testpath = str(tmp_path)\n\n with pytest.raises(spack.error.SpackError):\n fetcher = GitFetchStrategy(git=\"file:///not-a-real-git-repo\")\n with Stage(fetcher, path=testpath):\n fetcher.fetch()\n\n\n@pytest.mark.parametrize(\"type_of_test\", [\"default\", \"branch\", \"tag\", \"commit\"])\n@pytest.mark.parametrize(\"secure\", [True, False])\ndef test_fetch(\n git,\n type_of_test,\n secure,\n mock_git_repository,\n default_mock_concretization,\n mutable_mock_repo,\n git_version,\n monkeypatch,\n):\n \"\"\"Tries to:\n\n 1. Fetch the repo using a fetch strategy constructed with\n supplied args (they depend on type_of_test).\n 2. Check if the test_file is in the checked out repository.\n 3. Assert that the repository is at the revision supplied.\n 4. Add and remove some files, then reset the repo, and\n ensure it's all there again.\n \"\"\"\n # Retrieve the right test parameters\n t = mock_git_repository.checks[type_of_test]\n h = mock_git_repository.hash\n\n pkg_class = spack.repo.PATH.get_pkg_class(\"git-test\")\n # This would fail using the default-no-per-version-git check but that\n # isn't included in this test\n monkeypatch.delattr(pkg_class, \"git\")\n\n # Construct the package under test\n s = default_mock_concretization(\"git-test\")\n monkeypatch.setitem(s.package.versions, Version(\"git\"), t.args)\n\n if type_of_test == \"commit\":\n s.variants[\"commit\"] = SingleValuedVariant(\"commit\", t.args[\"commit\"])\n\n # Enter the stage directory and check some properties\n with s.package.stage:\n with spack.config.override(\"config:verify_ssl\", secure):\n s.package.do_stage()\n\n with working_dir(s.package.stage.source_path):\n assert h(\"HEAD\") == h(t.revision)\n\n file_path = os.path.join(s.package.stage.source_path, t.file)\n assert os.path.isdir(s.package.stage.source_path)\n assert os.path.isfile(file_path)\n\n os.unlink(file_path)\n assert not os.path.isfile(file_path)\n\n untracked_file = \"foobarbaz\"\n touch(untracked_file)\n assert os.path.isfile(untracked_file)\n s.package.do_restage()\n assert not os.path.isfile(untracked_file)\n\n assert os.path.isdir(s.package.stage.source_path)\n assert os.path.isfile(file_path)\n\n assert h(\"HEAD\") == h(t.revision)\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_fetch_pkg_attr_submodule_init(\n mock_git_repository, default_mock_concretization, mutable_mock_repo, monkeypatch, mock_stage\n):\n \"\"\"In this case the version() args do not contain a 'git' URL, so\n the fetcher must be assembled using the Package-level 'git' attribute.\n This test ensures that the submodules are properly initialized and the\n expected branch file is present.\n \"\"\"\n\n t = mock_git_repository.checks[\"default-no-per-version-git\"]\n pkg_class = spack.repo.PATH.get_pkg_class(\"git-test\")\n # For this test, the version args don't specify 'git' (which is\n # the majority of version specifications)\n monkeypatch.setattr(pkg_class, \"git\", mock_git_repository.url)\n\n # Construct the package under test\n s = default_mock_concretization(\"git-test\")\n monkeypatch.setitem(s.package.versions, Version(\"git\"), t.args)\n\n s.package.do_stage()\n collected_fnames = set()\n for root, dirs, files in os.walk(s.package.stage.source_path):\n collected_fnames.update(files)\n # The submodules generate files with the prefix \"r0_file_\"\n assert {\"r0_file_0\", \"r0_file_1\", t.file} < collected_fnames\n\n\n@pytest.mark.skipif(\n str(spack.platforms.host()) == \"windows\",\n reason=(\n \"Git fails to clone because the src/dst paths\"\n \" are too long: the name of the staging directory\"\n \" for ad-hoc Git commit versions is longer than\"\n \" other staged sources\"\n ),\n)\n@pytest.mark.disable_clean_stage_check\ndef test_adhoc_version_submodules(\n mock_git_repository,\n config,\n mutable_mock_repo,\n monkeypatch,\n mock_stage,\n override_git_repos_cache_path,\n):\n t = mock_git_repository.checks[\"tag\"]\n # Construct the package under test\n pkg_class = spack.repo.PATH.get_pkg_class(\"git-test\")\n monkeypatch.setitem(pkg_class.versions, Version(\"git\"), t.args)\n monkeypatch.setattr(pkg_class, \"git\", mock_git_repository.url, raising=False)\n\n spec = spack.concretize.concretize_one(\n Spec(\"git-test@{0}\".format(mock_git_repository.unversioned_commit))\n )\n spec.package.do_stage()\n collected_fnames = set()\n for root, dirs, files in os.walk(spec.package.stage.source_path):\n collected_fnames.update(files)\n # The submodules generate files with the prefix \"r0_file_\"\n assert set([\"r0_file_0\", \"r0_file_1\"]) < collected_fnames\n\n\n@pytest.mark.parametrize(\"type_of_test\", [\"branch\", \"commit\"])\ndef test_debug_fetch(\n mock_packages, type_of_test, mock_git_repository, default_mock_concretization, monkeypatch\n):\n \"\"\"Fetch the repo with debug enabled.\"\"\"\n # Retrieve the right test parameters\n t = mock_git_repository.checks[type_of_test]\n\n # Construct the package under test\n s = default_mock_concretization(\"git-test\")\n monkeypatch.setitem(s.package.versions, Version(\"git\"), t.args)\n\n # Fetch then ensure source path exists\n with s.package.stage:\n with spack.config.override(\"config:debug\", True):\n s.package.do_fetch()\n assert os.path.isdir(s.package.stage.source_path)\n\n\ndef test_git_extra_fetch(git, tmp_path: pathlib.Path):\n \"\"\"Ensure a fetch after 'expanding' is effectively a no-op.\"\"\"\n testpath = str(tmp_path)\n\n fetcher = GitFetchStrategy(git=\"file:///not-a-real-git-repo\")\n with Stage(fetcher, path=testpath) as stage:\n mkdirp(stage.source_path)\n fetcher.fetch() # Use fetcher to fetch for code coverage\n shutil.rmtree(stage.source_path)\n\n\ndef test_needs_stage(git):\n \"\"\"Trigger a NoStageError when attempt a fetch without a stage.\"\"\"\n with pytest.raises(\n spack.fetch_strategy.NoStageError, match=r\"set_stage.*before calling fetch\"\n ):\n fetcher = GitFetchStrategy(git=\"file:///not-a-real-git-repo\")\n fetcher.fetch()\n\n\n@pytest.mark.parametrize(\"get_full_repo\", [True, False])\n@pytest.mark.parametrize(\"use_commit\", [True, False])\ndef test_get_full_repo(\n get_full_repo,\n use_commit,\n git_version,\n mock_git_repository,\n default_mock_concretization,\n mutable_mock_repo,\n monkeypatch,\n):\n \"\"\"Ensure that we can clone a full repository.\"\"\"\n\n if git_version < Version(min_opt_string):\n pytest.skip(\"Not testing get_full_repo for older git {0}\".format(git_version))\n\n # newer git allows for direct commit fetching\n can_use_direct_commit = git_version >= Version(min_direct_commit)\n\n secure = True\n type_of_test = \"tag-branch\"\n\n t = mock_git_repository.checks[type_of_test]\n\n spec_string = \"git-test\"\n\n s = default_mock_concretization(spec_string)\n\n args = copy.copy(t.args)\n args[\"get_full_repo\"] = get_full_repo\n monkeypatch.setitem(s.package.versions, Version(\"git\"), args)\n\n if use_commit:\n git_exe = mock_git_repository.git_exe\n url = mock_git_repository.url\n commit = git_exe(\"ls-remote\", url, t.revision, output=str).strip().split()[0]\n s.variants[\"commit\"] = SingleValuedVariant(\"commit\", commit)\n if can_use_direct_commit:\n path = mock_git_repository.path\n git_exe(\"-C\", path, \"config\", \"uploadpack.allowReachableSHA1InWant\", \"true\")\n\n with s.package.stage:\n with spack.config.override(\"config:verify_ssl\", secure):\n s.package.do_stage()\n with working_dir(s.package.stage.source_path):\n branches = mock_git_repository.git_exe(\"branch\", \"-a\", output=str).splitlines()\n nbranches = len(branches)\n commits = mock_git_repository.git_exe(\n \"log\",\n \"--graph\",\n \"--pretty=format:%h -%d %s (%ci) <%an>\",\n \"--abbrev-commit\",\n output=str,\n ).splitlines()\n ncommits = len(commits)\n\n if get_full_repo:\n # default branch commit, plus checkout commit\n assert ncommits == 2, commits\n assert nbranches >= 5, branches\n else:\n assert ncommits == 1, commits\n if can_use_direct_commit:\n if use_commit:\n # only commit (detached state)\n assert nbranches == 1, branches\n else:\n # tag, commit (detached state)\n assert nbranches == 2, branches\n else:\n if use_commit:\n # default branch, tag, commit (detached state)\n # git does not have a rewind, avoid messing with git history by\n # accepting detachment\n assert nbranches == 3, branches\n else:\n # default branch plus tag\n assert nbranches == 2, branches\n\n\n@pytest.mark.disable_clean_stage_check\n@pytest.mark.parametrize(\"submodules\", [True, False])\ndef test_gitsubmodule(\n submodules, mock_git_repository, default_mock_concretization, mutable_mock_repo, monkeypatch\n):\n \"\"\"\n Test GitFetchStrategy behavior with submodules. This package\n has a `submodules` property which is always True: when a specific\n version also indicates to include submodules, this should not\n interfere; if the specific version explicitly requests that\n submodules *not* be initialized, this should override the\n Package-level request.\n \"\"\"\n type_of_test = \"tag-branch\"\n t = mock_git_repository.checks[type_of_test]\n\n # Construct the package under test\n s = default_mock_concretization(\"git-test\")\n args = copy.copy(t.args)\n args[\"submodules\"] = submodules\n monkeypatch.setitem(s.package.versions, Version(\"git\"), args)\n s.package.do_stage()\n with working_dir(s.package.stage.source_path):\n for submodule_count in range(2):\n file_path = os.path.join(\n s.package.stage.source_path,\n \"third_party/submodule{0}/r0_file_{0}\".format(submodule_count),\n )\n if submodules:\n assert os.path.isfile(file_path)\n else:\n assert not os.path.isfile(file_path)\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_gitsubmodules_callable(\n mock_git_repository, default_mock_concretization, mutable_mock_repo, monkeypatch\n):\n \"\"\"\n Test GitFetchStrategy behavior with submodules selected after concretization\n \"\"\"\n\n def submodules_callback(package):\n assert isinstance(package, PackageBase)\n name = \"third_party/submodule0\"\n return [name]\n\n type_of_test = \"tag-branch\"\n t = mock_git_repository.checks[type_of_test]\n\n # Construct the package under test\n s = default_mock_concretization(\"git-test\")\n args = copy.copy(t.args)\n args[\"submodules\"] = submodules_callback\n monkeypatch.setitem(s.package.versions, Version(\"git\"), args)\n s.package.do_stage()\n with working_dir(s.package.stage.source_path):\n file_path = os.path.join(s.package.stage.source_path, \"third_party/submodule0/r0_file_0\")\n assert os.path.isfile(file_path)\n file_path = os.path.join(s.package.stage.source_path, \"third_party/submodule1/r0_file_1\")\n assert not os.path.isfile(file_path)\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_gitsubmodules_delete(\n mock_git_repository, default_mock_concretization, mutable_mock_repo, monkeypatch\n):\n \"\"\"\n Test GitFetchStrategy behavior with submodules_delete\n \"\"\"\n type_of_test = \"tag-branch\"\n t = mock_git_repository.checks[type_of_test]\n\n # Construct the package under test\n s = default_mock_concretization(\"git-test\")\n args = copy.copy(t.args)\n args[\"submodules\"] = True\n args[\"submodules_delete\"] = [\"third_party/submodule0\", \"third_party/submodule1\"]\n monkeypatch.setitem(s.package.versions, Version(\"git\"), args)\n s.package.do_stage()\n with working_dir(s.package.stage.source_path):\n file_path = os.path.join(s.package.stage.source_path, \"third_party/submodule0\")\n assert not os.path.isdir(file_path)\n file_path = os.path.join(s.package.stage.source_path, \"third_party/submodule1\")\n assert not os.path.isdir(file_path)\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_gitsubmodules_falsey(\n mock_git_repository, default_mock_concretization, mutable_mock_repo, monkeypatch\n):\n \"\"\"\n Test GitFetchStrategy behavior when callable submodules returns Falsey\n \"\"\"\n\n def submodules_callback(package):\n assert isinstance(package, PackageBase)\n return False\n\n type_of_test = \"tag-branch\"\n t = mock_git_repository.checks[type_of_test]\n\n # Construct the package under test\n s = default_mock_concretization(\"git-test\")\n args = copy.copy(t.args)\n args[\"submodules\"] = submodules_callback\n monkeypatch.setitem(s.package.versions, Version(\"git\"), args)\n s.package.do_stage()\n with working_dir(s.package.stage.source_path):\n file_path = os.path.join(s.package.stage.source_path, \"third_party/submodule0/r0_file_0\")\n assert not os.path.isfile(file_path)\n file_path = os.path.join(s.package.stage.source_path, \"third_party/submodule1/r0_file_1\")\n assert not os.path.isfile(file_path)\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_git_sparse_paths_partial_clone(\n mock_git_repository, git_version, default_mock_concretization, mutable_mock_repo, monkeypatch\n):\n \"\"\"\n Test partial clone of repository when using git_sparse_paths property\n \"\"\"\n type_of_test = \"many-directories\"\n sparse_paths = [\"dir0\"]\n omitted_paths = [\"dir1\", \"dir2\"]\n t = mock_git_repository.checks[type_of_test]\n args = copy.copy(t.args)\n args[\"git_sparse_paths\"] = sparse_paths\n s = default_mock_concretization(\"git-test\")\n monkeypatch.setitem(s.package.versions, Version(\"git\"), args)\n s.package.do_stage()\n with working_dir(s.package.stage.source_path):\n # top level directory files are cloned via sparse-checkout\n assert os.path.isfile(\"r0_file\")\n\n for p in sparse_paths:\n assert os.path.isdir(p)\n\n if git_version < Version(\"2.26.0.0\"):\n # older versions of git should fall back to a full clone\n for p in omitted_paths:\n assert os.path.isdir(p)\n else:\n for p in omitted_paths:\n assert not os.path.isdir(p)\n\n # fixture file is in the sparse-path expansion tree\n assert os.path.isfile(t.file)\n\n\n@pytest.mark.regression(\"50699\")\ndef test_git_sparse_path_have_unique_mirror_projections(\n git, mock_git_repository, mutable_mock_repo, monkeypatch, mutable_config\n):\n \"\"\"\n Confirm two packages with different sparse paths but the same git commit\n have different mirror projections so tarfiles in the mirror are unique\n and don't get overwritten\n \"\"\"\n repo_path = mock_git_repository.path\n monkeypatch.setattr(\n spack.package_base.PackageBase, \"git\", pathlib.Path(repo_path).as_uri(), raising=False\n )\n gold_commit = git(\"-C\", repo_path, \"rev-parse\", \"many_dirs\", output=str).strip()\n s_a = spack.concretize.concretize_one(f\"git-sparse-a commit={gold_commit}\")\n s_b = spack.concretize.concretize_one(f\"git-sparse-b commit={gold_commit}\")\n assert s_a.package.stage[0].mirror_layout.path != s_b.package.stage[0].mirror_layout.path\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_commit_variant_clone(\n git, default_mock_concretization, mutable_mock_repo, mock_git_version_info, monkeypatch\n):\n\n repo_path, filename, commits = mock_git_version_info\n test_commit = commits[-2]\n s = default_mock_concretization(\"git-test\")\n args = {\"git\": pathlib.Path(repo_path).as_uri()}\n monkeypatch.setitem(s.package.versions, Version(\"git\"), args)\n s.variants[\"commit\"] = SingleValuedVariant(\"commit\", test_commit)\n s.package.do_stage()\n with working_dir(s.package.stage.source_path):\n assert git(\"rev-parse\", \"HEAD\", output=str, error=str).strip() == test_commit\n" }, { "path": "lib/spack/spack/test/graph.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport io\n\nimport spack.concretize\nimport spack.graph\n\n\ndef test_dynamic_dot_graph_mpileaks(default_mock_concretization):\n \"\"\"Test dynamically graphing the mpileaks package.\"\"\"\n s = default_mock_concretization(\"mpileaks\")\n stream = io.StringIO()\n spack.graph.graph_dot([s], out=stream)\n dot = stream.getvalue()\n\n nodes_to_check = [\"mpileaks\", \"mpi\", \"callpath\", \"dyninst\", \"libdwarf\", \"libelf\"]\n hashes, builder = {}, spack.graph.SimpleDAG()\n for name in nodes_to_check:\n current = s[name]\n current_hash = current.dag_hash()\n hashes[name] = current_hash\n node_options = builder.node_entry(current)[1]\n assert node_options in dot\n\n dependencies_to_check = [\n (\"dyninst\", \"libdwarf\"),\n (\"callpath\", \"dyninst\"),\n (\"mpileaks\", \"mpi\"),\n (\"libdwarf\", \"libelf\"),\n (\"callpath\", \"mpi\"),\n (\"mpileaks\", \"callpath\"),\n (\"dyninst\", \"libelf\"),\n ]\n for parent, child in dependencies_to_check:\n assert ' \"{0}\" -> \"{1}\"\\n'.format(hashes[parent], hashes[child]) in dot\n\n\ndef test_ascii_graph_mpileaks(config, mock_packages, monkeypatch):\n monkeypatch.setattr(spack.graph.AsciiGraph, \"_node_label\", lambda self, node: node.name)\n s = spack.concretize.concretize_one(\"mpileaks\")\n\n stream = io.StringIO()\n graph = spack.graph.AsciiGraph()\n graph.write(s, out=stream, color=False)\n graph_str = stream.getvalue()\n graph_str = \"\\n\".join([line.rstrip() for line in graph_str.split(\"\\n\")])\n\n assert (\n graph_str\n == r\"\"\"o mpileaks\n|\\\n| |\\\n| | |\\\n| | | |\\\n| | | | o callpath\n| |_|_|/|\n|/| |_|/|\n| |/| |/|\n| | |/|/|\n| | | | o dyninst\n| | |_|/|\n| |/| |/|\n| | |/|/|\n| | | | |\\\no | | | | | mpich\n|\\| | | | |\n|\\ \\ \\ \\ \\ \\\n| |_|/ / / /\n|/| | | | |\n| |/ / / /\n| | | | o libdwarf\n| |_|_|/|\n|/| |_|/|\n| |/| |/|\n| | |/|/\n| | | o libelf\n| |_|/|\n|/| |/|\n| |/|/\n| o | compiler-wrapper\n| /\n| o gcc-runtime\n|/\no gcc\n\"\"\"\n or graph_str\n == r\"\"\"o mpileaks\n|\\\n| |\\\n| | |\\\n| | | o callpath\n| |_|/|\n|/| |/|\n| |/|/|\n| | | o dyninst\n| | |/|\n| |/|/|\n| | | |\\\no | | | | mpich\n|\\| | | |\n| |/ / /\n|/| | |\n| | | o libdwarf\n| |_|/|\n|/| |/|\n| |/|/\n| | o libelf\n| |/|\n|/|/\n| o gcc-runtime\n|/\no gcc\n\"\"\"\n )\n" }, { "path": "lib/spack/spack/test/hg_fetch.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport pathlib\n\nimport pytest\n\nimport spack.concretize\nimport spack.config\nfrom spack.fetch_strategy import HgFetchStrategy\nfrom spack.llnl.util.filesystem import mkdirp, touch, working_dir\nfrom spack.stage import Stage\nfrom spack.util.executable import which\nfrom spack.version import Version\n\n# Test functionality covered is supported on Windows, but currently failing\n# and expected to be fixed\npytestmark = [\n pytest.mark.skipif(not which(\"hg\"), reason=\"requires mercurial to be installed\"),\n pytest.mark.not_on_windows(\"Failing on Win\"),\n]\n\n\n@pytest.mark.parametrize(\"type_of_test\", [\"default\", \"rev0\"])\n@pytest.mark.parametrize(\"secure\", [True, False])\ndef test_fetch(type_of_test, secure, mock_hg_repository, config, mutable_mock_repo, monkeypatch):\n \"\"\"Tries to:\n\n 1. Fetch the repo using a fetch strategy constructed with\n supplied args (they depend on type_of_test).\n 2. Check if the test_file is in the checked out repository.\n 3. Assert that the repository is at the revision supplied.\n 4. Add and remove some files, then reset the repo, and\n ensure it's all there again.\n \"\"\"\n # Retrieve the right test parameters\n t = mock_hg_repository.checks[type_of_test]\n h = mock_hg_repository.hash\n\n # Construct the package under test\n s = spack.concretize.concretize_one(\"hg-test\")\n monkeypatch.setitem(s.package.versions, Version(\"hg\"), t.args)\n\n # Enter the stage directory and check some properties\n with s.package.stage:\n with spack.config.override(\"config:verify_ssl\", secure):\n s.package.do_stage()\n\n with working_dir(s.package.stage.source_path):\n assert h() == t.revision\n\n file_path = os.path.join(s.package.stage.source_path, t.file)\n assert os.path.isdir(s.package.stage.source_path)\n assert os.path.isfile(file_path)\n\n os.unlink(file_path)\n assert not os.path.isfile(file_path)\n\n untracked_file = \"foobarbaz\"\n touch(untracked_file)\n assert os.path.isfile(untracked_file)\n s.package.do_restage()\n assert not os.path.isfile(untracked_file)\n\n assert os.path.isdir(s.package.stage.source_path)\n assert os.path.isfile(file_path)\n\n assert h() == t.revision\n\n\ndef test_hg_extra_fetch(tmp_path: pathlib.Path):\n \"\"\"Ensure a fetch after expanding is effectively a no-op.\"\"\"\n testpath = str(tmp_path)\n\n fetcher = HgFetchStrategy(hg=\"file:///not-a-real-hg-repo\")\n with Stage(fetcher, path=testpath) as stage:\n source_path = stage.source_path\n mkdirp(source_path)\n fetcher.fetch()\n" }, { "path": "lib/spack/spack/test/hooks/absolutify_elf_sonames.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\nimport os\nimport pathlib\n\nimport pytest\n\nimport spack.llnl.util.filesystem as fs\nimport spack.platforms\nfrom spack.hooks.absolutify_elf_sonames import SharedLibrariesVisitor, find_and_patch_sonames\nfrom spack.util.executable import Executable\n\n\ndef skip_unless_linux(f):\n return pytest.mark.skipif(\n str(spack.platforms.real_host()) != \"linux\", reason=\"only tested on linux for now\"\n )(f)\n\n\nclass ExecutableIntercept:\n def __init__(self):\n self.calls = []\n\n def __call__(self, *args, **kwargs):\n self.calls.append(args)\n\n @property\n def returncode(self):\n return 0\n\n\n@pytest.mark.requires_executables(\"gcc\")\n@skip_unless_linux\ndef test_shared_libraries_visitor(tmp_path: pathlib.Path):\n \"\"\"Integration test for soname rewriting\"\"\"\n gcc = Executable(\"gcc\")\n\n # Create a directory structure like this:\n # ./no-soname.so # just a shared library without a soname\n # ./soname.so # a shared library with a soname\n # ./executable.so # an executable masquerading as a shared lib\n # ./libskipme.so # a shared library with a soname\n # ./mydir/parent_dir -> .. # a symlinked dir, causing a cycle\n # ./mydir/skip_symlink -> ../libskipme # a symlink to a library\n\n with fs.working_dir(str(tmp_path)):\n with open(\"hello.c\", \"w\", encoding=\"utf-8\") as f:\n f.write(\"int main(){return 0;}\")\n gcc(\"hello.c\", \"-o\", \"no-soname.so\", \"--shared\")\n gcc(\"hello.c\", \"-o\", \"soname.so\", \"--shared\", \"-Wl,-soname,example.so\")\n gcc(\"hello.c\", \"-pie\", \"-o\", \"executable.so\")\n gcc(\"hello.c\", \"-o\", \"libskipme.so\", \"-Wl,-soname,libskipme.so\")\n os.mkdir(\"my_dir\")\n os.symlink(\"..\", os.path.join(\"my_dir\", \"parent_dir\"))\n os.symlink(os.path.join(\"..\", \"libskipme.so\"), os.path.join(\"my_dir\", \"skip_symlink\"))\n\n # Visit the whole prefix, but exclude `skip_symlink`\n visitor = SharedLibrariesVisitor(exclude_list=[\"skip_symlink\"])\n fs.visit_directory_tree(str(tmp_path), visitor)\n relative_paths = visitor.get_shared_libraries_relative_paths()\n\n assert \"no-soname.so\" in relative_paths\n assert \"soname.so\" in relative_paths\n assert \"executable.so\" not in relative_paths\n assert \"libskipme.so\" not in relative_paths\n\n # Run the full hook of finding libs and setting sonames.\n patchelf = ExecutableIntercept()\n find_and_patch_sonames(str(tmp_path), [\"skip_symlink\"], patchelf)\n assert len(patchelf.calls) == 2\n elf_1 = tmp_path / \"no-soname.so\"\n elf_2 = tmp_path / \"soname.so\"\n assert (\"--set-soname\", str(elf_1), str(elf_1)) in patchelf.calls\n assert (\"--set-soname\", str(elf_2), str(elf_2)) in patchelf.calls\n" }, { "path": "lib/spack/spack/test/install.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport pathlib\nimport shutil\nimport sys\n\nimport pytest\n\nimport spack.build_environment\nimport spack.builder\nimport spack.concretize\nimport spack.config\nimport spack.database\nimport spack.error\nimport spack.installer\nimport spack.llnl.util.filesystem as fs\nimport spack.mirrors.mirror\nimport spack.mirrors.utils\nimport spack.package_base\nimport spack.patch\nimport spack.repo\nimport spack.store\nimport spack.util.spack_json as sjson\nfrom spack import binary_distribution\nfrom spack.error import InstallError\nfrom spack.installer import PackageInstaller\nfrom spack.package_base import (\n PackageBase,\n PackageStillNeededError,\n _spack_build_envfile,\n _spack_build_logfile,\n _spack_configure_argsfile,\n spack_times_log,\n)\nfrom spack.spec import Spec\n\n\ndef find_nothing(*args):\n raise spack.repo.UnknownPackageError(\"Repo package access is disabled for test\")\n\n\ndef test_install_and_uninstall(install_mockery, mock_fetch, monkeypatch):\n spec = spack.concretize.concretize_one(\"trivial-install-test-package\")\n\n PackageInstaller([spec.package], explicit=True).install()\n assert spec.installed\n\n spec.package.do_uninstall()\n assert not spec.installed\n\n\n@pytest.mark.regression(\"11870\")\ndef test_uninstall_non_existing_package(install_mockery, mock_fetch, monkeypatch):\n \"\"\"Ensure that we can uninstall a package that has been deleted from the repo\"\"\"\n spec = spack.concretize.concretize_one(\"trivial-install-test-package\")\n\n PackageInstaller([spec.package], explicit=True).install()\n assert spec.installed\n\n # Mock deletion of the package\n spec._package = None\n monkeypatch.setattr(spack.repo.PATH, \"get\", find_nothing)\n with pytest.raises(spack.repo.UnknownPackageError):\n spec.package\n\n # Ensure we can uninstall it\n PackageBase.uninstall_by_spec(spec)\n assert not spec.installed\n\n\ndef test_pkg_attributes(install_mockery, mock_fetch, monkeypatch):\n # Get a basic concrete spec for the dummy package.\n spec = spack.concretize.concretize_one(\"attributes-foo-app ^attributes-foo\")\n assert spec.concrete\n\n pkg = spec.package\n PackageInstaller([pkg], explicit=True).install()\n foo = \"attributes-foo\"\n assert spec[\"bar\"].prefix == spec[foo].prefix\n assert spec[\"baz\"].prefix == spec[foo].prefix\n\n assert spec[foo].home == spec[foo].prefix\n assert spec[\"bar\"].home == spec[foo].home\n assert spec[\"baz\"].home == spec[foo].prefix.baz\n\n foo_headers = spec[foo].headers\n # assert foo_headers.basenames == ['foo.h']\n assert foo_headers.directories == [spec[foo].home.include]\n bar_headers = spec[\"bar\"].headers\n # assert bar_headers.basenames == ['bar.h']\n assert bar_headers.directories == [spec[\"bar\"].home.include]\n baz_headers = spec[\"baz\"].headers\n # assert baz_headers.basenames == ['baz.h']\n assert baz_headers.directories == [spec[\"baz\"].home.include]\n\n lib_suffix = \".so\"\n if sys.platform == \"win32\":\n lib_suffix = \".dll\"\n elif sys.platform == \"darwin\":\n lib_suffix = \".dylib\"\n\n foo_libs = spec[foo].libs\n assert foo_libs.basenames == [\"libFoo\" + lib_suffix]\n assert foo_libs.directories == [spec[foo].home.lib64]\n bar_libs = spec[\"bar\"].libs\n assert bar_libs.basenames == [\"libFooBar\" + lib_suffix]\n assert bar_libs.directories == [spec[\"bar\"].home.lib64]\n baz_libs = spec[\"baz\"].libs\n assert baz_libs.basenames == [\"libFooBaz\" + lib_suffix]\n assert baz_libs.directories == [spec[\"baz\"].home.lib]\n\n\ndef mock_remove_prefix(*args):\n raise MockInstallError(\"Intentional error\", \"Mock remove_prefix method intentionally fails\")\n\n\nclass RemovePrefixChecker:\n def __init__(self, wrapped_rm_prefix):\n self.removed = False\n self.wrapped_rm_prefix = wrapped_rm_prefix\n\n def remove_prefix(self):\n self.removed = True\n self.wrapped_rm_prefix()\n\n\ndef test_partial_install_delete_prefix_and_stage(install_mockery, mock_fetch, working_env):\n s = spack.concretize.concretize_one(\"canfail\")\n s.package.succeed = False\n\n instance_rm_prefix = s.package.remove_prefix\n\n s.package.remove_prefix = mock_remove_prefix\n with pytest.raises(MockInstallError):\n PackageInstaller([s.package], explicit=True).install()\n assert os.path.isdir(s.package.prefix)\n rm_prefix_checker = RemovePrefixChecker(instance_rm_prefix)\n s.package.remove_prefix = rm_prefix_checker.remove_prefix\n\n # must clear failure markings for the package before re-installing it\n spack.store.STORE.failure_tracker.clear(s, True)\n\n s.package.succeed = True\n spack.builder._BUILDERS.clear() # the builder is cached with a copy of the pkg's __dict__.\n\n PackageInstaller([s.package], explicit=True, restage=True).install()\n assert rm_prefix_checker.removed\n assert s.package.spec.installed\n\n\n@pytest.mark.not_on_windows(\"Fails spuriously on Windows\")\n@pytest.mark.disable_clean_stage_check\ndef test_failing_overwrite_install_should_keep_previous_installation(\n mock_fetch, install_mockery, working_env\n):\n \"\"\"\n Make sure that whenever `spack install --overwrite` fails, spack restores\n the original install prefix instead of cleaning it.\n \"\"\"\n # Do a successful install\n s = spack.concretize.concretize_one(\"canfail\")\n s.package.succeed = True\n\n # Do a failing overwrite install\n PackageInstaller([s.package], explicit=True).install()\n s.package.succeed = False\n spack.builder._BUILDERS.clear() # the builder is cached with a copy of the pkg's __dict__.\n kwargs = {\"overwrite\": [s.dag_hash()]}\n\n with pytest.raises(Exception):\n PackageInstaller([s.package], explicit=True, **kwargs).install()\n\n assert s.package.spec.installed\n assert os.path.exists(s.prefix)\n\n\ndef test_dont_add_patches_to_installed_package(install_mockery, mock_fetch, monkeypatch):\n dependency = spack.concretize.concretize_one(\"dependency-install\")\n PackageInstaller([dependency.package], explicit=True).install()\n\n dependency_hash = dependency.dag_hash()\n dependent = spack.concretize.concretize_one(\"dependent-install ^/\" + dependency_hash)\n\n monkeypatch.setitem(\n dependency.package.patches,\n \"dependency-install\",\n [spack.patch.UrlPatch(dependent.package, \"file://fake.patch\", sha256=\"unused-hash\")],\n )\n\n assert dependent[\"dependency-install\"] == dependency\n\n\ndef test_installed_dependency_request_conflicts(install_mockery, mock_fetch, mutable_mock_repo):\n dependency = spack.concretize.concretize_one(\"dependency-install\")\n PackageInstaller([dependency.package], explicit=True).install()\n\n dependency_hash = dependency.dag_hash()\n dependent = Spec(\"conflicting-dependent ^/\" + dependency_hash)\n with pytest.raises(spack.error.UnsatisfiableSpecError):\n spack.concretize.concretize_one(dependent)\n\n\ndef test_install_times(install_mockery, mock_fetch, mutable_mock_repo):\n \"\"\"Test install times added.\"\"\"\n spec = spack.concretize.concretize_one(\"dev-build-test-install-phases\")\n PackageInstaller([spec.package], explicit=True).install()\n\n # Ensure dependency directory exists after the installation.\n install_times = os.path.join(spec.package.prefix, \".spack\", spack_times_log)\n assert os.path.isfile(install_times)\n\n # Ensure the phases are included\n with open(install_times, \"r\", encoding=\"utf-8\") as timefile:\n times = sjson.load(timefile.read())\n\n # The order should be maintained\n phases = [x[\"name\"] for x in times[\"phases\"]]\n assert phases == [\"stage\", \"one\", \"two\", \"three\", \"install\", \"post-install\"]\n assert all(isinstance(x[\"seconds\"], float) for x in times[\"phases\"])\n\n\n@pytest.fixture()\ndef install_upstream(tmp_path_factory: pytest.TempPathFactory, gen_mock_layout, install_mockery):\n \"\"\"Provides a function that installs a specified set of specs to an\n upstream database. The function returns a store which points to the\n upstream, as well as the upstream layout (for verifying that dependent\n installs are using the upstream installs).\n \"\"\"\n mock_db_root = str(tmp_path_factory.mktemp(\"mock_db_root\"))\n upstream_layout = gen_mock_layout(\"a\")\n prepared_db = spack.database.Database(mock_db_root, layout=upstream_layout)\n spack.config.CONFIG.push_scope(\n spack.config.InternalConfigScope(\n name=\"install-upstream-fixture\",\n data={\"upstreams\": {\"mock1\": {\"install_tree\": prepared_db.root}}},\n )\n )\n\n def _install_upstream(*specs):\n for spec_str in specs:\n prepared_db.add(spack.concretize.concretize_one(spec_str))\n downstream_root = str(tmp_path_factory.mktemp(\"mock_downstream_db_root\"))\n return downstream_root, upstream_layout\n\n return _install_upstream\n\n\ndef test_installed_upstream_external(install_upstream, mock_fetch):\n \"\"\"Check that when a dependency package is recorded as installed in\n an upstream database that it is not reinstalled.\n \"\"\"\n store_root, _ = install_upstream(\"externaltool\")\n with spack.store.use_store(store_root):\n dependent = spack.concretize.concretize_one(\"externaltest\")\n\n new_dependency = dependent[\"externaltool\"]\n assert new_dependency.external\n assert new_dependency.prefix == os.path.sep + os.path.join(\"path\", \"to\", \"external_tool\")\n\n PackageInstaller([dependent.package], explicit=True).install()\n\n assert not os.path.exists(new_dependency.prefix)\n assert os.path.exists(dependent.prefix)\n\n\ndef test_installed_upstream(install_upstream, mock_fetch):\n \"\"\"Check that when a dependency package is recorded as installed in\n an upstream database that it is not reinstalled.\n \"\"\"\n store_root, upstream_layout = install_upstream(\"dependency-install\")\n with spack.store.use_store(store_root):\n dependency = spack.concretize.concretize_one(\"dependency-install\")\n dependent = spack.concretize.concretize_one(\"dependent-install\")\n\n new_dependency = dependent[\"dependency-install\"]\n assert new_dependency.installed_upstream\n assert new_dependency.prefix == upstream_layout.path_for_spec(dependency)\n\n PackageInstaller([dependent.package], explicit=True).install()\n\n assert not os.path.exists(new_dependency.prefix)\n assert os.path.exists(dependent.prefix)\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_partial_install_keep_prefix(install_mockery, mock_fetch, monkeypatch, working_env):\n s = spack.concretize.concretize_one(\"canfail\")\n s.package.succeed = False\n\n # If remove_prefix is called at any point in this test, that is an error\n monkeypatch.setattr(spack.package_base.PackageBase, \"remove_prefix\", mock_remove_prefix)\n with pytest.raises(spack.build_environment.ChildError):\n PackageInstaller([s.package], explicit=True, keep_prefix=True).install()\n assert os.path.exists(s.package.prefix)\n\n # must clear failure markings for the package before re-installing it\n spack.store.STORE.failure_tracker.clear(s, True)\n\n s.package.succeed = True\n spack.builder._BUILDERS.clear() # the builder is cached with a copy of the pkg's __dict__.\n PackageInstaller([s.package], explicit=True, keep_prefix=True).install()\n assert s.package.spec.installed\n\n\ndef test_second_install_no_overwrite_first(install_mockery, mock_fetch, monkeypatch):\n s = spack.concretize.concretize_one(\"canfail\")\n monkeypatch.setattr(spack.package_base.PackageBase, \"remove_prefix\", mock_remove_prefix)\n\n s.package.succeed = True\n PackageInstaller([s.package], explicit=True).install()\n assert s.package.spec.installed\n\n # If Package.install is called after this point, it will fail\n s.package.succeed = False\n PackageInstaller([s.package], explicit=True).install()\n\n\ndef test_install_prefix_collision_fails(config, mock_fetch, mock_packages, tmp_path: pathlib.Path):\n \"\"\"\n Test that different specs with coinciding install prefixes will fail\n to install.\n \"\"\"\n projections = {\"projections\": {\"all\": \"one-prefix-per-package-{name}\"}}\n with spack.store.use_store(str(tmp_path), extra_data=projections):\n with spack.config.override(\"config:checksum\", False):\n pkg_a = spack.concretize.concretize_one(\"libelf@0.8.13\").package\n pkg_b = spack.concretize.concretize_one(\"libelf@0.8.12\").package\n PackageInstaller([pkg_a], explicit=True, fake=True).install()\n\n with pytest.raises(InstallError, match=\"Install prefix collision\"):\n PackageInstaller([pkg_b], explicit=True, fake=True).install()\n\n\ndef test_store(install_mockery, mock_fetch):\n spec = spack.concretize.concretize_one(\"cmake-client\")\n pkg = spec.package\n PackageInstaller([pkg], fake=True, explicit=True).install()\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_failing_build(install_mockery, mock_fetch):\n spec = spack.concretize.concretize_one(\"failing-build\")\n pkg = spec.package\n\n with pytest.raises(spack.build_environment.ChildError, match=\"Expected failure\"):\n PackageInstaller([pkg], explicit=True).install()\n\n\nclass MockInstallError(spack.error.SpackError):\n pass\n\n\ndef test_uninstall_by_spec_errors(mutable_database):\n \"\"\"Test exceptional cases with the uninstall command.\"\"\"\n\n # Try to uninstall a spec that has not been installed\n spec = spack.concretize.concretize_one(\"dependent-install\")\n with pytest.raises(InstallError, match=\"is not installed\"):\n PackageBase.uninstall_by_spec(spec)\n\n # Try an unforced uninstall of a spec with dependencies\n rec = mutable_database.get_record(\"mpich\")\n with pytest.raises(PackageStillNeededError, match=\"Cannot uninstall\"):\n PackageBase.uninstall_by_spec(rec.spec)\n\n\n@pytest.mark.disable_clean_stage_check\n@pytest.mark.use_package_hash\ndef test_nosource_pkg_install(install_mockery, mock_fetch, mock_packages, capfd, ensure_debug):\n \"\"\"Test install phases with the nosource package.\"\"\"\n spec = spack.concretize.concretize_one(\"nosource\")\n pkg = spec.package\n\n # Make sure install works even though there is no associated code.\n PackageInstaller([pkg], explicit=True).install()\n out = capfd.readouterr()\n assert \"Installing dependency-install\" in out[0]\n\n # Make sure a warning for missing code is issued\n assert \"Missing a source id for nosource\" in out[1]\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_nosource_bundle_pkg_install(\n install_mockery, mock_fetch, mock_packages, capfd, ensure_debug\n):\n \"\"\"Test install phases with the nosource-bundle package.\"\"\"\n spec = spack.concretize.concretize_one(\"nosource-bundle\")\n pkg = spec.package\n\n # Make sure install works even though there is no associated code.\n PackageInstaller([pkg], explicit=True).install()\n out = capfd.readouterr()\n assert \"Installing dependency-install\" in out[0]\n\n # Make sure a warning for missing code is *not* issued\n assert \"Missing a source id for nosource\" not in out[1]\n\n\ndef test_nosource_pkg_install_post_install(install_mockery, mock_fetch, mock_packages):\n \"\"\"Test install phases with the nosource package with post-install.\"\"\"\n spec = spack.concretize.concretize_one(\"nosource-install\")\n pkg = spec.package\n\n # Make sure both the install and post-install package methods work.\n PackageInstaller([pkg], explicit=True).install()\n\n # Ensure the file created in the package's `install` method exists.\n install_txt = os.path.join(spec.prefix, \"install.txt\")\n assert os.path.isfile(install_txt)\n\n # Ensure the file created in the package's `post-install` method exists.\n post_install_txt = os.path.join(spec.prefix, \"post-install.txt\")\n assert os.path.isfile(post_install_txt)\n\n\ndef test_pkg_build_paths(install_mockery):\n # Get a basic concrete spec for the trivial install package.\n spec = spack.concretize.concretize_one(\"trivial-install-test-package\")\n assert spec.package.log_path.endswith(_spack_build_logfile)\n assert spec.package.env_path.endswith(_spack_build_envfile)\n\n\ndef test_pkg_install_paths(install_mockery):\n # Get a basic concrete spec for the trivial install package.\n spec = spack.concretize.concretize_one(\"trivial-install-test-package\")\n\n log_path = os.path.join(spec.prefix, \".spack\", _spack_build_logfile + \".gz\")\n assert spec.package.install_log_path == log_path\n\n env_path = os.path.join(spec.prefix, \".spack\", _spack_build_envfile)\n assert spec.package.install_env_path == env_path\n\n args_path = os.path.join(spec.prefix, \".spack\", _spack_configure_argsfile)\n assert spec.package.install_configure_args_path == args_path\n\n # Backward compatibility checks\n log_dir = os.path.dirname(log_path)\n fs.mkdirp(log_dir)\n with fs.working_dir(log_dir):\n # Start with the older of the previous install log filenames\n older_log = \"build.out\"\n fs.touch(older_log)\n assert spec.package.install_log_path.endswith(older_log)\n\n # Now check the newer install log filename\n last_log = \"build.txt\"\n fs.rename(older_log, last_log)\n assert spec.package.install_log_path.endswith(last_log)\n\n # Check the old install environment file\n last_env = \"build.env\"\n fs.rename(last_log, last_env)\n assert spec.package.install_env_path.endswith(last_env)\n\n # Cleanup\n shutil.rmtree(log_dir)\n\n\ndef test_log_install_without_build_files(install_mockery):\n \"\"\"Test the installer log function when no build files are present.\"\"\"\n # Get a basic concrete spec for the trivial install package.\n spec = spack.concretize.concretize_one(\"trivial-install-test-package\")\n\n # Attempt installing log without the build log file\n with pytest.raises(OSError, match=\"No such file or directory\"):\n spack.installer.log(spec.package)\n\n\ndef test_log_install_with_build_files(install_mockery, monkeypatch):\n \"\"\"Test the installer's log function when have build files.\"\"\"\n config_log = \"config.log\"\n\n # Retain the original function for use in the monkey patch that is used\n # to raise an exception under the desired condition for test coverage.\n orig_install_fn = fs.install\n\n def _install(src, dest):\n orig_install_fn(src, dest)\n if src.endswith(config_log):\n raise Exception(\"Mock log install error\")\n\n monkeypatch.setattr(fs, \"install\", _install)\n\n spec = spack.concretize.concretize_one(\"trivial-install-test-package\")\n\n # Set up mock build files and try again to include archive failure\n log_path = spec.package.log_path\n log_dir = os.path.dirname(log_path)\n fs.mkdirp(log_dir)\n with fs.working_dir(log_dir):\n fs.touch(log_path)\n fs.touch(spec.package.env_path)\n fs.touch(spec.package.env_mods_path)\n fs.touch(spec.package.configure_args_path)\n\n install_path = os.path.dirname(spec.package.install_log_path)\n fs.mkdirp(install_path)\n\n source = spec.package.stage.source_path\n config = os.path.join(source, \"config.log\")\n fs.touchp(config)\n monkeypatch.setattr(\n type(spec.package), \"archive_files\", [\"missing\", \"..\", config], raising=False\n )\n\n spack.installer.log(spec.package)\n\n assert os.path.exists(spec.package.install_log_path)\n assert os.path.exists(spec.package.install_env_path)\n assert os.path.exists(spec.package.install_configure_args_path)\n\n archive_dir = os.path.join(install_path, \"archived-files\")\n source_dir = os.path.dirname(source)\n rel_config = os.path.relpath(config, source_dir)\n\n assert os.path.exists(os.path.join(archive_dir, rel_config))\n assert not os.path.exists(os.path.join(archive_dir, \"missing\"))\n\n expected_errs = [\"OUTSIDE SOURCE PATH\", \"FAILED TO ARCHIVE\"] # for '..' # for rel_config\n with open(os.path.join(archive_dir, \"errors.txt\"), \"r\", encoding=\"utf-8\") as fd:\n for ln, expected in zip(fd, expected_errs):\n assert expected in ln\n\n # Cleanup\n shutil.rmtree(log_dir)\n\n\ndef test_unconcretized_install(install_mockery, mock_fetch, mock_packages):\n \"\"\"Test attempts to perform install phases with unconcretized spec.\"\"\"\n spec = Spec(\"trivial-install-test-package\")\n pkg_cls = spack.repo.PATH.get_pkg_class(spec.name)\n\n with pytest.raises(ValueError, match=\"must have a concrete spec\"):\n PackageInstaller([pkg_cls(spec)], explicit=True).install()\n\n with pytest.raises(ValueError, match=\"only patch concrete packages\"):\n pkg_cls(spec).do_patch()\n\n\ndef test_install_error():\n try:\n msg = \"test install error\"\n long_msg = \"this is the long version of test install error\"\n raise InstallError(msg, long_msg=long_msg)\n except Exception as exc:\n assert exc.__class__.__name__ == \"InstallError\"\n assert exc.message == msg\n assert exc.long_message == long_msg\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_empty_install_sanity_check_prefix(\n monkeypatch, install_mockery, mock_fetch, mock_packages\n):\n \"\"\"Test empty install triggers sanity_check_prefix.\"\"\"\n spec = spack.concretize.concretize_one(\"failing-empty-install\")\n with pytest.raises(spack.build_environment.ChildError, match=\"Nothing was installed\"):\n PackageInstaller([spec.package], explicit=True).install()\n\n\ndef test_install_from_binary_with_missing_patch_succeeds(\n temporary_store: spack.store.Store, mutable_config, tmp_path: pathlib.Path, mock_packages\n):\n \"\"\"If a patch is missing in the local package repository, but was present when building and\n pushing the package to a binary cache, installation from that binary cache shouldn't error out\n because of the missing patch.\"\"\"\n # Create a spec s with non-existing patches\n s = spack.concretize.concretize_one(\"trivial-install-test-package\")\n patches = [\"a\" * 64]\n s_dict = s.to_dict()\n s_dict[\"spec\"][\"nodes\"][0][\"patches\"] = patches\n s_dict[\"spec\"][\"nodes\"][0][\"parameters\"][\"patches\"] = patches\n s = Spec.from_dict(s_dict)\n\n # Create an install dir for it\n os.makedirs(os.path.join(s.prefix, \".spack\"))\n with open(os.path.join(s.prefix, \".spack\", \"spec.json\"), \"w\", encoding=\"utf-8\") as f:\n s.to_json(f)\n\n # And register it in the database\n temporary_store.db.add(s, explicit=True)\n\n # Push it to a binary cache\n mirror = spack.mirrors.mirror.Mirror.from_local_path(str(tmp_path / \"my_build_cache\"))\n with binary_distribution.make_uploader(mirror=mirror) as uploader:\n uploader.push_or_raise([s])\n\n # Now re-install it.\n s.package.do_uninstall()\n assert not temporary_store.db.query_local_by_spec_hash(s.dag_hash())\n\n # Source install: fails, we don't have the patch.\n with pytest.raises(spack.error.SpecError, match=\"Couldn't find patch for package\"):\n PackageInstaller([s.package], explicit=True).install()\n\n # Binary install: succeeds, we don't need the patch.\n spack.mirrors.utils.add(mirror)\n PackageInstaller(\n [s.package],\n explicit=True,\n root_policy=\"cache_only\",\n dependencies_policy=\"cache_only\",\n unsigned=True,\n ).install()\n\n assert temporary_store.db.query_local_by_spec_hash(s.dag_hash())\n" }, { "path": "lib/spack/spack/test/installer.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport glob\nimport os\nimport pathlib\nimport shutil\nimport sys\nfrom typing import List, Optional, Union\n\nimport py\nimport pytest\n\nimport spack.binary_distribution\nimport spack.concretize\nimport spack.database\nimport spack.deptypes as dt\nimport spack.error\nimport spack.hooks\nimport spack.installer as inst\nimport spack.installer_dispatch\nimport spack.llnl.util.filesystem as fs\nimport spack.llnl.util.lock as ulk\nimport spack.llnl.util.tty as tty\nimport spack.package_base\nimport spack.package_prefs as prefs\nimport spack.repo\nimport spack.report\nimport spack.spec\nimport spack.store\nimport spack.util.lock as lk\nfrom spack.main import SpackCommand\nfrom spack.test.conftest import RepoBuilder\n\n\ndef _mock_repo(root, namespace):\n \"\"\"Create an empty repository at the specified root\n\n Args:\n root (str): path to the mock repository root\n namespace (str): mock repo's namespace\n \"\"\"\n repodir = py.path.local(root) if isinstance(root, str) else root\n repodir.ensure(spack.repo.packages_dir_name, dir=True)\n yaml = repodir.join(\"repo.yaml\")\n yaml.write(\n f\"\"\"\nrepo:\n namespace: {namespace}\n\"\"\"\n )\n\n\ndef _noop(*args, **kwargs):\n \"\"\"Generic monkeypatch no-op routine.\"\"\"\n\n\ndef _none(*args, **kwargs):\n \"\"\"Generic monkeypatch function that always returns None.\"\"\"\n return None\n\n\ndef _not_locked(installer, lock_type, pkg):\n \"\"\"Generic monkeypatch function for _ensure_locked to return no lock\"\"\"\n tty.msg(\"{0} locked {1}\".format(lock_type, pkg.spec.name))\n return lock_type, None\n\n\ndef _true(*args, **kwargs):\n \"\"\"Generic monkeypatch function that always returns True.\"\"\"\n return True\n\n\ndef create_build_task(\n pkg: spack.package_base.PackageBase, install_args: Optional[dict] = None\n) -> inst.BuildTask:\n request = inst.BuildRequest(pkg, {} if install_args is None else install_args)\n return inst.BuildTask(pkg, request=request, status=inst.BuildStatus.QUEUED)\n\n\ndef create_installer(\n specs: Union[List[str], List[spack.spec.Spec]], install_args: Optional[dict] = None\n) -> inst.PackageInstaller:\n \"\"\"Create an installer instance for a list of specs or package names that will be\n concretized.\"\"\"\n _specs = [spack.concretize.concretize_one(s) if isinstance(s, str) else s for s in specs]\n _install_args = {} if install_args is None else install_args\n return inst.PackageInstaller([spec.package for spec in _specs], **_install_args)\n\n\n@pytest.mark.parametrize(\n \"sec,result\",\n [(86400, \"24h\"), (3600, \"1h\"), (60, \"1m\"), (1.802, \"1.80s\"), (3723.456, \"1h 2m 3.46s\")],\n)\ndef test_hms(sec, result):\n assert inst._hms(sec) == result\n\n\ndef test_get_dependent_ids(install_mockery, mock_packages):\n # Concretize the parent package, which handle dependency too\n spec = spack.concretize.concretize_one(\"pkg-a\")\n assert spec.concrete\n\n pkg_id = inst.package_id(spec)\n\n # Grab the sole dependency of 'a', which is 'b'\n dep = spec.dependencies()[0]\n\n # Ensure the parent package is a dependent of the dependency package\n assert pkg_id in inst.get_dependent_ids(dep)\n\n\ndef test_install_msg(monkeypatch):\n \"\"\"Test results of call to install_msg based on debug level.\"\"\"\n name = \"some-package\"\n pid = 123456\n install_msg = \"Installing {0}\".format(name)\n\n monkeypatch.setattr(tty, \"_debug\", 0)\n assert inst.install_msg(name, pid, None) == install_msg\n\n install_status = inst.InstallStatus(1)\n expected = \"{0} [0/1]\".format(install_msg)\n assert inst.install_msg(name, pid, install_status) == expected\n\n monkeypatch.setattr(tty, \"_debug\", 1)\n assert inst.install_msg(name, pid, None) == install_msg\n\n # Expect the PID to be added at debug level 2\n monkeypatch.setattr(tty, \"_debug\", 2)\n expected = \"{0}: {1}\".format(pid, install_msg)\n assert inst.install_msg(name, pid, None) == expected\n\n\ndef test_install_from_cache_errors(install_mockery):\n \"\"\"Test to ensure cover install from cache errors.\"\"\"\n spec = spack.concretize.concretize_one(\"trivial-install-test-package\")\n assert spec.concrete\n\n # Check with cache-only\n with pytest.raises(\n spack.error.InstallError, match=\"No binary found when cache-only was specified\"\n ):\n inst.PackageInstaller(\n [spec.package], root_policy=\"cache_only\", dependencies_policy=\"cache_only\"\n ).install()\n assert not spec.package.installed_from_binary_cache\n\n # Check when don't expect to install only from binary cache\n assert not inst._install_from_cache(spec.package, explicit=True, unsigned=False)\n assert not spec.package.installed_from_binary_cache\n\n\ndef test_install_from_cache_ok(install_mockery, monkeypatch):\n \"\"\"Test to ensure cover _install_from_cache to the return.\"\"\"\n spec = spack.concretize.concretize_one(\"trivial-install-test-package\")\n monkeypatch.setattr(inst, \"_try_install_from_binary_cache\", _true)\n monkeypatch.setattr(spack.hooks, \"post_install\", _noop)\n\n assert inst._install_from_cache(spec.package, explicit=True, unsigned=False)\n\n\ndef test_process_external_package_module(install_mockery, monkeypatch, capfd):\n \"\"\"Test to simply cover the external module message path.\"\"\"\n spec = spack.concretize.concretize_one(\"trivial-install-test-package\")\n assert spec.concrete\n\n # Ensure take the external module path WITHOUT any changes to the database\n monkeypatch.setattr(spack.database.Database, \"get_record\", _none)\n\n spec.external_path = \"/actual/external/path/not/checked\"\n spec.external_modules = [\"unchecked_module\"]\n inst._process_external_package(spec.package, False)\n\n out = capfd.readouterr()[0]\n assert \"has external module in {0}\".format(spec.external_modules) in out\n\n\ndef test_process_binary_cache_tarball_tar(install_mockery, monkeypatch, capfd):\n \"\"\"Tests of _process_binary_cache_tarball with a tar file.\"\"\"\n\n def _spec(spec, unsigned=False, mirrors_for_spec=None):\n return spec\n\n # Skip binary distribution functionality since assume tested elsewhere\n monkeypatch.setattr(spack.binary_distribution, \"download_tarball\", _spec)\n monkeypatch.setattr(spack.binary_distribution, \"extract_tarball\", _noop)\n\n # Skip database updates\n monkeypatch.setattr(spack.database.Database, \"add\", _noop)\n\n spec = spack.concretize.concretize_one(\"pkg-a\")\n assert inst._process_binary_cache_tarball(spec.package, explicit=False, unsigned=False)\n\n out = capfd.readouterr()[0]\n assert \"Extracting pkg-a\" in out\n assert \"from binary cache\" in out\n\n\ndef test_try_install_from_binary_cache(install_mockery, mock_packages, monkeypatch):\n \"\"\"Test return false when no match exists in the mirror\"\"\"\n spec = spack.concretize.concretize_one(\"mpich\")\n result = inst._try_install_from_binary_cache(spec.package, False, False)\n assert not result\n\n\ndef test_installer_repr(install_mockery):\n installer = create_installer([\"trivial-install-test-package\"])\n\n irep = installer.__repr__()\n assert irep.startswith(installer.__class__.__name__)\n assert \"installed=\" in irep\n assert \"failed=\" in irep\n\n\ndef test_installer_str(install_mockery):\n installer = create_installer([\"trivial-install-test-package\"])\n\n istr = str(installer)\n assert \"#tasks=0\" in istr\n assert \"installed (0)\" in istr\n assert \"failed (0)\" in istr\n\n\ndef test_installer_prune_built_build_deps(install_mockery, monkeypatch, repo_builder: RepoBuilder):\n r\"\"\"\n Ensure that build dependencies of installed deps are pruned\n from installer package queues.\n\n (a)\n / \\\n / \\\n (b) (c) <--- is installed already so we should\n \\ / | \\ prune (f) from this install since\n \\ / | \\ it is *only* needed to build (b)\n (d) (e) (f)\n\n Thus since (c) is already installed our build_pq dag should\n only include four packages. [(a), (b), (c), (d), (e)]\n \"\"\"\n\n def _mock_installed(self):\n return self.name == \"pkg-c\"\n\n # Mock the installed property to say that (c) is installed\n monkeypatch.setattr(spack.spec.Spec, \"installed\", property(_mock_installed))\n\n # Create mock repository with packages (a), (b), (c), (d), and (e)\n repo_builder.add_package(\n \"pkg-a\", dependencies=[(\"pkg-b\", \"build\", None), (\"pkg-c\", \"build\", None)]\n )\n repo_builder.add_package(\"pkg-b\", dependencies=[(\"pkg-d\", \"build\", None)])\n repo_builder.add_package(\n \"pkg-c\",\n dependencies=[(\"pkg-d\", \"build\", None), (\"pkg-e\", \"all\", None), (\"pkg-f\", \"build\", None)],\n )\n repo_builder.add_package(\"pkg-d\")\n repo_builder.add_package(\"pkg-e\")\n repo_builder.add_package(\"pkg-f\")\n\n with spack.repo.use_repositories(repo_builder.root):\n installer = create_installer([\"pkg-a\"])\n\n installer._init_queue()\n\n # Assert that (c) is not in the build_pq\n result = {task.pkg_id[:5] for _, task in installer.build_pq}\n expected = {\"pkg-a\", \"pkg-b\", \"pkg-c\", \"pkg-d\", \"pkg-e\"}\n assert result == expected\n\n\ndef test_check_before_phase_error(install_mockery):\n s = spack.concretize.concretize_one(\"trivial-install-test-package\")\n s.package.stop_before_phase = \"beforephase\"\n with pytest.raises(inst.BadInstallPhase) as exc_info:\n inst._check_last_phase(s.package)\n\n err = str(exc_info.value)\n assert \"is not a valid phase\" in err\n assert s.package.stop_before_phase in err\n\n\ndef test_check_last_phase_error(install_mockery):\n s = spack.concretize.concretize_one(\"trivial-install-test-package\")\n s.package.stop_before_phase = None\n s.package.last_phase = \"badphase\"\n with pytest.raises(inst.BadInstallPhase) as exc_info:\n inst._check_last_phase(s.package)\n\n err = str(exc_info.value)\n assert \"is not a valid phase\" in err\n assert s.package.last_phase in err\n\n\ndef test_installer_ensure_ready_errors(install_mockery, monkeypatch):\n installer = create_installer([\"trivial-install-test-package\"])\n spec = installer.build_requests[0].pkg.spec\n\n fmt = r\"cannot be installed locally.*{0}\"\n # Force an external package error\n path, modules = spec.external_path, spec.external_modules\n spec.external_path = \"/actual/external/path/not/checked\"\n spec.external_modules = [\"unchecked_module\"]\n msg = fmt.format(\"is external\")\n with pytest.raises(inst.ExternalPackageError, match=msg):\n installer._ensure_install_ready(spec.package)\n\n # Force an upstream package error\n spec.external_path, spec.external_modules = path, modules\n monkeypatch.setattr(spack.spec.Spec, \"installed_upstream\", True)\n msg = fmt.format(\"is upstream\")\n with pytest.raises(inst.UpstreamPackageError, match=msg):\n installer._ensure_install_ready(spec.package)\n\n # Force an install lock error, which should occur naturally since\n # we are calling an internal method prior to any lock-related setup\n monkeypatch.setattr(spack.spec.Spec, \"installed_upstream\", False)\n assert len(installer.locks) == 0\n with pytest.raises(inst.InstallLockError, match=fmt.format(\"not locked\")):\n installer._ensure_install_ready(spec.package)\n\n\ndef test_ensure_locked_err(install_mockery, monkeypatch, tmp_path: pathlib.Path, capfd):\n \"\"\"Test _ensure_locked when a non-lock exception is raised.\"\"\"\n mock_err_msg = \"Mock exception error\"\n\n def _raise(lock, timeout=None):\n raise RuntimeError(mock_err_msg)\n\n installer = create_installer([\"trivial-install-test-package\"])\n spec = installer.build_requests[0].pkg.spec\n\n monkeypatch.setattr(ulk.Lock, \"acquire_read\", _raise)\n with fs.working_dir(str(tmp_path)):\n with pytest.raises(RuntimeError):\n installer._ensure_locked(\"read\", spec.package)\n\n out = str(capfd.readouterr()[1])\n assert \"Failed to acquire a read lock\" in out\n assert mock_err_msg in out\n\n\ndef test_ensure_locked_have(install_mockery, tmp_path: pathlib.Path, capfd):\n \"\"\"Test _ensure_locked when already have lock.\"\"\"\n installer = create_installer([\"trivial-install-test-package\"], {})\n spec = installer.build_requests[0].pkg.spec\n pkg_id = inst.package_id(spec)\n\n with fs.working_dir(str(tmp_path)):\n # Test \"downgrade\" of a read lock (to a read lock)\n lock = lk.Lock(\"./test\", default_timeout=1e-9, desc=\"test\")\n lock_type = \"read\"\n tpl = (lock_type, lock)\n installer.locks[pkg_id] = tpl\n assert installer._ensure_locked(lock_type, spec.package) == tpl\n\n # Test \"upgrade\" of a read lock without read count to a write\n lock_type = \"write\"\n err = \"Cannot upgrade lock\"\n with pytest.raises(ulk.LockUpgradeError, match=err):\n installer._ensure_locked(lock_type, spec.package)\n\n out = str(capfd.readouterr()[1])\n assert \"Failed to upgrade to a write lock\" in out\n assert \"exception when releasing read lock\" in out\n\n # Test \"upgrade\" of the read lock *with* read count to a write\n lock._reads = 1\n tpl = (lock_type, lock)\n assert installer._ensure_locked(lock_type, spec.package) == tpl\n\n # Test \"downgrade\" of the write lock to a read lock\n lock_type = \"read\"\n tpl = (lock_type, lock)\n assert installer._ensure_locked(lock_type, spec.package) == tpl\n\n\n@pytest.mark.parametrize(\"lock_type,reads,writes\", [(\"read\", 1, 0), (\"write\", 0, 1)])\ndef test_ensure_locked_new_lock(install_mockery, tmp_path: pathlib.Path, lock_type, reads, writes):\n installer = create_installer([\"pkg-a\"], {})\n spec = installer.build_requests[0].pkg.spec\n with fs.working_dir(str(tmp_path)):\n ltype, lock = installer._ensure_locked(lock_type, spec.package)\n assert ltype == lock_type\n assert lock is not None\n assert lock._reads == reads\n assert lock._writes == writes\n\n\ndef test_ensure_locked_new_warn(install_mockery, monkeypatch, capfd):\n orig_pl = spack.database.SpecLocker.lock\n\n def _pl(db, spec, timeout):\n lock = orig_pl(db, spec, timeout)\n lock.default_timeout = 1e-9 if timeout is None else None\n return lock\n\n installer = create_installer([\"pkg-a\"], {})\n spec = installer.build_requests[0].pkg.spec\n\n monkeypatch.setattr(spack.database.SpecLocker, \"lock\", _pl)\n\n lock_type = \"read\"\n ltype, lock = installer._ensure_locked(lock_type, spec.package)\n assert ltype == lock_type\n assert lock is not None\n\n out = str(capfd.readouterr()[1])\n assert \"Expected prefix lock timeout\" in out\n\n\ndef test_package_id_err(install_mockery):\n s = spack.spec.Spec(\"trivial-install-test-package\")\n with pytest.raises(ValueError, match=\"spec is not concretized\"):\n inst.package_id(s)\n\n\ndef test_package_id_ok(install_mockery):\n spec = spack.concretize.concretize_one(\"trivial-install-test-package\")\n assert spec.concrete\n assert spec.name in inst.package_id(spec)\n\n\ndef test_fake_install(install_mockery):\n spec = spack.concretize.concretize_one(\"trivial-install-test-package\")\n assert spec.concrete\n\n pkg = spec.package\n inst._do_fake_install(pkg)\n assert os.path.isdir(pkg.prefix.lib)\n\n\ndef test_dump_packages_deps_ok(install_mockery, tmp_path: pathlib.Path, mock_packages):\n \"\"\"Test happy path for dump_packages with dependencies.\"\"\"\n\n spec_name = \"simple-inheritance\"\n spec = spack.concretize.concretize_one(spec_name)\n inst.dump_packages(spec, str(tmp_path))\n\n repo = mock_packages.repos[0]\n dest_pkg = repo.filename_for_package_name(spec_name)\n assert os.path.isfile(dest_pkg)\n\n\ndef test_dump_packages_deps_errs(install_mockery, tmp_path: pathlib.Path, monkeypatch, capfd):\n \"\"\"Test error paths for dump_packages with dependencies.\"\"\"\n orig_bpp = spack.store.STORE.layout.build_packages_path\n orig_dirname = spack.repo.Repo.dirname_for_package_name\n repo_err_msg = \"Mock dirname_for_package_name\"\n\n def bpp_path(spec):\n # Perform the original function\n source = orig_bpp(spec)\n # Mock the required directory structure for the repository\n _mock_repo(os.path.join(source, spec.namespace), spec.namespace)\n return source\n\n def _repoerr(repo, name):\n if name == \"cmake\":\n raise spack.repo.RepoError(repo_err_msg)\n else:\n return orig_dirname(repo, name)\n\n # Now mock the creation of the required directory structure to cover\n # the try-except block\n monkeypatch.setattr(spack.store.STORE.layout, \"build_packages_path\", bpp_path)\n\n spec = spack.concretize.concretize_one(\"simple-inheritance\")\n path = str(tmp_path)\n\n # The call to install_tree will raise the exception since not mocking\n # creation of dependency package files within *install* directories.\n with pytest.raises(OSError, match=path if sys.platform != \"win32\" else \"\"):\n inst.dump_packages(spec, path)\n\n # Now try the error path, which requires the mock directory structure\n # above\n monkeypatch.setattr(spack.repo.Repo, \"dirname_for_package_name\", _repoerr)\n with pytest.raises(spack.repo.RepoError, match=repo_err_msg):\n inst.dump_packages(spec, path)\n\n out = str(capfd.readouterr()[1])\n assert \"Couldn't copy in provenance for cmake\" in out\n\n\ndef test_clear_failures_success(tmp_path: pathlib.Path):\n \"\"\"Test the clear_failures happy path.\"\"\"\n failures = spack.database.FailureTracker(str(tmp_path), default_timeout=0.1)\n\n spec = spack.spec.Spec(\"pkg-a\")\n spec._mark_concrete()\n\n # Set up a test prefix failure lock\n failures.mark(spec)\n assert failures.has_failed(spec)\n\n # Now clear failure tracking\n failures.clear_all()\n\n # Ensure there are no cached failure locks or failure marks\n assert len(failures.locker.locks) == 0\n assert len(os.listdir(failures.dir)) == 0\n\n # Ensure the core directory and failure lock file still exist\n assert os.path.isdir(failures.dir)\n\n # Locks on windows are a no-op\n if sys.platform != \"win32\":\n assert os.path.isfile(failures.locker.lock_path)\n\n\n@pytest.mark.not_on_windows(\"chmod does not prevent removal on Win\")\n@pytest.mark.skipif(fs.getuid() == 0, reason=\"user is root\")\ndef test_clear_failures_errs(tmp_path: pathlib.Path, capfd):\n \"\"\"Test the clear_failures exception paths.\"\"\"\n failures = spack.database.FailureTracker(str(tmp_path), default_timeout=0.1)\n spec = spack.spec.Spec(\"pkg-a\")\n spec._mark_concrete()\n failures.mark(spec)\n\n # Make the file marker not writeable, so that clearing_failures fails\n failures.dir.chmod(0o000)\n\n # Clear failure tracking\n failures.clear_all()\n\n # Ensure expected warning generated\n out = str(capfd.readouterr()[1])\n assert \"Unable to remove failure\" in out\n failures.dir.chmod(0o750)\n\n\ndef test_combine_phase_logs(tmp_path: pathlib.Path):\n \"\"\"Write temporary files, and assert that combine phase logs works\n to combine them into one file. We aren't currently using this function,\n but it's available when the logs are refactored to be written separately.\n \"\"\"\n log_files = [\"configure-out.txt\", \"install-out.txt\", \"build-out.txt\"]\n phase_log_files = []\n\n # Create and write to dummy phase log files\n for log_file in log_files:\n phase_log_file = tmp_path / log_file\n with open(phase_log_file, \"w\", encoding=\"utf-8\") as plf:\n plf.write(\"Output from %s\\n\" % log_file)\n phase_log_files.append(str(phase_log_file))\n\n # This is the output log we will combine them into\n combined_log = tmp_path / \"combined-out.txt\"\n inst.combine_phase_logs(phase_log_files, str(combined_log))\n with open(combined_log, \"r\", encoding=\"utf-8\") as log_file:\n out = log_file.read()\n\n # Ensure each phase log file is represented\n for log_file in log_files:\n assert \"Output from %s\\n\" % log_file in out\n\n\ndef test_combine_phase_logs_does_not_care_about_encoding(tmp_path: pathlib.Path):\n # this is invalid utf-8 at a minimum\n data = b\"\\x00\\xf4\\xbf\\x00\\xbf\\xbf\"\n input = [str(tmp_path / \"a\"), str(tmp_path / \"b\")]\n output = str(tmp_path / \"c\")\n\n for path in input:\n with open(path, \"wb\") as f:\n f.write(data)\n\n inst.combine_phase_logs(input, output)\n\n with open(output, \"rb\") as f:\n assert f.read() == data * 2\n\n\ndef test_check_deps_status_install_failure(install_mockery):\n \"\"\"Tests that checking the dependency status on a request to install\n 'a' fails, if we mark the dependency as failed.\n \"\"\"\n s = spack.concretize.concretize_one(\"pkg-a\")\n for dep in s.traverse(root=False):\n spack.store.STORE.failure_tracker.mark(dep)\n\n installer = create_installer([\"pkg-a\"], {})\n request = installer.build_requests[0]\n\n with pytest.raises(spack.error.InstallError, match=\"install failure\"):\n installer._check_deps_status(request)\n\n\ndef test_check_deps_status_write_locked(install_mockery, monkeypatch):\n installer = create_installer([\"pkg-a\"], {})\n request = installer.build_requests[0]\n\n # Ensure the lock is not acquired\n monkeypatch.setattr(inst.PackageInstaller, \"_ensure_locked\", _not_locked)\n\n with pytest.raises(spack.error.InstallError, match=\"write locked by another\"):\n installer._check_deps_status(request)\n\n\ndef test_check_deps_status_external(install_mockery, monkeypatch):\n installer = create_installer([\"pkg-a\"], {})\n request = installer.build_requests[0]\n\n # Mock the dependencies as external so assumed to be installed\n monkeypatch.setattr(spack.spec.Spec, \"external\", True)\n installer._check_deps_status(request)\n\n for dep in request.spec.traverse(root=False):\n assert inst.package_id(dep) in installer.installed\n\n\ndef test_check_deps_status_upstream(install_mockery, monkeypatch):\n installer = create_installer([\"pkg-a\"], {})\n request = installer.build_requests[0]\n\n # Mock the known dependencies as installed upstream\n monkeypatch.setattr(spack.spec.Spec, \"installed_upstream\", True)\n installer._check_deps_status(request)\n\n for dep in request.spec.traverse(root=False):\n assert inst.package_id(dep) in installer.installed\n\n\ndef test_prepare_for_install_on_installed(install_mockery, monkeypatch):\n \"\"\"Test of _prepare_for_install's early return for installed task path.\"\"\"\n installer = create_installer([\"dependent-install\"], {})\n request = installer.build_requests[0]\n\n install_args = {\"keep_prefix\": True, \"keep_stage\": True, \"restage\": False}\n task = create_build_task(request.pkg, install_args)\n installer.installed.add(task.pkg_id)\n\n monkeypatch.setattr(inst.PackageInstaller, \"_ensure_install_ready\", _noop)\n installer._prepare_for_install(task)\n\n\ndef test_installer_init_requests(install_mockery):\n \"\"\"Test of installer initial requests.\"\"\"\n spec_name = \"dependent-install\"\n installer = create_installer([spec_name], {})\n\n # There is only one explicit request in this case\n assert len(installer.build_requests) == 1\n request = installer.build_requests[0]\n assert request.pkg.name == spec_name\n\n\ndef false(*args, **kwargs):\n return False\n\n\ndef test_rewire_task_no_tarball(monkeypatch, mock_packages):\n spec = spack.concretize.concretize_one(\"splice-t\")\n dep = spack.concretize.concretize_one(\"splice-h+foo\")\n out = spec.splice(dep)\n\n rewire_task = inst.RewireTask(out.package, inst.BuildRequest(out.package, {}))\n monkeypatch.setattr(inst, \"_process_binary_cache_tarball\", false)\n monkeypatch.setattr(spack.report.InstallRecord, \"succeed\", lambda x: None)\n\n assert rewire_task.complete() == inst.ExecuteResult.MISSING_BUILD_SPEC\n\n\n@pytest.mark.parametrize(\"transitive\", [True, False])\ndef test_install_spliced(install_mockery, mock_fetch, monkeypatch, transitive):\n \"\"\"Test installing a spliced spec\"\"\"\n spec = spack.concretize.concretize_one(\"splice-t\")\n dep = spack.concretize.concretize_one(\"splice-h+foo\")\n\n # Do the splice.\n out = spec.splice(dep, transitive)\n installer = create_installer([out], {\"verbose\": True, \"fail_fast\": True})\n installer.install()\n for node in out.traverse():\n assert node.installed\n assert node.build_spec.installed\n\n\n@pytest.mark.parametrize(\"transitive\", [True, False])\ndef test_install_spliced_build_spec_installed(install_mockery, mock_fetch, transitive):\n \"\"\"Test installing a spliced spec with the build spec already installed\"\"\"\n spec = spack.concretize.concretize_one(\"splice-t\")\n dep = spack.concretize.concretize_one(\"splice-h+foo\")\n\n # Do the splice.\n out = spec.splice(dep, transitive)\n inst.PackageInstaller([out.build_spec.package]).install()\n\n installer = create_installer([out], {\"verbose\": True, \"fail_fast\": True})\n installer._init_queue()\n for _, task in installer.build_pq:\n assert isinstance(task, inst.RewireTask if task.pkg.spec.spliced else inst.BuildTask)\n installer.install()\n for node in out.traverse():\n assert node.installed\n assert node.build_spec.installed\n\n\n# Unit tests should not be affected by the user's managed environments\n@pytest.mark.not_on_windows(\"lacking windows support for binary installs\")\n@pytest.mark.parametrize(\"transitive\", [True, False])\n@pytest.mark.parametrize(\n \"root_str\", [\"splice-t^splice-h~foo\", \"splice-h~foo\", \"splice-vt^splice-a\"]\n)\ndef test_install_splice_root_from_binary(\n mutable_mock_env_path, install_mockery, mock_fetch, temporary_mirror, transitive, root_str\n):\n \"\"\"Test installing a spliced spec with the root available in binary cache\"\"\"\n # Test splicing and rewiring a spec with the same name, different hash.\n original_spec = spack.concretize.concretize_one(root_str)\n spec_to_splice = spack.concretize.concretize_one(\"splice-h+foo\")\n\n inst.PackageInstaller([original_spec.package, spec_to_splice.package]).install()\n\n out = original_spec.splice(spec_to_splice, transitive)\n\n buildcache = SpackCommand(\"buildcache\")\n buildcache(\n \"push\",\n \"--unsigned\",\n \"--update-index\",\n temporary_mirror,\n str(original_spec),\n str(spec_to_splice),\n )\n\n uninstall = SpackCommand(\"uninstall\")\n uninstall(\"-ay\")\n\n inst.PackageInstaller([out.package], unsigned=True).install()\n\n assert len(spack.store.STORE.db.query()) == len(list(out.traverse()))\n\n\nclass MockInstallStatus(inst.InstallStatus):\n def next_pkg(self, *args, **kwargs):\n pass\n\n def set_term_title(self, *args, **kwargs):\n pass\n\n def get_progress(self):\n return \"1/1\"\n\n\nclass MockTermStatusLine(inst.TermStatusLine):\n def add(self, *args, **kwargs):\n pass\n\n def clear(self):\n pass\n\n\ndef test_installing_task_use_cache(install_mockery, monkeypatch):\n installer = create_installer([\"trivial-install-test-package\"], {})\n request = installer.build_requests[0]\n task = create_build_task(request.pkg)\n install_status = MockInstallStatus(1)\n term_status = MockTermStatusLine(True)\n\n monkeypatch.setattr(inst, \"_install_from_cache\", _true)\n installer.start_task(task, install_status, term_status)\n installer.complete_task(task, install_status)\n assert request.pkg_id in installer.installed\n\n\ndef test_install_task_requeue_build_specs(install_mockery, monkeypatch):\n \"\"\"Check that a missing build_spec spec is added by _complete_task.\"\"\"\n\n # This test also ensures coverage of most of the new\n # _requeue_with_build_spec_tasks method.\n def _missing(*args, **kwargs):\n return inst.ExecuteResult.MISSING_BUILD_SPEC\n\n # Set the configuration to ensure _requeue_with_build_spec_tasks actually\n # does something.\n installer = create_installer([\"depb\"], {})\n installer._init_queue()\n request = installer.build_requests[0]\n task = create_build_task(request.pkg)\n\n # Drop one of the specs so its task is missing before _complete_task\n popped_task = installer._pop_ready_task()\n assert inst.package_id(popped_task.pkg.spec) not in installer.build_tasks\n\n monkeypatch.setattr(task, \"complete\", _missing)\n installer._complete_task(task, None)\n\n # Ensure the dropped task/spec was added back by _install_task\n assert inst.package_id(popped_task.pkg.spec) in installer.build_tasks\n\n\ndef test_release_lock_write_n_exception(install_mockery, tmp_path: pathlib.Path, capfd):\n \"\"\"Test _release_lock for supposed write lock with exception.\"\"\"\n installer = create_installer([\"trivial-install-test-package\"], {})\n\n pkg_id = \"test\"\n with fs.working_dir(str(tmp_path)):\n lock = lk.Lock(\"./test\", default_timeout=1e-9, desc=\"test\")\n installer.locks[pkg_id] = (\"write\", lock)\n assert lock._writes == 0\n\n installer._release_lock(pkg_id)\n out = str(capfd.readouterr()[1])\n msg = \"exception when releasing write lock for {0}\".format(pkg_id)\n assert msg in out\n\n\n@pytest.mark.parametrize(\"installed\", [True, False])\ndef test_push_task_skip_processed(install_mockery, installed):\n \"\"\"Test to ensure skip re-queueing a processed package.\"\"\"\n installer = create_installer([\"pkg-a\"], {})\n assert len(list(installer.build_tasks)) == 0\n\n # Mark the package as installed OR failed\n task = create_build_task(installer.build_requests[0].pkg)\n if installed:\n installer.installed.add(task.pkg_id)\n else:\n installer.failed[task.pkg_id] = None\n\n installer._push_task(task)\n\n assert len(list(installer.build_tasks)) == 0\n\n\ndef test_requeue_task(install_mockery, capfd):\n \"\"\"Test to ensure cover _requeue_task.\"\"\"\n installer = create_installer([\"pkg-a\"], {})\n task = create_build_task(installer.build_requests[0].pkg)\n\n # temporarily set tty debug messages on so we can test output\n current_debug_level = tty.debug_level()\n tty.set_debug(1)\n installer._requeue_task(task, None)\n tty.set_debug(current_debug_level)\n\n ids = list(installer.build_tasks)\n assert len(ids) == 1\n qtask = installer.build_tasks[ids[0]]\n assert qtask.status == inst.BuildStatus.INSTALLING\n assert qtask.sequence > task.sequence\n assert qtask.attempts == task.attempts + 1\n\n out = capfd.readouterr()[1]\n assert \"Installing pkg-a\" in out\n assert \" in progress by another process\" in out\n\n\ndef test_cleanup_all_tasks(install_mockery, monkeypatch):\n \"\"\"Test to ensure cover _cleanup_all_tasks.\"\"\"\n\n def _mktask(pkg):\n return create_build_task(pkg)\n\n def _rmtask(installer, pkg_id):\n raise RuntimeError(\"Raise an exception to test except path\")\n\n installer = create_installer([\"pkg-a\"], {})\n spec = installer.build_requests[0].pkg.spec\n\n # Cover task removal happy path\n installer.build_tasks[\"pkg-a\"] = _mktask(spec.package)\n installer._cleanup_all_tasks()\n assert len(installer.build_tasks) == 0\n\n # Cover task removal exception path\n installer.build_tasks[\"pkg-a\"] = _mktask(spec.package)\n monkeypatch.setattr(inst.PackageInstaller, \"_remove_task\", _rmtask)\n installer._cleanup_all_tasks()\n assert len(installer.build_tasks) == 1\n\n\ndef test_setup_install_dir_grp(install_mockery, monkeypatch, capfd):\n \"\"\"Test _setup_install_dir's group change.\"\"\"\n mock_group = \"mockgroup\"\n mock_chgrp_msg = \"Changing group for {0} to {1}\"\n\n def _get_group(spec):\n return mock_group\n\n def _chgrp(path, group, follow_symlinks=True):\n tty.msg(mock_chgrp_msg.format(path, group))\n\n monkeypatch.setattr(prefs, \"get_package_group\", _get_group)\n monkeypatch.setattr(fs, \"chgrp\", _chgrp)\n\n build_task = create_build_task(\n spack.concretize.concretize_one(\"trivial-install-test-package\").package\n )\n spec = build_task.request.pkg.spec\n\n fs.touchp(spec.prefix)\n metadatadir = spack.store.STORE.layout.metadata_path(spec)\n # Regex matching with Windows style paths typically fails\n # so we skip the match check here\n if sys.platform == \"win32\":\n metadatadir = None\n # Should fail with a \"not a directory\" error\n with pytest.raises(OSError, match=metadatadir):\n build_task._setup_install_dir(spec.package)\n\n out = str(capfd.readouterr()[0])\n\n expected_msg = mock_chgrp_msg.format(spec.prefix, mock_group)\n assert expected_msg in out\n\n\ndef test_cleanup_failed_err(install_mockery, tmp_path: pathlib.Path, monkeypatch, capfd):\n \"\"\"Test _cleanup_failed exception path.\"\"\"\n msg = \"Fake release_write exception\"\n\n def _raise_except(lock):\n raise RuntimeError(msg)\n\n installer = create_installer([\"trivial-install-test-package\"], {})\n\n monkeypatch.setattr(lk.Lock, \"release_write\", _raise_except)\n pkg_id = \"test\"\n with fs.working_dir(str(tmp_path)):\n lock = lk.Lock(\"./test\", default_timeout=1e-9, desc=\"test\")\n installer.failed[pkg_id] = lock\n\n installer._cleanup_failed(pkg_id)\n out = str(capfd.readouterr()[1])\n assert \"exception when removing failure tracking\" in out\n assert msg in out\n\n\ndef test_update_failed_no_dependent_task(install_mockery):\n \"\"\"Test _update_failed with missing dependent build tasks.\"\"\"\n installer = create_installer([\"dependent-install\"], {})\n spec = installer.build_requests[0].pkg.spec\n\n for dep in spec.traverse(root=False):\n task = create_build_task(dep.package)\n installer._update_failed(task, mark=False)\n assert installer.failed[task.pkg_id] is None\n\n\ndef test_install_uninstalled_deps(install_mockery, monkeypatch, capfd):\n \"\"\"Test install with uninstalled dependencies.\"\"\"\n installer = create_installer([\"parallel-package-a\"], {})\n\n # Skip the actual installation and any status updates\n monkeypatch.setattr(inst.Task, \"start\", _noop)\n monkeypatch.setattr(inst.Task, \"poll\", _noop)\n monkeypatch.setattr(inst.Task, \"complete\", _noop)\n monkeypatch.setattr(inst.PackageInstaller, \"_update_installed\", _noop)\n monkeypatch.setattr(inst.PackageInstaller, \"_update_failed\", _noop)\n\n msg = \"Cannot proceed with parallel-package-a\"\n with pytest.raises(spack.error.InstallError, match=msg):\n installer.install()\n\n out = str(capfd.readouterr())\n assert \"Detected uninstalled dependencies for\" in out\n\n\ndef test_install_failed(install_mockery, monkeypatch, capfd):\n \"\"\"Test install with failed install.\"\"\"\n installer = create_installer([\"parallel-package-a\"], {})\n\n # Make sure the package is identified as failed\n monkeypatch.setattr(spack.database.FailureTracker, \"has_failed\", _true)\n\n with pytest.raises(spack.error.InstallError, match=\"request failed\"):\n installer.install()\n\n out = str(capfd.readouterr())\n assert installer.build_requests[0].pkg_id in out\n assert \"failed to install\" in out\n\n\ndef test_install_failed_not_fast(install_mockery, monkeypatch, capfd):\n \"\"\"Test install with failed install.\"\"\"\n installer = create_installer([\"parallel-package-a\"], {\"fail_fast\": False})\n\n # Make sure the package is identified as failed\n monkeypatch.setattr(spack.database.FailureTracker, \"has_failed\", _true)\n\n with pytest.raises(spack.error.InstallError, match=\"request failed\"):\n installer.install()\n\n out = str(capfd.readouterr())\n assert \"failed to install\" in out\n assert \"Skipping build of parallel-package-a\" in out\n\n\ndef _interrupt(installer, task, install_status, **kwargs):\n if task.pkg.name == \"pkg-a\":\n raise KeyboardInterrupt(\"mock keyboard interrupt for pkg-a\")\n else:\n return installer._real_install_task(task, None)\n # installer.installed.add(task.pkg.name)\n\n\ndef test_install_fail_on_interrupt(install_mockery, mock_fetch, monkeypatch):\n \"\"\"Test ctrl-c interrupted install.\"\"\"\n spec_name = \"pkg-a\"\n err_msg = \"mock keyboard interrupt for {0}\".format(spec_name)\n installer = create_installer([spec_name], {\"fake\": True})\n setattr(inst.PackageInstaller, \"_real_install_task\", inst.PackageInstaller._complete_task)\n # Raise a KeyboardInterrupt error to trigger early termination\n monkeypatch.setattr(inst.PackageInstaller, \"_complete_task\", _interrupt)\n\n with pytest.raises(KeyboardInterrupt, match=err_msg):\n installer.install()\n\n assert not any(i.startswith(\"pkg-a-\") for i in installer.installed)\n assert any(\n i.startswith(\"pkg-b-\") for i in installer.installed\n ) # ensure dependency of a is 'installed'\n\n\nclass MyBuildException(Exception):\n pass\n\n\n_old_complete_task = None\n\n\ndef _install_fail_my_build_exception(installer, task, install_status, **kwargs):\n if task.pkg.name == \"pkg-a\":\n raise MyBuildException(\"mock internal package build error for pkg-a\")\n else:\n _old_complete_task(installer, task, install_status)\n\n\ndef test_install_fail_single(install_mockery, mock_fetch, monkeypatch):\n \"\"\"Test expected results for failure of single package.\"\"\"\n global _old_complete_task\n\n installer = create_installer([\"pkg-a\"], {\"fake\": True})\n\n # Raise a KeyboardInterrupt error to trigger early termination\n _old_complete_task = inst.PackageInstaller._complete_task\n monkeypatch.setattr(inst.PackageInstaller, \"_complete_task\", _install_fail_my_build_exception)\n\n with pytest.raises(MyBuildException, match=\"mock internal package build error for pkg-a\"):\n installer.install()\n\n # ensure dependency of a is 'installed' and a is not\n assert any(pkg_id.startswith(\"pkg-b-\") for pkg_id in installer.installed)\n assert not any(pkg_id.startswith(\"pkg-a-\") for pkg_id in installer.installed)\n\n\ndef test_install_fail_multi(install_mockery, mock_fetch, monkeypatch):\n \"\"\"Test expected results for failure of multiple packages.\"\"\"\n global _old_complete_task\n installer = create_installer([\"pkg-a\", \"pkg-c\"], {\"fake\": True})\n\n # Raise a KeyboardInterrupt error to trigger early termination\n _old_complete_task = inst.PackageInstaller._complete_task\n monkeypatch.setattr(inst.PackageInstaller, \"_complete_task\", _install_fail_my_build_exception)\n\n with pytest.raises(spack.error.InstallError, match=\"Installation request failed\"):\n installer.install()\n\n # ensure the the second spec installed but not the first\n assert any(pkg_id.startswith(\"pkg-c-\") for pkg_id in installer.installed)\n assert not any(pkg_id.startswith(\"pkg-a-\") for pkg_id in installer.installed)\n\n\ndef test_install_fail_fast_on_detect(install_mockery, monkeypatch, capfd):\n \"\"\"Test fail_fast install when an install failure is detected.\"\"\"\n a = spack.concretize.concretize_one(\"parallel-package-a\")\n\n a_id = inst.package_id(a)\n b_id = inst.package_id(a[\"parallel-package-b\"])\n c_id = inst.package_id(a[\"parallel-package-c\"])\n\n installer = create_installer([a], {\"fail_fast\": True})\n # Make sure all packages are identified as failed\n # This will prevent a and b from installing, which will cause the build of c to be skipped\n # and the active processes to be killed.\n monkeypatch.setattr(spack.database.FailureTracker, \"has_failed\", _true)\n\n installer.max_active_tasks = 2\n with pytest.raises(spack.error.InstallError, match=\"after first install failure\"):\n installer.install()\n\n assert b_id in installer.failed, \"Expected b to be marked as failed\"\n assert c_id in installer.failed, \"Expected c to be marked as failed\"\n assert a_id not in installer.installed, (\n \"Package a cannot install due to its dependencies failing\"\n )\n # check that b's active process got killed when c failed\n\n assert f\"{b_id} failed to install\" in capfd.readouterr().err\n\n\ndef _test_install_fail_fast_on_except_patch(installer, **kwargs):\n \"\"\"Helper for test_install_fail_fast_on_except.\"\"\"\n # This is a module-scope function and not a local function because it\n # needs to be pickleable.\n raise RuntimeError(\"mock patch failure\")\n\n\n@pytest.mark.disable_clean_stage_check\ndef test_install_fail_fast_on_except(install_mockery, monkeypatch, capfd):\n \"\"\"Test fail_fast install when an install failure results from an error.\"\"\"\n installer = create_installer([\"pkg-a\"], {\"fail_fast\": True})\n\n # Raise a non-KeyboardInterrupt exception to trigger fast failure.\n #\n # This will prevent b from installing, which will cause the build of a\n # to be skipped.\n monkeypatch.setattr(\n spack.package_base.PackageBase, \"do_patch\", _test_install_fail_fast_on_except_patch\n )\n\n with pytest.raises(spack.error.InstallError, match=\"mock patch failure\"):\n installer.install()\n\n out = str(capfd.readouterr())\n assert \"Skipping build of pkg-a\" in out\n\n\ndef test_install_lock_failures(install_mockery, monkeypatch, capfd):\n \"\"\"Cover basic install lock failure handling in a single pass.\"\"\"\n\n # Note: this test relies on installing a package with no dependencies\n def _requeued(installer, task, install_status):\n tty.msg(\"requeued {0}\".format(task.pkg.spec.name))\n\n installer = create_installer([\"pkg-c\"], {})\n\n # Ensure never acquire a lock\n monkeypatch.setattr(inst.PackageInstaller, \"_ensure_locked\", _not_locked)\n\n # Ensure don't continually requeue the task\n monkeypatch.setattr(inst.PackageInstaller, \"_requeue_task\", _requeued)\n\n with pytest.raises(spack.error.InstallError, match=\"request failed\"):\n installer.install()\n\n out = capfd.readouterr()[0]\n expected = [\"write locked\", \"read locked\", \"requeued\"]\n for exp, ln in zip(expected, out.split(\"\\n\")):\n assert exp in ln\n\n\ndef test_install_lock_installed_requeue(install_mockery, monkeypatch, capfd):\n \"\"\"Cover basic install handling for installed package.\"\"\"\n # Note: this test relies on installing a package with no dependencies\n concrete_spec = spack.concretize.concretize_one(\"pkg-c\")\n pkg_id = inst.package_id(concrete_spec)\n installer = create_installer([concrete_spec])\n\n def _prep(installer, task):\n installer.installed.add(pkg_id)\n tty.msg(f\"{pkg_id} is installed\")\n\n # also do not allow the package to be locked again\n monkeypatch.setattr(inst.PackageInstaller, \"_ensure_locked\", _not_locked)\n\n def _requeued(installer, task, install_status):\n tty.msg(f\"requeued {inst.package_id(task.pkg.spec)}\")\n\n # Flag the package as installed\n monkeypatch.setattr(inst.PackageInstaller, \"_prepare_for_install\", _prep)\n\n # Ensure don't continually requeue the task\n monkeypatch.setattr(inst.PackageInstaller, \"_requeue_task\", _requeued)\n\n with pytest.raises(spack.error.InstallError, match=\"request failed\"):\n installer.install()\n\n assert pkg_id not in installer.installed\n\n expected = [\"is installed\", \"read locked\", \"requeued\"]\n for exp, ln in zip(expected, capfd.readouterr().out.splitlines()):\n assert exp in ln\n\n\ndef test_install_read_locked_requeue(install_mockery, monkeypatch, capfd):\n \"\"\"Cover basic read lock handling for uninstalled package with requeue.\"\"\"\n # Note: this test relies on installing a package with no dependencies\n orig_fn = inst.PackageInstaller._ensure_locked\n\n def _read(installer, lock_type, pkg):\n tty.msg(\"{0}->read locked {1}\".format(lock_type, pkg.spec.name))\n return orig_fn(installer, \"read\", pkg)\n\n def _prep(installer, task):\n tty.msg(\"preparing {0}\".format(task.pkg.spec.name))\n assert task.pkg.spec.name not in installer.installed\n\n def _requeued(installer, task, install_status):\n tty.msg(\"requeued {0}\".format(task.pkg.spec.name))\n\n # Force a read lock\n monkeypatch.setattr(inst.PackageInstaller, \"_ensure_locked\", _read)\n\n # Flag the package as installed\n monkeypatch.setattr(inst.PackageInstaller, \"_prepare_for_install\", _prep)\n\n # Ensure don't continually requeue the task\n monkeypatch.setattr(inst.PackageInstaller, \"_requeue_task\", _requeued)\n\n installer = create_installer([\"pkg-c\"], {})\n\n with pytest.raises(spack.error.InstallError, match=\"request failed\"):\n installer.install()\n\n assert \"b\" not in installer.installed\n\n out = capfd.readouterr()[0]\n expected = [\"write->read locked\", \"preparing\", \"requeued\"]\n for exp, ln in zip(expected, out.split(\"\\n\")):\n assert exp in ln\n\n\ndef test_install_skip_patch(install_mockery, mock_fetch):\n \"\"\"Test the path skip_patch install path.\"\"\"\n # Note: this test relies on installing a package with no dependencies\n installer = create_installer([\"pkg-c\"], {\"fake\": False, \"skip_patch\": True})\n installer.install()\n assert inst.package_id(installer.build_requests[0].pkg.spec) in installer.installed\n\n\ndef test_install_implicit(install_mockery, mock_fetch):\n \"\"\"Test the path skip_patch install path.\"\"\"\n spec_name = \"trivial-install-test-package\"\n installer = create_installer([spec_name], {\"fake\": False})\n pkg = installer.build_requests[0].pkg\n assert not create_build_task(pkg, {\"explicit\": []}).explicit\n assert create_build_task(pkg, {\"explicit\": [pkg.spec.dag_hash()]}).explicit\n assert not create_build_task(pkg).explicit\n\n\n# Install that wipes the prefix directory\ndef wipe_prefix(pkg, install_args):\n shutil.rmtree(pkg.prefix, ignore_errors=True)\n fs.mkdirp(pkg.prefix)\n raise Exception(\"Some fatal install error\")\n\n\ndef fail(*args, **kwargs):\n assert False\n\n\ndef test_overwrite_install_backup_success(monkeypatch, temporary_store, config, mock_packages):\n \"\"\"\n When doing an overwrite install that fails, Spack should restore the backup\n of the original prefix, and leave the original spec marked installed.\n \"\"\"\n # Get a build task. TODO: Refactor this to avoid calling internal methods.\n # This task relies on installing something with no dependencies\n installer = create_installer([\"pkg-c\"])\n installer._init_queue()\n task = installer._pop_task()\n install_status = MockInstallStatus(1)\n term_status = MockTermStatusLine(True)\n\n # Make sure the install prefix exists with some trivial file\n installed_file = os.path.join(task.pkg.prefix, \"some_file\")\n fs.touchp(installed_file)\n\n monkeypatch.setattr(inst, \"build_process\", wipe_prefix)\n\n # Make sure the package is not marked uninstalled\n monkeypatch.setattr(spack.store.STORE.db, \"remove\", fail)\n # Make sure that the installer does an overwrite install\n monkeypatch.setattr(task, \"_install_action\", inst.InstallAction.OVERWRITE)\n\n # Installation should throw the installation exception, not the backup\n # failure.\n installer.start_task(task, install_status, term_status)\n with pytest.raises(Exception, match=\"Some fatal install error\"):\n installer.complete_task(task, install_status)\n\n # Check that the original file is back.\n assert os.path.exists(installed_file)\n\n\n# Install that removes the backup directory, which is at the same level as\n# the prefix, starting with .backup\ndef remove_backup(pkg, install_args):\n backup_glob = os.path.join(os.path.dirname(os.path.normpath(pkg.prefix)), \".backup*\")\n for backup in glob.iglob(backup_glob):\n shutil.rmtree(backup)\n raise Exception(\"Some fatal install error\")\n\n\ndef test_overwrite_install_backup_failure(monkeypatch, temporary_store, config, mock_packages):\n \"\"\"\n When doing an overwrite install that fails, Spack should try to recover the\n original prefix. If that fails, the spec is lost, and it should be removed\n from the database.\n \"\"\"\n # Get a build task. TODO: refactor this to avoid calling internal methods\n installer = create_installer([\"pkg-c\"])\n installer._init_queue()\n task = installer._pop_task()\n install_status = MockInstallStatus(1)\n term_status = MockTermStatusLine(True)\n\n # Make sure the install prefix exists\n installed_file = os.path.join(task.pkg.prefix, \"some_file\")\n fs.touchp(installed_file)\n monkeypatch.setattr(inst, \"build_process\", remove_backup)\n\n # Make sure that the installer does an overwrite install\n monkeypatch.setattr(task, \"_install_action\", inst.InstallAction.OVERWRITE)\n\n # Make sure that `remove` was called on the database after an unsuccessful\n # attempt to restore the backup.\n # This error is raised while handling the original install error\n installer.start_task(task, install_status, term_status)\n with pytest.raises(Exception, match=\"No such spec in database\"):\n installer.complete_task(task, install_status)\n\n\ndef test_term_status_line():\n # Smoke test for TermStatusLine; to actually test output it would be great\n # to pass a StringIO instance, but we use tty.msg() internally which does not\n # accept that. `with log_output(buf)` doesn't really work because it trims output\n # and we actually want to test for escape sequences etc.\n x = inst.TermStatusLine(enabled=True)\n x.add(\"pkg-a\")\n x.add(\"pkg-b\")\n x.clear()\n\n\n@pytest.mark.parametrize(\"explicit\", [True, False])\ndef test_single_external_implicit_install(install_mockery, explicit):\n pkg = \"trivial-install-test-package\"\n s = spack.concretize.concretize_one(pkg)\n s.external_path = \"/usr\"\n args = {\"explicit\": [s.dag_hash()] if explicit else []}\n create_installer([s], args).install()\n assert spack.store.STORE.db.get_record(pkg).explicit == explicit\n\n\ndef test_overwrite_install_does_install_build_deps(install_mockery, mock_fetch):\n \"\"\"When overwrite installing something from sources, build deps should be installed.\"\"\"\n s = spack.concretize.concretize_one(\"dtrun3\")\n create_installer([s]).install()\n\n # Verify there is a pure build dep\n edge = s.edges_to_dependencies(name=\"dtbuild3\").pop()\n assert edge.depflag == dt.BUILD\n build_dep = edge.spec\n\n # Uninstall the build dep\n build_dep.package.do_uninstall()\n\n # Overwrite install the root dtrun3\n create_installer([s], {\"overwrite\": [s.dag_hash()]}).install()\n\n # Verify that the build dep was also installed.\n assert build_dep.installed\n\n\n@pytest.mark.parametrize(\"run_tests\", [True, False])\ndef test_print_install_test_log_skipped(install_mockery, mock_packages, capfd, run_tests):\n \"\"\"Confirm printing of install log skipped if not run/no failures.\"\"\"\n name = \"trivial-install-test-package\"\n s = spack.concretize.concretize_one(name)\n pkg = s.package\n\n pkg.run_tests = run_tests\n inst.print_install_test_log(pkg)\n out = capfd.readouterr()[0]\n assert out == \"\"\n\n\ndef test_print_install_test_log_failures(\n tmp_path: pathlib.Path, install_mockery, mock_packages, ensure_debug, capfd\n):\n \"\"\"Confirm expected outputs when there are test failures.\"\"\"\n name = \"trivial-install-test-package\"\n s = spack.concretize.concretize_one(name)\n pkg = s.package\n\n # Missing test log is an error\n pkg.run_tests = True\n pkg.tester.test_log_file = str(tmp_path / \"test-log.txt\")\n pkg.tester.add_failure(AssertionError(\"test\"), \"test-failure\")\n inst.print_install_test_log(pkg)\n err = capfd.readouterr()[1]\n assert \"no test log file\" in err\n\n # Having test log results in path being output\n fs.touch(pkg.tester.test_log_file)\n inst.print_install_test_log(pkg)\n out = capfd.readouterr()[0]\n assert \"See test results at\" in out\n\n\ndef test_fallback_to_old_installer_for_splicing(monkeypatch, mock_packages, mutable_config):\n \"\"\"Test that the old installer is used for spliced specs (unsupported in the new installer)\"\"\"\n mutable_config.set(\"config:installer\", \"new\")\n spec = spack.concretize.concretize_one(\"splice-t\")\n dep = spack.concretize.concretize_one(\"splice-h+foo\")\n out = spec.splice(dep)\n assert isinstance(\n spack.installer_dispatch.create_installer([out.package]), inst.PackageInstaller\n )\n" }, { "path": "lib/spack/spack/test/installer_build_graph.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Tests for BuildGraph class in new_installer\"\"\"\n\nimport sys\nfrom typing import Dict, List, Tuple, Union\n\nimport pytest\n\nif sys.platform == \"win32\":\n pytest.skip(\"Skipping new installer tests on Windows\", allow_module_level=True)\n\nimport spack.deptypes as dt\nimport spack.error\nimport spack.traverse\nfrom spack.new_installer import BuildGraph\nfrom spack.spec import Spec\nfrom spack.store import Store\n\n\ndef create_dag(\n nodes: List[str], edges: List[Tuple[str, str, Union[dt.DepType, Tuple[dt.DepType, ...]]]]\n) -> Dict[str, Spec]:\n \"\"\"\n Create a DAG of concrete specs, as a mapping from package name to Spec.\n\n Arguments:\n nodes: list of unique package names\n edges: list of tuples (parent, child, deptype)\n \"\"\"\n specs = {name: Spec(name) for name in nodes}\n for parent, child, deptypes in edges:\n depflag = deptypes if isinstance(deptypes, dt.DepFlag) else dt.canonicalize(deptypes)\n specs[parent].add_dependency_edge(specs[child], depflag=depflag, virtuals=())\n\n # Mark all specs as concrete\n for spec in specs.values():\n spec._mark_concrete()\n\n return specs\n\n\ndef install_spec_in_db(spec: Spec, store: Store):\n \"\"\"Helper to install a spec in the database for testing.\"\"\"\n prefix = store.layout.path_for_spec(spec)\n spec.set_prefix(prefix)\n # Use the layout to create a proper installation directory structure\n store.layout.create_install_directory(spec)\n store.db.add(spec, explicit=False)\n\n\n@pytest.fixture\ndef mock_specs():\n \"\"\"Create a set of mock specs for testing.\n\n DAG structure:\n root -> dep1 -> dep2\n root -> dep3\n \"\"\"\n return create_dag(\n nodes=[\"root\", \"dep1\", \"dep2\", \"dep3\"],\n edges=[\n (\"root\", \"dep1\", (\"build\", \"link\")),\n (\"root\", \"dep3\", (\"build\", \"link\")),\n (\"dep1\", \"dep2\", (\"build\", \"link\")),\n ],\n )\n\n\n@pytest.fixture\ndef diamond_dag():\n \"\"\"Create a diamond-shaped DAG to test shared dependencies.\n\n DAG structure:\n root -> dep1 -> shared\n root -> dep2 -> shared\n \"\"\"\n return create_dag(\n nodes=[\"root\", \"dep1\", \"dep2\", \"shared\"],\n edges=[\n (\"root\", \"dep1\", (\"build\", \"link\")),\n (\"root\", \"dep2\", (\"build\", \"link\")),\n (\"dep1\", \"shared\", (\"build\", \"link\")),\n (\"dep2\", \"shared\", (\"build\", \"link\")),\n ],\n )\n\n\n@pytest.fixture\ndef specs_with_build_deps():\n \"\"\"Create specs with different dependency types for testing build dep filtering.\n\n DAG structure:\n root -> link_dep (link only)\n root -> build_dep (build only)\n root -> all_dep (build, link, run)\n \"\"\"\n return create_dag(\n nodes=[\"root\", \"link_dep\", \"build_dep\", \"all_dep\"],\n edges=[\n (\"root\", \"link_dep\", \"link\"),\n (\"root\", \"build_dep\", \"build\"),\n (\"root\", \"all_dep\", (\"build\", \"link\", \"run\")),\n ],\n )\n\n\n@pytest.fixture\ndef complex_pruning_dag():\n \"\"\"Create a complex DAG for testing re-parenting logic.\n\n DAG structure:\n parent1 -> middle -> child1\n parent2 -> middle -> child2\n\n When 'middle' is installed and pruned, both parent1 and parent2 should\n become direct parents of both child1 and child2 (full Cartesian product).\n \"\"\"\n return create_dag(\n nodes=[\"parent1\", \"parent2\", \"middle\", \"child1\", \"child2\"],\n edges=[\n (\"parent1\", \"middle\", (\"build\", \"link\")),\n (\"parent2\", \"middle\", (\"build\", \"link\")),\n (\"middle\", \"child1\", (\"build\", \"link\")),\n (\"middle\", \"child2\", (\"build\", \"link\")),\n ],\n )\n\n\nclass TestBuildGraph:\n \"\"\"Tests for the BuildGraph class.\"\"\"\n\n def test_basic_graph_construction(self, mock_specs: Dict[str, Spec], temporary_store: Store):\n \"\"\"Test basic graph construction with all specs to be installed.\"\"\"\n graph = BuildGraph(\n specs=[mock_specs[\"root\"]],\n root_policy=\"auto\",\n dependencies_policy=\"auto\",\n include_build_deps=False,\n install_package=True,\n install_deps=True,\n database=temporary_store.db,\n )\n\n # Root should be in roots set\n assert mock_specs[\"root\"].dag_hash() in graph.roots\n # All uninstalled specs should be in nodes\n assert len(graph.nodes) == 4 # root, dep1, dep2, dep3\n # Root should have 2 children (dep1, dep3)\n assert len(graph.parent_to_child[mock_specs[\"root\"].dag_hash()]) == 2\n\n def test_install_package_only_mode(self, mock_specs: Dict[str, Spec], temporary_store: Store):\n \"\"\"Test that install_package=False removes root specs from graph.\"\"\"\n graph = BuildGraph(\n specs=[mock_specs[\"root\"]],\n root_policy=\"auto\",\n dependencies_policy=\"auto\",\n include_build_deps=False,\n install_package=False, # Only install dependencies\n install_deps=True,\n database=temporary_store.db,\n )\n\n # Root should NOT be in nodes when install_package=False\n assert mock_specs[\"root\"].dag_hash() not in graph.nodes\n # But its dependencies should be\n assert mock_specs[\"dep1\"].dag_hash() in graph.nodes\n\n def test_install_deps_false_with_uninstalled_deps(\n self, mock_specs: Dict[str, Spec], temporary_store: Store\n ):\n \"\"\"Test that install_deps=False raises error when dependencies are not installed.\"\"\"\n # Should raise error because dependencies are not installed\n with pytest.raises(\n spack.error.InstallError, match=\"package only mode.*dependency.*not installed\"\n ):\n BuildGraph(\n specs=[mock_specs[\"root\"]],\n root_policy=\"auto\",\n dependencies_policy=\"auto\",\n include_build_deps=False,\n install_package=True,\n install_deps=False, # Don't install dependencies\n database=temporary_store.db,\n )\n\n def test_multiple_roots(self, mock_specs: Dict[str, Spec], temporary_store: Store):\n \"\"\"Test graph construction with multiple root specs.\"\"\"\n graph = BuildGraph(\n specs=[mock_specs[\"root\"], mock_specs[\"dep1\"]],\n root_policy=\"auto\",\n dependencies_policy=\"auto\",\n include_build_deps=False,\n install_package=True,\n install_deps=True,\n database=temporary_store.db,\n )\n\n # Both should be in roots\n assert mock_specs[\"root\"].dag_hash() in graph.roots\n assert mock_specs[\"dep1\"].dag_hash() in graph.roots\n\n def test_parent_child_mappings(self, mock_specs: Dict[str, Spec], temporary_store: Store):\n \"\"\"Test that parent-child mappings are correctly constructed.\"\"\"\n spec_root = mock_specs[\"root\"]\n graph = BuildGraph(\n specs=[spec_root],\n root_policy=\"auto\",\n dependencies_policy=\"auto\",\n include_build_deps=False,\n install_package=True,\n install_deps=True,\n database=temporary_store.db,\n )\n\n # Verify parent_to_child and child_to_parent are inverse mappings\n for parent, children in graph.parent_to_child.items():\n for child in children:\n assert child in graph.child_to_parent\n assert parent in graph.child_to_parent[child]\n\n def test_diamond_dag_with_shared_dependency(\n self, diamond_dag: Dict[str, Spec], temporary_store: Store\n ):\n \"\"\"Test graph construction with a diamond DAG where a dependency has multiple parents.\"\"\"\n graph = BuildGraph(\n specs=[diamond_dag[\"root\"]],\n root_policy=\"auto\",\n dependencies_policy=\"auto\",\n include_build_deps=False,\n install_package=True,\n install_deps=True,\n database=temporary_store.db,\n )\n\n # Shared dependency should have two parents\n shared_hash = diamond_dag[\"shared\"].dag_hash()\n assert len(graph.child_to_parent[shared_hash]) == 2\n # Both dep1 and dep2 should be parents of shared\n assert diamond_dag[\"dep1\"].dag_hash() in graph.child_to_parent[shared_hash]\n assert diamond_dag[\"dep2\"].dag_hash() in graph.child_to_parent[shared_hash]\n\n def test_pruning_installed_specs(self, mock_specs: Dict[str, Spec], temporary_store: Store):\n \"\"\"Test that installed specs are correctly pruned from the graph.\"\"\"\n # Install dep2 in the database\n dep2 = mock_specs[\"dep2\"]\n install_spec_in_db(dep2, temporary_store)\n\n graph = BuildGraph(\n specs=[mock_specs[\"root\"]],\n root_policy=\"auto\",\n dependencies_policy=\"auto\",\n include_build_deps=False,\n install_package=True,\n install_deps=True,\n database=temporary_store.db,\n )\n\n # dep2 should be pruned since it's installed\n assert dep2.dag_hash() not in graph.nodes\n # But dep1 (its parent) should still be in the graph\n assert mock_specs[\"dep1\"].dag_hash() in graph.nodes\n # And dep1 should have no children (since dep2 was pruned)\n assert len(graph.parent_to_child[mock_specs[\"dep1\"].dag_hash()]) == 0\n\n def test_pruning_with_shared_dependency_partially_installed(\n self, diamond_dag: Dict[str, Spec], temporary_store: Store\n ):\n \"\"\"Test that pruning a shared dependency correctly updates all parents.\"\"\"\n # Install the shared dependency\n shared = diamond_dag[\"shared\"]\n install_spec_in_db(shared, temporary_store)\n graph = BuildGraph(\n specs=[diamond_dag[\"root\"]],\n root_policy=\"auto\",\n dependencies_policy=\"auto\",\n include_build_deps=False,\n install_package=True,\n install_deps=True,\n database=temporary_store.db,\n )\n\n # Shared should be pruned\n assert shared.dag_hash() not in graph.nodes\n # Both dep1 and dep2 should have no children\n assert len(graph.parent_to_child[diamond_dag[\"dep1\"].dag_hash()]) == 0\n assert len(graph.parent_to_child[diamond_dag[\"dep2\"].dag_hash()]) == 0\n\n def test_overwrite_set_prevents_pruning(\n self, mock_specs: Dict[str, Spec], temporary_store: Store\n ):\n \"\"\"Test that specs in overwrite_set are not pruned even if installed.\"\"\"\n # Install dep2 in the database\n dep2 = mock_specs[\"dep2\"]\n install_spec_in_db(dep2, temporary_store)\n\n # Create graph with dep2 in the overwrite set\n graph = BuildGraph(\n specs=[mock_specs[\"root\"]],\n root_policy=\"auto\",\n dependencies_policy=\"auto\",\n include_build_deps=False,\n install_package=True,\n install_deps=True,\n database=temporary_store.db,\n overwrite_set={dep2.dag_hash()},\n )\n\n # dep2 should NOT be pruned since it's in overwrite_set\n assert dep2.dag_hash() in graph.nodes\n # dep1 should still have dep2 as a child\n assert dep2.dag_hash() in graph.parent_to_child[mock_specs[\"dep1\"].dag_hash()]\n # dep2 should have dep1 as a parent\n assert mock_specs[\"dep1\"].dag_hash() in graph.child_to_parent[dep2.dag_hash()]\n\n def test_installed_root_excludes_build_deps_even_when_requested(\n self, specs_with_build_deps: Dict[str, Spec], temporary_store: Store\n ):\n \"\"\"Test that installed root specs never include build deps, even with\n include_build_deps=True.\"\"\"\n root = specs_with_build_deps[\"root\"]\n install_spec_in_db(root, temporary_store)\n\n graph = BuildGraph(\n specs=[root],\n root_policy=\"auto\",\n dependencies_policy=\"auto\",\n include_build_deps=True, # Should be ignored for installed root\n install_package=True,\n install_deps=True,\n database=temporary_store.db,\n )\n\n # build_dep should NOT be in the graph (installed root never needs build deps)\n assert specs_with_build_deps[\"build_dep\"].dag_hash() not in graph.nodes\n # link_dep and all_dep should be in the graph (link/run deps)\n assert specs_with_build_deps[\"link_dep\"].dag_hash() in graph.nodes\n assert specs_with_build_deps[\"all_dep\"].dag_hash() in graph.nodes\n\n def test_cache_only_excludes_build_deps(\n self, specs_with_build_deps: Dict[str, Spec], temporary_store: Store\n ):\n \"\"\"Test that cache_only policy excludes build deps when include_build_deps=False.\"\"\"\n specs = [specs_with_build_deps[\"root\"]]\n graph = BuildGraph(\n specs=specs,\n root_policy=\"cache_only\",\n dependencies_policy=\"auto\",\n include_build_deps=False, # exclude build deps when possible\n install_package=True,\n install_deps=True,\n database=temporary_store.db,\n )\n\n assert specs_with_build_deps[\"build_dep\"].dag_hash() not in graph.nodes\n assert specs_with_build_deps[\"link_dep\"].dag_hash() in graph.nodes\n assert specs_with_build_deps[\"all_dep\"].dag_hash() in graph.nodes\n\n # Verify that the entire graph has a prefix assigned, which avoids that the subprocess has\n # to obtain a read lock on the database.\n for s in spack.traverse.traverse_nodes(specs):\n assert s._prefix is not None\n\n def test_cache_only_includes_build_deps_when_requested(\n self, specs_with_build_deps: Dict[str, Spec], temporary_store: Store\n ):\n \"\"\"Test that cache_only policy includes build deps when include_build_deps=True.\"\"\"\n graph = BuildGraph(\n specs=[specs_with_build_deps[\"root\"]],\n root_policy=\"cache_only\",\n dependencies_policy=\"cache_only\",\n include_build_deps=True,\n install_package=True,\n install_deps=True,\n database=temporary_store.db,\n )\n\n # All dependencies should be in the graph, including build_dep\n assert specs_with_build_deps[\"build_dep\"].dag_hash() in graph.nodes\n assert specs_with_build_deps[\"link_dep\"].dag_hash() in graph.nodes\n assert specs_with_build_deps[\"all_dep\"].dag_hash() in graph.nodes\n\n def test_install_deps_false_with_all_deps_installed(\n self, mock_specs: Dict[str, Spec], temporary_store: Store\n ):\n \"\"\"Test successful package-only install when all dependencies are already installed.\"\"\"\n # Install all dependencies\n for dep_name in [\"dep1\", \"dep2\", \"dep3\"]:\n install_spec_in_db(mock_specs[dep_name], temporary_store)\n\n # Should succeed since all dependencies are installed\n graph = BuildGraph(\n specs=[mock_specs[\"root\"]],\n root_policy=\"auto\",\n dependencies_policy=\"auto\",\n include_build_deps=False,\n install_package=True,\n install_deps=False,\n database=temporary_store.db,\n )\n\n # Only the root should be in the graph\n assert len(graph.nodes) == 1\n assert mock_specs[\"root\"].dag_hash() in graph.nodes\n # Root should have no children (all deps pruned)\n assert len(graph.parent_to_child.get(mock_specs[\"root\"].dag_hash(), [])) == 0\n\n def test_pruning_creates_cartesian_product_of_connections(\n self, complex_pruning_dag: Dict[str, Spec], temporary_store: Store\n ):\n \"\"\"Test that pruning creates full Cartesian product of parent-child connections.\n\n When a node with multiple parents and multiple children is pruned,\n all parents should be connected to all children (parents x children).\n\n DAG structure:\n parent1 -> middle -> child1\n parent2 -> middle -> child2\n\n After pruning 'middle':\n parent1 -> child1\n parent1 -> child2\n parent2 -> child1\n parent2 -> child2\n \"\"\"\n # Install the middle node\n middle = complex_pruning_dag[\"middle\"]\n install_spec_in_db(middle, temporary_store)\n\n # Use parent1 as the root to build the graph\n graph = BuildGraph(\n specs=[complex_pruning_dag[\"parent1\"], complex_pruning_dag[\"parent2\"]],\n root_policy=\"auto\",\n dependencies_policy=\"auto\",\n include_build_deps=False,\n install_package=True,\n install_deps=True,\n database=temporary_store.db,\n )\n\n parent1_hash = complex_pruning_dag[\"parent1\"].dag_hash()\n parent2_hash = complex_pruning_dag[\"parent2\"].dag_hash()\n middle_hash = middle.dag_hash()\n child1_hash = complex_pruning_dag[\"child1\"].dag_hash()\n child2_hash = complex_pruning_dag[\"child2\"].dag_hash()\n\n # middle should be pruned since it's installed\n assert middle_hash not in graph.nodes\n\n # All other nodes should be in the graph\n assert parent1_hash in graph.nodes\n assert parent2_hash in graph.nodes\n assert child1_hash in graph.nodes\n assert child2_hash in graph.nodes\n\n # Verify full Cartesian product: each parent should be connected to each child\n # parent1 -> child1, child2\n assert child1_hash in graph.parent_to_child[parent1_hash]\n assert child2_hash in graph.parent_to_child[parent1_hash]\n\n # parent2 -> child1, child2\n assert child1_hash in graph.parent_to_child[parent2_hash]\n assert child2_hash in graph.parent_to_child[parent2_hash]\n\n # Verify reverse mapping: each child should have both parents\n # child1 <- parent1, parent2\n assert parent1_hash in graph.child_to_parent[child1_hash]\n assert parent2_hash in graph.child_to_parent[child1_hash]\n\n # child2 <- parent1, parent2\n assert parent1_hash in graph.child_to_parent[child2_hash]\n assert parent2_hash in graph.child_to_parent[child2_hash]\n\n # middle should not appear in any parent-child relationships\n assert middle_hash not in graph.parent_to_child\n assert middle_hash not in graph.child_to_parent\n\n def test_empty_graph_all_specs_installed(\n self, mock_specs: Dict[str, Spec], temporary_store: Store\n ):\n \"\"\"Test that the graph is empty when all specs are already installed.\"\"\"\n # Install all specs in the DAG\n for spec_name in [\"root\", \"dep1\", \"dep2\", \"dep3\"]:\n install_spec_in_db(mock_specs[spec_name], temporary_store)\n\n graph = BuildGraph(\n specs=[mock_specs[\"root\"]],\n root_policy=\"auto\",\n dependencies_policy=\"auto\",\n include_build_deps=False,\n install_package=True,\n install_deps=True,\n database=temporary_store.db,\n )\n\n # All nodes should be pruned, resulting in an empty graph\n assert len(graph.nodes) == 0\n assert len(graph.parent_to_child) == 0\n assert len(graph.child_to_parent) == 0\n\n def test_empty_graph_install_package_false_all_deps_installed(\n self, mock_specs: Dict[str, Spec], temporary_store: Store\n ):\n \"\"\"Test empty graph when install_package=False and all dependencies are installed.\"\"\"\n # Install all dependencies (but not the root)\n for dep_name in [\"dep1\", \"dep2\", \"dep3\"]:\n install_spec_in_db(mock_specs[dep_name], temporary_store)\n\n graph = BuildGraph(\n specs=[mock_specs[\"root\"]],\n root_policy=\"auto\",\n dependencies_policy=\"auto\",\n include_build_deps=False,\n install_package=False, # Don't install the root\n install_deps=True,\n database=temporary_store.db,\n )\n\n # Root is pruned because install_package=False\n # Dependencies are pruned because they're installed\n # Result: empty graph\n assert len(graph.nodes) == 0\n assert len(graph.parent_to_child) == 0\n assert len(graph.child_to_parent) == 0\n\n def test_pruning_leaf_node(self, mock_specs: Dict[str, Spec], temporary_store: Store):\n \"\"\"Test that pruning a leaf node (no children) works correctly.\n\n This ensures the pruning logic handles the boundary condition where\n a node has no children to re-wire.\n \"\"\"\n # Install dep2, which is a leaf node (no children)\n dep2 = mock_specs[\"dep2\"]\n install_spec_in_db(dep2, temporary_store)\n\n graph = BuildGraph(\n specs=[mock_specs[\"root\"]],\n root_policy=\"auto\",\n dependencies_policy=\"auto\",\n include_build_deps=False,\n install_package=True,\n install_deps=True,\n database=temporary_store.db,\n )\n\n dep2_hash = dep2.dag_hash()\n dep1_hash = mock_specs[\"dep1\"].dag_hash()\n\n # dep2 should be pruned\n assert dep2_hash not in graph.nodes\n # dep1 (parent of dep2) should have no children now\n assert len(graph.parent_to_child[dep1_hash]) == 0\n # dep2 should not appear in any mappings\n assert dep2_hash not in graph.parent_to_child\n assert dep2_hash not in graph.child_to_parent\n\n def test_pruning_root_node_with_install_package_false(\n self, mock_specs: Dict[str, Spec], temporary_store: Store\n ):\n \"\"\"Test that pruning a root node (no parents in the context) works correctly.\n\n When install_package=False, root nodes are marked for pruning. This ensures\n the pruning logic handles the boundary condition where a node has no parents.\n \"\"\"\n graph = BuildGraph(\n specs=[mock_specs[\"dep1\"]],\n root_policy=\"auto\",\n dependencies_policy=\"auto\",\n include_build_deps=False,\n install_package=False, # Prune the root\n install_deps=True,\n database=temporary_store.db,\n )\n\n dep1_hash = mock_specs[\"dep1\"].dag_hash()\n dep2_hash = mock_specs[\"dep2\"].dag_hash()\n\n # dep1 should be pruned (it's the root and install_package=False)\n assert dep1_hash not in graph.nodes\n # dep2 (child of dep1) should still be in the graph\n assert dep2_hash in graph.nodes\n # dep2 should have no parents now (its only parent was pruned)\n assert not graph.child_to_parent.get(dep2_hash)\n # dep1 should not appear in any mappings\n assert dep1_hash not in graph.parent_to_child\n assert dep1_hash not in graph.child_to_parent\n\n\n@pytest.fixture\ndef specs_with_test_deps():\n \"\"\"Create specs with test-typed dependencies.\n\n DAG structure:\n root -> dep (link) + test_dep (test)\n dep -> dep_test_dep (test)\n \"\"\"\n return create_dag(\n nodes=[\"root\", \"dep\", \"test_dep\", \"dep_test_dep\"],\n edges=[\n (\"root\", \"dep\", (\"build\", \"link\")),\n (\"root\", \"test_dep\", \"test\"),\n (\"dep\", \"dep_test_dep\", \"test\"),\n ],\n )\n\n\nclass TestBuildGraphTestDeps:\n \"\"\"Tests for BuildGraph handling of TEST-typed dependencies.\"\"\"\n\n def test_tests_false_excludes_test_deps(\n self, specs_with_test_deps: Dict[str, Spec], temporary_store: Store\n ):\n \"\"\"Test that tests=False excludes TEST-typed dependencies.\"\"\"\n graph = BuildGraph(\n specs=[specs_with_test_deps[\"root\"]],\n root_policy=\"auto\",\n dependencies_policy=\"auto\",\n include_build_deps=True,\n install_package=True,\n install_deps=True,\n database=temporary_store.db,\n tests=False,\n )\n\n assert specs_with_test_deps[\"dep\"].dag_hash() in graph.nodes\n assert specs_with_test_deps[\"test_dep\"].dag_hash() not in graph.nodes\n assert specs_with_test_deps[\"dep_test_dep\"].dag_hash() not in graph.nodes\n\n def test_tests_root_includes_test_deps_for_root(\n self, specs_with_test_deps: Dict[str, Spec], temporary_store: Store\n ):\n \"\"\"Test that tests=[root_name] includes test deps only for the root package.\"\"\"\n graph = BuildGraph(\n specs=[specs_with_test_deps[\"root\"]],\n root_policy=\"auto\",\n dependencies_policy=\"auto\",\n include_build_deps=True,\n install_package=True,\n install_deps=True,\n database=temporary_store.db,\n tests=[\"root\"],\n )\n\n assert specs_with_test_deps[\"dep\"].dag_hash() in graph.nodes\n assert specs_with_test_deps[\"test_dep\"].dag_hash() in graph.nodes\n # dep's test dep is NOT included because tests=[\"root\"] only applies to \"root\"\n assert specs_with_test_deps[\"dep_test_dep\"].dag_hash() not in graph.nodes\n\n def test_tests_all_includes_test_deps_for_all(\n self, specs_with_test_deps: Dict[str, Spec], temporary_store: Store\n ):\n \"\"\"Test that tests=True includes TEST-typed deps for all packages.\"\"\"\n graph = BuildGraph(\n specs=[specs_with_test_deps[\"root\"]],\n root_policy=\"auto\",\n dependencies_policy=\"auto\",\n include_build_deps=True,\n install_package=True,\n install_deps=True,\n database=temporary_store.db,\n tests=True,\n )\n\n assert specs_with_test_deps[\"dep\"].dag_hash() in graph.nodes\n assert specs_with_test_deps[\"test_dep\"].dag_hash() in graph.nodes\n assert specs_with_test_deps[\"dep_test_dep\"].dag_hash() in graph.nodes\n\n def test_mark_explicit_spec_excludes_build_only_deps(\n self, specs_with_build_deps: Dict[str, Spec], temporary_store: Store\n ):\n \"\"\"An installed-implicit spec in explicit_set should only traverse link/run deps,\n not build-only deps.\"\"\"\n root = specs_with_build_deps[\"root\"]\n install_spec_in_db(root, temporary_store)\n assert temporary_store.db._data[root.dag_hash()].explicit is False\n graph = BuildGraph(\n specs=[root],\n root_policy=\"auto\",\n dependencies_policy=\"auto\",\n include_build_deps=True,\n install_package=True,\n install_deps=True,\n database=temporary_store.db,\n explicit_set={root.dag_hash()},\n )\n # root should be in graph (not pruned) because it needs to be marked explicit.\n assert root.dag_hash() in graph.nodes\n # build-only dep should NOT be pulled in since root is already installed.\n assert specs_with_build_deps[\"build_dep\"].dag_hash() not in graph.nodes\n" }, { "path": "lib/spack/spack/test/installer_tui.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Tests for the BuildStatus terminal UI in new_installer.py\"\"\"\n\nimport sys\n\nimport pytest\n\nif sys.platform == \"win32\":\n pytest.skip(\"No Windows support\", allow_module_level=True)\n\n\nimport io\nimport os\nfrom multiprocessing import Pipe\nfrom typing import List, Optional, Tuple\n\nimport spack.new_installer as inst\nfrom spack.new_installer import BuildStatus, StdinReader\n\n\nclass MockConnection:\n \"\"\"Mock multiprocessing.Connection for testing\"\"\"\n\n def fileno(self):\n return -1\n\n\nclass MockSpec:\n \"\"\"Minimal mock for spack.spec.Spec\"\"\"\n\n def __init__(\n self, name: str, version: str = \"1.0\", external: bool = False, prefix: Optional[str] = None\n ) -> None:\n self.name = name\n self.version = version\n self.external = external\n self.prefix = prefix or f\"/fake/prefix/{name}\"\n self._hash = name # Simple hash based on name\n\n def dag_hash(self, length: Optional[int] = None) -> str:\n if length:\n return self._hash[:length]\n return self._hash\n\n\nclass SimpleTextIOWrapper(io.TextIOWrapper):\n \"\"\"TextIOWrapper around a BytesIO buffer for testing of stdout behavior\"\"\"\n\n def __init__(self, tty: bool) -> None:\n self._buffer = io.BytesIO()\n self._tty = tty\n super().__init__(self._buffer, encoding=\"utf-8\", line_buffering=True)\n\n def isatty(self) -> bool:\n return self._tty\n\n def getvalue(self) -> str:\n self.flush()\n return self._buffer.getvalue().decode(\"utf-8\")\n\n def clear(self):\n self.flush()\n self._buffer.truncate(0)\n self._buffer.seek(0)\n\n\ndef create_build_status(\n is_tty: bool = True,\n terminal_cols: int = 80,\n terminal_rows: int = 24,\n total: int = 0,\n verbose: bool = False,\n filter_padding: bool = False,\n color: Optional[bool] = None,\n) -> Tuple[BuildStatus, List[float], SimpleTextIOWrapper]:\n \"\"\"Helper function to create BuildStatus with mocked dependencies\"\"\"\n fake_stdout = SimpleTextIOWrapper(tty=is_tty)\n # Easy way to set the current time in tests before running UI updates\n time_values = [0.0]\n\n def mock_get_time():\n return time_values[-1]\n\n def mock_get_terminal_size():\n return os.terminal_size((terminal_cols, terminal_rows))\n\n status = BuildStatus(\n total=total,\n stdout=fake_stdout,\n get_terminal_size=mock_get_terminal_size,\n get_time=mock_get_time,\n is_tty=is_tty,\n verbose=verbose,\n filter_padding=filter_padding,\n color=color,\n )\n\n return status, time_values, fake_stdout\n\n\ndef add_mock_builds(status: BuildStatus, count: int) -> List[MockSpec]:\n \"\"\"Helper function to add builds to a BuildStatus instance\"\"\"\n specs = [MockSpec(f\"pkg{i}\", f\"{i}.0\") for i in range(count)]\n for spec in specs:\n status.add_build(spec, explicit=True, control_w_conn=MockConnection()) # type: ignore\n return specs\n\n\nclass TestBasicStateManagement:\n \"\"\"Test basic state management operations\"\"\"\n\n def test_on_resize(self):\n \"\"\"Test that on_resize sets terminal_size_changed and update() fetches lazily\"\"\"\n sizes = [os.terminal_size((80, 24))]\n fake_stdout = SimpleTextIOWrapper(tty=True)\n status = BuildStatus(\n total=0, stdout=fake_stdout, get_terminal_size=lambda: sizes[-1], is_tty=True\n )\n # terminal_size_changed is True from __init__; terminal_size is placeholder\n assert status.terminal_size_changed is True\n\n # After on_resize the flag stays set and dirty is True\n sizes.append(os.terminal_size((120, 40)))\n status.on_resize()\n assert status.terminal_size_changed is True\n assert status.dirty is True\n\n # The actual size is fetched lazily on the first update()\n status.update()\n assert status.terminal_size == os.terminal_size((120, 40))\n assert status.terminal_size_changed is False\n\n def test_add_build(self):\n \"\"\"Test that add_build adds builds correctly\"\"\"\n status, _, _ = create_build_status(total=2)\n spec1 = MockSpec(\"pkg1\", \"1.0\")\n spec2 = MockSpec(\"pkg2\", \"2.0\")\n\n status.add_build(spec1, explicit=True, control_w_conn=MockConnection())\n assert len(status.builds) == 1\n assert spec1.dag_hash() in status.builds\n assert status.builds[spec1.dag_hash()].name == \"pkg1\"\n assert status.builds[spec1.dag_hash()].explicit is True\n assert status.dirty is True\n\n status.add_build(spec2, explicit=False, control_w_conn=MockConnection())\n assert len(status.builds) == 2\n assert spec2.dag_hash() in status.builds\n assert status.builds[spec2.dag_hash()].explicit is False\n\n def test_update_state_transitions(self):\n \"\"\"Test that update_state transitions states properly\"\"\"\n status, fake_time, _ = create_build_status()\n (spec,) = add_mock_builds(status, 1)\n build_id = spec.dag_hash()\n\n # Update to 'building' state\n status.update_state(build_id, \"building\")\n assert status.builds[build_id].state == \"building\"\n assert status.builds[build_id].progress_percent is None\n assert status.completed == 0\n\n # Update to 'finished' state\n status.update_state(build_id, \"finished\")\n assert status.builds[build_id].state == \"finished\"\n assert status.completed == 1\n assert status.builds[build_id].finished_time == fake_time[0] + inst.CLEANUP_TIMEOUT\n\n def test_update_state_failed(self):\n \"\"\"Test that failed state increments completed counter\"\"\"\n status, fake_time, _ = create_build_status()\n (spec,) = add_mock_builds(status, 1)\n build_id = spec.dag_hash()\n\n status.update_state(build_id, \"failed\")\n assert status.builds[build_id].state == \"failed\"\n assert status.completed == 1\n assert status.builds[build_id].finished_time == fake_time[0] + inst.CLEANUP_TIMEOUT\n\n def test_parse_log_summary(self, tmp_path):\n \"\"\"Test that parse_log_summary parses the build log and stores the summary.\"\"\"\n status, _, _ = create_build_status()\n (spec,) = add_mock_builds(status, 1)\n build_id = spec.dag_hash()\n\n # Create a fake log file with an error\n log_file = tmp_path / \"build.log\"\n log_file.write_text(\"error: something went wrong\\n\")\n\n status.builds[build_id].log_path = str(log_file)\n status.parse_log_summary(build_id)\n assert status.builds[build_id].log_summary is not None\n assert \"error\" in status.builds[build_id].log_summary.lower()\n\n def test_parse_log_summary_no_log_path(self):\n \"\"\"Test that parse_log_summary is a no-op when log_path is not set.\"\"\"\n status, _, _ = create_build_status()\n (spec,) = add_mock_builds(status, 1)\n build_id = spec.dag_hash()\n\n status.parse_log_summary(build_id)\n assert status.builds[build_id].log_summary is None\n\n def test_parse_log_summary_missing_file(self, tmp_path):\n \"\"\"Test that parse_log_summary is a no-op when log file doesn't exist.\"\"\"\n status, _, _ = create_build_status()\n (spec,) = add_mock_builds(status, 1)\n build_id = spec.dag_hash()\n\n status.builds[build_id].log_path = str(tmp_path / \"nonexistent.log\")\n status.parse_log_summary(build_id)\n assert status.builds[build_id].log_summary is None\n\n def test_update_progress(self):\n \"\"\"Test that update_progress updates percentages\"\"\"\n status, _, _ = create_build_status()\n (spec,) = add_mock_builds(status, 1)\n build_id = spec.dag_hash()\n\n # Update progress\n status.update_progress(build_id, 50, 100)\n assert status.builds[build_id].progress_percent == 50\n assert status.dirty is True\n\n # Same percentage shouldn't mark dirty again\n status.dirty = False\n status.update_progress(build_id, 50, 100)\n assert status.dirty is False\n\n # Different percentage should mark dirty\n status.update_progress(build_id, 75, 100)\n assert status.builds[build_id].progress_percent == 75\n assert status.dirty is True\n\n def test_completion_counter(self):\n \"\"\"Test that completion counter increments correctly\"\"\"\n status, _, _ = create_build_status(total=3)\n specs = add_mock_builds(status, 3)\n\n assert status.completed == 0\n\n status.update_state(specs[0].dag_hash(), \"finished\")\n assert status.completed == 1\n\n status.update_state(specs[1].dag_hash(), \"failed\")\n assert status.completed == 2\n\n status.update_state(specs[2].dag_hash(), \"finished\")\n assert status.completed == 3\n\n\nclass TestOutputRendering:\n \"\"\"Test output rendering for TTY and non-TTY modes\"\"\"\n\n def test_non_tty_output(self):\n \"\"\"Test that non-TTY mode prints simple state changes\"\"\"\n status, _, fake_stdout = create_build_status(is_tty=False)\n spec = MockSpec(\"mypackage\", \"1.0\")\n\n status.add_build(spec, explicit=True, control_w_conn=MockConnection())\n build_id = spec.dag_hash()\n\n status.update_state(build_id, \"finished\")\n\n output = fake_stdout.getvalue()\n assert \"[+]\" in output\n assert \"mypackage\" in output\n assert \"1.0\" in output\n assert \"/fake/prefix/mypackage\" in output # prefix is shown for finished builds\n # Non-TTY output should not contain ANSI escape codes\n assert \"\\033[\" not in output\n\n def test_tty_output_contains_ansi(self):\n \"\"\"Test that TTY mode produces ANSI codes\"\"\"\n status, _, fake_stdout = create_build_status()\n add_mock_builds(status, 1)\n\n # Call update to render\n status.update()\n\n output = fake_stdout.getvalue()\n # Should contain ANSI escape sequences\n assert \"\\033[\" in output\n # Should contain progress header\n assert \"Progress:\" in output\n\n def test_no_output_when_not_dirty(self):\n \"\"\"Test that update() skips rendering when not dirty\"\"\"\n status, _, fake_stdout = create_build_status()\n add_mock_builds(status, 1)\n status.update()\n\n # Clear stdout and mark not dirty\n fake_stdout.clear()\n status.dirty = False\n\n # Update should not produce output\n status.update()\n assert fake_stdout.getvalue() == \"\"\n\n def test_update_throttling(self):\n \"\"\"Test that update() throttles redraws\"\"\"\n status, fake_time, fake_stdout = create_build_status()\n add_mock_builds(status, 1)\n\n # First update at time 0\n fake_time[0] = 0.0\n status.update()\n first_output = fake_stdout.getvalue()\n assert first_output != \"\"\n\n # Mark dirty and try to update immediately\n fake_stdout.clear()\n status.dirty = True\n fake_time[0] = 0.01 # Very small time advance\n\n # Should be throttled (next_update not reached)\n status.update()\n assert fake_stdout.getvalue() == \"\"\n\n # Advance time past throttle and try again\n fake_time[0] = 1.0\n status.update()\n assert fake_stdout.getvalue() != \"\"\n\n def test_cursor_movement_vs_newlines(self):\n \"\"\"Test that finished builds get newlines, active builds get cursor movements\"\"\"\n status, fake_time, fake_stdout = create_build_status(total=5)\n specs = add_mock_builds(status, 3)\n\n # First update renders 3 active builds\n fake_time[0] = 0.0\n status.update()\n output1 = fake_stdout.getvalue()\n\n # Count newlines (\\n) and cursor movements (\\033[1B\\r = move down 1 line)\n newlines1 = output1.count(\"\\n\")\n cursor_moves1 = output1.count(\"\\033[1B\\r\")\n\n # Initially all lines should be newlines (nothing in history yet)\n assert newlines1 > 0\n assert cursor_moves1 == 0\n\n # Now finish 2 builds and add 2 more\n fake_stdout.clear()\n fake_time[0] = inst.CLEANUP_TIMEOUT + 0.1\n status.update_state(specs[0].dag_hash(), \"finished\")\n status.update_state(specs[1].dag_hash(), \"finished\")\n\n spec4 = MockSpec(\"pkg3\", \"3.0\")\n spec5 = MockSpec(\"pkg4\", \"4.0\")\n status.add_build(spec4, explicit=True, control_w_conn=MockConnection())\n status.add_build(spec5, explicit=True, control_w_conn=MockConnection())\n\n # Second update: finished builds persist (newlines), active area updates (cursor moves)\n status.update()\n output2 = fake_stdout.getvalue()\n\n newlines2 = output2.count(\"\\n\")\n cursor_moves2 = output2.count(\"\\033[1B\\r\")\n\n # Should have newlines for the 2 finished builds persisted to history\n # and cursor movements for the active area (header + 3 active builds)\n assert newlines2 > 0, \"Should have newlines for finished builds\"\n assert cursor_moves2 > 0, \"Should have cursor movements for active area\"\n\n # Finished builds should be printed with newlines\n assert \"pkg0\" in output2\n assert \"pkg1\" in output2\n\n\nclass TestTimeBasedBehavior:\n \"\"\"Test time-based behaviors like spinner and cleanup\"\"\"\n\n def test_spinner_updates(self):\n \"\"\"Test that spinner advances over time\"\"\"\n status, fake_time, _ = create_build_status()\n add_mock_builds(status, 1)\n\n # Initial spinner index\n initial_index = status.spinner_index\n\n # Advance time past spinner interval\n fake_time[0] = inst.SPINNER_INTERVAL + 0.01\n status.update()\n\n # Spinner should have advanced\n assert status.spinner_index == (initial_index + 1) % len(status.spinner_chars)\n\n def test_finished_package_cleanup(self):\n \"\"\"Test that finished packages are cleaned up after timeout\"\"\"\n status, fake_time, _ = create_build_status()\n (spec,) = add_mock_builds(status, 1)\n build_id = spec.dag_hash()\n\n # Mark as finished\n fake_time[0] = 0.0\n status.update_state(build_id, \"finished\")\n\n # Build should still be in active builds\n assert build_id in status.builds\n assert len(status.finished_builds) == 0\n\n # Advance time past cleanup timeout\n fake_time[0] = inst.CLEANUP_TIMEOUT + 0.01\n status.update()\n\n # Build should now be moved to finished_builds and removed from active\n assert build_id not in status.builds\n # Note: finished_builds is cleared after rendering, so check it happened via side effects\n assert status.dirty or build_id not in status.builds\n\n def test_failed_packages_not_cleaned_up(self):\n \"\"\"Test that failed packages stay in active builds\"\"\"\n status, fake_time, _ = create_build_status()\n (spec,) = add_mock_builds(status, 1)\n build_id = spec.dag_hash()\n\n # Mark as failed\n fake_time[0] = 0.0\n status.update_state(build_id, \"failed\")\n\n # Advance time past cleanup timeout\n fake_time[0] = inst.CLEANUP_TIMEOUT + 0.01\n status.update()\n\n # Failed build should remain in active builds\n assert build_id in status.builds\n\n\nclass TestSearchAndFilter:\n \"\"\"Test search mode and filtering\"\"\"\n\n def test_enter_search_mode(self):\n \"\"\"Test that enter_search enables search mode\"\"\"\n status, _, _ = create_build_status()\n assert status.search_mode is False\n\n status.enter_search()\n assert status.search_mode is True\n assert status.dirty is True\n\n def test_search_input_printable(self):\n \"\"\"Test that printable characters are added to search term\"\"\"\n status, _, _ = create_build_status()\n status.enter_search()\n\n status.search_input(\"a\")\n assert status.search_term == \"a\"\n\n status.search_input(\"b\")\n assert status.search_term == \"ab\"\n\n status.search_input(\"c\")\n assert status.search_term == \"abc\"\n\n def test_search_input_backspace(self):\n \"\"\"Test that backspace removes characters\"\"\"\n status, _, _ = create_build_status()\n status.enter_search()\n\n status.search_input(\"a\")\n status.search_input(\"b\")\n status.search_input(\"c\")\n assert status.search_term == \"abc\"\n\n status.search_input(\"\\x7f\") # Backspace\n assert status.search_term == \"ab\"\n\n status.search_input(\"\\b\") # Alternative backspace\n assert status.search_term == \"a\"\n\n def test_search_input_escape(self):\n \"\"\"Test that escape exits search mode\"\"\"\n status, _, _ = create_build_status()\n status.enter_search()\n status.search_input(\"test\")\n\n status.search_input(\"\\x1b\") # Escape\n assert status.search_mode is False\n assert status.search_term == \"\"\n\n def test_is_displayed_filters_by_name(self):\n \"\"\"Test that _is_displayed filters by package name\"\"\"\n status, _, _ = create_build_status(total=3)\n\n spec1 = MockSpec(\"package-foo\", \"1.0\")\n spec2 = MockSpec(\"package-bar\", \"1.0\")\n spec3 = MockSpec(\"other\", \"1.0\")\n\n status.add_build(spec1, explicit=True, control_w_conn=MockConnection())\n status.add_build(spec2, explicit=True, control_w_conn=MockConnection())\n status.add_build(spec3, explicit=True, control_w_conn=MockConnection())\n\n build1 = status.builds[spec1.dag_hash()]\n build2 = status.builds[spec2.dag_hash()]\n build3 = status.builds[spec3.dag_hash()]\n\n # No search term: all displayed\n status.search_term = \"\"\n assert status._is_displayed(build1)\n assert status._is_displayed(build2)\n assert status._is_displayed(build3)\n\n # Search for \"package\"\n status.search_term = \"package\"\n assert status._is_displayed(build1)\n assert status._is_displayed(build2)\n assert not status._is_displayed(build3)\n\n # Search for \"foo\"\n status.search_term = \"foo\"\n assert status._is_displayed(build1)\n assert not status._is_displayed(build2)\n assert not status._is_displayed(build3)\n\n def test_is_displayed_filters_by_hash(self):\n \"\"\"Test that _is_displayed filters by hash prefix\"\"\"\n status, _, _ = create_build_status(total=2)\n\n spec1 = MockSpec(\"pkg1\", \"1.0\")\n spec1._hash = \"abc123\"\n spec2 = MockSpec(\"pkg2\", \"1.0\")\n spec2._hash = \"def456\"\n\n status.add_build(spec1, explicit=True, control_w_conn=MockConnection())\n status.add_build(spec2, explicit=True, control_w_conn=MockConnection())\n\n build1 = status.builds[spec1.dag_hash()]\n build2 = status.builds[spec2.dag_hash()]\n\n # Search by hash prefix\n status.search_term = \"abc\"\n assert status._is_displayed(build1)\n assert not status._is_displayed(build2)\n\n status.search_term = \"def\"\n assert not status._is_displayed(build1)\n assert status._is_displayed(build2)\n\n\nclass TestNavigation:\n \"\"\"Test navigation between builds\"\"\"\n\n def test_get_next_basic(self):\n \"\"\"Test basic next/previous navigation\"\"\"\n status, _, _ = create_build_status(total=3)\n specs = add_mock_builds(status, 3)\n\n # Get first build\n first_id = status._get_next(1)\n assert first_id == specs[0].dag_hash()\n\n # Set tracked and get next\n status.tracked_build_id = first_id\n next_id = status._get_next(1)\n assert next_id == specs[1].dag_hash()\n\n # Get next again\n status.tracked_build_id = next_id\n next_id = status._get_next(1)\n assert next_id == specs[2].dag_hash()\n\n # Wrap around\n status.tracked_build_id = next_id\n next_id = status._get_next(1)\n assert next_id == specs[0].dag_hash()\n\n def test_get_next_previous(self):\n \"\"\"Test backward navigation\"\"\"\n status, _, _ = create_build_status(total=3)\n specs = add_mock_builds(status, 3)\n\n # Start at second build\n status.tracked_build_id = specs[1].dag_hash()\n\n # Go backward\n prev_id = status._get_next(-1)\n assert prev_id == specs[0].dag_hash()\n\n # Go backward again (wrap around)\n status.tracked_build_id = prev_id\n prev_id = status._get_next(-1)\n assert prev_id == specs[2].dag_hash()\n\n def test_get_next_with_filter(self):\n \"\"\"Test navigation respects search filter\"\"\"\n status, _, _ = create_build_status(total=4)\n\n specs = [\n MockSpec(\"package-a\", \"1.0\"),\n MockSpec(\"package-b\", \"1.0\"),\n MockSpec(\"other-c\", \"1.0\"),\n MockSpec(\"package-d\", \"1.0\"),\n ]\n for spec in specs:\n status.add_build(spec, explicit=True, control_w_conn=MockConnection())\n\n # Filter to only \"package-*\"\n status.search_term = \"package\"\n\n # Should only navigate through matching builds\n first_id = status._get_next(1)\n assert first_id and first_id == specs[0].dag_hash()\n\n status.tracked_build_id = first_id\n next_id = status._get_next(1)\n assert next_id and next_id == specs[1].dag_hash()\n\n status.tracked_build_id = next_id\n next_id = status._get_next(1)\n # Should skip \"other-c\" and go to \"package-d\"\n assert next_id and next_id == specs[3].dag_hash()\n\n def test_get_next_skips_finished(self):\n \"\"\"Test that navigation skips finished builds\"\"\"\n status, _, _ = create_build_status(total=3)\n specs = add_mock_builds(status, 3)\n\n # Mark middle build as finished\n status.update_state(specs[1].dag_hash(), \"finished\")\n\n # Navigate from first\n status.tracked_build_id = specs[0].dag_hash()\n next_id = status._get_next(1)\n # Should skip finished build and go to third\n assert next_id == specs[2].dag_hash()\n\n def test_get_next_no_matching(self):\n \"\"\"Test that _get_next returns None when no builds match\"\"\"\n status, _, _ = create_build_status(total=2)\n specs = add_mock_builds(status, 2)\n\n # Mark both as finished\n for spec in specs:\n status.update_state(spec.dag_hash(), \"finished\")\n\n # Should return None since no unfinished builds\n result = status._get_next(1)\n assert result is None\n\n def test_get_next_fallback_when_tracked_filtered_out(self):\n \"\"\"Test that _get_next falls back correctly when tracked build no longer matches filter\"\"\"\n status, _, _ = create_build_status(total=3)\n\n specs = [\n MockSpec(\"package-a\", \"1.0\"),\n MockSpec(\"package-b\", \"1.0\"),\n MockSpec(\"other-c\", \"1.0\"),\n ]\n for spec in specs:\n status.add_build(spec, explicit=True, control_w_conn=MockConnection())\n\n # Start tracking \"other-c\"\n status.tracked_build_id = specs[2].dag_hash()\n\n # Now apply a filter that excludes the tracked build\n status.search_term = \"package\"\n\n # _get_next should fall back to first matching build (forward)\n next_id = status._get_next(1)\n assert next_id == specs[0].dag_hash()\n\n # Test backward direction, should fall back to last matching build\n status.tracked_build_id = specs[2].dag_hash() # Reset to filtered-out build\n prev_id = status._get_next(-1)\n assert prev_id == specs[1].dag_hash()\n\n\nclass TestTerminalSizes:\n \"\"\"Test behavior with different terminal sizes\"\"\"\n\n def test_small_terminal_truncation(self):\n \"\"\"Test that output is truncated for small terminals\"\"\"\n status, _, fake_stdout = create_build_status(total=10, terminal_cols=80, terminal_rows=10)\n\n # Add more builds than can fit on screen\n add_mock_builds(status, 10)\n\n status.update()\n output = fake_stdout.getvalue()\n\n # Should contain \"more...\" message indicating truncation\n assert \"more...\" in output\n\n def test_large_terminal_no_truncation(self):\n \"\"\"Test that all builds shown on large terminal\"\"\"\n status, _, fake_stdout = create_build_status(total=3, terminal_cols=120)\n add_mock_builds(status, 3)\n\n status.update()\n output = fake_stdout.getvalue()\n\n # Should not contain truncation message\n assert \"more...\" not in output\n # Should contain all package names\n for i in range(3):\n assert f\"pkg{i}\" in output\n\n def test_narrow_terminal_short_header(self):\n \"\"\"Test that narrow terminals get shortened header\"\"\"\n status, _, fake_stdout = create_build_status(total=1, terminal_cols=40)\n add_mock_builds(status, 1)\n\n status.update()\n output = fake_stdout.getvalue()\n\n # Should not contain the full header with hints\n assert \"filter\" not in output\n # But should contain progress\n assert \"Progress:\" in output\n\n\nclass TestBuildInfo:\n \"\"\"Test the BuildInfo dataclass\"\"\"\n\n def test_build_info_creation(self):\n \"\"\"Test that BuildInfo is created correctly\"\"\"\n spec = MockSpec(\"mypackage\", \"1.0\")\n\n build_info = inst.BuildInfo(spec, explicit=True, control_w_conn=MockConnection())\n\n assert build_info.name == \"mypackage\"\n assert build_info.version == \"1.0\"\n assert build_info.explicit is True\n assert build_info.external is False\n assert build_info.state == \"starting\"\n assert build_info.finished_time is None\n assert build_info.progress_percent is None\n\n def test_build_info_external_package(self):\n \"\"\"Test BuildInfo for external package\"\"\"\n spec = MockSpec(\"external-pkg\", \"1.0\", external=True)\n\n build_info = inst.BuildInfo(spec, explicit=False, control_w_conn=MockConnection())\n\n assert build_info.external is True\n\n\nclass TestLogFollowing:\n \"\"\"Test log following and print_logs functionality\"\"\"\n\n def test_print_logs_when_following(self):\n \"\"\"Test that logs are printed when following a specific build\"\"\"\n status, _, fake_stdout = create_build_status()\n (spec,) = add_mock_builds(status, 1)\n build_id = spec.dag_hash()\n\n # Switch to log-following mode\n status.overview_mode = False\n status.tracked_build_id = build_id\n\n # Send some log data\n log_data = b\"Building package...\\nRunning tests...\\n\"\n status.print_logs(build_id, log_data)\n\n # Check that logs were echoed to stdout\n assert fake_stdout._buffer.getvalue() == log_data\n\n def test_print_logs_discarded_when_in_overview_mode(self):\n \"\"\"Test that logs are discarded when in overview mode\"\"\"\n status, _, fake_stdout = create_build_status()\n (spec,) = add_mock_builds(status, 1)\n build_id = spec.dag_hash()\n\n # Stay in overview mode\n assert status.overview_mode is True\n\n # Try to print logs\n log_data = b\"Should not be printed\\n\"\n status.print_logs(build_id, log_data)\n\n # Nothing should be printed\n assert fake_stdout.getvalue() == \"\"\n\n def test_print_logs_discarded_when_not_tracked(self):\n \"\"\"Test that logs from non-tracked builds are discarded\"\"\"\n status, _, fake_stdout = create_build_status(total=2)\n spec1, spec2 = add_mock_builds(status, 2)\n\n # Switch to log-following mode for spec1\n status.overview_mode = False\n status.tracked_build_id = spec1.dag_hash()\n\n # Try to print logs from spec2 (not tracked)\n log_data = b\"Logs from pkg2\\n\"\n status.print_logs(spec2.dag_hash(), log_data)\n\n # Nothing should be printed since we're tracking pkg1, not pkg2\n assert fake_stdout.getvalue() == \"\"\n\n def test_can_navigate_to_failed_build(self):\n \"\"\"Test that navigating to a failed build shows log summary and path\"\"\"\n status, _, fake_stdout = create_build_status(total=3)\n specs = add_mock_builds(status, 3)\n\n # Mark the middle build as failed and set log info\n status.update_state(specs[1].dag_hash(), \"failed\")\n build_info = status.builds[specs[1].dag_hash()]\n build_info.log_summary = \"Error: something went wrong\\n\"\n build_info.log_path = \"/tmp/spack/pkg1.log\"\n\n # Navigate from pkg0 to next -- should land on failed pkg1\n status.tracked_build_id = specs[0].dag_hash()\n next_id = status._get_next(1)\n assert next_id == specs[1].dag_hash()\n\n # Actually navigate to it\n status.next(1)\n output = fake_stdout.getvalue()\n assert \"Log summary of pkg1\" in output\n assert \"Error: something went wrong\" in output\n assert \"/tmp/spack/pkg1.log\" in output\n\n def test_navigation_skips_finished_build(self):\n \"\"\"Test that navigation skips successfully finished builds\"\"\"\n status, _, _ = create_build_status(total=3)\n specs = add_mock_builds(status, 3)\n\n # Mark the middle build as finished (successful)\n status.update_state(specs[1].dag_hash(), \"finished\")\n\n # Try to get next build, should skip the finished one\n status.tracked_build_id = specs[0].dag_hash()\n next_id = status._get_next(1)\n\n assert next_id == specs[2].dag_hash()\n\n\nclass TestNavigationIntegration:\n \"\"\"Test the next() method and navigation between builds\"\"\"\n\n def test_next_switches_from_overview_to_logs(self):\n \"\"\"Test that next() switches from overview mode to log-following mode\"\"\"\n status, _, fake_stdout = create_build_status(total=2)\n specs = add_mock_builds(status, 2)\n\n # Start in overview mode\n assert status.overview_mode is True\n assert status.tracked_build_id == \"\"\n\n # Call next() to start following first build\n status.next()\n\n # Should have switched to log-following mode\n assert status.overview_mode is False\n assert status.tracked_build_id == specs[0].dag_hash()\n\n # Should have printed \"Following logs\" message\n output = fake_stdout.getvalue()\n assert \"Following logs of\" in output\n assert \"pkg0\" in output\n\n def test_next_cycles_through_builds(self):\n \"\"\"Test that next() cycles through multiple builds\"\"\"\n status, _, fake_stdout = create_build_status(total=3)\n specs = add_mock_builds(status, 3)\n\n # Start following first build\n status.next()\n assert status.tracked_build_id == specs[0].dag_hash()\n\n fake_stdout.clear()\n\n # Navigate to next\n status.next(1)\n assert status.tracked_build_id == specs[1].dag_hash()\n assert \"pkg1\" in fake_stdout.getvalue()\n\n fake_stdout.clear()\n\n # Navigate to next (third build)\n status.next(1)\n assert status.tracked_build_id == specs[2].dag_hash()\n assert \"pkg2\" in fake_stdout.getvalue()\n\n fake_stdout.clear()\n\n # Navigate to next (should wrap to first)\n status.next(1)\n assert status.tracked_build_id == specs[0].dag_hash()\n assert \"pkg0\" in fake_stdout.getvalue()\n\n def test_next_backward_navigation(self):\n \"\"\"Test that next(-1) navigates backward\"\"\"\n status, _, _ = create_build_status(total=3)\n specs = add_mock_builds(status, 3)\n\n # Start at first build\n status.next()\n assert status.tracked_build_id == specs[0].dag_hash()\n\n # Go backward (should wrap to last)\n status.next(-1)\n assert status.tracked_build_id == specs[2].dag_hash()\n\n # Go backward again\n status.next(-1)\n assert status.tracked_build_id == specs[1].dag_hash()\n\n def test_next_does_nothing_when_no_builds(self):\n \"\"\"Test that next() does nothing when no unfinished builds exist\"\"\"\n status, _, _ = create_build_status(total=1)\n (spec,) = add_mock_builds(status, 1)\n\n # Mark as finished\n status.update_state(spec.dag_hash(), \"finished\")\n\n # Try to navigate\n initial_mode = status.overview_mode\n initial_tracked = status.tracked_build_id\n\n status.next()\n\n # Nothing should change\n assert status.overview_mode == initial_mode\n assert status.tracked_build_id == initial_tracked\n\n def test_next_does_nothing_when_same_build(self):\n \"\"\"Test that next() doesn't re-print when already on the same build\"\"\"\n status, _, fake_stdout = create_build_status(total=1)\n (spec,) = add_mock_builds(status, 1)\n\n # Start following\n status.next()\n assert status.tracked_build_id == spec.dag_hash()\n\n # Clear output\n fake_stdout.clear()\n\n # Try to navigate to \"next\" (which is the same build)\n status.next()\n\n # Should not print anything\n assert fake_stdout.getvalue() == \"\"\n\n\nclass TestToggle:\n \"\"\"Test toggle() method for switching between overview and log-following modes\"\"\"\n\n def test_toggle_from_overview_calls_next(self):\n \"\"\"Test that toggle() from overview mode calls next()\"\"\"\n status, _, fake_stdout = create_build_status(total=2)\n add_mock_builds(status, 2)\n\n # Start in overview mode\n assert status.overview_mode is True\n\n # Toggle should call next()\n status.toggle()\n\n # Should now be following logs\n assert status.overview_mode is False\n assert status.tracked_build_id != \"\"\n assert \"Following logs of\" in fake_stdout.getvalue()\n\n def test_toggle_from_logs_returns_to_overview(self):\n \"\"\"Test that toggle() from log-following mode returns to overview\"\"\"\n status, _, _ = create_build_status(total=2)\n add_mock_builds(status, 2)\n\n # Switch to log-following mode first\n status.next()\n assert status.overview_mode is False\n tracked_id = status.tracked_build_id\n assert tracked_id != \"\"\n\n # Set some search state to verify cleanup\n status.search_term = \"test\"\n status.search_mode = True\n status.active_area_rows = 5\n\n # Toggle back to overview\n status.toggle()\n\n # Should be back in overview mode with cleaned state\n assert status.overview_mode is True\n assert status.tracked_build_id == \"\"\n assert status.search_term == \"\"\n assert status.search_mode is False\n assert status.active_area_rows == 0\n assert status.dirty is True\n\n def test_update_state_finished_triggers_toggle_when_tracking(self):\n \"\"\"Test that finishing a tracked build triggers toggle back to overview\"\"\"\n status, _, _ = create_build_status(total=2)\n specs = add_mock_builds(status, 2)\n\n # Start tracking first build\n status.next()\n assert status.overview_mode is False\n assert status.tracked_build_id == specs[0].dag_hash()\n\n # Mark the tracked build as finished\n status.update_state(specs[0].dag_hash(), \"finished\")\n\n # Should have toggled back to overview mode\n assert status.overview_mode is True\n assert status.tracked_build_id == \"\"\n\n def test_partial_line_newline_on_toggle_and_next(self):\n \"\"\"Ensure newline is inserted before mode transitions when log doesn't end with newline.\"\"\"\n status, _, fake_stdout = create_build_status(total=2)\n specs = add_mock_builds(status, 2)\n build_a, build_b = specs[0].dag_hash(), specs[1].dag_hash()\n\n # Follow a build, toggle back and forth between logs and overview mode, and receive logs\n # that may or may not end with newlines.\n status.next()\n status.print_logs(build_a, b\"checking for foo...\")\n status.toggle()\n status.next()\n status.print_logs(build_a, b\"checking for bar... yes\\n\")\n status.next(1)\n status.print_logs(build_b, b\"checking for baz...\")\n status.next(-1)\n\n written = fake_stdout.getvalue()\n\n # There shouldn't be any double newlines:\n assert \"\\n\\n\" not in written\n\n # All partial and newline-terminated logs should be present with appropriate newlines:\n assert \"checking for foo...\\n\" in written\n assert \"checking for bar... yes\\n\" in written\n assert \"checking for baz...\\n\" in written\n\n @pytest.mark.parametrize(\"filter_padding\", [True, False])\n def test_print_logs_filters_padding(self, filter_padding):\n \"\"\"print_logs strips path-padding placeholders before writing to stdout.\"\"\"\n status, _, fake_stdout = create_build_status(filter_padding=filter_padding)\n (spec,) = add_mock_builds(status, 1)\n build_id = spec.dag_hash()\n log_output = b\"--with-foo=/base/__spack_path_placeholder__/__spack_path_placeholder__/bin\"\n\n # track the build and print logs with the relevant path.\n status.overview_mode = False\n status.tracked_build_id = build_id\n status.print_logs(build_id, log_output)\n written = fake_stdout._buffer.getvalue()\n\n if filter_padding:\n assert written == b\"--with-foo=/base/[padded-to-59-chars]/bin\"\n else:\n assert written == log_output\n\n @pytest.mark.parametrize(\"filter_padding\", [True, False])\n def test_prefix_padding_filter_in_status(self, filter_padding):\n \"\"\"Test that prefix in status indicator applies padding filter.\"\"\"\n padded_prefix = \"/base/__spack_path_placeholder__/__spack_path_placeholder__/mypackage\"\n status, _, fake_stdout = create_build_status(is_tty=False, filter_padding=filter_padding)\n spec = MockSpec(\"mypackage\", \"1.0\", prefix=padded_prefix)\n status.add_build(spec, explicit=True, control_w_conn=MockConnection())\n build_id = spec.dag_hash()\n status.update_state(build_id, \"finished\")\n output = fake_stdout.getvalue()\n common = f\"[+] {spec.dag_hash(7)} {spec.name}@{spec.version}\"\n if filter_padding:\n assert output == f\"{common} /base/[padded-to-59-chars]/mypackage\\n\"\n else:\n assert output == f\"{common} {padded_prefix}\\n\"\n\n\nclass TestSearchFilteringIntegration:\n \"\"\"Test search mode with display filtering\"\"\"\n\n def test_search_mode_filters_displayed_builds(self):\n \"\"\"Test that search mode actually filters what's displayed\"\"\"\n status, _, fake_stdout = create_build_status(total=4)\n\n specs = [\n MockSpec(\"package-foo\", \"1.0\"),\n MockSpec(\"package-bar\", \"2.0\"),\n MockSpec(\"other-thing\", \"3.0\"),\n MockSpec(\"package-baz\", \"4.0\"),\n ]\n for spec in specs:\n status.add_build(spec, explicit=True, control_w_conn=MockConnection())\n\n # Enter search mode and search for \"package\"\n status.enter_search()\n assert status.search_mode is True\n\n for character in \"package\":\n status.search_input(character)\n\n assert status.search_term == \"package\"\n\n # Update to render\n status.update()\n output = fake_stdout.getvalue()\n\n # Should contain filtered builds\n assert \"package-foo\" in output\n assert \"package-bar\" in output\n assert \"package-baz\" in output\n # Should not contain the filtered-out build\n assert \"other-thing\" not in output\n\n # Should show filter prompt\n assert \"filter>\" in output\n assert status.search_term in output\n\n def test_search_mode_with_navigation(self):\n \"\"\"Test that navigation respects search filter\"\"\"\n status, _, _ = create_build_status(total=4)\n\n specs = [\n MockSpec(\"package-a\", \"1.0\"),\n MockSpec(\"other-b\", \"2.0\"),\n MockSpec(\"package-c\", \"3.0\"),\n MockSpec(\"other-d\", \"4.0\"),\n ]\n for spec in specs:\n status.add_build(spec, explicit=True, control_w_conn=MockConnection())\n\n # Set search term to filter for \"package\"\n status.search_term = \"package\"\n\n # Start navigating, should only go through \"package-a\" and \"package-c\"\n status.next()\n assert status.tracked_build_id == specs[0].dag_hash() # package-a\n\n status.next(1)\n # Should skip other-b and go to package-c\n assert status.tracked_build_id == specs[2].dag_hash() # package-c\n\n status.next(1)\n # Should wrap around to package-a\n assert status.tracked_build_id == specs[0].dag_hash() # package-a\n\n def test_search_input_enter_navigates_to_next(self):\n \"\"\"Test that pressing enter in search mode navigates to next match\"\"\"\n status, _, _ = create_build_status(total=3)\n specs = add_mock_builds(status, 3)\n\n # Enter search mode\n status.enter_search()\n for character in \"pkg\":\n status.search_input(character)\n\n # Press enter (should navigate to first match)\n status.search_input(\"\\r\")\n\n # Should have started following first matching build\n assert status.overview_mode is False\n assert status.tracked_build_id == specs[0].dag_hash()\n\n def test_clearing_search_shows_all_builds(self):\n \"\"\"Test that clearing search term shows all builds again\"\"\"\n status, _, fake_stdout = create_build_status(total=3)\n\n specs = [\n MockSpec(\"package-a\", \"1.0\"),\n MockSpec(\"other-b\", \"2.0\"),\n MockSpec(\"package-c\", \"3.0\"),\n ]\n for spec in specs:\n status.add_build(spec, explicit=True, control_w_conn=MockConnection())\n\n # Enter search and type something\n status.enter_search()\n status.search_input(\"p\")\n status.search_input(\"a\")\n status.search_input(\"c\")\n assert status.search_term == \"pac\"\n\n # Clear it with backspace\n status.search_input(\"\\x7f\") # backspace\n status.search_input(\"\\x7f\") # backspace\n status.search_input(\"\\x7f\") # backspace\n assert status.search_term == \"\"\n\n # Update to render\n status.update()\n output = fake_stdout.getvalue()\n\n # All builds should be visible now\n assert \"package-a\" in output\n assert \"other-b\" in output\n assert \"package-c\" in output\n\n\nclass TestEdgeCases:\n \"\"\"Test edge cases and error conditions\"\"\"\n\n def test_empty_build_list(self):\n \"\"\"Test update with no builds\"\"\"\n status, _, fake_stdout = create_build_status(total=0)\n\n status.update()\n output = fake_stdout.getvalue()\n\n # Should render header but no builds\n assert \"Progress:\" in output\n assert \"0/0\" in output\n\n def test_no_header_with_finalize(self):\n \"\"\"Test that we don't print a header with finalize=True\"\"\"\n status, _, fake_stdout = create_build_status(total=2, color=False)\n spec_a, spec_b = add_mock_builds(status, 2)\n status.update_state(spec_a.dag_hash(), \"finished\")\n status.update_state(spec_b.dag_hash(), \"failed\")\n status.update(finalize=True)\n\n output = fake_stdout.getvalue()\n\n # Should not contain header\n assert \"Progress:\" not in output\n\n # Should contain final status lines for both builds\n assert f\"[+] {spec_a.dag_hash(7)} {spec_a.name}@{spec_a.version}\" in output\n assert f\"[x] {spec_b.dag_hash(7)} {spec_b.name}@{spec_b.version}\" in output\n\n def test_all_builds_finished(self):\n \"\"\"Test when all builds are finished\"\"\"\n status, fake_time, _ = create_build_status(total=2)\n specs = add_mock_builds(status, 2)\n\n # Mark all as finished\n for spec in specs:\n status.update_state(spec.dag_hash(), \"finished\")\n\n # Advance time and update\n fake_time[0] = inst.CLEANUP_TIMEOUT + 0.01\n status.update()\n\n # All should be cleaned up\n assert len(status.builds) == 0\n assert status.completed == 2\n\n def test_update_progress_rounds_correctly(self):\n \"\"\"Test that progress percentage rounding works\"\"\"\n status, _, _ = create_build_status()\n (spec,) = add_mock_builds(status, 1)\n build_id = spec.dag_hash()\n\n # Test rounding\n status.update_progress(build_id, 1, 3)\n assert status.builds[build_id].progress_percent == 33 # int(100/3)\n\n status.update_progress(build_id, 2, 3)\n assert status.builds[build_id].progress_percent == 66 # int(200/3)\n\n status.update_progress(build_id, 3, 3)\n assert status.builds[build_id].progress_percent == 100\n\n\nclass TestBuildStatusVerbose:\n \"\"\"Tests for verbose non-TTY log tracking in BuildStatus.\"\"\"\n\n def test_verbose_tracks_first_build(self):\n \"\"\"First add_build() in verbose non-TTY mode sets tracked_build_id and enables echoing.\"\"\"\n bs, _, _ = create_build_status(is_tty=False, verbose=True, total=4)\n spec = MockSpec(\"trivial-install-test-package\", \"1.0\")\n\n r_conn, w_conn = Pipe(duplex=False)\n\n with r_conn, w_conn:\n bs.add_build(spec, explicit=True, control_w_conn=w_conn)\n\n assert bs.tracked_build_id == spec.dag_hash()\n written = os.read(r_conn.fileno(), 1)\n assert written == b\"1\"\n\n def test_verbose_does_not_track_when_already_tracking(self):\n \"\"\"Second add_build() while already tracking does not switch tracking.\"\"\"\n bs, _, _ = create_build_status(is_tty=False, verbose=True, total=4)\n spec1 = MockSpec(\"pkg1\", \"1.0\")\n spec2 = MockSpec(\"pkg2\", \"1.0\")\n\n r1, w1 = Pipe(duplex=False)\n r2, w2 = Pipe(duplex=False)\n with r1, w1, r2, w2:\n bs.add_build(spec1, explicit=True, control_w_conn=w1)\n first_tracked = bs.tracked_build_id\n\n bs.add_build(spec2, explicit=False, control_w_conn=w2)\n assert bs.tracked_build_id == first_tracked\n assert bs.tracked_build_id == spec1.dag_hash()\n\n # Second build should not have received b\"1\"\n assert not r2.poll(), \"Second build should not be enabled\"\n\n def test_verbose_switches_on_finish(self):\n \"\"\"After the tracked build finishes, tracked_build_id is cleared.\"\"\"\n bs, _, _ = create_build_status(is_tty=False, verbose=True, total=4)\n spec = MockSpec(\"trivial-install-test-package\", \"1.0\")\n\n r_conn, w_conn = Pipe(duplex=False)\n\n with r_conn, w_conn:\n bs.add_build(spec, explicit=True, control_w_conn=w_conn)\n assert bs.tracked_build_id == spec.dag_hash()\n\n bs.update_state(spec.dag_hash(), \"finished\")\n assert bs.tracked_build_id == \"\"\n\n def test_verbose_print_logs_tracked(self):\n \"\"\"print_logs() for the tracked build writes to stdout.\"\"\"\n bs, _, stdout = create_build_status(is_tty=False, verbose=True, total=1)\n spec = MockSpec(\"trivial-install-test-package\", \"1.0\")\n\n r_conn, w_conn = Pipe(duplex=False)\n\n with r_conn, w_conn:\n bs.add_build(spec, explicit=True, control_w_conn=w_conn)\n bs.print_logs(spec.dag_hash(), b\"hello log\\n\")\n\n stdout.flush()\n assert stdout.buffer.getvalue() == b\"hello log\\n\"\n\n def test_verbose_print_logs_untracked(self):\n \"\"\"print_logs() for an untracked build discards data.\"\"\"\n bs, _, stdout = create_build_status(is_tty=False, verbose=True, total=2)\n spec1 = MockSpec(\"pkg1\", \"1.0\")\n spec2 = MockSpec(\"pkg2\", \"1.0\")\n\n r1, w1 = Pipe(duplex=False)\n\n with r1, w1:\n bs.add_build(spec1, explicit=True, control_w_conn=w1)\n bs.add_build(spec2, explicit=False, control_w_conn=None)\n\n # Only spec1 is tracked; spec2 logs should be discarded\n bs.print_logs(spec2.dag_hash(), b\"ignored\\n\")\n\n stdout.flush()\n assert stdout.buffer.getvalue() == b\"\"\n\n def test_verbose_tty_no_effect(self):\n \"\"\"In TTY mode, add_build() does not set tracked_build_id automatically.\"\"\"\n bs, _, _ = create_build_status(is_tty=True, verbose=True, total=4)\n spec = MockSpec(\"trivial-install-test-package\", \"1.0\")\n\n r_conn, w_conn = Pipe(duplex=False)\n\n with r_conn, w_conn:\n bs.add_build(spec, explicit=True, control_w_conn=w_conn)\n assert bs.tracked_build_id == \"\"\n\n\nclass TestBuildStatusColor:\n \"\"\"Tests that BuildStatus respects the explicit color=True/False parameter.\"\"\"\n\n def test_non_tty_finished_color_true_emits_green(self):\n \"\"\"color=True in non-TTY mode: finished line has per-component ANSI colors.\"\"\"\n spec = MockSpec(\"pkg\", \"1.0\")\n status, _, stdout = create_build_status(is_tty=False, total=1, color=True)\n status.add_build(spec, explicit=True)\n status.update_state(spec.dag_hash(), \"finished\")\n # green indicator, reset, dark-gray hash\n assert stdout.getvalue().startswith(\"\\033[32m[+]\\033[0m \\033[0;90m\")\n\n def test_non_tty_failed_color_true_emits_red(self):\n \"\"\"color=True in non-TTY mode: failed line has per-component ANSI colors.\"\"\"\n spec = MockSpec(\"pkg\", \"1.0\")\n status, _, stdout = create_build_status(is_tty=False, total=1, color=True)\n status.add_build(spec, explicit=True)\n status.update_state(spec.dag_hash(), \"failed\")\n # red indicator, reset, dark-gray hash\n assert stdout.getvalue().startswith(\"\\033[31m[x]\\033[0m \\033[0;90m\")\n\n def test_non_tty_finished_color_false_no_ansi(self):\n \"\"\"color=False in non-TTY mode: finished line has no ANSI escape codes.\"\"\"\n spec = MockSpec(\"pkg\", \"1.0\")\n status, _, stdout = create_build_status(is_tty=False, total=1, color=False)\n status.add_build(spec, explicit=True)\n status.update_state(spec.dag_hash(), \"finished\")\n assert \"\\033[\" not in stdout.getvalue()\n\n\nclass TestTargetJobs:\n \"\"\"Test set_jobs and its effect on the header.\"\"\"\n\n def test_set_jobs_marks_dirty(self):\n \"\"\"set_jobs with a new value should update target_jobs and mark dirty.\"\"\"\n status, _, _ = create_build_status()\n status.dirty = False\n status.set_jobs(3, 2)\n assert status.actual_jobs == 3\n assert status.target_jobs == 2\n assert status.dirty is True\n status.set_jobs(2, 2)\n assert status.actual_jobs == 2\n assert status.target_jobs == 2\n\n def test_set_jobs_same_value_no_dirty(self):\n \"\"\"set_jobs with the same value should not mark dirty.\"\"\"\n status, _, _ = create_build_status()\n status.set_jobs(5, 5)\n status.dirty = False\n status.set_jobs(5, 5)\n assert status.dirty is False\n\n def test_header_shows_target_jobs(self):\n \"\"\"The rendered header should contain the target_jobs count and the word 'jobs'.\"\"\"\n status, _, fake_stdout = create_build_status(total=1)\n add_mock_builds(status, 1)\n status.set_jobs(4, 4)\n status.update()\n output = fake_stdout.getvalue()\n assert \"4\" in output\n assert \"jobs\" in output\n\n def test_header_shows_arrow_when_pending(self):\n \"\"\"When actual != target, the header should show 'actual=>target jobs'.\"\"\"\n status, _, fake_stdout = create_build_status(total=1)\n add_mock_builds(status, 1)\n status.set_jobs(4, 2)\n status.update()\n output = fake_stdout.getvalue()\n assert \"4=>2\" in output\n\n\nclass TestHeadlessMode:\n \"\"\"Test that headless mode suppresses terminal output.\"\"\"\n\n def test_update_suppressed_when_headless(self):\n \"\"\"update() should not write anything when headless is True.\"\"\"\n status, time_values, stdout = create_build_status(is_tty=True, total=1)\n add_mock_builds(status, 1)\n status.headless = True\n time_values.append(10.0)\n status.update()\n assert stdout.getvalue() == \"\"\n\n def test_print_logs_suppressed_when_headless(self):\n \"\"\"print_logs() should discard data when headless is True.\"\"\"\n status, _, stdout = create_build_status(is_tty=True, total=1)\n specs = add_mock_builds(status, 1)\n status.tracked_build_id = specs[0].dag_hash()\n status.headless = True\n status.print_logs(specs[0].dag_hash(), b\"hello world\\n\")\n assert stdout.getvalue() == \"\"\n\n def test_update_state_non_tty_suppressed_when_headless(self):\n \"\"\"update_state() non-TTY output should be suppressed when headless.\"\"\"\n status, _, stdout = create_build_status(is_tty=False, total=1)\n spec = MockSpec(\"pkg\", \"1.0\")\n status.add_build(spec, explicit=True)\n status.headless = True\n stdout.clear()\n status.update_state(spec.dag_hash(), \"finished\")\n assert stdout.getvalue() == \"\"\n\n def test_update_works_after_headless_cleared(self):\n \"\"\"update() should work normally once headless is cleared.\"\"\"\n status, time_values, stdout = create_build_status(is_tty=True, total=1, color=False)\n add_mock_builds(status, 1)\n status.headless = True\n time_values.append(10.0)\n status.update()\n assert stdout.getvalue() == \"\"\n # Clear headless and verify output resumes\n status.headless = False\n status.dirty = True\n status.update()\n assert \"[/] pkg0 pkg0@0.0 starting\" in stdout.getvalue()\n\n\nclass TestStdinReader:\n def test_basic_ascii(self):\n r, w = os.pipe()\n try:\n reader = StdinReader(r)\n os.write(w, b\"abc\")\n assert reader.read() == \"abc\"\n finally:\n os.close(r)\n os.close(w)\n\n def test_ansi_stripping(self):\n r, w = os.pipe()\n try:\n reader = StdinReader(r)\n os.write(w, b\"hello\\x1b[Aworld\\x1b[B!\")\n assert reader.read() == \"helloworld!\"\n finally:\n os.close(r)\n os.close(w)\n\n def test_multibyte_utf8(self):\n r, w = os.pipe()\n try:\n reader = StdinReader(r)\n encoded = \"é\".encode(\"utf-8\") # 0xc3 0xa9\n os.write(w, encoded[:1])\n # First read: incomplete char, decoder buffers it\n result1 = reader.read()\n os.write(w, encoded[1:])\n result2 = reader.read()\n assert result1 + result2 == \"é\"\n finally:\n os.close(r)\n os.close(w)\n\n def test_oserror_returns_empty(self):\n r, w = os.pipe()\n os.close(w)\n os.close(r)\n reader = StdinReader(r)\n assert reader.read() == \"\"\n" }, { "path": "lib/spack/spack/test/jobserver.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport sys\n\nimport pytest\n\nif sys.platform == \"win32\":\n pytest.skip(\"Jobserver tests are not supported on Windows\", allow_module_level=True)\n\nimport fcntl\nimport os\nimport pathlib\nimport stat\n\nfrom spack.new_installer import (\n JobServer,\n create_jobserver_fifo,\n get_jobserver_config,\n open_existing_jobserver_fifo,\n)\nfrom spack.spec import Spec\n\n\nclass TestGetJobserverConfig:\n \"\"\"Test parsing of MAKEFLAGS for jobserver configuration.\"\"\"\n\n def test_empty_makeflags(self):\n \"\"\"Empty MAKEFLAGS should return None.\"\"\"\n assert get_jobserver_config(\"\") is None\n\n def test_no_jobserver_flag(self):\n \"\"\"MAKEFLAGS without jobserver flag should return None.\"\"\"\n assert get_jobserver_config(\" -j4 --silent\") is None\n\n def test_fifo_format_new(self):\n \"\"\"Parse new FIFO format\"\"\"\n assert get_jobserver_config(\" -j4 --jobserver-auth=fifo:/tmp/my_fifo\") == \"/tmp/my_fifo\"\n\n def test_pipe_format_new(self):\n \"\"\"Parse new pipe format\"\"\"\n assert get_jobserver_config(\" -j4 --jobserver-auth=3,4\") == (3, 4)\n\n def test_pipe_format_old(self):\n \"\"\"Parse old pipe format (on old versions of gmake this was not publicized)\"\"\"\n assert get_jobserver_config(\" -j4 --jobserver-fds=5,6\") == (5, 6)\n\n def test_multiple_flags_last_wins(self):\n \"\"\"When multiple jobserver flags exist, last one wins.\"\"\"\n makeflags = \" --jobserver-fds=3,4 --jobserver-auth=fifo:/tmp/fifo --jobserver-auth=7,8\"\n assert get_jobserver_config(makeflags) == (7, 8)\n\n def test_invalid_format(self):\n assert get_jobserver_config(\" --jobserver-auth=3\") is None\n assert get_jobserver_config(\" --jobserver-auth=a,b\") is None\n assert get_jobserver_config(\" --jobserver-auth=3,b\") is None\n assert get_jobserver_config(\" --jobserver-auth=3,4,5\") is None\n assert get_jobserver_config(\" --jobserver-auth=\") is None\n\n\nclass TestCreateJobserverFifo:\n \"\"\"Test FIFO creation for jobserver.\"\"\"\n\n def test_creates_fifo(self):\n \"\"\"Should create a FIFO with correct properties.\"\"\"\n r, w, path = create_jobserver_fifo(4)\n try:\n assert os.path.exists(path)\n assert stat.S_ISFIFO(os.stat(path).st_mode)\n assert (os.stat(path).st_mode & 0o777) == 0o600\n assert fcntl.fcntl(r, fcntl.F_GETFD) != -1\n assert fcntl.fcntl(w, fcntl.F_GETFD) != -1\n assert fcntl.fcntl(r, fcntl.F_GETFL) & os.O_NONBLOCK\n finally:\n os.close(r)\n os.close(w)\n os.unlink(path)\n os.rmdir(os.path.dirname(path))\n\n def test_writes_correct_tokens(self):\n \"\"\"Should write num_jobs - 1 tokens.\"\"\"\n r, w, path = create_jobserver_fifo(5)\n try:\n assert os.read(r, 10) == b\"++++\" # 4 tokens for 5 jobs\n finally:\n os.close(r)\n os.close(w)\n os.unlink(path)\n os.rmdir(os.path.dirname(path))\n\n def test_single_job_no_tokens(self):\n \"\"\"Single job should write 0 tokens.\"\"\"\n r, w, path = create_jobserver_fifo(1)\n try:\n with pytest.raises(BlockingIOError):\n os.read(r, 10) # No tokens for 1 job\n finally:\n os.close(r)\n os.close(w)\n os.unlink(path)\n os.rmdir(os.path.dirname(path))\n\n\nclass TestOpenExistingJobserverFifo:\n \"\"\"Test opening existing jobserver FIFOs.\"\"\"\n\n def test_opens_existing_fifo(self, tmp_path: pathlib.Path):\n \"\"\"Should successfully open an existing FIFO.\"\"\"\n fifo_path = str(tmp_path / \"test_fifo\")\n os.mkfifo(fifo_path, 0o600)\n\n result = open_existing_jobserver_fifo(fifo_path)\n assert result is not None\n\n r, w = result\n assert fcntl.fcntl(r, fcntl.F_GETFD) != -1\n assert fcntl.fcntl(w, fcntl.F_GETFD) != -1\n assert fcntl.fcntl(r, fcntl.F_GETFL) & os.O_NONBLOCK\n\n os.close(r)\n os.close(w)\n\n def test_returns_none_for_missing_fifo(self, tmp_path: pathlib.Path):\n \"\"\"Should return None if FIFO doesn't exist.\"\"\"\n result = open_existing_jobserver_fifo(str(tmp_path / \"nonexistent_fifo\"))\n assert result is None\n\n\n#: Constant that's larger than the number of jobs used in tests.\nALL_TOKENS = 100\n\n\nclass TestJobServer:\n \"\"\"Test JobServer class functionality.\"\"\"\n\n def test_creates_new_jobserver(self):\n \"\"\"Should create a new FIFO-based jobserver when none exists.\"\"\"\n js = JobServer(4)\n\n try:\n assert js.created is True\n assert js.fifo_path is not None\n assert os.path.exists(js.fifo_path)\n assert js.tokens_acquired == 0\n assert fcntl.fcntl(js.r, fcntl.F_GETFD) != -1\n assert fcntl.fcntl(js.w, fcntl.F_GETFD) != -1\n finally:\n js.close()\n\n def test_attaches_to_existing_fifo(self):\n \"\"\"Should attach to existing FIFO jobserver from environment.\"\"\"\n js1 = JobServer(4)\n assert js1.fifo_path\n\n try:\n fifo_config = get_jobserver_config(f\" -j4 --jobserver-auth=fifo:{js1.fifo_path}\")\n assert fifo_config == js1.fifo_path\n\n result = open_existing_jobserver_fifo(js1.fifo_path)\n assert result is not None\n\n r, w = result\n os.close(r)\n os.close(w)\n\n finally:\n js1.close()\n\n def test_acquire_tokens(self):\n \"\"\"Should acquire tokens from jobserver.\"\"\"\n js = JobServer(5)\n\n try:\n assert js.acquire(2) == 2\n assert js.tokens_acquired == 2\n\n assert js.acquire(2) == 2\n assert js.tokens_acquired == 4\n\n assert js.acquire(2) == 0\n assert js.tokens_acquired == 4\n\n finally:\n js.close()\n\n def test_release_tokens(self):\n \"\"\"Should release tokens back to jobserver.\"\"\"\n js = JobServer(5)\n\n try:\n assert js.acquire(2) == 2\n assert js.tokens_acquired == 2\n\n js.release()\n assert js.tokens_acquired == 1\n\n assert js.acquire(1) == 1\n assert js.tokens_acquired == 2\n\n finally:\n js.close()\n\n def test_release_without_tokens_is_noop(self):\n \"\"\"Releasing without acquired tokens should be a no-op.\"\"\"\n js = JobServer(4)\n\n try:\n assert js.tokens_acquired == 0\n js.release()\n assert js.tokens_acquired == 0\n finally:\n js.close()\n\n def test_makeflags_fifo_gmake_44(self):\n \"\"\"Should return FIFO format for gmake >= 4.4.\"\"\"\n js = JobServer(8)\n\n try:\n flags = js.makeflags(Spec(\"gmake@=4.4\"))\n assert flags == f\" -j8 --jobserver-auth=fifo:{js.fifo_path}\"\n finally:\n js.close()\n\n def test_makeflags_pipe_gmake_40(self):\n \"\"\"Should return pipe format for gmake 4.0-4.3.\"\"\"\n js = JobServer(8)\n\n try:\n flags = js.makeflags(Spec(\"gmake@=4.0\"))\n assert flags == f\" -j8 --jobserver-auth={js.r},{js.w}\"\n finally:\n js.close()\n\n def test_makeflags_old_format_gmake_3(self):\n \"\"\"Should return old --jobserver-fds format for gmake < 4.0.\"\"\"\n js = JobServer(8)\n\n try:\n flags = js.makeflags(Spec(\"gmake@=3.9\"))\n assert flags == f\" -j8 --jobserver-fds={js.r},{js.w}\"\n finally:\n js.close()\n\n def test_makeflags_no_gmake(self):\n \"\"\"Should return FIFO format when no gmake (modern default).\"\"\"\n js = JobServer(6)\n\n try:\n flags = js.makeflags(None)\n assert flags == f\" -j6 --jobserver-auth=fifo:{js.fifo_path}\"\n finally:\n js.close()\n\n def test_close_removes_created_fifo(self):\n \"\"\"Should remove FIFO and directory if created by this instance.\"\"\"\n js = JobServer(4)\n fifo_path = js.fifo_path\n assert fifo_path and os.path.exists(fifo_path)\n js.close()\n assert not os.path.exists(os.path.dirname(fifo_path))\n\n def test_file_descriptors_are_inheritable(self):\n \"\"\"Should set file descriptors as inheritable for child processes.\"\"\"\n js = JobServer(4)\n\n try:\n assert os.get_inheritable(js.r)\n assert os.get_inheritable(js.w)\n finally:\n js.close()\n\n def test_connection_objects_exist(self):\n \"\"\"Should create Connection objects for fd inheritance.\"\"\"\n js = JobServer(4)\n\n try:\n assert js.r_conn is not None and js.r_conn.fileno() == js.r\n assert js.w_conn is not None and js.w_conn.fileno() == js.w\n finally:\n js.close()\n\n def test_close_warns_when_spack_holds_tokens(self):\n \"\"\"Should warn when Spack closes the jobserver while still holding acquired tokens.\"\"\"\n js = JobServer(4)\n js.acquire(1) # Spack acquires a token without releasing it\n with pytest.warns(UserWarning, match=\"Spack failed to release jobserver tokens\"):\n js.close()\n\n def test_close_warns_when_subprocess_holds_tokens(self):\n \"\"\"Should warn when a subprocess acquired a token but never released it.\"\"\"\n js1 = JobServer(4)\n os.read(js1.r, 1) # A subprocess acquires a token without releasing it\n with pytest.warns(UserWarning, match=\"1 jobserver token was not released\"):\n js1.close()\n\n js2 = JobServer(4)\n os.read(js2.r, 2) # A subprocess acquires two tokens without releasing them\n with pytest.warns(UserWarning, match=\"2 jobserver tokens were not released\"):\n js2.close()\n\n def test_has_target_parallelism(self):\n \"\"\"has_target_parallelism() should be True initially.\"\"\"\n js = JobServer(4)\n try:\n assert js.has_target_parallelism() is True\n js.target_jobs = js.num_jobs - 1\n assert js.has_target_parallelism() is False\n finally:\n js.close()\n\n def test_increase_parallelism_not_created(self):\n \"\"\"increase_parallelism() should be a no-op when not self.created.\"\"\"\n # Simulate an externally attached jobserver by patching created after construction.\n js = JobServer(3)\n try:\n original_num = js.num_jobs\n original_target = js.target_jobs\n js.created = False\n js.increase_parallelism()\n assert js.num_jobs == original_num\n assert js.target_jobs == original_target\n js.decrease_parallelism()\n assert js.num_jobs == original_num\n assert js.target_jobs == original_target\n finally:\n js.created = True # restore so close() works\n js.close()\n\n def test_increase_parallelism(self):\n \"\"\"increase_parallelism() should increment num_jobs and target_jobs and add a token.\"\"\"\n js = JobServer(3)\n try:\n original_num = js.num_jobs\n original_target = js.target_jobs\n js.increase_parallelism()\n assert js.num_jobs == original_num + 1\n assert js.target_jobs == original_target + 1\n # Verify the \"js.num_jobs - 1 tokens in the pipe\" invariant.\n assert js.acquire(ALL_TOKENS) + 1 == js.num_jobs\n finally:\n js.close()\n\n def test_decrease_parallelism_at_floor(self):\n \"\"\"decrease_parallelism() should not go below target_jobs == 1.\"\"\"\n js = JobServer(1)\n try:\n # target_jobs starts at 1\n assert js.target_jobs == 1\n js.decrease_parallelism()\n assert js.target_jobs == 1\n finally:\n js.close()\n\n def test_decrease_parallelism_token_available(self):\n \"\"\"When pipe has tokens, decrease_parallelism discards one immediately.\"\"\"\n js = JobServer(3)\n try:\n # 3-job server starts with 2 tokens in the pipe.\n original_num = js.num_jobs\n js.decrease_parallelism()\n assert js.target_jobs == original_num - 1\n assert js.num_jobs == original_num - 1\n assert js.acquire(ALL_TOKENS) + 1 == js.num_jobs\n finally:\n js.close()\n\n def test_decrease_parallelism_no_token_available(self):\n \"\"\"When all tokens are held, decrease_parallelism defers the discard.\"\"\"\n js = JobServer(3)\n try:\n # Drain the pipe so no tokens are available for immediate discard.\n assert js.acquire(ALL_TOKENS) == js.num_jobs - 1\n original_num = js.num_jobs\n js.decrease_parallelism()\n # target_jobs decremented but num_jobs unchanged (no token to discard yet).\n assert js.target_jobs == original_num - 1\n assert js.num_jobs == original_num\n finally:\n js.close()\n\n def test_maybe_discard_tokens_noop_at_target(self):\n \"\"\"maybe_discard_tokens() should be a no-op when num_jobs == target_jobs.\"\"\"\n js = JobServer(3)\n try:\n original_num = js.num_jobs\n js.maybe_discard_tokens() # to_discard == 0\n assert js.num_jobs == original_num\n finally:\n js.close()\n\n def test_maybe_discard_tokens_discards_when_available(self):\n \"\"\"maybe_discard_tokens() should consume tokens from the pipe.\"\"\"\n js = JobServer(4)\n try:\n # Manually set target lower to create a discard requirement.\n js.target_jobs = js.num_jobs - 2\n original_num = js.num_jobs\n js.maybe_discard_tokens()\n assert js.num_jobs < original_num\n finally:\n js.close()\n\n def test_maybe_discard_tokens_noop_on_blocking(self):\n \"\"\"maybe_discard_tokens() should not raise when pipe is empty.\"\"\"\n js = JobServer(3)\n try:\n # Drain all tokens from the pipe (simulates subprocesses holding them).\n assert js.acquire(ALL_TOKENS) == js.num_jobs - 1\n original_num = js.num_jobs\n # Artificially lower target so a discard is requested, but pipe is empty.\n js.target_jobs = js.num_jobs - 1\n js.maybe_discard_tokens() # Should not raise; num_jobs unchanged.\n assert js.num_jobs == original_num\n finally:\n js.close()\n\n def test_release_discards_token_when_target_below_num(self):\n \"\"\"release() should discard a token (not return it) when target_jobs < num_jobs.\"\"\"\n js = JobServer(4)\n try:\n # Acquire a token.\n assert js.acquire(1) == 1\n assert js.tokens_acquired == 1\n # Manually lower target to simulate a pending decrease.\n js.target_jobs = js.num_jobs - 1\n original_num = js.num_jobs\n # Drain the free tokens from the pipe so we can count them after.\n drained = os.read(js.r, ALL_TOKENS)\n # Release should discard the token (decrement num_jobs) instead of writing to pipe.\n js.release()\n assert js.tokens_acquired == 0\n assert js.num_jobs == original_num - 1\n # Pipe should remain empty (nothing written back).\n with pytest.raises(BlockingIOError):\n os.read(js.r, 1)\n finally:\n # Restore drained tokens so close() can clean up cleanly.\n os.write(js.w, drained)\n js.close()\n" }, { "path": "lib/spack/spack/test/link_paths.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport os\nimport re\nimport sys\n\nimport pytest\n\nimport spack.compilers.libraries\nimport spack.paths\nfrom spack.compilers.libraries import parse_non_system_link_dirs\n\ndrive = \"\"\nif sys.platform == \"win32\":\n match = re.search(r\"[A-Za-z]:\", spack.paths.test_path)\n if match:\n drive = match.group()\nroot = drive + os.sep\n\n#: directory with sample compiler data\ndatadir = os.path.join(spack.paths.test_path, \"data\", \"compiler_verbose_output\")\n\n\n@pytest.fixture(autouse=True)\ndef allow_nonexistent_paths(monkeypatch):\n # Allow nonexistent paths to be detected as part of the output\n # for testing purposes.\n monkeypatch.setattr(spack.compilers.libraries, \"filter_non_existing_dirs\", lambda x: x)\n\n\ndef check_link_paths(filename, paths):\n with open(os.path.join(datadir, filename), encoding=\"utf-8\") as file:\n output = file.read()\n detected_paths = parse_non_system_link_dirs(output)\n\n actual = detected_paths\n expected = paths\n\n missing_paths = list(x for x in expected if x not in actual)\n assert not missing_paths\n\n extra_paths = list(x for x in actual if x not in expected)\n assert not extra_paths\n\n assert actual == expected\n\n\ndef test_icc16_link_paths():\n prefix = os.path.join(root, \"usr\", \"tce\", \"packages\")\n check_link_paths(\n \"icc-16.0.3.txt\",\n [\n os.path.join(\n prefix,\n \"intel\",\n \"intel-16.0.3\",\n \"compilers_and_libraries_2016.3.210\",\n \"linux\",\n \"compiler\",\n \"lib\",\n \"intel64_lin\",\n ),\n os.path.join(\n prefix, \"gcc\", \"gcc-4.9.3\", \"lib64\", \"gcc\", \"x86_64-unknown-linux-gnu\", \"4.9.3\"\n ),\n os.path.join(prefix, \"gcc\", \"gcc-4.9.3\", \"lib64\"),\n ],\n )\n\n\ndef test_gcc7_link_paths():\n check_link_paths(\"gcc-7.3.1.txt\", [])\n\n\ndef test_clang4_link_paths():\n check_link_paths(\"clang-4.0.1.txt\", [])\n\n\ndef test_xl_link_paths():\n check_link_paths(\n \"xl-13.1.5.txt\",\n [\n os.path.join(root, \"opt\", \"ibm\", \"xlsmp\", \"4.1.5\", \"lib\"),\n os.path.join(root, \"opt\", \"ibm\", \"xlmass\", \"8.1.5\", \"lib\"),\n os.path.join(root, \"opt\", \"ibm\", \"xlC\", \"13.1.5\", \"lib\"),\n ],\n )\n\n\ndef test_cce_link_paths():\n gcc = os.path.join(root, \"opt\", \"gcc\")\n cray = os.path.join(root, \"opt\", \"cray\")\n check_link_paths(\n \"cce-8.6.5.txt\",\n [\n os.path.join(gcc, \"6.1.0\", \"snos\", \"lib64\"),\n os.path.join(cray, \"dmapp\", \"default\", \"lib64\"),\n os.path.join(cray, \"pe\", \"mpt\", \"7.7.0\", \"gni\", \"mpich-cray\", \"8.6\", \"lib\"),\n os.path.join(cray, \"pe\", \"libsci\", \"17.12.1\", \"CRAY\", \"8.6\", \"x86_64\", \"lib\"),\n os.path.join(cray, \"rca\", \"2.2.16-6.0.5.0_15.34__g5e09e6d.ari\", \"lib64\"),\n os.path.join(cray, \"pe\", \"pmi\", \"5.0.13\", \"lib64\"),\n os.path.join(cray, \"xpmem\", \"2.2.4-6.0.5.0_4.8__g35d5e73.ari\", \"lib64\"),\n os.path.join(cray, \"dmapp\", \"7.1.1-6.0.5.0_49.8__g1125556.ari\", \"lib64\"),\n os.path.join(cray, \"ugni\", \"6.0.14-6.0.5.0_16.9__g19583bb.ari\", \"lib64\"),\n os.path.join(cray, \"udreg\", \"2.3.2-6.0.5.0_13.12__ga14955a.ari\", \"lib64\"),\n os.path.join(cray, \"alps\", \"6.5.28-6.0.5.0_18.6__g13a91b6.ari\", \"lib64\"),\n os.path.join(cray, \"pe\", \"atp\", \"2.1.1\", \"libApp\"),\n os.path.join(cray, \"pe\", \"cce\", \"8.6.5\", \"cce\", \"x86_64\", \"lib\"),\n os.path.join(cray, \"wlm_detect\", \"1.3.2-6.0.5.0_3.1__g388ccd5.ari\", \"lib64\"),\n os.path.join(gcc, \"6.1.0\", \"snos\", \"lib\", \"gcc\", \"x86_64-suse-linux\", \"6.1.0\"),\n os.path.join(\n cray, \"pe\", \"cce\", \"8.6.5\", \"binutils\", \"x86_64\", \"x86_64-unknown-linux-gnu\", \"lib\"\n ),\n ],\n )\n\n\ndef test_clang_apple_ld_link_paths():\n check_link_paths(\n \"clang-9.0.0-apple-ld.txt\",\n [\n os.path.join(\n root,\n \"Applications\",\n \"Xcode.app\",\n \"Contents\",\n \"Developer\",\n \"Platforms\",\n \"MacOSX.platform\",\n \"Developer\",\n \"SDKs\",\n \"MacOSX10.13.sdk\",\n \"usr\",\n \"lib\",\n )\n ],\n )\n\n\ndef test_nag_mixed_gcc_gnu_ld_link_paths():\n # This is a test of a mixed NAG/GCC toolchain, i.e. 'cxx' is set to g++ and\n # is used for the rpath detection. The reference compiler output is a\n # result of\n # '/path/to/gcc/bin/g++ -Wl,-v ./main.c'.\n prefix = os.path.join(\n root,\n \"scratch\",\n \"local1\",\n \"spack\",\n \"opt\",\n \"spack\",\n \"gcc-6.3.0-haswell\",\n \"gcc-6.5.0-4sdjgrs\",\n )\n\n check_link_paths(\n \"collect2-6.3.0-gnu-ld.txt\",\n [\n os.path.join(prefix, \"lib\", \"gcc\", \"x86_64-pc-linux-gnu\", \"6.5.0\"),\n os.path.join(prefix, \"lib64\"),\n os.path.join(prefix, \"lib\"),\n ],\n )\n\n\ndef test_nag_link_paths():\n # This is a test of a NAG-only toolchain, i.e. 'cc' and 'cxx' are empty,\n # and therefore 'fc' is used for the rpath detection). The reference\n # compiler output is a result of\n # 'nagfor -Wc=/path/to/gcc/bin/gcc -Wl,-v ./main.c'.\n prefix = os.path.join(\n root,\n \"scratch\",\n \"local1\",\n \"spack\",\n \"opt\",\n \"spack\",\n \"gcc-6.3.0-haswell\",\n \"gcc-6.5.0-4sdjgrs\",\n )\n\n check_link_paths(\n \"nag-6.2-gcc-6.5.0.txt\",\n [\n os.path.join(prefix, \"lib\", \"gcc\", \"x86_64-pc-linux-gnu\", \"6.5.0\"),\n os.path.join(prefix, \"lib64\"),\n os.path.join(prefix, \"lib\"),\n ],\n )\n\n\ndef test_obscure_parsing_rules():\n paths = [\n os.path.join(root, \"first\", \"path\"),\n os.path.join(root, \"second\", \"path\"),\n os.path.join(root, \"third\", \"path\"),\n ]\n\n # TODO: add a comment explaining why this happens\n if sys.platform == \"win32\":\n paths.remove(os.path.join(root, \"second\", \"path\"))\n\n check_link_paths(\"obscure-parsing-rules.txt\", paths)\n" }, { "path": "lib/spack/spack/test/llnl/llnl_string.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport pytest\n\nimport spack.llnl.string\n\n\n@pytest.mark.parametrize(\n \"arguments,expected\",\n [\n ((0, \"thing\"), \"0 things\"),\n ((1, \"thing\"), \"1 thing\"),\n ((2, \"thing\"), \"2 things\"),\n ((1, \"thing\", \"wombats\"), \"1 thing\"),\n ((2, \"thing\", \"wombats\"), \"2 wombats\"),\n ((2, \"thing\", \"wombats\", False), \"wombats\"),\n ],\n)\ndef test_plural(arguments, expected):\n assert spack.llnl.string.plural(*arguments) == expected\n\n\n@pytest.mark.parametrize(\n \"arguments,expected\",\n [(([\"one\", \"two\"],), [\"'one'\", \"'two'\"]), (([\"one\", \"two\"], \"^\"), [\"^one^\", \"^two^\"])],\n)\ndef test_quote(arguments, expected):\n assert spack.llnl.string.quote(*arguments) == expected\n\n\n@pytest.mark.parametrize(\n \"input,expected_and,expected_or\",\n [\n ([\"foo\"], \"foo\", \"foo\"),\n ([\"foo\", \"bar\"], \"foo and bar\", \"foo or bar\"),\n ([\"foo\", \"bar\", \"baz\"], \"foo, bar, and baz\", \"foo, bar, or baz\"),\n ],\n)\ndef test_comma_and_or(input, expected_and, expected_or):\n assert spack.llnl.string.comma_and(input) == expected_and\n assert spack.llnl.string.comma_or(input) == expected_or\n" }, { "path": "lib/spack/spack/test/llnl/url.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Tests for spack.llnl.url functions\"\"\"\n\nimport itertools\n\nimport pytest\n\nimport spack.llnl.url\n\n\n@pytest.fixture(params=spack.llnl.url.ALLOWED_ARCHIVE_TYPES)\ndef archive_and_expected(request):\n archive_name = \".\".join([\"Foo\", request.param])\n return archive_name, request.param\n\n\ndef test_get_extension(archive_and_expected):\n \"\"\"Tests that we can predict correctly known extensions for simple cases.\"\"\"\n archive, expected = archive_and_expected\n result = spack.llnl.url.extension_from_path(archive)\n assert result == expected\n\n\ndef test_get_bad_extension():\n \"\"\"Tests that a bad extension returns None\"\"\"\n result = spack.llnl.url.extension_from_path(\"Foo.cxx\")\n assert result is None\n\n\n@pytest.mark.parametrize(\n \"url,expected\",\n [\n # No suffix\n (\"rgb-1.0.6\", \"rgb-1.0.6\"),\n # Misleading prefix\n (\"jpegsrc.v9b\", \"jpegsrc.v9b\"),\n (\"turbolinux702\", \"turbolinux702\"),\n (\"converge_install_2.3.16\", \"converge_install_2.3.16\"),\n # Download type - code, source\n (\"cistem-1.0.0-beta-source-code\", \"cistem-1.0.0-beta\"),\n # Download type - src\n (\"apache-ant-1.9.7-src\", \"apache-ant-1.9.7\"),\n (\"go1.7.4.src\", \"go1.7.4\"),\n # Download type - source\n (\"bowtie2-2.2.5-source\", \"bowtie2-2.2.5\"),\n (\"grib_api-1.17.0-Source\", \"grib_api-1.17.0\"),\n # Download type - full\n (\"julia-0.4.3-full\", \"julia-0.4.3\"),\n # Download type - bin\n (\"apache-maven-3.3.9-bin\", \"apache-maven-3.3.9\"),\n # Download type - binary\n (\"Jmol-14.8.0-binary\", \"Jmol-14.8.0\"),\n # Download type - gem\n (\"rubysl-date-2.0.9.gem\", \"rubysl-date-2.0.9\"),\n # Download type - tar\n (\"gromacs-4.6.1-tar\", \"gromacs-4.6.1\"),\n # Download type - sh\n (\"Miniconda2-4.3.11-Linux-x86_64.sh\", \"Miniconda2-4.3.11\"),\n # Download version - release\n (\"v1.0.4-release\", \"v1.0.4\"),\n # Download version - stable\n (\"libevent-2.0.21-stable\", \"libevent-2.0.21\"),\n # Download version - final\n (\"2.6.7-final\", \"2.6.7\"),\n # Download version - rel\n (\"v1.9.5.1rel\", \"v1.9.5.1\"),\n # Download version - orig\n (\"dash_0.5.5.1.orig\", \"dash_0.5.5.1\"),\n # Download version - plus\n (\"ncbi-blast-2.6.0+-src\", \"ncbi-blast-2.6.0\"),\n # License\n (\"cppad-20170114.gpl\", \"cppad-20170114\"),\n # Arch\n (\"pcraster-4.1.0_x86-64\", \"pcraster-4.1.0\"),\n (\"dislin-11.0.linux.i586_64\", \"dislin-11.0\"),\n (\"PAGIT.V1.01.64bit\", \"PAGIT.V1.01\"),\n # OS - linux\n (\"astyle_2.04_linux\", \"astyle_2.04\"),\n # OS - unix\n (\"install-tl-unx\", \"install-tl\"),\n # OS - macos\n (\"astyle_1.23_macosx\", \"astyle_1.23\"),\n (\"haxe-2.08-osx\", \"haxe-2.08\"),\n # PyPI - wheel\n (\"wheel-1.2.3-py3-none-any\", \"wheel-1.2.3\"),\n (\"wheel-1.2.3-py2.py3-none-any\", \"wheel-1.2.3\"),\n (\"wheel-1.2.3-cp38-abi3-macosx_10_12_x86_64\", \"wheel-1.2.3\"),\n (\"entrypoints-0.2.2-py2.py3-none-any\", \"entrypoints-0.2.2\"),\n (\n \"numpy-1.12.0-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.\"\n \"macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64\",\n \"numpy-1.12.0\",\n ),\n # Combinations of multiple patterns - bin, release\n (\"rocketmq-all-4.5.2-bin-release\", \"rocketmq-all-4.5.2\"),\n # Combinations of multiple patterns - all\n (\"p7zip_9.04_src_all\", \"p7zip_9.04\"),\n # Combinations of multiple patterns - run\n (\"cuda_8.0.44_linux.run\", \"cuda_8.0.44\"),\n # Combinations of multiple patterns - file\n (\"ack-2.14-single-file\", \"ack-2.14\"),\n # Combinations of multiple patterns - jar\n (\"antlr-3.4-complete.jar\", \"antlr-3.4\"),\n # Combinations of multiple patterns - oss\n (\"tbb44_20160128oss_src_0\", \"tbb44_20160128\"),\n # Combinations of multiple patterns - darwin\n (\"ghc-7.0.4-x86_64-apple-darwin\", \"ghc-7.0.4\"),\n (\"ghc-7.0.4-i386-apple-darwin\", \"ghc-7.0.4\"),\n # Combinations of multiple patterns - centos\n (\"sratoolkit.2.8.2-1-centos_linux64\", \"sratoolkit.2.8.2-1\"),\n # Combinations of multiple patterns - arch\n (\n \"VizGlow_v2.2alpha17-R21November2016-Linux-x86_64-Install\",\n \"VizGlow_v2.2alpha17-R21November2016\",\n ),\n (\"jdk-8u92-linux-x64\", \"jdk-8u92\"),\n (\"cuda_6.5.14_linux_64.run\", \"cuda_6.5.14\"),\n (\"Mathematica_12.0.0_LINUX.sh\", \"Mathematica_12.0.0\"),\n (\"trf407b.linux64\", \"trf407b\"),\n # Combinations of multiple patterns - with\n (\"mafft-7.221-with-extensions-src\", \"mafft-7.221\"),\n (\"spark-2.0.0-bin-without-hadoop\", \"spark-2.0.0\"),\n (\"conduit-v0.3.0-src-with-blt\", \"conduit-v0.3.0\"),\n # Combinations of multiple patterns - rock\n (\"bitlib-23-2.src.rock\", \"bitlib-23-2\"),\n # Combinations of multiple patterns - public\n (\"dakota-6.3-public.src\", \"dakota-6.3\"),\n # Combinations of multiple patterns - universal\n (\"synergy-1.3.6p2-MacOSX-Universal\", \"synergy-1.3.6p2\"),\n # Combinations of multiple patterns - dynamic\n (\"snptest_v2.5.2_linux_x86_64_dynamic\", \"snptest_v2.5.2\"),\n # Combinations of multiple patterns - other\n (\"alglib-3.11.0.cpp.gpl\", \"alglib-3.11.0\"),\n (\"hpcviewer-2019.08-linux.gtk.x86_64\", \"hpcviewer-2019.08\"),\n (\"apache-mxnet-src-1.3.0-incubating\", \"apache-mxnet-src-1.3.0\"),\n ],\n)\ndef test_url_strip_version_suffixes(url, expected):\n stripped = spack.llnl.url.strip_version_suffixes(url)\n assert stripped == expected\n\n\ndef test_strip_compression_extension(archive_and_expected):\n archive, extension = archive_and_expected\n stripped = spack.llnl.url.strip_compression_extension(archive)\n if extension == \"zip\":\n assert stripped == \"Foo.zip\"\n stripped = spack.llnl.url.strip_compression_extension(archive, \"zip\")\n assert stripped == \"Foo\"\n elif extension == \"whl\":\n assert stripped == \"Foo.whl\"\n elif (\n extension.lower() == \"tar\"\n or extension in spack.llnl.url.CONTRACTION_MAP\n or extension\n in [\n \".\".join(ext)\n for ext in itertools.product(\n spack.llnl.url.PREFIX_EXTENSIONS, spack.llnl.url.EXTENSIONS\n )\n ]\n ):\n assert stripped == \"Foo.tar\" or stripped == \"Foo.TAR\"\n else:\n assert stripped == \"Foo\"\n\n\ndef test_allowed_archive(archive_and_expected):\n archive, _ = archive_and_expected\n assert spack.llnl.url.allowed_archive(archive)\n" }, { "path": "lib/spack/spack/test/llnl/util/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n" }, { "path": "lib/spack/spack/test/llnl/util/argparsewriter.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Tests for ``llnl/util/argparsewriter.py``\n\nThese tests are fairly minimal, and ArgparseWriter is more extensively\ntested in ``cmd/commands.py``.\n\"\"\"\n\nimport pytest\n\nimport spack.llnl.util.argparsewriter as aw\nimport spack.main\n\nparser = spack.main.make_argument_parser()\nspack.main.add_all_commands(parser)\n\n\ndef test_format_not_overridden():\n with pytest.raises(TypeError):\n aw.ArgparseWriter(\"spack\")\n" }, { "path": "lib/spack/spack/test/llnl/util/file_list.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport fnmatch\nimport os\nimport sys\n\nimport pytest\n\nimport spack.paths\nfrom spack.llnl.util.filesystem import HeaderList, LibraryList, find_headers, find_libraries\n\n\n@pytest.fixture()\ndef library_list():\n \"\"\"Returns an instance of LibraryList.\"\"\"\n # Test all valid extensions: ['.a', '.dylib', '.so']\n libs = (\n [\n \"/dir1/liblapack.a\",\n \"/dir2/libpython3.6.dylib\", # name may contain periods\n \"/dir1/libblas.a\",\n \"/dir3/libz.so\",\n \"libmpi.so.20.10.1\", # shared object libraries may be versioned\n ]\n if sys.platform != \"win32\"\n else [\n \"/dir1/liblapack.lib\",\n \"/dir2/libpython3.6.dll\",\n \"/dir1/libblas.lib\",\n \"/dir3/libz.dll\",\n \"libmpi.dll.20.10.1\",\n ]\n )\n\n return LibraryList(libs)\n\n\n@pytest.fixture()\ndef header_list():\n \"\"\"Returns an instance of header list\"\"\"\n # Test all valid extensions: ['.h', '.hpp', '.hh', '.cuh']\n headers = [\n \"/dir1/Python.h\",\n \"/dir2/date.time.h\",\n \"/dir1/pyconfig.hpp\",\n \"/dir3/core.hh\",\n \"pymem.cuh\",\n ]\n h = HeaderList(headers)\n h.add_macro(\"-DBOOST_LIB_NAME=boost_regex\")\n h.add_macro(\"-DBOOST_DYN_LINK\")\n return h\n\n\n# TODO: Remove below when spack.llnl.util.filesystem.find_libraries becomes spec aware\nplat_static_ext = \"lib\" if sys.platform == \"win32\" else \"a\"\n\n\nplat_shared_ext = \"dll\" if sys.platform == \"win32\" else \"so\"\n\n\nplat_apple_shared_ext = \"dylib\"\n\n\nclass TestLibraryList:\n def test_repr(self, library_list):\n x = eval(repr(library_list))\n assert library_list == x\n\n def test_joined_and_str(self, library_list):\n s1 = library_list.joined()\n expected = \" \".join(\n [\n \"/dir1/liblapack.%s\" % plat_static_ext,\n \"/dir2/libpython3.6.%s\"\n % (plat_apple_shared_ext if sys.platform != \"win32\" else \"dll\"),\n \"/dir1/libblas.%s\" % plat_static_ext,\n \"/dir3/libz.%s\" % plat_shared_ext,\n \"libmpi.%s.20.10.1\" % plat_shared_ext,\n ]\n )\n assert s1 == expected\n\n s2 = str(library_list)\n assert s1 == s2\n\n s3 = library_list.joined(\";\")\n expected = \";\".join(\n [\n \"/dir1/liblapack.%s\" % plat_static_ext,\n \"/dir2/libpython3.6.%s\"\n % (plat_apple_shared_ext if sys.platform != \"win32\" else \"dll\"),\n \"/dir1/libblas.%s\" % plat_static_ext,\n \"/dir3/libz.%s\" % plat_shared_ext,\n \"libmpi.%s.20.10.1\" % plat_shared_ext,\n ]\n )\n assert s3 == expected\n\n def test_flags(self, library_list):\n search_flags = library_list.search_flags\n assert \"-L/dir1\" in search_flags\n assert \"-L/dir2\" in search_flags\n assert \"-L/dir3\" in search_flags\n assert isinstance(search_flags, str)\n assert search_flags == \"-L/dir1 -L/dir2 -L/dir3\"\n\n link_flags = library_list.link_flags\n assert \"-llapack\" in link_flags\n assert \"-lpython3.6\" in link_flags\n assert \"-lblas\" in link_flags\n assert \"-lz\" in link_flags\n assert \"-lmpi\" in link_flags\n assert isinstance(link_flags, str)\n assert link_flags == \"-llapack -lpython3.6 -lblas -lz -lmpi\"\n\n ld_flags = library_list.ld_flags\n assert isinstance(ld_flags, str)\n assert ld_flags == search_flags + \" \" + link_flags\n\n def test_paths_manipulation(self, library_list):\n names = library_list.names\n assert names == [\"lapack\", \"python3.6\", \"blas\", \"z\", \"mpi\"]\n\n directories = library_list.directories\n assert directories == [\"/dir1\", \"/dir2\", \"/dir3\"]\n\n def test_get_item(self, library_list):\n a = library_list[0]\n assert a == \"/dir1/liblapack.%s\" % plat_static_ext\n\n b = library_list[:]\n assert type(b) is type(library_list)\n assert library_list == b\n assert library_list is not b\n\n def test_add(self, library_list):\n pylist = [\n \"/dir1/liblapack.%s\" % plat_static_ext, # removed from the final list\n \"/dir2/libmpi.%s\" % plat_shared_ext,\n \"/dir4/libnew.%s\" % plat_static_ext,\n ]\n another = LibraryList(pylist)\n both = library_list + another\n assert len(both) == 7\n\n # Invariant\n assert both == both + both\n\n # Always produce an instance of LibraryList\n assert type(library_list + pylist) is type(library_list)\n assert type(pylist + library_list) is type(library_list)\n\n\nclass TestHeaderList:\n def test_repr(self, header_list):\n x = eval(repr(header_list))\n assert header_list == x\n\n def test_joined_and_str(self, header_list):\n s1 = header_list.joined()\n expected = \" \".join(\n [\n \"/dir1/Python.h\",\n \"/dir2/date.time.h\",\n \"/dir1/pyconfig.hpp\",\n \"/dir3/core.hh\",\n \"pymem.cuh\",\n ]\n )\n assert s1 == expected\n\n s2 = str(header_list)\n assert s1 == s2\n\n s3 = header_list.joined(\";\")\n expected = \";\".join(\n [\n \"/dir1/Python.h\",\n \"/dir2/date.time.h\",\n \"/dir1/pyconfig.hpp\",\n \"/dir3/core.hh\",\n \"pymem.cuh\",\n ]\n )\n assert s3 == expected\n\n def test_flags(self, header_list):\n include_flags = header_list.include_flags\n assert \"-I/dir1\" in include_flags\n assert \"-I/dir2\" in include_flags\n assert \"-I/dir3\" in include_flags\n assert isinstance(include_flags, str)\n assert include_flags == \"-I/dir1 -I/dir2 -I/dir3\"\n\n macros = header_list.macro_definitions\n assert \"-DBOOST_LIB_NAME=boost_regex\" in macros\n assert \"-DBOOST_DYN_LINK\" in macros\n assert isinstance(macros, str)\n assert macros == \"-DBOOST_LIB_NAME=boost_regex -DBOOST_DYN_LINK\"\n\n cpp_flags = header_list.cpp_flags\n assert isinstance(cpp_flags, str)\n assert cpp_flags == include_flags + \" \" + macros\n\n def test_paths_manipulation(self, header_list):\n names = header_list.names\n assert names == [\"Python\", \"date.time\", \"pyconfig\", \"core\", \"pymem\"]\n\n directories = header_list.directories\n assert directories == [\"/dir1\", \"/dir2\", \"/dir3\"]\n\n def test_get_item(self, header_list):\n a = header_list[0]\n assert a == \"/dir1/Python.h\"\n\n b = header_list[:]\n assert type(b) is type(header_list)\n assert header_list == b\n assert header_list is not b\n\n def test_add(self, header_list):\n pylist = [\n \"/dir1/Python.h\", # removed from the final list\n \"/dir2/pyconfig.hpp\",\n \"/dir4/date.time.h\",\n ]\n another = HeaderList(pylist)\n h = header_list + another\n assert len(h) == 7\n\n # Invariant : l == l + l\n assert h == h + h\n\n # Always produce an instance of HeaderList\n assert type(header_list + pylist) is type(header_list)\n assert type(pylist + header_list) is type(header_list)\n\n\n#: Directory where the data for the test below is stored\nsearch_dir = os.path.join(spack.paths.test_path, \"data\", \"directory_search\")\n\n\n@pytest.mark.parametrize(\n \"lib_list,kwargs\",\n [\n ([\"liba\"], {\"shared\": True, \"recursive\": True}),\n ([\"liba\"], {\"shared\": False, \"recursive\": True}),\n ([\"libc\", \"liba\"], {\"shared\": True, \"recursive\": True}),\n ([\"liba\", \"libc\"], {\"shared\": False, \"recursive\": True}),\n ([\"libc\", \"libb\", \"liba\"], {\"shared\": True, \"recursive\": True}),\n ([\"liba\", \"libb\", \"libc\"], {\"shared\": False, \"recursive\": True}),\n ],\n)\ndef test_library_type_search(lib_list, kwargs):\n results = find_libraries(lib_list, search_dir, **kwargs)\n assert len(results) != 0\n for result in results:\n lib_type_ext = plat_shared_ext\n if not kwargs[\"shared\"]:\n lib_type_ext = plat_static_ext\n assert result.endswith(lib_type_ext) or (\n kwargs[\"shared\"] and result.endswith(plat_apple_shared_ext)\n )\n\n\n@pytest.mark.parametrize(\n \"search_fn,search_list,root,kwargs\",\n [\n (find_libraries, \"liba\", search_dir, {\"recursive\": True}),\n (find_libraries, [\"liba\"], search_dir, {\"recursive\": True}),\n (find_libraries, \"libb\", search_dir, {\"recursive\": True}),\n (find_libraries, [\"libc\"], search_dir, {\"recursive\": True}),\n (find_libraries, [\"libc\", \"liba\"], search_dir, {\"recursive\": True}),\n (find_libraries, [\"liba\", \"libc\"], search_dir, {\"recursive\": True}),\n (find_libraries, [\"libc\", \"libb\", \"liba\"], search_dir, {\"recursive\": True}),\n (find_libraries, [\"liba\", \"libc\"], search_dir, {\"recursive\": True}),\n (\n find_libraries,\n [\"libc\", \"libb\", \"liba\"],\n search_dir,\n {\"recursive\": True, \"shared\": False},\n ),\n (find_headers, \"a\", search_dir, {\"recursive\": True}),\n (find_headers, [\"a\"], search_dir, {\"recursive\": True}),\n (find_headers, \"b\", search_dir, {\"recursive\": True}),\n (find_headers, [\"c\"], search_dir, {\"recursive\": True}),\n (find_headers, [\"c\", \"a\"], search_dir, {\"recursive\": True}),\n (find_headers, [\"a\", \"c\"], search_dir, {\"recursive\": True}),\n (find_headers, [\"c\", \"b\", \"a\"], search_dir, {\"recursive\": True}),\n (find_headers, [\"a\", \"c\"], search_dir, {\"recursive\": True}),\n (find_libraries, [\"liba\", \"libd\"], os.path.join(search_dir, \"b\"), {\"recursive\": False}),\n (find_headers, [\"b\", \"d\"], os.path.join(search_dir, \"b\"), {\"recursive\": False}),\n ],\n)\ndef test_searching_order(search_fn, search_list, root, kwargs):\n # Test search\n result = search_fn(search_list, root, **kwargs)\n\n # The tests are set-up so that something is always found\n assert len(result) != 0\n\n # Now reverse the result and start discarding things\n # as soon as you have matches. In the end the list should\n # be emptied.\n rlist = list(reversed(result))\n\n # At this point make sure the search list is a sequence\n if isinstance(search_list, str):\n search_list = [search_list]\n\n # Discard entries in the order they appear in search list\n for x in search_list:\n try:\n while fnmatch.fnmatch(rlist[-1], x) or x in rlist[-1]:\n rlist.pop()\n except IndexError:\n # List is empty\n pass\n\n # List should be empty here\n assert len(rlist) == 0\n" }, { "path": "lib/spack/spack/test/llnl/util/filesystem.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Tests for ``llnl/util/filesystem.py``\"\"\"\n\nimport filecmp\nimport os\nimport pathlib\nimport shutil\nimport stat\nimport sys\nfrom contextlib import contextmanager\n\nimport pytest\n\nimport spack.llnl.util.filesystem as fs\nimport spack.paths\n\n\n@pytest.fixture()\ndef stage(tmp_path_factory: pytest.TempPathFactory):\n \"\"\"Creates a stage with the directory structure for the tests.\"\"\"\n\n s = tmp_path_factory.mktemp(\"filesystem_test\")\n\n with fs.working_dir(s):\n # Create source file hierarchy\n fs.touchp(\"source/1\")\n fs.touchp(\"source/a/b/2\")\n fs.touchp(\"source/a/b/3\")\n fs.touchp(\"source/c/4\")\n fs.touchp(\"source/c/d/5\")\n fs.touchp(\"source/c/d/6\")\n fs.touchp(\"source/c/d/e/7\")\n fs.touchp(\"source/g/h/i/8\")\n fs.touchp(\"source/g/h/i/9\")\n fs.touchp(\"source/g/i/j/10\")\n\n # Create symlinks\n fs.symlink(os.path.abspath(\"source/1\"), \"source/2\")\n fs.symlink(\"b/2\", \"source/a/b2\")\n fs.symlink(\"a/b\", \"source/f\")\n\n # Create destination directory\n fs.mkdirp(\"dest\")\n\n yield s\n\n\nclass TestCopy:\n \"\"\"Tests for ``filesystem.copy``\"\"\"\n\n def test_file_dest(self, stage):\n \"\"\"Test using a filename as the destination.\"\"\"\n\n with fs.working_dir(str(stage)):\n fs.copy(\"source/1\", \"dest/1\")\n\n assert os.path.exists(\"dest/1\")\n\n def test_dir_dest(self, stage):\n \"\"\"Test using a directory as the destination.\"\"\"\n\n with fs.working_dir(str(stage)):\n fs.copy(\"source/1\", \"dest\")\n\n assert os.path.exists(\"dest/1\")\n\n def test_glob_src(self, stage):\n \"\"\"Test using a glob as the source.\"\"\"\n\n with fs.working_dir(str(stage)):\n fs.copy(\"source/a/*/*\", \"dest\")\n\n assert os.path.exists(\"dest/2\")\n assert os.path.exists(\"dest/3\")\n\n def test_non_existing_src(self, stage):\n \"\"\"Test using a non-existing source.\"\"\"\n\n with fs.working_dir(str(stage)):\n with pytest.raises(OSError, match=\"No such file or directory\"):\n fs.copy(\"source/none\", \"dest\")\n\n def test_multiple_src_file_dest(self, stage):\n \"\"\"Test a glob that matches multiple source files and a dest\n that is not a directory.\"\"\"\n\n with fs.working_dir(str(stage)):\n match = \".* matches multiple files but .* is not a directory\"\n with pytest.raises(ValueError, match=match):\n fs.copy(\"source/a/*/*\", \"dest/1\")\n\n\ndef check_added_exe_permissions(src, dst):\n src_mode = os.stat(src).st_mode\n dst_mode = os.stat(dst).st_mode\n for perm in [stat.S_IXUSR, stat.S_IXGRP, stat.S_IXOTH]:\n if src_mode & perm:\n assert dst_mode & perm\n\n\nclass TestInstall:\n \"\"\"Tests for ``filesystem.install``\"\"\"\n\n def test_file_dest(self, stage):\n \"\"\"Test using a filename as the destination.\"\"\"\n\n with fs.working_dir(str(stage)):\n fs.install(\"source/1\", \"dest/1\")\n\n assert os.path.exists(\"dest/1\")\n check_added_exe_permissions(\"source/1\", \"dest/1\")\n\n def test_dir_dest(self, stage):\n \"\"\"Test using a directory as the destination.\"\"\"\n\n with fs.working_dir(str(stage)):\n fs.install(\"source/1\", \"dest\")\n\n assert os.path.exists(\"dest/1\")\n check_added_exe_permissions(\"source/1\", \"dest/1\")\n\n def test_glob_src(self, stage):\n \"\"\"Test using a glob as the source.\"\"\"\n\n with fs.working_dir(str(stage)):\n fs.install(\"source/a/*/*\", \"dest\")\n\n assert os.path.exists(\"dest/2\")\n assert os.path.exists(\"dest/3\")\n check_added_exe_permissions(\"source/a/b/2\", \"dest/2\")\n check_added_exe_permissions(\"source/a/b/3\", \"dest/3\")\n\n def test_non_existing_src(self, stage):\n \"\"\"Test using a non-existing source.\"\"\"\n\n with fs.working_dir(str(stage)):\n with pytest.raises(OSError, match=\"No such file or directory\"):\n fs.install(\"source/none\", \"dest\")\n\n def test_multiple_src_file_dest(self, stage):\n \"\"\"Test a glob that matches multiple source files and a dest\n that is not a directory.\"\"\"\n\n with fs.working_dir(str(stage)):\n match = \".* matches multiple files but .* is not a directory\"\n with pytest.raises(ValueError, match=match):\n fs.install(\"source/a/*/*\", \"dest/1\")\n\n\nclass TestCopyTree:\n \"\"\"Tests for ``filesystem.copy_tree``\"\"\"\n\n def test_existing_dir(self, stage):\n \"\"\"Test copying to an existing directory.\"\"\"\n\n with fs.working_dir(str(stage)):\n fs.copy_tree(\"source\", \"dest\")\n\n assert os.path.exists(\"dest/a/b/2\")\n\n def test_non_existing_dir(self, stage):\n \"\"\"Test copying to a non-existing directory.\"\"\"\n\n with fs.working_dir(str(stage)):\n fs.copy_tree(\"source\", \"dest/sub/directory\")\n\n assert os.path.exists(\"dest/sub/directory/a/b/2\")\n\n def test_symlinks_true(self, stage):\n \"\"\"Test copying with symlink preservation.\"\"\"\n\n with fs.working_dir(str(stage)):\n fs.copy_tree(\"source\", \"dest\", symlinks=True)\n\n assert os.path.exists(\"dest/2\")\n assert fs.islink(\"dest/2\")\n\n assert os.path.exists(\"dest/a/b2\")\n with fs.working_dir(\"dest/a\"):\n assert os.path.exists(fs.readlink(\"b2\"))\n\n assert os.path.realpath(\"dest/f/2\") == os.path.abspath(\"dest/a/b/2\")\n assert os.path.realpath(\"dest/2\") == os.path.abspath(\"dest/1\")\n\n def test_symlinks_true_ignore(self, stage):\n \"\"\"Test copying when specifying relative paths that should be ignored\"\"\"\n with fs.working_dir(str(stage)):\n ignore = lambda p: p in [os.path.join(\"c\", \"d\", \"e\"), \"a\"]\n fs.copy_tree(\"source\", \"dest\", symlinks=True, ignore=ignore)\n assert not os.path.exists(\"dest/a\")\n assert os.path.exists(\"dest/c/d\")\n assert not os.path.exists(\"dest/c/d/e\")\n\n def test_symlinks_false(self, stage):\n \"\"\"Test copying without symlink preservation.\"\"\"\n\n with fs.working_dir(str(stage)):\n fs.copy_tree(\"source\", \"dest\", symlinks=False)\n\n assert os.path.exists(\"dest/2\")\n if sys.platform != \"win32\":\n assert not os.path.islink(\"dest/2\")\n\n def test_glob_src(self, stage):\n \"\"\"Test using a glob as the source.\"\"\"\n\n with fs.working_dir(str(stage)):\n fs.copy_tree(\"source/g/*\", \"dest\")\n\n assert os.path.exists(\"dest/i/8\")\n assert os.path.exists(\"dest/i/9\")\n assert os.path.exists(\"dest/j/10\")\n\n def test_non_existing_src(self, stage):\n \"\"\"Test using a non-existing source.\"\"\"\n\n with fs.working_dir(str(stage)):\n with pytest.raises(OSError, match=\"No such file or directory\"):\n fs.copy_tree(\"source/none\", \"dest\")\n\n def test_parent_dir(self, stage):\n \"\"\"Test source as a parent directory of destination.\"\"\"\n\n with fs.working_dir(str(stage)):\n match = \"Cannot copy ancestor directory\"\n with pytest.raises(ValueError, match=match):\n fs.copy_tree(\"source\", \"source/sub/directory\")\n\n\nclass TestInstallTree:\n \"\"\"Tests for ``filesystem.install_tree``\"\"\"\n\n def test_existing_dir(self, stage):\n \"\"\"Test installing to an existing directory.\"\"\"\n\n with fs.working_dir(str(stage)):\n fs.install_tree(\"source\", \"dest\")\n\n assert os.path.exists(\"dest/a/b/2\")\n check_added_exe_permissions(\"source/a/b/2\", \"dest/a/b/2\")\n\n def test_non_existing_dir(self, stage):\n \"\"\"Test installing to a non-existing directory.\"\"\"\n\n with fs.working_dir(str(stage)):\n fs.install_tree(\"source\", \"dest/sub/directory\")\n\n assert os.path.exists(\"dest/sub/directory/a/b/2\")\n check_added_exe_permissions(\"source/a/b/2\", \"dest/sub/directory/a/b/2\")\n\n def test_symlinks_true(self, stage):\n \"\"\"Test installing with symlink preservation.\"\"\"\n\n with fs.working_dir(str(stage)):\n fs.install_tree(\"source\", \"dest\", symlinks=True)\n\n assert os.path.exists(\"dest/2\")\n if sys.platform != \"win32\":\n assert os.path.islink(\"dest/2\")\n check_added_exe_permissions(\"source/2\", \"dest/2\")\n\n def test_symlinks_false(self, stage):\n \"\"\"Test installing without symlink preservation.\"\"\"\n\n with fs.working_dir(str(stage)):\n fs.install_tree(\"source\", \"dest\", symlinks=False)\n\n assert os.path.exists(\"dest/2\")\n if sys.platform != \"win32\":\n assert not os.path.islink(\"dest/2\")\n check_added_exe_permissions(\"source/2\", \"dest/2\")\n\n @pytest.mark.not_on_windows(\"Broken symlinks not allowed on Windows\")\n def test_allow_broken_symlinks(self, stage):\n \"\"\"Test installing with a broken symlink.\"\"\"\n with fs.working_dir(str(stage)):\n fs.symlink(\"nonexistant.txt\", \"source/broken\")\n fs.install_tree(\"source\", \"dest\", symlinks=True)\n assert os.path.islink(\"dest/broken\")\n assert not os.path.exists(fs.readlink(\"dest/broken\"))\n\n def test_glob_src(self, stage):\n \"\"\"Test using a glob as the source.\"\"\"\n\n with fs.working_dir(str(stage)):\n fs.install_tree(\"source/g/*\", \"dest\")\n\n assert os.path.exists(\"dest/i/8\")\n assert os.path.exists(\"dest/i/9\")\n assert os.path.exists(\"dest/j/10\")\n check_added_exe_permissions(\"source/g/h/i/8\", \"dest/i/8\")\n check_added_exe_permissions(\"source/g/h/i/9\", \"dest/i/9\")\n check_added_exe_permissions(\"source/g/i/j/10\", \"dest/j/10\")\n\n def test_non_existing_src(self, stage):\n \"\"\"Test using a non-existing source.\"\"\"\n\n with fs.working_dir(str(stage)):\n with pytest.raises(OSError, match=\"No such file or directory\"):\n fs.install_tree(\"source/none\", \"dest\")\n\n def test_parent_dir(self, stage):\n \"\"\"Test source as a parent directory of destination.\"\"\"\n\n with fs.working_dir(str(stage)):\n match = \"Cannot copy ancestor directory\"\n with pytest.raises(ValueError, match=match):\n fs.install_tree(\"source\", \"source/sub/directory\")\n\n\ndef test_paths_containing_libs(dirs_with_libfiles):\n lib_to_dirs, all_dirs = dirs_with_libfiles\n\n assert set(fs.paths_containing_libs(all_dirs, [\"libgfortran\"])) == set(\n lib_to_dirs[\"libgfortran\"]\n )\n\n assert set(fs.paths_containing_libs(all_dirs, [\"libirc\"])) == set(lib_to_dirs[\"libirc\"])\n\n\ndef test_move_transaction_commit(tmp_path: pathlib.Path):\n lib_dir = tmp_path / \"lib\"\n lib_dir.mkdir()\n fake_library = lib_dir / \"libfoo.so\"\n fake_library.write_text(\"Just some fake content.\")\n\n with fs.replace_directory_transaction(str(lib_dir)) as backup:\n assert os.path.isdir(backup)\n lib_dir.mkdir()\n fake_library = lib_dir / \"libfoo.so\"\n fake_library.write_text(\"Other content.\")\n\n assert not os.path.lexists(backup)\n with open(str(lib_dir / \"libfoo.so\"), \"r\", encoding=\"utf-8\") as f:\n assert \"Other content.\" == f.read()\n\n\ndef test_move_transaction_rollback(tmp_path: pathlib.Path):\n lib_dir = tmp_path / \"lib\"\n lib_dir.mkdir()\n fake_library = lib_dir / \"libfoo.so\"\n fake_library.write_text(\"Initial content.\")\n\n try:\n with fs.replace_directory_transaction(str(lib_dir)) as backup:\n assert os.path.isdir(backup)\n lib_dir.mkdir()\n fake_library = lib_dir / \"libfoo.so\"\n fake_library.write_text(\"New content.\")\n raise RuntimeError(\"\")\n except RuntimeError:\n pass\n\n assert not os.path.lexists(backup)\n with open(str(lib_dir / \"libfoo.so\"), \"r\", encoding=\"utf-8\") as f:\n assert \"Initial content.\" == f.read()\n\n\n@pytest.mark.regression(\"10601\")\n@pytest.mark.regression(\"10603\")\ndef test_recursive_search_of_headers_from_prefix(installation_dir_with_headers):\n # Try to inspect recursively from and ensure we don't get\n # subdirectories of the '/include' path\n prefix = str(installation_dir_with_headers)\n header_list = fs.find_all_headers(prefix)\n\n include_dirs = header_list.directories\n\n if sys.platform == \"win32\":\n header_list = [header.replace(\"/\", \"\\\\\") for header in header_list]\n include_dirs = [dir.replace(\"/\", \"\\\\\") for dir in include_dirs]\n\n # Check that the header files we expect are all listed\n assert os.path.join(prefix, \"include\", \"ex3.h\") in header_list\n assert os.path.join(prefix, \"include\", \"boost\", \"ex3.h\") in header_list\n assert os.path.join(prefix, \"path\", \"to\", \"ex1.h\") in header_list\n assert os.path.join(prefix, \"path\", \"to\", \"subdir\", \"ex2.h\") in header_list\n\n # Check that when computing directories we exclude /include/boost\n assert os.path.join(prefix, \"include\") in include_dirs\n assert os.path.join(prefix, \"include\", \"boost\") not in include_dirs\n assert os.path.join(prefix, \"path\", \"to\") in include_dirs\n assert os.path.join(prefix, \"path\", \"to\", \"subdir\") in include_dirs\n\n\nif sys.platform == \"win32\":\n dir_list = [\n ([\"C:/pfx/include/foo.h\", \"C:/pfx/include/subdir/foo.h\"], [\"C:/pfx/include\"]),\n ([\"C:/pfx/include/foo.h\", \"C:/pfx/subdir/foo.h\"], [\"C:/pfx/include\", \"C:/pfx/subdir\"]),\n (\n [\"C:/pfx/include/subdir/foo.h\", \"C:/pfx/subdir/foo.h\"],\n [\"C:/pfx/include\", \"C:/pfx/subdir\"],\n ),\n ]\nelse:\n dir_list = [\n ([\"/pfx/include/foo.h\", \"/pfx/include/subdir/foo.h\"], [\"/pfx/include\"]),\n ([\"/pfx/include/foo.h\", \"/pfx/subdir/foo.h\"], [\"/pfx/include\", \"/pfx/subdir\"]),\n ([\"/pfx/include/subdir/foo.h\", \"/pfx/subdir/foo.h\"], [\"/pfx/include\", \"/pfx/subdir\"]),\n ]\n\n\n@pytest.mark.parametrize(\"list_of_headers,expected_directories\", dir_list)\ndef test_computation_of_header_directories(list_of_headers, expected_directories):\n hl = fs.HeaderList(list_of_headers)\n assert hl.directories == expected_directories\n\n\ndef test_headers_directory_setter():\n if sys.platform == \"win32\":\n root = r\"C:\\pfx\\include\\subdir\"\n else:\n root = \"/pfx/include/subdir\"\n hl = fs.HeaderList([root + \"/foo.h\", root + \"/bar.h\"])\n\n # Set directories using a list\n hl.directories = [root]\n assert hl.directories == [root]\n\n # If it's a single directory it's fine to not wrap it into a list\n # when setting the property\n hl.directories = root\n assert hl.directories == [root]\n\n # Paths are normalized, so it doesn't matter how many backslashes etc.\n # are present in the original directory being used\n if sys.platform == \"win32\":\n # TODO: Test with \\\\'s\n hl.directories = \"C:/pfx/include//subdir\"\n else:\n hl.directories = \"/pfx/include//subdir/\"\n assert hl.directories == [root]\n\n # Setting an empty list is allowed and returns an empty list\n hl.directories = []\n assert hl.directories == []\n\n # Setting directories to None also returns an empty list\n hl.directories = None\n assert hl.directories == []\n\n\nif sys.platform == \"win32\":\n # TODO: Test \\\\s\n paths = [\n (r\"C:\\user\\root\", None, ([\"C:\\\\\", r\"C:\\user\", r\"C:\\user\\root\"], \"\", [])),\n (r\"C:\\user\\root\", \"C:\\\\\", ([], \"C:\\\\\", [r\"C:\\user\", r\"C:\\user\\root\"])),\n (r\"C:\\user\\root\", r\"user\", ([\"C:\\\\\"], r\"C:\\user\", [r\"C:\\user\\root\"])),\n (r\"C:\\user\\root\", r\"root\", ([\"C:\\\\\", r\"C:\\user\"], r\"C:\\user\\root\", [])),\n (r\"relative\\path\", None, ([r\"relative\", r\"relative\\path\"], \"\", [])),\n (r\"relative\\path\", r\"relative\", ([], r\"relative\", [r\"relative\\path\"])),\n (r\"relative\\path\", r\"path\", ([r\"relative\"], r\"relative\\path\", [])),\n ]\nelse:\n paths = [\n (\"/tmp/user/root\", None, ([\"/tmp\", \"/tmp/user\", \"/tmp/user/root\"], \"\", [])),\n (\"/tmp/user/root\", \"tmp\", ([], \"/tmp\", [\"/tmp/user\", \"/tmp/user/root\"])),\n (\"/tmp/user/root\", \"user\", ([\"/tmp\"], \"/tmp/user\", [\"/tmp/user/root\"])),\n (\"/tmp/user/root\", \"root\", ([\"/tmp\", \"/tmp/user\"], \"/tmp/user/root\", [])),\n (\"relative/path\", None, ([\"relative\", \"relative/path\"], \"\", [])),\n (\"relative/path\", \"relative\", ([], \"relative\", [\"relative/path\"])),\n (\"relative/path\", \"path\", ([\"relative\"], \"relative/path\", [])),\n ]\n\n\n@pytest.mark.parametrize(\"path,entry,expected\", paths)\ndef test_partition_path(path, entry, expected):\n assert fs.partition_path(path, entry) == expected\n\n\nif sys.platform == \"win32\":\n path_list = [\n (\"\", []),\n (r\".\\some\\sub\\dir\", [r\".\\some\", r\".\\some\\sub\", r\".\\some\\sub\\dir\"]),\n (r\"another\\sub\\dir\", [r\"another\", r\"another\\sub\", r\"another\\sub\\dir\"]),\n ]\nelse:\n path_list = [\n (\"\", []),\n (\"/tmp/user/dir\", [\"/tmp\", \"/tmp/user\", \"/tmp/user/dir\"]),\n (\"./some/sub/dir\", [\"./some\", \"./some/sub\", \"./some/sub/dir\"]),\n (\"another/sub/dir\", [\"another\", \"another/sub\", \"another/sub/dir\"]),\n ]\n\n\n@pytest.mark.parametrize(\"path,expected\", path_list)\ndef test_prefixes(path, expected):\n assert fs.prefixes(path) == expected\n\n\n@pytest.mark.regression(\"7358\")\n@pytest.mark.parametrize(\n \"regex,replacement,filename,keyword_args\",\n [\n (r\"\\\", \"\", \"x86_cpuid_info.c\", {}),\n (r\"CDIR\", \"CURRENT_DIRECTORY\", \"selfextract.bsx\", {\"stop_at\": \"__ARCHIVE_BELOW__\"}),\n ],\n)\ndef test_filter_files_with_different_encodings(\n regex, replacement, filename, tmp_path: pathlib.Path, keyword_args\n):\n # All files given as input to this test must satisfy the pre-requisite\n # that the 'replacement' string is not present in the file initially and\n # that there's at least one match for the regex\n original_file = os.path.join(spack.paths.test_path, \"data\", \"filter_file\", filename)\n target_file = os.path.join(str(tmp_path), filename)\n shutil.copy(original_file, target_file)\n # This should not raise exceptions\n fs.filter_file(regex, replacement, target_file, **keyword_args)\n # Check the strings have been replaced\n with open(target_file, mode=\"r\", encoding=\"utf-8\", errors=\"surrogateescape\") as f:\n assert replacement in f.read()\n\n\n@pytest.mark.not_on_windows(\"chgrp isn't used on Windows\")\ndef test_chgrp_dont_set_group_if_already_set(tmp_path: pathlib.Path, monkeypatch):\n with fs.working_dir(str(tmp_path)):\n os.mkdir(\"test-dir_chgrp_dont_set_group_if_already_set\")\n\n def _fail(*args, **kwargs):\n raise Exception(\"chrgrp should not be called\")\n\n class FakeStat(object):\n def __init__(self, gid):\n self.st_gid = gid\n\n original_stat = os.stat\n\n def _stat(*args, **kwargs):\n path = args[0]\n if path == \"test-dir_chgrp_dont_set_group_if_already_set\":\n return FakeStat(gid=1001)\n else:\n # Monkeypatching stat can interfere with post-test cleanup, so for\n # paths that aren't part of the test, we want the original behavior\n # of stat\n return original_stat(*args, **kwargs)\n\n monkeypatch.setattr(os, \"chown\", _fail)\n monkeypatch.setattr(os, \"lchown\", _fail)\n monkeypatch.setattr(os, \"stat\", _stat)\n\n with fs.working_dir(str(tmp_path)):\n with pytest.raises(Exception):\n fs.chgrp(\"test-dir_chgrp_dont_set_group_if_already_set\", 1002)\n fs.chgrp(\"test-dir_chgrp_dont_set_group_if_already_set\", 1001)\n\n\ndef test_filter_files_multiple(tmp_path: pathlib.Path):\n # All files given as input to this test must satisfy the pre-requisite\n # that the 'replacement' string is not present in the file initially and\n # that there's at least one match for the regex\n original_file = os.path.join(spack.paths.test_path, \"data\", \"filter_file\", \"x86_cpuid_info.c\")\n target_file = os.path.join(str(tmp_path), \"x86_cpuid_info.c\")\n shutil.copy(original_file, target_file)\n # This should not raise exceptions\n fs.filter_file(r\"\\\", \"\", target_file)\n fs.filter_file(r\"\\\", \"\", target_file)\n fs.filter_file(r\"\\\", \"\", target_file)\n # Check the strings have been replaced\n with open(target_file, mode=\"r\", encoding=\"utf-8\", errors=\"surrogateescape\") as f:\n assert \"\" not in f.read()\n assert \"\" not in f.read()\n assert \"\" not in f.read()\n\n\ndef test_filter_files_start_stop(tmp_path: pathlib.Path):\n original_file = os.path.join(spack.paths.test_path, \"data\", \"filter_file\", \"start_stop.txt\")\n target_file = os.path.join(str(tmp_path), \"start_stop.txt\")\n shutil.copy(original_file, target_file)\n # None of the following should happen:\n # - filtering starts after A is found in the file:\n fs.filter_file(\"A\", \"X\", target_file, string=True, start_at=\"B\")\n # - filtering starts exactly when B is found:\n fs.filter_file(\"B\", \"X\", target_file, string=True, start_at=\"B\")\n # - filtering stops before D is found:\n fs.filter_file(\"D\", \"X\", target_file, string=True, stop_at=\"C\")\n\n assert filecmp.cmp(original_file, target_file)\n\n # All of the following should happen:\n fs.filter_file(\"A\", \"X\", target_file, string=True)\n fs.filter_file(\"B\", \"X\", target_file, string=True, start_at=\"X\", stop_at=\"C\")\n fs.filter_file(r\"C|D\", \"X\", target_file, start_at=\"X\", stop_at=\"E\")\n\n with open(target_file, mode=\"r\", encoding=\"utf-8\") as f:\n assert all(\"X\" == line.strip() for line in f.readlines())\n\n\n# Each test input is a tuple of entries which prescribe\n# - the 'subdirs' to be created from tmp_path\n# - the 'files' in that directory\n# - what is to be removed\n@pytest.mark.parametrize(\n \"files_or_dirs\",\n [\n # Remove a file over the two that are present\n [{\"subdirs\": None, \"files\": [\"spack.lock\", \"spack.yaml\"], \"remove\": [\"spack.lock\"]}],\n # Remove the entire directory where two files are stored\n [{\"subdirs\": \"myenv\", \"files\": [\"spack.lock\", \"spack.yaml\"], \"remove\": [\"myenv\"]}],\n # Combine a mix of directories and files\n [\n {\"subdirs\": None, \"files\": [\"spack.lock\", \"spack.yaml\"], \"remove\": [\"spack.lock\"]},\n {\"subdirs\": \"myenv\", \"files\": [\"spack.lock\", \"spack.yaml\"], \"remove\": [\"myenv\"]},\n ],\n # Multiple subdirectories, remove root\n [\n {\"subdirs\": \"work/myenv1\", \"files\": [\"spack.lock\", \"spack.yaml\"], \"remove\": []},\n {\"subdirs\": \"work/myenv2\", \"files\": [\"spack.lock\", \"spack.yaml\"], \"remove\": [\"work\"]},\n ],\n # Multiple subdirectories, remove each one\n [\n {\n \"subdirs\": \"work/myenv1\",\n \"files\": [\"spack.lock\", \"spack.yaml\"],\n \"remove\": [\"work/myenv1\"],\n },\n {\n \"subdirs\": \"work/myenv2\",\n \"files\": [\"spack.lock\", \"spack.yaml\"],\n \"remove\": [\"work/myenv2\"],\n },\n ],\n # Remove files with the same name in different directories\n [\n {\n \"subdirs\": \"work/myenv1\",\n \"files\": [\"spack.lock\", \"spack.yaml\"],\n \"remove\": [\"work/myenv1/spack.lock\"],\n },\n {\n \"subdirs\": \"work/myenv2\",\n \"files\": [\"spack.lock\", \"spack.yaml\"],\n \"remove\": [\"work/myenv2/spack.lock\"],\n },\n ],\n # Remove first the directory, then a file within the directory\n [\n {\n \"subdirs\": \"myenv\",\n \"files\": [\"spack.lock\", \"spack.yaml\"],\n \"remove\": [\"myenv\", \"myenv/spack.lock\"],\n }\n ],\n # Remove first a file within a directory, then the directory\n [\n {\n \"subdirs\": \"myenv\",\n \"files\": [\"spack.lock\", \"spack.yaml\"],\n \"remove\": [\"myenv/spack.lock\", \"myenv\"],\n }\n ],\n ],\n)\n@pytest.mark.regression(\"18441\")\ndef test_safe_remove(files_or_dirs, tmp_path: pathlib.Path):\n # Create a fake directory structure as prescribed by test input\n to_be_removed, to_be_checked = [], []\n for entry in files_or_dirs:\n # Create relative dir\n subdirs = entry[\"subdirs\"]\n if not subdirs:\n dir_path = tmp_path\n else:\n dir_path = tmp_path\n for subdir in subdirs.split(\"/\"):\n dir_path = dir_path / subdir\n dir_path.mkdir(parents=True, exist_ok=True)\n\n # Create files in the directory\n files = entry[\"files\"]\n for f in files:\n abspath = str(dir_path / f)\n to_be_checked.append(abspath)\n fs.touch(abspath)\n\n # List of things to be removed\n for r in entry[\"remove\"]:\n to_be_removed.append(str(tmp_path / r))\n\n # Assert that files are deleted in the context block,\n # mock a failure by raising an exception\n with pytest.raises(RuntimeError):\n with fs.safe_remove(*to_be_removed):\n for entry in to_be_removed:\n assert not os.path.exists(entry)\n raise RuntimeError(\"Mock a failure\")\n\n # Assert that files are restored\n for entry in to_be_checked:\n assert os.path.exists(entry)\n\n\n@pytest.mark.regression(\"18441\")\ndef test_content_of_files_with_same_name(tmp_path: pathlib.Path):\n # Create two subdirectories containing a file with the same name,\n # differentiate the files by their content\n myenv1_dir = tmp_path / \"myenv1\"\n myenv1_dir.mkdir()\n file1 = myenv1_dir / \"spack.lock\"\n file1.write_text(\"file1\")\n\n myenv2_dir = tmp_path / \"myenv2\"\n myenv2_dir.mkdir()\n file2 = myenv2_dir / \"spack.lock\"\n file2.write_text(\"file2\")\n\n # Use 'safe_remove' to remove the two files\n with pytest.raises(RuntimeError):\n with fs.safe_remove(str(file1), str(file2)):\n raise RuntimeError(\"Mock a failure\")\n\n # Check both files have been restored correctly\n # and have not been mixed\n assert file1.read_text().strip() == \"file1\"\n assert file2.read_text().strip() == \"file2\"\n\n\ndef test_keep_modification_time(tmp_path: pathlib.Path):\n file1 = tmp_path / \"file1\"\n file1.touch()\n file2 = tmp_path / \"file2\"\n file2.touch()\n\n # Shift the modification time of the file 10 seconds back:\n stat1 = file1.stat()\n mtime1 = stat1.st_mtime - 10\n os.utime(file1, (stat1.st_atime, mtime1))\n\n with fs.keep_modification_time(str(file1), str(file2), \"non-existing-file\"):\n file1.write_text(\"file1\")\n file2.unlink()\n\n # Assert that the modifications took place the modification time has not\n # changed;\n assert file1.read_text().strip() == \"file1\"\n assert not file2.exists()\n assert int(mtime1) == int(file1.stat().st_mtime)\n\n\ndef test_temporary_dir_context_manager():\n previous_dir = os.path.realpath(os.getcwd())\n with fs.temporary_dir() as tmp_dir:\n assert previous_dir != os.path.realpath(os.getcwd())\n assert os.path.realpath(str(tmp_dir)) == os.path.realpath(os.getcwd())\n\n\n@pytest.mark.not_on_windows(\"No shebang on Windows\")\ndef test_is_nonsymlink_exe_with_shebang(tmp_path: pathlib.Path):\n with fs.working_dir(str(tmp_path)):\n # Create an executable with shebang.\n with open(\"executable_script\", \"wb\") as f:\n f.write(b\"#!/interpreter\")\n os.chmod(\"executable_script\", 0o100775)\n\n with open(\"executable_but_not_script\", \"wb\") as f:\n f.write(b\"#/not-a-shebang\")\n os.chmod(\"executable_but_not_script\", 0o100775)\n\n with open(\"not_executable_with_shebang\", \"wb\") as f:\n f.write(b\"#!/interpreter\")\n os.chmod(\"not_executable_with_shebang\", 0o100664)\n\n os.symlink(\"executable_script\", \"symlink_to_executable_script\")\n\n assert fs.is_nonsymlink_exe_with_shebang(\"executable_script\")\n assert not fs.is_nonsymlink_exe_with_shebang(\"executable_but_not_script\")\n assert not fs.is_nonsymlink_exe_with_shebang(\"not_executable_with_shebang\")\n assert not fs.is_nonsymlink_exe_with_shebang(\"symlink_to_executable_script\")\n\n\nclass RegisterVisitor(fs.BaseDirectoryVisitor):\n \"\"\"A directory visitor that keeps track of all visited paths\"\"\"\n\n def __init__(self, root, follow_dirs=True, follow_symlink_dirs=True):\n self.files = []\n self.dirs_before = []\n self.symlinked_dirs_before = []\n self.dirs_after = []\n self.symlinked_dirs_after = []\n\n self.root = root\n self.follow_dirs = follow_dirs\n self.follow_symlink_dirs = follow_symlink_dirs\n\n def check(self, root, rel_path, depth):\n # verify the (root, rel_path, depth) make sense.\n assert root == self.root and depth + 1 == len(rel_path.split(os.sep))\n\n def visit_file(self, root, rel_path, depth):\n self.check(root, rel_path, depth)\n self.files.append(rel_path)\n\n def visit_symlinked_file(self, root, rel_path, depth):\n self.visit_file(root, rel_path, depth)\n\n def before_visit_dir(self, root, rel_path, depth):\n self.check(root, rel_path, depth)\n self.dirs_before.append(rel_path)\n return self.follow_dirs\n\n def before_visit_symlinked_dir(self, root, rel_path, depth):\n self.check(root, rel_path, depth)\n self.symlinked_dirs_before.append(rel_path)\n return self.follow_symlink_dirs\n\n def after_visit_dir(self, root, rel_path, depth):\n self.check(root, rel_path, depth)\n self.dirs_after.append(rel_path)\n\n def after_visit_symlinked_dir(self, root, rel_path, depth):\n self.check(root, rel_path, depth)\n self.symlinked_dirs_after.append(rel_path)\n\n\n@pytest.mark.not_on_windows(\"Requires symlinks\")\ndef test_visit_directory_tree_follow_all(noncyclical_dir_structure):\n root = str(noncyclical_dir_structure)\n visitor = RegisterVisitor(root, follow_dirs=True, follow_symlink_dirs=True)\n fs.visit_directory_tree(root, visitor)\n j = os.path.join\n assert visitor.files == [\n j(\"a\", \"file_1\"),\n j(\"a\", \"to_c\", \"dangling_link\"),\n j(\"a\", \"to_c\", \"file_2\"),\n j(\"a\", \"to_file_1\"),\n j(\"b\", \"file_1\"),\n j(\"b\", \"to_c\", \"dangling_link\"),\n j(\"b\", \"to_c\", \"file_2\"),\n j(\"b\", \"to_file_1\"),\n j(\"c\", \"dangling_link\"),\n j(\"c\", \"file_2\"),\n j(\"file_3\"),\n ]\n assert visitor.dirs_before == [j(\"a\"), j(\"a\", \"d\"), j(\"b\", \"d\"), j(\"c\")]\n assert visitor.dirs_after == [j(\"a\", \"d\"), j(\"a\"), j(\"b\", \"d\"), j(\"c\")]\n assert visitor.symlinked_dirs_before == [j(\"a\", \"to_c\"), j(\"b\"), j(\"b\", \"to_c\")]\n assert visitor.symlinked_dirs_after == [j(\"a\", \"to_c\"), j(\"b\", \"to_c\"), j(\"b\")]\n\n\n@pytest.mark.not_on_windows(\"Requires symlinks\")\ndef test_visit_directory_tree_follow_dirs(noncyclical_dir_structure):\n root = str(noncyclical_dir_structure)\n visitor = RegisterVisitor(root, follow_dirs=True, follow_symlink_dirs=False)\n fs.visit_directory_tree(root, visitor)\n j = os.path.join\n assert visitor.files == [\n j(\"a\", \"file_1\"),\n j(\"a\", \"to_file_1\"),\n j(\"c\", \"dangling_link\"),\n j(\"c\", \"file_2\"),\n j(\"file_3\"),\n ]\n assert visitor.dirs_before == [j(\"a\"), j(\"a\", \"d\"), j(\"c\")]\n assert visitor.dirs_after == [j(\"a\", \"d\"), j(\"a\"), j(\"c\")]\n assert visitor.symlinked_dirs_before == [j(\"a\", \"to_c\"), j(\"b\")]\n assert not visitor.symlinked_dirs_after\n\n\n@pytest.mark.not_on_windows(\"Requires symlinks\")\ndef test_visit_directory_tree_follow_none(noncyclical_dir_structure):\n root = str(noncyclical_dir_structure)\n visitor = RegisterVisitor(root, follow_dirs=False, follow_symlink_dirs=False)\n fs.visit_directory_tree(root, visitor)\n j = os.path.join\n assert visitor.files == [j(\"file_3\")]\n assert visitor.dirs_before == [j(\"a\"), j(\"c\")]\n assert not visitor.dirs_after\n assert visitor.symlinked_dirs_before == [j(\"b\")]\n assert not visitor.symlinked_dirs_after\n\n\n@pytest.mark.regression(\"29687\")\n@pytest.mark.parametrize(\"initial_mode\", [stat.S_IRUSR | stat.S_IXUSR, stat.S_IWGRP])\n@pytest.mark.not_on_windows(\"Windows might change permissions\")\ndef test_remove_linked_tree_doesnt_change_file_permission(tmp_path: pathlib.Path, initial_mode):\n # Here we test that a failed call to remove_linked_tree, due to passing a file\n # as an argument instead of a directory, doesn't leave the file with different\n # permissions as a side effect of trying to handle the error.\n file_instead_of_dir = tmp_path / \"foo\"\n file_instead_of_dir.touch()\n file_instead_of_dir.chmod(initial_mode)\n initial_stat = os.stat(str(file_instead_of_dir))\n fs.remove_linked_tree(str(file_instead_of_dir))\n final_stat = os.stat(str(file_instead_of_dir))\n assert final_stat == initial_stat\n\n\ndef test_filesummary(tmp_path: pathlib.Path):\n p = str(tmp_path / \"xyz\")\n with open(p, \"wb\") as f:\n f.write(b\"abcdefghijklmnopqrstuvwxyz\")\n\n assert fs.filesummary(p, print_bytes=8) == (26, b\"abcdefgh...stuvwxyz\")\n assert fs.filesummary(p, print_bytes=13) == (26, b\"abcdefghijklmnopqrstuvwxyz\")\n assert fs.filesummary(p, print_bytes=100) == (26, b\"abcdefghijklmnopqrstuvwxyz\")\n\n\n@pytest.mark.parametrize(\"bfs_depth\", [1, 2, 10])\ndef test_find_first_file(tmp_path: pathlib.Path, bfs_depth):\n # Create a structure: a/a/a/{file1,file2}, b/a, c/a, d/{a,file1}\n (tmp_path / \"a\" / \"a\" / \"a\").mkdir(parents=True)\n (tmp_path / \"b\" / \"a\").mkdir(parents=True)\n (tmp_path / \"c\" / \"a\").mkdir(parents=True)\n (tmp_path / \"d\" / \"a\").mkdir(parents=True)\n (tmp_path / \"e\").mkdir()\n\n fs.touch(str(tmp_path / \"a\" / \"a\" / \"a\" / \"file1\"))\n fs.touch(str(tmp_path / \"a\" / \"a\" / \"a\" / \"file2\"))\n fs.touch(str(tmp_path / \"d\" / \"file1\"))\n\n root = str(tmp_path)\n\n # Iterative deepening: should find low-depth file1.\n f1 = fs.find_first(root, \"file*\", bfs_depth=bfs_depth)\n assert f1 is not None and os.path.samefile(f1, os.path.join(root, \"d\", \"file1\"))\n\n assert fs.find_first(root, \"nonexisting\", bfs_depth=bfs_depth) is None\n\n f2 = fs.find_first(root, [\"nonexisting\", \"file2\"], bfs_depth=bfs_depth)\n assert f2 is not None and os.path.samefile(f2, os.path.join(root, \"a\", \"a\", \"a\", \"file2\"))\n\n # Should find first dir\n f3 = fs.find_first(root, \"a\", bfs_depth=bfs_depth)\n assert f3 is not None and os.path.samefile(f3, os.path.join(root, \"a\"))\n\n\ndef test_rename_dest_exists(tmp_path: pathlib.Path):\n @contextmanager\n def setup_test_files():\n a_dir = tmp_path / \"a\"\n a_dir.mkdir()\n a = a_dir / \"file1\"\n b = a_dir / \"file2\"\n fs.touchp(str(a))\n fs.touchp(str(b))\n with open(a, \"w\", encoding=\"utf-8\") as oa, open(b, \"w\", encoding=\"utf-8\") as ob:\n oa.write(\"I am A\")\n ob.write(\"I am B\")\n yield a, b\n shutil.rmtree(str(a_dir))\n\n @contextmanager\n def setup_test_dirs():\n d_dir = tmp_path / \"d\"\n d_dir.mkdir()\n a = d_dir / \"a\"\n b = d_dir / \"b\"\n fs.mkdirp(str(a))\n fs.mkdirp(str(b))\n yield a, b\n shutil.rmtree(str(d_dir))\n\n # test standard behavior of rename\n # smoke test\n with setup_test_files() as files:\n a, b = files\n fs.rename(str(a), str(b))\n assert os.path.exists(b)\n assert not os.path.exists(a)\n with open(b, \"r\", encoding=\"utf-8\") as ob:\n content = ob.read()\n assert content == \"I am A\"\n\n # test relatitve paths\n # another sanity check/smoke test\n with setup_test_files() as files:\n a, b = files\n with fs.working_dir(str(tmp_path)):\n fs.rename(os.path.join(\"a\", \"file1\"), os.path.join(\"a\", \"file2\"))\n assert os.path.exists(b)\n assert not os.path.exists(a)\n with open(b, \"r\", encoding=\"utf-8\") as ob:\n content = ob.read()\n assert content == \"I am A\"\n\n # Test rename symlinks to same file\n a_dir = tmp_path / \"a\"\n a_dir.mkdir()\n c = a_dir / \"file1\"\n a = a_dir / \"link1\"\n b = a_dir / \"link2\"\n fs.touchp(str(c))\n fs.symlink(str(c), str(a))\n fs.symlink(str(c), str(b))\n fs.rename(str(a), str(b))\n assert os.path.exists(b)\n assert not os.path.exists(a)\n assert os.path.realpath(str(b)) == str(c)\n shutil.rmtree(str(a_dir))\n\n # test rename onto itself\n a_dir = tmp_path / \"a\"\n a_dir.mkdir()\n a = a_dir / \"file1\"\n b = a\n fs.touchp(str(a))\n with open(a, \"w\", encoding=\"utf-8\") as oa:\n oa.write(\"I am A\")\n fs.rename(str(a), str(b))\n # check a, or b, doesn't matter, same file\n assert os.path.exists(a)\n # ensure original file was not duplicated\n assert len(os.listdir(str(a_dir))) == 1\n with open(a, \"r\", encoding=\"utf-8\") as oa:\n assert oa.read()\n shutil.rmtree(str(a_dir))\n\n # test rename onto symlink\n # to directory from symlink to directory\n # (this is something spack does when regenerating views)\n with setup_test_dirs() as dirs:\n a, b = dirs\n f_dir = tmp_path / \"f\"\n f_dir.mkdir()\n link1 = f_dir / \"link1\"\n link2 = f_dir / \"link2\"\n fs.symlink(str(a), str(link1))\n fs.symlink(str(b), str(link2))\n fs.rename(str(link1), str(link2))\n assert os.path.exists(link2)\n assert os.path.realpath(str(link2)) == str(a)\n shutil.rmtree(str(f_dir))\n\n\n@pytest.mark.only_windows(\"Test is for Windows specific behavior\")\ndef test_windows_sfn(tmp_path: pathlib.Path):\n # first check some standard Windows locations\n # we know require sfn names\n # this is basically a smoke test\n # ensure spaces are replaced + path abbreviated\n assert fs.windows_sfn(\"C:\\\\Program Files (x86)\") == \"C:\\\\PROGRA~2\"\n # ensure path without spaces is still properly shortened\n assert fs.windows_sfn(\"C:\\\\ProgramData\") == \"C:\\\\PROGRA~3\"\n\n # test user created paths\n # ensure longer path with spaces is properly abbreviated\n a = tmp_path / \"d\" / \"this is a test\" / \"a\" / \"still test\"\n # ensure longer path is properly abbreviated\n b = tmp_path / \"d\" / \"long_path_with_no_spaces\" / \"more_long_path\"\n # ensure path not in need of abbreviation is properly roundtripped\n c = tmp_path / \"d\" / \"this\" / \"is\" / \"short\"\n # ensure paths that are the same in the first six letters\n # are incremented post tilde\n d = tmp_path / \"d\" / \"longerpath1\"\n e = tmp_path / \"d\" / \"longerpath2\"\n fs.mkdirp(str(a))\n fs.mkdirp(str(b))\n fs.mkdirp(str(c))\n fs.mkdirp(str(d))\n fs.mkdirp(str(e))\n # check only for path of path we can control,\n # pytest prefix may or may not be mangled by windows_sfn\n # based on user/pytest config\n assert \"d\\\\THISIS~1\\\\a\\\\STILLT~1\" in fs.windows_sfn(str(a))\n assert \"d\\\\LONG_P~1\\\\MORE_L~1\" in fs.windows_sfn(str(b))\n assert \"d\\\\this\\\\is\\\\short\" in fs.windows_sfn(str(c))\n assert \"d\\\\LONGER~1\" in fs.windows_sfn(str(d))\n assert \"d\\\\LONGER~2\" in fs.windows_sfn(str(e))\n shutil.rmtree(str(tmp_path / \"d\"))\n\n\n@pytest.fixture\ndef dir_structure_with_things_to_find(tmp_path_factory: pytest.TempPathFactory):\n \"\"\"\n /\n dir_one/\n file_one\n dir_two/\n dir_three/\n dir_four/\n file_two\n file_three\n file_four\n \"\"\"\n tmp_main_dir = tmp_path_factory.mktemp(\"dir_structure_with_things_to_find\")\n dir_one = tmp_main_dir / \"dir_one\"\n dir_one.mkdir()\n dir_two = tmp_main_dir / \"dir_two\"\n dir_two.mkdir()\n dir_three = tmp_main_dir / \"dir_three\"\n dir_three.mkdir()\n dir_four = dir_three / \"dir_four\"\n dir_four.mkdir()\n\n locations = {}\n file_one = dir_one / \"file_one\"\n file_one.touch()\n locations[\"file_one\"] = str(file_one)\n\n file_two = dir_four / \"file_two\"\n file_two.touch()\n locations[\"file_two\"] = str(file_two)\n\n file_three = dir_three / \"file_three\"\n file_three.touch()\n locations[\"file_three\"] = str(file_three)\n\n file_four = tmp_main_dir / \"file_four\"\n file_four.touch()\n locations[\"file_four\"] = str(file_four)\n\n return str(tmp_main_dir), locations\n\n\ndef test_find_path_glob_matches(dir_structure_with_things_to_find):\n root, locations = dir_structure_with_things_to_find\n # both file name and path match\n assert (\n fs.find(root, \"file_two\")\n == fs.find(root, \"*/*/file_two\")\n == fs.find(root, \"dir_t*/*/*two\")\n == [locations[\"file_two\"]]\n )\n # ensure that * does not match directory separators\n assert fs.find(root, \"dir*file_two\") == []\n # ensure that file name matches after / are matched from the start of the file name\n assert fs.find(root, \"*/ile_two\") == []\n # file name matches exist, but not with these paths\n assert fs.find(root, \"dir_one/*/*two\") == fs.find(root, \"*/*/*/*/file_two\") == []\n\n\ndef test_find_max_depth(dir_structure_with_things_to_find):\n root, locations = dir_structure_with_things_to_find\n\n # Make sure the paths we use to verify are absolute\n assert os.path.isabs(locations[\"file_one\"])\n\n assert set(fs.find(root, \"file_*\", max_depth=0)) == {locations[\"file_four\"]}\n assert set(fs.find(root, \"file_*\", max_depth=1)) == {\n locations[\"file_one\"],\n locations[\"file_three\"],\n locations[\"file_four\"],\n }\n assert set(fs.find(root, \"file_two\", max_depth=2)) == {locations[\"file_two\"]}\n assert not set(fs.find(root, \"file_two\", max_depth=1))\n assert set(fs.find(root, \"file_two\")) == {locations[\"file_two\"]}\n assert set(fs.find(root, \"file_*\")) == set(locations.values())\n\n\ndef test_find_max_depth_relative(dir_structure_with_things_to_find):\n \"\"\"find_max_depth should return absolute paths even if the provided path is relative.\"\"\"\n root, locations = dir_structure_with_things_to_find\n with fs.working_dir(root):\n assert set(fs.find(\".\", \"file_*\", max_depth=0)) == {locations[\"file_four\"]}\n assert set(fs.find(\".\", \"file_two\", max_depth=2)) == {locations[\"file_two\"]}\n\n\n@pytest.mark.parametrize(\"recursive,max_depth\", [(False, -1), (False, 1)])\ndef test_max_depth_and_recursive_errors(tmp_path: pathlib.Path, recursive, max_depth):\n root = str(tmp_path)\n error_str = \"cannot be set if recursive is False\"\n with pytest.raises(ValueError, match=error_str):\n fs.find(root, [\"some_file\"], recursive=recursive, max_depth=max_depth)\n\n with pytest.raises(ValueError, match=error_str):\n fs.find_libraries([\"some_lib\"], root, recursive=recursive, max_depth=max_depth)\n\n\n@pytest.fixture(params=[True, False])\ndef complex_dir_structure(request, tmp_path_factory: pytest.TempPathFactory):\n \"\"\"\n \"lx-dy\" means \"level x, directory y\"\n \"lx-fy\" means \"level x, file y\"\n \"lx-sy\" means \"level x, symlink y\"\n\n /\n l1-d1/\n l2-d1/\n l3-d2/\n l4-f1\n l3-d4/\n l4-f2\n l3-s1 -> l1-d2 # points to directory above l2-d1\n l3-s3 -> l1-d1 # cyclic link\n l1-d2/\n l2-d2/\n l3-f3\n l2-f1\n l2-s3 -> l2-d2\n l1-s3 -> l3-d4 # a link that \"skips\" a directory level\n l1-s4 -> l2-s3 # a link to a link to a dir\n \"\"\"\n use_junctions = request.param\n if sys.platform == \"win32\" and not use_junctions and not fs._windows_can_symlink():\n pytest.skip(\"This Windows instance is not configured with symlink support\")\n elif sys.platform != \"win32\" and use_junctions:\n pytest.skip(\"Junctions are a Windows-only feature\")\n\n tmp_complex_dir = tmp_path_factory.mktemp(\"complex_dir_structure\")\n l1_d1 = tmp_complex_dir / \"l1-d1\"\n l1_d1.mkdir()\n l2_d1 = l1_d1 / \"l2-d1\"\n l2_d1.mkdir()\n l3_d2 = l2_d1 / \"l3-d2\"\n l3_d2.mkdir()\n l3_d4 = l2_d1 / \"l3-d4\"\n l3_d4.mkdir()\n l1_d2 = tmp_complex_dir / \"l1-d2\"\n l1_d2.mkdir()\n l2_d2 = l1_d2 / \"l2-d2\"\n l2_d2.mkdir()\n\n if sys.platform == \"win32\" and use_junctions:\n link_fn = fs._windows_create_junction\n else:\n link_fn = os.symlink\n\n link_fn(str(l1_d2), str(l2_d1 / \"l3-s1\"))\n link_fn(str(l1_d1), str(l2_d1 / \"l3-s3\"))\n link_fn(str(l3_d4), str(tmp_complex_dir / \"l1-s3\"))\n l2_s3 = l1_d2 / \"l2-s3\"\n link_fn(str(l2_d2), str(l2_s3))\n link_fn(str(l2_s3), str(tmp_complex_dir / \"l1-s4\"))\n\n # Create files\n l4_f1 = l3_d2 / \"l4-f1\"\n l4_f1.touch()\n l4_f2 = l3_d4 / \"l4-f2\"\n l4_f2.touch()\n l2_f1 = l1_d2 / \"l2-f1\"\n l2_f1.touch()\n l3_f3 = l2_d2 / \"l3-f3\"\n l3_f3.touch()\n\n locations = {\n \"l4-f1\": str(l4_f1),\n \"l4-f2-full\": str(l4_f2),\n \"l4-f2-link\": str(tmp_complex_dir / \"l1-s3\" / \"l4-f2\"),\n \"l2-f1\": str(l2_f1),\n \"l2-f1-link\": str(tmp_complex_dir / \"l1-d1\" / \"l2-d1\" / \"l3-s1\" / \"l2-f1\"),\n \"l3-f3-full\": str(l3_f3),\n \"l3-f3-link-l1\": str(tmp_complex_dir / \"l1-s4\" / \"l3-f3\"),\n }\n\n return str(tmp_complex_dir), locations\n\n\ndef test_find_max_depth_symlinks(complex_dir_structure):\n root, locations = complex_dir_structure\n root = pathlib.Path(root)\n assert set(fs.find(root, \"l4-f1\")) == {locations[\"l4-f1\"]}\n assert set(fs.find(root / \"l1-s3\", \"l4-f2\", max_depth=0)) == {locations[\"l4-f2-link\"]}\n assert set(fs.find(root / \"l1-d1\", \"l2-f1\")) == {locations[\"l2-f1-link\"]}\n # File is accessible via symlink and subdir, the link path will be\n # searched first, and the directory will not be searched again when\n # it is encountered the second time (via not-link) in the traversal\n assert set(fs.find(root, \"l4-f2\")) == {locations[\"l4-f2-link\"]}\n # File is accessible only via the dir, so the full file path should\n # be reported\n assert set(fs.find(root / \"l1-d1\", \"l4-f2\")) == {locations[\"l4-f2-full\"]}\n # Check following links to links\n assert set(fs.find(root, \"l3-f3\")) == {locations[\"l3-f3-link-l1\"]}\n\n\ndef test_find_max_depth_multiple_and_repeated_entry_points(complex_dir_structure):\n root, locations = complex_dir_structure\n\n fst = str(pathlib.Path(root) / \"l1-d1\" / \"l2-d1\")\n snd = str(pathlib.Path(root) / \"l1-d2\")\n nonexistent = str(pathlib.Path(root) / \"nonexistent\")\n\n assert set(fs.find([fst, snd, fst, snd, nonexistent], [\"l*-f*\"], max_depth=1)) == {\n locations[\"l2-f1\"],\n locations[\"l4-f1\"],\n locations[\"l4-f2-full\"],\n locations[\"l3-f3-full\"],\n }\n\n\ndef test_multiple_patterns(complex_dir_structure):\n root, _ = complex_dir_structure\n paths = fs.find(root, [\"l2-f1\", \"l*-d*/l3-f3\", \"*-f*\", \"*/*-f*\"])\n # There shouldn't be duplicate results with multiple, overlapping patterns\n assert len(set(paths)) == len(paths)\n # All files should be found\n filenames = [os.path.basename(p) for p in paths]\n assert set(filenames) == {\"l2-f1\", \"l3-f3\", \"l4-f1\", \"l4-f2\"}\n # They are ordered by first matching pattern (this is a bit of an implementation detail,\n # and we could decide to change the exact order in the future)\n assert filenames[0] == \"l2-f1\"\n assert filenames[1] == \"l3-f3\"\n\n\ndef test_find_input_types(tmp_path: pathlib.Path):\n \"\"\"test that find only accepts sequences and instances of pathlib.Path and str for root, and\n only sequences and instances of str for patterns. In principle mypy catches these issues, but\n it is not enabled on all call-sites.\"\"\"\n (tmp_path / \"file.txt\").write_text(\"\")\n assert (\n fs.find(tmp_path, \"file.txt\")\n == fs.find(str(tmp_path), \"file.txt\")\n == fs.find([tmp_path, str(tmp_path)], \"file.txt\")\n == fs.find((tmp_path, str(tmp_path)), \"file.txt\")\n == fs.find(tmp_path, \"file.txt\")\n == fs.find(tmp_path, [\"file.txt\"])\n == fs.find(tmp_path, (\"file.txt\",))\n == [str(tmp_path / \"file.txt\")]\n )\n\n with pytest.raises(TypeError):\n fs.find(tmp_path, pathlib.Path(\"file.txt\")) # type: ignore\n\n with pytest.raises(TypeError):\n fs.find(1, \"file.txt\") # type: ignore\n\n\ndef test_edit_in_place_through_temporary_file(tmp_path: pathlib.Path):\n (tmp_path / \"example.txt\").write_text(\"Hello\")\n current_ino = os.stat(tmp_path / \"example.txt\").st_ino\n with fs.edit_in_place_through_temporary_file(str(tmp_path / \"example.txt\")) as temporary:\n os.unlink(temporary)\n with open(temporary, \"w\", encoding=\"utf-8\") as f:\n f.write(\"World\")\n assert (tmp_path / \"example.txt\").read_text() == \"World\"\n assert os.stat(tmp_path / \"example.txt\").st_ino == current_ino\n" }, { "path": "lib/spack/spack/test/llnl/util/lang.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport pathlib\nimport re\nimport sys\nfrom datetime import datetime, timedelta\n\nimport pytest\n\nimport spack.llnl.util.lang\nfrom spack.llnl.util.lang import (\n Singleton,\n SingletonInstantiationError,\n dedupe,\n match_predicate,\n memoized,\n pretty_date,\n)\n\n\n@pytest.fixture()\ndef now():\n return datetime.now()\n\n\n@pytest.fixture()\ndef module_path(tmp_path: pathlib.Path):\n m = tmp_path / \"foo.py\"\n content = \"\"\"\nimport os\n\nvalue = 1\npath = os.path.join('/usr', 'bin')\n\"\"\"\n m.write_text(content)\n\n yield str(m)\n\n # Don't leave garbage in the module system\n if \"foo\" in sys.modules:\n del sys.modules[\"foo\"]\n\n\ndef test_pretty_date():\n \"\"\"Make sure pretty_date prints the right dates.\"\"\"\n now = datetime.now()\n\n just_now = now - timedelta(seconds=5)\n assert pretty_date(just_now, now) == \"just now\"\n\n seconds = now - timedelta(seconds=30)\n assert pretty_date(seconds, now) == \"30 seconds ago\"\n\n a_minute = now - timedelta(seconds=60)\n assert pretty_date(a_minute, now) == \"a minute ago\"\n\n minutes = now - timedelta(seconds=1800)\n assert pretty_date(minutes, now) == \"30 minutes ago\"\n\n an_hour = now - timedelta(hours=1)\n assert pretty_date(an_hour, now) == \"an hour ago\"\n\n hours = now - timedelta(hours=2)\n assert pretty_date(hours, now) == \"2 hours ago\"\n\n yesterday = now - timedelta(days=1)\n assert pretty_date(yesterday, now) == \"yesterday\"\n\n days = now - timedelta(days=3)\n assert pretty_date(days, now) == \"3 days ago\"\n\n a_week = now - timedelta(weeks=1)\n assert pretty_date(a_week, now) == \"a week ago\"\n\n weeks = now - timedelta(weeks=2)\n assert pretty_date(weeks, now) == \"2 weeks ago\"\n\n a_month = now - timedelta(days=30)\n assert pretty_date(a_month, now) == \"a month ago\"\n\n months = now - timedelta(days=60)\n assert pretty_date(months, now) == \"2 months ago\"\n\n a_year = now - timedelta(days=365)\n assert pretty_date(a_year, now) == \"a year ago\"\n\n years = now - timedelta(days=365 * 2)\n assert pretty_date(years, now) == \"2 years ago\"\n\n\n@pytest.mark.parametrize(\n \"delta,pretty_string\",\n [\n (timedelta(days=1), \"a day ago\"),\n (timedelta(days=1), \"yesterday\"),\n (timedelta(days=1), \"1 day ago\"),\n (timedelta(weeks=1), \"1 week ago\"),\n (timedelta(weeks=3), \"3 weeks ago\"),\n (timedelta(days=30), \"1 month ago\"),\n (timedelta(days=730), \"2 years ago\"),\n ],\n)\ndef test_pretty_string_to_date_delta(now, delta, pretty_string):\n t1 = now - delta\n t2 = spack.llnl.util.lang.pretty_string_to_date(pretty_string, now)\n assert t1 == t2\n\n\n@pytest.mark.parametrize(\n \"format,pretty_string\",\n [\n (\"%Y\", \"2018\"),\n (\"%Y-%m\", \"2015-03\"),\n (\"%Y-%m-%d\", \"2015-03-28\"),\n (\"%Y-%m-%d %H:%M\", \"2015-03-28 11:12\"),\n (\"%Y-%m-%d %H:%M:%S\", \"2015-03-28 23:34:45\"),\n ],\n)\ndef test_pretty_string_to_date(format, pretty_string):\n t1 = datetime.strptime(pretty_string, format)\n t2 = spack.llnl.util.lang.pretty_string_to_date(pretty_string, now)\n assert t1 == t2\n\n\ndef test_pretty_seconds():\n assert spack.llnl.util.lang.pretty_seconds(2.1) == \"2.100s\"\n assert spack.llnl.util.lang.pretty_seconds(2.1 / 1000) == \"2.100ms\"\n assert spack.llnl.util.lang.pretty_seconds(2.1 / 1000 / 1000) == \"2.100us\"\n assert spack.llnl.util.lang.pretty_seconds(2.1 / 1000 / 1000 / 1000) == \"2.100ns\"\n assert spack.llnl.util.lang.pretty_seconds(2.1 / 1000 / 1000 / 1000 / 10) == \"0.210ns\"\n\n\ndef test_pretty_duration():\n assert spack.llnl.util.lang.pretty_duration(0) == \"0s\"\n assert spack.llnl.util.lang.pretty_duration(45) == \"45s\"\n assert spack.llnl.util.lang.pretty_duration(60) == \"1m00s\"\n assert spack.llnl.util.lang.pretty_duration(125) == \"2m05s\"\n assert spack.llnl.util.lang.pretty_duration(3600) == \"1h00m\"\n assert spack.llnl.util.lang.pretty_duration(3661) == \"1h01m\"\n\n\ndef test_match_predicate():\n matcher = match_predicate(lambda x: True)\n assert matcher(\"foo\")\n assert matcher(\"bar\")\n assert matcher(\"baz\")\n\n matcher = match_predicate([\"foo\", \"bar\"])\n assert matcher(\"foo\")\n assert matcher(\"bar\")\n assert not matcher(\"baz\")\n\n matcher = match_predicate(r\"^(foo|bar)$\")\n assert matcher(\"foo\")\n assert matcher(\"bar\")\n assert not matcher(\"baz\")\n\n with pytest.raises(ValueError):\n matcher = match_predicate(object())\n matcher(\"foo\")\n\n\ndef test_load_modules_from_file(module_path):\n # Check prerequisites\n assert \"foo\" not in sys.modules\n\n # Check that the module is loaded correctly from file\n foo = spack.llnl.util.lang.load_module_from_file(\"foo\", module_path)\n assert \"foo\" in sys.modules\n assert foo.value == 1\n assert foo.path == os.path.join(\"/usr\", \"bin\")\n\n # Check that the module is not reloaded a second time on subsequent calls\n foo.value = 2\n foo = spack.llnl.util.lang.load_module_from_file(\"foo\", module_path)\n assert \"foo\" in sys.modules\n assert foo.value == 2\n assert foo.path == os.path.join(\"/usr\", \"bin\")\n\n\ndef test_uniq():\n assert [1, 2, 3] == spack.llnl.util.lang.uniq([1, 2, 3])\n assert [1, 2, 3] == spack.llnl.util.lang.uniq([1, 1, 1, 1, 2, 2, 2, 3, 3])\n assert [1, 2, 1] == spack.llnl.util.lang.uniq([1, 1, 1, 1, 2, 2, 2, 1, 1])\n assert [] == spack.llnl.util.lang.uniq([])\n\n\ndef test_key_ordering():\n \"\"\"Ensure that key ordering works correctly.\"\"\"\n\n with pytest.raises(TypeError):\n\n @spack.llnl.util.lang.key_ordering\n class ClassThatHasNoCmpKeyMethod:\n # this will raise b/c it does not define _cmp_key\n pass\n\n @spack.llnl.util.lang.key_ordering\n class KeyComparable:\n def __init__(self, t):\n self.t = t\n\n def _cmp_key(self):\n return self.t\n\n a = KeyComparable((1, 2, 3))\n a2 = KeyComparable((1, 2, 3))\n b = KeyComparable((2, 3, 4))\n b2 = KeyComparable((2, 3, 4))\n\n assert a == a\n assert a == a2\n assert a2 == a\n\n assert b == b\n assert b == b2\n assert b2 == b\n\n assert a != b\n\n assert a < b\n assert b > a\n\n assert a <= b\n assert b >= a\n\n assert a <= a\n assert a <= a2\n assert b >= b\n assert b >= b2\n\n assert hash(a) != hash(b)\n assert hash(a) == hash(a)\n assert hash(a) == hash(a2)\n assert hash(b) == hash(b)\n assert hash(b) == hash(b2)\n\n\n@pytest.mark.parametrize(\"args, kwargs\", [((1,), {}), ((), {\"a\": 3}), ((1,), {\"a\": 3})])\ndef test_memoized(args, kwargs):\n @memoized\n def f(*args, **kwargs):\n return \"return-value\"\n\n assert f(*args, **kwargs) == \"return-value\"\n assert f(*args, **kwargs) == \"return-value\"\n assert f.cache_info().hits == 1\n\n\n@pytest.mark.parametrize(\"args, kwargs\", [(([1],), {}), ((), {\"a\": [1]})])\ndef test_memoized_unhashable(args, kwargs):\n \"\"\"Check that an exception is raised clearly\"\"\"\n\n @memoized\n def f(*args, **kwargs):\n return None\n\n with pytest.raises(TypeError, match=\"unhashable type:\"):\n f(*args, **kwargs)\n\n\ndef test_dedupe():\n assert [x for x in dedupe([1, 2, 1, 3, 2])] == [1, 2, 3]\n assert [x for x in dedupe([1, -2, 1, 3, 2], key=abs)] == [1, -2, 3]\n\n\ndef test_grouped_exception():\n h = spack.llnl.util.lang.GroupedExceptionHandler()\n\n def inner():\n raise ValueError(\"wow!\")\n\n with h.forward(\"inner method\"):\n inner()\n\n with h.forward(\"top-level\"):\n raise TypeError(\"ok\")\n\n\ndef test_grouped_exception_base_type():\n h = spack.llnl.util.lang.GroupedExceptionHandler()\n\n with h.forward(\"catch-runtime-error\", RuntimeError):\n raise NotImplementedError()\n\n with pytest.raises(NotImplementedError):\n with h.forward(\"catch-value-error\", ValueError):\n raise NotImplementedError()\n\n message = h.grouped_message(with_tracebacks=False)\n assert \"catch-runtime-error\" in message\n assert \"catch-value-error\" not in message\n\n\ndef test_class_level_constant_value():\n \"\"\"Tests that the Const descriptor does not allow overwriting the value from an instance\"\"\"\n\n class _SomeClass:\n CONST_VALUE = spack.llnl.util.lang.Const(10)\n\n with pytest.raises(TypeError, match=\"not support assignment\"):\n _SomeClass().CONST_VALUE = 11\n\n\ndef test_deprecated_property():\n \"\"\"Tests the behavior of the DeprecatedProperty descriptor, which is can be used when\n deprecating an attribute.\n \"\"\"\n\n class _Deprecated(spack.llnl.util.lang.DeprecatedProperty):\n def factory(self, instance, owner):\n return 46\n\n class _SomeClass:\n deprecated = _Deprecated(\"deprecated\")\n\n # Default behavior is to just return the deprecated value\n s = _SomeClass()\n assert s.deprecated == 46\n\n # When setting error_level to 1 the attribute warns\n _SomeClass.deprecated.error_lvl = 1\n with pytest.warns(UserWarning):\n assert s.deprecated == 46\n\n # When setting error_level to 2 an exception is raised\n _SomeClass.deprecated.error_lvl = 2\n with pytest.raises(AttributeError):\n _ = s.deprecated\n\n\ndef test_fnmatch_multiple():\n named_patterns = {\"a\": \"libf*o.so\", \"b\": \"libb*r.so\"}\n regex = re.compile(spack.llnl.util.lang.fnmatch_translate_multiple(named_patterns))\n\n a = regex.match(\"libfoo.so\")\n assert a and a.group(\"a\") == \"libfoo.so\"\n\n b = regex.match(\"libbar.so\")\n assert b and b.group(\"b\") == \"libbar.so\"\n\n assert not regex.match(\"libfoo.so.1\")\n assert not regex.match(\"libbar.so.1\")\n assert not regex.match(\"libfoo.solibbar.so\")\n assert not regex.match(\"libbaz.so\")\n\n\ndef _attr_error_factory():\n raise AttributeError(\"Could not make something\")\n\n\ndef test_singleton_instantiation_attr_failure():\n \"\"\"\n If an AttributeError occurs during the instantiation of a Singleton\n object, we want to see that error.\n \"\"\"\n x = Singleton(_attr_error_factory)\n with pytest.raises(SingletonInstantiationError) as last_exception:\n x.something\n\n def follow_exceptions(e):\n while e:\n yield e\n e = e.__cause__ or e.__context__\n\n assert any(\n \"Could not make something\" in str(e) for e in follow_exceptions(last_exception.value)\n )\n\n\nclass TestPriorityOrderedMapping:\n @pytest.mark.parametrize(\n \"elements,expected\",\n [\n # Push out-of-order with explicit, and different, priorities\n ([(\"b\", 2), (\"a\", 1), (\"d\", 4), (\"c\", 3)], [\"a\", \"b\", \"c\", \"d\"]),\n # Push in-order with priority=None\n ([(\"a\", None), (\"b\", None), (\"c\", None), (\"d\", None)], [\"a\", \"b\", \"c\", \"d\"]),\n # Mix explicit and implicit priorities\n ([(\"b\", 2), (\"c\", None), (\"a\", 1), (\"d\", None)], [\"a\", \"b\", \"c\", \"d\"]),\n ([(\"b\", 10), (\"c\", None), (\"a\", -20), (\"d\", None)], [\"a\", \"b\", \"c\", \"d\"]),\n ([(\"b\", 10), (\"c\", None), (\"a\", 20), (\"d\", None)], [\"b\", \"c\", \"a\", \"d\"]),\n # Adding the same key twice with different priorities\n ([(\"b\", 10), (\"c\", None), (\"a\", 20), (\"d\", None), (\"a\", -20)], [\"a\", \"b\", \"c\", \"d\"]),\n # Adding the same key twice, no priorities\n ([(\"b\", None), (\"a\", None), (\"b\", None)], [\"a\", \"b\"]),\n ],\n )\n def test_iteration_order(self, elements, expected):\n \"\"\"Tests that the iteration order respects priorities, no matter the insertion order.\"\"\"\n m = spack.llnl.util.lang.PriorityOrderedMapping()\n for key, priority in elements:\n m.add(key, value=None, priority=priority)\n assert list(m) == expected\n\n def test_reverse_iteration(self):\n \"\"\"Tests that we can conveniently use reverse iteration\"\"\"\n m = spack.llnl.util.lang.PriorityOrderedMapping()\n for key, value in [(\"a\", 1), (\"b\", 2), (\"c\", 3)]:\n m.add(key, value=value)\n\n assert list(m) == [\"a\", \"b\", \"c\"]\n assert list(reversed(m)) == [\"c\", \"b\", \"a\"]\n\n assert list(m.keys()) == [\"a\", \"b\", \"c\"]\n assert list(m.reversed_keys()) == [\"c\", \"b\", \"a\"]\n\n assert list(m.values()) == [1, 2, 3]\n assert list(m.reversed_values()) == [3, 2, 1]\n" }, { "path": "lib/spack/spack/test/llnl/util/link_tree.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport pathlib\nimport sys\n\nimport pytest\n\nimport spack.llnl.util.filesystem\nfrom spack.llnl.util.filesystem import (\n _windows_can_symlink,\n islink,\n mkdirp,\n readlink,\n symlink,\n touchp,\n visit_directory_tree,\n working_dir,\n)\nfrom spack.llnl.util.link_tree import DestinationMergeVisitor, LinkTree, SourceMergeVisitor\n\n\n@pytest.fixture\ndef stage(tmp_path: pathlib.Path):\n touchp(str(tmp_path / \"source\" / \"1\"))\n touchp(str(tmp_path / \"source\" / \"a\" / \"b\" / \"2\"))\n touchp(str(tmp_path / \"source\" / \"a\" / \"b\" / \"3\"))\n touchp(str(tmp_path / \"source\" / \"c\" / \"4\"))\n touchp(str(tmp_path / \"source\" / \"c\" / \"d\" / \"5\"))\n touchp(str(tmp_path / \"source\" / \"c\" / \"d\" / \"6\"))\n touchp(str(tmp_path / \"source\" / \"c\" / \"d\" / \"e\" / \"7\"))\n yield str(tmp_path)\n\n\ndef check_file_link(filename: str, expected_target: str):\n assert os.path.isfile(filename)\n assert islink(filename)\n if sys.platform != \"win32\" or spack.llnl.util.filesystem._windows_can_symlink():\n assert os.path.abspath(os.path.realpath(filename)) == os.path.abspath(expected_target)\n\n\n@pytest.mark.parametrize(\"run_as_root\", [True, False] if sys.platform == \"win32\" else [False])\ndef test_merge_to_new_directory(stage: str, monkeypatch, run_as_root: bool):\n if sys.platform == \"win32\":\n if run_as_root and not _windows_can_symlink():\n pytest.skip(\"Skipping portion of test which required dev-mode privileges.\")\n\n monkeypatch.setattr(\n spack.llnl.util.filesystem, \"_windows_can_symlink\", lambda: run_as_root\n )\n\n link_tree = LinkTree(os.path.join(stage, \"source\"))\n\n with working_dir(stage):\n link_tree.merge(\"dest\")\n\n files = [\n (\"dest/1\", \"source/1\"),\n (\"dest/a/b/2\", \"source/a/b/2\"),\n (\"dest/a/b/3\", \"source/a/b/3\"),\n (\"dest/c/4\", \"source/c/4\"),\n (\"dest/c/d/5\", \"source/c/d/5\"),\n (\"dest/c/d/6\", \"source/c/d/6\"),\n (\"dest/c/d/e/7\", \"source/c/d/e/7\"),\n ]\n\n for dest, source in files:\n check_file_link(dest, source)\n assert os.path.isabs(readlink(dest))\n\n link_tree.unmerge(\"dest\")\n\n assert not os.path.exists(\"dest\")\n\n\n@pytest.mark.parametrize(\"run_as_root\", [True, False] if sys.platform == \"win32\" else [False])\ndef test_merge_to_new_directory_relative(stage: str, monkeypatch, run_as_root: bool):\n if sys.platform == \"win32\":\n if run_as_root and not _windows_can_symlink():\n pytest.skip(\"Skipping portion of test which required dev-mode privileges.\")\n\n monkeypatch.setattr(\n spack.llnl.util.filesystem, \"_windows_can_symlink\", lambda: run_as_root\n )\n\n link_tree = LinkTree(os.path.join(stage, \"source\"))\n with working_dir(stage):\n link_tree.merge(\"dest\", relative=True)\n\n files = [\n (\"dest/1\", \"source/1\"),\n (\"dest/a/b/2\", \"source/a/b/2\"),\n (\"dest/a/b/3\", \"source/a/b/3\"),\n (\"dest/c/4\", \"source/c/4\"),\n (\"dest/c/d/5\", \"source/c/d/5\"),\n (\"dest/c/d/6\", \"source/c/d/6\"),\n (\"dest/c/d/e/7\", \"source/c/d/e/7\"),\n ]\n\n for dest, source in files:\n check_file_link(dest, source)\n # Hard links/junctions are inherently absolute.\n if sys.platform != \"win32\" or run_as_root:\n assert not os.path.isabs(readlink(dest))\n\n link_tree.unmerge(\"dest\")\n\n assert not os.path.exists(\"dest\")\n\n\n@pytest.mark.parametrize(\"run_as_root\", [True, False] if sys.platform == \"win32\" else [False])\ndef test_merge_to_existing_directory(stage: str, monkeypatch, run_as_root):\n if sys.platform == \"win32\":\n if run_as_root and not _windows_can_symlink():\n pytest.skip(\"Skipping portion of test which required dev-mode privileges.\")\n\n monkeypatch.setattr(\n spack.llnl.util.filesystem, \"_windows_can_symlink\", lambda: run_as_root\n )\n\n link_tree = LinkTree(os.path.join(stage, \"source\"))\n\n with working_dir(stage):\n touchp(\"dest/x\")\n touchp(\"dest/a/b/y\")\n\n link_tree.merge(\"dest\")\n\n files = [\n (\"dest/1\", \"source/1\"),\n (\"dest/a/b/2\", \"source/a/b/2\"),\n (\"dest/a/b/3\", \"source/a/b/3\"),\n (\"dest/c/4\", \"source/c/4\"),\n (\"dest/c/d/5\", \"source/c/d/5\"),\n (\"dest/c/d/6\", \"source/c/d/6\"),\n (\"dest/c/d/e/7\", \"source/c/d/e/7\"),\n ]\n for dest, source in files:\n check_file_link(dest, source)\n\n assert os.path.isfile(\"dest/x\")\n assert os.path.isfile(\"dest/a/b/y\")\n\n link_tree.unmerge(\"dest\")\n\n assert os.path.isfile(\"dest/x\")\n assert os.path.isfile(\"dest/a/b/y\")\n\n for dest, _ in files:\n assert not os.path.isfile(dest)\n\n\ndef test_merge_with_empty_directories(stage: str):\n link_tree = LinkTree(os.path.join(stage, \"source\"))\n with working_dir(stage):\n mkdirp(\"dest/f/g\")\n mkdirp(\"dest/a/b/h\")\n\n link_tree.merge(\"dest\")\n link_tree.unmerge(\"dest\")\n\n assert not os.path.exists(\"dest/1\")\n assert not os.path.exists(\"dest/a/b/2\")\n assert not os.path.exists(\"dest/a/b/3\")\n assert not os.path.exists(\"dest/c/4\")\n assert not os.path.exists(\"dest/c/d/5\")\n assert not os.path.exists(\"dest/c/d/6\")\n assert not os.path.exists(\"dest/c/d/e/7\")\n\n assert os.path.isdir(\"dest/a/b/h\")\n assert os.path.isdir(\"dest/f/g\")\n\n\ndef test_ignore(stage: str):\n link_tree = LinkTree(os.path.join(stage, \"source\"))\n with working_dir(stage):\n touchp(\"source/.spec\")\n touchp(\"dest/.spec\")\n\n link_tree.merge(\"dest\", ignore=lambda x: x == \".spec\")\n link_tree.unmerge(\"dest\", ignore=lambda x: x == \".spec\")\n\n assert not os.path.exists(\"dest/1\")\n assert not os.path.exists(\"dest/a\")\n assert not os.path.exists(\"dest/c\")\n\n assert os.path.isfile(\"source/.spec\")\n assert os.path.isfile(\"dest/.spec\")\n\n\ndef test_source_merge_visitor_does_not_follow_symlinked_dirs_at_depth(tmp_path: pathlib.Path):\n \"\"\"Given an dir structure like this::\n\n .\n `-- a\n |-- b\n | |-- c\n | | |-- d\n | | | `-- file\n | | `-- symlink_d -> d\n | `-- symlink_c -> c\n `-- symlink_b -> b\n\n The SoureMergeVisitor will expand symlinked dirs to directories, but only\n to fixed depth, to avoid exponential explosion. In our current defaults,\n symlink_b will be expanded, but symlink_c and symlink_d will not.\n \"\"\"\n j = os.path.join\n with working_dir(str(tmp_path)):\n os.mkdir(j(\"a\"))\n os.mkdir(j(\"a\", \"b\"))\n os.mkdir(j(\"a\", \"b\", \"c\"))\n os.mkdir(j(\"a\", \"b\", \"c\", \"d\"))\n symlink(j(\"b\"), j(\"a\", \"symlink_b\"))\n symlink(j(\"c\"), j(\"a\", \"b\", \"symlink_c\"))\n symlink(j(\"d\"), j(\"a\", \"b\", \"c\", \"symlink_d\"))\n with open(j(\"a\", \"b\", \"c\", \"d\", \"file\"), \"wb\"):\n pass\n\n visitor = SourceMergeVisitor()\n visit_directory_tree(str(tmp_path), visitor)\n assert [p for p in visitor.files.keys()] == [\n j(\"a\", \"b\", \"c\", \"d\", \"file\"),\n j(\"a\", \"b\", \"c\", \"symlink_d\"), # treated as a file, not expanded\n j(\"a\", \"b\", \"symlink_c\"), # treated as a file, not expanded\n j(\"a\", \"symlink_b\", \"c\", \"d\", \"file\"), # symlink_b was expanded\n j(\"a\", \"symlink_b\", \"c\", \"symlink_d\"), # symlink_b was expanded\n j(\"a\", \"symlink_b\", \"symlink_c\"), # symlink_b was expanded\n ]\n assert [p for p in visitor.directories.keys()] == [\n j(\"a\"),\n j(\"a\", \"b\"),\n j(\"a\", \"b\", \"c\"),\n j(\"a\", \"b\", \"c\", \"d\"),\n j(\"a\", \"symlink_b\"),\n j(\"a\", \"symlink_b\", \"c\"),\n j(\"a\", \"symlink_b\", \"c\", \"d\"),\n ]\n\n\ndef test_source_merge_visitor_cant_be_cyclical(tmp_path: pathlib.Path):\n \"\"\"Given an dir structure like this::\n\n .\n |-- a\n | `-- symlink_b -> ../b\n | `-- symlink_symlink_b -> symlink_b\n `-- b\n `-- symlink_a -> ../a\n\n The SoureMergeVisitor will not expand `a/symlink_b`, `a/symlink_symlink_b` and\n `b/symlink_a` to avoid recursion. The general rule is: only expand symlinked dirs\n pointing deeper into the directory structure.\n \"\"\"\n j = os.path.join\n with working_dir(str(tmp_path)):\n os.mkdir(j(\"a\"))\n os.mkdir(j(\"b\"))\n\n symlink(j(\"..\", \"b\"), j(\"a\", \"symlink_b\"))\n symlink(j(\"symlink_b\"), j(\"a\", \"symlink_b_b\"))\n symlink(j(\"..\", \"a\"), j(\"b\", \"symlink_a\"))\n\n visitor = SourceMergeVisitor()\n visit_directory_tree(str(tmp_path), visitor)\n assert [p for p in visitor.files.keys()] == [\n j(\"a\", \"symlink_b\"),\n j(\"a\", \"symlink_b_b\"),\n j(\"b\", \"symlink_a\"),\n ]\n assert [p for p in visitor.directories.keys()] == [j(\"a\"), j(\"b\")]\n\n\ndef test_destination_merge_visitor_always_errors_on_symlinked_dirs(tmp_path: pathlib.Path):\n \"\"\"When merging prefixes into a non-empty destination folder, and\n this destination folder has a symlinked dir where the prefix has a dir,\n we should never merge any files there, but register a fatal error.\"\"\"\n j = os.path.join\n\n # Here example_a and example_b are symlinks.\n dst_path = tmp_path / \"dst\"\n dst_path.mkdir()\n with working_dir(str(dst_path)):\n os.mkdir(\"a\")\n os.symlink(\"a\", \"example_a\")\n os.symlink(\"a\", \"example_b\")\n\n # Here example_a is a directory, and example_b is a (non-expanded) symlinked\n # directory.\n src_path = tmp_path / \"src\"\n src_path.mkdir()\n with working_dir(str(src_path)):\n os.mkdir(\"example_a\")\n with open(j(\"example_a\", \"file\"), \"wb\"):\n pass\n os.symlink(\"..\", \"example_b\")\n\n visitor = SourceMergeVisitor()\n visit_directory_tree(str(src_path), visitor)\n visit_directory_tree(str(dst_path), DestinationMergeVisitor(visitor))\n\n assert visitor.fatal_conflicts\n conflicts = [c.dst for c in visitor.fatal_conflicts]\n assert \"example_a\" in conflicts\n assert \"example_b\" in conflicts\n\n\ndef test_destination_merge_visitor_file_dir_clashes(tmp_path: pathlib.Path):\n \"\"\"Tests whether non-symlink file-dir and dir-file clashes as registered as fatal\n errors\"\"\"\n a_path = tmp_path / \"a\"\n a_path.mkdir()\n with working_dir(str(a_path)):\n os.mkdir(\"example\")\n\n b_path = tmp_path / \"b\"\n b_path.mkdir()\n with working_dir(str(b_path)):\n with open(\"example\", \"wb\"):\n pass\n\n a_to_b = SourceMergeVisitor()\n visit_directory_tree(str(a_path), a_to_b)\n visit_directory_tree(str(b_path), DestinationMergeVisitor(a_to_b))\n assert a_to_b.fatal_conflicts\n assert a_to_b.fatal_conflicts[0].dst == \"example\"\n\n b_to_a = SourceMergeVisitor()\n visit_directory_tree(str(b_path), b_to_a)\n visit_directory_tree(str(a_path), DestinationMergeVisitor(b_to_a))\n assert b_to_a.fatal_conflicts\n assert b_to_a.fatal_conflicts[0].dst == \"example\"\n\n\n@pytest.mark.parametrize(\"normalize\", [True, False])\ndef test_source_merge_visitor_handles_same_file_gracefully(\n tmp_path: pathlib.Path, normalize: bool\n):\n \"\"\"Symlinked files/dirs from one prefix to the other are not file or fatal conflicts, they are\n resolved by taking the underlying file/dir, and this does not depend on the order prefixes\n are visited.\"\"\"\n\n def u(path: str) -> str:\n return path.upper() if normalize else path\n\n (tmp_path / \"a\").mkdir()\n (tmp_path / \"a\" / \"file\").write_bytes(b\"hello\")\n (tmp_path / \"a\" / \"dir\").mkdir()\n (tmp_path / \"a\" / \"dir\" / \"foo\").write_bytes(b\"hello\")\n\n (tmp_path / \"b\").mkdir()\n (tmp_path / \"b\" / u(\"file\")).symlink_to(tmp_path / \"a\" / \"file\")\n (tmp_path / \"b\" / u(\"dir\")).symlink_to(tmp_path / \"a\" / \"dir\")\n (tmp_path / \"b\" / \"bar\").write_bytes(b\"hello\")\n\n visitor_1 = SourceMergeVisitor(normalize_paths=normalize)\n visitor_1.set_projection(str(tmp_path / \"view\"))\n for p in (\"a\", \"b\"):\n visit_directory_tree(str(tmp_path / p), visitor_1)\n\n visitor_2 = SourceMergeVisitor(normalize_paths=normalize)\n visitor_2.set_projection(str(tmp_path / \"view\"))\n for p in (\"b\", \"a\"):\n visit_directory_tree(str(tmp_path / p), visitor_2)\n\n assert not visitor_1.file_conflicts and not visitor_2.file_conflicts\n assert not visitor_1.fatal_conflicts and not visitor_2.fatal_conflicts\n assert (\n sorted(visitor_1.files.items())\n == sorted(visitor_2.files.items())\n == [\n (str(tmp_path / \"view\" / \"bar\"), (str(tmp_path / \"b\"), \"bar\")),\n (str(tmp_path / \"view\" / \"dir\" / \"foo\"), (str(tmp_path / \"a\"), f\"dir{os.sep}foo\")),\n (str(tmp_path / \"view\" / \"file\"), (str(tmp_path / \"a\"), \"file\")),\n ]\n )\n assert visitor_1.directories[str(tmp_path / \"view\" / \"dir\")] == (str(tmp_path / \"a\"), \"dir\")\n assert visitor_2.directories[str(tmp_path / \"view\" / \"dir\")] == (str(tmp_path / \"a\"), \"dir\")\n\n\ndef test_source_merge_visitor_deals_with_dangling_symlinks(tmp_path: pathlib.Path):\n \"\"\"When a file and a dangling symlink conflict, this should be handled like a file conflict.\"\"\"\n (tmp_path / \"dir_a\").mkdir()\n os.symlink(\"non-existent\", str(tmp_path / \"dir_a\" / \"file\"))\n\n (tmp_path / \"dir_b\").mkdir()\n (tmp_path / \"dir_b\" / \"file\").write_bytes(b\"data\")\n\n visitor = SourceMergeVisitor()\n visitor.set_projection(str(tmp_path / \"view\"))\n\n visit_directory_tree(str(tmp_path / \"dir_a\"), visitor)\n visit_directory_tree(str(tmp_path / \"dir_b\"), visitor)\n\n # Check that a conflict was registered.\n assert len(visitor.file_conflicts) == 1\n conflict = visitor.file_conflicts[0]\n assert conflict.src_a == str(tmp_path / \"dir_a\" / \"file\")\n assert conflict.src_b == str(tmp_path / \"dir_b\" / \"file\")\n assert conflict.dst == str(tmp_path / \"view\" / \"file\")\n\n # The first file encountered should be listed.\n assert visitor.files == {str(tmp_path / \"view\" / \"file\"): (str(tmp_path / \"dir_a\"), \"file\")}\n\n\n@pytest.mark.parametrize(\"normalize\", [True, False])\ndef test_source_visitor_file_file(tmp_path: pathlib.Path, normalize: bool):\n (tmp_path / \"a\").mkdir()\n (tmp_path / \"b\").mkdir()\n (tmp_path / \"a\" / \"file\").write_bytes(b\"\")\n (tmp_path / \"b\" / \"FILE\").write_bytes(b\"\")\n\n v = SourceMergeVisitor(normalize_paths=normalize)\n for p in (\"a\", \"b\"):\n visit_directory_tree(str(tmp_path / p), v)\n\n if normalize:\n assert len(v.files) == 1\n assert len(v.directories) == 0\n assert \"file\" in v.files # first file wins\n assert len(v.file_conflicts) == 1\n else:\n assert len(v.files) == 2\n assert len(v.directories) == 0\n assert \"file\" in v.files and \"FILE\" in v.files\n assert not v.fatal_conflicts\n assert not v.file_conflicts\n\n\n@pytest.mark.parametrize(\"normalize\", [True, False])\ndef test_source_visitor_file_dir(tmp_path: pathlib.Path, normalize: bool):\n (tmp_path / \"a\").mkdir()\n (tmp_path / \"a\" / \"file\").write_bytes(b\"\")\n (tmp_path / \"b\").mkdir()\n (tmp_path / \"b\" / \"FILE\").mkdir()\n v1 = SourceMergeVisitor(normalize_paths=normalize)\n for p in (\"a\", \"b\"):\n visit_directory_tree(str(tmp_path / p), v1)\n v2 = SourceMergeVisitor(normalize_paths=normalize)\n for p in (\"b\", \"a\"):\n visit_directory_tree(str(tmp_path / p), v2)\n\n assert not v1.file_conflicts and not v2.file_conflicts\n\n if normalize:\n assert len(v1.fatal_conflicts) == len(v2.fatal_conflicts) == 1\n else:\n assert len(v1.files) == len(v2.files) == 1\n assert \"file\" in v1.files and \"file\" in v2.files\n assert len(v1.directories) == len(v2.directories) == 1\n assert \"FILE\" in v1.directories and \"FILE\" in v2.directories\n assert not v1.fatal_conflicts and not v2.fatal_conflicts\n\n\n@pytest.mark.parametrize(\"normalize\", [True, False])\ndef test_source_visitor_dir_dir(tmp_path: pathlib.Path, normalize: bool):\n (tmp_path / \"a\").mkdir()\n (tmp_path / \"a\" / \"dir\").mkdir()\n (tmp_path / \"b\").mkdir()\n (tmp_path / \"b\" / \"DIR\").mkdir()\n v = SourceMergeVisitor(normalize_paths=normalize)\n for p in (\"a\", \"b\"):\n visit_directory_tree(str(tmp_path / p), v)\n\n assert not v.files\n assert not v.fatal_conflicts\n assert not v.file_conflicts\n\n if normalize:\n assert len(v.directories) == 1\n assert \"dir\" in v.directories\n else:\n assert len(v.directories) == 2\n assert \"DIR\" in v.directories and \"dir\" in v.directories\n\n\n@pytest.mark.parametrize(\"normalize\", [True, False])\ndef test_dst_visitor_file_file(tmp_path: pathlib.Path, normalize: bool):\n (tmp_path / \"a\").mkdir()\n (tmp_path / \"b\").mkdir()\n (tmp_path / \"a\" / \"file\").write_bytes(b\"\")\n (tmp_path / \"b\" / \"FILE\").write_bytes(b\"\")\n\n src = SourceMergeVisitor(normalize_paths=normalize)\n visit_directory_tree(str(tmp_path / \"a\"), src)\n visit_directory_tree(str(tmp_path / \"b\"), DestinationMergeVisitor(src))\n\n assert len(src.files) == 1\n assert len(src.directories) == 0\n assert \"file\" in src.files\n assert not src.file_conflicts\n\n if normalize:\n assert len(src.fatal_conflicts) == 1\n assert \"FILE\" in [c.dst for c in src.fatal_conflicts]\n else:\n assert not src.fatal_conflicts\n\n\n@pytest.mark.parametrize(\"normalize\", [True, False])\ndef test_dst_visitor_file_dir(tmp_path: pathlib.Path, normalize: bool):\n (tmp_path / \"a\").mkdir()\n (tmp_path / \"a\" / \"file\").write_bytes(b\"\")\n (tmp_path / \"b\").mkdir()\n (tmp_path / \"b\" / \"FILE\").mkdir()\n src1 = SourceMergeVisitor(normalize_paths=normalize)\n visit_directory_tree(str(tmp_path / \"a\"), src1)\n visit_directory_tree(str(tmp_path / \"b\"), DestinationMergeVisitor(src1))\n src2 = SourceMergeVisitor(normalize_paths=normalize)\n visit_directory_tree(str(tmp_path / \"b\"), src2)\n visit_directory_tree(str(tmp_path / \"a\"), DestinationMergeVisitor(src2))\n\n assert len(src1.files) == 1\n assert \"file\" in src1.files\n assert not src1.directories\n assert not src2.file_conflicts\n assert len(src2.directories) == 1\n\n if normalize:\n assert len(src1.fatal_conflicts) == 1\n assert \"FILE\" in [c.dst for c in src1.fatal_conflicts]\n assert not src2.files\n assert len(src2.fatal_conflicts) == 1\n assert \"file\" in [c.dst for c in src2.fatal_conflicts]\n else:\n assert not src1.fatal_conflicts and not src2.fatal_conflicts\n assert not src1.file_conflicts and not src2.file_conflicts\n" }, { "path": "lib/spack/spack/test/llnl/util/lock.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"These tests ensure that our lock works correctly.\n\nRun with pytest::\n\n pytest lib/spack/spack/test/llnl/util/lock.py\n\nYou can use this to test whether your shared filesystem properly supports\nPOSIX reader-writer locking with byte ranges through fcntl.\n\nIf you want to test on multiple filesystems, you can modify the\n``locations`` list below. By default it looks like this::\n\n locations = [\n tempfile.gettempdir(), # standard tmp directory (potentially local)\n '/nfs/tmp2/%u', # NFS tmp mount\n '/p/lscratch*/%u' # Lustre scratch mount\n ]\n\nAdd names and paths for your preferred filesystem mounts to test on them;\nthe tests are parametrized to run on all the filesystems listed in this\ndict.\n\n\"\"\"\n\nimport collections\nimport errno\nimport getpass\nimport glob\nimport multiprocessing\nimport os\nimport pathlib\nimport shutil\nimport socket\nimport stat\nimport sys\nimport tempfile\nimport traceback\nfrom contextlib import contextmanager\nfrom multiprocessing import Barrier, Process, Queue\n\nimport pytest\n\nimport spack.llnl.util.lock as lk\nfrom spack.llnl.util.filesystem import getuid, touch, working_dir\n\nif sys.platform != \"win32\":\n import fcntl\n\npytestmark = pytest.mark.not_on_windows(\"does not run on windows\")\n\n\n#\n# This test can be run with MPI. MPI is \"enabled\" if we can import\n# mpi4py and the number of total MPI processes is greater than 1.\n# Otherwise it just runs as a node-local test.\n#\n# NOTE: MPI mode is different from node-local mode in that node-local\n# mode will spawn its own test processes, while MPI mode assumes you've\n# run this script as a SPMD application. In MPI mode, no additional\n# processes are spawned, and you need to ensure that you mpirun the\n# script with enough processes for all the multiproc_test cases below.\n#\n# If you don't run with enough processes, tests that require more\n# processes than you currently have will be skipped.\n#\nmpi = False\ncomm = None\ntry:\n from mpi4py import MPI\n\n comm = MPI.COMM_WORLD\n if comm.size > 1:\n mpi = True\nexcept ImportError:\n pass\n\n\n#: This is a list of filesystem locations to test locks in. Paths are\n#: expanded so that %u is replaced with the current username. '~' is also\n#: legal and will be expanded to the user's home directory.\n#:\n#: Tests are skipped for directories that don't exist, so you'll need to\n#: update this with the locations of NFS, Lustre, and other mounts on your\n#: system.\nlocations = [\n tempfile.gettempdir(),\n os.path.join(\"/nfs/tmp2/\", getpass.getuser()),\n os.path.join(\"/p/lscratch*/\", getpass.getuser()),\n]\n\n#: This is the longest a failed multiproc test will take.\n#: Barriers will time out and raise an exception after this interval.\n#: In MPI mode, barriers don't time out (they hang). See mpi_multiproc_test.\nbarrier_timeout = 5\n\n#: This is the lock timeout for expected failures.\n#: This may need to be higher for some filesystems.\nlock_fail_timeout = 0.1\n\n\ndef make_readable(*paths):\n # TODO: From os.chmod doc:\n # \"Note Although Windows supports chmod(), you can only\n # set the file's read-only flag with it (via the stat.S_IWRITE and\n # stat.S_IREAD constants or a corresponding integer value). All other\n # bits are ignored.\"\n for path in paths:\n if sys.platform != \"win32\":\n mode = 0o555 if os.path.isdir(path) else 0o444\n else:\n mode = stat.S_IREAD\n os.chmod(path, mode)\n\n\ndef make_writable(*paths):\n for path in paths:\n if sys.platform != \"win32\":\n mode = 0o755 if os.path.isdir(path) else 0o744\n else:\n mode = stat.S_IWRITE\n os.chmod(path, mode)\n\n\n@contextmanager\ndef read_only(*paths):\n modes = [os.stat(p).st_mode for p in paths]\n make_readable(*paths)\n\n yield\n\n for path, mode in zip(paths, modes):\n os.chmod(path, mode)\n\n\n@pytest.fixture(scope=\"session\", params=locations)\ndef lock_test_directory(request):\n \"\"\"This fixture causes tests to be executed for many different mounts.\n\n See the ``locations`` dict above for details.\n \"\"\"\n return request.param\n\n\n@pytest.fixture(scope=\"session\")\ndef lock_dir(lock_test_directory):\n parent = next(\n (p for p in glob.glob(lock_test_directory) if os.path.exists(p) and os.access(p, os.W_OK)),\n None,\n )\n if not parent:\n # Skip filesystems that don't exist or aren't writable\n pytest.skip(\"requires filesystem: '%s'\" % lock_test_directory)\n elif mpi and parent == tempfile.gettempdir():\n # Skip local tmp test for MPI runs\n pytest.skip(\"skipping local tmp directory for MPI test.\")\n\n tempdir = None\n if not mpi or comm.rank == 0:\n tempdir = tempfile.mkdtemp(dir=parent)\n if mpi:\n tempdir = comm.bcast(tempdir)\n\n yield tempdir\n\n if mpi:\n # rank 0 may get here before others, in which case it'll try to\n # remove the directory while other processes try to re-create the\n # lock. This will give errno 39: directory not empty. Use a\n # barrier to ensure everyone is done first.\n comm.barrier()\n\n if not mpi or comm.rank == 0:\n make_writable(tempdir)\n shutil.rmtree(tempdir)\n\n\n@pytest.fixture\ndef private_lock_path(lock_dir):\n \"\"\"In MPI mode, this is a private lock for each rank in a multiproc test.\n\n For other modes, it is the same as a shared lock.\n \"\"\"\n lock_file = os.path.join(lock_dir, \"lockfile\")\n if mpi:\n lock_file += \".%s\" % comm.rank\n\n yield lock_file\n\n if os.path.exists(lock_file):\n make_writable(lock_dir, lock_file)\n os.unlink(lock_file)\n\n\n@pytest.fixture\ndef lock_path(lock_dir):\n \"\"\"This lock is shared among all processes in a multiproc test.\"\"\"\n lock_file = os.path.join(lock_dir, \"lockfile\")\n\n yield lock_file\n\n if os.path.exists(lock_file):\n make_writable(lock_dir, lock_file)\n os.unlink(lock_file)\n\n\ndef test_poll_interval_generator():\n interval_iter = iter(lk.Lock._poll_interval_generator(_wait_times=[1, 2, 3]))\n intervals = list(next(interval_iter) for i in range(100))\n assert intervals == [1] * 20 + [2] * 40 + [3] * 40\n\n\ndef local_multiproc_test(*functions, **kwargs):\n \"\"\"Order some processes using simple barrier synchronization.\"\"\"\n b = Barrier(len(functions), timeout=barrier_timeout)\n\n args = (b,) + tuple(kwargs.get(\"extra_args\", ()))\n procs = [Process(target=f, args=args, name=f.__name__) for f in functions]\n\n for p in procs:\n p.start()\n\n for p in procs:\n p.join()\n\n assert all(p.exitcode == 0 for p in procs)\n\n\ndef mpi_multiproc_test(*functions):\n \"\"\"SPMD version of multiproc test.\n\n This needs to be run like so:\n\n srun spack test lock\n\n Each process executes its corresponding function. This is different\n from ``multiproc_test`` above, which spawns the processes. This will\n skip tests if there are too few processes to run them.\n \"\"\"\n procs = len(functions)\n if procs > comm.size:\n pytest.skip(\"requires at least %d MPI processes\" % procs)\n\n comm.Barrier() # barrier before each MPI test\n\n include = comm.rank < len(functions)\n subcomm = comm.Split(include)\n\n class subcomm_barrier:\n \"\"\"Stand-in for multiproc barrier for MPI-parallel jobs.\"\"\"\n\n def wait(self):\n subcomm.Barrier()\n\n if include:\n try:\n functions[subcomm.rank](subcomm_barrier())\n except BaseException:\n # aborting is the best we can do for MPI tests without\n # hanging, since we're using MPI barriers. This will fail\n # early and it loses the nice pytest output, but at least it\n # gets use a stacktrace on the processes that failed.\n traceback.print_exc()\n comm.Abort()\n subcomm.Free()\n\n comm.Barrier() # barrier after each MPI test.\n\n\n#: ``multiproc_test()`` should be called by tests below.\n#: ``multiproc_test()`` will work for either MPI runs or for local runs.\nmultiproc_test = mpi_multiproc_test if mpi else local_multiproc_test\n\n\n#\n# Process snippets below can be composed into tests.\n#\nclass AcquireWrite:\n def __init__(self, lock_path, start=0, length=0):\n self.lock_path = lock_path\n self.start = start\n self.length = length\n\n @property\n def __name__(self):\n return self.__class__.__name__\n\n def __call__(self, barrier):\n lock = lk.Lock(self.lock_path, start=self.start, length=self.length)\n lock.acquire_write() # grab exclusive lock\n barrier.wait()\n barrier.wait() # hold the lock until timeout in other procs.\n\n\nclass AcquireRead:\n def __init__(self, lock_path, start=0, length=0):\n self.lock_path = lock_path\n self.start = start\n self.length = length\n\n @property\n def __name__(self):\n return self.__class__.__name__\n\n def __call__(self, barrier):\n lock = lk.Lock(self.lock_path, start=self.start, length=self.length)\n lock.acquire_read() # grab shared lock\n barrier.wait()\n barrier.wait() # hold the lock until timeout in other procs.\n\n\nclass TimeoutWrite:\n def __init__(self, lock_path, start=0, length=0):\n self.lock_path = lock_path\n self.start = start\n self.length = length\n\n @property\n def __name__(self):\n return self.__class__.__name__\n\n def __call__(self, barrier):\n lock = lk.Lock(self.lock_path, start=self.start, length=self.length)\n barrier.wait() # wait for lock acquire in first process\n with pytest.raises(lk.LockTimeoutError):\n lock.acquire_write(lock_fail_timeout)\n barrier.wait()\n\n\nclass TimeoutRead:\n def __init__(self, lock_path, start=0, length=0):\n self.lock_path = lock_path\n self.start = start\n self.length = length\n\n @property\n def __name__(self):\n return self.__class__.__name__\n\n def __call__(self, barrier):\n lock = lk.Lock(self.lock_path, start=self.start, length=self.length)\n barrier.wait() # wait for lock acquire in first process\n with pytest.raises(lk.LockTimeoutError):\n lock.acquire_read(lock_fail_timeout)\n barrier.wait()\n\n\n#\n# Test that exclusive locks on other processes time out when an\n# exclusive lock is held.\n#\ndef test_write_lock_timeout_on_write(lock_path):\n multiproc_test(AcquireWrite(lock_path), TimeoutWrite(lock_path))\n\n\ndef test_write_lock_timeout_on_write_2(lock_path):\n multiproc_test(AcquireWrite(lock_path), TimeoutWrite(lock_path), TimeoutWrite(lock_path))\n\n\ndef test_write_lock_timeout_on_write_3(lock_path):\n multiproc_test(\n AcquireWrite(lock_path),\n TimeoutWrite(lock_path),\n TimeoutWrite(lock_path),\n TimeoutWrite(lock_path),\n )\n\n\ndef test_write_lock_timeout_on_write_ranges(lock_path):\n multiproc_test(AcquireWrite(lock_path, 0, 1), TimeoutWrite(lock_path, 0, 1))\n\n\ndef test_write_lock_timeout_on_write_ranges_2(lock_path):\n multiproc_test(\n AcquireWrite(lock_path, 0, 64),\n AcquireWrite(lock_path, 65, 1),\n TimeoutWrite(lock_path, 0, 1),\n TimeoutWrite(lock_path, 63, 1),\n )\n\n\ndef test_write_lock_timeout_on_write_ranges_3(lock_path):\n multiproc_test(\n AcquireWrite(lock_path, 0, 1),\n AcquireWrite(lock_path, 1, 1),\n TimeoutWrite(lock_path),\n TimeoutWrite(lock_path),\n TimeoutWrite(lock_path),\n )\n\n\ndef test_write_lock_timeout_on_write_ranges_4(lock_path):\n multiproc_test(\n AcquireWrite(lock_path, 0, 1),\n AcquireWrite(lock_path, 1, 1),\n AcquireWrite(lock_path, 2, 456),\n AcquireWrite(lock_path, 500, 64),\n TimeoutWrite(lock_path),\n TimeoutWrite(lock_path),\n TimeoutWrite(lock_path),\n )\n\n\n#\n# Test that shared locks on other processes time out when an\n# exclusive lock is held.\n#\ndef test_read_lock_timeout_on_write(lock_path):\n multiproc_test(AcquireWrite(lock_path), TimeoutRead(lock_path))\n\n\ndef test_read_lock_timeout_on_write_2(lock_path):\n multiproc_test(AcquireWrite(lock_path), TimeoutRead(lock_path), TimeoutRead(lock_path))\n\n\ndef test_read_lock_timeout_on_write_3(lock_path):\n multiproc_test(\n AcquireWrite(lock_path),\n TimeoutRead(lock_path),\n TimeoutRead(lock_path),\n TimeoutRead(lock_path),\n )\n\n\ndef test_read_lock_timeout_on_write_ranges(lock_path):\n \"\"\"small write lock, read whole file.\"\"\"\n multiproc_test(AcquireWrite(lock_path, 0, 1), TimeoutRead(lock_path))\n\n\ndef test_read_lock_timeout_on_write_ranges_2(lock_path):\n \"\"\"small write lock, small read lock\"\"\"\n multiproc_test(AcquireWrite(lock_path, 0, 1), TimeoutRead(lock_path, 0, 1))\n\n\ndef test_read_lock_timeout_on_write_ranges_3(lock_path):\n \"\"\"two write locks, overlapping read locks\"\"\"\n multiproc_test(\n AcquireWrite(lock_path, 0, 1),\n AcquireWrite(lock_path, 64, 128),\n TimeoutRead(lock_path, 0, 1),\n TimeoutRead(lock_path, 128, 256),\n )\n\n\n#\n# Test that exclusive locks time out when shared locks are held.\n#\ndef test_write_lock_timeout_on_read(lock_path):\n multiproc_test(AcquireRead(lock_path), TimeoutWrite(lock_path))\n\n\ndef test_write_lock_timeout_on_read_2(lock_path):\n multiproc_test(AcquireRead(lock_path), TimeoutWrite(lock_path), TimeoutWrite(lock_path))\n\n\ndef test_write_lock_timeout_on_read_3(lock_path):\n multiproc_test(\n AcquireRead(lock_path),\n TimeoutWrite(lock_path),\n TimeoutWrite(lock_path),\n TimeoutWrite(lock_path),\n )\n\n\ndef test_write_lock_timeout_on_read_ranges(lock_path):\n multiproc_test(AcquireRead(lock_path, 0, 1), TimeoutWrite(lock_path))\n\n\ndef test_write_lock_timeout_on_read_ranges_2(lock_path):\n multiproc_test(AcquireRead(lock_path, 0, 1), TimeoutWrite(lock_path, 0, 1))\n\n\ndef test_write_lock_timeout_on_read_ranges_3(lock_path):\n multiproc_test(\n AcquireRead(lock_path, 0, 1),\n AcquireRead(lock_path, 10, 1),\n TimeoutWrite(lock_path, 0, 1),\n TimeoutWrite(lock_path, 10, 1),\n )\n\n\ndef test_write_lock_timeout_on_read_ranges_4(lock_path):\n multiproc_test(\n AcquireRead(lock_path, 0, 64),\n TimeoutWrite(lock_path, 10, 1),\n TimeoutWrite(lock_path, 32, 1),\n )\n\n\ndef test_write_lock_timeout_on_read_ranges_5(lock_path):\n multiproc_test(\n AcquireRead(lock_path, 64, 128),\n TimeoutWrite(lock_path, 65, 1),\n TimeoutWrite(lock_path, 127, 1),\n TimeoutWrite(lock_path, 90, 10),\n )\n\n\n#\n# Test that exclusive locks time while lots of shared locks are held.\n#\ndef test_write_lock_timeout_with_multiple_readers_2_1(lock_path):\n multiproc_test(AcquireRead(lock_path), AcquireRead(lock_path), TimeoutWrite(lock_path))\n\n\ndef test_write_lock_timeout_with_multiple_readers_2_2(lock_path):\n multiproc_test(\n AcquireRead(lock_path),\n AcquireRead(lock_path),\n TimeoutWrite(lock_path),\n TimeoutWrite(lock_path),\n )\n\n\ndef test_write_lock_timeout_with_multiple_readers_3_1(lock_path):\n multiproc_test(\n AcquireRead(lock_path),\n AcquireRead(lock_path),\n AcquireRead(lock_path),\n TimeoutWrite(lock_path),\n )\n\n\ndef test_write_lock_timeout_with_multiple_readers_3_2(lock_path):\n multiproc_test(\n AcquireRead(lock_path),\n AcquireRead(lock_path),\n AcquireRead(lock_path),\n TimeoutWrite(lock_path),\n TimeoutWrite(lock_path),\n )\n\n\ndef test_write_lock_timeout_with_multiple_readers_2_1_ranges(lock_path):\n multiproc_test(\n AcquireRead(lock_path, 0, 10), AcquireRead(lock_path, 2, 10), TimeoutWrite(lock_path, 5, 5)\n )\n\n\ndef test_write_lock_timeout_with_multiple_readers_2_3_ranges(lock_path):\n multiproc_test(\n AcquireRead(lock_path, 0, 10),\n AcquireRead(lock_path, 5, 15),\n TimeoutWrite(lock_path, 0, 1),\n TimeoutWrite(lock_path, 11, 3),\n TimeoutWrite(lock_path, 7, 1),\n )\n\n\ndef test_write_lock_timeout_with_multiple_readers_3_1_ranges(lock_path):\n multiproc_test(\n AcquireRead(lock_path, 0, 5),\n AcquireRead(lock_path, 5, 5),\n AcquireRead(lock_path, 10, 5),\n TimeoutWrite(lock_path, 0, 15),\n )\n\n\ndef test_write_lock_timeout_with_multiple_readers_3_2_ranges(lock_path):\n multiproc_test(\n AcquireRead(lock_path, 0, 5),\n AcquireRead(lock_path, 5, 5),\n AcquireRead(lock_path, 10, 5),\n TimeoutWrite(lock_path, 3, 10),\n TimeoutWrite(lock_path, 5, 1),\n )\n\n\n@pytest.mark.skipif(getuid() == 0, reason=\"user is root\")\ndef test_read_lock_on_read_only_lockfile(lock_dir, lock_path):\n \"\"\"read-only directory, read-only lockfile.\"\"\"\n touch(lock_path)\n with read_only(lock_path, lock_dir):\n lock = lk.Lock(lock_path)\n\n with lk.ReadTransaction(lock):\n pass\n\n with pytest.raises(lk.LockROFileError):\n with lk.WriteTransaction(lock):\n pass\n\n\ndef test_read_lock_read_only_dir_writable_lockfile(lock_dir, lock_path):\n \"\"\"read-only directory, writable lockfile.\"\"\"\n touch(lock_path)\n with read_only(lock_dir):\n lock = lk.Lock(lock_path)\n\n with lk.ReadTransaction(lock):\n pass\n\n with lk.WriteTransaction(lock):\n pass\n\n\n@pytest.mark.skipif(False if sys.platform == \"win32\" else getuid() == 0, reason=\"user is root\")\ndef test_read_lock_no_lockfile(lock_dir, lock_path):\n \"\"\"read-only directory, no lockfile (so can't create).\"\"\"\n with read_only(lock_dir):\n lock = lk.Lock(lock_path)\n\n with pytest.raises(lk.CantCreateLockError):\n with lk.ReadTransaction(lock):\n pass\n\n with pytest.raises(lk.CantCreateLockError):\n with lk.WriteTransaction(lock):\n pass\n\n\ndef test_upgrade_read_to_write(private_lock_path):\n \"\"\"Test that a read lock can be upgraded to a write lock.\n\n Note that to upgrade a read lock to a write lock, you have the be the\n only holder of a read lock. Client code needs to coordinate that for\n shared locks. For this test, we use a private lock just to test that an\n upgrade is possible.\n \"\"\"\n # ensure lock file exists the first time, so we open it read-only\n # to begin with.\n touch(private_lock_path)\n\n lock = lk.Lock(private_lock_path)\n assert lock._reads == 0\n assert lock._writes == 0\n\n lock.acquire_read()\n assert lock._reads == 1\n assert lock._writes == 0\n assert lock._file_ref.fh.mode == \"rb+\"\n\n lock.acquire_write()\n assert lock._reads == 1\n assert lock._writes == 1\n assert lock._file_ref.fh.mode == \"rb+\"\n\n lock.release_write()\n assert lock._reads == 1\n assert lock._writes == 0\n assert lock._file_ref.fh.mode == \"rb+\"\n\n lock.release_read()\n assert lock._reads == 0\n assert lock._writes == 0\n assert not lock._file_ref.fh.closed # recycle the file handle for next lock\n\n\ndef test_release_write_downgrades_to_shared(private_lock_path):\n \"\"\"Releasing a write lock while a read lock is held must downgrade the POSIX lock\n from exclusive to shared, allowing other processes to acquire read locks.\"\"\"\n lock = lk.Lock(private_lock_path)\n lock.acquire_read()\n lock.acquire_write()\n lock.release_write()\n assert lock._reads == 1\n assert lock._writes == 0\n\n ctx = multiprocessing.get_context()\n q = ctx.Queue()\n\n # Another process must be able to acquire a shared read lock concurrently.\n p = ctx.Process(target=_child_try_acquire_read, args=(private_lock_path, q))\n p.start()\n p.join()\n assert q.get() is True\n\n # But must not be able to acquire an exclusive write lock.\n p = ctx.Process(target=_child_try_acquire_write, args=(private_lock_path, q))\n p.start()\n p.join()\n assert q.get() is False\n\n lock.release_read()\n assert lock._reads == 0\n assert lock._writes == 0\n\n\n@pytest.mark.skipif(getuid() == 0, reason=\"user is root\")\ndef test_upgrade_read_to_write_fails_with_readonly_file(private_lock_path):\n \"\"\"Test that read-only file can be read-locked but not write-locked.\"\"\"\n # ensure lock file exists the first time\n touch(private_lock_path)\n\n # open it read-only to begin with.\n with read_only(private_lock_path):\n lock = lk.Lock(private_lock_path)\n assert lock._reads == 0\n assert lock._writes == 0\n\n lock.acquire_read()\n assert lock._reads == 1\n assert lock._writes == 0\n assert lock._file_ref.fh.mode == \"rb\"\n\n # upgrade to write here\n with pytest.raises(lk.LockROFileError):\n lock.acquire_write()\n\n\nclass ComplexAcquireAndRelease:\n def __init__(self, lock_path):\n self.lock_path = lock_path\n\n def p1(self, barrier):\n lock = lk.Lock(self.lock_path)\n\n lock.acquire_write()\n barrier.wait() # ---------------------------------------- 1\n # others test timeout\n barrier.wait() # ---------------------------------------- 2\n lock.release_write() # release and others acquire read\n barrier.wait() # ---------------------------------------- 3\n with pytest.raises(lk.LockTimeoutError):\n lock.acquire_write(lock_fail_timeout)\n lock.acquire_read()\n barrier.wait() # ---------------------------------------- 4\n lock.release_read()\n barrier.wait() # ---------------------------------------- 5\n\n # p2 upgrades read to write\n barrier.wait() # ---------------------------------------- 6\n with pytest.raises(lk.LockTimeoutError):\n lock.acquire_write(lock_fail_timeout)\n with pytest.raises(lk.LockTimeoutError):\n lock.acquire_read(lock_fail_timeout)\n barrier.wait() # ---------------------------------------- 7\n # p2 releases write and read\n barrier.wait() # ---------------------------------------- 8\n\n # p3 acquires read\n barrier.wait() # ---------------------------------------- 9\n # p3 upgrades read to write\n barrier.wait() # ---------------------------------------- 10\n with pytest.raises(lk.LockTimeoutError):\n lock.acquire_write(lock_fail_timeout)\n with pytest.raises(lk.LockTimeoutError):\n lock.acquire_read(lock_fail_timeout)\n barrier.wait() # ---------------------------------------- 11\n # p3 releases locks\n barrier.wait() # ---------------------------------------- 12\n lock.acquire_read()\n barrier.wait() # ---------------------------------------- 13\n lock.release_read()\n\n def p2(self, barrier):\n lock = lk.Lock(self.lock_path)\n\n # p1 acquires write\n barrier.wait() # ---------------------------------------- 1\n with pytest.raises(lk.LockTimeoutError):\n lock.acquire_write(lock_fail_timeout)\n with pytest.raises(lk.LockTimeoutError):\n lock.acquire_read(lock_fail_timeout)\n barrier.wait() # ---------------------------------------- 2\n lock.acquire_read()\n barrier.wait() # ---------------------------------------- 3\n # p1 tests shared read\n barrier.wait() # ---------------------------------------- 4\n # others release reads\n barrier.wait() # ---------------------------------------- 5\n\n lock.acquire_write() # upgrade read to write\n barrier.wait() # ---------------------------------------- 6\n # others test timeout\n barrier.wait() # ---------------------------------------- 7\n lock.release_write() # release read AND write (need both)\n lock.release_read()\n barrier.wait() # ---------------------------------------- 8\n\n # p3 acquires read\n barrier.wait() # ---------------------------------------- 9\n # p3 upgrades read to write\n barrier.wait() # ---------------------------------------- 10\n with pytest.raises(lk.LockTimeoutError):\n lock.acquire_write(lock_fail_timeout)\n with pytest.raises(lk.LockTimeoutError):\n lock.acquire_read(lock_fail_timeout)\n barrier.wait() # ---------------------------------------- 11\n # p3 releases locks\n barrier.wait() # ---------------------------------------- 12\n lock.acquire_read()\n barrier.wait() # ---------------------------------------- 13\n lock.release_read()\n\n def p3(self, barrier):\n lock = lk.Lock(self.lock_path)\n\n # p1 acquires write\n barrier.wait() # ---------------------------------------- 1\n with pytest.raises(lk.LockTimeoutError):\n lock.acquire_write(lock_fail_timeout)\n with pytest.raises(lk.LockTimeoutError):\n lock.acquire_read(lock_fail_timeout)\n barrier.wait() # ---------------------------------------- 2\n lock.acquire_read()\n barrier.wait() # ---------------------------------------- 3\n # p1 tests shared read\n barrier.wait() # ---------------------------------------- 4\n lock.release_read()\n barrier.wait() # ---------------------------------------- 5\n\n # p2 upgrades read to write\n barrier.wait() # ---------------------------------------- 6\n with pytest.raises(lk.LockTimeoutError):\n lock.acquire_write(lock_fail_timeout)\n with pytest.raises(lk.LockTimeoutError):\n lock.acquire_read(lock_fail_timeout)\n barrier.wait() # ---------------------------------------- 7\n # p2 releases write & read\n barrier.wait() # ---------------------------------------- 8\n\n lock.acquire_read()\n barrier.wait() # ---------------------------------------- 9\n lock.acquire_write()\n barrier.wait() # ---------------------------------------- 10\n # others test timeout\n barrier.wait() # ---------------------------------------- 11\n lock.release_read() # release read AND write in opposite\n lock.release_write() # order from before on p2\n barrier.wait() # ---------------------------------------- 12\n lock.acquire_read()\n barrier.wait() # ---------------------------------------- 13\n lock.release_read()\n\n\n#\n# Longer test case that ensures locks are reusable. Ordering is\n# enforced by barriers throughout -- steps are shown with numbers.\n#\ndef test_complex_acquire_and_release_chain(lock_path):\n test_chain = ComplexAcquireAndRelease(lock_path)\n multiproc_test(test_chain.p1, test_chain.p2, test_chain.p3)\n\n\nclass AssertLock(lk.Lock):\n \"\"\"Test lock class that marks acquire/release events.\"\"\"\n\n def __init__(self, lock_path, vals):\n super().__init__(lock_path)\n self.vals = vals\n\n # assert hooks for subclasses\n assert_acquire_read = lambda self: None\n assert_acquire_write = lambda self: None\n assert_release_read = lambda self: None\n assert_release_write = lambda self: None\n\n def acquire_read(self, timeout=None):\n self.assert_acquire_read()\n result = super().acquire_read(timeout)\n self.vals[\"acquired_read\"] = True\n return result\n\n def acquire_write(self, timeout=None):\n self.assert_acquire_write()\n result = super().acquire_write(timeout)\n self.vals[\"acquired_write\"] = True\n return result\n\n def release_read(self, release_fn=None):\n self.assert_release_read()\n result = super().release_read(release_fn)\n self.vals[\"released_read\"] = True\n return result\n\n def release_write(self, release_fn=None):\n self.assert_release_write()\n result = super().release_write(release_fn)\n self.vals[\"released_write\"] = True\n return result\n\n\n@pytest.mark.parametrize(\n \"transaction,type\", [(lk.ReadTransaction, \"read\"), (lk.WriteTransaction, \"write\")]\n)\ndef test_transaction(lock_path, transaction, type):\n class MockLock(AssertLock):\n def assert_acquire_read(self):\n assert not vals[\"entered_fn\"]\n assert not vals[\"exited_fn\"]\n\n def assert_release_read(self):\n assert vals[\"entered_fn\"]\n assert not vals[\"exited_fn\"]\n\n def assert_acquire_write(self):\n assert not vals[\"entered_fn\"]\n assert not vals[\"exited_fn\"]\n\n def assert_release_write(self):\n assert vals[\"entered_fn\"]\n assert not vals[\"exited_fn\"]\n\n def enter_fn():\n # assert enter_fn is called while lock is held\n assert vals[\"acquired_%s\" % type]\n vals[\"entered_fn\"] = True\n\n def exit_fn(t, v, tb):\n # assert exit_fn is called while lock is held\n assert not vals[\"released_%s\" % type]\n vals[\"exited_fn\"] = True\n vals[\"exception\"] = t or v or tb\n\n vals = collections.defaultdict(lambda: False)\n lock = MockLock(lock_path, vals)\n\n with transaction(lock, acquire=enter_fn, release=exit_fn):\n assert vals[\"acquired_%s\" % type]\n assert not vals[\"released_%s\" % type]\n\n assert vals[\"entered_fn\"]\n assert vals[\"exited_fn\"]\n assert vals[\"acquired_%s\" % type]\n assert vals[\"released_%s\" % type]\n assert not vals[\"exception\"]\n\n\n@pytest.mark.parametrize(\n \"transaction,type\", [(lk.ReadTransaction, \"read\"), (lk.WriteTransaction, \"write\")]\n)\ndef test_transaction_with_exception(lock_path, transaction, type):\n class MockLock(AssertLock):\n def assert_acquire_read(self):\n assert not vals[\"entered_fn\"]\n assert not vals[\"exited_fn\"]\n\n def assert_release_read(self):\n assert vals[\"entered_fn\"]\n assert not vals[\"exited_fn\"]\n\n def assert_acquire_write(self):\n assert not vals[\"entered_fn\"]\n assert not vals[\"exited_fn\"]\n\n def assert_release_write(self):\n assert vals[\"entered_fn\"]\n assert not vals[\"exited_fn\"]\n\n def enter_fn():\n assert vals[\"acquired_%s\" % type]\n vals[\"entered_fn\"] = True\n\n def exit_fn(t, v, tb):\n assert not vals[\"released_%s\" % type]\n vals[\"exited_fn\"] = True\n vals[\"exception\"] = t or v or tb\n return exit_result\n\n exit_result = False\n vals = collections.defaultdict(lambda: False)\n lock = MockLock(lock_path, vals)\n\n with pytest.raises(Exception):\n with transaction(lock, acquire=enter_fn, release=exit_fn):\n raise Exception()\n\n assert vals[\"entered_fn\"]\n assert vals[\"exited_fn\"]\n assert vals[\"exception\"]\n\n # test suppression of exceptions from exit_fn\n exit_result = True\n vals.clear()\n\n # should not raise now.\n with transaction(lock, acquire=enter_fn, release=exit_fn):\n raise Exception()\n\n assert vals[\"entered_fn\"]\n assert vals[\"exited_fn\"]\n assert vals[\"exception\"]\n\n\ndef test_nested_write_transaction(lock_path):\n \"\"\"Ensure that the outermost write transaction writes.\"\"\"\n\n def write(t, v, tb):\n vals[\"wrote\"] = True\n\n vals = collections.defaultdict(lambda: False)\n lock = AssertLock(lock_path, vals)\n\n # write/write\n with lk.WriteTransaction(lock, release=write):\n assert not vals[\"wrote\"]\n with lk.WriteTransaction(lock, release=write):\n assert not vals[\"wrote\"]\n assert not vals[\"wrote\"]\n assert vals[\"wrote\"]\n\n # read/write\n vals.clear()\n with lk.ReadTransaction(lock):\n assert not vals[\"wrote\"]\n with lk.WriteTransaction(lock, release=write):\n assert not vals[\"wrote\"]\n assert vals[\"wrote\"]\n\n # write/read/write\n vals.clear()\n with lk.WriteTransaction(lock, release=write):\n assert not vals[\"wrote\"]\n with lk.ReadTransaction(lock):\n assert not vals[\"wrote\"]\n with lk.WriteTransaction(lock, release=write):\n assert not vals[\"wrote\"]\n assert not vals[\"wrote\"]\n assert not vals[\"wrote\"]\n assert vals[\"wrote\"]\n\n # read/write/read/write\n vals.clear()\n with lk.ReadTransaction(lock):\n with lk.WriteTransaction(lock, release=write):\n assert not vals[\"wrote\"]\n with lk.ReadTransaction(lock):\n assert not vals[\"wrote\"]\n with lk.WriteTransaction(lock, release=write):\n assert not vals[\"wrote\"]\n assert not vals[\"wrote\"]\n assert not vals[\"wrote\"]\n assert vals[\"wrote\"]\n\n\ndef test_nested_reads(lock_path):\n \"\"\"Ensure that write transactions won't re-read data.\"\"\"\n\n def read():\n vals[\"read\"] += 1\n\n vals = collections.defaultdict(lambda: 0)\n lock = AssertLock(lock_path, vals)\n\n # read/read\n vals.clear()\n assert vals[\"read\"] == 0\n with lk.ReadTransaction(lock, acquire=read):\n assert vals[\"read\"] == 1\n with lk.ReadTransaction(lock, acquire=read):\n assert vals[\"read\"] == 1\n\n # write/write\n vals.clear()\n assert vals[\"read\"] == 0\n with lk.WriteTransaction(lock, acquire=read):\n assert vals[\"read\"] == 1\n with lk.WriteTransaction(lock, acquire=read):\n assert vals[\"read\"] == 1\n\n # read/write\n vals.clear()\n assert vals[\"read\"] == 0\n with lk.ReadTransaction(lock, acquire=read):\n assert vals[\"read\"] == 1\n with lk.WriteTransaction(lock, acquire=read):\n assert vals[\"read\"] == 1\n\n # write/read/write\n vals.clear()\n assert vals[\"read\"] == 0\n with lk.WriteTransaction(lock, acquire=read):\n assert vals[\"read\"] == 1\n with lk.ReadTransaction(lock, acquire=read):\n assert vals[\"read\"] == 1\n with lk.WriteTransaction(lock, acquire=read):\n assert vals[\"read\"] == 1\n\n # read/write/read/write\n vals.clear()\n assert vals[\"read\"] == 0\n with lk.ReadTransaction(lock, acquire=read):\n assert vals[\"read\"] == 1\n with lk.WriteTransaction(lock, acquire=read):\n assert vals[\"read\"] == 1\n with lk.ReadTransaction(lock, acquire=read):\n assert vals[\"read\"] == 1\n with lk.WriteTransaction(lock, acquire=read):\n assert vals[\"read\"] == 1\n\n\nclass LockDebugOutput:\n def __init__(self, lock_path):\n self.lock_path = lock_path\n self.host = socket.gethostname()\n\n def p1(self, barrier, q1, q2):\n # exchange pids\n p1_pid = os.getpid()\n q1.put(p1_pid)\n p2_pid = q2.get()\n\n # set up lock\n lock = lk.Lock(self.lock_path, debug=True)\n\n with lk.WriteTransaction(lock):\n # p1 takes write lock and writes pid/host to file\n barrier.wait() # ------------------------------------ 1\n\n assert lock.pid == p1_pid\n assert lock.host == self.host\n\n # wait for p2 to verify contents of file\n barrier.wait() # ---------------------------------------- 2\n\n # wait for p2 to take a write lock\n barrier.wait() # ---------------------------------------- 3\n\n # verify pid/host info again\n with lk.ReadTransaction(lock):\n assert lock.old_pid == p1_pid\n assert lock.old_host == self.host\n\n assert lock.pid == p2_pid\n assert lock.host == self.host\n\n barrier.wait() # ---------------------------------------- 4\n\n def p2(self, barrier, q1, q2):\n # exchange pids\n p2_pid = os.getpid()\n p1_pid = q1.get()\n q2.put(p2_pid)\n\n # set up lock\n lock = lk.Lock(self.lock_path, debug=True)\n\n # p1 takes write lock and writes pid/host to file\n barrier.wait() # ---------------------------------------- 1\n\n # verify that p1 wrote information to lock file\n with lk.ReadTransaction(lock):\n assert lock.pid == p1_pid\n assert lock.host == self.host\n\n barrier.wait() # ---------------------------------------- 2\n\n # take a write lock on the file and verify pid/host info\n with lk.WriteTransaction(lock):\n assert lock.old_pid == p1_pid\n assert lock.old_host == self.host\n\n assert lock.pid == p2_pid\n assert lock.host == self.host\n\n barrier.wait() # ------------------------------------ 3\n\n # wait for p1 to verify pid/host info\n barrier.wait() # ---------------------------------------- 4\n\n\ndef test_lock_debug_output(lock_path):\n test_debug = LockDebugOutput(lock_path)\n q1, q2 = Queue(), Queue()\n local_multiproc_test(test_debug.p2, test_debug.p1, extra_args=(q1, q2))\n\n\ndef test_lock_with_no_parent_directory(tmp_path: pathlib.Path):\n \"\"\"Make sure locks work even when their parent directory does not exist.\"\"\"\n with working_dir(str(tmp_path)):\n lock = lk.Lock(\"foo/bar/baz/lockfile\")\n with lk.WriteTransaction(lock):\n pass\n\n\ndef test_lock_in_current_directory(tmp_path: pathlib.Path):\n \"\"\"Make sure locks work even when their parent directory does not exist.\"\"\"\n with working_dir(str(tmp_path)):\n # test we can create a lock in the current directory\n lock = lk.Lock(\"lockfile\")\n for i in range(10):\n with lk.ReadTransaction(lock):\n pass\n with lk.WriteTransaction(lock):\n pass\n\n # and that we can do the same thing after it's already there\n lock = lk.Lock(\"lockfile\")\n for i in range(10):\n with lk.ReadTransaction(lock):\n pass\n with lk.WriteTransaction(lock):\n pass\n\n\ndef test_attempts_str():\n assert lk._attempts_str(0, 0) == \"\"\n assert lk._attempts_str(0.12, 1) == \"\"\n assert lk._attempts_str(12.345, 2) == \" after 12.345s and 2 attempts\"\n\n\ndef test_lock_str():\n lock = lk.Lock(\"lockfile\")\n lockstr = str(lock)\n assert \"lockfile[0:0]\" in lockstr\n assert \"timeout=None\" in lockstr\n assert \"#reads=0, #writes=0\" in lockstr\n\n\ndef test_downgrade_write_okay(tmp_path: pathlib.Path):\n \"\"\"Test the lock write-to-read downgrade operation.\"\"\"\n with working_dir(str(tmp_path)):\n lock = lk.Lock(\"lockfile\")\n lock.acquire_write()\n lock.downgrade_write_to_read()\n assert lock._reads == 1\n assert lock._writes == 0\n lock.release_read()\n\n\ndef test_downgrade_write_fails(tmp_path: pathlib.Path):\n \"\"\"Test failing the lock write-to-read downgrade operation.\"\"\"\n with working_dir(str(tmp_path)):\n lock = lk.Lock(\"lockfile\")\n lock.acquire_read()\n msg = \"Cannot downgrade lock from write to read on file: lockfile\"\n with pytest.raises(lk.LockDowngradeError, match=msg):\n lock.downgrade_write_to_read()\n lock.release_read()\n\n\n@pytest.mark.parametrize(\n \"err_num,err_msg\",\n [\n (errno.EACCES, \"Fake EACCES error\"),\n (errno.EAGAIN, \"Fake EAGAIN error\"),\n (errno.ENOENT, \"Fake ENOENT error\"),\n ],\n)\ndef test_poll_lock_exception(tmp_path: pathlib.Path, monkeypatch, err_num, err_msg):\n \"\"\"Test poll lock exception handling.\"\"\"\n\n def _lockf(fd, cmd, len, start, whence):\n raise OSError(err_num, err_msg)\n\n with working_dir(str(tmp_path)):\n lockfile = \"lockfile\"\n lock = lk.Lock(lockfile)\n lock.acquire_read()\n\n monkeypatch.setattr(fcntl, \"lockf\", _lockf)\n\n if err_num in [errno.EAGAIN, errno.EACCES]:\n assert not lock._poll_lock(fcntl.LOCK_EX)\n else:\n with pytest.raises(OSError, match=err_msg):\n lock._poll_lock(fcntl.LOCK_EX)\n\n monkeypatch.undo()\n lock.release_read()\n\n\ndef test_upgrade_read_okay(tmp_path: pathlib.Path):\n \"\"\"Test the lock read-to-write upgrade operation.\"\"\"\n with working_dir(str(tmp_path)):\n lock = lk.Lock(\"lockfile\")\n lock.acquire_read()\n lock.upgrade_read_to_write()\n assert lock._reads == 0\n assert lock._writes == 1\n lock.release_write()\n\n\ndef test_upgrade_read_fails(tmp_path: pathlib.Path):\n \"\"\"Test failing the lock read-to-write upgrade operation.\"\"\"\n with working_dir(str(tmp_path)):\n lock = lk.Lock(\"lockfile\")\n lock.acquire_write()\n msg = \"Cannot upgrade lock from read to write on file: lockfile\"\n with pytest.raises(lk.LockUpgradeError, match=msg):\n lock.upgrade_read_to_write()\n lock.release_write()\n\n\n@pytest.mark.parametrize(\"acquire\", [\"acquire_write\", \"acquire_read\"])\ndef test_acquire_after_fork(tmp_path: pathlib.Path, acquire: str):\n \"\"\"After fork, acquire_write/read must not silently succeed due to inherited counters.\"\"\"\n try:\n ctx = multiprocessing.get_context(\"fork\")\n except ValueError:\n pytest.skip(\"fork start method not available on this platform\")\n\n lockfile = str(tmp_path / \"lockfile\")\n lock = lk.Lock(lockfile)\n result = ctx.Queue()\n\n def child():\n assert lock._writes == 1 # due to forking, but POSIX lock is NOT held by this process\n try:\n if acquire == \"acquire_write\":\n lock.acquire_write(lock_fail_timeout)\n elif acquire == \"acquire_read\":\n lock.acquire_read(lock_fail_timeout)\n else:\n assert False # should never get here\n result.put(\"no_error\")\n except lk.LockTimeoutError:\n result.put(\"timed_out\")\n\n lock.acquire_write()\n try:\n p = ctx.Process(target=child)\n p.start()\n p.join()\n assert result.get() == \"timed_out\"\n finally:\n lock.release_write()\n\n\ndef _child_try_acquire_write(lock_path: str, result_queue):\n lock = lk.Lock(lock_path)\n result_queue.put(lock.try_acquire_write())\n\n\ndef _child_try_acquire_read(lock_path: str, result_queue):\n lock = lk.Lock(lock_path)\n result_queue.put(lock.try_acquire_read())\n\n\ndef test_try_acquire_read(tmp_path: pathlib.Path):\n \"\"\"Test non-blocking try_acquire_read.\"\"\"\n lock = lk.Lock(str(tmp_path / \"lockfile\"))\n\n # Succeeds on unlocked lock\n assert lock.try_acquire_read() is True\n assert lock._reads == 1\n\n # Succeeds again (nested)\n assert lock.try_acquire_read() is True\n assert lock._reads == 2\n\n lock.release_read()\n lock.release_read()\n ctx = multiprocessing.get_context()\n\n # Fails when another process holds an exclusive write lock\n lock.acquire_write()\n try:\n q = ctx.Queue()\n p = ctx.Process(target=_child_try_acquire_read, args=(str(tmp_path / \"lockfile\"), q))\n p.start()\n p.join()\n assert q.get() is False\n finally:\n lock.release_write()\n\n\ndef test_try_acquire_write(tmp_path: pathlib.Path):\n \"\"\"Test non-blocking try_acquire_write.\"\"\"\n lock = lk.Lock(str(tmp_path / \"lockfile\"))\n ctx = multiprocessing.get_context()\n\n # Succeeds on unlocked lock\n assert lock.try_acquire_write() is True\n assert lock._writes == 1\n\n # Succeeds again (nested)\n assert lock.try_acquire_write() is True\n assert lock._writes == 2\n\n lock.release_write()\n lock.release_write()\n\n # Fails when another process holds a write lock\n lock.acquire_write()\n try:\n q = ctx.Queue()\n p = ctx.Process(target=_child_try_acquire_write, args=(str(tmp_path / \"lockfile\"), q))\n p.start()\n p.join()\n assert q.get() is False\n finally:\n lock.release_write()\n\n # Fails when another process holds a read lock\n lock.acquire_read()\n try:\n q = ctx.Queue()\n p = ctx.Process(target=_child_try_acquire_write, args=(str(tmp_path / \"lockfile\"), q))\n p.start()\n p.join()\n assert q.get() is False\n finally:\n lock.release_read()\n\n\ndef _child_fails_to_acquire_read(_lock: lk.Lock):\n try:\n _lock.acquire_read(timeout=1e-9)\n except lk.LockTimeoutError:\n return\n assert False, \"Child process should not have been able to acquire read lock\"\n\n\ndef test_read_after_write_does_not_accidentally_downgrade(tmp_path: pathlib.Path):\n \"\"\"Test that acquiring a read lock after a write lock does not accidentally downgrade the\n write lock, by having another process attempt to acquire a read lock.\"\"\"\n lock = lk.Lock(str(tmp_path / \"lockfile\"))\n lock.acquire_write()\n lock.acquire_read() # should not downgrade the write lock\n try:\n # No matter the start method, the child process shouldn't be able to acquire a read lock.\n p = multiprocessing.Process(target=_child_fails_to_acquire_read, args=(lock,))\n p.start()\n p.join()\n assert p.exitcode == 0\n finally:\n lock.release_read()\n lock.release_write()\n" }, { "path": "lib/spack/spack/test/llnl/util/symlink.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Tests for Windows symlink functionality.\"\"\"\n\nimport os\nimport pathlib\nimport tempfile\n\nimport pytest\n\nimport spack.llnl.util.filesystem as fs\n\n\ndef test_symlink_file(tmp_path: pathlib.Path):\n \"\"\"Test the symlink functionality on all operating systems for a file\"\"\"\n with fs.working_dir(str(tmp_path)):\n test_dir = str(tmp_path)\n fd, real_file = tempfile.mkstemp(prefix=\"real\", suffix=\".txt\", dir=test_dir)\n link_file = str(tmp_path / \"link.txt\")\n assert os.path.exists(link_file) is False\n fs.symlink(real_file, link_file)\n assert os.path.exists(link_file)\n assert fs.islink(link_file)\n\n\ndef test_symlink_dir(tmp_path: pathlib.Path):\n \"\"\"Test the symlink functionality on all operating systems for a directory\"\"\"\n with fs.working_dir(str(tmp_path)):\n test_dir = str(tmp_path)\n real_dir = os.path.join(test_dir, \"real_dir\")\n link_dir = os.path.join(test_dir, \"link_dir\")\n os.mkdir(real_dir)\n fs.symlink(real_dir, link_dir)\n assert os.path.exists(link_dir)\n assert fs.islink(link_dir)\n\n\n@pytest.mark.only_windows(\"Test is for Windows specific behavior\")\ndef test_symlink_source_not_exists(tmp_path: pathlib.Path):\n \"\"\"Test the symlink method for the case where a source path does not exist\"\"\"\n with fs.working_dir(str(tmp_path)):\n test_dir = str(tmp_path)\n real_dir = os.path.join(test_dir, \"real_dir\")\n link_dir = os.path.join(test_dir, \"link_dir\")\n with pytest.raises(fs.SymlinkError):\n fs._windows_symlink(real_dir, link_dir)\n\n\ndef test_symlink_src_relative_to_link(tmp_path: pathlib.Path):\n \"\"\"Test the symlink functionality where the source value exists relative to the link\n but not relative to the cwd\"\"\"\n with fs.working_dir(str(tmp_path)):\n subdir_1 = tmp_path / \"a\"\n subdir_2 = os.path.join(subdir_1, \"b\")\n link_dir = os.path.join(subdir_1, \"c\")\n\n os.mkdir(subdir_1)\n os.mkdir(subdir_2)\n\n fd, real_file = tempfile.mkstemp(prefix=\"real\", suffix=\".txt\", dir=subdir_2)\n link_file = os.path.join(subdir_1, \"link.txt\")\n\n fs.symlink(f\"b/{os.path.basename(real_file)}\", f\"a/{os.path.basename(link_file)}\")\n assert os.path.exists(link_file)\n assert fs.islink(link_file)\n # Check dirs\n assert not os.path.lexists(link_dir)\n fs.symlink(\"b\", \"a/c\")\n assert os.path.lexists(link_dir)\n\n\n@pytest.mark.only_windows(\"Test is for Windows specific behavior\")\ndef test_symlink_src_not_relative_to_link(tmp_path: pathlib.Path):\n \"\"\"Test the symlink functionality where the source value does not exist relative to\n the link and not relative to the cwd. NOTE that this symlink api call is EXPECTED to raise\n a fs.SymlinkError exception that we catch.\"\"\"\n with fs.working_dir(str(tmp_path)):\n test_dir = str(tmp_path)\n subdir_1 = os.path.join(test_dir, \"a\")\n subdir_2 = os.path.join(subdir_1, \"b\")\n link_dir = os.path.join(subdir_1, \"c\")\n os.mkdir(subdir_1)\n os.mkdir(subdir_2)\n fd, real_file = tempfile.mkstemp(prefix=\"real\", suffix=\".txt\", dir=subdir_2)\n link_file = str(tmp_path / \"link.txt\")\n # Expected SymlinkError because source path does not exist relative to link path\n with pytest.raises(fs.SymlinkError):\n fs._windows_symlink(\n f\"d/{os.path.basename(real_file)}\", f\"a/{os.path.basename(link_file)}\"\n )\n assert not os.path.exists(link_file)\n # Check dirs\n assert not os.path.lexists(link_dir)\n with pytest.raises(fs.SymlinkError):\n fs._windows_symlink(\"d\", \"a/c\")\n assert not os.path.lexists(link_dir)\n\n\n@pytest.mark.only_windows(\"Test is for Windows specific behavior\")\ndef test_symlink_link_already_exists(tmp_path: pathlib.Path):\n \"\"\"Test the symlink method for the case where a link already exists\"\"\"\n with fs.working_dir(str(tmp_path)):\n test_dir = str(tmp_path)\n real_dir = os.path.join(test_dir, \"real_dir\")\n link_dir = os.path.join(test_dir, \"link_dir\")\n os.mkdir(real_dir)\n fs._windows_symlink(real_dir, link_dir)\n assert os.path.exists(link_dir)\n with pytest.raises(fs.SymlinkError):\n fs._windows_symlink(real_dir, link_dir)\n\n\n@pytest.mark.skipif(not fs._windows_can_symlink(), reason=\"Test requires elevated privileges\")\n@pytest.mark.only_windows(\"Test is for Windows specific behavior\")\ndef test_symlink_win_file(tmp_path: pathlib.Path):\n \"\"\"Check that symlink makes a symlink file when run with elevated permissions\"\"\"\n with fs.working_dir(str(tmp_path)):\n test_dir = str(tmp_path)\n fd, real_file = tempfile.mkstemp(prefix=\"real\", suffix=\".txt\", dir=test_dir)\n link_file = str(tmp_path / \"link.txt\")\n fs.symlink(real_file, link_file)\n # Verify that all expected conditions are met\n assert os.path.exists(link_file)\n assert fs.islink(link_file)\n assert os.path.islink(link_file)\n assert not fs._windows_is_hardlink(link_file)\n assert not fs._windows_is_junction(link_file)\n\n\n@pytest.mark.skipif(not fs._windows_can_symlink(), reason=\"Test requires elevated privileges\")\n@pytest.mark.only_windows(\"Test is for Windows specific behavior\")\ndef test_symlink_win_dir(tmp_path: pathlib.Path):\n \"\"\"Check that symlink makes a symlink dir when run with elevated permissions\"\"\"\n with fs.working_dir(str(tmp_path)):\n test_dir = str(tmp_path)\n real_dir = os.path.join(test_dir, \"real\")\n link_dir = os.path.join(test_dir, \"link\")\n os.mkdir(real_dir)\n fs.symlink(real_dir, link_dir)\n # Verify that all expected conditions are met\n assert os.path.exists(link_dir)\n assert fs.islink(link_dir)\n assert os.path.islink(link_dir)\n assert not fs._windows_is_hardlink(link_dir)\n assert not fs._windows_is_junction(link_dir)\n\n\n@pytest.mark.only_windows(\"Test is for Windows specific behavior\")\ndef test_windows_create_junction(tmp_path: pathlib.Path):\n \"\"\"Test the _windows_create_junction method\"\"\"\n with fs.working_dir(str(tmp_path)):\n test_dir = str(tmp_path)\n junction_real_dir = os.path.join(test_dir, \"real_dir\")\n junction_link_dir = os.path.join(test_dir, \"link_dir\")\n os.mkdir(junction_real_dir)\n fs._windows_create_junction(junction_real_dir, junction_link_dir)\n # Verify that all expected conditions are met\n assert os.path.exists(junction_link_dir)\n assert fs._windows_is_junction(junction_link_dir)\n assert fs.islink(junction_link_dir)\n assert not os.path.islink(junction_link_dir)\n\n\n@pytest.mark.only_windows(\"Test is for Windows specific behavior\")\ndef test_windows_create_hard_link(tmp_path: pathlib.Path):\n \"\"\"Test the _windows_create_hard_link method\"\"\"\n with fs.working_dir(str(tmp_path)):\n test_dir = str(tmp_path)\n fd, real_file = tempfile.mkstemp(prefix=\"real\", suffix=\".txt\", dir=test_dir)\n link_file = str(tmp_path / \"link.txt\")\n fs._windows_create_hard_link(real_file, link_file)\n # Verify that all expected conditions are met\n assert os.path.exists(link_file)\n assert fs._windows_is_hardlink(real_file)\n assert fs._windows_is_hardlink(link_file)\n assert fs.islink(link_file)\n assert not os.path.islink(link_file)\n\n\n@pytest.mark.only_windows(\"Test is for Windows specific behavior\")\ndef test_windows_create_link_dir(tmp_path: pathlib.Path):\n \"\"\"Test the functionality of the windows_create_link method with a directory\n which should result in making a junction.\n \"\"\"\n with fs.working_dir(str(tmp_path)):\n test_dir = str(tmp_path)\n real_dir = os.path.join(test_dir, \"real\")\n link_dir = os.path.join(test_dir, \"link\")\n os.mkdir(real_dir)\n fs._windows_create_link(real_dir, link_dir)\n # Verify that all expected conditions are met\n assert os.path.exists(link_dir)\n assert fs.islink(link_dir)\n assert not fs._windows_is_hardlink(link_dir)\n assert fs._windows_is_junction(link_dir)\n assert not os.path.islink(link_dir)\n\n\n@pytest.mark.only_windows(\"Test is for Windows specific behavior\")\ndef test_windows_create_link_file(tmp_path: pathlib.Path):\n \"\"\"Test the functionality of the windows_create_link method with a file\n which should result in the creation of a hard link. It also tests the\n functionality of the symlink islink infrastructure.\n \"\"\"\n with fs.working_dir(str(tmp_path)):\n test_dir = str(tmp_path)\n fd, real_file = tempfile.mkstemp(prefix=\"real\", suffix=\".txt\", dir=test_dir)\n link_file = str(tmp_path / \"link.txt\")\n fs._windows_create_link(real_file, link_file)\n # Verify that all expected conditions are met\n assert fs._windows_is_hardlink(link_file)\n assert fs.islink(link_file)\n assert not fs._windows_is_junction(link_file)\n\n\n@pytest.mark.only_windows(\"Test is for Windows specific behavior\")\ndef test_windows_read_link(tmp_path: pathlib.Path):\n \"\"\"Makes sure readlink can read the link source for hard links and\n junctions on windows.\"\"\"\n with fs.working_dir(str(tmp_path)):\n real_dir_1 = \"real_dir_1\"\n real_dir_2 = \"real_dir_2\"\n link_dir_1 = \"link_dir_1\"\n link_dir_2 = \"link_dir_2\"\n os.mkdir(real_dir_1)\n os.mkdir(real_dir_2)\n\n # Create a file and a directory\n _, real_file_1 = tempfile.mkstemp(prefix=\"real_1\", suffix=\".txt\", dir=\".\")\n _, real_file_2 = tempfile.mkstemp(prefix=\"real_2\", suffix=\".txt\", dir=\".\")\n link_file_1 = \"link_1.txt\"\n link_file_2 = \"link_2.txt\"\n\n # Make hard link/junction\n fs._windows_create_hard_link(real_file_1, link_file_1)\n fs._windows_create_hard_link(real_file_2, link_file_2)\n fs._windows_create_junction(real_dir_1, link_dir_1)\n fs._windows_create_junction(real_dir_2, link_dir_2)\n\n assert fs.readlink(link_file_1) == os.path.abspath(real_file_1)\n assert fs.readlink(link_file_2) == os.path.abspath(real_file_2)\n assert fs.readlink(link_dir_1) == os.path.abspath(real_dir_1)\n assert fs.readlink(link_dir_2) == os.path.abspath(real_dir_2)\n" }, { "path": "lib/spack/spack/test/llnl/util/tty/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n" }, { "path": "lib/spack/spack/test/llnl/util/tty/colify.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport re\nimport sys\n\nimport pytest\n\nfrom spack.llnl.util.tty.colify import colify, colify_table\n\n# table as 3 rows x 6 columns\nlorem_table = [\n [\"There\", \"are\", \"many\", \"variations\", \"of\", \"passages\"],\n [\"of\", \"Lorem\", \"Ipsum\", \"available\", \"but\", \"many\"],\n [\"have\", \"suffered\", \"alteration\", \"in\", \"some\", \"form\"],\n]\n\n# width of each column in above table\nlorem_table_col_starts = [0, 7, 17, 29, 41, 47]\n\n# table in a single list\nlorem_words = lorem_table[0] + lorem_table[1] + lorem_table[2]\n\n\n@pytest.mark.parametrize(\"console_cols\", [10, 20, 40, 60, 80, 100, 120])\ndef test_fixed_column_table(console_cols, capfd):\n \"ensure output is a fixed table regardless of size\"\n colify_table(lorem_table, output=sys.stdout, console_cols=console_cols)\n output, _ = capfd.readouterr()\n\n # 3 rows\n assert output.strip().count(\"\\n\") == 2\n\n # right spacing\n lines = output.strip().split(\"\\n\")\n for line in lines:\n assert [line[w - 1] for w in lorem_table_col_starts[1:]] == [\" \"] * 5\n\n # same data\n stripped_lines = [re.sub(r\"\\s+\", \" \", line.strip()) for line in lines]\n assert stripped_lines == [\" \".join(row) for row in lorem_table]\n\n\n@pytest.mark.parametrize(\n \"console_cols,expected_rows,expected_cols\",\n [\n (10, 18, 1),\n (20, 18, 1),\n (40, 5, 4),\n (60, 3, 6),\n (80, 2, 9),\n (100, 2, 9),\n (120, 2, 9),\n (140, 1, 18),\n ],\n)\ndef test_variable_width_columns(console_cols, expected_rows, expected_cols, capfd):\n colify(lorem_words, tty=True, output=sys.stdout, console_cols=console_cols)\n output, _ = capfd.readouterr()\n\n print(output)\n # expected rows\n assert output.strip().count(\"\\n\") == expected_rows - 1\n\n # right cols\n lines = output.strip().split(\"\\n\")\n assert all(len(re.split(r\"\\s+\", line)) <= expected_cols for line in lines)\n\n # padding between columns\n rows = [re.split(r\"\\s+\", line) for line in lines]\n cols = list(zip(*rows))\n\n max_col_widths = [max(len(s) for s in col) for col in cols]\n col_start = 0\n for w in max_col_widths:\n col_start += w + 2 # plus padding\n\n # verify that every column boundary is at max width + padding\n assert all(\n [\n line[col_start - 1] == \" \" and line[col_start] != \" \"\n for line in lines\n if col_start < len(line)\n ]\n )\n" }, { "path": "lib/spack/spack/test/llnl/util/tty/color.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport re\nimport textwrap\n\nimport pytest\n\nimport spack.llnl.util.tty.color as color\nfrom spack.llnl.util.tty.color import cescape, colorize, csub\n\ntest_text = [\n \"@r{The quick brown fox jumps over the lazy yellow dog.\",\n \"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt \"\n \"ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco \"\n \"laboris nisi ut aliquip ex ea commodo consequat.}\",\n \"@c{none, gfx1010, gfx1011, gfx1012, gfx1013, gfx1030, gfx1031, gfx1032, gfx1033, gfx1034}\",\n \"none, @c{gfx1010}, gfx1011, @r{gfx1012}, gfx1013, @b{gfx1030}, gfx1031, gfx1032, gfx1033\",\n \"@c{none, 10, 100, 100a, 100f, 101, 101a, 101f, 103, 103a, 103f, 11, 12, 120, 120a, 120f}\",\n \"@c{none, 10, 100, 100a, 100f, 101, 101a, 101f, 103, 103a, 103f, 11, 12, 120}\",\n \"none, @c{10}, @b{100}, 100a, @r{100f}, 101, @g{101a}, 101f, @c{103}, 103a, 103f\"\n \"@g{build}, @c{link}, @r{run}\",\n]\n\n\n@pytest.mark.parametrize(\"cols\", list(range(30, 101, 10)))\n@pytest.mark.parametrize(\"text\", test_text)\n@pytest.mark.parametrize(\"indent\", [0, 4, 8])\ndef test_color_wrap(cols, text, indent):\n colorized = color.colorize(text, color=True) # True to force color\n plain = color.csub(colorized)\n\n spaces = indent * \" \"\n\n color_wrapped = \" \".join(\n color.cwrap(colorized, width=cols, initial_indent=spaces, subsequent_indent=spaces)\n )\n plain_cwrapped = \" \".join(\n color.cwrap(plain, width=cols, initial_indent=spaces, subsequent_indent=spaces)\n )\n wrapped = \" \".join(\n textwrap.wrap(plain, width=cols, initial_indent=spaces, subsequent_indent=spaces)\n )\n\n # make sure the concatenated, non-indented wrapped version is the same as the\n # original, modulo any spaces consumed while wrapping.\n assert re.sub(r\"\\s+\", \" \", color_wrapped).lstrip() == re.sub(r\"\\s+\", \" \", colorized)\n\n # make sure we wrap the same as textwrap\n assert color.csub(color_wrapped) == wrapped\n assert plain_cwrapped == wrapped\n\n\ndef test_cescape_at_sign_roundtrip():\n \"\"\"cescape followed by colorize should not double-escape '@' inside color blocks.\"\"\"\n raw = 'if spec.satisfies(\"@:25.1\"):'\n colorized = colorize(\"@R{%s}\" % cescape(raw), color=True)\n assert csub(colorized) == raw\n\n\ndef test_cescape_multiple_at_signs_roundtrip():\n \"\"\"Multiple consecutive '@' characters should survive a cescape/colorize roundtrip.\"\"\"\n raw = \"foo @@@@@bar\"\n colorized = colorize(\"@R{%s}\" % cescape(raw), color=True)\n assert csub(colorized) == raw\n\n\ndef test_colorize_top_level_consecutive_escaped_ats():\n \"\"\"Consecutive @@ at the top level (outside braces) must each unescape independently.\"\"\"\n assert colorize(\"@@@@\", color=False) == \"@@\"\n assert colorize(\"@@@@@@\", color=False) == \"@@@\"\n" }, { "path": "lib/spack/spack/test/llnl/util/tty/log.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport contextlib\nimport pathlib\nimport sys\nfrom types import ModuleType\nfrom typing import Optional\n\nimport pytest\n\nimport spack.llnl.util.tty.log as log\nfrom spack.llnl.util.filesystem import working_dir\nfrom spack.util.executable import Executable\n\ntermios: Optional[ModuleType] = None\ntry:\n import termios as term_mod\n\n termios = term_mod\nexcept ImportError:\n pass\n\n\npytestmark = pytest.mark.not_on_windows(\"does not run on windows\")\n\n\n@contextlib.contextmanager\ndef nullcontext():\n yield\n\n\ndef test_log_python_output_with_echo(capfd, tmp_path: pathlib.Path):\n with working_dir(str(tmp_path)):\n with log.log_output(\"foo.txt\", echo=True):\n print(\"logged\")\n\n # foo.txt has output\n with open(\"foo.txt\", encoding=\"utf-8\") as f:\n assert f.read() == \"logged\\n\"\n\n # output is also echoed.\n assert capfd.readouterr()[0] == \"logged\\n\"\n\n\ndef test_log_python_output_without_echo(capfd, tmp_path: pathlib.Path):\n with working_dir(str(tmp_path)):\n with log.log_output(\"foo.txt\"):\n print(\"logged\")\n\n # foo.txt has output\n with open(\"foo.txt\", encoding=\"utf-8\") as f:\n assert f.read() == \"logged\\n\"\n\n # nothing on stdout or stderr\n assert capfd.readouterr()[0] == \"\"\n\n\ndef test_log_python_output_with_invalid_utf8(capfd, tmp_path: pathlib.Path):\n tmp_file = str(tmp_path / \"foo.txt\")\n with log.log_output(tmp_file, echo=True):\n sys.stdout.buffer.write(b\"\\xc3helloworld\\n\")\n\n # we should be able to read this as valid utf-8\n with open(tmp_file, \"r\", encoding=\"utf-8\") as f:\n assert f.read() == \"�helloworld\\n\"\n\n assert capfd.readouterr().out == \"�helloworld\\n\"\n\n\ndef test_log_python_output_and_echo_output(capfd, tmp_path: pathlib.Path):\n with working_dir(str(tmp_path)):\n # echo two lines\n with log.log_output(\"foo.txt\") as logger:\n with logger.force_echo():\n print(\"force echo\")\n print(\"logged\")\n\n # log file contains everything\n with open(\"foo.txt\", encoding=\"utf-8\") as f:\n assert f.read() == \"force echo\\nlogged\\n\"\n\n # only force-echo'd stuff is in output\n assert capfd.readouterr()[0] == \"force echo\\n\"\n\n\ndef test_log_output_with_control_codes(capfd, tmp_path: pathlib.Path):\n with working_dir(str(tmp_path)):\n with log.log_output(\"foo.txt\"):\n # Print a sample of formatted GCC error output\n # Line obtained from the file generated by running gcc on a nonexistent file:\n # gcc -fdiagnostics-color=always ./test.cpp 2>test.log\n csi = \"\\x1b[\"\n print(\n f\"{csi}01m{csi}Kgcc:{csi}m{csi}K {csi}01;31m{csi}Kerror: {csi}m{csi}K./test.cpp:\"\n )\n\n with open(\"foo.txt\", encoding=\"utf-8\") as f:\n assert f.read() == \"gcc: error: ./test.cpp:\\n\"\n\n\ndef _log_filter_fn(string):\n return string.replace(\"foo\", \"bar\")\n\n\ndef test_log_output_with_filter(capfd, tmp_path: pathlib.Path):\n with working_dir(str(tmp_path)):\n with log.log_output(\"foo.txt\", filter_fn=_log_filter_fn):\n print(\"foo blah\")\n print(\"blah foo\")\n print(\"foo foo\")\n\n # foo.txt output is not filtered\n with open(\"foo.txt\", encoding=\"utf-8\") as f:\n assert f.read() == \"foo blah\\nblah foo\\nfoo foo\\n\"\n\n # output is not echoed\n assert capfd.readouterr()[0] == \"\"\n\n # now try with echo\n with working_dir(str(tmp_path)):\n with log.log_output(\"foo.txt\", echo=True, filter_fn=_log_filter_fn):\n print(\"foo blah\")\n print(\"blah foo\")\n print(\"foo foo\")\n\n # foo.txt output is still not filtered\n with open(\"foo.txt\", encoding=\"utf-8\") as f:\n assert f.read() == \"foo blah\\nblah foo\\nfoo foo\\n\"\n\n # echoed output is filtered.\n assert capfd.readouterr()[0] == \"bar blah\\nblah bar\\nbar bar\\n\"\n\n\ndef test_log_output_with_filter_and_append(capfd, tmp_path: pathlib.Path):\n with working_dir(str(tmp_path)):\n with log.log_output(\"foo.txt\", filter_fn=_log_filter_fn):\n print(\"foo blah\")\n print(\"blah foo\")\n print(\"foo foo\")\n\n with open(\"foo.txt\", encoding=\"utf-8\") as f:\n assert f.read() == \"foo blah\\nblah foo\\nfoo foo\\n\"\n\n with log.log_output(\"foo.txt\", filter_fn=_log_filter_fn, append=True):\n print(\"more foo more blah\")\n\n with open(\"foo.txt\", encoding=\"utf-8\") as f:\n assert f.read() == \"foo blah\\nblah foo\\nfoo foo\\nmore foo more blah\\n\"\n\n\ndef test_log_subproc_and_echo_output(capfd, tmp_path: pathlib.Path):\n python = Executable(sys.executable)\n\n with working_dir(str(tmp_path)):\n with log.log_output(\"foo.txt\") as logger:\n with logger.force_echo():\n python(\"-c\", \"print('echo')\")\n print(\"logged\")\n\n # Check log file content\n with open(\"foo.txt\", encoding=\"utf-8\") as f:\n assert f.read() == \"echo\\nlogged\\n\"\n\n # Check captured output (echoed content)\n # Note: 'logged' is not echoed because force_echo() scope ended\n assert capfd.readouterr()[0] == \"echo\\n\"\n" }, { "path": "lib/spack/spack/test/llnl/util/tty/tty.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\n\nimport pytest\n\nimport spack.llnl.util.tty as tty\n\n\ndef test_get_timestamp(monkeypatch):\n \"\"\"Ensure the results of get_timestamp are reasonable.\"\"\"\n\n # Debug disabled should return an empty string\n monkeypatch.setattr(tty, \"_debug\", 0)\n assert not tty.get_timestamp(False), \"Expected an empty string\"\n\n # Debug disabled but force the timestamp should return a string\n assert tty.get_timestamp(True), \"Expected a timestamp/non-empty string\"\n\n pid_str = \" {0}\".format(os.getpid())\n\n # Level 1 debugging should return a timestamp WITHOUT the pid\n monkeypatch.setattr(tty, \"_debug\", 1)\n out_str = tty.get_timestamp(False)\n assert out_str and pid_str not in out_str, \"Expected no PID in results\"\n\n # Level 2 debugging should also return a timestamp WITH the pid\n monkeypatch.setattr(tty, \"_debug\", 2)\n out_str = tty.get_timestamp(False)\n assert out_str and pid_str in out_str, \"Expected PID in results\"\n\n\n@pytest.mark.parametrize(\n \"msg,enabled,trace,newline\",\n [\n (\"\", False, False, False), # Nothing is output\n (Exception(\"\"), True, False, True), # Exception output\n (\"trace\", True, True, False), # stacktrace output\n (\"newline\", True, False, True), # newline in output\n (\"no newline\", True, False, False), # no newline output\n ],\n)\ndef test_msg(capfd, monkeypatch, enabled, msg, trace, newline):\n \"\"\"Ensure the output from msg with options is appropriate.\"\"\"\n\n # temporarily use the parameterized settings\n monkeypatch.setattr(tty, \"_msg_enabled\", enabled)\n monkeypatch.setattr(tty, \"_stacktrace\", trace)\n\n expected = [msg if isinstance(msg, str) else \"Exception: \"]\n if newline:\n expected[0] = \"{0}\\n\".format(expected[0])\n if trace:\n expected.insert(0, \".py\")\n\n tty.msg(msg, newline=newline)\n out = capfd.readouterr()[0]\n for msg in expected:\n assert msg in out\n\n\n@pytest.mark.parametrize(\n \"msg,trace,wrap\",\n [\n (Exception(\"\"), False, False), # Exception output\n (\"trace\", True, False), # stacktrace output\n (\"wrap\", False, True), # wrap in output\n ],\n)\ndef test_info(capfd, monkeypatch, msg, trace, wrap):\n \"\"\"Ensure the output from info with options is appropriate.\"\"\"\n\n # temporarily use the parameterized settings\n monkeypatch.setattr(tty, \"_stacktrace\", trace)\n\n expected = [msg if isinstance(msg, str) else \"Exception: \"]\n if trace:\n expected.insert(0, \".py\")\n\n extra = (\n \"This extra argument *should* make for a sufficiently long line\"\n \" that needs to be wrapped if the option is enabled.\"\n )\n args = [msg, extra]\n\n num_newlines = 3 if wrap else 2\n\n tty.info(*args, wrap=wrap, countback=3)\n out = capfd.readouterr()[0]\n for msg in expected:\n assert msg in out\n\n assert out.count(\"\\n\") == num_newlines\n" }, { "path": "lib/spack/spack/test/main.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\nimport os\nimport os.path\nimport pathlib\n\nimport pytest\n\nimport spack\nimport spack.config\nimport spack.environment as ev\nimport spack.error\nimport spack.llnl.util.filesystem as fs\nimport spack.main\nimport spack.paths\nimport spack.platforms\nimport spack.util.executable as exe\nimport spack.util.git\nimport spack.util.spack_yaml as syaml\n\npytestmark = pytest.mark.not_on_windows(\n \"Test functionality supported but tests are failing on Win\"\n)\n\n\n@pytest.fixture(autouse=True)\ndef _clear_commit_cache():\n spack.get_spack_commit.cache_clear()\n\n\ndef test_version_git_nonsense_output(tmp_path: pathlib.Path, working_env, monkeypatch):\n git = tmp_path / \"git\"\n with open(git, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"#!/bin/sh\necho --|not a hash|----\n\"\"\"\n )\n fs.set_executable(str(git))\n\n monkeypatch.setattr(spack.util.git, \"git\", lambda: exe.which(str(git)))\n assert spack.spack_version == spack.get_version()\n\n\ndef test_version_git_fails(tmp_path: pathlib.Path, working_env, monkeypatch):\n git = tmp_path / \"git\"\n with open(git, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"#!/bin/sh\necho 26552533be04e83e66be2c28e0eb5011cb54e8fa\nexit 1\n\"\"\"\n )\n fs.set_executable(str(git))\n\n monkeypatch.setattr(spack.util.git, \"git\", lambda: exe.which(str(git)))\n assert spack.spack_version == spack.get_version()\n\n\ndef test_git_sha_output(tmp_path: pathlib.Path, working_env, monkeypatch):\n git = tmp_path / \"git\"\n sha = \"26552533be04e83e66be2c28e0eb5011cb54e8fa\"\n with open(git, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"#!/bin/sh\necho {0}\n\"\"\".format(sha)\n )\n fs.set_executable(str(git))\n\n monkeypatch.setattr(spack.util.git, \"git\", lambda: exe.which(str(git)))\n expected = \"{0} ({1})\".format(spack.spack_version, sha)\n assert expected == spack.get_version()\n\n\ndef test_get_version_no_repo(tmp_path: pathlib.Path, monkeypatch):\n monkeypatch.setattr(spack.paths, \"prefix\", str(tmp_path))\n assert spack.spack_version == spack.get_version()\n\n\ndef test_get_version_no_git(working_env, monkeypatch):\n monkeypatch.setattr(spack.util.git, \"git\", lambda: None)\n assert spack.spack_version == spack.get_version()\n\n\ndef test_main_calls_get_version(capfd, working_env, monkeypatch):\n # act like git is not found in the PATH\n monkeypatch.setattr(spack.util.git, \"git\", lambda: None)\n\n # make sure we get a bare version (without commit) when this happens\n spack.main.main([\"-V\"])\n out, err = capfd.readouterr()\n assert spack.spack_version == out.strip()\n\n\ndef test_unrecognized_top_level_flag():\n assert spack.main.main([\"-o\", \"mirror\", \"list\"]) != 0\n\n\ndef test_get_version_bad_git(tmp_path: pathlib.Path, working_env, monkeypatch):\n bad_git = str(tmp_path / \"git\")\n with open(bad_git, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"#!/bin/sh\nexit 1\n\"\"\"\n )\n fs.set_executable(bad_git)\n\n monkeypatch.setattr(spack.util.git, \"git\", lambda: exe.which(bad_git))\n assert spack.spack_version == spack.get_version()\n\n\ndef test_bad_command_line_scopes(tmp_path: pathlib.Path, config):\n cfg = spack.config.Configuration()\n file_path = tmp_path / \"file_instead_of_dir\"\n non_existing_path = tmp_path / \"non_existing_dir\"\n\n file_path.write_text(\"\")\n\n with pytest.raises(spack.error.ConfigError):\n spack.main.add_command_line_scopes(cfg, [str(file_path)])\n\n with pytest.raises(spack.error.ConfigError):\n spack.main.add_command_line_scopes(cfg, [str(non_existing_path)])\n\n\ndef test_add_command_line_scopes(tmp_path: pathlib.Path, mutable_config):\n config_yaml = str(tmp_path / \"config.yaml\")\n with open(config_yaml, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nconfig:\n verify_ssl: False\n dirty: False\n\"\"\"\n )\n\n spack.main.add_command_line_scopes(mutable_config, [str(tmp_path)])\n assert mutable_config.get(\"config:verify_ssl\") is False\n assert mutable_config.get(\"config:dirty\") is False\n\n\ndef test_add_command_line_scope_env(tmp_path: pathlib.Path, mutable_mock_env_path):\n \"\"\"Test whether --config-scope works, either by name or path.\"\"\"\n managed_env = ev.create(\"example\").manifest_path\n\n with open(managed_env, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n config:\n install_tree:\n root: /tmp/first\n\"\"\"\n )\n\n with open(tmp_path / \"spack.yaml\", \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nspack:\n config:\n install_tree:\n root: /tmp/second\n\"\"\"\n )\n\n config = spack.config.Configuration()\n spack.main.add_command_line_scopes(config, [\"example\", str(tmp_path)])\n assert len(config.scopes) == 2\n assert config.get(\"config:install_tree:root\") == \"/tmp/second\"\n\n config = spack.config.Configuration()\n spack.main.add_command_line_scopes(config, [str(tmp_path), \"example\"])\n assert len(config.scopes) == 2\n assert config.get(\"config:install_tree:root\") == \"/tmp/first\"\n\n assert ev.active_environment() is None # shouldn't cause an environment to be activated\n\n\ndef test_include_cfg(mock_low_high_config, write_config_file, tmp_path: pathlib.Path):\n cfg1_path = str(tmp_path / \"include1.yaml\")\n with open(cfg1_path, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nconfig:\n verify_ssl: False\n dirty: True\npackages:\n python:\n require:\n - spec: \"@3.11:\"\n\"\"\"\n )\n\n def python_cfg(_spec):\n return f\"\"\"\\\npackages:\n python:\n require:\n - spec: {_spec}\n\"\"\"\n\n def write_python_cfg(_spec, _cfg_name):\n cfg_path = str(tmp_path / _cfg_name)\n with open(cfg_path, \"w\", encoding=\"utf-8\") as f:\n f.write(python_cfg(_spec))\n return cfg_path\n\n # This config will not be included\n cfg2_path = write_python_cfg(\"+shared\", \"include2.yaml\")\n\n # The config will point to this using substitutable variables,\n # namely $os; we expect that Spack resolves these variables\n # into the actual path of the config\n this_os = spack.platforms.host().default_os\n cfg3_expanded_path = os.path.join(str(tmp_path), f\"{this_os}\", \"include3.yaml\")\n fs.mkdirp(os.path.dirname(cfg3_expanded_path))\n with open(cfg3_expanded_path, \"w\", encoding=\"utf-8\") as f:\n f.write(python_cfg(\"+ssl\"))\n cfg3_abstract_path = os.path.join(str(tmp_path), \"$os\", \"include3.yaml\")\n\n # This will be included unconditionally\n cfg4_path = write_python_cfg(\"+tk\", \"include4.yaml\")\n\n # This config will not exist, and the config will explicitly\n # allow this\n cfg5_path = os.path.join(str(tmp_path), \"non-existent.yaml\")\n\n include_entries = [\n {\"path\": f\"{cfg1_path}\", \"when\": f'os == \"{this_os}\"'},\n {\"path\": f\"{cfg2_path}\", \"when\": \"False\"},\n {\"path\": cfg3_abstract_path},\n cfg4_path,\n {\"path\": cfg5_path, \"optional\": True},\n ]\n include_cfg = {\"include\": include_entries}\n filename = write_config_file(\"include\", include_cfg, \"low\")\n\n assert not spack.config.get(\"config:dirty\")\n\n spack.main.add_command_line_scopes(mock_low_high_config, [os.path.dirname(filename)])\n\n assert spack.config.get(\"config:dirty\")\n python_reqs = spack.config.get(\"packages\")[\"python\"][\"require\"]\n req_specs = set(x[\"spec\"] for x in python_reqs)\n assert req_specs == set([\"@3.11:\", \"+ssl\", \"+tk\"])\n\n\ndef test_include_duplicate_source(mutable_config):\n \"\"\"Check precedence when include.yaml files have the same path.\"\"\"\n include_yaml = \"debug.yaml\"\n include_list = {\"include\": [f\"./{include_yaml}\"]}\n\n system_filename = mutable_config.get_config_filename(\"system\", \"include\")\n site_filename = mutable_config.get_config_filename(\"site\", \"include\")\n\n def write_configs(include_path, debug_data):\n fs.mkdirp(os.path.dirname(include_path))\n with open(include_path, \"w\", encoding=\"utf-8\") as f:\n syaml.dump_config(include_list, f)\n\n debug_path = fs.join_path(os.path.dirname(include_path), include_yaml)\n with open(debug_path, \"w\", encoding=\"utf-8\") as f:\n syaml.dump_config(debug_data, f)\n\n system_config = {\"config\": {\"debug\": False}}\n write_configs(system_filename, system_config)\n spack.main.add_command_line_scopes(mutable_config, [os.path.dirname(system_filename)])\n\n site_config = {\"config\": {\"debug\": True}}\n write_configs(site_filename, site_config)\n spack.main.add_command_line_scopes(mutable_config, [os.path.dirname(site_filename)])\n\n # Ensure takes the last value of the option pushed onto the stack\n assert mutable_config.get(\"config:debug\") == site_config[\"config\"][\"debug\"]\n\n\ndef test_include_recurse_limit(tmp_path: pathlib.Path, mutable_config):\n \"\"\"Ensure hit the recursion limit.\"\"\"\n include_yaml = \"include.yaml\"\n include_list = {\"include\": [f\"./{include_yaml}\"]}\n\n include_path = str(tmp_path / include_yaml)\n with open(include_path, \"w\", encoding=\"utf-8\") as f:\n syaml.dump_config(include_list, f)\n\n with pytest.raises(spack.config.RecursiveIncludeError, match=\"recursion exceeded\"):\n spack.main.add_command_line_scopes(mutable_config, [os.path.dirname(include_path)])\n\n\n# TODO: Fix this once recursive includes are processed in the expected order.\n@pytest.mark.parametrize(\"child,expected\", [(\"b\", True), (\"c\", False)])\ndef test_include_recurse_diamond(tmp_path: pathlib.Path, mutable_config, child, expected):\n \"\"\"Demonstrate include parent's value overrides that of child in diamond include.\n\n Check that the value set by b or c overrides that set by d.\n \"\"\"\n configs_root = tmp_path / \"configs\"\n configs_root.mkdir()\n\n def write(path, contents):\n with open(path, \"w\", encoding=\"utf-8\") as f:\n f.write(contents)\n\n def debug_contents(value):\n return f\"config:\\n debug: {value}\\n\"\n\n def include_contents(paths):\n indent = \"\\n - \"\n values = indent.join([str(p) for p in paths])\n return f\"include:{indent}{values}\"\n\n a_yaml = tmp_path / \"a.yaml\"\n b_yaml = configs_root / \"b.yaml\"\n c_yaml = configs_root / \"c.yaml\"\n d_yaml = configs_root / \"d.yaml\"\n debug_yaml = configs_root / \"enable_debug.yaml\"\n\n write(debug_yaml, debug_contents(\"true\"))\n\n a_contents = f\"\"\"\\\ninclude:\n- {b_yaml}\n- {c_yaml}\n\"\"\"\n write(a_yaml, a_contents)\n write(d_yaml, debug_contents(\"false\"))\n\n write(b_yaml, include_contents([debug_yaml, d_yaml] if child == \"b\" else [d_yaml]))\n write(c_yaml, include_contents([debug_yaml, d_yaml] if child == \"c\" else [d_yaml]))\n\n spack.main.add_command_line_scopes(mutable_config, [str(tmp_path)])\n\n try:\n assert mutable_config.get(\"config:debug\") is expected\n except AssertionError:\n pytest.xfail(\"recursive includes are not processed in the expected order\")\n" }, { "path": "lib/spack/spack/test/make_executable.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"\nTests for Spack's built-in parallel make support.\n\nThis just tests whether the right args are getting passed to make.\n\"\"\"\n\nimport os\nimport pathlib\n\nimport pytest\n\nfrom spack.build_environment import MakeExecutable\nfrom spack.util.environment import path_put_first\n\npytestmark = pytest.mark.not_on_windows(\"MakeExecutable not supported on Windows\")\n\n\n@pytest.fixture(autouse=True)\ndef make_executable(tmp_path: pathlib.Path, working_env):\n make_exe = tmp_path / \"make\"\n with open(make_exe, \"w\", encoding=\"utf-8\") as f:\n f.write(\"#!/bin/sh\\n\")\n f.write('echo \"$@\"')\n os.chmod(make_exe, 0o700)\n\n path_put_first(\"PATH\", [str(tmp_path)])\n\n\ndef test_make_normal():\n make = MakeExecutable(\"make\", jobs=8)\n assert make(output=str).strip() == \"-j8\"\n assert make(\"install\", output=str).strip() == \"-j8 install\"\n\n\ndef test_make_explicit():\n make = MakeExecutable(\"make\", jobs=8)\n assert make(parallel=True, output=str).strip() == \"-j8\"\n assert make(\"install\", parallel=True, output=str).strip() == \"-j8 install\"\n\n\ndef test_make_one_job():\n make = MakeExecutable(\"make\", jobs=1)\n assert make(output=str).strip() == \"-j1\"\n assert make(\"install\", output=str).strip() == \"-j1 install\"\n\n\ndef test_make_parallel_false():\n make = MakeExecutable(\"make\", jobs=8)\n assert make(parallel=False, output=str).strip() == \"-j1\"\n assert make(\"install\", parallel=False, output=str).strip() == \"-j1 install\"\n\n\ndef test_make_parallel_disabled(monkeypatch):\n make = MakeExecutable(\"make\", jobs=8)\n\n monkeypatch.setenv(\"SPACK_NO_PARALLEL_MAKE\", \"true\")\n assert make(output=str).strip() == \"-j1\"\n assert make(\"install\", output=str).strip() == \"-j1 install\"\n\n monkeypatch.setenv(\"SPACK_NO_PARALLEL_MAKE\", \"1\")\n assert make(output=str).strip() == \"-j1\"\n assert make(\"install\", output=str).strip() == \"-j1 install\"\n\n # These don't disable (false and random string)\n monkeypatch.setenv(\"SPACK_NO_PARALLEL_MAKE\", \"false\")\n assert make(output=str).strip() == \"-j8\"\n assert make(\"install\", output=str).strip() == \"-j8 install\"\n\n monkeypatch.setenv(\"SPACK_NO_PARALLEL_MAKE\", \"foobar\")\n assert make(output=str).strip() == \"-j8\"\n assert make(\"install\", output=str).strip() == \"-j8 install\"\n\n\ndef test_make_parallel_precedence(monkeypatch):\n make = MakeExecutable(\"make\", jobs=8)\n\n # These should work\n monkeypatch.setenv(\"SPACK_NO_PARALLEL_MAKE\", \"true\")\n assert make(parallel=True, output=str).strip() == \"-j1\"\n assert make(\"install\", parallel=True, output=str).strip() == \"-j1 install\"\n\n monkeypatch.setenv(\"SPACK_NO_PARALLEL_MAKE\", \"1\")\n assert make(parallel=True, output=str).strip() == \"-j1\"\n assert make(\"install\", parallel=True, output=str).strip() == \"-j1 install\"\n\n # These don't disable (false and random string)\n monkeypatch.setenv(\"SPACK_NO_PARALLEL_MAKE\", \"false\")\n assert make(parallel=True, output=str).strip() == \"-j8\"\n assert make(\"install\", parallel=True, output=str).strip() == \"-j8 install\"\n\n monkeypatch.setenv(\"SPACK_NO_PARALLEL_MAKE\", \"foobar\")\n assert make(parallel=True, output=str).strip() == \"-j8\"\n assert make(\"install\", parallel=True, output=str).strip() == \"-j8 install\"\n\n\ndef test_make_jobs_env():\n make = MakeExecutable(\"make\", jobs=8)\n dump_env = {}\n assert make(output=str, jobs_env=\"MAKE_PARALLELISM\", _dump_env=dump_env).strip() == \"-j8\"\n assert dump_env[\"MAKE_PARALLELISM\"] == \"8\"\n\n\ndef test_make_jobserver(monkeypatch):\n make = MakeExecutable(\"make\", jobs=8)\n monkeypatch.setenv(\"MAKEFLAGS\", \"--jobserver-auth=X,Y\")\n assert make(output=str).strip() == \"\"\n assert make(parallel=False, output=str).strip() == \"-j1\"\n\n\ndef test_make_jobserver_not_supported(monkeypatch):\n make = MakeExecutable(\"make\", jobs=8, supports_jobserver=False)\n monkeypatch.setenv(\"MAKEFLAGS\", \"--jobserver-auth=X,Y\")\n # Currently fallback on default job count, Maybe it should force -j1 ?\n assert make(output=str).strip() == \"-j8\"\n" }, { "path": "lib/spack/spack/test/mirror.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport filecmp\nimport os\nimport pathlib\n\nimport pytest\n\nimport spack.caches\nimport spack.cmd.mirror\nimport spack.concretize\nimport spack.config\nimport spack.fetch_strategy\nimport spack.mirrors.layout\nimport spack.mirrors.mirror\nimport spack.mirrors.utils\nimport spack.patch\nimport spack.stage\nimport spack.util.spack_json as sjson\nimport spack.util.url as url_util\nfrom spack.cmd.common.arguments import mirror_name_or_url\nfrom spack.llnl.util.filesystem import resolve_link_target_relative_to_the_link, working_dir\nfrom spack.spec import Spec\nfrom spack.util.executable import which\nfrom spack.util.spack_yaml import SpackYAMLError\n\npytestmark = [pytest.mark.usefixtures(\"mutable_config\", \"mutable_mock_repo\")]\n\n# paths in repos that shouldn't be in the mirror tarballs.\nexclude = [\".hg\", \".git\", \".svn\"]\n\n\nrepos = {}\n\n\ndef set_up_package(name, repository, url_attr):\n \"\"\"Set up a mock package to be mirrored.\n Each package needs us to:\n\n 1. Set up a mock repo/archive to fetch from.\n 2. Point the package's version args at that repo.\n \"\"\"\n # Set up packages to point at mock repos.\n s = spack.concretize.concretize_one(name)\n repos[name] = repository\n\n # change the fetch args of the first (only) version.\n assert len(s.package.versions) == 1\n\n v = next(iter(s.package.versions))\n s.package.versions[v][url_attr] = repository.url\n\n\ndef check_mirror():\n with spack.stage.Stage(\"spack-mirror-test\") as stage:\n mirror_root = os.path.join(stage.path, \"test-mirror\")\n # register mirror with spack config\n mirrors = {\"spack-mirror-test\": url_util.path_to_file_url(mirror_root)}\n with spack.config.override(\"mirrors\", mirrors):\n with spack.config.override(\"config:checksum\", False):\n specs = [spack.concretize.concretize_one(x) for x in repos]\n spack.cmd.mirror.create(mirror_root, specs)\n\n # Stage directory exists\n assert os.path.isdir(mirror_root)\n\n for spec in specs:\n fetcher = spec.package.fetcher\n per_package_ref = os.path.join(spec.name, \"-\".join([spec.name, str(spec.version)]))\n mirror_layout = spack.mirrors.layout.default_mirror_layout(\n fetcher, per_package_ref\n )\n expected_path = os.path.join(mirror_root, mirror_layout.path)\n assert os.path.exists(expected_path)\n\n # Now try to fetch each package.\n for name, mock_repo in repos.items():\n spec = spack.concretize.concretize_one(name)\n pkg = spec.package\n\n with spack.config.override(\"config:checksum\", False):\n with pkg.stage:\n pkg.do_stage(mirror_only=True)\n\n # Compare the original repo with the expanded archive\n original_path = mock_repo.path\n if \"svn\" in name:\n # have to check out the svn repo to compare.\n original_path = os.path.join(mock_repo.path, \"checked_out\")\n\n svn = which(\"svn\", required=True)\n svn(\"checkout\", mock_repo.url, original_path)\n\n dcmp = filecmp.dircmp(original_path, pkg.stage.source_path)\n\n # make sure there are no new files in the expanded\n # tarball\n assert not dcmp.right_only\n # and that all original files are present.\n assert all(left in exclude for left in dcmp.left_only)\n\n\ndef test_url_mirror(mock_archive):\n set_up_package(\"trivial-install-test-package\", mock_archive, \"url\")\n check_mirror()\n repos.clear()\n\n\ndef test_git_mirror(git, mock_git_repository):\n set_up_package(\"git-test\", mock_git_repository, \"git\")\n check_mirror()\n repos.clear()\n\n\ndef test_svn_mirror(mock_svn_repository):\n set_up_package(\"svn-test\", mock_svn_repository, \"svn\")\n check_mirror()\n repos.clear()\n\n\ndef test_hg_mirror(mock_hg_repository):\n set_up_package(\"hg-test\", mock_hg_repository, \"hg\")\n check_mirror()\n repos.clear()\n\n\ndef test_all_mirror(mock_git_repository, mock_svn_repository, mock_hg_repository, mock_archive):\n set_up_package(\"git-test\", mock_git_repository, \"git\")\n set_up_package(\"svn-test\", mock_svn_repository, \"svn\")\n set_up_package(\"hg-test\", mock_hg_repository, \"hg\")\n set_up_package(\"trivial-install-test-package\", mock_archive, \"url\")\n check_mirror()\n repos.clear()\n\n\n@pytest.mark.parametrize(\n \"mirror\",\n [\n spack.mirrors.mirror.Mirror(\n {\"fetch\": \"https://example.com/fetch\", \"push\": \"https://example.com/push\"}\n )\n ],\n)\ndef test_roundtrip_mirror(mirror: spack.mirrors.mirror.Mirror):\n mirror_yaml = mirror.to_yaml()\n assert spack.mirrors.mirror.Mirror.from_yaml(mirror_yaml) == mirror\n mirror_json = mirror.to_json()\n assert spack.mirrors.mirror.Mirror.from_json(mirror_json) == mirror\n\n\n@pytest.mark.parametrize(\n \"invalid_yaml\", [\"playing_playlist: {{ action }} playlist {{ playlist_name }}\"]\n)\ndef test_invalid_yaml_mirror(invalid_yaml):\n with pytest.raises(SpackYAMLError, match=\"error parsing YAML\") as e:\n spack.mirrors.mirror.Mirror.from_yaml(invalid_yaml)\n assert invalid_yaml in str(e.value)\n\n\n@pytest.mark.parametrize(\"invalid_json, error_message\", [(\"{13:\", \"Expecting property name\")])\ndef test_invalid_json_mirror(invalid_json, error_message):\n with pytest.raises(sjson.SpackJSONError) as e:\n spack.mirrors.mirror.Mirror.from_json(invalid_json)\n exc_msg = str(e.value)\n assert exc_msg.startswith(\"error parsing JSON mirror:\")\n assert error_message in exc_msg\n\n\n@pytest.mark.parametrize(\n \"mirror_collection\",\n [\n spack.mirrors.mirror.MirrorCollection(\n mirrors={\n \"example-mirror\": spack.mirrors.mirror.Mirror(\n \"https://example.com/fetch\", \"https://example.com/push\"\n ).to_dict()\n }\n )\n ],\n)\ndef test_roundtrip_mirror_collection(mirror_collection):\n mirror_collection_yaml = mirror_collection.to_yaml()\n assert (\n spack.mirrors.mirror.MirrorCollection.from_yaml(mirror_collection_yaml)\n == mirror_collection\n )\n mirror_collection_json = mirror_collection.to_json()\n assert (\n spack.mirrors.mirror.MirrorCollection.from_json(mirror_collection_json)\n == mirror_collection\n )\n\n\n@pytest.mark.parametrize(\n \"invalid_yaml\", [\"playing_playlist: {{ action }} playlist {{ playlist_name }}\"]\n)\ndef test_invalid_yaml_mirror_collection(invalid_yaml):\n with pytest.raises(SpackYAMLError, match=\"error parsing YAML\") as e:\n spack.mirrors.mirror.MirrorCollection.from_yaml(invalid_yaml)\n assert invalid_yaml in str(e.value)\n\n\n@pytest.mark.parametrize(\"invalid_json, error_message\", [(\"{13:\", \"Expecting property name\")])\ndef test_invalid_json_mirror_collection(invalid_json, error_message):\n with pytest.raises(sjson.SpackJSONError) as e:\n spack.mirrors.mirror.MirrorCollection.from_json(invalid_json)\n exc_msg = str(e.value)\n assert exc_msg.startswith(\"error parsing JSON mirror collection:\")\n assert error_message in exc_msg\n\n\ndef test_mirror_archive_paths_no_version(mock_packages, mock_archive):\n spec = spack.concretize.concretize_one(\n Spec(\"trivial-install-test-package@=nonexistingversion\")\n )\n fetcher = spack.fetch_strategy.URLFetchStrategy(url=mock_archive.url)\n spack.mirrors.layout.default_mirror_layout(fetcher, \"per-package-ref\", spec)\n\n\ndef test_mirror_with_url_patches(mock_packages, monkeypatch):\n spec = spack.concretize.concretize_one(\"patch-several-dependencies\")\n files_cached_in_mirror = set()\n\n def record_store(_class, fetcher, relative_dst, cosmetic_path=None):\n files_cached_in_mirror.add(os.path.basename(relative_dst))\n\n def successful_fetch(_class):\n with open(_class.stage.save_filename, \"w\", encoding=\"utf-8\"):\n pass\n\n def successful_expand(_class):\n expanded_path = os.path.join(_class.stage.path, spack.stage._source_path_subdir)\n os.mkdir(expanded_path)\n with open(os.path.join(expanded_path, \"test.patch\"), \"w\", encoding=\"utf-8\"):\n pass\n\n def successful_apply(*args, **kwargs):\n pass\n\n def successful_make_alias(*args, **kwargs):\n pass\n\n with spack.stage.Stage(\"spack-mirror-test\") as stage:\n mirror_root = os.path.join(stage.path, \"test-mirror\")\n\n monkeypatch.setattr(spack.fetch_strategy.URLFetchStrategy, \"fetch\", successful_fetch)\n monkeypatch.setattr(spack.fetch_strategy.URLFetchStrategy, \"expand\", successful_expand)\n monkeypatch.setattr(spack.patch, \"apply_patch\", successful_apply)\n monkeypatch.setattr(spack.caches.MirrorCache, \"store\", record_store)\n monkeypatch.setattr(\n spack.mirrors.layout.DefaultLayout, \"make_alias\", successful_make_alias\n )\n\n with spack.config.override(\"config:checksum\", False):\n spack.cmd.mirror.create(mirror_root, list(spec.traverse()))\n\n assert {\n \"abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234\",\n \"abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd.gz\",\n }.issubset(files_cached_in_mirror)\n\n\nclass MockFetcher:\n \"\"\"Mock fetcher object which implements the necessary functionality for\n testing MirrorCache\n \"\"\"\n\n @staticmethod\n def archive(dst):\n with open(dst, \"w\", encoding=\"utf-8\"):\n pass\n\n\n@pytest.mark.regression(\"14067\")\ndef test_mirror_layout_make_alias(tmp_path: pathlib.Path):\n \"\"\"Confirm that the cosmetic symlink created in the mirror cache (which may\n be relative) targets the storage path correctly.\n \"\"\"\n alias = os.path.join(\"zlib\", \"zlib-1.2.11.tar.gz\")\n path = os.path.join(\"_source-cache\", \"archive\", \"c3\", \"c3e5.tar.gz\")\n cache = spack.caches.MirrorCache(root=str(tmp_path), skip_unstable_versions=False)\n layout = spack.mirrors.layout.DefaultLayout(alias, path)\n\n cache.store(MockFetcher(), layout.path)\n layout.make_alias(cache.root)\n\n link_target = resolve_link_target_relative_to_the_link(os.path.join(cache.root, layout.alias))\n assert os.path.exists(link_target)\n assert os.path.normpath(link_target) == os.path.join(cache.root, layout.path)\n\n\n@pytest.mark.regression(\"31627\")\n@pytest.mark.parametrize(\n \"specs,expected_specs\",\n [\n ([\"pkg-a\"], [\"pkg-a@=1.0\", \"pkg-a@=2.0\"]),\n ([\"pkg-a\", \"brillig\"], [\"pkg-a@=1.0\", \"pkg-a@=2.0\", \"brillig@=1.0.0\", \"brillig@=2.0.0\"]),\n ],\n)\ndef test_get_all_versions(specs, expected_specs):\n specs = [Spec(s) for s in specs]\n output_list = spack.mirrors.utils.get_all_versions(specs)\n output_list = [str(x) for x in output_list]\n # Compare sets since order is not important\n assert set(output_list) == set(expected_specs)\n\n\ndef test_update_1():\n # No change\n m = spack.mirrors.mirror.Mirror(\"https://example.com\")\n assert not m.update({\"url\": \"https://example.com\"})\n assert m.to_dict() == \"https://example.com\"\n\n\ndef test_update_2():\n # Change URL, shouldn't expand to {\"url\": ...} dict.\n m = spack.mirrors.mirror.Mirror(\"https://example.com\")\n assert m.update({\"url\": \"https://example.org\"})\n assert m.to_dict() == \"https://example.org\"\n assert m.fetch_url == \"https://example.org\"\n assert m.push_url == \"https://example.org\"\n\n\ndef test_update_3():\n # Change fetch url, ensure minimal config\n m = spack.mirrors.mirror.Mirror(\"https://example.com\")\n assert m.update({\"url\": \"https://example.org\"}, \"fetch\")\n assert m.to_dict() == {\"url\": \"https://example.com\", \"fetch\": \"https://example.org\"}\n assert m.fetch_url == \"https://example.org\"\n assert m.push_url == \"https://example.com\"\n\n\ndef test_update_4():\n # Change push url, ensure minimal config\n m = spack.mirrors.mirror.Mirror(\"https://example.com\")\n assert m.update({\"url\": \"https://example.org\"}, \"push\")\n assert m.to_dict() == {\"url\": \"https://example.com\", \"push\": \"https://example.org\"}\n assert m.push_url == \"https://example.org\"\n assert m.fetch_url == \"https://example.com\"\n\n\n@pytest.mark.parametrize(\"direction\", [\"fetch\", \"push\"])\ndef test_update_connection_params(direction, monkeypatch):\n \"\"\"Test whether new connection params expand the mirror config to a dict.\"\"\"\n m = spack.mirrors.mirror.Mirror(\"https://example.com\", \"example\")\n\n assert m.update(\n {\n \"url\": \"http://example.org\",\n \"access_pair\": [\"username\", \"password\"],\n \"access_token\": \"token\",\n \"profile\": \"profile\",\n \"endpoint_url\": \"https://example.com\",\n },\n direction,\n )\n\n assert m.to_dict() == {\n \"url\": \"https://example.com\",\n direction: {\n \"url\": \"http://example.org\",\n \"access_pair\": [\"username\", \"password\"],\n \"access_token\": \"token\",\n \"profile\": \"profile\",\n \"endpoint_url\": \"https://example.com\",\n },\n }\n assert m.get_access_pair(direction) == (\"username\", \"password\")\n assert m.get_access_token(direction) == \"token\"\n assert m.get_profile(direction) == \"profile\"\n assert m.get_endpoint_url(direction) == \"https://example.com\"\n\n # Expand environment variables\n os.environ[\"_SPACK_TEST_PAIR_USERNAME\"] = \"expanded_username\"\n os.environ[\"_SPACK_TEST_PAIR_PASSWORD\"] = \"expanded_password\"\n os.environ[\"_SPACK_TEST_TOKEN\"] = \"expanded_token\"\n\n assert m.update(\n {\n \"access_pair\": {\n \"id_variable\": \"_SPACK_TEST_PAIR_USERNAME\",\n \"secret_variable\": \"_SPACK_TEST_PAIR_PASSWORD\",\n }\n },\n direction,\n )\n\n assert m.to_dict() == {\n \"url\": \"https://example.com\",\n direction: {\n \"url\": \"http://example.org\",\n \"access_pair\": {\n \"id_variable\": \"_SPACK_TEST_PAIR_USERNAME\",\n \"secret_variable\": \"_SPACK_TEST_PAIR_PASSWORD\",\n },\n \"access_token\": \"token\",\n \"profile\": \"profile\",\n \"endpoint_url\": \"https://example.com\",\n },\n }\n\n assert m.get_access_pair(direction) == (\"expanded_username\", \"expanded_password\")\n\n assert m.update(\n {\n \"access_pair\": {\"id\": \"username\", \"secret_variable\": \"_SPACK_TEST_PAIR_PASSWORD\"},\n \"access_token_variable\": \"_SPACK_TEST_TOKEN\",\n },\n direction,\n )\n\n assert m.to_dict() == {\n \"url\": \"https://example.com\",\n direction: {\n \"url\": \"http://example.org\",\n \"access_pair\": {\"id\": \"username\", \"secret_variable\": \"_SPACK_TEST_PAIR_PASSWORD\"},\n \"access_token_variable\": \"_SPACK_TEST_TOKEN\",\n \"profile\": \"profile\",\n \"endpoint_url\": \"https://example.com\",\n },\n }\n\n assert m.get_access_pair(direction) == (\"username\", \"expanded_password\")\n assert m.get_access_token(direction) == \"expanded_token\"\n\n\ndef test_mirror_name_or_url_dir_parsing(tmp_path: pathlib.Path):\n curdir = tmp_path / \"mirror\"\n curdir.mkdir()\n\n with working_dir(curdir):\n assert mirror_name_or_url(\".\").fetch_url == curdir.as_uri()\n assert mirror_name_or_url(\"..\").fetch_url == tmp_path.as_uri()\n" }, { "path": "lib/spack/spack/test/module_parsing.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport pathlib\n\nimport pytest\n\nimport spack.util.module_cmd\nfrom spack.util.module_cmd import (\n get_path_args_from_module_line,\n get_path_from_module_contents,\n module,\n path_from_modules,\n)\n\npytestmark = pytest.mark.not_on_windows(\"Tests fail on Windows\")\n\ntest_module_lines = [\n \"prepend-path LD_LIBRARY_PATH /path/to/lib\",\n \"setenv MOD_DIR /path/to\",\n \"setenv LDFLAGS -Wl,-rpath/path/to/lib\",\n \"setenv LDFLAGS -L/path/to/lib\",\n \"prepend-path PATH /path/to/bin\",\n]\n\n\ndef test_module_function_change_env(tmp_path: pathlib.Path):\n environb = {b\"TEST_MODULE_ENV_VAR\": b\"TEST_FAIL\", b\"NOT_AFFECTED\": b\"NOT_AFFECTED\"}\n src_file = tmp_path / \"src_me\"\n src_file.write_text(\"export TEST_MODULE_ENV_VAR=TEST_SUCCESS\\n\")\n module(\"load\", str(src_file), module_template=f\". {src_file} 2>&1\", environb=environb)\n assert environb[b\"TEST_MODULE_ENV_VAR\"] == b\"TEST_SUCCESS\"\n assert environb[b\"NOT_AFFECTED\"] == b\"NOT_AFFECTED\"\n\n\ndef test_module_function_change_env_with_module_src_cmd(tmp_path: pathlib.Path):\n environb = {\n b\"MODULESHOME\": b\"here\",\n b\"TEST_MODULE_ENV_VAR\": b\"TEST_FAIL\",\n b\"TEST_ANOTHER_MODULE_ENV_VAR\": b\"TEST_FAIL\",\n b\"NOT_AFFECTED\": b\"NOT_AFFECTED\",\n }\n src_file = tmp_path / \"src_me\"\n src_file.write_text(\"export TEST_MODULE_ENV_VAR=TEST_SUCCESS\\n\")\n module_src_file = tmp_path / \"src_me_too\"\n module_src_file.write_text(\"export TEST_ANOTHER_MODULE_ENV_VAR=TEST_SUCCESS\\n\")\n module(\"load\", str(src_file), module_template=f\". {src_file} 2>&1\", environb=environb)\n module(\n \"load\",\n str(src_file),\n module_template=f\". {src_file} 2>&1\",\n module_src_cmd=f\". {module_src_file} 2>&1; \",\n environb=environb,\n )\n assert environb[b\"TEST_MODULE_ENV_VAR\"] == b\"TEST_SUCCESS\"\n assert environb[b\"TEST_ANOTHER_MODULE_ENV_VAR\"] == b\"TEST_SUCCESS\"\n assert environb[b\"NOT_AFFECTED\"] == b\"NOT_AFFECTED\"\n\n\ndef test_module_function_change_env_without_moduleshome_no_module_src_cmd(tmp_path: pathlib.Path):\n environb = {\n b\"TEST_MODULE_ENV_VAR\": b\"TEST_FAIL\",\n b\"TEST_ANOTHER_MODULE_ENV_VAR\": b\"TEST_FAIL\",\n b\"NOT_AFFECTED\": b\"NOT_AFFECTED\",\n }\n src_file = tmp_path / \"src_me\"\n src_file.write_text(\"export TEST_MODULE_ENV_VAR=TEST_SUCCESS\\n\")\n module_src_file = tmp_path / \"src_me_too\"\n module_src_file.write_text(\"export TEST_ANOTHER_MODULE_ENV_VAR=TEST_SUCCESS\\n\")\n module(\"load\", str(src_file), module_template=f\". {src_file} 2>&1\", environb=environb)\n module(\n \"load\",\n str(src_file),\n module_template=f\". {src_file} 2>&1\",\n module_src_cmd=f\". {module_src_file} 2>&1; \",\n environb=environb,\n )\n assert environb[b\"TEST_MODULE_ENV_VAR\"] == b\"TEST_SUCCESS\"\n assert environb[b\"TEST_ANOTHER_MODULE_ENV_VAR\"] == b\"TEST_FAIL\"\n assert environb[b\"NOT_AFFECTED\"] == b\"NOT_AFFECTED\"\n\n\ndef test_module_function_no_change(tmp_path: pathlib.Path):\n src_file = str(tmp_path / \"src_me\")\n with open(src_file, \"w\", encoding=\"utf-8\") as f:\n f.write(\"echo TEST_MODULE_FUNCTION_PRINT\")\n\n old_env = os.environ.copy()\n text = module(\"show\", src_file, module_template=\". {0} 2>&1\".format(src_file))\n\n assert text == \"TEST_MODULE_FUNCTION_PRINT\\n\"\n assert os.environ == old_env\n\n\ndef test_get_path_from_module_faked(monkeypatch):\n for line in test_module_lines:\n\n def fake_module(*args):\n return line\n\n monkeypatch.setattr(spack.util.module_cmd, \"module\", fake_module)\n\n path = path_from_modules([\"mod\"])\n assert path == \"/path/to\"\n\n\ndef test_get_path_from_module_contents():\n # A line with \"MODULEPATH\" appears early on, and the test confirms that it\n # is not extracted as the package's path\n module_show_output = \"\"\"\nos.environ[\"MODULEPATH\"] = \"/path/to/modules1:/path/to/modules2\";\n----------------------------------------------------------------------------\n /root/cmake/3.9.2.lua:\n----------------------------------------------------------------------------\nhelp([[CMake Version 3.9.2\n]])\nwhatis(\"Name: CMake\")\nwhatis(\"Version: 3.9.2\")\nwhatis(\"Category: Tools\")\nwhatis(\"URL: https://cmake.org/\")\nprepend_path(\"LD_LIBRARY_PATH\",\"/bad/path\")\nprepend_path(\"PATH\",\"/path/to/cmake-3.9.2/bin:/other/bad/path\")\nprepend_path(\"MANPATH\",\"/path/to/cmake/cmake-3.9.2/share/man\")\nprepend_path(\"LD_LIBRARY_PATH\",\"/path/to/cmake-3.9.2/lib64\")\n\"\"\"\n module_show_lines = module_show_output.split(\"\\n\")\n\n # PATH and LD_LIBRARY_PATH outvote MANPATH and the other PATH and\n # LD_LIBRARY_PATH entries\n assert (\n get_path_from_module_contents(module_show_lines, \"cmake-3.9.2\") == \"/path/to/cmake-3.9.2\"\n )\n\n\ndef test_get_path_from_empty_module():\n assert get_path_from_module_contents(\"\", \"test\") is None\n\n\ndef test_pkg_dir_from_module_name():\n module_show_lines = [\"setenv FOO_BAR_DIR /path/to/foo-bar\"]\n\n assert get_path_from_module_contents(module_show_lines, \"foo-bar\") == \"/path/to/foo-bar\"\n\n assert get_path_from_module_contents(module_show_lines, \"foo-bar/1.0\") == \"/path/to/foo-bar\"\n\n\ndef test_get_argument_from_module_line():\n simple_lines = [\n \"prepend-path LD_LIBRARY_PATH /lib/path\",\n \"prepend-path LD_LIBRARY_PATH /lib/path\",\n \"prepend_path('PATH' , '/lib/path')\",\n 'prepend_path( \"PATH\" , \"/lib/path\" )',\n 'prepend_path(\"PATH\",' + \"'/lib/path')\",\n ]\n\n complex_lines = [\n \"prepend-path LD_LIBRARY_PATH /lib/path:/pkg/path\",\n \"prepend-path LD_LIBRARY_PATH /lib/path:/pkg/path\",\n \"prepend_path('PATH' , '/lib/path:/pkg/path')\",\n 'prepend_path( \"PATH\" , \"/lib/path:/pkg/path\" )',\n 'prepend_path(\"PATH\",' + \"'/lib/path:/pkg/path')\",\n ]\n\n bad_lines = [\"prepend_path(PATH,/lib/path)\", \"prepend-path (LD_LIBRARY_PATH) /lib/path\"]\n\n assert all(get_path_args_from_module_line(x) == [\"/lib/path\"] for x in simple_lines)\n assert all(\n get_path_args_from_module_line(x) == [\"/lib/path\", \"/pkg/path\"] for x in complex_lines\n )\n for bl in bad_lines:\n with pytest.raises(ValueError):\n get_path_args_from_module_line(bl)\n\n\n# lmod is entirely unsupported on Windows\ndef test_lmod_quote_parsing():\n lines = ['setenv(\"SOME_PARTICULAR_DIR\",\"-L/opt/cray/pe/mpich/8.1.4/gtl/lib\")']\n result = get_path_from_module_contents(lines, \"some-module\")\n assert \"/opt/cray/pe/mpich/8.1.4/gtl\" == result\n" }, { "path": "lib/spack/spack/test/modules/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n" }, { "path": "lib/spack/spack/test/modules/common.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport os\nimport pickle\nimport stat\n\nimport pytest\n\nimport spack.cmd.modules\nimport spack.concretize\nimport spack.config\nimport spack.error\nimport spack.modules\nimport spack.modules.common\nimport spack.modules.tcl\nimport spack.package_base\nimport spack.package_prefs\nimport spack.repo\nfrom spack.installer import PackageInstaller\nfrom spack.llnl.util.filesystem import readlink\nfrom spack.modules.common import UpstreamModuleIndex\n\npytestmark = [\n pytest.mark.not_on_windows(\"does not run on windows\"),\n pytest.mark.usefixtures(\"mock_modules_root\"),\n]\n\n\ndef test_update_dictionary_extending_list():\n target = {\"foo\": {\"a\": 1, \"b\": 2, \"d\": 4}, \"bar\": [1, 2, 4], \"baz\": \"foobar\"}\n update = {\"foo\": {\"c\": 3}, \"bar\": [3], \"baz\": \"foobaz\", \"newkey\": {\"d\": 4}}\n spack.modules.common.update_dictionary_extending_lists(target, update)\n assert len(target) == 4\n assert len(target[\"foo\"]) == 4\n assert len(target[\"bar\"]) == 4\n assert target[\"baz\"] == \"foobaz\"\n\n\n@pytest.fixture()\ndef mock_module_defaults(monkeypatch):\n def impl(*args):\n # No need to patch both types because neither override base\n monkeypatch.setattr(\n spack.modules.common.BaseConfiguration, \"defaults\", [arg for arg in args]\n )\n\n return impl\n\n\n@pytest.fixture()\ndef mock_package_perms(monkeypatch):\n perms = stat.S_IRGRP | stat.S_IWGRP\n monkeypatch.setattr(spack.package_prefs, \"get_package_permissions\", lambda spec: perms)\n\n yield perms\n\n\ndef test_modules_written_with_proper_permissions(\n mock_module_filename, mock_package_perms, mock_packages, config\n):\n spec = spack.concretize.concretize_one(\"mpileaks\")\n\n # The code tested is common to all module types, but has to be tested from\n # one. Tcl picked at random\n generator = spack.modules.tcl.TclModulefileWriter(spec, \"default\")\n generator.write()\n\n assert mock_package_perms & os.stat(mock_module_filename).st_mode == mock_package_perms\n\n\n@pytest.mark.parametrize(\"module_type\", [\"tcl\", \"lmod\"])\ndef test_modules_default_symlink(\n module_type, mock_packages, mock_module_filename, mock_module_defaults, config\n):\n spec = spack.concretize.concretize_one(\"mpileaks@2.3\")\n mock_module_defaults(spec.format(\"{name}{@version}\"), True)\n\n generator_cls = spack.modules.module_types[module_type]\n generator = generator_cls(spec, \"default\")\n generator.write()\n\n link_path = os.path.join(os.path.dirname(mock_module_filename), \"default\")\n assert os.path.islink(link_path)\n assert readlink(link_path) == mock_module_filename\n\n generator.remove()\n assert not os.path.lexists(link_path)\n\n\nclass MockDb:\n def __init__(self, db_ids, spec_hash_to_db):\n self.upstream_dbs = db_ids\n self.spec_hash_to_db = spec_hash_to_db\n\n def db_for_spec_hash(self, spec_hash):\n return self.spec_hash_to_db.get(spec_hash)\n\n\nclass MockSpec:\n def __init__(self, unique_id):\n self.unique_id = unique_id\n\n def dag_hash(self):\n return self.unique_id\n\n\ndef test_upstream_module_index():\n s1 = MockSpec(\"spec-1\")\n s2 = MockSpec(\"spec-2\")\n s3 = MockSpec(\"spec-3\")\n s4 = MockSpec(\"spec-4\")\n\n tcl_module_index = \"\"\"\\\nmodule_index:\n {0}:\n path: /path/to/a\n use_name: a\n\"\"\".format(s1.dag_hash())\n\n module_indices = [{\"tcl\": spack.modules.common._read_module_index(tcl_module_index)}, {}]\n\n dbs = [\"d0\", \"d1\"]\n\n mock_db = MockDb(dbs, {s1.dag_hash(): \"d0\", s2.dag_hash(): \"d1\", s3.dag_hash(): \"d0\"})\n upstream_index = UpstreamModuleIndex(mock_db, module_indices)\n\n m1 = upstream_index.upstream_module(s1, \"tcl\")\n assert m1.path == \"/path/to/a\"\n\n # No modules are defined for the DB associated with s2\n assert not upstream_index.upstream_module(s2, \"tcl\")\n\n # Modules are defined for the index associated with s1, but none are\n # defined for the requested type\n assert not upstream_index.upstream_module(s1, \"lmod\")\n\n # A module is registered with a DB and the associated module index has\n # modules of the specified type defined, but not for the requested spec\n assert not upstream_index.upstream_module(s3, \"tcl\")\n\n # The spec isn't recorded as installed in any of the DBs\n with pytest.raises(spack.error.SpackError):\n upstream_index.upstream_module(s4, \"tcl\")\n\n\ndef test_get_module_upstream():\n s1 = MockSpec(\"spec-1\")\n\n tcl_module_index = \"\"\"\\\nmodule_index:\n {0}:\n path: /path/to/a\n use_name: a\n\"\"\".format(s1.dag_hash())\n\n module_indices = [{}, {\"tcl\": spack.modules.common._read_module_index(tcl_module_index)}]\n\n dbs = [\"d0\", \"d1\"]\n\n mock_db = MockDb(dbs, {s1.dag_hash(): \"d1\"})\n upstream_index = UpstreamModuleIndex(mock_db, module_indices)\n\n setattr(s1, \"installed_upstream\", True)\n try:\n old_index = spack.modules.common.upstream_module_index\n spack.modules.common.upstream_module_index = upstream_index\n\n m1_path = spack.modules.get_module(\"tcl\", s1, True)\n assert m1_path == \"/path/to/a\"\n finally:\n spack.modules.common.upstream_module_index = old_index\n\n\n@pytest.mark.regression(\"14347\")\ndef test_load_installed_package_not_in_repo(install_mockery, mock_fetch, monkeypatch):\n \"\"\"Test that installed packages that have been removed are still loadable\"\"\"\n spec = spack.concretize.concretize_one(\"trivial-install-test-package\")\n PackageInstaller([spec.package], explicit=True).install()\n spack.modules.module_types[\"tcl\"](spec, \"default\", True).write()\n\n def find_nothing(*args):\n raise spack.repo.UnknownPackageError(\"Repo package access is disabled for test\")\n\n # Mock deletion of the package\n spec._package = None\n monkeypatch.setattr(spack.repo.PATH, \"get\", find_nothing)\n with pytest.raises(spack.repo.UnknownPackageError):\n spec.package\n\n module_path = spack.modules.get_module(\"tcl\", spec, True)\n assert module_path\n\n spack.package_base.PackageBase.uninstall_by_spec(spec)\n\n\n@pytest.mark.regression(\"37649\")\ndef test_check_module_set_name(mutable_config):\n \"\"\"Tests that modules set name are validated correctly and an error is reported if the\n name we require does not exist or is reserved by the configuration.\"\"\"\n # Minimal modules.yaml config.\n spack.config.set(\n \"modules\",\n {\n \"prefix_inspections\": {\"./bin\": [\"PATH\"]},\n # module sets\n \"first\": {},\n \"second\": {},\n },\n )\n\n # Valid module set name\n spack.cmd.modules.check_module_set_name(\"first\")\n\n # Invalid module set names\n msg = \"Valid module set names are\"\n with pytest.raises(spack.error.ConfigError, match=msg):\n spack.cmd.modules.check_module_set_name(\"prefix_inspections\")\n\n with pytest.raises(spack.error.ConfigError, match=msg):\n spack.cmd.modules.check_module_set_name(\"third\")\n\n\n@pytest.mark.parametrize(\"module_type\", [\"tcl\", \"lmod\"])\ndef test_module_writers_are_pickleable(default_mock_concretization, module_type):\n s = default_mock_concretization(\"mpileaks\")\n writer = spack.modules.module_types[module_type](s, \"default\")\n assert pickle.loads(pickle.dumps(writer)).spec == s\n" }, { "path": "lib/spack/spack/test/modules/conftest.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport pathlib\n\nimport pytest\n\nimport spack.concretize\nimport spack.modules.lmod\nimport spack.modules.tcl\nimport spack.spec\n\n\n@pytest.fixture()\ndef modulefile_content(request):\n \"\"\"Returns a function that generates the content of a module file as a list of lines.\"\"\"\n writer_cls = getattr(request.module, \"writer_cls\")\n\n def _impl(spec_like, module_set_name=\"default\", explicit=True):\n if isinstance(spec_like, str):\n spec_like = spack.spec.Spec(spec_like)\n spec = spack.concretize.concretize_one(spec_like)\n generator = writer_cls(spec, module_set_name, explicit)\n generator.write(overwrite=True)\n written_module = pathlib.Path(generator.layout.filename)\n content = written_module.read_text(encoding=\"utf-8\").splitlines()\n generator.remove()\n return content\n\n return _impl\n\n\n@pytest.fixture()\ndef factory(request, mock_modules_root):\n \"\"\"Given a spec string, returns an instance of the writer and the corresponding spec.\"\"\"\n writer_cls = getattr(request.module, \"writer_cls\")\n\n def _mock(spec_string, module_set_name=\"default\", explicit=True):\n spec = spack.concretize.concretize_one(spec_string)\n return writer_cls(spec, module_set_name, explicit), spec\n\n return _mock\n\n\n@pytest.fixture()\ndef mock_module_filename(monkeypatch, tmp_path: pathlib.Path):\n filename = tmp_path / \"module\"\n # Set for both module types so we can test both\n monkeypatch.setattr(spack.modules.lmod.LmodFileLayout, \"filename\", str(filename))\n monkeypatch.setattr(spack.modules.tcl.TclFileLayout, \"filename\", str(filename))\n yield str(filename)\n" }, { "path": "lib/spack/spack/test/modules/lmod.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport pathlib\n\nimport pytest\n\nimport spack.vendor.archspec.cpu\n\nimport spack.concretize\nimport spack.config\nimport spack.environment as ev\nimport spack.main\nimport spack.modules.common\nimport spack.modules.lmod\nimport spack.spec\nimport spack.util.environment\n\nmpich_spec_string = \"mpich@3.0.4\"\nmpileaks_spec_string = \"mpileaks\"\nlibdwarf_spec_string = \"libdwarf arch=x64-linux\"\n\ninstall = spack.main.SpackCommand(\"install\")\n\n#: Class of the writer tested in this module\nwriter_cls = spack.modules.lmod.LmodModulefileWriter\n\npytestmark = [\n pytest.mark.not_on_windows(\"does not run on windows\"),\n pytest.mark.usefixtures(\"mock_modules_root\"),\n]\n\n\n@pytest.fixture(params=[\"clang@=15.0.0\", \"gcc@=10.2.1\"])\ndef compiler(request):\n return request.param\n\n\n@pytest.fixture(\n params=[\n (\"mpich@3.0.4\", (\"mpi\",), True, False),\n (\"mpich@3.0.1\", [], True, True),\n (\"openblas@0.2.15\", (\"blas\",), True, False),\n (\"openblas-with-lapack@0.2.15\", (\"blas\", \"lapack\"), True, False),\n (\"mpileaks@2.3\", (\"mpi\",), True, False),\n (\"mpileaks@2.1\", [], True, False),\n (\"py-extension1@2.0\", (\"python\",), False, True),\n (\"python@3.8.0\", (\"python\",), False, True),\n ]\n)\ndef provider(request):\n return request.param\n\n\n@pytest.mark.usefixtures(\"mutable_config\", \"mock_packages\")\nclass TestLmod:\n @pytest.mark.regression(\"37788\")\n @pytest.mark.parametrize(\"modules_config\", [\"core_compilers\", \"core_compilers_at_equal\"])\n def test_layout_for_specs_compiled_with_core_compilers(\n self, modules_config, module_configuration, factory\n ):\n \"\"\"Tests that specs compiled with core compilers are in the 'Core' folder. Also tests that\n we can use both ``compiler@version`` and ``compiler@=version`` to specify a core compiler.\n \"\"\"\n module_configuration(modules_config)\n module, spec = factory(\"libelf%clang@15.0.0\")\n assert \"Core\" in module.layout.available_path_parts\n\n def test_file_layout(self, compiler, provider, factory, module_configuration):\n \"\"\"Tests the layout of files in the hierarchy is the one expected.\"\"\"\n module_configuration(\"complex_hierarchy\")\n spec_string, services, use_compiler, place_in_core = provider\n\n # Non-python specs add compiler\n factory_string = spec_string\n if use_compiler:\n factory_string += \"%\" + compiler\n\n module, spec = factory(factory_string)\n\n layout = module.layout\n\n # Check that the services provided are in the hierarchy\n for s in services:\n assert s in layout.conf.hierarchy_tokens\n\n # Check that the compiler part of the path has no hash and that it\n # is transformed to r\"Core\" if the compiler is listed among core\n # compilers\n # Check that specs listed as core_specs are transformed to \"Core\"\n # Check that specs with no hierarchy components are transformed to \"Core\"\n if \"clang@=15.0.0\" in factory_string or place_in_core:\n assert \"Core\" in layout.available_path_parts\n else:\n assert compiler.replace(\"@=\", \"/\") in layout.available_path_parts\n\n # Check that the provider part instead has always an hash even if\n # hash has been disallowed in the configuration file\n path_parts = layout.available_path_parts\n service_part = spec_string.replace(\"@\", \"/\")\n service_part = \"-\".join([service_part, layout.spec.dag_hash(length=7)])\n\n if \"mpi\" in spec:\n # It's a user, not a provider, so create the provider string\n service_part = layout.spec[\"mpi\"].format(\"{name}/{version}-{hash:7}\")\n elif \"python\" in spec:\n # It's a user, not a provider, so create the provider string\n service_part = layout.spec[\"python\"].format(\"{name}/{version}-{hash:7}\")\n else:\n # Only relevant for providers, not users, of virtuals\n assert service_part in path_parts\n\n # Check that multi-providers have repetitions in path parts\n repetitions = len([x for x in path_parts if service_part == x])\n if spec_string == \"openblas-with-lapack@0.2.15\":\n assert repetitions == 2\n elif spec_string == \"mpileaks@2.1\":\n assert repetitions == 0\n else:\n assert repetitions == 1\n\n def test_compilers_provided_different_name(\n self, factory, module_configuration, compiler_factory\n ):\n with spack.config.override(\n \"packages\", {\"llvm\": {\"externals\": [compiler_factory(spec=\"llvm@3.3 +clang\")]}}\n ):\n module_configuration(\"complex_hierarchy\")\n module, spec = factory(\"intel-oneapi-compilers%clang@3.3\")\n\n provides = module.conf.provides\n\n assert \"compiler\" in provides\n assert provides[\"compiler\"] == spack.spec.Spec(\"intel-oneapi-compilers@=3.0\")\n\n @pytest.mark.parametrize(\"language\", [\"c\", \"cxx\", \"fortran\"])\n def test_compiler_language_virtuals(self, factory, module_configuration, language):\n \"\"\"Tests all compiler virtuals for hierarchical module placement.\"\"\"\n module_configuration(\"complex_hierarchy\")\n module, spec = factory(f\"single-language-virtual +{language} %{language}=gcc@=10.2.1\")\n\n requires = module.conf.requires\n\n assert \"gcc@=10.2.1\" in requires[\"compiler\"]\n\n def test_simple_case(self, modulefile_content, module_configuration):\n \"\"\"Tests the generation of a simple Lua module file.\"\"\"\n\n module_configuration(\"autoload_direct\")\n content = modulefile_content(mpich_spec_string)\n\n assert \"-- -*- lua -*-\" in content\n assert \"whatis([[Name : mpich]])\" in content\n assert \"whatis([[Version : 3.0.4]])\" in content\n assert 'family(\"mpi\")' in content\n\n def test_autoload_direct(self, modulefile_content, module_configuration):\n \"\"\"Tests the automatic loading of direct dependencies.\"\"\"\n\n module_configuration(\"autoload_direct\")\n content = modulefile_content(mpileaks_spec_string)\n\n assert len([x for x in content if \"depends_on(\" in x]) == 3\n\n def test_autoload_all(self, modulefile_content, module_configuration):\n \"\"\"Tests the automatic loading of all dependencies.\"\"\"\n\n module_configuration(\"autoload_all\")\n content = modulefile_content(mpileaks_spec_string)\n\n assert len([x for x in content if \"depends_on(\" in x]) == 6\n\n def test_alter_environment(self, modulefile_content, module_configuration):\n \"\"\"Tests modifications to run-time environment.\"\"\"\n\n module_configuration(\"alter_environment\")\n content = modulefile_content(\"mpileaks platform=test target=x86_64\")\n\n assert len([x for x in content if x.startswith('prepend_path(\"CMAKE_PREFIX_PATH\"')]) == 0\n assert len([x for x in content if 'setenv(\"FOO\", \"foo\")' in x]) == 1\n assert len([x for x in content if 'unsetenv(\"BAR\")' in x]) == 1\n\n content = modulefile_content(\"libdwarf platform=test target=core2\")\n\n assert len([x for x in content if x.startswith('prepend_path(\"CMAKE_PREFIX_PATH\"')]) == 0\n assert len([x for x in content if 'setenv(\"FOO\", \"foo\")' in x]) == 0\n assert len([x for x in content if 'unsetenv(\"BAR\")' in x]) == 0\n\n def test_prepend_path_separator(self, modulefile_content, module_configuration):\n \"\"\"Tests that we can use custom delimiters to manipulate path lists.\"\"\"\n\n module_configuration(\"module_path_separator\")\n content = modulefile_content(\"module-path-separator\")\n\n assert len([x for x in content if 'append_path(\"COLON\", \"foo\", \":\")' in x]) == 1\n assert len([x for x in content if 'prepend_path(\"COLON\", \"foo\", \":\")' in x]) == 1\n assert len([x for x in content if 'remove_path(\"COLON\", \"foo\", \":\")' in x]) == 1\n assert len([x for x in content if 'append_path(\"SEMICOLON\", \"bar\", \";\")' in x]) == 1\n assert len([x for x in content if 'prepend_path(\"SEMICOLON\", \"bar\", \";\")' in x]) == 1\n assert len([x for x in content if 'remove_path(\"SEMICOLON\", \"bar\", \";\")' in x]) == 1\n assert len([x for x in content if 'append_path(\"SPACE\", \"qux\", \" \")' in x]) == 1\n assert len([x for x in content if 'remove_path(\"SPACE\", \"qux\", \" \")' in x]) == 1\n\n @pytest.mark.regression(\"11355\")\n def test_manpath_setup(self, modulefile_content, module_configuration):\n \"\"\"Tests specific setup of MANPATH environment variable.\"\"\"\n\n module_configuration(\"autoload_direct\")\n\n # no manpath set by module\n content = modulefile_content(\"mpileaks\")\n assert len([x for x in content if 'append_path(\"MANPATH\", \"\", \":\")' in x]) == 0\n\n # manpath set by module with prepend_path\n content = modulefile_content(\"module-manpath-prepend\")\n assert (\n len([x for x in content if 'prepend_path(\"MANPATH\", \"/path/to/man\", \":\")' in x]) == 1\n )\n assert (\n len([x for x in content if 'prepend_path(\"MANPATH\", \"/path/to/share/man\", \":\")' in x])\n == 1\n )\n assert len([x for x in content if 'append_path(\"MANPATH\", \"\", \":\")' in x]) == 1\n\n # manpath set by module with append_path\n content = modulefile_content(\"module-manpath-append\")\n assert len([x for x in content if 'append_path(\"MANPATH\", \"/path/to/man\", \":\")' in x]) == 1\n assert len([x for x in content if 'append_path(\"MANPATH\", \"\", \":\")' in x]) == 1\n\n # manpath set by module with setenv\n content = modulefile_content(\"module-manpath-setenv\")\n assert len([x for x in content if 'setenv(\"MANPATH\", \"/path/to/man\")' in x]) == 1\n assert len([x for x in content if 'append_path(\"MANPATH\", \"\", \":\")' in x]) == 0\n\n @pytest.mark.regression(\"29578\")\n def test_setenv_raw_value(self, modulefile_content, module_configuration):\n \"\"\"Tests that we can set environment variable value without formatting it.\"\"\"\n\n module_configuration(\"autoload_direct\")\n content = modulefile_content(\"module-setenv-raw\")\n\n assert len([x for x in content if 'setenv(\"FOO\", \"{{name}}, {name}, {{}}, {}\")' in x]) == 1\n\n @pytest.mark.skipif(\n str(spack.vendor.archspec.cpu.host().family) != \"x86_64\",\n reason=\"test data is specific for x86_64\",\n )\n def test_help_message(self, modulefile_content, module_configuration):\n \"\"\"Tests the generation of module help message.\"\"\"\n\n module_configuration(\"autoload_direct\")\n content = modulefile_content(\"mpileaks target=core2\")\n\n help_msg = (\n \"help([[Name : mpileaks]])\"\n \"help([[Version: 2.3]])\"\n \"help([[Target : core2]])\"\n \"help()\"\n \"help([[Mpileaks is a mock package that passes audits]])\"\n )\n assert help_msg in \"\".join(content)\n\n content = modulefile_content(\"libdwarf target=core2\")\n\n help_msg = (\n \"help([[Name : libdwarf]])\"\n \"help([[Version: 20130729]])\"\n \"help([[Target : core2]])\"\n \"depends_on(\"\n )\n assert help_msg in \"\".join(content)\n\n content = modulefile_content(\"module-long-help target=core2\")\n\n help_msg = (\n \"help([[Name : module-long-help]])\"\n \"help([[Version: 1.0]])\"\n \"help([[Target : core2]])\"\n \"help()\"\n \"help([[Package to test long description message generated in modulefile.\"\n \"Message too long is wrapped over multiple lines.]])\"\n )\n assert help_msg in \"\".join(content)\n\n def test_exclude(self, modulefile_content, module_configuration):\n \"\"\"Tests excluding the generation of selected modules.\"\"\"\n module_configuration(\"exclude\")\n content = modulefile_content(mpileaks_spec_string)\n\n assert len([x for x in content if \"depends_on(\" in x]) == 2\n\n def test_no_hash(self, factory, module_configuration):\n \"\"\"Makes sure that virtual providers (in the hierarchy) always\n include a hash. Make sure that the module file for the spec\n does not include a hash if hash_length is 0.\n \"\"\"\n\n module_configuration(\"no_hash\")\n module, spec = factory(mpileaks_spec_string)\n path = module.layout.filename\n mpi_spec = spec[\"mpi\"]\n\n mpi_element = \"{0}/{1}-{2}/\".format(\n mpi_spec.name, mpi_spec.version, mpi_spec.dag_hash(length=7)\n )\n\n assert mpi_element in path\n\n mpileaks_spec = spec\n mpileaks_element = \"{0}/{1}.lua\".format(mpileaks_spec.name, mpileaks_spec.version)\n\n assert path.endswith(mpileaks_element)\n\n def test_no_core_compilers(self, factory, module_configuration):\n \"\"\"Ensures that missing 'core_compilers' in the configuration file\n raises the right exception.\n \"\"\"\n\n # In this case we miss the entry completely\n module_configuration(\"missing_core_compilers\")\n\n module, spec = factory(mpileaks_spec_string)\n with pytest.raises(spack.modules.lmod.CoreCompilersNotFoundError):\n module.write()\n\n # Here we have an empty list\n module_configuration(\"core_compilers_empty\")\n\n module, spec = factory(mpileaks_spec_string)\n with pytest.raises(spack.modules.lmod.CoreCompilersNotFoundError):\n module.write()\n\n def test_conflicts(self, modulefile_content, module_configuration):\n \"\"\"Tests adding conflicts to the module.\"\"\"\n\n # This configuration has no error, so check the conflicts directives\n # are there\n module_configuration(\"conflicts\")\n content = modulefile_content(\"mpileaks\")\n\n assert len([x for x in content if x.startswith(\"conflict\")]) == 2\n assert len([x for x in content if x == 'conflict(\"mpileaks\")']) == 1\n assert len([x for x in content if x == 'conflict(\"intel/14.0.1\")']) == 1\n\n def test_inconsistent_conflict_in_modules_yaml(self, modulefile_content, module_configuration):\n \"\"\"Tests inconsistent conflict definition in `modules.yaml`.\"\"\"\n\n # This configuration is inconsistent, check an error is raised\n module_configuration(\"wrong_conflicts\")\n with pytest.raises(spack.modules.common.ModulesError):\n modulefile_content(\"mpileaks\")\n\n def test_override_template_in_package(self, modulefile_content, module_configuration):\n \"\"\"Tests overriding a template from and attribute in the package.\"\"\"\n\n module_configuration(\"autoload_direct\")\n content = modulefile_content(\"override-module-templates\")\n\n assert \"Override successful!\" in content\n\n def test_override_template_in_modules_yaml(\n self, modulefile_content, module_configuration, host_architecture_str\n ):\n \"\"\"Tests overriding a template from `modules.yaml`\"\"\"\n module_configuration(\"override_template\")\n\n content = modulefile_content(\"override-module-templates\")\n assert \"Override even better!\" in content\n\n content = modulefile_content(f\"mpileaks target={host_architecture_str}\")\n assert \"Override even better!\" in content\n\n def test_external_configure_args(self, factory):\n # If this package is detected as an external, its configure option line\n # in the module file starts with 'unknown'\n writer, spec = factory(\"externaltool\")\n\n assert \"unknown\" in writer.context.configure_options\n\n def test_guess_core_compilers(self, factory, module_configuration, monkeypatch):\n \"\"\"Check that we can guess core compilers.\"\"\"\n\n # In this case we miss the entry completely\n module_configuration(\"missing_core_compilers\")\n\n # Our mock paths must be detected as system paths\n monkeypatch.setattr(spack.util.environment, \"SYSTEM_DIRS\", [\"/path/bin\"])\n\n # We don't want to really write into user configuration\n # when running tests\n def no_op_set(*args, **kwargs):\n pass\n\n monkeypatch.setattr(spack.config, \"set\", no_op_set)\n\n # Assert we have core compilers now\n writer, _ = factory(mpileaks_spec_string)\n assert writer.conf.core_compilers\n\n @pytest.mark.parametrize(\n \"spec_str\", [\"mpileaks target=nocona\", \"mpileaks target=core2\", \"mpileaks target=x86_64\"]\n )\n @pytest.mark.regression(\"13005\")\n def test_only_generic_microarchitectures_in_root(\n self, spec_str, factory, module_configuration\n ):\n module_configuration(\"complex_hierarchy\")\n writer, spec = factory(spec_str)\n\n assert str(spec.target.family) in writer.layout.arch_dirname\n if spec.target.family != spec.target:\n assert str(spec.target) not in writer.layout.arch_dirname\n\n def test_projections_specific(self, factory, module_configuration):\n \"\"\"Tests reading the correct naming scheme.\"\"\"\n\n # This configuration has no error, so check the conflicts directives\n # are there\n module_configuration(\"projections\")\n\n # Test we read the expected configuration for the naming scheme\n writer, _ = factory(\"mpileaks\")\n expected = {\"all\": \"{name}/v{version}\", \"mpileaks\": \"{name}-mpiprojection\"}\n\n assert writer.conf.projections == expected\n projection = writer.spec.format(writer.conf.projections[\"mpileaks\"])\n assert projection in writer.layout.use_name\n\n def test_projections_all(self, factory, module_configuration):\n \"\"\"Tests reading the correct naming scheme.\"\"\"\n\n # This configuration has no error, so check the conflicts directives\n # are there\n module_configuration(\"projections\")\n\n # Test we read the expected configuration for the naming scheme\n writer, _ = factory(\"libelf\")\n expected = {\"all\": \"{name}/v{version}\", \"mpileaks\": \"{name}-mpiprojection\"}\n\n assert writer.conf.projections == expected\n projection = writer.spec.format(writer.conf.projections[\"all\"])\n assert projection in writer.layout.use_name\n\n def test_modules_relative_to_view(\n self,\n tmp_path: pathlib.Path,\n modulefile_content,\n module_configuration,\n install_mockery,\n mock_fetch,\n ):\n with ev.create_in_dir(str(tmp_path), with_view=True) as e:\n module_configuration(\"with_view\")\n install(\"--fake\", \"--add\", \"cmake\")\n\n spec = spack.concretize.concretize_one(\"cmake\")\n\n content = modulefile_content(\"cmake\")\n expected = e.default_view.get_projection_for_spec(spec)\n # Rather than parse all lines, ensure all prefixes in the content\n # point to the right one\n assert any(expected in line for line in content)\n assert not any(spec.prefix in line for line in content)\n\n def test_modules_no_arch(self, factory, module_configuration):\n module_configuration(\"no_arch\")\n module, spec = factory(mpileaks_spec_string)\n path = module.layout.filename\n\n assert str(spec.os) not in path\n\n def test_hide_implicits(self, module_configuration, temporary_store):\n \"\"\"Tests the addition and removal of hide command in modulerc.\"\"\"\n module_configuration(\"hide_implicits\")\n\n spec = spack.concretize.concretize_one(\"mpileaks@2.3\")\n\n # mpileaks is defined as implicit, thus hide command should appear in modulerc\n writer = writer_cls(spec, \"default\", False)\n writer.write()\n assert os.path.exists(writer.layout.modulerc)\n with open(writer.layout.modulerc, encoding=\"utf-8\") as f:\n content = [line.strip() for line in f.readlines()]\n hide_implicit_mpileaks = f'hide_version(\"{writer.layout.use_name}\")'\n assert len([x for x in content if hide_implicit_mpileaks == x]) == 1\n\n # The direct dependencies are all implicitly installed, and they should all be hidden,\n # except for mpich, which is provider for mpi, which is in the hierarchy, and therefore\n # can't be hidden. All other hidden modules should have a 7 character hash (the config\n # hash_length = 0 only applies to exposed modules).\n with open(writer.layout.filename, encoding=\"utf-8\") as f:\n depends_statements = [line.strip() for line in f.readlines() if \"depends_on\" in line]\n for dep in spec.dependencies(deptype=(\"link\", \"run\")):\n if dep.satisfies(\"mpi\"):\n assert not any(dep.dag_hash(7) in line for line in depends_statements)\n else:\n assert any(dep.dag_hash(7) in line for line in depends_statements)\n\n # when mpileaks becomes explicit, its file name changes (hash_length = 0), meaning an\n # extra module file is created; the old one still exists and remains hidden.\n writer = writer_cls(spec, \"default\", True)\n writer.write()\n assert os.path.exists(writer.layout.modulerc)\n with open(writer.layout.modulerc, encoding=\"utf-8\") as f:\n content = [line.strip() for line in f.readlines()]\n assert hide_implicit_mpileaks in content # old, implicit mpileaks is still hidden\n assert f'hide_version(\"{writer.layout.use_name}\")' not in content\n\n # after removing both the implicit and explicit module, the modulerc file would be empty\n # and should be removed.\n writer_cls(spec, \"default\", False).remove()\n writer_cls(spec, \"default\", True).remove()\n assert not os.path.exists(writer.layout.modulerc)\n assert not os.path.exists(writer.layout.filename)\n\n # implicit module is removed\n writer = writer_cls(spec, \"default\", False)\n writer.write()\n assert os.path.exists(writer.layout.filename)\n assert os.path.exists(writer.layout.modulerc)\n writer.remove()\n assert not os.path.exists(writer.layout.modulerc)\n assert not os.path.exists(writer.layout.filename)\n\n # three versions of mpileaks are implicit\n writer = writer_cls(spec, \"default\", False)\n writer.write(overwrite=True)\n spec_alt1 = spack.concretize.concretize_one(\"mpileaks@2.2\")\n spec_alt2 = spack.concretize.concretize_one(\"mpileaks@2.1\")\n writer_alt1 = writer_cls(spec_alt1, \"default\", False)\n writer_alt1.write(overwrite=True)\n writer_alt2 = writer_cls(spec_alt2, \"default\", False)\n writer_alt2.write(overwrite=True)\n assert os.path.exists(writer.layout.modulerc)\n with open(writer.layout.modulerc, encoding=\"utf-8\") as f:\n content = [line.strip() for line in f.readlines()]\n hide_cmd = f'hide_version(\"{writer.layout.use_name}\")'\n hide_cmd_alt1 = f'hide_version(\"{writer_alt1.layout.use_name}\")'\n hide_cmd_alt2 = f'hide_version(\"{writer_alt2.layout.use_name}\")'\n assert len([x for x in content if hide_cmd == x]) == 1\n assert len([x for x in content if hide_cmd_alt1 == x]) == 1\n assert len([x for x in content if hide_cmd_alt2 == x]) == 1\n\n # one version is removed\n writer_alt1.remove()\n assert os.path.exists(writer.layout.modulerc)\n with open(writer.layout.modulerc, encoding=\"utf-8\") as f:\n content = [line.strip() for line in f.readlines()]\n assert len([x for x in content if hide_cmd == x]) == 1\n assert len([x for x in content if hide_cmd_alt1 == x]) == 0\n assert len([x for x in content if hide_cmd_alt2 == x]) == 1\n" }, { "path": "lib/spack/spack/test/modules/tcl.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\n\nimport pytest\n\nimport spack.vendor.archspec.cpu\n\nimport spack.concretize\nimport spack.modules.common\nimport spack.modules.tcl\n\nmpich_spec_string = \"mpich@3.0.4\"\nmpileaks_spec_string = \"mpileaks\"\nlibdwarf_spec_string = \"libdwarf target=x86_64\"\n\n#: Class of the writer tested in this module\nwriter_cls = spack.modules.tcl.TclModulefileWriter\n\npytestmark = [\n pytest.mark.not_on_windows(\"does not run on windows\"),\n pytest.mark.usefixtures(\"mock_modules_root\"),\n]\n\n\n@pytest.mark.usefixtures(\"mutable_config\", \"mock_packages\", \"mock_module_filename\")\nclass TestTcl:\n def test_simple_case(self, modulefile_content, module_configuration):\n \"\"\"Tests the generation of a simple Tcl module file.\"\"\"\n\n module_configuration(\"autoload_direct\")\n content = modulefile_content(mpich_spec_string)\n\n assert \"module-whatis {mpich @3.0.4}\" in content\n\n def test_autoload_direct(self, modulefile_content, module_configuration):\n \"\"\"Tests the automatic loading of direct dependencies.\"\"\"\n\n module_configuration(\"autoload_direct\")\n content = modulefile_content(mpileaks_spec_string)\n\n assert (\n len([x for x in content if \"if {![llength [info commands depends-on]]} {\" in x]) == 1\n )\n assert len([x for x in content if \" proc depends-on {args} {\" in x]) == 1\n assert len([x for x in content if \" module load {*}$args\" in x]) == 1\n # depends-on command defined once and used 3 times\n assert len([x for x in content if \"depends-on \" in x]) == 4\n\n # dtbuild1 has\n # - 1 ('run',) dependency\n # - 1 ('build','link') dependency\n # - 1 ('build',) dependency\n # Just make sure the 'build' dependency is not there\n content = modulefile_content(\"dtbuild1\")\n\n assert (\n len([x for x in content if \"if {![llength [info commands depends-on]]} {\" in x]) == 1\n )\n assert len([x for x in content if \" proc depends-on {args} {\" in x]) == 1\n assert len([x for x in content if \" module load {*}$args\" in x]) == 1\n # depends-on command defined once and used twice\n assert len([x for x in content if \"depends-on \" in x]) == 3\n\n # The configuration file sets the verbose keyword to False\n messages = [x for x in content if 'puts stderr \"Autoloading' in x]\n assert len(messages) == 0\n\n def test_autoload_all(self, modulefile_content, module_configuration):\n \"\"\"Tests the automatic loading of all dependencies.\"\"\"\n\n module_configuration(\"autoload_all\")\n content = modulefile_content(mpileaks_spec_string)\n\n assert (\n len([x for x in content if \"if {![llength [info commands depends-on]]} {\" in x]) == 1\n )\n assert len([x for x in content if \" proc depends-on {args} {\" in x]) == 1\n assert len([x for x in content if \" module load {*}$args\" in x]) == 1\n # depends-on command defined once and used 6 times\n assert len([x for x in content if \"depends-on \" in x]) == 7\n\n # dtbuild1 has\n # - 1 ('run',) dependency\n # - 1 ('build','link') dependency\n # - 1 ('build',) dependency\n # Just make sure the 'build' dependency is not there\n content = modulefile_content(\"dtbuild1\")\n\n assert (\n len([x for x in content if \"if {![llength [info commands depends-on]]} {\" in x]) == 1\n )\n assert len([x for x in content if \" proc depends-on {args} {\" in x]) == 1\n assert len([x for x in content if \" module load {*}$args\" in x]) == 1\n # depends-on command defined once and used twice\n assert len([x for x in content if \"depends-on \" in x]) == 3\n\n def test_prerequisites_direct(\n self, modulefile_content, module_configuration, host_architecture_str\n ):\n \"\"\"Tests asking direct dependencies as prerequisites.\"\"\"\n\n module_configuration(\"prerequisites_direct\")\n content = modulefile_content(f\"mpileaks target={host_architecture_str}\")\n\n assert len([x for x in content if \"prereq\" in x]) == 3\n\n def test_prerequisites_all(\n self, modulefile_content, module_configuration, host_architecture_str\n ):\n \"\"\"Tests asking all dependencies as prerequisites.\"\"\"\n\n module_configuration(\"prerequisites_all\")\n content = modulefile_content(f\"mpileaks target={host_architecture_str}\")\n\n assert len([x for x in content if \"prereq\" in x]) == 6\n\n def test_alter_environment(self, modulefile_content, module_configuration):\n \"\"\"Tests modifications to run-time environment.\"\"\"\n\n module_configuration(\"alter_environment\")\n content = modulefile_content(\"mpileaks platform=test target=x86_64\")\n\n assert len([x for x in content if x.startswith(\"prepend-path CMAKE_PREFIX_PATH\")]) == 0\n assert len([x for x in content if \"setenv FOO {foo}\" in x]) == 1\n assert len([x for x in content if \"setenv OMPI_MCA_mpi_leave_pinned {1}\" in x]) == 1\n assert len([x for x in content if \"setenv OMPI_MCA_MPI_LEAVE_PINNED {1}\" in x]) == 0\n assert len([x for x in content if \"unsetenv BAR\" in x]) == 1\n assert len([x for x in content if \"setenv MPILEAKS_ROOT\" in x]) == 1\n\n content = modulefile_content(\"libdwarf platform=test target=core2\")\n\n assert len([x for x in content if x.startswith(\"prepend-path CMAKE_PREFIX_PATH\")]) == 0\n assert len([x for x in content if \"setenv FOO {foo}\" in x]) == 0\n assert len([x for x in content if \"unsetenv BAR\" in x]) == 0\n assert len([x for x in content if \"depends-on foo/bar\" in x]) == 1\n assert len([x for x in content if \"setenv LIBDWARF_ROOT\" in x]) == 1\n\n def test_prepend_path_separator(self, modulefile_content, module_configuration):\n \"\"\"Tests that we can use custom delimiters to manipulate path lists.\"\"\"\n\n module_configuration(\"module_path_separator\")\n content = modulefile_content(\"module-path-separator\")\n\n assert len([x for x in content if \"append-path -d {:} COLON {foo}\" in x]) == 1\n assert len([x for x in content if \"prepend-path -d {:} COLON {foo}\" in x]) == 1\n assert len([x for x in content if \"remove-path -d {:} COLON {foo}\" in x]) == 1\n assert len([x for x in content if \"append-path -d {;} SEMICOLON {bar}\" in x]) == 1\n assert len([x for x in content if \"prepend-path -d {;} SEMICOLON {bar}\" in x]) == 1\n assert len([x for x in content if \"remove-path -d {;} SEMICOLON {bar}\" in x]) == 1\n assert len([x for x in content if \"append-path -d { } SPACE {qux}\" in x]) == 1\n assert len([x for x in content if \"remove-path -d { } SPACE {qux}\" in x]) == 1\n\n @pytest.mark.regression(\"11355\")\n def test_manpath_setup(self, modulefile_content, module_configuration):\n \"\"\"Tests specific setup of MANPATH environment variable.\"\"\"\n\n module_configuration(\"autoload_direct\")\n\n # no manpath set by module\n content = modulefile_content(\"mpileaks\")\n assert len([x for x in content if \"append-path MANPATH {}\" in x]) == 0\n\n # manpath set by module with prepend-path\n content = modulefile_content(\"module-manpath-prepend\")\n assert len([x for x in content if \"prepend-path -d {:} MANPATH {/path/to/man}\" in x]) == 1\n assert (\n len([x for x in content if \"prepend-path -d {:} MANPATH {/path/to/share/man}\" in x])\n == 1\n )\n assert len([x for x in content if \"append-path MANPATH {}\" in x]) == 1\n\n # manpath set by module with append-path\n content = modulefile_content(\"module-manpath-append\")\n assert len([x for x in content if \"append-path -d {:} MANPATH {/path/to/man}\" in x]) == 1\n assert len([x for x in content if \"append-path MANPATH {}\" in x]) == 1\n\n # manpath set by module with setenv\n content = modulefile_content(\"module-manpath-setenv\")\n assert len([x for x in content if \"setenv MANPATH {/path/to/man}\" in x]) == 1\n assert len([x for x in content if \"append-path MANPATH {}\" in x]) == 0\n\n @pytest.mark.regression(\"29578\")\n def test_setenv_raw_value(self, modulefile_content, module_configuration):\n \"\"\"Tests that we can set environment variable value without formatting it.\"\"\"\n\n module_configuration(\"autoload_direct\")\n content = modulefile_content(\"module-setenv-raw\")\n\n assert len([x for x in content if \"setenv FOO {{{name}}, {name}, {{}}, {}}\" in x]) == 1\n\n @pytest.mark.skipif(\n str(spack.vendor.archspec.cpu.host().family) != \"x86_64\",\n reason=\"test data is specific for x86_64\",\n )\n def test_help_message(self, modulefile_content, module_configuration):\n \"\"\"Tests the generation of module help message.\"\"\"\n\n module_configuration(\"autoload_direct\")\n content = modulefile_content(\"mpileaks target=core2\")\n\n help_msg = (\n \"proc ModulesHelp { } {\"\n \" puts stderr {Name : mpileaks}\"\n \" puts stderr {Version: 2.3}\"\n \" puts stderr {Target : core2}\"\n \" puts stderr {}\"\n \" puts stderr {Mpileaks is a mock package that passes audits}\"\n \"}\"\n )\n assert help_msg in \"\".join(content)\n\n content = modulefile_content(\"libdwarf target=core2\")\n\n help_msg = (\n \"proc ModulesHelp { } {\"\n \" puts stderr {Name : libdwarf}\"\n \" puts stderr {Version: 20130729}\"\n \" puts stderr {Target : core2}\"\n \"}\"\n )\n assert help_msg in \"\".join(content)\n\n content = modulefile_content(\"module-long-help target=core2\")\n\n help_msg = (\n \"proc ModulesHelp { } {\"\n \" puts stderr {Name : module-long-help}\"\n \" puts stderr {Version: 1.0}\"\n \" puts stderr {Target : core2}\"\n \" puts stderr {}\"\n \" puts stderr {Package to test long description message generated in modulefile.}\"\n \" puts stderr {Message too long is wrapped over multiple lines.}\"\n \"}\"\n )\n assert help_msg in \"\".join(content)\n\n def test_exclude(self, modulefile_content, module_configuration, host_architecture_str):\n \"\"\"Tests excluding the generation of selected modules.\"\"\"\n\n module_configuration(\"exclude\")\n content = modulefile_content(\"mpileaks ^zmpi\")\n\n # depends-on command defined once and used twice\n assert len([x for x in content if \"depends-on \" in x]) == 3\n\n with pytest.raises(FileNotFoundError):\n modulefile_content(f\"callpath target={host_architecture_str}\")\n\n content = modulefile_content(f\"zmpi target={host_architecture_str}\")\n\n # depends-on command defined once and used twice\n assert len([x for x in content if \"depends-on \" in x]) == 3\n\n def test_naming_scheme_compat(self, factory, module_configuration):\n \"\"\"Tests backwards compatibility for naming_scheme key\"\"\"\n module_configuration(\"naming_scheme\")\n\n # Test we read the expected configuration for the naming scheme\n writer, _ = factory(\"mpileaks\")\n expected = {\"all\": \"{name}/{version}-{compiler.name}\"}\n\n assert writer.conf.projections == expected\n projection = writer.spec.format(writer.conf.projections[\"all\"])\n assert projection in writer.layout.use_name\n\n def test_projections_specific(self, factory, module_configuration):\n \"\"\"Tests reading the correct naming scheme.\"\"\"\n\n # This configuration has no error, so check the conflicts directives\n # are there\n module_configuration(\"projections\")\n\n # Test we read the expected configuration for the naming scheme\n writer, _ = factory(\"mpileaks\")\n expected = {\"all\": \"{name}/{version}-{compiler.name}\", \"mpileaks\": \"{name}-mpiprojection\"}\n\n assert writer.conf.projections == expected\n projection = writer.spec.format(writer.conf.projections[\"mpileaks\"])\n assert projection in writer.layout.use_name\n\n def test_projections_all(self, factory, module_configuration):\n \"\"\"Tests reading the correct naming scheme.\"\"\"\n\n # This configuration has no error, so check the conflicts directives\n # are there\n module_configuration(\"projections\")\n\n # Test we read the expected configuration for the naming scheme\n writer, _ = factory(\"libelf\")\n expected = {\"all\": \"{name}/{version}-{compiler.name}\", \"mpileaks\": \"{name}-mpiprojection\"}\n\n assert writer.conf.projections == expected\n projection = writer.spec.format(writer.conf.projections[\"all\"])\n assert projection in writer.layout.use_name\n\n def test_invalid_naming_scheme(self, factory, module_configuration):\n \"\"\"Tests the evaluation of an invalid naming scheme.\"\"\"\n\n module_configuration(\"invalid_naming_scheme\")\n\n # Test that having invalid tokens in the naming scheme raises\n # a RuntimeError\n writer, _ = factory(\"mpileaks\")\n with pytest.raises(RuntimeError):\n writer.layout.use_name\n\n def test_invalid_token_in_env_name(self, factory, module_configuration):\n \"\"\"Tests setting environment variables with an invalid name.\"\"\"\n\n module_configuration(\"invalid_token_in_env_var_name\")\n\n writer, _ = factory(\"mpileaks\")\n with pytest.raises(RuntimeError):\n writer.write()\n\n def test_conflicts(self, modulefile_content, module_configuration):\n \"\"\"Tests adding conflicts to the module.\"\"\"\n\n # This configuration has no error, so check the conflicts directives\n # are there\n module_configuration(\"conflicts\")\n content = modulefile_content(\"mpileaks\")\n\n assert len([x for x in content if x.startswith(\"conflict\")]) == 2\n assert len([x for x in content if x == \"conflict mpileaks\"]) == 1\n assert len([x for x in content if x == \"conflict intel/14.0.1\"]) == 1\n\n def test_inconsistent_conflict_in_modules_yaml(self, modulefile_content, module_configuration):\n \"\"\"Tests inconsistent conflict definition in `modules.yaml`.\"\"\"\n\n # This configuration is inconsistent, check an error is raised\n module_configuration(\"wrong_conflicts\")\n with pytest.raises(spack.modules.common.ModulesError):\n modulefile_content(\"mpileaks\")\n\n def test_module_index(\n self, module_configuration, factory, tmp_path_factory: pytest.TempPathFactory\n ):\n module_configuration(\"suffix\")\n\n w1, s1 = factory(\"mpileaks\")\n w2, s2 = factory(\"callpath\")\n w3, s3 = factory(\"openblas\")\n\n test_root = str(tmp_path_factory.mktemp(\"module-root\"))\n\n spack.modules.common.generate_module_index(test_root, [w1, w2])\n\n index = spack.modules.common.read_module_index(test_root)\n\n assert index[s1.dag_hash()].use_name == w1.layout.use_name\n assert index[s2.dag_hash()].path == w2.layout.filename\n\n spack.modules.common.generate_module_index(test_root, [w3])\n\n index = spack.modules.common.read_module_index(test_root)\n\n assert len(index) == 3\n assert index[s1.dag_hash()].use_name == w1.layout.use_name\n assert index[s2.dag_hash()].path == w2.layout.filename\n\n spack.modules.common.generate_module_index(test_root, [w3], overwrite=True)\n\n index = spack.modules.common.read_module_index(test_root)\n\n assert len(index) == 1\n assert index[s3.dag_hash()].use_name == w3.layout.use_name\n\n def test_suffixes(self, module_configuration, factory):\n \"\"\"Tests adding suffixes to module file name.\"\"\"\n module_configuration(\"suffix\")\n\n writer, spec = factory(\"mpileaks+debug target=x86_64\")\n assert \"foo\" in writer.layout.use_name\n assert \"foo-foo\" not in writer.layout.use_name\n\n writer, spec = factory(\"mpileaks~debug target=x86_64\")\n assert \"foo-bar\" in writer.layout.use_name\n assert \"baz\" not in writer.layout.use_name\n\n writer, spec = factory(\"mpileaks~debug+opt target=x86_64\")\n assert \"baz-foo-bar\" in writer.layout.use_name\n\n def test_suffixes_format(self, module_configuration, factory):\n \"\"\"Tests adding suffixes as spec format string to module file name.\"\"\"\n module_configuration(\"suffix-format\")\n\n writer, spec = factory(\"mpileaks +debug target=x86_64 ^mpich@3.0.4\")\n assert \"debug=True\" in writer.layout.use_name\n assert \"mpi=mpich-v3.0.4\" in writer.layout.use_name\n\n def test_setup_environment(self, modulefile_content, module_configuration):\n \"\"\"Tests the internal set-up of run-time environment.\"\"\"\n\n module_configuration(\"suffix\")\n content = modulefile_content(\"mpileaks\")\n\n assert len([x for x in content if \"setenv FOOBAR\" in x]) == 1\n assert len([x for x in content if \"setenv FOOBAR {mpileaks}\" in x]) == 1\n\n spec = spack.concretize.concretize_one(\"mpileaks\")\n content = modulefile_content(spec[\"callpath\"])\n\n assert len([x for x in content if \"setenv FOOBAR\" in x]) == 1\n assert len([x for x in content if \"setenv FOOBAR {callpath}\" in x]) == 1\n\n def test_override_config(self, module_configuration, factory):\n \"\"\"Tests overriding some sections of the configuration file.\"\"\"\n module_configuration(\"override_config\")\n\n writer, spec = factory(\"mpileaks~opt target=x86_64\")\n assert \"mpich-static\" in writer.layout.use_name\n assert \"over\" not in writer.layout.use_name\n assert \"ridden\" not in writer.layout.use_name\n\n writer, spec = factory(\"mpileaks+opt target=x86_64\")\n assert \"over-ridden\" in writer.layout.use_name\n assert \"mpich\" not in writer.layout.use_name\n assert \"static\" not in writer.layout.use_name\n\n def test_override_template_in_package(self, modulefile_content, module_configuration):\n \"\"\"Tests overriding a template from and attribute in the package.\"\"\"\n\n module_configuration(\"autoload_direct\")\n content = modulefile_content(\"override-module-templates\")\n\n assert \"Override successful!\" in content\n\n def test_override_template_in_modules_yaml(\n self, modulefile_content, module_configuration, host_architecture_str\n ):\n \"\"\"Tests overriding a template from `modules.yaml`\"\"\"\n module_configuration(\"override_template\")\n\n content = modulefile_content(\"override-module-templates\")\n assert \"Override even better!\" in content\n\n content = modulefile_content(f\"mpileaks target={host_architecture_str}\")\n assert \"Override even better!\" in content\n\n def test_extend_context(self, modulefile_content, module_configuration):\n \"\"\"Tests using a package defined context\"\"\"\n module_configuration(\"autoload_direct\")\n content = modulefile_content(\"override-context-templates\")\n\n assert 'puts stderr \"sentence from package\"' in content\n\n short_description = \"module-whatis {This package updates the context for Tcl modulefiles.}\"\n assert short_description in content\n\n @pytest.mark.regression(\"4400\")\n @pytest.mark.db\n def test_hide_implicits_no_arg(self, module_configuration, mutable_database):\n module_configuration(\"exclude_implicits\")\n\n # mpileaks has been installed explicitly when setting up\n # the tests database\n mpileaks_specs = mutable_database.query(\"mpileaks\")\n for item in mpileaks_specs:\n writer = writer_cls(item, \"default\")\n assert not writer.conf.excluded\n\n # callpath is a dependency of mpileaks, and has been pulled\n # in implicitly\n callpath_specs = mutable_database.query(\"callpath\")\n for item in callpath_specs:\n writer = writer_cls(item, \"default\")\n assert writer.conf.excluded\n\n @pytest.mark.regression(\"12105\")\n def test_hide_implicits_with_arg(self, module_configuration):\n module_configuration(\"exclude_implicits\")\n\n # mpileaks is defined as explicit with explicit argument set on writer\n mpileaks_spec = spack.concretize.concretize_one(\"mpileaks\")\n writer = writer_cls(mpileaks_spec, \"default\", True)\n assert not writer.conf.excluded\n\n # callpath is defined as implicit with explicit argument set on writer\n callpath_spec = spack.concretize.concretize_one(\"callpath\")\n writer = writer_cls(callpath_spec, \"default\", False)\n assert writer.conf.excluded\n\n @pytest.mark.regression(\"9624\")\n def test_autoload_with_constraints(self, modulefile_content, module_configuration):\n \"\"\"Tests the automatic loading of direct dependencies.\"\"\"\n\n module_configuration(\"autoload_with_constraints\")\n\n # Test the mpileaks that should have the autoloaded dependencies\n content = modulefile_content(\"mpileaks ^mpich2\")\n # depends-on command defined once and used 3 times\n assert len([x for x in content if \"depends-on \" in x]) == 4\n\n # Test the mpileaks that should NOT have the autoloaded dependencies\n content = modulefile_content(\"mpileaks ^mpich\")\n assert (\n len([x for x in content if \"if {![llength [info commands depends-on]]} {\" in x]) == 0\n )\n assert len([x for x in content if \" proc depends-on {args} {\" in x]) == 0\n assert len([x for x in content if \" module load {*}$args\" in x]) == 0\n assert len([x for x in content if \"depends-on \" in x]) == 0\n\n def test_modules_no_arch(self, factory, module_configuration):\n module_configuration(\"no_arch\")\n module, spec = factory(mpileaks_spec_string)\n path = module.layout.filename\n\n assert str(spec.os) not in path\n\n def test_hide_implicits(self, module_configuration, temporary_store):\n \"\"\"Tests the addition and removal of hide command in modulerc.\"\"\"\n module_configuration(\"hide_implicits\")\n\n spec = spack.concretize.concretize_one(\"mpileaks@2.3\")\n\n # mpileaks is defined as implicit, thus hide command should appear in modulerc\n writer = writer_cls(spec, \"default\", False)\n writer.write()\n assert os.path.exists(writer.layout.modulerc)\n with open(writer.layout.modulerc, encoding=\"utf-8\") as f:\n content = [line.strip() for line in f.readlines()]\n hide_implicit_mpileaks = f\"module-hide --soft --hidden-loaded {writer.layout.use_name}\"\n assert len([x for x in content if hide_implicit_mpileaks == x]) == 1\n\n # The direct dependencies are all implicit, and they should have depends-on with fixed\n # 7 character hash, even though the config is set to hash_length = 0.\n with open(writer.layout.filename, encoding=\"utf-8\") as f:\n depends_statements = [line.strip() for line in f.readlines() if \"depends-on\" in line]\n for dep in spec.dependencies(deptype=(\"link\", \"run\")):\n assert any(dep.dag_hash(7) in line for line in depends_statements)\n\n # when mpileaks becomes explicit, its file name changes (hash_length = 0), meaning an\n # extra module file is created; the old one still exists and remains hidden.\n writer = writer_cls(spec, \"default\", True)\n writer.write()\n assert os.path.exists(writer.layout.modulerc)\n with open(writer.layout.modulerc, encoding=\"utf-8\") as f:\n content = [line.strip() for line in f.readlines()]\n assert hide_implicit_mpileaks in content # old, implicit mpileaks is still hidden\n assert f\"module-hide --soft --hidden-loaded {writer.layout.use_name}\" not in content\n\n # after removing both the implicit and explicit module, the modulerc file would be empty\n # and should be removed.\n writer_cls(spec, \"default\", False).remove()\n writer_cls(spec, \"default\", True).remove()\n assert not os.path.exists(writer.layout.modulerc)\n assert not os.path.exists(writer.layout.filename)\n\n # implicit module is removed\n writer = writer_cls(spec, \"default\", False)\n writer.write()\n assert os.path.exists(writer.layout.filename)\n assert os.path.exists(writer.layout.modulerc)\n writer.remove()\n assert not os.path.exists(writer.layout.modulerc)\n assert not os.path.exists(writer.layout.filename)\n\n # three versions of mpileaks are implicit\n writer = writer_cls(spec, \"default\", False)\n writer.write(overwrite=True)\n spec_alt1 = spack.concretize.concretize_one(\"mpileaks@2.2\")\n spec_alt2 = spack.concretize.concretize_one(\"mpileaks@2.1\")\n writer_alt1 = writer_cls(spec_alt1, \"default\", False)\n writer_alt1.write(overwrite=True)\n writer_alt2 = writer_cls(spec_alt2, \"default\", False)\n writer_alt2.write(overwrite=True)\n assert os.path.exists(writer.layout.modulerc)\n with open(writer.layout.modulerc, encoding=\"utf-8\") as f:\n content = [line.strip() for line in f.readlines()]\n hide_cmd = f\"module-hide --soft --hidden-loaded {writer.layout.use_name}\"\n hide_cmd_alt1 = f\"module-hide --soft --hidden-loaded {writer_alt1.layout.use_name}\"\n hide_cmd_alt2 = f\"module-hide --soft --hidden-loaded {writer_alt2.layout.use_name}\"\n assert len([x for x in content if hide_cmd == x]) == 1\n assert len([x for x in content if hide_cmd_alt1 == x]) == 1\n assert len([x for x in content if hide_cmd_alt2 == x]) == 1\n\n # one version is removed\n writer_alt1.remove()\n assert os.path.exists(writer.layout.modulerc)\n with open(writer.layout.modulerc, encoding=\"utf-8\") as f:\n content = [line.strip() for line in f.readlines()]\n assert len([x for x in content if hide_cmd == x]) == 1\n assert len([x for x in content if hide_cmd_alt1 == x]) == 0\n assert len([x for x in content if hide_cmd_alt2 == x]) == 1\n" }, { "path": "lib/spack/spack/test/multimethod.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Test for multi_method dispatch.\"\"\"\n\nimport pytest\n\nimport spack.concretize\nimport spack.platforms\nfrom spack.multimethod import NoSuchMethodError\n\npytestmark = [\n pytest.mark.usefixtures(\"mock_packages\", \"config\"),\n pytest.mark.not_on_windows(\"Not running on windows\"),\n]\n\n\n@pytest.fixture(scope=\"module\", params=[\"multimethod\", \"multimethod-inheritor\"])\ndef pkg_name(request):\n \"\"\"Make tests run on both multimethod and multimethod-inheritor.\n\n This means we test all of our @when methods on a class that uses them\n directly, AND on a class that inherits them.\n \"\"\"\n return request.param\n\n\ndef test_no_version_match(pkg_name):\n spec = spack.concretize.concretize_one(pkg_name + \"@2.0\")\n with pytest.raises(NoSuchMethodError):\n spec.package.no_version_2()\n\n\n@pytest.mark.parametrize(\n \"constraint_str,method_name,expected_result\",\n [\n # Only one version match these constraints\n (\"@1.0\", \"no_version_2\", 1),\n (\"@3.0\", \"no_version_2\", 3),\n (\"@4.0\", \"no_version_2\", 4),\n # These constraints overlap, in which case the first match wins\n (\"@2.0\", \"version_overlap\", 1),\n (\"@5.0\", \"version_overlap\", 2),\n # These constraints are on the version of a virtual dependency\n (\"^mpich@3.0.4\", \"mpi_version\", 3),\n (\"^mpich2@1.2\", \"mpi_version\", 2),\n (\"^mpich@1.0\", \"mpi_version\", 1),\n # Undefined mpi versions\n (\"^mpich@=0.4\", \"mpi_version\", 1),\n (\"^mpich@=1.4\", \"mpi_version\", 1),\n # Constraints on compilers with a default\n (\"%gcc\", \"has_a_default\", \"gcc\"),\n (\"%clang\", \"has_a_default\", \"clang\"),\n (\"%gcc@9\", \"has_a_default\", \"default\"),\n # Constraints on dependencies\n (\"^zmpi\", \"different_by_dep\", \"zmpi\"),\n (\"^mpich\", \"different_by_dep\", \"mpich\"),\n # Constraints on virtual dependencies\n (\"^mpich2\", \"different_by_virtual_dep\", 2),\n (\"^mpich@1.0\", \"different_by_virtual_dep\", 1),\n # Multimethod with base classes\n (\"@1\", \"base_method\", \"base_method\"),\n # Boolean\n (\"\", \"boolean_true_first\", \"True\"),\n (\"\", \"boolean_false_first\", \"True\"),\n ],\n)\ndef test_multimethod_calls(\n pkg_name, constraint_str, method_name, expected_result, default_mock_concretization\n):\n s = default_mock_concretization(f\"{pkg_name}{constraint_str}\")\n msg = f\"Method {method_name} from {s} is giving a wrong result\"\n assert getattr(s.package, method_name)() == expected_result, msg\n\n\ndef test_target_match(pkg_name):\n platform = spack.platforms.host()\n targets = list(platform.targets.values())\n for target in targets[:-1]:\n s = spack.concretize.concretize_one(pkg_name + \" target=\" + target.name)\n assert s.package.different_by_target() == target.name\n\n s = spack.concretize.concretize_one(pkg_name + \" target=\" + targets[-1].name)\n if len(targets) == 1:\n assert s.package.different_by_target() == targets[-1].name\n else:\n with pytest.raises(NoSuchMethodError):\n s.package.different_by_target()\n\n\n@pytest.mark.parametrize(\n \"spec_str,method_name,expected_result\",\n [\n # This is overridden in the second case\n (\"multimethod@3\", \"base_method\", \"multimethod\"),\n (\"multimethod-inheritor@3\", \"base_method\", \"multimethod-inheritor\"),\n # Here we have a mix of inherited and overridden methods\n (\"multimethod-inheritor@1.0\", \"inherited_and_overridden\", \"inheritor@1.0\"),\n (\"multimethod-inheritor@2.0\", \"inherited_and_overridden\", \"base@2.0\"),\n (\"multimethod@1.0\", \"inherited_and_overridden\", \"base@1.0\"),\n (\"multimethod@2.0\", \"inherited_and_overridden\", \"base@2.0\"),\n # Diamond-like inheritance (even though the MRO linearize everything)\n (\"multimethod-diamond@1.0\", \"diamond_inheritance\", \"base_package\"),\n (\"multimethod-base@=1.0\", \"diamond_inheritance\", \"base_package\"),\n (\"multimethod-diamond@2.0\", \"diamond_inheritance\", \"first_parent\"),\n (\"multimethod-inheritor@2.0\", \"diamond_inheritance\", \"first_parent\"),\n (\"multimethod-diamond@=3.0\", \"diamond_inheritance\", \"second_parent\"),\n (\"multimethod-diamond-parent@=3.0\", \"diamond_inheritance\", \"second_parent\"),\n (\"multimethod-diamond@4.0\", \"diamond_inheritance\", \"subclass\"),\n ],\n)\ndef test_multimethod_calls_and_inheritance(spec_str, method_name, expected_result):\n s = spack.concretize.concretize_one(spec_str)\n assert getattr(s.package, method_name)() == expected_result\n" }, { "path": "lib/spack/spack/test/namespace_trie.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport pytest\n\nimport spack.util.naming\n\n\n@pytest.fixture()\ndef trie():\n return spack.util.naming.NamespaceTrie()\n\n\ndef test_add_single(trie):\n trie[\"foo\"] = \"bar\"\n\n assert trie.is_prefix(\"foo\")\n assert trie.has_value(\"foo\")\n assert trie[\"foo\"] == \"bar\"\n\n\ndef test_add_multiple(trie):\n trie[\"foo.bar\"] = \"baz\"\n\n assert not trie.has_value(\"foo\")\n assert trie.is_prefix(\"foo\")\n\n assert trie.is_prefix(\"foo.bar\")\n assert trie.has_value(\"foo.bar\")\n assert trie[\"foo.bar\"] == \"baz\"\n\n assert not trie.is_prefix(\"foo.bar.baz\")\n assert not trie.has_value(\"foo.bar.baz\")\n\n\ndef test_add_three(trie):\n # add a three-level namespace\n trie[\"foo.bar.baz\"] = \"quux\"\n\n assert trie.is_prefix(\"foo\")\n assert not trie.has_value(\"foo\")\n\n assert trie.is_prefix(\"foo.bar\")\n assert not trie.has_value(\"foo.bar\")\n\n assert trie.is_prefix(\"foo.bar.baz\")\n assert trie.has_value(\"foo.bar.baz\")\n assert trie[\"foo.bar.baz\"] == \"quux\"\n\n assert not trie.is_prefix(\"foo.bar.baz.quux\")\n assert not trie.has_value(\"foo.bar.baz.quux\")\n\n # Try to add a second element in a prefix namespace\n trie[\"foo.bar\"] = \"blah\"\n\n assert trie.is_prefix(\"foo\")\n assert not trie.has_value(\"foo\")\n\n assert trie.is_prefix(\"foo.bar\")\n assert trie.has_value(\"foo.bar\")\n assert trie[\"foo.bar\"] == \"blah\"\n\n assert trie.is_prefix(\"foo.bar.baz\")\n assert trie.has_value(\"foo.bar.baz\")\n assert trie[\"foo.bar.baz\"] == \"quux\"\n\n assert not trie.is_prefix(\"foo.bar.baz.quux\")\n assert not trie.has_value(\"foo.bar.baz.quux\")\n\n\ndef test_add_none_single(trie):\n trie[\"foo\"] = None\n assert trie.is_prefix(\"foo\")\n assert trie.has_value(\"foo\")\n assert trie[\"foo\"] is None\n\n assert not trie.is_prefix(\"foo.bar\")\n assert not trie.has_value(\"foo.bar\")\n\n\ndef test_add_none_multiple(trie):\n trie[\"foo.bar\"] = None\n\n assert trie.is_prefix(\"foo\")\n assert not trie.has_value(\"foo\")\n\n assert trie.is_prefix(\"foo.bar\")\n assert trie.has_value(\"foo.bar\")\n assert trie[\"foo.bar\"] is None\n\n assert not trie.is_prefix(\"foo.bar.baz\")\n assert not trie.has_value(\"foo.bar.baz\")\n" }, { "path": "lib/spack/spack/test/new_installer.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Tests for the new_installer.py module\"\"\"\n\nimport pathlib\nimport sys\nimport time\n\nimport pytest\n\nif sys.platform == \"win32\":\n pytest.skip(\"No Windows support\", allow_module_level=True)\n\nimport spack.spec\nfrom spack.new_installer import (\n OVERWRITE_GARBAGE_SUFFIX,\n JobServer,\n PackageInstaller,\n PrefixPivoter,\n _node_to_roots,\n schedule_builds,\n)\nfrom spack.test.traverse import create_dag\n\n\n@pytest.fixture\ndef existing_prefix(tmp_path: pathlib.Path) -> pathlib.Path:\n \"\"\"Creates a standard existing prefix with content.\"\"\"\n prefix = tmp_path / \"existing_prefix\"\n prefix.mkdir()\n (prefix / \"old_file\").write_text(\"old content\")\n return prefix\n\n\nclass TestPrefixPivoter:\n \"\"\"Tests for the PrefixPivoter class.\"\"\"\n\n def test_no_existing_prefix(self, tmp_path: pathlib.Path):\n \"\"\"Test installation when prefix doesn't exist yet.\"\"\"\n prefix = tmp_path / \"new_prefix\"\n\n with PrefixPivoter(str(prefix)):\n prefix.mkdir()\n (prefix / \"installed_file\").write_text(\"content\")\n\n assert prefix.exists()\n assert (prefix / \"installed_file\").read_text() == \"content\"\n\n def test_existing_prefix_success_cleans_up_old_prefix(\n self, tmp_path: pathlib.Path, existing_prefix: pathlib.Path\n ):\n \"\"\"Test that an existing prefix is moved aside, and cleaned up on success.\"\"\"\n with PrefixPivoter(str(existing_prefix)):\n assert not existing_prefix.exists()\n existing_prefix.mkdir()\n (existing_prefix / \"new_file\").write_text(\"new content\")\n\n assert existing_prefix.exists()\n assert (existing_prefix / \"new_file\").exists()\n assert not (existing_prefix / \"old_file\").exists()\n # Only the existing_prefix directory should remain\n assert len(list(tmp_path.iterdir())) == 1\n\n def test_existing_prefix_failure_restores_original_prefix(\n self, tmp_path: pathlib.Path, existing_prefix: pathlib.Path\n ):\n \"\"\"Test that the original prefix is restored when installation fails.\"\"\"\n with pytest.raises(RuntimeError, match=\"simulated failure\"):\n with PrefixPivoter(str(existing_prefix), keep_prefix=False):\n existing_prefix.mkdir()\n (existing_prefix / \"partial_file\").write_text(\"partial\")\n raise RuntimeError(\"simulated failure\")\n\n assert existing_prefix.exists()\n assert (existing_prefix / \"old_file\").read_text() == \"old content\"\n assert not (existing_prefix / \"partial_file\").exists()\n # Only the original prefix should remain\n assert len(list(tmp_path.iterdir())) == 1\n\n def test_existing_prefix_failure_no_partial_prefix_created(\n self, existing_prefix: pathlib.Path\n ):\n \"\"\"Test restoration when failure occurs before the build creates the prefix dir.\"\"\"\n with pytest.raises(RuntimeError, match=\"early failure\"):\n with PrefixPivoter(str(existing_prefix)):\n raise RuntimeError(\"early failure\")\n\n assert existing_prefix.exists()\n assert (existing_prefix / \"old_file\").read_text() == \"old content\"\n\n def test_no_existing_prefix_success(self, tmp_path: pathlib.Path):\n \"\"\"Test that a fresh install with no pre-existing prefix works fine.\"\"\"\n prefix = tmp_path / \"new_prefix\"\n with PrefixPivoter(str(prefix)):\n prefix.mkdir()\n (prefix / \"installed_file\").write_text(\"content\")\n\n assert prefix.exists()\n # Only the new_prefix directory should remain\n assert len(list(tmp_path.iterdir())) == 1\n\n def test_keep_prefix_true_with_existing_prefix_keeps_failed_install(\n self, tmp_path: pathlib.Path, existing_prefix: pathlib.Path\n ):\n \"\"\"Test that keep_prefix=True keeps the failed install and discards the backup.\"\"\"\n with pytest.raises(RuntimeError, match=\"simulated failure\"):\n with PrefixPivoter(str(existing_prefix), keep_prefix=True):\n existing_prefix.mkdir()\n (existing_prefix / \"partial_file\").write_text(\"partial content\")\n raise RuntimeError(\"simulated failure\")\n\n # The failed prefix should be kept (not the original)\n assert existing_prefix.exists()\n assert (existing_prefix / \"partial_file\").exists()\n assert not (existing_prefix / \"old_file\").exists()\n # Backup should have been removed\n assert len(list(tmp_path.iterdir())) == 1\n\n def test_keep_prefix_false_removes_failed_install(self, tmp_path: pathlib.Path):\n \"\"\"Test that keep_prefix=False removes the failed installation (no pre-existing prefix).\"\"\"\n prefix = tmp_path / \"new_prefix\"\n\n with pytest.raises(RuntimeError, match=\"simulated failure\"):\n with PrefixPivoter(str(prefix), keep_prefix=False):\n prefix.mkdir()\n (prefix / \"partial_file\").write_text(\"partial content\")\n raise RuntimeError(\"simulated failure\")\n\n # Failed prefix should be removed\n assert not prefix.exists()\n # Nothing should remain\n assert len(list(tmp_path.iterdir())) == 0\n\n def test_keep_prefix_true_no_existing_prefix(self, tmp_path: pathlib.Path):\n \"\"\"Test failure with keep_prefix=True when no prefix existed beforehand.\"\"\"\n prefix = tmp_path / \"new_prefix\"\n\n with pytest.raises(RuntimeError, match=\"simulated failure\"):\n with PrefixPivoter(str(prefix), keep_prefix=True):\n prefix.mkdir()\n (prefix / \"partial_file\").write_text(\"partial content\")\n raise RuntimeError(\"simulated failure\")\n\n # The failed prefix should be kept\n assert prefix.exists()\n assert (prefix / \"partial_file\").exists()\n # No backup should exist\n assert len(list(tmp_path.iterdir())) == 1\n\n def test_failure_no_prefix_created(self, tmp_path: pathlib.Path):\n \"\"\"Test failure when the prefix directory was never created.\"\"\"\n prefix = tmp_path / \"new_prefix\"\n\n with pytest.raises(RuntimeError, match=\"simulated failure\"):\n with PrefixPivoter(str(prefix), keep_prefix=False):\n # Do NOT create the prefix directory\n raise RuntimeError(\"simulated failure\")\n\n # Prefix should not exist\n assert not prefix.exists()\n # Nothing should remain\n assert len(list(tmp_path.iterdir())) == 0\n\n\nclass FailingPrefixPivoter(PrefixPivoter):\n \"\"\"Test subclass that can simulate filesystem failures.\"\"\"\n\n def __init__(\n self,\n prefix: str,\n keep_prefix: bool = False,\n fail_on_restore: bool = False,\n fail_on_move_garbage: bool = False,\n ):\n super().__init__(prefix, keep_prefix)\n self.fail_on_restore = fail_on_restore\n self.fail_on_move_garbage = fail_on_move_garbage\n self.restore_rename_count = 0\n\n def _rename(self, src: str, dst: str) -> None:\n if (\n self.fail_on_restore\n and self.tmp_prefix\n and src == self.tmp_prefix\n and dst == self.prefix\n ):\n self.restore_rename_count += 1\n raise OSError(\"Simulated rename failure during restore\")\n\n if self.fail_on_move_garbage and dst.endswith(OVERWRITE_GARBAGE_SUFFIX):\n raise OSError(\"Simulated rename failure moving to garbage\")\n\n super()._rename(src, dst)\n\n\nclass TestPrefixPivoterFailureRecovery:\n \"\"\"Tests for edge cases and failure recovery in PrefixPivoter.\"\"\"\n\n def test_restore_failure_leaves_backup(\n self, tmp_path: pathlib.Path, existing_prefix: pathlib.Path\n ):\n \"\"\"Test that if restoration fails, the backup is not deleted.\"\"\"\n pivoter = FailingPrefixPivoter(str(existing_prefix), fail_on_restore=True)\n\n with pytest.raises(OSError, match=\"Simulated rename failure during restore\"):\n with pivoter:\n existing_prefix.mkdir()\n (existing_prefix / \"partial_file\").write_text(\"partial\")\n raise RuntimeError(\"simulated failure\")\n\n assert pivoter.restore_rename_count > 0\n # Backup directory should still exist (plus the failed prefix)\n assert len(list(tmp_path.iterdir())) == 2\n\n def test_garbage_move_failure_leaves_backup(\n self, tmp_path: pathlib.Path, existing_prefix: pathlib.Path\n ):\n \"\"\"Test that if moving the failed install to garbage fails, the backup is preserved.\"\"\"\n pivoter = FailingPrefixPivoter(str(existing_prefix), fail_on_move_garbage=True)\n\n with pytest.raises(OSError, match=\"Simulated rename failure moving to garbage\"):\n with pivoter:\n existing_prefix.mkdir()\n (existing_prefix / \"partial_file\").write_text(\"partial\")\n raise RuntimeError(\"simulated failure\")\n\n assert (existing_prefix / \"partial_file\").exists()\n # Backup directory, failed prefix, and empty garbage directory should exist\n assert len(list(tmp_path.iterdir())) == 3\n\n\nclass TestPackageInstallerConstructor:\n \"\"\"Tests for PackageInstaller constructor, especially capacity initialization.\"\"\"\n\n def test_capacity_explicit_concurrent_packages(self, temporary_store, mock_packages):\n \"\"\"Test that capacity is set correctly when concurrent_packages is explicitly provided.\"\"\"\n spec = spack.spec.Spec(\"trivial-install-test-package\")\n spec._mark_concrete()\n assert PackageInstaller([spec.package], concurrent_packages=5).capacity == 5\n assert PackageInstaller([spec.package], concurrent_packages=1).capacity == 1\n\n def test_capacity_from_config_default_one(\n self, temporary_store, mock_packages, mutable_config\n ):\n \"\"\"Test that config value of 0 is treated as unlimited.\"\"\"\n mutable_config.set(\"config:concurrent_packages\", 0)\n spec = spack.spec.Spec(\"trivial-install-test-package\")\n spec._mark_concrete()\n assert PackageInstaller([spec.package]).capacity == sys.maxsize\n\n def test_capacity_from_config_non_zero(self, temporary_store, mock_packages, mutable_config):\n \"\"\"Test that non-0 config values are used as-is.\"\"\"\n mutable_config.set(\"config:concurrent_packages\", 1)\n spec = spack.spec.Spec(\"trivial-install-test-package\")\n spec._mark_concrete()\n assert PackageInstaller([spec.package]).capacity == 1\n\n\nclass _FakeBuildGraph:\n \"\"\"Minimal stand-in for BuildGraph in schedule_builds unit tests.\n\n Provides the two interface points that schedule_builds calls:\n - .nodes (dict: dag_hash -> Spec)\n - .enqueue_parents(dag_hash, pending_builds)\n \"\"\"\n\n def __init__(self, specs):\n self.nodes = {spec.dag_hash(): spec for spec in specs}\n\n def enqueue_parents(self, dag_hash, pending_builds):\n \"\"\"Remove dag_hash from nodes; no parents in these simple unit tests.\"\"\"\n self.nodes.pop(dag_hash, None)\n\n\nclass TestScheduleBuilds:\n \"\"\"Unit tests for the module-level schedule_builds() function.\"\"\"\n\n def _make_spec(self, name):\n \"\"\"Return a minimal concrete spec suitable for locking and DB queries.\"\"\"\n spec = spack.spec.Spec(name)\n spec._mark_concrete()\n return spec\n\n def _mark_installed(self, spec, store):\n \"\"\"Create the install directory structure and register the spec in the DB as installed.\"\"\"\n store.layout.create_install_directory(spec)\n store.db.add(spec, explicit=True)\n\n def test_not_installed_no_running_starts_build(self, temporary_store, mock_packages):\n \"\"\"A fresh spec with no running builds is added to to_start.\"\"\"\n spec = self._make_spec(\"trivial-install-test-package\")\n pending = [spec.dag_hash()]\n bg = _FakeBuildGraph([spec])\n jobserver = JobServer(num_jobs=2)\n try:\n result = schedule_builds(\n pending,\n bg,\n temporary_store.db,\n temporary_store.prefix_locker,\n overwrite=set(),\n overwrite_time=0.0,\n capacity=1,\n needs_jobserver_token=False,\n jobserver=jobserver,\n explicit=set(),\n )\n assert not result.blocked\n assert len(result.to_start) == 1\n assert result.to_start[0][0] == spec.dag_hash()\n assert not result.newly_installed\n assert not pending # removed from the pending list\n finally:\n for _, lock in result.to_start:\n lock.release_write()\n jobserver.close()\n\n def test_already_installed_yields_newly_installed(self, temporary_store, mock_packages):\n \"\"\"A spec already in the DB is returned in newly_installed, not in to_start.\"\"\"\n spec = self._make_spec(\"trivial-install-test-package\")\n self._mark_installed(spec, temporary_store)\n pending = [spec.dag_hash()]\n bg = _FakeBuildGraph([spec])\n jobserver = JobServer(num_jobs=2)\n try:\n result = schedule_builds(\n pending,\n bg,\n temporary_store.db,\n temporary_store.prefix_locker,\n overwrite=set(),\n overwrite_time=0.0,\n capacity=1,\n needs_jobserver_token=False,\n jobserver=jobserver,\n explicit=set(),\n )\n assert not result.blocked\n assert not result.to_start\n assert len(result.newly_installed) == 1\n assert result.newly_installed[0][0] == spec.dag_hash()\n assert not pending # removed from the pending list\n finally:\n for _, _, lock in result.newly_installed:\n lock.release_read()\n jobserver.close()\n\n def test_no_jobserver_token_returns_empty(self, temporary_store, mock_packages):\n \"\"\"When has_running_builds=True and no token is available, nothing is started.\"\"\"\n spec = self._make_spec(\"trivial-install-test-package\")\n pending = [spec.dag_hash()]\n bg = _FakeBuildGraph([spec])\n # num_jobs=1 writes 0 tokens to the FIFO. Only the implicit token exists.\n jobserver = JobServer(num_jobs=1)\n try:\n result = schedule_builds(\n pending,\n bg,\n temporary_store.db,\n temporary_store.prefix_locker,\n overwrite=set(),\n overwrite_time=0.0,\n capacity=2,\n needs_jobserver_token=True,\n jobserver=jobserver,\n explicit=set(),\n )\n assert not result.blocked\n assert not result.to_start\n assert not result.newly_installed\n assert len(pending) == 1\n finally:\n jobserver.close()\n\n def test_all_locked_returns_blocked(self, temporary_store, mock_packages, monkeypatch):\n \"\"\"When all pending specs are locked externally, blocked_on_locks is True.\"\"\"\n spec = self._make_spec(\"trivial-install-test-package\")\n pending = [spec.dag_hash()]\n bg = _FakeBuildGraph([spec])\n jobserver = JobServer(num_jobs=2)\n # Pre-register the lock in the prefix_locker cache, then patch try_acquire to fail.\n lock = temporary_store.prefix_locker.lock(spec)\n monkeypatch.setattr(lock, \"try_acquire_write\", lambda: False)\n monkeypatch.setattr(lock, \"try_acquire_read\", lambda: False)\n try:\n result = schedule_builds(\n pending,\n bg,\n temporary_store.db,\n temporary_store.prefix_locker,\n overwrite=set(),\n overwrite_time=0.0,\n capacity=2,\n needs_jobserver_token=False,\n jobserver=jobserver,\n explicit=set(),\n )\n assert result.blocked\n assert not result.to_start\n assert not result.newly_installed\n assert len(pending) == 1\n finally:\n jobserver.close()\n\n def test_overwrite_installed_spec_is_started(self, temporary_store, mock_packages):\n \"\"\"A spec in the overwrite set is scheduled even when already installed.\"\"\"\n spec = self._make_spec(\"trivial-install-test-package\")\n self._mark_installed(spec, temporary_store)\n pending = [spec.dag_hash()]\n bg = _FakeBuildGraph([spec])\n jobserver = JobServer(num_jobs=2)\n try:\n result = schedule_builds(\n pending,\n bg,\n temporary_store.db,\n temporary_store.prefix_locker,\n overwrite={spec.dag_hash()},\n overwrite_time=time.time() + 100,\n capacity=1,\n needs_jobserver_token=False,\n jobserver=jobserver,\n explicit=set(),\n )\n assert not result.blocked\n assert len(result.to_start) == 1\n assert result.to_start[0][0] == spec.dag_hash()\n assert not result.newly_installed\n finally:\n for _, lock in result.to_start:\n lock.release_write()\n jobserver.close()\n\n def test_mixed_locked_unlocked(self, temporary_store, mock_packages, monkeypatch):\n \"\"\"Only the unlocked spec enters to_start when one spec is externally locked.\"\"\"\n spec_a = self._make_spec(\"trivial-install-test-package\")\n spec_b = self._make_spec(\"trivial-smoke-test\")\n pending = [spec_a.dag_hash(), spec_b.dag_hash()]\n bg = _FakeBuildGraph([spec_a, spec_b])\n jobserver = JobServer(num_jobs=4)\n # Patch spec_a's lock to always fail, simulating an external write lock.\n lock_a = temporary_store.prefix_locker.lock(spec_a)\n monkeypatch.setattr(lock_a, \"try_acquire_write\", lambda: False)\n monkeypatch.setattr(lock_a, \"try_acquire_read\", lambda: False)\n try:\n result = schedule_builds(\n pending,\n bg,\n temporary_store.db,\n temporary_store.prefix_locker,\n overwrite=set(),\n overwrite_time=0.0,\n capacity=2,\n needs_jobserver_token=False,\n jobserver=jobserver,\n explicit=set(),\n )\n assert not result.blocked # spec_b was schedulable\n started_hashes = {h for h, _ in result.to_start}\n assert spec_b.dag_hash() in started_hashes\n assert spec_a.dag_hash() not in started_hashes\n assert not result.newly_installed\n finally:\n for _, lock in result.to_start:\n lock.release_write()\n jobserver.close()\n\n def test_write_locked_read_locked_installed_yields_newly_installed(\n self, temporary_store, mock_packages, monkeypatch\n ):\n \"\"\"Write lock fails but read lock succeeds and spec is installed: treated as done.\n\n Simulates the case where another process finished building and downgraded its write lock\n to a read lock. The spec should appear in newly_installed. blocked remains True because no\n write lock was obtained, preventing the jobserver from firing unnecessarily.\n \"\"\"\n spec = self._make_spec(\"trivial-install-test-package\")\n self._mark_installed(spec, temporary_store)\n pending = [spec.dag_hash()]\n bg = _FakeBuildGraph([spec])\n jobserver = JobServer(num_jobs=2)\n lock = temporary_store.prefix_locker.lock(spec)\n monkeypatch.setattr(lock, \"try_acquire_write\", lambda: False)\n try:\n result = schedule_builds(\n pending,\n bg,\n temporary_store.db,\n temporary_store.prefix_locker,\n overwrite=set(),\n overwrite_time=0.0,\n capacity=2,\n needs_jobserver_token=False,\n jobserver=jobserver,\n explicit=set(),\n )\n assert result.blocked # no write lock was obtained; jobserver should not fire\n assert not result.to_start\n assert len(result.newly_installed) == 1\n dag_hash, installed_spec, lock = result.newly_installed[0]\n assert dag_hash == spec.dag_hash()\n assert installed_spec == spec\n assert not pending # spec was removed from pending\n finally:\n for _, _, lock in result.newly_installed:\n lock.release_read()\n jobserver.close()\n\n def test_write_locked_read_locked_not_installed_still_blocked(\n self, temporary_store, mock_packages, monkeypatch\n ):\n \"\"\"Write lock fails, read lock succeeds, but spec is not in DB: retry later.\n\n Simulates the case where a concurrent process was killed mid-build. The read lock is\n released and the spec stays in pending; blocked should remain True.\n \"\"\"\n spec = self._make_spec(\"trivial-install-test-package\")\n pending = [spec.dag_hash()]\n bg = _FakeBuildGraph([spec])\n jobserver = JobServer(num_jobs=2)\n lock = temporary_store.prefix_locker.lock(spec)\n monkeypatch.setattr(lock, \"try_acquire_write\", lambda: False)\n try:\n result = schedule_builds(\n pending,\n bg,\n temporary_store.db,\n temporary_store.prefix_locker,\n overwrite=set(),\n overwrite_time=0.0,\n capacity=2,\n needs_jobserver_token=False,\n jobserver=jobserver,\n explicit=set(),\n )\n assert result.blocked\n assert not result.to_start\n assert not result.newly_installed\n assert pending == [spec.dag_hash()] # spec stays in pending for retry\n finally:\n jobserver.close()\n\n def test_overwrite_handled_by_concurrent_process(self, temporary_store, mock_packages):\n \"\"\"When a spec in overwrite was installed AFTER overwrite_time, another process did it.\"\"\"\n spec = self._make_spec(\"trivial-install-test-package\")\n self._mark_installed(spec, temporary_store) # installation_time = now()\n pending = [spec.dag_hash()]\n bg = _FakeBuildGraph([spec])\n jobserver = JobServer(num_jobs=2)\n try:\n result = schedule_builds(\n pending,\n bg,\n temporary_store.db,\n temporary_store.prefix_locker,\n overwrite={spec.dag_hash()},\n overwrite_time=0.0, # earlier than now()\n capacity=1,\n needs_jobserver_token=False,\n jobserver=jobserver,\n explicit=set(),\n )\n assert not result.blocked\n assert not result.to_start\n assert len(result.newly_installed) == 1\n assert result.newly_installed[0][0] == spec.dag_hash()\n finally:\n for _, _, lock in result.newly_installed:\n lock.release_read()\n jobserver.close()\n\n def test_installed_implicit_explicit_set_produces_db_update(\n self, temporary_store, mock_packages\n ):\n \"\"\"An installed-implicit spec in explicit set produces a DbUpdate.\"\"\"\n spec = self._make_spec(\"trivial-install-test-package\")\n temporary_store.layout.create_install_directory(spec)\n temporary_store.db.add(spec, explicit=False)\n pending = [spec.dag_hash()]\n bg = _FakeBuildGraph([spec])\n jobserver = JobServer(num_jobs=2)\n try:\n result = schedule_builds(\n pending,\n bg,\n temporary_store.db,\n temporary_store.prefix_locker,\n overwrite=set(),\n overwrite_time=0.0,\n capacity=1,\n needs_jobserver_token=False,\n jobserver=jobserver,\n explicit={spec.dag_hash()},\n )\n assert len(result.to_mark_explicit) == 1\n assert result.to_mark_explicit[0].spec is spec\n assert len(result.newly_installed) == 1\n finally:\n for _, _, lock in result.newly_installed:\n lock.release_read()\n jobserver.close()\n\n\ndef test_nodes_to_roots():\n \"\"\"Independent roots don't reach each other's exclusive nodes.\"\"\"\n # A - B and C - D are disconnected graphs, A, B and C are \"roots\".\n specs = create_dag(nodes=[\"A\", \"B\", \"C\", \"D\"], edges=[(\"A\", \"B\", \"all\"), (\"C\", \"D\", \"all\")])\n a, b, c, d = specs[\"A\"], specs[\"B\"], specs[\"C\"], specs[\"D\"]\n node_to_roots = _node_to_roots([a, b, c])\n assert node_to_roots[a.dag_hash()] == frozenset([a.dag_hash()])\n assert node_to_roots[b.dag_hash()] == frozenset([a.dag_hash(), b.dag_hash()])\n assert node_to_roots[c.dag_hash()] == frozenset([c.dag_hash()])\n assert node_to_roots[d.dag_hash()] == frozenset([c.dag_hash()])\n\n\ndef test_nodes_to_roots_shared_dependency():\n \"\"\"A dependency shared by two roots is attributed to both.\"\"\"\n specs = create_dag(nodes=[\"A\", \"B\", \"C\"], edges=[(\"A\", \"C\", \"all\"), (\"B\", \"C\", \"all\")])\n a, b, c = specs[\"A\"], specs[\"B\"], specs[\"C\"]\n node_to_roots = _node_to_roots([a, b])\n assert node_to_roots[a.dag_hash()] == frozenset([a.dag_hash()])\n assert node_to_roots[b.dag_hash()] == frozenset([b.dag_hash()])\n assert node_to_roots[c.dag_hash()] == frozenset([a.dag_hash(), b.dag_hash()])\n" }, { "path": "lib/spack/spack/test/oci/image.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport pytest\n\nfrom spack.oci.image import Digest, ImageReference\n\n\n@pytest.mark.parametrize(\n \"image_ref, expected\",\n [\n (\n f\"example.com:1234/a/b/c:tag@sha256:{'a' * 64}\",\n (\"example.com:1234\", \"a/b/c\", \"tag\", Digest.from_sha256(\"a\" * 64)),\n ),\n (\"example.com:1234/a/b/c:tag\", (\"example.com:1234\", \"a/b/c\", \"tag\", None)),\n (\"example.com:1234/a/b/c\", (\"example.com:1234\", \"a/b/c\", \"latest\", None)),\n (\n f\"example.com:1234/a/b/c@sha256:{'a' * 64}\",\n (\"example.com:1234\", \"a/b/c\", \"latest\", Digest.from_sha256(\"a\" * 64)),\n ),\n # ipv4\n (\"1.2.3.4:1234/a/b/c:tag\", (\"1.2.3.4:1234\", \"a/b/c\", \"tag\", None)),\n # ipv6\n (\"[2001:db8::1]:1234/a/b/c:tag\", (\"[2001:db8::1]:1234\", \"a/b/c\", \"tag\", None)),\n # Follow docker rules for parsing\n (\"ubuntu:22.04\", (\"index.docker.io\", \"library/ubuntu\", \"22.04\", None)),\n (\"myname/myimage:abc\", (\"index.docker.io\", \"myname/myimage\", \"abc\", None)),\n (\"myname:1234/myimage:abc\", (\"myname:1234\", \"myimage\", \"abc\", None)),\n (\"localhost/myimage:abc\", (\"localhost\", \"myimage\", \"abc\", None)),\n (\"localhost:1234/myimage:abc\", (\"localhost:1234\", \"myimage\", \"abc\", None)),\n (\n \"example.com/UPPERCASE/lowercase:AbC\",\n (\"example.com\", \"uppercase/lowercase\", \"AbC\", None),\n ),\n ],\n)\ndef test_name_parsing(image_ref, expected):\n x = ImageReference.from_string(image_ref)\n assert (x.domain, x.name, x.tag, x.digest) == expected\n\n\n@pytest.mark.parametrize(\n \"image_ref\",\n [\n # wrong order of tag and sha\n f\"example.com:1234/a/b/c@sha256:{'a' * 64}:tag\",\n # double tag\n \"example.com:1234/a/b/c:tag:tag\",\n # empty tag\n \"example.com:1234/a/b/c:\",\n # empty digest\n \"example.com:1234/a/b/c@sha256:\",\n # unsupported digest algorithm\n f\"example.com:1234/a/b/c@sha512:{'a' * 128}\",\n # invalid digest length\n f\"example.com:1234/a/b/c@sha256:{'a' * 63}\",\n # whitespace\n \"example.com:1234/a/b/c :tag\",\n \"example.com:1234/a/b/c: tag\",\n \"example.com:1234/a/b/c:tag \",\n \" example.com:1234/a/b/c:tag\",\n # broken ipv4\n \"1.2..3:1234/a/b/c:tag\",\n ],\n)\ndef test_parsing_failure(image_ref):\n with pytest.raises(ValueError):\n ImageReference.from_string(image_ref)\n\n\ndef test_digest():\n valid_digest = \"1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef\"\n\n # Test string roundtrip\n assert str(Digest.from_string(f\"sha256:{valid_digest}\")) == f\"sha256:{valid_digest}\"\n\n # Invalid digest length\n with pytest.raises(ValueError):\n Digest.from_string(\"sha256:abcdef\")\n\n # Missing algorithm\n with pytest.raises(ValueError):\n Digest.from_string(valid_digest)\n\n\ndef test_url_with_scheme():\n \"\"\"Test that scheme=http translates to http:// URLs\"\"\"\n http = ImageReference.from_string(\"localhost:1234/myimage:abc\", scheme=\"http\")\n https = ImageReference.from_string(\"localhost:1234/myimage:abc\", scheme=\"https\")\n default = ImageReference.from_string(\"localhost:1234/myimage:abc\")\n\n assert http != https\n assert https == default\n\n assert http.manifest_url() == \"http://localhost:1234/v2/myimage/manifests/abc\"\n assert https.manifest_url() == \"https://localhost:1234/v2/myimage/manifests/abc\"\n assert default.manifest_url() == \"https://localhost:1234/v2/myimage/manifests/abc\"\n" }, { "path": "lib/spack/spack/test/oci/integration_test.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n# These are slow integration tests that do concretization, install, tarballing\n# and compression. They still use an in-memory OCI registry.\n\nimport hashlib\nimport json\nimport os\nimport pathlib\nimport re\nimport urllib.error\nfrom contextlib import contextmanager\n\nimport pytest\n\nimport spack.binary_distribution\nimport spack.database\nimport spack.deptypes as dt\nimport spack.environment as ev\nimport spack.error\nimport spack.oci.opener\nimport spack.spec\nimport spack.traverse\nfrom spack.main import SpackCommand\nfrom spack.oci.image import Digest, ImageReference, default_config, default_manifest\nfrom spack.oci.oci import blob_exists, get_manifest_and_config, upload_blob, upload_manifest\nfrom spack.test.oci.mock_registry import DummyServer, InMemoryOCIRegistry, create_opener\nfrom spack.util.archive import gzip_compressed_tarfile\n\nbuildcache = SpackCommand(\"buildcache\")\nmirror = SpackCommand(\"mirror\")\nenv = SpackCommand(\"env\")\ninstall = SpackCommand(\"install\")\n\n\n@contextmanager\ndef oci_servers(*servers: DummyServer):\n old_opener = spack.oci.opener.urlopen\n spack.oci.opener.urlopen = create_opener(*servers).open\n yield\n spack.oci.opener.urlopen = old_opener\n\n\ndef test_buildcache_push_command(mutable_database):\n with oci_servers(InMemoryOCIRegistry(\"example.com\")):\n mirror(\"add\", \"oci-test\", \"oci://example.com/image\")\n\n # Push the package(s) to the OCI registry\n buildcache(\"push\", \"--update-index\", \"oci-test\", \"mpileaks^mpich\")\n\n # Remove mpileaks from the database\n matches = mutable_database.query_local(\"mpileaks^mpich\")\n assert len(matches) == 1\n spec = matches[0]\n spec.package.do_uninstall()\n\n # Reinstall mpileaks from the OCI registry\n buildcache(\"install\", \"--unsigned\", \"mpileaks^mpich\")\n\n # Now it should be installed again\n assert spec.installed\n\n # And let's check that the bin/mpileaks executable is there\n assert os.path.exists(os.path.join(spec.prefix, \"bin\", \"mpileaks\"))\n\n\ndef test_buildcache_tag(install_mockery, mock_fetch, mutable_mock_env_path):\n \"\"\"Tests whether we can create an OCI image from a full environment with multiple roots.\"\"\"\n env(\"create\", \"test\")\n with ev.read(\"test\"):\n install(\"--fake\", \"--add\", \"libelf\")\n install(\"--fake\", \"--add\", \"trivial-install-test-package\")\n\n registry = InMemoryOCIRegistry(\"example.com\")\n\n with oci_servers(registry):\n mirror(\"add\", \"oci-test\", \"oci://example.com/image\")\n\n with ev.read(\"test\"):\n buildcache(\"push\", \"--tag\", \"full_env\", \"oci-test\")\n\n name = ImageReference.from_string(\"example.com/image:full_env\")\n\n with ev.read(\"test\") as e:\n specs = [\n x\n for x in spack.traverse.traverse_nodes(\n e.concrete_roots(), deptype=dt.LINK | dt.RUN\n )\n if not x.external\n ]\n\n manifest, config = get_manifest_and_config(name)\n\n # without a base image, we should have one layer per spec\n assert len(manifest[\"layers\"]) == len(specs)\n\n # Now create yet another tag, but with just a single selected spec as root. This should\n # also test the case where Spack doesn't have to upload any binaries, it just has to create\n # a new tag.\n libelf = next(s for s in specs if s.name == \"libelf\")\n with ev.read(\"test\"):\n # Get libelf spec\n buildcache(\"push\", \"--tag\", \"single_spec\", \"oci-test\", libelf.format(\"libelf{/hash}\"))\n\n name = ImageReference.from_string(\"example.com/image:single_spec\")\n manifest, config = get_manifest_and_config(name)\n assert len(manifest[\"layers\"]) == len(\n [x for x in libelf.traverse(deptype=dt.LINK | dt.RUN) if not x.external]\n )\n\n\ndef test_buildcache_push_with_base_image_command(mutable_database, tmp_path: pathlib.Path):\n \"\"\"Test that we can push a package with a base image to an OCI registry.\n\n This test is a bit involved, cause we have to create a small base image.\"\"\"\n\n registry_src = InMemoryOCIRegistry(\"src.example.com\")\n registry_dst = InMemoryOCIRegistry(\"dst.example.com\")\n\n base_image = ImageReference.from_string(\"src.example.com/my-base-image:latest\")\n\n with oci_servers(registry_src, registry_dst):\n mirror(\"add\", \"oci-test\", \"oci://dst.example.com/image\")\n\n # TODO: simplify creation of images...\n # We create a rootfs.tar.gz, a config file and a manifest file,\n # and upload those.\n\n config, manifest = default_config(architecture=\"amd64\", os=\"linux\"), default_manifest()\n\n # Create a small rootfs\n rootfs = tmp_path / \"rootfs\"\n rootfs.mkdir()\n (rootfs / \"bin\").mkdir()\n (rootfs / \"bin\" / \"sh\").touch()\n\n # Create a tarball of it.\n tarball = tmp_path / \"base.tar.gz\"\n with gzip_compressed_tarfile(str(tarball)) as (tar, tar_gz_checksum, tar_checksum):\n tar.add(str(rootfs), arcname=\".\")\n\n tar_gz_digest = Digest.from_sha256(tar_gz_checksum.hexdigest())\n tar_digest = Digest.from_sha256(tar_checksum.hexdigest())\n\n # Save the config file\n config[\"rootfs\"][\"diff_ids\"] = [str(tar_digest)]\n config_file = tmp_path / \"config.json\"\n config_file.write_text(json.dumps(config), encoding=\"utf-8\")\n\n config_digest = Digest.from_sha256(hashlib.sha256(config_file.read_bytes()).hexdigest())\n\n # Register the layer in the manifest\n manifest[\"layers\"].append(\n {\n \"mediaType\": \"application/vnd.oci.image.layer.v1.tar+gzip\",\n \"digest\": str(tar_gz_digest),\n \"size\": tarball.stat().st_size,\n }\n )\n manifest[\"config\"][\"digest\"] = str(config_digest)\n manifest[\"config\"][\"size\"] = config_file.stat().st_size\n\n # Upload the layer and config file\n upload_blob(base_image, str(tarball), tar_gz_digest)\n upload_blob(base_image, str(config_file), config_digest)\n\n # Upload the manifest\n upload_manifest(base_image, manifest)\n\n # END TODO\n\n # Finally... use it as a base image\n buildcache(\"push\", \"--base-image\", str(base_image), \"oci-test\", \"mpileaks^mpich\")\n\n # Figure out what tag was produced\n tag = next(tag for _, tag in registry_dst.manifests.keys() if tag.startswith(\"mpileaks-\"))\n assert tag is not None\n\n # Fetch the manifest and config\n dst_image = ImageReference.from_string(f\"dst.example.com/image:{tag}\")\n retrieved_manifest, retrieved_config = get_manifest_and_config(dst_image)\n\n # Check that the media type is OCI\n assert retrieved_manifest[\"mediaType\"] == \"application/vnd.oci.image.manifest.v1+json\"\n assert (\n retrieved_manifest[\"config\"][\"mediaType\"] == \"application/vnd.oci.image.config.v1+json\"\n )\n\n # Check that the base image layer is first.\n assert retrieved_manifest[\"layers\"][0][\"digest\"] == str(tar_gz_digest)\n assert retrieved_config[\"rootfs\"][\"diff_ids\"][0] == str(tar_digest)\n\n # And also check that we have layers for each link-run dependency\n matches = mutable_database.query_local(\"mpileaks^mpich\")\n assert len(matches) == 1\n spec = matches[0]\n\n num_runtime_deps = len(list(spec.traverse(root=True, deptype=(\"link\", \"run\"))))\n\n # One base layer + num_runtime_deps\n assert len(retrieved_manifest[\"layers\"]) == 1 + num_runtime_deps\n\n # And verify that all layers including the base layer are present\n for layer in retrieved_manifest[\"layers\"]:\n assert blob_exists(dst_image, digest=Digest.from_string(layer[\"digest\"]))\n assert layer[\"mediaType\"] == \"application/vnd.oci.image.layer.v1.tar+gzip\"\n\n\ndef test_uploading_with_base_image_in_docker_image_manifest_v2_format(\n tmp_path: pathlib.Path, mutable_database\n):\n \"\"\"If the base image uses an old manifest schema, Spack should also use that.\n That is necessary for container images to work with Apptainer, which is rather strict about\n mismatching manifest/layer types.\"\"\"\n\n registry_src = InMemoryOCIRegistry(\"src.example.com\")\n registry_dst = InMemoryOCIRegistry(\"dst.example.com\")\n\n base_image = ImageReference.from_string(\"src.example.com/my-base-image:latest\")\n\n with oci_servers(registry_src, registry_dst):\n mirror(\"add\", \"oci-test\", \"oci://dst.example.com/image\")\n\n # Create a dummy base image (blob, config, manifest) in registry A in the Docker Image\n # Manifest V2 format.\n rootfs = tmp_path / \"rootfs\"\n (rootfs / \"bin\").mkdir(parents=True)\n (rootfs / \"bin\" / \"sh\").write_text(\"hello world\")\n tarball = tmp_path / \"base.tar.gz\"\n with gzip_compressed_tarfile(str(tarball)) as (tar, tar_gz_checksum, tar_checksum):\n tar.add(rootfs, arcname=\".\")\n tar_gz_digest = Digest.from_sha256(tar_gz_checksum.hexdigest())\n tar_digest = Digest.from_sha256(tar_checksum.hexdigest())\n upload_blob(base_image, str(tarball), tar_gz_digest)\n config = {\n \"created\": \"2015-10-31T22:22:56.015925234Z\",\n \"author\": \"Foo \",\n \"architecture\": \"amd64\",\n \"os\": \"linux\",\n \"config\": {\n \"User\": \"foo\",\n \"Memory\": 2048,\n \"MemorySwap\": 4096,\n \"CpuShares\": 8,\n \"ExposedPorts\": {\"8080/tcp\": {}},\n \"Env\": [\"PATH=/usr/bin:/bin\"],\n \"Entrypoint\": [\"/bin/sh\"],\n \"Cmd\": [\"-c\", \"'echo hello world'\"],\n \"Volumes\": {\"/x\": {}},\n \"WorkingDir\": \"/\",\n },\n \"rootfs\": {\"diff_ids\": [str(tar_digest)], \"type\": \"layers\"},\n \"history\": [\n {\n \"created\": \"2015-10-31T22:22:54.690851953Z\",\n \"created_by\": \"/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /\", # noqa: E501\n }\n ],\n }\n config_file = tmp_path / \"config.json\"\n config_file.write_text(json.dumps(config))\n config_digest = Digest.from_sha256(hashlib.sha256(config_file.read_bytes()).hexdigest())\n upload_blob(base_image, str(config_file), config_digest)\n manifest = {\n \"schemaVersion\": 2,\n \"mediaType\": \"application/vnd.docker.distribution.manifest.v2+json\",\n \"config\": {\n \"mediaType\": \"application/vnd.docker.container.image.v1+json\",\n \"size\": config_file.stat().st_size,\n \"digest\": str(config_digest),\n },\n \"layers\": [\n {\n \"mediaType\": \"application/vnd.docker.image.rootfs.diff.tar.gzip\",\n \"size\": tarball.stat().st_size,\n \"digest\": str(tar_gz_digest),\n }\n ],\n }\n upload_manifest(base_image, manifest)\n\n # Finally upload some package to registry B with registry A's image as base\n buildcache(\"push\", \"--base-image\", str(base_image), \"oci-test\", \"mpileaks^mpich\")\n\n # Should have some manifests uploaded to registry B now.\n assert registry_dst.manifests\n\n # Verify that all manifest are in the Docker Image Manifest V2 format, not OCI.\n # And also check that we're not using annotations, which is an OCI-only \"feature\".\n for m in registry_dst.manifests.values():\n assert m[\"mediaType\"] == \"application/vnd.docker.distribution.manifest.v2+json\"\n assert m[\"config\"][\"mediaType\"] == \"application/vnd.docker.container.image.v1+json\"\n for layer in m[\"layers\"]:\n assert layer[\"mediaType\"] == \"application/vnd.docker.image.rootfs.diff.tar.gzip\"\n assert \"annotations\" not in m\n\n\ndef test_best_effort_upload(mutable_database: spack.database.Database, monkeypatch):\n \"\"\"Failure to upload a blob or manifest should not prevent others from being uploaded -- it\n should be a best-effort operation. If any runtime dep fails to upload, it results in a missing\n layer for dependents. But we do still create manifests for dependents, so that the build cache\n is maximally useful. (The downside is that container images are not runnable).\"\"\"\n\n _push_blob = spack.binary_distribution._oci_push_pkg_blob\n _push_manifest = spack.binary_distribution._oci_put_manifest\n\n def push_blob(image_ref, spec, tmpdir):\n # fail to upload the blob of mpich\n if spec.name == \"mpich\":\n raise Exception(\"Blob Server Error\")\n return _push_blob(image_ref, spec, tmpdir)\n\n def put_manifest(base_images, checksums, image_ref, tmpdir, extra_config, annotations, *specs):\n # fail to upload the manifest of libdwarf\n if \"libdwarf\" in (s.name for s in specs):\n raise Exception(\"Manifest Server Error\")\n return _push_manifest(\n base_images, checksums, image_ref, tmpdir, extra_config, annotations, *specs\n )\n\n monkeypatch.setattr(spack.binary_distribution, \"_oci_push_pkg_blob\", push_blob)\n monkeypatch.setattr(spack.binary_distribution, \"_oci_put_manifest\", put_manifest)\n\n mirror(\"add\", \"oci-test\", \"oci://example.com/image\")\n registry = InMemoryOCIRegistry(\"example.com\")\n image = ImageReference.from_string(\"example.com/image\")\n\n with oci_servers(registry):\n with pytest.raises(spack.error.SpackError, match=\"The following 2 errors occurred\") as e:\n buildcache(\"push\", \"--update-index\", \"oci-test\", \"mpileaks^mpich\")\n\n # mpich's blob failed to upload and libdwarf's manifest failed to upload\n assert re.search(\"mpich.+: Exception: Blob Server Error\", e.value)\n assert re.search(\"libdwarf.+: Exception: Manifest Server Error\", e.value)\n\n mpileaks: spack.spec.Spec = mutable_database.query_local(\"mpileaks^mpich\")[0]\n\n without_manifest = (\"mpich\", \"libdwarf\")\n\n # Verify that manifests of mpich/libdwarf are missing due to upload failure.\n for name in without_manifest:\n tagged_img = image.with_tag(spack.binary_distribution._oci_default_tag(mpileaks[name]))\n with pytest.raises(urllib.error.HTTPError, match=\"404\"):\n get_manifest_and_config(tagged_img)\n\n # Collect the layer digests of successfully uploaded packages. Every package should refer\n # to its own tarballs and those of its runtime deps that were uploaded.\n pkg_to_all_digests = {}\n pkg_to_own_digest = {}\n for s in mpileaks.traverse():\n if s.name in without_manifest:\n continue\n\n if s.external:\n continue\n\n # This should not raise a 404.\n manifest, _ = get_manifest_and_config(\n image.with_tag(spack.binary_distribution._oci_default_tag(s))\n )\n\n # Collect layer digests\n pkg_to_all_digests[s.name] = {layer[\"digest\"] for layer in manifest[\"layers\"]}\n pkg_to_own_digest[s.name] = manifest[\"layers\"][-1][\"digest\"]\n\n # Verify that all packages reference blobs of their runtime deps that uploaded fine.\n for s in mpileaks.traverse():\n if s.name in without_manifest:\n continue\n\n if s.external:\n continue\n\n expected_digests = {\n pkg_to_own_digest[t.name]\n for t in s.traverse(deptype=(\"link\", \"run\"), root=True)\n if t.name not in without_manifest\n }\n\n # Test with issubset, cause we don't have the blob of libdwarf as it has no manifest.\n assert expected_digests and expected_digests.issubset(pkg_to_all_digests[s.name])\n" }, { "path": "lib/spack/spack/test/oci/mock_registry.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\nimport base64\nimport hashlib\nimport io\nimport json\nimport re\nimport urllib.error\nimport urllib.parse\nimport urllib.request\nimport uuid\nfrom typing import Callable, Dict, List, Optional, Pattern, Tuple\nfrom urllib.request import Request\n\nimport spack.oci.oci\nfrom spack.oci.image import Digest\nfrom spack.oci.opener import OCIAuthHandler\nfrom spack.test.conftest import MockHTTPResponse\n\n\nclass MiddlewareError(Exception):\n \"\"\"Thrown in a handler to return a response early.\"\"\"\n\n def __init__(self, response: MockHTTPResponse):\n self.response = response\n\n\nclass Router:\n \"\"\"This class is a small router for requests to the OCI registry.\n\n It is used to dispatch requests to a handler, and middleware can be\n used to transform requests, as well as return responses early\n (e.g. for authentication).\"\"\"\n\n def __init__(self) -> None:\n self.routes: List[Tuple[str, Pattern, Callable]] = []\n self.middleware: List[Callable[[Request], Request]] = []\n\n def handle(self, req: Request) -> MockHTTPResponse:\n \"\"\"Dispatch a request to a handler.\"\"\"\n result = urllib.parse.urlparse(req.full_url)\n\n # Apply middleware\n try:\n for handler in self.middleware:\n req = handler(req)\n except MiddlewareError as e:\n return e.response\n\n for method, path_regex, handler in self.routes:\n if method != req.get_method():\n continue\n match = re.fullmatch(path_regex, result.path)\n if not match:\n continue\n\n return handler(req, **match.groupdict())\n\n return MockHTTPResponse(404, \"Not found\")\n\n def register(self, method, path: str, handler: Callable):\n self.routes.append((method, re.compile(path), handler))\n\n def add_middleware(self, handler: Callable[[Request], Request]):\n self.middleware.append(handler)\n\n\nclass DummyServer:\n def __init__(self, domain: str) -> None:\n # The domain of the server, e.g. \"registry.example.com\"\n self.domain = domain\n\n # List of (method, url) tuples\n self.requests: List[Tuple[str, str]] = []\n\n # Dispatches requests to handlers\n self.router = Router()\n\n # Always install a request logger\n self.router.add_middleware(self.log_request)\n\n def handle(self, req: Request) -> MockHTTPResponse:\n return self.router.handle(req)\n\n def log_request(self, req: Request):\n path = urllib.parse.urlparse(req.full_url).path\n self.requests.append((req.get_method(), path))\n return req\n\n def clear_log(self):\n self.requests = []\n\n\nclass InMemoryOCIRegistry(DummyServer):\n \"\"\"This implements the basic OCI registry API, but in memory.\n\n It supports two types of blob uploads:\n 1. POST + PUT: the client first starts a session with POST, then does a large PUT request\n 2. POST: the client does a single POST request with the whole blob\n\n Option 2 is not supported by all registries, so we allow to disable it,\n with allow_single_post=False.\n\n A third option is to use the chunked upload, but this is not implemented here, because\n it's typically a major performance hit in upload speed, so we're not using it in Spack.\"\"\"\n\n def __init__(\n self, domain: str, allow_single_post: bool = True, tags_per_page: int = 100\n ) -> None:\n super().__init__(domain)\n self.router.register(\"GET\", r\"/v2/\", self.index)\n self.router.register(\"HEAD\", r\"/v2/(?P.+)/blobs/(?P.+)\", self.head_blob)\n self.router.register(\"POST\", r\"/v2/(?P.+)/blobs/uploads/\", self.start_session)\n self.router.register(\"PUT\", r\"/upload\", self.put_session)\n self.router.register(\"PUT\", r\"/v2/(?P.+)/manifests/(?P.+)\", self.put_manifest)\n self.router.register(\"GET\", r\"/v2/(?P.+)/manifests/(?P.+)\", self.get_manifest)\n self.router.register(\"GET\", r\"/v2/(?P.+)/blobs/(?P.+)\", self.get_blob)\n self.router.register(\"GET\", r\"/v2/(?P.+)/tags/list\", self.list_tags)\n\n # If True, allow single POST upload, not all registries support this\n self.allow_single_post = allow_single_post\n\n # How many tags are returned in a single request\n self.tags_per_page = tags_per_page\n\n # Used for POST + PUT upload. This is a map from session ID to image name\n self.sessions: Dict[str, str] = {}\n\n # Set of sha256:... digests that are known to the registry\n self.blobs: Dict[str, bytes] = {}\n\n # Map from (name, tag) to manifest\n self.manifests: Dict[Tuple[str, str], dict] = {}\n\n def index(self, req: Request):\n return MockHTTPResponse.with_json(200, \"OK\", body={})\n\n def head_blob(self, req: Request, name: str, digest: str):\n if digest in self.blobs:\n return MockHTTPResponse(200, \"OK\", headers={\"Content-Length\": \"1234\"})\n return MockHTTPResponse(404, \"Not found\")\n\n def get_blob(self, req: Request, name: str, digest: str):\n if digest in self.blobs:\n return MockHTTPResponse(200, \"OK\", body=io.BytesIO(self.blobs[digest]))\n return MockHTTPResponse(404, \"Not found\")\n\n def start_session(self, req: Request, name: str):\n id = str(uuid.uuid4())\n self.sessions[id] = name\n\n # Check if digest is present (single monolithic upload)\n result = urllib.parse.urlparse(req.full_url)\n query = urllib.parse.parse_qs(result.query)\n\n if self.allow_single_post and \"digest\" in query:\n return self.handle_upload(\n req, name=name, digest=Digest.from_string(query[\"digest\"][0])\n )\n\n return MockHTTPResponse(202, \"Accepted\", headers={\"Location\": f\"/upload?uuid={id}\"})\n\n def put_session(self, req: Request):\n # Do the upload.\n result = urllib.parse.urlparse(req.full_url)\n query = urllib.parse.parse_qs(result.query)\n\n # uuid param should be preserved, and digest should be present\n assert \"uuid\" in query and len(query[\"uuid\"]) == 1\n assert \"digest\" in query and len(query[\"digest\"]) == 1\n\n id = query[\"uuid\"][0]\n assert id in self.sessions\n\n name, digest = self.sessions[id], Digest.from_string(query[\"digest\"][0])\n\n response = self.handle_upload(req, name=name, digest=digest)\n\n # End the session\n del self.sessions[id]\n\n return response\n\n def put_manifest(self, req: Request, name: str, ref: str):\n # In requests, Python runs header.capitalize().\n content_type = req.get_header(\"Content-type\")\n assert content_type in spack.oci.oci.all_content_type\n\n index_or_manifest = json.loads(self._require_data(req))\n\n # Verify that we have all blobs (layers for manifest, manifests for index)\n if content_type in spack.oci.oci.manifest_content_type:\n for layer in index_or_manifest[\"layers\"]:\n assert layer[\"digest\"] in self.blobs, \"Missing blob while uploading manifest\"\n\n else:\n for manifest in index_or_manifest[\"manifests\"]:\n assert (name, manifest[\"digest\"]) in self.manifests, (\n \"Missing manifest while uploading index\"\n )\n\n self.manifests[(name, ref)] = index_or_manifest\n\n return MockHTTPResponse(\n 201, \"Created\", headers={\"Location\": f\"/v2/{name}/manifests/{ref}\"}\n )\n\n def get_manifest(self, req: Request, name: str, ref: str):\n if (name, ref) not in self.manifests:\n return MockHTTPResponse(404, \"Not found\")\n\n manifest_or_index = self.manifests[(name, ref)]\n\n return MockHTTPResponse.with_json(\n 200,\n \"OK\",\n headers={\"Content-type\": manifest_or_index[\"mediaType\"]},\n body=manifest_or_index,\n )\n\n def _require_data(self, req: Request) -> bytes:\n \"\"\"Extract request.data, it's type remains a mystery\"\"\"\n assert req.data is not None\n\n if hasattr(req.data, \"read\"):\n return req.data.read()\n elif isinstance(req.data, bytes):\n return req.data\n\n raise ValueError(\"req.data should be bytes or have a read() method\")\n\n def handle_upload(self, req: Request, name: str, digest: Digest):\n \"\"\"Verify the digest, save the blob, return created status\"\"\"\n data = self._require_data(req)\n assert hashlib.sha256(data).hexdigest() == digest.digest\n self.blobs[str(digest)] = data\n return MockHTTPResponse(201, \"Created\", headers={\"Location\": f\"/v2/{name}/blobs/{digest}\"})\n\n def list_tags(self, req: Request, name: str):\n # Paginate using Link headers, this was added to the spec in the following commit:\n # https://github.com/opencontainers/distribution-spec/commit/2ed79d930ecec11dd755dc8190409a3b10f01ca9\n\n # List all tags, exclude digests.\n all_tags = sorted(\n _tag for _name, _tag in self.manifests.keys() if _name == name and \":\" not in _tag\n )\n\n query = urllib.parse.parse_qs(urllib.parse.urlparse(req.full_url).query)\n\n n = int(query[\"n\"][0]) if \"n\" in query else self.tags_per_page\n\n if \"last\" in query:\n try:\n offset = all_tags.index(query[\"last\"][0]) + 1\n except ValueError:\n return MockHTTPResponse(404, \"Not found\")\n else:\n offset = 0\n\n tags = all_tags[offset : offset + n]\n\n if offset + n < len(all_tags):\n headers = {\"Link\": f'; rel=\"next\"'}\n else:\n headers = None\n\n return MockHTTPResponse.with_json(200, \"OK\", headers=headers, body={\"tags\": tags})\n\n\nclass DummyServerUrllibHandler(urllib.request.BaseHandler):\n \"\"\"Glue between urllib and DummyServer, routing requests to\n the correct mock server for a given domain.\"\"\"\n\n def __init__(self) -> None:\n self.servers: Dict[str, DummyServer] = {}\n\n def add_server(self, domain: str, api: DummyServer):\n self.servers[domain] = api\n return self\n\n def https_open(self, req: Request):\n domain = urllib.parse.urlparse(req.full_url).netloc\n\n if domain not in self.servers:\n return MockHTTPResponse(404, \"Not found\")\n\n return self.servers[domain].handle(req)\n\n\nclass InMemoryOCIRegistryWithBearerAuth(InMemoryOCIRegistry):\n \"\"\"This is another in-memory OCI registry requiring bearer token authentication.\"\"\"\n\n def __init__(\n self, domain, token: Optional[str], realm: str, allow_single_post: bool = True\n ) -> None:\n super().__init__(domain, allow_single_post)\n self.token = token # token to accept\n self.realm = realm # url to the authorization server\n self.router.add_middleware(self.authenticate)\n\n def authenticate(self, req: Request):\n # Any request needs an Authorization header\n authorization = req.get_header(\"Authorization\")\n\n if authorization is None:\n raise MiddlewareError(self.unauthorized())\n\n # Ensure that the token is correct\n assert authorization.startswith(\"Bearer \")\n token = authorization[7:]\n\n if token != self.token:\n raise MiddlewareError(self.unauthorized())\n\n return req\n\n def unauthorized(self):\n return MockHTTPResponse(\n 401,\n \"Unauthorized\",\n {\n \"www-authenticate\": f'Bearer realm=\"{self.realm}\",'\n f'service=\"{self.domain}\",'\n 'scope=\"repository:spack-registry:pull,push\"'\n },\n )\n\n\nclass InMemoryOCIRegistryWithBasicAuth(InMemoryOCIRegistry):\n \"\"\"This is another in-memory OCI registry requiring basic authentication.\"\"\"\n\n def __init__(\n self, domain, username: str, password: str, realm: str, allow_single_post: bool = True\n ) -> None:\n super().__init__(domain, allow_single_post)\n self.username = username\n self.password = password\n self.realm = realm\n self.router.add_middleware(self.authenticate)\n\n def authenticate(self, req: Request):\n # Any request needs an Authorization header\n authorization = req.get_header(\"Authorization\")\n\n if authorization is None:\n raise MiddlewareError(self.unauthorized())\n\n # Ensure that the username and password are correct\n assert authorization.startswith(\"Basic \")\n auth = base64.b64decode(authorization[6:]).decode(\"utf-8\")\n username, password = auth.split(\":\", 1)\n\n if username != self.username or password != self.password:\n raise MiddlewareError(self.unauthorized())\n\n return req\n\n def unauthorized(self):\n return MockHTTPResponse(\n 401, \"Unauthorized\", {\"www-authenticate\": f'Basic realm=\"{self.realm}\"'}\n )\n\n\nclass MockBearerTokenServer(DummyServer):\n \"\"\"Simulates a basic server that hands out bearer tokens\n at the /login endpoint for the following services:\n public.example.com, which doesn't require Basic Auth\n private.example.com, which requires Basic Auth, with user:pass\n \"\"\"\n\n def __init__(self, domain: str) -> None:\n super().__init__(domain)\n self.router.register(\"GET\", \"/login\", self.login)\n\n def login(self, req: Request):\n url = urllib.parse.urlparse(req.full_url)\n query_params = urllib.parse.parse_qs(url.query)\n\n # Verify query params, from the www-authenticate header\n assert query_params[\"client_id\"] == [\"spack\"]\n assert len(query_params[\"service\"]) == 1\n assert query_params[\"scope\"] == [\"repository:spack-registry:pull,push\"]\n\n service = query_params[\"service\"][0]\n\n if service == \"public.example.com\":\n return self.public_auth(req)\n elif service == \"private.example.com\":\n return self.private_auth(req)\n elif service == \"oauth.example.com\":\n return self.oauth_auth(req)\n\n return MockHTTPResponse(404, \"Not found\")\n\n def public_auth(self, req: Request):\n # No need to login with username and password for the public registry\n assert req.get_header(\"Authorization\") is None\n return MockHTTPResponse.with_json(200, \"OK\", body={\"token\": \"public_token\"})\n\n def oauth_auth(self, req: Request):\n assert req.get_header(\"Authorization\") is None\n return MockHTTPResponse.with_json(200, \"OK\", body={\"access_token\": \"oauth_token\"})\n\n def private_auth(self, req: Request):\n # For the private registry we need to login with username and password\n auth_value = req.get_header(\"Authorization\")\n\n if (\n auth_value is None\n or not auth_value.startswith(\"Basic \")\n or base64.b64decode(auth_value[6:]) != b\"user:pass\"\n ):\n return MockHTTPResponse(401, \"Unauthorized\")\n\n return MockHTTPResponse.with_json(200, \"OK\", body={\"token\": \"private_token\"})\n\n\ndef create_opener(*servers: DummyServer, credentials_provider=None):\n \"\"\"Creates a mock opener, that can be used to fake requests to a list\n of servers.\"\"\"\n opener = urllib.request.OpenerDirector()\n handler = DummyServerUrllibHandler()\n for server in servers:\n handler.add_server(server.domain, server)\n opener.add_handler(handler)\n opener.add_handler(urllib.request.HTTPDefaultErrorHandler())\n opener.add_handler(urllib.request.HTTPErrorProcessor())\n if credentials_provider is not None:\n opener.add_handler(OCIAuthHandler(credentials_provider))\n return opener\n" }, { "path": "lib/spack/spack/test/oci/urlopen.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\nimport hashlib\nimport json\nimport pathlib\nimport random\nimport urllib.error\nimport urllib.parse\nimport urllib.request\nfrom urllib.request import Request\n\nimport pytest\n\nimport spack.mirrors.mirror\nfrom spack.oci.image import Digest, ImageReference, default_config, default_manifest\nfrom spack.oci.oci import (\n copy_missing_layers,\n get_manifest_and_config,\n image_from_mirror,\n list_tags,\n upload_blob,\n upload_manifest,\n)\nfrom spack.oci.opener import (\n Challenge,\n RealmServiceScope,\n UsernamePassword,\n _get_basic_challenge,\n _get_bearer_challenge,\n credentials_from_mirrors,\n default_retry,\n parse_www_authenticate,\n)\nfrom spack.test.conftest import MockHTTPResponse\nfrom spack.test.oci.mock_registry import (\n DummyServer,\n DummyServerUrllibHandler,\n InMemoryOCIRegistry,\n InMemoryOCIRegistryWithBasicAuth,\n InMemoryOCIRegistryWithBearerAuth,\n MiddlewareError,\n MockBearerTokenServer,\n create_opener,\n)\n\n\ndef test_parse_www_authenticate():\n \"\"\"Test parsing of valid WWW-Authenticate header, check whether it's\n decomposed into a list of challenges with correct scheme and parameters\n according to RFC 7235 section 4.1\"\"\"\n www_authenticate = 'Bearer realm=\"https://spack.io/authenticate\",service=\"spack-registry\",scope=\"repository:spack-registry:pull,push\"'\n assert parse_www_authenticate(www_authenticate) == [\n Challenge(\n \"bearer\",\n [\n (\"realm\", \"https://spack.io/authenticate\"),\n (\"service\", \"spack-registry\"),\n (\"scope\", \"repository:spack-registry:pull,push\"),\n ],\n )\n ]\n\n assert parse_www_authenticate(\"Bearer\") == [Challenge(\"bearer\")]\n assert parse_www_authenticate(\"MethodA, MethodB,MethodC\") == [\n Challenge(\"methoda\"),\n Challenge(\"methodb\"),\n Challenge(\"methodc\"),\n ]\n\n assert parse_www_authenticate(\n 'Digest realm=\"Digest Realm\", nonce=\"1234567890\", algorithm=MD5, qop=\"auth\"'\n ) == [\n Challenge(\n \"digest\",\n [\n (\"realm\", \"Digest Realm\"),\n (\"nonce\", \"1234567890\"),\n (\"algorithm\", \"MD5\"),\n (\"qop\", \"auth\"),\n ],\n )\n ]\n\n assert parse_www_authenticate(\n r'Newauth realm=\"apps\", type=1, title=\"Login to \\\"apps\\\"\", Basic realm=\"simple\"'\n ) == [\n Challenge(\"newauth\", [(\"realm\", \"apps\"), (\"type\", \"1\"), (\"title\", 'Login to \"apps\"')]),\n Challenge(\"basic\", [(\"realm\", \"simple\")]),\n ]\n\n assert parse_www_authenticate(r'BASIC REALM=\"simple\"') == [\n Challenge(\"basic\", [(\"realm\", \"simple\")])\n ]\n\n\n@pytest.mark.parametrize(\n \"invalid_str\",\n [\n # Not comma separated\n \"SchemeA SchemeB SchemeC\",\n # Unexpected eof\n \"SchemeA, SchemeB, SchemeC, \",\n # Invalid auth param or scheme\n r\"Scheme x=y, \",\n # Unexpected eof\n \"Scheme key=\",\n # Invalid token\n r'\"Bearer\"',\n # Invalid token\n r'Scheme\"xyz\"',\n # No auth param\n r\"Scheme \",\n ],\n)\ndef test_invalid_www_authenticate(invalid_str):\n with pytest.raises(ValueError):\n parse_www_authenticate(invalid_str)\n\n\ndef test_get_basic_challenge():\n \"\"\"Test extracting Basic challenge from a list of challenges\"\"\"\n\n # No basic challenge\n assert (\n _get_basic_challenge(\n [\n Challenge(\n \"bearer\",\n [\n (\"realm\", \"https://spack.io/authenticate\"),\n (\"service\", \"spack-registry\"),\n (\"scope\", \"repository:spack-registry:pull,push\"),\n ],\n ),\n Challenge(\n \"digest\",\n [\n (\"realm\", \"Digest Realm\"),\n (\"nonce\", \"1234567890\"),\n (\"algorithm\", \"MD5\"),\n (\"qop\", \"auth\"),\n ],\n ),\n ]\n )\n is None\n )\n\n # Multiple challenges, should pick the basic one and return its realm.\n assert (\n _get_basic_challenge(\n [\n Challenge(\n \"dummy\",\n [\n (\"realm\", \"https://example.com/\"),\n (\"service\", \"service\"),\n (\"scope\", \"scope\"),\n ],\n ),\n Challenge(\"basic\", [(\"realm\", \"simple\")]),\n Challenge(\n \"bearer\",\n [\n (\"realm\", \"https://spack.io/authenticate\"),\n (\"service\", \"spack-registry\"),\n (\"scope\", \"repository:spack-registry:pull,push\"),\n ],\n ),\n ]\n )\n == \"simple\"\n )\n\n\ndef test_get_bearer_challenge():\n \"\"\"Test extracting Bearer challenge from a list of challenges\"\"\"\n\n # Only an incomplete bearer challenge, missing service and scope, not usable.\n assert (\n _get_bearer_challenge(\n [\n Challenge(\"bearer\", [(\"realm\", \"https://spack.io/authenticate\")]),\n Challenge(\"basic\", [(\"realm\", \"simple\")]),\n Challenge(\n \"Digest\",\n [\n (\"realm\", \"Digest Realm\"),\n (\"nonce\", \"1234567890\"),\n (\"algorithm\", \"MD5\"),\n (\"qop\", \"auth\"),\n ],\n ),\n ]\n )\n is None\n )\n\n # Multiple challenges, should pick the bearer one.\n assert _get_bearer_challenge(\n [\n Challenge(\n \"dummy\",\n [(\"realm\", \"https://example.com/\"), (\"service\", \"service\"), (\"scope\", \"scope\")],\n ),\n Challenge(\n \"bearer\",\n [\n (\"realm\", \"https://spack.io/authenticate\"),\n (\"service\", \"spack-registry\"),\n (\"scope\", \"repository:spack-registry:pull,push\"),\n ],\n ),\n ]\n ) == RealmServiceScope(\n \"https://spack.io/authenticate\", \"spack-registry\", \"repository:spack-registry:pull,push\"\n )\n\n\n@pytest.mark.parametrize(\n \"image_ref,token\",\n [\n (\"public.example.com/spack-registry:latest\", \"public_token\"),\n (\"private.example.com/spack-registry:latest\", \"private_token\"),\n (\"oauth.example.com/spack-registry:latest\", \"oauth_token\"),\n ],\n)\ndef test_automatic_oci_bearer_authentication(image_ref: str, token: str):\n image = ImageReference.from_string(image_ref)\n\n def credentials_provider(domain: str):\n return UsernamePassword(\"user\", \"pass\") if domain == \"private.example.com\" else None\n\n opener = create_opener(\n InMemoryOCIRegistryWithBearerAuth(\n image.domain, token=token, realm=\"https://auth.example.com/login\"\n ),\n MockBearerTokenServer(\"auth.example.com\"),\n credentials_provider=credentials_provider,\n )\n\n # Run this twice, as it will triggers a code path that caches the bearer token\n assert opener.open(image.endpoint()).status == 200\n assert opener.open(image.endpoint()).status == 200\n\n\ndef test_automatic_oci_basic_authentication():\n image = ImageReference.from_string(\"private.example.com/image\")\n server = InMemoryOCIRegistryWithBasicAuth(\n image.domain, username=\"user\", password=\"pass\", realm=\"example.com\"\n )\n\n # With correct credentials we should get a 200\n opener_with_correct_auth = create_opener(\n server, credentials_provider=lambda domain: UsernamePassword(\"user\", \"pass\")\n )\n assert opener_with_correct_auth.open(image.endpoint()).status == 200\n\n # With wrong credentials we should get a 401\n opener_with_wrong_auth = create_opener(\n server, credentials_provider=lambda domain: UsernamePassword(\"wrong\", \"wrong\")\n )\n with pytest.raises(urllib.error.HTTPError) as e:\n opener_with_wrong_auth.open(image.endpoint())\n assert e.value.getcode() == 401\n\n\ndef test_wrong_credentials():\n \"\"\"Test that when wrong credentials are rejected by the auth server, we\n get a 401 error.\"\"\"\n credentials_provider = lambda domain: UsernamePassword(\"wrong\", \"wrong\")\n image = ImageReference.from_string(\"private.example.com/image\")\n opener = create_opener(\n InMemoryOCIRegistryWithBearerAuth(\n image.domain, token=\"something\", realm=\"https://auth.example.com/login\"\n ),\n MockBearerTokenServer(\"auth.example.com\"),\n credentials_provider=credentials_provider,\n )\n\n with pytest.raises(urllib.error.HTTPError) as e:\n opener.open(image.endpoint())\n\n assert e.value.getcode() == 401\n\n\ndef test_wrong_bearer_token_returned_by_auth_server():\n \"\"\"When the auth server returns a wrong bearer token, we should get a 401 error\n when the request we attempt fails. We shouldn't go in circles getting a 401 from\n the registry, then a non-working token from the auth server, then a 401 from the\n registry, etc.\"\"\"\n image = ImageReference.from_string(\"private.example.com/image\")\n opener = create_opener(\n InMemoryOCIRegistryWithBearerAuth(\n image.domain,\n token=\"other_token_than_token_server_provides\",\n realm=\"https://auth.example.com/login\",\n ),\n MockBearerTokenServer(\"auth.example.com\"),\n credentials_provider=lambda domain: UsernamePassword(\"user\", \"pass\"),\n )\n\n with pytest.raises(urllib.error.HTTPError) as e:\n opener.open(image.endpoint())\n\n assert e.value.getcode() == 401\n\n\nclass TrivialAuthServer(DummyServer):\n \"\"\"A trivial auth server that hands out a bearer token at GET /login.\"\"\"\n\n def __init__(self, domain: str, token: str) -> None:\n super().__init__(domain)\n self.router.register(\"GET\", \"/login\", self.login)\n self.token = token\n\n def login(self, req: Request):\n return MockHTTPResponse.with_json(200, \"OK\", body={\"token\": self.token})\n\n\ndef test_registry_with_short_lived_bearer_tokens():\n \"\"\"An issued bearer token is mostly opaque to the client, but typically\n it embeds a short-lived expiration date. To speed up requests to a registry,\n it's good not to authenticate on every request, but to cache the bearer token,\n however: we have to deal with the case of an expired bearer token.\n\n Here we test that when the bearer token expires, we authenticate again, and\n when the token is still valid, we don't re-authenticate.\"\"\"\n\n image = ImageReference.from_string(\"private.example.com/image\")\n credentials_provider = lambda domain: UsernamePassword(\"user\", \"pass\")\n\n auth_server = TrivialAuthServer(\"auth.example.com\", token=\"token\")\n registry_server = InMemoryOCIRegistryWithBearerAuth(\n image.domain, token=\"token\", realm=\"https://auth.example.com/login\"\n )\n urlopen = create_opener(\n registry_server, auth_server, credentials_provider=credentials_provider\n ).open\n\n # First request, should work with token \"token\"\n assert urlopen(image.endpoint()).status == 200\n\n # Invalidate the token on the registry\n registry_server.token = \"new_token\"\n auth_server.token = \"new_token\"\n\n # Second request: reusing the cached token should fail\n # but in the background we will get a new token from the auth server\n assert urlopen(image.endpoint()).status == 200\n\n # Subsequent requests should work with the same token, let's do two more\n assert urlopen(image.endpoint()).status == 200\n assert urlopen(image.endpoint()).status == 200\n\n # And finally, we should see that we've issues exactly two requests to the auth server\n assert auth_server.requests == [(\"GET\", \"/login\"), (\"GET\", \"/login\")]\n\n # Whereas we've done more requests to the registry\n assert registry_server.requests == [\n (\"GET\", \"/v2/\"), # 1: without bearer token\n (\"GET\", \"/v2/\"), # 2: retry with bearer token\n (\"GET\", \"/v2/\"), # 3: with incorrect bearer token\n (\"GET\", \"/v2/\"), # 4: retry with new bearer token\n (\"GET\", \"/v2/\"), # 5: with recycled correct bearer token\n (\"GET\", \"/v2/\"), # 6: with recycled correct bearer token\n ]\n\n\nclass InMemoryRegistryWithUnsupportedAuth(InMemoryOCIRegistry):\n \"\"\"A registry that does set a WWW-Authenticate header, but\n with a challenge we don't support.\"\"\"\n\n def __init__(self, domain: str, allow_single_post: bool = True, www_authenticate=None) -> None:\n self.www_authenticate = www_authenticate\n super().__init__(domain, allow_single_post)\n self.router.add_middleware(self.unsupported_auth_method)\n\n def unsupported_auth_method(self, req: Request):\n headers = {}\n if self.www_authenticate:\n headers[\"WWW-Authenticate\"] = self.www_authenticate\n raise MiddlewareError(MockHTTPResponse(401, \"Unauthorized\", headers=headers))\n\n\n@pytest.mark.parametrize(\n \"www_authenticate,error_message\",\n [\n # missing service and scope\n ('Bearer realm=\"https://auth.example.com/login\"', \"unsupported authentication scheme\"),\n # missing realm\n (\"Basic\", \"unsupported authentication scheme\"),\n # multiple unsupported challenges\n (\n \"CustomChallenge method=unsupported, OtherChallenge method=x,param=y\",\n \"unsupported authentication scheme\",\n ),\n # no challenge\n (None, \"missing WWW-Authenticate header\"),\n # malformed challenge, missing quotes\n (\"Bearer realm=https://auth.example.com\", \"malformed WWW-Authenticate header\"),\n # http instead of https\n ('Bearer realm=\"http://auth.example.com\",scope=x,service=y', \"insecure http connection\"),\n ],\n)\ndef test_auth_method_we_cannot_handle_is_error(www_authenticate, error_message):\n # We can only handle WWW-Authenticate with a Bearer challenge\n image = ImageReference.from_string(\"private.example.com/image\")\n urlopen = create_opener(\n InMemoryRegistryWithUnsupportedAuth(image.domain, www_authenticate=www_authenticate),\n TrivialAuthServer(\"auth.example.com\", token=\"token\"),\n credentials_provider=lambda domain: UsernamePassword(\"user\", \"pass\"),\n ).open\n\n with pytest.raises(urllib.error.HTTPError, match=error_message) as e:\n urlopen(image.endpoint())\n assert e.value.getcode() == 401\n\n\n# Parametrize over single POST vs POST + PUT.\n@pytest.mark.parametrize(\"client_single_request\", [True, False])\n@pytest.mark.parametrize(\"server_single_request\", [True, False])\ndef test_oci_registry_upload(tmp_path: pathlib.Path, client_single_request, server_single_request):\n opener = urllib.request.OpenerDirector()\n opener.add_handler(\n DummyServerUrllibHandler().add_server(\n \"example.com\", InMemoryOCIRegistry(server_single_request)\n )\n )\n opener.add_handler(urllib.request.HTTPDefaultErrorHandler())\n opener.add_handler(urllib.request.HTTPErrorProcessor())\n\n # Create a small blob\n blob = tmp_path / \"blob\"\n blob.write_text(\"Hello world!\")\n\n image = ImageReference.from_string(\"example.com/image:latest\")\n digest = Digest.from_sha256(hashlib.sha256(blob.read_bytes()).hexdigest())\n\n # Set small file size larger than the blob iff we're doing single request\n small_file_size = 1024 if client_single_request else 0\n\n # Upload once, should actually upload\n assert upload_blob(\n ref=image,\n file=str(blob),\n digest=digest,\n small_file_size=small_file_size,\n _urlopen=opener.open,\n )\n\n # Second time should exit as it exists\n assert not upload_blob(\n ref=image,\n file=str(blob),\n digest=digest,\n small_file_size=small_file_size,\n _urlopen=opener.open,\n )\n\n # Force upload should upload again\n assert upload_blob(\n ref=image,\n file=str(blob),\n digest=digest,\n force=True,\n small_file_size=small_file_size,\n _urlopen=opener.open,\n )\n\n\ndef test_copy_missing_layers(tmp_path: pathlib.Path, config):\n \"\"\"Test copying layers from one registry to another.\n Creates 3 blobs, 1 config and 1 manifest in registry A\n and copies layers to registry B. Then checks that all\n layers are present in registry B. Finally it runs the copy\n again and checks that no new layers are uploaded.\"\"\"\n\n # NOTE: config fixture is used to disable default source mirrors\n # which are used in Stage(...). Otherwise this test doesn't really\n # rely on globals.\n\n src = ImageReference.from_string(\"a.example.com/image:x\")\n dst = ImageReference.from_string(\"b.example.com/image:y\")\n\n src_registry = InMemoryOCIRegistry(src.domain)\n dst_registry = InMemoryOCIRegistry(dst.domain)\n\n urlopen = create_opener(src_registry, dst_registry).open\n\n # TODO: make it a bit easier to create bunch of blobs + config + manifest?\n\n # Create a few blobs and a config file\n blobs = [tmp_path / f\"blob{i}\" for i in range(3)]\n\n for i, blob in enumerate(blobs):\n blob.write_text(f\"Blob {i}\")\n\n digests = [Digest.from_sha256(hashlib.sha256(blob.read_bytes()).hexdigest()) for blob in blobs]\n\n config = default_config(architecture=\"amd64\", os=\"linux\")\n configfile = tmp_path / \"config.json\"\n configfile.write_text(json.dumps(config))\n config_digest = Digest.from_sha256(hashlib.sha256(configfile.read_bytes()).hexdigest())\n\n for blob, digest in zip(blobs, digests):\n upload_blob(src, str(blob), digest, _urlopen=urlopen)\n upload_blob(src, str(configfile), config_digest, _urlopen=urlopen)\n\n # Then create a manifest referencing them\n manifest = default_manifest()\n\n for blob, digest in zip(blobs, digests):\n manifest[\"layers\"].append(\n {\n \"mediaType\": \"application/vnd.oci.image.layer.v1.tar+gzip\",\n \"digest\": str(digest),\n \"size\": blob.stat().st_size,\n }\n )\n\n manifest[\"config\"] = {\n \"mediaType\": \"application/vnd.oci.image.config.v1+json\",\n \"digest\": str(config_digest),\n \"size\": configfile.stat().st_size,\n }\n\n upload_manifest(src, manifest, _urlopen=urlopen)\n\n # Finally, copy the image from src to dst\n copy_missing_layers(src, dst, architecture=\"amd64\", _urlopen=urlopen)\n\n # Check that all layers (not config) were copied and identical\n assert len(dst_registry.blobs) == len(blobs)\n for blob, digest in zip(blobs, digests):\n assert dst_registry.blobs.get(str(digest)) == blob.read_bytes()\n\n is_upload = lambda method, path: method == \"POST\" and path == \"/v2/image/blobs/uploads/\"\n is_exists = lambda method, path: method == \"HEAD\" and path.startswith(\"/v2/image/blobs/\")\n\n # Check that exactly 3 uploads were initiated, and that we don't do\n # double existence checks when uploading.\n assert sum(is_upload(method, path) for method, path in dst_registry.requests) == 3\n assert sum(is_exists(method, path) for method, path in dst_registry.requests) == 3\n\n # Check that re-uploading skips existing layers.\n dst_registry.clear_log()\n copy_missing_layers(src, dst, architecture=\"amd64\", _urlopen=urlopen)\n\n # Check that no uploads were initiated, only existence checks were done.\n assert sum(is_upload(method, path) for method, path in dst_registry.requests) == 0\n assert sum(is_exists(method, path) for method, path in dst_registry.requests) == 3\n\n\ndef test_image_from_mirror():\n mirror = spack.mirrors.mirror.Mirror(\"oci://example.com/image\")\n assert image_from_mirror(mirror) == ImageReference.from_string(\"example.com/image\")\n\n\ndef test_image_from_mirror_with_http_scheme():\n image = image_from_mirror(spack.mirrors.mirror.Mirror({\"url\": \"oci+http://example.com/image\"}))\n assert image.scheme == \"http\"\n assert image.with_tag(\"latest\").scheme == \"http\"\n assert image.with_digest(f\"sha256:{1234:064x}\").scheme == \"http\"\n\n\ndef test_image_reference_str():\n \"\"\"Test that with_digest() works with Digest and str.\"\"\"\n digest_str = f\"sha256:{1234:064x}\"\n digest = Digest.from_string(digest_str)\n\n img = ImageReference.from_string(\"example.com/image\")\n\n assert str(img.with_digest(digest)) == f\"example.com/image:latest@{digest}\"\n assert str(img.with_digest(digest_str)) == f\"example.com/image:latest@{digest}\"\n assert str(img.with_tag(\"hello\")) == \"example.com/image:hello\"\n assert str(img.with_tag(\"hello\").with_digest(digest)) == f\"example.com/image:hello@{digest}\"\n\n\n@pytest.mark.parametrize(\n \"image\",\n [\n # white space issue\n \" example.com/image\",\n # not alpha-numeric\n \"hello#world:latest\",\n ],\n)\ndef test_image_reference_invalid(image):\n with pytest.raises(ValueError, match=\"Invalid image reference\"):\n ImageReference.from_string(image)\n\n\ndef test_default_credentials_provider():\n \"\"\"The default credentials provider uses a collection of configured\n mirrors.\"\"\"\n\n mirrors = [\n # OCI mirror with push credentials\n spack.mirrors.mirror.Mirror(\n {\"url\": \"oci://a.example.com/image\", \"push\": {\"access_pair\": [\"user.a\", \"pass.a\"]}}\n ),\n # Not an OCI mirror\n spack.mirrors.mirror.Mirror(\n {\"url\": \"https://b.example.com/image\", \"access_pair\": [\"user.b\", \"pass.b\"]}\n ),\n # No credentials\n spack.mirrors.mirror.Mirror(\"oci://c.example.com/image\"),\n # Top-level credentials\n spack.mirrors.mirror.Mirror(\n {\"url\": \"oci://d.example.com/image\", \"access_pair\": [\"user.d\", \"pass.d\"]}\n ),\n # Dockerhub short reference\n spack.mirrors.mirror.Mirror(\n {\"url\": \"oci://user/image\", \"access_pair\": [\"dockerhub_user\", \"dockerhub_pass\"]}\n ),\n # Localhost (not a dockerhub short reference)\n spack.mirrors.mirror.Mirror(\n {\"url\": \"oci://localhost/image\", \"access_pair\": [\"user.localhost\", \"pass.localhost\"]}\n ),\n ]\n\n assert credentials_from_mirrors(\"a.example.com\", mirrors=mirrors) == UsernamePassword(\n \"user.a\", \"pass.a\"\n )\n assert credentials_from_mirrors(\"b.example.com\", mirrors=mirrors) is None\n assert credentials_from_mirrors(\"c.example.com\", mirrors=mirrors) is None\n assert credentials_from_mirrors(\"d.example.com\", mirrors=mirrors) == UsernamePassword(\n \"user.d\", \"pass.d\"\n )\n assert credentials_from_mirrors(\"index.docker.io\", mirrors=mirrors) == UsernamePassword(\n \"dockerhub_user\", \"dockerhub_pass\"\n )\n assert credentials_from_mirrors(\"localhost\", mirrors=mirrors) == UsernamePassword(\n \"user.localhost\", \"pass.localhost\"\n )\n\n\ndef test_manifest_index(tmp_path: pathlib.Path):\n \"\"\"Test obtaining manifest + config from a registry\n that has an index\"\"\"\n urlopen = create_opener(InMemoryOCIRegistry(\"registry.example.com\")).open\n\n img = ImageReference.from_string(\"registry.example.com/image\")\n\n # Create two config files and manifests, for different architectures\n manifest_descriptors = []\n manifest_and_config = {}\n for arch in (\"amd64\", \"arm64\"):\n file = tmp_path / f\"config_{arch}.json\"\n config = default_config(architecture=arch, os=\"linux\")\n file.write_text(json.dumps(config))\n config_digest = Digest.from_sha256(hashlib.sha256(file.read_bytes()).hexdigest())\n assert upload_blob(img, str(file), config_digest, _urlopen=urlopen)\n manifest = {\n \"schemaVersion\": 2,\n \"mediaType\": \"application/vnd.oci.image.manifest.v1+json\",\n \"config\": {\n \"mediaType\": \"application/vnd.oci.image.config.v1+json\",\n \"digest\": str(config_digest),\n \"size\": file.stat().st_size,\n },\n \"layers\": [],\n }\n manifest_digest, manifest_size = upload_manifest(\n img, manifest, tag=False, _urlopen=urlopen\n )\n\n manifest_descriptors.append(\n {\n \"mediaType\": \"application/vnd.oci.image.manifest.v1+json\",\n \"platform\": {\"architecture\": arch, \"os\": \"linux\"},\n \"digest\": str(manifest_digest),\n \"size\": manifest_size,\n }\n )\n\n manifest_and_config[arch] = (manifest, config)\n\n # And a single index.\n index = {\n \"schemaVersion\": 2,\n \"mediaType\": \"application/vnd.oci.image.index.v1+json\",\n \"manifests\": manifest_descriptors,\n }\n\n upload_manifest(img, index, tag=True, _urlopen=urlopen)\n\n # Check that we fetcht the correct manifest and config for each architecture\n for arch in (\"amd64\", \"arm64\"):\n assert (\n get_manifest_and_config(img, architecture=arch, _urlopen=urlopen)\n == manifest_and_config[arch]\n )\n\n # Also test max recursion\n with pytest.raises(Exception, match=\"Maximum recursion depth reached\"):\n get_manifest_and_config(img, architecture=\"amd64\", recurse=0, _urlopen=urlopen)\n\n\nclass BrokenServer(DummyServer):\n \"\"\"Dummy server that returns 500 and 429 errors twice before succeeding\"\"\"\n\n def __init__(self, domain: str) -> None:\n super().__init__(domain)\n self.router.register(\"GET\", r\"/internal-server-error/\", self.internal_server_error_twice)\n self.router.register(\"GET\", r\"/rate-limit/\", self.rate_limit_twice)\n self.router.register(\"GET\", r\"/not-found/\", self.not_found)\n self.count_500 = 0\n self.count_429 = 0\n\n def internal_server_error_twice(self, request: Request):\n self.count_500 += 1\n if self.count_500 < 3:\n return MockHTTPResponse(500, \"Internal Server Error\")\n else:\n return MockHTTPResponse(200, \"OK\")\n\n def rate_limit_twice(self, request: Request):\n self.count_429 += 1\n if self.count_429 < 3:\n return MockHTTPResponse(429, \"Rate Limit Exceeded\")\n else:\n return MockHTTPResponse(200, \"OK\")\n\n def not_found(self, request: Request):\n return MockHTTPResponse(404, \"Not Found\")\n\n\n@pytest.mark.parametrize(\n \"url,max_retries,expect_failure,expect_requests\",\n [\n # 500s should be retried\n (\"https://example.com/internal-server-error/\", 2, True, 2),\n (\"https://example.com/internal-server-error/\", 5, False, 3),\n # 429s should be retried\n (\"https://example.com/rate-limit/\", 2, True, 2),\n (\"https://example.com/rate-limit/\", 5, False, 3),\n # 404s shouldn't be retried\n (\"https://example.com/not-found/\", 3, True, 1),\n ],\n)\ndef test_retry(url, max_retries, expect_failure, expect_requests):\n server = BrokenServer(\"example.com\")\n urlopen = create_opener(server).open\n sleep_time = []\n dont_sleep = lambda t: sleep_time.append(t) # keep track of sleep times\n\n try:\n response = default_retry(urlopen, retries=max_retries, sleep=dont_sleep)(url)\n except urllib.error.HTTPError as e:\n if not expect_failure:\n assert False, f\"Unexpected HTTPError: {e}\"\n else:\n if expect_failure:\n assert False, \"Expected HTTPError, but none was raised\"\n assert response.status == 200\n\n assert len(server.requests) == expect_requests\n assert sleep_time == [2**i for i in range(expect_requests - 1)]\n\n\ndef test_list_tags():\n # Follows a relatively new rewording of the OCI distribution spec, which is not yet tagged.\n # https://github.com/opencontainers/distribution-spec/commit/2ed79d930ecec11dd755dc8190409a3b10f01ca9\n N = 20\n urlopen = create_opener(InMemoryOCIRegistry(\"example.com\", tags_per_page=5)).open\n image = ImageReference.from_string(\"example.com/image\")\n to_tag = lambda i: f\"tag-{i:02}\"\n\n # Create N tags in arbitrary order\n _tags_to_create = [to_tag(i) for i in range(N)]\n random.shuffle(_tags_to_create)\n for tag in _tags_to_create:\n upload_manifest(image.with_tag(tag), default_manifest(), tag=True, _urlopen=urlopen)\n\n # list_tags should return all tags from all pages in order\n tags = list_tags(image, urlopen)\n assert len(tags) == N\n assert [to_tag(i) for i in range(N)] == tags\n\n # Test a single request, which should give the first 5 tags\n assert json.loads(urlopen(image.tags_url()).read())[\"tags\"] == [to_tag(i) for i in range(5)]\n\n # Test response at an offset, which should exclude the `last` tag.\n assert json.loads(urlopen(image.tags_url() + f\"?last={to_tag(N - 3)}\").read())[\"tags\"] == [\n to_tag(i) for i in range(N - 2, N)\n ]\n" }, { "path": "lib/spack/spack/test/optional_deps.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport pytest\n\nimport spack.concretize\nfrom spack.spec import Spec\n\n\n@pytest.fixture(\n params=[\n # Normalize simple conditionals\n (\"optional-dep-test\", {\"optional-dep-test\": None}),\n (\"optional-dep-test~a\", {\"optional-dep-test~a\": None}),\n (\"optional-dep-test+a\", {\"optional-dep-test+a\": {\"pkg-a\": None}}),\n (\"optional-dep-test a=true\", {\"optional-dep-test a=true\": {\"pkg-a\": None}}),\n (\"optional-dep-test a=true\", {\"optional-dep-test+a\": {\"pkg-a\": None}}),\n (\"optional-dep-test@1.1\", {\"optional-dep-test@1.1\": {\"pkg-b\": None}}),\n (\"optional-dep-test%intel\", {\"optional-dep-test%intel\": {\"pkg-c\": None}}),\n (\n \"optional-dep-test%intel@64.1\",\n {\"optional-dep-test%intel@64.1\": {\"pkg-c\": None, \"pkg-d\": None}},\n ),\n (\n \"optional-dep-test%intel@64.1.2\",\n {\"optional-dep-test%intel@64.1.2\": {\"pkg-c\": None, \"pkg-d\": None}},\n ),\n (\"optional-dep-test%clang@35\", {\"optional-dep-test%clang@35\": {\"pkg-e\": None}}),\n # Normalize multiple conditionals\n (\"optional-dep-test+a@1.1\", {\"optional-dep-test+a@1.1\": {\"pkg-a\": None, \"pkg-b\": None}}),\n (\n \"optional-dep-test+a%intel\",\n {\"optional-dep-test+a%intel\": {\"pkg-a\": None, \"pkg-c\": None}},\n ),\n (\n \"optional-dep-test@1.1%intel\",\n {\"optional-dep-test@1.1%intel\": {\"pkg-b\": None, \"pkg-c\": None}},\n ),\n (\n \"optional-dep-test@1.1+a%intel@64.1.2\",\n {\n \"optional-dep-test@1.1+a%intel@64.1.2\": {\n \"pkg-a\": None,\n \"pkg-b\": None,\n \"pkg-c\": None,\n \"pkg-d\": None,\n }\n },\n ),\n (\n \"optional-dep-test@1.1+a%clang@36.5\",\n {\"optional-dep-test@1.1+a%clang@36.5\": {\"pkg-b\": None, \"pkg-a\": None, \"pkg-e\": None}},\n ),\n # Chained MPI\n (\n \"optional-dep-test-2+mpi\",\n {\"optional-dep-test-2+mpi\": {\"optional-dep-test+mpi\": {\"mpi\": None}}},\n ),\n # Each of these dependencies comes from a conditional\n # dependency on another. This requires iterating to evaluate\n # the whole chain.\n (\n \"optional-dep-test+f\",\n {\"optional-dep-test+f\": {\"pkg-f\": None, \"pkg-g\": None, \"mpi\": None}},\n ),\n ]\n)\ndef spec_and_expected(request):\n \"\"\"Parameters for the normalization test.\"\"\"\n spec, d = request.param\n return spec, Spec.from_literal(d)\n\n\ndef test_default_variant(config, mock_packages):\n spec = spack.concretize.concretize_one(\"optional-dep-test-3\")\n assert \"pkg-a\" in spec\n\n spec = spack.concretize.concretize_one(\"optional-dep-test-3~var\")\n assert \"pkg-a\" in spec\n\n spec = spack.concretize.concretize_one(\"optional-dep-test-3+var\")\n assert \"pkg-b\" in spec\n" }, { "path": "lib/spack/spack/test/package_class.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Test class methods on PackageBase objects.\n\nThis doesn't include methods on package *instances* (like do_patch(),\netc.). Only methods like ``possible_dependencies()`` that deal with the\nstatic DSL metadata for packages.\n\"\"\"\n\nimport os\nimport pathlib\nimport shutil\n\nimport pytest\n\nimport spack.binary_distribution\nimport spack.concretize\nimport spack.deptypes as dt\nimport spack.error\nimport spack.install_test\nimport spack.llnl.util.filesystem as fs\nimport spack.package_base\nimport spack.spec\nimport spack.store\nimport spack.subprocess_context\nfrom spack.error import InstallError\nfrom spack.package_base import PackageBase\nfrom spack.solver.input_analysis import NoStaticAnalysis, StaticAnalysis\nfrom spack.version import Version\n\n\n@pytest.fixture(scope=\"module\")\ndef compiler_names(mock_packages_repo):\n return [spec.name for spec in mock_packages_repo.providers_for(\"c\")]\n\n\n@pytest.fixture()\ndef mpileaks_possible_deps(mock_packages, mpi_names, compiler_names):\n possible = {\n \"callpath\": set([\"dyninst\"] + mpi_names + compiler_names),\n \"low-priority-provider\": set(),\n \"dyninst\": set([\"libdwarf\", \"libelf\"] + compiler_names),\n \"fake\": set(),\n \"gcc\": set(compiler_names),\n \"intel-parallel-studio\": set(),\n \"libdwarf\": set([\"libelf\"] + compiler_names),\n \"libelf\": set(compiler_names),\n \"llvm\": {\"gcc\", \"llvm\"},\n \"mpich\": set(compiler_names),\n \"mpich2\": set(compiler_names),\n \"mpileaks\": set([\"callpath\"] + mpi_names + compiler_names),\n \"multi-provider-mpi\": set(),\n \"zmpi\": set([\"fake\"] + compiler_names),\n \"compiler-with-deps\": set([\"binutils-for-test\", \"zlib\"] + compiler_names),\n \"binutils-for-test\": set([\"zlib\"] + compiler_names),\n \"zlib\": set(),\n }\n return possible\n\n\n@pytest.fixture(params=[NoStaticAnalysis, StaticAnalysis])\ndef mock_inspector(config, mock_packages, request):\n inspector_cls = request.param\n if inspector_cls is NoStaticAnalysis:\n return inspector_cls(configuration=config, repo=mock_packages)\n return inspector_cls(\n configuration=config,\n repo=mock_packages,\n store=spack.store.STORE,\n binary_index=spack.binary_distribution.BINARY_INDEX,\n )\n\n\n@pytest.fixture\ndef mpi_names(mock_inspector):\n return [spec.name for spec in mock_inspector.providers_for(\"mpi\")]\n\n\n@pytest.mark.parametrize(\n \"pkg_name,fn_kwargs,expected\",\n [\n (\n \"mpileaks\",\n {\"expand_virtuals\": True, \"allowed_deps\": dt.ALL},\n {\n \"fake\",\n \"mpileaks\",\n \"gcc\",\n \"llvm\",\n \"compiler-with-deps\",\n \"binutils-for-test\",\n \"zlib\",\n \"multi-provider-mpi\",\n \"callpath\",\n \"dyninst\",\n \"mpich2\",\n \"libdwarf\",\n \"zmpi\",\n \"low-priority-provider\",\n \"intel-parallel-studio\",\n \"mpich\",\n \"libelf\",\n },\n ),\n (\n \"mpileaks\",\n {\"expand_virtuals\": False, \"allowed_deps\": dt.ALL},\n {\"callpath\", \"dyninst\", \"libdwarf\", \"libelf\", \"mpileaks\"},\n ),\n (\n \"mpileaks\",\n {\"expand_virtuals\": False, \"allowed_deps\": dt.ALL, \"transitive\": False},\n {\"callpath\", \"mpileaks\"},\n ),\n (\"dtbuild1\", {\"allowed_deps\": dt.LINK | dt.RUN}, {\"dtbuild1\", \"dtrun2\", \"dtlink2\"}),\n (\"dtbuild1\", {\"allowed_deps\": dt.BUILD}, {\"dtbuild1\", \"dtbuild2\", \"dtlink2\"}),\n (\"dtbuild1\", {\"allowed_deps\": dt.LINK}, {\"dtbuild1\", \"dtlink2\"}),\n ],\n)\ndef test_possible_dependencies(pkg_name, fn_kwargs, expected, mock_inspector):\n \"\"\"Tests possible nodes of mpileaks, under different scenarios.\"\"\"\n result, *_ = mock_inspector.possible_dependencies(pkg_name, **fn_kwargs)\n assert expected == result\n\n\ndef test_possible_dependencies_virtual(mock_inspector, mock_packages, mpi_names):\n expected = set(mpi_names)\n for name in mpi_names:\n expected.update(\n dep\n for dep in mock_packages.get_pkg_class(name).dependencies_by_name()\n if not mock_packages.is_virtual(dep)\n )\n expected.update(s.name for s in mock_packages.providers_for(\"c\"))\n\n real_pkgs, *_ = mock_inspector.possible_dependencies(\n \"mpi\", transitive=False, allowed_deps=dt.ALL\n )\n assert expected == real_pkgs\n\n\ndef test_possible_dependencies_missing(mock_inspector):\n result, *_ = mock_inspector.possible_dependencies(\"missing-dependency\", allowed_deps=dt.ALL)\n assert \"this-is-a-missing-dependency\" not in result\n\n\ndef test_possible_dependencies_with_multiple_classes(\n mock_inspector, mock_packages, mpileaks_possible_deps\n):\n pkgs = [\"dt-diamond\", \"mpileaks\"]\n expected = set(mpileaks_possible_deps)\n expected.update({\"dt-diamond\", \"dt-diamond-left\", \"dt-diamond-right\", \"dt-diamond-bottom\"})\n\n real_pkgs, *_ = mock_inspector.possible_dependencies(*pkgs, allowed_deps=dt.ALL)\n assert set(expected) == real_pkgs\n\n\ndef setup_install_test(source_paths, test_root):\n \"\"\"\n Set up the install test by creating sources and install test roots.\n\n The convention used here is to create an empty file if the path name\n ends with an extension otherwise, a directory is created.\n \"\"\"\n fs.mkdirp(test_root)\n for path in source_paths:\n if os.path.splitext(path)[1]:\n fs.touchp(path)\n else:\n fs.mkdirp(path)\n\n\n@pytest.mark.parametrize(\n \"spec,sources,extras,expect\",\n [\n (\n \"pkg-a\",\n [\"example/a.c\"], # Source(s)\n [\"example/a.c\"], # Extra test source\n [\"example/a.c\"],\n ), # Test install dir source(s)\n (\n \"pkg-b\",\n [\"test/b.cpp\", \"test/b.hpp\", \"example/b.txt\"], # Source(s)\n [\"test\"], # Extra test source\n [\"test/b.cpp\", \"test/b.hpp\"],\n ), # Test install dir source\n (\n \"pkg-c\",\n [\"examples/a.py\", \"examples/b.py\", \"examples/c.py\", \"tests/d.py\"],\n [\"examples/b.py\", \"tests\"],\n [\"examples/b.py\", \"tests/d.py\"],\n ),\n ],\n)\ndef test_cache_extra_sources(install_mockery, spec, sources, extras, expect):\n \"\"\"Test the package's cache extra test sources helper function.\"\"\"\n s = spack.concretize.concretize_one(spec)\n\n source_path = s.package.stage.source_path\n srcs = [fs.join_path(source_path, src) for src in sources]\n test_root = spack.install_test.install_test_root(s.package)\n setup_install_test(srcs, test_root)\n\n emsg_dir = \"Expected {0} to be a directory\"\n emsg_file = \"Expected {0} to be a file\"\n for src in srcs:\n assert os.path.exists(src), \"Expected {0} to exist\".format(src)\n if os.path.splitext(src)[1]:\n assert os.path.isfile(src), emsg_file.format(src)\n else:\n assert os.path.isdir(src), emsg_dir.format(src)\n\n spack.install_test.cache_extra_test_sources(s.package, extras)\n\n src_dests = [fs.join_path(test_root, src) for src in sources]\n exp_dests = [fs.join_path(test_root, e) for e in expect]\n poss_dests = set(src_dests) | set(exp_dests)\n\n msg = \"Expected {0} to{1} exist\"\n for pd in poss_dests:\n if pd in exp_dests:\n assert os.path.exists(pd), msg.format(pd, \"\")\n if os.path.splitext(pd)[1]:\n assert os.path.isfile(pd), emsg_file.format(pd)\n else:\n assert os.path.isdir(pd), emsg_dir.format(pd)\n else:\n assert not os.path.exists(pd), msg.format(pd, \" not\")\n\n # Perform a little cleanup\n shutil.rmtree(os.path.dirname(source_path))\n\n\ndef test_cache_extra_sources_fails(install_mockery, tmp_path: pathlib.Path):\n s = spack.concretize.concretize_one(\"pkg-a\")\n\n with pytest.raises(InstallError) as exc_info:\n spack.install_test.cache_extra_test_sources(s.package, [str(tmp_path), \"no-such-file\"])\n\n errors = str(exc_info.value)\n assert f\"'{tmp_path}') must be relative\" in errors\n assert \"'no-such-file') for the copy does not exist\" in errors\n\n\ndef test_package_exes_and_libs():\n with pytest.raises(spack.error.SpackError, match=\"defines both\"):\n\n class BadDetectablePackage(PackageBase):\n executables = [\"findme\"]\n libraries = [\"libFindMe.a\"]\n\n\ndef test_package_url_and_urls():\n UrlsPackage = type(\n \"URLsPackage\",\n (PackageBase,),\n {\n \"__module__\": \"spack.pkg.builtin.urls_package\",\n \"url\": \"https://www.example.com/url-package-1.0.tgz\",\n \"urls\": [\"https://www.example.com/archive\"],\n },\n )\n\n s = spack.spec.Spec(\"urls-package\")\n with pytest.raises(ValueError, match=\"defines both\"):\n UrlsPackage(s)\n\n\ndef test_package_license():\n LicensedPackage = type(\n \"LicensedPackage\", (PackageBase,), {\"__module__\": \"spack.pkg.builtin.licensed_package\"}\n )\n\n pkg = LicensedPackage(spack.spec.Spec(\"licensed-package\"))\n assert pkg.global_license_file is None\n\n pkg.license_files = [\"license.txt\"]\n assert os.path.basename(pkg.global_license_file) == pkg.license_files[0]\n\n\nclass BaseTestPackage(PackageBase):\n extendees = {} # currently a required attribute for is_extension()\n\n\ndef test_package_version_fails():\n s = spack.spec.Spec(\"pkg-a\")\n pkg = BaseTestPackage(s)\n with pytest.raises(ValueError, match=\"does not have a concrete version\"):\n pkg.version()\n\n\ndef test_package_tester_fails():\n s = spack.spec.Spec(\"pkg-a\")\n pkg = BaseTestPackage(s)\n with pytest.raises(ValueError, match=\"without concrete version\"):\n pkg.tester()\n\n\ndef test_package_fetcher_fails():\n s = spack.spec.Spec(\"pkg-a\")\n pkg = BaseTestPackage(s)\n with pytest.raises(ValueError, match=\"without concrete version\"):\n pkg.fetcher\n\n\ndef test_package_test_no_compilers(mock_packages, monkeypatch, capfd):\n \"\"\"Ensures that a test which needs the compiler, and build dependencies, to run, is skipped\n if no compiler is available.\n \"\"\"\n s = spack.spec.Spec(\"pkg-a\")\n pkg = BaseTestPackage(s)\n pkg.test_requires_compiler = True\n pkg.do_test()\n error = capfd.readouterr()[1]\n assert \"Skipping tests for package\" in error\n\n\ndef test_package_subscript(default_mock_concretization):\n \"\"\"Tests that we can use the subscript notation on packages, and that it returns a package\"\"\"\n root = default_mock_concretization(\"mpileaks\")\n root_pkg = root.package\n\n # Subscript of a virtual\n assert isinstance(root_pkg[\"mpi\"], spack.package_base.PackageBase)\n\n # Subscript on concrete\n for d in root.traverse():\n assert isinstance(root_pkg[d.name], spack.package_base.PackageBase)\n\n\ndef test_deserialize_preserves_package_attribute(default_mock_concretization):\n x = default_mock_concretization(\"mpileaks\").package\n assert x.spec._package is x\n\n y = spack.subprocess_context.deserialize(spack.subprocess_context.serialize(x))\n assert y.spec._package is y\n\n\n@pytest.mark.require_provenance\ndef test_git_provenance_commit_version(default_mock_concretization):\n spec = default_mock_concretization(\"git-ref-package@stable\")\n assert spec.satisfies(f\"commit={'c' * 40}\")\n\n\n@pytest.mark.parametrize(\"version\", (\"main\", \"tag\", \"annotated-tag\"))\n@pytest.mark.parametrize(\"pre_stage\", (True, False))\n@pytest.mark.require_provenance\n@pytest.mark.disable_clean_stage_check\ndef test_git_provenance_find_commit_ls_remote(\n git, mock_git_repository, mock_packages, config, monkeypatch, version, pre_stage\n):\n repo_path = mock_git_repository.path\n monkeypatch.setattr(\n spack.package_base.PackageBase, \"git\", f\"file://{repo_path}\", raising=False\n )\n\n spec_str = f\"git-test-commit@{version}\"\n\n if pre_stage:\n spack.concretize.concretize_one(spec_str).package.do_stage(False)\n else:\n # explicitly disable ability to use stage or mirror, force url path\n monkeypatch.setattr(\n spack.package_base.PackageBase, \"do_fetch\", lambda *args, **kwargs: None\n )\n\n spec = spack.concretize.concretize_one(spec_str)\n\n if pre_stage:\n # confirmation that we actually had an expanded stage to query with ls-remote\n assert spec.package.stage.expanded\n\n vattrs = spec.package.versions[spec.version]\n git_ref = vattrs.get(\"tag\") or vattrs.get(\"branch\")\n # add the ^{} suffix to the ref so it redirects to the first parent git object\n # for branches and lightweight tags the suffix makes no difference since it is\n # always a commit SHA, but for annotated tags the SHA shifts from the tag SHA\n # back to the commit SHA, which is what we want\n actual_commit = git(\n \"-C\", repo_path, \"rev-parse\", f\"{git_ref}^{{}}\", output=str, error=str\n ).strip()\n assert spec.variants[\"commit\"].value == actual_commit\n\n\n@pytest.mark.require_provenance\n@pytest.mark.disable_clean_stage_check\ndef test_git_provenance_cant_resolve_commit(mock_packages, monkeypatch, config, capfd, tmp_path):\n \"\"\"Fail all attempts to resolve git commits\"\"\"\n repo_path = str(tmp_path / \"non_existent\")\n monkeypatch.setattr(spack.package_base.PackageBase, \"git\", repo_path, raising=False)\n monkeypatch.setattr(spack.package_base.PackageBase, \"do_fetch\", lambda *args, **kwargs: None)\n spec = spack.concretize.concretize_one(\"git-ref-package@develop\")\n captured = capfd.readouterr()\n assert \"commit\" not in spec.variants\n assert \"Warning: Unable to resolve the git commit\" in captured.err\n\n\n@pytest.mark.parametrize(\n \"pkg_name,preferred_version\",\n [\n # This package has a deprecated v1.1.0 which should not be the preferred\n (\"deprecated_versions\", \"1.0.0\"),\n # Python has v2.7.11 marked as preferred and newer v3 versions\n (\"python\", \"2.7.11\"),\n # This package has various versions, some deprecated, plus \"main\" and \"develop\"\n (\"git-ref-package\", \"3.0.1\"),\n ],\n)\ndef test_package_preferred_version(mock_packages, config, pkg_name, preferred_version):\n \"\"\"Tests retrieving the preferred version of a package.\"\"\"\n pkg_cls = mock_packages.get_pkg_class(pkg_name)\n assert spack.package_base.preferred_version(pkg_cls) == Version(preferred_version)\n" }, { "path": "lib/spack/spack/test/packages.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport importlib\nimport os\nimport pathlib\nimport sys\n\nimport pytest\n\nimport spack.concretize\nimport spack.directives\nimport spack.error\nimport spack.fetch_strategy\nimport spack.package\nimport spack.package_base\nimport spack.repo\nfrom spack.paths import mock_packages_path\nfrom spack.spec import Spec\nfrom spack.util.naming import pkg_name_to_class_name\nfrom spack.version import VersionChecksumError\n\n\nclass MyPrependFileLoader(spack.repo._PrependFileLoader):\n \"\"\"Skip explicit prepending of 'spack_repo.builtin.build_systems' import.\"\"\"\n\n def __init__(self, fullname, repo, package_name):\n super().__init__(fullname, repo, package_name)\n self.prepend = b\"\"\n\n\ndef pkg_factory(name):\n \"\"\"Return a package object tied to an abstract spec\"\"\"\n pkg_cls = spack.repo.PATH.get_pkg_class(name)\n return pkg_cls(Spec(name))\n\n\n@pytest.mark.usefixtures(\"config\", \"mock_packages\")\nclass TestPackage:\n def test_load_package(self):\n spack.repo.PATH.get_pkg_class(\"mpich\")\n\n def test_package_name(self):\n pkg_cls = spack.repo.PATH.get_pkg_class(\"mpich\")\n assert pkg_cls.name == \"mpich\"\n\n def test_package_filename(self):\n repo = spack.repo.from_path(mock_packages_path)\n filename = repo.filename_for_package_name(\"mpich\")\n assert filename == os.path.join(mock_packages_path, \"packages\", \"mpich\", \"package.py\")\n\n def test_nonexisting_package_filename(self):\n repo = spack.repo.from_path(mock_packages_path)\n filename = repo.filename_for_package_name(\"some-nonexisting-package\")\n assert filename == os.path.join(\n mock_packages_path, \"packages\", \"some_nonexisting_package\", \"package.py\"\n )\n\n def test_package_class_names(self):\n assert \"Mpich\" == pkg_name_to_class_name(\"mpich\")\n assert \"PmgrCollective\" == pkg_name_to_class_name(\"pmgr_collective\")\n assert \"PmgrCollective\" == pkg_name_to_class_name(\"pmgr-collective\")\n assert \"Pmgrcollective\" == pkg_name_to_class_name(\"PmgrCollective\")\n assert \"_3db\" == pkg_name_to_class_name(\"3db\")\n assert \"_True\" == pkg_name_to_class_name(\"true\") # reserved keyword\n assert \"_False\" == pkg_name_to_class_name(\"false\") # reserved keyword\n assert \"_None\" == pkg_name_to_class_name(\"none\") # reserved keyword\n assert \"Finally\" == pkg_name_to_class_name(\"finally\") # `Finally` is not reserved\n\n # Below tests target direct imports of spack packages from the spack.pkg namespace\n def test_import_package(self, tmp_path: pathlib.Path, monkeypatch):\n monkeypatch.setattr(spack.repo, \"_PrependFileLoader\", MyPrependFileLoader)\n root, _ = spack.repo.create_repo(str(tmp_path), \"testing_repo\", package_api=(1, 0))\n pkg_path = pathlib.Path(root) / \"packages\" / \"mpich\" / \"package.py\"\n pkg_path.parent.mkdir(parents=True)\n pkg_path.write_text(\"foo = 1\")\n\n with spack.repo.use_repositories(root):\n importlib.import_module(\"spack.pkg.testing_repo\")\n assert importlib.import_module(\"spack.pkg.testing_repo.mpich\").foo == 1\n\n del sys.modules[\"spack.pkg.testing_repo\"]\n del sys.modules[\"spack.pkg.testing_repo.mpich\"]\n\n def test_inheritance_of_directives(self):\n pkg_cls = spack.repo.PATH.get_pkg_class(\"simple-inheritance\")\n\n # Check dictionaries that should have been filled by directives\n dependencies = pkg_cls.dependencies_by_name()\n assert len(dependencies) == 4\n assert \"cmake\" in dependencies\n assert \"openblas\" in dependencies\n assert \"mpi\" in dependencies\n assert len(pkg_cls.provided) == 2\n\n # Check that Spec instantiation behaves as we expect\n s = spack.concretize.concretize_one(\"simple-inheritance\")\n assert \"^cmake\" in s\n assert \"^openblas\" in s\n assert \"+openblas\" in s\n assert \"mpi\" in s\n\n s = spack.concretize.concretize_one(\"simple-inheritance~openblas\")\n assert \"^cmake\" in s\n assert \"^openblas\" not in s\n assert \"~openblas\" in s\n assert \"mpi\" in s\n\n @pytest.mark.regression(\"11844\")\n def test_inheritance_of_patches(self):\n # Will error if inheritor package cannot find inherited patch files\n _ = spack.concretize.concretize_one(\"patch-inheritance\")\n\n\n@pytest.mark.regression(\"2737\")\ndef test_urls_for_versions(mock_packages, config):\n \"\"\"Version directive without a 'url' argument should use default url.\"\"\"\n for spec_str in (\"url-override@0.9.0\", \"url-override@1.0.0\"):\n s = spack.concretize.concretize_one(spec_str)\n url = s.package.url_for_version(\"0.9.0\")\n assert url == \"http://www.anothersite.org/uo-0.9.0.tgz\"\n\n url = s.package.url_for_version(\"1.0.0\")\n assert url == \"http://www.doesnotexist.org/url_override-1.0.0.tar.gz\"\n\n url = s.package.url_for_version(\"0.8.1\")\n assert url == \"http://www.doesnotexist.org/url_override-0.8.1.tar.gz\"\n\n\ndef test_url_for_version_with_no_urls(mock_packages, config):\n spec = Spec(\"git-test\")\n pkg_cls = spack.repo.PATH.get_pkg_class(spec.name)\n with pytest.raises(spack.error.NoURLError):\n pkg_cls(spec).url_for_version(\"1.0\")\n\n with pytest.raises(spack.error.NoURLError):\n pkg_cls(spec).url_for_version(\"1.1\")\n\n\n@pytest.mark.skip(reason=\"spack.build_systems moved out of spack/spack\")\ndef test_custom_cmake_prefix_path(mock_packages, config):\n pass\n # spec = spack.concretize.concretize_one(\"depends-on-define-cmake-prefix-paths\")\n # assert spack.build_systems.cmake.get_cmake_prefix_path(spec.package) == [\n # spec[\"define-cmake-prefix-paths\"].prefix.test\n # ]\n\n\ndef test_url_for_version_with_only_overrides(mock_packages, config):\n s = spack.concretize.concretize_one(\"url-only-override\")\n\n # these exist and should just take the URL provided in the package\n assert s.package.url_for_version(\"1.0.0\") == \"http://a.example.com/url_override-1.0.0.tar.gz\"\n assert s.package.url_for_version(\"0.9.0\") == \"http://b.example.com/url_override-0.9.0.tar.gz\"\n assert s.package.url_for_version(\"0.8.1\") == \"http://c.example.com/url_override-0.8.1.tar.gz\"\n\n # these don't exist but should still work, even if there are only overrides\n assert s.package.url_for_version(\"1.0.5\") == \"http://a.example.com/url_override-1.0.5.tar.gz\"\n assert s.package.url_for_version(\"0.9.5\") == \"http://b.example.com/url_override-0.9.5.tar.gz\"\n assert s.package.url_for_version(\"0.8.5\") == \"http://c.example.com/url_override-0.8.5.tar.gz\"\n assert s.package.url_for_version(\"0.7.0\") == \"http://c.example.com/url_override-0.7.0.tar.gz\"\n\n\ndef test_url_for_version_with_only_overrides_with_gaps(mock_packages, config):\n s = spack.concretize.concretize_one(\"url-only-override-with-gaps\")\n\n # same as for url-only-override -- these are specific\n assert s.package.url_for_version(\"1.0.0\") == \"http://a.example.com/url_override-1.0.0.tar.gz\"\n assert s.package.url_for_version(\"0.9.0\") == \"http://b.example.com/url_override-0.9.0.tar.gz\"\n assert s.package.url_for_version(\"0.8.1\") == \"http://c.example.com/url_override-0.8.1.tar.gz\"\n\n # these don't have specific URLs, but should still work by extrapolation\n assert s.package.url_for_version(\"1.0.5\") == \"http://a.example.com/url_override-1.0.5.tar.gz\"\n assert s.package.url_for_version(\"0.9.5\") == \"http://b.example.com/url_override-0.9.5.tar.gz\"\n assert s.package.url_for_version(\"0.8.5\") == \"http://c.example.com/url_override-0.8.5.tar.gz\"\n assert s.package.url_for_version(\"0.7.0\") == \"http://c.example.com/url_override-0.7.0.tar.gz\"\n\n\n@pytest.mark.usefixtures(\"mock_packages\", \"config\")\n@pytest.mark.parametrize(\n \"spec_str,expected_type,expected_url\",\n [\n (\n \"git-top-level\",\n spack.fetch_strategy.GitFetchStrategy,\n \"https://example.com/some/git/repo\",\n ),\n (\n \"svn-top-level\",\n spack.fetch_strategy.SvnFetchStrategy,\n \"https://example.com/some/svn/repo\",\n ),\n (\"hg-top-level\", spack.fetch_strategy.HgFetchStrategy, \"https://example.com/some/hg/repo\"),\n ],\n)\ndef test_fetcher_url(spec_str, expected_type, expected_url):\n \"\"\"Ensure that top-level git attribute can be used as a default.\"\"\"\n fetcher = spack.fetch_strategy.for_package_version(pkg_factory(spec_str), \"1.0\")\n assert isinstance(fetcher, expected_type)\n assert fetcher.url == expected_url\n\n\n@pytest.mark.usefixtures(\"mock_packages\", \"config\")\n@pytest.mark.parametrize(\n \"spec_str,version_str,exception_type\",\n [\n # Non-url-package\n (\"git-top-level\", \"1.1\", spack.fetch_strategy.ExtrapolationError),\n # Two VCS specified together\n (\"git-url-svn-top-level\", \"1.0\", spack.fetch_strategy.FetcherConflict),\n (\"git-svn-top-level\", \"1.0\", spack.fetch_strategy.FetcherConflict),\n ],\n)\ndef test_fetcher_errors(spec_str, version_str, exception_type):\n \"\"\"Verify that we can't extrapolate versions for non-URL packages.\"\"\"\n with pytest.raises(exception_type):\n spack.fetch_strategy.for_package_version(pkg_factory(spec_str), version_str)\n\n\n@pytest.mark.usefixtures(\"mock_packages\", \"config\")\n@pytest.mark.parametrize(\n \"version_str,expected_url,digest\",\n [\n (\"2.0\", \"https://example.com/some/tarball-2.0.tar.gz\", \"20\"),\n (\"2.1\", \"https://example.com/some/tarball-2.1.tar.gz\", \"21\"),\n (\"2.2\", \"https://www.example.com/foo2.2.tar.gz\", \"22\"),\n (\"2.3\", \"https://www.example.com/foo2.3.tar.gz\", \"23\"),\n ],\n)\ndef test_git_url_top_level_url_versions(version_str, expected_url, digest):\n \"\"\"Test URL fetch strategy inference when url is specified with git.\"\"\"\n # leading 62 zeros of sha256 hash\n leading_zeros = \"0\" * 62\n\n fetcher = spack.fetch_strategy.for_package_version(\n pkg_factory(\"git-url-top-level\"), version_str\n )\n assert isinstance(fetcher, spack.fetch_strategy.URLFetchStrategy)\n assert fetcher.url == expected_url\n assert fetcher.digest == leading_zeros + digest\n\n\n@pytest.mark.usefixtures(\"mock_packages\", \"config\")\n@pytest.mark.parametrize(\n \"version_str,tag,commit,branch\",\n [\n (\"3.0\", \"v3.0\", None, None),\n (\"3.1\", \"v3.1\", \"abc31\", None),\n (\"3.2\", None, None, \"releases/v3.2\"),\n (\"3.3\", None, \"abc33\", \"releases/v3.3\"),\n (\"3.4\", None, \"abc34\", None),\n (\"submodules\", None, None, None),\n (\"develop\", None, None, \"develop\"),\n ],\n)\ndef test_git_url_top_level_git_versions(version_str, tag, commit, branch):\n \"\"\"Test git fetch strategy inference when url is specified with git.\"\"\"\n fetcher = spack.fetch_strategy.for_package_version(\n pkg_factory(\"git-url-top-level\"), version_str\n )\n assert isinstance(fetcher, spack.fetch_strategy.GitFetchStrategy)\n assert fetcher.url == \"https://example.com/some/git/repo\"\n assert fetcher.tag == tag\n assert fetcher.commit == commit\n assert fetcher.branch == branch\n assert fetcher.url == pkg_factory(\"git-url-top-level\").git\n\n\n@pytest.mark.usefixtures(\"mock_packages\", \"config\")\n@pytest.mark.parametrize(\"version_str\", [\"1.0\", \"1.1\", \"1.2\", \"1.3\"])\ndef test_git_url_top_level_conflicts(version_str):\n \"\"\"Test git fetch strategy inference when url is specified with git.\"\"\"\n with pytest.raises(spack.fetch_strategy.FetcherConflict):\n spack.fetch_strategy.for_package_version(pkg_factory(\"git-url-top-level\"), version_str)\n\n\ndef test_rpath_args(mutable_database):\n \"\"\"Test a package's rpath_args property.\"\"\"\n\n rec = mutable_database.get_record(\"mpich\")\n\n rpath_args = rec.spec.package.rpath_args\n assert \"-rpath\" in rpath_args\n assert \"mpich\" in rpath_args\n\n\ndef test_bundle_version_checksum(mock_directive_bundle, clear_directive_functions):\n \"\"\"Test raising exception on a version checksum with a bundle package.\"\"\"\n with pytest.raises(VersionChecksumError, match=\"Checksums not allowed\"):\n version = spack.directives.version(\"1.0\", checksum=\"1badpkg\")\n version(mock_directive_bundle)\n\n\ndef test_bundle_patch_directive(mock_directive_bundle, clear_directive_functions):\n \"\"\"Test raising exception on a patch directive with a bundle package.\"\"\"\n with pytest.raises(\n spack.directives.UnsupportedPackageDirective, match=\"Patches are not allowed\"\n ):\n patch = spack.directives.patch(\"mock/patch.txt\")\n patch(mock_directive_bundle)\n\n\n@pytest.mark.usefixtures(\"mock_packages\", \"config\")\n@pytest.mark.parametrize(\n \"version_str,digest_end,extra_options\",\n [\n (\"1.0\", \"10\", {\"timeout\": 42, \"cookie\": \"foobar\"}),\n (\"1.1\", \"11\", {\"timeout\": 65}),\n (\"1.2\", \"12\", {\"cookie\": \"baz\"}),\n ],\n)\ndef test_fetch_options(version_str, digest_end, extra_options):\n \"\"\"Test fetch options inference.\"\"\"\n leading_zeros = \"000000000000000000000000000000\"\n fetcher = spack.fetch_strategy.for_package_version(pkg_factory(\"fetch-options\"), version_str)\n assert isinstance(fetcher, spack.fetch_strategy.URLFetchStrategy)\n assert fetcher.digest == leading_zeros + digest_end\n assert fetcher.extra_options == extra_options\n\n\ndef test_package_deprecated_version(mock_packages, mock_fetch, mock_stage):\n spec = Spec(\"deprecated-versions\")\n pkg_cls = spack.repo.PATH.get_pkg_class(spec.name)\n\n assert spack.package_base.deprecated_version(pkg_cls, \"1.1.0\")\n assert not spack.package_base.deprecated_version(pkg_cls, \"1.0.0\")\n\n\ndef test_package_can_have_sparse_checkout_properties(mock_packages, mock_fetch, mock_stage):\n spec = Spec(\"git-sparsepaths-pkg\")\n pkg_cls = spack.repo.PATH.get_pkg_class(spec.name)\n assert hasattr(pkg_cls, \"git_sparse_paths\")\n\n fetcher = spack.fetch_strategy.for_package_version(pkg_cls(spec), \"1.0\")\n assert isinstance(fetcher, spack.fetch_strategy.GitFetchStrategy)\n assert hasattr(fetcher, \"git_sparse_paths\")\n assert fetcher.git_sparse_paths == pkg_cls.git_sparse_paths\n\n\ndef test_package_can_have_sparse_checkout_properties_with_commit_version(\n mock_packages, mock_fetch, mock_stage\n):\n spec = Spec(\"git-sparsepaths-pkg commit=abcdefg\")\n pkg_cls = spack.repo.PATH.get_pkg_class(spec.name)\n assert hasattr(pkg_cls, \"git_sparse_paths\")\n\n fetcher = spack.fetch_strategy.for_package_version(pkg_cls(spec), \"1.0\")\n assert isinstance(fetcher, spack.fetch_strategy.GitFetchStrategy)\n assert hasattr(fetcher, \"git_sparse_paths\")\n assert fetcher.git_sparse_paths == pkg_cls.git_sparse_paths\n\n\ndef test_package_can_have_sparse_checkout_properties_with_gitversion(\n mock_packages, mock_fetch, mock_stage\n):\n spec = Spec(\"git-sparsepaths-pkg\")\n pkg_cls = spack.repo.PATH.get_pkg_class(spec.name)\n assert hasattr(pkg_cls, \"git_sparse_paths\")\n\n version = \"git.foo=1.0\"\n fetcher = spack.fetch_strategy.for_package_version(pkg_cls(spec), version)\n assert isinstance(fetcher, spack.fetch_strategy.GitFetchStrategy)\n assert hasattr(fetcher, \"git_sparse_paths\")\n assert fetcher.git_sparse_paths == pkg_cls.git_sparse_paths\n\n\ndef test_package_version_can_have_sparse_checkout_properties(\n mock_packages, mock_fetch, mock_stage\n):\n spec = Spec(\"git-sparsepaths-version\")\n pkg_cls = spack.repo.PATH.get_pkg_class(spec.name)\n\n fetcher = spack.fetch_strategy.for_package_version(pkg_cls(spec), version=\"1.0\")\n assert isinstance(fetcher, spack.fetch_strategy.GitFetchStrategy)\n assert fetcher.git_sparse_paths == [\"foo\", \"bar\"]\n\n fetcher = spack.fetch_strategy.for_package_version(pkg_cls(spec), version=\"0.9\")\n assert isinstance(fetcher, spack.fetch_strategy.GitFetchStrategy)\n assert fetcher.git_sparse_paths is None\n\n\ndef test_package_can_depend_on_commit_of_dependency(mock_packages, config):\n spec = spack.concretize.concretize_one(Spec(\"git-ref-commit-dep@1.0.0\"))\n assert spec.satisfies(f\"^git-ref-package commit={'a' * 40}\")\n assert \"surgical\" not in spec[\"git-ref-package\"].variants\n\n\ndef test_package_condtional_variants_may_depend_on_commit(mock_packages, config):\n spec = spack.concretize.concretize_one(Spec(\"git-ref-commit-dep@develop\"))\n assert spec.satisfies(f\"^git-ref-package commit={'b' * 40}\")\n conditional_variant = spec[\"git-ref-package\"].variants.get(\"surgical\", None)\n assert conditional_variant\n assert conditional_variant.value\n\n\ndef test_commit_variant_finds_matches_for_commit_versions(mock_packages, config):\n \"\"\"\n test conditional dependence on `when='commit='`\n git-ref-commit-dep variant commit-selector depends on a specific commit of git-ref-package\n that commit is associated with the stable version of git-ref-package\n \"\"\"\n spec = spack.concretize.concretize_one(Spec(\"git-ref-commit-dep+commit-selector\"))\n assert spec.satisfies(f\"^git-ref-package commit={'c' * 40}\")\n\n\ndef test_pkg_name_can_only_be_derived_when_package_module():\n \"\"\"When the module prefix is not spack_repo (or legacy spack.pkg) we cannot derive\n a package name.\"\"\"\n ExamplePackage = type(\n \"ExamplePackage\",\n (spack.package_base.PackageBase,),\n {\"__module__\": \"not.a.spack.repo.packages.example_package.package\"},\n )\n\n with pytest.raises(ValueError, match=\"Package ExamplePackage is not a known Spack package\"):\n ExamplePackage.name\n\n\ndef test_spack_package_api_versioning():\n \"\"\"Test that the symbols in spack.package.api match the public API.\"\"\"\n assert spack.package.__all__ == [\n symbol for symbols in spack.package.api.values() for symbol in symbols\n ]\n" }, { "path": "lib/spack/spack/test/packaging.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"\nThis test checks the binary packaging infrastructure\n\"\"\"\n\nimport argparse\nimport os\nimport pathlib\nimport platform\nimport shutil\nimport urllib.error\nfrom collections import OrderedDict\n\nimport pytest\n\nimport spack.binary_distribution\nimport spack.cmd.buildcache as buildcache\nimport spack.cmd.mirror\nimport spack.concretize\nimport spack.config\nimport spack.error\nimport spack.fetch_strategy\nimport spack.package_base\nimport spack.stage\nimport spack.util.gpg\nimport spack.util.url as url_util\nfrom spack.fetch_strategy import URLFetchStrategy\nfrom spack.installer import PackageInstaller\nfrom spack.llnl.util import filesystem as fs\nfrom spack.llnl.util.filesystem import readlink, symlink\nfrom spack.paths import mock_gpg_keys_path\nfrom spack.relocate import _macho_find_paths, relocate_links, relocate_text\n\npytestmark = pytest.mark.not_on_windows(\"does not run on windows\")\n\n\n@pytest.mark.usefixtures(\"install_mockery\", \"mock_gnupghome\")\ndef test_buildcache(mock_archive, tmp_path: pathlib.Path, monkeypatch, mutable_config):\n # Install a test package\n spec = spack.concretize.concretize_one(\"trivial-install-test-package\")\n monkeypatch.setattr(spec.package, \"fetcher\", URLFetchStrategy(url=mock_archive.url))\n PackageInstaller([spec.package], explicit=True).install()\n pkghash = \"/\" + str(spec.dag_hash(7))\n\n # Put some non-relocatable file in there\n dummy_txt = pathlib.Path(spec.prefix) / \"dummy.txt\"\n dummy_txt.write_text(spec.prefix)\n\n # Create an absolute symlink\n linkname = os.path.join(spec.prefix, \"link_to_dummy.txt\")\n symlink(dummy_txt, linkname)\n\n # Create the build cache and put it directly into the mirror\n mirror_path = str(tmp_path / \"test-mirror\")\n spack.cmd.mirror.create(mirror_path, specs=[])\n\n # register mirror with spack config\n mirrors = {\"spack-mirror-test\": url_util.path_to_file_url(mirror_path)}\n spack.config.set(\"mirrors\", mirrors)\n\n with spack.stage.Stage(mirrors[\"spack-mirror-test\"], name=\"build_cache\", keep=True):\n parser = argparse.ArgumentParser()\n buildcache.setup_parser(parser)\n\n create_args = [\"create\", \"-f\", \"--rebuild-index\", mirror_path, pkghash]\n # Create a private key to sign package with if gpg2 available\n spack.util.gpg.create(\n name=\"test key 1\",\n expires=\"0\",\n email=\"spack@googlegroups.com\",\n comment=\"Spack test key\",\n )\n\n args = parser.parse_args(create_args)\n buildcache.buildcache(parser, args)\n # trigger overwrite warning\n buildcache.buildcache(parser, args)\n\n # Uninstall the package\n spec.package.do_uninstall(force=True)\n\n install_args = [\"install\", \"-f\", pkghash]\n args = parser.parse_args(install_args)\n # Test install\n buildcache.buildcache(parser, args)\n\n files = os.listdir(spec.prefix)\n\n assert \"link_to_dummy.txt\" in files\n assert \"dummy.txt\" in files\n\n # Validate the relocation information\n buildinfo = spack.binary_distribution.read_buildinfo_file(spec.prefix)\n assert buildinfo[\"relocate_textfiles\"] == [\"dummy.txt\"]\n assert buildinfo[\"relocate_links\"] == [\"link_to_dummy.txt\"]\n\n args = parser.parse_args([\"keys\"])\n buildcache.buildcache(parser, args)\n\n args = parser.parse_args([\"list\"])\n buildcache.buildcache(parser, args)\n\n args = parser.parse_args([\"list\"])\n buildcache.buildcache(parser, args)\n\n args = parser.parse_args([\"list\", \"trivial\"])\n buildcache.buildcache(parser, args)\n\n # Copy a key to the mirror to have something to download\n shutil.copyfile(mock_gpg_keys_path + \"/external.key\", mirror_path + \"/external.key\")\n\n args = parser.parse_args([\"keys\"])\n buildcache.buildcache(parser, args)\n\n args = parser.parse_args([\"keys\", \"-f\"])\n buildcache.buildcache(parser, args)\n\n args = parser.parse_args([\"keys\", \"-i\", \"-t\"])\n buildcache.buildcache(parser, args)\n\n\ndef test_relocate_text(tmp_path: pathlib.Path):\n \"\"\"Tests that a text file containing the original directory of an installation, can be\n relocated to a target directory.\n \"\"\"\n original_dir = \"/home/spack/opt/spack\"\n relocation_dir = \"/opt/rh/devtoolset/\"\n dummy_txt = tmp_path / \"dummy.txt\"\n dummy_txt.write_text(original_dir)\n\n relocate_text([str(dummy_txt)], {original_dir: relocation_dir})\n text = dummy_txt.read_text()\n\n assert relocation_dir in text\n assert original_dir not in text\n\n\ndef test_relocate_links(tmp_path: pathlib.Path):\n (tmp_path / \"new_prefix_a\").mkdir()\n\n own_prefix_path = str(tmp_path / \"prefix_a\" / \"file\")\n dep_prefix_path = str(tmp_path / \"prefix_b\" / \"file\")\n new_own_prefix_path = str(tmp_path / \"new_prefix_a\" / \"file\")\n new_dep_prefix_path = str(tmp_path / \"new_prefix_b\" / \"file\")\n system_path = os.path.join(os.path.sep, \"system\", \"path\")\n\n fs.touchp(own_prefix_path)\n fs.touchp(new_own_prefix_path)\n fs.touchp(dep_prefix_path)\n fs.touchp(new_dep_prefix_path)\n\n # Old prefixes to new prefixes\n prefix_to_prefix = OrderedDict(\n [\n # map /prefix_a -> /new_prefix_a\n (str(tmp_path / \"prefix_a\"), str(tmp_path / \"new_prefix_a\")),\n # map /prefix_b -> /new_prefix_b\n (str(tmp_path / \"prefix_b\"), str(tmp_path / \"new_prefix_b\")),\n # map -> /fallback/path -- this is just to see we respect order.\n (str(tmp_path), os.path.join(os.path.sep, \"fallback\", \"path\")),\n ]\n )\n\n with fs.working_dir(str(tmp_path / \"new_prefix_a\")):\n # To be relocated\n os.symlink(own_prefix_path, \"to_self\")\n os.symlink(dep_prefix_path, \"to_dependency\")\n\n # To be ignored\n os.symlink(system_path, \"to_system\")\n os.symlink(\"relative\", \"to_self_but_relative\")\n\n relocate_links([\"to_self\", \"to_dependency\", \"to_system\"], prefix_to_prefix)\n\n # These two are relocated\n assert readlink(\"to_self\") == str(tmp_path / \"new_prefix_a\" / \"file\")\n assert readlink(\"to_dependency\") == str(tmp_path / \"new_prefix_b\" / \"file\")\n\n # These two are not.\n assert readlink(\"to_system\") == system_path\n assert readlink(\"to_self_but_relative\") == \"relative\"\n\n\ndef test_replace_paths(tmp_path: pathlib.Path):\n with fs.working_dir(str(tmp_path)):\n suffix = \"dylib\" if platform.system().lower() == \"darwin\" else \"so\"\n hash_a = \"53moz6jwnw3xpiztxwhc4us26klribws\"\n hash_b = \"tk62dzu62kd4oh3h3heelyw23hw2sfee\"\n hash_c = \"hdkhduizmaddpog6ewdradpobnbjwsjl\"\n hash_d = \"hukkosc7ahff7o65h6cdhvcoxm57d4bw\"\n hash_loco = \"zy4oigsc4eovn5yhr2lk4aukwzoespob\"\n\n prefix2hash = {}\n\n old_spack_dir = os.path.join(f\"{tmp_path}\", \"Users\", \"developer\", \"spack\")\n fs.mkdirp(old_spack_dir)\n\n oldprefix_a = os.path.join(f\"{old_spack_dir}\", f\"pkgA-{hash_a}\")\n oldlibdir_a = os.path.join(f\"{oldprefix_a}\", \"lib\")\n fs.mkdirp(oldlibdir_a)\n prefix2hash[str(oldprefix_a)] = hash_a\n\n oldprefix_b = os.path.join(f\"{old_spack_dir}\", f\"pkgB-{hash_b}\")\n oldlibdir_b = os.path.join(f\"{oldprefix_b}\", \"lib\")\n fs.mkdirp(oldlibdir_b)\n prefix2hash[str(oldprefix_b)] = hash_b\n\n oldprefix_c = os.path.join(f\"{old_spack_dir}\", f\"pkgC-{hash_c}\")\n oldlibdir_c = os.path.join(f\"{oldprefix_c}\", \"lib\")\n oldlibdir_cc = os.path.join(f\"{oldlibdir_c}\", \"C\")\n fs.mkdirp(oldlibdir_c)\n prefix2hash[str(oldprefix_c)] = hash_c\n\n oldprefix_d = os.path.join(f\"{old_spack_dir}\", f\"pkgD-{hash_d}\")\n oldlibdir_d = os.path.join(f\"{oldprefix_d}\", \"lib\")\n fs.mkdirp(oldlibdir_d)\n prefix2hash[str(oldprefix_d)] = hash_d\n\n oldprefix_local = os.path.join(f\"{tmp_path}\", \"usr\", \"local\")\n oldlibdir_local = os.path.join(f\"{oldprefix_local}\", \"lib\")\n fs.mkdirp(oldlibdir_local)\n prefix2hash[str(oldprefix_local)] = hash_loco\n libfile_a = f\"libA.{suffix}\"\n libfile_b = f\"libB.{suffix}\"\n libfile_c = f\"libC.{suffix}\"\n libfile_d = f\"libD.{suffix}\"\n libfile_loco = f\"libloco.{suffix}\"\n old_libnames = [\n os.path.join(oldlibdir_a, libfile_a),\n os.path.join(oldlibdir_b, libfile_b),\n os.path.join(oldlibdir_c, libfile_c),\n os.path.join(oldlibdir_d, libfile_d),\n os.path.join(oldlibdir_local, libfile_loco),\n ]\n\n for old_libname in old_libnames:\n with open(old_libname, \"a\", encoding=\"utf-8\"):\n os.utime(old_libname, None)\n\n hash2prefix = dict()\n\n new_spack_dir = os.path.join(f\"{tmp_path}\", \"Users\", \"Shared\", \"spack\")\n fs.mkdirp(new_spack_dir)\n\n prefix_a = os.path.join(new_spack_dir, f\"pkgA-{hash_a}\")\n libdir_a = os.path.join(prefix_a, \"lib\")\n fs.mkdirp(libdir_a)\n hash2prefix[hash_a] = str(prefix_a)\n\n prefix_b = os.path.join(new_spack_dir, f\"pkgB-{hash_b}\")\n libdir_b = os.path.join(prefix_b, \"lib\")\n fs.mkdirp(libdir_b)\n hash2prefix[hash_b] = str(prefix_b)\n\n prefix_c = os.path.join(new_spack_dir, f\"pkgC-{hash_c}\")\n libdir_c = os.path.join(prefix_c, \"lib\")\n libdir_cc = os.path.join(libdir_c, \"C\")\n fs.mkdirp(libdir_cc)\n hash2prefix[hash_c] = str(prefix_c)\n\n prefix_d = os.path.join(new_spack_dir, f\"pkgD-{hash_d}\")\n libdir_d = os.path.join(prefix_d, \"lib\")\n fs.mkdirp(libdir_d)\n hash2prefix[hash_d] = str(prefix_d)\n\n prefix_local = os.path.join(f\"{tmp_path}\", \"usr\", \"local\")\n libdir_local = os.path.join(prefix_local, \"lib\")\n fs.mkdirp(libdir_local)\n hash2prefix[hash_loco] = str(prefix_local)\n\n new_libnames = [\n os.path.join(libdir_a, libfile_a),\n os.path.join(libdir_b, libfile_b),\n os.path.join(libdir_cc, libfile_c),\n os.path.join(libdir_d, libfile_d),\n os.path.join(libdir_local, libfile_loco),\n ]\n\n for new_libname in new_libnames:\n with open(new_libname, \"a\", encoding=\"utf-8\"):\n os.utime(new_libname, None)\n\n prefix2prefix = dict()\n for prefix, hash in prefix2hash.items():\n prefix2prefix[prefix] = hash2prefix[hash]\n\n out_dict = _macho_find_paths(\n [oldlibdir_a, oldlibdir_b, oldlibdir_c, oldlibdir_cc, oldlibdir_local],\n [\n os.path.join(oldlibdir_a, libfile_a),\n os.path.join(oldlibdir_b, libfile_b),\n os.path.join(oldlibdir_local, libfile_loco),\n ],\n os.path.join(oldlibdir_cc, libfile_c),\n prefix2prefix,\n )\n assert out_dict == {\n oldlibdir_a: libdir_a,\n oldlibdir_b: libdir_b,\n oldlibdir_c: libdir_c,\n oldlibdir_cc: libdir_cc,\n libdir_local: libdir_local,\n os.path.join(oldlibdir_a, libfile_a): os.path.join(libdir_a, libfile_a),\n os.path.join(oldlibdir_b, libfile_b): os.path.join(libdir_b, libfile_b),\n os.path.join(oldlibdir_local, libfile_loco): os.path.join(libdir_local, libfile_loco),\n os.path.join(oldlibdir_cc, libfile_c): os.path.join(libdir_cc, libfile_c),\n }\n\n out_dict = _macho_find_paths(\n [oldlibdir_a, oldlibdir_b, oldlibdir_c, oldlibdir_cc, oldlibdir_local],\n [\n os.path.join(oldlibdir_a, libfile_a),\n os.path.join(oldlibdir_b, libfile_b),\n os.path.join(oldlibdir_cc, libfile_c),\n os.path.join(oldlibdir_local, libfile_loco),\n ],\n None,\n prefix2prefix,\n )\n assert out_dict == {\n oldlibdir_a: libdir_a,\n oldlibdir_b: libdir_b,\n oldlibdir_c: libdir_c,\n oldlibdir_cc: libdir_cc,\n libdir_local: libdir_local,\n os.path.join(oldlibdir_a, libfile_a): os.path.join(libdir_a, libfile_a),\n os.path.join(oldlibdir_b, libfile_b): os.path.join(libdir_b, libfile_b),\n os.path.join(oldlibdir_local, libfile_loco): os.path.join(libdir_local, libfile_loco),\n os.path.join(oldlibdir_cc, libfile_c): os.path.join(libdir_cc, libfile_c),\n }\n\n out_dict = _macho_find_paths(\n [oldlibdir_a, oldlibdir_b, oldlibdir_c, oldlibdir_cc, oldlibdir_local],\n [\n f\"@rpath/{libfile_a}\",\n f\"@rpath/{libfile_b}\",\n f\"@rpath/{libfile_c}\",\n f\"@rpath/{libfile_loco}\",\n ],\n None,\n prefix2prefix,\n )\n\n assert out_dict == {\n f\"@rpath/{libfile_a}\": f\"@rpath/{libfile_a}\",\n f\"@rpath/{libfile_b}\": f\"@rpath/{libfile_b}\",\n f\"@rpath/{libfile_c}\": f\"@rpath/{libfile_c}\",\n f\"@rpath/{libfile_loco}\": f\"@rpath/{libfile_loco}\",\n oldlibdir_a: libdir_a,\n oldlibdir_b: libdir_b,\n oldlibdir_c: libdir_c,\n oldlibdir_cc: libdir_cc,\n libdir_local: libdir_local,\n }\n\n out_dict = _macho_find_paths(\n [oldlibdir_a, oldlibdir_b, oldlibdir_d, oldlibdir_local],\n [f\"@rpath/{libfile_a}\", f\"@rpath/{libfile_b}\", f\"@rpath/{libfile_loco}\"],\n None,\n prefix2prefix,\n )\n assert out_dict == {\n f\"@rpath/{libfile_a}\": f\"@rpath/{libfile_a}\",\n f\"@rpath/{libfile_b}\": f\"@rpath/{libfile_b}\",\n f\"@rpath/{libfile_loco}\": f\"@rpath/{libfile_loco}\",\n oldlibdir_a: libdir_a,\n oldlibdir_b: libdir_b,\n oldlibdir_d: libdir_d,\n libdir_local: libdir_local,\n }\n\n\n@pytest.fixture()\ndef mock_download(monkeypatch):\n \"\"\"Mock a failing download strategy.\"\"\"\n\n class FailedDownloadStrategy(spack.fetch_strategy.FetchStrategy):\n def mirror_id(self):\n return None\n\n def fetch(self):\n raise spack.fetch_strategy.FailedDownloadError(\n urllib.error.URLError(\"This FetchStrategy always fails\")\n )\n\n @property\n def fake_fn(self):\n return FailedDownloadStrategy()\n\n monkeypatch.setattr(spack.package_base.PackageBase, \"fetcher\", fake_fn)\n\n\n@pytest.mark.parametrize(\n \"manual,instr\", [(False, False), (False, True), (True, False), (True, True)]\n)\n@pytest.mark.disable_clean_stage_check\ndef test_manual_download(mock_download, default_mock_concretization, monkeypatch, manual, instr):\n \"\"\"\n Ensure expected fetcher fail message based on manual download and instr.\n \"\"\"\n\n @property\n def _instr(pkg):\n return f\"Download instructions for {pkg.spec.name}\"\n\n spec = default_mock_concretization(\"pkg-a\")\n spec.package.manual_download = manual\n if instr:\n monkeypatch.setattr(spack.package_base.PackageBase, \"download_instr\", _instr)\n\n expected = spec.package.download_instr if manual else \"All fetchers failed\"\n with pytest.raises(spack.error.FetchError, match=expected):\n spec.package.do_fetch()\n\n\n@pytest.fixture()\ndef fetching_not_allowed(monkeypatch):\n class FetchingNotAllowed(spack.fetch_strategy.FetchStrategy):\n def mirror_id(self):\n return None\n\n def fetch(self):\n raise Exception(\"Sources are fetched but shouldn't have been\")\n\n monkeypatch.setattr(spack.package_base.PackageBase, \"fetcher\", FetchingNotAllowed())\n\n\ndef test_fetch_without_code_is_noop(default_mock_concretization, fetching_not_allowed):\n \"\"\"do_fetch for packages without code should be a no-op\"\"\"\n pkg = default_mock_concretization(\"pkg-a\").package\n pkg.has_code = False\n pkg.do_fetch()\n\n\ndef test_fetch_external_package_is_noop(default_mock_concretization, fetching_not_allowed):\n \"\"\"do_fetch for packages without code should be a no-op\"\"\"\n spec = default_mock_concretization(\"pkg-a\")\n spec.external_path = \"/some/where\"\n assert spec.external\n spec.package.do_fetch()\n\n\n@pytest.mark.parametrize(\n \"relocation_dict\",\n [\n {\"/foo/bar/baz\": \"/a/b/c\", \"/foo/bar\": \"/a/b\"},\n # Ensure correctness does not depend on the ordering of the dict\n {\"/foo/bar\": \"/a/b\", \"/foo/bar/baz\": \"/a/b/c\"},\n ],\n)\ndef test_macho_relocation_with_changing_projection(relocation_dict):\n \"\"\"Tests that prefix relocation is computed correctly when the prefixes to be relocated\n contain a directory and its subdirectories.\n\n This happens when relocating to a new place AND changing the store projection. In that case we\n might have a relocation dict like:\n\n /foo/bar/baz/ -> /a/b/c\n /foo/bar -> /a/b\n\n What we need to check is that we don't end up in situations where we relocate to a mixture of\n the two schemes, like /a/b/baz.\n \"\"\"\n original_rpath = \"/foo/bar/baz/abcdef\"\n result = _macho_find_paths(\n [original_rpath], deps=[], idpath=None, prefix_to_prefix=relocation_dict\n )\n assert result[original_rpath] == \"/a/b/c/abcdef\"\n" }, { "path": "lib/spack/spack/test/patch.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport collections\nimport filecmp\nimport os\nimport pathlib\nimport shutil\nimport sys\n\nimport pytest\n\nimport spack.concretize\nimport spack.deptypes as dt\nimport spack.error\nimport spack.fetch_strategy\nimport spack.patch\nimport spack.paths\nimport spack.repo\nimport spack.spec\nimport spack.stage\nimport spack.util.url as url_util\nfrom spack.llnl.util.filesystem import mkdirp, touch, working_dir\nfrom spack.spec import Spec\nfrom spack.stage import Stage\nfrom spack.util.executable import Executable\n\n# various sha256 sums (using variables for legibility)\n# many file based shas will differ between Windows and other platforms\n# due to the use of carriage returns ('\\r\\n') in Windows line endings\n\n# files with contents 'foo', 'bar', and 'baz'\nfoo_sha256 = (\n \"b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c\"\n if sys.platform != \"win32\"\n else \"bf874c7dd3a83cf370fdc17e496e341de06cd596b5c66dbf3c9bb7f6c139e3ee\"\n)\nbar_sha256 = (\n \"7d865e959b2466918c9863afca942d0fb89d7c9ac0c99bafc3749504ded97730\"\n if sys.platform != \"win32\"\n else \"556ddc69a75d0be0ecafc82cd4657666c8063f13d762282059c39ff5dbf18116\"\n)\nbaz_sha256 = (\n \"bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c\"\n if sys.platform != \"win32\"\n else \"d30392e66c636a063769cbb1db08cd3455a424650d4494db6379d73ea799582b\"\n)\nbiz_sha256 = (\n \"a69b288d7393261e613c276c6d38a01461028291f6e381623acc58139d01f54d\"\n if sys.platform != \"win32\"\n else \"2f2b087a8f84834fd03d4d1d5b43584011e869e4657504ef3f8b0a672a5c222e\"\n)\n\n# url patches\n# url shas are the same on Windows\nurl1_sha256 = \"abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234\"\nurl2_sha256 = \"1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd\"\nurl2_archive_sha256 = \"abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd\"\n\nplatform_url_sha = (\n \"252c0af58be3d90e5dc5e0d16658434c9efa5d20a5df6c10bf72c2d77f780866\"\n if sys.platform != \"win32\"\n else \"ecf44a8244a486e9ef5f72c6cb622f99718dcd790707ac91af0b8c9a4ab7a2bb\"\n)\n\n\n@pytest.fixture()\ndef mock_patch_stage(tmp_path_factory: pytest.TempPathFactory, monkeypatch):\n # Don't disrupt the spack install directory with tests.\n mock_path = str(tmp_path_factory.mktemp(\"mock-patch-stage\"))\n monkeypatch.setattr(spack.stage, \"_stage_root\", mock_path)\n return mock_path\n\n\ndata_path = os.path.join(spack.paths.test_path, \"data\", \"patch\")\n\n\n@pytest.mark.not_on_windows(\"Line ending conflict on Windows\")\n@pytest.mark.parametrize(\n \"filename, sha256, archive_sha256\",\n [\n # compressed patch -- needs sha256 and archive_256\n (\n os.path.join(data_path, \"foo.tgz\"),\n \"252c0af58be3d90e5dc5e0d16658434c9efa5d20a5df6c10bf72c2d77f780866\",\n \"4e8092a161ec6c3a1b5253176fcf33ce7ba23ee2ff27c75dbced589dabacd06e\",\n ),\n # uncompressed patch -- needs only sha256\n (os.path.join(data_path, \"foo.patch\"), platform_url_sha, None),\n ],\n)\ndef test_url_patch(mock_packages, mock_patch_stage, filename, sha256, archive_sha256, config):\n # Make a patch object\n url = url_util.path_to_file_url(filename)\n s = spack.concretize.concretize_one(\"patch\")\n\n # make a stage\n with Stage(url) as stage: # TODO: url isn't used; maybe refactor Stage\n stage.mirror_path = mock_patch_stage\n\n mkdirp(stage.source_path)\n with working_dir(stage.source_path):\n # write a file to be patched\n with open(\"foo.txt\", \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nfirst line\nsecond line\n\"\"\"\n )\n # save it for later comparison\n shutil.copyfile(\"foo.txt\", \"foo-original.txt\")\n # write the expected result of patching.\n with open(\"foo-expected.txt\", \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nzeroth line\nfirst line\nthird line\n\"\"\"\n )\n # apply the patch and compare files\n patch = spack.patch.UrlPatch(s.package, url, sha256=sha256, archive_sha256=archive_sha256)\n patch_stage = Stage(patch.fetcher())\n with patch_stage:\n patch_stage.create()\n patch_stage.fetch()\n patch_stage.expand_archive()\n spack.patch.apply_patch(\n stage.source_path,\n patch_stage.single_file,\n patch.level,\n patch.working_dir,\n patch.reverse,\n )\n\n with working_dir(stage.source_path):\n assert filecmp.cmp(\"foo.txt\", \"foo-expected.txt\")\n\n # apply the patch in reverse and compare files\n patch = spack.patch.UrlPatch(\n s.package, url, sha256=sha256, archive_sha256=archive_sha256, reverse=True\n )\n patch_stage = Stage(patch.fetcher())\n with patch_stage:\n patch_stage.create()\n patch_stage.fetch()\n patch_stage.expand_archive()\n spack.patch.apply_patch(\n stage.source_path,\n patch_stage.single_file,\n patch.level,\n patch.working_dir,\n patch.reverse,\n )\n\n with working_dir(stage.source_path):\n assert filecmp.cmp(\"foo.txt\", \"foo-original.txt\")\n\n\ndef test_patch_in_spec(mock_packages, config):\n \"\"\"Test whether patches in a package appear in the spec.\"\"\"\n spec = spack.concretize.concretize_one(\"patch\")\n assert \"patches\" in list(spec.variants.keys())\n\n # Here the order is bar, foo, baz. Note that MV variants order\n # lexicographically based on the hash, not on the position of the\n # patch directive.\n assert (bar_sha256, foo_sha256, baz_sha256) == spec.variants[\"patches\"].value\n\n assert (foo_sha256, bar_sha256, baz_sha256) == tuple(\n spec.variants[\"patches\"]._patches_in_order_of_appearance\n )\n\n\ndef test_stale_patch_cache_falls_back_to_fresh(mock_packages, config):\n \"\"\"spec.patches returns correct patches even when the stale in-memory cache is wrong.\"\"\"\n spec = spack.concretize.concretize_one(\"patch@=1.0\")\n pkg_cls = spack.repo.PATH.get_pkg_class(\"patch\")\n\n # Inject a stale PatchCache: foo_sha256 points to a non-existent patch file\n stale_cache = spack.patch.PatchCache(repository=spack.repo.PATH)\n stale_cache.index = {\n foo_sha256: {\n pkg_cls.fullname: {\n \"owner\": pkg_cls.fullname,\n \"relative_path\": \"stale_wrong.patch\",\n \"level\": 1,\n \"working_dir\": \".\",\n \"reverse\": False,\n }\n }\n }\n spack.repo.PATH._patch_index = stale_cache\n spack.repo.PATH._index_is_fresh = False\n\n patches = spec.patches\n\n assert len(patches) == 2\n assert {p.relative_path for p in patches} == {\"foo.patch\", \"baz.patch\"}\n\n\ndef test_patch_mixed_versions_subset_constraint(mock_packages, config):\n \"\"\"If we have a package with mixed x.y and x.y.z versions, make sure that\n a patch applied to a version range of x.y.z versions is not applied to\n an x.y version.\n \"\"\"\n spec1 = spack.concretize.concretize_one(\"patch@1.0.1\")\n assert biz_sha256 in spec1.variants[\"patches\"].value\n\n spec2 = spack.concretize.concretize_one(\"patch@=1.0\")\n assert biz_sha256 not in spec2.variants[\"patches\"].value\n\n\ndef test_patch_order(mock_packages, config):\n spec = spack.concretize.concretize_one(\"dep-diamond-patch-top\")\n\n mid2_sha256 = (\n \"mid21234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234\"\n if sys.platform != \"win32\"\n else \"mid21234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234\"\n )\n mid1_sha256 = (\n \"0b62284961dab49887e31319843431ee5b037382ac02c4fe436955abef11f094\"\n if sys.platform != \"win32\"\n else \"aeb16c4dec1087e39f2330542d59d9b456dd26d791338ae6d80b6ffd10c89dfa\"\n )\n top_sha256 = (\n \"f7de2947c64cb6435e15fb2bef359d1ed5f6356b2aebb7b20535e3772904e6db\"\n if sys.platform != \"win32\"\n else \"ff34cb21271d16dbf928374f610bb5dd593d293d311036ddae86c4846ff79070\"\n )\n\n dep = spec[\"patch\"]\n patch_order = dep.variants[\"patches\"]._patches_in_order_of_appearance\n # 'mid2' comes after 'mid1' alphabetically\n # 'top' comes after 'mid1'/'mid2' alphabetically\n # 'patch' comes last of all specs in the dag, alphabetically, so the\n # patches of 'patch' to itself are applied last. The patches applied by\n # 'patch' are ordered based on their appearance in the package.py file\n expected_order = (mid1_sha256, mid2_sha256, top_sha256, foo_sha256, bar_sha256, baz_sha256)\n\n assert expected_order == tuple(patch_order)\n\n\ndef test_nested_directives(mock_packages):\n \"\"\"Ensure pkg data structures are set up properly by nested directives.\"\"\"\n # this ensures that the patch() directive results were removed\n # properly from the DirectiveMeta._directives_to_be_executed list\n package = spack.repo.PATH.get_pkg_class(\"patch-several-dependencies\")\n assert len(package.patches) == 0\n\n # this ensures that results of dependency patches were properly added\n # to Dependency objects.\n\n # package.dependencies is keyed by three when clauses\n assert package.dependencies.keys() == {Spec(), Spec(\"+foo\"), Spec(\"@1.0\")}\n\n # fake and libelf are unconditional dependencies\n when_unconditional = package.dependencies[Spec()]\n assert when_unconditional.keys() == {\"fake\", \"libelf\"}\n # fake has two unconditional URL patches\n assert when_unconditional[\"fake\"].patches.keys() == {Spec()}\n assert len(when_unconditional[\"fake\"].patches[Spec()]) == 2\n # libelf has one unconditional patch\n assert when_unconditional[\"libelf\"].patches.keys() == {Spec()}\n assert len(when_unconditional[\"libelf\"].patches[Spec()]) == 1\n\n # there are multiple depends_on directives for libelf under the +foo when clause; these must be\n # reduced to a single Dependency object.\n when_foo = package.dependencies[Spec(\"+foo\")]\n assert when_foo.keys() == {\"libelf\"}\n assert when_foo[\"libelf\"].spec == Spec(\"libelf@0.8.10\")\n assert when_foo[\"libelf\"].depflag == dt.BUILD | dt.LINK\n # there is one unconditional patch for libelf under the +foo when clause\n assert len(when_foo[\"libelf\"].patches) == 1\n assert len(when_foo[\"libelf\"].patches[Spec()]) == 1\n\n # libdwarf is a dependency when @1.0 with two patches applied from a single depends_on\n # statement, one conditional on the libdwarf version\n when_1_0 = package.dependencies[Spec(\"@1.0\")]\n assert when_1_0.keys() == {\"libdwarf\"}\n assert when_1_0[\"libdwarf\"].patches.keys() == {Spec(), Spec(\"@20111030\")}\n assert len(when_1_0[\"libdwarf\"].patches[Spec()]) == 1\n assert len(when_1_0[\"libdwarf\"].patches[Spec(\"@20111030\")]) == 1\n\n\n@pytest.mark.not_on_windows(\"Test requires Autotools\")\ndef test_patched_dependency(mock_packages, install_mockery, mock_fetch):\n \"\"\"Test whether patched dependencies work.\"\"\"\n spec = spack.concretize.concretize_one(\"patch-a-dependency\")\n assert \"patches\" in list(spec[\"libelf\"].variants.keys())\n\n # make sure the patch makes it into the dependency spec\n t_sha = (\n \"c45c1564f70def3fc1a6e22139f62cb21cd190cc3a7dbe6f4120fa59ce33dcb8\"\n if sys.platform != \"win32\"\n else \"3c5b65abcd6a3b2c714dbf7c31ff65fe3748a1adc371f030c283007ca5534f11\"\n )\n assert (t_sha,) == spec[\"libelf\"].variants[\"patches\"].value\n\n # make sure the patch in the dependent's directory is applied to the\n # dependency\n libelf = spec[\"libelf\"]\n pkg = libelf.package\n pkg.do_patch()\n with pkg.stage:\n with working_dir(pkg.stage.source_path):\n # output a Makefile with 'echo Patched!' as the default target\n configure = Executable(\"./configure\")\n configure()\n\n # Make sure the Makefile contains the patched text\n with open(\"Makefile\", encoding=\"utf-8\") as mf:\n assert \"Patched!\" in mf.read()\n\n\ndef trigger_bad_patch(pkg):\n if not os.path.isdir(pkg.stage.source_path):\n os.makedirs(pkg.stage.source_path)\n bad_file = os.path.join(pkg.stage.source_path, \".spack_patch_failed\")\n touch(bad_file)\n return bad_file\n\n\ndef test_patch_failure_develop_spec_exits_gracefully(\n mock_packages, install_mockery, mock_fetch, tmp_path: pathlib.Path, mock_stage\n):\n \"\"\"ensure that a failing patch does not trigger exceptions for develop specs\"\"\"\n\n spec = spack.concretize.concretize_one(f\"patch-a-dependency ^libelf dev_path={tmp_path}\")\n libelf = spec[\"libelf\"]\n assert \"patches\" in list(libelf.variants.keys())\n pkg = libelf.package\n with pkg.stage:\n bad_patch_indicator = trigger_bad_patch(pkg)\n assert os.path.isfile(bad_patch_indicator)\n pkg.do_patch()\n # success if no exceptions raised\n\n\ndef test_patch_failure_restages(mock_packages, install_mockery, mock_fetch):\n \"\"\"\n ensure that a failing patch does not trigger exceptions\n for non-develop specs and the source gets restaged\n \"\"\"\n spec = spack.concretize.concretize_one(\"patch-a-dependency\")\n pkg = spec[\"libelf\"].package\n with pkg.stage:\n bad_patch_indicator = trigger_bad_patch(pkg)\n assert os.path.isfile(bad_patch_indicator)\n pkg.do_patch()\n assert not os.path.isfile(bad_patch_indicator)\n\n\ndef test_multiple_patched_dependencies(mock_packages, config):\n \"\"\"Test whether multiple patched dependencies work.\"\"\"\n spec = spack.concretize.concretize_one(\"patch-several-dependencies\")\n\n # basic patch on libelf\n assert \"patches\" in list(spec[\"libelf\"].variants.keys())\n # foo\n assert (foo_sha256,) == spec[\"libelf\"].variants[\"patches\"].value\n\n # URL patches\n assert \"patches\" in list(spec[\"fake\"].variants.keys())\n # urlpatch.patch, urlpatch.patch.gz\n assert (url2_sha256, url1_sha256) == spec[\"fake\"].variants[\"patches\"].value\n\n\ndef test_conditional_patched_dependencies(mock_packages, config):\n \"\"\"Test whether conditional patched dependencies work.\"\"\"\n spec = spack.concretize.concretize_one(\"patch-several-dependencies @1.0\")\n\n # basic patch on libelf\n assert \"patches\" in list(spec[\"libelf\"].variants.keys())\n # foo\n assert (foo_sha256,) == spec[\"libelf\"].variants[\"patches\"].value\n\n # conditional patch on libdwarf\n assert \"patches\" in list(spec[\"libdwarf\"].variants.keys())\n # bar\n assert (bar_sha256,) == spec[\"libdwarf\"].variants[\"patches\"].value\n # baz is conditional on libdwarf version\n assert baz_sha256 not in spec[\"libdwarf\"].variants[\"patches\"].value\n\n # URL patches\n assert \"patches\" in list(spec[\"fake\"].variants.keys())\n # urlpatch.patch, urlpatch.patch.gz\n assert (url2_sha256, url1_sha256) == spec[\"fake\"].variants[\"patches\"].value\n\n\ndef check_multi_dependency_patch_specs(\n libelf,\n libdwarf,\n fake,\n owner,\n package_dir, # specs\n): # parent spec properties\n \"\"\"Validate patches on dependencies of patch-several-dependencies.\"\"\"\n # basic patch on libelf\n assert \"patches\" in list(libelf.variants.keys())\n # foo\n assert foo_sha256 in libelf.variants[\"patches\"].value\n\n # conditional patch on libdwarf\n assert \"patches\" in list(libdwarf.variants.keys())\n # bar\n assert bar_sha256 in libdwarf.variants[\"patches\"].value\n # baz is conditional on libdwarf version (no guarantee on order w/conds)\n assert baz_sha256 in libdwarf.variants[\"patches\"].value\n\n def get_patch(spec, ending):\n return next(p for p in spec.patches if p.path_or_url.endswith(ending))\n\n # make sure file patches are reconstructed properly\n foo_patch = get_patch(libelf, \"foo.patch\")\n bar_patch = get_patch(libdwarf, \"bar.patch\")\n baz_patch = get_patch(libdwarf, \"baz.patch\")\n\n assert foo_patch.owner == owner\n assert foo_patch.path == os.path.join(package_dir, \"foo.patch\")\n assert foo_patch.sha256 == foo_sha256\n\n assert bar_patch.owner == \"builtin_mock.patch-several-dependencies\"\n assert bar_patch.path == os.path.join(package_dir, \"bar.patch\")\n assert bar_patch.sha256 == bar_sha256\n\n assert baz_patch.owner == \"builtin_mock.patch-several-dependencies\"\n assert baz_patch.path == os.path.join(package_dir, \"baz.patch\")\n assert baz_patch.sha256 == baz_sha256\n\n # URL patches\n assert \"patches\" in list(fake.variants.keys())\n # urlpatch.patch, urlpatch.patch.gz\n assert (url2_sha256, url1_sha256) == fake.variants[\"patches\"].value\n\n url1_patch = get_patch(fake, \"urlpatch.patch\")\n url2_patch = get_patch(fake, \"urlpatch2.patch.gz\")\n\n assert url1_patch.owner == \"builtin_mock.patch-several-dependencies\"\n assert url1_patch.url == \"http://example.com/urlpatch.patch\"\n assert url1_patch.sha256 == url1_sha256\n\n assert url2_patch.owner == \"builtin_mock.patch-several-dependencies\"\n assert url2_patch.url == \"http://example.com/urlpatch2.patch.gz\"\n assert url2_patch.sha256 == url2_sha256\n assert url2_patch.archive_sha256 == url2_archive_sha256\n\n\ndef test_conditional_patched_deps_with_conditions(mock_packages, config):\n \"\"\"Test whether conditional patched dependencies with conditions work.\"\"\"\n spec = spack.concretize.concretize_one(\n Spec(\"patch-several-dependencies @1.0 ^libdwarf@20111030\")\n )\n\n libelf = spec[\"libelf\"]\n libdwarf = spec[\"libdwarf\"]\n fake = spec[\"fake\"]\n\n check_multi_dependency_patch_specs(\n libelf, libdwarf, fake, \"builtin_mock.patch-several-dependencies\", spec.package.package_dir\n )\n\n\ndef test_write_and_read_sub_dags_with_patched_deps(mock_packages, config):\n \"\"\"Test whether patched dependencies are still correct after writing and\n reading a sub-DAG of a concretized Spec.\n \"\"\"\n spec = spack.concretize.concretize_one(\n Spec(\"patch-several-dependencies @1.0 ^libdwarf@20111030\")\n )\n\n # write to YAML and read back in -- new specs will *only* contain\n # their sub-DAGs, and won't contain the dependent that patched them\n libelf = spack.spec.Spec.from_yaml(spec[\"libelf\"].to_yaml())\n libdwarf = spack.spec.Spec.from_yaml(spec[\"libdwarf\"].to_yaml())\n fake = spack.spec.Spec.from_yaml(spec[\"fake\"].to_yaml())\n\n # make sure we can still read patches correctly for these specs\n check_multi_dependency_patch_specs(\n libelf, libdwarf, fake, \"builtin_mock.patch-several-dependencies\", spec.package.package_dir\n )\n\n\ndef test_patch_no_file():\n # Give it the attributes we need to construct the error message\n FakePackage = collections.namedtuple(\"FakePackage\", [\"name\", \"namespace\", \"fullname\"])\n fp = FakePackage(\"fake-package\", \"test\", \"fake-package\")\n with pytest.raises(ValueError, match=\"FilePatch:\"):\n spack.patch.FilePatch(fp, \"nonexistent_file\", 0, \"\")\n\n patch = spack.patch.Patch(fp, \"nonexistent_file\", 0, \"\")\n patch.path = \"test\"\n with pytest.raises(spack.error.NoSuchPatchError, match=\"No such patch:\"):\n spack.patch.apply_patch(Stage(\"https://example.com/foo.patch\").source_path, patch.path)\n\n\ndef test_patch_no_sha256():\n # Give it the attributes we need to construct the error message\n FakePackage = collections.namedtuple(\"FakePackage\", [\"name\", \"namespace\", \"fullname\"])\n fp = FakePackage(\"fake-package\", \"test\", \"fake-package\")\n url = url_util.path_to_file_url(\"foo.tgz\")\n match = \"Compressed patches require 'archive_sha256' and patch 'sha256' attributes: file://\"\n with pytest.raises(spack.error.PatchDirectiveError, match=match):\n spack.patch.UrlPatch(fp, url, sha256=\"\", archive_sha256=\"\")\n match = \"URL patches require a sha256 checksum\"\n with pytest.raises(spack.error.PatchDirectiveError, match=match):\n spack.patch.UrlPatch(fp, url, sha256=\"\", archive_sha256=\"abc\")\n\n\n@pytest.mark.parametrize(\"level\", [-1, 0.0, \"1\"])\ndef test_invalid_level(level):\n # Give it the attributes we need to construct the error message\n FakePackage = collections.namedtuple(\"FakePackage\", [\"name\", \"namespace\"])\n fp = FakePackage(\"fake-package\", \"test\")\n with pytest.raises(ValueError, match=\"Patch level needs to be a non-negative integer.\"):\n spack.patch.Patch(fp, \"nonexistent_file\", level, \"\")\n\n\ndef test_equality():\n FakePackage = collections.namedtuple(\"FakePackage\", [\"name\", \"namespace\", \"fullname\"])\n fp = FakePackage(\"fake-package\", \"test\", \"fake-package\")\n patch1 = spack.patch.UrlPatch(fp, \"nonexistent_url1\", sha256=\"abc\")\n patch2 = spack.patch.UrlPatch(fp, \"nonexistent_url2\", sha256=\"def\")\n assert patch1 == patch1\n assert patch1 != patch2\n assert patch1 != \"not a patch\"\n\n\ndef test_sha256_setter(mock_packages, mock_patch_stage, config):\n path = os.path.join(data_path, \"foo.patch\")\n s = spack.concretize.concretize_one(\"patch\")\n patch = spack.patch.FilePatch(s.package, path, level=1, working_dir=\".\")\n patch.sha256 = \"abc\"\n\n\ndef test_invalid_from_dict(mock_packages, config):\n dictionary = {}\n with pytest.raises(ValueError, match=\"Invalid patch dictionary:\"):\n spack.patch.from_dict(dictionary, mock_packages)\n\n dictionary = {\"owner\": \"patch\"}\n with pytest.raises(ValueError, match=\"Invalid patch dictionary:\"):\n spack.patch.from_dict(dictionary, mock_packages)\n\n dictionary = {\n \"owner\": \"patch\",\n \"relative_path\": \"foo.patch\",\n \"level\": 1,\n \"working_dir\": \".\",\n \"reverse\": False,\n \"sha256\": bar_sha256,\n }\n with pytest.raises(spack.fetch_strategy.ChecksumError, match=\"sha256 checksum failed for\"):\n spack.patch.from_dict(dictionary, mock_packages)\n" }, { "path": "lib/spack/spack/test/permissions.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport pathlib\nimport stat\n\nimport pytest\n\nimport spack.llnl.util.filesystem as fs\nfrom spack.util.file_permissions import InvalidPermissionsError, set_permissions\n\npytestmark = pytest.mark.not_on_windows(\"chmod unsupported on Windows\")\n\n\ndef ensure_known_group(path):\n \"\"\"Ensure that the group of a file is one that's actually in our group list.\n\n On systems with remote groups, the primary user group may be remote and may not\n exist on the local system (i.e., it might just be a number). Trying to use chmod to\n setgid can fail silently in situations like this.\n \"\"\"\n uid = os.getuid()\n gid = fs.group_ids(uid)[0]\n os.chown(path, uid, gid)\n\n\ndef test_chmod_real_entries_ignores_suid_sgid(tmp_path: pathlib.Path):\n path = tmp_path / \"file\"\n path.touch()\n mode = stat.S_ISUID | stat.S_ISGID | stat.S_ISVTX\n os.chmod(str(path), mode)\n mode = os.stat(str(path)).st_mode # adds a high bit we aren't concerned with\n\n perms = stat.S_IRWXU\n set_permissions(str(path), perms)\n\n assert os.stat(str(path)).st_mode == mode | perms & ~stat.S_IXUSR\n\n\ndef test_chmod_rejects_group_writable_suid(tmp_path: pathlib.Path):\n path = tmp_path / \"file\"\n path.touch()\n mode = stat.S_ISUID\n fs.chmod_x(str(path), mode)\n\n perms = stat.S_IWGRP\n with pytest.raises(InvalidPermissionsError):\n set_permissions(str(path), perms)\n\n\ndef test_chmod_rejects_world_writable_suid(tmp_path: pathlib.Path):\n path = tmp_path / \"file\"\n path.touch()\n mode = stat.S_ISUID\n fs.chmod_x(str(path), mode)\n\n perms = stat.S_IWOTH\n with pytest.raises(InvalidPermissionsError):\n set_permissions(str(path), perms)\n\n\ndef test_chmod_rejects_world_writable_sgid(tmp_path: pathlib.Path):\n path = tmp_path / \"file\"\n path.touch()\n ensure_known_group(str(path))\n\n mode = stat.S_ISGID\n fs.chmod_x(str(path), mode)\n\n perms = stat.S_IWOTH\n with pytest.raises(InvalidPermissionsError):\n set_permissions(str(path), perms)\n" }, { "path": "lib/spack/spack/test/projections.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nfrom datetime import date\n\nimport spack.projections\nimport spack.spec\n\n\ndef test_projection_expansion(mock_packages, monkeypatch):\n \"\"\"Test that env variables and spack config variables are expanded in projections\"\"\"\n\n monkeypatch.setenv(\"FOO_ENV_VAR\", \"test-string\")\n projections = {\"all\": \"{name}-{version}/$FOO_ENV_VAR/$date\"}\n spec = spack.spec.Spec(\"fake@1.0\")\n projection = spack.projections.get_projection(projections, spec)\n assert \"{name}-{version}/test-string/%s\" % date.today().strftime(\"%Y-%m-%d\") == projection\n" }, { "path": "lib/spack/spack/test/provider_index.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Tests for provider index cache files.\n\nTests assume that mock packages provide this::\n\n {'blas': {\n blas: set([netlib-blas, openblas, openblas-with-lapack])},\n 'lapack': {lapack: set([netlib-lapack, openblas-with-lapack])},\n 'mpi': {mpi@:1: set([mpich@:1]),\n mpi@:2.0: set([mpich2]),\n mpi@:2.1: set([mpich2@1.1:]),\n mpi@:2.2: set([mpich2@1.2:]),\n mpi@:3: set([mpich@3:]),\n mpi@:10.0: set([zmpi])},\n 'stuff': {stuff: set([externalvirtual])}}\n\"\"\"\n\nimport io\n\nimport spack.repo\nfrom spack.provider_index import ProviderIndex\nfrom spack.spec import Spec\n\n\ndef test_provider_index_round_trip(mock_packages):\n p = ProviderIndex(specs=spack.repo.all_package_names(), repository=spack.repo.PATH)\n\n ostream = io.StringIO()\n p.to_json(ostream)\n\n istream = io.StringIO(ostream.getvalue())\n q = ProviderIndex.from_json(istream, repository=spack.repo.PATH)\n\n assert p == q\n\n\ndef test_providers_for_simple(mock_packages):\n p = ProviderIndex(specs=spack.repo.all_package_names(), repository=spack.repo.PATH)\n\n blas_providers = p.providers_for(\"blas\")\n assert Spec(\"netlib-blas\") in blas_providers\n assert Spec(\"openblas\") in blas_providers\n assert Spec(\"openblas-with-lapack\") in blas_providers\n\n lapack_providers = p.providers_for(\"lapack\")\n assert Spec(\"netlib-lapack\") in lapack_providers\n assert Spec(\"openblas-with-lapack\") in lapack_providers\n\n\ndef test_mpi_providers(mock_packages):\n p = ProviderIndex(specs=spack.repo.all_package_names(), repository=spack.repo.PATH)\n\n mpi_2_providers = p.providers_for(\"mpi@2\")\n assert Spec(\"mpich2\") in mpi_2_providers\n assert Spec(\"mpich@3:\") in mpi_2_providers\n\n mpi_3_providers = p.providers_for(\"mpi@3\")\n assert Spec(\"mpich2\") not in mpi_3_providers\n assert Spec(\"mpich@3:\") in mpi_3_providers\n assert Spec(\"zmpi\") in mpi_3_providers\n\n\ndef test_equal(mock_packages):\n p = ProviderIndex(specs=spack.repo.all_package_names(), repository=spack.repo.PATH)\n q = ProviderIndex(specs=spack.repo.all_package_names(), repository=spack.repo.PATH)\n assert p == q\n\n\ndef test_copy(mock_packages):\n p = ProviderIndex(specs=spack.repo.all_package_names(), repository=spack.repo.PATH)\n q = p.copy()\n assert p == q\n\n\ndef test_remove_providers(mock_packages):\n \"\"\"Test removing providers from the index.\"\"\"\n p = ProviderIndex(specs=[\"mpich\"], repository=spack.repo.PATH)\n # Check that mpich is a provider for mpi\n providers = p.providers_for(\"mpi\")\n assert any(spec.name == \"mpich\" for spec in providers)\n p.remove_providers({\"mpich\"})\n # After removal, mpich should no longer be a provider for mpi\n providers = p.providers_for(\"mpi\")\n assert not any(spec.name == \"mpich\" for spec in providers)\n" }, { "path": "lib/spack/spack/test/relocate.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport pathlib\nimport re\nimport shutil\nimport subprocess\nimport tempfile\n\nimport pytest\n\nimport spack.platforms\nimport spack.relocate\nimport spack.relocate_text as relocate_text\nimport spack.util.executable\n\npytestmark = pytest.mark.not_on_windows(\"Tests fail on Windows\")\n\n\ndef skip_unless_linux(f):\n return pytest.mark.skipif(\n str(spack.platforms.real_host()) != \"linux\",\n reason=\"implementation currently requires linux\",\n )(f)\n\n\ndef rpaths_for(new_binary):\n \"\"\"Return the RPATHs or RUNPATHs of a binary.\"\"\"\n patchelf = spack.util.executable.which(\"patchelf\", required=True)\n output = patchelf(\"--print-rpath\", str(new_binary), output=str)\n return output.strip()\n\n\ndef text_in_bin(text, binary):\n with open(str(binary), \"rb\") as f:\n data = f.read()\n f.seek(0)\n pat = re.compile(text.encode(\"utf-8\"))\n if not pat.search(data):\n return False\n return True\n\n\n@pytest.fixture()\ndef make_dylib(tmp_path_factory: pytest.TempPathFactory):\n \"\"\"Create a shared library with unfriendly qualities.\n\n - Writes the same rpath twice\n - Writes its install path as an absolute path\n \"\"\"\n cc = spack.util.executable.which(\"cc\", required=True)\n\n def _factory(abs_install_name=\"abs\", extra_rpaths=[]):\n assert all(extra_rpaths)\n\n tmpdir = tmp_path_factory.mktemp(\n abs_install_name + \"-\".join(extra_rpaths).replace(\"/\", \"\")\n )\n src = tmpdir / \"foo.c\"\n src.write_text(\"int foo() { return 1; }\\n\")\n\n filename = \"foo.dylib\"\n lib = tmpdir / filename\n\n args = [\"-shared\", str(src), \"-o\", str(lib)]\n rpaths = list(extra_rpaths)\n if abs_install_name.startswith(\"abs\"):\n args += [\"-install_name\", str(lib)]\n else:\n args += [\"-install_name\", \"@rpath/\" + filename]\n\n if abs_install_name.endswith(\"rpath\"):\n rpaths.append(str(tmpdir))\n\n args.extend(\"-Wl,-rpath,\" + s for s in rpaths)\n\n cc(*args)\n\n return (str(tmpdir), filename)\n\n return _factory\n\n\n@pytest.fixture()\ndef make_object_file(tmp_path: pathlib.Path):\n cc = spack.util.executable.which(\"cc\", required=True)\n\n def _factory():\n src = tmp_path / \"bar.c\"\n src.write_text(\"int bar() { return 2; }\\n\")\n\n filename = \"bar.o\"\n lib = tmp_path / filename\n\n args = [\"-c\", str(src), \"-o\", str(lib)]\n\n cc(*args)\n\n return (str(tmp_path), filename)\n\n return _factory\n\n\n@pytest.fixture()\ndef copy_binary(prefix_like):\n \"\"\"Returns a function that copies a binary somewhere and\n returns the new location.\n \"\"\"\n\n def _copy_somewhere(orig_binary):\n # Create a temporary directory\n temp_dir = pathlib.Path(tempfile.mkdtemp())\n new_root = temp_dir / prefix_like\n new_root.mkdir(parents=True, exist_ok=True)\n new_binary = new_root / \"main.x\"\n shutil.copy(str(orig_binary), str(new_binary))\n return new_binary\n\n return _copy_somewhere\n\n\n@pytest.mark.requires_executables(\"patchelf\", \"gcc\")\n@skip_unless_linux\ndef test_relocate_text_bin(binary_with_rpaths, prefix_like):\n prefix = \"/usr/\" + prefix_like\n prefix_bytes = prefix.encode(\"utf-8\")\n new_prefix = \"/foo/\" + prefix_like\n new_prefix_bytes = new_prefix.encode(\"utf-8\")\n # Compile an \"Hello world!\" executable and set RPATHs\n executable = binary_with_rpaths(rpaths=[prefix + \"/lib\", prefix + \"/lib64\"])\n\n # Relocate the RPATHs\n spack.relocate.relocate_text_bin([str(executable)], {prefix_bytes: new_prefix_bytes})\n\n # Some compilers add rpaths so ensure changes included in final result\n assert \"%s/lib:%s/lib64\" % (new_prefix, new_prefix) in rpaths_for(executable)\n\n\n@pytest.mark.requires_executables(\"patchelf\", \"gcc\")\n@skip_unless_linux\ndef test_relocate_elf_binaries_absolute_paths(binary_with_rpaths, copy_binary, prefix_tmpdir):\n # Create an executable, set some RPATHs, copy it to another location\n lib_dir = prefix_tmpdir / \"lib\"\n lib_dir.mkdir()\n orig_binary = binary_with_rpaths(rpaths=[str(lib_dir), \"/usr/lib64\"])\n new_binary = copy_binary(orig_binary)\n\n spack.relocate.relocate_elf_binaries(\n binaries=[str(new_binary)], prefix_to_prefix={str(orig_binary.parent): \"/foo\"}\n )\n\n # Some compilers add rpaths so ensure changes included in final result\n assert \"/foo/lib:/usr/lib64\" in rpaths_for(new_binary)\n\n\n@pytest.mark.requires_executables(\"patchelf\", \"gcc\")\n@skip_unless_linux\ndef test_relocate_text_bin_with_message(binary_with_rpaths, copy_binary, prefix_tmpdir):\n lib_dir = prefix_tmpdir / \"lib\"\n lib_dir.mkdir()\n lib64_dir = prefix_tmpdir / \"lib64\"\n lib64_dir.mkdir()\n orig_binary = binary_with_rpaths(\n rpaths=[str(lib_dir), str(lib64_dir), \"/opt/local/lib\"], message=str(prefix_tmpdir)\n )\n new_binary = copy_binary(orig_binary)\n\n # Check original directory is in the executable and the new one is not\n assert text_in_bin(str(prefix_tmpdir), new_binary)\n assert not text_in_bin(str(new_binary.parent), new_binary)\n\n # Check this call succeed\n orig_path_bytes = str(orig_binary.parent).encode(\"utf-8\")\n new_path_bytes = str(new_binary.parent).encode(\"utf-8\")\n\n spack.relocate.relocate_text_bin([str(new_binary)], {orig_path_bytes: new_path_bytes})\n\n # Check original directory is not there anymore and it was\n # substituted with the new one\n assert not text_in_bin(str(prefix_tmpdir), new_binary)\n assert text_in_bin(str(new_binary.parent), new_binary)\n\n\ndef test_relocate_text_bin_raise_if_new_prefix_is_longer(tmp_path: pathlib.Path):\n short_prefix = b\"/short\"\n long_prefix = b\"/much/longer\"\n fpath = str(tmp_path / \"fakebin\")\n with open(fpath, \"w\", encoding=\"utf-8\") as f:\n f.write(\"/short\")\n with pytest.raises(relocate_text.BinaryTextReplaceError):\n spack.relocate.relocate_text_bin([fpath], {short_prefix: long_prefix})\n\n\n@pytest.mark.requires_executables(\"install_name_tool\", \"cc\")\ndef test_fixup_macos_rpaths(make_dylib, make_object_file):\n # Get Apple Clang major version for XCode 15+ linker behavior\n try:\n result = subprocess.check_output([\"cc\", \"--version\"], universal_newlines=True)\n version_match = re.search(r\"Apple clang version (\\d+)\", result)\n assert version_match, \"Apple Clang version not found in output\"\n xcode_major_version = int(version_match.group(1))\n except Exception:\n pytest.xfail(\"cannot determine Apple Clang major version\")\n return\n\n # For each of these tests except for the \"correct\" case, the first fixup\n # should make changes, and the second fixup should be a null-op.\n fixup_rpath = spack.relocate.fixup_macos_rpath\n\n no_rpath = []\n duplicate_rpaths = [\"/usr\", \"/usr\"]\n bad_rpath = [\"/nonexistent/path\"]\n\n # Non-relocatable library id and duplicate rpaths\n (root, filename) = make_dylib(\"abs\", duplicate_rpaths)\n # XCode 15 ships a new linker that takes care of deduplication\n if xcode_major_version < 15:\n assert fixup_rpath(root, filename)\n assert not fixup_rpath(root, filename)\n\n # Hardcoded but relocatable library id (but we do NOT relocate)\n (root, filename) = make_dylib(\"abs_with_rpath\", no_rpath)\n assert not fixup_rpath(root, filename)\n\n # Library id uses rpath but there are extra duplicate rpaths\n (root, filename) = make_dylib(\"rpath\", duplicate_rpaths)\n # XCode 15 ships a new linker that takes care of deduplication\n if xcode_major_version < 15:\n assert fixup_rpath(root, filename)\n assert not fixup_rpath(root, filename)\n\n # Shared library was constructed with relocatable id from the get-go\n (root, filename) = make_dylib(\"rpath\", no_rpath)\n assert not fixup_rpath(root, filename)\n\n # Non-relocatable library id\n (root, filename) = make_dylib(\"abs\", no_rpath)\n assert not fixup_rpath(root, filename)\n\n # Relocatable with executable paths and loader paths\n (root, filename) = make_dylib(\"rpath\", [\"@executable_path/../lib\", \"@loader_path\"])\n assert not fixup_rpath(root, filename)\n\n # Non-relocatable library id but nonexistent rpath\n (root, filename) = make_dylib(\"abs\", bad_rpath)\n assert fixup_rpath(root, filename)\n assert not fixup_rpath(root, filename)\n\n # Duplicate nonexistent rpath will need *two* passes\n (root, filename) = make_dylib(\"rpath\", bad_rpath * 2)\n assert fixup_rpath(root, filename)\n # XCode 15 ships a new linker that takes care of deduplication\n if xcode_major_version < 15:\n assert fixup_rpath(root, filename)\n assert not fixup_rpath(root, filename)\n\n # Test on an object file, which *also* has type 'application/x-mach-binary'\n # but should be ignored (no ID headers, no RPATH)\n # (this is a corner case for GCC installation)\n (root, filename) = make_object_file()\n assert not fixup_rpath(root, filename)\n" }, { "path": "lib/spack/spack/test/relocate_text.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport io\nfrom collections import OrderedDict\n\nimport pytest\n\nimport spack.relocate_text as relocate_text\n\n\ndef test_text_relocation_regex_is_safe():\n # Test whether prefix regex is properly escaped\n string = b\"This does not match /a/, but this does: /[a-z]/.\"\n assert relocate_text.utf8_path_to_binary_regex(\"/[a-z]/\").search(string).group(0) == b\"/[a-z]/\"\n\n\ndef test_utf8_paths_to_single_binary_regex():\n regex = relocate_text.utf8_paths_to_single_binary_regex(\n [\"/first/path\", \"/second/path\", \"/safe/[a-z]\"]\n )\n # Match nothing\n assert not regex.search(b\"text /neither/first/path text /the/second/path text\")\n\n # Match first\n string = b\"contains both /first/path/subdir and /second/path/sub\"\n assert regex.search(string).group(0) == b\"/first/path/subdir\"\n\n # Match second\n string = b\"contains both /not/first/path/subdir but /second/path/subdir\"\n assert regex.search(string).group(0) == b\"/second/path/subdir\"\n\n # Match \"unsafe\" dir name\n string = b\"don't match /safe/a/path but do match /safe/[a-z]/file\"\n assert regex.search(string).group(0) == b\"/safe/[a-z]/file\"\n\n\ndef test_ordered_replacement():\n # This tests whether binary text replacement respects order, so that\n # a long package prefix is replaced before a shorter sub-prefix like\n # the root of the spack store (as a fallback).\n def replace_and_expect(prefix_map, before, after=None, suffix_safety_size=7):\n f = io.BytesIO(before)\n relocater = relocate_text.BinaryFilePrefixReplacer(\n OrderedDict(prefix_map), suffix_safety_size\n )\n relocater.apply_to_file(f)\n f.seek(0)\n assert f.read() == after\n\n # The case of having a non-null terminated common suffix.\n replace_and_expect(\n [\n (b\"/old-spack/opt/specific-package\", b\"/first/specific-package\"),\n (b\"/old-spack/opt\", b\"/sec/spack/opt\"),\n ],\n b\"Binary with /old-spack/opt/specific-package and /old-spack/opt\",\n b\"Binary with /////////first/specific-package and /sec/spack/opt\",\n suffix_safety_size=7,\n )\n\n # The case of having a direct null terminated common suffix.\n replace_and_expect(\n [\n (b\"/old-spack/opt/specific-package\", b\"/first/specific-package\"),\n (b\"/old-spack/opt\", b\"/sec/spack/opt\"),\n ],\n b\"Binary with /old-spack/opt/specific-package\\0 and /old-spack/opt\\0\",\n b\"Binary with /////////first/specific-package\\0 and /sec/spack/opt\\0\",\n suffix_safety_size=7,\n )\n\n # Testing the order of operations (not null terminated, long enough common suffix)\n replace_and_expect(\n [\n (b\"/old-spack/opt\", b\"/s/spack/opt\"),\n (b\"/old-spack/opt/specific-package\", b\"/first/specific-package\"),\n ],\n b\"Binary with /old-spack/opt/specific-package and /old-spack/opt\",\n b\"Binary with ///s/spack/opt/specific-package and ///s/spack/opt\",\n suffix_safety_size=7,\n )\n\n # Testing the order of operations (null terminated, long enough common suffix)\n replace_and_expect(\n [\n (b\"/old-spack/opt\", b\"/s/spack/opt\"),\n (b\"/old-spack/opt/specific-package\", b\"/first/specific-package\"),\n ],\n b\"Binary with /old-spack/opt/specific-package\\0 and /old-spack/opt\\0\",\n b\"Binary with ///s/spack/opt/specific-package\\0 and ///s/spack/opt\\0\",\n suffix_safety_size=7,\n )\n\n # Null terminated within the lookahead window, common suffix long enough\n replace_and_expect(\n [(b\"/old-spack/opt/specific-package\", b\"/opt/specific-XXXXage\")],\n b\"Binary with /old-spack/opt/specific-package/sub\\0 data\",\n b\"Binary with ///////////opt/specific-XXXXage/sub\\0 data\",\n suffix_safety_size=7,\n )\n\n # Null terminated within the lookahead window, common suffix too short, but\n # shortening is enough to spare more than 7 bytes of old suffix.\n replace_and_expect(\n [(b\"/old-spack/opt/specific-package\", b\"/opt/specific-XXXXXge\")],\n b\"Binary with /old-spack/opt/specific-package/sub\\0 data\",\n b\"Binary with /opt/specific-XXXXXge/sub\\0ckage/sub\\0 data\", # ckage/sub = 9 bytes\n suffix_safety_size=7,\n )\n\n # Null terminated within the lookahead window, common suffix too short,\n # shortening leaves exactly 7 suffix bytes untouched, amazing!\n replace_and_expect(\n [(b\"/old-spack/opt/specific-package\", b\"/spack/specific-XXXXXge\")],\n b\"Binary with /old-spack/opt/specific-package/sub\\0 data\",\n b\"Binary with /spack/specific-XXXXXge/sub\\0age/sub\\0 data\", # age/sub = 7 bytes\n suffix_safety_size=7,\n )\n\n # Null terminated within the lookahead window, common suffix too short,\n # shortening doesn't leave space for 7 bytes, sad!\n error_msg = \"Cannot replace {!r} with {!r} in the C-string {!r}.\".format(\n b\"/old-spack/opt/specific-package\",\n b\"/snacks/specific-XXXXXge\",\n b\"/old-spack/opt/specific-package/sub\",\n )\n with pytest.raises(relocate_text.CannotShrinkCString, match=error_msg):\n replace_and_expect(\n [(b\"/old-spack/opt/specific-package\", b\"/snacks/specific-XXXXXge\")],\n b\"Binary with /old-spack/opt/specific-package/sub\\0 data\",\n # expect failure!\n suffix_safety_size=7,\n )\n\n # Check that it works when changing suffix_safety_size.\n replace_and_expect(\n [(b\"/old-spack/opt/specific-package\", b\"/snacks/specific-XXXXXXe\")],\n b\"Binary with /old-spack/opt/specific-package/sub\\0 data\",\n b\"Binary with /snacks/specific-XXXXXXe/sub\\0ge/sub\\0 data\",\n suffix_safety_size=6,\n )\n\n # Finally check the case of no shortening but a long enough common suffix.\n replace_and_expect(\n [(b\"pkg-gwixwaalgczp6\", b\"pkg-zkesfralgczp6\")],\n b\"Binary with pkg-gwixwaalgczp6/config\\0 data\",\n b\"Binary with pkg-zkesfralgczp6/config\\0 data\",\n suffix_safety_size=7,\n )\n\n # Too short matching suffix, identical string length\n error_msg = \"Cannot replace {!r} with {!r} in the C-string {!r}.\".format(\n b\"pkg-gwixwaxlgczp6\", b\"pkg-zkesfrzlgczp6\", b\"pkg-gwixwaxlgczp6\"\n )\n with pytest.raises(relocate_text.CannotShrinkCString, match=error_msg):\n replace_and_expect(\n [(b\"pkg-gwixwaxlgczp6\", b\"pkg-zkesfrzlgczp6\")],\n b\"Binary with pkg-gwixwaxlgczp6\\0 data\",\n # expect failure\n suffix_safety_size=7,\n )\n\n # Finally, make sure that the regex is not greedily finding the LAST null byte\n # it should find the first null byte in the window. In this test we put one null\n # at a distance where we can't keep a long enough suffix, and one where we can,\n # so we should expect failure when the first null is used.\n error_msg = \"Cannot replace {!r} with {!r} in the C-string {!r}.\".format(\n b\"pkg-abcdef\", b\"pkg-xyzabc\", b\"pkg-abcdef\"\n )\n with pytest.raises(relocate_text.CannotShrinkCString, match=error_msg):\n replace_and_expect(\n [(b\"pkg-abcdef\", b\"pkg-xyzabc\")],\n b\"Binary with pkg-abcdef\\0/xx\\0\", # def\\0/xx is 7 bytes.\n # expect failure\n suffix_safety_size=7,\n )\n\n\ndef test_inplace_text_replacement():\n def replace_and_expect(prefix_to_prefix, before: bytes, after: bytes):\n f = io.BytesIO(before)\n replacer = relocate_text.TextFilePrefixReplacer(OrderedDict(prefix_to_prefix))\n replacer.apply_to_file(f)\n f.seek(0)\n assert f.read() == after\n\n replace_and_expect(\n [\n (b\"/first/prefix\", b\"/first-replacement/prefix\"),\n (b\"/second/prefix\", b\"/second-replacement/prefix\"),\n ],\n b\"Example: /first/prefix/subdir and /second/prefix/subdir\",\n b\"Example: /first-replacement/prefix/subdir and /second-replacement/prefix/subdir\",\n )\n\n replace_and_expect(\n [\n (b\"/replace/in/order\", b\"/first\"),\n (b\"/replace/in\", b\"/second\"),\n (b\"/replace\", b\"/third\"),\n ],\n b\"/replace/in/order/x /replace/in/y /replace/z\",\n b\"/first/x /second/y /third/z\",\n )\n\n replace_and_expect(\n [\n (b\"/replace\", b\"/third\"),\n (b\"/replace/in\", b\"/second\"),\n (b\"/replace/in/order\", b\"/first\"),\n ],\n b\"/replace/in/order/x /replace/in/y /replace/z\",\n b\"/third/in/order/x /third/in/y /third/z\",\n )\n\n replace_and_expect(\n [(b\"/my/prefix\", b\"/replacement\")],\n b\"/dont/replace/my/prefix #!/dont/replace/my/prefix\",\n b\"/dont/replace/my/prefix #!/dont/replace/my/prefix\",\n )\n\n replace_and_expect(\n [(b\"/my/prefix\", b\"/replacement\")],\n b\"Install path: /my/prefix.\",\n b\"Install path: /replacement.\",\n )\n\n replace_and_expect([(b\"/my/prefix\", b\"/replacement\")], b\"#!/my/prefix\", b\"#!/replacement\")\n\n\ndef test_relocate_text_filters_redundant_entries():\n # Test that we're filtering identical old / new paths, since that's a waste.\n mapping = OrderedDict([(\"/hello\", \"/hello\"), (\"/world\", \"/world\")])\n replacer_1 = relocate_text.BinaryFilePrefixReplacer.from_strings_or_bytes(mapping)\n replacer_2 = relocate_text.TextFilePrefixReplacer.from_strings_or_bytes(mapping)\n assert not replacer_1.prefix_to_prefix\n assert not replacer_2.prefix_to_prefix\n" }, { "path": "lib/spack/spack/test/repo.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport os\nimport pathlib\n\nimport pytest\n\nimport spack.environment\nimport spack.package_base\nimport spack.paths\nimport spack.repo\nimport spack.schema.repos\nimport spack.spec\nimport spack.util.executable\nimport spack.util.file_cache\nimport spack.util.lock\nimport spack.util.naming\nfrom spack.test.conftest import RepoBuilder\nfrom spack.util.naming import valid_module_name\n\n\n@pytest.fixture(params=[\"packages\", \"\", \"foo\"])\ndef extra_repo(tmp_path_factory: pytest.TempPathFactory, request):\n repo_namespace = \"extra_test_repo\"\n repo_dir = tmp_path_factory.mktemp(repo_namespace)\n cache_dir = tmp_path_factory.mktemp(\"cache\")\n (repo_dir / request.param).mkdir(parents=True, exist_ok=True)\n if request.param == \"packages\":\n (repo_dir / \"repo.yaml\").write_text(\n \"\"\"\nrepo:\n namespace: extra_test_repo\n\"\"\"\n )\n else:\n (repo_dir / \"repo.yaml\").write_text(\n f\"\"\"\nrepo:\n namespace: extra_test_repo\n subdirectory: '{request.param}'\n\"\"\"\n )\n repo_cache = spack.util.file_cache.FileCache(cache_dir)\n return spack.repo.Repo(str(repo_dir), cache=repo_cache), request.param\n\n\ndef test_repo_getpkg(mutable_mock_repo):\n mutable_mock_repo.get_pkg_class(\"pkg-a\")\n mutable_mock_repo.get_pkg_class(\"builtin_mock.pkg-a\")\n\n\ndef test_repo_multi_getpkg(mutable_mock_repo, extra_repo):\n mutable_mock_repo.put_first(extra_repo[0])\n mutable_mock_repo.get_pkg_class(\"pkg-a\")\n mutable_mock_repo.get_pkg_class(\"builtin_mock.pkg-a\")\n\n\ndef test_repo_multi_getpkgclass(mutable_mock_repo, extra_repo):\n mutable_mock_repo.put_first(extra_repo[0])\n mutable_mock_repo.get_pkg_class(\"pkg-a\")\n mutable_mock_repo.get_pkg_class(\"builtin_mock.pkg-a\")\n\n\ndef test_repo_pkg_with_unknown_namespace(mutable_mock_repo):\n with pytest.raises(spack.repo.UnknownNamespaceError):\n mutable_mock_repo.get_pkg_class(\"unknown.pkg-a\")\n\n\ndef test_repo_unknown_pkg(mutable_mock_repo):\n with pytest.raises(spack.repo.UnknownPackageError):\n mutable_mock_repo.get_pkg_class(\"builtin_mock.nonexistentpackage\")\n\n\ndef test_repo_last_mtime(mock_packages):\n mtime_with_package_py = [\n (os.path.getmtime(p.module.__file__), p.module.__file__)\n for p in spack.repo.PATH.all_package_classes()\n ]\n repo_mtime = spack.repo.PATH.last_mtime()\n max_mtime, max_file = max(mtime_with_package_py)\n if max_mtime > repo_mtime:\n modified_after = \"\\n \".join(\n f\"{path} ({mtime})\" for mtime, path in mtime_with_package_py if mtime > repo_mtime\n )\n assert max_mtime <= repo_mtime, (\n f\"the following files were modified while running tests:\\n {modified_after}\"\n )\n assert max_mtime == repo_mtime, f\"last_mtime incorrect for {max_file}\"\n\n\ndef test_repo_invisibles(mutable_mock_repo, extra_repo):\n with open(\n os.path.join(extra_repo[0].root, extra_repo[1], \".invisible\"), \"w\", encoding=\"utf-8\"\n ):\n pass\n extra_repo[0].all_package_names()\n\n\n@pytest.mark.regression(\"24552\")\ndef test_all_package_names_is_cached_correctly(mock_packages):\n assert \"mpi\" in spack.repo.all_package_names(include_virtuals=True)\n assert \"mpi\" not in spack.repo.all_package_names(include_virtuals=False)\n\n\n@pytest.mark.regression(\"29203\")\ndef test_use_repositories_doesnt_change_class(mock_packages):\n \"\"\"Test that we don't create the same package module and class multiple times\n when swapping repositories.\n \"\"\"\n zlib_cls_outer = spack.repo.PATH.get_pkg_class(\"zlib\")\n current_paths = [r.root for r in spack.repo.PATH.repos]\n with spack.repo.use_repositories(*current_paths):\n zlib_cls_inner = spack.repo.PATH.get_pkg_class(\"zlib\")\n assert id(zlib_cls_inner) == id(zlib_cls_outer)\n\n\ndef test_absolute_import_spack_packages_as_python_modules(mock_packages):\n import spack_repo.builtin_mock.packages.mpileaks.package # type: ignore[import]\n\n assert hasattr(spack_repo.builtin_mock.packages.mpileaks.package, \"Mpileaks\")\n assert isinstance(\n spack_repo.builtin_mock.packages.mpileaks.package.Mpileaks, spack.package_base.PackageMeta\n )\n assert issubclass(\n spack_repo.builtin_mock.packages.mpileaks.package.Mpileaks, spack.package_base.PackageBase\n )\n\n\ndef test_relative_import_spack_packages_as_python_modules(mock_packages):\n from spack_repo.builtin_mock.packages.mpileaks.package import Mpileaks\n\n assert isinstance(Mpileaks, spack.package_base.PackageMeta)\n assert issubclass(Mpileaks, spack.package_base.PackageBase)\n\n\ndef test_get_all_mock_packages(mock_packages):\n \"\"\"Get the mock packages once each too.\"\"\"\n for name in mock_packages.all_package_names():\n mock_packages.get_pkg_class(name)\n\n\ndef test_repo_path_handles_package_removal(mock_packages, repo_builder: RepoBuilder):\n repo_builder.add_package(\"pkg-c\")\n with spack.repo.use_repositories(repo_builder.root, override=False) as repos:\n r = repos.repo_for_pkg(\"pkg-c\")\n assert r.namespace == repo_builder.namespace\n\n repo_builder.remove(\"pkg-c\")\n with spack.repo.use_repositories(repo_builder.root, override=False) as repos:\n r = repos.repo_for_pkg(\"pkg-c\")\n assert r.namespace == \"builtin_mock\"\n\n\ndef test_repo_dump_virtuals(\n tmp_path: pathlib.Path, mutable_mock_repo, mock_packages, ensure_debug, capfd\n):\n # Start with a package-less virtual\n vspec = spack.spec.Spec(\"something\")\n mutable_mock_repo.dump_provenance(vspec, str(tmp_path))\n captured = capfd.readouterr()[1]\n assert \"does not have a package\" in captured\n\n # Now with a virtual with a package\n vspec = spack.spec.Spec(\"externalvirtual\")\n mutable_mock_repo.dump_provenance(vspec, str(tmp_path))\n captured = capfd.readouterr()[1]\n assert \"Installing\" in captured\n assert \"package.py\" in os.listdir(str(tmp_path)), \"Expected the virtual's package to be copied\"\n\n\n@pytest.mark.parametrize(\"repos\", [[\"mock\"], [\"extra\"], [\"mock\", \"extra\"], [\"extra\", \"mock\"]])\ndef test_repository_construction_doesnt_use_globals(\n nullify_globals, tmp_path: pathlib.Path, repos, repo_builder: RepoBuilder\n):\n def _repo_descriptors(repos):\n descriptors = {}\n for entry in repos:\n if entry == \"mock\":\n descriptors[\"builtin_mock\"] = spack.repo.LocalRepoDescriptor(\n \"builtin_mock\", spack.paths.mock_packages_path\n )\n if entry == \"extra\":\n repo_dir = tmp_path / \"extra_mock\"\n repo_dir.mkdir()\n descriptors[repo_builder.namespace] = spack.repo.LocalRepoDescriptor(\n repo_builder.namespace, repo_builder.root\n )\n return spack.repo.RepoDescriptors(descriptors)\n\n descriptors = _repo_descriptors(repos)\n\n repo_cache = spack.util.file_cache.FileCache(tmp_path / \"cache\")\n repo_path = spack.repo.RepoPath.from_descriptors(descriptors, cache=repo_cache)\n assert len(repo_path.repos) == len(descriptors)\n assert [x.namespace for x in repo_path.repos] == list(descriptors.keys())\n\n\n@pytest.mark.parametrize(\"method_name\", [\"dirname_for_package_name\", \"filename_for_package_name\"])\ndef test_path_computation_with_names(method_name, mock_packages_repo):\n \"\"\"Tests that repositories can compute the correct paths when using both fully qualified\n names and unqualified names.\n \"\"\"\n repo_path = spack.repo.RepoPath(mock_packages_repo)\n method = getattr(repo_path, method_name)\n unqualified = method(\"mpileaks\")\n qualified = method(\"builtin_mock.mpileaks\")\n assert qualified == unqualified\n\n\ndef test_use_repositories_and_import():\n \"\"\"Tests that use_repositories changes the import search too\"\"\"\n import spack.paths\n\n repo_dir = pathlib.Path(spack.paths.test_repos_path)\n with spack.repo.use_repositories(str(repo_dir / \"spack_repo\" / \"compiler_runtime_test\")):\n import spack_repo.compiler_runtime_test.packages.gcc_runtime.package # type: ignore[import] # noqa: E501\n\n with spack.repo.use_repositories(str(repo_dir / \"spack_repo\" / \"builtin_mock\")):\n import spack_repo.builtin_mock.packages.cmake.package # type: ignore[import] # noqa: F401\n\n\n@pytest.mark.usefixtures(\"nullify_globals\")\nclass TestRepo:\n \"\"\"Test that the Repo class work correctly, and does not depend on globals,\n except the REPOS_FINDER.\n \"\"\"\n\n def test_creation(self, mock_test_cache):\n repo = spack.repo.Repo(spack.paths.mock_packages_path, cache=mock_test_cache)\n assert repo.config_file.endswith(\"repo.yaml\")\n assert repo.namespace == \"builtin_mock\"\n\n @pytest.mark.parametrize(\n \"name,expected\", [(\"mpi\", True), (\"mpich\", False), (\"mpileaks\", False)]\n )\n def test_is_virtual(self, name, expected, mock_test_cache):\n repo = spack.repo.Repo(spack.paths.mock_packages_path, cache=mock_test_cache)\n assert repo.is_virtual(name) is expected\n assert repo.is_virtual_safe(name) is expected\n\n repo_path = spack.repo.RepoPath(repo)\n assert repo_path.is_virtual(name) is expected\n assert repo_path.is_virtual_safe(name) is expected\n\n @pytest.mark.parametrize(\n \"module_name,pkg_name\",\n [\n (\"dla_future\", \"dla-future\"),\n (\"num7zip\", \"7zip\"),\n # If no package is there, None is returned\n (\"unknown\", None),\n ],\n )\n def test_real_name(self, module_name, pkg_name, mock_test_cache, tmp_path: pathlib.Path):\n \"\"\"Test that we can correctly compute the 'real' name of a package, from the one\n used to import the Python module.\n \"\"\"\n path, _ = spack.repo.create_repo(str(tmp_path), package_api=(1, 0))\n if pkg_name is not None:\n pkg_path = pathlib.Path(path) / \"packages\" / pkg_name / \"package.py\"\n pkg_path.parent.mkdir(parents=True)\n pkg_path.write_text(\"\")\n repo = spack.repo.Repo(\n path, cache=spack.util.file_cache.FileCache(str(tmp_path / \"cache\"))\n )\n assert repo.real_name(module_name) == pkg_name\n\n @pytest.mark.parametrize(\"name\", [\"mpileaks\", \"7zip\", \"dla-future\"])\n def test_get(self, name, mock_test_cache):\n repo = spack.repo.Repo(spack.paths.mock_packages_path, cache=mock_test_cache)\n mock_spec = spack.spec.Spec(name)\n mock_spec._mark_concrete()\n pkg = repo.get(mock_spec)\n assert pkg.__class__ == repo.get_pkg_class(name)\n\n @pytest.mark.parametrize(\"virtual_name,expected\", [(\"mpi\", [\"mpich\", \"zmpi\"])])\n def test_providers(self, virtual_name, expected, mock_test_cache):\n repo = spack.repo.Repo(spack.paths.mock_packages_path, cache=mock_test_cache)\n provider_names = {x.name for x in repo.providers_for(virtual_name)}\n assert provider_names.issuperset(expected)\n\n @pytest.mark.parametrize(\n \"extended,expected\",\n [(\"python\", [\"py-extension1\", \"python-venv\"]), (\"perl\", [\"perl-extension\"])],\n )\n def test_extensions(self, extended, expected, mock_test_cache):\n repo = spack.repo.Repo(spack.paths.mock_packages_path, cache=mock_test_cache)\n repo_path = spack.repo.RepoPath(repo)\n for instance in (repo, repo_path):\n provider_names = {x.name for x in instance.extensions_for(extended)}\n assert provider_names.issuperset(expected)\n\n def test_all_package_names(self, mock_test_cache):\n repo = spack.repo.Repo(spack.paths.mock_packages_path, cache=mock_test_cache)\n repo_path = spack.repo.RepoPath(repo)\n\n for instance in (repo, repo_path):\n all_names = instance.all_package_names(include_virtuals=True)\n real_names = instance.all_package_names(include_virtuals=False)\n assert set(all_names).issuperset(real_names)\n for name in set(all_names) - set(real_names):\n assert instance.is_virtual(name)\n assert instance.is_virtual_safe(name)\n\n def test_packages_with_tags(self, mock_test_cache):\n repo = spack.repo.Repo(spack.paths.mock_packages_path, cache=mock_test_cache)\n repo_path = spack.repo.RepoPath(repo)\n\n for instance in (repo, repo_path):\n r1 = instance.packages_with_tags(\"tag1\")\n r2 = instance.packages_with_tags(\"tag1\", \"tag2\")\n assert \"mpich\" in r1 and \"mpich\" in r2\n assert \"mpich2\" in r1 and \"mpich2\" not in r2\n assert r2.issubset(r1)\n\n\n@pytest.mark.usefixtures(\"nullify_globals\")\nclass TestRepoPath:\n def test_creation_from_string(self, mock_test_cache):\n repo = spack.repo.RepoPath.from_descriptors(\n spack.repo.RepoDescriptors(\n {\n \"builtin_mock\": spack.repo.LocalRepoDescriptor(\n \"builtin_mock\", spack.paths.mock_packages_path\n )\n }\n ),\n cache=mock_test_cache,\n )\n assert len(repo.repos) == 1\n assert repo.by_namespace[\"builtin_mock\"] is repo.repos[0]\n\n def test_get_repo(self, mock_test_cache):\n repo = spack.repo.RepoPath.from_descriptors(\n spack.repo.RepoDescriptors(\n {\n \"builtin_mock\": spack.repo.LocalRepoDescriptor(\n \"builtin_mock\", spack.paths.mock_packages_path\n )\n }\n ),\n cache=mock_test_cache,\n )\n # builtin_mock is there\n assert repo.get_repo(\"builtin_mock\") is repo.repos[0]\n # foo is not there, raise\n with pytest.raises(spack.repo.UnknownNamespaceError):\n repo.get_repo(\"foo\")\n\n\ndef test_parse_package_api_version():\n \"\"\"Test that we raise an error if a repository has a version that is not supported.\"\"\"\n # valid version\n assert spack.repo._parse_package_api_version(\n {\"api\": \"v1.2\"}, min_api=(1, 0), max_api=(2, 3)\n ) == (1, 2)\n # too new and too old\n with pytest.raises(\n spack.repo.BadRepoError,\n match=r\"Package API v2.4 is not supported .* \\(must be between v1.0 and v2.3\\)\",\n ):\n spack.repo._parse_package_api_version({\"api\": \"v2.4\"}, min_api=(1, 0), max_api=(2, 3))\n with pytest.raises(\n spack.repo.BadRepoError,\n match=r\"Package API v0.9 is not supported .* \\(must be between v1.0 and v2.3\\)\",\n ):\n spack.repo._parse_package_api_version({\"api\": \"v0.9\"}, min_api=(1, 0), max_api=(2, 3))\n # default to v1.0 if not specified\n assert spack.repo._parse_package_api_version({}, min_api=(1, 0), max_api=(2, 3)) == (1, 0)\n # if v1.0 support is dropped we should also raise\n with pytest.raises(\n spack.repo.BadRepoError,\n match=r\"Package API v1.0 is not supported .* \\(must be between v2.0 and v2.3\\)\",\n ):\n spack.repo._parse_package_api_version({}, min_api=(2, 0), max_api=(2, 3))\n # finally test invalid input\n with pytest.raises(spack.repo.BadRepoError, match=\"Invalid Package API version\"):\n spack.repo._parse_package_api_version({\"api\": \"v2\"}, min_api=(1, 0), max_api=(3, 3))\n with pytest.raises(spack.repo.BadRepoError, match=\"Invalid Package API version\"):\n spack.repo._parse_package_api_version({\"api\": 2.0}, min_api=(1, 0), max_api=(3, 3))\n\n\ndef test_repo_package_api_version(tmp_path: pathlib.Path):\n \"\"\"Test that we can specify the API version of a repository.\"\"\"\n (tmp_path / \"example\" / \"packages\").mkdir(parents=True)\n (tmp_path / \"example\" / \"repo.yaml\").write_text(\n \"\"\"\\\nrepo:\n namespace: example\n\"\"\"\n )\n cache = spack.util.file_cache.FileCache(tmp_path / \"cache\")\n assert spack.repo.Repo(str(tmp_path / \"example\"), cache=cache).package_api == (1, 0)\n\n\ndef test_mod_to_pkg_name_and_reverse():\n # In repo v1 the dirname/module name is the package name\n assert spack.util.naming.pkg_dir_to_pkg_name(\"zlib_ng\", package_api=(1, 0)) == \"zlib_ng\"\n assert (\n spack.util.naming.pkg_dir_to_pkg_name(\"_3example_4\", package_api=(1, 0)) == \"_3example_4\"\n )\n assert spack.util.naming.pkg_name_to_pkg_dir(\"zlib_ng\", package_api=(1, 0)) == \"zlib_ng\"\n assert (\n spack.util.naming.pkg_name_to_pkg_dir(\"_3example_4\", package_api=(1, 0)) == \"_3example_4\"\n )\n\n # In repo v2 there is a 1-1 mapping between module and package names\n assert spack.util.naming.pkg_dir_to_pkg_name(\"_3example_4\", package_api=(2, 0)) == \"3example-4\"\n assert spack.util.naming.pkg_dir_to_pkg_name(\"zlib_ng\", package_api=(2, 0)) == \"zlib-ng\"\n assert spack.util.naming.pkg_name_to_pkg_dir(\"zlib-ng\", package_api=(2, 0)) == \"zlib_ng\"\n assert spack.util.naming.pkg_name_to_pkg_dir(\"3example-4\", package_api=(2, 0)) == \"_3example_4\"\n\n # reserved names need an underscore\n assert spack.util.naming.pkg_dir_to_pkg_name(\"_finally\", package_api=(2, 0)) == \"finally\"\n assert spack.util.naming.pkg_dir_to_pkg_name(\"_assert\", package_api=(2, 0)) == \"assert\"\n assert spack.util.naming.pkg_name_to_pkg_dir(\"finally\", package_api=(2, 0)) == \"_finally\"\n assert spack.util.naming.pkg_name_to_pkg_dir(\"assert\", package_api=(2, 0)) == \"_assert\"\n\n # reserved names are case sensitive, so true/false/none are ok\n assert spack.util.naming.pkg_dir_to_pkg_name(\"true\", package_api=(2, 0)) == \"true\"\n assert spack.util.naming.pkg_dir_to_pkg_name(\"none\", package_api=(2, 0)) == \"none\"\n assert spack.util.naming.pkg_name_to_pkg_dir(\"true\", package_api=(2, 0)) == \"true\"\n assert spack.util.naming.pkg_name_to_pkg_dir(\"none\", package_api=(2, 0)) == \"none\"\n\n\ndef test_repo_v2_invalid_module_name(tmp_path: pathlib.Path, capfd):\n # Create a repo with a v2 structure\n root, _ = spack.repo.create_repo(str(tmp_path), namespace=\"repo_1\", package_api=(2, 0))\n repo_dir = pathlib.Path(root)\n\n # Create two invalid module names\n (repo_dir / \"packages\" / \"zlib-ng\").mkdir()\n (repo_dir / \"packages\" / \"zlib-ng\" / \"package.py\").write_text(\n \"\"\"\nfrom spack.package import PackageBase\n\nclass ZlibNg(PackageBase):\n pass\n\"\"\"\n )\n (repo_dir / \"packages\" / \"UPPERCASE\").mkdir()\n (repo_dir / \"packages\" / \"UPPERCASE\" / \"package.py\").write_text(\n \"\"\"\nfrom spack.package import PackageBase\n\nclass Uppercase(PackageBase):\n pass\n\"\"\"\n )\n\n with spack.repo.use_repositories(str(repo_dir)) as repo:\n assert len(repo.all_package_names()) == 0\n\n stderr = capfd.readouterr().err\n assert \"cannot be used because `zlib-ng` is not a valid Spack package module name\" in stderr\n assert \"cannot be used because `UPPERCASE` is not a valid Spack package module name\" in stderr\n\n\ndef test_repo_v2_module_and_class_to_package_name(tmp_path: pathlib.Path):\n # Create a repo with a v2 structure\n root, _ = spack.repo.create_repo(str(tmp_path), namespace=\"repo_2\", package_api=(2, 0))\n repo_dir = pathlib.Path(root)\n\n # Create an invalid module name\n (repo_dir / \"packages\" / \"_1example_2_test\").mkdir()\n (repo_dir / \"packages\" / \"_1example_2_test\" / \"package.py\").write_text(\n \"\"\"\nfrom spack.package import PackageBase\n\nclass _1example2Test(PackageBase):\n pass\n\"\"\"\n )\n\n with spack.repo.use_repositories(str(repo_dir)) as repo:\n assert repo.exists(\"1example-2-test\")\n pkg_cls = repo.get_pkg_class(\"1example-2-test\")\n assert pkg_cls.name == \"1example-2-test\"\n assert pkg_cls.module.__name__ == \"spack_repo.repo_2.packages._1example_2_test.package\"\n\n\ndef test_valid_module_name_v2():\n api = (2, 0)\n\n # no hyphens\n assert not valid_module_name(\"zlib-ng\", api)\n\n # cannot start with a number\n assert not valid_module_name(\"7zip\", api)\n\n # no consecutive underscores\n assert not valid_module_name(\"zlib__ng\", api)\n\n # reserved names\n assert not valid_module_name(\"finally\", api)\n assert not valid_module_name(\"assert\", api)\n\n # cannot contain uppercase\n assert not valid_module_name(\"False\", api)\n assert not valid_module_name(\"zlib_NG\", api)\n\n # reserved names are allowed when preceded by underscore\n assert valid_module_name(\"_finally\", api)\n assert valid_module_name(\"_assert\", api)\n\n # digits are allowed when preceded by underscore\n assert valid_module_name(\"_1example_2_test\", api)\n\n # underscore is not allowed unless followed by reserved name or digit\n assert not valid_module_name(\"_zlib\", api)\n assert not valid_module_name(\"_false\", api)\n\n\ndef test_namespace_is_optional_in_v2(tmp_path: pathlib.Path):\n \"\"\"Test that a repo without a namespace is valid in v2.\"\"\"\n repo_yaml_dir = tmp_path / \"spack_repo\" / \"foo\" / \"bar\" / \"baz\"\n (repo_yaml_dir / \"packages\").mkdir(parents=True)\n (repo_yaml_dir / \"repo.yaml\").write_text(\n \"\"\"\\\nrepo:\n api: v2.0\n\"\"\"\n )\n\n cache = spack.util.file_cache.FileCache(tmp_path / \"cache\")\n repo = spack.repo.Repo(str(repo_yaml_dir), cache=cache)\n\n assert repo.namespace == \"foo.bar.baz\"\n assert repo.full_namespace == \"spack_repo.foo.bar.baz.packages\"\n assert repo.root == str(repo_yaml_dir)\n assert repo.packages_path == str(repo_yaml_dir / \"packages\")\n assert repo.python_path == str(tmp_path)\n assert repo.package_api == (2, 0)\n\n\ndef test_subdir_in_v2():\n \"\"\"subdir cannot be . or empty in v2, because otherwise we cannot statically distinguish\n between namespace and subdir.\"\"\"\n with pytest.raises(spack.repo.BadRepoError, match=\"Use a symlink packages -> . instead\"):\n spack.repo._validate_and_normalize_subdir(subdir=\"\", root=\"root\", package_api=(2, 0))\n\n with pytest.raises(spack.repo.BadRepoError, match=\"Use a symlink packages -> . instead\"):\n spack.repo._validate_and_normalize_subdir(subdir=\".\", root=\"root\", package_api=(2, 0))\n\n with pytest.raises(spack.repo.BadRepoError, match=\"Expected a directory name, not a path\"):\n subdir = os.path.join(\"a\", \"b\")\n spack.repo._validate_and_normalize_subdir(subdir=subdir, root=\"root\", package_api=(2, 0))\n\n with pytest.raises(spack.repo.BadRepoError, match=\"Must be a valid Python module name\"):\n spack.repo._validate_and_normalize_subdir(subdir=\"123\", root=\"root\", package_api=(2, 0))\n\n\ndef test_is_package_module():\n assert spack.repo.is_package_module(\"spack.pkg.something.something\")\n assert spack.repo.is_package_module(\"spack_repo.foo.bar.baz.package\")\n assert not spack.repo.is_package_module(\"spack_repo.builtin.build_systems.cmake\")\n assert not spack.repo.is_package_module(\"spack.something.else\")\n\n\ndef test_environment_activation_updates_repo_path(tmp_path: pathlib.Path):\n \"\"\"Test that the environment activation updates the repo path correctly.\"\"\"\n repo_root, _ = spack.repo.create_repo(str(tmp_path / \"foo\"), namespace=\"bar\")\n (tmp_path / \"spack.yaml\").write_text(\n \"\"\"\\\nspack:\n repos:\n bar: $env/foo/spack_repo/bar\n\"\"\"\n )\n env = spack.environment.Environment(tmp_path)\n\n with env:\n assert any(os.path.samefile(repo_root, r.root) for r in spack.repo.PATH.repos)\n\n assert not any(os.path.samefile(repo_root, r.root) for r in spack.repo.PATH.repos)\n\n with env:\n assert any(os.path.samefile(repo_root, r.root) for r in spack.repo.PATH.repos)\n\n assert not any(os.path.samefile(repo_root, r.root) for r in spack.repo.PATH.repos)\n\n\ndef test_repo_update(tmp_path: pathlib.Path):\n existing_root, _ = spack.repo.create_repo(str(tmp_path), namespace=\"foo\")\n nonexisting_root = str(tmp_path / \"nonexisting\")\n config = {\"repos\": [existing_root, nonexisting_root]}\n assert spack.schema.repos.update(config)\n assert config[\"repos\"] == {\n \"foo\": existing_root\n # non-existing root is removed for simplicity; would be a warning otherwise.\n }\n\n\ndef test_mock_builtin_repo(mock_packages):\n assert spack.repo.builtin_repo() is spack.repo.PATH.get_repo(\"builtin_mock\")\n\n\ndef test_parse_config_descriptor_git_1(tmp_path: pathlib.Path):\n descriptor = spack.repo.parse_config_descriptor(\n name=\"name\",\n descriptor={\n \"git\": str(tmp_path / \"repo.git\"),\n \"destination\": str(tmp_path / \"some/destination\"),\n },\n lock=spack.util.lock.Lock(str(tmp_path / \"x\"), enable=False),\n )\n\n assert isinstance(descriptor, spack.repo.RemoteRepoDescriptor)\n assert descriptor.name == \"name\"\n assert descriptor.repository == str(tmp_path / \"repo.git\")\n assert descriptor.destination == str(tmp_path / \"some/destination\")\n assert descriptor.relative_paths is None\n\n\ndef test_parse_config_descriptor_git_2(tmp_path: pathlib.Path):\n descriptor = spack.repo.parse_config_descriptor(\n name=\"name\",\n descriptor={\"git\": str(tmp_path / \"repo.git\"), \"paths\": [\"some/path\"]},\n lock=spack.util.lock.Lock(str(tmp_path / \"x\"), enable=False),\n )\n assert isinstance(descriptor, spack.repo.RemoteRepoDescriptor)\n assert descriptor.relative_paths == [\"some/path\"]\n\n\ndef test_remote_descriptor_no_git(tmp_path: pathlib.Path):\n \"\"\"Test that descriptor fails without git.\"\"\"\n descriptor = spack.repo.parse_config_descriptor(\n name=\"name\",\n descriptor={\n \"git\": str(tmp_path / \"repo.git\"),\n \"destination\": str(tmp_path / \"some/destination\"),\n },\n lock=spack.util.lock.Lock(str(tmp_path / \"x\"), enable=False),\n )\n\n descriptor.initialize(fetch=True, git=None)\n\n assert isinstance(descriptor, spack.repo.RemoteRepoDescriptor)\n assert descriptor.error == \"Git executable not found\"\n\n\ndef test_remote_descriptor_update_no_git(tmp_path: pathlib.Path):\n \"\"\"Test that descriptor fails without git.\"\"\"\n descriptor = spack.repo.parse_config_descriptor(\n name=\"name\",\n descriptor={\n \"git\": str(tmp_path / \"repo.git\"),\n \"destination\": str(tmp_path / \"some/destination\"),\n },\n lock=spack.util.lock.Lock(str(tmp_path / \"x\"), enable=False),\n )\n\n assert isinstance(descriptor, spack.repo.RemoteRepoDescriptor)\n\n with pytest.raises(spack.repo.RepoError, match=\"Git executable not found\"):\n descriptor.update(git=None)\n\n\ndef test_parse_config_descriptor_local(tmp_path: pathlib.Path):\n descriptor = spack.repo.parse_config_descriptor(\n name=\"name\",\n descriptor=str(tmp_path / \"local_repo\"),\n lock=spack.util.lock.Lock(str(tmp_path / \"x\"), enable=False),\n )\n assert isinstance(descriptor, spack.repo.LocalRepoDescriptor)\n assert descriptor.name == \"name\"\n assert descriptor.path == str(tmp_path / \"local_repo\")\n\n\ndef test_parse_config_descriptor_no_git(tmp_path: pathlib.Path):\n \"\"\"Test that we can parse a descriptor without a git key.\"\"\"\n with pytest.raises(RuntimeError, match=\"Invalid configuration for repository\"):\n spack.repo.parse_config_descriptor(\n name=\"name\",\n descriptor={\"destination\": str(tmp_path / \"some/destination\"), \"paths\": [\"some/path\"]},\n lock=spack.util.lock.Lock(str(tmp_path / \"x\"), enable=False),\n )\n\n\ndef test_repo_descriptors_construct(tmp_path: pathlib.Path):\n \"\"\"Test the RepoDescriptors construct function. Ensure it does not raise when we cannot\n construct a Repo instance, e.g. due to missing repo.yaml file. Check that it parses the\n spack-repo-index.yaml file both when newly initialized and when already cloned.\"\"\"\n\n lock = spack.util.lock.Lock(str(tmp_path / \"x\"), enable=False)\n cache = spack.util.file_cache.FileCache(str(tmp_path / \"cache\"))\n\n # Construct 3 identical descriptors\n descriptors_1, descriptors_2, descriptors_3 = [\n {\n \"foo\": spack.repo.RemoteRepoDescriptor(\n name=\"foo\",\n repository=str(tmp_path / \"foo.git\"),\n destination=str(tmp_path / \"foo_destination\"),\n branch=None,\n tag=None,\n commit=None,\n relative_paths=None,\n lock=lock,\n )\n }\n for _ in range(3)\n ]\n\n repos_1 = spack.repo.RepoDescriptors(descriptors_1) # type: ignore\n repos_2 = spack.repo.RepoDescriptors(descriptors_2) # type: ignore\n repos_3 = spack.repo.RepoDescriptors(descriptors_3) # type: ignore\n\n class MockGit(spack.util.executable.Executable):\n def __init__(self):\n pass\n\n def __call__(self, *args, **kwargs) -> str: # type: ignore\n action = args[0]\n\n if action == \"ls-remote\":\n return \"\"\"\\\na8eff4da7aab59bbf5996ac1720954bf82443247 HEAD\n165c479984b94051c982a6be1bd850f8bae02858 refs/heads/feature-branch\na8eff4da7aab59bbf5996ac1720954bf82443247 refs/heads/develop\n3bd0276ab0491552247fa055921a23d2ffd9443c refs/heads/releases/v0.20\"\"\"\n\n elif action == \"rev-parse\":\n return \"develop\"\n\n elif action == \"config\":\n return \"origin\"\n\n elif action == \"init\":\n # The git repo needs a .git subdir\n os.makedirs(os.path.join(\".git\"))\n\n elif action == \"checkout\":\n # The spack-repo-index.yaml is optional; we test Spack reads from it.\n with open(os.path.join(\"spack-repo-index.yaml\"), \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nrepo_index:\n paths:\n - spack_repo/foo\n\"\"\"\n )\n\n return \"\"\n\n repo_path_1, errors_1 = repos_1.construct(cache=cache, find_git=MockGit)\n\n # Verify it cannot construct a Repo instance, and that this does *not* throw, since that would\n # break Spack very early on. Instead, an error is returned. Also verify that\n # relative_paths is read from spack-repo-index.yaml.\n assert len(repo_path_1.repos) == 0\n assert len(errors_1) == 1\n assert all(\"No repo.yaml\" in str(err) for err in errors_1.values()), errors_1\n assert descriptors_1[\"foo\"].relative_paths == [\"spack_repo/foo\"]\n # Verify that the default branch was detected from ls-remote\n assert descriptors_1[\"foo\"].branch == \"develop\"\n\n # Do the same test with another instance: it should *not* clone a second time.\n repo_path_2, errors_2 = repos_2.construct(cache=cache, find_git=MockGit)\n assert len(repo_path_2.repos) == 0\n assert len(errors_2) == 1\n assert all(\"No repo.yaml\" in str(err) for err in errors_2.values()), errors_2\n assert descriptors_1[\"foo\"].relative_paths == [\"spack_repo/foo\"]\n\n # Finally fill the repo with an actual repo and check that the repo can be constructed.\n spack.repo.create_repo(str(tmp_path / \"foo_destination\"), \"foo\")\n repo_path_3, errors_3 = repos_3.construct(cache=cache, find_git=MockGit)\n assert not errors_3\n assert len(repo_path_3.repos) == 1\n assert repo_path_3.repos[0].namespace == \"foo\"\n\n\ndef test_repo_descriptors_update(tmp_path: pathlib.Path):\n \"\"\"Test the RepoDescriptors construct function. Ensure it does not raise when we cannot\n construct a Repo instance, e.g. due to missing repo.yaml file. Check that it parses the\n spack-repo-index.yaml file both when newly initialized and when already cloned.\"\"\"\n\n lock = spack.util.lock.Lock(str(tmp_path / \"x\"), enable=False)\n cache = spack.util.file_cache.FileCache(str(tmp_path / \"cache\"))\n\n # Construct 3 identical descriptors\n descriptors_1, descriptors_2, descriptors_3, descriptors_4 = [\n {\n \"foo\": spack.repo.RemoteRepoDescriptor(\n name=\"foo\",\n repository=str(tmp_path / \"foo.git\"),\n destination=str(tmp_path / \"foo_destination\"),\n branch=\"develop\" if i == 0 else None,\n tag=\"v1.0\" if i == 1 else None,\n commit=\"abc123\" if i == 2 else None,\n relative_paths=None,\n lock=lock,\n )\n }\n for i in range(4)\n ]\n\n repos_1 = spack.repo.RepoDescriptors(descriptors_1) # type: ignore\n repos_2 = spack.repo.RepoDescriptors(descriptors_2) # type: ignore\n repos_3 = spack.repo.RepoDescriptors(descriptors_3) # type: ignore\n repos_4 = spack.repo.RepoDescriptors(descriptors_4) # type: ignore\n\n class MockGit(spack.util.executable.Executable):\n def __init__(self):\n pass\n\n def __call__(self, *args, **kwargs) -> str: # type: ignore\n action = args[0]\n\n if action == \"ls-remote\":\n return \"\"\"\\\na8eff4da7aab59bbf5996ac1720954bf82443247 HEAD\n165c479984b94051c982a6be1bd850f8bae02858 refs/heads/feature-branch\na8eff4da7aab59bbf5996ac1720954bf82443247 refs/heads/develop\n3bd0276ab0491552247fa055921a23d2ffd9443c refs/heads/releases/v0.20\"\"\"\n\n elif action == \"rev-parse\":\n return \"develop\"\n\n elif action == \"config\":\n return \"origin\"\n\n elif action == \"init\":\n # The git repo needs a .git subdir\n os.makedirs(os.path.join(\".git\"))\n\n elif action == \"checkout\":\n # The spack-repo-index.yaml is optional; we test Spack reads from it.\n with open(os.path.join(\"spack-repo-index.yaml\"), \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"\\\nrepo_index:\n paths:\n - spack_repo/foo\n\"\"\"\n )\n\n return \"\"\n\n spack.repo.create_repo(str(tmp_path / \"foo_destination\"), \"foo\")\n\n # branch develop\n _, errors_1 = repos_1.construct(cache=cache, find_git=MockGit)\n assert not errors_1\n for descriptor in repos_1.values():\n descriptor.update(git=MockGit())\n\n # tag v1.0\n _, errors_2 = repos_2.construct(cache=cache, find_git=MockGit)\n assert not errors_2\n for descriptor in repos_2.values():\n descriptor.update(git=MockGit())\n\n # commit abc123\n _, errors_3 = repos_3.construct(cache=cache, find_git=MockGit)\n assert not errors_3\n for descriptor in repos_3.values():\n descriptor.update(git=MockGit())\n\n # default branch\n _, errors_4 = repos_4.construct(cache=cache, find_git=MockGit)\n assert not errors_4\n for descriptor in repos_4.values():\n descriptor.update(git=MockGit())\n\n # Rerun construction after initialization to test early exit logic\n _, errors_4 = repos_4.construct(cache=cache, find_git=MockGit)\n assert not errors_4\n\n\ndef test_repo_descriptors_update_invalid(tmp_path: pathlib.Path):\n \"\"\"Test the RepoDescriptors construct function. Ensure it does not raise when we cannot\n construct a Repo instance, e.g. due to missing repo.yaml file. Check that it parses the\n spack-repo-index.yaml file both when newly initialized and when already cloned.\"\"\"\n\n lock = spack.util.lock.Lock(str(tmp_path / \"x\"), enable=False)\n cache = spack.util.file_cache.FileCache(str(tmp_path / \"cache\"))\n\n # Construct 3 identical descriptors\n descriptors_1 = {\n \"foo\": spack.repo.RemoteRepoDescriptor(\n name=\"foo\",\n repository=str(tmp_path / \"foo.git\"),\n destination=str(tmp_path / \"foo_destination\"),\n branch=None,\n tag=None,\n commit=None,\n relative_paths=None,\n lock=lock,\n )\n }\n\n repos_1 = spack.repo.RepoDescriptors(descriptors_1) # type: ignore\n\n class MockGitInvalidRemote(spack.util.executable.Executable):\n def __init__(self):\n pass\n\n def __call__(self, *args, **kwargs) -> str: # type: ignore\n action = args[0]\n\n if action == \"ls-remote\":\n # HEAD ref exists, but no default branch (i.e. no refs/heads/*)\n return \"a8eff4da7aab59bbf5996ac1720954bf82443247 HEAD\"\n\n return \"\"\n\n class MockGitFailed(spack.util.executable.Executable):\n def __init__(self):\n pass\n\n def __call__(self, *args, **kwargs) -> str: # type: ignore\n raise spack.util.executable.ProcessError(\"failed\")\n\n spack.repo.create_repo(str(tmp_path / \"foo_destination\"), \"foo\")\n\n _, errors_1 = repos_1.construct(cache=cache, find_git=MockGitFailed)\n assert len(errors_1) == 1\n assert all(\"Failed to clone repository\" in str(err) for err in errors_1.values()), errors_1\n\n with pytest.raises(spack.repo.RepoError, match=\"Unable to locate a default branch\"):\n for descriptor in repos_1.values():\n descriptor.update(git=MockGitInvalidRemote())\n\n\ndef test_repo_use_bad_import(config, repo_builder: RepoBuilder):\n \"\"\"Demonstrate failure when attempt to get the class for package containing\n a failing import (e.g., missing repository).\"\"\"\n package_py = pathlib.Path(repo_builder._recipe_filename(\"importer\"))\n package_py.parent.mkdir(parents=True)\n package_py.write_text(\n \"\"\"\\\nfrom spack_repo.missing.packages import base\nfrom spack.package import *\n\n\nclass Importer(PackageBase):\n homepage = \"https://www.bad-importer.com\"\n url = \"https://www.bad-importer.com/v1.0.tar.gz\"\n\n version(\"1.0\", md5=\"0123456789abcdef0123456789abcdef\")\n\"\"\",\n encoding=\"utf-8\",\n )\n\n with spack.repo.use_repositories(repo_builder.root):\n with pytest.raises(spack.repo.RepoError, match=\"cannot load\"):\n spack.repo.PATH.get_pkg_class(\"importer\")\n\n\ndef test_repo_use_bad_syntax(config, repo_builder: RepoBuilder):\n \"\"\"Demonstrate failure when attempt to get class for package with invalid syntax.\"\"\"\n package_py = pathlib.Path(repo_builder._recipe_filename(\"erroneous\"))\n package_py.parent.mkdir(parents=True)\n package_py.write_text(\"class 123: pass\", encoding=\"utf-8\")\n\n with spack.repo.use_repositories(repo_builder.root):\n with pytest.raises(spack.repo.RepoError):\n spack.repo.PATH.get_pkg_class(\"erroneous\")\n\n\ndef test_unknownpkgerror_match_fails():\n \"\"\"Ensure fails with basic message when get_close_matches fails.\"\"\"\n\n def _get_close_matches(*args, **kwargs):\n raise MemoryError(\"Too many packages to compare\")\n\n # Confirm that the error indicates there were no matches (default).\n exception = spack.repo.UnknownPackageError(\"pkg_a\", get_close_matches=_get_close_matches)\n assert \"mean one of the following\" not in str(exception)\n\n\ndef test_unknownpkgerror_str_repo():\n \"\"\"Ensure reasonable error message when repo is a string.\"\"\"\n assert \"not found in repository\" in str(spack.repo.UnknownPackageError(\"pkg_a\", \"my_repo\"))\n" }, { "path": "lib/spack/spack/test/reporters.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport os\nimport pathlib\n\nimport pytest\n\nimport spack.llnl.util.filesystem as fs\nimport spack.llnl.util.tty as tty\nimport spack.reporters.extract\nfrom spack.install_test import TestStatus\nfrom spack.reporters import CDash, CDashConfiguration\n\n# Use a path variable to appease Spack style line length checks\nfake_install_prefix = fs.join_path(\n os.sep,\n \"usr\",\n \"spack\",\n \"spack\",\n \"opt\",\n \"spack\",\n \"linux-rhel7-broadwell\",\n \"intel-19.0.4.227\",\n \"fake-1.0\",\n)\nfake_install_test_root = fs.join_path(fake_install_prefix, \".spack\", \"test\")\nfake_test_cache = fs.join_path(\n \"usr\", \"spack\", \".spack\", \"test\", \"abcdefg\", \"fake-1.0-abcdefg\", \"cache\", \"fake\"\n)\n\n\ndef test_reporters_extract_basics():\n # This test has a description, command, and status\n fake_bin = fs.join_path(fake_install_prefix, \"bin\", \"fake\")\n name = \"test_no_status\"\n desc = \"basic description\"\n status = TestStatus.PASSED\n outputs = \"\"\"\n==> Testing package fake-1.0-abcdefg\n==> [2022-02-15-18:44:21.250165] test: {0}: {1}\n==> [2022-02-15-18:44:21.250200] '{2}'\n{3}: {0}\n\"\"\".format(name, desc, fake_bin, status).splitlines()\n\n parts = spack.reporters.extract.extract_test_parts(\"fake\", outputs)\n assert len(parts) == 1\n assert parts[0][\"command\"] == \"{0}\".format(fake_bin)\n assert parts[0][\"desc\"] == desc\n assert parts[0][\"loglines\"] == [\"{0}: {1}\".format(status, name)]\n assert parts[0][\"status\"] == status.lower()\n\n\ndef test_reporters_extract_no_parts(capfd):\n # This test ticks three boxes:\n # 1) has Installing, which is skipped;\n # 2) does not define any test parts;\n # 3) has a status value without a part so generates a warning\n status = TestStatus.NO_TESTS\n outputs = \"\"\"\n==> Testing package fake-1.0-abcdefg\n==> [2022-02-11-17:14:38.875259] Installing {0} to {1}\n{2}\n\"\"\".format(fake_install_test_root, fake_test_cache, status).splitlines()\n\n parts = spack.reporters.extract.extract_test_parts(\"fake\", outputs)\n err = capfd.readouterr()[1]\n\n assert len(parts) == 1\n assert parts[0][\"status\"] == \"notrun\"\n assert \"No part to add status\" in err\n\n\ndef test_reporters_extract_missing_desc():\n # This test parts with and without descriptions *and* a test part that has\n # multiple commands\n fake_bin = fs.join_path(fake_install_prefix, \"bin\", \"importer\")\n names = [\"test_fake_bin\", \"test_fake_util\", \"test_multiple_commands\"]\n descs = [\"\", \"import fake util module\", \"\"]\n failed = TestStatus.FAILED\n passed = TestStatus.PASSED\n results = [passed, failed, passed]\n outputs = \"\"\"\n==> Testing package fake-1.0-abcdefg\n==> [2022-02-15-18:44:21.250165] test: {0}: {1}\n==> [2022-02-15-18:44:21.250170] '{5}' '-c' 'import fake.bin'\n{2}: {0}\n==> [2022-02-15-18:44:21.250185] test: {3}: {4}\n==> [2022-02-15-18:44:21.250200] '{5}' '-c' 'import fake.util'\n{6}: {3}\n==> [2022-02-15-18:44:21.250205] test: {7}: {8}\n==> [2022-02-15-18:44:21.250210] 'exe1 1'\n==> [2022-02-15-18:44:21.250250] 'exe2 2'\n{9}: {7}\n\"\"\".format(\n names[0],\n descs[0],\n results[0],\n names[1],\n descs[1],\n fake_bin,\n results[1],\n names[2],\n descs[2],\n results[2],\n ).splitlines()\n\n parts = spack.reporters.extract.extract_test_parts(\"fake\", outputs)\n\n assert len(parts) == 3\n for i, (name, desc, status) in enumerate(zip(names, descs, results)):\n assert parts[i][\"name\"] == name\n assert parts[i][\"desc\"] == desc\n assert parts[i][\"status\"] == status.lower()\n assert parts[2][\"command\"] == \"exe1 1; exe2 2\"\n\n\n@pytest.mark.parametrize(\"state\", [(\"not installed\"), (\"external\")])\ndef test_reporters_extract_skipped(state):\n expected = \"Skipped {0} package\".format(state)\n outputs = \"\"\"\n==> Testing package fake-1.0-abcdefg\n{0}\n\"\"\".format(expected).splitlines()\n\n parts = spack.reporters.extract.extract_test_parts(\"fake\", outputs)\n\n assert len(parts) == 1\n\n assert parts[0][\"completed\"] == spack.reporters.extract.completed[\"skipped\"]\n\n\ndef test_reporters_skip_new():\n outputs = \"\"\"\n==> [2023-04-06-15:55:13.094025] test: test_skip:\nSKIPPED: test_skip: Package must be built with +python\n==> [2023-04-06-15:55:13.540029] Completed testing\n==> [2023-04-06-15:55:13.540275]\n======================= SUMMARY: fake-1.0-abcdefg ========================\nfake::test_skip .. SKIPPED\n=========================== 1 skipped of 1 part ==========================\n\"\"\".splitlines()\n\n parts = spack.reporters.extract.extract_test_parts(\"fake\", outputs)\n\n assert len(parts) == 1\n part = parts[0]\n assert part[\"name\"] == \"test_skip\"\n assert part[\"status\"] == \"skipped\"\n assert part[\"completed\"] == \"Completed\"\n assert part[\"loglines\"][0].startswith(\"SKIPPED:\")\n\n\ndef test_reporters_report_for_package_no_stdout(tmp_path: pathlib.Path, monkeypatch, capfd):\n class MockCDash(CDash):\n def upload(*args, **kwargs):\n # Just return (Do NOT try to upload the report to the fake site)\n return\n\n configuration = CDashConfiguration(\n upload_url=\"https://fake-upload\",\n packages=\"fake-package\",\n build=\"fake-cdash-build\",\n site=\"fake-site\",\n buildstamp=None,\n track=\"fake-track\",\n )\n monkeypatch.setattr(tty, \"_debug\", 1)\n\n reporter = MockCDash(configuration=configuration)\n pkg_data = {\"name\": \"fake-package\"}\n reporter.test_report_for_package(str(tmp_path), pkg_data, 0)\n err = capfd.readouterr()[1]\n assert \"Skipping report for\" in err\n assert \"No generated output\" in err\n\n\ndef test_cdash_reporter_truncates_build_name_if_too_long():\n build_name = \"a\" * 190\n extra_long_build_name = build_name + \"a\"\n configuration = CDashConfiguration(\n upload_url=\"https://fake-upload\",\n packages=\"fake-package\",\n build=extra_long_build_name,\n site=\"fake-site\",\n buildstamp=None,\n track=\"fake-track\",\n )\n\n reporter = CDash(configuration=configuration)\n new_build_name = reporter.report_build_name(\"fake-package\")\n\n assert new_build_name != extra_long_build_name\n assert new_build_name == build_name\n" }, { "path": "lib/spack/spack/test/rewiring.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport filecmp\nimport os\nimport sys\n\nimport pytest\n\nimport spack.concretize\nimport spack.deptypes as dt\nimport spack.rewiring\nimport spack.store\nfrom spack.installer import PackageInstaller\nfrom spack.test.relocate import text_in_bin\n\nif sys.platform == \"darwin\":\n required_executables = [\"/usr/bin/clang++\", \"install_name_tool\"]\nelse:\n required_executables = [\"g++\", \"patchelf\"]\n\n\ndef check_spliced_spec_prefixes(spliced_spec):\n \"\"\"check the file in the prefix has the correct paths\"\"\"\n for node in spliced_spec.traverse(root=True):\n text_file_path = os.path.join(node.prefix, node.name)\n with open(text_file_path, \"r\", encoding=\"utf-8\") as f:\n text = f.read()\n for modded_spec in node.traverse(root=True, deptype=dt.ALL & ~dt.BUILD):\n assert modded_spec.prefix in text\n\n\n@pytest.mark.requires_executables(*required_executables)\n@pytest.mark.parametrize(\"transitive\", [True, False])\ndef test_rewire_db(mock_fetch, install_mockery, transitive):\n \"\"\"Tests basic rewiring without binary executables.\"\"\"\n spec = spack.concretize.concretize_one(\"splice-t^splice-h~foo\")\n dep = spack.concretize.concretize_one(\"splice-h+foo\")\n PackageInstaller([spec.package, dep.package], explicit=True).install()\n spliced_spec = spec.splice(dep, transitive=transitive)\n assert spec.dag_hash() != spliced_spec.dag_hash()\n\n spack.rewiring.rewire(spliced_spec)\n\n # check that the prefix exists\n assert os.path.exists(spliced_spec.prefix)\n\n # test that it made it into the database\n rec = spack.store.STORE.db.get_record(spliced_spec)\n installed_in_db = rec.installed if rec else False\n assert installed_in_db\n\n # check for correct prefix paths\n check_spliced_spec_prefixes(spliced_spec)\n\n\n@pytest.mark.requires_executables(*required_executables)\n@pytest.mark.parametrize(\"transitive\", [True, False])\ndef test_rewire_bin(mock_fetch, install_mockery, transitive):\n \"\"\"Tests basic rewiring with binary executables.\"\"\"\n spec = spack.concretize.concretize_one(\"quux\")\n dep = spack.concretize.concretize_one(\"garply cflags=-g\")\n PackageInstaller([spec.package, dep.package], explicit=True).install()\n spliced_spec = spec.splice(dep, transitive=transitive)\n\n assert spec.dag_hash() != spliced_spec.dag_hash()\n\n spack.rewiring.rewire(spliced_spec)\n\n # check that the prefix exists\n assert os.path.exists(spliced_spec.prefix)\n\n # test that it made it into the database\n rec = spack.store.STORE.db.get_record(spliced_spec)\n installed_in_db = rec.installed if rec else False\n assert installed_in_db\n\n # check the file in the prefix has the correct paths\n bin_names = {\"garply\": \"garplinator\", \"corge\": \"corgegator\", \"quux\": \"quuxifier\"}\n for node in spliced_spec.traverse(root=True):\n for dep in node.traverse(root=True):\n bin_file_path = os.path.join(dep.prefix.bin, bin_names[dep.name])\n assert text_in_bin(dep.prefix, bin_file_path)\n\n\n@pytest.mark.requires_executables(*required_executables)\ndef test_rewire_writes_new_metadata(mock_fetch, install_mockery):\n \"\"\"Tests that new metadata was written during a rewire.\n Accuracy of metadata is left to other tests.\"\"\"\n spec = spack.concretize.concretize_one(\"quux\")\n dep = spack.concretize.concretize_one(\"garply cflags=-g\")\n PackageInstaller([spec.package, dep.package], explicit=True).install()\n spliced_spec = spec.splice(dep, transitive=True)\n spack.rewiring.rewire(spliced_spec)\n\n # test install manifests\n for node in spliced_spec.traverse(root=True):\n spack.store.STORE.layout.ensure_installed(node)\n manifest_file_path = os.path.join(\n node.prefix,\n spack.store.STORE.layout.metadata_dir,\n spack.store.STORE.layout.manifest_file_name,\n )\n assert os.path.exists(manifest_file_path)\n orig_node = spec[node.name]\n if node == orig_node:\n continue\n orig_manifest_file_path = os.path.join(\n orig_node.prefix,\n spack.store.STORE.layout.metadata_dir,\n spack.store.STORE.layout.manifest_file_name,\n )\n assert os.path.exists(orig_manifest_file_path)\n assert not filecmp.cmp(orig_manifest_file_path, manifest_file_path, shallow=False)\n specfile_path = os.path.join(\n node.prefix,\n spack.store.STORE.layout.metadata_dir,\n spack.store.STORE.layout.spec_file_name,\n )\n assert os.path.exists(specfile_path)\n orig_specfile_path = os.path.join(\n orig_node.prefix,\n spack.store.STORE.layout.metadata_dir,\n spack.store.STORE.layout.spec_file_name,\n )\n assert os.path.exists(orig_specfile_path)\n assert not filecmp.cmp(orig_specfile_path, specfile_path, shallow=False)\n\n\n@pytest.mark.requires_executables(*required_executables)\n@pytest.mark.parametrize(\"transitive\", [True, False])\ndef test_uninstall_rewired_spec(mock_fetch, install_mockery, transitive):\n \"\"\"Test that rewired packages can be uninstalled as normal.\"\"\"\n spec = spack.concretize.concretize_one(\"quux\")\n dep = spack.concretize.concretize_one(\"garply cflags=-g\")\n PackageInstaller([spec.package, dep.package], explicit=True).install()\n spliced_spec = spec.splice(dep, transitive=transitive)\n spack.rewiring.rewire(spliced_spec)\n spliced_spec.package.do_uninstall()\n assert len(spack.store.STORE.db.query(spliced_spec)) == 0\n assert not os.path.exists(spliced_spec.prefix)\n\n\n@pytest.mark.requires_executables(*required_executables)\ndef test_rewire_not_installed_fails(mock_fetch, install_mockery):\n \"\"\"Tests error when an attempt is made to rewire a package that was not\n previously installed.\"\"\"\n spec = spack.concretize.concretize_one(\"quux\")\n dep = spack.concretize.concretize_one(\"garply cflags=-g\")\n spliced_spec = spec.splice(dep, False)\n with pytest.raises(\n spack.rewiring.PackageNotInstalledError,\n match=\"failed due to missing install of build spec\",\n ):\n spack.rewiring.rewire(spliced_spec)\n\n\ndef test_rewire_virtual(mock_fetch, install_mockery):\n \"\"\"Check installed package can successfully splice an alternate virtual implementation\"\"\"\n dep = \"splice-a\"\n alt_dep = \"splice-h\"\n\n spec = spack.concretize.concretize_one(f\"splice-vt^{dep}\")\n alt_spec = spack.concretize.concretize_one(alt_dep)\n\n PackageInstaller([spec.package, alt_spec.package]).install()\n\n spliced_spec = spec.splice(alt_spec, True)\n spack.rewiring.rewire(spliced_spec)\n\n # Confirm the original spec still has the original virtual implementation.\n assert spec.satisfies(f\"^{dep}\")\n\n # Confirm the spliced spec uses the new virtual implementation.\n assert spliced_spec.satisfies(f\"^{alt_dep}\")\n\n # check for correct prefix paths\n check_spliced_spec_prefixes(spliced_spec)\n" }, { "path": "lib/spack/spack/test/s3_fetch.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport pathlib\n\nimport spack.fetch_strategy as spack_fs\nimport spack.stage as spack_stage\n\n\ndef test_s3fetchstrategy_downloaded(tmp_path: pathlib.Path):\n \"\"\"Ensure fetch with archive file already downloaded is a noop.\"\"\"\n archive = tmp_path / \"s3.tar.gz\"\n\n class Archived_S3FS(spack_fs.S3FetchStrategy):\n @property\n def archive_file(self):\n return archive\n\n fetcher = Archived_S3FS(url=\"s3://example/s3.tar.gz\")\n with spack_stage.Stage(fetcher, path=str(tmp_path)):\n fetcher.fetch()\n" }, { "path": "lib/spack/spack/test/sbang.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"\\\nTest that Spack's shebang filtering works correctly.\n\"\"\"\n\nimport filecmp\nimport os\nimport pathlib\nimport shutil\nimport stat\nimport sys\nimport tempfile\n\nimport pytest\n\nimport spack.config\nimport spack.hooks.sbang as sbang\nimport spack.llnl.util.filesystem as fs\nimport spack.store\nimport spack.util.spack_yaml as syaml\nfrom spack.util.executable import which\n\nif sys.platform != \"win32\":\n import grp\n\n\npytestmark = pytest.mark.not_on_windows(\"does not run on windows\")\n\n\ntoo_long = sbang.system_shebang_limit + 1\n\n\nshort_line = \"#!/this/is/short/bin/bash\\n\"\nlong_line = \"#!/this/\" + (\"x\" * too_long) + \"/is/long\\n\"\n\nlua_line = \"#!/this/\" + (\"x\" * too_long) + \"/is/lua\\n\"\nlua_in_text = (\"line\\n\") * 100 + \"lua\\n\" + (\"line\\n\" * 100)\nlua_line_patched = \"--!/this/\" + (\"x\" * too_long) + \"/is/lua\\n\"\n\nluajit_line = \"#!/this/\" + (\"x\" * too_long) + \"/is/luajit\\n\"\nluajit_in_text = (\"line\\n\") * 100 + \"lua\\n\" + (\"line\\n\" * 100)\nluajit_line_patched = \"--!/this/\" + (\"x\" * too_long) + \"/is/luajit\\n\"\n\nnode_line = \"#!/this/\" + (\"x\" * too_long) + \"/is/node\\n\"\nnode_in_text = (\"line\\n\") * 100 + \"lua\\n\" + (\"line\\n\" * 100)\nnode_line_patched = \"//!/this/\" + (\"x\" * too_long) + \"/is/node\\n\"\n\nphp_line = \"#!/this/\" + (\"x\" * too_long) + \"/is/php\\n\"\nphp_in_text = (\"line\\n\") * 100 + \"php\\n\" + (\"line\\n\" * 100)\nphp_line_patched = \"\\n\"\n\nlast_line = \"last!\\n\"\n\n\n@pytest.fixture # type: ignore[no-redef]\ndef sbang_line():\n yield \"#!/bin/sh %s/bin/sbang\\n\" % spack.store.STORE.layout.root\n\n\nclass ScriptDirectory:\n \"\"\"Directory full of test scripts to run sbang instrumentation on.\"\"\"\n\n def __init__(self, sbang_line):\n self.tempdir = tempfile.mkdtemp()\n\n self.directory = os.path.join(self.tempdir, \"dir\")\n fs.mkdirp(self.directory)\n\n # Script with short shebang\n self.short_shebang = os.path.join(self.tempdir, \"short\")\n with open(self.short_shebang, \"w\", encoding=\"utf-8\") as f:\n f.write(short_line)\n f.write(last_line)\n self.make_executable(self.short_shebang)\n\n # Script with long shebang\n self.long_shebang = os.path.join(self.tempdir, \"long\")\n with open(self.long_shebang, \"w\", encoding=\"utf-8\") as f:\n f.write(long_line)\n f.write(last_line)\n self.make_executable(self.long_shebang)\n\n # Non-executable script with long shebang\n self.nonexec_long_shebang = os.path.join(self.tempdir, \"nonexec_long\")\n with open(self.nonexec_long_shebang, \"w\", encoding=\"utf-8\") as f:\n f.write(long_line)\n f.write(last_line)\n\n # Lua script with long shebang\n self.lua_shebang = os.path.join(self.tempdir, \"lua\")\n with open(self.lua_shebang, \"w\", encoding=\"utf-8\") as f:\n f.write(lua_line)\n f.write(last_line)\n self.make_executable(self.lua_shebang)\n\n # Lua occurring in text, not in shebang\n self.lua_textbang = os.path.join(self.tempdir, \"lua_in_text\")\n with open(self.lua_textbang, \"w\", encoding=\"utf-8\") as f:\n f.write(short_line)\n f.write(lua_in_text)\n f.write(last_line)\n self.make_executable(self.lua_textbang)\n\n # Luajit script with long shebang\n self.luajit_shebang = os.path.join(self.tempdir, \"luajit\")\n with open(self.luajit_shebang, \"w\", encoding=\"utf-8\") as f:\n f.write(luajit_line)\n f.write(last_line)\n self.make_executable(self.luajit_shebang)\n\n # Luajit occurring in text, not in shebang\n self.luajit_textbang = os.path.join(self.tempdir, \"luajit_in_text\")\n with open(self.luajit_textbang, \"w\", encoding=\"utf-8\") as f:\n f.write(short_line)\n f.write(luajit_in_text)\n f.write(last_line)\n self.make_executable(self.luajit_textbang)\n\n # Node script with long shebang\n self.node_shebang = os.path.join(self.tempdir, \"node\")\n with open(self.node_shebang, \"w\", encoding=\"utf-8\") as f:\n f.write(node_line)\n f.write(last_line)\n self.make_executable(self.node_shebang)\n\n # Node occurring in text, not in shebang\n self.node_textbang = os.path.join(self.tempdir, \"node_in_text\")\n with open(self.node_textbang, \"w\", encoding=\"utf-8\") as f:\n f.write(short_line)\n f.write(node_in_text)\n f.write(last_line)\n self.make_executable(self.node_textbang)\n\n # php script with long shebang\n self.php_shebang = os.path.join(self.tempdir, \"php\")\n with open(self.php_shebang, \"w\", encoding=\"utf-8\") as f:\n f.write(php_line)\n f.write(last_line)\n self.make_executable(self.php_shebang)\n\n # php occurring in text, not in shebang\n self.php_textbang = os.path.join(self.tempdir, \"php_in_text\")\n with open(self.php_textbang, \"w\", encoding=\"utf-8\") as f:\n f.write(short_line)\n f.write(php_in_text)\n f.write(last_line)\n self.make_executable(self.php_textbang)\n\n # Script already using sbang.\n self.has_sbang = os.path.join(self.tempdir, \"shebang\")\n with open(self.has_sbang, \"w\", encoding=\"utf-8\") as f:\n f.write(sbang_line)\n f.write(long_line)\n f.write(last_line)\n self.make_executable(self.has_sbang)\n\n # Fake binary file.\n self.binary = os.path.join(self.tempdir, \"binary\")\n tar = which(\"tar\", required=True)\n tar(\"czf\", self.binary, self.has_sbang)\n self.make_executable(self.binary)\n\n def destroy(self):\n shutil.rmtree(self.tempdir, ignore_errors=True)\n\n def make_executable(self, path):\n # make a file executable\n st = os.stat(path)\n executable_mode = st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH\n os.chmod(path, executable_mode)\n\n st = os.stat(path)\n assert oct(executable_mode) == oct(st.st_mode & executable_mode)\n\n\n@pytest.fixture\ndef script_dir(sbang_line):\n sdir = ScriptDirectory(sbang_line)\n yield sdir\n sdir.destroy()\n\n\n@pytest.mark.parametrize(\n \"shebang,interpreter\",\n [\n (b\"#!/path/to/interpreter argument\\n\", b\"/path/to/interpreter\"),\n (b\"#! /path/to/interpreter truncated-argum\", b\"/path/to/interpreter\"),\n (b\"#! \\t \\t/path/to/interpreter\\t \\targument\", b\"/path/to/interpreter\"),\n (b\"#! \\t \\t /path/to/interpreter\", b\"/path/to/interpreter\"),\n (b\"#!/path/to/interpreter\\0\", b\"/path/to/interpreter\"),\n (b\"#!/path/to/interpreter multiple args\\n\", b\"/path/to/interpreter\"),\n (b\"#!\\0/path/to/interpreter arg\\n\", None),\n (b\"#!\\n/path/to/interpreter arg\\n\", None),\n (b\"#!\", None),\n ],\n)\ndef test_shebang_interpreter_regex(shebang, interpreter):\n assert sbang.get_interpreter(shebang) == interpreter\n\n\ndef test_shebang_handling(script_dir, sbang_line):\n sbang.filter_shebangs_in_directory(script_dir.tempdir)\n\n # Make sure this is untouched\n with open(script_dir.short_shebang, \"r\", encoding=\"utf-8\") as f:\n assert f.readline() == short_line\n assert f.readline() == last_line\n\n # Make sure this got patched.\n with open(script_dir.long_shebang, \"r\", encoding=\"utf-8\") as f:\n assert f.readline() == sbang_line\n assert f.readline() == long_line\n assert f.readline() == last_line\n\n # Make sure this is untouched\n with open(script_dir.nonexec_long_shebang, \"r\", encoding=\"utf-8\") as f:\n assert f.readline() == long_line\n assert f.readline() == last_line\n\n # Make sure this got patched.\n with open(script_dir.lua_shebang, \"r\", encoding=\"utf-8\") as f:\n assert f.readline() == sbang_line\n assert f.readline() == lua_line_patched\n assert f.readline() == last_line\n\n # Make sure this got patched.\n with open(script_dir.luajit_shebang, \"r\", encoding=\"utf-8\") as f:\n assert f.readline() == sbang_line\n assert f.readline() == luajit_line_patched\n assert f.readline() == last_line\n\n # Make sure this got patched.\n with open(script_dir.node_shebang, \"r\", encoding=\"utf-8\") as f:\n assert f.readline() == sbang_line\n assert f.readline() == node_line_patched\n assert f.readline() == last_line\n\n assert filecmp.cmp(script_dir.lua_textbang, os.path.join(script_dir.tempdir, \"lua_in_text\"))\n assert filecmp.cmp(\n script_dir.luajit_textbang, os.path.join(script_dir.tempdir, \"luajit_in_text\")\n )\n assert filecmp.cmp(script_dir.node_textbang, os.path.join(script_dir.tempdir, \"node_in_text\"))\n assert filecmp.cmp(script_dir.php_textbang, os.path.join(script_dir.tempdir, \"php_in_text\"))\n\n # Make sure this is untouched\n with open(script_dir.has_sbang, \"r\", encoding=\"utf-8\") as f:\n assert f.readline() == sbang_line\n assert f.readline() == long_line\n assert f.readline() == last_line\n\n\ndef test_shebang_handles_non_writable_files(script_dir, sbang_line):\n # make a file non-writable\n st = os.stat(script_dir.long_shebang)\n not_writable_mode = st.st_mode & ~stat.S_IWRITE\n os.chmod(script_dir.long_shebang, not_writable_mode)\n\n test_shebang_handling(script_dir, sbang_line)\n\n st = os.stat(script_dir.long_shebang)\n assert oct(not_writable_mode) == oct(st.st_mode)\n\n\n@pytest.fixture(scope=\"function\")\ndef configure_group_perms():\n # On systems with remote groups, the primary user group may be remote\n # and grp does not act on remote groups.\n # To ensure we find a group we can operate on, we get take the first group\n # listed which has the current user as a member.\n gid = fs.group_ids(os.getuid())[0]\n group_name = grp.getgrgid(gid).gr_name\n\n conf = syaml.load_config(\n \"\"\"\\\nall:\n permissions:\n read: world\n write: group\n group: {0}\n\"\"\".format(group_name)\n )\n spack.config.set(\"packages\", conf, scope=\"user\")\n\n yield\n\n\n@pytest.fixture(scope=\"function\")\ndef configure_user_perms():\n conf = syaml.load_config(\n \"\"\"\\\nall:\n permissions:\n read: world\n write: user\n\"\"\"\n )\n spack.config.set(\"packages\", conf, scope=\"user\")\n\n yield\n\n\ndef check_sbang_installation(group=False):\n sbang_path = sbang.sbang_install_path()\n sbang_bin_dir = os.path.dirname(sbang_path)\n assert sbang_path.startswith(spack.store.STORE.unpadded_root)\n\n assert os.path.exists(sbang_path)\n assert fs.is_exe(sbang_path)\n\n status = os.stat(sbang_bin_dir)\n mode = status.st_mode & 0o777\n if group:\n assert mode == 0o775, \"Unexpected {0}\".format(oct(mode))\n else:\n assert mode == 0o755, \"Unexpected {0}\".format(oct(mode))\n\n status = os.stat(sbang_path)\n mode = status.st_mode & 0o777\n if group:\n assert mode == 0o775, \"Unexpected {0}\".format(oct(mode))\n else:\n assert mode == 0o755, \"Unexpected {0}\".format(oct(mode))\n\n\ndef run_test_install_sbang(group):\n sbang_path = sbang.sbang_install_path()\n sbang_bin_dir = os.path.dirname(sbang_path)\n\n assert sbang_path.startswith(spack.store.STORE.unpadded_root)\n assert not os.path.exists(sbang_bin_dir)\n\n sbang.install_sbang()\n check_sbang_installation(group)\n\n # put an invalid file in for sbang\n fs.mkdirp(sbang_bin_dir)\n with open(sbang_path, \"w\", encoding=\"utf-8\") as f:\n f.write(\"foo\")\n\n sbang.install_sbang()\n check_sbang_installation(group)\n\n # install again and make sure sbang is still fine\n sbang.install_sbang()\n check_sbang_installation(group)\n\n\ndef test_install_group_sbang(install_mockery, configure_group_perms):\n run_test_install_sbang(True)\n\n\ndef test_install_user_sbang(install_mockery, configure_user_perms):\n run_test_install_sbang(False)\n\n\ndef test_install_sbang_too_long(tmp_path: pathlib.Path):\n root = str(tmp_path)\n num_extend = sbang.system_shebang_limit - len(root) - len(\"/bin/sbang\")\n long_path = root\n while num_extend > 1:\n add = min(num_extend, 255)\n long_path = os.path.join(long_path, \"e\" * add)\n num_extend -= add\n with spack.store.use_store(long_path):\n with pytest.raises(sbang.SbangPathError) as exc_info:\n sbang.sbang_install_path()\n\n err = str(exc_info.value)\n assert \"root is too long\" in err\n assert \"exceeds limit\" in err\n assert \"cannot patch\" in err\n\n\ndef test_sbang_hook_skips_nonexecutable_blobs(tmp_path: pathlib.Path):\n # Write a binary blob to non-executable.sh, with a long interpreter \"path\"\n # consisting of invalid UTF-8. The latter is technically not really necessary for\n # the test, but binary blobs accidentally starting with b'#!' usually do not contain\n # valid UTF-8, so we also ensure that Spack does not attempt to decode as UTF-8.\n contents = b\"#!\" + b\"\\x80\" * sbang.system_shebang_limit\n file = str(tmp_path / \"non-executable.sh\")\n with open(file, \"wb\") as f:\n f.write(contents)\n\n sbang.filter_shebangs_in_directory(str(tmp_path))\n\n # Make sure there is no sbang shebang.\n with open(file, \"rb\") as f:\n assert b\"sbang\" not in f.readline()\n\n\ndef test_sbang_handles_non_utf8_files(tmp_path: pathlib.Path):\n # We have an executable with a copyright sign as filename\n contents = b\"#!\" + b\"\\xa9\" * sbang.system_shebang_limit + b\"\\nand another symbol: \\xa9\"\n\n # Make sure it's indeed valid latin1 but invalid utf-8.\n assert contents.decode(\"latin1\")\n with pytest.raises(UnicodeDecodeError):\n contents.decode(\"utf-8\")\n\n # Put it in an executable file\n file = str(tmp_path / \"latin1.sh\")\n with open(file, \"wb\") as f:\n f.write(contents)\n\n # Run sbang\n assert sbang.filter_shebang(file)\n\n with open(file, \"rb\") as f:\n new_contents = f.read()\n\n assert contents in new_contents\n assert b\"sbang\" in new_contents\n\n\n@pytest.fixture\ndef shebang_limits_system_8_spack_16():\n system_limit, sbang.system_shebang_limit = sbang.system_shebang_limit, 8\n spack_limit, sbang.spack_shebang_limit = sbang.spack_shebang_limit, 16\n yield\n sbang.system_shebang_limit = system_limit\n sbang.spack_shebang_limit = spack_limit\n\n\ndef test_shebang_exceeds_spack_shebang_limit(\n shebang_limits_system_8_spack_16, tmp_path: pathlib.Path\n):\n \"\"\"Tests whether shebangs longer than Spack's limit are skipped\"\"\"\n file = str(tmp_path / \"longer_than_spack_limit.sh\")\n with open(file, \"wb\") as f:\n f.write(b\"#!\" + b\"x\" * sbang.spack_shebang_limit)\n\n # Then Spack shouldn't try to add a shebang\n assert not sbang.filter_shebang(file)\n\n with open(file, \"rb\") as f:\n assert b\"sbang\" not in f.read()\n\n\ndef test_sbang_hook_handles_non_writable_files_preserving_permissions(tmp_path: pathlib.Path):\n path = str(tmp_path / \"file.sh\")\n with open(path, \"w\", encoding=\"utf-8\") as f:\n f.write(long_line)\n os.chmod(path, 0o555)\n sbang.filter_shebang(path)\n with open(path, \"r\", encoding=\"utf-8\") as f:\n assert \"sbang\" in f.readline()\n assert os.stat(path).st_mode & 0o777 == 0o555\n" }, { "path": "lib/spack/spack/test/schema.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport importlib\nimport os\n\nimport pytest\n\nfrom spack.vendor import jsonschema\n\nimport spack.schema\nimport spack.schema.env\nimport spack.util.spack_yaml as syaml\nfrom spack.llnl.util.lang import list_modules\n\n_draft_07_with_spack_extensions = {\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Core schema meta-schema\",\n \"definitions\": {\n \"schemaArray\": {\"type\": \"array\", \"minItems\": 1, \"items\": {\"$ref\": \"#\"}},\n \"nonNegativeInteger\": {\"type\": \"integer\", \"minimum\": 0},\n \"nonNegativeIntegerDefault0\": {\n \"allOf\": [{\"$ref\": \"#/definitions/nonNegativeInteger\"}, {\"default\": 0}]\n },\n \"simpleTypes\": {\n \"enum\": [\"array\", \"boolean\", \"integer\", \"null\", \"number\", \"object\", \"string\"]\n },\n \"stringArray\": {\n \"type\": \"array\",\n \"items\": {\"type\": \"string\"},\n \"uniqueItems\": True,\n \"default\": [],\n },\n },\n \"type\": [\"object\", \"boolean\"],\n \"properties\": {\n \"$id\": {\"type\": \"string\", \"format\": \"uri-reference\"},\n \"$schema\": {\"type\": \"string\", \"format\": \"uri\"},\n \"$ref\": {\"type\": \"string\", \"format\": \"uri-reference\"},\n \"$comment\": {\"type\": \"string\"},\n \"title\": {\"type\": \"string\"},\n \"description\": {\"type\": \"string\"},\n \"default\": True,\n \"readOnly\": {\"type\": \"boolean\", \"default\": False},\n \"writeOnly\": {\"type\": \"boolean\", \"default\": False},\n \"examples\": {\"type\": \"array\", \"items\": True},\n \"multipleOf\": {\"type\": \"number\", \"exclusiveMinimum\": 0},\n \"maximum\": {\"type\": \"number\"},\n \"exclusiveMaximum\": {\"type\": \"number\"},\n \"minimum\": {\"type\": \"number\"},\n \"exclusiveMinimum\": {\"type\": \"number\"},\n \"maxLength\": {\"$ref\": \"#/definitions/nonNegativeInteger\"},\n \"minLength\": {\"$ref\": \"#/definitions/nonNegativeIntegerDefault0\"},\n \"pattern\": {\"type\": \"string\", \"format\": \"regex\"},\n \"additionalItems\": {\"$ref\": \"#\"},\n \"items\": {\n \"anyOf\": [{\"$ref\": \"#\"}, {\"$ref\": \"#/definitions/schemaArray\"}],\n \"default\": True,\n },\n \"maxItems\": {\"$ref\": \"#/definitions/nonNegativeInteger\"},\n \"minItems\": {\"$ref\": \"#/definitions/nonNegativeIntegerDefault0\"},\n \"uniqueItems\": {\"type\": \"boolean\", \"default\": False},\n \"contains\": {\"$ref\": \"#\"},\n \"maxProperties\": {\"$ref\": \"#/definitions/nonNegativeInteger\"},\n \"minProperties\": {\"$ref\": \"#/definitions/nonNegativeIntegerDefault0\"},\n \"required\": {\"$ref\": \"#/definitions/stringArray\"},\n \"additionalProperties\": {\"$ref\": \"#\"},\n \"definitions\": {\"type\": \"object\", \"additionalProperties\": {\"$ref\": \"#\"}, \"default\": {}},\n \"properties\": {\"type\": \"object\", \"additionalProperties\": {\"$ref\": \"#\"}, \"default\": {}},\n \"patternProperties\": {\n \"type\": \"object\",\n \"additionalProperties\": {\"$ref\": \"#\"},\n \"propertyNames\": {\"format\": \"regex\"},\n \"default\": {},\n },\n \"dependencies\": {\n \"type\": \"object\",\n \"additionalProperties\": {\n \"anyOf\": [{\"$ref\": \"#\"}, {\"$ref\": \"#/definitions/stringArray\"}]\n },\n },\n \"propertyNames\": {\"$ref\": \"#\"},\n \"const\": True,\n \"enum\": {\"type\": \"array\", \"items\": True, \"minItems\": 1, \"uniqueItems\": True},\n \"type\": {\n \"anyOf\": [\n {\"$ref\": \"#/definitions/simpleTypes\"},\n {\n \"type\": \"array\",\n \"items\": {\"$ref\": \"#/definitions/simpleTypes\"},\n \"minItems\": 1,\n \"uniqueItems\": True,\n },\n ]\n },\n \"format\": {\"type\": \"string\"},\n \"contentMediaType\": {\"type\": \"string\"},\n \"contentEncoding\": {\"type\": \"string\"},\n \"if\": {\"$ref\": \"#\"},\n \"then\": {\"$ref\": \"#\"},\n \"else\": {\"$ref\": \"#\"},\n \"allOf\": {\"$ref\": \"#/definitions/schemaArray\"},\n \"anyOf\": {\"$ref\": \"#/definitions/schemaArray\"},\n \"oneOf\": {\"$ref\": \"#/definitions/schemaArray\"},\n \"not\": {\"$ref\": \"#\"},\n # What follows is two Spack extensions to JSON Schema Draft 7:\n # deprecatedProperties and additionalKeysAreSpecs\n \"deprecatedProperties\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"names\": {\n \"type\": \"array\",\n \"items\": {\"type\": \"string\"},\n \"minItems\": 1,\n \"uniqueItems\": True,\n },\n \"message\": {\"type\": \"string\"},\n \"error\": {\"type\": \"boolean\"},\n },\n \"required\": [\"names\", \"message\"],\n \"additionalProperties\": False,\n },\n },\n \"additionalKeysAreSpecs\": {\"type\": \"boolean\"},\n },\n \"default\": True,\n # note: not in draft-07, this is for catching typos\n \"additionalProperties\": False,\n}\n\n\n@pytest.fixture()\ndef validate_spec_schema():\n return {\n \"type\": \"object\",\n \"additionalKeysAreSpecs\": True,\n \"patternProperties\": {r\"\\w[\\w-]*\": {\"type\": \"string\"}},\n }\n\n\n@pytest.fixture()\ndef module_suffixes_schema():\n return {\n \"type\": \"object\",\n \"properties\": {\n \"tcl\": {\n \"type\": \"object\",\n \"patternProperties\": {\n r\"\\w[\\w-]*\": {\n \"type\": \"object\",\n \"properties\": {\n \"suffixes\": {\n \"additionalKeysAreSpecs\": True,\n \"patternProperties\": {r\"\\w[\\w-]*\": {\"type\": \"string\"}},\n }\n },\n }\n },\n }\n },\n }\n\n\n@pytest.mark.regression(\"9857\")\ndef test_validate_spec(validate_spec_schema):\n v = spack.schema.Validator(validate_spec_schema)\n data = {\"foo@3.7\": \"bar\"}\n\n # Validate good data (the key is a spec)\n v.validate(data)\n\n # Check that invalid data throws\n data[\"^python@3.7@\"] = \"baz\"\n with pytest.raises(jsonschema.ValidationError, match=\"is not a valid spec\"):\n v.validate(data)\n\n\n@pytest.mark.regression(\"9857\")\ndef test_module_suffixes(module_suffixes_schema):\n v = spack.schema.Validator(module_suffixes_schema)\n data = {\"tcl\": {\"all\": {\"suffixes\": {\"^python@2.7@\": \"py2.7\"}}}}\n\n with pytest.raises(jsonschema.ValidationError, match=\"is not a valid spec\"):\n v.validate(data)\n\n\ndef test_deprecated_properties(module_suffixes_schema):\n # Test that an error is reported when 'error: True'\n msg_fmt = r\"{name} is deprecated\"\n module_suffixes_schema[\"deprecatedProperties\"] = [\n {\"names\": [\"tcl\"], \"message\": msg_fmt, \"error\": True}\n ]\n v = spack.schema.Validator(module_suffixes_schema)\n data = {\"tcl\": {\"all\": {\"suffixes\": {\"^python\": \"py\"}}}}\n\n expected_match = \"tcl is deprecated\"\n with pytest.raises(jsonschema.ValidationError, match=expected_match):\n v.validate(data)\n\n # Test that just a warning is reported when 'error: False'\n module_suffixes_schema[\"deprecatedProperties\"] = [\n {\"names\": [\"tcl\"], \"message\": msg_fmt, \"error\": False}\n ]\n v = spack.schema.Validator(module_suffixes_schema)\n data = {\"tcl\": {\"all\": {\"suffixes\": {\"^python\": \"py\"}}}}\n # The next validation doesn't raise anymore\n v.validate(data)\n\n\ndef test_ordereddict_merge_order():\n \"\"\" \"Test that source keys come before dest keys in merge_yaml results.\"\"\"\n source = syaml.syaml_dict([(\"k1\", \"v1\"), (\"k2\", \"v2\"), (\"k3\", \"v3\")])\n\n dest = syaml.syaml_dict([(\"k4\", \"v4\"), (\"k3\", \"WRONG\"), (\"k5\", \"v5\")])\n\n result = spack.schema.merge_yaml(dest, source)\n assert \"WRONG\" not in result.values()\n\n expected_keys = [\"k1\", \"k2\", \"k3\", \"k4\", \"k5\"]\n expected_items = [(\"k1\", \"v1\"), (\"k2\", \"v2\"), (\"k3\", \"v3\"), (\"k4\", \"v4\"), (\"k5\", \"v5\")]\n assert expected_keys == list(result.keys())\n assert expected_items == list(result.items())\n\n\ndef test_list_merge_order():\n \"\"\" \"Test that source lists are prepended to dest.\"\"\"\n source = [\"a\", \"b\", \"c\"]\n dest = [\"d\", \"e\", \"f\"]\n\n result = spack.schema.merge_yaml(dest, source)\n\n assert [\"a\", \"b\", \"c\", \"d\", \"e\", \"f\"] == result\n\n\ndef test_spack_schemas_are_valid():\n \"\"\"Test that the Spack schemas in spack.schema.*.schema are valid under JSON Schema Draft 7\n with Spack extensions *only*.\"\"\"\n # Collect schema submodules, and verify we have at least a few known ones\n schema_submodules = (\n importlib.import_module(f\"spack.schema.{name}\")\n for name in list_modules(os.path.dirname(spack.schema.__file__))\n )\n schemas = {m.__name__: m.schema for m in schema_submodules if hasattr(m, \"schema\")}\n assert set(schemas) >= {\"spack.schema.config\", \"spack.schema.packages\", \"spack.schema.modules\"}\n\n # Validate them using the meta-schema\n for module_name, module_schema in schemas.items():\n try:\n jsonschema.validate(module_schema, _draft_07_with_spack_extensions)\n except jsonschema.ValidationError as e:\n raise RuntimeError(f\"Invalid JSON schema in {module_name}: {e.message}\") from e\n\n\ndef test_env_schema_update_wrong_type():\n \"\"\"Confirm passing the wrong type to env.update() results in no changes.\"\"\"\n assert not spack.schema.env.update([\"a/b\"])\n" }, { "path": "lib/spack/spack/test/spack_yaml.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Test Spack's custom YAML format.\"\"\"\n\nimport io\nimport pathlib\n\nimport pytest\n\nimport spack.util.spack_yaml as syaml\nfrom spack.util.spack_yaml import DictWithLineInfo\n\n\n@pytest.fixture()\ndef data():\n \"\"\"Returns the data loaded from a test file\"\"\"\n test_file = \"\"\"\\\nconfig_file:\n x86_64:\n foo: /path/to/foo\n bar: /path/to/bar\n baz: /path/to/baz\n some_list:\n - item 1\n - item 2\n - item 3\n another_list:\n [ 1, 2, 3 ]\n some_key: some_string\n\"\"\"\n return syaml.load_config(test_file)\n\n\ndef test_parse(data):\n expected = {\n \"config_file\": syaml.syaml_dict(\n [\n (\n \"x86_64\",\n syaml.syaml_dict(\n [(\"foo\", \"/path/to/foo\"), (\"bar\", \"/path/to/bar\"), (\"baz\", \"/path/to/baz\")]\n ),\n ),\n (\"some_list\", [\"item 1\", \"item 2\", \"item 3\"]),\n (\"another_list\", [1, 2, 3]),\n (\"some_key\", \"some_string\"),\n ]\n )\n }\n\n assert data == expected\n\n\ndef test_dict_order(data):\n expected_order = [\"x86_64\", \"some_list\", \"another_list\", \"some_key\"]\n assert list(data[\"config_file\"].keys()) == expected_order\n\n expected_order = [\"foo\", \"bar\", \"baz\"]\n assert list(data[\"config_file\"][\"x86_64\"].keys()) == expected_order\n\n\ndef test_line_numbers(data):\n def check(obj, start_line, end_line):\n assert obj._start_mark.line == start_line\n assert obj._end_mark.line == end_line\n\n check(data, 0, 12)\n check(data[\"config_file\"], 1, 12)\n check(data[\"config_file\"][\"x86_64\"], 2, 5)\n check(data[\"config_file\"][\"x86_64\"][\"foo\"], 2, 2)\n check(data[\"config_file\"][\"x86_64\"][\"bar\"], 3, 3)\n check(data[\"config_file\"][\"x86_64\"][\"baz\"], 4, 4)\n check(data[\"config_file\"][\"some_list\"], 6, 9)\n check(data[\"config_file\"][\"some_list\"][0], 6, 6)\n check(data[\"config_file\"][\"some_list\"][1], 7, 7)\n check(data[\"config_file\"][\"some_list\"][2], 8, 8)\n check(data[\"config_file\"][\"another_list\"], 10, 10)\n check(data[\"config_file\"][\"some_key\"], 11, 11)\n\n\ndef test_yaml_aliases():\n aliased_list_1 = [\"foo\"]\n aliased_list_2 = []\n dict_with_aliases = {\n \"a\": aliased_list_1,\n \"b\": aliased_list_1,\n \"c\": aliased_list_1,\n \"d\": aliased_list_2,\n \"e\": aliased_list_2,\n \"f\": aliased_list_2,\n }\n stringio = io.StringIO()\n syaml.dump(dict_with_aliases, stream=stringio)\n\n # ensure no YAML aliases appear in syaml dumps.\n assert \"*id\" not in stringio.getvalue()\n\n\n@pytest.mark.parametrize(\n \"initial_content,expected_final_content\",\n [\n # List are dumped indented as the outer attribute\n (\n \"\"\"spack:\n #foo\n specs:\n # bar\n - zlib\n\"\"\",\n None,\n ),\n (\n \"\"\"spack:\n #foo\n specs:\n # bar\n - zlib\n\"\"\",\n \"\"\"spack:\n #foo\n specs:\n # bar\n - zlib\n\"\"\",\n ),\n ],\n)\n@pytest.mark.not_on_windows(reason=\"fails on Windows\")\ndef test_round_trip_configuration(initial_content, expected_final_content, tmp_path: pathlib.Path):\n \"\"\"Test that configuration can be loaded and dumped without too many changes\"\"\"\n file = tmp_path / \"test.yaml\"\n file.write_text(initial_content)\n final_content = io.StringIO()\n\n data = syaml.load_config(file)\n syaml.dump_config(data, stream=final_content)\n\n if expected_final_content is None:\n expected_final_content = initial_content\n\n assert final_content.getvalue() == expected_final_content\n\n\ndef test_sorted_dict():\n assert syaml.sorted_dict(\n {\n \"z\": 0,\n \"y\": [{\"x\": 0, \"w\": [2, 1, 0]}, 0],\n \"v\": ({\"u\": 0, \"t\": 0, \"s\": 0}, 0, {\"r\": 0, \"q\": 0}),\n \"p\": 0,\n }\n ) == {\n \"p\": 0,\n \"v\": ({\"s\": 0, \"t\": 0, \"u\": 0}, 0, {\"q\": 0, \"r\": 0}),\n \"y\": [{\"w\": [2, 1, 0], \"x\": 0}, 0],\n \"z\": 0,\n }\n\n\ndef test_deepcopy_to_native():\n yaml = \"\"\"\\\na:\n b: 1\n c: 1.0\n d:\n - e: false\n - f: null\n - \"string\"\n 2.0: \"float key\"\n 1: \"int key\"\n\"\"\"\n allowed_types = {str, int, float, bool, type(None), DictWithLineInfo, list}\n original = syaml.load(yaml)\n copied = syaml.deepcopy_as_builtin(original)\n assert original == copied\n assert type(copied[\"a\"][\"b\"]) is int\n assert type(copied[\"a\"][\"c\"]) is float\n assert type(copied[\"a\"][\"d\"][0][\"e\"]) is bool # edge case: bool is subclass of int\n assert type(copied[\"a\"][\"d\"][1][\"f\"]) is type(None)\n assert type(copied[\"a\"][\"d\"][2]) is str\n\n stack = [copied]\n while stack:\n obj = stack.pop()\n assert type(obj) in allowed_types\n if type(obj) is DictWithLineInfo:\n stack.extend(obj.keys())\n stack.extend(obj.values())\n elif type(obj) is list:\n stack.extend(obj)\n" }, { "path": "lib/spack/spack/test/spec_dag.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"\nThese tests check Spec DAG operations using dummy packages.\n\"\"\"\n\nimport pytest\n\nimport spack.concretize\nimport spack.deptypes as dt\nimport spack.error\nimport spack.installer\nimport spack.repo\nimport spack.solver.asp\nimport spack.util.hash as hashutil\nimport spack.version\nfrom spack.dependency import Dependency\nfrom spack.spec import Spec\nfrom spack.test.conftest import RepoBuilder\n\n\ndef check_links(spec_to_check):\n for spec in spec_to_check.traverse():\n for dependent in spec.dependents():\n assert dependent.edges_to_dependencies(name=spec.name)\n\n for dependency in spec.dependencies():\n assert dependency.edges_from_dependents(name=spec.name)\n\n\n@pytest.fixture()\ndef saved_deps():\n \"\"\"Returns a dictionary to save the dependencies.\"\"\"\n return {}\n\n\n@pytest.fixture()\ndef set_dependency(saved_deps, monkeypatch):\n \"\"\"Returns a function that alters the dependency information\n for a package in the ``saved_deps`` fixture.\n \"\"\"\n\n def _mock(pkg_name, spec):\n \"\"\"Alters dependence information for a package.\n\n Adds a dependency on to pkg. Use this to mock up constraints.\n \"\"\"\n spec = Spec(spec)\n # Save original dependencies before making any changes.\n pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)\n if pkg_name not in saved_deps:\n saved_deps[pkg_name] = (pkg_cls, pkg_cls.dependencies.copy())\n\n cond = Spec(pkg_cls.name)\n dependency = Dependency(pkg_cls, spec)\n monkeypatch.setitem(pkg_cls.dependencies, cond, {spec.name: dependency})\n\n return _mock\n\n\n@pytest.mark.usefixtures(\"config\")\ndef test_test_deptype(repo_builder: RepoBuilder):\n \"\"\"Ensure that test-only dependencies are only included for specified\n packages in the following spec DAG::\n\n w\n /|\n x y\n |\n z\n\n w->y deptypes are (link, build), w->x and y->z deptypes are (test)\n \"\"\"\n repo_builder.add_package(\"x\")\n repo_builder.add_package(\"z\")\n repo_builder.add_package(\"y\", dependencies=[(\"z\", \"test\", None)])\n repo_builder.add_package(\"w\", dependencies=[(\"x\", \"test\", None), (\"y\", None, None)])\n\n with spack.repo.use_repositories(repo_builder.root):\n spec = spack.concretize.concretize_one(\"w\", tests=(\"w\",))\n assert \"x\" in spec\n assert \"z\" not in spec\n\n\ndef test_installed_deps(monkeypatch, install_mockery):\n \"\"\"Ensure that concrete specs and their build deps don't constrain solves.\n\n Preinstall a package ``c`` that has a constrained build dependency on ``d``, then\n install ``a`` and ensure that neither:\n\n * ``c``'s package constraints, nor\n * the concrete ``c``'s build dependencies\n\n constrain ``a``'s dependency on ``d``.\n\n \"\"\"\n # see installed-deps-[abcde] test packages.\n # a\n # / \\\n # b c b --> d build/link\n # |\\ /| b --> e build/link\n # |/ \\| c --> d build\n # d e c --> e build/link\n #\n a, b, c, d, e = [f\"installed-deps-{s}\" for s in \"abcde\"]\n\n # install C, which will force d's version to be 2\n # BUT d is only a build dependency of C, so it won't constrain\n # link/run dependents of C when C is depended on as an existing\n # (concrete) installation.\n c_spec = spack.concretize.concretize_one(c)\n assert c_spec[d].version == spack.version.Version(\"2\")\n\n spack.installer.PackageInstaller([c_spec.package], fake=True, explicit=True).install()\n\n # install A, which depends on B, C, D, and E, and force A to\n # use the installed C. It should *not* force A to use the installed D\n # *if* we're doing a fresh installation.\n a_spec = spack.concretize.concretize_one(f\"{a} ^/{c_spec.dag_hash()}\")\n assert spack.version.Version(\"2\") == a_spec[c][d].version\n assert spack.version.Version(\"2\") == a_spec[e].version\n assert spack.version.Version(\"3\") == a_spec[b][d].version\n assert spack.version.Version(\"3\") == a_spec[d].version\n\n\n@pytest.mark.usefixtures(\"config\")\ndef test_specify_preinstalled_dep(monkeypatch, repo_builder: RepoBuilder):\n \"\"\"Specify the use of a preinstalled package during concretization with a\n transitive dependency that is only supplied by the preinstalled package.\n \"\"\"\n repo_builder.add_package(\"pkg-c\")\n repo_builder.add_package(\"pkg-b\", dependencies=[(\"pkg-c\", None, None)])\n repo_builder.add_package(\"pkg-a\", dependencies=[(\"pkg-b\", None, None)])\n\n with spack.repo.use_repositories(repo_builder.root):\n b_spec = spack.concretize.concretize_one(\"pkg-b\")\n monkeypatch.setattr(Spec, \"installed\", property(lambda x: x.name != \"pkg-a\"))\n\n a_spec = Spec(\"pkg-a\")\n a_spec._add_dependency(b_spec, depflag=dt.BUILD | dt.LINK, virtuals=())\n a_spec = spack.concretize.concretize_one(a_spec)\n\n assert {x.name for x in a_spec.traverse()} == {\"pkg-a\", \"pkg-b\", \"pkg-c\"}\n\n\n@pytest.mark.usefixtures(\"config\")\n@pytest.mark.parametrize(\n \"spec_str,expr_str,expected\",\n [(\"x ^y@2\", \"y@2\", True), (\"x@1\", \"y\", False), (\"x\", \"y@3\", True)],\n)\ndef test_conditional_dep_with_user_constraints(\n spec_str, expr_str, expected, repo_builder: RepoBuilder\n):\n \"\"\"This sets up packages X->Y such that X depends on Y conditionally. It\n then constructs a Spec with X but with no constraints on X, so that the\n initial normalization pass cannot determine whether the constraints are\n met to add the dependency; this checks whether a user-specified constraint\n on Y is applied properly.\n \"\"\"\n repo_builder.add_package(\"y\")\n repo_builder.add_package(\"x\", dependencies=[(\"y\", None, \"x@2:\")])\n\n with spack.repo.use_repositories(repo_builder.root):\n spec = spack.concretize.concretize_one(spec_str)\n result = expr_str in spec\n assert result is expected, \"{0} in {1}\".format(expr_str, spec)\n\n\n@pytest.mark.usefixtures(\"mutable_mock_repo\", \"config\")\nclass TestSpecDag:\n def test_conflicting_package_constraints(self, set_dependency):\n set_dependency(\"mpileaks\", \"mpich@1.0\")\n set_dependency(\"callpath\", \"mpich@2.0\")\n\n spec = Spec(\"mpileaks ^mpich ^callpath ^dyninst ^libelf ^libdwarf\")\n\n with pytest.raises(spack.error.UnsatisfiableSpecError):\n spack.concretize.concretize_one(spec)\n\n @pytest.mark.parametrize(\n \"pairs,traverse_kwargs\",\n [\n # Preorder node traversal\n (\n [\n (0, \"mpileaks\"),\n (1, \"callpath\"),\n (2, \"compiler-wrapper\"),\n (2, \"dyninst\"),\n (3, \"gcc\"),\n (3, \"gcc-runtime\"),\n (3, \"libdwarf\"),\n (4, \"libelf\"),\n (2, \"zmpi\"),\n (3, \"fake\"),\n ],\n {},\n ),\n # Preorder edge traversal\n (\n [\n (0, \"mpileaks\"),\n (1, \"callpath\"),\n (2, \"compiler-wrapper\"),\n (2, \"dyninst\"),\n (3, \"compiler-wrapper\"),\n (3, \"gcc\"),\n (3, \"gcc-runtime\"),\n (4, \"gcc\"),\n (3, \"libdwarf\"),\n (4, \"compiler-wrapper\"),\n (4, \"gcc\"),\n (4, \"gcc-runtime\"),\n (4, \"libelf\"),\n (5, \"compiler-wrapper\"),\n (5, \"gcc\"),\n (5, \"gcc-runtime\"),\n (3, \"libelf\"),\n (2, \"gcc\"),\n (2, \"gcc-runtime\"),\n (2, \"zmpi\"),\n (3, \"compiler-wrapper\"),\n (3, \"fake\"),\n (3, \"gcc\"),\n (3, \"gcc-runtime\"),\n (1, \"compiler-wrapper\"),\n (1, \"gcc\"),\n (1, \"gcc-runtime\"),\n (1, \"zmpi\"),\n ],\n {\"cover\": \"edges\"},\n ),\n # Preorder path traversal\n (\n [\n (0, \"mpileaks\"),\n (1, \"callpath\"),\n (2, \"compiler-wrapper\"),\n (2, \"dyninst\"),\n (3, \"compiler-wrapper\"),\n (3, \"gcc\"),\n (3, \"gcc-runtime\"),\n (4, \"gcc\"),\n (3, \"libdwarf\"),\n (4, \"compiler-wrapper\"),\n (4, \"gcc\"),\n (4, \"gcc-runtime\"),\n (5, \"gcc\"),\n (4, \"libelf\"),\n (5, \"compiler-wrapper\"),\n (5, \"gcc\"),\n (5, \"gcc-runtime\"),\n (6, \"gcc\"),\n (3, \"libelf\"),\n (4, \"compiler-wrapper\"),\n (4, \"gcc\"),\n (4, \"gcc-runtime\"),\n (5, \"gcc\"),\n (2, \"gcc\"),\n (2, \"gcc-runtime\"),\n (3, \"gcc\"),\n (2, \"zmpi\"),\n (3, \"compiler-wrapper\"),\n (3, \"fake\"),\n (3, \"gcc\"),\n (3, \"gcc-runtime\"),\n (4, \"gcc\"),\n (1, \"compiler-wrapper\"),\n (1, \"gcc\"),\n (1, \"gcc-runtime\"),\n (2, \"gcc\"),\n (1, \"zmpi\"),\n (2, \"compiler-wrapper\"),\n (2, \"fake\"),\n (2, \"gcc\"),\n (2, \"gcc-runtime\"),\n (3, \"gcc\"),\n ],\n {\"cover\": \"paths\"},\n ),\n # Postorder node traversal\n (\n [\n (2, \"compiler-wrapper\"),\n (3, \"gcc\"),\n (3, \"gcc-runtime\"),\n (4, \"libelf\"),\n (3, \"libdwarf\"),\n (2, \"dyninst\"),\n (3, \"fake\"),\n (2, \"zmpi\"),\n (1, \"callpath\"),\n (0, \"mpileaks\"),\n ],\n {\"order\": \"post\"},\n ),\n # Postorder edge traversal\n (\n [\n (2, \"compiler-wrapper\"),\n (3, \"compiler-wrapper\"),\n (3, \"gcc\"),\n (4, \"gcc\"),\n (3, \"gcc-runtime\"),\n (4, \"compiler-wrapper\"),\n (4, \"gcc\"),\n (4, \"gcc-runtime\"),\n (5, \"compiler-wrapper\"),\n (5, \"gcc\"),\n (5, \"gcc-runtime\"),\n (4, \"libelf\"),\n (3, \"libdwarf\"),\n (3, \"libelf\"),\n (2, \"dyninst\"),\n (2, \"gcc\"),\n (2, \"gcc-runtime\"),\n (3, \"compiler-wrapper\"),\n (3, \"fake\"),\n (3, \"gcc\"),\n (3, \"gcc-runtime\"),\n (2, \"zmpi\"),\n (1, \"callpath\"),\n (1, \"compiler-wrapper\"),\n (1, \"gcc\"),\n (1, \"gcc-runtime\"),\n (1, \"zmpi\"),\n (0, \"mpileaks\"),\n ],\n {\"cover\": \"edges\", \"order\": \"post\"},\n ),\n # Postorder path traversal\n (\n [\n (2, \"compiler-wrapper\"),\n (3, \"compiler-wrapper\"),\n (3, \"gcc\"),\n (4, \"gcc\"),\n (3, \"gcc-runtime\"),\n (4, \"compiler-wrapper\"),\n (4, \"gcc\"),\n (5, \"gcc\"),\n (4, \"gcc-runtime\"),\n (5, \"compiler-wrapper\"),\n (5, \"gcc\"),\n (6, \"gcc\"),\n (5, \"gcc-runtime\"),\n (4, \"libelf\"),\n (3, \"libdwarf\"),\n (4, \"compiler-wrapper\"),\n (4, \"gcc\"),\n (5, \"gcc\"),\n (4, \"gcc-runtime\"),\n (3, \"libelf\"),\n (2, \"dyninst\"),\n (2, \"gcc\"),\n (3, \"gcc\"),\n (2, \"gcc-runtime\"),\n (3, \"compiler-wrapper\"),\n (3, \"fake\"),\n (3, \"gcc\"),\n (4, \"gcc\"),\n (3, \"gcc-runtime\"),\n (2, \"zmpi\"),\n (1, \"callpath\"),\n (1, \"compiler-wrapper\"),\n (1, \"gcc\"),\n (2, \"gcc\"),\n (1, \"gcc-runtime\"),\n (2, \"compiler-wrapper\"),\n (2, \"fake\"),\n (2, \"gcc\"),\n (3, \"gcc\"),\n (2, \"gcc-runtime\"),\n (1, \"zmpi\"),\n (0, \"mpileaks\"),\n ],\n {\"cover\": \"paths\", \"order\": \"post\"},\n ),\n ],\n )\n def test_traversal(self, pairs, traverse_kwargs, default_mock_concretization):\n r\"\"\"Tests different traversals of the following graph\n\n o mpileaks@2.3/3qeg7jx\n |\\\n | |\\\n | | |\\\n | | | |\\\n | | | | |\\\n | | | | | o callpath@1.0/4gilijr\n | |_|_|_|/|\n |/| |_|_|/|\n | |/| |_|/|\n | | |/| |/|\n | | | |/|/|\n | | | | | o dyninst@8.2/u4oymb3\n | | |_|_|/|\n | |/| |_|/|\n | | |/| |/|\n | | | |/|/|\n | | | | | |\\\n o | | | | | | mpich@3.0.4/g734fu6\n |\\| | | | | |\n |\\ \\ \\ \\ \\ \\ \\\n | |_|/ / / / /\n |/| | | | | |\n | |\\ \\ \\ \\ \\ \\\n | | |_|/ / / /\n | |/| | | | |\n | | |/ / / /\n | | | | | o libdwarf@20130729/q5r7l2r\n | |_|_|_|/|\n |/| |_|_|/|\n | |/| |_|/|\n | | |/| |/|\n | | | |/|/\n | | | | o libelf@0.8.13/i2x6pya\n | |_|_|/|\n |/| |_|/|\n | |/| |/|\n | | |/|/\n | | o | compiler-wrapper@1.0/njdili2\n | | /\n o | | gcc-runtime@10.5.0/iyytqeo\n |\\| |\n | |/\n |/|\n | o gcc@10.5.0/ljeisd4\n |\n o glibc@2.31/tbyn33w\n \"\"\"\n dag = default_mock_concretization(\"mpileaks ^zmpi\")\n names = [x for _, x in pairs]\n\n traversal = dag.traverse(**traverse_kwargs, depth=True)\n assert [(x, y.name) for x, y in traversal] == pairs\n\n traversal = dag.traverse(**traverse_kwargs)\n assert [x.name for x in traversal] == names\n\n def test_dependents_and_dependencies_are_correct(self):\n spec = Spec.from_literal(\n {\n \"mpileaks\": {\n \"callpath\": {\n \"dyninst\": {\"libdwarf\": {\"libelf\": None}, \"libelf\": None},\n \"mpi\": None,\n },\n \"mpi\": None,\n }\n }\n )\n check_links(spec)\n concrete = spack.concretize.concretize_one(spec)\n check_links(concrete)\n\n @pytest.mark.parametrize(\n \"constraint_str,spec_str\",\n [\n (\"mpich@1.0\", \"mpileaks ^mpich@3.0\"),\n (\"mpich%gcc\", \"mpileaks ^mpich%intel\"),\n (\"mpich%gcc@2.0\", \"mpileaks ^mpich%gcc@3.0\"),\n ],\n )\n def test_unsatisfiable_cases(self, set_dependency, constraint_str, spec_str):\n \"\"\"Tests that synthetic cases of conflicting requirements raise an UnsatisfiableSpecError\n when concretizing.\n \"\"\"\n set_dependency(\"mpileaks\", constraint_str)\n with pytest.raises(spack.error.UnsatisfiableSpecError):\n spack.concretize.concretize_one(spec_str)\n\n @pytest.mark.parametrize(\n \"spec_str\", [\"libelf ^mpich\", \"libelf ^libdwarf\", \"mpich ^dyninst ^libelf\"]\n )\n def test_invalid_dep(self, spec_str):\n spec = Spec(spec_str)\n with pytest.raises(spack.solver.asp.InvalidDependencyError):\n spack.concretize.concretize_one(spec)\n\n def test_equal(self):\n # Different spec structures to test for equality\n flat = Spec.from_literal({\"mpileaks ^callpath ^libelf ^libdwarf\": None})\n\n flat_init = Spec.from_literal(\n {\"mpileaks\": {\"callpath\": None, \"libdwarf\": None, \"libelf\": None}}\n )\n\n flip_flat = Spec.from_literal(\n {\"mpileaks\": {\"libelf\": None, \"libdwarf\": None, \"callpath\": None}}\n )\n\n dag = Spec.from_literal({\"mpileaks\": {\"callpath\": {\"libdwarf\": {\"libelf\": None}}}})\n\n flip_dag = Spec.from_literal({\"mpileaks\": {\"callpath\": {\"libelf\": {\"libdwarf\": None}}}})\n\n # All these are equal to each other with regular ==\n specs = (flat, flat_init, flip_flat, dag, flip_dag)\n for lhs, rhs in zip(specs, specs):\n assert lhs == rhs\n assert str(lhs) == str(rhs)\n\n # Same DAGs constructed different ways are equal\n assert flat.eq_dag(flat_init)\n\n # order at same level does not matter -- (dep on same parent)\n assert flat.eq_dag(flip_flat)\n\n # DAGs should be unequal if nesting is different\n assert not flat.eq_dag(dag)\n assert not flat.eq_dag(flip_dag)\n assert not flip_flat.eq_dag(dag)\n assert not flip_flat.eq_dag(flip_dag)\n assert not dag.eq_dag(flip_dag)\n\n def test_contains(self):\n spec = Spec(\"mpileaks ^mpi ^libelf@1.8.11 ^libdwarf\")\n assert Spec(\"mpi\") in spec\n assert Spec(\"libelf\") in spec\n assert Spec(\"libelf@1.8.11\") in spec\n assert Spec(\"libelf@1.8.12\") not in spec\n assert Spec(\"libdwarf\") in spec\n assert Spec(\"libgoblin\") not in spec\n assert Spec(\"mpileaks\") in spec\n\n def test_copy_simple(self):\n orig = Spec(\"mpileaks\")\n copy = orig.copy()\n check_links(copy)\n\n assert orig == copy\n assert orig.eq_dag(copy)\n assert orig._concrete == copy._concrete\n\n # ensure no shared nodes bt/w orig and copy.\n orig_ids = set(id(s) for s in orig.traverse())\n copy_ids = set(id(s) for s in copy.traverse())\n assert not orig_ids.intersection(copy_ids)\n\n def test_copy_concretized(self):\n orig = spack.concretize.concretize_one(\"mpileaks\")\n copy = orig.copy()\n\n check_links(copy)\n\n assert orig == copy\n assert orig.eq_dag(copy)\n assert orig._concrete == copy._concrete\n\n # ensure no shared nodes bt/w orig and copy.\n orig_ids = set(id(s) for s in orig.traverse())\n copy_ids = set(id(s) for s in copy.traverse())\n assert not orig_ids.intersection(copy_ids)\n\n def test_copy_through_spec_build_interface(self):\n \"\"\"Check that copying dependencies using id(node) as a fast identifier of the\n node works when the spec is wrapped in a SpecBuildInterface object.\n \"\"\"\n s = spack.concretize.concretize_one(\"mpileaks\")\n\n c0 = s.copy()\n assert c0 == s\n\n # Single indirection\n c1 = s[\"mpileaks\"].copy()\n assert c0 == c1 == s\n\n # Double indirection\n c2 = s[\"mpileaks\"][\"mpileaks\"].copy()\n assert c0 == c1 == c2 == s\n\n # Here is the graph with deptypes labeled (assume all packages have a 'dt'\n # prefix). Arrows are marked with the deptypes ('b' for 'build', 'l' for\n # 'link', 'r' for 'run').\n\n # use -bl-> top\n\n # top -b-> build1\n # top -bl-> link1\n # top -r-> run1\n\n # build1 -b-> build2\n # build1 -bl-> link2\n # build1 -r-> run2\n\n # link1 -bl-> link3\n\n # run1 -bl-> link5\n # run1 -r-> run3\n\n # link3 -b-> build2\n # link3 -bl-> link4\n\n # run3 -b-> build3\n\n @pytest.mark.parametrize(\n \"spec_str,deptypes,expected\",\n [\n (\n \"dtuse\",\n (\"build\", \"link\"),\n [\n \"dtuse\",\n \"dttop\",\n \"dtbuild1\",\n \"dtbuild2\",\n \"dtlink2\",\n \"dtlink1\",\n \"dtlink3\",\n \"dtlink4\",\n ],\n ),\n (\n \"dttop\",\n (\"build\", \"link\"),\n [\"dttop\", \"dtbuild1\", \"dtbuild2\", \"dtlink2\", \"dtlink1\", \"dtlink3\", \"dtlink4\"],\n ),\n (\n \"dttop\",\n all,\n [\n \"dttop\",\n \"dtbuild1\",\n \"dtbuild2\",\n \"dtlink2\",\n \"dtrun2\",\n \"dtlink1\",\n \"dtlink3\",\n \"dtlink4\",\n \"dtrun1\",\n \"dtlink5\",\n \"dtrun3\",\n \"dtbuild3\",\n ],\n ),\n (\"dttop\", \"run\", [\"dttop\", \"dtrun1\", \"dtrun3\"]),\n ],\n )\n def test_deptype_traversal(self, spec_str, deptypes, expected):\n dag = spack.concretize.concretize_one(spec_str)\n traversal = dag.traverse(deptype=deptypes)\n assert [x.name for x in traversal] == expected\n\n def test_hash_bits(self):\n \"\"\"Ensure getting first n bits of a base32-encoded DAG hash works.\"\"\"\n\n # RFC 4648 base32 decode table\n b32 = dict((j, i) for i, j in enumerate(\"abcdefghijklmnopqrstuvwxyz\"))\n b32.update(dict((j, i) for i, j in enumerate(\"234567\", 26)))\n\n # some package hashes\n tests = [\n \"35orsd4cenv743hg4i5vxha2lzayycby\",\n \"6kfqtj7dap3773rxog6kkmoweix5gpwo\",\n \"e6h6ff3uvmjbq3azik2ckr6ckwm3depv\",\n \"snz2juf4ij7sv77cq3vs467q6acftmur\",\n \"4eg47oedi5bbkhpoxw26v3oe6vamkfd7\",\n \"vrwabwj6umeb5vjw6flx2rnft3j457rw\",\n ]\n\n for test_hash in tests:\n # string containing raw bits of hash ('1' and '0')\n expected = \"\".join([format(b32[c], \"#07b\").replace(\"0b\", \"\") for c in test_hash])\n\n for bits in (1, 2, 3, 4, 7, 8, 9, 16, 64, 117, 128, 160):\n actual_int = hashutil.base32_prefix_bits(test_hash, bits)\n fmt = \"#0%sb\" % (bits + 2)\n actual = format(actual_int, fmt).replace(\"0b\", \"\")\n\n assert expected[:bits] == actual\n\n with pytest.raises(ValueError):\n hashutil.base32_prefix_bits(test_hash, 161)\n\n with pytest.raises(ValueError):\n hashutil.base32_prefix_bits(test_hash, 256)\n\n def test_traversal_directions(self):\n \"\"\"Make sure child and parent traversals of specs work.\"\"\"\n # Mock spec - d is used for a diamond dependency\n spec = Spec.from_literal(\n {\"a\": {\"b\": {\"c\": {\"d\": None}, \"e\": None}, \"f\": {\"g\": {\"d\": None}}}}\n )\n\n assert [\"a\", \"b\", \"c\", \"d\", \"e\", \"f\", \"g\"] == [\n s.name for s in spec.traverse(direction=\"children\")\n ]\n\n assert [\"g\", \"f\", \"a\"] == [s.name for s in spec[\"g\"].traverse(direction=\"parents\")]\n\n assert [\"d\", \"c\", \"b\", \"a\", \"g\", \"f\"] == [\n s.name for s in spec[\"d\"].traverse(direction=\"parents\")\n ]\n\n def test_edge_traversals(self):\n \"\"\"Make sure child and parent traversals of specs work.\"\"\"\n # Mock spec - d is used for a diamond dependency\n spec = Spec.from_literal(\n {\"a\": {\"b\": {\"c\": {\"d\": None}, \"e\": None}, \"f\": {\"g\": {\"d\": None}}}}\n )\n\n assert [\"a\", \"b\", \"c\", \"d\", \"e\", \"f\", \"g\"] == [\n s.name for s in spec.traverse(direction=\"children\")\n ]\n\n assert [\"g\", \"f\", \"a\"] == [s.name for s in spec[\"g\"].traverse(direction=\"parents\")]\n\n assert [\"d\", \"c\", \"b\", \"a\", \"g\", \"f\"] == [\n s.name for s in spec[\"d\"].traverse(direction=\"parents\")\n ]\n\n def test_copy_dependencies(self):\n s1 = Spec(\"mpileaks ^mpich2@1.1\")\n s2 = s1.copy()\n\n assert \"^mpich2@1.1\" in s2\n assert \"^mpich2\" in s2\n\n def test_construct_spec_with_deptypes(self):\n \"\"\"Ensure that it is possible to construct a spec with explicit\n dependency types.\"\"\"\n s = Spec.from_literal(\n {\"a\": {\"b\": {\"c:build\": None}, \"d\": {\"e:build,link\": {\"f:run\": None}}}}\n )\n\n assert s[\"b\"].edges_to_dependencies(name=\"c\")[0].depflag == dt.BUILD\n assert s[\"d\"].edges_to_dependencies(name=\"e\")[0].depflag == dt.BUILD | dt.LINK\n assert s[\"e\"].edges_to_dependencies(name=\"f\")[0].depflag == dt.RUN\n # The subscript follows link/run transitive deps or direct build/test deps, therefore\n # we need an extra step to get to \"c\"\n assert s[\"b\"][\"c\"].edges_from_dependents(name=\"b\")[0].depflag == dt.BUILD\n assert s[\"e\"].edges_from_dependents(name=\"d\")[0].depflag == dt.BUILD | dt.LINK\n assert s[\"f\"].edges_from_dependents(name=\"e\")[0].depflag == dt.RUN\n\n def check_diamond_deptypes(self, spec):\n \"\"\"Validate deptypes in dt-diamond spec.\n\n This ensures that concretization works properly when two packages\n depend on the same dependency in different ways.\n\n \"\"\"\n assert (\n spec[\"dt-diamond\"].edges_to_dependencies(name=\"dt-diamond-left\")[0].depflag\n == dt.BUILD | dt.LINK\n )\n assert (\n spec[\"dt-diamond\"].edges_to_dependencies(name=\"dt-diamond-right\")[0].depflag\n == dt.BUILD | dt.LINK\n )\n assert (\n spec[\"dt-diamond-left\"].edges_to_dependencies(name=\"dt-diamond-bottom\")[0].depflag\n == dt.BUILD\n )\n assert (\n spec[\"dt-diamond-right\"].edges_to_dependencies(name=\"dt-diamond-bottom\")[0].depflag\n == dt.BUILD | dt.LINK | dt.RUN\n )\n\n def test_concretize_deptypes(self):\n \"\"\"Ensure that dependency types are preserved after concretization.\"\"\"\n s = spack.concretize.concretize_one(\"dt-diamond\")\n self.check_diamond_deptypes(s)\n\n def test_copy_deptypes(self):\n \"\"\"Ensure that dependency types are preserved by spec copy.\"\"\"\n s1 = spack.concretize.concretize_one(\"dt-diamond\")\n self.check_diamond_deptypes(s1)\n s2 = s1.copy()\n self.check_diamond_deptypes(s2)\n\n def test_getitem_query(self):\n s = spack.concretize.concretize_one(\"mpileaks\")\n\n # Check a query to a non-virtual package\n a = s[\"callpath\"]\n\n query = a.last_query\n assert query.name == \"callpath\"\n assert len(query.extra_parameters) == 0\n assert not query.isvirtual\n\n # Check a query to a virtual package\n a = s[\"mpi\"]\n\n query = a.last_query\n assert query.name == \"mpi\"\n assert len(query.extra_parameters) == 0\n assert query.isvirtual\n\n # Check a query to a virtual package with\n # extra parameters after query\n a = s[\"mpi:cxx,fortran\"]\n\n query = a.last_query\n assert query.name == \"mpi\"\n assert len(query.extra_parameters) == 2\n assert \"cxx\" in query.extra_parameters\n assert \"fortran\" in query.extra_parameters\n assert query.isvirtual\n\n def test_getitem_exceptional_paths(self):\n s = spack.concretize.concretize_one(\"mpileaks\")\n # Needed to get a proxy object\n q = s[\"mpileaks\"]\n\n # Test that the attribute is read-only\n with pytest.raises(AttributeError):\n q.libs = \"foo\"\n\n with pytest.raises(AttributeError):\n q.libs\n\n def test_canonical_deptype(self):\n # special values\n assert dt.canonicalize(all) == dt.ALL\n assert dt.canonicalize(\"all\") == dt.ALL\n\n with pytest.raises(ValueError):\n dt.canonicalize(None)\n with pytest.raises(ValueError):\n dt.canonicalize([None])\n\n # everything in all_types is canonical\n for v in dt.ALL_TYPES:\n assert dt.canonicalize(v) == dt.flag_from_string(v)\n\n # tuples\n assert dt.canonicalize((\"build\",)) == dt.BUILD\n assert dt.canonicalize((\"build\", \"link\", \"run\")) == dt.BUILD | dt.LINK | dt.RUN\n assert dt.canonicalize((\"build\", \"link\")) == dt.BUILD | dt.LINK\n assert dt.canonicalize((\"build\", \"run\")) == dt.BUILD | dt.RUN\n\n # lists\n assert dt.canonicalize([\"build\", \"link\", \"run\"]) == dt.BUILD | dt.LINK | dt.RUN\n assert dt.canonicalize([\"build\", \"link\"]) == dt.BUILD | dt.LINK\n assert dt.canonicalize([\"build\", \"run\"]) == dt.BUILD | dt.RUN\n\n # sorting\n assert dt.canonicalize((\"run\", \"build\", \"link\")) == dt.BUILD | dt.LINK | dt.RUN\n assert dt.canonicalize((\"run\", \"link\", \"build\")) == dt.BUILD | dt.LINK | dt.RUN\n assert dt.canonicalize((\"run\", \"link\")) == dt.LINK | dt.RUN\n assert dt.canonicalize((\"link\", \"build\")) == dt.BUILD | dt.LINK\n\n # deduplication\n assert dt.canonicalize((\"run\", \"run\", \"link\")) == dt.RUN | dt.LINK\n assert dt.canonicalize((\"run\", \"link\", \"link\")) == dt.RUN | dt.LINK\n\n # can't put 'all' in tuple or list\n with pytest.raises(ValueError):\n dt.canonicalize([\"all\"])\n with pytest.raises(ValueError):\n dt.canonicalize((\"all\",))\n\n # invalid values\n with pytest.raises(ValueError):\n dt.canonicalize(\"foo\")\n with pytest.raises(ValueError):\n dt.canonicalize((\"foo\", \"bar\"))\n with pytest.raises(ValueError):\n dt.canonicalize((\"foo\",))\n\n def test_invalid_literal_spec(self):\n # Can't give type 'build' to a top-level spec\n with pytest.raises(spack.error.SpecSyntaxError):\n Spec.from_literal({\"foo:build\": None})\n\n # Can't use more than one ':' separator\n with pytest.raises(KeyError):\n Spec.from_literal({\"foo\": {\"bar:build:link\": None}})\n\n def test_spec_tree_respect_deptypes(self):\n # Version-test-root uses version-test-pkg as a build dependency\n s = spack.concretize.concretize_one(\"version-test-root\")\n out = s.tree(deptypes=\"all\")\n assert \"version-test-pkg\" in out\n out = s.tree(deptypes=(\"link\", \"run\"))\n assert \"version-test-pkg\" not in out\n\n @pytest.mark.parametrize(\n \"query,expected_length,expected_satisfies\",\n [\n ({\"virtuals\": [\"mpi\"]}, 1, [\"mpich\", \"mpi\"]),\n ({\"depflag\": dt.BUILD}, 4, [\"mpich\", \"mpi\", \"callpath\"]),\n ({\"depflag\": dt.BUILD, \"virtuals\": [\"mpi\"]}, 1, [\"mpich\", \"mpi\"]),\n ({\"depflag\": dt.LINK}, 3, [\"mpich\", \"mpi\", \"callpath\"]),\n ({\"depflag\": dt.BUILD | dt.LINK}, 5, [\"mpich\", \"mpi\", \"callpath\"]),\n ({\"virtuals\": [\"lapack\"]}, 0, []),\n ],\n )\n def test_query_dependency_edges(\n self, default_mock_concretization, query, expected_length, expected_satisfies\n ):\n \"\"\"Tests querying edges to dependencies on the following DAG:\n\n - [ ] mpileaks@2.3\n - [bl ] ^callpath@1.0\n - [bl ] ^dyninst@8.2\n - [bl ] ^libdwarf@20130729\n - [bl ] ^libelf@0.8.13\n [e] [b ] ^gcc@10.1.0\n - [ l ] ^gcc-runtime@10.1.0\n - [bl ] ^mpich@3.0.4~debug\n \"\"\"\n mpileaks = default_mock_concretization(\"mpileaks\")\n edges = mpileaks.edges_to_dependencies(**query)\n assert len(edges) == expected_length\n for constraint in expected_satisfies:\n assert any(x.spec.satisfies(constraint) for x in edges)\n\n def test_query_dependents_edges(self, default_mock_concretization):\n \"\"\"Tests querying edges from dependents\"\"\"\n mpileaks = default_mock_concretization(\"mpileaks\")\n mpich = mpileaks[\"mpich\"]\n\n # Recover the root with 2 different queries\n edges_of_link_type = mpich.edges_from_dependents(depflag=dt.LINK)\n edges_with_mpi = mpich.edges_from_dependents(virtuals=[\"mpi\"])\n assert edges_with_mpi == edges_of_link_type\n\n # Check a node depended upon by 2 parents\n assert len(mpileaks[\"libelf\"].edges_from_dependents(depflag=dt.LINK)) == 2\n\n\ndef test_tree_cover_nodes_reduce_deptype():\n \"\"\"Test that tree output with deptypes sticks to the sub-dag of interest, instead of looking\n at in-edges from nodes not reachable from the root.\"\"\"\n a, b, c, d = Spec(\"a\"), Spec(\"b\"), Spec(\"c\"), Spec(\"d\")\n a.add_dependency_edge(d, depflag=dt.BUILD, virtuals=())\n a.add_dependency_edge(b, depflag=dt.LINK, virtuals=())\n b.add_dependency_edge(d, depflag=dt.LINK, virtuals=())\n c.add_dependency_edge(d, depflag=dt.RUN | dt.TEST, virtuals=())\n assert (\n a.tree(cover=\"nodes\", show_types=True)\n == \"\"\"\\\n[ ] a\n[ l ] ^b\n[bl ] ^d\n\"\"\"\n )\n assert (\n c.tree(cover=\"nodes\", show_types=True)\n == \"\"\"\\\n[ ] c\n[ rt] ^d\n\"\"\"\n )\n\n\ndef test_synthetic_construction_of_split_dependencies_from_same_package(mock_packages, config):\n # Construct in a synthetic way (i.e. without using the solver)\n # the following spec:\n #\n # pkg-b\n # build / \\ link,run\n # pkg-c@2.0 pkg-c@1.0\n #\n # To demonstrate that a spec can now hold two direct\n # dependencies from the same package\n root = spack.concretize.concretize_one(\"pkg-b\")\n link_run_spec = spack.concretize.concretize_one(\"pkg-c@=1.0\")\n build_spec = spack.concretize.concretize_one(\"pkg-c@=2.0\")\n\n root.add_dependency_edge(link_run_spec, depflag=dt.LINK, virtuals=())\n root.add_dependency_edge(link_run_spec, depflag=dt.RUN, virtuals=())\n root.add_dependency_edge(build_spec, depflag=dt.BUILD, virtuals=())\n\n # Check dependencies from the perspective of root\n assert len(root.dependencies()) == 5\n assert len([x for x in root.dependencies() if x.name == \"pkg-c\"]) == 2\n\n assert \"@2.0\" in root.dependencies(name=\"pkg-c\", deptype=dt.BUILD)[0]\n assert \"@1.0\" in root.dependencies(name=\"pkg-c\", deptype=dt.LINK | dt.RUN)[0]\n\n # Check parent from the perspective of the dependencies\n assert len(build_spec.dependents()) == 1\n assert len(link_run_spec.dependents()) == 1\n assert build_spec.dependents() == link_run_spec.dependents()\n assert build_spec != link_run_spec\n\n\ndef test_synthetic_construction_bootstrapping(mock_packages, config):\n # Construct the following spec:\n #\n # pkg-b@2.0\n # | build\n # pkg-b@1.0\n #\n root = spack.concretize.concretize_one(\"pkg-b@=2.0\")\n bootstrap = spack.concretize.concretize_one(\"pkg-b@=1.0\")\n\n root.add_dependency_edge(bootstrap, depflag=dt.BUILD, virtuals=())\n\n assert len([x for x in root.dependencies() if x.name == \"pkg-b\"]) == 1\n assert root.name == \"pkg-b\"\n\n\ndef test_addition_of_different_deptypes_in_multiple_calls(mock_packages, config):\n # Construct the following spec:\n #\n # pkg-b@2.0\n # | build,link,run\n # pkg-b@1.0\n #\n # with three calls and check we always have a single edge\n root = spack.concretize.concretize_one(\"pkg-b@=2.0\")\n bootstrap = spack.concretize.concretize_one(\"pkg-b@=1.0\")\n\n for current_depflag in (dt.BUILD, dt.LINK, dt.RUN):\n root.add_dependency_edge(bootstrap, depflag=current_depflag, virtuals=())\n\n # Check edges in dependencies\n assert len(root.edges_to_dependencies(name=\"pkg-b\")) == 1\n forward_edge = root.edges_to_dependencies(depflag=current_depflag, name=\"pkg-b\")[0]\n assert current_depflag & forward_edge.depflag\n assert id(forward_edge.parent) == id(root)\n assert id(forward_edge.spec) == id(bootstrap)\n\n # Check edges from dependents\n assert len(bootstrap.edges_from_dependents()) == 1\n backward_edge = bootstrap.edges_from_dependents(depflag=current_depflag)[0]\n assert current_depflag & backward_edge.depflag\n assert id(backward_edge.parent) == id(root)\n assert id(backward_edge.spec) == id(bootstrap)\n\n\n@pytest.mark.parametrize(\n \"c1_depflag,c2_depflag\",\n [(dt.LINK, dt.BUILD | dt.LINK), (dt.LINK | dt.RUN, dt.BUILD | dt.LINK)],\n)\ndef test_adding_same_deptype_with_the_same_name_raises(\n mock_packages, config, c1_depflag, c2_depflag\n):\n p = spack.concretize.concretize_one(\"pkg-b@=2.0\")\n c1 = spack.concretize.concretize_one(\"pkg-b@=1.0\")\n c2 = spack.concretize.concretize_one(\"pkg-b@=2.0\")\n\n p.add_dependency_edge(c1, depflag=c1_depflag, virtuals=())\n with pytest.raises(spack.error.SpackError):\n p.add_dependency_edge(c2, depflag=c2_depflag, virtuals=())\n\n\n@pytest.mark.regression(\"33499\")\ndef test_indexing_prefers_direct_or_transitive_link_deps():\n \"\"\"Tests whether spec indexing prefers direct/transitive link/run type deps over deps of\n build/test deps.\n \"\"\"\n root = Spec(\"root\")\n\n # Use a and z to since we typically traverse by edges sorted alphabetically.\n a1 = Spec(\"a1\")\n a2 = Spec(\"a2\")\n z1 = Spec(\"z1\")\n z2 = Spec(\"z2\")\n\n # Same package, different spec.\n z3_flavor_1 = Spec(\"z3 +through_a1\")\n z3_flavor_2 = Spec(\"z3 +through_z1\")\n\n root.add_dependency_edge(a1, depflag=dt.BUILD | dt.TEST, virtuals=())\n\n # unique package as a dep of a build/run/test type dep.\n a1.add_dependency_edge(a2, depflag=dt.ALL, virtuals=())\n a1.add_dependency_edge(z3_flavor_1, depflag=dt.ALL, virtuals=())\n\n # chain of link type deps root -> z1 -> z2 -> z3\n root.add_dependency_edge(z1, depflag=dt.LINK, virtuals=())\n z1.add_dependency_edge(z2, depflag=dt.LINK, virtuals=())\n z2.add_dependency_edge(z3_flavor_2, depflag=dt.LINK, virtuals=())\n\n # Indexing should prefer the link-type dep.\n assert \"through_z1\" in root[\"z3\"].variants\n assert \"through_a1\" in a1[\"z3\"].variants\n\n # Ensure that only the runtime sub-DAG can be searched\n with pytest.raises(KeyError):\n root[\"a2\"]\n\n # Check consistency of __contains__ with __getitem__\n assert \"z3 +through_z1\" in root\n assert \"z3 +through_a1\" in a1\n assert \"a2\" not in root\n\n\ndef test_getitem_sticks_to_subdag():\n \"\"\"Test that indexing on Spec by virtual does not traverse outside the dag, which happens in\n the unlikely case someone would rewrite __getitem__ in terms of edges_from_dependents instead\n of edges_to_dependencies.\"\"\"\n x, y, z = Spec(\"x\"), Spec(\"y\"), Spec(\"z\")\n x.add_dependency_edge(z, depflag=dt.LINK, virtuals=(\"virtual\",))\n y.add_dependency_edge(z, depflag=dt.LINK, virtuals=())\n assert x[\"virtual\"].name == \"z\"\n with pytest.raises(KeyError):\n y[\"virtual\"]\n\n\ndef test_getitem_finds_transitive_virtual():\n x, y, z = Spec(\"x\"), Spec(\"y\"), Spec(\"z\")\n x.add_dependency_edge(z, depflag=dt.LINK, virtuals=())\n x.add_dependency_edge(y, depflag=dt.LINK, virtuals=())\n y.add_dependency_edge(z, depflag=dt.LINK, virtuals=(\"virtual\",))\n assert x[\"virtual\"].name == \"z\"\n" }, { "path": "lib/spack/spack/test/spec_list.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport itertools\n\nimport pytest\n\nimport spack.concretize\nfrom spack.environment.list import SpecListParser\nfrom spack.installer import PackageInstaller\nfrom spack.spec import Spec\n\nDEFAULT_EXPANSION = [\n \"mpileaks\",\n \"zmpi@1.0\",\n \"mpich@3.0\",\n {\"matrix\": [[\"hypre\"], [\"%gcc@4.5.0\", \"%clang@3.3\"]]},\n \"libelf\",\n]\n\nDEFAULT_CONSTRAINTS = [\n [Spec(\"mpileaks\")],\n [Spec(\"zmpi@1.0\")],\n [Spec(\"mpich@3.0\")],\n [Spec(\"hypre\"), Spec(\"%gcc@4.5.0\")],\n [Spec(\"hypre\"), Spec(\"%clang@3.3\")],\n [Spec(\"libelf\")],\n]\n\nDEFAULT_SPECS = [\n Spec(\"mpileaks\"),\n Spec(\"zmpi@1.0\"),\n Spec(\"mpich@3.0\"),\n Spec(\"hypre%gcc@4.5.0\"),\n Spec(\"hypre%clang@3.3\"),\n Spec(\"libelf\"),\n]\n\n\n@pytest.fixture()\ndef parser_and_speclist():\n \"\"\"Default configuration of parser and user spec list for tests\"\"\"\n parser = SpecListParser()\n parser.parse_definitions(\n data=[\n {\"gccs\": [\"%gcc@4.5.0\"]},\n {\"clangs\": [\"%clang@3.3\"]},\n {\"mpis\": [\"zmpi@1.0\", \"mpich@3.0\"]},\n ]\n )\n result = parser.parse_user_specs(\n name=\"specs\",\n yaml_list=[\"mpileaks\", \"$mpis\", {\"matrix\": [[\"hypre\"], [\"$gccs\", \"$clangs\"]]}, \"libelf\"],\n )\n return parser, result\n\n\n@pytest.mark.usefixtures(\"mock_packages\")\nclass TestSpecList:\n @pytest.mark.regression(\"28749\")\n @pytest.mark.parametrize(\n \"specs,expected\",\n [\n # Constraints are ordered randomly\n (\n [\n {\n \"matrix\": [\n [\"^zmpi\"],\n [\"%gcc@4.5.0\"],\n [\"hypre\", \"libelf\"],\n [\"~shared\"],\n [\"cflags=-O3\", 'cflags=\"-g -O0\"'],\n [\"^foo\"],\n ]\n }\n ],\n [\n \"hypre cflags=-O3 ~shared %gcc@4.5.0 ^foo ^zmpi\",\n 'hypre cflags=\"-g -O0\" ~shared %gcc@4.5.0 ^foo ^zmpi',\n \"libelf cflags=-O3 ~shared %gcc@4.5.0 ^foo ^zmpi\",\n 'libelf cflags=\"-g -O0\" ~shared %gcc@4.5.0 ^foo ^zmpi',\n ],\n ),\n # A constraint affects both the root and a dependency\n (\n [{\"matrix\": [[\"version-test-root\"], [\"%gcc\"], [\"^version-test-pkg%gcc\"]]}],\n [\"version-test-root%gcc ^version-test-pkg%gcc\"],\n ),\n ],\n )\n def test_spec_list_constraint_ordering(self, specs, expected):\n result = SpecListParser().parse_user_specs(name=\"specs\", yaml_list=specs)\n assert result.specs == [Spec(x) for x in expected]\n\n def test_mock_spec_list(self, parser_and_speclist):\n \"\"\"Tests expected properties on the default mock spec list\"\"\"\n parser, mock_list = parser_and_speclist\n assert mock_list.specs_as_yaml_list == DEFAULT_EXPANSION\n assert mock_list.specs_as_constraints == DEFAULT_CONSTRAINTS\n assert mock_list.specs == DEFAULT_SPECS\n\n def test_spec_list_add(self, parser_and_speclist):\n parser, mock_list = parser_and_speclist\n mock_list.add(\"libdwarf\")\n\n assert mock_list.specs_as_yaml_list == DEFAULT_EXPANSION + [\"libdwarf\"]\n assert mock_list.specs_as_constraints == DEFAULT_CONSTRAINTS + [[Spec(\"libdwarf\")]]\n assert mock_list.specs == DEFAULT_SPECS + [Spec(\"libdwarf\")]\n\n def test_spec_list_remove(self, parser_and_speclist):\n parser, mock_list = parser_and_speclist\n mock_list.remove(\"libelf\")\n\n assert mock_list.specs_as_yaml_list + [\"libelf\"] == DEFAULT_EXPANSION\n assert mock_list.specs_as_constraints + [[Spec(\"libelf\")]] == DEFAULT_CONSTRAINTS\n assert mock_list.specs + [Spec(\"libelf\")] == DEFAULT_SPECS\n\n def test_spec_list_extension(self, parser_and_speclist):\n parser, mock_list = parser_and_speclist\n other_list = parser.parse_user_specs(\n name=\"specs\", yaml_list=[{\"matrix\": [[\"callpath\"], [\"%intel@18\"]]}]\n )\n mock_list.extend(other_list)\n\n assert mock_list.specs_as_yaml_list == (DEFAULT_EXPANSION + other_list.specs_as_yaml_list)\n assert mock_list.specs == DEFAULT_SPECS + other_list.specs\n\n def test_spec_list_nested_matrices(self, parser_and_speclist):\n parser, _ = parser_and_speclist\n\n inner_matrix = [{\"matrix\": [[\"zlib\", \"libelf\"], [\"%gcc\", \"%intel\"]]}]\n outer_addition = [\"+shared\", \"~shared\"]\n outer_matrix = [{\"matrix\": [inner_matrix, outer_addition]}]\n result = parser.parse_user_specs(name=\"specs\", yaml_list=outer_matrix)\n\n expected_components = itertools.product(\n [\"zlib\", \"libelf\"], [\"%gcc\", \"%intel\"], [\"+shared\", \"~shared\"]\n )\n\n def _reduce(*, combo):\n root = Spec(combo[0])\n for x in combo[1:]:\n root.constrain(x)\n return root\n\n expected = [_reduce(combo=combo) for combo in expected_components]\n assert set(result.specs) == set(expected)\n\n @pytest.mark.regression(\"16897\")\n def test_spec_list_recursion_specs_as_constraints(self):\n input = [\"mpileaks\", \"$mpis\", {\"matrix\": [[\"hypre\"], [\"$%gccs\", \"$%clangs\"]]}, \"libelf\"]\n\n definitions = [\n {\"gccs\": [\"gcc@4.5.0\"]},\n {\"clangs\": [\"clang@3.3\"]},\n {\"mpis\": [\"zmpi@1.0\", \"mpich@3.0\"]},\n ]\n\n parser = SpecListParser()\n parser.parse_definitions(data=definitions)\n result = parser.parse_user_specs(name=\"specs\", yaml_list=input)\n\n assert result.specs_as_yaml_list == DEFAULT_EXPANSION\n assert result.specs_as_constraints == DEFAULT_CONSTRAINTS\n assert result.specs == DEFAULT_SPECS\n\n @pytest.mark.regression(\"16841\")\n def test_spec_list_matrix_exclude(self):\n parser = SpecListParser()\n result = parser.parse_user_specs(\n name=\"specs\",\n yaml_list=[\n {\n \"matrix\": [[\"multivalue-variant\"], [\"foo=bar\", \"foo=baz\"]],\n \"exclude\": [\"foo=bar\"],\n }\n ],\n )\n assert len(result.specs) == 1\n\n def test_spec_list_exclude_with_abstract_hashes(self, install_mockery):\n # Put mpich in the database so it can be referred to by hash.\n mpich_1 = spack.concretize.concretize_one(\"mpich+debug\")\n mpich_2 = spack.concretize.concretize_one(\"mpich~debug\")\n PackageInstaller([mpich_1.package, mpich_2.package], explicit=True, fake=True).install()\n\n # Create matrix and exclude +debug, which excludes the first mpich after its abstract hash\n # is resolved.\n parser = SpecListParser()\n result = parser.parse_user_specs(\n name=\"specs\",\n yaml_list=[\n {\n \"matrix\": [\n [\"mpileaks\"],\n [\"^callpath\"],\n [f\"^mpich/{mpich_1.dag_hash(5)}\", f\"^mpich/{mpich_2.dag_hash(5)}\"],\n ],\n \"exclude\": [\"^mpich+debug\"],\n }\n ],\n )\n\n # Ensure that only mpich~debug is selected, and that the assembled spec remains abstract.\n assert len(result.specs) == 1\n assert result.specs[0] == Spec(f\"mpileaks ^callpath ^mpich/{mpich_2.dag_hash(5)}\")\n\n @pytest.mark.regression(\"51703\")\n def test_exclusion_with_conditional_dependencies(self):\n \"\"\"Tests that we can exclude some spec using conditional dependencies in the exclusion.\"\"\"\n parser = SpecListParser()\n result = parser.parse_user_specs(\n name=\"specs\",\n yaml_list=[\n {\n \"matrix\": [[\"libunwind\"], [\"%[when=%c]c=gcc\", \"%[when=%c]c=llvm\"]],\n \"exclude\": [\"libunwind %[when=%c]c=gcc\"],\n }\n ],\n )\n assert len(result.specs) == 1\n" }, { "path": "lib/spack/spack/test/spec_semantics.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport pathlib\n\nimport pytest\n\nimport spack.concretize\nimport spack.deptypes as dt\nimport spack.directives\nimport spack.llnl.util.lang\nimport spack.package_base\nimport spack.paths\nimport spack.repo\nimport spack.solver.asp\nimport spack.spec\nimport spack.spec_parser\nimport spack.store\nimport spack.variant\nimport spack.version as vn\nfrom spack.enums import PropagationPolicy\nfrom spack.error import SpecError, UnsatisfiableSpecError\nfrom spack.llnl.util.tty.color import colorize\nfrom spack.spec import ArchSpec, DependencySpec, Spec, SpecFormatSigilError, SpecFormatStringError\nfrom spack.variant import (\n InvalidVariantValueError,\n MultipleValuesInExclusiveVariantError,\n UnknownVariantError,\n)\n\n\n@pytest.fixture()\ndef setup_complex_splice(monkeypatch):\n r\"\"\"Fixture to set up splicing for two complex specs.\n\n a_red is a spec in which every node has the variant color=red\n c_blue is a spec in which every node has the variant color=blue\n\n a_red structure:\n a -\n / \\ \\\n b c \\\n /|\\ / \\ |\n e | d g@2\n \\|/\n g@1\n\n c_blue structure:\n c\n /|\\\n d f \\\n / |\\ \\\n g@2 e \\ \\\n \\| /\n g@3\n\n This is not intended for use in tests that use virtuals, so ``_splice_match`` is monkeypatched\n to avoid needing package files for each spec.\n \"\"\"\n\n def splice_match(self, other, self_root, other_root):\n return self.name == other.name\n\n def virtuals_provided(self, root):\n return []\n\n monkeypatch.setattr(Spec, \"_splice_match\", splice_match)\n monkeypatch.setattr(Spec, \"_virtuals_provided\", virtuals_provided)\n\n g1_red = Spec(\"pkg-g color=red\")\n g1_red.versions = vn.VersionList([vn.Version(\"1\")])\n g2_red = Spec(\"pkg-g color=red\")\n g2_red.versions = vn.VersionList([vn.Version(\"2\")])\n g2_blue = Spec(\"pkg-g color=blue\")\n g2_blue.versions = vn.VersionList([vn.Version(\"2\")])\n g3_blue = Spec(\"pkg-g color=blue\")\n g3_blue.versions = vn.VersionList([vn.Version(\"3\")])\n\n depflag = dt.LINK | dt.BUILD\n e_red = Spec(\"pkg-e color=red\")\n e_red._add_dependency(g1_red, depflag=depflag, virtuals=())\n e_blue = Spec(\"pkg-e color=blue\")\n e_blue._add_dependency(g3_blue, depflag=depflag, virtuals=())\n\n d_red = Spec(\"pkg-d color=red\")\n d_red._add_dependency(g1_red, depflag=depflag, virtuals=())\n d_blue = Spec(\"pkg-d color=blue\")\n d_blue._add_dependency(g2_blue, depflag=depflag, virtuals=())\n\n b_red = Spec(\"pkg-b color=red\")\n b_red._add_dependency(e_red, depflag=depflag, virtuals=())\n b_red._add_dependency(d_red, depflag=depflag, virtuals=())\n b_red._add_dependency(g1_red, depflag=depflag, virtuals=())\n\n f_blue = Spec(\"pkg-f color=blue\")\n f_blue._add_dependency(e_blue, depflag=depflag, virtuals=())\n f_blue._add_dependency(g3_blue, depflag=depflag, virtuals=())\n\n c_red = Spec(\"pkg-c color=red\")\n c_red._add_dependency(d_red, depflag=depflag, virtuals=())\n c_red._add_dependency(g2_red, depflag=depflag, virtuals=())\n c_blue = Spec(\"pkg-c color=blue\")\n c_blue._add_dependency(d_blue, depflag=depflag, virtuals=())\n c_blue._add_dependency(f_blue, depflag=depflag, virtuals=())\n c_blue._add_dependency(g3_blue, depflag=depflag, virtuals=())\n\n a_red = Spec(\"pkg-a color=red\")\n a_red._add_dependency(b_red, depflag=depflag, virtuals=())\n a_red._add_dependency(c_red, depflag=depflag, virtuals=())\n a_red._add_dependency(g2_red, depflag=depflag, virtuals=())\n\n for spec in [e_red, e_blue, d_red, d_blue, b_red, f_blue, c_red, c_blue, a_red]:\n spec.versions = vn.VersionList([vn.Version(\"1\")])\n\n a_red._mark_concrete()\n c_blue._mark_concrete()\n\n return a_red, c_blue\n\n\n@pytest.mark.usefixtures(\"config\", \"mock_packages\")\nclass TestSpecSemantics:\n \"\"\"Test satisfies(), intersects(), constrain() and other semantic operations on specs.\"\"\"\n\n @pytest.mark.parametrize(\n \"lhs,rhs,expected\",\n [\n (\"libelf@0.8.13\", \"@0:1\", \"libelf@0.8.13\"),\n (\"libdwarf^libelf@0.8.13\", \"^libelf@0:1\", \"libdwarf^libelf@0.8.13\"),\n (\"libelf\", Spec(), \"libelf\"),\n (\"libdwarf\", Spec(), \"libdwarf\"),\n (\"%intel\", Spec(), \"%intel\"),\n (\"^mpi\", Spec(), \"^mpi\"),\n (\"+debug\", Spec(), \"+debug\"),\n (\"@3:\", Spec(), \"@3:\"),\n # Versions\n (\"libelf@0:2.5\", \"libelf@2.1:3\", \"libelf@2.1:2.5\"),\n (\"libelf@0:2.5%gcc@2:4.6\", \"libelf@2.1:3%gcc@4.5:4.7\", \"libelf@2.1:2.5%gcc@4.5:4.6\"),\n # Namespaces\n (\"builtin.mpich\", \"mpich\", \"builtin.mpich\"),\n (\"builtin.mock.mpich\", \"mpich\", \"builtin.mock.mpich\"),\n (\"builtin.mpich\", \"builtin.mpich\", \"builtin.mpich\"),\n (\"mpileaks ^builtin.mock.mpich\", \"^mpich\", \"mpileaks ^builtin.mock.mpich\"),\n # Virtual dependencies are fully resolved during concretization, so we can constrain\n # abstract specs but that would result in a new node\n (\"mpileaks ^builtin.mock.mpich\", \"^mpi\", \"mpileaks ^mpi ^builtin.mock.mpich\"),\n (\n \"mpileaks ^builtin.mock.mpich\",\n \"^builtin.mock.mpich\",\n \"mpileaks ^builtin.mock.mpich\",\n ),\n # Compilers\n (\"foo%gcc\", \"%gcc\", \"foo%gcc\"),\n (\"foo%intel\", \"%intel\", \"foo%intel\"),\n (\"foo%gcc\", \"%gcc@4.7.2\", \"foo%gcc@4.7.2\"),\n (\"foo%intel\", \"%intel@4.7.2\", \"foo%intel@4.7.2\"),\n (\"foo%gcc@4.5\", \"%gcc@4.4:4.6\", \"foo%gcc@4.5\"),\n (\"foo@2.0%gcc@4.5\", \"@1:3%gcc@4.4:4.6\", \"foo@2.0%gcc@4.5\"),\n (\"foo %gcc@4.7.3\", \"%gcc@4.7\", \"foo %gcc@4.7.3\"),\n (\"libelf %gcc@4.4.7\", \"libelf %gcc@4.4.7\", \"libelf %gcc@4.4.7\"),\n (\"libelf\", \"libelf %gcc@4.4.7\", \"libelf %gcc@4.4.7\"),\n # Architecture\n (\"foo platform=test\", \"platform=test\", \"foo platform=test\"),\n (\"foo platform=linux\", \"platform=linux\", \"foo platform=linux\"),\n (\n \"foo platform=test\",\n \"platform=test target=frontend\",\n \"foo platform=test target=frontend\",\n ),\n (\n \"foo platform=test\",\n \"platform=test os=frontend target=frontend\",\n \"foo platform=test os=frontend target=frontend\",\n ),\n (\n \"foo platform=test os=frontend target=frontend\",\n \"platform=test\",\n \"foo platform=test os=frontend target=frontend\",\n ),\n (\"foo arch=test-None-None\", \"platform=test\", \"foo platform=test\"),\n (\n \"foo arch=test-None-frontend\",\n \"platform=test target=frontend\",\n \"foo platform=test target=frontend\",\n ),\n (\n \"foo arch=test-frontend-frontend\",\n \"platform=test os=frontend target=frontend\",\n \"foo platform=test os=frontend target=frontend\",\n ),\n (\n \"foo arch=test-frontend-frontend\",\n \"platform=test\",\n \"foo platform=test os=frontend target=frontend\",\n ),\n (\n \"foo platform=test target=backend os=backend\",\n \"platform=test target=backend os=backend\",\n \"foo platform=test target=backend os=backend\",\n ),\n (\n \"libelf target=default_target os=default_os\",\n \"libelf target=default_target os=default_os\",\n \"libelf target=default_target os=default_os\",\n ),\n # Dependencies\n (\"mpileaks ^mpich\", \"^mpich\", \"mpileaks ^mpich\"),\n (\"mpileaks ^mpich@2.0\", \"^mpich@1:3\", \"mpileaks ^mpich@2.0\"),\n (\n \"mpileaks ^mpich@2.0 ^callpath@1.5\",\n \"^mpich@1:3 ^callpath@1.4:1.6\",\n \"mpileaks^mpich@2.0^callpath@1.5\",\n ),\n (\"mpileaks ^mpi\", \"^mpi\", \"mpileaks ^mpi\"),\n (\"mpileaks ^mpi\", \"^mpich\", \"mpileaks ^mpi ^mpich\"),\n (\"mpileaks^mpi@1.5\", \"^mpi@1.2:1.6\", \"mpileaks^mpi@1.5\"),\n (\"mpileaks^mpi@2:\", \"^mpich\", \"mpileaks^mpi@2: ^mpich\"),\n (\"mpileaks^mpi@2:\", \"^mpich@3.0.4\", \"mpileaks^mpi@2: ^mpich@3.0.4\"),\n # Variants\n (\"mpich+foo\", \"mpich+foo\", \"mpich+foo\"),\n (\"mpich++foo\", \"mpich++foo\", \"mpich++foo\"),\n (\"mpich~foo\", \"mpich~foo\", \"mpich~foo\"),\n (\"mpich~~foo\", \"mpich~~foo\", \"mpich~~foo\"),\n (\"mpich foo=1\", \"mpich foo=1\", \"mpich foo=1\"),\n (\"mpich foo==1\", \"mpich foo==1\", \"mpich foo==1\"),\n (\"mpich+foo\", \"mpich foo=True\", \"mpich+foo\"),\n (\"mpich++foo\", \"mpich foo=True\", \"mpich+foo\"),\n (\"mpich foo=true\", \"mpich+foo\", \"mpich+foo\"),\n (\"mpich foo==true\", \"mpich++foo\", \"mpich++foo\"),\n (\"mpich~foo\", \"mpich foo=FALSE\", \"mpich~foo\"),\n (\"mpich~~foo\", \"mpich foo=FALSE\", \"mpich~foo\"),\n (\"mpich foo=False\", \"mpich~foo\", \"mpich~foo\"),\n (\"mpich foo==False\", \"mpich~foo\", \"mpich~foo\"),\n (\"mpich foo=*\", \"mpich~foo\", \"mpich~foo\"),\n (\"mpich+foo\", \"mpich foo=*\", \"mpich+foo\"),\n (\n 'multivalue-variant foo=\"bar,baz\"',\n \"multivalue-variant foo=bar,baz\",\n \"multivalue-variant foo=bar,baz\",\n ),\n (\n 'multivalue-variant foo=\"bar,baz\"',\n \"multivalue-variant foo=*\",\n \"multivalue-variant foo=bar,baz\",\n ),\n (\n 'multivalue-variant foo=\"bar,baz\"',\n \"multivalue-variant foo=bar\",\n \"multivalue-variant foo=bar,baz\",\n ),\n (\n 'multivalue-variant foo=\"bar,baz\"',\n \"multivalue-variant foo=baz\",\n \"multivalue-variant foo=bar,baz\",\n ),\n (\n 'multivalue-variant foo=\"bar,baz,barbaz\"',\n \"multivalue-variant foo=bar,baz\",\n \"multivalue-variant foo=bar,baz,barbaz\",\n ),\n (\n 'multivalue-variant foo=\"bar,baz\"',\n 'foo=\"baz,bar\"', # Order of values doesn't matter\n \"multivalue-variant foo=bar,baz\",\n ),\n (\"mpich+foo\", \"mpich\", \"mpich+foo\"),\n (\"mpich~foo\", \"mpich\", \"mpich~foo\"),\n (\"mpich foo=1\", \"mpich\", \"mpich foo=1\"),\n (\"mpich\", \"mpich++foo\", \"mpich++foo\"),\n (\"libelf+debug\", \"libelf+foo\", \"libelf+debug+foo\"),\n (\"libelf+debug\", \"libelf+debug+foo\", \"libelf+debug+foo\"),\n (\"libelf debug=2\", \"libelf foo=1\", \"libelf debug=2 foo=1\"),\n (\"libelf debug=2\", \"libelf debug=2 foo=1\", \"libelf debug=2 foo=1\"),\n (\"libelf+debug\", \"libelf~foo\", \"libelf+debug~foo\"),\n (\"libelf+debug\", \"libelf+debug~foo\", \"libelf+debug~foo\"),\n (\"libelf++debug\", \"libelf+debug+foo\", \"libelf+debug+foo\"),\n (\"libelf debug==2\", \"libelf foo=1\", \"libelf debug==2 foo=1\"),\n (\"libelf debug==2\", \"libelf debug=2 foo=1\", \"libelf debug=2 foo=1\"),\n (\"libelf++debug\", \"libelf++debug~foo\", \"libelf++debug~foo\"),\n (\"libelf foo=bar,baz\", \"libelf foo=*\", \"libelf foo=bar,baz\"),\n (\"libelf foo=*\", \"libelf foo=bar,baz\", \"libelf foo=bar,baz\"),\n (\n 'multivalue-variant foo=\"bar\"',\n 'multivalue-variant foo=\"baz\"',\n 'multivalue-variant foo=\"bar,baz\"',\n ),\n (\n 'multivalue-variant foo=\"bar,barbaz\"',\n 'multivalue-variant foo=\"baz\"',\n 'multivalue-variant foo=\"bar,baz,barbaz\"',\n ),\n # Namespace (special case, but like variants\n (\"builtin.libelf\", \"namespace=builtin\", \"builtin.libelf\"),\n (\"libelf\", \"namespace=builtin\", \"builtin.libelf\"),\n # Flags\n (\"mpich \", 'mpich cppflags=\"-O3\"', 'mpich cppflags=\"-O3\"'),\n (\n 'mpich cppflags=\"-O3 -Wall\"',\n 'mpich cppflags=\"-O3 -Wall\"',\n 'mpich cppflags=\"-O3 -Wall\"',\n ),\n ('mpich cppflags==\"-O3\"', 'mpich cppflags==\"-O3\"', 'mpich cppflags==\"-O3\"'),\n (\n 'libelf cflags=\"-O3\"',\n 'libelf cppflags=\"-Wall\"',\n 'libelf cflags=\"-O3\" cppflags=\"-Wall\"',\n ),\n (\n 'libelf cflags=\"-O3\"',\n 'libelf cppflags==\"-Wall\"',\n 'libelf cflags=\"-O3\" cppflags==\"-Wall\"',\n ),\n (\n 'libelf cflags==\"-O3\"',\n 'libelf cppflags==\"-Wall\"',\n 'libelf cflags==\"-O3\" cppflags==\"-Wall\"',\n ),\n (\n 'libelf cflags=\"-O3\"',\n 'libelf cflags=\"-O3\" cppflags=\"-Wall\"',\n 'libelf cflags=\"-O3\" cppflags=\"-Wall\"',\n ),\n (\n \"libelf patches=ba5e334fe247335f3a116decfb5284100791dc302b5571ff5e664d8f9a6806c2\",\n \"libelf patches=ba5e3\", # constrain by a patch sha256 prefix\n # TODO: the result below is not ideal. Prefix satisfies() works for patches, but\n # constrain() isn't similarly special-cased to do the same thing\n (\n \"libelf patches=ba5e3,\"\n \"ba5e334fe247335f3a116decfb5284100791dc302b5571ff5e664d8f9a6806c2\"\n ),\n ),\n # deptypes on direct deps\n (\n \"mpileaks %[deptypes=build] mpich\",\n \"mpileaks %[deptypes=link] mpich\",\n \"mpileaks %[deptypes=build,link] mpich\",\n ),\n # conditional edges\n (\n \"libelf\",\n \"%[when='%c' virtuals=c]gcc ^[when='+mpi' virtuals=mpi]mpich\",\n \"libelf %[when='%c' virtuals=c]gcc ^[when='+mpi' virtuals=mpi]mpich\",\n ),\n (\n \"libelf %[when='%c' virtuals=c]gcc\",\n \"%[when='%c' virtuals=c]gcc@10.3.1\",\n \"libelf%[when='%c' virtuals=c]gcc@10.3.1\",\n ),\n (\n \"libelf %[when='%c' virtuals=c]gcc\",\n \"%[when='%c' virtuals=c]gcc@10.3.1 ^[when='+mpi'] mpich\",\n \"libelf%[when='%c' virtuals=c]gcc@10.3.1 ^[when='+mpi']mpich\",\n ),\n (\n \"libelf %[when='%c' virtuals=c]gcc\",\n \"%[when='%cxx' virtuals=cxx]gcc@10.3.1\",\n \"libelf%[when='%c' virtuals=c]gcc %[when='%cxx' virtuals=cxx]gcc@10.3.1\",\n ),\n (\n \"libelf %[when='+c' virtuals=c]gcc\",\n \"%[when='%c' virtuals=c]gcc@10.3.1\",\n \"libelf %[when='+c' virtuals=c]gcc %[when='%c' virtuals=c]gcc@10.3.1\",\n ),\n ],\n )\n def test_abstract_specs_can_constrain_each_other(self, lhs, rhs, expected):\n \"\"\"Test that lhs and rhs intersect with each other, and that they can be constrained\n with each other. Also check that the constrained result match the expected spec.\n \"\"\"\n lhs, rhs, expected = Spec(lhs), Spec(rhs), Spec(expected)\n\n assert lhs.intersects(rhs)\n assert rhs.intersects(lhs)\n\n c1, c2 = lhs.copy(), rhs.copy()\n c1.constrain(rhs)\n c2.constrain(lhs)\n assert c1 == c2\n assert c1 == expected\n\n @pytest.mark.parametrize(\n \"lhs,rhs,expected_lhs,expected_rhs,propagated_lhs,propagated_rhs\",\n [\n (\n 'mpich cppflags=\"-O3\"',\n 'mpich cppflags=\"-O2\"',\n 'mpich cppflags=\"-O3 -O2\"',\n 'mpich cppflags=\"-O2 -O3\"',\n [],\n [],\n ),\n (\n 'mpich cflags=\"-O3 -g\"',\n 'mpich cflags==\"-O3\"',\n 'mpich cflags=\"-O3 -g\"',\n 'mpich cflags=\"-O3 -g\"',\n [],\n [],\n ),\n (\n 'mpich cflags==\"-O3 -g\"',\n 'mpich cflags==\"-O3\"',\n 'mpich cflags==\"-O3 -g\"',\n 'mpich cflags==\"-O3 -g\"',\n [(\"cflags\", \"-O3\"), (\"cflags\", \"-g\")],\n [(\"cflags\", \"-O3\"), (\"cflags\", \"-g\")],\n ),\n ],\n )\n def test_constrain_compiler_flags(\n self, lhs, rhs, expected_lhs, expected_rhs, propagated_lhs, propagated_rhs\n ):\n \"\"\"Constraining is asymmetric for compiler flags.\"\"\"\n lhs, rhs, expected_lhs, expected_rhs = (\n Spec(lhs),\n Spec(rhs),\n Spec(expected_lhs),\n Spec(expected_rhs),\n )\n\n assert lhs.intersects(rhs)\n assert rhs.intersects(lhs)\n\n c1, c2 = lhs.copy(), rhs.copy()\n c1.constrain(rhs)\n c2.constrain(lhs)\n\n assert c1 == expected_lhs\n assert c2 == expected_rhs\n for x in [c1, c2]:\n assert x.satisfies(lhs)\n assert x.satisfies(rhs)\n\n def _propagated_flags(_spec):\n result = set()\n for flagtype in _spec.compiler_flags:\n for flag in _spec.compiler_flags[flagtype]:\n if flag.propagate:\n result.add((flagtype, flag))\n return result\n\n assert set(propagated_lhs) <= _propagated_flags(c1)\n assert set(propagated_rhs) <= _propagated_flags(c2)\n\n def test_constrain_specs_by_hash(self, default_mock_concretization, database):\n \"\"\"Test that Specs specified only by their hashes can constrain each other.\"\"\"\n mpich_dag_hash = \"/\" + database.query_one(\"mpich\").dag_hash()\n spec = Spec(mpich_dag_hash[:7])\n assert spec.constrain(Spec(mpich_dag_hash)) is False\n assert spec.abstract_hash == mpich_dag_hash[1:]\n\n def test_mismatched_constrain_spec_by_hash(self, default_mock_concretization, database):\n \"\"\"Test that Specs specified only by their incompatible hashes fail appropriately.\"\"\"\n lhs = \"/\" + database.query_one(\"callpath ^mpich\").dag_hash()\n rhs = \"/\" + database.query_one(\"callpath ^mpich2\").dag_hash()\n with pytest.raises(spack.spec.InvalidHashError):\n Spec(lhs).constrain(Spec(rhs))\n with pytest.raises(spack.spec.InvalidHashError):\n Spec(lhs[:7]).constrain(Spec(rhs))\n\n @pytest.mark.parametrize(\n \"lhs,rhs\", [(\"libelf\", Spec()), (\"libelf\", \"@0:1\"), (\"libelf\", \"@0:1 %gcc\")]\n )\n def test_concrete_specs_which_satisfies_abstract(self, lhs, rhs, default_mock_concretization):\n \"\"\"Test that constraining an abstract spec by a compatible concrete one makes the\n abstract spec concrete, and equal to the one it was constrained with.\n \"\"\"\n lhs, rhs = default_mock_concretization(lhs), Spec(rhs)\n\n assert lhs.intersects(rhs)\n assert rhs.intersects(lhs)\n assert lhs.satisfies(rhs)\n assert not rhs.satisfies(lhs)\n\n assert lhs.constrain(rhs) is False\n assert rhs.constrain(lhs) is True\n\n assert rhs.concrete\n assert lhs.satisfies(rhs)\n assert rhs.satisfies(lhs)\n assert lhs == rhs\n\n @pytest.mark.parametrize(\n \"lhs,rhs\",\n [\n (\"foo platform=linux\", \"platform=test os=redhat6 target=x86\"),\n (\"foo os=redhat6\", \"platform=test os=debian6 target=x86_64\"),\n (\"foo target=x86_64\", \"platform=test os=redhat6 target=x86\"),\n (\"foo%gcc@4.3\", \"%gcc@4.4:4.6\"),\n (\"foo@4.0%gcc\", \"@1:3%gcc\"),\n (\"foo@4.0%gcc@4.5\", \"@1:3%gcc@4.4:4.6\"),\n (\"builtin.mock.mpich\", \"builtin.mpich\"),\n (\"mpileaks ^builtin.mock.mpich\", \"^builtin.mpich\"),\n (\"mpileaks^mpich@1.2\", \"^mpich@2.0\"),\n (\"mpileaks^mpich@4.0^callpath@1.5\", \"^mpich@1:3^callpath@1.4:1.6\"),\n (\"mpileaks^mpich@2.0^callpath@1.7\", \"^mpich@1:3^callpath@1.4:1.6\"),\n (\"mpileaks^mpich@4.0^callpath@1.7\", \"^mpich@1:3^callpath@1.4:1.6\"),\n (\"mpileaks^mpi@3\", \"^mpi@1.2:1.6\"),\n (\"mpileaks^mpi@3:\", \"^mpich2@1.4\"),\n (\"mpileaks^mpi@3:\", \"^mpich2\"),\n (\"mpileaks^mpi@3:\", \"^mpich@1.0\"),\n (\"mpich~foo\", \"mpich+foo\"),\n (\"mpich+foo\", \"mpich~foo\"),\n (\"mpich foo=True\", \"mpich foo=False\"),\n (\"mpich~~foo\", \"mpich++foo\"),\n (\"mpich++foo\", \"mpich~~foo\"),\n (\"mpich foo==True\", \"mpich foo==False\"),\n (\"libelf@0:2.0\", \"libelf@2.1:3\"),\n (\"libelf@0:2.5%gcc@4.8:4.9\", \"libelf@2.1:3%gcc@4.5:4.7\"),\n (\"libelf+debug\", \"libelf~debug\"),\n (\"libelf+debug~foo\", \"libelf+debug+foo\"),\n (\"libelf debug=True\", \"libelf debug=False\"),\n (\"namespace=builtin.mock\", \"namespace=builtin\"),\n ],\n )\n def test_constraining_abstract_specs_with_empty_intersection(self, lhs, rhs):\n \"\"\"Check that two abstract specs with an empty intersection cannot be constrained\n with each other.\n \"\"\"\n lhs, rhs = Spec(lhs), Spec(rhs)\n\n assert not lhs.intersects(rhs)\n assert not rhs.intersects(lhs)\n\n with pytest.raises(UnsatisfiableSpecError):\n lhs.constrain(rhs)\n\n with pytest.raises(UnsatisfiableSpecError):\n rhs.constrain(lhs)\n\n @pytest.mark.parametrize(\n \"lhs,rhs\",\n [\n (\"mpich\", \"mpich +foo\"),\n (\"mpich\", \"mpich~foo\"),\n (\"mpich\", \"mpich foo=1\"),\n (\"multivalue-variant foo=bar\", \"multivalue-variant +foo\"),\n (\"multivalue-variant foo=bar\", \"multivalue-variant ~foo\"),\n (\"multivalue-variant fee=bar\", \"multivalue-variant fee=baz\"),\n ],\n )\n def test_concrete_specs_which_do_not_satisfy_abstract(\n self, lhs, rhs, default_mock_concretization\n ):\n lhs, rhs = default_mock_concretization(lhs), Spec(rhs)\n\n assert lhs.intersects(rhs) is False\n assert rhs.intersects(lhs) is False\n assert not lhs.satisfies(rhs)\n assert not rhs.satisfies(lhs)\n\n with pytest.raises(UnsatisfiableSpecError):\n assert lhs.constrain(rhs)\n\n with pytest.raises(UnsatisfiableSpecError):\n assert rhs.constrain(lhs)\n\n @pytest.mark.parametrize(\n \"lhs,rhs\", [(\"mpich\", \"mpich++foo\"), (\"mpich\", \"mpich~~foo\"), (\"mpich\", \"mpich foo==1\")]\n )\n def test_concrete_specs_which_satisfy_abstract(self, lhs, rhs, default_mock_concretization):\n lhs, rhs = default_mock_concretization(lhs), Spec(rhs)\n\n assert lhs.intersects(rhs)\n assert rhs.intersects(lhs)\n assert lhs.satisfies(rhs)\n\n s1 = lhs.copy()\n s1.constrain(rhs)\n assert s1 == lhs and s1.satisfies(lhs)\n\n s2 = rhs.copy()\n s2.constrain(lhs)\n assert s2 == lhs and s2.satisfies(lhs)\n\n @pytest.mark.parametrize(\n \"lhs,rhs,expected,constrained\",\n [\n # hdf5++mpi satisfies hdf5, and vice versa, because of the non-contradiction semantic\n (\"hdf5++mpi\", \"hdf5\", True, \"hdf5++mpi\"),\n (\"hdf5\", \"hdf5++mpi\", True, \"hdf5++mpi\"),\n # Same holds true for arbitrary propagated variants\n (\"hdf5++mpi\", \"hdf5++shared\", True, \"hdf5++mpi++shared\"),\n # Here hdf5+mpi satisfies hdf5++mpi but not vice versa\n (\"hdf5++mpi\", \"hdf5+mpi\", False, \"hdf5+mpi\"),\n (\"hdf5+mpi\", \"hdf5++mpi\", True, \"hdf5+mpi\"),\n # Non contradiction is violated\n (\"hdf5 ^foo~mpi\", \"hdf5++mpi\", False, \"hdf5++mpi ^foo~mpi\"),\n (\"hdf5++mpi\", \"hdf5 ^foo~mpi\", False, \"hdf5++mpi ^foo~mpi\"),\n ],\n )\n def test_abstract_specs_with_propagation(self, lhs, rhs, expected, constrained):\n \"\"\"Tests (and documents) behavior of variant propagation on abstract specs.\n\n Propagated variants do not comply with subset semantic, making it difficult to give\n precise definitions. Here we document the behavior that has been decided for the\n practical cases we face.\n \"\"\"\n lhs, rhs, constrained = Spec(lhs), Spec(rhs), Spec(constrained)\n assert lhs.satisfies(rhs) is expected\n\n c = lhs.copy()\n c.constrain(rhs)\n assert c == constrained\n\n c = rhs.copy()\n c.constrain(lhs)\n assert c == constrained\n\n def test_basic_satisfies_conditional_dep(self, default_mock_concretization):\n \"\"\"Tests basic semantic of satisfies with conditional dependencies, on a concrete spec\"\"\"\n concrete = default_mock_concretization(\"mpileaks ^mpich\")\n\n # This branch exists, so the condition is met, and is satisfied\n assert concrete.satisfies(\"^[virtuals=mpi] mpich\")\n assert concrete.satisfies(\"^[when='^notapackage' virtuals=mpi] mpich\")\n assert concrete.satisfies(\"^[when='^mpi' virtuals=mpi] mpich\")\n\n # This branch does not exist, but the condition is not met\n assert not concrete.satisfies(\"^zmpi\")\n assert concrete.satisfies(\"^[when='^notapackage'] zmpi\")\n assert not concrete.satisfies(\"^[when='^mpi'] zmpi\")\n\n def test_concrete_satisfies_does_not_consult_repo(\n self, default_mock_concretization, monkeypatch\n ):\n \"\"\"Tests that `satisfies()` on a concrete lhs doesn't need the provider index, when the rhs\n contains a virtual name.\n \"\"\"\n concrete = default_mock_concretization(\"mpileaks ^mpich\")\n\n # Reset the index, will raise if the `_provider_index` is ever removed as an attribute\n monkeypatch.setattr(spack.repo.PATH, \"_provider_index\", None)\n\n # Basic match and mismatch cases.\n assert concrete.satisfies(\"mpileaks\")\n assert not concrete.satisfies(\"zlib\")\n\n # Virtuals on a direct edge\n assert concrete.satisfies(\"%mpi\")\n assert concrete.satisfies(\"%mpi@3\")\n assert not concrete.satisfies(\"%mpi@5\")\n assert concrete.satisfies(\"%mpi=mpich\")\n assert not concrete.satisfies(\"%lapack\")\n\n # Virtuals on a transitive edge\n assert concrete.satisfies(\"^mpi\")\n assert concrete.satisfies(\"^mpi=mpich\")\n assert not concrete.satisfies(\"^lapack\")\n\n # Concrete spec asking about one of its concrete deps.\n mpich = concrete[\"mpich\"]\n assert mpich.satisfies(\"mpich\")\n assert mpich.satisfies(\"mpi\")\n\n # We should not create again the index\n assert spack.repo.PATH._provider_index is None\n\n def test_concrete_contains_does_not_consult_repo(\n self, default_mock_concretization, monkeypatch\n ):\n \"\"\"Tests that `foo in spec` on a concrete spec doesn't need the provider index, when the\n item contains a virtual name.\n \"\"\"\n concrete = default_mock_concretization(\"mpileaks ^mpich\")\n\n # Reset the index, will raise if the `_provider_index` is ever removed as an attribute\n monkeypatch.setattr(spack.repo.PATH, \"_provider_index\", None)\n\n assert \"mpi\" in concrete\n assert \"c\" in concrete\n\n # We should not create again the index\n assert spack.repo.PATH._provider_index is None\n\n def test_abstract_satisfies_with_lhs_provider_rhs_virtual(self):\n \"\"\"If the left-hand side mentions a provider among dependencies and the right-hand side\n mentions a virtual among its deps, we only have satisfaction if the edge attribute\n specifies this virtual is provided.\"\"\"\n assert not Spec(\"mpileaks ^mpich\").satisfies(\"mpileaks ^mpi\")\n assert not Spec(\"mpileaks %mpich\").satisfies(\"mpileaks %mpi\")\n assert Spec(\"mpileaks ^[virtuals=mpi] mpich\").satisfies(\"mpileaks ^mpi\")\n assert Spec(\"mpileaks %[virtuals=mpi] mpich\").satisfies(\"mpileaks ^mpi\")\n assert Spec(\"mpileaks %[virtuals=mpi] mpich\").satisfies(\"mpileaks %mpi\")\n\n def test_concrete_checks_on_virtual_names_dont_need_repo(\n self, default_mock_concretization, monkeypatch\n ):\n \"\"\"Tests that ``%mpi`` or similar on a concrete spec doesn't need the repo\"\"\"\n concrete = default_mock_concretization(\"mpileaks ^mpich\")\n\n # We don't need the repo\n monkeypatch.setattr(spack.repo, \"PATH\", None)\n\n assert concrete.satisfies(\"%mpi\")\n assert concrete.satisfies(\"%c\")\n assert concrete.satisfies(\"%c=gcc\")\n assert concrete.satisfies(\"%mpi=mpich\")\n\n assert not concrete.satisfies(\"%c,mpi=mpich\")\n\n def test_satisfies_single_valued_variant(self):\n \"\"\"Tests that the case reported in\n https://github.com/spack/spack/pull/2386#issuecomment-282147639\n is handled correctly.\n \"\"\"\n a = spack.concretize.concretize_one(\"pkg-a foobar=bar\")\n\n assert a.satisfies(\"foobar=bar\")\n assert a.satisfies(\"foobar=*\")\n\n # Assert that an autospec generated from a literal\n # gives the right result for a single valued variant\n assert \"foobar=bar\" in a\n assert \"foobar==bar\" in a\n assert \"foobar=baz\" not in a\n assert \"foobar=fee\" not in a\n\n # ... and for a multi valued variant\n assert \"foo=bar\" in a\n\n # Check that conditional dependencies are treated correctly\n assert \"^pkg-b\" in a\n\n def test_unsatisfied_single_valued_variant(self):\n a = spack.concretize.concretize_one(\"pkg-a foobar=baz\")\n assert \"^pkg-b\" not in a\n\n mv = spack.concretize.concretize_one(\"multivalue-variant\")\n assert \"pkg-a@1.0\" not in mv\n\n def test_indirect_unsatisfied_single_valued_variant(self):\n spec = spack.concretize.concretize_one(\"singlevalue-variant-dependent\")\n assert \"pkg-a@1.0\" not in spec\n\n def test_satisfied_namespace(self):\n spec = spack.concretize.concretize_one(\"zlib\")\n assert spec.satisfies(\"namespace=builtin_mock\")\n assert not spec.satisfies(\"namespace=builtin\")\n\n @pytest.mark.parametrize(\n \"spec_string\",\n [\n \"tcl namespace==foobar\",\n \"tcl arch==foobar\",\n \"tcl os==foobar\",\n \"tcl patches==foobar\",\n \"tcl dev_path==foobar\",\n ],\n )\n def test_propagate_reserved_variant_names(self, spec_string):\n with pytest.raises(spack.spec_parser.SpecParsingError, match=\"Propagation\"):\n Spec(spec_string)\n\n def test_multivalued_variant_1(self, default_mock_concretization):\n # Semantics for a multi-valued variant is different\n # Depending on whether the spec is concrete or not\n\n a = default_mock_concretization(\"multivalue-variant foo=bar\")\n b = Spec(\"multivalue-variant foo=bar,baz\")\n assert not a.satisfies(b)\n\n def test_multivalued_variant_2(self):\n a = Spec(\"multivalue-variant foo=bar\")\n b = Spec(\"multivalue-variant foo=bar,baz\")\n # The specs are abstract and they **could** be constrained\n assert b.satisfies(a) and not a.satisfies(b)\n # An abstract spec can instead be constrained\n assert a.constrain(b)\n\n def test_multivalued_variant_3(self, default_mock_concretization):\n a = default_mock_concretization(\"multivalue-variant foo=bar,baz\")\n b = Spec(\"multivalue-variant foo=bar,baz,quux\")\n assert not a.satisfies(b)\n\n def test_multivalued_variant_4(self):\n a = Spec(\"multivalue-variant foo=bar,baz\")\n b = Spec(\"multivalue-variant foo=bar,baz,quux\")\n # The specs are abstract and they **could** be constrained\n assert a.intersects(b)\n # An abstract spec can instead be constrained\n assert a.constrain(b)\n # ...but will fail during concretization if there are\n # values in the variant that are not allowed\n with pytest.raises(InvalidVariantValueError):\n spack.concretize.concretize_one(a)\n\n def test_multivalued_variant_5(self):\n # This time we'll try to set a single-valued variant\n a = Spec(\"multivalue-variant fee=bar\")\n b = Spec(\"multivalue-variant fee=baz\")\n # The specs are abstract and they **could** be constrained,\n # as before concretization I don't know which type of variant\n # I have (if it is not a BV)\n assert a.intersects(b)\n # A variant cannot be parsed as single-valued until we try to\n # concretize. This means that we can constrain the variant above\n assert a.constrain(b)\n # ...but will fail during concretization if there are\n # multiple values set\n with pytest.raises(MultipleValuesInExclusiveVariantError):\n spack.concretize.concretize_one(a)\n\n def test_copy_satisfies_transitive(self):\n spec = spack.concretize.concretize_one(\"dttop\")\n copy = spec.copy()\n for s, t in zip(spec.traverse(), copy.traverse()):\n assert s.satisfies(t)\n assert t.satisfies(s)\n\n def test_intersects_virtual(self):\n assert Spec(\"mpich\").intersects(Spec(\"mpi\"))\n assert Spec(\"mpich2\").intersects(Spec(\"mpi\"))\n assert Spec(\"zmpi\").intersects(Spec(\"mpi\"))\n\n def test_intersects_virtual_providers(self):\n \"\"\"Tests that we can always intersect virtual providers from abstract specs.\n Concretization will give meaning to virtuals, and eventually forbid certain\n configurations.\n \"\"\"\n assert Spec(\"netlib-lapack ^openblas\").intersects(\"netlib-lapack ^openblas\")\n assert Spec(\"netlib-lapack ^netlib-blas\").intersects(\"netlib-lapack ^openblas\")\n assert Spec(\"netlib-lapack ^openblas\").intersects(\"netlib-lapack ^netlib-blas\")\n assert Spec(\"netlib-lapack ^netlib-blas\").intersects(\"netlib-lapack ^netlib-blas\")\n\n def test_intersectable_concrete_specs_must_have_the_same_hash(self):\n \"\"\"Ensure that concrete specs are matched *exactly* by hash.\"\"\"\n s1 = spack.concretize.concretize_one(\"mpileaks\")\n s2 = s1.copy()\n\n assert s1.satisfies(s2)\n assert s2.satisfies(s1)\n assert s1.intersects(s2)\n\n # Simulate specs that were installed before and after a change to\n # Spack's hashing algorithm. This just reverses s2's hash.\n s2._hash = s1.dag_hash()[-1::-1]\n\n assert not s1.satisfies(s2)\n assert not s2.satisfies(s1)\n assert not s1.intersects(s2)\n\n # ========================================================================\n # Indexing specs\n # ========================================================================\n def test_self_index(self):\n s = Spec(\"callpath\")\n assert s[\"callpath\"] == s\n\n def test_dep_index(self, default_mock_concretization):\n \"\"\"Tests __getitem__ and __contains__ for specs.\"\"\"\n s = default_mock_concretization(\"callpath\")\n\n assert s[\"callpath\"] == s\n\n # Real dependencies\n for key in (\"dyninst\", \"libdwarf\", \"libelf\"):\n assert isinstance(s[key], Spec)\n assert s[key].name == key\n assert key in s\n\n # Virtual dependencies\n assert s[\"mpi\"].name == \"mpich\"\n assert \"mpi\" in s\n\n @pytest.mark.usefixtures(\"config\")\n def test_virtual_index(self):\n s = spack.concretize.concretize_one(\"callpath\")\n s_mpich = spack.concretize.concretize_one(\"callpath ^mpich\")\n s_mpich2 = spack.concretize.concretize_one(\"callpath ^mpich2\")\n s_zmpi = spack.concretize.concretize_one(\"callpath ^zmpi\")\n\n assert s[\"mpi\"].name != \"mpi\"\n assert s_mpich[\"mpi\"].name == \"mpich\"\n assert s_mpich2[\"mpi\"].name == \"mpich2\"\n assert s_zmpi[\"zmpi\"].name == \"zmpi\"\n\n for spec in [s, s_mpich, s_mpich2, s_zmpi]:\n assert \"mpi\" in spec\n\n @pytest.mark.parametrize(\n \"lhs,rhs\",\n [\n (\"libelf\", \"@1.0\"),\n (\"libelf\", \"@1.0:5.0\"),\n (\"libelf\", \"%gcc\"),\n (\"libelf%gcc\", \"%gcc@4.5\"),\n (\"libelf\", \"+debug\"),\n (\"libelf\", \"debug=*\"),\n (\"libelf\", \"~debug\"),\n (\"libelf\", \"debug=2\"),\n (\"libelf\", 'cppflags=\"-O3\"'),\n (\"libelf\", 'cppflags==\"-O3\"'),\n (\"libelf^foo\", \"libelf^foo@1.0\"),\n (\"libelf^foo\", \"libelf^foo@1.0:5.0\"),\n (\"libelf^foo\", \"libelf^foo%gcc\"),\n (\"libelf^foo%gcc\", \"libelf^foo%gcc@4.5\"),\n (\"libelf^foo\", \"libelf^foo+debug\"),\n (\"libelf^foo\", \"libelf^foo~debug\"),\n (\"libelf\", \"^foo\"),\n (\"mpileaks ^callpath %gcc@14\", \"mpileaks ^callpath %gcc@14.1\"),\n (\"mpileaks %[deptypes=build] mpich\", \"mpileaks %[deptypes=link] mpich\"),\n (\"mpileaks %mpich\", \"mpileaks %[deptypes=link] mpich\"),\n ],\n )\n def test_lhs_is_changed_when_constraining(self, lhs, rhs):\n lhs, rhs = Spec(lhs), Spec(rhs)\n\n assert lhs.intersects(rhs)\n assert rhs.intersects(lhs)\n assert not lhs.satisfies(rhs)\n\n assert lhs.constrain(rhs) is True\n assert lhs.satisfies(rhs)\n\n @pytest.mark.parametrize(\n \"lhs,rhs\",\n [\n (\"libelf\", \"libelf\"),\n (\"libelf@1.0\", \"@1.0\"),\n (\"libelf@1.0:5.0\", \"@1.0:5.0\"),\n (\"libelf%gcc\", \"%gcc\"),\n (\"libelf%gcc@4.5\", \"%gcc@4.5\"),\n (\"libelf+debug\", \"+debug\"),\n (\"libelf~debug\", \"~debug\"),\n (\"libelf debug=2\", \"debug=2\"),\n (\"libelf debug=2\", \"debug=*\"),\n ('libelf cppflags=\"-O3\"', 'cppflags=\"-O3\"'),\n ('libelf cppflags==\"-O3\"', 'cppflags==\"-O3\"'),\n (\"libelf^foo@1.0\", \"libelf^foo@1.0\"),\n (\"libelf^foo@1.0:5.0\", \"libelf^foo@1.0:5.0\"),\n (\"libelf^foo%gcc\", \"libelf^foo%gcc\"),\n (\"libelf^foo%gcc@4.5\", \"libelf^foo%gcc@4.5\"),\n (\"libelf^foo+debug\", \"libelf^foo+debug\"),\n (\"libelf^foo~debug\", \"libelf^foo~debug\"),\n ('libelf^foo cppflags=\"-O3\"', 'libelf^foo cppflags=\"-O3\"'),\n (\"mpileaks ^callpath %gcc@14.1\", \"mpileaks ^callpath %gcc@14\"),\n (\"mpileaks %[deptypes=build] gcc@14.1\", \"mpileaks %gcc@14\"),\n ],\n )\n def test_lhs_is_not_changed_when_constraining(self, lhs, rhs):\n lhs, rhs = Spec(lhs), Spec(rhs)\n assert lhs.intersects(rhs)\n assert rhs.intersects(lhs)\n assert lhs.satisfies(rhs)\n assert lhs.constrain(rhs) is False\n\n def test_exceptional_paths_for_constructor(self):\n with pytest.raises(TypeError):\n Spec((1, 2))\n\n with pytest.raises(ValueError):\n Spec(\"libelf foo\")\n\n def test_spec_formatting(self, default_mock_concretization):\n spec = default_mock_concretization(\"multivalue-variant cflags=-O2\")\n\n # Testing named strings ie {string} and whether we get\n # the correct component\n # Mixed case intentional to test both\n # Fields are as follow\n # fmt_str: the format string to test\n # sigil: the portion that is a sigil (may be empty string)\n # prop: the property to get\n # component: subcomponent of spec from which to get property\n package_segments = [\n (\"{NAME}\", \"\", \"name\", lambda spec: spec),\n (\"{VERSION}\", \"\", \"version\", lambda spec: spec),\n (\"{compiler}\", \"\", \"compiler\", lambda spec: spec),\n (\"{compiler_flags}\", \"\", \"compiler_flags\", lambda spec: spec),\n (\"{variants}\", \"\", \"variants\", lambda spec: spec),\n (\"{architecture}\", \"\", \"architecture\", lambda spec: spec),\n (\"{@VERSIONS}\", \"@\", \"versions\", lambda spec: spec),\n (\"{%compiler}\", \"%\", \"compiler\", lambda spec: spec),\n (\"{arch=architecture}\", \"arch=\", \"architecture\", lambda spec: spec),\n (\"{namespace=namespace}\", \"namespace=\", \"namespace\", lambda spec: spec),\n (\"{compiler.name}\", \"\", \"name\", lambda spec: spec.compiler),\n (\"{compiler.version}\", \"\", \"version\", lambda spec: spec.compiler),\n (\n \"{compiler.version.up_to_1}\",\n \"\",\n \"up_to_1\",\n lambda spec: spec.compiler.version.up_to(1),\n ),\n (\"{%compiler.name}\", \"%\", \"name\", lambda spec: spec.compiler),\n (\"{@compiler.version}\", \"@\", \"version\", lambda spec: spec.compiler),\n (\"{architecture.platform}\", \"\", \"platform\", lambda spec: spec.architecture),\n (\"{architecture.os}\", \"\", \"os\", lambda spec: spec.architecture),\n (\"{architecture.target}\", \"\", \"target\", lambda spec: spec.architecture),\n (\"{prefix}\", \"\", \"prefix\", lambda spec: spec),\n (\"{external}\", \"\", \"external\", lambda spec: spec), # test we print \"False\"\n ]\n\n hash_segments = [\n (\"{hash:7}\", \"\", lambda s: s.dag_hash(7)),\n (\"{/hash}\", \"/\", lambda s: \"/\" + s.dag_hash()),\n ]\n\n variants_segments = [\n (\"{variants.debug}\", spec, \"debug\"),\n (\"{variants.foo}\", spec, \"foo\"),\n (\"{^pkg-a.variants.bvv}\", spec[\"pkg-a\"], \"bvv\"),\n (\"{^pkg-a.variants.foo}\", spec[\"pkg-a\"], \"foo\"),\n ]\n\n other_segments = [\n (\"{spack_root}\", spack.paths.spack_root),\n (\"{spack_install}\", spack.store.STORE.layout.root),\n ]\n\n def depify(depname, fmt_str, sigil):\n sig = len(sigil)\n opening = fmt_str[: 1 + sig]\n closing = fmt_str[1 + sig :]\n return spec[depname], opening + f\"^{depname}.\" + closing\n\n def check_prop(check_spec, fmt_str, prop, getter):\n actual = spec.format(fmt_str)\n expected = getter(check_spec)\n assert actual == str(expected).strip()\n\n for named_str, sigil, prop, get_component in package_segments:\n getter = lambda s: sigil + str(getattr(get_component(s), prop, \"\"))\n check_prop(spec, named_str, prop, getter)\n mpi, fmt_str = depify(\"mpi\", named_str, sigil)\n check_prop(mpi, fmt_str, prop, getter)\n\n for named_str, sigil, getter in hash_segments:\n assert spec.format(named_str) == getter(spec)\n callpath, fmt_str = depify(\"callpath\", named_str, sigil)\n assert spec.format(fmt_str) == getter(callpath)\n\n for named_str, test_spec, variant_name in variants_segments:\n assert test_spec.format(named_str) == str(test_spec.variants[variant_name])\n assert test_spec.format(named_str[:-1] + \".value}\") == str(\n test_spec.variants[variant_name].value\n )\n\n for named_str, expected in other_segments:\n actual = spec.format(named_str)\n assert expected == actual\n\n @pytest.mark.parametrize(\n \"fmt_str\",\n [\n \"{name}\",\n \"{version}\",\n \"{@version}\",\n \"{namespace}\",\n \"{ namespace=namespace}\",\n \"{ namespace =namespace}\",\n \"{ name space =namespace}\",\n \"{arch}\",\n \"{architecture}\",\n \"{arch=architecture}\",\n \"{ arch=architecture}\",\n \"{ arch =architecture}\",\n ],\n )\n def test_spec_format_null_attributes(self, fmt_str):\n \"\"\"Ensure that attributes format to empty strings when their values are null.\"\"\"\n spec = spack.spec.Spec()\n assert spec.format(fmt_str) == \"\"\n\n def test_spec_formatting_spaces_in_key(self, default_mock_concretization):\n spec = default_mock_concretization(\"multivalue-variant cflags=-O2\")\n\n # test that spaces are preserved, if they come after some other text, otherwise\n # they are trimmed.\n # TODO: should we be trimming whitespace from formats? Probably not.\n assert spec.format(\"x{ arch=architecture}\") == f\"x arch={spec.architecture}\"\n assert spec.format(\"x{ namespace=namespace}\") == f\"x namespace={spec.namespace}\"\n assert spec.format(\"x{ name space =namespace}\") == f\"x name space ={spec.namespace}\"\n assert spec.format(\"x{ os =os}\") == f\"x os ={spec.os}\"\n\n @pytest.mark.parametrize(\n \"fmt_str\", [\"{@name}\", \"{@version.concrete}\", \"{%compiler.version}\", \"{/hashd}\"]\n )\n def test_spec_formatting_sigil_mismatches(self, default_mock_concretization, fmt_str):\n spec = default_mock_concretization(\"multivalue-variant cflags=-O2\")\n\n with pytest.raises(SpecFormatSigilError):\n spec.format(fmt_str)\n\n @pytest.mark.parametrize(\n \"fmt_str\",\n [\n r\"{}\",\n r\"name}\",\n r\"\\{name}\",\n r\"{name\",\n r\"{name\\}\",\n r\"{_concrete}\",\n r\"{dag_hash}\",\n r\"{foo}\",\n r\"{+variants.debug}\",\n r\"{variants.this_variant_does_not_exist}\",\n ],\n )\n def test_spec_formatting_bad_formats(self, default_mock_concretization, fmt_str):\n spec = default_mock_concretization(\"multivalue-variant cflags=-O2\")\n with pytest.raises(SpecFormatStringError):\n spec.format(fmt_str)\n\n def test_wildcard_is_invalid_variant_value(self):\n \"\"\"The spec string x=* is parsed as a multi-valued variant with values the empty set.\n That excludes * as a literal variant value.\"\"\"\n with pytest.raises(spack.spec_parser.SpecParsingError, match=\"cannot use reserved value\"):\n Spec(\"multivalue-variant foo=*,bar\")\n\n def test_errors_in_variant_directive(self):\n variant = spack.directives.variant.__wrapped__\n\n class Pkg:\n name = \"PKG\"\n\n # We can't use names that are reserved by Spack\n fn = variant(\"patches\")\n with pytest.raises(spack.directives.DirectiveError) as exc_info:\n fn(Pkg())\n assert \"The name 'patches' is reserved\" in str(exc_info.value)\n\n # We can't have conflicting definitions for arguments\n fn = variant(\"foo\", values=spack.variant.any_combination_of(\"fee\", \"foom\"), default=\"bar\")\n with pytest.raises(spack.directives.DirectiveError) as exc_info:\n fn(Pkg())\n assert \" it is handled by an attribute of the 'values' argument\" in str(exc_info.value)\n\n # We can't leave None as a default value\n fn = variant(\"foo\", default=None)\n with pytest.raises(spack.directives.DirectiveError) as exc_info:\n fn(Pkg())\n assert \"either a default was not explicitly set, or 'None' was used\" in str(exc_info.value)\n\n # We can't use an empty string as a default value\n fn = variant(\"foo\", default=\"\")\n with pytest.raises(spack.directives.DirectiveError) as exc_info:\n fn(Pkg())\n assert \"the default cannot be an empty string\" in str(exc_info.value)\n\n def test_abstract_spec_prefix_error(self):\n spec = Spec(\"libelf\")\n\n with pytest.raises(SpecError):\n spec.prefix\n\n def test_forwarding_of_architecture_attributes(self):\n spec = spack.concretize.concretize_one(\"libelf target=x86_64\")\n\n # Check that we can still access each member through\n # the architecture attribute\n assert \"test\" in spec.architecture\n assert \"debian\" in spec.architecture\n assert \"x86_64\" in spec.architecture\n\n # Check that we forward the platform and os attribute correctly\n assert spec.platform == \"test\"\n assert spec.os == \"debian6\"\n\n # Check that the target is also forwarded correctly and supports\n # all the operators we expect\n assert spec.target == \"x86_64\"\n assert spec.target.family == \"x86_64\"\n assert \"avx512\" not in spec.target\n assert spec.target < \"broadwell\"\n\n @pytest.mark.parametrize(\"transitive\", [True, False])\n def test_splice(self, transitive, default_mock_concretization):\n # Tests the new splice function in Spec using a somewhat simple case\n # with a variant with a conditional dependency.\n spec = default_mock_concretization(\"splice-t\")\n dep = default_mock_concretization(\"splice-h+foo\")\n\n # Sanity checking that these are not the same thing.\n assert dep.dag_hash() != spec[\"splice-h\"].dag_hash()\n\n # Do the splice.\n out = spec.splice(dep, transitive)\n\n # Returned spec should still be concrete.\n assert out.concrete\n\n # Traverse the spec and assert that all dependencies are accounted for.\n for node in spec.traverse():\n assert node.name in out\n\n # If the splice worked, then the dag hash of the spliced dep should\n # now match the dag hash of the build spec of the dependency from the\n # returned spec.\n out_h_build = out[\"splice-h\"].build_spec\n assert out_h_build.dag_hash() == dep.dag_hash()\n\n # Transitivity should determine whether the transitive dependency was\n # changed.\n expected_z = dep[\"splice-z\"] if transitive else spec[\"splice-z\"]\n assert out[\"splice-z\"].dag_hash() == expected_z.dag_hash()\n\n # Sanity check build spec of out should be the original spec.\n assert out[\"splice-t\"].build_spec.dag_hash() == spec[\"splice-t\"].dag_hash()\n\n # Finally, the spec should know it's been spliced:\n assert out.spliced\n\n def test_splice_intransitive_complex(self, setup_complex_splice):\n a_red, c_blue = setup_complex_splice\n\n spliced = a_red.splice(c_blue, transitive=False)\n assert spliced.satisfies(\n \"pkg-a color=red ^pkg-b color=red ^pkg-c color=blue \"\n \"^pkg-d color=red ^pkg-e color=red ^pkg-f color=blue ^pkg-g@2 color=red\"\n )\n assert set(spliced.dependencies(deptype=dt.BUILD)) == set()\n assert spliced.build_spec == a_red\n\n # We cannot check spliced[\"b\"].build_spec is spliced[\"b\"] because Spec.__getitem__ creates\n # a new wrapper object on each invocation. So we select once and check on that object\n # For the rest of the unchanged specs we will just check the s._build_spec is None.\n b = spliced[\"pkg-b\"]\n assert b == a_red[\"pkg-b\"]\n assert b.build_spec is b\n assert set(b.dependents()) == {spliced}\n\n assert spliced[\"pkg-c\"].satisfies(\n \"pkg-c color=blue ^pkg-d color=red ^pkg-e color=red \"\n \"^pkg-f color=blue ^pkg-g@2 color=red\"\n )\n assert set(spliced[\"pkg-c\"].dependencies(deptype=dt.BUILD)) == set()\n assert spliced[\"pkg-c\"].build_spec == c_blue\n assert set(spliced[\"pkg-c\"].dependents()) == {spliced}\n\n assert spliced[\"pkg-d\"] == a_red[\"pkg-d\"]\n assert spliced[\"pkg-d\"]._build_spec is None\n # Since D had a parent changed, it has a split edge for link vs build dependent\n # note: spliced[\"b\"] == b_red, referenced differently to preserve logic\n assert set(spliced[\"pkg-d\"].dependents()) == {\n spliced[\"pkg-b\"],\n spliced[\"pkg-c\"],\n a_red[\"pkg-c\"],\n }\n assert set(spliced[\"pkg-d\"].dependents(deptype=dt.BUILD)) == {\n a_red[\"pkg-b\"],\n a_red[\"pkg-c\"],\n }\n\n assert spliced[\"pkg-e\"] == a_red[\"pkg-e\"]\n assert spliced[\"pkg-e\"]._build_spec is None\n # Because a copy of e is used, it does not have dependnets in the original specs\n assert set(spliced[\"pkg-e\"].dependents()) == {spliced[\"pkg-b\"], spliced[\"pkg-f\"]}\n # Build dependent edge to f because f originally depended on the e this was copied from\n assert set(spliced[\"pkg-e\"].dependents(deptype=dt.BUILD)) == {spliced[\"pkg-b\"]}\n\n assert spliced[\"pkg-f\"].satisfies(\"pkg-f color=blue ^pkg-e color=red ^pkg-g@2 color=red\")\n assert set(spliced[\"pkg-f\"].dependencies(deptype=dt.BUILD)) == set()\n assert spliced[\"pkg-f\"].build_spec == c_blue[\"pkg-f\"]\n assert set(spliced[\"pkg-f\"].dependents()) == {spliced[\"pkg-c\"]}\n\n # spliced[\"pkg-g\"] is g2, but spliced[\"pkg-b\"][\"pkg-g\"] is g1\n assert spliced[\"pkg-g\"] == a_red[\"pkg-g\"]\n assert spliced[\"pkg-g\"]._build_spec is None\n assert set(spliced[\"pkg-g\"].dependents(deptype=dt.LINK)) == {\n spliced,\n spliced[\"pkg-c\"],\n spliced[\"pkg-f\"],\n a_red[\"pkg-c\"],\n }\n\n assert spliced[\"pkg-b\"][\"pkg-g\"] == a_red[\"pkg-b\"][\"pkg-g\"]\n assert spliced[\"pkg-b\"][\"pkg-g\"]._build_spec is None\n assert set(spliced[\"pkg-b\"][\"pkg-g\"].dependents()) == {\n spliced[\"pkg-b\"],\n spliced[\"pkg-d\"],\n spliced[\"pkg-e\"],\n }\n\n for edge in spliced.traverse_edges(cover=\"edges\", deptype=dt.LINK | dt.RUN):\n # traverse_edges creates a synthetic edge with no deptypes to the root\n if edge.depflag:\n depflag = dt.LINK\n if not edge.parent.spliced:\n depflag |= dt.BUILD\n assert edge.depflag == depflag\n\n def test_splice_transitive_complex(self, setup_complex_splice):\n a_red, c_blue = setup_complex_splice\n\n spliced = a_red.splice(c_blue, transitive=True)\n assert spliced.satisfies(\n \"pkg-a color=red ^pkg-b color=red ^pkg-c color=blue ^pkg-d color=blue \"\n \"^pkg-e color=blue ^pkg-f color=blue ^pkg-g@3 color=blue\"\n )\n assert set(spliced.dependencies(deptype=dt.BUILD)) == set()\n assert spliced.build_spec == a_red\n\n assert spliced[\"pkg-b\"].satisfies(\n \"pkg-b color=red ^pkg-d color=blue ^pkg-e color=blue ^pkg-g@2 color=blue\"\n )\n assert set(spliced[\"pkg-b\"].dependencies(deptype=dt.BUILD)) == set()\n assert spliced[\"pkg-b\"].build_spec == a_red[\"pkg-b\"]\n assert set(spliced[\"pkg-b\"].dependents()) == {spliced}\n\n # We cannot check spliced[\"c\"].build_spec is spliced[\"c\"] because Spec.__getitem__ creates\n # a new wrapper object on each invocation. So we select once and check on that object\n # For the rest of the unchanged specs we will just check the s._build_spec is None.\n c = spliced[\"pkg-c\"]\n assert c == c_blue\n assert c.build_spec is c\n assert set(c.dependents()) == {spliced}\n\n assert spliced[\"pkg-d\"] == c_blue[\"pkg-d\"]\n assert spliced[\"pkg-d\"]._build_spec is None\n assert set(spliced[\"pkg-d\"].dependents()) == {spliced[\"pkg-b\"], spliced[\"pkg-c\"]}\n\n assert spliced[\"pkg-e\"] == c_blue[\"pkg-e\"]\n assert spliced[\"pkg-e\"]._build_spec is None\n assert set(spliced[\"pkg-e\"].dependents()) == {spliced[\"pkg-b\"], spliced[\"pkg-f\"]}\n\n assert spliced[\"pkg-f\"] == c_blue[\"pkg-f\"]\n assert spliced[\"pkg-f\"]._build_spec is None\n assert set(spliced[\"pkg-f\"].dependents()) == {spliced[\"pkg-c\"]}\n\n # spliced[\"g\"] is g3, but spliced[\"d\"][\"g\"] is g1\n assert spliced[\"pkg-g\"] == c_blue[\"pkg-g\"]\n assert spliced[\"pkg-g\"]._build_spec is None\n assert set(spliced[\"pkg-g\"].dependents(deptype=dt.LINK)) == {\n spliced,\n spliced[\"pkg-b\"],\n spliced[\"pkg-c\"],\n spliced[\"pkg-e\"],\n spliced[\"pkg-f\"],\n }\n # Because a copy of g3 is used, it does not have dependents in the original specs\n # It has build dependents on these spliced specs because it is an unchanged dependency\n # for them\n assert set(spliced[\"pkg-g\"].dependents(deptype=dt.BUILD)) == {\n spliced[\"pkg-c\"],\n spliced[\"pkg-e\"],\n spliced[\"pkg-f\"],\n }\n\n assert spliced[\"pkg-d\"][\"pkg-g\"] == c_blue[\"pkg-d\"][\"pkg-g\"]\n assert spliced[\"pkg-d\"][\"pkg-g\"]._build_spec is None\n assert set(spliced[\"pkg-d\"][\"pkg-g\"].dependents()) == {spliced[\"pkg-d\"]}\n\n for edge in spliced.traverse_edges(cover=\"edges\", deptype=dt.LINK | dt.RUN):\n # traverse_edges creates a synthetic edge with no deptypes to the root\n if edge.depflag:\n depflag = dt.LINK\n if not edge.parent.spliced:\n depflag |= dt.BUILD\n assert edge.depflag == depflag\n\n @pytest.mark.parametrize(\"transitive\", [True, False])\n def test_splice_with_cached_hashes(self, default_mock_concretization, transitive):\n spec = default_mock_concretization(\"splice-t\")\n dep = default_mock_concretization(\"splice-h+foo\")\n\n # monkeypatch hashes so we can test that they are cached\n spec._hash = \"aaaaaa\"\n dep._hash = \"bbbbbb\"\n spec[\"splice-h\"]._hash = \"cccccc\"\n spec[\"splice-z\"]._hash = \"dddddd\"\n dep[\"splice-z\"]._hash = \"eeeeee\"\n\n out = spec.splice(dep, transitive=transitive)\n out_z_expected = (dep if transitive else spec)[\"splice-z\"]\n\n assert out.dag_hash() != spec.dag_hash()\n assert (out[\"splice-h\"].dag_hash() == dep.dag_hash()) == transitive\n assert out[\"splice-z\"].dag_hash() == out_z_expected.dag_hash()\n\n @pytest.mark.parametrize(\"transitive\", [True, False])\n def test_splice_input_unchanged(self, default_mock_concretization, transitive):\n spec = default_mock_concretization(\"splice-t\")\n dep = default_mock_concretization(\"splice-h+foo\")\n orig_spec_hash = spec.dag_hash()\n orig_dep_hash = dep.dag_hash()\n spec.splice(dep, transitive)\n # Post-splice, dag hash should still be different; no changes should be\n # made to these specs.\n assert spec.dag_hash() == orig_spec_hash\n assert dep.dag_hash() == orig_dep_hash\n\n @pytest.mark.parametrize(\"transitive\", [True, False])\n def test_splice_subsequent(self, default_mock_concretization, transitive):\n spec = default_mock_concretization(\"splice-t\")\n dep = default_mock_concretization(\"splice-h+foo\")\n out = spec.splice(dep, transitive)\n\n # Now we attempt a second splice.\n dep = default_mock_concretization(\"splice-z+bar\")\n\n # Transitivity shouldn't matter since Splice Z has no dependencies.\n out2 = out.splice(dep, transitive)\n assert out2.concrete\n assert out2[\"splice-z\"].dag_hash() != spec[\"splice-z\"].dag_hash()\n assert out2[\"splice-z\"].dag_hash() != out[\"splice-z\"].dag_hash()\n assert out2[\"splice-t\"].build_spec.dag_hash() == spec[\"splice-t\"].dag_hash()\n assert out2.spliced\n\n @pytest.mark.parametrize(\"transitive\", [True, False])\n def test_splice_dict(self, default_mock_concretization, transitive):\n spec = default_mock_concretization(\"splice-t\")\n dep = default_mock_concretization(\"splice-h+foo\")\n out = spec.splice(dep, transitive)\n\n # Sanity check all hashes are unique...\n assert spec.dag_hash() != dep.dag_hash()\n assert out.dag_hash() != dep.dag_hash()\n assert out.dag_hash() != spec.dag_hash()\n node_list = out.to_dict()[\"spec\"][\"nodes\"]\n root_nodes = [n for n in node_list if n[\"hash\"] == out.dag_hash()]\n build_spec_nodes = [n for n in node_list if n[\"hash\"] == spec.dag_hash()]\n assert spec.dag_hash() == out.build_spec.dag_hash()\n assert len(root_nodes) == 1\n assert len(build_spec_nodes) == 1\n\n @pytest.mark.parametrize(\"transitive\", [True, False])\n def test_splice_dict_roundtrip(self, default_mock_concretization, transitive):\n spec = default_mock_concretization(\"splice-t\")\n dep = default_mock_concretization(\"splice-h+foo\")\n out = spec.splice(dep, transitive)\n\n # Sanity check all hashes are unique...\n assert spec.dag_hash() != dep.dag_hash()\n assert out.dag_hash() != dep.dag_hash()\n assert out.dag_hash() != spec.dag_hash()\n out_rt_spec = Spec.from_dict(out.to_dict()) # rt is \"round trip\"\n assert out_rt_spec.dag_hash() == out.dag_hash()\n out_rt_spec_bld_hash = out_rt_spec.build_spec.dag_hash()\n out_rt_spec_h_bld_hash = out_rt_spec[\"splice-h\"].build_spec.dag_hash()\n out_rt_spec_z_bld_hash = out_rt_spec[\"splice-z\"].build_spec.dag_hash()\n\n # In any case, the build spec for splice-t (root) should point to the\n # original spec, preserving build provenance.\n assert spec.dag_hash() == out_rt_spec_bld_hash\n assert out_rt_spec.dag_hash() != out_rt_spec_bld_hash\n\n # The build spec for splice-h should always point to the introduced\n # spec, since that is the spec spliced in.\n assert dep[\"splice-h\"].dag_hash() == out_rt_spec_h_bld_hash\n\n # The build spec for splice-z will depend on whether or not the splice\n # was transitive.\n expected_z_bld_hash = (\n dep[\"splice-z\"].dag_hash() if transitive else spec[\"splice-z\"].dag_hash()\n )\n assert expected_z_bld_hash == out_rt_spec_z_bld_hash\n\n @pytest.mark.parametrize(\n \"spec,constraint,expected_result\",\n [\n (\"libelf target=haswell\", \"target=broadwell\", False),\n (\"libelf target=haswell\", \"target=haswell\", True),\n (\"libelf target=haswell\", \"target=x86_64:\", True),\n (\"libelf target=haswell\", \"target=:haswell\", True),\n (\"libelf target=haswell\", \"target=icelake,:nocona\", False),\n (\"libelf target=haswell\", \"target=haswell,:nocona\", True),\n # Check that a single target is not treated as the start\n # or the end of an open range\n (\"libelf target=haswell\", \"target=x86_64\", False),\n (\"libelf target=x86_64\", \"target=haswell\", False),\n ],\n )\n @pytest.mark.regression(\"13111\")\n def test_target_constraints(self, spec, constraint, expected_result):\n s = Spec(spec)\n assert s.intersects(constraint) is expected_result\n\n @pytest.mark.regression(\"13124\")\n def test_error_message_unknown_variant(self):\n s = Spec(\"mpileaks +unknown\")\n with pytest.raises(UnknownVariantError):\n spack.concretize.concretize_one(s)\n\n @pytest.mark.regression(\"18527\")\n def test_satisfies_dependencies_ordered(self):\n d = Spec(\"zmpi ^fake\")\n s = Spec(\"mpileaks\")\n s._add_dependency(d, depflag=0, virtuals=())\n assert s.satisfies(\"mpileaks ^zmpi ^fake\")\n\n @pytest.mark.parametrize(\"transitive\", [True, False])\n def test_splice_swap_names(self, default_mock_concretization, transitive):\n spec = default_mock_concretization(\"splice-vt\")\n dep = default_mock_concretization(\"splice-a+foo\")\n out = spec.splice(dep, transitive)\n assert dep.name in out\n assert transitive == (\"+foo\" in out[\"splice-z\"])\n\n @pytest.mark.parametrize(\"transitive\", [True, False])\n def test_splice_swap_names_mismatch_virtuals(self, default_mock_concretization, transitive):\n vt = default_mock_concretization(\"splice-vt\")\n vh = default_mock_concretization(\"splice-vh+foo\")\n with pytest.raises(spack.spec.SpliceError, match=\"virtual\"):\n vt.splice(vh, transitive)\n\n def test_adaptor_optflags(self):\n \"\"\"Tests that we can obtain the list of optflags, and debugflags,\n from the compiler adaptor, and that this list is taken from the\n appropriate compiler package.\n \"\"\"\n # pkg-a depends on c, so only the gcc compiler should be chosen\n spec = spack.concretize.concretize_one(Spec(\"pkg-a %gcc\"))\n assert \"-Otestopt\" in spec.package.compiler.opt_flags\n # This is not set, make sure we get an empty list\n for x in spec.package.compiler.debug_flags:\n pass\n\n def test_spec_override(self):\n init_spec = Spec(\"pkg-a foo=baz foobar=baz cflags=-O3 cxxflags=-O1\")\n change_spec = Spec(\"pkg-a foo=fee cflags=-O2\")\n new_spec = spack.concretize.concretize_one(Spec.override(init_spec, change_spec))\n assert \"foo=fee\" in new_spec\n # This check fails without concretizing: apparently if both specs are\n # abstract, then the spec will always be considered to satisfy\n # 'variant=value' (regardless of whether it in fact does).\n assert \"foo=baz\" not in new_spec\n assert \"foobar=baz\" in new_spec\n assert new_spec.compiler_flags[\"cflags\"] == [\"-O2\"]\n assert new_spec.compiler_flags[\"cxxflags\"] == [\"-O1\"]\n\n def test_spec_override_with_nonexisting_variant(self):\n init_spec = Spec(\"pkg-a foo=baz foobar=baz cflags=-O3 cxxflags=-O1\")\n change_spec = Spec(\"pkg-a baz=fee\")\n with pytest.raises(ValueError):\n Spec.override(init_spec, change_spec)\n\n def test_spec_override_with_variant_not_in_init_spec(self):\n init_spec = Spec(\"pkg-a foo=baz foobar=baz cflags=-O3 cxxflags=-O1\")\n change_spec = Spec(\"pkg-a +bvv ~lorem_ipsum\")\n new_spec = spack.concretize.concretize_one(Spec.override(init_spec, change_spec))\n assert \"+bvv\" in new_spec\n assert \"~lorem_ipsum\" in new_spec\n\n @pytest.mark.parametrize(\n \"spec_str,specs_in_dag\",\n [\n (\"hdf5 ^[virtuals=mpi] mpich\", [(\"mpich\", \"mpich\"), (\"mpi\", \"mpich\")]),\n # Try different combinations with packages that provides a\n # disjoint set of virtual dependencies\n (\n \"netlib-scalapack ^mpich ^openblas-with-lapack\",\n [\n (\"mpi\", \"mpich\"),\n (\"lapack\", \"openblas-with-lapack\"),\n (\"blas\", \"openblas-with-lapack\"),\n ],\n ),\n (\n \"netlib-scalapack ^[virtuals=mpi] mpich ^openblas-with-lapack\",\n [\n (\"mpi\", \"mpich\"),\n (\"lapack\", \"openblas-with-lapack\"),\n (\"blas\", \"openblas-with-lapack\"),\n ],\n ),\n (\n \"netlib-scalapack ^mpich ^[virtuals=lapack] openblas-with-lapack\",\n [\n (\"mpi\", \"mpich\"),\n (\"lapack\", \"openblas-with-lapack\"),\n (\"blas\", \"openblas-with-lapack\"),\n ],\n ),\n (\n \"netlib-scalapack ^[virtuals=mpi] mpich ^[virtuals=lapack] openblas-with-lapack\",\n [\n (\"mpi\", \"mpich\"),\n (\"lapack\", \"openblas-with-lapack\"),\n (\"blas\", \"openblas-with-lapack\"),\n ],\n ),\n # Test that we can mix dependencies that provide an overlapping\n # sets of virtual dependencies\n (\n \"netlib-scalapack ^[virtuals=mpi] intel-parallel-studio \"\n \"^[virtuals=lapack] openblas-with-lapack\",\n [\n (\"mpi\", \"intel-parallel-studio\"),\n (\"lapack\", \"openblas-with-lapack\"),\n (\"blas\", \"openblas-with-lapack\"),\n ],\n ),\n (\n \"netlib-scalapack ^[virtuals=mpi] intel-parallel-studio ^openblas-with-lapack\",\n [\n (\"mpi\", \"intel-parallel-studio\"),\n (\"lapack\", \"openblas-with-lapack\"),\n (\"blas\", \"openblas-with-lapack\"),\n ],\n ),\n (\n \"netlib-scalapack ^intel-parallel-studio ^[virtuals=lapack] openblas-with-lapack\",\n [\n (\"mpi\", \"intel-parallel-studio\"),\n (\"lapack\", \"openblas-with-lapack\"),\n (\"blas\", \"openblas-with-lapack\"),\n ],\n ),\n # Test that we can bind more than one virtual to the same provider\n (\n \"netlib-scalapack ^[virtuals=lapack,blas] openblas-with-lapack\",\n [(\"lapack\", \"openblas-with-lapack\"), (\"blas\", \"openblas-with-lapack\")],\n ),\n ],\n )\n def test_virtual_deps_bindings(self, default_mock_concretization, spec_str, specs_in_dag):\n s = default_mock_concretization(spec_str)\n for label, expected in specs_in_dag:\n assert label in s\n assert s[label].satisfies(expected), label\n\n @pytest.mark.parametrize(\n \"spec_str\",\n [\n # openblas-with-lapack needs to provide blas and lapack together\n \"netlib-scalapack ^[virtuals=blas] intel-parallel-studio ^openblas-with-lapack\",\n # intel-* provides blas and lapack together, openblas can provide blas only\n \"netlib-scalapack ^[virtuals=lapack] intel-parallel-studio ^openblas\",\n ],\n )\n def test_unsatisfiable_virtual_deps_bindings(self, spec_str):\n with pytest.raises(spack.solver.asp.UnsatisfiableSpecError):\n spack.concretize.concretize_one(spec_str)\n\n @pytest.mark.parametrize(\n \"spec_str,abstract_tests,concrete_tests\",\n [\n # Ensure the 'when=+debug' is referred to 'callpath', and not to 'mpileaks',\n # and that we can concretize the spec despite 'callpath' has no debug variant\n (\n \"mpileaks+debug ^callpath %[when=+debug virtuals=mpi] zmpi\",\n [\n (\"^zmpi\", False),\n (\"^mpich\", False),\n (\"mpileaks+debug %[when=+debug virtuals=mpi] zmpi\", False),\n ],\n [(\"^zmpi\", False), (\"^[virtuals=mpi] mpich\", True)],\n ),\n # Ensure we don't skip conditional edges when testing because we associate them\n # with the wrong node (e.g. mpileaks instead of mpich)\n (\n \"mpileaks~debug ^mpich+debug %[when=+debug virtuals=c] llvm\",\n [(\"^mpich+debug %[when=+debug virtuals=c] gcc\", False)],\n [(\"^mpich %[virtuals=c] gcc\", False), (\"^mpich %[virtuals=c] llvm\", True)],\n ),\n ],\n )\n def test_conditional_dependencies_satisfies(\n self, spec_str, abstract_tests, concrete_tests, default_mock_concretization\n ):\n \"\"\"Tests satisfaction semantics for conditional specs, in different scenarios.\"\"\"\n s = Spec(spec_str)\n for c, result in abstract_tests:\n assert s.satisfies(c) is result\n\n concrete = default_mock_concretization(spec_str)\n for c, result in concrete_tests:\n assert concrete.satisfies(c) is result\n\n\n@pytest.mark.parametrize(\n \"spec_str,format_str,expected\",\n [\n (\"git-test@git.foo/bar\", \"{name}-{version}\", str(pathlib.Path(\"git-test-git.foo_bar\"))),\n (\"git-test@git.foo/bar\", \"{name}-{version}-{/hash}\", None),\n (\"git-test@git.foo/bar\", \"{name}/{version}\", str(pathlib.Path(\"git-test\", \"git.foo_bar\"))),\n # {compiler} is 'none' if a package does not depend on C, C++, or Fortran\n (\n f\"git-test@{'a' * 40}=1.0%gcc\",\n \"{name}/{version}/{compiler}\",\n str(pathlib.Path(\"git-test\", f\"{'a' * 40}_1.0\", \"none\")),\n ),\n (\n \"git-test@git.foo/bar=1.0%gcc\",\n \"{name}/{version}/{compiler}\",\n str(pathlib.Path(\"git-test\", \"git.foo_bar_1.0\", \"none\")),\n ),\n ],\n)\ndef test_spec_format_path(spec_str, format_str, expected, mock_git_test_package):\n _check_spec_format_path(spec_str, format_str, expected)\n\n\ndef _check_spec_format_path(spec_str, format_str, expected, path_ctor=None):\n spec = Spec(spec_str)\n if not expected:\n with pytest.raises((spack.spec.SpecFormatPathError, spack.spec.SpecFormatStringError)):\n spec.format_path(format_str, _path_ctor=path_ctor)\n else:\n formatted = spec.format_path(format_str, _path_ctor=path_ctor)\n assert formatted == expected\n\n\n@pytest.mark.parametrize(\n \"spec_str,format_str,expected\",\n [\n (\n \"git-test@git.foo/bar\",\n r\"C:\\\\installroot\\{name}\\{version}\",\n r\"C:\\installroot\\git-test\\git.foo_bar\",\n ),\n (\n \"git-test@git.foo/bar\",\n r\"\\\\hostname\\sharename\\{name}\\{version}\",\n r\"\\\\hostname\\sharename\\git-test\\git.foo_bar\",\n ),\n # leading '/' is preserved on windows but converted to '\\'\n # note that it's still not \"absolute\" -- absolute windows paths start with a drive.\n (\n \"git-test@git.foo/bar\",\n r\"/installroot/{name}/{version}\",\n r\"\\installroot\\git-test\\git.foo_bar\",\n ),\n ],\n)\ndef test_spec_format_path_windows(spec_str, format_str, expected, mock_git_test_package):\n _check_spec_format_path(spec_str, format_str, expected, path_ctor=pathlib.PureWindowsPath)\n\n\n@pytest.mark.parametrize(\n \"spec_str,format_str,expected\",\n [\n (\n \"git-test@git.foo/bar\",\n r\"/installroot/{name}/{version}\",\n \"/installroot/git-test/git.foo_bar\",\n ),\n (\n \"git-test@git.foo/bar\",\n r\"//installroot/{name}/{version}\",\n \"//installroot/git-test/git.foo_bar\",\n ),\n # This is likely unintentional on Linux: Firstly, \"\\\" is not a\n # path separator for POSIX, so this is treated as a single path\n # component (containing literal \"\\\" characters); secondly,\n # Spec.format treats \"\\\" as an escape character, so is\n # discarded (unless directly following another \"\\\")\n (\n \"git-test@git.foo/bar\",\n r\"C:\\\\installroot\\package-{name}-{version}\",\n r\"C__installrootpackage-git-test-git.foo_bar\",\n ),\n # \"\\\" is not a POSIX separator, and Spec.format treats \"\\{\" as a literal\n # \"{\", which means that the resulting format string is invalid\n (\"git-test@git.foo/bar\", r\"package\\{name}\\{version}\", None),\n ],\n)\ndef test_spec_format_path_posix(spec_str, format_str, expected, mock_git_test_package):\n _check_spec_format_path(spec_str, format_str, expected, path_ctor=pathlib.PurePosixPath)\n\n\n@pytest.mark.regression(\"3887\")\n@pytest.mark.parametrize(\"spec_str\", [\"py-extension2\", \"extension1\", \"perl-extension\"])\ndef test_is_extension_after_round_trip_to_dict(config, mock_packages, spec_str):\n # x is constructed directly from string, y from a\n # round-trip to dict representation\n x = spack.concretize.concretize_one(spec_str)\n y = Spec.from_dict(x.to_dict())\n\n # Using 'y' since the round-trip make us lose build dependencies\n for d in y.traverse():\n assert x[d.name].package.is_extension == y[d.name].package.is_extension\n\n\ndef test_malformed_spec_dict():\n # FIXME: This test was really testing the specific implementation with an ad-hoc test\n with pytest.raises(SpecError, match=\"malformed\"):\n Spec.from_dict(\n {\"spec\": {\"_meta\": {\"version\": 2}, \"nodes\": [{\"dependencies\": {\"name\": \"foo\"}}]}}\n )\n\n\ndef test_spec_dict_hashless_dep():\n # FIXME: This test was really testing the specific implementation with an ad-hoc test\n with pytest.raises(SpecError, match=\"Couldn't parse\"):\n Spec.from_dict(\n {\n \"spec\": {\n \"_meta\": {\"version\": 2},\n \"nodes\": [\n {\"name\": \"foo\", \"hash\": \"thehash\", \"dependencies\": [{\"name\": \"bar\"}]}\n ],\n }\n }\n )\n\n\n@pytest.mark.parametrize(\n \"anonymous,named,expected\",\n [\n (\"+plumed\", \"gromacs\", \"gromacs+plumed\"),\n (\"+plumed ^plumed%gcc\", \"gromacs\", \"gromacs+plumed ^plumed%gcc\"),\n (\"+plumed\", \"builtin.gromacs\", \"builtin.gromacs+plumed\"),\n ],\n)\ndef test_merge_anonymous_spec_with_named_spec(anonymous, named, expected):\n s = Spec(anonymous)\n changed = s.constrain(named)\n assert changed\n assert s == Spec(expected)\n\n\ndef test_spec_installed(default_mock_concretization, database):\n \"\"\"Test whether Spec.installed works.\"\"\"\n # a known installed spec should say that it's installed\n specs = database.query()\n spec = specs[0]\n assert spec.installed\n assert spec.copy().installed\n\n # an abstract spec should say it's not installed\n spec = Spec(\"not-a-real-package\")\n assert not spec.installed\n\n # pkg-a is not in the mock DB and is not installed\n spec = default_mock_concretization(\"pkg-a\")\n assert not spec.installed\n\n\n@pytest.mark.regression(\"30678\")\ndef test_call_dag_hash_on_old_dag_hash_spec(mock_packages, default_mock_concretization):\n # create a concrete spec\n a = default_mock_concretization(\"pkg-a\")\n dag_hashes = {spec.name: spec.dag_hash() for spec in a.traverse()}\n\n # make it look like an old DAG hash spec with no package hash on the spec.\n for spec in a.traverse():\n assert spec.concrete\n spec._package_hash = None\n\n for spec in a.traverse():\n assert dag_hashes[spec.name] == spec.dag_hash()\n\n with pytest.raises(ValueError, match=\"Cannot call package_hash()\"):\n spec.package_hash()\n\n\ndef test_spec_trim(mock_packages, config):\n top = spack.concretize.concretize_one(\"dt-diamond\")\n top.trim(\"dt-diamond-left\")\n remaining = {x.name for x in top.traverse()}\n assert {\n \"compiler-wrapper\",\n \"dt-diamond\",\n \"dt-diamond-right\",\n \"dt-diamond-bottom\",\n \"gcc-runtime\",\n \"gcc\",\n } == remaining\n\n top.trim(\"dt-diamond-right\")\n remaining = {x.name for x in top.traverse()}\n assert {\"compiler-wrapper\", \"dt-diamond\", \"gcc-runtime\", \"gcc\"} == remaining\n\n\n@pytest.mark.regression(\"30861\")\ndef test_concretize_partial_old_dag_hash_spec(mock_packages, config):\n # create an \"old\" spec with no package hash\n bottom = spack.concretize.concretize_one(\"dt-diamond-bottom\")\n delattr(bottom, \"_package_hash\")\n\n dummy_hash = \"zd4m26eis2wwbvtyfiliar27wkcv3ehk\"\n bottom._hash = dummy_hash\n\n # add it to an abstract spec as a dependency\n top = Spec(\"dt-diamond\")\n top.add_dependency_edge(bottom, depflag=0, virtuals=())\n\n # concretize with the already-concrete dependency\n top = spack.concretize.concretize_one(top)\n\n for spec in top.traverse():\n assert spec.concrete\n\n # make sure dag_hash is untouched\n assert spec[\"dt-diamond-bottom\"].dag_hash() == dummy_hash\n assert spec[\"dt-diamond-bottom\"]._hash == dummy_hash\n\n # make sure package hash is NOT recomputed\n assert not getattr(spec[\"dt-diamond-bottom\"], \"_package_hash\", None)\n\n\ndef test_package_hash_affects_dunder_and_dag_hash(mock_packages, default_mock_concretization):\n a1 = default_mock_concretization(\"pkg-a\")\n a2 = default_mock_concretization(\"pkg-a\")\n\n assert hash(a1) == hash(a2)\n assert a1.dag_hash() == a2.dag_hash()\n\n a1.clear_caches()\n a2.clear_caches()\n\n # tweak the dag hash of one of these specs\n new_hash = \"00000000000000000000000000000000\"\n if new_hash == a1._package_hash:\n new_hash = \"11111111111111111111111111111111\"\n a1._package_hash = new_hash\n\n assert hash(a1) != hash(a2)\n assert a1.dag_hash() != a2.dag_hash()\n\n\ndef test_intersects_and_satisfies_on_concretized_spec(default_mock_concretization):\n \"\"\"Test that a spec obtained by concretizing an abstract spec, satisfies the abstract spec\n but not vice-versa.\n \"\"\"\n a1 = default_mock_concretization(\"pkg-a@1.0\")\n a2 = Spec(\"pkg-a@1.0\")\n\n assert a1.intersects(a2)\n assert a2.intersects(a1)\n assert a1.satisfies(a2)\n assert not a2.satisfies(a1)\n\n\n@pytest.mark.parametrize(\n \"abstract_spec,spec_str\",\n [\n (\"v1-provider\", \"v1-consumer ^conditional-provider+disable-v1\"),\n (\"conditional-provider\", \"v1-consumer ^conditional-provider+disable-v1\"),\n (\"^v1-provider\", \"v1-consumer ^conditional-provider+disable-v1\"),\n (\"^conditional-provider\", \"v1-consumer ^conditional-provider+disable-v1\"),\n ],\n)\n@pytest.mark.regression(\"35597\")\ndef test_abstract_provider_in_spec(abstract_spec, spec_str, default_mock_concretization):\n s = default_mock_concretization(spec_str)\n assert abstract_spec in s\n\n\n@pytest.mark.parametrize(\n \"lhs,rhs,expected\", [(\"a\", \"a\", True), (\"a\", \"a@1.0\", True), (\"a@1.0\", \"a\", False)]\n)\ndef test_abstract_contains_semantic(lhs, rhs, expected, mock_packages):\n s, t = Spec(lhs), Spec(rhs)\n result = s in t\n assert result is expected\n\n\n@pytest.mark.parametrize(\n \"factory,lhs_str,rhs_str,results\",\n [\n # Architecture\n (ArchSpec, \"None-ubuntu20.04-None\", \"None-None-x86_64\", (True, False, False)),\n (ArchSpec, \"None-ubuntu20.04-None\", \"linux-None-x86_64\", (True, False, False)),\n (ArchSpec, \"None-None-x86_64:\", \"linux-None-haswell\", (True, False, True)),\n (ArchSpec, \"None-None-x86_64:haswell\", \"linux-None-icelake\", (False, False, False)),\n (ArchSpec, \"linux-None-None\", \"linux-None-None\", (True, True, True)),\n (ArchSpec, \"darwin-None-None\", \"linux-None-None\", (False, False, False)),\n (ArchSpec, \"None-ubuntu20.04-None\", \"None-ubuntu20.04-None\", (True, True, True)),\n (ArchSpec, \"None-ubuntu20.04-None\", \"None-ubuntu22.04-None\", (False, False, False)),\n # Compiler\n (Spec, \"gcc\", \"clang\", (False, False, False)),\n (Spec, \"gcc\", \"gcc@5\", (True, False, True)),\n (Spec, \"gcc@5\", \"gcc@5.3\", (True, False, True)),\n (Spec, \"gcc@5\", \"gcc@5-tag\", (True, False, True)),\n # Flags (flags are a map, so for convenience we initialize a full Spec)\n # Note: the semantic is that of sv variants, not mv variants\n (Spec, \"cppflags=-foo\", \"cppflags=-bar\", (True, False, False)),\n (Spec, \"cppflags='-bar -foo'\", \"cppflags=-bar\", (True, True, False)),\n (Spec, \"cppflags=-foo\", \"cppflags=-foo\", (True, True, True)),\n (Spec, \"cppflags=-foo\", \"cflags=-foo\", (True, False, False)),\n # Versions\n (Spec, \"@0.94h\", \"@:0.94i\", (True, True, False)),\n # Different virtuals intersect if there is at least package providing both\n (Spec, \"mpi\", \"lapack\", (True, False, False)),\n (Spec, \"mpi\", \"pkgconfig\", (False, False, False)),\n # Intersection among target ranges for different architectures\n (Spec, \"target=x86_64:\", \"target=ppc64le:\", (False, False, False)),\n (Spec, \"target=x86_64:\", \"target=:power9\", (False, False, False)),\n (Spec, \"target=:haswell\", \"target=:power9\", (False, False, False)),\n (Spec, \"target=:haswell\", \"target=ppc64le:\", (False, False, False)),\n # Intersection among target ranges for the same architecture\n (Spec, \"target=:haswell\", \"target=x86_64:\", (True, True, True)),\n (Spec, \"target=:haswell\", \"target=x86_64_v4:\", (False, False, False)),\n # Edge case of uarch that split in a diamond structure, from a common ancestor\n (Spec, \"target=:cascadelake\", \"target=:cannonlake\", (False, False, False)),\n # Spec with compilers\n (Spec, \"mpileaks %gcc@5\", \"mpileaks %gcc@6\", (False, False, False)),\n (Spec, \"mpileaks ^callpath %gcc@5\", \"mpileaks ^callpath %gcc@6\", (False, False, False)),\n (Spec, \"mpileaks ^callpath %gcc@5\", \"mpileaks ^callpath %gcc@5.4\", (True, False, True)),\n ],\n)\ndef test_intersects_and_satisfies(mock_packages, factory, lhs_str, rhs_str, results):\n lhs = factory(lhs_str)\n rhs = factory(rhs_str)\n\n intersects, lhs_satisfies_rhs, rhs_satisfies_lhs = results\n\n assert lhs.intersects(rhs) is intersects\n assert rhs.intersects(lhs) is lhs.intersects(rhs)\n\n assert lhs.satisfies(rhs) is lhs_satisfies_rhs\n assert rhs.satisfies(lhs) is rhs_satisfies_lhs\n\n\n@pytest.mark.parametrize(\n \"factory,lhs_str,rhs_str,result,constrained_str\",\n [\n # Architecture\n (ArchSpec, \"None-ubuntu20.04-None\", \"None-None-x86_64\", True, \"None-ubuntu20.04-x86_64\"),\n (ArchSpec, \"None-None-x86_64\", \"None-None-x86_64\", False, \"None-None-x86_64\"),\n (\n ArchSpec,\n \"None-None-x86_64:icelake\",\n \"None-None-x86_64:icelake\",\n False,\n \"None-None-x86_64:icelake\",\n ),\n (ArchSpec, \"None-ubuntu20.04-None\", \"linux-None-x86_64\", True, \"linux-ubuntu20.04-x86_64\"),\n (\n ArchSpec,\n \"None-ubuntu20.04-nocona:haswell\",\n \"None-None-x86_64:icelake\",\n False,\n \"None-ubuntu20.04-nocona:haswell\",\n ),\n (\n ArchSpec,\n \"None-ubuntu20.04-nocona,haswell\",\n \"None-None-x86_64:icelake\",\n False,\n \"None-ubuntu20.04-nocona,haswell\",\n ),\n # Compiler\n (Spec, \"foo %gcc@5\", \"foo %gcc@5-tag\", True, \"foo %gcc@5-tag\"),\n (Spec, \"foo %gcc@5\", \"foo %gcc@5\", False, \"foo %gcc@5\"),\n # Flags\n (Spec, \"cppflags=-foo\", \"cppflags=-foo\", False, \"cppflags=-foo\"),\n (Spec, \"cppflags=-foo\", \"cflags=-foo\", True, \"cppflags=-foo cflags=-foo\"),\n # Target ranges\n (Spec, \"target=x86_64:\", \"target=x86_64:\", False, \"target=x86_64:\"),\n (Spec, \"target=x86_64:\", \"target=:haswell\", True, \"target=x86_64:haswell\"),\n (\n Spec,\n \"target=x86_64:haswell\",\n \"target=x86_64_v2:icelake\",\n True,\n \"target=x86_64_v2:haswell\",\n ),\n ],\n)\ndef test_constrain(factory, lhs_str, rhs_str, result, constrained_str):\n lhs = factory(lhs_str)\n rhs = factory(rhs_str)\n\n assert lhs.constrain(rhs) is result\n assert lhs == factory(constrained_str)\n\n # The intersection must be the same, so check that invariant too\n lhs = factory(lhs_str)\n rhs = factory(rhs_str)\n rhs.constrain(lhs)\n assert rhs == factory(constrained_str)\n\n\ndef test_constrain_dependencies_copies(mock_packages):\n \"\"\"Tests that constraining a spec with new deps makes proper copies, and does not accidentally\n share dependency instances, leading to corruption of unrelated Spec instances.\"\"\"\n x = Spec(\"root\")\n y = Spec(\"^foo\")\n z = Spec(\"%foo +bar\")\n assert x.constrain(y)\n assert x == Spec(\"root ^foo\")\n assert x.constrain(z)\n assert x == Spec(\"root %foo +bar\")\n assert not x.constrain(Spec(\"root %foo +bar\")) # no new constraints\n # now, double check that we did not mutate `y` after constraining `x` with `z`.\n assert y == Spec(\"^foo\")\n\n\ndef test_abstract_hash_intersects_and_satisfies(default_mock_concretization):\n concrete: Spec = default_mock_concretization(\"pkg-a\")\n hash = concrete.dag_hash()\n hash_5 = hash[:5]\n hash_6 = hash[:6]\n # abstract hash that doesn't have a common prefix with the others.\n hash_other = f\"{'a' if hash_5[0] == 'b' else 'b'}{hash_5[1:]}\"\n\n abstract_5 = Spec(f\"pkg-a/{hash_5}\")\n abstract_6 = Spec(f\"pkg-a/{hash_6}\")\n abstract_none = Spec(f\"pkg-a/{hash_other}\")\n abstract = Spec(\"pkg-a\")\n\n def assert_subset(a: Spec, b: Spec):\n assert a.intersects(b) and b.intersects(a) and a.satisfies(b) and not b.satisfies(a)\n\n def assert_disjoint(a: Spec, b: Spec):\n assert (\n not a.intersects(b)\n and not b.intersects(a)\n and not a.satisfies(b)\n and not b.satisfies(a)\n )\n\n # left-hand side is more constrained, so its\n # concretization space is a subset of the right-hand side's\n assert_subset(concrete, abstract_5)\n assert_subset(abstract_6, abstract_5)\n assert_subset(abstract_5, abstract)\n\n # disjoint concretization space\n assert_disjoint(abstract_none, concrete)\n assert_disjoint(abstract_none, abstract_5)\n\n\ndef test_edge_equality_does_not_depend_on_virtual_order():\n \"\"\"Tests that two edges that are constructed with just a different order of the virtuals in\n the input parameters are equal to each other.\n \"\"\"\n parent, child = Spec(\"parent\"), Spec(\"child\")\n edge1 = DependencySpec(parent, child, depflag=0, virtuals=(\"mpi\", \"lapack\"))\n edge2 = DependencySpec(parent, child, depflag=0, virtuals=(\"lapack\", \"mpi\"))\n assert edge1 == edge2\n assert tuple(sorted(edge1.virtuals)) == edge1.virtuals\n assert tuple(sorted(edge2.virtuals)) == edge1.virtuals\n\n\ndef test_update_virtuals():\n parent, child = Spec(\"parent\"), Spec(\"child\")\n edge = DependencySpec(parent, child, depflag=0, virtuals=(\"mpi\", \"lapack\"))\n assert edge.update_virtuals(\"blas\")\n assert edge.virtuals == (\"blas\", \"lapack\", \"mpi\")\n assert edge.update_virtuals((\"c\", \"fortran\", \"mpi\", \"lapack\"))\n assert edge.virtuals == (\"blas\", \"c\", \"fortran\", \"lapack\", \"mpi\")\n assert not edge.update_virtuals(\"mpi\")\n assert not edge.update_virtuals((\"c\", \"fortran\", \"mpi\", \"lapack\"))\n assert edge.virtuals == (\"blas\", \"c\", \"fortran\", \"lapack\", \"mpi\")\n\n\ndef test_virtual_queries_work_for_strings_and_lists():\n \"\"\"Ensure that ``dependencies()`` works with both virtuals=str and virtuals=[str, ...].\"\"\"\n parent, child = Spec(\"parent\"), Spec(\"child\")\n parent._add_dependency(\n child,\n depflag=dt.BUILD,\n virtuals=(\"cxx\", \"fortran\"), # multi-char dep names\n )\n\n assert not parent.dependencies(virtuals=\"c\") # not in virtuals but shares a char with cxx\n\n for lang in [\"cxx\", \"fortran\"]:\n assert parent.dependencies(virtuals=lang) # string arg\n assert parent.edges_to_dependencies(virtuals=lang) # string arg\n\n assert parent.dependencies(virtuals=[lang]) # list arg\n assert parent.edges_to_dependencies(virtuals=[lang]) # string arg\n\n\ndef test_old_format_strings_trigger_error(default_mock_concretization):\n s = spack.concretize.concretize_one(\"pkg-a\")\n with pytest.raises(SpecFormatStringError):\n s.format(\"${PACKAGE}-${VERSION}-${HASH}\")\n\n\n@pytest.mark.regression(\"47362\")\n@pytest.mark.parametrize(\n \"lhs,rhs\",\n [\n (\"hdf5 +mpi\", \"hdf5++mpi\"),\n (\"hdf5 cflags==-g\", \"hdf5 cflags=-g\"),\n (\"hdf5 +mpi ++shared\", \"hdf5+mpi +shared\"),\n (\"hdf5 +mpi cflags==-g\", \"hdf5++mpi cflag=-g\"),\n ],\n)\ndef test_equality_discriminate_on_propagation(lhs, rhs):\n \"\"\"Tests that == can discriminate abstract specs based on their 'propagation' status\"\"\"\n s, t = Spec(lhs), Spec(rhs)\n assert s != t\n assert len({s, t}) == 2\n\n\ndef test_comparison_multivalued_variants():\n assert Spec(\"x=a\") < Spec(\"x=a,b\") < Spec(\"x==a,b\") < Spec(\"x==a,b,c\")\n\n\n@pytest.mark.parametrize(\n \"specs_in_expected_order\",\n [\n (\"a\", \"b\", \"c\", \"d\", \"e\"),\n (\"a@1.0\", \"a@2.0\", \"b\", \"c@3.0\", \"c@4.0\"),\n (\"a^d\", \"b^c\", \"c^b\", \"d^a\"),\n (\"e^a\", \"e^b\", \"e^c\", \"e^d\"),\n (\"e^a@1.0\", \"e^a@2.0\", \"e^a@3.0\", \"e^a@4.0\"),\n (\"e^a@1.0 +a\", \"e^a@1.0 +b\", \"e^a@1.0 +c\", \"e^a@1.0 +c\"),\n (\"a^b%c\", \"a^b%d\", \"a^b%e\", \"a^b%f\"),\n (\"a^b%c@1.0\", \"a^b%c@2.0\", \"a^b%c@3.0\", \"a^b%c@4.0\"),\n (\"a^b%c@1.0 +a\", \"a^b%c@1.0 +b\", \"a^b%c@1.0 +c\", \"a^b%c@1.0 +d\"),\n (\"a cflags=-O1\", \"a cflags=-O2\", \"a cflags=-O3\"),\n (\"a %cmake@1.0 ^b %cmake@2.0\", \"a %cmake@2.0 ^b %cmake@1.0\"),\n (\"a^b^c^d\", \"a^b^c^e\", \"a^b^c^f\"),\n (\"a^b^c^d\", \"a^b^c^e\", \"a^b^c^e\", \"a^b^c^f\"),\n (\"a%b%c%d\", \"a%b%c%e\", \"a%b%c%e\", \"a%b%c%f\"),\n (\"d.a\", \"c.b\", \"b.c\", \"a.d\"), # names before namespaces\n ],\n)\ndef test_spec_ordering(specs_in_expected_order):\n specs_in_expected_order = [Spec(s) for s in specs_in_expected_order]\n assert sorted(specs_in_expected_order) == specs_in_expected_order\n assert sorted(reversed(specs_in_expected_order)) == specs_in_expected_order\n\n for i in range(len(specs_in_expected_order) - 1):\n lhs, rhs = specs_in_expected_order[i : i + 2]\n assert lhs <= rhs\n assert (lhs < rhs and lhs != rhs) or lhs == rhs\n assert rhs >= lhs\n assert (rhs > lhs and rhs != lhs) or rhs == lhs\n\n\nEMPTY_VER = vn.VersionList(\":\")\nEMPTY_VAR = Spec().variants\nEMPTY_FLG = Spec().compiler_flags\n\n\n@pytest.mark.parametrize(\n \"spec,expected_tuplified\",\n [\n # simple, no dependencies\n [(\"a\"), (((\"a\", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None),), ())],\n # with some node attributes\n [\n (\"a@1.0 +foo cflags='-O3 -g'\"),\n (\n (\n (\n \"a\",\n None,\n vn.VersionList([\"1.0\"]),\n Spec(\"+foo\").variants,\n Spec(\"cflags='-O3 -g'\").compiler_flags,\n None,\n None,\n None,\n ),\n ),\n (),\n ),\n ],\n # single edge case\n [\n (\"a^b\"),\n (\n (\n (\"a\", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None),\n (\"b\", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None),\n ),\n ((0, 1, 0, (), False, Spec()),),\n ),\n ],\n # root with multiple deps\n [\n (\"a^b^c^d\"),\n (\n (\n (\"a\", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None),\n (\"b\", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None),\n (\"c\", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None),\n (\"d\", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None),\n ),\n (\n (0, 1, 0, (), False, Spec()),\n (0, 2, 0, (), False, Spec()),\n (0, 3, 0, (), False, Spec()),\n ),\n ),\n ],\n # root with multiple build deps\n [\n (\"a%b%c%d\"),\n (\n (\n (\"a\", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None),\n (\"b\", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None),\n (\"c\", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None),\n (\"d\", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None),\n ),\n (\n (0, 1, 0, (), True, Spec()),\n (0, 2, 0, (), True, Spec()),\n (0, 3, 0, (), True, Spec()),\n ),\n ),\n ],\n # dependencies with dependencies\n [\n (\"a ^b %c %d ^e %f %g\"),\n (\n (\n (\"a\", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None),\n (\"b\", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None),\n (\"e\", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None),\n (\"c\", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None),\n (\"d\", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None),\n (\"f\", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None),\n (\"g\", None, EMPTY_VER, EMPTY_VAR, EMPTY_FLG, None, None, None),\n ),\n (\n (0, 1, 0, (), False, Spec()),\n (0, 2, 0, (), False, Spec()),\n (1, 3, 0, (), True, Spec()),\n (1, 4, 0, (), True, Spec()),\n (2, 5, 0, (), True, Spec()),\n (2, 6, 0, (), True, Spec()),\n ),\n ),\n ],\n ],\n)\ndef test_spec_canonical_comparison_form(spec, expected_tuplified):\n \"\"\"Tests a few expected canonical comparison form of specs\"\"\"\n assert spack.llnl.util.lang.tuplify(Spec(spec)._cmp_iter) == expected_tuplified\n\n\ndef test_comparison_after_breaking_hash_change():\n # We simulate a breaking change in DAG hash computation in Spack. We have two specs that are\n # entirely equal modulo DAG hash. When deserializing these specs, we don't want them to compare\n # as equal, because DAG hash is used throughout in Spack to distinguish between specs\n # (e.g. database, build caches, install dir).\n s = Spec(\"example@=1.0\")\n s._mark_concrete(True)\n\n # compute the dag hash and a change to it\n dag_hash = s.dag_hash()\n new_dag_hash = f\"{'b' if dag_hash[0] == 'a' else 'a'}{dag_hash[1:]}\"\n\n before_breakage = s.to_dict()\n after_breakage = s.to_dict()\n after_breakage[\"spec\"][\"nodes\"][0][\"hash\"] = new_dag_hash\n assert before_breakage != after_breakage\n\n x = Spec.from_dict(before_breakage)\n y = Spec.from_dict(after_breakage)\n assert x != y\n assert len({x, y}) == 2\n\n\ndef test_satisfies_and_subscript_with_compilers(default_mock_concretization):\n \"\"\"Tests the semantic of \"satisfies\" and __getitem__ for the following spec:\n\n [ ] multivalue-variant@2.3\n [bl ] ^callpath@1.0\n [bl ] ^dyninst@8.2\n [bl ] ^libdwarf@20130729\n [bl ] ^libelf@0.8.13\n [b ] ^gcc@10.2.1\n [ l ] ^gcc-runtime@10.2.1\n [bl ] ^mpich@3.0.4\n [bl ] ^pkg-a@2.0\n [b ] ^gmake@4.4\n [bl ] ^pkg-b@1.0\n \"\"\"\n s = default_mock_concretization(\"multivalue-variant\")\n\n # Check a direct build/link dependency\n assert s.satisfies(\"^pkg-a\")\n assert s.dependencies(name=\"pkg-a\")[0] == s[\"pkg-a\"]\n\n # Transitive build/link dependency\n assert s.satisfies(\"^libelf\")\n assert s[\"libdwarf\"].dependencies(name=\"libelf\")[0] == s[\"libelf\"]\n\n # Direct build dependencies\n assert s.satisfies(\"^[virtuals=c] gcc\")\n assert s.satisfies(\"%[virtuals=c] gcc\")\n assert s.dependencies(name=\"gcc\")[0] == s[\"gcc\"]\n assert s.dependencies(name=\"gcc\")[0] == s[\"c\"]\n\n # Transitive build dependencies\n assert not s.satisfies(\"^gmake\")\n\n # \"gmake\" is not in the link/run subdag + direct build deps\n with pytest.raises(KeyError):\n _ = s[\"gmake\"]\n\n # We need to pass through \"pkg-a\" to get \"gmake\" with [] notation\n assert s[\"pkg-a\"].dependencies(name=\"gmake\")[0] == s[\"pkg-a\"][\"gmake\"]\n\n\n@pytest.mark.parametrize(\n \"spec_str,spec_fmt,expected\",\n [\n # Depends on C\n (\"mpileaks\", \"{name}-{compiler.name}\", \"mpileaks-gcc\"),\n (\"mpileaks\", \"{name}-{compiler.name}-{compiler.version}\", \"mpileaks-gcc-10.2.1\"),\n # No compiler\n (\"pkg-c\", \"{name}-{compiler.name}\", \"pkg-c-none\"),\n (\"pkg-c\", \"{name}-{compiler.name}-{compiler.version}\", \"pkg-c-none-none\"),\n ],\n)\ndef test_spec_format_with_compiler_adaptors(\n spec_str, spec_fmt, expected, default_mock_concretization\n):\n \"\"\"Tests the output of spec format, when involving `Spec.compiler` adaptors\"\"\"\n s = default_mock_concretization(spec_str)\n assert s.format(spec_fmt) == expected\n\n\n@pytest.mark.parametrize(\n \"lhs,rhs,expected\",\n [\n (\"mpich %gcc\", \"mpich %gcc\", True),\n (\"mpich %gcc\", \"mpich ^gcc\", False),\n (\"mpich ^callpath %gcc\", \"mpich %gcc ^callpath\", False),\n ],\n)\ndef test_specs_equality(lhs, rhs, expected):\n \"\"\"Tests the semantic of == for abstract specs\"\"\"\n lhs, rhs = Spec(lhs), Spec(rhs)\n assert (lhs == rhs) is expected\n\n\ndef test_edge_equality_accounts_for_when_condition():\n \"\"\"Tests that edges can be distinguished by their 'when' condition.\"\"\"\n parent, child = Spec(\"parent\"), Spec(\"child\")\n edge1 = DependencySpec(parent, child, depflag=0, virtuals=(), when=Spec(\"%c\"))\n edge2 = DependencySpec(parent, child, depflag=0, virtuals=())\n assert edge1 != edge2\n\n\ndef test_long_spec():\n \"\"\"Test that long_spec preserves dependency types and has correct ordering.\"\"\"\n assert Spec(\"foo %m %l ^k %n %j\").long_spec == \"foo %l %m ^k %j %n\"\n\n\n@pytest.mark.parametrize(\n \"constraints,expected\",\n [\n # Anonymous specs without dependencies\n ([\"+baz\", \"+bar\"], \"+baz+bar\"),\n ([\"@2.0:\", \"@:5.1\", \"+bar\"], \"@2.0:5.1 +bar\"),\n # Anonymous specs with dependencies\n ([\"^mpich@3.2\", \"^mpich@:4.0+foo\"], \"^mpich@3.2 +foo\"),\n # Mix a real package with a virtual one. This test\n # should fail if we start using the repository\n ([\"^mpich@3.2\", \"^mpi+foo\"], \"^mpich@3.2 ^mpi+foo\"),\n # Non direct dependencies + direct dependencies\n ([\"^mpich\", \"%mpich\"], \"%mpich\"),\n ([\"^foo\", \"^bar %foo\"], \"^foo ^bar%foo\"),\n ([\"^foo\", \"%bar %foo\"], \"%bar%foo\"),\n ],\n)\ndef test_constrain_symbolically(constraints, expected):\n \"\"\"Tests the semantics of constraining a spec when we don't resolve virtuals.\"\"\"\n merged = Spec()\n for c in constraints:\n merged._constrain_symbolically(c)\n assert merged == Spec(expected)\n\n reverse_order = Spec()\n for c in reversed(constraints):\n reverse_order._constrain_symbolically(c)\n assert reverse_order == Spec(expected)\n\n\n@pytest.mark.parametrize(\n \"parent_str,child_str,kwargs,expected_str,expected_repr\",\n [\n (\n \"mpileaks\",\n \"callpath\",\n {\"virtuals\": ()},\n \"mpileaks ^callpath\",\n \"DependencySpec('mpileaks', 'callpath', depflag=0, virtuals=())\",\n ),\n (\n \"mpileaks\",\n \"callpath\",\n {\"virtuals\": (\"mpi\", \"lapack\")},\n \"mpileaks ^[virtuals=lapack,mpi] callpath\",\n \"DependencySpec('mpileaks', 'callpath', depflag=0, virtuals=('lapack', 'mpi'))\",\n ),\n (\n \"\",\n \"callpath\",\n {\"virtuals\": (\"mpi\", \"lapack\"), \"direct\": True},\n \" %[virtuals=lapack,mpi] callpath\",\n \"DependencySpec('', 'callpath', depflag=0, virtuals=('lapack', 'mpi'), direct=True)\",\n ),\n (\n \"\",\n \"callpath\",\n {\n \"virtuals\": (\"mpi\", \"lapack\"),\n \"direct\": True,\n \"propagation\": PropagationPolicy.PREFERENCE,\n },\n \" %%[virtuals=lapack,mpi] callpath\",\n \"DependencySpec('', 'callpath', depflag=0, virtuals=('lapack', 'mpi'), direct=True,\"\n \" propagation=PropagationPolicy.PREFERENCE)\",\n ),\n (\n \"\",\n \"callpath\",\n {\"virtuals\": (), \"direct\": True, \"propagation\": PropagationPolicy.PREFERENCE},\n \" %%callpath\",\n \"DependencySpec('', 'callpath', depflag=0, virtuals=(), direct=True,\"\n \" propagation=PropagationPolicy.PREFERENCE)\",\n ),\n (\n \"mpileaks+foo\",\n \"callpath+bar\",\n {\"virtuals\": (), \"direct\": True, \"propagation\": PropagationPolicy.PREFERENCE},\n \"mpileaks+foo %%callpath+bar\",\n \"DependencySpec('mpileaks+foo', 'callpath+bar', depflag=0, virtuals=(), direct=True,\"\n \" propagation=PropagationPolicy.PREFERENCE)\",\n ),\n ],\n)\ndef test_edge_representation(parent_str, child_str, kwargs, expected_str, expected_repr):\n \"\"\"Tests the string representations of edges.\"\"\"\n parent = Spec(parent_str) or Spec()\n child = Spec(child_str) or Spec()\n edge = DependencySpec(parent, child, depflag=0, **kwargs)\n assert str(edge) == expected_str\n assert repr(edge) == expected_repr\n\n\n@pytest.mark.parametrize(\n \"spec_str,assertions\",\n [\n # Check =* semantics for a \"regular\" variant\n (\"mpileaks foo=abc\", [(\"foo=*\", True), (\"bar=*\", False)]),\n # Check the semantics for architecture related key value pairs\n (\n \"mpileaks\",\n [\n (\"target=*\", False),\n (\"os=*\", False),\n (\"platform=*\", False),\n (\"target=* platform=*\", False),\n ],\n ),\n (\n \"mpileaks target=x86_64\",\n [\n (\"target=*\", True),\n (\"os=*\", False),\n (\"platform=*\", False),\n (\"target=* platform=*\", False),\n ],\n ),\n (\"mpileaks os=debian6\", [(\"target=*\", False), (\"os=*\", True), (\"platform=*\", False)]),\n (\"mpileaks platform=linux\", [(\"target=*\", False), (\"os=*\", False), (\"platform=*\", True)]),\n (\"mpileaks platform=linux\", [(\"target=*\", False), (\"os=*\", False), (\"platform=*\", True)]),\n (\n \"mpileaks platform=linux target=x86_64\",\n [\n (\"target=*\", True),\n (\"os=*\", False),\n (\"platform=*\", True),\n (\"target=* platform=*\", True),\n ],\n ),\n ],\n)\ndef test_attribute_existence_in_satisfies(spec_str, assertions, mock_packages, config):\n \"\"\"Tests the semantics of =* when used in Spec.satisfies\"\"\"\n s = Spec(spec_str)\n for test, expected in assertions:\n assert s.satisfies(test) is expected\n\n\n@pytest.mark.regression(\"51768\")\n@pytest.mark.parametrize(\"spec_str\", [\"mpi\", \"%mpi\", \"^mpi\", \"%foo\", \"%c=gcc\", \"%[when=%c]c=gcc\"])\ndef test_specs_semantics_on_self(spec_str, mock_packages, config):\n \"\"\"Tests that an abstract spec satisfies and intersects with itself.\"\"\"\n s = Spec(spec_str)\n assert s.satisfies(s)\n assert s.intersects(s)\n\n\n@pytest.mark.parametrize(\n \"spec_str,expected_fmt\",\n [\n (\"mpileaks@2.2\", \"mpileaks@_R{@=2.2}\"),\n (\"mpileaks@2.3\", \"mpileaks@c{@=2.3}\"),\n (\"mpileaks+debug\", \"@_R{+debug}\"),\n ],\n)\ndef test_highlighting_spec_parts(spec_str, expected_fmt, default_mock_concretization):\n \"\"\"Tests correct highlighting of non-default versions and variants\"\"\"\n s = default_mock_concretization(spec_str)\n expected = colorize(expected_fmt, color=True)\n colorized_str = s.format(\n color=True,\n highlight_version_fn=spack.package_base.non_preferred_version,\n highlight_variant_fn=spack.package_base.non_default_variant,\n )\n assert expected in colorized_str\n" }, { "path": "lib/spack/spack/test/spec_syntax.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport itertools\nimport os\nimport pathlib\nimport re\nimport sys\n\nimport pytest\n\nimport spack.binary_distribution\nimport spack.cmd\nimport spack.concretize\nimport spack.error\nimport spack.llnl.util.filesystem as fs\nimport spack.platforms.test\nimport spack.repo\nimport spack.solver.asp\nimport spack.spec\nfrom spack.spec_parser import (\n UNIX_FILENAME,\n WINDOWS_FILENAME,\n SpecParser,\n SpecParsingError,\n SpecTokenizationError,\n SpecTokens,\n expand_toolchains,\n parse_one_or_raise,\n)\nfrom spack.tokenize import Token\n\nSKIP_ON_WINDOWS = pytest.mark.skipif(sys.platform == \"win32\", reason=\"Unix style path on Windows\")\n\nSKIP_ON_UNIX = pytest.mark.skipif(sys.platform != \"win32\", reason=\"Windows style path on Unix\")\n\n\ndef simple_package_name(name):\n \"\"\"A simple package name in canonical form\"\"\"\n return name, [Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=name)], name\n\n\ndef dependency_with_version(text):\n root, rest = text.split(\"^\")\n dependency, version = rest.split(\"@\")\n return (\n text,\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=root.strip()),\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=dependency.strip()),\n Token(SpecTokens.VERSION, value=f\"@{version}\"),\n ],\n text,\n )\n\n\n@pytest.fixture()\ndef specfile_for(default_mock_concretization):\n def _specfile_for(spec_str, filename):\n s = default_mock_concretization(spec_str)\n is_json = str(filename).endswith(\".json\")\n is_yaml = str(filename).endswith(\".yaml\")\n if not is_json and not is_yaml:\n raise ValueError(\"wrong extension used for specfile\")\n\n with filename.open(\"w\") as f:\n if is_json:\n f.write(s.to_json())\n else:\n f.write(s.to_yaml())\n return s\n\n return _specfile_for\n\n\n@pytest.mark.parametrize(\n \"spec_str,tokens,expected_roundtrip\",\n [\n # Package names\n simple_package_name(\"mvapich\"),\n simple_package_name(\"mvapich_foo\"),\n simple_package_name(\"_mvapich_foo\"),\n simple_package_name(\"3dtk\"),\n simple_package_name(\"ns-3-dev\"),\n # Single token anonymous specs\n (\"@2.7\", [Token(SpecTokens.VERSION, value=\"@2.7\")], \"@2.7\"),\n (\"@2.7:\", [Token(SpecTokens.VERSION, value=\"@2.7:\")], \"@2.7:\"),\n (\"@:2.7\", [Token(SpecTokens.VERSION, value=\"@:2.7\")], \"@:2.7\"),\n (\"+foo\", [Token(SpecTokens.BOOL_VARIANT, value=\"+foo\")], \"+foo\"),\n (\"~foo\", [Token(SpecTokens.BOOL_VARIANT, value=\"~foo\")], \"~foo\"),\n (\"-foo\", [Token(SpecTokens.BOOL_VARIANT, value=\"-foo\")], \"~foo\"),\n (\n \"platform=test\",\n [Token(SpecTokens.KEY_VALUE_PAIR, value=\"platform=test\")],\n \"platform=test\",\n ),\n # Multiple tokens anonymous specs\n (\n \"%intel\",\n [\n Token(SpecTokens.DEPENDENCY, value=\"%\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, \"intel\"),\n ],\n \"%intel\",\n ),\n (\n \"languages=go @4.2:\",\n [\n Token(SpecTokens.KEY_VALUE_PAIR, value=\"languages=go\"),\n Token(SpecTokens.VERSION, value=\"@4.2:\"),\n ],\n \"@4.2: languages=go\",\n ),\n (\n \"@4.2: languages=go\",\n [\n Token(SpecTokens.VERSION, value=\"@4.2:\"),\n Token(SpecTokens.KEY_VALUE_PAIR, value=\"languages=go\"),\n ],\n \"@4.2: languages=go\",\n ),\n (\n \"^zlib\",\n [\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"zlib\"),\n ],\n \"^zlib\",\n ),\n # Specs with simple dependencies\n (\n \"openmpi ^hwloc\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"openmpi\"),\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"hwloc\"),\n ],\n \"openmpi ^hwloc\",\n ),\n (\n \"openmpi ^hwloc ^libunwind\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"openmpi\"),\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"hwloc\"),\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"libunwind\"),\n ],\n \"openmpi ^hwloc ^libunwind\",\n ),\n (\n \"openmpi ^hwloc^libunwind\",\n [ # White spaces are tested\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"openmpi\"),\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"hwloc\"),\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"libunwind\"),\n ],\n \"openmpi ^hwloc ^libunwind\",\n ),\n # Version after compiler\n (\n \"foo @2.0 %bar@1.0\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"foo\"),\n Token(SpecTokens.VERSION, value=\"@2.0\"),\n Token(SpecTokens.DEPENDENCY, value=\"%\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"bar\"),\n Token(SpecTokens.VERSION, value=\"@1.0\"),\n ],\n \"foo@2.0 %bar@1.0\",\n ),\n # Single dependency with version\n dependency_with_version(\"openmpi ^hwloc@1.2e6\"),\n dependency_with_version(\"openmpi ^hwloc@1.2e6:\"),\n dependency_with_version(\"openmpi ^hwloc@:1.4b7-rc3\"),\n dependency_with_version(\"openmpi ^hwloc@1.2e6:1.4b7-rc3\"),\n # Complex specs with multiple constraints\n (\n \"mvapich_foo ^_openmpi@1.2:1.4,1.6+debug~qt_4 %intel@12.1 ^stackwalker@8.1_1e\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"mvapich_foo\"),\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"_openmpi\"),\n Token(SpecTokens.VERSION, value=\"@1.2:1.4,1.6\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"+debug\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"~qt_4\"),\n Token(SpecTokens.DEPENDENCY, value=\"%\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"intel\"),\n Token(SpecTokens.VERSION, value=\"@12.1\"),\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"stackwalker\"),\n Token(SpecTokens.VERSION, value=\"@8.1_1e\"),\n ],\n \"mvapich_foo ^_openmpi@1.2:1.4,1.6+debug~qt_4 %intel@12.1 ^stackwalker@8.1_1e\",\n ),\n (\n \"mvapich_foo ^_openmpi@1.2:1.4,1.6~qt_4 debug=2 %intel@12.1 ^stackwalker@8.1_1e\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"mvapich_foo\"),\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"_openmpi\"),\n Token(SpecTokens.VERSION, value=\"@1.2:1.4,1.6\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"~qt_4\"),\n Token(SpecTokens.KEY_VALUE_PAIR, value=\"debug=2\"),\n Token(SpecTokens.DEPENDENCY, value=\"%\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"intel\"),\n Token(SpecTokens.VERSION, value=\"@12.1\"),\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"stackwalker\"),\n Token(SpecTokens.VERSION, value=\"@8.1_1e\"),\n ],\n \"mvapich_foo ^_openmpi@1.2:1.4,1.6~qt_4 debug=2 %intel@12.1 ^stackwalker@8.1_1e\",\n ),\n (\n \"mvapich_foo ^_openmpi@1.2:1.4,1.6 cppflags=-O3 +debug~qt_4 %intel@12.1 \"\n \"^stackwalker@8.1_1e\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"mvapich_foo\"),\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"_openmpi\"),\n Token(SpecTokens.VERSION, value=\"@1.2:1.4,1.6\"),\n Token(SpecTokens.KEY_VALUE_PAIR, value=\"cppflags=-O3\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"+debug\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"~qt_4\"),\n Token(SpecTokens.DEPENDENCY, value=\"%\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"intel\"),\n Token(SpecTokens.VERSION, value=\"@12.1\"),\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"stackwalker\"),\n Token(SpecTokens.VERSION, value=\"@8.1_1e\"),\n ],\n \"mvapich_foo ^_openmpi@1.2:1.4,1.6 cppflags=-O3 +debug~qt_4 %intel@12.1\"\n \" ^stackwalker@8.1_1e\",\n ),\n # Specs containing YAML or JSON in the package name\n (\n \"yaml-cpp@0.1.8%intel@12.1 ^boost@3.1.4\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"yaml-cpp\"),\n Token(SpecTokens.VERSION, value=\"@0.1.8\"),\n Token(SpecTokens.DEPENDENCY, value=\"%\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"intel\"),\n Token(SpecTokens.VERSION, value=\"@12.1\"),\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"boost\"),\n Token(SpecTokens.VERSION, value=\"@3.1.4\"),\n ],\n \"yaml-cpp@0.1.8 %intel@12.1 ^boost@3.1.4\",\n ),\n (\n r\"builtin.yaml-cpp%gcc\",\n [\n Token(SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME, value=\"builtin.yaml-cpp\"),\n Token(SpecTokens.DEPENDENCY, value=\"%\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"gcc\"),\n ],\n \"yaml-cpp %gcc\",\n ),\n (\n r\"testrepo.yaml-cpp%gcc\",\n [\n Token(SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME, value=\"testrepo.yaml-cpp\"),\n Token(SpecTokens.DEPENDENCY, value=\"%\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"gcc\"),\n ],\n \"yaml-cpp %gcc\",\n ),\n (\n r\"builtin.yaml-cpp@0.1.8%gcc@7.2.0 ^boost@3.1.4\",\n [\n Token(SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME, value=\"builtin.yaml-cpp\"),\n Token(SpecTokens.VERSION, value=\"@0.1.8\"),\n Token(SpecTokens.DEPENDENCY, value=\"%\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"gcc\"),\n Token(SpecTokens.VERSION, value=\"@7.2.0\"),\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"boost\"),\n Token(SpecTokens.VERSION, value=\"@3.1.4\"),\n ],\n \"yaml-cpp@0.1.8 %gcc@7.2.0 ^boost@3.1.4\",\n ),\n (\n r\"builtin.yaml-cpp ^testrepo.boost ^zlib\",\n [\n Token(SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME, value=\"builtin.yaml-cpp\"),\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME, value=\"testrepo.boost\"),\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"zlib\"),\n ],\n \"yaml-cpp ^boost ^zlib\",\n ),\n # Canonicalization of the string representation\n (\n r\"mvapich ^stackwalker ^_openmpi\", # Dependencies are reordered\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"mvapich\"),\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"stackwalker\"),\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"_openmpi\"),\n ],\n \"mvapich ^_openmpi ^stackwalker\",\n ),\n (\n r\"y~f+e~d+c~b+a\", # Variants are reordered\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"y\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"~f\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"+e\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"~d\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"+c\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"~b\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"+a\"),\n ],\n \"y+a~b+c~d+e~f\",\n ),\n # Things that evaluate to Spec()\n # TODO: consider making these format to \"*\" instead of \"\"\n (\"@:\", [Token(SpecTokens.VERSION, value=\"@:\")], r\"\"),\n (\"*\", [Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"*\")], r\"\"),\n # virtual assignment on a dep of an anonymous spec (more of these later)\n (\n \"%foo=bar\",\n [Token(SpecTokens.DEPENDENCY, value=\"%foo=bar\", virtuals=\"foo\", substitute=\"bar\")],\n \"%foo=bar\",\n ),\n (\n \"^foo=bar\",\n [Token(SpecTokens.DEPENDENCY, value=\"^foo=bar\", virtuals=\"foo\", substitute=\"bar\")],\n \"^foo=bar\",\n ),\n # anonymous dependencies with variants\n (\n \"^*foo=bar\",\n [\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"*\"),\n Token(SpecTokens.KEY_VALUE_PAIR, value=\"foo=bar\"),\n ],\n \"^*foo=bar\",\n ),\n (\n \"%*foo=bar\",\n [\n Token(SpecTokens.DEPENDENCY, value=\"%\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"*\"),\n Token(SpecTokens.KEY_VALUE_PAIR, value=\"foo=bar\"),\n ],\n \"%*foo=bar\",\n ),\n (\n \"^*+foo\",\n [\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"*\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"+foo\"),\n ],\n \"^+foo\",\n ),\n (\n \"^*~foo\",\n [\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"*\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"~foo\"),\n ],\n \"^~foo\",\n ),\n (\n \"%*+foo\",\n [\n Token(SpecTokens.DEPENDENCY, value=\"%\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"*\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"+foo\"),\n ],\n \"%+foo\",\n ),\n (\n \"%*~foo\",\n [\n Token(SpecTokens.DEPENDENCY, value=\"%\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"*\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"~foo\"),\n ],\n \"%~foo\",\n ),\n # version range and list\n (\"@1.6,1.2:1.4\", [Token(SpecTokens.VERSION, value=\"@1.6,1.2:1.4\")], r\"@1.2:1.4,1.6\"),\n (\n r\"os=fe\", # Various translations associated with the architecture\n [Token(SpecTokens.KEY_VALUE_PAIR, value=\"os=fe\")],\n \"platform=test os=debian6\",\n ),\n (\n r\"os=default_os\",\n [Token(SpecTokens.KEY_VALUE_PAIR, value=\"os=default_os\")],\n \"platform=test os=debian6\",\n ),\n (\n r\"target=be\",\n [Token(SpecTokens.KEY_VALUE_PAIR, value=\"target=be\")],\n f\"platform=test target={spack.platforms.test.Test.default}\",\n ),\n (\n r\"target=default_target\",\n [Token(SpecTokens.KEY_VALUE_PAIR, value=\"target=default_target\")],\n f\"platform=test target={spack.platforms.test.Test.default}\",\n ),\n (\n r\"platform=linux\",\n [Token(SpecTokens.KEY_VALUE_PAIR, value=\"platform=linux\")],\n r\"platform=linux\",\n ),\n # Version hash pair\n (\n rf\"develop-branch-version@{'abc12' * 8}=develop\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"develop-branch-version\"),\n Token(SpecTokens.VERSION_HASH_PAIR, value=f\"@{'abc12' * 8}=develop\"),\n ],\n rf\"develop-branch-version@{'abc12' * 8}=develop\",\n ),\n # Redundant specs\n (\n r\"x ^y@foo ^y@foo\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"x\"),\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"y\"),\n Token(SpecTokens.VERSION, value=\"@foo\"),\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"y\"),\n Token(SpecTokens.VERSION, value=\"@foo\"),\n ],\n r\"x ^y@foo\",\n ),\n (\n r\"x ^y@foo ^y+bar\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"x\"),\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"y\"),\n Token(SpecTokens.VERSION, value=\"@foo\"),\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"y\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"+bar\"),\n ],\n r\"x ^y@foo+bar\",\n ),\n (\n r\"x ^y@foo +bar ^y@foo\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"x\"),\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"y\"),\n Token(SpecTokens.VERSION, value=\"@foo\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"+bar\"),\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"y\"),\n Token(SpecTokens.VERSION, value=\"@foo\"),\n ],\n r\"x ^y@foo+bar\",\n ),\n # Ambiguous variant specification\n (\n r\"_openmpi +debug-qt_4\", # Parse as a single bool variant\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"_openmpi\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"+debug-qt_4\"),\n ],\n r\"_openmpi+debug-qt_4\",\n ),\n (\n r\"_openmpi +debug -qt_4\", # Parse as two variants\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"_openmpi\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"+debug\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"-qt_4\"),\n ],\n r\"_openmpi+debug~qt_4\",\n ),\n (\n r\"_openmpi +debug~qt_4\", # Parse as two variants\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"_openmpi\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"+debug\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"~qt_4\"),\n ],\n r\"_openmpi+debug~qt_4\",\n ),\n # Key value pairs with \":\" and \",\" in the value\n (\n r\"target=:broadwell,icelake\",\n [Token(SpecTokens.KEY_VALUE_PAIR, value=\"target=:broadwell,icelake\")],\n r\"target=:broadwell,icelake\",\n ),\n # Hash pair version followed by a variant\n (\n f\"develop-branch-version@git.{'a' * 40}=develop+var1+var2\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"develop-branch-version\"),\n Token(SpecTokens.VERSION_HASH_PAIR, value=f\"@git.{'a' * 40}=develop\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"+var1\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"+var2\"),\n ],\n f\"develop-branch-version@git.{'a' * 40}=develop+var1+var2\",\n ),\n # Compiler with version ranges\n (\n \"%gcc@10.2.1:\",\n [\n Token(SpecTokens.DEPENDENCY, value=\"%\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"gcc\"),\n Token(SpecTokens.VERSION, value=\"@10.2.1:\"),\n ],\n \"%gcc@10.2.1:\",\n ),\n (\n \"%gcc@:10.2.1\",\n [\n Token(SpecTokens.DEPENDENCY, value=\"%\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"gcc\"),\n Token(SpecTokens.VERSION, value=\"@:10.2.1\"),\n ],\n \"%gcc@:10.2.1\",\n ),\n (\n \"%gcc@10.2.1:12.1.0\",\n [\n Token(SpecTokens.DEPENDENCY, value=\"%\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"gcc\"),\n Token(SpecTokens.VERSION, value=\"@10.2.1:12.1.0\"),\n ],\n \"%gcc@10.2.1:12.1.0\",\n ),\n (\n \"%gcc@10.1.0,12.2.1:\",\n [\n Token(SpecTokens.DEPENDENCY, value=\"%\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"gcc\"),\n Token(SpecTokens.VERSION, value=\"@10.1.0,12.2.1:\"),\n ],\n \"%gcc@10.1.0,12.2.1:\",\n ),\n (\n \"%gcc@:8.4.3,10.2.1:12.1.0\",\n [\n Token(SpecTokens.DEPENDENCY, value=\"%\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"gcc\"),\n Token(SpecTokens.VERSION, value=\"@:8.4.3,10.2.1:12.1.0\"),\n ],\n \"%gcc@:8.4.3,10.2.1:12.1.0\",\n ),\n # Special key value arguments\n (\"dev_path=*\", [Token(SpecTokens.KEY_VALUE_PAIR, value=\"dev_path=*\")], \"dev_path='*'\"),\n (\n \"dev_path=none\",\n [Token(SpecTokens.KEY_VALUE_PAIR, value=\"dev_path=none\")],\n \"dev_path=none\",\n ),\n (\n \"dev_path=../relpath/work\",\n [Token(SpecTokens.KEY_VALUE_PAIR, value=\"dev_path=../relpath/work\")],\n \"dev_path=../relpath/work\",\n ),\n (\n \"dev_path=/abspath/work\",\n [Token(SpecTokens.KEY_VALUE_PAIR, value=\"dev_path=/abspath/work\")],\n \"dev_path=/abspath/work\",\n ),\n # One liner for flags like 'a=b=c' that are injected\n (\n \"cflags=a=b=c\",\n [Token(SpecTokens.KEY_VALUE_PAIR, value=\"cflags=a=b=c\")],\n \"cflags='a=b=c'\",\n ),\n (\n \"cflags=a=b=c\",\n [Token(SpecTokens.KEY_VALUE_PAIR, value=\"cflags=a=b=c\")],\n \"cflags='a=b=c'\",\n ),\n (\n \"cflags=a=b=c+~\",\n [Token(SpecTokens.KEY_VALUE_PAIR, value=\"cflags=a=b=c+~\")],\n \"cflags='a=b=c+~'\",\n ),\n (\n \"cflags=-Wl,a,b,c\",\n [Token(SpecTokens.KEY_VALUE_PAIR, value=\"cflags=-Wl,a,b,c\")],\n \"cflags=-Wl,a,b,c\",\n ),\n # Multi quoted\n (\n 'cflags==\"-O3 -g\"',\n [Token(SpecTokens.PROPAGATED_KEY_VALUE_PAIR, value='cflags==\"-O3 -g\"')],\n \"cflags=='-O3 -g'\",\n ),\n # Whitespace is allowed in version lists\n (\"@1.2:1.4 , 1.6 \", [Token(SpecTokens.VERSION, value=\"@1.2:1.4 , 1.6\")], \"@1.2:1.4,1.6\"),\n # But not in ranges. `a@1:` and `b` are separate specs, not a single `a@1:b`.\n (\n \"a@1: b\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"a\"),\n Token(SpecTokens.VERSION, value=\"@1:\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"b\"),\n ],\n \"a@1:\",\n ),\n (\n \"+ debug % intel @ 12.1:12.6\",\n [\n Token(SpecTokens.BOOL_VARIANT, value=\"+ debug\"),\n Token(SpecTokens.DEPENDENCY, value=\"%\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"intel\"),\n Token(SpecTokens.VERSION, value=\"@ 12.1:12.6\"),\n ],\n \"+debug %intel@12.1:12.6\",\n ),\n (\n \"@ 12.1:12.6 + debug - qt_4\",\n [\n Token(SpecTokens.VERSION, value=\"@ 12.1:12.6\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"+ debug\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"- qt_4\"),\n ],\n \"@12.1:12.6+debug~qt_4\",\n ),\n (\n \"@10.4.0:10,11.3.0:target=aarch64:\",\n [\n Token(SpecTokens.VERSION, value=\"@10.4.0:10,11.3.0:\"),\n Token(SpecTokens.KEY_VALUE_PAIR, value=\"target=aarch64:\"),\n ],\n \"@10.4.0:10,11.3.0: target=aarch64:\",\n ),\n (\n \"@:0.4 % nvhpc\",\n [\n Token(SpecTokens.VERSION, value=\"@:0.4\"),\n Token(SpecTokens.DEPENDENCY, value=\"%\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"nvhpc\"),\n ],\n \"@:0.4 %nvhpc\",\n ),\n (\n \"^[virtuals=mpi] openmpi\",\n [\n Token(SpecTokens.START_EDGE_PROPERTIES, value=\"^[\"),\n Token(SpecTokens.KEY_VALUE_PAIR, value=\"virtuals=mpi\"),\n Token(SpecTokens.END_EDGE_PROPERTIES, value=\"]\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"openmpi\"),\n ],\n \"^mpi=openmpi\",\n ),\n (\n \"^mpi=openmpi\",\n [\n Token(\n SpecTokens.DEPENDENCY,\n value=\"^mpi=openmpi\",\n virtuals=\"mpi\",\n substitute=\"openmpi\",\n )\n ],\n \"^mpi=openmpi\",\n ),\n # Allow merging attributes, if deptypes match\n (\n \"^[virtuals=mpi] openmpi+foo ^[virtuals=lapack] openmpi+bar\",\n [\n Token(SpecTokens.START_EDGE_PROPERTIES, value=\"^[\"),\n Token(SpecTokens.KEY_VALUE_PAIR, value=\"virtuals=mpi\"),\n Token(SpecTokens.END_EDGE_PROPERTIES, value=\"]\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"openmpi\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"+foo\"),\n Token(SpecTokens.START_EDGE_PROPERTIES, value=\"^[\"),\n Token(SpecTokens.KEY_VALUE_PAIR, value=\"virtuals=lapack\"),\n Token(SpecTokens.END_EDGE_PROPERTIES, value=\"]\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"openmpi\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"+bar\"),\n ],\n \"^lapack,mpi=openmpi+bar+foo\",\n ),\n (\n \"^lapack,mpi=openmpi+foo+bar\",\n [\n Token(\n SpecTokens.DEPENDENCY,\n value=\"^lapack,mpi=openmpi\",\n virtuals=\"lapack,mpi\",\n substitute=\"openmpi\",\n ),\n Token(SpecTokens.BOOL_VARIANT, value=\"+foo\"),\n Token(SpecTokens.BOOL_VARIANT, value=\"+bar\"),\n ],\n \"^lapack,mpi=openmpi+bar+foo\",\n ),\n (\n \"^[deptypes=link,build] zlib\",\n [\n Token(SpecTokens.START_EDGE_PROPERTIES, value=\"^[\"),\n Token(SpecTokens.KEY_VALUE_PAIR, value=\"deptypes=link,build\"),\n Token(SpecTokens.END_EDGE_PROPERTIES, value=\"]\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"zlib\"),\n ],\n \"^[deptypes=build,link] zlib\",\n ),\n (\n \"^[deptypes=link] zlib ^[deptypes=build] zlib\",\n [\n Token(SpecTokens.START_EDGE_PROPERTIES, value=\"^[\"),\n Token(SpecTokens.KEY_VALUE_PAIR, value=\"deptypes=link\"),\n Token(SpecTokens.END_EDGE_PROPERTIES, value=\"]\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"zlib\"),\n Token(SpecTokens.START_EDGE_PROPERTIES, value=\"^[\"),\n Token(SpecTokens.KEY_VALUE_PAIR, value=\"deptypes=build\"),\n Token(SpecTokens.END_EDGE_PROPERTIES, value=\"]\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"zlib\"),\n ],\n \"^[deptypes=link] zlib ^[deptypes=build] zlib\",\n ),\n (\n \"git-test@git.foo/bar\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, \"git-test\"),\n Token(SpecTokens.GIT_VERSION, \"@git.foo/bar\"),\n ],\n \"git-test@git.foo/bar\",\n ),\n # Variant propagation\n (\n \"zlib ++foo\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, \"zlib\"),\n Token(SpecTokens.PROPAGATED_BOOL_VARIANT, \"++foo\"),\n ],\n \"zlib++foo\",\n ),\n (\n \"zlib ~~foo\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, \"zlib\"),\n Token(SpecTokens.PROPAGATED_BOOL_VARIANT, \"~~foo\"),\n ],\n \"zlib~~foo\",\n ),\n (\n \"zlib foo==bar\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, \"zlib\"),\n Token(SpecTokens.PROPAGATED_KEY_VALUE_PAIR, \"foo==bar\"),\n ],\n \"zlib foo==bar\",\n ),\n # Compilers specifying virtuals\n (\n \"zlib %[virtuals=c] gcc\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, \"zlib\"),\n Token(SpecTokens.START_EDGE_PROPERTIES, value=\"%[\"),\n Token(SpecTokens.KEY_VALUE_PAIR, value=\"virtuals=c\"),\n Token(SpecTokens.END_EDGE_PROPERTIES, value=\"]\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"gcc\"),\n ],\n \"zlib %c=gcc\",\n ),\n (\n \"zlib %c=gcc\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, \"zlib\"),\n Token(SpecTokens.DEPENDENCY, value=\"%c=gcc\", virtuals=\"c\", substitute=\"gcc\"),\n ],\n \"zlib %c=gcc\",\n ),\n (\n \"zlib %[virtuals=c,cxx] gcc\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, \"zlib\"),\n Token(SpecTokens.START_EDGE_PROPERTIES, value=\"%[\"),\n Token(SpecTokens.KEY_VALUE_PAIR, value=\"virtuals=c,cxx\"),\n Token(SpecTokens.END_EDGE_PROPERTIES, value=\"]\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"gcc\"),\n ],\n \"zlib %c,cxx=gcc\",\n ),\n (\n \"zlib %c,cxx=gcc\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, \"zlib\"),\n Token(\n SpecTokens.DEPENDENCY, value=\"%c,cxx=gcc\", virtuals=\"c,cxx\", substitute=\"gcc\"\n ),\n ],\n \"zlib %c,cxx=gcc\",\n ),\n (\n \"zlib %[virtuals=c,cxx] gcc@14.1\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, \"zlib\"),\n Token(SpecTokens.START_EDGE_PROPERTIES, value=\"%[\"),\n Token(SpecTokens.KEY_VALUE_PAIR, value=\"virtuals=c,cxx\"),\n Token(SpecTokens.END_EDGE_PROPERTIES, value=\"]\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"gcc\"),\n Token(SpecTokens.VERSION, value=\"@14.1\"),\n ],\n \"zlib %c,cxx=gcc@14.1\",\n ),\n (\n \"zlib %c,cxx=gcc@14.1\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, \"zlib\"),\n Token(\n SpecTokens.DEPENDENCY, value=\"%c,cxx=gcc\", virtuals=\"c,cxx\", substitute=\"gcc\"\n ),\n Token(SpecTokens.VERSION, value=\"@14.1\"),\n ],\n \"zlib %c,cxx=gcc@14.1\",\n ),\n (\n \"zlib %[virtuals=fortran] gcc@14.1 %[virtuals=c,cxx] clang\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, \"zlib\"),\n Token(SpecTokens.START_EDGE_PROPERTIES, value=\"%[\"),\n Token(SpecTokens.KEY_VALUE_PAIR, value=\"virtuals=fortran\"),\n Token(SpecTokens.END_EDGE_PROPERTIES, value=\"]\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"gcc\"),\n Token(SpecTokens.VERSION, value=\"@14.1\"),\n Token(SpecTokens.START_EDGE_PROPERTIES, value=\"%[\"),\n Token(SpecTokens.KEY_VALUE_PAIR, value=\"virtuals=c,cxx\"),\n Token(SpecTokens.END_EDGE_PROPERTIES, value=\"]\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"clang\"),\n ],\n \"zlib %fortran=gcc@14.1 %c,cxx=clang\",\n ),\n (\n \"zlib %fortran=gcc@14.1 %c,cxx=clang\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, \"zlib\"),\n Token(\n SpecTokens.DEPENDENCY,\n value=\"%fortran=gcc\",\n virtuals=\"fortran\",\n substitute=\"gcc\",\n ),\n Token(SpecTokens.VERSION, value=\"@14.1\"),\n Token(\n SpecTokens.DEPENDENCY,\n value=\"%c,cxx=clang\",\n virtuals=\"c,cxx\",\n substitute=\"clang\",\n ),\n ],\n \"zlib %fortran=gcc@14.1 %c,cxx=clang\",\n ),\n # test := and :== syntax for key value pairs\n (\n \"gcc languages:=c,c++\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, \"gcc\"),\n Token(SpecTokens.KEY_VALUE_PAIR, \"languages:=c,c++\"),\n ],\n \"gcc languages:='c,c++'\",\n ),\n (\n \"gcc languages:==c,c++\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, \"gcc\"),\n Token(SpecTokens.PROPAGATED_KEY_VALUE_PAIR, \"languages:==c,c++\"),\n ],\n \"gcc languages:=='c,c++'\",\n ),\n # test etc. after %\n (\n \"mvapich %gcc languages:=c,c++ target=x86_64\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, \"mvapich\"),\n Token(SpecTokens.DEPENDENCY, \"%\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, \"gcc\"),\n Token(SpecTokens.KEY_VALUE_PAIR, \"languages:=c,c++\"),\n Token(SpecTokens.KEY_VALUE_PAIR, \"target=x86_64\"),\n ],\n \"mvapich %gcc languages:='c,c++' target=x86_64\",\n ),\n # Test conditional dependencies\n (\n \"foo ^[when='%c' virtuals=c] gcc\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, \"foo\"),\n Token(SpecTokens.START_EDGE_PROPERTIES, \"^[\"),\n Token(SpecTokens.KEY_VALUE_PAIR, \"when='%c'\"),\n Token(SpecTokens.KEY_VALUE_PAIR, \"virtuals=c\"),\n Token(SpecTokens.END_EDGE_PROPERTIES, \"]\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, \"gcc\"),\n ],\n \"foo ^[when='%c'] c=gcc\",\n ),\n (\n \"foo ^[when='%c' virtuals=c]gcc\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, \"foo\"),\n Token(SpecTokens.START_EDGE_PROPERTIES, \"^[\"),\n Token(SpecTokens.KEY_VALUE_PAIR, \"when='%c'\"),\n Token(SpecTokens.KEY_VALUE_PAIR, \"virtuals=c\"),\n Token(SpecTokens.END_EDGE_PROPERTIES, \"]\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, \"gcc\"),\n ],\n \"foo ^[when='%c'] c=gcc\",\n ),\n (\n \"foo ^[when='%c'] c=gcc\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, \"foo\"),\n Token(SpecTokens.START_EDGE_PROPERTIES, \"^[\"),\n Token(SpecTokens.KEY_VALUE_PAIR, \"when='%c'\"),\n Token(SpecTokens.END_EDGE_PROPERTIES, \"] c=gcc\", virtuals=\"c\", substitute=\"gcc\"),\n ],\n \"foo ^[when='%c'] c=gcc\",\n ),\n # Test dependency propagation\n (\n \"foo %%gcc\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, \"foo\"),\n Token(SpecTokens.DEPENDENCY, \"%%\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, \"gcc\"),\n ],\n \"foo %%gcc\",\n ),\n (\n \"foo %%c,cxx=gcc\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, \"foo\"),\n Token(SpecTokens.DEPENDENCY, \"%%c,cxx=gcc\", virtuals=\"c,cxx\", substitute=\"gcc\"),\n ],\n \"foo %%c,cxx=gcc\",\n ),\n (\n \"foo %%[when='%c'] c=gcc\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, \"foo\"),\n Token(SpecTokens.START_EDGE_PROPERTIES, \"%%[\"),\n Token(SpecTokens.KEY_VALUE_PAIR, \"when='%c'\"),\n Token(SpecTokens.END_EDGE_PROPERTIES, \"] c=gcc\", virtuals=\"c\", substitute=\"gcc\"),\n ],\n \"foo %%[when='%c'] c=gcc\",\n ),\n (\n \"foo %%[when='%c' virtuals=c] gcc\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, \"foo\"),\n Token(SpecTokens.START_EDGE_PROPERTIES, \"%%[\"),\n Token(SpecTokens.KEY_VALUE_PAIR, \"when='%c'\"),\n Token(SpecTokens.KEY_VALUE_PAIR, \"virtuals=c\"),\n Token(SpecTokens.END_EDGE_PROPERTIES, \"]\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, \"gcc\"),\n ],\n \"foo %%[when='%c'] c=gcc\",\n ),\n ],\n)\ndef test_parse_single_spec(spec_str, tokens, expected_roundtrip, mock_git_test_package):\n parser = SpecParser(spec_str)\n assert tokens == parser.tokens()\n assert expected_roundtrip == str(parser.next_spec())\n\n\n@pytest.mark.parametrize(\n \"text,tokens,expected_specs\",\n [\n (\n \"mvapich emacs\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"mvapich\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"emacs\"),\n ],\n [\"mvapich\", \"emacs\"],\n ),\n (\n \"mvapich cppflags='-O3 -fPIC' emacs\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"mvapich\"),\n Token(SpecTokens.KEY_VALUE_PAIR, value=\"cppflags='-O3 -fPIC'\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"emacs\"),\n ],\n [\"mvapich cppflags='-O3 -fPIC'\", \"emacs\"],\n ),\n (\n \"mvapich cppflags=-O3 emacs\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"mvapich\"),\n Token(SpecTokens.KEY_VALUE_PAIR, value=\"cppflags=-O3\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"emacs\"),\n ],\n [\"mvapich cppflags=-O3\", \"emacs\"],\n ),\n (\n \"mvapich emacs @1.1.1 cflags=-O3 %intel\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"mvapich\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"emacs\"),\n Token(SpecTokens.VERSION, value=\"@1.1.1\"),\n Token(SpecTokens.KEY_VALUE_PAIR, value=\"cflags=-O3\"),\n Token(SpecTokens.DEPENDENCY, value=\"%\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"intel\"),\n ],\n [\"mvapich\", \"emacs @1.1.1 cflags=-O3 %intel\"],\n ),\n (\n 'mvapich cflags=\"-O3 -fPIC\" emacs^ncurses%intel',\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"mvapich\"),\n Token(SpecTokens.KEY_VALUE_PAIR, value='cflags=\"-O3 -fPIC\"'),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"emacs\"),\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"ncurses\"),\n Token(SpecTokens.DEPENDENCY, value=\"%\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"intel\"),\n ],\n ['mvapich cflags=\"-O3 -fPIC\"', \"emacs ^ncurses%intel\"],\n ),\n (\n \"mvapich %gcc languages=c,c++ emacs ^ncurses%gcc languages:=c\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"mvapich\"),\n Token(SpecTokens.DEPENDENCY, value=\"%\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"gcc\"),\n Token(SpecTokens.KEY_VALUE_PAIR, value=\"languages=c,c++\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"emacs\"),\n Token(SpecTokens.DEPENDENCY, value=\"^\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"ncurses\"),\n Token(SpecTokens.DEPENDENCY, value=\"%\"),\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"gcc\"),\n Token(SpecTokens.KEY_VALUE_PAIR, value=\"languages:=c\"),\n ],\n [\"mvapich %gcc languages=c,c++\", \"emacs ^ncurses%gcc languages:=c\"],\n ),\n ],\n)\ndef test_parse_multiple_specs(text, tokens, expected_specs):\n total_parser = SpecParser(text)\n assert total_parser.tokens() == tokens\n\n for single_spec_text in expected_specs:\n single_spec_parser = SpecParser(single_spec_text)\n assert str(total_parser.next_spec()) == str(single_spec_parser.next_spec())\n\n\n@pytest.mark.parametrize(\n \"args,expected\",\n [\n # Test that CLI-quoted flags/variant values are preserved\n ([\"zlib\", \"cflags=-O3 -g\", \"+bar\", \"baz\"], \"zlib cflags='-O3 -g' +bar baz\"),\n # Test that CLI-quoted propagated flags/variant values are preserved\n ([\"zlib\", \"cflags==-O3 -g\", \"+bar\", \"baz\"], \"zlib cflags=='-O3 -g' +bar baz\"),\n # An entire string passed on the CLI with embedded quotes also works\n ([\"zlib cflags='-O3 -g' +bar baz\"], \"zlib cflags='-O3 -g' +bar baz\"),\n # Entire string *without* quoted flags splits -O3/-g (-g interpreted as a variant)\n ([\"zlib cflags=-O3 -g +bar baz\"], \"zlib cflags=-O3 +bar~g baz\"),\n # If the entirety of \"-O3 -g +bar baz\" is quoted on the CLI, it's all taken as flags\n ([\"zlib\", \"cflags=-O3 -g +bar baz\"], \"zlib cflags='-O3 -g +bar baz'\"),\n # If the string doesn't start with key=, it needs internal quotes for flags\n ([\"zlib\", \" cflags=-O3 -g +bar baz\"], \"zlib cflags=-O3 +bar~g baz\"),\n # Internal quotes for quoted CLI args are considered part of *one* arg\n ([\"zlib\", 'cflags=\"-O3 -g\" +bar baz'], \"\"\"zlib cflags='\"-O3 -g\" +bar baz'\"\"\"),\n # Use double quotes if internal single quotes are present\n ([\"zlib\", \"cflags='-O3 -g' +bar baz\"], '''zlib cflags=\"'-O3 -g' +bar baz\"'''),\n # Use single quotes and escape single quotes with internal single and double quotes\n ([\"zlib\", \"cflags='-O3 -g' \\\"+bar baz\\\"\"], 'zlib cflags=\"\\'-O3 -g\\' \\\\\"+bar baz\\\\\"\"'),\n # Ensure that empty strings are handled correctly on CLI\n ([\"zlib\", \"ldflags=\", \"+pic\"], \"zlib+pic\"),\n # These flags are assumed to be quoted by the shell, but the space doesn't matter because\n # flags are space-separated.\n ([\"zlib\", \"ldflags= +pic\"], \"zlib ldflags='+pic'\"),\n ([\"ldflags= +pic\"], \"ldflags='+pic'\"),\n # If the name is not a flag name, the space is preserved verbatim, because variant values\n # are comma-separated.\n ([\"zlib\", \"foo= +pic\"], \"zlib foo=' +pic'\"),\n ([\"foo= +pic\"], \"foo=' +pic'\"),\n # You can ensure no quotes are added parse_specs() by starting your string with space,\n # but you still need to quote empty strings properly.\n ([\" ldflags= +pic\"], SpecTokenizationError),\n ([\" ldflags=\", \"+pic\"], SpecTokenizationError),\n ([\" ldflags='' +pic\"], \"+pic\"),\n ([\" ldflags=''\", \"+pic\"], \"+pic\"),\n # Ensure that empty strings are handled properly in quoted strings\n ([\"zlib ldflags='' +pic\"], \"zlib+pic\"),\n # Ensure that $ORIGIN is handled correctly\n ([\"zlib\", \"ldflags=-Wl,-rpath=$ORIGIN/_libs\"], \"zlib ldflags='-Wl,-rpath=$ORIGIN/_libs'\"),\n # Ensure that passing escaped quotes on the CLI raises a tokenization error\n ([\"zlib\", '\"-g', '-O2\"'], SpecTokenizationError),\n ],\n)\ndef test_cli_spec_roundtrip(args, expected):\n if isinstance(expected, type) and issubclass(expected, BaseException):\n with pytest.raises(expected):\n spack.cmd.parse_specs(args)\n return\n\n specs = spack.cmd.parse_specs(args)\n output_string = \" \".join(str(spec) for spec in specs)\n assert expected == output_string\n\n\n@pytest.mark.parametrize(\n [\"spec_str\", \"toolchain\", \"expected_roundtrip\"],\n [\n (\n \"foo%my_toolchain\",\n {\"my_toolchain\": \"%[when='%c' virtuals=c]gcc\"},\n [\"foo %[when='%c'] c=gcc\"],\n ),\n (\"foo%my_toolchain\", {\"my_toolchain\": \"%[when='%c'] c=gcc\"}, [\"foo %[when='%c'] c=gcc\"]),\n (\n \"foo%my_toolchain\",\n {\"my_toolchain\": \"+bar cflags=baz %[when='%c' virtuals=c]gcc\"},\n [\"foo cflags=baz +bar %[when='%c'] c=gcc\"],\n ),\n (\n \"foo%my_toolchain\",\n {\"my_toolchain\": \"+bar cflags=baz %[when='%c']c=gcc\"},\n [\"foo cflags=baz +bar %[when='%c'] c=gcc\"],\n ),\n (\n \"foo%my_toolchain2\",\n {\"my_toolchain2\": \"%[when='%c' virtuals=c]gcc %[when='+mpi' virtuals=mpi]mpich\"},\n [\"foo %[when='%c'] c=gcc %[when='+mpi'] mpi=mpich\"],\n ),\n (\n \"foo%my_toolchain2\",\n {\"my_toolchain2\": \"%[when='%c'] c=gcc %[when='+mpi'] mpi=mpich\"},\n [\"foo %[when='%c'] c=gcc %[when='+mpi'] mpi=mpich\"],\n ),\n (\n \"foo%my_toolchain bar%my_toolchain2\",\n {\n \"my_toolchain\": \"%[when='%c' virtuals=c]gcc\",\n \"my_toolchain2\": \"%[when='%c' virtuals=c]gcc %[when='+mpi' virtuals=mpi]mpich\",\n },\n [\"foo %[when='%c'] c=gcc\", \"bar %[when='%c'] c=gcc %[when='+mpi'] mpi=mpich\"],\n ),\n (\n \"foo%my_toolchain bar%my_toolchain2\",\n {\n \"my_toolchain\": \"%[when='%c'] c=gcc\",\n \"my_toolchain2\": \"%[when='%c'] c=gcc %[when='+mpi']mpi=mpich\",\n },\n [\"foo %[when='%c'] c=gcc\", \"bar %[when='%c'] c=gcc %[when='+mpi'] mpi=mpich\"],\n ),\n (\n \"foo%my_toolchain2\",\n {\n \"my_toolchain2\": [\n {\"spec\": \"%[virtuals=c]gcc\", \"when\": \"%c\"},\n {\"spec\": \"%[virtuals=mpi]mpich\", \"when\": \"+mpi\"},\n ]\n },\n [\"foo %[when='%c'] c=gcc %[when='+mpi'] mpi=mpich\"],\n ),\n (\n \"foo%my_toolchain2\",\n {\n \"my_toolchain2\": [\n {\"spec\": \"%c=gcc\", \"when\": \"%c\"},\n {\"spec\": \"%mpi=mpich\", \"when\": \"+mpi\"},\n ]\n },\n [\"foo %[when='%c'] c=gcc %[when='+mpi'] mpi=mpich\"],\n ),\n (\n \"foo%my_toolchain2\",\n {\"my_toolchain2\": [{\"spec\": \"%[virtuals=c]gcc %[virtuals=mpi]mpich\", \"when\": \"%c\"}]},\n [\"foo %[when='%c'] c=gcc %[when='%c'] mpi=mpich\"],\n ),\n (\n \"foo%my_toolchain2\",\n {\"my_toolchain2\": [{\"spec\": \"%c=gcc %mpi=mpich\", \"when\": \"%c\"}]},\n [\"foo %[when='%c'] c=gcc %[when='%c'] mpi=mpich\"],\n ),\n # Test that we don't get caching wrong in the parser\n (\n \"foo %gcc-mpich ^bar%gcc-mpich\",\n {\n \"gcc-mpich\": [\n {\"spec\": \"%[virtuals=c] gcc\", \"when\": \"%c\"},\n {\"spec\": \"%[virtuals=mpi] mpich\", \"when\": \"%mpi\"},\n ]\n },\n [\n \"foo %[when='%c'] c=gcc %[when='%mpi'] mpi=mpich \"\n \"^bar %[when='%c'] c=gcc %[when='%mpi'] mpi=mpich\"\n ],\n ),\n (\n \"foo %gcc-mpich ^bar%gcc-mpich\",\n {\n \"gcc-mpich\": [\n {\"spec\": \"%c=gcc\", \"when\": \"%c\"},\n {\"spec\": \"%mpi=mpich\", \"when\": \"%mpi\"},\n ]\n },\n [\n \"foo %[when='%c'] c=gcc %[when='%mpi'] mpi=mpich \"\n \"^bar %[when='%c'] c=gcc %[when='%mpi'] mpi=mpich\"\n ],\n ),\n ],\n)\ndef test_parse_toolchain(spec_str, toolchain, expected_roundtrip, mutable_config):\n \"\"\"Tests that toolchains are expanded correctly\"\"\"\n parser = SpecParser(spec_str)\n for expected in expected_roundtrip:\n result = parser.next_spec()\n expand_toolchains(result, toolchain)\n assert expected == str(result)\n\n\n@pytest.mark.parametrize(\n \"text,expected_in_error\",\n [\n (\"x@@1.2\", r\"x@@1.2\\n ^\"),\n (\"y ^x@@1.2\", r\"y ^x@@1.2\\n ^\"),\n (\"x@1.2::\", r\"x@1.2::\\n ^\"),\n (\"x::\", r\"x::\\n ^^\"),\n (\"cflags=''-Wl,a,b,c''\", r\"cflags=''-Wl,a,b,c''\\n ^ ^ ^ ^^\"),\n (\"@1.2: develop = foo\", r\"@1.2: develop = foo\\n ^^\"),\n (\"@1.2:develop = foo\", r\"@1.2:develop = foo\\n ^^\"),\n ],\n)\ndef test_error_reporting(text, expected_in_error):\n parser = SpecParser(text)\n with pytest.raises(SpecTokenizationError) as exc:\n parser.tokens()\n\n assert expected_in_error in str(exc), parser.tokens()\n\n\n@pytest.mark.parametrize(\n \"text,tokens\",\n [\n (\"/abcde\", [Token(SpecTokens.DAG_HASH, value=\"/abcde\")]),\n (\n \"foo/abcde\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"foo\"),\n Token(SpecTokens.DAG_HASH, value=\"/abcde\"),\n ],\n ),\n (\n \"foo@1.2.3 /abcde\",\n [\n Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value=\"foo\"),\n Token(SpecTokens.VERSION, value=\"@1.2.3\"),\n Token(SpecTokens.DAG_HASH, value=\"/abcde\"),\n ],\n ),\n ],\n)\ndef test_spec_by_hash_tokens(text, tokens):\n parser = SpecParser(text)\n assert parser.tokens() == tokens\n\n\n@pytest.mark.db\ndef test_spec_by_hash(database, monkeypatch, config):\n mpileaks = database.query_one(\"mpileaks ^zmpi\")\n b = spack.concretize.concretize_one(\"pkg-b\")\n monkeypatch.setattr(spack.binary_distribution, \"update_cache_and_get_specs\", lambda: [b])\n\n hash_str = f\"/{mpileaks.dag_hash()}\"\n parsed_spec = SpecParser(hash_str).next_spec()\n parsed_spec.replace_hash()\n assert parsed_spec == mpileaks\n\n short_hash_str = f\"/{mpileaks.dag_hash()[:5]}\"\n parsed_spec = SpecParser(short_hash_str).next_spec()\n parsed_spec.replace_hash()\n assert parsed_spec == mpileaks\n\n name_version_and_hash = f\"{mpileaks.name}@{mpileaks.version} /{mpileaks.dag_hash()[:5]}\"\n parsed_spec = SpecParser(name_version_and_hash).next_spec()\n parsed_spec.replace_hash()\n assert parsed_spec == mpileaks\n\n b_hash = f\"/{b.dag_hash()}\"\n parsed_spec = SpecParser(b_hash).next_spec()\n parsed_spec.replace_hash()\n assert parsed_spec == b\n\n\n@pytest.mark.db\ndef test_dep_spec_by_hash(database, config):\n mpileaks_zmpi = database.query_one(\"mpileaks ^zmpi\")\n zmpi = database.query_one(\"zmpi\")\n fake = database.query_one(\"fake\")\n\n assert \"fake\" in mpileaks_zmpi\n assert \"zmpi\" in mpileaks_zmpi\n\n mpileaks_hash_fake = SpecParser(f\"mpileaks ^/{fake.dag_hash()} ^zmpi\").next_spec()\n mpileaks_hash_fake.replace_hash()\n assert \"fake\" in mpileaks_hash_fake\n assert mpileaks_hash_fake[\"fake\"] == fake\n assert \"zmpi\" in mpileaks_hash_fake\n assert mpileaks_hash_fake[\"zmpi\"] == spack.spec.Spec(\"zmpi\")\n\n mpileaks_hash_zmpi = SpecParser(f\"mpileaks ^ /{zmpi.dag_hash()}\").next_spec()\n mpileaks_hash_zmpi.replace_hash()\n assert \"zmpi\" in mpileaks_hash_zmpi\n assert mpileaks_hash_zmpi[\"zmpi\"] == zmpi\n\n mpileaks_hash_fake_and_zmpi = SpecParser(\n f\"mpileaks ^/{fake.dag_hash()[:4]} ^ /{zmpi.dag_hash()[:5]}\"\n ).next_spec()\n mpileaks_hash_fake_and_zmpi.replace_hash()\n assert \"zmpi\" in mpileaks_hash_fake_and_zmpi\n assert mpileaks_hash_fake_and_zmpi[\"zmpi\"] == zmpi\n\n assert \"fake\" in mpileaks_hash_fake_and_zmpi\n assert mpileaks_hash_fake_and_zmpi[\"fake\"] == fake\n\n\n@pytest.mark.db\ndef test_multiple_specs_with_hash(database, config):\n mpileaks_zmpi = database.query_one(\"mpileaks ^zmpi\")\n callpath_mpich2 = database.query_one(\"callpath ^mpich2\")\n\n # name + hash + separate hash\n specs = SpecParser(\n f\"mpileaks /{mpileaks_zmpi.dag_hash()} /{callpath_mpich2.dag_hash()}\"\n ).all_specs()\n assert len(specs) == 2\n\n # 2 separate hashes\n specs = SpecParser(f\"/{mpileaks_zmpi.dag_hash()} /{callpath_mpich2.dag_hash()}\").all_specs()\n assert len(specs) == 2\n\n # 2 separate hashes + name\n specs = SpecParser(\n f\"/{mpileaks_zmpi.dag_hash()} /{callpath_mpich2.dag_hash()} callpath\"\n ).all_specs()\n assert len(specs) == 3\n\n # hash + 2 names\n specs = SpecParser(f\"/{mpileaks_zmpi.dag_hash()} callpath callpath\").all_specs()\n assert len(specs) == 3\n\n # hash + name + hash\n specs = SpecParser(\n f\"/{mpileaks_zmpi.dag_hash()} callpath /{callpath_mpich2.dag_hash()}\"\n ).all_specs()\n assert len(specs) == 2\n\n\n@pytest.mark.db\ndef test_ambiguous_hash(mutable_database):\n \"\"\"Test that abstract hash ambiguity is delayed until concretization.\n In the past this ambiguity error would happen during parse time.\"\"\"\n\n # This is a very sketchy as manually setting hashes easily breaks invariants\n x1 = spack.concretize.concretize_one(\"pkg-a\")\n x2 = x1.copy()\n x1._hash = \"xyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\"\n x2._hash = \"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"\n\n assert x1 != x2 # doesn't hold when only the dag hash is modified.\n\n mutable_database.add(x1)\n mutable_database.add(x2)\n\n # ambiguity in first hash character\n s1 = SpecParser(\"/x\").next_spec()\n with pytest.raises(spack.spec.AmbiguousHashError):\n s1.lookup_hash()\n\n # ambiguity in first hash character AND spec name\n s2 = SpecParser(\"pkg-a/x\").next_spec()\n with pytest.raises(spack.spec.AmbiguousHashError):\n s2.lookup_hash()\n\n\n@pytest.mark.db\ndef test_invalid_hash(database, config):\n zmpi = database.query_one(\"zmpi\")\n mpich = database.query_one(\"mpich\")\n\n # name + incompatible hash\n with pytest.raises(spack.spec.InvalidHashError):\n parsed_spec = SpecParser(f\"zmpi /{mpich.dag_hash()}\").next_spec()\n parsed_spec.replace_hash()\n with pytest.raises(spack.spec.InvalidHashError):\n parsed_spec = SpecParser(f\"mpich /{zmpi.dag_hash()}\").next_spec()\n parsed_spec.replace_hash()\n\n # name + dep + incompatible hash\n with pytest.raises(spack.spec.InvalidHashError):\n parsed_spec = SpecParser(f\"mpileaks ^zmpi /{mpich.dag_hash()}\").next_spec()\n parsed_spec.replace_hash()\n\n\ndef test_invalid_hash_dep(database, config):\n mpich = database.query_one(\"mpich\")\n hash = mpich.dag_hash()\n with pytest.raises(spack.spec.InvalidHashError):\n spack.spec.Spec(f\"callpath ^zlib/{hash}\").replace_hash()\n\n\n@pytest.mark.db\ndef test_nonexistent_hash(database, config):\n \"\"\"Ensure we get errors for non existent hashes.\"\"\"\n specs = database.query()\n\n # This hash shouldn't be in the test DB. What are the odds :)\n no_such_hash = \"aaaaaaaaaaaaaaa\"\n hashes = [s._hash for s in specs]\n assert no_such_hash not in [h[: len(no_such_hash)] for h in hashes]\n\n with pytest.raises(spack.spec.InvalidHashError):\n parsed_spec = SpecParser(f\"/{no_such_hash}\").next_spec()\n parsed_spec.replace_hash()\n\n\n@pytest.mark.parametrize(\n \"spec1,spec2,constraint\",\n [\n (\"zlib\", \"hdf5\", None),\n (\"zlib+shared\", \"zlib~shared\", \"+shared\"),\n (\"hdf5+mpi^zmpi\", \"hdf5~mpi\", \"^zmpi\"),\n (\"hdf5+mpi^mpich+debug\", \"hdf5+mpi^mpich~debug\", \"^mpich+debug\"),\n ],\n)\ndef test_disambiguate_hash_by_spec(spec1, spec2, constraint, mock_packages, monkeypatch, config):\n spec1_concrete = spack.concretize.concretize_one(spec1)\n spec2_concrete = spack.concretize.concretize_one(spec2)\n\n spec1_concrete._hash = \"spec1\"\n spec2_concrete._hash = \"spec2\"\n\n monkeypatch.setattr(\n spack.binary_distribution,\n \"update_cache_and_get_specs\",\n lambda: [spec1_concrete, spec2_concrete],\n )\n\n # Ordering is tricky -- for constraints we want after, for names we want before\n if not constraint:\n spec = spack.spec.Spec(spec1 + \"/spec\")\n else:\n spec = spack.spec.Spec(\"/spec\" + constraint)\n\n assert spec.lookup_hash() == spec1_concrete\n\n\n@pytest.mark.parametrize(\n \"text,match_string\",\n [\n # Duplicate variants\n (\"x@1.2+debug+debug\", \"variant\"),\n (\"x ^y@1.2+debug debug=true\", \"variant\"),\n (\"x ^y@1.2 debug=false debug=true\", \"variant\"),\n (\"x ^y@1.2 debug=false ~debug\", \"variant\"),\n # Multiple versions\n (\"x@1.2@2.3\", \"version\"),\n (\"x@1.2:2.3@1.4\", \"version\"),\n (\"x@1.2@2.3:2.4\", \"version\"),\n (\"x@1.2@2.3,2.4\", \"version\"),\n (\"x@1.2 +foo~bar @2.3\", \"version\"),\n (\"x@1.2%y@1.2@2.3:2.4\", \"version\"),\n # Duplicate dependency\n (\"x ^y@1 ^y@2\", \"Cannot depend on incompatible specs\"),\n # Duplicate Architectures\n (\"x arch=linux-rhel7-x86_64 arch=linux-rhel7-x86_64\", \"two architectures\"),\n (\"x arch=linux-rhel7-x86_64 arch=linux-rhel7-ppc64le\", \"two architectures\"),\n (\"x arch=linux-rhel7-ppc64le arch=linux-rhel7-x86_64\", \"two architectures\"),\n (\"y ^x arch=linux-rhel7-x86_64 arch=linux-rhel7-x86_64\", \"two architectures\"),\n (\"y ^x arch=linux-rhel7-x86_64 arch=linux-rhel7-ppc64le\", \"two architectures\"),\n (\"x os=redhat6 os=debian6\", \"'os'\"),\n (\"x os=debian6 os=redhat6\", \"'os'\"),\n (\"x target=core2 target=x86_64\", \"'target'\"),\n (\"x target=x86_64 target=core2\", \"'target'\"),\n (\"x platform=test platform=test\", \"'platform'\"),\n # TODO: these two seem wrong: need to change how arch is initialized (should fail on os)\n (\"x os=debian6 platform=test target=default_target os=redhat6\", \"two architectures\"),\n (\"x target=default_target platform=test os=redhat6 os=debian6\", \"'platform'\"),\n # Dependencies\n (\"^[@foo] zlib\", \"edge attributes\"),\n (\"x ^[deptypes=link]foo ^[deptypes=run]foo\", \"conflicting dependency types\"),\n (\"x ^[deptypes=build,link]foo ^[deptypes=link]foo\", \"conflicting dependency types\"),\n # TODO: Remove this as soon as use variants are added and we can parse custom attributes\n (\"^[foo=bar] zlib\", \"edge attributes\"),\n # Propagating reserved names generates a parse error\n (\"x namespace==foo.bar.baz\", \"Propagation\"),\n (\"x arch==linux-rhel9-x86_64\", \"Propagation\"),\n (\"x architecture==linux-rhel9-x86_64\", \"Propagation\"),\n (\"x os==rhel9\", \"Propagation\"),\n (\"x operating_system==rhel9\", \"Propagation\"),\n (\"x target==x86_64\", \"Propagation\"),\n (\"x dev_path==/foo/bar/baz\", \"Propagation\"),\n (\"x patches==abcde12345,12345abcde\", \"Propagation\"),\n ],\n)\ndef test_error_conditions(text, match_string):\n with pytest.raises(SpecParsingError, match=match_string):\n SpecParser(text).next_spec()\n\n\n@pytest.mark.parametrize(\n \"text,exc_cls\",\n [\n # Specfile related errors\n pytest.param(\n \"/bogus/path/libdwarf.yaml\", spack.error.NoSuchSpecFileError, marks=SKIP_ON_WINDOWS\n ),\n pytest.param(\n \"../../libdwarf.yaml\", spack.error.NoSuchSpecFileError, marks=SKIP_ON_WINDOWS\n ),\n pytest.param(\"./libdwarf.yaml\", spack.error.NoSuchSpecFileError, marks=SKIP_ON_WINDOWS),\n pytest.param(\n \"libfoo ^/bogus/path/libdwarf.yaml\",\n spack.error.NoSuchSpecFileError,\n marks=SKIP_ON_WINDOWS,\n ),\n pytest.param(\n \"libfoo ^../../libdwarf.yaml\", spack.error.NoSuchSpecFileError, marks=SKIP_ON_WINDOWS\n ),\n pytest.param(\n \"libfoo ^./libdwarf.yaml\", spack.error.NoSuchSpecFileError, marks=SKIP_ON_WINDOWS\n ),\n pytest.param(\n \"/bogus/path/libdwarf.yamlfoobar\",\n spack.error.NoSuchSpecFileError,\n marks=SKIP_ON_WINDOWS,\n ),\n pytest.param(\n \"libdwarf^/bogus/path/libelf.yamlfoobar ^/path/to/bogus.yaml\",\n spack.error.NoSuchSpecFileError,\n marks=SKIP_ON_WINDOWS,\n ),\n pytest.param(\n \"c:\\\\bogus\\\\path\\\\libdwarf.yaml\", spack.error.NoSuchSpecFileError, marks=SKIP_ON_UNIX\n ),\n pytest.param(\"..\\\\..\\\\libdwarf.yaml\", spack.error.NoSuchSpecFileError, marks=SKIP_ON_UNIX),\n pytest.param(\".\\\\libdwarf.yaml\", spack.error.NoSuchSpecFileError, marks=SKIP_ON_UNIX),\n pytest.param(\n \"libfoo ^c:\\\\bogus\\\\path\\\\libdwarf.yaml\",\n spack.error.NoSuchSpecFileError,\n marks=SKIP_ON_UNIX,\n ),\n pytest.param(\n \"libfoo ^..\\\\..\\\\libdwarf.yaml\", spack.error.NoSuchSpecFileError, marks=SKIP_ON_UNIX\n ),\n pytest.param(\n \"libfoo ^.\\\\libdwarf.yaml\", spack.error.NoSuchSpecFileError, marks=SKIP_ON_UNIX\n ),\n pytest.param(\n \"c:\\\\bogus\\\\path\\\\libdwarf.yamlfoobar\",\n spack.error.SpecFilenameError,\n marks=SKIP_ON_UNIX,\n ),\n pytest.param(\n \"libdwarf^c:\\\\bogus\\\\path\\\\libelf.yamlfoobar ^c:\\\\path\\\\to\\\\bogus.yaml\",\n spack.error.SpecFilenameError,\n marks=SKIP_ON_UNIX,\n ),\n ],\n)\ndef test_specfile_error_conditions_windows(text, exc_cls):\n with pytest.raises(exc_cls):\n SpecParser(text).all_specs()\n\n\n@pytest.mark.parametrize(\n \"filename,regex\",\n [\n (r\"c:\\abs\\windows\\\\path.yaml\", WINDOWS_FILENAME),\n (r\".\\\\relative\\\\dot\\\\win\\\\path.yaml\", WINDOWS_FILENAME),\n (r\"relative\\\\windows\\\\path.yaml\", WINDOWS_FILENAME),\n (\"/absolute/path/to/file.yaml\", UNIX_FILENAME),\n (\"relative/path/to/file.yaml\", UNIX_FILENAME),\n (\"./dot/rel/to/file.yaml\", UNIX_FILENAME),\n ],\n)\ndef test_specfile_parsing(filename, regex):\n match = re.match(regex, filename)\n assert match\n assert match.end() == len(filename)\n\n\ndef test_parse_specfile_simple(specfile_for, tmp_path: pathlib.Path):\n specfile = tmp_path / \"libdwarf.json\"\n s = specfile_for(\"libdwarf\", specfile)\n\n spec = SpecParser(str(specfile)).next_spec()\n assert spec == s\n\n # Check we can mix literal and spec-file in text\n specs = SpecParser(f\"mvapich_foo {str(specfile)}\").all_specs()\n assert len(specs) == 2\n\n\n@pytest.mark.parametrize(\"filename\", [\"libelf.yaml\", \"libelf.json\"])\ndef test_parse_filename_missing_slash_as_spec(specfile_for, tmp_path: pathlib.Path, filename):\n \"\"\"Ensure that libelf(.yaml|.json) parses as a spec, NOT a file.\"\"\"\n specfile = tmp_path / filename\n specfile_for(filename.split(\".\")[0], specfile)\n\n # Move to where the specfile is located so that libelf.yaml is there\n with fs.working_dir(str(tmp_path)):\n specs = SpecParser(\"libelf.yaml\").all_specs()\n assert len(specs) == 1\n\n spec = specs[0]\n assert spec.name == \"yaml\"\n assert spec.namespace == \"libelf\"\n assert spec.fullname == \"libelf.yaml\"\n\n # Check that if we concretize this spec, we get a good error\n # message that mentions we might've meant a file.\n with pytest.raises(spack.repo.UnknownEntityError) as exc_info:\n spack.concretize.concretize_one(spec)\n assert exc_info.value.long_message\n assert (\n \"Did you mean to specify a filename with './libelf.yaml'?\" in exc_info.value.long_message\n )\n\n # make sure that only happens when the spec ends in yaml\n with pytest.raises(spack.solver.asp.UnsatisfiableSpecError) as exc_info:\n spack.concretize.concretize_one(\"builtin_mock.doesnotexist\")\n assert not exc_info.value.long_message or (\n \"Did you mean to specify a filename with\" not in exc_info.value.long_message\n )\n\n\ndef test_parse_specfile_dependency(default_mock_concretization, tmp_path: pathlib.Path):\n \"\"\"Ensure we can use a specfile as a dependency\"\"\"\n s = default_mock_concretization(\"libdwarf\")\n\n specfile = tmp_path / \"libelf.json\"\n with open(specfile, \"w\", encoding=\"utf-8\") as f:\n f.write(s[\"libelf\"].to_json())\n\n # Make sure we can use yaml path as dependency, e.g.:\n # \"spack spec libdwarf ^ /path/to/libelf.json\"\n spec = SpecParser(f\"libdwarf ^ {str(specfile)}\").next_spec()\n assert spec and spec[\"libelf\"] == s[\"libelf\"]\n\n with fs.working_dir(str(tmp_path)):\n # Make sure this also works: \"spack spec ./libelf.yaml\"\n spec = SpecParser(f\"libdwarf^.{os.path.sep}{specfile.name}\").next_spec()\n assert spec and spec[\"libelf\"] == s[\"libelf\"]\n\n # Should also be accepted: \"spack spec ..//libelf.yaml\"\n spec = SpecParser(\n f\"libdwarf^..{os.path.sep}{specfile.parent.name}{os.path.sep}{specfile.name}\"\n ).next_spec()\n assert spec and spec[\"libelf\"] == s[\"libelf\"]\n\n\ndef test_parse_specfile_relative_paths(specfile_for, tmp_path: pathlib.Path):\n specfile = tmp_path / \"libdwarf.json\"\n s = specfile_for(\"libdwarf\", specfile)\n\n basename = specfile.name\n parent_dir = specfile.parent\n\n with fs.working_dir(str(parent_dir)):\n # Make sure this also works: \"spack spec ./libelf.yaml\"\n spec = SpecParser(f\".{os.path.sep}{basename}\").next_spec()\n assert spec == s\n\n # Should also be accepted: \"spack spec ..//libelf.yaml\"\n spec = SpecParser(f\"..{os.path.sep}{parent_dir.name}{os.path.sep}{basename}\").next_spec()\n assert spec == s\n\n # Should also handle mixed clispecs and relative paths, e.g.:\n # \"spack spec mvapich_foo ..//libelf.yaml\"\n specs = SpecParser(\n f\"mvapich_foo ..{os.path.sep}{parent_dir.name}{os.path.sep}{basename}\"\n ).all_specs()\n assert len(specs) == 2\n assert specs[1] == s\n\n\ndef test_parse_specfile_relative_subdir_path(specfile_for, tmp_path: pathlib.Path):\n subdir = tmp_path / \"subdir\"\n subdir.mkdir()\n specfile = subdir / \"libdwarf.json\"\n s = specfile_for(\"libdwarf\", specfile)\n\n with fs.working_dir(str(tmp_path)):\n spec = SpecParser(f\"subdir{os.path.sep}{specfile.name}\").next_spec()\n assert spec == s\n\n\n@pytest.mark.regression(\"20310\")\ndef test_compare_abstract_specs():\n \"\"\"Spec comparisons must be valid for abstract specs.\n\n Check that the spec cmp_key appropriately handles comparing specs for\n which some attributes are None in exactly one of two specs\n \"\"\"\n # Add fields in order they appear in `Spec._cmp_node`\n constraints = [\n \"foo\",\n \"foo.foo\",\n \"foo.foo@foo\",\n \"foo.foo@foo+foo\",\n \"foo.foo@foo+foo arch=foo-foo-foo\",\n \"foo.foo@foo+foo arch=foo-foo-foo %foo\",\n \"foo.foo@foo+foo arch=foo-foo-foo cflags=foo %foo\",\n ]\n specs = [SpecParser(s).next_spec() for s in constraints]\n\n for a, b in itertools.product(specs, repeat=2):\n # Check that we can compare without raising an error\n assert a <= b or b < a\n\n\n@pytest.mark.parametrize(\n \"lhs_str,rhs_str,expected\",\n [\n # Git shasum vs generic develop\n (\n f\"develop-branch-version@git.{'a' * 40}=develop\",\n \"develop-branch-version@develop\",\n (True, True, False),\n ),\n # Two different shasums\n (\n f\"develop-branch-version@git.{'a' * 40}=develop\",\n f\"develop-branch-version@git.{'b' * 40}=develop\",\n (False, False, False),\n ),\n # Git shasum vs. git tag\n (\n f\"develop-branch-version@git.{'a' * 40}=develop\",\n \"develop-branch-version@git.0.2.15=develop\",\n (False, False, False),\n ),\n # Git tag vs. generic develop\n (\n \"develop-branch-version@git.0.2.15=develop\",\n \"develop-branch-version@develop\",\n (True, True, False),\n ),\n ],\n)\ndef test_git_ref_spec_equivalences(mock_packages, lhs_str, rhs_str, expected):\n lhs = SpecParser(lhs_str).next_spec()\n rhs = SpecParser(rhs_str).next_spec()\n intersect, lhs_sat_rhs, rhs_sat_lhs = expected\n\n assert lhs.intersects(rhs) is intersect\n assert rhs.intersects(lhs) is intersect\n assert lhs.satisfies(rhs) is lhs_sat_rhs\n assert rhs.satisfies(lhs) is rhs_sat_lhs\n\n\n@pytest.mark.regression(\"32471\")\n@pytest.mark.parametrize(\"spec_str\", [\"target=x86_64\", \"os=redhat6\", \"target=x86_64:\"])\ndef test_platform_is_none_if_not_present(spec_str):\n s = SpecParser(spec_str).next_spec()\n assert s.architecture.platform is None, s\n\n\ndef test_parse_one_or_raise_error_message():\n with pytest.raises(ValueError) as exc:\n parse_one_or_raise(\" x y z\")\n\n msg = \"\"\"\\\nexpected a single spec, but got more:\n x y z\n ^\\\n\"\"\"\n\n assert str(exc.value) == msg\n\n with pytest.raises(ValueError, match=\"expected a single spec, but got none\"):\n parse_one_or_raise(\" \")\n\n\n@pytest.mark.parametrize(\n \"input_args,expected\",\n [\n # mpileaks %[virtuals=c deptypes=build] gcc\n (\n [\"mpileaks\", \"%[virtuals=c\", \"deptypes=build]\", \"gcc\"],\n [\"mpileaks %[virtuals=c deptypes=build] gcc\"],\n ),\n # mpileaks %[ virtuals=c deptypes=build] gcc\n (\n [\"mpileaks\", \"%[\", \"virtuals=c\", \"deptypes=build]\", \"gcc\"],\n [\"mpileaks %[virtuals=c deptypes=build] gcc\"],\n ),\n # mpileaks %[ virtuals=c deptypes=build ] gcc\n (\n [\"mpileaks\", \"%[\", \"virtuals=c\", \"deptypes=build\", \"]\", \"gcc\"],\n [\"mpileaks %[virtuals=c deptypes=build] gcc\"],\n ),\n ],\n)\ndef test_parse_multiple_edge_attributes(input_args, expected):\n \"\"\"Tests that we can parse correctly multiple edge attributes within square brackets,\n from the command line.\n\n The input are strings as they would be parsed from argparse.REMAINDER\n \"\"\"\n s, *_ = spack.cmd.parse_specs(input_args)\n for c in expected:\n assert s.satisfies(c)\n" }, { "path": "lib/spack/spack/test/spec_yaml.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Test YAML and JSON serialization for specs.\n\nThe YAML and JSON formats preserve DAG information in the spec.\n\n\"\"\"\n\nimport collections\nimport collections.abc\nimport gzip\nimport io\nimport json\nimport os\nimport pathlib\nimport pickle\n\nimport pytest\n\nimport spack.vendor.ruamel.yaml\n\nimport spack.concretize\nimport spack.config\nimport spack.hash_types as ht\nimport spack.paths\nimport spack.repo\nimport spack.spec\nimport spack.util.spack_json as sjson\nimport spack.util.spack_yaml as syaml\nfrom spack.spec import Spec, save_dependency_specfiles\nfrom spack.test.conftest import RepoBuilder\nfrom spack.util.spack_yaml import SpackYAMLError, syaml_dict\n\n\ndef check_yaml_round_trip(spec):\n yaml_text = spec.to_yaml()\n spec_from_yaml = Spec.from_yaml(yaml_text)\n assert spec.eq_dag(spec_from_yaml)\n\n\ndef check_json_round_trip(spec):\n json_text = spec.to_json()\n spec_from_json = Spec.from_json(json_text)\n assert spec.eq_dag(spec_from_json)\n\n\ndef test_read_spec_from_signed_json():\n spec_dir = os.path.join(spack.paths.test_path, \"data\", \"mirrors\", \"signed_json\")\n file_name = (\n \"linux-ubuntu18.04-haswell-gcc-8.4.0-\"\n \"zlib-1.2.12-g7otk5dra3hifqxej36m5qzm7uyghqgb.spec.json.sig\"\n )\n spec_path = os.path.join(spec_dir, file_name)\n\n def check_spec(spec_to_check):\n assert spec_to_check.name == \"zlib\"\n assert spec_to_check._hash == \"g7otk5dra3hifqxej36m5qzm7uyghqgb\"\n\n with open(spec_path, encoding=\"utf-8\") as fd:\n s = Spec.from_signed_json(fd)\n check_spec(s)\n\n with open(spec_path, encoding=\"utf-8\") as fd:\n s = Spec.from_signed_json(fd.read())\n check_spec(s)\n\n\n@pytest.mark.parametrize(\n \"invalid_yaml\", [\"playing_playlist: {{ action }} playlist {{ playlist_name }}\"]\n)\ndef test_invalid_yaml_spec(invalid_yaml):\n with pytest.raises(SpackYAMLError, match=\"error parsing YAML\") as e:\n Spec.from_yaml(invalid_yaml)\n assert invalid_yaml in str(e)\n\n\n@pytest.mark.parametrize(\"invalid_json, error_message\", [(\"{13:\", \"Expecting property name\")])\ndef test_invalid_json_spec(invalid_json, error_message):\n with pytest.raises(sjson.SpackJSONError) as e:\n Spec.from_json(invalid_json)\n exc_msg = str(e.value)\n assert exc_msg.startswith(\"error parsing JSON spec:\")\n assert error_message in exc_msg\n\n\n@pytest.mark.parametrize(\n \"abstract_spec\",\n [\n # Externals\n \"externaltool\",\n \"externaltest\",\n # Ambiguous version spec\n \"mpileaks@1.0:5.0,6.1,7.3+debug~opt\",\n # Variants\n \"mpileaks+debug~opt\",\n 'multivalue-variant foo=\"bar,baz\"',\n # Virtuals on edges\n \"callpath\",\n \"mpileaks\",\n ],\n)\ndef test_roundtrip_concrete_specs(abstract_spec, default_mock_concretization):\n check_yaml_round_trip(Spec(abstract_spec))\n check_json_round_trip(Spec(abstract_spec))\n concrete_spec = default_mock_concretization(abstract_spec)\n check_yaml_round_trip(concrete_spec)\n check_json_round_trip(concrete_spec)\n\n\ndef test_yaml_subdag(config, mock_packages):\n spec = spack.concretize.concretize_one(\"mpileaks^mpich+debug\")\n yaml_spec = Spec.from_yaml(spec.to_yaml())\n json_spec = Spec.from_json(spec.to_json())\n\n for dep in (\"callpath\", \"mpich\", \"dyninst\", \"libdwarf\", \"libelf\"):\n assert spec[dep].eq_dag(yaml_spec[dep])\n assert spec[dep].eq_dag(json_spec[dep])\n\n\n@pytest.mark.parametrize(\"spec_str\", [\"mpileaks ^zmpi\", \"dttop\", \"dtuse\"])\ndef test_using_ordered_dict(default_mock_concretization, spec_str):\n \"\"\"Checks that we use syaml_dicts for spec serialization.\n\n Necessary to make sure that dag_hash is stable across python\n versions and processes.\n \"\"\"\n\n def descend_and_check(iterable, level=0):\n if isinstance(iterable, collections.abc.Mapping):\n assert type(iterable) in (syaml_dict, dict)\n return descend_and_check(iterable.values(), level=level + 1)\n max_level = level\n for value in iterable:\n if isinstance(value, collections.abc.Iterable) and not isinstance(value, str):\n nlevel = descend_and_check(value, level=level + 1)\n if nlevel > max_level:\n max_level = nlevel\n return max_level\n\n s = default_mock_concretization(spec_str)\n level = descend_and_check(s.to_node_dict())\n # level just makes sure we are doing something here\n assert level >= 5\n\n\n@pytest.mark.parametrize(\"spec_str\", [\"mpileaks ^zmpi\", \"dttop\", \"dtuse\"])\ndef test_ordered_read_not_required_for_consistent_dag_hash(\n spec_str, mutable_config: spack.config.Configuration, mock_packages\n):\n \"\"\"Make sure ordered serialization isn't required to preserve hashes.\n\n For consistent hashes, we require that YAML and JSON serializations have their keys in a\n deterministic order. However, we don't want to require them to be serialized in order. This\n ensures that is not required.\"\"\"\n\n # Make sure that `extra_attributes` of externals is order independent for hashing.\n extra_attributes = {\n \"compilers\": {\"c\": \"/some/path/bin/cc\", \"cxx\": \"/some/path/bin/c++\"},\n \"foo\": \"bar\",\n \"baz\": \"qux\",\n }\n mutable_config.set(\n \"packages:dtuse\",\n {\n \"buildable\": False,\n \"externals\": [\n {\"spec\": \"dtuse@=1.0\", \"prefix\": \"/usr\", \"extra_attributes\": extra_attributes}\n ],\n },\n )\n\n spec = spack.concretize.concretize_one(spec_str)\n\n if spec_str == \"dtuse\":\n assert spec.external and spec.extra_attributes == extra_attributes\n\n spec_dict = spec.to_dict(hash=ht.dag_hash)\n spec_yaml = spec.to_yaml()\n spec_json = spec.to_json()\n\n # Make a spec with dict keys reversed recursively\n spec_dict_rev = reverse_all_dicts(spec_dict)\n\n # Dump to YAML and JSON\n yaml_string = syaml.dump(spec_dict, default_flow_style=False)\n yaml_string_rev = syaml.dump(spec_dict_rev, default_flow_style=False)\n json_string = sjson.dump(spec_dict)\n json_string_rev = sjson.dump(spec_dict_rev)\n\n # spec yaml is ordered like the spec dict\n assert yaml_string == spec_yaml\n assert json_string == spec_json\n\n # reversed string is different from the original, so it *would* generate a different hash\n assert yaml_string != yaml_string_rev\n assert json_string != json_string_rev\n\n # build specs from the \"wrongly\" ordered data\n from_yaml = Spec.from_yaml(yaml_string)\n from_json = Spec.from_json(json_string)\n from_yaml_rev = Spec.from_yaml(yaml_string_rev)\n from_json_rev = Spec.from_json(json_string_rev)\n\n # Strip spec if we stripped the yaml\n spec = spec.copy(deps=ht.dag_hash.depflag)\n\n # specs and their hashes are equal to the original\n assert (\n spec.dag_hash()\n == from_yaml.dag_hash()\n == from_json.dag_hash()\n == from_yaml_rev.dag_hash()\n == from_json_rev.dag_hash()\n )\n assert spec == from_yaml == from_json == from_yaml_rev == from_json_rev\n\n\ndef reverse_all_dicts(data):\n \"\"\"Descend into data and reverse all the dictionaries\"\"\"\n if isinstance(data, dict):\n return type(data)((k, reverse_all_dicts(v)) for k, v in reversed(list(data.items())))\n elif isinstance(data, (list, tuple)):\n return type(data)(reverse_all_dicts(elt) for elt in data)\n return data\n\n\ndef check_specs_equal(original_spec, spec_yaml_path):\n with open(spec_yaml_path, \"r\", encoding=\"utf-8\") as fd:\n spec_yaml = fd.read()\n spec_from_yaml = Spec.from_yaml(spec_yaml)\n return original_spec.eq_dag(spec_from_yaml)\n\n\ndef test_save_dependency_spec_jsons_subset(\n tmp_path: pathlib.Path, config, repo_builder: RepoBuilder\n):\n output_path = tmp_path / \"spec_jsons\"\n output_path.mkdir()\n\n repo_builder.add_package(\"pkg-g\")\n repo_builder.add_package(\"pkg-f\")\n repo_builder.add_package(\"pkg-e\")\n repo_builder.add_package(\"pkg-d\", dependencies=[(\"pkg-f\", None, None), (\"pkg-g\", None, None)])\n repo_builder.add_package(\"pkg-c\")\n repo_builder.add_package(\"pkg-b\", dependencies=[(\"pkg-d\", None, None), (\"pkg-e\", None, None)])\n repo_builder.add_package(\"pkg-a\", dependencies=[(\"pkg-b\", None, None), (\"pkg-c\", None, None)])\n\n with spack.repo.use_repositories(repo_builder.root):\n spec_a = spack.concretize.concretize_one(\"pkg-a\")\n b_spec = spec_a[\"pkg-b\"]\n c_spec = spec_a[\"pkg-c\"]\n\n save_dependency_specfiles(spec_a, str(output_path), [Spec(\"pkg-b\"), Spec(\"pkg-c\")])\n\n assert check_specs_equal(b_spec, str(output_path / \"pkg-b.json\"))\n assert check_specs_equal(c_spec, str(output_path / \"pkg-c.json\"))\n\n\ndef test_legacy_yaml(install_mockery, mock_packages):\n \"\"\"Tests a simple legacy YAML with a dependency and ensures spec survives\n concretization.\"\"\"\n yaml = \"\"\"\nspec:\n- a:\n version: '2.0'\n arch:\n platform: linux\n platform_os: rhel7\n target: x86_64\n compiler:\n name: gcc\n version: 8.3.0\n namespace: builtin.mock\n parameters:\n bvv: true\n foo:\n - bar\n foobar: bar\n cflags: []\n cppflags: []\n cxxflags: []\n fflags: []\n ldflags: []\n ldlibs: []\n dependencies:\n b:\n hash: iaapywazxgetn6gfv2cfba353qzzqvhn\n type:\n - build\n - link\n hash: obokmcsn3hljztrmctbscmqjs3xclazz\n full_hash: avrk2tqsnzxeabmxa6r776uq7qbpeufv\n build_hash: obokmcsn3hljztrmctbscmqjs3xclazy\n- b:\n version: '1.0'\n arch:\n platform: linux\n platform_os: rhel7\n target: x86_64\n compiler:\n name: gcc\n version: 8.3.0\n namespace: builtin.mock\n parameters:\n cflags: []\n cppflags: []\n cxxflags: []\n fflags: []\n ldflags: []\n ldlibs: []\n hash: iaapywazxgetn6gfv2cfba353qzzqvhn\n full_hash: qvsxvlmjaothtpjluqijv7qfnni3kyyg\n build_hash: iaapywazxgetn6gfv2cfba353qzzqvhy\n\"\"\"\n spec = Spec.from_yaml(yaml)\n concrete_spec = spack.concretize.concretize_one(spec)\n assert concrete_spec.eq_dag(spec)\n\n\n#: A well ordered Spec dictionary, using ``OrderdDict``.\n#: Any operation that transforms Spec dictionaries should\n#: preserve this order.\nordered_spec = collections.OrderedDict(\n [\n (\n \"arch\",\n collections.OrderedDict(\n [\n (\"platform\", \"darwin\"),\n (\"platform_os\", \"bigsur\"),\n (\n \"target\",\n collections.OrderedDict(\n [\n (\n \"features\",\n [\n \"adx\",\n \"aes\",\n \"avx\",\n \"avx2\",\n \"bmi1\",\n \"bmi2\",\n \"clflushopt\",\n \"f16c\",\n \"fma\",\n \"mmx\",\n \"movbe\",\n \"pclmulqdq\",\n \"popcnt\",\n \"rdrand\",\n \"rdseed\",\n \"sse\",\n \"sse2\",\n \"sse4_1\",\n \"sse4_2\",\n \"ssse3\",\n \"xsavec\",\n \"xsaveopt\",\n ],\n ),\n (\"generation\", 0),\n (\"name\", \"skylake\"),\n (\"parents\", [\"broadwell\"]),\n (\"vendor\", \"GenuineIntel\"),\n ]\n ),\n ),\n ]\n ),\n ),\n (\"compiler\", collections.OrderedDict([(\"name\", \"apple-clang\"), (\"version\", \"13.0.0\")])),\n (\"name\", \"zlib\"),\n (\"namespace\", \"builtin\"),\n (\n \"parameters\",\n collections.OrderedDict(\n [\n (\"cflags\", []),\n (\"cppflags\", []),\n (\"cxxflags\", []),\n (\"fflags\", []),\n (\"ldflags\", []),\n (\"ldlibs\", []),\n (\"optimize\", True),\n (\"pic\", True),\n (\"shared\", True),\n ]\n ),\n ),\n (\"version\", \"1.2.11\"),\n ]\n)\n\n\n@pytest.mark.parametrize(\n \"specfile,expected_hash,reader_cls\",\n [\n # First version supporting JSON format for specs\n (\"specfiles/hdf5.v013.json.gz\", \"vglgw4reavn65vx5d4dlqn6rjywnq76d\", spack.spec.SpecfileV1),\n # Introduces full hash in the format, still has 3 hashes\n (\"specfiles/hdf5.v016.json.gz\", \"stp45yvzte43xdauknaj3auxlxb4xvzs\", spack.spec.SpecfileV1),\n # Introduces \"build_specs\", see https://github.com/spack/spack/pull/22845\n (\"specfiles/hdf5.v017.json.gz\", \"xqh5iyjjtrp2jw632cchacn3l7vqzf3m\", spack.spec.SpecfileV2),\n # Use \"full hash\" everywhere, see https://github.com/spack/spack/pull/28504\n (\"specfiles/hdf5.v019.json.gz\", \"iulacrbz7o5v5sbj7njbkyank3juh6d3\", spack.spec.SpecfileV3),\n # Add properties on edges, see https://github.com/spack/spack/pull/34821\n (\"specfiles/hdf5.v020.json.gz\", \"vlirlcgazhvsvtundz4kug75xkkqqgou\", spack.spec.SpecfileV4),\n ],\n)\ndef test_load_json_specfiles(specfile, expected_hash, reader_cls):\n fullpath = os.path.join(spack.paths.test_path, \"data\", specfile)\n with gzip.open(fullpath, \"rt\", encoding=\"utf-8\") as f:\n data = json.load(f)\n\n s1 = Spec.from_dict(data)\n s2 = reader_cls.load(data)\n\n assert s2.dag_hash() == expected_hash\n assert s1.dag_hash() == s2.dag_hash()\n assert s1 == s2\n assert Spec.from_json(s2.to_json()).dag_hash() == s2.dag_hash()\n\n openmpi_edges = s2.edges_to_dependencies(name=\"openmpi\")\n assert len(openmpi_edges) == 1\n\n # Check that virtuals have been reconstructed for specfiles conforming to\n # version 4 on.\n if reader_cls.SPEC_VERSION >= spack.spec.SpecfileV4.SPEC_VERSION:\n assert \"mpi\" in openmpi_edges[0].virtuals\n\n # The virtuals attribute must be a tuple, when read from a\n # JSON or YAML file, not a list\n for edge in s2.traverse_edges():\n assert isinstance(edge.virtuals, tuple), edge\n\n # Ensure we can format {compiler} tokens\n assert s2.format(\"{compiler}\") != \"none\"\n assert s2.format(\"{compiler.name}\") == \"gcc\"\n assert s2.format(\"{compiler.version}\") != \"none\"\n\n # Ensure satisfies works with compilers and direct dependencies\n assert s2.satisfies(\"%gcc\")\n assert s2.satisfies(\"%gcc@9.4.0\")\n assert s2.satisfies(\"%zlib\")\n\n\ndef test_anchorify_1():\n \"\"\"Test that anchorify replaces duplicate values with references to a single instance, and\n that that results in anchors in the output YAML.\"\"\"\n before = {\"a\": [1, 2, 3], \"b\": [1, 2, 3]}\n after = {\"a\": [1, 2, 3], \"b\": [1, 2, 3]}\n syaml.anchorify(after)\n assert before == after\n assert after[\"a\"] is after[\"b\"]\n\n # Check if anchors are used\n out = io.StringIO()\n spack.vendor.ruamel.yaml.YAML().dump(after, out)\n assert (\n out.getvalue()\n == \"\"\"\\\na: &id001\n- 1\n- 2\n- 3\nb: *id001\n\"\"\"\n )\n\n\ndef test_anchorify_2():\n before = {\"a\": {\"b\": {\"c\": True}}, \"d\": {\"b\": {\"c\": True}}, \"e\": {\"c\": True}}\n after = {\"a\": {\"b\": {\"c\": True}}, \"d\": {\"b\": {\"c\": True}}, \"e\": {\"c\": True}}\n syaml.anchorify(after)\n assert before == after\n assert after[\"a\"] is after[\"d\"]\n assert after[\"a\"][\"b\"] is after[\"e\"]\n\n # Check if anchors are used\n out = io.StringIO()\n spack.vendor.ruamel.yaml.YAML().dump(after, out)\n assert (\n out.getvalue()\n == \"\"\"\\\na: &id001\n b: &id002\n c: true\nd: *id001\ne: *id002\n\"\"\"\n )\n\n\n@pytest.mark.parametrize(\n \"spec_str\",\n [\n \"hdf5 ++mpi\",\n \"hdf5 cflags==-g\",\n \"hdf5 foo==bar\",\n \"hdf5~~mpi++shared\",\n \"hdf5 cflags==-g foo==bar cxxflags==-O3\",\n \"hdf5 cflags=-g foo==bar cxxflags==-O3\",\n \"hdf5%gcc\",\n \"hdf5%cmake\",\n \"hdf5^gcc\",\n \"hdf5^cmake\",\n ],\n)\ndef test_pickle_roundtrip_for_abstract_specs(spec_str):\n \"\"\"Tests that abstract specs correctly round trip when pickled.\n\n This test compares both spec objects and their string representation, due to some\n inconsistencies in how `Spec.__eq__` is implemented.\n \"\"\"\n s = spack.spec.Spec(spec_str)\n t = pickle.loads(pickle.dumps(s))\n assert s == t\n assert str(s) == str(t)\n\n\ndef test_specfile_alias_is_updated():\n \"\"\"Tests that the SpecfileLatest alias gets updated on a Specfile version bump\"\"\"\n specfile_class_name = f\"SpecfileV{spack.spec.SPECFILE_FORMAT_VERSION}\"\n specfile_cls = getattr(spack.spec, specfile_class_name)\n assert specfile_cls is spack.spec.SpecfileLatest\n\n\n@pytest.mark.parametrize(\"spec_str\", [\"mpileaks %gcc\", \"mpileaks ^zmpi ^callpath%gcc\"])\ndef test_direct_edges_and_round_tripping_to_dict(spec_str, default_mock_concretization):\n \"\"\"Tests that we preserve edge information when round-tripping to dict\"\"\"\n original = Spec(spec_str)\n reconstructed = Spec.from_dict(original.to_dict())\n assert original == reconstructed\n assert original.to_dict() == reconstructed.to_dict()\n\n concrete = default_mock_concretization(spec_str)\n concrete_reconstructed = Spec.from_dict(concrete.to_dict())\n assert concrete == concrete_reconstructed\n assert concrete.to_dict() == concrete_reconstructed.to_dict()\n\n # Ensure we don't get 'direct' in concrete JSON specs, for the time being\n d = concrete.to_dict()\n for node in d[\"spec\"][\"nodes\"]:\n if \"dependencies\" not in node:\n continue\n for dependency_data in node[\"dependencies\"]:\n assert \"direct\" not in dependency_data[\"parameters\"]\n\n\ndef test_pickle_preserves_identity_and_prefix(default_mock_concretization):\n \"\"\"When pickling multiple specs that share dependencies, the identity of those dependencies\n should be preserved when unpickling.\"\"\"\n mpileaks_before: Spec = default_mock_concretization(\"mpileaks\")\n callpath_before = mpileaks_before.dependencies(\"callpath\")[0]\n callpath_before.set_prefix(\"/fake/prefix/callpath\")\n specs_before = [mpileaks_before, callpath_before]\n specs_after = pickle.loads(pickle.dumps(specs_before))\n mpileaks_after, callpath_after = specs_after\n\n # Test whether the mpileaks<->callpath link is preserved and corresponds to the same object\n assert mpileaks_after is callpath_after.dependents(\"mpileaks\")[0]\n assert callpath_after is mpileaks_after.dependencies(\"callpath\")[0]\n\n # Test that we have the exact same number of unique Spec objects before and after pickling\n num_unique_specs = lambda specs: len({id(s) for r in specs for s in r.traverse()})\n assert num_unique_specs(specs_before) == num_unique_specs(specs_after)\n\n # Test that the specs are the same as dicts\n assert mpileaks_before.to_dict() == mpileaks_after.to_dict()\n" }, { "path": "lib/spack/spack/test/stage.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Test that the Stage class works correctly.\"\"\"\n\nimport collections\nimport errno\nimport getpass\nimport os\nimport pathlib\nimport shutil\nimport stat\nimport sys\n\nimport pytest\n\nimport spack.config\nimport spack.error\nimport spack.fetch_strategy\nimport spack.stage\nimport spack.util.executable\nimport spack.util.path\nimport spack.util.url as url_util\nfrom spack.llnl.util.filesystem import getuid, mkdirp, partition_path, readlink, touch, working_dir\nfrom spack.resource import Resource\nfrom spack.stage import DevelopStage, ResourceStage, Stage, StageComposite\nfrom spack.util.path import canonicalize_path\n\n# The following values are used for common fetch and stage mocking fixtures:\n_archive_base = \"test-files\"\n_archive_fn = \"%s.tar.gz\" % _archive_base\n_extra_fn = \"extra.sh\"\n_hidden_fn = \".hidden\"\n_readme_fn = \"README.txt\"\n\n_extra_contents = \"#!/bin/sh\\n\"\n_hidden_contents = \"\"\n_readme_contents = \"hello world!\\n\"\n\n# TODO: Replace the following with an enum once guarantee supported (or\n# include enum34 for python versions < 3.4.\n_include_readme = 1\n_include_hidden = 2\n_include_extra = 3\n\n\n# Mock fetch directories are expected to appear as follows:\n#\n# TMPDIR/\n# _archive_fn archive_url = file:///path/to/_archive_fn\n#\n# Mock expanded stage directories are expected to have one of two forms,\n# depending on how the tarball expands. Non-exploding tarballs are expected\n# to have the following structure:\n#\n# TMPDIR/ temp stage dir\n# spack-src/ well-known stage source directory\n# _readme_fn Optional test_readme (contains _readme_contents)\n# _hidden_fn Optional hidden file (contains _hidden_contents)\n# _archive_fn archive_url = file:///path/to/_archive_fn\n#\n# while exploding tarball directories are expected to be structured as follows:\n#\n# TMPDIR/ temp stage dir\n# spack-src/ well-known stage source directory\n# archive_name/ archive dir\n# _readme_fn test_readme (contains _readme_contents)\n# _extra_fn test_extra file (contains _extra_contents)\n# _archive_fn archive_url = file:///path/to/_archive_fn\n#\n\n\n@pytest.fixture\ndef clear_stage_root(monkeypatch):\n \"\"\"Ensure spack.stage._stage_root is not set at test start.\"\"\"\n monkeypatch.setattr(spack.stage, \"_stage_root\", None)\n yield\n\n\ndef check_expand_archive(stage, stage_name, expected_file_list):\n \"\"\"\n Ensure the expanded archive directory contains the expected structure and\n files as described in the module-level comments above.\n \"\"\"\n stage_path = get_stage_path(stage, stage_name)\n archive_dir = spack.stage._source_path_subdir\n\n stage_contents = os.listdir(stage_path)\n assert _archive_fn in stage_contents\n assert archive_dir in stage_contents\n\n source_path = os.path.join(stage_path, archive_dir)\n assert source_path == stage.source_path\n\n source_contents = os.listdir(source_path)\n\n for _include in expected_file_list:\n if _include == _include_hidden:\n # The hidden file represent the HFS metadata associated with Mac\n # OS X tar files so is expected to be in the same directory as\n # the archive directory.\n assert _hidden_fn in stage_contents\n\n fn = os.path.join(stage_path, _hidden_fn)\n contents = _hidden_contents\n\n elif _include == _include_readme:\n # The standard README.txt file will be in the source directory if\n # the tarball didn't explode; otherwise, it will be in the\n # original archive subdirectory of it.\n if _archive_base in source_contents:\n fn = os.path.join(source_path, _archive_base, _readme_fn)\n else:\n fn = os.path.join(source_path, _readme_fn)\n contents = _readme_contents\n\n elif _include == _include_extra:\n assert _extra_fn in source_contents\n\n fn = os.path.join(source_path, _extra_fn)\n contents = _extra_contents\n\n else:\n assert False\n\n assert os.path.isfile(fn)\n with open(fn, encoding=\"utf-8\") as _file:\n assert _file.read() == contents\n\n\ndef check_fetch(stage, stage_name):\n \"\"\"\n Ensure the fetch resulted in a properly placed archive file as described in\n the module-level comments.\n \"\"\"\n stage_path = get_stage_path(stage, stage_name)\n assert _archive_fn in os.listdir(stage_path)\n assert os.path.join(stage_path, _archive_fn) == stage.fetcher.archive_file\n\n\ndef check_destroy(stage, stage_name):\n \"\"\"Figure out whether a stage was destroyed correctly.\"\"\"\n stage_path = get_stage_path(stage, stage_name)\n\n # check that the stage dir/link was removed.\n assert not os.path.exists(stage_path)\n\n # tmp stage needs to remove tmp dir too.\n target = os.path.realpath(stage_path)\n assert not os.path.exists(target)\n\n\ndef check_setup(stage, stage_name, archive):\n \"\"\"Figure out whether a stage was set up correctly.\"\"\"\n stage_path = get_stage_path(stage, stage_name)\n\n # Ensure stage was created in the spack stage directory\n assert os.path.isdir(stage_path)\n\n # Make sure it points to a valid directory\n target = os.path.realpath(stage_path)\n assert os.path.isdir(target)\n assert not os.path.islink(target)\n\n # Make sure the directory is in the place we asked it to\n # be (see setUp, tearDown, and use_tmp)\n assert target.startswith(str(archive.stage_path))\n\n\ndef get_stage_path(stage, stage_name):\n \"\"\"Figure out where a stage should be living. This depends on\n whether it's named.\n \"\"\"\n stage_path = spack.stage.get_stage_root()\n if stage_name is not None:\n # If it is a named stage, we know where the stage should be\n return os.path.join(stage_path, stage_name)\n else:\n # If it's unnamed, ensure that we ran mkdtemp in the right spot.\n assert stage.path is not None\n assert stage.path.startswith(stage_path)\n return stage.path\n\n\n# TODO: Revisit use of the following fixture (and potentially leveraging\n# the `mock_stage` path in `mock_stage_archive`) per discussions in\n# #12857. See also #13065.\n@pytest.fixture\ndef tmp_build_stage_dir(tmp_path: pathlib.Path, clear_stage_root):\n \"\"\"Use a temporary test directory for the stage root.\"\"\"\n test_path = str(tmp_path / \"stage\")\n with spack.config.override(\"config:build_stage\", test_path):\n yield tmp_path, spack.stage.get_stage_root()\n\n shutil.rmtree(test_path)\n\n\n@pytest.fixture\ndef mock_stage_archive(tmp_build_stage_dir):\n \"\"\"Create the directories and files for the staged mock archive.\"\"\"\n\n # Mock up a stage area that looks like this:\n #\n # tmp_path/ test_files_dir\n # stage/ test_stage_path (where stage should be)\n # <_archive_base>/ archive_dir_path\n # <_readme_fn> Optional test_readme (contains _readme_contents)\n # <_extra_fn> Optional extra file (contains _extra_contents)\n # <_hidden_fn> Optional hidden file (contains _hidden_contents)\n # <_archive_fn> archive_url = file:///path/to/<_archive_fn>\n #\n def create_stage_archive(expected_file_list=[_include_readme]):\n tmp_build_dir, test_stage_path = tmp_build_stage_dir\n mkdirp(test_stage_path)\n\n # Create the archive directory and associated file\n archive_dir = tmp_build_dir / _archive_base\n archive = tmp_build_dir / _archive_fn\n archive_url = url_util.path_to_file_url(str(archive))\n archive_dir.mkdir(exist_ok=True)\n\n # Create the optional files as requested and make sure expanded\n # archive peers are included.\n tar_args = [\"czf\", str(_archive_fn), _archive_base]\n for _include in expected_file_list:\n if _include == _include_hidden:\n # The hidden file case stands in for the way Mac OS X tar files\n # represent HFS metadata. Locate in the same directory as the\n # archive file.\n tar_args.append(_hidden_fn)\n fn, contents = (tmp_build_dir / _hidden_fn, _hidden_contents)\n\n elif _include == _include_readme:\n # The usual README.txt file is contained in the archive dir.\n fn, contents = (archive_dir / _readme_fn, _readme_contents)\n\n elif _include == _include_extra:\n # The extra file stands in for exploding tar files so needs\n # to be in the same directory as the archive file.\n tar_args.append(_extra_fn)\n fn, contents = (tmp_build_dir / _extra_fn, _extra_contents)\n else:\n break\n\n fn.write_text(contents)\n\n # Create the archive file\n with working_dir(str(tmp_build_dir)):\n tar = spack.util.executable.which(\"tar\", required=True)\n tar(*tar_args)\n\n Archive = collections.namedtuple(\"Archive\", [\"url\", \"tmpdir\", \"stage_path\", \"archive_dir\"])\n return Archive(\n url=archive_url,\n tmpdir=tmp_build_dir,\n stage_path=test_stage_path,\n archive_dir=archive_dir,\n )\n\n return create_stage_archive\n\n\n@pytest.fixture\ndef mock_noexpand_resource(tmp_path: pathlib.Path):\n \"\"\"Set up a non-expandable resource in the tmp_path prior to staging.\"\"\"\n test_resource = tmp_path / \"resource-no-expand.sh\"\n test_resource.write_text(\"an example resource\")\n return str(test_resource)\n\n\n@pytest.fixture\ndef mock_expand_resource(tmp_path: pathlib.Path):\n \"\"\"Sets up an expandable resource in tmp_path prior to staging.\"\"\"\n # Mock up an expandable resource:\n #\n # tmp_path/ test_files_dir\n # resource-expand/ resource source dir\n # resource-file.txt resource contents (contains 'test content')\n # resource.tar.gz archive of resource content\n #\n subdir = \"resource-expand\"\n resource_dir = tmp_path / subdir\n resource_dir.mkdir()\n\n archive_name = \"resource.tar.gz\"\n archive = tmp_path / archive_name\n archive_url = url_util.path_to_file_url(str(archive))\n\n filename = \"resource-file.txt\"\n test_file = resource_dir / filename\n test_file.write_text(\"test content\\n\")\n\n with working_dir(str(tmp_path)):\n tar = spack.util.executable.which(\"tar\", required=True)\n tar(\"czf\", str(archive_name), subdir)\n\n MockResource = collections.namedtuple(\"MockResource\", [\"url\", \"files\"])\n\n return MockResource(archive_url, [filename])\n\n\n@pytest.fixture\ndef composite_stage_with_expanding_resource(mock_stage_archive, mock_expand_resource):\n \"\"\"Sets up a composite for expanding resources prior to staging.\"\"\"\n composite_stage = StageComposite()\n archive = mock_stage_archive()\n root_stage = Stage(archive.url)\n composite_stage.append(root_stage)\n\n test_resource_fetcher = spack.fetch_strategy.from_kwargs(url=mock_expand_resource.url)\n # Specify that the resource files are to be placed in the 'resource-dir'\n # directory\n test_resource = Resource(\"test_resource\", test_resource_fetcher, \"\", \"resource-dir\")\n resource_stage = ResourceStage(test_resource_fetcher, root_stage, test_resource)\n composite_stage.append(resource_stage)\n return composite_stage, root_stage, resource_stage, mock_expand_resource\n\n\n@pytest.fixture\ndef failing_search_fn():\n \"\"\"Returns a search function that fails! Always!\"\"\"\n\n def _mock():\n raise Exception(\"This should not have been called\")\n\n return _mock\n\n\nclass FailingFetchStrategy(spack.fetch_strategy.FetchStrategy):\n def fetch(self):\n raise spack.fetch_strategy.FailedDownloadError(\n \"\", \"This implementation of FetchStrategy always fails\"\n )\n\n\n@pytest.fixture\ndef search_fn():\n \"\"\"Returns a search function that always succeeds.\"\"\"\n\n class _Mock:\n performed_search = False\n\n def __call__(self):\n self.performed_search = True\n return []\n\n return _Mock()\n\n\ndef check_stage_dir_perms(prefix, path):\n \"\"\"Check the stage directory perms to ensure match expectations.\"\"\"\n # Ensure the path's subdirectories -- to `$user` -- have their parent's\n # perms while those from `$user` on are owned and restricted to the\n # user.\n assert path.startswith(prefix)\n\n user = getpass.getuser()\n prefix_status = os.stat(prefix)\n uid = getuid()\n\n # Obtain lists of ancestor and descendant paths of the $user node, if any.\n #\n # Skip processing prefix ancestors since no guarantee they will be in the\n # required group (e.g. $TEMPDIR on HPC machines).\n skip = prefix if prefix.endswith(os.sep) else prefix + os.sep\n group_paths, user_node, user_paths = partition_path(path.replace(skip, \"\"), user)\n\n for p in group_paths:\n p_status = os.stat(os.path.join(prefix, p))\n assert p_status.st_gid == prefix_status.st_gid\n assert p_status.st_mode == prefix_status.st_mode\n\n # Add the path ending with the $user node to the user paths to ensure paths\n # from $user (on down) meet the ownership and permission requirements.\n if user_node:\n user_paths.insert(0, user_node)\n\n for p in user_paths:\n p_status = os.stat(os.path.join(prefix, p))\n assert uid == p_status.st_uid\n assert p_status.st_mode & stat.S_IRWXU == stat.S_IRWXU\n\n\n@pytest.mark.usefixtures(\"mock_packages\")\nclass TestStage:\n stage_name = \"spack-test-stage\"\n\n def test_setup_and_destroy_name_with_tmp(self, mock_stage_archive):\n archive = mock_stage_archive()\n with Stage(archive.url, name=self.stage_name) as stage:\n check_setup(stage, self.stage_name, archive)\n check_destroy(stage, self.stage_name)\n\n def test_setup_and_destroy_name_without_tmp(self, mock_stage_archive):\n archive = mock_stage_archive()\n with Stage(archive.url, name=self.stage_name) as stage:\n check_setup(stage, self.stage_name, archive)\n check_destroy(stage, self.stage_name)\n\n def test_setup_and_destroy_no_name_with_tmp(self, mock_stage_archive):\n archive = mock_stage_archive()\n with Stage(archive.url) as stage:\n check_setup(stage, None, archive)\n check_destroy(stage, None)\n\n def test_noexpand_stage_file(self, mock_stage_archive, mock_noexpand_resource):\n \"\"\"When creating a stage with a nonexpanding URL, the 'archive_file'\n property of the stage should refer to the path of that file.\n \"\"\"\n test_noexpand_fetcher = spack.fetch_strategy.from_kwargs(\n url=url_util.path_to_file_url(mock_noexpand_resource), expand=False\n )\n with Stage(test_noexpand_fetcher) as stage:\n stage.fetch()\n stage.expand_archive()\n assert os.path.exists(stage.archive_file)\n\n @pytest.mark.disable_clean_stage_check\n def test_composite_stage_with_noexpand_resource(\n self, mock_stage_archive, mock_noexpand_resource\n ):\n archive = mock_stage_archive()\n composite_stage = StageComposite()\n root_stage = Stage(archive.url)\n composite_stage.append(root_stage)\n\n resource_dst_name = \"resource-dst-name.sh\"\n test_resource_fetcher = spack.fetch_strategy.from_kwargs(\n url=url_util.path_to_file_url(mock_noexpand_resource), expand=False\n )\n test_resource = Resource(\"test_resource\", test_resource_fetcher, resource_dst_name, None)\n resource_stage = ResourceStage(test_resource_fetcher, root_stage, test_resource)\n composite_stage.append(resource_stage)\n\n composite_stage.create()\n composite_stage.fetch()\n composite_stage.expand_archive()\n assert composite_stage.expanded # Archive is expanded\n\n assert os.path.exists(os.path.join(composite_stage.source_path, resource_dst_name))\n\n @pytest.mark.disable_clean_stage_check\n def test_composite_stage_with_expand_resource(self, composite_stage_with_expanding_resource):\n (composite_stage, root_stage, resource_stage, mock_resource) = (\n composite_stage_with_expanding_resource\n )\n\n composite_stage.create()\n composite_stage.fetch()\n composite_stage.expand_archive()\n\n assert composite_stage.expanded # Archive is expanded\n\n for fname in mock_resource.files:\n file_path = os.path.join(root_stage.source_path, \"resource-dir\", fname)\n assert os.path.exists(file_path)\n\n # Perform a little cleanup\n shutil.rmtree(root_stage.path)\n\n @pytest.mark.disable_clean_stage_check\n def test_composite_stage_with_expand_resource_default_placement(\n self, composite_stage_with_expanding_resource\n ):\n \"\"\"For a resource which refers to a compressed archive which expands\n to a directory, check that by default the resource is placed in\n the source_path of the root stage with the name of the decompressed\n directory.\n \"\"\"\n\n (composite_stage, root_stage, resource_stage, mock_resource) = (\n composite_stage_with_expanding_resource\n )\n\n resource_stage.resource.placement = None\n\n composite_stage.create()\n composite_stage.fetch()\n composite_stage.expand_archive()\n\n for fname in mock_resource.files:\n file_path = os.path.join(root_stage.source_path, \"resource-expand\", fname)\n assert os.path.exists(file_path)\n\n # Perform a little cleanup\n shutil.rmtree(root_stage.path)\n\n def test_setup_and_destroy_no_name_without_tmp(self, mock_stage_archive):\n archive = mock_stage_archive()\n with Stage(archive.url) as stage:\n check_setup(stage, None, archive)\n check_destroy(stage, None)\n\n @pytest.mark.parametrize(\"debug\", [False, True])\n def test_fetch(self, mock_stage_archive, debug):\n archive = mock_stage_archive()\n with spack.config.override(\"config:debug\", debug):\n with Stage(archive.url, name=self.stage_name) as stage:\n stage.fetch()\n check_setup(stage, self.stage_name, archive)\n check_fetch(stage, self.stage_name)\n check_destroy(stage, self.stage_name)\n\n def test_no_search_if_default_succeeds(self, mock_stage_archive, failing_search_fn):\n archive = mock_stage_archive()\n stage = Stage(archive.url, name=self.stage_name, search_fn=failing_search_fn)\n with stage:\n stage.fetch()\n check_destroy(stage, self.stage_name)\n\n def test_no_search_mirror_only(self, failing_search_fn):\n stage = Stage(FailingFetchStrategy(), name=self.stage_name, search_fn=failing_search_fn)\n with stage:\n try:\n stage.fetch(mirror_only=True)\n except spack.error.FetchError:\n pass\n check_destroy(stage, self.stage_name)\n\n @pytest.mark.parametrize(\n \"err_msg,expected\",\n [\n (\"Fetch from fetch.test.com\", \"Fetch from fetch.test.com\"),\n (None, \"All fetchers failed\"),\n ],\n )\n def test_search_if_default_fails(self, search_fn, err_msg, expected):\n stage = Stage(FailingFetchStrategy(), name=self.stage_name, search_fn=search_fn)\n\n with stage:\n with pytest.raises(spack.error.FetchError, match=expected):\n stage.fetch(mirror_only=False, err_msg=err_msg)\n\n check_destroy(stage, self.stage_name)\n assert search_fn.performed_search\n\n def test_ensure_one_stage_entry(self, mock_stage_archive):\n archive = mock_stage_archive()\n with Stage(archive.url, name=self.stage_name) as stage:\n stage.fetch()\n stage_path = get_stage_path(stage, self.stage_name)\n spack.fetch_strategy._ensure_one_stage_entry(stage_path)\n check_destroy(stage, self.stage_name)\n\n @pytest.mark.parametrize(\n \"expected_file_list\",\n [\n [],\n [_include_readme],\n [_include_extra, _include_readme],\n [_include_hidden, _include_readme],\n ],\n )\n def test_expand_archive(self, expected_file_list, mock_stage_archive):\n archive = mock_stage_archive(expected_file_list)\n with Stage(archive.url, name=self.stage_name) as stage:\n stage.fetch()\n check_setup(stage, self.stage_name, archive)\n check_fetch(stage, self.stage_name)\n stage.expand_archive()\n check_expand_archive(stage, self.stage_name, expected_file_list)\n check_destroy(stage, self.stage_name)\n\n def test_expand_archive_extra_expand(self, mock_stage_archive):\n \"\"\"Test expand with an extra expand after expand (i.e., no-op).\"\"\"\n archive = mock_stage_archive()\n with Stage(archive.url, name=self.stage_name) as stage:\n stage.fetch()\n check_setup(stage, self.stage_name, archive)\n check_fetch(stage, self.stage_name)\n stage.expand_archive()\n stage.fetcher.expand()\n check_expand_archive(stage, self.stage_name, [_include_readme])\n check_destroy(stage, self.stage_name)\n\n def test_restage(self, mock_stage_archive):\n archive = mock_stage_archive()\n with Stage(archive.url, name=self.stage_name) as stage:\n stage.fetch()\n stage.expand_archive()\n\n with working_dir(stage.source_path):\n check_expand_archive(stage, self.stage_name, [_include_readme])\n\n # Try to make a file in the old archive dir\n with open(\"foobar\", \"w\", encoding=\"utf-8\") as file:\n file.write(\"this file is to be destroyed.\")\n\n assert \"foobar\" in os.listdir(stage.source_path)\n\n # Make sure the file is not there after restage.\n stage.restage()\n check_fetch(stage, self.stage_name)\n assert \"foobar\" not in os.listdir(stage.source_path)\n check_destroy(stage, self.stage_name)\n\n def test_no_keep_without_exceptions(self, mock_stage_archive):\n archive = mock_stage_archive()\n stage = Stage(archive.url, name=self.stage_name, keep=False)\n with stage:\n pass\n check_destroy(stage, self.stage_name)\n\n @pytest.mark.disable_clean_stage_check\n def test_keep_without_exceptions(self, mock_stage_archive):\n archive = mock_stage_archive()\n stage = Stage(archive.url, name=self.stage_name, keep=True)\n with stage:\n pass\n path = get_stage_path(stage, self.stage_name)\n assert os.path.isdir(path)\n\n @pytest.mark.disable_clean_stage_check\n def test_no_keep_with_exceptions(self, mock_stage_archive):\n class ThisMustFailHere(Exception):\n pass\n\n archive = mock_stage_archive()\n stage = Stage(archive.url, name=self.stage_name, keep=False)\n try:\n with stage:\n raise ThisMustFailHere()\n\n except ThisMustFailHere:\n path = get_stage_path(stage, self.stage_name)\n assert os.path.isdir(path)\n\n @pytest.mark.disable_clean_stage_check\n def test_keep_exceptions(self, mock_stage_archive):\n class ThisMustFailHere(Exception):\n pass\n\n archive = mock_stage_archive()\n stage = Stage(archive.url, name=self.stage_name, keep=True)\n try:\n with stage:\n raise ThisMustFailHere()\n\n except ThisMustFailHere:\n path = get_stage_path(stage, self.stage_name)\n assert os.path.isdir(path)\n\n def test_source_path_available(self, mock_stage_archive):\n \"\"\"Ensure source path available but does not exist on instantiation.\"\"\"\n archive = mock_stage_archive()\n stage = Stage(archive.url, name=self.stage_name)\n\n source_path = stage.source_path\n assert source_path\n assert source_path.endswith(spack.stage._source_path_subdir)\n assert not os.path.exists(source_path)\n\n @pytest.mark.not_on_windows(\"Windows file permission erroring is not yet supported\")\n @pytest.mark.skipif(getuid() == 0, reason=\"user is root\")\n def test_first_accessible_path(self, tmp_path: pathlib.Path):\n \"\"\"Test _first_accessible_path names.\"\"\"\n spack_dir = tmp_path / \"paths\"\n name = str(spack_dir)\n files = [os.path.join(os.path.sep, \"no\", \"such\", \"path\"), name]\n\n # Ensure the tmp_path path is returned since the user should have access\n path = spack.stage._first_accessible_path(files)\n assert path == name\n assert os.path.isdir(path)\n check_stage_dir_perms(str(tmp_path), path)\n\n # Ensure an existing path is returned\n spack_subdir = spack_dir / \"existing\"\n spack_subdir.mkdir(parents=True)\n subdir = str(spack_subdir)\n path = spack.stage._first_accessible_path([subdir])\n assert path == subdir\n\n # Ensure a path with a `$user` node has the right permissions\n # for its subdirectories.\n user = getpass.getuser()\n user_dir = spack_dir / user / \"has\" / \"paths\"\n user_path = str(user_dir)\n path = spack.stage._first_accessible_path([user_path])\n assert path == user_path\n check_stage_dir_perms(str(tmp_path), path)\n\n # Cleanup\n shutil.rmtree(str(name))\n\n def test_create_stage_root(self, tmp_path: pathlib.Path, no_path_access):\n \"\"\"Test create_stage_root permissions.\"\"\"\n test_dir = tmp_path / \"path\"\n test_path = str(test_dir)\n\n try:\n if getpass.getuser() in str(test_path).split(os.sep):\n # Simply ensure directory created if tmp_path includes user\n spack.stage.create_stage_root(test_path)\n assert os.path.exists(test_path)\n\n p_stat = os.stat(test_path)\n assert p_stat.st_mode & stat.S_IRWXU == stat.S_IRWXU\n else:\n # Ensure an OS Error is raised on created, non-user directory\n with pytest.raises(OSError) as exc_info:\n spack.stage.create_stage_root(test_path)\n\n assert exc_info.value.errno == errno.EACCES\n finally:\n try:\n shutil.rmtree(test_path)\n except OSError:\n pass\n\n def test_resolve_paths(self, monkeypatch):\n \"\"\"Test _resolve_paths.\"\"\"\n assert spack.stage._resolve_paths([]) == []\n\n user = \"testuser\"\n monkeypatch.setattr(spack.util.path, \"get_user\", lambda: user)\n\n # Test that user is appended to path if not present (except on Windows)\n if sys.platform == \"win32\":\n path = r\"C:\\spack-test\\a\\b\\c\"\n expected = path\n else:\n path = \"/spack-test/a/b/c\"\n expected = os.path.join(path, user)\n\n assert spack.stage._resolve_paths([path]) == [expected]\n\n # Test that user is NOT appended if already present\n if sys.platform == \"win32\":\n path_with_user = rf\"C:\\spack-test\\spack-{user}\\stage\"\n else:\n path_with_user = f\"/spack-test/spack-{user}/stage\"\n\n assert spack.stage._resolve_paths([path_with_user]) == [path_with_user]\n\n canonicalized_tempdir = canonicalize_path(\"$tempdir\")\n temp_has_user = user in canonicalized_tempdir.split(os.sep)\n paths = [\n os.path.join(\"$tempdir\", \"stage\"),\n os.path.join(\"$tempdir\", \"$user\"),\n os.path.join(\"$tempdir\", \"$user\", \"$user\"),\n os.path.join(\"$tempdir\", \"$user\", \"stage\", \"$user\"),\n ]\n\n res_paths = [canonicalize_path(p) for p in paths]\n if temp_has_user:\n res_paths[1] = canonicalized_tempdir\n res_paths[2] = os.path.join(canonicalized_tempdir, user)\n res_paths[3] = os.path.join(canonicalized_tempdir, \"stage\", user)\n elif sys.platform != \"win32\":\n res_paths[0] = os.path.join(res_paths[0], user)\n\n assert spack.stage._resolve_paths(paths) == res_paths\n\n @pytest.mark.not_on_windows(\"Windows file permission erroring is not yet supported\")\n @pytest.mark.skipif(getuid() == 0, reason=\"user is root\")\n def test_get_stage_root_bad_path(self, clear_stage_root):\n \"\"\"Ensure an invalid stage path root raises a StageError.\"\"\"\n with spack.config.override(\"config:build_stage\", \"/no/such/path\"):\n with pytest.raises(spack.stage.StageError, match=\"No accessible stage paths in\"):\n spack.stage.get_stage_root()\n\n # Make sure the cached stage path values are unchanged.\n assert spack.stage._stage_root is None\n\n @pytest.mark.parametrize(\n \"path,purged\",\n [\n (\"spack-stage-1234567890abcdef1234567890abcdef\", True),\n (\"spack-stage-anything-goes-here\", True),\n (\"stage-spack\", False),\n ],\n )\n def test_stage_purge(self, tmp_path: pathlib.Path, clear_stage_root, path, purged):\n \"\"\"Test purging of stage directories.\"\"\"\n stage_config_path = str(tmp_path / \"stage\")\n\n with spack.config.override(\"config:build_stage\", stage_config_path):\n stage_root = spack.stage.get_stage_root()\n\n test_dir = pathlib.Path(stage_root) / path\n test_dir.mkdir(parents=True)\n test_path = str(test_dir)\n\n spack.stage.purge()\n\n if purged:\n assert not os.path.exists(test_path)\n else:\n assert os.path.exists(test_path)\n shutil.rmtree(test_path)\n\n def test_stage_constructor_no_fetcher(self):\n \"\"\"Ensure Stage constructor with no URL or fetch strategy fails.\"\"\"\n with pytest.raises(ValueError):\n with Stage(None):\n pass\n\n def test_stage_constructor_with_path(self, tmp_path: pathlib.Path):\n \"\"\"Ensure Stage constructor with a path uses it.\"\"\"\n testpath = str(tmp_path)\n with Stage(\"file:///does-not-exist\", path=testpath) as stage:\n assert stage.path == testpath\n\n\ndef _create_files_from_tree(base, tree):\n for name, content in tree.items():\n sub_base = os.path.join(base, name)\n if isinstance(content, dict):\n os.mkdir(sub_base)\n _create_files_from_tree(sub_base, content)\n else:\n assert (content is None) or (isinstance(content, str))\n with open(sub_base, \"w\", encoding=\"utf-8\") as f:\n if content:\n f.write(content)\n\n\ndef _create_tree_from_dir_recursive(path):\n if os.path.islink(path):\n return readlink(path)\n elif os.path.isdir(path):\n tree = {}\n for name in os.listdir(path):\n sub_path = os.path.join(path, name)\n tree[name] = _create_tree_from_dir_recursive(sub_path)\n return tree\n else:\n with open(path, \"r\", encoding=\"utf-8\") as f:\n content = f.read() or None\n return content\n\n\n@pytest.fixture\ndef develop_path(tmp_path: pathlib.Path):\n dir_structure = {\"a1\": {\"b1\": None, \"b2\": \"b1content\"}, \"a2\": None}\n srcdir = str(tmp_path / \"test-src\")\n os.mkdir(srcdir)\n _create_files_from_tree(srcdir, dir_structure)\n yield dir_structure, srcdir\n\n\nclass TestDevelopStage:\n def test_sanity_check_develop_path(self, develop_path):\n _, srcdir = develop_path\n with open(os.path.join(srcdir, \"a1\", \"b2\"), encoding=\"utf-8\") as f:\n assert f.read() == \"b1content\"\n\n assert os.path.exists(os.path.join(srcdir, \"a2\"))\n\n def test_develop_stage(self, develop_path, tmp_build_stage_dir):\n \"\"\"Check that (a) develop stages update the given\n `dev_path` with a symlink that points to the stage dir and\n (b) that destroying the stage does not destroy `dev_path`\n \"\"\"\n devtree, srcdir = develop_path\n stage = DevelopStage(\"test-stage\", srcdir, reference_link=\"link-to-stage\")\n assert not os.path.exists(stage.reference_link)\n stage.create()\n assert os.path.exists(stage.reference_link)\n srctree1 = _create_tree_from_dir_recursive(stage.source_path)\n assert os.path.samefile(srctree1[\"link-to-stage\"], stage.path)\n del srctree1[\"link-to-stage\"]\n assert srctree1 == devtree\n\n stage.destroy()\n assert not os.path.exists(stage.reference_link)\n # Make sure destroying the stage doesn't change anything\n # about the path\n assert not os.path.exists(stage.path)\n srctree2 = _create_tree_from_dir_recursive(srcdir)\n assert srctree2 == devtree\n\n def test_develop_stage_without_reference_link(self, develop_path, tmp_build_stage_dir):\n \"\"\"Check that develop stages can be created without creating a reference link\"\"\"\n devtree, srcdir = develop_path\n stage = DevelopStage(\"test-stage\", srcdir, reference_link=None)\n stage.create()\n srctree1 = _create_tree_from_dir_recursive(stage.source_path)\n assert srctree1 == devtree\n\n stage.destroy()\n # Make sure destroying the stage doesn't change anything\n # about the path\n assert not os.path.exists(stage.path)\n srctree2 = _create_tree_from_dir_recursive(srcdir)\n assert srctree2 == devtree\n\n\ndef test_stage_create_replace_path(tmp_build_stage_dir):\n \"\"\"Ensure stage creation replaces a non-directory path.\"\"\"\n _, test_stage_path = tmp_build_stage_dir\n mkdirp(test_stage_path)\n\n nondir = os.path.join(test_stage_path, \"afile\")\n touch(nondir)\n path = url_util.path_to_file_url(str(nondir))\n\n stage = Stage(path, name=\"afile\")\n stage.create()\n\n # Ensure the stage path is \"converted\" to a directory\n assert os.path.isdir(nondir)\n\n\ndef test_cannot_access(capfd):\n \"\"\"Ensure can_access dies with the expected error.\"\"\"\n with pytest.raises(SystemExit):\n # It's far more portable to use a non-existent filename.\n spack.stage.ensure_access(\"/no/such/file\")\n\n captured = capfd.readouterr()\n assert \"Insufficient permissions\" in str(captured)\n\n\ndef test_override_keep_in_composite_stage():\n stage_1 = Stage(\"file:///does-not-exist\", keep=True)\n stage_2 = Stage(\"file:///does-not-exist\", keep=False)\n stage_3 = Stage(\"file:///does-not-exist\", keep=True)\n stages = spack.stage.StageComposite.from_iterable((stage_1, stage_2, stage_3))\n\n # The getter for the composite stage just returns the value of the first stage\n # its just there so we have a setter too.\n assert stages.keep\n assert stage_1.keep\n assert not stage_2.keep\n assert stage_3.keep\n\n # This should override all stages\n stages.keep = False\n assert not stages.keep\n assert not stage_1.keep\n assert not stage_2.keep\n assert not stage_3.keep\n" }, { "path": "lib/spack/spack/test/svn_fetch.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport pathlib\n\nimport pytest\n\nimport spack.concretize\nimport spack.config\nfrom spack.fetch_strategy import SvnFetchStrategy\nfrom spack.llnl.util.filesystem import mkdirp, touch, working_dir\nfrom spack.stage import Stage\nfrom spack.util.executable import which\nfrom spack.version import Version\n\npytestmark = [\n pytest.mark.skipif(\n not which(\"svn\") or not which(\"svnadmin\"), reason=\"requires subversion to be installed\"\n ),\n pytest.mark.not_on_windows(\"does not run on windows\"),\n]\n\n\n@pytest.mark.parametrize(\"type_of_test\", [\"default\", \"rev0\"])\n@pytest.mark.parametrize(\"secure\", [True, False])\ndef test_fetch(type_of_test, secure, mock_svn_repository, config, mutable_mock_repo, monkeypatch):\n \"\"\"Tries to:\n\n 1. Fetch the repo using a fetch strategy constructed with\n supplied args (they depend on type_of_test).\n 2. Check if the test_file is in the checked out repository.\n 3. Assert that the repository is at the revision supplied.\n 4. Add and remove some files, then reset the repo, and\n ensure it's all there again.\n \"\"\"\n # Retrieve the right test parameters\n t = mock_svn_repository.checks[type_of_test]\n h = mock_svn_repository.hash\n\n # Construct the package under test\n s = spack.concretize.concretize_one(\"svn-test\")\n monkeypatch.setitem(s.package.versions, Version(\"svn\"), t.args)\n\n # Enter the stage directory and check some properties\n with s.package.stage:\n with spack.config.override(\"config:verify_ssl\", secure):\n s.package.do_stage()\n\n with working_dir(s.package.stage.source_path):\n assert h() == t.revision\n\n file_path = os.path.join(s.package.stage.source_path, t.file)\n assert os.path.isdir(s.package.stage.source_path)\n assert os.path.isfile(file_path)\n\n os.unlink(file_path)\n assert not os.path.isfile(file_path)\n\n untracked_file = \"foobarbaz\"\n touch(untracked_file)\n assert os.path.isfile(untracked_file)\n s.package.do_restage()\n assert not os.path.isfile(untracked_file)\n\n assert os.path.isdir(s.package.stage.source_path)\n assert os.path.isfile(file_path)\n\n assert h() == t.revision\n\n\ndef test_svn_extra_fetch(tmp_path: pathlib.Path):\n \"\"\"Ensure a fetch after downloading is effectively a no-op.\"\"\"\n testpath = str(tmp_path)\n\n fetcher = SvnFetchStrategy(svn=\"file:///not-a-real-svn-repo\")\n assert fetcher is not None\n\n with Stage(fetcher, path=testpath) as stage:\n assert stage is not None\n\n source_path = stage.source_path\n mkdirp(source_path)\n\n fetcher.fetch()\n" }, { "path": "lib/spack/spack/test/tag.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Tests for tag index cache files.\"\"\"\n\nimport io\n\nimport pytest\n\nimport spack.cmd.tags\nimport spack.repo\nimport spack.tag\nfrom spack.main import SpackCommand\n\ninstall = SpackCommand(\"install\")\n\n# Alternate representation\ntags_json = \"\"\"\n {\n \"tags\": {\n \"no-version\": [\n \"noversion\",\n \"noversion-bundle\"\n ],\n \"no-source\": [\n \"nosource\"\n ]\n }\n }\n \"\"\"\n\nmore_tags_json = \"\"\"\n {\n \"tags\": {\n \"merge\": [\n \"check\"\n ]\n }\n }\n \"\"\"\n\n\ndef test_tag_get_all_available(mock_packages):\n for skip in [False, True]:\n all_pkgs = spack.cmd.tags.packages_with_tags([\"tag1\", \"tag2\", \"tag3\"], False, skip)\n assert sorted(all_pkgs[\"tag1\"]) == [\"mpich\", \"mpich2\"]\n assert all_pkgs[\"tag2\"] == [\"mpich\"]\n assert all_pkgs[\"tag3\"] == [\"mpich2\"]\n\n\ndef ensure_tags_results_equal(results, expected):\n if expected:\n assert sorted(results.keys()) == sorted(expected.keys())\n for tag in results:\n assert sorted(results[tag]) == sorted(expected[tag])\n else:\n assert results == expected\n\n\n@pytest.mark.parametrize(\n \"tags,expected\",\n [\n ([\"tag1\"], {\"tag1\": [\"mpich\", \"mpich2\"]}),\n ([\"tag2\"], {\"tag2\": [\"mpich\"]}),\n ([\"tag3\"], {\"tag3\": [\"mpich2\"]}),\n ([\"nosuchpackage\"], {\"nosuchpackage\": {}}),\n ],\n)\ndef test_tag_get_available(tags, expected, mock_packages):\n # Ensure results for all tags\n all_tag_pkgs = spack.cmd.tags.packages_with_tags(tags, False, False)\n ensure_tags_results_equal(all_tag_pkgs, expected)\n\n # Ensure results for tags expecting results since skipping otherwise\n only_pkgs = spack.cmd.tags.packages_with_tags(tags, False, True)\n if expected[tags[0]]:\n ensure_tags_results_equal(only_pkgs, expected)\n else:\n assert not only_pkgs\n\n\ndef test_tag_get_installed_packages(mock_packages, mock_archive, mock_fetch, install_mockery):\n install(\"--fake\", \"mpich\")\n\n for skip in [False, True]:\n all_pkgs = spack.cmd.tags.packages_with_tags([\"tag1\", \"tag2\", \"tag3\"], True, skip)\n assert sorted(all_pkgs[\"tag1\"]) == [\"mpich\"]\n assert all_pkgs[\"tag2\"] == [\"mpich\"]\n assert skip or all_pkgs[\"tag3\"] == []\n\n\ndef test_tag_index_round_trip(mock_packages):\n # Assumes at least two packages -- mpich and mpich2 -- have tags\n mock_index = spack.repo.PATH.tag_index\n assert mock_index.tags\n\n ostream = io.StringIO()\n mock_index.to_json(ostream)\n\n istream = io.StringIO(ostream.getvalue())\n new_index = spack.tag.TagIndex.from_json(istream)\n\n assert mock_index.tags == new_index.tags\n\n\ndef test_tag_equal(mock_packages):\n first_index = spack.tag.TagIndex.from_json(io.StringIO(tags_json))\n second_index = spack.tag.TagIndex.from_json(io.StringIO(tags_json))\n\n assert first_index.tags == second_index.tags\n\n\ndef test_tag_merge(mock_packages):\n first_index = spack.tag.TagIndex.from_json(io.StringIO(tags_json))\n second_index = spack.tag.TagIndex.from_json(io.StringIO(more_tags_json))\n\n assert first_index != second_index\n\n tags1 = list(first_index.tags.keys())\n tags2 = list(second_index.tags.keys())\n all_tags = sorted(list(set(tags1 + tags2)))\n\n first_index.merge(second_index)\n tag_keys = sorted(first_index.tags.keys())\n assert tag_keys == all_tags\n\n # Merge again to make sure the index does not retain duplicates\n first_index.merge(second_index)\n tag_keys = sorted(first_index.tags.keys())\n assert tag_keys == all_tags\n\n\ndef test_tag_not_dict(mock_packages):\n list_json = \"[]\"\n with pytest.raises(spack.tag.TagIndexError) as e:\n spack.tag.TagIndex.from_json(io.StringIO(list_json))\n assert \"not a dict\" in str(e)\n\n\ndef test_tag_no_tags(mock_packages):\n pkg_json = '{\"packages\": []}'\n with pytest.raises(spack.tag.TagIndexError) as e:\n spack.tag.TagIndex.from_json(io.StringIO(pkg_json))\n assert \"does not start with\" in str(e)\n\n\ndef test_tag_update_package(mock_packages):\n mock_index = mock_packages.tag_index\n index = spack.tag.TagIndex()\n index.update_packages(set(spack.repo.all_package_names()), repo=mock_packages)\n\n ensure_tags_results_equal(mock_index.tags, index.tags)\n" }, { "path": "lib/spack/spack/test/tengine.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\nimport pytest\n\nimport spack.config\nimport spack.tengine as tengine\nfrom spack.util.path import canonicalize_path\n\n\nclass TestContext:\n class A(tengine.Context):\n @tengine.context_property\n def foo(self):\n return 1\n\n class B(tengine.Context):\n @tengine.context_property\n def bar(self):\n return 2\n\n class C(A, B):\n @tengine.context_property\n def foobar(self):\n return 3\n\n @tengine.context_property\n def foo(self):\n return 10\n\n def test_to_dict(self):\n \"\"\"Tests that all the context properties in a hierarchy are considered\n when building the context dictionary.\n \"\"\"\n\n # A derives directly from Context\n a = TestContext.A()\n d = a.to_dict()\n\n assert len(d) == 1\n assert \"foo\" in d\n assert d[\"foo\"] == 1\n\n # So does B\n b = TestContext.B()\n d = b.to_dict()\n\n assert len(d) == 1\n assert \"bar\" in d\n assert d[\"bar\"] == 2\n\n # C derives from both and overrides 'foo'\n c = TestContext.C()\n d = c.to_dict()\n\n assert len(d) == 3\n for x in (\"foo\", \"bar\", \"foobar\"):\n assert x in d\n\n assert d[\"foo\"] == 10\n assert d[\"bar\"] == 2\n assert d[\"foobar\"] == 3\n\n\n@pytest.mark.usefixtures(\"config\")\nclass TestTengineEnvironment:\n def test_template_retrieval(self):\n \"\"\"Tests the template retrieval mechanism hooked into config files\"\"\"\n # Check the directories are correct\n template_dirs = spack.config.get(\"config:template_dirs\")\n template_dirs = tuple([canonicalize_path(x) for x in template_dirs])\n assert len(template_dirs) == 3\n\n env = tengine.make_environment(template_dirs)\n\n # Retrieve a.txt, which resides in the second\n # template directory specified in the mock configuration\n template = env.get_template(\"a.txt\")\n text = template.render({\"word\": \"world\"})\n assert \"Hello world!\" == text\n\n # Retrieve b.txt, which resides in the third\n # template directory specified in the mock configuration\n template = env.get_template(\"b.txt\")\n text = template.render({\"word\": \"world\"})\n assert \"Howdy world!\" == text\n" }, { "path": "lib/spack/spack/test/test_suite.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport collections\nimport os\nimport pathlib\nimport sys\n\nimport pytest\n\nimport spack.concretize\nimport spack.config\nimport spack.install_test\nimport spack.spec\nimport spack.util.executable\nfrom spack.install_test import TestStatus\nfrom spack.llnl.util.filesystem import touch\nfrom spack.util.executable import which\n\n\ndef _true(*args, **kwargs):\n \"\"\"Generic monkeypatch function that always returns True.\"\"\"\n return True\n\n\ndef ensure_results(filename, expected, present=True):\n assert os.path.exists(filename)\n with open(filename, \"r\", encoding=\"utf-8\") as fd:\n lines = fd.readlines()\n have = False\n for line in lines:\n if expected in line:\n have = True\n break\n if present:\n assert have, f\"Expected '{expected}' in the file\"\n else:\n assert not have, f\"Expected '{expected}' NOT to be in the file\"\n\n\ndef test_test_log_name(mock_packages, config):\n \"\"\"Ensure test log path is reasonable.\"\"\"\n spec = spack.concretize.concretize_one(\"libdwarf\")\n\n test_name = \"test_name\"\n\n test_suite = spack.install_test.TestSuite([spec], test_name)\n logfile = test_suite.log_file_for_spec(spec)\n\n assert test_suite.stage in logfile\n assert test_suite.test_log_name(spec) in logfile\n\n\ndef test_test_ensure_stage(mock_test_stage, mock_packages):\n \"\"\"Make sure test stage directory is properly set up.\"\"\"\n spec = spack.concretize.concretize_one(\"libdwarf\")\n\n test_name = \"test_name\"\n\n test_suite = spack.install_test.TestSuite([spec], test_name)\n test_suite.ensure_stage()\n\n assert os.path.isdir(test_suite.stage)\n assert mock_test_stage in test_suite.stage\n\n\ndef test_write_test_result(mock_packages, mock_test_stage):\n \"\"\"Ensure test results written to a results file.\"\"\"\n spec = spack.concretize.concretize_one(\"libdwarf\")\n result = \"TEST\"\n test_name = \"write-test\"\n\n test_suite = spack.install_test.TestSuite([spec], test_name)\n test_suite.ensure_stage()\n results_file = test_suite.results_file\n test_suite.write_test_result(spec, result)\n\n with open(results_file, \"r\", encoding=\"utf-8\") as f:\n lines = f.readlines()\n assert len(lines) == 1\n\n msg = lines[0]\n assert result in msg\n assert spec.name in msg\n\n\ndef test_test_not_installed(mock_packages, install_mockery, mock_test_stage):\n \"\"\"Attempt to perform stand-alone test for not_installed package.\"\"\"\n spec = spack.concretize.concretize_one(\"trivial-smoke-test\")\n test_suite = spack.install_test.TestSuite([spec])\n\n test_suite()\n\n ensure_results(test_suite.results_file, \"SKIPPED\")\n ensure_results(test_suite.log_file_for_spec(spec), \"Skipped not installed\")\n\n\n@pytest.mark.parametrize(\n \"arguments,status,msg\",\n [({}, TestStatus.SKIPPED, \"Skipped\"), ({\"externals\": True}, TestStatus.NO_TESTS, \"No tests\")],\n)\ndef test_test_external(\n mock_packages, install_mockery, mock_test_stage, monkeypatch, arguments, status, msg\n):\n name = \"trivial-smoke-test\"\n spec = spack.concretize.concretize_one(name)\n spec.external_path = \"/path/to/external/{0}\".format(name)\n\n monkeypatch.setattr(spack.spec.Spec, \"installed\", _true)\n\n test_suite = spack.install_test.TestSuite([spec])\n test_suite(**arguments)\n\n ensure_results(test_suite.results_file, str(status))\n if arguments:\n ensure_results(test_suite.log_file_for_spec(spec), msg)\n\n\ndef test_test_stage_caches(mock_packages, install_mockery, mock_test_stage):\n def ensure_current_cache_fail(test_suite):\n with pytest.raises(spack.install_test.TestSuiteSpecError):\n _ = test_suite.current_test_cache_dir\n\n with pytest.raises(spack.install_test.TestSuiteSpecError):\n _ = test_suite.current_test_data_dir\n\n spec = spack.concretize.concretize_one(\"libelf\")\n test_suite = spack.install_test.TestSuite([spec], \"test-cache\")\n\n # Check no current specs yield failure\n ensure_current_cache_fail(test_suite)\n\n # Check no current base spec yields failure\n test_suite.current_base_spec = None\n test_suite.current_test_spec = spec\n ensure_current_cache_fail(test_suite)\n\n # Check no current test spec yields failure\n test_suite.current_base_spec = spec\n test_suite.current_test_spec = None\n ensure_current_cache_fail(test_suite)\n\n\ndef test_test_spec_run_once(mock_packages, install_mockery, mock_test_stage):\n spec = spack.concretize.concretize_one(\"libelf\")\n test_suite = spack.install_test.TestSuite([spec], \"test-dups\")\n (test_suite.specs[0]).package.test_suite = test_suite\n\n with pytest.raises(spack.install_test.TestSuiteFailure):\n test_suite()\n\n\n@pytest.mark.not_on_windows(\"Cannot find echo executable\")\ndef test_test_spec_passes(mock_packages, install_mockery, mock_test_stage, monkeypatch):\n spec = spack.concretize.concretize_one(\"simple-standalone-test\")\n monkeypatch.setattr(spack.spec.Spec, \"installed\", _true)\n test_suite = spack.install_test.TestSuite([spec])\n test_suite()\n\n ensure_results(test_suite.results_file, \"PASSED\")\n ensure_results(test_suite.log_file_for_spec(spec), \"simple stand-alone\")\n ensure_results(test_suite.log_file_for_spec(spec), \"standalone-ifc\", present=False)\n\n\ndef test_get_test_suite():\n assert not spack.install_test.get_test_suite(\"nothing\")\n\n\ndef test_get_test_suite_no_name(mock_packages, mock_test_stage):\n with pytest.raises(spack.install_test.TestSuiteNameError) as exc_info:\n spack.install_test.get_test_suite(\"\")\n\n assert \"name is required\" in str(exc_info)\n\n\ndef test_get_test_suite_too_many(mock_packages, mock_test_stage):\n test_suites = []\n name = \"duplicate-alias\"\n\n def add_suite(package):\n spec = spack.concretize.concretize_one(package)\n suite = spack.install_test.TestSuite([spec], name)\n suite.ensure_stage()\n spack.install_test.write_test_suite_file(suite)\n test_suites.append(suite)\n\n add_suite(\"libdwarf\")\n suite = spack.install_test.get_test_suite(name)\n assert suite.alias == name\n\n add_suite(\"libelf\")\n with pytest.raises(spack.install_test.TestSuiteNameError) as exc_info:\n spack.install_test.get_test_suite(name)\n assert \"many suites named\" in str(exc_info)\n\n\n@pytest.mark.parametrize(\n \"virtuals,expected\",\n [(False, [\"Mpich.test_mpich\"]), (True, [\"Mpi.test_hello\", \"Mpich.test_mpich\"])],\n)\ndef test_test_function_names(mock_packages, install_mockery, virtuals, expected):\n \"\"\"Confirm test_function_names works as expected with/without virtuals.\"\"\"\n spec = spack.concretize.concretize_one(\"mpich\")\n tests = spack.install_test.test_function_names(spec.package, add_virtuals=virtuals)\n assert sorted(tests) == sorted(expected)\n\n\ndef test_test_functions_pkgless(mock_packages, install_mockery, ensure_debug, capfd):\n \"\"\"Confirm works for package providing a package-less virtual.\"\"\"\n spec = spack.concretize.concretize_one(\"simple-standalone-test\")\n fns = spack.install_test.test_functions(spec.package, add_virtuals=True)\n out = capfd.readouterr()\n assert len(fns) == 2, \"Expected two test functions\"\n for f in fns:\n assert f[1].__name__ in [\"test_echo\", \"test_skip\"]\n assert \"virtual does not appear to have a package file\" in out[1]\n\n\n# TODO: This test should go away when compilers as dependencies is supported\ndef test_test_virtuals():\n \"\"\"Confirm virtuals picks up non-unique, provided compilers.\"\"\"\n\n # This is an unrealistic case but it is set up to retrieve all possible\n # virtual names in a single call.\n def satisfies(spec):\n return True\n\n # Ensure spec will pick up the llvm+clang virtual compiler package names.\n VirtualSpec = collections.namedtuple(\"VirtualSpec\", [\"name\", \"satisfies\"])\n vspec = VirtualSpec(\"llvm\", satisfies)\n\n # Ensure the package name is in the list that provides c, cxx, and fortran\n # to pick up the three associated compilers and that virtuals provided will\n # be deduped.\n MyPackage = collections.namedtuple(\"MyPackage\", [\"name\", \"spec\", \"virtuals_provided\"])\n pkg = MyPackage(\"gcc\", vspec, [vspec, vspec])\n\n # This check assumes the method will not provide a unique set of compilers\n v_names = spack.install_test.virtuals(pkg)\n for name, number in [(\"c\", 2), (\"cxx\", 2), (\"fortran\", 1), (\"llvm\", 1)]:\n assert v_names.count(name) == number, \"Expected {0} of '{1}'\".format(number, name)\n\n\ndef test_package_copy_test_files_fails(mock_packages):\n \"\"\"Confirm copy_test_files fails as expected without package or test_suite.\"\"\"\n vspec = spack.spec.Spec(\"something\")\n\n # Try without a package\n with pytest.raises(spack.install_test.TestSuiteError) as exc_info:\n spack.install_test.copy_test_files(None, vspec)\n assert \"without a package\" in str(exc_info)\n\n # Try with a package without a test suite\n MyPackage = collections.namedtuple(\"MyPackage\", [\"name\", \"spec\", \"test_suite\"])\n pkg = MyPackage(\"SomePackage\", vspec, None)\n\n with pytest.raises(spack.install_test.TestSuiteError) as exc_info:\n spack.install_test.copy_test_files(pkg, vspec)\n assert \"test suite is missing\" in str(exc_info)\n\n\ndef test_package_copy_test_files_skips(mock_packages, ensure_debug, capfd):\n \"\"\"Confirm copy_test_files errors as expected if no package class found.\"\"\"\n # Try with a non-concrete spec and package with a test suite\n MockSuite = collections.namedtuple(\"TestSuite\", [\"specs\"])\n MyPackage = collections.namedtuple(\"MyPackage\", [\"name\", \"spec\", \"test_suite\"])\n vspec = spack.spec.Spec(\"something\")\n pkg = MyPackage(\"SomePackage\", vspec, MockSuite([]))\n spack.install_test.copy_test_files(pkg, vspec)\n out = capfd.readouterr()[1]\n assert \"skipping test data copy\" in out\n assert \"no package class found\" in out\n\n\ndef test_process_test_parts(mock_packages):\n \"\"\"Confirm process_test_parts fails as expected without package or test_suite.\"\"\"\n # Try without a package\n with pytest.raises(spack.install_test.TestSuiteError) as exc_info:\n spack.install_test.process_test_parts(None, [])\n assert \"without a package\" in str(exc_info)\n\n # Try with a package without a test suite\n MyPackage = collections.namedtuple(\"MyPackage\", [\"name\", \"test_suite\"])\n pkg = MyPackage(\"SomePackage\", None)\n\n with pytest.raises(spack.install_test.TestSuiteError) as exc_info:\n spack.install_test.process_test_parts(pkg, [])\n assert \"test suite is missing\" in str(exc_info)\n\n\ndef test_test_part_fail(tmp_path: pathlib.Path, install_mockery, mock_fetch, mock_test_stage):\n \"\"\"Confirm test_part with a ProcessError results in FAILED status.\"\"\"\n s = spack.concretize.concretize_one(\"trivial-smoke-test\")\n pkg = s.package\n pkg.tester.test_log_file = str(tmp_path / \"test-log.txt\")\n touch(pkg.tester.test_log_file)\n\n name = \"test_fail\"\n with spack.install_test.test_part(pkg, name, \"fake ProcessError\"):\n raise spack.util.executable.ProcessError(\"Mock failure\")\n\n for part_name, status in pkg.tester.test_parts.items():\n assert part_name.endswith(name)\n assert status == TestStatus.FAILED\n\n\ndef test_test_part_pass(install_mockery, mock_fetch, mock_test_stage):\n \"\"\"Confirm test_part that succeeds results in PASSED status.\"\"\"\n s = spack.concretize.concretize_one(\"trivial-smoke-test\")\n pkg = s.package\n\n name = \"test_echo\"\n msg = \"nothing\"\n with spack.install_test.test_part(pkg, name, \"echo\"):\n if sys.platform == \"win32\":\n print(msg)\n else:\n echo = which(\"echo\", required=True)\n echo(msg)\n\n for part_name, status in pkg.tester.test_parts.items():\n assert part_name.endswith(name)\n assert status == TestStatus.PASSED\n\n\ndef test_test_part_skip(install_mockery, mock_fetch, mock_test_stage):\n \"\"\"Confirm test_part that raises SkipTest results in test status SKIPPED.\"\"\"\n s = spack.concretize.concretize_one(\"trivial-smoke-test\")\n pkg = s.package\n\n name = \"test_skip\"\n with spack.install_test.test_part(pkg, name, \"raise SkipTest\"):\n raise spack.install_test.SkipTest(\"Skipping the test\")\n\n for part_name, status in pkg.tester.test_parts.items():\n assert part_name.endswith(name)\n assert status == TestStatus.SKIPPED\n\n\ndef test_test_part_missing_exe_fail_fast(\n tmp_path: pathlib.Path, install_mockery, mock_fetch, mock_test_stage\n):\n \"\"\"Confirm test_part with fail fast enabled raises exception.\"\"\"\n s = spack.concretize.concretize_one(\"trivial-smoke-test\")\n pkg = s.package\n pkg.tester.test_log_file = str(tmp_path / \"test-log.txt\")\n touch(pkg.tester.test_log_file)\n\n name = \"test_fail_fast\"\n with spack.config.override(\"config:fail_fast\", True):\n with pytest.raises(spack.install_test.TestFailure, match=\"object is not callable\"):\n with spack.install_test.test_part(pkg, name, \"fail fast\"):\n missing = which(\"no-possible-program\")\n missing() # type: ignore\n\n test_parts = pkg.tester.test_parts\n assert len(test_parts) == 1\n for part_name, status in test_parts.items():\n assert part_name.endswith(name)\n assert status == TestStatus.FAILED\n\n\ndef test_test_part_missing_exe(\n tmp_path: pathlib.Path, install_mockery, mock_fetch, mock_test_stage\n):\n \"\"\"Confirm test_part with missing executable fails.\"\"\"\n s = spack.concretize.concretize_one(\"trivial-smoke-test\")\n pkg = s.package\n pkg.tester.test_log_file = str(tmp_path / \"test-log.txt\")\n touch(pkg.tester.test_log_file)\n\n name = \"test_missing_exe\"\n with spack.install_test.test_part(pkg, name, \"missing exe\"):\n missing = which(\"no-possible-program\")\n missing() # type: ignore\n\n test_parts = pkg.tester.test_parts\n assert len(test_parts) == 1\n for part_name, status in test_parts.items():\n assert part_name.endswith(name)\n assert status == TestStatus.FAILED\n\n\n# TODO (embedded test parts): Update this once embedded test part tracking\n# TODO (embedded test parts): properly handles the nested context managers.\n@pytest.mark.parametrize(\n \"current,substatuses,expected\",\n [\n (TestStatus.PASSED, [TestStatus.PASSED, TestStatus.PASSED], TestStatus.PASSED),\n (TestStatus.FAILED, [TestStatus.PASSED, TestStatus.PASSED], TestStatus.FAILED),\n (TestStatus.SKIPPED, [TestStatus.PASSED, TestStatus.PASSED], TestStatus.SKIPPED),\n (TestStatus.NO_TESTS, [TestStatus.PASSED, TestStatus.PASSED], TestStatus.NO_TESTS),\n (TestStatus.PASSED, [TestStatus.PASSED, TestStatus.SKIPPED], TestStatus.PASSED),\n (TestStatus.PASSED, [TestStatus.PASSED, TestStatus.FAILED], TestStatus.FAILED),\n (TestStatus.PASSED, [TestStatus.SKIPPED, TestStatus.SKIPPED], TestStatus.SKIPPED),\n ],\n)\ndef test_embedded_test_part_status(\n install_mockery, mock_fetch, mock_test_stage, current, substatuses, expected\n):\n \"\"\"Check to ensure the status of the enclosing test part reflects summary of embedded parts.\"\"\"\n\n s = spack.concretize.concretize_one(\"trivial-smoke-test\")\n pkg = s.package\n base_name = \"test_example\"\n part_name = f\"{pkg.__class__.__name__}::{base_name}\"\n\n pkg.tester.test_parts[part_name] = current\n for i, status in enumerate(substatuses):\n pkg.tester.test_parts[f\"{part_name}_{i}\"] = status\n\n pkg.tester.status(base_name, current)\n assert pkg.tester.test_parts[part_name] == expected\n\n\n@pytest.mark.parametrize(\n \"statuses,expected\",\n [\n ([TestStatus.PASSED, TestStatus.PASSED], TestStatus.PASSED),\n ([TestStatus.PASSED, TestStatus.SKIPPED], TestStatus.PASSED),\n ([TestStatus.PASSED, TestStatus.FAILED], TestStatus.FAILED),\n ([TestStatus.SKIPPED, TestStatus.SKIPPED], TestStatus.SKIPPED),\n ([], TestStatus.NO_TESTS),\n ],\n)\ndef test_write_tested_status(\n tmp_path: pathlib.Path, install_mockery, mock_fetch, mock_test_stage, statuses, expected\n):\n \"\"\"Check to ensure the status of the enclosing test part reflects summary of embedded parts.\"\"\"\n s = spack.concretize.concretize_one(\"trivial-smoke-test\")\n pkg = s.package\n for i, status in enumerate(statuses):\n pkg.tester.test_parts[f\"test_{i}\"] = status\n pkg.tester.counts[status] += 1\n\n pkg.tester.tested_file = str(tmp_path / \"test-log.txt\")\n pkg.tester.write_tested_status()\n with open(pkg.tester.tested_file, \"r\", encoding=\"utf-8\") as f:\n status = int(f.read().strip(\"\\n\"))\n assert TestStatus(status) == expected\n\n\n@pytest.mark.regression(\"37840\")\ndef test_write_tested_status_no_repeats(\n tmp_path: pathlib.Path, install_mockery, mock_fetch, mock_test_stage\n):\n \"\"\"Emulate re-running the same stand-alone tests a second time.\"\"\"\n s = spack.concretize.concretize_one(\"trivial-smoke-test\")\n pkg = s.package\n statuses = [TestStatus.PASSED, TestStatus.PASSED]\n for i, status in enumerate(statuses):\n pkg.tester.test_parts[f\"test_{i}\"] = status\n pkg.tester.counts[status] += 1\n\n pkg.tester.tested_file = str(tmp_path / \"test-log.txt\")\n pkg.tester.write_tested_status()\n pkg.tester.write_tested_status()\n\n # The test should NOT result in a ValueError: invalid literal for int()\n # with base 10: '2\\n2' (i.e., the results being appended instead of\n # written to the file).\n with open(pkg.tester.tested_file, \"r\", encoding=\"utf-8\") as f:\n status_no = int(f.read().strip(\"\\n\"))\n assert TestStatus(status_no) == TestStatus.PASSED\n\n\ndef test_check_special_outputs(tmp_path: pathlib.Path):\n \"\"\"This test covers two related helper methods\"\"\"\n contents = \"\"\"CREATE TABLE packages (\nname varchar(80) primary key,\nhas_code integer,\nurl varchar(160));\nINSERT INTO packages VALUES('sqlite',1,'https://www.sqlite.org');\nINSERT INTO packages VALUES('readline',1,'https://tiswww.case.edu/php/chet/readline/rltop.html');\nINSERT INTO packages VALUES('xsdk',0,'http://xsdk.info');\nCOMMIT;\n\"\"\"\n filename = tmp_path / \"special.txt\"\n with open(filename, \"w\", encoding=\"utf-8\") as f:\n f.write(contents)\n\n expected = spack.install_test.get_escaped_text_output(str(filename))\n spack.install_test.check_outputs(expected, contents)\n\n # Let's also cover case where something expected is NOT in the output\n expected.append(\"should not find me\")\n with pytest.raises(RuntimeError, match=\"Expected\"):\n spack.install_test.check_outputs(expected, contents)\n\n\ndef test_find_required_file(tmp_path: pathlib.Path):\n filename = \"myexe\"\n for d in [\"a\", \"b\"]:\n path = tmp_path / d / filename\n os.makedirs(path.parent, exist_ok=True)\n path.touch()\n path = tmp_path / \"c\" / \"d\" / filename\n os.makedirs(path.parent, exist_ok=True)\n path.touch()\n\n # First just find a single path\n results = spack.install_test.find_required_file(\n str(tmp_path / \"c\"), filename, expected=1, recursive=True\n )\n assert isinstance(results, str)\n\n # Ensure none file if do not recursively search that directory\n with pytest.raises(spack.install_test.SkipTest, match=\"Expected 1\"):\n spack.install_test.find_required_file(\n str(tmp_path / \"c\"), filename, expected=1, recursive=False\n )\n\n # Now make sure we get all of the files\n results = spack.install_test.find_required_file(\n str(tmp_path), filename, expected=3, recursive=True\n )\n assert isinstance(results, list) and len(results) == 3\n\n\ndef test_packagetest_fails(mock_packages):\n MyPackage = collections.namedtuple(\"MyPackage\", [\"spec\"])\n\n s = spack.spec.Spec(\"pkg-a\")\n pkg = MyPackage(s)\n with pytest.raises(ValueError, match=\"require a concrete package\"):\n spack.install_test.PackageTest(pkg)\n" }, { "path": "lib/spack/spack/test/traverse.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport pytest\n\nimport spack.deptypes as dt\nimport spack.traverse as traverse\nfrom spack.spec import Spec\n\n\ndef create_dag(nodes, edges):\n \"\"\"\n Arguments:\n nodes: list of package names\n edges: list of tuples (from, to, deptype)\n Returns:\n dict: mapping from package name to abstract Spec with proper deps.\n \"\"\"\n specs = {name: Spec(name) for name in nodes}\n for parent, child, deptypes in edges:\n depflag = deptypes if isinstance(deptypes, dt.DepFlag) else dt.canonicalize(deptypes)\n specs[parent].add_dependency_edge(specs[child], depflag=depflag, virtuals=())\n return specs\n\n\n@pytest.fixture()\ndef abstract_specs_dtuse():\n return create_dag(\n nodes=[\n \"dtbuild1\",\n \"dtbuild2\",\n \"dtbuild3\",\n \"dtlink1\",\n \"dtlink2\",\n \"dtlink3\",\n \"dtlink4\",\n \"dtlink5\",\n \"dtrun1\",\n \"dtrun2\",\n \"dtrun3\",\n \"dttop\",\n \"dtuse\",\n ],\n edges=[\n (\"dtbuild1\", \"dtbuild2\", (\"build\")),\n (\"dtbuild1\", \"dtlink2\", (\"build\", \"link\")),\n (\"dtbuild1\", \"dtrun2\", (\"run\")),\n (\"dtlink1\", \"dtlink3\", (\"build\", \"link\")),\n (\"dtlink3\", \"dtbuild2\", (\"build\")),\n (\"dtlink3\", \"dtlink4\", (\"build\", \"link\")),\n (\"dtrun1\", \"dtlink5\", (\"build\", \"link\")),\n (\"dtrun1\", \"dtrun3\", (\"run\")),\n (\"dtrun3\", \"dtbuild3\", (\"build\")),\n (\"dttop\", \"dtbuild1\", (\"build\",)),\n (\"dttop\", \"dtlink1\", (\"build\", \"link\")),\n (\"dttop\", \"dtrun1\", (\"run\")),\n (\"dtuse\", \"dttop\", (\"build\", \"link\")),\n ],\n )\n\n\n@pytest.fixture()\ndef abstract_specs_dt_diamond():\n return create_dag(\n nodes=[\"dt-diamond\", \"dt-diamond-left\", \"dt-diamond-right\", \"dt-diamond-bottom\"],\n edges=[\n (\"dt-diamond\", \"dt-diamond-left\", (\"build\", \"link\")),\n (\"dt-diamond\", \"dt-diamond-right\", (\"build\", \"link\")),\n (\"dt-diamond-right\", \"dt-diamond-bottom\", (\"build\", \"link\", \"run\")),\n (\"dt-diamond-left\", \"dt-diamond-bottom\", (\"build\")),\n ],\n )\n\n\n@pytest.fixture()\ndef abstract_specs_chain():\n # Chain a -> b -> c -> d with skip connections\n # from a -> c and a -> d.\n return create_dag(\n nodes=[\"chain-a\", \"chain-b\", \"chain-c\", \"chain-d\"],\n edges=[\n (\"chain-a\", \"chain-b\", (\"build\", \"link\")),\n (\"chain-b\", \"chain-c\", (\"build\", \"link\")),\n (\"chain-c\", \"chain-d\", (\"build\", \"link\")),\n (\"chain-a\", \"chain-c\", (\"build\", \"link\")),\n (\"chain-a\", \"chain-d\", (\"build\", \"link\")),\n ],\n )\n\n\n@pytest.mark.parametrize(\"direction\", (\"children\", \"parents\"))\n@pytest.mark.parametrize(\"deptype\", (\"all\", (\"link\", \"build\"), (\"run\", \"link\")))\ndef test_all_orders_traverse_the_same_nodes(direction, deptype, abstract_specs_dtuse):\n # Test whether all graph traversal methods visit the same set of vertices.\n # When testing cover=nodes, the traversal methods may reach the same vertices\n # through different edges, so we're using traverse_nodes here to only verify the\n # vertices.\n #\n # NOTE: root=False currently means \"yield nodes discovered at depth > 0\",\n # meaning that depth first search will yield dtlink5 as it is first found through\n # dtuse, whereas breadth first search considers dtlink5 at depth 0 and does not\n # yield it since it is a root. Therefore, we only use root=True.\n # (The inconsistency cannot be resolved by making root=False mean \"don't yield\n # vertices without in-edges\", since this is not how it's used; it's typically used\n # as \"skip the input specs\".)\n specs = [abstract_specs_dtuse[\"dtuse\"], abstract_specs_dtuse[\"dtlink5\"]]\n kwargs = {\"root\": True, \"direction\": direction, \"deptype\": deptype, \"cover\": \"nodes\"}\n\n def nodes(order):\n s = traverse.traverse_nodes(specs, order=order, **kwargs)\n return sorted(list(s))\n\n assert nodes(\"pre\") == nodes(\"post\") == nodes(\"breadth\") == nodes(\"topo\")\n\n\n@pytest.mark.parametrize(\"direction\", (\"children\", \"parents\"))\n@pytest.mark.parametrize(\"root\", (True, False))\n@pytest.mark.parametrize(\"deptype\", (\"all\", (\"link\", \"build\"), (\"run\", \"link\")))\ndef test_all_orders_traverse_the_same_edges(direction, root, deptype, abstract_specs_dtuse):\n # Test whether all graph traversal methods visit the same set of edges.\n # All edges should be returned, including the artificial edges to the input\n # specs when root=True.\n specs = [abstract_specs_dtuse[\"dtuse\"], abstract_specs_dtuse[\"dtlink5\"]]\n kwargs = {\"root\": root, \"direction\": direction, \"deptype\": deptype, \"cover\": \"edges\"}\n\n def edges(order):\n s = traverse.traverse_edges(specs, order=order, **kwargs)\n return sorted(list(s))\n\n assert edges(\"pre\") == edges(\"post\") == edges(\"breadth\") == edges(\"topo\")\n\n\ndef test_breadth_first_traversal(abstract_specs_dtuse):\n # That that depth of discovery is non-decreasing\n s = abstract_specs_dtuse[\"dttop\"]\n depths = [\n depth\n for (depth, _) in traverse.traverse_nodes(\n [s], order=\"breadth\", key=lambda s: s.name, depth=True\n )\n ]\n assert depths == sorted(depths)\n\n\ndef test_breadth_first_deptype_traversal(abstract_specs_dtuse):\n s = abstract_specs_dtuse[\"dtuse\"]\n\n names = [\"dtuse\", \"dttop\", \"dtbuild1\", \"dtlink1\", \"dtbuild2\", \"dtlink2\", \"dtlink3\", \"dtlink4\"]\n\n traversal = traverse.traverse_nodes([s], order=\"breadth\", key=id, deptype=(\"build\", \"link\"))\n assert [x.name for x in traversal] == names\n\n\ndef test_breadth_firsrt_traversal_deptype_with_builddeps(abstract_specs_dtuse):\n s = abstract_specs_dtuse[\"dttop\"]\n\n names = [\"dttop\", \"dtbuild1\", \"dtlink1\", \"dtbuild2\", \"dtlink2\", \"dtlink3\", \"dtlink4\"]\n\n traversal = traverse.traverse_nodes([s], order=\"breadth\", key=id, deptype=(\"build\", \"link\"))\n assert [x.name for x in traversal] == names\n\n\ndef test_breadth_first_traversal_deptype_full(abstract_specs_dtuse):\n s = abstract_specs_dtuse[\"dttop\"]\n\n names = [\n \"dttop\",\n \"dtbuild1\",\n \"dtlink1\",\n \"dtrun1\",\n \"dtbuild2\",\n \"dtlink2\",\n \"dtrun2\",\n \"dtlink3\",\n \"dtlink5\",\n \"dtrun3\",\n \"dtlink4\",\n \"dtbuild3\",\n ]\n\n traversal = traverse.traverse_nodes([s], order=\"breadth\", key=id, deptype=\"all\")\n assert [x.name for x in traversal] == names\n\n\ndef test_breadth_first_traversal_deptype_run(abstract_specs_dtuse):\n s = abstract_specs_dtuse[\"dttop\"]\n names = [\"dttop\", \"dtrun1\", \"dtrun3\"]\n traversal = traverse.traverse_nodes([s], order=\"breadth\", key=id, deptype=\"run\")\n assert [x.name for x in traversal] == names\n\n\ndef test_breadth_first_traversal_reverse(abstract_specs_dt_diamond):\n gen = traverse.traverse_nodes(\n [abstract_specs_dt_diamond[\"dt-diamond-bottom\"]],\n order=\"breadth\",\n key=id,\n direction=\"parents\",\n depth=True,\n )\n assert [(depth, spec.name) for (depth, spec) in gen] == [\n (0, \"dt-diamond-bottom\"),\n (1, \"dt-diamond-left\"),\n (1, \"dt-diamond-right\"),\n (2, \"dt-diamond\"),\n ]\n\n\ndef test_breadth_first_traversal_multiple_input_specs(abstract_specs_dt_diamond):\n # With DFS, the branch dt-diamond -> dt-diamond-left -> dt-diamond-bottom\n # is followed, with BFS, dt-diamond-bottom should be traced through the second\n # input spec dt-diamond-right at depth 1 instead.\n input_specs = [\n abstract_specs_dt_diamond[\"dt-diamond\"],\n abstract_specs_dt_diamond[\"dt-diamond-right\"],\n ]\n gen = traverse.traverse_edges(input_specs, order=\"breadth\", key=id, depth=True, root=False)\n assert [(depth, edge.parent.name, edge.spec.name) for (depth, edge) in gen] == [\n (1, \"dt-diamond\", \"dt-diamond-left\"), # edge from first input spec \"to\" depth 1\n (1, \"dt-diamond-right\", \"dt-diamond-bottom\"), # edge from second input spec \"to\" depth 1\n ]\n\n\ndef test_breadth_first_versus_depth_first_tree(abstract_specs_chain):\n \"\"\"\n The packages chain-a, chain-b, chain-c, chain-d have the following DAG:\n a --> b --> c --> d # a chain\n a --> c # and \"skip\" connections\n a --> d\n Here we test at what depth the nodes are discovered when using BFS vs DFS.\n \"\"\"\n s = abstract_specs_chain[\"chain-a\"]\n\n # BFS should find all nodes as direct deps\n assert [\n (depth, edge.spec.name)\n for (depth, edge) in traverse.traverse_tree([s], cover=\"nodes\", depth_first=False)\n ] == [(0, \"chain-a\"), (1, \"chain-b\"), (1, \"chain-c\"), (1, \"chain-d\")]\n\n # DFS will discover all nodes along the chain a -> b -> c -> d.\n assert [\n (depth, edge.spec.name)\n for (depth, edge) in traverse.traverse_tree([s], cover=\"nodes\", depth_first=True)\n ] == [(0, \"chain-a\"), (1, \"chain-b\"), (2, \"chain-c\"), (3, \"chain-d\")]\n\n # When covering all edges, we should never exceed depth 2 in BFS.\n assert [\n (depth, edge.spec.name)\n for (depth, edge) in traverse.traverse_tree([s], cover=\"edges\", depth_first=False)\n ] == [\n (0, \"chain-a\"),\n (1, \"chain-b\"),\n (2, \"chain-c\"),\n (1, \"chain-c\"),\n (2, \"chain-d\"),\n (1, \"chain-d\"),\n ]\n\n # In DFS we see the chain again.\n assert [\n (depth, edge.spec.name)\n for (depth, edge) in traverse.traverse_tree([s], cover=\"edges\", depth_first=True)\n ] == [\n (0, \"chain-a\"),\n (1, \"chain-b\"),\n (2, \"chain-c\"),\n (3, \"chain-d\"),\n (1, \"chain-c\"),\n (1, \"chain-d\"),\n ]\n\n\n@pytest.mark.parametrize(\"cover\", [\"nodes\", \"edges\"])\n@pytest.mark.parametrize(\"depth_first\", [True, False])\ndef test_tree_traversal_with_key(cover, depth_first, abstract_specs_chain):\n \"\"\"Compare two multisource traversals of the same DAG. In one case the DAG consists of unique\n Spec instances, in the second case there are identical copies of nodes and edges. Traversal\n should be equivalent when nodes are identified by dag_hash.\"\"\"\n a = abstract_specs_chain[\"chain-a\"]\n c = abstract_specs_chain[\"chain-c\"]\n kwargs = {\"cover\": cover, \"depth_first\": depth_first}\n dag_hash = lambda s: s.dag_hash()\n\n # Traverse DAG spanned by a unique set of Spec instances\n first = traverse.traverse_tree([a, c], key=id, **kwargs)\n\n # Traverse equivalent DAG with copies of Spec instances included, keyed by dag hash.\n second = traverse.traverse_tree([a, c.copy()], key=dag_hash, **kwargs)\n\n # Check that the same nodes are discovered at the same depth\n node_at_depth_first = [(depth, dag_hash(edge.spec)) for (depth, edge) in first]\n node_at_depth_second = [(depth, dag_hash(edge.spec)) for (depth, edge) in second]\n assert node_at_depth_first == node_at_depth_second\n\n\ndef test_breadth_first_versus_depth_first_printing(abstract_specs_chain):\n \"\"\"Test breadth-first versus depth-first tree printing.\"\"\"\n s = abstract_specs_chain[\"chain-a\"]\n\n args = {\"format\": \"{name}\", \"color\": False}\n\n dfs_tree_nodes = \"\"\"\\\nchain-a\n ^chain-b\n ^chain-c\n ^chain-d\n\"\"\"\n assert s.tree(depth_first=True, **args) == dfs_tree_nodes\n\n bfs_tree_nodes = \"\"\"\\\nchain-a\n ^chain-b\n ^chain-c\n ^chain-d\n\"\"\"\n assert s.tree(depth_first=False, **args) == bfs_tree_nodes\n\n dfs_tree_edges = \"\"\"\\\nchain-a\n ^chain-b\n ^chain-c\n ^chain-d\n ^chain-c\n ^chain-d\n\"\"\"\n assert s.tree(depth_first=True, cover=\"edges\", **args) == dfs_tree_edges\n\n bfs_tree_edges = \"\"\"\\\nchain-a\n ^chain-b\n ^chain-c\n ^chain-c\n ^chain-d\n ^chain-d\n\"\"\"\n assert s.tree(depth_first=False, cover=\"edges\", **args) == bfs_tree_edges\n\n\n@pytest.fixture()\ndef abstract_specs_toposort():\n # Create a graph that both BFS and DFS would not traverse in topo order, given the\n # default edge ordering (by target spec name). Roots are {A, E} in forward order\n # and {F, G} in backward order.\n # forward: DFS([A, E]) traverses [A, B, F, G, C, D, E] (not topo since C < B)\n # forward: BFS([A, E]) traverses [A, E, B, C, D, F, G] (not topo since C < B)\n # reverse: DFS([F, G]) traverses [F, B, A, D, C, E, G] (not topo since D < A)\n # reverse: BFS([F, G]) traverses [F, G, B, A, D, C, E] (not topo since D < A)\n # E\n # | A\n # | | \\\n # | C |\n # \\ | |\n # D |\n # | /\n # B\n # / \\\n # F G\n return create_dag(\n nodes=[\"A\", \"B\", \"C\", \"D\", \"E\", \"F\", \"G\"],\n edges=(\n (\"A\", \"B\", \"all\"),\n (\"A\", \"C\", \"all\"),\n (\"B\", \"F\", \"all\"),\n (\"B\", \"G\", \"all\"),\n (\"C\", \"D\", \"all\"),\n (\"D\", \"B\", \"all\"),\n (\"E\", \"D\", \"all\"),\n ),\n )\n\n\ndef test_traverse_nodes_topo(abstract_specs_toposort):\n # Test whether we get topologically ordered specs when using traverse_nodes with\n # order=topo and cover=nodes.\n nodes = abstract_specs_toposort\n\n def test_topo(input_specs, direction=\"children\"):\n # Ensure the invariant that all parents of specs[i] are in specs[0:i]\n specs = list(\n traverse.traverse_nodes(input_specs, order=\"topo\", cover=\"nodes\", direction=direction)\n )\n reverse = \"parents\" if direction == \"children\" else \"children\"\n for i in range(len(specs)):\n parents = specs[i].traverse(cover=\"nodes\", direction=reverse, root=False)\n assert set(list(parents)).issubset(set(specs[:i]))\n\n # Traverse forward from roots A and E and a non-root D. Notice that adding D has no\n # effect, it's just to make the test case a bit more complicated, as D is a starting\n # point for traversal, but it's also discovered as a descendant of E and A.\n test_topo([nodes[\"D\"], nodes[\"E\"], nodes[\"A\"]], direction=\"children\")\n\n # Traverse reverse from leafs F and G and non-leaf D\n test_topo([nodes[\"F\"], nodes[\"D\"], nodes[\"G\"]], direction=\"parents\")\n\n\ndef test_traverse_edges_topo(abstract_specs_toposort):\n # Test the invariant that for each node in-edges precede out-edges when\n # using traverse_edges with order=topo.\n nodes = abstract_specs_toposort\n input_specs = [nodes[\"E\"], nodes[\"A\"]]\n\n # Collect pairs of (parent spec name, child spec name)\n edges = [\n (e.parent.name, e.spec.name)\n for e in traverse.traverse_edges(input_specs, order=\"topo\", cover=\"edges\", root=False)\n ]\n\n # See figure above, we have 7 edges (excluding artificial ones to the root)\n assert set(edges) == set(\n [(\"A\", \"B\"), (\"A\", \"C\"), (\"B\", \"F\"), (\"B\", \"G\"), (\"C\", \"D\"), (\"D\", \"B\"), (\"E\", \"D\")]\n )\n\n # Verify that all in-edges precede all out-edges\n for node in nodes.keys():\n in_edge_indices = [i for (i, (parent, child)) in enumerate(edges) if node == child]\n out_edge_indices = [i for (i, (parent, child)) in enumerate(edges) if node == parent]\n if in_edge_indices and out_edge_indices:\n assert max(in_edge_indices) < min(out_edge_indices)\n\n\ndef test_traverse_nodes_no_deps(abstract_specs_dtuse):\n \"\"\"Traversing nodes without deps should be the same as deduplicating the input specs. This may\n not look useful, but can be used to avoid a branch on the call site in which it's otherwise\n easy to forget to deduplicate input specs.\"\"\"\n inputs = [\n abstract_specs_dtuse[\"dtuse\"],\n abstract_specs_dtuse[\"dtlink5\"],\n abstract_specs_dtuse[\"dtuse\"], # <- duplicate\n ]\n outputs = [x for x in traverse.traverse_nodes(inputs, deptype=dt.NONE)]\n assert outputs == [abstract_specs_dtuse[\"dtuse\"], abstract_specs_dtuse[\"dtlink5\"]]\n\n\n@pytest.mark.parametrize(\"cover\", [\"nodes\", \"edges\"])\ndef test_topo_is_bfs_for_trees(cover):\n \"\"\"For trees, both DFS and BFS produce a topological order, but BFS is the most sensible for\n our applications, where we typically want to avoid that transitive dependencies shadow direct\n dependencies in global search paths, etc. This test ensures that for trees, the default topo\n order coincides with BFS.\"\"\"\n binary_tree = create_dag(\n nodes=[\"A\", \"B\", \"C\", \"D\", \"E\", \"F\", \"G\"],\n edges=(\n (\"A\", \"B\", \"all\"),\n (\"A\", \"C\", \"all\"),\n (\"B\", \"D\", \"all\"),\n (\"B\", \"E\", \"all\"),\n (\"C\", \"F\", \"all\"),\n (\"C\", \"G\", \"all\"),\n ),\n )\n\n assert list(traverse.traverse_nodes([binary_tree[\"A\"]], order=\"topo\", cover=cover)) == list(\n traverse.traverse_nodes([binary_tree[\"A\"]], order=\"breadth\", cover=cover)\n )\n\n\n@pytest.mark.parametrize(\"roots\", [[\"A\"], [\"A\", \"B\"], [\"B\", \"A\"], [\"A\", \"B\", \"A\"]])\n@pytest.mark.parametrize(\"order\", [\"breadth\", \"post\", \"pre\"])\n@pytest.mark.parametrize(\"include_root\", [True, False])\ndef test_mixed_depth_visitor(roots, order, include_root):\n \"\"\"Test that the MixedDepthVisitor lists unique edges that are reachable either directly from\n roots through build type edges, or transitively through link type edges. The tests ensures that\n unique edges are listed exactly once.\"\"\"\n my_graph = create_dag(\n nodes=[\"A\", \"B\", \"C\", \"D\", \"E\", \"F\", \"G\", \"H\", \"I\"],\n edges=(\n (\"A\", \"B\", dt.LINK | dt.RUN),\n (\"A\", \"C\", dt.BUILD),\n (\"A\", \"D\", dt.BUILD | dt.RUN),\n (\"A\", \"H\", dt.LINK),\n (\"A\", \"I\", dt.RUN),\n (\"B\", \"D\", dt.BUILD | dt.LINK),\n (\"C\", \"E\", dt.BUILD | dt.LINK | dt.RUN),\n (\"D\", \"F\", dt.LINK),\n (\"D\", \"G\", dt.BUILD | dt.RUN),\n (\"H\", \"B\", dt.LINK),\n ),\n )\n starting_points = traverse.with_artificial_edges([my_graph[root] for root in roots])\n visitor = traverse.MixedDepthVisitor(direct=dt.BUILD, transitive=dt.LINK)\n\n if order == \"pre\":\n edges = traverse.traverse_depth_first_edges_generator(\n starting_points, visitor, post_order=False, root=include_root\n )\n elif order == \"post\":\n edges = traverse.traverse_depth_first_edges_generator(\n starting_points, visitor, post_order=True, root=include_root\n )\n elif order == \"breadth\":\n edges = traverse.traverse_breadth_first_edges_generator(\n starting_points, visitor, root=include_root\n )\n\n artificial_edges = [(None, root) for root in roots] if include_root else []\n simple_edges = [\n (None if edge.parent is None else edge.parent.name, edge.spec.name) for edge in edges\n ]\n\n # make sure that every edge is listed exactly once and that the right edges are listed\n assert len(simple_edges) == len(set(simple_edges))\n assert set(simple_edges) == {\n # the roots\n *artificial_edges,\n (\"A\", \"B\"),\n (\"A\", \"C\"),\n (\"A\", \"D\"),\n (\"A\", \"H\"),\n (\"B\", \"D\"),\n (\"D\", \"F\"),\n (\"H\", \"B\"),\n }\n" }, { "path": "lib/spack/spack/test/url_fetch.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport collections\nimport filecmp\nimport os\nimport pathlib\nimport sys\nimport urllib.error\n\nimport pytest\n\nimport spack.concretize\nimport spack.config\nimport spack.error\nimport spack.fetch_strategy as fs\nimport spack.llnl.util.tty as tty\nimport spack.url\nimport spack.util.crypto as crypto\nimport spack.util.web as web_util\nimport spack.version\nfrom spack.llnl.util.filesystem import is_exe, working_dir\nfrom spack.stage import Stage\nfrom spack.util.executable import which\n\n\n@pytest.fixture\ndef missing_curl(monkeypatch):\n def require_curl():\n raise spack.error.FetchError(\"curl is required but not found\")\n\n monkeypatch.setattr(web_util, \"require_curl\", require_curl)\n\n\n@pytest.fixture(params=list(crypto.hashes.keys()))\ndef checksum_type(request):\n return request.param\n\n\n@pytest.fixture\ndef pkg_factory():\n Pkg = collections.namedtuple(\n \"Pkg\",\n [\n \"url_for_version\",\n \"all_urls_for_version\",\n \"find_valid_url_for_version\",\n \"urls\",\n \"url\",\n \"versions\",\n \"fetch_options\",\n ],\n )\n\n def factory(url, urls, fetch_options={}):\n def fn(v):\n main_url = url or urls[0]\n return spack.url.substitute_version(main_url, v)\n\n def fn_urls(v):\n urls_loc = urls or [url]\n return [spack.url.substitute_version(u, v) for u in urls_loc]\n\n return Pkg(\n find_valid_url_for_version=fn,\n url_for_version=fn,\n all_urls_for_version=fn_urls,\n url=url,\n urls=(urls,),\n versions=collections.defaultdict(dict),\n fetch_options=fetch_options,\n )\n\n return factory\n\n\n@pytest.mark.parametrize(\"method\", [\"curl\", \"urllib\"])\ndef test_urlfetchstrategy_bad_url(tmp_path: pathlib.Path, mutable_config, method):\n \"\"\"Ensure fetch with bad URL fails as expected.\"\"\"\n mutable_config.set(\"config:url_fetch_method\", method)\n fetcher = fs.URLFetchStrategy(url=(tmp_path / \"does-not-exist\").as_uri())\n\n with Stage(fetcher, path=str(tmp_path / \"stage\")):\n with pytest.raises(fs.FailedDownloadError) as exc:\n fetcher.fetch()\n\n assert len(exc.value.exceptions) == 1\n exception = exc.value.exceptions[0]\n\n if method == \"curl\":\n assert isinstance(exception, spack.error.FetchError)\n assert \"Curl failed with error 37\" in str(exception) # FILE_COULDNT_READ_FILE\n elif method == \"urllib\":\n assert isinstance(exception, urllib.error.URLError)\n assert isinstance(exception.reason, FileNotFoundError)\n\n\ndef test_fetch_options(tmp_path: pathlib.Path, mock_archive):\n with spack.config.override(\"config:url_fetch_method\", \"curl\"):\n fetcher = fs.URLFetchStrategy(\n url=mock_archive.url, fetch_options={\"cookie\": \"True\", \"timeout\": 10}\n )\n\n with Stage(fetcher, path=str(tmp_path)):\n assert fetcher.archive_file is None\n fetcher.fetch()\n archive_file = fetcher.archive_file\n assert archive_file is not None\n assert filecmp.cmp(archive_file, mock_archive.archive_file)\n\n\ndef test_fetch_curl_options(tmp_path: pathlib.Path, mock_archive, monkeypatch):\n with spack.config.override(\"config:url_fetch_method\", \"curl -k -q\"):\n fetcher = fs.URLFetchStrategy(\n url=mock_archive.url, fetch_options={\"cookie\": \"True\", \"timeout\": 10}\n )\n\n def check_args(*args, **kwargs):\n # Raise StopIteration to avoid running the rest of the fetch method\n # args[0] is `which curl`, next two are our config options\n assert args[1:3] == (\"-k\", \"-q\")\n raise StopIteration\n\n monkeypatch.setattr(type(fetcher.curl), \"__call__\", check_args)\n\n with Stage(fetcher, path=str(tmp_path)):\n assert fetcher.archive_file is None\n with pytest.raises(StopIteration):\n fetcher.fetch()\n\n\n@pytest.mark.parametrize(\"_fetch_method\", [\"curl\", \"urllib\"])\ndef test_archive_file_errors(tmp_path: pathlib.Path, mutable_config, mock_archive, _fetch_method):\n \"\"\"Ensure FetchStrategy commands may only be used as intended\"\"\"\n with spack.config.override(\"config:url_fetch_method\", _fetch_method):\n fetcher = fs.URLFetchStrategy(url=mock_archive.url)\n with Stage(fetcher, path=str(tmp_path)) as stage:\n assert fetcher.archive_file is None\n with pytest.raises(fs.NoArchiveFileError):\n fetcher.archive(str(tmp_path))\n with pytest.raises(fs.NoArchiveFileError):\n fetcher.expand()\n with pytest.raises(fs.NoArchiveFileError):\n fetcher.reset()\n stage.fetch()\n with pytest.raises(fs.NoDigestError):\n fetcher.check()\n archive_file = fetcher.archive_file\n assert archive_file is not None\n assert filecmp.cmp(archive_file, mock_archive.archive_file)\n\n\nfiles = [(\".tar.gz\", \"z\"), (\".tgz\", \"z\")]\nif sys.platform != \"win32\":\n files += [(\".tar.bz2\", \"j\"), (\".tbz2\", \"j\"), (\".tar.xz\", \"J\"), (\".txz\", \"J\")]\n\n\n@pytest.mark.parametrize(\"secure\", [True, False])\n@pytest.mark.parametrize(\"_fetch_method\", [\"curl\", \"urllib\"])\n@pytest.mark.parametrize(\"mock_archive\", files, indirect=True)\ndef test_fetch(\n mock_archive,\n secure,\n _fetch_method,\n checksum_type,\n default_mock_concretization,\n mutable_mock_repo,\n):\n \"\"\"Fetch an archive and make sure we can checksum it.\"\"\"\n algo = crypto.hash_fun_for_algo(checksum_type)()\n with open(mock_archive.archive_file, \"rb\") as f:\n algo.update(f.read())\n checksum = algo.hexdigest()\n\n # Get a spec and tweak the test package with new checksum params\n s = default_mock_concretization(\"url-test\")\n s.package.url = mock_archive.url\n s.package.versions[spack.version.Version(\"test\")] = {\n checksum_type: checksum,\n \"url\": s.package.url,\n }\n\n # Enter the stage directory and check some properties\n with s.package.stage:\n with spack.config.override(\"config:verify_ssl\", secure):\n with spack.config.override(\"config:url_fetch_method\", _fetch_method):\n s.package.do_stage()\n with working_dir(s.package.stage.source_path):\n assert os.path.exists(\"configure\")\n assert is_exe(\"configure\")\n\n with open(\"configure\", encoding=\"utf-8\") as f:\n contents = f.read()\n assert contents.startswith(\"#!/bin/sh\")\n assert \"echo Building...\" in contents\n\n\n@pytest.mark.parametrize(\n \"spec,url,digest\",\n [\n (\"url-list-test @=0.0.0\", \"foo-0.0.0.tar.gz\", \"00000000000000000000000000000000\"),\n (\"url-list-test @=1.0.0\", \"foo-1.0.0.tar.gz\", \"00000000000000000000000000000100\"),\n (\"url-list-test @=3.0\", \"foo-3.0.tar.gz\", \"00000000000000000000000000000030\"),\n (\"url-list-test @=4.5\", \"foo-4.5.tar.gz\", \"00000000000000000000000000000450\"),\n (\"url-list-test @=2.0.0b2\", \"foo-2.0.0b2.tar.gz\", \"000000000000000000000000000200b2\"),\n (\"url-list-test @=3.0a1\", \"foo-3.0a1.tar.gz\", \"000000000000000000000000000030a1\"),\n (\"url-list-test @=4.5-rc5\", \"foo-4.5-rc5.tar.gz\", \"000000000000000000000000000045c5\"),\n ],\n)\n@pytest.mark.parametrize(\"_fetch_method\", [\"curl\", \"urllib\"])\ndef test_from_list_url(mock_packages, config, spec, url, digest, _fetch_method):\n \"\"\"\n Test URLs in the url-list-test package, which means they should\n have checksums in the package.\n \"\"\"\n with spack.config.override(\"config:url_fetch_method\", _fetch_method):\n s = spack.concretize.concretize_one(spec)\n fetch_strategy = fs.from_list_url(s.package)\n assert isinstance(fetch_strategy, fs.URLFetchStrategy)\n assert os.path.basename(fetch_strategy.url) == url\n assert fetch_strategy.digest == digest\n assert fetch_strategy.extra_options == {}\n s.package.fetch_options = {\"timeout\": 60}\n fetch_strategy = fs.from_list_url(s.package)\n assert fetch_strategy.extra_options == {\"timeout\": 60}\n\n\n@pytest.mark.parametrize(\"_fetch_method\", [\"curl\", \"urllib\"])\n@pytest.mark.parametrize(\n \"requested_version,tarball,digest\",\n [\n # These versions are in the web data path (test/data/web/4.html), but not in the\n # url-list-test package. We expect Spack to generate a URL with the new version.\n (\"=4.5.0\", \"foo-4.5.0.tar.gz\", None),\n (\"=2.0.0\", \"foo-2.0.0.tar.gz\", None),\n ],\n)\ndef test_new_version_from_list_url(\n mock_packages, config, _fetch_method, requested_version, tarball, digest\n):\n \"\"\"Test non-specific URLs from the url-list-test package.\"\"\"\n with spack.config.override(\"config:url_fetch_method\", _fetch_method):\n s = spack.concretize.concretize_one(f\"url-list-test @{requested_version}\")\n fetch_strategy = fs.from_list_url(s.package)\n\n assert isinstance(fetch_strategy, fs.URLFetchStrategy)\n assert os.path.basename(fetch_strategy.url) == tarball\n assert fetch_strategy.digest == digest\n assert fetch_strategy.extra_options == {}\n s.package.fetch_options = {\"timeout\": 60}\n fetch_strategy = fs.from_list_url(s.package)\n assert fetch_strategy.extra_options == {\"timeout\": 60}\n\n\ndef test_nosource_from_list_url(mock_packages, config):\n \"\"\"This test confirms BundlePackages do not have list url.\"\"\"\n s = spack.concretize.concretize_one(\"nosource\")\n fetch_strategy = fs.from_list_url(s.package)\n assert fetch_strategy is None\n\n\ndef test_hash_detection(checksum_type):\n algo = crypto.hash_fun_for_algo(checksum_type)()\n h = \"f\" * (algo.digest_size * 2) # hex -> bytes\n checker = crypto.Checker(h)\n assert checker.hash_name == checksum_type\n\n\ndef test_unknown_hash(checksum_type):\n with pytest.raises(ValueError):\n crypto.Checker(\"a\")\n\n\n@pytest.mark.skipif(which(\"curl\") is None, reason=\"Urllib does not have built-in status bar\")\ndef test_url_with_status_bar(tmp_path: pathlib.Path, mock_archive, monkeypatch, capfd):\n \"\"\"Ensure fetch with status bar option succeeds.\"\"\"\n\n def is_true():\n return True\n\n testpath = str(tmp_path)\n\n monkeypatch.setattr(sys.stdout, \"isatty\", is_true)\n monkeypatch.setattr(tty, \"msg_enabled\", is_true)\n with spack.config.override(\"config:url_fetch_method\", \"curl\"):\n fetcher = fs.URLFetchStrategy(url=mock_archive.url)\n with Stage(fetcher, path=testpath) as stage:\n assert fetcher.archive_file is None\n stage.fetch()\n\n status = capfd.readouterr()[1]\n assert \"##### 100\" in status\n\n\n@pytest.mark.parametrize(\"_fetch_method\", [\"curl\", \"urllib\"])\ndef test_url_extra_fetch(tmp_path: pathlib.Path, mutable_config, mock_archive, _fetch_method):\n \"\"\"Ensure a fetch after downloading is effectively a no-op.\"\"\"\n mutable_config.set(\"config:url_fetch_method\", _fetch_method)\n fetcher = fs.URLFetchStrategy(url=mock_archive.url)\n with Stage(fetcher, path=str(tmp_path)) as stage:\n assert fetcher.archive_file is None\n stage.fetch()\n archive_file = fetcher.archive_file\n assert archive_file is not None\n assert filecmp.cmp(archive_file, mock_archive.archive_file)\n fetcher.fetch()\n\n\n@pytest.mark.parametrize(\n \"url,urls,version,expected\",\n [\n (\n None,\n [\n \"https://ftpmirror.gnu.org/autoconf/autoconf-2.69.tar.gz\",\n \"https://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz\",\n ],\n \"2.62\",\n [\n \"https://ftpmirror.gnu.org/autoconf/autoconf-2.62.tar.gz\",\n \"https://ftp.gnu.org/gnu/autoconf/autoconf-2.62.tar.gz\",\n ],\n )\n ],\n)\n@pytest.mark.parametrize(\"_fetch_method\", [\"curl\", \"urllib\"])\ndef test_candidate_urls(pkg_factory, url, urls, version, expected, _fetch_method):\n \"\"\"Tests that candidate urls include mirrors and that they go through\n pattern matching and substitution for versions.\n \"\"\"\n with spack.config.override(\"config:url_fetch_method\", _fetch_method):\n pkg = pkg_factory(url, urls)\n f = fs._from_merged_attrs(fs.URLFetchStrategy, pkg, version)\n assert f.candidate_urls == expected\n assert f.extra_options == {}\n pkg = pkg_factory(url, urls, fetch_options={\"timeout\": 60})\n f = fs._from_merged_attrs(fs.URLFetchStrategy, pkg, version)\n assert f.extra_options == {\"timeout\": 60}\n\n\n@pytest.mark.regression(\"19673\")\ndef test_missing_curl(tmp_path: pathlib.Path, missing_curl, mutable_config, monkeypatch):\n \"\"\"Ensure a fetch involving missing curl package reports the error.\"\"\"\n mutable_config.set(\"config:url_fetch_method\", \"curl\")\n fetcher = fs.URLFetchStrategy(url=\"http://example.com/file.tar.gz\")\n with pytest.raises(spack.error.FetchError, match=\"curl is required but not found\"):\n with Stage(fetcher, path=str(tmp_path)) as stage:\n stage.fetch()\n\n\ndef test_url_fetch_text_without_url():\n with pytest.raises(spack.error.FetchError, match=\"URL is required\"):\n web_util.fetch_url_text(None)\n\n\ndef test_url_fetch_text_curl_failures(mutable_config, missing_curl, monkeypatch):\n \"\"\"Check fetch_url_text if URL's curl is missing.\"\"\"\n mutable_config.set(\"config:url_fetch_method\", \"curl\")\n with pytest.raises(spack.error.FetchError, match=\"curl is required but not found\"):\n web_util.fetch_url_text(\"https://example.com/\")\n\n\ndef test_url_check_curl_errors():\n \"\"\"Check that standard curl error returncodes raise expected errors.\"\"\"\n # Check returncode 22 (i.e., 404)\n with pytest.raises(spack.error.FetchError, match=\"not found\"):\n web_util.check_curl_code(22)\n\n # Check returncode 60 (certificate error)\n with pytest.raises(spack.error.FetchError, match=\"invalid certificate\"):\n web_util.check_curl_code(60)\n\n\ndef test_url_missing_curl(mutable_config, missing_curl, monkeypatch):\n \"\"\"Check url_exists failures if URL's curl is missing.\"\"\"\n mutable_config.set(\"config:url_fetch_method\", \"curl\")\n with pytest.raises(spack.error.FetchError, match=\"curl is required but not found\"):\n web_util.url_exists(\"https://example.com/\")\n\n\ndef test_url_fetch_text_urllib_web_error(mutable_config, monkeypatch):\n def _raise_web_error(*args, **kwargs):\n raise web_util.SpackWebError(\"bad url\")\n\n monkeypatch.setattr(web_util, \"read_from_url\", _raise_web_error)\n mutable_config.set(\"config:url_fetch_method\", \"urllib\")\n\n with pytest.raises(spack.error.FetchError, match=\"fetch failed\"):\n web_util.fetch_url_text(\"https://example.com/\")\n" }, { "path": "lib/spack/spack/test/url_parse.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Tests Spack's ability to parse the name and version of a package\nbased on its URL.\n\"\"\"\n\nimport os\n\nimport pytest\n\nfrom spack.url import (\n UndetectableVersionError,\n parse_name_and_version,\n parse_name_offset,\n parse_version_offset,\n strip_name_suffixes,\n substitute_version,\n)\nfrom spack.version import Version\n\n\n@pytest.mark.parametrize(\n \"url,version,expected\",\n [\n # No suffix\n (\"rgb-1.0.6\", \"1.0.6\", \"rgb\"),\n (\"nauty26r7\", \"26r7\", \"nauty\"),\n (\"PAGIT.V1.01\", \"1.01\", \"PAGIT\"),\n (\"AmpliconNoiseV1.29\", \"1.29\", \"AmpliconNoise\"),\n # Download type - install\n (\"converge_install_2.3.16\", \"2.3.16\", \"converge\"),\n # Download type - src\n (\"jpegsrc.v9b\", \"9b\", \"jpeg\"),\n (\"blatSrc35\", \"35\", \"blat\"),\n # Download type - open\n (\"RepeatMasker-open-4-0-7\", \"4-0-7\", \"RepeatMasker\"),\n # Download type - archive\n (\"coinhsl-archive-2014.01.17\", \"2014.01.17\", \"coinhsl\"),\n # Download type - std\n (\"ghostscript-fonts-std-8.11\", \"8.11\", \"ghostscript-fonts\"),\n # Download type - bin\n (\"GapCloser-bin-v1.12-r6\", \"1.12-r6\", \"GapCloser\"),\n # Download type - software\n (\"orthomclSoftware-v2.0.9\", \"2.0.9\", \"orthomcl\"),\n # Download version - release\n (\"cbench_release_1.3.0.tar.gz\", \"1.3.0\", \"cbench\"),\n # Download version - snapshot\n (\"gts-snapshot-121130\", \"121130\", \"gts\"),\n # Download version - distrib\n (\"zoltan_distrib_v3.83\", \"3.83\", \"zoltan\"),\n # Download version - latest\n (\"Platypus-latest\", \"N/A\", \"Platypus\"),\n # Download version - complex\n (\"qt-everywhere-opensource-src-5.7.0\", \"5.7.0\", \"qt\"),\n # Arch\n (\"VESTA-x86_64\", \"3.4.6\", \"VESTA\"),\n # VCS - bazaar\n (\"libvterm-0+bzr681\", \"681\", \"libvterm\"),\n # License - gpl\n (\"PyQt-x11-gpl-4.11.3\", \"4.11.3\", \"PyQt\"),\n (\"PyQt4_gpl_x11-4.12.3\", \"4.12.3\", \"PyQt4\"),\n ],\n)\ndef test_url_strip_name_suffixes(url, version, expected):\n stripped = strip_name_suffixes(url, version)\n assert stripped == expected\n\n\n@pytest.mark.parametrize(\n \"name,noffset,ver,voffset,path\",\n [\n # Name in path\n (\"antlr\", 25, \"2.7.7\", 40, \"https://github.com/antlr/antlr/tarball/v2.7.7\"),\n # Name in stem\n (\"gmp\", 32, \"6.0.0a\", 36, \"https://gmplib.org/download/gmp/gmp-6.0.0a.tar.bz2\"),\n # Name in suffix\n # Don't think I've ever seen one of these before\n # We don't look for it, so it would probably fail anyway\n # Version in path\n (\n \"nextflow\",\n 31,\n \"0.20.1\",\n 59,\n \"https://github.com/nextflow-io/nextflow/releases/download/v0.20.1/nextflow\",\n ),\n (\n \"hpcviewer\",\n 30,\n \"2024.02\",\n 51,\n \"https://gitlab.com/hpctoolkit/hpcviewer/-/releases/2024.02/downloads/hpcviewer.tgz\",\n ),\n # Version in stem\n (\"zlib\", 24, \"1.2.10\", 29, \"http://zlib.net/fossils/zlib-1.2.10.tar.gz\"),\n (\n \"slepc\",\n 51,\n \"3.6.2\",\n 57,\n \"http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz\",\n ),\n (\n \"cloog\",\n 61,\n \"0.18.1\",\n 67,\n \"http://www.bastoul.net/cloog/pages/download/count.php3?url=./cloog-0.18.1.tar.gz\",\n ),\n (\n \"libxc\",\n 58,\n \"2.2.2\",\n 64,\n \"http://www.tddft.org/programs/octopus/down.php?file=libxc/libxc-2.2.2.tar.gz\",\n ),\n # Version in suffix\n (\n \"swiftsim\",\n 36,\n \"0.3.0\",\n 76,\n \"http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0\",\n ),\n (\n \"swiftsim\",\n 55,\n \"0.3.0\",\n 95,\n \"https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0\",\n ),\n (\n \"sionlib\",\n 30,\n \"1.7.1\",\n 59,\n \"http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1\",\n ),\n # Regex in name\n (\"voro++\", 40, \"0.4.6\", 47, \"http://math.lbl.gov/voro++/download/dir/voro++-0.4.6.tar.gz\"),\n # SourceForge download\n (\n \"glew\",\n 55,\n \"2.0.0\",\n 60,\n \"https://sourceforge.net/projects/glew/files/glew/2.0.0/glew-2.0.0.tgz/download\",\n ),\n ],\n)\ndef test_url_parse_offset(name, noffset, ver, voffset, path):\n \"\"\"Tests that the name, version and offsets are computed correctly.\n\n Args:\n name (str): expected name\n noffset (int): name offset\n ver (str): expected version\n voffset (int): version offset\n path (str): url to be parsed\n \"\"\"\n # Make sure parse_name_offset and parse_name_version are working\n v, vstart, vlen, vi, vre = parse_version_offset(path)\n n, nstart, nlen, ni, nre = parse_name_offset(path, v)\n\n assert n == name\n assert v == ver\n assert nstart == noffset\n assert vstart == voffset\n\n\n@pytest.mark.parametrize(\n \"name,version,url\",\n [\n # Common Repositories - github downloads\n # name/archive/ver.ver\n (\"nco\", \"4.6.2\", \"https://github.com/nco/nco/archive/4.6.2.tar.gz\"),\n # name/archive/vver.ver\n (\"vim\", \"8.0.0134\", \"https://github.com/vim/vim/archive/v8.0.0134.tar.gz\"),\n # name/archive/name-ver.ver\n (\"oce\", \"0.18\", \"https://github.com/tpaviot/oce/archive/OCE-0.18.tar.gz\"),\n # name/releases/download/vver/name-ver.ver\n (\n \"libmesh\",\n \"1.0.0\",\n \"https://github.com/libMesh/libmesh/releases/download/v1.0.0/libmesh-1.0.0.tar.bz2\",\n ),\n # name/tarball/vver.ver\n (\"git\", \"2.7.1\", \"https://github.com/git/git/tarball/v2.7.1\"),\n # name/zipball/vver.ver\n (\"git\", \"2.7.1\", \"https://github.com/git/git/zipball/v2.7.1\"),\n # Common Repositories - gitlab downloads\n # name/repository/archive.ext?ref=vver.ver\n (\n \"swiftsim\",\n \"0.3.0\",\n \"http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0\",\n ),\n # /api/v4/projects/NAMESPACE%2Fname/repository/archive.ext?sha=vver.ver\n (\n \"swiftsim\",\n \"0.3.0\",\n \"https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0\",\n ),\n # name/repository/archive.ext?ref=name-ver.ver\n (\n \"icet\",\n \"1.2.3\",\n \"https://gitlab.kitware.com/icet/icet/repository/archive.tar.gz?ref=IceT-1.2.3\",\n ),\n # /api/v4/projects/NAMESPACE%2Fname/repository/archive.ext?sha=name-ver.ver\n (\n \"icet\",\n \"1.2.3\",\n \"https://gitlab.kitware.com/api/v4/projects/icet%2Ficet/repository/archive.tar.bz2?sha=IceT-1.2.3\",\n ),\n # Common Repositories - bitbucket downloads\n # name/get/ver.ver\n (\"eigen\", \"3.2.7\", \"https://bitbucket.org/eigen/eigen/get/3.2.7.tar.bz2\"),\n # name/get/vver.ver\n (\"hoomd-blue\", \"1.3.3\", \"https://bitbucket.org/glotzer/hoomd-blue/get/v1.3.3.tar.bz2\"),\n # name/downloads/name-ver.ver\n (\n \"dolfin\",\n \"2016.1.0\",\n \"https://bitbucket.org/fenics-project/dolfin/downloads/dolfin-2016.1.0.tar.gz\",\n ),\n # Common Repositories - sourceforge downloads\n # name-ver.ver\n (\"libpng\", \"1.6.27\", \"http://download.sourceforge.net/libpng/libpng-1.6.27.tar.gz\"),\n (\n \"lcms2\",\n \"2.6\",\n \"http://downloads.sourceforge.net/project/lcms/lcms/2.6/lcms2-2.6.tar.gz\",\n ),\n (\"modules\", \"3.2.10\", \"http://prdownloads.sourceforge.net/modules/modules-3.2.10.tar.gz\"),\n # name-ver.ver.ext/download\n (\n \"glew\",\n \"2.0.0\",\n \"https://sourceforge.net/projects/glew/files/glew/2.0.0/glew-2.0.0.tgz/download\",\n ),\n # Common Repositories - cran downloads\n # name.name_ver.ver-ver.ver\n (\"TH.data\", \"1.0-8\", \"https://cran.r-project.org/src/contrib/TH.data_1.0-8.tar.gz\"),\n (\"knitr\", \"1.14\", \"https://cran.rstudio.com/src/contrib/knitr_1.14.tar.gz\"),\n (\"devtools\", \"1.12.0\", \"https://cloud.r-project.org/src/contrib/devtools_1.12.0.tar.gz\"),\n # Common Repositories - pypi downloads\n # name.name_name-ver.ver\n (\"3to2\", \"1.1.1\", \"https://pypi.python.org/packages/source/3/3to2/3to2-1.1.1.zip\"),\n (\n \"mpmath\",\n \"0.19\",\n \"https://pypi.python.org/packages/source/m/mpmath/mpmath-all-0.19.tar.gz\",\n ),\n (\n \"pandas\",\n \"0.16.0\",\n \"https://pypi.python.org/packages/source/p/pandas/pandas-0.16.0.tar.gz#md5=bfe311f05dc0c351f8955fbd1e296e73\",\n ),\n (\n \"sphinx_rtd_theme\",\n \"0.1.10a0\",\n \"https://pypi.python.org/packages/da/6b/1b75f13d8aa3333f19c6cdf1f0bc9f52ea739cae464fbee050307c121857/sphinx_rtd_theme-0.1.10a0.tar.gz\",\n ),\n (\n \"backports.ssl_match_hostname\",\n \"3.5.0.1\",\n \"https://pypi.io/packages/source/b/backports.ssl_match_hostname/backports.ssl_match_hostname-3.5.0.1.tar.gz\",\n ),\n # Common Repositories - bazaar downloads\n (\"libvterm\", \"681\", \"http://www.leonerd.org.uk/code/libvterm/libvterm-0+bzr681.tar.gz\"),\n # Common Tarball Formats\n # 1st Pass: Simplest case\n # Assume name contains no digits and version contains no letters\n # name-ver.ver\n (\"libpng\", \"1.6.37\", \"http://download.sourceforge.net/libpng/libpng-1.6.37.tar.gz\"),\n # 2nd Pass: Version only\n # Assume version contains no letters\n # ver.ver\n (\"eigen\", \"3.2.7\", \"https://bitbucket.org/eigen/eigen/get/3.2.7.tar.bz2\"),\n # ver.ver-ver\n (\n \"ImageMagick\",\n \"7.0.2-7\",\n \"https://github.com/ImageMagick/ImageMagick/archive/7.0.2-7.tar.gz\",\n ),\n # vver.ver\n (\"CGNS\", \"3.3.0\", \"https://github.com/CGNS/CGNS/archive/v3.3.0.tar.gz\"),\n # vver_ver\n (\n \"luafilesystem\",\n \"1_6_3\",\n \"https://github.com/keplerproject/luafilesystem/archive/v1_6_3.tar.gz\",\n ),\n # 3rd Pass: No separator characters are used\n # Assume name contains no digits\n # namever\n (\"turbolinux\", \"702\", \"file://{0}/turbolinux702.tar.gz\".format(os.getcwd())),\n (\"nauty\", \"26r7\", \"http://pallini.di.uniroma1.it/nauty26r7.tar.gz\"),\n # 4th Pass: A single separator character is used\n # Assume name contains no digits\n # name-name-ver-ver\n (\n \"Trilinos\",\n \"12-10-1\",\n \"https://github.com/trilinos/Trilinos/archive/trilinos-release-12-10-1.tar.gz\",\n ),\n (\n \"panda\",\n \"2016-03-07\",\n \"http://comopt.ifi.uni-heidelberg.de/software/PANDA/downloads/panda-2016-03-07.tar\",\n ),\n (\"gts\", \"121130\", \"http://gts.sourceforge.net/tarballs/gts-snapshot-121130.tar.gz\"),\n (\"cdd\", \"061a\", \"http://www.cs.mcgill.ca/~fukuda/download/cdd/cdd-061a.tar.gz\"),\n # name_name_ver_ver\n (\n \"tinyxml\",\n \"2_6_2\",\n \"https://sourceforge.net/projects/tinyxml/files/tinyxml/2.6.2/tinyxml_2_6_2.tar.gz\",\n ),\n (\n \"boost\",\n \"1_55_0\",\n \"http://downloads.sourceforge.net/project/boost/boost/1.55.0/boost_1_55_0.tar.bz2\",\n ),\n (\"yorick\", \"2_2_04\", \"https://github.com/dhmunro/yorick/archive/y_2_2_04.tar.gz\"),\n (\n \"tbb\",\n \"44_20160413\",\n \"https://www.threadingbuildingblocks.org/sites/default/files/software_releases/source/tbb44_20160413oss_src.tgz\",\n ),\n # name.name.ver.ver\n (\"prank\", \"150803\", \"http://wasabiapp.org/download/prank/prank.source.150803.tgz\"),\n (\"jpeg\", \"9b\", \"http://www.ijg.org/files/jpegsrc.v9b.tar.gz\"),\n (\"openjpeg\", \"2.1\", \"https://github.com/uclouvain/openjpeg/archive/version.2.1.tar.gz\"),\n # name.namever.ver\n (\n \"atlas\",\n \"3.11.34\",\n \"http://sourceforge.net/projects/math-atlas/files/Developer%20%28unstable%29/3.11.34/atlas3.11.34.tar.bz2\",\n ),\n (\n \"visit\",\n \"2.10.1\",\n \"http://portal.nersc.gov/project/visit/releases/2.10.1/visit2.10.1.tar.gz\",\n ),\n (\"geant\", \"4.10.01.p03\", \"http://geant4.cern.ch/support/source/geant4.10.01.p03.tar.gz\"),\n (\"tcl\", \"8.6.5\", \"http://prdownloads.sourceforge.net/tcl/tcl8.6.5-src.tar.gz\"),\n # 5th Pass: Two separator characters are used\n # Name may contain digits, version may contain letters\n # name-name-ver.ver\n (\"m4\", \"1.4.17\", \"https://ftp.gnu.org/gnu/m4/m4-1.4.17.tar.gz\"),\n (\"gmp\", \"6.0.0a\", \"https://gmplib.org/download/gmp/gmp-6.0.0a.tar.bz2\"),\n (\n \"LaunchMON\",\n \"1.0.2\",\n \"https://github.com/LLNL/LaunchMON/releases/download/v1.0.2/launchmon-v1.0.2.tar.gz\",\n ),\n # name-ver-ver.ver\n (\"libedit\", \"20150325-3.1\", \"http://thrysoee.dk/editline/libedit-20150325-3.1.tar.gz\"),\n # name-name-ver_ver\n (\"icu4c\", \"57_1\", \"http://download.icu-project.org/files/icu4c/57.1/icu4c-57_1-src.tgz\"),\n # name_name_ver.ver\n (\n \"superlu_dist\",\n \"4.1\",\n \"http://crd-legacy.lbl.gov/~xiaoye/SuperLU/superlu_dist_4.1.tar.gz\",\n ),\n (\"pexsi\", \"0.9.0\", \"https://math.berkeley.edu/~linlin/pexsi/download/pexsi_v0.9.0.tar.gz\"),\n # name_name.ver.ver\n (\"fer\", \"696\", \"ftp://ftp.pmel.noaa.gov/ferret/pub/source/fer_source.v696.tar.gz\"),\n # name_name_ver-ver\n (\n \"Bridger\",\n \"2014-12-01\",\n \"https://downloads.sourceforge.net/project/rnaseqassembly/Bridger_r2014-12-01.tar.gz\",\n ),\n # name-name-ver.ver-ver.ver\n (\n \"sowing\",\n \"1.1.23-p1\",\n \"http://ftp.mcs.anl.gov/pub/petsc/externalpackages/sowing-1.1.23-p1.tar.gz\",\n ),\n (\n \"bib2xhtml\",\n \"3.0-15-gf506\",\n \"http://www.spinellis.gr/sw/textproc/bib2xhtml/bib2xhtml-v3.0-15-gf506.tar.gz\",\n ),\n # namever.ver-ver.ver\n (\n \"go\",\n \"1.4-bootstrap-20161024\",\n \"https://storage.googleapis.com/golang/go1.4-bootstrap-20161024.tar.gz\",\n ),\n # 6th Pass: All three separator characters are used\n # Name may contain digits, version may contain letters\n # name_name-ver.ver\n (\n \"the_silver_searcher\",\n \"0.32.0\",\n \"http://geoff.greer.fm/ag/releases/the_silver_searcher-0.32.0.tar.gz\",\n ),\n (\n \"sphinx_rtd_theme\",\n \"0.1.10a0\",\n \"https://pypi.python.org/packages/source/s/sphinx_rtd_theme/sphinx_rtd_theme-0.1.10a0.tar.gz\",\n ),\n # name.name_ver.ver-ver.ver\n (\"TH.data\", \"1.0-8\", \"https://cran.r-project.org/src/contrib/TH.data_1.0-8.tar.gz\"),\n (\"XML\", \"3.98-1.4\", \"https://cran.r-project.org/src/contrib/XML_3.98-1.4.tar.gz\"),\n # name-name-ver.ver_ver.ver\n (\n \"pypar\",\n \"2.1.5_108\",\n \"https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/pypar/pypar-2.1.5_108.tgz\",\n ),\n # name-namever.ver_ver.ver\n (\n \"STAR-CCM+\",\n \"11.06.010_02\",\n \"file://{0}/STAR-CCM+11.06.010_02_linux-x86_64.tar.gz\".format(os.getcwd()),\n ),\n # name-name_name-ver.ver\n (\n \"PerlIO-utf8_strict\",\n \"0.002\",\n \"http://search.cpan.org/CPAN/authors/id/L/LE/LEONT/PerlIO-utf8_strict-0.002.tar.gz\",\n ),\n # Various extensions\n # .tar.gz\n (\n \"libXcursor\",\n \"1.1.14\",\n \"https://www.x.org/archive/individual/lib/libXcursor-1.1.14.tar.gz\",\n ),\n # .tar.bz2\n (\"mpfr\", \"4.0.1\", \"https://ftpmirror.gnu.org/mpfr/mpfr-4.0.1.tar.bz2\"),\n # .tar.xz\n (\"pkgconf\", \"1.5.4\", \"http://distfiles.dereferenced.org/pkgconf/pkgconf-1.5.4.tar.xz\"),\n # .tar.Z\n (\n \"Gblocks\",\n \"0.91b\",\n \"http://molevol.cmima.csic.es/castresana/Gblocks/Gblocks_Linux64_0.91b.tar.Z\",\n ),\n # .tar.zip\n (\n \"bcl2fastq2\",\n \"2.19.1.403\",\n \"ftp://webdata2:webdata2@ussd-ftp.illumina.com/downloads/software/bcl2fastq/bcl2fastq2-v2.19.1.403-tar.zip\",\n ),\n # .tar, .TAR\n (\n \"python-meep\",\n \"1.4.2\",\n \"https://launchpad.net/python-meep/1.4/1.4/+download/python-meep-1.4.2.tar\",\n ),\n (\n \"python-meep\",\n \"1.4.2\",\n \"https://launchpad.net/python-meep/1.4/1.4/+download/python-meep-1.4.2.TAR\",\n ),\n # .gz\n (\"libXcursor\", \"1.1.14\", \"https://www.x.org/archive/individual/lib/libXcursor-1.1.14.gz\"),\n # .bz2\n (\"mpfr\", \"4.0.1\", \"https://ftpmirror.gnu.org/mpfr/mpfr-4.0.1.bz2\"),\n # .xz\n (\"pkgconf\", \"1.5.4\", \"http://distfiles.dereferenced.org/pkgconf/pkgconf-1.5.4.xz\"),\n # .Z\n (\n \"Gblocks\",\n \"0.91b\",\n \"http://molevol.cmima.csic.es/castresana/Gblocks/Gblocks_Linux64_0.91b.Z\",\n ),\n # .zip\n (\"bliss\", \"0.73\", \"http://www.tcs.hut.fi/Software/bliss/bliss-0.73.zip\"),\n # .tgz\n (\"ADOL-C\", \"2.6.1\", \"http://www.coin-or.org/download/source/ADOL-C/ADOL-C-2.6.1.tgz\"),\n # .tbz\n (\"mpfr\", \"4.0.1\", \"https://ftpmirror.gnu.org/mpfr/mpfr-4.0.1.tbz\"),\n # .tbz2\n (\"mpfr\", \"4.0.1\", \"https://ftpmirror.gnu.org/mpfr/mpfr-4.0.1.tbz2\"),\n # .txz\n (\"kim-api\", \"2.1.0\", \"https://s3.openkim.org/kim-api/kim-api-2.1.0.txz\"),\n # 8th Pass: Query strings\n # suffix queries\n (\n \"swiftsim\",\n \"0.3.0\",\n \"http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0\",\n ),\n (\n \"swiftsim\",\n \"0.3.0\",\n \"https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0\",\n ),\n (\"sionlib\", \"1.7.1\", \"http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1\"),\n (\"jube2\", \"2.2.2\", \"https://apps.fz-juelich.de/jsc/jube/jube2/download.php?version=2.2.2\"),\n (\n \"archive\",\n \"1.0.0\",\n \"https://code.ornl.gov/eck/papyrus/repository/archive.tar.bz2?ref=v1.0.0\",\n ),\n (\n \"VecGeom\",\n \"0.3.rc\",\n \"https://gitlab.cern.ch/api/v4/projects/VecGeom%2FVecGeom/repository/archive.tar.gz?sha=v0.3.rc\",\n ),\n (\n \"parsplice\",\n \"1.1\",\n \"https://gitlab.com/api/v4/projects/exaalt%2Fparsplice/repository/archive.tar.gz?sha=v1.1\",\n ),\n (\n \"busco\",\n \"2.0.1\",\n \"https://gitlab.com/api/v4/projects/ezlab%2Fbusco/repository/archive.tar.gz?sha=2.0.1\",\n ),\n (\n \"libaec\",\n \"1.0.2\",\n \"https://gitlab.dkrz.de/api/v4/projects/k202009%2Flibaec/repository/archive.tar.gz?sha=v1.0.2\",\n ),\n (\n \"icet\",\n \"2.1.1\",\n \"https://gitlab.kitware.com/api/v4/projects/icet%2Ficet/repository/archive.tar.bz2?sha=IceT-2.1.1\",\n ),\n (\n \"vtk-m\",\n \"1.3.0\",\n \"https://gitlab.kitware.com/api/v4/projects/vtk%2Fvtk-m/repository/archive.tar.gz?sha=v1.3.0\",\n ),\n (\n \"GATK\",\n \"3.8-1-0-gf15c1c3ef\",\n \"https://software.broadinstitute.org/gatk/download/auth?package=GATK-archive&version=3.8-1-0-gf15c1c3ef\",\n ),\n # stem queries\n (\n \"slepc\",\n \"3.6.2\",\n \"http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz\",\n ),\n (\n \"otf\",\n \"1.12.5salmon\",\n \"http://wwwpub.zih.tu-dresden.de/%7Emlieber/dcount/dcount.php?package=otf&get=OTF-1.12.5salmon.tar.gz\",\n ),\n (\n \"eospac\",\n \"6.4.0beta.1\",\n \"http://laws-green.lanl.gov/projects/data/eos/get_file.php?package=eospac&filename=eospac_v6.4.0beta.1_r20171213193219.tgz\",\n ),\n (\n \"vampirtrace\",\n \"5.14.4\",\n \"http://wwwpub.zih.tu-dresden.de/~mlieber/dcount/dcount.php?package=vampirtrace&get=VampirTrace-5.14.4.tar.gz\",\n ),\n (\"EvtGen\", \"01.07.00\", \"https://evtgen.hepforge.org/downloads?f=EvtGen-01.07.00.tar.gz\"),\n # (we don't actually look for these, they are picked up\n # during the preliminary stem parsing)\n (\"octopus\", \"6.0\", \"http://octopus-code.org/down.php?file=6.0/octopus-6.0.tar.gz\"),\n (\n \"cloog\",\n \"0.18.1\",\n \"http://www.bastoul.net/cloog/pages/download/count.php3?url=./cloog-0.18.1.tar.gz\",\n ),\n (\n \"libxc\",\n \"2.2.2\",\n \"http://www.tddft.org/programs/octopus/down.php?file=libxc/libxc-2.2.2.tar.gz\",\n ),\n (\n \"cistem\",\n \"1.0.0-beta\",\n \"https://cistem.org/system/tdf/upload3/cistem-1.0.0-beta-source-code.tar.gz?file=1&type=cistem_details&id=37&force=0\",\n ),\n (\n \"Magics\",\n \"4.1.0\",\n \"https://confluence.ecmwf.int/download/attachments/3473464/Magics-4.1.0-Source.tar.gz?api=v2\",\n ),\n (\n \"grib_api\",\n \"1.17.0\",\n \"https://software.ecmwf.int/wiki/download/attachments/3473437/grib_api-1.17.0-Source.tar.gz?api=v2\",\n ),\n (\n \"eccodes\",\n \"2.2.0\",\n \"https://software.ecmwf.int/wiki/download/attachments/45757960/eccodes-2.2.0-Source.tar.gz?api=v2\",\n ),\n (\n \"SWFFT\",\n \"1.0\",\n \"https://xgitlab.cels.anl.gov/api/v4/projects/hacc%2FSWFFT/repository/archive.tar.gz?sha=v1.0\",\n ),\n # 9th Pass: Version in path\n # github.com/repo/name/releases/download/name-vver/name\n (\n \"nextflow\",\n \"0.20.1\",\n \"https://github.com/nextflow-io/nextflow/releases/download/v0.20.1/nextflow\",\n ),\n # ver/name\n (\n \"ncbi\",\n \"2.2.26\",\n \"ftp://ftp.ncbi.nlm.nih.gov/blast/executables/legacy.NOTSUPPORTED/2.2.26/ncbi.tar.gz\",\n ),\n # Other tests for corner cases\n # single character name\n (\"R\", \"3.3.2\", \"https://cloud.r-project.org/src/base/R-3/R-3.3.2.tar.gz\"),\n # name starts with digit\n (\"3to2\", \"1.1.1\", \"https://pypi.python.org/packages/source/3/3to2/3to2-1.1.1.zip\"),\n # plus in name\n (\n \"gtk+\",\n \"2.24.31\",\n \"http://ftp.gnome.org/pub/gnome/sources/gtk+/2.24/gtk+-2.24.31.tar.xz\",\n ),\n (\"voro++\", \"0.4.6\", \"http://math.lbl.gov/voro++/download/dir/voro++-0.4.6.tar.gz\"),\n # Name comes before download.php\n (\"sionlib\", \"1.7.1\", \"http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1\"),\n # Ignore download.php\n (\n \"slepc\",\n \"3.6.2\",\n \"http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz\",\n ),\n (\n \"ScientificPython\",\n \"2.8.1\",\n \"https://sourcesup.renater.fr/frs/download.php/file/4411/ScientificPython-2.8.1.tar.gz\",\n ),\n # gloox beta style\n (\"gloox\", \"1.0-beta7\", \"http://camaya.net/download/gloox-1.0-beta7.tar.bz2\"),\n # sphinx beta style\n (\"sphinx\", \"1.10-beta\", \"http://sphinxsearch.com/downloads/sphinx-1.10-beta.tar.gz\"),\n # ruby version style\n (\"ruby\", \"1.9.1-p243\", \"ftp://ftp.ruby-lang.org/pub/ruby/1.9/ruby-1.9.1-p243.tar.gz\"),\n # rc style\n (\n \"libvorbis\",\n \"1.2.2rc1\",\n \"http://downloads.xiph.org/releases/vorbis/libvorbis-1.2.2rc1.tar.bz2\",\n ),\n # dash rc style\n (\"js\", \"1.8.0-rc1\", \"http://ftp.mozilla.org/pub/mozilla.org/js/js-1.8.0-rc1.tar.gz\"),\n # apache version style\n (\n \"apache-cassandra\",\n \"1.2.0-rc2\",\n \"http://www.apache.org/dyn/closer.cgi?path=/cassandra/1.2.0/apache-cassandra-1.2.0-rc2-bin.tar.gz\",\n ),\n # xaw3d version\n (\"Xaw3d\", \"1.5E\", \"ftp://ftp.visi.com/users/hawkeyd/X/Xaw3d-1.5E.tar.gz\"),\n # fann version\n (\n \"fann\",\n \"2.1.0beta\",\n \"http://downloads.sourceforge.net/project/fann/fann/2.1.0beta/fann-2.1.0beta.zip\",\n ),\n # imap version\n (\"imap\", \"2007f\", \"ftp://ftp.cac.washington.edu/imap/imap-2007f.tar.gz\"),\n # suite3270 version\n (\n \"suite3270\",\n \"3.3.12ga7\",\n \"http://sourceforge.net/projects/x3270/files/x3270/3.3.12ga7/suite3270-3.3.12ga7-src.tgz\",\n ),\n # scalasca version\n (\n \"cube\",\n \"4.2.3\",\n \"http://apps.fz-juelich.de/scalasca/releases/cube/4.2/dist/cube-4.2.3.tar.gz\",\n ),\n (\n \"cube\",\n \"4.3-TP1\",\n \"http://apps.fz-juelich.de/scalasca/releases/cube/4.3/dist/cube-4.3-TP1.tar.gz\",\n ),\n # github raw url\n (\n \"CLAMR\",\n \"2.0.7\",\n \"https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true\",\n ),\n # luaposix version\n (\n \"luaposix\",\n \"33.4.0\",\n \"https://github.com/luaposix/luaposix/archive/release-v33.4.0.tar.gz\",\n ),\n # nco version\n (\"nco\", \"4.6.2-beta03\", \"https://github.com/nco/nco/archive/4.6.2-beta03.tar.gz\"),\n (\"nco\", \"4.6.3-alpha04\", \"https://github.com/nco/nco/archive/4.6.3-alpha04.tar.gz\"),\n ],\n)\ndef test_url_parse_name_and_version(name, version, url):\n # Make sure correct name and version are extracted.\n parsed_name, parsed_version = parse_name_and_version(url)\n assert parsed_name == name\n assert parsed_version == Version(version)\n\n # Make sure Spack formulates the right URL when we try to\n # build one with a specific version.\n assert url == substitute_version(url, version)\n\n\n@pytest.mark.parametrize(\n \"not_detectable_url\",\n [\n \"http://www.netlib.org/blas/blast-forum/cblas.tgz\",\n \"http://www.netlib.org/voronoi/triangle.zip\",\n ],\n)\ndef test_no_version(not_detectable_url):\n with pytest.raises(UndetectableVersionError):\n parse_name_and_version(not_detectable_url)\n" }, { "path": "lib/spack/spack/test/url_substitution.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Tests Spack's ability to substitute a different version into a URL.\"\"\"\n\nimport os\n\nimport pytest\n\nimport spack.url\n\n\n@pytest.mark.parametrize(\n \"base_url,version,expected\",\n [\n # Ensures that substituting the same version results in the same URL\n (\n \"http://www.mr511.de/software/libelf-0.8.13.tar.gz\",\n \"0.8.13\",\n \"http://www.mr511.de/software/libelf-0.8.13.tar.gz\",\n ),\n # Test a completely different version syntax\n (\n \"http://www.prevanders.net/libdwarf-20130729.tar.gz\",\n \"8.12\",\n \"http://www.prevanders.net/libdwarf-8.12.tar.gz\",\n ),\n # Test a URL where the version appears twice\n # It should get substituted both times\n (\n \"https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz\",\n \"2.1.3\",\n \"https://github.com/hpc/mpileaks/releases/download/v2.1.3/mpileaks-2.1.3.tar.gz\",\n ),\n # Test now with a partial prefix earlier in the URL\n # This is hard to figure out so Spack only substitutes\n # the last instance of the version\n (\n \"https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.1.0.tar.bz2\",\n \"2.2.0\",\n \"https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.2.0.tar.bz2\",\n ),\n (\n \"https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.1.0.tar.bz2\",\n \"2.2\",\n \"https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.2.tar.bz2\",\n ),\n # No separator between the name and version of the package\n (\n \"file://{0}/turbolinux702.tar.gz\".format(os.getcwd()),\n \"703\",\n \"file://{0}/turbolinux703.tar.gz\".format(os.getcwd()),\n ),\n (\n \"https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true\",\n \"2.0.7\",\n \"https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true\",\n ),\n (\n \"https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true\",\n \"4.7\",\n \"https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v4.7.tgz?raw=true\",\n ),\n # Package name contains regex characters\n (\n \"http://math.lbl.gov/voro++/download/dir/voro++-0.4.6.tar.gz\",\n \"1.2.3\",\n \"http://math.lbl.gov/voro++/download/dir/voro++-1.2.3.tar.gz\",\n ),\n ],\n)\ndef test_url_substitution(base_url, version, expected):\n computed = spack.url.substitute_version(base_url, version)\n assert computed == expected\n" }, { "path": "lib/spack/spack/test/util/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n" }, { "path": "lib/spack/spack/test/util/archive.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport gzip\nimport hashlib\nimport os\nimport shutil\nimport tarfile\nfrom pathlib import Path, PurePath\n\nimport pytest\n\nimport spack.util.crypto\nimport spack.version\nfrom spack.llnl.util.filesystem import working_dir\nfrom spack.util.archive import (\n gzip_compressed_tarfile,\n reproducible_tarfile_from_prefix,\n retrieve_commit_from_archive,\n)\n\n\ndef test_gzip_compressed_tarball_is_reproducible(tmp_path: Path, monkeypatch):\n \"\"\"Test gzip_compressed_tarfile and reproducible_tarfile_from_prefix for reproducibility\"\"\"\n\n with working_dir(str(tmp_path)):\n # Create a few directories\n root = Path(\"root\")\n dir_a = root / \"a\"\n dir_b = root / \"b\"\n root.mkdir(mode=0o777)\n dir_a.mkdir(mode=0o777)\n dir_b.mkdir(mode=0o777)\n\n (root / \"y\").touch()\n (root / \"x\").touch()\n\n (dir_a / \"executable\").touch(mode=0o777)\n (dir_a / \"data\").touch(mode=0o666)\n (dir_a / \"symlink_file\").symlink_to(\"data\")\n (dir_a / \"symlink_dir\").symlink_to(PurePath(\"..\", \"b\"))\n try:\n os.link(dir_a / \"executable\", dir_a / \"hardlink\")\n hardlink_support = True\n except OSError:\n hardlink_support = False\n\n (dir_b / \"executable\").touch(mode=0o777)\n (dir_b / \"data\").touch(mode=0o666)\n (dir_b / \"symlink_file\").symlink_to(\"data\")\n (dir_b / \"symlink_dir\").symlink_to(PurePath(\"..\", \"a\"))\n\n # Create the first tarball\n with gzip_compressed_tarfile(\"fst.tar.gz\") as (tar, gzip_checksum_1, tarfile_checksum_1):\n reproducible_tarfile_from_prefix(tar, \"root\")\n\n # Expected mode for non-dirs is 644 if not executable, 755 if executable. Better to compute\n # that as we don't know the umask of the user running the test.\n expected_mode = lambda name: (\n 0o755 if Path(*name.split(\"/\")).lstat().st_mode & 0o100 else 0o644\n )\n\n # Verify the tarball contents\n with tarfile.open(\"fst.tar.gz\", \"r:gz\") as tar:\n # Directories (mode is always 755)\n for dir in (\"root\", \"root/a\", \"root/b\"):\n m = tar.getmember(dir)\n assert m.isdir()\n assert m.mode == 0o755\n assert m.uid == m.gid == 0\n assert m.uname == m.gname == \"\"\n\n # Non-executable regular files\n for file in (\n \"root/x\",\n \"root/y\",\n \"root/a/data\",\n \"root/b/data\",\n \"root/a/executable\",\n \"root/b/executable\",\n ):\n m = tar.getmember(file)\n assert m.isreg()\n assert m.mode == expected_mode(file)\n assert m.uid == m.gid == 0\n assert m.uname == m.gname == \"\"\n\n # Symlinks\n for file in (\n \"root/a/symlink_file\",\n \"root/a/symlink_dir\",\n \"root/b/symlink_file\",\n \"root/b/symlink_dir\",\n ):\n m = tar.getmember(file)\n assert m.issym()\n assert m.mode == 0o755\n assert m.uid == m.gid == m.mtime == 0\n assert m.uname == m.gname == \"\"\n\n # Verify the symlink targets. Notice that symlink targets are copied verbatim. That\n # means the value is platform specific for relative symlinks within the current prefix,\n # as on Windows they'd be ..\\a and ..\\b instead of ../a and ../b. So, reproducilility\n # is only guaranteed per-platform currently.\n assert PurePath(tar.getmember(\"root/a/symlink_file\").linkname) == PurePath(\"data\")\n assert PurePath(tar.getmember(\"root/b/symlink_file\").linkname) == PurePath(\"data\")\n assert PurePath(tar.getmember(\"root/a/symlink_dir\").linkname) == PurePath(\"..\", \"b\")\n assert PurePath(tar.getmember(\"root/b/symlink_dir\").linkname) == PurePath(\"..\", \"a\")\n\n # Check hardlink if supported\n if hardlink_support:\n m = tar.getmember(\"root/a/hardlink\")\n assert m.islnk()\n assert m.mode == expected_mode(\"root/a/hardlink\")\n assert m.uid == m.gid == 0\n assert m.uname == m.gname == \"\"\n # Hardlink targets are always in posix format, as they reference a file that exists\n # in the tarball.\n assert m.linkname == \"root/a/executable\"\n\n # Finally verify if entries are ordered by (is_dir, name)\n assert [t.name for t in tar.getmembers()] == [\n \"root\",\n \"root/x\",\n \"root/y\",\n \"root/a\",\n \"root/a/data\",\n \"root/a/executable\",\n *([\"root/a/hardlink\"] if hardlink_support else []),\n \"root/a/symlink_dir\",\n \"root/a/symlink_file\",\n \"root/b\",\n \"root/b/data\",\n \"root/b/executable\",\n \"root/b/symlink_dir\",\n \"root/b/symlink_file\",\n ]\n\n # Delete the current root dir, extract the first tarball, create a second\n shutil.rmtree(root)\n with tarfile.open(\"fst.tar.gz\", \"r:gz\") as tar:\n tar.extractall()\n\n # Create the second tarball\n with gzip_compressed_tarfile(\"snd.tar.gz\") as (tar, gzip_checksum_2, tarfile_checksum_2):\n reproducible_tarfile_from_prefix(tar, \"root\")\n\n # Verify the .tar.gz checksums are identical and correct\n assert (\n gzip_checksum_1.hexdigest()\n == gzip_checksum_2.hexdigest()\n == spack.util.crypto.checksum(hashlib.sha256, \"fst.tar.gz\")\n == spack.util.crypto.checksum(hashlib.sha256, \"snd.tar.gz\")\n )\n\n # Verify the .tar checksums are identical and correct\n with gzip.open(\"fst.tar.gz\", \"rb\") as f, gzip.open(\"snd.tar.gz\", \"rb\") as g:\n assert (\n tarfile_checksum_1.hexdigest()\n == tarfile_checksum_2.hexdigest()\n == spack.util.crypto.checksum_stream(hashlib.sha256, f) # type: ignore\n == spack.util.crypto.checksum_stream(hashlib.sha256, g) # type: ignore\n )\n\n\ndef test_reproducible_tarfile_from_prefix_path_to_name(tmp_path: Path):\n prefix = tmp_path / \"example\"\n prefix.mkdir()\n (prefix / \"file1\").write_bytes(b\"file\")\n (prefix / \"file2\").write_bytes(b\"file\")\n\n def map_prefix(path: str) -> str:\n \"\"\"maps / to some/common/prefix/\"\"\"\n p = PurePath(path)\n assert p.parts[: len(prefix.parts)] == prefix.parts, f\"{path} is not under {prefix}\"\n return PurePath(\"some\", \"common\", \"prefix\", *p.parts[len(prefix.parts) :]).as_posix()\n\n with tarfile.open(tmp_path / \"example.tar\", \"w\") as tar:\n reproducible_tarfile_from_prefix(\n tar,\n str(tmp_path / \"example\"),\n include_parent_directories=True,\n path_to_name=map_prefix,\n )\n\n with tarfile.open(tmp_path / \"example.tar\", \"r\") as tar:\n assert [t.name for t in tar.getmembers() if t.isdir()] == [\n \"some\",\n \"some/common\",\n \"some/common/prefix\",\n ]\n assert [t.name for t in tar.getmembers() if t.isfile()] == [\n \"some/common/prefix/file1\",\n \"some/common/prefix/file2\",\n ]\n\n\n@pytest.mark.parametrize(\"ref\", (\"test-branch\", \"test-tag\", \"annotated-tag\"))\ndef test_get_commits_from_archive(mock_git_repository, tmp_path: Path, ref):\n git_exe = mock_git_repository.git_exe\n with working_dir(str(tmp_path)):\n archive_file = str(tmp_path / \"archive.tar.gz\")\n path_to_name = lambda path: PurePath(path).relative_to(mock_git_repository.path).as_posix()\n\n # round trip the git repo, the desired ref will always be checked out\n git_exe(\"-C\", mock_git_repository.path, \"checkout\", ref)\n with gzip_compressed_tarfile(archive_file) as (tar, _, _):\n reproducible_tarfile_from_prefix(\n tar=tar, prefix=mock_git_repository.path, path_to_name=path_to_name\n )\n git_exe(\n \"-C\",\n mock_git_repository.path,\n \"checkout\",\n mock_git_repository.checks[\"default\"].revision,\n )\n commit = retrieve_commit_from_archive(archive_file, ref)\n assert commit\n assert spack.version.is_git_commit_sha(commit)\n\n\ndef test_can_tell_if_archive_has_git(mock_git_repository, tmp_path: Path):\n with working_dir(str(tmp_path)):\n archive_file = str(tmp_path / \"archive.tar.gz\")\n path_to_name = lambda path: PurePath(path).relative_to(mock_git_repository.path).as_posix()\n exclude = lambda entry: \".git\" in PurePath(entry.path).parts\n with gzip_compressed_tarfile(archive_file) as (tar, _, _):\n reproducible_tarfile_from_prefix(\n tar=tar, prefix=mock_git_repository.path, path_to_name=path_to_name, skip=exclude\n )\n with pytest.raises(AssertionError) as err:\n retrieve_commit_from_archive(archive_file, \"main\")\n assert \"does not contain git data\" in str(err.value)\n" }, { "path": "lib/spack/spack/test/util/compression.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport io\nimport os\nimport pathlib\nimport shutil\nimport tarfile\nfrom itertools import product\n\nimport pytest\n\nimport spack.llnl.url\nfrom spack.llnl.util.filesystem import working_dir\nfrom spack.paths import spack_root\nfrom spack.util import compression\nfrom spack.util.executable import CommandNotFoundError\n\ndatadir = os.path.join(spack_root, \"lib\", \"spack\", \"spack\", \"test\", \"data\", \"compression\")\n\next_archive = {\n ext: f\"Foo.{ext}\" for ext in spack.llnl.url.ALLOWED_ARCHIVE_TYPES if \"TAR\" not in ext\n}\n# Spack does not use Python native handling for tarballs or zip\n# Don't test tarballs or zip in native test\nnative_archive_list = [\n key for key in ext_archive.keys() if \"tar\" not in key and \"zip\" not in key and \"whl\" not in key\n]\n\n\n@pytest.fixture\ndef compr_support_check(monkeypatch):\n monkeypatch.setattr(compression, \"LZMA_SUPPORTED\", False)\n monkeypatch.setattr(compression, \"GZIP_SUPPORTED\", False)\n monkeypatch.setattr(compression, \"BZ2_SUPPORTED\", False)\n\n\n@pytest.fixture\ndef archive_file_and_extension(tmp_path_factory: pytest.TempPathFactory, request):\n \"\"\"Copy example archive to temp directory into an extension-less file for test\"\"\"\n archive_file_stub = os.path.join(datadir, \"Foo\")\n extension, add_extension = request.param\n tmpdir = tmp_path_factory.mktemp(\"compression\")\n tmp_archive_file = os.path.join(\n str(tmpdir), \"Foo\" + ((\".\" + extension) if add_extension else \"\")\n )\n shutil.copy(archive_file_stub + \".\" + extension, tmp_archive_file)\n return (tmp_archive_file, extension)\n\n\n@pytest.mark.parametrize(\n \"archive_file_and_extension\", product(native_archive_list, [True, False]), indirect=True\n)\ndef test_native_unpacking(tmp_path_factory: pytest.TempPathFactory, archive_file_and_extension):\n archive_file, extension = archive_file_and_extension\n util = compression.decompressor_for(archive_file, extension)\n tmpdir = tmp_path_factory.mktemp(\"comp_test\")\n with working_dir(str(tmpdir)):\n assert not os.listdir(os.getcwd())\n util(archive_file)\n files = os.listdir(os.getcwd())\n assert len(files) == 1\n with open(files[0], \"r\", encoding=\"utf-8\") as f:\n contents = f.read()\n assert \"TEST\" in contents\n\n\n@pytest.mark.not_on_windows(\"Only Python unpacking available on Windows\")\n@pytest.mark.parametrize(\n \"archive_file_and_extension\",\n [(ext, True) for ext in ext_archive.keys() if \"whl\" not in ext],\n indirect=True,\n)\ndef test_system_unpacking(\n tmp_path_factory: pytest.TempPathFactory, archive_file_and_extension, compr_support_check\n):\n # actually run test\n archive_file, _ = archive_file_and_extension\n util = compression.decompressor_for(archive_file)\n tmpdir = tmp_path_factory.mktemp(\"system_comp_test\")\n with working_dir(str(tmpdir)):\n assert not os.listdir(os.getcwd())\n util(archive_file)\n files = os.listdir(os.getcwd())\n assert len(files) == 1\n with open(files[0], \"r\", encoding=\"utf-8\") as f:\n contents = f.read()\n assert \"TEST\" in contents\n\n\ndef test_unallowed_extension():\n # use a cxx file as python files included for the test\n # are picked up by the linter and break style checks\n bad_ext_archive = \"Foo.cxx\"\n with pytest.raises(CommandNotFoundError):\n compression.decompressor_for(bad_ext_archive)\n\n\n@pytest.mark.parametrize(\"ext\", [\"gz\", \"bz2\", \"xz\"])\ndef test_file_type_check_does_not_advance_stream(tmp_path: pathlib.Path, ext):\n # Create a tarball compressed with the given format\n path = str(tmp_path / \"compressed_tarball\")\n\n try:\n if ext == \"gz\":\n tar = tarfile.open(path, \"w:gz\")\n elif ext == \"bz2\":\n tar = tarfile.open(path, \"w:bz2\")\n elif ext == \"xz\":\n tar = tarfile.open(path, \"w:xz\")\n else:\n assert False, f\"Unsupported extension: {ext}\"\n except tarfile.CompressionError:\n pytest.skip(f\"Cannot create tar.{ext} files\")\n\n with tar:\n tar.addfile(tarfile.TarInfo(\"test.txt\"), fileobj=io.BytesIO(b\"test\"))\n\n # Classify the file from its magic bytes, and check that the stream is not advanced\n with open(path, \"rb\") as f:\n computed_ext = compression.extension_from_magic_numbers_by_stream(f, decompress=False)\n assert computed_ext == ext\n assert f.tell() == 0\n computed_ext = compression.extension_from_magic_numbers_by_stream(f, decompress=True)\n assert computed_ext == f\"tar.{ext}\"\n assert f.tell() == 0\n" }, { "path": "lib/spack/spack/test/util/editor.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport sys\n\nimport pytest\n\nimport spack.util.editor as ed\nfrom spack.llnl.util.filesystem import set_executable\n\npytestmark = [\n pytest.mark.usefixtures(\"working_env\"),\n pytest.mark.not_on_windows(\"editor not implemented on windows\"),\n]\n\n\n# env vars that control the editor\nEDITOR_VARS = [\"SPACK_EDITOR\", \"VISUAL\", \"EDITOR\"]\n\n\n@pytest.fixture(scope=\"module\", autouse=True)\ndef clean_env_vars():\n \"\"\"Unset all editor env vars before tests.\"\"\"\n for var in EDITOR_VARS:\n if var in os.environ:\n del os.environ[var]\n\n\n@pytest.fixture(autouse=True)\ndef working_editor_test_env(working_env):\n \"\"\"Don't leak environment variables between functions here.\"\"\"\n\n\n# parameterized fixture for editor var names\n@pytest.fixture(params=EDITOR_VARS)\ndef editor_var(request):\n return request.param\n\n\ndef _make_exe(tmp_path_factory: pytest.TempPathFactory, name, contents=None):\n if sys.platform == \"win32\":\n name += \".exe\"\n exe_dir = tmp_path_factory.mktemp(f\"{name}_exe\")\n path = exe_dir / name\n if contents is not None:\n path.write_text(f\"#!/bin/sh\\n{contents}\\n\", encoding=\"utf-8\")\n set_executable(str(path))\n return str(path)\n\n\n@pytest.fixture(scope=\"session\")\ndef good_exe(tmp_path_factory: pytest.TempPathFactory):\n return _make_exe(tmp_path_factory, \"good\", \"exit 0\")\n\n\n@pytest.fixture(scope=\"session\")\ndef bad_exe(tmp_path_factory: pytest.TempPathFactory):\n return _make_exe(tmp_path_factory, \"bad\", \"exit 1\")\n\n\n@pytest.fixture(scope=\"session\")\ndef nosuch_exe(tmp_path_factory: pytest.TempPathFactory):\n return _make_exe(tmp_path_factory, \"nosuch\")\n\n\n@pytest.fixture(scope=\"session\")\ndef vim_exe(tmp_path_factory: pytest.TempPathFactory):\n return _make_exe(tmp_path_factory, \"vim\", \"exit 0\")\n\n\n@pytest.fixture(scope=\"session\")\ndef gvim_exe(tmp_path_factory: pytest.TempPathFactory):\n return _make_exe(tmp_path_factory, \"gvim\", \"exit 0\")\n\n\ndef test_find_exe_from_env_var(good_exe):\n os.environ[\"EDITOR\"] = good_exe\n assert ed._find_exe_from_env_var(\"EDITOR\") == (good_exe, [good_exe])\n\n\ndef test_find_exe_from_env_var_with_args(good_exe):\n os.environ[\"EDITOR\"] = good_exe + \" a b c\"\n assert ed._find_exe_from_env_var(\"EDITOR\") == (good_exe, [good_exe, \"a\", \"b\", \"c\"])\n\n\ndef test_find_exe_from_env_var_bad_path(nosuch_exe):\n os.environ[\"EDITOR\"] = nosuch_exe\n assert ed._find_exe_from_env_var(\"FOO\") == (None, [])\n\n\ndef test_editor_gvim_special_case(gvim_exe):\n os.environ[\"EDITOR\"] = gvim_exe\n\n def assert_exec(exe, args):\n assert exe == gvim_exe\n assert args == [gvim_exe, \"-f\", \"/path/to/file\"]\n return 0\n\n assert ed.editor(\"/path/to/file\", exec_fn=assert_exec)\n\n os.environ[\"EDITOR\"] = gvim_exe + \" -f\"\n assert ed.editor(\"/path/to/file\", exec_fn=assert_exec)\n\n\ndef test_editor_precedence(good_exe, gvim_exe, vim_exe, bad_exe):\n \"\"\"Ensure we prefer editor variables in order of precedence.\"\"\"\n os.environ[\"SPACK_EDITOR\"] = good_exe\n os.environ[\"VISUAL\"] = gvim_exe\n os.environ[\"EDITOR\"] = vim_exe\n correct_exe = good_exe\n\n def assert_callback(exe, args):\n result = ed.executable(exe, args)\n if result == 0:\n assert exe == correct_exe\n return result\n\n ed.editor(exec_fn=assert_callback)\n\n os.environ[\"SPACK_EDITOR\"] = bad_exe\n correct_exe = gvim_exe\n ed.editor(exec_fn=assert_callback)\n\n os.environ[\"VISUAL\"] = bad_exe\n correct_exe = vim_exe\n ed.editor(exec_fn=assert_callback)\n\n\ndef test_find_exe_from_env_var_no_editor():\n if \"FOO\" in os.environ:\n os.environ.unset(\"FOO\")\n assert ed._find_exe_from_env_var(\"FOO\") == (None, [])\n\n\ndef test_editor(editor_var, good_exe):\n os.environ[editor_var] = good_exe\n\n def assert_exec(exe, args):\n assert exe == good_exe\n assert args == [good_exe, \"/path/to/file\"]\n return 0\n\n ed.editor(\"/path/to/file\", exec_fn=assert_exec)\n\n\ndef test_editor_visual_bad(good_exe, bad_exe):\n os.environ[\"VISUAL\"] = bad_exe\n os.environ[\"EDITOR\"] = good_exe\n\n def assert_exec(exe, args):\n if exe == bad_exe:\n raise OSError()\n\n assert exe == good_exe\n assert args == [good_exe, \"/path/to/file\"]\n return 0\n\n ed.editor(\"/path/to/file\", exec_fn=assert_exec)\n\n\ndef test_editor_no_visual(good_exe):\n os.environ[\"EDITOR\"] = good_exe\n\n def assert_exec(exe, args):\n assert exe == good_exe\n assert args == [good_exe, \"/path/to/file\"]\n return 0\n\n ed.editor(\"/path/to/file\", exec_fn=assert_exec)\n\n\ndef test_editor_no_visual_with_args(good_exe):\n # editor has extra args in the var (e.g., emacs -nw)\n os.environ[\"EDITOR\"] = good_exe + \" -nw --foo\"\n\n def assert_exec(exe, args):\n assert exe == good_exe\n assert args == [good_exe, \"-nw\", \"--foo\", \"/path/to/file\"]\n return 0\n\n ed.editor(\"/path/to/file\", exec_fn=assert_exec)\n\n\ndef test_editor_both_bad(nosuch_exe, vim_exe):\n os.environ[\"VISUAL\"] = nosuch_exe\n os.environ[\"EDITOR\"] = nosuch_exe\n\n os.environ[\"PATH\"] = \"%s%s%s\" % (os.path.dirname(vim_exe), os.pathsep, os.environ[\"PATH\"])\n\n def assert_exec(exe, args):\n assert exe == vim_exe\n assert args == [vim_exe, \"/path/to/file\"]\n return 0\n\n ed.editor(\"/path/to/file\", exec_fn=assert_exec)\n\n\ndef test_no_editor():\n os.environ[\"PATH\"] = \"\"\n\n def assert_exec(exe, args):\n assert False\n\n with pytest.raises(OSError, match=r\"No text editor found.*\"):\n ed.editor(\"/path/to/file\", exec_fn=assert_exec)\n\n def assert_exec(exe, args):\n return False\n\n with pytest.raises(OSError, match=r\"No text editor found.*\"):\n ed.editor(\"/path/to/file\", exec_fn=assert_exec)\n\n\ndef test_exec_fn_executable(editor_var, good_exe, bad_exe):\n \"\"\"Make sure editor() works with ``ed.executable`` as well as execv\"\"\"\n os.environ[editor_var] = good_exe\n assert ed.editor(exec_fn=ed.executable)\n\n os.environ[editor_var] = bad_exe\n with pytest.raises(OSError, match=r\"No text editor found.*\"):\n ed.editor(exec_fn=ed.executable)\n" }, { "path": "lib/spack/spack/test/util/elf.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\nimport io\nimport pathlib\n\nimport pytest\n\nimport spack.llnl.util.filesystem as fs\nimport spack.platforms\nimport spack.util.elf as elf\nimport spack.util.executable\nfrom spack.hooks.drop_redundant_rpaths import drop_redundant_rpaths\n\n\n# note that our elf parser is platform independent... but I guess creating an elf file\n# is slightly more difficult with system tools on non-linux.\ndef skip_unless_linux(f):\n return pytest.mark.skipif(\n str(spack.platforms.real_host()) != \"linux\",\n reason=\"implementation currently requires linux\",\n )(f)\n\n\n@pytest.mark.requires_executables(\"gcc\")\n@skip_unless_linux\n@pytest.mark.parametrize(\n \"linker_flag,is_runpath\",\n [(\"-Wl,--disable-new-dtags\", False), (\"-Wl,--enable-new-dtags\", True)],\n)\ndef test_elf_parsing_shared_linking(linker_flag, is_runpath, tmp_path: pathlib.Path):\n gcc = spack.util.executable.which(\"gcc\", required=True)\n\n with fs.working_dir(str(tmp_path)):\n # Create a library to link to so we can force a dynamic section in an ELF file\n with open(\"foo.c\", \"w\", encoding=\"utf-8\") as f:\n f.write(\"int foo(){return 0;}\")\n with open(\"bar.c\", \"w\", encoding=\"utf-8\") as f:\n f.write(\"int foo(); int _start(){return foo();}\")\n\n # Create library and executable linking to it.\n gcc(\"-shared\", \"-o\", \"libfoo.so\", \"-Wl,-soname,libfoo.so.1\", \"-nostdlib\", \"foo.c\")\n gcc(\n \"-o\",\n \"bar\",\n linker_flag,\n \"-Wl,-rpath,/first\",\n \"-Wl,-rpath,/second\",\n \"-Wl,--no-as-needed\",\n \"-nostdlib\",\n \"libfoo.so\",\n \"bar.c\",\n \"-o\",\n \"bar\",\n )\n\n with open(\"libfoo.so\", \"rb\") as f:\n foo_parsed = elf.parse_elf(f, interpreter=True, dynamic_section=True)\n\n assert not foo_parsed.has_pt_interp\n assert foo_parsed.has_pt_dynamic\n assert not foo_parsed.has_rpath\n assert not foo_parsed.has_needed\n assert foo_parsed.has_soname\n assert foo_parsed.dt_soname_str == b\"libfoo.so.1\"\n\n with open(\"bar\", \"rb\") as f:\n bar_parsed = elf.parse_elf(f, interpreter=True, dynamic_section=True)\n\n assert bar_parsed.has_pt_interp\n assert bar_parsed.has_pt_dynamic\n assert bar_parsed.has_rpath\n assert bar_parsed.has_needed\n assert not bar_parsed.has_soname\n assert bar_parsed.dt_rpath_str == b\"/first:/second\"\n assert bar_parsed.dt_needed_strs == [b\"libfoo.so.1\"]\n\n\ndef test_broken_elf():\n # No elf magic\n with pytest.raises(elf.ElfParsingError, match=\"Not an ELF file\"):\n elf.parse_elf(io.BytesIO(b\"x\"))\n\n # Incomplete ELF header\n with pytest.raises(elf.ElfParsingError, match=\"Not an ELF file\"):\n elf.parse_elf(io.BytesIO(b\"\\x7fELF\"))\n\n # Invalid class\n with pytest.raises(elf.ElfParsingError, match=\"Invalid class\"):\n elf.parse_elf(io.BytesIO(b\"\\x7fELF\\x09\\x01\" + b\"\\x00\" * 10))\n\n # Invalid data type\n with pytest.raises(elf.ElfParsingError, match=\"Invalid data type\"):\n elf.parse_elf(io.BytesIO(b\"\\x7fELF\\x01\\x09\" + b\"\\x00\" * 10))\n\n # 64-bit needs at least 64 bytes of header; this is only 56 bytes\n with pytest.raises(elf.ElfParsingError, match=\"ELF header malformed\"):\n elf.parse_elf(io.BytesIO(b\"\\x7fELF\\x02\\x01\" + b\"\\x00\" * 50))\n\n # 32-bit needs at least 52 bytes of header; this is only 46 bytes\n with pytest.raises(elf.ElfParsingError, match=\"ELF header malformed\"):\n elf.parse_elf(io.BytesIO(b\"\\x7fELF\\x01\\x01\" + b\"\\x00\" * 40))\n\n # Not a ET_DYN/ET_EXEC on a 32-bit LE ELF\n with pytest.raises(elf.ElfParsingError, match=\"Not an ET_DYN or ET_EXEC\"):\n elf.parse_elf(io.BytesIO(b\"\\x7fELF\\x01\\x01\" + (b\"\\x00\" * 10) + b\"\\x09\" + (b\"\\x00\" * 35)))\n\n\ndef test_parser_doesnt_deal_with_nonzero_offset():\n # Currently we don't have logic to parse ELF files at nonzero offsets in a file\n # This could be useful when e.g. modifying an ELF file inside a tarball or so,\n # but currently we cannot.\n elf_at_offset_one = io.BytesIO(b\"\\x00\\x7fELF\\x01\\x01\" + b\"\\x00\" * 10)\n elf_at_offset_one.read(1)\n with pytest.raises(elf.ElfParsingError, match=\"Cannot parse at a nonzero offset\"):\n elf.parse_elf(elf_at_offset_one)\n\n\ndef test_only_header():\n # When passing only_header=True parsing a file that is literally just a header\n # without any sections/segments should not error.\n\n # 32 bit\n elf_32 = elf.parse_elf(io.BytesIO(b\"\\x7fELF\\x01\\x01\" + b\"\\x00\" * 46), only_header=True)\n assert not elf_32.is_64_bit\n assert elf_32.is_little_endian\n\n # 64 bit\n elf_64 = elf.parse_elf(io.BytesIO(b\"\\x7fELF\\x02\\x01\" + b\"\\x00\" * 58), only_header=True)\n assert elf_64.is_64_bit\n assert elf_64.is_little_endian\n\n\n@pytest.mark.requires_executables(\"gcc\")\n@skip_unless_linux\ndef test_elf_get_and_replace_rpaths_and_pt_interp(binary_with_rpaths):\n long_paths = [\"/very/long/prefix-a/x\", \"/very/long/prefix-b/y\"]\n executable = str(\n binary_with_rpaths(rpaths=long_paths, dynamic_linker=\"/very/long/prefix-b/lib/ld.so\")\n )\n\n # Before\n assert elf.get_rpaths(executable) == long_paths\n\n replacements = {\n b\"/very/long/prefix-a\": b\"/short-a\",\n b\"/very/long/prefix-b\": b\"/short-b\",\n b\"/very/long\": b\"/dont\",\n }\n\n # Replace once: should modify the file.\n assert elf.substitute_rpath_and_pt_interp_in_place_or_raise(executable, replacements)\n\n # Replace twice: nothing to be done.\n assert not elf.substitute_rpath_and_pt_interp_in_place_or_raise(executable, replacements)\n\n # Verify the rpaths were modified correctly\n assert elf.get_rpaths(executable) == [\"/short-a/x\", \"/short-b/y\"]\n assert elf.get_interpreter(executable) == \"/short-b/lib/ld.so\"\n\n # Going back to long rpaths should fail, since we've added trailing \\0\n # bytes, and replacement can't assume it can write back in repeated null\n # bytes -- it may correspond to zero-length strings for example.\n with pytest.raises(elf.ElfCStringUpdatesFailed) as info:\n elf.substitute_rpath_and_pt_interp_in_place_or_raise(\n executable, {b\"/short-a\": b\"/very/long/prefix-a\", b\"/short-b\": b\"/very/long/prefix-b\"}\n )\n\n assert info.value.rpath is not None\n assert info.value.pt_interp is not None\n assert info.value.rpath.old_value == b\"/short-a/x:/short-b/y\"\n assert info.value.rpath.new_value == b\"/very/long/prefix-a/x:/very/long/prefix-b/y\"\n assert info.value.pt_interp.old_value == b\"/short-b/lib/ld.so\"\n assert info.value.pt_interp.new_value == b\"/very/long/prefix-b/lib/ld.so\"\n\n\n@pytest.mark.requires_executables(\"gcc\")\n@skip_unless_linux\ndef test_drop_redundant_rpath(tmp_path: pathlib.Path, binary_with_rpaths):\n \"\"\"Test the post install hook that drops redundant rpath entries\"\"\"\n\n # Use existing and non-existing dirs in tmp_path\n non_existing_dirs = [str(tmp_path / \"a\"), str(tmp_path / \"b\")]\n existing_dirs = [str(tmp_path / \"c\"), str(tmp_path / \"d\")]\n all_dirs = non_existing_dirs + existing_dirs\n\n (tmp_path / \"c\").mkdir()\n (tmp_path / \"d\").mkdir()\n\n # Create a binary with rpaths to both existing and non-existing dirs\n binary = binary_with_rpaths(rpaths=all_dirs)\n\n # Verify that the binary has all the rpaths\n # sometimes compilers add extra rpaths, so we test for a subset\n all_rpaths = elf.get_rpaths(binary)\n assert all_rpaths and set(all_dirs).issubset(all_rpaths)\n\n # Test whether the right rpaths are dropped\n drop_redundant_rpaths(binary)\n new_rpaths = elf.get_rpaths(binary)\n assert new_rpaths and set(existing_dirs).issubset(new_rpaths)\n assert set(non_existing_dirs).isdisjoint(new_rpaths)\n\n\ndef test_elf_invalid_e_shnum(tmp_path: pathlib.Path):\n # from llvm/test/Object/Inputs/invalid-e_shnum.elf\n path = tmp_path / \"invalid-e_shnum.elf\"\n with open(path, \"wb\") as file:\n file.write(\n b\"\\x7fELF\\x02\\x010000000000\\x03\\x00>\\x0000000000000000000000\"\n b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x000000000000@\\x000000\"\n )\n with open(path, \"rb\") as file, pytest.raises(elf.ElfParsingError):\n elf.parse_elf(file)\n" }, { "path": "lib/spack/spack/test/util/environment.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Test Spack's environment utility functions.\"\"\"\n\nimport os\nimport pathlib\nimport sys\n\nimport pytest\n\nimport spack.util.environment as envutil\n\n\n@pytest.fixture()\ndef prepare_environment_for_tests():\n if \"TEST_ENV_VAR\" in os.environ:\n del os.environ[\"TEST_ENV_VAR\"]\n yield\n del os.environ[\"TEST_ENV_VAR\"]\n\n\ndef test_is_system_path():\n sys_path = \"C:\\\\Users\" if sys.platform == \"win32\" else \"/usr/bin\"\n assert envutil.is_system_path(sys_path)\n assert not envutil.is_system_path(\"/nonsense_path/bin\")\n assert not envutil.is_system_path(\"\")\n assert not envutil.is_system_path(None)\n\n\nif sys.platform == \"win32\":\n test_paths = [\n \"C:\\\\Users\",\n \"C:\\\\\",\n \"C:\\\\ProgramData\",\n \"C:\\\\nonsense_path\",\n \"C:\\\\Program Files\",\n \"C:\\\\nonsense_path\\\\extra\\\\bin\",\n ]\nelse:\n test_paths = [\n \"/usr/bin\",\n \"/nonsense_path/lib\",\n \"/usr/local/lib\",\n \"/bin\",\n \"/nonsense_path/extra/bin\",\n \"/usr/lib64\",\n ]\n\n\ndef test_filter_system_paths():\n nonsense_prefix = \"C:\\\\nonsense_path\" if sys.platform == \"win32\" else \"/nonsense_path\"\n expected = [p for p in test_paths if p.startswith(nonsense_prefix)]\n filtered = envutil.filter_system_paths(test_paths)\n assert expected == filtered\n\n\ndef deprioritize_system_paths():\n expected = [p for p in test_paths if p.startswith(\"/nonsense_path\")]\n expected.extend([p for p in test_paths if not p.startswith(\"/nonsense_path\")])\n filtered = envutil.deprioritize_system_paths(test_paths)\n assert expected == filtered\n\n\ndef test_prune_duplicate_paths():\n test_paths = [\"/a/b\", \"/a/c\", \"/a/b\", \"/a/a\", \"/a/c\", \"/a/a/..\"]\n expected = [\"/a/b\", \"/a/c\", \"/a/a\", \"/a/a/..\"]\n assert expected == envutil.prune_duplicate_paths(test_paths)\n\n\ndef test_get_path(prepare_environment_for_tests):\n os.environ[\"TEST_ENV_VAR\"] = os.pathsep.join([\"/a\", \"/b\", \"/c/d\"])\n expected = [\"/a\", \"/b\", \"/c/d\"]\n assert envutil.get_path(\"TEST_ENV_VAR\") == expected\n\n\ndef test_env_flag(prepare_environment_for_tests):\n assert not envutil.env_flag(\"TEST_NO_ENV_VAR\")\n os.environ[\"TEST_ENV_VAR\"] = \"1\"\n assert envutil.env_flag(\"TEST_ENV_VAR\")\n os.environ[\"TEST_ENV_VAR\"] = \"TRUE\"\n assert envutil.env_flag(\"TEST_ENV_VAR\")\n os.environ[\"TEST_ENV_VAR\"] = \"True\"\n assert envutil.env_flag(\"TEST_ENV_VAR\")\n os.environ[\"TEST_ENV_VAR\"] = \"TRue\"\n assert envutil.env_flag(\"TEST_ENV_VAR\")\n os.environ[\"TEST_ENV_VAR\"] = \"true\"\n assert envutil.env_flag(\"TEST_ENV_VAR\")\n os.environ[\"TEST_ENV_VAR\"] = \"27\"\n assert not envutil.env_flag(\"TEST_ENV_VAR\")\n os.environ[\"TEST_ENV_VAR\"] = \"-2.3\"\n assert not envutil.env_flag(\"TEST_ENV_VAR\")\n os.environ[\"TEST_ENV_VAR\"] = \"0\"\n assert not envutil.env_flag(\"TEST_ENV_VAR\")\n os.environ[\"TEST_ENV_VAR\"] = \"False\"\n assert not envutil.env_flag(\"TEST_ENV_VAR\")\n os.environ[\"TEST_ENV_VAR\"] = \"false\"\n assert not envutil.env_flag(\"TEST_ENV_VAR\")\n os.environ[\"TEST_ENV_VAR\"] = \"garbage\"\n assert not envutil.env_flag(\"TEST_ENV_VAR\")\n\n\ndef test_path_set(prepare_environment_for_tests):\n envutil.path_set(\"TEST_ENV_VAR\", [\"/a\", \"/a/b\", \"/a/a\"])\n assert os.environ[\"TEST_ENV_VAR\"] == \"/a\" + os.pathsep + \"/a/b\" + os.pathsep + \"/a/a\"\n\n\ndef test_path_put_first(prepare_environment_for_tests):\n envutil.path_set(\"TEST_ENV_VAR\", test_paths)\n expected = [\"/usr/bin\", \"/new_nonsense_path/a/b\"]\n expected.extend([p for p in test_paths if p != \"/usr/bin\"])\n envutil.path_put_first(\"TEST_ENV_VAR\", expected)\n assert envutil.get_path(\"TEST_ENV_VAR\") == expected\n\n\n@pytest.mark.parametrize(\"shell\", [\"pwsh\", \"bat\"] if sys.platform == \"win32\" else [\"bash\"])\ndef test_dump_environment(prepare_environment_for_tests, shell_as, shell, tmp_path: pathlib.Path):\n test_paths = \"/a:/b/x:/b/c\"\n os.environ[\"TEST_ENV_VAR\"] = test_paths\n dumpfile_path = str(tmp_path / \"envdump.txt\")\n envutil.dump_environment(dumpfile_path)\n with open(dumpfile_path, \"r\", encoding=\"utf-8\") as dumpfile:\n if shell == \"pwsh\":\n assert \"$Env:TEST_ENV_VAR={}\\n\".format(test_paths) in list(dumpfile)\n elif shell == \"bat\":\n assert 'set \"TEST_ENV_VAR={}\"\\n'.format(test_paths) in list(dumpfile)\n else:\n assert \"TEST_ENV_VAR={0}; export TEST_ENV_VAR\\n\".format(test_paths) in list(dumpfile)\n\n\ndef test_reverse_environment_modifications(working_env):\n prepend_val = os.sep + os.path.join(\"new\", \"path\", \"prepended\")\n append_val = os.sep + os.path.join(\"new\", \"path\", \"appended\")\n\n start_env = {\n \"PREPEND_PATH\": prepend_val + os.pathsep + os.path.join(\"path\", \"to\", \"prepend\", \"to\"),\n \"APPEND_PATH\": os.path.sep\n + os.path.join(\"path\", \"to\", \"append\", \"to\" + os.pathsep + append_val),\n \"UNSET\": \"var_to_unset\",\n \"APPEND_FLAGS\": \"flags to append to\",\n }\n\n to_reverse = envutil.EnvironmentModifications()\n\n to_reverse.prepend_path(\"PREPEND_PATH\", prepend_val)\n to_reverse.append_path(\"APPEND_PATH\", append_val)\n to_reverse.set_path(\"SET_PATH\", [\"/one/set/path\", \"/two/set/path\"])\n to_reverse.set(\"SET\", \"a var\")\n to_reverse.unset(\"UNSET\")\n to_reverse.append_flags(\"APPEND_FLAGS\", \"more_flags\")\n\n reversal = to_reverse.reversed()\n\n os.environ.clear()\n os.environ.update(start_env)\n\n to_reverse.apply_modifications()\n reversal.apply_modifications()\n\n start_env.pop(\"UNSET\")\n assert os.environ == start_env\n\n\ndef test_shell_modifications_are_properly_escaped():\n \"\"\"Test that variable values are properly escaped so that they can safely be eval'd.\"\"\"\n changes = envutil.EnvironmentModifications()\n changes.set(\"VAR\", \"$PATH\")\n changes.append_path(\"VAR\", \"$ANOTHER_PATH\")\n changes.set(\"RM_RF\", \"$(rm -rf /)\")\n\n script = changes.shell_modifications(shell=\"sh\")\n assert f\"export VAR='$PATH{os.pathsep}$ANOTHER_PATH'\" in script\n assert \"export RM_RF='$(rm -rf /)'\" in script\n" }, { "path": "lib/spack/spack/test/util/executable.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport pathlib\nimport sys\nfrom typing import List\n\nimport pytest\n\nimport spack.llnl.util.filesystem as fs\nimport spack.main\nimport spack.util.executable as ex\nfrom spack.hooks.sbang import filter_shebangs_in_directory\n\n\ndef test_read_unicode(tmp_path: pathlib.Path, working_env):\n with fs.working_dir(str(tmp_path)):\n script_name = \"print_unicode.py\"\n script_args: List[str] = []\n # read the unicode back in and see whether things work\n if sys.platform == \"win32\":\n script = ex.Executable(\"%s\" % (sys.executable))\n script_args.append(script_name)\n else:\n script = ex.Executable(\"./%s\" % script_name)\n\n os.environ[\"LD_LIBRARY_PATH\"] = spack.main.spack_ld_library_path\n # make a script that prints some unicode\n with open(script_name, \"w\", encoding=\"utf-8\") as f:\n f.write(\n \"\"\"#!{0}\nprint(u'\\\\xc3')\n\"\"\".format(sys.executable)\n )\n\n # make it executable\n fs.set_executable(script_name)\n filter_shebangs_in_directory(\".\", [script_name])\n\n assert \"\\xc3\" == script(*script_args, output=str).strip()\n\n\ndef test_which_relative_path_with_slash(tmp_path: pathlib.Path, working_env):\n (tmp_path / \"exe\").touch()\n path = str(tmp_path / \"exe\")\n\n os.environ[\"PATH\"] = \"\"\n\n with fs.working_dir(str(tmp_path)):\n no_exe = ex.which(\".{0}exe\".format(os.path.sep))\n assert no_exe is None\n if sys.platform == \"win32\":\n # These checks are for 'executable' files, Windows\n # determines this by file extension.\n path += \".exe\"\n (tmp_path / \"exe.exe\").touch()\n else:\n fs.set_executable(path)\n\n exe = ex.which(\".{0}exe\".format(os.path.sep), required=True)\n assert exe.path == path\n\n\ndef test_which_with_slash_ignores_path(tmp_path: pathlib.Path, working_env):\n (tmp_path / \"exe\").touch()\n (tmp_path / \"bin\").mkdir()\n (tmp_path / \"bin\" / \"exe\").touch()\n\n path = str(tmp_path / \"exe\")\n wrong_path = str(tmp_path / \"bin\" / \"exe\")\n os.environ[\"PATH\"] = str(tmp_path / \"bin\")\n\n with fs.working_dir(str(tmp_path)):\n if sys.platform == \"win32\":\n # For Windows, need to create files with .exe after any assert is none tests\n (tmp_path / \"exe.exe\").touch()\n (tmp_path / \"bin\" / \"exe.exe\").touch()\n path = path + \".exe\"\n wrong_path = wrong_path + \".exe\"\n else:\n fs.set_executable(path)\n fs.set_executable(wrong_path)\n\n exe = ex.which(\".{0}exe\".format(os.path.sep), required=True)\n assert exe.path == path\n\n\ndef test_which(tmp_path: pathlib.Path, monkeypatch):\n monkeypatch.setenv(\"PATH\", str(tmp_path))\n assert ex.which(\"spack-test-exe\") is None\n\n with pytest.raises(ex.CommandNotFoundError):\n ex.which(\"spack-test-exe\", required=True)\n\n path = str(tmp_path / \"spack-test-exe\")\n\n with fs.working_dir(str(tmp_path)):\n if sys.platform == \"win32\":\n # For Windows, need to create files with .exe after any assert is none tests\n (tmp_path / \"spack-test-exe.exe\").touch()\n path += \".exe\"\n else:\n fs.touch(\"spack-test-exe\")\n fs.set_executable(\"spack-test-exe\")\n\n exe = ex.which(\"spack-test-exe\")\n assert exe is not None\n assert exe.path == path\n\n\n@pytest.fixture\ndef make_script_exe(tmp_path: pathlib.Path):\n if sys.platform == \"win32\":\n pytest.skip(\"Can't test #!/bin/sh scripts on Windows.\")\n\n def make_script(name, contents):\n script = tmp_path / f\"{name}.sh\"\n with script.open(\"w\", encoding=\"utf-8\") as f:\n f.write(\"#!/bin/sh\\n\")\n f.write(contents)\n f.write(\"\\n\")\n fs.set_executable(str(script))\n\n return ex.Executable(str(script))\n\n return make_script\n\n\ndef test_exe_fail(make_script_exe):\n fail = make_script_exe(\"fail\", \"exit 107\")\n with pytest.raises(ex.ProcessError):\n fail()\n assert fail.returncode == 107\n\n\ndef test_exe_success(make_script_exe):\n succeed = make_script_exe(\"fail\", \"exit 0\")\n succeed()\n assert succeed.returncode == 0\n\n\ndef test_exe_timeout(make_script_exe):\n timeout = make_script_exe(\"timeout\", \"sleep 100\")\n with pytest.raises(ex.ProcessError):\n timeout(timeout=1)\n assert timeout.returncode == 1\n\n\ndef test_exe_not_exist(tmp_path: pathlib.Path):\n fail = ex.Executable(str(tmp_path / \"foo\")) # doesn't exist\n with pytest.raises(ex.ProcessError):\n fail()\n assert fail.returncode == 1\n\n\ndef test_construct_from_pathlib(mock_executable):\n \"\"\"Tests that we can construct an executable from a pathlib.Path object\"\"\"\n expected = \"Hello world!\"\n path = mock_executable(\"hello\", output=f\"echo {expected}\\n\")\n hello = ex.Executable(path)\n assert expected in hello(output=str)\n\n\ndef test_exe_disallows_str_split_as_input(mock_executable):\n path = mock_executable(\"hello\", output=\"echo hi\\n\")\n hello = ex.Executable(path)\n with pytest.raises(ValueError):\n hello(input=str.split)\n\n\ndef test_exe_disallows_callable_as_output(mock_executable):\n path = mock_executable(\"hello\", output=\"echo hi\\n\")\n hello = ex.Executable(path)\n with pytest.raises(ValueError):\n hello(output=lambda line: line)\n" }, { "path": "lib/spack/spack/test/util/file_cache.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Test Spack's FileCache.\"\"\"\n\nimport os\nimport pathlib\n\nimport pytest\n\nimport spack.llnl.util.filesystem as fs\nfrom spack.util.file_cache import CacheError, FileCache\n\n\n@pytest.fixture()\ndef file_cache(tmp_path: pathlib.Path):\n \"\"\"Returns a properly initialized FileCache instance\"\"\"\n return FileCache(str(tmp_path))\n\n\ndef test_write_and_read_cache_file(file_cache):\n \"\"\"Test writing then reading a cached file.\"\"\"\n with file_cache.write_transaction(\"test.yaml\") as (old, new):\n assert old is None\n assert new is not None\n new.write(\"foobar\\n\")\n\n with file_cache.read_transaction(\"test.yaml\") as stream:\n text = stream.read()\n assert text == \"foobar\\n\"\n\n\ndef test_read_before_init(file_cache):\n with file_cache.read_transaction(\"test.yaml\") as stream:\n assert stream is None\n\n\n@pytest.mark.not_on_windows(\"Locks not supported on Windows\")\ndef test_failed_write_and_read_cache_file(file_cache):\n \"\"\"Test failing to write then attempting to read a cached file.\"\"\"\n with pytest.raises(RuntimeError, match=r\"^foobar$\"):\n with file_cache.write_transaction(\"test.yaml\") as (old, new):\n assert old is None\n assert new is not None\n raise RuntimeError(\"foobar\")\n\n # Cache dir should have exactly one (lock) file\n assert os.listdir(file_cache.root) == [\".lock\"]\n\n # File does not exist\n assert not os.path.exists(file_cache.cache_path(\"test.yaml\"))\n\n\ndef test_write_and_remove_cache_file(file_cache):\n \"\"\"Test two write transactions on a cached file. Then try to remove an\n entry from it.\n \"\"\"\n\n with file_cache.write_transaction(\"test.yaml\") as (old, new):\n assert old is None\n assert new is not None\n new.write(\"foobar\\n\")\n\n with file_cache.write_transaction(\"test.yaml\") as (old, new):\n assert old is not None\n text = old.read()\n assert text == \"foobar\\n\"\n assert new is not None\n new.write(\"barbaz\\n\")\n\n with file_cache.read_transaction(\"test.yaml\") as stream:\n text = stream.read()\n assert text == \"barbaz\\n\"\n\n file_cache.remove(\"test.yaml\")\n\n # After removal the file should not exist\n assert not os.path.exists(file_cache.cache_path(\"test.yaml\"))\n\n # Whether the lock file exists is more of an implementation detail, on Linux they\n # continue to exist, on Windows they don't.\n # assert os.path.exists(file_cache._lock_path('test.yaml'))\n\n\n@pytest.mark.not_on_windows(\"Not supported on Windows (yet)\")\n@pytest.mark.skipif(fs.getuid() == 0, reason=\"user is root\")\ndef test_bad_cache_permissions(file_cache, request):\n \"\"\"Test that transactions raise CacheError on permission problems.\"\"\"\n relpath = fs.join_path(\"test-dir\", \"read-only-file.txt\")\n cachefile = file_cache.cache_path(relpath)\n fs.touchp(cachefile)\n\n # A directory where a file is expected raises CacheError on read\n with pytest.raises(CacheError, match=\"not a file\"):\n with file_cache.read_transaction(os.path.dirname(relpath)) as _:\n pass\n\n # A directory where a file is expected raises CacheError on write\n with pytest.raises(CacheError, match=\"not a file\"):\n with file_cache.write_transaction(os.path.dirname(relpath)) as _:\n pass\n\n # A non-readable file raises CacheError on read\n os.chmod(cachefile, 0o200)\n request.addfinalizer(lambda c=cachefile: os.chmod(c, 0o600))\n with pytest.raises(CacheError, match=\"Cannot access cache file\"):\n with file_cache.read_transaction(relpath) as _:\n pass\n\n # A read-only parent directory raises CacheError on write\n relpath2 = fs.join_path(\"test-dir\", \"another-file.txxt\")\n parent = str(file_cache.cache_path(relpath2).parent)\n os.chmod(parent, 0o400)\n request.addfinalizer(lambda p=parent: os.chmod(p, 0o700))\n with pytest.raises(CacheError):\n with file_cache.write_transaction(relpath2) as _:\n pass\n\n\n@pytest.mark.regression(\"31475\")\ndef test_delete_is_idempotent(file_cache):\n \"\"\"Deleting a non-existent key should be idempotent, to simplify life when\n running delete with multiple processes\"\"\"\n file_cache.remove(\"test.yaml\")\n" }, { "path": "lib/spack/spack/test/util/git.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport pathlib\nfrom typing import Optional\n\nimport pytest\n\nimport spack.util.executable as exe\nimport spack.util.git\nfrom spack.llnl.util.filesystem import working_dir\n\n\ndef test_git_not_found(monkeypatch):\n def _mock_find_git() -> Optional[str]:\n return None\n\n monkeypatch.setattr(spack.util.git, \"_find_git\", _mock_find_git)\n\n git = spack.util.git.git(required=False)\n assert git is None\n\n with pytest.raises(exe.CommandNotFoundError):\n spack.util.git.git(required=True)\n\n\ndef test_modified_files(mock_git_package_changes):\n repo, filename, commits = mock_git_package_changes\n\n with working_dir(repo.packages_path):\n files = spack.util.git.get_modified_files(from_ref=\"HEAD~1\", to_ref=\"HEAD\")\n assert len(files) == 1\n assert files[0] == filename\n\n\ndef test_init_git_repo(git, mock_git_version_info, tmp_path: pathlib.Path):\n \"\"\"Test that init_git_repo creates a new repo with remote but doesn't checkout.\"\"\"\n repo, _, _ = mock_git_version_info\n\n with working_dir(str(tmp_path / \"test_git_init_repo\"), create=True):\n spack.util.git.init_git_repo(repo)\n\n # Verify repo was initialized but no commits checked out yet\n assert \"No commits yet\" in git(\"status\", output=str)\n\n\ndef test_pull_checkout_commit_any_remote(git, tmp_path: pathlib.Path, mock_git_version_info):\n repo, _, commits = mock_git_version_info\n destination = str(tmp_path / \"test_git_checkout_commit\")\n\n with working_dir(destination, create=True):\n spack.util.git.init_git_repo(repo)\n spack.util.git.pull_checkout_commit(commits[0])\n\n assert commits[0] in git(\"rev-parse\", \"HEAD\", output=str)\n\n\ndef test_pull_checkout_commit_specific_remote(git, tmp_path: pathlib.Path, mock_git_version_info):\n \"\"\"Test fetching a specific commit from a specific remote.\"\"\"\n repo, _, commits = mock_git_version_info\n destination = str(tmp_path / \"test_git_checkout_commit_from_remote\")\n\n with working_dir(destination, create=True):\n spack.util.git.init_git_repo(repo)\n spack.util.git.pull_checkout_commit(commits[0], remote=\"origin\", depth=1)\n\n assert commits[0] in git(\"rev-parse\", \"HEAD\", output=str)\n\n\ndef test_pull_checkout_tag(git, tmp_path: pathlib.Path, mock_git_version_info):\n repo, _, _ = mock_git_version_info\n destination = str(tmp_path / \"test_git_checkout_tag\")\n\n with working_dir(destination, create=True):\n spack.util.git.init_git_repo(repo)\n spack.util.git.pull_checkout_tag(\"v1.1\")\n\n assert \"v1.1\" in git(\"describe\", \"--exact-match\", \"--tags\", output=str)\n\n\ndef test_pull_checkout_branch(git, tmp_path: pathlib.Path, mock_git_version_info):\n repo, _, _ = mock_git_version_info\n destination = str(tmp_path / \"test_git_checkout_branch\")\n\n with working_dir(destination, create=True):\n spack.util.git.init_git_repo(repo)\n spack.util.git.pull_checkout_branch(\"1.x\")\n\n assert \"1.x\" in git(\"rev-parse\", \"--abbrev-ref\", \"HEAD\", output=str)\n\n with open(\"file.txt\", \"w\", encoding=\"utf-8\") as f:\n f.write(\"hi harmen\")\n\n with pytest.raises(exe.ProcessError):\n spack.util.git.pull_checkout_branch(\"main\")\n\n\n@pytest.mark.parametrize(\n \"input,answer\",\n (\n [\"git version 1.7.1\", (1, 7, 1)],\n [\"git version 2.34.1.windows.2\", (2, 34, 1)],\n [\"git version 2.50.1 (Apple Git-155)\", (2, 50, 1)],\n [\"git version 1.2.3.4.150.abcd10\", (1, 2, 3, 4, 150)],\n ),\n)\ndef test_extract_git_version(mock_util_executable, input, answer):\n _, _, registered_responses = mock_util_executable\n registered_responses[\"--version\"] = input\n git = spack.util.git.GitExecutable()\n assert git.version == answer\n\n\ndef test_mock_git_exe(mock_util_executable):\n log, should_fail, _ = mock_util_executable\n should_fail.append(\"clone\")\n git = spack.util.git.GitExecutable()\n with pytest.raises(exe.ProcessError):\n git(\"clone\")\n assert git.returncode == 1\n git(\"status\")\n assert git.returncode == 0\n assert \"clone\" in \"\\n\".join(log)\n assert \"status\" in \"\\n\".join(log)\n\n\n@pytest.mark.parametrize(\"git_version\", (\"1.5.0\", \"1.3.0\"))\ndef test_git_exe_conditional_option(mock_util_executable, git_version):\n log, _, registered_responses = mock_util_executable\n min_version = (1, 4, 1)\n registered_responses[\"git --version\"] = git_version\n git = spack.util.git.GitExecutable(\"git\")\n mock_opt = spack.util.git.VersionConditionalOption(\"--maybe\", min_version=min_version)\n args = mock_opt(git.version)\n if git.version >= min_version:\n assert \"--maybe\" in args\n else:\n assert not args\n\n\n@pytest.mark.parametrize(\n \"git_version,ommitted_opts\",\n ((\"2.18.0\", [\"--filter=blob:none\"]), (\"1.8.0\", [\"--filter=blob:none\", \"--depth\"])),\n)\ndef test_git_init_fetch_ommissions(mock_util_executable, git_version, ommitted_opts):\n log, _, registered_responses = mock_util_executable\n registered_responses[\"git --version\"] = git_version\n git = spack.util.git.GitExecutable(\"git\")\n url = \"https://foo.git\"\n ref = \"v1.2.3\"\n spack.util.git.git_init_fetch(url, ref, git_exe=git)\n for opt in ommitted_opts:\n assert all(opt not in call for call in log)\n" }, { "path": "lib/spack/spack/test/util/ld_so_conf.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport pathlib\nimport sys\n\nimport pytest\n\nimport spack.util.ld_so_conf as ld_so_conf\n\n\n@pytest.mark.skipif(sys.platform == \"win32\", reason=\"Unix path\")\ndef test_ld_so_conf_parsing(tmp_path: pathlib.Path):\n cwd = os.getcwd()\n (tmp_path / \"subdir\").mkdir()\n\n # Entrypoint config file\n with open(str(tmp_path / \"main.conf\"), \"wb\") as f:\n f.write(b\" \\n\")\n f.write(b\"include subdir/*.conf\\n\")\n f.write(b\"include non-existent/file\\n\")\n f.write(b\"include #nope\\n\")\n f.write(b\"include \\n\")\n f.write(b\"include\\t\\n\")\n f.write(b\"include\\n\")\n f.write(b\"/main.conf/lib # and a comment\\n\")\n f.write(b\"relative/path\\n\\n\")\n f.write(b\"#/skip/me\\n\")\n\n # Should be parsed: subdir/first.conf\n with open(str(tmp_path / \"subdir\" / \"first.conf\"), \"wb\") as f:\n f.write(b\"/first.conf/lib\")\n\n # Should be parsed: subdir/second.conf\n with open(str(tmp_path / \"subdir\" / \"second.conf\"), \"wb\") as f:\n f.write(b\"/second.conf/lib\")\n\n # Not matching subdir/*.conf\n with open(str(tmp_path / \"subdir\" / \"third\"), \"wb\") as f:\n f.write(b\"/third/lib\")\n\n paths = ld_so_conf.parse_ld_so_conf(str(tmp_path / \"main.conf\"))\n\n assert len(paths) == 3\n assert \"/main.conf/lib\" in paths\n assert \"/first.conf/lib\" in paths\n assert \"/second.conf/lib\" in paths\n\n # Make sure globbing didn't change the working dir\n assert os.getcwd() == cwd\n\n\ndef test_host_dynamic_linker_search_paths():\n assert {\"/usr/lib\", \"/usr/lib64\", \"/lib\", \"/lib64\"}.issubset(\n ld_so_conf.host_dynamic_linker_search_paths()\n )\n" }, { "path": "lib/spack/spack/test/util/log_parser.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport io\nimport pathlib\n\nfrom spack.llnl.util.tty.color import color_when\nfrom spack.util.ctest_log_parser import CTestLogParser\nfrom spack.util.log_parse import make_log_context\n\n\ndef test_log_parser(tmp_path: pathlib.Path):\n log_file = tmp_path / \"log.txt\"\n\n with log_file.open(\"w\") as f:\n f.write(\n \"\"\"#!/bin/sh\\n\nchecking build system type... x86_64-apple-darwin16.6.0\nchecking host system type... x86_64-apple-darwin16.6.0\nerror: weird_error.c:145: something weird happened E\nchecking for gcc... /Users/gamblin2/src/spack/lib/spack/env/clang/clang\nchecking whether the C compiler works... yes\n/var/tmp/build/foo.py:60: warning: some weird warning W\nchecking for C compiler default output file name... a.out\nld: fatal: linker thing happened E\nchecking for suffix of executables...\nconfigure: error: in /path/to/some/file: E\nconfigure: error: cannot run C compiled programs. E\n\"\"\"\n )\n\n parser = CTestLogParser()\n errors, warnings = parser.parse(str(log_file))\n\n assert len(errors) == 4\n assert all(e.text.endswith(\"E\") for e in errors)\n\n assert len(warnings) == 1\n assert all(w.text.endswith(\"W\") for w in warnings)\n\n\ndef test_log_parser_stream():\n \"\"\"parse() accepts a file-like object.\"\"\"\n log = io.StringIO(\n \"error: weird_error.c:145: something weird happened E\\n\"\n \"checking for gcc... irrelevant line\\n\"\n \"/var/tmp/build/foo.py:60: warning: some weird warning W\\n\"\n )\n parser = CTestLogParser()\n errors, warnings = parser.parse(log)\n\n assert len(errors) == 1\n assert errors[0].text.endswith(\"E\")\n assert len(warnings) == 1\n assert warnings[0].text.endswith(\"W\")\n\n\ndef test_log_parser_preserves_leading_whitespace():\n \"\"\"Leading whitespace (e.g. compiler caret underlines) must not be stripped.\"\"\"\n log = io.StringIO(\n \"/path/to/file.c:10: error: use of undeclared identifier 'x'\\n\"\n \" int y = x + 1;\\n\"\n \" ^\\n\"\n )\n parser = CTestLogParser()\n errors, _ = parser.parse(log, context=6)\n\n assert len(errors) == 1\n assert errors[0].post_context[0] == \" int y = x + 1;\"\n assert errors[0].post_context[1] == \" ^\"\n\n\ndef test_make_log_context_merges_overlapping_events(tmp_path: pathlib.Path):\n \"\"\"Overlapping or adjacent context windows should produce a single merged block.\"\"\"\n\n # Two errors close together: lines 5 and 10 with context=3 means windows overlap.\n lines = [f\"line {i}\\n\" for i in range(1, 21)]\n lines[4] = \"error: first problem\\n\" # line 5\n lines[9] = \"error: second problem\\n\" # line 10\n\n log_file = tmp_path / \"log.txt\"\n log_file.write_text(\"\".join(lines))\n\n parser = CTestLogParser()\n errors, warnings = parser.parse(str(log_file), context=3)\n\n log_events = sorted([*errors, *warnings], key=lambda e: e.line_no)\n output = make_log_context(log_events)\n\n # Should be exactly one header for the merged block, not two.\n assert output.count(\"-- lines\") == 1\n\n # The header should cover the full merged range.\n assert \"-- lines 2 to 13 --\" in output\n\n\ndef test_make_log_context_warning_in_error_context_keeps_yellow(tmp_path: pathlib.Path):\n \"\"\"A warning line inside an error's context window must be highlighted yellow, not red.\"\"\"\n # Line 5 = error, line 8 = warning, context=3 so error window covers lines 2-11\n # meaning the warning at line 8 falls inside the error's context.\n lines = [f\"line {i}\\n\" for i in range(1, 16)]\n lines[4] = \"error: something broke\\n\" # line 5\n lines[7] = \"/tmp/foo.c:1: warning: something fishy\\n\" # line 8\n\n log_file = tmp_path / \"log.txt\"\n log_file.write_text(\"\".join(lines))\n\n parser = CTestLogParser()\n errors, warnings = parser.parse(str(log_file), context=3)\n\n assert len(errors) == len(warnings) == 1\n\n log_events = sorted([*errors, *warnings], key=lambda e: e.line_no)\n\n with color_when(\"always\"):\n output = make_log_context(log_events)\n\n # The error line should be red (ANSI 91), the warning yellow (ANSI 93).\n assert \"\\x1b[0;91m> \" in output and \"something broke\" in output\n assert \"\\x1b[0;93m> \" in output and \"something fishy\" in output\n\n\ndef test_log_parser_non_utf8_bytes(tmp_path: pathlib.Path):\n \"\"\"parse() does not raise UnicodeDecodeError on non-UTF-8 log files.\"\"\"\n log_file = tmp_path / \"log.bin\"\n log_file.write_bytes(b\"checking things...\\nerror: \\x80\\xff something broke\\ndone\\n\")\n parser = CTestLogParser()\n errors, _ = parser.parse(str(log_file))\n assert len(errors) == 1\n" }, { "path": "lib/spack/spack/test/util/module_cmd.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\n\nimport pytest\n\nimport spack.util.module_cmd\n\n\n@pytest.mark.not_on_windows(\"Module files are not supported on Windows\")\ndef test_load_module_success(monkeypatch, working_env):\n \"\"\"Test that load_module properly handles successful module loads.\n\n This is a very lightweight test that only confirms that successful\n loads are not flagged as failed.\"\"\"\n\n # Mock the module function to simulate a successful module load\n def mock_module(*args, **kwargs):\n if args[0] == \"show\":\n return \"\"\n elif args[0] == \"load\":\n # Simulate successful module load by adding to LOADEDMODULES\n current_modules = os.environ.get(\"LOADEDMODULES\", \"\")\n if current_modules:\n os.environ[\"LOADEDMODULES\"] = f\"{current_modules}:{args[1]}\"\n else:\n os.environ[\"LOADEDMODULES\"] = args[1]\n\n monkeypatch.setattr(spack.util.module_cmd, \"module\", mock_module)\n\n # This should succeed\n spack.util.module_cmd.load_module(\"test_module\")\n spack.util.module_cmd.load_module(\"test_module_2\")\n\n # Confirm LOADEDMODULES was modified\n assert \"test_module:test_module_2\" in os.environ[\"LOADEDMODULES\"]\n\n\n@pytest.mark.not_on_windows(\"Module files are not supported on Windows\")\ndef test_load_module_failure(monkeypatch, working_env):\n \"\"\"Test that load_module raises an exception when a module load fails.\"\"\"\n\n # Mock the module function to simulate a failed module load\n def mock_module(*args, **kwargs):\n if args[0] == \"show\":\n return \"\"\n elif args[0] == \"load\":\n # Simulate module load failure by not changing LOADEDMODULES\n pass\n\n monkeypatch.setattr(spack.util.module_cmd, \"module\", mock_module)\n\n # This should fail with ModuleLoadError\n with pytest.raises(spack.util.module_cmd.ModuleLoadError):\n spack.util.module_cmd.load_module(\"non_existent_module\")\n" }, { "path": "lib/spack/spack/test/util/package_hash.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport ast\nimport os\n\nimport pytest\n\nimport spack.concretize\nimport spack.directives_meta\nimport spack.paths\nimport spack.repo\nimport spack.util.package_hash as ph\nfrom spack.spec import Spec\nfrom spack.util.unparse import unparse\n\ndatadir = os.path.join(spack.paths.test_path, \"data\", \"unparse\")\n\n\ndef compare_sans_name(eq, spec1, spec2):\n content1 = ph.canonical_source(spec1)\n content1 = content1.replace(spack.repo.PATH.get_pkg_class(spec1.name).__name__, \"TestPackage\")\n content2 = ph.canonical_source(spec2)\n content2 = content2.replace(spack.repo.PATH.get_pkg_class(spec2.name).__name__, \"TestPackage\")\n if eq:\n assert content1 == content2\n else:\n assert content1 != content2\n\n\ndef compare_hash_sans_name(eq, spec1, spec2):\n content1 = ph.canonical_source(spec1)\n pkg_cls1 = spack.repo.PATH.get_pkg_class(spec1.name)\n content1 = content1.replace(pkg_cls1.__name__, \"TestPackage\")\n hash1 = pkg_cls1(spec1).content_hash(content=content1)\n\n content2 = ph.canonical_source(spec2)\n pkg_cls2 = spack.repo.PATH.get_pkg_class(spec2.name)\n content2 = content2.replace(pkg_cls2.__name__, \"TestPackage\")\n hash2 = pkg_cls2(spec2).content_hash(content=content2)\n\n assert (hash1 == hash2) == eq\n\n\ndef test_hash(mock_packages, config):\n ph.package_hash(Spec(\"hash-test1@=1.2\"))\n\n\ndef test_different_variants(mock_packages, config):\n spec1 = Spec(\"hash-test1@=1.2 +variantx\")\n spec2 = Spec(\"hash-test1@=1.2 +varianty\")\n assert ph.package_hash(spec1) == ph.package_hash(spec2)\n\n\ndef test_all_same_but_name(mock_packages, config):\n spec1 = Spec(\"hash-test1@=1.2\")\n spec2 = Spec(\"hash-test2@=1.2\")\n compare_sans_name(True, spec1, spec2)\n\n spec1 = Spec(\"hash-test1@=1.2 +varianty\")\n spec2 = Spec(\"hash-test2@=1.2 +varianty\")\n compare_sans_name(True, spec1, spec2)\n\n\ndef test_all_same_but_archive_hash(mock_packages, config):\n \"\"\"\n Archive hash is not intended to be reflected in Package hash.\n \"\"\"\n spec1 = Spec(\"hash-test1@=1.3\")\n spec2 = Spec(\"hash-test2@=1.3\")\n compare_sans_name(True, spec1, spec2)\n\n\ndef test_all_same_but_patch_contents(mock_packages, config):\n spec1 = Spec(\"hash-test1@=1.1\")\n spec2 = Spec(\"hash-test2@=1.1\")\n compare_sans_name(True, spec1, spec2)\n\n\ndef test_all_same_but_patches_to_apply(mock_packages, config):\n spec1 = Spec(\"hash-test1@=1.4\")\n spec2 = Spec(\"hash-test2@=1.4\")\n compare_sans_name(True, spec1, spec2)\n\n\ndef test_all_same_but_install(mock_packages, config):\n spec1 = Spec(\"hash-test1@=1.5\")\n spec2 = Spec(\"hash-test2@=1.5\")\n compare_sans_name(False, spec1, spec2)\n\n\ndef test_content_hash_all_same_but_patch_contents(mock_packages, config):\n spec1 = spack.concretize.concretize_one(\"hash-test1@1.1\")\n spec2 = spack.concretize.concretize_one(\"hash-test2@1.1\")\n compare_hash_sans_name(False, spec1, spec2)\n\n\ndef test_content_hash_not_concretized(mock_packages, config):\n \"\"\"Check that Package.content_hash() works on abstract specs.\"\"\"\n # these are different due to the package hash\n spec1 = Spec(\"hash-test1@=1.1\")\n spec2 = Spec(\"hash-test2@=1.3\")\n compare_hash_sans_name(False, spec1, spec2)\n\n # at v1.1 these are actually the same package when @when's are removed\n # and the name isn't considered\n spec1 = Spec(\"hash-test1@=1.1\")\n spec2 = Spec(\"hash-test2@=1.1\")\n compare_hash_sans_name(True, spec1, spec2)\n\n # these end up being different b/c we can't eliminate much of the package.py\n # without a version.\n spec1 = Spec(\"hash-test1\")\n spec2 = Spec(\"hash-test2\")\n compare_hash_sans_name(False, spec1, spec2)\n\n\ndef test_content_hash_different_variants(mock_packages, config):\n spec1 = spack.concretize.concretize_one(\"hash-test1@1.2 +variantx\")\n spec2 = spack.concretize.concretize_one(\"hash-test2@1.2 ~variantx\")\n compare_hash_sans_name(True, spec1, spec2)\n\n\ndef test_content_hash_cannot_get_details_from_ast(mock_packages, config):\n \"\"\"Packages hash-test1 and hash-test3 would be considered the same\n except that hash-test3 conditionally executes a phase based on\n a \"when\" directive that Spack cannot evaluate by examining the\n AST. This test ensures that Spack can compute a content hash\n for hash-test3. If Spack cannot determine when a phase applies,\n it adds it by default, so the test also ensures that the hashes\n differ where Spack includes a phase on account of AST-examination\n failure.\n \"\"\"\n spec3 = spack.concretize.concretize_one(\"hash-test1@1.7\")\n spec4 = spack.concretize.concretize_one(\"hash-test3@1.7\")\n compare_hash_sans_name(False, spec3, spec4)\n\n\ndef test_content_hash_all_same_but_archive_hash(mock_packages, config):\n spec1 = spack.concretize.concretize_one(\"hash-test1@1.3\")\n spec2 = spack.concretize.concretize_one(\"hash-test2@1.3\")\n compare_hash_sans_name(False, spec1, spec2)\n\n\ndef test_content_hash_parse_dynamic_function_call(mock_packages, config):\n spec = spack.concretize.concretize_one(\"hash-test4\")\n spec.package.content_hash()\n\n\nmany_strings = '''\\\n\"\"\"ONE\"\"\"\n\"\"\"TWO\"\"\"\n\nvar = \"THREE\" # make sure this is not removed\n\n\"FOUR\"\n\nclass ManyDocstrings:\n \"\"\"FIVE\"\"\"\n \"\"\"SIX\"\"\"\n\n x = \"SEVEN\"\n\n def method1():\n \"\"\"EIGHT\"\"\"\n\n print(\"NINE\")\n\n \"TEN\"\n for i in range(10):\n print(i)\n\n def method2():\n \"\"\"ELEVEN\"\"\"\n return \"TWELVE\"\n\ndef empty_func():\n \"\"\"THIRTEEN\"\"\"\n'''\n\nmany_strings_no_docstrings = \"\"\"\\\nvar = 'THREE'\n\nclass ManyDocstrings:\n x = 'SEVEN'\n\n def method1():\n print('NINE')\n for i in range(10):\n print(i)\n\n def method2():\n return 'TWELVE'\n\ndef empty_func():\n pass\n\"\"\"\n\n\ndef test_remove_docstrings():\n tree = ast.parse(many_strings)\n tree = ph.RemoveDocstrings().visit(tree)\n\n unparsed = unparse(tree, py_ver_consistent=True)\n assert unparsed == many_strings_no_docstrings\n\n\nmany_directives = \"\"\"\\\n\nclass HasManyDirectives:\n{directives}\n\n def foo():\n # just a method to get in the way\n pass\n\n{directives}\n\"\"\".format(\n directives=\"\\n\".join(\" %s()\" % name for name in spack.directives_meta.directive_names)\n)\n\n\ndef test_remove_all_directives():\n \"\"\"Ensure all directives are removed from packages before hashing.\"\"\"\n for name in spack.directives_meta.directive_names:\n assert name in many_directives\n\n tree = ast.parse(many_directives)\n spec = Spec(\"has-many-directives\")\n tree = ph.RemoveDirectives(spec).visit(tree)\n unparsed = unparse(tree, py_ver_consistent=True)\n\n for name in spack.directives_meta.directive_names:\n assert name not in unparsed\n\n\nmany_attributes = \"\"\"\\\nclass HasManyMetadataAttributes:\n homepage = \"https://example.com\"\n url = \"https://example.com/foo.tar.gz\"\n git = \"https://example.com/foo/bar.git\"\n\n maintainers(\"alice\", \"bob\")\n tags = [\"foo\", \"bar\", \"baz\"]\n\n depends_on(\"foo\")\n conflicts(\"foo\")\n\"\"\"\n\n\nmany_attributes_canonical = \"\"\"\\\nclass HasManyMetadataAttributes:\n pass\n\"\"\"\n\n\ndef test_remove_spack_attributes():\n tree = ast.parse(many_attributes)\n spec = Spec(\"has-many-metadata-attributes\")\n tree = ph.RemoveDirectives(spec).visit(tree)\n unparsed = unparse(tree, py_ver_consistent=True)\n\n assert unparsed == many_attributes_canonical\n\n\ncomplex_package_logic = \"\"\"\\\nclass ComplexPackageLogic:\n for variant in [\"+foo\", \"+bar\", \"+baz\"]:\n conflicts(\"quux\" + variant)\n\n for variant in [\"+foo\", \"+bar\", \"+baz\"]:\n # logic in the loop prevents our dumb analyzer from having it removed. This\n # is uncommon so we don't (yet?) implement logic to detect that spec is unused.\n print(\"oops can't remove this.\")\n conflicts(\"quux\" + variant)\n\n # Hard to make a while loop that makes sense, so ignore the infinite loop here.\n # Likely nobody uses while instead of for, but we test it just in case.\n while x <= 10:\n depends_on(\"garply@%d.0\" % x)\n\n # all of these should go away, as they only contain directives\n with when(\"@10.0\"):\n depends_on(\"foo\")\n with when(\"+bar\"):\n depends_on(\"bar\")\n with when(\"+baz\"):\n depends_on(\"baz\")\n\n # this whole statement should disappear\n if sys.platform == \"linux\":\n conflicts(\"baz@9.0\")\n\n # the else block here should disappear\n if sys.platform == \"linux\":\n print(\"foo\")\n else:\n conflicts(\"foo@9.0\")\n\n # both blocks of this statement should disappear\n if sys.platform == \"darwin\":\n conflicts(\"baz@10.0\")\n else:\n conflicts(\"bar@10.0\")\n\n # This one is complicated as the body goes away but the else block doesn't.\n # Again, this could be optimized, but we're just testing removal logic here.\n if sys.platform() == \"darwin\":\n conflicts(\"baz@10.0\")\n else:\n print(\"oops can't remove this.\")\n conflicts(\"bar@10.0\")\n\"\"\"\n\n\ncomplex_package_logic_filtered = \"\"\"\\\nclass ComplexPackageLogic:\n for variant in ['+foo', '+bar', '+baz']:\n print(\"oops can't remove this.\")\n if sys.platform == 'linux':\n print('foo')\n if sys.platform() == 'darwin':\n pass\n else:\n print(\"oops can't remove this.\")\n\"\"\"\n\n\ndef test_remove_complex_package_logic_filtered():\n tree = ast.parse(complex_package_logic)\n spec = Spec(\"has-many-metadata-attributes\")\n tree = ph.RemoveDirectives(spec).visit(tree)\n unparsed = unparse(tree, py_ver_consistent=True)\n\n assert unparsed == complex_package_logic_filtered\n\n\n@pytest.mark.parametrize(\n \"package_spec,expected_hash\",\n [\n (\"amdfftw\", \"tivb752zddjgvfkogfs7cnnvp5olj6co\"),\n (\"grads\", \"lomrsppasfxegyamz4r33zgwiqkveftv\"),\n (\"llvm\", \"paicamlvy5jkgxw4xnacaxahrixe3f3i\"),\n # has @when(\"@4.1.0\") and raw unicode literals\n (\"mfem\", \"slf5qyyyhuj66mo5lpuhkrs35akh2zck\"),\n (\"mfem@4.0.0\", \"slf5qyyyhuj66mo5lpuhkrs35akh2zck\"),\n (\"mfem@4.1.0\", \"6tjbezoh2aquz6gmvoz7jf6j6lib65m2\"),\n # has @when(\"@1.5.0:\")\n (\"py-torch\", \"m3ucsddqr7hjevtgx4cad34nrtqgyjfg\"),\n (\"py-torch@1.0\", \"m3ucsddqr7hjevtgx4cad34nrtqgyjfg\"),\n (\"py-torch@1.6\", \"insaxs6bq34rvyhajdbyr4wddqeqb2t3\"),\n # has a print with multiple arguments\n (\"legion\", \"bq2etsik5l6pbryxmbhfhzynci56ruy4\"),\n # has nested `with when()` blocks and loops\n (\"trilinos\", \"ojbtbu3p6gpa42sbilblo2ioanvhouxu\"),\n ],\n)\ndef test_package_hash_consistency(package_spec, expected_hash):\n \"\"\"Ensure that that package hash is consistent python version to version.\n\n We assume these tests run across all supported Python versions in CI, and we ensure\n consistency with recorded hashes for some well known inputs.\n\n If this fails, then something about the way the python AST works has likely changed.\n If Spack is running in a new python version, we might need to modify the unparser to\n handle it. If not, then something has become inconsistent about the way we unparse\n Python code across versions.\n\n \"\"\"\n spec = Spec(package_spec)\n filename = os.path.join(datadir, \"%s.txt\" % spec.name)\n with open(filename, \"rb\") as f:\n source = f.read()\n h = ph.package_hash(spec, source=source)\n assert expected_hash == h\n\n\nmany_multimethods = \"\"\"\\\nclass Pkg:\n def foo(self):\n print(\"ONE\")\n\n @when(\"@1.0\")\n def foo(self):\n print(\"TWO\")\n\n @when(\"@2.0\")\n @when(sys.platform == \"darwin\")\n def foo(self):\n print(\"THREE\")\n\n @when(\"@3.0\")\n def foo(self):\n print(\"FOUR\")\n\n # this one should always stay\n @run_after(\"install\")\n def some_function(self):\n print(\"FIVE\")\n\"\"\"\n\n\nmore_dynamic_multimethods = \"\"\"\\\nclass Pkg:\n @when(sys.platform == \"darwin\")\n def foo(self):\n print(\"ONE\")\n\n @when(\"@1.0\")\n def foo(self):\n print(\"TWO\")\n\n # this one isn't dynamic, but an int fails the Spec parse,\n # so it's kept because it has to be evaluated at runtime.\n @when(\"@2.0\")\n @when(1)\n def foo(self):\n print(\"THREE\")\n\n @when(\"@3.0\")\n def foo(self):\n print(\"FOUR\")\n\n # this one should always stay\n @run_after(\"install\")\n def some_function(self):\n print(\"FIVE\")\n\"\"\"\n\n\n@pytest.mark.parametrize(\n \"spec_str,source,expected,not_expected\",\n [\n # all are false but the default\n (\"pkg@4.0\", many_multimethods, [\"ONE\", \"FIVE\"], [\"TWO\", \"THREE\", \"FOUR\"]),\n # we know first @when overrides default and others are false\n (\"pkg@1.0\", many_multimethods, [\"TWO\", \"FIVE\"], [\"ONE\", \"THREE\", \"FOUR\"]),\n # we know last @when overrides default and others are false\n (\"pkg@3.0\", many_multimethods, [\"FOUR\", \"FIVE\"], [\"ONE\", \"TWO\", \"THREE\"]),\n # we don't know if default or THREE will win, include both\n (\"pkg@2.0\", many_multimethods, [\"ONE\", \"THREE\", \"FIVE\"], [\"TWO\", \"FOUR\"]),\n # we know the first one is the only one that can win.\n (\"pkg@4.0\", more_dynamic_multimethods, [\"ONE\", \"FIVE\"], [\"TWO\", \"THREE\", \"FOUR\"]),\n # now we have to include ONE and TWO because ONE may win dynamically.\n (\"pkg@1.0\", more_dynamic_multimethods, [\"ONE\", \"TWO\", \"FIVE\"], [\"THREE\", \"FOUR\"]),\n # we know FOUR is true and TWO and THREE are false, but ONE may\n # still win dynamically.\n (\"pkg@3.0\", more_dynamic_multimethods, [\"ONE\", \"FOUR\", \"FIVE\"], [\"TWO\", \"THREE\"]),\n # TWO and FOUR can't be satisfied, but ONE or THREE could win\n (\"pkg@2.0\", more_dynamic_multimethods, [\"ONE\", \"THREE\", \"FIVE\"], [\"TWO\", \"FOUR\"]),\n ],\n)\ndef test_multimethod_resolution(spec_str, source, expected, not_expected):\n filtered = ph.canonical_source(Spec(spec_str), source=source)\n for item in expected:\n assert item in filtered\n for item in not_expected:\n assert item not in filtered\n" }, { "path": "lib/spack/spack/test/util/path.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport sys\n\nimport pytest\n\nimport spack.config\nimport spack.llnl.util.tty as tty\nimport spack.util.path as sup\n\n#: Some lines with lots of placeholders\npadded_lines = [\n \"==> [2021-06-23-15:59:05.020387] './configure' '--prefix=/Users/gamblin2/padding-log-test/opt/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_pla/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga\", # noqa: E501\n \"/Users/gamblin2/Workspace/spack/lib/spack/env/clang/clang -dynamiclib -install_name /Users/gamblin2/padding-log-test/opt/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_pla/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga/lib/libz.1.dylib -compatibility_version 1 -current_version 1.2.11 -fPIC -O2 -fPIC -DHAVE_HIDDEN -o libz.1.2.11.dylib adler32.lo crc32.lo deflate.lo infback.lo inffast.lo inflate.lo inftrees.lo trees.lo zutil.lo compress.lo uncompr.lo gzclose.lo gzlib.lo gzread.lo gzwrite.lo -lc\", # noqa: E501\n \"rm -f /Users/gamblin2/padding-log-test/opt/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_pla/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga/lib/libz.a\", # noqa: E501\n \"rm -f /Users/gamblin2/padding-log-test/opt/__spack_path_placeholder__/__spack_path_placeholder___/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga/lib/libz.a\", # noqa: E501\n]\n\n\n#: unpadded versions of padded_lines, with [padded-to-X-chars] replacing the padding\nfixed_lines = [\n \"==> [2021-06-23-15:59:05.020387] './configure' '--prefix=/Users/gamblin2/padding-log-test/opt/[padded-to-512-chars]/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga\", # noqa: E501\n \"/Users/gamblin2/Workspace/spack/lib/spack/env/clang/clang -dynamiclib -install_name /Users/gamblin2/padding-log-test/opt/[padded-to-512-chars]/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga/lib/libz.1.dylib -compatibility_version 1 -current_version 1.2.11 -fPIC -O2 -fPIC -DHAVE_HIDDEN -o libz.1.2.11.dylib adler32.lo crc32.lo deflate.lo infback.lo inffast.lo inflate.lo inftrees.lo trees.lo zutil.lo compress.lo uncompr.lo gzclose.lo gzlib.lo gzread.lo gzwrite.lo -lc\", # noqa: E501\n \"rm -f /Users/gamblin2/padding-log-test/opt/[padded-to-512-chars]/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga/lib/libz.a\", # noqa: E501\n \"rm -f /Users/gamblin2/padding-log-test/opt/[padded-to-91-chars]/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga/lib/libz.a\", # noqa: E501\n]\n\n\ndef test_sanitize_filename():\n \"\"\"Test filtering illegal characters out of potential filenames\"\"\"\n sanitized = sup.sanitize_filename(\"\"\"acd/?e:f\"g|h*i.\\0txt\"\"\")\n if sys.platform == \"win32\":\n assert sanitized == \"a_b_cd__e_f_g_h_i._txt\"\n else:\n assert sanitized == \"\"\"acd_?e:f\"g|h*i._txt\"\"\"\n\n\n# This class pertains to path string padding manipulation specifically\n# which is used for binary caching. This functionality is not supported\n# on Windows as of yet.\n@pytest.mark.not_on_windows(\"Padding functionality unsupported on Windows\")\n@pytest.mark.parametrize(\"as_bytes\", [False, True])\nclass TestPathPadding:\n @pytest.fixture(autouse=True)\n def setup(self, as_bytes: bool):\n #: The filter function, either for bytes or str\n self.filter = sup.padding_filter_bytes if as_bytes else sup.padding_filter\n #: A converter of str -> bytes if we're testing the bytes filter\n self.convert = lambda s: s.encode(\"ascii\") if as_bytes else s\n\n @pytest.mark.parametrize(\"padded,fixed\", zip(padded_lines, fixed_lines))\n def test_padding_substitution(self, padded, fixed):\n \"\"\"Ensure that all padded lines are unpadded correctly.\"\"\"\n assert self.convert(fixed) == self.filter(self.convert(padded))\n\n def test_no_substitution(self):\n \"\"\"Ensure that a line not containing one full path placeholder\n is not modified.\"\"\"\n partial = \"--prefix=/Users/gamblin2/padding-log-test/opt/__spack_path_pla/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga'\" # noqa: E501\n p = self.convert(partial)\n assert self.filter(p) is p # Test fast-path identity\n\n def test_short_substitution(self):\n \"\"\"Ensure that a single placeholder path component is replaced\"\"\"\n short = \"--prefix=/Users/gamblin2/padding-log-test/opt/__spack_path_placeholder__/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga'\" # noqa: E501\n short_subst = \"--prefix=/Users/gamblin2/padding-log-test/opt/[padded-to-63-chars]/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga'\" # noqa: E501\n assert self.convert(short_subst) == self.filter(self.convert(short))\n\n def test_partial_substitution(self):\n \"\"\"Ensure that a single placeholder path component is replaced\"\"\"\n short = \"--prefix=/Users/gamblin2/padding-log-test/opt/__spack_path_placeholder__/__spack_p/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga'\" # noqa: E501\n short_subst = \"--prefix=/Users/gamblin2/padding-log-test/opt/[padded-to-73-chars]/darwin-bigsur-skylake/apple-clang-12.0.5/zlib-1.2.11-74mwnxgn6nujehpyyalhwizwojwn5zga'\" # noqa: E501\n assert self.convert(short_subst) == self.filter(self.convert(short))\n\n def test_longest_prefix_re(self):\n \"\"\"Test that longest_prefix_re generates correct regular expressions.\"\"\"\n assert \"(s(?:t(?:r(?:i(?:ng?)?)?)?)?)\" == sup.longest_prefix_re(\"string\", capture=True)\n assert \"(?:s(?:t(?:r(?:i(?:ng?)?)?)?)?)\" == sup.longest_prefix_re(\"string\", capture=False)\n\n\n@pytest.mark.not_on_windows(\"Padding functionality unsupported on Windows\")\ndef test_output_filtering(capfd, install_mockery, mutable_config):\n \"\"\"Test filtering padding out of tty messages.\"\"\"\n long_path = \"/\" + \"/\".join([sup.SPACK_PATH_PADDING_CHARS] * 200)\n padding_string = \"[padded-to-%d-chars]\" % len(long_path)\n\n # test filtering when padding is enabled\n with spack.config.override(\"config:install_tree\", {\"padded_length\": 256}):\n # tty.msg with filtering on the first argument\n with sup.filter_padding():\n tty.msg(\"here is a long path: %s/with/a/suffix\" % long_path)\n out, err = capfd.readouterr()\n assert padding_string in out\n\n # tty.msg with filtering on a laterargument\n with sup.filter_padding():\n tty.msg(\"here is a long path:\", \"%s/with/a/suffix\" % long_path)\n out, err = capfd.readouterr()\n assert padding_string in out\n\n # tty.error with filtering on the first argument\n with sup.filter_padding():\n tty.error(\"here is a long path: %s/with/a/suffix\" % long_path)\n out, err = capfd.readouterr()\n assert padding_string in err\n\n # tty.error with filtering on a later argument\n with sup.filter_padding():\n tty.error(\"here is a long path:\", \"%s/with/a/suffix\" % long_path)\n out, err = capfd.readouterr()\n assert padding_string in err\n\n # test no filtering\n tty.msg(\"here is a long path: %s/with/a/suffix\" % long_path)\n out, err = capfd.readouterr()\n assert padding_string not in out\n\n\n@pytest.mark.not_on_windows(\"Padding functionality unsupported on Windows\")\ndef test_pad_on_path_sep_boundary():\n \"\"\"Ensure that padded paths do not end with path separator.\"\"\"\n pad_length = len(sup.SPACK_PATH_PADDING_CHARS)\n padded_length = 128\n remainder = padded_length % (pad_length + 1)\n path = \"a\" * (remainder - 1)\n result = sup.add_padding(path, padded_length)\n assert 128 == len(result) and not result.endswith(os.path.sep)\n\n\n@pytest.mark.parametrize(\"debug\", [1, 2])\ndef test_path_debug_padded_filter(debug, monkeypatch):\n \"\"\"Ensure padded filter works as expected with different debug levels.\"\"\"\n fmt = \"{0}{1}{2}{1}{3}\"\n prefix = \"[+] {0}home{0}user{0}install\".format(os.sep)\n suffix = \"mypackage\"\n string = fmt.format(prefix, os.sep, os.sep.join([sup.SPACK_PATH_PADDING_CHARS] * 2), suffix)\n expected = (\n fmt.format(prefix, os.sep, \"[padded-to-{0}-chars]\".format(72), suffix)\n if debug <= 1 and sys.platform != \"win32\"\n else string\n )\n\n monkeypatch.setattr(tty, \"_debug\", debug)\n with spack.config.override(\"config:install_tree\", {\"padded_length\": 128}):\n assert expected == sup.debug_padded_filter(string)\n\n\n@pytest.mark.not_on_windows(\"Unix path\")\ndef test_canonicalize_file_unix():\n assert sup.canonicalize_path(\"/home/spack/path/to/file.txt\") == \"/home/spack/path/to/file.txt\"\n assert sup.canonicalize_path(\"file:///home/another/config.yaml\") == \"/home/another/config.yaml\"\n\n\n@pytest.mark.only_windows(\"Windows path\")\ndef test_canonicalize_file_windows():\n assert sup.canonicalize_path(r\"C:\\Files (x86)\\Windows\\10\") == r\"C:\\Files (x86)\\Windows\\10\"\n assert sup.canonicalize_path(r\"E:/spack stage\") == r\"E:\\spack stage\"\n\n\ndef test_canonicalize_file_relative():\n assert sup.canonicalize_path(\"path/to.txt\") == os.path.join(os.getcwd(), \"path\", \"to.txt\")\n" }, { "path": "lib/spack/spack/test/util/prefix.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Tests various features of :py:class:`spack.util.prefix.Prefix`\"\"\"\n\nimport os\n\nfrom spack.util.prefix import Prefix\n\n\ndef test_prefix_attributes():\n \"\"\"Test normal prefix attributes like ``prefix.bin``\"\"\"\n prefix = Prefix(os.sep + \"usr\")\n\n assert prefix.bin == os.sep + os.path.join(\"usr\", \"bin\")\n assert prefix.lib == os.sep + os.path.join(\"usr\", \"lib\")\n assert prefix.include == os.sep + os.path.join(\"usr\", \"include\")\n\n\ndef test_prefix_join():\n \"\"\"Test prefix join ``prefix.join(...)``\"\"\"\n prefix = Prefix(os.sep + \"usr\")\n\n a1 = prefix.join(\"a_{0}\".format(1)).lib64\n a2 = prefix.join(\"a-{0}\".format(1)).lib64\n a3 = prefix.join(\"a.{0}\".format(1)).lib64\n\n assert a1 == os.sep + os.path.join(\"usr\", \"a_1\", \"lib64\")\n assert a2 == os.sep + os.path.join(\"usr\", \"a-1\", \"lib64\")\n assert a3 == os.sep + os.path.join(\"usr\", \"a.1\", \"lib64\")\n\n assert isinstance(a1, Prefix)\n assert isinstance(a2, Prefix)\n assert isinstance(a3, Prefix)\n\n p1 = prefix.bin.join(\"executable.sh\")\n p2 = prefix.share.join(\"pkg-config\").join(\"foo.pc\")\n p3 = prefix.join(\"dashed-directory\").foo\n\n assert p1 == os.sep + os.path.join(\"usr\", \"bin\", \"executable.sh\")\n assert p2 == os.sep + os.path.join(\"usr\", \"share\", \"pkg-config\", \"foo.pc\")\n assert p3 == os.sep + os.path.join(\"usr\", \"dashed-directory\", \"foo\")\n\n assert isinstance(p1, Prefix)\n assert isinstance(p2, Prefix)\n assert isinstance(p3, Prefix)\n\n\ndef test_multilevel_attributes():\n \"\"\"Test attributes of attributes, like ``prefix.share.man``\"\"\"\n prefix = Prefix(os.sep + \"usr\" + os.sep)\n\n assert prefix.share.man == os.sep + os.path.join(\"usr\", \"share\", \"man\")\n assert prefix.man.man8 == os.sep + os.path.join(\"usr\", \"man\", \"man8\")\n assert prefix.foo.bar.baz == os.sep + os.path.join(\"usr\", \"foo\", \"bar\", \"baz\")\n\n share = prefix.share\n\n assert isinstance(share, Prefix)\n assert share.man == os.sep + os.path.join(\"usr\", \"share\", \"man\")\n\n\ndef test_string_like_behavior():\n \"\"\"Test string-like behavior of the prefix object\"\"\"\n prefix = Prefix(\"/usr\")\n\n assert prefix == \"/usr\"\n assert isinstance(prefix, str)\n\n assert prefix + \"/bin\" == \"/usr/bin\"\n assert \"--prefix=%s\" % prefix == \"--prefix=/usr\"\n assert \"--prefix={0}\".format(prefix) == \"--prefix=/usr\"\n\n assert prefix.find(\"u\", 1)\n assert prefix.upper() == \"/USR\"\n assert prefix.lstrip(\"/\") == \"usr\"\n" }, { "path": "lib/spack/spack/test/util/remote_file_cache.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport os.path\nimport pathlib\nimport sys\n\nimport pytest\n\nimport spack.config\nimport spack.llnl.util.tty as tty\nimport spack.util.remote_file_cache as rfc_util\nfrom spack.llnl.util.filesystem import join_path\n\ngithub_url = \"https://github.com/fake/fake/{0}/develop\"\ngitlab_url = \"https://gitlab.fake.io/user/repo/-/blob/config/defaults\"\n\n\n@pytest.mark.parametrize(\n \"path,err\",\n [\n (\"ssh://git@github.com:spack/\", \"Unsupported URL scheme\"),\n (\"bad:///this/is/a/file/url/include.yaml\", \"Invalid URL scheme\"),\n ],\n)\ndef test_rfc_local_path_bad_scheme(path, err):\n with pytest.raises(ValueError, match=err):\n _ = rfc_util.local_path(path, \"\")\n\n\n@pytest.mark.not_on_windows(\"Unix path\")\ndef test_rfc_local_file_unix():\n assert rfc_util.local_path(\"/a/b/c/d/e/config.py\", \"\") == \"/a/b/c/d/e/config.py\"\n assert (\n rfc_util.local_path(\"file:///this/is/a/file/url/include.yaml\", \"\")\n == \"/this/is/a/file/url/include.yaml\"\n )\n\n\n@pytest.mark.only_windows(\"Windows path\")\ndef test_rfc_local_file_windows():\n assert rfc_util.local_path(r\"C:\\Files (x86)\\Windows\\10\", \"\") == r\"C:\\Files (x86)\\Windows\\10\"\n assert rfc_util.local_path(r\"D:/spack stage\", \"\") == r\"D:\\spack stage\"\n\n\ndef test_rfc_local_file_relative():\n path = \"relative/packages.txt\"\n expected = os.path.join(os.getcwd(), \"relative\", \"packages.txt\")\n assert rfc_util.local_path(path, \"\") == expected\n\n\ndef test_rfc_remote_local_path_no_dest():\n path = f\"{gitlab_url}/packages.yaml\"\n with pytest.raises(ValueError, match=\"Requires the destination argument\"):\n _ = rfc_util.local_path(path, \"\")\n\n\npackages_yaml_sha256 = (\n \"8d428c600b215e3b4a207a08236659dfc2c9ae2782c35943a00ee4204a135702\"\n if sys.platform != \"win32\"\n else \"6c094ec3ee1eb5068860cdd97d8da965bf281be29e60ab9afc8f6e4d72d24f21\"\n)\n\n\n@pytest.mark.parametrize(\n \"url,sha256,err,msg\",\n [\n (\n f\"{join_path(github_url.format('tree'), 'config.yaml')}\",\n \"\",\n ValueError,\n \"Requires sha256\",\n ),\n # This is the packages.yaml in lib/spack/spack/test/data/config\n (f\"{gitlab_url}/packages.yaml\", packages_yaml_sha256, None, \"\"),\n (f\"{gitlab_url}/packages.yaml\", \"abcdef\", ValueError, \"does not match\"),\n (f\"{github_url.format('blob')}/README.md\", \"\", OSError, \"No such\"),\n (github_url.format(\"tree\"), \"\", OSError, \"No such\"),\n (\"\", \"\", ValueError, \"argument is required\"),\n ],\n)\ndef test_rfc_remote_local_path(\n tmp_path: pathlib.Path, mutable_empty_config, mock_fetch_url_text, url, sha256, err, msg\n):\n def _has_content(filename):\n # The first element of all configuration files for this test happen to\n # be the basename of the file so this check leverages that feature. If\n # that changes, then this check will need to change accordingly.\n element = f\"{os.path.splitext(os.path.basename(filename))[0]}:\"\n with open(filename, \"r\", encoding=\"utf-8\") as fd:\n for line in fd:\n if element in line:\n return True\n tty.debug(f\"Expected {element} in '{filename}'\")\n return False\n\n dest_dir = join_path(str(tmp_path), \"cache\")\n\n if err is not None:\n with spack.config.override(\"config:url_fetch_method\", \"curl\"):\n with pytest.raises(err, match=msg):\n rfc_util.local_path(url, sha256, dest_dir)\n else:\n with spack.config.override(\"config:url_fetch_method\", \"curl\"):\n path = rfc_util.local_path(url, sha256, dest_dir)\n assert os.path.exists(path)\n # Ensure correct file is \"fetched\"\n assert os.path.basename(path) == os.path.basename(url)\n # Ensure contents of the file contains expected config element\n assert _has_content(path)\n" }, { "path": "lib/spack/spack/test/util/spack_lock_wrapper.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Tests for Spack's wrapper module around spack.llnl.util.lock.\"\"\"\n\nimport os\nimport pathlib\n\nimport pytest\n\nimport spack.error\nimport spack.util.lock as lk\nfrom spack.llnl.util.filesystem import getuid, group_ids\n\n\ndef test_disable_locking(tmp_path: pathlib.Path):\n \"\"\"Ensure that locks do no real locking when disabled.\"\"\"\n lock_path = str(tmp_path / \"lockfile\")\n lock = lk.Lock(lock_path, enable=False)\n\n lock.acquire_read()\n assert not os.path.exists(lock_path)\n\n lock.acquire_write()\n assert not os.path.exists(lock_path)\n\n lock.release_write()\n assert not os.path.exists(lock_path)\n\n lock.release_read()\n assert not os.path.exists(lock_path)\n\n\n# \"Disable\" mock_stage fixture to avoid subdir permissions issues on cleanup.\n@pytest.mark.nomockstage\n@pytest.mark.skipif(getuid() == 0, reason=\"user is root\")\ndef test_lock_checks_user(tmp_path: pathlib.Path):\n \"\"\"Ensure lock checks work with a self-owned, self-group repo.\"\"\"\n uid = getuid()\n if uid not in group_ids():\n pytest.skip(\"user has no group with gid == uid\")\n\n # self-owned, own group\n os.chown(tmp_path, uid, uid)\n\n # safe\n path = str(tmp_path)\n tmp_path.chmod(0o744)\n lk.check_lock_safety(path)\n\n # safe\n tmp_path.chmod(0o774)\n lk.check_lock_safety(path)\n\n # unsafe\n tmp_path.chmod(0o777)\n with pytest.raises(spack.error.SpackError):\n lk.check_lock_safety(path)\n\n # safe\n tmp_path.chmod(0o474)\n lk.check_lock_safety(path)\n\n # safe\n tmp_path.chmod(0o477)\n lk.check_lock_safety(path)\n\n\n# \"Disable\" mock_stage fixture to avoid subdir permissions issues on cleanup.\n@pytest.mark.nomockstage\ndef test_lock_checks_group(tmp_path: pathlib.Path):\n \"\"\"Ensure lock checks work with a self-owned, non-self-group repo.\"\"\"\n uid = getuid()\n gid = next((g for g in group_ids() if g != uid), None)\n if not gid:\n pytest.skip(\"user has no group with gid != uid\")\n return\n\n # self-owned, another group\n os.chown(tmp_path, uid, gid)\n\n # safe\n path = str(tmp_path)\n tmp_path.chmod(0o744)\n lk.check_lock_safety(path)\n\n # unsafe\n tmp_path.chmod(0o774)\n with pytest.raises(spack.error.SpackError):\n lk.check_lock_safety(path)\n\n # unsafe\n tmp_path.chmod(0o777)\n with pytest.raises(spack.error.SpackError):\n lk.check_lock_safety(path)\n\n # safe\n tmp_path.chmod(0o474)\n lk.check_lock_safety(path)\n\n # safe\n tmp_path.chmod(0o477)\n lk.check_lock_safety(path)\n" }, { "path": "lib/spack/spack/test/util/spack_yaml.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport re\n\nimport spack.config\nfrom spack.main import SpackCommand\n\nconfig_cmd = SpackCommand(\"config\")\n\n\ndef get_config_line(pattern, lines):\n \"\"\"Get a configuration line that matches a particular pattern.\"\"\"\n line = next((x for x in lines if re.search(pattern, x)), None)\n assert line is not None, \"no such line!\"\n return line\n\n\ndef check_blame(element, file_name, line=None):\n \"\"\"Check that `config blame config` gets right file/line for an element.\n\n This runs `spack config blame config` and scrapes the output for a\n particular YAML key. It then checks that the requested file/line info\n is also on that line.\n\n Line is optional; if it is ``None`` we just check for the\n ``file_name``, which may just be a name for a special config scope\n like ``_builtin`` or ``command_line``.\n \"\"\"\n output = config_cmd(\"blame\", \"config\")\n\n blame_lines = output.rstrip().split(\"\\n\")\n element_line = get_config_line(element + \":\", blame_lines)\n\n annotation = file_name\n if line is not None:\n annotation += \":%d\" % line\n\n assert file_name in element_line\n\n\ndef test_config_blame(config):\n \"\"\"check blame info for elements in mock configuration.\"\"\"\n config_file = config.get_config_filename(\"site\", \"config\")\n\n check_blame(\"install_tree\", config_file, 2)\n check_blame(\"source_cache\", config_file, 11)\n check_blame(\"misc_cache\", config_file, 12)\n check_blame(\"verify_ssl\", config_file, 13)\n check_blame(\"checksum\", config_file, 14)\n check_blame(\"dirty\", config_file, 15)\n\n\ndef test_config_blame_with_override(config):\n \"\"\"check blame for an element from an override scope\"\"\"\n config_file = config.get_config_filename(\"site\", \"config\")\n\n with spack.config.override(\"config:install_tree\", {\"root\": \"foobar\"}):\n check_blame(\"install_tree\", \"overrides\")\n\n check_blame(\"source_cache\", config_file, 11)\n check_blame(\"misc_cache\", config_file, 12)\n check_blame(\"verify_ssl\", config_file, 13)\n check_blame(\"checksum\", config_file, 14)\n check_blame(\"dirty\", config_file, 15)\n\n\ndef test_config_blame_defaults():\n \"\"\"check blame for an element from an override scope\"\"\"\n files = {}\n\n def get_file_lines(filename):\n if filename not in files:\n with open(filename, \"r\", encoding=\"utf-8\") as f:\n files[filename] = [\"\"] + f.read().split(\"\\n\")\n return files[filename]\n\n config_blame = config_cmd(\"blame\", \"config\")\n for line in config_blame.split(\"\\n\"):\n # currently checking only simple lines with dict keys\n match = re.match(r\"^([^:]+):(\\d+)\\s+([^:]+):\\s+(.*)\", line)\n\n # check that matches are on the lines they say they are\n if match:\n filename, line, key, val = match.groups()\n line = int(line)\n lines = get_file_lines(filename)\n assert key in lines[line]\n\n val = val.strip(\"'\\\"\")\n printed_line = lines[line]\n if val.lower() in (\"true\", \"false\"):\n val = val.lower()\n printed_line = printed_line.lower()\n\n assert val in printed_line, filename\n" }, { "path": "lib/spack/spack/test/util/timer.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport json\nfrom io import StringIO\n\nimport spack.util.timer as timer\n\n\nclass Tick:\n \"\"\"Timer that increments the seconds passed by 1\n every time tick is called.\"\"\"\n\n def __init__(self):\n self.time = 0.0\n\n def tick(self):\n self.time += 1\n return self.time\n\n\ndef test_timer():\n # 0\n t = timer.Timer(now=Tick().tick)\n\n # 1 (restart)\n t.start()\n\n # 2\n t.start(\"wrapped\")\n\n # 3\n t.start(\"first\")\n\n # 4\n t.stop(\"first\")\n assert t.duration(\"first\") == 1.0\n\n # 5\n t.start(\"second\")\n\n # 6\n t.stop(\"second\")\n assert t.duration(\"second\") == 1.0\n\n # 7-8\n with t.measure(\"third\"):\n pass\n assert t.duration(\"third\") == 1.0\n\n # 9\n t.stop(\"wrapped\")\n assert t.duration(\"wrapped\") == 7.0\n\n # tick 10-13\n t.start(\"not-stopped\")\n assert t.duration(\"not-stopped\") == 1.0\n assert t.duration(\"not-stopped\") == 2.0\n assert t.duration(\"not-stopped\") == 3.0\n\n # 14\n assert t.duration() == 13.0\n\n # 15\n t.stop()\n assert t.duration() == 14.0\n\n\ndef test_timer_stop_stops_all():\n # Ensure that timer.stop() effectively stops all timers.\n\n # 0\n t = timer.Timer(now=Tick().tick)\n\n # 1\n t.start(\"first\")\n\n # 2\n t.start(\"second\")\n\n # 3\n t.start(\"third\")\n\n # 4\n t.stop()\n\n assert t.duration(\"first\") == 3.0\n assert t.duration(\"second\") == 2.0\n assert t.duration(\"third\") == 1.0\n assert t.duration() == 4.0\n\n\ndef test_stopping_unstarted_timer_is_no_error():\n t = timer.Timer(now=Tick().tick)\n assert t.duration(\"hello\") == 0.0\n t.stop(\"hello\")\n assert t.duration(\"hello\") == 0.0\n\n\ndef test_timer_write():\n text_buffer = StringIO()\n json_buffer = StringIO()\n\n # 0\n t = timer.Timer(now=Tick().tick)\n\n # 1\n t.start(\"timer\")\n\n # 2\n t.stop(\"timer\")\n\n # 3\n t.stop()\n\n t.write_tty(text_buffer)\n t.write_json(json_buffer)\n\n output = text_buffer.getvalue().splitlines()\n assert \"timer\" in output[0]\n assert \"1.000s\" in output[0]\n assert \"total\" in output[1]\n assert \"3.000s\" in output[1]\n\n deserialized = json.loads(json_buffer.getvalue())\n assert deserialized == {\n \"phases\": [{\"name\": \"timer\", \"path\": \"timer\", \"seconds\": 1.0, \"count\": 1}],\n \"total\": 3.0,\n }\n\n\ndef test_null_timer():\n # Just ensure that the interface of the noop-timer doesn't break at some point\n buffer = StringIO()\n t = timer.NullTimer()\n t.start()\n t.start(\"first\")\n t.stop(\"first\")\n with t.measure(\"second\"):\n pass\n t.stop()\n assert t.duration(\"first\") == 0.0\n assert t.duration() == 0.0\n assert not t.phases\n t.write_json(buffer)\n t.write_tty(buffer)\n assert not buffer.getvalue()\n" }, { "path": "lib/spack/spack/test/util/unparse/__init__.py", "content": "# Copyright (c) 2014-2021, Simon Percivall and Spack Project Developers.\n#\n# SPDX-License-Identifier: Python-2.0\n" }, { "path": "lib/spack/spack/test/util/unparse/unparse.py", "content": "# Copyright (c) 2014-2021, Simon Percivall and Spack Project Developers.\n#\n# SPDX-License-Identifier: Python-2.0\n\nimport ast\nimport os\nimport sys\nimport tokenize\n\nimport pytest\n\nimport spack.util.unparse\n\npytestmark = pytest.mark.not_on_windows(\"Test module unsupported on Windows\")\n\n\ndef read_pyfile(filename):\n \"\"\"Read and return the contents of a Python source file (as a\n string), taking into account the file encoding.\"\"\"\n with open(filename, \"rb\") as pyfile:\n encoding = tokenize.detect_encoding(pyfile.readline)[0]\n with open(filename, \"r\", encoding=encoding) as pyfile:\n source = pyfile.read()\n return source\n\n\ncode_parseable_in_all_parser_modes = \"\"\"\\\n(a + b + c) * (d + e + f)\n\"\"\"\n\nfor_else = \"\"\"\\\ndef f():\n for x in range(10):\n break\n else:\n y = 2\n z = 3\n\"\"\"\n\nwhile_else = \"\"\"\\\ndef g():\n while True:\n break\n else:\n y = 2\n z = 3\n\"\"\"\n\nrelative_import = \"\"\"\\\nfrom . import fred\nfrom .. import barney\nfrom .australia import shrimp as prawns\n\"\"\"\n\nimport_many = \"\"\"\\\nimport fred, barney\n\"\"\"\n\nnonlocal_ex = \"\"\"\\\ndef f():\n x = 1\n def g():\n nonlocal x\n x = 2\n y = 7\n def h():\n nonlocal x, y\n\"\"\"\n\n# also acts as test for 'except ... as ...'\nraise_from = \"\"\"\\\ntry:\n 1 / 0\nexcept ZeroDivisionError as e:\n raise ArithmeticError from e\n\"\"\"\n\nasync_comprehensions_and_generators = \"\"\"\\\nasync def async_function():\n my_set = {i async for i in aiter() if i % 2}\n my_list = [i async for i in aiter() if i % 2]\n my_dict = {i: -i async for i in aiter() if i % 2}\n my_gen = (i ** 2 async for i in agen())\n my_other_gen = (i - 1 async for i in agen() if i % 2)\n\"\"\"\n\nclass_decorator = \"\"\"\\\n@f1(arg)\n@f2\nclass Foo: pass\n\"\"\"\n\nelif1 = \"\"\"\\\nif cond1:\n suite1\nelif cond2:\n suite2\nelse:\n suite3\n\"\"\"\n\nelif2 = \"\"\"\\\nif cond1:\n suite1\nelif cond2:\n suite2\n\"\"\"\n\ntry_except_finally = \"\"\"\\\ntry:\n suite1\nexcept ex1:\n suite2\nexcept ex2:\n suite3\nelse:\n suite4\nfinally:\n suite5\n\"\"\"\n\nwith_simple = \"\"\"\\\nwith f():\n suite1\n\"\"\"\n\nwith_as = \"\"\"\\\nwith f() as x:\n suite1\n\"\"\"\n\nwith_two_items = \"\"\"\\\nwith f() as x, g() as y:\n suite1\n\"\"\"\n\na_repr = \"\"\"\\\n`{}`\n\"\"\"\n\ncomplex_f_string = '''\\\nf\\'\\'\\'-{f\"\"\"*{f\"+{f'.{x}.'}+\"}*\"\"\"}-\\'\\'\\'\n'''\n\nasync_function_def = \"\"\"\\\nasync def f():\n suite1\n\"\"\"\n\nasync_for = \"\"\"\\\nasync def f():\n async for _ in reader:\n suite1\n\"\"\"\n\nasync_with = \"\"\"\\\nasync def f():\n async with g():\n suite1\n\"\"\"\n\nasync_with_as = \"\"\"\\\nasync def f():\n async with g() as x:\n suite1\n\"\"\"\n\n\nmatch_literal = \"\"\"\\\nmatch status:\n case 400:\n return \"Bad request\"\n case 404 | 418:\n return \"Not found\"\n case _:\n return \"Something's wrong with the internet\"\n\"\"\"\n\nmatch_with_noop = \"\"\"\\\nmatch status:\n case 400:\n return \"Bad request\"\n\"\"\"\n\nmatch_literal_and_variable = \"\"\"\\\nmatch point:\n case (0, 0):\n print(\"Origin\")\n case (0, y):\n print(f\"Y={y}\")\n case (x, 0):\n print(f\"X={x}\")\n case (x, y):\n print(f\"X={x}, Y={y}\")\n case _:\n raise ValueError(\"Not a point\")\n\"\"\"\n\n\nmatch_classes = \"\"\"\\\nclass Point:\n x: int\n y: int\n\ndef location(point):\n match point:\n case Point(x=0, y=0):\n print(\"Origin is the point's location.\")\n case Point(x=0, y=y):\n print(f\"Y={y} and the point is on the y-axis.\")\n case Point(x=x, y=0):\n print(f\"X={x} and the point is on the x-axis.\")\n case Point():\n print(\"The point is located somewhere else on the plane.\")\n case _:\n print(\"Not a point\")\n\"\"\"\n\nmatch_nested = \"\"\"\\\nmatch points:\n case []:\n print(\"No points in the list.\")\n case [Point(0, 0)]:\n print(\"The origin is the only point in the list.\")\n case [Point(x, y)]:\n print(f\"A single point {x}, {y} is in the list.\")\n case [Point(0, y1), Point(0, y2)]:\n print(f\"Two points on the Y axis at {y1}, {y2} are in the list.\")\n case _:\n print(\"Something else is found in the list.\")\n\"\"\"\n\n\ndef check_ast_roundtrip(code1, filename=\"internal\", mode=\"exec\"):\n ast1 = compile(str(code1), filename, mode, ast.PyCF_ONLY_AST)\n code2 = spack.util.unparse.unparse(ast1)\n\n ast2 = compile(code2, filename, mode, ast.PyCF_ONLY_AST)\n\n error_msg = \"Failed to roundtrip {} [mode={}]\".format(filename, mode)\n assert ast.dump(ast1) == ast.dump(ast2), error_msg\n\n\ndef test_core_lib_files():\n \"\"\"Roundtrip source files from the Python core libs.\"\"\"\n test_directories = [\n os.path.join(\n getattr(sys, \"real_prefix\", sys.prefix), \"lib\", \"python%s.%s\" % sys.version_info[:2]\n )\n ]\n\n names = []\n for test_dir in test_directories:\n for n in os.listdir(test_dir):\n if n.endswith(\".py\") and not n.startswith(\"bad\"):\n names.append(os.path.join(test_dir, n))\n\n for filename in names:\n source = read_pyfile(filename)\n check_ast_roundtrip(source)\n\n\n@pytest.mark.skipif(sys.version_info[:2] < (3, 6), reason=\"Only for Python 3.6 or greater\")\ndef test_simple_fstring():\n check_ast_roundtrip(\"f'{x}'\")\n\n\n@pytest.mark.skipif(sys.version_info[:2] < (3, 6), reason=\"Only for Python 3.6 or greater\")\ndef test_fstrings():\n # See issue 25180\n check_ast_roundtrip(r\"\"\"f'{f\"{0}\"*3}'\"\"\")\n check_ast_roundtrip(r\"\"\"f'{f\"{y}\"*3}'\"\"\")\n check_ast_roundtrip(\"\"\"f''\"\"\")\n check_ast_roundtrip('''f\"\"\"'end' \"quote\\\\\"\"\"\"''')\n\n\n@pytest.mark.skipif(sys.version_info[:2] < (3, 6), reason=\"Only for Python 3.6 or greater\")\ndef test_fstrings_complicated():\n # See issue 28002\n check_ast_roundtrip(\"\"\"f'''{\"'\"}'''\"\"\")\n check_ast_roundtrip('''f\\'\\'\\'-{f\"\"\"*{f\"+{f'.{x}.'}+\"}*\"\"\"}-\\'\\'\\'''')\n check_ast_roundtrip('''f\\'\\'\\'-{f\"\"\"*{f\"+{f'.{x}.'}+\"}*\"\"\"}-'single quote\\\\'\\'\\'\\'''')\n check_ast_roundtrip(\"f\\\"\\\"\\\"{'''\\n'''}\\\"\\\"\\\"\")\n check_ast_roundtrip(\"f\\\"\\\"\\\"{g('''\\n''')}\\\"\\\"\\\"\")\n check_ast_roundtrip('''f\"a\\\\r\\\\nb\"''')\n check_ast_roundtrip('''f\"\\\\u2028{'x'}\"''')\n\n\ndef test_parser_modes():\n for mode in [\"exec\", \"single\", \"eval\"]:\n check_ast_roundtrip(code_parseable_in_all_parser_modes, mode=mode)\n\n\ndef test_del_statement():\n check_ast_roundtrip(\"del x, y, z\")\n\n\ndef test_shifts():\n check_ast_roundtrip(\"45 << 2\")\n check_ast_roundtrip(\"13 >> 7\")\n\n\ndef test_for_else():\n check_ast_roundtrip(for_else)\n\n\ndef test_while_else():\n check_ast_roundtrip(while_else)\n\n\ndef test_unary_parens():\n check_ast_roundtrip(\"(-1)**7\")\n check_ast_roundtrip(\"(-1.)**8\")\n check_ast_roundtrip(\"(-1j)**6\")\n check_ast_roundtrip(\"not True or False\")\n check_ast_roundtrip(\"True or not False\")\n\n\n@pytest.mark.skipif(sys.version_info >= (3, 6), reason=\"Only works for Python < 3.6\")\ndef test_integer_parens():\n check_ast_roundtrip(\"3 .__abs__()\")\n\n\ndef test_huge_float():\n check_ast_roundtrip(\"1e1000\")\n check_ast_roundtrip(\"-1e1000\")\n check_ast_roundtrip(\"1e1000j\")\n check_ast_roundtrip(\"-1e1000j\")\n\n\ndef test_min_int30():\n check_ast_roundtrip(str(-(2**31)))\n check_ast_roundtrip(str(-(2**63)))\n\n\ndef test_imaginary_literals():\n check_ast_roundtrip(\"7j\")\n check_ast_roundtrip(\"-7j\")\n check_ast_roundtrip(\"0j\")\n check_ast_roundtrip(\"-0j\")\n\n\ndef test_negative_zero():\n check_ast_roundtrip(\"-0\")\n check_ast_roundtrip(\"-(0)\")\n check_ast_roundtrip(\"-0b0\")\n check_ast_roundtrip(\"-(0b0)\")\n check_ast_roundtrip(\"-0o0\")\n check_ast_roundtrip(\"-(0o0)\")\n check_ast_roundtrip(\"-0x0\")\n check_ast_roundtrip(\"-(0x0)\")\n\n\ndef test_lambda_parentheses():\n check_ast_roundtrip(\"(lambda: int)()\")\n\n\ndef test_chained_comparisons():\n check_ast_roundtrip(\"1 < 4 <= 5\")\n check_ast_roundtrip(\"a is b is c is not d\")\n\n\ndef test_function_arguments():\n check_ast_roundtrip(\"def f(): pass\")\n check_ast_roundtrip(\"def f(a): pass\")\n check_ast_roundtrip(\"def f(b = 2): pass\")\n check_ast_roundtrip(\"def f(a, b): pass\")\n check_ast_roundtrip(\"def f(a, b = 2): pass\")\n check_ast_roundtrip(\"def f(a = 5, b = 2): pass\")\n check_ast_roundtrip(\"def f(*args, **kwargs): pass\")\n check_ast_roundtrip(\"def f(*, a = 1, b = 2): pass\")\n check_ast_roundtrip(\"def f(*, a = 1, b): pass\")\n check_ast_roundtrip(\"def f(*, a, b = 2): pass\")\n check_ast_roundtrip(\"def f(a, b = None, *, c, **kwds): pass\")\n check_ast_roundtrip(\"def f(a=2, *args, c=5, d, **kwds): pass\")\n\n\ndef test_relative_import():\n check_ast_roundtrip(relative_import)\n\n\ndef test_import_many():\n check_ast_roundtrip(import_many)\n\n\ndef test_nonlocal():\n check_ast_roundtrip(nonlocal_ex)\n\n\ndef test_raise_from():\n check_ast_roundtrip(raise_from)\n\n\ndef test_bytes():\n check_ast_roundtrip(\"b'123'\")\n\n\n@pytest.mark.skipif(sys.version_info < (3, 6), reason=\"Not supported < 3.6\")\ndef test_formatted_value():\n check_ast_roundtrip('f\"{value}\"')\n check_ast_roundtrip('f\"{value!s}\"')\n check_ast_roundtrip('f\"{value:4}\"')\n check_ast_roundtrip('f\"{value!s:4}\"')\n\n\n@pytest.mark.skipif(sys.version_info < (3, 6), reason=\"Not supported < 3.6\")\ndef test_joined_str():\n check_ast_roundtrip('f\"{key}={value!s}\"')\n check_ast_roundtrip('f\"{key}={value!r}\"')\n check_ast_roundtrip('f\"{key}={value!a}\"')\n\n\n@pytest.mark.skipif(sys.version_info != (3, 6, 0), reason=\"Only supported on 3.6.0\")\ndef test_joined_str_361():\n check_ast_roundtrip('f\"{key:4}={value!s}\"')\n check_ast_roundtrip('f\"{key:02}={value!r}\"')\n check_ast_roundtrip('f\"{key:6}={value!a}\"')\n check_ast_roundtrip('f\"{key:4}={value:#06x}\"')\n check_ast_roundtrip('f\"{key:02}={value:#06x}\"')\n check_ast_roundtrip('f\"{key:6}={value:#06x}\"')\n check_ast_roundtrip('f\"{key:4}={value!s:#06x}\"')\n check_ast_roundtrip('f\"{key:4}={value!r:#06x}\"')\n check_ast_roundtrip('f\"{key:4}={value!a:#06x}\"')\n\n\n@pytest.mark.skipif(sys.version_info[:2] < (3, 6), reason=\"Only for Python 3.6 or greater\")\ndef test_complex_f_string():\n check_ast_roundtrip(complex_f_string)\n\n\ndef test_annotations():\n check_ast_roundtrip(\"def f(a : int): pass\")\n check_ast_roundtrip(\"def f(a: int = 5): pass\")\n check_ast_roundtrip(\"def f(*args: [int]): pass\")\n check_ast_roundtrip(\"def f(**kwargs: dict): pass\")\n check_ast_roundtrip(\"def f() -> None: pass\")\n\n\n@pytest.mark.skipif(sys.version_info < (2, 7), reason=\"Not supported < 2.7\")\ndef test_set_literal():\n check_ast_roundtrip(\"{'a', 'b', 'c'}\")\n\n\n@pytest.mark.skipif(sys.version_info < (2, 7), reason=\"Not supported < 2.7\")\ndef test_set_comprehension():\n check_ast_roundtrip(\"{x for x in range(5)}\")\n\n\n@pytest.mark.skipif(sys.version_info < (2, 7), reason=\"Not supported < 2.7\")\ndef test_dict_comprehension():\n check_ast_roundtrip(\"{x: x*x for x in range(10)}\")\n\n\n@pytest.mark.skipif(sys.version_info < (3, 6), reason=\"Not supported < 3.6\")\ndef test_dict_with_unpacking():\n check_ast_roundtrip(\"{**x}\")\n check_ast_roundtrip(\"{a: b, **x}\")\n\n\n@pytest.mark.skipif(sys.version_info < (3, 6), reason=\"Not supported < 3.6\")\ndef test_async_comp_and_gen_in_async_function():\n check_ast_roundtrip(async_comprehensions_and_generators)\n\n\n@pytest.mark.skipif(sys.version_info < (3, 7), reason=\"Not supported < 3.7\")\ndef test_async_comprehension():\n check_ast_roundtrip(\"{i async for i in aiter() if i % 2}\")\n check_ast_roundtrip(\"[i async for i in aiter() if i % 2]\")\n check_ast_roundtrip(\"{i: -i async for i in aiter() if i % 2}\")\n\n\n@pytest.mark.skipif(sys.version_info < (3, 7), reason=\"Not supported < 3.7\")\ndef test_async_generator_expression():\n check_ast_roundtrip(\"(i ** 2 async for i in agen())\")\n check_ast_roundtrip(\"(i - 1 async for i in agen() if i % 2)\")\n\n\ndef test_class_decorators():\n check_ast_roundtrip(class_decorator)\n\n\ndef test_class_definition():\n check_ast_roundtrip(\"class A(metaclass=type, *[], **{}): pass\")\n\n\ndef test_elifs():\n check_ast_roundtrip(elif1)\n check_ast_roundtrip(elif2)\n\n\ndef test_try_except_finally():\n check_ast_roundtrip(try_except_finally)\n\n\ndef test_starred_assignment():\n check_ast_roundtrip(\"a, *b, c = seq\")\n check_ast_roundtrip(\"a, (*b, c) = seq\")\n check_ast_roundtrip(\"a, *b[0], c = seq\")\n check_ast_roundtrip(\"a, *(b, c) = seq\")\n\n\n@pytest.mark.skipif(sys.version_info < (3, 6), reason=\"Not supported < 3.6\")\ndef test_variable_annotation():\n check_ast_roundtrip(\"a: int\")\n check_ast_roundtrip(\"a: int = 0\")\n check_ast_roundtrip(\"a: int = None\")\n check_ast_roundtrip(\"some_list: List[int]\")\n check_ast_roundtrip(\"some_list: List[int] = []\")\n check_ast_roundtrip(\"t: Tuple[int, ...] = (1, 2, 3)\")\n check_ast_roundtrip(\"(a): int\")\n check_ast_roundtrip(\"(a): int = 0\")\n check_ast_roundtrip(\"(a): int = None\")\n\n\ndef test_with_simple():\n check_ast_roundtrip(with_simple)\n\n\ndef test_with_as():\n check_ast_roundtrip(with_as)\n\n\n@pytest.mark.skipif(sys.version_info < (2, 7), reason=\"Not supported < 2.7\")\ndef test_with_two_items():\n check_ast_roundtrip(with_two_items)\n\n\n@pytest.mark.skipif(sys.version_info < (3, 5), reason=\"Not supported < 3.5\")\ndef test_async_function_def():\n check_ast_roundtrip(async_function_def)\n\n\n@pytest.mark.skipif(sys.version_info < (3, 5), reason=\"Not supported < 3.5\")\ndef test_async_for():\n check_ast_roundtrip(async_for)\n\n\n@pytest.mark.skipif(sys.version_info < (3, 5), reason=\"Not supported < 3.5\")\ndef test_async_with():\n check_ast_roundtrip(async_with)\n\n\n@pytest.mark.skipif(sys.version_info < (3, 5), reason=\"Not supported < 3.5\")\ndef test_async_with_as():\n check_ast_roundtrip(async_with_as)\n\n\n@pytest.mark.skipif(sys.version_info < (3, 10), reason=\"Not supported < 3.10\")\n@pytest.mark.parametrize(\n \"literal\",\n [match_literal, match_with_noop, match_literal_and_variable, match_classes, match_nested],\n)\ndef test_match_literal(literal):\n check_ast_roundtrip(literal)\n\n\n@pytest.mark.skipif(sys.version_info < (3, 14), reason=\"Not supported < 3.14\")\ndef test_tstrings():\n check_ast_roundtrip(\"t'foo'\")\n check_ast_roundtrip(\"t'foo {bar}'\")\n check_ast_roundtrip(\"t'foo {bar!s:.2f}'\")\n check_ast_roundtrip(\"t'{a + b}'\")\n check_ast_roundtrip(\"t'{a + b:x}'\")\n check_ast_roundtrip(\"t'{a + b!s}'\")\n check_ast_roundtrip(\"t'{ {a}}'\")\n check_ast_roundtrip(\"t'{ {a}=}'\")\n check_ast_roundtrip(\"t'{{a}}'\")\n check_ast_roundtrip(\"t''\")\n\n\ndef test_subscript_with_tuple():\n \"\"\"Test change in visit_Subscript/visit_Index is_non_empty_tuple.\"\"\"\n check_ast_roundtrip(\"a[()]\")\n check_ast_roundtrip(\"a[b]\")\n check_ast_roundtrip(\"a[(*b,)]\")\n check_ast_roundtrip(\"a[(1, 2)]\")\n check_ast_roundtrip(\"a[(1, *b)]\")\n\n\n@pytest.mark.skipif(sys.version_info < (3, 11), reason=\"Not supported < 3.11\")\ndef test_subscript_without_tuple():\n \"\"\"Test change in visit_Subscript/visit_Index is_non_empty_tuple.\"\"\"\n check_ast_roundtrip(\"a[*b]\")\n check_ast_roundtrip(\"a[1, *b]\")\n\n\ndef test_attribute_on_int():\n check_ast_roundtrip(\"1 .__abs__()\")\n" }, { "path": "lib/spack/spack/test/util/util_gpg.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport pathlib\n\nimport pytest\n\nimport spack.util.gpg\n\n\n@pytest.fixture()\ndef has_socket_dir():\n spack.util.gpg.init()\n return bool(spack.util.gpg.SOCKET_DIR)\n\n\ndef test_parse_gpg_output_case_one():\n # Two keys, fingerprint for primary keys, but not subkeys\n output = \"\"\"sec::2048:1:AAAAAAAAAAAAAAAA:AAAAAAAAAA:AAAAAAAAAA:::::::::\nfpr:::::::::XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX:\nuid:::::::AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA::Joe (Test) :\nssb::2048:1:AAAAAAAAAAAAAAAA:AAAAAAAAAA::::::::::\nsec::2048:1:AAAAAAAAAAAAAAAA:AAAAAAAAAA:AAAAAAAAAA:::::::::\nfpr:::::::::YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY:\nuid:::::::AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA::Joe (Test) :\nssb::2048:1:AAAAAAAAAAAAAAAA:AAAAAAAAAA::::::::::\n\"\"\"\n keys = spack.util.gpg._parse_secret_keys_output(output)\n\n assert len(keys) == 2\n assert keys[0] == \"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n assert keys[1] == \"YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY\"\n\n\ndef test_parse_gpg_output_case_two():\n # One key, fingerprint for primary key as well as subkey\n output = \"\"\"sec:-:2048:1:AAAAAAAAAA:AAAAAAAA:::-:::escaESCA:::+:::23::0:\nfpr:::::::::XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX:\ngrp:::::::::AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA:\nuid:-::::AAAAAAAAA::AAAAAAAAA::Joe (Test) ::::::::::0:\nssb:-:2048:1:AAAAAAAAA::::::esa:::+:::23:\nfpr:::::::::YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY:\ngrp:::::::::AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA:\n\"\"\"\n keys = spack.util.gpg._parse_secret_keys_output(output)\n\n assert len(keys) == 1\n assert keys[0] == \"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n\n\ndef test_parse_gpg_output_case_three():\n # Two keys, fingerprint for primary keys as well as subkeys\n output = \"\"\"sec::2048:1:AAAAAAAAAAAAAAAA:AAAAAAAAAA:AAAAAAAAAA:::::::::\nfpr:::::::::WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW:\nuid:::::::AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA::Joe (Test) :\nssb::2048:1:AAAAAAAAAAAAAAAA:AAAAAAAAAA::::::::::\nfpr:::::::::XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX:\nsec::2048:1:AAAAAAAAAAAAAAAA:AAAAAAAAAA:AAAAAAAAAA:::::::::\nfpr:::::::::YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY:\nuid:::::::AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA::Joe (Test) :\nssb::2048:1:AAAAAAAAAAAAAAAA:AAAAAAAAAA::::::::::\nfpr:::::::::ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ:\"\"\"\n\n keys = spack.util.gpg._parse_secret_keys_output(output)\n\n assert len(keys) == 2\n assert keys[0] == \"WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW\"\n assert keys[1] == \"YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY\"\n\n\n@pytest.mark.requires_executables(\"gpg2\")\ndef test_really_long_gnupghome_dir(tmp_path: pathlib.Path, has_socket_dir):\n if not has_socket_dir:\n pytest.skip(\"This test requires /var/run/user/$(id -u)\")\n\n N = 960\n tdir = str(tmp_path)\n while len(tdir) < N:\n tdir = os.path.join(tdir, \"filler\")\n\n tdir = tdir[:N].rstrip(os.sep)\n tdir += \"0\" * (N - len(tdir))\n\n with spack.util.gpg.gnupghome_override(tdir):\n spack.util.gpg.create(\n name=\"Spack testing 1\", email=\"test@spack.io\", comment=\"Spack testing key\", expires=\"0\"\n )\n spack.util.gpg.list(True, True)\n" }, { "path": "lib/spack/spack/test/util/util_url.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Test Spack's URL handling utility functions.\"\"\"\n\nimport os\nimport pathlib\nimport urllib.parse\n\nimport pytest\n\nimport spack.util.path\nimport spack.util.url as url_util\nfrom spack.llnl.util.filesystem import working_dir\n\n\ndef test_url_local_file_path(tmp_path: pathlib.Path):\n # Create a file\n path = str(tmp_path / \"hello.txt\")\n with open(path, \"wb\") as f:\n f.write(b\"hello world\")\n\n assert url_util.path_to_file_url(path).startswith(\"file://\")\n\n # Go from path -> url -> path.\n roundtrip = url_util.local_file_path(url_util.path_to_file_url(path))\n\n # Verify it's the same file.\n assert os.path.samefile(roundtrip, path)\n\n # Test if it accepts urlparse objects\n parsed = urllib.parse.urlparse(url_util.path_to_file_url(path))\n assert os.path.samefile(url_util.local_file_path(parsed), path)\n\n\ndef test_url_local_file_path_no_file_scheme():\n assert url_util.local_file_path(\"https://example.com/hello.txt\") is None\n assert url_util.local_file_path(\"C:\\\\Program Files\\\\hello.txt\") is None\n\n\ndef test_relative_path_to_file_url(tmp_path: pathlib.Path):\n # Create a file\n path = str(tmp_path / \"hello.txt\")\n with open(path, \"wb\") as f:\n f.write(b\"hello world\")\n\n with working_dir(str(tmp_path)):\n roundtrip = url_util.local_file_path(url_util.path_to_file_url(\"hello.txt\"))\n assert os.path.samefile(roundtrip, path)\n\n\n@pytest.mark.parametrize(\"resolve_href\", [True, False])\n@pytest.mark.parametrize(\"scheme\", [\"http\", \"s3\", \"gs\", \"file\", \"oci\"])\ndef test_url_join_absolute(scheme, resolve_href):\n \"\"\"Test that joining a URL with an absolute path works the same for schemes we care about, and\n whether we work in web browser mode or not.\"\"\"\n netloc = \"\" if scheme == \"file\" else \"example.com\"\n a1 = url_util.join(f\"{scheme}://{netloc}/a/b/c\", \"/d/e/f\", resolve_href=resolve_href)\n a2 = url_util.join(f\"{scheme}://{netloc}/a/b/c\", \"/d\", \"e\", \"f\", resolve_href=resolve_href)\n assert a1 == a2 == f\"{scheme}://{netloc}/d/e/f\"\n\n b1 = url_util.join(f\"{scheme}://{netloc}/a\", \"https://b.com/b\", resolve_href=resolve_href)\n b2 = url_util.join(f\"{scheme}://{netloc}/a\", \"https://b.com\", \"b\", resolve_href=resolve_href)\n assert b1 == b2 == \"https://b.com/b\"\n\n\n@pytest.mark.parametrize(\"scheme\", [\"http\", \"s3\", \"gs\"])\ndef test_url_join_up(scheme):\n \"\"\"Test that the netloc component is preserved when going .. up in the path.\"\"\"\n a1 = url_util.join(f\"{scheme}://netloc/a/b.html\", \"c\", resolve_href=True)\n assert a1 == f\"{scheme}://netloc/a/c\"\n b1 = url_util.join(f\"{scheme}://netloc/a/b.html\", \"../c\", resolve_href=True)\n b2 = url_util.join(f\"{scheme}://netloc/a/b.html\", \"..\", \"c\", resolve_href=True)\n assert b1 == b2 == f\"{scheme}://netloc/c\"\n c1 = url_util.join(f\"{scheme}://netloc/a/b.html\", \"../../c\", resolve_href=True)\n c2 = url_util.join(f\"{scheme}://netloc/a/b.html\", \"..\", \"..\", \"c\", resolve_href=True)\n assert c1 == c2 == f\"{scheme}://netloc/c\"\n\n d1 = url_util.join(f\"{scheme}://netloc/a/b\", \"c\", resolve_href=False)\n assert d1 == f\"{scheme}://netloc/a/b/c\"\n d2 = url_util.join(f\"{scheme}://netloc/a/b\", \"../c\", resolve_href=False)\n d3 = url_util.join(f\"{scheme}://netloc/a/b\", \"..\", \"c\", resolve_href=False)\n assert d2 == d3 == f\"{scheme}://netloc/a/c\"\n e1 = url_util.join(f\"{scheme}://netloc/a/b\", \"../../c\", resolve_href=False)\n e2 = url_util.join(f\"{scheme}://netloc/a/b\", \"..\", \"..\", \"c\", resolve_href=False)\n assert e1 == e2 == f\"{scheme}://netloc/c\"\n f1 = url_util.join(f\"{scheme}://netloc/a/b\", \"../../../c\", resolve_href=False)\n f2 = url_util.join(f\"{scheme}://netloc/a/b\", \"..\", \"..\", \"..\", \"c\", resolve_href=False)\n assert f1 == f2 == f\"{scheme}://netloc/c\"\n\n\n@pytest.mark.parametrize(\"scheme\", [\"http\", \"https\", \"ftp\", \"s3\", \"gs\", \"file\"])\ndef test_url_join_resolve_href(scheme):\n \"\"\"test that `resolve_href=True` behaves like a web browser at the base page, and\n `resolve_href=False` behaves like joining paths in a file system at the base directory.\"\"\"\n # these are equivalent because of the trailing /\n netloc = \"\" if scheme == \"file\" else \"netloc\"\n a1 = url_util.join(f\"{scheme}://{netloc}/my/path/\", \"other/path\", resolve_href=True)\n a2 = url_util.join(f\"{scheme}://{netloc}/my/path/\", \"other\", \"path\", resolve_href=True)\n assert a1 == a2 == f\"{scheme}://{netloc}/my/path/other/path\"\n b1 = url_util.join(f\"{scheme}://{netloc}/my/path\", \"other/path\", resolve_href=False)\n b2 = url_util.join(f\"{scheme}://{netloc}/my/path\", \"other\", \"path\", resolve_href=False)\n assert b1 == b2 == f\"{scheme}://{netloc}/my/path/other/path\"\n\n # this is like a web browser: relative to /my.\n c1 = url_util.join(f\"{scheme}://{netloc}/my/path\", \"other/path\", resolve_href=True)\n c2 = url_util.join(f\"{scheme}://{netloc}/my/path\", \"other\", \"path\", resolve_href=True)\n assert c1 == c2 == f\"{scheme}://{netloc}/my/other/path\"\n\n\ndef test_default_download_name():\n url = \"https://example.com:1234/path/to/file.txt;params?abc=def#file=blob.tar\"\n filename = url_util.default_download_filename(url)\n assert filename == spack.util.path.sanitize_filename(filename)\n\n\ndef test_default_download_name_dot_dot():\n \"\"\"Avoid that downloaded files get names computed as ., .. or any hidden file.\"\"\"\n assert url_util.default_download_filename(\"https://example.com/.\") == \"_\"\n assert url_util.default_download_filename(\"https://example.com/..\") == \"_.\"\n assert url_util.default_download_filename(\"https://example.com/.abcdef\") == \"_abcdef\"\n\n\ndef test_parse_link_rel_next():\n parse = url_util.parse_link_rel_next\n assert parse(r'; rel=\"next\"') == \"/abc\"\n assert parse(r'; x=y; rel=\"next\", ; x=y; rel=\"prev\"') == \"/abc\"\n assert parse(r'; rel=\"prev\"; x=y, ; x=y; rel=\"next\"') == \"/def\"\n\n # example from RFC5988\n assert (\n parse(\n r\"\"\"; title*=UTF-8'de'letztes%20Kapitel; rel=\"previous\",\"\"\"\n r\"\"\"; title*=UTF-8'de'n%c3%a4chstes%20Kapitel; rel=\"next\" \"\"\"\n )\n == \"/TheBook/chapter4\"\n )\n\n assert (\n parse(r\"\"\"; key=\";a=b, ; e=f\"; rel=\"next\" \"\"\")\n == \"https://example.com/example\"\n )\n\n assert parse(\"https://example.com/example\") is None\n assert parse(\" None:\n # Value with invalid type\n a = VariantMap()\n with pytest.raises(TypeError):\n a[\"foo\"] = 2\n\n # Duplicate variant\n a[\"foo\"] = MultiValuedVariant(\"foo\", (\"bar\", \"baz\"))\n with pytest.raises(DuplicateVariantError):\n a[\"foo\"] = MultiValuedVariant(\"foo\", (\"bar\",))\n\n with pytest.raises(DuplicateVariantError):\n a[\"foo\"] = SingleValuedVariant(\"foo\", \"bar\")\n\n with pytest.raises(DuplicateVariantError):\n a[\"foo\"] = BoolValuedVariant(\"foo\", True)\n\n # Non matching names between key and vspec.name\n with pytest.raises(KeyError):\n a[\"bar\"] = MultiValuedVariant(\"foo\", (\"bar\",))\n\n def test_set_item(self) -> None:\n # Check that all the three types of variants are accepted\n a = VariantMap()\n\n a[\"foo\"] = BoolValuedVariant(\"foo\", True)\n a[\"bar\"] = SingleValuedVariant(\"bar\", \"baz\")\n a[\"foobar\"] = MultiValuedVariant(\"foobar\", (\"a\", \"b\", \"c\", \"d\", \"e\"))\n\n def test_substitute(self) -> None:\n # Check substitution of a key that exists\n a = VariantMap()\n a[\"foo\"] = BoolValuedVariant(\"foo\", True)\n a.substitute(SingleValuedVariant(\"foo\", \"bar\"))\n\n # Trying to substitute something that is not\n # in the map will raise a KeyError\n with pytest.raises(KeyError):\n a.substitute(BoolValuedVariant(\"bar\", True))\n\n def test_satisfies_and_constrain(self) -> None:\n # foo=bar foobar=fee feebar=foo\n a = Spec()\n a.variants[\"foo\"] = MultiValuedVariant(\"foo\", (\"bar\",))\n a.variants[\"foobar\"] = SingleValuedVariant(\"foobar\", \"fee\")\n a.variants[\"feebar\"] = SingleValuedVariant(\"feebar\", \"foo\")\n\n # foo=bar,baz foobar=fee shared=True\n b = Spec()\n b.variants[\"foo\"] = MultiValuedVariant(\"foo\", (\"bar\", \"baz\"))\n b.variants[\"foobar\"] = SingleValuedVariant(\"foobar\", \"fee\")\n b.variants[\"shared\"] = BoolValuedVariant(\"shared\", True)\n\n # concrete, different values do not intersect / satisfy each other\n assert not a.intersects(b) and not b.intersects(a)\n assert not a.satisfies(b) and not b.satisfies(a)\n\n # foo=bar,baz foobar=fee feebar=foo shared=True\n c = Spec()\n c.variants[\"foo\"] = MultiValuedVariant(\"foo\", (\"bar\", \"baz\"))\n c.variants[\"foobar\"] = SingleValuedVariant(\"foobar\", \"fee\")\n c.variants[\"feebar\"] = SingleValuedVariant(\"feebar\", \"foo\")\n c.variants[\"shared\"] = BoolValuedVariant(\"shared\", True)\n\n # concrete values cannot be constrained\n with pytest.raises(spack.variant.UnsatisfiableVariantSpecError):\n a._constrain_variants(b)\n\n def test_copy(self) -> None:\n a = VariantMap()\n a[\"foo\"] = BoolValuedVariant(\"foo\", True)\n a[\"bar\"] = SingleValuedVariant(\"bar\", \"baz\")\n a[\"foobar\"] = MultiValuedVariant(\"foobar\", (\"a\", \"b\", \"c\", \"d\", \"e\"))\n\n c = a.copy()\n assert a == c\n\n def test_str(self) -> None:\n c = VariantMap()\n c[\"foo\"] = MultiValuedVariant(\"foo\", (\"bar\", \"baz\"))\n c[\"foobar\"] = SingleValuedVariant(\"foobar\", \"fee\")\n c[\"feebar\"] = SingleValuedVariant(\"feebar\", \"foo\")\n c[\"shared\"] = BoolValuedVariant(\"shared\", True)\n assert str(c) == \"+shared feebar=foo foo:=bar,baz foobar=fee\"\n\n\ndef test_disjoint_set_initialization_errors():\n # Constructing from non-disjoint sets should raise an exception\n with pytest.raises(spack.error.SpecError) as exc_info:\n disjoint_sets((\"a\", \"b\"), (\"b\", \"c\"))\n assert \"sets in input must be disjoint\" in str(exc_info.value)\n\n # A set containing the reserved item 'none' along with other items\n # should raise an exception\n with pytest.raises(spack.error.SpecError) as exc_info:\n disjoint_sets((\"a\", \"b\"), (\"none\", \"c\"))\n assert \"The value 'none' represents the empty set,\" in str(exc_info.value)\n\n\ndef test_disjoint_set_initialization():\n # Test that no error is thrown when the sets are disjoint\n d = disjoint_sets((\"a\",), (\"b\", \"c\"), (\"e\", \"f\"))\n\n assert d.default == \"none\"\n assert d.multi is True\n assert list(d) == [\"none\", \"a\", \"b\", \"c\", \"e\", \"f\"]\n\n\ndef test_disjoint_set_fluent_methods():\n # Construct an object without the empty set\n d = disjoint_sets((\"a\",), (\"b\", \"c\"), (\"e\", \"f\")).prohibit_empty_set()\n assert (\"none\",) not in d.sets\n\n # Call this 2 times to check that no matter whether\n # the empty set was allowed or not before, the state\n # returned is consistent.\n for _ in range(2):\n d = d.allow_empty_set()\n assert (\"none\",) in d.sets\n assert \"none\" in d\n assert \"none\" in [x for x in d]\n assert \"none\" in d.feature_values\n\n # Marking a value as 'non-feature' removes it from the\n # list of feature values, but not for the items returned\n # when iterating over the object.\n d = d.with_non_feature_values(\"none\")\n assert \"none\" in d\n assert \"none\" in [x for x in d]\n assert \"none\" not in d.feature_values\n\n # Call this 2 times to check that no matter whether\n # the empty set was allowed or not before, the state\n # returned is consistent.\n for _ in range(2):\n d = d.prohibit_empty_set()\n assert (\"none\",) not in d.sets\n assert \"none\" not in d\n assert \"none\" not in [x for x in d]\n assert \"none\" not in d.feature_values\n\n\n@pytest.mark.regression(\"32694\")\n@pytest.mark.parametrize(\"other\", [True, False])\ndef test_conditional_value_comparable_to_bool(other):\n value = spack.variant.ConditionalValue(\"98\", when=Spec(\"@1.0\"))\n comparison = value == other\n assert comparison is False\n\n\n@pytest.mark.regression(\"40405\")\ndef test_wild_card_valued_variants_equivalent_to_str():\n \"\"\"\n There was a bug prioro to PR 40406 in that variants with wildcard values \"*\"\n were being overwritten in the variant constructor.\n The expected/appropriate behavior is for it to behave like value=str and this\n test demonstrates that the two are now equivalent\n \"\"\"\n str_var = spack.variant.Variant(\n name=\"str_var\",\n default=\"none\",\n values=str,\n description=\"str variant\",\n multi=True,\n validator=None,\n )\n\n wild_var = spack.variant.Variant(\n name=\"wild_var\",\n default=\"none\",\n values=\"*\",\n description=\"* variant\",\n multi=True,\n validator=None,\n )\n\n several_arbitrary_values = (\"doe\", \"re\", \"mi\")\n # \"*\" case\n wild_output = wild_var.make_variant(*several_arbitrary_values)\n wild_var.validate_or_raise(wild_output, \"test-package\")\n # str case\n str_output = str_var.make_variant(*several_arbitrary_values)\n str_var.validate_or_raise(str_output, \"test-package\")\n # equivalence each instance already validated\n assert str_output.value == wild_output.value\n\n\ndef test_variant_definitions(mock_packages):\n pkg = spack.repo.PATH.get_pkg_class(\"variant-values\")\n\n # two variant names\n assert len(pkg.variant_names()) == 2\n assert \"build_system\" in pkg.variant_names()\n assert \"v\" in pkg.variant_names()\n\n # this name doesn't exist\n assert len(pkg.variant_definitions(\"no-such-variant\")) == 0\n\n # there are 4 definitions but one is completely shadowed by another\n assert len(pkg.variants) == 4\n\n # variant_items ignores the shadowed definition\n assert len(list(pkg.variant_items())) == 3\n\n # variant_definitions also ignores the shadowed definition\n defs = [vdef for _, vdef in pkg.variant_definitions(\"v\")]\n assert len(defs) == 2\n assert defs[0].default == \"foo\"\n assert defs[0].values == (\"foo\",)\n\n assert defs[1].default == \"bar\"\n assert defs[1].values == (\"foo\", \"bar\")\n\n\n@pytest.mark.parametrize(\n \"pkg_name,value,spec,def_ids\",\n [\n (\"variant-values\", \"foo\", \"\", [0, 1]),\n (\"variant-values\", \"bar\", \"\", [1]),\n (\"variant-values\", \"foo\", \"@1.0\", [0]),\n (\"variant-values\", \"foo\", \"@2.0\", [1]),\n (\"variant-values\", \"foo\", \"@3.0\", [1]),\n (\"variant-values\", \"foo\", \"@4.0\", []),\n (\"variant-values\", \"bar\", \"@2.0\", [1]),\n (\"variant-values\", \"bar\", \"@3.0\", [1]),\n (\"variant-values\", \"bar\", \"@4.0\", []),\n # now with a global override\n (\"variant-values-override\", \"bar\", \"\", [0]),\n (\"variant-values-override\", \"bar\", \"@1.0\", [0]),\n (\"variant-values-override\", \"bar\", \"@2.0\", [0]),\n (\"variant-values-override\", \"bar\", \"@3.0\", [0]),\n (\"variant-values-override\", \"bar\", \"@4.0\", [0]),\n (\"variant-values-override\", \"baz\", \"\", [0]),\n (\"variant-values-override\", \"baz\", \"@2.0\", [0]),\n (\"variant-values-override\", \"baz\", \"@3.0\", [0]),\n (\"variant-values-override\", \"baz\", \"@4.0\", [0]),\n ],\n)\ndef test_prevalidate_variant_value(mock_packages, pkg_name, value, spec, def_ids):\n pkg = spack.repo.PATH.get_pkg_class(pkg_name)\n\n all_defs = [vdef for _, vdef in pkg.variant_definitions(\"v\")]\n\n valid_defs = spack.variant.prevalidate_variant_value(\n pkg, SingleValuedVariant(\"v\", value), spack.spec.Spec(spec)\n )\n assert len(valid_defs) == len(def_ids)\n\n for vdef, i in zip(valid_defs, def_ids):\n assert vdef is all_defs[i]\n\n\n@pytest.mark.parametrize(\n \"pkg_name,value,spec\",\n [\n (\"variant-values\", \"baz\", \"\"),\n (\"variant-values\", \"bar\", \"@1.0\"),\n (\"variant-values\", \"bar\", \"@4.0\"),\n (\"variant-values\", \"baz\", \"@3.0\"),\n (\"variant-values\", \"baz\", \"@4.0\"),\n # and with override\n (\"variant-values-override\", \"foo\", \"\"),\n (\"variant-values-override\", \"foo\", \"@1.0\"),\n (\"variant-values-override\", \"foo\", \"@2.0\"),\n (\"variant-values-override\", \"foo\", \"@3.0\"),\n (\"variant-values-override\", \"foo\", \"@4.0\"),\n ],\n)\ndef test_strict_invalid_variant_values(mock_packages, pkg_name, value, spec):\n pkg = spack.repo.PATH.get_pkg_class(pkg_name)\n\n with pytest.raises(spack.variant.InvalidVariantValueError):\n spack.variant.prevalidate_variant_value(\n pkg, SingleValuedVariant(\"v\", value), spack.spec.Spec(spec), strict=True\n )\n\n\n@pytest.mark.parametrize(\n \"pkg_name,spec,satisfies,def_id\",\n [\n (\"variant-values\", \"@1.0\", \"v=foo\", 0),\n (\"variant-values\", \"@2.0\", \"v=bar\", 1),\n (\"variant-values\", \"@3.0\", \"v=bar\", 1),\n (\"variant-values-override\", \"@1.0\", \"v=baz\", 0),\n (\"variant-values-override\", \"@2.0\", \"v=baz\", 0),\n (\"variant-values-override\", \"@3.0\", \"v=baz\", 0),\n ],\n)\ndef test_concretize_variant_default_with_multiple_defs(\n mock_packages, config, pkg_name, spec, satisfies, def_id\n):\n pkg = spack.repo.PATH.get_pkg_class(pkg_name)\n pkg_defs = [vdef for _, vdef in pkg.variant_definitions(\"v\")]\n\n spec = spack.concretize.concretize_one(f\"{pkg_name}{spec}\")\n assert spec.satisfies(satisfies)\n assert spec.package.get_variant(\"v\") is pkg_defs[def_id]\n\n\n@pytest.mark.parametrize(\n \"spec,variant_name,narrowed_type\",\n [\n # dev_path is a special case\n (\"foo dev_path=/path/to/source\", \"dev_path\", spack.variant.VariantType.SINGLE),\n # reserved name: won't be touched\n (\"foo patches=2349dc44\", \"patches\", spack.variant.VariantType.MULTI),\n # simple case -- one definition applies\n (\"variant-values@1.0 v=foo\", \"v\", spack.variant.VariantType.SINGLE),\n # simple, but with bool valued variant\n (\"pkg-a bvv=true\", \"bvv\", spack.variant.VariantType.BOOL),\n # takes the second definition, which overrides the single-valued one\n (\"variant-values@2.0 v=bar\", \"v\", spack.variant.VariantType.MULTI),\n ],\n)\ndef test_substitute_abstract_variants_narrowing(mock_packages, spec, variant_name, narrowed_type):\n spec = Spec(spec)\n spack.spec.substitute_abstract_variants(spec)\n assert spec.variants[variant_name].type == narrowed_type\n\n\ndef test_substitute_abstract_variants_failure(mock_packages):\n with pytest.raises(spack.spec.InvalidVariantForSpecError):\n # variant doesn't exist at version\n spack.spec.substitute_abstract_variants(Spec(\"variant-values@4.0 v=bar\"))\n\n\ndef test_abstract_variant_satisfies_abstract_abstract():\n # rhs should be a subset of lhs\n assert Spec(\"foo=bar\").satisfies(\"foo=bar\")\n assert Spec(\"foo=bar,baz\").satisfies(\"foo=bar\")\n assert Spec(\"foo=bar,baz\").satisfies(\"foo=bar,baz\")\n assert not Spec(\"foo=bar\").satisfies(\"foo=baz\")\n assert not Spec(\"foo=bar\").satisfies(\"foo=bar,baz\")\n assert Spec(\"foo=bar\").satisfies(\"foo=*\") # rhs empty set\n assert Spec(\"foo=*\").satisfies(\"foo=*\") # lhs and rhs empty set\n assert not Spec(\"foo=*\").satisfies(\"foo=bar\") # lhs empty set, rhs not\n\n\ndef test_abstract_variant_satisfies_concrete_abstract():\n # rhs should be a subset of lhs\n assert Spec(\"foo:=bar\").satisfies(\"foo=bar\")\n assert Spec(\"foo:=bar,baz\").satisfies(\"foo=bar\")\n assert Spec(\"foo:=bar,baz\").satisfies(\"foo=bar,baz\")\n assert not Spec(\"foo:=bar\").satisfies(\"foo=baz\")\n assert not Spec(\"foo:=bar\").satisfies(\"foo=bar,baz\")\n assert Spec(\"foo:=bar\").satisfies(\"foo=*\") # rhs empty set\n\n\ndef test_abstract_variant_satisfies_abstract_concrete():\n # always false since values can be added to the lhs\n assert not Spec(\"foo=bar\").satisfies(\"foo:=bar\")\n assert not Spec(\"foo=bar,baz\").satisfies(\"foo:=bar\")\n assert not Spec(\"foo=bar,baz\").satisfies(\"foo:=bar,baz\")\n assert not Spec(\"foo=bar\").satisfies(\"foo:=baz\")\n assert not Spec(\"foo=bar\").satisfies(\"foo:=bar,baz\")\n assert not Spec(\"foo=*\").satisfies(\"foo:=bar\") # lhs empty set\n\n\ndef test_abstract_variant_satisfies_concrete_concrete():\n # concrete values only satisfy each other when equal\n assert Spec(\"foo:=bar\").satisfies(\"foo:=bar\")\n assert not Spec(\"foo:=bar,baz\").satisfies(\"foo:=bar\")\n assert not Spec(\"foo:=bar\").satisfies(\"foo:=bar,baz\")\n assert Spec(\"foo:=bar,baz\").satisfies(\"foo:=bar,baz\")\n\n\ndef test_abstract_variant_intersects_abstract_abstract():\n # always true since the union of values satisfies both\n assert Spec(\"foo=bar\").intersects(\"foo=bar\")\n assert Spec(\"foo=bar,baz\").intersects(\"foo=bar\")\n assert Spec(\"foo=bar,baz\").intersects(\"foo=bar,baz\")\n assert Spec(\"foo=bar\").intersects(\"foo=baz\")\n assert Spec(\"foo=bar\").intersects(\"foo=bar,baz\")\n assert Spec(\"foo=bar\").intersects(\"foo=*\") # rhs empty set\n assert Spec(\"foo=*\").intersects(\"foo=*\") # lhs and rhs empty set\n assert Spec(\"foo=*\").intersects(\"foo=bar\") # lhs empty set, rhs not\n\n\ndef test_abstract_variant_intersects_concrete_abstract():\n assert Spec(\"foo:=bar\").intersects(\"foo=bar\")\n assert Spec(\"foo:=bar,baz\").intersects(\"foo=bar\")\n assert Spec(\"foo:=bar,baz\").intersects(\"foo=bar,baz\")\n assert not Spec(\"foo:=bar\").intersects(\"foo=baz\") # rhs has at least baz, lhs has not\n assert not Spec(\"foo:=bar\").intersects(\"foo=bar,baz\") # rhs has at least baz, lhs has not\n assert Spec(\"foo:=bar\").intersects(\"foo=*\") # rhs empty set\n\n\ndef test_abstract_variant_intersects_abstract_concrete():\n assert Spec(\"foo=bar\").intersects(\"foo:=bar\")\n assert not Spec(\"foo=bar,baz\").intersects(\"foo:=bar\") # lhs has at least baz, rhs has not\n assert Spec(\"foo=bar,baz\").intersects(\"foo:=bar,baz\")\n assert not Spec(\"foo=bar\").intersects(\"foo:=baz\") # lhs has at least bar, rhs has not\n assert Spec(\"foo=bar\").intersects(\"foo:=bar,baz\")\n assert Spec(\"foo=*\").intersects(\"foo:=bar\") # lhs empty set\n\n\ndef test_abstract_variant_intersects_concrete_concrete():\n # concrete values only intersect each other when equal\n assert Spec(\"foo:=bar\").intersects(\"foo:=bar\")\n assert not Spec(\"foo:=bar,baz\").intersects(\"foo:=bar\")\n assert not Spec(\"foo:=bar\").intersects(\"foo:=bar,baz\")\n assert Spec(\"foo:=bar,baz\").intersects(\"foo:=bar,baz\")\n\n\ndef test_abstract_variant_constrain_abstract_abstract():\n s1 = Spec(\"foo=bar\")\n s2 = Spec(\"foo=*\")\n assert s1.constrain(\"foo=baz\")\n assert s1 == Spec(\"foo=bar,baz\")\n assert s2.constrain(\"foo=baz\")\n assert s2 == Spec(\"foo=baz\")\n\n\ndef test_abstract_variant_constrain_abstract_concrete_fail():\n with pytest.raises(UnsatisfiableVariantSpecError):\n Spec(\"foo=bar\").constrain(\"foo:=baz\")\n\n\ndef test_abstract_variant_constrain_abstract_concrete_ok():\n s1 = Spec(\"foo=bar\")\n s2 = Spec(\"foo=*\")\n assert s1.constrain(\"foo:=bar\") # the change is concreteness\n assert s1 == Spec(\"foo:=bar\")\n assert s2.constrain(\"foo:=bar\")\n assert s2 == Spec(\"foo:=bar\")\n\n\ndef test_abstract_variant_constrain_concrete_concrete_fail():\n with pytest.raises(UnsatisfiableVariantSpecError):\n Spec(\"foo:=bar\").constrain(\"foo:=bar,baz\")\n\n\ndef test_abstract_variant_constrain_concrete_concrete_ok():\n s = Spec(\"foo:=bar\")\n assert not s.constrain(\"foo:=bar\") # no change\n\n\ndef test_abstract_variant_constrain_concrete_abstract_fail():\n s = Spec(\"foo:=bar\")\n with pytest.raises(UnsatisfiableVariantSpecError):\n s.constrain(\"foo=baz\")\n\n\ndef test_abstract_variant_constrain_concrete_abstract_ok():\n s = Spec(\"foo:=bar,baz\")\n assert not s.constrain(\"foo=bar\") # no change in value or concreteness\n assert not s.constrain(\"foo=*\")\n\n\ndef test_patches_variant():\n \"\"\"patches=x,y,z is a variant with special satisfies behavior when the rhs is abstract; it\n allows string prefix matching of the lhs.\"\"\"\n assert Spec(\"patches:=abcdef\").satisfies(\"patches=ab\")\n assert Spec(\"patches:=abcdef\").satisfies(\"patches=abcdef\")\n assert not Spec(\"patches:=abcdef\").satisfies(\"patches=xyz\")\n assert Spec(\"patches:=abcdef,xyz\").satisfies(\"patches=xyz\")\n assert not Spec(\"patches:=abcdef\").satisfies(\"patches=abcdefghi\")\n\n # but when the rhs is concrete, it must match exactly\n assert Spec(\"patches:=abcdef\").satisfies(\"patches:=abcdef\")\n assert not Spec(\"patches:=abcdef\").satisfies(\"patches:=ab\")\n assert not Spec(\"patches:=abcdef,xyz\").satisfies(\"patches:=abc,xyz\")\n assert not Spec(\"patches:=abcdef\").satisfies(\"patches:=abcdefghi\")\n\n\ndef test_constrain_narrowing():\n s = Spec(\"foo=*\")\n assert s.variants[\"foo\"].type == spack.variant.VariantType.MULTI\n assert not s.variants[\"foo\"].concrete\n s.constrain(\"+foo\")\n assert s.variants[\"foo\"].type == spack.variant.VariantType.BOOL\n assert s.variants[\"foo\"].concrete\n" }, { "path": "lib/spack/spack/test/verification.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Tests for the `spack.verify` module\"\"\"\n\nimport os\nimport pathlib\nimport shutil\nimport stat\n\nimport pytest\n\nimport spack.llnl.util.filesystem as fs\nimport spack.spec\nimport spack.store\nimport spack.util.spack_json as sjson\nimport spack.verify\nfrom spack.llnl.util.filesystem import symlink\n\npytestmark = pytest.mark.not_on_windows(\"Tests fail on Win\")\n\n\ndef test_link_manifest_entry(tmp_path: pathlib.Path):\n # Test that symlinks are properly checked against the manifest.\n # Test that the appropriate errors are generated when the check fails.\n file = tmp_path / \"file\"\n file.touch()\n link = tmp_path / \"link\"\n os.symlink(str(file), str(link))\n\n data = spack.verify.create_manifest_entry(str(link))\n assert data[\"dest\"] == str(file)\n assert all(x in data for x in (\"mode\", \"owner\", \"group\"))\n\n results = spack.verify.check_entry(str(link), data)\n assert not results.has_errors()\n\n file2 = tmp_path / \"file2\"\n file2.touch()\n os.remove(str(link))\n os.symlink(str(file2), str(link))\n\n results = spack.verify.check_entry(str(link), data)\n assert results.has_errors()\n assert str(link) in results.errors\n assert results.errors[str(link)] == [\"link\"]\n\n\ndef test_dir_manifest_entry(tmp_path: pathlib.Path):\n # Test that directories are properly checked against the manifest.\n # Test that the appropriate errors are generated when the check fails.\n dirent = str(tmp_path / \"dir\")\n fs.mkdirp(dirent)\n\n data = spack.verify.create_manifest_entry(dirent)\n assert stat.S_ISDIR(data[\"mode\"])\n assert all(x in data for x in (\"mode\", \"owner\", \"group\"))\n\n results = spack.verify.check_entry(dirent, data)\n assert not results.has_errors()\n\n data[\"mode\"] = \"garbage\"\n\n results = spack.verify.check_entry(dirent, data)\n assert results.has_errors()\n assert dirent in results.errors\n assert results.errors[dirent] == [\"mode\"]\n\n\ndef test_file_manifest_entry(tmp_path: pathlib.Path):\n # Test that files are properly checked against the manifest.\n # Test that the appropriate errors are generated when the check fails.\n orig_str = \"This is a file\"\n new_str = \"The file has changed\"\n\n file = str(tmp_path / \"dir\")\n with open(file, \"w\", encoding=\"utf-8\") as f:\n f.write(orig_str)\n\n data = spack.verify.create_manifest_entry(file)\n assert stat.S_ISREG(data[\"mode\"])\n assert data[\"size\"] == len(orig_str)\n assert all(x in data for x in (\"owner\", \"group\"))\n\n results = spack.verify.check_entry(file, data)\n assert not results.has_errors()\n\n data[\"mode\"] = 0x99999\n\n results = spack.verify.check_entry(file, data)\n assert results.has_errors()\n assert file in results.errors\n assert results.errors[file] == [\"mode\"]\n\n with open(file, \"w\", encoding=\"utf-8\") as f:\n f.write(new_str)\n\n data[\"mode\"] = os.stat(file).st_mode\n\n results = spack.verify.check_entry(file, data)\n\n expected = [\"size\", \"hash\"]\n mtime = os.stat(file).st_mtime\n if mtime != data[\"time\"]:\n expected.append(\"mtime\")\n\n assert results.has_errors()\n assert file in results.errors\n assert sorted(results.errors[file]) == sorted(expected)\n\n\ndef test_check_chmod_manifest_entry(tmp_path: pathlib.Path):\n # Check that the verification properly identifies errors for files whose\n # permissions have been modified.\n file = str(tmp_path / \"dir\")\n with open(file, \"w\", encoding=\"utf-8\") as f:\n f.write(\"This is a file\")\n\n data = spack.verify.create_manifest_entry(file)\n\n os.chmod(file, data[\"mode\"] - 1)\n\n results = spack.verify.check_entry(file, data)\n assert results.has_errors()\n assert file in results.errors\n assert results.errors[file] == [\"mode\"]\n\n\ndef test_check_prefix_manifest(tmp_path: pathlib.Path):\n # Test the verification of an entire prefix and its contents\n prefix_path = tmp_path / \"prefix\"\n prefix = str(prefix_path)\n\n spec = spack.spec.Spec(\"libelf\")\n spec._mark_concrete()\n spec.set_prefix(prefix)\n\n results = spack.verify.check_spec_manifest(spec)\n assert results.has_errors()\n assert prefix in results.errors\n assert results.errors[prefix] == [\"manifest missing\"]\n\n metadata_dir = str(prefix_path / \".spack\")\n bin_dir = str(prefix_path / \"bin\")\n other_dir = str(prefix_path / \"other\")\n\n for d in (metadata_dir, bin_dir, other_dir):\n fs.mkdirp(d)\n\n file = os.path.join(other_dir, \"file\")\n with open(file, \"w\", encoding=\"utf-8\") as f:\n f.write(\"I'm a little file short and stout\")\n\n link = os.path.join(bin_dir, \"run\")\n symlink(file, link)\n\n spack.verify.write_manifest(spec)\n results = spack.verify.check_spec_manifest(spec)\n assert not results.has_errors()\n\n os.remove(link)\n malware = os.path.join(metadata_dir, \"hiddenmalware\")\n with open(malware, \"w\", encoding=\"utf-8\") as f:\n f.write(\"Foul evil deeds\")\n\n results = spack.verify.check_spec_manifest(spec)\n assert results.has_errors()\n assert all(x in results.errors for x in (malware, link))\n assert len(results.errors) == 2\n\n assert results.errors[link] == [\"deleted\"]\n assert results.errors[malware] == [\"added\"]\n\n manifest_file = os.path.join(\n spec.prefix,\n spack.store.STORE.layout.metadata_dir,\n spack.store.STORE.layout.manifest_file_name,\n )\n with open(manifest_file, \"w\", encoding=\"utf-8\") as f:\n f.write(\"{This) string is not proper json\")\n\n results = spack.verify.check_spec_manifest(spec)\n assert results.has_errors()\n assert results.errors[spec.prefix] == [\"manifest corrupted\"]\n\n\ndef test_single_file_verification(tmp_path: pathlib.Path):\n # Test the API to verify a single file, including finding the package\n # to which it belongs\n filedir = tmp_path / \"a\" / \"b\" / \"c\" / \"d\"\n filepath = filedir / \"file\"\n metadir = tmp_path / spack.store.STORE.layout.metadata_dir\n\n fs.mkdirp(str(filedir))\n fs.mkdirp(str(metadir))\n\n with open(str(filepath), \"w\", encoding=\"utf-8\") as f:\n f.write(\"I'm a file\")\n\n data = spack.verify.create_manifest_entry(str(filepath))\n\n manifest_file = os.path.join(metadir, spack.store.STORE.layout.manifest_file_name)\n\n with open(manifest_file, \"w\", encoding=\"utf-8\") as f:\n sjson.dump({str(filepath): data}, f)\n\n results = spack.verify.check_file_manifest(str(filepath))\n assert not results.has_errors()\n\n os.utime(str(filepath), (0, 0))\n with open(str(filepath), \"w\", encoding=\"utf-8\") as f:\n f.write(\"I changed.\")\n\n results = spack.verify.check_file_manifest(str(filepath))\n\n expected = [\"hash\"]\n mtime = os.stat(str(filepath)).st_mtime\n if mtime != data[\"time\"]:\n expected.append(\"mtime\")\n\n assert results.has_errors()\n assert str(filepath) in results.errors\n assert sorted(results.errors[str(filepath)]) == sorted(expected)\n\n shutil.rmtree(str(metadir))\n results = spack.verify.check_file_manifest(filepath)\n assert results.has_errors()\n assert results.errors[filepath] == [\"not owned by any package\"]\n" }, { "path": "lib/spack/spack/test/versions.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"These version tests were taken from the RPM source code.\nWe try to maintain compatibility with RPM's version semantics\nwhere it makes sense.\n\"\"\"\n\nimport os\nimport pathlib\n\nimport pytest\n\nimport spack.concretize\nimport spack.package_base\nimport spack.spec\nimport spack.version\nfrom spack.llnl.util.filesystem import working_dir\nfrom spack.version import (\n ClosedOpenRange,\n EmptyRangeError,\n GitVersion,\n StandardVersion,\n Version,\n VersionList,\n VersionLookupError,\n VersionRange,\n is_git_version,\n ver,\n)\nfrom spack.version.git_ref_lookup import SEMVER_REGEX\n\n\ndef assert_ver_lt(a, b):\n \"\"\"Asserts the results of comparisons when 'a' is less than 'b'.\"\"\"\n a, b = ver(a), ver(b)\n assert a < b\n assert a <= b\n assert a != b\n assert not a == b\n assert not a > b\n assert not a >= b\n\n\ndef assert_ver_gt(a, b):\n \"\"\"Asserts the results of comparisons when 'a' is greater than 'b'.\"\"\"\n a, b = ver(a), ver(b)\n assert a > b\n assert a >= b\n assert a != b\n assert not a == b\n assert not a < b\n assert not a <= b\n\n\ndef assert_ver_eq(a, b):\n \"\"\"Asserts the results of comparisons when 'a' is equal to 'b'.\"\"\"\n a, b = ver(a), ver(b)\n assert not a > b\n assert a >= b\n assert not a != b\n assert a == b\n assert not a < b\n assert a <= b\n\n\ndef assert_in(needle, haystack):\n \"\"\"Asserts that 'needle' is in 'haystack'.\"\"\"\n assert ver(needle) in ver(haystack)\n\n\ndef assert_not_in(needle, haystack):\n \"\"\"Asserts that 'needle' is not in 'haystack'.\"\"\"\n assert ver(needle) not in ver(haystack)\n\n\ndef assert_canonical(canonical_list, version_list):\n \"\"\"Asserts that a redundant list is reduced to canonical form.\"\"\"\n assert ver(canonical_list) == ver(version_list)\n\n\ndef assert_overlaps(v1, v2):\n \"\"\"Asserts that two version ranges overlaps.\"\"\"\n assert ver(v1).overlaps(ver(v2))\n\n\ndef assert_no_overlap(v1, v2):\n \"\"\"Asserts that two version ranges do not overlap.\"\"\"\n assert not ver(v1).overlaps(ver(v2))\n\n\ndef assert_satisfies(v1, v2):\n \"\"\"Asserts that 'v1' satisfies 'v2'.\"\"\"\n assert ver(v1).satisfies(ver(v2))\n\n\ndef assert_does_not_satisfy(v1, v2):\n \"\"\"Asserts that 'v1' does not satisfy 'v2'.\"\"\"\n assert not ver(v1).satisfies(ver(v2))\n\n\ndef check_intersection(expected, a, b):\n \"\"\"Asserts that 'a' intersect 'b' == 'expected'.\"\"\"\n assert ver(expected) == ver(a).intersection(ver(b))\n\n\ndef check_union(expected, a, b):\n \"\"\"Asserts that 'a' union 'b' == 'expected'.\"\"\"\n assert ver(expected) == ver(a).union(ver(b))\n\n\ndef test_string_prefix():\n assert_ver_eq(\"=xsdk-0.2.0\", \"=xsdk-0.2.0\")\n assert_ver_lt(\"=xsdk-0.2.0\", \"=xsdk-0.3\")\n assert_ver_gt(\"=xsdk-0.3\", \"=xsdk-0.2.0\")\n\n\ndef test_two_segments():\n assert_ver_eq(\"=1.0\", \"=1.0\")\n assert_ver_lt(\"=1.0\", \"=2.0\")\n assert_ver_gt(\"=2.0\", \"=1.0\")\n\n\ndef test_develop():\n assert_ver_eq(\"=develop\", \"=develop\")\n assert_ver_eq(\"=develop.local\", \"=develop.local\")\n assert_ver_lt(\"=1.0\", \"=develop\")\n assert_ver_gt(\"=develop\", \"=1.0\")\n assert_ver_eq(\"=1.develop\", \"=1.develop\")\n assert_ver_lt(\"=1.1\", \"=1.develop\")\n assert_ver_gt(\"=1.develop\", \"=1.0\")\n assert_ver_gt(\"=0.5.develop\", \"=0.5\")\n assert_ver_lt(\"=0.5\", \"=0.5.develop\")\n assert_ver_lt(\"=1.develop\", \"=2.1\")\n assert_ver_gt(\"=2.1\", \"=1.develop\")\n assert_ver_lt(\"=1.develop.1\", \"=1.develop.2\")\n assert_ver_gt(\"=1.develop.2\", \"=1.develop.1\")\n assert_ver_lt(\"=develop.1\", \"=develop.2\")\n assert_ver_gt(\"=develop.2\", \"=develop.1\")\n # other +infinity versions\n assert_ver_gt(\"=master\", \"=9.0\")\n assert_ver_gt(\"=head\", \"=9.0\")\n assert_ver_gt(\"=trunk\", \"=9.0\")\n assert_ver_gt(\"=develop\", \"=9.0\")\n # hierarchical develop-like versions\n assert_ver_gt(\"=develop\", \"=master\")\n assert_ver_gt(\"=master\", \"=head\")\n assert_ver_gt(\"=head\", \"=trunk\")\n assert_ver_gt(\"=9.0\", \"=system\")\n # not develop\n assert_ver_lt(\"=mydevelopmentnightmare\", \"=1.1\")\n assert_ver_lt(\"=1.mydevelopmentnightmare\", \"=1.1\")\n assert_ver_gt(\"=1.1\", \"=1.mydevelopmentnightmare\")\n\n\ndef test_isdevelop():\n assert ver(\"=develop\").isdevelop()\n assert ver(\"=develop.1\").isdevelop()\n assert ver(\"=develop.local\").isdevelop()\n assert ver(\"=master\").isdevelop()\n assert ver(\"=head\").isdevelop()\n assert ver(\"=trunk\").isdevelop()\n assert ver(\"=1.develop\").isdevelop()\n assert ver(\"=1.develop.2\").isdevelop()\n assert not ver(\"=1.1\").isdevelop()\n assert not ver(\"=1.mydevelopmentnightmare.3\").isdevelop()\n assert not ver(\"=mydevelopmentnightmare.3\").isdevelop()\n\n\ndef test_three_segments():\n assert_ver_eq(\"=2.0.1\", \"=2.0.1\")\n assert_ver_lt(\"=2.0\", \"=2.0.1\")\n assert_ver_gt(\"=2.0.1\", \"=2.0\")\n\n\ndef test_alpha():\n # TODO: not sure whether I like this. 2.0.1a is *usually*\n # TODO: less than 2.0.1, but special-casing it makes version\n # TODO: comparison complicated. See version.py\n assert_ver_eq(\"=2.0.1a\", \"=2.0.1a\")\n assert_ver_gt(\"=2.0.1a\", \"=2.0.1\")\n assert_ver_lt(\"=2.0.1\", \"=2.0.1a\")\n\n\ndef test_patch():\n assert_ver_eq(\"=5.5p1\", \"=5.5p1\")\n assert_ver_lt(\"=5.5p1\", \"=5.5p2\")\n assert_ver_gt(\"=5.5p2\", \"=5.5p1\")\n assert_ver_eq(\"=5.5p10\", \"=5.5p10\")\n assert_ver_lt(\"=5.5p1\", \"=5.5p10\")\n assert_ver_gt(\"=5.5p10\", \"=5.5p1\")\n\n\ndef test_num_alpha_with_no_separator():\n assert_ver_lt(\"=10xyz\", \"=10.1xyz\")\n assert_ver_gt(\"=10.1xyz\", \"=10xyz\")\n assert_ver_eq(\"=xyz10\", \"=xyz10\")\n assert_ver_lt(\"=xyz10\", \"=xyz10.1\")\n assert_ver_gt(\"=xyz10.1\", \"=xyz10\")\n\n\ndef test_alpha_with_dots():\n assert_ver_eq(\"=xyz.4\", \"=xyz.4\")\n assert_ver_lt(\"=xyz.4\", \"=8\")\n assert_ver_gt(\"=8\", \"=xyz.4\")\n assert_ver_lt(\"=xyz.4\", \"=2\")\n assert_ver_gt(\"=2\", \"=xyz.4\")\n\n\ndef test_nums_and_patch():\n assert_ver_lt(\"=5.5p2\", \"=5.6p1\")\n assert_ver_gt(\"=5.6p1\", \"=5.5p2\")\n assert_ver_lt(\"=5.6p1\", \"=6.5p1\")\n assert_ver_gt(\"=6.5p1\", \"=5.6p1\")\n\n\ndef test_prereleases():\n # pre-releases are special: they are less than final releases\n assert_ver_lt(\"=6.0alpha\", \"=6.0alpha0\")\n assert_ver_lt(\"=6.0alpha0\", \"=6.0alpha1\")\n assert_ver_lt(\"=6.0alpha1\", \"=6.0alpha2\")\n assert_ver_lt(\"=6.0alpha2\", \"=6.0beta\")\n assert_ver_lt(\"=6.0beta\", \"=6.0beta0\")\n assert_ver_lt(\"=6.0beta0\", \"=6.0beta1\")\n assert_ver_lt(\"=6.0beta1\", \"=6.0beta2\")\n assert_ver_lt(\"=6.0beta2\", \"=6.0rc\")\n assert_ver_lt(\"=6.0rc\", \"=6.0rc0\")\n assert_ver_lt(\"=6.0rc0\", \"=6.0rc1\")\n assert_ver_lt(\"=6.0rc1\", \"=6.0rc2\")\n assert_ver_lt(\"=6.0rc2\", \"=6.0\")\n\n\ndef test_alpha_beta():\n # these are not pre-releases, but ordinary string components.\n assert_ver_gt(\"=10b2\", \"=10a1\")\n assert_ver_lt(\"=10a2\", \"=10b2\")\n\n\ndef test_double_alpha():\n assert_ver_eq(\"=1.0aa\", \"=1.0aa\")\n assert_ver_lt(\"=1.0a\", \"=1.0aa\")\n assert_ver_gt(\"=1.0aa\", \"=1.0a\")\n\n\ndef test_padded_numbers():\n assert_ver_eq(\"=10.0001\", \"=10.0001\")\n assert_ver_eq(\"=10.0001\", \"=10.1\")\n assert_ver_eq(\"=10.1\", \"=10.0001\")\n assert_ver_lt(\"=10.0001\", \"=10.0039\")\n assert_ver_gt(\"=10.0039\", \"=10.0001\")\n\n\ndef test_close_numbers():\n assert_ver_lt(\"=4.999.9\", \"=5.0\")\n assert_ver_gt(\"=5.0\", \"=4.999.9\")\n\n\ndef test_date_stamps():\n assert_ver_eq(\"=20101121\", \"=20101121\")\n assert_ver_lt(\"=20101121\", \"=20101122\")\n assert_ver_gt(\"=20101122\", \"=20101121\")\n\n\ndef test_underscores():\n assert_ver_eq(\"=2_0\", \"=2_0\")\n assert_ver_eq(\"=2.0\", \"=2_0\")\n assert_ver_eq(\"=2_0\", \"=2.0\")\n assert_ver_eq(\"=2-0\", \"=2_0\")\n assert_ver_eq(\"=2_0\", \"=2-0\")\n\n\ndef test_rpm_oddities():\n assert_ver_eq(\"=1b.fc17\", \"=1b.fc17\")\n assert_ver_lt(\"=1b.fc17\", \"=1.fc17\")\n assert_ver_gt(\"=1.fc17\", \"=1b.fc17\")\n assert_ver_eq(\"=1g.fc17\", \"=1g.fc17\")\n assert_ver_gt(\"=1g.fc17\", \"=1.fc17\")\n assert_ver_lt(\"=1.fc17\", \"=1g.fc17\")\n\n\n# Stuff below here is not taken from RPM's tests and is\n# unique to spack\ndef test_version_ranges():\n assert_ver_lt(\"1.2:1.4\", \"1.6\")\n assert_ver_gt(\"1.6\", \"1.2:1.4\")\n assert_ver_eq(\"1.2:1.4\", \"1.2:1.4\")\n assert ver(\"1.2:1.4\") != ver(\"1.2:1.6\")\n\n assert_ver_lt(\"1.2:1.4\", \"1.5:1.6\")\n assert_ver_gt(\"1.5:1.6\", \"1.2:1.4\")\n\n\ndef test_version_range_with_prereleases():\n # 1.2.1: means from the 1.2.1 release onwards\n assert_does_not_satisfy(\"1.2.1alpha1\", \"1.2.1:\")\n assert_does_not_satisfy(\"1.2.1beta2\", \"1.2.1:\")\n assert_does_not_satisfy(\"1.2.1rc3\", \"1.2.1:\")\n\n # Pre-releases of 1.2.1 are included in the 1.2.0: range\n assert_satisfies(\"1.2.1alpha1\", \"1.2.0:\")\n assert_satisfies(\"1.2.1beta1\", \"1.2.0:\")\n assert_satisfies(\"1.2.1rc3\", \"1.2.0:\")\n\n # In Spack 1.2 and 1.2.0 are distinct with 1.2 < 1.2.0. So a lowerbound on 1.2 includes\n # pre-releases of 1.2.0 as well.\n assert_satisfies(\"1.2.0alpha1\", \"1.2:\")\n assert_satisfies(\"1.2.0beta2\", \"1.2:\")\n assert_satisfies(\"1.2.0rc3\", \"1.2:\")\n\n # An upperbound :1.1 does not include 1.2.0 pre-releases\n assert_does_not_satisfy(\"1.2.0alpha1\", \":1.1\")\n assert_does_not_satisfy(\"1.2.0beta2\", \":1.1\")\n assert_does_not_satisfy(\"1.2.0rc3\", \":1.1\")\n\n assert_satisfies(\"1.2.0alpha1\", \":1.2\")\n assert_satisfies(\"1.2.0beta2\", \":1.2\")\n assert_satisfies(\"1.2.0rc3\", \":1.2\")\n\n # You can also construct ranges from prereleases\n assert_satisfies(\"1.2.0alpha2:1.2.0beta1\", \"1.2.0alpha1:1.2.0beta2\")\n assert_satisfies(\"1.2.0\", \"1.2.0alpha1:\")\n assert_satisfies(\"=1.2.0\", \"1.2.0alpha1:\")\n assert_does_not_satisfy(\"=1.2.0\", \":1.2.0rc345\")\n\n\ndef test_contains():\n assert_in(\"=1.3\", \"1.2:1.4\")\n assert_in(\"=1.2.5\", \"1.2:1.4\")\n assert_in(\"=1.3.5\", \"1.2:1.4\")\n assert_in(\"=1.3.5-7\", \"1.2:1.4\")\n assert_not_in(\"=1.1\", \"1.2:1.4\")\n assert_not_in(\"=1.5\", \"1.2:1.4\")\n assert_not_in(\"=1.5\", \"1.5.1:1.6\")\n assert_not_in(\"=1.5\", \"1.5.1:\")\n\n assert_in(\"=1.4.2\", \"1.2:1.4\")\n assert_not_in(\"=1.4.2\", \"1.2:1.4.0\")\n\n assert_in(\"=1.2.8\", \"1.2.7:1.4\")\n assert_in(\"1.2.7:1.4\", \":\")\n assert_not_in(\"=1.2.5\", \"1.2.7:1.4\")\n\n assert_in(\"=1.4.1\", \"1.2.7:1.4\")\n assert_not_in(\"=1.4.1\", \"1.2.7:1.4.0\")\n\n\ndef test_in_list():\n assert_in(\"1.2\", [\"1.5\", \"1.2\", \"1.3\"])\n assert_in(\"1.2.5\", [\"1.5\", \"1.2:1.3\"])\n assert_in(\"1.5\", [\"1.5\", \"1.2:1.3\"])\n assert_not_in(\"1.4\", [\"1.5\", \"1.2:1.3\"])\n\n assert_in(\"1.2.5:1.2.7\", [\":\"])\n assert_in(\"1.2.5:1.2.7\", [\"1.5\", \"1.2:1.3\"])\n assert_not_in(\"1.2.5:1.5\", [\"1.5\", \"1.2:1.3\"])\n assert_not_in(\"1.1:1.2.5\", [\"1.5\", \"1.2:1.3\"])\n\n\ndef test_ranges_overlap():\n assert_overlaps(\"1.2\", \"1.2\")\n assert_overlaps(\"1.2.1\", \"1.2.1\")\n assert_overlaps(\"1.2.1b\", \"1.2.1b\")\n\n assert_overlaps(\"1.2:1.7\", \"1.6:1.9\")\n assert_overlaps(\":1.7\", \"1.6:1.9\")\n assert_overlaps(\":1.7\", \":1.9\")\n assert_overlaps(\":1.7\", \"1.6:\")\n assert_overlaps(\"1.2:\", \"1.6:1.9\")\n assert_overlaps(\"1.2:\", \":1.9\")\n assert_overlaps(\"1.2:\", \"1.6:\")\n assert_overlaps(\":\", \":\")\n assert_overlaps(\":\", \"1.6:1.9\")\n assert_overlaps(\"1.6:1.9\", \":\")\n\n\ndef test_overlap_with_containment():\n assert_in(\"1.6.5\", \"1.6\")\n assert_in(\"1.6.5\", \":1.6\")\n\n assert_overlaps(\"1.6.5\", \":1.6\")\n assert_overlaps(\":1.6\", \"1.6.5\")\n\n assert_not_in(\":1.6\", \"1.6.5\")\n assert_in(\"1.6.5\", \":1.6\")\n\n\ndef test_lists_overlap():\n assert_overlaps(\"1.2b:1.7,5\", \"1.6:1.9,1\")\n assert_overlaps(\"1,2,3,4,5\", \"3,4,5,6,7\")\n assert_overlaps(\"1,2,3,4,5\", \"5,6,7\")\n assert_overlaps(\"1,2,3,4,5\", \"5:7\")\n assert_overlaps(\"1,2,3,4,5\", \"3, 6:7\")\n assert_overlaps(\"1, 2, 4, 6.5\", \"3, 6:7\")\n assert_overlaps(\"1, 2, 4, 6.5\", \":, 5, 8\")\n assert_overlaps(\"1, 2, 4, 6.5\", \":\")\n assert_no_overlap(\"1, 2, 4\", \"3, 6:7\")\n assert_no_overlap(\"1,2,3,4,5\", \"6,7\")\n assert_no_overlap(\"1,2,3,4,5\", \"6:7\")\n\n\ndef test_canonicalize_list():\n assert_canonical([\"1.2\", \"1.3\", \"1.4\"], [\"1.2\", \"1.3\", \"1.3\", \"1.4\"])\n\n assert_canonical([\"1.2\", \"1.3:1.4\"], [\"1.2\", \"1.3\", \"1.3:1.4\"])\n\n assert_canonical([\"1.2\", \"1.3:1.4\"], [\"1.2\", \"1.3:1.4\", \"1.4\"])\n\n assert_canonical([\"1.3:1.4\"], [\"1.3:1.4\", \"1.3\", \"1.3.1\", \"1.3.9\", \"1.4\"])\n\n assert_canonical([\"1.3:1.4\"], [\"1.3\", \"1.3.1\", \"1.3.9\", \"1.4\", \"1.3:1.4\"])\n\n assert_canonical([\"1.3:1.5\"], [\"1.3\", \"1.3.1\", \"1.3.9\", \"1.4:1.5\", \"1.3:1.4\"])\n\n assert_canonical([\"1.3:1.5\"], [\"1.3, 1.3.1,1.3.9,1.4:1.5,1.3:1.4\"])\n\n assert_canonical([\"1.3:1.5\"], [\"1.3, 1.3.1,1.3.9,1.4 : 1.5 , 1.3 : 1.4\"])\n\n assert_canonical([\":\"], [\":,1.3, 1.3.1,1.3.9,1.4 : 1.5 , 1.3 : 1.4\"])\n\n\ndef test_intersection():\n check_intersection(\"2.5\", \"1.0:2.5\", \"2.5:3.0\")\n check_intersection(\"2.5:2.7\", \"1.0:2.7\", \"2.5:3.0\")\n check_intersection(\"0:1\", \":\", \"0:1\")\n\n check_intersection([\"1.0\", \"2.5:2.7\"], [\"1.0:2.7\"], [\"2.5:3.0\", \"1.0\"])\n check_intersection([\"2.5:2.7\"], [\"1.1:2.7\"], [\"2.5:3.0\", \"1.0\"])\n check_intersection([\"0:1\"], [\":\"], [\"0:1\"])\n\n check_intersection([\"=ref=1.0\", \"=1.1\"], [\"=ref=1.0\", \"1.1\"], [\"1:1.0\", \"=1.1\"])\n\n\ndef test_intersect_with_containment():\n check_intersection(\"1.6.5\", \"1.6.5\", \":1.6\")\n check_intersection(\"1.6.5\", \":1.6\", \"1.6.5\")\n\n check_intersection(\"1.6:1.6.5\", \":1.6.5\", \"1.6\")\n check_intersection(\"1.6:1.6.5\", \"1.6\", \":1.6.5\")\n\n check_intersection(\"11.2\", \"11\", \"11.2\")\n check_intersection(\"11.2\", \"11.2\", \"11\")\n\n\ndef test_union_with_containment():\n check_union(\":1.6\", \"1.6.5\", \":1.6\")\n check_union(\":1.6\", \":1.6\", \"1.6.5\")\n\n check_union(\":1.6\", \":1.6.5\", \"1.6\")\n check_union(\":1.6\", \"1.6\", \":1.6.5\")\n\n check_union(\":\", \"1.0:\", \":2.0\")\n\n check_union(\"1:4\", \"1:3\", \"2:4\")\n check_union(\"1:4\", \"2:4\", \"1:3\")\n\n # Tests successor/predecessor case.\n check_union(\"1:4\", \"1:2\", \"3:4\")\n\n check_union([\"1:1.0\", \"1.1\"], [\"=ref=1.0\", \"1.1\"], [\"1:1.0\", \"=1.1\"])\n\n\ndef test_basic_version_satisfaction():\n assert_satisfies(\"4.7.3\", \"4.7.3\")\n\n assert_satisfies(\"4.7.3\", \"4.7\")\n assert_satisfies(\"4.7.3v2\", \"4.7\")\n assert_satisfies(\"4.7v6\", \"4.7\")\n\n assert_satisfies(\"4.7.3\", \"4\")\n assert_satisfies(\"4.7.3v2\", \"4\")\n assert_satisfies(\"4.7v6\", \"4\")\n\n assert_does_not_satisfy(\"4.8.0\", \"4.9\")\n assert_does_not_satisfy(\"4.8\", \"4.9\")\n assert_does_not_satisfy(\"4\", \"4.9\")\n\n\ndef test_basic_version_satisfaction_in_lists():\n assert_satisfies([\"4.7.3\"], [\"4.7.3\"])\n\n assert_satisfies([\"4.7.3\"], [\"4.7\"])\n assert_satisfies([\"4.7.3v2\"], [\"4.7\"])\n assert_satisfies([\"4.7v6\"], [\"4.7\"])\n\n assert_satisfies([\"4.7.3\"], [\"4\"])\n assert_satisfies([\"4.7.3v2\"], [\"4\"])\n assert_satisfies([\"4.7v6\"], [\"4\"])\n\n assert_does_not_satisfy([\"4.8.0\"], [\"4.9\"])\n assert_does_not_satisfy([\"4.8\"], [\"4.9\"])\n assert_does_not_satisfy([\"4\"], [\"4.9\"])\n\n\ndef test_version_range_satisfaction():\n assert_satisfies(\"4.7b6\", \"4.3:4.7\")\n assert_satisfies(\"4.3.0\", \"4.3:4.7\")\n assert_satisfies(\"4.3.2\", \"4.3:4.7\")\n\n assert_does_not_satisfy(\"4.8.0\", \"4.3:4.7\")\n assert_does_not_satisfy(\"4.3\", \"4.4:4.7\")\n\n assert_satisfies(\"4.7b6\", \"4.3:4.7\")\n assert_does_not_satisfy(\"4.8.0\", \"4.3:4.7\")\n\n\ndef test_version_range_satisfaction_in_lists():\n assert_satisfies([\"4.7b6\"], [\"4.3:4.7\"])\n assert_satisfies([\"4.3.0\"], [\"4.3:4.7\"])\n assert_satisfies([\"4.3.2\"], [\"4.3:4.7\"])\n\n assert_does_not_satisfy([\"4.8.0\"], [\"4.3:4.7\"])\n assert_does_not_satisfy([\"4.3\"], [\"4.4:4.7\"])\n\n assert_satisfies([\"4.7b6\"], [\"4.3:4.7\"])\n assert_does_not_satisfy([\"4.8.0\"], [\"4.3:4.7\"])\n\n\ndef test_satisfaction_with_lists():\n assert_satisfies(\"4.7\", \"4.3, 4.6, 4.7\")\n assert_satisfies(\"4.7.3\", \"4.3, 4.6, 4.7\")\n assert_satisfies(\"4.6.5\", \"4.3, 4.6, 4.7\")\n assert_satisfies(\"4.6.5.2\", \"4.3, 4.6, 4.7\")\n\n assert_does_not_satisfy(\"4\", \"4.3, 4.6, 4.7\")\n assert_does_not_satisfy(\"4.8.0\", \"4.2, 4.3:4.7\")\n\n assert_satisfies(\"4.8.0\", \"4.2, 4.3:4.8\")\n assert_satisfies(\"4.8.2\", \"4.2, 4.3:4.8\")\n\n\ndef test_formatted_strings():\n versions = (\n \"1.2.3b\",\n \"1_2_3b\",\n \"1-2-3b\",\n \"1.2-3b\",\n \"1.2_3b\",\n \"1-2.3b\",\n \"1-2_3b\",\n \"1_2.3b\",\n \"1_2-3b\",\n )\n for item in versions:\n v = Version(item)\n assert v.dotted.string == \"1.2.3b\"\n assert v.dashed.string == \"1-2-3b\"\n assert v.underscored.string == \"1_2_3b\"\n assert v.joined.string == \"123b\"\n\n assert v.dotted.dashed.string == \"1-2-3b\"\n assert v.dotted.underscored.string == \"1_2_3b\"\n assert v.dotted.dotted.string == \"1.2.3b\"\n assert v.dotted.joined.string == \"123b\"\n\n\ndef test_dotted_numeric_string():\n assert Version(\"1a2b3\").dotted_numeric_string == \"1.0.2.0.3\"\n assert Version(\"1a2b3alpha4\").dotted_numeric_string == \"1.0.2.0.3.0.4\"\n\n\ndef test_up_to():\n v = Version(\"1.23-4_5b\")\n\n assert v.up_to(1).string == \"1\"\n assert v.up_to(2).string == \"1.23\"\n assert v.up_to(3).string == \"1.23-4\"\n assert v.up_to(4).string == \"1.23-4_5\"\n assert v.up_to(5).string == \"1.23-4_5b\"\n\n assert v.up_to(-1).string == \"1.23-4_5\"\n assert v.up_to(-2).string == \"1.23-4\"\n assert v.up_to(-3).string == \"1.23\"\n assert v.up_to(-4).string == \"1\"\n\n assert v.up_to(2).dotted.string == \"1.23\"\n assert v.up_to(2).dashed.string == \"1-23\"\n assert v.up_to(2).underscored.string == \"1_23\"\n assert v.up_to(2).joined.string == \"123\"\n\n assert v.dotted.up_to(2).string == \"1.23\" == v.up_to(2).dotted.string\n assert v.dashed.up_to(2).string == \"1-23\" == v.up_to(2).dashed.string\n assert v.underscored.up_to(2).string == \"1_23\"\n assert v.up_to(2).underscored.string == \"1_23\"\n\n assert v.up_to(2).up_to(1).string == \"1\"\n\n\ndef test_repr_and_str():\n def check_repr_and_str(vrs):\n a = Version(vrs)\n assert repr(a) == f'Version(\"{vrs}\")'\n b = eval(repr(a))\n assert a == b\n assert str(a) == vrs\n assert str(a) == str(b)\n\n check_repr_and_str(\"1.2.3\")\n check_repr_and_str(\"R2016a\")\n check_repr_and_str(\"R2016a.2-3_4\")\n\n\ndef test_str_and_hash_version_range():\n \"\"\"Test that precomputed string and hash values are consistent with computed ones.\"\"\"\n x = ver(\"1.2:3.4\")\n assert isinstance(x, ClosedOpenRange)\n # Test that precomputed str() and hash() are assigned\n assert x._string is not None and x._hash is not None\n _str = str(x)\n _hash = hash(x)\n assert x._string == _str and x._hash == _hash\n # Ensure computed values match precomputed ones\n x._string = None\n x._hash = None\n assert _str == str(x)\n assert _hash == hash(x)\n\n\n@pytest.mark.parametrize(\n \"version_str\", [\"1.2string3\", \"1.2-3xyz_4-alpha.5\", \"1.2beta\", \"1_x_rc-4\"]\n)\ndef test_stringify_version(version_str):\n v = Version(version_str)\n v.string = None\n assert str(v) == version_str\n\n v.string = None\n assert v.string == version_str\n\n\ndef test_len():\n a = Version(\"1.2.3.4\")\n assert len(a) == len(a.version[0])\n assert len(a) == 4\n b = Version(\"2018.0\")\n assert len(b) == 2\n\n\ndef test_get_item():\n a = Version(\"0.1_2-3\")\n assert isinstance(a[1], int)\n # Test slicing\n b = a[0:2]\n assert isinstance(b, StandardVersion)\n assert b == Version(\"0.1\")\n assert repr(b) == 'Version(\"0.1\")'\n assert str(b) == \"0.1\"\n b = a[0:3]\n assert isinstance(b, StandardVersion)\n assert b == Version(\"0.1_2\")\n assert repr(b) == 'Version(\"0.1_2\")'\n assert str(b) == \"0.1_2\"\n b = a[1:]\n assert isinstance(b, StandardVersion)\n assert b == Version(\"1_2-3\")\n assert repr(b) == 'Version(\"1_2-3\")'\n assert str(b) == \"1_2-3\"\n # Raise TypeError on tuples\n with pytest.raises(TypeError):\n b.__getitem__(1, 2)\n\n\ndef test_list_highest():\n vl = VersionList([\"=master\", \"=1.2.3\", \"=develop\", \"=3.4.5\", \"=foobar\"])\n assert vl.highest() == Version(\"develop\")\n assert vl.lowest() == Version(\"foobar\")\n assert vl.highest_numeric() == Version(\"3.4.5\")\n\n vl2 = VersionList([\"=master\", \"=develop\"])\n assert vl2.highest_numeric() is None\n assert vl2.preferred() == Version(\"develop\")\n assert vl2.lowest() == Version(\"master\")\n\n\n@pytest.mark.parametrize(\"version_str\", [\"foo 1.2.0\", \"!\", \"1!2\", \"=1.2.0\"])\ndef test_invalid_versions(version_str):\n \"\"\"Ensure invalid versions are rejected with a ValueError\"\"\"\n with pytest.raises(ValueError):\n Version(version_str)\n\n\ndef test_versions_from_git(git, mock_git_version_info, monkeypatch, mock_packages):\n repo_path, filename, commits = mock_git_version_info\n monkeypatch.setattr(\n spack.package_base.PackageBase, \"git\", pathlib.Path(repo_path).as_uri(), raising=False\n )\n\n for commit in commits:\n spec = spack.spec.Spec(\"git-test-commit@%s\" % commit)\n version: GitVersion = spec.version\n comparator = [str(v) if not isinstance(v, int) else v for v in version.ref_version]\n\n with working_dir(repo_path):\n git(\"checkout\", commit)\n\n with open(os.path.join(repo_path, filename), \"r\", encoding=\"utf-8\") as f:\n expected = f.read()\n\n assert str(comparator) == expected\n\n\n@pytest.mark.parametrize(\n \"commit_idx,expected_satisfies,expected_not_satisfies\",\n [\n # Spec based on earliest commit\n (-1, (\"@:0\",), (\"@1.0\",)),\n # Spec based on second commit (same as version 1.0)\n (-2, (\"@1.0\",), (\"@1.1:\",)),\n # Spec based on 4th commit (in timestamp order)\n (-4, (\"@1.1\", \"@1.0:1.2\"), tuple()),\n ],\n)\ndef test_git_hash_comparisons(\n mock_git_version_info,\n install_mockery,\n mock_packages,\n monkeypatch,\n commit_idx,\n expected_satisfies,\n expected_not_satisfies,\n):\n \"\"\"Check that hashes compare properly to versions\"\"\"\n repo_path, filename, commits = mock_git_version_info\n monkeypatch.setattr(\n spack.package_base.PackageBase, \"git\", pathlib.Path(repo_path).as_uri(), raising=False\n )\n\n spec = spack.concretize.concretize_one(\n spack.spec.Spec(f\"git-test-commit@{commits[commit_idx]}\")\n )\n for item in expected_satisfies:\n assert spec.satisfies(item)\n\n for item in expected_not_satisfies:\n assert not spec.satisfies(item)\n\n\ndef test_git_ref_comparisons(mock_git_version_info, install_mockery, mock_packages, monkeypatch):\n \"\"\"Check that hashes compare properly to versions\"\"\"\n repo_path, filename, commits = mock_git_version_info\n monkeypatch.setattr(\n spack.package_base.PackageBase, \"git\", pathlib.Path(repo_path).as_uri(), raising=False\n )\n\n # Spec based on tag v1.0\n spec_tag = spack.concretize.concretize_one(\"git-test-commit@git.v1.0\")\n assert spec_tag.satisfies(\"@1.0\")\n assert not spec_tag.satisfies(\"@1.1:\")\n assert str(spec_tag.version) == \"git.v1.0=1.0\"\n\n # Spec based on branch 1.x\n spec_branch = spack.concretize.concretize_one(\"git-test-commit@git.1.x\")\n assert spec_branch.satisfies(\"@1.2\")\n assert spec_branch.satisfies(\"@1.1:1.3\")\n assert str(spec_branch.version) == \"git.1.x=1.2\"\n\n\ndef test_git_branch_with_slash():\n class MockLookup(object):\n def get(self, ref):\n assert ref == \"feature/bar\"\n return \"1.2\", 0\n\n v = spack.version.from_string(\"git.feature/bar\")\n assert isinstance(v, GitVersion)\n v.attach_lookup(MockLookup())\n\n # Create a version range\n test_number_version = spack.version.from_string(\"1.2\")\n v.satisfies(test_number_version)\n\n serialized = VersionList([v]).to_dict()\n v_deserialized = VersionList.from_dict(serialized)\n assert v_deserialized[0].ref == \"feature/bar\"\n\n\n@pytest.mark.parametrize(\n \"string,git\",\n [\n (\"1.2.9\", False),\n (\"gitmain\", False),\n (\"git.foo\", True),\n (\"git.abcdabcdabcdabcdabcdabcdabcdabcdabcdabcd\", True),\n (\"abcdabcdabcdabcdabcdabcdabcdabcdabcdabcd\", True),\n ],\n)\ndef test_version_git_vs_base(string, git):\n assert is_git_version(string) == git\n assert isinstance(Version(string), GitVersion) == git\n\n\ndef test_version_range_nonempty():\n assert Version(\"1.2.9\") in VersionRange(\"1.2.0\", \"1.2\")\n assert Version(\"1.1.1\") in ver(\"1.0:1\")\n\n\ndef test_empty_version_range_raises():\n with pytest.raises(EmptyRangeError, match=\"2:1.0 is an empty range\"):\n assert VersionRange(\"2\", \"1.0\")\n with pytest.raises(EmptyRangeError, match=\"2:1.0 is an empty range\"):\n assert ver(\"2:1.0\")\n\n\ndef test_version_empty_slice():\n \"\"\"Check an empty slice to confirm get \"empty\" version instead of\n an IndexError (#25953).\n \"\"\"\n assert Version(\"1.\")[1:] == Version(\"\")\n\n\ndef test_version_wrong_idx_type():\n \"\"\"Ensure exception raised if attempt to use non-integer index.\"\"\"\n v = Version(\"1.1\")\n with pytest.raises(TypeError):\n v[\"0:\"]\n\n\n@pytest.mark.regression(\"29170\")\ndef test_version_range_satisfies_means_nonempty_intersection():\n x = VersionRange(\"3.7.0\", \"3\")\n y = VersionRange(\"3.6.0\", \"3.6.0\")\n assert not x.satisfies(y)\n assert not y.satisfies(x)\n\n\ndef test_version_list_with_range_and_concrete_version_is_not_concrete():\n v = VersionList([Version(\"3.1\"), VersionRange(Version(\"3.1.1\"), Version(\"3.1.2\"))])\n assert not v.concrete\n\n\n@pytest.mark.parametrize(\n \"git_ref, std_version\",\n ((\"foo\", \"develop\"), (\"a\" * 40, \"develop\"), (\"a\" * 40, None), (\"v1.2.0\", \"1.2.0\")),\n)\ndef test_git_versions_store_ref_requests(git_ref, std_version):\n \"\"\"\n User requested ref's should be known on creation\n Commit and standard version may not be known until concretization\n To be concrete a GitVersion must have a commit and standard version\n \"\"\"\n if std_version:\n vstring = f\"git.{git_ref}={std_version}\"\n else:\n vstring = git_ref\n\n v = Version(vstring)\n\n assert isinstance(v, GitVersion)\n assert v.ref == git_ref\n\n if std_version:\n assert v.std_version == Version(std_version)\n\n if v.is_commit:\n assert v.ref == v.commit_sha\n\n\n@pytest.mark.parametrize(\n \"vstring, eq_vstring, is_commit\",\n (\n (\"abc12\" * 8 + \"=develop\", \"develop\", True),\n (\"git.\" + \"abc12\" * 8 + \"=main\", \"main\", True),\n (\"a\" * 40 + \"=develop\", \"develop\", True),\n (\"b\" * 40 + \"=3.2\", \"3.2\", True),\n (\"git.foo=3.2\", \"3.2\", False),\n ),\n)\ndef test_git_ref_can_be_assigned_a_version(vstring, eq_vstring, is_commit):\n v = Version(vstring)\n v_equivalent = Version(eq_vstring)\n assert v.is_commit == is_commit\n assert not v._ref_lookup\n assert v_equivalent == v.ref_version\n\n\n@pytest.mark.parametrize(\n \"lhs_str,rhs_str,expected\",\n [\n # StandardVersion\n (\"4.7.3\", \"4.7.3\", (True, True, True)),\n (\"4.7.3\", \"4.7\", (True, True, False)),\n (\"4.7.3\", \"4\", (True, True, False)),\n (\"4.7.3\", \"4.8\", (False, False, False)),\n # GitVersion\n (f\"git.{'a' * 40}=develop\", \"develop\", (True, True, False)),\n (f\"git.{'a' * 40}=develop\", f\"git.{'a' * 40}=develop\", (True, True, True)),\n (f\"git.{'a' * 40}=develop\", f\"git.{'b' * 40}=develop\", (False, False, False)),\n ],\n)\ndef test_version_intersects_satisfies_semantic(lhs_str, rhs_str, expected):\n lhs, rhs = ver(lhs_str), ver(rhs_str)\n intersect, lhs_sat_rhs, rhs_sat_lhs = expected\n\n assert lhs.intersects(rhs) is intersect\n assert lhs.intersects(rhs) is rhs.intersects(lhs)\n assert lhs.satisfies(rhs) is lhs_sat_rhs\n assert rhs.satisfies(lhs) is rhs_sat_lhs\n\n\n@pytest.mark.parametrize(\n \"spec_str,tested_intersects,tested_satisfies\",\n [\n (\n \"git-test-commit@git.1.x\",\n [(\"@:2\", True), (\"@:1\", True), (\"@:0\", False), (\"@1.3:\", False)],\n [(\"@:2\", True), (\"@:1\", True), (\"@:0\", False), (\"@1.3:\", False)],\n ),\n (\n \"git-test-commit@git.v2.0\",\n [(\"@:2\", True), (\"@:1\", False), (\"@:0\", False), (\"@1.3:\", True)],\n [(\"@:2\", True), (\"@:1\", False), (\"@:0\", False), (\"@1.3:\", True)],\n ),\n ],\n)\ndef test_git_versions_without_explicit_reference(\n spec_str,\n tested_intersects,\n tested_satisfies,\n mock_git_version_info,\n mock_packages,\n monkeypatch,\n):\n repo_path, filename, commits = mock_git_version_info\n monkeypatch.setattr(\n spack.package_base.PackageBase, \"git\", pathlib.Path(repo_path).as_uri(), raising=False\n )\n spec = spack.spec.Spec(spec_str)\n\n for test_str, expected in tested_intersects:\n assert spec.intersects(test_str) is expected, test_str\n\n for test_str, expected in tested_intersects:\n assert spec.intersects(test_str) is expected, test_str\n\n\ndef test_total_order_versions_and_ranges():\n # The set of version ranges and individual versions are comparable, which is used in\n # VersionList. The comparison across types is based on default version comparison\n # of StandardVersion, GitVersion.ref_version, and ClosedOpenRange.lo.\n\n # StandardVersion / GitVersion (at equal ref version)\n assert_ver_lt(\"=1.2\", \"git.ref=1.2\")\n assert_ver_gt(\"git.ref=1.2\", \"=1.2\")\n\n # StandardVersion / GitVersion (at different ref versions)\n assert_ver_lt(\"git.ref=1.2\", \"=1.3\")\n assert_ver_gt(\"=1.3\", \"git.ref=1.2\")\n assert_ver_lt(\"=1.2\", \"git.ref=1.3\")\n assert_ver_gt(\"git.ref=1.3\", \"=1.2\")\n\n # GitVersion / ClosedOpenRange (at equal ref/lo version)\n assert_ver_lt(\"git.ref=1.2\", \"1.2\")\n assert_ver_gt(\"1.2\", \"git.ref=1.2\")\n\n # GitVersion / ClosedOpenRange (at different ref/lo version)\n assert_ver_lt(\"git.ref=1.2\", \"1.3\")\n assert_ver_gt(\"1.3\", \"git.ref=1.2\")\n assert_ver_lt(\"1.2\", \"git.ref=1.3\")\n assert_ver_gt(\"git.ref=1.3\", \"1.2\")\n\n # StandardVersion / ClosedOpenRange (at equal lo version)\n assert_ver_lt(\"=1.2\", \"1.2\")\n assert_ver_gt(\"1.2\", \"=1.2\")\n\n # StandardVersion / ClosedOpenRange (at different lo version)\n assert_ver_lt(\"=1.2\", \"1.3\")\n assert_ver_gt(\"1.3\", \"=1.2\")\n assert_ver_lt(\"1.2\", \"=1.3\")\n assert_ver_gt(\"=1.3\", \"1.2\")\n\n\ndef test_git_version_accessors():\n \"\"\"Test whether iteration, indexing, slicing, dotted, dashed, and underscored works for\n GitVersion.\"\"\"\n v = GitVersion(\"my_branch=1.2-3\")\n assert [x for x in v] == [1, 2, 3]\n assert v[0] == 1\n assert v[1] == 2\n assert v[2] == 3\n assert v[0:2] == Version(\"1.2\")\n assert v[0:10] == Version(\"1.2.3\")\n assert str(v.dotted) == \"1.2.3\"\n assert str(v.dashed) == \"1-2-3\"\n assert str(v.underscored) == \"1_2_3\"\n assert v.up_to(1) == Version(\"1\")\n assert v.up_to(2) == Version(\"1.2\")\n assert len(v) == 3\n assert not v.isdevelop()\n assert GitVersion(\"my_branch=develop\").isdevelop()\n\n\ndef test_boolness_of_versions():\n # We do implement __len__, but at the end of the day versions are used as elements in\n # the first place, not as lists of version components. So VersionList(...).concrete\n # should be truthy even when there are no version components.\n assert bool(Version(\"1.2\"))\n assert bool(Version(\"1.2\").up_to(0))\n\n # bool(GitVersion) shouldn't trigger a ref lookup.\n assert bool(GitVersion(\"a\" * 40))\n\n\ndef test_version_list_normalization():\n # Git versions and ordinary versions can live together in a VersionList\n assert len(VersionList([\"=1.2\", \"ref=1.2\"])) == 2\n\n # But when a range is added, the only disjoint bit is the range.\n assert VersionList([\"=1.2\", \"ref=1.2\", \"ref=1.3\", \"1.2:1.3\"]) == VersionList([\"1.2:1.3\"])\n\n # Also test normalization when using ver.\n assert ver(\"=1.0,ref=1.0,1.0:2.0\") == ver([\"1.0:2.0\"])\n assert ver(\"=1.0,1.0:2.0,ref=1.0\") == ver([\"1.0:2.0\"])\n assert ver(\"1.0:2.0,=1.0,ref=1.0\") == ver([\"1.0:2.0\"])\n\n\ndef test_version_list_connected_union_of_disjoint_ranges():\n # Make sure that we also simplify lists of ranges if their intersection is empty, but their\n # union is connected.\n assert ver(\"1.0:2.0,2.1,2.2:3,4:6\") == ver([\"1.0:6\"])\n assert ver(\"1.0:1.2,1.3:2\") == ver(\"1.0:1.5,1.6:2\")\n\n\n@pytest.mark.parametrize(\"version\", [\"=1.2\", \"git.ref=1.2\", \"1.2\"])\ndef test_version_comparison_with_list_fails(version):\n vlist = VersionList([\"=1.3\"])\n\n with pytest.raises(TypeError):\n version < vlist\n\n with pytest.raises(TypeError):\n vlist < version\n\n with pytest.raises(TypeError):\n version <= vlist\n\n with pytest.raises(TypeError):\n vlist <= version\n\n with pytest.raises(TypeError):\n version >= vlist\n\n with pytest.raises(TypeError):\n vlist >= version\n\n with pytest.raises(TypeError):\n version > vlist\n\n with pytest.raises(TypeError):\n vlist > version\n\n\ndef test_inclusion_upperbound():\n is_specific = spack.spec.Spec(\"x@=1.2\")\n is_range = spack.spec.Spec(\"x@1.2\")\n upperbound = spack.spec.Spec(\"x@:1.2.0\")\n\n # The exact version is included in the range\n assert is_specific.satisfies(upperbound)\n\n # But the range 1.2:1.2 is not, since it includes for example 1.2.1\n assert not is_range.satisfies(upperbound)\n\n # They do intersect of course.\n assert is_specific.intersects(upperbound) and is_range.intersects(upperbound)\n\n\n@pytest.mark.not_on_windows(\"Not supported on Windows (yet)\")\ndef test_git_version_repo_attached_after_serialization(\n mock_git_version_info, mock_packages, config, monkeypatch\n):\n \"\"\"Test that a GitVersion instance can be serialized and deserialized\n without losing its repository reference.\n \"\"\"\n repo_path, _, commits = mock_git_version_info\n monkeypatch.setattr(\n spack.package_base.PackageBase, \"git\", \"file://%s\" % repo_path, raising=False\n )\n spec = spack.concretize.concretize_one(f\"git-test-commit@{commits[-2]}\")\n\n # Before serialization, the repo is attached\n assert spec.satisfies(\"@1.0\")\n\n # After serialization, the repo is still attached\n assert spack.spec.Spec.from_dict(spec.to_dict()).satisfies(\"@1.0\")\n\n\n@pytest.mark.not_on_windows(\"Not supported on Windows (yet)\")\ndef test_resolved_git_version_is_shown_in_str(\n mock_git_version_info, mock_packages, config, monkeypatch\n):\n \"\"\"Test that a GitVersion from a commit without a user supplied version is printed\n as =, and not just .\"\"\"\n repo_path, _, commits = mock_git_version_info\n monkeypatch.setattr(\n spack.package_base.PackageBase, \"git\", \"file://%s\" % repo_path, raising=False\n )\n commit = commits[-3]\n spec = spack.concretize.concretize_one(f\"git-test-commit@{commit}\")\n\n assert spec.version.satisfies(ver(\"1.0\"))\n assert str(spec.version) == f\"{commit}=1.0-git.1\"\n\n\ndef test_unresolvable_git_versions_error(config, mock_packages):\n \"\"\"Test that VersionLookupError is raised when a git prop is not set on a package.\"\"\"\n with pytest.raises(VersionLookupError):\n # The package exists, but does not have a git property set. When dereferencing\n # the version, we should get VersionLookupError, not a generic AttributeError.\n spack.spec.Spec(f\"git-test-commit@{'a' * 40}\").version.ref_version\n\n\n@pytest.mark.parametrize(\n \"tag,expected\",\n [\n (\"v100.2.3\", \"100.2.3\"),\n (\"v1.2.3\", \"1.2.3\"),\n (\"v1.2.3-pre.release+build.1\", \"1.2.3-pre.release+build.1\"),\n (\"v1.2.3+build.1\", \"1.2.3+build.1\"),\n (\"v1.2.3+build_1\", None),\n (\"v1.2.3-pre.release\", \"1.2.3-pre.release\"),\n (\"v1.2.3-pre_release\", None),\n (\"1.2.3\", \"1.2.3\"),\n (\"1.2.3.\", None),\n ],\n)\ndef test_semver_regex(tag, expected):\n result = SEMVER_REGEX.search(tag)\n if expected is None:\n assert result is None\n else:\n assert result.group() == expected\n" }, { "path": "lib/spack/spack/test/views.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport pathlib\n\nimport pytest\n\nimport spack.concretize\nfrom spack.directory_layout import DirectoryLayout\nfrom spack.filesystem_view import SimpleFilesystemView, YamlFilesystemView\nfrom spack.installer import PackageInstaller\nfrom spack.spec import Spec\n\n\ndef test_remove_extensions_ordered(install_mockery, mock_fetch, tmp_path: pathlib.Path):\n view_dir = str(tmp_path / \"view\")\n layout = DirectoryLayout(view_dir)\n view = YamlFilesystemView(view_dir, layout)\n e2 = spack.concretize.concretize_one(\"extension2\")\n PackageInstaller([e2.package], explicit=True).install()\n view.add_specs(e2)\n\n e1 = e2[\"extension1\"]\n view.remove_specs(e1, e2)\n\n\n@pytest.mark.regression(\"32456\")\ndef test_view_with_spec_not_contributing_files(mock_packages, tmp_path: pathlib.Path):\n view_dir = str(tmp_path / \"view\")\n os.mkdir(view_dir)\n\n layout = DirectoryLayout(view_dir)\n view = SimpleFilesystemView(view_dir, layout)\n\n a = Spec(\"pkg-a\")\n b = Spec(\"pkg-b\")\n a.set_prefix(str(tmp_path / \"a\"))\n b.set_prefix(str(tmp_path / \"b\"))\n a._mark_concrete()\n b._mark_concrete()\n\n # Create directory structure for a and b, and view\n os.makedirs(a.prefix.subdir)\n os.makedirs(b.prefix.subdir)\n os.makedirs(os.path.join(a.prefix, \".spack\"))\n os.makedirs(os.path.join(b.prefix, \".spack\"))\n\n # Add files to b's prefix, but not to a's\n with open(b.prefix.file, \"w\", encoding=\"utf-8\") as f:\n f.write(\"file 1\")\n\n with open(b.prefix.subdir.file, \"w\", encoding=\"utf-8\") as f:\n f.write(\"file 2\")\n\n # In previous versions of Spack we incorrectly called add_files_to_view\n # with b's merge map. It shouldn't be called at all, since a has no\n # files to add to the view.\n def pkg_a_add_files_to_view(view, merge_map, skip_if_exists=True):\n assert False, \"There shouldn't be files to add\"\n\n a.package.add_files_to_view = pkg_a_add_files_to_view\n\n # Create view and see if files are linked.\n view.add_specs(a, b)\n assert os.path.lexists(os.path.join(view_dir, \"file\"))\n assert os.path.lexists(os.path.join(view_dir, \"subdir\", \"file\"))\n" }, { "path": "lib/spack/spack/test/web.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport collections\nimport email.message\nimport os\nimport pathlib\nimport pickle\nimport ssl\nimport urllib.request\nfrom typing import Dict\n\nimport pytest\n\nimport spack.config\nimport spack.llnl.util.tty as tty\nimport spack.mirrors.mirror\nimport spack.paths\nimport spack.url\nimport spack.util.s3\nimport spack.util.url as url_util\nimport spack.util.web\nfrom spack.llnl.util.filesystem import working_dir\nfrom spack.version import Version\n\n\ndef _create_url(relative_url):\n web_data_path = os.path.join(spack.paths.test_path, \"data\", \"web\")\n return url_util.path_to_file_url(os.path.join(web_data_path, relative_url))\n\n\nroot = _create_url(\"index.html\")\nroot_tarball = _create_url(\"foo-0.0.0.tar.gz\")\npage_1 = _create_url(\"1.html\")\npage_2 = _create_url(\"2.html\")\npage_3 = _create_url(\"3.html\")\npage_4 = _create_url(\"4.html\")\n\nroot_with_fragment = _create_url(\"index_with_fragment.html\")\nroot_with_javascript = _create_url(\"index_with_javascript.html\")\n\n\n@pytest.mark.parametrize(\n \"depth,expected_found,expected_not_found,expected_text\",\n [\n (\n 0,\n {\"pages\": [root], \"links\": [page_1]},\n {\"pages\": [page_1, page_2, page_3, page_4], \"links\": [root, page_2, page_3, page_4]},\n {root: \"This is the root page.\"},\n ),\n (\n 1,\n {\"pages\": [root, page_1], \"links\": [page_1, page_2]},\n {\"pages\": [page_2, page_3, page_4], \"links\": [root, page_3, page_4]},\n {root: \"This is the root page.\", page_1: \"This is page 1.\"},\n ),\n (\n 2,\n {\"pages\": [root, page_1, page_2], \"links\": [page_1, page_2, page_3, page_4]},\n {\"pages\": [page_3, page_4], \"links\": [root]},\n {root: \"This is the root page.\", page_1: \"This is page 1.\", page_2: \"This is page 2.\"},\n ),\n (\n 3,\n {\n \"pages\": [root, page_1, page_2, page_3, page_4],\n \"links\": [root, page_1, page_2, page_3, page_4],\n },\n {\"pages\": [], \"links\": []},\n {\n root: \"This is the root page.\",\n page_1: \"This is page 1.\",\n page_2: \"This is page 2.\",\n page_3: \"This is page 3.\",\n page_4: \"This is page 4.\",\n },\n ),\n ],\n)\ndef test_spider(depth, expected_found, expected_not_found, expected_text):\n pages, links = spack.util.web.spider(root, depth=depth)\n\n for page in expected_found[\"pages\"]:\n assert page in pages\n\n for page in expected_not_found[\"pages\"]:\n assert page not in pages\n\n for link in expected_found[\"links\"]:\n assert link in links\n\n for link in expected_not_found[\"links\"]:\n assert link not in links\n\n for page, text in expected_text.items():\n assert text in pages[page]\n\n\ndef test_spider_no_response(monkeypatch):\n # Mock the absence of a response\n monkeypatch.setattr(spack.util.web, \"read_from_url\", lambda x, y: (None, None, None))\n pages, links, _, _ = spack.util.web._spider(root, collect_nested=False, _visited=set())\n assert not pages and not links\n\n\ndef test_find_versions_of_archive_0():\n versions = spack.url.find_versions_of_archive(root_tarball, root, list_depth=0)\n assert Version(\"0.0.0\") in versions\n\n\ndef test_find_versions_of_archive_1():\n versions = spack.url.find_versions_of_archive(root_tarball, root, list_depth=1)\n assert Version(\"0.0.0\") in versions\n assert Version(\"1.0.0\") in versions\n\n\ndef test_find_versions_of_archive_2():\n versions = spack.url.find_versions_of_archive(root_tarball, root, list_depth=2)\n assert Version(\"0.0.0\") in versions\n assert Version(\"1.0.0\") in versions\n assert Version(\"2.0.0\") in versions\n\n\ndef test_find_exotic_versions_of_archive_2():\n versions = spack.url.find_versions_of_archive(root_tarball, root, list_depth=2)\n # up for grabs to make this better.\n assert Version(\"2.0.0b2\") in versions\n\n\ndef test_find_versions_of_archive_3():\n versions = spack.url.find_versions_of_archive(root_tarball, root, list_depth=3)\n assert Version(\"0.0.0\") in versions\n assert Version(\"1.0.0\") in versions\n assert Version(\"2.0.0\") in versions\n assert Version(\"3.0\") in versions\n assert Version(\"4.5\") in versions\n\n\ndef test_find_exotic_versions_of_archive_3():\n versions = spack.url.find_versions_of_archive(root_tarball, root, list_depth=3)\n assert Version(\"2.0.0b2\") in versions\n assert Version(\"3.0a1\") in versions\n assert Version(\"4.5-rc5\") in versions\n\n\ndef test_find_versions_of_archive_with_fragment():\n versions = spack.url.find_versions_of_archive(root_tarball, root_with_fragment, list_depth=0)\n assert Version(\"5.0.0\") in versions\n\n\ndef test_find_versions_of_archive_with_javascript():\n versions = spack.url.find_versions_of_archive(root_tarball, root_with_javascript, list_depth=0)\n assert Version(\"5.0.0\") in versions\n\n\ndef test_get_header():\n headers = {\"Content-type\": \"text/plain\"}\n\n # looking up headers should just work like a plain dict\n # lookup when there is an entry with the right key\n assert spack.util.web.get_header(headers, \"Content-type\") == \"text/plain\"\n\n # looking up headers should still work if there is a fuzzy match\n assert spack.util.web.get_header(headers, \"contentType\") == \"text/plain\"\n\n # ...unless there is an exact match for the \"fuzzy\" spelling.\n headers[\"contentType\"] = \"text/html\"\n assert spack.util.web.get_header(headers, \"contentType\") == \"text/html\"\n\n # If lookup has to fallback to fuzzy matching and there are more than one\n # fuzzy match, the result depends on the internal ordering of the given\n # mapping\n headers = collections.OrderedDict()\n headers[\"Content-type\"] = \"text/plain\"\n headers[\"contentType\"] = \"text/html\"\n\n assert spack.util.web.get_header(headers, \"CONTENT_TYPE\") == \"text/plain\"\n del headers[\"Content-type\"]\n assert spack.util.web.get_header(headers, \"CONTENT_TYPE\") == \"text/html\"\n\n # Same as above, but different ordering\n headers = collections.OrderedDict()\n headers[\"contentType\"] = \"text/html\"\n headers[\"Content-type\"] = \"text/plain\"\n\n assert spack.util.web.get_header(headers, \"CONTENT_TYPE\") == \"text/html\"\n del headers[\"contentType\"]\n assert spack.util.web.get_header(headers, \"CONTENT_TYPE\") == \"text/plain\"\n\n # If there isn't even a fuzzy match, raise KeyError\n with pytest.raises(KeyError):\n spack.util.web.get_header(headers, \"ContentLength\")\n\n\ndef test_etag_parser():\n # This follows rfc7232 to some extent, relaxing the quote requirement.\n assert spack.util.web.parse_etag('\"abcdef\"') == \"abcdef\"\n assert spack.util.web.parse_etag(\"abcdef\") == \"abcdef\"\n\n # No empty tags\n assert spack.util.web.parse_etag(\"\") is None\n\n # No quotes or spaces allowed\n assert spack.util.web.parse_etag('\"abcdef\"ghi\"') is None\n assert spack.util.web.parse_etag('\"abc def\"') is None\n assert spack.util.web.parse_etag(\"abc def\") is None\n\n\ndef test_list_url(tmp_path: pathlib.Path):\n testpath = str(tmp_path)\n testpath_url = url_util.path_to_file_url(testpath)\n\n os.mkdir(os.path.join(testpath, \"dir\"))\n\n with open(os.path.join(testpath, \"file-0.txt\"), \"w\", encoding=\"utf-8\"):\n pass\n with open(os.path.join(testpath, \"file-1.txt\"), \"w\", encoding=\"utf-8\"):\n pass\n with open(os.path.join(testpath, \"file-2.txt\"), \"w\", encoding=\"utf-8\"):\n pass\n\n with open(os.path.join(testpath, \"dir\", \"another-file.txt\"), \"w\", encoding=\"utf-8\"):\n pass\n\n list_url = lambda recursive: list(\n sorted(spack.util.web.list_url(testpath_url, recursive=recursive))\n )\n\n assert list_url(False) == [\"file-0.txt\", \"file-1.txt\", \"file-2.txt\"]\n\n assert list_url(True) == [\"dir/another-file.txt\", \"file-0.txt\", \"file-1.txt\", \"file-2.txt\"]\n\n\nclass MockPages:\n def search(self, *args, **kwargs):\n return [{\"Key\": \"keyone\"}, {\"Key\": \"keytwo\"}, {\"Key\": \"keythree\"}]\n\n\nclass MockPaginator:\n def paginate(self, *args, **kwargs):\n return MockPages()\n\n\nclass MockClientError(Exception):\n def __init__(self):\n self.response = {\n \"Error\": {\"Code\": \"NoSuchKey\"},\n \"ResponseMetadata\": {\"HTTPStatusCode\": 404},\n }\n\n\nclass MockS3Client:\n def get_paginator(self, *args, **kwargs):\n return MockPaginator()\n\n def delete_objects(self, *args, **kwargs):\n return {\n \"Errors\": [{\"Key\": \"keyone\", \"Message\": \"Access Denied\"}],\n \"Deleted\": [{\"Key\": \"keytwo\"}, {\"Key\": \"keythree\"}],\n }\n\n def delete_object(self, *args, **kwargs):\n pass\n\n def get_object(self, Bucket=None, Key=None):\n self.ClientError = MockClientError\n if Bucket == \"my-bucket\" and Key == \"subdirectory/my-file\":\n return {\"ResponseMetadata\": {\"HTTPHeaders\": {}}}\n raise self.ClientError\n\n def head_object(self, Bucket=None, Key=None):\n self.ClientError = MockClientError\n if Bucket == \"my-bucket\" and Key == \"subdirectory/my-file\":\n return {\"ResponseMetadata\": {\"HTTPHeaders\": {}}}\n raise self.ClientError\n\n\ndef test_gather_s3_information(monkeypatch):\n mirror = spack.mirrors.mirror.Mirror(\n {\n \"fetch\": {\n \"access_token\": \"AAAAAAA\",\n \"profile\": \"SPacKDeV\",\n \"access_pair\": (\"SPA\", \"CK\"),\n \"endpoint_url\": \"https://127.0.0.1:8888\",\n },\n \"push\": {\n \"access_token\": \"AAAAAAA\",\n \"profile\": \"SPacKDeV\",\n \"access_pair\": (\"SPA\", \"CK\"),\n \"endpoint_url\": \"https://127.0.0.1:8888\",\n },\n }\n )\n\n session_args, client_args = spack.util.s3.get_mirror_s3_connection_info(mirror, \"push\")\n\n # Session args are used to create the S3 Session object\n assert \"aws_session_token\" in session_args\n assert session_args.get(\"aws_session_token\") == \"AAAAAAA\"\n assert \"aws_access_key_id\" in session_args\n assert session_args.get(\"aws_access_key_id\") == \"SPA\"\n assert \"aws_secret_access_key\" in session_args\n assert session_args.get(\"aws_secret_access_key\") == \"CK\"\n assert \"profile_name\" in session_args\n assert session_args.get(\"profile_name\") == \"SPacKDeV\"\n\n # In addition to the session object, use the client_args to create the s3\n # Client object\n assert \"endpoint_url\" in client_args\n\n\ndef test_remove_s3_url(monkeypatch, capfd):\n fake_s3_url = \"s3://my-bucket/subdirectory/mirror\"\n\n def get_s3_session(url, method=\"fetch\"):\n return MockS3Client()\n\n monkeypatch.setattr(spack.util.web, \"get_s3_session\", get_s3_session)\n\n current_debug_level = tty.debug_level()\n tty.set_debug(1)\n\n spack.util.web.remove_url(fake_s3_url, recursive=True)\n err = capfd.readouterr()[1]\n\n tty.set_debug(current_debug_level)\n\n assert \"Failed to delete keyone (Access Denied)\" in err\n assert \"Deleted keythree\" in err\n assert \"Deleted keytwo\" in err\n\n\ndef test_s3_url_exists(monkeypatch):\n def get_s3_session(url, method=\"fetch\"):\n return MockS3Client()\n\n monkeypatch.setattr(spack.util.s3, \"get_s3_session\", get_s3_session)\n\n fake_s3_url_exists = \"s3://my-bucket/subdirectory/my-file\"\n assert spack.util.web.url_exists(fake_s3_url_exists)\n\n fake_s3_url_does_not_exist = \"s3://my-bucket/subdirectory/my-notfound-file\"\n assert not spack.util.web.url_exists(fake_s3_url_does_not_exist)\n\n\ndef test_s3_url_parsing():\n assert spack.util.s3._parse_s3_endpoint_url(\"example.com\") == \"https://example.com\"\n assert spack.util.s3._parse_s3_endpoint_url(\"http://example.com\") == \"http://example.com\"\n\n\ndef test_detailed_http_error_pickle(tmp_path: pathlib.Path):\n (tmp_path / \"response\").write_text(\"response\")\n\n headers = email.message.Message()\n headers.add_header(\"Content-Type\", \"text/plain\")\n\n # Use a temporary file object as a response body\n with open(str(tmp_path / \"response\"), \"rb\") as f:\n error = spack.util.web.DetailedHTTPError(\n urllib.request.Request(\"http://example.com\"), 404, \"Not Found\", headers, f\n )\n\n deserialized = pickle.loads(pickle.dumps(error))\n\n assert isinstance(deserialized, spack.util.web.DetailedHTTPError)\n assert deserialized.code == 404\n assert deserialized.filename == \"http://example.com\"\n assert deserialized.reason == \"Not Found\"\n assert str(deserialized.info()) == str(headers)\n assert str(deserialized) == str(error)\n\n\n@pytest.fixture()\ndef ssl_scrubbed_env(mutable_config, monkeypatch):\n \"\"\"clear out environment variables that could give false positives for SSL Cert tests\"\"\"\n monkeypatch.delenv(\"SSL_CERT_FILE\", raising=False)\n monkeypatch.delenv(\"SSL_CERT_DIR\", raising=False)\n monkeypatch.delenv(\"CURL_CA_BUNDLE\", raising=False)\n spack.config.set(\"config:verify_ssl\", True)\n\n\n@pytest.mark.parametrize(\n \"cert_path,cert_creator\",\n [\n pytest.param(\n lambda base_path: os.path.join(base_path, \"mock_cert.crt\"),\n lambda cert_path: open(cert_path, \"w\", encoding=\"utf-8\").close(),\n id=\"cert_file\",\n ),\n pytest.param(\n lambda base_path: os.path.join(base_path, \"mock_cert\"),\n lambda cert_path: os.mkdir(cert_path),\n id=\"cert_directory\",\n ),\n ],\n)\ndef test_ssl_urllib(\n cert_path, cert_creator, tmp_path: pathlib.Path, ssl_scrubbed_env, mutable_config, monkeypatch\n):\n \"\"\"\n create a proposed cert type and then verify that they exist inside ssl's checks\n \"\"\"\n spack.config.set(\"config:url_fetch_method\", \"urllib\")\n\n def mock_verify_locations(self, cafile, capath, cadata):\n \"\"\"overwrite ssl's verification to simply check for valid file/path\"\"\"\n assert cafile or capath\n if cafile:\n assert os.path.isfile(cafile)\n if capath:\n assert os.path.isdir(capath)\n\n monkeypatch.setattr(ssl.SSLContext, \"load_verify_locations\", mock_verify_locations)\n\n with working_dir(str(tmp_path)):\n mock_cert = cert_path(str(tmp_path))\n cert_creator(mock_cert)\n spack.config.set(\"config:ssl_certs\", mock_cert)\n\n assert mock_cert == spack.config.get(\"config:ssl_certs\", None)\n\n ssl_context = spack.util.web.ssl_create_default_context()\n assert ssl_context.verify_mode == ssl.CERT_REQUIRED\n\n\n@pytest.mark.parametrize(\"cert_exists\", [True, False], ids=[\"exists\", \"missing\"])\ndef test_ssl_curl_cert_file(\n cert_exists, tmp_path: pathlib.Path, ssl_scrubbed_env, mutable_config, monkeypatch\n):\n \"\"\"\n Assure that if a valid cert file is specified curl executes\n with CURL_CA_BUNDLE in the env\n \"\"\"\n spack.config.set(\"config:url_fetch_method\", \"curl\")\n with working_dir(str(tmp_path)):\n mock_cert = str(tmp_path / \"mock_cert.crt\")\n spack.config.set(\"config:ssl_certs\", mock_cert)\n if cert_exists:\n open(mock_cert, \"w\", encoding=\"utf-8\").close()\n assert os.path.isfile(mock_cert)\n curl = spack.util.web.require_curl()\n\n # arbitrary call to query the run env\n dump_env: Dict[str, str] = {}\n curl(\"--help\", output=str, _dump_env=dump_env)\n\n if cert_exists:\n assert dump_env[\"CURL_CA_BUNDLE\"] == mock_cert\n else:\n assert \"CURL_CA_BUNDLE\" not in dump_env\n\n\n@pytest.mark.parametrize(\n \"error_code,num_errors,max_retries,expect_failure\",\n [\n (500, 2, 5, False), # transient, enough retries\n (500, 2, 2, True), # transient, not enough retries\n (429, 2, 5, False), # rate limit, enough retries\n (404, 1, 5, True), # not transient, never retried\n ],\n)\ndef test_retry_on_transient_error(error_code, num_errors, max_retries, expect_failure):\n import urllib.error\n\n call_count = 0\n sleep_times = []\n\n def flaky_func():\n nonlocal call_count\n call_count += 1\n if call_count <= num_errors:\n raise urllib.error.HTTPError(\n url=\"https://example.com\", code=error_code, msg=\"err\", hdrs={}, fp=None\n )\n return \"ok\"\n\n retrying = spack.util.web.retry_on_transient_error(\n flaky_func, retries=max_retries, sleep=sleep_times.append\n )\n\n if expect_failure:\n with pytest.raises(urllib.error.HTTPError):\n retrying()\n else:\n assert retrying() == \"ok\"\n assert sleep_times == [2**i for i in range(num_errors)]\n\n\ndef test_retry_on_transient_error_non_oserror():\n \"\"\"Non-OSError exceptions with transient names (e.g. botocore) should be retried.\"\"\"\n\n class ResponseStreamingError(Exception):\n pass\n\n call_count = 0\n sleep_times = []\n\n def flaky_func():\n nonlocal call_count\n call_count += 1\n if call_count <= 2:\n raise ResponseStreamingError(\"IncompleteRead\")\n return \"ok\"\n\n retrying = spack.util.web.retry_on_transient_error(\n flaky_func, retries=5, sleep=sleep_times.append\n )\n\n assert retrying() == \"ok\"\n assert call_count == 3\n assert sleep_times == [1, 2]\n" }, { "path": "lib/spack/spack/tokenize.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"This module provides building blocks for tokenizing strings. Users can define tokens by\ninheriting from TokenBase and defining tokens as ordered enum members. The Tokenizer class can then\nbe used to iterate over tokens in a string.\"\"\"\n\nimport enum\nimport re\nfrom typing import Generator, Match, Optional, Type\n\n\nclass TokenBase(enum.Enum):\n \"\"\"Base class for an enum type with a regex value\"\"\"\n\n def __new__(cls, *args, **kwargs):\n value = len(cls.__members__) + 1\n obj = object.__new__(cls)\n obj._value_ = value\n return obj\n\n def __init__(self, regex):\n self.regex = regex\n\n def __str__(self):\n return f\"{self._name_}\"\n\n\nclass Token:\n \"\"\"Represents tokens; generated from input by lexer and fed to parse().\"\"\"\n\n __slots__ = \"kind\", \"value\", \"start\", \"end\", \"subvalues\"\n\n def __init__(self, kind: TokenBase, value: str, start: int = 0, end: int = 0, **kwargs):\n self.kind = kind\n self.value = value\n self.start = start\n self.end = end\n self.subvalues = kwargs if kwargs else None\n\n def __repr__(self):\n return str(self)\n\n def __str__(self):\n parts = [self.kind, self.value]\n if self.subvalues:\n parts += [self.subvalues]\n return f\"({', '.join(f'`{p}`' for p in parts)})\"\n\n def __eq__(self, other):\n return (\n self.kind == other.kind\n and self.value == other.value\n and self.subvalues == other.subvalues\n )\n\n\ndef token_match_regex(token: TokenBase):\n \"\"\"Generate a regular expression that matches the provided token and its subvalues.\n\n This will extract named capture groups from the provided regex and prefix them with\n token name, so they can coexist together in a larger, joined regular expression.\n\n Returns:\n A regex with a capture group for the token and rewritten capture groups for any subvalues.\n\n \"\"\"\n pairs = []\n\n def replace(m):\n subvalue_name = m.group(1)\n token_prefixed_subvalue_name = f\"{token.name}_{subvalue_name}\"\n pairs.append((subvalue_name, token_prefixed_subvalue_name))\n return f\"(?P<{token_prefixed_subvalue_name}>\"\n\n # rewrite all subvalue capture groups so they're prefixed with the token name\n rewritten_token_regex = re.sub(r\"\\(\\?P<([^>]+)>\", replace, token.regex)\n\n # construct a regex that matches the token as a whole *and* the subvalue capture groups\n token_regex = f\"(?P<{token}>{rewritten_token_regex})\"\n return token_regex, pairs\n\n\nclass Tokenizer:\n def __init__(self, tokens: Type[TokenBase]):\n self.tokens = tokens\n\n # tokens can have named subexpressions, if their regexes define named capture groups.\n # record this so we can associate them with the token\n self.token_subvalues = {}\n\n parts = []\n for token in tokens:\n token_regex, pairs = token_match_regex(token)\n parts.append(token_regex)\n if pairs:\n self.token_subvalues[token.name] = pairs\n\n self.regex = re.compile(\"|\".join(parts))\n\n def tokenize(self, text: str) -> Generator[Token, None, None]:\n if not text:\n return\n\n scanner = self.regex.scanner(text) # type: ignore[attr-defined]\n m: Optional[Match] = None\n for m in iter(scanner.match, None):\n # The following two assertions are to help mypy\n msg = (\n \"unexpected value encountered during parsing. Please submit a bug report \"\n \"at https://github.com/spack/spack/issues/new/choose\"\n )\n assert m is not None, msg\n assert m.lastgroup is not None, msg\n\n token = Token(self.tokens.__members__[m.lastgroup], m.group(), m.start(), m.end())\n\n # add any subvalues to the token\n subvalues = self.token_subvalues.get(m.lastgroup)\n if subvalues:\n if any(m.group(rewritten) for subval, rewritten in subvalues):\n token.subvalues = {\n subval: m.group(rewritten) for subval, rewritten in subvalues\n }\n\n yield token\n" }, { "path": "lib/spack/spack/traverse.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nfrom collections import defaultdict, deque\nfrom typing import (\n TYPE_CHECKING,\n Any,\n Callable,\n Iterable,\n List,\n NamedTuple,\n Optional,\n Sequence,\n Set,\n Tuple,\n Union,\n overload,\n)\n\nfrom spack.vendor.typing_extensions import Literal\n\nimport spack.deptypes as dt\n\nif TYPE_CHECKING:\n import spack.spec\n\n# Export only the high-level API.\n__all__ = [\"traverse_edges\", \"traverse_nodes\", \"traverse_tree\"]\n\n\n#: Data class that stores a directed edge together with depth at\n#: which the target vertex was found. It is passed to ``accept``\n#: and ``neighbors`` of visitors, so they can decide whether to\n#: follow the edge or not.\nclass EdgeAndDepth(NamedTuple):\n edge: \"spack.spec.DependencySpec\"\n depth: int\n\n\n# Sort edges by name first, then abstract hash, then full edge comparison to break ties\ndef sort_edges(edges):\n edges.sort(key=lambda edge: (edge.spec.name or \"\", edge.spec.abstract_hash or \"\", edge))\n return edges\n\n\nclass BaseVisitor:\n \"\"\"A simple visitor that accepts all edges unconditionally and follows all\n edges to dependencies of a given ``deptype``.\"\"\"\n\n def __init__(self, depflag: dt.DepFlag = dt.ALL):\n self.depflag = depflag\n\n def accept(self, item):\n \"\"\"\n Arguments:\n item (EdgeAndDepth): Provides the depth and the edge through which the\n node was discovered\n\n Returns:\n bool: Returns ``True`` if the node is accepted. When ``False``, this\n indicates that the node won't be yielded by iterators and dependencies\n are not followed.\n \"\"\"\n return True\n\n def neighbors(self, item):\n return sort_edges(item.edge.spec.edges_to_dependencies(depflag=self.depflag))\n\n\nclass ReverseVisitor:\n \"\"\"A visitor that reverses the arrows in the DAG, following dependents.\"\"\"\n\n def __init__(self, visitor, depflag: dt.DepFlag = dt.ALL):\n self.visitor = visitor\n self.depflag = depflag\n\n def accept(self, item):\n return self.visitor.accept(item)\n\n def neighbors(self, item):\n \"\"\"Return dependents, note that we actually flip the edge direction to allow\n generic programming\"\"\"\n spec = item.edge.spec\n return sort_edges(\n [edge.flip() for edge in spec.edges_from_dependents(depflag=self.depflag)]\n )\n\n\nclass CoverNodesVisitor:\n \"\"\"A visitor that traverses each node once.\"\"\"\n\n def __init__(self, visitor, key=id, visited=None):\n self.visitor = visitor\n self.key = key\n self.visited = set() if visited is None else visited\n\n def accept(self, item):\n # Covering nodes means: visit nodes once and only once.\n key = self.key(item.edge.spec)\n\n if key in self.visited:\n return False\n\n accept = self.visitor.accept(item)\n self.visited.add(key)\n return accept\n\n def neighbors(self, item):\n return self.visitor.neighbors(item)\n\n\nclass CoverEdgesVisitor:\n \"\"\"A visitor that traverses all edges once.\"\"\"\n\n def __init__(self, visitor, key=id, visited=None):\n self.visitor = visitor\n self.visited = set() if visited is None else visited\n self.key = key\n\n def accept(self, item):\n return self.visitor.accept(item)\n\n def neighbors(self, item):\n # Covering edges means: drop dependencies of visited nodes.\n key = self.key(item.edge.spec)\n\n if key in self.visited:\n return []\n\n self.visited.add(key)\n return self.visitor.neighbors(item)\n\n\nclass MixedDepthVisitor:\n \"\"\"Visits all unique edges of the sub-DAG induced by direct dependencies of type ``direct``\n and transitive dependencies of type ``transitive``. An example use for this is traversing build\n type dependencies non-recursively, and link dependencies recursively.\"\"\"\n\n def __init__(\n self,\n *,\n direct: dt.DepFlag,\n transitive: dt.DepFlag,\n key: Callable[[\"spack.spec.Spec\"], Any] = id,\n ) -> None:\n self.direct_type = direct\n self.transitive_type = transitive\n self.key = key\n self.seen: Set[Any] = set()\n self.seen_roots: Set[Any] = set()\n\n def accept(self, item: EdgeAndDepth) -> bool:\n # Do not accept duplicate root nodes. This only happens if the user starts iterating from\n # multiple roots and lists one of the roots multiple times.\n if item.edge.parent is None:\n node_id = self.key(item.edge.spec)\n if node_id in self.seen_roots:\n return False\n self.seen_roots.add(node_id)\n return True\n\n def neighbors(self, item: EdgeAndDepth) -> List[EdgeAndDepth]:\n # If we're here through an artificial source node, it's a root, and we return all\n # direct_type and transitive_type edges. If we're here through a transitive_type edge, we\n # return all transitive_type edges. To avoid returning the same edge twice:\n # 1. If we had already encountered the current node through a transitive_type edge, we\n # don't need to return transitive_type edges again.\n # 2. If we encounter the current node through a direct_type edge, and we had already seen\n # it through a transitive_type edge, only return the non-transitive_type, direct_type\n # edges.\n node_id = self.key(item.edge.spec)\n seen = node_id in self.seen\n is_root = item.edge.parent is None\n follow_transitive = is_root or bool(item.edge.depflag & self.transitive_type)\n follow = self.direct_type if is_root else dt.NONE\n\n if follow_transitive and not seen:\n follow |= self.transitive_type\n self.seen.add(node_id)\n elif follow == dt.NONE:\n return []\n\n edges = item.edge.spec.edges_to_dependencies(depflag=follow)\n\n # filter direct_type edges already followed before because they were also transitive_type.\n if seen:\n edges = [edge for edge in edges if not edge.depflag & self.transitive_type]\n\n return sort_edges(edges)\n\n\ndef get_visitor_from_args(\n cover, direction, depflag: Union[dt.DepFlag, dt.DepTypes], key=id, visited=None, visitor=None\n):\n \"\"\"\n Create a visitor object from common keyword arguments.\n\n Arguments:\n cover (str): Determines how extensively to cover the dag. Possible values:\n ``nodes`` -- Visit each unique node in the dag only once.\n ``edges`` -- If a node has been visited once but is reached along a\n new path, it's accepted, but not recursively followed. This traverses\n each 'edge' in the DAG once.\n ``paths`` -- Explore every unique path reachable from the root.\n This descends into visited subtrees and will accept nodes multiple\n times if they're reachable by multiple paths.\n direction (str): ``children`` or ``parents``. If ``children``, does a traversal\n of this spec's children. If ``parents``, traverses upwards in the DAG\n towards the root.\n deptype: allowed dependency types\n key: function that takes a spec and outputs a key for uniqueness test.\n visited (set or None): a set of nodes not to follow (when using cover=nodes/edges)\n visitor: An initial visitor that is used for composition.\n\n Returns:\n A visitor\n \"\"\"\n if not isinstance(depflag, dt.DepFlag):\n depflag = dt.canonicalize(depflag)\n visitor = visitor or BaseVisitor(depflag)\n if cover == \"nodes\":\n visitor = CoverNodesVisitor(visitor, key, visited)\n elif cover == \"edges\":\n visitor = CoverEdgesVisitor(visitor, key, visited)\n if direction == \"parents\":\n visitor = ReverseVisitor(visitor, depflag)\n return visitor\n\n\ndef with_artificial_edges(specs):\n \"\"\"Initialize a deque of edges from an artificial root node to the root specs.\"\"\"\n from spack.spec import DependencySpec\n\n return deque(\n EdgeAndDepth(edge=DependencySpec(parent=None, spec=s, depflag=0, virtuals=()), depth=0)\n for s in specs\n )\n\n\ndef traverse_depth_first_edges_generator(edges, visitor, post_order=False, root=True, depth=False):\n \"\"\"Generator that takes explores a DAG in depth-first fashion starting from\n a list of edges. Note that typically DFS would take a vertex not a list of edges,\n but the API is like this so we don't have to create an artificial root node when\n traversing from multiple roots in a DAG.\n\n Arguments:\n edges (list): List of EdgeAndDepth instances\n visitor: class instance implementing accept() and neighbors()\n post_order (bool): Whether to yield nodes when backtracking\n root (bool): whether to yield at depth 0\n depth (bool): when ``True`` yield a tuple of depth and edge, otherwise only the\n edge.\n \"\"\"\n for edge in edges:\n if not visitor.accept(edge):\n continue\n\n yield_me = root or edge.depth > 0\n\n # Pre\n if yield_me and not post_order:\n yield (edge.depth, edge.edge) if depth else edge.edge\n\n neighbors = [EdgeAndDepth(edge=n, depth=edge.depth + 1) for n in visitor.neighbors(edge)]\n\n # This extra branch is just for efficiency.\n if len(neighbors) > 0:\n for item in traverse_depth_first_edges_generator(\n neighbors, visitor, post_order, root, depth\n ):\n yield item\n\n # Post\n if yield_me and post_order:\n yield (edge.depth, edge.edge) if depth else edge.edge\n\n\ndef traverse_breadth_first_edges_generator(queue: deque, visitor, root=True, depth=False):\n while len(queue) > 0:\n edge = queue.popleft()\n\n # If the visitor doesn't accept the node, we don't yield it nor follow its edges.\n if not visitor.accept(edge):\n continue\n\n if root or edge.depth > 0:\n yield (edge.depth, edge.edge) if depth else edge.edge\n\n for e in visitor.neighbors(edge):\n queue.append(EdgeAndDepth(e, edge.depth + 1))\n\n\ndef traverse_breadth_first_with_visitor(specs, visitor):\n \"\"\"Performs breadth first traversal for a list of specs (not a generator).\n\n Arguments:\n specs (list): List of Spec instances.\n visitor: object that implements accept and neighbors interface, see\n for example BaseVisitor.\n \"\"\"\n queue = with_artificial_edges(specs)\n while len(queue) > 0:\n edge = queue.popleft()\n\n # If the visitor doesn't accept the node, we don't traverse it further.\n if not visitor.accept(edge):\n continue\n\n for e in visitor.neighbors(edge):\n queue.append(EdgeAndDepth(e, edge.depth + 1))\n\n\ndef traverse_depth_first_with_visitor(edges, visitor):\n \"\"\"Traverse a DAG in depth-first fashion using a visitor, starting from\n a list of edges. Note that typically DFS would take a vertex not a list of edges,\n but the API is like this so we don't have to create an artificial root node when\n traversing from multiple roots in a DAG.\n\n Arguments:\n edges (list): List of EdgeAndDepth instances\n visitor: class instance implementing accept(), pre(), post() and neighbors()\n \"\"\"\n for edge in edges:\n if not visitor.accept(edge):\n continue\n\n visitor.pre(edge)\n\n neighbors = [EdgeAndDepth(edge=e, depth=edge.depth + 1) for e in visitor.neighbors(edge)]\n\n traverse_depth_first_with_visitor(neighbors, visitor)\n\n visitor.post(edge)\n\n\n# Helper functions for generating a tree using breadth-first traversal\n\n\ndef breadth_first_to_tree_edges(roots, deptype=\"all\", key=id):\n \"\"\"This produces an adjacency list (with edges) and a map of parents.\n There may be nodes that are reached through multiple edges. To print as\n a tree, one should use the parents dict to verify if the path leading to\n the node is through the correct parent. If not, the branch should be\n truncated.\"\"\"\n edges = defaultdict(list)\n parents = dict()\n\n for edge in traverse_edges(roots, order=\"breadth\", cover=\"edges\", deptype=deptype, key=key):\n parent_id = None if edge.parent is None else key(edge.parent)\n child_id = key(edge.spec)\n edges[parent_id].append(edge)\n if child_id not in parents:\n parents[child_id] = parent_id\n\n return edges, parents\n\n\ndef breadth_first_to_tree_nodes(roots, deptype=\"all\", key=id):\n \"\"\"This produces a list of edges that forms a tree; every node has no more\n that one incoming edge.\"\"\"\n edges = defaultdict(list)\n\n for edge in traverse_edges(roots, order=\"breadth\", cover=\"nodes\", deptype=deptype, key=key):\n parent_id = None if edge.parent is None else key(edge.parent)\n edges[parent_id].append(edge)\n\n return edges\n\n\ndef traverse_breadth_first_tree_edges(parent_id, edges, parents, key=id, depth=0):\n \"\"\"Do a depth-first search on edges generated by bread-first traversal,\n which can be used to produce a tree.\"\"\"\n for edge in edges[parent_id]:\n yield (depth, edge)\n\n child_id = key(edge.spec)\n\n # Don't follow further if we're not the parent\n if parents[child_id] != parent_id:\n continue\n\n yield from traverse_breadth_first_tree_edges(child_id, edges, parents, key, depth + 1)\n\n\ndef traverse_breadth_first_tree_nodes(parent_id, edges, key=id, depth=0):\n for edge in edges[parent_id]:\n yield (depth, edge)\n for item in traverse_breadth_first_tree_nodes(key(edge.spec), edges, key, depth + 1):\n yield item\n\n\ndef traverse_topo_edges_generator(edges, visitor, key=id, root=True, all_edges=False):\n \"\"\"\n Returns a list of edges in topological order, in the sense that all in-edges of a vertex appear\n before all out-edges.\n\n Arguments:\n edges (list): List of EdgeAndDepth instances\n visitor: visitor that produces unique edges defining the (sub)DAG of interest.\n key: function that takes a spec and outputs a key for uniqueness test.\n root (bool): Yield the root nodes themselves\n all_edges (bool): When ``False`` only one in-edge per node is returned, when\n ``True`` all reachable edges are returned.\n \"\"\"\n # Topo order used to be implemented using a DFS visitor, which was relatively efficient in that\n # it would visit nodes only once, and it was composable. In practice however it would yield a\n # DFS order on DAGs that are trees, which is undesirable in many cases. For example, a list of\n # search paths for trees is better in BFS order, so that direct dependencies are listed first.\n # That way a transitive dependency cannot shadow a direct one. So, here we collect the sub-DAG\n # of interest and then compute a topological order that is the most breadth-first possible.\n\n # maps node identifier to the number of remaining in-edges\n in_edge_count = defaultdict(int)\n # maps parent identifier to a list of edges, where None is a special identifier\n # for the artificial root/source.\n node_to_edges = defaultdict(list)\n for edge in traverse_breadth_first_edges_generator(edges, visitor, root=True, depth=False):\n in_edge_count[key(edge.spec)] += 1\n parent_id = key(edge.parent) if edge.parent is not None else None\n node_to_edges[parent_id].append(edge)\n\n queue = deque((None,))\n\n while queue:\n for edge in node_to_edges[queue.popleft()]:\n child_id = key(edge.spec)\n in_edge_count[child_id] -= 1\n\n should_yield = root or edge.parent is not None\n\n if all_edges and should_yield:\n yield edge\n\n if in_edge_count[child_id] == 0:\n if not all_edges and should_yield:\n yield edge\n queue.append(key(edge.spec))\n\n\n# High-level API: traverse_edges, traverse_nodes, traverse_tree.\n\nOrderType = Literal[\"pre\", \"post\", \"breadth\", \"topo\"]\nCoverType = Literal[\"nodes\", \"edges\", \"paths\"]\nDirectionType = Literal[\"children\", \"parents\"]\n\n\n@overload\ndef traverse_edges(\n specs: Sequence[\"spack.spec.Spec\"],\n *,\n root: bool = ...,\n order: OrderType = ...,\n cover: CoverType = ...,\n direction: DirectionType = ...,\n deptype: Union[dt.DepFlag, dt.DepTypes] = ...,\n depth: Literal[False] = False,\n key: Callable[[\"spack.spec.Spec\"], Any] = ...,\n visited: Optional[Set[Any]] = ...,\n) -> Iterable[\"spack.spec.DependencySpec\"]: ...\n\n\n@overload\ndef traverse_edges(\n specs: Sequence[\"spack.spec.Spec\"],\n *,\n root: bool = ...,\n order: OrderType = ...,\n cover: CoverType = ...,\n direction: DirectionType = ...,\n deptype: Union[dt.DepFlag, dt.DepTypes] = ...,\n depth: Literal[True],\n key: Callable[[\"spack.spec.Spec\"], Any] = ...,\n visited: Optional[Set[Any]] = ...,\n) -> Iterable[Tuple[int, \"spack.spec.DependencySpec\"]]: ...\n\n\n@overload\ndef traverse_edges(\n specs: Sequence[\"spack.spec.Spec\"],\n *,\n root: bool = ...,\n order: OrderType = ...,\n cover: CoverType = ...,\n direction: DirectionType = ...,\n deptype: Union[dt.DepFlag, dt.DepTypes] = ...,\n depth: bool,\n key: Callable[[\"spack.spec.Spec\"], Any] = ...,\n visited: Optional[Set[Any]] = ...,\n) -> Iterable[Union[\"spack.spec.DependencySpec\", Tuple[int, \"spack.spec.DependencySpec\"]]]: ...\n\n\ndef traverse_edges(\n specs: Sequence[\"spack.spec.Spec\"],\n root: bool = True,\n order: OrderType = \"pre\",\n cover: CoverType = \"nodes\",\n direction: DirectionType = \"children\",\n deptype: Union[dt.DepFlag, dt.DepTypes] = \"all\",\n depth: bool = False,\n key: Callable[[\"spack.spec.Spec\"], Any] = id,\n visited: Optional[Set[Any]] = None,\n) -> Iterable[Union[\"spack.spec.DependencySpec\", Tuple[int, \"spack.spec.DependencySpec\"]]]:\n \"\"\"\n Iterable of edges from the DAG, starting from a list of root specs.\n\n Arguments:\n\n specs: List of root specs (considered to be depth 0)\n root: Yield the root nodes themselves\n order: What order of traversal to use in the DAG. For depth-first search this can be\n ``pre`` or ``post``. For BFS this should be ``breadth``. For topological order use\n ``topo``\n cover: Determines how extensively to cover the dag. Possible values:\n ``nodes`` -- Visit each unique node in the dag only once.\n ``edges`` -- If a node has been visited once but is reached along a new path, it's\n accepted, but not recursively followed. This traverses each 'edge' in the DAG once.\n ``paths`` -- Explore every unique path reachable from the root. This descends into\n visited subtrees and will accept nodes multiple times if they're reachable by multiple\n paths.\n direction: ``children`` or ``parents``. If ``children``, does a traversal of this spec's\n children. If ``parents``, traverses upwards in the DAG towards the root.\n deptype: allowed dependency types\n depth: When ``False``, yield just edges. When ``True`` yield the tuple (depth, edge), where\n depth corresponds to the depth at which edge.spec was discovered.\n key: function that takes a spec and outputs a key for uniqueness test.\n visited: a set of nodes not to follow\n\n Returns:\n An iterable of ``DependencySpec`` if depth is ``False`` or a tuple of\n ``(depth, DependencySpec)`` if depth is ``True``.\n \"\"\"\n # validate input\n if order == \"topo\":\n if cover == \"paths\":\n raise ValueError(\"cover=paths not supported for order=topo\")\n if visited is not None:\n raise ValueError(\"visited set not implemented for order=topo\")\n elif order not in (\"post\", \"pre\", \"breadth\"):\n raise ValueError(f\"Unknown order {order}\")\n\n # In topo traversal we need to construct a sub-DAG including all unique edges even if we are\n # yielding a subset of them, hence \"edges\".\n _cover = \"edges\" if order == \"topo\" else cover\n visitor = get_visitor_from_args(_cover, direction, deptype, key, visited)\n root_edges = with_artificial_edges(specs)\n\n # Depth-first\n if order == \"pre\" or order == \"post\":\n return traverse_depth_first_edges_generator(\n root_edges, visitor, order == \"post\", root, depth\n )\n elif order == \"breadth\":\n return traverse_breadth_first_edges_generator(root_edges, visitor, root, depth)\n elif order == \"topo\":\n return traverse_topo_edges_generator(\n root_edges, visitor, key, root, all_edges=cover == \"edges\"\n )\n\n\n@overload\ndef traverse_nodes(\n specs: Sequence[\"spack.spec.Spec\"],\n *,\n root: bool = ...,\n order: OrderType = ...,\n cover: CoverType = ...,\n direction: DirectionType = ...,\n deptype: Union[dt.DepFlag, dt.DepTypes] = ...,\n depth: Literal[False] = False,\n key: Callable[[\"spack.spec.Spec\"], Any] = ...,\n visited: Optional[Set[Any]] = ...,\n) -> Iterable[\"spack.spec.Spec\"]: ...\n\n\n@overload\ndef traverse_nodes(\n specs: Sequence[\"spack.spec.Spec\"],\n *,\n root: bool = ...,\n order: OrderType = ...,\n cover: CoverType = ...,\n direction: DirectionType = ...,\n deptype: Union[dt.DepFlag, dt.DepTypes] = ...,\n depth: Literal[True],\n key: Callable[[\"spack.spec.Spec\"], Any] = ...,\n visited: Optional[Set[Any]] = ...,\n) -> Iterable[Tuple[int, \"spack.spec.Spec\"]]: ...\n\n\n@overload\ndef traverse_nodes(\n specs: Sequence[\"spack.spec.Spec\"],\n *,\n root: bool = ...,\n order: OrderType = ...,\n cover: CoverType = ...,\n direction: DirectionType = ...,\n deptype: Union[dt.DepFlag, dt.DepTypes] = ...,\n depth: bool,\n key: Callable[[\"spack.spec.Spec\"], Any] = ...,\n visited: Optional[Set[Any]] = ...,\n) -> Iterable[Union[\"spack.spec.Spec\", Tuple[int, \"spack.spec.Spec\"]]]: ...\n\n\ndef traverse_nodes(\n specs: Sequence[\"spack.spec.Spec\"],\n *,\n root: bool = True,\n order: OrderType = \"pre\",\n cover: CoverType = \"nodes\",\n direction: DirectionType = \"children\",\n deptype: Union[dt.DepFlag, dt.DepTypes] = \"all\",\n depth: bool = False,\n key: Callable[[\"spack.spec.Spec\"], Any] = id,\n visited: Optional[Set[Any]] = None,\n) -> Iterable[Union[\"spack.spec.Spec\", Tuple[int, \"spack.spec.Spec\"]]]:\n \"\"\"\n Iterable of specs from the DAG, starting from a list of root specs.\n\n Arguments:\n specs: List of root specs (considered to be depth 0)\n root: Yield the root nodes themselves\n order: What order of traversal to use in the DAG. For depth-first search this can be\n ``pre`` or ``post``. For BFS this should be ``breadth``.\n cover: Determines how extensively to cover the dag. Possible values:\n ``nodes`` -- Visit each unique node in the dag only once.\n ``edges`` -- If a node has been visited once but is reached along a new path, it's\n accepted, but not recursively followed. This traverses each 'edge' in the DAG once.\n ``paths`` -- Explore every unique path reachable from the root. This descends into\n visited subtrees and will accept nodes multiple times if they're reachable by multiple\n paths.\n direction: ``children`` or ``parents``. If ``children``, does a traversal of this spec's\n children. If ``parents``, traverses upwards in the DAG towards the root.\n deptype: allowed dependency types\n depth: When ``False``, yield just edges. When ``True`` yield the tuple ``(depth, edge)``,\n where depth corresponds to the depth at which ``edge.spec`` was discovered.\n key: function that takes a spec and outputs a key for uniqueness test.\n visited: a set of nodes not to follow\n\n Yields:\n By default :class:`~spack.spec.Spec`, or a tuple ``(depth, Spec)`` if depth is\n set to ``True``.\n \"\"\"\n for item in traverse_edges(\n specs,\n root=root,\n order=order,\n cover=cover,\n direction=direction,\n deptype=deptype,\n depth=depth,\n key=key,\n visited=visited,\n ):\n yield (item[0], item[1].spec) if depth else item.spec # type: ignore\n\n\ndef traverse_tree(\n specs: Sequence[\"spack.spec.Spec\"],\n cover: CoverType = \"nodes\",\n deptype: Union[dt.DepFlag, dt.DepTypes] = \"all\",\n key: Callable[[\"spack.spec.Spec\"], Any] = id,\n depth_first: bool = True,\n) -> Iterable[Tuple[int, \"spack.spec.DependencySpec\"]]:\n \"\"\"\n Generator that yields ``(depth, DependencySpec)`` tuples in the depth-first\n pre-order, so that a tree can be printed from it.\n\n Arguments:\n\n specs: List of root specs (considered to be depth 0)\n cover: Determines how extensively to cover the dag. Possible values:\n ``nodes`` -- Visit each unique node in the dag only once.\n ``edges`` -- If a node has been visited once but is reached along a\n new path, it's accepted, but not recursively followed. This traverses each 'edge' in\n the DAG once.\n ``paths`` -- Explore every unique path reachable from the root. This descends into\n visited subtrees and will accept nodes multiple times if they're reachable by multiple\n paths.\n deptype: allowed dependency types\n key: function that takes a spec and outputs a key for uniqueness test.\n depth_first: Explore the tree in depth-first or breadth-first order. When setting\n ``depth_first=True`` and ``cover=nodes``, each spec only occurs once at the shallowest\n level, which is useful when rendering the tree in a terminal.\n\n Returns:\n A generator that yields ``(depth, DependencySpec)`` tuples in such an order that a tree can\n be printed.\n \"\"\"\n # BFS only makes sense when going over edges and nodes, for paths the tree is\n # identical to DFS, which is much more efficient then.\n if not depth_first and cover == \"edges\":\n edges, parents = breadth_first_to_tree_edges(specs, deptype, key)\n return traverse_breadth_first_tree_edges(None, edges, parents, key)\n elif not depth_first and cover == \"nodes\":\n edges = breadth_first_to_tree_nodes(specs, deptype, key)\n return traverse_breadth_first_tree_nodes(None, edges, key)\n\n return traverse_edges(specs, order=\"pre\", cover=cover, deptype=deptype, key=key, depth=True)\n\n\ndef by_dag_hash(s: \"spack.spec.Spec\") -> str:\n \"\"\"Used very often as a key function for traversals.\"\"\"\n return s.dag_hash()\n" }, { "path": "lib/spack/spack/url.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"\nThis module has methods for parsing names and versions of packages from URLs.\nThe idea is to allow package creators to supply nothing more than the\ndownload location of the package, and figure out version and name information\nfrom there.\n\n**Example:** when spack is given the following URL:\n\n.. code-block::\n\n https://www.hdfgroup.org/ftp/HDF/releases/HDF4.2.12/src/hdf-4.2.12.tar.gz\n\nIt can figure out that the package name is ``hdf``, and that it is at version\n``4.2.12``. This is useful for making the creation of packages simple: a user\njust supplies a URL and skeleton code is generated automatically.\n\nSpack can also figure out that it can most likely download 4.2.6 at this URL:\n\n.. code-block::\n\n https://www.hdfgroup.org/ftp/HDF/releases/HDF4.2.6/src/hdf-4.2.6.tar.gz\n\nThis is useful if a user asks for a package at a particular version number;\nspack doesn't need anyone to tell it where to get the tarball even though\nit's never been told about that version before.\n\"\"\"\n\nimport io\nimport os\nimport pathlib\nimport re\nfrom typing import Any, Dict, Optional, Sequence, Tuple, Union\n\nimport spack.error\nimport spack.llnl.url\nimport spack.util.web\nimport spack.version\nfrom spack.llnl.path import convert_to_posix_path\nfrom spack.llnl.util.tty.color import cescape, colorize\n\n#\n# Note: We call the input to most of these functions a \"path\" but the functions\n# work on paths and URLs. There's not a good word for both of these, but\n# \"path\" seemed like the most generic term.\n#\n\n\ndef strip_name_suffixes(path: str, version: Union[str, spack.version.StandardVersion]) -> str:\n \"\"\"Most tarballs contain a package name followed by a version number.\n However, some also contain extraneous information in-between the name\n and version:\n\n * ``rgb-1.0.6``\n * ``converge_install_2.3.16``\n * ``jpegsrc.v9b``\n\n These strings are not part of the package name and should be ignored.\n This function strips the version number and any extraneous suffixes\n off and returns the remaining string. The goal is that the name is\n always the last thing in ``path``:\n\n * ``rgb``\n * ``converge``\n * ``jpeg``\n\n Args:\n path: The filename or URL for the package\n version: The version detected for this URL\n\n Returns:\n str: The ``path`` with any extraneous suffixes removed\n \"\"\"\n # NOTE: This could be done with complicated regexes in parse_name_offset\n # NOTE: The problem is that we would have to add these regexes to every\n # NOTE: single name regex. Easier to just strip them off permanently\n\n suffix_regexes = [\n # Strip off the version and anything after it\n # name-ver\n # name_ver\n # name.ver\n r\"[._-][rvV]?\" + str(version) + \".*\",\n # namever\n r\"V?\" + str(version) + \".*\",\n # Download type\n r\"install\",\n r\"[Ss]rc\",\n r\"(open)?[Ss]ources?\",\n r\"[._-]open\",\n r\"[._-]archive\",\n r\"[._-]std\",\n r\"[._-]bin\",\n r\"Software\",\n # Download version\n r\"release\",\n r\"snapshot\",\n r\"distrib\",\n r\"everywhere\",\n r\"latest\",\n # Arch\n r\"Linux(64)?\",\n r\"x86_64\",\n # VCS\n r\"0\\+bzr\",\n # License\n r\"gpl\",\n # Needs to come before and after gpl, appears in both orders\n r\"[._-]x11\",\n r\"gpl\",\n ]\n\n for regex in suffix_regexes:\n # Remove the suffix from the end of the path\n # This may be done multiple times\n path = re.sub(\"[._-]?\" + regex + \"$\", \"\", path)\n\n return path\n\n\ndef parse_version_offset(path: str) -> Tuple[str, int, int, int, str]:\n \"\"\"Try to extract a version string from a filename or URL.\n\n Args:\n path (str): The filename or URL for the package\n\n Returns:\n A tuple containing\n\n * version of the package\n * first index of version\n * length of version string\n * the index of the matching regex\n * the matching regex\n\n Raises:\n UndetectableVersionError: If the URL does not match any regexes\n \"\"\"\n original_path = path\n\n # path: The prefix of the URL, everything before the ext and suffix\n # ext: The file extension\n # suffix: Any kind of query string that begins with a '?'\n path, ext, suffix = spack.llnl.url.split_url_extension(path)\n\n # stem: Everything from path after the final '/'\n original_stem = os.path.basename(path)\n\n # Try to strip off anything after the version number\n stem = spack.llnl.url.strip_version_suffixes(original_stem)\n\n # Assumptions:\n #\n # 1. version always comes after the name\n # 2. separators include '-', '_', and '.'\n # 3. names can contain A-Z, a-z, 0-9, '+', separators\n # 4. versions can contain A-Z, a-z, 0-9, separators\n # 5. versions always start with a digit\n # 6. versions are often prefixed by a 'v' or 'r' character\n # 7. separators are most reliable to determine name/version boundaries\n\n # List of the following format:\n #\n # [\n # (regex, string),\n # ...\n # ]\n #\n # The first regex that matches string will be used to determine\n # the version of the package. Therefore, hyperspecific regexes should\n # come first while generic, catch-all regexes should come last.\n # With that said, regular expressions are slow, so if possible, put\n # ones that only catch one or two URLs at the bottom.\n version_regexes = [\n # 1st Pass: Simplest case\n # Assume name contains no digits and version contains no letters\n # e.g. libpng-1.6.27\n (r\"^[a-zA-Z+._-]+[._-]v?(\\d[\\d._-]*)$\", stem),\n # 2nd Pass: Version only\n # Assume version contains no letters\n # ver\n # e.g. 3.2.7, 7.0.2-7, v3.3.0, v1_6_3\n (r\"^v?(\\d[\\d._-]*)$\", stem),\n # 3rd Pass: No separator characters are used\n # Assume name contains no digits\n # namever\n # e.g. turbolinux702, nauty26r7\n (r\"^[a-zA-Z+]*(\\d[\\da-zA-Z]*)$\", stem),\n # 4th Pass: A single separator character is used\n # Assume name contains no digits\n # name-name-ver-ver\n # e.g. panda-2016-03-07, gts-snapshot-121130, cdd-061a\n (r\"^[a-zA-Z+-]*(\\d[\\da-zA-Z-]*)$\", stem),\n # name_name_ver_ver\n # e.g. tinyxml_2_6_2, boost_1_55_0, tbb2017_20161128\n (r\"^[a-zA-Z+_]*(\\d[\\da-zA-Z_]*)$\", stem),\n # name.name.ver.ver\n # e.g. prank.source.150803, jpegsrc.v9b, atlas3.11.34, geant4.10.01.p03\n (r\"^[a-zA-Z+.]*(\\d[\\da-zA-Z.]*)$\", stem),\n # 5th Pass: Two separator characters are used\n # Name may contain digits, version may contain letters\n # name-name-ver.ver\n # e.g. m4-1.4.17, gmp-6.0.0a, launchmon-v1.0.2\n (r\"^[a-zA-Z\\d+-]+-v?(\\d[\\da-zA-Z.]*)$\", stem),\n # name-name-ver_ver\n # e.g. icu4c-57_1\n (r\"^[a-zA-Z\\d+-]+-v?(\\d[\\da-zA-Z_]*)$\", stem),\n # name_name_ver.ver\n # e.g. superlu_dist_4.1, pexsi_v0.9.0\n (r\"^[a-zA-Z\\d+_]+_v?(\\d[\\da-zA-Z.]*)$\", stem),\n # name_name.ver.ver\n # e.g. fer_source.v696\n (r\"^[a-zA-Z\\d+_]+\\.v?(\\d[\\da-zA-Z.]*)$\", stem),\n # name_ver-ver\n # e.g. Bridger_r2014-12-01\n (r\"^[a-zA-Z\\d+]+_r?(\\d[\\da-zA-Z-]*)$\", stem),\n # name-name-ver.ver-ver.ver\n # e.g. sowing-1.1.23-p1, bib2xhtml-v3.0-15-gf506, 4.6.3-alpha04\n (r\"^(?:[a-zA-Z\\d+-]+-)?v?(\\d[\\da-zA-Z.-]*)$\", stem),\n # namever.ver-ver.ver\n # e.g. go1.4-bootstrap-20161024\n (r\"^[a-zA-Z+]+v?(\\d[\\da-zA-Z.-]*)$\", stem),\n # 6th Pass: All three separator characters are used\n # Name may contain digits, version may contain letters\n # name_name-ver.ver\n # e.g. the_silver_searcher-0.32.0, sphinx_rtd_theme-0.1.10a0\n (r\"^[a-zA-Z\\d+_]+-v?(\\d[\\da-zA-Z.]*)$\", stem),\n # name.name_ver.ver-ver.ver\n # e.g. TH.data_1.0-8, XML_3.98-1.4\n (r\"^[a-zA-Z\\d+.]+_v?(\\d[\\da-zA-Z.-]*)$\", stem),\n # name-name-ver.ver_ver.ver\n # e.g. pypar-2.1.5_108\n (r\"^[a-zA-Z\\d+-]+-v?(\\d[\\da-zA-Z._]*)$\", stem),\n # name.name_name-ver.ver\n # e.g. tap.py-1.6, backports.ssl_match_hostname-3.5.0.1\n (r\"^[a-zA-Z\\d+._]+-v?(\\d[\\da-zA-Z.]*)$\", stem),\n # name-namever.ver_ver.ver\n # e.g. STAR-CCM+11.06.010_02\n (r\"^[a-zA-Z+-]+(\\d[\\da-zA-Z._]*)$\", stem),\n # name-name_name-ver.ver\n # e.g. PerlIO-utf8_strict-0.002\n (r\"^[a-zA-Z\\d+_-]+-v?(\\d[\\da-zA-Z.]*)$\", stem),\n # 7th Pass: Specific VCS\n # bazaar\n # e.g. libvterm-0+bzr681\n (r\"bzr(\\d[\\da-zA-Z._-]*)$\", stem),\n # 8th Pass: Query strings\n # e.g. https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0\n # e.g. https://gitlab.kitware.com/api/v4/projects/icet%2Ficet/repository/archive.tar.bz2?sha=IceT-2.1.1\n # e.g. http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0\n # e.g. http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1\n # e.g. https://software.broadinstitute.org/gatk/download/auth?package=GATK-archive&version=3.8-1-0-gf15c1c3ef\n (r\"[?&](?:sha|ref|version)=[a-zA-Z\\d+-]*[_-]?v?(\\d[\\da-zA-Z._-]*)$\", suffix),\n # e.g. http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz\n # e.g. http://laws-green.lanl.gov/projects/data/eos/get_file.php?package=eospac&filename=eospac_v6.4.0beta.1_r20171213193219.tgz\n # e.g. https://evtgen.hepforge.org/downloads?f=EvtGen-01.07.00.tar.gz\n # e.g. http://wwwpub.zih.tu-dresden.de/%7Emlieber/dcount/dcount.php?package=otf&get=OTF-1.12.5salmon.tar.gz\n (r\"[?&](?:filename|f|get)=[a-zA-Z\\d+-]+[_-]v?(\\d[\\da-zA-Z.]*)\", stem),\n # 9th Pass: Version in path\n # github.com/repo/name/releases/download/vver/name\n # e.g. https://github.com/nextflow-io/nextflow/releases/download/v0.20.1/nextflow\n # e.g. https://gitlab.com/hpctoolkit/hpcviewer/-/releases/2024.02/downloads/hpcviewer.tgz\n (r\"github\\.com/[^/]+/[^/]+/releases/download/[a-zA-Z+._-]*v?(\\d[\\da-zA-Z._-]*)/\", path),\n (r\"gitlab\\.com/[^/]+/.+/-/releases/[a-zA-Z+._-]*v?(\\d[\\da-zA-Z._-]*)/downloads/\", path),\n # e.g. ftp://ftp.ncbi.nlm.nih.gov/blast/executables/legacy.NOTSUPPORTED/2.2.26/ncbi.tar.gz\n (r\"(\\d[\\da-zA-Z._-]*)/[^/]+$\", path),\n ]\n\n for i, version_regex in enumerate(version_regexes):\n regex, match_string = version_regex\n match = re.search(regex, match_string)\n if match and match.group(1) is not None:\n version = match.group(1)\n start = match.start(1)\n\n # If we matched from the stem or suffix, we need to add offset\n offset = 0\n if match_string is stem:\n offset = len(path) - len(original_stem)\n elif match_string is suffix:\n offset = len(path)\n if ext:\n offset += len(ext) + 1 # .tar.gz is converted to tar.gz\n start += offset\n\n return version, start, len(version), i, regex\n\n raise UndetectableVersionError(original_path)\n\n\ndef parse_version(path: str) -> spack.version.StandardVersion:\n \"\"\"Try to extract a version string from a filename or URL.\n\n Args:\n path: The filename or URL for the package\n\n Returns: The version of the package\n\n Raises:\n UndetectableVersionError: If the URL does not match any regexes\n \"\"\"\n version, start, length, i, regex = parse_version_offset(path)\n return spack.version.StandardVersion.from_string(version)\n\n\ndef parse_name_offset(\n path: str, v: Optional[Union[str, spack.version.StandardVersion]] = None\n) -> Tuple[str, int, int, int, str]:\n \"\"\"Try to determine the name of a package from its filename or URL.\n\n Args:\n path: The filename or URL for the package\n v: The version of the package\n\n Returns:\n A tuple containing\n\n * name of the package\n * first index of name\n * length of name\n * the index of the matching regex\n * the matching regex\n\n Raises:\n UndetectableNameError: If the URL does not match any regexes\n \"\"\"\n original_path = path\n\n # We really need to know the version of the package\n # This helps us prevent collisions between the name and version\n if v is None:\n try:\n v = parse_version(path)\n except UndetectableVersionError:\n # Not all URLs contain a version. We still want to be able\n # to determine a name if possible.\n v = \"unknown\"\n\n # path: The prefix of the URL, everything before the ext and suffix\n # ext: The file extension\n # suffix: Any kind of query string that begins with a '?'\n path, ext, suffix = spack.llnl.url.split_url_extension(path)\n\n # stem: Everything from path after the final '/'\n original_stem = os.path.basename(path)\n\n # Try to strip off anything after the package name\n stem = strip_name_suffixes(original_stem, v)\n\n # List of the following format:\n #\n # [\n # (regex, string),\n # ...\n # ]\n #\n # The first regex that matches string will be used to determine\n # the name of the package. Therefore, hyperspecific regexes should\n # come first while generic, catch-all regexes should come last.\n # With that said, regular expressions are slow, so if possible, put\n # ones that only catch one or two URLs at the bottom.\n name_regexes = [\n # 1st Pass: Common repositories\n # GitHub: github.com/repo/name/\n # e.g. https://github.com/nco/nco/archive/4.6.2.tar.gz\n (r\"github\\.com/[^/]+/([^/]+)\", path),\n # GitLab API endpoint: gitlab.*/api/v4/projects/NAMESPACE%2Fname/\n # e.g. https://gitlab.cosma.dur.ac.uk/api/v4/projects/swift%2Fswiftsim/repository/archive.tar.gz?sha=v0.3.0\n (r\"gitlab[^/]+/api/v4/projects/[^/]+%2F([^/]+)\", path),\n # GitLab non-API endpoint: gitlab.*/repo/name/\n # e.g. http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0\n (r\"gitlab[^/]+/(?!api/v4/projects)[^/]+/([^/]+)\", path),\n # Bitbucket: bitbucket.org/repo/name/\n # e.g. https://bitbucket.org/glotzer/hoomd-blue/get/v1.3.3.tar.bz2\n (r\"bitbucket\\.org/[^/]+/([^/]+)\", path),\n # PyPI: pypi.(python.org|io)/packages/source/first-letter/name/\n # e.g. https://pypi.python.org/packages/source/m/mpmath/mpmath-all-0.19.tar.gz\n # e.g. https://pypi.io/packages/source/b/backports.ssl_match_hostname/backports.ssl_match_hostname-3.5.0.1.tar.gz\n (r\"pypi\\.(?:python\\.org|io)/packages/source/[A-Za-z\\d]/([^/]+)\", path),\n # 2nd Pass: Query strings\n # ?filename=name-ver.ver\n # e.g. http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz\n (r\"\\?filename=([A-Za-z\\d+-]+)$\", stem),\n # ?f=name-ver.ver\n # e.g. https://evtgen.hepforge.org/downloads?f=EvtGen-01.07.00.tar.gz\n (r\"\\?f=([A-Za-z\\d+-]+)$\", stem),\n # ?package=name\n # e.g. http://wwwpub.zih.tu-dresden.de/%7Emlieber/dcount/dcount.php?package=otf&get=OTF-1.12.5salmon.tar.gz\n (r\"\\?package=([A-Za-z\\d+-]+)\", stem),\n # ?package=name-version\n (r\"\\?package=([A-Za-z\\d]+)\", suffix),\n # download.php\n # e.g. http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1\n (r\"([^/]+)/download.php$\", path),\n # 3rd Pass: Name followed by version in archive\n (r\"^([A-Za-z\\d+\\._-]+)$\", stem),\n ]\n\n for i, name_regex in enumerate(name_regexes):\n regex, match_string = name_regex\n match = re.search(regex, match_string)\n if match:\n name = match.group(1)\n start = match.start(1)\n\n # If we matched from the stem or suffix, we need to add offset\n offset = 0\n if match_string is stem:\n offset = len(path) - len(original_stem)\n elif match_string is suffix:\n offset = len(path)\n if ext:\n offset += len(ext) + 1 # .tar.gz is converted to tar.gz\n start += offset\n\n return name, start, len(name), i, regex\n\n raise UndetectableNameError(original_path)\n\n\ndef parse_name(path, ver=None):\n \"\"\"Try to determine the name of a package from its filename or URL.\n\n Args:\n path (str): The filename or URL for the package\n ver (str): The version of the package\n\n Returns:\n str: The name of the package\n\n Raises:\n UndetectableNameError: If the URL does not match any regexes\n \"\"\"\n name, start, length, i, regex = parse_name_offset(path, ver)\n return name\n\n\ndef parse_name_and_version(path: str) -> Tuple[str, spack.version.StandardVersion]:\n \"\"\"Try to determine the name of a package and extract its version\n from its filename or URL.\n\n Args:\n path: The filename or URL for the package\n\n Returns:\n tuple: a tuple containing the package (name, version)\n\n Raises:\n UndetectableVersionError: If the URL does not match any regexes\n UndetectableNameError: If the URL does not match any regexes\n \"\"\"\n ver = parse_version(path)\n name = parse_name(path, ver)\n return (name, ver)\n\n\ndef find_all(substring, string):\n \"\"\"Returns a list containing the indices of\n every occurrence of substring in string.\"\"\"\n\n occurrences = []\n index = 0\n while index < len(string):\n index = string.find(substring, index)\n if index == -1:\n break\n occurrences.append(index)\n index += len(substring)\n\n return occurrences\n\n\ndef substitution_offsets(path):\n \"\"\"This returns offsets for substituting versions and names in the\n provided path. It is a helper for :func:`substitute_version`.\n \"\"\"\n # Get name and version offsets\n try:\n ver, vs, vl, vi, vregex = parse_version_offset(path)\n name, ns, nl, ni, nregex = parse_name_offset(path, ver)\n except UndetectableNameError:\n return (None, -1, -1, (), ver, vs, vl, (vs,))\n except UndetectableVersionError:\n try:\n name, ns, nl, ni, nregex = parse_name_offset(path)\n return (name, ns, nl, (ns,), None, -1, -1, ())\n except UndetectableNameError:\n return (None, -1, -1, (), None, -1, -1, ())\n\n # Find the index of every occurrence of name and ver in path\n name_offsets = find_all(name, path)\n ver_offsets = find_all(ver, path)\n\n return (name, ns, nl, name_offsets, ver, vs, vl, ver_offsets)\n\n\ndef wildcard_version(path):\n \"\"\"Find the version in the supplied path, and return a regular expression\n that will match this path with any version in its place.\n \"\"\"\n # Get version so we can replace it with a wildcard\n version = parse_version(path)\n\n # Split path by versions\n vparts = path.split(str(version))\n\n # Replace each version with a generic capture group to find versions\n # and escape everything else so it's not interpreted as a regex\n result = r\"(\\d.*)\".join(re.escape(vp) for vp in vparts)\n\n return result\n\n\ndef substitute_version(path: str, new_version) -> str:\n \"\"\"Given a URL or archive name, find the version in the path and\n substitute the new version for it. Replace all occurrences of\n the version *if* they don't overlap with the package name.\n\n Simple example:\n\n .. code-block:: pycon\n\n >>> substitute_version(\"http://www.mr511.de/software/libelf-0.8.13.tar.gz\", \"2.9.3\")\n \"http://www.mr511.de/software/libelf-2.9.3.tar.gz\"\n\n Complex example:\n\n .. code-block:: pycon\n\n >>> substitute_version(\"https://www.hdfgroup.org/ftp/HDF/releases/HDF4.2.12/src/hdf-4.2.12.tar.gz\", \"2.3\")\n \"https://www.hdfgroup.org/ftp/HDF/releases/HDF2.3/src/hdf-2.3.tar.gz\"\n \"\"\" # noqa: E501\n (name, ns, nl, noffs, ver, vs, vl, voffs) = substitution_offsets(path)\n\n new_path = \"\"\n last = 0\n for vo in voffs:\n new_path += path[last:vo]\n new_path += str(new_version)\n last = vo + vl\n\n new_path += path[last:]\n return new_path\n\n\ndef color_url(path, **kwargs):\n \"\"\"Color the parts of the url according to Spack's parsing.\n\n Colors are:\n\n * Cyan: The version found by :func:`parse_version_offset`.\n * Red: The name found by :func:`parse_name_offset`.\n * Green: Instances of version string from :func:`substitute_version`.\n * Magenta: Instances of the name (protected from substitution).\n\n Args:\n path (str): The filename or URL for the package\n errors (bool): Append parse errors at end of string.\n subs (bool): Color substitutions as well as parsed name/version.\n \"\"\"\n # Allow URLs containing @ and }\n path = cescape(path)\n\n errors = kwargs.get(\"errors\", False)\n subs = kwargs.get(\"subs\", False)\n\n (name, ns, nl, noffs, ver, vs, vl, voffs) = substitution_offsets(path)\n\n nends = [no + nl - 1 for no in noffs]\n vends = [vo + vl - 1 for vo in voffs]\n\n nerr = verr = 0\n out = io.StringIO()\n for i in range(len(path)):\n if i == vs:\n out.write(\"@c\")\n verr += 1\n elif i == ns:\n out.write(\"@r\")\n nerr += 1\n elif subs:\n if i in voffs:\n out.write(\"@g\")\n elif i in noffs:\n out.write(\"@m\")\n\n out.write(path[i])\n\n if i == vs + vl - 1:\n out.write(\"@.\")\n verr += 1\n elif i == ns + nl - 1:\n out.write(\"@.\")\n nerr += 1\n elif subs:\n if i in vends or i in nends:\n out.write(\"@.\")\n\n if errors:\n if nerr == 0:\n out.write(\" @r{[no name]}\")\n if verr == 0:\n out.write(\" @r{[no version]}\")\n if nerr == 1:\n out.write(\" @r{[incomplete name]}\")\n if verr == 1:\n out.write(\" @r{[incomplete version]}\")\n\n return colorize(out.getvalue())\n\n\ndef find_versions_of_archive(\n archive_urls: Union[str, Sequence[str]],\n list_url: Optional[str] = None,\n list_depth: int = 0,\n concurrency: Optional[int] = 32,\n reference_package: Optional[Any] = None,\n) -> Dict[spack.version.StandardVersion, str]:\n \"\"\"Scrape web pages for new versions of a tarball. This function prefers URLs in the\n following order: links found on the scraped page that match a url generated by the\n reference package, found and in the archive_urls list, found and derived from those\n in the archive_urls list, and if none are found for a version then the item in the\n archive_urls list is included for the version.\n\n Args:\n archive_urls: URL or sequence of URLs for different versions of a package. Typically these\n are just the tarballs from the package file itself. By default, this searches the\n parent directories of archives.\n list_url: URL for a listing of archives. Spack will scrape these pages for download links\n that look like the archive URL.\n list_depth: max depth to follow links on list_url pages. Defaults to 0.\n concurrency: maximum number of concurrent requests\n reference_package: a spack package used as a reference for url detection. Uses the\n url_for_version method on the package to produce reference urls which, if found, are\n preferred.\n \"\"\"\n if isinstance(archive_urls, str):\n archive_urls = [archive_urls]\n\n # Generate a list of list_urls based on archive urls and any\n # explicitly listed list_url in the package\n list_urls = set()\n if list_url is not None:\n list_urls.add(list_url)\n for aurl in archive_urls:\n list_urls |= spack.llnl.url.find_list_urls(aurl)\n\n # Add '/' to the end of the URL. Some web servers require this.\n additional_list_urls = set()\n for lurl in list_urls:\n if not lurl.endswith(\"/\"):\n additional_list_urls.add(lurl + \"/\")\n list_urls |= additional_list_urls\n\n # Grab some web pages to scrape.\n _, links = spack.util.web.spider(list_urls, depth=list_depth, concurrency=concurrency)\n\n # Scrape them for archive URLs\n regexes = []\n for aurl in archive_urls:\n # This creates a regex from the URL with a capture group for\n # the version part of the URL. The capture group is converted\n # to a generic wildcard, so we can use this to extract things\n # on a page that look like archive URLs.\n url_regex = wildcard_version(aurl)\n\n # We'll be a bit more liberal and just look for the archive\n # part, not the full path.\n # this is a URL so it is a posixpath even on Windows\n url_regex = pathlib.PurePosixPath(url_regex).name\n\n # We need to add a / to the beginning of the regex to prevent\n # Spack from picking up similarly named packages like:\n # https://cran.r-project.org/src/contrib/pls_2.6-0.tar.gz\n # https://cran.r-project.org/src/contrib/enpls_5.7.tar.gz\n # https://cran.r-project.org/src/contrib/autopls_1.3.tar.gz\n # https://cran.r-project.org/src/contrib/matrixpls_1.0.4.tar.gz\n url_regex = \"/\" + url_regex\n\n # We need to add a $ anchor to the end of the regex to prevent\n # Spack from picking up signature files like:\n # .asc\n # .md5\n # .sha256\n # .sig\n # However, SourceForge downloads still need to end in '/download'.\n url_regex += r\"(\\/download)?\"\n # PyPI adds #sha256=... to the end of the URL\n url_regex += \"(#sha256=.*)?\"\n url_regex += \"$\"\n\n regexes.append(url_regex)\n\n regexes = [re.compile(r) for r in regexes]\n # Build a dict version -> URL from any links that match the wildcards.\n # Walk through archive_url links first.\n # Any conflicting versions will be overwritten by the list_url links.\n versions: Dict[spack.version.StandardVersion, str] = {}\n matched = set()\n for url in sorted(links):\n url = convert_to_posix_path(url)\n if any(r.search(url) for r in regexes):\n try:\n ver = parse_version(url)\n if ver in matched:\n continue\n versions[ver] = url\n # prevent this version from getting overwritten\n if reference_package is not None:\n if url == reference_package.url_for_version(ver):\n matched.add(ver)\n else:\n extrapolated_urls = [substitute_version(u, ver) for u in archive_urls]\n if url in extrapolated_urls:\n matched.add(ver)\n except UndetectableVersionError:\n continue\n\n for url in archive_urls:\n url = convert_to_posix_path(url)\n ver = parse_version(url)\n if ver not in versions:\n versions[ver] = url\n\n return versions\n\n\nclass UrlParseError(spack.error.SpackError):\n \"\"\"Raised when the URL module can't parse something correctly.\"\"\"\n\n def __init__(self, msg, path):\n super().__init__(msg)\n self.path = path\n\n\nclass UndetectableVersionError(UrlParseError):\n \"\"\"Raised when we can't parse a version from a string.\"\"\"\n\n def __init__(self, path):\n super().__init__(\"Couldn't detect version in: \" + path, path)\n\n\nclass UndetectableNameError(UrlParseError):\n \"\"\"Raised when we can't parse a package name from a string.\"\"\"\n\n def __init__(self, path):\n super().__init__(\"Couldn't parse package name in: \" + path, path)\n" }, { "path": "lib/spack/spack/url_buildcache.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport enum\nimport fnmatch\nimport gzip\nimport io\nimport json\nimport os\nimport re\nimport shutil\nimport urllib.parse\nfrom contextlib import closing, contextmanager\nfrom datetime import datetime\nfrom pathlib import Path\nfrom tempfile import TemporaryDirectory\nfrom typing import Any, Callable, Dict, List, Optional, Tuple, Type\n\nimport spack.vendor.jsonschema\n\nimport spack.config as config\nimport spack.database\nimport spack.error\nimport spack.hash_types as ht\nimport spack.llnl.util.filesystem as fsys\nimport spack.llnl.util.tty as tty\nimport spack.mirrors.mirror\nimport spack.spec\nimport spack.stage\nimport spack.util.crypto\nimport spack.util.gpg\nimport spack.util.url as url_util\nimport spack.util.web as web_util\nfrom spack.schema.url_buildcache_manifest import schema as buildcache_manifest_schema\nfrom spack.util.archive import ChecksumWriter\nfrom spack.util.crypto import hash_fun_for_algo\nfrom spack.util.executable import which\n\n#: The build cache layout version that this version of Spack creates.\n#: Version 3: Introduces content-addressable tarballs\nCURRENT_BUILD_CACHE_LAYOUT_VERSION = 3\n\n#: The name of the default buildcache index manifest file\nINDEX_MANIFEST_FILE = \"index.manifest.json\"\n\n\nclass BuildcacheComponent(enum.Enum):\n \"\"\"Enumeration of the kinds of things that live in a URL buildcache\n\n These enums serve two purposes: They allow different buildcache layout\n versions to specify different relative location of these entities, and\n they're used to map buildcache objects to their respective media types.\n \"\"\"\n\n # manifest files\n MANIFEST = enum.auto()\n # metadata file for a binary package\n SPEC = enum.auto()\n # things that live in the blobs directory\n BLOB = enum.auto()\n # binary mirror index\n INDEX = enum.auto()\n # public key used for verifying signed binary packages\n KEY = enum.auto()\n # index of all public keys found in the mirror\n KEY_INDEX = enum.auto()\n # compressed archive of spec installation directory\n TARBALL = enum.auto()\n # binary mirror descriptor file\n LAYOUT_JSON = enum.auto()\n\n\nclass BlobRecord:\n \"\"\"Class to describe a single data element (blob) from a manifest\"\"\"\n\n def __init__(\n self,\n content_length: int,\n media_type: str,\n compression_alg: str,\n checksum_alg: str,\n checksum: str,\n ) -> None:\n self.content_length = content_length\n self.media_type = media_type\n self.compression_alg = compression_alg\n self.checksum_alg = checksum_alg\n self.checksum = checksum\n\n @classmethod\n def from_dict(cls, record_dict):\n return BlobRecord(\n record_dict[\"contentLength\"],\n record_dict[\"mediaType\"],\n record_dict[\"compression\"],\n record_dict[\"checksumAlgorithm\"],\n record_dict[\"checksum\"],\n )\n\n def to_dict(self):\n return {\n \"contentLength\": self.content_length,\n \"mediaType\": self.media_type,\n \"compression\": self.compression_alg,\n \"checksumAlgorithm\": self.checksum_alg,\n \"checksum\": self.checksum,\n }\n\n\nclass BuildcacheManifest:\n \"\"\"A class to represent a buildcache manifest, which consists of a version\n number and an array of data blobs, each of which is represented by a\n BlobRecord.\"\"\"\n\n def __init__(self, layout_version: int, data: Optional[List[BlobRecord]] = None):\n self.version: int = layout_version\n if data:\n self.data: List[BlobRecord] = [\n BlobRecord(\n rec.content_length,\n rec.media_type,\n rec.compression_alg,\n rec.checksum_alg,\n rec.checksum,\n )\n for rec in data\n ]\n else:\n self.data = []\n\n def to_dict(self):\n return {\"version\": self.version, \"data\": [rec.to_dict() for rec in self.data]}\n\n @classmethod\n def from_dict(cls, manifest_json: Dict[str, Any]) -> \"BuildcacheManifest\":\n spack.vendor.jsonschema.validate(manifest_json, buildcache_manifest_schema)\n return BuildcacheManifest(\n layout_version=manifest_json[\"version\"],\n data=[BlobRecord.from_dict(blob_json) for blob_json in manifest_json[\"data\"]],\n )\n\n def get_blob_records(self, media_type: str) -> List[BlobRecord]:\n \"\"\"Return any blob records from the manifest matching the given media type\"\"\"\n matches: List[BlobRecord] = []\n\n for record in self.data:\n if record.media_type == media_type:\n matches.append(record)\n\n if matches:\n return matches\n\n raise NoSuchBlobException(f\"Manifest has no blobs of type {media_type}\")\n\n\nclass URLBuildcacheEntry:\n \"\"\"A class for managing URL-style buildcache entries\n\n This class manages access to a versioned buildcache entry by providing\n a means to download both the metadata (spec file) and compressed archive.\n It also provides methods for accessing the paths/urls associated with\n buildcache entries.\n\n Starting with buildcache layout version 3, it is not possible to know\n the full path to a compressed archive without either building it locally,\n or else fetching and reading the metadata first. This class provides api\n for fetching the metadata, as well as fetching the archive, and it enforces\n the need to fetch the metadata first.\n\n To help with downloading, this class manages two spack.spec.Stage objects\n internally, which must be destroyed when finished. Specifically, if you\n call either of the following methods on an instance, you must eventually also\n call destroy()::\n\n fetch_metadata()\n fetch_archive()\n\n This class also provides generic manifest and blob management api, and it\n can be used to fetch and push other kinds of buildcache entries aside from\n just binary packages. It can be used to work with public keys, buildcache\n indices, and any other type of data represented as a manifest which refers\n to blobs of data.\n\n \"\"\"\n\n SPEC_URL_REGEX = re.compile(r\"(.+)/v([\\d]+)/manifests/.+\")\n LAYOUT_VERSION = 3\n BUILDCACHE_INDEX_MEDIATYPE = f\"application/vnd.spack.db.v{spack.database._DB_VERSION}+json\"\n SPEC_MEDIATYPE = f\"application/vnd.spack.spec.v{spack.spec.SPECFILE_FORMAT_VERSION}+json\"\n TARBALL_MEDIATYPE = \"application/vnd.spack.install.v2.tar+gzip\"\n PUBLIC_KEY_MEDIATYPE = \"application/pgp-keys\"\n PUBLIC_KEY_INDEX_MEDIATYPE = \"application/vnd.spack.keyindex.v1+json\"\n BUILDCACHE_INDEX_FILE = \"index.manifest.json\"\n COMPONENT_PATHS = {\n BuildcacheComponent.MANIFEST: [f\"v{LAYOUT_VERSION}\", \"manifests\"],\n BuildcacheComponent.BLOB: [\"blobs\"],\n BuildcacheComponent.INDEX: [f\"v{LAYOUT_VERSION}\", \"manifests\", \"index\"],\n BuildcacheComponent.KEY: [f\"v{LAYOUT_VERSION}\", \"manifests\", \"key\"],\n BuildcacheComponent.SPEC: [f\"v{LAYOUT_VERSION}\", \"manifests\", \"spec\"],\n BuildcacheComponent.KEY_INDEX: [f\"v{LAYOUT_VERSION}\", \"manifests\", \"key\"],\n BuildcacheComponent.TARBALL: [\"blobs\"],\n BuildcacheComponent.LAYOUT_JSON: [f\"v{LAYOUT_VERSION}\", \"layout.json\"],\n }\n\n def __init__(\n self, mirror_url: str, spec: Optional[spack.spec.Spec] = None, allow_unsigned: bool = False\n ):\n \"\"\"Lazily initialize the object\"\"\"\n self.mirror_url: str = mirror_url\n self.spec: Optional[spack.spec.Spec] = spec\n self.allow_unsigned: bool = allow_unsigned\n self.manifest: Optional[BuildcacheManifest] = None\n self.remote_manifest_url: str = \"\"\n self.stages: Dict[BlobRecord, spack.stage.Stage] = {}\n\n @classmethod\n def get_layout_version(cls) -> int:\n \"\"\"Returns the layout version of this class\"\"\"\n return cls.LAYOUT_VERSION\n\n @classmethod\n def check_layout_json_exists(cls, mirror_url: str) -> bool:\n \"\"\"Return True if layout.json exists in the expected location, False otherwise\"\"\"\n layout_json_url = url_util.join(\n mirror_url, *cls.get_relative_path_components(BuildcacheComponent.LAYOUT_JSON)\n )\n return web_util.url_exists(layout_json_url)\n\n @classmethod\n def maybe_push_layout_json(cls, mirror_url: str) -> None:\n \"\"\"This function does nothing if layout.json already exists, otherwise it\n pushes layout.json to the expected location in the mirror\"\"\"\n if cls.check_layout_json_exists(mirror_url):\n return\n\n layout_contents = {\"signing\": \"gpg\"}\n\n with TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir:\n local_layout_path = os.path.join(tmpdir, \"layout.json\")\n with open(local_layout_path, \"w\", encoding=\"utf-8\") as fd:\n json.dump(layout_contents, fd)\n remote_layout_url = url_util.join(\n mirror_url, *cls.get_relative_path_components(BuildcacheComponent.LAYOUT_JSON)\n )\n web_util.push_to_url(local_layout_path, remote_layout_url, keep_original=False)\n\n @classmethod\n def get_base_url(cls, manifest_url: str) -> str:\n \"\"\"Given any manifest url (i.e. one containing ``v3/manifests/``) return the\n base part of the url\"\"\"\n rematch = cls.SPEC_URL_REGEX.match(manifest_url)\n if not rematch:\n raise BuildcacheEntryError(f\"Unable to parse spec url: {manifest_url}\")\n return rematch.group(1)\n\n @classmethod\n def get_index_url(cls, mirror_url: str, view: Optional[str] = None):\n return url_util.join(\n mirror_url,\n *cls.get_relative_path_components(BuildcacheComponent.INDEX),\n url_util.join(view or \"\", cls.BUILDCACHE_INDEX_FILE),\n )\n\n @classmethod\n def get_relative_path_components(cls, component: BuildcacheComponent) -> List[str]:\n \"\"\"Given any type of buildcache component, return its relative location within\n a mirror as a list path elements\"\"\"\n return cls.COMPONENT_PATHS[component]\n\n @classmethod\n def get_manifest_filename(cls, spec: spack.spec.Spec) -> str:\n \"\"\"Given a concrete spec, compute and return the name (i.e. basename) of\n the manifest file representing it\"\"\"\n spec_formatted = spec.format_path(\"{name}-{version}-{hash}\")\n return f\"{spec_formatted}.spec.manifest.json\"\n\n @classmethod\n def get_manifest_url(cls, spec: spack.spec.Spec, mirror_url: str) -> str:\n \"\"\"Given a concrete spec and a base url, return the full url where the\n spec manifest should be found\"\"\"\n path_components = cls.get_relative_path_components(BuildcacheComponent.SPEC)\n return url_util.join(\n mirror_url, *path_components, spec.name, cls.get_manifest_filename(spec)\n )\n\n @classmethod\n def get_buildcache_component_include_pattern(\n cls, buildcache_component: BuildcacheComponent\n ) -> str:\n \"\"\"Given a buildcache component, return the glob pattern that can be used\n to match it in a directory listing. If None is provided, return a catch-all\n pattern that will match all buildcache components.\"\"\"\n if buildcache_component is BuildcacheComponent.MANIFEST:\n return \"*.manifest.json\"\n elif buildcache_component == BuildcacheComponent.SPEC:\n return \"*.spec.manifest.json\"\n elif buildcache_component == BuildcacheComponent.INDEX:\n return \".*index.manifest.json\"\n elif buildcache_component == BuildcacheComponent.KEY:\n return \"*.key.manifest.json\"\n elif buildcache_component == BuildcacheComponent.KEY_INDEX:\n return \"keys.manifest.json\"\n\n raise BuildcacheEntryError(f\"Not a manifest component: {buildcache_component}\")\n\n @classmethod\n def component_to_media_type(cls, component: BuildcacheComponent) -> str:\n \"\"\"Mapping from buildcache component to media type\"\"\"\n if component == BuildcacheComponent.SPEC:\n return cls.SPEC_MEDIATYPE\n elif component == BuildcacheComponent.TARBALL:\n return cls.TARBALL_MEDIATYPE\n elif component == BuildcacheComponent.INDEX:\n return cls.BUILDCACHE_INDEX_MEDIATYPE\n elif component == BuildcacheComponent.KEY:\n return cls.PUBLIC_KEY_MEDIATYPE\n elif component == BuildcacheComponent.KEY_INDEX:\n return cls.PUBLIC_KEY_INDEX_MEDIATYPE\n\n raise BuildcacheEntryError(f\"Not a blob component: {component}\")\n\n def get_local_spec_path(self) -> str:\n \"\"\"Convenience method to return the local path of a fetched spec file\"\"\"\n return self.get_staged_blob_path(self.get_blob_record(BuildcacheComponent.SPEC))\n\n def get_local_archive_path(self) -> str:\n \"\"\"Convenience method to return the local path of a fetched tarball\"\"\"\n return self.get_staged_blob_path(self.get_blob_record(BuildcacheComponent.TARBALL))\n\n def get_blob_record(self, blob_type: BuildcacheComponent) -> BlobRecord:\n \"\"\"Return the first blob record of the given type. Assumes the manifest has\n already been fetched.\"\"\"\n if not self.manifest:\n raise BuildcacheEntryError(\"Read manifest before accessing blob records\")\n\n records = self.manifest.get_blob_records(self.component_to_media_type(blob_type))\n\n if len(records) == 0:\n raise BuildcacheEntryError(f\"Manifest has no blob record of type {blob_type}\")\n\n return records[0]\n\n def check_blob_exists(self, record: BlobRecord) -> bool:\n \"\"\"Return True if the blob given by record exists on the mirror, False otherwise\"\"\"\n blob_url = self.get_blob_url(self.mirror_url, record)\n return web_util.url_exists(blob_url)\n\n @classmethod\n def get_blob_path_components(cls, record: BlobRecord) -> List[str]:\n \"\"\"Given a BlobRecord, return the relative path of the blob within a mirror\n as a list of path components\"\"\"\n return [\n *cls.get_relative_path_components(BuildcacheComponent.BLOB),\n record.checksum_alg,\n record.checksum[:2],\n record.checksum,\n ]\n\n @classmethod\n def get_blob_url(cls, mirror_url: str, record: BlobRecord) -> str:\n \"\"\"Return the full url of the blob given by record\"\"\"\n return url_util.join(mirror_url, *cls.get_blob_path_components(record))\n\n def fetch_blob(self, record: BlobRecord) -> str:\n \"\"\"Given a blob record, find associated blob in the manifest and stage it\n\n Returns the local path to the staged blob\n \"\"\"\n if record not in self.stages:\n blob_url = self.get_blob_url(self.mirror_url, record)\n blob_stage = spack.stage.Stage(blob_url)\n\n # Fetch the blob, or else cleanup and exit early\n try:\n blob_stage.create()\n blob_stage.fetch()\n except spack.error.FetchError as e:\n self.destroy()\n raise BuildcacheEntryError(f\"Unable to fetch blob from {blob_url}\") from e\n\n # Raises if checksum does not match expectation\n validate_checksum(blob_stage.save_filename, record.checksum_alg, record.checksum)\n\n self.stages[record] = blob_stage\n\n return self.get_staged_blob_path(record)\n\n def get_staged_blob_path(self, record: BlobRecord) -> str:\n \"\"\"Convenience method to return the local path of a staged blob\"\"\"\n if record not in self.stages:\n raise BuildcacheEntryError(f\"Blob not staged: {record}\")\n\n return self.stages[record].save_filename\n\n def exists(self, components: List[BuildcacheComponent]) -> bool:\n \"\"\"Check whether blobs exist for all specified components\n\n Returns True if there is a blob present in the mirror for every\n given component type.\n \"\"\"\n try:\n self.read_manifest()\n except BuildcacheEntryError:\n return False\n\n if not self.manifest:\n return False\n\n for component in components:\n component_blobs = self.manifest.get_blob_records(\n self.component_to_media_type(component)\n )\n\n if len(component_blobs) == 0:\n return False\n\n if not self.check_blob_exists(component_blobs[0]):\n return False\n\n return True\n\n @classmethod\n def verify_and_extract_manifest(cls, manifest_contents: str, verify: bool = False) -> dict:\n \"\"\"Possibly verify clearsig, then extract contents and return as json\"\"\"\n magic_string = \"-----BEGIN PGP SIGNED MESSAGE-----\"\n if manifest_contents.startswith(magic_string):\n if verify:\n # Try to verify and raise if we fail\n with TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir:\n manifest_path = os.path.join(tmpdir, \"manifest.json.sig\")\n with open(manifest_path, \"w\", encoding=\"utf-8\") as fd:\n fd.write(manifest_contents)\n if not try_verify(manifest_path):\n raise NoVerifyException(\"Signature could not be verified\")\n\n return spack.spec.Spec.extract_json_from_clearsig(manifest_contents)\n elif verify:\n raise NoVerifyException(\"Required signature was not found on manifest\")\n return json.loads(manifest_contents)\n\n def read_manifest(self, manifest_url: Optional[str] = None) -> BuildcacheManifest:\n \"\"\"Read and process the the buildcache entry manifest.\n\n If no manifest url is provided, build the url from the internal spec and\n base push url.\"\"\"\n\n if self.manifest:\n if not manifest_url or manifest_url == self.remote_manifest_url:\n # We already have a manifest, so now calling this method without a specific\n # manifest url, or with the same one we have internally, then skip reading\n # again, and just return the manifest we already read.\n return self.manifest\n\n self.manifest = None\n\n if not manifest_url:\n if not self.spec or not self.mirror_url:\n raise BuildcacheEntryError(\n \"Either manifest url or spec and mirror are required to read manifest\"\n )\n manifest_url = self.get_manifest_url(self.spec, self.mirror_url)\n\n self.remote_manifest_url = manifest_url\n manifest_contents = \"\"\n\n try:\n manifest_contents = web_util.read_text(manifest_url)\n except (web_util.SpackWebError, OSError) as e:\n raise BuildcacheEntryError(f\"Error reading manifest at {manifest_url}\") from e\n\n if not manifest_contents:\n raise BuildcacheEntryError(\"Unable to read manifest or manifest empty\")\n\n manifest_contents = self.verify_and_extract_manifest(\n manifest_contents, verify=not self.allow_unsigned\n )\n\n self.manifest = BuildcacheManifest.from_dict(manifest_contents)\n\n if self.manifest.version != 3:\n raise BuildcacheEntryError(\"Layout version mismatch in fetched manifest\")\n\n return self.manifest\n\n def fetch_metadata(self) -> dict:\n \"\"\"Retrieve metadata for the spec, returns the validated spec dict\"\"\"\n if not self.manifest:\n # Reading the manifest will either successfully compute the remote\n # spec url, or else raise an exception\n self.read_manifest()\n\n local_specfile_path = self.fetch_blob(self.get_blob_record(BuildcacheComponent.SPEC))\n\n # Check spec file for validity and read it, or else cleanup and exit early\n try:\n spec_dict, _ = get_valid_spec_file(local_specfile_path, self.get_layout_version())\n except InvalidMetadataFile as e:\n self.destroy()\n raise BuildcacheEntryError(\"Buildcache entry does not have valid metadata file\") from e\n\n return spec_dict\n\n def fetch_archive(self) -> str:\n \"\"\"Retrieve the archive file and return the local archive file path\"\"\"\n if not self.manifest:\n # Raises if problems encountered, including not being able to verify signagure\n self.read_manifest()\n\n return self.fetch_blob(self.get_blob_record(BuildcacheComponent.TARBALL))\n\n def get_archive_stage(self) -> Optional[spack.stage.Stage]:\n return self.stages[self.get_blob_record(BuildcacheComponent.TARBALL)]\n\n def remove(self):\n \"\"\"Remove a binary package (spec file and tarball) and the associated\n manifest from the mirror.\"\"\"\n if self.manifest:\n try:\n web_util.remove_url(self.remote_manifest_url)\n except Exception as e:\n tty.debug(f\"Failed to remove previous manfifest: {e}\")\n\n try:\n web_util.remove_url(\n self.get_blob_url(\n self.mirror_url, self.get_blob_record(BuildcacheComponent.TARBALL)\n )\n )\n except Exception as e:\n tty.debug(f\"Failed to remove previous archive: {e}\")\n\n try:\n web_util.remove_url(\n self.get_blob_url(\n self.mirror_url, self.get_blob_record(BuildcacheComponent.SPEC)\n )\n )\n except Exception as e:\n tty.debug(f\"Failed to remove previous metadata: {e}\")\n\n self.manifest = None\n\n @classmethod\n def push_blob(cls, mirror_url: str, blob_path: str, record: BlobRecord) -> None:\n \"\"\"Push the blob_path file to mirror as a blob represented by the given\n record\"\"\"\n blob_destination_url = cls.get_blob_url(mirror_url, record)\n web_util.push_to_url(blob_path, blob_destination_url, keep_original=False)\n\n @classmethod\n def push_manifest(\n cls,\n mirror_url: str,\n manifest_name: str,\n manifest: BuildcacheManifest,\n tmpdir: str,\n component_type: BuildcacheComponent = BuildcacheComponent.SPEC,\n signing_key: Optional[str] = None,\n ) -> None:\n \"\"\"Given a BuildcacheManifest, push it to the mirror using the given manifest\n name. The component_type is used to indicate what type of thing the manifest\n represents, so it can be placed in the correct relative path within the mirror.\n If a signing_key is provided, it will be used to clearsign the manifest before\n pushing it.\"\"\"\n # write the manifest to a temporary location\n manifest_file_name = f\"{manifest_name}.manifest.json\"\n manifest_path = os.path.join(tmpdir, manifest_file_name)\n os.makedirs(os.path.dirname(manifest_path), exist_ok=True)\n with open(manifest_path, \"w\", encoding=\"utf-8\") as f:\n json.dump(manifest.to_dict(), f, indent=0, separators=(\",\", \":\"))\n # Note: when using gpg clear sign, we need to avoid long lines (19995\n # chars). If lines are longer, they are truncated without error. So,\n # here we still add newlines, but no indent, so save on file size and\n # line length.\n\n if signing_key:\n manifest_path = sign_file(signing_key, manifest_path)\n\n manifest_destination_url = url_util.join(\n mirror_url, *cls.get_relative_path_components(component_type), manifest_file_name\n )\n\n web_util.push_to_url(manifest_path, manifest_destination_url, keep_original=False)\n\n @classmethod\n def push_local_file_as_blob(\n cls,\n local_file_path: str,\n mirror_url: str,\n manifest_name: str,\n component_type: BuildcacheComponent,\n compression: str = \"none\",\n ) -> None:\n \"\"\"Convenience method to push a local file to a mirror as a blob. Both manifest\n and blob are pushed as a component of the given component_type. If ``compression``\n is ``\"gzip\"`` the blob will be compressed before pushing, otherwise it will be pushed\n uncompressed.\"\"\"\n cache_class = get_url_buildcache_class()\n checksum_algo = \"sha256\"\n blob_to_push = local_file_path\n\n with TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir:\n blob_to_push = os.path.join(tmpdir, os.path.basename(local_file_path))\n\n with compression_writer(blob_to_push, compression, checksum_algo) as (\n fout,\n checker,\n ), open(local_file_path, \"rb\") as fin:\n shutil.copyfileobj(fin, fout)\n\n record = BlobRecord(\n checker.length,\n cache_class.component_to_media_type(component_type),\n compression,\n checksum_algo,\n checker.hexdigest(),\n )\n manifest = BuildcacheManifest(\n layout_version=CURRENT_BUILD_CACHE_LAYOUT_VERSION, data=[record]\n )\n cls.push_blob(mirror_url, blob_to_push, record)\n cls.push_manifest(\n mirror_url, manifest_name, manifest, tmpdir, component_type=component_type\n )\n\n def push_binary_package(\n self,\n spec: spack.spec.Spec,\n tarball_path: str,\n checksum_algorithm: str,\n tarball_checksum: str,\n tmpdir: str,\n signing_key: Optional[str],\n ) -> None:\n \"\"\"Convenience method to push tarball, specfile, and manifest to the remote mirror\n\n Pushing should only be done after checking for the pre-existence of a\n buildcache entry for this spec, and represents a force push if one is\n found. Thus, any pre-existing files are first removed.\n \"\"\"\n\n spec_dict = spec.to_dict(hash=ht.dag_hash)\n # TODO: Remove this key once oci buildcache no longer uses it\n spec_dict[\"buildcache_layout_version\"] = 2\n tarball_content_length = os.stat(tarball_path).st_size\n compression = \"gzip\"\n\n # Delete the previously existing version\n self.remove()\n\n if not self.remote_manifest_url:\n self.remote_manifest_url = self.get_manifest_url(spec, self.mirror_url)\n\n # Any previous archive/tarball is gone, compute the path to the new one\n remote_archive_url = url_util.join(\n self.mirror_url,\n *self.get_relative_path_components(BuildcacheComponent.BLOB),\n checksum_algorithm,\n tarball_checksum[:2],\n tarball_checksum,\n )\n\n # push the archive/tarball blob to the remote\n web_util.push_to_url(tarball_path, remote_archive_url, keep_original=False)\n\n # Clear out the previous data, then add a record for the new blob\n blobs: List[BlobRecord] = []\n blobs.append(\n BlobRecord(\n tarball_content_length,\n self.TARBALL_MEDIATYPE,\n compression,\n checksum_algorithm,\n tarball_checksum,\n )\n )\n\n # compress the spec dict and compute its checksum\n specfile = os.path.join(tmpdir, f\"{spec.dag_hash()}.spec.json\")\n metadata_checksum, metadata_size = compressed_json_from_dict(\n specfile, spec_dict, checksum_algorithm\n )\n\n # Any previous metadata blob is gone, compute the path to the new one\n remote_spec_url = url_util.join(\n self.mirror_url,\n *self.get_relative_path_components(BuildcacheComponent.BLOB),\n checksum_algorithm,\n metadata_checksum[:2],\n metadata_checksum,\n )\n\n # push the metadata/spec blob to the remote\n web_util.push_to_url(specfile, remote_spec_url, keep_original=False)\n\n blobs.append(\n BlobRecord(\n metadata_size,\n self.SPEC_MEDIATYPE,\n compression,\n checksum_algorithm,\n metadata_checksum,\n )\n )\n\n # generate the manifest\n manifest = {\n \"version\": self.get_layout_version(),\n \"data\": [record.to_dict() for record in blobs],\n }\n\n # write the manifest to a temporary location\n manifest_path = os.path.join(tmpdir, f\"{spec.dag_hash()}.manifest.json\")\n with open(manifest_path, \"w\", encoding=\"utf-8\") as f:\n json.dump(manifest, f, indent=0, separators=(\",\", \":\"))\n # Note: when using gpg clear sign, we need to avoid long lines (19995\n # chars). If lines are longer, they are truncated without error. So,\n # here we still add newlines, but no indent, so save on file size and\n # line length.\n\n # possibly sign the manifest\n if signing_key:\n manifest_path = sign_file(signing_key, manifest_path)\n\n # Push the manifest file to the remote. The remote manifest url for\n # a given concrete spec is fixed, so we don't have to recompute it,\n # even if we deleted the pre-existing one.\n web_util.push_to_url(manifest_path, self.remote_manifest_url, keep_original=False)\n\n def destroy(self):\n \"\"\"Destroy any existing stages\"\"\"\n for blob_stage in self.stages.values():\n blob_stage.destroy()\n\n self.stages = {}\n\n\nclass URLBuildcacheEntryV2(URLBuildcacheEntry):\n \"\"\"This class exists to provide read-only support for reading older buildcache\n layouts in a way that is transparent to binary_distribution code responsible for\n downloading and extracting binary packages. Since support for layout v2 is\n read-only, and since v2 did not have support for manifests and blobs, many class\n and instance methods are overridden simply to raise, hopefully making the intended\n use and limitations of the class clear to developers.\"\"\"\n\n SPEC_URL_REGEX = re.compile(r\"(.+)/build_cache/.+\")\n LAYOUT_VERSION = 2\n BUILDCACHE_INDEX_FILE = \"index.json\"\n COMPONENT_PATHS = {\n BuildcacheComponent.MANIFEST: [\"build_cache\"],\n BuildcacheComponent.BLOB: [\"build_cache\"],\n BuildcacheComponent.INDEX: [\"build_cache\"],\n BuildcacheComponent.KEY: [\"build_cache\", \"_pgp\"],\n BuildcacheComponent.SPEC: [\"build_cache\"],\n BuildcacheComponent.KEY_INDEX: [\"build_cache\", \"_pgp\"],\n BuildcacheComponent.TARBALL: [\"build_cache\"],\n BuildcacheComponent.LAYOUT_JSON: [\"build_cache\", \"layout.json\"],\n }\n\n def __init__(\n self,\n push_url_base: str,\n spec: Optional[spack.spec.Spec] = None,\n allow_unsigned: bool = False,\n ):\n \"\"\"Lazily initialize the object\"\"\"\n self.mirror_url: str = push_url_base\n self.spec: Optional[spack.spec.Spec] = spec\n self.allow_unsigned: bool = allow_unsigned\n\n self.has_metadata: bool = False\n self.has_tarball: bool = False\n self.has_signed: bool = False\n self.has_unsigned: bool = False\n self.spec_stage: Optional[spack.stage.Stage] = None\n self.local_specfile_path: str = \"\"\n self.archive_stage: Optional[spack.stage.Stage] = None\n self.local_archive_path: str = \"\"\n\n self.remote_spec_url: str = \"\"\n self.remote_archive_url: str = \"\"\n self.remote_archive_checksum_algorithm: str = \"\"\n self.remote_archive_checksum_hash: str = \"\"\n self.spec_dict: Dict[Any, Any] = {}\n\n self._checked_signed = False\n self._checked_unsigned = False\n self._checked_exists = False\n\n @classmethod\n def get_layout_version(cls) -> int:\n return cls.LAYOUT_VERSION\n\n @classmethod\n def maybe_push_layout_json(cls, mirror_url: str) -> None:\n raise BuildcacheEntryError(\"spack can no longer write to v2 buildcaches\")\n\n def _get_spec_url(\n self, spec: spack.spec.Spec, mirror_url: str, ext: str = \".spec.json.sig\"\n ) -> str:\n spec_formatted = spec.format_path(\n \"{architecture}-{compiler.name}-{compiler.version}-{name}-{version}-{hash}\"\n )\n path_components = self.get_relative_path_components(BuildcacheComponent.SPEC)\n return url_util.join(mirror_url, *path_components, f\"{spec_formatted}{ext}\")\n\n def _get_tarball_url(self, spec: spack.spec.Spec, mirror_url: str) -> str:\n directory_name = spec.format_path(\n \"{architecture}/{compiler.name}-{compiler.version}/{name}-{version}\"\n )\n spec_formatted = spec.format_path(\n \"{architecture}-{compiler.name}-{compiler.version}-{name}-{version}-{hash}\"\n )\n filename = f\"{spec_formatted}.spack\"\n return url_util.join(\n mirror_url,\n *self.get_relative_path_components(BuildcacheComponent.BLOB),\n directory_name,\n filename,\n )\n\n def _check_metadata_exists(self):\n if not self.spec:\n return\n\n if not self._checked_signed:\n signed_url = self._get_spec_url(self.spec, self.mirror_url, ext=\".spec.json.sig\")\n if web_util.url_exists(signed_url):\n self.remote_spec_url = signed_url\n self.has_signed = True\n self._checked_signed = True\n\n if not self.has_signed and not self._checked_unsigned:\n unsigned_url = self._get_spec_url(self.spec, self.mirror_url, ext=\".spec.json\")\n if web_util.url_exists(unsigned_url):\n self.remote_spec_url = unsigned_url\n self.has_unsigned = True\n self._checked_unsigned = True\n\n def exists(self, components: List[BuildcacheComponent]) -> bool:\n if not self.spec:\n return False\n\n if (\n len(components) != 2\n or BuildcacheComponent.SPEC not in components\n or BuildcacheComponent.TARBALL not in components\n ):\n return False\n\n self._check_metadata_exists()\n if not self.has_signed and not self.has_unsigned:\n return False\n\n if not web_util.url_exists(self._get_tarball_url(self.spec, self.mirror_url)):\n return False\n\n return True\n\n def fetch_metadata(self) -> dict:\n \"\"\"Retrieve the v2 specfile for the spec, yields the validated spec+ dict\"\"\"\n if self.spec_dict:\n # Only fetch the metadata once\n return self.spec_dict\n\n self._check_metadata_exists()\n\n if not self.remote_spec_url:\n raise BuildcacheEntryError(f\"Mirror {self.mirror_url} does not have metadata for spec\")\n\n if not self.allow_unsigned and self.has_unsigned:\n raise BuildcacheEntryError(\n f\"Mirror {self.mirror_url} does not have signed metadata for spec\"\n )\n\n self.spec_stage = spack.stage.Stage(self.remote_spec_url)\n\n # Fetch the spec file, or else cleanup and exit early\n try:\n self.spec_stage.create()\n self.spec_stage.fetch()\n except spack.error.FetchError as e:\n self.destroy()\n raise BuildcacheEntryError(\n f\"Unable to fetch metadata from {self.remote_spec_url}\"\n ) from e\n\n self.local_specfile_path = self.spec_stage.save_filename\n\n if not self.allow_unsigned and not try_verify(self.local_specfile_path):\n raise NoVerifyException(f\"Signature on {self.remote_spec_url} could not be verified\")\n\n # Check spec file for validity and read it, or else cleanup and exit early\n try:\n spec_dict, _ = get_valid_spec_file(self.local_specfile_path, self.get_layout_version())\n except InvalidMetadataFile as e:\n self.destroy()\n raise BuildcacheEntryError(\"Buildcache entry does not have valid metadata file\") from e\n\n try:\n self.spec = spack.spec.Spec.from_dict(spec_dict)\n except Exception as err:\n raise BuildcacheEntryError(\"Fetched spec dict does not contain valid spec\") from err\n\n self.spec_dict = spec_dict\n\n # Retrieve the alg and hash from the spec dict, use them to build the path to\n # the tarball.\n if \"binary_cache_checksum\" not in self.spec_dict:\n raise BuildcacheEntryError(\"Provided spec dict must contain 'binary_cache_checksum'\")\n\n bchecksum = self.spec_dict[\"binary_cache_checksum\"]\n\n if \"hash_algorithm\" not in bchecksum or \"hash\" not in bchecksum:\n raise BuildcacheEntryError(\n \"Provided spec dict contains invalid 'binary_cache_checksum'\"\n )\n\n self.remote_archive_checksum_algorithm = bchecksum[\"hash_algorithm\"]\n self.remote_archive_checksum_hash = bchecksum[\"hash\"]\n self.remote_archive_url = self._get_tarball_url(self.spec, self.mirror_url)\n\n return self.spec_dict\n\n def fetch_archive(self) -> str:\n self.fetch_metadata()\n\n # Adding this, we can avoid passing a dictionary of stages around the\n # install logic, and in fact completely avoid fetching the metadata in\n # the new (v3) approach.\n if self.spec_stage:\n self.spec_stage.destroy()\n self.spec_stage = None\n\n self.archive_stage = spack.stage.Stage(self.remote_archive_url)\n\n # Fetch the archive file, or else cleanup and exit early\n try:\n self.archive_stage.create()\n self.archive_stage.fetch()\n except spack.error.FetchError as e:\n self.destroy()\n raise BuildcacheEntryError(\n f\"Unable to fetch archive from {self.remote_archive_url}\"\n ) from e\n\n self.local_archive_path = self.archive_stage.save_filename\n\n # Raises if checksum does not match expected\n validate_checksum(\n self.local_archive_path,\n self.remote_archive_checksum_algorithm,\n self.remote_archive_checksum_hash,\n )\n\n return self.local_archive_path\n\n def get_archive_stage(self) -> Optional[spack.stage.Stage]:\n return self.archive_stage\n\n @classmethod\n def get_manifest_filename(cls, spec: spack.spec.Spec) -> str:\n raise BuildcacheEntryError(\"v2 buildcache entries do not have a manifest file\")\n\n @classmethod\n def get_manifest_url(cls, spec: spack.spec.Spec, mirror_url: str) -> str:\n raise BuildcacheEntryError(\"v2 buildcache entries do not have a manifest url\")\n\n @classmethod\n def get_buildcache_component_include_pattern(\n cls, buildcache_component: BuildcacheComponent\n ) -> str:\n raise BuildcacheEntryError(\"v2 buildcache entries do not have a manifest file\")\n\n def read_manifest(self, manifest_url: Optional[str] = None) -> BuildcacheManifest:\n raise BuildcacheEntryError(\"v2 buildcache entries do not have a manifest file\")\n\n def remove(self):\n raise BuildcacheEntryError(\"Spack cannot delete v2 buildcache entries\")\n\n def get_blob_record(self, blob_type: BuildcacheComponent) -> BlobRecord:\n raise BuildcacheEntryError(\"v2 buildcache layout is unaware of manifests and blobs\")\n\n def check_blob_exists(self, record: BlobRecord) -> bool:\n raise BuildcacheEntryError(\"v2 buildcache layout is unaware of manifests and blobs\")\n\n @classmethod\n def get_blob_path_components(cls, record: BlobRecord) -> List[str]:\n raise BuildcacheEntryError(\"v2 buildcache layout is unaware of manifests and blobs\")\n\n @classmethod\n def get_blob_url(cls, mirror_url: str, record: BlobRecord) -> str:\n raise BuildcacheEntryError(\"v2 buildcache layout is unaware of manifests and blobs\")\n\n def fetch_blob(self, record: BlobRecord) -> str:\n raise BuildcacheEntryError(\"v2 buildcache layout is unaware of manifests and blobs\")\n\n def get_staged_blob_path(self, record: BlobRecord) -> str:\n raise BuildcacheEntryError(\"v2 buildcache layout is unaware of manifests and blobs\")\n\n @classmethod\n def verify_and_extract_manifest(cls, manifest_contents: str, verify: bool = False) -> dict:\n raise BuildcacheEntryError(\"v2 buildcache entries do not have a manifest file\")\n\n @classmethod\n def push_blob(cls, mirror_url: str, blob_path: str, record: BlobRecord) -> None:\n raise BuildcacheEntryError(\"v2 buildcache layout is unaware of manifests and blobs\")\n\n @classmethod\n def push_manifest(\n cls,\n mirror_url: str,\n manifest_name: str,\n manifest: BuildcacheManifest,\n tmpdir: str,\n component_type: BuildcacheComponent = BuildcacheComponent.SPEC,\n signing_key: Optional[str] = None,\n ) -> None:\n raise BuildcacheEntryError(\"v2 buildcache layout is unaware of manifests and blobs\")\n\n @classmethod\n def push_local_file_as_blob(\n cls,\n local_file_path: str,\n mirror_url: str,\n manifest_name: str,\n component_type: BuildcacheComponent,\n compression: str = \"none\",\n ) -> None:\n raise BuildcacheEntryError(\"v2 buildcache layout is unaware of manifests and blobs\")\n\n def push_binary_package(\n self,\n spec: spack.spec.Spec,\n tarball_path: str,\n checksum_algorithm: str,\n tarball_checksum: str,\n tmpdir: str,\n signing_key: Optional[str],\n ) -> None:\n raise BuildcacheEntryError(\"Spack can no longer push v2 buildcache entries\")\n\n def destroy(self):\n if self.archive_stage:\n self.archive_stage.destroy()\n self.archive_stage = None\n if self.spec_stage:\n self.spec_stage.destroy()\n self.spec_stage = None\n\n\ndef get_url_buildcache_class(\n layout_version: int = CURRENT_BUILD_CACHE_LAYOUT_VERSION,\n) -> Type[URLBuildcacheEntry]:\n \"\"\"Given a layout version, return the class responsible for managing access\n to buildcache entries of that version\"\"\"\n if layout_version == 2:\n return URLBuildcacheEntryV2\n elif layout_version == 3:\n return URLBuildcacheEntry\n else:\n raise UnknownBuildcacheLayoutError(\n f\"Cannot create buildcache class for unknown layout version {layout_version}\"\n )\n\n\ndef check_mirror_for_layout(mirror: spack.mirrors.mirror.Mirror):\n \"\"\"Check specified mirror, and warn if missing layout.json\"\"\"\n cache_class = get_url_buildcache_class()\n if not cache_class.check_layout_json_exists(mirror.fetch_url):\n msg = (\n f\"Configured mirror {mirror.name} is missing layout.json and has either \\n\"\n \" never been pushed or is of an old layout version. If it's the latter, \\n\"\n \" consider running 'spack buildcache migrate' or rebuilding the specs in \\n\"\n \" in this mirror.\"\n )\n tty.warn(msg)\n\n\ndef _entries_from_cache_aws_cli(url: str, tmpspecsdir: str, component_type: BuildcacheComponent):\n \"\"\"Use aws cli to sync all manifests into a local temporary directory.\n\n Args:\n url: prefix of the build cache on s3\n tmpspecsdir: path to temporary directory to use for writing files\n component_type: type of buildcache component to sync (spec, index, key, etc.)\n\n Return:\n A tuple where the first item is a list of local file paths pointing\n to the manifests that should be read from the mirror, and the\n second item is a function taking a url or file path and returning\n a :class:`URLBuildcacheEntry` for that manifest.\n \"\"\"\n read_fn = None\n file_list = None\n aws = which(\"aws\")\n\n cache_class = get_url_buildcache_class(layout_version=CURRENT_BUILD_CACHE_LAYOUT_VERSION)\n if not aws:\n tty.warn(\"Failed to use aws s3 sync to retrieve specs, falling back to parallel fetch\")\n return file_list, read_fn\n\n def file_read_method(manifest_path: str) -> URLBuildcacheEntry:\n cache_entry = cache_class(mirror_url=url, allow_unsigned=True)\n cache_entry.read_manifest(manifest_url=manifest_path)\n return cache_entry\n\n include_pattern = cache_class.get_buildcache_component_include_pattern(component_type)\n component_prefix = cache_class.get_relative_path_components(component_type)\n\n sync_command_args = [\n \"s3\",\n \"sync\",\n \"--exclude\",\n \"*\",\n \"--include\",\n include_pattern,\n url_util.join(url, *component_prefix),\n tmpspecsdir,\n ]\n\n # Use aws s3 ls to get mtimes of manifests\n ls_command_args = [\"s3\", \"ls\", \"--recursive\", url]\n s3_ls_regex = re.compile(r\"^(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2})\\s+\\d+\\s+(.+)$\")\n\n filename_to_mtime: Dict[str, float] = {}\n\n tty.debug(f\"Using aws s3 sync to download manifests from {url} to {tmpspecsdir}\")\n\n try:\n aws(*sync_command_args, output=os.devnull, error=os.devnull)\n file_list = fsys.find(tmpspecsdir, [include_pattern])\n read_fn = file_read_method\n\n # Use `aws s3 ls` to get mtimes of manifests\n for line in aws(*ls_command_args, output=str, error=os.devnull).splitlines():\n match = s3_ls_regex.match(line)\n if match:\n # Parse the url and use the S3 path of the file to derive the\n # local path of the file (i.e. where `aws s3 sync` put it).\n parsed_url = urllib.parse.urlparse(url)\n s3_path = parsed_url.path.lstrip(\"/\")\n filename = match.group(2)\n if s3_path and filename.startswith(s3_path):\n filename = filename[len(s3_path) :].lstrip(\"/\")\n local_path = url_util.join(tmpspecsdir, filename)\n\n if Path(local_path).exists():\n filename_to_mtime[url_util.path_to_file_url(local_path)] = datetime.strptime(\n match.group(1), \"%Y-%m-%d %H:%M:%S\"\n ).timestamp()\n except Exception as e:\n tty.warn(\"Failed to use aws s3 sync to retrieve specs, falling back to parallel fetch\")\n raise e\n\n return filename_to_mtime, read_fn\n\n\ndef _entries_from_cache_fallback(url: str, tmpspecsdir: str, component_type: BuildcacheComponent):\n \"\"\"Use spack.util.web module to get a list of all the manifests at the remote url.\n\n Args:\n url: Base url of mirror (location of manifest files)\n tmpspecsdir: path to temporary directory to use for writing files\n component_type: type of buildcache component to sync (spec, index, key, etc.)\n\n Return:\n A tuple where the first item is a list of absolute file paths or\n urls pointing to the manifests that should be read from the mirror,\n and the second item is a function taking a url or file path of a manifest and\n returning a :class:`URLBuildcacheEntry` for that manifest.\n \"\"\"\n read_fn = None\n filename_to_mtime = None\n\n cache_class = get_url_buildcache_class(layout_version=CURRENT_BUILD_CACHE_LAYOUT_VERSION)\n\n def url_read_method(manifest_url: str) -> URLBuildcacheEntry:\n cache_entry = cache_class(mirror_url=url, allow_unsigned=True)\n cache_entry.read_manifest(manifest_url)\n return cache_entry\n\n try:\n filename_to_mtime = {}\n component_path_parts = cache_class.get_relative_path_components(component_type)\n component_prefix: str = url_util.join(url, *component_path_parts)\n component_pattern = cache_class.get_buildcache_component_include_pattern(component_type)\n for entry in web_util.list_url(component_prefix, recursive=True):\n if fnmatch.fnmatch(entry, component_pattern):\n entry_url = url_util.join(component_prefix, entry)\n stat_result = web_util.stat_url(entry_url)\n if stat_result is not None:\n filename_to_mtime[entry_url] = stat_result[1] # mtime is second element\n read_fn = url_read_method\n except Exception as err:\n # If we got some kind of S3 (access denied or other connection error), the first non\n # boto-specific class in the exception is Exception. Just print a warning and return\n tty.warn(f\"Encountered problem listing packages at {url}: {err}\")\n\n return filename_to_mtime, read_fn\n\n\ndef get_entries_from_cache(url: str, tmpspecsdir: str, component_type: BuildcacheComponent):\n \"\"\"Get a list of all the manifests in the mirror and a function to read them.\n\n Args:\n url: Base url of mirror (location of spec files)\n tmpspecsdir: Temporary location for writing files\n component_type: type of buildcache component to sync (spec, index, key, etc.)\n\n Return:\n A tuple where the first item is a list of absolute file paths or\n urls pointing to the manifests that should be read from the mirror,\n and the second item is a function taking a url or file path and\n returning a :class:`URLBuildcacheEntry` for that manifest.\n \"\"\"\n callbacks: List[Callable] = []\n if url.startswith(\"s3://\"):\n callbacks.append(_entries_from_cache_aws_cli)\n\n callbacks.append(_entries_from_cache_fallback)\n\n for specs_from_cache_fn in callbacks:\n file_to_mtime_mapping, read_fn = specs_from_cache_fn(url, tmpspecsdir, component_type)\n if file_to_mtime_mapping:\n return file_to_mtime_mapping, read_fn\n\n raise ListMirrorSpecsError(\"Failed to get list of entries from {0}\".format(url))\n\n\ndef validate_checksum(file_path, checksum_algorithm, expected_checksum) -> None:\n \"\"\"Compute the checksum of the given file and raise if invalid\"\"\"\n local_checksum = spack.util.crypto.checksum(hash_fun_for_algo(checksum_algorithm), file_path)\n\n if local_checksum != expected_checksum:\n size, contents = fsys.filesummary(file_path)\n raise spack.error.NoChecksumException(\n file_path, size, contents, checksum_algorithm, expected_checksum, local_checksum\n )\n\n\ndef _get_compressor(compression: str, writable: io.BufferedIOBase) -> io.BufferedIOBase:\n if compression == \"gzip\":\n return gzip.GzipFile(filename=\"\", mode=\"wb\", compresslevel=6, mtime=0, fileobj=writable)\n elif compression == \"none\":\n return writable\n else:\n raise BuildcacheEntryError(f\"Unknown compression type: {compression}\")\n\n\n@contextmanager\ndef compression_writer(output_path: str, compression: str, checksum_algo: str):\n \"\"\"Create and return a writer capable of writing compressed data. Available\n options for ``compression`` are ``\"gzip\"`` or ``\"none\"``, ``checksum_algo`` is used to pick\n the checksum algorithm used by the :class:`~spack.util.archive.ChecksumWriter`.\n\n Yields:\n A tuple containing\n\n * An :class:`io.BufferedIOBase` writer that can compress (or not) as it writes\n * A :class:`~spack.util.archive.ChecksumWriter` that provides checksum and length of\n written data\n \"\"\"\n with open(output_path, \"wb\") as writer, ChecksumWriter(\n fileobj=writer, algorithm=hash_fun_for_algo(checksum_algo)\n ) as checksum_writer, closing(\n _get_compressor(compression, checksum_writer)\n ) as compress_writer:\n yield compress_writer, checksum_writer\n\n\ndef compressed_json_from_dict(\n output_path: str, spec_dict: dict, checksum_algo: str\n) -> Tuple[str, int]:\n \"\"\"Compress the spec dict and write it to the given path\n\n Return the checksum (using the given algorithm) and size on disk of the file\n \"\"\"\n with compression_writer(output_path, \"gzip\", checksum_algo) as (\n f_bin,\n checker,\n ), io.TextIOWrapper(f_bin, encoding=\"utf-8\") as f_txt:\n json.dump(spec_dict, f_txt, separators=(\",\", \":\"))\n\n return checker.hexdigest(), checker.length\n\n\ndef get_valid_spec_file(path: str, max_supported_layout: int) -> Tuple[Dict, int]:\n \"\"\"Read and validate a spec file, returning the spec dict with its layout version, or raising\n InvalidMetadataFile if invalid.\"\"\"\n try:\n with open(path, \"rb\") as f:\n binary_content = f.read()\n except OSError as e:\n raise InvalidMetadataFile(f\"No such file: {path}\") from e\n\n # Decompress spec file if necessary\n if binary_content[:2] == b\"\\x1f\\x8b\":\n binary_content = gzip.decompress(binary_content)\n\n try:\n as_string = binary_content.decode(\"utf-8\")\n if path.endswith(\".json.sig\"):\n spec_dict = spack.spec.Spec.extract_json_from_clearsig(as_string)\n else:\n spec_dict = json.loads(as_string)\n except Exception as e:\n raise InvalidMetadataFile(f\"Could not parse {path} due to: {e}\") from e\n\n # Ensure this version is not too new.\n try:\n layout_version = int(spec_dict.get(\"buildcache_layout_version\", 0))\n except ValueError as e:\n raise InvalidMetadataFile(\"Could not parse layout version\") from e\n\n if layout_version > max_supported_layout:\n raise InvalidMetadataFile(\n f\"Layout version {layout_version} is too new for this version of Spack\"\n )\n\n return spec_dict, layout_version\n\n\ndef sign_file(key: str, file_path: str) -> str:\n \"\"\"sign and return the path to the signed file\"\"\"\n signed_file_path = f\"{file_path}.sig\"\n spack.util.gpg.sign(key, file_path, signed_file_path, clearsign=True)\n return signed_file_path\n\n\ndef try_verify(specfile_path):\n \"\"\"Utility function to attempt to verify a local file. Assumes the\n file is a clearsigned signature file.\n\n Args:\n specfile_path (str): Path to file to be verified.\n\n Returns:\n ``True`` if the signature could be verified, ``False`` otherwise.\n \"\"\"\n suppress = config.get(\"config:suppress_gpg_warnings\", False)\n\n try:\n spack.util.gpg.verify(specfile_path, suppress_warnings=suppress)\n except Exception:\n return False\n\n return True\n\n\nclass MirrorMetadata:\n \"\"\"Simple class to hold a mirror url and a buildcache layout version\n\n This class is used by BinaryCacheIndex to produce a key used to keep\n track of downloaded/processed buildcache index files from remote mirrors\n in some layout version.\"\"\"\n\n __slots__ = (\"url\", \"version\", \"view\")\n\n def __init__(self, url: str, version: int, view: Optional[str] = None):\n self.url = url\n self.version = version\n self.view = view\n\n def __str__(self):\n s = f\"{self.url}__v{self.version}\"\n if self.view:\n s += f\"__{self.view}\"\n return s\n\n def __eq__(self, other):\n if not isinstance(other, MirrorMetadata):\n return NotImplemented\n return self.url == other.url and self.version == other.version and self.view == other.view\n\n def __hash__(self):\n return hash((self.url, self.version, self.view))\n\n @classmethod\n def from_string(cls, s: str):\n m = re.match(r\"^(.*)__v([0-9]+)(?:__(.*))?$\", s)\n if not m:\n raise MirrorMetadataError(f\"Malformed string {s}\")\n\n url, version, view = m.groups()\n return cls(url, int(version), view)\n\n def strip_view(self) -> \"MirrorMetadata\":\n return MirrorMetadata(self.url, self.version)\n\n\nclass InvalidMetadataFile(spack.error.SpackError):\n \"\"\"Raised when spack encounters a spec file it cannot understand or process\"\"\"\n\n\nclass BuildcacheEntryError(spack.error.SpackError):\n \"\"\"Raised for problems finding or accessing binary cache entry on mirror\"\"\"\n\n\nclass NoSuchBlobException(spack.error.SpackError):\n \"\"\"Raised when manifest does have some requested type of requested type\"\"\"\n\n\nclass NoVerifyException(BuildcacheEntryError):\n \"\"\"Raised if file fails signature verification\"\"\"\n\n\nclass UnknownBuildcacheLayoutError(BuildcacheEntryError):\n \"\"\"Raised when unrecognized buildcache layout version is encountered\"\"\"\n\n\nclass ListMirrorSpecsError(spack.error.SpackError):\n \"\"\"Raised when unable to retrieve list of specs from the mirror\"\"\"\n\n\nclass MirrorMetadataError(spack.error.SpackError):\n \"\"\"Raised when unable to interpret a MirrorMetadata string\"\"\"\n" }, { "path": "lib/spack/spack/user_environment.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport os\nimport re\nimport sys\n\nimport spack.build_environment\nimport spack.config\nimport spack.spec\nimport spack.util.environment as environment\nfrom spack import traverse\nfrom spack.context import Context\n\n#: Environment variable name Spack uses to track individually loaded packages\nspack_loaded_hashes_var = \"SPACK_LOADED_HASHES\"\n\n\ndef prefix_inspections(platform: str) -> dict:\n \"\"\"Get list of prefix inspections for platform\n\n Arguments:\n platform: the name of the platform to consider. The platform determines what environment\n variables Spack will use for some inspections.\n\n Returns:\n A dictionary mapping subdirectory names to lists of environment variables to modify with\n that directory if it exists.\n \"\"\"\n inspections = spack.config.get(\"modules:prefix_inspections\")\n if isinstance(inspections, dict):\n return inspections\n\n inspections = {\n \"bin\": [\"PATH\"],\n \"man\": [\"MANPATH\"],\n \"share/man\": [\"MANPATH\"],\n \"share/aclocal\": [\"ACLOCAL_PATH\"],\n \"lib/pkgconfig\": [\"PKG_CONFIG_PATH\"],\n \"lib64/pkgconfig\": [\"PKG_CONFIG_PATH\"],\n \"share/pkgconfig\": [\"PKG_CONFIG_PATH\"],\n \"\": [\"CMAKE_PREFIX_PATH\"],\n }\n\n if platform == \"darwin\":\n inspections[\"lib\"] = [\"DYLD_FALLBACK_LIBRARY_PATH\"]\n inspections[\"lib64\"] = [\"DYLD_FALLBACK_LIBRARY_PATH\"]\n\n return inspections\n\n\ndef unconditional_environment_modifications(view):\n \"\"\"List of environment (shell) modifications to be processed for view.\n\n This list does not depend on the specs in this environment\"\"\"\n env = environment.EnvironmentModifications()\n\n for subdir, vars in prefix_inspections(sys.platform).items():\n full_subdir = os.path.join(view.root, subdir)\n for var in vars:\n env.prepend_path(var, full_subdir)\n\n return env\n\n\ndef project_env_mods(\n *specs: spack.spec.Spec, view, env: environment.EnvironmentModifications\n) -> None:\n \"\"\"Given a list of environment modifications, project paths changes to the view.\"\"\"\n prefix_to_prefix = {\n str(s.prefix): view.get_projection_for_spec(s) for s in specs if not s.external\n }\n # Avoid empty regex if all external\n if not prefix_to_prefix:\n return\n prefix_regex = re.compile(\"|\".join(re.escape(p) for p in prefix_to_prefix.keys()))\n for mod in env.env_modifications:\n if isinstance(mod, environment.NameValueModifier):\n mod.value = prefix_regex.sub(lambda m: prefix_to_prefix[m.group(0)], mod.value)\n\n\ndef environment_modifications_for_specs(\n *specs: spack.spec.Spec, view=None, set_package_py_globals: bool = True\n):\n \"\"\"List of environment (shell) modifications to be processed for spec.\n\n This list is specific to the location of the spec or its projection in\n the view.\n\n Args:\n specs: spec(s) for which to list the environment modifications\n view: view associated with the spec passed as first argument\n set_package_py_globals: whether or not to set the global variables in the\n package.py files (this may be problematic when using buildcaches that have\n been built on a different but compatible OS)\n \"\"\"\n env = environment.EnvironmentModifications()\n topo_ordered = list(\n traverse.traverse_nodes(specs, root=True, deptype=(\"run\", \"link\"), order=\"topo\")\n )\n\n # Static environment changes (prefix inspections)\n for s in reversed(topo_ordered):\n static = environment.inspect_path(\n s.prefix, prefix_inspections(s.platform), exclude=environment.is_system_path\n )\n env.extend(static)\n\n # Dynamic environment changes (setup_run_environment etc)\n setup_context = spack.build_environment.SetupContext(*specs, context=Context.RUN)\n if set_package_py_globals:\n setup_context.set_all_package_py_globals()\n env.extend(setup_context.get_env_modifications())\n\n # Apply view projections if any.\n if view:\n project_env_mods(*topo_ordered, view=view, env=env)\n\n return env\n" }, { "path": "lib/spack/spack/util/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n" }, { "path": "lib/spack/spack/util/archive.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport errno\nimport hashlib\nimport io\nimport os\nimport pathlib\nimport tarfile\nfrom contextlib import closing, contextmanager\nfrom gzip import GzipFile\nfrom typing import Callable, Dict, Generator, List, Tuple\n\nfrom spack.llnl.util import tty\nfrom spack.llnl.util.filesystem import readlink\nfrom spack.util.git import is_git_commit_sha\n\n\nclass ChecksumWriter(io.BufferedIOBase):\n \"\"\"Checksum writer computes a checksum while writing to a file.\"\"\"\n\n myfileobj = None\n\n def __init__(self, fileobj, algorithm=hashlib.sha256):\n self.fileobj = fileobj\n self.hasher = algorithm()\n self.length = 0\n\n def hexdigest(self):\n return self.hasher.hexdigest()\n\n def write(self, data):\n if isinstance(data, (bytes, bytearray)):\n length = len(data)\n else:\n data = memoryview(data)\n length = data.nbytes\n\n if length > 0:\n self.fileobj.write(data)\n self.hasher.update(data)\n\n self.length += length\n\n return length\n\n def read(self, size=-1):\n raise OSError(errno.EBADF, \"read() on write-only object\")\n\n def read1(self, size=-1):\n raise OSError(errno.EBADF, \"read1() on write-only object\")\n\n def peek(self, n):\n raise OSError(errno.EBADF, \"peek() on write-only object\")\n\n @property\n def closed(self):\n return self.fileobj is None\n\n def close(self):\n fileobj = self.fileobj\n if fileobj is None:\n return\n self.fileobj.close()\n self.fileobj = None\n\n def flush(self):\n self.fileobj.flush()\n\n def fileno(self):\n return self.fileobj.fileno()\n\n def rewind(self):\n raise OSError(\"Can't rewind while computing checksum\")\n\n def readable(self):\n return False\n\n def writable(self):\n return True\n\n def seekable(self):\n return True\n\n def tell(self):\n return self.fileobj.tell()\n\n def seek(self, offset, whence=io.SEEK_SET):\n # In principle forward seek is possible with b\"0\" padding,\n # but this is not implemented.\n if offset == 0 and whence == io.SEEK_CUR:\n return\n raise OSError(\"Can't seek while computing checksum\")\n\n def readline(self, size=-1):\n raise OSError(errno.EBADF, \"readline() on write-only object\")\n\n\n@contextmanager\ndef gzip_compressed_tarfile(\n path: str,\n) -> Generator[Tuple[tarfile.TarFile, ChecksumWriter, ChecksumWriter], None, None]:\n \"\"\"Create a reproducible, gzip compressed tarfile, and keep track of shasums of both the\n compressed and uncompressed tarfile. Reproducibility is achieved by normalizing the gzip header\n (no file name and zero mtime).\n\n Yields:\n A tuple of three elements\n\n * :class:`tarfile.TarFile`: tarfile object\n * :class:`ChecksumWriter`: checksum of the gzip compressed tarfile\n * :class:`ChecksumWriter`: checksum of the uncompressed tarfile\n \"\"\"\n # Create gzip compressed tarball of the install prefix\n # 1) Use explicit empty filename and mtime 0 for gzip header reproducibility.\n # If the filename=\"\" is dropped, Python will use fileobj.name instead.\n # This should effectively mimic `gzip --no-name`.\n # 2) On AMD Ryzen 3700X and an SSD disk, we have the following on compression speed:\n # compresslevel=6 gzip default: llvm takes 4mins, roughly 2.1GB\n # compresslevel=9 python default: llvm takes 12mins, roughly 2.1GB\n # So we follow gzip.\n with open(path, \"wb\") as f, ChecksumWriter(f) as gzip_checksum, closing(\n GzipFile(filename=\"\", mode=\"wb\", compresslevel=6, mtime=0, fileobj=gzip_checksum)\n ) as gzip_file, ChecksumWriter(gzip_file) as tarfile_checksum, tarfile.TarFile(\n name=\"\", mode=\"w\", fileobj=tarfile_checksum\n ) as tar:\n yield tar, gzip_checksum, tarfile_checksum\n\n\ndef default_path_to_name(path: str) -> str:\n \"\"\"Converts a path to a tarfile name, which uses posix path separators.\"\"\"\n p = pathlib.PurePath(path)\n # Drop the leading slash on posix and the drive letter on windows, and always format as a\n # posix path.\n return pathlib.PurePath(*p.parts[1:]).as_posix() if p.is_absolute() else p.as_posix()\n\n\ndef default_add_file(tar: tarfile.TarFile, file_info: tarfile.TarInfo, path: str) -> None:\n with open(path, \"rb\") as f:\n tar.addfile(file_info, f)\n\n\ndef default_add_link(tar: tarfile.TarFile, file_info: tarfile.TarInfo, path: str) -> None:\n tar.addfile(file_info)\n\n\ndef reproducible_tarfile_from_prefix(\n tar: tarfile.TarFile,\n prefix: str,\n *,\n include_parent_directories: bool = False,\n skip: Callable[[os.DirEntry], bool] = lambda entry: False,\n path_to_name: Callable[[str], str] = default_path_to_name,\n add_file: Callable[[tarfile.TarFile, tarfile.TarInfo, str], None] = default_add_file,\n add_symlink: Callable[[tarfile.TarFile, tarfile.TarInfo, str], None] = default_add_link,\n add_hardlink: Callable[[tarfile.TarFile, tarfile.TarInfo, str], None] = default_add_link,\n) -> None:\n \"\"\"Create a tarball from a given directory. Only adds regular files, symlinks and dirs.\n Skips devices, fifos. Preserves hardlinks. Normalizes permissions like git. Tar entries are\n added in depth-first pre-order, with dir entries partitioned by file | dir, and sorted\n lexicographically, for reproducibility. Partitioning ensures only one dir is in memory at a\n time, and sorting improves compression.\n\n Args:\n tar: tarfile object opened in write mode\n prefix: path to directory to tar (either absolute or relative)\n include_parent_directories: whether to include every directory leading up to ``prefix`` in\n the tarball\n skip: function that receives a DirEntry and returns True if the entry should be skipped,\n whether it is a file or directory. Default implementation does not skip anything.\n path_to_name: function that converts a path string to a tarfile entry name, which should be\n in posix format. Not only is it necessary to transform paths in certain cases, such as\n windows path to posix format, but it can also be used to prepend a directory to each\n entry even if it does not exist on the filesystem. The default implementation drops the\n leading slash on posix and the drive letter on windows for absolute paths, and formats\n as a posix.\"\"\"\n\n hardlink_to_tarinfo_name: Dict[Tuple[int, int], str] = dict()\n\n if include_parent_directories:\n parent_dirs = reversed(pathlib.PurePosixPath(path_to_name(prefix)).parents)\n next(parent_dirs) # skip the root: slices are supported from python 3.10\n for parent_dir in parent_dirs:\n dir_info = tarfile.TarInfo(str(parent_dir))\n dir_info.type = tarfile.DIRTYPE\n dir_info.mode = 0o755\n tar.addfile(dir_info)\n\n dir_stack = [prefix]\n new_dirs: List[str] = []\n while dir_stack:\n dir = dir_stack.pop()\n new_dirs.clear()\n\n # Add the dir before its contents\n dir_info = tarfile.TarInfo(path_to_name(dir))\n dir_info.type = tarfile.DIRTYPE\n dir_info.mode = 0o755\n tar.addfile(dir_info)\n\n # Sort by name: reproducible & improves compression\n with os.scandir(dir) as it:\n entries = sorted(it, key=lambda entry: entry.name)\n\n for entry in entries:\n if skip(entry):\n continue\n\n if entry.is_dir(follow_symlinks=False):\n new_dirs.append(entry.path)\n continue\n\n file_info = tarfile.TarInfo(path_to_name(entry.path))\n\n if entry.is_symlink():\n file_info.type = tarfile.SYMTYPE\n file_info.linkname = readlink(entry.path)\n # According to POSIX: \"the value of the file mode bits returned in the\n # st_mode field of the stat structure is unspecified.\" So we set it to\n # something sensible without lstat'ing the link.\n file_info.mode = 0o755\n add_symlink(tar, file_info, entry.path)\n\n elif entry.is_file(follow_symlinks=False):\n # entry.stat has zero (st_ino, st_dev, st_nlink) on Windows: use lstat.\n s = os.lstat(entry.path)\n\n # Normalize permissions like git\n file_info.mode = 0o755 if s.st_mode & 0o100 else 0o644\n\n # Deduplicate hardlinks\n if s.st_nlink > 1:\n ident = (s.st_dev, s.st_ino)\n if ident in hardlink_to_tarinfo_name:\n file_info.type = tarfile.LNKTYPE\n file_info.linkname = hardlink_to_tarinfo_name[ident]\n add_hardlink(tar, file_info, entry.path)\n continue\n hardlink_to_tarinfo_name[ident] = file_info.name\n\n # If file not yet seen, copy it\n file_info.type = tarfile.REGTYPE\n file_info.size = s.st_size\n add_file(tar, file_info, entry.path)\n\n dir_stack.extend(reversed(new_dirs)) # we pop, so reverse to stay alphabetical\n\n\ndef retrieve_commit_from_archive(archive_path, ref):\n \"\"\"Extract git data from an archive with out expanding it\n\n Open the archive and searches for .git/HEAD. Return if HEAD is a commit (detached head or tag)\n Otherwise attempt to read the ref that .git/HEAD is pointing to and return the commit\n associated with it.\n \"\"\"\n if not os.path.isfile(archive_path):\n raise FileNotFoundError(f\"The file {archive_path} does not exist\")\n\n try:\n with tarfile.open(archive_path, \"r\") as tar:\n names = tar.getnames()\n # since we always have a prefix and can't guarantee the value we need this lookup.\n prefix = \"\"\n for name in names:\n if name.endswith(\".git\"):\n prefix = name[:-4]\n break\n if f\"{prefix}.git/HEAD\" in names:\n head = tar.extractfile(f\"{prefix}.git/HEAD\").read().decode(\"utf-8\").strip()\n if is_git_commit_sha(head):\n # detached HEAD/ lightweight tag\n return head\n else:\n # refs in had have the format \"ref \"\n ref = head.split()[1]\n contents = (\n tar.extractfile(f\"{prefix}.git/{ref}\").read().decode(\"utf-8\").strip()\n )\n if is_git_commit_sha(contents):\n return contents\n except tarfile.ReadError:\n tty.warn(f\"Archive {archive_path} does not appear to contain git data\")\n return\n" }, { "path": "lib/spack/spack/util/compression.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport errno\nimport inspect\nimport io\nimport os\nimport shutil\nimport sys\nfrom typing import Any, BinaryIO, Callable, Dict, List, Optional\n\nimport spack.llnl.url\nfrom spack.error import SpackError\nfrom spack.llnl.util import tty\nfrom spack.util.executable import CommandNotFoundError, which\n\ntry:\n import bz2 # noqa\n\n BZ2_SUPPORTED = True\nexcept ImportError:\n BZ2_SUPPORTED = False\n\n\ntry:\n import gzip # noqa\n\n GZIP_SUPPORTED = True\nexcept ImportError:\n GZIP_SUPPORTED = False\n\n\ntry:\n import lzma # noqa # novermin\n\n LZMA_SUPPORTED = True\nexcept ImportError:\n LZMA_SUPPORTED = False\n\n\ndef _system_untar(archive_file: str, remove_archive_file: bool = False) -> str:\n \"\"\"Returns path to unarchived tar file. Untars archive via system tar.\n\n Args:\n archive_file (str): absolute path to the archive to be extracted.\n Can be one of .tar(.[gz|bz2|xz|Z]) or .(tgz|tbz|tbz2|txz).\n \"\"\"\n archive_file_no_ext = spack.llnl.url.strip_extension(archive_file)\n outfile = os.path.basename(archive_file_no_ext)\n if archive_file_no_ext == archive_file:\n # the archive file has no extension. Tar on windows cannot untar onto itself\n # archive_file can be a tar file (which causes the problem on windows) but it can\n # also have other extensions (on Unix) such as tgz, tbz2, ...\n archive_file = archive_file_no_ext + \"-input\"\n shutil.move(archive_file_no_ext, archive_file)\n tar = which(\"tar\", required=True)\n # GNU tar's --no-same-owner is not as portable, -o works for BSD tar too. This flag is relevant\n # when extracting archives as root, where tar attempts to set original ownership of files. This\n # is redundant when distributing tarballs, as the tarballs are created on different systems\n # than where they are extracted. In certain cases like rootless containers, setting original\n # ownership is known to fail, so we need to disable it.\n tar.add_default_arg(\"-oxf\")\n tar(archive_file)\n if remove_archive_file:\n # remove input file to prevent two stage\n # extractions from being treated as exploding\n # archives by the fetcher\n os.remove(archive_file)\n return outfile\n\n\ndef _bunzip2(archive_file: str) -> str:\n \"\"\"Returns path to decompressed file.\n Uses Python's bz2 module to decompress bz2 compressed archives\n Fall back to system utility failing to find Python module `bz2`\n\n Args:\n archive_file: absolute path to the bz2 archive to be decompressed\n \"\"\"\n if BZ2_SUPPORTED:\n return _py_bunzip(archive_file)\n else:\n return _system_bunzip(archive_file)\n\n\ndef _py_bunzip(archive_file: str) -> str:\n \"\"\"Returns path to decompressed file.\n Decompresses bz2 compressed archives/files via python's bz2 module\"\"\"\n decompressed_file = os.path.basename(\n spack.llnl.url.strip_compression_extension(archive_file, \"bz2\")\n )\n working_dir = os.getcwd()\n archive_out = os.path.join(working_dir, decompressed_file)\n f_bz = bz2.BZ2File(archive_file, mode=\"rb\")\n with open(archive_out, \"wb\") as ar:\n shutil.copyfileobj(f_bz, ar)\n f_bz.close()\n return archive_out\n\n\ndef _system_bunzip(archive_file: str) -> str:\n \"\"\"Returns path to decompressed file.\n Decompresses bz2 compressed archives/files via system bzip2 utility\"\"\"\n compressed_file_name = os.path.basename(archive_file)\n decompressed_file = os.path.basename(\n spack.llnl.url.strip_compression_extension(archive_file, \"bz2\")\n )\n working_dir = os.getcwd()\n archive_out = os.path.join(working_dir, decompressed_file)\n copy_path = os.path.join(working_dir, compressed_file_name)\n shutil.copy(archive_file, copy_path)\n bunzip2 = which(\"bunzip2\", required=True)\n bunzip2.add_default_arg(\"-q\")\n bunzip2(copy_path)\n return archive_out\n\n\ndef _gunzip(archive_file: str) -> str:\n \"\"\"Returns path to gunzip'd file. Decompresses `.gz` extensions. Prefer native Python\n `gzip` module. Falling back to system utility gunzip. Like gunzip, but extracts in the current\n working directory instead of in-place.\n\n Args:\n archive_file: absolute path of the file to be decompressed\n \"\"\"\n return _py_gunzip(archive_file) if GZIP_SUPPORTED else _system_gunzip(archive_file)\n\n\ndef _py_gunzip(archive_file: str) -> str:\n \"\"\"Returns path to gunzip'd file. Decompresses `.gz` compressed archives via python gzip\n module\"\"\"\n decompressed_file = os.path.basename(\n spack.llnl.url.strip_compression_extension(archive_file, \"gz\")\n )\n working_dir = os.getcwd()\n destination_abspath = os.path.join(working_dir, decompressed_file)\n f_in = gzip.open(archive_file, \"rb\")\n with open(destination_abspath, \"wb\") as f_out:\n shutil.copyfileobj(f_in, f_out)\n f_in.close()\n return destination_abspath\n\n\ndef _system_gunzip(archive_file: str) -> str:\n \"\"\"Returns path to gunzip'd file. Decompresses `.gz` compressed files via system gzip\"\"\"\n archive_file_no_ext = spack.llnl.url.strip_compression_extension(archive_file)\n if archive_file_no_ext == archive_file:\n # the zip file has no extension. On Unix gunzip cannot unzip onto itself\n archive_file = archive_file + \".gz\"\n shutil.move(archive_file_no_ext, archive_file)\n decompressed_file = os.path.basename(archive_file_no_ext)\n working_dir = os.getcwd()\n destination_abspath = os.path.join(working_dir, decompressed_file)\n compressed_file = os.path.basename(archive_file)\n copy_path = os.path.join(working_dir, compressed_file)\n shutil.copy(archive_file, copy_path)\n gzip = which(\"gzip\", required=True)\n gzip.add_default_arg(\"-d\")\n gzip(copy_path)\n return destination_abspath\n\n\ndef _do_nothing(archive_file: str) -> None:\n return None\n\n\ndef _unzip(archive_file: str) -> str:\n \"\"\"Returns path to extracted zip archive. Extract Zipfile, searching for unzip system\n executable. If unavailable, search for 'tar' executable on system and use instead.\n\n Args:\n archive_file: absolute path of the file to be decompressed\n \"\"\"\n if sys.platform == \"win32\":\n return _system_untar(archive_file)\n unzip = which(\"unzip\", required=True)\n unzip.add_default_arg(\"-q\")\n unzip(archive_file)\n return os.path.basename(spack.llnl.url.strip_extension(archive_file, extension=\"zip\"))\n\n\ndef _system_unZ(archive_file: str) -> str:\n \"\"\"Returns path to decompressed file\n Decompress UNIX compress style compression\n Utilizes gunzip on unix and 7zip on Windows\n \"\"\"\n if sys.platform == \"win32\":\n return _system_7zip(archive_file)\n return _system_gunzip(archive_file)\n\n\ndef _lzma_decomp(archive_file):\n \"\"\"Returns path to decompressed xz file. Decompress lzma compressed files. Prefer Python native\n lzma module, but fall back on command line xz tooling to find available Python support.\"\"\"\n return _py_lzma(archive_file) if LZMA_SUPPORTED else _xz(archive_file)\n\n\ndef _win_compressed_tarball_handler(decompressor: Callable[[str], str]) -> Callable[[str], str]:\n \"\"\"Returns function pointer to two stage decompression\n and extraction method\n Decompress and extract compressed tarballs on Windows.\n This method uses a decompression method in conjunction with\n the tar utility to perform decompression and extraction in\n a two step process first using decompressor to decompress,\n and tar to extract.\n\n The motivation for this method is Windows tar utility's lack\n of access to the xz tool (unsupported natively on Windows) but\n can be installed manually or via spack\n \"\"\"\n\n def unarchive(archive_file: str):\n # perform intermediate extraction step\n # record name of new archive so we can extract\n decomped_tarball = decompressor(archive_file)\n # run tar on newly decomped archive\n outfile = _system_untar(decomped_tarball, remove_archive_file=True)\n return outfile\n\n return unarchive\n\n\ndef _py_lzma(archive_file: str) -> str:\n \"\"\"Returns path to decompressed .xz files. Decompress lzma compressed .xz files via Python\n lzma module.\"\"\"\n decompressed_file = os.path.basename(\n spack.llnl.url.strip_compression_extension(archive_file, \"xz\")\n )\n archive_out = os.path.join(os.getcwd(), decompressed_file)\n with open(archive_out, \"wb\") as ar:\n with lzma.open(archive_file) as lar:\n shutil.copyfileobj(lar, ar)\n return archive_out\n\n\ndef _xz(archive_file):\n \"\"\"Returns path to decompressed xz files. Decompress lzma compressed .xz files via xz command\n line tool.\"\"\"\n decompressed_file = os.path.basename(\n spack.llnl.url.strip_extension(archive_file, extension=\"xz\")\n )\n working_dir = os.getcwd()\n destination_abspath = os.path.join(working_dir, decompressed_file)\n compressed_file = os.path.basename(archive_file)\n copy_path = os.path.join(working_dir, compressed_file)\n shutil.copy(archive_file, copy_path)\n xz = which(\"xz\", required=True)\n xz.add_default_arg(\"-d\")\n xz(copy_path)\n return destination_abspath\n\n\ndef _system_7zip(archive_file):\n \"\"\"Returns path to decompressed file\n Unpack/decompress with 7z executable\n 7z is able to handle a number file extensions however\n it may not be available on system.\n Without 7z, Windows users with certain versions of Python may\n be unable to extract .xz files, and all Windows users will be unable\n to extract .Z files. If we cannot find 7z either externally or a\n Spack installed copy, we fail, but inform the user that 7z can\n be installed via `spack install 7zip`\n Args:\n archive_file (str): absolute path of file to be unarchived\n \"\"\"\n outfile = os.path.basename(spack.llnl.url.strip_compression_extension(archive_file))\n _7z = which(\"7z\")\n if not _7z:\n raise CommandNotFoundError(\n \"7z unavailable, unable to extract %s files. 7z can be installed via Spack\"\n % spack.llnl.url.extension_from_path(archive_file)\n )\n _7z.add_default_arg(\"e\")\n _7z(archive_file)\n return outfile\n\n\ndef decompressor_for(path: str, extension: Optional[str] = None):\n \"\"\"Returns appropriate decompression/extraction algorithm function pointer\n for provided extension. If extension is none, it is computed\n from the ``path`` and the decompression function is derived\n from that information.\"\"\"\n if not extension:\n extension = extension_from_magic_numbers(path, decompress=True)\n\n if not extension or not spack.llnl.url.allowed_archive(extension):\n raise CommandNotFoundError(\n f\"Cannot extract {path}, unrecognized file extension: '{extension}'\"\n )\n if sys.platform == \"win32\":\n return decompressor_for_win(extension)\n else:\n return decompressor_for_nix(extension)\n\n\ndef decompressor_for_nix(extension: str) -> Callable[[str], Any]:\n \"\"\"Returns a function pointer to appropriate decompression algorithm based on extension type\n and unix specific considerations i.e. a reasonable expectation system utils like gzip, bzip2,\n and xz are available\n\n Args:\n extension: path of the archive file requiring decompression\n \"\"\"\n extension_to_decompressor: Dict[str, Callable[[str], Any]] = {\n \"zip\": _unzip,\n \"gz\": _gunzip,\n \"bz2\": _bunzip2,\n \"Z\": _system_unZ, # no builtin support for .Z files\n \"xz\": _lzma_decomp,\n \"whl\": _do_nothing,\n }\n\n return extension_to_decompressor.get(extension, _system_untar)\n\n\ndef _determine_py_decomp_archive_strategy(extension: str) -> Optional[Callable[[str], Any]]:\n \"\"\"Returns appropriate python based decompression strategy\n based on extension type\"\"\"\n extension_to_decompressor: Dict[str, Callable[[str], str]] = {\n \"gz\": _py_gunzip,\n \"bz2\": _py_bunzip,\n \"xz\": _py_lzma,\n }\n return extension_to_decompressor.get(extension, None)\n\n\ndef decompressor_for_win(extension: str) -> Callable[[str], Any]:\n \"\"\"Returns a function pointer to appropriate decompression\n algorithm based on extension type and Windows specific considerations\n\n Windows natively vendors *only* tar, no other archive/compression utilities\n So we must rely exclusively on Python module support for all compression\n operations, tar for tarballs and zip files, and 7zip for Z compressed archives\n and files as Python does not provide support for the UNIX compress algorithm\n \"\"\"\n extension = spack.llnl.url.expand_contracted_extension(extension)\n extension_to_decompressor: Dict[str, Callable[[str], Any]] = {\n # Windows native tar can handle .zip extensions, use standard unzip method\n \"zip\": _unzip,\n # if extension is standard tarball, invoke Windows native tar\n \"tar\": _system_untar,\n # Python does not have native support of any kind for .Z files. In these cases, we rely on\n # 7zip, which must be installed outside of Spack and added to the PATH or externally\n # detected\n \"Z\": _system_unZ,\n \"xz\": _lzma_decomp,\n \"whl\": _do_nothing,\n }\n\n decompressor = extension_to_decompressor.get(extension)\n if decompressor:\n return decompressor\n\n # Windows vendors no native decompression tools, attempt to derive Python based decompression\n # strategy. Expand extension from abbreviated ones, i.e. tar.gz from .tgz\n compression_extension = spack.llnl.url.compression_ext_from_compressed_archive(extension)\n decompressor = (\n _determine_py_decomp_archive_strategy(compression_extension)\n if compression_extension\n else None\n )\n if not decompressor:\n raise SpackError(\n \"Spack was unable to determine a proper decompression strategy for\"\n f\"valid extension: {extension}\"\n \"This is a bug, please file an issue at https://github.com/spack/spack/issues\"\n )\n if \"tar\" not in extension:\n return decompressor\n\n return _win_compressed_tarball_handler(decompressor)\n\n\nclass FileTypeInterface:\n \"\"\"Base interface class for describing and querying file type information. FileType describes\n information about a single file type such as typical extension and byte header properties,\n and provides an interface to check a given file against said type based on magic number.\n\n This class should be subclassed each time a new type is to be described.\n\n Subclasses should each describe a different type of file. In order to do so, they must define\n the extension string, magic number, and header offset (if non zero). If a class has multiple\n magic numbers, it will need to override the method describing that file type's magic numbers\n and the method that checks a types magic numbers against a given file's.\"\"\"\n\n OFFSET = 0\n extension: str\n name: str\n\n @classmethod\n def magic_numbers(cls) -> List[bytes]:\n \"\"\"Return a list of all potential magic numbers for a filetype\"\"\"\n return [\n value for name, value in inspect.getmembers(cls) if name.startswith(\"_MAGIC_NUMBER\")\n ]\n\n @classmethod\n def header_size(cls) -> int:\n \"\"\"Return size of largest magic number associated with file type\"\"\"\n return max(len(x) for x in cls.magic_numbers())\n\n def matches_magic(self, stream: BinaryIO) -> bool:\n \"\"\"Returns true if the stream matches the current file type by any of its magic numbers.\n Resets stream to original position.\n\n Args:\n stream: file byte stream\n \"\"\"\n # move to location of magic bytes\n offset = stream.tell()\n stream.seek(self.OFFSET)\n magic_bytes = stream.read(self.header_size())\n stream.seek(offset)\n return any(magic_bytes.startswith(magic) for magic in self.magic_numbers())\n\n\nclass CompressedFileTypeInterface(FileTypeInterface):\n \"\"\"Interface class for FileTypes that include compression information\"\"\"\n\n def peek(self, stream: BinaryIO, num_bytes: int) -> Optional[io.BytesIO]:\n \"\"\"This method returns the first num_bytes of a decompressed stream. Returns None if no\n builtin support for decompression.\"\"\"\n return None\n\n\ndef _decompressed_peek(\n decompressed_stream: io.BufferedIOBase, stream: BinaryIO, num_bytes: int\n) -> io.BytesIO:\n # Read the first num_bytes of the decompressed stream, do not advance the stream position.\n pos = stream.tell()\n data = decompressed_stream.read(num_bytes)\n stream.seek(pos)\n return io.BytesIO(data)\n\n\nclass BZipFileType(CompressedFileTypeInterface):\n _MAGIC_NUMBER = b\"\\x42\\x5a\\x68\"\n extension = \"bz2\"\n name = \"bzip2 compressed data\"\n\n def peek(self, stream: BinaryIO, num_bytes: int) -> Optional[io.BytesIO]:\n if BZ2_SUPPORTED:\n return _decompressed_peek(bz2.BZ2File(stream), stream, num_bytes)\n return None\n\n\nclass ZCompressedFileType(CompressedFileTypeInterface):\n _MAGIC_NUMBER_LZW = b\"\\x1f\\x9d\"\n _MAGIC_NUMBER_LZH = b\"\\x1f\\xa0\"\n extension = \"Z\"\n name = \"compress'd data\"\n\n\nclass GZipFileType(CompressedFileTypeInterface):\n _MAGIC_NUMBER = b\"\\x1f\\x8b\\x08\"\n extension = \"gz\"\n name = \"gzip compressed data\"\n\n def peek(self, stream: BinaryIO, num_bytes: int) -> Optional[io.BytesIO]:\n if GZIP_SUPPORTED:\n return _decompressed_peek(gzip.GzipFile(fileobj=stream), stream, num_bytes)\n return None\n\n\nclass LzmaFileType(CompressedFileTypeInterface):\n _MAGIC_NUMBER = b\"\\xfd7zXZ\"\n extension = \"xz\"\n name = \"xz compressed data\"\n\n def peek(self, stream: BinaryIO, num_bytes: int) -> Optional[io.BytesIO]:\n if LZMA_SUPPORTED:\n return _decompressed_peek(lzma.LZMAFile(stream), stream, num_bytes)\n return None\n\n\nclass TarFileType(FileTypeInterface):\n OFFSET = 257\n _MAGIC_NUMBER_GNU = b\"ustar \\0\"\n _MAGIC_NUMBER_POSIX = b\"ustar\\x0000\"\n extension = \"tar\"\n name = \"tar archive\"\n\n\nclass ZipFleType(FileTypeInterface):\n _MAGIC_NUMBER = b\"PK\\003\\004\"\n extension = \"zip\"\n name = \"Zip archive data\"\n\n\n#: Maximum number of bytes to read from a file to determine any archive type. Tar is the largest.\nMAX_BYTES_ARCHIVE_HEADER = TarFileType.OFFSET + TarFileType.header_size()\n\n#: Collection of supported archive and compression file type identifier classes.\nSUPPORTED_FILETYPES: List[FileTypeInterface] = [\n BZipFileType(),\n ZCompressedFileType(),\n GZipFileType(),\n LzmaFileType(),\n TarFileType(),\n ZipFleType(),\n]\n\n\ndef _extension_of_compressed_file(\n file_type: CompressedFileTypeInterface, stream: BinaryIO\n) -> Optional[str]:\n \"\"\"Retrieves the extension of a file after decompression from its magic numbers, if it can be\n decompressed.\"\"\"\n # To classify the file we only need to decompress the first so many bytes.\n decompressed_magic = file_type.peek(stream, MAX_BYTES_ARCHIVE_HEADER)\n\n if not decompressed_magic:\n return None\n\n return extension_from_magic_numbers_by_stream(decompressed_magic, decompress=False)\n\n\ndef extension_from_magic_numbers_by_stream(\n stream: BinaryIO, decompress: bool = False\n) -> Optional[str]:\n \"\"\"Returns the typical extension for the opened file, without leading ``.``, based on its magic\n numbers.\n\n If the stream does not represent file type recognized by Spack (see\n :py:data:`SUPPORTED_FILETYPES`), the method will return None\n\n Args:\n stream: stream representing a file on system\n decompress: if True, compressed files are checked for archive types beneath compression.\n For example tar.gz if True versus only gz if False.\"\"\"\n for file_type in SUPPORTED_FILETYPES:\n if not file_type.matches_magic(stream):\n continue\n ext = file_type.extension\n if decompress and isinstance(file_type, CompressedFileTypeInterface):\n uncompressed_ext = _extension_of_compressed_file(file_type, stream)\n if not uncompressed_ext:\n tty.debug(\n \"Cannot derive file extension from magic number;\"\n \" falling back to original file name.\"\n )\n return spack.llnl.url.extension_from_path(stream.name)\n ext = f\"{uncompressed_ext}.{ext}\"\n tty.debug(f\"File extension {ext} successfully derived by magic number.\")\n return ext\n return None\n\n\ndef _maybe_abbreviate_extension(path: str, extension: str) -> str:\n \"\"\"If the file is a compressed tar archive, return the abbreviated extension t[xz|gz|bz2|bz]\n instead of tar.[xz|gz|bz2|bz] if the file's original name also has an abbreviated extension.\"\"\"\n if not extension.startswith(\"tar.\"):\n return extension\n abbr = f\"t{extension[4:]}\"\n return abbr if spack.llnl.url.has_extension(path, abbr) else extension\n\n\ndef extension_from_magic_numbers(path: str, decompress: bool = False) -> Optional[str]:\n \"\"\"Return typical extension without leading ``.`` of a compressed file or archive at the given\n path, based on its magic numbers, similar to the ``file`` utility. Notice that the extension\n returned from this function may not coincide with the file's given extension.\n\n Args:\n path: file to determine extension of\n decompress: If True, method will peek into decompressed file to check for archive file\n types. If False, the method will return only the top-level extension (for example\n ``gz`` and not ``tar.gz``).\n Returns:\n Spack recognized archive file extension as determined by file's magic number and file name.\n If file is not on system or is of a type not recognized by Spack as an archive or\n compression type, None is returned. If the file is classified as a compressed tarball, the\n extension is abbreviated (for instance ``tgz`` not ``tar.gz``) if that matches the file's\n given extension.\n \"\"\"\n try:\n with open(path, \"rb\") as f:\n ext = extension_from_magic_numbers_by_stream(f, decompress)\n except OSError as e:\n if e.errno == errno.ENOENT:\n return None\n raise\n\n # Return the extension derived from the magic number if possible.\n if ext:\n return _maybe_abbreviate_extension(path, ext)\n\n # Otherwise, use the extension from the file name.\n return spack.llnl.url.extension_from_path(path)\n" }, { "path": "lib/spack/spack/util/cpus.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport multiprocessing\nimport os\n\n\ndef cpus_available():\n \"\"\"\n Returns the number of CPUs available for the current process, or the number\n of physical CPUs when that information cannot be retrieved. The number\n of available CPUs might differ from the number of physical CPUs when\n using spack through Slurm or container runtimes.\n \"\"\"\n try:\n return len(os.sched_getaffinity(0)) # novermin\n except Exception:\n return multiprocessing.cpu_count()\n" }, { "path": "lib/spack/spack/util/crypto.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport hashlib\nfrom typing import BinaryIO, Callable, Dict, Optional\n\nimport spack.llnl.util.tty as tty\n\nHashFactory = Callable[[], \"hashlib._Hash\"]\n\n#: Set of hash algorithms that Spack can use, mapped to digest size in bytes\nhashes = {\"sha256\": 32, \"md5\": 16, \"sha1\": 20, \"sha224\": 28, \"sha384\": 48, \"sha512\": 64}\n# Note: keys are ordered by popularity for earliest return in ``hash_key in version_dict`` checks.\n\n\n#: size of hash digests in bytes, mapped to algorithm names\n_size_to_hash = dict((v, k) for k, v in hashes.items())\n\n\n#: List of deprecated hash functions. On some systems, these cannot be\n#: used without special options to hashlib.\n_deprecated_hash_algorithms = [\"md5\"]\n\n\n#: cache of hash functions generated\n_hash_functions: Dict[str, HashFactory] = {}\n\n\nclass DeprecatedHash:\n def __init__(self, hash_alg, alert_fn, disable_security_check):\n self.hash_alg = hash_alg\n self.alert_fn = alert_fn\n self.disable_security_check = disable_security_check\n\n def __call__(self, disable_alert=False):\n if not disable_alert:\n self.alert_fn(\n \"Deprecation warning: {0} checksums will not be\"\n \" supported in future Spack releases.\".format(self.hash_alg)\n )\n if self.disable_security_check:\n return hashlib.new(self.hash_alg, usedforsecurity=False) # novermin\n else:\n return hashlib.new(self.hash_alg)\n\n\ndef hash_fun_for_algo(algo: str) -> HashFactory:\n \"\"\"Get a function that can perform the specified hash algorithm.\"\"\"\n fun = _hash_functions.get(algo)\n if fun:\n return fun\n elif algo not in _deprecated_hash_algorithms:\n _hash_functions[algo] = getattr(hashlib, algo)\n else:\n try:\n deprecated_fun = DeprecatedHash(algo, tty.debug, disable_security_check=False)\n\n # call once to get a ValueError if usedforsecurity is needed\n deprecated_fun(disable_alert=True)\n except ValueError:\n # Some systems may support the 'usedforsecurity' option\n # so try with that (but display a warning when it is used)\n deprecated_fun = DeprecatedHash(algo, tty.warn, disable_security_check=True)\n _hash_functions[algo] = deprecated_fun\n return _hash_functions[algo]\n\n\ndef hash_algo_for_digest(hexdigest: str) -> str:\n \"\"\"Gets name of the hash algorithm for a hex digest.\"\"\"\n algo = _size_to_hash.get(len(hexdigest) // 2)\n if algo is None:\n raise ValueError(f\"Spack knows no hash algorithm for this digest: {hexdigest}\")\n return algo\n\n\ndef hash_fun_for_digest(hexdigest: str) -> HashFactory:\n \"\"\"Gets a hash function corresponding to a hex digest.\"\"\"\n return hash_fun_for_algo(hash_algo_for_digest(hexdigest))\n\n\ndef checksum_stream(hashlib_algo: HashFactory, fp: BinaryIO, *, block_size: int = 2**20) -> str:\n \"\"\"Returns a hex digest of the stream generated using given algorithm from hashlib.\"\"\"\n hasher = hashlib_algo()\n while True:\n data = fp.read(block_size)\n if not data:\n break\n hasher.update(data)\n return hasher.hexdigest()\n\n\ndef checksum(hashlib_algo: HashFactory, filename: str, *, block_size: int = 2**20) -> str:\n \"\"\"Returns a hex digest of the filename generated using an algorithm from hashlib.\"\"\"\n with open(filename, \"rb\") as f:\n return checksum_stream(hashlib_algo, f, block_size=block_size)\n\n\nclass Checker:\n \"\"\"A checker checks files against one particular hex digest.\n It will automatically determine what hashing algorithm\n to used based on the length of the digest it's initialized\n with. e.g., if the digest is 32 hex characters long this will\n use md5.\n\n Example: know your tarball should hash to ``abc123``. You want\n to check files against this. You would use this class like so::\n\n hexdigest = 'abc123'\n checker = Checker(hexdigest)\n success = checker.check('downloaded.tar.gz')\n\n After the call to check, the actual checksum is available in\n checker.sum, in case it's needed for error output.\n\n You can trade read performance and memory usage by\n adjusting the block_size optional arg. By default it's\n a 1MB (2**20 bytes) buffer.\n \"\"\"\n\n def __init__(self, hexdigest: str, **kwargs) -> None:\n self.block_size = kwargs.get(\"block_size\", 2**20)\n self.hexdigest = hexdigest\n self.sum: Optional[str] = None\n self.hash_fun = hash_fun_for_digest(hexdigest)\n\n @property\n def hash_name(self) -> str:\n \"\"\"Get the name of the hash function this Checker is using.\"\"\"\n return self.hash_fun().name.lower()\n\n def check(self, filename: str) -> bool:\n \"\"\"Read the file with the specified name and check its checksum\n against self.hexdigest. Return True if they match, False\n otherwise. Actual checksum is stored in self.sum.\n \"\"\"\n self.sum = checksum(self.hash_fun, filename, block_size=self.block_size)\n return self.sum == self.hexdigest\n\n\ndef prefix_bits(byte_array, bits):\n \"\"\"Return the first bits of a byte array as an integer.\"\"\"\n b2i = lambda b: b # In Python 3, indexing byte_array gives int\n result = 0\n n = 0\n for i, b in enumerate(byte_array):\n n += 8\n result = (result << 8) | b2i(b)\n if n >= bits:\n break\n\n result >>= n - bits\n return result\n\n\ndef bit_length(num):\n \"\"\"Number of bits required to represent an integer in binary.\"\"\"\n s = bin(num)\n s = s.lstrip(\"-0b\")\n return len(s)\n" }, { "path": "lib/spack/spack/util/ctest_log_parser.py", "content": "# pylint: skip-file\n# -----------------------------------------------------------------------------\n# CMake - Cross Platform Makefile Generator\n# Copyright 2000-2017 Kitware, Inc. and Contributors\n# All rights reserved.\n#\n# Redistribution and use in source and binary forms, with or without\n# modification, are permitted provided that the following conditions\n# are met:\n#\n# * Redistributions of source code must retain the above copyright\n# notice, this list of conditions and the following disclaimer.\n#\n# * Redistributions in binary form must reproduce the above copyright\n# notice, this list of conditions and the following disclaimer in the\n# documentation and/or other materials provided with the distribution.\n#\n# * Neither the name of Kitware, Inc. nor the names of Contributors\n# may be used to endorse or promote products derived from this\n# software without specific prior written permission.\n#\n# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n# \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n#\n# -----------------------------------------------------------------------------\n#\n# The above copyright and license notice applies to distributions of\n# CMake in source and binary form. Third-party software packages supplied\n# with CMake under compatible licenses provide their own copyright notices\n# documented in corresponding subdirectories or source files.\n#\n# -----------------------------------------------------------------------------\n#\n# CMake was initially developed by Kitware with the following sponsorship:\n#\n# * National Library of Medicine at the National Institutes of Health\n# as part of the Insight Segmentation and Registration Toolkit (ITK).\n#\n# * US National Labs (Los Alamos, Livermore, Sandia) ASC Parallel\n# Visualization Initiative.\n#\n# * National Alliance for Medical Image Computing (NAMIC) is funded by the\n# National Institutes of Health through the NIH Roadmap for Medical\n# Research, Grant U54 EB005149.\n#\n# * Kitware, Inc.\n# -----------------------------------------------------------------------------\n\"\"\"Functions to parse build logs and extract error messages.\n\nThis is a python port of the regular expressions CTest uses to parse log\nfiles here:\n\n.. code-block::\n\n https://github.com/Kitware/CMake/blob/master/Source/CTest/cmCTestBuildHandler.cxx\n\nThis file takes the regexes verbatim from there and adds some parsing\nalgorithms that duplicate the way CTest scrapes log files. To keep this\nup to date with CTest, just make sure the ``*_matches`` and\n``*_exceptions`` lists are kept up to date with CTest's build handler.\n\"\"\"\n\nimport io\nimport math\nimport re\nimport time\nfrom collections import deque\nfrom contextlib import contextmanager\nfrom typing import List, TextIO, Tuple, Union\n\n_error_matches = [\n \"^FAIL: \",\n \"^FATAL: \",\n \"^failed \",\n \"FAILED\",\n \"Failed test\",\n \"^[Bb]us [Ee]rror\",\n \"^[Ss]egmentation [Vv]iolation\",\n \"^[Ss]egmentation [Ff]ault\",\n \"Permission [Dd]enied\",\n \"permission [Dd]enied\",\n \":[0-9]+: [^ \\\\t]\",\n \": error[ \\\\t]*[0-9]+[ \\\\t]*:\",\n \"^Error ([0-9]+):\",\n \"^Fatal\",\n \"^[Ee]rror: \",\n \"^Error \",\n \" ERROR: \",\n '^\"[^\"]+\", line [0-9]+: [^Ww]',\n \"^cc[^C]*CC: ERROR File = ([^,]+), Line = ([0-9]+)\",\n \"^ld([^:])*:([ \\\\t])*ERROR([^:])*:\",\n \"^ild:([ \\\\t])*\\\\(undefined symbol\\\\)\",\n \": (error|fatal error|catastrophic error)\",\n \": (Error:|error|undefined reference|multiply defined)\",\n \"\\\\([^\\\\)]+\\\\) ?: (error|fatal error|catastrophic error)\",\n \"^fatal error C[0-9]+:\",\n \": syntax error \",\n \"^collect2: ld returned 1 exit status\",\n \"ld terminated with signal\",\n \"Unsatisfied symbol\",\n \"^Unresolved:\",\n \"Undefined symbol\",\n \"^Undefined[ \\\\t]+first referenced\",\n \"^CMake Error\",\n \":[ \\\\t]cannot find\",\n \":[ \\\\t]can't find\",\n \": \\\\*\\\\*\\\\* No rule to make target [`'].*\\\\'. Stop\",\n \": \\\\*\\\\*\\\\* No targets specified and no makefile found\",\n \": Invalid loader fixup for symbol\",\n \": Invalid fixups exist\",\n \": Can't find library for\",\n \": internal link edit command failed\",\n \": Unrecognized option [`'].*\\\\'\",\n '\", line [0-9]+\\\\.[0-9]+: [0-9]+-[0-9]+ \\\\([^WI]\\\\)',\n \"ld: 0706-006 Cannot find or open library file: -l \",\n \"ild: \\\\(argument error\\\\) can't find library argument ::\",\n \"^could not be found and will not be loaded.\",\n \"^WARNING: '.*' is missing on your system\",\n \"s:616 string too big\",\n \"make: Fatal error: \",\n \"ld: 0711-993 Error occurred while writing to the output file:\",\n \"ld: fatal: \",\n \"final link failed:\",\n \"make: \\\\*\\\\*\\\\*.*Error\",\n \"make\\\\[.*\\\\]: \\\\*\\\\*\\\\*.*Error\",\n \"\\\\*\\\\*\\\\* Error code\",\n \"nternal error:\",\n \"Makefile:[0-9]+: \\\\*\\\\*\\\\* .* Stop\\\\.\",\n \": No such file or directory\",\n \": Invalid argument\",\n \"^The project cannot be built\\\\.\",\n \"^\\\\[ERROR\\\\]\",\n \"^Command .* failed with exit code\",\n]\n\n_error_exceptions = [\n \"instantiated from \",\n \"candidates are:\",\n \": warning\",\n \": WARNING\",\n \": \\\\(Warning\\\\)\",\n \": note\",\n \" ok\",\n \"Note:\",\n \":[ \\\\t]+Where:\",\n \":[0-9]+: Warning\",\n \"------ Build started: .* ------\",\n]\n\n#: Regexes to match file/line numbers in error/warning messages\n_warning_matches = [\n \":[0-9]+: warning:\",\n \":[0-9]+: note:\",\n \"^cc[^C]*CC: WARNING File = ([^,]+), Line = ([0-9]+)\",\n \"^ld([^:])*:([ \\\\t])*WARNING([^:])*:\",\n \": warning [0-9]+:\",\n '^\"[^\"]+\", line [0-9]+: [Ww](arning|arnung)',\n \": warning[ \\\\t]*[0-9]+[ \\\\t]*:\",\n \"^(Warning|Warnung) ([0-9]+):\",\n \"^(Warning|Warnung)[ :]\",\n \"WARNING: \",\n \": warning\",\n '\", line [0-9]+\\\\.[0-9]+: [0-9]+-[0-9]+ \\\\([WI]\\\\)',\n \"^cxx: Warning:\",\n \"file: .* has no symbols\",\n \":[0-9]+: (Warning|Warnung)\",\n \"\\\\([0-9]*\\\\): remark #[0-9]*\",\n '\".*\", line [0-9]+: remark\\\\([0-9]*\\\\):',\n \"cc-[0-9]* CC: REMARK File = .*, Line = [0-9]*\",\n \"^CMake Warning\",\n \"^\\\\[WARNING\\\\]\",\n]\n\n#: Regexes to match file/line numbers in error/warning messages\n_warning_exceptions = [\n \"/usr/.*/X11/Xlib\\\\.h:[0-9]+: war.*: ANSI C\\\\+\\\\+ forbids declaration\",\n \"/usr/.*/X11/Xutil\\\\.h:[0-9]+: war.*: ANSI C\\\\+\\\\+ forbids declaration\",\n \"/usr/.*/X11/XResource\\\\.h:[0-9]+: war.*: ANSI C\\\\+\\\\+ forbids declaration\",\n \"WARNING 84 :\",\n \"WARNING 47 :\",\n \"warning: Clock skew detected. Your build may be incomplete.\",\n \"/usr/openwin/include/GL/[^:]+:\",\n \"bind_at_load\",\n \"XrmQGetResource\",\n \"IceFlush\",\n \"warning LNK4089: all references to [^ \\\\t]+ discarded by .OPT:REF\",\n \"ld32: WARNING 85: definition of dataKey in\",\n 'cc: warning 422: Unknown option \"\\\\+b',\n \"_with_warning_C\",\n]\n\n#: Regexes to match file/line numbers in error/warning messages\n_file_line_matches = [\n \"^Warning W[0-9]+ ([a-zA-Z.\\\\:/0-9_+ ~-]+) ([0-9]+):\",\n \"^([a-zA-Z./0-9_+ ~-]+):([0-9]+):\",\n \"^([a-zA-Z.\\\\:/0-9_+ ~-]+)\\\\(([0-9]+)\\\\)\",\n \"^[0-9]+>([a-zA-Z.\\\\:/0-9_+ ~-]+)\\\\(([0-9]+)\\\\)\",\n \"^([a-zA-Z./0-9_+ ~-]+)\\\\(([0-9]+)\\\\)\",\n '\"([a-zA-Z./0-9_+ ~-]+)\", line ([0-9]+)',\n \"File = ([a-zA-Z./0-9_+ ~-]+), Line = ([0-9]+)\",\n]\n\n\nclass LogEvent:\n \"\"\"Class representing interesting events (e.g., errors) in a build log.\"\"\"\n\n #: color name when rendering in the terminal\n color = \"\"\n\n def __init__(\n self,\n text,\n line_no,\n source_file=None,\n source_line_no=None,\n pre_context=None,\n post_context=None,\n ):\n self.text = text\n self.line_no = line_no\n self.source_file = (source_file,)\n self.source_line_no = (source_line_no,)\n self.pre_context = pre_context if pre_context is not None else []\n self.post_context = post_context if post_context is not None else []\n self.repeat_count = 0\n\n @property\n def start(self):\n \"\"\"First line in the log with text for the event or its context.\"\"\"\n return self.line_no - len(self.pre_context)\n\n @property\n def end(self):\n \"\"\"Last line in the log with text for event or its context.\"\"\"\n return self.line_no + len(self.post_context) + 1\n\n def __getitem__(self, line_no):\n \"\"\"Index event text and context by actual line number in file.\"\"\"\n if line_no == self.line_no:\n return self.text\n elif line_no < self.line_no:\n return self.pre_context[line_no - self.line_no]\n elif line_no > self.line_no:\n return self.post_context[line_no - self.line_no - 1]\n\n def __str__(self):\n \"\"\"Returns event lines and context.\"\"\"\n out = io.StringIO()\n for i in range(self.start, self.end):\n if i == self.line_no:\n out.write(\" >> %-6d%s\" % (i, self[i]))\n else:\n out.write(\" %-6d%s\" % (i, self[i]))\n return out.getvalue()\n\n\nclass BuildError(LogEvent):\n \"\"\"LogEvent subclass for build errors.\"\"\"\n\n color = \"R\"\n\n\nclass BuildWarning(LogEvent):\n \"\"\"LogEvent subclass for build warnings.\"\"\"\n\n color = \"Y\"\n\n\ndef chunks(xs, n):\n \"\"\"Divide xs into n approximately-even chunks.\"\"\"\n chunksize = int(math.ceil(len(xs) / n))\n return [xs[i : i + chunksize] for i in range(0, len(xs), chunksize)]\n\n\n@contextmanager\ndef _time(times, i):\n start = time.time()\n yield\n end = time.time()\n times[i] += end - start\n\n\ndef _match(matches, exceptions, line):\n \"\"\"True if line matches a regex in matches and none in exceptions.\"\"\"\n return any(m.search(line) for m in matches) and not any(e.search(line) for e in exceptions)\n\n\ndef _profile_match(matches, exceptions, line, match_times, exc_times):\n \"\"\"Profiled version of match().\n\n Timing is expensive so we have two whole functions. This is much\n longer because we have to break up the ``any()`` calls.\n\n \"\"\"\n for i, m in enumerate(matches):\n with _time(match_times, i):\n if m.search(line):\n break\n else:\n return False\n\n for i, m in enumerate(exceptions):\n with _time(exc_times, i):\n if m.search(line):\n return False\n else:\n return True\n\n\ndef _parse(stream, profile, context):\n def compile(regex_array):\n return [re.compile(regex) for regex in regex_array]\n\n error_matches = compile(_error_matches)\n error_exceptions = compile(_error_exceptions)\n warning_matches = compile(_warning_matches)\n warning_exceptions = compile(_warning_exceptions)\n file_line_matches = compile(_file_line_matches)\n\n matcher, _ = _match, []\n timings = []\n if profile:\n matcher = _profile_match\n timings = [\n [0.0] * len(error_matches),\n [0.0] * len(error_exceptions),\n [0.0] * len(warning_matches),\n [0.0] * len(warning_exceptions),\n ]\n\n errors = []\n warnings = []\n # rolling window of recent lines\n pre_context = deque(maxlen=context)\n # list of (event, remaining_post_context_lines)\n pending_events: List[Tuple[LogEvent, int]] = []\n\n for i, line in enumerate(stream):\n rstripped_line = line.rstrip()\n\n # feed this line into every event still collecting post_context\n if pending_events:\n active_events = []\n for event, remaining in pending_events:\n event.post_context.append(rstripped_line)\n if remaining > 1:\n active_events.append((event, remaining - 1))\n elif isinstance(event, BuildError):\n errors.append(event)\n else:\n warnings.append(event)\n pending_events = active_events\n\n # use CTest's regular expressions to scrape the log for events\n if matcher(error_matches, error_exceptions, line, *timings[:2]):\n event = BuildError(rstripped_line, i + 1)\n elif matcher(warning_matches, warning_exceptions, line, *timings[2:]):\n event = BuildWarning(rstripped_line, i + 1)\n else:\n pre_context.append(rstripped_line)\n continue\n\n event.pre_context = list(pre_context)\n event.post_context = []\n\n # get file/line number for the event, if possible\n for flm in file_line_matches:\n match = flm.search(line)\n if match:\n event.source_file, event.source_line_no = match.groups()\n break\n\n if context > 0:\n pending_events.append((event, context))\n elif isinstance(event, BuildError):\n errors.append(event)\n else:\n warnings.append(event)\n\n pre_context.append(rstripped_line)\n\n # flush events whose post_context window extends past EOF\n for event, _ in pending_events:\n if isinstance(event, BuildError):\n errors.append(event)\n else:\n warnings.append(event)\n\n return errors, warnings, timings\n\n\nclass CTestLogParser:\n \"\"\"Log file parser that extracts errors and warnings.\"\"\"\n\n def __init__(self, profile=False):\n # whether to record timing information\n self.timings = []\n self.profile = profile\n\n def print_timings(self):\n \"\"\"Print out profile of time spent in different regular expressions.\"\"\"\n\n def stringify(elt):\n return elt if isinstance(elt, str) else elt.pattern\n\n index = 0\n for name, arr in [\n (\"error_matches\", _error_matches),\n (\"error_exceptions\", _error_exceptions),\n (\"warning_matches\", _warning_matches),\n (\"warning_exceptions\", _warning_exceptions),\n ]:\n print()\n print(name)\n for i, elt in enumerate(arr):\n print(\"%16.2f %s\" % (self.timings[index][i] * 1e6, stringify(elt)))\n index += 1\n\n def parse(\n self, stream: Union[str, TextIO], context: int = 6\n ) -> Tuple[List[BuildError], List[BuildWarning]]:\n \"\"\"Parse a log file by searching each line for errors and warnings.\n\n Args:\n stream: filename or stream to read from\n context: lines of context to extract around each log event\n\n Returns:\n two lists containing :class:`BuildError` and :class:`BuildWarning` objects.\n \"\"\"\n if isinstance(stream, str):\n with open(stream, encoding=\"utf-8\", errors=\"replace\") as f:\n return self.parse(f, context)\n\n errors, warnings, self.timings = _parse(stream, self.profile, context)\n return errors, warnings\n" }, { "path": "lib/spack/spack/util/editor.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Module for finding the user's preferred text editor.\n\nDefines one function, editor(), which invokes the editor defined by the\nuser's VISUAL environment variable if set. We fall back to the editor\ndefined by the EDITOR environment variable if VISUAL is not set or the\nspecified editor fails (e.g. no DISPLAY for a graphical editor). If\nneither variable is set, we fall back to one of several common editors,\nraising an OSError if we are unable to find one.\n\"\"\"\n\nimport os\nimport shlex\nfrom typing import Callable, List\n\nimport spack.config\nimport spack.llnl.util.tty as tty\nimport spack.util.executable\n\n#: editors to try if VISUAL and EDITOR are not set\n_default_editors = [\"vim\", \"vi\", \"emacs\", \"nano\", \"notepad\"]\n\n\ndef _find_exe_from_env_var(var: str):\n \"\"\"Find an executable from an environment variable.\n\n Args:\n var (str): environment variable name\n\n Returns:\n (str or None, list): executable string (or None if not found) and\n arguments parsed from the env var\n \"\"\"\n # try to get the environment variable\n exe = os.environ.get(var)\n if not exe:\n return None, []\n\n # split env var into executable and args if needed\n args = shlex.split(str(exe))\n\n if not args:\n return None, []\n\n exe = spack.util.executable.which_string(args[0])\n args = [exe] + args[1:]\n return exe, args\n\n\ndef executable(exe: str, args: List[str]) -> int:\n \"\"\"Wrapper that makes ``spack.util.executable.Executable`` look like ``os.execv()``.\n\n Use this with ``editor()`` if you want it to return instead of running ``execv``.\n \"\"\"\n cmd = spack.util.executable.Executable(exe)\n cmd(*args[1:], fail_on_error=False)\n return cmd.returncode\n\n\ndef editor(*args: str, exec_fn: Callable[[str, List[str]], int] = os.execv) -> bool:\n \"\"\"Invoke the user's editor.\n\n This will try to execute the following, in order:\n\n 1. ``$VISUAL ``: the \"visual\" editor (per POSIX)\n 2. ``$EDITOR ``: the regular editor (per POSIX)\n 3. some default editor (see ``_default_editors``) with \n\n If an environment variable isn't defined, it is skipped. If it\n points to something that can't be executed, we'll print a\n warning. And if we can't find anything that can be executed after\n searching the full list above, we'll raise an error.\n\n Arguments:\n args: args to pass to editor\n\n exec_fn: invoke this function to run; use ``spack.util.editor.executable`` if you\n want something that returns, instead of the default ``os.execv()``.\n \"\"\"\n\n def try_exec(exe, args, var=None):\n \"\"\"Try to execute an editor with execv, and warn if it fails.\n\n Returns: (bool) False if the editor failed, ideally does not\n return if ``execv`` succeeds, and ``True`` if the\n ``exec`` does return successfully.\n \"\"\"\n # gvim runs in the background by default so we force it to run\n # in the foreground to ensure it gets attention.\n if \"gvim\" in exe and \"-f\" not in args:\n exe, *rest = args\n args = [exe, \"-f\"] + rest\n\n try:\n return exec_fn(exe, args) == 0\n\n except (OSError, spack.util.executable.ProcessError) as e:\n if spack.config.get(\"config:debug\"):\n raise\n\n # Show variable we were trying to use, if it's from one\n if var:\n exe = \"$%s (%s)\" % (var, exe)\n tty.warn(\"Could not execute %s due to error:\" % exe, str(e))\n return False\n\n def try_env_var(var):\n \"\"\"Find an editor from an environment variable and try to exec it.\n\n This will warn if the variable points to something is not\n executable, or if there is an error when trying to exec it.\n \"\"\"\n if var not in os.environ:\n return False\n\n exe, editor_args = _find_exe_from_env_var(var)\n if not exe:\n tty.warn(\"$%s is not an executable:\" % var, os.environ[var])\n return False\n\n full_args = editor_args + list(args)\n return try_exec(exe, full_args, var)\n\n # try standard environment variables\n if try_env_var(\"SPACK_EDITOR\"):\n return True\n if try_env_var(\"VISUAL\"):\n return True\n if try_env_var(\"EDITOR\"):\n return True\n\n # nothing worked -- try the first default we can find don't bother\n # trying them all -- if we get here and one fails, something is\n # probably much more deeply wrong with the environment.\n exe = spack.util.executable.which_string(*_default_editors)\n if exe and try_exec(exe, [exe] + list(args)):\n return True\n\n # Fail if nothing could be found\n raise OSError(\n \"No text editor found! Please set the VISUAL and/or EDITOR \"\n \"environment variable(s) to your preferred text editor.\"\n )\n" }, { "path": "lib/spack/spack/util/elf.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport bisect\nimport re\nimport struct\nfrom struct import calcsize, unpack, unpack_from\nfrom typing import BinaryIO, Callable, Dict, List, NamedTuple, Optional, Pattern, Tuple\n\n\nclass ElfHeader(NamedTuple):\n e_type: int\n e_machine: int\n e_version: int\n e_entry: int\n e_phoff: int\n e_shoff: int\n e_flags: int\n e_ehsize: int\n e_phentsize: int\n e_phnum: int\n e_shentsize: int\n e_shnum: int\n e_shstrndx: int\n\n\nclass SectionHeader(NamedTuple):\n sh_name: int\n sh_type: int\n sh_flags: int\n sh_addr: int\n sh_offset: int\n sh_size: int\n sh_link: int\n sh_info: int\n sh_addralign: int\n sh_entsize: int\n\n\nclass ProgramHeader32(NamedTuple):\n p_type: int\n p_offset: int\n p_vaddr: int\n p_paddr: int\n p_filesz: int\n p_memsz: int\n p_flags: int\n p_align: int\n\n\nclass ProgramHeader64(NamedTuple):\n p_type: int\n p_flags: int\n p_offset: int\n p_vaddr: int\n p_paddr: int\n p_filesz: int\n p_memsz: int\n p_align: int\n\n\nclass ELF_CONSTANTS:\n MAGIC = b\"\\x7fELF\"\n CLASS32 = 1\n CLASS64 = 2\n DATA2LSB = 1\n DATA2MSB = 2\n ET_EXEC = 2\n ET_DYN = 3\n PT_LOAD = 1\n PT_DYNAMIC = 2\n PT_INTERP = 3\n DT_NULL = 0\n DT_NEEDED = 1\n DT_STRTAB = 5\n DT_SONAME = 14\n DT_RPATH = 15\n DT_RUNPATH = 29\n SHT_STRTAB = 3\n\n\nclass ElfFile:\n \"\"\"Parsed ELF file.\"\"\"\n\n is_64_bit: bool\n is_little_endian: bool\n byte_order: str\n elf_hdr: ElfHeader\n pt_load: List[Tuple[int, int]]\n has_pt_interp: bool\n pt_interp_p_offset: int\n pt_interp_p_filesz: int\n pt_interp_str: bytes\n has_pt_dynamic: bool\n pt_dynamic_p_offset: int\n pt_dynamic_p_filesz: int\n pt_dynamic_strtab_offset: int\n has_rpath: bool\n dt_rpath_offset: int\n dt_rpath_str: bytes\n rpath_strtab_offset: int\n is_runpath: bool\n has_needed: bool\n dt_needed_strtab_offsets: List[int]\n dt_needed_strs: List[bytes]\n has_soname: bool\n dt_soname_strtab_offset: int\n dt_soname_str: bytes\n\n __slots__ = [\n \"is_64_bit\",\n \"is_little_endian\",\n \"byte_order\",\n \"elf_hdr\",\n \"pt_load\",\n # pt_interp\n \"has_pt_interp\",\n \"pt_interp_p_offset\",\n \"pt_interp_p_filesz\",\n \"pt_interp_str\",\n # pt_dynamic\n \"has_pt_dynamic\",\n \"pt_dynamic_p_offset\",\n \"pt_dynamic_p_filesz\",\n \"pt_dynamic_strtab_offset\", # string table for dynamic section\n # rpath\n \"has_rpath\",\n \"dt_rpath_offset\",\n \"dt_rpath_str\",\n \"rpath_strtab_offset\",\n \"is_runpath\",\n # dt needed\n \"has_needed\",\n \"dt_needed_strtab_offsets\",\n \"dt_needed_strs\",\n # dt soname\n \"has_soname\",\n \"dt_soname_strtab_offset\",\n \"dt_soname_str\",\n ]\n\n def __init__(self):\n self.dt_needed_strtab_offsets = []\n self.has_soname = False\n self.has_rpath = False\n self.has_needed = False\n self.pt_load = []\n self.has_pt_dynamic = False\n self.has_pt_interp = False\n\n\ndef parse_c_string(byte_string: bytes, start: int = 0) -> bytes:\n \"\"\"\n Retrieve a C-string at a given offset in a byte string\n\n Arguments:\n byte_string: String\n start: Offset into the string\n\n Returns:\n bytes: A copy of the C-string excluding the terminating null byte\n \"\"\"\n str_end = byte_string.find(b\"\\0\", start)\n if str_end == -1:\n raise ElfParsingError(\"C-string is not null terminated\")\n return byte_string[start:str_end]\n\n\ndef read_exactly(f: BinaryIO, num_bytes: int, msg: str) -> bytes:\n \"\"\"\n Read exactly num_bytes at the current offset, otherwise raise\n a parsing error with the given error message.\n\n Arguments:\n f: file handle\n num_bytes: Number of bytes to read\n msg: Error to show when bytes cannot be read\n\n Returns:\n bytes: the ``num_bytes`` bytes that were read.\n \"\"\"\n data = f.read(num_bytes)\n if len(data) != num_bytes:\n raise ElfParsingError(msg)\n return data\n\n\ndef parse_program_headers(f: BinaryIO, elf: ElfFile) -> None:\n \"\"\"\n Parse program headers\n\n Arguments:\n f: file handle\n elf: ELF file parser data\n \"\"\"\n # Forward to the program header\n try:\n f.seek(elf.elf_hdr.e_phoff)\n except OSError:\n raise ElfParsingError(\"Could not seek to program header\")\n\n # Here we have to make a mapping from virtual address to offset in the file.\n ph_fmt = elf.byte_order + (\"LLQQQQQQ\" if elf.is_64_bit else \"LLLLLLLL\")\n ph_size = calcsize(ph_fmt)\n ph_num = elf.elf_hdr.e_phnum\n\n # Read all program headers in one go\n data = read_exactly(f, ph_num * ph_size, \"Malformed program header\")\n\n ProgramHeader = ProgramHeader64 if elf.is_64_bit else ProgramHeader32\n\n for i in range(ph_num):\n # mypy currently does not understand the union of two named tuples with equal fields\n ph = ProgramHeader(*unpack_from(ph_fmt, data, i * ph_size))\n\n # Skip segments of size 0; we don't distinguish between missing segment and\n # empty segments. I've see an empty PT_DYNAMIC section for an ELF file that\n # contained debug data.\n if ph.p_filesz == 0: # type: ignore\n continue\n\n # For PT_LOAD entries: Save offsets and virtual addrs of the loaded ELF segments\n # This way we can map offsets by virtual address to offsets in the file.\n if ph.p_type == ELF_CONSTANTS.PT_LOAD: # type: ignore\n elf.pt_load.append((ph.p_offset, ph.p_vaddr)) # type: ignore\n\n elif ph.p_type == ELF_CONSTANTS.PT_INTERP: # type: ignore\n elf.pt_interp_p_offset = ph.p_offset # type: ignore\n elf.pt_interp_p_filesz = ph.p_filesz # type: ignore\n elf.has_pt_interp = True\n\n elif ph.p_type == ELF_CONSTANTS.PT_DYNAMIC: # type: ignore\n elf.pt_dynamic_p_offset = ph.p_offset # type: ignore\n elf.pt_dynamic_p_filesz = ph.p_filesz # type: ignore\n elf.has_pt_dynamic = True\n\n # The linker sorts PT_LOAD segments by vaddr, but let's do it just to be sure, since\n # patchelf for example has a flag to leave them in an arbitrary order.\n elf.pt_load.sort(key=lambda x: x[1])\n\n\ndef parse_pt_interp(f: BinaryIO, elf: ElfFile) -> None:\n \"\"\"\n Parse the interpreter (i.e. absolute path to the dynamic linker)\n\n Arguments:\n f: file handle\n elf: ELF file parser data\n \"\"\"\n try:\n f.seek(elf.pt_interp_p_offset)\n except OSError:\n raise ElfParsingError(\"Could not seek to PT_INTERP entry\")\n data = read_exactly(f, elf.pt_interp_p_filesz, \"Malformed PT_INTERP entry\")\n elf.pt_interp_str = parse_c_string(data)\n\n\ndef find_strtab_size_at_offset(f: BinaryIO, elf: ElfFile, offset: int) -> int:\n \"\"\"\n Retrieve the size of a string table section at a particular known offset\n\n Arguments:\n f: file handle\n elf: ELF file parser data\n offset: offset of the section in the file (i.e. ``sh_offset``)\n\n Returns:\n int: the size of the string table in bytes\n \"\"\"\n section_hdr_fmt = elf.byte_order + (\"LLQQQQLLQQ\" if elf.is_64_bit else \"LLLLLLLLLL\")\n section_hdr_size = calcsize(section_hdr_fmt)\n try:\n f.seek(elf.elf_hdr.e_shoff)\n except OSError:\n raise ElfParsingError(\"Could not seek to section header table\")\n for _ in range(elf.elf_hdr.e_shnum):\n data = read_exactly(f, section_hdr_size, \"Malformed section header\")\n sh = SectionHeader(*unpack(section_hdr_fmt, data))\n if sh.sh_type == ELF_CONSTANTS.SHT_STRTAB and sh.sh_offset == offset:\n return sh.sh_size\n\n raise ElfParsingError(\"Could not determine strtab size\")\n\n\ndef retrieve_strtab(f: BinaryIO, elf: ElfFile, offset: int) -> bytes:\n \"\"\"\n Read a full string table at the given offset, which\n requires looking it up in the section headers.\n\n Arguments:\n elf: ELF file parser data\n vaddr: virtual address\n\n Returns: file offset\n \"\"\"\n size = find_strtab_size_at_offset(f, elf, offset)\n try:\n f.seek(offset)\n except OSError:\n raise ElfParsingError(\"Could not seek to string table\")\n return read_exactly(f, size, \"Could not read string table\")\n\n\ndef vaddr_to_offset(elf: ElfFile, vaddr: int) -> int:\n \"\"\"\n Given a virtual address, find the corresponding offset in the ELF file itself.\n\n Arguments:\n elf: ELF file parser data\n vaddr: virtual address\n \"\"\"\n idx = bisect.bisect_right([p_vaddr for (p_offset, p_vaddr) in elf.pt_load], vaddr) - 1\n p_offset, p_vaddr = elf.pt_load[idx]\n return p_offset - p_vaddr + vaddr\n\n\ndef parse_pt_dynamic(f: BinaryIO, elf: ElfFile) -> None:\n \"\"\"\n Parse the dynamic section of an ELF file\n\n Arguments:\n f: file handle\n elf: ELF file parse data\n \"\"\"\n dynamic_array_fmt = elf.byte_order + (\"qQ\" if elf.is_64_bit else \"lL\")\n dynamic_array_size = calcsize(dynamic_array_fmt)\n\n current_offset = elf.pt_dynamic_p_offset\n count_rpath = 0\n count_runpath = 0\n count_strtab = 0\n\n try:\n f.seek(elf.pt_dynamic_p_offset)\n except OSError:\n raise ElfParsingError(\"Could not seek to PT_DYNAMIC entry\")\n\n # In case of broken ELF files, don't read beyond the advertised size.\n for _ in range(elf.pt_dynamic_p_filesz // dynamic_array_size):\n data = read_exactly(f, dynamic_array_size, \"Malformed dynamic array entry\")\n tag, val = unpack(dynamic_array_fmt, data)\n if tag == ELF_CONSTANTS.DT_NULL:\n break\n elif tag == ELF_CONSTANTS.DT_RPATH:\n count_rpath += 1\n elf.rpath_strtab_offset = val\n elf.dt_rpath_offset = current_offset\n elf.is_runpath = False\n elf.has_rpath = True\n elif tag == ELF_CONSTANTS.DT_RUNPATH:\n count_runpath += 1\n elf.rpath_strtab_offset = val\n elf.dt_rpath_offset = current_offset\n elf.is_runpath = True\n elf.has_rpath = True\n elif tag == ELF_CONSTANTS.DT_STRTAB:\n count_strtab += 1\n strtab_vaddr = val\n elif tag == ELF_CONSTANTS.DT_NEEDED:\n elf.has_needed = True\n elf.dt_needed_strtab_offsets.append(val)\n elif tag == ELF_CONSTANTS.DT_SONAME:\n elf.has_soname = True\n elf.dt_soname_strtab_offset = val\n current_offset += dynamic_array_size\n\n # No rpath/runpath, that happens.\n if count_rpath == count_runpath == 0:\n elf.has_rpath = False\n elif count_rpath + count_runpath != 1:\n raise ElfParsingError(\"Could not find a unique rpath/runpath.\")\n\n if count_strtab != 1:\n raise ElfParsingError(\"Could not find a unique strtab of for the dynamic section strings\")\n\n # Nothing to retrieve, so don't bother getting the string table.\n if not (elf.has_rpath or elf.has_soname or elf.has_needed):\n return\n\n elf.pt_dynamic_strtab_offset = vaddr_to_offset(elf, strtab_vaddr)\n string_table = retrieve_strtab(f, elf, elf.pt_dynamic_strtab_offset)\n\n if elf.has_needed:\n elf.dt_needed_strs = list(\n parse_c_string(string_table, offset) for offset in elf.dt_needed_strtab_offsets\n )\n\n if elf.has_soname:\n elf.dt_soname_str = parse_c_string(string_table, elf.dt_soname_strtab_offset)\n\n if elf.has_rpath:\n elf.dt_rpath_str = parse_c_string(string_table, elf.rpath_strtab_offset)\n\n\ndef parse_header(f: BinaryIO, elf: ElfFile) -> None:\n # Read the 32/64 bit class independent part of the header and validate\n e_ident = f.read(16)\n\n # Require ELF magic bytes.\n if len(e_ident) != 16 or e_ident[:4] != ELF_CONSTANTS.MAGIC:\n raise ElfParsingError(\"Not an ELF file\")\n\n # Defensively require a valid class and data.\n e_ident_class, e_ident_data = e_ident[4], e_ident[5]\n\n if e_ident_class not in (ELF_CONSTANTS.CLASS32, ELF_CONSTANTS.CLASS64):\n raise ElfParsingError(\"Invalid class found\")\n\n if e_ident_data not in (ELF_CONSTANTS.DATA2LSB, ELF_CONSTANTS.DATA2MSB):\n raise ElfParsingError(\"Invalid data type\")\n\n elf.is_64_bit = e_ident_class == ELF_CONSTANTS.CLASS64\n elf.is_little_endian = e_ident_data == ELF_CONSTANTS.DATA2LSB\n\n # Set up byte order and types for unpacking\n elf.byte_order = \"<\" if elf.is_little_endian else \">\"\n\n # Parse the rest of the header\n elf_header_fmt = elf.byte_order + (\"HHLQQQLHHHHHH\" if elf.is_64_bit else \"HHLLLLLHHHHHH\")\n hdr_size = calcsize(elf_header_fmt)\n data = read_exactly(f, hdr_size, \"ELF header malformed\")\n elf.elf_hdr = ElfHeader(*unpack(elf_header_fmt, data))\n\n\ndef _do_parse_elf(\n f: BinaryIO, interpreter: bool = True, dynamic_section: bool = True, only_header: bool = False\n) -> ElfFile:\n # We don't (yet?) allow parsing ELF files at a nonzero offset, we just\n # jump to absolute offsets as they are specified in the ELF file.\n if f.tell() != 0:\n raise ElfParsingError(\"Cannot parse at a nonzero offset\")\n\n elf = ElfFile()\n parse_header(f, elf)\n\n if only_header:\n return elf\n\n # We don't handle anything but executables and shared libraries now.\n if elf.elf_hdr.e_type not in (ELF_CONSTANTS.ET_EXEC, ELF_CONSTANTS.ET_DYN):\n raise ElfParsingError(\"Not an ET_DYN or ET_EXEC type\")\n\n parse_program_headers(f, elf)\n\n # Parse PT_INTERP section\n if interpreter and elf.has_pt_interp:\n parse_pt_interp(f, elf)\n\n # Parse PT_DYNAMIC section.\n if dynamic_section and elf.has_pt_dynamic and len(elf.pt_load) > 0:\n parse_pt_dynamic(f, elf)\n\n return elf\n\n\ndef parse_elf(\n f: BinaryIO,\n interpreter: bool = False,\n dynamic_section: bool = False,\n only_header: bool = False,\n) -> ElfFile:\n \"\"\"Given a file handle ``f`` for an ELF file opened in binary mode, return an\n :class:`~spack.util.elf.ElfFile` object with the parsed contents.\"\"\"\n try:\n return _do_parse_elf(f, interpreter, dynamic_section, only_header)\n except (DeprecationWarning, struct.error):\n # According to the docs old versions of Python can throw DeprecationWarning\n # instead of struct.error.\n raise ElfParsingError(\"Malformed ELF file\")\n\n\ndef get_rpaths(path: str) -> Optional[List[str]]:\n \"\"\"Returns list of rpaths of the given file as UTF-8 strings, or None if not set.\"\"\"\n try:\n with open(path, \"rb\") as f:\n elf = parse_elf(f, interpreter=False, dynamic_section=True)\n return elf.dt_rpath_str.decode(\"utf-8\").split(\":\") if elf.has_rpath else None\n except ElfParsingError:\n return None\n\n\ndef get_interpreter(path: str) -> Optional[str]:\n \"\"\"Returns the interpreter of the given file as UTF-8 string, or None if not set.\"\"\"\n try:\n with open(path, \"rb\") as f:\n elf = parse_elf(f, interpreter=True, dynamic_section=False)\n return elf.pt_interp_str.decode(\"utf-8\") if elf.has_pt_interp else None\n except ElfParsingError:\n return None\n\n\ndef _delete_dynamic_array_entry(\n f: BinaryIO, elf: ElfFile, should_delete: Callable[[int, int], bool]\n) -> None:\n try:\n f.seek(elf.pt_dynamic_p_offset)\n except OSError:\n raise ElfParsingError(\"Could not seek to PT_DYNAMIC entry\")\n dynamic_array_fmt = elf.byte_order + (\"qQ\" if elf.is_64_bit else \"lL\")\n dynamic_array_size = calcsize(dynamic_array_fmt)\n new_offset = elf.pt_dynamic_p_offset # points to the new dynamic array\n old_offset = elf.pt_dynamic_p_offset # points to the current dynamic array\n for _ in range(elf.pt_dynamic_p_filesz // dynamic_array_size):\n data = read_exactly(f, dynamic_array_size, \"Malformed dynamic array entry\")\n tag, val = unpack(dynamic_array_fmt, data)\n\n if tag == ELF_CONSTANTS.DT_NULL or not should_delete(tag, val):\n if new_offset != old_offset:\n f.seek(new_offset)\n f.write(data)\n f.seek(old_offset + dynamic_array_size)\n new_offset += dynamic_array_size\n\n if tag == ELF_CONSTANTS.DT_NULL:\n break\n\n old_offset += dynamic_array_size\n\n\ndef delete_rpath(path: str) -> None:\n \"\"\"Modifies a binary to remove the rpath. It zeros out the rpath string and also drops the\n ``DT_RPATH`` / ``DT_RUNPATH`` entry from the dynamic section, so it doesn't show up in\n ``readelf -d file``, nor in ``strings file``.\"\"\"\n with open(path, \"rb+\") as f:\n elf = parse_elf(f, interpreter=False, dynamic_section=True)\n\n if not elf.has_rpath:\n return\n\n # Zero out the rpath *string* in the binary\n new_rpath_string = b\"\\x00\" * len(elf.dt_rpath_str)\n rpath_offset = elf.pt_dynamic_strtab_offset + elf.rpath_strtab_offset\n f.seek(rpath_offset)\n f.write(new_rpath_string)\n\n # Delete DT_RPATH / DT_RUNPATH entries from the dynamic section\n _delete_dynamic_array_entry(\n f, elf, lambda tag, _: tag == ELF_CONSTANTS.DT_RPATH or tag == ELF_CONSTANTS.DT_RUNPATH\n )\n\n\ndef delete_needed_from_elf(f: BinaryIO, elf: ElfFile, needed: bytes) -> None:\n \"\"\"Delete a needed library from the dynamic section of an ELF file\"\"\"\n if not elf.has_needed or needed not in elf.dt_needed_strs:\n return\n\n offset = elf.dt_needed_strtab_offsets[elf.dt_needed_strs.index(needed)]\n\n _delete_dynamic_array_entry(\n f, elf, lambda tag, val: tag == ELF_CONSTANTS.DT_NEEDED and val == offset\n )\n\n\nclass CStringType:\n PT_INTERP = 1\n RPATH = 2\n\n\nclass UpdateCStringAction:\n def __init__(self, old_value: bytes, new_value: bytes, offset: int):\n self.old_value = old_value\n self.new_value = new_value\n self.offset = offset\n\n @property\n def inplace(self) -> bool:\n return len(self.new_value) <= len(self.old_value)\n\n def apply(self, f: BinaryIO) -> None:\n assert self.inplace\n\n f.seek(self.offset)\n f.write(self.new_value)\n\n # We zero out the bits we shortened because (a) it should be a\n # C-string and (b) it's nice not to have spurious parts of old\n # paths in the output of `strings file`. Note that we're all\n # good when pad == 0; the original terminating null is used.\n f.write(b\"\\x00\" * (len(self.old_value) - len(self.new_value)))\n\n\ndef _get_rpath_substitution(\n elf: ElfFile, regex: Pattern, substitutions: Dict[bytes, bytes]\n) -> Optional[UpdateCStringAction]:\n \"\"\"Make rpath substitutions in-place.\"\"\"\n # If there's no RPATH, then there's no need to replace anything.\n if not elf.has_rpath:\n return None\n\n # Get the non-empty rpaths. Sometimes there's a bunch of trailing\n # colons ::::: used for padding, we don't add them back to make it\n # more likely that the string doesn't grow.\n rpaths = list(filter(len, elf.dt_rpath_str.split(b\":\")))\n\n num_rpaths = len(rpaths)\n\n if num_rpaths == 0:\n return None\n\n changed = False\n for i in range(num_rpaths):\n old_rpath = rpaths[i]\n match = regex.match(old_rpath)\n if match:\n changed = True\n rpaths[i] = substitutions[match.group()] + old_rpath[match.end() :]\n\n # Nothing to replace!\n if not changed:\n return None\n\n return UpdateCStringAction(\n old_value=elf.dt_rpath_str,\n new_value=b\":\".join(rpaths),\n # The rpath is at a given offset in the string table used by the dynamic section.\n offset=elf.pt_dynamic_strtab_offset + elf.rpath_strtab_offset,\n )\n\n\ndef _get_pt_interp_substitution(\n elf: ElfFile, regex: Pattern, substitutions: Dict[bytes, bytes]\n) -> Optional[UpdateCStringAction]:\n \"\"\"Make interpreter substitutions in-place.\"\"\"\n if not elf.has_pt_interp:\n return None\n\n match = regex.match(elf.pt_interp_str)\n if not match:\n return None\n\n return UpdateCStringAction(\n old_value=elf.pt_interp_str,\n new_value=substitutions[match.group()] + elf.pt_interp_str[match.end() :],\n offset=elf.pt_interp_p_offset,\n )\n\n\ndef substitute_rpath_and_pt_interp_in_place_or_raise(\n path: str, substitutions: Dict[bytes, bytes]\n) -> bool:\n \"\"\"Returns true if the rpath and interpreter were modified, false if there was nothing to do.\n Raises ElfCStringUpdatesFailed if the ELF file cannot be updated in-place. This exception\n contains a list of actions to perform with other tools. The file is left untouched in this\n case.\"\"\"\n regex = re.compile(b\"|\".join(re.escape(p) for p in substitutions.keys()))\n\n try:\n with open(path, \"rb+\") as f:\n elf = parse_elf(f, interpreter=True, dynamic_section=True)\n\n # Get the actions to perform.\n rpath = _get_rpath_substitution(elf, regex, substitutions)\n pt_interp = _get_pt_interp_substitution(elf, regex, substitutions)\n\n # Nothing to do.\n if not rpath and not pt_interp:\n return False\n\n # If we can't update in-place, leave it to other tools, don't do partial updates.\n if rpath and not rpath.inplace or pt_interp and not pt_interp.inplace:\n raise ElfCStringUpdatesFailed(rpath, pt_interp)\n\n # Otherwise, apply the updates.\n if rpath:\n rpath.apply(f)\n\n if pt_interp:\n pt_interp.apply(f)\n\n return True\n\n except ElfParsingError:\n # This just means the file wasn't an elf file, so there's no point\n # in updating its rpath anyways; ignore this problem.\n return False\n\n\ndef pt_interp(path: str) -> Optional[str]:\n \"\"\"Retrieve the interpreter of an executable at ``path``.\"\"\"\n try:\n with open(path, \"rb\") as f:\n elf = parse_elf(f, interpreter=True)\n except (OSError, ElfParsingError):\n return None\n\n if not elf.has_pt_interp:\n return None\n\n return elf.pt_interp_str.decode(\"utf-8\")\n\n\ndef get_elf_compat(path: str) -> Tuple[bool, bool, int]:\n \"\"\"Get a triplet (is_64_bit, is_little_endian, e_machine) from an ELF file, which can be used\n to see if two ELF files are compatible.\"\"\"\n # On ELF platforms supporting, we try to be a bit smarter when it comes to shared\n # libraries, by dropping those that are not host compatible.\n with open(path, \"rb\") as f:\n elf = parse_elf(f, only_header=True)\n return (elf.is_64_bit, elf.is_little_endian, elf.elf_hdr.e_machine)\n\n\nclass ElfCStringUpdatesFailed(Exception):\n def __init__(\n self, rpath: Optional[UpdateCStringAction], pt_interp: Optional[UpdateCStringAction]\n ):\n self.rpath = rpath\n self.pt_interp = pt_interp\n\n\nclass ElfParsingError(Exception):\n pass\n" }, { "path": "lib/spack/spack/util/environment.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Set, unset or modify environment variables.\"\"\"\n\nimport collections\nimport contextlib\nimport inspect\nimport json\nimport os\nimport pathlib\nimport pickle\nimport re\nimport shlex\nimport subprocess\nimport sys\nimport warnings\nfrom typing import Any, Callable, Dict, Iterable, List, MutableMapping, Optional, Tuple, Union\n\nimport spack.error\nfrom spack.llnl.path import path_to_os_path, system_path_filter\nfrom spack.llnl.util import tty\nfrom spack.llnl.util.lang import dedupe\n\n# List is invariant, so List[str] is not a subtype of List[Union[str, pathlib.PurePath]].\n# Sequence is covariant, but because str itself is a subtype of Sequence[str], we cannot exclude it\n# in the type hint. So, use an awkward union type to allow (mixed) str and PurePath items.\nListOfPaths = Union[List[str], List[pathlib.PurePath], List[Union[str, pathlib.PurePath]]]\n\n\nif sys.platform == \"win32\":\n SYSTEM_PATHS = [\n \"C:\\\\\",\n \"C:\\\\Program Files\",\n \"C:\\\\Program Files (x86)\",\n \"C:\\\\Users\",\n \"C:\\\\ProgramData\",\n ]\n SUFFIXES = []\n DEFAULT_SHELL = os.environ.get(\"SPACK_SHELL\", \"bat\")\nelse:\n SYSTEM_PATHS = [\"/\", \"/usr\", \"/usr/local\"]\n SUFFIXES = [\"bin\", \"bin64\", \"include\", \"lib\", \"lib64\"]\n DEFAULT_SHELL = \"sh\"\n\nSYSTEM_DIRS = [os.path.join(p, s) for s in SUFFIXES for p in SYSTEM_PATHS] + SYSTEM_PATHS\n\n#: used in the compiler wrapper's ``/usr/lib|/usr/lib64|...)`` case entry\nSYSTEM_DIR_CASE_ENTRY = \"|\".join(sorted(f'\"{d}{suff}\"' for d in SYSTEM_DIRS for suff in (\"\", \"/\")))\n\n_SHELL_SET_STRINGS = {\n \"sh\": \"export {0}={1};\\n\",\n \"csh\": \"setenv {0} {1};\\n\",\n \"fish\": \"set -gx {0} {1};\\n\",\n \"bat\": 'set \"{0}={1}\"\\n',\n \"pwsh\": \"$Env:{0}='{1}'\\n\",\n}\n\n\n_SHELL_UNSET_STRINGS = {\n \"sh\": \"unset {0};\\n\",\n \"csh\": \"unsetenv {0};\\n\",\n \"fish\": \"set -e {0};\\n\",\n \"bat\": 'set \"{0}=\"\\n',\n \"pwsh\": \"Set-Item -Path Env:{0}\\n\",\n}\n\n\nTRACING_ENABLED = False\n\nPath = str\nModificationList = List[Union[\"NameModifier\", \"NameValueModifier\"]]\n\n\ndef is_system_path(path: Path) -> bool:\n \"\"\"Returns True if the argument is a system path, False otherwise.\"\"\"\n return bool(path) and (os.path.normpath(path) in SYSTEM_DIRS)\n\n\ndef filter_system_paths(paths: Iterable[Path]) -> List[Path]:\n \"\"\"Returns a copy of the input where system paths are filtered out.\"\"\"\n return [p for p in paths if not is_system_path(p)]\n\n\ndef deprioritize_system_paths(paths: List[Path]) -> List[Path]:\n \"\"\"Reorders input paths by putting system paths at the end of the list, otherwise\n preserving order.\n \"\"\"\n return list(sorted(paths, key=is_system_path))\n\n\ndef prune_duplicate_paths(paths: List[Path]) -> List[Path]:\n \"\"\"Returns the input list with duplicates removed, otherwise preserving order.\"\"\"\n return list(dedupe(paths))\n\n\ndef get_path(name: str) -> List[Path]:\n \"\"\"Given the name of an environment variable containing multiple\n paths separated by :data:`os.pathsep`, returns a list of the paths.\n \"\"\"\n path = os.environ.get(name, \"\").strip()\n if path:\n return path.split(os.pathsep)\n return []\n\n\ndef env_flag(name: str) -> bool:\n \"\"\"Given the name of an environment variable, returns True if the lowercase value is set to\n ``true`` or to ``1``, False otherwise.\n \"\"\"\n if name in os.environ:\n value = os.environ[name].lower()\n return value in (\"true\", \"1\")\n return False\n\n\ndef path_set(var_name: str, directories: List[Path]):\n \"\"\"Sets the variable passed as input to the :data:`os.pathsep` joined list of directories.\"\"\"\n path_str = os.pathsep.join(str(dir) for dir in directories)\n os.environ[var_name] = path_str\n\n\ndef path_put_first(var_name: str, directories: List[Path]):\n \"\"\"Puts the provided directories first in the path, adding them\n if they're not already there.\n \"\"\"\n path = os.environ.get(var_name, \"\").split(os.pathsep)\n\n for directory in directories:\n if directory in path:\n path.remove(directory)\n\n new_path = list(directories) + list(path)\n path_set(var_name, new_path)\n\n\nBASH_FUNCTION_FINDER = re.compile(r\"BASH_FUNC_(.*?)\\(\\)\")\n\n\ndef _win_env_var_to_set_line(var: str, val: str) -> str:\n is_pwsh = os.environ.get(\"SPACK_SHELL\", None) == \"pwsh\"\n env_set_phrase = f\"$Env:{var}={val}\" if is_pwsh else f'set \"{var}={val}\"'\n return env_set_phrase\n\n\ndef _nix_env_var_to_source_line(var: str, val: str) -> str:\n if var.startswith(\"BASH_FUNC\"):\n source_line = \"function {fname}{decl}; export -f {fname}\".format(\n fname=BASH_FUNCTION_FINDER.sub(r\"\\1\", var), decl=val\n )\n else:\n source_line = f\"{var}={shlex.quote(val)}; export {var}\"\n return source_line\n\n\ndef _env_var_to_source_line(var: str, val: str) -> str:\n if sys.platform == \"win32\":\n return _win_env_var_to_set_line(var, val)\n else:\n return _nix_env_var_to_source_line(var, val)\n\n\n@system_path_filter(arg_slice=slice(1))\ndef dump_environment(path: Path, environment: Optional[MutableMapping[str, str]] = None):\n \"\"\"Dump an environment dictionary to a source-able file.\n\n Args:\n path: path of the file to write\n environment: environment to be written. If None os.environ is used.\n \"\"\"\n use_env = environment or os.environ\n hidden_vars = {\"PS1\", \"PWD\", \"OLDPWD\", \"TERM_SESSION_ID\"}\n\n file_descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)\n with os.fdopen(file_descriptor, \"w\", encoding=\"utf-8\") as env_file:\n for var, val in sorted(use_env.items()):\n env_file.write(\n \"\".join(\n [\"#\" if var in hidden_vars else \"\", _env_var_to_source_line(var, val), \"\\n\"]\n )\n )\n\n\n@system_path_filter(arg_slice=slice(1))\ndef pickle_environment(path: Path, environment: Optional[Dict[str, str]] = None):\n \"\"\"Pickle an environment dictionary to a file.\"\"\"\n with open(path, \"wb\") as pickle_file:\n pickle.dump(dict(environment if environment else os.environ), pickle_file, protocol=2)\n\n\n@contextlib.contextmanager\ndef set_env(**kwargs):\n \"\"\"Temporarily sets and restores environment variables. Variables can be set as keyword\n arguments to this function.\n\n .. note::\n\n If the goal is to set environment variables for a subprocess, it is strongly recommended to\n use the ``extra_env`` argument of :func:`spack.util.executable.Executable.__call__` instead\n of this function.\n\n This function is intended to modify the *current* process's environment (which is an unsafe\n operation in general).\n \"\"\"\n saved = {}\n for var, value in kwargs.items():\n if var in os.environ:\n saved[var] = os.environ[var]\n\n if value is None:\n if var in os.environ:\n del os.environ[var]\n else:\n os.environ[var] = value\n\n yield\n\n for var, value in kwargs.items():\n if var in saved:\n os.environ[var] = saved[var]\n else:\n if var in os.environ:\n del os.environ[var]\n\n\nclass Trace:\n \"\"\"Trace information on a function call\"\"\"\n\n __slots__ = (\"filename\", \"lineno\", \"context\")\n\n def __init__(self, *, filename: str, lineno: int, context: str):\n self.filename = filename\n self.lineno = lineno\n self.context = context\n\n def __str__(self):\n return f\"{self.context} at {self.filename}:{self.lineno}\"\n\n def __repr__(self):\n return f\"Trace(filename={self.filename}, lineno={self.lineno}, context={self.context})\"\n\n\nclass NameModifier:\n \"\"\"Base class for modifiers that act on the environment variable as a whole, and thus\n store just its name\n \"\"\"\n\n __slots__ = (\"name\", \"separator\", \"trace\")\n\n def __init__(self, name: str, *, separator: str = os.pathsep, trace: Optional[Trace] = None):\n self.name = name.upper() if sys.platform == \"win32\" else name\n self.separator = separator\n self.trace = trace\n\n def __eq__(self, other: object):\n if not isinstance(other, NameModifier):\n return NotImplemented\n return self.name == other.name\n\n def execute(self, env: MutableMapping[str, str]):\n \"\"\"Apply the modification to the mapping passed as input\"\"\"\n raise NotImplementedError(\"must be implemented by derived classes\")\n\n\nclass NameValueModifier:\n \"\"\"Base class for modifiers that modify the value of an environment variable.\"\"\"\n\n __slots__ = (\"name\", \"value\", \"separator\", \"trace\")\n\n def __init__(\n self, name: str, value: str, *, separator: str = os.pathsep, trace: Optional[Trace] = None\n ):\n self.name = name.upper() if sys.platform == \"win32\" else name\n self.value = value\n self.separator = separator\n self.trace = trace\n\n def __eq__(self, other: object):\n if not isinstance(other, NameValueModifier):\n return NotImplemented\n return (\n self.name == other.name\n and self.value == other.value\n and self.separator == other.separator\n )\n\n def execute(self, env: MutableMapping[str, str]):\n \"\"\"Apply the modification to the mapping passed as input\"\"\"\n raise NotImplementedError(\"must be implemented by derived classes\")\n\n\nclass NamePathModifier(NameValueModifier):\n \"\"\"Base class for modifiers that modify the value of an environment variable\n that is a path.\"\"\"\n\n __slots__ = ()\n\n def __init__(\n self,\n name: str,\n value: Union[str, pathlib.PurePath],\n *,\n separator: str = os.pathsep,\n trace: Optional[Trace] = None,\n ):\n super().__init__(name, str(value), separator=separator, trace=trace)\n\n\nclass SetEnv(NameValueModifier):\n __slots__ = (\"force\", \"raw\")\n\n def __init__(\n self,\n name: str,\n value: str,\n *,\n trace: Optional[Trace] = None,\n force: bool = False,\n raw: bool = False,\n ):\n super().__init__(name, value, trace=trace)\n self.force = force\n self.raw = raw\n\n def execute(self, env: MutableMapping[str, str]):\n tty.debug(f\"SetEnv: {self.name}={self.value}\", level=3)\n env[self.name] = self.value\n\n\nclass AppendFlagsEnv(NameValueModifier):\n def execute(self, env: MutableMapping[str, str]):\n tty.debug(f\"AppendFlagsEnv: {self.name}={self.value}\", level=3)\n if self.name in env and env[self.name]:\n env[self.name] += self.separator + self.value\n else:\n env[self.name] = self.value\n\n\nclass UnsetEnv(NameModifier):\n def execute(self, env: MutableMapping[str, str]):\n tty.debug(f\"UnsetEnv: {self.name}\", level=3)\n # Avoid throwing if the variable was not set\n env.pop(self.name, None)\n\n\nclass RemoveFlagsEnv(NameValueModifier):\n def execute(self, env: MutableMapping[str, str]):\n tty.debug(f\"RemoveFlagsEnv: {self.name}-{self.value}\", level=3)\n environment_value = env.get(self.name, \"\")\n flags = environment_value.split(self.separator) if environment_value else []\n flags = [f for f in flags if f != self.value]\n env[self.name] = self.separator.join(flags)\n\n\nclass SetPath(NameValueModifier):\n def __init__(\n self,\n name: str,\n value: ListOfPaths,\n *,\n separator: str = os.pathsep,\n trace: Optional[Trace] = None,\n ):\n super().__init__(\n name, separator.join(str(x) for x in value), separator=separator, trace=trace\n )\n\n def execute(self, env: MutableMapping[str, str]):\n tty.debug(f\"SetPath: {self.name}={self.value}\", level=3)\n env[self.name] = self.value\n\n\nclass AppendPath(NamePathModifier):\n def execute(self, env: MutableMapping[str, str]):\n tty.debug(f\"AppendPath: {self.name}+{self.value}\", level=3)\n environment_value = env.get(self.name, \"\")\n directories = environment_value.split(self.separator) if environment_value else []\n directories.append(path_to_os_path(os.path.normpath(self.value)).pop())\n env[self.name] = self.separator.join(directories)\n\n\nclass PrependPath(NamePathModifier):\n def execute(self, env: MutableMapping[str, str]):\n tty.debug(f\"PrependPath: {self.name}+{self.value}\", level=3)\n environment_value = env.get(self.name, \"\")\n directories = environment_value.split(self.separator) if environment_value else []\n directories = [path_to_os_path(os.path.normpath(self.value)).pop()] + directories\n env[self.name] = self.separator.join(directories)\n\n\nclass RemoveFirstPath(NamePathModifier):\n def execute(self, env: MutableMapping[str, str]):\n tty.debug(f\"RemoveFirstPath: {self.name}-{self.value}\", level=3)\n environment_value = env.get(self.name, \"\")\n directories = environment_value.split(self.separator)\n directories = [path_to_os_path(os.path.normpath(x)).pop() for x in directories]\n val = path_to_os_path(os.path.normpath(self.value)).pop()\n if val in directories:\n directories.remove(val)\n env[self.name] = self.separator.join(directories)\n\n\nclass RemoveLastPath(NamePathModifier):\n def execute(self, env: MutableMapping[str, str]):\n tty.debug(f\"RemoveLastPath: {self.name}-{self.value}\", level=3)\n environment_value = env.get(self.name, \"\")\n directories = environment_value.split(self.separator)[::-1]\n directories = [path_to_os_path(os.path.normpath(x)).pop() for x in directories]\n val = path_to_os_path(os.path.normpath(self.value)).pop()\n if val in directories:\n directories.remove(val)\n env[self.name] = self.separator.join(directories[::-1])\n\n\nclass RemovePath(NamePathModifier):\n def execute(self, env: MutableMapping[str, str]):\n tty.debug(f\"RemovePath: {self.name}-{self.value}\", level=3)\n environment_value = env.get(self.name, \"\")\n directories = environment_value.split(self.separator)\n directories = [\n path_to_os_path(os.path.normpath(x)).pop()\n for x in directories\n if x != path_to_os_path(os.path.normpath(self.value)).pop()\n ]\n env[self.name] = self.separator.join(directories)\n\n\nclass DeprioritizeSystemPaths(NameModifier):\n def execute(self, env: MutableMapping[str, str]):\n tty.debug(f\"DeprioritizeSystemPaths: {self.name}\", level=3)\n environment_value = env.get(self.name, \"\")\n directories = environment_value.split(self.separator) if environment_value else []\n directories = deprioritize_system_paths(\n [path_to_os_path(os.path.normpath(x)).pop() for x in directories]\n )\n env[self.name] = self.separator.join(directories)\n\n\nclass PruneDuplicatePaths(NameModifier):\n def execute(self, env: MutableMapping[str, str]):\n tty.debug(f\"PruneDuplicatePaths: {self.name}\", level=3)\n environment_value = env.get(self.name, \"\")\n directories = environment_value.split(self.separator) if environment_value else []\n directories = prune_duplicate_paths(\n [path_to_os_path(os.path.normpath(x)).pop() for x in directories]\n )\n env[self.name] = self.separator.join(directories)\n\n\ndef _validate_path_value(name: str, value: Any) -> Union[str, pathlib.PurePath]:\n \"\"\"Ensure the value for an env variable is string or path\"\"\"\n types = (str, pathlib.PurePath)\n if isinstance(value, types):\n return value\n types_str = \" or \".join([f\"`{t.__name__}`\" for t in types])\n warnings.warn(\n f\"when setting environment variable {name}={value}: value is of type \"\n f\"`{type(value).__name__}`, but {types_str} was expected. This is deprecated and will be \"\n f\"an error in Spack v1.0\",\n spack.error.SpackAPIWarning,\n stacklevel=3,\n )\n return str(value)\n\n\ndef _validate_value(name: str, value: Any) -> str:\n \"\"\"Ensure the value for an env variable is a string\"\"\"\n if isinstance(value, str):\n return value\n warnings.warn(\n f\"when setting environment variable {name}={value}: value is of type \"\n f\"`{type(value).__name__}`, but `str` was expected. This is deprecated and will be an \"\n \"error in Spack v1.0\",\n spack.error.SpackAPIWarning,\n stacklevel=3,\n )\n return str(value)\n\n\nclass EnvironmentModifications:\n \"\"\"\n Tracks and applies a sequence of environment variable modifications.\n\n This class provides a high-level interface for building up a list of environment changes,\n such as setting, unsetting, appending, prepending, or removing values from environment\n variables. Modifications are stored and can be applied to a given environment dictionary, or\n rendered as shell code.\n\n Package authors typically receive an instance of this class and call :meth:`set`,\n :meth:`unset`, :meth:`prepend_path`, :meth:`remove_path`, etc., to queue up modifications.\n Spack runs :meth:`apply_modifications` to apply these modifications to the environment when\n needed.\n\n Modifications can be grouped by variable name, reversed (where possible), validated for\n suspicious patterns, and extended from other instances. The class also supports tracing the\n origin of modifications for debugging.\n\n Example:\n\n .. code-block:: python\n\n env = EnvironmentModifications()\n env.set(\"FOO\", \"bar\")\n env.prepend_path(\"PATH\", \"/custom/bin\")\n env.apply_modifications() # applies changes to os.environ\n \"\"\"\n\n def __init__(\n self, other: Optional[\"EnvironmentModifications\"] = None, traced: Union[None, bool] = None\n ):\n \"\"\"Initializes a new instance, copying commands from 'other' if it is not None.\n\n Args:\n other: list of environment modifications to be extended (optional)\n traced: enable or disable stack trace inspection to log the origin\n of the environment modifications\n \"\"\"\n self.traced = TRACING_ENABLED if traced is None else bool(traced)\n self.env_modifications: List[Union[NameModifier, NameValueModifier]] = []\n if other is not None:\n self.extend(other)\n\n def __iter__(self):\n return iter(self.env_modifications)\n\n def __len__(self):\n return len(self.env_modifications)\n\n def extend(self, other: \"EnvironmentModifications\"):\n \"\"\"Extends the current instance with modifications from another instance.\"\"\"\n self._check_other(other)\n self.env_modifications.extend(other.env_modifications)\n\n @staticmethod\n def _check_other(other: \"EnvironmentModifications\"):\n if not isinstance(other, EnvironmentModifications):\n raise TypeError(\"other must be an instance of EnvironmentModifications\")\n\n def _trace(self) -> Optional[Trace]:\n \"\"\"Returns a trace object if tracing is enabled, else None.\"\"\"\n if not self.traced:\n return None\n\n stack = inspect.stack()\n try:\n _, filename, lineno, _, context, index = stack[2]\n assert index is not None, \"index must be an integer\"\n current_context = context[index].strip() if context is not None else \"unknown context\"\n except Exception:\n filename = \"unknown file\"\n lineno = -1\n current_context = \"unknown context\"\n\n return Trace(filename=filename, lineno=lineno, context=current_context)\n\n def set(self, name: str, value: str, *, force: bool = False, raw: bool = False) -> None:\n \"\"\"Stores a request to set an environment variable.\n\n Args:\n name: name of the environment variable\n value: value of the environment variable\n force: if True, audit will not consider this modification a warning\n raw: if True, format of value string is skipped\n \"\"\"\n value = _validate_value(name, value)\n item = SetEnv(name, value, trace=self._trace(), force=force, raw=raw)\n self.env_modifications.append(item)\n\n def append_flags(self, name: str, value: str, sep: str = \" \") -> None:\n \"\"\"Stores a request to append flags to an environment variable.\n\n Args:\n name: name of the environment variable\n value: flags to be appended\n sep: separator for the flags (default: ``\" \"``)\n \"\"\"\n value = _validate_value(name, value)\n item = AppendFlagsEnv(name, value, separator=sep, trace=self._trace())\n self.env_modifications.append(item)\n\n def unset(self, name: str) -> None:\n \"\"\"Stores a request to unset an environment variable.\n\n Args:\n name: name of the environment variable\n \"\"\"\n item = UnsetEnv(name, trace=self._trace())\n self.env_modifications.append(item)\n\n def remove_flags(self, name: str, value: str, sep: str = \" \") -> None:\n \"\"\"Stores a request to remove flags from an environment variable\n\n Args:\n name: name of the environment variable\n value: flags to be removed\n sep: separator for the flags (default: ``\" \"``)\n \"\"\"\n value = _validate_value(name, value)\n item = RemoveFlagsEnv(name, value, separator=sep, trace=self._trace())\n self.env_modifications.append(item)\n\n def set_path(self, name: str, elements: ListOfPaths, separator: str = os.pathsep) -> None:\n \"\"\"Stores a request to set an environment variable to a list of paths,\n separated by a character defined in input.\n\n Args:\n name: name of the environment variable\n elements: ordered list paths\n separator: separator for the paths (default: :data:`os.pathsep`)\n \"\"\"\n elements = [_validate_path_value(name, x) for x in elements]\n item = SetPath(name, elements, separator=separator, trace=self._trace())\n self.env_modifications.append(item)\n\n def append_path(\n self, name: str, path: Union[str, pathlib.PurePath], separator: str = os.pathsep\n ) -> None:\n \"\"\"Stores a request to append a path to list of paths.\n\n Args:\n name: name of the environment variable\n path: path to be appended\n separator: separator for the paths (default: :data:`os.pathsep`)\n \"\"\"\n path = _validate_path_value(name, path)\n item = AppendPath(name, path, separator=separator, trace=self._trace())\n self.env_modifications.append(item)\n\n def prepend_path(\n self, name: str, path: Union[str, pathlib.PurePath], separator: str = os.pathsep\n ) -> None:\n \"\"\"Stores a request to prepend a path to list of paths.\n\n Args:\n name: name of the environment variable\n path: path to be prepended\n separator: separator for the paths (default: :data:`os.pathsep`)\n \"\"\"\n path = _validate_path_value(name, path)\n item = PrependPath(name, path, separator=separator, trace=self._trace())\n self.env_modifications.append(item)\n\n def remove_first_path(\n self, name: str, path: Union[str, pathlib.PurePath], separator: str = os.pathsep\n ) -> None:\n \"\"\"Stores a request to remove first instance of path from a list of paths.\n\n Args:\n name: name of the environment variable\n path: path to be removed\n separator: separator for the paths (default: :data:`os.pathsep`)\n \"\"\"\n path = _validate_path_value(name, path)\n item = RemoveFirstPath(name, path, separator=separator, trace=self._trace())\n self.env_modifications.append(item)\n\n def remove_last_path(\n self, name: str, path: Union[str, pathlib.PurePath], separator: str = os.pathsep\n ) -> None:\n \"\"\"Stores a request to remove last instance of path from a list of paths.\n\n Args:\n name: name of the environment variable\n path: path to be removed\n separator: separator for the paths (default: :data:`os.pathsep`)\n \"\"\"\n path = _validate_path_value(name, path)\n item = RemoveLastPath(name, path, separator=separator, trace=self._trace())\n self.env_modifications.append(item)\n\n def remove_path(\n self, name: str, path: Union[str, pathlib.PurePath], separator: str = os.pathsep\n ) -> None:\n \"\"\"Stores a request to remove a path from a list of paths.\n\n Args:\n name: name of the environment variable\n path: path to be removed\n separator: separator for the paths (default: :data:`os.pathsep`)\n \"\"\"\n path = _validate_path_value(name, path)\n item = RemovePath(name, path, separator=separator, trace=self._trace())\n self.env_modifications.append(item)\n\n def deprioritize_system_paths(self, name: str, separator: str = os.pathsep) -> None:\n \"\"\"Stores a request to deprioritize system paths in a path list,\n otherwise preserving the order.\n\n Args:\n name: name of the environment variable\n separator: separator for the paths (default: :data:`os.pathsep`)\n \"\"\"\n item = DeprioritizeSystemPaths(name, separator=separator, trace=self._trace())\n self.env_modifications.append(item)\n\n def prune_duplicate_paths(self, name: str, separator: str = os.pathsep) -> None:\n \"\"\"Stores a request to remove duplicates from a path list, otherwise\n preserving the order.\n\n Args:\n name: name of the environment variable\n separator: separator for the paths (default: :data:`os.pathsep`)\n \"\"\"\n item = PruneDuplicatePaths(name, separator=separator, trace=self._trace())\n self.env_modifications.append(item)\n\n def group_by_name(self) -> Dict[str, ModificationList]:\n \"\"\"Returns a dict of the current modifications keyed by variable name.\"\"\"\n modifications = collections.defaultdict(list)\n for item in self:\n modifications[item.name].append(item)\n return modifications\n\n def drop(self, *name) -> bool:\n \"\"\"Drop all modifications to the variable with the given name.\"\"\"\n old_mods = self.env_modifications\n new_mods = [x for x in self.env_modifications if x.name not in name]\n self.env_modifications = new_mods\n\n return len(old_mods) != len(new_mods)\n\n def is_unset(self, variable_name: str) -> bool:\n \"\"\"Returns :data:`True` if the last modification to a variable is to unset it,\n :data:`False` otherwise.\"\"\"\n modifications = self.group_by_name()\n if variable_name not in modifications:\n return False\n\n # The last modification must unset the variable for it to be considered unset\n return isinstance(modifications[variable_name][-1], UnsetEnv)\n\n def clear(self):\n \"\"\"Clears the current list of modifications.\"\"\"\n self.env_modifications = []\n\n def reversed(self) -> \"EnvironmentModifications\":\n \"\"\"Returns the EnvironmentModifications object that will reverse self\n\n Only creates reversals for additions to the environment, as reversing\n :meth:`unset` and :meth:`remove_path` modifications is impossible.\n\n Reversible operations are :meth:`set`, :meth:`prepend_path`, :meth:`append_path`,\n :meth:`set_path`, and :meth:`append_flags`.\n \"\"\"\n rev = EnvironmentModifications()\n\n for envmod in reversed(self.env_modifications):\n if isinstance(envmod, SetEnv):\n tty.debug(\"Reversing `Set` environment operation may lose the original value\")\n rev.unset(envmod.name)\n elif isinstance(envmod, AppendPath):\n rev.remove_last_path(envmod.name, envmod.value)\n elif isinstance(envmod, PrependPath):\n rev.remove_first_path(envmod.name, envmod.value)\n elif isinstance(envmod, SetPath):\n tty.debug(\"Reversing `SetPath` environment operation may lose the original value\")\n rev.unset(envmod.name)\n elif isinstance(envmod, AppendFlagsEnv):\n rev.remove_flags(envmod.name, envmod.value)\n else:\n tty.debug(\n f\"Skipping reversal of irreversible operation {type(envmod)} {envmod.name}\"\n )\n\n return rev\n\n def apply_modifications(self, env: Optional[MutableMapping[str, str]] = None):\n \"\"\"Applies the modifications to the environment.\n\n Args:\n env: environment to be modified. If None, :obj:`os.environ` will be used.\n \"\"\"\n env = os.environ if env is None else env\n\n modifications = self.group_by_name()\n for _, actions in sorted(modifications.items()):\n for modifier in actions:\n modifier.execute(env)\n\n def shell_modifications(\n self,\n shell: str = DEFAULT_SHELL,\n explicit: bool = False,\n env: Optional[MutableMapping[str, str]] = None,\n ) -> str:\n \"\"\"Return shell code to apply the modifications.\"\"\"\n modifications = self.group_by_name()\n\n env = os.environ if env is None else env\n new_env = dict(env.items())\n\n for _, actions in sorted(modifications.items()):\n for modifier in actions:\n modifier.execute(new_env)\n\n if \"MANPATH\" in new_env and not new_env[\"MANPATH\"].endswith(os.pathsep):\n new_env[\"MANPATH\"] += os.pathsep\n\n cmds = \"\"\n\n for name in sorted(set(modifications)):\n new = new_env.get(name, None)\n old = env.get(name, None)\n if explicit or new != old:\n if new is None:\n cmds += _SHELL_UNSET_STRINGS[shell].format(name)\n else:\n value = new_env[name]\n if shell not in (\"bat\", \"pwsh\"):\n value = shlex.quote(value)\n cmd = _SHELL_SET_STRINGS[shell].format(name, value)\n cmds += cmd\n return cmds\n\n @staticmethod\n def from_sourcing_file(\n filename: Path, *arguments: str, **kwargs: Any\n ) -> \"EnvironmentModifications\":\n \"\"\"Returns the environment modifications that have the same effect as\n sourcing the input file in a shell.\n\n Args:\n filename: the file to be sourced\n *arguments: arguments to pass on the command line\n\n Keyword Args:\n shell (str): the shell to use (default: ``bash``)\n shell_options (str): options passed to the shell (default: ``-c``)\n source_command (str): the command to run (default: ``source``)\n suppress_output (str): redirect used to suppress output of command\n (default: ``&> /dev/null``)\n concatenate_on_success (str): operator used to execute a command\n only when the previous command succeeds (default: ``&&``)\n exclude ([str or re.Pattern[str]]): ignore any modifications of these\n variables (default: [])\n include ([str or re.Pattern[str]]): always respect modifications of these\n variables (default: []). Supersedes any excluded variables.\n clean (bool): in addition to removing empty entries,\n also remove duplicate entries (default: False).\n \"\"\"\n tty.debug(f\"EnvironmentModifications.from_sourcing_file: {filename}\")\n # Check if the file actually exists\n if not os.path.isfile(filename):\n msg = f\"Trying to source non-existing file: {filename}\"\n raise RuntimeError(msg)\n\n # Prepare include and exclude lists of environment variable names\n exclude = kwargs.get(\"exclude\", [])\n include = kwargs.get(\"include\", [])\n clean = kwargs.get(\"clean\", False)\n\n # Other variables unrelated to sourcing a file\n exclude.extend(\n [\n # Bash internals\n \"SHLVL\",\n \"_\",\n \"PWD\",\n \"OLDPWD\",\n \"PS1\",\n \"PS2\",\n \"ENV\",\n # Environment Modules or Lmod\n \"LOADEDMODULES\",\n \"_LMFILES_\",\n \"MODULEPATH\",\n \"MODULERCFILE\",\n \"BASH_FUNC_ml()\",\n \"BASH_FUNC_module()\",\n # Environment Modules-specific configuration\n \"MODULESHOME\",\n \"BASH_FUNC__module_raw()\",\n r\"MODULES_(.*)\",\n r\"__MODULES_(.*)\",\n r\"(\\w*)_mod(quar|share)\",\n # Lmod-specific configuration\n r\"LMOD_(.*)\",\n ]\n )\n\n before_kwargs = {**kwargs}\n if sys.platform == \"win32\":\n # Windows cannot source os.devnull, but it can echo from it\n # so we override the \"source\" action in the method that\n # extracts the env (environment_after_sourcing_files)\n if \"source_command\" not in kwargs:\n before_kwargs[\"source_command\"] = \"echo\"\n\n # Compute the environments before and after sourcing\n\n # First look at the environment after doing nothing to\n # establish baseline\n before = sanitize(\n environment_after_sourcing_files(os.devnull, **before_kwargs),\n exclude=exclude,\n include=include,\n )\n file_and_args = (filename,) + arguments\n after = sanitize(\n environment_after_sourcing_files(file_and_args, **kwargs),\n exclude=exclude,\n include=include,\n )\n\n # Delegate to the other factory\n return EnvironmentModifications.from_environment_diff(before, after, clean)\n\n @staticmethod\n def from_environment_diff(\n before: MutableMapping[str, str], after: MutableMapping[str, str], clean: bool = False\n ) -> \"EnvironmentModifications\":\n \"\"\"Constructs the environment modifications from the diff of two environments.\n\n Args:\n before: environment before the modifications are applied\n after: environment after the modifications are applied\n clean: in addition to removing empty entries, also remove duplicate entries\n \"\"\"\n # Fill the EnvironmentModifications instance\n env = EnvironmentModifications()\n # New variables\n new_variables = list(set(after) - set(before))\n # Variables that have been unset\n unset_variables = list(set(before) - set(after))\n # Variables that have been modified\n common_variables = set(before).intersection(set(after))\n modified_variables = [x for x in common_variables if before[x] != after[x]]\n # Consistent output order - looks nicer, easier comparison...\n new_variables.sort()\n unset_variables.sort()\n modified_variables.sort()\n\n def return_separator_if_any(*args):\n separators = [os.pathsep] if sys.platform == \"win32\" else [\":\", \";\"]\n for separator in separators:\n for arg in args:\n if separator in arg:\n return separator\n return None\n\n # Add variables to env.\n # Assume that variables with 'PATH' in the name or that contain\n # separators like ':' or ';' are more likely to be paths\n for variable_name in new_variables:\n sep = return_separator_if_any(after[variable_name])\n if sep:\n env.prepend_path(variable_name, after[variable_name], separator=sep)\n elif \"PATH\" in variable_name:\n env.prepend_path(variable_name, after[variable_name])\n else:\n # We just need to set the variable to the new value\n env.set(variable_name, after[variable_name])\n\n for variable_name in unset_variables:\n env.unset(variable_name)\n\n for variable_name in modified_variables:\n value_before = before[variable_name]\n value_after = after[variable_name]\n sep = return_separator_if_any(value_before, value_after)\n if sep:\n before_list = value_before.split(sep)\n after_list = value_after.split(sep)\n\n # Filter out empty strings\n before_list = list(filter(None, before_list))\n after_list = list(filter(None, after_list))\n\n # Remove duplicate entries (worse matching, bloats env)\n if clean:\n before_list = list(dedupe(before_list))\n after_list = list(dedupe(after_list))\n # The reassembled cleaned entries\n value_before = sep.join(before_list)\n value_after = sep.join(after_list)\n\n # Paths that have been removed\n remove_list = [ii for ii in before_list if ii not in after_list]\n # Check that nothing has been added in the middle of\n # before_list\n remaining_list = [ii for ii in before_list if ii in after_list]\n try:\n start = after_list.index(remaining_list[0])\n end = after_list.index(remaining_list[-1])\n search = sep.join(after_list[start : end + 1])\n except IndexError:\n env.prepend_path(variable_name, value_after)\n continue\n\n if search not in value_before:\n # We just need to set the variable to the new value\n env.prepend_path(variable_name, value_after)\n else:\n try:\n prepend_list = after_list[:start]\n prepend_list.reverse() # Preserve order after prepend\n except KeyError:\n prepend_list = []\n try:\n append_list = after_list[end + 1 :]\n except KeyError:\n append_list = []\n\n for item in remove_list:\n env.remove_path(variable_name, item)\n for item in append_list:\n env.append_path(variable_name, item)\n for item in prepend_list:\n env.prepend_path(variable_name, item)\n else:\n # We just need to set the variable to the new value\n env.set(variable_name, value_after)\n\n return env\n\n\ndef _set_or_unset_not_first(\n variable: str, changes: ModificationList, errstream: Callable[[str], None]\n):\n \"\"\"Check if we are going to set or unset something after other\n modifications have already been requested.\n \"\"\"\n indexes = [\n ii\n for ii, item in enumerate(changes)\n if ii != 0 and isinstance(item, (SetEnv, UnsetEnv)) and not getattr(item, \"force\", False)\n ]\n if indexes:\n good = \"\\t \\t{}\"\n nogood = \"\\t--->\\t{}\"\n errstream(f\"Different requests to set/unset '{variable}' have been found\")\n for idx, item in enumerate(changes):\n print_format = nogood if idx in indexes else good\n errstream(print_format.format(item.trace))\n\n\ndef validate(env: EnvironmentModifications, errstream: Callable[[str], None]):\n \"\"\"Validates the environment modifications to check for the presence of\n suspicious patterns. Prompts a warning for everything that was found.\n\n Current checks:\n - set or unset variables after other changes on the same variable\n\n Args:\n env: list of environment modifications\n errstream: callable to log error messages\n \"\"\"\n if not env.traced:\n return\n modifications = env.group_by_name()\n for variable, list_of_changes in sorted(modifications.items()):\n _set_or_unset_not_first(variable, list_of_changes, errstream)\n\n\ndef inspect_path(\n root: Path,\n inspections: MutableMapping[str, List[str]],\n exclude: Optional[Callable[[Path], bool]] = None,\n) -> EnvironmentModifications:\n \"\"\"Inspects ``root`` to search for the subdirectories in ``inspections``.\n Adds every path found to a list of prepend-path commands and returns it.\n\n Args:\n root: absolute path where to search for subdirectories\n inspections: maps relative paths to a list of environment\n variables that will be modified if the path exists. The\n modifications are not performed immediately, but stored in a\n command object that is returned to client\n exclude: optional callable. If present it must accept an\n absolute path and return True if it should be excluded from the\n inspection\n\n Examples:\n\n The following lines execute an inspection in ``/usr`` to search for\n ``/usr/include`` and ``/usr/lib64``. If found we want to prepend\n ``/usr/include`` to ``CPATH`` and ``/usr/lib64`` to ``MY_LIB64_PATH``.\n\n .. code-block:: python\n\n # Set up the dictionary containing the inspection\n inspections = {\n \"include\": [\"CPATH\"],\n \"lib64\": [\"MY_LIB64_PATH\"]\n }\n\n # Get back the list of command needed to modify the environment\n env = inspect_path(\"/usr\", inspections)\n\n # Eventually execute the commands\n env.apply_modifications()\n \"\"\"\n if exclude is None:\n exclude = lambda x: False\n\n env = EnvironmentModifications()\n # Inspect the prefix to check for the existence of common directories\n for relative_path, variables in inspections.items():\n expected = os.path.join(root, os.path.normpath(relative_path))\n\n if os.path.isdir(expected) and not exclude(expected):\n for variable in variables:\n env.prepend_path(variable, expected)\n\n return env\n\n\n@contextlib.contextmanager\ndef preserve_environment(*variables: str):\n \"\"\"Ensures that the value of the environment variables passed as\n arguments is the same before entering to the context manager and after\n exiting it.\n\n Variables that are unset before entering the context manager will be\n explicitly unset on exit.\n\n Args:\n variables: list of environment variables to be preserved\n \"\"\"\n cache = {}\n for var in variables:\n # The environment variable to be preserved might not be there.\n # In that case store None as a placeholder.\n cache[var] = os.environ.get(var, None)\n\n yield\n\n for var in variables:\n value = cache[var]\n msg = \"[PRESERVE_ENVIRONMENT]\"\n if value is not None:\n # Print a debug statement if the value changed\n if var not in os.environ:\n msg += ' {0} was unset, will be reset to \"{1}\"'\n tty.debug(msg.format(var, value))\n elif os.environ[var] != value:\n msg += ' {0} was set to \"{1}\", will be reset to \"{2}\"'\n tty.debug(msg.format(var, os.environ[var], value))\n os.environ[var] = value\n elif var in os.environ:\n msg += ' {0} was set to \"{1}\", will be unset'\n tty.debug(msg.format(var, os.environ[var]))\n del os.environ[var]\n\n\ndef environment_after_sourcing_files(\n *files: Union[Path, Tuple[str, ...]], **kwargs\n) -> Dict[str, str]:\n \"\"\"Returns a dictionary with the environment that one would have\n after sourcing the files passed as argument.\n\n Args:\n *files: each item can either be a string containing the path\n of the file to be sourced or a sequence, where the first element\n is the file to be sourced and the remaining are arguments to be\n passed to the command line\n\n Keyword Args:\n env (dict): the initial environment (default: current environment)\n shell (str): the shell to use (default: ``/bin/bash`` or ``cmd.exe`` (Windows))\n shell_options (str): options passed to the shell (default: ``-c`` or ``/C`` (Windows))\n source_command (str): the command to run (default: ``source``)\n suppress_output (str): redirect used to suppress output of command\n (default: ``&> /dev/null``)\n concatenate_on_success (str): operator used to execute a command\n only when the previous command succeeds (default: ``&&``)\n \"\"\"\n # Set the shell executable that will be used to source files\n if sys.platform == \"win32\":\n shell_cmd = kwargs.get(\"shell\", \"cmd.exe\")\n shell_options = kwargs.get(\"shell_options\", \"/C\")\n suppress_output = kwargs.get(\"suppress_output\", \"> nul\")\n source_command = kwargs.get(\"source_command\", \"\")\n else:\n shell_cmd = kwargs.get(\"shell\", \"/bin/bash\")\n shell_options = kwargs.get(\"shell_options\", \"-c\")\n suppress_output = kwargs.get(\"suppress_output\", \"&> /dev/null\")\n source_command = kwargs.get(\"source_command\", \"source\")\n concatenate_on_success = kwargs.get(\"concatenate_on_success\", \"&&\")\n\n def _source_single_file(file_and_args, environment):\n shell_options_list = shell_options.split()\n\n source_file = [source_command]\n source_file.extend(x for x in file_and_args)\n source_file = \" \".join(source_file)\n\n dump_cmd = \"import os, json; print(json.dumps(dict(os.environ)))\"\n dump_environment_cmd = sys.executable + f' -E -c \"{dump_cmd}\"'\n\n # Try to source the file\n source_file_arguments = \" \".join(\n [source_file, suppress_output, concatenate_on_success, dump_environment_cmd]\n )\n\n # Popens argument processing can break command invocations\n # on Windows, compose to a string to avoid said processing\n cmd = [shell_cmd, *shell_options_list, source_file_arguments]\n cmd = \" \".join(cmd) if sys.platform == \"win32\" else cmd\n\n with subprocess.Popen(\n cmd, env=environment, stdout=subprocess.PIPE, stderr=subprocess.PIPE\n ) as shell:\n output, _ = shell.communicate()\n\n return json.loads(output)\n\n current_environment = kwargs.get(\"env\", dict(os.environ))\n for file in files:\n # Normalize the input to the helper function\n if isinstance(file, str):\n file = (file,)\n\n current_environment = _source_single_file(file, environment=current_environment)\n\n return current_environment\n\n\ndef sanitize(\n environment: MutableMapping[str, str], exclude: List[str], include: List[str]\n) -> Dict[str, str]:\n \"\"\"Returns a copy of the input dictionary where all the keys that\n match an excluded pattern and don't match an included pattern are\n removed.\n\n Args:\n environment (dict): input dictionary\n exclude (list): literals or regex patterns to be excluded\n include (list): literals or regex patterns to be included\n \"\"\"\n\n def set_intersection(fullset, *args):\n # A set intersection using string literals and regexs\n meta = \"[\" + re.escape(\"[$()*?[]^{|}\") + \"]\"\n subset = fullset & set(args) # As literal\n for name in args:\n if re.search(meta, name):\n pattern = re.compile(name)\n for k in fullset:\n if re.match(pattern, k):\n subset.add(k)\n return subset\n\n # Don't modify input, make a copy instead\n environment = dict(environment)\n\n # include supersedes any excluded items\n prune = set_intersection(set(environment), *exclude)\n prune -= set_intersection(prune, *include)\n for k in prune:\n environment.pop(k, None)\n\n return environment\n" }, { "path": "lib/spack/spack/util/executable.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport io\nimport os\nimport re\nimport shlex\nimport subprocess\nimport sys\nfrom pathlib import Path, PurePath\nfrom typing import BinaryIO, Callable, Dict, List, Optional, Sequence, Tuple, Type, Union, overload\n\nfrom spack.vendor.typing_extensions import Literal\n\nimport spack.error\nimport spack.llnl.util.tty as tty\nfrom spack.util.environment import EnvironmentModifications\n\n__all__ = [\"Executable\", \"which\", \"which_string\", \"ProcessError\"]\n\nOutType = Union[Optional[BinaryIO], str, Type[str], Callable]\n\n\ndef _process_cmd_output(\n out: bytes,\n err: bytes,\n output: OutType,\n error: OutType,\n encoding: str = \"ISO-8859-1\" if sys.platform == \"win32\" else \"utf-8\",\n) -> Optional[str]:\n if output is str or output is str.split or error is str or error is str.split:\n result = \"\"\n if output is str or output is str.split:\n outstr = out.decode(encoding)\n result += outstr\n if output is str.split:\n sys.stdout.write(outstr)\n if error is str or error is str.split:\n errstr = err.decode(encoding)\n result += errstr\n if error is str.split:\n sys.stderr.write(errstr)\n return result\n else:\n return None\n\n\ndef _streamify_output(arg: OutType, name: str) -> Tuple[Union[int, BinaryIO, None], bool]:\n if isinstance(arg, str):\n return open(arg, \"wb\"), True\n elif arg is str or arg is str.split:\n return subprocess.PIPE, False\n elif callable(arg):\n raise ValueError(f\"`{name}` must be a stream, a filename, or `str`/`str.split`\")\n else:\n return arg, False\n\n\nclass Executable:\n \"\"\"\n Represent an executable file that can be run as a subprocess.\n\n This class provides a simple interface for running executables with custom arguments and\n environment variables. It supports setting default arguments and environment modifications,\n copying instances, and running commands with various options for input/output/error handling.\n\n Example usage:\n\n .. code-block:: python\n\n ls = Executable(\"ls\")\n ls.add_default_arg(\"-l\")\n ls.add_default_env(\"LC_ALL\", \"C\")\n output = ls(\"-a\", output=str) # Run 'ls -l -a' and capture output as string\n \"\"\"\n\n def __init__(self, name: Union[str, Path]) -> None:\n file_path = str(Path(name))\n if sys.platform != \"win32\" and isinstance(name, str) and name.startswith(\".\"):\n # pathlib strips the ./ from relative paths so it must be added back\n file_path = os.path.join(\".\", file_path)\n\n self.exe = [file_path]\n self._default_env: Dict[str, str] = {}\n self._default_envmod = EnvironmentModifications()\n #: Return code of the last executed command.\n self.returncode: int = 1 # 1 until proven successful\n #: Whether to warn users that quotes are not needed, as Spack does not use a shell.\n self.ignore_quotes: bool = False\n\n def add_default_arg(self, *args: str) -> None:\n \"\"\"Add default argument(s) to the command.\"\"\"\n self.exe.extend(args)\n\n def with_default_args(self, *args: str) -> \"Executable\":\n \"\"\"Same as add_default_arg, but returns a copy of the executable.\"\"\"\n new = self.copy()\n new.add_default_arg(*args)\n return new\n\n def copy(self) -> \"Executable\":\n \"\"\"Return a copy of this Executable.\"\"\"\n new = Executable(self.exe[0])\n new.exe[:] = self.exe\n new._default_env.update(self._default_env)\n new._default_envmod.extend(self._default_envmod)\n return new\n\n def add_default_env(self, key: str, value: str) -> None:\n \"\"\"Set an environment variable when the command is run.\n\n Parameters:\n key: The environment variable to set\n value: The value to set it to\n \"\"\"\n self._default_env[key] = value\n\n def add_default_envmod(self, envmod: EnvironmentModifications) -> None:\n \"\"\"Set an :class:`spack.util.environment.EnvironmentModifications` to use when the command\n is run.\"\"\"\n self._default_envmod.extend(envmod)\n\n @property\n def command(self) -> str:\n \"\"\"Returns the entire command-line string\"\"\"\n return \" \".join(self.exe)\n\n @property\n def name(self) -> str:\n \"\"\"Returns the executable name\"\"\"\n return PurePath(self.path).name\n\n @property\n def path(self) -> str:\n \"\"\"Returns the executable path\"\"\"\n return str(PurePath(self.exe[0]))\n\n @overload\n def __call__(\n self,\n *args: str,\n fail_on_error: bool = ...,\n ignore_errors: Union[int, Sequence[int]] = ...,\n ignore_quotes: Optional[bool] = ...,\n timeout: Optional[int] = ...,\n env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ...,\n extra_env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ...,\n input: Optional[BinaryIO] = ...,\n output: Union[Optional[BinaryIO], str] = ...,\n error: Union[Optional[BinaryIO], str] = ...,\n _dump_env: Optional[Dict[str, str]] = ...,\n ) -> None: ...\n\n @overload\n def __call__(\n self,\n *args: str,\n fail_on_error: bool = ...,\n ignore_errors: Union[int, Sequence[int]] = ...,\n ignore_quotes: Optional[bool] = ...,\n timeout: Optional[int] = ...,\n env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ...,\n extra_env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ...,\n input: Optional[BinaryIO] = ...,\n output: Union[Type[str], Callable], # str or str.split\n error: OutType = ...,\n _dump_env: Optional[Dict[str, str]] = ...,\n ) -> str: ...\n\n @overload\n def __call__(\n self,\n *args: str,\n fail_on_error: bool = ...,\n ignore_errors: Union[int, Sequence[int]] = ...,\n ignore_quotes: Optional[bool] = ...,\n timeout: Optional[int] = ...,\n env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ...,\n extra_env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ...,\n input: Optional[BinaryIO] = ...,\n output: OutType = ...,\n error: Union[Type[str], Callable], # str or str.split\n _dump_env: Optional[Dict[str, str]] = ...,\n ) -> str: ...\n\n def __call__(\n self,\n *args: str,\n fail_on_error: bool = True,\n ignore_errors: Union[int, Sequence[int]] = (),\n ignore_quotes: Optional[bool] = None,\n timeout: Optional[int] = None,\n env: Optional[Union[Dict[str, str], EnvironmentModifications]] = None,\n extra_env: Optional[Union[Dict[str, str], EnvironmentModifications]] = None,\n input: Optional[BinaryIO] = None,\n output: OutType = None,\n error: OutType = None,\n _dump_env: Optional[Dict[str, str]] = None,\n ) -> Optional[str]:\n \"\"\"Runs this executable in a subprocess.\n\n Parameters:\n *args: command-line arguments to the executable to run\n fail_on_error: if True, raises an exception if the subprocess returns an error\n The return code is available as :attr:`returncode`\n ignore_errors: a sequence of error codes to ignore. If these codes are returned, this\n process will not raise an exception, even if ``fail_on_error`` is set to ``True``\n ignore_quotes: if False, warn users that quotes are not needed, as Spack does not\n use a shell. If None, use :attr:`ignore_quotes`.\n timeout: the number of seconds to wait before killing the child process\n env: the environment with which to run the executable\n extra_env: extra items to add to the environment (neither requires nor precludes env)\n input: where to read stdin from\n output: where to send stdout\n error: where to send stderr\n _dump_env: dict to be set to the environment actually used (envisaged for\n testing purposes only)\n\n Accepted values for ``input``, ``output``, and ``error``:\n\n * Python streams: open Python file objects or ``os.devnull``\n * :obj:`str`: the Python string **type**. If you set these to :obj:`str`,\n output and error will be written to pipes and returned as a string.\n If both ``output`` and ``error`` are set to :obj:`str`, then one string\n is returned containing output concatenated with error. Not valid\n for ``input``.\n * :obj:`str.split`: the split method of the Python string type.\n Behaves the same as :obj:`str`, except that value is also written to\n ``stdout`` or ``stderr``.\n\n For ``output`` and ``error`` it's also accepted to pass a string with a filename, which\n will be automatically opened for writing.\n\n By default, the subprocess inherits the parent's file descriptors.\n \"\"\"\n # Setup default environment\n current_environment = os.environ.copy() if env is None else {}\n self._default_envmod.apply_modifications(current_environment)\n current_environment.update(self._default_env)\n\n # Apply env argument\n if isinstance(env, EnvironmentModifications):\n env.apply_modifications(current_environment)\n elif env:\n current_environment.update(env)\n\n # Apply extra env\n if isinstance(extra_env, EnvironmentModifications):\n extra_env.apply_modifications(current_environment)\n elif extra_env is not None:\n current_environment.update(extra_env)\n\n if _dump_env is not None:\n _dump_env.clear()\n _dump_env.update(current_environment)\n\n if ignore_quotes is None:\n ignore_quotes = self.ignore_quotes\n\n # If they just want to ignore one error code, make it a tuple.\n if isinstance(ignore_errors, int):\n ignore_errors = (ignore_errors,)\n\n if input is str or input is str.split:\n raise ValueError(\"Cannot use `str` or `str.split` as input stream.\")\n elif isinstance(input, str):\n istream, close_istream = open(input, \"rb\"), True\n else:\n istream, close_istream = input, False\n\n ostream, close_ostream = _streamify_output(output, \"output\")\n estream, close_estream = _streamify_output(error, \"error\")\n\n if not ignore_quotes:\n quoted_args = [arg for arg in args if re.search(r'^\".*\"$|^\\'.*\\'$', arg)]\n if quoted_args:\n tty.warn(\n \"Quotes in command arguments can confuse scripts like configure.\",\n \"The following arguments may cause problems when executed:\",\n str(\"\\n\".join([\" \" + arg for arg in quoted_args])),\n \"Quotes aren't needed because spack doesn't use a shell. \"\n \"Consider removing them.\",\n \"If multiple levels of quotation are required, use `ignore_quotes=True`.\",\n )\n\n cmd = self.exe + list(args)\n\n cmd_line_string = \" \".join(shlex.quote(arg) for arg in cmd)\n tty.debug(cmd_line_string)\n\n result = None\n try:\n proc = subprocess.Popen(\n cmd,\n stdin=istream,\n stderr=estream,\n stdout=ostream,\n env=current_environment,\n close_fds=False,\n )\n except OSError as e:\n message = \"Command: \" + cmd_line_string\n if \" \" in self.exe[0]:\n message += \"\\nDid you mean to add a space to the command?\"\n\n raise ProcessError(f\"{self.exe[0]}: {e.strerror}\", message)\n\n try:\n out, err = proc.communicate(timeout=timeout)\n result = _process_cmd_output(out, err, output, error)\n rc = self.returncode = proc.returncode\n if fail_on_error and rc != 0 and (rc not in ignore_errors):\n long_msg = cmd_line_string\n if result:\n # If the output is not captured in the result, it will have\n # been stored either in the specified files (e.g. if\n # 'output' specifies a file) or written to the parent's\n # stdout/stderr (e.g. if 'output' is not specified)\n long_msg += \"\\n\" + result\n\n raise ProcessError(f\"Command exited with status {proc.returncode}:\", long_msg)\n\n except subprocess.TimeoutExpired as te:\n proc.kill()\n out, err = proc.communicate()\n result = _process_cmd_output(out, err, output, error)\n long_msg = cmd_line_string + f\"\\n{result}\"\n if fail_on_error:\n raise ProcessTimeoutError(\n f\"\\nProcess timed out after {timeout}s. \"\n \"We expected the following command to run quickly but it did not, \"\n f\"please report this as an issue: {long_msg}\",\n long_message=long_msg,\n ) from te\n\n finally:\n # The isinstance checks are only needed for type checking.\n if close_ostream and isinstance(ostream, io.IOBase):\n ostream.close()\n if close_estream and isinstance(estream, io.IOBase):\n estream.close()\n if close_istream and isinstance(istream, io.IOBase):\n istream.close()\n\n return result\n\n def __eq__(self, other):\n return hasattr(other, \"exe\") and self.exe == other.exe\n\n def __hash__(self):\n return hash((type(self),) + tuple(self.exe))\n\n def __repr__(self):\n return f\"\"\n\n def __str__(self):\n return \" \".join(self.exe)\n\n\n@overload\ndef which_string(\n *args: str, path: Optional[Union[List[str], str]] = ..., required: Literal[True]\n) -> str: ...\n\n\n@overload\ndef which_string(\n *args: str, path: Optional[Union[List[str], str]] = ..., required: bool = ...\n) -> Optional[str]: ...\n\n\ndef which_string(\n *args: str, path: Optional[Union[List[str], str]] = None, required: bool = False\n) -> Optional[str]:\n \"\"\"Like :func:`which`, but returns a string instead of an :class:`Executable`.\"\"\"\n if path is None:\n path = os.environ.get(\"PATH\", \"\")\n\n if isinstance(path, list):\n paths = [Path(str(x)) for x in path]\n\n if isinstance(path, str):\n paths = [Path(x) for x in path.split(os.pathsep)]\n\n def get_candidate_items(search_item):\n if sys.platform == \"win32\" and not search_item.suffix:\n return [search_item.parent / (search_item.name + ext) for ext in [\".exe\", \".bat\"]]\n\n return [Path(search_item)]\n\n def add_extra_search_paths(paths):\n with_parents = []\n with_parents.extend(paths)\n if sys.platform == \"win32\":\n for p in paths:\n if p.name == \"bin\":\n with_parents.append(p.parent)\n return with_parents\n\n for search_item in args:\n search_paths = []\n search_paths.extend(paths)\n if search_item.startswith(\".\"):\n # we do this because pathlib will strip any leading ./\n search_paths.insert(0, Path.cwd())\n search_paths = add_extra_search_paths(search_paths)\n\n candidate_items = get_candidate_items(Path(search_item))\n\n for candidate_item in candidate_items:\n for directory in search_paths:\n exe = directory / candidate_item\n try:\n if exe.is_file() and os.access(str(exe), os.X_OK):\n return str(exe)\n except OSError:\n pass\n\n if required:\n raise CommandNotFoundError(f\"spack requires '{args[0]}'. Make sure it is in your path.\")\n\n return None\n\n\n@overload\ndef which(\n *args: str, path: Optional[Union[List[str], str]] = ..., required: Literal[True]\n) -> Executable: ...\n\n\n@overload\ndef which(\n *args: str, path: Optional[Union[List[str], str]] = ..., required: bool = ...\n) -> Optional[Executable]: ...\n\n\ndef which(\n *args: str, path: Optional[Union[List[str], str]] = None, required: bool = False\n) -> Optional[Executable]:\n \"\"\"Finds an executable in the path like command-line which.\n\n If given multiple executables, returns the first one that is found.\n If no executables are found, returns None.\n\n Parameters:\n *args: one or more executables to search for\n path: the path to search. Defaults to ``PATH``\n required: if set to :data:`True`, raise an error if executable not found\n\n Returns:\n The first executable that is found in the path or :data:`None` if not found.\n \"\"\"\n exe = which_string(*args, path=path, required=required)\n return Executable(exe) if exe is not None else None\n\n\nclass ProcessError(spack.error.SpackError):\n \"\"\"Raised when :class:`Executable` exits with an error code.\"\"\"\n\n\nclass ProcessTimeoutError(ProcessError):\n \"\"\"Raised when :class:`Executable` calls with a specified timeout exceed that time.\"\"\"\n\n\nclass CommandNotFoundError(spack.error.SpackError):\n \"\"\"Raised when :func:`which()` can't find a required executable.\"\"\"\n" }, { "path": "lib/spack/spack/util/file_cache.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport hashlib\nimport os\nimport pathlib\nimport shutil\nimport tempfile\nfrom contextlib import contextmanager\nfrom typing import IO, Dict, Iterator, Optional, Tuple, Union\n\nfrom spack.error import SpackError\nfrom spack.llnl.util.filesystem import rename\nfrom spack.util.lock import Lock\n\n\ndef _maybe_open(path: Union[str, pathlib.Path]) -> Optional[IO[str]]:\n try:\n return open(path, \"r\", encoding=\"utf-8\")\n except IsADirectoryError:\n raise CacheError(\"Cache file is not a file: %s\" % path)\n except PermissionError:\n raise CacheError(\"Cannot access cache file: %s\" % path)\n except FileNotFoundError:\n return None\n\n\ndef _open_temp(context_dir: Union[str, pathlib.Path]) -> Tuple[IO[str], str]:\n \"\"\"Open a temporary file in a directory\n\n This implementation minimizes the number of system calls for the case\n the target directory already exists.\n \"\"\"\n try:\n fd, path = tempfile.mkstemp(dir=context_dir)\n except FileNotFoundError:\n os.makedirs(context_dir, exist_ok=True)\n fd, path = tempfile.mkstemp(dir=context_dir)\n\n stream = os.fdopen(fd, \"w\", encoding=\"utf-8\")\n return stream, path\n\n\nclass ReadContextManager:\n def __init__(self, path: Union[str, pathlib.Path]) -> None:\n self.path = path\n\n def __enter__(self) -> Optional[IO[str]]:\n \"\"\"Return a file object for the cache if it exists.\"\"\"\n self.cache_file = _maybe_open(self.path)\n return self.cache_file\n\n def __exit__(self, type, value, traceback):\n if self.cache_file:\n self.cache_file.close()\n\n\nclass WriteContextManager:\n def __init__(self, path: Union[str, pathlib.Path]) -> None:\n self.path = path\n\n def __enter__(self) -> Tuple[Optional[IO[str]], IO[str]]:\n \"\"\"Return (old_file, new_file) file objects, where old_file is optional.\"\"\"\n try:\n self.old_file = _maybe_open(self.path)\n self.new_file, self.tmp_path = _open_temp(os.path.dirname(self.path))\n except PermissionError:\n if self.old_file:\n self.old_file.close()\n raise CacheError(f\"Insufficient permissions to write to file cache at {self.path}\")\n return self.old_file, self.new_file\n\n def __exit__(self, type, value, traceback):\n if self.old_file:\n self.old_file.close()\n self.new_file.close()\n\n if value:\n try:\n os.remove(self.tmp_path)\n except OSError:\n pass\n else:\n rename(self.tmp_path, self.path)\n\n\nclass FileCache:\n \"\"\"This class manages cached data in the filesystem.\n\n - Cache files are fetched and stored by unique keys. Keys can be relative\n paths, so that there can be some hierarchy in the cache.\n\n - The FileCache handles locking cache files for reading and writing, so\n client code need not manage locks for cache entries.\n\n \"\"\"\n\n def __init__(self, root: Union[str, pathlib.Path], timeout=120):\n \"\"\"Create a file cache object.\n\n This will create the cache directory if it does not exist yet.\n\n Args:\n root: specifies the root directory where the cache stores files\n\n timeout: when there is contention among multiple Spack processes\n for cache files, this specifies how long Spack should wait\n before assuming that there is a deadlock.\n \"\"\"\n if isinstance(root, str):\n root = pathlib.Path(root)\n self.root = root\n self.root.mkdir(parents=True, exist_ok=True)\n\n self.lock_path = self.root / \".lock\"\n self._locks: Dict[Union[pathlib.Path, str], Lock] = {}\n self.lock_timeout = timeout\n\n def destroy(self):\n \"\"\"Remove all files under the cache root.\"\"\"\n for f in self.root.iterdir():\n if f.is_dir():\n shutil.rmtree(f, True)\n else:\n f.unlink()\n\n def cache_path(self, key: Union[str, pathlib.Path]):\n \"\"\"Path to the file in the cache for a particular key.\"\"\"\n return self.root / key\n\n def _get_lock_offsets(self, key: str) -> Tuple[int, int]:\n \"\"\"Hash function to determine byte-range offsets for a key. Returns (start, length) for\n the lock.\"\"\"\n hasher = hashlib.sha256(key.encode(\"utf-8\"))\n hash_int = int.from_bytes(hasher.digest()[:8], \"little\")\n start_offset = hash_int % (2**63 - 1)\n return start_offset, 1\n\n def _get_lock(self, key: Union[str, pathlib.Path]):\n \"\"\"Create a lock for a key using byte-range offsets.\"\"\"\n key_str = str(key)\n\n if key_str not in self._locks:\n start, length = self._get_lock_offsets(key_str)\n self._locks[key_str] = Lock(\n str(self.lock_path),\n start=start,\n length=length,\n default_timeout=self.lock_timeout,\n desc=f\"key:{key_str}\",\n )\n return self._locks[key_str]\n\n @contextmanager\n def read_transaction(self, key: Union[str, pathlib.Path]) -> Iterator[Optional[IO[str]]]:\n \"\"\"Get a read transaction on a file cache item.\n\n Returns a context manager that yields an open file object for reading,\n or None if the cache file does not exist. You can use it like this::\n\n with file_cache_object.read_transaction(key) as cache_file:\n if cache_file is not None:\n cache_file.read()\n\n \"\"\"\n lock = self._get_lock(key)\n lock.acquire_read()\n try:\n with ReadContextManager(self.cache_path(key)) as f:\n yield f\n finally:\n lock.release_read()\n\n @contextmanager\n def write_transaction(\n self, key: Union[str, pathlib.Path]\n ) -> Iterator[Tuple[Optional[IO[str]], IO[str]]]:\n \"\"\"Get a write transaction on a file cache item.\n\n Returns a context manager that yields (old_file, new_file) where old_file\n is the existing cache file (or None), and new_file is a writable temporary\n file. Once the context manager exits cleanly, moves the temporary file\n into place atomically.\n\n \"\"\"\n path = self.cache_path(key)\n lock = self._get_lock(key)\n try:\n lock.acquire_write()\n except PermissionError:\n raise CacheError(f\"Insufficient permissions to write to file cache at {path}\")\n try:\n with WriteContextManager(str(path)) as (old, new):\n yield old, new\n finally:\n lock.release_write()\n\n def remove(self, key: Union[str, pathlib.Path]):\n file = self.cache_path(key)\n lock = self._get_lock(key)\n lock.acquire_write()\n try:\n file.unlink()\n except FileNotFoundError:\n pass\n finally:\n lock.release_write()\n\n\nclass CacheError(SpackError):\n pass\n" }, { "path": "lib/spack/spack/util/file_permissions.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport stat as st\n\nimport spack.llnl.util.filesystem as fs\nimport spack.package_prefs as pp\nfrom spack.error import SpackError\n\n\ndef set_permissions_by_spec(path, spec):\n # Get permissions for spec\n if os.path.isdir(path):\n perms = pp.get_package_dir_permissions(spec)\n else:\n perms = pp.get_package_permissions(spec)\n group = pp.get_package_group(spec)\n\n set_permissions(path, perms, group)\n\n\ndef set_permissions(path, perms, group=None):\n # Preserve higher-order bits of file permissions\n perms |= os.stat(path).st_mode & (st.S_ISUID | st.S_ISGID | st.S_ISVTX)\n\n # Do not let users create world/group writable suid binaries\n if perms & st.S_ISUID:\n if perms & st.S_IWOTH:\n raise InvalidPermissionsError(\"Attempting to set suid with world writable\")\n if perms & st.S_IWGRP:\n raise InvalidPermissionsError(\"Attempting to set suid with group writable\")\n # Or world writable sgid binaries\n if perms & st.S_ISGID:\n if perms & st.S_IWOTH:\n raise InvalidPermissionsError(\"Attempting to set sgid with world writable\")\n\n fs.chmod_x(path, perms)\n\n if group:\n fs.chgrp(path, group, follow_symlinks=False)\n\n\nclass InvalidPermissionsError(SpackError):\n \"\"\"Error class for invalid permission setters\"\"\"\n" }, { "path": "lib/spack/spack/util/filesystem.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"\nUtilities for interacting with files,\nlike those in spack.llnl.util.filesystem, but which require logic from spack.util\n\"\"\"\n\nimport glob\nimport os\n\nfrom spack.llnl.util.filesystem import edit_in_place_through_temporary_file\nfrom spack.util.executable import Executable\n\n\ndef fix_darwin_install_name(path: str) -> None:\n \"\"\"Fix install name of dynamic libraries on Darwin to have full path.\n\n There are two parts of this task:\n\n 1. Use ``install_name -id ...`` to change install name of a single lib\n 2. Use ``install_name -change ...`` to change the cross linking between libs.\n The function assumes that all libraries are in one folder and currently won't follow\n subfolders.\n\n Parameters:\n path: directory in which ``.dylib`` files are located\n \"\"\"\n libs = glob.glob(os.path.join(path, \"*.dylib\"))\n install_name_tool = Executable(\"install_name_tool\")\n otool = Executable(\"otool\")\n for lib in libs:\n args = [\"-id\", lib]\n long_deps = otool(\"-L\", lib, output=str).split(\"\\n\")\n deps = [dep.partition(\" \")[0][1::] for dep in long_deps[2:-1]]\n # fix all dependencies:\n for dep in deps:\n for loc in libs:\n # We really want to check for either\n # dep == os.path.basename(loc) or\n # dep == join_path(builddir, os.path.basename(loc)),\n # but we don't know builddir (nor how symbolic links look\n # in builddir). We thus only compare the basenames.\n if os.path.basename(dep) == os.path.basename(loc):\n args.extend((\"-change\", dep, loc))\n break\n\n with edit_in_place_through_temporary_file(lib) as tmp:\n install_name_tool(*args, tmp)\n" }, { "path": "lib/spack/spack/util/format.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\ndef get_version_lines(version_hashes_dict: dict) -> str:\n \"\"\"\n Renders out a set of versions like those found in a package's\n package.py file for a given set of versions and hashes.\n\n Args:\n version_hashes_dict: A dictionary of the form: version -> checksum.\n\n Returns: Rendered version lines.\n \"\"\"\n version_lines = []\n\n for v, h in version_hashes_dict.items():\n version_lines.append(f' version(\"{v}\", sha256=\"{h}\")')\n\n return \"\\n\".join(version_lines)\n" }, { "path": "lib/spack/spack/util/gcs.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"\nThis file contains the definition of the GCS Blob storage Class used to\nintegrate GCS Blob storage with spack buildcache.\n\"\"\"\n\nimport os\nimport sys\nimport urllib.parse\nimport urllib.response\nfrom typing import List\nfrom urllib.error import URLError\nfrom urllib.request import BaseHandler\n\nimport spack.llnl.util.tty as tty\n\n\ndef gcs_client():\n \"\"\"Create a GCS client\n Creates an authenticated GCS client to access GCS buckets and blobs\n \"\"\"\n\n try:\n import google.auth\n from google.cloud import storage\n except ImportError as ex:\n tty.error(\n \"{0}, google-cloud-storage python module is missing.\".format(ex)\n + \" Please install to use the gs:// backend.\"\n )\n sys.exit(1)\n\n storage_credentials, storage_project = google.auth.default()\n storage_client = storage.Client(storage_project, storage_credentials)\n return storage_client\n\n\nclass GCSBucket:\n \"\"\"GCS Bucket Object\n Create a wrapper object for a GCS Bucket. Provides methods to wrap spack\n related tasks, such as destroy.\n \"\"\"\n\n def __init__(self, url, client=None):\n \"\"\"Constructor for GCSBucket objects\n\n Args:\n url (str): The url pointing to the GCS bucket to build an object out of\n client (google.cloud.storage.client.Client): A pre-defined storage\n client that will be used to access the GCS bucket.\n \"\"\"\n if url.scheme != \"gs\":\n raise ValueError(\n \"Can not create GCS bucket connection with scheme {SCHEME}\".format(\n SCHEME=url.scheme\n )\n )\n self.url = url\n self.name = self.url.netloc\n if self.url.path[0] == \"/\":\n self.prefix = self.url.path[1:]\n else:\n self.prefix = self.url.path\n\n self.client = client or gcs_client()\n\n self.bucket = None\n tty.debug(\"New GCS bucket:\")\n tty.debug(\" name: {0}\".format(self.name))\n tty.debug(\" prefix: {0}\".format(self.prefix))\n\n def exists(self):\n from google.cloud.exceptions import NotFound\n\n if not self.bucket:\n try:\n self.bucket = self.client.bucket(self.name)\n except NotFound as ex:\n tty.error(\"{0}, Failed check for bucket existence\".format(ex))\n sys.exit(1)\n return self.bucket is not None\n\n def create(self):\n if not self.bucket:\n self.bucket = self.client.create_bucket(self.name)\n\n def get_blob(self, blob_path):\n if self.exists():\n return self.bucket.get_blob(blob_path)\n return None\n\n def blob(self, blob_path):\n if self.exists():\n return self.bucket.blob(blob_path)\n return None\n\n def get_all_blobs(self, recursive: bool = True, relative: bool = True) -> List[str]:\n \"\"\"Get a list of all blobs\n\n Returns: a list of all blobs within this bucket.\n\n Args:\n relative: If true (default), print blob paths relative to 'build_cache' directory.\n If false, print absolute blob paths (useful for destruction of bucket)\n \"\"\"\n tty.debug(\"Getting GCS blobs... Recurse {0} -- Rel: {1}\".format(recursive, relative))\n\n converter = self._relative_blob_name if relative else str\n\n blob_list: List[str] = []\n\n if self.exists():\n all_blobs = self.bucket.list_blobs(prefix=self.prefix)\n\n base_dirs = len(self.prefix.split(\"/\")) + 1\n\n for blob in all_blobs:\n if not recursive:\n num_dirs = len(blob.name.split(\"/\"))\n if num_dirs <= base_dirs:\n blob_list.append(converter(blob.name))\n else:\n blob_list.append(converter(blob.name))\n\n return blob_list\n\n def _relative_blob_name(self, blob_name):\n return os.path.relpath(blob_name, self.prefix)\n\n def destroy(self, recursive=False, **kwargs):\n \"\"\"Bucket destruction method\n\n Deletes all blobs within the bucket, and then deletes the bucket itself.\n\n Uses GCS Batch operations to bundle several delete operations together.\n \"\"\"\n from google.cloud.exceptions import NotFound\n\n tty.debug(\"Bucket.destroy(recursive={0})\".format(recursive))\n try:\n bucket_blobs = self.get_all_blobs(recursive=recursive, relative=False)\n batch_size = 1000\n\n num_blobs = len(bucket_blobs)\n for i in range(0, num_blobs, batch_size):\n with self.client.batch():\n for j in range(i, min(i + batch_size, num_blobs)):\n blob = self.blob(bucket_blobs[j])\n blob.delete()\n except NotFound as ex:\n tty.error(\"{0}, Could not delete a blob in bucket {1}.\".format(ex, self.name))\n sys.exit(1)\n\n\nclass GCSBlob:\n \"\"\"GCS Blob object\n\n Wraps some blob methods for spack functionality\n \"\"\"\n\n def __init__(self, url, client=None):\n self.url = url\n if url.scheme != \"gs\":\n raise ValueError(\n \"Can not create GCS blob connection with scheme: {SCHEME}\".format(\n SCHEME=url.scheme\n )\n )\n\n self.client = client or gcs_client()\n\n self.bucket = GCSBucket(url)\n\n self.blob_path = self.url.path.lstrip(\"/\")\n\n tty.debug(\"New GCSBlob\")\n tty.debug(\" blob_path = {0}\".format(self.blob_path))\n\n if not self.bucket.exists():\n tty.warn(\"The bucket {0} does not exist, it will be created\".format(self.bucket.name))\n self.bucket.create()\n\n def get(self):\n return self.bucket.get_blob(self.blob_path)\n\n def exists(self):\n from google.cloud.exceptions import NotFound\n\n try:\n blob = self.bucket.blob(self.blob_path)\n exists = blob.exists()\n except NotFound:\n return False\n\n return exists\n\n def delete_blob(self):\n from google.cloud.exceptions import NotFound\n\n try:\n blob = self.bucket.blob(self.blob_path)\n blob.delete()\n except NotFound as ex:\n tty.error(\"{0}, Could not delete gcs blob {1}\".format(ex, self.blob_path))\n\n def upload_to_blob(self, local_file_path):\n blob = self.bucket.blob(self.blob_path)\n blob.upload_from_filename(local_file_path)\n\n def get_blob_byte_stream(self):\n return self.bucket.get_blob(self.blob_path).open(mode=\"rb\")\n\n def get_blob_headers(self):\n blob = self.bucket.get_blob(self.blob_path)\n\n headers = {\n \"Content-type\": blob.content_type,\n \"Content-encoding\": blob.content_encoding,\n \"Content-language\": blob.content_language,\n \"MD5Hash\": blob.md5_hash,\n }\n\n return headers\n\n\ndef gcs_open(req, *args, **kwargs):\n \"\"\"Open a reader stream to a blob object on GCS\"\"\"\n url = urllib.parse.urlparse(req.get_full_url())\n gcsblob = GCSBlob(url)\n\n if not gcsblob.exists():\n raise URLError(\"GCS blob {0} does not exist\".format(gcsblob.blob_path))\n stream = gcsblob.get_blob_byte_stream()\n headers = gcsblob.get_blob_headers()\n\n return urllib.response.addinfourl(stream, headers, url)\n\n\nclass GCSHandler(BaseHandler):\n def gs_open(self, req):\n return gcs_open(req)\n" }, { "path": "lib/spack/spack/util/git.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Single util module where Spack should get a git executable.\"\"\"\n\nimport os\nimport re\nimport shutil\nimport sys\nfrom typing import List, Optional, overload\n\nfrom spack.vendor.typing_extensions import Literal\n\nimport spack.llnl.util.filesystem as fs\nimport spack.llnl.util.lang\nimport spack.util.executable as exe\nfrom spack.util.environment import EnvironmentModifications\n\n# regex for a commit version\nCOMMIT_VERSION = re.compile(r\"^[a-f0-9]{40}$\")\n\n# regex for a git version to extract only the numeric parts\nGIT_VERSION = re.compile(r\"(\\d+(?:\\.\\d+)*)\")\n\n\ndef is_git_commit_sha(string: str) -> bool:\n return len(string) == 40 and bool(COMMIT_VERSION.match(string))\n\n\n@spack.llnl.util.lang.memoized\ndef _find_git() -> Optional[str]:\n \"\"\"Find the git executable in the system path.\"\"\"\n return exe.which_string(\"git\", required=False)\n\n\ndef extract_git_version_str(git_exe: exe.Executable) -> str:\n match = re.search(GIT_VERSION, git_exe(\"--version\", output=str))\n return match.group(1) if match else \"\"\n\n\nclass GitExecutable(exe.Executable):\n \"\"\"Specialized executable that encodes the git version for optimized option selection\"\"\"\n\n def __init__(self, name=None):\n if not name:\n name = _find_git()\n super().__init__(name)\n self._version = None\n\n @property\n def version(self):\n # lazy init git version\n if not self._version:\n v_string = extract_git_version_str(self)\n self._version = tuple(int(i) for i in v_string.split(\".\"))\n return self._version\n\n\nclass VersionConditionalOption:\n def __init__(self, key, value=None, min_version=(0, 0, 0), max_version=(99, 99, 99)):\n self.key = key\n self.value = value\n self.min_version = min_version\n self.max_version = max_version\n\n def __call__(self, exe_version, value=None) -> List:\n if (self.min_version <= exe_version) and (self.max_version >= exe_version):\n option = [self.key]\n if value:\n option.append(value)\n elif self.value:\n option.append(self.value)\n return option\n else:\n return []\n\n\n# The earliest git version where we start trying to optimize clones\n# git@1.8.5 is when branch could also accept tag so we don't have to track ref types as closely\n# This also corresponds to system git on RHEL7\nMIN_OPT_VERSION = (1, 8, 5, 2)\nMIN_DIRECT_COMMIT_FETCH = (2, 5, 0)\n\n# Technically the flags existed earlier but we are pruning our logic to 1.8.5 or greater\nBRANCH = VersionConditionalOption(\"--branch\", min_version=MIN_OPT_VERSION)\nSINGLE_BRANCH = VersionConditionalOption(\"--single-branch\", min_version=MIN_OPT_VERSION)\nNO_SINGLE_BRANCH = VersionConditionalOption(\"--no-single-branch\", min_version=MIN_OPT_VERSION)\n# Depth was introduced in 1.7.11 but isn't worth much without the --branch options\nDEPTH = VersionConditionalOption(\"--depth\", 1, min_version=MIN_OPT_VERSION)\n\nFILTER_BLOB_NONE = VersionConditionalOption(\"--filter=blob:none\", min_version=(2, 19, 0))\nNO_CHECKOUT = VersionConditionalOption(\"--no-checkout\", min_version=(2, 34, 0))\n# technically sparse-checkout was added in 2.25, but we go forward since the model we use only\n# works with the `--cone` option\nSPARSE_CHECKOUT = VersionConditionalOption(\"sparse-checkout\", \"set\", min_version=(2, 34, 0))\n\n\n@overload\ndef git(required: Literal[True]) -> GitExecutable: ...\n\n\n@overload\ndef git(required: bool = ...) -> Optional[GitExecutable]: ...\n\n\ndef git(required: bool = False) -> Optional[GitExecutable]:\n \"\"\"Get a git executable.\n\n The returned executable automatically unsets ``GIT_EXTERNAL_DIFF`` and ``GIT_DIFF_OPTS``\n environment variables that can interfere with spack git diff operations.\n\n Args:\n required (bool): if True, raises CommandNotFoundError when git is not found\n\n Returns: GitExecutable, or None if git is not found and required is False\n \"\"\"\n git_path = _find_git()\n\n if not git_path:\n if required:\n raise exe.CommandNotFoundError(\"spack requires 'git'. Make sure it is in your path.\")\n return None\n\n git = GitExecutable(git_path)\n\n # If we're running under pytest, add this to ignore the fix for CVE-2022-39253 in\n # git 2.38.1+. Do this in one place; we need git to do this in all parts of Spack.\n if \"pytest\" in sys.modules:\n git.add_default_arg(\"-c\", \"protocol.file.allow=always\")\n\n # Block environment variables that can interfere with git diff operations\n # this can cause problems for spack ci verify-versions and spack repo show-version-updates\n env_blocklist = EnvironmentModifications()\n env_blocklist.unset(\"GIT_EXTERNAL_DIFF\")\n env_blocklist.unset(\"GIT_DIFF_OPTS\")\n git.add_default_envmod(env_blocklist)\n\n return git\n\n\ndef init_git_repo(\n repository: str, remote: str = \"origin\", git_exe: Optional[exe.Executable] = None\n):\n \"\"\"Initialize a new Git repository and configure it with a remote.\"\"\"\n git_exe = git_exe or git(required=True)\n\n git_exe(\"init\", \"--quiet\", output=str)\n git_exe(\"remote\", \"add\", remote, repository)\n # versions of git prior to v2.24 may not have the manyFiles feature\n # so we should ignore errors here on older versions of git\n git_exe(\"config\", \"feature.manyFiles\", \"true\", ignore_errors=True)\n\n\ndef pull_checkout_commit(\n commit: str,\n remote: Optional[str] = None,\n depth: Optional[int] = None,\n git_exe: Optional[exe.Executable] = None,\n):\n \"\"\"Checkout the specified commit (fetched if necessary).\"\"\"\n git_exe = git_exe or git(required=True)\n\n # Do not do any fetching if the commit is already present.\n try:\n git_exe(\"checkout\", \"--quiet\", commit, error=os.devnull)\n return\n except exe.ProcessError:\n pass\n\n # First try to fetch the specific commit from a specific remote. This allows fixed depth, but\n # the server needs to support it.\n if remote is not None:\n try:\n flags = [] if depth is None else [f\"--depth={depth}\"]\n git_exe(\"fetch\", \"--quiet\", \"--progress\", *flags, remote, commit, error=os.devnull)\n git_exe(\"checkout\", \"--quiet\", commit)\n return\n except exe.ProcessError:\n pass\n\n # Fall back to fetching all while unshallowing, to guarantee we get the commit. The depth flag\n # is equivalent to --unshallow, and needed cause git can pedantically error with\n # \"--unshallow on a complete repository does not make sense\".\n remote_flag = \"--all\" if remote is None else remote\n git_exe(\"fetch\", \"--quiet\", \"--progress\", \"--depth=2147483647\", remote_flag)\n git_exe(\"checkout\", \"--quiet\", commit)\n\n\ndef pull_checkout_tag(\n tag: str,\n remote: str = \"origin\",\n depth: Optional[int] = None,\n git_exe: Optional[exe.Executable] = None,\n):\n \"\"\"Fetch tags with specified depth and checkout the given tag.\"\"\"\n git_exe = git_exe or git(required=True)\n\n fetch_args = [\"--quiet\", \"--progress\", \"--tags\"]\n if depth is not None:\n if depth <= 0:\n raise ValueError(\"depth must be a positive integer\")\n fetch_args.append(f\"--depth={depth}\")\n\n git_exe(\"fetch\", *fetch_args, remote)\n git_exe(\"checkout\", tag)\n\n\ndef pull_checkout_branch(\n branch: str,\n remote: str = \"origin\",\n depth: Optional[int] = None,\n git_exe: Optional[exe.Executable] = None,\n):\n \"\"\"Fetch and checkout branch, then rebase with remote tracking branch.\"\"\"\n git_exe = git_exe or git(required=True)\n\n fetch_args = [\"--quiet\", \"--progress\"]\n if depth:\n if depth <= 0:\n raise ValueError(\"depth must be a positive integer\")\n fetch_args.append(f\"--depth={depth}\")\n\n git_exe(\"fetch\", *fetch_args, remote, f\"refs/heads/{branch}:refs/remotes/{remote}/{branch}\")\n git_exe(\"checkout\", \"--quiet\", branch)\n\n try:\n git_exe(\"rebase\", \"--quiet\", f\"{remote}/{branch}\")\n except exe.ProcessError:\n git_exe(\"rebase\", \"--abort\", fail_on_error=False, error=str, output=str)\n raise\n\n\ndef get_modified_files(\n from_ref: str = \"HEAD~1\", to_ref: str = \"HEAD\", git_exe: Optional[exe.Executable] = None\n) -> List[str]:\n \"\"\"Get a list of files modified between ``from_ref`` and ``to_ref``\n Args:\n from_ref (str): oldest git ref, defaults to ``HEAD~1``\n to_ref (str): newer git ref, defaults to ``HEAD``\n Returns: list of file paths\n \"\"\"\n git_exe = git_exe or git(required=True)\n\n stdout = git_exe(\"diff\", \"--name-only\", from_ref, to_ref, output=str)\n\n return stdout.split()\n\n\ndef get_commit_sha(path: str, ref: str) -> Optional[str]:\n \"\"\"Get a commit sha for an arbitrary ref using ls-remote\"\"\"\n\n # search for matching branch, annotated tag's commit, then lightweight tag\n ref_list = [f\"refs/heads/{ref}\", f\"refs/tags/{ref}^{{}}\", f\"refs/tags/{ref}\"]\n\n if os.path.isdir(path):\n # for the filesystem an unpacked mirror could be in a detached state from a depth 1 clone\n # only reference there will be HEAD\n ref_list.append(\"HEAD\")\n\n for try_ref in ref_list:\n # this command enabled in git@1.7 so no version checking supplied (1.7 released in 2009)\n try:\n query = git(required=True)(\n \"ls-remote\",\n path,\n try_ref,\n output=str,\n error=os.devnull,\n extra_env={\"GIT_TERMINAL_PROMPT\": \"0\"},\n )\n\n if query:\n return query.strip().split()[0]\n except exe.ProcessError:\n continue\n\n return None\n\n\ndef _exec_git_commands(git_exe, cmds, debug, dest=None):\n dest_args = [\"-C\", dest] if dest else []\n error_stream = sys.stdout if debug else os.devnull # swallow extra output for non-debug\n for cmd in cmds:\n git_exe(*dest_args, *cmd, error=error_stream)\n\n\ndef _exec_git_commands_unique_dir(git_exe, cmds, debug, dest=None):\n if dest:\n # mimic creating a dir and clean up if there is a failure like git clone\n assert not os.path.isdir(dest)\n os.mkdir(dest)\n try:\n _exec_git_commands(git_exe, cmds, debug, dest)\n except exe.ProcessError:\n shutil.rmtree(\n dest, ignore_errors=False, onerror=fs.readonly_file_handler(ignore_errors=True)\n )\n raise\n else:\n _exec_git_commands(git_exe, cmds, debug, dest)\n\n\ndef protocol_supports_shallow_clone(url):\n \"\"\"Shallow clone operations (``--depth #``) are not supported by the basic\n HTTP protocol or by no-protocol file specifications.\n Use (e.g.) ``https://`` or ``file://`` instead.\"\"\"\n return not (url.startswith(\"http://\") or url.startswith(\"/\"))\n\n\ndef git_init_fetch(url, ref, depth=None, debug=False, dest=None, git_exe=None):\n \"\"\"Utilize ``git init`` and then ``git fetch`` for a minimal clone of a single git ref\n This method runs git init, repo add, fetch to get a minimal set of source data.\n Profiling has shown this method can be 10-20% less storage than purely using sparse-checkout,\n and is even smaller than git clone --depth 1. This makes it the preferred method for single\n commit checkouts and source mirror population.\n\n There is a trade off since less git data means less flexibility with additional git operations.\n Technically adding the remote is not necessary, but we do it since there are test cases where\n we may want to fetch additional data.\n\n Checkout is explicitly deferred to a second method so we can intercept and add sparse-checkout\n options uniformly whether we use `git clone` or `init fetch`\n \"\"\"\n git_exe = git_exe or git(required=True)\n version = git_exe.version\n # minimum criteria for fetching a single commit, but also requires server to be configured\n # fall-back to a process error so an old git version or a fetch failure from an nonsupporting\n # server can be caught the same way.\n if ref and is_git_commit_sha(ref) and version < MIN_DIRECT_COMMIT_FETCH:\n raise exe.ProcessError(\"Git older than 2.5 detected, can't fetch commit directly\")\n init = [\"init\"]\n remote = [\"remote\", \"add\", \"origin\", url]\n config = [\"config\", \"remote.origin.fetch\", \"+refs/heads/*:origin/refs/*\"]\n fetch = [\"fetch\"]\n\n if not debug:\n fetch.append(\"--quiet\")\n if depth and protocol_supports_shallow_clone(url):\n fetch.extend(DEPTH(version, str(depth)))\n\n filter_args = FILTER_BLOB_NONE(version)\n if filter_args:\n fetch.extend(filter_args)\n fetch.extend([url, ref])\n\n partial_clone = [\"config\", \"extensions.partialClone\", \"true\"] if filter_args else None\n if partial_clone is not None:\n cmds = [init, partial_clone, remote, config, fetch]\n else:\n cmds = [init, remote, config, fetch]\n _exec_git_commands_unique_dir(git_exe, cmds, debug, dest)\n\n\ndef git_checkout(\n ref: Optional[str] = None,\n sparse_paths: List[str] = [],\n debug: bool = False,\n dest: Optional[str] = None,\n git_exe: Optional[GitExecutable] = None,\n):\n \"\"\"A generic method for running ``git checkout`` that integrates sparse-checkout\n Several methods in this module explicitly delay checkout so sparse-checkout can be called.\n It is intended to be used with ``git clone --no-checkout`` or ``git init && git fetch``.\n There is minimal impact to performance since the initial clone operation filters blobs and\n has to download a minimal subset of git data.\n \"\"\"\n git_exe = git_exe or git(required=True)\n checkout = [\"checkout\"]\n sparse_checkout = SPARSE_CHECKOUT(git_exe.version)\n\n if not debug:\n checkout.append(\"--quiet\")\n if ref:\n checkout.append(ref)\n\n cmds = []\n if sparse_paths and sparse_checkout:\n sparse_checkout.extend([*sparse_paths, \"--cone\"])\n cmds.append(sparse_checkout)\n\n cmds.append(checkout)\n _exec_git_commands(git_exe, cmds, debug, dest)\n\n\ndef git_clone(\n url: str,\n ref: Optional[str] = None,\n full_repo: bool = False,\n depth: Optional[int] = None,\n debug: bool = False,\n dest: Optional[str] = None,\n git_exe: Optional[GitExecutable] = None,\n):\n \"\"\"A git clone that prefers deferring expensive blob fetching for modern git installations\n This is our fallback method for capturing more git data than the ``init && fetch`` model.\n It is still optimized to capture a minimal set of ``./.git`` data and expects to be paired with\n a call to ``git checkout`` to fully download the source code.\n \"\"\"\n git_exe = git_exe or git(required=True)\n version = git_exe.version\n clone = [\"clone\"]\n # only need fetch if it's a really old git so we don't fail a checkout\n old = version < MIN_OPT_VERSION\n fetch = [\"fetch\"]\n\n if not debug:\n clone.append(\"--quiet\")\n fetch.append(\"--quiet\")\n\n if not old and depth and not full_repo and protocol_supports_shallow_clone(url):\n clone.extend(DEPTH(version, str(depth)))\n\n if full_repo:\n if old:\n fetch.extend([\"--all\"])\n else:\n clone.extend(NO_SINGLE_BRANCH(version))\n elif ref and not is_git_commit_sha(ref):\n if old:\n fetch.extend([\"origin\", ref])\n else:\n clone.extend([*SINGLE_BRANCH(version), *BRANCH(version, ref)])\n\n clone.extend([*FILTER_BLOB_NONE(version), *NO_CHECKOUT(version), url])\n\n if dest:\n clone.append(dest)\n _exec_git_commands(git_exe, [clone], debug)\n if old:\n _exec_git_commands(git_exe, [fetch], debug, dest)\n" }, { "path": "lib/spack/spack/util/gpg.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport contextlib\nimport errno\nimport functools\nimport os\nimport re\nfrom typing import List\n\nimport spack.error\nimport spack.llnl.util.filesystem\nimport spack.paths\nimport spack.util.executable\nimport spack.version\n\n#: Executable instance for \"gpg\", initialized lazily\nGPG = None\n#: Executable instance for \"gpgconf\", initialized lazily\nGPGCONF = None\n#: Socket directory required if a non default home directory is used\nSOCKET_DIR = None\n#: GNUPGHOME environment variable in the context of this Python module\nGNUPGHOME = None\n\n\ndef clear():\n \"\"\"Reset the global state to uninitialized.\"\"\"\n global GPG, GPGCONF, SOCKET_DIR, GNUPGHOME\n GPG, GPGCONF, SOCKET_DIR, GNUPGHOME = None, None, None, None\n\n\ndef init(gnupghome=None, force=False):\n \"\"\"Initialize the global objects in the module, if not set.\n\n When calling any gpg executable, the GNUPGHOME environment\n variable is set to:\n\n 1. The value of the ``gnupghome`` argument, if not None\n 2. The value of the \"SPACK_GNUPGHOME\" environment variable, if set\n 3. The default gpg path for Spack otherwise\n\n Args:\n gnupghome (str): value to be used for GNUPGHOME when calling\n GnuPG executables\n force (bool): if True forces the re-initialization even if the\n global objects are set already\n \"\"\"\n global GPG, GPGCONF, SOCKET_DIR, GNUPGHOME\n import spack.bootstrap\n\n if force:\n clear()\n\n # If the executables are already set, there's nothing to do\n if GPG and GNUPGHOME:\n return\n\n # Set the value of GNUPGHOME to be used in this module\n GNUPGHOME = gnupghome or os.getenv(\"SPACK_GNUPGHOME\") or spack.paths.gpg_path\n\n # Set the executable objects for \"gpg\" and \"gpgconf\"\n with spack.bootstrap.ensure_bootstrap_configuration():\n spack.bootstrap.ensure_gpg_in_path_or_raise()\n GPG, GPGCONF = _gpg(), _gpgconf()\n\n GPG.add_default_env(\"GNUPGHOME\", GNUPGHOME)\n if GPGCONF:\n GPGCONF.add_default_env(\"GNUPGHOME\", GNUPGHOME)\n # Set the socket dir if not using GnuPG defaults\n SOCKET_DIR = _socket_dir(GPGCONF)\n\n # Make sure that the GNUPGHOME exists\n if not os.path.exists(GNUPGHOME):\n os.makedirs(GNUPGHOME)\n os.chmod(GNUPGHOME, 0o700)\n\n if not os.path.isdir(GNUPGHOME):\n msg = 'GNUPGHOME \"{0}\" exists and is not a directory'.format(GNUPGHOME)\n raise SpackGPGError(msg)\n\n if SOCKET_DIR is not None:\n GPGCONF(\"--create-socketdir\")\n\n\ndef _autoinit(func):\n \"\"\"Decorator to ensure that global variables have been initialized before\n running the decorated function.\n\n Args:\n func (callable): decorated function\n \"\"\"\n\n @functools.wraps(func)\n def _wrapped(*args, **kwargs):\n init()\n return func(*args, **kwargs)\n\n return _wrapped\n\n\n@contextlib.contextmanager\ndef gnupghome_override(dir):\n \"\"\"Set the GNUPGHOME to a new location for this context.\n\n Args:\n dir (str): new value for GNUPGHOME\n \"\"\"\n global GPG, GPGCONF, SOCKET_DIR, GNUPGHOME\n\n # Store backup values\n _GPG, _GPGCONF = GPG, GPGCONF\n _SOCKET_DIR, _GNUPGHOME = SOCKET_DIR, GNUPGHOME\n clear()\n\n # Clear global state\n init(gnupghome=dir, force=True)\n\n yield\n\n clear()\n GPG, GPGCONF = _GPG, _GPGCONF\n SOCKET_DIR, GNUPGHOME = _SOCKET_DIR, _GNUPGHOME\n\n\ndef _parse_secret_keys_output(output: str) -> List[str]:\n keys: List[str] = []\n found_sec = False\n for line in output.split(\"\\n\"):\n if found_sec:\n if line.startswith(\"fpr\"):\n keys.append(line.split(\":\")[9])\n found_sec = False\n elif line.startswith(\"ssb\"):\n found_sec = False\n elif line.startswith(\"sec\"):\n found_sec = True\n return keys\n\n\ndef _parse_public_keys_output(output):\n \"\"\"\n Returns a list of public keys with their fingerprints\n \"\"\"\n keys = []\n found_pub = False\n current_pub_key = \"\"\n for line in output.split(\"\\n\"):\n if found_pub:\n if line.startswith(\"fpr\"):\n keys.append((current_pub_key, line.split(\":\")[9]))\n found_pub = False\n elif line.startswith(\"ssb\"):\n found_pub = False\n elif line.startswith(\"pub\"):\n current_pub_key = line.split(\":\")[4]\n found_pub = True\n return keys\n\n\ndef _get_unimported_public_keys(output):\n keys = []\n for line in output.split(\"\\n\"):\n if line.startswith(\"pub\"):\n keys.append(line.split(\":\")[4])\n return keys\n\n\nclass SpackGPGError(spack.error.SpackError):\n \"\"\"Class raised when GPG errors are detected.\"\"\"\n\n\n@_autoinit\ndef create(**kwargs):\n \"\"\"Create a new key pair.\"\"\"\n r, w = os.pipe()\n with contextlib.closing(os.fdopen(r, \"r\")) as r:\n with contextlib.closing(os.fdopen(w, \"w\")) as w:\n w.write(\n \"\"\"\nKey-Type: rsa\nKey-Length: 4096\nKey-Usage: sign\nName-Real: %(name)s\nName-Email: %(email)s\nName-Comment: %(comment)s\nExpire-Date: %(expires)s\n%%no-protection\n%%commit\n\"\"\"\n % kwargs\n )\n GPG(\"--gen-key\", \"--batch\", input=r)\n\n\n@_autoinit\ndef signing_keys(*args) -> List[str]:\n \"\"\"Return the keys that can be used to sign binaries.\"\"\"\n assert GPG\n output: str = GPG(\"--list-secret-keys\", \"--with-colons\", \"--fingerprint\", *args, output=str)\n return _parse_secret_keys_output(output)\n\n\n@_autoinit\ndef public_keys_to_fingerprint(*args):\n \"\"\"Return the keys that can be used to verify binaries.\"\"\"\n output = GPG(\"--list-public-keys\", \"--with-colons\", \"--fingerprint\", *args, output=str)\n return _parse_public_keys_output(output)\n\n\n@_autoinit\ndef public_keys(*args):\n \"\"\"Return a list of fingerprints\"\"\"\n keys_and_fpr = public_keys_to_fingerprint(*args)\n return [key_and_fpr[1] for key_and_fpr in keys_and_fpr]\n\n\n@_autoinit\ndef export_keys(location, keys, secret=False):\n \"\"\"Export public keys to a location passed as argument.\n\n Args:\n location (str): where to export the keys\n keys (list): keys to be exported\n secret (bool): whether to export secret keys or not\n \"\"\"\n if secret:\n GPG(\"--export-secret-keys\", \"--armor\", \"--output\", location, *keys)\n else:\n GPG(\"--batch\", \"--yes\", \"--armor\", \"--export\", \"--output\", location, *keys)\n\n\n@_autoinit\ndef trust(keyfile):\n \"\"\"Import a public key from a file and trust it.\n\n Args:\n keyfile (str): file with the public key\n \"\"\"\n # Get the public keys we are about to import\n output = GPG(\"--with-colons\", keyfile, output=str, error=str)\n keys = _get_unimported_public_keys(output)\n\n # Import them\n GPG(\"--batch\", \"--import\", keyfile)\n\n # Set trust to ultimate\n key_to_fpr = dict(public_keys_to_fingerprint())\n for key in keys:\n # Skip over keys we cannot find a fingerprint for.\n if key not in key_to_fpr:\n continue\n\n fpr = key_to_fpr[key]\n r, w = os.pipe()\n with contextlib.closing(os.fdopen(r, \"r\")) as r:\n with contextlib.closing(os.fdopen(w, \"w\")) as w:\n w.write(\"{0}:6:\\n\".format(fpr))\n GPG(\"--import-ownertrust\", input=r)\n\n\n@_autoinit\ndef untrust(signing, *keys):\n \"\"\"Delete known keys.\n\n Args:\n signing (bool): if True deletes the secret keys\n *keys: keys to be deleted\n \"\"\"\n if signing:\n skeys = signing_keys(*keys)\n GPG(\"--batch\", \"--yes\", \"--delete-secret-keys\", *skeys)\n\n pkeys = public_keys(*keys)\n GPG(\"--batch\", \"--yes\", \"--delete-keys\", *pkeys)\n\n\n@_autoinit\ndef sign(key, file, output, clearsign=False):\n \"\"\"Sign a file with a key.\n\n Args:\n key: key to be used to sign\n file (str): file to be signed\n output (str): output file (either the clearsigned file or\n the detached signature)\n clearsign (bool): if True wraps the document in an ASCII-armored\n signature, if False creates a detached signature\n \"\"\"\n signopt = \"--clearsign\" if clearsign else \"--detach-sign\"\n GPG(signopt, \"--armor\", \"--local-user\", key, \"--output\", output, file)\n\n\n@_autoinit\ndef verify(signature, file=None, suppress_warnings=False):\n \"\"\"Verify the signature on a file.\n\n Args:\n signature (str): signature of the file (or clearsigned file)\n file (str): file to be verified. If None, then signature is\n assumed to be a clearsigned file.\n suppress_warnings (bool): whether or not to suppress warnings\n from GnuPG\n \"\"\"\n args = [signature]\n if file:\n args.append(file)\n kwargs = {\"error\": str} if suppress_warnings else {}\n GPG(\"--verify\", *args, **kwargs)\n\n\n@_autoinit\ndef list(trusted, signing):\n \"\"\"List known keys.\n\n Args:\n trusted (bool): if True list public keys\n signing (bool): if True list private keys\n \"\"\"\n if trusted:\n GPG(\"--list-public-keys\")\n\n if signing:\n GPG(\"--list-secret-keys\")\n\n\ndef _verify_exe_or_raise(exe):\n msg = (\n \"Spack requires gpgconf version >= 2\\n\"\n \" To install a suitable version using Spack, run\\n\"\n \" spack install gnupg@2:\\n\"\n \" and load it by running\\n\"\n \" spack load gnupg@2:\"\n )\n if not exe:\n raise SpackGPGError(msg)\n\n output = exe(\"--version\", output=str)\n match = re.search(r\"^gpg(conf)? \\(GnuPG(?:/MacGPG2)?\\) (.*)$\", output, re.M)\n if not match:\n raise SpackGPGError('Could not determine \"{0}\" version'.format(exe.name))\n\n if spack.version.Version(match.group(2)) < spack.version.Version(\"2\"):\n raise SpackGPGError(msg)\n\n\ndef _gpgconf():\n exe = spack.util.executable.which(\"gpgconf\", \"gpg2conf\", \"gpgconf2\")\n _verify_exe_or_raise(exe)\n\n # ensure that the gpgconf we found can run \"gpgconf --create-socketdir\"\n try:\n exe(\"--dry-run\", \"--create-socketdir\", output=os.devnull, error=os.devnull)\n except spack.util.executable.ProcessError:\n # no dice\n exe = None\n\n return exe\n\n\ndef _gpg():\n exe = spack.util.executable.which(\"gpg2\", \"gpg\")\n _verify_exe_or_raise(exe)\n return exe\n\n\ndef _socket_dir(gpgconf):\n # Try to ensure that (/var)/run/user/$(id -u) exists so that\n # `gpgconf --create-socketdir` can be run later.\n #\n # NOTE(opadron): This action helps prevent a large class of\n # \"file-name-too-long\" errors in gpg.\n\n # If there is no suitable gpgconf, don't even bother trying to\n # pre-create a user run dir.\n if not gpgconf:\n return None\n\n result = None\n for var_run in (\"/run\", \"/var/run\"):\n if not os.path.exists(var_run):\n continue\n\n var_run_user = os.path.join(var_run, \"user\")\n try:\n if not os.path.exists(var_run_user):\n os.mkdir(var_run_user)\n os.chmod(var_run_user, 0o777)\n\n user_dir = os.path.join(var_run_user, str(spack.llnl.util.filesystem.getuid()))\n\n if not os.path.exists(user_dir):\n os.mkdir(user_dir)\n os.chmod(user_dir, 0o700)\n\n # If the above operation fails due to lack of permissions, then\n # just carry on without running gpgconf and hope for the best.\n #\n # NOTE(opadron): Without a dir in which to create a socket for IPC,\n # gnupg may fail if GNUPGHOME is set to a path that\n # is too long, where \"too long\" in this context is\n # actually quite short; somewhere in the\n # neighborhood of more than 100 characters.\n #\n # TODO(opadron): Maybe a warning should be printed in this case?\n except OSError as exc:\n if exc.errno not in (errno.EPERM, errno.EACCES):\n raise\n user_dir = None\n\n # return the last iteration that provides a usable user run dir\n if user_dir is not None:\n result = user_dir\n\n return result\n" }, { "path": "lib/spack/spack/util/hash.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport base64\nimport hashlib\n\nimport spack.util.crypto\n\n\ndef b32_hash(content):\n \"\"\"Return the b32 encoded sha1 hash of the input string as a string.\"\"\"\n sha = hashlib.sha1(content.encode(\"utf-8\"))\n b32_hash = base64.b32encode(sha.digest()).lower()\n b32_hash = b32_hash.decode(\"utf-8\")\n return b32_hash\n\n\ndef base32_prefix_bits(hash_string, bits):\n \"\"\"Return the first bits of a base32 string as an integer.\"\"\"\n if bits > len(hash_string) * 5:\n raise ValueError(\"Too many bits! Requested %d bit prefix of '%s'.\" % (bits, hash_string))\n\n hash_bytes = base64.b32decode(hash_string, casefold=True)\n return spack.util.crypto.prefix_bits(hash_bytes, bits)\n" }, { "path": "lib/spack/spack/util/ld_so_conf.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport glob\nimport os\nimport re\nimport sys\n\nimport spack.util.elf as elf_utils\nfrom spack.llnl.util.lang import dedupe\n\n\ndef parse_ld_so_conf(conf_file=\"/etc/ld.so.conf\"):\n \"\"\"Parse glibc style ld.so.conf file, which specifies default search paths for the\n dynamic linker. This can in principle also be used for musl libc.\n\n Arguments:\n conf_file (str or bytes): Path to config file\n\n Returns:\n list: List of absolute search paths\n \"\"\"\n # Parse in binary mode since it's faster\n is_bytes = isinstance(conf_file, bytes)\n if not is_bytes:\n conf_file = conf_file.encode(\"utf-8\")\n\n # For globbing in Python2 we need to chdir.\n cwd = os.getcwd()\n try:\n paths = _process_ld_so_conf_queue([conf_file])\n finally:\n os.chdir(cwd)\n\n return list(paths) if is_bytes else [p.decode(\"utf-8\") for p in paths]\n\n\ndef _process_ld_so_conf_queue(queue):\n include_regex = re.compile(b\"include\\\\s\")\n paths = []\n while queue:\n p = queue.pop(0)\n\n try:\n with open(p, \"rb\") as f:\n lines = f.readlines()\n except OSError:\n continue\n\n for line in lines:\n # Strip comments\n comment = line.find(b\"#\")\n if comment != -1:\n line = line[:comment]\n\n # Skip empty lines\n line = line.strip()\n if not line:\n continue\n\n is_include = include_regex.match(line) is not None\n\n # If not an include, it's a literal path (no globbing here).\n if not is_include:\n # We only allow absolute search paths.\n if os.path.isabs(line):\n paths.append(line)\n continue\n\n # Finally handle includes.\n include_path = line[8:].strip()\n if not include_path:\n continue\n\n cwd = os.path.dirname(p)\n os.chdir(cwd)\n queue.extend(os.path.join(cwd, p) for p in glob.glob(include_path))\n\n return dedupe(paths)\n\n\ndef get_conf_file_from_dynamic_linker(dynamic_linker_name):\n # We basically assume everything is glibc, except musl.\n if \"ld-musl-\" not in dynamic_linker_name:\n return \"ld.so.conf\"\n\n # Musl has a dynamic loader of the form ld-musl-.so.1\n # and a corresponding config file ld-musl-.path\n idx = dynamic_linker_name.find(\".\")\n if idx != -1:\n return dynamic_linker_name[:idx] + \".path\"\n\n\ndef host_dynamic_linker_search_paths():\n \"\"\"Retrieve the current host runtime search paths for shared libraries;\n for GNU and musl Linux we try to retrieve the dynamic linker from the\n current Python interpreter and then find the corresponding config file\n (e.g. ld.so.conf or ld-musl-.path). Similar can be done for\n BSD and others, but this is not implemented yet. The default paths\n are always returned. We don't check if the listed directories exist.\"\"\"\n default_paths = [\"/usr/lib\", \"/usr/lib64\", \"/lib\", \"/lib64\"]\n\n # Currently only for Linux (gnu/musl)\n if not sys.platform.startswith(\"linux\"):\n return default_paths\n\n # If everything fails, try this standard glibc path.\n conf_file = \"/etc/ld.so.conf\"\n\n # Try to improve on the default conf path by retrieving the location of the\n # dynamic linker from our current Python interpreter, and figure out the\n # config file location from there.\n try:\n with open(sys.executable, \"rb\") as f:\n elf = elf_utils.parse_elf(f, dynamic_section=False, interpreter=True)\n\n # If we have a dynamic linker, try to retrieve the config file relative\n # to its prefix.\n if elf.has_pt_interp:\n dynamic_linker = elf.pt_interp_str.decode(\"utf-8\")\n dynamic_linker_name = os.path.basename(dynamic_linker)\n conf_name = get_conf_file_from_dynamic_linker(dynamic_linker_name)\n\n # Typically it is /lib/ld.so, but on Gentoo Prefix it is something\n # like /lib/ld.so. And on Debian /lib64 is actually\n # a symlink to /usr/lib64. So, best effort attempt is to just strip\n # two path components and join with etc/ld.so.conf.\n possible_prefix = os.path.dirname(os.path.dirname(dynamic_linker))\n possible_conf = os.path.join(possible_prefix, \"etc\", conf_name)\n\n if os.path.exists(possible_conf):\n conf_file = possible_conf\n except (OSError, elf_utils.ElfParsingError):\n pass\n\n # Note: ld_so_conf doesn't error if the file does not exist.\n return list(dedupe(parse_ld_so_conf(conf_file) + default_paths))\n" }, { "path": "lib/spack/spack/util/libc.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport os\nimport re\nimport shlex\nimport sys\nfrom subprocess import PIPE, run\nfrom typing import Dict, List, Optional\n\nimport spack.spec\nimport spack.util.elf\nfrom spack.llnl.util.lang import memoized\n\n#: Pattern to distinguish glibc from other libc implementations\nGLIBC_PATTERN = r\"\\b(?:Free Software Foundation|Roland McGrath|Ulrich Depper)\\b\"\n\n\ndef _env() -> Dict[str, str]:\n \"\"\"Currently only set LC_ALL=C without clearing further environment variables\"\"\"\n return {**os.environ, \"LC_ALL\": \"C\"}\n\n\ndef _libc_from_ldd(ldd: str) -> Optional[\"spack.spec.Spec\"]:\n try:\n result = run([ldd, \"--version\"], stdout=PIPE, stderr=PIPE, check=False, env=_env())\n stdout = result.stdout.decode(\"utf-8\")\n except Exception:\n return None\n\n # The string \"Free Software Foundation\" is sometimes translated and not detected, but the names\n # of the authors are typically present.\n if not re.search(GLIBC_PATTERN, stdout):\n return None\n\n version_str = re.match(r\".+\\(.+\\) (.+)\", stdout)\n if not version_str:\n return None\n try:\n return spack.spec.Spec(f\"glibc@={version_str.group(1)}\")\n except Exception:\n return None\n\n\ndef default_search_paths_from_dynamic_linker(dynamic_linker: str) -> List[str]:\n \"\"\"If the dynamic linker is glibc at a certain version, we can query the hard-coded library\n search paths\"\"\"\n try:\n result = run([dynamic_linker, \"--help\"], stdout=PIPE, stderr=PIPE, check=False, env=_env())\n assert result.returncode == 0\n out = result.stdout.decode(\"utf-8\")\n except Exception:\n return []\n\n return [\n match.group(1).strip()\n for match in re.finditer(r\"^ (/.+) \\(system search path\\)$\", out, re.MULTILINE)\n ]\n\n\ndef libc_from_dynamic_linker(dynamic_linker: str) -> Optional[\"spack.spec.Spec\"]:\n \"\"\"Get the libc spec from the dynamic linker path.\"\"\"\n maybe_spec = _libc_from_dynamic_linker(dynamic_linker)\n if maybe_spec:\n return maybe_spec.copy()\n return None\n\n\n@memoized\ndef _libc_from_dynamic_linker(dynamic_linker: str) -> Optional[\"spack.spec.Spec\"]:\n if not os.path.exists(dynamic_linker):\n return None\n\n # The dynamic linker is usually installed in the same /lib(64)?/ld-*.so path across all\n # distros. The rest of libc is elsewhere, e.g. /usr. Typically the dynamic linker is then\n # a symlink into /usr/lib, which we use to for determining the actual install prefix of\n # libc.\n realpath = os.path.realpath(dynamic_linker)\n\n prefix = os.path.dirname(realpath)\n # Remove the multiarch suffix if it exists\n if os.path.basename(prefix) not in (\"lib\", \"lib64\"):\n prefix = os.path.dirname(prefix)\n\n # Non-standard install layout -- just bail.\n if os.path.basename(prefix) not in (\"lib\", \"lib64\"):\n return None\n\n prefix = os.path.dirname(prefix)\n\n # Now try to figure out if glibc or musl, which is the only ones we support.\n # In recent glibc we can simply execute the dynamic loader. In musl that's always the case.\n try:\n result = run(\n [dynamic_linker, \"--version\"], stdout=PIPE, stderr=PIPE, check=False, env=_env()\n )\n stdout = result.stdout.decode(\"utf-8\")\n stderr = result.stderr.decode(\"utf-8\")\n except Exception:\n return None\n\n # musl prints to stderr\n if stderr.startswith(\"musl libc\"):\n version_str = re.search(r\"^Version (.+)$\", stderr, re.MULTILINE)\n if not version_str:\n return None\n try:\n spec = spack.spec.Spec(f\"musl@={version_str.group(1)}\")\n spec.external_path = prefix\n return spec\n except Exception:\n return None\n elif re.search(GLIBC_PATTERN, stdout):\n # output is like \"ld.so (...) stable release version 2.33.\"\n match = re.search(r\"version (\\d+\\.\\d+(?:\\.\\d+)?)\", stdout)\n if not match:\n return None\n try:\n version = match.group(1)\n spec = spack.spec.Spec(f\"glibc@={version}\")\n spec.external_path = prefix\n return spec\n except Exception:\n return None\n else:\n # Could not get the version by running the dynamic linker directly. Instead locate `ldd`\n # relative to the dynamic linker.\n ldd = os.path.join(prefix, \"bin\", \"ldd\")\n if not os.path.exists(ldd):\n # If `/lib64/ld.so` was not a symlink to `/usr/lib/ld.so` we can try to use /usr as\n # prefix. This is the case on ubuntu 18.04 where /lib != /usr/lib.\n if prefix != \"/\":\n return None\n prefix = \"/usr\"\n ldd = os.path.join(prefix, \"bin\", \"ldd\")\n if not os.path.exists(ldd):\n return None\n maybe_spec = _libc_from_ldd(ldd)\n if not maybe_spec:\n return None\n maybe_spec.external_path = prefix\n return maybe_spec\n\n\ndef libc_from_current_python_process() -> Optional[\"spack.spec.Spec\"]:\n if not sys.executable:\n return None\n\n dynamic_linker = spack.util.elf.pt_interp(sys.executable)\n\n if not dynamic_linker:\n return None\n\n return libc_from_dynamic_linker(dynamic_linker)\n\n\ndef parse_dynamic_linker(output: str):\n \"\"\"Parse ``-dynamic-linker /path/to/ld.so`` from compiler output\"\"\"\n for line in reversed(output.splitlines()):\n if \"-dynamic-linker\" not in line:\n continue\n args = shlex.split(line)\n\n for idx in reversed(range(1, len(args))):\n arg = args[idx]\n if arg == \"-dynamic-linker\" or args == \"--dynamic-linker\":\n return args[idx + 1]\n elif arg.startswith(\"--dynamic-linker=\") or arg.startswith(\"-dynamic-linker=\"):\n return arg.split(\"=\", 1)[1]\n" }, { "path": "lib/spack/spack/util/lock.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Wrapper for ``spack.llnl.util.lock`` allows locking to be enabled/disabled.\"\"\"\n\nimport os\nimport stat\nimport sys\nfrom typing import Optional, Tuple\n\nimport spack.error\nfrom spack.llnl.util.lock import Lock as Llnl_lock\nfrom spack.llnl.util.lock import (\n LockError,\n LockTimeoutError,\n LockUpgradeError,\n ReadTransaction,\n WriteTransaction,\n)\n\n\nclass Lock(Llnl_lock):\n \"\"\"Lock that can be disabled.\n\n This overrides the ``_lock()`` and ``_unlock()`` methods from\n ``spack.llnl.util.lock`` so that all the lock API calls will succeed, but\n the actual locking mechanism can be disabled via ``_enable_locks``.\n \"\"\"\n\n def __init__(\n self,\n path: str,\n *,\n start: int = 0,\n length: int = 0,\n default_timeout: Optional[float] = None,\n debug: bool = False,\n desc: str = \"\",\n enable: bool = True,\n ) -> None:\n self._enable = sys.platform != \"win32\" and enable\n super().__init__(\n path,\n start=start,\n length=length,\n default_timeout=default_timeout,\n debug=debug,\n desc=desc,\n )\n\n def _reaffirm_lock(self) -> None:\n if self._enable:\n super()._reaffirm_lock()\n\n def _lock(self, op: int, timeout: Optional[float] = 0.0) -> Tuple[float, int]:\n if self._enable:\n return super()._lock(op, timeout)\n return 0.0, 0\n\n def _poll_lock(self, op: int) -> bool:\n if self._enable:\n return super()._poll_lock(op)\n return True\n\n def _unlock(self) -> None:\n \"\"\"Unlock call that always succeeds.\"\"\"\n if self._enable:\n super()._unlock()\n\n def cleanup(self, *args) -> None:\n if self._enable:\n super().cleanup(*args)\n\n\ndef check_lock_safety(path: str) -> None:\n \"\"\"Do some extra checks to ensure disabling locks is safe.\n\n This will raise an error if ``path`` can is group- or world-writable\n AND the current user can write to the directory (i.e., if this user\n AND others could write to the path).\n\n This is intended to run on the Spack prefix, but can be run on any\n path for testing.\n \"\"\"\n if os.access(path, os.W_OK):\n stat_result = os.stat(path)\n uid, gid = stat_result.st_uid, stat_result.st_gid\n mode = stat_result[stat.ST_MODE]\n\n writable = None\n if (mode & stat.S_IWGRP) and (uid != gid):\n # spack is group-writeable and the group is not the owner\n writable = \"group\"\n elif mode & stat.S_IWOTH:\n # spack is world-writeable\n writable = \"world\"\n\n if writable:\n msg = f\"Refusing to disable locks: spack is {writable}-writable.\"\n long_msg = (\n f\"Running a shared spack without locks is unsafe. You must \"\n f\"restrict permissions on {path} or enable locks.\"\n )\n raise spack.error.SpackError(msg, long_msg)\n\n\n__all__ = [\n \"LockError\",\n \"LockTimeoutError\",\n \"LockUpgradeError\",\n \"ReadTransaction\",\n \"WriteTransaction\",\n \"Lock\",\n \"check_lock_safety\",\n]\n" }, { "path": "lib/spack/spack/util/log_parse.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport io\nfrom typing import List, TextIO, Union\n\nfrom spack.llnl.util.tty.color import cescape, colorize\nfrom spack.util.ctest_log_parser import CTestLogParser, LogEvent\n\n__all__ = [\"parse_log_events\", \"make_log_context\"]\n\n\ndef parse_log_events(stream: Union[str, TextIO], context: int = 6, profile: bool = False):\n \"\"\"Extract interesting events from a log file as a list of LogEvent.\n\n Args:\n stream: build log name or file object\n context: lines of context to extract around each log event\n profile: print out profile information for parsing\n\n Returns:\n two lists containing :class:`~spack.util.ctest_log_parser.BuildError` and\n :class:`~spack.util.ctest_log_parser.BuildWarning` objects.\n\n This is a wrapper around :class:`~spack.util.ctest_log_parser.CTestLogParser` that\n lazily constructs a single ``CTestLogParser`` object. This ensures\n that all the regex compilation is only done once.\n \"\"\"\n parser = getattr(parse_log_events, \"ctest_parser\", None)\n if parser is None:\n parser = CTestLogParser(profile=profile)\n setattr(parse_log_events, \"ctest_parser\", parser)\n\n result = parser.parse(stream, context)\n if profile:\n parser.print_timings()\n return result\n\n\n#: lazily constructed CTest log parser\nparse_log_events.ctest_parser = None # type: ignore[attr-defined]\n\n\ndef make_log_context(log_events: List[LogEvent]) -> str:\n \"\"\"Get error context from a log file.\n\n Args:\n log_events: list of events created by ``ctest_log_parser.parse()``\n\n Returns:\n str: context from the build log with errors highlighted\n\n Parses the log file for lines containing errors, and prints them out with context.\n Errors are highlighted in red and warnings in yellow. Events are sorted by line number.\n \"\"\"\n event_colors = {e.line_no: e.color for e in log_events}\n log_events = sorted(log_events, key=lambda e: e.line_no)\n\n out = io.StringIO()\n next_line = 1\n block_start = -1\n block_lines: List[str] = []\n\n def flush_block():\n block_end = block_start + len(block_lines) - 1\n out.write(colorize(\"@c{-- lines %d to %d --}\\n\" % (block_start, block_end)))\n out.writelines(block_lines)\n block_lines.clear()\n\n for event in log_events:\n start = event.start\n\n if start < next_line:\n start = next_line\n elif block_lines:\n flush_block()\n\n if not block_lines:\n block_start = start\n\n for i in range(start, event.end):\n if i in event_colors:\n color = event_colors[i]\n block_lines.append(colorize(\"@%s{> %s}\\n\" % (color, cescape(event[i]))))\n else:\n block_lines.append(\" %s\\n\" % event[i])\n\n next_line = event.end\n\n if block_lines:\n flush_block()\n\n return out.getvalue()\n" }, { "path": "lib/spack/spack/util/module_cmd.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"\nThis module contains routines related to the module command for accessing and\nparsing environment modules.\n\"\"\"\n\nimport os\nimport re\nimport subprocess\nfrom typing import MutableMapping, Optional\n\nimport spack.llnl.util.tty as tty\nfrom spack.error import SpackError\n\n# This list is not exhaustive. Currently we only use load and unload\n# If we need another option that changes the environment, add it here.\nmodule_change_commands = [\"load\", \"swap\", \"unload\", \"purge\", \"use\", \"unuse\"]\n\n# This awk script is a posix alternative to `env -0`\nawk_cmd = r\"\"\"awk 'BEGIN{for(name in ENVIRON)\"\"\" r\"\"\"printf(\"%s=%s%c\", name, ENVIRON[name], 0)}'\"\"\"\n\n\ndef module(\n *args: str,\n module_template: Optional[str] = None,\n module_src_cmd: Optional[str] = None,\n environb: Optional[MutableMapping[bytes, bytes]] = None,\n):\n \"\"\"Run the ``module`` shell function in a ``/bin/bash`` subprocess, and either collect its\n changes to environment variables and apply them in the current process (for ``module load``,\n ``module swap``, etc.), or return its output as a string (for ``module show``, etc.).\n\n This requires ``/bin/bash`` to be available on the system and ``awk`` to be in ``PATH``.\n\n Args:\n args: Command line arguments for the module command.\n environb: (Binary) environment variables dictionary. If not provided, the current\n process's environment is modified.\n \"\"\"\n module_cmd = module_template or (\"module \" + \" \".join(args))\n environb = environb or os.environb\n if b\"MODULESHOME\" in environb:\n module_cmd = module_src_cmd or \"source $MODULESHOME/init/bash; \" + module_cmd\n\n if args[0] in module_change_commands:\n # Suppress module output\n module_cmd += r\" >/dev/null 2>&1; \" + awk_cmd\n module_p = subprocess.Popen(\n module_cmd,\n stdout=subprocess.PIPE,\n stderr=subprocess.STDOUT,\n shell=True,\n executable=\"/bin/bash\",\n env=environb,\n )\n\n new_environb = {}\n output = module_p.communicate()[0]\n\n # Loop over each environment variable key=value byte string\n for entry in output.strip(b\"\\0\").split(b\"\\0\"):\n # Split variable name and value\n parts = entry.split(b\"=\", 1)\n if len(parts) != 2:\n continue\n new_environb[parts[0]] = parts[1]\n\n # Update os.environ with new dict\n environb.clear()\n environb.update(new_environb) # novermin\n\n else:\n # Simply execute commands that don't change state and return output\n module_p = subprocess.Popen(\n module_cmd,\n stdout=subprocess.PIPE,\n stderr=subprocess.STDOUT,\n shell=True,\n executable=\"/bin/bash\",\n )\n # Decode and str to return a string object in both python 2 and 3\n return str(module_p.communicate()[0].decode())\n\n\ndef load_module(mod):\n \"\"\"Takes a module name and removes modules until it is possible to\n load that module. It then loads the provided module. Depends on the\n modulecmd implementation of modules used in cray and lmod.\n\n Raises:\n ModuleLoadError: if the module could not be loaded\n \"\"\"\n tty.debug(\"module_cmd.load_module: {0}\".format(mod))\n # Read the module and remove any conflicting modules\n # We do this without checking that they are already installed\n # for ease of programming because unloading a module that is not\n # loaded does nothing.\n text = module(\"show\", mod).split()\n for i, word in enumerate(text):\n if word == \"conflict\":\n module(\"unload\", text[i + 1])\n\n # Store the LOADEDMODULES before trying to load the new module\n loaded_modules_before = os.environ.get(\"LOADEDMODULES\", \"\")\n\n # Load the module now that there are no conflicts\n # Some module systems use stdout and some use stderr\n module(\"load\", mod)\n\n # Check if the module was actually loaded by comparing LOADEDMODULES\n loaded_modules_after = os.environ.get(\"LOADEDMODULES\", \"\")\n\n # If LOADEDMODULES didn't change, the module wasn't loaded\n if loaded_modules_before == loaded_modules_after:\n raise ModuleLoadError(mod)\n\n\ndef get_path_args_from_module_line(line):\n if \"(\" in line and \")\" in line:\n # Determine which lua quote symbol is being used for the argument\n comma_index = line.index(\",\")\n cline = line[comma_index:]\n try:\n quote_index = min(cline.find(q) for q in ['\"', \"'\"] if q in cline)\n lua_quote = cline[quote_index]\n except ValueError:\n # Change error text to describe what is going on.\n raise ValueError(\"No lua quote symbol found in lmod module line.\")\n words_and_symbols = line.split(lua_quote)\n path_arg = words_and_symbols[-2]\n else:\n # The path arg is the 3rd \"word\" of the line in a Tcl module\n # OPERATION VAR_NAME PATH_ARG\n words = line.split()\n if len(words) > 2:\n path_arg = words[2]\n else:\n return []\n\n paths = path_arg.split(\":\")\n return paths\n\n\ndef path_from_modules(modules):\n \"\"\"Inspect a list of Tcl modules for entries that indicate the absolute\n path at which the library supported by said module can be found.\n\n Args:\n modules (list): module files to be loaded to get an external package\n\n Returns:\n Guess of the prefix path where the package\n \"\"\"\n assert isinstance(modules, list), 'the \"modules\" argument must be a list'\n\n best_choice = None\n for module_name in modules:\n # Read the current module and return a candidate path\n text = module(\"show\", module_name).split(\"\\n\")\n candidate_path = get_path_from_module_contents(text, module_name)\n\n if candidate_path and not os.path.exists(candidate_path):\n msg = \"Extracted path from module does not exist [module={0}, path={1}]\"\n tty.warn(msg.format(module_name, candidate_path))\n\n # If anything is found, then it's the best choice. This means\n # that we give preference to the last module to be loaded\n # for packages requiring to load multiple modules in sequence\n best_choice = candidate_path or best_choice\n return best_choice\n\n\ndef get_path_from_module_contents(text, module_name):\n tty.debug(\"Module name: \" + module_name)\n pkg_var_prefix = module_name.replace(\"-\", \"_\").upper()\n components = pkg_var_prefix.split(\"/\")\n # For modules with multiple components like foo/1.0.1, retrieve the package\n # name \"foo\" from the module name\n if len(components) > 1:\n pkg_var_prefix = components[-2]\n tty.debug(\"Package directory variable prefix: \" + pkg_var_prefix)\n\n path_occurrences = {}\n\n def strip_path(path, endings):\n for ending in endings:\n if path.endswith(ending):\n return path[: -len(ending)]\n if path.endswith(ending + \"/\"):\n return path[: -(len(ending) + 1)]\n return path\n\n def match_pattern_and_strip(line, pattern, strip=[]):\n if re.search(pattern, line):\n paths = get_path_args_from_module_line(line)\n for path in paths:\n path = strip_path(path, strip)\n path_occurrences[path] = path_occurrences.get(path, 0) + 1\n\n def match_flag_and_strip(line, flag, strip=[]):\n flag_idx = line.find(flag)\n if flag_idx >= 0:\n # Search for the first occurrence of any separator marking the end of\n # the path.\n separators = (\" \", '\"', \"'\")\n occurrences = [line.find(s, flag_idx) for s in separators]\n indices = [idx for idx in occurrences if idx >= 0]\n if indices:\n path = line[flag_idx + len(flag) : min(indices)]\n else:\n path = line[flag_idx + len(flag) :]\n path = strip_path(path, strip)\n path_occurrences[path] = path_occurrences.get(path, 0) + 1\n\n lib_endings = [\"/lib64\", \"/lib\"]\n bin_endings = [\"/bin\"]\n man_endings = [\"/share/man\", \"/man\"]\n\n for line in text:\n # Check entries of LD_LIBRARY_PATH and CRAY_LD_LIBRARY_PATH\n pattern = r\"\\W(CRAY_)?LD_LIBRARY_PATH\"\n match_pattern_and_strip(line, pattern, lib_endings)\n\n # Check {name}_DIR entries\n pattern = r\"\\W{0}_DIR\".format(pkg_var_prefix)\n match_pattern_and_strip(line, pattern)\n\n # Check {name}_ROOT entries\n pattern = r\"\\W{0}_ROOT\".format(pkg_var_prefix)\n match_pattern_and_strip(line, pattern)\n\n # Check entries that update the PATH variable\n pattern = r\"\\WPATH\"\n match_pattern_and_strip(line, pattern, bin_endings)\n\n # Check entries that update the MANPATH variable\n pattern = r\"MANPATH\"\n match_pattern_and_strip(line, pattern, man_endings)\n\n # Check entries that add a `-rpath` flag to a variable\n match_flag_and_strip(line, \"-rpath\", lib_endings)\n\n # Check entries that add a `-L` flag to a variable\n match_flag_and_strip(line, \"-L\", lib_endings)\n\n # Whichever path appeared most in the module, we assume is the correct path\n if len(path_occurrences) > 0:\n return max(path_occurrences.items(), key=lambda x: x[1])[0]\n\n # Unable to find path in module\n return None\n\n\nclass ModuleLoadError(SpackError):\n \"\"\"Raised when a module cannot be loaded.\"\"\"\n\n def __init__(self, module):\n super().__init__(f\"Module '{module}' could not be loaded.\")\n" }, { "path": "lib/spack/spack/util/naming.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport io\nimport itertools\nimport re\nimport string\nfrom typing import List, Tuple\n\n__all__ = [\n \"pkg_name_to_class_name\",\n \"valid_module_name\",\n \"possible_spack_module_names\",\n \"simplify_name\",\n \"NamespaceTrie\",\n]\n\n#: see keyword.kwlist: https://github.com/python/cpython/blob/main/Lib/keyword.py\nRESERVED_NAMES_ONLY_LOWERCASE = frozenset(\n (\n \"and\",\n \"as\",\n \"assert\",\n \"async\",\n \"await\",\n \"break\",\n \"class\",\n \"continue\",\n \"def\",\n \"del\",\n \"elif\",\n \"else\",\n \"except\",\n \"finally\",\n \"for\",\n \"from\",\n \"global\",\n \"if\",\n \"import\",\n \"in\",\n \"is\",\n \"lambda\",\n \"nonlocal\",\n \"not\",\n \"or\",\n \"pass\",\n \"raise\",\n \"return\",\n \"try\",\n \"while\",\n \"with\",\n \"yield\",\n )\n)\n\nRESERVED_NAMES_LIST_MIXED_CASE = (\"False\", \"None\", \"True\")\n\n# Valid module names can contain '-' but can't start with it.\n_VALID_MODULE_RE_V1 = re.compile(r\"^\\w[\\w-]*$\")\n\n_VALID_MODULE_RE_V2 = re.compile(r\"^[a-z_][a-z0-9_]*$\")\n\n\ndef pkg_name_to_class_name(pkg_name: str):\n \"\"\"Convert a Spack package name to a class name, based on\n `PEP-8 `_:\n\n * Module and package names use lowercase_with_underscores.\n * Class names use the CapWords convention.\n\n Not all package names are valid Python identifiers:\n\n * They can contain ``-``, but cannot start with ``-``.\n * They can start with numbers, e.g. ``3proxy``.\n\n This function converts from the package name to the class convention by removing ``_`` and\n ``-``, and converting surrounding lowercase text to CapWords. If package name starts with a\n number, the class name returned will be prepended with ``_`` to make a valid Python identifier.\n \"\"\"\n class_name = re.sub(r\"[-_]+\", \"-\", pkg_name)\n class_name = string.capwords(class_name, \"-\")\n class_name = class_name.replace(\"-\", \"\")\n\n # Ensure that the class name is a valid Python identifier\n if re.match(r\"^[0-9]\", class_name) or class_name in RESERVED_NAMES_LIST_MIXED_CASE:\n class_name = f\"_{class_name}\"\n\n return class_name\n\n\ndef pkg_dir_to_pkg_name(dirname: str, package_api: Tuple[int, int]) -> str:\n \"\"\"Translate a package dir (pkg_dir/package.py) to its corresponding package name\"\"\"\n if package_api < (2, 0):\n return dirname\n return dirname.lstrip(\"_\").replace(\"_\", \"-\")\n\n\ndef pkg_name_to_pkg_dir(name: str, package_api: Tuple[int, int]) -> str:\n \"\"\"Translate a package name to its corresponding package dir (pkg_dir/package.py)\"\"\"\n if package_api < (2, 0):\n return name\n name = name.replace(\"-\", \"_\")\n if re.match(r\"^[0-9]\", name) or name in RESERVED_NAMES_ONLY_LOWERCASE:\n name = f\"_{name}\"\n return name\n\n\ndef possible_spack_module_names(python_mod_name: str) -> List[str]:\n \"\"\"Given a Python module name, return a list of all possible spack module\n names that could correspond to it.\"\"\"\n mod_name = re.sub(r\"^num(\\d)\", r\"\\1\", python_mod_name)\n\n parts = re.split(r\"(_)\", mod_name)\n options = [[\"_\", \"-\"]] * mod_name.count(\"_\")\n\n results: List[str] = []\n for subs in itertools.product(*options):\n s = list(parts)\n s[1::2] = subs\n results.append(\"\".join(s))\n\n return results\n\n\ndef simplify_name(name: str) -> str:\n \"\"\"Simplify package name to only lowercase, digits, and dashes.\n\n Simplifies a name which may include uppercase letters, periods,\n underscores, and pluses. In general, we want our package names to\n only contain lowercase letters, digits, and dashes.\n\n Args:\n name (str): The original name of the package\n\n Returns:\n str: The new name of the package\n \"\"\"\n # Convert CamelCase to Dashed-Names\n # e.g. ImageMagick -> Image-Magick\n # e.g. SuiteSparse -> Suite-Sparse\n # name = re.sub('([a-z])([A-Z])', r'\\1-\\2', name)\n\n # Rename Intel downloads\n # e.g. l_daal, l_ipp, l_mkl -> daal, ipp, mkl\n if name.startswith(\"l_\"):\n name = name[2:]\n\n # Convert UPPERCASE to lowercase\n # e.g. SAMRAI -> samrai\n name = name.lower()\n\n # Replace '_' and '.' with '-'\n # e.g. backports.ssl_match_hostname -> backports-ssl-match-hostname\n name = name.replace(\"_\", \"-\")\n name = name.replace(\".\", \"-\")\n\n # Replace \"++\" with \"pp\" and \"+\" with \"-plus\"\n # e.g. gtk+ -> gtk-plus\n # e.g. voro++ -> voropp\n name = name.replace(\"++\", \"pp\")\n name = name.replace(\"+\", \"-plus\")\n\n # Simplify Lua package names\n # We don't want \"lua\" to occur multiple times in the name\n name = re.sub(\"^(lua)([^-])\", r\"\\1-\\2\", name)\n\n # Simplify Bio++ package names\n name = re.sub(\"^(bpp)([^-])\", r\"\\1-\\2\", name)\n\n return name\n\n\ndef valid_module_name(mod_name: str, package_api: Tuple[int, int]) -> bool:\n \"\"\"Return whether mod_name is valid for use in Spack.\"\"\"\n if package_api < (2, 0):\n return bool(_VALID_MODULE_RE_V1.match(mod_name))\n elif not _VALID_MODULE_RE_V2.match(mod_name) or \"__\" in mod_name:\n return False\n elif mod_name.startswith(\"_\"):\n # it can only start with an underscore if followed by digit or reserved name\n return mod_name[1:] in RESERVED_NAMES_ONLY_LOWERCASE or mod_name[1].isdigit()\n else:\n return mod_name not in RESERVED_NAMES_ONLY_LOWERCASE\n\n\nclass NamespaceTrie:\n class Element:\n def __init__(self, value):\n self.value = value\n\n def __init__(self, separator=\".\"):\n self._subspaces = {}\n self._value = None\n self._sep = separator\n\n def __setitem__(self, namespace, value):\n first, sep, rest = namespace.partition(self._sep)\n\n if not first:\n self._value = NamespaceTrie.Element(value)\n return\n\n if first not in self._subspaces:\n self._subspaces[first] = NamespaceTrie()\n\n self._subspaces[first][rest] = value\n\n def _get_helper(self, namespace, full_name):\n first, sep, rest = namespace.partition(self._sep)\n if not first:\n if not self._value:\n raise KeyError(\"Can't find namespace '%s' in trie\" % full_name)\n return self._value.value\n elif first not in self._subspaces:\n raise KeyError(\"Can't find namespace '%s' in trie\" % full_name)\n else:\n return self._subspaces[first]._get_helper(rest, full_name)\n\n def __getitem__(self, namespace):\n return self._get_helper(namespace, namespace)\n\n def is_prefix(self, namespace):\n \"\"\"True if the namespace has a value, or if it's the prefix of one that\n does.\"\"\"\n first, sep, rest = namespace.partition(self._sep)\n if not first:\n return True\n elif first not in self._subspaces:\n return False\n else:\n return self._subspaces[first].is_prefix(rest)\n\n def is_leaf(self, namespace):\n \"\"\"True if this namespace has no children in the trie.\"\"\"\n first, sep, rest = namespace.partition(self._sep)\n if not first:\n return bool(self._subspaces)\n elif first not in self._subspaces:\n return False\n else:\n return self._subspaces[first].is_leaf(rest)\n\n def has_value(self, namespace):\n \"\"\"True if there is a value set for the given namespace.\"\"\"\n first, sep, rest = namespace.partition(self._sep)\n if not first:\n return self._value is not None\n elif first not in self._subspaces:\n return False\n else:\n return self._subspaces[first].has_value(rest)\n\n def __contains__(self, namespace):\n \"\"\"Returns whether a value has been set for the namespace.\"\"\"\n return self.has_value(namespace)\n\n def _str_helper(self, stream, level=0):\n indent = level * \" \"\n for name in sorted(self._subspaces):\n stream.write(indent + name + \"\\n\")\n if self._value:\n stream.write(indent + \" \" + repr(self._value.value))\n stream.write(self._subspaces[name]._str_helper(stream, level + 1))\n\n def __str__(self):\n stream = io.StringIO()\n self._str_helper(stream)\n return stream.getvalue()\n" }, { "path": "lib/spack/spack/util/package_hash.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport ast\nimport sys\nfrom typing import Any, Dict, List, Optional, Tuple\n\nimport spack.directives_meta\nimport spack.error\nimport spack.fetch_strategy\nimport spack.repo\nimport spack.spec\nimport spack.util.hash\nfrom spack.util.unparse import unparse\n\nif sys.version_info >= (3, 8):\n\n def unused_string(node: ast.AST) -> bool:\n \"\"\"Criteria for unassigned body strings.\"\"\"\n return (\n isinstance(node, ast.Expr)\n and isinstance(node.value, ast.Constant)\n and isinstance(node.value.value, str)\n )\n\nelse:\n\n def unused_string(node: ast.AST) -> bool:\n \"\"\"Criteria for unassigned body strings.\"\"\"\n return isinstance(node, ast.Expr) and isinstance(node.value, ast.Str)\n\n\nclass RemoveDocstrings(ast.NodeTransformer):\n \"\"\"Transformer that removes docstrings from a Python AST.\n\n This removes *all* strings that aren't on the RHS of an assignment statement from\n the body of functions, classes, and modules -- even if they're not directly after\n the declaration.\n\n \"\"\"\n\n def remove_docstring(self, node):\n if node.body:\n node.body = [child for child in node.body if not unused_string(child)]\n if not node.body:\n node.body = [ast.Pass()]\n\n self.generic_visit(node)\n return node\n\n def visit_FunctionDef(self, node):\n return self.remove_docstring(node)\n\n def visit_ClassDef(self, node):\n return self.remove_docstring(node)\n\n def visit_Module(self, node):\n return self.remove_docstring(node)\n\n\nclass RemoveDirectives(ast.NodeTransformer):\n \"\"\"Remove Spack directives from a package AST.\n\n This removes Spack directives (e.g., ``depends_on``, ``conflicts``, etc.) and\n metadata attributes (e.g., ``tags``, ``homepage``, ``url``) in a top-level class\n definition within a ``package.py``, but it does not modify nested classes or\n functions.\n\n If removing directives causes a ``for``, ``with``, or ``while`` statement to have an\n empty body, we remove the entire statement. Similarly, If removing directives causes\n an ``if`` statement to have an empty body or ``else`` block, we'll remove the block\n (or replace the body with ``pass`` if there is an ``else`` block but no body).\n\n \"\"\"\n\n def __init__(self, spec):\n #: List of attributes to be excluded from a package's hash.\n self.metadata_attrs = [s.url_attr for s in spack.fetch_strategy.all_strategies] + [\n \"homepage\",\n \"url\",\n \"urls\",\n \"list_url\",\n \"extendable\",\n \"parallel\",\n \"make_jobs\",\n \"maintainers\",\n \"tags\",\n ]\n\n self.spec = spec\n self.in_classdef = False # used to avoid nested classdefs\n\n def visit_Expr(self, node):\n # Directives are represented in the AST as named function call expressions (as\n # opposed to function calls through a variable callback). We remove them.\n #\n # Note that changes to directives (e.g., a preferred version change or a hash\n # change on an archive) are already represented in the spec *outside* the\n # package hash.\n return (\n None\n if (\n node.value\n and isinstance(node.value, ast.Call)\n and isinstance(node.value.func, ast.Name)\n and node.value.func.id in spack.directives_meta.directive_names\n )\n else node\n )\n\n def visit_Assign(self, node):\n # Remove assignments to metadata attributes, b/c they don't affect the build.\n return (\n None\n if (\n node.targets\n and isinstance(node.targets[0], ast.Name)\n and node.targets[0].id in self.metadata_attrs\n )\n else node\n )\n\n def visit_With(self, node):\n self.generic_visit(node) # visit children\n return node if node.body else None # remove with statement if it has no body\n\n def visit_For(self, node):\n self.generic_visit(node) # visit children\n return node if node.body else None # remove loop if it has no body\n\n def visit_While(self, node):\n self.generic_visit(node) # visit children\n return node if node.body else None # remove loop if it has no body\n\n def visit_If(self, node):\n self.generic_visit(node)\n\n # an empty orelse is ignored by unparsing, but an empty body with a full orelse\n # ends up unparsing as a syntax error, so we replace the empty body into `pass`.\n if not node.body:\n if node.orelse:\n node.body = [ast.Pass()]\n else:\n return None\n\n # if the node has a body, it's valid python code with or without an orelse\n return node\n\n def visit_FunctionDef(self, node):\n # do not descend into function definitions\n return node\n\n def visit_ClassDef(self, node):\n # packages are always top-level, and we do not descend\n # into nested class defs and their attributes\n if self.in_classdef:\n return node\n\n # guard against recursive class definitions\n self.in_classdef = True\n self.generic_visit(node)\n self.in_classdef = False\n\n # replace class definition with `pass` if it's empty (e.g., packages that only\n # have directives b/c they subclass a build system class)\n if not node.body:\n node.body = [ast.Pass()]\n\n return node\n\n\ndef _is_when_decorator(node: ast.Call) -> bool:\n \"\"\"Check if the node is a @when decorator.\"\"\"\n return isinstance(node.func, ast.Name) and node.func.id == \"when\" and len(node.args) == 1\n\n\nclass TagMultiMethods(ast.NodeVisitor):\n \"\"\"Tag @when-decorated methods in a package AST.\"\"\"\n\n def __init__(self, spec: spack.spec.Spec) -> None:\n self.spec = spec\n # map from function name to (implementation, condition_list) tuples\n self.methods: Dict[str, List[Tuple[ast.FunctionDef, List[Optional[bool]]]]] = {}\n\n if sys.version_info >= (3, 8):\n\n def _get_when_condition(self, node: ast.expr) -> Optional[Any]:\n \"\"\"Extract the first argument of a @when decorator.\"\"\"\n return node.value if isinstance(node, ast.Constant) else None\n\n else:\n\n def _get_when_condition(self, node: ast.expr) -> Optional[Any]:\n \"\"\"Extract the first argument of a @when decorator.\"\"\"\n if isinstance(node, ast.Str):\n return node.s\n elif isinstance(node, ast.NameConstant):\n return node.value\n return None\n\n def _evaluate_decorator(self, dec: ast.AST) -> Optional[bool]:\n \"\"\"Evaluates a single decorator node. Returns True/False if it's a statically evaluatable\n @when decorator, otherwise returns None.\"\"\"\n if not isinstance(dec, ast.Call) or not _is_when_decorator(dec):\n return None\n\n # Extract from the @when() decorator.\n cond = self._get_when_condition(dec.args[0])\n\n # Statically evaluate the condition if possible. If not, return None.\n if isinstance(cond, str):\n try:\n return self.spec.satisfies(cond)\n except Exception:\n return None\n elif isinstance(cond, bool):\n return cond\n else:\n return None\n\n def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.AST:\n conditions = [self._evaluate_decorator(dec) for dec in node.decorator_list]\n\n # anything defined without conditions will overwrite prior definitions\n if not conditions:\n self.methods[node.name] = []\n\n # add all discovered conditions on this node to the node list\n self.methods.setdefault(node.name, []).append((node, conditions))\n\n # don't modify the AST -- return the untouched function node\n return node\n\n\nclass ResolveMultiMethods(ast.NodeTransformer):\n \"\"\"Remove multi-methods when we know statically that they won't be used.\n\n Say we have multi-methods like this::\n\n class SomePackage:\n def foo(self): print(\"implementation 1\")\n\n @when(\"@1.0\")\n def foo(self): print(\"implementation 2\")\n\n @when(\"@2.0\")\n @when(sys.platform == \"darwin\")\n def foo(self): print(\"implementation 3\")\n\n @when(\"@3.0\")\n def foo(self): print(\"implementation 4\")\n\n The multimethod that will be chosen at runtime depends on the package spec and on\n whether we're on the darwin platform *at build time* (the darwin condition for\n implementation 3 is dynamic). We know the package spec statically; we don't know\n statically what the runtime environment will be. We need to include things that can\n possibly affect package behavior in the package hash, and we want to exclude things\n when we know that they will not affect package behavior.\n\n If we're at version 4.0, we know that implementation 1 will win, because some @when\n for 2, 3, and 4 will be ``False``. We should only include implementation 1.\n\n If we're at version 1.0, we know that implementation 2 will win, because it\n overrides implementation 1. We should only include implementation 2.\n\n If we're at version 3.0, we know that implementation 4 will win, because it\n overrides implementation 1 (the default), and some @when on all others will be\n False.\n\n If we're at version 2.0, it's a bit more complicated. We know we can remove\n implementations 2 and 4, because their @when's will never be satisfied. But, the\n choice between implementations 1 and 3 will happen at runtime (this is a bad example\n because the spec itself has platform information, and we should prefer to use that,\n but we allow arbitrary boolean expressions in @when's, so this example suffices).\n For this case, we end up needing to include *both* implementation 1 and 3 in the\n package hash, because either could be chosen.\n\n \"\"\"\n\n def __init__(self, methods):\n self.methods = methods\n\n def resolve(self, impl_conditions):\n \"\"\"Given list of nodes and conditions, figure out which node will be chosen.\"\"\"\n result = []\n default = None\n for impl, conditions in impl_conditions:\n # if there's a default implementation with no conditions, remember that.\n if not conditions:\n default = impl\n result.append(default)\n continue\n\n # any known-false @when means the method won't be used\n if any(c is False for c in conditions):\n continue\n\n # anything with all known-true conditions will be picked if it's first\n if all(c is True for c in conditions):\n if result and result[0] is default:\n return [impl] # we know the first MM will always win\n # if anything dynamic comes before it we don't know if it'll win,\n # so just let this result get appended\n\n # anything else has to be determined dynamically, so add it to a list\n result.append(impl)\n\n # if nothing was picked, the last definition wins.\n return result\n\n def visit_FunctionDef(self, node: ast.FunctionDef) -> Optional[ast.FunctionDef]:\n # if the function def wasn't visited on the first traversal there is a problem\n assert node.name in self.methods, \"Inconsistent package traversal!\"\n\n # if the function is a multimethod, need to resolve it statically\n impl_conditions = self.methods[node.name]\n\n resolutions = self.resolve(impl_conditions)\n if not any(r is node for r in resolutions):\n # multimethod did not resolve to this function; remove it\n return None\n\n # if we get here, this function is a possible resolution for a multi-method.\n # it might be the only one, or there might be several that have to be evaluated\n # dynamcially. Either way, we include the function.\n\n # strip the when decorators (preserve the rest)\n node.decorator_list = [\n dec\n for dec in node.decorator_list\n if not (isinstance(dec, ast.Call) and _is_when_decorator(dec))\n ]\n return node\n\n\ndef canonical_source(\n spec: spack.spec.Spec, filter_multimethods: bool = True, source: Optional[bytes] = None\n) -> str:\n \"\"\"Get canonical source for a spec's package.py by unparsing its AST.\n\n Arguments:\n filter_multimethods: By default, filter multimethods out of the AST if they are known\n statically to be unused. Supply False to disable.\n source: Optionally provide a string to read python code from.\n \"\"\"\n return unparse(package_ast(spec, filter_multimethods, source=source), py_ver_consistent=True)\n\n\ndef package_hash(spec: spack.spec.Spec, source: Optional[bytes] = None) -> str:\n \"\"\"Get a hash of a package's canonical source code.\n\n This function is used to determine whether a spec needs a rebuild when a\n package's source code changes.\n\n Arguments:\n source: Optionally provide a string to read python code from.\n\n \"\"\"\n source = canonical_source(spec, filter_multimethods=True, source=source)\n return spack.util.hash.b32_hash(source)\n\n\ndef package_ast(\n spec: spack.spec.Spec, filter_multimethods: bool = True, source: Optional[bytes] = None\n) -> ast.AST:\n \"\"\"Get the AST for the ``package.py`` file corresponding to ``spec``.\n\n Arguments:\n filter_multimethods: By default, filter multimethods out of the AST if they are known\n statically to be unused. Supply False to disable.\n source: Optionally provide a string to read python code from.\n \"\"\"\n if source is None:\n filename = spack.repo.PATH.filename_for_package_name(spec.name)\n with open(filename, \"rb\") as f:\n source = f.read()\n\n # create an AST\n root = ast.parse(source)\n\n # remove docstrings, comments, and directives from the package AST\n root = RemoveDocstrings().visit(root)\n root = RemoveDirectives(spec).visit(root)\n\n if filter_multimethods:\n # visit nodes and build up a dictionary of methods (no need to assign)\n tagger = TagMultiMethods(spec)\n tagger.visit(root)\n\n # transform AST using tagged methods\n root = ResolveMultiMethods(tagger.methods).visit(root)\n\n return root\n\n\nclass PackageHashError(spack.error.SpackError):\n \"\"\"Raised for all errors encountered during package hashing.\"\"\"\n" }, { "path": "lib/spack/spack/util/parallel.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport concurrent.futures\nimport multiprocessing\nimport os\nimport sys\nimport traceback\nfrom typing import Optional\n\nimport spack.config\n\n#: Used in tests to disable parallelism, as tests themselves are parallelized\nENABLE_PARALLELISM = sys.platform != \"win32\"\n\n\nclass ErrorFromWorker:\n \"\"\"Wrapper class to report an error from a worker process\"\"\"\n\n def __init__(self, exc_cls, exc, tb):\n \"\"\"Create an error object from an exception raised from\n the worker process.\n\n The attributes of the process error objects are all strings\n as they are easy to send over a pipe.\n\n Args:\n exc: exception raised from the worker process\n \"\"\"\n self.pid = os.getpid()\n self.error_message = str(exc)\n self.stacktrace_message = \"\".join(traceback.format_exception(exc_cls, exc, tb))\n\n @property\n def stacktrace(self):\n msg = \"[PID={0.pid}] {0.stacktrace_message}\"\n return msg.format(self)\n\n def __str__(self):\n return self.error_message\n\n\nclass Task:\n \"\"\"Wrapped task that trap every Exception and return it as an\n ErrorFromWorker object.\n\n We are using a wrapper class instead of a decorator since the class\n is pickleable, while a decorator with an inner closure is not.\n \"\"\"\n\n def __init__(self, func):\n self.func = func\n\n def __call__(self, *args, **kwargs):\n try:\n value = self.func(*args, **kwargs)\n except Exception:\n value = ErrorFromWorker(*sys.exc_info())\n return value\n\n\ndef imap_unordered(\n f, list_of_args, *, processes: int, maxtaskperchild: Optional[int] = None, debug=False\n):\n \"\"\"Wrapper around multiprocessing.Pool.imap_unordered.\n\n Args:\n f: function to apply\n list_of_args: list of tuples of args for the task\n processes: maximum number of processes allowed\n debug: if False, raise an exception containing just the error messages\n from workers, if True an exception with complete stacktraces\n maxtaskperchild: number of tasks to be executed by a child before being\n killed and substituted\n\n Raises:\n RuntimeError: if any error occurred in the worker processes\n \"\"\"\n\n if not ENABLE_PARALLELISM or len(list_of_args) <= 1:\n yield from map(f, list_of_args)\n return\n\n from spack.subprocess_context import GlobalStateMarshaler\n\n marshaler = GlobalStateMarshaler()\n with multiprocessing.Pool(\n processes, initializer=marshaler.restore, maxtasksperchild=maxtaskperchild\n ) as p:\n for result in p.imap_unordered(Task(f), list_of_args):\n if isinstance(result, ErrorFromWorker):\n raise RuntimeError(result.stacktrace if debug else str(result))\n yield result\n\n\nclass SequentialExecutor(concurrent.futures.Executor):\n \"\"\"Executor that runs tasks sequentially in the current thread.\"\"\"\n\n def submit(self, fn, *args, **kwargs):\n \"\"\"Submit a function to be executed.\"\"\"\n future = concurrent.futures.Future()\n try:\n future.set_result(fn(*args, **kwargs))\n except Exception as e:\n future.set_exception(e)\n return future\n\n\ndef make_concurrent_executor(\n jobs: Optional[int] = None, *, require_fork: bool = True\n) -> concurrent.futures.Executor:\n \"\"\"Create a concurrent executor. If require_fork is True, then the executor is sequential\n if the platform does not enable forking as the default start method. Effectively\n require_fork=True makes the executor sequential in the current process on Windows, macOS, and\n Linux from Python 3.14+ (which changes defaults)\"\"\"\n\n if (\n not ENABLE_PARALLELISM\n or (require_fork and multiprocessing.get_start_method() != \"fork\")\n or sys.version_info[:2] == (3, 6)\n ):\n return SequentialExecutor()\n\n from spack.subprocess_context import GlobalStateMarshaler\n\n jobs = jobs or spack.config.determine_number_of_jobs(parallel=True)\n marshaler = GlobalStateMarshaler()\n return concurrent.futures.ProcessPoolExecutor(jobs, initializer=marshaler.restore) # novermin\n" }, { "path": "lib/spack/spack/util/path.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Utilities for managing paths in Spack.\n\nTODO: this is really part of spack.config. Consolidate it.\n\"\"\"\n\nimport contextlib\nimport getpass\nimport os\nimport pathlib\nimport re\nimport subprocess\nimport sys\nimport tempfile\nfrom datetime import date\nfrom typing import Optional, Union\n\nimport spack.llnl.util.tty as tty\nimport spack.util.spack_yaml as syaml\nfrom spack.llnl.util.lang import memoized\n\n__all__ = [\"substitute_config_variables\", \"substitute_path_variables\", \"canonicalize_path\"]\n\n\ndef architecture():\n # break circular import\n import spack.platforms\n import spack.spec\n\n host_platform = spack.platforms.host()\n host_os = host_platform.default_operating_system()\n host_target = host_platform.default_target()\n\n return spack.spec.ArchSpec((str(host_platform), str(host_os), str(host_target)))\n\n\ndef get_user():\n # User pwd where available because it accounts for effective uids when using ksu and similar\n try:\n # user pwd for unix systems\n import pwd\n\n return pwd.getpwuid(os.geteuid()).pw_name\n except ImportError:\n # fallback on getpass\n return getpass.getuser()\n\n\n# return value for replacements with no match\nNOMATCH = object()\n\n\n# Substitutions to perform\ndef replacements():\n # break circular imports\n import spack\n import spack.environment as ev\n import spack.paths\n\n arch = architecture()\n\n return {\n \"spack\": lambda: spack.paths.prefix,\n \"user\": lambda: get_user(),\n \"tempdir\": lambda: tempfile.gettempdir(),\n \"user_cache_path\": lambda: spack.paths.user_cache_path,\n \"spack_instance_id\": lambda: spack.paths.spack_instance_id,\n \"architecture\": lambda: arch,\n \"arch\": lambda: arch,\n \"platform\": lambda: arch.platform,\n \"operating_system\": lambda: arch.os,\n \"os\": lambda: arch.os,\n \"target\": lambda: arch.target,\n \"target_family\": lambda: arch.target.family,\n \"date\": lambda: date.today().strftime(\"%Y-%m-%d\"),\n \"env\": lambda: ev.active_environment().path if ev.active_environment() else NOMATCH,\n \"spack_short_version\": lambda: spack.get_short_version(),\n }\n\n\n# This is intended to be longer than the part of the install path\n# spack generates from the root path we give it. Included in the\n# estimate:\n#\n# os-arch -> 30\n# compiler -> 30\n# package name -> 50 (longest is currently 47 characters)\n# version -> 20\n# hash -> 32\n# buffer -> 138\n# ---------------------\n# total -> 300\nSPACK_MAX_INSTALL_PATH_LENGTH = 300\n\n#: Padded paths comprise directories with this name (or some prefix of it). :\n#: It starts with two underscores to make it unlikely that prefix matches would\n#: include some other component of the installation path.\nSPACK_PATH_PADDING_CHARS = \"__spack_path_placeholder__\"\n\n#: Bytes equivalent of SPACK_PATH_PADDING_CHARS.\nSPACK_PATH_PADDING_BYTES = SPACK_PATH_PADDING_CHARS.encode(\"ascii\")\n\n#: Special padding char if the padded string would otherwise end with a path\n#: separator (since the path separator would otherwise get collapsed out,\n#: causing inconsistent padding).\nSPACK_PATH_PADDING_EXTRA_CHAR = \"_\"\n\n\ndef win_exe_ext():\n return r\"(?:\\.bat|\\.exe)\"\n\n\ndef sanitize_filename(filename: str) -> str:\n \"\"\"\n Replaces unsupported characters (for the host) in a filename with underscores.\n\n Criteria for legal files based on\n https://en.wikipedia.org/wiki/Filename#Comparison_of_filename_limitations\n\n Args:\n filename: string containing filename to be created on the host filesystem\n\n Return:\n filename that can be created on the host filesystem\n \"\"\"\n if sys.platform != \"win32\":\n # Only disallow null bytes and directory separators.\n return re.sub(\"[\\0/]\", \"_\", filename)\n\n # On Windows, things are more involved.\n # NOTE: this is incomplete, missing reserved names\n return re.sub(r'[\\x00-\\x1F\\x7F\"*/:<>?\\\\|]', \"_\", filename)\n\n\n@memoized\ndef get_system_path_max():\n # Choose a conservative default\n sys_max_path_length = 256\n if sys.platform == \"win32\":\n sys_max_path_length = 260\n else:\n try:\n path_max_proc = subprocess.Popen(\n [\"getconf\", \"PATH_MAX\", \"/\"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT\n )\n proc_output = str(path_max_proc.communicate()[0].decode())\n sys_max_path_length = int(proc_output)\n except (ValueError, subprocess.CalledProcessError, OSError):\n tty.msg(\n \"Unable to find system max path length, using: {0}\".format(sys_max_path_length)\n )\n\n return sys_max_path_length\n\n\ndef substitute_config_variables(path):\n \"\"\"Substitute placeholders into paths.\n\n Spack allows paths in configs to have some placeholders, as follows:\n\n - $env The active Spack environment.\n - $spack The Spack instance's prefix\n - $tempdir Default temporary directory returned by tempfile.gettempdir()\n - $user The current user's username\n - $user_cache_path The user cache directory (~/.spack, unless overridden)\n - $spack_instance_id Hash that distinguishes Spack instances on the filesystem\n - $architecture The spack architecture triple for the current system\n - $arch The spack architecture triple for the current system\n - $platform The spack platform for the current system\n - $os The OS of the current system\n - $operating_system The OS of the current system\n - $target The ISA target detected for the system\n - $target_family The family of the target detected for the system\n - $date The current date (YYYY-MM-DD)\n - $spack_short_version The spack short version\n\n These are substituted case-insensitively into the path, and users can\n use either ``$var`` or ``${var}`` syntax for the variables. $env is only\n replaced if there is an active environment, and should only be used in\n environment yaml files.\n \"\"\"\n _replacements = replacements()\n\n # Look up replacements\n def repl(match):\n m = match.group(0)\n key = m.strip(\"${}\").lower()\n repl = _replacements.get(key, lambda: m)()\n return m if repl is NOMATCH else str(repl)\n\n # Replace $var or ${var}.\n return re.sub(r\"(\\$\\w+\\b|\\$\\{\\w+\\})\", repl, path)\n\n\ndef substitute_path_variables(path):\n \"\"\"Substitute config vars, expand environment vars, expand user home.\"\"\"\n path = substitute_config_variables(path)\n path = os.path.expandvars(path)\n path = os.path.expanduser(path)\n return path\n\n\ndef _get_padding_string(length):\n spack_path_padding_size = len(SPACK_PATH_PADDING_CHARS)\n num_reps = int(length / (spack_path_padding_size + 1))\n extra_chars = length % (spack_path_padding_size + 1)\n reps_list = [SPACK_PATH_PADDING_CHARS for i in range(num_reps)]\n reps_list.append(SPACK_PATH_PADDING_CHARS[:extra_chars])\n padding = os.path.sep.join(reps_list)\n if padding.endswith(os.path.sep):\n padding = padding[: len(padding) - 1] + SPACK_PATH_PADDING_EXTRA_CHAR\n return padding\n\n\ndef add_padding(path, length):\n \"\"\"Add padding subdirectories to path until total is length characters\n\n Returns the padded path. If path is length - 1 or more characters long,\n returns path. If path is length - 1 characters, warns that it is not\n padding to length\n\n Assumes path does not have a trailing path separator\"\"\"\n padding_length = length - len(path)\n if padding_length == 1:\n # The only 1 character addition we can make to a path is `/`\n # Spack internally runs normpath, so `foo/` will be reduced to `foo`\n # Even if we removed this behavior from Spack, the user could normalize\n # the path, removing the additional `/`.\n # Because we can't expect one character of padding to show up in the\n # resulting binaries, we warn the user and do not pad by a single char\n tty.warn(\"Cannot pad path by exactly one character.\")\n if padding_length <= 0:\n return path\n\n # we subtract 1 from the padding_length to account for the path separator\n # coming from os.path.join below\n padding = _get_padding_string(padding_length - 1)\n\n return os.path.join(path, padding)\n\n\ndef canonicalize_path(path: str, default_wd: Optional[str] = None) -> str:\n \"\"\"Same as substitute_path_variables, but also take absolute path.\n\n If the string is a yaml object with file annotations, make absolute paths\n relative to that file's directory.\n Otherwise, use ``default_wd`` if specified, otherwise ``os.getcwd()``\n\n Arguments:\n path: path being converted as needed\n default_wd: optional working directory/root for non-yaml string paths\n\n Returns: An absolute path or non-file URL with path variable substitution\n \"\"\"\n import urllib.parse\n import urllib.request\n\n # Get file in which path was written in case we need to make it absolute\n # relative to that path.\n filename = None\n if isinstance(path, syaml.syaml_str):\n filename = os.path.dirname(path._start_mark.name) # type: ignore[attr-defined]\n assert path._start_mark.name == path._end_mark.name # type: ignore[attr-defined]\n\n path = substitute_path_variables(path)\n\n # Ensure properly process a Windows path\n win_path = pathlib.PureWindowsPath(path)\n if win_path.drive:\n # Assume only absolute paths are supported with a Windows drive\n # (though DOS does allow drive-relative paths).\n return os.path.normpath(str(win_path))\n\n # Now process linux-like paths and remote URLs\n url = urllib.parse.urlparse(path)\n url_path = urllib.request.url2pathname(url.path)\n if url.scheme:\n if url.scheme != \"file\":\n # Have a remote URL so simply return it with substitutions\n return path\n\n # Drop the URL scheme from the local path\n path = url_path\n\n if os.path.isabs(path):\n return os.path.normpath(path)\n\n # Have a relative path so prepend the appropriate dir to make it absolute\n if filename:\n # Prepend the directory of the syaml path\n return os.path.normpath(os.path.join(filename, path))\n\n # Prepend the default, if provided, or current working directory.\n base = default_wd or os.getcwd()\n tty.debug(f\"Using working directory {base} as base for abspath\")\n return os.path.normpath(os.path.join(base, path))\n\n\ndef longest_prefix_re(string, capture=True):\n \"\"\"Return a regular expression that matches a the longest possible prefix of string.\n\n i.e., if the input string is ``the_quick_brown_fox``, then::\n\n m = re.compile(longest_prefix('the_quick_brown_fox'))\n m.match('the_').group(1) == 'the_'\n m.match('the_quick').group(1) == 'the_quick'\n m.match('the_quick_brown_fox').group(1) == 'the_quick_brown_fox'\n m.match('the_xquick_brown_fox').group(1) == 'the_'\n m.match('the_quickx_brown_fox').group(1) == 'the_quick'\n\n \"\"\"\n if len(string) < 2:\n return string\n\n return \"(%s%s%s?)\" % (\n \"\" if capture else \"?:\",\n string[0],\n longest_prefix_re(string[1:], capture=False),\n )\n\n\ndef _build_padding_re(as_bytes: bool = False):\n \"\"\"Build and return a compiled regex for filtering path padding placeholders.\"\"\"\n pad = re.escape(SPACK_PATH_PADDING_CHARS)\n extra = SPACK_PATH_PADDING_EXTRA_CHAR\n longest_prefix = longest_prefix_re(SPACK_PATH_PADDING_CHARS, capture=False)\n\n regex = (\n r\"((?:/[^/\\s]*)*?)\" # zero or more leading non-whitespace path components\n r\"(?:/{pad})+\" # the padding string repeated one or more times\n # trailing prefix of padding as path component\n r\"(?:/{longest_prefix}|/{longest_prefix}{extra})?(?=/)\"\n )\n regex = regex.replace(\"/\", re.escape(os.sep))\n regex = regex.format(pad=pad, extra=extra, longest_prefix=longest_prefix)\n\n if as_bytes:\n return re.compile(regex.encode(\"ascii\"))\n else:\n return re.compile(regex)\n\n\nclass _PaddingFilter:\n \"\"\"Callable that filters path-padding placeholders from a string or bytes buffer.\n\n This turns paths like this:\n\n /foo/bar/__spack_path_placeholder__/__spack_path_placeholder__/...\n\n Into paths like this:\n\n /foo/bar/[padded-to-512-chars]/...\n\n Where ``padded-to-512-chars`` indicates that the prefix was padded with\n placeholders until it hit 512 characters. The actual value of this number\n depends on what the ``install_tree``'s ``padded_length`` is configured to.\n\n For a path to match and be filtered, the placeholder must appear in its\n entirety at least one time. e.g., \"/spack/\" would not be filtered, but\n \"/__spack_path_placeholder__/spack/\" would be.\n\n Note that only the first padded path in the string is filtered.\n \"\"\"\n\n __slots__ = (\"_re\", \"_needle\", \"_fmt\")\n\n def __init__(self, as_bytes: bool = False) -> None:\n self._re = _build_padding_re(as_bytes=as_bytes)\n if as_bytes:\n self._needle: Union[str, bytes] = SPACK_PATH_PADDING_BYTES\n self._fmt: Union[str, bytes] = b\"%b\" + os.sep.encode(\"ascii\") + b\"[padded-to-%d-chars]\"\n else:\n self._needle = SPACK_PATH_PADDING_CHARS\n self._fmt = \"%s\" + os.sep + \"[padded-to-%d-chars]\"\n\n def _replace(self, match):\n return self._fmt % (match.group(1), len(match.group(0)))\n\n def __call__(self, data):\n if self._needle not in data:\n return data\n return self._re.sub(self._replace, data)\n\n\n#: Callable that filters path-padding placeholders from strings\npadding_filter = _PaddingFilter(as_bytes=False)\n\n#: Callable that filters path-padding placeholders from bytes buffers\npadding_filter_bytes = _PaddingFilter(as_bytes=True)\n\n\n@contextlib.contextmanager\ndef filter_padding():\n \"\"\"Context manager to safely disable path padding in all Spack output.\n\n This is needed because Spack's debug output gets extremely long when we use a\n long padded installation path.\n \"\"\"\n # circular import\n import spack.config\n\n padding = spack.config.get(\"config:install_tree:padded_length\", None)\n if padding:\n # filter out all padding from the install command output\n with tty.output_filter(padding_filter):\n yield\n else:\n yield # no-op: don't filter unless padding is actually enabled\n\n\ndef debug_padded_filter(string, level=1):\n \"\"\"\n Return string, path padding filtered if debug level and not windows\n\n Args:\n string (str): string containing path\n level (int): maximum debug level value for filtering (e.g., 1\n means filter path padding if the current debug level is 0 or 1\n but return the original string if it is 2 or more)\n\n Returns (str): filtered string if current debug level does not exceed\n level and not windows; otherwise, unfiltered string\n \"\"\"\n if sys.platform == \"win32\":\n return string\n\n return padding_filter(string) if tty.debug_level() <= level else string\n" }, { "path": "lib/spack/spack/util/pattern.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\nclass Bunch:\n \"\"\"Carries a bunch of named attributes (from Alex Martelli bunch)\"\"\"\n\n def __init__(self, **kwargs):\n self.__dict__.update(kwargs)\n\n\nclass Args(Bunch):\n \"\"\"Subclass of Bunch to write argparse args more naturally.\"\"\"\n\n def __init__(self, *flags, **kwargs):\n super().__init__(flags=tuple(flags), kwargs=kwargs)\n" }, { "path": "lib/spack/spack/util/prefix.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"\nThis file contains utilities for managing the installation prefix of a package.\n\"\"\"\n\nimport os\nfrom typing import Dict\n\n\nclass Prefix(str):\n \"\"\"This class represents an installation prefix, but provides useful attributes for referring\n to directories inside the prefix.\n\n Attributes of this object are created on the fly when you request them, so any of the following\n are valid:\n\n >>> prefix = Prefix(\"/usr\")\n >>> prefix.bin\n /usr/bin\n >>> prefix.lib64\n /usr/lib64\n >>> prefix.share.man\n /usr/share/man\n >>> prefix.foo.bar.baz\n /usr/foo/bar/baz\n >>> prefix.join(\"dashed-directory\").bin64\n /usr/dashed-directory/bin64\n\n Prefix objects behave identically to strings. In fact, they subclass :class:`str`, so operators\n like ``+`` are legal::\n\n print(\"foobar \" + prefix)\n\n This prints ``foobar /usr``. All of this is meant to make custom installs easy.\n \"\"\"\n\n def __getattr__(self, name: str) -> \"Prefix\":\n \"\"\"Concatenate a string to a prefix.\n\n Useful for strings that are valid variable names.\n\n Args:\n name: the string to append to the prefix\n\n Returns:\n the newly created installation prefix\n \"\"\"\n return Prefix(os.path.join(self, name))\n\n def join(self, string: str) -> \"Prefix\": # type: ignore[override]\n \"\"\"Concatenate a string to a prefix.\n\n Useful for strings that are not valid variable names. This includes strings containing\n characters like ``-`` and ``.``.\n\n Args:\n string: the string to append to the prefix\n\n Returns:\n the newly created installation prefix\n \"\"\"\n return Prefix(os.path.join(self, string))\n\n def __getstate__(self) -> Dict[str, str]:\n \"\"\"Control how object is pickled.\n\n Returns:\n current state of the object\n \"\"\"\n return self.__dict__\n\n def __setstate__(self, state: Dict[str, str]) -> None:\n \"\"\"Control how object is unpickled.\n\n Args:\n new state of the object\n \"\"\"\n self.__dict__.update(state)\n" }, { "path": "lib/spack/spack/util/remote_file_cache.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport hashlib\nimport os.path\nimport pathlib\nimport shutil\nimport tempfile\nimport urllib.parse\nimport urllib.request\nfrom typing import Optional\n\nimport spack.llnl.util.tty as tty\nimport spack.util.crypto\nfrom spack.llnl.util.filesystem import copy, join_path, mkdirp\nfrom spack.util.path import canonicalize_path\nfrom spack.util.url import validate_scheme\n\n\ndef raw_github_gitlab_url(url: str) -> str:\n \"\"\"Transform a github URL to the raw form to avoid undesirable html.\n\n Args:\n url: url to be converted to raw form\n\n Returns:\n Raw github/gitlab url or the original url\n \"\"\"\n # Note we rely on GitHub to redirect the 'raw' URL returned here to the\n # actual URL under https://raw.githubusercontent.com/ with '/blob'\n # removed and or, '/blame' if needed.\n if \"github\" in url or \"gitlab\" in url:\n return url.replace(\"/blob/\", \"/raw/\")\n\n return url\n\n\ndef fetch_remote_text_file(url: str, dest_dir: str) -> str:\n \"\"\"Retrieve the text file from the url into the destination directory.\n\n Arguments:\n url: URL for the remote text file\n dest_dir: destination directory in which to stage the file locally\n\n Returns:\n Path to the fetched file\n\n Raises:\n ValueError: if there are missing required arguments\n \"\"\"\n from spack.util.web import fetch_url_text # circular import\n\n if not url:\n raise ValueError(\"Cannot retrieve the remote file without the URL\")\n\n raw_url = raw_github_gitlab_url(url)\n tty.debug(f\"Fetching file from {raw_url} into {dest_dir}\")\n\n return fetch_url_text(raw_url, dest_dir=dest_dir)\n\n\ndef local_path(raw_path: str, sha256: str, dest: Optional[str] = None) -> str:\n \"\"\"Determine the actual path and, if remote, stage its contents locally.\n\n Args:\n raw_path: raw path with possible variables needing substitution\n sha256: the expected sha256 if the file is remote\n dest: destination path\n\n Returns: resolved, normalized local path\n\n Raises:\n ValueError: missing or mismatched arguments, unsupported URL scheme\n \"\"\"\n if not raw_path:\n raise ValueError(\"path argument is required to cache remote files\")\n\n file_schemes = [\"\", \"file\"]\n\n # Allow paths (and URLs) to contain spack config/environment variables,\n # etc.\n path = canonicalize_path(raw_path, dest)\n\n # Save off the Windows drive of the canonicalized path (since now absolute)\n # to ensure recognized by URL parsing as a valid file \"scheme\".\n win_path = pathlib.PureWindowsPath(path)\n if win_path.drive:\n file_schemes.append(win_path.drive.lower().strip(\":\"))\n\n url = urllib.parse.urlparse(path)\n\n # Path isn't remote so return normalized, absolute path with substitutions.\n if url.scheme in file_schemes:\n return os.path.normpath(path)\n\n # If scheme is not valid, path is not a supported url.\n if validate_scheme(url.scheme):\n # Fetch files from supported URL schemes.\n if url.scheme in (\"http\", \"https\", \"ftp\"):\n if not dest:\n raise ValueError(\"Requires the destination argument to cache remote files\")\n assert os.path.isabs(dest), (\n f\"Remote file destination '{dest}' must be an absolute path\"\n )\n\n # Stage the remote configuration file\n tmpdir = tempfile.mkdtemp()\n try:\n staged_path = fetch_remote_text_file(path, tmpdir)\n\n # Ensure the sha256 is expected.\n checksum = spack.util.crypto.checksum(hashlib.sha256, staged_path)\n if sha256 and checksum != sha256:\n raise ValueError(\n f\"Actual sha256 ('{checksum}') does not match expected ('{sha256}')\"\n )\n\n # Help the user by reporting the required checksum.\n if not sha256:\n raise ValueError(f\"Requires sha256 ('{checksum}') to cache remote files.\")\n\n # Copy the file to the destination directory\n dest_dir = join_path(dest, checksum)\n if not os.path.exists(dest_dir):\n mkdirp(dest_dir)\n\n cache_path = join_path(dest_dir, os.path.basename(staged_path))\n copy(staged_path, cache_path)\n tty.debug(f\"Cached {raw_path} in {cache_path}\")\n\n # Stash the associated URL to aid with debugging\n with open(join_path(dest_dir, \"source_url.txt\"), \"w\", encoding=\"utf-8\") as f:\n f.write(f\"{raw_path}\\n\")\n\n return cache_path\n\n except ValueError as err:\n tty.warn(f\"Unable to cache {raw_path}: {str(err)}\")\n raise\n\n finally:\n shutil.rmtree(tmpdir)\n\n raise ValueError(f\"Unsupported URL scheme ({url.scheme}) in {raw_path}\")\n\n else:\n raise ValueError(f\"Invalid URL scheme ({url.scheme}) in {raw_path}\")\n" }, { "path": "lib/spack/spack/util/s3.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\nimport os\nimport urllib.error\nimport urllib.parse\nimport urllib.request\nimport urllib.response\nfrom io import BufferedReader, BytesIO, IOBase\nfrom typing import Any, Dict, Tuple\n\nimport spack.config\n\n#: Map (mirror name, method) tuples to s3 client instances.\ns3_client_cache: Dict[Tuple[str, str], Any] = dict()\n\n\ndef get_s3_session(url, method=\"fetch\"):\n # import boto and friends as late as possible. We don't want to require boto as a\n # dependency unless the user actually wants to access S3 mirrors.\n from boto3 import Session\n from botocore import UNSIGNED\n from botocore.client import Config\n from botocore.exceptions import ClientError\n\n # Circular dependency\n from spack.mirrors.mirror import MirrorCollection\n\n global s3_client_cache\n\n # Parse the URL if not already done.\n if not isinstance(url, urllib.parse.ParseResult):\n url = urllib.parse.urlparse(url)\n url_str = url.geturl()\n\n def get_mirror_url(mirror):\n return mirror.fetch_url if method == \"fetch\" else mirror.push_url\n\n # Get all configured mirrors that could match.\n all_mirrors = MirrorCollection()\n mirrors = [\n (name, mirror)\n for name, mirror in all_mirrors.items()\n if url_str.startswith(get_mirror_url(mirror))\n ]\n\n if not mirrors:\n name, mirror = None, {}\n else:\n # In case we have more than one mirror, we pick the longest matching url.\n # The heuristic being that it's more specific, and you can have different\n # credentials for a sub-bucket (if that is a thing).\n name, mirror = max(\n mirrors, key=lambda name_and_mirror: len(get_mirror_url(name_and_mirror[1]))\n )\n\n key = (name, method)\n\n # Did we already create a client for this? Then return it.\n if key in s3_client_cache:\n return s3_client_cache[key]\n\n # Otherwise, create it.\n s3_connection, s3_client_args = get_mirror_s3_connection_info(mirror, method)\n\n session = Session(**s3_connection)\n # if no access credentials provided above, then access anonymously\n if not session.get_credentials():\n s3_client_args[\"config\"] = Config(signature_version=UNSIGNED)\n\n client = session.client(\"s3\", **s3_client_args)\n client.ClientError = ClientError\n\n # Cache the client.\n s3_client_cache[key] = client\n return client\n\n\ndef _parse_s3_endpoint_url(endpoint_url):\n if not urllib.parse.urlparse(endpoint_url, scheme=\"\").scheme:\n endpoint_url = \"://\".join((\"https\", endpoint_url))\n\n return endpoint_url\n\n\ndef get_mirror_s3_connection_info(mirror, method):\n \"\"\"Create s3 config for session/client from a Mirror instance (or just set defaults\n when no mirror is given.)\"\"\"\n from spack.mirrors.mirror import Mirror\n\n s3_connection = {}\n s3_client_args = {\"use_ssl\": spack.config.get(\"config:verify_ssl\")}\n\n # access token\n if isinstance(mirror, Mirror):\n credentials = mirror.get_credentials(method)\n if credentials:\n if \"access_token\" in credentials:\n s3_connection[\"aws_session_token\"] = credentials[\"access_token\"]\n\n if \"access_pair\" in credentials:\n s3_connection[\"aws_access_key_id\"] = credentials[\"access_pair\"][0]\n s3_connection[\"aws_secret_access_key\"] = credentials[\"access_pair\"][1]\n\n if \"profile\" in credentials:\n s3_connection[\"profile_name\"] = credentials[\"profile\"]\n\n # endpoint url\n endpoint_url = mirror.get_endpoint_url(method) or os.environ.get(\"S3_ENDPOINT_URL\")\n else:\n endpoint_url = os.environ.get(\"S3_ENDPOINT_URL\")\n\n if endpoint_url:\n s3_client_args[\"endpoint_url\"] = _parse_s3_endpoint_url(endpoint_url)\n\n return s3_connection, s3_client_args\n\n\n# NOTE(opadron): Workaround issue in boto where its StreamingBody\n# implementation is missing several APIs expected from IOBase. These missing\n# APIs prevent the streams returned by boto from being passed as-are along to\n# urllib.\n#\n# https://github.com/boto/botocore/issues/879\n# https://github.com/python/cpython/pull/3249\nclass WrapStream(BufferedReader):\n def __init__(self, raw):\n # In botocore >=1.23.47, StreamingBody inherits from IOBase, so we\n # only add missing attributes in older versions.\n # https://github.com/boto/botocore/commit/a624815eabac50442ed7404f3c4f2664cd0aa784\n if not isinstance(raw, IOBase):\n raw.readable = lambda: True\n raw.writable = lambda: False\n raw.seekable = lambda: False\n raw.closed = False\n raw.flush = lambda: None\n super().__init__(raw)\n\n def detach(self):\n self.raw = None\n\n def read(self, *args, **kwargs):\n return self.raw.read(*args, **kwargs)\n\n def __getattr__(self, key):\n return getattr(self.raw, key)\n\n\ndef _s3_open(url, method=\"GET\"):\n parsed = urllib.parse.urlparse(url)\n s3 = get_s3_session(url, method=\"fetch\")\n\n bucket = parsed.netloc\n key = parsed.path\n\n if key.startswith(\"/\"):\n key = key[1:]\n\n if method not in (\"GET\", \"HEAD\"):\n raise urllib.error.URLError(\n \"Only GET and HEAD verbs are currently supported for the s3:// scheme\"\n )\n\n try:\n if method == \"GET\":\n obj = s3.get_object(Bucket=bucket, Key=key)\n # NOTE(opadron): Apply workaround here (see above)\n stream = WrapStream(obj[\"Body\"])\n elif method == \"HEAD\":\n obj = s3.head_object(Bucket=bucket, Key=key)\n stream = BytesIO()\n except s3.ClientError as e:\n raise urllib.error.URLError(e) from e\n\n headers = obj[\"ResponseMetadata\"][\"HTTPHeaders\"]\n\n return url, headers, stream\n\n\nclass UrllibS3Handler(urllib.request.BaseHandler):\n def s3_open(self, req):\n orig_url = req.get_full_url()\n url, headers, stream = _s3_open(orig_url, method=req.get_method())\n return urllib.response.addinfourl(stream, headers, url)\n" }, { "path": "lib/spack/spack/util/socket.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Optimized Spack implementations of methods from socket module.\"\"\"\n\nimport socket\n\nimport spack.llnl.util.lang\n\n\n@spack.llnl.util.lang.memoized\ndef _gethostname():\n \"\"\"Memoized version of `getfqdn()`.\n\n If we call `getfqdn()` too many times, DNS can be very slow. We only need to call it\n one time per process, so we cache it here.\n\n \"\"\"\n return socket.gethostname()\n" }, { "path": "lib/spack/spack/util/spack_json.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Simple wrapper around JSON to guarantee consistent use of load/dump.\"\"\"\n\nimport json\nfrom typing import Any, Dict, Optional\n\nimport spack.error\n\n__all__ = [\"load\", \"dump\", \"SpackJSONError\"]\n\n_json_dump_args = {\"indent\": None, \"separators\": (\",\", \":\")}\n_pretty_dump_args = {\"indent\": \" \", \"separators\": (\", \", \": \")}\n\n\ndef load(stream: Any) -> Dict:\n \"\"\"Spack JSON needs to be ordered to support specs.\"\"\"\n if isinstance(stream, str):\n return json.loads(stream)\n return json.load(stream)\n\n\ndef dump(data: Dict, stream: Optional[Any] = None, pretty: bool = False) -> Optional[str]:\n \"\"\"Dump JSON with a reasonable amount of indentation and separation.\"\"\"\n dump_args = _pretty_dump_args if pretty else _json_dump_args\n if stream is None:\n return json.dumps(data, **dump_args) # type: ignore[arg-type]\n json.dump(data, stream, **dump_args) # type: ignore[arg-type]\n return None\n\n\nclass SpackJSONError(spack.error.SpackError):\n \"\"\"Raised when there are issues with JSON parsing.\"\"\"\n\n def __init__(self, msg: str, json_error: BaseException):\n super().__init__(msg, str(json_error))\n" }, { "path": "lib/spack/spack/util/spack_yaml.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Enhanced YAML parsing for Spack.\n\n- ``load()`` preserves YAML Marks on returned objects -- this allows\n us to access file and line information later.\n\n- ``Our load methods use ``OrderedDict`` class instead of YAML's\n default unordered dict.\n\n\"\"\"\n\nimport ctypes\nimport enum\nimport functools\nimport io\nimport re\nfrom typing import IO, Any, Callable, Dict, List, Optional, Union\n\nfrom spack.vendor.ruamel.yaml import YAML, comments, constructor, emitter, error, representer\n\nimport spack.error\nfrom spack.llnl.util.tty.color import cextra, clen, colorize\n\n# Only export load and dump\n__all__ = [\"load\", \"dump\", \"SpackYAMLError\"]\n\n\n# Make new classes so we can add custom attributes.\nclass syaml_dict(dict):\n pass\n\n\nclass syaml_list(list):\n pass\n\n\nclass syaml_str(str):\n pass\n\n\nclass syaml_int(int):\n pass\n\n\n#: mapping from syaml type -> primitive type\nsyaml_types = {syaml_str: str, syaml_int: int, syaml_dict: dict, syaml_list: list}\n\n\nmarkable_types = set(syaml_types) | {comments.CommentedSeq, comments.CommentedMap}\n\n\ndef syaml_type(obj):\n \"\"\"Get the corresponding syaml wrapper type for a primitive type.\n\n Return:\n (object): syaml-typed copy of object, or the obj if no wrapper\n \"\"\"\n for syaml_t, t in syaml_types.items():\n if type(obj) is not bool and isinstance(obj, t):\n return syaml_t(obj) if type(obj) is not syaml_t else obj\n return obj\n\n\nclass DictWithLineInfo(dict):\n \"\"\"A dictionary that preserves YAML line information.\"\"\"\n\n __slots__ = (\"line_info\",)\n\n def __init__(self, *args, line_info: str = \"\", **kwargs):\n super().__init__(*args, **kwargs)\n self.line_info = line_info\n\n\ndef _represent_dict_with_line_info(dumper, data):\n return dumper.represent_dict(data)\n\n\ndef deepcopy_as_builtin(obj: Any, *, line_info: bool = False) -> Any:\n \"\"\"Deep copies a YAML object as built-in types (dict, list, str, int, ...).\n\n Args:\n obj: object to be copied\n line_info: if ``True``, add line information to the copied object\n \"\"\"\n if isinstance(obj, str):\n return str(obj)\n elif isinstance(obj, dict):\n result = DictWithLineInfo()\n result.update(\n {\n deepcopy_as_builtin(k): deepcopy_as_builtin(v, line_info=line_info)\n for k, v in obj.items()\n }\n )\n if line_info:\n result.line_info = _line_info(obj)\n return result\n elif isinstance(obj, list):\n return [deepcopy_as_builtin(x, line_info=line_info) for x in obj]\n elif isinstance(obj, bool):\n return bool(obj)\n elif isinstance(obj, int):\n return int(obj)\n elif isinstance(obj, float):\n return float(obj)\n elif obj is None:\n return obj\n raise ValueError(f\"cannot convert {type(obj)} to built-in type\")\n\n\ndef markable(obj):\n \"\"\"Whether an object can be marked.\"\"\"\n return type(obj) in markable_types\n\n\ndef mark(obj, node):\n \"\"\"Add start and end markers to an object.\"\"\"\n if hasattr(node, \"start_mark\"):\n obj._start_mark = node.start_mark\n elif hasattr(node, \"_start_mark\"):\n obj._start_mark = node._start_mark\n if hasattr(node, \"end_mark\"):\n obj._end_mark = node.end_mark\n elif hasattr(node, \"_end_mark\"):\n obj._end_mark = node._end_mark\n\n\ndef marked(obj):\n \"\"\"Whether an object has been marked by spack_yaml.\"\"\"\n return (\n hasattr(obj, \"_start_mark\")\n and obj._start_mark\n or hasattr(obj, \"_end_mark\")\n and obj._end_mark\n )\n\n\nclass OrderedLineConstructor(constructor.RoundTripConstructor):\n \"\"\"YAML loader specifically intended for reading Spack configuration\n files. It preserves order and line numbers. It also has special-purpose\n logic for handling dictionary keys that indicate a Spack config\n override: namely any key that contains an \"extra\" ':' character.\n\n Mappings read in by this loader behave like an ordered dict.\n Sequences, mappings, and strings also have new attributes,\n ``_start_mark`` and ``_end_mark``, that preserve YAML line\n information in the output data.\n\n \"\"\"\n\n #\n # Override construct_yaml_* so that we can apply _start_mark/_end_mark to\n # them. The superclass returns CommentedMap/CommentedSeq objects that we\n # can add attributes to (and we depend on their behavior to preserve\n # comments).\n #\n # The inherited sequence/dictionary constructors return empty instances\n # and fill in with mappings later. We preserve this behavior.\n #\n\n def construct_yaml_str(self, node):\n value = super().construct_yaml_str(node)\n # There is no specific marker to indicate that we are parsing a key,\n # so this assumes we are talking about a Spack config override key if\n # it ends with a ':' and does not contain a '@' (which can appear\n # in config values that refer to Spack specs)\n if value and value.endswith(\":\") and \"@\" not in value:\n value = syaml_str(value[:-1])\n value.override = True\n else:\n value = syaml_str(value)\n mark(value, node)\n return value\n\n def construct_yaml_seq(self, node):\n gen = super().construct_yaml_seq(node)\n data = next(gen)\n if markable(data):\n mark(data, node)\n yield data\n for x in gen:\n pass\n\n def construct_yaml_map(self, node):\n gen = super().construct_yaml_map(node)\n data = next(gen)\n if markable(data):\n mark(data, node)\n yield data\n for x in gen:\n pass\n\n\n# register above new constructors\nOrderedLineConstructor.add_constructor(\n \"tag:yaml.org,2002:map\", OrderedLineConstructor.construct_yaml_map\n)\nOrderedLineConstructor.add_constructor(\n \"tag:yaml.org,2002:seq\", OrderedLineConstructor.construct_yaml_seq\n)\nOrderedLineConstructor.add_constructor(\n \"tag:yaml.org,2002:str\", OrderedLineConstructor.construct_yaml_str\n)\n\n\nclass OrderedLineRepresenter(representer.RoundTripRepresenter):\n \"\"\"Representer that preserves ordering and formats ``syaml_*`` objects.\n\n This representer preserves insertion ordering ``syaml_dict`` objects\n when they're written out. It also has some custom formatters\n for ``syaml_*`` objects so that they are formatted like their\n regular Python equivalents, instead of ugly YAML pyobjects.\n \"\"\"\n\n def ignore_aliases(self, _data):\n \"\"\"Make the dumper NEVER print YAML aliases.\"\"\"\n return True\n\n def represent_data(self, data):\n result = super().represent_data(data)\n if data is None:\n result.value = syaml_str(\"null\")\n return result\n\n def represent_str(self, data):\n if hasattr(data, \"override\") and data.override:\n data = data + \":\"\n return super().represent_str(data)\n\n\nclass SafeRepresenter(representer.RoundTripRepresenter):\n def ignore_aliases(self, _data):\n \"\"\"Make the dumper NEVER print YAML aliases.\"\"\"\n return True\n\n\n# Make our special objects look like normal YAML ones.\nrepresenter.RoundTripRepresenter.add_representer(\n syaml_dict, representer.RoundTripRepresenter.represent_dict\n)\nrepresenter.RoundTripRepresenter.add_representer(\n syaml_list, representer.RoundTripRepresenter.represent_list\n)\nrepresenter.RoundTripRepresenter.add_representer(\n syaml_int, representer.RoundTripRepresenter.represent_int\n)\nrepresenter.RoundTripRepresenter.add_representer(\n syaml_str, representer.RoundTripRepresenter.represent_str\n)\nOrderedLineRepresenter.add_representer(syaml_str, OrderedLineRepresenter.represent_str)\n\n\n#: Max integer helps avoid passing too large a value to cyaml.\nmaxint = 2 ** (ctypes.sizeof(ctypes.c_int) * 8 - 1) - 1\n\n\ndef return_string_when_no_stream(func):\n @functools.wraps(func)\n def wrapper(data, stream=None, **kwargs):\n if stream:\n return func(data, stream=stream, **kwargs)\n stream = io.StringIO()\n func(data, stream=stream, **kwargs)\n return stream.getvalue()\n\n return wrapper\n\n\n@return_string_when_no_stream\ndef dump(data, stream=None, default_flow_style=False):\n handler = ConfigYAML(yaml_type=YAMLType.GENERIC_YAML)\n handler.yaml.default_flow_style = default_flow_style\n handler.yaml.width = maxint\n return handler.dump(data, stream=stream)\n\n\ndef _line_info(obj):\n \"\"\"Format a mark as : information.\"\"\"\n m = get_mark_from_yaml_data(obj)\n if m is None:\n return \"\"\n if m.line:\n return f\"{m.name}:{m.line:d}\"\n return m.name\n\n\n#: Global for interactions between LineAnnotationDumper and dump_annotated().\n#: This is nasty but YAML doesn't give us many ways to pass arguments --\n#: yaml.dump() takes a class (not an instance) and instantiates the dumper\n#: itself, so we can't just pass an instance\n_ANNOTATIONS: List[str] = []\n\n\nclass LineAnnotationRepresenter(OrderedLineRepresenter):\n \"\"\"Representer that generates per-line annotations.\n\n Annotations are stored in the ``_annotations`` global. After one\n dump pass, the strings in ``_annotations`` will correspond one-to-one\n with the lines output by the dumper.\n\n LineAnnotationDumper records blame information after each line is\n generated. As each line is parsed, it saves file/line info for each\n object printed. At the end of each line, it creates an annotation\n based on the saved mark and stores it in ``_annotations``.\n\n For an example of how to use this, see ``dump_annotated()``, which\n writes to a ``StringIO`` then joins the lines from that with\n annotations.\n \"\"\"\n\n def represent_data(self, data):\n \"\"\"Force syaml_str to be passed through with marks.\"\"\"\n result = super().represent_data(data)\n if data is None:\n result.value = syaml_str(\"null\")\n elif isinstance(result.value, str):\n result.value = syaml_str(data)\n if markable(result.value):\n mark(result.value, data)\n return result\n\n\nclass LineAnnotationEmitter(emitter.Emitter):\n saved = None\n\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n del _ANNOTATIONS[:]\n self.colors = \"KgrbmcyGRBMCY\"\n self.filename_colors = {}\n\n def process_scalar(self):\n super().process_scalar()\n if marked(self.event.value):\n self.saved = self.event.value\n\n def write_line_break(self, data=None):\n super().write_line_break(data)\n if self.saved is None:\n _ANNOTATIONS.append(colorize(\"@K{---}\"))\n return\n\n # append annotations at the end of each line\n if self.saved:\n mark = self.saved._start_mark\n\n color = self.filename_colors.get(mark.name)\n if not color:\n ncolors = len(self.colors)\n color = self.colors[len(self.filename_colors) % ncolors]\n self.filename_colors[mark.name] = color\n\n fmt = \"@%s{%%s}\" % color\n ann = fmt % mark.name\n if mark.line is not None:\n ann += \":@c{%s}\" % (mark.line + 1)\n _ANNOTATIONS.append(colorize(ann))\n else:\n _ANNOTATIONS.append(\"\")\n\n def write_comment(self, comment, pre=False):\n pass\n\n\nclass YAMLType(enum.Enum):\n \"\"\"YAML configurations handled by Spack\"\"\"\n\n #: Generic YAML configuration\n GENERIC_YAML = enum.auto()\n #: A Spack config file with overrides\n SPACK_CONFIG_FILE = enum.auto()\n #: A Spack config file with line annotations\n ANNOTATED_SPACK_CONFIG_FILE = enum.auto()\n\n\nclass ConfigYAML:\n \"\"\"Handles the loading and dumping of Spack's YAML files.\"\"\"\n\n def __init__(self, yaml_type: YAMLType) -> None:\n self.yaml = YAML(typ=\"rt\", pure=True)\n if yaml_type == YAMLType.GENERIC_YAML:\n self.yaml.Representer = SafeRepresenter\n elif yaml_type == YAMLType.ANNOTATED_SPACK_CONFIG_FILE:\n self.yaml.Representer = LineAnnotationRepresenter\n self.yaml.Emitter = LineAnnotationEmitter\n self.yaml.Constructor = OrderedLineConstructor\n else:\n self.yaml.Representer = OrderedLineRepresenter\n self.yaml.Constructor = OrderedLineConstructor\n self.yaml.Representer.add_representer(DictWithLineInfo, _represent_dict_with_line_info)\n\n def load(self, stream: IO):\n \"\"\"Loads the YAML data from a stream and returns it.\n\n Args:\n stream: stream to load from.\n\n Raises:\n SpackYAMLError: if anything goes wrong while loading\n \"\"\"\n try:\n return self.yaml.load(stream)\n\n except error.MarkedYAMLError as e:\n msg = \"error parsing YAML\"\n error_mark = e.context_mark if e.context_mark else e.problem_mark\n if error_mark:\n line, column = error_mark.line, error_mark.column\n filename = error_mark.name\n msg += f\": near {filename}, {str(line)}, {str(column)}\"\n else:\n filename = stream.name\n msg += f\": {stream.name}\"\n msg += f\": {e.problem}\"\n\n raise SpackYAMLError(msg, e, filename) from e\n\n except Exception as e:\n msg = \"cannot load Spack YAML configuration\"\n filename = stream.name\n raise SpackYAMLError(msg, e, filename) from e\n\n def dump(self, data, stream: Optional[IO] = None, *, transform=None) -> None:\n \"\"\"Dumps the YAML data to a stream.\n\n Args:\n data: data to be dumped\n stream: stream to dump the data into.\n\n Raises:\n SpackYAMLError: if anything goes wrong while dumping\n \"\"\"\n try:\n return self.yaml.dump(data, stream=stream, transform=transform)\n except Exception as e:\n msg = \"cannot dump Spack YAML configuration\"\n filename = stream.name if stream else None\n raise SpackYAMLError(msg, str(e), filename) from e\n\n def as_string(self, data) -> str:\n \"\"\"Returns a string representing the YAML data passed as input.\"\"\"\n result = io.StringIO()\n self.dump(data, stream=result)\n return result.getvalue()\n\n\ndef load_config(str_or_file):\n \"\"\"Load but modify the loader instance so that it will add __line__\n attributes to the returned object.\"\"\"\n handler = ConfigYAML(yaml_type=YAMLType.SPACK_CONFIG_FILE)\n return handler.load(str_or_file)\n\n\ndef load(*args, **kwargs):\n handler = ConfigYAML(yaml_type=YAMLType.GENERIC_YAML)\n return handler.load(*args, **kwargs)\n\n\n@return_string_when_no_stream\ndef dump_config(data, stream, *, default_flow_style=False, blame=False):\n if blame:\n handler = ConfigYAML(yaml_type=YAMLType.ANNOTATED_SPACK_CONFIG_FILE)\n handler.yaml.default_flow_style = default_flow_style\n handler.yaml.width = maxint\n return _dump_annotated(handler, data, stream)\n\n handler = ConfigYAML(yaml_type=YAMLType.SPACK_CONFIG_FILE)\n handler.yaml.default_flow_style = default_flow_style\n handler.yaml.width = maxint\n return handler.dump(data, stream)\n\n\ndef _dump_annotated(handler, data, stream=None):\n sio = io.StringIO()\n handler.dump(data, sio)\n\n # write_line_break() is not called by YAML for empty lines, so we\n # skip empty lines here with \\n+.\n lines = re.split(r\"\\n+\", sio.getvalue().rstrip())\n\n getvalue = None\n if stream is None:\n stream = io.StringIO()\n getvalue = stream.getvalue\n\n # write out annotations and lines, accounting for color\n width = max(clen(a) for a in _ANNOTATIONS)\n formats = [\"%%-%ds %%s\\n\" % (width + cextra(a)) for a in _ANNOTATIONS]\n\n for fmt, annotation, line in zip(formats, _ANNOTATIONS, lines):\n stream.write(fmt % (annotation, line))\n\n if getvalue:\n return getvalue()\n\n\ndef sorted_dict(data):\n \"\"\"Descend into data and sort all dictionary keys.\"\"\"\n if isinstance(data, dict):\n return type(data)((k, sorted_dict(v)) for k, v in sorted(data.items()))\n elif isinstance(data, (list, tuple)):\n return type(data)(sorted_dict(v) for v in data)\n return data\n\n\ndef extract_comments(data):\n \"\"\"Extract and returns comments from some YAML data\"\"\"\n return getattr(data, comments.Comment.attrib, None)\n\n\ndef set_comments(data, *, data_comments):\n \"\"\"Set comments on some YAML data\"\"\"\n return setattr(data, comments.Comment.attrib, data_comments)\n\n\ndef name_mark(name):\n \"\"\"Returns a mark with just a name\"\"\"\n return error.StringMark(name, None, None, None, None, None)\n\n\ndef anchorify(data: Union[dict, list], identifier: Callable[[Any], str] = repr) -> None:\n \"\"\"Replace identical dict/list branches in tree with references to earlier instances. The YAML\n serializer generate anchors for them, resulting in small yaml files.\"\"\"\n anchors: Dict[str, Union[dict, list]] = {}\n stack: List[Union[dict, list]] = [data]\n\n while stack:\n item = stack.pop()\n\n for key, value in item.items() if isinstance(item, dict) else enumerate(item):\n if not isinstance(value, (dict, list)):\n continue\n\n id = identifier(value)\n anchor = anchors.get(id)\n\n if anchor is None:\n anchors[id] = value\n stack.append(value)\n else:\n item[key] = anchor # replace with reference\n\n\nclass SpackYAMLError(spack.error.SpackError):\n \"\"\"Raised when there are issues with YAML parsing.\"\"\"\n\n def __init__(self, msg, yaml_error, filename=None):\n self.filename = filename\n super().__init__(msg, str(yaml_error))\n\n\ndef get_mark_from_yaml_data(obj):\n \"\"\"Try to get ``spack.util.spack_yaml`` mark from YAML data.\n\n We try the object, and if that fails we try its first member (if it's a container).\n\n Returns:\n mark if one is found, otherwise None.\n \"\"\"\n # mark of object itelf\n mark = getattr(obj, \"_start_mark\", None)\n if mark:\n return mark\n\n # mark of first member if it is a container\n if isinstance(obj, (list, dict)):\n first_member = next(iter(obj), None)\n if first_member:\n mark = getattr(first_member, \"_start_mark\", None)\n\n return mark\n" }, { "path": "lib/spack/spack/util/timer.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"Debug signal handler: prints a stack trace and enters interpreter.\n\n``register_interrupt_handler()`` enables a ctrl-C handler that prints\na stack trace and drops the user into an interpreter.\n\n\"\"\"\n\nimport collections\nimport sys\nimport time\nfrom contextlib import contextmanager\nfrom typing import Callable, Dict, List\n\nimport spack.util.spack_json as sjson\nfrom spack.llnl.util.lang import pretty_seconds_formatter\n\nTimerEvent = collections.namedtuple(\"TimerEvent\", (\"time\", \"running\", \"label\"))\nTimeTracker = collections.namedtuple(\"TimeTracker\", (\"total\", \"start\", \"count\", \"path\"))\n\n#: name for the global timer (used in start(), stop(), duration() without arguments)\nglobal_timer_name = \"_global\"\n\n\nclass BaseTimer:\n def start(self, name=None):\n pass\n\n def stop(self, name=None):\n pass\n\n def duration(self, name=None):\n return 0.0\n\n @contextmanager\n def measure(self, name):\n yield self\n\n @property\n def phases(self):\n return []\n\n def write_json(self, out=sys.stdout):\n pass\n\n def write_tty(self, out=sys.stdout):\n pass\n\n\nclass NullTimer(BaseTimer):\n \"\"\"Timer interface that does nothing, useful in for \"tell\n don't ask\" style code when timers are optional.\"\"\"\n\n pass\n\n\nclass Timer(BaseTimer):\n \"\"\"Simple interval timer\"\"\"\n\n def __init__(self, now: Callable[[], float] = time.time):\n \"\"\"\n Arguments:\n now: function that gives the seconds since e.g. epoch\n \"\"\"\n self._now = now\n self._timers: Dict[str, TimeTracker] = {}\n self._timer_stack: List[str] = []\n\n self._events: List[TimerEvent] = []\n # Push start event\n self._events.append(TimerEvent(self._now(), True, global_timer_name))\n\n def start(self, name=global_timer_name):\n \"\"\"\n Start or restart a named timer, or the global timer when no name is given.\n\n Arguments:\n name (str): Optional name of the timer. When no name is passed, the\n global timer is started.\n \"\"\"\n self._events.append(TimerEvent(self._now(), True, name))\n\n def stop(self, name=global_timer_name):\n \"\"\"\n Stop a named timer, or all timers when no name is given. Stopping a\n timer that has not started has no effect.\n\n Arguments:\n name (str): Optional name of the timer. When no name is passed, all\n timers are stopped.\n \"\"\"\n self._events.append(TimerEvent(self._now(), False, name))\n\n def duration(self, name=global_timer_name):\n \"\"\"\n Get the time in seconds of a named timer, or the total time if no\n name is passed. The duration is always 0 for timers that have not been\n started, no error is raised.\n\n Arguments:\n name (str): (Optional) name of the timer\n\n Returns:\n float: duration of timer.\n \"\"\"\n self._flatten()\n if name in self._timers:\n if name in self._timer_stack:\n return self._timers[name].total + (self._now() - self._timers[name].start)\n return self._timers[name].total\n else:\n return 0.0\n\n @contextmanager\n def measure(self, name):\n \"\"\"\n Context manager that allows you to time a block of code.\n\n Arguments:\n name (str): Name of the timer\n \"\"\"\n self.start(name)\n yield self\n self.stop(name)\n\n @property\n def phases(self):\n \"\"\"Get all named timers (excluding the global/total timer)\"\"\"\n self._flatten()\n return [k for k in self._timers.keys() if not k == global_timer_name]\n\n def _flatten(self):\n for event in self._events:\n if event.running:\n if event.label not in self._timer_stack:\n self._timer_stack.append(event.label)\n # Only start the timer if it is on top of the stack\n # restart doesn't work after a subtimer is started\n if event.label == self._timer_stack[-1]:\n timer_path = \"/\".join(self._timer_stack[1:])\n tracker = self._timers.get(\n event.label, TimeTracker(0.0, event.time, 0, timer_path)\n )\n assert tracker.path == timer_path\n self._timers[event.label] = TimeTracker(\n tracker.total, event.time, tracker.count, tracker.path\n )\n else: # if not event.running:\n if event.label in self._timer_stack:\n index = self._timer_stack.index(event.label)\n for label in self._timer_stack[index:]:\n tracker = self._timers[label]\n self._timers[label] = TimeTracker(\n tracker.total + (event.time - tracker.start),\n None,\n tracker.count + 1,\n tracker.path,\n )\n self._timer_stack = self._timer_stack[: max(0, index)]\n # clear events\n self._events = []\n\n def write_json(self, out=sys.stdout, extra_attributes={}):\n \"\"\"Write a json object with times to file\"\"\"\n self._flatten()\n data = {\n \"total\": self._timers[global_timer_name].total,\n \"phases\": [\n {\n \"name\": phase,\n \"path\": self._timers[phase].path,\n \"seconds\": self._timers[phase].total,\n \"count\": self._timers[phase].count,\n }\n for phase in self.phases\n ],\n }\n if extra_attributes:\n data.update(extra_attributes)\n if out:\n out.write(sjson.dump(data))\n else:\n return data\n\n def write_tty(self, out=sys.stdout):\n \"\"\"Write a human-readable summary of timings (depth is 1)\"\"\"\n self._flatten()\n\n times = [self.duration(p) for p in self.phases]\n\n # Get a consistent unit for the time\n pretty_seconds = pretty_seconds_formatter(max(times))\n\n # Tuples of (phase, time) including total.\n formatted = list(zip(self.phases, times))\n formatted.append((\"total\", self.duration()))\n\n # Write to out\n for name, duration in formatted:\n out.write(f\" {name:10s} {pretty_seconds(duration):>10s}\\n\")\n\n\n#: instance of a do-nothing timer\nNULL_TIMER = NullTimer()\n" }, { "path": "lib/spack/spack/util/typing.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.: object\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Extra support for type checking in Spack.\n\nProtocols here that have runtime overhead should be set to ``object`` when\n``TYPE_CHECKING`` is not enabled, as they can incur unreasonable runtime overheads.\n\nIn particular, Protocols intended for use on objects that have many ``isinstance()``\ncalls can be very expensive.\n\n\"\"\"\n\nfrom typing import TYPE_CHECKING, Any\n\nfrom spack.vendor.typing_extensions import Protocol\n\nif TYPE_CHECKING:\n\n class SupportsRichComparison(Protocol):\n \"\"\"Objects that support =, !=, <, <=, >, and >=.\"\"\"\n\n def __eq__(self, other: Any) -> bool:\n raise NotImplementedError\n\n def __ne__(self, other: Any) -> bool:\n raise NotImplementedError\n\n def __lt__(self, other: Any) -> bool:\n raise NotImplementedError\n\n def __le__(self, other: Any) -> bool:\n raise NotImplementedError\n\n def __gt__(self, other: Any) -> bool:\n raise NotImplementedError\n\n def __ge__(self, other: Any) -> bool:\n raise NotImplementedError\n\nelse:\n SupportsRichComparison = object\n" }, { "path": "lib/spack/spack/util/unparse/LICENSE", "content": "LICENSE\n=======\n\nCopyright (c) 2014, Simon Percivall\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:\n\n* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.\n\n* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.\n\n* Neither the name of AST Unparser nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n\nPYTHON SOFTWARE FOUNDATION LICENSE VERSION 2\n--------------------------------------------\n\n1. This LICENSE AGREEMENT is between the Python Software Foundation\n(\"PSF\"), and the Individual or Organization (\"Licensee\") accessing and\notherwise using this software (\"Python\") in source or binary form and\nits associated documentation.\n\n2. Subject to the terms and conditions of this License Agreement, PSF hereby\ngrants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,\nanalyze, test, perform and/or display publicly, prepare derivative works,\ndistribute, and otherwise use Python alone or in any derivative version,\nprovided, however, that PSF's License Agreement and PSF's notice of copyright,\ni.e., \"Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,\n2011, 2012, 2013, 2014 Python Software Foundation; All Rights Reserved\" are retained\nin Python alone or in any derivative version prepared by Licensee.\n\n3. In the event Licensee prepares a derivative work that is based on\nor incorporates Python or any part thereof, and wants to make\nthe derivative work available to others as provided herein, then\nLicensee hereby agrees to include in any such work a brief summary of\nthe changes made to Python.\n\n4. PSF is making Python available to Licensee on an \"AS IS\"\nbasis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR\nIMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND\nDISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS\nFOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT\nINFRINGE ANY THIRD PARTY RIGHTS.\n\n5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON\nFOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS\nA RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,\nOR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.\n\n6. This License Agreement will automatically terminate upon a material\nbreach of its terms and conditions.\n\n7. Nothing in this License Agreement shall be deemed to create any\nrelationship of agency, partnership, or joint venture between PSF and\nLicensee. This License Agreement does not grant permission to use PSF\ntrademarks or trade name in a trademark sense to endorse or promote\nproducts or services of Licensee, or any third party.\n\n8. By copying, installing or otherwise using Python, Licensee\nagrees to be bound by the terms and conditions of this License\nAgreement.\n" }, { "path": "lib/spack/spack/util/unparse/__init__.py", "content": "# Copyright (c) 2014-2021, Simon Percivall and Spack Project Developers.\n#\n# SPDX-License-Identifier: Python-2.0\n\nfrom .unparser import Unparser\n\n__version__ = \"1.6.3\"\n\n\ndef unparse(tree, py_ver_consistent=False):\n unparser = Unparser(py_ver_consistent=py_ver_consistent)\n return unparser.visit(tree) + \"\\n\"\n" }, { "path": "lib/spack/spack/util/unparse/unparser.py", "content": "# Copyright (c) 2014-2021, Simon Percivall and Spack Project Developers.\n#\n# SPDX-License-Identifier: Python-2.0\n\"Usage: unparse.py \"\n\nimport ast\nimport sys\nfrom ast import AST, FormattedValue, If, JoinedStr, Name, Tuple\nfrom contextlib import contextmanager\nfrom enum import IntEnum, auto\nfrom typing import Optional\n\n\n# TODO: if we require Python 3.7, use its `nullcontext()`\n@contextmanager\ndef nullcontext():\n yield\n\n\ndef is_non_empty_non_star_tuple(slice_value):\n \"\"\"True for `(1, 2)`, False for `()` and `(1, *b)`\"\"\"\n return (\n isinstance(slice_value, Tuple)\n and slice_value.elts\n and not any(isinstance(elt, ast.Starred) for elt in slice_value.elts)\n )\n\n\ndef iter_fields(node):\n \"\"\"\n Yield a tuple of ``(fieldname, value)`` for each field in ``node._fields``\n that is present on *node*.\n \"\"\"\n for field in node._fields:\n try:\n yield field, getattr(node, field)\n except AttributeError:\n pass\n\n\nclass NodeVisitor(object):\n \"\"\"\n A node visitor base class that walks the abstract syntax tree and calls a\n visitor function for every node found. This function may return a value\n which is forwarded by the `visit` method.\n\n This class is meant to be subclassed, with the subclass adding visitor\n methods.\n\n Per default the visitor functions for the nodes are ``'visit_'`` +\n class name of the node. So a `TryFinally` node visit function would\n be `visit_TryFinally`. This behavior can be changed by overriding\n the `visit` method. If no visitor function exists for a node\n (return value `None`) the `generic_visit` visitor is used instead.\n\n Don't use the `NodeVisitor` if you want to apply changes to nodes during\n traversing. For this a special visitor exists (`NodeTransformer`) that\n allows modifications.\n \"\"\"\n\n def visit(self, node):\n \"\"\"Visit a node.\"\"\"\n method = \"visit_\" + node.__class__.__name__\n visitor = getattr(self, method, self.generic_visit)\n return visitor(node)\n\n def generic_visit(self, node):\n \"\"\"Called if no explicit visitor function exists for a node.\"\"\"\n for field, value in iter_fields(node):\n if isinstance(value, list):\n for item in value:\n if isinstance(item, AST):\n self.visit(item)\n elif isinstance(value, AST):\n self.visit(value)\n\n\n# Large float and imaginary literals get turned into infinities in the AST.\n# We unparse those infinities to INFSTR.\n_INFSTR = \"1e\" + repr(sys.float_info.max_10_exp + 1)\n\n\nclass _Precedence(IntEnum):\n \"\"\"Precedence table that originated from python grammar.\"\"\"\n\n NAMED_EXPR = auto() # := \n TUPLE = auto() # , \n YIELD = auto() # 'yield', 'yield from'\n TEST = auto() # 'if'-'else', 'lambda'\n OR = auto() # 'or'\n AND = auto() # 'and'\n NOT = auto() # 'not'\n CMP = auto() # '<', '>', '==', '>=', '<=', '!=',\n # 'in', 'not in', 'is', 'is not'\n EXPR = auto()\n BOR = EXPR # '|'\n BXOR = auto() # '^'\n BAND = auto() # '&'\n SHIFT = auto() # '<<', '>>'\n ARITH = auto() # '+', '-'\n TERM = auto() # '*', '@', '/', '%', '//'\n FACTOR = auto() # unary '+', '-', '~'\n POWER = auto() # '**'\n AWAIT = auto() # 'await'\n ATOM = auto()\n\n def next(self):\n try:\n return self.__class__(self + 1)\n except ValueError:\n return self\n\n\n_SINGLE_QUOTES = (\"'\", '\"')\n_MULTI_QUOTES = ('\"\"\"', \"'''\")\n_ALL_QUOTES = (*_SINGLE_QUOTES, *_MULTI_QUOTES)\n\n\nclass Unparser(NodeVisitor):\n \"\"\"Methods in this class recursively traverse an AST and\n output source code for the abstract syntax; original formatting\n is disregarded.\"\"\"\n\n def __init__(self, py_ver_consistent=False, _avoid_backslashes=False):\n self._source = []\n self._precedences = {}\n self._type_ignores = {}\n self._indent = 0\n self._in_try_star = False\n self._py_ver_consistent = py_ver_consistent\n self._avoid_backslashes = _avoid_backslashes\n\n def interleave(self, inter, f, seq):\n \"\"\"Call f on each item in seq, calling inter() in between.\"\"\"\n seq = iter(seq)\n try:\n f(next(seq))\n except StopIteration:\n pass\n else:\n for x in seq:\n inter()\n f(x)\n\n def items_view(self, traverser, items):\n \"\"\"Traverse and separate the given *items* with a comma and append it to\n the buffer. If *items* is a single item sequence, a trailing comma\n will be added.\"\"\"\n if len(items) == 1:\n traverser(items[0])\n self.write(\",\")\n else:\n self.interleave(lambda: self.write(\", \"), traverser, items)\n\n def maybe_newline(self):\n \"\"\"Adds a newline if it isn't the start of generated source\"\"\"\n if self._source:\n self.write(\"\\n\")\n\n def fill(self, text=\"\"):\n \"\"\"Indent a piece of text and append it, according to the current\n indentation level\"\"\"\n self.maybe_newline()\n self.write(\" \" * self._indent + text)\n\n def write(self, *text):\n \"\"\"Add new source parts\"\"\"\n self._source.extend(text)\n\n @contextmanager\n def buffered(self, buffer=None):\n if buffer is None:\n buffer = []\n\n original_source = self._source\n self._source = buffer\n yield buffer\n self._source = original_source\n\n @contextmanager\n def block(self, *, extra=None):\n \"\"\"A context manager for preparing the source for blocks. It adds\n the character':', increases the indentation on enter and decreases\n the indentation on exit. If *extra* is given, it will be directly\n appended after the colon character.\n \"\"\"\n self.write(\":\")\n if extra:\n self.write(extra)\n self._indent += 1\n yield\n self._indent -= 1\n\n @contextmanager\n def delimit(self, start, end):\n \"\"\"A context manager for preparing the source for expressions. It adds\n *start* to the buffer and enters, after exit it adds *end*.\"\"\"\n\n self.write(start)\n yield\n self.write(end)\n\n def delimit_if(self, start, end, condition):\n if condition:\n return self.delimit(start, end)\n else:\n return nullcontext()\n\n def require_parens(self, precedence, node):\n \"\"\"Shortcut to adding precedence related parens\"\"\"\n return self.delimit_if(\"(\", \")\", self.get_precedence(node) > precedence)\n\n def get_precedence(self, node):\n return self._precedences.get(node, _Precedence.TEST)\n\n def set_precedence(self, precedence, *nodes):\n for node in nodes:\n self._precedences[node] = precedence\n\n def get_raw_docstring(self, node):\n \"\"\"If a docstring node is found in the body of the *node* parameter,\n return that docstring node, None otherwise.\n\n Logic mirrored from ``_PyAST_GetDocString``.\"\"\"\n if (\n not isinstance(node, (ast.AsyncFunctionDef, ast.FunctionDef, ast.ClassDef, ast.Module))\n or len(node.body) < 1\n ):\n return None\n node = node.body[0]\n if not isinstance(node, ast.Expr):\n return None\n node = node.value\n if _is_str_literal(node):\n return node\n\n def get_type_comment(self, node):\n # Python 3.8 introduced type_comment\n # (enabled on compile(... ast.PyCF_TYPE_COMMENTS))\n comment = self._type_ignores.get(node.lineno) or getattr(node, \"type_comment\", None)\n if comment is not None:\n return f\" # type: {comment}\"\n\n def traverse(self, node):\n if isinstance(node, list):\n for item in node:\n self.traverse(item)\n else:\n super().visit(node)\n\n # Note: as visit() resets the output text, do NOT rely on\n # NodeVisitor.generic_visit to handle any nodes (as it calls back in to\n # the subclass visit() method, which resets self._source to an empty list)\n def visit(self, node):\n \"\"\"Outputs a source code string that, if converted back to an ast\n (using ast.parse) will generate an AST equivalent to *node*\"\"\"\n self._source = []\n self.traverse(node)\n return \"\".join(self._source)\n\n def _write_docstring_and_traverse_body(self, node):\n docstring = self.get_raw_docstring(node)\n if docstring:\n self._write_docstring(docstring)\n self.traverse(node.body[1:])\n else:\n self.traverse(node.body)\n\n def visit_Module(self, node):\n # Python 3.8 introduced types\n self._type_ignores = {\n ignore.lineno: f\"ignore{ignore.tag}\" for ignore in getattr(node, \"type_ignores\", ())\n }\n self._write_docstring_and_traverse_body(node)\n self._type_ignores.clear()\n\n def visit_FunctionType(self, node):\n with self.delimit(\"(\", \")\"):\n self.interleave(lambda: self.write(\", \"), self.traverse, node.argtypes)\n\n self.write(\" -> \")\n self.traverse(node.returns)\n\n def visit_Expr(self, node):\n self.fill()\n self.set_precedence(_Precedence.YIELD, node.value)\n self.traverse(node.value)\n\n def visit_NamedExpr(self, node):\n with self.require_parens(_Precedence.NAMED_EXPR, node):\n self.set_precedence(_Precedence.ATOM, node.target, node.value)\n self.traverse(node.target)\n self.write(\" := \")\n self.traverse(node.value)\n\n def visit_Import(self, node):\n self.fill(\"import \")\n self.interleave(lambda: self.write(\", \"), self.traverse, node.names)\n\n def visit_ImportFrom(self, node):\n self.fill(\"from \")\n self.write(\".\" * (node.level or 0))\n if node.module:\n self.write(node.module)\n self.write(\" import \")\n self.interleave(lambda: self.write(\", \"), self.traverse, node.names)\n\n def visit_Assign(self, node):\n self.fill()\n for target in node.targets:\n self.set_precedence(_Precedence.TUPLE, target)\n self.traverse(target)\n self.write(\" = \")\n self.traverse(node.value)\n type_comment = self.get_type_comment(node)\n if type_comment:\n self.write(type_comment)\n\n def visit_AugAssign(self, node):\n self.fill()\n self.traverse(node.target)\n self.write(\" \" + self.binop[node.op.__class__.__name__] + \"= \")\n self.traverse(node.value)\n\n def visit_AnnAssign(self, node):\n self.fill()\n with self.delimit_if(\"(\", \")\", not node.simple and isinstance(node.target, Name)):\n self.traverse(node.target)\n self.write(\": \")\n self.traverse(node.annotation)\n if node.value:\n self.write(\" = \")\n self.traverse(node.value)\n\n def visit_Return(self, node):\n self.fill(\"return\")\n if node.value:\n self.write(\" \")\n self.traverse(node.value)\n\n def visit_Pass(self, node):\n self.fill(\"pass\")\n\n def visit_Break(self, node):\n self.fill(\"break\")\n\n def visit_Continue(self, node):\n self.fill(\"continue\")\n\n def visit_Delete(self, node):\n self.fill(\"del \")\n self.interleave(lambda: self.write(\", \"), self.traverse, node.targets)\n\n def visit_Assert(self, node):\n self.fill(\"assert \")\n self.traverse(node.test)\n if node.msg:\n self.write(\", \")\n self.traverse(node.msg)\n\n def visit_Global(self, node):\n self.fill(\"global \")\n self.interleave(lambda: self.write(\", \"), self.write, node.names)\n\n def visit_Nonlocal(self, node):\n self.fill(\"nonlocal \")\n self.interleave(lambda: self.write(\", \"), self.write, node.names)\n\n def visit_Await(self, node):\n with self.require_parens(_Precedence.AWAIT, node):\n self.write(\"await\")\n if node.value:\n self.write(\" \")\n self.set_precedence(_Precedence.ATOM, node.value)\n self.traverse(node.value)\n\n def visit_Yield(self, node):\n with self.require_parens(_Precedence.YIELD, node):\n self.write(\"yield\")\n if node.value:\n self.write(\" \")\n self.set_precedence(_Precedence.ATOM, node.value)\n self.traverse(node.value)\n\n def visit_YieldFrom(self, node):\n with self.require_parens(_Precedence.YIELD, node):\n self.write(\"yield from \")\n if not node.value:\n raise ValueError(\"Node can't be used without a value attribute.\")\n self.set_precedence(_Precedence.ATOM, node.value)\n self.traverse(node.value)\n\n def visit_Raise(self, node):\n self.fill(\"raise\")\n if not node.exc:\n if node.cause:\n raise ValueError(\"Node can't use cause without an exception.\")\n return\n self.write(\" \")\n self.traverse(node.exc)\n if node.cause:\n self.write(\" from \")\n self.traverse(node.cause)\n\n def do_visit_try(self, node):\n self.fill(\"try\")\n with self.block():\n self.traverse(node.body)\n for ex in node.handlers:\n self.traverse(ex)\n if node.orelse:\n self.fill(\"else\")\n with self.block():\n self.traverse(node.orelse)\n if node.finalbody:\n self.fill(\"finally\")\n with self.block():\n self.traverse(node.finalbody)\n\n def visit_Try(self, node):\n prev_in_try_star = self._in_try_star\n try:\n self._in_try_star = False\n self.do_visit_try(node)\n finally:\n self._in_try_star = prev_in_try_star\n\n def visit_TryStar(self, node):\n prev_in_try_star = self._in_try_star\n try:\n self._in_try_star = True\n self.do_visit_try(node)\n finally:\n self._in_try_star = prev_in_try_star\n\n def visit_ExceptHandler(self, node):\n self.fill(\"except*\" if self._in_try_star else \"except\")\n if node.type:\n self.write(\" \")\n self.traverse(node.type)\n if node.name:\n self.write(\" as \")\n self.write(node.name)\n with self.block():\n self.traverse(node.body)\n\n def visit_ClassDef(self, node):\n self.maybe_newline()\n for deco in node.decorator_list:\n self.fill(\"@\")\n self.traverse(deco)\n self.fill(\"class \" + node.name)\n if hasattr(node, \"type_params\"):\n self._type_params_helper(node.type_params)\n with self.delimit_if(\"(\", \")\", condition=node.bases or node.keywords):\n comma = False\n for e in node.bases:\n if comma:\n self.write(\", \")\n else:\n comma = True\n self.traverse(e)\n for e in node.keywords:\n if comma:\n self.write(\", \")\n else:\n comma = True\n self.traverse(e)\n\n with self.block():\n self._write_docstring_and_traverse_body(node)\n\n def visit_FunctionDef(self, node):\n self._function_helper(node, \"def\")\n\n def visit_AsyncFunctionDef(self, node):\n self._function_helper(node, \"async def\")\n\n def _function_helper(self, node, fill_suffix):\n self.maybe_newline()\n for deco in node.decorator_list:\n self.fill(\"@\")\n self.traverse(deco)\n def_str = fill_suffix + \" \" + node.name\n self.fill(def_str)\n if hasattr(node, \"type_params\"):\n self._type_params_helper(node.type_params)\n with self.delimit(\"(\", \")\"):\n self.traverse(node.args)\n if node.returns:\n self.write(\" -> \")\n self.traverse(node.returns)\n with self.block(extra=self.get_type_comment(node)):\n self._write_docstring_and_traverse_body(node)\n\n def _type_params_helper(self, type_params):\n if type_params is not None and len(type_params) > 0:\n with self.delimit(\"[\", \"]\"):\n self.interleave(lambda: self.write(\", \"), self.traverse, type_params)\n\n def visit_TypeVar(self, node):\n self.write(node.name)\n if node.bound:\n self.write(\": \")\n self.traverse(node.bound)\n # Python 3.13 introduced default_value\n if getattr(node, \"default_value\", False):\n self.write(\" = \")\n self.traverse(node.default_value)\n\n def visit_TypeVarTuple(self, node):\n self.write(\"*\" + node.name)\n # Python 3.13 introduced default_value\n if getattr(node, \"default_value\", False):\n self.write(\" = \")\n self.traverse(node.default_value)\n\n def visit_ParamSpec(self, node):\n self.write(\"**\" + node.name)\n # Python 3.13 introduced default_value\n if getattr(node, \"default_value\", False):\n self.write(\" = \")\n self.traverse(node.default_value)\n\n def visit_TypeAlias(self, node):\n self.fill(\"type \")\n self.traverse(node.name)\n self._type_params_helper(node.type_params)\n self.write(\" = \")\n self.traverse(node.value)\n\n def visit_For(self, node):\n self._for_helper(\"for \", node)\n\n def visit_AsyncFor(self, node):\n self._for_helper(\"async for \", node)\n\n def _for_helper(self, fill, node):\n self.fill(fill)\n self.set_precedence(_Precedence.TUPLE, node.target)\n self.traverse(node.target)\n self.write(\" in \")\n self.traverse(node.iter)\n with self.block(extra=self.get_type_comment(node)):\n self.traverse(node.body)\n if node.orelse:\n self.fill(\"else\")\n with self.block():\n self.traverse(node.orelse)\n\n def visit_If(self, node):\n self.fill(\"if \")\n self.traverse(node.test)\n with self.block():\n self.traverse(node.body)\n # collapse nested ifs into equivalent elifs.\n while node.orelse and len(node.orelse) == 1 and isinstance(node.orelse[0], If):\n node = node.orelse[0]\n self.fill(\"elif \")\n self.traverse(node.test)\n with self.block():\n self.traverse(node.body)\n # final else\n if node.orelse:\n self.fill(\"else\")\n with self.block():\n self.traverse(node.orelse)\n\n def visit_While(self, node):\n self.fill(\"while \")\n self.traverse(node.test)\n with self.block():\n self.traverse(node.body)\n if node.orelse:\n self.fill(\"else\")\n with self.block():\n self.traverse(node.orelse)\n\n def visit_With(self, node):\n self.fill(\"with \")\n self.interleave(lambda: self.write(\", \"), self.traverse, node.items)\n with self.block(extra=self.get_type_comment(node)):\n self.traverse(node.body)\n\n def visit_AsyncWith(self, node):\n self.fill(\"async with \")\n self.interleave(lambda: self.write(\", \"), self.traverse, node.items)\n with self.block(extra=self.get_type_comment(node)):\n self.traverse(node.body)\n\n def _str_literal_helper(\n self, string, *, quote_types=_ALL_QUOTES, escape_special_whitespace=False\n ):\n \"\"\"Helper for writing string literals, minimizing escapes.\n Returns the tuple (string literal to write, possible quote types).\n \"\"\"\n\n def escape_char(c):\n # \\n and \\t are non-printable, but we only escape them if\n # escape_special_whitespace is True\n if not escape_special_whitespace and c in \"\\n\\t\":\n return c\n # Always escape backslashes and other non-printable characters\n if c == \"\\\\\" or not c.isprintable():\n return c.encode(\"unicode_escape\").decode(\"ascii\")\n return c\n\n escaped_string = \"\".join(map(escape_char, string))\n possible_quotes = quote_types\n if \"\\n\" in escaped_string:\n possible_quotes = [q for q in possible_quotes if q in _MULTI_QUOTES]\n possible_quotes = [q for q in possible_quotes if q not in escaped_string]\n if not possible_quotes:\n # If there aren't any possible_quotes, fallback to using repr\n # on the original string. Try to use a quote from quote_types,\n # e.g., so that we use triple quotes for docstrings.\n string = repr(string)\n quote = next((q for q in quote_types if string[0] in q), string[0])\n return string[1:-1], [quote]\n if escaped_string:\n # Sort so that we prefer '''\"''' over \"\"\"\\\"\"\"\"\n possible_quotes.sort(key=lambda q: q[0] == escaped_string[-1])\n # If we're using triple quotes and we'd need to escape a final\n # quote, escape it\n if possible_quotes[0][0] == escaped_string[-1]:\n assert len(possible_quotes[0]) == 3\n escaped_string = escaped_string[:-1] + \"\\\\\" + escaped_string[-1]\n return escaped_string, possible_quotes\n\n def _write_str_avoiding_backslashes(self, string, *, quote_types=_ALL_QUOTES):\n \"\"\"Write string literal value with a best effort attempt to avoid backslashes.\"\"\"\n string, quote_types = self._str_literal_helper(string, quote_types=quote_types)\n quote_type = quote_types[0]\n self.write(f\"{quote_type}{string}{quote_type}\")\n\n # Python < 3.8. Num, Str, Bytes, NameConstant, Ellipsis replaced with Constant\n # https://github.com/python/cpython/commit/3f22811fef73aec848d961593d95fa877f77ecbf\n if sys.version_info < (3, 8):\n\n def visit_Num(self, node):\n repr_n = repr(node.n)\n self.write(repr_n.replace(\"inf\", _INFSTR))\n\n def visit_Str(self, node):\n self._write_constant(node.s)\n\n def visit_Bytes(self, node):\n self.write(repr(node.s))\n\n def visit_NameConstant(self, node):\n self.write(repr(node.value))\n\n def visit_Ellipsis(self, node):\n self.write(\"...\")\n\n def _ftstring_helper(self, parts):\n new_parts = []\n quote_types = list(_ALL_QUOTES)\n fallback_to_repr = False\n for value, is_constant in parts:\n # Python 3.12 allows `f'{''}'`.\n # But we unparse to `f'{\"\"}'` for < 3.12 compat.\n if True:\n value, new_quote_types = self._str_literal_helper(\n value, quote_types=quote_types, escape_special_whitespace=is_constant\n )\n if set(new_quote_types).isdisjoint(quote_types):\n fallback_to_repr = True\n break\n quote_types = new_quote_types\n elif \"\\n\" in value:\n quote_types = [q for q in quote_types if q in _MULTI_QUOTES]\n assert quote_types\n new_parts.append(value)\n\n if fallback_to_repr:\n # If we weren't able to find a quote type that works for all parts\n # of the JoinedStr, fallback to using repr and triple single quotes.\n quote_types = [\"'''\"]\n new_parts.clear()\n for value, is_constant in parts:\n # Python 3.12 allows `f'{''}'`.\n # We need to unparse to `f'{\"\"}'` for < 3.12 compat.\n if True:\n value = repr('\"' + value) # force repr to use single quotes\n expected_prefix = \"'\\\"\"\n assert value.startswith(expected_prefix), repr(value)\n value = value[len(expected_prefix) : -1]\n new_parts.append(value)\n\n value = \"\".join(new_parts)\n quote_type = quote_types[0]\n self.write(f\"{quote_type}{value}{quote_type}\")\n\n def _write_ftstring(self, node, prefix):\n self.write(prefix)\n # Python 3.12 added support for backslashes inside format parts.\n # We need to keep adding backslashes for python < 3.11 compat.\n if self._avoid_backslashes:\n with self.buffered() as buffer:\n self._write_ftstring_inner(node)\n return self._write_str_avoiding_backslashes(\"\".join(buffer))\n fstring_parts = []\n for value in node.values:\n with self.buffered() as buffer:\n self._write_ftstring_inner(value)\n fstring_parts.append((\"\".join(buffer), _is_str_literal(value)))\n self._ftstring_helper(fstring_parts)\n\n def visit_JoinedStr(self, node):\n self._write_ftstring(node, \"f\")\n\n def visit_TemplateStr(self, node):\n self._write_ftstring(node, \"t\")\n\n def _write_ftstring_inner(self, node, is_format_spec=False):\n if isinstance(node, JoinedStr):\n # for both the f-string itself, and format_spec\n for value in node.values:\n self._write_ftstring_inner(value, is_format_spec=is_format_spec)\n elif isinstance(node, FormattedValue):\n self.visit_FormattedValue(node)\n elif _is_interpolation(node):\n self.visit_Interpolation(node)\n else: # str literal\n maybe_string = _get_str_literal_value(node)\n if maybe_string is None:\n raise ValueError(f\"Unexpected node inside JoinedStr, {node!r}\")\n\n value = maybe_string.replace(\"{\", \"{{\").replace(\"}\", \"}}\")\n\n if is_format_spec:\n value = value.replace(\"\\\\\", \"\\\\\\\\\")\n value = value.replace(\"'\", \"\\\\'\")\n value = value.replace('\"', '\\\\\"')\n value = value.replace(\"\\n\", \"\\\\n\")\n self.write(value)\n\n def _unparse_interpolation_value(self, inner):\n # Python <= 3.11 does not support backslashes inside format parts\n unparser = type(self)(_avoid_backslashes=True)\n unparser.set_precedence(_Precedence.TEST.next(), inner)\n return unparser.visit(inner)\n\n def _write_interpolation(self, node, use_str_attr=False):\n with self.delimit(\"{\", \"}\"):\n if use_str_attr:\n expr = node.str\n else:\n expr = self._unparse_interpolation_value(node.value)\n # Python <= 3.11 does not support backslash in formats part\n if \"\\\\\" in expr:\n raise ValueError(\n \"Unable to avoid backslash in f-string expression part (python 3.11)\"\n )\n if expr.startswith(\"{\"):\n # Separate pair of opening brackets as \"{ {\"\n self.write(\" \")\n self.write(expr)\n if node.conversion != -1:\n self.write(f\"!{chr(node.conversion)}\")\n if node.format_spec:\n self.write(\":\")\n self._write_ftstring_inner(node.format_spec, is_format_spec=True)\n\n def visit_FormattedValue(self, node):\n self._write_interpolation(node)\n\n def visit_Interpolation(self, node):\n # If `str` is set to `None`, use the `value` to generate the source code.\n self._write_interpolation(node, use_str_attr=node.str is not None)\n\n def visit_Name(self, node):\n self.write(node.id)\n\n def _write_docstring(self, node):\n self.fill()\n # Don't emit `u\"\"` because it's not avail in python AST <= 3.7\n # Ubuntu 18's Python 3.6 doesn't have \"kind\"\n if not self._py_ver_consistent and getattr(node, \"kind\", None) == \"u\":\n self.write(\"u\")\n # Python 3.8 replaced Str with Constant\n value = _get_str_literal_value(node)\n if value is None:\n raise ValueError(f\"Node {node!r} is not a string literal.\")\n self._write_str_avoiding_backslashes(value, quote_types=_MULTI_QUOTES)\n\n def _write_constant(self, value):\n if isinstance(value, (float, complex)):\n # Substitute overflowing decimal literal for AST infinities,\n # and inf - inf for NaNs.\n self.write(\n repr(value).replace(\"inf\", _INFSTR).replace(\"nan\", f\"({_INFSTR}-{_INFSTR})\")\n )\n # Python <= 3.11 does not support backslashes inside format parts\n elif self._avoid_backslashes and isinstance(value, str):\n self._write_str_avoiding_backslashes(value)\n else:\n self.write(repr(value))\n\n def visit_Constant(self, node):\n value = node.value\n if isinstance(value, tuple):\n with self.delimit(\"(\", \")\"):\n self.items_view(self._write_constant, value)\n elif value is ...:\n self.write(\"...\")\n else:\n # Don't emit `u\"\"` because it's not avail in python AST <= 3.7\n # Ubuntu 18's Python 3.6 doesn't have \"kind\"\n if not self._py_ver_consistent and getattr(node, \"kind\", None) == \"u\":\n self.write(\"u\")\n self._write_constant(node.value)\n\n def visit_List(self, node):\n with self.delimit(\"[\", \"]\"):\n self.interleave(lambda: self.write(\", \"), self.traverse, node.elts)\n\n def visit_ListComp(self, node):\n with self.delimit(\"[\", \"]\"):\n self.traverse(node.elt)\n for gen in node.generators:\n self.traverse(gen)\n\n def visit_GeneratorExp(self, node):\n with self.delimit(\"(\", \")\"):\n self.traverse(node.elt)\n for gen in node.generators:\n self.traverse(gen)\n\n def visit_SetComp(self, node):\n with self.delimit(\"{\", \"}\"):\n self.traverse(node.elt)\n for gen in node.generators:\n self.traverse(gen)\n\n def visit_DictComp(self, node):\n with self.delimit(\"{\", \"}\"):\n self.traverse(node.key)\n self.write(\": \")\n self.traverse(node.value)\n for gen in node.generators:\n self.traverse(gen)\n\n def visit_comprehension(self, node):\n if node.is_async:\n self.write(\" async for \")\n else:\n self.write(\" for \")\n self.set_precedence(_Precedence.TUPLE, node.target)\n self.traverse(node.target)\n self.write(\" in \")\n self.set_precedence(_Precedence.TEST.next(), node.iter, *node.ifs)\n self.traverse(node.iter)\n for if_clause in node.ifs:\n self.write(\" if \")\n self.traverse(if_clause)\n\n def visit_IfExp(self, node):\n with self.require_parens(_Precedence.TEST, node):\n self.set_precedence(_Precedence.TEST.next(), node.body, node.test)\n self.traverse(node.body)\n self.write(\" if \")\n self.traverse(node.test)\n self.write(\" else \")\n self.set_precedence(_Precedence.TEST, node.orelse)\n self.traverse(node.orelse)\n\n def visit_Set(self, node):\n if node.elts:\n with self.delimit(\"{\", \"}\"):\n self.interleave(lambda: self.write(\", \"), self.traverse, node.elts)\n else:\n # `{}` would be interpreted as a dictionary literal, and\n # `set` might be shadowed. Thus:\n self.write(\"{*()}\")\n\n def visit_Dict(self, node):\n def write_key_value_pair(k, v):\n self.traverse(k)\n self.write(\": \")\n self.traverse(v)\n\n def write_item(item):\n k, v = item\n if k is None:\n # for dictionary unpacking operator in dicts {**{'y': 2}}\n # see PEP 448 for details\n self.write(\"**\")\n self.set_precedence(_Precedence.EXPR, v)\n self.traverse(v)\n else:\n write_key_value_pair(k, v)\n\n with self.delimit(\"{\", \"}\"):\n self.interleave(lambda: self.write(\", \"), write_item, zip(node.keys, node.values))\n\n def visit_Tuple(self, node):\n with self.delimit_if(\n \"(\",\n \")\",\n # Don't drop redundant parenthesis to mimic python <= 3.10\n self._py_ver_consistent\n or len(node.elts) == 0\n or self.get_precedence(node) > _Precedence.TUPLE,\n ):\n self.items_view(self.traverse, node.elts)\n\n unop = {\"Invert\": \"~\", \"Not\": \"not\", \"UAdd\": \"+\", \"USub\": \"-\"}\n unop_precedence = {\n \"not\": _Precedence.NOT,\n \"~\": _Precedence.FACTOR,\n \"+\": _Precedence.FACTOR,\n \"-\": _Precedence.FACTOR,\n }\n\n def visit_UnaryOp(self, node):\n operator = self.unop[node.op.__class__.__name__]\n operator_precedence = self.unop_precedence[operator]\n with self.require_parens(operator_precedence, node):\n self.write(operator)\n # factor prefixes (+, -, ~) shouldn't be separated\n # from the value they belong, (e.g: +1 instead of + 1)\n if operator_precedence is not _Precedence.FACTOR:\n self.write(\" \")\n self.set_precedence(operator_precedence, node.operand)\n self.traverse(node.operand)\n\n binop = {\n \"Add\": \"+\",\n \"Sub\": \"-\",\n \"Mult\": \"*\",\n \"MatMult\": \"@\",\n \"Div\": \"/\",\n \"Mod\": \"%\",\n \"LShift\": \"<<\",\n \"RShift\": \">>\",\n \"BitOr\": \"|\",\n \"BitXor\": \"^\",\n \"BitAnd\": \"&\",\n \"FloorDiv\": \"//\",\n \"Pow\": \"**\",\n }\n\n binop_precedence = {\n \"+\": _Precedence.ARITH,\n \"-\": _Precedence.ARITH,\n \"*\": _Precedence.TERM,\n \"@\": _Precedence.TERM,\n \"/\": _Precedence.TERM,\n \"%\": _Precedence.TERM,\n \"<<\": _Precedence.SHIFT,\n \">>\": _Precedence.SHIFT,\n \"|\": _Precedence.BOR,\n \"^\": _Precedence.BXOR,\n \"&\": _Precedence.BAND,\n \"//\": _Precedence.TERM,\n \"**\": _Precedence.POWER,\n }\n\n binop_rassoc = frozenset((\"**\",))\n\n def visit_BinOp(self, node):\n operator = self.binop[node.op.__class__.__name__]\n operator_precedence = self.binop_precedence[operator]\n with self.require_parens(operator_precedence, node):\n if operator in self.binop_rassoc:\n left_precedence = operator_precedence.next()\n right_precedence = operator_precedence\n else:\n left_precedence = operator_precedence\n right_precedence = operator_precedence.next()\n\n self.set_precedence(left_precedence, node.left)\n self.traverse(node.left)\n self.write(f\" {operator} \")\n self.set_precedence(right_precedence, node.right)\n self.traverse(node.right)\n\n cmpops = {\n \"Eq\": \"==\",\n \"NotEq\": \"!=\",\n \"Lt\": \"<\",\n \"LtE\": \"<=\",\n \"Gt\": \">\",\n \"GtE\": \">=\",\n \"Is\": \"is\",\n \"IsNot\": \"is not\",\n \"In\": \"in\",\n \"NotIn\": \"not in\",\n }\n\n def visit_Compare(self, node):\n with self.require_parens(_Precedence.CMP, node):\n self.set_precedence(_Precedence.CMP.next(), node.left, *node.comparators)\n self.traverse(node.left)\n for o, e in zip(node.ops, node.comparators):\n self.write(\" \" + self.cmpops[o.__class__.__name__] + \" \")\n self.traverse(e)\n\n boolops = {\"And\": \"and\", \"Or\": \"or\"}\n boolop_precedence = {\"and\": _Precedence.AND, \"or\": _Precedence.OR}\n\n def visit_BoolOp(self, node):\n operator = self.boolops[node.op.__class__.__name__]\n operator_precedence = self.boolop_precedence[operator]\n\n def increasing_level_traverse(node):\n nonlocal operator_precedence\n operator_precedence = operator_precedence.next()\n self.set_precedence(operator_precedence, node)\n self.traverse(node)\n\n with self.require_parens(operator_precedence, node):\n s = f\" {operator} \"\n self.interleave(lambda: self.write(s), increasing_level_traverse, node.values)\n\n def visit_Attribute(self, node: ast.Attribute):\n self.set_precedence(_Precedence.ATOM, node.value)\n self.traverse(node.value)\n # Special case: 3.__abs__() is a syntax error, so if node.value\n # is an integer literal then we need to either parenthesize\n # it or add an extra space to get 3 .__abs__().\n if _is_int_literal(node.value):\n self.write(\" \")\n self.write(\".\")\n self.write(node.attr)\n\n def visit_Call(self, node):\n self.set_precedence(_Precedence.ATOM, node.func)\n self.traverse(node.func)\n with self.delimit(\"(\", \")\"):\n comma = False\n for e in node.args:\n if comma:\n self.write(\", \")\n else:\n comma = True\n self.traverse(e)\n for e in node.keywords:\n if comma:\n self.write(\", \")\n else:\n comma = True\n self.traverse(e)\n\n def visit_Subscript(self, node):\n def is_non_empty_tuple(slice_value):\n return isinstance(slice_value, Tuple) and slice_value.elts\n\n self.set_precedence(_Precedence.ATOM, node.value)\n self.traverse(node.value)\n with self.delimit(\"[\", \"]\"):\n # Python >= 3.11 supports `a[42, *b]` (same AST as a[(42, *b)]),\n # but this is syntax error in 3.10.\n # So, always emit parenthesis `a[(42, *b)]`\n if is_non_empty_non_star_tuple(node.slice):\n self.items_view(self.traverse, node.slice.elts)\n else:\n self.traverse(node.slice)\n\n def visit_Starred(self, node):\n self.write(\"*\")\n self.set_precedence(_Precedence.EXPR, node.value)\n self.traverse(node.value)\n\n # Python 3.9 simplified Subscript(Index(value)) to Subscript(value)\n # https://github.com/python/cpython/commit/13d52c268699f199a8e917a0f1dc4c51e5346c42\n def visit_Index(self, node):\n if is_non_empty_non_star_tuple(node.value):\n self.items_view(self.traverse, node.value.elts)\n else:\n self.traverse(node.value)\n\n def visit_Slice(self, node):\n if node.lower:\n self.traverse(node.lower)\n self.write(\":\")\n if node.upper:\n self.traverse(node.upper)\n if node.step:\n self.write(\":\")\n self.traverse(node.step)\n\n def visit_Match(self, node):\n self.fill(\"match \")\n self.traverse(node.subject)\n with self.block():\n for case in node.cases:\n self.traverse(case)\n\n # Python 3.9 replaced ExtSlice(slices) with Tuple(slices, Load())\n # https://github.com/python/cpython/commit/13d52c268699f199a8e917a0f1dc4c51e5346c42\n def visit_ExtSlice(self, node):\n self.interleave(lambda: self.write(\", \"), self.traverse, node.dims)\n\n def visit_arg(self, node):\n self.write(node.arg)\n if node.annotation:\n self.write(\": \")\n self.traverse(node.annotation)\n\n def visit_arguments(self, node):\n first = True\n # normal arguments\n # Python 3.8 introduced position-only arguments (PEP 570)\n all_args = getattr(node, \"posonlyargs\", []) + node.args\n defaults = [None] * (len(all_args) - len(node.defaults)) + node.defaults\n for index, elements in enumerate(zip(all_args, defaults), 1):\n a, d = elements\n if first:\n first = False\n else:\n self.write(\", \")\n self.traverse(a)\n if d:\n self.write(\"=\")\n self.traverse(d)\n # Python 3.8 introduced position-only arguments (PEP 570)\n if index == len(getattr(node, \"posonlyargs\", ())):\n self.write(\", /\")\n\n # varargs, or bare '*' if no varargs but keyword-only arguments present\n if node.vararg or node.kwonlyargs:\n if first:\n first = False\n else:\n self.write(\", \")\n self.write(\"*\")\n if node.vararg:\n self.write(node.vararg.arg)\n if node.vararg.annotation:\n self.write(\": \")\n self.traverse(node.vararg.annotation)\n\n # keyword-only arguments\n if node.kwonlyargs:\n for a, d in zip(node.kwonlyargs, node.kw_defaults):\n self.write(\", \")\n self.traverse(a)\n if d:\n self.write(\"=\")\n self.traverse(d)\n\n # kwargs\n if node.kwarg:\n if first:\n first = False\n else:\n self.write(\", \")\n self.write(\"**\" + node.kwarg.arg)\n if node.kwarg.annotation:\n self.write(\": \")\n self.traverse(node.kwarg.annotation)\n\n def visit_keyword(self, node):\n if node.arg is None:\n self.write(\"**\")\n else:\n self.write(node.arg)\n self.write(\"=\")\n self.traverse(node.value)\n\n def visit_Lambda(self, node):\n with self.require_parens(_Precedence.TEST, node):\n self.write(\"lambda\")\n with self.buffered() as buffer:\n self.traverse(node.args)\n # Don't omit extra space to keep old package hash\n # (extra space was removed in python 3.11)\n if buffer or self._py_ver_consistent:\n self.write(\" \", *buffer)\n self.write(\": \")\n self.set_precedence(_Precedence.TEST, node.body)\n self.traverse(node.body)\n\n def visit_alias(self, node):\n self.write(node.name)\n if node.asname:\n self.write(\" as \" + node.asname)\n\n def visit_withitem(self, node):\n self.traverse(node.context_expr)\n if node.optional_vars:\n self.write(\" as \")\n self.traverse(node.optional_vars)\n\n def visit_match_case(self, node):\n self.fill(\"case \")\n self.traverse(node.pattern)\n if node.guard:\n self.write(\" if \")\n self.traverse(node.guard)\n with self.block():\n self.traverse(node.body)\n\n def visit_MatchValue(self, node):\n self.traverse(node.value)\n\n def visit_MatchSingleton(self, node):\n self._write_constant(node.value)\n\n def visit_MatchSequence(self, node):\n with self.delimit(\"[\", \"]\"):\n self.interleave(lambda: self.write(\", \"), self.traverse, node.patterns)\n\n def visit_MatchStar(self, node):\n name = node.name\n if name is None:\n name = \"_\"\n self.write(f\"*{name}\")\n\n def visit_MatchMapping(self, node):\n def write_key_pattern_pair(pair):\n k, p = pair\n self.traverse(k)\n self.write(\": \")\n self.traverse(p)\n\n with self.delimit(\"{\", \"}\"):\n keys = node.keys\n self.interleave(\n lambda: self.write(\", \"),\n write_key_pattern_pair,\n # (zip strict is >= Python 3.10)\n zip(keys, node.patterns),\n )\n rest = node.rest\n if rest is not None:\n if keys:\n self.write(\", \")\n self.write(f\"**{rest}\")\n\n def visit_MatchClass(self, node):\n self.set_precedence(_Precedence.ATOM, node.cls)\n self.traverse(node.cls)\n with self.delimit(\"(\", \")\"):\n patterns = node.patterns\n self.interleave(lambda: self.write(\", \"), self.traverse, patterns)\n attrs = node.kwd_attrs\n if attrs:\n\n def write_attr_pattern(pair):\n attr, pattern = pair\n self.write(f\"{attr}=\")\n self.traverse(pattern)\n\n if patterns:\n self.write(\", \")\n self.interleave(\n lambda: self.write(\", \"),\n write_attr_pattern,\n # (zip strict is >= Python 3.10)\n zip(attrs, node.kwd_patterns),\n )\n\n def visit_MatchAs(self, node):\n name = node.name\n pattern = node.pattern\n if name is None:\n self.write(\"_\")\n elif pattern is None:\n self.write(node.name)\n else:\n with self.require_parens(_Precedence.TEST, node):\n self.set_precedence(_Precedence.BOR, node.pattern)\n self.traverse(node.pattern)\n self.write(f\" as {node.name}\")\n\n def visit_MatchOr(self, node):\n with self.require_parens(_Precedence.BOR, node):\n self.set_precedence(_Precedence.BOR.next(), *node.patterns)\n self.interleave(lambda: self.write(\" | \"), self.traverse, node.patterns)\n\n\nif sys.version_info >= (3, 8):\n\n def _is_int_literal(node: ast.AST) -> bool:\n \"\"\"Check if a node represents a literal int.\"\"\"\n return isinstance(node, ast.Constant) and isinstance(node.value, int)\n\n def _is_str_literal(node: ast.AST) -> bool:\n \"\"\"Check if a node represents a literal str.\"\"\"\n return isinstance(node, ast.Constant) and isinstance(node.value, str)\n\n def _get_str_literal_value(node: ast.AST) -> Optional[str]:\n \"\"\"Get the string value of a literal str node.\"\"\"\n if isinstance(node, ast.Constant) and isinstance(node.value, str):\n return node.value\n return None\n\nelse:\n\n def _is_int_literal(node: ast.AST) -> bool:\n \"\"\"Check if a node represents a literal int.\"\"\"\n return isinstance(node, ast.Num) and isinstance(node.n, int)\n\n def _is_str_literal(node: ast.AST) -> bool:\n \"\"\"Check if a node represents a literal str.\"\"\"\n return isinstance(node, ast.Str)\n\n def _get_str_literal_value(node: ast.AST) -> Optional[str]:\n \"\"\"Get the string value of a literal str node.\"\"\"\n return node.s if isinstance(node, ast.Str) else None\n\n\nif sys.version_info >= (3, 14):\n\n def _is_interpolation(node: ast.AST) -> bool:\n \"\"\"Check if a node represents a template string literal.\"\"\"\n return isinstance(node, ast.Interpolation)\n\nelse:\n\n def _is_interpolation(node: ast.AST) -> bool:\n \"\"\"Check if a node represents a template string literal.\"\"\"\n return False\n" }, { "path": "lib/spack/spack/util/url.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"\nUtility functions for parsing, formatting, and manipulating URLs.\n\"\"\"\n\nimport posixpath\nimport re\nimport urllib.parse\nimport urllib.request\nfrom pathlib import Path\nfrom typing import Optional\n\nfrom spack.util.path import sanitize_filename\n\n\ndef validate_scheme(scheme):\n \"\"\"Returns true if the URL scheme is generally known to Spack. This function\n helps mostly in validation of paths vs urls, as Windows paths such as\n C:/x/y/z (with backward not forward slash) may parse as a URL with scheme\n C and path /x/y/z.\"\"\"\n return scheme in (\"file\", \"http\", \"https\", \"ftp\", \"s3\", \"gs\", \"ssh\", \"git\", \"oci\")\n\n\ndef local_file_path(url):\n \"\"\"Get a local file path from a url.\n\n If url is a ``file://`` URL, return the absolute path to the local\n file or directory referenced by it. Otherwise, return None.\n \"\"\"\n if isinstance(url, str):\n url = urllib.parse.urlparse(url)\n\n if url.scheme == \"file\":\n return urllib.request.url2pathname(url.path)\n\n return None\n\n\ndef path_to_file_url(path):\n return Path(path).absolute().as_uri()\n\n\ndef file_url_string_to_path(url):\n return urllib.request.url2pathname(urllib.parse.urlparse(url).path)\n\n\ndef is_path_instead_of_url(path_or_url):\n \"\"\"Historically some config files and spack commands used paths\n where urls should be used. This utility can be used to validate\n and promote paths to urls.\"\"\"\n return not validate_scheme(urllib.parse.urlparse(path_or_url).scheme)\n\n\ndef format(parsed_url):\n \"\"\"Format a URL string\n\n Returns a canonicalized format of the given URL as a string.\n \"\"\"\n if isinstance(parsed_url, str):\n parsed_url = urllib.parse.urlparse(parsed_url)\n\n return parsed_url.geturl()\n\n\ndef join(base: str, *components: str, resolve_href: bool = False, **kwargs) -> str:\n \"\"\"Convenience wrapper around :func:`urllib.parse.urljoin`, with a few differences:\n\n 1. By default ``resolve_href=False``, which makes the function like :func:`os.path.join`.\n For example ``https://example.com/a/b + c/d = https://example.com/a/b/c/d``. If\n ``resolve_href=True``, the behavior is how a browser would resolve the URL:\n ``https://example.com/a/c/d``.\n 2. ``s3://``, ``gs://``, ``oci://`` URLs are joined like ``http://`` URLs.\n 3. It accepts multiple components for convenience. Note that ``components[1:]`` are treated as\n literal path components and appended to ``components[0]`` separated by slashes.\"\"\"\n # Ensure a trailing slash in the path component of the base URL to get os.path.join-like\n # behavior instead of web browser behavior.\n if not resolve_href:\n parsed = urllib.parse.urlparse(base)\n if not parsed.path.endswith(\"/\"):\n base = parsed._replace(path=f\"{parsed.path}/\").geturl()\n old_netloc = urllib.parse.uses_netloc\n old_relative = urllib.parse.uses_relative\n try:\n # NOTE: we temporarily modify urllib internals so s3 and gs schemes are treated like http.\n # This is non-portable, and may be forward incompatible with future cpython versions.\n urllib.parse.uses_netloc = [*old_netloc, \"s3\", \"gs\", \"oci\", \"oci+http\"] # type: ignore\n urllib.parse.uses_relative = [*old_relative, \"s3\", \"gs\", \"oci\", \"oci+http\"] # type: ignore\n return urllib.parse.urljoin(base, \"/\".join(components), **kwargs)\n finally:\n urllib.parse.uses_netloc = old_netloc # type: ignore\n urllib.parse.uses_relative = old_relative # type: ignore\n\n\ndef default_download_filename(url: str) -> str:\n \"\"\"This method computes a default file name for a given URL.\n Note that it makes no request, so this is not the same as the\n option curl -O, which uses the remote file name from the response\n header.\"\"\"\n parsed_url = urllib.parse.urlparse(url)\n # Only use the last path component + params + query + fragment\n name = urllib.parse.urlunparse(\n parsed_url._replace(scheme=\"\", netloc=\"\", path=posixpath.basename(parsed_url.path))\n )\n valid_name = sanitize_filename(name)\n\n # Don't download to hidden files please\n if valid_name[0] == \".\":\n valid_name = \"_\" + valid_name[1:]\n\n return valid_name\n\n\ndef parse_link_rel_next(link_value: str) -> Optional[str]:\n \"\"\"Return the next link from a Link header value, if any.\"\"\"\n\n # Relaxed version of RFC5988\n uri = re.compile(r\"\\s*<([^>]+)>\\s*\")\n param_key = r\"[^;=\\s]+\"\n quoted_string = r\"\\\"([^\\\"]+)\\\"\"\n unquoted_param_value = r\"([^;,\\s]+)\"\n param = re.compile(rf\";\\s*({param_key})\\s*=\\s*(?:{quoted_string}|{unquoted_param_value})\\s*\")\n\n data = link_value\n\n # Parse a list of ; key=value; key=value, ; key=value; key=value, ... links.\n while True:\n uri_match = re.match(uri, data)\n if not uri_match:\n break\n uri_reference = uri_match.group(1)\n data = data[uri_match.end() :]\n\n # Parse parameter list\n while True:\n param_match = re.match(param, data)\n if not param_match:\n break\n key, quoted_value, unquoted_value = param_match.groups()\n value = quoted_value or unquoted_value\n data = data[param_match.end() :]\n\n if key == \"rel\" and value == \"next\":\n return uri_reference\n\n if not data.startswith(\",\"):\n break\n\n data = data[1:]\n\n return None\n" }, { "path": "lib/spack/spack/util/web.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nimport email.message\nimport errno\nimport functools\nimport io\nimport json\nimport os\nimport re\nimport shutil\nimport socket\nimport ssl\nimport stat\nimport sys\nimport time\nimport traceback\nimport urllib.parse\nfrom html.parser import HTMLParser\nfrom http.client import IncompleteRead\nfrom pathlib import Path, PurePosixPath\nfrom typing import IO, Callable, Dict, Iterable, List, Optional, Set, Tuple, TypeVar, Union\nfrom urllib.error import HTTPError, URLError\nfrom urllib.request import HTTPDefaultErrorHandler, HTTPSHandler, Request, build_opener\n\nfrom spack.vendor.typing_extensions import ParamSpec\n\nimport spack\nimport spack.config\nimport spack.error\nimport spack.llnl.url\nimport spack.util.executable\nimport spack.util.parallel\nimport spack.util.path\nimport spack.util.url as url_util\nfrom spack.llnl.util import lang, tty\nfrom spack.llnl.util.filesystem import mkdirp, rename, working_dir\n\nfrom .executable import CommandNotFoundError, Executable\nfrom .gcs import GCSBlob, GCSBucket, GCSHandler\nfrom .s3 import UrllibS3Handler, get_s3_session\n\n\ndef is_transient_error(e: Exception) -> bool:\n \"\"\"Return True for HTTP/network errors that are worth retrying.\"\"\"\n\n if isinstance(e, HTTPError) and (500 <= e.code < 600 or e.code == 429):\n return True\n if isinstance(e, URLError) and isinstance(e.reason, socket.timeout):\n return True\n if isinstance(e, (socket.timeout, IncompleteRead)):\n return True\n # exceptions not inherited from the above used in urllib3 and botocore.\n if type(e).__name__ in (\n \"ConnectionClosedError\",\n \"IncompleteReadError\",\n \"ProtocolError\",\n \"ReadTimeoutError\",\n \"ResponseStreamingError\",\n ):\n return True\n return False\n\n\n_P = ParamSpec(\"_P\")\n_R = TypeVar(\"_R\")\n\n\ndef retry_on_transient_error(\n f: Callable[_P, _R], retries: int = 5, sleep: Optional[Callable[[float], None]] = None\n) -> Callable[_P, _R]:\n \"\"\"Retry a function on transient HTTP/network errors with exponential backoff.\"\"\"\n sleep = sleep or time.sleep\n\n @functools.wraps(f)\n def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R:\n for i in range(retries):\n try:\n return f(*args, **kwargs)\n except Exception as e:\n if i + 1 != retries and is_transient_error(e):\n sleep(2**i) # type: ignore[misc] # mypy still thinks it's possibly None.\n continue\n raise\n raise AssertionError(\"unreachable\")\n\n return wrapper\n\n\nclass DetailedHTTPError(HTTPError):\n def __init__(\n self, req: Request, code: int, msg: str, hdrs: email.message.Message, fp: Optional[IO]\n ) -> None:\n self.req = req\n super().__init__(req.get_full_url(), code, msg, hdrs, fp)\n\n def __str__(self):\n # Note: HTTPError, is actually a kind of non-seekable response object, so\n # best not to read the response body here (even if it may include a human-readable\n # error message).\n # Note: use self.filename, not self.url, because the latter requires fp to be an\n # IO object, which is not the case after unpickling.\n return f\"{self.req.get_method()} {self.filename} returned {self.code}: {self.msg}\"\n\n def __reduce__(self):\n # fp is an IO object and not picklable, the rest should be.\n return DetailedHTTPError, (self.req, self.code, self.msg, self.hdrs, None)\n\n\nclass DetailedURLError(URLError):\n def __init__(self, req: Request, reason):\n super().__init__(reason)\n self.req = req\n\n def __str__(self):\n return f\"{self.req.get_method()} {self.req.get_full_url()} errored with: {self.reason}\"\n\n def __reduce__(self):\n return DetailedURLError, (self.req, self.reason)\n\n\nclass SpackHTTPDefaultErrorHandler(HTTPDefaultErrorHandler):\n def http_error_default(self, req, fp, code, msg, hdrs):\n raise DetailedHTTPError(req, code, msg, hdrs, fp)\n\n\nclass SpackHTTPSHandler(HTTPSHandler):\n \"\"\"A custom HTTPS handler that shows more detailed error messages on connection failure.\"\"\"\n\n def https_open(self, req):\n try:\n return super().https_open(req)\n except HTTPError:\n raise\n except URLError as e:\n raise DetailedURLError(req, e.reason) from e\n\n\ndef custom_ssl_certs() -> Optional[Tuple[bool, str]]:\n \"\"\"Returns a tuple (is_file, path) if custom SSL certifates are configured and valid.\"\"\"\n ssl_certs = spack.config.get(\"config:ssl_certs\")\n if not ssl_certs:\n return None\n path = spack.util.path.substitute_path_variables(ssl_certs)\n if not os.path.isabs(path):\n tty.debug(f\"certs: relative path not allowed: {path}\")\n return None\n try:\n st = os.stat(path)\n except OSError as e:\n tty.debug(f\"certs: error checking path {path}: {e}\")\n return None\n\n file_type = stat.S_IFMT(st.st_mode)\n\n if file_type != stat.S_IFREG and file_type != stat.S_IFDIR:\n tty.debug(f\"certs: not a file or directory: {path}\")\n return None\n\n return (file_type == stat.S_IFREG, path)\n\n\ndef ssl_create_default_context():\n \"\"\"Create the default SSL context for urllib with custom certificates if configured.\"\"\"\n certs = custom_ssl_certs()\n if certs is None:\n return ssl.create_default_context()\n is_file, path = certs\n if is_file:\n tty.debug(f\"urllib: certs: using cafile {path}\")\n return ssl.create_default_context(cafile=path)\n else:\n tty.debug(f\"urllib: certs: using capath {path}\")\n return ssl.create_default_context(capath=path)\n\n\ndef set_curl_env_for_ssl_certs(curl: Executable) -> None:\n \"\"\"configure curl to use custom certs in a file at runtime. See:\n https://curl.se/docs/sslcerts.html item 4\"\"\"\n certs = custom_ssl_certs()\n if certs is None:\n return\n is_file, path = certs\n if not is_file:\n tty.debug(f\"curl: {path} is not a file: default certs will be used.\")\n return\n tty.debug(f\"curl: using CURL_CA_BUNDLE={path}\")\n curl.add_default_env(\"CURL_CA_BUNDLE\", path)\n\n\ndef _urlopen():\n s3 = UrllibS3Handler()\n gcs = GCSHandler()\n error_handler = SpackHTTPDefaultErrorHandler()\n\n # One opener with HTTPS ssl enabled\n with_ssl = build_opener(\n s3, gcs, SpackHTTPSHandler(context=ssl_create_default_context()), error_handler\n )\n\n # One opener with HTTPS ssl disabled\n without_ssl = build_opener(\n s3, gcs, SpackHTTPSHandler(context=ssl._create_unverified_context()), error_handler\n )\n\n # And dynamically dispatch based on the config:verify_ssl.\n def dispatch_open(fullurl, data=None, timeout=None):\n opener = with_ssl if spack.config.get(\"config:verify_ssl\", True) else without_ssl\n timeout = timeout or spack.config.get(\"config:connect_timeout\", 10)\n return opener.open(fullurl, data, timeout)\n\n return dispatch_open\n\n\n#: Dispatches to the correct OpenerDirector.open, based on Spack configuration.\nurlopen = lang.Singleton(_urlopen)\n\n#: User-Agent used in Request objects\nSPACK_USER_AGENT = \"Spackbot/{0}\".format(spack.spack_version)\n\n\n# Also, HTMLParseError is deprecated and never raised.\nclass HTMLParseError(Exception):\n pass\n\n\nclass LinkParser(HTMLParser):\n \"\"\"This parser just takes an HTML page and strips out the hrefs on the\n links, as well as some javascript tags used on GitLab servers.\n Good enough for a really simple spider.\"\"\"\n\n def __init__(self):\n super().__init__()\n self.links = []\n\n def handle_starttag(self, tag, attrs):\n if tag == \"a\":\n self.links.extend(val for key, val in attrs if key == \"href\")\n\n # GitLab uses a javascript function to place dropdown links:\n #
# noqa: E501\n if tag == \"div\" and (\"class\", \"js-source-code-dropdown\") in attrs:\n try:\n links_str = next(val for key, val in attrs if key == \"data-download-links\")\n links = json.loads(links_str)\n self.links.extend(x[\"path\"] for x in links)\n except Exception:\n pass\n\n\nclass ExtractMetadataParser(HTMLParser):\n \"\"\"This parser takes an HTML page and selects the include-fragments,\n used on GitHub, https://github.github.io/include-fragment-element,\n as well as a possible base url.\"\"\"\n\n def __init__(self):\n super().__init__()\n self.fragments = []\n self.base_url = None\n\n def handle_starttag(self, tag, attrs):\n # \n if tag == \"include-fragment\":\n for attr, val in attrs:\n if attr == \"src\":\n self.fragments.append(val)\n\n # \n elif tag == \"base\":\n for attr, val in attrs:\n if attr == \"href\":\n self.base_url = val\n\n\ndef read_from_url(url, accept_content_type=None):\n if isinstance(url, str):\n url = urllib.parse.urlparse(url)\n\n # Timeout in seconds for web requests\n request = Request(url.geturl(), headers={\"User-Agent\": SPACK_USER_AGENT})\n\n try:\n response = urlopen(request)\n except OSError as e:\n raise SpackWebError(f\"Download of {url.geturl()} failed: {e.__class__.__name__}: {e}\")\n\n if accept_content_type:\n try:\n content_type = get_header(response.headers, \"Content-type\")\n reject_content_type = not content_type.startswith(accept_content_type)\n except KeyError:\n content_type = None\n reject_content_type = True\n\n if reject_content_type:\n msg = \"ignoring page {}\".format(url.geturl())\n if content_type:\n msg += \" with content type {}\".format(content_type)\n tty.debug(msg)\n return None, None, None\n\n return response.url, response.headers, response\n\n\ndef _read_text(url: str) -> str:\n request = Request(url, headers={\"User-Agent\": SPACK_USER_AGENT})\n with urlopen(request) as response:\n return io.TextIOWrapper(response, encoding=\"utf-8\").read()\n\n\ndef _read_json(url: str):\n request = Request(url, headers={\"User-Agent\": SPACK_USER_AGENT})\n with urlopen(request) as response:\n return json.load(response)\n\n\n_read_text_with_retry = retry_on_transient_error(_read_text)\n_read_json_with_retry = retry_on_transient_error(_read_json)\n\n\ndef read_text(url: str) -> str:\n \"\"\"Fetch url and return the response body decoded as UTF-8 text.\"\"\"\n try:\n return _read_text_with_retry(url)\n except Exception as e:\n raise SpackWebError(f\"Download of {url} failed: {e.__class__.__name__}: {e}\")\n\n\ndef read_json(url: str):\n \"\"\"Fetch url and return the response body parsed as JSON.\"\"\"\n try:\n return _read_json_with_retry(url)\n except Exception as e:\n raise SpackWebError(f\"Download of {url} failed: {e.__class__.__name__}: {e}\")\n\n\ndef push_to_url(local_file_path, remote_path, keep_original=True, extra_args=None):\n remote_url = urllib.parse.urlparse(remote_path)\n if remote_url.scheme == \"file\":\n remote_file_path = url_util.local_file_path(remote_url)\n mkdirp(os.path.dirname(remote_file_path))\n if keep_original:\n shutil.copy(local_file_path, remote_file_path)\n else:\n try:\n rename(local_file_path, remote_file_path)\n except OSError as e:\n if e.errno == errno.EXDEV:\n # NOTE(opadron): The above move failed because it crosses\n # filesystem boundaries. Copy the file (plus original\n # metadata), and then delete the original. This operation\n # needs to be done in separate steps.\n shutil.copy2(local_file_path, remote_file_path)\n os.remove(local_file_path)\n else:\n raise\n\n elif remote_url.scheme == \"s3\":\n if extra_args is None:\n extra_args = {}\n\n remote_path = remote_url.path\n while remote_path.startswith(\"/\"):\n remote_path = remote_path[1:]\n\n s3 = get_s3_session(remote_url, method=\"push\")\n s3.upload_file(local_file_path, remote_url.netloc, remote_path, ExtraArgs=extra_args)\n\n if not keep_original:\n os.remove(local_file_path)\n\n elif remote_url.scheme == \"gs\":\n gcs = GCSBlob(remote_url)\n gcs.upload_to_blob(local_file_path)\n if not keep_original:\n os.remove(local_file_path)\n\n else:\n raise NotImplementedError(f\"Unrecognized URL scheme: {remote_url.scheme}\")\n\n\ndef base_curl_fetch_args(url, timeout=0):\n \"\"\"Return the basic fetch arguments typically used in calls to curl.\n\n The arguments include those for ensuring behaviors such as failing on\n errors for codes over 400, printing HTML headers, resolving 3xx redirects,\n status or failure handling, and connection timeouts.\n\n It also uses the following configuration option to set an additional\n argument as needed:\n\n * config:connect_timeout (int): connection timeout\n * config:verify_ssl (str): Perform SSL verification\n\n Arguments:\n url (str): URL whose contents will be fetched\n timeout (int): Connection timeout, which is only used if higher than\n config:connect_timeout\n\n Returns (list): list of argument strings\n \"\"\"\n curl_args = [\n \"-f\", # fail on >400 errors\n \"-D\",\n \"-\", # \"-D -\" prints out HTML headers\n \"-L\", # resolve 3xx redirects\n url,\n ]\n if not spack.config.get(\"config:verify_ssl\"):\n curl_args.append(\"-k\")\n\n if sys.stdout.isatty() and tty.msg_enabled():\n curl_args.append(\"-#\") # status bar when using a tty\n else:\n curl_args.append(\"-sS\") # show errors if fail\n\n connect_timeout = spack.config.get(\"config:connect_timeout\", 10)\n if timeout:\n connect_timeout = max(int(connect_timeout), int(timeout))\n if connect_timeout > 0:\n curl_args.extend([\"--connect-timeout\", str(connect_timeout)])\n\n return curl_args\n\n\ndef check_curl_code(returncode: int) -> None:\n \"\"\"Check standard return code failures for provided arguments.\n\n Arguments:\n returncode: curl return code\n\n Raises FetchError if the curl returncode indicates failure\n \"\"\"\n if returncode == 0:\n return\n elif returncode == 22:\n # This is a 404. Curl will print the error.\n raise spack.error.FetchError(\"URL was not found!\")\n elif returncode == 60:\n # This is a certificate error. Suggest spack -k\n raise spack.error.FetchError(\n \"Curl was unable to fetch due to invalid certificate. \"\n \"This is either an attack, or your cluster's SSL \"\n \"configuration is bad. If you believe your SSL \"\n \"configuration is bad, you can try running spack -k, \"\n \"which will not check SSL certificates.\"\n \"Use this at your own risk.\"\n )\n\n raise spack.error.FetchError(f\"Curl failed with error {returncode}\")\n\n\ndef require_curl() -> Executable:\n try:\n path = spack.util.executable.which_string(\"curl\", required=True)\n except CommandNotFoundError as e:\n raise spack.error.FetchError(f\"curl is required but not found: {e}\") from e\n curl = spack.util.executable.Executable(path)\n set_curl_env_for_ssl_certs(curl)\n return curl\n\n\ndef fetch_url_text(url, curl: Optional[Executable] = None, dest_dir=\".\"):\n \"\"\"Retrieves text-only URL content using the configured fetch method.\n It determines the fetch method from:\n\n * config:url_fetch_method (str): fetch method to use (e.g., 'curl')\n\n If the method is ``curl``, it also uses the following configuration\n options:\n\n * config:connect_timeout (int): connection time out\n * config:verify_ssl (str): Perform SSL verification\n\n Arguments:\n url (str): URL whose contents are to be fetched\n curl (spack.util.executable.Executable or None): (optional) curl\n executable if curl is the configured fetch method\n dest_dir (str): (optional) destination directory for fetched text\n file\n\n Returns (str or None): path to the fetched file\n\n Raises FetchError if the curl returncode indicates failure\n \"\"\"\n if not url:\n raise spack.error.FetchError(\"A URL is required to fetch its text\")\n\n tty.debug(\"Fetching text at {0}\".format(url))\n\n filename = os.path.basename(url)\n path = os.path.join(dest_dir, filename)\n\n fetch_method = spack.config.get(\"config:url_fetch_method\")\n tty.debug(\"Using '{0}' to fetch {1} into {2}\".format(fetch_method, url, path))\n if fetch_method and fetch_method.startswith(\"curl\"):\n curl_exe = curl or require_curl()\n curl_args = fetch_method.split()[1:] + [\"-O\"]\n curl_args.extend(base_curl_fetch_args(url))\n\n # Curl automatically downloads file contents as filename\n with working_dir(dest_dir, create=True):\n _ = curl_exe(*curl_args, fail_on_error=False, output=os.devnull)\n check_curl_code(curl_exe.returncode)\n\n return path\n\n else:\n try:\n output = read_text(url)\n if output:\n with working_dir(dest_dir, create=True):\n with open(filename, \"w\", encoding=\"utf-8\") as f:\n f.write(output)\n\n return path\n\n except (SpackWebError, OSError, ValueError) as err:\n raise spack.error.FetchError(f\"Urllib fetch failed: {err}\")\n\n return None\n\n\ndef _url_exists_urllib_impl(url):\n with urlopen(\n Request(url, method=\"HEAD\", headers={\"User-Agent\": SPACK_USER_AGENT}),\n timeout=spack.config.get(\"config:connect_timeout\", 10),\n ) as _:\n pass\n\n\n_url_exists_urllib = retry_on_transient_error(_url_exists_urllib_impl)\n\n\ndef url_exists(url, curl=None):\n \"\"\"Determines whether url exists.\n\n A scheme-specific process is used for Google Storage (``gs``) and Amazon\n Simple Storage Service (``s3``) URLs; otherwise, the configured fetch\n method defined by ``config:url_fetch_method`` is used.\n\n Arguments:\n url (str): URL whose existence is being checked\n curl (spack.util.executable.Executable or None): (optional) curl\n executable if curl is the configured fetch method\n\n Returns (bool): True if it exists; False otherwise.\n \"\"\"\n tty.debug(\"Checking existence of {0}\".format(url))\n url_result = urllib.parse.urlparse(url)\n\n # Use curl if configured to do so\n fetch_method = spack.config.get(\"config:url_fetch_method\", \"urllib\")\n use_curl = fetch_method.startswith(\"curl\") and url_result.scheme not in (\"gs\", \"s3\")\n if use_curl:\n curl_exe = curl or require_curl()\n\n # Telling curl to fetch the first byte (-r 0-0) is supposed to be\n # portable.\n curl_args = fetch_method.split()[1:] + [\"--stderr\", \"-\", \"-s\", \"-f\", \"-r\", \"0-0\", url]\n if not spack.config.get(\"config:verify_ssl\"):\n curl_args.append(\"-k\")\n _ = curl_exe(*curl_args, fail_on_error=False, output=os.devnull)\n return curl_exe.returncode == 0\n\n # Otherwise use urllib.\n try:\n _url_exists_urllib(url)\n return True\n except Exception as e:\n tty.debug(f\"Failure reading {url}: {e}\")\n return False\n\n\ndef _debug_print_delete_results(result):\n if \"Deleted\" in result:\n for d in result[\"Deleted\"]:\n tty.debug(\"Deleted {0}\".format(d[\"Key\"]))\n if \"Errors\" in result:\n for e in result[\"Errors\"]:\n tty.debug(\"Failed to delete {0} ({1})\".format(e[\"Key\"], e[\"Message\"]))\n\n\ndef remove_url(url, recursive=False):\n url = urllib.parse.urlparse(url)\n\n local_path = url_util.local_file_path(url)\n if local_path:\n if recursive:\n shutil.rmtree(local_path)\n else:\n os.remove(local_path)\n return\n\n if url.scheme == \"s3\":\n # Try to find a mirror for potential connection information\n s3 = get_s3_session(url, method=\"push\")\n bucket = url.netloc\n if recursive:\n # Because list_objects_v2 can only return up to 1000 items\n # at a time, we have to paginate to make sure we get it all\n prefix = url.path.strip(\"/\")\n paginator = s3.get_paginator(\"list_objects_v2\")\n pages = paginator.paginate(Bucket=bucket, Prefix=prefix)\n\n delete_request = {\"Objects\": []}\n for item in pages.search(\"Contents\"):\n if not item:\n continue\n\n delete_request[\"Objects\"].append({\"Key\": item[\"Key\"]})\n\n # Make sure we do not try to hit S3 with a list of more\n # than 1000 items\n if len(delete_request[\"Objects\"]) >= 1000:\n r = s3.delete_objects(Bucket=bucket, Delete=delete_request)\n _debug_print_delete_results(r)\n delete_request = {\"Objects\": []}\n\n # Delete any items that remain\n if len(delete_request[\"Objects\"]):\n r = s3.delete_objects(Bucket=bucket, Delete=delete_request)\n _debug_print_delete_results(r)\n else:\n s3.delete_object(Bucket=bucket, Key=url.path.lstrip(\"/\"))\n return\n\n elif url.scheme == \"gs\":\n if recursive:\n bucket = GCSBucket(url)\n bucket.destroy(recursive=recursive)\n else:\n blob = GCSBlob(url)\n blob.delete_blob()\n return\n\n # Don't even try for other URL schemes.\n\n\ndef _iter_s3_contents(contents, prefix):\n for entry in contents:\n key = entry[\"Key\"]\n\n if not key.startswith(\"/\"):\n key = \"/\" + key\n\n key = os.path.relpath(key, prefix)\n\n if key == \".\":\n continue\n\n yield key\n\n\ndef _list_s3_objects(client, bucket, prefix, num_entries, start_after=None):\n list_args = dict(Bucket=bucket, Prefix=prefix[1:], MaxKeys=num_entries)\n\n if start_after is not None:\n list_args[\"StartAfter\"] = start_after\n\n result = client.list_objects_v2(**list_args)\n\n last_key = None\n if result[\"IsTruncated\"]:\n last_key = result[\"Contents\"][-1][\"Key\"]\n\n iter = _iter_s3_contents(result[\"Contents\"], prefix)\n\n return iter, last_key\n\n\ndef _iter_s3_prefix(client, url, num_entries=1024):\n key = None\n bucket = url.netloc\n prefix = re.sub(r\"^/*\", \"/\", url.path)\n\n while True:\n contents, key = _list_s3_objects(client, bucket, prefix, num_entries, start_after=key)\n\n for x in contents:\n yield x\n\n if not key:\n break\n\n\ndef _iter_local_prefix(path):\n for root, _, files in os.walk(path):\n for f in files:\n yield os.path.relpath(os.path.join(root, f), path)\n\n\ndef list_url(url, recursive=False):\n url = urllib.parse.urlparse(url)\n local_path = url_util.local_file_path(url)\n\n if local_path:\n if recursive:\n # convert backslash to forward slash as required for URLs\n return [str(PurePosixPath(Path(p))) for p in _iter_local_prefix(local_path)]\n return [\n subpath\n for subpath in os.listdir(local_path)\n if os.path.isfile(os.path.join(local_path, subpath))\n ]\n\n if url.scheme == \"s3\":\n s3 = get_s3_session(url, method=\"fetch\")\n if recursive:\n return list(_iter_s3_prefix(s3, url))\n\n return list(set(key.split(\"/\", 1)[0] for key in _iter_s3_prefix(s3, url)))\n\n elif url.scheme == \"gs\":\n gcs = GCSBucket(url)\n return gcs.get_all_blobs(recursive=recursive)\n\n\ndef stat_url(url: str) -> Optional[Tuple[int, float]]:\n \"\"\"Get stat result for a URL.\n\n Args:\n url: URL to get stat result for\n Returns:\n A tuple of (size, mtime) if the URL exists, None otherwise.\n \"\"\"\n parsed_url = urllib.parse.urlparse(url)\n\n if parsed_url.scheme == \"file\":\n local_file_path = url_util.local_file_path(parsed_url)\n assert isinstance(local_file_path, str)\n try:\n url_stat = Path(local_file_path).stat()\n except FileNotFoundError:\n return None\n return url_stat.st_size, url_stat.st_mtime\n\n elif parsed_url.scheme == \"s3\":\n s3_bucket = parsed_url.netloc\n s3_key = parsed_url.path.lstrip(\"/\")\n\n s3 = get_s3_session(url, method=\"fetch\")\n\n try:\n head_request = s3.head_object(Bucket=s3_bucket, Key=s3_key)\n except s3.ClientError as e:\n if e.response[\"Error\"][\"Code\"] == \"404\":\n return None\n raise e\n\n mtime = head_request[\"LastModified\"].timestamp()\n size = head_request[\"ContentLength\"]\n return size, mtime\n\n else:\n raise NotImplementedError(f\"Unrecognized URL scheme: {parsed_url.scheme}\")\n\n\ndef spider(\n root_urls: Union[str, Iterable[str]], depth: int = 0, concurrency: Optional[int] = None\n):\n \"\"\"Get web pages from root URLs.\n\n If depth is specified (e.g., depth=2), then this will also follow up to levels\n of links from each root.\n\n Args:\n root_urls: root urls used as a starting point for spidering\n depth: level of recursion into links\n concurrency: number of simultaneous requests that can be sent\n\n Returns:\n A dict of pages visited (URL) mapped to their full text and the set of visited links.\n \"\"\"\n if isinstance(root_urls, str):\n root_urls = [root_urls]\n\n current_depth = 0\n pages, links, spider_args = {}, set(), []\n\n _visited: Set[str] = set()\n go_deeper = current_depth < depth\n for root_str in root_urls:\n root = urllib.parse.urlparse(root_str)\n spider_args.append((root, go_deeper, _visited))\n\n with spack.util.parallel.make_concurrent_executor(concurrency, require_fork=False) as tp:\n while current_depth <= depth:\n tty.debug(\n f\"SPIDER: [depth={current_depth}, max_depth={depth}, urls={len(spider_args)}]\"\n )\n results = [tp.submit(_spider, *one_search_args) for one_search_args in spider_args]\n spider_args = []\n go_deeper = current_depth < depth\n for future in results:\n sub_pages, sub_links, sub_spider_args, sub_visited = future.result()\n _visited.update(sub_visited)\n sub_spider_args = [(x, go_deeper, _visited) for x in sub_spider_args]\n pages.update(sub_pages)\n links.update(sub_links)\n spider_args.extend(sub_spider_args)\n\n current_depth += 1\n\n return pages, links\n\n\ndef _spider(url: urllib.parse.ParseResult, collect_nested: bool, _visited: Set[str]):\n \"\"\"Fetches URL and any pages it links to.\n\n Prints out a warning only if the root can't be fetched; it ignores errors with pages\n that the root links to.\n\n Args:\n url: url being fetched and searched for links\n collect_nested: whether we want to collect arguments for nested spidering on the\n links found in this url\n _visited: links already visited\n\n Returns:\n A tuple of:\n - pages: dict of pages visited (URL) mapped to their full text.\n - links: set of links encountered while visiting the pages.\n - spider_args: argument for subsequent call to spider\n - visited: updated set of visited urls\n \"\"\"\n pages: Dict[str, str] = {} # dict from page URL -> text content.\n links: Set[str] = set() # set of all links seen on visited pages.\n subcalls: List[str] = []\n\n try:\n response_url, _, response = read_from_url(url, \"text/html\")\n if not response_url or not response:\n return pages, links, subcalls, _visited\n\n with response:\n page = io.TextIOWrapper(response, encoding=\"utf-8\").read()\n pages[response_url] = page\n\n # Parse out the include-fragments in the page\n # https://github.github.io/include-fragment-element\n metadata_parser = ExtractMetadataParser()\n metadata_parser.feed(page)\n\n # Change of base URL due to tag\n response_url = metadata_parser.base_url or response_url\n\n fragments = set()\n while metadata_parser.fragments:\n raw_link = metadata_parser.fragments.pop()\n abs_link = url_util.join(response_url, raw_link.strip(), resolve_href=True)\n\n fragment_response_url = None\n try:\n # This seems to be text/html, though text/fragment+html is also used\n fragment_response_url, _, fragment_response = read_from_url(abs_link, \"text/html\")\n except Exception as e:\n msg = f\"Error reading fragment: {(type(e), str(e))}:{traceback.format_exc()}\"\n tty.debug(msg)\n\n if not fragment_response_url or not fragment_response:\n continue\n\n with fragment_response:\n fragment = io.TextIOWrapper(fragment_response, encoding=\"utf-8\").read()\n fragments.add(fragment)\n\n pages[fragment_response_url] = fragment\n\n # Parse out the links in the page and all fragments\n link_parser = LinkParser()\n link_parser.feed(page)\n for fragment in fragments:\n link_parser.feed(fragment)\n\n while link_parser.links:\n raw_link = link_parser.links.pop()\n abs_link = url_util.join(response_url, raw_link.strip(), resolve_href=True)\n links.add(abs_link)\n\n # Skip stuff that looks like an archive\n if any(raw_link.endswith(s) for s in spack.llnl.url.ALLOWED_ARCHIVE_TYPES):\n continue\n\n # Skip already-visited links\n if abs_link in _visited:\n continue\n\n # If we're not at max depth, follow links.\n if collect_nested:\n subcalls.append(abs_link)\n _visited.add(abs_link)\n\n except OSError as e:\n tty.debug(f\"[SPIDER] Unable to read: {url}\")\n tty.debug(str(e), level=2)\n if isinstance(e, URLError) and isinstance(e.reason, ssl.SSLError):\n tty.warn(\n \"Spack was unable to fetch url list due to a \"\n \"certificate verification problem. You can try \"\n \"running spack -k, which will not check SSL \"\n \"certificates. Use this at your own risk.\"\n )\n\n except HTMLParseError as e:\n # This error indicates that Python's HTML parser sucks.\n msg = \"Got an error parsing HTML.\"\n tty.warn(msg, url, \"HTMLParseError: \" + str(e))\n\n except Exception as e:\n # Other types of errors are completely ignored,\n # except in debug mode\n tty.debug(f\"Error in _spider: {type(e)}:{str(e)}\", traceback.format_exc())\n\n finally:\n tty.debug(f\"SPIDER: [url={url}]\")\n\n return pages, links, subcalls, _visited\n\n\ndef get_header(headers, header_name):\n \"\"\"Looks up a dict of headers for the given header value.\n\n Looks up a dict of headers, [headers], for a header value given by\n [header_name]. Returns headers[header_name] if header_name is in headers.\n Otherwise, the first fuzzy match is returned, if any.\n\n This fuzzy matching is performed by discarding word separators and\n capitalization, so that for example, \"Content-length\", \"content_length\",\n \"conTENtLength\", etc., all match. In the case of multiple fuzzy-matches,\n the returned value is the \"first\" such match given the underlying mapping's\n ordering, or unspecified if no such ordering is defined.\n\n If header_name is not in headers, and no such fuzzy match exists, then a\n KeyError is raised.\n \"\"\"\n\n def unfuzz(header):\n return re.sub(r\"[ _-]\", \"\", header).lower()\n\n try:\n return headers[header_name]\n except KeyError:\n unfuzzed_header_name = unfuzz(header_name)\n for header, value in headers.items():\n if unfuzz(header) == unfuzzed_header_name:\n return value\n raise\n\n\ndef parse_etag(header_value):\n \"\"\"Parse a strong etag from an ETag: header value.\n We don't allow for weakness indicators because it's unclear\n what that means for cache invalidation.\"\"\"\n if header_value is None:\n return None\n\n # First follow rfc7232 section 2.3 mostly:\n # ETag = entity-tag\n # entity-tag = [ weak ] opaque-tag\n # weak = %x57.2F ; \"W/\", case-sensitive\n # opaque-tag = DQUOTE *etagc DQUOTE\n # etagc = %x21 / %x23-7E / obs-text\n # ; VCHAR except double quotes, plus obs-text\n # obs-text = %x80-FF\n\n # That means quotes are required.\n valid = re.match(r'\"([\\x21\\x23-\\x7e\\x80-\\xFF]+)\"$', header_value)\n if valid:\n return valid.group(1)\n\n # However, not everybody adheres to the RFC (some servers send\n # wrong etags, but also s3:// is simply a different standard).\n # In that case, it's common that quotes are omitted, everything\n # else stays the same.\n valid = re.match(r\"([\\x21\\x23-\\x7e\\x80-\\xFF]+)$\", header_value)\n\n return valid.group(1) if valid else None\n\n\nclass SpackWebError(spack.error.SpackError):\n \"\"\"Superclass for Spack web spidering errors.\"\"\"\n\n\nclass NoNetworkConnectionError(SpackWebError):\n \"\"\"Raised when an operation can't get an internet connection.\"\"\"\n\n def __init__(self, message, url):\n super().__init__(\"No network connection: \" + str(message), \"URL was: \" + str(url))\n self.url = url\n" }, { "path": "lib/spack/spack/util/windows_registry.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"\nUtility module for dealing with Windows Registry.\n\"\"\"\n\nimport os\nimport re\nimport sys\nfrom contextlib import contextmanager\n\nfrom spack.llnl.util import tty\n\nif sys.platform == \"win32\":\n import winreg\n\n\nclass RegistryValue:\n \"\"\"\n Class defining a Windows registry entry\n \"\"\"\n\n def __init__(self, name, value, parent_key):\n self.path = name\n self.value = value\n self.key = parent_key\n\n\nclass RegistryKey:\n \"\"\"\n Class wrapping a Windows registry key\n \"\"\"\n\n def __init__(self, name, handle):\n self.path = name\n self.name = os.path.split(name)[-1]\n self._handle = handle\n self._keys = []\n self._values = {}\n\n @property\n def values(self):\n \"\"\"Returns all subvalues of this key as RegistryValue objects in dictionary\n of value name : RegistryValue object\n \"\"\"\n self._gather_value_info()\n return self._values\n\n @property\n def subkeys(self):\n \"\"\"Returns list of all subkeys of this key as RegistryKey objects\"\"\"\n self._gather_subkey_info()\n return self._keys\n\n @property\n def hkey(self):\n return self._handle\n\n @contextmanager\n def winreg_error_handler(self, name, *args, **kwargs):\n try:\n yield\n except OSError as err:\n # Expected errors that occur on occasion, these are easily\n # debug-able and have sufficiently verbose reporting and obvious cause\n # [WinError 2]: the system cannot find the file specified - lookup item does\n # not exist\n # [WinError 5]: Access is denied - user not in key's ACL\n if hasattr(err, \"winerror\") and err.winerror in (5, 2):\n raise err\n # Other OS errors are more difficult to diagnose, so we wrap them in some extra\n # reporting\n raise InvalidRegistryOperation(name, err, *args, **kwargs) from err\n\n def OpenKeyEx(self, subname, **kwargs):\n \"\"\"Convenience wrapper around winreg.OpenKeyEx\"\"\"\n tty.debug(\n f\"[WINREG ACCESS] Accessing Reg Key {self.path}/{subname} with\"\n f\" {kwargs.get('access', 'default')} access\"\n )\n with self.winreg_error_handler(\"OpenKeyEx\", subname, **kwargs):\n return winreg.OpenKeyEx(self.hkey, subname, **kwargs)\n\n def QueryInfoKey(self):\n \"\"\"Convenience wrapper around winreg.QueryInfoKey\"\"\"\n tty.debug(f\"[WINREG ACCESS] Obtaining key,value information from key {self.path}\")\n with self.winreg_error_handler(\"QueryInfoKey\"):\n return winreg.QueryInfoKey(self.hkey)\n\n def EnumKey(self, index):\n \"\"\"Convenience wrapper around winreg.EnumKey\"\"\"\n tty.debug(\n \"[WINREG ACCESS] Obtaining name of subkey at index \"\n f\"{index} from registry key {self.path}\"\n )\n with self.winreg_error_handler(\"EnumKey\", index):\n return winreg.EnumKey(self.hkey, index)\n\n def EnumValue(self, index):\n \"\"\"Convenience wrapper around winreg.EnumValue\"\"\"\n tty.debug(\n f\"[WINREG ACCESS] Obtaining value at index {index} from registry key {self.path}\"\n )\n with self.winreg_error_handler(\"EnumValue\", index):\n return winreg.EnumValue(self.hkey, index)\n\n def QueryValueEx(self, name, **kwargs):\n \"\"\"Convenience wrapper around winreg.QueryValueEx\"\"\"\n tty.debug(f\"[WINREG ACCESS] Obtaining value {name} from registry key {self.path}\")\n with self.winreg_error_handler(\"QueryValueEx\", name, **kwargs):\n return winreg.QueryValueEx(self.hkey, name, **kwargs)\n\n def __str__(self):\n return self.name\n\n def _gather_subkey_info(self):\n \"\"\"Composes all subkeys into a list for access\"\"\"\n if self._keys:\n return\n sub_keys, _, _ = self.QueryInfoKey()\n for i in range(sub_keys):\n sub_name = self.EnumKey(i)\n try:\n sub_handle = self.OpenKeyEx(sub_name, access=winreg.KEY_READ)\n self._keys.append(RegistryKey(os.path.join(self.path, sub_name), sub_handle))\n except OSError as e:\n if hasattr(e, \"winerror\") and e.winerror == 5:\n # This is a permission error, we can't read this key\n # move on\n pass\n else:\n raise\n\n def _gather_value_info(self):\n \"\"\"Compose all values for this key into a dict of form value name: RegistryValue Object\"\"\"\n if self._values:\n return\n _, values, _ = self.QueryInfoKey()\n for i in range(values):\n value_name, value_data, _ = self.EnumValue(i)\n self._values[value_name] = RegistryValue(value_name, value_data, self)\n\n def get_subkey(self, sub_key):\n \"\"\"Returns subkey of name sub_key in a RegistryKey objects\"\"\"\n return RegistryKey(\n os.path.join(self.path, sub_key), self.OpenKeyEx(sub_key, access=winreg.KEY_READ)\n )\n\n def get_value(self, val_name):\n \"\"\"Returns value associated with this key in RegistryValue object\"\"\"\n return RegistryValue(val_name, self.QueryValueEx(val_name)[0], self)\n\n\nclass _HKEY_CONSTANT(RegistryKey):\n \"\"\"Subclass of RegistryKey to represent the prebaked, always open registry HKEY constants\"\"\"\n\n def __init__(self, hkey_constant):\n hkey_name = hkey_constant\n # This class is instantiated at module import time\n # on non Windows platforms, winreg would not have been\n # imported. For this reason we can't reference winreg yet,\n # so handle is none for now to avoid invalid references to a module.\n # _handle provides a workaround to prevent null references to self.handle\n # when coupled with the handle property.\n super(_HKEY_CONSTANT, self).__init__(hkey_name, None)\n\n def _get_hkey(self, key):\n return getattr(winreg, key)\n\n @property\n def hkey(self):\n if not self._handle:\n self._handle = self._get_hkey(self.path)\n return self._handle\n\n\nclass HKEY:\n \"\"\"\n Predefined, open registry HKEYs\n From the Microsoft docs:\n An application must open a key before it can read data from the registry.\n To open a key, an application must supply a handle to another key in\n the registry that is already open. The system defines predefined keys\n that are always open. Predefined keys help an application navigate in\n the registry.\"\"\"\n\n HKEY_CLASSES_ROOT = _HKEY_CONSTANT(\"HKEY_CLASSES_ROOT\")\n HKEY_CURRENT_USER = _HKEY_CONSTANT(\"HKEY_CURRENT_USER\")\n HKEY_USERS = _HKEY_CONSTANT(\"HKEY_USERS\")\n HKEY_LOCAL_MACHINE = _HKEY_CONSTANT(\"HKEY_LOCAL_MACHINE\")\n HKEY_CURRENT_CONFIG = _HKEY_CONSTANT(\"HKEY_CURRENT_CONFIG\")\n HKEY_PERFORMANCE_DATA = _HKEY_CONSTANT(\"HKEY_PERFORMANCE_DATA\")\n\n\nclass WindowsRegistryView:\n \"\"\"\n Interface to provide access, querying, and searching to Windows registry entries.\n This class represents a single key entrypoint into the Windows registry\n and provides an interface to this key's values, its subkeys, and those subkey's values.\n This class cannot be used to move freely about the registry, only subkeys/values of\n the root key used to instantiate this class.\n \"\"\"\n\n def __init__(self, key, root_key=HKEY.HKEY_CURRENT_USER):\n \"\"\"Constructs a Windows Registry entrypoint to key provided\n root_key should be an already open root key or an hkey constant if provided\n\n Args:\n key (str): registry key to provide root for registry key for this clas\n root_key: Already open registry key or HKEY constant to provide access into\n the Windows registry. Registry access requires an already open key\n to get an entrypoint, the HKEY constants are always open, or an already\n open key can be used instead.\n \"\"\"\n if sys.platform != \"win32\":\n raise RuntimeError(\n \"Cannot instantiate Windows Registry class on non Windows platforms\"\n )\n self.key = key\n self.root = root_key\n self._reg = None\n\n class KeyMatchConditions:\n @staticmethod\n def regex_matcher(subkey_name):\n return lambda x: re.match(subkey_name, x.name)\n\n @staticmethod\n def name_matcher(subkey_name):\n return lambda x: subkey_name == x.name\n\n @contextmanager\n def invalid_reg_ref_error_handler(self):\n try:\n yield\n except FileNotFoundError as e:\n if sys.platform == \"win32\" and e.winerror == 2:\n tty.debug(\"Key %s at position %s does not exist\" % (self.key, str(self.root)))\n else:\n raise e\n\n def __bool__(self):\n return self.reg != -1\n\n def _load_key(self):\n try:\n self._reg = self.root.get_subkey(self.key)\n except FileNotFoundError as e:\n if sys.platform == \"win32\" and e.winerror == 2:\n self._reg = -1\n tty.debug(\"Key %s at position %s does not exist\" % (self.key, str(self.root)))\n else:\n raise e\n\n def _valid_reg_check(self):\n if self.reg == -1:\n tty.debug(f\"[WINREG ACCESS] Cannot perform operation for nonexistent key {self.key}\")\n return False\n return True\n\n def _regex_match_subkeys(self, subkey):\n r_subkey = re.compile(subkey)\n return [key for key in self.get_subkeys() if r_subkey.match(key.name)]\n\n @property\n def reg(self):\n if not self._reg:\n self._load_key()\n return self._reg\n\n def get_value(self, value_name):\n \"\"\"Return registry value corresponding to provided argument (if it exists)\"\"\"\n if not self._valid_reg_check():\n raise RegistryError(f\"Cannot query value from invalid key {self.key}\")\n with self.invalid_reg_ref_error_handler():\n return self.reg.get_value(value_name)\n\n def get_subkey(self, subkey_name):\n if not self._valid_reg_check():\n raise RegistryError(f\"Cannot query subkey from invalid key {self.key}\")\n with self.invalid_reg_ref_error_handler():\n return self.reg.get_subkey(subkey_name)\n\n def get_subkeys(self):\n if not self._valid_reg_check():\n raise RegistryError(f\"Cannot query subkeys from invalid key {self.key}\")\n with self.invalid_reg_ref_error_handler():\n return self.reg.subkeys\n\n def get_matching_subkeys(self, subkey_name):\n \"\"\"Returns all subkeys regex matching subkey name\n\n Note: this method obtains only direct subkeys of the given key and does not\n descend to transitive subkeys. For this behavior, see ``find_matching_subkeys``\"\"\"\n self._regex_match_subkeys(subkey_name)\n\n def get_values(self):\n if not self._valid_reg_check():\n raise RegistryError(f\"Cannot query values from invalid key {self.key}\")\n with self.invalid_reg_ref_error_handler():\n return self.reg.values\n\n def _traverse_subkeys(self, stop_condition, collect_all_matching=False, recursive=True):\n \"\"\"Perform simple BFS of subkeys, returning the key\n that successfully triggers the stop condition.\n Args:\n stop_condition: lambda or function pointer that takes a single argument\n a key and returns a boolean value based on that key\n collect_all_matching: boolean value, if True, the traversal collects and returns\n all keys meeting stop condition. If false, once stop\n condition is met, the key that triggered the condition '\n is returned.\n recursive: boolean value, if True perform a recursive search of subkeys\n Return:\n the key if stop_condition is triggered, or None if not\n \"\"\"\n collection = []\n if not self._valid_reg_check():\n raise InvalidKeyError(self.key)\n with self.invalid_reg_ref_error_handler():\n queue = self.reg.subkeys\n for key in queue:\n if stop_condition(key):\n if collect_all_matching:\n collection.append(key)\n else:\n return key\n if recursive:\n queue.extend(key.subkeys)\n return collection if collection else None\n\n def find_subkey(self, subkey_name: str, recursive: bool = True):\n \"\"\"Perform a BFS of subkeys until desired key is found\n Returns None or RegistryKey object corresponding to requested key name\n\n Args:\n subkey_name: subkey to be searched for\n recursive: perform a recursive search\n Return:\n the desired subkey as a RegistryKey object, or none\n \"\"\"\n return self._traverse_subkeys(\n WindowsRegistryView.KeyMatchConditions.name_matcher(subkey_name), recursive=recursive\n )\n\n def find_matching_subkey(self, subkey_name: str, recursive: bool = True):\n \"\"\"Perform a BFS of subkeys until a key matching subkey name regex is found\n Returns None or the first RegistryKey object corresponding to requested key name\n\n Args:\n subkey_name: subkey to be searched for\n recursive: perform a recursive search\n Return:\n the desired subkey as a RegistryKey object, or none\n \"\"\"\n return self._traverse_subkeys(\n WindowsRegistryView.KeyMatchConditions.regex_matcher(subkey_name), recursive=recursive\n )\n\n def find_subkeys(self, subkey_name: str, recursive: bool = True):\n \"\"\"Exactly the same as find_subkey, except this function tries to match\n a regex to multiple keys\n\n Args:\n subkey_name: subkey to be searched for\n Return:\n the desired subkeys as a list of RegistryKey object, or none\n \"\"\"\n kwargs = {\"collect_all_matching\": True, \"recursive\": recursive}\n return self._traverse_subkeys(\n WindowsRegistryView.KeyMatchConditions.regex_matcher(subkey_name), **kwargs\n )\n\n def find_value(self, val_name: str, recursive: bool = True):\n \"\"\"\n If non recursive, return RegistryValue object corresponding to name\n\n Args:\n val_name: name of value desired from registry\n recursive: optional argument, if True, the registry is searched recursively\n for the value of name val_name, else only the current key is searched\n Return:\n The desired registry value as a RegistryValue object if it exists, otherwise, None\n \"\"\"\n if not recursive:\n return self.get_value(val_name)\n\n else:\n key = self._traverse_subkeys(lambda x: val_name in x.values)\n if not key:\n return None\n else:\n return key.values[val_name]\n\n\nclass RegistryError(Exception):\n \"\"\"RunTime Error concerning the Windows Registry\"\"\"\n\n\nclass InvalidKeyError(RegistryError):\n \"\"\"Runtime Error describing issue with invalid key access to Windows registry\"\"\"\n\n def __init__(self, key):\n message = f\"Cannot query invalid key: {key}\"\n super().__init__(message)\n\n\nclass InvalidRegistryOperation(RegistryError):\n \"\"\"A Runtime Error encountered when a registry operation is invalid for\n an indeterminate reason\"\"\"\n\n def __init__(self, name, e, *args, **kwargs):\n message = (\n f\"Windows registry operations: {name} encountered error: {str(e)}\"\n \"\\nMethod invoked with parameters:\\n\"\n )\n message += \"\\n\\t\".join([f\"{k}:{v}\" for k, v in kwargs.items()])\n message += \"\\n\"\n message += \"\\n\\t\".join(args)\n super().__init__(self, message)\n" }, { "path": "lib/spack/spack/variant.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\n\"\"\"The variant module contains data structures that are needed to manage\nvariants both in packages and in specs.\n\"\"\"\n\nimport collections.abc\nimport enum\nimport functools\nimport inspect\nimport itertools\nfrom typing import (\n TYPE_CHECKING,\n Any,\n Callable,\n Collection,\n Iterable,\n List,\n Optional,\n Set,\n Tuple,\n Type,\n Union,\n)\n\nimport spack.error\nimport spack.llnl.util.lang as lang\nimport spack.llnl.util.tty.color\nimport spack.spec_parser\n\nif TYPE_CHECKING:\n import spack.package_base\n import spack.spec\n\n#: These are variant names used by Spack internally; packages can't use them\nRESERVED_NAMES = {\n \"arch\",\n \"architecture\",\n \"branch\",\n \"commit\",\n \"dev_path\",\n \"namespace\",\n \"operating_system\",\n \"os\",\n \"patches\",\n \"platform\",\n \"ref\",\n \"tag\",\n \"target\",\n}\n\n\nclass VariantType(enum.IntEnum):\n \"\"\"Enum representing the three concrete variant types.\"\"\"\n\n BOOL = 1\n SINGLE = 2\n MULTI = 3\n INDICATOR = 4 # special type for placeholder variant values\n\n @property\n def string(self) -> str:\n \"\"\"Convert the variant type to a string.\"\"\"\n if self == VariantType.BOOL:\n return \"bool\"\n elif self == VariantType.SINGLE:\n return \"single\"\n elif self == VariantType.MULTI:\n return \"multi\"\n else:\n return \"indicator\"\n\n\nclass Variant:\n \"\"\"Represents a variant definition, created by the ``variant()`` directive.\n\n There can be multiple definitions of the same variant, and they are given precedence\n by order of appearance in the package. Later definitions have higher precedence.\n Similarly, definitions in derived classes have higher precedence than those in their\n superclasses.\n\n \"\"\"\n\n name: str\n default: Union[bool, str]\n description: str\n values: Optional[Collection] #: if None, valid values are defined only by validators\n multi: bool\n single_value_validator: Callable\n group_validator: Optional[Callable]\n sticky: bool\n precedence: int\n\n def __init__(\n self,\n name: str,\n *,\n default: Union[bool, str],\n description: str,\n values: Union[Collection, Callable] = (True, False),\n multi: bool = False,\n validator: Optional[Callable] = None,\n sticky: bool = False,\n precedence: int = 0,\n ):\n \"\"\"Initialize a package variant.\n\n Args:\n name: name of the variant\n default: default value for the variant, used when nothing is explicitly specified\n description: purpose of the variant\n values: sequence of allowed values or a callable accepting a single value as argument\n and returning True if the value is good, False otherwise\n multi: whether multiple values are allowed\n validator: optional callable that can be used to perform additional validation\n sticky: if true the variant is set to the default value at concretization time\n precedence: int indicating precedence of this variant definition in the solve\n (definition with highest precedence is used when multiple definitions are possible)\n \"\"\"\n self.name = name\n self.default = default\n self.description = str(description)\n\n self.values = None\n if values == \"*\":\n # wildcard is a special case to make it easy to say any value is ok\n self.single_value_validator = lambda v: True\n\n elif isinstance(values, type):\n # supplying a type means any value *of that type*\n def isa_type(v):\n try:\n values(v)\n return True\n except ValueError:\n return False\n\n self.single_value_validator = isa_type\n\n elif callable(values):\n # If 'values' is a callable, assume it is a single value\n # validator and reset the values to be explicit during debug\n self.single_value_validator = values\n else:\n # Otherwise, assume values is the set of allowed explicit values\n values = _flatten(values)\n self.values = values\n self.single_value_validator = lambda v: v in values\n\n self.multi = multi\n self.group_validator = validator\n self.sticky = sticky\n self.precedence = precedence\n\n def values_defined_by_validator(self) -> bool:\n return self.values is None\n\n def validate_or_raise(self, vspec: \"VariantValue\", pkg_name: str):\n \"\"\"Validate a variant spec against this package variant. Raises an\n exception if any error is found.\n\n Args:\n vspec: variant spec to be validated\n pkg_name: the name of the package class that required this validation (for errors)\n\n Raises:\n InconsistentValidationError: if ``vspec.name != self.name``\n\n MultipleValuesInExclusiveVariantError: if ``vspec`` has\n multiple values but ``self.multi == False``\n\n InvalidVariantValueError: if ``vspec.value`` contains\n invalid values\n \"\"\"\n # Check the name of the variant\n if self.name != vspec.name:\n raise InconsistentValidationError(vspec, self)\n\n # If the value is exclusive there must be at most one\n value = vspec.values\n if not self.multi and len(value) != 1:\n raise MultipleValuesInExclusiveVariantError(vspec, pkg_name)\n\n # Check and record the values that are not allowed\n invalid_vals = \", \".join(\n f\"'{v}'\" for v in value if v != \"*\" and self.single_value_validator(v) is False\n )\n if invalid_vals:\n raise InvalidVariantValueError(\n f\"invalid values for variant '{self.name}' in package {pkg_name}: {invalid_vals}\\n\"\n )\n\n # Validate the group of values if needed\n if self.group_validator is not None and value != (\"*\",):\n self.group_validator(pkg_name, self.name, value)\n\n @property\n def allowed_values(self):\n \"\"\"Returns a string representation of the allowed values for\n printing purposes\n\n Returns:\n str: representation of the allowed values\n \"\"\"\n # Join an explicit set of allowed values\n if self.values is not None:\n v = tuple(str(x) for x in self.values)\n return \", \".join(v)\n # In case we were given a single-value validator\n # print the docstring\n docstring = inspect.getdoc(self.single_value_validator)\n v = docstring if docstring else \"\"\n return v\n\n def make_default(self) -> \"VariantValue\":\n \"\"\"Factory that creates a variant holding the default value(s).\"\"\"\n variant = VariantValue.from_string_or_bool(self.name, self.default)\n variant.type = self.variant_type\n return variant\n\n def make_variant(self, *value: Union[str, bool]) -> \"VariantValue\":\n \"\"\"Factory that creates a variant holding the value(s) passed.\"\"\"\n return VariantValue(self.variant_type, self.name, value)\n\n @property\n def variant_type(self) -> VariantType:\n \"\"\"String representation of the type of this variant (single/multi/bool)\"\"\"\n if self.multi:\n return VariantType.MULTI\n elif self.values == (True, False):\n return VariantType.BOOL\n else:\n return VariantType.SINGLE\n\n def __str__(self) -> str:\n return (\n f\"Variant('{self.name}', \"\n f\"default='{self.default}', \"\n f\"description='{self.description}', \"\n f\"values={self.values}, \"\n f\"multi={self.multi}, \"\n f\"single_value_validator={self.single_value_validator}, \"\n f\"group_validator={self.group_validator}, \"\n f\"sticky={self.sticky}, \"\n f\"precedence={self.precedence})\"\n )\n\n\ndef _flatten(values) -> Collection:\n \"\"\"Flatten instances of _ConditionalVariantValues for internal representation\"\"\"\n if isinstance(values, DisjointSetsOfValues):\n return values\n\n flattened: List = []\n for item in values:\n if isinstance(item, ConditionalVariantValues):\n flattened.extend(item)\n else:\n flattened.append(item)\n # There are parts of the variant checking mechanism that expect to find tuples\n # here, so it is important to convert the type once we flattened the values.\n return tuple(flattened)\n\n\n#: Type for value of a variant\nValueType = Tuple[Union[bool, str], ...]\n\n#: Type of variant value when output for JSON, YAML, etc.\nSerializedValueType = Union[str, bool, List[Union[str, bool]]]\n\n\n@lang.lazy_lexicographic_ordering\nclass VariantValue:\n \"\"\"A VariantValue is a key-value pair that represents a variant. It can have zero or more\n values. Values have set semantics, so they are unordered and unique. The variant type can\n be narrowed from multi to single to boolean, this limits the number of values that can be\n stored in the variant. Multi-valued variants can either be concrete or abstract: abstract\n means that the variant takes at least the values specified, but may take more when concretized.\n Concrete means that the variant takes exactly the values specified. Lastly, a variant can be\n marked as propagating, which means that it should be propagated to dependencies.\"\"\"\n\n name: str\n propagate: bool\n concrete: bool\n type: VariantType\n _values: ValueType\n\n slots = (\"name\", \"propagate\", \"concrete\", \"type\", \"_values\")\n\n def __init__(\n self,\n type: VariantType,\n name: str,\n value: ValueType,\n *,\n propagate: bool = False,\n concrete: bool = False,\n ) -> None:\n self.name = name\n self.type = type\n self.propagate = propagate\n # only multi-valued variants can be abstract\n self.concrete = concrete or type in (VariantType.BOOL, VariantType.SINGLE)\n\n # Invokes property setter\n self.set(*value)\n\n @staticmethod\n def from_node_dict(\n name: str, value: Union[str, List[str]], *, propagate: bool = False, abstract: bool = False\n ) -> \"VariantValue\":\n \"\"\"Reconstruct a variant from a node dict.\"\"\"\n if isinstance(value, list):\n return VariantValue(\n VariantType.MULTI, name, tuple(value), propagate=propagate, concrete=not abstract\n )\n\n # todo: is this necessary? not literal true / false in json/yaml?\n elif str(value).upper() == \"TRUE\" or str(value).upper() == \"FALSE\":\n return VariantValue(\n VariantType.BOOL, name, (str(value).upper() == \"TRUE\",), propagate=propagate\n )\n\n return VariantValue(VariantType.SINGLE, name, (value,), propagate=propagate)\n\n @staticmethod\n def from_string_or_bool(\n name: str, value: Union[str, bool], *, propagate: bool = False, concrete: bool = False\n ) -> \"VariantValue\":\n if value is True or value is False:\n return VariantValue(VariantType.BOOL, name, (value,), propagate=propagate)\n\n elif value.upper() in (\"TRUE\", \"FALSE\"):\n return VariantValue(\n VariantType.BOOL, name, (value.upper() == \"TRUE\",), propagate=propagate\n )\n\n elif value == \"*\":\n return VariantValue(VariantType.MULTI, name, (), propagate=propagate)\n\n return VariantValue(\n VariantType.MULTI,\n name,\n tuple(value.split(\",\")),\n propagate=propagate,\n concrete=concrete,\n )\n\n @staticmethod\n def from_concretizer(name: str, value: str, type: str) -> \"VariantValue\":\n \"\"\"Reconstruct a variant from concretizer output.\"\"\"\n if type == \"bool\":\n return VariantValue(VariantType.BOOL, name, (value == \"True\",))\n elif type == \"multi\":\n return VariantValue(VariantType.MULTI, name, (value,), concrete=True)\n else:\n return VariantValue(VariantType.SINGLE, name, (value,))\n\n def yaml_entry(self) -> Tuple[str, SerializedValueType]:\n \"\"\"Returns a (key, value) tuple suitable to be an entry in a yaml dict.\n\n Returns:\n tuple: (name, value_representation)\n \"\"\"\n if self.type == VariantType.MULTI:\n return self.name, list(self.values)\n return self.name, self.values[0]\n\n @property\n def values(self) -> ValueType:\n return self._values\n\n @property\n def value(self) -> Union[ValueType, bool, str]:\n return self._values[0] if self.type != VariantType.MULTI else self._values\n\n def set(self, *value: Union[bool, str]) -> None:\n \"\"\"Set the value(s) of the variant.\"\"\"\n if len(value) > 1:\n value = tuple(sorted(set(value)))\n\n if self.type != VariantType.MULTI:\n if len(value) != 1:\n raise MultipleValuesInExclusiveVariantError(self)\n unwrapped = value[0]\n if self.type == VariantType.BOOL and unwrapped not in (True, False):\n raise ValueError(\n f\"cannot set a boolean variant to a value that is not a boolean: {unwrapped}\"\n )\n\n if \"*\" in value:\n raise InvalidVariantValueError(\"cannot use reserved value '*'\")\n\n self._values = value\n\n def _cmp_iter(self) -> Iterable:\n yield self.name\n yield self.propagate\n yield self.concrete\n yield from (str(v) for v in self.values)\n\n def copy(self) -> \"VariantValue\":\n return VariantValue(\n self.type, self.name, self.values, propagate=self.propagate, concrete=self.concrete\n )\n\n def satisfies(self, other: \"VariantValue\") -> bool:\n \"\"\"The lhs satisfies the rhs if all possible concretizations of lhs are also\n possible concretizations of rhs.\"\"\"\n if self.name != other.name:\n return False\n\n if not other.concrete:\n # rhs abstract means the lhs must at least contain its values.\n # special-case patches with rhs abstract: their values may be prefixes of the lhs\n # values.\n if self.name == \"patches\":\n return all(\n isinstance(v, str)\n and any(isinstance(w, str) and w.startswith(v) for w in self.values)\n for v in other.values\n )\n return all(v in self for v in other.values)\n if self.concrete:\n # both concrete: they must be equal\n return self.values == other.values\n return False\n\n def intersects(self, other: \"VariantValue\") -> bool:\n \"\"\"True iff there exists a concretization that satisfies both lhs and rhs.\"\"\"\n if self.name != other.name:\n return False\n if self.concrete:\n if other.concrete:\n return self.values == other.values\n return all(v in self for v in other.values)\n if other.concrete:\n return all(v in other for v in self.values)\n # both abstract: the union is a valid concretization of both\n return True\n\n def constrain(self, other: \"VariantValue\") -> bool:\n \"\"\"Constrain self with other if they intersect. Returns true iff self was changed.\"\"\"\n if not self.intersects(other):\n raise UnsatisfiableVariantSpecError(self, other)\n old_values = self.values\n self.set(*self.values, *other.values)\n changed = old_values != self.values\n if self.propagate and not other.propagate:\n self.propagate = False\n changed = True\n if not self.concrete and other.concrete:\n self.concrete = True\n changed = True\n if self.type > other.type:\n self.type = other.type\n changed = True\n return changed\n\n def append(self, value: Union[str, bool]) -> None:\n self.set(*self.values, value)\n\n def __contains__(self, item: Union[str, bool]) -> bool:\n return item in self.values\n\n def __str__(self) -> str:\n # boolean variants are printed +foo or ~foo\n if self.type == VariantType.BOOL:\n sigil = \"+\" if self.value else \"~\"\n if self.propagate:\n sigil *= 2\n return f\"{sigil}{self.name}\"\n\n # concrete multi-valued foo:=bar,baz\n concrete = \":\" if self.type == VariantType.MULTI and self.concrete else \"\"\n delim = \"==\" if self.propagate else \"=\"\n if not self.values:\n value_str = \"*\"\n elif self.name == \"patches\" and self.concrete:\n value_str = \",\".join(str(x)[:7] for x in self.values)\n else:\n value_str = \",\".join(str(x) for x in self.values)\n return f\"{self.name}{concrete}{delim}{spack.spec_parser.quote_if_needed(value_str)}\"\n\n def __repr__(self):\n return (\n f\"VariantValue({self.type!r}, {self.name!r}, {self.values!r}, \"\n f\"propagate={self.propagate!r}, concrete={self.concrete!r})\"\n )\n\n\ndef MultiValuedVariant(name: str, value: ValueType, propagate: bool = False) -> VariantValue:\n return VariantValue(VariantType.MULTI, name, value, propagate=propagate, concrete=True)\n\n\ndef SingleValuedVariant(\n name: str, value: Union[bool, str], propagate: bool = False\n) -> VariantValue:\n return VariantValue(VariantType.SINGLE, name, (value,), propagate=propagate)\n\n\ndef BoolValuedVariant(name: str, value: bool, propagate: bool = False) -> VariantValue:\n return VariantValue(VariantType.BOOL, name, (value,), propagate=propagate)\n\n\nclass VariantValueRemoval(VariantValue):\n \"\"\"Indicator class for Spec.mutate to remove a variant\"\"\"\n\n def __init__(self, name):\n super().__init__(VariantType.INDICATOR, name, (None,))\n\n\n# The class below inherit from Sequence to disguise as a tuple and comply\n# with the semantic expected by the 'values' argument of the variant directive\nclass DisjointSetsOfValues(collections.abc.Sequence):\n \"\"\"Allows combinations from one of many mutually exclusive sets.\n\n The value ``('none',)`` is reserved to denote the empty set\n and therefore no other set can contain the item ``'none'``.\n\n Args:\n *sets (list): mutually exclusive sets of values\n \"\"\"\n\n _empty_set = (\"none\",)\n\n def __init__(self, *sets: Tuple[str, ...]) -> None:\n self.sets = [tuple(_flatten(x)) for x in sets]\n\n # 'none' is a special value and can appear only in a set of a single element\n if any(\"none\" in s and s != self._empty_set for s in self.sets):\n raise spack.error.SpecError(\n \"The value 'none' represents the empty set, and must appear alone in a set. \"\n \"Use the method 'allow_empty_set' to add it.\"\n )\n\n # Sets should not intersect with each other\n cumulated: Set[str] = set()\n for current_set in self.sets:\n if not cumulated.isdisjoint(current_set):\n duplicates = \", \".join(sorted(cumulated.intersection(current_set)))\n raise spack.error.SpecError(\n f\"sets in input must be disjoint, but {duplicates} appeared more than once\"\n )\n cumulated.update(current_set)\n\n #: Attribute used to track values which correspond to\n #: features which can be enabled or disabled as understood by the\n #: package's build system.\n self.feature_values = tuple(itertools.chain.from_iterable(self.sets))\n self.default = None\n self.multi = True\n self.error_fmt = (\n \"this variant accepts combinations of values from \"\n \"exactly one of the following sets '{values}' \"\n \"@*r{{[{package}, variant '{variant}']}}\"\n )\n\n def with_default(self, default):\n \"\"\"Sets the default value and returns self.\"\"\"\n self.default = default\n return self\n\n def with_error(self, error_fmt):\n \"\"\"Sets the error message format and returns self.\"\"\"\n self.error_fmt = error_fmt\n return self\n\n def with_non_feature_values(self, *values):\n \"\"\"Marks a few values as not being tied to a feature.\"\"\"\n self.feature_values = tuple(x for x in self.feature_values if x not in values)\n return self\n\n def allow_empty_set(self):\n \"\"\"Adds the empty set to the current list of disjoint sets.\"\"\"\n if self._empty_set in self.sets:\n return self\n\n # Create a new object to be returned\n object_with_empty_set = type(self)((\"none\",), *self.sets)\n object_with_empty_set.error_fmt = self.error_fmt\n object_with_empty_set.feature_values = self.feature_values + (\"none\",)\n return object_with_empty_set\n\n def prohibit_empty_set(self):\n \"\"\"Removes the empty set from the current list of disjoint sets.\"\"\"\n if self._empty_set not in self.sets:\n return self\n\n # Create a new object to be returned\n sets = [s for s in self.sets if s != self._empty_set]\n object_without_empty_set = type(self)(*sets)\n object_without_empty_set.error_fmt = self.error_fmt\n object_without_empty_set.feature_values = tuple(\n x for x in self.feature_values if x != \"none\"\n )\n return object_without_empty_set\n\n def __getitem__(self, idx):\n return tuple(itertools.chain.from_iterable(self.sets))[idx]\n\n def __len__(self):\n return sum(len(x) for x in self.sets)\n\n @property\n def validator(self):\n def _disjoint_set_validator(pkg_name, variant_name, values):\n # If for any of the sets, all the values are in it return True\n if any(all(x in s for x in values) for s in self.sets):\n return\n\n format_args = {\"variant\": variant_name, \"package\": pkg_name, \"values\": values}\n msg = self.error_fmt + \" @*r{{[{package}, variant '{variant}']}}\"\n msg = spack.llnl.util.tty.color.colorize(msg.format(**format_args))\n raise spack.error.SpecError(msg)\n\n return _disjoint_set_validator\n\n\ndef _a_single_value_or_a_combination(single_value: str, *values: str) -> DisjointSetsOfValues:\n error = f\"the value '{single_value}' is mutually exclusive with any of the other values\"\n return (\n DisjointSetsOfValues((single_value,), values)\n .with_default(single_value)\n .with_error(error)\n .with_non_feature_values(single_value)\n )\n\n\n# TODO: The factories below are used by package writers to set values of\n# TODO: multi-valued variants. It could be worthwhile to gather them in\n# TODO: a common namespace (like 'multi') in the future.\n\n\ndef any_combination_of(*values: str) -> DisjointSetsOfValues:\n \"\"\"Multi-valued variant that allows either any combination of the specified values, or none\n at all (using ``variant=none``). The literal value ``none`` is used as sentinel for the empty\n set, since in the spec DSL we have to always specify a value for a variant.\n\n It is up to the package implementation to handle the value ``none`` specially, if at all.\n\n See also :func:`auto_or_any_combination_of` and :func:`disjoint_sets`.\n\n Args:\n *values: allowed variant values\n\n Example::\n\n variant(\"cuda_arch\", values=any_combination_of(\"10\", \"11\"))\n\n Returns:\n a properly initialized instance of :class:`~spack.variant.DisjointSetsOfValues`\n \"\"\"\n return _a_single_value_or_a_combination(\"none\", *values)\n\n\ndef auto_or_any_combination_of(*values: str) -> DisjointSetsOfValues:\n \"\"\"Multi-valued variant that allows any combination of a set of values (but not the empty set)\n or ``auto``.\n\n See also :func:`any_combination_of` and :func:`disjoint_sets`.\n\n Args:\n *values: allowed variant values\n\n Example::\n\n variant(\n \"file_systems\",\n values=auto_or_any_combination_of(\"lustre\", \"gpfs\", \"nfs\", \"ufs\"),\n )\n\n Returns:\n a properly initialized instance of :class:`~spack.variant.DisjointSetsOfValues`\n \"\"\"\n return _a_single_value_or_a_combination(\"auto\", *values)\n\n\ndef disjoint_sets(*sets: Tuple[str, ...]) -> DisjointSetsOfValues:\n \"\"\"Multi-valued variant that allows any combination picking from one of multiple disjoint sets\n of values, and also allows the user to specify ``none`` to choose none of them.\n\n It is up to the package implementation to handle the value ``none`` specially, if at all.\n\n See also :func:`any_combination_of` and :func:`auto_or_any_combination_of`.\n\n Args:\n *sets: sets of allowed values, each set is a tuple of strings\n\n Returns:\n a properly initialized instance of :class:`~spack.variant.DisjointSetsOfValues`\n \"\"\"\n return DisjointSetsOfValues(*sets).allow_empty_set().with_default(\"none\")\n\n\n@functools.total_ordering\nclass ConditionalValue:\n \"\"\"Conditional value for a variant.\"\"\"\n\n value: Any\n\n # optional because statically disabled values (when=False) are set to None\n # when=True results in spack.spec.Spec()\n when: Optional[\"spack.spec.Spec\"]\n\n def __init__(self, value: Any, when: Optional[\"spack.spec.Spec\"]):\n self.value = value\n self.when = when\n\n def __repr__(self):\n return f\"ConditionalValue({self.value}, when={self.when})\"\n\n def __str__(self):\n return str(self.value)\n\n def __hash__(self):\n # Needed to allow testing the presence of a variant in a set by its value\n return hash(self.value)\n\n def __eq__(self, other):\n if isinstance(other, (str, bool)):\n return self.value == other\n return self.value == other.value\n\n def __lt__(self, other):\n if isinstance(other, str):\n return self.value < other\n return self.value < other.value\n\n\ndef prevalidate_variant_value(\n pkg_cls: \"Type[spack.package_base.PackageBase]\",\n variant: VariantValue,\n spec: Optional[\"spack.spec.Spec\"] = None,\n strict: bool = False,\n) -> List[Variant]:\n \"\"\"Do as much validation of a variant value as is possible before concretization.\n\n This checks that the variant value is valid for *some* definition of the variant, and\n it raises if we know *before* concretization that the value cannot occur. On success\n it returns the variant definitions for which the variant is valid.\n\n Arguments:\n pkg_cls: package in which variant is (potentially multiply) defined\n variant: variant spec with value to validate\n spec: optionally restrict validation only to variants defined for this spec\n strict: if True, raise an exception if no variant definition is valid for any\n constraint on the spec.\n\n Return:\n list of variant definitions that will accept the given value. List will be empty\n only if the variant is a reserved variant.\n \"\"\"\n # do not validate non-user variants or optional variants\n if variant.name in RESERVED_NAMES or variant.propagate:\n return []\n\n # raise if there is no definition at all\n if not pkg_cls.has_variant(variant.name):\n raise UnknownVariantError(\n f\"No such variant '{variant.name}' in package {pkg_cls.name}\", [variant.name]\n )\n\n # do as much prevalidation as we can -- check only those\n # variants whose when constraint intersects this spec\n errors = []\n possible_definitions = []\n valid_definitions = []\n\n for when, pkg_variant_def in pkg_cls.variant_definitions(variant.name):\n if spec and not spec.intersects(when):\n continue\n possible_definitions.append(pkg_variant_def)\n\n try:\n pkg_variant_def.validate_or_raise(variant, pkg_cls.name)\n valid_definitions.append(pkg_variant_def)\n except spack.error.SpecError as e:\n errors.append(e)\n\n # value is valid for at least one definition -- return them all\n if valid_definitions:\n return valid_definitions\n\n # no when spec intersected, so no possible definition for the variant in this configuration\n if strict and not possible_definitions:\n when_clause = f\" when {spec}\" if spec else \"\"\n raise InvalidVariantValueError(\n f\"variant '{variant.name}' does not exist for '{pkg_cls.name}'{when_clause}\"\n )\n\n # There are only no errors if we're not strict and there are no possible_definitions.\n # We are strict for audits but not for specs on the CLI or elsewhere. Being strict\n # in these cases would violate our rule of being able to *talk* about any configuration,\n # regardless of what the package.py currently says.\n if not errors:\n return []\n\n # if there is just one error, raise the specific error\n if len(errors) == 1:\n raise errors[0]\n\n # otherwise combine all the errors and raise them together\n raise InvalidVariantValueError(\n \"multiple variant issues:\", \"\\n\".join(e.message for e in errors)\n )\n\n\nclass ConditionalVariantValues(lang.TypedMutableSequence):\n \"\"\"A list, just with a different type\"\"\"\n\n\nclass DuplicateVariantError(spack.error.SpecError):\n \"\"\"Raised when the same variant occurs in a spec twice.\"\"\"\n\n\nclass UnknownVariantError(spack.error.SpecError):\n \"\"\"Raised when an unknown variant occurs in a spec.\"\"\"\n\n def __init__(self, msg: str, unknown_variants: List[str]):\n super().__init__(msg)\n self.unknown_variants = unknown_variants\n\n\nclass InconsistentValidationError(spack.error.SpecError):\n \"\"\"Raised if the wrong validator is used to validate a variant.\"\"\"\n\n def __init__(self, vspec, variant):\n msg = 'trying to validate variant \"{0.name}\" with the validator of \"{1.name}\"'\n super().__init__(msg.format(vspec, variant))\n\n\nclass MultipleValuesInExclusiveVariantError(spack.error.SpecError, ValueError):\n \"\"\"Raised when multiple values are present in a variant that wants\n only one.\n \"\"\"\n\n def __init__(self, variant: VariantValue, pkg_name: Optional[str] = None):\n pkg_info = \"\" if pkg_name is None else f\" in package '{pkg_name}'\"\n msg = f\"multiple values are not allowed for variant '{variant.name}'{pkg_info}\"\n\n super().__init__(msg.format(variant, pkg_info))\n\n\nclass InvalidVariantValueError(spack.error.SpecError):\n \"\"\"Raised when variants have invalid values.\"\"\"\n\n\nclass UnsatisfiableVariantSpecError(spack.error.UnsatisfiableSpecError):\n \"\"\"Raised when a spec variant conflicts with package constraints.\"\"\"\n\n def __init__(self, provided, required):\n super().__init__(provided, required, \"variant\")\n" }, { "path": "lib/spack/spack/vendor/__init__.py", "content": "# Copyright Spack Project Developers. See COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n" }, { "path": "lib/spack/spack/vendor/_pyrsistent_version.py", "content": "__version__ = '0.18.0'\n" }, { "path": "lib/spack/spack/vendor/altgraph/Dot.py", "content": "\"\"\"\nspack.vendor.altgraph.Dot - Interface to the dot language\n============================================\n\nThe :py:mod:`~spack.vendor.altgraph.Dot` module provides a simple interface to the\nfile format used in the\n`graphviz `_\nprogram. The module is intended to offload the most tedious part of the process\n(the **dot** file generation) while transparently exposing most of its\nfeatures.\n\nTo display the graphs or to generate image files the\n`graphviz `_\npackage needs to be installed on the system, moreover the :command:`dot` and\n:command:`dotty` programs must be accesible in the program path so that they\ncan be ran from processes spawned within the module.\n\nExample usage\n-------------\n\nHere is a typical usage::\n\n from spack.vendor.altgraph import Graph, Dot\n\n # create a graph\n edges = [ (1,2), (1,3), (3,4), (3,5), (4,5), (5,4) ]\n graph = Graph.Graph(edges)\n\n # create a dot representation of the graph\n dot = Dot.Dot(graph)\n\n # display the graph\n dot.display()\n\n # save the dot representation into the mydot.dot file\n dot.save_dot(file_name='mydot.dot')\n\n # save dot file as gif image into the graph.gif file\n dot.save_img(file_name='graph', file_type='gif')\n\nDirected graph and non-directed graph\n-------------------------------------\n\nDot class can use for both directed graph and non-directed graph\nby passing ``graphtype`` parameter.\n\nExample::\n\n # create directed graph(default)\n dot = Dot.Dot(graph, graphtype=\"digraph\")\n\n # create non-directed graph\n dot = Dot.Dot(graph, graphtype=\"graph\")\n\nCustomizing the output\n----------------------\n\nThe graph drawing process may be customized by passing\nvalid :command:`dot` parameters for the nodes and edges. For a list of all\nparameters see the `graphviz `_\ndocumentation.\n\nExample::\n\n # customizing the way the overall graph is drawn\n dot.style(size='10,10', rankdir='RL', page='5, 5' , ranksep=0.75)\n\n # customizing node drawing\n dot.node_style(1, label='BASE_NODE',shape='box', color='blue' )\n dot.node_style(2, style='filled', fillcolor='red')\n\n # customizing edge drawing\n dot.edge_style(1, 2, style='dotted')\n dot.edge_style(3, 5, arrowhead='dot', label='binds', labelangle='90')\n dot.edge_style(4, 5, arrowsize=2, style='bold')\n\n\n.. note::\n\n dotty (invoked via :py:func:`~spack.vendor.altgraph.Dot.display`) may not be able to\n display all graphics styles. To verify the output save it to an image file\n and look at it that way.\n\nValid attributes\n----------------\n\n - dot styles, passed via the :py:meth:`Dot.style` method::\n\n rankdir = 'LR' (draws the graph horizontally, left to right)\n ranksep = number (rank separation in inches)\n\n - node attributes, passed via the :py:meth:`Dot.node_style` method::\n\n style = 'filled' | 'invisible' | 'diagonals' | 'rounded'\n shape = 'box' | 'ellipse' | 'circle' | 'point' | 'triangle'\n\n - edge attributes, passed via the :py:meth:`Dot.edge_style` method::\n\n style = 'dashed' | 'dotted' | 'solid' | 'invis' | 'bold'\n arrowhead = 'box' | 'crow' | 'diamond' | 'dot' | 'inv' | 'none'\n | 'tee' | 'vee'\n weight = number (the larger the number the closer the nodes will be)\n\n - valid `graphviz colors\n `_\n\n - for more details on how to control the graph drawing process see the\n `graphviz reference\n `_.\n\"\"\"\nimport os\nimport warnings\n\nfrom spack.vendor.altgraph import GraphError\n\n\nclass Dot(object):\n \"\"\"\n A class providing a **graphviz** (dot language) representation\n allowing a fine grained control over how the graph is being\n displayed.\n\n If the :command:`dot` and :command:`dotty` programs are not in the current\n system path their location needs to be specified in the contructor.\n \"\"\"\n\n def __init__(\n self,\n graph=None,\n nodes=None,\n edgefn=None,\n nodevisitor=None,\n edgevisitor=None,\n name=\"G\",\n dot=\"dot\",\n dotty=\"dotty\",\n neato=\"neato\",\n graphtype=\"digraph\",\n ):\n \"\"\"\n Initialization.\n \"\"\"\n self.name, self.attr = name, {}\n\n assert graphtype in [\"graph\", \"digraph\"]\n self.type = graphtype\n\n self.temp_dot = \"tmp_dot.dot\"\n self.temp_neo = \"tmp_neo.dot\"\n\n self.dot, self.dotty, self.neato = dot, dotty, neato\n\n # self.nodes: node styles\n # self.edges: edge styles\n self.nodes, self.edges = {}, {}\n\n if graph is not None and nodes is None:\n nodes = graph\n if graph is not None and edgefn is None:\n\n def edgefn(node, graph=graph):\n return graph.out_nbrs(node)\n\n if nodes is None:\n nodes = ()\n\n seen = set()\n for node in nodes:\n if nodevisitor is None:\n style = {}\n else:\n style = nodevisitor(node)\n if style is not None:\n self.nodes[node] = {}\n self.node_style(node, **style)\n seen.add(node)\n if edgefn is not None:\n for head in seen:\n for tail in (n for n in edgefn(head) if n in seen):\n if edgevisitor is None:\n edgestyle = {}\n else:\n edgestyle = edgevisitor(head, tail)\n if edgestyle is not None:\n if head not in self.edges:\n self.edges[head] = {}\n self.edges[head][tail] = {}\n self.edge_style(head, tail, **edgestyle)\n\n def style(self, **attr):\n \"\"\"\n Changes the overall style\n \"\"\"\n self.attr = attr\n\n def display(self, mode=\"dot\"):\n \"\"\"\n Displays the current graph via dotty\n \"\"\"\n\n if mode == \"neato\":\n self.save_dot(self.temp_neo)\n neato_cmd = \"%s -o %s %s\" % (self.neato, self.temp_dot, self.temp_neo)\n os.system(neato_cmd)\n else:\n self.save_dot(self.temp_dot)\n\n plot_cmd = \"%s %s\" % (self.dotty, self.temp_dot)\n os.system(plot_cmd)\n\n def node_style(self, node, **kwargs):\n \"\"\"\n Modifies a node style to the dot representation.\n \"\"\"\n if node not in self.edges:\n self.edges[node] = {}\n self.nodes[node] = kwargs\n\n def all_node_style(self, **kwargs):\n \"\"\"\n Modifies all node styles\n \"\"\"\n for node in self.nodes:\n self.node_style(node, **kwargs)\n\n def edge_style(self, head, tail, **kwargs):\n \"\"\"\n Modifies an edge style to the dot representation.\n \"\"\"\n if tail not in self.nodes:\n raise GraphError(\"invalid node %s\" % (tail,))\n\n try:\n if tail not in self.edges[head]:\n self.edges[head][tail] = {}\n self.edges[head][tail] = kwargs\n except KeyError:\n raise GraphError(\"invalid edge %s -> %s \" % (head, tail))\n\n def iterdot(self):\n # write graph title\n if self.type == \"digraph\":\n yield \"digraph %s {\\n\" % (self.name,)\n elif self.type == \"graph\":\n yield \"graph %s {\\n\" % (self.name,)\n\n else:\n raise GraphError(\"unsupported graphtype %s\" % (self.type,))\n\n # write overall graph attributes\n for attr_name, attr_value in sorted(self.attr.items()):\n yield '%s=\"%s\";' % (attr_name, attr_value)\n yield \"\\n\"\n\n # some reusable patterns\n cpatt = '%s=\"%s\",' # to separate attributes\n epatt = \"];\\n\" # to end attributes\n\n # write node attributes\n for node_name, node_attr in sorted(self.nodes.items()):\n yield '\\t\"%s\" [' % (node_name,)\n for attr_name, attr_value in sorted(node_attr.items()):\n yield cpatt % (attr_name, attr_value)\n yield epatt\n\n # write edge attributes\n for head in sorted(self.edges):\n for tail in sorted(self.edges[head]):\n if self.type == \"digraph\":\n yield '\\t\"%s\" -> \"%s\" [' % (head, tail)\n else:\n yield '\\t\"%s\" -- \"%s\" [' % (head, tail)\n for attr_name, attr_value in sorted(self.edges[head][tail].items()):\n yield cpatt % (attr_name, attr_value)\n yield epatt\n\n # finish file\n yield \"}\\n\"\n\n def __iter__(self):\n return self.iterdot()\n\n def save_dot(self, file_name=None):\n \"\"\"\n Saves the current graph representation into a file\n \"\"\"\n\n if not file_name:\n warnings.warn(DeprecationWarning, \"always pass a file_name\")\n file_name = self.temp_dot\n\n with open(file_name, \"w\") as fp:\n for chunk in self.iterdot():\n fp.write(chunk)\n\n def save_img(self, file_name=None, file_type=\"gif\", mode=\"dot\"):\n \"\"\"\n Saves the dot file as an image file\n \"\"\"\n\n if not file_name:\n warnings.warn(DeprecationWarning, \"always pass a file_name\")\n file_name = \"out\"\n\n if mode == \"neato\":\n self.save_dot(self.temp_neo)\n neato_cmd = \"%s -o %s %s\" % (self.neato, self.temp_dot, self.temp_neo)\n os.system(neato_cmd)\n plot_cmd = self.dot\n else:\n self.save_dot(self.temp_dot)\n plot_cmd = self.dot\n\n file_name = \"%s.%s\" % (file_name, file_type)\n create_cmd = \"%s -T%s %s -o %s\" % (\n plot_cmd,\n file_type,\n self.temp_dot,\n file_name,\n )\n os.system(create_cmd)\n" }, { "path": "lib/spack/spack/vendor/altgraph/Graph.py", "content": "\"\"\"\nspack.vendor.altgraph.Graph - Base Graph class\n=================================\n\n..\n #--Version 2.1\n #--Bob Ippolito October, 2004\n\n #--Version 2.0\n #--Istvan Albert June, 2004\n\n #--Version 1.0\n #--Nathan Denny, May 27, 1999\n\"\"\"\n\nfrom collections import deque\n\nfrom spack.vendor.altgraph import GraphError\n\n\nclass Graph(object):\n \"\"\"\n The Graph class represents a directed graph with *N* nodes and *E* edges.\n\n Naming conventions:\n\n - the prefixes such as *out*, *inc* and *all* will refer to methods\n that operate on the outgoing, incoming or all edges of that node.\n\n For example: :py:meth:`inc_degree` will refer to the degree of the node\n computed over the incoming edges (the number of neighbours linking to\n the node).\n\n - the prefixes such as *forw* and *back* will refer to the\n orientation of the edges used in the method with respect to the node.\n\n For example: :py:meth:`forw_bfs` will start at the node then use the\n outgoing edges to traverse the graph (goes forward).\n \"\"\"\n\n def __init__(self, edges=None):\n \"\"\"\n Initialization\n \"\"\"\n\n self.next_edge = 0\n self.nodes, self.edges = {}, {}\n self.hidden_edges, self.hidden_nodes = {}, {}\n\n if edges is not None:\n for item in edges:\n if len(item) == 2:\n head, tail = item\n self.add_edge(head, tail)\n elif len(item) == 3:\n head, tail, data = item\n self.add_edge(head, tail, data)\n else:\n raise GraphError(\"Cannot create edge from %s\" % (item,))\n\n def __repr__(self):\n return \"\" % (\n self.number_of_nodes(),\n self.number_of_edges(),\n )\n\n def add_node(self, node, node_data=None):\n \"\"\"\n Adds a new node to the graph. Arbitrary data can be attached to the\n node via the node_data parameter. Adding the same node twice will be\n silently ignored.\n\n The node must be a hashable value.\n \"\"\"\n #\n # the nodes will contain tuples that will store incoming edges,\n # outgoing edges and data\n #\n # index 0 -> incoming edges\n # index 1 -> outgoing edges\n\n if node in self.hidden_nodes:\n # Node is present, but hidden\n return\n\n if node not in self.nodes:\n self.nodes[node] = ([], [], node_data)\n\n def add_edge(self, head_id, tail_id, edge_data=1, create_nodes=True):\n \"\"\"\n Adds a directed edge going from head_id to tail_id.\n Arbitrary data can be attached to the edge via edge_data.\n It may create the nodes if adding edges between nonexisting ones.\n\n :param head_id: head node\n :param tail_id: tail node\n :param edge_data: (optional) data attached to the edge\n :param create_nodes: (optional) creates the head_id or tail_id\n node in case they did not exist\n \"\"\"\n # shorcut\n edge = self.next_edge\n\n # add nodes if on automatic node creation\n if create_nodes:\n self.add_node(head_id)\n self.add_node(tail_id)\n\n # update the corresponding incoming and outgoing lists in the nodes\n # index 0 -> incoming edges\n # index 1 -> outgoing edges\n\n try:\n self.nodes[tail_id][0].append(edge)\n self.nodes[head_id][1].append(edge)\n except KeyError:\n raise GraphError(\"Invalid nodes %s -> %s\" % (head_id, tail_id))\n\n # store edge information\n self.edges[edge] = (head_id, tail_id, edge_data)\n\n self.next_edge += 1\n\n def hide_edge(self, edge):\n \"\"\"\n Hides an edge from the graph. The edge may be unhidden at some later\n time.\n \"\"\"\n try:\n head_id, tail_id, edge_data = self.hidden_edges[edge] = self.edges[edge]\n self.nodes[tail_id][0].remove(edge)\n self.nodes[head_id][1].remove(edge)\n del self.edges[edge]\n except KeyError:\n raise GraphError(\"Invalid edge %s\" % edge)\n\n def hide_node(self, node):\n \"\"\"\n Hides a node from the graph. The incoming and outgoing edges of the\n node will also be hidden. The node may be unhidden at some later time.\n \"\"\"\n try:\n all_edges = self.all_edges(node)\n self.hidden_nodes[node] = (self.nodes[node], all_edges)\n for edge in all_edges:\n self.hide_edge(edge)\n del self.nodes[node]\n except KeyError:\n raise GraphError(\"Invalid node %s\" % node)\n\n def restore_node(self, node):\n \"\"\"\n Restores a previously hidden node back into the graph and restores\n all of its incoming and outgoing edges.\n \"\"\"\n try:\n self.nodes[node], all_edges = self.hidden_nodes[node]\n for edge in all_edges:\n self.restore_edge(edge)\n del self.hidden_nodes[node]\n except KeyError:\n raise GraphError(\"Invalid node %s\" % node)\n\n def restore_edge(self, edge):\n \"\"\"\n Restores a previously hidden edge back into the graph.\n \"\"\"\n try:\n head_id, tail_id, data = self.hidden_edges[edge]\n self.nodes[tail_id][0].append(edge)\n self.nodes[head_id][1].append(edge)\n self.edges[edge] = head_id, tail_id, data\n del self.hidden_edges[edge]\n except KeyError:\n raise GraphError(\"Invalid edge %s\" % edge)\n\n def restore_all_edges(self):\n \"\"\"\n Restores all hidden edges.\n \"\"\"\n for edge in list(self.hidden_edges.keys()):\n try:\n self.restore_edge(edge)\n except GraphError:\n pass\n\n def restore_all_nodes(self):\n \"\"\"\n Restores all hidden nodes.\n \"\"\"\n for node in list(self.hidden_nodes.keys()):\n self.restore_node(node)\n\n def __contains__(self, node):\n \"\"\"\n Test whether a node is in the graph\n \"\"\"\n return node in self.nodes\n\n def edge_by_id(self, edge):\n \"\"\"\n Returns the edge that connects the head_id and tail_id nodes\n \"\"\"\n try:\n head, tail, data = self.edges[edge]\n except KeyError:\n head, tail = None, None\n raise GraphError(\"Invalid edge %s\" % edge)\n\n return (head, tail)\n\n def edge_by_node(self, head, tail):\n \"\"\"\n Returns the edge that connects the head_id and tail_id nodes\n \"\"\"\n for edge in self.out_edges(head):\n if self.tail(edge) == tail:\n return edge\n return None\n\n def number_of_nodes(self):\n \"\"\"\n Returns the number of nodes\n \"\"\"\n return len(self.nodes)\n\n def number_of_edges(self):\n \"\"\"\n Returns the number of edges\n \"\"\"\n return len(self.edges)\n\n def __iter__(self):\n \"\"\"\n Iterates over all nodes in the graph\n \"\"\"\n return iter(self.nodes)\n\n def node_list(self):\n \"\"\"\n Return a list of the node ids for all visible nodes in the graph.\n \"\"\"\n return list(self.nodes.keys())\n\n def edge_list(self):\n \"\"\"\n Returns an iterator for all visible nodes in the graph.\n \"\"\"\n return list(self.edges.keys())\n\n def number_of_hidden_edges(self):\n \"\"\"\n Returns the number of hidden edges\n \"\"\"\n return len(self.hidden_edges)\n\n def number_of_hidden_nodes(self):\n \"\"\"\n Returns the number of hidden nodes\n \"\"\"\n return len(self.hidden_nodes)\n\n def hidden_node_list(self):\n \"\"\"\n Returns the list with the hidden nodes\n \"\"\"\n return list(self.hidden_nodes.keys())\n\n def hidden_edge_list(self):\n \"\"\"\n Returns a list with the hidden edges\n \"\"\"\n return list(self.hidden_edges.keys())\n\n def describe_node(self, node):\n \"\"\"\n return node, node data, outgoing edges, incoming edges for node\n \"\"\"\n incoming, outgoing, data = self.nodes[node]\n return node, data, outgoing, incoming\n\n def describe_edge(self, edge):\n \"\"\"\n return edge, edge data, head, tail for edge\n \"\"\"\n head, tail, data = self.edges[edge]\n return edge, data, head, tail\n\n def node_data(self, node):\n \"\"\"\n Returns the data associated with a node\n \"\"\"\n return self.nodes[node][2]\n\n def edge_data(self, edge):\n \"\"\"\n Returns the data associated with an edge\n \"\"\"\n return self.edges[edge][2]\n\n def update_edge_data(self, edge, edge_data):\n \"\"\"\n Replace the edge data for a specific edge\n \"\"\"\n self.edges[edge] = self.edges[edge][0:2] + (edge_data,)\n\n def head(self, edge):\n \"\"\"\n Returns the node of the head of the edge.\n \"\"\"\n return self.edges[edge][0]\n\n def tail(self, edge):\n \"\"\"\n Returns node of the tail of the edge.\n \"\"\"\n return self.edges[edge][1]\n\n def out_nbrs(self, node):\n \"\"\"\n List of nodes connected by outgoing edges\n \"\"\"\n return [self.tail(n) for n in self.out_edges(node)]\n\n def inc_nbrs(self, node):\n \"\"\"\n List of nodes connected by incoming edges\n \"\"\"\n return [self.head(n) for n in self.inc_edges(node)]\n\n def all_nbrs(self, node):\n \"\"\"\n List of nodes connected by incoming and outgoing edges\n \"\"\"\n return list(dict.fromkeys(self.inc_nbrs(node) + self.out_nbrs(node)))\n\n def out_edges(self, node):\n \"\"\"\n Returns a list of the outgoing edges\n \"\"\"\n try:\n return list(self.nodes[node][1])\n except KeyError:\n raise GraphError(\"Invalid node %s\" % node)\n\n def inc_edges(self, node):\n \"\"\"\n Returns a list of the incoming edges\n \"\"\"\n try:\n return list(self.nodes[node][0])\n except KeyError:\n raise GraphError(\"Invalid node %s\" % node)\n\n def all_edges(self, node):\n \"\"\"\n Returns a list of incoming and outging edges.\n \"\"\"\n return set(self.inc_edges(node) + self.out_edges(node))\n\n def out_degree(self, node):\n \"\"\"\n Returns the number of outgoing edges\n \"\"\"\n return len(self.out_edges(node))\n\n def inc_degree(self, node):\n \"\"\"\n Returns the number of incoming edges\n \"\"\"\n return len(self.inc_edges(node))\n\n def all_degree(self, node):\n \"\"\"\n The total degree of a node\n \"\"\"\n return self.inc_degree(node) + self.out_degree(node)\n\n def _topo_sort(self, forward=True):\n \"\"\"\n Topological sort.\n\n Returns a list of nodes where the successors (based on outgoing and\n incoming edges selected by the forward parameter) of any given node\n appear in the sequence after that node.\n \"\"\"\n topo_list = []\n queue = deque()\n indeg = {}\n\n # select the operation that will be performed\n if forward:\n get_edges = self.out_edges\n get_degree = self.inc_degree\n get_next = self.tail\n else:\n get_edges = self.inc_edges\n get_degree = self.out_degree\n get_next = self.head\n\n for node in self.node_list():\n degree = get_degree(node)\n if degree:\n indeg[node] = degree\n else:\n queue.append(node)\n\n while queue:\n curr_node = queue.popleft()\n topo_list.append(curr_node)\n for edge in get_edges(curr_node):\n tail_id = get_next(edge)\n if tail_id in indeg:\n indeg[tail_id] -= 1\n if indeg[tail_id] == 0:\n queue.append(tail_id)\n\n if len(topo_list) == len(self.node_list()):\n valid = True\n else:\n # the graph has cycles, invalid topological sort\n valid = False\n\n return (valid, topo_list)\n\n def forw_topo_sort(self):\n \"\"\"\n Topological sort.\n\n Returns a list of nodes where the successors (based on outgoing edges)\n of any given node appear in the sequence after that node.\n \"\"\"\n return self._topo_sort(forward=True)\n\n def back_topo_sort(self):\n \"\"\"\n Reverse topological sort.\n\n Returns a list of nodes where the successors (based on incoming edges)\n of any given node appear in the sequence after that node.\n \"\"\"\n return self._topo_sort(forward=False)\n\n def _bfs_subgraph(self, start_id, forward=True):\n \"\"\"\n Private method creates a subgraph in a bfs order.\n\n The forward parameter specifies whether it is a forward or backward\n traversal.\n \"\"\"\n if forward:\n get_bfs = self.forw_bfs\n get_nbrs = self.out_nbrs\n else:\n get_bfs = self.back_bfs\n get_nbrs = self.inc_nbrs\n\n g = Graph()\n bfs_list = get_bfs(start_id)\n for node in bfs_list:\n g.add_node(node)\n\n for node in bfs_list:\n for nbr_id in get_nbrs(node):\n if forward:\n g.add_edge(node, nbr_id)\n else:\n g.add_edge(nbr_id, node)\n\n return g\n\n def forw_bfs_subgraph(self, start_id):\n \"\"\"\n Creates and returns a subgraph consisting of the breadth first\n reachable nodes based on their outgoing edges.\n \"\"\"\n return self._bfs_subgraph(start_id, forward=True)\n\n def back_bfs_subgraph(self, start_id):\n \"\"\"\n Creates and returns a subgraph consisting of the breadth first\n reachable nodes based on the incoming edges.\n \"\"\"\n return self._bfs_subgraph(start_id, forward=False)\n\n def iterdfs(self, start, end=None, forward=True):\n \"\"\"\n Collecting nodes in some depth first traversal.\n\n The forward parameter specifies whether it is a forward or backward\n traversal.\n \"\"\"\n visited, stack = {start}, deque([start])\n\n if forward:\n get_edges = self.out_edges\n get_next = self.tail\n else:\n get_edges = self.inc_edges\n get_next = self.head\n\n while stack:\n curr_node = stack.pop()\n yield curr_node\n if curr_node == end:\n break\n for edge in sorted(get_edges(curr_node)):\n tail = get_next(edge)\n if tail not in visited:\n visited.add(tail)\n stack.append(tail)\n\n def iterdata(self, start, end=None, forward=True, condition=None):\n \"\"\"\n Perform a depth-first walk of the graph (as ``iterdfs``)\n and yield the item data of every node where condition matches. The\n condition callback is only called when node_data is not None.\n \"\"\"\n\n visited, stack = {start}, deque([start])\n\n if forward:\n get_edges = self.out_edges\n get_next = self.tail\n else:\n get_edges = self.inc_edges\n get_next = self.head\n\n get_data = self.node_data\n\n while stack:\n curr_node = stack.pop()\n curr_data = get_data(curr_node)\n if curr_data is not None:\n if condition is not None and not condition(curr_data):\n continue\n yield curr_data\n if curr_node == end:\n break\n for edge in get_edges(curr_node):\n tail = get_next(edge)\n if tail not in visited:\n visited.add(tail)\n stack.append(tail)\n\n def _iterbfs(self, start, end=None, forward=True):\n \"\"\"\n The forward parameter specifies whether it is a forward or backward\n traversal. Returns a list of tuples where the first value is the hop\n value the second value is the node id.\n \"\"\"\n queue, visited = deque([(start, 0)]), {start}\n\n # the direction of the bfs depends on the edges that are sampled\n if forward:\n get_edges = self.out_edges\n get_next = self.tail\n else:\n get_edges = self.inc_edges\n get_next = self.head\n\n while queue:\n curr_node, curr_step = queue.popleft()\n yield (curr_node, curr_step)\n if curr_node == end:\n break\n for edge in get_edges(curr_node):\n tail = get_next(edge)\n if tail not in visited:\n visited.add(tail)\n queue.append((tail, curr_step + 1))\n\n def forw_bfs(self, start, end=None):\n \"\"\"\n Returns a list of nodes in some forward BFS order.\n\n Starting from the start node the breadth first search proceeds along\n outgoing edges.\n \"\"\"\n return [node for node, step in self._iterbfs(start, end, forward=True)]\n\n def back_bfs(self, start, end=None):\n \"\"\"\n Returns a list of nodes in some backward BFS order.\n\n Starting from the start node the breadth first search proceeds along\n incoming edges.\n \"\"\"\n return [node for node, _ in self._iterbfs(start, end, forward=False)]\n\n def forw_dfs(self, start, end=None):\n \"\"\"\n Returns a list of nodes in some forward DFS order.\n\n Starting with the start node the depth first search proceeds along\n outgoing edges.\n \"\"\"\n return list(self.iterdfs(start, end, forward=True))\n\n def back_dfs(self, start, end=None):\n \"\"\"\n Returns a list of nodes in some backward DFS order.\n\n Starting from the start node the depth first search proceeds along\n incoming edges.\n \"\"\"\n return list(self.iterdfs(start, end, forward=False))\n\n def connected(self):\n \"\"\"\n Returns :py:data:`True` if the graph's every node can be reached from\n every other node.\n \"\"\"\n node_list = self.node_list()\n for node in node_list:\n bfs_list = self.forw_bfs(node)\n if len(bfs_list) != len(node_list):\n return False\n return True\n\n def clust_coef(self, node):\n \"\"\"\n Computes and returns the local clustering coefficient of node.\n\n The local cluster coefficient is proportion of the actual number of\n edges between neighbours of node and the maximum number of edges\n between those neighbours.\n\n See \"Local Clustering Coefficient\" on\n \n for a formal definition.\n \"\"\"\n num = 0\n nbr_set = set(self.out_nbrs(node))\n\n if node in nbr_set:\n nbr_set.remove(node) # loop defense\n\n for nbr in nbr_set:\n sec_set = set(self.out_nbrs(nbr))\n if nbr in sec_set:\n sec_set.remove(nbr) # loop defense\n num += len(nbr_set & sec_set)\n\n nbr_num = len(nbr_set)\n if nbr_num:\n clust_coef = float(num) / (nbr_num * (nbr_num - 1))\n else:\n clust_coef = 0.0\n return clust_coef\n\n def get_hops(self, start, end=None, forward=True):\n \"\"\"\n Computes the hop distance to all nodes centered around a node.\n\n First order neighbours are at hop 1, their neigbours are at hop 2 etc.\n Uses :py:meth:`forw_bfs` or :py:meth:`back_bfs` depending on the value\n of the forward parameter. If the distance between all neighbouring\n nodes is 1 the hop number corresponds to the shortest distance between\n the nodes.\n\n :param start: the starting node\n :param end: ending node (optional). When not specified will search the\n whole graph.\n :param forward: directionality parameter (optional).\n If C{True} (default) it uses L{forw_bfs} otherwise L{back_bfs}.\n :return: returns a list of tuples where each tuple contains the\n node and the hop.\n\n Typical usage::\n\n >>> print (graph.get_hops(1, 8))\n >>> [(1, 0), (2, 1), (3, 1), (4, 2), (5, 3), (7, 4), (8, 5)]\n # node 1 is at 0 hops\n # node 2 is at 1 hop\n # ...\n # node 8 is at 5 hops\n \"\"\"\n if forward:\n return list(self._iterbfs(start=start, end=end, forward=True))\n else:\n return list(self._iterbfs(start=start, end=end, forward=False))\n" }, { "path": "lib/spack/spack/vendor/altgraph/GraphAlgo.py", "content": "\"\"\"\nspack.vendor.altgraph.GraphAlgo - Graph algorithms\n=====================================\n\"\"\"\nfrom spack.vendor.altgraph import GraphError\n\n\ndef dijkstra(graph, start, end=None):\n \"\"\"\n Dijkstra's algorithm for shortest paths\n\n `David Eppstein, UC Irvine, 4 April 2002\n `_\n\n `Python Cookbook Recipe\n `_\n\n Find shortest paths from the start node to all nodes nearer than or\n equal to the end node.\n\n Dijkstra's algorithm is only guaranteed to work correctly when all edge\n lengths are positive. This code does not verify this property for all\n edges (only the edges examined until the end vertex is reached), but will\n correctly compute shortest paths even for some graphs with negative edges,\n and will raise an exception if it discovers that a negative edge has\n caused it to make a mistake.\n\n Adapted to spack.vendor.altgraph by Istvan Albert, Pennsylvania State University -\n June, 9 2004\n \"\"\"\n D = {} # dictionary of final distances\n P = {} # dictionary of predecessors\n Q = _priorityDictionary() # estimated distances of non-final vertices\n Q[start] = 0\n\n for v in Q:\n D[v] = Q[v]\n if v == end:\n break\n\n for w in graph.out_nbrs(v):\n edge_id = graph.edge_by_node(v, w)\n vwLength = D[v] + graph.edge_data(edge_id)\n if w in D:\n if vwLength < D[w]:\n raise GraphError(\n \"Dijkstra: found better path to already-final vertex\"\n )\n elif w not in Q or vwLength < Q[w]:\n Q[w] = vwLength\n P[w] = v\n\n return (D, P)\n\n\ndef shortest_path(graph, start, end):\n \"\"\"\n Find a single shortest path from the *start* node to the *end* node.\n The input has the same conventions as dijkstra(). The output is a list of\n the nodes in order along the shortest path.\n\n **Note that the distances must be stored in the edge data as numeric data**\n \"\"\"\n\n D, P = dijkstra(graph, start, end)\n Path = []\n while 1:\n Path.append(end)\n if end == start:\n break\n end = P[end]\n Path.reverse()\n return Path\n\n\n#\n# Utility classes and functions\n#\nclass _priorityDictionary(dict):\n \"\"\"\n Priority dictionary using binary heaps (internal use only)\n\n David Eppstein, UC Irvine, 8 Mar 2002\n\n Implements a data structure that acts almost like a dictionary, with\n two modifications:\n\n 1. D.smallest() returns the value x minimizing D[x]. For this to\n work correctly, all values D[x] stored in the dictionary must be\n comparable.\n\n 2. iterating \"for x in D\" finds and removes the items from D in sorted\n order. Each item is not removed until the next item is requested,\n so D[x] will still return a useful value until the next iteration\n of the for-loop. Each operation takes logarithmic amortized time.\n \"\"\"\n\n def __init__(self):\n \"\"\"\n Initialize priorityDictionary by creating binary heap of pairs\n (value,key). Note that changing or removing a dict entry will not\n remove the old pair from the heap until it is found by smallest()\n or until the heap is rebuilt.\n \"\"\"\n self.__heap = []\n dict.__init__(self)\n\n def smallest(self):\n \"\"\"\n Find smallest item after removing deleted items from front of heap.\n \"\"\"\n if len(self) == 0:\n raise IndexError(\"smallest of empty priorityDictionary\")\n heap = self.__heap\n while heap[0][1] not in self or self[heap[0][1]] != heap[0][0]:\n lastItem = heap.pop()\n insertionPoint = 0\n while 1:\n smallChild = 2 * insertionPoint + 1\n if (\n smallChild + 1 < len(heap)\n and heap[smallChild] > heap[smallChild + 1]\n ):\n smallChild += 1\n if smallChild >= len(heap) or lastItem <= heap[smallChild]:\n heap[insertionPoint] = lastItem\n break\n heap[insertionPoint] = heap[smallChild]\n insertionPoint = smallChild\n return heap[0][1]\n\n def __iter__(self):\n \"\"\"\n Create destructive sorted iterator of priorityDictionary.\n \"\"\"\n\n def iterfn():\n while len(self) > 0:\n x = self.smallest()\n yield x\n del self[x]\n\n return iterfn()\n\n def __setitem__(self, key, val):\n \"\"\"\n Change value stored in dictionary and add corresponding pair to heap.\n Rebuilds the heap if the number of deleted items gets large, to avoid\n memory leakage.\n \"\"\"\n dict.__setitem__(self, key, val)\n heap = self.__heap\n if len(heap) > 2 * len(self):\n self.__heap = [(v, k) for k, v in self.items()]\n self.__heap.sort()\n else:\n newPair = (val, key)\n insertionPoint = len(heap)\n heap.append(None)\n while insertionPoint > 0 and newPair < heap[(insertionPoint - 1) // 2]:\n heap[insertionPoint] = heap[(insertionPoint - 1) // 2]\n insertionPoint = (insertionPoint - 1) // 2\n heap[insertionPoint] = newPair\n\n def setdefault(self, key, val):\n \"\"\"\n Reimplement setdefault to pass through our customized __setitem__.\n \"\"\"\n if key not in self:\n self[key] = val\n return self[key]\n" }, { "path": "lib/spack/spack/vendor/altgraph/GraphStat.py", "content": "\"\"\"\nspack.vendor.altgraph.GraphStat - Functions providing various graph statistics\n=================================================================\n\"\"\"\n\n\ndef degree_dist(graph, limits=(0, 0), bin_num=10, mode=\"out\"):\n \"\"\"\n Computes the degree distribution for a graph.\n\n Returns a list of tuples where the first element of the tuple is the\n center of the bin representing a range of degrees and the second element\n of the tuple are the number of nodes with the degree falling in the range.\n\n Example::\n\n ....\n \"\"\"\n\n deg = []\n if mode == \"inc\":\n get_deg = graph.inc_degree\n else:\n get_deg = graph.out_degree\n\n for node in graph:\n deg.append(get_deg(node))\n\n if not deg:\n return []\n\n results = _binning(values=deg, limits=limits, bin_num=bin_num)\n\n return results\n\n\n_EPS = 1.0 / (2.0 ** 32)\n\n\ndef _binning(values, limits=(0, 0), bin_num=10):\n \"\"\"\n Bins data that falls between certain limits, if the limits are (0, 0) the\n minimum and maximum values are used.\n\n Returns a list of tuples where the first element of the tuple is the\n center of the bin and the second element of the tuple are the counts.\n \"\"\"\n if limits == (0, 0):\n min_val, max_val = min(values) - _EPS, max(values) + _EPS\n else:\n min_val, max_val = limits\n\n # get bin size\n bin_size = (max_val - min_val) / float(bin_num)\n bins = [0] * (bin_num)\n\n # will ignore these outliers for now\n for value in values:\n try:\n if (value - min_val) >= 0:\n index = int((value - min_val) / float(bin_size))\n bins[index] += 1\n except IndexError:\n pass\n\n # make it ready for an x,y plot\n result = []\n center = (bin_size / 2) + min_val\n for i, y in enumerate(bins):\n x = center + bin_size * i\n result.append((x, y))\n\n return result\n" }, { "path": "lib/spack/spack/vendor/altgraph/GraphUtil.py", "content": "\"\"\"\nspack.vendor.altgraph.GraphUtil - Utility classes and functions\n==================================================\n\"\"\"\n\nimport random\nfrom collections import deque\n\nfrom spack.vendor.altgraph import Graph, GraphError\n\n\ndef generate_random_graph(node_num, edge_num, self_loops=False, multi_edges=False):\n \"\"\"\n Generates and returns a :py:class:`~spack.vendor.altgraph.Graph.Graph` instance with\n *node_num* nodes randomly connected by *edge_num* edges.\n \"\"\"\n g = Graph.Graph()\n\n if not multi_edges:\n if self_loops:\n max_edges = node_num * node_num\n else:\n max_edges = node_num * (node_num - 1)\n\n if edge_num > max_edges:\n raise GraphError(\"inconsistent arguments to 'generate_random_graph'\")\n\n nodes = range(node_num)\n\n for node in nodes:\n g.add_node(node)\n\n while 1:\n head = random.choice(nodes)\n tail = random.choice(nodes)\n\n # loop defense\n if head == tail and not self_loops:\n continue\n\n # multiple edge defense\n if g.edge_by_node(head, tail) is not None and not multi_edges:\n continue\n\n # add the edge\n g.add_edge(head, tail)\n if g.number_of_edges() >= edge_num:\n break\n\n return g\n\n\ndef generate_scale_free_graph(steps, growth_num, self_loops=False, multi_edges=False):\n \"\"\"\n Generates and returns a :py:class:`~spack.vendor.altgraph.Graph.Graph` instance that\n will have *steps* \\\\* *growth_num* nodes and a scale free (powerlaw)\n connectivity. Starting with a fully connected graph with *growth_num*\n nodes at every step *growth_num* nodes are added to the graph and are\n connected to existing nodes with a probability proportional to the degree\n of these existing nodes.\n \"\"\"\n # The code doesn't seem to do what the documentation claims.\n graph = Graph.Graph()\n\n # initialize the graph\n store = []\n for i in range(growth_num):\n for j in range(i + 1, growth_num):\n store.append(i)\n store.append(j)\n graph.add_edge(i, j)\n\n # generate\n for node in range(growth_num, steps * growth_num):\n graph.add_node(node)\n while graph.out_degree(node) < growth_num:\n nbr = random.choice(store)\n\n # loop defense\n if node == nbr and not self_loops:\n continue\n\n # multi edge defense\n if graph.edge_by_node(node, nbr) and not multi_edges:\n continue\n\n graph.add_edge(node, nbr)\n\n for nbr in graph.out_nbrs(node):\n store.append(node)\n store.append(nbr)\n\n return graph\n\n\ndef filter_stack(graph, head, filters):\n \"\"\"\n Perform a walk in a depth-first order starting\n at *head*.\n\n Returns (visited, removes, orphans).\n\n * visited: the set of visited nodes\n * removes: the list of nodes where the node\n data does not all *filters*\n * orphans: tuples of (last_good, node),\n where node is not in removes, is directly\n reachable from a node in *removes* and\n *last_good* is the closest upstream node that is not\n in *removes*.\n \"\"\"\n\n visited, removes, orphans = {head}, set(), set()\n stack = deque([(head, head)])\n get_data = graph.node_data\n get_edges = graph.out_edges\n get_tail = graph.tail\n\n while stack:\n last_good, node = stack.pop()\n data = get_data(node)\n if data is not None:\n for filtfunc in filters:\n if not filtfunc(data):\n removes.add(node)\n break\n else:\n last_good = node\n for edge in get_edges(node):\n tail = get_tail(edge)\n if last_good is not node:\n orphans.add((last_good, tail))\n if tail not in visited:\n visited.add(tail)\n stack.append((last_good, tail))\n\n orphans = [(lg, tl) for (lg, tl) in orphans if tl not in removes]\n\n return visited, removes, orphans\n" }, { "path": "lib/spack/spack/vendor/altgraph/LICENSE", "content": "Copyright (c) 2004 Istvan Albert unless otherwise noted.\nCopyright (c) 2006-2010 Bob Ippolito\nCopyright (2) 2010-2020 Ronald Oussoren, et. al.\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to\ndeal in the Software without restriction, including without limitation the\nrights to use, copy, modify, merge, publish, distribute, sublicense,\nand/or sell copies of the Software, and to permit persons to whom the\nSoftware is furnished to do so.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\nFROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\nIN THE SOFTWARE.\n" }, { "path": "lib/spack/spack/vendor/altgraph/ObjectGraph.py", "content": "\"\"\"\nspack.vendor.altgraph.ObjectGraph - Graph of objects with an identifier\n==========================================================\n\nA graph of objects that have a \"graphident\" attribute.\ngraphident is the key for the object in the graph\n\"\"\"\n\nfrom spack.vendor.altgraph import GraphError\nfrom spack.vendor.altgraph.Graph import Graph\nfrom spack.vendor.altgraph.GraphUtil import filter_stack\n\n\nclass ObjectGraph(object):\n \"\"\"\n A graph of objects that have a \"graphident\" attribute.\n graphident is the key for the object in the graph\n \"\"\"\n\n def __init__(self, graph=None, debug=0):\n if graph is None:\n graph = Graph()\n self.graphident = self\n self.graph = graph\n self.debug = debug\n self.indent = 0\n graph.add_node(self, None)\n\n def __repr__(self):\n return \"<%s>\" % (type(self).__name__,)\n\n def flatten(self, condition=None, start=None):\n \"\"\"\n Iterate over the subgraph that is entirely reachable by condition\n starting from the given start node or the ObjectGraph root\n \"\"\"\n if start is None:\n start = self\n start = self.getRawIdent(start)\n return self.graph.iterdata(start=start, condition=condition)\n\n def nodes(self):\n for ident in self.graph:\n node = self.graph.node_data(ident)\n if node is not None:\n yield self.graph.node_data(ident)\n\n def get_edges(self, node):\n if node is None:\n node = self\n start = self.getRawIdent(node)\n _, _, outraw, incraw = self.graph.describe_node(start)\n\n def iter_edges(lst, n):\n seen = set()\n for tpl in (self.graph.describe_edge(e) for e in lst):\n ident = tpl[n]\n if ident not in seen:\n yield self.findNode(ident)\n seen.add(ident)\n\n return iter_edges(outraw, 3), iter_edges(incraw, 2)\n\n def edgeData(self, fromNode, toNode):\n if fromNode is None:\n fromNode = self\n start = self.getRawIdent(fromNode)\n stop = self.getRawIdent(toNode)\n edge = self.graph.edge_by_node(start, stop)\n return self.graph.edge_data(edge)\n\n def updateEdgeData(self, fromNode, toNode, edgeData):\n if fromNode is None:\n fromNode = self\n start = self.getRawIdent(fromNode)\n stop = self.getRawIdent(toNode)\n edge = self.graph.edge_by_node(start, stop)\n self.graph.update_edge_data(edge, edgeData)\n\n def filterStack(self, filters):\n \"\"\"\n Filter the ObjectGraph in-place by removing all edges to nodes that\n do not match every filter in the given filter list\n\n Returns a tuple containing the number of:\n (nodes_visited, nodes_removed, nodes_orphaned)\n \"\"\"\n visited, removes, orphans = filter_stack(self.graph, self, filters)\n\n for last_good, tail in orphans:\n self.graph.add_edge(last_good, tail, edge_data=\"orphan\")\n\n for node in removes:\n self.graph.hide_node(node)\n\n return len(visited) - 1, len(removes), len(orphans)\n\n def removeNode(self, node):\n \"\"\"\n Remove the given node from the graph if it exists\n \"\"\"\n ident = self.getIdent(node)\n if ident is not None:\n self.graph.hide_node(ident)\n\n def removeReference(self, fromnode, tonode):\n \"\"\"\n Remove all edges from fromnode to tonode\n \"\"\"\n if fromnode is None:\n fromnode = self\n fromident = self.getIdent(fromnode)\n toident = self.getIdent(tonode)\n if fromident is not None and toident is not None:\n while True:\n edge = self.graph.edge_by_node(fromident, toident)\n if edge is None:\n break\n self.graph.hide_edge(edge)\n\n def getIdent(self, node):\n \"\"\"\n Get the graph identifier for a node\n \"\"\"\n ident = self.getRawIdent(node)\n if ident is not None:\n return ident\n node = self.findNode(node)\n if node is None:\n return None\n return node.graphident\n\n def getRawIdent(self, node):\n \"\"\"\n Get the identifier for a node object\n \"\"\"\n if node is self:\n return node\n ident = getattr(node, \"graphident\", None)\n return ident\n\n def __contains__(self, node):\n return self.findNode(node) is not None\n\n def findNode(self, node):\n \"\"\"\n Find the node on the graph\n \"\"\"\n ident = self.getRawIdent(node)\n if ident is None:\n ident = node\n try:\n return self.graph.node_data(ident)\n except KeyError:\n return None\n\n def addNode(self, node):\n \"\"\"\n Add a node to the graph referenced by the root\n \"\"\"\n self.msg(4, \"addNode\", node)\n\n try:\n self.graph.restore_node(node.graphident)\n except GraphError:\n self.graph.add_node(node.graphident, node)\n\n def createReference(self, fromnode, tonode, edge_data=None):\n \"\"\"\n Create a reference from fromnode to tonode\n \"\"\"\n if fromnode is None:\n fromnode = self\n fromident, toident = self.getIdent(fromnode), self.getIdent(tonode)\n if fromident is None or toident is None:\n return\n self.msg(4, \"createReference\", fromnode, tonode, edge_data)\n self.graph.add_edge(fromident, toident, edge_data=edge_data)\n\n def createNode(self, cls, name, *args, **kw):\n \"\"\"\n Add a node of type cls to the graph if it does not already exist\n by the given name\n \"\"\"\n m = self.findNode(name)\n if m is None:\n m = cls(name, *args, **kw)\n self.addNode(m)\n return m\n\n def msg(self, level, s, *args):\n \"\"\"\n Print a debug message with the given level\n \"\"\"\n if s and level <= self.debug:\n print(\"%s%s %s\" % (\" \" * self.indent, s, \" \".join(map(repr, args))))\n\n def msgin(self, level, s, *args):\n \"\"\"\n Print a debug message and indent\n \"\"\"\n if level <= self.debug:\n self.msg(level, s, *args)\n self.indent = self.indent + 1\n\n def msgout(self, level, s, *args):\n \"\"\"\n Dedent and print a debug message\n \"\"\"\n if level <= self.debug:\n self.indent = self.indent - 1\n self.msg(level, s, *args)\n" }, { "path": "lib/spack/spack/vendor/altgraph/__init__.py", "content": "\"\"\"\nspack.vendor.altgraph - a python graph library\n=================================\n\nspack.vendor.altgraph is a fork of `graphlib `_ tailored\nto use newer Python 2.3+ features, including additional support used by the\npy2app suite (modulegraph and spack.vendor.macholib, specifically).\n\nspack.vendor.altgraph is a python based graph (network) representation and manipulation\npackage. It has started out as an extension to the\n`graph_lib module\n`_\nwritten by Nathan Denny it has been significantly optimized and expanded.\n\nThe :class:`spack.vendor.altgraph.Graph.Graph` class is loosely modeled after the\n`LEDA `_\n(Library of Efficient Datatypes) representation. The library\nincludes methods for constructing graphs, BFS and DFS traversals,\ntopological sort, finding connected components, shortest paths as well as a\nnumber graph statistics functions. The library can also visualize graphs\nvia `graphviz `_.\n\nThe package contains the following modules:\n\n - the :py:mod:`spack.vendor.altgraph.Graph` module contains the\n :class:`~spack.vendor.altgraph.Graph.Graph` class that stores the graph data\n\n - the :py:mod:`spack.vendor.altgraph.GraphAlgo` module implements graph algorithms\n operating on graphs (:py:class:`~spack.vendor.altgraph.Graph.Graph`} instances)\n\n - the :py:mod:`spack.vendor.altgraph.GraphStat` module contains functions for\n computing statistical measures on graphs\n\n - the :py:mod:`spack.vendor.altgraph.GraphUtil` module contains functions for\n generating, reading and saving graphs\n\n - the :py:mod:`spack.vendor.altgraph.Dot` module contains functions for displaying\n graphs via `graphviz `_\n\n - the :py:mod:`spack.vendor.altgraph.ObjectGraph` module implements a graph of\n objects with a unique identifier\n\nInstallation\n------------\n\nDownload and unpack the archive then type::\n\n python setup.py install\n\nThis will install the library in the default location. For instructions on\nhow to customize the install procedure read the output of::\n\n python setup.py --help install\n\nTo verify that the code works run the test suite::\n\n python setup.py test\n\nExample usage\n-------------\n\nLets assume that we want to analyze the graph below (links to the full picture)\nGRAPH_IMG. Our script then might look the following way::\n\n from spack.vendor.altgraph import Graph, GraphAlgo, Dot\n\n # these are the edges\n edges = [ (1,2), (2,4), (1,3), (2,4), (3,4), (4,5), (6,5),\n (6,14), (14,15), (6, 15), (5,7), (7, 8), (7,13), (12,8),\n (8,13), (11,12), (11,9), (13,11), (9,13), (13,10) ]\n\n # creates the graph\n graph = Graph.Graph()\n for head, tail in edges:\n graph.add_edge(head, tail)\n\n # do a forward bfs from 1 at most to 20\n print(graph.forw_bfs(1))\n\nThis will print the nodes in some breadth first order::\n\n [1, 2, 3, 4, 5, 7, 8, 13, 11, 10, 12, 9]\n\nIf we wanted to get the hop-distance from node 1 to node 8\nwe coud write::\n\n print(graph.get_hops(1, 8))\n\nThis will print the following::\n\n [(1, 0), (2, 1), (3, 1), (4, 2), (5, 3), (7, 4), (8, 5)]\n\nNode 1 is at 0 hops since it is the starting node, nodes 2,3 are 1 hop away ...\nnode 8 is 5 hops away. To find the shortest distance between two nodes you\ncan use::\n\n print(GraphAlgo.shortest_path(graph, 1, 12))\n\nIt will print the nodes on one (if there are more) the shortest paths::\n\n [1, 2, 4, 5, 7, 13, 11, 12]\n\nTo display the graph we can use the GraphViz backend::\n\n dot = Dot.Dot(graph)\n\n # display the graph on the monitor\n dot.display()\n\n # save it in an image file\n dot.save_img(file_name='graph', file_type='gif')\n\n\n\n..\n @author: U{Istvan Albert}\n\n @license: MIT License\n\n Copyright (c) 2004 Istvan Albert unless otherwise noted.\n\n Permission is hereby granted, free of charge, to any person obtaining a copy\n of this software and associated documentation files (the \"Software\"), to\n deal in the Software without restriction, including without limitation the\n rights to use, copy, modify, merge, publish, distribute, sublicense,\n and/or sell copies of the Software, and to permit persons to whom the\n Software is furnished to do so.\n\n THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n IN THE SOFTWARE.\n @requires: Python 2.3 or higher\n\n @newfield contributor: Contributors:\n @contributor: U{Reka Albert }\n\n\"\"\"\n\n__version__ = \"0.17.3\"\n\n\nclass GraphError(ValueError):\n pass\n" }, { "path": "lib/spack/spack/vendor/archspec/LICENSE", "content": "The MIT License (MIT)\n\nCopyright (c) 2014 Anders Høst\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software is furnished to do so,\nsubject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\nFOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\nCOPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\nIN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\nCONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" }, { "path": "lib/spack/spack/vendor/archspec/LICENSE-APACHE", "content": "\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "lib/spack/spack/vendor/archspec/LICENSE-MIT", "content": "Copyright 2019-2020 Lawrence Livermore National Security, LLC and other\nArchspec Project Developers. See the top-level COPYRIGHT file for details.\n\nPermission is hereby granted, free of charge, to any person obtaining a\ncopy of this software and associated documentation files (the \"Software\"),\nto deal in the Software without restriction, including without limitation\nthe rights to use, copy, modify, merge, publish, distribute, sublicense,\nand/or sell copies of the Software, and to permit persons to whom the\nSoftware is furnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in\nall copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\nFROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\nDEALINGS IN THE SOFTWARE.\n" }, { "path": "lib/spack/spack/vendor/archspec/__init__.py", "content": "\"\"\"Init file to avoid namespace packages\"\"\"\n\n__version__ = \"0.2.5\"\n" }, { "path": "lib/spack/spack/vendor/archspec/__main__.py", "content": "\"\"\"\nRun the `archspec` CLI as a module.\n\"\"\"\n\nimport sys\n\nfrom .cli import main\n\nsys.exit(main())\n" }, { "path": "lib/spack/spack/vendor/archspec/cli.py", "content": "# Copyright 2019-2020 Lawrence Livermore National Security, LLC and other\n# Archspec Project Developers. See the top-level COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"\narchspec command line interface\n\"\"\"\n\nimport argparse\nimport typing\n\nfrom . import __version__ as archspec_version\nfrom .cpu import host\n\n\ndef _make_parser() -> argparse.ArgumentParser:\n parser = argparse.ArgumentParser(\n \"archspec\",\n description=\"archspec command line interface\",\n add_help=False,\n )\n parser.add_argument(\n \"--version\",\n \"-V\",\n help=\"Show the version and exit.\",\n action=\"version\",\n version=f\"archspec, version {archspec_version}\",\n )\n parser.add_argument(\"--help\", \"-h\", help=\"Show the help and exit.\", action=\"help\")\n\n subcommands = parser.add_subparsers(\n title=\"command\",\n metavar=\"COMMAND\",\n dest=\"command\",\n )\n\n cpu_command = subcommands.add_parser(\n \"cpu\",\n help=\"archspec command line interface for CPU\",\n description=\"archspec command line interface for CPU\",\n )\n cpu_command.set_defaults(run=cpu)\n\n return parser\n\n\ndef cpu() -> int:\n \"\"\"Run the `archspec cpu` subcommand.\"\"\"\n try:\n print(host())\n except FileNotFoundError as exc:\n print(exc)\n return 1\n return 0\n\n\ndef main(argv: typing.Optional[typing.List[str]] = None) -> int:\n \"\"\"Run the `archspec` command line interface.\"\"\"\n parser = _make_parser()\n\n try:\n args = parser.parse_args(argv)\n except SystemExit as err:\n return err.code\n\n if args.command is None:\n parser.print_help()\n return 0\n\n return args.run()\n" }, { "path": "lib/spack/spack/vendor/archspec/cpu/__init__.py", "content": "# Copyright 2019-2020 Lawrence Livermore National Security, LLC and other\n# Archspec Project Developers. See the top-level COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"The \"cpu\" package permits to query and compare different\nCPU microarchitectures.\n\"\"\"\nfrom .detect import brand_string, host\nfrom .microarchitecture import (\n TARGETS,\n InvalidCompilerVersion,\n Microarchitecture,\n UnsupportedMicroarchitecture,\n generic_microarchitecture,\n version_components,\n)\n\n__all__ = [\n \"brand_string\",\n \"host\",\n \"TARGETS\",\n \"InvalidCompilerVersion\",\n \"Microarchitecture\",\n \"UnsupportedMicroarchitecture\",\n \"generic_microarchitecture\",\n \"version_components\",\n]\n" }, { "path": "lib/spack/spack/vendor/archspec/cpu/alias.py", "content": "# Copyright 2019-2020 Lawrence Livermore National Security, LLC and other\n# Archspec Project Developers. See the top-level COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Aliases for microarchitecture features.\"\"\"\nfrom typing import Callable, Dict\n\nfrom .schema import TARGETS_JSON, LazyDictionary\n\n_FEATURE_ALIAS_PREDICATE: Dict[str, Callable] = {}\n\n\nclass FeatureAliasTest:\n \"\"\"A test that must be passed for a feature alias to succeed.\n\n Args:\n rules (dict): dictionary of rules to be met. Each key must be a\n valid alias predicate\n \"\"\"\n\n # pylint: disable=too-few-public-methods\n def __init__(self, rules):\n self.rules = rules\n self.predicates = []\n for name, args in rules.items():\n self.predicates.append(_FEATURE_ALIAS_PREDICATE[name](args))\n\n def __call__(self, microarchitecture):\n return all(feature_test(microarchitecture) for feature_test in self.predicates)\n\n\ndef _feature_aliases():\n \"\"\"Returns the dictionary of all defined feature aliases.\"\"\"\n json_data = TARGETS_JSON[\"feature_aliases\"]\n aliases = {}\n for alias, rules in json_data.items():\n aliases[alias] = FeatureAliasTest(rules)\n return aliases\n\n\nFEATURE_ALIASES = LazyDictionary(_feature_aliases)\n\n\ndef alias_predicate(func):\n \"\"\"Decorator to register a predicate that can be used to evaluate\n feature aliases.\n \"\"\"\n name = func.__name__\n\n # Check we didn't register anything else with the same name\n if name in _FEATURE_ALIAS_PREDICATE:\n msg = f'the alias predicate \"{name}\" already exists'\n raise KeyError(msg)\n\n _FEATURE_ALIAS_PREDICATE[name] = func\n\n return func\n\n\n@alias_predicate\ndef reason(_):\n \"\"\"This predicate returns always True and it's there to allow writing\n a documentation string in the JSON file to explain why an alias is needed.\n \"\"\"\n return lambda x: True\n\n\n@alias_predicate\ndef any_of(list_of_features):\n \"\"\"Returns a predicate that is True if any of the feature in the\n list is in the microarchitecture being tested, False otherwise.\n \"\"\"\n\n def _impl(microarchitecture):\n return any(x in microarchitecture for x in list_of_features)\n\n return _impl\n\n\n@alias_predicate\ndef families(list_of_families):\n \"\"\"Returns a predicate that is True if the architecture family of\n the microarchitecture being tested is in the list, False otherwise.\n \"\"\"\n\n def _impl(microarchitecture):\n return str(microarchitecture.family) in list_of_families\n\n return _impl\n" }, { "path": "lib/spack/spack/vendor/archspec/cpu/detect.py", "content": "# Copyright 2019-2020 Lawrence Livermore National Security, LLC and other\n# Archspec Project Developers. See the top-level COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Detection of CPU microarchitectures\"\"\"\nimport collections\nimport os\nimport platform\nimport re\nimport struct\nimport subprocess\nimport warnings\nfrom typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union\n\nfrom ..vendor.cpuid.cpuid import CPUID\nfrom .microarchitecture import TARGETS, Microarchitecture, generic_microarchitecture\nfrom .schema import CPUID_JSON, TARGETS_JSON\n\n#: Mapping from operating systems to chain of commands\n#: to obtain a dictionary of raw info on the current cpu\nINFO_FACTORY = collections.defaultdict(list)\n\n#: Mapping from micro-architecture families (x86_64, ppc64le, etc.) to\n#: functions checking the compatibility of the host with a given target\nCOMPATIBILITY_CHECKS: Dict[str, Callable[[Microarchitecture, Microarchitecture], bool]] = {}\n\n# Constants for commonly used architectures\nX86_64 = \"x86_64\"\nAARCH64 = \"aarch64\"\nPPC64LE = \"ppc64le\"\nPPC64 = \"ppc64\"\nRISCV64 = \"riscv64\"\n\n\ndef detection(operating_system: str):\n \"\"\"Decorator to mark functions that are meant to return partial information on the current cpu.\n\n Args:\n operating_system: operating system where this function can be used.\n \"\"\"\n\n def decorator(factory):\n INFO_FACTORY[operating_system].append(factory)\n return factory\n\n return decorator\n\n\ndef partial_uarch(\n name: str = \"\",\n vendor: str = \"\",\n features: Optional[Set[str]] = None,\n generation: int = 0,\n cpu_part: str = \"\",\n) -> Microarchitecture:\n \"\"\"Construct a partial microarchitecture, from information gathered during system scan.\"\"\"\n return Microarchitecture(\n name=name,\n parents=[],\n vendor=vendor,\n features=features or set(),\n compilers={},\n generation=generation,\n cpu_part=cpu_part,\n )\n\n\n@detection(operating_system=\"Linux\")\ndef proc_cpuinfo() -> Microarchitecture:\n \"\"\"Returns a partial Microarchitecture, obtained from scanning ``/proc/cpuinfo``\"\"\"\n data: Dict[str, Any] = {}\n with open(\"/proc/cpuinfo\") as file: # pylint: disable=unspecified-encoding\n for line in file:\n key, separator, value = line.partition(\":\")\n\n # If there's no separator and info was already populated\n # according to what's written here:\n #\n # http://www.linfo.org/proc_cpuinfo.html\n #\n # we are on a blank line separating two cpus. Exit early as\n # we want to read just the first entry in /proc/cpuinfo\n if separator != \":\" and data:\n break\n\n data[key.strip()] = value.strip()\n\n architecture = _machine()\n if architecture == X86_64:\n return partial_uarch(\n vendor=data.get(\"vendor_id\", \"generic\"), features=_feature_set(data, key=\"flags\")\n )\n\n if architecture == AARCH64:\n return partial_uarch(\n vendor=_canonicalize_aarch64_vendor(data),\n features=_feature_set(data, key=\"Features\"),\n cpu_part=data.get(\"CPU part\", \"\"),\n )\n\n if architecture in (PPC64LE, PPC64):\n generation_match = re.search(r\"POWER(\\d+)\", data.get(\"cpu\", \"\"))\n # There might be no match under emulated environments. For instance\n # emulating a ppc64le with QEMU and Docker still reports the host\n # /proc/cpuinfo and not a Power\n if generation_match is None:\n return partial_uarch(generation=0)\n\n try:\n generation = int(generation_match.group(1))\n except ValueError:\n generation = 0\n return partial_uarch(generation=generation)\n\n if architecture == RISCV64:\n if data.get(\"uarch\") == \"sifive,u74-mc\":\n data[\"uarch\"] = \"u74mc\"\n return partial_uarch(name=data.get(\"uarch\", RISCV64))\n\n return generic_microarchitecture(architecture)\n\n\nclass CpuidInfoCollector:\n \"\"\"Collects the information we need on the host CPU from cpuid\"\"\"\n\n # pylint: disable=too-few-public-methods\n def __init__(self):\n self.cpuid = CPUID()\n\n registers = self.cpuid.registers_for(**CPUID_JSON[\"vendor\"][\"input\"])\n self.highest_basic_support = registers.eax\n self.vendor = struct.pack(\"III\", registers.ebx, registers.edx, registers.ecx).decode(\n \"utf-8\"\n )\n\n registers = self.cpuid.registers_for(**CPUID_JSON[\"highest_extension_support\"][\"input\"])\n self.highest_extension_support = registers.eax\n\n self.features = self._features()\n\n def _features(self):\n result = set()\n\n def check_features(data):\n registers = self.cpuid.registers_for(**data[\"input\"])\n for feature_check in data[\"bits\"]:\n current = getattr(registers, feature_check[\"register\"])\n if self._is_bit_set(current, feature_check[\"bit\"]):\n result.add(feature_check[\"name\"])\n\n for call_data in CPUID_JSON[\"flags\"]:\n if call_data[\"input\"][\"eax\"] > self.highest_basic_support:\n continue\n check_features(call_data)\n\n for call_data in CPUID_JSON[\"extension-flags\"]:\n if call_data[\"input\"][\"eax\"] > self.highest_extension_support:\n continue\n check_features(call_data)\n\n return result\n\n def _is_bit_set(self, register: int, bit: int) -> bool:\n mask = 1 << bit\n return register & mask > 0\n\n def brand_string(self) -> Optional[str]:\n \"\"\"Returns the brand string, if available.\"\"\"\n if self.highest_extension_support < 0x80000004:\n return None\n\n r1 = self.cpuid.registers_for(eax=0x80000002, ecx=0)\n r2 = self.cpuid.registers_for(eax=0x80000003, ecx=0)\n r3 = self.cpuid.registers_for(eax=0x80000004, ecx=0)\n result = struct.pack(\n \"IIIIIIIIIIII\",\n r1.eax,\n r1.ebx,\n r1.ecx,\n r1.edx,\n r2.eax,\n r2.ebx,\n r2.ecx,\n r2.edx,\n r3.eax,\n r3.ebx,\n r3.ecx,\n r3.edx,\n ).decode(\"utf-8\")\n return result.strip(\"\\x00\")\n\n\n@detection(operating_system=\"Windows\")\ndef cpuid_info():\n \"\"\"Returns a partial Microarchitecture, obtained from running the cpuid instruction\"\"\"\n architecture = _machine()\n if architecture == X86_64:\n data = CpuidInfoCollector()\n return partial_uarch(vendor=data.vendor, features=data.features)\n\n return generic_microarchitecture(architecture)\n\n\ndef _check_output(args, env):\n with subprocess.Popen(args, stdout=subprocess.PIPE, env=env) as proc:\n output = proc.communicate()[0]\n return str(output.decode(\"utf-8\"))\n\n\nWINDOWS_MAPPING = {\n \"AMD64\": X86_64,\n \"ARM64\": AARCH64,\n}\n\n\ndef _machine() -> str:\n \"\"\"Return the machine architecture we are on\"\"\"\n operating_system = platform.system()\n\n # If we are not on Darwin or Windows, trust what Python tells us\n if operating_system not in (\"Darwin\", \"Windows\"):\n return platform.machine()\n\n # Normalize windows specific names\n if operating_system == \"Windows\":\n platform_machine = platform.machine()\n return WINDOWS_MAPPING.get(platform_machine, platform_machine)\n\n # On Darwin it might happen that we are on M1, but using an interpreter\n # built for x86_64. In that case \"platform.machine() == 'x86_64'\", so we\n # need to fix that.\n #\n # See: https://bugs.python.org/issue42704\n output = _check_output(\n [\"sysctl\", \"-n\", \"machdep.cpu.brand_string\"], env=_ensure_bin_usrbin_in_path()\n ).strip()\n\n if \"Apple\" in output:\n # Note that a native Python interpreter on Apple M1 would return\n # \"arm64\" instead of \"aarch64\". Here we normalize to the latter.\n return AARCH64\n\n return X86_64\n\n\n@detection(operating_system=\"Darwin\")\ndef sysctl_info() -> Microarchitecture:\n \"\"\"Returns a raw info dictionary parsing the output of sysctl.\"\"\"\n child_environment = _ensure_bin_usrbin_in_path()\n\n def sysctl(*args: str) -> str:\n return _check_output([\"sysctl\", *args], env=child_environment).strip()\n\n if _machine() == X86_64:\n raw_features = sysctl(\n \"-n\",\n \"machdep.cpu.features\",\n \"machdep.cpu.leaf7_features\",\n \"machdep.cpu.extfeatures\",\n )\n features = set(raw_features.lower().split())\n\n # Flags detected on Darwin turned to their linux counterpart\n for darwin_flags, linux_flags in TARGETS_JSON[\"conversions\"][\"darwin_flags\"].items():\n if all(x in features for x in darwin_flags.split()):\n features.update(linux_flags.split())\n\n return partial_uarch(vendor=sysctl(\"-n\", \"machdep.cpu.vendor\"), features=features)\n\n model = \"unknown\"\n model_str = sysctl(\"-n\", \"machdep.cpu.brand_string\").lower()\n if \"m4\" in model_str:\n model = \"m4\"\n elif \"m3\" in model_str:\n model = \"m3\"\n elif \"m2\" in model_str:\n model = \"m2\"\n elif \"m1\" in model_str:\n model = \"m1\"\n elif \"apple\" in model_str:\n model = \"m1\"\n\n return partial_uarch(name=model, vendor=\"Apple\")\n\n\ndef _ensure_bin_usrbin_in_path():\n # Make sure that /sbin and /usr/sbin are in PATH as sysctl is usually found there\n child_environment = dict(os.environ.items())\n search_paths = child_environment.get(\"PATH\", \"\").split(os.pathsep)\n for additional_path in (\"/sbin\", \"/usr/sbin\"):\n if additional_path not in search_paths:\n search_paths.append(additional_path)\n child_environment[\"PATH\"] = os.pathsep.join(search_paths)\n return child_environment\n\n\ndef _canonicalize_aarch64_vendor(data: Dict[str, str]) -> str:\n \"\"\"Adjust the vendor field to make it human-readable\"\"\"\n if \"CPU implementer\" not in data:\n return \"generic\"\n\n # Mapping numeric codes to vendor (ARM). This list is a merge from\n # different sources:\n #\n # https://github.com/karelzak/util-linux/blob/master/sys-utils/lscpu-arm.c\n # https://developer.arm.com/docs/ddi0487/latest/arm-architecture-reference-manual-armv8-for-armv8-a-architecture-profile\n # https://github.com/gcc-mirror/gcc/blob/master/gcc/config/aarch64/aarch64-cores.def\n # https://patchwork.kernel.org/patch/10524949/\n arm_vendors = TARGETS_JSON[\"conversions\"][\"arm_vendors\"]\n arm_code = data[\"CPU implementer\"]\n return arm_vendors.get(arm_code, arm_code)\n\n\ndef _feature_set(data: Dict[str, str], key: str) -> Set[str]:\n return set(data.get(key, \"\").split())\n\n\ndef detected_info() -> Microarchitecture:\n \"\"\"Returns a partial Microarchitecture with information on the CPU of the current host.\n\n This function calls all the viable factories one after the other until there's one that is\n able to produce the requested information. Falls-back to a generic microarchitecture, if none\n of the calls succeed.\n \"\"\"\n # pylint: disable=broad-except\n for factory in INFO_FACTORY[platform.system()]:\n try:\n return factory()\n except Exception as exc:\n warnings.warn(str(exc))\n\n return generic_microarchitecture(_machine())\n\n\ndef compatible_microarchitectures(info: Microarchitecture) -> List[Microarchitecture]:\n \"\"\"Returns an unordered list of known micro-architectures that are compatible with the\n partial Microarchitecture passed as input.\n \"\"\"\n architecture_family = _machine()\n # If a tester is not registered, assume no known target is compatible with the host\n tester = COMPATIBILITY_CHECKS.get(architecture_family, lambda x, y: False)\n return [x for x in TARGETS.values() if tester(info, x)] or [\n generic_microarchitecture(architecture_family)\n ]\n\n\ndef host() -> Microarchitecture:\n \"\"\"Detects the host micro-architecture and returns it.\"\"\"\n # Retrieve information on the host's cpu\n info = detected_info()\n\n # Get a list of possible candidates for this micro-architecture\n candidates = compatible_microarchitectures(info)\n\n # Sorting criteria for candidates\n def sorting_fn(item):\n return len(item.ancestors), len(item.features)\n\n # Get the best generic micro-architecture\n generic_candidates = [c for c in candidates if c.vendor == \"generic\"]\n best_generic = max(generic_candidates, key=sorting_fn)\n\n # Relevant for AArch64. Filter on \"cpu_part\" if we have any match\n if info.cpu_part != \"\" and any(c for c in candidates if info.cpu_part == c.cpu_part):\n candidates = [c for c in candidates if info.cpu_part == c.cpu_part]\n\n # Filter the candidates to be descendant of the best generic candidate.\n # This is to avoid that the lack of a niche feature that can be disabled\n # from e.g. BIOS prevents detection of a reasonably performant architecture\n candidates = [c for c in candidates if c > best_generic]\n\n # If we don't have candidates, return the best generic micro-architecture\n if not candidates:\n return best_generic\n\n # Reverse sort of the depth for the inheritance tree among only targets we\n # can use. This gets the newest target we satisfy.\n return max(candidates, key=sorting_fn)\n\n\ndef compatibility_check(architecture_family: Union[str, Tuple[str, ...]]):\n \"\"\"Decorator to register a function as a proper compatibility check.\n\n A compatibility check function takes a partial Microarchitecture object as a first argument,\n and an arbitrary target Microarchitecture as the second argument. It returns True if the\n target is compatible with the first argument, False otherwise.\n\n Args:\n architecture_family: architecture family for which this test can be used\n \"\"\"\n # Turn the argument into something iterable\n if isinstance(architecture_family, str):\n architecture_family = (architecture_family,)\n\n def decorator(func):\n COMPATIBILITY_CHECKS.update({family: func for family in architecture_family})\n return func\n\n return decorator\n\n\n@compatibility_check(architecture_family=(PPC64LE, PPC64))\ndef compatibility_check_for_power(info, target):\n \"\"\"Compatibility check for PPC64 and PPC64LE architectures.\"\"\"\n # We can use a target if it descends from our machine type, and our\n # generation (9 for POWER9, etc.) is at least its generation.\n arch_root = TARGETS[_machine()]\n return (\n target == arch_root or arch_root in target.ancestors\n ) and target.generation <= info.generation\n\n\n@compatibility_check(architecture_family=X86_64)\ndef compatibility_check_for_x86_64(info, target):\n \"\"\"Compatibility check for x86_64 architectures.\"\"\"\n # We can use a target if it descends from our machine type, is from our\n # vendor, and we have all of its features\n arch_root = TARGETS[X86_64]\n return (\n (target == arch_root or arch_root in target.ancestors)\n and target.vendor in (info.vendor, \"generic\")\n and target.features.issubset(info.features)\n )\n\n\n@compatibility_check(architecture_family=AARCH64)\ndef compatibility_check_for_aarch64(info, target):\n \"\"\"Compatibility check for AARCH64 architectures.\"\"\"\n # At the moment, it's not clear how to detect compatibility with\n # a specific version of the architecture\n if target.vendor == \"generic\" and target.name != AARCH64:\n return False\n\n arch_root = TARGETS[AARCH64]\n arch_root_and_vendor = arch_root == target.family and target.vendor in (\n info.vendor,\n \"generic\",\n )\n\n # On macOS it seems impossible to get all the CPU features\n # with syctl info, but for ARM we can get the exact model\n if platform.system() == \"Darwin\":\n model = TARGETS[info.name]\n return arch_root_and_vendor and (target == model or target in model.ancestors)\n\n return arch_root_and_vendor and target.features.issubset(info.features)\n\n\n@compatibility_check(architecture_family=RISCV64)\ndef compatibility_check_for_riscv64(info, target):\n \"\"\"Compatibility check for riscv64 architectures.\"\"\"\n arch_root = TARGETS[RISCV64]\n return (target == arch_root or arch_root in target.ancestors) and (\n target.name == info.name or target.vendor == \"generic\"\n )\n\n\ndef brand_string() -> Optional[str]:\n \"\"\"Returns the brand string of the host, if detected, or None.\"\"\"\n if platform.system() == \"Darwin\":\n return _check_output(\n [\"sysctl\", \"-n\", \"machdep.cpu.brand_string\"], env=_ensure_bin_usrbin_in_path()\n ).strip()\n\n if host().family == X86_64:\n return CpuidInfoCollector().brand_string()\n\n return None\n" }, { "path": "lib/spack/spack/vendor/archspec/cpu/microarchitecture.py", "content": "# Copyright 2019-2020 Lawrence Livermore National Security, LLC and other\n# Archspec Project Developers. See the top-level COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Types and functions to manage information on CPU microarchitectures.\"\"\"\nimport functools\nimport platform\nimport re\nimport sys\nimport warnings\nfrom typing import IO, Any, Dict, List, Optional, Set, Tuple, Union\n\nfrom . import schema\nfrom .alias import FEATURE_ALIASES\nfrom .schema import LazyDictionary\n\n\ndef coerce_target_names(func):\n \"\"\"Decorator that automatically converts a known target name to a proper\n Microarchitecture object.\n \"\"\"\n\n @functools.wraps(func)\n def _impl(self, other):\n if isinstance(other, str):\n if other not in TARGETS:\n msg = '\"{0}\" is not a valid target name'\n raise ValueError(msg.format(other))\n other = TARGETS[other]\n\n return func(self, other)\n\n return _impl\n\n\nclass Microarchitecture:\n \"\"\"A specific CPU micro-architecture\"\"\"\n\n # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-instance-attributes\n #: Aliases for micro-architecture's features\n feature_aliases = FEATURE_ALIASES\n\n def __init__(\n self,\n name: str,\n parents: List[\"Microarchitecture\"],\n vendor: str,\n features: Set[str],\n compilers: Dict[str, List[Dict[str, str]]],\n generation: int = 0,\n cpu_part: str = \"\",\n ):\n \"\"\"\n Args:\n name: name of the micro-architecture (e.g. ``icelake``)\n parents: list of parent micro-architectures, if any. Parenthood is considered by\n cpu features and not chronologically. As such, each micro-architecture is\n compatible with its ancestors. For example, ``skylake``, which has ``broadwell``\n as a parent, supports running binaries optimized for ``broadwell``.\n vendor: vendor of the micro-architecture\n features: supported CPU flags. Note that the semantic of the flags in this field might\n vary among architectures, if at all present. For instance, x86_64 processors will\n list all the flags supported by a given CPU, while Arm processors will list instead\n only the flags that have been added on top of the base model for the current\n micro-architecture.\n compilers: compiler support to generate tuned code for this micro-architecture. This\n dictionary has as keys names of supported compilers, while values are a list of\n dictionaries with fields:\n\n * name: name of the micro-architecture according to the compiler. This is the name\n passed to the ``-march`` option or similar. Not needed if it is the same as\n ``self.name``.\n * versions: versions that support this micro-architecture.\n * flags: flags to be passed to the compiler to generate optimized code\n\n generation: generation of the micro-architecture, if relevant.\n cpu_part: cpu part of the architecture, if relevant.\n \"\"\"\n self.name = name\n self.parents = parents\n self.vendor = vendor\n self.features = features\n self.compilers = compilers\n # Only relevant for PowerPC\n self.generation = generation\n # Only relevant for AArch64\n self.cpu_part = cpu_part\n\n # Cache the \"ancestor\" computation\n self._ancestors: Optional[List[\"Microarchitecture\"]] = None\n # Cache the \"generic\" computation\n self._generic: Optional[\"Microarchitecture\"] = None\n # Cache the \"family\" computation\n self._family: Optional[\"Microarchitecture\"] = None\n\n # ssse3 implies sse3; on Linux sse3 is not mentioned in /proc/cpuinfo, so add it ad-hoc.\n if \"ssse3\" in self.features:\n self.features.add(\"sse3\")\n\n @property\n def ancestors(self) -> List[\"Microarchitecture\"]:\n \"\"\"All the ancestors of this microarchitecture.\"\"\"\n if self._ancestors is None:\n value = self.parents[:]\n for parent in self.parents:\n value.extend(a for a in parent.ancestors if a not in value)\n self._ancestors = value\n return self._ancestors\n\n def _to_set(self) -> Set[str]:\n \"\"\"Returns a set of the nodes in this microarchitecture DAG.\"\"\"\n # This function is used to implement subset semantics with\n # comparison operators\n return set([str(self)] + [str(x) for x in self.ancestors])\n\n @coerce_target_names\n def __eq__(self, other: Union[str, \"Microarchitecture\"]) -> bool:\n if not isinstance(other, Microarchitecture):\n return NotImplemented\n\n return (\n self.name == other.name\n and self.vendor == other.vendor\n and self.features == other.features\n and self.parents == other.parents # avoid ancestors here\n and self.compilers == other.compilers\n and self.generation == other.generation\n and self.cpu_part == other.cpu_part\n )\n\n def __hash__(self) -> int:\n return hash(self.name)\n\n @coerce_target_names\n def __ne__(self, other: Union[str, \"Microarchitecture\"]) -> bool:\n return not self == other\n\n @coerce_target_names\n def __lt__(self, other: Union[str, \"Microarchitecture\"]) -> bool:\n if not isinstance(other, Microarchitecture):\n return NotImplemented\n\n return self._to_set() < other._to_set()\n\n @coerce_target_names\n def __le__(self, other: Union[str, \"Microarchitecture\"]) -> bool:\n return (self == other) or (self < other)\n\n @coerce_target_names\n def __gt__(self, other: Union[str, \"Microarchitecture\"]) -> bool:\n if not isinstance(other, Microarchitecture):\n return NotImplemented\n\n return self._to_set() > other._to_set()\n\n @coerce_target_names\n def __ge__(self, other: Union[str, \"Microarchitecture\"]) -> bool:\n return (self == other) or (self > other)\n\n def __repr__(self) -> str:\n return f\"{self.__class__.__name__}({self.name!r})\"\n\n def __str__(self) -> str:\n return self.name\n\n def tree(self, fp: IO[str] = sys.stdout, indent: int = 4) -> None:\n \"\"\"Format the partial order of this microarchitecture's ancestors as a tree.\"\"\"\n seen: Set[str] = set()\n stack: List[Tuple[int, Microarchitecture]] = [(0, self)]\n while stack:\n level, current = stack.pop()\n print(f\"{'':>{level}}{current.name}\", file=fp)\n\n if current.name in seen:\n continue\n\n for parent in reversed(current.parents):\n stack.append((level + indent, parent))\n\n def __contains__(self, feature: str) -> bool:\n # Feature must be of a string type, so be defensive about that\n if not isinstance(feature, str):\n msg = \"only objects of string types are accepted [got {0}]\"\n raise TypeError(msg.format(str(type(feature))))\n\n # Here we look first in the raw features, and fall-back to\n # feature aliases if not match was found\n if feature in self.features:\n return True\n\n # Check if the alias is defined, if not it will return False\n match_alias = Microarchitecture.feature_aliases.get(feature, lambda x: False)\n return match_alias(self)\n\n @property\n def family(self) -> \"Microarchitecture\":\n \"\"\"Returns the architecture family a given target belongs to\"\"\"\n if self._family is None:\n roots = [x for x in [self] + self.ancestors if not x.ancestors]\n msg = \"a target is expected to belong to just one architecture family\"\n msg += f\"[found {', '.join(str(x) for x in roots)}]\"\n assert len(roots) == 1, msg\n self._family = roots.pop()\n\n return self._family\n\n @property\n def generic(self) -> \"Microarchitecture\":\n \"\"\"Returns the best generic architecture that is compatible with self\"\"\"\n if self._generic is None:\n generics = [x for x in [self] + self.ancestors if x.vendor == \"generic\"]\n self._generic = max(generics, key=lambda x: len(x.ancestors))\n return self._generic\n\n def to_dict(self) -> Dict[str, Any]:\n \"\"\"Returns a dictionary representation of this object.\"\"\"\n return {\n \"name\": str(self.name),\n \"vendor\": str(self.vendor),\n \"features\": sorted(str(x) for x in self.features),\n \"generation\": self.generation,\n \"parents\": [str(x) for x in self.parents],\n \"compilers\": self.compilers,\n \"cpupart\": self.cpu_part,\n }\n\n @staticmethod\n def from_dict(data) -> \"Microarchitecture\":\n \"\"\"Construct a microarchitecture from a dictionary representation.\"\"\"\n return Microarchitecture(\n name=data[\"name\"],\n parents=[TARGETS[x] for x in data[\"parents\"]],\n vendor=data[\"vendor\"],\n features=set(data[\"features\"]),\n compilers=data.get(\"compilers\", {}),\n generation=data.get(\"generation\", 0),\n cpu_part=data.get(\"cpupart\", \"\"),\n )\n\n def optimization_flags(self, compiler: str, version: str) -> str:\n \"\"\"Returns a string containing the optimization flags that needs to be used to produce\n code optimized for this micro-architecture.\n\n The version is expected to be a string of dot-separated digits.\n\n If there is no information on the compiler passed as an argument, the function returns an\n empty string. If it is known that the compiler version we want to use does not support\n this architecture, the function raises an exception.\n\n Args:\n compiler: name of the compiler to be used\n version: version of the compiler to be used\n\n Raises:\n UnsupportedMicroarchitecture: if the requested compiler does not support\n this micro-architecture.\n ValueError: if the version doesn't match the expected format\n \"\"\"\n # If we don't have information on compiler at all return an empty string\n if compiler not in self.family.compilers:\n return \"\"\n\n # If we have information, but it stops before this\n # microarchitecture, fall back to the best known target\n if compiler not in self.compilers:\n best_target = [x for x in self.ancestors if compiler in x.compilers][0]\n msg = (\n \"'{0}' compiler is known to optimize up to the '{1}'\"\n \" microarchitecture in the '{2}' architecture family\"\n )\n msg = msg.format(compiler, best_target, best_target.family)\n raise UnsupportedMicroarchitecture(msg)\n\n # Check that the version matches the expected format\n if not re.match(r\"^(?:\\d+\\.)*\\d+$\", version):\n msg = (\n \"invalid format for the compiler version argument. \"\n \"Only dot separated digits are allowed.\"\n )\n raise InvalidCompilerVersion(msg)\n\n # If we have information on this compiler we need to check the\n # version being used\n compiler_info = self.compilers[compiler]\n\n def satisfies_constraint(entry, version):\n min_version, max_version = entry[\"versions\"].split(\":\")\n\n # Check version suffixes\n min_version, _ = version_components(min_version)\n max_version, _ = version_components(max_version)\n version, _ = version_components(version)\n\n # Assume compiler versions fit into semver\n def tuplify(ver):\n return tuple(int(y) for y in ver.split(\".\"))\n\n version = tuplify(version)\n if min_version:\n min_version = tuplify(min_version)\n if min_version > version:\n return False\n\n if max_version:\n max_version = tuplify(max_version)\n if max_version < version:\n return False\n\n return True\n\n for compiler_entry in compiler_info:\n if satisfies_constraint(compiler_entry, version):\n flags_fmt = compiler_entry[\"flags\"]\n # If there's no field name, use the name of the\n # micro-architecture\n compiler_entry.setdefault(\"name\", self.name)\n\n # Check if we need to emit a warning\n warning_message = compiler_entry.get(\"warnings\", None)\n if warning_message:\n warnings.warn(warning_message)\n\n flags = flags_fmt.format(**compiler_entry)\n return flags\n\n msg = \"cannot produce optimized binary for micro-architecture '{0}' with {1}@{2}\"\n if compiler_info:\n versions = [x[\"versions\"] for x in compiler_info]\n msg += f' [supported compiler versions are {\", \".join(versions)}]'\n else:\n msg += \" [no supported compiler versions]\"\n msg = msg.format(self.name, compiler, version)\n raise UnsupportedMicroarchitecture(msg)\n\n\ndef generic_microarchitecture(name: str) -> Microarchitecture:\n \"\"\"Returns a generic micro-architecture with no vendor and no features.\n\n Args:\n name: name of the micro-architecture\n \"\"\"\n return Microarchitecture(name, parents=[], vendor=\"generic\", features=set(), compilers={})\n\n\ndef version_components(version: str) -> Tuple[str, str]:\n \"\"\"Decomposes the version passed as input in version number and\n suffix and returns them.\n\n If the version number or the suffix are not present, an empty\n string is returned.\n\n Args:\n version: version to be decomposed into its components\n \"\"\"\n match = re.match(r\"([\\d.]*)(-?)(.*)\", str(version))\n if not match:\n return \"\", \"\"\n\n version_number = match.group(1)\n suffix = match.group(3)\n\n return version_number, suffix\n\n\ndef _known_microarchitectures():\n \"\"\"Returns a dictionary of the known micro-architectures. If the\n current host platform is unknown, add it too as a generic target.\n \"\"\"\n\n def fill_target_from_dict(name, data, targets):\n \"\"\"Recursively fills targets by adding the micro-architecture\n passed as argument and all its ancestors.\n\n Args:\n name (str): micro-architecture to be added to targets.\n data (dict): raw data loaded from JSON.\n targets (dict): dictionary that maps micro-architecture names\n to ``Microarchitecture`` objects\n \"\"\"\n values = data[name]\n\n # Get direct parents of target\n parent_names = values[\"from\"]\n for parent in parent_names:\n # Recursively fill parents so they exist before we add them\n if parent in targets:\n continue\n fill_target_from_dict(parent, data, targets)\n parents = [targets.get(parent) for parent in parent_names]\n\n vendor = values[\"vendor\"]\n features = set(values[\"features\"])\n compilers = values.get(\"compilers\", {})\n generation = values.get(\"generation\", 0)\n cpu_part = values.get(\"cpupart\", \"\")\n\n targets[name] = Microarchitecture(\n name, parents, vendor, features, compilers, generation=generation, cpu_part=cpu_part\n )\n\n known_targets = {}\n data = schema.TARGETS_JSON[\"microarchitectures\"]\n for name in data:\n if name in known_targets:\n # name was already brought in as ancestor to a target\n continue\n fill_target_from_dict(name, data, known_targets)\n\n # Add the host platform if not present\n host_platform = platform.machine()\n known_targets.setdefault(host_platform, generic_microarchitecture(host_platform))\n\n return known_targets\n\n\n#: Dictionary of known micro-architectures\nTARGETS = LazyDictionary(_known_microarchitectures)\n\n\nclass ArchspecError(Exception):\n \"\"\"Base class for errors within archspec\"\"\"\n\n\nclass UnsupportedMicroarchitecture(ArchspecError, ValueError):\n \"\"\"Raised if a compiler version does not support optimization for a given\n micro-architecture.\n \"\"\"\n\n\nclass InvalidCompilerVersion(ArchspecError, ValueError):\n \"\"\"Raised when an invalid format is used for compiler versions in archspec.\"\"\"\n" }, { "path": "lib/spack/spack/vendor/archspec/cpu/schema.py", "content": "# Copyright 2019-2020 Lawrence Livermore National Security, LLC and other\n# Archspec Project Developers. See the top-level COPYRIGHT file for details.\n#\n# SPDX-License-Identifier: (Apache-2.0 OR MIT)\n\"\"\"Global objects with the content of the microarchitecture\nJSON file and its schema\n\"\"\"\nimport collections.abc\nimport json\nimport os\nimport pathlib\nfrom typing import Optional, Tuple\n\n\nclass LazyDictionary(collections.abc.MutableMapping):\n \"\"\"Lazy dictionary that gets constructed on first access to any object key\n\n Args:\n factory (callable): factory function to construct the dictionary\n \"\"\"\n\n def __init__(self, factory, *args, **kwargs):\n self.factory = factory\n self.args = args\n self.kwargs = kwargs\n self._data = None\n\n @property\n def data(self):\n \"\"\"Returns the lazily constructed dictionary\"\"\"\n if self._data is None:\n self._data = self.factory(*self.args, **self.kwargs)\n return self._data\n\n def __getitem__(self, key):\n return self.data[key]\n\n def __setitem__(self, key, value):\n self.data[key] = value\n\n def __delitem__(self, key):\n del self.data[key]\n\n def __iter__(self):\n return iter(self.data)\n\n def __len__(self):\n return len(self.data)\n\n\n#: Environment variable that might point to a directory with a user defined JSON file\nDIR_FROM_ENVIRONMENT = \"ARCHSPEC_CPU_DIR\"\n\n#: Environment variable that might point to a directory with extensions to JSON files\nEXTENSION_DIR_FROM_ENVIRONMENT = \"ARCHSPEC_EXTENSION_CPU_DIR\"\n\n\ndef _json_file(\n filename: str, allow_custom: bool = False\n) -> Tuple[pathlib.Path, Optional[pathlib.Path]]:\n \"\"\"Given a filename, returns the absolute path for the main JSON file, and an\n optional absolute path for an extension JSON file.\n\n Args:\n filename: filename for the JSON file\n allow_custom: if True, allows overriding the location where the file resides\n \"\"\"\n json_dir = pathlib.Path(__file__).parent / \"..\" / \"json\" / \"cpu\"\n if allow_custom and DIR_FROM_ENVIRONMENT in os.environ:\n json_dir = pathlib.Path(os.environ[DIR_FROM_ENVIRONMENT])\n json_dir = json_dir.absolute()\n json_file = json_dir / filename\n\n extension_file = None\n if allow_custom and EXTENSION_DIR_FROM_ENVIRONMENT in os.environ:\n extension_dir = pathlib.Path(os.environ[EXTENSION_DIR_FROM_ENVIRONMENT])\n extension_dir.absolute()\n extension_file = extension_dir / filename\n\n return json_file, extension_file\n\n\ndef _load(json_file: pathlib.Path, extension_file: pathlib.Path):\n with open(json_file, \"r\", encoding=\"utf-8\") as file:\n data = json.load(file)\n\n if not extension_file or not extension_file.exists():\n return data\n\n with open(extension_file, \"r\", encoding=\"utf-8\") as file:\n extension_data = json.load(file)\n\n top_level_sections = list(data.keys())\n for key in top_level_sections:\n if key not in extension_data:\n continue\n\n data[key].update(extension_data[key])\n\n return data\n\n\n#: In memory representation of the data in microarchitectures.json, loaded on first access\nTARGETS_JSON = LazyDictionary(_load, *_json_file(\"microarchitectures.json\", allow_custom=True))\n\n#: JSON schema for microarchitectures.json, loaded on first access\nTARGETS_JSON_SCHEMA = LazyDictionary(_load, *_json_file(\"microarchitectures_schema.json\"))\n\n#: Information on how to call 'cpuid' to get information on the HOST CPU\nCPUID_JSON = LazyDictionary(_load, *_json_file(\"cpuid.json\", allow_custom=True))\n\n#: JSON schema for cpuid.json, loaded on first access\nCPUID_JSON_SCHEMA = LazyDictionary(_load, *_json_file(\"cpuid_schema.json\"))\n" }, { "path": "lib/spack/spack/vendor/archspec/json/COPYRIGHT", "content": "Intellectual Property Notice\n------------------------------\n\nArchspec is licensed under the Apache License, Version 2.0 (LICENSE-APACHE\nor http://www.apache.org/licenses/LICENSE-2.0) or the MIT license,\n(LICENSE-MIT or http://opensource.org/licenses/MIT), at your option.\n\nCopyrights and patents in the Archspec project are retained by contributors.\nNo copyright assignment is required to contribute to Archspec.\n\n\nSPDX usage\n------------\n\nIndividual files contain SPDX tags instead of the full license text.\nThis enables machine processing of license information based on the SPDX\nLicense Identifiers that are available here: https://spdx.org/licenses/\n\nFiles that are dual-licensed as Apache-2.0 OR MIT contain the following\ntext in the license header:\n\n SPDX-License-Identifier: (Apache-2.0 OR MIT)\n" }, { "path": "lib/spack/spack/vendor/archspec/json/LICENSE-APACHE", "content": "\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "lib/spack/spack/vendor/archspec/json/LICENSE-MIT", "content": "Copyright 2019-2020 Lawrence Livermore National Security, LLC and other\nArchspec Project Developers. See the top-level COPYRIGHT file for details.\n\nPermission is hereby granted, free of charge, to any person obtaining a\ncopy of this software and associated documentation files (the \"Software\"),\nto deal in the Software without restriction, including without limitation\nthe rights to use, copy, modify, merge, publish, distribute, sublicense,\nand/or sell copies of the Software, and to permit persons to whom the\nSoftware is furnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in\nall copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\nFROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\nDEALINGS IN THE SOFTWARE.\n" }, { "path": "lib/spack/spack/vendor/archspec/json/NOTICE", "content": "This work was produced under the auspices of the U.S. Department of\nEnergy by Lawrence Livermore National Laboratory under Contract\nDE-AC52-07NA27344.\n\nThis work was prepared as an account of work sponsored by an agency of\nthe United States Government. Neither the United States Government nor\nLawrence Livermore National Security, LLC, nor any of their employees\nmakes any warranty, expressed or implied, or assumes any legal liability\nor responsibility for the accuracy, completeness, or usefulness of any\ninformation, apparatus, product, or process disclosed, or represents that\nits use would not infringe privately owned rights.\n\nReference herein to any specific commercial product, process, or service\nby trade name, trademark, manufacturer, or otherwise does not necessarily\nconstitute or imply its endorsement, recommendation, or favoring by the\nUnited States Government or Lawrence Livermore National Security, LLC.\n\nThe views and opinions of authors expressed herein do not necessarily\nstate or reflect those of the United States Government or Lawrence\nLivermore National Security, LLC, and shall not be used for advertising\nor product endorsement purposes.\n" }, { "path": "lib/spack/spack/vendor/archspec/json/README.md", "content": "[![](https://github.com/archspec/archspec-json/workflows/JSON%20Validation/badge.svg)](https://github.com/archspec/archspec-json/actions)\n\n# Archspec-json\n\nThe [archspec-json](https://github.com/archspec/archspec-json) repository is part of the\n[Archspec](https://github.com/archspec) project. It contains data on various architectural\naspects of a platform stored in JSON format and is meant to be used as a base to develop\nlanguage specific APIs.\n\nCurrently the repository contains the following JSON files:\n```console\ncpu/\n├── cpuid.json # Contains information on CPUID calls to retrieve vendor and features on x86_64\n├── cpuid_schema.json # Schema for the file above\n├── microarchitectures.json # Contains information on CPU microarchitectures\n└── microarchitectures_schema.json # Schema for the file above\n ```\n\n\n## License\n\nArchspec is distributed under the terms of both the MIT license and the\nApache License (Version 2.0). Users may choose either license, at their\noption.\n\nAll new contributions must be made under both the MIT and Apache-2.0\nlicenses.\n\nSee [LICENSE-MIT](https://github.com/archspec/archspec-json/blob/master/LICENSE-MIT),\n[LICENSE-APACHE](https://github.com/archspec/archspec-json/blob/master/LICENSE-APACHE),\n[COPYRIGHT](https://github.com/archspec/archspec-json/blob/master/COPYRIGHT), and\n[NOTICE](https://github.com/archspec/archspec-json/blob/master/NOTICE) for details.\n\nSPDX-License-Identifier: (Apache-2.0 OR MIT)\n\nLLNL-CODE-811653\n" }, { "path": "lib/spack/spack/vendor/archspec/json/cpu/cpuid.json", "content": "{\n \"vendor\": {\n \"description\": \"https://en.wikipedia.org/wiki/CPUID#EAX=0:_Highest_Function_Parameter_and_Manufacturer_ID\",\n \"input\": {\n \"eax\": 0,\n \"ecx\": 0\n }\n },\n \"highest_extension_support\": {\n \"description\": \"https://en.wikipedia.org/wiki/CPUID#EAX=80000000h:_Get_Highest_Extended_Function_Implemented\",\n \"input\": {\n \"eax\": 2147483648,\n \"ecx\": 0\n }\n },\n \"flags\": [\n {\n \"description\": \"https://en.wikipedia.org/wiki/CPUID#EAX=1:_Processor_Info_and_Feature_Bits\",\n \"input\": {\n \"eax\": 1,\n \"ecx\": 0\n },\n \"bits\": [\n {\n \"name\": \"fpu\",\n \"register\": \"edx\",\n \"bit\": 0\n },\n {\n \"name\": \"vme\",\n \"register\": \"edx\",\n \"bit\": 1\n },\n {\n \"name\": \"de\",\n \"register\": \"edx\",\n \"bit\": 2\n },\n {\n \"name\": \"pse\",\n \"register\": \"edx\",\n \"bit\": 3\n },\n {\n \"name\": \"tsc\",\n \"register\": \"edx\",\n \"bit\": 4\n },\n {\n \"name\": \"msr\",\n \"register\": \"edx\",\n \"bit\": 5\n },\n {\n \"name\": \"pae\",\n \"register\": \"edx\",\n \"bit\": 6\n },\n {\n \"name\": \"mce\",\n \"register\": \"edx\",\n \"bit\": 7\n },\n {\n \"name\": \"cx8\",\n \"register\": \"edx\",\n \"bit\": 8\n },\n {\n \"name\": \"apic\",\n \"register\": \"edx\",\n \"bit\": 9\n },\n {\n \"name\": \"sep\",\n \"register\": \"edx\",\n \"bit\": 11\n },\n {\n \"name\": \"mtrr\",\n \"register\": \"edx\",\n \"bit\": 12\n },\n {\n \"name\": \"pge\",\n \"register\": \"edx\",\n \"bit\": 13\n },\n {\n \"name\": \"mca\",\n \"register\": \"edx\",\n \"bit\": 14\n },\n {\n \"name\": \"cmov\",\n \"register\": \"edx\",\n \"bit\": 15\n },\n {\n \"name\": \"pat\",\n \"register\": \"edx\",\n \"bit\": 16\n },\n {\n \"name\": \"pse36\",\n \"register\": \"edx\",\n \"bit\": 17\n },\n {\n \"name\": \"pn\",\n \"register\": \"edx\",\n \"bit\": 18\n },\n {\n \"name\": \"clflush\",\n \"register\": \"edx\",\n \"bit\": 19\n },\n {\n \"name\": \"dts\",\n \"register\": \"edx\",\n \"bit\": 21\n },\n {\n \"name\": \"acpi\",\n \"register\": \"edx\",\n \"bit\": 22\n },\n {\n \"name\": \"mmx\",\n \"register\": \"edx\",\n \"bit\": 23\n },\n {\n \"name\": \"fxsr\",\n \"register\": \"edx\",\n \"bit\": 24\n },\n {\n \"name\": \"sse\",\n \"register\": \"edx\",\n \"bit\": 25\n },\n {\n \"name\": \"sse2\",\n \"register\": \"edx\",\n \"bit\": 26\n },\n {\n \"name\": \"ss\",\n \"register\": \"edx\",\n \"bit\": 27\n },\n {\n \"name\": \"ht\",\n \"register\": \"edx\",\n \"bit\": 28\n },\n {\n \"name\": \"tm\",\n \"register\": \"edx\",\n \"bit\": 29\n },\n {\n \"name\": \"ia64\",\n \"register\": \"edx\",\n \"bit\": 30\n },\n {\n \"name\": \"pbe\",\n \"register\": \"edx\",\n \"bit\": 31\n },\n {\n \"name\": \"pni\",\n \"register\": \"ecx\",\n \"bit\": 0\n },\n {\n \"name\": \"pclmulqdq\",\n \"register\": \"ecx\",\n \"bit\": 1\n },\n {\n \"name\": \"dtes64\",\n \"register\": \"ecx\",\n \"bit\": 2\n },\n {\n \"name\": \"monitor\",\n \"register\": \"ecx\",\n \"bit\": 3\n },\n {\n \"name\": \"ds_cpl\",\n \"register\": \"ecx\",\n \"bit\": 4\n },\n {\n \"name\": \"vmx\",\n \"register\": \"ecx\",\n \"bit\": 5\n },\n {\n \"name\": \"smx\",\n \"register\": \"ecx\",\n \"bit\": 6\n },\n {\n \"name\": \"est\",\n \"register\": \"ecx\",\n \"bit\": 7\n },\n {\n \"name\": \"tm2\",\n \"register\": \"ecx\",\n \"bit\": 8\n },\n {\n \"name\": \"ssse3\",\n \"register\": \"ecx\",\n \"bit\": 9\n },\n {\n \"name\": \"cid\",\n \"register\": \"ecx\",\n \"bit\": 10\n },\n {\n \"name\": \"fma\",\n \"register\": \"ecx\",\n \"bit\": 12\n },\n {\n \"name\": \"cx16\",\n \"register\": \"ecx\",\n \"bit\": 13\n },\n {\n \"name\": \"xtpr\",\n \"register\": \"ecx\",\n \"bit\": 14\n },\n {\n \"name\": \"pdcm\",\n \"register\": \"ecx\",\n \"bit\": 15\n },\n {\n \"name\": \"pcid\",\n \"register\": \"ecx\",\n \"bit\": 17\n },\n {\n \"name\": \"dca\",\n \"register\": \"ecx\",\n \"bit\": 18\n },\n {\n \"name\": \"sse4_1\",\n \"register\": \"ecx\",\n \"bit\": 19\n },\n {\n \"name\": \"sse4_2\",\n \"register\": \"ecx\",\n \"bit\": 20\n },\n {\n \"name\": \"x2apic\",\n \"register\": \"ecx\",\n \"bit\": 21\n },\n {\n \"name\": \"movbe\",\n \"register\": \"ecx\",\n \"bit\": 22\n },\n {\n \"name\": \"popcnt\",\n \"register\": \"ecx\",\n \"bit\": 23\n },\n {\n \"name\": \"tscdeadline\",\n \"register\": \"ecx\",\n \"bit\": 24\n },\n {\n \"name\": \"aes\",\n \"register\": \"ecx\",\n \"bit\": 25\n },\n {\n \"name\": \"xsave\",\n \"register\": \"ecx\",\n \"bit\": 26\n },\n {\n \"name\": \"osxsave\",\n \"register\": \"ecx\",\n \"bit\": 27\n },\n {\n \"name\": \"avx\",\n \"register\": \"ecx\",\n \"bit\": 28\n },\n {\n \"name\": \"f16c\",\n \"register\": \"ecx\",\n \"bit\": 29\n },\n {\n \"name\": \"rdrand\",\n \"register\": \"ecx\",\n \"bit\": 30\n },\n {\n \"name\": \"hypervisor\",\n \"register\": \"ecx\",\n \"bit\": 31\n }\n ]\n },\n {\n \"description\": \"https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features\",\n \"input\": {\n \"eax\": 7,\n \"ecx\": 0\n },\n \"bits\": [\n {\n \"name\": \"fsgsbase\",\n \"register\": \"ebx\",\n \"bit\": 0\n },\n {\n \"name\": \"sgx\",\n \"register\": \"ebx\",\n \"bit\": 2\n },\n {\n \"name\": \"bmi1\",\n \"register\": \"ebx\",\n \"bit\": 3\n },\n {\n \"name\": \"hle\",\n \"register\": \"ebx\",\n \"bit\": 4\n },\n {\n \"name\": \"avx2\",\n \"register\": \"ebx\",\n \"bit\": 5\n },\n {\n \"name\": \"fdp-excptn-only\",\n \"register\": \"ebx\",\n \"bit\": 6\n },\n {\n \"name\": \"smep\",\n \"register\": \"ebx\",\n \"bit\": 7\n },\n {\n \"name\": \"bmi2\",\n \"register\": \"ebx\",\n \"bit\": 8\n },\n {\n \"name\": \"erms\",\n \"register\": \"ebx\",\n \"bit\": 9\n },\n {\n \"name\": \"invpcid\",\n \"register\": \"ebx\",\n \"bit\": 10\n },\n {\n \"name\": \"rtm\",\n \"register\": \"ebx\",\n \"bit\": 11\n },\n {\n \"name\": \"pqm\",\n \"register\": \"ebx\",\n \"bit\": 12\n },\n {\n \"name\": \"mpx\",\n \"register\": \"ebx\",\n \"bit\": 14\n },\n {\n \"name\": \"pqe\",\n \"register\": \"ebx\",\n \"bit\": 15\n },\n {\n \"name\": \"avx512f\",\n \"register\": \"ebx\",\n \"bit\": 16\n },\n {\n \"name\": \"avx512dq\",\n \"register\": \"ebx\",\n \"bit\": 17\n },\n {\n \"name\": \"rdseed\",\n \"register\": \"ebx\",\n \"bit\": 18\n },\n {\n \"name\": \"adx\",\n \"register\": \"ebx\",\n \"bit\": 19\n },\n {\n \"name\": \"smap\",\n \"register\": \"ebx\",\n \"bit\": 20\n },\n {\n \"name\": \"avx512ifma\",\n \"register\": \"ebx\",\n \"bit\": 21\n },\n {\n \"name\": \"pcommit\",\n \"register\": \"ebx\",\n \"bit\": 22\n },\n {\n \"name\": \"clflushopt\",\n \"register\": \"ebx\",\n \"bit\": 23\n },\n {\n \"name\": \"clwb\",\n \"register\": \"ebx\",\n \"bit\": 24\n },\n {\n \"name\": \"intel_pt\",\n \"register\": \"ebx\",\n \"bit\": 25\n },\n {\n \"name\": \"avx512pf\",\n \"register\": \"ebx\",\n \"bit\": 26\n },\n {\n \"name\": \"avx512er\",\n \"register\": \"ebx\",\n \"bit\": 27\n },\n {\n \"name\": \"avx512cd\",\n \"register\": \"ebx\",\n \"bit\": 28\n },\n {\n \"name\": \"sha_ni\",\n \"register\": \"ebx\",\n \"bit\": 29\n },\n {\n \"name\": \"avx512bw\",\n \"register\": \"ebx\",\n \"bit\": 30\n },\n {\n \"name\": \"avx512vl\",\n \"register\": \"ebx\",\n \"bit\": 31\n },\n {\n \"name\": \"prefetchwt1\",\n \"register\": \"ecx\",\n \"bit\": 0\n },\n {\n \"name\": \"avx512vbmi\",\n \"register\": \"ecx\",\n \"bit\": 1\n },\n {\n \"name\": \"umip\",\n \"register\": \"ecx\",\n \"bit\": 2\n },\n {\n \"name\": \"pku\",\n \"register\": \"ecx\",\n \"bit\": 3\n },\n {\n \"name\": \"ospke\",\n \"register\": \"ecx\",\n \"bit\": 4\n },\n {\n \"name\": \"waitpkg\",\n \"register\": \"ecx\",\n \"bit\": 5\n },\n {\n \"name\": \"avx512_vbmi2\",\n \"register\": \"ecx\",\n \"bit\": 6\n },\n {\n \"name\": \"cet_ss\",\n \"register\": \"ecx\",\n \"bit\": 7\n },\n {\n \"name\": \"gfni\",\n \"register\": \"ecx\",\n \"bit\": 8\n },\n {\n \"name\": \"vaes\",\n \"register\": \"ecx\",\n \"bit\": 9\n },\n {\n \"name\": \"vpclmulqdq\",\n \"register\": \"ecx\",\n \"bit\": 10\n },\n {\n \"name\": \"avx512_vnni\",\n \"register\": \"ecx\",\n \"bit\": 11\n },\n {\n \"name\": \"avx512_bitalg\",\n \"register\": \"ecx\",\n \"bit\": 12\n },\n {\n \"name\": \"tme\",\n \"register\": \"ecx\",\n \"bit\": 13\n },\n {\n \"name\": \"avx512_vpopcntdq\",\n \"register\": \"ecx\",\n \"bit\": 14\n },\n {\n \"name\": \"rdpid\",\n \"register\": \"ecx\",\n \"bit\": 22\n },\n {\n \"name\": \"cldemote\",\n \"register\": \"ecx\",\n \"bit\": 25\n },\n {\n \"name\": \"movdiri\",\n \"register\": \"ecx\",\n \"bit\": 27\n },\n {\n \"name\": \"movdir64b\",\n \"register\": \"ecx\",\n \"bit\": 28\n },\n {\n \"name\": \"enqcmd\",\n \"register\": \"ecx\",\n \"bit\": 29\n },\n {\n \"name\": \"sgx_lc\",\n \"register\": \"ecx\",\n \"bit\": 30\n },\n {\n \"name\": \"pks\",\n \"register\": \"ecx\",\n \"bit\": 31\n },\n {\n \"name\": \"fsrm\",\n \"register\": \"edx\",\n \"bit\": 4\n },\n {\n \"name\": \"avx512_vp2intersect\",\n \"register\": \"edx\",\n \"bit\": 8\n },\n {\n \"name\": \"md_clear\",\n \"register\": \"edx\",\n \"bit\": 10\n },\n {\n \"name\": \"serialize\",\n \"register\": \"edx\",\n \"bit\": 14\n },\n {\n \"name\": \"tsxldtrk\",\n \"register\": \"edx\",\n \"bit\": 16\n },\n {\n \"name\": \"amx_bf16\",\n \"register\": \"edx\",\n \"bit\": 22\n },\n {\n \"name\": \"avx512_fp16\",\n \"register\": \"edx\",\n \"bit\": 23\n },\n {\n \"name\": \"amx_tile\",\n \"register\": \"edx\",\n \"bit\": 24\n },\n {\n \"name\": \"amx_int8\",\n \"register\": \"edx\",\n \"bit\": 25\n },\n {\n \"name\": \"ssbd\",\n \"register\": \"edx\",\n \"bit\": 31\n }\n ]\n },\n {\n \"description\": \"https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features\",\n \"input\": {\n \"eax\": 7,\n \"ecx\": 1\n },\n \"bits\": [\n {\n \"name\": \"sha512\",\n \"register\": \"eax\",\n \"bit\": 0\n },\n {\n \"name\": \"sm3\",\n \"register\": \"eax\",\n \"bit\": 1\n },\n {\n \"name\": \"sm4\",\n \"register\": \"eax\",\n \"bit\": 2\n },\n {\n \"name\": \"rao_int\",\n \"register\": \"eax\",\n \"bit\": 3\n },\n {\n \"name\": \"avx_vnni\",\n \"register\": \"eax\",\n \"bit\": 4\n },\n {\n \"name\": \"avx512_bf16\",\n \"register\": \"eax\",\n \"bit\": 5\n },\n {\n \"name\": \"cmpccxadd\",\n \"register\": \"eax\",\n \"bit\": 7\n },\n {\n \"name\": \"arch_perfmon_ext\",\n \"register\": \"eax\",\n \"bit\": 8\n },\n {\n \"name\": \"fzrm\",\n \"register\": \"eax\",\n \"bit\": 10\n },\n {\n \"name\": \"fsrs\",\n \"register\": \"eax\",\n \"bit\": 11\n },\n {\n \"name\": \"fsrc\",\n \"register\": \"eax\",\n \"bit\": 12\n },\n {\n \"name\": \"lkgs\",\n \"register\": \"eax\",\n \"bit\": 18\n },\n {\n \"name\": \"amx_fp16\",\n \"register\": \"eax\",\n \"bit\": 21\n },\n {\n \"name\": \"avx_ifma\",\n \"register\": \"eax\",\n \"bit\": 23\n },\n {\n \"name\": \"lam\",\n \"register\": \"eax\",\n \"bit\": 26\n }\n ]\n },\n {\n \"description\": \"https://en.wikipedia.org/wiki/CPUID#EAX=0Dh:_XSAVE_features_and_state-components\",\n \"input\": {\n \"eax\": 13,\n \"ecx\": 1\n },\n \"bits\": [\n {\n \"name\": \"xsaveopt\",\n \"register\": \"eax\",\n \"bit\": 0\n },\n {\n \"name\": \"xsavec\",\n \"register\": \"eax\",\n \"bit\": 1\n },\n {\n \"name\": \"xgetbv1\",\n \"register\": \"eax\",\n \"bit\": 2\n },\n {\n \"name\": \"xsaves\",\n \"register\": \"eax\",\n \"bit\": 3\n },\n {\n \"name\": \"xfd\",\n \"register\": \"eax\",\n \"bit\": 4\n }\n ]\n }\n ],\n \"extension-flags\": [\n {\n \"description\": \"https://en.wikipedia.org/wiki/CPUID#EAX=0Dh:_XSAVE_features_and_state-components\",\n \"input\": {\n \"eax\": 2147483649,\n \"ecx\": 0\n },\n \"bits\": [\n {\n \"name\": \"fpu\",\n \"register\": \"edx\",\n \"bit\": 0\n },\n {\n \"name\": \"vme\",\n \"register\": \"edx\",\n \"bit\": 1\n },\n {\n \"name\": \"de\",\n \"register\": \"edx\",\n \"bit\": 2\n },\n {\n \"name\": \"pse\",\n \"register\": \"edx\",\n \"bit\": 3\n },\n {\n \"name\": \"tsc\",\n \"register\": \"edx\",\n \"bit\": 4\n },\n {\n \"name\": \"msr\",\n \"register\": \"edx\",\n \"bit\": 5\n },\n {\n \"name\": \"pae\",\n \"register\": \"edx\",\n \"bit\": 6\n },\n {\n \"name\": \"mce\",\n \"register\": \"edx\",\n \"bit\": 7\n },\n {\n \"name\": \"cx8\",\n \"register\": \"edx\",\n \"bit\": 8\n },\n {\n \"name\": \"apic\",\n \"register\": \"edx\",\n \"bit\": 9\n },\n {\n \"name\": \"syscall\",\n \"register\": \"edx\",\n \"bit\": 10\n },\n {\n \"name\": \"syscall\",\n \"register\": \"edx\",\n \"bit\": 11\n },\n {\n \"name\": \"mtrr\",\n \"register\": \"edx\",\n \"bit\": 12\n },\n {\n \"name\": \"pge\",\n \"register\": \"edx\",\n \"bit\": 13\n },\n {\n \"name\": \"mca\",\n \"register\": \"edx\",\n \"bit\": 14\n },\n {\n \"name\": \"cmov\",\n \"register\": \"edx\",\n \"bit\": 15\n },\n {\n \"name\": \"pat\",\n \"register\": \"edx\",\n \"bit\": 16\n },\n {\n \"name\": \"pse36\",\n \"register\": \"edx\",\n \"bit\": 17\n },\n {\n \"name\": \"mp\",\n \"register\": \"edx\",\n \"bit\": 19\n },\n {\n \"name\": \"nx\",\n \"register\": \"edx\",\n \"bit\": 20\n },\n {\n \"name\": \"mmxext\",\n \"register\": \"edx\",\n \"bit\": 22\n },\n {\n \"name\": \"mmx\",\n \"register\": \"edx\",\n \"bit\": 23\n },\n {\n \"name\": \"fxsr\",\n \"register\": \"edx\",\n \"bit\": 24\n },\n {\n \"name\": \"fxsr_opt\",\n \"register\": \"edx\",\n \"bit\": 25\n },\n {\n \"name\": \"pdpe1gp\",\n \"register\": \"edx\",\n \"bit\": 26\n },\n {\n \"name\": \"rdtscp\",\n \"register\": \"edx\",\n \"bit\": 27\n },\n {\n \"name\": \"lm\",\n \"register\": \"edx\",\n \"bit\": 29\n },\n {\n \"name\": \"3dnowext\",\n \"register\": \"edx\",\n \"bit\": 30\n },\n {\n \"name\": \"3dnow\",\n \"register\": \"edx\",\n \"bit\": 31\n },\n {\n \"name\": \"lahf_lm\",\n \"register\": \"ecx\",\n \"bit\": 0\n },\n {\n \"name\": \"cmp_legacy\",\n \"register\": \"ecx\",\n \"bit\": 1\n },\n {\n \"name\": \"svm\",\n \"register\": \"ecx\",\n \"bit\": 2\n },\n {\n \"name\": \"extapic\",\n \"register\": \"ecx\",\n \"bit\": 3\n },\n {\n \"name\": \"cr8_legacy\",\n \"register\": \"ecx\",\n \"bit\": 4\n },\n {\n \"name\": \"abm\",\n \"register\": \"ecx\",\n \"bit\": 5\n },\n {\n \"name\": \"sse4a\",\n \"register\": \"ecx\",\n \"bit\": 6\n },\n {\n \"name\": \"misalignsse\",\n \"register\": \"ecx\",\n \"bit\": 7\n },\n {\n \"name\": \"3dnowprefetch\",\n \"register\": \"ecx\",\n \"bit\": 8\n },\n {\n \"name\": \"osvw\",\n \"register\": \"ecx\",\n \"bit\": 9\n },\n {\n \"name\": \"ibs\",\n \"register\": \"ecx\",\n \"bit\": 10\n },\n {\n \"name\": \"xop\",\n \"register\": \"ecx\",\n \"bit\": 11\n },\n {\n \"name\": \"skinit\",\n \"register\": \"ecx\",\n \"bit\": 12\n },\n {\n \"name\": \"wdt\",\n \"register\": \"ecx\",\n \"bit\": 13\n },\n {\n \"name\": \"lwp\",\n \"register\": \"ecx\",\n \"bit\": 15\n },\n {\n \"name\": \"fma4\",\n \"register\": \"ecx\",\n \"bit\": 16\n },\n {\n \"name\": \"tce\",\n \"register\": \"ecx\",\n \"bit\": 17\n },\n {\n \"name\": \"nodeid_msr\",\n \"register\": \"ecx\",\n \"bit\": 19\n },\n {\n \"name\": \"tbm\",\n \"register\": \"ecx\",\n \"bit\": 21\n },\n {\n \"name\": \"topoext\",\n \"register\": \"ecx\",\n \"bit\": 22\n },\n {\n \"name\": \"perfctr_core\",\n \"register\": \"ecx\",\n \"bit\": 23\n },\n {\n \"name\": \"perfctr_nb\",\n \"register\": \"ecx\",\n \"bit\": 24\n },\n {\n \"name\": \"dbx\",\n \"register\": \"ecx\",\n \"bit\": 26\n },\n {\n \"name\": \"perftsc\",\n \"register\": \"ecx\",\n \"bit\": 27\n },\n {\n \"name\": \"pci_l2i\",\n \"register\": \"ecx\",\n \"bit\": 28\n },\n {\n \"name\": \"mwaitx\",\n \"register\": \"ecx\",\n \"bit\": 29\n }\n ]\n }\n ]\n}\n" }, { "path": "lib/spack/spack/vendor/archspec/json/cpu/cpuid_schema.json", "content": "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Schema for microarchitecture definitions and feature aliases\",\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"vendor\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"description\": {\n \"type\": \"string\"\n },\n \"input\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"eax\": {\n \"type\": \"integer\"\n },\n \"ecx\": {\n \"type\": \"integer\"\n }\n }\n }\n }\n },\n \"highest_extension_support\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"description\": {\n \"type\": \"string\"\n },\n \"input\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"eax\": {\n \"type\": \"integer\"\n },\n \"ecx\": {\n \"type\": \"integer\"\n }\n }\n }\n }\n },\n \"flags\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"description\": {\n \"type\": \"string\"\n },\n \"input\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"eax\": {\n \"type\": \"integer\"\n },\n \"ecx\": {\n \"type\": \"integer\"\n }\n }\n },\n \"bits\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"name\": {\n \"type\": \"string\"\n },\n \"register\": {\n \"type\": \"string\"\n },\n \"bit\": {\n \"type\": \"integer\"\n }\n }\n }\n }\n }\n }\n },\n \"extension-flags\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"description\": {\n \"type\": \"string\"\n },\n \"input\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"eax\": {\n \"type\": \"integer\"\n },\n \"ecx\": {\n \"type\": \"integer\"\n }\n }\n },\n \"bits\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"name\": {\n \"type\": \"string\"\n },\n \"register\": {\n \"type\": \"string\"\n },\n \"bit\": {\n \"type\": \"integer\"\n }\n }\n }\n }\n }\n }\n }\n }\n}\n" }, { "path": "lib/spack/spack/vendor/archspec/json/cpu/microarchitectures.json", "content": "{\n \"microarchitectures\": {\n \"x86\": {\n \"from\": [],\n \"vendor\": \"generic\",\n \"features\": []\n },\n \"i686\": {\n \"from\": [\n \"x86\"\n ],\n \"vendor\": \"GenuineIntel\",\n \"features\": []\n },\n \"pentium2\": {\n \"from\": [\n \"i686\"\n ],\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\"\n ]\n },\n \"pentium3\": {\n \"from\": [\n \"pentium2\"\n ],\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\"\n ]\n },\n \"pentium4\": {\n \"from\": [\n \"pentium3\"\n ],\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\"\n ]\n },\n \"prescott\": {\n \"from\": [\n \"pentium4\"\n ],\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"sse3\"\n ]\n },\n \"x86_64\": {\n \"from\": [],\n \"vendor\": \"generic\",\n \"features\": [],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"4.2.0:\",\n \"name\": \"x86-64\",\n \"flags\": \"-march={name} -mtune=generic\"\n },\n {\n \"versions\": \":4.1.2\",\n \"name\": \"x86-64\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"apple-clang\": [\n {\n \"versions\": \":\",\n \"name\": \"x86-64\",\n \"flags\": \"-march={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \":\",\n \"name\": \"x86-64\",\n \"flags\": \"-march={name} -mtune=generic\"\n }\n ],\n \"aocc\": [\n {\n \"versions\": \"2.2:\",\n \"name\": \"x86-64\",\n \"flags\": \"-march={name} -mtune=generic\"\n }\n ],\n \"intel\": [\n {\n \"versions\": \":\",\n \"name\": \"pentium4\",\n \"flags\": \"-march={name} -mtune=generic\"\n }\n ],\n \"oneapi\": [\n {\n \"versions\": \":\",\n \"name\": \"x86-64\",\n \"flags\": \"-march={name} -mtune=generic\"\n }\n ],\n \"dpcpp\": [\n {\n \"versions\": \":\",\n \"name\": \"x86-64\",\n \"flags\": \"-march={name} -mtune=generic\"\n }\n ],\n \"nvhpc\": []\n }\n },\n \"x86_64_v2\": {\n \"from\": [\n \"x86_64\"\n ],\n \"vendor\": \"generic\",\n \"features\": [\n \"cx16\",\n \"lahf_lm\",\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"popcnt\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"11.1:\",\n \"name\": \"x86-64-v2\",\n \"flags\": \"-march={name} -mtune=generic\"\n },\n {\n \"versions\": \"4.6:11.0\",\n \"name\": \"x86-64\",\n \"flags\": \"-march={name} -mtune=generic -mcx16 -msahf -mpopcnt -msse3 -msse4.1 -msse4.2 -mssse3\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"12.0:\",\n \"name\": \"x86-64-v2\",\n \"flags\": \"-march={name} -mtune=generic\"\n },\n {\n \"versions\": \"3.9:11.1\",\n \"name\": \"x86-64\",\n \"flags\": \"-march={name} -mtune=generic -mcx16 -msahf -mpopcnt -msse3 -msse4.1 -msse4.2 -mssse3\"\n }\n ],\n \"aocc\": [\n {\n \"versions\": \"2.2:\",\n \"name\": \"x86-64-v2\",\n \"flags\": \"-march={name} -mtune=generic\"\n }\n ],\n \"intel\": [\n {\n \"versions\": \"16.0:\",\n \"name\": \"corei7\",\n \"flags\": \"-march={name} -mtune=generic -mpopcnt\"\n }\n ],\n \"oneapi\": [\n {\n \"versions\": \"2021.2.0:\",\n \"name\": \"x86-64-v2\",\n \"flags\": \"-march={name} -mtune=generic\"\n }\n ],\n \"dpcpp\": [\n {\n \"versions\": \"2021.2.0:\",\n \"name\": \"x86-64-v2\",\n \"flags\": \"-march={name} -mtune=generic\"\n }\n ],\n \"nvhpc\": []\n }\n },\n \"x86_64_v3\": {\n \"from\": [\n \"x86_64_v2\"\n ],\n \"vendor\": \"generic\",\n \"features\": [\n \"cx16\",\n \"lahf_lm\",\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"popcnt\",\n \"avx\",\n \"avx2\",\n \"bmi1\",\n \"bmi2\",\n \"f16c\",\n \"fma\",\n \"abm\",\n \"movbe\",\n \"xsave\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"11.1:\",\n \"name\": \"x86-64-v3\",\n \"flags\": \"-march={name} -mtune=generic\"\n },\n {\n \"versions\": \"4.8:11.0\",\n \"name\": \"x86-64\",\n \"flags\": \"-march={name} -mtune=generic -mcx16 -msahf -mpopcnt -msse3 -msse4.1 -msse4.2 -mssse3 -mavx -mavx2 -mbmi -mbmi2 -mf16c -mfma -mlzcnt -mmovbe -mxsave\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"12.0:\",\n \"name\": \"x86-64-v3\",\n \"flags\": \"-march={name} -mtune=generic\"\n },\n {\n \"versions\": \"3.9:11.1\",\n \"name\": \"x86-64\",\n \"flags\": \"-march={name} -mtune=generic -mcx16 -msahf -mpopcnt -msse3 -msse4.1 -msse4.2 -mssse3 -mavx -mavx2 -mbmi -mbmi2 -mf16c -mfma -mlzcnt -mmovbe -mxsave\"\n }\n ],\n \"aocc\": [\n {\n \"versions\": \"2.2:\",\n \"name\": \"x86-64-v3\",\n \"flags\": \"-march={name} -mtune=generic\"\n }\n ],\n \"apple-clang\": [\n {\n \"versions\": \"8.0:\",\n \"name\": \"x86-64\",\n \"flags\": \"-march={name} -mtune=generic -mcx16 -msahf -mpopcnt -msse3 -msse4.1 -msse4.2 -mssse3 -mavx -mavx2 -mbmi -mbmi2 -mf16c -mfma -mlzcnt -mmovbe -mxsave\"\n }\n ],\n \"intel\": [\n {\n \"versions\": \"16.0:\",\n \"name\": \"core-avx2\",\n \"flags\": \"-march={name} -mtune={name} -fma -mf16c\"\n }\n ],\n \"oneapi\": [\n {\n \"versions\": \"2021.2.0:\",\n \"name\": \"x86-64-v3\",\n \"flags\": \"-march={name} -mtune=generic\"\n }\n ],\n \"dpcpp\": [\n {\n \"versions\": \"2021.2.0:\",\n \"name\": \"x86-64-v3\",\n \"flags\": \"-march={name} -mtune=generic\"\n }\n ],\n \"nvhpc\": [\n {\n \"versions\": \":\",\n \"name\": \"px\",\n \"flags\": \"-tp {name} -mpopcnt -msse3 -msse4.1 -msse4.2 -mssse3 -mavx -mavx2 -mbmi -mbmi2 -mf16c -mfma -mlzcnt -mxsave\"\n }\n ]\n }\n },\n \"x86_64_v4\": {\n \"from\": [\n \"x86_64_v3\"\n ],\n \"vendor\": \"generic\",\n \"features\": [\n \"cx16\",\n \"lahf_lm\",\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"popcnt\",\n \"avx\",\n \"avx2\",\n \"bmi1\",\n \"bmi2\",\n \"f16c\",\n \"fma\",\n \"abm\",\n \"movbe\",\n \"xsave\",\n \"avx512f\",\n \"avx512bw\",\n \"avx512cd\",\n \"avx512dq\",\n \"avx512vl\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"11.1:\",\n \"name\": \"x86-64-v4\",\n \"flags\": \"-march={name} -mtune=generic\"\n },\n {\n \"versions\": \"6.0:11.0\",\n \"name\": \"x86-64\",\n \"flags\": \"-march={name} -mtune=generic -mcx16 -msahf -mpopcnt -msse3 -msse4.1 -msse4.2 -mssse3 -mavx -mavx2 -mbmi -mbmi2 -mf16c -mfma -mlzcnt -mmovbe -mxsave -mavx512f -mavx512bw -mavx512cd -mavx512dq -mavx512vl\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"12.0:\",\n \"name\": \"x86-64-v4\",\n \"flags\": \"-march={name} -mtune=generic\"\n },\n {\n \"versions\": \"3.9:11.1\",\n \"name\": \"x86-64\",\n \"flags\": \"-march={name} -mtune=generic -mcx16 -msahf -mpopcnt -msse3 -msse4.1 -msse4.2 -mssse3 -mavx -mavx2 -mbmi -mbmi2 -mf16c -mfma -mlzcnt -mmovbe -mxsave -mavx512f -mavx512bw -mavx512cd -mavx512dq -mavx512vl\"\n }\n ],\n \"aocc\": [\n {\n \"versions\": \"4:\",\n \"name\": \"x86-64-v4\",\n \"flags\": \"-march={name} -mtune=generic\"\n }\n ],\n \"apple-clang\": [\n {\n \"versions\": \"8.0:\",\n \"name\": \"x86-64\",\n \"flags\": \"-march={name} -mtune=generic -mcx16 -msahf -mpopcnt -msse3 -msse4.1 -msse4.2 -mssse3 -mavx -mavx2 -mbmi -mbmi2 -mf16c -mfma -mlzcnt -mmovbe -mxsave -mavx512f -mavx512bw -mavx512cd -mavx512dq -mavx512vl\"\n }\n ],\n \"intel\": [\n {\n \"versions\": \"16.0:\",\n \"name\": \"skylake-avx512\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"oneapi\": [\n {\n \"versions\": \"2021.2.0:\",\n \"name\": \"x86-64-v4\",\n \"flags\": \"-march={name} -mtune=generic\"\n }\n ],\n \"dpcpp\": [\n {\n \"versions\": \"2021.2.0:\",\n \"name\": \"x86-64-v4\",\n \"flags\": \"-march={name} -mtune=generic\"\n }\n ],\n \"nvhpc\": [\n {\n \"versions\": \":\",\n \"name\": \"px\",\n \"flags\": \"-tp {name} -mpopcnt -msse3 -msse4.1 -msse4.2 -mssse3 -mavx -mavx2 -mbmi -mbmi2 -mf16c -mfma -mlzcnt -mxsave -mavx512f -mavx512bw -mavx512cd -mavx512dq -mavx512vl\"\n }\n ]\n }\n },\n \"nocona\": {\n \"from\": [\n \"x86_64\"\n ],\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"sse3\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"4.0.4:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"3.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"aocc\": [\n {\n \"versions\": \"2.2:\",\n \"flags\": \"-march={name} -mtune=generic\"\n }\n ],\n \"apple-clang\": [\n {\n \"versions\": \"8.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"intel\": [\n {\n \"versions\": \"16.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"oneapi\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"dpcpp\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"nvhpc\": []\n }\n },\n \"core2\": {\n \"from\": [\n \"nocona\"\n ],\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"ssse3\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"4.3.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"3.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"aocc\": [\n {\n \"versions\": \"2.2:\",\n \"flags\": \"-march={name} -mtune=generic\"\n }\n ],\n \"apple-clang\": [\n {\n \"versions\": \"8.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"intel\": [\n {\n \"versions\": \"16.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"oneapi\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"dpcpp\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"nvhpc\": []\n }\n },\n \"nehalem\": {\n \"from\": [\n \"core2\",\n \"x86_64_v2\"\n ],\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"popcnt\",\n \"lahf_lm\",\n \"cx16\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"4.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n {\n \"versions\": \"4.6:4.8.5\",\n \"name\": \"corei7\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"3.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"aocc\": [\n {\n \"versions\": \"2.2:\",\n \"flags\": \"-march={name} -mtune=generic\"\n }\n ],\n \"apple-clang\": [\n {\n \"versions\": \"8.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"intel\": [\n {\n \"versions\": \"16.0:\",\n \"name\": \"corei7\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"oneapi\": [\n {\n \"versions\": \":\",\n \"name\": \"corei7\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"dpcpp\": [\n {\n \"versions\": \":\",\n \"name\": \"corei7\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"nvhpc\": []\n }\n },\n \"westmere\": {\n \"from\": [\n \"nehalem\"\n ],\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"popcnt\",\n \"lahf_lm\",\n \"cx16\",\n \"aes\",\n \"pclmulqdq\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"4.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"3.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"aocc\": [\n {\n \"versions\": \"2.2:\",\n \"flags\": \"-march={name} -mtune=generic\"\n }\n ],\n \"apple-clang\": [\n {\n \"versions\": \"8.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"intel\": [\n {\n \"versions\": \"16.0:\",\n \"name\": \"corei7\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"oneapi\": [\n {\n \"versions\": \":\",\n \"name\": \"corei7\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"dpcpp\": [\n {\n \"versions\": \":\",\n \"name\": \"corei7\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"nvhpc\": []\n }\n },\n \"sandybridge\": {\n \"from\": [\n \"westmere\"\n ],\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"popcnt\",\n \"lahf_lm\",\n \"cx16\",\n \"aes\",\n \"pclmulqdq\",\n \"avx\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"4.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n {\n \"versions\": \"4.6:4.8.5\",\n \"name\": \"corei7-avx\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"3.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"aocc\": [\n {\n \"versions\": \"2.2:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"apple-clang\": [\n {\n \"versions\": \"8.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"intel\": [\n {\n \"versions\": \"16.0:17.9.0\",\n \"name\": \"corei7-avx\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n {\n \"versions\": \"18.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"oneapi\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"dpcpp\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"nvhpc\": [\n {\n \"versions\": \":\",\n \"flags\": \"-tp {name}\"\n }\n ]\n }\n },\n \"ivybridge\": {\n \"from\": [\n \"sandybridge\"\n ],\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"popcnt\",\n \"lahf_lm\",\n \"cx16\",\n \"aes\",\n \"pclmulqdq\",\n \"avx\",\n \"rdrand\",\n \"f16c\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"4.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n {\n \"versions\": \"4.6:4.8.5\",\n \"name\": \"core-avx-i\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"3.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"aocc\": [\n {\n \"versions\": \"2.2:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"apple-clang\": [\n {\n \"versions\": \"8.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"intel\": [\n {\n \"versions\": \"16.0:17.9.0\",\n \"name\": \"core-avx-i\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n {\n \"versions\": \"18.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"oneapi\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"dpcpp\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"nvhpc\": [\n {\n \"versions\": \":\",\n \"flags\": \"-tp {name}\"\n }\n ]\n }\n },\n \"haswell\": {\n \"from\": [\n \"ivybridge\",\n \"x86_64_v3\"\n ],\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"popcnt\",\n \"abm\",\n \"lahf_lm\",\n \"xsave\",\n \"cx16\",\n \"aes\",\n \"pclmulqdq\",\n \"avx\",\n \"rdrand\",\n \"f16c\",\n \"movbe\",\n \"fma\",\n \"avx2\",\n \"bmi1\",\n \"bmi2\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"4.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n {\n \"versions\": \"4.8:4.8.5\",\n \"name\": \"core-avx2\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"3.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"aocc\": [\n {\n \"versions\": \"2.2:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"apple-clang\": [\n {\n \"versions\": \"8.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"intel\": [\n {\n \"versions\": \"16.0:17.9.0\",\n \"name\": \"core-avx2\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n {\n \"versions\": \"18.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"oneapi\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"dpcpp\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"nvhpc\": [\n {\n \"versions\": \":\",\n \"flags\": \"-tp {name}\"\n }\n ]\n }\n },\n \"broadwell\": {\n \"from\": [\n \"haswell\"\n ],\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"abm\",\n \"popcnt\",\n \"xsave\",\n \"lahf_lm\",\n \"cx16\",\n \"aes\",\n \"pclmulqdq\",\n \"avx\",\n \"rdrand\",\n \"f16c\",\n \"movbe\",\n \"fma\",\n \"avx2\",\n \"bmi1\",\n \"bmi2\",\n \"rdseed\",\n \"adx\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"4.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"3.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"aocc\": [\n {\n \"versions\": \"2.2:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"apple-clang\": [\n {\n \"versions\": \"8.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"intel\": [\n {\n \"versions\": \"18.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"oneapi\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"dpcpp\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"nvhpc\": [\n {\n \"versions\": \":\",\n \"name\": \"haswell\",\n \"flags\": \"-tp {name}\"\n }\n ]\n }\n },\n \"skylake\": {\n \"from\": [\n \"broadwell\"\n ],\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"popcnt\",\n \"abm\",\n \"lahf_lm\",\n \"xsave\",\n \"cx16\",\n \"aes\",\n \"pclmulqdq\",\n \"avx\",\n \"rdrand\",\n \"f16c\",\n \"movbe\",\n \"fma\",\n \"avx2\",\n \"bmi1\",\n \"bmi2\",\n \"rdseed\",\n \"adx\",\n \"clflushopt\",\n \"xsavec\",\n \"xsaveopt\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"6.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"3.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"aocc\": [\n {\n \"versions\": \"2.2:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"apple-clang\": [\n {\n \"versions\": \"8.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"intel\": [\n {\n \"versions\": \"18.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"oneapi\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"dpcpp\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"nvhpc\": [\n {\n \"versions\": \":\",\n \"name\": \"haswell\",\n \"flags\": \"-tp {name}\"\n }\n ]\n }\n },\n \"mic_knl\": {\n \"from\": [\n \"broadwell\"\n ],\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"popcnt\",\n \"abm\",\n \"lahf_lm\",\n \"cx16\",\n \"aes\",\n \"xsave\",\n \"pclmulqdq\",\n \"avx\",\n \"rdrand\",\n \"f16c\",\n \"movbe\",\n \"avx2\",\n \"fma\",\n \"avx2\",\n \"bmi1\",\n \"bmi2\",\n \"rdseed\",\n \"adx\",\n \"avx512f\",\n \"avx512pf\",\n \"avx512er\",\n \"avx512cd\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"5.1:\",\n \"name\": \"knl\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"3.9:\",\n \"name\": \"knl\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"aocc\": [\n {\n \"versions\": \"2.2:\",\n \"name\": \"knl\",\n \"flags\": \"-march={name} -mtune=generic\"\n }\n ],\n \"apple-clang\": [\n {\n \"versions\": \"8.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"intel\": [\n {\n \"versions\": \"18.0:2021.2\",\n \"name\": \"knl\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"oneapi\": [\n {\n \"versions\": \":2021.2\",\n \"name\": \"knl\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"dpcpp\": [\n {\n \"versions\": \":2021.2\",\n \"name\": \"knl\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ]\n }\n },\n \"skylake_avx512\": {\n \"from\": [\n \"skylake\",\n \"x86_64_v4\"\n ],\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"popcnt\",\n \"abm\",\n \"lahf_lm\",\n \"cx16\",\n \"aes\",\n \"pclmulqdq\",\n \"avx\",\n \"rdrand\",\n \"f16c\",\n \"movbe\",\n \"fma\",\n \"avx2\",\n \"bmi1\",\n \"bmi2\",\n \"rdseed\",\n \"adx\",\n \"clflushopt\",\n \"xsave\",\n \"xsavec\",\n \"xsaveopt\",\n \"avx512f\",\n \"clwb\",\n \"avx512vl\",\n \"avx512bw\",\n \"avx512dq\",\n \"avx512cd\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"name\": \"skylake-avx512\",\n \"versions\": \"6.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"3.9:\",\n \"name\": \"skylake-avx512\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"aocc\": [\n {\n \"versions\": \"2.2:\",\n \"name\": \"skylake-avx512\",\n \"flags\": \"-march={name} -mtune=generic\"\n }\n ],\n \"apple-clang\": [\n {\n \"versions\": \"8.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"intel\": [\n {\n \"versions\": \"18.0:\",\n \"name\": \"skylake-avx512\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"oneapi\": [\n {\n \"versions\": \":\",\n \"name\": \"skylake-avx512\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"dpcpp\": [\n {\n \"versions\": \":\",\n \"name\": \"skylake-avx512\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"nvhpc\": [\n {\n \"versions\": \":\",\n \"name\": \"skylake\",\n \"flags\": \"-tp {name}\"\n }\n ]\n }\n },\n \"cannonlake\": {\n \"from\": [\n \"skylake\"\n ],\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"popcnt\",\n \"abm\",\n \"lahf_lm\",\n \"cx16\",\n \"aes\",\n \"pclmulqdq\",\n \"avx\",\n \"rdrand\",\n \"f16c\",\n \"movbe\",\n \"fma\",\n \"avx2\",\n \"bmi1\",\n \"bmi2\",\n \"rdseed\",\n \"adx\",\n \"clflushopt\",\n \"xsave\",\n \"xsavec\",\n \"xsaveopt\",\n \"avx512f\",\n \"avx512vl\",\n \"avx512bw\",\n \"avx512dq\",\n \"avx512cd\",\n \"avx512vbmi\",\n \"avx512ifma\",\n \"sha_ni\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"8.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"3.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"aocc\": [\n {\n \"versions\": \"2.2:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"apple-clang\": [\n {\n \"versions\": \"8.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"intel\": [\n {\n \"versions\": \"18.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"oneapi\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"dpcpp\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"nvhpc\": [\n {\n \"versions\": \":\",\n \"name\": \"skylake\",\n \"flags\": \"-tp {name}\"\n }\n ]\n }\n },\n \"cascadelake\": {\n \"from\": [\n \"skylake_avx512\"\n ],\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"popcnt\",\n \"abm\",\n \"lahf_lm\",\n \"cx16\",\n \"aes\",\n \"pclmulqdq\",\n \"avx\",\n \"rdrand\",\n \"f16c\",\n \"movbe\",\n \"fma\",\n \"avx2\",\n \"bmi1\",\n \"bmi2\",\n \"rdseed\",\n \"adx\",\n \"clflushopt\",\n \"xsave\",\n \"xsavec\",\n \"xsaveopt\",\n \"avx512f\",\n \"clwb\",\n \"avx512vl\",\n \"avx512bw\",\n \"avx512dq\",\n \"avx512cd\",\n \"avx512_vnni\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"9.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"8.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"aocc\": [\n {\n \"versions\": \"2.2:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"apple-clang\": [\n {\n \"versions\": \"11.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"intel\": [\n {\n \"versions\": \"19.0.1:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"oneapi\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"dpcpp\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"nvhpc\": [\n {\n \"versions\": \":\",\n \"name\": \"skylake\",\n \"flags\": \"-tp {name}\"\n }\n ]\n }\n },\n \"icelake\": {\n \"from\": [\n \"cascadelake\",\n \"cannonlake\"\n ],\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"popcnt\",\n \"abm\",\n \"lahf_lm\",\n \"cx16\",\n \"aes\",\n \"sha_ni\",\n \"pclmulqdq\",\n \"avx\",\n \"rdrand\",\n \"f16c\",\n \"movbe\",\n \"fma\",\n \"avx2\",\n \"bmi1\",\n \"bmi2\",\n \"rdseed\",\n \"adx\",\n \"clflushopt\",\n \"xsave\",\n \"xsavec\",\n \"xsaveopt\",\n \"avx512f\",\n \"avx512vl\",\n \"avx512bw\",\n \"avx512dq\",\n \"avx512cd\",\n \"avx512vbmi\",\n \"avx512ifma\",\n \"sha_ni\",\n \"clwb\",\n \"rdpid\",\n \"gfni\",\n \"avx512_vbmi2\",\n \"avx512_vpopcntdq\",\n \"avx512_bitalg\",\n \"avx512_vnni\",\n \"vpclmulqdq\",\n \"vaes\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"name\": \"icelake-client\",\n \"versions\": \"8.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"7.0:\",\n \"name\": \"icelake-client\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n {\n \"versions\": \"6.0:6.9\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"aocc\": [\n {\n \"versions\": \"2.2:\",\n \"name\": \"icelake-client\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"apple-clang\": [\n {\n \"versions\": \"10.0.1:\",\n \"name\": \"icelake-client\",\n \"flags\": \"-march={name} -mtune={name}\"\n },\n {\n \"versions\": \"10.0.0:10.0.99\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"intel\": [\n {\n \"versions\": \"18.0:\",\n \"name\": \"icelake-client\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"oneapi\": [\n {\n \"versions\": \":\",\n \"name\": \"icelake-client\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"dpcpp\": [\n {\n \"versions\": \":\",\n \"name\": \"icelake-client\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"nvhpc\": [\n {\n \"versions\": \":\",\n \"name\": \"skylake\",\n \"flags\": \"-tp {name}\"\n }\n ]\n }\n },\n \"sapphirerapids\": {\n \"from\": [\n \"icelake\"\n ],\n \"vendor\": \"GenuineIntel\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"popcnt\",\n \"abm\",\n \"lahf_lm\",\n \"cx16\",\n \"sha_ni\",\n \"aes\",\n \"pclmulqdq\",\n \"avx\",\n \"rdrand\",\n \"f16c\",\n \"movbe\",\n \"fma\",\n \"avx2\",\n \"bmi1\",\n \"bmi2\",\n \"rdseed\",\n \"adx\",\n \"clflushopt\",\n \"xsave\",\n \"xsavec\",\n \"xsaveopt\",\n \"avx512f\",\n \"avx512vl\",\n \"avx512bw\",\n \"avx512dq\",\n \"avx512cd\",\n \"avx512vbmi\",\n \"avx512ifma\",\n \"sha_ni\",\n \"clwb\",\n \"rdpid\",\n \"gfni\",\n \"avx512_vbmi2\",\n \"avx512_vpopcntdq\",\n \"avx512_bitalg\",\n \"avx512_vnni\",\n \"vpclmulqdq\",\n \"vaes\",\n \"avx512_bf16\",\n \"cldemote\",\n \"movdir64b\",\n \"movdiri\",\n \"serialize\",\n \"waitpkg\",\n \"amx_bf16\",\n \"amx_tile\",\n \"amx_int8\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"11.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"12.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"intel\": [\n {\n \"versions\": \"2021.2:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"oneapi\": [\n {\n \"versions\": \"2021.2:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"dpcpp\": [\n {\n \"versions\": \"2021.2:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ]\n }\n },\n \"k10\": {\n \"from\": [\n \"x86_64\"\n ],\n \"vendor\": \"AuthenticAMD\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"sse4a\",\n \"abm\",\n \"cx16\",\n \"3dnow\",\n \"3dnowext\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"name\": \"amdfam10\",\n \"versions\": \"4.3:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"3.9:\",\n \"name\": \"amdfam10\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"aocc\": [\n {\n \"versions\": \"2.2:\",\n \"name\": \"amdfam10\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"intel\": [\n {\n \"versions\": \"16.0:\",\n \"warnings\": \"Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors\",\n \"flags\": \"-msse2\"\n }\n ],\n \"oneapi\": [\n {\n \"versions\": \":\",\n \"warnings\": \"Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors\",\n \"flags\": \"-msse2\"\n }\n ],\n \"dpcpp\": [\n {\n \"versions\": \":\",\n \"warnings\": \"Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors\",\n \"flags\": \"-msse2\"\n }\n ],\n \"nvhpc\": []\n }\n },\n \"bulldozer\": {\n \"from\": [\n \"x86_64_v2\"\n ],\n \"vendor\": \"AuthenticAMD\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"sse4a\",\n \"popcnt\",\n \"lahf_lm\",\n \"cx16\",\n \"xsave\",\n \"abm\",\n \"avx\",\n \"xop\",\n \"fma4\",\n \"aes\",\n \"pclmulqdq\",\n \"cx16\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"name\": \"bdver1\",\n \"versions\": \"4.7:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"3.9:\",\n \"name\": \"bdver1\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"aocc\": [\n {\n \"versions\": \"2.2:\",\n \"name\": \"bdver1\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"intel\": [\n {\n \"versions\": \"16.0:\",\n \"warnings\": \"Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors\",\n \"flags\": \"-msse3\"\n }\n ],\n \"oneapi\": [\n {\n \"versions\": \":\",\n \"warnings\": \"Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors\",\n \"flags\": \"-msse3\"\n }\n ],\n \"dpcpp\": [\n {\n \"versions\": \":\",\n \"warnings\": \"Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors\",\n \"flags\": \"-msse3\"\n }\n ],\n \"nvhpc\": [\n {\n \"versions\": \":\",\n \"flags\": \"-tp {name}\"\n }\n ]\n }\n },\n \"piledriver\": {\n \"from\": [\n \"bulldozer\"\n ],\n \"vendor\": \"AuthenticAMD\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"sse4a\",\n \"popcnt\",\n \"lahf_lm\",\n \"cx16\",\n \"xsave\",\n \"abm\",\n \"avx\",\n \"xop\",\n \"fma4\",\n \"aes\",\n \"pclmulqdq\",\n \"cx16\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"bmi1\",\n \"f16c\",\n \"fma\",\n \"tbm\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"name\": \"bdver2\",\n \"versions\": \"4.7:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"3.9:\",\n \"name\": \"bdver2\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"aocc\": [\n {\n \"versions\": \"2.2:\",\n \"name\": \"bdver2\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"intel\": [\n {\n \"versions\": \"16.0:\",\n \"warnings\": \"Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors\",\n \"flags\": \"-msse3\"\n }\n ],\n \"oneapi\": [\n {\n \"versions\": \":\",\n \"warnings\": \"Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors\",\n \"flags\": \"-msse3\"\n }\n ],\n \"dpcpp\": [\n {\n \"versions\": \":\",\n \"warnings\": \"Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors\",\n \"flags\": \"-msse3\"\n }\n ],\n \"nvhpc\": [\n {\n \"versions\": \":\",\n \"flags\": \"-tp {name}\"\n }\n ]\n }\n },\n \"steamroller\": {\n \"from\": [\n \"piledriver\"\n ],\n \"vendor\": \"AuthenticAMD\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"sse4a\",\n \"popcnt\",\n \"lahf_lm\",\n \"cx16\",\n \"xsave\",\n \"abm\",\n \"avx\",\n \"xop\",\n \"fma4\",\n \"aes\",\n \"pclmulqdq\",\n \"cx16\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"bmi1\",\n \"f16c\",\n \"fma\",\n \"fsgsbase\",\n \"tbm\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"name\": \"bdver3\",\n \"versions\": \"4.8:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"3.9:\",\n \"name\": \"bdver3\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"aocc\": [\n {\n \"versions\": \"2.2:\",\n \"name\": \"bdver3\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"intel\": [\n {\n \"versions\": \"16.0:\",\n \"warnings\": \"Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors\",\n \"flags\": \"-msse4.2\"\n }\n ],\n \"oneapi\": [\n {\n \"versions\": \":\",\n \"warnings\": \"Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors\",\n \"flags\": \"-msse4.2\"\n }\n ],\n \"dpcpp\": [\n {\n \"versions\": \":\",\n \"warnings\": \"Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors\",\n \"flags\": \"-msse4.2\"\n }\n ],\n \"nvhpc\": [\n {\n \"versions\": \":\",\n \"name\": \"piledriver\",\n \"flags\": \"-tp {name}\"\n }\n ]\n }\n },\n \"excavator\": {\n \"from\": [\n \"steamroller\",\n \"x86_64_v3\"\n ],\n \"vendor\": \"AuthenticAMD\",\n \"features\": [\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"sse4a\",\n \"popcnt\",\n \"lahf_lm\",\n \"cx16\",\n \"xsave\",\n \"abm\",\n \"avx\",\n \"xop\",\n \"fma4\",\n \"aes\",\n \"pclmulqdq\",\n \"cx16\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"bmi1\",\n \"f16c\",\n \"fma\",\n \"fsgsbase\",\n \"bmi2\",\n \"avx2\",\n \"movbe\",\n \"tbm\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"name\": \"bdver4\",\n \"versions\": \"4.9:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"3.9:\",\n \"name\": \"bdver4\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"aocc\": [\n {\n \"versions\": \"2.2:\",\n \"name\": \"bdver4\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"intel\": [\n {\n \"versions\": \"16.0:\",\n \"warnings\": \"Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors\",\n \"name\": \"core-avx2\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"oneapi\": [\n {\n \"versions\": \":\",\n \"warnings\": \"Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors\",\n \"name\": \"core-avx2\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"dpcpp\": [\n {\n \"versions\": \":\",\n \"warnings\": \"Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors\",\n \"name\": \"core-avx2\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"nvhpc\": [\n {\n \"versions\": \":\",\n \"name\": \"piledriver\",\n \"flags\": \"-tp {name}\"\n }\n ]\n }\n },\n \"zen\": {\n \"from\": [\n \"x86_64_v3\"\n ],\n \"vendor\": \"AuthenticAMD\",\n \"features\": [\n \"bmi1\",\n \"bmi2\",\n \"f16c\",\n \"fma\",\n \"fsgsbase\",\n \"avx\",\n \"avx2\",\n \"rdseed\",\n \"clzero\",\n \"aes\",\n \"pclmulqdq\",\n \"cx16\",\n \"movbe\",\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"sse4a\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"abm\",\n \"xsave\",\n \"xsavec\",\n \"xsaveopt\",\n \"clflushopt\",\n \"popcnt\",\n \"lahf_lm\",\n \"cx16\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"name\": \"znver1\",\n \"versions\": \"6.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"4.0:\",\n \"name\": \"znver1\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"aocc\": [\n {\n \"versions\": \"2.2:\",\n \"name\": \"znver1\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"intel\": [\n {\n \"versions\": \"16.0:\",\n \"warnings\": \"Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors\",\n \"name\": \"core-avx2\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"oneapi\": [\n {\n \"versions\": \":\",\n \"warnings\": \"Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors\",\n \"name\": \"core-avx2\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"dpcpp\": [\n {\n \"versions\": \":\",\n \"warnings\": \"Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors\",\n \"name\": \"core-avx2\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"nvhpc\": [\n {\n \"versions\": \":\",\n \"flags\": \"-tp {name}\"\n }\n ]\n }\n },\n \"zen2\": {\n \"from\": [\n \"zen\"\n ],\n \"vendor\": \"AuthenticAMD\",\n \"features\": [\n \"bmi1\",\n \"bmi2\",\n \"f16c\",\n \"fma\",\n \"fsgsbase\",\n \"avx\",\n \"avx2\",\n \"rdseed\",\n \"clzero\",\n \"aes\",\n \"pclmulqdq\",\n \"cx16\",\n \"movbe\",\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"sse4a\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"abm\",\n \"xsave\",\n \"xsavec\",\n \"xsaveopt\",\n \"clflushopt\",\n \"popcnt\",\n \"clwb\",\n \"lahf_lm\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"name\": \"znver2\",\n \"versions\": \"9.0:\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"9.0:\",\n \"name\": \"znver2\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"aocc\": [\n {\n \"versions\": \"2.2:\",\n \"name\": \"znver2\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"intel\": [\n {\n \"versions\": \"16.0:\",\n \"warnings\": \"Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors\",\n \"name\": \"core-avx2\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"oneapi\": [\n {\n \"versions\": \":\",\n \"warnings\": \"Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors\",\n \"name\": \"core-avx2\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"dpcpp\": [\n {\n \"versions\": \":\",\n \"warnings\": \"Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors\",\n \"name\": \"core-avx2\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"nvhpc\": [\n {\n \"versions\": \"20.5:\",\n \"flags\": \"-tp {name}\"\n }\n ]\n }\n },\n \"zen3\": {\n \"from\": [\n \"zen2\"\n ],\n \"vendor\": \"AuthenticAMD\",\n \"features\": [\n \"bmi1\",\n \"bmi2\",\n \"f16c\",\n \"fma\",\n \"fsgsbase\",\n \"avx\",\n \"avx2\",\n \"rdseed\",\n \"clzero\",\n \"aes\",\n \"pclmulqdq\",\n \"cx16\",\n \"movbe\",\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"sse4a\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"abm\",\n \"xsave\",\n \"xsavec\",\n \"xsaveopt\",\n \"clflushopt\",\n \"popcnt\",\n \"lahf_lm\",\n \"clwb\",\n \"vaes\",\n \"vpclmulqdq\",\n \"pku\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"10.3:\",\n \"name\": \"znver3\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"12.0:\",\n \"name\": \"znver3\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"aocc\": [\n {\n \"versions\": \"3.0:\",\n \"name\": \"znver3\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"intel\": [\n {\n \"versions\": \"16.0:\",\n \"warnings\": \"Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors\",\n \"name\": \"core-avx2\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"oneapi\": [\n {\n \"versions\": \":\",\n \"warnings\": \"Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors\",\n \"name\": \"core-avx2\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"dpcpp\": [\n {\n \"versions\": \":\",\n \"warnings\": \"Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors\",\n \"name\": \"core-avx2\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"nvhpc\": [\n {\n \"versions\": \"21.11:\",\n \"flags\": \"-tp {name}\"\n }\n ]\n }\n },\n \"zen4\": {\n \"from\": [\n \"zen3\",\n \"x86_64_v4\"\n ],\n \"vendor\": \"AuthenticAMD\",\n \"features\": [\n \"bmi1\",\n \"bmi2\",\n \"f16c\",\n \"fma\",\n \"fsgsbase\",\n \"avx\",\n \"avx2\",\n \"rdseed\",\n \"clzero\",\n \"aes\",\n \"pclmulqdq\",\n \"cx16\",\n \"movbe\",\n \"mmx\",\n \"sse\",\n \"sse2\",\n \"sse4a\",\n \"ssse3\",\n \"sse4_1\",\n \"sse4_2\",\n \"abm\",\n \"xsave\",\n \"xsavec\",\n \"xsaveopt\",\n \"clflushopt\",\n \"popcnt\",\n \"lahf_lm\",\n \"clwb\",\n \"vaes\",\n \"vpclmulqdq\",\n \"pku\",\n \"gfni\",\n \"flush_l1d\",\n \"avx512f\",\n \"avx512dq\",\n \"avx512ifma\",\n \"avx512cd\",\n \"avx512bw\",\n \"avx512vl\",\n \"avx512_bf16\",\n \"avx512vbmi\",\n \"avx512_vbmi2\",\n \"avx512_vnni\",\n \"avx512_bitalg\",\n \"avx512_vpopcntdq\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"10.3:12.2\",\n \"name\": \"znver3\",\n \"flags\": \"-march={name} -mtune={name} -mavx512f -mavx512dq -mavx512ifma -mavx512cd -mavx512bw -mavx512vl -mavx512vbmi -mavx512vbmi2 -mavx512vnni -mavx512bitalg\"\n },\n {\n \"versions\": \"12.3:\",\n \"name\": \"znver4\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"12.0:15.9\",\n \"name\": \"znver3\",\n \"flags\": \"-march={name} -mtune={name} -mavx512f -mavx512dq -mavx512ifma -mavx512cd -mavx512bw -mavx512vl -mavx512vbmi -mavx512vbmi2 -mavx512vnni -mavx512bitalg\"\n },\n {\n \"versions\": \"16.0:\",\n \"name\": \"znver4\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"aocc\": [\n {\n \"versions\": \"3.0:3.9\",\n \"name\": \"znver3\",\n \"flags\": \"-march={name} -mtune={name} -mavx512f -mavx512dq -mavx512ifma -mavx512cd -mavx512bw -mavx512vl -mavx512vbmi -mavx512vbmi2 -mavx512vnni -mavx512bitalg\",\n \"warnings\": \"Zen4 processors are not fully supported by AOCC versions < 4.0. For optimal performance please upgrade to a newer version of AOCC\"\n },\n {\n \"versions\": \"4.0:\",\n \"name\": \"znver4\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"nvhpc\": [\n {\n \"versions\": \"21.11:23.8\",\n \"name\": \"zen3\",\n \"flags\": \"-tp {name}\",\n \"warnings\": \"zen4 is not fully supported by nvhpc versions < 23.9, falling back to zen3\"\n },\n {\n \"versions\": \"23.9:\",\n \"flags\": \"-tp {name}\"\n }\n ]\n }\n },\n \"zen5\": {\n \"from\": [\n \"zen4\"\n ],\n \"vendor\": \"AuthenticAMD\",\n \"features\": [\n \"abm\",\n \"aes\",\n \"avx\",\n \"avx2\",\n \"avx512_bf16\",\n \"avx512_bitalg\",\n \"avx512bw\",\n \"avx512cd\",\n \"avx512dq\",\n \"avx512f\",\n \"avx512ifma\",\n \"avx512vbmi\",\n \"avx512_vbmi2\",\n \"avx512vl\",\n \"avx512_vnni\",\n \"avx512_vp2intersect\",\n \"avx512_vpopcntdq\",\n \"avx_vnni\",\n \"bmi1\",\n \"bmi2\",\n \"clflushopt\",\n \"clwb\",\n \"clzero\",\n \"cx16\",\n \"f16c\",\n \"flush_l1d\",\n \"fma\",\n \"fsgsbase\",\n \"gfni\",\n \"ibrs_enhanced\",\n \"mmx\",\n \"movbe\",\n \"movdir64b\",\n \"lahf_lm\",\n \"movdiri\",\n \"pclmulqdq\",\n \"popcnt\",\n \"pku\",\n \"rdseed\",\n \"sse\",\n \"sse2\",\n \"sse4_1\",\n \"sse4_2\",\n \"sse4a\",\n \"ssse3\",\n \"tsc_adjust\",\n \"vaes\",\n \"vpclmulqdq\",\n \"xsave\",\n \"xsavec\",\n \"xsaveopt\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"14.1:\",\n \"name\": \"znver5\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"aocc\": [\n {\n \"versions\": \"5.0:\",\n \"name\": \"znver5\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"19.1:\",\n \"name\": \"znver5\",\n \"flags\": \"-march={name} -mtune={name}\"\n }\n ]\n }\n },\n \"ppc64\": {\n \"from\": [],\n \"vendor\": \"generic\",\n \"features\": [],\n \"compilers\": {\n \"gcc\": [\n {\n \"name\": \"powerpc64\",\n \"versions\": \":\",\n \"flags\": \"-mcpu={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \":\",\n \"flags\": \"-mcpu={name} -mtune={name}\"\n }\n ]\n }\n },\n \"power7\": {\n \"from\": [\n \"ppc64\"\n ],\n \"vendor\": \"IBM\",\n \"generation\": 7,\n \"features\": [],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"4.4:\",\n \"flags\": \"-mcpu={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"3.9:\",\n \"flags\": \"-mcpu={name} -mtune={name}\"\n }\n ]\n }\n },\n \"power8\": {\n \"from\": [\n \"power7\"\n ],\n \"vendor\": \"IBM\",\n \"generation\": 8,\n \"features\": [],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"4.9:\",\n \"flags\": \"-mcpu={name} -mtune={name}\"\n },\n {\n \"versions\": \"4.8:4.8.5\",\n \"warnings\": \"Using GCC 4.8 to optimize for Power 8 might not work if you are not on Red Hat Enterprise Linux 7, where a custom backport of the feature has been done. Upstream support from GCC starts in version 4.9\",\n \"flags\": \"-mcpu={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"3.9:\",\n \"flags\": \"-mcpu={name} -mtune={name}\"\n }\n ]\n }\n },\n \"power9\": {\n \"from\": [\n \"power8\"\n ],\n \"vendor\": \"IBM\",\n \"generation\": 9,\n \"features\": [],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"6.0:\",\n \"flags\": \"-mcpu={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"3.9:\",\n \"flags\": \"-mcpu={name} -mtune={name}\"\n }\n ]\n }\n },\n \"power10\": {\n \"from\": [\n \"power9\"\n ],\n \"vendor\": \"IBM\",\n \"generation\": 10,\n \"features\": [],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"11.1:\",\n \"flags\": \"-mcpu={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"11.0:\",\n \"flags\": \"-mcpu={name} -mtune={name}\"\n }\n ]\n }\n },\n \"ppc64le\": {\n \"from\": [],\n \"vendor\": \"generic\",\n \"features\": [],\n \"compilers\": {\n \"gcc\": [\n {\n \"name\": \"powerpc64le\",\n \"versions\": \"4.8:\",\n \"flags\": \"-mcpu={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \":\",\n \"flags\": \"-mcpu={name} -mtune={name}\"\n }\n ],\n \"nvhpc\": []\n }\n },\n \"power8le\": {\n \"from\": [\n \"ppc64le\"\n ],\n \"vendor\": \"IBM\",\n \"generation\": 8,\n \"features\": [],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"4.9:\",\n \"name\": \"power8\",\n \"flags\": \"-mcpu={name} -mtune={name}\"\n },\n {\n \"versions\": \"4.8:4.8.5\",\n \"warnings\": \"Using GCC 4.8 to optimize for Power 8 might not work if you are not on Red Hat Enterprise Linux 7, where a custom backport of the feature has been done. Upstream support from GCC starts in version 4.9\",\n \"name\": \"power8\",\n \"flags\": \"-mcpu={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"3.9:\",\n \"family\": \"ppc64le\",\n \"name\": \"power8\",\n \"flags\": \"-mcpu={name} -mtune={name}\"\n }\n ],\n \"nvhpc\": [\n {\n \"versions\": \":\",\n \"name\": \"pwr8\",\n \"flags\": \"-tp {name}\"\n }\n ]\n }\n },\n \"power9le\": {\n \"from\": [\n \"power8le\"\n ],\n \"vendor\": \"IBM\",\n \"generation\": 9,\n \"features\": [],\n \"compilers\": {\n \"gcc\": [\n {\n \"name\": \"power9\",\n \"versions\": \"6.0:\",\n \"flags\": \"-mcpu={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"3.9:\",\n \"family\": \"ppc64le\",\n \"name\": \"power9\",\n \"flags\": \"-mcpu={name} -mtune={name}\"\n }\n ],\n \"nvhpc\": [\n {\n \"versions\": \":\",\n \"name\": \"pwr9\",\n \"flags\": \"-tp {name}\"\n }\n ]\n }\n },\n \"power10le\": {\n \"from\": [\n \"power9le\"\n ],\n \"vendor\": \"IBM\",\n \"generation\": 10,\n \"features\": [],\n \"compilers\": {\n \"gcc\": [\n {\n \"name\": \"power10\",\n \"versions\": \"11.1:\",\n \"flags\": \"-mcpu={name} -mtune={name}\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"11.0:\",\n \"family\": \"ppc64le\",\n \"name\": \"power10\",\n \"flags\": \"-mcpu={name} -mtune={name}\"\n }\n ]\n }\n },\n \"aarch64\": {\n \"from\": [],\n \"vendor\": \"generic\",\n \"features\": [],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"4.8.0:\",\n \"flags\": \"-march=armv8-a -mtune=generic\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march=armv8-a -mtune=generic\"\n }\n ],\n \"apple-clang\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march=armv8-a -mtune=generic\"\n }\n ],\n \"arm\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march=armv8-a -mtune=generic\"\n }\n ],\n \"nvhpc\": []\n }\n },\n \"armv8.1a\": {\n \"from\": [\n \"aarch64\"\n ],\n \"vendor\": \"generic\",\n \"features\": [],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"5:\",\n \"flags\": \"-march=armv8.1-a -mtune=generic\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march=armv8.1-a -mtune=generic\"\n }\n ],\n \"apple-clang\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march=armv8.1-a -mtune=generic\"\n }\n ],\n \"arm\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march=armv8.1-a -mtune=generic\"\n }\n ]\n }\n },\n \"armv8.2a\": {\n \"from\": [\n \"armv8.1a\"\n ],\n \"vendor\": \"generic\",\n \"features\": [],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"6:\",\n \"flags\": \"-march=armv8.2-a -mtune=generic\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march=armv8.2-a -mtune=generic\"\n }\n ],\n \"apple-clang\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march=armv8.2-a -mtune=generic\"\n }\n ],\n \"arm\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march=armv8.2-a -mtune=generic\"\n }\n ]\n }\n },\n \"armv8.3a\": {\n \"from\": [\n \"armv8.2a\"\n ],\n \"vendor\": \"generic\",\n \"features\": [],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"6:\",\n \"flags\": \"-march=armv8.3-a -mtune=generic\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"6:\",\n \"flags\": \"-march=armv8.3-a -mtune=generic\"\n }\n ],\n \"apple-clang\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march=armv8.3-a -mtune=generic\"\n }\n ],\n \"arm\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march=armv8.3-a -mtune=generic\"\n }\n ]\n }\n },\n \"armv8.4a\": {\n \"from\": [\n \"armv8.3a\"\n ],\n \"vendor\": \"generic\",\n \"features\": [],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"8:\",\n \"flags\": \"-march=armv8.4-a -mtune=generic\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"8:\",\n \"flags\": \"-march=armv8.4-a -mtune=generic\"\n }\n ],\n \"apple-clang\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march=armv8.4-a -mtune=generic\"\n }\n ],\n \"arm\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march=armv8.4-a -mtune=generic\"\n }\n ]\n }\n },\n \"armv8.5a\": {\n \"from\": [\n \"armv8.4a\"\n ],\n \"vendor\": \"generic\",\n \"features\": [],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"9:\",\n \"flags\": \"-march=armv8.5-a -mtune=generic\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"11:\",\n \"flags\": \"-march=armv8.5-a -mtune=generic\"\n }\n ],\n \"apple-clang\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march=armv8.5-a -mtune=generic\"\n }\n ],\n \"arm\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march=armv8.5-a -mtune=generic\"\n }\n ]\n }\n },\n \"armv8.6a\": {\n \"from\": [\n \"armv8.5a\"\n ],\n \"vendor\": \"generic\",\n \"features\": [],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"10.1:\",\n \"flags\": \"-march=armv8.6-a -mtune=generic\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"11:\",\n \"flags\": \"-march=armv8.6-a -mtune=generic\"\n }\n ]\n }\n },\n \"armv9.0a\": {\n \"from\": [\n \"armv8.5a\"\n ],\n \"vendor\": \"generic\",\n \"features\": [],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"12:\",\n \"flags\": \"-march=armv9-a -mtune=generic\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"14:\",\n \"flags\": \"-march=armv9-a -mtune=generic\"\n }\n ],\n \"apple-clang\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march=armv9-a -mtune=generic\"\n }\n ],\n \"arm\": [\n {\n \"versions\": \":\",\n \"flags\": \"-march=armv9-a -mtune=generic\"\n }\n ]\n }\n },\n \"thunderx2\": {\n \"from\": [\n \"armv8.1a\"\n ],\n \"vendor\": \"Cavium\",\n \"features\": [\n \"fp\",\n \"asimd\",\n \"evtstrm\",\n \"aes\",\n \"pmull\",\n \"sha1\",\n \"sha2\",\n \"crc32\",\n \"atomics\",\n \"cpuid\",\n \"asimdrdm\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"4.8:4.8.9\",\n \"flags\": \"-march=armv8-a\"\n },\n {\n \"versions\": \"4.9:5.9\",\n \"flags\": \"-march=armv8-a+crc+crypto\"\n },\n {\n \"versions\": \"6:6.9\",\n \"flags\": \"-march=armv8.1-a+crc+crypto\"\n },\n {\n \"versions\": \"7:\",\n \"flags\": \"-mcpu=thunderx2t99\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"3.9:4.9\",\n \"flags\": \"-march=armv8.1-a+crc+crypto\"\n },\n {\n \"versions\": \"5:\",\n \"flags\": \"-mcpu=thunderx2t99\"\n }\n ]\n },\n \"cpupart\": \"0x0af\"\n },\n \"a64fx\": {\n \"from\": [\n \"armv8.2a\"\n ],\n \"vendor\": \"Fujitsu\",\n \"features\": [\n \"fp\",\n \"asimd\",\n \"evtstrm\",\n \"sha1\",\n \"sha2\",\n \"crc32\",\n \"atomics\",\n \"cpuid\",\n \"asimdrdm\",\n \"fphp\",\n \"asimdhp\",\n \"fcma\",\n \"dcpop\",\n \"sve\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"4.8:4.8.9\",\n \"flags\": \"-march=armv8-a\"\n },\n {\n \"versions\": \"4.9:5.9\",\n \"flags\": \"-march=armv8-a+crc+crypto\"\n },\n {\n \"versions\": \"6:6.9\",\n \"flags\": \"-march=armv8.1-a+crc+crypto\"\n },\n {\n \"versions\": \"7:7.9\",\n \"flags\": \"-march=armv8.2-a+crc+crypto+fp16\"\n },\n {\n \"versions\": \"8:10.2\",\n \"flags\": \"-march=armv8.2-a+crc+sha2+fp16+sve -msve-vector-bits=512\"\n },\n {\n \"versions\": \"10.3:\",\n \"flags\": \"-mcpu=a64fx -msve-vector-bits=512\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"3.9:4.9\",\n \"flags\": \"-march=armv8.2-a+crc+sha2+fp16\"\n },\n {\n \"versions\": \"5:10\",\n \"flags\": \"-march=armv8.2-a+crc+sha2+fp16+sve\"\n },\n {\n \"versions\": \"11:\",\n \"flags\": \"-mcpu=a64fx\"\n }\n ],\n \"arm\": [\n {\n \"versions\": \"20:\",\n \"flags\": \"-march=armv8.2-a+crc+crypto+fp16+sve\"\n }\n ]\n },\n \"cpupart\": \"0x001\"\n },\n \"cortex_a72\": {\n \"from\": [\n \"aarch64\"\n ],\n \"vendor\": \"ARM\",\n \"features\": [\n \"fp\",\n \"asimd\",\n \"evtstrm\",\n \"aes\",\n \"pmull\",\n \"sha1\",\n \"sha2\",\n \"crc32\",\n \"cpuid\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"4.8:4.8.9\",\n \"flags\": \"-march=armv8-a\"\n },\n {\n \"versions\": \"4.9:5.9\",\n \"flags\": \"-march=armv8-a+crc+crypto\"\n },\n {\n \"versions\": \"6:\",\n \"flags\": \"-mcpu=cortex-a72\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"3.9:\",\n \"flags\": \"-mcpu=cortex-a72\"\n }\n ]\n },\n \"cpupart\": \"0xd08\"\n },\n \"neoverse_n1\": {\n \"from\": [\n \"cortex_a72\",\n \"armv8.2a\"\n ],\n \"vendor\": \"ARM\",\n \"features\": [\n \"fp\",\n \"asimd\",\n \"evtstrm\",\n \"aes\",\n \"pmull\",\n \"sha1\",\n \"sha2\",\n \"crc32\",\n \"atomics\",\n \"fphp\",\n \"asimdhp\",\n \"cpuid\",\n \"asimdrdm\",\n \"lrcpc\",\n \"dcpop\",\n \"asimddp\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"4.8:4.8.9\",\n \"flags\": \"-march=armv8-a\"\n },\n {\n \"versions\": \"4.9:5.9\",\n \"flags\": \"-march=armv8-a+crc+crypto\"\n },\n {\n \"versions\": \"6:6.9\",\n \"flags\": \"-march=armv8.1-a\"\n },\n {\n \"versions\": \"7:7.9\",\n \"flags\": \"-march=armv8.2-a+fp16 -mtune=cortex-a72\"\n },\n {\n \"versions\": \"8.0:8.0\",\n \"flags\": \"-march=armv8.2-a+fp16+dotprod+crypto -mtune=cortex-a72\"\n },\n {\n \"versions\": \"8.1:8.9\",\n \"flags\": \"-march=armv8.2-a+fp16+rcpc+dotprod+crypto -mtune=cortex-a72\"\n },\n {\n \"versions\": \"9.0:\",\n \"flags\": \"-mcpu=neoverse-n1\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"3.9:4.9\",\n \"flags\": \"-march=armv8.2-a+fp16+crc+crypto\"\n },\n {\n \"versions\": \"5:\",\n \"flags\": \"-march=armv8.2-a+fp16+rcpc+dotprod+crypto\"\n },\n {\n \"versions\": \"10:\",\n \"flags\": \"-mcpu=neoverse-n1\"\n }\n ],\n \"arm\": [\n {\n \"versions\": \"20:21.9\",\n \"flags\": \"-march=armv8.2-a+fp16+rcpc+dotprod+crypto\"\n },\n {\n \"versions\": \"22:\",\n \"flags\": \"-mcpu=neoverse-n1\"\n }\n ],\n \"nvhpc\": [\n {\n \"versions\": \"22.5:\",\n \"name\": \"neoverse-n1\",\n \"flags\": \"-tp {name}\"\n }\n ]\n },\n \"cpupart\": \"0xd0c\"\n },\n \"neoverse_v1\": {\n \"from\": [\n \"neoverse_n1\",\n \"armv8.4a\"\n ],\n \"vendor\": \"ARM\",\n \"features\": [\n \"fp\",\n \"asimd\",\n \"evtstrm\",\n \"aes\",\n \"pmull\",\n \"sha1\",\n \"sha2\",\n \"crc32\",\n \"atomics\",\n \"fphp\",\n \"asimdhp\",\n \"cpuid\",\n \"asimdrdm\",\n \"jscvt\",\n \"fcma\",\n \"lrcpc\",\n \"dcpop\",\n \"sha3\",\n \"asimddp\",\n \"sha512\",\n \"sve\",\n \"asimdfhm\",\n \"dit\",\n \"uscat\",\n \"ilrcpc\",\n \"flagm\",\n \"dcpodp\",\n \"svei8mm\",\n \"svebf16\",\n \"i8mm\",\n \"bf16\",\n \"dgh\",\n \"rng\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"4.8:4.8.9\",\n \"flags\": \"-march=armv8-a\"\n },\n {\n \"versions\": \"4.9:5.9\",\n \"flags\": \"-march=armv8-a+crc+crypto\"\n },\n {\n \"versions\": \"6:6.9\",\n \"flags\": \"-march=armv8.1-a\"\n },\n {\n \"versions\": \"7:7.9\",\n \"flags\": \"-march=armv8.2-a+crypto+fp16 -mtune=cortex-a72\"\n },\n {\n \"versions\": \"8.0:8.4\",\n \"flags\": \"-march=armv8.2-a+fp16+dotprod+crypto -mtune=cortex-a72\"\n },\n {\n \"versions\": \"8.5:8.9\",\n \"flags\": \"-mcpu=neoverse-v1\"\n },\n {\n \"versions\": \"9.0:9.3\",\n \"flags\": \"-march=armv8.2-a+fp16+dotprod+crypto -mtune=cortex-a72\"\n },\n {\n \"versions\": \"9.4:9.9\",\n \"flags\": \"-mcpu=neoverse-v1\"\n },\n {\n \"versions\": \"10.0:10.1\",\n \"flags\": \"-march=armv8.2-a+fp16+dotprod+crypto -mtune=cortex-a72\"\n },\n {\n \"versions\": \"10.2:10.2.99\",\n \"flags\": \"-mcpu=zeus\"\n },\n {\n \"versions\": \"10.3:\",\n \"flags\": \"-mcpu=neoverse-v1\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"3.9:4.9\",\n \"flags\": \"-march=armv8.2-a+fp16+crc+crypto\"\n },\n {\n \"versions\": \"5:10\",\n \"flags\": \"-march=armv8.2-a+fp16+rcpc+dotprod+crypto\"\n },\n {\n \"versions\": \"11:\",\n \"flags\": \"-march=armv8.4-a+sve+fp16+bf16+crypto+i8mm+rng\"\n },\n {\n \"versions\": \"12:\",\n \"flags\": \"-mcpu=neoverse-v1\"\n }\n ],\n \"arm\": [\n {\n \"versions\": \"20:21.9\",\n \"flags\": \"-march=armv8.2-a+sve+fp16+rcpc+dotprod+crypto\"\n },\n {\n \"versions\": \"22:\",\n \"flags\": \"-mcpu=neoverse-v1\"\n }\n ],\n \"nvhpc\": [\n {\n \"versions\": \"22.5:\",\n \"name\": \"neoverse-n1\",\n \"flags\": \"-tp {name}\"\n }\n ]\n },\n \"cpupart\": \"0xd40\"\n },\n \"neoverse_v2\": {\n \"from\": [\n \"neoverse_n1\",\n \"armv9.0a\"\n ],\n \"vendor\": \"ARM\",\n \"features\": [\n \"fp\",\n \"asimd\",\n \"evtstrm\",\n \"aes\",\n \"pmull\",\n \"sha1\",\n \"sha2\",\n \"crc32\",\n \"atomics\",\n \"fphp\",\n \"asimdhp\",\n \"cpuid\",\n \"asimdrdm\",\n \"jscvt\",\n \"fcma\",\n \"lrcpc\",\n \"dcpop\",\n \"sha3\",\n \"asimddp\",\n \"sha512\",\n \"sve\",\n \"asimdfhm\",\n \"uscat\",\n \"ilrcpc\",\n \"flagm\",\n \"sb\",\n \"dcpodp\",\n \"sve2\",\n \"flagm2\",\n \"frint\",\n \"svei8mm\",\n \"svebf16\",\n \"i8mm\",\n \"bf16\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"4.8:5.99\",\n \"flags\": \"-march=armv8-a\"\n },\n {\n \"versions\": \"6:6.99\",\n \"flags\": \"-march=armv8.1-a\"\n },\n {\n \"versions\": \"7.0:7.99\",\n \"flags\": \"-march=armv8.2-a -mtune=cortex-a72\"\n },\n {\n \"versions\": \"8.0:8.99\",\n \"flags\": \"-march=armv8.4-a+sve -mtune=cortex-a72\"\n },\n {\n \"versions\": \"9.0:9.99\",\n \"flags\": \"-march=armv8.5-a+sve -mtune=cortex-a76\"\n },\n {\n \"versions\": \"10.0:11.3.99\",\n \"flags\": \"-march=armv8.5-a+sve+sve2+i8mm+bf16 -mtune=cortex-a77\"\n },\n {\n \"versions\": \"11.4:11.99\",\n \"flags\": \"-mcpu=neoverse-v2\"\n },\n {\n \"versions\": \"12.0:12.2.99\",\n \"flags\": \"-march=armv9-a+i8mm+bf16 -mtune=cortex-a710\"\n },\n {\n \"versions\": \"12.3:\",\n \"flags\": \"-mcpu=neoverse-v2\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"9.0:10.99\",\n \"flags\": \"-march=armv8.5-a+sve\"\n },\n {\n \"versions\": \"11.0:13.99\",\n \"flags\": \"-march=armv8.5-a+sve+sve2+i8mm+bf16\"\n },\n {\n \"versions\": \"14.0:15.99\",\n \"flags\": \"-march=armv9-a+i8mm+bf16\"\n },\n {\n \"versions\": \"16.0:\",\n \"flags\": \"-mcpu=neoverse-v2\"\n }\n ],\n \"arm\": [\n {\n \"versions\": \"23.04.0:\",\n \"flags\": \"-mcpu=neoverse-v2\"\n }\n ],\n \"nvhpc\": [\n {\n \"versions\": \"23.3:\",\n \"name\": \"neoverse-v2\",\n \"flags\": \"-tp {name}\"\n }\n ]\n },\n \"cpupart\": \"0xd4f\"\n },\n \"neoverse_n2\": {\n \"from\": [\n \"neoverse_n1\",\n \"armv9.0a\"\n ],\n \"vendor\": \"ARM\",\n \"features\": [\n \"fp\",\n \"asimd\",\n \"evtstrm\",\n \"aes\",\n \"pmull\",\n \"sha1\",\n \"sha2\",\n \"crc32\",\n \"atomics\",\n \"fphp\",\n \"asimdhp\",\n \"cpuid\",\n \"asimdrdm\",\n \"jscvt\",\n \"fcma\",\n \"lrcpc\",\n \"dcpop\",\n \"sha3\",\n \"asimddp\",\n \"sha512\",\n \"sve\",\n \"asimdfhm\",\n \"uscat\",\n \"ilrcpc\",\n \"flagm\",\n \"sb\",\n \"dcpodp\",\n \"sve2\",\n \"flagm2\",\n \"frint\",\n \"svei8mm\",\n \"svebf16\",\n \"i8mm\",\n \"bf16\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"4.8:5.99\",\n \"flags\": \"-march=armv8-a\"\n },\n {\n \"versions\": \"6:6.99\",\n \"flags\": \"-march=armv8.1-a\"\n },\n {\n \"versions\": \"7.0:7.99\",\n \"flags\": \"-march=armv8.2-a -mtune=cortex-a72\"\n },\n {\n \"versions\": \"8.0:8.99\",\n \"flags\": \"-march=armv8.4-a+sve -mtune=cortex-a72\"\n },\n {\n \"versions\": \"9.0:9.99\",\n \"flags\": \"-march=armv8.5-a+sve -mtune=cortex-a76\"\n },\n {\n \"versions\": \"10.0:10.99\",\n \"flags\": \"-march=armv8.5-a+sve+sve2+i8mm+bf16 -mtune=cortex-a77\"\n },\n {\n \"versions\": \"11.0:\",\n \"flags\": \"-mcpu=neoverse-n2\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"9.0:10.99\",\n \"flags\": \"-march=armv8.5-a+sve\"\n },\n {\n \"versions\": \"11.0:13.99\",\n \"flags\": \"-march=armv8.5-a+sve+sve2+i8mm+bf16\"\n },\n {\n \"versions\": \"14.0:15.99\",\n \"flags\": \"-march=armv9-a+i8mm+bf16\"\n },\n {\n \"versions\": \"16.0:\",\n \"flags\": \"-mcpu=neoverse-n2\"\n }\n ],\n \"arm\": [\n {\n \"versions\": \"23.04.0:\",\n \"flags\": \"-mcpu=neoverse-n2\"\n }\n ],\n \"nvhpc\": [\n {\n \"versions\": \"23.3:\",\n \"name\": \"neoverse-n1\",\n \"flags\": \"-tp {name}\"\n }\n ]\n },\n \"cpupart\": \"0xd49\"\n },\n \"m1\": {\n \"from\": [\n \"armv8.4a\"\n ],\n \"vendor\": \"Apple\",\n \"features\": [\n \"fp\",\n \"asimd\",\n \"evtstrm\",\n \"aes\",\n \"pmull\",\n \"sha1\",\n \"sha2\",\n \"crc32\",\n \"atomics\",\n \"fphp\",\n \"asimdhp\",\n \"cpuid\",\n \"asimdrdm\",\n \"jscvt\",\n \"fcma\",\n \"lrcpc\",\n \"dcpop\",\n \"sha3\",\n \"asimddp\",\n \"sha512\",\n \"asimdfhm\",\n \"dit\",\n \"uscat\",\n \"ilrcpc\",\n \"flagm\",\n \"ssbs\",\n \"sb\",\n \"paca\",\n \"pacg\",\n \"dcpodp\",\n \"flagm2\",\n \"frint\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"8.0:\",\n \"flags\": \"-march=armv8.4-a -mtune=generic\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"9.0:12.0\",\n \"flags\": \"-march=armv8.4-a\"\n },\n {\n \"versions\": \"13.0:\",\n \"flags\": \"-mcpu=apple-m1\"\n }\n ],\n \"apple-clang\": [\n {\n \"versions\": \"11.0:12.5\",\n \"flags\": \"-march=armv8.4-a\"\n },\n {\n \"versions\": \"13.0:\",\n \"flags\": \"-mcpu=apple-m1\"\n }\n ]\n },\n \"cpupart\": \"0x022\"\n },\n \"m2\": {\n \"from\": [\n \"m1\",\n \"armv8.5a\"\n ],\n \"vendor\": \"Apple\",\n \"features\": [\n \"fp\",\n \"asimd\",\n \"evtstrm\",\n \"aes\",\n \"pmull\",\n \"sha1\",\n \"sha2\",\n \"crc32\",\n \"atomics\",\n \"fphp\",\n \"asimdhp\",\n \"cpuid\",\n \"asimdrdm\",\n \"jscvt\",\n \"fcma\",\n \"lrcpc\",\n \"dcpop\",\n \"sha3\",\n \"asimddp\",\n \"sha512\",\n \"asimdfhm\",\n \"dit\",\n \"uscat\",\n \"ilrcpc\",\n \"flagm\",\n \"ssbs\",\n \"sb\",\n \"paca\",\n \"pacg\",\n \"dcpodp\",\n \"flagm2\",\n \"frint\",\n \"ecv\",\n \"bf16\",\n \"i8mm\",\n \"bti\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"8.0:\",\n \"flags\": \"-march=armv8.5-a -mtune=generic\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"9.0:12.0\",\n \"flags\": \"-march=armv8.5-a\"\n },\n {\n \"versions\": \"13.0:\",\n \"flags\": \"-mcpu=apple-m1\"\n },\n {\n \"versions\": \"16.0:\",\n \"flags\": \"-mcpu=apple-m2\"\n }\n ],\n \"apple-clang\": [\n {\n \"versions\": \"11.0:12.5\",\n \"flags\": \"-march=armv8.5-a\"\n },\n {\n \"versions\": \"13.0:14.0.2\",\n \"flags\": \"-mcpu=apple-m1\"\n },\n {\n \"versions\": \"14.0.2:\",\n \"flags\": \"-mcpu=apple-m2\"\n }\n ]\n },\n \"cpupart\": \"0x032\"\n },\n \"m3\": {\n \"from\": [\n \"m2\",\n \"armv8.6a\"\n ],\n \"vendor\": \"Apple\",\n \"features\": [\n \"fp\",\n \"asimd\",\n \"evtstrm\",\n \"aes\",\n \"pmull\",\n \"sha1\",\n \"sha2\",\n \"crc32\",\n \"atomics\",\n \"fphp\",\n \"asimdhp\",\n \"cpuid\",\n \"asimdrdm\",\n \"jscvt\",\n \"fcma\",\n \"lrcpc\",\n \"dcpop\",\n \"sha3\",\n \"asimddp\",\n \"sha512\",\n \"asimdfhm\",\n \"dit\",\n \"uscat\",\n \"ilrcpc\",\n \"flagm\",\n \"ssbs\",\n \"sb\",\n \"paca\",\n \"pacg\",\n \"dcpodp\",\n \"flagm2\",\n \"frint\",\n \"ecv\",\n \"bf16\",\n \"i8mm\",\n \"bti\"\n ],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"8.0:\",\n \"flags\": \"-march=armv8.5-a -mtune=generic\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"9.0:12.0\",\n \"flags\": \"-march=armv8.5-a\"\n },\n {\n \"versions\": \"13.0:\",\n \"flags\": \"-mcpu=apple-m1\"\n },\n {\n \"versions\": \"16.0:\",\n \"flags\": \"-mcpu=apple-m3\"\n }\n ],\n \"apple-clang\": [\n {\n \"versions\": \"11.0:12.5\",\n \"flags\": \"-march=armv8.5-a\"\n },\n {\n \"versions\": \"13.0:14.0.2\",\n \"flags\": \"-mcpu=apple-m1\"\n },\n {\n \"versions\": \"14.0.2:15\",\n \"flags\": \"-mcpu=apple-m2\"\n },\n {\n \"versions\": \"16:\",\n \"flags\": \"-mcpu=apple-m3\"\n }\n ]\n },\n \"cpupart\": \"Unknown\"\n },\n \"m4\": {\n \"from\": [\n \"m3\",\n \"armv8.6a\"\n ],\n \"vendor\": \"Apple\",\n \"features\": [\n \"fp\",\n \"asimd\",\n \"evtstrm\",\n \"aes\",\n \"pmull\",\n \"sha1\",\n \"sha2\",\n \"crc32\",\n \"atomics\",\n \"fphp\",\n \"asimdhp\",\n \"cpuid\",\n \"asimdrdm\",\n \"jscvt\",\n \"fcma\",\n \"lrcpc\",\n \"dcpop\",\n \"sha3\",\n \"asimddp\",\n \"sha512\",\n \"asimdfhm\",\n \"dit\",\n \"uscat\",\n \"ilrcpc\",\n \"flagm\",\n \"ssbs\",\n \"sb\",\n \"paca\",\n \"pacg\",\n \"dcpodp\",\n \"flagm2\",\n \"frint\",\n \"ecv\",\n \"bf16\",\n \"i8mm\",\n \"bti\",\n \"sme\",\n \"sme2\"\n ],\n \"compilers\": {\n \"clang\": [\n {\n \"versions\": \"9.0:12.0\",\n \"flags\": \"-march=armv8.5-a\"\n },\n {\n \"versions\": \"13.0:\",\n \"flags\": \"-mcpu=apple-m1\"\n },\n {\n \"versions\": \"16.0:18\",\n \"flags\": \"-mcpu=apple-m3\"\n },\n {\n \"versions\": \"19:\",\n \"flags\": \"-mcpu=apple-m4\"\n }\n ],\n \"apple-clang\": [\n {\n \"versions\": \"11.0:12.5\",\n \"flags\": \"-march=armv8.5-a\"\n },\n {\n \"versions\": \"13.0:14.0.2\",\n \"flags\": \"-mcpu=apple-m1\"\n },\n {\n \"versions\": \"14.0.2:15\",\n \"flags\": \"-mcpu=apple-m2\"\n },\n {\n \"versions\": \"16:\",\n \"flags\": \"-mcpu=apple-m3\"\n },\n {\n \"versions\": \"17:\",\n \"flags\": \"-mcpu=apple-m4\"\n }\n ]\n },\n \"cpupart\": \"Unknown\"\n },\n \"arm\": {\n \"from\": [],\n \"vendor\": \"generic\",\n \"features\": [],\n \"compilers\": {\n \"clang\": [\n {\n \"versions\": \":\",\n \"family\": \"arm\",\n \"flags\": \"-march={family} -mcpu=generic\"\n }\n ]\n }\n },\n \"ppc\": {\n \"from\": [],\n \"vendor\": \"generic\",\n \"features\": [],\n \"compilers\": {}\n },\n \"ppcle\": {\n \"from\": [],\n \"vendor\": \"generic\",\n \"features\": [],\n \"compilers\": {}\n },\n \"sparc\": {\n \"from\": [],\n \"vendor\": \"generic\",\n \"features\": [],\n \"compilers\": {}\n },\n \"sparc64\": {\n \"from\": [],\n \"vendor\": \"generic\",\n \"features\": [],\n \"compilers\": {}\n },\n \"riscv64\": {\n \"from\": [],\n \"vendor\": \"generic\",\n \"features\": [],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"7.1:\",\n \"flags\": \"-march=rv64gc\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"9.0:\",\n \"flags\": \"-march=rv64gc\"\n }\n ]\n }\n },\n \"u74mc\": {\n \"from\": [\n \"riscv64\"\n ],\n \"vendor\": \"SiFive\",\n \"features\": [],\n \"compilers\": {\n \"gcc\": [\n {\n \"versions\": \"10.2:\",\n \"flags\": \"-march=rv64gc -mtune=sifive-7-series\"\n }\n ],\n \"clang\": [\n {\n \"versions\": \"12.0:\",\n \"flags\": \"-march=rv64gc -mtune=sifive-7-series\"\n }\n ]\n }\n }\n },\n \"feature_aliases\": {\n \"sse3\": {\n \"reason\": \"ssse3 is a superset of sse3 and might be the only one listed\",\n \"any_of\": [\n \"ssse3\"\n ]\n },\n \"avx512\": {\n \"reason\": \"avx512 indicates generic support for any of the avx512 instruction sets\",\n \"any_of\": [\n \"avx512f\",\n \"avx512vl\",\n \"avx512bw\",\n \"avx512dq\",\n \"avx512cd\"\n ]\n },\n \"altivec\": {\n \"reason\": \"altivec is supported by Power PC architectures, but might not be listed in features\",\n \"families\": [\n \"ppc64le\",\n \"ppc64\"\n ]\n },\n \"vsx\": {\n \"reason\": \"VSX alitvec extensions are supported by PowerISA from v2.06 (Power7+), but might not be listed in features\",\n \"families\": [\n \"ppc64le\",\n \"ppc64\"\n ]\n },\n \"fma\": {\n \"reason\": \"FMA has been supported by PowerISA since Power1, but might not be listed in features\",\n \"families\": [\n \"ppc64le\",\n \"ppc64\"\n ]\n },\n \"sse4.1\": {\n \"reason\": \"permits to refer to sse4_1 also as sse4.1\",\n \"any_of\": [\n \"sse4_1\"\n ]\n },\n \"sse4.2\": {\n \"reason\": \"permits to refer to sse4_2 also as sse4.2\",\n \"any_of\": [\n \"sse4_2\"\n ]\n },\n \"neon\": {\n \"reason\": \"NEON is required in all standard ARMv8 implementations\",\n \"families\": [\n \"aarch64\"\n ]\n }\n },\n \"conversions\": {\n \"description\": \"Conversions that map some platform specific values to canonical values\",\n \"arm_vendors\": {\n \"0x41\": \"ARM\",\n \"0x42\": \"Broadcom\",\n \"0x43\": \"Cavium\",\n \"0x44\": \"DEC\",\n \"0x46\": \"Fujitsu\",\n \"0x48\": \"HiSilicon\",\n \"0x49\": \"Infineon Technologies AG\",\n \"0x4d\": \"Motorola\",\n \"0x4e\": \"Nvidia\",\n \"0x50\": \"APM\",\n \"0x51\": \"Qualcomm\",\n \"0x53\": \"Samsung\",\n \"0x56\": \"Marvell\",\n \"0x61\": \"Apple\",\n \"0x66\": \"Faraday\",\n \"0x68\": \"HXT\",\n \"0x69\": \"Intel\"\n },\n \"darwin_flags\": {\n \"sse4.1\": \"sse4_1\",\n \"sse4.2\": \"sse4_2\",\n \"avx1.0\": \"avx\",\n \"lahf\": \"lahf_lm\",\n \"sha\": \"sha_ni\",\n \"popcnt lzcnt\": \"abm\",\n \"clfsopt\": \"clflushopt\",\n \"xsave\": \"xsavec xsaveopt\"\n }\n }\n}\n" }, { "path": "lib/spack/spack/vendor/archspec/json/cpu/microarchitectures_schema.json", "content": "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Schema for microarchitecture definitions and feature aliases\",\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"microarchitectures\": {\n \"type\": \"object\",\n \"patternProperties\": {\n \"([\\\\w]*)\": {\n \"type\": \"object\",\n \"properties\": {\n \"from\": {\n \"$comment\": \"More than one parent\",\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"vendor\": {\n \"type\": \"string\"\n },\n \"features\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"compilers\": {\n \"type\": \"object\",\n \"patternProperties\": {\n \"([\\\\w]*)\": {\n \"$comment\": \"Permit multiple entries since compilers change options across versions\",\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"versions\": {\n \"type\": \"string\"\n },\n \"name\": {\n \"type\": \"string\"\n },\n \"flags\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"versions\",\n \"flags\"\n ]\n }\n }\n }\n },\n \"cpupart\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"from\",\n \"vendor\",\n \"features\"\n ]\n }\n }\n },\n \"feature_aliases\": {\n \"type\": \"object\",\n \"patternProperties\": {\n \"([\\\\w]*)\": {\n \"type\": \"object\",\n \"properties\": {\n \"reason\": {\n \"$comment\": \"Comment containing the reason why an alias is there\",\n \"type\": \"string\"\n },\n \"any_of\": {\n \"$comment\": \"The alias is true if any of the items is a feature of the target\",\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"families\": {\n \"$comment\": \"The alias is true if the family of the target is in this list\",\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n }\n },\n \"additionalProperties\": false\n }\n }\n },\n \"conversions\": {\n \"type\": \"object\",\n \"properties\": {\n \"description\": {\n \"type\": \"string\"\n },\n \"arm_vendors\": {\n \"type\": \"object\"\n },\n \"darwin_flags\": {\n \"type\": \"object\"\n }\n },\n \"additionalProperties\": false\n }\n }\n}\n" }, { "path": "lib/spack/spack/vendor/archspec/vendor/cpuid/LICENSE", "content": "The MIT License (MIT)\n\nCopyright (c) 2014 Anders Høst\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software is furnished to do so,\nsubject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\nFOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\nCOPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\nIN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\nCONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" }, { "path": "lib/spack/spack/vendor/archspec/vendor/cpuid/README.md", "content": "cpuid.py\n========\n\nNow, this is silly!\n\nPure Python library for accessing information about x86 processors\nby querying the [CPUID](http://en.wikipedia.org/wiki/CPUID)\ninstruction. Well, not exactly pure Python...\n\nIt works by allocating a small piece of virtual memory, copying\na raw x86 function to that memory, giving the memory execute\npermissions and then calling the memory as a function. The injected\nfunction executes the CPUID instruction and copies the result back\nto a ctypes.Structure where is can be read by Python.\n\nIt should work fine on both 32 and 64 bit versions of Windows and Linux\nrunning x86 processors. Apple OS X and other BSD systems should also work,\nnot tested though...\n\n\nWhy?\n----\nFor poops and giggles. Plus, having access to a low-level feature\nwithout having to compile a C wrapper is pretty neat.\n\n\nExamples\n--------\nGetting info with eax=0:\n\n import cpuid\n\n q = cpuid.CPUID()\n eax, ebx, ecx, edx = q(0)\n\nRunning the files:\n\n $ python example.py \n Vendor ID : GenuineIntel\n CPU name : Intel(R) Xeon(R) CPU W3550 @ 3.07GHz\n \n Vector instructions supported:\n SSE : Yes\n SSE2 : Yes\n SSE3 : Yes\n SSSE3 : Yes\n SSE4.1 : Yes\n SSE4.2 : Yes\n SSE4a : --\n AVX : --\n AVX2 : --\n \n $ python cpuid.py\n CPUID A B C D\n 00000000 0000000b 756e6547 6c65746e 49656e69\n 00000001 000106a5 00100800 009ce3bd bfebfbff\n 00000002 55035a01 00f0b2e4 00000000 09ca212c\n 00000003 00000000 00000000 00000000 00000000\n 00000004 00000000 00000000 00000000 00000000\n 00000005 00000040 00000040 00000003 00001120\n 00000006 00000003 00000002 00000001 00000000\n 00000007 00000000 00000000 00000000 00000000\n 00000008 00000000 00000000 00000000 00000000\n 00000009 00000000 00000000 00000000 00000000\n 0000000a 07300403 00000044 00000000 00000603\n 0000000b 00000000 00000000 00000095 00000000\n 80000000 80000008 00000000 00000000 00000000\n 80000001 00000000 00000000 00000001 28100800\n 80000002 65746e49 2952286c 6f655820 2952286e\n 80000003 55504320 20202020 20202020 57202020\n 80000004 30353533 20402020 37302e33 007a4847\n 80000005 00000000 00000000 00000000 00000000\n 80000006 00000000 00000000 01006040 00000000\n 80000007 00000000 00000000 00000000 00000100\n 80000008 00003024 00000000 00000000 00000000\n\n" }, { "path": "lib/spack/spack/vendor/archspec/vendor/cpuid/cpuid.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (c) 2024 Anders Høst\n#\n\nfrom __future__ import print_function\n\nimport platform\nimport os\nimport ctypes\nfrom ctypes import c_uint32, c_long, c_ulong, c_size_t, c_void_p, POINTER, CFUNCTYPE\n\n# Posix x86_64:\n# Three first call registers : RDI, RSI, RDX\n# Volatile registers : RAX, RCX, RDX, RSI, RDI, R8-11\n\n# Windows x86_64:\n# Three first call registers : RCX, RDX, R8\n# Volatile registers : RAX, RCX, RDX, R8-11\n\n# cdecl 32 bit:\n# Three first call registers : Stack (%esp)\n# Volatile registers : EAX, ECX, EDX\n\n_POSIX_64_OPC = [\n 0x53, # push %rbx\n 0x89, 0xf0, # mov %esi,%eax\n 0x89, 0xd1, # mov %edx,%ecx\n 0x0f, 0xa2, # cpuid\n 0x89, 0x07, # mov %eax,(%rdi)\n 0x89, 0x5f, 0x04, # mov %ebx,0x4(%rdi)\n 0x89, 0x4f, 0x08, # mov %ecx,0x8(%rdi)\n 0x89, 0x57, 0x0c, # mov %edx,0xc(%rdi)\n 0x5b, # pop %rbx\n 0xc3 # retq\n]\n\n_WINDOWS_64_OPC = [\n 0x53, # push %rbx\n 0x89, 0xd0, # mov %edx,%eax\n 0x49, 0x89, 0xc9, # mov %rcx,%r9\n 0x44, 0x89, 0xc1, # mov %r8d,%ecx\n 0x0f, 0xa2, # cpuid\n 0x41, 0x89, 0x01, # mov %eax,(%r9)\n 0x41, 0x89, 0x59, 0x04, # mov %ebx,0x4(%r9)\n 0x41, 0x89, 0x49, 0x08, # mov %ecx,0x8(%r9)\n 0x41, 0x89, 0x51, 0x0c, # mov %edx,0xc(%r9)\n 0x5b, # pop %rbx\n 0xc3 # retq\n]\n\n_CDECL_32_OPC = [\n 0x53, # push %ebx\n 0x57, # push %edi\n 0x8b, 0x7c, 0x24, 0x0c, # mov 0xc(%esp),%edi\n 0x8b, 0x44, 0x24, 0x10, # mov 0x10(%esp),%eax\n 0x8b, 0x4c, 0x24, 0x14, # mov 0x14(%esp),%ecx\n 0x0f, 0xa2, # cpuid\n 0x89, 0x07, # mov %eax,(%edi)\n 0x89, 0x5f, 0x04, # mov %ebx,0x4(%edi)\n 0x89, 0x4f, 0x08, # mov %ecx,0x8(%edi)\n 0x89, 0x57, 0x0c, # mov %edx,0xc(%edi)\n 0x5f, # pop %edi\n 0x5b, # pop %ebx\n 0xc3 # ret\n]\n\nis_windows = os.name == \"nt\"\nis_64bit = ctypes.sizeof(ctypes.c_voidp) == 8\n\n\nclass CPUID_struct(ctypes.Structure):\n _register_names = (\"eax\", \"ebx\", \"ecx\", \"edx\")\n _fields_ = [(r, c_uint32) for r in _register_names]\n\n def __getitem__(self, item):\n if item not in self._register_names:\n raise KeyError(item)\n return getattr(self, item)\n\n def __repr__(self):\n return \"eax=0x{:x}, ebx=0x{:x}, ecx=0x{:x}, edx=0x{:x}\".format(self.eax, self.ebx, self.ecx, self.edx)\n\n\nclass CPUID(object):\n def __init__(self):\n if platform.machine() not in (\"AMD64\", \"x86_64\", \"x86\", \"i686\"):\n raise SystemError(\"Only available for x86\")\n\n if is_windows:\n if is_64bit:\n # VirtualAlloc seems to fail under some weird\n # circumstances when ctypes.windll.kernel32 is\n # used under 64 bit Python. CDLL fixes this.\n self.win = ctypes.CDLL(\"kernel32.dll\")\n opc = _WINDOWS_64_OPC\n else:\n # Here ctypes.windll.kernel32 is needed to get the\n # right DLL. Otherwise it will fail when running\n # 32 bit Python on 64 bit Windows.\n self.win = ctypes.windll.kernel32\n opc = _CDECL_32_OPC\n else:\n opc = _POSIX_64_OPC if is_64bit else _CDECL_32_OPC\n\n size = len(opc)\n code = (ctypes.c_ubyte * size)(*opc)\n\n if is_windows:\n self.win.VirtualAlloc.restype = c_void_p\n self.win.VirtualAlloc.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_ulong, ctypes.c_ulong]\n self.addr = self.win.VirtualAlloc(None, size, 0x1000, 0x40)\n if not self.addr:\n raise MemoryError(\"Could not allocate RWX memory\")\n ctypes.memmove(self.addr, code, size)\n else:\n from mmap import (\n mmap,\n MAP_PRIVATE,\n MAP_ANONYMOUS,\n PROT_WRITE,\n PROT_READ,\n PROT_EXEC,\n )\n self.mm = mmap(\n -1,\n size,\n flags=MAP_PRIVATE | MAP_ANONYMOUS,\n prot=PROT_WRITE | PROT_READ | PROT_EXEC,\n )\n self.mm.write(code)\n self.addr = ctypes.addressof(ctypes.c_int.from_buffer(self.mm))\n\n func_type = CFUNCTYPE(None, POINTER(CPUID_struct), c_uint32, c_uint32)\n self.func_ptr = func_type(self.addr)\n\n def __call__(self, eax, ecx=0):\n struct = self.registers_for(eax=eax, ecx=ecx)\n return struct.eax, struct.ebx, struct.ecx, struct.edx\n\n def registers_for(self, eax, ecx=0):\n \"\"\"Calls cpuid with eax and ecx set as the input arguments, and returns a structure\n containing eax, ebx, ecx, and edx.\n \"\"\"\n struct = CPUID_struct()\n self.func_ptr(struct, eax, ecx)\n return struct\n\n def __del__(self):\n if is_windows:\n self.win.VirtualFree.restype = c_long\n self.win.VirtualFree.argtypes = [c_void_p, c_size_t, c_ulong]\n self.win.VirtualFree(self.addr, 0, 0x8000)\n else:\n self.mm.close()\n\n\n\nif __name__ == \"__main__\":\n def valid_inputs():\n cpuid = CPUID()\n for eax in (0x0, 0x80000000):\n highest, _, _, _ = cpuid(eax)\n while eax <= highest:\n regs = cpuid(eax)\n yield (eax, regs)\n eax += 1\n\n\n print(\" \".join(x.ljust(8) for x in (\"CPUID\", \"A\", \"B\", \"C\", \"D\")).strip())\n for eax, regs in valid_inputs():\n print(\"%08x\" % eax, \" \".join(\"%08x\" % reg for reg in regs))\n" }, { "path": "lib/spack/spack/vendor/archspec/vendor/cpuid/example.py", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright (c) 2024 Anders Høst\n#\n\nfrom __future__ import print_function\n\nimport struct\nimport cpuid\n\n\ndef cpu_vendor(cpu):\n _, b, c, d = cpu(0)\n return struct.pack(\"III\", b, d, c).decode(\"utf-8\")\n\n\ndef cpu_name(cpu):\n name = \"\".join((struct.pack(\"IIII\", *cpu(0x80000000 + i)).decode(\"utf-8\")\n for i in range(2, 5)))\n\n return name.split('\\x00', 1)[0]\n\n\ndef is_set(cpu, leaf, subleaf, reg_idx, bit):\n \"\"\"\n @param {leaf} %eax\n @param {sublead} %ecx, 0 in most cases\n @param {reg_idx} idx of [%eax, %ebx, %ecx, %edx], 0-based\n @param {bit} bit of reg selected by {reg_idx}, 0-based\n \"\"\"\n\n regs = cpu(leaf, subleaf)\n\n if (1 << bit) & regs[reg_idx]:\n return \"Yes\"\n else:\n return \"--\"\n\n\nif __name__ == \"__main__\":\n cpu = cpuid.CPUID()\n\n print(\"Vendor ID : %s\" % cpu_vendor(cpu))\n print(\"CPU name : %s\" % cpu_name(cpu))\n print()\n print(\"Vector instructions supported:\")\n print(\"SSE : %s\" % is_set(cpu, 1, 0, 3, 25))\n print(\"SSE2 : %s\" % is_set(cpu, 1, 0, 3, 26))\n print(\"SSE3 : %s\" % is_set(cpu, 1, 0, 2, 0))\n print(\"SSSE3 : %s\" % is_set(cpu, 1, 0, 2, 9))\n print(\"SSE4.1 : %s\" % is_set(cpu, 1, 0, 2, 19))\n print(\"SSE4.2 : %s\" % is_set(cpu, 1, 0, 2, 20))\n print(\"SSE4a : %s\" % is_set(cpu, 0x80000001, 0, 2, 6))\n print(\"AVX : %s\" % is_set(cpu, 1, 0, 2, 28))\n print(\"AVX2 : %s\" % is_set(cpu, 7, 0, 1, 5))\n print(\"BMI1 : %s\" % is_set(cpu, 7, 0, 1, 3))\n print(\"BMI2 : %s\" % is_set(cpu, 7, 0, 1, 8))\n # Intel RDT CMT/MBM\n print(\"L3 Monitoring : %s\" % is_set(cpu, 0xf, 0, 3, 1))\n print(\"L3 Occupancy : %s\" % is_set(cpu, 0xf, 1, 3, 0))\n print(\"L3 Total BW : %s\" % is_set(cpu, 0xf, 1, 3, 1))\n print(\"L3 Local BW : %s\" % is_set(cpu, 0xf, 1, 3, 2))\n" }, { "path": "lib/spack/spack/vendor/attr/__init__.py", "content": "# SPDX-License-Identifier: MIT\n\n\nimport sys\n\nfrom functools import partial\n\nfrom . import converters, exceptions, filters, setters, validators\nfrom ._cmp import cmp_using\nfrom ._config import get_run_validators, set_run_validators\nfrom ._funcs import asdict, assoc, astuple, evolve, has, resolve_types\nfrom ._make import (\n NOTHING,\n Attribute,\n Factory,\n attrib,\n attrs,\n fields,\n fields_dict,\n make_class,\n validate,\n)\nfrom ._version_info import VersionInfo\n\n\n__version__ = \"22.1.0\"\n__version_info__ = VersionInfo._from_version_string(__version__)\n\n__title__ = \"attrs\"\n__description__ = \"Classes Without Boilerplate\"\n__url__ = \"https://www.attrs.org/\"\n__uri__ = __url__\n__doc__ = __description__ + \" <\" + __uri__ + \">\"\n\n__author__ = \"Hynek Schlawack\"\n__email__ = \"hs@ox.cx\"\n\n__license__ = \"MIT\"\n__copyright__ = \"Copyright (c) 2015 Hynek Schlawack\"\n\n\ns = attributes = attrs\nib = attr = attrib\ndataclass = partial(attrs, auto_attribs=True) # happy Easter ;)\n\n__all__ = [\n \"Attribute\",\n \"Factory\",\n \"NOTHING\",\n \"asdict\",\n \"assoc\",\n \"astuple\",\n \"attr\",\n \"attrib\",\n \"attributes\",\n \"attrs\",\n \"cmp_using\",\n \"converters\",\n \"evolve\",\n \"exceptions\",\n \"fields\",\n \"fields_dict\",\n \"filters\",\n \"get_run_validators\",\n \"has\",\n \"ib\",\n \"make_class\",\n \"resolve_types\",\n \"s\",\n \"set_run_validators\",\n \"setters\",\n \"validate\",\n \"validators\",\n]\n\nif sys.version_info[:2] >= (3, 6):\n from ._next_gen import define, field, frozen, mutable # noqa: F401\n\n __all__.extend((\"define\", \"field\", \"frozen\", \"mutable\"))\n" }, { "path": "lib/spack/spack/vendor/attr/__init__.pyi", "content": "import sys\n\nfrom typing import (\n Any,\n Callable,\n ClassVar,\n Dict,\n Generic,\n List,\n Mapping,\n Optional,\n Protocol,\n Sequence,\n Tuple,\n Type,\n TypeVar,\n Union,\n overload,\n)\n\n# `import X as X` is required to make these public\nfrom . import converters as converters\nfrom . import exceptions as exceptions\nfrom . import filters as filters\nfrom . import setters as setters\nfrom . import validators as validators\nfrom ._cmp import cmp_using as cmp_using\nfrom ._version_info import VersionInfo\n\n__version__: str\n__version_info__: VersionInfo\n__title__: str\n__description__: str\n__url__: str\n__uri__: str\n__author__: str\n__email__: str\n__license__: str\n__copyright__: str\n\n_T = TypeVar(\"_T\")\n_C = TypeVar(\"_C\", bound=type)\n\n_EqOrderType = Union[bool, Callable[[Any], Any]]\n_ValidatorType = Callable[[Any, Attribute[_T], _T], Any]\n_ConverterType = Callable[[Any], Any]\n_FilterType = Callable[[Attribute[_T], _T], bool]\n_ReprType = Callable[[Any], str]\n_ReprArgType = Union[bool, _ReprType]\n_OnSetAttrType = Callable[[Any, Attribute[Any], Any], Any]\n_OnSetAttrArgType = Union[\n _OnSetAttrType, List[_OnSetAttrType], setters._NoOpType\n]\n_FieldTransformer = Callable[\n [type, List[Attribute[Any]]], List[Attribute[Any]]\n]\n# FIXME: in reality, if multiple validators are passed they must be in a list\n# or tuple, but those are invariant and so would prevent subtypes of\n# _ValidatorType from working when passed in a list or tuple.\n_ValidatorArgType = Union[_ValidatorType[_T], Sequence[_ValidatorType[_T]]]\n\n# A protocol to be able to statically accept an attrs class.\nclass AttrsInstance(Protocol):\n __attrs_attrs__: ClassVar[Any]\n\n# _make --\n\nNOTHING: object\n\n# NOTE: Factory lies about its return type to make this possible:\n# `x: List[int] # = Factory(list)`\n# Work around mypy issue #4554 in the common case by using an overload.\nif sys.version_info >= (3, 8):\n from typing import Literal\n @overload\n def Factory(factory: Callable[[], _T]) -> _T: ...\n @overload\n def Factory(\n factory: Callable[[Any], _T],\n takes_self: Literal[True],\n ) -> _T: ...\n @overload\n def Factory(\n factory: Callable[[], _T],\n takes_self: Literal[False],\n ) -> _T: ...\n\nelse:\n @overload\n def Factory(factory: Callable[[], _T]) -> _T: ...\n @overload\n def Factory(\n factory: Union[Callable[[Any], _T], Callable[[], _T]],\n takes_self: bool = ...,\n ) -> _T: ...\n\n# Static type inference support via __dataclass_transform__ implemented as per:\n# https://github.com/microsoft/pyright/blob/1.1.135/specs/dataclass_transforms.md\n# This annotation must be applied to all overloads of \"define\" and \"attrs\"\n#\n# NOTE: This is a typing construct and does not exist at runtime. Extensions\n# wrapping attrs decorators should declare a separate __dataclass_transform__\n# signature in the extension module using the specification linked above to\n# provide pyright support.\ndef __dataclass_transform__(\n *,\n eq_default: bool = True,\n order_default: bool = False,\n kw_only_default: bool = False,\n field_descriptors: Tuple[Union[type, Callable[..., Any]], ...] = (()),\n) -> Callable[[_T], _T]: ...\n\nclass Attribute(Generic[_T]):\n name: str\n default: Optional[_T]\n validator: Optional[_ValidatorType[_T]]\n repr: _ReprArgType\n cmp: _EqOrderType\n eq: _EqOrderType\n order: _EqOrderType\n hash: Optional[bool]\n init: bool\n converter: Optional[_ConverterType]\n metadata: Dict[Any, Any]\n type: Optional[Type[_T]]\n kw_only: bool\n on_setattr: _OnSetAttrType\n def evolve(self, **changes: Any) -> \"Attribute[Any]\": ...\n\n# NOTE: We had several choices for the annotation to use for type arg:\n# 1) Type[_T]\n# - Pros: Handles simple cases correctly\n# - Cons: Might produce less informative errors in the case of conflicting\n# TypeVars e.g. `attr.ib(default='bad', type=int)`\n# 2) Callable[..., _T]\n# - Pros: Better error messages than #1 for conflicting TypeVars\n# - Cons: Terrible error messages for validator checks.\n# e.g. attr.ib(type=int, validator=validate_str)\n# -> error: Cannot infer function type argument\n# 3) type (and do all of the work in the mypy plugin)\n# - Pros: Simple here, and we could customize the plugin with our own errors.\n# - Cons: Would need to write mypy plugin code to handle all the cases.\n# We chose option #1.\n\n# `attr` lies about its return type to make the following possible:\n# attr() -> Any\n# attr(8) -> int\n# attr(validator=) -> Whatever the callable expects.\n# This makes this type of assignments possible:\n# x: int = attr(8)\n#\n# This form catches explicit None or no default but with no other arguments\n# returns Any.\n@overload\ndef attrib(\n default: None = ...,\n validator: None = ...,\n repr: _ReprArgType = ...,\n cmp: Optional[_EqOrderType] = ...,\n hash: Optional[bool] = ...,\n init: bool = ...,\n metadata: Optional[Mapping[Any, Any]] = ...,\n type: None = ...,\n converter: None = ...,\n factory: None = ...,\n kw_only: bool = ...,\n eq: Optional[_EqOrderType] = ...,\n order: Optional[_EqOrderType] = ...,\n on_setattr: Optional[_OnSetAttrArgType] = ...,\n) -> Any: ...\n\n# This form catches an explicit None or no default and infers the type from the\n# other arguments.\n@overload\ndef attrib(\n default: None = ...,\n validator: Optional[_ValidatorArgType[_T]] = ...,\n repr: _ReprArgType = ...,\n cmp: Optional[_EqOrderType] = ...,\n hash: Optional[bool] = ...,\n init: bool = ...,\n metadata: Optional[Mapping[Any, Any]] = ...,\n type: Optional[Type[_T]] = ...,\n converter: Optional[_ConverterType] = ...,\n factory: Optional[Callable[[], _T]] = ...,\n kw_only: bool = ...,\n eq: Optional[_EqOrderType] = ...,\n order: Optional[_EqOrderType] = ...,\n on_setattr: Optional[_OnSetAttrArgType] = ...,\n) -> _T: ...\n\n# This form catches an explicit default argument.\n@overload\ndef attrib(\n default: _T,\n validator: Optional[_ValidatorArgType[_T]] = ...,\n repr: _ReprArgType = ...,\n cmp: Optional[_EqOrderType] = ...,\n hash: Optional[bool] = ...,\n init: bool = ...,\n metadata: Optional[Mapping[Any, Any]] = ...,\n type: Optional[Type[_T]] = ...,\n converter: Optional[_ConverterType] = ...,\n factory: Optional[Callable[[], _T]] = ...,\n kw_only: bool = ...,\n eq: Optional[_EqOrderType] = ...,\n order: Optional[_EqOrderType] = ...,\n on_setattr: Optional[_OnSetAttrArgType] = ...,\n) -> _T: ...\n\n# This form covers type=non-Type: e.g. forward references (str), Any\n@overload\ndef attrib(\n default: Optional[_T] = ...,\n validator: Optional[_ValidatorArgType[_T]] = ...,\n repr: _ReprArgType = ...,\n cmp: Optional[_EqOrderType] = ...,\n hash: Optional[bool] = ...,\n init: bool = ...,\n metadata: Optional[Mapping[Any, Any]] = ...,\n type: object = ...,\n converter: Optional[_ConverterType] = ...,\n factory: Optional[Callable[[], _T]] = ...,\n kw_only: bool = ...,\n eq: Optional[_EqOrderType] = ...,\n order: Optional[_EqOrderType] = ...,\n on_setattr: Optional[_OnSetAttrArgType] = ...,\n) -> Any: ...\n@overload\ndef field(\n *,\n default: None = ...,\n validator: None = ...,\n repr: _ReprArgType = ...,\n hash: Optional[bool] = ...,\n init: bool = ...,\n metadata: Optional[Mapping[Any, Any]] = ...,\n converter: None = ...,\n factory: None = ...,\n kw_only: bool = ...,\n eq: Optional[bool] = ...,\n order: Optional[bool] = ...,\n on_setattr: Optional[_OnSetAttrArgType] = ...,\n) -> Any: ...\n\n# This form catches an explicit None or no default and infers the type from the\n# other arguments.\n@overload\ndef field(\n *,\n default: None = ...,\n validator: Optional[_ValidatorArgType[_T]] = ...,\n repr: _ReprArgType = ...,\n hash: Optional[bool] = ...,\n init: bool = ...,\n metadata: Optional[Mapping[Any, Any]] = ...,\n converter: Optional[_ConverterType] = ...,\n factory: Optional[Callable[[], _T]] = ...,\n kw_only: bool = ...,\n eq: Optional[_EqOrderType] = ...,\n order: Optional[_EqOrderType] = ...,\n on_setattr: Optional[_OnSetAttrArgType] = ...,\n) -> _T: ...\n\n# This form catches an explicit default argument.\n@overload\ndef field(\n *,\n default: _T,\n validator: Optional[_ValidatorArgType[_T]] = ...,\n repr: _ReprArgType = ...,\n hash: Optional[bool] = ...,\n init: bool = ...,\n metadata: Optional[Mapping[Any, Any]] = ...,\n converter: Optional[_ConverterType] = ...,\n factory: Optional[Callable[[], _T]] = ...,\n kw_only: bool = ...,\n eq: Optional[_EqOrderType] = ...,\n order: Optional[_EqOrderType] = ...,\n on_setattr: Optional[_OnSetAttrArgType] = ...,\n) -> _T: ...\n\n# This form covers type=non-Type: e.g. forward references (str), Any\n@overload\ndef field(\n *,\n default: Optional[_T] = ...,\n validator: Optional[_ValidatorArgType[_T]] = ...,\n repr: _ReprArgType = ...,\n hash: Optional[bool] = ...,\n init: bool = ...,\n metadata: Optional[Mapping[Any, Any]] = ...,\n converter: Optional[_ConverterType] = ...,\n factory: Optional[Callable[[], _T]] = ...,\n kw_only: bool = ...,\n eq: Optional[_EqOrderType] = ...,\n order: Optional[_EqOrderType] = ...,\n on_setattr: Optional[_OnSetAttrArgType] = ...,\n) -> Any: ...\n@overload\n@__dataclass_transform__(order_default=True, field_descriptors=(attrib, field))\ndef attrs(\n maybe_cls: _C,\n these: Optional[Dict[str, Any]] = ...,\n repr_ns: Optional[str] = ...,\n repr: bool = ...,\n cmp: Optional[_EqOrderType] = ...,\n hash: Optional[bool] = ...,\n init: bool = ...,\n slots: bool = ...,\n frozen: bool = ...,\n weakref_slot: bool = ...,\n str: bool = ...,\n auto_attribs: bool = ...,\n kw_only: bool = ...,\n cache_hash: bool = ...,\n auto_exc: bool = ...,\n eq: Optional[_EqOrderType] = ...,\n order: Optional[_EqOrderType] = ...,\n auto_detect: bool = ...,\n collect_by_mro: bool = ...,\n getstate_setstate: Optional[bool] = ...,\n on_setattr: Optional[_OnSetAttrArgType] = ...,\n field_transformer: Optional[_FieldTransformer] = ...,\n match_args: bool = ...,\n) -> _C: ...\n@overload\n@__dataclass_transform__(order_default=True, field_descriptors=(attrib, field))\ndef attrs(\n maybe_cls: None = ...,\n these: Optional[Dict[str, Any]] = ...,\n repr_ns: Optional[str] = ...,\n repr: bool = ...,\n cmp: Optional[_EqOrderType] = ...,\n hash: Optional[bool] = ...,\n init: bool = ...,\n slots: bool = ...,\n frozen: bool = ...,\n weakref_slot: bool = ...,\n str: bool = ...,\n auto_attribs: bool = ...,\n kw_only: bool = ...,\n cache_hash: bool = ...,\n auto_exc: bool = ...,\n eq: Optional[_EqOrderType] = ...,\n order: Optional[_EqOrderType] = ...,\n auto_detect: bool = ...,\n collect_by_mro: bool = ...,\n getstate_setstate: Optional[bool] = ...,\n on_setattr: Optional[_OnSetAttrArgType] = ...,\n field_transformer: Optional[_FieldTransformer] = ...,\n match_args: bool = ...,\n) -> Callable[[_C], _C]: ...\n@overload\n@__dataclass_transform__(field_descriptors=(attrib, field))\ndef define(\n maybe_cls: _C,\n *,\n these: Optional[Dict[str, Any]] = ...,\n repr: bool = ...,\n hash: Optional[bool] = ...,\n init: bool = ...,\n slots: bool = ...,\n frozen: bool = ...,\n weakref_slot: bool = ...,\n str: bool = ...,\n auto_attribs: bool = ...,\n kw_only: bool = ...,\n cache_hash: bool = ...,\n auto_exc: bool = ...,\n eq: Optional[bool] = ...,\n order: Optional[bool] = ...,\n auto_detect: bool = ...,\n getstate_setstate: Optional[bool] = ...,\n on_setattr: Optional[_OnSetAttrArgType] = ...,\n field_transformer: Optional[_FieldTransformer] = ...,\n match_args: bool = ...,\n) -> _C: ...\n@overload\n@__dataclass_transform__(field_descriptors=(attrib, field))\ndef define(\n maybe_cls: None = ...,\n *,\n these: Optional[Dict[str, Any]] = ...,\n repr: bool = ...,\n hash: Optional[bool] = ...,\n init: bool = ...,\n slots: bool = ...,\n frozen: bool = ...,\n weakref_slot: bool = ...,\n str: bool = ...,\n auto_attribs: bool = ...,\n kw_only: bool = ...,\n cache_hash: bool = ...,\n auto_exc: bool = ...,\n eq: Optional[bool] = ...,\n order: Optional[bool] = ...,\n auto_detect: bool = ...,\n getstate_setstate: Optional[bool] = ...,\n on_setattr: Optional[_OnSetAttrArgType] = ...,\n field_transformer: Optional[_FieldTransformer] = ...,\n match_args: bool = ...,\n) -> Callable[[_C], _C]: ...\n\nmutable = define\nfrozen = define # they differ only in their defaults\n\ndef fields(cls: Type[AttrsInstance]) -> Any: ...\ndef fields_dict(cls: Type[AttrsInstance]) -> Dict[str, Attribute[Any]]: ...\ndef validate(inst: AttrsInstance) -> None: ...\ndef resolve_types(\n cls: _C,\n globalns: Optional[Dict[str, Any]] = ...,\n localns: Optional[Dict[str, Any]] = ...,\n attribs: Optional[List[Attribute[Any]]] = ...,\n) -> _C: ...\n\n# TODO: add support for returning a proper attrs class from the mypy plugin\n# we use Any instead of _CountingAttr so that e.g. `make_class('Foo',\n# [attr.ib()])` is valid\ndef make_class(\n name: str,\n attrs: Union[List[str], Tuple[str, ...], Dict[str, Any]],\n bases: Tuple[type, ...] = ...,\n repr_ns: Optional[str] = ...,\n repr: bool = ...,\n cmp: Optional[_EqOrderType] = ...,\n hash: Optional[bool] = ...,\n init: bool = ...,\n slots: bool = ...,\n frozen: bool = ...,\n weakref_slot: bool = ...,\n str: bool = ...,\n auto_attribs: bool = ...,\n kw_only: bool = ...,\n cache_hash: bool = ...,\n auto_exc: bool = ...,\n eq: Optional[_EqOrderType] = ...,\n order: Optional[_EqOrderType] = ...,\n collect_by_mro: bool = ...,\n on_setattr: Optional[_OnSetAttrArgType] = ...,\n field_transformer: Optional[_FieldTransformer] = ...,\n) -> type: ...\n\n# _funcs --\n\n# TODO: add support for returning TypedDict from the mypy plugin\n# FIXME: asdict/astuple do not honor their factory args. Waiting on one of\n# these:\n# https://github.com/python/mypy/issues/4236\n# https://github.com/python/typing/issues/253\n# XXX: remember to fix attrs.asdict/astuple too!\ndef asdict(\n inst: AttrsInstance,\n recurse: bool = ...,\n filter: Optional[_FilterType[Any]] = ...,\n dict_factory: Type[Mapping[Any, Any]] = ...,\n retain_collection_types: bool = ...,\n value_serializer: Optional[\n Callable[[type, Attribute[Any], Any], Any]\n ] = ...,\n tuple_keys: Optional[bool] = ...,\n) -> Dict[str, Any]: ...\n\n# TODO: add support for returning NamedTuple from the mypy plugin\ndef astuple(\n inst: AttrsInstance,\n recurse: bool = ...,\n filter: Optional[_FilterType[Any]] = ...,\n tuple_factory: Type[Sequence[Any]] = ...,\n retain_collection_types: bool = ...,\n) -> Tuple[Any, ...]: ...\ndef has(cls: type) -> bool: ...\ndef assoc(inst: _T, **changes: Any) -> _T: ...\ndef evolve(inst: _T, **changes: Any) -> _T: ...\n\n# _config --\n\ndef set_run_validators(run: bool) -> None: ...\ndef get_run_validators() -> bool: ...\n\n# aliases --\n\ns = attributes = attrs\nib = attr = attrib\ndataclass = attrs # Technically, partial(attrs, auto_attribs=True) ;)\n" }, { "path": "lib/spack/spack/vendor/attr/_cmp.py", "content": "# SPDX-License-Identifier: MIT\n\n\nimport functools\nimport types\n\nfrom ._make import _make_ne\n\n\n_operation_names = {\"eq\": \"==\", \"lt\": \"<\", \"le\": \"<=\", \"gt\": \">\", \"ge\": \">=\"}\n\n\ndef cmp_using(\n eq=None,\n lt=None,\n le=None,\n gt=None,\n ge=None,\n require_same_type=True,\n class_name=\"Comparable\",\n):\n \"\"\"\n Create a class that can be passed into `attr.ib`'s ``eq``, ``order``, and\n ``cmp`` arguments to customize field comparison.\n\n The resulting class will have a full set of ordering methods if\n at least one of ``{lt, le, gt, ge}`` and ``eq`` are provided.\n\n :param Optional[callable] eq: `callable` used to evaluate equality\n of two objects.\n :param Optional[callable] lt: `callable` used to evaluate whether\n one object is less than another object.\n :param Optional[callable] le: `callable` used to evaluate whether\n one object is less than or equal to another object.\n :param Optional[callable] gt: `callable` used to evaluate whether\n one object is greater than another object.\n :param Optional[callable] ge: `callable` used to evaluate whether\n one object is greater than or equal to another object.\n\n :param bool require_same_type: When `True`, equality and ordering methods\n will return `NotImplemented` if objects are not of the same type.\n\n :param Optional[str] class_name: Name of class. Defaults to 'Comparable'.\n\n See `comparison` for more details.\n\n .. versionadded:: 21.1.0\n \"\"\"\n\n body = {\n \"__slots__\": [\"value\"],\n \"__init__\": _make_init(),\n \"_requirements\": [],\n \"_is_comparable_to\": _is_comparable_to,\n }\n\n # Add operations.\n num_order_functions = 0\n has_eq_function = False\n\n if eq is not None:\n has_eq_function = True\n body[\"__eq__\"] = _make_operator(\"eq\", eq)\n body[\"__ne__\"] = _make_ne()\n\n if lt is not None:\n num_order_functions += 1\n body[\"__lt__\"] = _make_operator(\"lt\", lt)\n\n if le is not None:\n num_order_functions += 1\n body[\"__le__\"] = _make_operator(\"le\", le)\n\n if gt is not None:\n num_order_functions += 1\n body[\"__gt__\"] = _make_operator(\"gt\", gt)\n\n if ge is not None:\n num_order_functions += 1\n body[\"__ge__\"] = _make_operator(\"ge\", ge)\n\n type_ = types.new_class(\n class_name, (object,), {}, lambda ns: ns.update(body)\n )\n\n # Add same type requirement.\n if require_same_type:\n type_._requirements.append(_check_same_type)\n\n # Add total ordering if at least one operation was defined.\n if 0 < num_order_functions < 4:\n if not has_eq_function:\n # functools.total_ordering requires __eq__ to be defined,\n # so raise early error here to keep a nice stack.\n raise ValueError(\n \"eq must be define is order to complete ordering from \"\n \"lt, le, gt, ge.\"\n )\n type_ = functools.total_ordering(type_)\n\n return type_\n\n\ndef _make_init():\n \"\"\"\n Create __init__ method.\n \"\"\"\n\n def __init__(self, value):\n \"\"\"\n Initialize object with *value*.\n \"\"\"\n self.value = value\n\n return __init__\n\n\ndef _make_operator(name, func):\n \"\"\"\n Create operator method.\n \"\"\"\n\n def method(self, other):\n if not self._is_comparable_to(other):\n return NotImplemented\n\n result = func(self.value, other.value)\n if result is NotImplemented:\n return NotImplemented\n\n return result\n\n method.__name__ = \"__%s__\" % (name,)\n method.__doc__ = \"Return a %s b. Computed by attrs.\" % (\n _operation_names[name],\n )\n\n return method\n\n\ndef _is_comparable_to(self, other):\n \"\"\"\n Check whether `other` is comparable to `self`.\n \"\"\"\n for func in self._requirements:\n if not func(self, other):\n return False\n return True\n\n\ndef _check_same_type(self, other):\n \"\"\"\n Return True if *self* and *other* are of the same type, False otherwise.\n \"\"\"\n return other.value.__class__ is self.value.__class__\n" }, { "path": "lib/spack/spack/vendor/attr/_cmp.pyi", "content": "from typing import Any, Callable, Optional, Type\n\n_CompareWithType = Callable[[Any, Any], bool]\n\ndef cmp_using(\n eq: Optional[_CompareWithType],\n lt: Optional[_CompareWithType],\n le: Optional[_CompareWithType],\n gt: Optional[_CompareWithType],\n ge: Optional[_CompareWithType],\n require_same_type: bool,\n class_name: str,\n) -> Type: ...\n" }, { "path": "lib/spack/spack/vendor/attr/_compat.py", "content": "# SPDX-License-Identifier: MIT\n\n\nimport inspect\nimport platform\nimport sys\nimport threading\nimport types\nimport warnings\n\nfrom collections.abc import Mapping, Sequence # noqa\n\n\nPYPY = platform.python_implementation() == \"PyPy\"\nPY36 = sys.version_info[:2] >= (3, 6)\nHAS_F_STRINGS = PY36\nPY310 = sys.version_info[:2] >= (3, 10)\n\n\nif PYPY or PY36:\n ordered_dict = dict\nelse:\n from collections import OrderedDict\n\n ordered_dict = OrderedDict\n\n\ndef just_warn(*args, **kw):\n warnings.warn(\n \"Running interpreter doesn't sufficiently support code object \"\n \"introspection. Some features like bare super() or accessing \"\n \"__class__ will not work with slotted classes.\",\n RuntimeWarning,\n stacklevel=2,\n )\n\n\nclass _AnnotationExtractor:\n \"\"\"\n Extract type annotations from a callable, returning None whenever there\n is none.\n \"\"\"\n\n __slots__ = [\"sig\"]\n\n def __init__(self, callable):\n try:\n self.sig = inspect.signature(callable)\n except (ValueError, TypeError): # inspect failed\n self.sig = None\n\n def get_first_param_type(self):\n \"\"\"\n Return the type annotation of the first argument if it's not empty.\n \"\"\"\n if not self.sig:\n return None\n\n params = list(self.sig.parameters.values())\n if params and params[0].annotation is not inspect.Parameter.empty:\n return params[0].annotation\n\n return None\n\n def get_return_type(self):\n \"\"\"\n Return the return type if it's not empty.\n \"\"\"\n if (\n self.sig\n and self.sig.return_annotation is not inspect.Signature.empty\n ):\n return self.sig.return_annotation\n\n return None\n\n\ndef make_set_closure_cell():\n \"\"\"Return a function of two arguments (cell, value) which sets\n the value stored in the closure cell `cell` to `value`.\n \"\"\"\n # pypy makes this easy. (It also supports the logic below, but\n # why not do the easy/fast thing?)\n if PYPY:\n\n def set_closure_cell(cell, value):\n cell.__setstate__((value,))\n\n return set_closure_cell\n\n # Otherwise gotta do it the hard way.\n\n # Create a function that will set its first cellvar to `value`.\n def set_first_cellvar_to(value):\n x = value\n return\n\n # This function will be eliminated as dead code, but\n # not before its reference to `x` forces `x` to be\n # represented as a closure cell rather than a local.\n def force_x_to_be_a_cell(): # pragma: no cover\n return x\n\n try:\n # Extract the code object and make sure our assumptions about\n # the closure behavior are correct.\n co = set_first_cellvar_to.__code__\n if co.co_cellvars != (\"x\",) or co.co_freevars != ():\n raise AssertionError # pragma: no cover\n\n # Convert this code object to a code object that sets the\n # function's first _freevar_ (not cellvar) to the argument.\n if sys.version_info >= (3, 8):\n\n def set_closure_cell(cell, value):\n cell.cell_contents = value\n\n else:\n args = [co.co_argcount]\n args.append(co.co_kwonlyargcount)\n args.extend(\n [\n co.co_nlocals,\n co.co_stacksize,\n co.co_flags,\n co.co_code,\n co.co_consts,\n co.co_names,\n co.co_varnames,\n co.co_filename,\n co.co_name,\n co.co_firstlineno,\n co.co_lnotab,\n # These two arguments are reversed:\n co.co_cellvars,\n co.co_freevars,\n ]\n )\n set_first_freevar_code = types.CodeType(*args)\n\n def set_closure_cell(cell, value):\n # Create a function using the set_first_freevar_code,\n # whose first closure cell is `cell`. Calling it will\n # change the value of that cell.\n setter = types.FunctionType(\n set_first_freevar_code, {}, \"setter\", (), (cell,)\n )\n # And call it to set the cell.\n setter(value)\n\n # Make sure it works on this interpreter:\n def make_func_with_cell():\n x = None\n\n def func():\n return x # pragma: no cover\n\n return func\n\n cell = make_func_with_cell().__closure__[0]\n set_closure_cell(cell, 100)\n if cell.cell_contents != 100:\n raise AssertionError # pragma: no cover\n\n except Exception:\n return just_warn\n else:\n return set_closure_cell\n\n\nset_closure_cell = make_set_closure_cell()\n\n# Thread-local global to track attrs instances which are already being repr'd.\n# This is needed because there is no other (thread-safe) way to pass info\n# about the instances that are already being repr'd through the call stack\n# in order to ensure we don't perform infinite recursion.\n#\n# For instance, if an instance contains a dict which contains that instance,\n# we need to know that we're already repr'ing the outside instance from within\n# the dict's repr() call.\n#\n# This lives here rather than in _make.py so that the functions in _make.py\n# don't have a direct reference to the thread-local in their globals dict.\n# If they have such a reference, it breaks cloudpickle.\nrepr_context = threading.local()\n" }, { "path": "lib/spack/spack/vendor/attr/_config.py", "content": "# SPDX-License-Identifier: MIT\n\n\n__all__ = [\"set_run_validators\", \"get_run_validators\"]\n\n_run_validators = True\n\n\ndef set_run_validators(run):\n \"\"\"\n Set whether or not validators are run. By default, they are run.\n\n .. deprecated:: 21.3.0 It will not be removed, but it also will not be\n moved to new ``attrs`` namespace. Use `attrs.validators.set_disabled()`\n instead.\n \"\"\"\n if not isinstance(run, bool):\n raise TypeError(\"'run' must be bool.\")\n global _run_validators\n _run_validators = run\n\n\ndef get_run_validators():\n \"\"\"\n Return whether or not validators are run.\n\n .. deprecated:: 21.3.0 It will not be removed, but it also will not be\n moved to new ``attrs`` namespace. Use `attrs.validators.get_disabled()`\n instead.\n \"\"\"\n return _run_validators\n" }, { "path": "lib/spack/spack/vendor/attr/_funcs.py", "content": "# SPDX-License-Identifier: MIT\n\n\nimport copy\n\nfrom ._make import NOTHING, _obj_setattr, fields\nfrom .exceptions import AttrsAttributeNotFoundError\n\n\ndef asdict(\n inst,\n recurse=True,\n filter=None,\n dict_factory=dict,\n retain_collection_types=False,\n value_serializer=None,\n):\n \"\"\"\n Return the ``attrs`` attribute values of *inst* as a dict.\n\n Optionally recurse into other ``attrs``-decorated classes.\n\n :param inst: Instance of an ``attrs``-decorated class.\n :param bool recurse: Recurse into classes that are also\n ``attrs``-decorated.\n :param callable filter: A callable whose return code determines whether an\n attribute or element is included (``True``) or dropped (``False``). Is\n called with the `attrs.Attribute` as the first argument and the\n value as the second argument.\n :param callable dict_factory: A callable to produce dictionaries from. For\n example, to produce ordered dictionaries instead of normal Python\n dictionaries, pass in ``collections.OrderedDict``.\n :param bool retain_collection_types: Do not convert to ``list`` when\n encountering an attribute whose type is ``tuple`` or ``set``. Only\n meaningful if ``recurse`` is ``True``.\n :param Optional[callable] value_serializer: A hook that is called for every\n attribute or dict key/value. It receives the current instance, field\n and value and must return the (updated) value. The hook is run *after*\n the optional *filter* has been applied.\n\n :rtype: return type of *dict_factory*\n\n :raise attr.exceptions.NotAnAttrsClassError: If *cls* is not an ``attrs``\n class.\n\n .. versionadded:: 16.0.0 *dict_factory*\n .. versionadded:: 16.1.0 *retain_collection_types*\n .. versionadded:: 20.3.0 *value_serializer*\n .. versionadded:: 21.3.0 If a dict has a collection for a key, it is\n serialized as a tuple.\n \"\"\"\n attrs = fields(inst.__class__)\n rv = dict_factory()\n for a in attrs:\n v = getattr(inst, a.name)\n if filter is not None and not filter(a, v):\n continue\n\n if value_serializer is not None:\n v = value_serializer(inst, a, v)\n\n if recurse is True:\n if has(v.__class__):\n rv[a.name] = asdict(\n v,\n recurse=True,\n filter=filter,\n dict_factory=dict_factory,\n retain_collection_types=retain_collection_types,\n value_serializer=value_serializer,\n )\n elif isinstance(v, (tuple, list, set, frozenset)):\n cf = v.__class__ if retain_collection_types is True else list\n rv[a.name] = cf(\n [\n _asdict_anything(\n i,\n is_key=False,\n filter=filter,\n dict_factory=dict_factory,\n retain_collection_types=retain_collection_types,\n value_serializer=value_serializer,\n )\n for i in v\n ]\n )\n elif isinstance(v, dict):\n df = dict_factory\n rv[a.name] = df(\n (\n _asdict_anything(\n kk,\n is_key=True,\n filter=filter,\n dict_factory=df,\n retain_collection_types=retain_collection_types,\n value_serializer=value_serializer,\n ),\n _asdict_anything(\n vv,\n is_key=False,\n filter=filter,\n dict_factory=df,\n retain_collection_types=retain_collection_types,\n value_serializer=value_serializer,\n ),\n )\n for kk, vv in v.items()\n )\n else:\n rv[a.name] = v\n else:\n rv[a.name] = v\n return rv\n\n\ndef _asdict_anything(\n val,\n is_key,\n filter,\n dict_factory,\n retain_collection_types,\n value_serializer,\n):\n \"\"\"\n ``asdict`` only works on attrs instances, this works on anything.\n \"\"\"\n if getattr(val.__class__, \"__attrs_attrs__\", None) is not None:\n # Attrs class.\n rv = asdict(\n val,\n recurse=True,\n filter=filter,\n dict_factory=dict_factory,\n retain_collection_types=retain_collection_types,\n value_serializer=value_serializer,\n )\n elif isinstance(val, (tuple, list, set, frozenset)):\n if retain_collection_types is True:\n cf = val.__class__\n elif is_key:\n cf = tuple\n else:\n cf = list\n\n rv = cf(\n [\n _asdict_anything(\n i,\n is_key=False,\n filter=filter,\n dict_factory=dict_factory,\n retain_collection_types=retain_collection_types,\n value_serializer=value_serializer,\n )\n for i in val\n ]\n )\n elif isinstance(val, dict):\n df = dict_factory\n rv = df(\n (\n _asdict_anything(\n kk,\n is_key=True,\n filter=filter,\n dict_factory=df,\n retain_collection_types=retain_collection_types,\n value_serializer=value_serializer,\n ),\n _asdict_anything(\n vv,\n is_key=False,\n filter=filter,\n dict_factory=df,\n retain_collection_types=retain_collection_types,\n value_serializer=value_serializer,\n ),\n )\n for kk, vv in val.items()\n )\n else:\n rv = val\n if value_serializer is not None:\n rv = value_serializer(None, None, rv)\n\n return rv\n\n\ndef astuple(\n inst,\n recurse=True,\n filter=None,\n tuple_factory=tuple,\n retain_collection_types=False,\n):\n \"\"\"\n Return the ``attrs`` attribute values of *inst* as a tuple.\n\n Optionally recurse into other ``attrs``-decorated classes.\n\n :param inst: Instance of an ``attrs``-decorated class.\n :param bool recurse: Recurse into classes that are also\n ``attrs``-decorated.\n :param callable filter: A callable whose return code determines whether an\n attribute or element is included (``True``) or dropped (``False``). Is\n called with the `attrs.Attribute` as the first argument and the\n value as the second argument.\n :param callable tuple_factory: A callable to produce tuples from. For\n example, to produce lists instead of tuples.\n :param bool retain_collection_types: Do not convert to ``list``\n or ``dict`` when encountering an attribute which type is\n ``tuple``, ``dict`` or ``set``. Only meaningful if ``recurse`` is\n ``True``.\n\n :rtype: return type of *tuple_factory*\n\n :raise attr.exceptions.NotAnAttrsClassError: If *cls* is not an ``attrs``\n class.\n\n .. versionadded:: 16.2.0\n \"\"\"\n attrs = fields(inst.__class__)\n rv = []\n retain = retain_collection_types # Very long. :/\n for a in attrs:\n v = getattr(inst, a.name)\n if filter is not None and not filter(a, v):\n continue\n if recurse is True:\n if has(v.__class__):\n rv.append(\n astuple(\n v,\n recurse=True,\n filter=filter,\n tuple_factory=tuple_factory,\n retain_collection_types=retain,\n )\n )\n elif isinstance(v, (tuple, list, set, frozenset)):\n cf = v.__class__ if retain is True else list\n rv.append(\n cf(\n [\n astuple(\n j,\n recurse=True,\n filter=filter,\n tuple_factory=tuple_factory,\n retain_collection_types=retain,\n )\n if has(j.__class__)\n else j\n for j in v\n ]\n )\n )\n elif isinstance(v, dict):\n df = v.__class__ if retain is True else dict\n rv.append(\n df(\n (\n astuple(\n kk,\n tuple_factory=tuple_factory,\n retain_collection_types=retain,\n )\n if has(kk.__class__)\n else kk,\n astuple(\n vv,\n tuple_factory=tuple_factory,\n retain_collection_types=retain,\n )\n if has(vv.__class__)\n else vv,\n )\n for kk, vv in v.items()\n )\n )\n else:\n rv.append(v)\n else:\n rv.append(v)\n\n return rv if tuple_factory is list else tuple_factory(rv)\n\n\ndef has(cls):\n \"\"\"\n Check whether *cls* is a class with ``attrs`` attributes.\n\n :param type cls: Class to introspect.\n :raise TypeError: If *cls* is not a class.\n\n :rtype: bool\n \"\"\"\n return getattr(cls, \"__attrs_attrs__\", None) is not None\n\n\ndef assoc(inst, **changes):\n \"\"\"\n Copy *inst* and apply *changes*.\n\n :param inst: Instance of a class with ``attrs`` attributes.\n :param changes: Keyword changes in the new copy.\n\n :return: A copy of inst with *changes* incorporated.\n\n :raise attr.exceptions.AttrsAttributeNotFoundError: If *attr_name* couldn't\n be found on *cls*.\n :raise attr.exceptions.NotAnAttrsClassError: If *cls* is not an ``attrs``\n class.\n\n .. deprecated:: 17.1.0\n Use `attrs.evolve` instead if you can.\n This function will not be removed du to the slightly different approach\n compared to `attrs.evolve`.\n \"\"\"\n import warnings\n\n warnings.warn(\n \"assoc is deprecated and will be removed after 2018/01.\",\n DeprecationWarning,\n stacklevel=2,\n )\n new = copy.copy(inst)\n attrs = fields(inst.__class__)\n for k, v in changes.items():\n a = getattr(attrs, k, NOTHING)\n if a is NOTHING:\n raise AttrsAttributeNotFoundError(\n \"{k} is not an attrs attribute on {cl}.\".format(\n k=k, cl=new.__class__\n )\n )\n _obj_setattr(new, k, v)\n return new\n\n\ndef evolve(inst, **changes):\n \"\"\"\n Create a new instance, based on *inst* with *changes* applied.\n\n :param inst: Instance of a class with ``attrs`` attributes.\n :param changes: Keyword changes in the new copy.\n\n :return: A copy of inst with *changes* incorporated.\n\n :raise TypeError: If *attr_name* couldn't be found in the class\n ``__init__``.\n :raise attr.exceptions.NotAnAttrsClassError: If *cls* is not an ``attrs``\n class.\n\n .. versionadded:: 17.1.0\n \"\"\"\n cls = inst.__class__\n attrs = fields(cls)\n for a in attrs:\n if not a.init:\n continue\n attr_name = a.name # To deal with private attributes.\n init_name = attr_name if attr_name[0] != \"_\" else attr_name[1:]\n if init_name not in changes:\n changes[init_name] = getattr(inst, attr_name)\n\n return cls(**changes)\n\n\ndef resolve_types(cls, globalns=None, localns=None, attribs=None):\n \"\"\"\n Resolve any strings and forward annotations in type annotations.\n\n This is only required if you need concrete types in `Attribute`'s *type*\n field. In other words, you don't need to resolve your types if you only\n use them for static type checking.\n\n With no arguments, names will be looked up in the module in which the class\n was created. If this is not what you want, e.g. if the name only exists\n inside a method, you may pass *globalns* or *localns* to specify other\n dictionaries in which to look up these names. See the docs of\n `typing.get_type_hints` for more details.\n\n :param type cls: Class to resolve.\n :param Optional[dict] globalns: Dictionary containing global variables.\n :param Optional[dict] localns: Dictionary containing local variables.\n :param Optional[list] attribs: List of attribs for the given class.\n This is necessary when calling from inside a ``field_transformer``\n since *cls* is not an ``attrs`` class yet.\n\n :raise TypeError: If *cls* is not a class.\n :raise attr.exceptions.NotAnAttrsClassError: If *cls* is not an ``attrs``\n class and you didn't pass any attribs.\n :raise NameError: If types cannot be resolved because of missing variables.\n\n :returns: *cls* so you can use this function also as a class decorator.\n Please note that you have to apply it **after** `attrs.define`. That\n means the decorator has to come in the line **before** `attrs.define`.\n\n .. versionadded:: 20.1.0\n .. versionadded:: 21.1.0 *attribs*\n\n \"\"\"\n # Since calling get_type_hints is expensive we cache whether we've\n # done it already.\n if getattr(cls, \"__attrs_types_resolved__\", None) != cls:\n import typing\n\n hints = typing.get_type_hints(cls, globalns=globalns, localns=localns)\n for field in fields(cls) if attribs is None else attribs:\n if field.name in hints:\n # Since fields have been frozen we must work around it.\n _obj_setattr(field, \"type\", hints[field.name])\n # We store the class we resolved so that subclasses know they haven't\n # been resolved.\n cls.__attrs_types_resolved__ = cls\n\n # Return the class so you can use it as a decorator too.\n return cls\n" }, { "path": "lib/spack/spack/vendor/attr/_make.py", "content": "# SPDX-License-Identifier: MIT\n\nimport copy\nimport linecache\nimport sys\nimport types\nimport typing\n\nfrom operator import itemgetter\n\n# We need to import _compat itself in addition to the _compat members to avoid\n# having the thread-local in the globals here.\nfrom . import _compat, _config, setters\nfrom ._compat import (\n HAS_F_STRINGS,\n PY310,\n PYPY,\n _AnnotationExtractor,\n ordered_dict,\n set_closure_cell,\n)\nfrom .exceptions import (\n DefaultAlreadySetError,\n FrozenInstanceError,\n NotAnAttrsClassError,\n UnannotatedAttributeError,\n)\n\n\n# This is used at least twice, so cache it here.\n_obj_setattr = object.__setattr__\n_init_converter_pat = \"__attr_converter_%s\"\n_init_factory_pat = \"__attr_factory_{}\"\n_tuple_property_pat = (\n \" {attr_name} = _attrs_property(_attrs_itemgetter({index}))\"\n)\n_classvar_prefixes = (\n \"typing.ClassVar\",\n \"t.ClassVar\",\n \"ClassVar\",\n \"spack.vendor.typing_extensions.ClassVar\",\n)\n# we don't use a double-underscore prefix because that triggers\n# name mangling when trying to create a slot for the field\n# (when slots=True)\n_hash_cache_field = \"_attrs_cached_hash\"\n\n_empty_metadata_singleton = types.MappingProxyType({})\n\n# Unique object for unequivocal getattr() defaults.\n_sentinel = object()\n\n_ng_default_on_setattr = setters.pipe(setters.convert, setters.validate)\n\n\nclass _Nothing:\n \"\"\"\n Sentinel class to indicate the lack of a value when ``None`` is ambiguous.\n\n ``_Nothing`` is a singleton. There is only ever one of it.\n\n .. versionchanged:: 21.1.0 ``bool(NOTHING)`` is now False.\n \"\"\"\n\n _singleton = None\n\n def __new__(cls):\n if _Nothing._singleton is None:\n _Nothing._singleton = super().__new__(cls)\n return _Nothing._singleton\n\n def __repr__(self):\n return \"NOTHING\"\n\n def __bool__(self):\n return False\n\n\nNOTHING = _Nothing()\n\"\"\"\nSentinel to indicate the lack of a value when ``None`` is ambiguous.\n\"\"\"\n\n\nclass _CacheHashWrapper(int):\n \"\"\"\n An integer subclass that pickles / copies as None\n\n This is used for non-slots classes with ``cache_hash=True``, to avoid\n serializing a potentially (even likely) invalid hash value. Since ``None``\n is the default value for uncalculated hashes, whenever this is copied,\n the copy's value for the hash should automatically reset.\n\n See GH #613 for more details.\n \"\"\"\n\n def __reduce__(self, _none_constructor=type(None), _args=()):\n return _none_constructor, _args\n\n\ndef attrib(\n default=NOTHING,\n validator=None,\n repr=True,\n cmp=None,\n hash=None,\n init=True,\n metadata=None,\n type=None,\n converter=None,\n factory=None,\n kw_only=False,\n eq=None,\n order=None,\n on_setattr=None,\n):\n \"\"\"\n Create a new attribute on a class.\n\n .. warning::\n\n Does *not* do anything unless the class is also decorated with\n `attr.s`!\n\n :param default: A value that is used if an ``attrs``-generated ``__init__``\n is used and no value is passed while instantiating or the attribute is\n excluded using ``init=False``.\n\n If the value is an instance of `attrs.Factory`, its callable will be\n used to construct a new value (useful for mutable data types like lists\n or dicts).\n\n If a default is not set (or set manually to `attrs.NOTHING`), a value\n *must* be supplied when instantiating; otherwise a `TypeError`\n will be raised.\n\n The default can also be set using decorator notation as shown below.\n\n :type default: Any value\n\n :param callable factory: Syntactic sugar for\n ``default=attr.Factory(factory)``.\n\n :param validator: `callable` that is called by ``attrs``-generated\n ``__init__`` methods after the instance has been initialized. They\n receive the initialized instance, the :func:`~attrs.Attribute`, and the\n passed value.\n\n The return value is *not* inspected so the validator has to throw an\n exception itself.\n\n If a `list` is passed, its items are treated as validators and must\n all pass.\n\n Validators can be globally disabled and re-enabled using\n `get_run_validators`.\n\n The validator can also be set using decorator notation as shown below.\n\n :type validator: `callable` or a `list` of `callable`\\\\ s.\n\n :param repr: Include this attribute in the generated ``__repr__``\n method. If ``True``, include the attribute; if ``False``, omit it. By\n default, the built-in ``repr()`` function is used. To override how the\n attribute value is formatted, pass a ``callable`` that takes a single\n value and returns a string. Note that the resulting string is used\n as-is, i.e. it will be used directly *instead* of calling ``repr()``\n (the default).\n :type repr: a `bool` or a `callable` to use a custom function.\n\n :param eq: If ``True`` (default), include this attribute in the\n generated ``__eq__`` and ``__ne__`` methods that check two instances\n for equality. To override how the attribute value is compared,\n pass a ``callable`` that takes a single value and returns the value\n to be compared.\n :type eq: a `bool` or a `callable`.\n\n :param order: If ``True`` (default), include this attributes in the\n generated ``__lt__``, ``__le__``, ``__gt__`` and ``__ge__`` methods.\n To override how the attribute value is ordered,\n pass a ``callable`` that takes a single value and returns the value\n to be ordered.\n :type order: a `bool` or a `callable`.\n\n :param cmp: Setting *cmp* is equivalent to setting *eq* and *order* to the\n same value. Must not be mixed with *eq* or *order*.\n :type cmp: a `bool` or a `callable`.\n\n :param Optional[bool] hash: Include this attribute in the generated\n ``__hash__`` method. If ``None`` (default), mirror *eq*'s value. This\n is the correct behavior according the Python spec. Setting this value\n to anything else than ``None`` is *discouraged*.\n :param bool init: Include this attribute in the generated ``__init__``\n method. It is possible to set this to ``False`` and set a default\n value. In that case this attributed is unconditionally initialized\n with the specified default value or factory.\n :param callable converter: `callable` that is called by\n ``attrs``-generated ``__init__`` methods to convert attribute's value\n to the desired format. It is given the passed-in value, and the\n returned value will be used as the new value of the attribute. The\n value is converted before being passed to the validator, if any.\n :param metadata: An arbitrary mapping, to be used by third-party\n components. See `extending_metadata`.\n :param type: The type of the attribute. In Python 3.6 or greater, the\n preferred method to specify the type is using a variable annotation\n (see :pep:`526`).\n This argument is provided for backward compatibility.\n Regardless of the approach used, the type will be stored on\n ``Attribute.type``.\n\n Please note that ``attrs`` doesn't do anything with this metadata by\n itself. You can use it as part of your own code or for\n `static type checking `.\n :param kw_only: Make this attribute keyword-only (Python 3+)\n in the generated ``__init__`` (if ``init`` is ``False``, this\n parameter is ignored).\n :param on_setattr: Allows to overwrite the *on_setattr* setting from\n `attr.s`. If left `None`, the *on_setattr* value from `attr.s` is used.\n Set to `attrs.setters.NO_OP` to run **no** `setattr` hooks for this\n attribute -- regardless of the setting in `attr.s`.\n :type on_setattr: `callable`, or a list of callables, or `None`, or\n `attrs.setters.NO_OP`\n\n .. versionadded:: 15.2.0 *convert*\n .. versionadded:: 16.3.0 *metadata*\n .. versionchanged:: 17.1.0 *validator* can be a ``list`` now.\n .. versionchanged:: 17.1.0\n *hash* is ``None`` and therefore mirrors *eq* by default.\n .. versionadded:: 17.3.0 *type*\n .. deprecated:: 17.4.0 *convert*\n .. versionadded:: 17.4.0 *converter* as a replacement for the deprecated\n *convert* to achieve consistency with other noun-based arguments.\n .. versionadded:: 18.1.0\n ``factory=f`` is syntactic sugar for ``default=attr.Factory(f)``.\n .. versionadded:: 18.2.0 *kw_only*\n .. versionchanged:: 19.2.0 *convert* keyword argument removed.\n .. versionchanged:: 19.2.0 *repr* also accepts a custom callable.\n .. deprecated:: 19.2.0 *cmp* Removal on or after 2021-06-01.\n .. versionadded:: 19.2.0 *eq* and *order*\n .. versionadded:: 20.1.0 *on_setattr*\n .. versionchanged:: 20.3.0 *kw_only* backported to Python 2\n .. versionchanged:: 21.1.0\n *eq*, *order*, and *cmp* also accept a custom callable\n .. versionchanged:: 21.1.0 *cmp* undeprecated\n \"\"\"\n eq, eq_key, order, order_key = _determine_attrib_eq_order(\n cmp, eq, order, True\n )\n\n if hash is not None and hash is not True and hash is not False:\n raise TypeError(\n \"Invalid value for hash. Must be True, False, or None.\"\n )\n\n if factory is not None:\n if default is not NOTHING:\n raise ValueError(\n \"The `default` and `factory` arguments are mutually \"\n \"exclusive.\"\n )\n if not callable(factory):\n raise ValueError(\"The `factory` argument must be a callable.\")\n default = Factory(factory)\n\n if metadata is None:\n metadata = {}\n\n # Apply syntactic sugar by auto-wrapping.\n if isinstance(on_setattr, (list, tuple)):\n on_setattr = setters.pipe(*on_setattr)\n\n if validator and isinstance(validator, (list, tuple)):\n validator = and_(*validator)\n\n if converter and isinstance(converter, (list, tuple)):\n converter = pipe(*converter)\n\n return _CountingAttr(\n default=default,\n validator=validator,\n repr=repr,\n cmp=None,\n hash=hash,\n init=init,\n converter=converter,\n metadata=metadata,\n type=type,\n kw_only=kw_only,\n eq=eq,\n eq_key=eq_key,\n order=order,\n order_key=order_key,\n on_setattr=on_setattr,\n )\n\n\ndef _compile_and_eval(script, globs, locs=None, filename=\"\"):\n \"\"\"\n \"Exec\" the script with the given global (globs) and local (locs) variables.\n \"\"\"\n bytecode = compile(script, filename, \"exec\")\n eval(bytecode, globs, locs)\n\n\ndef _make_method(name, script, filename, globs):\n \"\"\"\n Create the method with the script given and return the method object.\n \"\"\"\n locs = {}\n\n # In order of debuggers like PDB being able to step through the code,\n # we add a fake linecache entry.\n count = 1\n base_filename = filename\n while True:\n linecache_tuple = (\n len(script),\n None,\n script.splitlines(True),\n filename,\n )\n old_val = linecache.cache.setdefault(filename, linecache_tuple)\n if old_val == linecache_tuple:\n break\n else:\n filename = \"{}-{}>\".format(base_filename[:-1], count)\n count += 1\n\n _compile_and_eval(script, globs, locs, filename)\n\n return locs[name]\n\n\ndef _make_attr_tuple_class(cls_name, attr_names):\n \"\"\"\n Create a tuple subclass to hold `Attribute`s for an `attrs` class.\n\n The subclass is a bare tuple with properties for names.\n\n class MyClassAttributes(tuple):\n __slots__ = ()\n x = property(itemgetter(0))\n \"\"\"\n attr_class_name = \"{}Attributes\".format(cls_name)\n attr_class_template = [\n \"class {}(tuple):\".format(attr_class_name),\n \" __slots__ = ()\",\n ]\n if attr_names:\n for i, attr_name in enumerate(attr_names):\n attr_class_template.append(\n _tuple_property_pat.format(index=i, attr_name=attr_name)\n )\n else:\n attr_class_template.append(\" pass\")\n globs = {\"_attrs_itemgetter\": itemgetter, \"_attrs_property\": property}\n _compile_and_eval(\"\\n\".join(attr_class_template), globs)\n return globs[attr_class_name]\n\n\n# Tuple class for extracted attributes from a class definition.\n# `base_attrs` is a subset of `attrs`.\n_Attributes = _make_attr_tuple_class(\n \"_Attributes\",\n [\n # all attributes to build dunder methods for\n \"attrs\",\n # attributes that have been inherited\n \"base_attrs\",\n # map inherited attributes to their originating classes\n \"base_attrs_map\",\n ],\n)\n\n\ndef _is_class_var(annot):\n \"\"\"\n Check whether *annot* is a typing.ClassVar.\n\n The string comparison hack is used to avoid evaluating all string\n annotations which would put attrs-based classes at a performance\n disadvantage compared to plain old classes.\n \"\"\"\n annot = str(annot)\n\n # Annotation can be quoted.\n if annot.startswith((\"'\", '\"')) and annot.endswith((\"'\", '\"')):\n annot = annot[1:-1]\n\n return annot.startswith(_classvar_prefixes)\n\n\ndef _has_own_attribute(cls, attrib_name):\n \"\"\"\n Check whether *cls* defines *attrib_name* (and doesn't just inherit it).\n\n Requires Python 3.\n \"\"\"\n attr = getattr(cls, attrib_name, _sentinel)\n if attr is _sentinel:\n return False\n\n for base_cls in cls.__mro__[1:]:\n a = getattr(base_cls, attrib_name, None)\n if attr is a:\n return False\n\n return True\n\n\ndef _get_annotations(cls):\n \"\"\"\n Get annotations for *cls*.\n \"\"\"\n if _has_own_attribute(cls, \"__annotations__\"):\n return cls.__annotations__\n\n return {}\n\n\ndef _counter_getter(e):\n \"\"\"\n Key function for sorting to avoid re-creating a lambda for every class.\n \"\"\"\n return e[1].counter\n\n\ndef _collect_base_attrs(cls, taken_attr_names):\n \"\"\"\n Collect attr.ibs from base classes of *cls*, except *taken_attr_names*.\n \"\"\"\n base_attrs = []\n base_attr_map = {} # A dictionary of base attrs to their classes.\n\n # Traverse the MRO and collect attributes.\n for base_cls in reversed(cls.__mro__[1:-1]):\n for a in getattr(base_cls, \"__attrs_attrs__\", []):\n if a.inherited or a.name in taken_attr_names:\n continue\n\n a = a.evolve(inherited=True)\n base_attrs.append(a)\n base_attr_map[a.name] = base_cls\n\n # For each name, only keep the freshest definition i.e. the furthest at the\n # back. base_attr_map is fine because it gets overwritten with every new\n # instance.\n filtered = []\n seen = set()\n for a in reversed(base_attrs):\n if a.name in seen:\n continue\n filtered.insert(0, a)\n seen.add(a.name)\n\n return filtered, base_attr_map\n\n\ndef _collect_base_attrs_broken(cls, taken_attr_names):\n \"\"\"\n Collect attr.ibs from base classes of *cls*, except *taken_attr_names*.\n\n N.B. *taken_attr_names* will be mutated.\n\n Adhere to the old incorrect behavior.\n\n Notably it collects from the front and considers inherited attributes which\n leads to the buggy behavior reported in #428.\n \"\"\"\n base_attrs = []\n base_attr_map = {} # A dictionary of base attrs to their classes.\n\n # Traverse the MRO and collect attributes.\n for base_cls in cls.__mro__[1:-1]:\n for a in getattr(base_cls, \"__attrs_attrs__\", []):\n if a.name in taken_attr_names:\n continue\n\n a = a.evolve(inherited=True)\n taken_attr_names.add(a.name)\n base_attrs.append(a)\n base_attr_map[a.name] = base_cls\n\n return base_attrs, base_attr_map\n\n\ndef _transform_attrs(\n cls, these, auto_attribs, kw_only, collect_by_mro, field_transformer\n):\n \"\"\"\n Transform all `_CountingAttr`s on a class into `Attribute`s.\n\n If *these* is passed, use that and don't look for them on the class.\n\n *collect_by_mro* is True, collect them in the correct MRO order, otherwise\n use the old -- incorrect -- order. See #428.\n\n Return an `_Attributes`.\n \"\"\"\n cd = cls.__dict__\n anns = _get_annotations(cls)\n\n if these is not None:\n ca_list = [(name, ca) for name, ca in these.items()]\n\n if not isinstance(these, ordered_dict):\n ca_list.sort(key=_counter_getter)\n elif auto_attribs is True:\n ca_names = {\n name\n for name, attr in cd.items()\n if isinstance(attr, _CountingAttr)\n }\n ca_list = []\n annot_names = set()\n for attr_name, type in anns.items():\n if _is_class_var(type):\n continue\n annot_names.add(attr_name)\n a = cd.get(attr_name, NOTHING)\n\n if not isinstance(a, _CountingAttr):\n if a is NOTHING:\n a = attrib()\n else:\n a = attrib(default=a)\n ca_list.append((attr_name, a))\n\n unannotated = ca_names - annot_names\n if len(unannotated) > 0:\n raise UnannotatedAttributeError(\n \"The following `attr.ib`s lack a type annotation: \"\n + \", \".join(\n sorted(unannotated, key=lambda n: cd.get(n).counter)\n )\n + \".\"\n )\n else:\n ca_list = sorted(\n (\n (name, attr)\n for name, attr in cd.items()\n if isinstance(attr, _CountingAttr)\n ),\n key=lambda e: e[1].counter,\n )\n\n own_attrs = [\n Attribute.from_counting_attr(\n name=attr_name, ca=ca, type=anns.get(attr_name)\n )\n for attr_name, ca in ca_list\n ]\n\n if collect_by_mro:\n base_attrs, base_attr_map = _collect_base_attrs(\n cls, {a.name for a in own_attrs}\n )\n else:\n base_attrs, base_attr_map = _collect_base_attrs_broken(\n cls, {a.name for a in own_attrs}\n )\n\n if kw_only:\n own_attrs = [a.evolve(kw_only=True) for a in own_attrs]\n base_attrs = [a.evolve(kw_only=True) for a in base_attrs]\n\n attrs = base_attrs + own_attrs\n\n # Mandatory vs non-mandatory attr order only matters when they are part of\n # the __init__ signature and when they aren't kw_only (which are moved to\n # the end and can be mandatory or non-mandatory in any order, as they will\n # be specified as keyword args anyway). Check the order of those attrs:\n had_default = False\n for a in (a for a in attrs if a.init is not False and a.kw_only is False):\n if had_default is True and a.default is NOTHING:\n raise ValueError(\n \"No mandatory attributes allowed after an attribute with a \"\n \"default value or factory. Attribute in question: %r\" % (a,)\n )\n\n if had_default is False and a.default is not NOTHING:\n had_default = True\n\n if field_transformer is not None:\n attrs = field_transformer(cls, attrs)\n\n # Create AttrsClass *after* applying the field_transformer since it may\n # add or remove attributes!\n attr_names = [a.name for a in attrs]\n AttrsClass = _make_attr_tuple_class(cls.__name__, attr_names)\n\n return _Attributes((AttrsClass(attrs), base_attrs, base_attr_map))\n\n\nif PYPY:\n\n def _frozen_setattrs(self, name, value):\n \"\"\"\n Attached to frozen classes as __setattr__.\n \"\"\"\n if isinstance(self, BaseException) and name in (\n \"__cause__\",\n \"__context__\",\n ):\n BaseException.__setattr__(self, name, value)\n return\n\n raise FrozenInstanceError()\n\nelse:\n\n def _frozen_setattrs(self, name, value):\n \"\"\"\n Attached to frozen classes as __setattr__.\n \"\"\"\n raise FrozenInstanceError()\n\n\ndef _frozen_delattrs(self, name):\n \"\"\"\n Attached to frozen classes as __delattr__.\n \"\"\"\n raise FrozenInstanceError()\n\n\nclass _ClassBuilder:\n \"\"\"\n Iteratively build *one* class.\n \"\"\"\n\n __slots__ = (\n \"_attr_names\",\n \"_attrs\",\n \"_base_attr_map\",\n \"_base_names\",\n \"_cache_hash\",\n \"_cls\",\n \"_cls_dict\",\n \"_delete_attribs\",\n \"_frozen\",\n \"_has_pre_init\",\n \"_has_post_init\",\n \"_is_exc\",\n \"_on_setattr\",\n \"_slots\",\n \"_weakref_slot\",\n \"_wrote_own_setattr\",\n \"_has_custom_setattr\",\n )\n\n def __init__(\n self,\n cls,\n these,\n slots,\n frozen,\n weakref_slot,\n getstate_setstate,\n auto_attribs,\n kw_only,\n cache_hash,\n is_exc,\n collect_by_mro,\n on_setattr,\n has_custom_setattr,\n field_transformer,\n ):\n attrs, base_attrs, base_map = _transform_attrs(\n cls,\n these,\n auto_attribs,\n kw_only,\n collect_by_mro,\n field_transformer,\n )\n\n self._cls = cls\n self._cls_dict = dict(cls.__dict__) if slots else {}\n self._attrs = attrs\n self._base_names = {a.name for a in base_attrs}\n self._base_attr_map = base_map\n self._attr_names = tuple(a.name for a in attrs)\n self._slots = slots\n self._frozen = frozen\n self._weakref_slot = weakref_slot\n self._cache_hash = cache_hash\n self._has_pre_init = bool(getattr(cls, \"__attrs_pre_init__\", False))\n self._has_post_init = bool(getattr(cls, \"__attrs_post_init__\", False))\n self._delete_attribs = not bool(these)\n self._is_exc = is_exc\n self._on_setattr = on_setattr\n\n self._has_custom_setattr = has_custom_setattr\n self._wrote_own_setattr = False\n\n self._cls_dict[\"__attrs_attrs__\"] = self._attrs\n\n if frozen:\n self._cls_dict[\"__setattr__\"] = _frozen_setattrs\n self._cls_dict[\"__delattr__\"] = _frozen_delattrs\n\n self._wrote_own_setattr = True\n elif on_setattr in (\n _ng_default_on_setattr,\n setters.validate,\n setters.convert,\n ):\n has_validator = has_converter = False\n for a in attrs:\n if a.validator is not None:\n has_validator = True\n if a.converter is not None:\n has_converter = True\n\n if has_validator and has_converter:\n break\n if (\n (\n on_setattr == _ng_default_on_setattr\n and not (has_validator or has_converter)\n )\n or (on_setattr == setters.validate and not has_validator)\n or (on_setattr == setters.convert and not has_converter)\n ):\n # If class-level on_setattr is set to convert + validate, but\n # there's no field to convert or validate, pretend like there's\n # no on_setattr.\n self._on_setattr = None\n\n if getstate_setstate:\n (\n self._cls_dict[\"__getstate__\"],\n self._cls_dict[\"__setstate__\"],\n ) = self._make_getstate_setstate()\n\n def __repr__(self):\n return \"<_ClassBuilder(cls={cls})>\".format(cls=self._cls.__name__)\n\n def build_class(self):\n \"\"\"\n Finalize class based on the accumulated configuration.\n\n Builder cannot be used after calling this method.\n \"\"\"\n if self._slots is True:\n return self._create_slots_class()\n else:\n return self._patch_original_class()\n\n def _patch_original_class(self):\n \"\"\"\n Apply accumulated methods and return the class.\n \"\"\"\n cls = self._cls\n base_names = self._base_names\n\n # Clean class of attribute definitions (`attr.ib()`s).\n if self._delete_attribs:\n for name in self._attr_names:\n if (\n name not in base_names\n and getattr(cls, name, _sentinel) is not _sentinel\n ):\n try:\n delattr(cls, name)\n except AttributeError:\n # This can happen if a base class defines a class\n # variable and we want to set an attribute with the\n # same name by using only a type annotation.\n pass\n\n # Attach our dunder methods.\n for name, value in self._cls_dict.items():\n setattr(cls, name, value)\n\n # If we've inherited an attrs __setattr__ and don't write our own,\n # reset it to object's.\n if not self._wrote_own_setattr and getattr(\n cls, \"__attrs_own_setattr__\", False\n ):\n cls.__attrs_own_setattr__ = False\n\n if not self._has_custom_setattr:\n cls.__setattr__ = _obj_setattr\n\n return cls\n\n def _create_slots_class(self):\n \"\"\"\n Build and return a new class with a `__slots__` attribute.\n \"\"\"\n cd = {\n k: v\n for k, v in self._cls_dict.items()\n if k not in tuple(self._attr_names) + (\"__dict__\", \"__weakref__\")\n }\n\n # If our class doesn't have its own implementation of __setattr__\n # (either from the user or by us), check the bases, if one of them has\n # an attrs-made __setattr__, that needs to be reset. We don't walk the\n # MRO because we only care about our immediate base classes.\n # XXX: This can be confused by subclassing a slotted attrs class with\n # XXX: a non-attrs class and subclass the resulting class with an attrs\n # XXX: class. See `test_slotted_confused` for details. For now that's\n # XXX: OK with us.\n if not self._wrote_own_setattr:\n cd[\"__attrs_own_setattr__\"] = False\n\n if not self._has_custom_setattr:\n for base_cls in self._cls.__bases__:\n if base_cls.__dict__.get(\"__attrs_own_setattr__\", False):\n cd[\"__setattr__\"] = _obj_setattr\n break\n\n # Traverse the MRO to collect existing slots\n # and check for an existing __weakref__.\n existing_slots = dict()\n weakref_inherited = False\n for base_cls in self._cls.__mro__[1:-1]:\n if base_cls.__dict__.get(\"__weakref__\", None) is not None:\n weakref_inherited = True\n existing_slots.update(\n {\n name: getattr(base_cls, name)\n for name in getattr(base_cls, \"__slots__\", [])\n }\n )\n\n base_names = set(self._base_names)\n\n names = self._attr_names\n if (\n self._weakref_slot\n and \"__weakref__\" not in getattr(self._cls, \"__slots__\", ())\n and \"__weakref__\" not in names\n and not weakref_inherited\n ):\n names += (\"__weakref__\",)\n\n # We only add the names of attributes that aren't inherited.\n # Setting __slots__ to inherited attributes wastes memory.\n slot_names = [name for name in names if name not in base_names]\n # There are slots for attributes from current class\n # that are defined in parent classes.\n # As their descriptors may be overridden by a child class,\n # we collect them here and update the class dict\n reused_slots = {\n slot: slot_descriptor\n for slot, slot_descriptor in existing_slots.items()\n if slot in slot_names\n }\n slot_names = [name for name in slot_names if name not in reused_slots]\n cd.update(reused_slots)\n if self._cache_hash:\n slot_names.append(_hash_cache_field)\n cd[\"__slots__\"] = tuple(slot_names)\n\n cd[\"__qualname__\"] = self._cls.__qualname__\n\n # Create new class based on old class and our methods.\n cls = type(self._cls)(self._cls.__name__, self._cls.__bases__, cd)\n\n # The following is a fix for\n # . On Python 3,\n # if a method mentions `__class__` or uses the no-arg super(), the\n # compiler will bake a reference to the class in the method itself\n # as `method.__closure__`. Since we replace the class with a\n # clone, we rewrite these references so it keeps working.\n for item in cls.__dict__.values():\n if isinstance(item, (classmethod, staticmethod)):\n # Class- and staticmethods hide their functions inside.\n # These might need to be rewritten as well.\n closure_cells = getattr(item.__func__, \"__closure__\", None)\n elif isinstance(item, property):\n # Workaround for property `super()` shortcut (PY3-only).\n # There is no universal way for other descriptors.\n closure_cells = getattr(item.fget, \"__closure__\", None)\n else:\n closure_cells = getattr(item, \"__closure__\", None)\n\n if not closure_cells: # Catch None or the empty list.\n continue\n for cell in closure_cells:\n try:\n match = cell.cell_contents is self._cls\n except ValueError: # ValueError: Cell is empty\n pass\n else:\n if match:\n set_closure_cell(cell, cls)\n\n return cls\n\n def add_repr(self, ns):\n self._cls_dict[\"__repr__\"] = self._add_method_dunders(\n _make_repr(self._attrs, ns, self._cls)\n )\n return self\n\n def add_str(self):\n repr = self._cls_dict.get(\"__repr__\")\n if repr is None:\n raise ValueError(\n \"__str__ can only be generated if a __repr__ exists.\"\n )\n\n def __str__(self):\n return self.__repr__()\n\n self._cls_dict[\"__str__\"] = self._add_method_dunders(__str__)\n return self\n\n def _make_getstate_setstate(self):\n \"\"\"\n Create custom __setstate__ and __getstate__ methods.\n \"\"\"\n # __weakref__ is not writable.\n state_attr_names = tuple(\n an for an in self._attr_names if an != \"__weakref__\"\n )\n\n def slots_getstate(self):\n \"\"\"\n Automatically created by attrs.\n \"\"\"\n return tuple(getattr(self, name) for name in state_attr_names)\n\n hash_caching_enabled = self._cache_hash\n\n def slots_setstate(self, state):\n \"\"\"\n Automatically created by attrs.\n \"\"\"\n __bound_setattr = _obj_setattr.__get__(self, Attribute)\n for name, value in zip(state_attr_names, state):\n __bound_setattr(name, value)\n\n # The hash code cache is not included when the object is\n # serialized, but it still needs to be initialized to None to\n # indicate that the first call to __hash__ should be a cache\n # miss.\n if hash_caching_enabled:\n __bound_setattr(_hash_cache_field, None)\n\n return slots_getstate, slots_setstate\n\n def make_unhashable(self):\n self._cls_dict[\"__hash__\"] = None\n return self\n\n def add_hash(self):\n self._cls_dict[\"__hash__\"] = self._add_method_dunders(\n _make_hash(\n self._cls,\n self._attrs,\n frozen=self._frozen,\n cache_hash=self._cache_hash,\n )\n )\n\n return self\n\n def add_init(self):\n self._cls_dict[\"__init__\"] = self._add_method_dunders(\n _make_init(\n self._cls,\n self._attrs,\n self._has_pre_init,\n self._has_post_init,\n self._frozen,\n self._slots,\n self._cache_hash,\n self._base_attr_map,\n self._is_exc,\n self._on_setattr,\n attrs_init=False,\n )\n )\n\n return self\n\n def add_match_args(self):\n self._cls_dict[\"__match_args__\"] = tuple(\n field.name\n for field in self._attrs\n if field.init and not field.kw_only\n )\n\n def add_attrs_init(self):\n self._cls_dict[\"__attrs_init__\"] = self._add_method_dunders(\n _make_init(\n self._cls,\n self._attrs,\n self._has_pre_init,\n self._has_post_init,\n self._frozen,\n self._slots,\n self._cache_hash,\n self._base_attr_map,\n self._is_exc,\n self._on_setattr,\n attrs_init=True,\n )\n )\n\n return self\n\n def add_eq(self):\n cd = self._cls_dict\n\n cd[\"__eq__\"] = self._add_method_dunders(\n _make_eq(self._cls, self._attrs)\n )\n cd[\"__ne__\"] = self._add_method_dunders(_make_ne())\n\n return self\n\n def add_order(self):\n cd = self._cls_dict\n\n cd[\"__lt__\"], cd[\"__le__\"], cd[\"__gt__\"], cd[\"__ge__\"] = (\n self._add_method_dunders(meth)\n for meth in _make_order(self._cls, self._attrs)\n )\n\n return self\n\n def add_setattr(self):\n if self._frozen:\n return self\n\n sa_attrs = {}\n for a in self._attrs:\n on_setattr = a.on_setattr or self._on_setattr\n if on_setattr and on_setattr is not setters.NO_OP:\n sa_attrs[a.name] = a, on_setattr\n\n if not sa_attrs:\n return self\n\n if self._has_custom_setattr:\n # We need to write a __setattr__ but there already is one!\n raise ValueError(\n \"Can't combine custom __setattr__ with on_setattr hooks.\"\n )\n\n # docstring comes from _add_method_dunders\n def __setattr__(self, name, val):\n try:\n a, hook = sa_attrs[name]\n except KeyError:\n nval = val\n else:\n nval = hook(self, a, val)\n\n _obj_setattr(self, name, nval)\n\n self._cls_dict[\"__attrs_own_setattr__\"] = True\n self._cls_dict[\"__setattr__\"] = self._add_method_dunders(__setattr__)\n self._wrote_own_setattr = True\n\n return self\n\n def _add_method_dunders(self, method):\n \"\"\"\n Add __module__ and __qualname__ to a *method* if possible.\n \"\"\"\n try:\n method.__module__ = self._cls.__module__\n except AttributeError:\n pass\n\n try:\n method.__qualname__ = \".\".join(\n (self._cls.__qualname__, method.__name__)\n )\n except AttributeError:\n pass\n\n try:\n method.__doc__ = \"Method generated by attrs for class %s.\" % (\n self._cls.__qualname__,\n )\n except AttributeError:\n pass\n\n return method\n\n\ndef _determine_attrs_eq_order(cmp, eq, order, default_eq):\n \"\"\"\n Validate the combination of *cmp*, *eq*, and *order*. Derive the effective\n values of eq and order. If *eq* is None, set it to *default_eq*.\n \"\"\"\n if cmp is not None and any((eq is not None, order is not None)):\n raise ValueError(\"Don't mix `cmp` with `eq' and `order`.\")\n\n # cmp takes precedence due to bw-compatibility.\n if cmp is not None:\n return cmp, cmp\n\n # If left None, equality is set to the specified default and ordering\n # mirrors equality.\n if eq is None:\n eq = default_eq\n\n if order is None:\n order = eq\n\n if eq is False and order is True:\n raise ValueError(\"`order` can only be True if `eq` is True too.\")\n\n return eq, order\n\n\ndef _determine_attrib_eq_order(cmp, eq, order, default_eq):\n \"\"\"\n Validate the combination of *cmp*, *eq*, and *order*. Derive the effective\n values of eq and order. If *eq* is None, set it to *default_eq*.\n \"\"\"\n if cmp is not None and any((eq is not None, order is not None)):\n raise ValueError(\"Don't mix `cmp` with `eq' and `order`.\")\n\n def decide_callable_or_boolean(value):\n \"\"\"\n Decide whether a key function is used.\n \"\"\"\n if callable(value):\n value, key = True, value\n else:\n key = None\n return value, key\n\n # cmp takes precedence due to bw-compatibility.\n if cmp is not None:\n cmp, cmp_key = decide_callable_or_boolean(cmp)\n return cmp, cmp_key, cmp, cmp_key\n\n # If left None, equality is set to the specified default and ordering\n # mirrors equality.\n if eq is None:\n eq, eq_key = default_eq, None\n else:\n eq, eq_key = decide_callable_or_boolean(eq)\n\n if order is None:\n order, order_key = eq, eq_key\n else:\n order, order_key = decide_callable_or_boolean(order)\n\n if eq is False and order is True:\n raise ValueError(\"`order` can only be True if `eq` is True too.\")\n\n return eq, eq_key, order, order_key\n\n\ndef _determine_whether_to_implement(\n cls, flag, auto_detect, dunders, default=True\n):\n \"\"\"\n Check whether we should implement a set of methods for *cls*.\n\n *flag* is the argument passed into @attr.s like 'init', *auto_detect* the\n same as passed into @attr.s and *dunders* is a tuple of attribute names\n whose presence signal that the user has implemented it themselves.\n\n Return *default* if no reason for either for or against is found.\n \"\"\"\n if flag is True or flag is False:\n return flag\n\n if flag is None and auto_detect is False:\n return default\n\n # Logically, flag is None and auto_detect is True here.\n for dunder in dunders:\n if _has_own_attribute(cls, dunder):\n return False\n\n return default\n\n\ndef attrs(\n maybe_cls=None,\n these=None,\n repr_ns=None,\n repr=None,\n cmp=None,\n hash=None,\n init=None,\n slots=False,\n frozen=False,\n weakref_slot=True,\n str=False,\n auto_attribs=False,\n kw_only=False,\n cache_hash=False,\n auto_exc=False,\n eq=None,\n order=None,\n auto_detect=False,\n collect_by_mro=False,\n getstate_setstate=None,\n on_setattr=None,\n field_transformer=None,\n match_args=True,\n):\n r\"\"\"\n A class decorator that adds `dunder\n `_\\ -methods according to the\n specified attributes using `attr.ib` or the *these* argument.\n\n :param these: A dictionary of name to `attr.ib` mappings. This is\n useful to avoid the definition of your attributes within the class body\n because you can't (e.g. if you want to add ``__repr__`` methods to\n Django models) or don't want to.\n\n If *these* is not ``None``, ``attrs`` will *not* search the class body\n for attributes and will *not* remove any attributes from it.\n\n If *these* is an ordered dict (`dict` on Python 3.6+,\n `collections.OrderedDict` otherwise), the order is deduced from\n the order of the attributes inside *these*. Otherwise the order\n of the definition of the attributes is used.\n\n :type these: `dict` of `str` to `attr.ib`\n\n :param str repr_ns: When using nested classes, there's no way in Python 2\n to automatically detect that. Therefore it's possible to set the\n namespace explicitly for a more meaningful ``repr`` output.\n :param bool auto_detect: Instead of setting the *init*, *repr*, *eq*,\n *order*, and *hash* arguments explicitly, assume they are set to\n ``True`` **unless any** of the involved methods for one of the\n arguments is implemented in the *current* class (i.e. it is *not*\n inherited from some base class).\n\n So for example by implementing ``__eq__`` on a class yourself,\n ``attrs`` will deduce ``eq=False`` and will create *neither*\n ``__eq__`` *nor* ``__ne__`` (but Python classes come with a sensible\n ``__ne__`` by default, so it *should* be enough to only implement\n ``__eq__`` in most cases).\n\n .. warning::\n\n If you prevent ``attrs`` from creating the ordering methods for you\n (``order=False``, e.g. by implementing ``__le__``), it becomes\n *your* responsibility to make sure its ordering is sound. The best\n way is to use the `functools.total_ordering` decorator.\n\n\n Passing ``True`` or ``False`` to *init*, *repr*, *eq*, *order*,\n *cmp*, or *hash* overrides whatever *auto_detect* would determine.\n\n *auto_detect* requires Python 3. Setting it ``True`` on Python 2 raises\n an `attrs.exceptions.PythonTooOldError`.\n\n :param bool repr: Create a ``__repr__`` method with a human readable\n representation of ``attrs`` attributes..\n :param bool str: Create a ``__str__`` method that is identical to\n ``__repr__``. This is usually not necessary except for\n `Exception`\\ s.\n :param Optional[bool] eq: If ``True`` or ``None`` (default), add ``__eq__``\n and ``__ne__`` methods that check two instances for equality.\n\n They compare the instances as if they were tuples of their ``attrs``\n attributes if and only if the types of both classes are *identical*!\n :param Optional[bool] order: If ``True``, add ``__lt__``, ``__le__``,\n ``__gt__``, and ``__ge__`` methods that behave like *eq* above and\n allow instances to be ordered. If ``None`` (default) mirror value of\n *eq*.\n :param Optional[bool] cmp: Setting *cmp* is equivalent to setting *eq*\n and *order* to the same value. Must not be mixed with *eq* or *order*.\n :param Optional[bool] hash: If ``None`` (default), the ``__hash__`` method\n is generated according how *eq* and *frozen* are set.\n\n 1. If *both* are True, ``attrs`` will generate a ``__hash__`` for you.\n 2. If *eq* is True and *frozen* is False, ``__hash__`` will be set to\n None, marking it unhashable (which it is).\n 3. If *eq* is False, ``__hash__`` will be left untouched meaning the\n ``__hash__`` method of the base class will be used (if base class is\n ``object``, this means it will fall back to id-based hashing.).\n\n Although not recommended, you can decide for yourself and force\n ``attrs`` to create one (e.g. if the class is immutable even though you\n didn't freeze it programmatically) by passing ``True`` or not. Both of\n these cases are rather special and should be used carefully.\n\n See our documentation on `hashing`, Python's documentation on\n `object.__hash__`, and the `GitHub issue that led to the default \\\n behavior `_ for more\n details.\n :param bool init: Create a ``__init__`` method that initializes the\n ``attrs`` attributes. Leading underscores are stripped for the argument\n name. If a ``__attrs_pre_init__`` method exists on the class, it will\n be called before the class is initialized. If a ``__attrs_post_init__``\n method exists on the class, it will be called after the class is fully\n initialized.\n\n If ``init`` is ``False``, an ``__attrs_init__`` method will be\n injected instead. This allows you to define a custom ``__init__``\n method that can do pre-init work such as ``super().__init__()``,\n and then call ``__attrs_init__()`` and ``__attrs_post_init__()``.\n :param bool slots: Create a `slotted class ` that's more\n memory-efficient. Slotted classes are generally superior to the default\n dict classes, but have some gotchas you should know about, so we\n encourage you to read the `glossary entry `.\n :param bool frozen: Make instances immutable after initialization. If\n someone attempts to modify a frozen instance,\n `attr.exceptions.FrozenInstanceError` is raised.\n\n .. note::\n\n 1. This is achieved by installing a custom ``__setattr__`` method\n on your class, so you can't implement your own.\n\n 2. True immutability is impossible in Python.\n\n 3. This *does* have a minor a runtime performance `impact\n ` when initializing new instances. In other words:\n ``__init__`` is slightly slower with ``frozen=True``.\n\n 4. If a class is frozen, you cannot modify ``self`` in\n ``__attrs_post_init__`` or a self-written ``__init__``. You can\n circumvent that limitation by using\n ``object.__setattr__(self, \"attribute_name\", value)``.\n\n 5. Subclasses of a frozen class are frozen too.\n\n :param bool weakref_slot: Make instances weak-referenceable. This has no\n effect unless ``slots`` is also enabled.\n :param bool auto_attribs: If ``True``, collect :pep:`526`-annotated\n attributes (Python 3.6 and later only) from the class body.\n\n In this case, you **must** annotate every field. If ``attrs``\n encounters a field that is set to an `attr.ib` but lacks a type\n annotation, an `attr.exceptions.UnannotatedAttributeError` is\n raised. Use ``field_name: typing.Any = attr.ib(...)`` if you don't\n want to set a type.\n\n If you assign a value to those attributes (e.g. ``x: int = 42``), that\n value becomes the default value like if it were passed using\n ``attr.ib(default=42)``. Passing an instance of `attrs.Factory` also\n works as expected in most cases (see warning below).\n\n Attributes annotated as `typing.ClassVar`, and attributes that are\n neither annotated nor set to an `attr.ib` are **ignored**.\n\n .. warning::\n For features that use the attribute name to create decorators (e.g.\n `validators `), you still *must* assign `attr.ib` to\n them. Otherwise Python will either not find the name or try to use\n the default value to call e.g. ``validator`` on it.\n\n These errors can be quite confusing and probably the most common bug\n report on our bug tracker.\n\n :param bool kw_only: Make all attributes keyword-only (Python 3+)\n in the generated ``__init__`` (if ``init`` is ``False``, this\n parameter is ignored).\n :param bool cache_hash: Ensure that the object's hash code is computed\n only once and stored on the object. If this is set to ``True``,\n hashing must be either explicitly or implicitly enabled for this\n class. If the hash code is cached, avoid any reassignments of\n fields involved in hash code computation or mutations of the objects\n those fields point to after object creation. If such changes occur,\n the behavior of the object's hash code is undefined.\n :param bool auto_exc: If the class subclasses `BaseException`\n (which implicitly includes any subclass of any exception), the\n following happens to behave like a well-behaved Python exceptions\n class:\n\n - the values for *eq*, *order*, and *hash* are ignored and the\n instances compare and hash by the instance's ids (N.B. ``attrs`` will\n *not* remove existing implementations of ``__hash__`` or the equality\n methods. It just won't add own ones.),\n - all attributes that are either passed into ``__init__`` or have a\n default value are additionally available as a tuple in the ``args``\n attribute,\n - the value of *str* is ignored leaving ``__str__`` to base classes.\n :param bool collect_by_mro: Setting this to `True` fixes the way ``attrs``\n collects attributes from base classes. The default behavior is\n incorrect in certain cases of multiple inheritance. It should be on by\n default but is kept off for backward-compatibility.\n\n See issue `#428 `_ for\n more details.\n\n :param Optional[bool] getstate_setstate:\n .. note::\n This is usually only interesting for slotted classes and you should\n probably just set *auto_detect* to `True`.\n\n If `True`, ``__getstate__`` and\n ``__setstate__`` are generated and attached to the class. This is\n necessary for slotted classes to be pickleable. If left `None`, it's\n `True` by default for slotted classes and ``False`` for dict classes.\n\n If *auto_detect* is `True`, and *getstate_setstate* is left `None`,\n and **either** ``__getstate__`` or ``__setstate__`` is detected directly\n on the class (i.e. not inherited), it is set to `False` (this is usually\n what you want).\n\n :param on_setattr: A callable that is run whenever the user attempts to set\n an attribute (either by assignment like ``i.x = 42`` or by using\n `setattr` like ``setattr(i, \"x\", 42)``). It receives the same arguments\n as validators: the instance, the attribute that is being modified, and\n the new value.\n\n If no exception is raised, the attribute is set to the return value of\n the callable.\n\n If a list of callables is passed, they're automatically wrapped in an\n `attrs.setters.pipe`.\n :type on_setattr: `callable`, or a list of callables, or `None`, or\n `attrs.setters.NO_OP`\n\n :param Optional[callable] field_transformer:\n A function that is called with the original class object and all\n fields right before ``attrs`` finalizes the class. You can use\n this, e.g., to automatically add converters or validators to\n fields based on their types. See `transform-fields` for more details.\n\n :param bool match_args:\n If `True` (default), set ``__match_args__`` on the class to support\n :pep:`634` (Structural Pattern Matching). It is a tuple of all\n non-keyword-only ``__init__`` parameter names on Python 3.10 and later.\n Ignored on older Python versions.\n\n .. versionadded:: 16.0.0 *slots*\n .. versionadded:: 16.1.0 *frozen*\n .. versionadded:: 16.3.0 *str*\n .. versionadded:: 16.3.0 Support for ``__attrs_post_init__``.\n .. versionchanged:: 17.1.0\n *hash* supports ``None`` as value which is also the default now.\n .. versionadded:: 17.3.0 *auto_attribs*\n .. versionchanged:: 18.1.0\n If *these* is passed, no attributes are deleted from the class body.\n .. versionchanged:: 18.1.0 If *these* is ordered, the order is retained.\n .. versionadded:: 18.2.0 *weakref_slot*\n .. deprecated:: 18.2.0\n ``__lt__``, ``__le__``, ``__gt__``, and ``__ge__`` now raise a\n `DeprecationWarning` if the classes compared are subclasses of\n each other. ``__eq`` and ``__ne__`` never tried to compared subclasses\n to each other.\n .. versionchanged:: 19.2.0\n ``__lt__``, ``__le__``, ``__gt__``, and ``__ge__`` now do not consider\n subclasses comparable anymore.\n .. versionadded:: 18.2.0 *kw_only*\n .. versionadded:: 18.2.0 *cache_hash*\n .. versionadded:: 19.1.0 *auto_exc*\n .. deprecated:: 19.2.0 *cmp* Removal on or after 2021-06-01.\n .. versionadded:: 19.2.0 *eq* and *order*\n .. versionadded:: 20.1.0 *auto_detect*\n .. versionadded:: 20.1.0 *collect_by_mro*\n .. versionadded:: 20.1.0 *getstate_setstate*\n .. versionadded:: 20.1.0 *on_setattr*\n .. versionadded:: 20.3.0 *field_transformer*\n .. versionchanged:: 21.1.0\n ``init=False`` injects ``__attrs_init__``\n .. versionchanged:: 21.1.0 Support for ``__attrs_pre_init__``\n .. versionchanged:: 21.1.0 *cmp* undeprecated\n .. versionadded:: 21.3.0 *match_args*\n \"\"\"\n eq_, order_ = _determine_attrs_eq_order(cmp, eq, order, None)\n hash_ = hash # work around the lack of nonlocal\n\n if isinstance(on_setattr, (list, tuple)):\n on_setattr = setters.pipe(*on_setattr)\n\n def wrap(cls):\n is_frozen = frozen or _has_frozen_base_class(cls)\n is_exc = auto_exc is True and issubclass(cls, BaseException)\n has_own_setattr = auto_detect and _has_own_attribute(\n cls, \"__setattr__\"\n )\n\n if has_own_setattr and is_frozen:\n raise ValueError(\"Can't freeze a class with a custom __setattr__.\")\n\n builder = _ClassBuilder(\n cls,\n these,\n slots,\n is_frozen,\n weakref_slot,\n _determine_whether_to_implement(\n cls,\n getstate_setstate,\n auto_detect,\n (\"__getstate__\", \"__setstate__\"),\n default=slots,\n ),\n auto_attribs,\n kw_only,\n cache_hash,\n is_exc,\n collect_by_mro,\n on_setattr,\n has_own_setattr,\n field_transformer,\n )\n if _determine_whether_to_implement(\n cls, repr, auto_detect, (\"__repr__\",)\n ):\n builder.add_repr(repr_ns)\n if str is True:\n builder.add_str()\n\n eq = _determine_whether_to_implement(\n cls, eq_, auto_detect, (\"__eq__\", \"__ne__\")\n )\n if not is_exc and eq is True:\n builder.add_eq()\n if not is_exc and _determine_whether_to_implement(\n cls, order_, auto_detect, (\"__lt__\", \"__le__\", \"__gt__\", \"__ge__\")\n ):\n builder.add_order()\n\n builder.add_setattr()\n\n if (\n hash_ is None\n and auto_detect is True\n and _has_own_attribute(cls, \"__hash__\")\n ):\n hash = False\n else:\n hash = hash_\n if hash is not True and hash is not False and hash is not None:\n # Can't use `hash in` because 1 == True for example.\n raise TypeError(\n \"Invalid value for hash. Must be True, False, or None.\"\n )\n elif hash is False or (hash is None and eq is False) or is_exc:\n # Don't do anything. Should fall back to __object__'s __hash__\n # which is by id.\n if cache_hash:\n raise TypeError(\n \"Invalid value for cache_hash. To use hash caching,\"\n \" hashing must be either explicitly or implicitly \"\n \"enabled.\"\n )\n elif hash is True or (\n hash is None and eq is True and is_frozen is True\n ):\n # Build a __hash__ if told so, or if it's safe.\n builder.add_hash()\n else:\n # Raise TypeError on attempts to hash.\n if cache_hash:\n raise TypeError(\n \"Invalid value for cache_hash. To use hash caching,\"\n \" hashing must be either explicitly or implicitly \"\n \"enabled.\"\n )\n builder.make_unhashable()\n\n if _determine_whether_to_implement(\n cls, init, auto_detect, (\"__init__\",)\n ):\n builder.add_init()\n else:\n builder.add_attrs_init()\n if cache_hash:\n raise TypeError(\n \"Invalid value for cache_hash. To use hash caching,\"\n \" init must be True.\"\n )\n\n if (\n PY310\n and match_args\n and not _has_own_attribute(cls, \"__match_args__\")\n ):\n builder.add_match_args()\n\n return builder.build_class()\n\n # maybe_cls's type depends on the usage of the decorator. It's a class\n # if it's used as `@attrs` but ``None`` if used as `@attrs()`.\n if maybe_cls is None:\n return wrap\n else:\n return wrap(maybe_cls)\n\n\n_attrs = attrs\n\"\"\"\nInternal alias so we can use it in functions that take an argument called\n*attrs*.\n\"\"\"\n\n\ndef _has_frozen_base_class(cls):\n \"\"\"\n Check whether *cls* has a frozen ancestor by looking at its\n __setattr__.\n \"\"\"\n return cls.__setattr__ is _frozen_setattrs\n\n\ndef _generate_unique_filename(cls, func_name):\n \"\"\"\n Create a \"filename\" suitable for a function being generated.\n \"\"\"\n unique_filename = \"\".format(\n func_name,\n cls.__module__,\n getattr(cls, \"__qualname__\", cls.__name__),\n )\n return unique_filename\n\n\ndef _make_hash(cls, attrs, frozen, cache_hash):\n attrs = tuple(\n a for a in attrs if a.hash is True or (a.hash is None and a.eq is True)\n )\n\n tab = \" \"\n\n unique_filename = _generate_unique_filename(cls, \"hash\")\n type_hash = hash(unique_filename)\n # If eq is custom generated, we need to include the functions in globs\n globs = {}\n\n hash_def = \"def __hash__(self\"\n hash_func = \"hash((\"\n closing_braces = \"))\"\n if not cache_hash:\n hash_def += \"):\"\n else:\n hash_def += \", *\"\n\n hash_def += (\n \", _cache_wrapper=\"\n + \"__import__('attr._make')._make._CacheHashWrapper):\"\n )\n hash_func = \"_cache_wrapper(\" + hash_func\n closing_braces += \")\"\n\n method_lines = [hash_def]\n\n def append_hash_computation_lines(prefix, indent):\n \"\"\"\n Generate the code for actually computing the hash code.\n Below this will either be returned directly or used to compute\n a value which is then cached, depending on the value of cache_hash\n \"\"\"\n\n method_lines.extend(\n [\n indent + prefix + hash_func,\n indent + \" %d,\" % (type_hash,),\n ]\n )\n\n for a in attrs:\n if a.eq_key:\n cmp_name = \"_%s_key\" % (a.name,)\n globs[cmp_name] = a.eq_key\n method_lines.append(\n indent + \" %s(self.%s),\" % (cmp_name, a.name)\n )\n else:\n method_lines.append(indent + \" self.%s,\" % a.name)\n\n method_lines.append(indent + \" \" + closing_braces)\n\n if cache_hash:\n method_lines.append(tab + \"if self.%s is None:\" % _hash_cache_field)\n if frozen:\n append_hash_computation_lines(\n \"object.__setattr__(self, '%s', \" % _hash_cache_field, tab * 2\n )\n method_lines.append(tab * 2 + \")\") # close __setattr__\n else:\n append_hash_computation_lines(\n \"self.%s = \" % _hash_cache_field, tab * 2\n )\n method_lines.append(tab + \"return self.%s\" % _hash_cache_field)\n else:\n append_hash_computation_lines(\"return \", tab)\n\n script = \"\\n\".join(method_lines)\n return _make_method(\"__hash__\", script, unique_filename, globs)\n\n\ndef _add_hash(cls, attrs):\n \"\"\"\n Add a hash method to *cls*.\n \"\"\"\n cls.__hash__ = _make_hash(cls, attrs, frozen=False, cache_hash=False)\n return cls\n\n\ndef _make_ne():\n \"\"\"\n Create __ne__ method.\n \"\"\"\n\n def __ne__(self, other):\n \"\"\"\n Check equality and either forward a NotImplemented or\n return the result negated.\n \"\"\"\n result = self.__eq__(other)\n if result is NotImplemented:\n return NotImplemented\n\n return not result\n\n return __ne__\n\n\ndef _make_eq(cls, attrs):\n \"\"\"\n Create __eq__ method for *cls* with *attrs*.\n \"\"\"\n attrs = [a for a in attrs if a.eq]\n\n unique_filename = _generate_unique_filename(cls, \"eq\")\n lines = [\n \"def __eq__(self, other):\",\n \" if other.__class__ is not self.__class__:\",\n \" return NotImplemented\",\n ]\n\n # We can't just do a big self.x = other.x and... clause due to\n # irregularities like nan == nan is false but (nan,) == (nan,) is true.\n globs = {}\n if attrs:\n lines.append(\" return (\")\n others = [\" ) == (\"]\n for a in attrs:\n if a.eq_key:\n cmp_name = \"_%s_key\" % (a.name,)\n # Add the key function to the global namespace\n # of the evaluated function.\n globs[cmp_name] = a.eq_key\n lines.append(\n \" %s(self.%s),\"\n % (\n cmp_name,\n a.name,\n )\n )\n others.append(\n \" %s(other.%s),\"\n % (\n cmp_name,\n a.name,\n )\n )\n else:\n lines.append(\" self.%s,\" % (a.name,))\n others.append(\" other.%s,\" % (a.name,))\n\n lines += others + [\" )\"]\n else:\n lines.append(\" return True\")\n\n script = \"\\n\".join(lines)\n\n return _make_method(\"__eq__\", script, unique_filename, globs)\n\n\ndef _make_order(cls, attrs):\n \"\"\"\n Create ordering methods for *cls* with *attrs*.\n \"\"\"\n attrs = [a for a in attrs if a.order]\n\n def attrs_to_tuple(obj):\n \"\"\"\n Save us some typing.\n \"\"\"\n return tuple(\n key(value) if key else value\n for value, key in (\n (getattr(obj, a.name), a.order_key) for a in attrs\n )\n )\n\n def __lt__(self, other):\n \"\"\"\n Automatically created by attrs.\n \"\"\"\n if other.__class__ is self.__class__:\n return attrs_to_tuple(self) < attrs_to_tuple(other)\n\n return NotImplemented\n\n def __le__(self, other):\n \"\"\"\n Automatically created by attrs.\n \"\"\"\n if other.__class__ is self.__class__:\n return attrs_to_tuple(self) <= attrs_to_tuple(other)\n\n return NotImplemented\n\n def __gt__(self, other):\n \"\"\"\n Automatically created by attrs.\n \"\"\"\n if other.__class__ is self.__class__:\n return attrs_to_tuple(self) > attrs_to_tuple(other)\n\n return NotImplemented\n\n def __ge__(self, other):\n \"\"\"\n Automatically created by attrs.\n \"\"\"\n if other.__class__ is self.__class__:\n return attrs_to_tuple(self) >= attrs_to_tuple(other)\n\n return NotImplemented\n\n return __lt__, __le__, __gt__, __ge__\n\n\ndef _add_eq(cls, attrs=None):\n \"\"\"\n Add equality methods to *cls* with *attrs*.\n \"\"\"\n if attrs is None:\n attrs = cls.__attrs_attrs__\n\n cls.__eq__ = _make_eq(cls, attrs)\n cls.__ne__ = _make_ne()\n\n return cls\n\n\nif HAS_F_STRINGS:\n\n def _make_repr(attrs, ns, cls):\n unique_filename = _generate_unique_filename(cls, \"repr\")\n # Figure out which attributes to include, and which function to use to\n # format them. The a.repr value can be either bool or a custom\n # callable.\n attr_names_with_reprs = tuple(\n (a.name, (repr if a.repr is True else a.repr), a.init)\n for a in attrs\n if a.repr is not False\n )\n globs = {\n name + \"_repr\": r\n for name, r, _ in attr_names_with_reprs\n if r != repr\n }\n globs[\"_compat\"] = _compat\n globs[\"AttributeError\"] = AttributeError\n globs[\"NOTHING\"] = NOTHING\n attribute_fragments = []\n for name, r, i in attr_names_with_reprs:\n accessor = (\n \"self.\" + name\n if i\n else 'getattr(self, \"' + name + '\", NOTHING)'\n )\n fragment = (\n \"%s={%s!r}\" % (name, accessor)\n if r == repr\n else \"%s={%s_repr(%s)}\" % (name, name, accessor)\n )\n attribute_fragments.append(fragment)\n repr_fragment = \", \".join(attribute_fragments)\n\n if ns is None:\n cls_name_fragment = (\n '{self.__class__.__qualname__.rsplit(\">.\", 1)[-1]}'\n )\n else:\n cls_name_fragment = ns + \".{self.__class__.__name__}\"\n\n lines = [\n \"def __repr__(self):\",\n \" try:\",\n \" already_repring = _compat.repr_context.already_repring\",\n \" except AttributeError:\",\n \" already_repring = {id(self),}\",\n \" _compat.repr_context.already_repring = already_repring\",\n \" else:\",\n \" if id(self) in already_repring:\",\n \" return '...'\",\n \" else:\",\n \" already_repring.add(id(self))\",\n \" try:\",\n \" return f'%s(%s)'\" % (cls_name_fragment, repr_fragment),\n \" finally:\",\n \" already_repring.remove(id(self))\",\n ]\n\n return _make_method(\n \"__repr__\", \"\\n\".join(lines), unique_filename, globs=globs\n )\n\nelse:\n\n def _make_repr(attrs, ns, _):\n \"\"\"\n Make a repr method that includes relevant *attrs*, adding *ns* to the\n full name.\n \"\"\"\n\n # Figure out which attributes to include, and which function to use to\n # format them. The a.repr value can be either bool or a custom\n # callable.\n attr_names_with_reprs = tuple(\n (a.name, repr if a.repr is True else a.repr)\n for a in attrs\n if a.repr is not False\n )\n\n def __repr__(self):\n \"\"\"\n Automatically created by attrs.\n \"\"\"\n try:\n already_repring = _compat.repr_context.already_repring\n except AttributeError:\n already_repring = set()\n _compat.repr_context.already_repring = already_repring\n\n if id(self) in already_repring:\n return \"...\"\n real_cls = self.__class__\n if ns is None:\n class_name = real_cls.__qualname__.rsplit(\">.\", 1)[-1]\n else:\n class_name = ns + \".\" + real_cls.__name__\n\n # Since 'self' remains on the stack (i.e.: strongly referenced)\n # for the duration of this call, it's safe to depend on id(...)\n # stability, and not need to track the instance and therefore\n # worry about properties like weakref- or hash-ability.\n already_repring.add(id(self))\n try:\n result = [class_name, \"(\"]\n first = True\n for name, attr_repr in attr_names_with_reprs:\n if first:\n first = False\n else:\n result.append(\", \")\n result.extend(\n (name, \"=\", attr_repr(getattr(self, name, NOTHING)))\n )\n return \"\".join(result) + \")\"\n finally:\n already_repring.remove(id(self))\n\n return __repr__\n\n\ndef _add_repr(cls, ns=None, attrs=None):\n \"\"\"\n Add a repr method to *cls*.\n \"\"\"\n if attrs is None:\n attrs = cls.__attrs_attrs__\n\n cls.__repr__ = _make_repr(attrs, ns, cls)\n return cls\n\n\ndef fields(cls):\n \"\"\"\n Return the tuple of ``attrs`` attributes for a class.\n\n The tuple also allows accessing the fields by their names (see below for\n examples).\n\n :param type cls: Class to introspect.\n\n :raise TypeError: If *cls* is not a class.\n :raise attr.exceptions.NotAnAttrsClassError: If *cls* is not an ``attrs``\n class.\n\n :rtype: tuple (with name accessors) of `attrs.Attribute`\n\n .. versionchanged:: 16.2.0 Returned tuple allows accessing the fields\n by name.\n \"\"\"\n if not isinstance(cls, type):\n raise TypeError(\"Passed object must be a class.\")\n attrs = getattr(cls, \"__attrs_attrs__\", None)\n if attrs is None:\n raise NotAnAttrsClassError(\n \"{cls!r} is not an attrs-decorated class.\".format(cls=cls)\n )\n return attrs\n\n\ndef fields_dict(cls):\n \"\"\"\n Return an ordered dictionary of ``attrs`` attributes for a class, whose\n keys are the attribute names.\n\n :param type cls: Class to introspect.\n\n :raise TypeError: If *cls* is not a class.\n :raise attr.exceptions.NotAnAttrsClassError: If *cls* is not an ``attrs``\n class.\n\n :rtype: an ordered dict where keys are attribute names and values are\n `attrs.Attribute`\\\\ s. This will be a `dict` if it's\n naturally ordered like on Python 3.6+ or an\n :class:`~collections.OrderedDict` otherwise.\n\n .. versionadded:: 18.1.0\n \"\"\"\n if not isinstance(cls, type):\n raise TypeError(\"Passed object must be a class.\")\n attrs = getattr(cls, \"__attrs_attrs__\", None)\n if attrs is None:\n raise NotAnAttrsClassError(\n \"{cls!r} is not an attrs-decorated class.\".format(cls=cls)\n )\n return ordered_dict((a.name, a) for a in attrs)\n\n\ndef validate(inst):\n \"\"\"\n Validate all attributes on *inst* that have a validator.\n\n Leaves all exceptions through.\n\n :param inst: Instance of a class with ``attrs`` attributes.\n \"\"\"\n if _config._run_validators is False:\n return\n\n for a in fields(inst.__class__):\n v = a.validator\n if v is not None:\n v(inst, a, getattr(inst, a.name))\n\n\ndef _is_slot_cls(cls):\n return \"__slots__\" in cls.__dict__\n\n\ndef _is_slot_attr(a_name, base_attr_map):\n \"\"\"\n Check if the attribute name comes from a slot class.\n \"\"\"\n return a_name in base_attr_map and _is_slot_cls(base_attr_map[a_name])\n\n\ndef _make_init(\n cls,\n attrs,\n pre_init,\n post_init,\n frozen,\n slots,\n cache_hash,\n base_attr_map,\n is_exc,\n cls_on_setattr,\n attrs_init,\n):\n has_cls_on_setattr = (\n cls_on_setattr is not None and cls_on_setattr is not setters.NO_OP\n )\n\n if frozen and has_cls_on_setattr:\n raise ValueError(\"Frozen classes can't use on_setattr.\")\n\n needs_cached_setattr = cache_hash or frozen\n filtered_attrs = []\n attr_dict = {}\n for a in attrs:\n if not a.init and a.default is NOTHING:\n continue\n\n filtered_attrs.append(a)\n attr_dict[a.name] = a\n\n if a.on_setattr is not None:\n if frozen is True:\n raise ValueError(\"Frozen classes can't use on_setattr.\")\n\n needs_cached_setattr = True\n elif has_cls_on_setattr and a.on_setattr is not setters.NO_OP:\n needs_cached_setattr = True\n\n unique_filename = _generate_unique_filename(cls, \"init\")\n\n script, globs, annotations = _attrs_to_init_script(\n filtered_attrs,\n frozen,\n slots,\n pre_init,\n post_init,\n cache_hash,\n base_attr_map,\n is_exc,\n has_cls_on_setattr,\n attrs_init,\n )\n if cls.__module__ in sys.modules:\n # This makes typing.get_type_hints(CLS.__init__) resolve string types.\n globs.update(sys.modules[cls.__module__].__dict__)\n\n globs.update({\"NOTHING\": NOTHING, \"attr_dict\": attr_dict})\n\n if needs_cached_setattr:\n # Save the lookup overhead in __init__ if we need to circumvent\n # setattr hooks.\n globs[\"_setattr\"] = _obj_setattr\n\n init = _make_method(\n \"__attrs_init__\" if attrs_init else \"__init__\",\n script,\n unique_filename,\n globs,\n )\n init.__annotations__ = annotations\n\n return init\n\n\ndef _setattr(attr_name, value_var, has_on_setattr):\n \"\"\"\n Use the cached object.setattr to set *attr_name* to *value_var*.\n \"\"\"\n return \"_setattr(self, '%s', %s)\" % (attr_name, value_var)\n\n\ndef _setattr_with_converter(attr_name, value_var, has_on_setattr):\n \"\"\"\n Use the cached object.setattr to set *attr_name* to *value_var*, but run\n its converter first.\n \"\"\"\n return \"_setattr(self, '%s', %s(%s))\" % (\n attr_name,\n _init_converter_pat % (attr_name,),\n value_var,\n )\n\n\ndef _assign(attr_name, value, has_on_setattr):\n \"\"\"\n Unless *attr_name* has an on_setattr hook, use normal assignment. Otherwise\n relegate to _setattr.\n \"\"\"\n if has_on_setattr:\n return _setattr(attr_name, value, True)\n\n return \"self.%s = %s\" % (attr_name, value)\n\n\ndef _assign_with_converter(attr_name, value_var, has_on_setattr):\n \"\"\"\n Unless *attr_name* has an on_setattr hook, use normal assignment after\n conversion. Otherwise relegate to _setattr_with_converter.\n \"\"\"\n if has_on_setattr:\n return _setattr_with_converter(attr_name, value_var, True)\n\n return \"self.%s = %s(%s)\" % (\n attr_name,\n _init_converter_pat % (attr_name,),\n value_var,\n )\n\n\ndef _attrs_to_init_script(\n attrs,\n frozen,\n slots,\n pre_init,\n post_init,\n cache_hash,\n base_attr_map,\n is_exc,\n has_cls_on_setattr,\n attrs_init,\n):\n \"\"\"\n Return a script of an initializer for *attrs* and a dict of globals.\n\n The globals are expected by the generated script.\n\n If *frozen* is True, we cannot set the attributes directly so we use\n a cached ``object.__setattr__``.\n \"\"\"\n lines = []\n if pre_init:\n lines.append(\"self.__attrs_pre_init__()\")\n\n if frozen is True:\n if slots is True:\n fmt_setter = _setattr\n fmt_setter_with_converter = _setattr_with_converter\n else:\n # Dict frozen classes assign directly to __dict__.\n # But only if the attribute doesn't come from an ancestor slot\n # class.\n # Note _inst_dict will be used again below if cache_hash is True\n lines.append(\"_inst_dict = self.__dict__\")\n\n def fmt_setter(attr_name, value_var, has_on_setattr):\n if _is_slot_attr(attr_name, base_attr_map):\n return _setattr(attr_name, value_var, has_on_setattr)\n\n return \"_inst_dict['%s'] = %s\" % (attr_name, value_var)\n\n def fmt_setter_with_converter(\n attr_name, value_var, has_on_setattr\n ):\n if has_on_setattr or _is_slot_attr(attr_name, base_attr_map):\n return _setattr_with_converter(\n attr_name, value_var, has_on_setattr\n )\n\n return \"_inst_dict['%s'] = %s(%s)\" % (\n attr_name,\n _init_converter_pat % (attr_name,),\n value_var,\n )\n\n else:\n # Not frozen.\n fmt_setter = _assign\n fmt_setter_with_converter = _assign_with_converter\n\n args = []\n kw_only_args = []\n attrs_to_validate = []\n\n # This is a dictionary of names to validator and converter callables.\n # Injecting this into __init__ globals lets us avoid lookups.\n names_for_globals = {}\n annotations = {\"return\": None}\n\n for a in attrs:\n if a.validator:\n attrs_to_validate.append(a)\n\n attr_name = a.name\n has_on_setattr = a.on_setattr is not None or (\n a.on_setattr is not setters.NO_OP and has_cls_on_setattr\n )\n arg_name = a.name.lstrip(\"_\")\n\n has_factory = isinstance(a.default, Factory)\n if has_factory and a.default.takes_self:\n maybe_self = \"self\"\n else:\n maybe_self = \"\"\n\n if a.init is False:\n if has_factory:\n init_factory_name = _init_factory_pat.format(a.name)\n if a.converter is not None:\n lines.append(\n fmt_setter_with_converter(\n attr_name,\n init_factory_name + \"(%s)\" % (maybe_self,),\n has_on_setattr,\n )\n )\n conv_name = _init_converter_pat % (a.name,)\n names_for_globals[conv_name] = a.converter\n else:\n lines.append(\n fmt_setter(\n attr_name,\n init_factory_name + \"(%s)\" % (maybe_self,),\n has_on_setattr,\n )\n )\n names_for_globals[init_factory_name] = a.default.factory\n else:\n if a.converter is not None:\n lines.append(\n fmt_setter_with_converter(\n attr_name,\n \"attr_dict['%s'].default\" % (attr_name,),\n has_on_setattr,\n )\n )\n conv_name = _init_converter_pat % (a.name,)\n names_for_globals[conv_name] = a.converter\n else:\n lines.append(\n fmt_setter(\n attr_name,\n \"attr_dict['%s'].default\" % (attr_name,),\n has_on_setattr,\n )\n )\n elif a.default is not NOTHING and not has_factory:\n arg = \"%s=attr_dict['%s'].default\" % (arg_name, attr_name)\n if a.kw_only:\n kw_only_args.append(arg)\n else:\n args.append(arg)\n\n if a.converter is not None:\n lines.append(\n fmt_setter_with_converter(\n attr_name, arg_name, has_on_setattr\n )\n )\n names_for_globals[\n _init_converter_pat % (a.name,)\n ] = a.converter\n else:\n lines.append(fmt_setter(attr_name, arg_name, has_on_setattr))\n\n elif has_factory:\n arg = \"%s=NOTHING\" % (arg_name,)\n if a.kw_only:\n kw_only_args.append(arg)\n else:\n args.append(arg)\n lines.append(\"if %s is not NOTHING:\" % (arg_name,))\n\n init_factory_name = _init_factory_pat.format(a.name)\n if a.converter is not None:\n lines.append(\n \" \"\n + fmt_setter_with_converter(\n attr_name, arg_name, has_on_setattr\n )\n )\n lines.append(\"else:\")\n lines.append(\n \" \"\n + fmt_setter_with_converter(\n attr_name,\n init_factory_name + \"(\" + maybe_self + \")\",\n has_on_setattr,\n )\n )\n names_for_globals[\n _init_converter_pat % (a.name,)\n ] = a.converter\n else:\n lines.append(\n \" \" + fmt_setter(attr_name, arg_name, has_on_setattr)\n )\n lines.append(\"else:\")\n lines.append(\n \" \"\n + fmt_setter(\n attr_name,\n init_factory_name + \"(\" + maybe_self + \")\",\n has_on_setattr,\n )\n )\n names_for_globals[init_factory_name] = a.default.factory\n else:\n if a.kw_only:\n kw_only_args.append(arg_name)\n else:\n args.append(arg_name)\n\n if a.converter is not None:\n lines.append(\n fmt_setter_with_converter(\n attr_name, arg_name, has_on_setattr\n )\n )\n names_for_globals[\n _init_converter_pat % (a.name,)\n ] = a.converter\n else:\n lines.append(fmt_setter(attr_name, arg_name, has_on_setattr))\n\n if a.init is True:\n if a.type is not None and a.converter is None:\n annotations[arg_name] = a.type\n elif a.converter is not None:\n # Try to get the type from the converter.\n t = _AnnotationExtractor(a.converter).get_first_param_type()\n if t:\n annotations[arg_name] = t\n\n if attrs_to_validate: # we can skip this if there are no validators.\n names_for_globals[\"_config\"] = _config\n lines.append(\"if _config._run_validators is True:\")\n for a in attrs_to_validate:\n val_name = \"__attr_validator_\" + a.name\n attr_name = \"__attr_\" + a.name\n lines.append(\n \" %s(self, %s, self.%s)\" % (val_name, attr_name, a.name)\n )\n names_for_globals[val_name] = a.validator\n names_for_globals[attr_name] = a\n\n if post_init:\n lines.append(\"self.__attrs_post_init__()\")\n\n # because this is set only after __attrs_post_init__ is called, a crash\n # will result if post-init tries to access the hash code. This seemed\n # preferable to setting this beforehand, in which case alteration to\n # field values during post-init combined with post-init accessing the\n # hash code would result in silent bugs.\n if cache_hash:\n if frozen:\n if slots:\n # if frozen and slots, then _setattr defined above\n init_hash_cache = \"_setattr(self, '%s', %s)\"\n else:\n # if frozen and not slots, then _inst_dict defined above\n init_hash_cache = \"_inst_dict['%s'] = %s\"\n else:\n init_hash_cache = \"self.%s = %s\"\n lines.append(init_hash_cache % (_hash_cache_field, \"None\"))\n\n # For exceptions we rely on BaseException.__init__ for proper\n # initialization.\n if is_exc:\n vals = \",\".join(\"self.\" + a.name for a in attrs if a.init)\n\n lines.append(\"BaseException.__init__(self, %s)\" % (vals,))\n\n args = \", \".join(args)\n if kw_only_args:\n args += \"%s*, %s\" % (\n \", \" if args else \"\", # leading comma\n \", \".join(kw_only_args), # kw_only args\n )\n return (\n \"\"\"\\\ndef {init_name}(self, {args}):\n {lines}\n\"\"\".format(\n init_name=(\"__attrs_init__\" if attrs_init else \"__init__\"),\n args=args,\n lines=\"\\n \".join(lines) if lines else \"pass\",\n ),\n names_for_globals,\n annotations,\n )\n\n\nclass Attribute:\n \"\"\"\n *Read-only* representation of an attribute.\n\n The class has *all* arguments of `attr.ib` (except for ``factory``\n which is only syntactic sugar for ``default=Factory(...)`` plus the\n following:\n\n - ``name`` (`str`): The name of the attribute.\n - ``inherited`` (`bool`): Whether or not that attribute has been inherited\n from a base class.\n - ``eq_key`` and ``order_key`` (`typing.Callable` or `None`): The callables\n that are used for comparing and ordering objects by this attribute,\n respectively. These are set by passing a callable to `attr.ib`'s ``eq``,\n ``order``, or ``cmp`` arguments. See also :ref:`comparison customization\n `.\n\n Instances of this class are frequently used for introspection purposes\n like:\n\n - `fields` returns a tuple of them.\n - Validators get them passed as the first argument.\n - The :ref:`field transformer ` hook receives a list of\n them.\n\n .. versionadded:: 20.1.0 *inherited*\n .. versionadded:: 20.1.0 *on_setattr*\n .. versionchanged:: 20.2.0 *inherited* is not taken into account for\n equality checks and hashing anymore.\n .. versionadded:: 21.1.0 *eq_key* and *order_key*\n\n For the full version history of the fields, see `attr.ib`.\n \"\"\"\n\n __slots__ = (\n \"name\",\n \"default\",\n \"validator\",\n \"repr\",\n \"eq\",\n \"eq_key\",\n \"order\",\n \"order_key\",\n \"hash\",\n \"init\",\n \"metadata\",\n \"type\",\n \"converter\",\n \"kw_only\",\n \"inherited\",\n \"on_setattr\",\n )\n\n def __init__(\n self,\n name,\n default,\n validator,\n repr,\n cmp, # XXX: unused, remove along with other cmp code.\n hash,\n init,\n inherited,\n metadata=None,\n type=None,\n converter=None,\n kw_only=False,\n eq=None,\n eq_key=None,\n order=None,\n order_key=None,\n on_setattr=None,\n ):\n eq, eq_key, order, order_key = _determine_attrib_eq_order(\n cmp, eq_key or eq, order_key or order, True\n )\n\n # Cache this descriptor here to speed things up later.\n bound_setattr = _obj_setattr.__get__(self, Attribute)\n\n # Despite the big red warning, people *do* instantiate `Attribute`\n # themselves.\n bound_setattr(\"name\", name)\n bound_setattr(\"default\", default)\n bound_setattr(\"validator\", validator)\n bound_setattr(\"repr\", repr)\n bound_setattr(\"eq\", eq)\n bound_setattr(\"eq_key\", eq_key)\n bound_setattr(\"order\", order)\n bound_setattr(\"order_key\", order_key)\n bound_setattr(\"hash\", hash)\n bound_setattr(\"init\", init)\n bound_setattr(\"converter\", converter)\n bound_setattr(\n \"metadata\",\n (\n types.MappingProxyType(dict(metadata)) # Shallow copy\n if metadata\n else _empty_metadata_singleton\n ),\n )\n bound_setattr(\"type\", type)\n bound_setattr(\"kw_only\", kw_only)\n bound_setattr(\"inherited\", inherited)\n bound_setattr(\"on_setattr\", on_setattr)\n\n def __setattr__(self, name, value):\n raise FrozenInstanceError()\n\n @classmethod\n def from_counting_attr(cls, name, ca, type=None):\n # type holds the annotated value. deal with conflicts:\n if type is None:\n type = ca.type\n elif ca.type is not None:\n raise ValueError(\n \"Type annotation and type argument cannot both be present\"\n )\n inst_dict = {\n k: getattr(ca, k)\n for k in Attribute.__slots__\n if k\n not in (\n \"name\",\n \"validator\",\n \"default\",\n \"type\",\n \"inherited\",\n ) # exclude methods and deprecated alias\n }\n return cls(\n name=name,\n validator=ca._validator,\n default=ca._default,\n type=type,\n cmp=None,\n inherited=False,\n **inst_dict\n )\n\n # Don't use attr.evolve since fields(Attribute) doesn't work\n def evolve(self, **changes):\n \"\"\"\n Copy *self* and apply *changes*.\n\n This works similarly to `attr.evolve` but that function does not work\n with ``Attribute``.\n\n It is mainly meant to be used for `transform-fields`.\n\n .. versionadded:: 20.3.0\n \"\"\"\n new = copy.copy(self)\n\n new._setattrs(changes.items())\n\n return new\n\n # Don't use _add_pickle since fields(Attribute) doesn't work\n def __getstate__(self):\n \"\"\"\n Play nice with pickle.\n \"\"\"\n return tuple(\n getattr(self, name) if name != \"metadata\" else dict(self.metadata)\n for name in self.__slots__\n )\n\n def __setstate__(self, state):\n \"\"\"\n Play nice with pickle.\n \"\"\"\n self._setattrs(zip(self.__slots__, state))\n\n def _setattrs(self, name_values_pairs):\n bound_setattr = _obj_setattr.__get__(self, Attribute)\n for name, value in name_values_pairs:\n if name != \"metadata\":\n bound_setattr(name, value)\n else:\n bound_setattr(\n name,\n types.MappingProxyType(dict(value))\n if value\n else _empty_metadata_singleton,\n )\n\n\n_a = [\n Attribute(\n name=name,\n default=NOTHING,\n validator=None,\n repr=True,\n cmp=None,\n eq=True,\n order=False,\n hash=(name != \"metadata\"),\n init=True,\n inherited=False,\n )\n for name in Attribute.__slots__\n]\n\nAttribute = _add_hash(\n _add_eq(\n _add_repr(Attribute, attrs=_a),\n attrs=[a for a in _a if a.name != \"inherited\"],\n ),\n attrs=[a for a in _a if a.hash and a.name != \"inherited\"],\n)\n\n\nclass _CountingAttr:\n \"\"\"\n Intermediate representation of attributes that uses a counter to preserve\n the order in which the attributes have been defined.\n\n *Internal* data structure of the attrs library. Running into is most\n likely the result of a bug like a forgotten `@attr.s` decorator.\n \"\"\"\n\n __slots__ = (\n \"counter\",\n \"_default\",\n \"repr\",\n \"eq\",\n \"eq_key\",\n \"order\",\n \"order_key\",\n \"hash\",\n \"init\",\n \"metadata\",\n \"_validator\",\n \"converter\",\n \"type\",\n \"kw_only\",\n \"on_setattr\",\n )\n __attrs_attrs__ = tuple(\n Attribute(\n name=name,\n default=NOTHING,\n validator=None,\n repr=True,\n cmp=None,\n hash=True,\n init=True,\n kw_only=False,\n eq=True,\n eq_key=None,\n order=False,\n order_key=None,\n inherited=False,\n on_setattr=None,\n )\n for name in (\n \"counter\",\n \"_default\",\n \"repr\",\n \"eq\",\n \"order\",\n \"hash\",\n \"init\",\n \"on_setattr\",\n )\n ) + (\n Attribute(\n name=\"metadata\",\n default=None,\n validator=None,\n repr=True,\n cmp=None,\n hash=False,\n init=True,\n kw_only=False,\n eq=True,\n eq_key=None,\n order=False,\n order_key=None,\n inherited=False,\n on_setattr=None,\n ),\n )\n cls_counter = 0\n\n def __init__(\n self,\n default,\n validator,\n repr,\n cmp,\n hash,\n init,\n converter,\n metadata,\n type,\n kw_only,\n eq,\n eq_key,\n order,\n order_key,\n on_setattr,\n ):\n _CountingAttr.cls_counter += 1\n self.counter = _CountingAttr.cls_counter\n self._default = default\n self._validator = validator\n self.converter = converter\n self.repr = repr\n self.eq = eq\n self.eq_key = eq_key\n self.order = order\n self.order_key = order_key\n self.hash = hash\n self.init = init\n self.metadata = metadata\n self.type = type\n self.kw_only = kw_only\n self.on_setattr = on_setattr\n\n def validator(self, meth):\n \"\"\"\n Decorator that adds *meth* to the list of validators.\n\n Returns *meth* unchanged.\n\n .. versionadded:: 17.1.0\n \"\"\"\n if self._validator is None:\n self._validator = meth\n else:\n self._validator = and_(self._validator, meth)\n return meth\n\n def default(self, meth):\n \"\"\"\n Decorator that allows to set the default for an attribute.\n\n Returns *meth* unchanged.\n\n :raises DefaultAlreadySetError: If default has been set before.\n\n .. versionadded:: 17.1.0\n \"\"\"\n if self._default is not NOTHING:\n raise DefaultAlreadySetError()\n\n self._default = Factory(meth, takes_self=True)\n\n return meth\n\n\n_CountingAttr = _add_eq(_add_repr(_CountingAttr))\n\n\nclass Factory:\n \"\"\"\n Stores a factory callable.\n\n If passed as the default value to `attrs.field`, the factory is used to\n generate a new value.\n\n :param callable factory: A callable that takes either none or exactly one\n mandatory positional argument depending on *takes_self*.\n :param bool takes_self: Pass the partially initialized instance that is\n being initialized as a positional argument.\n\n .. versionadded:: 17.1.0 *takes_self*\n \"\"\"\n\n __slots__ = (\"factory\", \"takes_self\")\n\n def __init__(self, factory, takes_self=False):\n \"\"\"\n `Factory` is part of the default machinery so if we want a default\n value here, we have to implement it ourselves.\n \"\"\"\n self.factory = factory\n self.takes_self = takes_self\n\n def __getstate__(self):\n \"\"\"\n Play nice with pickle.\n \"\"\"\n return tuple(getattr(self, name) for name in self.__slots__)\n\n def __setstate__(self, state):\n \"\"\"\n Play nice with pickle.\n \"\"\"\n for name, value in zip(self.__slots__, state):\n setattr(self, name, value)\n\n\n_f = [\n Attribute(\n name=name,\n default=NOTHING,\n validator=None,\n repr=True,\n cmp=None,\n eq=True,\n order=False,\n hash=True,\n init=True,\n inherited=False,\n )\n for name in Factory.__slots__\n]\n\nFactory = _add_hash(_add_eq(_add_repr(Factory, attrs=_f), attrs=_f), attrs=_f)\n\n\ndef make_class(name, attrs, bases=(object,), **attributes_arguments):\n \"\"\"\n A quick way to create a new class called *name* with *attrs*.\n\n :param str name: The name for the new class.\n\n :param attrs: A list of names or a dictionary of mappings of names to\n attributes.\n\n If *attrs* is a list or an ordered dict (`dict` on Python 3.6+,\n `collections.OrderedDict` otherwise), the order is deduced from\n the order of the names or attributes inside *attrs*. Otherwise the\n order of the definition of the attributes is used.\n :type attrs: `list` or `dict`\n\n :param tuple bases: Classes that the new class will subclass.\n\n :param attributes_arguments: Passed unmodified to `attr.s`.\n\n :return: A new class with *attrs*.\n :rtype: type\n\n .. versionadded:: 17.1.0 *bases*\n .. versionchanged:: 18.1.0 If *attrs* is ordered, the order is retained.\n \"\"\"\n if isinstance(attrs, dict):\n cls_dict = attrs\n elif isinstance(attrs, (list, tuple)):\n cls_dict = {a: attrib() for a in attrs}\n else:\n raise TypeError(\"attrs argument must be a dict or a list.\")\n\n pre_init = cls_dict.pop(\"__attrs_pre_init__\", None)\n post_init = cls_dict.pop(\"__attrs_post_init__\", None)\n user_init = cls_dict.pop(\"__init__\", None)\n\n body = {}\n if pre_init is not None:\n body[\"__attrs_pre_init__\"] = pre_init\n if post_init is not None:\n body[\"__attrs_post_init__\"] = post_init\n if user_init is not None:\n body[\"__init__\"] = user_init\n\n type_ = types.new_class(name, bases, {}, lambda ns: ns.update(body))\n\n # For pickling to work, the __module__ variable needs to be set to the\n # frame where the class is created. Bypass this step in environments where\n # sys._getframe is not defined (Jython for example) or sys._getframe is not\n # defined for arguments greater than 0 (IronPython).\n try:\n type_.__module__ = sys._getframe(1).f_globals.get(\n \"__name__\", \"__main__\"\n )\n except (AttributeError, ValueError):\n pass\n\n # We do it here for proper warnings with meaningful stacklevel.\n cmp = attributes_arguments.pop(\"cmp\", None)\n (\n attributes_arguments[\"eq\"],\n attributes_arguments[\"order\"],\n ) = _determine_attrs_eq_order(\n cmp,\n attributes_arguments.get(\"eq\"),\n attributes_arguments.get(\"order\"),\n True,\n )\n\n return _attrs(these=cls_dict, **attributes_arguments)(type_)\n\n\n# These are required by within this module so we define them here and merely\n# import into .validators / .converters.\n\n\n@attrs(slots=True, hash=True)\nclass _AndValidator:\n \"\"\"\n Compose many validators to a single one.\n \"\"\"\n\n _validators = attrib()\n\n def __call__(self, inst, attr, value):\n for v in self._validators:\n v(inst, attr, value)\n\n\ndef and_(*validators):\n \"\"\"\n A validator that composes multiple validators into one.\n\n When called on a value, it runs all wrapped validators.\n\n :param callables validators: Arbitrary number of validators.\n\n .. versionadded:: 17.1.0\n \"\"\"\n vals = []\n for validator in validators:\n vals.extend(\n validator._validators\n if isinstance(validator, _AndValidator)\n else [validator]\n )\n\n return _AndValidator(tuple(vals))\n\n\ndef pipe(*converters):\n \"\"\"\n A converter that composes multiple converters into one.\n\n When called on a value, it runs all wrapped converters, returning the\n *last* value.\n\n Type annotations will be inferred from the wrapped converters', if\n they have any.\n\n :param callables converters: Arbitrary number of converters.\n\n .. versionadded:: 20.1.0\n \"\"\"\n\n def pipe_converter(val):\n for converter in converters:\n val = converter(val)\n\n return val\n\n if not converters:\n # If the converter list is empty, pipe_converter is the identity.\n A = typing.TypeVar(\"A\")\n pipe_converter.__annotations__ = {\"val\": A, \"return\": A}\n else:\n # Get parameter type from first converter.\n t = _AnnotationExtractor(converters[0]).get_first_param_type()\n if t:\n pipe_converter.__annotations__[\"val\"] = t\n\n # Get return type from last converter.\n rt = _AnnotationExtractor(converters[-1]).get_return_type()\n if rt:\n pipe_converter.__annotations__[\"return\"] = rt\n\n return pipe_converter\n" }, { "path": "lib/spack/spack/vendor/attr/_next_gen.py", "content": "# SPDX-License-Identifier: MIT\n\n\"\"\"\nThese are Python 3.6+-only and keyword-only APIs that call `attr.s` and\n`attr.ib` with different default values.\n\"\"\"\n\n\nfrom functools import partial\n\nfrom . import setters\nfrom ._funcs import asdict as _asdict\nfrom ._funcs import astuple as _astuple\nfrom ._make import (\n NOTHING,\n _frozen_setattrs,\n _ng_default_on_setattr,\n attrib,\n attrs,\n)\nfrom .exceptions import UnannotatedAttributeError\n\n\ndef define(\n maybe_cls=None,\n *,\n these=None,\n repr=None,\n hash=None,\n init=None,\n slots=True,\n frozen=False,\n weakref_slot=True,\n str=False,\n auto_attribs=None,\n kw_only=False,\n cache_hash=False,\n auto_exc=True,\n eq=None,\n order=False,\n auto_detect=True,\n getstate_setstate=None,\n on_setattr=None,\n field_transformer=None,\n match_args=True,\n):\n r\"\"\"\n Define an ``attrs`` class.\n\n Differences to the classic `attr.s` that it uses underneath:\n\n - Automatically detect whether or not *auto_attribs* should be `True` (c.f.\n *auto_attribs* parameter).\n - If *frozen* is `False`, run converters and validators when setting an\n attribute by default.\n - *slots=True*\n\n .. caution::\n\n Usually this has only upsides and few visible effects in everyday\n programming. But it *can* lead to some suprising behaviors, so please\n make sure to read :term:`slotted classes`.\n - *auto_exc=True*\n - *auto_detect=True*\n - *order=False*\n - Some options that were only relevant on Python 2 or were kept around for\n backwards-compatibility have been removed.\n\n Please note that these are all defaults and you can change them as you\n wish.\n\n :param Optional[bool] auto_attribs: If set to `True` or `False`, it behaves\n exactly like `attr.s`. If left `None`, `attr.s` will try to guess:\n\n 1. If any attributes are annotated and no unannotated `attrs.fields`\\ s\n are found, it assumes *auto_attribs=True*.\n 2. Otherwise it assumes *auto_attribs=False* and tries to collect\n `attrs.fields`\\ s.\n\n For now, please refer to `attr.s` for the rest of the parameters.\n\n .. versionadded:: 20.1.0\n .. versionchanged:: 21.3.0 Converters are also run ``on_setattr``.\n \"\"\"\n\n def do_it(cls, auto_attribs):\n return attrs(\n maybe_cls=cls,\n these=these,\n repr=repr,\n hash=hash,\n init=init,\n slots=slots,\n frozen=frozen,\n weakref_slot=weakref_slot,\n str=str,\n auto_attribs=auto_attribs,\n kw_only=kw_only,\n cache_hash=cache_hash,\n auto_exc=auto_exc,\n eq=eq,\n order=order,\n auto_detect=auto_detect,\n collect_by_mro=True,\n getstate_setstate=getstate_setstate,\n on_setattr=on_setattr,\n field_transformer=field_transformer,\n match_args=match_args,\n )\n\n def wrap(cls):\n \"\"\"\n Making this a wrapper ensures this code runs during class creation.\n\n We also ensure that frozen-ness of classes is inherited.\n \"\"\"\n nonlocal frozen, on_setattr\n\n had_on_setattr = on_setattr not in (None, setters.NO_OP)\n\n # By default, mutable classes convert & validate on setattr.\n if frozen is False and on_setattr is None:\n on_setattr = _ng_default_on_setattr\n\n # However, if we subclass a frozen class, we inherit the immutability\n # and disable on_setattr.\n for base_cls in cls.__bases__:\n if base_cls.__setattr__ is _frozen_setattrs:\n if had_on_setattr:\n raise ValueError(\n \"Frozen classes can't use on_setattr \"\n \"(frozen-ness was inherited).\"\n )\n\n on_setattr = setters.NO_OP\n break\n\n if auto_attribs is not None:\n return do_it(cls, auto_attribs)\n\n try:\n return do_it(cls, True)\n except UnannotatedAttributeError:\n return do_it(cls, False)\n\n # maybe_cls's type depends on the usage of the decorator. It's a class\n # if it's used as `@attrs` but ``None`` if used as `@attrs()`.\n if maybe_cls is None:\n return wrap\n else:\n return wrap(maybe_cls)\n\n\nmutable = define\nfrozen = partial(define, frozen=True, on_setattr=None)\n\n\ndef field(\n *,\n default=NOTHING,\n validator=None,\n repr=True,\n hash=None,\n init=True,\n metadata=None,\n converter=None,\n factory=None,\n kw_only=False,\n eq=None,\n order=None,\n on_setattr=None,\n):\n \"\"\"\n Identical to `attr.ib`, except keyword-only and with some arguments\n removed.\n\n .. versionadded:: 20.1.0\n \"\"\"\n return attrib(\n default=default,\n validator=validator,\n repr=repr,\n hash=hash,\n init=init,\n metadata=metadata,\n converter=converter,\n factory=factory,\n kw_only=kw_only,\n eq=eq,\n order=order,\n on_setattr=on_setattr,\n )\n\n\ndef asdict(inst, *, recurse=True, filter=None, value_serializer=None):\n \"\"\"\n Same as `attr.asdict`, except that collections types are always retained\n and dict is always used as *dict_factory*.\n\n .. versionadded:: 21.3.0\n \"\"\"\n return _asdict(\n inst=inst,\n recurse=recurse,\n filter=filter,\n value_serializer=value_serializer,\n retain_collection_types=True,\n )\n\n\ndef astuple(inst, *, recurse=True, filter=None):\n \"\"\"\n Same as `attr.astuple`, except that collections types are always retained\n and `tuple` is always used as the *tuple_factory*.\n\n .. versionadded:: 21.3.0\n \"\"\"\n return _astuple(\n inst=inst, recurse=recurse, filter=filter, retain_collection_types=True\n )\n" }, { "path": "lib/spack/spack/vendor/attr/_version_info.py", "content": "# SPDX-License-Identifier: MIT\n\n\nfrom functools import total_ordering\n\nfrom ._funcs import astuple\nfrom ._make import attrib, attrs\n\n\n@total_ordering\n@attrs(eq=False, order=False, slots=True, frozen=True)\nclass VersionInfo:\n \"\"\"\n A version object that can be compared to tuple of length 1--4:\n\n >>> attr.VersionInfo(19, 1, 0, \"final\") <= (19, 2)\n True\n >>> attr.VersionInfo(19, 1, 0, \"final\") < (19, 1, 1)\n True\n >>> vi = attr.VersionInfo(19, 2, 0, \"final\")\n >>> vi < (19, 1, 1)\n False\n >>> vi < (19,)\n False\n >>> vi == (19, 2,)\n True\n >>> vi == (19, 2, 1)\n False\n\n .. versionadded:: 19.2\n \"\"\"\n\n year = attrib(type=int)\n minor = attrib(type=int)\n micro = attrib(type=int)\n releaselevel = attrib(type=str)\n\n @classmethod\n def _from_version_string(cls, s):\n \"\"\"\n Parse *s* and return a _VersionInfo.\n \"\"\"\n v = s.split(\".\")\n if len(v) == 3:\n v.append(\"final\")\n\n return cls(\n year=int(v[0]), minor=int(v[1]), micro=int(v[2]), releaselevel=v[3]\n )\n\n def _ensure_tuple(self, other):\n \"\"\"\n Ensure *other* is a tuple of a valid length.\n\n Returns a possibly transformed *other* and ourselves as a tuple of\n the same length as *other*.\n \"\"\"\n\n if self.__class__ is other.__class__:\n other = astuple(other)\n\n if not isinstance(other, tuple):\n raise NotImplementedError\n\n if not (1 <= len(other) <= 4):\n raise NotImplementedError\n\n return astuple(self)[: len(other)], other\n\n def __eq__(self, other):\n try:\n us, them = self._ensure_tuple(other)\n except NotImplementedError:\n return NotImplemented\n\n return us == them\n\n def __lt__(self, other):\n try:\n us, them = self._ensure_tuple(other)\n except NotImplementedError:\n return NotImplemented\n\n # Since alphabetically \"dev0\" < \"final\" < \"post1\" < \"post2\", we don't\n # have to do anything special with releaselevel for now.\n return us < them\n" }, { "path": "lib/spack/spack/vendor/attr/_version_info.pyi", "content": "class VersionInfo:\n @property\n def year(self) -> int: ...\n @property\n def minor(self) -> int: ...\n @property\n def micro(self) -> int: ...\n @property\n def releaselevel(self) -> str: ...\n" }, { "path": "lib/spack/spack/vendor/attr/converters.py", "content": "# SPDX-License-Identifier: MIT\n\n\"\"\"\nCommonly useful converters.\n\"\"\"\n\n\nimport typing\n\nfrom ._compat import _AnnotationExtractor\nfrom ._make import NOTHING, Factory, pipe\n\n\n__all__ = [\n \"default_if_none\",\n \"optional\",\n \"pipe\",\n \"to_bool\",\n]\n\n\ndef optional(converter):\n \"\"\"\n A converter that allows an attribute to be optional. An optional attribute\n is one which can be set to ``None``.\n\n Type annotations will be inferred from the wrapped converter's, if it\n has any.\n\n :param callable converter: the converter that is used for non-``None``\n values.\n\n .. versionadded:: 17.1.0\n \"\"\"\n\n def optional_converter(val):\n if val is None:\n return None\n return converter(val)\n\n xtr = _AnnotationExtractor(converter)\n\n t = xtr.get_first_param_type()\n if t:\n optional_converter.__annotations__[\"val\"] = typing.Optional[t]\n\n rt = xtr.get_return_type()\n if rt:\n optional_converter.__annotations__[\"return\"] = typing.Optional[rt]\n\n return optional_converter\n\n\ndef default_if_none(default=NOTHING, factory=None):\n \"\"\"\n A converter that allows to replace ``None`` values by *default* or the\n result of *factory*.\n\n :param default: Value to be used if ``None`` is passed. Passing an instance\n of `attrs.Factory` is supported, however the ``takes_self`` option\n is *not*.\n :param callable factory: A callable that takes no parameters whose result\n is used if ``None`` is passed.\n\n :raises TypeError: If **neither** *default* or *factory* is passed.\n :raises TypeError: If **both** *default* and *factory* are passed.\n :raises ValueError: If an instance of `attrs.Factory` is passed with\n ``takes_self=True``.\n\n .. versionadded:: 18.2.0\n \"\"\"\n if default is NOTHING and factory is None:\n raise TypeError(\"Must pass either `default` or `factory`.\")\n\n if default is not NOTHING and factory is not None:\n raise TypeError(\n \"Must pass either `default` or `factory` but not both.\"\n )\n\n if factory is not None:\n default = Factory(factory)\n\n if isinstance(default, Factory):\n if default.takes_self:\n raise ValueError(\n \"`takes_self` is not supported by default_if_none.\"\n )\n\n def default_if_none_converter(val):\n if val is not None:\n return val\n\n return default.factory()\n\n else:\n\n def default_if_none_converter(val):\n if val is not None:\n return val\n\n return default\n\n return default_if_none_converter\n\n\ndef to_bool(val):\n \"\"\"\n Convert \"boolean\" strings (e.g., from env. vars.) to real booleans.\n\n Values mapping to :code:`True`:\n\n - :code:`True`\n - :code:`\"true\"` / :code:`\"t\"`\n - :code:`\"yes\"` / :code:`\"y\"`\n - :code:`\"on\"`\n - :code:`\"1\"`\n - :code:`1`\n\n Values mapping to :code:`False`:\n\n - :code:`False`\n - :code:`\"false\"` / :code:`\"f\"`\n - :code:`\"no\"` / :code:`\"n\"`\n - :code:`\"off\"`\n - :code:`\"0\"`\n - :code:`0`\n\n :raises ValueError: for any other value.\n\n .. versionadded:: 21.3.0\n \"\"\"\n if isinstance(val, str):\n val = val.lower()\n truthy = {True, \"true\", \"t\", \"yes\", \"y\", \"on\", \"1\", 1}\n falsy = {False, \"false\", \"f\", \"no\", \"n\", \"off\", \"0\", 0}\n try:\n if val in truthy:\n return True\n if val in falsy:\n return False\n except TypeError:\n # Raised when \"val\" is not hashable (e.g., lists)\n pass\n raise ValueError(\"Cannot convert value to bool: {}\".format(val))\n" }, { "path": "lib/spack/spack/vendor/attr/converters.pyi", "content": "from typing import Callable, Optional, TypeVar, overload\n\nfrom . import _ConverterType\n\n_T = TypeVar(\"_T\")\n\ndef pipe(*validators: _ConverterType) -> _ConverterType: ...\ndef optional(converter: _ConverterType) -> _ConverterType: ...\n@overload\ndef default_if_none(default: _T) -> _ConverterType: ...\n@overload\ndef default_if_none(*, factory: Callable[[], _T]) -> _ConverterType: ...\ndef to_bool(val: str) -> bool: ...\n" }, { "path": "lib/spack/spack/vendor/attr/exceptions.py", "content": "# SPDX-License-Identifier: MIT\n\n\nclass FrozenError(AttributeError):\n \"\"\"\n A frozen/immutable instance or attribute have been attempted to be\n modified.\n\n It mirrors the behavior of ``namedtuples`` by using the same error message\n and subclassing `AttributeError`.\n\n .. versionadded:: 20.1.0\n \"\"\"\n\n msg = \"can't set attribute\"\n args = [msg]\n\n\nclass FrozenInstanceError(FrozenError):\n \"\"\"\n A frozen instance has been attempted to be modified.\n\n .. versionadded:: 16.1.0\n \"\"\"\n\n\nclass FrozenAttributeError(FrozenError):\n \"\"\"\n A frozen attribute has been attempted to be modified.\n\n .. versionadded:: 20.1.0\n \"\"\"\n\n\nclass AttrsAttributeNotFoundError(ValueError):\n \"\"\"\n An ``attrs`` function couldn't find an attribute that the user asked for.\n\n .. versionadded:: 16.2.0\n \"\"\"\n\n\nclass NotAnAttrsClassError(ValueError):\n \"\"\"\n A non-``attrs`` class has been passed into an ``attrs`` function.\n\n .. versionadded:: 16.2.0\n \"\"\"\n\n\nclass DefaultAlreadySetError(RuntimeError):\n \"\"\"\n A default has been set using ``attr.ib()`` and is attempted to be reset\n using the decorator.\n\n .. versionadded:: 17.1.0\n \"\"\"\n\n\nclass UnannotatedAttributeError(RuntimeError):\n \"\"\"\n A class with ``auto_attribs=True`` has an ``attr.ib()`` without a type\n annotation.\n\n .. versionadded:: 17.3.0\n \"\"\"\n\n\nclass PythonTooOldError(RuntimeError):\n \"\"\"\n It was attempted to use an ``attrs`` feature that requires a newer Python\n version.\n\n .. versionadded:: 18.2.0\n \"\"\"\n\n\nclass NotCallableError(TypeError):\n \"\"\"\n A ``attr.ib()`` requiring a callable has been set with a value\n that is not callable.\n\n .. versionadded:: 19.2.0\n \"\"\"\n\n def __init__(self, msg, value):\n super(TypeError, self).__init__(msg, value)\n self.msg = msg\n self.value = value\n\n def __str__(self):\n return str(self.msg)\n" }, { "path": "lib/spack/spack/vendor/attr/exceptions.pyi", "content": "from typing import Any\n\nclass FrozenError(AttributeError):\n msg: str = ...\n\nclass FrozenInstanceError(FrozenError): ...\nclass FrozenAttributeError(FrozenError): ...\nclass AttrsAttributeNotFoundError(ValueError): ...\nclass NotAnAttrsClassError(ValueError): ...\nclass DefaultAlreadySetError(RuntimeError): ...\nclass UnannotatedAttributeError(RuntimeError): ...\nclass PythonTooOldError(RuntimeError): ...\n\nclass NotCallableError(TypeError):\n msg: str = ...\n value: Any = ...\n def __init__(self, msg: str, value: Any) -> None: ...\n" }, { "path": "lib/spack/spack/vendor/attr/filters.py", "content": "# SPDX-License-Identifier: MIT\n\n\"\"\"\nCommonly useful filters for `attr.asdict`.\n\"\"\"\n\nfrom ._make import Attribute\n\n\ndef _split_what(what):\n \"\"\"\n Returns a tuple of `frozenset`s of classes and attributes.\n \"\"\"\n return (\n frozenset(cls for cls in what if isinstance(cls, type)),\n frozenset(cls for cls in what if isinstance(cls, Attribute)),\n )\n\n\ndef include(*what):\n \"\"\"\n Include *what*.\n\n :param what: What to include.\n :type what: `list` of `type` or `attrs.Attribute`\\\\ s\n\n :rtype: `callable`\n \"\"\"\n cls, attrs = _split_what(what)\n\n def include_(attribute, value):\n return value.__class__ in cls or attribute in attrs\n\n return include_\n\n\ndef exclude(*what):\n \"\"\"\n Exclude *what*.\n\n :param what: What to exclude.\n :type what: `list` of classes or `attrs.Attribute`\\\\ s.\n\n :rtype: `callable`\n \"\"\"\n cls, attrs = _split_what(what)\n\n def exclude_(attribute, value):\n return value.__class__ not in cls and attribute not in attrs\n\n return exclude_\n" }, { "path": "lib/spack/spack/vendor/attr/filters.pyi", "content": "from typing import Any, Union\n\nfrom . import Attribute, _FilterType\n\ndef include(*what: Union[type, Attribute[Any]]) -> _FilterType[Any]: ...\ndef exclude(*what: Union[type, Attribute[Any]]) -> _FilterType[Any]: ...\n" }, { "path": "lib/spack/spack/vendor/attr/py.typed", "content": "" }, { "path": "lib/spack/spack/vendor/attr/setters.py", "content": "# SPDX-License-Identifier: MIT\n\n\"\"\"\nCommonly used hooks for on_setattr.\n\"\"\"\n\n\nfrom . import _config\nfrom .exceptions import FrozenAttributeError\n\n\ndef pipe(*setters):\n \"\"\"\n Run all *setters* and return the return value of the last one.\n\n .. versionadded:: 20.1.0\n \"\"\"\n\n def wrapped_pipe(instance, attrib, new_value):\n rv = new_value\n\n for setter in setters:\n rv = setter(instance, attrib, rv)\n\n return rv\n\n return wrapped_pipe\n\n\ndef frozen(_, __, ___):\n \"\"\"\n Prevent an attribute to be modified.\n\n .. versionadded:: 20.1.0\n \"\"\"\n raise FrozenAttributeError()\n\n\ndef validate(instance, attrib, new_value):\n \"\"\"\n Run *attrib*'s validator on *new_value* if it has one.\n\n .. versionadded:: 20.1.0\n \"\"\"\n if _config._run_validators is False:\n return new_value\n\n v = attrib.validator\n if not v:\n return new_value\n\n v(instance, attrib, new_value)\n\n return new_value\n\n\ndef convert(instance, attrib, new_value):\n \"\"\"\n Run *attrib*'s converter -- if it has one -- on *new_value* and return the\n result.\n\n .. versionadded:: 20.1.0\n \"\"\"\n c = attrib.converter\n if c:\n return c(new_value)\n\n return new_value\n\n\n# Sentinel for disabling class-wide *on_setattr* hooks for certain attributes.\n# autodata stopped working, so the docstring is inlined in the API docs.\nNO_OP = object()\n" }, { "path": "lib/spack/spack/vendor/attr/setters.pyi", "content": "from typing import Any, NewType, NoReturn, TypeVar, cast\n\nfrom . import Attribute, _OnSetAttrType\n\n_T = TypeVar(\"_T\")\n\ndef frozen(\n instance: Any, attribute: Attribute[Any], new_value: Any\n) -> NoReturn: ...\ndef pipe(*setters: _OnSetAttrType) -> _OnSetAttrType: ...\ndef validate(instance: Any, attribute: Attribute[_T], new_value: _T) -> _T: ...\n\n# convert is allowed to return Any, because they can be chained using pipe.\ndef convert(\n instance: Any, attribute: Attribute[Any], new_value: Any\n) -> Any: ...\n\n_NoOpType = NewType(\"_NoOpType\", object)\nNO_OP: _NoOpType\n" }, { "path": "lib/spack/spack/vendor/attr/validators.py", "content": "# SPDX-License-Identifier: MIT\n\n\"\"\"\nCommonly useful validators.\n\"\"\"\n\n\nimport operator\nimport re\n\nfrom contextlib import contextmanager\n\nfrom ._config import get_run_validators, set_run_validators\nfrom ._make import _AndValidator, and_, attrib, attrs\nfrom .exceptions import NotCallableError\n\n\ntry:\n Pattern = re.Pattern\nexcept AttributeError: # Python <3.7 lacks a Pattern type.\n Pattern = type(re.compile(\"\"))\n\n\n__all__ = [\n \"and_\",\n \"deep_iterable\",\n \"deep_mapping\",\n \"disabled\",\n \"ge\",\n \"get_disabled\",\n \"gt\",\n \"in_\",\n \"instance_of\",\n \"is_callable\",\n \"le\",\n \"lt\",\n \"matches_re\",\n \"max_len\",\n \"min_len\",\n \"optional\",\n \"provides\",\n \"set_disabled\",\n]\n\n\ndef set_disabled(disabled):\n \"\"\"\n Globally disable or enable running validators.\n\n By default, they are run.\n\n :param disabled: If ``True``, disable running all validators.\n :type disabled: bool\n\n .. warning::\n\n This function is not thread-safe!\n\n .. versionadded:: 21.3.0\n \"\"\"\n set_run_validators(not disabled)\n\n\ndef get_disabled():\n \"\"\"\n Return a bool indicating whether validators are currently disabled or not.\n\n :return: ``True`` if validators are currently disabled.\n :rtype: bool\n\n .. versionadded:: 21.3.0\n \"\"\"\n return not get_run_validators()\n\n\n@contextmanager\ndef disabled():\n \"\"\"\n Context manager that disables running validators within its context.\n\n .. warning::\n\n This context manager is not thread-safe!\n\n .. versionadded:: 21.3.0\n \"\"\"\n set_run_validators(False)\n try:\n yield\n finally:\n set_run_validators(True)\n\n\n@attrs(repr=False, slots=True, hash=True)\nclass _InstanceOfValidator:\n type = attrib()\n\n def __call__(self, inst, attr, value):\n \"\"\"\n We use a callable class to be able to change the ``__repr__``.\n \"\"\"\n if not isinstance(value, self.type):\n raise TypeError(\n \"'{name}' must be {type!r} (got {value!r} that is a \"\n \"{actual!r}).\".format(\n name=attr.name,\n type=self.type,\n actual=value.__class__,\n value=value,\n ),\n attr,\n self.type,\n value,\n )\n\n def __repr__(self):\n return \"\".format(\n type=self.type\n )\n\n\ndef instance_of(type):\n \"\"\"\n A validator that raises a `TypeError` if the initializer is called\n with a wrong type for this particular attribute (checks are performed using\n `isinstance` therefore it's also valid to pass a tuple of types).\n\n :param type: The type to check for.\n :type type: type or tuple of types\n\n :raises TypeError: With a human readable error message, the attribute\n (of type `attrs.Attribute`), the expected type, and the value it\n got.\n \"\"\"\n return _InstanceOfValidator(type)\n\n\n@attrs(repr=False, frozen=True, slots=True)\nclass _MatchesReValidator:\n pattern = attrib()\n match_func = attrib()\n\n def __call__(self, inst, attr, value):\n \"\"\"\n We use a callable class to be able to change the ``__repr__``.\n \"\"\"\n if not self.match_func(value):\n raise ValueError(\n \"'{name}' must match regex {pattern!r}\"\n \" ({value!r} doesn't)\".format(\n name=attr.name, pattern=self.pattern.pattern, value=value\n ),\n attr,\n self.pattern,\n value,\n )\n\n def __repr__(self):\n return \"\".format(\n pattern=self.pattern\n )\n\n\ndef matches_re(regex, flags=0, func=None):\n r\"\"\"\n A validator that raises `ValueError` if the initializer is called\n with a string that doesn't match *regex*.\n\n :param regex: a regex string or precompiled pattern to match against\n :param int flags: flags that will be passed to the underlying re function\n (default 0)\n :param callable func: which underlying `re` function to call. Valid options\n are `re.fullmatch`, `re.search`, and `re.match`; the default ``None``\n means `re.fullmatch`. For performance reasons, the pattern is always\n precompiled using `re.compile`.\n\n .. versionadded:: 19.2.0\n .. versionchanged:: 21.3.0 *regex* can be a pre-compiled pattern.\n \"\"\"\n valid_funcs = (re.fullmatch, None, re.search, re.match)\n if func not in valid_funcs:\n raise ValueError(\n \"'func' must be one of {}.\".format(\n \", \".join(\n sorted(\n e and e.__name__ or \"None\" for e in set(valid_funcs)\n )\n )\n )\n )\n\n if isinstance(regex, Pattern):\n if flags:\n raise TypeError(\n \"'flags' can only be used with a string pattern; \"\n \"pass flags to re.compile() instead\"\n )\n pattern = regex\n else:\n pattern = re.compile(regex, flags)\n\n if func is re.match:\n match_func = pattern.match\n elif func is re.search:\n match_func = pattern.search\n else:\n match_func = pattern.fullmatch\n\n return _MatchesReValidator(pattern, match_func)\n\n\n@attrs(repr=False, slots=True, hash=True)\nclass _ProvidesValidator:\n interface = attrib()\n\n def __call__(self, inst, attr, value):\n \"\"\"\n We use a callable class to be able to change the ``__repr__``.\n \"\"\"\n if not self.interface.providedBy(value):\n raise TypeError(\n \"'{name}' must provide {interface!r} which {value!r} \"\n \"doesn't.\".format(\n name=attr.name, interface=self.interface, value=value\n ),\n attr,\n self.interface,\n value,\n )\n\n def __repr__(self):\n return \"\".format(\n interface=self.interface\n )\n\n\ndef provides(interface):\n \"\"\"\n A validator that raises a `TypeError` if the initializer is called\n with an object that does not provide the requested *interface* (checks are\n performed using ``interface.providedBy(value)`` (see `zope.interface\n `_).\n\n :param interface: The interface to check for.\n :type interface: ``zope.interface.Interface``\n\n :raises TypeError: With a human readable error message, the attribute\n (of type `attrs.Attribute`), the expected interface, and the\n value it got.\n \"\"\"\n return _ProvidesValidator(interface)\n\n\n@attrs(repr=False, slots=True, hash=True)\nclass _OptionalValidator:\n validator = attrib()\n\n def __call__(self, inst, attr, value):\n if value is None:\n return\n\n self.validator(inst, attr, value)\n\n def __repr__(self):\n return \"\".format(\n what=repr(self.validator)\n )\n\n\ndef optional(validator):\n \"\"\"\n A validator that makes an attribute optional. An optional attribute is one\n which can be set to ``None`` in addition to satisfying the requirements of\n the sub-validator.\n\n :param validator: A validator (or a list of validators) that is used for\n non-``None`` values.\n :type validator: callable or `list` of callables.\n\n .. versionadded:: 15.1.0\n .. versionchanged:: 17.1.0 *validator* can be a list of validators.\n \"\"\"\n if isinstance(validator, list):\n return _OptionalValidator(_AndValidator(validator))\n return _OptionalValidator(validator)\n\n\n@attrs(repr=False, slots=True, hash=True)\nclass _InValidator:\n options = attrib()\n\n def __call__(self, inst, attr, value):\n try:\n in_options = value in self.options\n except TypeError: # e.g. `1 in \"abc\"`\n in_options = False\n\n if not in_options:\n raise ValueError(\n \"'{name}' must be in {options!r} (got {value!r})\".format(\n name=attr.name, options=self.options, value=value\n ),\n attr,\n self.options,\n value,\n )\n\n def __repr__(self):\n return \"\".format(\n options=self.options\n )\n\n\ndef in_(options):\n \"\"\"\n A validator that raises a `ValueError` if the initializer is called\n with a value that does not belong in the options provided. The check is\n performed using ``value in options``.\n\n :param options: Allowed options.\n :type options: list, tuple, `enum.Enum`, ...\n\n :raises ValueError: With a human readable error message, the attribute (of\n type `attrs.Attribute`), the expected options, and the value it\n got.\n\n .. versionadded:: 17.1.0\n .. versionchanged:: 22.1.0\n The ValueError was incomplete until now and only contained the human\n readable error message. Now it contains all the information that has\n been promised since 17.1.0.\n \"\"\"\n return _InValidator(options)\n\n\n@attrs(repr=False, slots=False, hash=True)\nclass _IsCallableValidator:\n def __call__(self, inst, attr, value):\n \"\"\"\n We use a callable class to be able to change the ``__repr__``.\n \"\"\"\n if not callable(value):\n message = (\n \"'{name}' must be callable \"\n \"(got {value!r} that is a {actual!r}).\"\n )\n raise NotCallableError(\n msg=message.format(\n name=attr.name, value=value, actual=value.__class__\n ),\n value=value,\n )\n\n def __repr__(self):\n return \"\"\n\n\ndef is_callable():\n \"\"\"\n A validator that raises a `attr.exceptions.NotCallableError` if the\n initializer is called with a value for this particular attribute\n that is not callable.\n\n .. versionadded:: 19.1.0\n\n :raises `attr.exceptions.NotCallableError`: With a human readable error\n message containing the attribute (`attrs.Attribute`) name,\n and the value it got.\n \"\"\"\n return _IsCallableValidator()\n\n\n@attrs(repr=False, slots=True, hash=True)\nclass _DeepIterable:\n member_validator = attrib(validator=is_callable())\n iterable_validator = attrib(\n default=None, validator=optional(is_callable())\n )\n\n def __call__(self, inst, attr, value):\n \"\"\"\n We use a callable class to be able to change the ``__repr__``.\n \"\"\"\n if self.iterable_validator is not None:\n self.iterable_validator(inst, attr, value)\n\n for member in value:\n self.member_validator(inst, attr, member)\n\n def __repr__(self):\n iterable_identifier = (\n \"\"\n if self.iterable_validator is None\n else \" {iterable!r}\".format(iterable=self.iterable_validator)\n )\n return (\n \"\"\n ).format(\n iterable_identifier=iterable_identifier,\n member=self.member_validator,\n )\n\n\ndef deep_iterable(member_validator, iterable_validator=None):\n \"\"\"\n A validator that performs deep validation of an iterable.\n\n :param member_validator: Validator(s) to apply to iterable members\n :param iterable_validator: Validator to apply to iterable itself\n (optional)\n\n .. versionadded:: 19.1.0\n\n :raises TypeError: if any sub-validators fail\n \"\"\"\n if isinstance(member_validator, (list, tuple)):\n member_validator = and_(*member_validator)\n return _DeepIterable(member_validator, iterable_validator)\n\n\n@attrs(repr=False, slots=True, hash=True)\nclass _DeepMapping:\n key_validator = attrib(validator=is_callable())\n value_validator = attrib(validator=is_callable())\n mapping_validator = attrib(default=None, validator=optional(is_callable()))\n\n def __call__(self, inst, attr, value):\n \"\"\"\n We use a callable class to be able to change the ``__repr__``.\n \"\"\"\n if self.mapping_validator is not None:\n self.mapping_validator(inst, attr, value)\n\n for key in value:\n self.key_validator(inst, attr, key)\n self.value_validator(inst, attr, value[key])\n\n def __repr__(self):\n return (\n \"\"\n ).format(key=self.key_validator, value=self.value_validator)\n\n\ndef deep_mapping(key_validator, value_validator, mapping_validator=None):\n \"\"\"\n A validator that performs deep validation of a dictionary.\n\n :param key_validator: Validator to apply to dictionary keys\n :param value_validator: Validator to apply to dictionary values\n :param mapping_validator: Validator to apply to top-level mapping\n attribute (optional)\n\n .. versionadded:: 19.1.0\n\n :raises TypeError: if any sub-validators fail\n \"\"\"\n return _DeepMapping(key_validator, value_validator, mapping_validator)\n\n\n@attrs(repr=False, frozen=True, slots=True)\nclass _NumberValidator:\n bound = attrib()\n compare_op = attrib()\n compare_func = attrib()\n\n def __call__(self, inst, attr, value):\n \"\"\"\n We use a callable class to be able to change the ``__repr__``.\n \"\"\"\n if not self.compare_func(value, self.bound):\n raise ValueError(\n \"'{name}' must be {op} {bound}: {value}\".format(\n name=attr.name,\n op=self.compare_op,\n bound=self.bound,\n value=value,\n )\n )\n\n def __repr__(self):\n return \"\".format(\n op=self.compare_op, bound=self.bound\n )\n\n\ndef lt(val):\n \"\"\"\n A validator that raises `ValueError` if the initializer is called\n with a number larger or equal to *val*.\n\n :param val: Exclusive upper bound for values\n\n .. versionadded:: 21.3.0\n \"\"\"\n return _NumberValidator(val, \"<\", operator.lt)\n\n\ndef le(val):\n \"\"\"\n A validator that raises `ValueError` if the initializer is called\n with a number greater than *val*.\n\n :param val: Inclusive upper bound for values\n\n .. versionadded:: 21.3.0\n \"\"\"\n return _NumberValidator(val, \"<=\", operator.le)\n\n\ndef ge(val):\n \"\"\"\n A validator that raises `ValueError` if the initializer is called\n with a number smaller than *val*.\n\n :param val: Inclusive lower bound for values\n\n .. versionadded:: 21.3.0\n \"\"\"\n return _NumberValidator(val, \">=\", operator.ge)\n\n\ndef gt(val):\n \"\"\"\n A validator that raises `ValueError` if the initializer is called\n with a number smaller or equal to *val*.\n\n :param val: Exclusive lower bound for values\n\n .. versionadded:: 21.3.0\n \"\"\"\n return _NumberValidator(val, \">\", operator.gt)\n\n\n@attrs(repr=False, frozen=True, slots=True)\nclass _MaxLengthValidator:\n max_length = attrib()\n\n def __call__(self, inst, attr, value):\n \"\"\"\n We use a callable class to be able to change the ``__repr__``.\n \"\"\"\n if len(value) > self.max_length:\n raise ValueError(\n \"Length of '{name}' must be <= {max}: {len}\".format(\n name=attr.name, max=self.max_length, len=len(value)\n )\n )\n\n def __repr__(self):\n return \"\".format(max=self.max_length)\n\n\ndef max_len(length):\n \"\"\"\n A validator that raises `ValueError` if the initializer is called\n with a string or iterable that is longer than *length*.\n\n :param int length: Maximum length of the string or iterable\n\n .. versionadded:: 21.3.0\n \"\"\"\n return _MaxLengthValidator(length)\n\n\n@attrs(repr=False, frozen=True, slots=True)\nclass _MinLengthValidator:\n min_length = attrib()\n\n def __call__(self, inst, attr, value):\n \"\"\"\n We use a callable class to be able to change the ``__repr__``.\n \"\"\"\n if len(value) < self.min_length:\n raise ValueError(\n \"Length of '{name}' must be => {min}: {len}\".format(\n name=attr.name, min=self.min_length, len=len(value)\n )\n )\n\n def __repr__(self):\n return \"\".format(min=self.min_length)\n\n\ndef min_len(length):\n \"\"\"\n A validator that raises `ValueError` if the initializer is called\n with a string or iterable that is shorter than *length*.\n\n :param int length: Minimum length of the string or iterable\n\n .. versionadded:: 22.1.0\n \"\"\"\n return _MinLengthValidator(length)\n" }, { "path": "lib/spack/spack/vendor/attr/validators.pyi", "content": "from typing import (\n Any,\n AnyStr,\n Callable,\n Container,\n ContextManager,\n Iterable,\n List,\n Mapping,\n Match,\n Optional,\n Pattern,\n Tuple,\n Type,\n TypeVar,\n Union,\n overload,\n)\n\nfrom . import _ValidatorType\nfrom . import _ValidatorArgType\n\n_T = TypeVar(\"_T\")\n_T1 = TypeVar(\"_T1\")\n_T2 = TypeVar(\"_T2\")\n_T3 = TypeVar(\"_T3\")\n_I = TypeVar(\"_I\", bound=Iterable)\n_K = TypeVar(\"_K\")\n_V = TypeVar(\"_V\")\n_M = TypeVar(\"_M\", bound=Mapping)\n\ndef set_disabled(run: bool) -> None: ...\ndef get_disabled() -> bool: ...\ndef disabled() -> ContextManager[None]: ...\n\n# To be more precise on instance_of use some overloads.\n# If there are more than 3 items in the tuple then we fall back to Any\n@overload\ndef instance_of(type: Type[_T]) -> _ValidatorType[_T]: ...\n@overload\ndef instance_of(type: Tuple[Type[_T]]) -> _ValidatorType[_T]: ...\n@overload\ndef instance_of(\n type: Tuple[Type[_T1], Type[_T2]]\n) -> _ValidatorType[Union[_T1, _T2]]: ...\n@overload\ndef instance_of(\n type: Tuple[Type[_T1], Type[_T2], Type[_T3]]\n) -> _ValidatorType[Union[_T1, _T2, _T3]]: ...\n@overload\ndef instance_of(type: Tuple[type, ...]) -> _ValidatorType[Any]: ...\ndef provides(interface: Any) -> _ValidatorType[Any]: ...\ndef optional(\n validator: Union[_ValidatorType[_T], List[_ValidatorType[_T]]]\n) -> _ValidatorType[Optional[_T]]: ...\ndef in_(options: Container[_T]) -> _ValidatorType[_T]: ...\ndef and_(*validators: _ValidatorType[_T]) -> _ValidatorType[_T]: ...\ndef matches_re(\n regex: Union[Pattern[AnyStr], AnyStr],\n flags: int = ...,\n func: Optional[\n Callable[[AnyStr, AnyStr, int], Optional[Match[AnyStr]]]\n ] = ...,\n) -> _ValidatorType[AnyStr]: ...\ndef deep_iterable(\n member_validator: _ValidatorArgType[_T],\n iterable_validator: Optional[_ValidatorType[_I]] = ...,\n) -> _ValidatorType[_I]: ...\ndef deep_mapping(\n key_validator: _ValidatorType[_K],\n value_validator: _ValidatorType[_V],\n mapping_validator: Optional[_ValidatorType[_M]] = ...,\n) -> _ValidatorType[_M]: ...\ndef is_callable() -> _ValidatorType[_T]: ...\ndef lt(val: _T) -> _ValidatorType[_T]: ...\ndef le(val: _T) -> _ValidatorType[_T]: ...\ndef ge(val: _T) -> _ValidatorType[_T]: ...\ndef gt(val: _T) -> _ValidatorType[_T]: ...\ndef max_len(length: int) -> _ValidatorType[_T]: ...\ndef min_len(length: int) -> _ValidatorType[_T]: ...\n" }, { "path": "lib/spack/spack/vendor/attrs/LICENSE", "content": "The MIT License (MIT)\n\nCopyright (c) 2015 Hynek Schlawack and the attrs contributors\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "lib/spack/spack/vendor/attrs/__init__.py", "content": "# SPDX-License-Identifier: MIT\n\nfrom spack.vendor.attr import (\n NOTHING,\n Attribute,\n Factory,\n __author__,\n __copyright__,\n __description__,\n __doc__,\n __email__,\n __license__,\n __title__,\n __url__,\n __version__,\n __version_info__,\n assoc,\n cmp_using,\n define,\n evolve,\n field,\n fields,\n fields_dict,\n frozen,\n has,\n make_class,\n mutable,\n resolve_types,\n validate,\n)\nfrom spack.vendor.attr._next_gen import asdict, astuple\n\nfrom . import converters, exceptions, filters, setters, validators\n\n\n__all__ = [\n \"__author__\",\n \"__copyright__\",\n \"__description__\",\n \"__doc__\",\n \"__email__\",\n \"__license__\",\n \"__title__\",\n \"__url__\",\n \"__version__\",\n \"__version_info__\",\n \"asdict\",\n \"assoc\",\n \"astuple\",\n \"Attribute\",\n \"cmp_using\",\n \"converters\",\n \"define\",\n \"evolve\",\n \"exceptions\",\n \"Factory\",\n \"field\",\n \"fields_dict\",\n \"fields\",\n \"filters\",\n \"frozen\",\n \"has\",\n \"make_class\",\n \"mutable\",\n \"NOTHING\",\n \"resolve_types\",\n \"setters\",\n \"validate\",\n \"validators\",\n]\n" }, { "path": "lib/spack/spack/vendor/attrs/__init__.pyi", "content": "from typing import (\n Any,\n Callable,\n Dict,\n Mapping,\n Optional,\n Sequence,\n Tuple,\n Type,\n)\n\n# Because we need to type our own stuff, we have to make everything from\n# attr explicitly public too.\nfrom attr import __author__ as __author__\nfrom attr import __copyright__ as __copyright__\nfrom attr import __description__ as __description__\nfrom attr import __email__ as __email__\nfrom attr import __license__ as __license__\nfrom attr import __title__ as __title__\nfrom attr import __url__ as __url__\nfrom attr import __version__ as __version__\nfrom attr import __version_info__ as __version_info__\nfrom attr import _FilterType\nfrom attr import assoc as assoc\nfrom attr import Attribute as Attribute\nfrom attr import cmp_using as cmp_using\nfrom attr import converters as converters\nfrom attr import define as define\nfrom attr import evolve as evolve\nfrom attr import exceptions as exceptions\nfrom attr import Factory as Factory\nfrom attr import field as field\nfrom attr import fields as fields\nfrom attr import fields_dict as fields_dict\nfrom attr import filters as filters\nfrom attr import frozen as frozen\nfrom attr import has as has\nfrom attr import make_class as make_class\nfrom attr import mutable as mutable\nfrom attr import NOTHING as NOTHING\nfrom attr import resolve_types as resolve_types\nfrom attr import setters as setters\nfrom attr import validate as validate\nfrom attr import validators as validators\n\n# TODO: see definition of attr.asdict/astuple\ndef asdict(\n inst: Any,\n recurse: bool = ...,\n filter: Optional[_FilterType[Any]] = ...,\n dict_factory: Type[Mapping[Any, Any]] = ...,\n retain_collection_types: bool = ...,\n value_serializer: Optional[\n Callable[[type, Attribute[Any], Any], Any]\n ] = ...,\n tuple_keys: bool = ...,\n) -> Dict[str, Any]: ...\n\n# TODO: add support for returning NamedTuple from the mypy plugin\ndef astuple(\n inst: Any,\n recurse: bool = ...,\n filter: Optional[_FilterType[Any]] = ...,\n tuple_factory: Type[Sequence[Any]] = ...,\n retain_collection_types: bool = ...,\n) -> Tuple[Any, ...]: ...\n" }, { "path": "lib/spack/spack/vendor/attrs/converters.py", "content": "# SPDX-License-Identifier: MIT\n\nfrom spack.vendor.attr.converters import * # noqa\n" }, { "path": "lib/spack/spack/vendor/attrs/exceptions.py", "content": "# SPDX-License-Identifier: MIT\n\nfrom spack.vendor.attr.exceptions import * # noqa\n" }, { "path": "lib/spack/spack/vendor/attrs/filters.py", "content": "# SPDX-License-Identifier: MIT\n\nfrom spack.vendor.attr.filters import * # noqa\n" }, { "path": "lib/spack/spack/vendor/attrs/py.typed", "content": "" }, { "path": "lib/spack/spack/vendor/attrs/setters.py", "content": "# SPDX-License-Identifier: MIT\n\nfrom spack.vendor.attr.setters import * # noqa\n" }, { "path": "lib/spack/spack/vendor/attrs/validators.py", "content": "# SPDX-License-Identifier: MIT\n\nfrom spack.vendor.attr.validators import * # noqa\n" }, { "path": "lib/spack/spack/vendor/distro/LICENSE", "content": "Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"{}\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright {yyyy} {name of copyright owner}\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n" }, { "path": "lib/spack/spack/vendor/distro/__init__.py", "content": "from .distro import (\n NORMALIZED_DISTRO_ID,\n NORMALIZED_LSB_ID,\n NORMALIZED_OS_ID,\n LinuxDistribution,\n __version__,\n build_number,\n codename,\n distro_release_attr,\n distro_release_info,\n id,\n info,\n like,\n linux_distribution,\n lsb_release_attr,\n lsb_release_info,\n major_version,\n minor_version,\n name,\n os_release_attr,\n os_release_info,\n uname_attr,\n uname_info,\n version,\n version_parts,\n)\n\n__all__ = [\n \"NORMALIZED_DISTRO_ID\",\n \"NORMALIZED_LSB_ID\",\n \"NORMALIZED_OS_ID\",\n \"LinuxDistribution\",\n \"build_number\",\n \"codename\",\n \"distro_release_attr\",\n \"distro_release_info\",\n \"id\",\n \"info\",\n \"like\",\n \"linux_distribution\",\n \"lsb_release_attr\",\n \"lsb_release_info\",\n \"major_version\",\n \"minor_version\",\n \"name\",\n \"os_release_attr\",\n \"os_release_info\",\n \"uname_attr\",\n \"uname_info\",\n \"version\",\n \"version_parts\",\n]\n\n__version__ = __version__\n" }, { "path": "lib/spack/spack/vendor/distro/__main__.py", "content": "from .distro import main\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "lib/spack/spack/vendor/distro/distro.py", "content": "#!/usr/bin/env python\n# Copyright 2015,2016,2017 Nir Cohen\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nThe ``distro`` package (``distro`` stands for Linux Distribution) provides\ninformation about the Linux distribution it runs on, such as a reliable\nmachine-readable distro ID, or version information.\n\nIt is the recommended replacement for Python's original\n:py:func:`platform.linux_distribution` function, but it provides much more\nfunctionality. An alternative implementation became necessary because Python\n3.5 deprecated this function, and Python 3.8 removed it altogether. Its\npredecessor function :py:func:`platform.dist` was already deprecated since\nPython 2.6 and removed in Python 3.8. Still, there are many cases in which\naccess to OS distribution information is needed. See `Python issue 1322\n`_ for more information.\n\"\"\"\n\nimport argparse\nimport json\nimport logging\nimport os\nimport re\nimport shlex\nimport subprocess\nimport sys\nimport warnings\nfrom typing import (\n Any,\n Callable,\n Dict,\n Iterable,\n Optional,\n Sequence,\n TextIO,\n Tuple,\n Type,\n)\n\ntry:\n from typing import TypedDict\nexcept ImportError:\n # Python 3.7\n TypedDict = dict\n\n__version__ = \"1.8.0\"\n\n\nclass VersionDict(TypedDict):\n major: str\n minor: str\n build_number: str\n\n\nclass InfoDict(TypedDict):\n id: str\n version: str\n version_parts: VersionDict\n like: str\n codename: str\n\n\n_UNIXCONFDIR = os.environ.get(\"UNIXCONFDIR\", \"/etc\")\n_UNIXUSRLIBDIR = os.environ.get(\"UNIXUSRLIBDIR\", \"/usr/lib\")\n_OS_RELEASE_BASENAME = \"os-release\"\n\n#: Translation table for normalizing the \"ID\" attribute defined in os-release\n#: files, for use by the :func:`distro.id` method.\n#:\n#: * Key: Value as defined in the os-release file, translated to lower case,\n#: with blanks translated to underscores.\n#:\n#: * Value: Normalized value.\nNORMALIZED_OS_ID = {\n \"ol\": \"oracle\", # Oracle Linux\n \"opensuse-leap\": \"opensuse\", # Newer versions of OpenSuSE report as opensuse-leap\n}\n\n#: Translation table for normalizing the \"Distributor ID\" attribute returned by\n#: the lsb_release command, for use by the :func:`distro.id` method.\n#:\n#: * Key: Value as returned by the lsb_release command, translated to lower\n#: case, with blanks translated to underscores.\n#:\n#: * Value: Normalized value.\nNORMALIZED_LSB_ID = {\n \"enterpriseenterpriseas\": \"oracle\", # Oracle Enterprise Linux 4\n \"enterpriseenterpriseserver\": \"oracle\", # Oracle Linux 5\n \"redhatenterpriseworkstation\": \"rhel\", # RHEL 6, 7 Workstation\n \"redhatenterpriseserver\": \"rhel\", # RHEL 6, 7 Server\n \"redhatenterprisecomputenode\": \"rhel\", # RHEL 6 ComputeNode\n}\n\n#: Translation table for normalizing the distro ID derived from the file name\n#: of distro release files, for use by the :func:`distro.id` method.\n#:\n#: * Key: Value as derived from the file name of a distro release file,\n#: translated to lower case, with blanks translated to underscores.\n#:\n#: * Value: Normalized value.\nNORMALIZED_DISTRO_ID = {\n \"redhat\": \"rhel\", # RHEL 6.x, 7.x\n}\n\n# Pattern for content of distro release file (reversed)\n_DISTRO_RELEASE_CONTENT_REVERSED_PATTERN = re.compile(\n r\"(?:[^)]*\\)(.*)\\()? *(?:STL )?([\\d.+\\-a-z]*\\d) *(?:esaeler *)?(.+)\"\n)\n\n# Pattern for base file name of distro release file\n_DISTRO_RELEASE_BASENAME_PATTERN = re.compile(r\"(\\w+)[-_](release|version)$\")\n\n# Base file names to be looked up for if _UNIXCONFDIR is not readable.\n_DISTRO_RELEASE_BASENAMES = [\n \"SuSE-release\",\n \"arch-release\",\n \"base-release\",\n \"centos-release\",\n \"fedora-release\",\n \"gentoo-release\",\n \"mageia-release\",\n \"mandrake-release\",\n \"mandriva-release\",\n \"mandrivalinux-release\",\n \"manjaro-release\",\n \"oracle-release\",\n \"redhat-release\",\n \"rocky-release\",\n \"sl-release\",\n \"slackware-version\",\n]\n\n# Base file names to be ignored when searching for distro release file\n_DISTRO_RELEASE_IGNORE_BASENAMES = (\n \"debian_version\",\n \"lsb-release\",\n \"oem-release\",\n _OS_RELEASE_BASENAME,\n \"system-release\",\n \"plesk-release\",\n \"iredmail-release\",\n)\n\n\ndef linux_distribution(full_distribution_name: bool = True) -> Tuple[str, str, str]:\n \"\"\"\n .. deprecated:: 1.6.0\n\n :func:`distro.linux_distribution()` is deprecated. It should only be\n used as a compatibility shim with Python's\n :py:func:`platform.linux_distribution()`. Please use :func:`distro.id`,\n :func:`distro.version` and :func:`distro.name` instead.\n\n Return information about the current OS distribution as a tuple\n ``(id_name, version, codename)`` with items as follows:\n\n * ``id_name``: If *full_distribution_name* is false, the result of\n :func:`distro.id`. Otherwise, the result of :func:`distro.name`.\n\n * ``version``: The result of :func:`distro.version`.\n\n * ``codename``: The extra item (usually in parentheses) after the\n os-release version number, or the result of :func:`distro.codename`.\n\n The interface of this function is compatible with the original\n :py:func:`platform.linux_distribution` function, supporting a subset of\n its parameters.\n\n The data it returns may not exactly be the same, because it uses more data\n sources than the original function, and that may lead to different data if\n the OS distribution is not consistent across multiple data sources it\n provides (there are indeed such distributions ...).\n\n Another reason for differences is the fact that the :func:`distro.id`\n method normalizes the distro ID string to a reliable machine-readable value\n for a number of popular OS distributions.\n \"\"\"\n warnings.warn(\n \"distro.linux_distribution() is deprecated. It should only be used as a \"\n \"compatibility shim with Python's platform.linux_distribution(). Please use \"\n \"distro.id(), distro.version() and distro.name() instead.\",\n DeprecationWarning,\n stacklevel=2,\n )\n return _distro.linux_distribution(full_distribution_name)\n\n\ndef id() -> str:\n \"\"\"\n Return the distro ID of the current distribution, as a\n machine-readable string.\n\n For a number of OS distributions, the returned distro ID value is\n *reliable*, in the sense that it is documented and that it does not change\n across releases of the distribution.\n\n This package maintains the following reliable distro ID values:\n\n ============== =========================================\n Distro ID Distribution\n ============== =========================================\n \"ubuntu\" Ubuntu\n \"debian\" Debian\n \"rhel\" RedHat Enterprise Linux\n \"centos\" CentOS\n \"fedora\" Fedora\n \"sles\" SUSE Linux Enterprise Server\n \"opensuse\" openSUSE\n \"amzn\" Amazon Linux\n \"arch\" Arch Linux\n \"buildroot\" Buildroot\n \"cloudlinux\" CloudLinux OS\n \"exherbo\" Exherbo Linux\n \"gentoo\" GenToo Linux\n \"ibm_powerkvm\" IBM PowerKVM\n \"kvmibm\" KVM for IBM z Systems\n \"linuxmint\" Linux Mint\n \"mageia\" Mageia\n \"mandriva\" Mandriva Linux\n \"parallels\" Parallels\n \"pidora\" Pidora\n \"raspbian\" Raspbian\n \"oracle\" Oracle Linux (and Oracle Enterprise Linux)\n \"scientific\" Scientific Linux\n \"slackware\" Slackware\n \"xenserver\" XenServer\n \"openbsd\" OpenBSD\n \"netbsd\" NetBSD\n \"freebsd\" FreeBSD\n \"midnightbsd\" MidnightBSD\n \"rocky\" Rocky Linux\n \"aix\" AIX\n \"guix\" Guix System\n ============== =========================================\n\n If you have a need to get distros for reliable IDs added into this set,\n or if you find that the :func:`distro.id` function returns a different\n distro ID for one of the listed distros, please create an issue in the\n `distro issue tracker`_.\n\n **Lookup hierarchy and transformations:**\n\n First, the ID is obtained from the following sources, in the specified\n order. The first available and non-empty value is used:\n\n * the value of the \"ID\" attribute of the os-release file,\n\n * the value of the \"Distributor ID\" attribute returned by the lsb_release\n command,\n\n * the first part of the file name of the distro release file,\n\n The so determined ID value then passes the following transformations,\n before it is returned by this method:\n\n * it is translated to lower case,\n\n * blanks (which should not be there anyway) are translated to underscores,\n\n * a normalization of the ID is performed, based upon\n `normalization tables`_. The purpose of this normalization is to ensure\n that the ID is as reliable as possible, even across incompatible changes\n in the OS distributions. A common reason for an incompatible change is\n the addition of an os-release file, or the addition of the lsb_release\n command, with ID values that differ from what was previously determined\n from the distro release file name.\n \"\"\"\n return _distro.id()\n\n\ndef name(pretty: bool = False) -> str:\n \"\"\"\n Return the name of the current OS distribution, as a human-readable\n string.\n\n If *pretty* is false, the name is returned without version or codename.\n (e.g. \"CentOS Linux\")\n\n If *pretty* is true, the version and codename are appended.\n (e.g. \"CentOS Linux 7.1.1503 (Core)\")\n\n **Lookup hierarchy:**\n\n The name is obtained from the following sources, in the specified order.\n The first available and non-empty value is used:\n\n * If *pretty* is false:\n\n - the value of the \"NAME\" attribute of the os-release file,\n\n - the value of the \"Distributor ID\" attribute returned by the lsb_release\n command,\n\n - the value of the \"\" field of the distro release file.\n\n * If *pretty* is true:\n\n - the value of the \"PRETTY_NAME\" attribute of the os-release file,\n\n - the value of the \"Description\" attribute returned by the lsb_release\n command,\n\n - the value of the \"\" field of the distro release file, appended\n with the value of the pretty version (\"\" and \"\"\n fields) of the distro release file, if available.\n \"\"\"\n return _distro.name(pretty)\n\n\ndef version(pretty: bool = False, best: bool = False) -> str:\n \"\"\"\n Return the version of the current OS distribution, as a human-readable\n string.\n\n If *pretty* is false, the version is returned without codename (e.g.\n \"7.0\").\n\n If *pretty* is true, the codename in parenthesis is appended, if the\n codename is non-empty (e.g. \"7.0 (Maipo)\").\n\n Some distributions provide version numbers with different precisions in\n the different sources of distribution information. Examining the different\n sources in a fixed priority order does not always yield the most precise\n version (e.g. for Debian 8.2, or CentOS 7.1).\n\n Some other distributions may not provide this kind of information. In these\n cases, an empty string would be returned. This behavior can be observed\n with rolling releases distributions (e.g. Arch Linux).\n\n The *best* parameter can be used to control the approach for the returned\n version:\n\n If *best* is false, the first non-empty version number in priority order of\n the examined sources is returned.\n\n If *best* is true, the most precise version number out of all examined\n sources is returned.\n\n **Lookup hierarchy:**\n\n In all cases, the version number is obtained from the following sources.\n If *best* is false, this order represents the priority order:\n\n * the value of the \"VERSION_ID\" attribute of the os-release file,\n * the value of the \"Release\" attribute returned by the lsb_release\n command,\n * the version number parsed from the \"\" field of the first line\n of the distro release file,\n * the version number parsed from the \"PRETTY_NAME\" attribute of the\n os-release file, if it follows the format of the distro release files.\n * the version number parsed from the \"Description\" attribute returned by\n the lsb_release command, if it follows the format of the distro release\n files.\n \"\"\"\n return _distro.version(pretty, best)\n\n\ndef version_parts(best: bool = False) -> Tuple[str, str, str]:\n \"\"\"\n Return the version of the current OS distribution as a tuple\n ``(major, minor, build_number)`` with items as follows:\n\n * ``major``: The result of :func:`distro.major_version`.\n\n * ``minor``: The result of :func:`distro.minor_version`.\n\n * ``build_number``: The result of :func:`distro.build_number`.\n\n For a description of the *best* parameter, see the :func:`distro.version`\n method.\n \"\"\"\n return _distro.version_parts(best)\n\n\ndef major_version(best: bool = False) -> str:\n \"\"\"\n Return the major version of the current OS distribution, as a string,\n if provided.\n Otherwise, the empty string is returned. The major version is the first\n part of the dot-separated version string.\n\n For a description of the *best* parameter, see the :func:`distro.version`\n method.\n \"\"\"\n return _distro.major_version(best)\n\n\ndef minor_version(best: bool = False) -> str:\n \"\"\"\n Return the minor version of the current OS distribution, as a string,\n if provided.\n Otherwise, the empty string is returned. The minor version is the second\n part of the dot-separated version string.\n\n For a description of the *best* parameter, see the :func:`distro.version`\n method.\n \"\"\"\n return _distro.minor_version(best)\n\n\ndef build_number(best: bool = False) -> str:\n \"\"\"\n Return the build number of the current OS distribution, as a string,\n if provided.\n Otherwise, the empty string is returned. The build number is the third part\n of the dot-separated version string.\n\n For a description of the *best* parameter, see the :func:`distro.version`\n method.\n \"\"\"\n return _distro.build_number(best)\n\n\ndef like() -> str:\n \"\"\"\n Return a space-separated list of distro IDs of distributions that are\n closely related to the current OS distribution in regards to packaging\n and programming interfaces, for example distributions the current\n distribution is a derivative from.\n\n **Lookup hierarchy:**\n\n This information item is only provided by the os-release file.\n For details, see the description of the \"ID_LIKE\" attribute in the\n `os-release man page\n `_.\n \"\"\"\n return _distro.like()\n\n\ndef codename() -> str:\n \"\"\"\n Return the codename for the release of the current OS distribution,\n as a string.\n\n If the distribution does not have a codename, an empty string is returned.\n\n Note that the returned codename is not always really a codename. For\n example, openSUSE returns \"x86_64\". This function does not handle such\n cases in any special way and just returns the string it finds, if any.\n\n **Lookup hierarchy:**\n\n * the codename within the \"VERSION\" attribute of the os-release file, if\n provided,\n\n * the value of the \"Codename\" attribute returned by the lsb_release\n command,\n\n * the value of the \"\" field of the distro release file.\n \"\"\"\n return _distro.codename()\n\n\ndef info(pretty: bool = False, best: bool = False) -> InfoDict:\n \"\"\"\n Return certain machine-readable information items about the current OS\n distribution in a dictionary, as shown in the following example:\n\n .. sourcecode:: python\n\n {\n 'id': 'rhel',\n 'version': '7.0',\n 'version_parts': {\n 'major': '7',\n 'minor': '0',\n 'build_number': ''\n },\n 'like': 'fedora',\n 'codename': 'Maipo'\n }\n\n The dictionary structure and keys are always the same, regardless of which\n information items are available in the underlying data sources. The values\n for the various keys are as follows:\n\n * ``id``: The result of :func:`distro.id`.\n\n * ``version``: The result of :func:`distro.version`.\n\n * ``version_parts -> major``: The result of :func:`distro.major_version`.\n\n * ``version_parts -> minor``: The result of :func:`distro.minor_version`.\n\n * ``version_parts -> build_number``: The result of\n :func:`distro.build_number`.\n\n * ``like``: The result of :func:`distro.like`.\n\n * ``codename``: The result of :func:`distro.codename`.\n\n For a description of the *pretty* and *best* parameters, see the\n :func:`distro.version` method.\n \"\"\"\n return _distro.info(pretty, best)\n\n\ndef os_release_info() -> Dict[str, str]:\n \"\"\"\n Return a dictionary containing key-value pairs for the information items\n from the os-release file data source of the current OS distribution.\n\n See `os-release file`_ for details about these information items.\n \"\"\"\n return _distro.os_release_info()\n\n\ndef lsb_release_info() -> Dict[str, str]:\n \"\"\"\n Return a dictionary containing key-value pairs for the information items\n from the lsb_release command data source of the current OS distribution.\n\n See `lsb_release command output`_ for details about these information\n items.\n \"\"\"\n return _distro.lsb_release_info()\n\n\ndef distro_release_info() -> Dict[str, str]:\n \"\"\"\n Return a dictionary containing key-value pairs for the information items\n from the distro release file data source of the current OS distribution.\n\n See `distro release file`_ for details about these information items.\n \"\"\"\n return _distro.distro_release_info()\n\n\ndef uname_info() -> Dict[str, str]:\n \"\"\"\n Return a dictionary containing key-value pairs for the information items\n from the distro release file data source of the current OS distribution.\n \"\"\"\n return _distro.uname_info()\n\n\ndef os_release_attr(attribute: str) -> str:\n \"\"\"\n Return a single named information item from the os-release file data source\n of the current OS distribution.\n\n Parameters:\n\n * ``attribute`` (string): Key of the information item.\n\n Returns:\n\n * (string): Value of the information item, if the item exists.\n The empty string, if the item does not exist.\n\n See `os-release file`_ for details about these information items.\n \"\"\"\n return _distro.os_release_attr(attribute)\n\n\ndef lsb_release_attr(attribute: str) -> str:\n \"\"\"\n Return a single named information item from the lsb_release command output\n data source of the current OS distribution.\n\n Parameters:\n\n * ``attribute`` (string): Key of the information item.\n\n Returns:\n\n * (string): Value of the information item, if the item exists.\n The empty string, if the item does not exist.\n\n See `lsb_release command output`_ for details about these information\n items.\n \"\"\"\n return _distro.lsb_release_attr(attribute)\n\n\ndef distro_release_attr(attribute: str) -> str:\n \"\"\"\n Return a single named information item from the distro release file\n data source of the current OS distribution.\n\n Parameters:\n\n * ``attribute`` (string): Key of the information item.\n\n Returns:\n\n * (string): Value of the information item, if the item exists.\n The empty string, if the item does not exist.\n\n See `distro release file`_ for details about these information items.\n \"\"\"\n return _distro.distro_release_attr(attribute)\n\n\ndef uname_attr(attribute: str) -> str:\n \"\"\"\n Return a single named information item from the distro release file\n data source of the current OS distribution.\n\n Parameters:\n\n * ``attribute`` (string): Key of the information item.\n\n Returns:\n\n * (string): Value of the information item, if the item exists.\n The empty string, if the item does not exist.\n \"\"\"\n return _distro.uname_attr(attribute)\n\n\ntry:\n from functools import cached_property\nexcept ImportError:\n # Python < 3.8\n class cached_property: # type: ignore\n \"\"\"A version of @property which caches the value. On access, it calls the\n underlying function and sets the value in `__dict__` so future accesses\n will not re-call the property.\n \"\"\"\n\n def __init__(self, f: Callable[[Any], Any]) -> None:\n self._fname = f.__name__\n self._f = f\n\n def __get__(self, obj: Any, owner: Type[Any]) -> Any:\n assert obj is not None, f\"call {self._fname} on an instance\"\n ret = obj.__dict__[self._fname] = self._f(obj)\n return ret\n\n\nclass LinuxDistribution:\n \"\"\"\n Provides information about a OS distribution.\n\n This package creates a private module-global instance of this class with\n default initialization arguments, that is used by the\n `consolidated accessor functions`_ and `single source accessor functions`_.\n By using default initialization arguments, that module-global instance\n returns data about the current OS distribution (i.e. the distro this\n package runs on).\n\n Normally, it is not necessary to create additional instances of this class.\n However, in situations where control is needed over the exact data sources\n that are used, instances of this class can be created with a specific\n distro release file, or a specific os-release file, or without invoking the\n lsb_release command.\n \"\"\"\n\n def __init__(\n self,\n include_lsb: Optional[bool] = None,\n os_release_file: str = \"\",\n distro_release_file: str = \"\",\n include_uname: Optional[bool] = None,\n root_dir: Optional[str] = None,\n include_oslevel: Optional[bool] = None,\n ) -> None:\n \"\"\"\n The initialization method of this class gathers information from the\n available data sources, and stores that in private instance attributes.\n Subsequent access to the information items uses these private instance\n attributes, so that the data sources are read only once.\n\n Parameters:\n\n * ``include_lsb`` (bool): Controls whether the\n `lsb_release command output`_ is included as a data source.\n\n If the lsb_release command is not available in the program execution\n path, the data source for the lsb_release command will be empty.\n\n * ``os_release_file`` (string): The path name of the\n `os-release file`_ that is to be used as a data source.\n\n An empty string (the default) will cause the default path name to\n be used (see `os-release file`_ for details).\n\n If the specified or defaulted os-release file does not exist, the\n data source for the os-release file will be empty.\n\n * ``distro_release_file`` (string): The path name of the\n `distro release file`_ that is to be used as a data source.\n\n An empty string (the default) will cause a default search algorithm\n to be used (see `distro release file`_ for details).\n\n If the specified distro release file does not exist, or if no default\n distro release file can be found, the data source for the distro\n release file will be empty.\n\n * ``include_uname`` (bool): Controls whether uname command output is\n included as a data source. If the uname command is not available in\n the program execution path the data source for the uname command will\n be empty.\n\n * ``root_dir`` (string): The absolute path to the root directory to use\n to find distro-related information files. Note that ``include_*``\n parameters must not be enabled in combination with ``root_dir``.\n\n * ``include_oslevel`` (bool): Controls whether (AIX) oslevel command\n output is included as a data source. If the oslevel command is not\n available in the program execution path the data source will be\n empty.\n\n Public instance attributes:\n\n * ``os_release_file`` (string): The path name of the\n `os-release file`_ that is actually used as a data source. The\n empty string if no distro release file is used as a data source.\n\n * ``distro_release_file`` (string): The path name of the\n `distro release file`_ that is actually used as a data source. The\n empty string if no distro release file is used as a data source.\n\n * ``include_lsb`` (bool): The result of the ``include_lsb`` parameter.\n This controls whether the lsb information will be loaded.\n\n * ``include_uname`` (bool): The result of the ``include_uname``\n parameter. This controls whether the uname information will\n be loaded.\n\n * ``include_oslevel`` (bool): The result of the ``include_oslevel``\n parameter. This controls whether (AIX) oslevel information will be\n loaded.\n\n * ``root_dir`` (string): The result of the ``root_dir`` parameter.\n The absolute path to the root directory to use to find distro-related\n information files.\n\n Raises:\n\n * :py:exc:`ValueError`: Initialization parameters combination is not\n supported.\n\n * :py:exc:`OSError`: Some I/O issue with an os-release file or distro\n release file.\n\n * :py:exc:`UnicodeError`: A data source has unexpected characters or\n uses an unexpected encoding.\n \"\"\"\n self.root_dir = root_dir\n self.etc_dir = os.path.join(root_dir, \"etc\") if root_dir else _UNIXCONFDIR\n self.usr_lib_dir = (\n os.path.join(root_dir, \"usr/lib\") if root_dir else _UNIXUSRLIBDIR\n )\n\n if os_release_file:\n self.os_release_file = os_release_file\n else:\n etc_dir_os_release_file = os.path.join(self.etc_dir, _OS_RELEASE_BASENAME)\n usr_lib_os_release_file = os.path.join(\n self.usr_lib_dir, _OS_RELEASE_BASENAME\n )\n\n # NOTE: The idea is to respect order **and** have it set\n # at all times for API backwards compatibility.\n if os.path.isfile(etc_dir_os_release_file) or not os.path.isfile(\n usr_lib_os_release_file\n ):\n self.os_release_file = etc_dir_os_release_file\n else:\n self.os_release_file = usr_lib_os_release_file\n\n self.distro_release_file = distro_release_file or \"\" # updated later\n\n is_root_dir_defined = root_dir is not None\n if is_root_dir_defined and (include_lsb or include_uname or include_oslevel):\n raise ValueError(\n \"Including subprocess data sources from specific root_dir is disallowed\"\n \" to prevent false information\"\n )\n self.include_lsb = (\n include_lsb if include_lsb is not None else not is_root_dir_defined\n )\n self.include_uname = (\n include_uname if include_uname is not None else not is_root_dir_defined\n )\n self.include_oslevel = (\n include_oslevel if include_oslevel is not None else not is_root_dir_defined\n )\n\n def __repr__(self) -> str:\n \"\"\"Return repr of all info\"\"\"\n return (\n \"LinuxDistribution(\"\n \"os_release_file={self.os_release_file!r}, \"\n \"distro_release_file={self.distro_release_file!r}, \"\n \"include_lsb={self.include_lsb!r}, \"\n \"include_uname={self.include_uname!r}, \"\n \"include_oslevel={self.include_oslevel!r}, \"\n \"root_dir={self.root_dir!r}, \"\n \"_os_release_info={self._os_release_info!r}, \"\n \"_lsb_release_info={self._lsb_release_info!r}, \"\n \"_distro_release_info={self._distro_release_info!r}, \"\n \"_uname_info={self._uname_info!r}, \"\n \"_oslevel_info={self._oslevel_info!r})\".format(self=self)\n )\n\n def linux_distribution(\n self, full_distribution_name: bool = True\n ) -> Tuple[str, str, str]:\n \"\"\"\n Return information about the OS distribution that is compatible\n with Python's :func:`platform.linux_distribution`, supporting a subset\n of its parameters.\n\n For details, see :func:`distro.linux_distribution`.\n \"\"\"\n return (\n self.name() if full_distribution_name else self.id(),\n self.version(),\n self._os_release_info.get(\"release_codename\") or self.codename(),\n )\n\n def id(self) -> str:\n \"\"\"Return the distro ID of the OS distribution, as a string.\n\n For details, see :func:`distro.id`.\n \"\"\"\n\n def normalize(distro_id: str, table: Dict[str, str]) -> str:\n distro_id = distro_id.lower().replace(\" \", \"_\")\n return table.get(distro_id, distro_id)\n\n distro_id = self.os_release_attr(\"id\")\n if distro_id:\n return normalize(distro_id, NORMALIZED_OS_ID)\n\n distro_id = self.lsb_release_attr(\"distributor_id\")\n if distro_id:\n return normalize(distro_id, NORMALIZED_LSB_ID)\n\n distro_id = self.distro_release_attr(\"id\")\n if distro_id:\n return normalize(distro_id, NORMALIZED_DISTRO_ID)\n\n distro_id = self.uname_attr(\"id\")\n if distro_id:\n return normalize(distro_id, NORMALIZED_DISTRO_ID)\n\n return \"\"\n\n def name(self, pretty: bool = False) -> str:\n \"\"\"\n Return the name of the OS distribution, as a string.\n\n For details, see :func:`distro.name`.\n \"\"\"\n name = (\n self.os_release_attr(\"name\")\n or self.lsb_release_attr(\"distributor_id\")\n or self.distro_release_attr(\"name\")\n or self.uname_attr(\"name\")\n )\n if pretty:\n name = self.os_release_attr(\"pretty_name\") or self.lsb_release_attr(\n \"description\"\n )\n if not name:\n name = self.distro_release_attr(\"name\") or self.uname_attr(\"name\")\n version = self.version(pretty=True)\n if version:\n name = f\"{name} {version}\"\n return name or \"\"\n\n def version(self, pretty: bool = False, best: bool = False) -> str:\n \"\"\"\n Return the version of the OS distribution, as a string.\n\n For details, see :func:`distro.version`.\n \"\"\"\n versions = [\n self.os_release_attr(\"version_id\"),\n self.lsb_release_attr(\"release\"),\n self.distro_release_attr(\"version_id\"),\n self._parse_distro_release_content(self.os_release_attr(\"pretty_name\")).get(\n \"version_id\", \"\"\n ),\n self._parse_distro_release_content(\n self.lsb_release_attr(\"description\")\n ).get(\"version_id\", \"\"),\n self.uname_attr(\"release\"),\n ]\n if self.uname_attr(\"id\").startswith(\"aix\"):\n # On AIX platforms, prefer oslevel command output.\n versions.insert(0, self.oslevel_info())\n elif self.id() == \"debian\" or \"debian\" in self.like().split():\n # On Debian-like, add debian_version file content to candidates list.\n versions.append(self._debian_version)\n version = \"\"\n if best:\n # This algorithm uses the last version in priority order that has\n # the best precision. If the versions are not in conflict, that\n # does not matter; otherwise, using the last one instead of the\n # first one might be considered a surprise.\n for v in versions:\n if v.count(\".\") > version.count(\".\") or version == \"\":\n version = v\n else:\n for v in versions:\n if v != \"\":\n version = v\n break\n if pretty and version and self.codename():\n version = f\"{version} ({self.codename()})\"\n return version\n\n def version_parts(self, best: bool = False) -> Tuple[str, str, str]:\n \"\"\"\n Return the version of the OS distribution, as a tuple of version\n numbers.\n\n For details, see :func:`distro.version_parts`.\n \"\"\"\n version_str = self.version(best=best)\n if version_str:\n version_regex = re.compile(r\"(\\d+)\\.?(\\d+)?\\.?(\\d+)?\")\n matches = version_regex.match(version_str)\n if matches:\n major, minor, build_number = matches.groups()\n return major, minor or \"\", build_number or \"\"\n return \"\", \"\", \"\"\n\n def major_version(self, best: bool = False) -> str:\n \"\"\"\n Return the major version number of the current distribution.\n\n For details, see :func:`distro.major_version`.\n \"\"\"\n return self.version_parts(best)[0]\n\n def minor_version(self, best: bool = False) -> str:\n \"\"\"\n Return the minor version number of the current distribution.\n\n For details, see :func:`distro.minor_version`.\n \"\"\"\n return self.version_parts(best)[1]\n\n def build_number(self, best: bool = False) -> str:\n \"\"\"\n Return the build number of the current distribution.\n\n For details, see :func:`distro.build_number`.\n \"\"\"\n return self.version_parts(best)[2]\n\n def like(self) -> str:\n \"\"\"\n Return the IDs of distributions that are like the OS distribution.\n\n For details, see :func:`distro.like`.\n \"\"\"\n return self.os_release_attr(\"id_like\") or \"\"\n\n def codename(self) -> str:\n \"\"\"\n Return the codename of the OS distribution.\n\n For details, see :func:`distro.codename`.\n \"\"\"\n try:\n # Handle os_release specially since distros might purposefully set\n # this to empty string to have no codename\n return self._os_release_info[\"codename\"]\n except KeyError:\n return (\n self.lsb_release_attr(\"codename\")\n or self.distro_release_attr(\"codename\")\n or \"\"\n )\n\n def info(self, pretty: bool = False, best: bool = False) -> InfoDict:\n \"\"\"\n Return certain machine-readable information about the OS\n distribution.\n\n For details, see :func:`distro.info`.\n \"\"\"\n return dict(\n id=self.id(),\n version=self.version(pretty, best),\n version_parts=dict(\n major=self.major_version(best),\n minor=self.minor_version(best),\n build_number=self.build_number(best),\n ),\n like=self.like(),\n codename=self.codename(),\n )\n\n def os_release_info(self) -> Dict[str, str]:\n \"\"\"\n Return a dictionary containing key-value pairs for the information\n items from the os-release file data source of the OS distribution.\n\n For details, see :func:`distro.os_release_info`.\n \"\"\"\n return self._os_release_info\n\n def lsb_release_info(self) -> Dict[str, str]:\n \"\"\"\n Return a dictionary containing key-value pairs for the information\n items from the lsb_release command data source of the OS\n distribution.\n\n For details, see :func:`distro.lsb_release_info`.\n \"\"\"\n return self._lsb_release_info\n\n def distro_release_info(self) -> Dict[str, str]:\n \"\"\"\n Return a dictionary containing key-value pairs for the information\n items from the distro release file data source of the OS\n distribution.\n\n For details, see :func:`distro.distro_release_info`.\n \"\"\"\n return self._distro_release_info\n\n def uname_info(self) -> Dict[str, str]:\n \"\"\"\n Return a dictionary containing key-value pairs for the information\n items from the uname command data source of the OS distribution.\n\n For details, see :func:`distro.uname_info`.\n \"\"\"\n return self._uname_info\n\n def oslevel_info(self) -> str:\n \"\"\"\n Return AIX' oslevel command output.\n \"\"\"\n return self._oslevel_info\n\n def os_release_attr(self, attribute: str) -> str:\n \"\"\"\n Return a single named information item from the os-release file data\n source of the OS distribution.\n\n For details, see :func:`distro.os_release_attr`.\n \"\"\"\n return self._os_release_info.get(attribute, \"\")\n\n def lsb_release_attr(self, attribute: str) -> str:\n \"\"\"\n Return a single named information item from the lsb_release command\n output data source of the OS distribution.\n\n For details, see :func:`distro.lsb_release_attr`.\n \"\"\"\n return self._lsb_release_info.get(attribute, \"\")\n\n def distro_release_attr(self, attribute: str) -> str:\n \"\"\"\n Return a single named information item from the distro release file\n data source of the OS distribution.\n\n For details, see :func:`distro.distro_release_attr`.\n \"\"\"\n return self._distro_release_info.get(attribute, \"\")\n\n def uname_attr(self, attribute: str) -> str:\n \"\"\"\n Return a single named information item from the uname command\n output data source of the OS distribution.\n\n For details, see :func:`distro.uname_attr`.\n \"\"\"\n return self._uname_info.get(attribute, \"\")\n\n @cached_property\n def _os_release_info(self) -> Dict[str, str]:\n \"\"\"\n Get the information items from the specified os-release file.\n\n Returns:\n A dictionary containing all information items.\n \"\"\"\n if os.path.isfile(self.os_release_file):\n with open(self.os_release_file, encoding=\"utf-8\") as release_file:\n return self._parse_os_release_content(release_file)\n return {}\n\n @staticmethod\n def _parse_os_release_content(lines: TextIO) -> Dict[str, str]:\n \"\"\"\n Parse the lines of an os-release file.\n\n Parameters:\n\n * lines: Iterable through the lines in the os-release file.\n Each line must be a unicode string or a UTF-8 encoded byte\n string.\n\n Returns:\n A dictionary containing all information items.\n \"\"\"\n props = {}\n lexer = shlex.shlex(lines, posix=True)\n lexer.whitespace_split = True\n\n tokens = list(lexer)\n for token in tokens:\n # At this point, all shell-like parsing has been done (i.e.\n # comments processed, quotes and backslash escape sequences\n # processed, multi-line values assembled, trailing newlines\n # stripped, etc.), so the tokens are now either:\n # * variable assignments: var=value\n # * commands or their arguments (not allowed in os-release)\n # Ignore any tokens that are not variable assignments\n if \"=\" in token:\n k, v = token.split(\"=\", 1)\n props[k.lower()] = v\n\n if \"version\" in props:\n # extract release codename (if any) from version attribute\n match = re.search(r\"\\((\\D+)\\)|,\\s*(\\D+)\", props[\"version\"])\n if match:\n release_codename = match.group(1) or match.group(2)\n props[\"codename\"] = props[\"release_codename\"] = release_codename\n\n if \"version_codename\" in props:\n # os-release added a version_codename field. Use that in\n # preference to anything else Note that some distros purposefully\n # do not have code names. They should be setting\n # version_codename=\"\"\n props[\"codename\"] = props[\"version_codename\"]\n elif \"ubuntu_codename\" in props:\n # Same as above but a non-standard field name used on older Ubuntus\n props[\"codename\"] = props[\"ubuntu_codename\"]\n\n return props\n\n @cached_property\n def _lsb_release_info(self) -> Dict[str, str]:\n \"\"\"\n Get the information items from the lsb_release command output.\n\n Returns:\n A dictionary containing all information items.\n \"\"\"\n if not self.include_lsb:\n return {}\n try:\n cmd = (\"lsb_release\", \"-a\")\n stdout = subprocess.check_output(cmd, stderr=subprocess.DEVNULL)\n # Command not found or lsb_release returned error\n except (OSError, subprocess.CalledProcessError):\n return {}\n content = self._to_str(stdout).splitlines()\n return self._parse_lsb_release_content(content)\n\n @staticmethod\n def _parse_lsb_release_content(lines: Iterable[str]) -> Dict[str, str]:\n \"\"\"\n Parse the output of the lsb_release command.\n\n Parameters:\n\n * lines: Iterable through the lines of the lsb_release output.\n Each line must be a unicode string or a UTF-8 encoded byte\n string.\n\n Returns:\n A dictionary containing all information items.\n \"\"\"\n props = {}\n for line in lines:\n kv = line.strip(\"\\n\").split(\":\", 1)\n if len(kv) != 2:\n # Ignore lines without colon.\n continue\n k, v = kv\n props.update({k.replace(\" \", \"_\").lower(): v.strip()})\n return props\n\n @cached_property\n def _uname_info(self) -> Dict[str, str]:\n if not self.include_uname:\n return {}\n try:\n cmd = (\"uname\", \"-rs\")\n stdout = subprocess.check_output(cmd, stderr=subprocess.DEVNULL)\n except OSError:\n return {}\n content = self._to_str(stdout).splitlines()\n return self._parse_uname_content(content)\n\n @cached_property\n def _oslevel_info(self) -> str:\n if not self.include_oslevel:\n return \"\"\n try:\n stdout = subprocess.check_output(\"oslevel\", stderr=subprocess.DEVNULL)\n except (OSError, subprocess.CalledProcessError):\n return \"\"\n return self._to_str(stdout).strip()\n\n @cached_property\n def _debian_version(self) -> str:\n try:\n with open(\n os.path.join(self.etc_dir, \"debian_version\"), encoding=\"ascii\"\n ) as fp:\n return fp.readline().rstrip()\n except FileNotFoundError:\n return \"\"\n\n @staticmethod\n def _parse_uname_content(lines: Sequence[str]) -> Dict[str, str]:\n if not lines:\n return {}\n props = {}\n match = re.search(r\"^([^\\s]+)\\s+([\\d\\.]+)\", lines[0].strip())\n if match:\n name, version = match.groups()\n\n # This is to prevent the Linux kernel version from\n # appearing as the 'best' version on otherwise\n # identifiable distributions.\n if name == \"Linux\":\n return {}\n props[\"id\"] = name.lower()\n props[\"name\"] = name\n props[\"release\"] = version\n return props\n\n @staticmethod\n def _to_str(bytestring: bytes) -> str:\n encoding = sys.getfilesystemencoding()\n return bytestring.decode(encoding)\n\n @cached_property\n def _distro_release_info(self) -> Dict[str, str]:\n \"\"\"\n Get the information items from the specified distro release file.\n\n Returns:\n A dictionary containing all information items.\n \"\"\"\n if self.distro_release_file:\n # If it was specified, we use it and parse what we can, even if\n # its file name or content does not match the expected pattern.\n distro_info = self._parse_distro_release_file(self.distro_release_file)\n basename = os.path.basename(self.distro_release_file)\n # The file name pattern for user-specified distro release files\n # is somewhat more tolerant (compared to when searching for the\n # file), because we want to use what was specified as best as\n # possible.\n match = _DISTRO_RELEASE_BASENAME_PATTERN.match(basename)\n else:\n try:\n with os.scandir(self.etc_dir) as it:\n etc_files = [\n p.path for p in it\n if p.is_file() and p.name not in _DISTRO_RELEASE_IGNORE_BASENAMES\n ]\n # We sort for repeatability in cases where there are multiple\n # distro specific files; e.g. CentOS, Oracle, Enterprise all\n # containing `redhat-release` on top of their own.\n etc_files.sort()\n except OSError:\n # This may occur when /etc is not readable but we can't be\n # sure about the *-release files. Check common entries of\n # /etc for information. If they turn out to not be there the\n # error is handled in `_parse_distro_release_file()`.\n etc_files = [\n os.path.join(self.etc_dir, basename)\n for basename in _DISTRO_RELEASE_BASENAMES\n ]\n\n for filepath in etc_files:\n match = _DISTRO_RELEASE_BASENAME_PATTERN.match(os.path.basename(filepath))\n if match is None:\n continue\n distro_info = self._parse_distro_release_file(filepath)\n # The name is always present if the pattern matches.\n if \"name\" not in distro_info:\n continue\n self.distro_release_file = filepath\n break\n else: # the loop didn't \"break\": no candidate.\n return {}\n\n if match is not None:\n distro_info[\"id\"] = match.group(1)\n\n # CloudLinux < 7: manually enrich info with proper id.\n if \"cloudlinux\" in distro_info.get(\"name\", \"\").lower():\n distro_info[\"id\"] = \"cloudlinux\"\n\n return distro_info\n\n def _parse_distro_release_file(self, filepath: str) -> Dict[str, str]:\n \"\"\"\n Parse a distro release file.\n\n Parameters:\n\n * filepath: Path name of the distro release file.\n\n Returns:\n A dictionary containing all information items.\n \"\"\"\n try:\n with open(filepath, encoding=\"utf-8\") as fp:\n # Only parse the first line. For instance, on SLES there\n # are multiple lines. We don't want them...\n return self._parse_distro_release_content(fp.readline())\n except OSError:\n # Ignore not being able to read a specific, seemingly version\n # related file.\n # See https://github.com/python-distro/distro/issues/162\n return {}\n\n @staticmethod\n def _parse_distro_release_content(line: str) -> Dict[str, str]:\n \"\"\"\n Parse a line from a distro release file.\n\n Parameters:\n * line: Line from the distro release file. Must be a unicode string\n or a UTF-8 encoded byte string.\n\n Returns:\n A dictionary containing all information items.\n \"\"\"\n matches = _DISTRO_RELEASE_CONTENT_REVERSED_PATTERN.match(line.strip()[::-1])\n distro_info = {}\n if matches:\n # regexp ensures non-None\n distro_info[\"name\"] = matches.group(3)[::-1]\n if matches.group(2):\n distro_info[\"version_id\"] = matches.group(2)[::-1]\n if matches.group(1):\n distro_info[\"codename\"] = matches.group(1)[::-1]\n elif line:\n distro_info[\"name\"] = line.strip()\n return distro_info\n\n\n_distro = LinuxDistribution()\n\n\ndef main() -> None:\n logger = logging.getLogger(__name__)\n logger.setLevel(logging.DEBUG)\n logger.addHandler(logging.StreamHandler(sys.stdout))\n\n parser = argparse.ArgumentParser(description=\"OS distro info tool\")\n parser.add_argument(\n \"--json\", \"-j\", help=\"Output in machine readable format\", action=\"store_true\"\n )\n\n parser.add_argument(\n \"--root-dir\",\n \"-r\",\n type=str,\n dest=\"root_dir\",\n help=\"Path to the root filesystem directory (defaults to /)\",\n )\n\n args = parser.parse_args()\n\n if args.root_dir:\n dist = LinuxDistribution(\n include_lsb=False,\n include_uname=False,\n include_oslevel=False,\n root_dir=args.root_dir,\n )\n else:\n dist = _distro\n\n if args.json:\n logger.info(json.dumps(dist.info(), indent=4, sort_keys=True))\n else:\n logger.info(\"Name: %s\", dist.name(pretty=True))\n distribution_version = dist.version(pretty=True)\n logger.info(\"Version: %s\", distribution_version)\n distribution_codename = dist.codename()\n logger.info(\"Codename: %s\", distribution_codename)\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "lib/spack/spack/vendor/distro/py.typed", "content": "" }, { "path": "lib/spack/spack/vendor/jinja2/LICENSE.rst", "content": "Copyright 2007 Pallets\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are\nmet:\n\n1. Redistributions of source code must retain the above copyright\n notice, this list of conditions and the following disclaimer.\n\n2. Redistributions in binary form must reproduce the above copyright\n notice, this list of conditions and the following disclaimer in the\n documentation and/or other materials provided with the distribution.\n\n3. Neither the name of the copyright holder nor the names of its\n contributors may be used to endorse or promote products derived from\n this software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n\"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\nLIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A\nPARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\nHOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\nSPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED\nTO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR\nPROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF\nLIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING\nNEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" }, { "path": "lib/spack/spack/vendor/jinja2/__init__.py", "content": "\"\"\"Jinja is a template engine written in pure Python. It provides a\nnon-XML syntax that supports inline expressions and an optional\nsandboxed environment.\n\"\"\"\nfrom .bccache import BytecodeCache as BytecodeCache\nfrom .bccache import FileSystemBytecodeCache as FileSystemBytecodeCache\nfrom .bccache import MemcachedBytecodeCache as MemcachedBytecodeCache\nfrom .environment import Environment as Environment\nfrom .environment import Template as Template\nfrom .exceptions import TemplateAssertionError as TemplateAssertionError\nfrom .exceptions import TemplateError as TemplateError\nfrom .exceptions import TemplateNotFound as TemplateNotFound\nfrom .exceptions import TemplateRuntimeError as TemplateRuntimeError\nfrom .exceptions import TemplatesNotFound as TemplatesNotFound\nfrom .exceptions import TemplateSyntaxError as TemplateSyntaxError\nfrom .exceptions import UndefinedError as UndefinedError\nfrom .filters import contextfilter\nfrom .filters import environmentfilter\nfrom .filters import evalcontextfilter\nfrom .loaders import BaseLoader as BaseLoader\nfrom .loaders import ChoiceLoader as ChoiceLoader\nfrom .loaders import DictLoader as DictLoader\nfrom .loaders import FileSystemLoader as FileSystemLoader\nfrom .loaders import FunctionLoader as FunctionLoader\nfrom .loaders import ModuleLoader as ModuleLoader\nfrom .loaders import PackageLoader as PackageLoader\nfrom .loaders import PrefixLoader as PrefixLoader\nfrom .runtime import ChainableUndefined as ChainableUndefined\nfrom .runtime import DebugUndefined as DebugUndefined\nfrom .runtime import make_logging_undefined as make_logging_undefined\nfrom .runtime import StrictUndefined as StrictUndefined\nfrom .runtime import Undefined as Undefined\nfrom .utils import clear_caches as clear_caches\nfrom .utils import contextfunction\nfrom .utils import environmentfunction\nfrom .utils import escape\nfrom .utils import evalcontextfunction\nfrom .utils import is_undefined as is_undefined\nfrom .utils import Markup\nfrom .utils import pass_context as pass_context\nfrom .utils import pass_environment as pass_environment\nfrom .utils import pass_eval_context as pass_eval_context\nfrom .utils import select_autoescape as select_autoescape\n\n__version__ = \"3.0.3\"\n" }, { "path": "lib/spack/spack/vendor/jinja2/_identifier.py", "content": "import re\n\n# generated by scripts/generate_identifier_pattern.py\npattern = re.compile(\n r\"[\\w·̀-ͯ·҃-֑҇-ׇֽֿׁׂׅׄؐ-ًؚ-ٰٟۖ-ۜ۟-۪ۤۧۨ-ܑۭܰ-݊ަ-ް߫-߳ࠖ-࠙ࠛ-ࠣࠥ-ࠧࠩ-࡙࠭-࡛ࣔ-ࣣ࣡-ःऺ-़ा-ॏ॑-ॗॢॣঁ-ঃ়া-ৄেৈো-্ৗৢৣਁ-ਃ਼ਾ-ੂੇੈੋ-੍ੑੰੱੵઁ-ઃ઼ા-ૅે-ૉો-્ૢૣଁ-ଃ଼ା-ୄେୈୋ-୍ୖୗୢୣஂா-ூெ-ைொ-்ௗఀ-ఃా-ౄె-ైొ-్ౕౖౢౣಁ-ಃ಼ಾ-ೄೆ-ೈೊ-್ೕೖೢೣഁ-ഃാ-ൄെ-ൈൊ-്ൗൢൣංඃ්ා-ුූෘ-ෟෲෳัิ-ฺ็-๎ັິ-ູົຼ່-ໍ༹༘༙༵༷༾༿ཱ-྄྆྇ྍ-ྗྙ-ྼ࿆ါ-ှၖ-ၙၞ-ၠၢ-ၤၧ-ၭၱ-ၴႂ-ႍႏႚ-ႝ፝-፟ᜒ-᜔ᜲ-᜴ᝒᝓᝲᝳ឴-៓៝᠋-᠍ᢅᢆᢩᤠ-ᤫᤰ-᤻ᨗ-ᨛᩕ-ᩞ᩠-᩿᩼᪰-᪽ᬀ-ᬄ᬴-᭄᭫-᭳ᮀ-ᮂᮡ-ᮭ᯦-᯳ᰤ-᰷᳐-᳔᳒-᳨᳭ᳲ-᳴᳸᳹᷀-᷵᷻-᷿‿⁀⁔⃐-⃥⃜⃡-⃰℘℮⳯-⵿⳱ⷠ-〪ⷿ-゙゚〯꙯ꙴ-꙽ꚞꚟ꛰꛱ꠂ꠆ꠋꠣ-ꠧꢀꢁꢴ-ꣅ꣠-꣱ꤦ-꤭ꥇ-꥓ꦀ-ꦃ꦳-꧀ꧥꨩ-ꨶꩃꩌꩍꩻ-ꩽꪰꪲ-ꪴꪷꪸꪾ꪿꫁ꫫ-ꫯꫵ꫶ꯣ-ꯪ꯬꯭ﬞ︀-️︠-︯︳︴﹍-﹏＿𐇽𐋠𐍶-𐍺𐨁-𐨃𐨅𐨆𐨌-𐨏𐨸-𐨿𐨺𐫦𐫥𑀀-𑀂𑀸-𑁆𑁿-𑂂𑂰-𑂺𑄀-𑄂𑄧-𑅳𑄴𑆀-𑆂𑆳-𑇊𑇀-𑇌𑈬-𑈷𑈾𑋟-𑋪𑌀-𑌃𑌼𑌾-𑍄𑍇𑍈𑍋-𑍍𑍗𑍢𑍣𑍦-𑍬𑍰-𑍴𑐵-𑑆𑒰-𑓃𑖯-𑖵𑖸-𑗀𑗜𑗝𑘰-𑙀𑚫-𑚷𑜝-𑜫𑰯-𑰶𑰸-𑰿𑲒-𑲧𑲩-𑲶𖫰-𖫴𖬰-𖬶𖽑-𖽾𖾏-𖾒𛲝𛲞𝅥-𝅩𝅭-𝅲𝅻-𝆂𝆅-𝆋𝆪-𝆭𝉂-𝉄𝨀-𝨶𝨻-𝩬𝩵𝪄𝪛-𝪟𝪡-𝪯𞀀-𞀆𞀈-𞀘𞀛-𞀡𞀣𞀤𞀦-𞣐𞀪-𞣖𞥄-𞥊󠄀-󠇯]+\" # noqa: B950\n)\n" }, { "path": "lib/spack/spack/vendor/jinja2/async_utils.py", "content": "import inspect\nimport typing as t\nfrom functools import wraps\n\nfrom .utils import _PassArg\nfrom .utils import pass_eval_context\n\nV = t.TypeVar(\"V\")\n\n\ndef async_variant(normal_func): # type: ignore\n def decorator(async_func): # type: ignore\n pass_arg = _PassArg.from_obj(normal_func)\n need_eval_context = pass_arg is None\n\n if pass_arg is _PassArg.environment:\n\n def is_async(args: t.Any) -> bool:\n return t.cast(bool, args[0].is_async)\n\n else:\n\n def is_async(args: t.Any) -> bool:\n return t.cast(bool, args[0].environment.is_async)\n\n @wraps(normal_func)\n def wrapper(*args, **kwargs): # type: ignore\n b = is_async(args)\n\n if need_eval_context:\n args = args[1:]\n\n if b:\n return async_func(*args, **kwargs)\n\n return normal_func(*args, **kwargs)\n\n if need_eval_context:\n wrapper = pass_eval_context(wrapper)\n\n wrapper.jinja_async_variant = True\n return wrapper\n\n return decorator\n\n\n_common_primitives = {int, float, bool, str, list, dict, tuple, type(None)}\n\n\nasync def auto_await(value: t.Union[t.Awaitable[\"V\"], \"V\"]) -> \"V\":\n # Avoid a costly call to isawaitable\n if type(value) in _common_primitives:\n return t.cast(\"V\", value)\n\n if inspect.isawaitable(value):\n return await t.cast(\"t.Awaitable[V]\", value)\n\n return t.cast(\"V\", value)\n\n\nasync def auto_aiter(\n iterable: \"t.Union[t.AsyncIterable[V], t.Iterable[V]]\",\n) -> \"t.AsyncIterator[V]\":\n if hasattr(iterable, \"__aiter__\"):\n async for item in t.cast(\"t.AsyncIterable[V]\", iterable):\n yield item\n else:\n for item in t.cast(\"t.Iterable[V]\", iterable):\n yield item\n\n\nasync def auto_to_list(\n value: \"t.Union[t.AsyncIterable[V], t.Iterable[V]]\",\n) -> t.List[\"V\"]:\n return [x async for x in auto_aiter(value)]\n" }, { "path": "lib/spack/spack/vendor/jinja2/bccache.py", "content": "\"\"\"The optional bytecode cache system. This is useful if you have very\ncomplex template situations and the compilation of all those templates\nslows down your application too much.\n\nSituations where this is useful are often forking web applications that\nare initialized on the first request.\n\"\"\"\nimport errno\nimport fnmatch\nimport marshal\nimport os\nimport pickle\nimport stat\nimport sys\nimport tempfile\nimport typing as t\nfrom hashlib import sha1\nfrom io import BytesIO\nfrom types import CodeType\n\nif t.TYPE_CHECKING:\n import spack.vendor.typing_extensions as te\n from .environment import Environment\n\n class _MemcachedClient(te.Protocol):\n def get(self, key: str) -> bytes:\n ...\n\n def set(self, key: str, value: bytes, timeout: t.Optional[int] = None) -> None:\n ...\n\n\nbc_version = 5\n# Magic bytes to identify Jinja bytecode cache files. Contains the\n# Python major and minor version to avoid loading incompatible bytecode\n# if a project upgrades its Python version.\nbc_magic = (\n b\"j2\"\n + pickle.dumps(bc_version, 2)\n + pickle.dumps((sys.version_info[0] << 24) | sys.version_info[1], 2)\n)\n\n\nclass Bucket:\n \"\"\"Buckets are used to store the bytecode for one template. It's created\n and initialized by the bytecode cache and passed to the loading functions.\n\n The buckets get an internal checksum from the cache assigned and use this\n to automatically reject outdated cache material. Individual bytecode\n cache subclasses don't have to care about cache invalidation.\n \"\"\"\n\n def __init__(self, environment: \"Environment\", key: str, checksum: str) -> None:\n self.environment = environment\n self.key = key\n self.checksum = checksum\n self.reset()\n\n def reset(self) -> None:\n \"\"\"Resets the bucket (unloads the bytecode).\"\"\"\n self.code: t.Optional[CodeType] = None\n\n def load_bytecode(self, f: t.BinaryIO) -> None:\n \"\"\"Loads bytecode from a file or file like object.\"\"\"\n # make sure the magic header is correct\n magic = f.read(len(bc_magic))\n if magic != bc_magic:\n self.reset()\n return\n # the source code of the file changed, we need to reload\n checksum = pickle.load(f)\n if self.checksum != checksum:\n self.reset()\n return\n # if marshal_load fails then we need to reload\n try:\n self.code = marshal.load(f)\n except (EOFError, ValueError, TypeError):\n self.reset()\n return\n\n def write_bytecode(self, f: t.BinaryIO) -> None:\n \"\"\"Dump the bytecode into the file or file like object passed.\"\"\"\n if self.code is None:\n raise TypeError(\"can't write empty bucket\")\n f.write(bc_magic)\n pickle.dump(self.checksum, f, 2)\n marshal.dump(self.code, f)\n\n def bytecode_from_string(self, string: bytes) -> None:\n \"\"\"Load bytecode from bytes.\"\"\"\n self.load_bytecode(BytesIO(string))\n\n def bytecode_to_string(self) -> bytes:\n \"\"\"Return the bytecode as bytes.\"\"\"\n out = BytesIO()\n self.write_bytecode(out)\n return out.getvalue()\n\n\nclass BytecodeCache:\n \"\"\"To implement your own bytecode cache you have to subclass this class\n and override :meth:`load_bytecode` and :meth:`dump_bytecode`. Both of\n these methods are passed a :class:`~spack.vendor.jinja2.bccache.Bucket`.\n\n A very basic bytecode cache that saves the bytecode on the file system::\n\n from os import path\n\n class MyCache(BytecodeCache):\n\n def __init__(self, directory):\n self.directory = directory\n\n def load_bytecode(self, bucket):\n filename = path.join(self.directory, bucket.key)\n if path.exists(filename):\n with open(filename, 'rb') as f:\n bucket.load_bytecode(f)\n\n def dump_bytecode(self, bucket):\n filename = path.join(self.directory, bucket.key)\n with open(filename, 'wb') as f:\n bucket.write_bytecode(f)\n\n A more advanced version of a filesystem based bytecode cache is part of\n Jinja.\n \"\"\"\n\n def load_bytecode(self, bucket: Bucket) -> None:\n \"\"\"Subclasses have to override this method to load bytecode into a\n bucket. If they are not able to find code in the cache for the\n bucket, it must not do anything.\n \"\"\"\n raise NotImplementedError()\n\n def dump_bytecode(self, bucket: Bucket) -> None:\n \"\"\"Subclasses have to override this method to write the bytecode\n from a bucket back to the cache. If it unable to do so it must not\n fail silently but raise an exception.\n \"\"\"\n raise NotImplementedError()\n\n def clear(self) -> None:\n \"\"\"Clears the cache. This method is not used by Jinja but should be\n implemented to allow applications to clear the bytecode cache used\n by a particular environment.\n \"\"\"\n\n def get_cache_key(\n self, name: str, filename: t.Optional[t.Union[str]] = None\n ) -> str:\n \"\"\"Returns the unique hash key for this template name.\"\"\"\n hash = sha1(name.encode(\"utf-8\"))\n\n if filename is not None:\n hash.update(f\"|{filename}\".encode())\n\n return hash.hexdigest()\n\n def get_source_checksum(self, source: str) -> str:\n \"\"\"Returns a checksum for the source.\"\"\"\n return sha1(source.encode(\"utf-8\")).hexdigest()\n\n def get_bucket(\n self,\n environment: \"Environment\",\n name: str,\n filename: t.Optional[str],\n source: str,\n ) -> Bucket:\n \"\"\"Return a cache bucket for the given template. All arguments are\n mandatory but filename may be `None`.\n \"\"\"\n key = self.get_cache_key(name, filename)\n checksum = self.get_source_checksum(source)\n bucket = Bucket(environment, key, checksum)\n self.load_bytecode(bucket)\n return bucket\n\n def set_bucket(self, bucket: Bucket) -> None:\n \"\"\"Put the bucket into the cache.\"\"\"\n self.dump_bytecode(bucket)\n\n\nclass FileSystemBytecodeCache(BytecodeCache):\n \"\"\"A bytecode cache that stores bytecode on the filesystem. It accepts\n two arguments: The directory where the cache items are stored and a\n pattern string that is used to build the filename.\n\n If no directory is specified a default cache directory is selected. On\n Windows the user's temp directory is used, on UNIX systems a directory\n is created for the user in the system temp directory.\n\n The pattern can be used to have multiple separate caches operate on the\n same directory. The default pattern is ``'__spack.vendor.jinja2_%s.cache'``. ``%s``\n is replaced with the cache key.\n\n >>> bcc = FileSystemBytecodeCache('/tmp/jinja_cache', '%s.cache')\n\n This bytecode cache supports clearing of the cache using the clear method.\n \"\"\"\n\n def __init__(\n self, directory: t.Optional[str] = None, pattern: str = \"__spack.vendor.jinja2_%s.cache\"\n ) -> None:\n if directory is None:\n directory = self._get_default_cache_dir()\n self.directory = directory\n self.pattern = pattern\n\n def _get_default_cache_dir(self) -> str:\n def _unsafe_dir() -> \"te.NoReturn\":\n raise RuntimeError(\n \"Cannot determine safe temp directory. You \"\n \"need to explicitly provide one.\"\n )\n\n tmpdir = tempfile.gettempdir()\n\n # On windows the temporary directory is used specific unless\n # explicitly forced otherwise. We can just use that.\n if os.name == \"nt\":\n return tmpdir\n if not hasattr(os, \"getuid\"):\n _unsafe_dir()\n\n dirname = f\"_spack.vendor.jinja2-cache-{os.getuid()}\"\n actual_dir = os.path.join(tmpdir, dirname)\n\n try:\n os.mkdir(actual_dir, stat.S_IRWXU)\n except OSError as e:\n if e.errno != errno.EEXIST:\n raise\n try:\n os.chmod(actual_dir, stat.S_IRWXU)\n actual_dir_stat = os.lstat(actual_dir)\n if (\n actual_dir_stat.st_uid != os.getuid()\n or not stat.S_ISDIR(actual_dir_stat.st_mode)\n or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU\n ):\n _unsafe_dir()\n except OSError as e:\n if e.errno != errno.EEXIST:\n raise\n\n actual_dir_stat = os.lstat(actual_dir)\n if (\n actual_dir_stat.st_uid != os.getuid()\n or not stat.S_ISDIR(actual_dir_stat.st_mode)\n or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU\n ):\n _unsafe_dir()\n\n return actual_dir\n\n def _get_cache_filename(self, bucket: Bucket) -> str:\n return os.path.join(self.directory, self.pattern % (bucket.key,))\n\n def load_bytecode(self, bucket: Bucket) -> None:\n filename = self._get_cache_filename(bucket)\n\n if os.path.exists(filename):\n with open(filename, \"rb\") as f:\n bucket.load_bytecode(f)\n\n def dump_bytecode(self, bucket: Bucket) -> None:\n with open(self._get_cache_filename(bucket), \"wb\") as f:\n bucket.write_bytecode(f)\n\n def clear(self) -> None:\n # imported lazily here because google app-engine doesn't support\n # write access on the file system and the function does not exist\n # normally.\n from os import remove\n\n files = fnmatch.filter(os.listdir(self.directory), self.pattern % (\"*\",))\n for filename in files:\n try:\n remove(os.path.join(self.directory, filename))\n except OSError:\n pass\n\n\nclass MemcachedBytecodeCache(BytecodeCache):\n \"\"\"This class implements a bytecode cache that uses a memcache cache for\n storing the information. It does not enforce a specific memcache library\n (tummy's memcache or cmemcache) but will accept any class that provides\n the minimal interface required.\n\n Libraries compatible with this class:\n\n - `cachelib `_\n - `python-memcached `_\n\n (Unfortunately the django cache interface is not compatible because it\n does not support storing binary data, only text. You can however pass\n the underlying cache client to the bytecode cache which is available\n as `django.core.cache.cache._client`.)\n\n The minimal interface for the client passed to the constructor is this:\n\n .. class:: MinimalClientInterface\n\n .. method:: set(key, value[, timeout])\n\n Stores the bytecode in the cache. `value` is a string and\n `timeout` the timeout of the key. If timeout is not provided\n a default timeout or no timeout should be assumed, if it's\n provided it's an integer with the number of seconds the cache\n item should exist.\n\n .. method:: get(key)\n\n Returns the value for the cache key. If the item does not\n exist in the cache the return value must be `None`.\n\n The other arguments to the constructor are the prefix for all keys that\n is added before the actual cache key and the timeout for the bytecode in\n the cache system. We recommend a high (or no) timeout.\n\n This bytecode cache does not support clearing of used items in the cache.\n The clear method is a no-operation function.\n\n .. versionadded:: 2.7\n Added support for ignoring memcache errors through the\n `ignore_memcache_errors` parameter.\n \"\"\"\n\n def __init__(\n self,\n client: \"_MemcachedClient\",\n prefix: str = \"spack.vendor.jinja2/bytecode/\",\n timeout: t.Optional[int] = None,\n ignore_memcache_errors: bool = True,\n ):\n self.client = client\n self.prefix = prefix\n self.timeout = timeout\n self.ignore_memcache_errors = ignore_memcache_errors\n\n def load_bytecode(self, bucket: Bucket) -> None:\n try:\n code = self.client.get(self.prefix + bucket.key)\n except Exception:\n if not self.ignore_memcache_errors:\n raise\n else:\n bucket.bytecode_from_string(code)\n\n def dump_bytecode(self, bucket: Bucket) -> None:\n key = self.prefix + bucket.key\n value = bucket.bytecode_to_string()\n\n try:\n if self.timeout is not None:\n self.client.set(key, value, self.timeout)\n else:\n self.client.set(key, value)\n except Exception:\n if not self.ignore_memcache_errors:\n raise\n" }, { "path": "lib/spack/spack/vendor/jinja2/compiler.py", "content": "\"\"\"Compiles nodes from the parser into Python code.\"\"\"\nimport typing as t\nfrom contextlib import contextmanager\nfrom functools import update_wrapper\nfrom io import StringIO\nfrom itertools import chain\nfrom keyword import iskeyword as is_python_keyword\n\nfrom spack.vendor.markupsafe import escape\nfrom spack.vendor.markupsafe import Markup\n\nfrom . import nodes\nfrom .exceptions import TemplateAssertionError\nfrom .idtracking import Symbols\nfrom .idtracking import VAR_LOAD_ALIAS\nfrom .idtracking import VAR_LOAD_PARAMETER\nfrom .idtracking import VAR_LOAD_RESOLVE\nfrom .idtracking import VAR_LOAD_UNDEFINED\nfrom .nodes import EvalContext\nfrom .optimizer import Optimizer\nfrom .utils import _PassArg\nfrom .utils import concat\nfrom .visitor import NodeVisitor\n\nif t.TYPE_CHECKING:\n import spack.vendor.typing_extensions as te\n from .environment import Environment\n\nF = t.TypeVar(\"F\", bound=t.Callable[..., t.Any])\n\noperators = {\n \"eq\": \"==\",\n \"ne\": \"!=\",\n \"gt\": \">\",\n \"gteq\": \">=\",\n \"lt\": \"<\",\n \"lteq\": \"<=\",\n \"in\": \"in\",\n \"notin\": \"not in\",\n}\n\n\ndef optimizeconst(f: F) -> F:\n def new_func(\n self: \"CodeGenerator\", node: nodes.Expr, frame: \"Frame\", **kwargs: t.Any\n ) -> t.Any:\n # Only optimize if the frame is not volatile\n if self.optimizer is not None and not frame.eval_ctx.volatile:\n new_node = self.optimizer.visit(node, frame.eval_ctx)\n\n if new_node != node:\n return self.visit(new_node, frame)\n\n return f(self, node, frame, **kwargs)\n\n return update_wrapper(t.cast(F, new_func), f)\n\n\ndef _make_binop(op: str) -> t.Callable[[\"CodeGenerator\", nodes.BinExpr, \"Frame\"], None]:\n @optimizeconst\n def visitor(self: \"CodeGenerator\", node: nodes.BinExpr, frame: Frame) -> None:\n if (\n self.environment.sandboxed\n and op in self.environment.intercepted_binops # type: ignore\n ):\n self.write(f\"environment.call_binop(context, {op!r}, \")\n self.visit(node.left, frame)\n self.write(\", \")\n self.visit(node.right, frame)\n else:\n self.write(\"(\")\n self.visit(node.left, frame)\n self.write(f\" {op} \")\n self.visit(node.right, frame)\n\n self.write(\")\")\n\n return visitor\n\n\ndef _make_unop(\n op: str,\n) -> t.Callable[[\"CodeGenerator\", nodes.UnaryExpr, \"Frame\"], None]:\n @optimizeconst\n def visitor(self: \"CodeGenerator\", node: nodes.UnaryExpr, frame: Frame) -> None:\n if (\n self.environment.sandboxed\n and op in self.environment.intercepted_unops # type: ignore\n ):\n self.write(f\"environment.call_unop(context, {op!r}, \")\n self.visit(node.node, frame)\n else:\n self.write(\"(\" + op)\n self.visit(node.node, frame)\n\n self.write(\")\")\n\n return visitor\n\n\ndef generate(\n node: nodes.Template,\n environment: \"Environment\",\n name: t.Optional[str],\n filename: t.Optional[str],\n stream: t.Optional[t.TextIO] = None,\n defer_init: bool = False,\n optimized: bool = True,\n) -> t.Optional[str]:\n \"\"\"Generate the python source for a node tree.\"\"\"\n if not isinstance(node, nodes.Template):\n raise TypeError(\"Can't compile non template nodes\")\n\n generator = environment.code_generator_class(\n environment, name, filename, stream, defer_init, optimized\n )\n generator.visit(node)\n\n if stream is None:\n return generator.stream.getvalue() # type: ignore\n\n return None\n\n\ndef has_safe_repr(value: t.Any) -> bool:\n \"\"\"Does the node have a safe representation?\"\"\"\n if value is None or value is NotImplemented or value is Ellipsis:\n return True\n\n if type(value) in {bool, int, float, complex, range, str, Markup}:\n return True\n\n if type(value) in {tuple, list, set, frozenset}:\n return all(has_safe_repr(v) for v in value)\n\n if type(value) is dict:\n return all(has_safe_repr(k) and has_safe_repr(v) for k, v in value.items())\n\n return False\n\n\ndef find_undeclared(\n nodes: t.Iterable[nodes.Node], names: t.Iterable[str]\n) -> t.Set[str]:\n \"\"\"Check if the names passed are accessed undeclared. The return value\n is a set of all the undeclared names from the sequence of names found.\n \"\"\"\n visitor = UndeclaredNameVisitor(names)\n try:\n for node in nodes:\n visitor.visit(node)\n except VisitorExit:\n pass\n return visitor.undeclared\n\n\nclass MacroRef:\n def __init__(self, node: t.Union[nodes.Macro, nodes.CallBlock]) -> None:\n self.node = node\n self.accesses_caller = False\n self.accesses_kwargs = False\n self.accesses_varargs = False\n\n\nclass Frame:\n \"\"\"Holds compile time information for us.\"\"\"\n\n def __init__(\n self,\n eval_ctx: EvalContext,\n parent: t.Optional[\"Frame\"] = None,\n level: t.Optional[int] = None,\n ) -> None:\n self.eval_ctx = eval_ctx\n\n # the parent of this frame\n self.parent = parent\n\n if parent is None:\n self.symbols = Symbols(level=level)\n\n # in some dynamic inheritance situations the compiler needs to add\n # write tests around output statements.\n self.require_output_check = False\n\n # inside some tags we are using a buffer rather than yield statements.\n # this for example affects {% filter %} or {% macro %}. If a frame\n # is buffered this variable points to the name of the list used as\n # buffer.\n self.buffer: t.Optional[str] = None\n\n # the name of the block we're in, otherwise None.\n self.block: t.Optional[str] = None\n\n else:\n self.symbols = Symbols(parent.symbols, level=level)\n self.require_output_check = parent.require_output_check\n self.buffer = parent.buffer\n self.block = parent.block\n\n # a toplevel frame is the root + soft frames such as if conditions.\n self.toplevel = False\n\n # the root frame is basically just the outermost frame, so no if\n # conditions. This information is used to optimize inheritance\n # situations.\n self.rootlevel = False\n\n # variables set inside of loops and blocks should not affect outer frames,\n # but they still needs to be kept track of as part of the active context.\n self.loop_frame = False\n self.block_frame = False\n\n # track whether the frame is being used in an if-statement or conditional\n # expression as it determines which errors should be raised during runtime\n # or compile time.\n self.soft_frame = False\n\n def copy(self) -> \"Frame\":\n \"\"\"Create a copy of the current one.\"\"\"\n rv = t.cast(Frame, object.__new__(self.__class__))\n rv.__dict__.update(self.__dict__)\n rv.symbols = self.symbols.copy()\n return rv\n\n def inner(self, isolated: bool = False) -> \"Frame\":\n \"\"\"Return an inner frame.\"\"\"\n if isolated:\n return Frame(self.eval_ctx, level=self.symbols.level + 1)\n return Frame(self.eval_ctx, self)\n\n def soft(self) -> \"Frame\":\n \"\"\"Return a soft frame. A soft frame may not be modified as\n standalone thing as it shares the resources with the frame it\n was created of, but it's not a rootlevel frame any longer.\n\n This is only used to implement if-statements and conditional\n expressions.\n \"\"\"\n rv = self.copy()\n rv.rootlevel = False\n rv.soft_frame = True\n return rv\n\n __copy__ = copy\n\n\nclass VisitorExit(RuntimeError):\n \"\"\"Exception used by the `UndeclaredNameVisitor` to signal a stop.\"\"\"\n\n\nclass DependencyFinderVisitor(NodeVisitor):\n \"\"\"A visitor that collects filter and test calls.\"\"\"\n\n def __init__(self) -> None:\n self.filters: t.Set[str] = set()\n self.tests: t.Set[str] = set()\n\n def visit_Filter(self, node: nodes.Filter) -> None:\n self.generic_visit(node)\n self.filters.add(node.name)\n\n def visit_Test(self, node: nodes.Test) -> None:\n self.generic_visit(node)\n self.tests.add(node.name)\n\n def visit_Block(self, node: nodes.Block) -> None:\n \"\"\"Stop visiting at blocks.\"\"\"\n\n\nclass UndeclaredNameVisitor(NodeVisitor):\n \"\"\"A visitor that checks if a name is accessed without being\n declared. This is different from the frame visitor as it will\n not stop at closure frames.\n \"\"\"\n\n def __init__(self, names: t.Iterable[str]) -> None:\n self.names = set(names)\n self.undeclared: t.Set[str] = set()\n\n def visit_Name(self, node: nodes.Name) -> None:\n if node.ctx == \"load\" and node.name in self.names:\n self.undeclared.add(node.name)\n if self.undeclared == self.names:\n raise VisitorExit()\n else:\n self.names.discard(node.name)\n\n def visit_Block(self, node: nodes.Block) -> None:\n \"\"\"Stop visiting a blocks.\"\"\"\n\n\nclass CompilerExit(Exception):\n \"\"\"Raised if the compiler encountered a situation where it just\n doesn't make sense to further process the code. Any block that\n raises such an exception is not further processed.\n \"\"\"\n\n\nclass CodeGenerator(NodeVisitor):\n def __init__(\n self,\n environment: \"Environment\",\n name: t.Optional[str],\n filename: t.Optional[str],\n stream: t.Optional[t.TextIO] = None,\n defer_init: bool = False,\n optimized: bool = True,\n ) -> None:\n if stream is None:\n stream = StringIO()\n self.environment = environment\n self.name = name\n self.filename = filename\n self.stream = stream\n self.created_block_context = False\n self.defer_init = defer_init\n self.optimizer: t.Optional[Optimizer] = None\n\n if optimized:\n self.optimizer = Optimizer(environment)\n\n # aliases for imports\n self.import_aliases: t.Dict[str, str] = {}\n\n # a registry for all blocks. Because blocks are moved out\n # into the global python scope they are registered here\n self.blocks: t.Dict[str, nodes.Block] = {}\n\n # the number of extends statements so far\n self.extends_so_far = 0\n\n # some templates have a rootlevel extends. In this case we\n # can safely assume that we're a child template and do some\n # more optimizations.\n self.has_known_extends = False\n\n # the current line number\n self.code_lineno = 1\n\n # registry of all filters and tests (global, not block local)\n self.tests: t.Dict[str, str] = {}\n self.filters: t.Dict[str, str] = {}\n\n # the debug information\n self.debug_info: t.List[t.Tuple[int, int]] = []\n self._write_debug_info: t.Optional[int] = None\n\n # the number of new lines before the next write()\n self._new_lines = 0\n\n # the line number of the last written statement\n self._last_line = 0\n\n # true if nothing was written so far.\n self._first_write = True\n\n # used by the `temporary_identifier` method to get new\n # unique, temporary identifier\n self._last_identifier = 0\n\n # the current indentation\n self._indentation = 0\n\n # Tracks toplevel assignments\n self._assign_stack: t.List[t.Set[str]] = []\n\n # Tracks parameter definition blocks\n self._param_def_block: t.List[t.Set[str]] = []\n\n # Tracks the current context.\n self._context_reference_stack = [\"context\"]\n\n @property\n def optimized(self) -> bool:\n return self.optimizer is not None\n\n # -- Various compilation helpers\n\n def fail(self, msg: str, lineno: int) -> \"te.NoReturn\":\n \"\"\"Fail with a :exc:`TemplateAssertionError`.\"\"\"\n raise TemplateAssertionError(msg, lineno, self.name, self.filename)\n\n def temporary_identifier(self) -> str:\n \"\"\"Get a new unique identifier.\"\"\"\n self._last_identifier += 1\n return f\"t_{self._last_identifier}\"\n\n def buffer(self, frame: Frame) -> None:\n \"\"\"Enable buffering for the frame from that point onwards.\"\"\"\n frame.buffer = self.temporary_identifier()\n self.writeline(f\"{frame.buffer} = []\")\n\n def return_buffer_contents(\n self, frame: Frame, force_unescaped: bool = False\n ) -> None:\n \"\"\"Return the buffer contents of the frame.\"\"\"\n if not force_unescaped:\n if frame.eval_ctx.volatile:\n self.writeline(\"if context.eval_ctx.autoescape:\")\n self.indent()\n self.writeline(f\"return Markup(concat({frame.buffer}))\")\n self.outdent()\n self.writeline(\"else:\")\n self.indent()\n self.writeline(f\"return concat({frame.buffer})\")\n self.outdent()\n return\n elif frame.eval_ctx.autoescape:\n self.writeline(f\"return Markup(concat({frame.buffer}))\")\n return\n self.writeline(f\"return concat({frame.buffer})\")\n\n def indent(self) -> None:\n \"\"\"Indent by one.\"\"\"\n self._indentation += 1\n\n def outdent(self, step: int = 1) -> None:\n \"\"\"Outdent by step.\"\"\"\n self._indentation -= step\n\n def start_write(self, frame: Frame, node: t.Optional[nodes.Node] = None) -> None:\n \"\"\"Yield or write into the frame buffer.\"\"\"\n if frame.buffer is None:\n self.writeline(\"yield \", node)\n else:\n self.writeline(f\"{frame.buffer}.append(\", node)\n\n def end_write(self, frame: Frame) -> None:\n \"\"\"End the writing process started by `start_write`.\"\"\"\n if frame.buffer is not None:\n self.write(\")\")\n\n def simple_write(\n self, s: str, frame: Frame, node: t.Optional[nodes.Node] = None\n ) -> None:\n \"\"\"Simple shortcut for start_write + write + end_write.\"\"\"\n self.start_write(frame, node)\n self.write(s)\n self.end_write(frame)\n\n def blockvisit(self, nodes: t.Iterable[nodes.Node], frame: Frame) -> None:\n \"\"\"Visit a list of nodes as block in a frame. If the current frame\n is no buffer a dummy ``if 0: yield None`` is written automatically.\n \"\"\"\n try:\n self.writeline(\"pass\")\n for node in nodes:\n self.visit(node, frame)\n except CompilerExit:\n pass\n\n def write(self, x: str) -> None:\n \"\"\"Write a string into the output stream.\"\"\"\n if self._new_lines:\n if not self._first_write:\n self.stream.write(\"\\n\" * self._new_lines)\n self.code_lineno += self._new_lines\n if self._write_debug_info is not None:\n self.debug_info.append((self._write_debug_info, self.code_lineno))\n self._write_debug_info = None\n self._first_write = False\n self.stream.write(\" \" * self._indentation)\n self._new_lines = 0\n self.stream.write(x)\n\n def writeline(\n self, x: str, node: t.Optional[nodes.Node] = None, extra: int = 0\n ) -> None:\n \"\"\"Combination of newline and write.\"\"\"\n self.newline(node, extra)\n self.write(x)\n\n def newline(self, node: t.Optional[nodes.Node] = None, extra: int = 0) -> None:\n \"\"\"Add one or more newlines before the next write.\"\"\"\n self._new_lines = max(self._new_lines, 1 + extra)\n if node is not None and node.lineno != self._last_line:\n self._write_debug_info = node.lineno\n self._last_line = node.lineno\n\n def signature(\n self,\n node: t.Union[nodes.Call, nodes.Filter, nodes.Test],\n frame: Frame,\n extra_kwargs: t.Optional[t.Mapping[str, t.Any]] = None,\n ) -> None:\n \"\"\"Writes a function call to the stream for the current node.\n A leading comma is added automatically. The extra keyword\n arguments may not include python keywords otherwise a syntax\n error could occur. The extra keyword arguments should be given\n as python dict.\n \"\"\"\n # if any of the given keyword arguments is a python keyword\n # we have to make sure that no invalid call is created.\n kwarg_workaround = any(\n is_python_keyword(t.cast(str, k))\n for k in chain((x.key for x in node.kwargs), extra_kwargs or ())\n )\n\n for arg in node.args:\n self.write(\", \")\n self.visit(arg, frame)\n\n if not kwarg_workaround:\n for kwarg in node.kwargs:\n self.write(\", \")\n self.visit(kwarg, frame)\n if extra_kwargs is not None:\n for key, value in extra_kwargs.items():\n self.write(f\", {key}={value}\")\n if node.dyn_args:\n self.write(\", *\")\n self.visit(node.dyn_args, frame)\n\n if kwarg_workaround:\n if node.dyn_kwargs is not None:\n self.write(\", **dict({\")\n else:\n self.write(\", **{\")\n for kwarg in node.kwargs:\n self.write(f\"{kwarg.key!r}: \")\n self.visit(kwarg.value, frame)\n self.write(\", \")\n if extra_kwargs is not None:\n for key, value in extra_kwargs.items():\n self.write(f\"{key!r}: {value}, \")\n if node.dyn_kwargs is not None:\n self.write(\"}, **\")\n self.visit(node.dyn_kwargs, frame)\n self.write(\")\")\n else:\n self.write(\"}\")\n\n elif node.dyn_kwargs is not None:\n self.write(\", **\")\n self.visit(node.dyn_kwargs, frame)\n\n def pull_dependencies(self, nodes: t.Iterable[nodes.Node]) -> None:\n \"\"\"Find all filter and test names used in the template and\n assign them to variables in the compiled namespace. Checking\n that the names are registered with the environment is done when\n compiling the Filter and Test nodes. If the node is in an If or\n CondExpr node, the check is done at runtime instead.\n\n .. versionchanged:: 3.0\n Filters and tests in If and CondExpr nodes are checked at\n runtime instead of compile time.\n \"\"\"\n visitor = DependencyFinderVisitor()\n\n for node in nodes:\n visitor.visit(node)\n\n for id_map, names, dependency in (self.filters, visitor.filters, \"filters\"), (\n self.tests,\n visitor.tests,\n \"tests\",\n ):\n for name in sorted(names):\n if name not in id_map:\n id_map[name] = self.temporary_identifier()\n\n # add check during runtime that dependencies used inside of executed\n # blocks are defined, as this step may be skipped during compile time\n self.writeline(\"try:\")\n self.indent()\n self.writeline(f\"{id_map[name]} = environment.{dependency}[{name!r}]\")\n self.outdent()\n self.writeline(\"except KeyError:\")\n self.indent()\n self.writeline(\"@internalcode\")\n self.writeline(f\"def {id_map[name]}(*unused):\")\n self.indent()\n self.writeline(\n f'raise TemplateRuntimeError(\"No {dependency[:-1]}'\n f' named {name!r} found.\")'\n )\n self.outdent()\n self.outdent()\n\n def enter_frame(self, frame: Frame) -> None:\n undefs = []\n for target, (action, param) in frame.symbols.loads.items():\n if action == VAR_LOAD_PARAMETER:\n pass\n elif action == VAR_LOAD_RESOLVE:\n self.writeline(f\"{target} = {self.get_resolve_func()}({param!r})\")\n elif action == VAR_LOAD_ALIAS:\n self.writeline(f\"{target} = {param}\")\n elif action == VAR_LOAD_UNDEFINED:\n undefs.append(target)\n else:\n raise NotImplementedError(\"unknown load instruction\")\n if undefs:\n self.writeline(f\"{' = '.join(undefs)} = missing\")\n\n def leave_frame(self, frame: Frame, with_python_scope: bool = False) -> None:\n if not with_python_scope:\n undefs = []\n for target in frame.symbols.loads:\n undefs.append(target)\n if undefs:\n self.writeline(f\"{' = '.join(undefs)} = missing\")\n\n def choose_async(self, async_value: str = \"async \", sync_value: str = \"\") -> str:\n return async_value if self.environment.is_async else sync_value\n\n def func(self, name: str) -> str:\n return f\"{self.choose_async()}def {name}\"\n\n def macro_body(\n self, node: t.Union[nodes.Macro, nodes.CallBlock], frame: Frame\n ) -> t.Tuple[Frame, MacroRef]:\n \"\"\"Dump the function def of a macro or call block.\"\"\"\n frame = frame.inner()\n frame.symbols.analyze_node(node)\n macro_ref = MacroRef(node)\n\n explicit_caller = None\n skip_special_params = set()\n args = []\n\n for idx, arg in enumerate(node.args):\n if arg.name == \"caller\":\n explicit_caller = idx\n if arg.name in (\"kwargs\", \"varargs\"):\n skip_special_params.add(arg.name)\n args.append(frame.symbols.ref(arg.name))\n\n undeclared = find_undeclared(node.body, (\"caller\", \"kwargs\", \"varargs\"))\n\n if \"caller\" in undeclared:\n # In older Jinja versions there was a bug that allowed caller\n # to retain the special behavior even if it was mentioned in\n # the argument list. However thankfully this was only really\n # working if it was the last argument. So we are explicitly\n # checking this now and error out if it is anywhere else in\n # the argument list.\n if explicit_caller is not None:\n try:\n node.defaults[explicit_caller - len(node.args)]\n except IndexError:\n self.fail(\n \"When defining macros or call blocks the \"\n 'special \"caller\" argument must be omitted '\n \"or be given a default.\",\n node.lineno,\n )\n else:\n args.append(frame.symbols.declare_parameter(\"caller\"))\n macro_ref.accesses_caller = True\n if \"kwargs\" in undeclared and \"kwargs\" not in skip_special_params:\n args.append(frame.symbols.declare_parameter(\"kwargs\"))\n macro_ref.accesses_kwargs = True\n if \"varargs\" in undeclared and \"varargs\" not in skip_special_params:\n args.append(frame.symbols.declare_parameter(\"varargs\"))\n macro_ref.accesses_varargs = True\n\n # macros are delayed, they never require output checks\n frame.require_output_check = False\n frame.symbols.analyze_node(node)\n self.writeline(f\"{self.func('macro')}({', '.join(args)}):\", node)\n self.indent()\n\n self.buffer(frame)\n self.enter_frame(frame)\n\n self.push_parameter_definitions(frame)\n for idx, arg in enumerate(node.args):\n ref = frame.symbols.ref(arg.name)\n self.writeline(f\"if {ref} is missing:\")\n self.indent()\n try:\n default = node.defaults[idx - len(node.args)]\n except IndexError:\n self.writeline(\n f'{ref} = undefined(\"parameter {arg.name!r} was not provided\",'\n f\" name={arg.name!r})\"\n )\n else:\n self.writeline(f\"{ref} = \")\n self.visit(default, frame)\n self.mark_parameter_stored(ref)\n self.outdent()\n self.pop_parameter_definitions()\n\n self.blockvisit(node.body, frame)\n self.return_buffer_contents(frame, force_unescaped=True)\n self.leave_frame(frame, with_python_scope=True)\n self.outdent()\n\n return frame, macro_ref\n\n def macro_def(self, macro_ref: MacroRef, frame: Frame) -> None:\n \"\"\"Dump the macro definition for the def created by macro_body.\"\"\"\n arg_tuple = \", \".join(repr(x.name) for x in macro_ref.node.args)\n name = getattr(macro_ref.node, \"name\", None)\n if len(macro_ref.node.args) == 1:\n arg_tuple += \",\"\n self.write(\n f\"Macro(environment, macro, {name!r}, ({arg_tuple}),\"\n f\" {macro_ref.accesses_kwargs!r}, {macro_ref.accesses_varargs!r},\"\n f\" {macro_ref.accesses_caller!r}, context.eval_ctx.autoescape)\"\n )\n\n def position(self, node: nodes.Node) -> str:\n \"\"\"Return a human readable position for the node.\"\"\"\n rv = f\"line {node.lineno}\"\n if self.name is not None:\n rv = f\"{rv} in {self.name!r}\"\n return rv\n\n def dump_local_context(self, frame: Frame) -> str:\n items_kv = \", \".join(\n f\"{name!r}: {target}\"\n for name, target in frame.symbols.dump_stores().items()\n )\n return f\"{{{items_kv}}}\"\n\n def write_commons(self) -> None:\n \"\"\"Writes a common preamble that is used by root and block functions.\n Primarily this sets up common local helpers and enforces a generator\n through a dead branch.\n \"\"\"\n self.writeline(\"resolve = context.resolve_or_missing\")\n self.writeline(\"undefined = environment.undefined\")\n # always use the standard Undefined class for the implicit else of\n # conditional expressions\n self.writeline(\"cond_expr_undefined = Undefined\")\n self.writeline(\"if 0: yield None\")\n\n def push_parameter_definitions(self, frame: Frame) -> None:\n \"\"\"Pushes all parameter targets from the given frame into a local\n stack that permits tracking of yet to be assigned parameters. In\n particular this enables the optimization from `visit_Name` to skip\n undefined expressions for parameters in macros as macros can reference\n otherwise unbound parameters.\n \"\"\"\n self._param_def_block.append(frame.symbols.dump_param_targets())\n\n def pop_parameter_definitions(self) -> None:\n \"\"\"Pops the current parameter definitions set.\"\"\"\n self._param_def_block.pop()\n\n def mark_parameter_stored(self, target: str) -> None:\n \"\"\"Marks a parameter in the current parameter definitions as stored.\n This will skip the enforced undefined checks.\n \"\"\"\n if self._param_def_block:\n self._param_def_block[-1].discard(target)\n\n def push_context_reference(self, target: str) -> None:\n self._context_reference_stack.append(target)\n\n def pop_context_reference(self) -> None:\n self._context_reference_stack.pop()\n\n def get_context_ref(self) -> str:\n return self._context_reference_stack[-1]\n\n def get_resolve_func(self) -> str:\n target = self._context_reference_stack[-1]\n if target == \"context\":\n return \"resolve\"\n return f\"{target}.resolve\"\n\n def derive_context(self, frame: Frame) -> str:\n return f\"{self.get_context_ref()}.derived({self.dump_local_context(frame)})\"\n\n def parameter_is_undeclared(self, target: str) -> bool:\n \"\"\"Checks if a given target is an undeclared parameter.\"\"\"\n if not self._param_def_block:\n return False\n return target in self._param_def_block[-1]\n\n def push_assign_tracking(self) -> None:\n \"\"\"Pushes a new layer for assignment tracking.\"\"\"\n self._assign_stack.append(set())\n\n def pop_assign_tracking(self, frame: Frame) -> None:\n \"\"\"Pops the topmost level for assignment tracking and updates the\n context variables if necessary.\n \"\"\"\n vars = self._assign_stack.pop()\n if (\n not frame.block_frame\n and not frame.loop_frame\n and not frame.toplevel\n or not vars\n ):\n return\n public_names = [x for x in vars if x[:1] != \"_\"]\n if len(vars) == 1:\n name = next(iter(vars))\n ref = frame.symbols.ref(name)\n if frame.loop_frame:\n self.writeline(f\"_loop_vars[{name!r}] = {ref}\")\n return\n if frame.block_frame:\n self.writeline(f\"_block_vars[{name!r}] = {ref}\")\n return\n self.writeline(f\"context.vars[{name!r}] = {ref}\")\n else:\n if frame.loop_frame:\n self.writeline(\"_loop_vars.update({\")\n elif frame.block_frame:\n self.writeline(\"_block_vars.update({\")\n else:\n self.writeline(\"context.vars.update({\")\n for idx, name in enumerate(vars):\n if idx:\n self.write(\", \")\n ref = frame.symbols.ref(name)\n self.write(f\"{name!r}: {ref}\")\n self.write(\"})\")\n if not frame.block_frame and not frame.loop_frame and public_names:\n if len(public_names) == 1:\n self.writeline(f\"context.exported_vars.add({public_names[0]!r})\")\n else:\n names_str = \", \".join(map(repr, public_names))\n self.writeline(f\"context.exported_vars.update(({names_str}))\")\n\n # -- Statement Visitors\n\n def visit_Template(\n self, node: nodes.Template, frame: t.Optional[Frame] = None\n ) -> None:\n assert frame is None, \"no root frame allowed\"\n eval_ctx = EvalContext(self.environment, self.name)\n\n from .runtime import exported, async_exported\n\n if self.environment.is_async:\n exported_names = sorted(exported + async_exported)\n else:\n exported_names = sorted(exported)\n\n self.writeline(\"from __future__ import generator_stop\") # Python < 3.7\n self.writeline(\"from spack.vendor.jinja2.runtime import \" + \", \".join(exported_names))\n\n # if we want a deferred initialization we cannot move the\n # environment into a local name\n envenv = \"\" if self.defer_init else \", environment=environment\"\n\n # do we have an extends tag at all? If not, we can save some\n # overhead by just not processing any inheritance code.\n have_extends = node.find(nodes.Extends) is not None\n\n # find all blocks\n for block in node.find_all(nodes.Block):\n if block.name in self.blocks:\n self.fail(f\"block {block.name!r} defined twice\", block.lineno)\n self.blocks[block.name] = block\n\n # find all imports and import them\n for import_ in node.find_all(nodes.ImportedName):\n if import_.importname not in self.import_aliases:\n imp = import_.importname\n self.import_aliases[imp] = alias = self.temporary_identifier()\n if \".\" in imp:\n module, obj = imp.rsplit(\".\", 1)\n self.writeline(f\"from {module} import {obj} as {alias}\")\n else:\n self.writeline(f\"import {imp} as {alias}\")\n\n # add the load name\n self.writeline(f\"name = {self.name!r}\")\n\n # generate the root render function.\n self.writeline(\n f\"{self.func('root')}(context, missing=missing{envenv}):\", extra=1\n )\n self.indent()\n self.write_commons()\n\n # process the root\n frame = Frame(eval_ctx)\n if \"self\" in find_undeclared(node.body, (\"self\",)):\n ref = frame.symbols.declare_parameter(\"self\")\n self.writeline(f\"{ref} = TemplateReference(context)\")\n frame.symbols.analyze_node(node)\n frame.toplevel = frame.rootlevel = True\n frame.require_output_check = have_extends and not self.has_known_extends\n if have_extends:\n self.writeline(\"parent_template = None\")\n self.enter_frame(frame)\n self.pull_dependencies(node.body)\n self.blockvisit(node.body, frame)\n self.leave_frame(frame, with_python_scope=True)\n self.outdent()\n\n # make sure that the parent root is called.\n if have_extends:\n if not self.has_known_extends:\n self.indent()\n self.writeline(\"if parent_template is not None:\")\n self.indent()\n if not self.environment.is_async:\n self.writeline(\"yield from parent_template.root_render_func(context)\")\n else:\n self.writeline(\n \"async for event in parent_template.root_render_func(context):\"\n )\n self.indent()\n self.writeline(\"yield event\")\n self.outdent()\n self.outdent(1 + (not self.has_known_extends))\n\n # at this point we now have the blocks collected and can visit them too.\n for name, block in self.blocks.items():\n self.writeline(\n f\"{self.func('block_' + name)}(context, missing=missing{envenv}):\",\n block,\n 1,\n )\n self.indent()\n self.write_commons()\n # It's important that we do not make this frame a child of the\n # toplevel template. This would cause a variety of\n # interesting issues with identifier tracking.\n block_frame = Frame(eval_ctx)\n block_frame.block_frame = True\n undeclared = find_undeclared(block.body, (\"self\", \"super\"))\n if \"self\" in undeclared:\n ref = block_frame.symbols.declare_parameter(\"self\")\n self.writeline(f\"{ref} = TemplateReference(context)\")\n if \"super\" in undeclared:\n ref = block_frame.symbols.declare_parameter(\"super\")\n self.writeline(f\"{ref} = context.super({name!r}, block_{name})\")\n block_frame.symbols.analyze_node(block)\n block_frame.block = name\n self.writeline(\"_block_vars = {}\")\n self.enter_frame(block_frame)\n self.pull_dependencies(block.body)\n self.blockvisit(block.body, block_frame)\n self.leave_frame(block_frame, with_python_scope=True)\n self.outdent()\n\n blocks_kv_str = \", \".join(f\"{x!r}: block_{x}\" for x in self.blocks)\n self.writeline(f\"blocks = {{{blocks_kv_str}}}\", extra=1)\n debug_kv_str = \"&\".join(f\"{k}={v}\" for k, v in self.debug_info)\n self.writeline(f\"debug_info = {debug_kv_str!r}\")\n\n def visit_Block(self, node: nodes.Block, frame: Frame) -> None:\n \"\"\"Call a block and register it for the template.\"\"\"\n level = 0\n if frame.toplevel:\n # if we know that we are a child template, there is no need to\n # check if we are one\n if self.has_known_extends:\n return\n if self.extends_so_far > 0:\n self.writeline(\"if parent_template is None:\")\n self.indent()\n level += 1\n\n if node.scoped:\n context = self.derive_context(frame)\n else:\n context = self.get_context_ref()\n\n if node.required:\n self.writeline(f\"if len(context.blocks[{node.name!r}]) <= 1:\", node)\n self.indent()\n self.writeline(\n f'raise TemplateRuntimeError(\"Required block {node.name!r} not found\")',\n node,\n )\n self.outdent()\n\n if not self.environment.is_async and frame.buffer is None:\n self.writeline(\n f\"yield from context.blocks[{node.name!r}][0]({context})\", node\n )\n else:\n self.writeline(\n f\"{self.choose_async()}for event in\"\n f\" context.blocks[{node.name!r}][0]({context}):\",\n node,\n )\n self.indent()\n self.simple_write(\"event\", frame)\n self.outdent()\n\n self.outdent(level)\n\n def visit_Extends(self, node: nodes.Extends, frame: Frame) -> None:\n \"\"\"Calls the extender.\"\"\"\n if not frame.toplevel:\n self.fail(\"cannot use extend from a non top-level scope\", node.lineno)\n\n # if the number of extends statements in general is zero so\n # far, we don't have to add a check if something extended\n # the template before this one.\n if self.extends_so_far > 0:\n\n # if we have a known extends we just add a template runtime\n # error into the generated code. We could catch that at compile\n # time too, but i welcome it not to confuse users by throwing the\n # same error at different times just \"because we can\".\n if not self.has_known_extends:\n self.writeline(\"if parent_template is not None:\")\n self.indent()\n self.writeline('raise TemplateRuntimeError(\"extended multiple times\")')\n\n # if we have a known extends already we don't need that code here\n # as we know that the template execution will end here.\n if self.has_known_extends:\n raise CompilerExit()\n else:\n self.outdent()\n\n self.writeline(\"parent_template = environment.get_template(\", node)\n self.visit(node.template, frame)\n self.write(f\", {self.name!r})\")\n self.writeline(\"for name, parent_block in parent_template.blocks.items():\")\n self.indent()\n self.writeline(\"context.blocks.setdefault(name, []).append(parent_block)\")\n self.outdent()\n\n # if this extends statement was in the root level we can take\n # advantage of that information and simplify the generated code\n # in the top level from this point onwards\n if frame.rootlevel:\n self.has_known_extends = True\n\n # and now we have one more\n self.extends_so_far += 1\n\n def visit_Include(self, node: nodes.Include, frame: Frame) -> None:\n \"\"\"Handles includes.\"\"\"\n if node.ignore_missing:\n self.writeline(\"try:\")\n self.indent()\n\n func_name = \"get_or_select_template\"\n if isinstance(node.template, nodes.Const):\n if isinstance(node.template.value, str):\n func_name = \"get_template\"\n elif isinstance(node.template.value, (tuple, list)):\n func_name = \"select_template\"\n elif isinstance(node.template, (nodes.Tuple, nodes.List)):\n func_name = \"select_template\"\n\n self.writeline(f\"template = environment.{func_name}(\", node)\n self.visit(node.template, frame)\n self.write(f\", {self.name!r})\")\n if node.ignore_missing:\n self.outdent()\n self.writeline(\"except TemplateNotFound:\")\n self.indent()\n self.writeline(\"pass\")\n self.outdent()\n self.writeline(\"else:\")\n self.indent()\n\n skip_event_yield = False\n if node.with_context:\n self.writeline(\n f\"{self.choose_async()}for event in template.root_render_func(\"\n \"template.new_context(context.get_all(), True,\"\n f\" {self.dump_local_context(frame)})):\"\n )\n elif self.environment.is_async:\n self.writeline(\n \"for event in (await template._get_default_module_async())\"\n \"._body_stream:\"\n )\n else:\n self.writeline(\"yield from template._get_default_module()._body_stream\")\n skip_event_yield = True\n\n if not skip_event_yield:\n self.indent()\n self.simple_write(\"event\", frame)\n self.outdent()\n\n if node.ignore_missing:\n self.outdent()\n\n def _import_common(\n self, node: t.Union[nodes.Import, nodes.FromImport], frame: Frame\n ) -> None:\n self.write(f\"{self.choose_async('await ')}environment.get_template(\")\n self.visit(node.template, frame)\n self.write(f\", {self.name!r}).\")\n\n if node.with_context:\n f_name = f\"make_module{self.choose_async('_async')}\"\n self.write(\n f\"{f_name}(context.get_all(), True, {self.dump_local_context(frame)})\"\n )\n else:\n self.write(f\"_get_default_module{self.choose_async('_async')}(context)\")\n\n def visit_Import(self, node: nodes.Import, frame: Frame) -> None:\n \"\"\"Visit regular imports.\"\"\"\n self.writeline(f\"{frame.symbols.ref(node.target)} = \", node)\n if frame.toplevel:\n self.write(f\"context.vars[{node.target!r}] = \")\n\n self._import_common(node, frame)\n\n if frame.toplevel and not node.target.startswith(\"_\"):\n self.writeline(f\"context.exported_vars.discard({node.target!r})\")\n\n def visit_FromImport(self, node: nodes.FromImport, frame: Frame) -> None:\n \"\"\"Visit named imports.\"\"\"\n self.newline(node)\n self.write(\"included_template = \")\n self._import_common(node, frame)\n var_names = []\n discarded_names = []\n for name in node.names:\n if isinstance(name, tuple):\n name, alias = name\n else:\n alias = name\n self.writeline(\n f\"{frame.symbols.ref(alias)} =\"\n f\" getattr(included_template, {name!r}, missing)\"\n )\n self.writeline(f\"if {frame.symbols.ref(alias)} is missing:\")\n self.indent()\n message = (\n \"the template {included_template.__name__!r}\"\n f\" (imported on {self.position(node)})\"\n f\" does not export the requested name {name!r}\"\n )\n self.writeline(\n f\"{frame.symbols.ref(alias)} = undefined(f{message!r}, name={name!r})\"\n )\n self.outdent()\n if frame.toplevel:\n var_names.append(alias)\n if not alias.startswith(\"_\"):\n discarded_names.append(alias)\n\n if var_names:\n if len(var_names) == 1:\n name = var_names[0]\n self.writeline(f\"context.vars[{name!r}] = {frame.symbols.ref(name)}\")\n else:\n names_kv = \", \".join(\n f\"{name!r}: {frame.symbols.ref(name)}\" for name in var_names\n )\n self.writeline(f\"context.vars.update({{{names_kv}}})\")\n if discarded_names:\n if len(discarded_names) == 1:\n self.writeline(f\"context.exported_vars.discard({discarded_names[0]!r})\")\n else:\n names_str = \", \".join(map(repr, discarded_names))\n self.writeline(\n f\"context.exported_vars.difference_update(({names_str}))\"\n )\n\n def visit_For(self, node: nodes.For, frame: Frame) -> None:\n loop_frame = frame.inner()\n loop_frame.loop_frame = True\n test_frame = frame.inner()\n else_frame = frame.inner()\n\n # try to figure out if we have an extended loop. An extended loop\n # is necessary if the loop is in recursive mode if the special loop\n # variable is accessed in the body if the body is a scoped block.\n extended_loop = (\n node.recursive\n or \"loop\"\n in find_undeclared(node.iter_child_nodes(only=(\"body\",)), (\"loop\",))\n or any(block.scoped for block in node.find_all(nodes.Block))\n )\n\n loop_ref = None\n if extended_loop:\n loop_ref = loop_frame.symbols.declare_parameter(\"loop\")\n\n loop_frame.symbols.analyze_node(node, for_branch=\"body\")\n if node.else_:\n else_frame.symbols.analyze_node(node, for_branch=\"else\")\n\n if node.test:\n loop_filter_func = self.temporary_identifier()\n test_frame.symbols.analyze_node(node, for_branch=\"test\")\n self.writeline(f\"{self.func(loop_filter_func)}(fiter):\", node.test)\n self.indent()\n self.enter_frame(test_frame)\n self.writeline(self.choose_async(\"async for \", \"for \"))\n self.visit(node.target, loop_frame)\n self.write(\" in \")\n self.write(self.choose_async(\"auto_aiter(fiter)\", \"fiter\"))\n self.write(\":\")\n self.indent()\n self.writeline(\"if \", node.test)\n self.visit(node.test, test_frame)\n self.write(\":\")\n self.indent()\n self.writeline(\"yield \")\n self.visit(node.target, loop_frame)\n self.outdent(3)\n self.leave_frame(test_frame, with_python_scope=True)\n\n # if we don't have an recursive loop we have to find the shadowed\n # variables at that point. Because loops can be nested but the loop\n # variable is a special one we have to enforce aliasing for it.\n if node.recursive:\n self.writeline(\n f\"{self.func('loop')}(reciter, loop_render_func, depth=0):\", node\n )\n self.indent()\n self.buffer(loop_frame)\n\n # Use the same buffer for the else frame\n else_frame.buffer = loop_frame.buffer\n\n # make sure the loop variable is a special one and raise a template\n # assertion error if a loop tries to write to loop\n if extended_loop:\n self.writeline(f\"{loop_ref} = missing\")\n\n for name in node.find_all(nodes.Name):\n if name.ctx == \"store\" and name.name == \"loop\":\n self.fail(\n \"Can't assign to special loop variable in for-loop target\",\n name.lineno,\n )\n\n if node.else_:\n iteration_indicator = self.temporary_identifier()\n self.writeline(f\"{iteration_indicator} = 1\")\n\n self.writeline(self.choose_async(\"async for \", \"for \"), node)\n self.visit(node.target, loop_frame)\n if extended_loop:\n self.write(f\", {loop_ref} in {self.choose_async('Async')}LoopContext(\")\n else:\n self.write(\" in \")\n\n if node.test:\n self.write(f\"{loop_filter_func}(\")\n if node.recursive:\n self.write(\"reciter\")\n else:\n if self.environment.is_async and not extended_loop:\n self.write(\"auto_aiter(\")\n self.visit(node.iter, frame)\n if self.environment.is_async and not extended_loop:\n self.write(\")\")\n if node.test:\n self.write(\")\")\n\n if node.recursive:\n self.write(\", undefined, loop_render_func, depth):\")\n else:\n self.write(\", undefined):\" if extended_loop else \":\")\n\n self.indent()\n self.enter_frame(loop_frame)\n\n self.writeline(\"_loop_vars = {}\")\n self.blockvisit(node.body, loop_frame)\n if node.else_:\n self.writeline(f\"{iteration_indicator} = 0\")\n self.outdent()\n self.leave_frame(\n loop_frame, with_python_scope=node.recursive and not node.else_\n )\n\n if node.else_:\n self.writeline(f\"if {iteration_indicator}:\")\n self.indent()\n self.enter_frame(else_frame)\n self.blockvisit(node.else_, else_frame)\n self.leave_frame(else_frame)\n self.outdent()\n\n # if the node was recursive we have to return the buffer contents\n # and start the iteration code\n if node.recursive:\n self.return_buffer_contents(loop_frame)\n self.outdent()\n self.start_write(frame, node)\n self.write(f\"{self.choose_async('await ')}loop(\")\n if self.environment.is_async:\n self.write(\"auto_aiter(\")\n self.visit(node.iter, frame)\n if self.environment.is_async:\n self.write(\")\")\n self.write(\", loop)\")\n self.end_write(frame)\n\n # at the end of the iteration, clear any assignments made in the\n # loop from the top level\n if self._assign_stack:\n self._assign_stack[-1].difference_update(loop_frame.symbols.stores)\n\n def visit_If(self, node: nodes.If, frame: Frame) -> None:\n if_frame = frame.soft()\n self.writeline(\"if \", node)\n self.visit(node.test, if_frame)\n self.write(\":\")\n self.indent()\n self.blockvisit(node.body, if_frame)\n self.outdent()\n for elif_ in node.elif_:\n self.writeline(\"elif \", elif_)\n self.visit(elif_.test, if_frame)\n self.write(\":\")\n self.indent()\n self.blockvisit(elif_.body, if_frame)\n self.outdent()\n if node.else_:\n self.writeline(\"else:\")\n self.indent()\n self.blockvisit(node.else_, if_frame)\n self.outdent()\n\n def visit_Macro(self, node: nodes.Macro, frame: Frame) -> None:\n macro_frame, macro_ref = self.macro_body(node, frame)\n self.newline()\n if frame.toplevel:\n if not node.name.startswith(\"_\"):\n self.write(f\"context.exported_vars.add({node.name!r})\")\n self.writeline(f\"context.vars[{node.name!r}] = \")\n self.write(f\"{frame.symbols.ref(node.name)} = \")\n self.macro_def(macro_ref, macro_frame)\n\n def visit_CallBlock(self, node: nodes.CallBlock, frame: Frame) -> None:\n call_frame, macro_ref = self.macro_body(node, frame)\n self.writeline(\"caller = \")\n self.macro_def(macro_ref, call_frame)\n self.start_write(frame, node)\n self.visit_Call(node.call, frame, forward_caller=True)\n self.end_write(frame)\n\n def visit_FilterBlock(self, node: nodes.FilterBlock, frame: Frame) -> None:\n filter_frame = frame.inner()\n filter_frame.symbols.analyze_node(node)\n self.enter_frame(filter_frame)\n self.buffer(filter_frame)\n self.blockvisit(node.body, filter_frame)\n self.start_write(frame, node)\n self.visit_Filter(node.filter, filter_frame)\n self.end_write(frame)\n self.leave_frame(filter_frame)\n\n def visit_With(self, node: nodes.With, frame: Frame) -> None:\n with_frame = frame.inner()\n with_frame.symbols.analyze_node(node)\n self.enter_frame(with_frame)\n for target, expr in zip(node.targets, node.values):\n self.newline()\n self.visit(target, with_frame)\n self.write(\" = \")\n self.visit(expr, frame)\n self.blockvisit(node.body, with_frame)\n self.leave_frame(with_frame)\n\n def visit_ExprStmt(self, node: nodes.ExprStmt, frame: Frame) -> None:\n self.newline(node)\n self.visit(node.node, frame)\n\n class _FinalizeInfo(t.NamedTuple):\n const: t.Optional[t.Callable[..., str]]\n src: t.Optional[str]\n\n @staticmethod\n def _default_finalize(value: t.Any) -> t.Any:\n \"\"\"The default finalize function if the environment isn't\n configured with one. Or, if the environment has one, this is\n called on that function's output for constants.\n \"\"\"\n return str(value)\n\n _finalize: t.Optional[_FinalizeInfo] = None\n\n def _make_finalize(self) -> _FinalizeInfo:\n \"\"\"Build the finalize function to be used on constants and at\n runtime. Cached so it's only created once for all output nodes.\n\n Returns a ``namedtuple`` with the following attributes:\n\n ``const``\n A function to finalize constant data at compile time.\n\n ``src``\n Source code to output around nodes to be evaluated at\n runtime.\n \"\"\"\n if self._finalize is not None:\n return self._finalize\n\n finalize: t.Optional[t.Callable[..., t.Any]]\n finalize = default = self._default_finalize\n src = None\n\n if self.environment.finalize:\n src = \"environment.finalize(\"\n env_finalize = self.environment.finalize\n pass_arg = {\n _PassArg.context: \"context\",\n _PassArg.eval_context: \"context.eval_ctx\",\n _PassArg.environment: \"environment\",\n }.get(\n _PassArg.from_obj(env_finalize) # type: ignore\n )\n finalize = None\n\n if pass_arg is None:\n\n def finalize(value: t.Any) -> t.Any:\n return default(env_finalize(value))\n\n else:\n src = f\"{src}{pass_arg}, \"\n\n if pass_arg == \"environment\":\n\n def finalize(value: t.Any) -> t.Any:\n return default(env_finalize(self.environment, value))\n\n self._finalize = self._FinalizeInfo(finalize, src)\n return self._finalize\n\n def _output_const_repr(self, group: t.Iterable[t.Any]) -> str:\n \"\"\"Given a group of constant values converted from ``Output``\n child nodes, produce a string to write to the template module\n source.\n \"\"\"\n return repr(concat(group))\n\n def _output_child_to_const(\n self, node: nodes.Expr, frame: Frame, finalize: _FinalizeInfo\n ) -> str:\n \"\"\"Try to optimize a child of an ``Output`` node by trying to\n convert it to constant, finalized data at compile time.\n\n If :exc:`Impossible` is raised, the node is not constant and\n will be evaluated at runtime. Any other exception will also be\n evaluated at runtime for easier debugging.\n \"\"\"\n const = node.as_const(frame.eval_ctx)\n\n if frame.eval_ctx.autoescape:\n const = escape(const)\n\n # Template data doesn't go through finalize.\n if isinstance(node, nodes.TemplateData):\n return str(const)\n\n return finalize.const(const) # type: ignore\n\n def _output_child_pre(\n self, node: nodes.Expr, frame: Frame, finalize: _FinalizeInfo\n ) -> None:\n \"\"\"Output extra source code before visiting a child of an\n ``Output`` node.\n \"\"\"\n if frame.eval_ctx.volatile:\n self.write(\"(escape if context.eval_ctx.autoescape else str)(\")\n elif frame.eval_ctx.autoescape:\n self.write(\"escape(\")\n else:\n self.write(\"str(\")\n\n if finalize.src is not None:\n self.write(finalize.src)\n\n def _output_child_post(\n self, node: nodes.Expr, frame: Frame, finalize: _FinalizeInfo\n ) -> None:\n \"\"\"Output extra source code after visiting a child of an\n ``Output`` node.\n \"\"\"\n self.write(\")\")\n\n if finalize.src is not None:\n self.write(\")\")\n\n def visit_Output(self, node: nodes.Output, frame: Frame) -> None:\n # If an extends is active, don't render outside a block.\n if frame.require_output_check:\n # A top-level extends is known to exist at compile time.\n if self.has_known_extends:\n return\n\n self.writeline(\"if parent_template is None:\")\n self.indent()\n\n finalize = self._make_finalize()\n body: t.List[t.Union[t.List[t.Any], nodes.Expr]] = []\n\n # Evaluate constants at compile time if possible. Each item in\n # body will be either a list of static data or a node to be\n # evaluated at runtime.\n for child in node.nodes:\n try:\n if not (\n # If the finalize function requires runtime context,\n # constants can't be evaluated at compile time.\n finalize.const\n # Unless it's basic template data that won't be\n # finalized anyway.\n or isinstance(child, nodes.TemplateData)\n ):\n raise nodes.Impossible()\n\n const = self._output_child_to_const(child, frame, finalize)\n except (nodes.Impossible, Exception):\n # The node was not constant and needs to be evaluated at\n # runtime. Or another error was raised, which is easier\n # to debug at runtime.\n body.append(child)\n continue\n\n if body and isinstance(body[-1], list):\n body[-1].append(const)\n else:\n body.append([const])\n\n if frame.buffer is not None:\n if len(body) == 1:\n self.writeline(f\"{frame.buffer}.append(\")\n else:\n self.writeline(f\"{frame.buffer}.extend((\")\n\n self.indent()\n\n for item in body:\n if isinstance(item, list):\n # A group of constant data to join and output.\n val = self._output_const_repr(item)\n\n if frame.buffer is None:\n self.writeline(\"yield \" + val)\n else:\n self.writeline(val + \",\")\n else:\n if frame.buffer is None:\n self.writeline(\"yield \", item)\n else:\n self.newline(item)\n\n # A node to be evaluated at runtime.\n self._output_child_pre(item, frame, finalize)\n self.visit(item, frame)\n self._output_child_post(item, frame, finalize)\n\n if frame.buffer is not None:\n self.write(\",\")\n\n if frame.buffer is not None:\n self.outdent()\n self.writeline(\")\" if len(body) == 1 else \"))\")\n\n if frame.require_output_check:\n self.outdent()\n\n def visit_Assign(self, node: nodes.Assign, frame: Frame) -> None:\n self.push_assign_tracking()\n self.newline(node)\n self.visit(node.target, frame)\n self.write(\" = \")\n self.visit(node.node, frame)\n self.pop_assign_tracking(frame)\n\n def visit_AssignBlock(self, node: nodes.AssignBlock, frame: Frame) -> None:\n self.push_assign_tracking()\n block_frame = frame.inner()\n # This is a special case. Since a set block always captures we\n # will disable output checks. This way one can use set blocks\n # toplevel even in extended templates.\n block_frame.require_output_check = False\n block_frame.symbols.analyze_node(node)\n self.enter_frame(block_frame)\n self.buffer(block_frame)\n self.blockvisit(node.body, block_frame)\n self.newline(node)\n self.visit(node.target, frame)\n self.write(\" = (Markup if context.eval_ctx.autoescape else identity)(\")\n if node.filter is not None:\n self.visit_Filter(node.filter, block_frame)\n else:\n self.write(f\"concat({block_frame.buffer})\")\n self.write(\")\")\n self.pop_assign_tracking(frame)\n self.leave_frame(block_frame)\n\n # -- Expression Visitors\n\n def visit_Name(self, node: nodes.Name, frame: Frame) -> None:\n if node.ctx == \"store\" and (\n frame.toplevel or frame.loop_frame or frame.block_frame\n ):\n if self._assign_stack:\n self._assign_stack[-1].add(node.name)\n ref = frame.symbols.ref(node.name)\n\n # If we are looking up a variable we might have to deal with the\n # case where it's undefined. We can skip that case if the load\n # instruction indicates a parameter which are always defined.\n if node.ctx == \"load\":\n load = frame.symbols.find_load(ref)\n if not (\n load is not None\n and load[0] == VAR_LOAD_PARAMETER\n and not self.parameter_is_undeclared(ref)\n ):\n self.write(\n f\"(undefined(name={node.name!r}) if {ref} is missing else {ref})\"\n )\n return\n\n self.write(ref)\n\n def visit_NSRef(self, node: nodes.NSRef, frame: Frame) -> None:\n # NSRefs can only be used to store values; since they use the normal\n # `foo.bar` notation they will be parsed as a normal attribute access\n # when used anywhere but in a `set` context\n ref = frame.symbols.ref(node.name)\n self.writeline(f\"if not isinstance({ref}, Namespace):\")\n self.indent()\n self.writeline(\n \"raise TemplateRuntimeError\"\n '(\"cannot assign attribute on non-namespace object\")'\n )\n self.outdent()\n self.writeline(f\"{ref}[{node.attr!r}]\")\n\n def visit_Const(self, node: nodes.Const, frame: Frame) -> None:\n val = node.as_const(frame.eval_ctx)\n if isinstance(val, float):\n self.write(str(val))\n else:\n self.write(repr(val))\n\n def visit_TemplateData(self, node: nodes.TemplateData, frame: Frame) -> None:\n try:\n self.write(repr(node.as_const(frame.eval_ctx)))\n except nodes.Impossible:\n self.write(\n f\"(Markup if context.eval_ctx.autoescape else identity)({node.data!r})\"\n )\n\n def visit_Tuple(self, node: nodes.Tuple, frame: Frame) -> None:\n self.write(\"(\")\n idx = -1\n for idx, item in enumerate(node.items):\n if idx:\n self.write(\", \")\n self.visit(item, frame)\n self.write(\",)\" if idx == 0 else \")\")\n\n def visit_List(self, node: nodes.List, frame: Frame) -> None:\n self.write(\"[\")\n for idx, item in enumerate(node.items):\n if idx:\n self.write(\", \")\n self.visit(item, frame)\n self.write(\"]\")\n\n def visit_Dict(self, node: nodes.Dict, frame: Frame) -> None:\n self.write(\"{\")\n for idx, item in enumerate(node.items):\n if idx:\n self.write(\", \")\n self.visit(item.key, frame)\n self.write(\": \")\n self.visit(item.value, frame)\n self.write(\"}\")\n\n visit_Add = _make_binop(\"+\")\n visit_Sub = _make_binop(\"-\")\n visit_Mul = _make_binop(\"*\")\n visit_Div = _make_binop(\"/\")\n visit_FloorDiv = _make_binop(\"//\")\n visit_Pow = _make_binop(\"**\")\n visit_Mod = _make_binop(\"%\")\n visit_And = _make_binop(\"and\")\n visit_Or = _make_binop(\"or\")\n visit_Pos = _make_unop(\"+\")\n visit_Neg = _make_unop(\"-\")\n visit_Not = _make_unop(\"not \")\n\n @optimizeconst\n def visit_Concat(self, node: nodes.Concat, frame: Frame) -> None:\n if frame.eval_ctx.volatile:\n func_name = \"(markup_join if context.eval_ctx.volatile else str_join)\"\n elif frame.eval_ctx.autoescape:\n func_name = \"markup_join\"\n else:\n func_name = \"str_join\"\n self.write(f\"{func_name}((\")\n for arg in node.nodes:\n self.visit(arg, frame)\n self.write(\", \")\n self.write(\"))\")\n\n @optimizeconst\n def visit_Compare(self, node: nodes.Compare, frame: Frame) -> None:\n self.write(\"(\")\n self.visit(node.expr, frame)\n for op in node.ops:\n self.visit(op, frame)\n self.write(\")\")\n\n def visit_Operand(self, node: nodes.Operand, frame: Frame) -> None:\n self.write(f\" {operators[node.op]} \")\n self.visit(node.expr, frame)\n\n @optimizeconst\n def visit_Getattr(self, node: nodes.Getattr, frame: Frame) -> None:\n if self.environment.is_async:\n self.write(\"(await auto_await(\")\n\n self.write(\"environment.getattr(\")\n self.visit(node.node, frame)\n self.write(f\", {node.attr!r})\")\n\n if self.environment.is_async:\n self.write(\"))\")\n\n @optimizeconst\n def visit_Getitem(self, node: nodes.Getitem, frame: Frame) -> None:\n # slices bypass the environment getitem method.\n if isinstance(node.arg, nodes.Slice):\n self.visit(node.node, frame)\n self.write(\"[\")\n self.visit(node.arg, frame)\n self.write(\"]\")\n else:\n if self.environment.is_async:\n self.write(\"(await auto_await(\")\n\n self.write(\"environment.getitem(\")\n self.visit(node.node, frame)\n self.write(\", \")\n self.visit(node.arg, frame)\n self.write(\")\")\n\n if self.environment.is_async:\n self.write(\"))\")\n\n def visit_Slice(self, node: nodes.Slice, frame: Frame) -> None:\n if node.start is not None:\n self.visit(node.start, frame)\n self.write(\":\")\n if node.stop is not None:\n self.visit(node.stop, frame)\n if node.step is not None:\n self.write(\":\")\n self.visit(node.step, frame)\n\n @contextmanager\n def _filter_test_common(\n self, node: t.Union[nodes.Filter, nodes.Test], frame: Frame, is_filter: bool\n ) -> t.Iterator[None]:\n if self.environment.is_async:\n self.write(\"await auto_await(\")\n\n if is_filter:\n self.write(f\"{self.filters[node.name]}(\")\n func = self.environment.filters.get(node.name)\n else:\n self.write(f\"{self.tests[node.name]}(\")\n func = self.environment.tests.get(node.name)\n\n # When inside an If or CondExpr frame, allow the filter to be\n # undefined at compile time and only raise an error if it's\n # actually called at runtime. See pull_dependencies.\n if func is None and not frame.soft_frame:\n type_name = \"filter\" if is_filter else \"test\"\n self.fail(f\"No {type_name} named {node.name!r}.\", node.lineno)\n\n pass_arg = {\n _PassArg.context: \"context\",\n _PassArg.eval_context: \"context.eval_ctx\",\n _PassArg.environment: \"environment\",\n }.get(\n _PassArg.from_obj(func) # type: ignore\n )\n\n if pass_arg is not None:\n self.write(f\"{pass_arg}, \")\n\n # Back to the visitor function to handle visiting the target of\n # the filter or test.\n yield\n\n self.signature(node, frame)\n self.write(\")\")\n\n if self.environment.is_async:\n self.write(\")\")\n\n @optimizeconst\n def visit_Filter(self, node: nodes.Filter, frame: Frame) -> None:\n with self._filter_test_common(node, frame, True):\n # if the filter node is None we are inside a filter block\n # and want to write to the current buffer\n if node.node is not None:\n self.visit(node.node, frame)\n elif frame.eval_ctx.volatile:\n self.write(\n f\"(Markup(concat({frame.buffer}))\"\n f\" if context.eval_ctx.autoescape else concat({frame.buffer}))\"\n )\n elif frame.eval_ctx.autoescape:\n self.write(f\"Markup(concat({frame.buffer}))\")\n else:\n self.write(f\"concat({frame.buffer})\")\n\n @optimizeconst\n def visit_Test(self, node: nodes.Test, frame: Frame) -> None:\n with self._filter_test_common(node, frame, False):\n self.visit(node.node, frame)\n\n @optimizeconst\n def visit_CondExpr(self, node: nodes.CondExpr, frame: Frame) -> None:\n frame = frame.soft()\n\n def write_expr2() -> None:\n if node.expr2 is not None:\n self.visit(node.expr2, frame)\n return\n\n self.write(\n f'cond_expr_undefined(\"the inline if-expression on'\n f\" {self.position(node)} evaluated to false and no else\"\n f' section was defined.\")'\n )\n\n self.write(\"(\")\n self.visit(node.expr1, frame)\n self.write(\" if \")\n self.visit(node.test, frame)\n self.write(\" else \")\n write_expr2()\n self.write(\")\")\n\n @optimizeconst\n def visit_Call(\n self, node: nodes.Call, frame: Frame, forward_caller: bool = False\n ) -> None:\n if self.environment.is_async:\n self.write(\"await auto_await(\")\n if self.environment.sandboxed:\n self.write(\"environment.call(context, \")\n else:\n self.write(\"context.call(\")\n self.visit(node.node, frame)\n extra_kwargs = {\"caller\": \"caller\"} if forward_caller else None\n loop_kwargs = {\"_loop_vars\": \"_loop_vars\"} if frame.loop_frame else {}\n block_kwargs = {\"_block_vars\": \"_block_vars\"} if frame.block_frame else {}\n if extra_kwargs:\n extra_kwargs.update(loop_kwargs, **block_kwargs)\n elif loop_kwargs or block_kwargs:\n extra_kwargs = dict(loop_kwargs, **block_kwargs)\n self.signature(node, frame, extra_kwargs)\n self.write(\")\")\n if self.environment.is_async:\n self.write(\")\")\n\n def visit_Keyword(self, node: nodes.Keyword, frame: Frame) -> None:\n self.write(node.key + \"=\")\n self.visit(node.value, frame)\n\n # -- Unused nodes for extensions\n\n def visit_MarkSafe(self, node: nodes.MarkSafe, frame: Frame) -> None:\n self.write(\"Markup(\")\n self.visit(node.expr, frame)\n self.write(\")\")\n\n def visit_MarkSafeIfAutoescape(\n self, node: nodes.MarkSafeIfAutoescape, frame: Frame\n ) -> None:\n self.write(\"(Markup if context.eval_ctx.autoescape else identity)(\")\n self.visit(node.expr, frame)\n self.write(\")\")\n\n def visit_EnvironmentAttribute(\n self, node: nodes.EnvironmentAttribute, frame: Frame\n ) -> None:\n self.write(\"environment.\" + node.name)\n\n def visit_ExtensionAttribute(\n self, node: nodes.ExtensionAttribute, frame: Frame\n ) -> None:\n self.write(f\"environment.extensions[{node.identifier!r}].{node.name}\")\n\n def visit_ImportedName(self, node: nodes.ImportedName, frame: Frame) -> None:\n self.write(self.import_aliases[node.importname])\n\n def visit_InternalName(self, node: nodes.InternalName, frame: Frame) -> None:\n self.write(node.name)\n\n def visit_ContextReference(\n self, node: nodes.ContextReference, frame: Frame\n ) -> None:\n self.write(\"context\")\n\n def visit_DerivedContextReference(\n self, node: nodes.DerivedContextReference, frame: Frame\n ) -> None:\n self.write(self.derive_context(frame))\n\n def visit_Continue(self, node: nodes.Continue, frame: Frame) -> None:\n self.writeline(\"continue\", node)\n\n def visit_Break(self, node: nodes.Break, frame: Frame) -> None:\n self.writeline(\"break\", node)\n\n def visit_Scope(self, node: nodes.Scope, frame: Frame) -> None:\n scope_frame = frame.inner()\n scope_frame.symbols.analyze_node(node)\n self.enter_frame(scope_frame)\n self.blockvisit(node.body, scope_frame)\n self.leave_frame(scope_frame)\n\n def visit_OverlayScope(self, node: nodes.OverlayScope, frame: Frame) -> None:\n ctx = self.temporary_identifier()\n self.writeline(f\"{ctx} = {self.derive_context(frame)}\")\n self.writeline(f\"{ctx}.vars = \")\n self.visit(node.context, frame)\n self.push_context_reference(ctx)\n\n scope_frame = frame.inner(isolated=True)\n scope_frame.symbols.analyze_node(node)\n self.enter_frame(scope_frame)\n self.blockvisit(node.body, scope_frame)\n self.leave_frame(scope_frame)\n self.pop_context_reference()\n\n def visit_EvalContextModifier(\n self, node: nodes.EvalContextModifier, frame: Frame\n ) -> None:\n for keyword in node.options:\n self.writeline(f\"context.eval_ctx.{keyword.key} = \")\n self.visit(keyword.value, frame)\n try:\n val = keyword.value.as_const(frame.eval_ctx)\n except nodes.Impossible:\n frame.eval_ctx.volatile = True\n else:\n setattr(frame.eval_ctx, keyword.key, val)\n\n def visit_ScopedEvalContextModifier(\n self, node: nodes.ScopedEvalContextModifier, frame: Frame\n ) -> None:\n old_ctx_name = self.temporary_identifier()\n saved_ctx = frame.eval_ctx.save()\n self.writeline(f\"{old_ctx_name} = context.eval_ctx.save()\")\n self.visit_EvalContextModifier(node, frame)\n for child in node.body:\n self.visit(child, frame)\n frame.eval_ctx.revert(saved_ctx)\n self.writeline(f\"context.eval_ctx.revert({old_ctx_name})\")\n" }, { "path": "lib/spack/spack/vendor/jinja2/constants.py", "content": "#: list of lorem ipsum words used by the lipsum() helper function\nLOREM_IPSUM_WORDS = \"\"\"\\\na ac accumsan ad adipiscing aenean aliquam aliquet amet ante aptent arcu at\nauctor augue bibendum blandit class commodo condimentum congue consectetuer\nconsequat conubia convallis cras cubilia cum curabitur curae cursus dapibus\ndiam dictum dictumst dignissim dis dolor donec dui duis egestas eget eleifend\nelementum elit enim erat eros est et etiam eu euismod facilisi facilisis fames\nfaucibus felis fermentum feugiat fringilla fusce gravida habitant habitasse hac\nhendrerit hymenaeos iaculis id imperdiet in inceptos integer interdum ipsum\njusto lacinia lacus laoreet lectus leo libero ligula litora lobortis lorem\nluctus maecenas magna magnis malesuada massa mattis mauris metus mi molestie\nmollis montes morbi mus nam nascetur natoque nec neque netus nibh nisi nisl non\nnonummy nostra nulla nullam nunc odio orci ornare parturient pede pellentesque\npenatibus per pharetra phasellus placerat platea porta porttitor posuere\npotenti praesent pretium primis proin pulvinar purus quam quis quisque rhoncus\nridiculus risus rutrum sagittis sapien scelerisque sed sem semper senectus sit\nsociis sociosqu sodales sollicitudin suscipit suspendisse taciti tellus tempor\ntempus tincidunt torquent tortor tristique turpis ullamcorper ultrices\nultricies urna ut varius vehicula vel velit venenatis vestibulum vitae vivamus\nviverra volutpat vulputate\"\"\"\n" }, { "path": "lib/spack/spack/vendor/jinja2/debug.py", "content": "import platform\nimport sys\nimport typing as t\nfrom types import CodeType\nfrom types import TracebackType\n\nfrom .exceptions import TemplateSyntaxError\nfrom .utils import internal_code\nfrom .utils import missing\n\nif t.TYPE_CHECKING:\n from .runtime import Context\n\n\ndef rewrite_traceback_stack(source: t.Optional[str] = None) -> BaseException:\n \"\"\"Rewrite the current exception to replace any tracebacks from\n within compiled template code with tracebacks that look like they\n came from the template source.\n\n This must be called within an ``except`` block.\n\n :param source: For ``TemplateSyntaxError``, the original source if\n known.\n :return: The original exception with the rewritten traceback.\n \"\"\"\n _, exc_value, tb = sys.exc_info()\n exc_value = t.cast(BaseException, exc_value)\n tb = t.cast(TracebackType, tb)\n\n if isinstance(exc_value, TemplateSyntaxError) and not exc_value.translated:\n exc_value.translated = True\n exc_value.source = source\n # Remove the old traceback, otherwise the frames from the\n # compiler still show up.\n exc_value.with_traceback(None)\n # Outside of runtime, so the frame isn't executing template\n # code, but it still needs to point at the template.\n tb = fake_traceback(\n exc_value, None, exc_value.filename or \"\", exc_value.lineno\n )\n else:\n # Skip the frame for the render function.\n tb = tb.tb_next\n\n stack = []\n\n # Build the stack of traceback object, replacing any in template\n # code with the source file and line information.\n while tb is not None:\n # Skip frames decorated with @internalcode. These are internal\n # calls that aren't useful in template debugging output.\n if tb.tb_frame.f_code in internal_code:\n tb = tb.tb_next\n continue\n\n template = tb.tb_frame.f_globals.get(\"__jinja_template__\")\n\n if template is not None:\n lineno = template.get_corresponding_lineno(tb.tb_lineno)\n fake_tb = fake_traceback(exc_value, tb, template.filename, lineno)\n stack.append(fake_tb)\n else:\n stack.append(tb)\n\n tb = tb.tb_next\n\n tb_next = None\n\n # Assign tb_next in reverse to avoid circular references.\n for tb in reversed(stack):\n tb_next = tb_set_next(tb, tb_next)\n\n return exc_value.with_traceback(tb_next)\n\n\ndef fake_traceback( # type: ignore\n exc_value: BaseException, tb: t.Optional[TracebackType], filename: str, lineno: int\n) -> TracebackType:\n \"\"\"Produce a new traceback object that looks like it came from the\n template source instead of the compiled code. The filename, line\n number, and location name will point to the template, and the local\n variables will be the current template context.\n\n :param exc_value: The original exception to be re-raised to create\n the new traceback.\n :param tb: The original traceback to get the local variables and\n code info from.\n :param filename: The template filename.\n :param lineno: The line number in the template source.\n \"\"\"\n if tb is not None:\n # Replace the real locals with the context that would be\n # available at that point in the template.\n locals = get_template_locals(tb.tb_frame.f_locals)\n locals.pop(\"__jinja_exception__\", None)\n else:\n locals = {}\n\n globals = {\n \"__name__\": filename,\n \"__file__\": filename,\n \"__jinja_exception__\": exc_value,\n }\n # Raise an exception at the correct line number.\n code: CodeType = compile(\n \"\\n\" * (lineno - 1) + \"raise __jinja_exception__\", filename, \"exec\"\n )\n\n # Build a new code object that points to the template file and\n # replaces the location with a block name.\n location = \"template\"\n\n if tb is not None:\n function = tb.tb_frame.f_code.co_name\n\n if function == \"root\":\n location = \"top-level template code\"\n elif function.startswith(\"block_\"):\n location = f\"block {function[6:]!r}\"\n\n if sys.version_info >= (3, 8):\n code = code.replace(co_name=location)\n else:\n code = CodeType(\n code.co_argcount,\n code.co_kwonlyargcount,\n code.co_nlocals,\n code.co_stacksize,\n code.co_flags,\n code.co_code,\n code.co_consts,\n code.co_names,\n code.co_varnames,\n code.co_filename,\n location,\n code.co_firstlineno,\n code.co_lnotab,\n code.co_freevars,\n code.co_cellvars,\n )\n\n # Execute the new code, which is guaranteed to raise, and return\n # the new traceback without this frame.\n try:\n exec(code, globals, locals)\n except BaseException:\n return sys.exc_info()[2].tb_next # type: ignore\n\n\ndef get_template_locals(real_locals: t.Mapping[str, t.Any]) -> t.Dict[str, t.Any]:\n \"\"\"Based on the runtime locals, get the context that would be\n available at that point in the template.\n \"\"\"\n # Start with the current template context.\n ctx: \"t.Optional[Context]\" = real_locals.get(\"context\")\n\n if ctx is not None:\n data: t.Dict[str, t.Any] = ctx.get_all().copy()\n else:\n data = {}\n\n # Might be in a derived context that only sets local variables\n # rather than pushing a context. Local variables follow the scheme\n # l_depth_name. Find the highest-depth local that has a value for\n # each name.\n local_overrides: t.Dict[str, t.Tuple[int, t.Any]] = {}\n\n for name, value in real_locals.items():\n if not name.startswith(\"l_\") or value is missing:\n # Not a template variable, or no longer relevant.\n continue\n\n try:\n _, depth_str, name = name.split(\"_\", 2)\n depth = int(depth_str)\n except ValueError:\n continue\n\n cur_depth = local_overrides.get(name, (-1,))[0]\n\n if cur_depth < depth:\n local_overrides[name] = (depth, value)\n\n # Modify the context with any derived context.\n for name, (_, value) in local_overrides.items():\n if value is missing:\n data.pop(name, None)\n else:\n data[name] = value\n\n return data\n\n\nif sys.version_info >= (3, 7):\n # tb_next is directly assignable as of Python 3.7\n def tb_set_next(\n tb: TracebackType, tb_next: t.Optional[TracebackType]\n ) -> TracebackType:\n tb.tb_next = tb_next\n return tb\n\n\nelif platform.python_implementation() == \"PyPy\":\n # PyPy might have special support, and won't work with ctypes.\n try:\n import tputil # type: ignore\n except ImportError:\n # Without tproxy support, use the original traceback.\n def tb_set_next(\n tb: TracebackType, tb_next: t.Optional[TracebackType]\n ) -> TracebackType:\n return tb\n\n else:\n # With tproxy support, create a proxy around the traceback that\n # returns the new tb_next.\n def tb_set_next(\n tb: TracebackType, tb_next: t.Optional[TracebackType]\n ) -> TracebackType:\n def controller(op): # type: ignore\n if op.opname == \"__getattribute__\" and op.args[0] == \"tb_next\":\n return tb_next\n\n return op.delegate()\n\n return tputil.make_proxy(controller, obj=tb) # type: ignore\n\n\nelse:\n # Use ctypes to assign tb_next at the C level since it's read-only\n # from Python.\n import ctypes\n\n class _CTraceback(ctypes.Structure):\n _fields_ = [\n # Extra PyObject slots when compiled with Py_TRACE_REFS.\n (\"PyObject_HEAD\", ctypes.c_byte * object().__sizeof__()),\n # Only care about tb_next as an object, not a traceback.\n (\"tb_next\", ctypes.py_object),\n ]\n\n def tb_set_next(\n tb: TracebackType, tb_next: t.Optional[TracebackType]\n ) -> TracebackType:\n c_tb = _CTraceback.from_address(id(tb))\n\n # Clear out the old tb_next.\n if tb.tb_next is not None:\n c_tb_next = ctypes.py_object(tb.tb_next)\n c_tb.tb_next = ctypes.py_object()\n ctypes.pythonapi.Py_DecRef(c_tb_next)\n\n # Assign the new tb_next.\n if tb_next is not None:\n c_tb_next = ctypes.py_object(tb_next)\n ctypes.pythonapi.Py_IncRef(c_tb_next)\n c_tb.tb_next = c_tb_next\n\n return tb\n" }, { "path": "lib/spack/spack/vendor/jinja2/defaults.py", "content": "import typing as t\n\nfrom .filters import FILTERS as DEFAULT_FILTERS # noqa: F401\nfrom .tests import TESTS as DEFAULT_TESTS # noqa: F401\nfrom .utils import Cycler\nfrom .utils import generate_lorem_ipsum\nfrom .utils import Joiner\nfrom .utils import Namespace\n\nif t.TYPE_CHECKING:\n import spack.vendor.typing_extensions as te\n\n# defaults for the parser / lexer\nBLOCK_START_STRING = \"{%\"\nBLOCK_END_STRING = \"%}\"\nVARIABLE_START_STRING = \"{{\"\nVARIABLE_END_STRING = \"}}\"\nCOMMENT_START_STRING = \"{#\"\nCOMMENT_END_STRING = \"#}\"\nLINE_STATEMENT_PREFIX: t.Optional[str] = None\nLINE_COMMENT_PREFIX: t.Optional[str] = None\nTRIM_BLOCKS = False\nLSTRIP_BLOCKS = False\nNEWLINE_SEQUENCE: \"te.Literal['\\\\n', '\\\\r\\\\n', '\\\\r']\" = \"\\n\"\nKEEP_TRAILING_NEWLINE = False\n\n# default filters, tests and namespace\n\nDEFAULT_NAMESPACE = {\n \"range\": range,\n \"dict\": dict,\n \"lipsum\": generate_lorem_ipsum,\n \"cycler\": Cycler,\n \"joiner\": Joiner,\n \"namespace\": Namespace,\n}\n\n# default policies\nDEFAULT_POLICIES: t.Dict[str, t.Any] = {\n \"compiler.ascii_str\": True,\n \"urlize.rel\": \"noopener\",\n \"urlize.target\": None,\n \"urlize.extra_schemes\": None,\n \"truncate.leeway\": 5,\n \"json.dumps_function\": None,\n \"json.dumps_kwargs\": {\"sort_keys\": True},\n \"ext.i18n.trimmed\": False,\n}\n" }, { "path": "lib/spack/spack/vendor/jinja2/environment.py", "content": "\"\"\"Classes for managing templates and their runtime and compile time\noptions.\n\"\"\"\nimport os\nimport sys\nimport typing\nimport typing as t\nimport weakref\nfrom collections import ChainMap\nfrom functools import lru_cache\nfrom functools import partial\nfrom functools import reduce\nfrom types import CodeType\n\nfrom spack.vendor.markupsafe import Markup\n\nfrom . import nodes\nfrom .compiler import CodeGenerator\nfrom .compiler import generate\nfrom .defaults import BLOCK_END_STRING\nfrom .defaults import BLOCK_START_STRING\nfrom .defaults import COMMENT_END_STRING\nfrom .defaults import COMMENT_START_STRING\nfrom .defaults import DEFAULT_FILTERS\nfrom .defaults import DEFAULT_NAMESPACE\nfrom .defaults import DEFAULT_POLICIES\nfrom .defaults import DEFAULT_TESTS\nfrom .defaults import KEEP_TRAILING_NEWLINE\nfrom .defaults import LINE_COMMENT_PREFIX\nfrom .defaults import LINE_STATEMENT_PREFIX\nfrom .defaults import LSTRIP_BLOCKS\nfrom .defaults import NEWLINE_SEQUENCE\nfrom .defaults import TRIM_BLOCKS\nfrom .defaults import VARIABLE_END_STRING\nfrom .defaults import VARIABLE_START_STRING\nfrom .exceptions import TemplateNotFound\nfrom .exceptions import TemplateRuntimeError\nfrom .exceptions import TemplatesNotFound\nfrom .exceptions import TemplateSyntaxError\nfrom .exceptions import UndefinedError\nfrom .lexer import get_lexer\nfrom .lexer import Lexer\nfrom .lexer import TokenStream\nfrom .nodes import EvalContext\nfrom .parser import Parser\nfrom .runtime import Context\nfrom .runtime import new_context\nfrom .runtime import Undefined\nfrom .utils import _PassArg\nfrom .utils import concat\nfrom .utils import consume\nfrom .utils import import_string\nfrom .utils import internalcode\nfrom .utils import LRUCache\nfrom .utils import missing\n\nif t.TYPE_CHECKING:\n import spack.vendor.typing_extensions as te\n from .bccache import BytecodeCache\n from .ext import Extension\n from .loaders import BaseLoader\n\n_env_bound = t.TypeVar(\"_env_bound\", bound=\"Environment\")\n\n\n# for direct template usage we have up to ten living environments\n@lru_cache(maxsize=10)\ndef get_spontaneous_environment(cls: t.Type[_env_bound], *args: t.Any) -> _env_bound:\n \"\"\"Return a new spontaneous environment. A spontaneous environment\n is used for templates created directly rather than through an\n existing environment.\n\n :param cls: Environment class to create.\n :param args: Positional arguments passed to environment.\n \"\"\"\n env = cls(*args)\n env.shared = True\n return env\n\n\ndef create_cache(\n size: int,\n) -> t.Optional[t.MutableMapping[t.Tuple[weakref.ref, str], \"Template\"]]:\n \"\"\"Return the cache class for the given size.\"\"\"\n if size == 0:\n return None\n\n if size < 0:\n return {}\n\n return LRUCache(size) # type: ignore\n\n\ndef copy_cache(\n cache: t.Optional[t.MutableMapping],\n) -> t.Optional[t.MutableMapping[t.Tuple[weakref.ref, str], \"Template\"]]:\n \"\"\"Create an empty copy of the given cache.\"\"\"\n if cache is None:\n return None\n\n if type(cache) is dict:\n return {}\n\n return LRUCache(cache.capacity) # type: ignore\n\n\ndef load_extensions(\n environment: \"Environment\",\n extensions: t.Sequence[t.Union[str, t.Type[\"Extension\"]]],\n) -> t.Dict[str, \"Extension\"]:\n \"\"\"Load the extensions from the list and bind it to the environment.\n Returns a dict of instantiated extensions.\n \"\"\"\n result = {}\n\n for extension in extensions:\n if isinstance(extension, str):\n extension = t.cast(t.Type[\"Extension\"], import_string(extension))\n\n result[extension.identifier] = extension(environment)\n\n return result\n\n\ndef _environment_config_check(environment: \"Environment\") -> \"Environment\":\n \"\"\"Perform a sanity check on the environment.\"\"\"\n assert issubclass(\n environment.undefined, Undefined\n ), \"'undefined' must be a subclass of 'spack.vendor.jinja2.Undefined'.\"\n assert (\n environment.block_start_string\n != environment.variable_start_string\n != environment.comment_start_string\n ), \"block, variable and comment start strings must be different.\"\n assert environment.newline_sequence in {\n \"\\r\",\n \"\\r\\n\",\n \"\\n\",\n }, \"'newline_sequence' must be one of '\\\\n', '\\\\r\\\\n', or '\\\\r'.\"\n return environment\n\n\nclass Environment:\n r\"\"\"The core component of Jinja is the `Environment`. It contains\n important shared variables like configuration, filters, tests,\n globals and others. Instances of this class may be modified if\n they are not shared and if no template was loaded so far.\n Modifications on environments after the first template was loaded\n will lead to surprising effects and undefined behavior.\n\n Here are the possible initialization parameters:\n\n `block_start_string`\n The string marking the beginning of a block. Defaults to ``'{%'``.\n\n `block_end_string`\n The string marking the end of a block. Defaults to ``'%}'``.\n\n `variable_start_string`\n The string marking the beginning of a print statement.\n Defaults to ``'{{'``.\n\n `variable_end_string`\n The string marking the end of a print statement. Defaults to\n ``'}}'``.\n\n `comment_start_string`\n The string marking the beginning of a comment. Defaults to ``'{#'``.\n\n `comment_end_string`\n The string marking the end of a comment. Defaults to ``'#}'``.\n\n `line_statement_prefix`\n If given and a string, this will be used as prefix for line based\n statements. See also :ref:`line-statements`.\n\n `line_comment_prefix`\n If given and a string, this will be used as prefix for line based\n comments. See also :ref:`line-statements`.\n\n .. versionadded:: 2.2\n\n `trim_blocks`\n If this is set to ``True`` the first newline after a block is\n removed (block, not variable tag!). Defaults to `False`.\n\n `lstrip_blocks`\n If this is set to ``True`` leading spaces and tabs are stripped\n from the start of a line to a block. Defaults to `False`.\n\n `newline_sequence`\n The sequence that starts a newline. Must be one of ``'\\r'``,\n ``'\\n'`` or ``'\\r\\n'``. The default is ``'\\n'`` which is a\n useful default for Linux and OS X systems as well as web\n applications.\n\n `keep_trailing_newline`\n Preserve the trailing newline when rendering templates.\n The default is ``False``, which causes a single newline,\n if present, to be stripped from the end of the template.\n\n .. versionadded:: 2.7\n\n `extensions`\n List of Jinja extensions to use. This can either be import paths\n as strings or extension classes. For more information have a\n look at :ref:`the extensions documentation `.\n\n `optimized`\n should the optimizer be enabled? Default is ``True``.\n\n `undefined`\n :class:`Undefined` or a subclass of it that is used to represent\n undefined values in the template.\n\n `finalize`\n A callable that can be used to process the result of a variable\n expression before it is output. For example one can convert\n ``None`` implicitly into an empty string here.\n\n `autoescape`\n If set to ``True`` the XML/HTML autoescaping feature is enabled by\n default. For more details about autoescaping see\n :class:`~spack.vendor.markupsafe.Markup`. As of Jinja 2.4 this can also\n be a callable that is passed the template name and has to\n return ``True`` or ``False`` depending on autoescape should be\n enabled by default.\n\n .. versionchanged:: 2.4\n `autoescape` can now be a function\n\n `loader`\n The template loader for this environment.\n\n `cache_size`\n The size of the cache. Per default this is ``400`` which means\n that if more than 400 templates are loaded the loader will clean\n out the least recently used template. If the cache size is set to\n ``0`` templates are recompiled all the time, if the cache size is\n ``-1`` the cache will not be cleaned.\n\n .. versionchanged:: 2.8\n The cache size was increased to 400 from a low 50.\n\n `auto_reload`\n Some loaders load templates from locations where the template\n sources may change (ie: file system or database). If\n ``auto_reload`` is set to ``True`` (default) every time a template is\n requested the loader checks if the source changed and if yes, it\n will reload the template. For higher performance it's possible to\n disable that.\n\n `bytecode_cache`\n If set to a bytecode cache object, this object will provide a\n cache for the internal Jinja bytecode so that templates don't\n have to be parsed if they were not changed.\n\n See :ref:`bytecode-cache` for more information.\n\n `enable_async`\n If set to true this enables async template execution which\n allows using async functions and generators.\n \"\"\"\n\n #: if this environment is sandboxed. Modifying this variable won't make\n #: the environment sandboxed though. For a real sandboxed environment\n #: have a look at spack.vendor.jinja2.sandbox. This flag alone controls the code\n #: generation by the compiler.\n sandboxed = False\n\n #: True if the environment is just an overlay\n overlayed = False\n\n #: the environment this environment is linked to if it is an overlay\n linked_to: t.Optional[\"Environment\"] = None\n\n #: shared environments have this set to `True`. A shared environment\n #: must not be modified\n shared = False\n\n #: the class that is used for code generation. See\n #: :class:`~spack.vendor.jinja2.compiler.CodeGenerator` for more information.\n code_generator_class: t.Type[\"CodeGenerator\"] = CodeGenerator\n\n #: the context class that is used for templates. See\n #: :class:`~spack.vendor.jinja2.runtime.Context` for more information.\n context_class: t.Type[Context] = Context\n\n template_class: t.Type[\"Template\"]\n\n def __init__(\n self,\n block_start_string: str = BLOCK_START_STRING,\n block_end_string: str = BLOCK_END_STRING,\n variable_start_string: str = VARIABLE_START_STRING,\n variable_end_string: str = VARIABLE_END_STRING,\n comment_start_string: str = COMMENT_START_STRING,\n comment_end_string: str = COMMENT_END_STRING,\n line_statement_prefix: t.Optional[str] = LINE_STATEMENT_PREFIX,\n line_comment_prefix: t.Optional[str] = LINE_COMMENT_PREFIX,\n trim_blocks: bool = TRIM_BLOCKS,\n lstrip_blocks: bool = LSTRIP_BLOCKS,\n newline_sequence: \"te.Literal['\\\\n', '\\\\r\\\\n', '\\\\r']\" = NEWLINE_SEQUENCE,\n keep_trailing_newline: bool = KEEP_TRAILING_NEWLINE,\n extensions: t.Sequence[t.Union[str, t.Type[\"Extension\"]]] = (),\n optimized: bool = True,\n undefined: t.Type[Undefined] = Undefined,\n finalize: t.Optional[t.Callable[..., t.Any]] = None,\n autoescape: t.Union[bool, t.Callable[[t.Optional[str]], bool]] = False,\n loader: t.Optional[\"BaseLoader\"] = None,\n cache_size: int = 400,\n auto_reload: bool = True,\n bytecode_cache: t.Optional[\"BytecodeCache\"] = None,\n enable_async: bool = False,\n ):\n # !!Important notice!!\n # The constructor accepts quite a few arguments that should be\n # passed by keyword rather than position. However it's important to\n # not change the order of arguments because it's used at least\n # internally in those cases:\n # - spontaneous environments (i18n extension and Template)\n # - unittests\n # If parameter changes are required only add parameters at the end\n # and don't change the arguments (or the defaults!) of the arguments\n # existing already.\n\n # lexer / parser information\n self.block_start_string = block_start_string\n self.block_end_string = block_end_string\n self.variable_start_string = variable_start_string\n self.variable_end_string = variable_end_string\n self.comment_start_string = comment_start_string\n self.comment_end_string = comment_end_string\n self.line_statement_prefix = line_statement_prefix\n self.line_comment_prefix = line_comment_prefix\n self.trim_blocks = trim_blocks\n self.lstrip_blocks = lstrip_blocks\n self.newline_sequence = newline_sequence\n self.keep_trailing_newline = keep_trailing_newline\n\n # runtime information\n self.undefined: t.Type[Undefined] = undefined\n self.optimized = optimized\n self.finalize = finalize\n self.autoescape = autoescape\n\n # defaults\n self.filters = DEFAULT_FILTERS.copy()\n self.tests = DEFAULT_TESTS.copy()\n self.globals = DEFAULT_NAMESPACE.copy()\n\n # set the loader provided\n self.loader = loader\n self.cache = create_cache(cache_size)\n self.bytecode_cache = bytecode_cache\n self.auto_reload = auto_reload\n\n # configurable policies\n self.policies = DEFAULT_POLICIES.copy()\n\n # load extensions\n self.extensions = load_extensions(self, extensions)\n\n self.is_async = enable_async\n _environment_config_check(self)\n\n def add_extension(self, extension: t.Union[str, t.Type[\"Extension\"]]) -> None:\n \"\"\"Adds an extension after the environment was created.\n\n .. versionadded:: 2.5\n \"\"\"\n self.extensions.update(load_extensions(self, [extension]))\n\n def extend(self, **attributes: t.Any) -> None:\n \"\"\"Add the items to the instance of the environment if they do not exist\n yet. This is used by :ref:`extensions ` to register\n callbacks and configuration values without breaking inheritance.\n \"\"\"\n for key, value in attributes.items():\n if not hasattr(self, key):\n setattr(self, key, value)\n\n def overlay(\n self,\n block_start_string: str = missing,\n block_end_string: str = missing,\n variable_start_string: str = missing,\n variable_end_string: str = missing,\n comment_start_string: str = missing,\n comment_end_string: str = missing,\n line_statement_prefix: t.Optional[str] = missing,\n line_comment_prefix: t.Optional[str] = missing,\n trim_blocks: bool = missing,\n lstrip_blocks: bool = missing,\n extensions: t.Sequence[t.Union[str, t.Type[\"Extension\"]]] = missing,\n optimized: bool = missing,\n undefined: t.Type[Undefined] = missing,\n finalize: t.Optional[t.Callable[..., t.Any]] = missing,\n autoescape: t.Union[bool, t.Callable[[t.Optional[str]], bool]] = missing,\n loader: t.Optional[\"BaseLoader\"] = missing,\n cache_size: int = missing,\n auto_reload: bool = missing,\n bytecode_cache: t.Optional[\"BytecodeCache\"] = missing,\n ) -> \"Environment\":\n \"\"\"Create a new overlay environment that shares all the data with the\n current environment except for cache and the overridden attributes.\n Extensions cannot be removed for an overlayed environment. An overlayed\n environment automatically gets all the extensions of the environment it\n is linked to plus optional extra extensions.\n\n Creating overlays should happen after the initial environment was set\n up completely. Not all attributes are truly linked, some are just\n copied over so modifications on the original environment may not shine\n through.\n \"\"\"\n args = dict(locals())\n del args[\"self\"], args[\"cache_size\"], args[\"extensions\"]\n\n rv = object.__new__(self.__class__)\n rv.__dict__.update(self.__dict__)\n rv.overlayed = True\n rv.linked_to = self\n\n for key, value in args.items():\n if value is not missing:\n setattr(rv, key, value)\n\n if cache_size is not missing:\n rv.cache = create_cache(cache_size)\n else:\n rv.cache = copy_cache(self.cache)\n\n rv.extensions = {}\n for key, value in self.extensions.items():\n rv.extensions[key] = value.bind(rv)\n if extensions is not missing:\n rv.extensions.update(load_extensions(rv, extensions))\n\n return _environment_config_check(rv)\n\n @property\n def lexer(self) -> Lexer:\n \"\"\"The lexer for this environment.\"\"\"\n return get_lexer(self)\n\n def iter_extensions(self) -> t.Iterator[\"Extension\"]:\n \"\"\"Iterates over the extensions by priority.\"\"\"\n return iter(sorted(self.extensions.values(), key=lambda x: x.priority))\n\n def getitem(\n self, obj: t.Any, argument: t.Union[str, t.Any]\n ) -> t.Union[t.Any, Undefined]:\n \"\"\"Get an item or attribute of an object but prefer the item.\"\"\"\n try:\n return obj[argument]\n except (AttributeError, TypeError, LookupError):\n if isinstance(argument, str):\n try:\n attr = str(argument)\n except Exception:\n pass\n else:\n try:\n return getattr(obj, attr)\n except AttributeError:\n pass\n return self.undefined(obj=obj, name=argument)\n\n def getattr(self, obj: t.Any, attribute: str) -> t.Any:\n \"\"\"Get an item or attribute of an object but prefer the attribute.\n Unlike :meth:`getitem` the attribute *must* be a string.\n \"\"\"\n try:\n return getattr(obj, attribute)\n except AttributeError:\n pass\n try:\n return obj[attribute]\n except (TypeError, LookupError, AttributeError):\n return self.undefined(obj=obj, name=attribute)\n\n def _filter_test_common(\n self,\n name: t.Union[str, Undefined],\n value: t.Any,\n args: t.Optional[t.Sequence[t.Any]],\n kwargs: t.Optional[t.Mapping[str, t.Any]],\n context: t.Optional[Context],\n eval_ctx: t.Optional[EvalContext],\n is_filter: bool,\n ) -> t.Any:\n if is_filter:\n env_map = self.filters\n type_name = \"filter\"\n else:\n env_map = self.tests\n type_name = \"test\"\n\n func = env_map.get(name) # type: ignore\n\n if func is None:\n msg = f\"No {type_name} named {name!r}.\"\n\n if isinstance(name, Undefined):\n try:\n name._fail_with_undefined_error()\n except Exception as e:\n msg = f\"{msg} ({e}; did you forget to quote the callable name?)\"\n\n raise TemplateRuntimeError(msg)\n\n args = [value, *(args if args is not None else ())]\n kwargs = kwargs if kwargs is not None else {}\n pass_arg = _PassArg.from_obj(func)\n\n if pass_arg is _PassArg.context:\n if context is None:\n raise TemplateRuntimeError(\n f\"Attempted to invoke a context {type_name} without context.\"\n )\n\n args.insert(0, context)\n elif pass_arg is _PassArg.eval_context:\n if eval_ctx is None:\n if context is not None:\n eval_ctx = context.eval_ctx\n else:\n eval_ctx = EvalContext(self)\n\n args.insert(0, eval_ctx)\n elif pass_arg is _PassArg.environment:\n args.insert(0, self)\n\n return func(*args, **kwargs)\n\n def call_filter(\n self,\n name: str,\n value: t.Any,\n args: t.Optional[t.Sequence[t.Any]] = None,\n kwargs: t.Optional[t.Mapping[str, t.Any]] = None,\n context: t.Optional[Context] = None,\n eval_ctx: t.Optional[EvalContext] = None,\n ) -> t.Any:\n \"\"\"Invoke a filter on a value the same way the compiler does.\n\n This might return a coroutine if the filter is running from an\n environment in async mode and the filter supports async\n execution. It's your responsibility to await this if needed.\n\n .. versionadded:: 2.7\n \"\"\"\n return self._filter_test_common(\n name, value, args, kwargs, context, eval_ctx, True\n )\n\n def call_test(\n self,\n name: str,\n value: t.Any,\n args: t.Optional[t.Sequence[t.Any]] = None,\n kwargs: t.Optional[t.Mapping[str, t.Any]] = None,\n context: t.Optional[Context] = None,\n eval_ctx: t.Optional[EvalContext] = None,\n ) -> t.Any:\n \"\"\"Invoke a test on a value the same way the compiler does.\n\n This might return a coroutine if the test is running from an\n environment in async mode and the test supports async execution.\n It's your responsibility to await this if needed.\n\n .. versionchanged:: 3.0\n Tests support ``@pass_context``, etc. decorators. Added\n the ``context`` and ``eval_ctx`` parameters.\n\n .. versionadded:: 2.7\n \"\"\"\n return self._filter_test_common(\n name, value, args, kwargs, context, eval_ctx, False\n )\n\n @internalcode\n def parse(\n self,\n source: str,\n name: t.Optional[str] = None,\n filename: t.Optional[str] = None,\n ) -> nodes.Template:\n \"\"\"Parse the sourcecode and return the abstract syntax tree. This\n tree of nodes is used by the compiler to convert the template into\n executable source- or bytecode. This is useful for debugging or to\n extract information from templates.\n\n If you are :ref:`developing Jinja extensions `\n this gives you a good overview of the node tree generated.\n \"\"\"\n try:\n return self._parse(source, name, filename)\n except TemplateSyntaxError:\n self.handle_exception(source=source)\n\n def _parse(\n self, source: str, name: t.Optional[str], filename: t.Optional[str]\n ) -> nodes.Template:\n \"\"\"Internal parsing function used by `parse` and `compile`.\"\"\"\n return Parser(self, source, name, filename).parse()\n\n def lex(\n self,\n source: str,\n name: t.Optional[str] = None,\n filename: t.Optional[str] = None,\n ) -> t.Iterator[t.Tuple[int, str, str]]:\n \"\"\"Lex the given sourcecode and return a generator that yields\n tokens as tuples in the form ``(lineno, token_type, value)``.\n This can be useful for :ref:`extension development `\n and debugging templates.\n\n This does not perform preprocessing. If you want the preprocessing\n of the extensions to be applied you have to filter source through\n the :meth:`preprocess` method.\n \"\"\"\n source = str(source)\n try:\n return self.lexer.tokeniter(source, name, filename)\n except TemplateSyntaxError:\n self.handle_exception(source=source)\n\n def preprocess(\n self,\n source: str,\n name: t.Optional[str] = None,\n filename: t.Optional[str] = None,\n ) -> str:\n \"\"\"Preprocesses the source with all extensions. This is automatically\n called for all parsing and compiling methods but *not* for :meth:`lex`\n because there you usually only want the actual source tokenized.\n \"\"\"\n return reduce(\n lambda s, e: e.preprocess(s, name, filename),\n self.iter_extensions(),\n str(source),\n )\n\n def _tokenize(\n self,\n source: str,\n name: t.Optional[str],\n filename: t.Optional[str] = None,\n state: t.Optional[str] = None,\n ) -> TokenStream:\n \"\"\"Called by the parser to do the preprocessing and filtering\n for all the extensions. Returns a :class:`~spack.vendor.jinja2.lexer.TokenStream`.\n \"\"\"\n source = self.preprocess(source, name, filename)\n stream = self.lexer.tokenize(source, name, filename, state)\n\n for ext in self.iter_extensions():\n stream = ext.filter_stream(stream) # type: ignore\n\n if not isinstance(stream, TokenStream):\n stream = TokenStream(stream, name, filename) # type: ignore\n\n return stream\n\n def _generate(\n self,\n source: nodes.Template,\n name: t.Optional[str],\n filename: t.Optional[str],\n defer_init: bool = False,\n ) -> str:\n \"\"\"Internal hook that can be overridden to hook a different generate\n method in.\n\n .. versionadded:: 2.5\n \"\"\"\n return generate( # type: ignore\n source,\n self,\n name,\n filename,\n defer_init=defer_init,\n optimized=self.optimized,\n )\n\n def _compile(self, source: str, filename: str) -> CodeType:\n \"\"\"Internal hook that can be overridden to hook a different compile\n method in.\n\n .. versionadded:: 2.5\n \"\"\"\n return compile(source, filename, \"exec\") # type: ignore\n\n @typing.overload\n def compile( # type: ignore\n self,\n source: t.Union[str, nodes.Template],\n name: t.Optional[str] = None,\n filename: t.Optional[str] = None,\n raw: \"te.Literal[False]\" = False,\n defer_init: bool = False,\n ) -> CodeType:\n ...\n\n @typing.overload\n def compile(\n self,\n source: t.Union[str, nodes.Template],\n name: t.Optional[str] = None,\n filename: t.Optional[str] = None,\n raw: \"te.Literal[True]\" = ...,\n defer_init: bool = False,\n ) -> str:\n ...\n\n @internalcode\n def compile(\n self,\n source: t.Union[str, nodes.Template],\n name: t.Optional[str] = None,\n filename: t.Optional[str] = None,\n raw: bool = False,\n defer_init: bool = False,\n ) -> t.Union[str, CodeType]:\n \"\"\"Compile a node or template source code. The `name` parameter is\n the load name of the template after it was joined using\n :meth:`join_path` if necessary, not the filename on the file system.\n the `filename` parameter is the estimated filename of the template on\n the file system. If the template came from a database or memory this\n can be omitted.\n\n The return value of this method is a python code object. If the `raw`\n parameter is `True` the return value will be a string with python\n code equivalent to the bytecode returned otherwise. This method is\n mainly used internally.\n\n `defer_init` is use internally to aid the module code generator. This\n causes the generated code to be able to import without the global\n environment variable to be set.\n\n .. versionadded:: 2.4\n `defer_init` parameter added.\n \"\"\"\n source_hint = None\n try:\n if isinstance(source, str):\n source_hint = source\n source = self._parse(source, name, filename)\n source = self._generate(source, name, filename, defer_init=defer_init)\n if raw:\n return source\n if filename is None:\n filename = \"